From b7b744c69f50b93f36d967e13575547872dcc57f Mon Sep 17 00:00:00 2001 From: Marcin Martinez Date: Thu, 17 Feb 2022 15:52:06 +0100 Subject: [PATCH 01/12] Add `dockerPrivateKey` option to specify the path to mounted SSH key --- index.js | 7 +++++-- lib/pip.js | 6 ++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/index.js b/index.js index 7741a7f8..c6577fe0 100644 --- a/index.js +++ b/index.js @@ -15,7 +15,6 @@ const { installAllRequirements } = require('./lib/pip'); const { pipfileToRequirements } = require('./lib/pipenv'); const { pyprojectTomlToRequirements } = require('./lib/poetry'); const { cleanup, cleanupCache } = require('./lib/clean'); - BbPromise.promisifyAll(fse); /** @@ -45,6 +44,7 @@ class ServerlessPythonRequirements { : this.serverless.service.provider.runtime || 'python', dockerizePip: false, dockerSsh: false, + dockerPrivateKey: null, dockerImage: null, dockerFile: null, dockerEnv: false, @@ -71,7 +71,10 @@ class ServerlessPythonRequirements { } if ( !options.dockerizePip && - (options.dockerSsh || options.dockerImage || options.dockerFile) + (options.dockerSsh || + options.dockerImage || + options.dockerFile || + options.dockerPrivateKey) ) { if (!this.warningLogged) { if (this.log) { diff --git a/lib/pip.js b/lib/pip.js index 7a0a0ceb..7b671010 100644 --- a/lib/pip.js +++ b/lib/pip.js @@ -275,10 +275,12 @@ async function installRequirements(targetFolder, pluginInstance) { dockerCmd.push('docker', 'run', '--rm', '-v', `${bindPath}:/var/task:z`); if (options.dockerSsh) { + const sshKeyPath = options.dockerPrivateKey || `${process.env.HOME}/.ssh/id_rsa`; + // Mount necessary ssh files to work with private repos dockerCmd.push( '-v', - `${process.env.HOME}/.ssh/id_rsa:/root/.ssh/id_rsa:z`, + `${sshKeyPath}:/root/.ssh/${sshKeyPath.split('/').at(-1)}:z`, '-v', `${process.env.HOME}/.ssh/known_hosts:/root/.ssh/known_hosts:z`, '-v', @@ -780,4 +782,4 @@ async function installAllRequirements() { } } -module.exports = { installAllRequirements }; +module.exports = { installAllRequirements }; \ No newline at end of file From 7db20e824496bbef303bda5da42b7a37de472a80 Mon Sep 17 00:00:00 2001 From: Marcin Martinez Date: Thu, 17 Feb 2022 15:53:27 +0100 Subject: [PATCH 02/12] Add test for `dockerPrivateKey`, update serverless.yml in tests/base --- test.js | 24 ++++++++++++++++++++++++ tests/base/id_ed25519 | 7 +++++++ tests/base/requirements-w-git-ssh.txt | 2 ++ tests/base/serverless.yml | 4 ++++ 4 files changed, 37 insertions(+) create mode 100644 tests/base/id_ed25519 create mode 100644 tests/base/requirements-w-git-ssh.txt diff --git a/test.js b/test.js index 11a7cce5..cf9e90dc 100644 --- a/test.js +++ b/test.js @@ -200,6 +200,30 @@ const canUseDocker = () => { // Skip if running on these platforms. const brokenOn = (...platforms) => platforms.indexOf(process.platform) != -1; +test( + 'can package private repo with dockerPrivateKey option', + async (t) => { + process.chdir('tests/base'); + const path = npm(['pack', '../..']); + npm(['i', path]); + sls(['package'], { + env: { + dockerizePip: true, + dockerSsh: true, + dockerPrivateKey: `${__dirname}${sep}tests${sep}base${sep}id_ed25519`, + fileName: 'requirements-w-git-ssh.txt', + }, + }); + const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); + t.true( + zipfiles.includes(`dockerPrivateKey_test/__init__.py`), + 'private repo accessed with specified key' + ); + t.end(); + }, + { skip: !canUseDocker() } +); + test( 'default pythonBin can package flask with default options', async (t) => { diff --git a/tests/base/id_ed25519 b/tests/base/id_ed25519 new file mode 100644 index 00000000..43b88e4d --- /dev/null +++ b/tests/base/id_ed25519 @@ -0,0 +1,7 @@ +-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW +QyNTUxOQAAACAns3vFG1eJtPnqVRylDQsyu4QGpkvqlH/mj8N6eYIqXAAAAKDM80pnzPNK +ZwAAAAtzc2gtZWQyNTUxOQAAACAns3vFG1eJtPnqVRylDQsyu4QGpkvqlH/mj8N6eYIqXA +AAAECYfiON2DfcWuu1uDsUs5INIh1adTbdwSmWa1khvvAawyeze8UbV4m0+epVHKUNCzK7 +hAamS+qUf+aPw3p5gipcAAAAG2Zpc2hlcm1hbm5ATWFyY2lucy1BaXIuaG9tZQEC +-----END OPENSSH PRIVATE KEY----- diff --git a/tests/base/requirements-w-git-ssh.txt b/tests/base/requirements-w-git-ssh.txt new file mode 100644 index 00000000..fb7c19db --- /dev/null +++ b/tests/base/requirements-w-git-ssh.txt @@ -0,0 +1,2 @@ +flask +git+ssh://git@github.com/martinezpl/dockerPrivateKey_test.git@v1.0.0#egg=dockerPrivateKey_test \ No newline at end of file diff --git a/tests/base/serverless.yml b/tests/base/serverless.yml index 37238158..2eae49d5 100644 --- a/tests/base/serverless.yml +++ b/tests/base/serverless.yml @@ -10,6 +10,8 @@ custom: pythonRequirements: zip: ${env:zip, self:custom.defaults.zip} dockerizePip: ${env:dockerizePip, self:custom.defaults.dockerizePip} + dockerSsh: ${env:dockerSsh, self:custom.defaults.dockerSsh} + dockerPrivateKey: ${env:dockerPrivateKey, self:custom.defaults.dockerPrivateKey} slim: ${env:slim, self:custom.defaults.slim} slimPatterns: ${file(./slimPatterns.yml):slimPatterns, self:custom.defaults.slimPatterns} slimPatternsAppendDefaults: ${env:slimPatternsAppendDefaults, self:custom.defaults.slimPatternsAppendDefaults} @@ -24,6 +26,8 @@ custom: slimPatternsAppendDefaults: true zip: false dockerizePip: false + dockerSsh: false + dockerPrivateKey: '' individually: false useStaticCache: true useDownloadCache: true From eb85d9f04691e7379fdd21c65584e30169d6e297 Mon Sep 17 00:00:00 2001 From: Marcin Martinez Date: Thu, 17 Feb 2022 15:53:41 +0100 Subject: [PATCH 03/12] Update README.md --- README.md | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index abe6a175..773637e4 100644 --- a/README.md +++ b/README.md @@ -77,8 +77,20 @@ custom: ``` The `dockerSsh` option will mount your `$HOME/.ssh/id_rsa` and `$HOME/.ssh/known_hosts` as a -volume in the docker container. If your SSH key is password protected, you can use `ssh-agent` -because `$SSH_AUTH_SOCK` is also mounted & the env var set. +volume in the docker container. + +In case you want to use a different key, you can specify the path (absolute) to it through `dockerPrivateKey` option: + +```yaml +custom: + pythonRequirements: + dockerizePip: true + dockerSsh: true + dockerPrivateKey: /home/.ssh/id_ed25519 +``` + +If your SSH key is password protected, you can use `ssh-agent` +because `$SSH_AUTH_SOCK` is also mounted & the env var is set. It is important that the host of your private repositories has already been added in your `$HOME/.ssh/known_hosts` file, as the install process will fail otherwise due to host authenticity failure. @@ -566,3 +578,4 @@ package: - [@jacksgt](https://github.com/jacksgt) - Fixing pip issues - [@lephuongbg](https://github.com/lephuongbg) - Fixing single function deployment - [@rileypriddle](https://github.com/rileypriddle) - Introducing schema validation for `module` property +- [@martinezpl](https://github.com/martinezpl) - Fixing test issues, adding `dockerPrivateKey` option From df5de8423f39cfff6665b2015544f08b5cbf93d5 Mon Sep 17 00:00:00 2001 From: Marcin Martinez Date: Thu, 17 Feb 2022 17:16:55 +0100 Subject: [PATCH 04/12] Use os.homedir() for multi-system compatibility --- lib/pip.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/pip.js b/lib/pip.js index 7b671010..8c31816a 100644 --- a/lib/pip.js +++ b/lib/pip.js @@ -275,14 +275,15 @@ async function installRequirements(targetFolder, pluginInstance) { dockerCmd.push('docker', 'run', '--rm', '-v', `${bindPath}:/var/task:z`); if (options.dockerSsh) { - const sshKeyPath = options.dockerPrivateKey || `${process.env.HOME}/.ssh/id_rsa`; + const homePath = require('os').homedir(); + const sshKeyPath = options.dockerPrivateKey || `${homePath}/.ssh/id_rsa`; // Mount necessary ssh files to work with private repos dockerCmd.push( '-v', `${sshKeyPath}:/root/.ssh/${sshKeyPath.split('/').at(-1)}:z`, '-v', - `${process.env.HOME}/.ssh/known_hosts:/root/.ssh/known_hosts:z`, + `${homePath}/.ssh/known_hosts:/root/.ssh/known_hosts:z`, '-v', `${process.env.SSH_AUTH_SOCK}:/tmp/ssh_sock:z`, '-e', From 0c6cf528376e5c11a6dc668a9a5785a477edb7f7 Mon Sep 17 00:00:00 2001 From: Marcin Martinez Date: Thu, 17 Feb 2022 18:26:37 +0100 Subject: [PATCH 05/12] Fix node.js v12 compatibility --- lib/pip.js | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/pip.js b/lib/pip.js index 8c31816a..d011f8df 100644 --- a/lib/pip.js +++ b/lib/pip.js @@ -276,12 +276,13 @@ async function installRequirements(targetFolder, pluginInstance) { dockerCmd.push('docker', 'run', '--rm', '-v', `${bindPath}:/var/task:z`); if (options.dockerSsh) { const homePath = require('os').homedir(); - const sshKeyPath = options.dockerPrivateKey || `${homePath}/.ssh/id_rsa`; - + const sshKeyPath = + options.dockerPrivateKey || `${homePath}/.ssh/id_rsa`; + // Mount necessary ssh files to work with private repos dockerCmd.push( '-v', - `${sshKeyPath}:/root/.ssh/${sshKeyPath.split('/').at(-1)}:z`, + `${sshKeyPath}:/root/.ssh/${sshKeyPath.split('/').splice(-1)[0]}:z`, '-v', `${homePath}/.ssh/known_hosts:/root/.ssh/known_hosts:z`, '-v', @@ -783,4 +784,4 @@ async function installAllRequirements() { } } -module.exports = { installAllRequirements }; \ No newline at end of file +module.exports = { installAllRequirements }; From 24e92afc973b19f9e5f303877f7f5637fd1a138a Mon Sep 17 00:00:00 2001 From: Marcin Martinez Date: Thu, 17 Feb 2022 18:29:53 +0100 Subject: [PATCH 06/12] Skip `dockerPrivateKey` test on win32 --- test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test.js b/test.js index cf9e90dc..07c9564d 100644 --- a/test.js +++ b/test.js @@ -221,7 +221,7 @@ test( ); t.end(); }, - { skip: !canUseDocker() } + { skip: !canUseDocker() || brokenOn('win32') } ); test( From 1c40bc8ddebe5797755fc9a02253627953a5d56f Mon Sep 17 00:00:00 2001 From: Marcin Martinez Date: Sat, 19 Feb 2022 13:20:20 +0100 Subject: [PATCH 07/12] Rerun tests --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 773637e4..63b1a32a 100644 --- a/README.md +++ b/README.md @@ -225,7 +225,7 @@ the names in `slimPatterns` #### Option not to strip binaries -In some cases, stripping binaries leads to problems like "ELF load command address/offset not properly aligned", even when done in the Docker environment. You can still slim down the package without `*.so` files with +In some cases, stripping binaries leads to problems like "ELF load command address/offset not properly aligned", even when done in the Docker environment. You can still slim down the package without `*.so` files with: ```yaml custom: From ef046d6751b18a5d27f075a4ba0aca8e56588888 Mon Sep 17 00:00:00 2001 From: Marcin Martinez Date: Thu, 3 Mar 2022 10:45:26 +0100 Subject: [PATCH 08/12] Change os.homedir() to process.env.HOME --- lib/pip.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pip.js b/lib/pip.js index d011f8df..fb0e492a 100644 --- a/lib/pip.js +++ b/lib/pip.js @@ -275,7 +275,7 @@ async function installRequirements(targetFolder, pluginInstance) { dockerCmd.push('docker', 'run', '--rm', '-v', `${bindPath}:/var/task:z`); if (options.dockerSsh) { - const homePath = require('os').homedir(); + const homePath = process.env.HOME; const sshKeyPath = options.dockerPrivateKey || `${homePath}/.ssh/id_rsa`; From 8500c319d86ba327aa91eb90d89c103b987cf62c Mon Sep 17 00:00:00 2001 From: Marcin Martinez Date: Fri, 4 Mar 2022 13:28:07 +0100 Subject: [PATCH 09/12] Try skipping the test to check the testing environment --- test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test.js b/test.js index 07c9564d..84a29666 100644 --- a/test.js +++ b/test.js @@ -221,7 +221,7 @@ test( ); t.end(); }, - { skip: !canUseDocker() || brokenOn('win32') } + { skip: !canUseDocker() || brokenOn('win32') || true } ); test( From eaf83cc83a6cf9d243d029f0c502a164728e8604 Mon Sep 17 00:00:00 2001 From: Marcin Martinez Date: Fri, 4 Mar 2022 15:17:41 +0100 Subject: [PATCH 10/12] Bring back os.homedir() --- lib/pip.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pip.js b/lib/pip.js index fb0e492a..d011f8df 100644 --- a/lib/pip.js +++ b/lib/pip.js @@ -275,7 +275,7 @@ async function installRequirements(targetFolder, pluginInstance) { dockerCmd.push('docker', 'run', '--rm', '-v', `${bindPath}:/var/task:z`); if (options.dockerSsh) { - const homePath = process.env.HOME; + const homePath = require('os').homedir(); const sshKeyPath = options.dockerPrivateKey || `${homePath}/.ssh/id_rsa`; From 33a9afa2c734768632197bf996cb9d7e36e9c8ba Mon Sep 17 00:00:00 2001 From: Marcin Martinez Date: Mon, 7 Mar 2022 16:40:15 +0100 Subject: [PATCH 11/12] Test docker command resolution --- test.js | 20 +++++++++++--------- tests/base/custom_ssh | 1 + tests/base/id_ed25519 | 7 ------- 3 files changed, 12 insertions(+), 16 deletions(-) create mode 100644 tests/base/custom_ssh delete mode 100644 tests/base/id_ed25519 diff --git a/test.js b/test.js index 84a29666..53d07342 100644 --- a/test.js +++ b/test.js @@ -32,11 +32,11 @@ const mkCommand = args, options ); - if (error) { + if (error && !options['noThrow']) { console.error(`Error running: ${quote([cmd, ...args])}`); // eslint-disable-line no-console throw error; } - if (status) { + if (status && !options['noThrow']) { console.error('STDOUT: ', stdout.toString()); // eslint-disable-line no-console console.error('STDERR: ', stderr.toString()); // eslint-disable-line no-console throw new Error( @@ -201,27 +201,29 @@ const canUseDocker = () => { const brokenOn = (...platforms) => platforms.indexOf(process.platform) != -1; test( - 'can package private repo with dockerPrivateKey option', + 'dockerPrivateKey option correctly resolves docker command', async (t) => { process.chdir('tests/base'); const path = npm(['pack', '../..']); npm(['i', path]); - sls(['package'], { + const stdout = sls(['package'], { + noThrow: true, env: { dockerizePip: true, dockerSsh: true, - dockerPrivateKey: `${__dirname}${sep}tests${sep}base${sep}id_ed25519`, + dockerPrivateKey: `${__dirname}${sep}tests${sep}base${sep}custom_ssh`, fileName: 'requirements-w-git-ssh.txt', }, }); - const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip'); t.true( - zipfiles.includes(`dockerPrivateKey_test/__init__.py`), - 'private repo accessed with specified key' + stdout.includes( + `-v ${__dirname}${sep}tests${sep}base${sep}custom_ssh:/root/.ssh/custom_ssh:z` + ), + 'docker command properly resolved' ); t.end(); }, - { skip: !canUseDocker() || brokenOn('win32') || true } + { skip: !canUseDocker() || brokenOn('win32') } ); test( diff --git a/tests/base/custom_ssh b/tests/base/custom_ssh new file mode 100644 index 00000000..8a7c4203 --- /dev/null +++ b/tests/base/custom_ssh @@ -0,0 +1 @@ +SOME KEY diff --git a/tests/base/id_ed25519 b/tests/base/id_ed25519 deleted file mode 100644 index 43b88e4d..00000000 --- a/tests/base/id_ed25519 +++ /dev/null @@ -1,7 +0,0 @@ ------BEGIN OPENSSH PRIVATE KEY----- -b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW -QyNTUxOQAAACAns3vFG1eJtPnqVRylDQsyu4QGpkvqlH/mj8N6eYIqXAAAAKDM80pnzPNK -ZwAAAAtzc2gtZWQyNTUxOQAAACAns3vFG1eJtPnqVRylDQsyu4QGpkvqlH/mj8N6eYIqXA -AAAECYfiON2DfcWuu1uDsUs5INIh1adTbdwSmWa1khvvAawyeze8UbV4m0+epVHKUNCzK7 -hAamS+qUf+aPw3p5gipcAAAAG2Zpc2hlcm1hbm5ATWFyY2lucy1BaXIuaG9tZQEC ------END OPENSSH PRIVATE KEY----- From e3f1c5da6a4d1f74455c2cecb4f5addf9cb20d54 Mon Sep 17 00:00:00 2001 From: Marcin Martinez Date: Mon, 7 Mar 2022 21:56:29 +0100 Subject: [PATCH 12/12] Fix test: no SSH key in use --- test.js | 5 +++-- tests/base/requirements-w-git-ssh.txt | 2 -- tests/base/serverless.yml | 2 ++ 3 files changed, 5 insertions(+), 4 deletions(-) delete mode 100644 tests/base/requirements-w-git-ssh.txt diff --git a/test.js b/test.js index 53d07342..b228805e 100644 --- a/test.js +++ b/test.js @@ -3,6 +3,7 @@ const glob = require('glob-all'); const JSZip = require('jszip'); const sha256File = require('sha256-file'); const tape = require('tape-promise/tape'); + const { chmodSync, removeSync, @@ -23,7 +24,7 @@ const mkCommand = (cmd) => (args, options = {}) => { options['env'] = Object.assign( - { SLS_DEBUG: 't' }, + { SLS_DEBUG: 'true' }, process.env, options['env'] ); @@ -212,7 +213,7 @@ test( dockerizePip: true, dockerSsh: true, dockerPrivateKey: `${__dirname}${sep}tests${sep}base${sep}custom_ssh`, - fileName: 'requirements-w-git-ssh.txt', + dockerImage: 'break the build to log the command', }, }); t.true( diff --git a/tests/base/requirements-w-git-ssh.txt b/tests/base/requirements-w-git-ssh.txt deleted file mode 100644 index fb7c19db..00000000 --- a/tests/base/requirements-w-git-ssh.txt +++ /dev/null @@ -1,2 +0,0 @@ -flask -git+ssh://git@github.com/martinezpl/dockerPrivateKey_test.git@v1.0.0#egg=dockerPrivateKey_test \ No newline at end of file diff --git a/tests/base/serverless.yml b/tests/base/serverless.yml index 2eae49d5..6526246c 100644 --- a/tests/base/serverless.yml +++ b/tests/base/serverless.yml @@ -12,6 +12,7 @@ custom: dockerizePip: ${env:dockerizePip, self:custom.defaults.dockerizePip} dockerSsh: ${env:dockerSsh, self:custom.defaults.dockerSsh} dockerPrivateKey: ${env:dockerPrivateKey, self:custom.defaults.dockerPrivateKey} + dockerImage: ${env:dockerImage, self:custom.defaults.dockerImage} slim: ${env:slim, self:custom.defaults.slim} slimPatterns: ${file(./slimPatterns.yml):slimPatterns, self:custom.defaults.slimPatterns} slimPatternsAppendDefaults: ${env:slimPatternsAppendDefaults, self:custom.defaults.slimPatternsAppendDefaults} @@ -28,6 +29,7 @@ custom: dockerizePip: false dockerSsh: false dockerPrivateKey: '' + dockerImage: '' individually: false useStaticCache: true useDownloadCache: true