From 25ca1802da0ddf51ff3fe21cb1e6419ef9549770 Mon Sep 17 00:00:00 2001 From: Jack Henschel Date: Mon, 9 Mar 2020 10:34:18 +0200 Subject: [PATCH 1/2] Mount the entire user SSH directory into build container This enables the user to use an key file format (RSA, ED25519, ...). Additionally, it allows more complex workflows (such as different SSH keys for specfic sites, such as Github or Bitbucket), since the .ssh/config file is also mounted into the container. Fixes https://github.com/UnitedIncome/serverless-python-requirements/issues/488 --- README.md | 2 +- lib/pip.js | 13 ++++++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 17107a00..d57defd3 100644 --- a/README.md +++ b/README.md @@ -71,7 +71,7 @@ custom: dockerSsh: true ``` -The `dockerSsh` option will mount your `$HOME/.ssh/id_rsa` and `$HOME/.ssh/known_hosts` as a +The `dockerSsh` option will mount your `$HOME/.ssh/` directory as a volume in the docker container. If your SSH key is password protected, you can use `ssh-agent` because `$SSH_AUTH_SOCK` is also mounted & the env var set. It is important that the host of your private repositories has already been added in your diff --git a/lib/pip.js b/lib/pip.js index 7d1777a4..0a68125c 100644 --- a/lib/pip.js +++ b/lib/pip.js @@ -204,14 +204,21 @@ function installRequirements(targetFolder, serverless, options) { // Mount necessary ssh files to work with private repos dockerCmd.push( '-v', - `${process.env.HOME}/.ssh/id_rsa:/root/.ssh/id_rsa:z`, - '-v', - `${process.env.HOME}/.ssh/known_hosts:/root/.ssh/known_hosts:z`, + `${process.env.HOME}/.ssh/:/root/.ssh/:z`, '-v', `${process.env.SSH_AUTH_SOCK}:/tmp/ssh_sock:z`, '-e', 'SSH_AUTH_SOCK=/tmp/ssh_sock' ); + + // If the user has a SSH_CONFIG file, it won't have the correct permissions + // inside the docker container, and the ssh command will fail with + // > Bad owner or permissions on /root/.ssh/config + // However, if the we specify the SSH_CONFIG file with -F explicitly, + // ssh does not check the ownership of the file. + if (fse.existsSync(`${process.env.HOME}/.ssh/config`)) { + dockerCmd.push('-e', 'GIT_SSH_COMMAND=ssh -F /root/.ssh/config'); + } } // If we want a download cache... From 0217946398c9e82363ca81962200bfcbb4fb4332 Mon Sep 17 00:00:00 2001 From: Jack Henschel Date: Mon, 16 Mar 2020 09:46:05 +0200 Subject: [PATCH 2/2] Add test to check whether the plugin mounts the entire ssh dir This test makes sure the plugin actually mounts the SSH directory into the container, if the dockerizePip and dockerSsh are given. It does this by creating a new known_hosts file and placing the RSA fingerprint of Github inside, then installing a package via Git, which requires this fingerprint to be present in known_hosts. If it is not present, the process will fail. This patch also adds minor debugging output, in case no requirements.txt is found. --- lib/pip.js | 3 +++ test.js | 33 ++++++++++++++++++++++++++- tests/base/requirements-w-git-ssh.txt | 3 +++ tests/base/serverless.yml | 4 ++-- 4 files changed, 40 insertions(+), 3 deletions(-) create mode 100644 tests/base/requirements-w-git-ssh.txt diff --git a/lib/pip.js b/lib/pip.js index 0a68125c..902aa941 100644 --- a/lib/pip.js +++ b/lib/pip.js @@ -491,6 +491,9 @@ function installRequirementsIfNeeded( // Skip requirements generation, if requirements file doesn't exist if (!requirementsFileExists(servicePath, options, fileName)) { + serverless.cli.log( + `Skipping generation of requirements: file ${fileName} not found` + ); return false; } diff --git a/test.js b/test.js index 0b589b99..35660b69 100644 --- a/test.js +++ b/test.js @@ -8,12 +8,14 @@ const { removeSync, readFile, copySync, + ensureFileSync, + appendFileSync, writeFileSync, statSync, pathExistsSync } = require('fs-extra'); const { quote } = require('shell-quote'); -const { sep } = require('path'); +const { sep, resolve } = require('path'); const { getUserCachePath, sha256Path } = require('./lib/shared'); @@ -579,6 +581,35 @@ test( { skip: !hasPython(2) } ); +test( + 'dockerSsh mounts entire ssh folder into docker', + t => { + process.chdir('tests/base'); + const path = npm(['pack', '../..']); + npm(['i', path]); + // create a known_hosts file with the rsa fingerprint of github + // the plugin should mount the entire .ssh directory into the container + const known_hosts_file = resolve(process.env.HOME, `./.ssh/known_hosts`); + ensureFileSync(known_hosts_file); + appendFileSync( + known_hosts_file, + 'github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==' + ); + // verify this by installing a requirement via git+ssh + sls([ + '--dockerizePip=true', + '--dockerSsh=true', + '--fileName=requirements-w-git-ssh.txt', + 'package' + ]); + const zipfiles = listZipFiles('.serverless/sls-py-req-test.zip'); + // check if the requirement is actually in the archive + t.true(zipfiles.includes(`boto3/__init__.py`), 'boto3 is packaged via ssh'); + t.end(); + }, + { skip: !canUseDocker() } +); + test( 'py2.7 can package flask with slim & dockerizePip & slimPatterns options', async t => { diff --git a/tests/base/requirements-w-git-ssh.txt b/tests/base/requirements-w-git-ssh.txt new file mode 100644 index 00000000..8fd6da43 --- /dev/null +++ b/tests/base/requirements-w-git-ssh.txt @@ -0,0 +1,3 @@ +flask +bottle +git+ssh://git@github.com/boto/boto3.git#egg=boto3 diff --git a/tests/base/serverless.yml b/tests/base/serverless.yml index 0763da0a..6250c96d 100644 --- a/tests/base/serverless.yml +++ b/tests/base/serverless.yml @@ -10,6 +10,7 @@ custom: pythonRequirements: zip: ${opt:zip, self:custom.defaults.zip} dockerizePip: ${opt:dockerizePip, self:custom.defaults.dockerizePip} + dockerSsh: ${opt:dockerSsh, self:custom.defaults.dockerSsh} slim: ${opt:slim, self:custom.defaults.slim} slimPatterns: ${file(./slimPatterns.yml):slimPatterns, self:custom.defaults.slimPatterns} slimPatternsAppendDefaults: ${opt:slimPatternsAppendDefaults, self:custom.defaults.slimPatternsAppendDefaults} @@ -24,6 +25,7 @@ custom: slimPatternsAppendDefaults: true zip: false dockerizePip: false + dockerSsh: false individually: false useStaticCache: true useDownloadCache: true @@ -49,5 +51,3 @@ functions: package: include: - 'fn2/**' - -