From 94e4c474bf423f9dbabb35918f4d815d7b1cb3f0 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 3 Aug 2022 20:13:28 -0500 Subject: [PATCH 01/27] PYTHON-2818 Add native support for AWS IAM Roles for service accounts, EKS in particular --- .evergreen/config.yml | 42 +++++++++++++++++++++++++++++++++++++++++- setup.py | 4 +++- 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index ac7f97f6fa..87002f2273 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -97,7 +97,7 @@ functions: # If this was a patch build, doing a fresh clone would not actually test the patch cp -R ${PROJECT_DIRECTORY}/ $DRIVERS_TOOLS else - git clone https://github.com/mongodb-labs/drivers-evergreen-tools.git $DRIVERS_TOOLS + git clone -b DRIVERS-1746 https://github.com/blink1073/drivers-evergreen-tools.git $DRIVERS_TOOLS fi echo "{ \"releases\": { \"default\": \"$MONGODB_BINARIES\" }}" > $MONGO_ORCHESTRATION_HOME/orchestration.config @@ -553,6 +553,11 @@ functions: "iam_auth_ec2_instance_account" : "${iam_auth_ec2_instance_account}", "iam_auth_ec2_instance_secret_access_key" : "${iam_auth_ec2_instance_secret_access_key}", "iam_auth_ec2_instance_profile" : "${iam_auth_ec2_instance_profile}" + + "iam_auth_assume_web_role_name": "$iam_auth_assume_web_role_name", + "iam_web_identity_issuer": "${iam_web_identity_issuer}", + "iam_web_identity_jwks_uri": "${iam_web_identity_jwks_uri}", + "iam_web_identity_token_file": "${iam_web_identity_jwks_uri}", } EOF @@ -642,6 +647,36 @@ functions: working_dir: "src" script: | ${PREPARE_SHELL} + rm "${PROJECT_DIRECTORY}/prepare_mongodb_aws.sh" + PYTHON_BINARY=${PYTHON_BINARY} ASSERT_NO_URI_CREDS=true .evergreen/run-mongodb-aws-test.sh + + "run aws auth test with aws web identity credentials": + - command: shell.exec + type: test + params: + working_dir: "src" + script: | + ${PREPARE_SHELL} + if [ "${skip_EC2_auth_test}" = "true" ]; then + echo "This platform does not support the web identity auth test, skipping..." + exit 0 + fi + cd ${DRIVERS_TOOLS}/.evergreen/auth_aws + . ./activate_venv.sh + mongo aws_e2e_web_identity.js + - command: shell.exec + type: test + params: + working_dir: "src" + script: | + ${PREPARE_SHELL} + + export AWS_ROLE_ARN=${iam_auth_assume_web_role_name} + export IDP_ISSUER=${iam_web_identity_issuer} + export IDP_JWKS_URI=${iam_web_identity_jwks_uri} + export AWS_WEB_IDENTITY_TOKEN_FILE=${iam_web_identity_token_file} + + rm "${PROJECT_DIRECTORY}/prepare_mongodb_aws.sh" PYTHON_BINARY=${PYTHON_BINARY} ASSERT_NO_URI_CREDS=true .evergreen/run-mongodb-aws-test.sh "run aws auth test with aws credentials as environment variables": @@ -1705,6 +1740,7 @@ tasks: - func: "run aws auth test with aws credentials as environment variables" - func: "run aws auth test with aws credentials and session token as environment variables" - func: "run aws auth test with aws EC2 credentials" + - func: "run aws auth test with aws web identity credentials" - func: "run aws ECS auth test" - name: "aws-auth-test-5.0" @@ -1721,6 +1757,7 @@ tasks: - func: "run aws auth test with aws credentials as environment variables" - func: "run aws auth test with aws credentials and session token as environment variables" - func: "run aws auth test with aws EC2 credentials" + - func: "run aws auth test with aws web identity credentials" - func: "run aws ECS auth test" - name: "aws-auth-test-6.0" @@ -1737,6 +1774,7 @@ tasks: - func: "run aws auth test with aws credentials as environment variables" - func: "run aws auth test with aws credentials and session token as environment variables" - func: "run aws auth test with aws EC2 credentials" + - func: "run aws auth test with aws web identity credentials" - func: "run aws ECS auth test" - name: "aws-auth-test-latest" @@ -1753,6 +1791,7 @@ tasks: - func: "run aws auth test with aws credentials as environment variables" - func: "run aws auth test with aws credentials and session token as environment variables" - func: "run aws auth test with aws EC2 credentials" + - func: "run aws auth test with aws web identity credentials" - func: "run aws ECS auth test" - name: "aws-auth-test-rapid" commands: @@ -1768,6 +1807,7 @@ tasks: - func: "run aws auth test with aws credentials as environment variables" - func: "run aws auth test with aws credentials and session token as environment variables" - func: "run aws auth test with aws EC2 credentials" + - func: "run aws auth test with aws web identity credentials" - func: "run aws ECS auth test" - name: load-balancer-test diff --git a/setup.py b/setup.py index 0e983e4642..90b8bf1fbb 100755 --- a/setup.py +++ b/setup.py @@ -280,7 +280,9 @@ def build_extension(self, ext): "ocsp": pyopenssl_reqs, "snappy": ["python-snappy"], "zstd": ["zstandard"], - "aws": ["pymongo-auth-aws<2.0.0"], + "aws": [ + "pymongo-auth-aws@https://github.com/mongodb/pymongo-auth-aws/archive/refs/heads/master.zip" + ], "srv": ["dnspython>=1.16.0,<3.0.0"], } From 376785f0ee7bccfee8948392f94c647ea3dd8da4 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 4 Aug 2022 10:30:45 -0500 Subject: [PATCH 02/27] limit tasks --- .evergreen/config.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 87002f2273..26042db6ca 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -1802,13 +1802,13 @@ tasks: TOPOLOGY: "server" VERSION: "rapid" - func: "add aws auth variables to file" - - func: "run aws auth test with regular aws credentials" - - func: "run aws auth test with assume role credentials" - - func: "run aws auth test with aws credentials as environment variables" - - func: "run aws auth test with aws credentials and session token as environment variables" - - func: "run aws auth test with aws EC2 credentials" + # - func: "run aws auth test with regular aws credentials" + # - func: "run aws auth test with assume role credentials" + # - func: "run aws auth test with aws credentials as environment variables" + # - func: "run aws auth test with aws credentials and session token as environment variables" + # - func: "run aws auth test with aws EC2 credentials" - func: "run aws auth test with aws web identity credentials" - - func: "run aws ECS auth test" + # - func: "run aws ECS auth test" - name: load-balancer-test commands: From 2563b8d02e4054199ba3149b0842cc87a04a955b Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 4 Aug 2022 10:54:30 -0500 Subject: [PATCH 03/27] fix syntax --- .evergreen/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 26042db6ca..d83b27fb3d 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -552,7 +552,7 @@ functions: "iam_auth_ec2_instance_account" : "${iam_auth_ec2_instance_account}", "iam_auth_ec2_instance_secret_access_key" : "${iam_auth_ec2_instance_secret_access_key}", - "iam_auth_ec2_instance_profile" : "${iam_auth_ec2_instance_profile}" + "iam_auth_ec2_instance_profile" : "${iam_auth_ec2_instance_profile}", "iam_auth_assume_web_role_name": "$iam_auth_assume_web_role_name", "iam_web_identity_issuer": "${iam_web_identity_issuer}", From 94e611b41ff97ff9cc04868b3b0b133761cc6255 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 4 Aug 2022 11:01:00 -0500 Subject: [PATCH 04/27] another try at config --- .evergreen/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index d83b27fb3d..73d27414da 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -554,10 +554,10 @@ functions: "iam_auth_ec2_instance_secret_access_key" : "${iam_auth_ec2_instance_secret_access_key}", "iam_auth_ec2_instance_profile" : "${iam_auth_ec2_instance_profile}", - "iam_auth_assume_web_role_name": "$iam_auth_assume_web_role_name", + "iam_auth_assume_web_role_name": "${iam_auth_assume_web_role_name}", "iam_web_identity_issuer": "${iam_web_identity_issuer}", "iam_web_identity_jwks_uri": "${iam_web_identity_jwks_uri}", - "iam_web_identity_token_file": "${iam_web_identity_jwks_uri}", + "iam_web_identity_token_file": "${iam_web_identity_token_file}", } EOF From 03f0bb06c1ba38931cd1634cfca9f17ad8062b30 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 4 Aug 2022 11:26:27 -0500 Subject: [PATCH 05/27] remove trailing comma --- .evergreen/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 73d27414da..51f9f4058c 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -557,7 +557,7 @@ functions: "iam_auth_assume_web_role_name": "${iam_auth_assume_web_role_name}", "iam_web_identity_issuer": "${iam_web_identity_issuer}", "iam_web_identity_jwks_uri": "${iam_web_identity_jwks_uri}", - "iam_web_identity_token_file": "${iam_web_identity_token_file}", + "iam_web_identity_token_file": "${iam_web_identity_token_file}" } EOF From 90d8af45dab6762ff3e0b4dbafcbdd8440fb3ce8 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 4 Aug 2022 14:56:28 -0500 Subject: [PATCH 06/27] fix creds usage --- .evergreen/config.yml | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 51f9f4058c..4ec6f24889 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -647,7 +647,7 @@ functions: working_dir: "src" script: | ${PREPARE_SHELL} - rm "${PROJECT_DIRECTORY}/prepare_mongodb_aws.sh" + rm "${PROJECT_DIRECTORY}/prepare_mongodb_aws.sh" || true PYTHON_BINARY=${PYTHON_BINARY} ASSERT_NO_URI_CREDS=true .evergreen/run-mongodb-aws-test.sh "run aws auth test with aws web identity credentials": @@ -664,19 +664,23 @@ functions: cd ${DRIVERS_TOOLS}/.evergreen/auth_aws . ./activate_venv.sh mongo aws_e2e_web_identity.js + - command: shell.exec + type: test + params: + working_dir: "src" + silent: true + script: | + # DO NOT ECHO WITH XTRACE (which PREPARE_SHELL does) + cat <<'EOF' > "${PROJECT_DIRECTORY}/prepare_mongodb_aws.sh" + AWS_ROLE_ARN=${iam_auth_assume_web_role_name} + AWS_WEB_IDENTITY_TOKEN_FILE=${iam_web_identity_token_file} + EOF - command: shell.exec type: test params: working_dir: "src" script: | ${PREPARE_SHELL} - - export AWS_ROLE_ARN=${iam_auth_assume_web_role_name} - export IDP_ISSUER=${iam_web_identity_issuer} - export IDP_JWKS_URI=${iam_web_identity_jwks_uri} - export AWS_WEB_IDENTITY_TOKEN_FILE=${iam_web_identity_token_file} - - rm "${PROJECT_DIRECTORY}/prepare_mongodb_aws.sh" PYTHON_BINARY=${PYTHON_BINARY} ASSERT_NO_URI_CREDS=true .evergreen/run-mongodb-aws-test.sh "run aws auth test with aws credentials as environment variables": From 4a6741011f47df396f21b1dee1dcdd711da11410 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 4 Aug 2022 15:04:59 -0500 Subject: [PATCH 07/27] ensure proper aws lib --- .evergreen/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 4ec6f24889..cd79482b67 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -681,6 +681,7 @@ functions: working_dir: "src" script: | ${PREPARE_SHELL} + ${PYTHON_BINARY} -m pip install https://github.com/mongodb/pymongo-auth-aws/archive/refs/heads/master.zip PYTHON_BINARY=${PYTHON_BINARY} ASSERT_NO_URI_CREDS=true .evergreen/run-mongodb-aws-test.sh "run aws auth test with aws credentials as environment variables": From 077b942ab7010ba30895110791f7be7c3b9ab0c4 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 4 Aug 2022 15:31:45 -0500 Subject: [PATCH 08/27] fix install --- .evergreen/config.yml | 1 - .evergreen/run-mongodb-aws-test.sh | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index cd79482b67..4ec6f24889 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -681,7 +681,6 @@ functions: working_dir: "src" script: | ${PREPARE_SHELL} - ${PYTHON_BINARY} -m pip install https://github.com/mongodb/pymongo-auth-aws/archive/refs/heads/master.zip PYTHON_BINARY=${PYTHON_BINARY} ASSERT_NO_URI_CREDS=true .evergreen/run-mongodb-aws-test.sh "run aws auth test with aws credentials as environment variables": diff --git a/.evergreen/run-mongodb-aws-test.sh b/.evergreen/run-mongodb-aws-test.sh index 9a33507cc8..fd394fd1e5 100755 --- a/.evergreen/run-mongodb-aws-test.sh +++ b/.evergreen/run-mongodb-aws-test.sh @@ -60,7 +60,8 @@ authtest () { else . venvaws/bin/activate fi - python -m pip install '.[aws]' + python -m pip install '.' + python -m pip install https://github.com/mongodb/pymongo-auth-aws/archive/refs/heads/master.zip python test/auth_aws/test_auth_aws.py deactivate rm -rf venvaws From 6ecd6230b47bf515017e31575209f6ef5f1c50ba Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 4 Aug 2022 15:50:07 -0500 Subject: [PATCH 09/27] debug --- .evergreen/config.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 4ec6f24889..430d22c7da 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -672,8 +672,8 @@ functions: script: | # DO NOT ECHO WITH XTRACE (which PREPARE_SHELL does) cat <<'EOF' > "${PROJECT_DIRECTORY}/prepare_mongodb_aws.sh" - AWS_ROLE_ARN=${iam_auth_assume_web_role_name} - AWS_WEB_IDENTITY_TOKEN_FILE=${iam_web_identity_token_file} + AWS_ROLE_ARN="${iam_auth_assume_web_role_name}" + AWS_WEB_IDENTITY_TOKEN_FILE="${iam_web_identity_token_file}" EOF - command: shell.exec type: test @@ -681,7 +681,9 @@ functions: working_dir: "src" script: | ${PREPARE_SHELL} + cat "${PROJECT_DIRECTORY}/prepare_mongodb_aws.sh" PYTHON_BINARY=${PYTHON_BINARY} ASSERT_NO_URI_CREDS=true .evergreen/run-mongodb-aws-test.sh + exit 1 # should remove the cat above "run aws auth test with aws credentials as environment variables": - command: shell.exec From b07605cc435b87311b0920b513700ce73e19dd33 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 4 Aug 2022 15:59:56 -0500 Subject: [PATCH 10/27] try an export --- .evergreen/config.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 430d22c7da..a52ef87c0c 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -672,8 +672,8 @@ functions: script: | # DO NOT ECHO WITH XTRACE (which PREPARE_SHELL does) cat <<'EOF' > "${PROJECT_DIRECTORY}/prepare_mongodb_aws.sh" - AWS_ROLE_ARN="${iam_auth_assume_web_role_name}" - AWS_WEB_IDENTITY_TOKEN_FILE="${iam_web_identity_token_file}" + export AWS_ROLE_ARN="${iam_auth_assume_web_role_name}" + export AWS_WEB_IDENTITY_TOKEN_FILE="${iam_web_identity_token_file}" EOF - command: shell.exec type: test @@ -681,9 +681,7 @@ functions: working_dir: "src" script: | ${PREPARE_SHELL} - cat "${PROJECT_DIRECTORY}/prepare_mongodb_aws.sh" PYTHON_BINARY=${PYTHON_BINARY} ASSERT_NO_URI_CREDS=true .evergreen/run-mongodb-aws-test.sh - exit 1 # should remove the cat above "run aws auth test with aws credentials as environment variables": - command: shell.exec From 885c030c9779dc306be9e69ec165c84d970e2808 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 4 Aug 2022 17:03:01 -0500 Subject: [PATCH 11/27] make rsa key a variable --- .evergreen/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index a52ef87c0c..e439dbcf4f 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -556,6 +556,7 @@ functions: "iam_auth_assume_web_role_name": "${iam_auth_assume_web_role_name}", "iam_web_identity_issuer": "${iam_web_identity_issuer}", + "iam_web_identity_rsa_key": "${iam_web_identity_rsa_key}", "iam_web_identity_jwks_uri": "${iam_web_identity_jwks_uri}", "iam_web_identity_token_file": "${iam_web_identity_token_file}" } From ae2d67e5c01bb51169fb8a99054b7e0a5be27feb Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 4 Aug 2022 17:04:19 -0500 Subject: [PATCH 12/27] clean up config --- .evergreen/config.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index e439dbcf4f..83f77b25fe 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -1807,11 +1807,11 @@ tasks: TOPOLOGY: "server" VERSION: "rapid" - func: "add aws auth variables to file" - # - func: "run aws auth test with regular aws credentials" - # - func: "run aws auth test with assume role credentials" - # - func: "run aws auth test with aws credentials as environment variables" - # - func: "run aws auth test with aws credentials and session token as environment variables" - # - func: "run aws auth test with aws EC2 credentials" + - func: "run aws auth test with regular aws credentials" + - func: "run aws auth test with assume role credentials" + - func: "run aws auth test with aws credentials as environment variables" + - func: "run aws auth test with aws credentials and session token as environment variables" + - func: "run aws auth test with aws EC2 credentials" - func: "run aws auth test with aws web identity credentials" # - func: "run aws ECS auth test" From ea4e36f82171ed120572fdb1af5240baebc4997b Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 8 Aug 2022 12:29:07 -0500 Subject: [PATCH 13/27] restore skipped task --- .evergreen/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 83f77b25fe..da3c51140d 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -1813,7 +1813,7 @@ tasks: - func: "run aws auth test with aws credentials and session token as environment variables" - func: "run aws auth test with aws EC2 credentials" - func: "run aws auth test with aws web identity credentials" - # - func: "run aws ECS auth test" + - func: "run aws ECS auth test" - name: load-balancer-test commands: From f6a5d2886563f8a2f8602793a03fba0bca4b4f10 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 29 Aug 2022 12:33:50 -0500 Subject: [PATCH 14/27] add test with AWS_ROLE_SESSION_NAME --- .evergreen/config.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index da3c51140d..d0a9531990 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -683,6 +683,25 @@ functions: script: | ${PREPARE_SHELL} PYTHON_BINARY=${PYTHON_BINARY} ASSERT_NO_URI_CREDS=true .evergreen/run-mongodb-aws-test.sh + - command: shell.exec + type: test + params: + working_dir: "src" + silent: true + script: | + # DO NOT ECHO WITH XTRACE (which PREPARE_SHELL does) + cat <<'EOF' > "${PROJECT_DIRECTORY}/prepare_mongodb_aws.sh" + export AWS_ROLE_ARN="${iam_auth_assume_web_role_name}" + export AWS_WEB_IDENTITY_TOKEN_FILE="${iam_web_identity_token_file}" + export AWS_ROLE_SESSION_NAME="test" + EOF + - command: shell.exec + type: test + params: + working_dir: "src" + script: | + ${PREPARE_SHELL} + PYTHON_BINARY=${PYTHON_BINARY} ASSERT_NO_URI_CREDS=true .evergreen/run-mongodb-aws-test.sh "run aws auth test with aws credentials as environment variables": - command: shell.exec From a365989b8cc5257337f74e06924624041c5790dc Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 30 Aug 2022 16:42:29 -0500 Subject: [PATCH 15/27] try with boto3 --- pymongo/auth_aws.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pymongo/auth_aws.py b/pymongo/auth_aws.py index 4b2af35ea4..c1d6cb656b 100644 --- a/pymongo/auth_aws.py +++ b/pymongo/auth_aws.py @@ -67,6 +67,15 @@ def _authenticate_aws(credentials, sock_info): credentials.mechanism_properties.aws_session_token, ) ) + import boto3 + session = boto3.Session() + if not credentials.username: + credentials = session.get_credentials() + ctx._credentials = AwsCredential( + credentials.access_key, + credentials.secret_key, + credentials.session_token + ) client_payload = ctx.step(None) client_first = SON( [("saslStart", 1), ("mechanism", "MONGODB-AWS"), ("payload", client_payload)] From 9c58ef74ccf31c958312f2b749562bbaa7bdd9d0 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 30 Aug 2022 17:02:22 -0500 Subject: [PATCH 16/27] try with released pymongo_auth_aws --- .evergreen/run-mongodb-aws-test.sh | 4 ++-- pymongo/auth_aws.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.evergreen/run-mongodb-aws-test.sh b/.evergreen/run-mongodb-aws-test.sh index fd394fd1e5..9ca27abbb1 100755 --- a/.evergreen/run-mongodb-aws-test.sh +++ b/.evergreen/run-mongodb-aws-test.sh @@ -60,8 +60,8 @@ authtest () { else . venvaws/bin/activate fi - python -m pip install '.' - python -m pip install https://github.com/mongodb/pymongo-auth-aws/archive/refs/heads/master.zip + python -m pip install '.[aws]' + # python -m pip install https://github.com/mongodb/pymongo-auth-aws/archive/refs/heads/master.zip python test/auth_aws/test_auth_aws.py deactivate rm -rf venvaws diff --git a/pymongo/auth_aws.py b/pymongo/auth_aws.py index c1d6cb656b..df963dd3ac 100644 --- a/pymongo/auth_aws.py +++ b/pymongo/auth_aws.py @@ -74,7 +74,7 @@ def _authenticate_aws(credentials, sock_info): ctx._credentials = AwsCredential( credentials.access_key, credentials.secret_key, - credentials.session_token + credentials.token ) client_payload = ctx.step(None) client_first = SON( From aa394bf407cf1309aecf8cb985f0517fe7adee17 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 30 Aug 2022 17:04:26 -0500 Subject: [PATCH 17/27] more assurances --- pymongo/auth_aws.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pymongo/auth_aws.py b/pymongo/auth_aws.py index df963dd3ac..ae39f5bd0a 100644 --- a/pymongo/auth_aws.py +++ b/pymongo/auth_aws.py @@ -68,9 +68,10 @@ def _authenticate_aws(credentials, sock_info): ) ) import boto3 - session = boto3.Session() if not credentials.username: + session = boto3.Session() credentials = session.get_credentials() + assert credentials.username is not None ctx._credentials = AwsCredential( credentials.access_key, credentials.secret_key, From 5c0dfd0b1fbe71576cf08a7c51ecd672ec62f932 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 30 Aug 2022 17:14:08 -0500 Subject: [PATCH 18/27] install boto3 --- .evergreen/run-mongodb-aws-ecs-test.sh | 2 +- .evergreen/run-mongodb-aws-test.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.evergreen/run-mongodb-aws-ecs-test.sh b/.evergreen/run-mongodb-aws-ecs-test.sh index 83f3975e9e..69c1670570 100755 --- a/.evergreen/run-mongodb-aws-ecs-test.sh +++ b/.evergreen/run-mongodb-aws-ecs-test.sh @@ -29,7 +29,7 @@ authtest () { $PYTHON --version $PYTHON -m pip install --upgrade wheel setuptools pip cd src - $PYTHON -m pip install '.[aws]' + $PYTHON -m pip install '.[aws]' boto3 $PYTHON test/auth_aws/test_auth_aws.py cd - } diff --git a/.evergreen/run-mongodb-aws-test.sh b/.evergreen/run-mongodb-aws-test.sh index 9ca27abbb1..c91671f1d7 100755 --- a/.evergreen/run-mongodb-aws-test.sh +++ b/.evergreen/run-mongodb-aws-test.sh @@ -60,7 +60,7 @@ authtest () { else . venvaws/bin/activate fi - python -m pip install '.[aws]' + python -m pip install '.[aws]' boto3 # python -m pip install https://github.com/mongodb/pymongo-auth-aws/archive/refs/heads/master.zip python test/auth_aws/test_auth_aws.py deactivate From 7e5ce35bec1ce5425e23c5e16cda5b723ab0bd29 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 30 Aug 2022 17:24:27 -0500 Subject: [PATCH 19/27] fix check --- pymongo/auth_aws.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymongo/auth_aws.py b/pymongo/auth_aws.py index ae39f5bd0a..73801ff00b 100644 --- a/pymongo/auth_aws.py +++ b/pymongo/auth_aws.py @@ -71,7 +71,7 @@ def _authenticate_aws(credentials, sock_info): if not credentials.username: session = boto3.Session() credentials = session.get_credentials() - assert credentials.username is not None + assert credentials.access_key is not None ctx._credentials = AwsCredential( credentials.access_key, credentials.secret_key, From 889611b7254a251dc657c3e93b8d5193659c5048 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 30 Aug 2022 17:25:30 -0500 Subject: [PATCH 20/27] more refactor --- pymongo/auth_aws.py | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/pymongo/auth_aws.py b/pymongo/auth_aws.py index 73801ff00b..dc5229a646 100644 --- a/pymongo/auth_aws.py +++ b/pymongo/auth_aws.py @@ -60,23 +60,27 @@ def _authenticate_aws(credentials, sock_info): raise ConfigurationError("MONGODB-AWS authentication requires MongoDB version 4.4 or later") try: - ctx = _AwsSaslContext( - AwsCredential( - credentials.username, - credentials.password, - credentials.mechanism_properties.aws_session_token, - ) - ) import boto3 if not credentials.username: session = boto3.Session() credentials = session.get_credentials() assert credentials.access_key is not None - ctx._credentials = AwsCredential( - credentials.access_key, - credentials.secret_key, - credentials.token + ctx = _AwsSaslContext( + AwsCredential( + credentials.access_key, + credentials.secret_key, + credentials.token, + ) ) + else: + ctx = _AwsSaslContext( + AwsCredential( + credentials.username, + credentials.password, + credentials.mechanism_properties.aws_session_token, + ) + ) + client_payload = ctx.step(None) client_first = SON( [("saslStart", 1), ("mechanism", "MONGODB-AWS"), ("payload", client_payload)] From 4e153620a44c18937e32ab2ad47079a4f77fe6a1 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 31 Aug 2022 11:03:49 -0500 Subject: [PATCH 21/27] run against boto3 --- .evergreen/run-mongodb-aws-ecs-test.sh | 3 ++- .evergreen/run-mongodb-aws-test.sh | 3 ++- pymongo/auth_aws.py | 25 ++++++------------------- 3 files changed, 10 insertions(+), 21 deletions(-) diff --git a/.evergreen/run-mongodb-aws-ecs-test.sh b/.evergreen/run-mongodb-aws-ecs-test.sh index 69c1670570..9d3bd7f629 100755 --- a/.evergreen/run-mongodb-aws-ecs-test.sh +++ b/.evergreen/run-mongodb-aws-ecs-test.sh @@ -29,7 +29,8 @@ authtest () { $PYTHON --version $PYTHON -m pip install --upgrade wheel setuptools pip cd src - $PYTHON -m pip install '.[aws]' boto3 + $PYTHON -m pip install '.' + $PYTHON -m pip install https://github.com/blink1073/pymongo-auth-aws/archive/refs/heads/PYTHON-3418.zip $PYTHON test/auth_aws/test_auth_aws.py cd - } diff --git a/.evergreen/run-mongodb-aws-test.sh b/.evergreen/run-mongodb-aws-test.sh index c91671f1d7..988c049ca3 100755 --- a/.evergreen/run-mongodb-aws-test.sh +++ b/.evergreen/run-mongodb-aws-test.sh @@ -60,8 +60,9 @@ authtest () { else . venvaws/bin/activate fi - python -m pip install '.[aws]' boto3 + python -m pip install '.' # python -m pip install https://github.com/mongodb/pymongo-auth-aws/archive/refs/heads/master.zip + python -m pip install https://github.com/blink1073/pymongo-auth-aws/archive/refs/heads/PYTHON-3418.zip python test/auth_aws/test_auth_aws.py deactivate rm -rf venvaws diff --git a/pymongo/auth_aws.py b/pymongo/auth_aws.py index dc5229a646..923c9f1492 100644 --- a/pymongo/auth_aws.py +++ b/pymongo/auth_aws.py @@ -60,26 +60,13 @@ def _authenticate_aws(credentials, sock_info): raise ConfigurationError("MONGODB-AWS authentication requires MongoDB version 4.4 or later") try: - import boto3 - if not credentials.username: - session = boto3.Session() - credentials = session.get_credentials() - assert credentials.access_key is not None - ctx = _AwsSaslContext( - AwsCredential( - credentials.access_key, - credentials.secret_key, - credentials.token, - ) - ) - else: - ctx = _AwsSaslContext( - AwsCredential( - credentials.username, - credentials.password, - credentials.mechanism_properties.aws_session_token, - ) + ctx = _AwsSaslContext( + AwsCredential( + credentials.username, + credentials.password, + credentials.mechanism_properties.aws_session_token, ) + ) client_payload = ctx.step(None) client_first = SON( From 115e702e2a797fdc7965570c0deba90413797c69 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 12 Sep 2022 11:05:51 -0500 Subject: [PATCH 22/27] try again --- .evergreen/config.yml | 7 +++++++ .evergreen/run-mongodb-aws-test.sh | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 9a1fe99525..13a5bc969a 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -702,6 +702,13 @@ functions: export AWS_WEB_IDENTITY_TOKEN_FILE="${iam_web_identity_token_file}" export AWS_ROLE_SESSION_NAME="test" EOF + - command: shell.exec + type: test + params: + working_dir: "src" + script: | + ${PREPARE_SHELL} + PYTHON_BINARY=${PYTHON_BINARY} ASSERT_NO_URI_CREDS=true .evergreen/run-mongodb-aws-test.sh "run aws auth test with aws credentials as environment variables": - command: shell.exec diff --git a/.evergreen/run-mongodb-aws-test.sh b/.evergreen/run-mongodb-aws-test.sh index 988c049ca3..ebe2bcd394 100755 --- a/.evergreen/run-mongodb-aws-test.sh +++ b/.evergreen/run-mongodb-aws-test.sh @@ -60,7 +60,7 @@ authtest () { else . venvaws/bin/activate fi - python -m pip install '.' + python -m pip install '.[aws]' # python -m pip install https://github.com/mongodb/pymongo-auth-aws/archive/refs/heads/master.zip python -m pip install https://github.com/blink1073/pymongo-auth-aws/archive/refs/heads/PYTHON-3418.zip python test/auth_aws/test_auth_aws.py From ee9b82bcc56fd6b634ee784335a9d84f999b6cbc Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 12 Sep 2022 11:16:04 -0500 Subject: [PATCH 23/27] skip non-ubunutu variants --- .evergreen/config.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 13a5bc969a..ec1c37152d 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -689,6 +689,10 @@ functions: working_dir: "src" script: | ${PREPARE_SHELL} + if [ "${skip_EC2_auth_test}" = "true" ]; then + echo "This platform does not support the web identity auth test, skipping..." + exit 0 + fi PYTHON_BINARY=${PYTHON_BINARY} ASSERT_NO_URI_CREDS=true .evergreen/run-mongodb-aws-test.sh - command: shell.exec type: test @@ -708,6 +712,10 @@ functions: working_dir: "src" script: | ${PREPARE_SHELL} + if [ "${skip_EC2_auth_test}" = "true" ]; then + echo "This platform does not support the web identity auth test, skipping..." + exit 0 + fi PYTHON_BINARY=${PYTHON_BINARY} ASSERT_NO_URI_CREDS=true .evergreen/run-mongodb-aws-test.sh "run aws auth test with aws credentials as environment variables": From 8778563cf2e8965cbad143e32551be5e0ac0eae4 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 12 Sep 2022 11:29:19 -0500 Subject: [PATCH 24/27] more explicit skip --- .evergreen/config.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index ec1c37152d..4e250068bc 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -689,7 +689,7 @@ functions: working_dir: "src" script: | ${PREPARE_SHELL} - if [ "${skip_EC2_auth_test}" = "true" ]; then + if [ "${skip_web_identity_auth_test}" = "true" ]; then echo "This platform does not support the web identity auth test, skipping..." exit 0 fi @@ -712,7 +712,7 @@ functions: working_dir: "src" script: | ${PREPARE_SHELL} - if [ "${skip_EC2_auth_test}" = "true" ]; then + if [ "${skip_web_identity_auth_test}" = "true" ]; then echo "This platform does not support the web identity auth test, skipping..." exit 0 fi @@ -1920,6 +1920,7 @@ axes: variables: skip_EC2_auth_test: true skip_ECS_auth_test: true + skip_web_identity_auth_test: true python3_binary: /Library/Frameworks/Python.framework/Versions/3.8/bin/python3 libmongocrypt_url: https://s3.amazonaws.com/mciuploads/libmongocrypt/macos/master/latest/libmongocrypt.tar.gz - id: macos-1100 @@ -1928,6 +1929,7 @@ axes: variables: skip_EC2_auth_test: true skip_ECS_auth_test: true + skip_web_identity_auth_test: true python3_binary: /Library/Frameworks/Python.framework/Versions/3.8/bin/python3 libmongocrypt_url: https://s3.amazonaws.com/mciuploads/libmongocrypt/macos/master/latest/libmongocrypt.tar.gz - id: rhel62 @@ -1991,6 +1993,7 @@ axes: batchtime: 10080 # 7 days variables: skip_ECS_auth_test: true + skip_web_identity_auth_test: true python3_binary: "C:/python/Python38/python.exe" venv_bin_dir: "Scripts" libmongocrypt_url: https://s3.amazonaws.com/mciuploads/libmongocrypt/windows-test/master/latest/libmongocrypt.tar.gz From 7ada61191430ab4ffad2b3927ed60c76a7986390 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 12 Sep 2022 11:54:41 -0500 Subject: [PATCH 25/27] more cleanup --- .evergreen/config.yml | 1 + .evergreen/run-mongodb-aws-test.sh | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 4e250068bc..8f3b19c38d 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -1993,6 +1993,7 @@ axes: batchtime: 10080 # 7 days variables: skip_ECS_auth_test: true + skip_EC2_auth_test: true skip_web_identity_auth_test: true python3_binary: "C:/python/Python38/python.exe" venv_bin_dir: "Scripts" diff --git a/.evergreen/run-mongodb-aws-test.sh b/.evergreen/run-mongodb-aws-test.sh index ebe2bcd394..bad085250e 100755 --- a/.evergreen/run-mongodb-aws-test.sh +++ b/.evergreen/run-mongodb-aws-test.sh @@ -60,7 +60,8 @@ authtest () { else . venvaws/bin/activate fi - python -m pip install '.[aws]' + python -m pip install --upgrade wheel + python -m pip install '.' # python -m pip install https://github.com/mongodb/pymongo-auth-aws/archive/refs/heads/master.zip python -m pip install https://github.com/blink1073/pymongo-auth-aws/archive/refs/heads/PYTHON-3418.zip python test/auth_aws/test_auth_aws.py From 4c4b801f20a24525261800c956e0e412205556f6 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 26 Oct 2022 20:12:02 -0500 Subject: [PATCH 26/27] address review --- pymongo/auth_aws.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pymongo/auth_aws.py b/pymongo/auth_aws.py index fa9e02fdb7..e84465ea66 100644 --- a/pymongo/auth_aws.py +++ b/pymongo/auth_aws.py @@ -78,7 +78,6 @@ def _authenticate_aws(credentials, sock_info): credentials.mechanism_properties.aws_session_token, ) ) - client_payload = ctx.step(None) client_first = SON( [("saslStart", 1), ("mechanism", "MONGODB-AWS"), ("payload", client_payload)] From 423366ba7de9705a645720875e06788215890c49 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 14 Nov 2022 06:57:15 -0600 Subject: [PATCH 27/27] remove branch pin --- .evergreen/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index b2f8d35829..b70da15779 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -97,7 +97,7 @@ functions: # If this was a patch build, doing a fresh clone would not actually test the patch cp -R ${PROJECT_DIRECTORY}/ $DRIVERS_TOOLS else - git clone -b DRIVERS-1746 https://github.com/blink1073/drivers-evergreen-tools.git $DRIVERS_TOOLS + git clone https://github.com/mongodb-labs/drivers-evergreen-tools.git $DRIVERS_TOOLS fi echo "{ \"releases\": { \"default\": \"$MONGODB_BINARIES\" }}" > $MONGO_ORCHESTRATION_HOME/orchestration.config