diff --git a/.evergreen/run-azurekms-fail-test.sh b/.evergreen/run-azurekms-fail-test.sh index 31ca30b3e2..eea84d42a9 100755 --- a/.evergreen/run-azurekms-fail-test.sh +++ b/.evergreen/run-azurekms-fail-test.sh @@ -2,10 +2,8 @@ set -o errexit # Exit the script with error if any of the commands fail HERE=$(dirname ${BASH_SOURCE:-$0}) . $DRIVERS_TOOLS/.evergreen/csfle/azurekms/setup-secrets.sh -export LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian11/master/latest/libmongocrypt.tar.gz SUCCESS=false TEST_FLE_AZURE_AUTO=1 bash $HERE/scripts/setup-tests.sh -PYTHON_BINARY=/opt/mongodbtoolchain/v4/bin/python3 \ - KEY_NAME="${AZUREKMS_KEYNAME}" \ +KEY_NAME="${AZUREKMS_KEYNAME}" \ KEY_VAULT_ENDPOINT="${AZUREKMS_KEYVAULTENDPOINT}" \ $HERE/just.sh test-eg bash $HERE/scripts/teardown-tests.sh diff --git a/.evergreen/run-azurekms-test.sh b/.evergreen/run-azurekms-test.sh index 27cb3fb315..8e6b050cb6 100755 --- a/.evergreen/run-azurekms-test.sh +++ b/.evergreen/run-azurekms-test.sh @@ -6,7 +6,6 @@ echo "Copying files ... begin" export AZUREKMS_RESOURCEGROUP=${AZUREKMS_RESOURCEGROUP} export AZUREKMS_VMNAME=${AZUREKMS_VMNAME} export AZUREKMS_PRIVATEKEYPATH=/tmp/testazurekms_privatekey -LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian11/master/latest/libmongocrypt.tar.gz # Set up the remote files to test. git add . git commit -m "add files" || true @@ -20,7 +19,7 @@ AZUREKMS_CMD="tar xf mongo-python-driver.tgz" \ $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh echo "Untarring file ... end" echo "Running test ... begin" -AZUREKMS_CMD="SUCCESS=true TEST_FLE_AZURE_AUTO=1 LIBMONGOCRYPT_URL=$LIBMONGOCRYPT_URL bash .evergreen/just.sh setup-test" \ +AZUREKMS_CMD="SUCCESS=true TEST_FLE_AZURE_AUTO=1 bash .evergreen/just.sh setup-test" \ $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh AZUREKMS_CMD="KEY_NAME=\"$AZUREKMS_KEYNAME\" KEY_VAULT_ENDPOINT=\"$AZUREKMS_KEYVAULTENDPOINT\" bash ./.evergreen/just.sh test-eg" \ $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh diff --git a/.evergreen/run-gcpkms-test.sh b/.evergreen/run-gcpkms-test.sh index 077ca0cb9f..a430f4e4f9 100755 --- a/.evergreen/run-gcpkms-test.sh +++ b/.evergreen/run-gcpkms-test.sh @@ -8,7 +8,6 @@ export GCPKMS_GCLOUD=${GCPKMS_GCLOUD} export GCPKMS_PROJECT=${GCPKMS_PROJECT} export GCPKMS_ZONE=${GCPKMS_ZONE} export GCPKMS_INSTANCENAME=${GCPKMS_INSTANCENAME} -LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian11/master/latest/libmongocrypt.tar.gz # Set up the remote files to test. git add . git commit -m "add files" || true @@ -19,7 +18,7 @@ echo "Untarring file ... begin" GCPKMS_CMD="tar xf mongo-python-driver.tgz" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh echo "Untarring file ... end" echo "Running test ... begin" -GCPKMS_CMD="SUCCESS=true TEST_FLE_GCP_AUTO=1 LIBMONGOCRYPT_URL=$LIBMONGOCRYPT_URL bash ./.evergreen/just.sh setup-test" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh +GCPKMS_CMD="SUCCESS=true TEST_FLE_GCP_AUTO=1 bash ./.evergreen/just.sh setup-test" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh GCPKMS_CMD="./.evergreen/just.sh test-eg" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh echo "Running test ... end" bash $HERE/scripts/teardown-tests.sh diff --git a/.evergreen/scripts/run-gcpkms-fail-test.sh b/.evergreen/scripts/run-gcpkms-fail-test.sh index 61f5e30ccc..8675c7c242 100755 --- a/.evergreen/scripts/run-gcpkms-fail-test.sh +++ b/.evergreen/scripts/run-gcpkms-fail-test.sh @@ -2,7 +2,5 @@ set -eu HERE=$(dirname ${BASH_SOURCE:-$0}) . $HERE/env.sh -export PYTHON_BINARY=/opt/mongodbtoolchain/v4/bin/python3 -export LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian11/master/latest/libmongocrypt.tar.gz SUCCESS=false TEST_FLE_GCP_AUTO=1 bash $HERE/setup-tests.sh bash ./.evergreen/just.sh test-eg diff --git a/.evergreen/scripts/setup-libmongocrypt.sh b/.evergreen/scripts/setup-libmongocrypt.sh deleted file mode 100755 index 775db07cb0..0000000000 --- a/.evergreen/scripts/setup-libmongocrypt.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/bin/bash -set -o errexit # Exit the script with error if any of the commands fail -set -o xtrace - -TARGET="" - -if [ "Windows_NT" = "${OS:-''}" ]; then # Magic variable in cygwin - # PYTHON-2808 Ensure this machine has the CA cert for google KMS. - powershell.exe "Invoke-WebRequest -URI https://oauth2.googleapis.com/" > /dev/null || true - TARGET="windows-test" -fi - -if [ "$(uname -s)" = "Darwin" ]; then - TARGET="macos" -fi - -if [ "$(uname -s)" = "Linux" ]; then - rhel_ver=$(awk -F'=' '/VERSION_ID/{ gsub(/"/,""); print $2}' /etc/os-release) - arch=$(uname -m) - echo "RHEL $rhel_ver $arch" - if [[ $rhel_ver =~ 7 ]]; then - TARGET="rhel-70-64-bit" - elif [[ $rhel_ver =~ 8 ]]; then - if [ "$arch" = "x86_64" ]; then - TARGET="rhel-80-64-bit" - elif [ "$arch" = "arm" ]; then - TARGET="rhel-82-arm64" - fi - fi -fi - -if [ -z "$LIBMONGOCRYPT_URL" ] && [ -n "$TARGET" ]; then - LIBMONGOCRYPT_URL="https://s3.amazonaws.com/mciuploads/libmongocrypt/$TARGET/master/latest/libmongocrypt.tar.gz" -fi - -if [ -z "$LIBMONGOCRYPT_URL" ]; then - echo "Cannot test client side encryption without LIBMONGOCRYPT_URL!" - exit 1 -fi -rm -rf libmongocrypt libmongocrypt.tar.gz -echo "Fetching $LIBMONGOCRYPT_URL..." -curl -O "$LIBMONGOCRYPT_URL" -echo "Fetching $LIBMONGOCRYPT_URL...done" -mkdir libmongocrypt -tar xzf libmongocrypt.tar.gz -C ./libmongocrypt -ls -la libmongocrypt -ls -la libmongocrypt/nocrypto - -if [ "Windows_NT" = "${OS:-''}" ]; then - # libmongocrypt's windows dll is not marked executable. - chmod +x libmongocrypt/nocrypto/bin/mongocrypt.dll -fi diff --git a/.evergreen/scripts/setup-tests.sh b/.evergreen/scripts/setup-tests.sh index 80fc717047..330469711c 100755 --- a/.evergreen/scripts/setup-tests.sh +++ b/.evergreen/scripts/setup-tests.sh @@ -58,4 +58,4 @@ fi . $ROOT_DIR/.evergreen/utils.sh PYTHON=${PYTHON_BINARY:-$(find_python3)} -$PYTHON $SCRIPT_DIR/setup-tests.py +$PYTHON $SCRIPT_DIR/setup_tests.py diff --git a/.evergreen/scripts/setup-tests.py b/.evergreen/scripts/setup_tests.py similarity index 79% rename from .evergreen/scripts/setup-tests.py rename to .evergreen/scripts/setup_tests.py index 4e14ff9830..07693d1e99 100644 --- a/.evergreen/scripts/setup-tests.py +++ b/.evergreen/scripts/setup_tests.py @@ -1,20 +1,26 @@ from __future__ import annotations import base64 +import dataclasses +import io import logging import os import platform import shlex +import shutil import stat import subprocess import sys +import tarfile from pathlib import Path from typing import Any +from urllib import request HERE = Path(__file__).absolute().parent ROOT = HERE.parent.parent ENV_FILE = HERE / "test-env.sh" DRIVERS_TOOLS = os.environ.get("DRIVERS_TOOLS", "").replace(os.sep, "/") +PLATFORM = "windows" if os.name == "nt" else sys.platform logging.basicConfig(level=logging.INFO) LOGGER = logging.getLogger(__name__) @@ -74,6 +80,13 @@ ) +@dataclasses.dataclass +class Distro: + name: str + version_id: str + arch: str + + def write_env(name: str, value: Any) -> None: with ENV_FILE.open("a", newline="\n") as fid: # Remove any existing quote chars. @@ -92,6 +105,69 @@ def run_command(cmd: str) -> None: LOGGER.info("Running command %s... done.", cmd) +def get_distro() -> Distro: + name = "" + version_id = "" + arch = platform.machine() + with open("/etc/os-release") as fid: + for line in fid.readlines(): + line = line.replace('"', "") # noqa: PLW2901 + if line.startswith("NAME="): + _, _, name = line.strip().partition("=") + if line.startswith("VERSION_ID="): + _, _, version_id = line.strip().partition("=") + return Distro(name=name, version_id=version_id, arch=arch) + + +def setup_libmongocrypt(): + target = "" + if PLATFORM == "windows": + # PYTHON-2808 Ensure this machine has the CA cert for google KMS. + if is_set("TEST_FLE_GCP_AUTO"): + run_command('powershell.exe "Invoke-WebRequest -URI https://oauth2.googleapis.com/"') + target = "windows-test" + + elif PLATFORM == "darwin": + target = "macos" + + else: + distro = get_distro() + if distro.name.startswith("Debian"): + target = f"debian{distro.version_id}" + elif distro.name.startswith("Red Hat"): + if distro.version_id.startswith("7"): + target = "rhel-70-64-bit" + elif distro.version_id.startswith("8"): + if distro.arch == "aarch64": + target = "rhel-82-arm64" + else: + target = "rhel-80-64-bit" + + if not is_set("LIBMONGOCRYPT_URL"): + if not target: + raise ValueError("Cannot find libmongocrypt target for current platform!") + url = f"https://s3.amazonaws.com/mciuploads/libmongocrypt/{target}/master/latest/libmongocrypt.tar.gz" + else: + url = os.environ["LIBMONGOCRYPT_URL"] + + shutil.rmtree(HERE / "libmongocrypt", ignore_errors=True) + + LOGGER.info(f"Fetching {url}...") + with request.urlopen(request.Request(url), timeout=15.0) as response: # noqa: S310 + if response.status == 200: + fileobj = io.BytesIO(response.read()) + with tarfile.open("libmongocrypt.tar.gz", fileobj=fileobj) as fid: + fid.extractall(Path.cwd() / "libmongocrypt") + LOGGER.info(f"Fetching {url}... done.") + + run_command("ls -la libmongocrypt") + run_command("ls -la libmongocrypt/nocrypto") + + if PLATFORM == "windows": + # libmongocrypt's windows dll is not marked executable. + run_command("chmod +x libmongocrypt/nocrypto/bin/mongocrypt.dll") + + def handle_test_env() -> None: AUTH = os.environ.get("AUTH", "noauth") SSL = os.environ.get("SSL", "nossl") @@ -156,7 +232,7 @@ def handle_test_env() -> None: write_env("PYMONGO_DISABLE_TEST_COMMANDS", "1") if is_set("TEST_ENTERPRISE_AUTH"): - if os.name == "nt": + if PLATFORM == "windows": LOGGER.info("Setting GSSAPI_PASS") write_env("GSSAPI_PASS", os.environ["SASL_PASS"]) write_env("GSSAPI_CANONICALIZE", "true") @@ -214,19 +290,19 @@ def handle_test_env() -> None: if is_set("TEST_ENCRYPTION") or is_set("TEST_FLE_AZURE_AUTO") or is_set("TEST_FLE_GCP_AUTO"): # Check for libmongocrypt download. if not (ROOT / "libmongocrypt").exists(): - run_command(f"bash {HERE.as_posix()}/setup-libmongocrypt.sh") + setup_libmongocrypt() # TODO: Test with 'pip install pymongocrypt' UV_ARGS.append("--group pymongocrypt_source") # Use the nocrypto build to avoid dependency issues with older windows/python versions. BASE = ROOT / "libmongocrypt/nocrypto" - if sys.platform == "linux": + if PLATFORM == "linux": if (BASE / "lib/libmongocrypt.so").exists(): PYMONGOCRYPT_LIB = BASE / "lib/libmongocrypt.so" else: PYMONGOCRYPT_LIB = BASE / "lib64/libmongocrypt.so" - elif sys.platform == "darwin": + elif PLATFORM == "darwin": PYMONGOCRYPT_LIB = BASE / "lib/libmongocrypt.dylib" else: PYMONGOCRYPT_LIB = BASE / "bin/mongocrypt.dll" @@ -244,7 +320,7 @@ def handle_test_env() -> None: if is_set("TEST_CRYPT_SHARED"): CRYPT_SHARED_DIR = Path(os.environ["CRYPT_SHARED_LIB_PATH"]).parent.as_posix() LOGGER.info("Using crypt_shared_dir %s", CRYPT_SHARED_DIR) - if os.name == "nt": + if PLATFORM == "windows": write_env("PATH", f"{CRYPT_SHARED_DIR}:$PATH") else: write_env(