Skip to content

Commit 1c7a7fe

Browse files
NoahStappJibola
andauthored
PYTHON-4721 - Create individualized scripts for all shell.exec commands (#1997)
Co-authored-by: Jib <[email protected]>
1 parent 906d021 commit 1c7a7fe

40 files changed

+683
-430
lines changed

.evergreen/config.yml

Lines changed: 303 additions & 407 deletions
Large diffs are not rendered by default.

.evergreen/hatch.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ else # Set up virtualenv before installing hatch
2929
# Ensure hatch does not write to user or global locations.
3030
touch hatch_config.toml
3131
HATCH_CONFIG=$(pwd)/hatch_config.toml
32-
if [ "Windows_NT" = "$OS" ]; then # Magic variable in cygwin
32+
if [ "Windows_NT" = "${OS:-}" ]; then # Magic variable in cygwin
3333
HATCH_CONFIG=$(cygpath -m "$HATCH_CONFIG")
3434
fi
3535
export HATCH_CONFIG

.evergreen/run-tests.sh

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -53,18 +53,18 @@ if [ -z "${NO_EXT:-}" ]; then
5353
fi
5454

5555
if [ "$AUTH" != "noauth" ]; then
56-
if [ ! -z "$TEST_DATA_LAKE" ]; then
56+
if [ -n "$TEST_DATA_LAKE" ]; then
5757
export DB_USER="mhuser"
5858
export DB_PASSWORD="pencil"
59-
elif [ ! -z "$TEST_SERVERLESS" ]; then
60-
source ${DRIVERS_TOOLS}/.evergreen/serverless/secrets-export.sh
59+
elif [ -n "$TEST_SERVERLESS" ]; then
60+
source "${DRIVERS_TOOLS}"/.evergreen/serverless/secrets-export.sh
6161
export DB_USER=$SERVERLESS_ATLAS_USER
6262
export DB_PASSWORD=$SERVERLESS_ATLAS_PASSWORD
6363
export MONGODB_URI="$SERVERLESS_URI"
6464
echo "MONGODB_URI=$MONGODB_URI"
6565
export SINGLE_MONGOS_LB_URI=$MONGODB_URI
6666
export MULTI_MONGOS_LB_URI=$MONGODB_URI
67-
elif [ ! -z "$TEST_AUTH_OIDC" ]; then
67+
elif [ -n "$TEST_AUTH_OIDC" ]; then
6868
export DB_USER=$OIDC_ADMIN_USER
6969
export DB_PASSWORD=$OIDC_ADMIN_PWD
7070
export DB_IP="$MONGODB_URI"
Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
#!/bin/bash
2+
3+
set -o xtrace
4+
mkdir out_dir
5+
# shellcheck disable=SC2156
6+
find "$MONGO_ORCHESTRATION_HOME" -name \*.log -exec sh -c 'x="{}"; mv $x $PWD/out_dir/$(basename $(dirname $x))_$(basename $x)' \;
7+
tar zcvf mongodb-logs.tar.gz -C out_dir/ .
8+
rm -rf out_dir
Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
#!/bin/bash
2+
3+
set -o xtrace
4+
5+
# Enable core dumps if enabled on the machine
6+
# Copied from https://github.com/mongodb/mongo/blob/master/etc/evergreen.yml
7+
if [ -f /proc/self/coredump_filter ]; then
8+
# Set the shell process (and its children processes) to dump ELF headers (bit 4),
9+
# anonymous shared mappings (bit 1), and anonymous private mappings (bit 0).
10+
echo 0x13 >/proc/self/coredump_filter
11+
12+
if [ -f /sbin/sysctl ]; then
13+
# Check that the core pattern is set explicitly on our distro image instead
14+
# of being the OS's default value. This ensures that coredump names are consistent
15+
# across distros and can be picked up by Evergreen.
16+
core_pattern=$(/sbin/sysctl -n "kernel.core_pattern")
17+
if [ "$core_pattern" = "dump_%e.%p.core" ]; then
18+
echo "Enabling coredumps"
19+
ulimit -c unlimited
20+
fi
21+
fi
22+
fi
23+
24+
if [ "$(uname -s)" = "Darwin" ]; then
25+
core_pattern_mac=$(/usr/sbin/sysctl -n "kern.corefile")
26+
if [ "$core_pattern_mac" = "dump_%N.%P.core" ]; then
27+
echo "Enabling coredumps"
28+
ulimit -c unlimited
29+
fi
30+
fi
31+
32+
if [ -n "${skip_crypt_shared}" ]; then
33+
export SKIP_CRYPT_SHARED=1
34+
fi
35+
36+
MONGODB_VERSION=${VERSION} \
37+
TOPOLOGY=${TOPOLOGY} \
38+
AUTH=${AUTH:-noauth} \
39+
SSL=${SSL:-nossl} \
40+
STORAGE_ENGINE=${STORAGE_ENGINE:-} \
41+
DISABLE_TEST_COMMANDS=${DISABLE_TEST_COMMANDS:-} \
42+
ORCHESTRATION_FILE=${ORCHESTRATION_FILE:-} \
43+
REQUIRE_API_VERSION=${REQUIRE_API_VERSION:-} \
44+
LOAD_BALANCER=${LOAD_BALANCER:-} \
45+
bash ${DRIVERS_TOOLS}/.evergreen/run-orchestration.sh
46+
# run-orchestration generates expansion file with the MONGODB_URI for the cluster
Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
#!/bin/bash
2+
3+
. .evergreen/scripts/env.sh
4+
set -x
5+
export BASE_SHA="$1"
6+
export HEAD_SHA="$2"
7+
bash .evergreen/run-import-time-test.sh

.evergreen/scripts/cleanup.sh

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
#!/bin/bash
2+
3+
if [ -f "$DRIVERS_TOOLS"/.evergreen/csfle/secrets-export.sh ]; then
4+
. .evergreen/hatch.sh encryption:teardown
5+
fi
6+
rm -rf "${DRIVERS_TOOLS}" || true
7+
rm -f ./secrets-export.sh || true

.evergreen/scripts/configure-env.sh

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
#!/bin/bash -ex
1+
#!/bin/bash -eux
22

33
# Get the current unique version of this checkout
44
# shellcheck disable=SC2154
@@ -29,7 +29,7 @@ fi
2929
export MONGO_ORCHESTRATION_HOME="$DRIVERS_TOOLS/.evergreen/orchestration"
3030
export MONGODB_BINARIES="$DRIVERS_TOOLS/mongodb/bin"
3131

32-
cat <<EOT > $SCRIPT_DIR/env.sh
32+
cat <<EOT > "$SCRIPT_DIR"/env.sh
3333
set -o errexit
3434
export PROJECT_DIRECTORY="$PROJECT_DIRECTORY"
3535
export CURRENT_VERSION="$CURRENT_VERSION"
@@ -38,6 +38,21 @@ export DRIVERS_TOOLS="$DRIVERS_TOOLS"
3838
export MONGO_ORCHESTRATION_HOME="$MONGO_ORCHESTRATION_HOME"
3939
export MONGODB_BINARIES="$MONGODB_BINARIES"
4040
export PROJECT_DIRECTORY="$PROJECT_DIRECTORY"
41+
export SETDEFAULTENCODING="${SETDEFAULTENCODING:-}"
42+
export SKIP_CSOT_TESTS="${SKIP_CSOT_TESTS:-}"
43+
export MONGODB_STARTED="${MONGODB_STARTED:-}"
44+
export DISABLE_TEST_COMMANDS="${DISABLE_TEST_COMMANDS:-}"
45+
export GREEN_FRAMEWORK="${GREEN_FRAMEWORK:-}"
46+
export NO_EXT="${NO_EXT:-}"
47+
export COVERAGE="${COVERAGE:-}"
48+
export COMPRESSORS="${COMPRESSORS:-}"
49+
export MONGODB_API_VERSION="${MONGODB_API_VERSION:-}"
50+
export SKIP_HATCH="${SKIP_HATCH:-}"
51+
export skip_crypt_shared="${skip_crypt_shared:-}"
52+
export STORAGE_ENGINE="${STORAGE_ENGINE:-}"
53+
export REQUIRE_API_VERSION="${REQUIRE_API_VERSION:-}"
54+
export skip_web_identity_auth_test="${skip_web_identity_auth_test:-}"
55+
export skip_ECS_auth_test="${skip_ECS_auth_test:-}"
4156
4257
export TMPDIR="$MONGO_ORCHESTRATION_HOME/db"
4358
export PATH="$MONGODB_BINARIES:$PATH"
Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
#!/bin/bash
2+
3+
# Download all the task coverage files.
4+
aws s3 cp --recursive s3://"$1"/coverage/"$2"/"$3"/coverage/ coverage/
Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
#!/bin/bash
2+
3+
set +x
4+
. src/.evergreen/scripts/env.sh
5+
# shellcheck disable=SC2044
6+
for filename in $(find $DRIVERS_TOOLS -name \*.json); do
7+
perl -p -i -e "s|ABSOLUTE_PATH_REPLACEMENT_TOKEN|$DRIVERS_TOOLS|g" $filename
8+
done
Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
#!/bin/bash
2+
3+
set +x
4+
. src/.evergreen/scripts/env.sh
5+
echo '{"results": [{ "status": "FAIL", "test_file": "Build", "log_raw": "No test-results.json found was created" } ]}' >$PROJECT_DIRECTORY/test-results.json
Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
#!/bin/bash
2+
3+
set -o xtrace
4+
file="$PROJECT_DIRECTORY/.evergreen/install-dependencies.sh"
5+
# Don't use ${file} syntax here because evergreen treats it as an empty expansion.
6+
[ -f "$file" ] && bash "$file" || echo "$file not available, skipping"
Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
#!/bin/bash
2+
3+
set +x
4+
. src/.evergreen/scripts/env.sh
5+
# shellcheck disable=SC2044
6+
for i in $(find "$DRIVERS_TOOLS"/.evergreen "$PROJECT_DIRECTORY"/.evergreen -name \*.sh); do
7+
chmod +x "$i"
8+
done
Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
#!/bin/bash
2+
3+
. src/.evergreen/scripts/env.sh
4+
set -o xtrace
5+
rm -rf $DRIVERS_TOOLS
6+
if [ "$PROJECT" = "drivers-tools" ]; then
7+
# If this was a patch build, doing a fresh clone would not actually test the patch
8+
cp -R $PROJECT_DIRECTORY/ $DRIVERS_TOOLS
9+
else
10+
git clone https://github.com/mongodb-labs/drivers-evergreen-tools.git $DRIVERS_TOOLS
11+
fi
12+
echo "{ \"releases\": { \"default\": \"$MONGODB_BINARIES\" }}" >$MONGO_ORCHESTRATION_HOME/orchestration.config

.evergreen/scripts/run-atlas-tests.sh

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
#!/bin/bash
2+
3+
# Disable xtrace for security reasons (just in case it was accidentally set).
4+
set +x
5+
set -o errexit
6+
bash "${DRIVERS_TOOLS}"/.evergreen/auth_aws/setup_secrets.sh drivers/atlas_connect
7+
TEST_ATLAS=1 bash "${PROJECT_DIRECTORY}"/.evergreen/hatch.sh test:test-eg
Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
#!/bin/bash
2+
3+
# shellcheck disable=SC2154
4+
if [ "${skip_ECS_auth_test}" = "true" ]; then
5+
echo "This platform does not support the ECS auth test, skipping..."
6+
exit 0
7+
fi
8+
set -ex
9+
cd "$DRIVERS_TOOLS"/.evergreen/auth_aws
10+
. ./activate-authawsvenv.sh
11+
. aws_setup.sh ecs
12+
export MONGODB_BINARIES="$MONGODB_BINARIES"
13+
export PROJECT_DIRECTORY="$PROJECT_DIRECTORY"
14+
python aws_tester.py ecs
15+
cd -

.evergreen/scripts/run-doctests.sh

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
#!/bin/bash
2+
3+
set -o xtrace
4+
PYTHON_BINARY=${PYTHON_BINARY} bash "${PROJECT_DIRECTORY}"/.evergreen/hatch.sh doctest:test
Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
#!/bin/bash
2+
3+
# Disable xtrace for security reasons (just in case it was accidentally set).
4+
set +x
5+
bash "${DRIVERS_TOOLS}"/.evergreen/auth_aws/setup_secrets.sh drivers/enterprise_auth
6+
TEST_ENTERPRISE_AUTH=1 AUTH=auth bash "${PROJECT_DIRECTORY}"/.evergreen/hatch.sh test:test-eg
Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
#!/bin/bash
2+
3+
. .evergreen/scripts/env.sh
4+
export PYTHON_BINARY=/opt/mongodbtoolchain/v4/bin/python3
5+
export LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian11/master/latest/libmongocrypt.tar.gz
6+
SKIP_SERVERS=1 bash ./.evergreen/setup-encryption.sh
7+
SUCCESS=false TEST_FLE_GCP_AUTO=1 ./.evergreen/hatch.sh test:test-eg

.evergreen/scripts/run-getdata.sh

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
#!/bin/bash
2+
3+
set -o xtrace
4+
. ${DRIVERS_TOOLS}/.evergreen/download-mongodb.sh || true
5+
get_distro || true
6+
echo $DISTRO
7+
echo $MARCH
8+
echo $OS
9+
uname -a || true
10+
ls /etc/*release* || true
11+
cc --version || true
12+
gcc --version || true
13+
clang --version || true
14+
gcov --version || true
15+
lcov --version || true
16+
llvm-cov --version || true
17+
echo $PATH
18+
ls -la /usr/local/Cellar/llvm/*/bin/ || true
19+
ls -la /usr/local/Cellar/ || true
20+
scan-build --version || true
21+
genhtml --version || true
22+
valgrind --version || true
Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
#!/bin/bash
2+
3+
MONGODB_URI=${MONGODB_URI} bash "${DRIVERS_TOOLS}"/.evergreen/run-load-balancer.sh start
Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
#!/bin/bash
2+
3+
set -o xtrace
4+
export PYTHON_BINARY=${PYTHON_BINARY}
5+
bash "${PROJECT_DIRECTORY}"/.evergreen/hatch.sh test:test-mockupdb

.evergreen/run-mod-wsgi-tests.sh renamed to .evergreen/scripts/run-mod-wsgi-tests.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ export MOD_WSGI_SO=/opt/python/mod_wsgi/python_version/$PYTHON_VERSION/mod_wsgi_
2828
export PYTHONHOME=/opt/python/$PYTHON_VERSION
2929
# If MOD_WSGI_EMBEDDED is set use the default embedded mode behavior instead
3030
# of daemon mode (WSGIDaemonProcess).
31-
if [ -n "$MOD_WSGI_EMBEDDED" ]; then
31+
if [ -n "${MOD_WSGI_EMBEDDED:-}" ]; then
3232
export MOD_WSGI_CONF=mod_wsgi_test_embedded.conf
3333
else
3434
export MOD_WSGI_CONF=mod_wsgi_test.conf

.evergreen/run-mongodb-aws-test.sh renamed to .evergreen/scripts/run-mongodb-aws-test.sh

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,10 +13,16 @@ set -o errexit # Exit the script with error if any of the commands fail
1313
# mechanism.
1414
# PYTHON_BINARY The Python version to use.
1515

16-
echo "Running MONGODB-AWS authentication tests"
16+
# shellcheck disable=SC2154
17+
if [ "${skip_EC2_auth_test:-}" = "true" ] && { [ "$1" = "ec2" ] || [ "$1" = "web-identity" ]; }; then
18+
echo "This platform does not support the EC2 auth test, skipping..."
19+
exit 0
20+
fi
21+
22+
echo "Running MONGODB-AWS authentication tests for $1"
1723

1824
# Handle credentials and environment setup.
19-
. $DRIVERS_TOOLS/.evergreen/auth_aws/aws_setup.sh $1
25+
. "$DRIVERS_TOOLS"/.evergreen/auth_aws/aws_setup.sh "$1"
2026

2127
# show test output
2228
set -x

.evergreen/scripts/run-ocsp-test.sh

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
#!/bin/bash
2+
3+
TEST_OCSP=1 \
4+
PYTHON_BINARY="${PYTHON_BINARY}" \
5+
CA_FILE="${DRIVERS_TOOLS}/.evergreen/ocsp/${OCSP_ALGORITHM}/ca.pem" \
6+
OCSP_TLS_SHOULD_SUCCEED="${OCSP_TLS_SHOULD_SUCCEED}" \
7+
bash "${PROJECT_DIRECTORY}"/.evergreen/hatch.sh test:test-eg
8+
bash "${DRIVERS_TOOLS}"/.evergreen/ocsp/teardown.sh

.evergreen/scripts/run-perf-tests.sh

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
#!/bin/bash
2+
3+
PROJECT_DIRECTORY=${PROJECT_DIRECTORY}
4+
bash "${PROJECT_DIRECTORY}"/.evergreen/run-perf-tests.sh

.evergreen/scripts/run-tests.sh

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
#!/bin/bash
2+
3+
# Disable xtrace
4+
set +x
5+
if [ -n "${MONGODB_STARTED}" ]; then
6+
export PYMONGO_MUST_CONNECT=true
7+
fi
8+
if [ -n "${DISABLE_TEST_COMMANDS}" ]; then
9+
export PYMONGO_DISABLE_TEST_COMMANDS=1
10+
fi
11+
if [ -n "${test_encryption}" ]; then
12+
# Disable xtrace (just in case it was accidentally set).
13+
set +x
14+
bash "${DRIVERS_TOOLS}"/.evergreen/csfle/await-servers.sh
15+
export TEST_ENCRYPTION=1
16+
if [ -n "${test_encryption_pyopenssl}" ]; then
17+
export TEST_ENCRYPTION_PYOPENSSL=1
18+
fi
19+
fi
20+
if [ -n "${test_crypt_shared}" ]; then
21+
export TEST_CRYPT_SHARED=1
22+
export CRYPT_SHARED_LIB_PATH=${CRYPT_SHARED_LIB_PATH}
23+
fi
24+
if [ -n "${test_pyopenssl}" ]; then
25+
export TEST_PYOPENSSL=1
26+
fi
27+
if [ -n "${SETDEFAULTENCODING}" ]; then
28+
export SETDEFAULTENCODING="${SETDEFAULTENCODING}"
29+
fi
30+
if [ -n "${test_loadbalancer}" ]; then
31+
export TEST_LOADBALANCER=1
32+
export SINGLE_MONGOS_LB_URI="${SINGLE_MONGOS_LB_URI}"
33+
export MULTI_MONGOS_LB_URI="${MULTI_MONGOS_LB_URI}"
34+
fi
35+
if [ -n "${test_serverless}" ]; then
36+
export TEST_SERVERLESS=1
37+
fi
38+
if [ -n "${TEST_INDEX_MANAGEMENT:-}" ]; then
39+
export TEST_INDEX_MANAGEMENT=1
40+
fi
41+
if [ -n "${SKIP_CSOT_TESTS}" ]; then
42+
export SKIP_CSOT_TESTS=1
43+
fi
44+
GREEN_FRAMEWORK=${GREEN_FRAMEWORK} \
45+
PYTHON_BINARY=${PYTHON_BINARY} \
46+
NO_EXT=${NO_EXT} \
47+
COVERAGE=${COVERAGE} \
48+
COMPRESSORS=${COMPRESSORS} \
49+
AUTH=${AUTH} \
50+
SSL=${SSL} \
51+
TEST_DATA_LAKE=${TEST_DATA_LAKE:-} \
52+
TEST_SUITES=${TEST_SUITES:-} \
53+
MONGODB_API_VERSION=${MONGODB_API_VERSION} \
54+
SKIP_HATCH=${SKIP_HATCH} \
55+
bash "${PROJECT_DIRECTORY}"/.evergreen/hatch.sh test:test-eg

.evergreen/scripts/run-with-env.sh

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
#!/bin/bash -eu
2+
3+
# Example use: bash run-with-env.sh run-tests.sh {args...}
4+
5+
# Parameter expansion to get just the current directory's name
6+
if [ "${PWD##*/}" == "src" ]; then
7+
. .evergreen/scripts/env.sh
8+
if [ -f ".evergreen/scripts/test-env.sh" ]; then
9+
. .evergreen/scripts/test-env.sh
10+
fi
11+
else
12+
. src/.evergreen/scripts/env.sh
13+
if [ -f "src/.evergreen/scripts/test-env.sh" ]; then
14+
. src/.evergreen/scripts/test-env.sh
15+
fi
16+
fi
17+
18+
set -eu
19+
20+
# shellcheck source=/dev/null
21+
. "$@"
Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
#!/bin/bash
2+
3+
if [ -n "${test_encryption}" ]; then
4+
./.evergreen/hatch.sh encryption:setup
5+
fi

0 commit comments

Comments
 (0)