From b78777d0ec8fa11e7f302c660de97bd919ce18e9 Mon Sep 17 00:00:00 2001 From: Rey Abolofia Date: Wed, 17 Apr 2024 14:45:02 -0700 Subject: [PATCH 01/10] Remove urllib3 dist-info files as well. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 12faa696..db8378b3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,7 +20,7 @@ RUN rm -rf ./python/lib/$runtime/site-packages/botocore* RUN rm -rf ./python/lib/$runtime/site-packages/setuptools RUN rm -rf ./python/lib/$runtime/site-packages/jsonschema/tests RUN find . -name 'libddwaf.so' -delete -RUN rm -rf ./python/lib/$runtime/site-packages/urllib3 +RUN rm -rf ./python/lib/$runtime/site-packages/urllib3* RUN rm ./python/lib/$runtime/site-packages/ddtrace/appsec/_iast/_taint_tracking/*.so RUN rm ./python/lib/$runtime/site-packages/ddtrace/appsec/_iast/_stacktrace*.so RUN rm ./python/lib/$runtime/site-packages/ddtrace/internal/datadog/profiling/libdd_wrapper.so From 6470ea68d6945993816b1271149cda1da31e047e Mon Sep 17 00:00:00 2001 From: Rey Abolofia Date: Wed, 17 Apr 2024 14:45:21 -0700 Subject: [PATCH 02/10] Pre-compile all python code and remove original .py files. --- Dockerfile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index db8378b3..79771462 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,9 +11,6 @@ WORKDIR /build COPY . . RUN pip install . -t ./python/lib/$runtime/site-packages -# Remove *.pyc files -RUN find ./python/lib/$runtime/site-packages -name \*.pyc -delete - # Remove botocore (40MB) to reduce package size. aws-xray-sdk # installs it, while it's already provided by the Lambda Runtime. RUN rm -rf ./python/lib/$runtime/site-packages/botocore* @@ -29,5 +26,9 @@ RUN rm ./python/lib/$runtime/site-packages/ddtrace/internal/datadog/profiling/st RUN find . -name "*.dist-info" -type d | xargs rm -rf +RUN python -m compileall -b ./python/lib/$runtime/site-packages +RUN find ./python/lib/$runtime/site-packages -name \*.py -delete +RUN find ./python/lib/$runtime/site-packages -name __pycache__ -type d -exec rm -r {} \+ + FROM scratch COPY --from=builder /build/python / From 1df454f3d556d141a251a876b3ef856f6f034500 Mon Sep 17 00:00:00 2001 From: Rey Abolofia Date: Wed, 17 Apr 2024 14:57:00 -0700 Subject: [PATCH 03/10] Compress at optimization level 2. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 79771462..aa8224f1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,7 +26,7 @@ RUN rm ./python/lib/$runtime/site-packages/ddtrace/internal/datadog/profiling/st RUN find . -name "*.dist-info" -type d | xargs rm -rf -RUN python -m compileall -b ./python/lib/$runtime/site-packages +RUN python -m compileall -o 2 -b ./python/lib/$runtime/site-packages RUN find ./python/lib/$runtime/site-packages -name \*.py -delete RUN find ./python/lib/$runtime/site-packages -name __pycache__ -type d -exec rm -r {} \+ From b9eb7358a3f544b8224afeb8ff46c6ce4226212f Mon Sep 17 00:00:00 2001 From: Rey Abolofia Date: Thu, 18 Apr 2024 09:54:36 -0700 Subject: [PATCH 04/10] Add PYTHONNODEBUGRANGES=1 to compile step. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index aa8224f1..89d8790b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,7 +26,7 @@ RUN rm ./python/lib/$runtime/site-packages/ddtrace/internal/datadog/profiling/st RUN find . -name "*.dist-info" -type d | xargs rm -rf -RUN python -m compileall -o 2 -b ./python/lib/$runtime/site-packages +RUN PYTHONNODEBUGRANGES=1 python -m compileall -o 2 -b ./python/lib/$runtime/site-packages RUN find ./python/lib/$runtime/site-packages -name \*.py -delete RUN find ./python/lib/$runtime/site-packages -name __pycache__ -type d -exec rm -r {} \+ From 34a3dc031b8770285a39bcafa978db2394c4367e Mon Sep 17 00:00:00 2001 From: Rey Abolofia Date: Fri, 19 Apr 2024 11:19:15 -0700 Subject: [PATCH 05/10] Add comment. --- Dockerfile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Dockerfile b/Dockerfile index 89d8790b..163a44e0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,6 +26,12 @@ RUN rm ./python/lib/$runtime/site-packages/ddtrace/internal/datadog/profiling/st RUN find . -name "*.dist-info" -type d | xargs rm -rf +# Precompile all .pyc files and remove .py files. This speeds up load time. +# Compile with optimization level 2 (-o 2) and PYTHONNODEBUGRANGES=1 to redtce +# size of .pyc files. +# See https://docs.python.org/3/tutorial/modules.html#compiled-python-files +# https://docs.python.org/3.11/using/cmdline.html#cmdoption-O +# https://docs.python.org/3/using/cmdline.html#envvar-PYTHONNODEBUGRANGES RUN PYTHONNODEBUGRANGES=1 python -m compileall -o 2 -b ./python/lib/$runtime/site-packages RUN find ./python/lib/$runtime/site-packages -name \*.py -delete RUN find ./python/lib/$runtime/site-packages -name __pycache__ -type d -exec rm -r {} \+ From 22e806868e59328462fd2a3115381f3ca35a38c5 Mon Sep 17 00:00:00 2001 From: Rey Abolofia Date: Fri, 19 Apr 2024 11:23:43 -0700 Subject: [PATCH 06/10] Use different commandline option to support py3.8 --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 163a44e0..55cc4dd7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,12 +27,12 @@ RUN find . -name "*.dist-info" -type d | xargs rm -rf # Precompile all .pyc files and remove .py files. This speeds up load time. -# Compile with optimization level 2 (-o 2) and PYTHONNODEBUGRANGES=1 to redtce +# Compile with optimization level 2 (-OO) and PYTHONNODEBUGRANGES=1 to redtce # size of .pyc files. # See https://docs.python.org/3/tutorial/modules.html#compiled-python-files # https://docs.python.org/3.11/using/cmdline.html#cmdoption-O # https://docs.python.org/3/using/cmdline.html#envvar-PYTHONNODEBUGRANGES -RUN PYTHONNODEBUGRANGES=1 python -m compileall -o 2 -b ./python/lib/$runtime/site-packages +RUN PYTHONNODEBUGRANGES=1 python -OO -m compileall -b ./python/lib/$runtime/site-packages RUN find ./python/lib/$runtime/site-packages -name \*.py -delete RUN find ./python/lib/$runtime/site-packages -name __pycache__ -type d -exec rm -r {} \+ From 4933d7ee10549d3f66171cc732147304c47e0768 Mon Sep 17 00:00:00 2001 From: Rey Abolofia Date: Fri, 19 Apr 2024 11:37:51 -0700 Subject: [PATCH 07/10] Update size limits. --- scripts/check_layer_size.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scripts/check_layer_size.sh b/scripts/check_layer_size.sh index 84752fa1..977c0283 100755 --- a/scripts/check_layer_size.sh +++ b/scripts/check_layer_size.sh @@ -7,10 +7,9 @@ # Compares layer size to threshold, and fails if below that threshold -# 7 mb size limit set -e -MAX_LAYER_COMPRESSED_SIZE_KB=$(expr 4 \* 1024) -MAX_LAYER_UNCOMPRESSED_SIZE_KB=$(expr 13 \* 1024) +MAX_LAYER_COMPRESSED_SIZE_KB=$(expr 5 \* 1024) +MAX_LAYER_UNCOMPRESSED_SIZE_KB=$(expr 12 \* 1024) LAYER_FILES_PREFIX="datadog_lambda_py" From f0afead628c6026bd593e7736b565dd6e5186b4f Mon Sep 17 00:00:00 2001 From: Rey Abolofia Date: Mon, 22 Apr 2024 09:36:35 -0700 Subject: [PATCH 08/10] Leave __init__.py contrib files. --- Dockerfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 55cc4dd7..e63e1a29 100644 --- a/Dockerfile +++ b/Dockerfile @@ -33,7 +33,10 @@ RUN find . -name "*.dist-info" -type d | xargs rm -rf # https://docs.python.org/3.11/using/cmdline.html#cmdoption-O # https://docs.python.org/3/using/cmdline.html#envvar-PYTHONNODEBUGRANGES RUN PYTHONNODEBUGRANGES=1 python -OO -m compileall -b ./python/lib/$runtime/site-packages -RUN find ./python/lib/$runtime/site-packages -name \*.py -delete +# remove all .py files except ddtrace/contrib/*/__init__.py which are necessary +# for ddtrace.patch to discover instrumationation packages. +RUN find ./python/lib/$runtime/site-packages -name \*.py | grep -v ddtrace/contrib | xargs rm -rf +RUN find ./python/lib/$runtime/site-packages/ddtrace/contrib -name \*.py | grep -v __init__ | xargs rm -rf RUN find ./python/lib/$runtime/site-packages -name __pycache__ -type d -exec rm -r {} \+ FROM scratch From 582e612eface29a121d654b5f2b52b6b858fab0d Mon Sep 17 00:00:00 2001 From: Rey Abolofia Date: Mon, 22 Apr 2024 10:20:01 -0700 Subject: [PATCH 09/10] Remove text like files. --- Dockerfile | 1 - ave_time.sh | 12 ++++++++++++ number1/__init__.py | 0 number1/patch.py | 0 number2/__init__.py | 0 number2/patch.py | 0 number3/__init__.py | 0 number3/patch.py | 0 number4/__init__.py | 0 number4/patch.py | 0 number5/__init__.py | 0 number5/patch.py | 0 testit.sh | 17 +++++++++++++++++ 13 files changed, 29 insertions(+), 1 deletion(-) create mode 100755 ave_time.sh create mode 100644 number1/__init__.py create mode 100644 number1/patch.py create mode 100644 number2/__init__.py create mode 100644 number2/patch.py create mode 100644 number3/__init__.py create mode 100644 number3/patch.py create mode 100644 number4/__init__.py create mode 100644 number4/patch.py create mode 100644 number5/__init__.py create mode 100644 number5/patch.py create mode 100755 testit.sh diff --git a/Dockerfile b/Dockerfile index e63e1a29..ed43e237 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,7 +25,6 @@ RUN rm ./python/lib/$runtime/site-packages/ddtrace/internal/datadog/profiling/dd RUN rm ./python/lib/$runtime/site-packages/ddtrace/internal/datadog/profiling/stack_v2/_stack_v2.*.so RUN find . -name "*.dist-info" -type d | xargs rm -rf - # Precompile all .pyc files and remove .py files. This speeds up load time. # Compile with optimization level 2 (-OO) and PYTHONNODEBUGRANGES=1 to redtce # size of .pyc files. diff --git a/ave_time.sh b/ave_time.sh new file mode 100755 index 00000000..ef061272 --- /dev/null +++ b/ave_time.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +set -e + +time ( + export DD_LAMBDA_HANDLER=builtins.print + + for _ in {1..100} + do + python -c "import datadog_lambda.handler" + done +) 2>&1 diff --git a/number1/__init__.py b/number1/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/number1/patch.py b/number1/patch.py new file mode 100644 index 00000000..e69de29b diff --git a/number2/__init__.py b/number2/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/number2/patch.py b/number2/patch.py new file mode 100644 index 00000000..e69de29b diff --git a/number3/__init__.py b/number3/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/number3/patch.py b/number3/patch.py new file mode 100644 index 00000000..e69de29b diff --git a/number4/__init__.py b/number4/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/number4/patch.py b/number4/patch.py new file mode 100644 index 00000000..e69de29b diff --git a/number5/__init__.py b/number5/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/number5/patch.py b/number5/patch.py new file mode 100644 index 00000000..e69de29b diff --git a/testit.sh b/testit.sh new file mode 100755 index 00000000..3d5a7c88 --- /dev/null +++ b/testit.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +set -e + +PYTHON_VERSION=3.9 ARCH=amd64 ./scripts/build_layers.sh + +cd .layers/ +unzip datadog_lambda_py-amd64-3.9.zip +cd - + +docker run -it \ + --platform=linux/amd64 \ + -v "$PWD"/.layers/python/lib/python3.9/site-packages:/opt/python/lib/python3.9/site-packages/ \ + --entrypoint='' \ + -e PYTHONPATH=/opt/python/lib/python3.9/site-packages \ + public.ecr.aws/lambda/python:3.9 \ + python /opt/python/lib/python3.9/site-packages/datadog_lambda/tracing.py From 43947e9915aa3fbc3e22713d9f15ae8cbbaee1d3 Mon Sep 17 00:00:00 2001 From: Rey Abolofia Date: Mon, 22 Apr 2024 10:57:54 -0700 Subject: [PATCH 10/10] oops --- ave_time.sh | 12 ------------ number1/__init__.py | 0 number1/patch.py | 0 number2/__init__.py | 0 number2/patch.py | 0 number3/__init__.py | 0 number3/patch.py | 0 number4/__init__.py | 0 number4/patch.py | 0 number5/__init__.py | 0 number5/patch.py | 0 testit.sh | 17 ----------------- 12 files changed, 29 deletions(-) delete mode 100755 ave_time.sh delete mode 100644 number1/__init__.py delete mode 100644 number1/patch.py delete mode 100644 number2/__init__.py delete mode 100644 number2/patch.py delete mode 100644 number3/__init__.py delete mode 100644 number3/patch.py delete mode 100644 number4/__init__.py delete mode 100644 number4/patch.py delete mode 100644 number5/__init__.py delete mode 100644 number5/patch.py delete mode 100755 testit.sh diff --git a/ave_time.sh b/ave_time.sh deleted file mode 100755 index ef061272..00000000 --- a/ave_time.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -set -e - -time ( - export DD_LAMBDA_HANDLER=builtins.print - - for _ in {1..100} - do - python -c "import datadog_lambda.handler" - done -) 2>&1 diff --git a/number1/__init__.py b/number1/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/number1/patch.py b/number1/patch.py deleted file mode 100644 index e69de29b..00000000 diff --git a/number2/__init__.py b/number2/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/number2/patch.py b/number2/patch.py deleted file mode 100644 index e69de29b..00000000 diff --git a/number3/__init__.py b/number3/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/number3/patch.py b/number3/patch.py deleted file mode 100644 index e69de29b..00000000 diff --git a/number4/__init__.py b/number4/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/number4/patch.py b/number4/patch.py deleted file mode 100644 index e69de29b..00000000 diff --git a/number5/__init__.py b/number5/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/number5/patch.py b/number5/patch.py deleted file mode 100644 index e69de29b..00000000 diff --git a/testit.sh b/testit.sh deleted file mode 100755 index 3d5a7c88..00000000 --- a/testit.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -set -e - -PYTHON_VERSION=3.9 ARCH=amd64 ./scripts/build_layers.sh - -cd .layers/ -unzip datadog_lambda_py-amd64-3.9.zip -cd - - -docker run -it \ - --platform=linux/amd64 \ - -v "$PWD"/.layers/python/lib/python3.9/site-packages:/opt/python/lib/python3.9/site-packages/ \ - --entrypoint='' \ - -e PYTHONPATH=/opt/python/lib/python3.9/site-packages \ - public.ecr.aws/lambda/python:3.9 \ - python /opt/python/lib/python3.9/site-packages/datadog_lambda/tracing.py