From 3bbce2b07aef69fa410c9af86816b4b3ae98bbb6 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Wed, 5 Jul 2023 13:49:10 +0200 Subject: [PATCH 1/8] chore: enable manual trigger ossf scorecard --- .github/workflows/ossf_scorecard.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ossf_scorecard.yml b/.github/workflows/ossf_scorecard.yml index 10e53f164ec..da72dc6a3ee 100644 --- a/.github/workflows/ossf_scorecard.yml +++ b/.github/workflows/ossf_scorecard.yml @@ -7,6 +7,7 @@ on: - cron: "0 9 * * *" push: branches: [$default-branch] + workflow_dispatch: permissions: read-all From ca41c69d080218bbc7c1ff5dd072e5ea7a93bb44 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Wed, 5 Jul 2023 14:00:24 +0200 Subject: [PATCH 2/8] chore(ci): enable fuzzing for each PR --- .github/workflows/on_pr_fuzzing.yml | 50 +++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 .github/workflows/on_pr_fuzzing.yml diff --git a/.github/workflows/on_pr_fuzzing.yml b/.github/workflows/on_pr_fuzzing.yml new file mode 100644 index 00000000000..511259fe631 --- /dev/null +++ b/.github/workflows/on_pr_fuzzing.yml @@ -0,0 +1,50 @@ +# Run fuzzing against changes in Pull Requests for bug and leaks detection +name: Fuzzing (ClusterFuzzLite) + +# PROCESS +# +# 1. Build a fuzzer for python for each sanitizer +# 2. Fuzz each sanitizer against each change in PR +# 3. Upload report as an artifact + +# NOTES +# +# More info: https://google.github.io/clusterfuzzlite/ + +on: + pull_request: + paths: + - '**' + +permissions: read-all + + +jobs: + fuzzing: + runs-on: ubuntu-latest + concurrency: + group: ${{ github.workflow }}-${{ matrix.sanitizer }}-${{ github.ref }} + cancel-in-progress: true + strategy: + fail-fast: false + matrix: + sanitizer: + - address # memory safety issues + - undefined # undefined behaviour (e.g., integer workflows, dangling pointers) + - memory # uninitialized memory (e.g., C-extensions) + steps: + - name: Build Fuzzers (${{ matrix.sanitizer }}) + id: build + uses: google/clusterfuzzlite/actions/build_fuzzers@1e163f06cba7820da5154ac9fe1a32d7fe6f73a3 # v1 + with: + language: python + github-token: ${{ secrets.GITHUB_TOKEN }} + sanitizer: ${{ matrix.sanitizer }} + - name: Run Fuzzers (${{ matrix.sanitizer }}) + id: run + uses: google/clusterfuzzlite/actions/run_fuzzers@v1 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + fuzz-seconds: 90 # shorter interval to not block PR; we run longer interval async for increased safety + mode: 'code-change' + sanitizer: ${{ matrix.sanitizer }} From 7bdbc8d2e81149f49192943650c13c5ef7b70004 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Wed, 5 Jul 2023 14:07:01 +0200 Subject: [PATCH 3/8] chore(ci): enable continuous fuzzing every 6 hours --- .github/workflows/on_pr_fuzzing.yml | 2 +- .github/workflows/on_schedule_fuzzing.yml | 44 +++++++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/on_schedule_fuzzing.yml diff --git a/.github/workflows/on_pr_fuzzing.yml b/.github/workflows/on_pr_fuzzing.yml index 511259fe631..a30257d8d40 100644 --- a/.github/workflows/on_pr_fuzzing.yml +++ b/.github/workflows/on_pr_fuzzing.yml @@ -42,7 +42,7 @@ jobs: sanitizer: ${{ matrix.sanitizer }} - name: Run Fuzzers (${{ matrix.sanitizer }}) id: run - uses: google/clusterfuzzlite/actions/run_fuzzers@v1 + uses: google/clusterfuzzlite/actions/run_fuzzers@1e163f06cba7820da5154ac9fe1a32d7fe6f73a3 # v1 with: github-token: ${{ secrets.GITHUB_TOKEN }} fuzz-seconds: 90 # shorter interval to not block PR; we run longer interval async for increased safety diff --git a/.github/workflows/on_schedule_fuzzing.yml b/.github/workflows/on_schedule_fuzzing.yml new file mode 100644 index 00000000000..e0daee7b301 --- /dev/null +++ b/.github/workflows/on_schedule_fuzzing.yml @@ -0,0 +1,44 @@ +# Run fuzzing against changes in Pull Requests for bug and leaks detection +name: Continuous Fuzzing (ClusterFuzzLite) + +# PROCESS +# +# 1. Build a fuzzer for python for each sanitizer +# 2. Fuzz each sanitizer against entire repository +# 3. Aggregate results and upload report as an artifact + +# NOTES +# +# More info: https://google.github.io/clusterfuzzlite/ + +on: + schedule: + - cron: '0 0/6 * * *' # Every 6th hour. + +permissions: read-all + +jobs: + fuzzing: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + sanitizer: + - address # memory safety issues + - undefined # undefined behaviour (e.g., integer workflows, dangling pointers) + - memory # uninitialized memory (e.g., C-extensions) + steps: + - name: Build Fuzzers (${{ matrix.sanitizer }}) + id: build + uses: google/clusterfuzzlite/actions/build_fuzzers@1e163f06cba7820da5154ac9fe1a32d7fe6f73a3 # v1 + with: + language: python + sanitizer: ${{ matrix.sanitizer }} + - name: Run Fuzzers (${{ matrix.sanitizer }}) + id: run + uses: google/clusterfuzzlite/actions/run_fuzzers@1e163f06cba7820da5154ac9fe1a32d7fe6f73a3 # v1 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + fuzz-seconds: 3600 + mode: 'batch' + sanitizer: ${{ matrix.sanitizer }} From 365f5978d7fa83b10c3aa78eead0d0187befccd5 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Wed, 5 Jul 2023 14:13:42 +0200 Subject: [PATCH 4/8] chore: fuzzing config --- .clusterfuzzlite/project.yaml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .clusterfuzzlite/project.yaml diff --git a/.clusterfuzzlite/project.yaml b/.clusterfuzzlite/project.yaml new file mode 100644 index 00000000000..aaa3e7a26a2 --- /dev/null +++ b/.clusterfuzzlite/project.yaml @@ -0,0 +1,2 @@ +# Config: https://google.github.io/clusterfuzzlite/build-integration/#projectyaml +language: python From c4811e3ec56098a4e831dd596642e645d4b5e76e Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Wed, 5 Jul 2023 14:18:23 +0200 Subject: [PATCH 5/8] chore: improve config --- .clusterfuzzlite/project.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.clusterfuzzlite/project.yaml b/.clusterfuzzlite/project.yaml index aaa3e7a26a2..a1eaa3b6871 100644 --- a/.clusterfuzzlite/project.yaml +++ b/.clusterfuzzlite/project.yaml @@ -1,2 +1,13 @@ # Config: https://google.github.io/clusterfuzzlite/build-integration/#projectyaml +main_repo: "https://github.com/aws-powertools/powertools-lambda-python.git" +homepage: "https://docs.powertools.aws.dev/lambda/python/latest/" language: python +primary_contact: "aws-lambda-powertools-feedback@amazon.com" +auto_ccs: + - "aws-lambda-powertools-feedback@amazon.com" +sanitizers: + - address + - undefined + - memory +architectures: + - x86_64 From 2f2082d8c20039cf92bf650bfb3f71d25e4b670f Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Wed, 5 Jul 2023 14:26:05 +0200 Subject: [PATCH 6/8] chore: try with minimal config --- .clusterfuzzlite/Dockerfile | 5 +++++ .clusterfuzzlite/build.sh | 6 ++++++ 2 files changed, 11 insertions(+) create mode 100644 .clusterfuzzlite/Dockerfile create mode 100644 .clusterfuzzlite/build.sh diff --git a/.clusterfuzzlite/Dockerfile b/.clusterfuzzlite/Dockerfile new file mode 100644 index 00000000000..0863abb397c --- /dev/null +++ b/.clusterfuzzlite/Dockerfile @@ -0,0 +1,5 @@ +# v1 +FROM gcr.io/oss-fuzz-base/base-builder-python@sha256:a7eb0875e2d7e96eb7baab4f6104b077dd8d5a9aabcde40c9251f8ad33de0e36 +COPY . $SRC/ +WORKDIR $SRC/aws_lambda_powertools +COPY .clusterfuzzlite/build.sh $SRC/ diff --git a/.clusterfuzzlite/build.sh b/.clusterfuzzlite/build.sh new file mode 100644 index 00000000000..4cd67386707 --- /dev/null +++ b/.clusterfuzzlite/build.sh @@ -0,0 +1,6 @@ +# Build and install project (using current CFLAGS, CXXFLAGS). +pip3 install --upgrade pip +pip3 install . +for fuzzer in $(find $SRC -name 'fuzz_*.py'); do + compile_python_fuzzer $fuzzer +done From 9840eb581c0e5ba4f388b947b4b084895d471aee Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Wed, 5 Jul 2023 14:30:33 +0200 Subject: [PATCH 7/8] chore: use recommended build now that we found the docs --- .clusterfuzzlite/build.sh | 32 +++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/.clusterfuzzlite/build.sh b/.clusterfuzzlite/build.sh index 4cd67386707..42fba28e53b 100644 --- a/.clusterfuzzlite/build.sh +++ b/.clusterfuzzlite/build.sh @@ -1,6 +1,32 @@ -# Build and install project (using current CFLAGS, CXXFLAGS). +# Build and install project (using current CFLAGS, CXXFLAGS). This is required +# for projects with C extensions so that they're built with the proper flags. +ls -l pip3 install --upgrade pip pip3 install . -for fuzzer in $(find $SRC -name 'fuzz_*.py'); do - compile_python_fuzzer $fuzzer + +# Build fuzzers into $OUT. These could be detected in other ways. +for fuzzer in $(find $SRC -name '*_fuzzer.py'); do + fuzzer_basename=$(basename -s .py $fuzzer) + fuzzer_package=${fuzzer_basename}.pkg + + # To avoid issues with Python version conflicts, or changes in environment + # over time, we use pyinstaller to create a standalone + # package. Though not necessarily required for reproducing issues, this is + # required to keep fuzzers working properly. + pyinstaller --distpath $OUT --onefile --name $fuzzer_package $fuzzer + + # Create execution wrapper. Atheris requires that certain libraries are + # preloaded, so this is also done here to ensure compatibility and simplify + # test case reproduction. Since this helper script is what will + # actually execute, it is also always required. + # NOTE: If you are fuzzing python-only code and do not have native C/C++ + # extensions, then remove the LD_PRELOAD line below as preloading sanitizer + # library is not required and can lead to unexpected startup crashes. + echo "#!/bin/sh +# LLVMFuzzerTestOneInput for fuzzer detection. +this_dir=\$(dirname \"\$0\") +LD_PRELOAD=\$this_dir/sanitizer_with_fuzzer.so \ +ASAN_OPTIONS=\$ASAN_OPTIONS:symbolize=1:external_symbolizer_path=\$this_dir/llvm-symbolizer:detect_leaks=0 \ +\$this_dir/$fuzzer_package \$@" >$OUT/$fuzzer_basename + chmod +x $OUT/$fuzzer_basename done From 88b72070d2227a667467d18003e5371131eaeb97 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Wed, 5 Jul 2023 14:33:23 +0200 Subject: [PATCH 8/8] chore: fix workdir --- .clusterfuzzlite/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.clusterfuzzlite/Dockerfile b/.clusterfuzzlite/Dockerfile index 0863abb397c..dfa932180c0 100644 --- a/.clusterfuzzlite/Dockerfile +++ b/.clusterfuzzlite/Dockerfile @@ -1,5 +1,5 @@ # v1 FROM gcr.io/oss-fuzz-base/base-builder-python@sha256:a7eb0875e2d7e96eb7baab4f6104b077dd8d5a9aabcde40c9251f8ad33de0e36 COPY . $SRC/ -WORKDIR $SRC/aws_lambda_powertools +WORKDIR $SRC COPY .clusterfuzzlite/build.sh $SRC/