Merge branch 'master' into fuzzers-upstream

Author: chrisjsewell

.github/workflows/benchmark.yml

@@ -9,7 +9,7 @@ jobs:
 
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
 
     - name: Set up Python 3.8
       uses: actions/setup-python@v4

.github/workflows/fuzz.yml

@@ -1,8 +1,16 @@
 name: fuzzing
+
+# This action runs fuzzing for a brief period of time,
+# only aginst the actual code added in the PR.
+# It is intended a relatively quick check,
+# to guard against code introducing crashes in the Markdown parsing,
+# which should in principle always run against any text input.
+# See: https://google.github.io/oss-fuzz/getting-started/continuous-integration/#how-it-works
+# Note, to reproduce a crash locally, copy to `testcase` file` and run: `tox -e fuzz`
+
 on:
-  push:
-    branches: [master]
   pull_request:
+    paths-ignore: ['docs/**', 'tests/**']
 
 jobs:
   Fuzzing:
@@ -19,7 +27,7 @@ jobs:
       with:
         oss-fuzz-project-name: 'markdown-it-py'
         language: python
-        fuzz-seconds: 600
+        fuzz-seconds: 60
     - name: Upload Crash
       uses: actions/upload-artifact@v3
       if: failure() && steps.build.outcome == 'success'

.github/workflows/tests.yml

@@ -19,7 +19,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Set up Python 3.8
       uses: actions/setup-python@v4
       with:
@@ -34,7 +34,7 @@ jobs:
         python-version: ['pypy-3.7', '3.7', '3.8', '3.9', '3.10', '3.11']
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v4
       with:
@@ -54,6 +54,7 @@ jobs:
         flags: pytests
         file: ./coverage.xml
         fail_ci_if_error: true
+        token: ${{ secrets.CODECOV_TOKEN }}
 
   test-plugins:
 
@@ -62,7 +63,7 @@ jobs:
       matrix:
         python-version: ['3.8']
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v4
       with:
@@ -82,7 +83,7 @@ jobs:
 
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
 
     - name: Set up Python 3.8
       uses: actions/setup-python@v4
@@ -112,7 +113,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Checkout source
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
     - name: Set up Python 3.8
       uses: actions/setup-python@v4
       with:

.pre-commit-config.yaml

@@ -29,7 +29,7 @@ repos:
     - id: isort
 
   - repo: https://github.com/psf/black
-    rev: 23.1.0
+    rev: 23.3.0
     hooks:
     - id: black
 
@@ -40,7 +40,7 @@ repos:
       additional_dependencies: [flake8-bugbear~=22.7]
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.0.1
+    rev: v1.2.0
     hooks:
     - id: mypy
       additional_dependencies: [mdurl]

docs/conf.py

@@ -102,7 +102,7 @@ def run_apidoc(app):
     this_folder = os.path.abspath(os.path.dirname(os.path.realpath(__file__)))
     api_folder = os.path.join(this_folder, "api")
     module_path = os.path.normpath(os.path.join(this_folder, "../"))
-    ignore_paths = ["../profiler.py", "../conftest.py", "../tests", "../benchmarking"]
+    ignore_paths = ["../scripts", "../conftest.py", "../tests", "../benchmarking"]
     ignore_paths = [
         os.path.normpath(os.path.join(this_folder, p)) for p in ignore_paths
     ]

scripts/build_fuzzers.py

@@ -0,0 +1,42 @@
+"""Build fuzzers idempotently in a given folder."""
+import argparse
+from pathlib import Path
+import subprocess
+
+
+def main():
+    """Build fuzzers idempotently in a given folder."""
+    parser = argparse.ArgumentParser()
+    parser.add_argument("folder")
+    args = parser.parse_args()
+    folder = Path(args.folder)
+    if not folder.exists():
+        print(f"Cloning google/oss-fuzz into: {folder}")
+        folder.mkdir(parents=True)
+        subprocess.check_call(
+            [
+                "git",
+                "clone",
+                "--single-branch",
+                "https://github.com/google/oss-fuzz",
+                str(folder),
+            ]
+        )
+    else:
+        print(f"Using google/oss-fuzz in: {folder}")
+    if not (folder / "build").exists():
+        print(f"Building fuzzers in: {folder / 'build'}")
+        subprocess.check_call(
+            [
+                "python",
+                str(folder / "infra" / "helper.py"),
+                "build_fuzzers",
+                "markdown-it-py",
+            ]
+        )
+    else:
+        print(f"Using existing fuzzers in: {folder / 'build'}")
+
+
+if __name__ == "__main__":
+    main()

profiler.py renamed to scripts/profiler.py

@@ -55,20 +55,17 @@ allowlist_externals =
     dot
 commands =
     mkdir -p "{toxworkdir}/prof"
-    python -m cProfile -o "{toxworkdir}/prof/output.pstats" profiler.py
+    python -m cProfile -o "{toxworkdir}/prof/output.pstats" scripts/profiler.py
     gprof2dot -f pstats -o "{toxworkdir}/prof/output.dot" "{toxworkdir}/prof/output.pstats"
     dot -Tsvg -o "{toxworkdir}/prof/output.svg" "{toxworkdir}/prof/output.dot"
     python -c 'import pathlib; print("profiler svg output under file://\{0\}".format(pathlib.Path(r"{toxworkdir}") / "prof" / "output.svg"))'
 
 [testenv:fuzz]
-description = run fuzzer
+description = run fuzzer on testcase file
+; See: https://google.github.io/oss-fuzz/
 deps = atheris
-allowlist_externals =
-    git
-commands_pre =
-    git clone --single-branch https://github.com/google/oss-fuzz {envtmpdir}/oss-fuzz
-    python {envtmpdir}/oss-fuzz/infra/helper.py build_fuzzers markdown-it-py
-commands = python {envtmpdir}/oss-fuzz/infra/helper.py run_fuzzer markdown-it-py fuzz_markdown
+commands_pre = python scripts/build_fuzzers.py {envdir}/oss-fuzz
+commands = python {envdir}/oss-fuzz/infra/helper.py reproduce markdown-it-py fuzz_markdown {posargs:testcase}
 
 [flake8]
 max-line-length = 100