Merge branch 'master' into code-block

Author: chrisjsewell

.github/workflows/benchmark.yml

@@ -9,7 +9,7 @@ jobs:
 
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
 
     - name: Set up Python 3.8
       uses: actions/setup-python@v4

.github/workflows/fuzz.yml

@@ -6,9 +6,11 @@ name: fuzzing
 # to guard against code introducing crashes in the Markdown parsing,
 # which should in principle always run against any text input.
 # See: https://google.github.io/oss-fuzz/getting-started/continuous-integration/#how-it-works
+# Note, to reproduce a crash locally, copy to `testcase` file` and run: `tox -e fuzz`
 
 on:
   pull_request:
+    paths-ignore: ['docs/**', 'tests/**']
 
 jobs:
   Fuzzing:

.github/workflows/tests.yml

@@ -19,7 +19,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Set up Python 3.8
       uses: actions/setup-python@v4
       with:
@@ -34,7 +34,7 @@ jobs:
         python-version: ['pypy-3.7', '3.7', '3.8', '3.9', '3.10', '3.11']
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v4
       with:
@@ -63,7 +63,7 @@ jobs:
       matrix:
         python-version: ['3.8']
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v4
       with:
@@ -83,7 +83,7 @@ jobs:
 
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
 
     - name: Set up Python 3.8
       uses: actions/setup-python@v4
@@ -113,7 +113,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Checkout source
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
     - name: Set up Python 3.8
       uses: actions/setup-python@v4
       with:

.pre-commit-config.yaml

@@ -29,7 +29,7 @@ repos:
     - id: isort
 
   - repo: https://github.com/psf/black
-    rev: 23.1.0
+    rev: 23.3.0
     hooks:
     - id: black
 
@@ -40,7 +40,7 @@ repos:
       additional_dependencies: [flake8-bugbear~=22.7]
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.0.1
+    rev: v1.2.0
     hooks:
     - id: mypy
       additional_dependencies: [mdurl]

docs/conf.py

@@ -102,7 +102,7 @@ def run_apidoc(app):
     this_folder = os.path.abspath(os.path.dirname(os.path.realpath(__file__)))
     api_folder = os.path.join(this_folder, "api")
     module_path = os.path.normpath(os.path.join(this_folder, "../"))
-    ignore_paths = ["../profiler.py", "../conftest.py", "../tests", "../benchmarking"]
+    ignore_paths = ["../scripts", "../conftest.py", "../tests", "../benchmarking"]
     ignore_paths = [
         os.path.normpath(os.path.join(this_folder, p)) for p in ignore_paths
     ]

scripts/build_fuzzers.py

@@ -0,0 +1,42 @@
+"""Build fuzzers idempotently in a given folder."""
+import argparse
+from pathlib import Path
+import subprocess
+
+
+def main():
+    """Build fuzzers idempotently in a given folder."""
+    parser = argparse.ArgumentParser()
+    parser.add_argument("folder")
+    args = parser.parse_args()
+    folder = Path(args.folder)
+    if not folder.exists():
+        print(f"Cloning google/oss-fuzz into: {folder}")
+        folder.mkdir(parents=True)
+        subprocess.check_call(
+            [
+                "git",
+                "clone",
+                "--single-branch",
+                "https://github.com/google/oss-fuzz",
+                str(folder),
+            ]
+        )
+    else:
+        print(f"Using google/oss-fuzz in: {folder}")
+    if not (folder / "build").exists():
+        print(f"Building fuzzers in: {folder / 'build'}")
+        subprocess.check_call(
+            [
+                "python",
+                str(folder / "infra" / "helper.py"),
+                "build_fuzzers",
+                "markdown-it-py",
+            ]
+        )
+    else:
+        print(f"Using existing fuzzers in: {folder / 'build'}")
+
+
+if __name__ == "__main__":
+    main()

profiler.py renamed to scripts/profiler.py

@@ -55,11 +55,18 @@ allowlist_externals =
     dot
 commands =
     mkdir -p "{toxworkdir}/prof"
-    python -m cProfile -o "{toxworkdir}/prof/output.pstats" profiler.py
+    python -m cProfile -o "{toxworkdir}/prof/output.pstats" scripts/profiler.py
     gprof2dot -f pstats -o "{toxworkdir}/prof/output.dot" "{toxworkdir}/prof/output.pstats"
     dot -Tsvg -o "{toxworkdir}/prof/output.svg" "{toxworkdir}/prof/output.dot"
     python -c 'import pathlib; print("profiler svg output under file://\{0\}".format(pathlib.Path(r"{toxworkdir}") / "prof" / "output.svg"))'
 
+[testenv:fuzz]
+description = run fuzzer on testcase file
+; See: https://google.github.io/oss-fuzz/
+deps = atheris
+commands_pre = python scripts/build_fuzzers.py {envdir}/oss-fuzz
+commands = python {envdir}/oss-fuzz/infra/helper.py reproduce markdown-it-py fuzz_markdown {posargs:testcase}
+
 [flake8]
 max-line-length = 100
 extend-ignore = E203