diff --git a/csmith-fuzzing/README.md b/csmith-fuzzing/README.md index 258ab5e71d..c7e8eaf9d7 100644 --- a/csmith-fuzzing/README.md +++ b/csmith-fuzzing/README.md @@ -18,13 +18,13 @@ tests in the bindings fail, then we report an issue containing the test case! ## Prerequisites -Requires `python3`, `csmith` and `bindgen` to be in `$PATH`. +Requires `python3`, `csmith`, and `creduce` to be in `$PATH`. -Many OS package managers have a `csmith` package: +Many OS package managers have `csmith` and `creduce` packages: ``` -$ sudo apt install csmith -$ brew install csmith +$ sudo apt install csmith creduce +$ brew install csmith creduce $ # Etc... ``` diff --git a/csmith-fuzzing/driver.py b/csmith-fuzzing/driver.py index 7dc6086c92..8bf76e0b4d 100755 --- a/csmith-fuzzing/driver.py +++ b/csmith-fuzzing/driver.py @@ -1,32 +1,44 @@ #!/usr/bin/env python3 -import os, sys +import argparse +import os +import re +import shlex +import sys from subprocess import run, SubprocessError, DEVNULL, PIPE from tempfile import NamedTemporaryFile -csmith_command = [ - "csmith", - "--no-checksum", - "--nomain", - "--max-block-size", "1", - "--max-block-depth", "1", -] +DESC = """ -def cat(path, title=None): - if not title: - title = path - print("-------------------- {} --------------------".format(title)) - run(["cat", path]) +A `csmith` fuzzing driver for `bindgen`. -def run_logged(cmd): - with NamedTemporaryFile() as stdout, NamedTemporaryFile() as stderr: - result = run(cmd, stdin=DEVNULL, stdout=stdout, stderr=stderr) - if result.returncode != 0: - print() - print("Error: '{}' exited with code {}".format(" ".join(cmd), result.returncode)) - cat(stdout.name, title="stdout") - cat(stdout.name, title="stderr") - return result +Generates random C source files with `csmith` and then passes them to `bindgen` +(via `predicate.py`). If `bindgen` can't emit bindings, `rustc` can't compile +those bindings, or the compiled bindings' layout tests fail, then the driver has +found a bug, and will report the problematic test case to you. + +""" + +parser = argparse.ArgumentParser( + formatter_class=argparse.RawDescriptionHelpFormatter, + description=DESC.strip()) + +parser.add_argument( + "--keep-going", + action="store_true", + help="Do not stop after finding a test case that exhibits a bug in `bindgen`. Instead, keep going.") + +CSMITH_ARGS="\ +--no-checksum \ +--nomain \ +--max-block-size 1 \ +--max-block-depth 1" + +parser.add_argument( + "--csmith-args", + type=str, + default=CSMITH_ARGS, + help="Pass this argument string to `csmith`. By default, very small functions are generated.") BINDGEN_ARGS = "--with-derive-partialeq \ --with-derive-eq \ @@ -35,31 +47,234 @@ def run_logged(cmd): --with-derive-hash \ --with-derive-default" +parser.add_argument( + "--bindgen-args", + type=str, + default=BINDGEN_ARGS, + help="Pass this argument string to `bindgen`. By default, all traits are derived.") + +parser.add_argument( + "--no-creduce", + action="store_false", + dest="creduce", + help="Do not run `creduce` on any buggy test case(s) discovered.") + +################################################################################ + +def cat(path, title=None): + if not title: + title = path + print("-------------------- {} --------------------".format(title)) + print() + print() + run(["cat", path]) + +def decode(f): + return f.decode(encoding="utf-8", errors="ignore") + +def run_logged(cmd): + result = run(cmd, stdin=DEVNULL, stdout=PIPE, stderr=PIPE) + result.stdout = decode(result.stdout) + result.stderr = decode(result.stderr) + if result.returncode != 0: + print() + print() + print("Error: {} exited with code {}".format(cmd, result.returncode)) + print() + print() + for line in result.stdout.splitlines(): + sys.stdout.write("+") + sys.stdout.write(line) + sys.stdout.write("\n") + for line in result.stderr.splitlines(): + sys.stderr.write("+") + sys.stderr.write(line) + sys.stderr.write("\n") + return result + def main(): - print("Fuzzing `bindgen` with C-Smith...\n") + os.environ["RUST_BACKTRACE"] = "full" + args = parser.parse_args() + + bindgen_args = args.bindgen_args + if bindgen_args.find(" -- ") == -1: + bindgen_args = bindgen_args + " -- " + bindgen_args = bindgen_args + " -I{}".format(os.path.abspath(os.path.dirname(sys.argv[0]))) + args.bindgen_args = bindgen_args + + print() + print() + print("Fuzzing `bindgen` with C-Smith...") + print() + print() iterations = 0 while True: print("\rIteration: {}".format(iterations), end="", flush=True) + iterations += 1 input = NamedTemporaryFile(delete=False, prefix="input-", suffix=".h") input.close() - result = run_logged(csmith_command + ["-o", input.name]) + result = run_logged(["csmith", "-o", input.name] + shlex.split(args.csmith_args)) if result.returncode != 0: exit(1) - result = run_logged([ + predicate_command = [ "./predicate.py", "--bindgen-args", - "{} -- -I{}".format(BINDGEN_ARGS, os.path.abspath(os.path.dirname(sys.argv[0]))), + args.bindgen_args, input.name - ]) + ] + result = run_logged(predicate_command) + if result.returncode != 0: - cat(input.name) + print() + print() + cat(input.name, title="Failing test case: {}".format(input.name)) + print() + print() + + if args.creduce: + creduce(args, input.name, result) + + print_issue_template(args, input.name, predicate_command, result) + + if args.keep_going: + continue exit(1) os.remove(input.name) - iterations += 1 + +RUSTC_ERROR_REGEX = re.compile(r".*(error\[.*].*)") +LAYOUT_TEST_FAILURE = re.compile(r".*(test bindgen_test_layout_.* \.\.\. FAILED)") + +def creduce(args, failing_test_case, result): + print() + print() + print("Reducing failing test case with `creduce`...") + + match = re.search(RUSTC_ERROR_REGEX, result.stderr) + if match: + error_msg = match.group(1) + print("...searching for \"{}\".".format(error_msg)) + return creduce_with_predicate_flags( + args, + failing_test_case, + "--bindgen-args '{}' --expect-compile-fail --rustc-grep '{}'".format( + args.bindgen_args, + re.escape(error_msg) + ) + ) + + match = re.search(LAYOUT_TEST_FAILURE, result.stdout) + if match: + layout_failure = match.group(1) + struct_name = layout_failure[len("test bindgen_test_layout_"):layout_failure.rindex(" ... FAILED")] + print("...searching for \"{}\".".format(layout_failure)) + return creduce_with_predicate_flags( + args, + failing_test_case, + "--bindgen-args '{}' --expect-layout-tests-fail --bindings-grep '{}' --layout-tests-grep '{}'".format( + args.bindgen_args, + re.escape(struct_name), + re.escape(layout_failure) + ) + ) + + print("...nevermind, don't know how to `creduce` this bug. Skipping.") + +def creduce_with_predicate_flags(args, failing_test_case, predicate_flags): + predicate = """ +#!/usr/bin/env bash +set -eu +{} {} {} + """.format( + os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), "predicate.py")), + predicate_flags, + os.path.basename(failing_test_case) + ) + + print("...and reducing with this script:") + print() + print() + print(predicate) + print() + print() + + predicate_path = failing_test_case + ".predicate.sh" + with open(predicate_path, "w") as p: + p.write(predicate) + os.chmod(predicate_path, 0o755) + + creduce_command = ["creduce", "--n", str(os.cpu_count()), predicate_path, failing_test_case] + print("Running:", creduce_command) + result = run(creduce_command) + if result.returncode == 0: + print() + print() + print("`creduce` reduced the failing test case to:") + print() + print() + cat(failing_test_case) + print() + print() + else: + print() + print() + print("`creduce` failed!") + if not args.keep_going: + sys.exit(1) + +def print_issue_template(args, failing_test_case, predicate_command, result): + test_case_contents = None + with open(failing_test_case, "r") as f: + test_case_contents = f.read() + + print(""" + +! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! +! File this issue at https://github.com/rust-lang-nursery/rust-bindgen/issues/new ! +! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! + + --------------- 8< --------------- 8< --------------- 8< --------------- + +This bug was found with `csmith` and `driver.py`. + +### Input Header + +```c +{} +``` + +### `bindgen` Invocation + +``` +$ {} +``` + +### Actual Results + +
+ +``` +{} +``` + +
+ +### Expected Results + +`bindgen` emits bindings OK, then `rustc` compiles those bindings OK, then the +compiled bindings' layout tests pass OK. + + --------------- 8< --------------- 8< --------------- 8< --------------- + + <3 <3 <3 Thank you! <3 <3 <3 + """.format( + test_case_contents, + " ".join(map(lambda s: "'{}'".format(s), predicate_command)), + result.stdout + result.stderr + )) if __name__ == "__main__": try: diff --git a/csmith-fuzzing/predicate.py b/csmith-fuzzing/predicate.py index 8896cd96a1..91e4e26d04 100755 --- a/csmith-fuzzing/predicate.py +++ b/csmith-fuzzing/predicate.py @@ -110,10 +110,16 @@ def exit_1(msg, child=None): print(msg) if child: - print("---------- stdout ----------------------------------------------") - print(decode(child.stdout)) - print("---------- stderr ----------------------------------------------") - print(decode(child.stderr)) + stdout = decode(child.stdout) + for line in stdout.splitlines(): + sys.stdout.write("+") + sys.stdout.write(line) + sys.stdout.write("\n") + stderr = decode(child.stderr) + for line in stderr.splitlines(): + sys.stderr.write("+") + sys.stderr.write(line) + sys.stderr.write("\n") raise ExitOne()