Skip to content

Automatically run creduce in csmith-fuzzing/driver.py #1080

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Oct 13, 2017
Merged

Automatically run creduce in csmith-fuzzing/driver.py #1080

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions csmith-fuzzing/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,13 @@ tests in the bindings fail, then we report an issue containing the test case!

## Prerequisites

Requires `python3`, `csmith` and `bindgen` to be in `$PATH`.
Requires `python3`, `csmith`, and `creduce` to be in `$PATH`.

Many OS package managers have a `csmith` package:
Many OS package managers have `csmith` and `creduce` packages:

```
$ sudo apt install csmith
$ brew install csmith
$ sudo apt install csmith creduce
$ brew install csmith creduce
$ # Etc...
```

Expand Down
273 changes: 244 additions & 29 deletions csmith-fuzzing/driver.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,32 +1,44 @@
#!/usr/bin/env python3

import os, sys
import argparse
import os
import re
import shlex
import sys
from subprocess import run, SubprocessError, DEVNULL, PIPE
from tempfile import NamedTemporaryFile

csmith_command = [
"csmith",
"--no-checksum",
"--nomain",
"--max-block-size", "1",
"--max-block-depth", "1",
]
DESC = """

def cat(path, title=None):
if not title:
title = path
print("-------------------- {} --------------------".format(title))
run(["cat", path])
A `csmith` fuzzing driver for `bindgen`.

def run_logged(cmd):
with NamedTemporaryFile() as stdout, NamedTemporaryFile() as stderr:
result = run(cmd, stdin=DEVNULL, stdout=stdout, stderr=stderr)
if result.returncode != 0:
print()
print("Error: '{}' exited with code {}".format(" ".join(cmd), result.returncode))
cat(stdout.name, title="stdout")
cat(stdout.name, title="stderr")
return result
Generates random C source files with `csmith` and then passes them to `bindgen`
(via `predicate.py`). If `bindgen` can't emit bindings, `rustc` can't compile
those bindings, or the compiled bindings' layout tests fail, then the driver has
found a bug, and will report the problematic test case to you.

"""

parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
description=DESC.strip())

parser.add_argument(
"--keep-going",
action="store_true",
help="Do not stop after finding a test case that exhibits a bug in `bindgen`. Instead, keep going.")

CSMITH_ARGS="\
--no-checksum \
--nomain \
--max-block-size 1 \
--max-block-depth 1"

parser.add_argument(
"--csmith-args",
type=str,
default=CSMITH_ARGS,
help="Pass this argument string to `csmith`. By default, very small functions are generated.")

BINDGEN_ARGS = "--with-derive-partialeq \
--with-derive-eq \
Expand All @@ -35,31 +47,234 @@ def run_logged(cmd):
--with-derive-hash \
--with-derive-default"

parser.add_argument(
"--bindgen-args",
type=str,
default=BINDGEN_ARGS,
help="Pass this argument string to `bindgen`. By default, all traits are derived.")

parser.add_argument(
"--no-creduce",
action="store_false",
dest="creduce",
help="Do not run `creduce` on any buggy test case(s) discovered.")

################################################################################

def cat(path, title=None):
if not title:
title = path
print("-------------------- {} --------------------".format(title))
print()
print()
run(["cat", path])

def decode(f):
return f.decode(encoding="utf-8", errors="ignore")

def run_logged(cmd):
result = run(cmd, stdin=DEVNULL, stdout=PIPE, stderr=PIPE)
result.stdout = decode(result.stdout)
result.stderr = decode(result.stderr)
if result.returncode != 0:
print()
print()
print("Error: {} exited with code {}".format(cmd, result.returncode))
print()
print()
for line in result.stdout.splitlines():
sys.stdout.write("+")
sys.stdout.write(line)
sys.stdout.write("\n")
for line in result.stderr.splitlines():
sys.stderr.write("+")
sys.stderr.write(line)
sys.stderr.write("\n")
return result

def main():
print("Fuzzing `bindgen` with C-Smith...\n")
os.environ["RUST_BACKTRACE"] = "full"
args = parser.parse_args()

bindgen_args = args.bindgen_args
if bindgen_args.find(" -- ") == -1:
bindgen_args = bindgen_args + " -- "
bindgen_args = bindgen_args + " -I{}".format(os.path.abspath(os.path.dirname(sys.argv[0])))
args.bindgen_args = bindgen_args

print()
print()
print("Fuzzing `bindgen` with C-Smith...")
print()
print()

iterations = 0
while True:
print("\rIteration: {}".format(iterations), end="", flush=True)
iterations += 1

input = NamedTemporaryFile(delete=False, prefix="input-", suffix=".h")
input.close()
result = run_logged(csmith_command + ["-o", input.name])
result = run_logged(["csmith", "-o", input.name] + shlex.split(args.csmith_args))
if result.returncode != 0:
exit(1)

result = run_logged([
predicate_command = [
"./predicate.py",
"--bindgen-args",
"{} -- -I{}".format(BINDGEN_ARGS, os.path.abspath(os.path.dirname(sys.argv[0]))),
args.bindgen_args,
input.name
])
]
result = run_logged(predicate_command)

if result.returncode != 0:
cat(input.name)
print()
print()
cat(input.name, title="Failing test case: {}".format(input.name))
print()
print()

if args.creduce:
creduce(args, input.name, result)

print_issue_template(args, input.name, predicate_command, result)

if args.keep_going:
continue
exit(1)

os.remove(input.name)
iterations += 1

RUSTC_ERROR_REGEX = re.compile(r".*(error\[.*].*)")
LAYOUT_TEST_FAILURE = re.compile(r".*(test bindgen_test_layout_.* \.\.\. FAILED)")

def creduce(args, failing_test_case, result):
print()
print()
print("Reducing failing test case with `creduce`...")

match = re.search(RUSTC_ERROR_REGEX, result.stderr)
if match:
error_msg = match.group(1)
print("...searching for \"{}\".".format(error_msg))
return creduce_with_predicate_flags(
args,
failing_test_case,
"--bindgen-args '{}' --expect-compile-fail --rustc-grep '{}'".format(
args.bindgen_args,
re.escape(error_msg)
)
)

match = re.search(LAYOUT_TEST_FAILURE, result.stdout)
if match:
layout_failure = match.group(1)
struct_name = layout_failure[len("test bindgen_test_layout_"):layout_failure.rindex(" ... FAILED")]
print("...searching for \"{}\".".format(layout_failure))
return creduce_with_predicate_flags(
args,
failing_test_case,
"--bindgen-args '{}' --expect-layout-tests-fail --bindings-grep '{}' --layout-tests-grep '{}'".format(
args.bindgen_args,
re.escape(struct_name),
re.escape(layout_failure)
)
)

print("...nevermind, don't know how to `creduce` this bug. Skipping.")

def creduce_with_predicate_flags(args, failing_test_case, predicate_flags):
predicate = """
#!/usr/bin/env bash
set -eu
{} {} {}
""".format(
os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), "predicate.py")),
predicate_flags,
os.path.basename(failing_test_case)
)

print("...and reducing with this script:")
print()
print()
print(predicate)
print()
print()

predicate_path = failing_test_case + ".predicate.sh"
with open(predicate_path, "w") as p:
p.write(predicate)
os.chmod(predicate_path, 0o755)

creduce_command = ["creduce", "--n", str(os.cpu_count()), predicate_path, failing_test_case]
print("Running:", creduce_command)
result = run(creduce_command)
if result.returncode == 0:
print()
print()
print("`creduce` reduced the failing test case to:")
print()
print()
cat(failing_test_case)
print()
print()
else:
print()
print()
print("`creduce` failed!")
if not args.keep_going:
sys.exit(1)

def print_issue_template(args, failing_test_case, predicate_command, result):
test_case_contents = None
with open(failing_test_case, "r") as f:
test_case_contents = f.read()

print("""

! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! !
! File this issue at https://github.com/rust-lang-nursery/rust-bindgen/issues/new !
! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! !

--------------- 8< --------------- 8< --------------- 8< ---------------

This bug was found with `csmith` and `driver.py`.

### Input Header

```c
{}
```

### `bindgen` Invocation

```
$ {}
```

### Actual Results

<details>

```
{}
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It looks like it will print not a reduced result but the original one.
For example: report will contain reduced header but stacktrace will be from bindings generated from original header.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, this is true. creduce hides the output of the predicate script as it runs it (since it is running many in parallel), so the only way to get the output on the reduced test case would be to run predicate.py once again. I didn't think it was really worth the trouble...

Do you think it's important?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Actually no, just nice to have!

```

</details>

### Expected Results

`bindgen` emits bindings OK, then `rustc` compiles those bindings OK, then the
compiled bindings' layout tests pass OK.

--------------- 8< --------------- 8< --------------- 8< ---------------

<3 <3 <3 Thank you! <3 <3 <3
""".format(
test_case_contents,
" ".join(map(lambda s: "'{}'".format(s), predicate_command)),
result.stdout + result.stderr
))

if __name__ == "__main__":
try:
Expand Down
14 changes: 10 additions & 4 deletions csmith-fuzzing/predicate.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,10 +110,16 @@ def exit_1(msg, child=None):
print(msg)

if child:
print("---------- stdout ----------------------------------------------")
print(decode(child.stdout))
print("---------- stderr ----------------------------------------------")
print(decode(child.stderr))
stdout = decode(child.stdout)
for line in stdout.splitlines():
sys.stdout.write("+")
sys.stdout.write(line)
sys.stdout.write("\n")
stderr = decode(child.stderr)
for line in stderr.splitlines():
sys.stderr.write("+")
sys.stderr.write(line)
sys.stderr.write("\n")

raise ExitOne()

Expand Down