Skip to content

Commit 1313995

Browse files
fitzgenfitzgen
committed
Automatically run creduce in csmith-fuzzing/driver.py
Reduced test cases FTW \o/
1 parent d5a5c50 commit 1313995

File tree

3 files changed

+258
-37
lines changed

3 files changed

+258
-37
lines changed

‎csmith-fuzzing/README.md

+4-4
Original file line numberDiff line numberDiff line change
@@ -18,13 +18,13 @@ tests in the bindings fail, then we report an issue containing the test case!
1818

1919
## Prerequisites
2020

21-
Requires `python3`, `csmith` and `bindgen` to be in `$PATH`.
21+
Requires `python3`, `csmith`, and `creduce` to be in `$PATH`.
2222

23-
Many OS package managers have a `csmith` package:
23+
Many OS package managers have `csmith` and `creduce` packages:
2424

2525
```
26-
$ sudo apt install csmith
27-
$ brew install csmith
26+
$ sudo apt install csmith creduce
27+
$ brew install csmith creduce
2828
$ # Etc...
2929
```
3030

‎csmith-fuzzing/driver.py

+244-29
Original file line numberDiff line numberDiff line change
@@ -1,32 +1,44 @@
11
#!/usr/bin/env python3
22

3-
import os, sys
3+
import argparse
4+
import os
5+
import re
6+
import shlex
7+
import sys
48
from subprocess import run, SubprocessError, DEVNULL, PIPE
59
from tempfile import NamedTemporaryFile
610

7-
csmith_command = [
8-
"csmith",
9-
"--no-checksum",
10-
"--nomain",
11-
"--max-block-size", "1",
12-
"--max-block-depth", "1",
13-
]
11+
DESC = """
1412
15-
def cat(path, title=None):
16-
if not title:
17-
title = path
18-
print("-------------------- {} --------------------".format(title))
19-
run(["cat", path])
13+
A `csmith` fuzzing driver for `bindgen`.
2014
21-
def run_logged(cmd):
22-
with NamedTemporaryFile() as stdout, NamedTemporaryFile() as stderr:
23-
result = run(cmd, stdin=DEVNULL, stdout=stdout, stderr=stderr)
24-
if result.returncode != 0:
25-
print()
26-
print("Error: '{}' exited with code {}".format(" ".join(cmd), result.returncode))
27-
cat(stdout.name, title="stdout")
28-
cat(stdout.name, title="stderr")
29-
return result
15+
Generates random C source files with `csmith` and then passes them to `bindgen`
16+
(via `predicate.py`). If `bindgen` can't emit bindings, `rustc` can't compile
17+
those bindings, or the compiled bindings' layout tests fail, then the driver has
18+
found a bug, and will report the problematic test case to you.
19+
20+
"""
21+
22+
parser = argparse.ArgumentParser(
23+
formatter_class=argparse.RawDescriptionHelpFormatter,
24+
description=DESC.strip())
25+
26+
parser.add_argument(
27+
"--keep-going",
28+
action="store_true",
29+
help="Do not stop after finding a test case that exhibits a bug in `bindgen`. Instead, keep going.")
30+
31+
CSMITH_ARGS="\
32+
--no-checksum \
33+
--nomain \
34+
--max-block-size 1 \
35+
--max-block-depth 1"
36+
37+
parser.add_argument(
38+
"--csmith-args",
39+
type=str,
40+
default=CSMITH_ARGS,
41+
help="Pass this argument string to `csmith`. By default, very small functions are generated.")
3042

3143
BINDGEN_ARGS = "--with-derive-partialeq \
3244
--with-derive-eq \
@@ -35,31 +47,234 @@ def run_logged(cmd):
3547
--with-derive-hash \
3648
--with-derive-default"
3749

50+
parser.add_argument(
51+
"--bindgen-args",
52+
type=str,
53+
default=BINDGEN_ARGS,
54+
help="Pass this argument string to `bindgen`. By default, all traits are derived.")
55+
56+
parser.add_argument(
57+
"--no-creduce",
58+
action="store_false",
59+
dest="creduce",
60+
help="Do not run `creduce` on any buggy test case(s) discovered.")
61+
62+
################################################################################
63+
64+
def cat(path, title=None):
65+
if not title:
66+
title = path
67+
print("-------------------- {} --------------------".format(title))
68+
print()
69+
print()
70+
run(["cat", path])
71+
72+
def decode(f):
73+
return f.decode(encoding="utf-8", errors="ignore")
74+
75+
def run_logged(cmd):
76+
result = run(cmd, stdin=DEVNULL, stdout=PIPE, stderr=PIPE)
77+
result.stdout = decode(result.stdout)
78+
result.stderr = decode(result.stderr)
79+
if result.returncode != 0:
80+
print()
81+
print()
82+
print("Error: {} exited with code {}".format(cmd, result.returncode))
83+
print()
84+
print()
85+
for line in result.stdout.splitlines():
86+
sys.stdout.write("+")
87+
sys.stdout.write(line)
88+
sys.stdout.write("\n")
89+
for line in result.stderr.splitlines():
90+
sys.stderr.write("+")
91+
sys.stderr.write(line)
92+
sys.stderr.write("\n")
93+
return result
94+
3895
def main():
39-
print("Fuzzing `bindgen` with C-Smith...\n")
96+
os.environ["RUST_BACKTRACE"] = "full"
97+
args = parser.parse_args()
98+
99+
bindgen_args = args.bindgen_args
100+
if bindgen_args.find(" -- ") == -1:
101+
bindgen_args = bindgen_args + " -- "
102+
bindgen_args = bindgen_args + " -I{}".format(os.path.abspath(os.path.dirname(sys.argv[0])))
103+
args.bindgen_args = bindgen_args
104+
105+
print()
106+
print()
107+
print("Fuzzing `bindgen` with C-Smith...")
108+
print()
109+
print()
40110

41111
iterations = 0
42112
while True:
43113
print("\rIteration: {}".format(iterations), end="", flush=True)
114+
iterations += 1
44115

45116
input = NamedTemporaryFile(delete=False, prefix="input-", suffix=".h")
46117
input.close()
47-
result = run_logged(csmith_command + ["-o", input.name])
118+
result = run_logged(["csmith", "-o", input.name] + shlex.split(args.csmith_args))
48119
if result.returncode != 0:
49120
exit(1)
50121

51-
result = run_logged([
122+
predicate_command = [
52123
"./predicate.py",
53124
"--bindgen-args",
54-
"{} -- -I{}".format(BINDGEN_ARGS, os.path.abspath(os.path.dirname(sys.argv[0]))),
125+
args.bindgen_args,
55126
input.name
56-
])
127+
]
128+
result = run_logged(predicate_command)
129+
57130
if result.returncode != 0:
58-
cat(input.name)
131+
print()
132+
print()
133+
cat(input.name, title="Failing test case: {}".format(input.name))
134+
print()
135+
print()
136+
137+
if args.creduce:
138+
creduce(args, input.name, result)
139+
140+
print_issue_template(args, input.name, predicate_command, result)
141+
142+
if args.keep_going:
143+
continue
59144
exit(1)
60145

61146
os.remove(input.name)
62-
iterations += 1
147+
148+
RUSTC_ERROR_REGEX = re.compile(r".*(error\[.*].*)")
149+
LAYOUT_TEST_FAILURE = re.compile(r".*(test bindgen_test_layout_.* \.\.\. FAILED)")
150+
151+
def creduce(args, failing_test_case, result):
152+
print()
153+
print()
154+
print("Reducing failing test case with `creduce`...")
155+
156+
match = re.search(RUSTC_ERROR_REGEX, result.stderr)
157+
if match:
158+
error_msg = match.group(1)
159+
print("...searching for \"{}\".".format(error_msg))
160+
return creduce_with_predicate_flags(
161+
args,
162+
failing_test_case,
163+
"--bindgen-args '{}' --expect-compile-fail --rustc-grep '{}'".format(
164+
args.bindgen_args,
165+
re.escape(error_msg)
166+
)
167+
)
168+
169+
match = re.search(LAYOUT_TEST_FAILURE, result.stdout)
170+
if match:
171+
layout_failure = match.group(1)
172+
struct_name = layout_failure[len("test bindgen_test_layout_"):layout_failure.rindex(" ... FAILED")]
173+
print("...searching for \"{}\".".format(layout_failure))
174+
return creduce_with_predicate_flags(
175+
args,
176+
failing_test_case,
177+
"--bindgen-args '{}' --expect-layout-tests-fail --bindings-grep '{}' --layout-tests-grep '{}'".format(
178+
args.bindgen_args,
179+
re.escape(struct_name),
180+
re.escape(layout_failure)
181+
)
182+
)
183+
184+
print("...nevermind, don't know how to `creduce` this bug. Skipping.")
185+
186+
def creduce_with_predicate_flags(args, failing_test_case, predicate_flags):
187+
predicate = """
188+
#!/usr/bin/env bash
189+
set -eu
190+
{} {} {}
191+
""".format(
192+
os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), "predicate.py")),
193+
predicate_flags,
194+
os.path.basename(failing_test_case)
195+
)
196+
197+
print("...and reducing with this script:")
198+
print()
199+
print()
200+
print(predicate)
201+
print()
202+
print()
203+
204+
predicate_path = failing_test_case + ".predicate.sh"
205+
with open(predicate_path, "w") as p:
206+
p.write(predicate)
207+
os.chmod(predicate_path, 0o755)
208+
209+
creduce_command = ["creduce", "--n", str(os.cpu_count()), predicate_path, failing_test_case]
210+
print("Running:", creduce_command)
211+
result = run(creduce_command)
212+
if result.returncode == 0:
213+
print()
214+
print()
215+
print("`creduce` reduced the failing test case to:")
216+
print()
217+
print()
218+
cat(failing_test_case)
219+
print()
220+
print()
221+
else:
222+
print()
223+
print()
224+
print("`creduce` failed!")
225+
if not args.keep_going:
226+
sys.exit(1)
227+
228+
def print_issue_template(args, failing_test_case, predicate_command, result):
229+
test_case_contents = None
230+
with open(failing_test_case, "r") as f:
231+
test_case_contents = f.read()
232+
233+
print("""
234+
235+
! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! !
236+
! File this issue at https://github.com/rust-lang-nursery/rust-bindgen/issues/new !
237+
! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! !
238+
239+
--------------- 8< --------------- 8< --------------- 8< ---------------
240+
241+
This bug was found with `csmith` and `driver.py`.
242+
243+
### Input Header
244+
245+
```c
246+
{}
247+
```
248+
249+
### `bindgen` Invocation
250+
251+
```
252+
$ {}
253+
```
254+
255+
### Actual Results
256+
257+
<details>
258+
259+
```
260+
{}
261+
```
262+
263+
</details>
264+
265+
### Expected Results
266+
267+
`bindgen` emits bindings OK, then `rustc` compiles those bindings OK, then the
268+
compiled bindings' layout tests pass OK.
269+
270+
--------------- 8< --------------- 8< --------------- 8< ---------------
271+
272+
<3 <3 <3 Thank you! <3 <3 <3
273+
""".format(
274+
test_case_contents,
275+
" ".join(map(lambda s: "'{}'".format(s), predicate_command)),
276+
result.stdout + result.stderr
277+
))
63278

64279
if __name__ == "__main__":
65280
try:

‎csmith-fuzzing/predicate.py

+10-4
Original file line numberDiff line numberDiff line change
@@ -110,10 +110,16 @@ def exit_1(msg, child=None):
110110
print(msg)
111111

112112
if child:
113-
print("---------- stdout ----------------------------------------------")
114-
print(decode(child.stdout))
115-
print("---------- stderr ----------------------------------------------")
116-
print(decode(child.stderr))
113+
stdout = decode(child.stdout)
114+
for line in stdout.splitlines():
115+
sys.stdout.write("+")
116+
sys.stdout.write(line)
117+
sys.stdout.write("\n")
118+
stderr = decode(child.stderr)
119+
for line in stderr.splitlines():
120+
sys.stderr.write("+")
121+
sys.stderr.write(line)
122+
sys.stderr.write("\n")
117123

118124
raise ExitOne()
119125

0 commit comments

Comments
 (0)