Skip to content

Commit 5cc050b

Browse files
Grahame Bowlandbrson
Grahame Bowland
authored and
brson
committed
Logfile output from tests; summarise in make check
Add an optional --logfile argument to std::test::test_main and to compiletest. Use this features and the new 'check-summary.py' script to summarise all the tests performed by the 'check' target. This is a short term fix for #2075.
1 parent 3aed498 commit 5cc050b

File tree

6 files changed

+122
-26
lines changed

6 files changed

+122
-26
lines changed

Diff for: ‎mk/clean.mk

+1
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ clean-misc:
3131
$(Q)rm -f $(CRATE_DEPFILES:%.d=%.d.tmp)
3232
$(Q)rm -Rf $(DOCS)
3333
$(Q)rm -Rf $(GENERATED)
34+
$(Q)rm -f tmp/*.log
3435
$(Q)rm -f rustllvm/$(CFG_RUSTLLVM) rustllvm/rustllvmbits.a
3536
$(Q)rm -f rt/$(CFG_RUNTIME)
3637
$(Q)find rustllvm rt -name '*.[odasS]' -delete

Diff for: ‎mk/tests.mk

+36-19
Original file line numberDiff line numberDiff line change
@@ -66,9 +66,11 @@ endif
6666
# Main test targets
6767
######################################################################
6868

69-
check: tidy all check-stage2 \
69+
check: tidy all check-stage2
70+
$(S)src/etc/check-summary.py tmp/*.log
7071

71-
check-full: tidy all check-stage1 check-stage2 check-stage3 \
72+
check-full: tidy all check-stage1 check-stage2 check-stage3
73+
$(S)src/etc/check-summary.py tmp/*.log
7274

7375
# Run the tidy script in multiple parts to avoid huge 'echo' commands
7476
ifdef CFG_NOTIDY
@@ -120,7 +122,6 @@ tidy:
120122
| xargs -n 10 python $(S)src/etc/tidy.py
121123
endif
122124

123-
124125
######################################################################
125126
# Extracting tests for docs
126127
######################################################################
@@ -232,7 +233,8 @@ $(3)/test/coretest.stage$(1)-$(2)$$(X): \
232233
check-stage$(1)-T-$(2)-H-$(3)-core-dummy: \
233234
$(3)/test/coretest.stage$(1)-$(2)$$(X)
234235
@$$(call E, run: $$<)
235-
$$(Q)$$(call CFG_RUN_TEST,$$<,$(2),$(3)) $$(TESTARGS)
236+
$$(Q)$$(call CFG_RUN_TEST,$$<,$(2),$(3)) $$(TESTARGS) \
237+
--logfile tmp/check-stage$(1)-T-$(2)-H-$(3)-core.log
236238

237239
# Rules for the standard library test runner
238240

@@ -245,7 +247,8 @@ $(3)/test/stdtest.stage$(1)-$(2)$$(X): \
245247
check-stage$(1)-T-$(2)-H-$(3)-std-dummy: \
246248
$(3)/test/stdtest.stage$(1)-$(2)$$(X)
247249
@$$(call E, run: $$<)
248-
$$(Q)$$(call CFG_RUN_TEST,$$<,$(2),$(3)) $$(TESTARGS)
250+
$$(Q)$$(call CFG_RUN_TEST,$$<,$(2),$(3)) $$(TESTARGS) \
251+
--logfile tmp/check-stage$(1)-T-$(2)-H-$(3)-std.log
249252

250253
# Rules for the rustc test runner
251254

@@ -260,7 +263,8 @@ $(3)/test/rustctest.stage$(1)-$(2)$$(X): \
260263
check-stage$(1)-T-$(2)-H-$(3)-rustc-dummy: \
261264
$(3)/test/rustctest.stage$(1)-$(2)$$(X)
262265
@$$(call E, run: $$<)
263-
$$(Q)$$(call CFG_RUN_TEST,$$<,$(2),$(3)) $$(TESTARGS)
266+
$$(Q)$$(call CFG_RUN_TEST,$$<,$(2),$(3)) $$(TESTARGS) \
267+
--logfile tmp/check-stage$(1)-T-$(2)-H-$(3)-rustc.log
264268

265269
# Rules for the rustdoc test runner
266270

@@ -276,7 +280,8 @@ $(3)/test/rustdoctest.stage$(1)-$(2)$$(X): \
276280
check-stage$(1)-T-$(2)-H-$(3)-rustdoc-dummy: \
277281
$(3)/test/rustdoctest.stage$(1)-$(2)$$(X)
278282
@$$(call E, run: $$<)
279-
$$(Q)$$(call CFG_RUN_TEST,$$<,$(2),$(3)) $$(TESTARGS)
283+
$$(Q)$$(call CFG_RUN_TEST,$$<,$(2),$(3)) $$(TESTARGS) \
284+
--logfile tmp/check-stage$(1)-T-$(2)-H-$(3)-rustdoc.log
280285

281286
# Rules for the cfail/rfail/rpass/bench/perf test runner
282287

@@ -365,87 +370,98 @@ check-stage$(1)-T-$(2)-H-$(3)-cfail-dummy: \
365370
$$(CFAIL_TESTS)
366371
@$$(call E, run cfail: $$<)
367372
$$(Q)$$(call CFG_RUN_CTEST,$(1),$$<,$(3)) \
368-
$$(CFAIL_ARGS$(1)-T-$(2)-H-$(3))
373+
$$(CFAIL_ARGS$(1)-T-$(2)-H-$(3)) \
374+
--logfile tmp/check-stage$(1)-T-$(2)-H-$(3)-cfail.log
369375

370376
check-stage$(1)-T-$(2)-H-$(3)-rfail-dummy: \
371377
$$(HBIN$(1)_H_$(3))/compiletest$$(X) \
372378
$$(SREQ$(1)_T_$(2)_H_$(3)) \
373379
$$(RFAIL_TESTS)
374380
@$$(call E, run rfail: $$<)
375381
$$(Q)$$(call CFG_RUN_CTEST,$(1),$$<,$(3)) \
376-
$$(RFAIL_ARGS$(1)-T-$(2)-H-$(3))
382+
$$(RFAIL_ARGS$(1)-T-$(2)-H-$(3)) \
383+
--logfile tmp/check-stage$(1)-T-$(2)-H-$(3)-rfail.log
377384

378385
check-stage$(1)-T-$(2)-H-$(3)-rpass-dummy: \
379386
$$(HBIN$(1)_H_$(3))/compiletest$$(X) \
380387
$$(SREQ$(1)_T_$(2)_H_$(3)) \
381388
$$(RPASS_TESTS)
382389
@$$(call E, run rpass: $$<)
383390
$$(Q)$$(call CFG_RUN_CTEST,$(1),$$<,$(3)) \
384-
$$(RPASS_ARGS$(1)-T-$(2)-H-$(3))
391+
$$(RPASS_ARGS$(1)-T-$(2)-H-$(3)) \
392+
--logfile tmp/check-stage$(1)-T-$(2)-H-$(3)-rpass.log
385393

386394
check-stage$(1)-T-$(2)-H-$(3)-bench-dummy: \
387395
$$(HBIN$(1)_H_$(3))/compiletest$$(X) \
388396
$$(SREQ$(1)_T_$(2)_H_$(3)) \
389397
$$(BENCH_TESTS)
390398
@$$(call E, run bench: $$<)
391399
$$(Q)$$(call CFG_RUN_CTEST,$(1),$$<,$(3)) \
392-
$$(BENCH_ARGS$(1)-T-$(2)-H-$(3))
400+
$$(BENCH_ARGS$(1)-T-$(2)-H-$(3)) \
401+
--logfile tmp/check-stage$(1)-T-$(2)-H-$(3)-bench.log
393402

394403
check-stage$(1)-T-$(2)-H-$(3)-perf-dummy: \
395404
$$(HBIN$(1)_H_$(3))/compiletest$$(X) \
396405
$$(SREQ$(1)_T_$(2)_H_$(3)) \
397406
$$(BENCH_TESTS)
398407
@$$(call E, perf: $$<)
399408
$$(Q)$$(call CFG_RUN_CTEST,$(1),$$<,$(3)) \
400-
$$(PERF_ARGS$(1)-T-$(2)-H-$(3))
409+
$$(PERF_ARGS$(1)-T-$(2)-H-$(3)) \
410+
--logfile tmp/check-stage$(1)-T-$(2)-H-$(3)-perf.log
401411

402412
check-stage$(1)-T-$(2)-H-$(3)-pretty-rpass-dummy: \
403413
$$(HBIN$(1)_H_$(3))/compiletest$$(X) \
404414
$$(SREQ$(1)_T_$(2)_H_$(3)) \
405415
$$(RPASS_TESTS)
406416
@$$(call E, run pretty-rpass: $$<)
407417
$$(Q)$$(call CFG_RUN_CTEST,$(1),$$<,$(3)) \
408-
$$(PRETTY_RPASS_ARGS$(1)-T-$(2)-H-$(3))
418+
$$(PRETTY_RPASS_ARGS$(1)-T-$(2)-H-$(3)) \
419+
--logfile tmp/check-stage$(1)-T-$(2)-H-$(3)-pretty-rpass.log
409420

410421
check-stage$(1)-T-$(2)-H-$(3)-pretty-rfail-dummy: \
411422
$$(HBIN$(1)_H_$(3))/compiletest$$(X) \
412423
$$(SREQ$(1)_T_$(2)_H_$(3)) \
413424
$$(RFAIL_TESTS)
414425
@$$(call E, run pretty-rfail: $$<)
415426
$$(Q)$$(call CFG_RUN_CTEST,$(1),$$<,$(3)) \
416-
$$(PRETTY_RFAIL_ARGS$(1)-T-$(2)-H-$(3))
427+
$$(PRETTY_RFAIL_ARGS$(1)-T-$(2)-H-$(3)) \
428+
--logfile tmp/check-stage$(1)-T-$(2)-H-$(3)-pretty-rfail.log
417429

418430
check-stage$(1)-T-$(2)-H-$(3)-pretty-bench-dummy: \
419431
$$(HBIN$(1)_H_$(3))/compiletest$$(X) \
420432
$$(SREQ$(1)_T_$(2)_H_$(3)) \
421433
$$(BENCH_TESTS)
422434
@$$(call E, run pretty-bench: $$<)
423435
$$(Q)$$(call CFG_RUN_CTEST,$(1),$$<,$(3)) \
424-
$$(PRETTY_BENCH_ARGS$(1)-T-$(2)-H-$(3))
436+
$$(PRETTY_BENCH_ARGS$(1)-T-$(2)-H-$(3)) \
437+
--logfile tmp/check-stage$(1)-T-$(2)-H-$(3)-pretty-bench.log
425438

426439
check-stage$(1)-T-$(2)-H-$(3)-pretty-pretty-dummy: \
427440
$$(HBIN$(1)_H_$(3))/compiletest$$(X) \
428441
$$(SREQ$(1)_T_$(2)_H_$(3)) \
429442
$$(PRETTY_TESTS)
430443
@$$(call E, run pretty-pretty: $$<)
431444
$$(Q)$$(call CFG_RUN_CTEST,$(1),$$<,$(3)) \
432-
$$(PRETTY_PRETTY_ARGS$(1)-T-$(2)-H-$(3))
445+
$$(PRETTY_PRETTY_ARGS$(1)-T-$(2)-H-$(3)) \
446+
--logfile tmp/check-stage$(1)-T-$(2)-H-$(3)-pretty-pretty.log
433447

434448
check-stage$(1)-T-$(2)-H-$(3)-doc-tutorial-dummy: \
435449
$$(HBIN$(1)_H_$(3))/compiletest$$(X) \
436450
$$(SREQ$(1)_T_$(2)_H_$(3)) \
437451
doc-tutorial-extract$(3)
438452
@$$(call E, run doc-tutorial: $$<)
439453
$$(Q)$$(call CFG_RUN_CTEST,$(1),$$<,$(3)) \
440-
$$(DOC_TUTORIAL_ARGS$(1)-T-$(2)-H-$(3))
454+
$$(DOC_TUTORIAL_ARGS$(1)-T-$(2)-H-$(3)) \
455+
--logfile tmp/check-stage$(1)-T-$(2)-H-$(3)-doc-tutorial.log
441456

442457
check-stage$(1)-T-$(2)-H-$(3)-doc-ref-dummy: \
443458
$$(HBIN$(1)_H_$(3))/compiletest$$(X) \
444459
$$(SREQ$(1)_T_$(2)_H_$(3)) \
445460
doc-ref-extract$(3)
446461
@$$(call E, run doc-ref: $$<)
447462
$$(Q)$$(call CFG_RUN_CTEST,$(1),$$<,$(3)) \
448-
$$(DOC_REF_ARGS$(1)-T-$(2)-H-$(3))
463+
$$(DOC_REF_ARGS$(1)-T-$(2)-H-$(3)) \
464+
--logfile tmp/check-stage$(1)-T-$(2)-H-$(3)-doc-ref.log
449465

450466
endef
451467

@@ -489,7 +505,8 @@ $(3)/test/$$(FT_DRIVER)-$(2)$$(X): \
489505
$(3)/test/$$(FT_DRIVER)-$(2).out: \
490506
$(3)/test/$$(FT_DRIVER)-$(2)$$(X) \
491507
$$(SREQ2_T_$(2)_H_$(3))
492-
$$(Q)$$(call CFG_RUN_TEST,$$<,$(2),$(3))
508+
$$(Q)$$(call CFG_RUN_TEST,$$<,$(2),$(3)) \
509+
--logfile tmp/$$(FT_DRIVER)-$(2).log
493510

494511
check-fast-T-$(2)-H-$(3): tidy \
495512
check-stage2-T-$(2)-H-$(3)-rustc \

Diff for: ‎src/compiletest/common.rs

+3
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,9 @@ type config = {
3333
// Only run tests that match this filter
3434
filter: option<str>,
3535

36+
// Write out a parseable log of tests that were run
37+
logfile: option<str>,
38+
3639
// A command line to prefix program execution with,
3740
// for running under valgrind
3841
runtool: option<str>,

Diff for: ‎src/compiletest/compiletest.rs

+10-2
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,8 @@ fn parse_config(args: [str]) -> config {
3535
getopts::reqopt("stage-id"),
3636
getopts::reqopt("mode"), getopts::optflag("ignored"),
3737
getopts::optopt("runtool"), getopts::optopt("rustcflags"),
38-
getopts::optflag("verbose")];
38+
getopts::optflag("verbose"),
39+
getopts::optopt("logfile")];
3940

4041
check (vec::is_not_empty(args));
4142
let args_ = vec::tail(args);
@@ -58,6 +59,7 @@ fn parse_config(args: [str]) -> config {
5859
if vec::len(match.free) > 0u {
5960
option::some(match.free[0])
6061
} else { option::none },
62+
logfile: getopts::opt_maybe_str(match, "logfile"),
6163
runtool: getopts::opt_maybe_str(match, "runtool"),
6264
rustcflags: getopts::opt_maybe_str(match, "rustcflags"),
6365
verbose: getopts::opt_present(match, "verbose")};
@@ -121,7 +123,13 @@ fn test_opts(config: config) -> test::test_opts {
121123
option::some(s) { option::some(s) }
122124
option::none { option::none }
123125
},
124-
run_ignored: config.run_ignored}
126+
run_ignored: config.run_ignored,
127+
logfile:
128+
alt config.logfile {
129+
option::some(s) { option::some(s) }
130+
option::none { option::none }
131+
}
132+
}
125133
}
126134

127135
fn make_tests(config: config) -> [test::test_desc] {

Diff for: ‎src/etc/check-summary.py

+32
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
#!/usr/bin/env python
2+
3+
import sys
4+
5+
if __name__ == '__main__':
6+
summaries = []
7+
def summarise(fname):
8+
summary = {}
9+
fd = open(fname)
10+
for line in fd:
11+
status, test = line.strip().split(' ', 1)
12+
if not summary.has_key(status):
13+
summary[status] = []
14+
summary[status].append(test)
15+
summaries.append((fname, summary))
16+
def count(t):
17+
return sum(map(lambda (f, s): len(s.get(t, [])), summaries))
18+
logfiles = sys.argv[1:]
19+
map(summarise, logfiles)
20+
ok = count('ok')
21+
failed = count('failed')
22+
ignored = count('ignored')
23+
print "summary of %d test logs: %d passed; %d failed; %d ignored" % \
24+
(len(logfiles), ok, failed, ignored)
25+
if failed > 0:
26+
print "failed tests:"
27+
for f, s in summaries:
28+
failures = s.get('failed', [])
29+
if len(failures) > 0:
30+
print " %s:" % (f)
31+
for test in failures:
32+
print " %s" % (test)

Diff for: ‎src/libstd/test.rs

+40-5
Original file line numberDiff line numberDiff line change
@@ -57,14 +57,15 @@ fn test_main(args: [str], tests: [test_desc]) {
5757
if !run_tests_console(opts, tests) { fail "Some tests failed"; }
5858
}
5959

60-
type test_opts = {filter: option<str>, run_ignored: bool};
60+
type test_opts = {filter: option<str>, run_ignored: bool,
61+
logfile: option<str>};
6162

6263
type opt_res = either<test_opts, str>;
6364

6465
// Parses command line arguments into test options
6566
fn parse_opts(args: [str]) -> opt_res {
6667
let args_ = vec::tail(args);
67-
let opts = [getopts::optflag("ignored")];
68+
let opts = [getopts::optflag("ignored"), getopts::optopt("logfile")];
6869
let match =
6970
alt getopts::getopts(args_, opts) {
7071
ok(m) { m }
@@ -77,8 +78,10 @@ fn parse_opts(args: [str]) -> opt_res {
7778
} else { option::none };
7879

7980
let run_ignored = getopts::opt_present(match, "ignored");
81+
let logfile = getopts::opt_maybe_str(match, "logfile");
8082

81-
let test_opts = {filter: filter, run_ignored: run_ignored};
83+
let test_opts = {filter: filter, run_ignored: run_ignored,
84+
logfile: logfile};
8285

8386
ret either::left(test_opts);
8487
}
@@ -87,6 +90,7 @@ enum test_result { tr_ok, tr_failed, tr_ignored, }
8790

8891
type console_test_state =
8992
@{out: io::writer,
93+
log_out: option<io::writer>,
9094
use_color: bool,
9195
mut total: uint,
9296
mut passed: uint,
@@ -106,6 +110,12 @@ fn run_tests_console(opts: test_opts,
106110
}
107111
te_wait(test) { st.out.write_str(#fmt["test %s ... ", test.name]); }
108112
te_result(test, result) {
113+
alt st.log_out {
114+
some(f) {
115+
write_log(f, result, test);
116+
}
117+
none {}
118+
}
109119
alt result {
110120
tr_ok {
111121
st.passed += 1u;
@@ -128,8 +138,21 @@ fn run_tests_console(opts: test_opts,
128138
}
129139
}
130140

141+
let log_out = alt opts.logfile {
142+
some(path) {
143+
alt io::file_writer(path, [io::create, io::truncate]) {
144+
result::ok(w) { some(w) }
145+
result::err(s) {
146+
fail(#fmt("can't open output file: %s", s))
147+
}
148+
}
149+
}
150+
none { none }
151+
};
152+
131153
let st =
132154
@{out: io::stdout(),
155+
log_out: log_out,
133156
use_color: use_color(),
134157
mut total: 0u,
135158
mut passed: 0u,
@@ -156,6 +179,15 @@ fn run_tests_console(opts: test_opts,
156179

157180
ret success;
158181

182+
fn write_log(out: io::writer, result: test_result, test: test_desc) {
183+
out.write_line(#fmt("%s %s",
184+
alt result {
185+
tr_ok { "ok" }
186+
tr_failed { "failed" }
187+
tr_ignored { "ignored" }
188+
}, test.name));
189+
}
190+
159191
fn write_ok(out: io::writer, use_color: bool) {
160192
write_pretty(out, "ok", term::color_green, use_color);
161193
}
@@ -209,6 +241,7 @@ fn should_sort_failures_before_printing_them() {
209241
210242
let st =
211243
@{out: writer,
244+
log_out: option::none,
212245
use_color: false,
213246
mut total: 0u,
214247
mut passed: 0u,
@@ -466,7 +499,8 @@ mod tests {
466499
// When we run ignored tests the test filter should filter out all the
467500
// unignored tests and flip the ignore flag on the rest to false
468501

469-
let opts = {filter: option::none, run_ignored: true};
502+
let opts = {filter: option::none, run_ignored: true,
503+
logfile: option::none};
470504
let tests =
471505
[{name: "1", fn: fn~() { }, ignore: true, should_fail: false},
472506
{name: "2", fn: fn~() { }, ignore: false, should_fail: false}];
@@ -479,7 +513,8 @@ mod tests {
479513

480514
#[test]
481515
fn sort_tests() {
482-
let opts = {filter: option::none, run_ignored: false};
516+
let opts = {filter: option::none, run_ignored: false,
517+
logfile: option::none};
483518

484519
let names =
485520
["sha1::test", "int::test_to_str", "int::test_pow",

0 commit comments

Comments
 (0)