Skip to content

Commit e24d665

Browse files
stmontgomerystmontgomery
authored
Merge pull request #320 from stmontgomery/assertForOverFulfill-disabled
[SR-12575] Over-fulfilling XCTestExpectation records test failure even when `assertForOverFulfill` is disabled
2 parents c21ac84 + dcaa306 commit e24d665

File tree

3 files changed

+49
-41
lines changed

3 files changed

+49
-41
lines changed

‎Sources/XCTest/Public/Asynchronous/XCTestExpectation.swift

+6-13
Original file line numberDiff line numberDiff line change
@@ -259,19 +259,12 @@ open class XCTestExpectation {
259259
// expectations have completed. Similarly, this should cause an
260260
// error as well.
261261

262-
if queue_isFulfilled {
263-
// FIXME: No regression tests exist for this feature. We may break it
264-
// without ever realizing (similar to `continueAfterFailure`).
265-
if _assertForOverFulfill {
266-
fatalError("API violation - multiple calls made to fulfill() for \(queue_expectationDescription)")
267-
}
268-
269-
if let testCase = XCTCurrentTestCase {
270-
testCase.recordFailure(
271-
description: "API violation - multiple calls made to XCTestExpectation.fulfill() for \(queue_expectationDescription).",
272-
at: sourceLocation,
273-
expected: false)
274-
}
262+
if queue_isFulfilled, _assertForOverFulfill, let testCase = XCTCurrentTestCase {
263+
testCase.recordFailure(
264+
description: "API violation - multiple calls made to XCTestExpectation.fulfill() for \(queue_expectationDescription).",
265+
at: sourceLocation,
266+
expected: false)
267+
275268
return nil
276269
}
277270

‎Tests/Functional/Asynchronous/Expectations/main.swift

+40-3
Original file line numberDiff line numberDiff line change
@@ -338,6 +338,39 @@ class ExpectationsTestCase: XCTestCase {
338338
waitForExpectations(timeout: 0.2)
339339
}
340340

341+
// PRAGMA MARK: - assertForOverFulfill
342+
343+
// CHECK: Test Case 'ExpectationsTestCase.test_assertForOverfulfill_disabled' started at \d+-\d+-\d+ \d+:\d+:\d+\.\d+
344+
// CHECK: Test Case 'ExpectationsTestCase.test_assertForOverfulfill_disabled' passed \(\d+\.\d+ seconds\)
345+
func test_assertForOverfulfill_disabled() {
346+
let foo = XCTestExpectation(description: "foo")
347+
XCTAssertFalse(foo.assertForOverFulfill, "assertForOverFulfill should be disabled by default")
348+
foo.fulfill()
349+
foo.fulfill()
350+
}
351+
352+
// CHECK: Test Case 'ExpectationsTestCase.test_assertForOverfulfill_failure' started at \d+-\d+-\d+ \d+:\d+:\d+\.\d+
353+
// CHECK: .*[/\\]Tests[/\\]Functional[/\\]Asynchronous[/\\]Expectations[/\\]main.swift:[[@LINE+7]]: error: ExpectationsTestCase.test_assertForOverfulfill_failure : API violation - multiple calls made to XCTestExpectation.fulfill\(\) for rob.
354+
// CHECK: .*[/\\]Tests[/\\]Functional[/\\]Asynchronous[/\\]Expectations[/\\]main.swift:[[@LINE+16]]: error: ExpectationsTestCase.test_assertForOverfulfill_failure : API violation - multiple calls made to XCTestExpectation.fulfill\(\) for rob.
355+
// CHECK: Test Case 'ExpectationsTestCase.test_assertForOverfulfill_failure' failed \(\d+\.\d+ seconds\)
356+
func test_assertForOverfulfill_failure() {
357+
let expectation = self.expectation(description: "rob")
358+
expectation.assertForOverFulfill = true
359+
expectation.fulfill()
360+
expectation.fulfill()
361+
// FIXME: The behavior here is subtly different from Objective-C XCTest.
362+
// Objective-C XCTest would stop executing the test on the line
363+
// above, and so would not report a failure for this line below.
364+
// In total, it would highlight one line as a failure in this
365+
// test.
366+
//
367+
// swift-corelibs-xctest continues to execute the test, and so
368+
// highlights both the lines above and below as failures.
369+
// This should be fixed such that the behavior is identical.
370+
expectation.fulfill()
371+
self.waitForExpectations(timeout: 0.1)
372+
}
373+
341374
// PRAGMA MARK: - Interrupted Waiters
342375

343376
// Disabled due to non-deterministic ordering of XCTWaiterDelegate callbacks, see [SR-10034] and <rdar://problem/49123061>
@@ -497,6 +530,10 @@ class ExpectationsTestCase: XCTestCase {
497530
("test_countedConditionPassBeforeWaiting", test_countedConditionPassBeforeWaiting),
498531
("test_countedConditionFail", test_countedConditionFail),
499532

533+
// assertForOverFulfill
534+
("test_assertForOverfulfill_disabled", test_assertForOverfulfill_disabled),
535+
("test_assertForOverfulfill_failure", test_assertForOverfulfill_failure),
536+
500537
// Interrupted Waiters
501538
// ("test_outerWaiterTimesOut_InnerWaitersAreInterrupted", test_outerWaiterTimesOut_InnerWaitersAreInterrupted),
502539
("test_outerWaiterCompletes_InnerWaiterTimesOut", test_outerWaiterCompletes_InnerWaiterTimesOut),
@@ -513,11 +550,11 @@ class ExpectationsTestCase: XCTestCase {
513550
}()
514551
}
515552
// CHECK: Test Suite 'ExpectationsTestCase' failed at \d+-\d+-\d+ \d+:\d+:\d+\.\d+
516-
// CHECK: \t Executed 30 tests, with 14 failures \(0 unexpected\) in \d+\.\d+ \(\d+\.\d+\) seconds
553+
// CHECK: \t Executed 32 tests, with 16 failures \(2 unexpected\) in \d+\.\d+ \(\d+\.\d+\) seconds
517554

518555
XCTMain([testCase(ExpectationsTestCase.allTests)])
519556

520557
// CHECK: Test Suite '.*\.xctest' failed at \d+-\d+-\d+ \d+:\d+:\d+\.\d+
521-
// CHECK: \t Executed 30 tests, with 14 failures \(0 unexpected\) in \d+\.\d+ \(\d+\.\d+\) seconds
558+
// CHECK: \t Executed 32 tests, with 16 failures \(2 unexpected\) in \d+\.\d+ \(\d+\.\d+\) seconds
522559
// CHECK: Test Suite 'All tests' failed at \d+-\d+-\d+ \d+:\d+:\d+\.\d+
523-
// CHECK: \t Executed 30 tests, with 14 failures \(0 unexpected\) in \d+\.\d+ \(\d+\.\d+\) seconds
560+
// CHECK: \t Executed 32 tests, with 16 failures \(2 unexpected\) in \d+\.\d+ \(\d+\.\d+\) seconds

‎Tests/Functional/Asynchronous/Misuse/main.swift

+3-25
Original file line numberDiff line numberDiff line change
@@ -28,41 +28,19 @@ class MisuseTestCase: XCTestCase {
2828
self.waitForExpectations(timeout: 0.1)
2929
}
3030

31-
// CHECK: Test Case 'MisuseTestCase.test_whenExpectationIsFulfilledMultipleTimes_fails' started at \d+-\d+-\d+ \d+:\d+:\d+\.\d+
32-
// CHECK: .*[/\\]Tests[/\\]Functional[/\\]Asynchronous[/\\]Misuse[/\\]main.swift:[[@LINE+6]]: error: MisuseTestCase.test_whenExpectationIsFulfilledMultipleTimes_fails : API violation - multiple calls made to XCTestExpectation.fulfill\(\) for rob.
33-
// CHECK: .*[/\\]Tests[/\\]Functional[/\\]Asynchronous[/\\]Misuse[/\\]main.swift:[[@LINE+15]]: error: MisuseTestCase.test_whenExpectationIsFulfilledMultipleTimes_fails : API violation - multiple calls made to XCTestExpectation.fulfill\(\) for rob.
34-
// CHECK: Test Case 'MisuseTestCase.test_whenExpectationIsFulfilledMultipleTimes_fails' failed \(\d+\.\d+ seconds\)
35-
func test_whenExpectationIsFulfilledMultipleTimes_fails() {
36-
let expectation = self.expectation(description: "rob")
37-
expectation.fulfill()
38-
expectation.fulfill()
39-
// FIXME: The behavior here is subtly different from Objective-C XCTest.
40-
// Objective-C XCTest would stop executing the test on the line
41-
// above, and so would not report a failure for this line below.
42-
// In total, it would highlight one line as a failure in this
43-
// test.
44-
//
45-
// swift-corelibs-xctest continues to execute the test, and so
46-
// highlights both the lines above and below as failures.
47-
// This should be fixed such that the behavior is identical.
48-
expectation.fulfill()
49-
self.waitForExpectations(timeout: 0.1)
50-
}
51-
5231
static var allTests = {
5332
return [
5433
("test_whenExpectationsAreMade_butNotWaitedFor_fails", test_whenExpectationsAreMade_butNotWaitedFor_fails),
5534
("test_whenNoExpectationsAreMade_butTheyAreWaitedFor_fails", test_whenNoExpectationsAreMade_butTheyAreWaitedFor_fails),
56-
("test_whenExpectationIsFulfilledMultipleTimes_fails", test_whenExpectationIsFulfilledMultipleTimes_fails),
5735
]
5836
}()
5937
}
6038
// CHECK: Test Suite 'MisuseTestCase' failed at \d+-\d+-\d+ \d+:\d+:\d+\.\d+
61-
// CHECK: \t Executed 3 tests, with 4 failures \(4 unexpected\) in \d+\.\d+ \(\d+\.\d+\) seconds
39+
// CHECK: \t Executed 2 tests, with 2 failures \(2 unexpected\) in \d+\.\d+ \(\d+\.\d+\) seconds
6240

6341
XCTMain([testCase(MisuseTestCase.allTests)])
6442

6543
// CHECK: Test Suite '.*\.xctest' failed at \d+-\d+-\d+ \d+:\d+:\d+\.\d+
66-
// CHECK: \t Executed 3 tests, with 4 failures \(4 unexpected\) in \d+\.\d+ \(\d+\.\d+\) seconds
44+
// CHECK: \t Executed 2 tests, with 2 failures \(2 unexpected\) in \d+\.\d+ \(\d+\.\d+\) seconds
6745
// CHECK: Test Suite 'All tests' failed at \d+-\d+-\d+ \d+:\d+:\d+\.\d+
68-
// CHECK: \t Executed 3 tests, with 4 failures \(4 unexpected\) in \d+\.\d+ \(\d+\.\d+\) seconds
46+
// CHECK: \t Executed 2 tests, with 2 failures \(2 unexpected\) in \d+\.\d+ \(\d+\.\d+\) seconds

0 commit comments

Comments
 (0)