Benchmarks shouldn't include test code

Author: lavalamp

internal/fixture/state.go

@@ -268,6 +268,32 @@ func (tc TestCase) Test(parser typed.ParseableType) error {
 	return tc.TestWithConverter(parser, &dummyConverter{})
 }
 
+// Bench runs the test-case using the given parser and a dummy converter, but
+// doesn't check exit conditions--see the comment for BenchWithConverter.
+func (tc TestCase) Bench(parser typed.ParseableType) error {
+	return tc.BenchWithConverter(parser, &dummyConverter{})
+}
+
+// BenchWithConverter runs the test-case using the given parser and converter,
+// but doesn't do any comparison operations aftewards; you should probably run
+// TestWithConverter once and reset the benchmark, to make sure the test case
+// actually passes..
+func (tc TestCase) BenchWithConverter(parser typed.ParseableType, converter merge.Converter) error {
+	state := State{
+		Updater: &merge.Updater{Converter: converter},
+		Parser:  parser,
+	}
+	// We currently don't have any test that converts, we can take
+	// care of that later.
+	for i, ops := range tc.Ops {
+		err := ops.run(&state)
+		if err != nil {
+			return fmt.Errorf("failed operation %d: %v", i, err)
+		}
+	}
+	return nil
+}
+
 // TestWithConverter runs the test-case using the given parser and converter.
 func (tc TestCase) TestWithConverter(parser typed.ParseableType, converter merge.Converter) error {
 	state := State{

merge/deduced_test.go

@@ -598,9 +598,15 @@ func BenchmarkDeducedSimple(b *testing.B) {
 		},
 	}
 
+	// Make sure this passes...
+	if err := test.Test(typed.DeducedParseableType); err != nil {
+		b.Fatal(err)
+	}
+
 	b.ReportAllocs()
+	b.ResetTimer()
 	for n := 0; n < b.N; n++ {
-		if err := test.Test(typed.DeducedParseableType); err != nil {
+		if err := test.Bench(typed.DeducedParseableType); err != nil {
 			b.Fatal(err)
 		}
 	}
@@ -701,9 +707,15 @@ func BenchmarkDeducedNested(b *testing.B) {
 			`,
 	}
 
+	// Make sure this passes...
+	if err := test.Test(typed.DeducedParseableType); err != nil {
+		b.Fatal(err)
+	}
+
 	b.ReportAllocs()
+	b.ResetTimer()
 	for n := 0; n < b.N; n++ {
-		if err := test.Test(typed.DeducedParseableType); err != nil {
+		if err := test.Bench(typed.DeducedParseableType); err != nil {
 			b.Fatal(err)
 		}
 	}
@@ -804,9 +816,15 @@ func BenchmarkDeducedNestedAcrossVersion(b *testing.B) {
 		`,
 	}
 
+	// Make sure this passes...
+	if err := test.Test(typed.DeducedParseableType); err != nil {
+		b.Fatal(err)
+	}
+
 	b.ReportAllocs()
+	b.ResetTimer()
 	for n := 0; n < b.N; n++ {
-		if err := test.Test(typed.DeducedParseableType); err != nil {
+		if err := test.Bench(typed.DeducedParseableType); err != nil {
 			b.Fatal(err)
 		}
 	}

merge/leaf_test.go

@@ -531,9 +531,15 @@ func BenchmarkLeafConflictAcrossVersion(b *testing.B) {
 		},
 	}
 
+	// Make sure this passes...
+	if err := test.Test(leafFieldsParser); err != nil {
+		b.Fatal(err)
+	}
+
 	b.ReportAllocs()
+	b.ResetTimer()
 	for n := 0; n < b.N; n++ {
-		if err := test.Test(leafFieldsParser); err != nil {
+		if err := test.Bench(leafFieldsParser); err != nil {
 			b.Fatal(err)
 		}
 	}

merge/multiple_appliers_test.go

@@ -1103,9 +1103,15 @@ func BenchmarkMultipleApplierRecursiveRealConversion(b *testing.B) {
 		},
 	}
 
+	// Make sure this passes...
+	if err := test.TestWithConverter(nestedTypeParser, repeatingConverter{nestedTypeParser}); err != nil {
+		b.Fatal(err)
+	}
+
 	b.ReportAllocs()
+	b.ResetTimer()
 	for n := 0; n < b.N; n++ {
-		if err := test.TestWithConverter(nestedTypeParser, repeatingConverter{nestedTypeParser}); err != nil {
+		if err := test.BenchWithConverter(nestedTypeParser, repeatingConverter{nestedTypeParser}); err != nil {
 			b.Fatal(err)
 		}
 	}