Benchmarks shouldn't include test code

Author: lavalamp

internal/fixture/state.go

@@ -269,6 +269,32 @@ func (tc TestCase) Test(parser typed.ParseableType) error {
 	return tc.TestWithConverter(parser, &dummyConverter{})
 }
 
+// Bench runs the test-case using the given parser and a dummy converter, but
+// doesn't check exit conditions--see the comment for BenchWithConverter.
+func (tc TestCase) Bench(parser typed.ParseableType) error {
+	return tc.BenchWithConverter(parser, &dummyConverter{})
+}
+
+// BenchWithConverter runs the test-case using the given parser and converter,
+// but doesn't do any comparison operations aftewards; you should probably run
+// TestWithConverter once and reset the benchmark, to make sure the test case
+// actually passes..
+func (tc TestCase) BenchWithConverter(parser typed.ParseableType, converter merge.Converter) error {
+	state := State{
+		Updater: &merge.Updater{Converter: converter},
+		Parser:  parser,
+	}
+	// We currently don't have any test that converts, we can take
+	// care of that later.
+	for i, ops := range tc.Ops {
+		err := ops.run(&state)
+		if err != nil {
+			return fmt.Errorf("failed operation %d: %v", i, err)
+		}
+	}
+	return nil
+}
+
 // TestWithConverter runs the test-case using the given parser and converter.
 func (tc TestCase) TestWithConverter(parser typed.ParseableType, converter merge.Converter) error {
 	state := State{

merge/deduced_test.go

@@ -598,7 +598,13 @@ func BenchmarkDeducedSimple(b *testing.B) {
 		},
 	}
 
+	// Make sure this passes...
+	if err := test.Test(typed.DeducedParseableType); err != nil {
+		b.Fatal(err)
+	}
+
 	b.ReportAllocs()
+	b.ResetTimer()
 	for n := 0; n < b.N; n++ {
 		if err := test.Test(typed.DeducedParseableType); err != nil {
 			b.Fatal(err)
@@ -701,7 +707,13 @@ func BenchmarkDeducedNested(b *testing.B) {
 			`,
 	}
 
+	// Make sure this passes...
+	if err := test.Test(typed.DeducedParseableType); err != nil {
+		b.Fatal(err)
+	}
+
 	b.ReportAllocs()
+	b.ResetTimer()
 	for n := 0; n < b.N; n++ {
 		if err := test.Test(typed.DeducedParseableType); err != nil {
 			b.Fatal(err)
@@ -804,7 +816,13 @@ func BenchmarkDeducedNestedAcrossVersion(b *testing.B) {
 		`,
 	}
 
+	// Make sure this passes...
+	if err := test.Test(typed.DeducedParseableType); err != nil {
+		b.Fatal(err)
+	}
+
 	b.ReportAllocs()
+	b.ResetTimer()
 	for n := 0; n < b.N; n++ {
 		if err := test.Test(typed.DeducedParseableType); err != nil {
 			b.Fatal(err)

merge/leaf_test.go

@@ -531,7 +531,13 @@ func BenchmarkLeafConflictAcrossVersion(b *testing.B) {
 		},
 	}
 
+	// Make sure this passes...
+	if err := test.Test(leafFieldsParser); err != nil {
+		b.Fatal(err)
+	}
+
 	b.ReportAllocs()
+	b.ResetTimer()
 	for n := 0; n < b.N; n++ {
 		if err := test.Test(leafFieldsParser); err != nil {
 			b.Fatal(err)

merge/multiple_appliers_test.go

@@ -1030,6 +1030,82 @@ func (r repeatingConverter) IsMissingVersionError(err error) bool {
 func BenchmarkMultipleApplierRecursiveRealConversion(b *testing.B) {
 	test := TestCase{
 		Ops: []Operation{
+			/*
+				Apply{
+					Manager: "startwide",
+					Object: `
+						mapOfMapsRecursive:
+						  x0:
+						  x1:
+						  x2:
+						  x3:
+						  x4:
+						  x5:
+						  x6:
+						  x7:
+						  x8:
+						  x9:
+						  x10:
+						  x11:
+						  x12:
+						  x13:
+						  x14:
+						  x15:
+						  x16:
+						  x17:
+						  x18:
+						  x19:
+						  x20:
+						  x21:
+						  x22:
+						  x23:
+						  x24:
+						  x25:
+						  x26:
+						  x27:
+						  x28:
+						  x29:
+					`,
+					APIVersion: "v1",
+				},
+				Update{
+					Manager: "startdeep",
+					Object: `
+						mapOfMapsRecursive:
+						  y0:
+						    x1:
+						      x2:
+						        x3:
+						          x4:
+						            x5:
+						              x6:
+						                x7:
+						                  x8:
+						                    x9:
+						                      x10:
+						                        x11:
+						                          x12:
+						                            x13:
+						                              x14:
+						                                x15:
+						                                  x16:
+						                                    x17:
+						                                      x18:
+						                                        x19:
+						                                          x20:
+						                                            x21:
+						                                              x22:
+						                                                x23:
+						                                                  x24:
+						                                                    x25:
+						                                                      x26:
+						                                                        x27:
+						                                                          x28:
+						                                                            x29:
+					`,
+					APIVersion: "v1",
+				},
+			*/
 			Apply{
 				Manager: "apply-one",
 				Object: `
@@ -1103,9 +1179,15 @@ func BenchmarkMultipleApplierRecursiveRealConversion(b *testing.B) {
 		},
 	}
 
+	// Make sure this passes...
+	if err := test.TestWithConverter(nestedTypeParser, repeatingConverter{nestedTypeParser}); err != nil {
+		b.Fatal(err)
+	}
+
 	b.ReportAllocs()
+	b.ResetTimer()
 	for n := 0; n < b.N; n++ {
-		if err := test.TestWithConverter(nestedTypeParser, repeatingConverter{nestedTypeParser}); err != nil {
+		if err := test.BenchWithConverter(nestedTypeParser, repeatingConverter{nestedTypeParser}); err != nil {
 			b.Fatal(err)
 		}
 	}