Merge pull request #365 from pohly/ktesting-timestamps

ktesting improvements

Author: serathius

internal/buffer/buffer.go

@@ -110,7 +110,8 @@ func (buf *Buffer) someDigits(i, d int) int {
 	return copy(buf.Tmp[i:], buf.Tmp[j:])
 }
 
-// FormatHeader formats a log header using the provided file name and line number.
+// FormatHeader formats a log header using the provided file name and line number
+// and writes it into the buffer.
 func (buf *Buffer) FormatHeader(s severity.Severity, file string, line int, now time.Time) {
 	if line < 0 {
 		line = 0 // not a real line number, but acceptable to someDigits
@@ -146,3 +147,30 @@ func (buf *Buffer) FormatHeader(s severity.Severity, file string, line int, now
 	buf.Tmp[n+2] = ' '
 	buf.Write(buf.Tmp[:n+3])
 }
+
+// SprintHeader formats a log header and returns a string. This is a simpler
+// version of FormatHeader for use in ktesting.
+func (buf *Buffer) SprintHeader(s severity.Severity, now time.Time) string {
+	if s > severity.FatalLog {
+		s = severity.InfoLog // for safety.
+	}
+
+	// Avoid Fprintf, for speed. The format is so simple that we can do it quickly by hand.
+	// It's worth about 3X. Fprintf is hard.
+	_, month, day := now.Date()
+	hour, minute, second := now.Clock()
+	// Lmmdd hh:mm:ss.uuuuuu threadid file:line]
+	buf.Tmp[0] = severity.Char[s]
+	buf.twoDigits(1, int(month))
+	buf.twoDigits(3, day)
+	buf.Tmp[5] = ' '
+	buf.twoDigits(6, hour)
+	buf.Tmp[8] = ':'
+	buf.twoDigits(9, minute)
+	buf.Tmp[11] = ':'
+	buf.twoDigits(12, second)
+	buf.Tmp[14] = '.'
+	buf.nDigits(6, 15, now.Nanosecond()/1000, '0')
+	buf.Tmp[21] = ']'
+	return string(buf.Tmp[:22])
+}

internal/serialize/keyvalues.go

@@ -95,9 +95,15 @@ func MergeKVs(first, second []interface{}) []interface{} {
 	return merged
 }
 
+type Formatter struct {
+	AnyToStringHook AnyToStringFunc
+}
+
+type AnyToStringFunc func(v interface{}) string
+
 // MergeKVsInto is a variant of MergeKVs which directly formats the key/value
 // pairs into a buffer.
-func MergeAndFormatKVs(b *bytes.Buffer, first, second []interface{}) {
+func (f Formatter) MergeAndFormatKVs(b *bytes.Buffer, first, second []interface{}) {
 	if len(first) == 0 && len(second) == 0 {
 		// Nothing to do at all.
 		return
@@ -107,7 +113,7 @@ func MergeAndFormatKVs(b *bytes.Buffer, first, second []interface{}) {
 		// Nothing to be overridden, second slice is well-formed
 		// and can be used directly.
 		for i := 0; i < len(second); i += 2 {
-			KVFormat(b, second[i], second[i+1])
+			f.KVFormat(b, second[i], second[i+1])
 		}
 		return
 	}
@@ -127,24 +133,28 @@ func MergeAndFormatKVs(b *bytes.Buffer, first, second []interface{}) {
 		if overrides[key] {
 			continue
 		}
-		KVFormat(b, key, first[i+1])
+		f.KVFormat(b, key, first[i+1])
 	}
 	// Round down.
 	l := len(second)
 	l = l / 2 * 2
 	for i := 1; i < l; i += 2 {
-		KVFormat(b, second[i-1], second[i])
+		f.KVFormat(b, second[i-1], second[i])
 	}
 	if len(second)%2 == 1 {
-		KVFormat(b, second[len(second)-1], missingValue)
+		f.KVFormat(b, second[len(second)-1], missingValue)
 	}
 }
 
+func MergeAndFormatKVs(b *bytes.Buffer, first, second []interface{}) {
+	Formatter{}.MergeAndFormatKVs(b, first, second)
+}
+
 const missingValue = "(MISSING)"
 
 // KVListFormat serializes all key/value pairs into the provided buffer.
 // A space gets inserted before the first pair and between each pair.
-func KVListFormat(b *bytes.Buffer, keysAndValues ...interface{}) {
+func (f Formatter) KVListFormat(b *bytes.Buffer, keysAndValues ...interface{}) {
 	for i := 0; i < len(keysAndValues); i += 2 {
 		var v interface{}
 		k := keysAndValues[i]
@@ -153,13 +163,17 @@ func KVListFormat(b *bytes.Buffer, keysAndValues ...interface{}) {
 		} else {
 			v = missingValue
 		}
-		KVFormat(b, k, v)
+		f.KVFormat(b, k, v)
 	}
 }
 
+func KVListFormat(b *bytes.Buffer, keysAndValues ...interface{}) {
+	Formatter{}.KVListFormat(b, keysAndValues...)
+}
+
 // KVFormat serializes one key/value pair into the provided buffer.
 // A space gets inserted before the pair.
-func KVFormat(b *bytes.Buffer, k, v interface{}) {
+func (f Formatter) KVFormat(b *bytes.Buffer, k, v interface{}) {
 	b.WriteByte(' ')
 	// Keys are assumed to be well-formed according to
 	// https://github.com/kubernetes/community/blob/master/contributors/devel/sig-instrumentation/migration-to-structured-logging.md#name-arguments
@@ -203,7 +217,7 @@ func KVFormat(b *bytes.Buffer, k, v interface{}) {
 		case string:
 			writeStringValue(b, true, value)
 		default:
-			writeStringValue(b, false, fmt.Sprintf("%+v", value))
+			writeStringValue(b, false, f.AnyToString(value))
 		}
 	case []byte:
 		// In https://github.com/kubernetes/klog/pull/237 it was decided
@@ -220,8 +234,20 @@ func KVFormat(b *bytes.Buffer, k, v interface{}) {
 		b.WriteByte('=')
 		b.WriteString(fmt.Sprintf("%+q", v))
 	default:
-		writeStringValue(b, false, fmt.Sprintf("%+v", v))
+		writeStringValue(b, false, f.AnyToString(v))
+	}
+}
+
+func KVFormat(b *bytes.Buffer, k, v interface{}) {
+	Formatter{}.KVFormat(b, k, v)
+}
+
+// AnyToString is the historic fallback formatter.
+func (f Formatter) AnyToString(v interface{}) string {
+	if f.AnyToStringHook != nil {
+		return f.AnyToStringHook(v)
 	}
+	return fmt.Sprintf("%+v", v)
 }
 
 // StringerToString converts a Stringer to a string,

ktesting/contextual_test.go

@@ -16,7 +16,8 @@ import (
 )
 
 func TestContextual(t *testing.T) {
-	logger, ctx := ktesting.NewTestContext(t)
+	var buffer ktesting.BufferTL
+	logger, ctx := ktesting.NewTestContext(&buffer)
 
 	doSomething(ctx)
 
@@ -33,8 +34,14 @@ func TestContextual(t *testing.T) {
 	}
 
 	actual := testingLogger.GetBuffer().String()
-	expected := `INFO hello world
-INFO foo: hello also from me
+	if actual != "" {
+		t.Errorf("testinglogger should not have buffered, got:\n%s", actual)
+	}
+
+	actual = buffer.String()
+	actual = headerRe.ReplaceAllString(actual, "${1}xxx] ")
+	expected := `Ixxx] hello world
+Ixxx] foo: hello also from me
 `
 	if actual != expected {
 		t.Errorf("mismatch in captured output, expected:\n%s\ngot:\n%s\n", expected, actual)

ktesting/example_test.go

@@ -25,9 +25,17 @@ import (
 )
 
 func ExampleUnderlier() {
-	logger := ktesting.NewLogger(ktesting.NopTL{}, ktesting.NewConfig(ktesting.Verbosity(4)))
+	logger := ktesting.NewLogger(ktesting.NopTL{},
+		ktesting.NewConfig(
+			ktesting.Verbosity(4),
+			ktesting.BufferLogs(true),
+			ktesting.AnyToString(func(value interface{}) string {
+				return fmt.Sprintf("### %+v ###", value)
+			}),
+		),
+	)
 
-	logger.Error(errors.New("failure"), "I failed", "what", "something")
+	logger.Error(errors.New("failure"), "I failed", "what", "something", "data", struct{ field int }{field: 1})
 	logger.WithValues("request", 42).WithValues("anotherValue", "fish").Info("hello world")
 	logger.WithValues("request", 42, "anotherValue", "fish").Info("hello world 2", "yetAnotherValue", "thanks")
 	logger.WithName("example").Info("with name")
@@ -57,15 +65,36 @@ func ExampleUnderlier() {
 	}
 
 	// Output:
-	// ERROR I failed err="failure" what="something"
-	// INFO hello world request=42 anotherValue="fish"
-	// INFO hello world 2 request=42 anotherValue="fish" yetAnotherValue="thanks"
+	// ERROR I failed err="failure" what="something" data=### {field:1} ###
+	// INFO hello world request=### 42 ### anotherValue="fish"
+	// INFO hello world 2 request=### 42 ### anotherValue="fish" yetAnotherValue="thanks"
 	// INFO example: with name
 	// INFO higher verbosity
 	//
-	// log entry #0: {Timestamp:0001-01-01 00:00:00 +0000 UTC Type:ERROR Prefix: Message:I failed Verbosity:0 Err:failure WithKVList:[] ParameterKVList:[what something]}
+	// log entry #0: {Timestamp:0001-01-01 00:00:00 +0000 UTC Type:ERROR Prefix: Message:I failed Verbosity:0 Err:failure WithKVList:[] ParameterKVList:[what something data {field:1}]}
 	// log entry #1: {Timestamp:0001-01-01 00:00:00 +0000 UTC Type:INFO Prefix: Message:hello world Verbosity:0 Err:<nil> WithKVList:[request 42 anotherValue fish] ParameterKVList:[]}
 	// log entry #2: {Timestamp:0001-01-01 00:00:00 +0000 UTC Type:INFO Prefix: Message:hello world 2 Verbosity:0 Err:<nil> WithKVList:[request 42 anotherValue fish] ParameterKVList:[yetAnotherValue thanks]}
 	// log entry #3: {Timestamp:0001-01-01 00:00:00 +0000 UTC Type:INFO Prefix:example Message:with name Verbosity:0 Err:<nil> WithKVList:[] ParameterKVList:[]}
 	// log entry #4: {Timestamp:0001-01-01 00:00:00 +0000 UTC Type:INFO Prefix: Message:higher verbosity Verbosity:4 Err:<nil> WithKVList:[] ParameterKVList:[]}
 }
+
+func ExampleNewLogger() {
+	var buffer ktesting.BufferTL
+	logger := ktesting.NewLogger(&buffer, ktesting.NewConfig())
+
+	logger.Error(errors.New("failure"), "I failed", "what", "something", "data", struct{ field int }{field: 1})
+	logger.V(5).Info("Logged at level 5.")
+	logger.V(6).Info("Not logged at level 6.")
+
+	testingLogger, ok := logger.GetSink().(ktesting.Underlier)
+	if !ok {
+		panic("Should have had a ktesting LogSink!?")
+	}
+	fmt.Printf(">> %s <<\n", testingLogger.GetBuffer().String())       // Should be empty.
+	fmt.Print(headerRe.ReplaceAllString(buffer.String(), "${1}...] ")) // Should not be empty.
+
+	// Output:
+	// >>  <<
+	// E...] I failed err="failure" what="something" data={field:1}
+	// I...] Logged at level 5.
+}

ktesting/options.go

@@ -20,6 +20,7 @@ import (
 	"flag"
 	"strconv"
 
+	"k8s.io/klog/v2/internal/serialize"
 	"k8s.io/klog/v2/internal/verbosity"
 )
 
@@ -47,9 +48,25 @@ type Config struct {
 type ConfigOption func(co *configOptions)
 
 type configOptions struct {
+	anyToString       serialize.AnyToStringFunc
 	verbosityFlagName string
 	vmoduleFlagName   string
 	verbosityDefault  int
+	bufferLogs        bool
+}
+
+// AnyToString overrides the default formatter for values that are not
+// supported directly by klog. The default is `fmt.Sprintf("%+v")`.
+// The formatter must not panic.
+//
+// # Experimental
+//
+// Notice: This function is EXPERIMENTAL and may be changed or removed in a
+// later release.
+func AnyToString(anyToString func(value interface{}) string) ConfigOption {
+	return func(co *configOptions) {
+		co.anyToString = anyToString
+	}
 }
 
 // VerbosityFlagName overrides the default -testing.v for the verbosity level.
@@ -94,6 +111,21 @@ func Verbosity(level int) ConfigOption {
 	}
 }
 
+// BufferLogs controls whether log entries are captured in memory in addition
+// to being printed. Off by default. Unit tests that want to verify that
+// log entries are emitted as expected can turn this on and then retrieve
+// the captured log through the Underlier LogSink interface.
+//
+// # Experimental
+//
+// Notice: This function is EXPERIMENTAL and may be changed or removed in a
+// later release.
+func BufferLogs(enabled bool) ConfigOption {
+	return func(co *configOptions) {
+		co.bufferLogs = enabled
+	}
+}
+
 // NewConfig returns a configuration with recommended defaults and optional
 // modifications. Command line flags are not bound to any FlagSet yet.
 //