diff --git a/connection/connection.go b/connection/connection.go index 1d8c6d1e..b87b1b40 100644 --- a/connection/connection.go +++ b/connection/connection.go @@ -109,7 +109,7 @@ func connect( grpc.WithBlock(), // Block until connection succeeds. grpc.WithChainUnaryInterceptor( LogGRPC, // Log all messages. - extendedCSIMetricsManager{metricsManager}.recordMetricsInterceptor, // Record metrics for each gRPC call. + ExtendedCSIMetricsManager{metricsManager}.RecordMetricsClientInterceptor, // Record metrics for each gRPC call. ), ) unixPrefix := "unix://" @@ -187,12 +187,13 @@ func LogGRPC(ctx context.Context, method string, req, reply interface{}, cc *grp return err } -type extendedCSIMetricsManager struct { +type ExtendedCSIMetricsManager struct { metrics.CSIMetricsManager } -// recordMetricsInterceptor is a gPRC unary interceptor for recording metrics for CSI operations. -func (cmm extendedCSIMetricsManager) recordMetricsInterceptor( +// RecordMetricsClientInterceptor is a gPRC unary interceptor for recording metrics for CSI operations +// in a gRPC client. +func (cmm ExtendedCSIMetricsManager) RecordMetricsClientInterceptor( ctx context.Context, method string, req, reply interface{}, @@ -209,3 +210,17 @@ func (cmm extendedCSIMetricsManager) recordMetricsInterceptor( ) return err } + +// RecordMetricsServerInterceptor is a gPRC unary interceptor for recording metrics for CSI operations +// in a gRCP server. +func (cmm ExtendedCSIMetricsManager) RecordMetricsServerInterceptor(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { + start := time.Now() + resp, err := handler(ctx, req) + duration := time.Since(start) + cmm.RecordMetrics( + info.FullMethod, /* operationName */ + err, /* operationErr */ + duration, /* operationDuration */ + ) + return resp, err +} diff --git a/connection/connection_test.go b/connection/connection_test.go index f03dad13..8c88bd0c 100644 --- a/connection/connection_test.go +++ b/connection/connection_test.go @@ -51,14 +51,34 @@ const ( serverSock = "server.sock" ) +type identityServer struct{} + +func (ids *identityServer) GetPluginInfo(ctx context.Context, req *csi.GetPluginInfoRequest) (*csi.GetPluginInfoResponse, error) { + return nil, status.Error(codes.Unimplemented, "Unimplemented") +} + +func (ids *identityServer) Probe(ctx context.Context, req *csi.ProbeRequest) (*csi.ProbeResponse, error) { + return nil, status.Error(codes.Unimplemented, "Unimplemented") +} + +func (ids *identityServer) GetPluginCapabilities(ctx context.Context, req *csi.GetPluginCapabilitiesRequest) (*csi.GetPluginCapabilitiesResponse, error) { + return nil, status.Error(codes.Unimplemented, "Unimplemented") +} + // startServer creates a gRPC server without any registered services. // The returned address can be used to connect to it. The cleanup // function stops it. It can be called multiple times. -func startServer(t *testing.T, tmp string, identity csi.IdentityServer, controller csi.ControllerServer) (string, func()) { +func startServer(t *testing.T, tmp string, identity csi.IdentityServer, controller csi.ControllerServer, cmm metrics.CSIMetricsManager) (string, func()) { addr := path.Join(tmp, serverSock) listener, err := net.Listen("unix", addr) require.NoError(t, err, "listening on %s", addr) - server := grpc.NewServer() + var opts []grpc.ServerOption + if cmm != nil { + opts = append(opts, + grpc.UnaryInterceptor(ExtendedCSIMetricsManager{cmm}.RecordMetricsServerInterceptor), + ) + } + server := grpc.NewServer(opts...) if identity != nil { csi.RegisterIdentityServer(server, identity) } @@ -85,7 +105,7 @@ func startServer(t *testing.T, tmp string, identity csi.IdentityServer, controll func TestConnect(t *testing.T) { tmp := tmpDir(t) defer os.RemoveAll(tmp) - addr, stopServer := startServer(t, tmp, nil, nil) + addr, stopServer := startServer(t, tmp, nil, nil, nil) defer stopServer() conn, err := Connect(addr, metrics.NewCSIMetricsManager("fake.csi.driver.io")) @@ -100,7 +120,7 @@ func TestConnect(t *testing.T) { func TestConnectUnix(t *testing.T) { tmp := tmpDir(t) defer os.RemoveAll(tmp) - addr, stopServer := startServer(t, tmp, nil, nil) + addr, stopServer := startServer(t, tmp, nil, nil, nil) defer stopServer() conn, err := Connect("unix:///"+addr, metrics.NewCSIMetricsManager("fake.csi.driver.io")) @@ -141,7 +161,7 @@ func TestWaitForServer(t *testing.T) { t.Logf("sleeping %s before starting server", delay) time.Sleep(delay) startTimeServer = time.Now() - _, stopServer = startServer(t, tmp, nil, nil) + _, stopServer = startServer(t, tmp, nil, nil, nil) }() conn, err := Connect(path.Join(tmp, serverSock), metrics.NewCSIMetricsManager("fake.csi.driver.io")) if assert.NoError(t, err, "connect via absolute path") { @@ -175,7 +195,7 @@ func TestTimout(t *testing.T) { func TestReconnect(t *testing.T) { tmp := tmpDir(t) defer os.RemoveAll(tmp) - addr, stopServer := startServer(t, tmp, nil, nil) + addr, stopServer := startServer(t, tmp, nil, nil, nil) defer func() { stopServer() }() @@ -202,7 +222,7 @@ func TestReconnect(t *testing.T) { } // No reconnection either when the server comes back. - _, stopServer = startServer(t, tmp, nil, nil) + _, stopServer = startServer(t, tmp, nil, nil, nil) // We need to give gRPC some time. It does not attempt to reconnect // immediately. If we send the method call too soon, the test passes // even though a later method call will go through again. @@ -220,7 +240,7 @@ func TestReconnect(t *testing.T) { func TestDisconnect(t *testing.T) { tmp := tmpDir(t) defer os.RemoveAll(tmp) - addr, stopServer := startServer(t, tmp, nil, nil) + addr, stopServer := startServer(t, tmp, nil, nil, nil) defer func() { stopServer() }() @@ -251,7 +271,7 @@ func TestDisconnect(t *testing.T) { } // No reconnection either when the server comes back. - _, stopServer = startServer(t, tmp, nil, nil) + _, stopServer = startServer(t, tmp, nil, nil, nil) // We need to give gRPC some time. It does not attempt to reconnect // immediately. If we send the method call too soon, the test passes // even though a later method call will go through again. @@ -271,7 +291,7 @@ func TestDisconnect(t *testing.T) { func TestExplicitReconnect(t *testing.T) { tmp := tmpDir(t) defer os.RemoveAll(tmp) - addr, stopServer := startServer(t, tmp, nil, nil) + addr, stopServer := startServer(t, tmp, nil, nil, nil) defer func() { stopServer() }() @@ -302,7 +322,7 @@ func TestExplicitReconnect(t *testing.T) { } // No reconnection either when the server comes back. - _, stopServer = startServer(t, tmp, nil, nil) + _, stopServer = startServer(t, tmp, nil, nil, nil) // We need to give gRPC some time. It does not attempt to reconnect // immediately. If we send the method call too soon, the test passes // even though a later method call will go through again. @@ -322,7 +342,10 @@ func TestExplicitReconnect(t *testing.T) { func TestConnectMetrics(t *testing.T) { tmp := tmpDir(t) defer os.RemoveAll(tmp) - addr, stopServer := startServer(t, tmp, nil, nil) + cmmServer := metrics.NewCSIMetricsManagerForPlugin("fake.csi.driver.io") + // We have to have a real implementation of the gRPC call, otherwise the metrics + // interceptor is not called. The CSI identity service is used because it's simple. + addr, stopServer := startServer(t, tmp, &identityServer{}, nil, cmmServer) defer stopServer() cmm := metrics.NewCSIMetricsManager("fake.csi.driver.io") @@ -332,7 +355,8 @@ func TestConnectMetrics(t *testing.T) { defer conn.Close() assert.Equal(t, connectivity.Ready, conn.GetState(), "connection ready") - if err := conn.Invoke(context.Background(), "/csi.v1.Controller/ControllerGetCapabilities", nil, nil); assert.Error(t, err) { + identityClient := csi.NewIdentityClient(conn) + if _, err := identityClient.GetPluginInfo(context.Background(), &csi.GetPluginInfoRequest{}); assert.Error(t, err) { errStatus, _ := status.FromError(err) assert.Equal(t, codes.Unimplemented, errStatus.Code(), "not implemented") } @@ -340,22 +364,22 @@ func TestConnectMetrics(t *testing.T) { expectedMetrics := `# HELP csi_sidecar_operations_seconds [ALPHA] Container Storage Interface operation duration with gRPC error code status total # TYPE csi_sidecar_operations_seconds histogram - csi_sidecar_operations_seconds_bucket{driver_name="fake.csi.driver.io",grpc_status_code="Unimplemented",method_name="/csi.v1.Controller/ControllerGetCapabilities",le="0.1"} 1 - csi_sidecar_operations_seconds_bucket{driver_name="fake.csi.driver.io",grpc_status_code="Unimplemented",method_name="/csi.v1.Controller/ControllerGetCapabilities",le="0.25"} 1 - csi_sidecar_operations_seconds_bucket{driver_name="fake.csi.driver.io",grpc_status_code="Unimplemented",method_name="/csi.v1.Controller/ControllerGetCapabilities",le="0.5"} 1 - csi_sidecar_operations_seconds_bucket{driver_name="fake.csi.driver.io",grpc_status_code="Unimplemented",method_name="/csi.v1.Controller/ControllerGetCapabilities",le="1"} 1 - csi_sidecar_operations_seconds_bucket{driver_name="fake.csi.driver.io",grpc_status_code="Unimplemented",method_name="/csi.v1.Controller/ControllerGetCapabilities",le="2.5"} 1 - csi_sidecar_operations_seconds_bucket{driver_name="fake.csi.driver.io",grpc_status_code="Unimplemented",method_name="/csi.v1.Controller/ControllerGetCapabilities",le="5"} 1 - csi_sidecar_operations_seconds_bucket{driver_name="fake.csi.driver.io",grpc_status_code="Unimplemented",method_name="/csi.v1.Controller/ControllerGetCapabilities",le="10"} 1 - csi_sidecar_operations_seconds_bucket{driver_name="fake.csi.driver.io",grpc_status_code="Unimplemented",method_name="/csi.v1.Controller/ControllerGetCapabilities",le="15"} 1 - csi_sidecar_operations_seconds_bucket{driver_name="fake.csi.driver.io",grpc_status_code="Unimplemented",method_name="/csi.v1.Controller/ControllerGetCapabilities",le="25"} 1 - csi_sidecar_operations_seconds_bucket{driver_name="fake.csi.driver.io",grpc_status_code="Unimplemented",method_name="/csi.v1.Controller/ControllerGetCapabilities",le="50"} 1 - csi_sidecar_operations_seconds_bucket{driver_name="fake.csi.driver.io",grpc_status_code="Unimplemented",method_name="/csi.v1.Controller/ControllerGetCapabilities",le="120"} 1 - csi_sidecar_operations_seconds_bucket{driver_name="fake.csi.driver.io",grpc_status_code="Unimplemented",method_name="/csi.v1.Controller/ControllerGetCapabilities",le="300"} 1 - csi_sidecar_operations_seconds_bucket{driver_name="fake.csi.driver.io",grpc_status_code="Unimplemented",method_name="/csi.v1.Controller/ControllerGetCapabilities",le="600"} 1 - csi_sidecar_operations_seconds_bucket{driver_name="fake.csi.driver.io",grpc_status_code="Unimplemented",method_name="/csi.v1.Controller/ControllerGetCapabilities",le="+Inf"} 1 - csi_sidecar_operations_seconds_sum{driver_name="fake.csi.driver.io",grpc_status_code="Unimplemented",method_name="/csi.v1.Controller/ControllerGetCapabilities"} 0 - csi_sidecar_operations_seconds_count{driver_name="fake.csi.driver.io",grpc_status_code="Unimplemented",method_name="/csi.v1.Controller/ControllerGetCapabilities"} 1 + csi_sidecar_operations_seconds_bucket{driver_name="fake.csi.driver.io",grpc_status_code="Unimplemented",method_name="/csi.v1.Identity/GetPluginInfo",le="0.1"} 1 + csi_sidecar_operations_seconds_bucket{driver_name="fake.csi.driver.io",grpc_status_code="Unimplemented",method_name="/csi.v1.Identity/GetPluginInfo",le="0.25"} 1 + csi_sidecar_operations_seconds_bucket{driver_name="fake.csi.driver.io",grpc_status_code="Unimplemented",method_name="/csi.v1.Identity/GetPluginInfo",le="0.5"} 1 + csi_sidecar_operations_seconds_bucket{driver_name="fake.csi.driver.io",grpc_status_code="Unimplemented",method_name="/csi.v1.Identity/GetPluginInfo",le="1"} 1 + csi_sidecar_operations_seconds_bucket{driver_name="fake.csi.driver.io",grpc_status_code="Unimplemented",method_name="/csi.v1.Identity/GetPluginInfo",le="2.5"} 1 + csi_sidecar_operations_seconds_bucket{driver_name="fake.csi.driver.io",grpc_status_code="Unimplemented",method_name="/csi.v1.Identity/GetPluginInfo",le="5"} 1 + csi_sidecar_operations_seconds_bucket{driver_name="fake.csi.driver.io",grpc_status_code="Unimplemented",method_name="/csi.v1.Identity/GetPluginInfo",le="10"} 1 + csi_sidecar_operations_seconds_bucket{driver_name="fake.csi.driver.io",grpc_status_code="Unimplemented",method_name="/csi.v1.Identity/GetPluginInfo",le="15"} 1 + csi_sidecar_operations_seconds_bucket{driver_name="fake.csi.driver.io",grpc_status_code="Unimplemented",method_name="/csi.v1.Identity/GetPluginInfo",le="25"} 1 + csi_sidecar_operations_seconds_bucket{driver_name="fake.csi.driver.io",grpc_status_code="Unimplemented",method_name="/csi.v1.Identity/GetPluginInfo",le="50"} 1 + csi_sidecar_operations_seconds_bucket{driver_name="fake.csi.driver.io",grpc_status_code="Unimplemented",method_name="/csi.v1.Identity/GetPluginInfo",le="120"} 1 + csi_sidecar_operations_seconds_bucket{driver_name="fake.csi.driver.io",grpc_status_code="Unimplemented",method_name="/csi.v1.Identity/GetPluginInfo",le="300"} 1 + csi_sidecar_operations_seconds_bucket{driver_name="fake.csi.driver.io",grpc_status_code="Unimplemented",method_name="/csi.v1.Identity/GetPluginInfo",le="600"} 1 + csi_sidecar_operations_seconds_bucket{driver_name="fake.csi.driver.io",grpc_status_code="Unimplemented",method_name="/csi.v1.Identity/GetPluginInfo",le="+Inf"} 1 + csi_sidecar_operations_seconds_sum{driver_name="fake.csi.driver.io",grpc_status_code="Unimplemented",method_name="/csi.v1.Identity/GetPluginInfo"} 0 + csi_sidecar_operations_seconds_count{driver_name="fake.csi.driver.io",grpc_status_code="Unimplemented",method_name="/csi.v1.Identity/GetPluginInfo"} 1 ` if err := testutil.GatherAndCompare( @@ -363,7 +387,17 @@ func TestConnectMetrics(t *testing.T) { // Ignore mismatches on csi_sidecar_operations_seconds_sum metric because execution time will vary from test to test. err = verifyMetricsError(t, err, "csi_sidecar_operations_seconds_sum") if err != nil { - t.Errorf("Expected metrics not found -- %v", err) + t.Errorf("Expected client metrics not found -- %v", err) + } + } + + expectedMetrics = strings.Replace(expectedMetrics, "csi_sidecar", metrics.SubsystemPlugin, -1) + if err := testutil.GatherAndCompare( + cmmServer.GetRegistry(), strings.NewReader(expectedMetrics)); err != nil { + // Ignore mismatches on csi_sidecar_operations_seconds_sum metric because execution time will vary from test to test. + err = verifyMetricsError(t, err, metrics.SubsystemPlugin+"_operations_seconds_sum") + if err != nil { + t.Errorf("Expected server metrics not found -- %v", err) } } } diff --git a/metrics/metrics.go b/metrics/metrics.go index 2e252d65..efa4b228 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -20,6 +20,7 @@ import ( "bufio" "fmt" "net/http" + "sort" "strings" "time" @@ -30,8 +31,18 @@ import ( ) const ( + // SubsystemSidecar is the default subsystem name in a metrics + // (= the prefix in the final metrics name). It is to be used + // by CSI sidecars. Using the same subsystem in different CSI + // drivers makes it possible to reuse dashboards because + // the metrics names will be identical. Data from different + // drivers can be selected via the "driver_name" tag. + SubsystemSidecar = "csi_sidecar" + // SubsystemPlugin is what CSI driver's should use as + // subsystem name. + SubsystemPlugin = "csi_plugin" + // Common metric strings - subsystem = "csi_sidecar" labelCSIDriverName = "driver_name" labelCSIOperationName = "method_name" labelGrpcStatusCode = "grpc_status_code" @@ -56,11 +67,23 @@ type CSIMetricsManager interface { // operationName - Name of the CSI operation. // operationErr - Error, if any, that resulted from execution of operation. // operationDuration - time it took for the operation to complete + // + // If WithLabelNames was used to define additional labels when constructing + // the manager, then WithLabelValues should be used to create a wrapper which + // holds the corresponding values before calling RecordMetrics of the wrapper. + // Labels with missing values are recorded as empty. RecordMetrics( operationName string, operationErr error, operationDuration time.Duration) + // WithLabelValues must be used to add the additional label + // values defined via WithLabelNames. When calling RecordMetrics + // without it or with too few values, the missing values are + // recorded as empty. WithLabelValues can be called multiple times + // and then accumulates values. + WithLabelValues(labels map[string]string) (CSIMetricsManager, error) + // SetDriverName is called to update the CSI driver name. This should be done // as soon as possible, otherwise metrics recorded by this manager will be // recorded with an "unknown-driver" driver_name. @@ -73,25 +96,109 @@ type CSIMetricsManager interface { StartMetricsEndpoint(metricsAddress, metricsPath string) } -// NewCSIMetricsManager creates and registers metrics for for CSI Sidecars and -// returns an object that can be used to trigger the metrics. +// MetricsManagerOption is used to pass optional configuration to a +// new metrics manager. +type MetricsManagerOption func(*csiMetricsManager) + +// WithSubsystem overrides the default subsystem name. +func WithSubsystem(subsystem string) MetricsManagerOption { + return func(cmm *csiMetricsManager) { + cmm.subsystem = subsystem + } +} + +// WithStabilityLevel overrides the default stability level. The recommended +// usage is to keep metrics at a lower level when csi-lib-utils switches +// to beta or GA. Overriding the alpha default with beta or GA is risky +// because the metrics can still change in the library. +func WithStabilityLevel(stabilityLevel metrics.StabilityLevel) MetricsManagerOption { + return func(cmm *csiMetricsManager) { + cmm.stabilityLevel = stabilityLevel + } +} + +// WithLabelNames defines labels for each sample that get added to the +// default labels (driver, method call, and gRPC result). This makes +// it possible to partition the histograms along additional +// dimensions. +// +// To record a metrics with additional values, use +// CSIMetricManager.WithLabelValues().RecordMetrics(). +func WithLabelNames(labelNames ...string) MetricsManagerOption { + return func(cmm *csiMetricsManager) { + cmm.additionalLabelNames = labelNames + } +} + +// WithLabels defines some label name and value pairs that are added to all +// samples. They get recorded sorted by name. +func WithLabels(labels map[string]string) MetricsManagerOption { + return func(cmm *csiMetricsManager) { + var l []label + for name, value := range labels { + l = append(l, label{name, value}) + } + sort.Slice(l, func(i, j int) bool { + return l[i].name < l[j].name + }) + cmm.additionalLabels = l + } +} + +// NewCSIMetricsManagerForSidecar creates and registers metrics for CSI Sidecars and +// returns an object that can be used to trigger the metrics. It uses "csi_sidecar" +// as subsystem. +// +// driverName - Name of the CSI driver against which this operation was executed. +// If unknown, leave empty, and use SetDriverName method to update later. +func NewCSIMetricsManagerForSidecar(driverName string) CSIMetricsManager { + return NewCSIMetricsManagerWithOptions(driverName) +} + +// NewCSIMetricsManager is provided for backwards-compatibility. +var NewCSIMetricsManager = NewCSIMetricsManagerForSidecar + +// NewCSIMetricsManagerForPlugin creates and registers metrics for CSI drivers and +// returns an object that can be used to trigger the metrics. It uses "csi_plugin" +// as subsystem. +// +// driverName - Name of the CSI driver against which this operation was executed. +// If unknown, leave empty, and use SetDriverName method to update later. +func NewCSIMetricsManagerForPlugin(driverName string) CSIMetricsManager { + return NewCSIMetricsManagerWithOptions(driverName, + WithSubsystem(SubsystemPlugin), + ) +} + +// NewCSIMetricsManagerWithOptions is a customizable constructor, to be used only +// if there are special needs like changing the default subsystems. +// // driverName - Name of the CSI driver against which this operation was executed. // If unknown, leave empty, and use SetDriverName method to update later. -func NewCSIMetricsManager(driverName string) CSIMetricsManager { +func NewCSIMetricsManagerWithOptions(driverName string, options ...MetricsManagerOption) CSIMetricsManager { cmm := csiMetricsManager{ - registry: metrics.NewKubeRegistry(), - csiOperationsLatencyMetric: metrics.NewHistogramVec( - &metrics.HistogramOpts{ - Subsystem: subsystem, - Name: operationsLatencyMetricName, - Help: operationsLatencyHelp, - Buckets: operationsLatencyBuckets, - StabilityLevel: metrics.ALPHA, - }, - []string{labelCSIDriverName, labelCSIOperationName, labelGrpcStatusCode}, - ), + registry: metrics.NewKubeRegistry(), + subsystem: SubsystemSidecar, + stabilityLevel: metrics.ALPHA, } - + for _, option := range options { + option(&cmm) + } + labels := []string{labelCSIDriverName, labelCSIOperationName, labelGrpcStatusCode} + labels = append(labels, cmm.additionalLabelNames...) + for _, label := range cmm.additionalLabels { + labels = append(labels, label.name) + } + cmm.csiOperationsLatencyMetric = metrics.NewHistogramVec( + &metrics.HistogramOpts{ + Subsystem: cmm.subsystem, + Name: operationsLatencyMetricName, + Help: operationsLatencyHelp, + Buckets: operationsLatencyBuckets, + StabilityLevel: cmm.stabilityLevel, + }, + labels, + ) cmm.SetDriverName(driverName) cmm.registerMetrics() return &cmm @@ -101,26 +208,105 @@ var _ CSIMetricsManager = &csiMetricsManager{} type csiMetricsManager struct { registry metrics.KubeRegistry + subsystem string + stabilityLevel metrics.StabilityLevel driverName string - csiOperationsMetric *metrics.CounterVec + additionalLabelNames []string + additionalLabels []label csiOperationsLatencyMetric *metrics.HistogramVec } +type label struct { + name, value string +} + func (cmm *csiMetricsManager) GetRegistry() metrics.KubeRegistry { return cmm.registry } -// RecordMetrics must be called upon CSI Operation completion to record -// the operation's metric. -// operationName - Name of the CSI operation. -// operationErr - Error, if any, that resulted from execution of operation. -// operationDuration - time it took for the operation to complete +// RecordMetrics implements CSIMetricsManager.RecordMetrics. func (cmm *csiMetricsManager) RecordMetrics( operationName string, operationErr error, operationDuration time.Duration) { - cmm.csiOperationsLatencyMetric.WithLabelValues( - cmm.driverName, operationName, getErrorCode(operationErr)).Observe(operationDuration.Seconds()) + cmm.recordMetricsWithLabels(operationName, operationErr, operationDuration, nil) +} + +// recordMetricsWithLabels is the internal implementation of RecordMetrics. +func (cmm *csiMetricsManager) recordMetricsWithLabels( + operationName string, + operationErr error, + operationDuration time.Duration, + labelValues map[string]string) { + values := []string{cmm.driverName, operationName, getErrorCode(operationErr)} + for _, name := range cmm.additionalLabelNames { + values = append(values, labelValues[name]) + } + for _, label := range cmm.additionalLabels { + values = append(values, label.value) + } + cmm.csiOperationsLatencyMetric.WithLabelValues(values...).Observe(operationDuration.Seconds()) +} + +type csiMetricsManagerWithValues struct { + *csiMetricsManager + + // additionalValues holds the values passed via WithLabelValues. + additionalValues map[string]string +} + +// WithLabelValues in the base metrics manager creates a fresh wrapper with no labels and let's +// that deal with adding the label values. +func (cmm *csiMetricsManager) WithLabelValues(labels map[string]string) (CSIMetricsManager, error) { + cmmv := &csiMetricsManagerWithValues{ + csiMetricsManager: cmm, + additionalValues: map[string]string{}, + } + return cmmv.WithLabelValues(labels) +} + +// WithLabelValues in the wrapper creates a wrapper which has all existing labels and +// adds the new ones, with error checking. Can be called multiple times. Each call then +// can add some new value(s). It is an error to overwrite an already set value. +// If RecordMetrics is called before setting all additional values, the missing ones will +// be empty. +func (cmmv *csiMetricsManagerWithValues) WithLabelValues(labels map[string]string) (CSIMetricsManager, error) { + extended := &csiMetricsManagerWithValues{ + csiMetricsManager: cmmv.csiMetricsManager, + additionalValues: map[string]string{}, + } + // We need to copy the old values to avoid modifying the map in cmmv. + for name, value := range cmmv.additionalValues { + extended.additionalValues[name] = value + } + // Now add all new values. + for name, value := range labels { + if !extended.haveAdditionalLabel(name) { + return nil, fmt.Errorf("label %q was not defined via WithLabelNames", name) + } + if v, ok := extended.additionalValues[name]; ok { + return nil, fmt.Errorf("label %q already has value %q", name, v) + } + extended.additionalValues[name] = value + } + return extended, nil +} + +func (cmm *csiMetricsManager) haveAdditionalLabel(name string) bool { + for _, n := range cmm.additionalLabelNames { + if n == name { + return true + } + } + return false +} + +// RecordMetrics passes the stored values as to the implementation. +func (cmmv *csiMetricsManagerWithValues) RecordMetrics( + operationName string, + operationErr error, + operationDuration time.Duration) { + cmmv.recordMetricsWithLabels(operationName, operationErr, operationDuration, cmmv.additionalValues) } // SetDriverName is called to update the CSI driver name. This should be done diff --git a/metrics/metrics_test.go b/metrics/metrics_test.go index 7d210837..7136f66a 100644 --- a/metrics/metrics_test.go +++ b/metrics/metrics_test.go @@ -25,13 +25,55 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + "k8s.io/component-base/metrics" "k8s.io/component-base/metrics/testutil" ) func TestRecordMetrics(t *testing.T) { + testcases := map[string]struct { + subsystem string + stabilityLevel metrics.StabilityLevel + }{ + "default": {}, + "sidecar": {subsystem: SubsystemSidecar}, + "driver": {subsystem: SubsystemPlugin}, + "other": {subsystem: "other"}, + "stable": {stabilityLevel: metrics.STABLE}, + "alpha": {stabilityLevel: metrics.ALPHA}, + } + + for name, tc := range testcases { + t.Run(name, func(t *testing.T) { + testRecordMetrics(t, tc.subsystem, tc.stabilityLevel) + }) + } +} + +func testRecordMetrics(t *testing.T, subsystem string, stabilityLevel metrics.StabilityLevel) { // Arrange - cmm := NewCSIMetricsManager( - "fake.csi.driver.io" /* driverName */) + var cmm CSIMetricsManager + driverName := "fake.csi.driver.io" + if stabilityLevel == "" { + // Cover the two dedicated calls. + switch subsystem { + case SubsystemSidecar: + cmm = NewCSIMetricsManagerForSidecar(driverName) + case SubsystemPlugin: + cmm = NewCSIMetricsManagerForPlugin(driverName) + } + } + if cmm == nil { + // The flexible construction is the fallback. + var options []MetricsManagerOption + if subsystem != "" { + options = append(options, WithSubsystem(subsystem)) + } + if stabilityLevel != "" { + options = append(options, WithStabilityLevel(stabilityLevel)) + } + cmm = NewCSIMetricsManagerWithOptions(driverName, options...) + } + operationDuration, _ := time.ParseDuration("20s") // Act @@ -60,6 +102,294 @@ func TestRecordMetrics(t *testing.T) { csi_sidecar_operations_seconds_sum{driver_name="fake.csi.driver.io",grpc_status_code="OK",method_name="/csi.v1.Controller/ControllerGetCapabilities"} 20 csi_sidecar_operations_seconds_count{driver_name="fake.csi.driver.io",grpc_status_code="OK",method_name="/csi.v1.Controller/ControllerGetCapabilities"} 1 ` + if subsystem != "" { + expectedMetrics = strings.Replace(expectedMetrics, "csi_sidecar", subsystem, -1) + } + if stabilityLevel != "" { + expectedMetrics = strings.Replace(expectedMetrics, "ALPHA", string(stabilityLevel), -1) + } + + if err := testutil.GatherAndCompare( + cmm.GetRegistry(), strings.NewReader(expectedMetrics)); err != nil { + t.Fatal(err) + } +} + +func TestFixedLabels(t *testing.T) { + // Arrange + cmm := NewCSIMetricsManagerWithOptions( + "", /* driverName */ + WithLabels(map[string]string{"a": "111", "b": "222"}), + ) + operationDuration, _ := time.ParseDuration("20s") + + // Act + cmm.RecordMetrics( + "myOperation", /* operationName */ + nil, /* operationErr */ + operationDuration /* operationDuration */) + + // Assert + expectedMetrics := `# HELP csi_sidecar_operations_seconds [ALPHA] Container Storage Interface operation duration with gRPC error code status total + # TYPE csi_sidecar_operations_seconds histogram + csi_sidecar_operations_seconds_bucket{a="111",b="222",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation",le="0.1"} 0 + csi_sidecar_operations_seconds_bucket{a="111",b="222",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation",le="0.25"} 0 + csi_sidecar_operations_seconds_bucket{a="111",b="222",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation",le="0.5"} 0 + csi_sidecar_operations_seconds_bucket{a="111",b="222",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation",le="1"} 0 + csi_sidecar_operations_seconds_bucket{a="111",b="222",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation",le="2.5"} 0 + csi_sidecar_operations_seconds_bucket{a="111",b="222",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation",le="5"} 0 + csi_sidecar_operations_seconds_bucket{a="111",b="222",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation",le="10"} 0 + csi_sidecar_operations_seconds_bucket{a="111",b="222",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation",le="15"} 0 + csi_sidecar_operations_seconds_bucket{a="111",b="222",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation",le="25"} 1 + csi_sidecar_operations_seconds_bucket{a="111",b="222",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation",le="50"} 1 + csi_sidecar_operations_seconds_bucket{a="111",b="222",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation",le="120"} 1 + csi_sidecar_operations_seconds_bucket{a="111",b="222",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation",le="300"} 1 + csi_sidecar_operations_seconds_bucket{a="111",b="222",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation",le="600"} 1 + csi_sidecar_operations_seconds_bucket{a="111",b="222",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation",le="+Inf"} 1 + csi_sidecar_operations_seconds_sum{a="111",b="222",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation"} 20 + csi_sidecar_operations_seconds_count{a="111",b="222",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation"} 1 + ` + + if err := testutil.GatherAndCompare( + cmm.GetRegistry(), strings.NewReader(expectedMetrics)); err != nil { + t.Fatal(err) + } +} + +func TestVaryingLabels(t *testing.T) { + // Arrange + cmm := NewCSIMetricsManagerWithOptions( + "", /* driverName */ + WithLabelNames("a", "b"), + ) + operationDuration, _ := time.ParseDuration("20s") + + // Act + cmmv, err := cmm.WithLabelValues(map[string]string{"a": "111"}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + cmmv, err = cmmv.WithLabelValues(map[string]string{"b": "222"}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + cmmv.RecordMetrics( + "myOperation", /* operationName */ + nil, /* operationErr */ + operationDuration /* operationDuration */) + + // Assert + expectedMetrics := `# HELP csi_sidecar_operations_seconds [ALPHA] Container Storage Interface operation duration with gRPC error code status total + # TYPE csi_sidecar_operations_seconds histogram + csi_sidecar_operations_seconds_bucket{a="111",b="222",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation",le="0.1"} 0 + csi_sidecar_operations_seconds_bucket{a="111",b="222",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation",le="0.25"} 0 + csi_sidecar_operations_seconds_bucket{a="111",b="222",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation",le="0.5"} 0 + csi_sidecar_operations_seconds_bucket{a="111",b="222",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation",le="1"} 0 + csi_sidecar_operations_seconds_bucket{a="111",b="222",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation",le="2.5"} 0 + csi_sidecar_operations_seconds_bucket{a="111",b="222",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation",le="5"} 0 + csi_sidecar_operations_seconds_bucket{a="111",b="222",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation",le="10"} 0 + csi_sidecar_operations_seconds_bucket{a="111",b="222",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation",le="15"} 0 + csi_sidecar_operations_seconds_bucket{a="111",b="222",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation",le="25"} 1 + csi_sidecar_operations_seconds_bucket{a="111",b="222",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation",le="50"} 1 + csi_sidecar_operations_seconds_bucket{a="111",b="222",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation",le="120"} 1 + csi_sidecar_operations_seconds_bucket{a="111",b="222",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation",le="300"} 1 + csi_sidecar_operations_seconds_bucket{a="111",b="222",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation",le="600"} 1 + csi_sidecar_operations_seconds_bucket{a="111",b="222",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation",le="+Inf"} 1 + csi_sidecar_operations_seconds_sum{a="111",b="222",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation"} 20 + csi_sidecar_operations_seconds_count{a="111",b="222",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation"} 1 + ` + + if err := testutil.GatherAndCompare( + cmm.GetRegistry(), strings.NewReader(expectedMetrics)); err != nil { + t.Fatal(err) + } +} + +func TestTwoVaryingLabels(t *testing.T) { + // Arrange + cmm := NewCSIMetricsManagerWithOptions( + "", /* driverName */ + WithLabelNames("a", "b"), + ) + operationDuration, _ := time.ParseDuration("20s") + + // Act + cmmv, err := cmm.WithLabelValues(map[string]string{"a": "111"}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + cmmv2, err := cmmv.WithLabelValues(map[string]string{"b": "222"}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + cmmv2.RecordMetrics( + "myOperation", /* operationName */ + nil, /* operationErr */ + operationDuration /* operationDuration */) + cmmv3, err := cmmv.WithLabelValues(map[string]string{"b": "xxx"}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + cmmv3.RecordMetrics( + "myOtherOperation", /* operationName */ + nil, /* operationErr */ + operationDuration /* operationDuration */) + + // Assert + expectedMetrics := `# HELP csi_sidecar_operations_seconds [ALPHA] Container Storage Interface operation duration with gRPC error code status total + # TYPE csi_sidecar_operations_seconds histogram + csi_sidecar_operations_seconds_bucket{a="111",b="222",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation",le="0.1"} 0 + csi_sidecar_operations_seconds_bucket{a="111",b="222",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation",le="0.25"} 0 + csi_sidecar_operations_seconds_bucket{a="111",b="222",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation",le="0.5"} 0 + csi_sidecar_operations_seconds_bucket{a="111",b="222",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation",le="1"} 0 + csi_sidecar_operations_seconds_bucket{a="111",b="222",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation",le="2.5"} 0 + csi_sidecar_operations_seconds_bucket{a="111",b="222",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation",le="5"} 0 + csi_sidecar_operations_seconds_bucket{a="111",b="222",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation",le="10"} 0 + csi_sidecar_operations_seconds_bucket{a="111",b="222",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation",le="15"} 0 + csi_sidecar_operations_seconds_bucket{a="111",b="222",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation",le="25"} 1 + csi_sidecar_operations_seconds_bucket{a="111",b="222",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation",le="50"} 1 + csi_sidecar_operations_seconds_bucket{a="111",b="222",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation",le="120"} 1 + csi_sidecar_operations_seconds_bucket{a="111",b="222",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation",le="300"} 1 + csi_sidecar_operations_seconds_bucket{a="111",b="222",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation",le="600"} 1 + csi_sidecar_operations_seconds_bucket{a="111",b="222",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation",le="+Inf"} 1 + csi_sidecar_operations_seconds_sum{a="111",b="222",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation"} 20 + csi_sidecar_operations_seconds_count{a="111",b="222",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation"} 1 + csi_sidecar_operations_seconds_bucket{a="111",b="xxx",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOtherOperation",le="0.1"} 0 + csi_sidecar_operations_seconds_bucket{a="111",b="xxx",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOtherOperation",le="0.25"} 0 + csi_sidecar_operations_seconds_bucket{a="111",b="xxx",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOtherOperation",le="0.5"} 0 + csi_sidecar_operations_seconds_bucket{a="111",b="xxx",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOtherOperation",le="1"} 0 + csi_sidecar_operations_seconds_bucket{a="111",b="xxx",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOtherOperation",le="2.5"} 0 + csi_sidecar_operations_seconds_bucket{a="111",b="xxx",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOtherOperation",le="5"} 0 + csi_sidecar_operations_seconds_bucket{a="111",b="xxx",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOtherOperation",le="10"} 0 + csi_sidecar_operations_seconds_bucket{a="111",b="xxx",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOtherOperation",le="15"} 0 + csi_sidecar_operations_seconds_bucket{a="111",b="xxx",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOtherOperation",le="25"} 1 + csi_sidecar_operations_seconds_bucket{a="111",b="xxx",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOtherOperation",le="50"} 1 + csi_sidecar_operations_seconds_bucket{a="111",b="xxx",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOtherOperation",le="120"} 1 + csi_sidecar_operations_seconds_bucket{a="111",b="xxx",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOtherOperation",le="300"} 1 + csi_sidecar_operations_seconds_bucket{a="111",b="xxx",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOtherOperation",le="600"} 1 + csi_sidecar_operations_seconds_bucket{a="111",b="xxx",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOtherOperation",le="+Inf"} 1 + csi_sidecar_operations_seconds_sum{a="111",b="xxx",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOtherOperation"} 20 + csi_sidecar_operations_seconds_count{a="111",b="xxx",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOtherOperation"} 1 + ` + + if err := testutil.GatherAndCompare( + cmm.GetRegistry(), strings.NewReader(expectedMetrics)); err != nil { + t.Fatal(err) + } +} + +func TestVaryingLabelsBackfill(t *testing.T) { + // Arrange + cmm := NewCSIMetricsManagerWithOptions( + "", /* driverName */ + WithLabelNames("a", "b"), + ) + operationDuration, _ := time.ParseDuration("20s") + + // Act + cmmv, err := cmm.WithLabelValues(map[string]string{"a": "111"}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + cmmv.RecordMetrics( + "myOperation", /* operationName */ + nil, /* operationErr */ + operationDuration /* operationDuration */) + + // Assert + expectedMetrics := `# HELP csi_sidecar_operations_seconds [ALPHA] Container Storage Interface operation duration with gRPC error code status total + # TYPE csi_sidecar_operations_seconds histogram + csi_sidecar_operations_seconds_bucket{a="111",b="",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation",le="0.1"} 0 + csi_sidecar_operations_seconds_bucket{a="111",b="",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation",le="0.25"} 0 + csi_sidecar_operations_seconds_bucket{a="111",b="",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation",le="0.5"} 0 + csi_sidecar_operations_seconds_bucket{a="111",b="",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation",le="1"} 0 + csi_sidecar_operations_seconds_bucket{a="111",b="",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation",le="2.5"} 0 + csi_sidecar_operations_seconds_bucket{a="111",b="",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation",le="5"} 0 + csi_sidecar_operations_seconds_bucket{a="111",b="",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation",le="10"} 0 + csi_sidecar_operations_seconds_bucket{a="111",b="",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation",le="15"} 0 + csi_sidecar_operations_seconds_bucket{a="111",b="",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation",le="25"} 1 + csi_sidecar_operations_seconds_bucket{a="111",b="",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation",le="50"} 1 + csi_sidecar_operations_seconds_bucket{a="111",b="",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation",le="120"} 1 + csi_sidecar_operations_seconds_bucket{a="111",b="",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation",le="300"} 1 + csi_sidecar_operations_seconds_bucket{a="111",b="",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation",le="600"} 1 + csi_sidecar_operations_seconds_bucket{a="111",b="",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation",le="+Inf"} 1 + csi_sidecar_operations_seconds_sum{a="111",b="",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation"} 20 + csi_sidecar_operations_seconds_count{a="111",b="",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation"} 1 + ` + + if err := testutil.GatherAndCompare( + cmm.GetRegistry(), strings.NewReader(expectedMetrics)); err != nil { + t.Fatal(err) + } +} + +func TestVaryingLabels_NameError(t *testing.T) { + cmm := NewCSIMetricsManagerWithOptions( + "", /* driverName */ + WithLabelNames("a", "b"), + ) + + _, err := cmm.WithLabelValues(map[string]string{"c": "111"}) + if err == nil { + t.Fatal("unexpected success") + } +} + +func TestVaryingLabels_OverwriteError(t *testing.T) { + cmm := NewCSIMetricsManagerWithOptions( + "", /* driverName */ + WithLabelNames("a", "b"), + ) + + cmmv, err := cmm.WithLabelValues(map[string]string{"a": "111", "b": "222"}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + _, err = cmmv.WithLabelValues(map[string]string{"a": "111"}) + if err == nil { + t.Fatal("unexpected success") + } +} + +func TestCombinedLabels(t *testing.T) { + // Arrange + cmm := NewCSIMetricsManagerWithOptions( + "", /* driverName */ + WithLabelNames("a", "b"), + WithLabels(map[string]string{"c": "333", "d": "444"}), + ) + operationDuration, _ := time.ParseDuration("20s") + + // Act + cmmv, err := cmm.WithLabelValues(map[string]string{"a": "111", "b": "222"}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + cmmv.RecordMetrics( + "myOperation", /* operationName */ + nil, /* operationErr */ + operationDuration /* operationDuration */) + + // Assert + expectedMetrics := `# HELP csi_sidecar_operations_seconds [ALPHA] Container Storage Interface operation duration with gRPC error code status total + # TYPE csi_sidecar_operations_seconds histogram + csi_sidecar_operations_seconds_bucket{a="111",b="222",c="333",d="444",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation",le="0.1"} 0 + csi_sidecar_operations_seconds_bucket{a="111",b="222",c="333",d="444",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation",le="0.25"} 0 + csi_sidecar_operations_seconds_bucket{a="111",b="222",c="333",d="444",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation",le="0.5"} 0 + csi_sidecar_operations_seconds_bucket{a="111",b="222",c="333",d="444",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation",le="1"} 0 + csi_sidecar_operations_seconds_bucket{a="111",b="222",c="333",d="444",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation",le="2.5"} 0 + csi_sidecar_operations_seconds_bucket{a="111",b="222",c="333",d="444",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation",le="5"} 0 + csi_sidecar_operations_seconds_bucket{a="111",b="222",c="333",d="444",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation",le="10"} 0 + csi_sidecar_operations_seconds_bucket{a="111",b="222",c="333",d="444",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation",le="15"} 0 + csi_sidecar_operations_seconds_bucket{a="111",b="222",c="333",d="444",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation",le="25"} 1 + csi_sidecar_operations_seconds_bucket{a="111",b="222",c="333",d="444",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation",le="50"} 1 + csi_sidecar_operations_seconds_bucket{a="111",b="222",c="333",d="444",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation",le="120"} 1 + csi_sidecar_operations_seconds_bucket{a="111",b="222",c="333",d="444",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation",le="300"} 1 + csi_sidecar_operations_seconds_bucket{a="111",b="222",c="333",d="444",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation",le="600"} 1 + csi_sidecar_operations_seconds_bucket{a="111",b="222",c="333",d="444",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation",le="+Inf"} 1 + csi_sidecar_operations_seconds_sum{a="111",b="222",c="333",d="444",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation"} 20 + csi_sidecar_operations_seconds_count{a="111",b="222",c="333",d="444",driver_name="unknown-driver",grpc_status_code="OK",method_name="myOperation"} 1 + ` if err := testutil.GatherAndCompare( cmm.GetRegistry(), strings.NewReader(expectedMetrics)); err != nil { @@ -69,7 +399,7 @@ func TestRecordMetrics(t *testing.T) { func TestRecordMetrics_NoDriverName(t *testing.T) { // Arrange - cmm := NewCSIMetricsManager( + cmm := NewCSIMetricsManagerForSidecar( "" /* driverName */) operationDuration, _ := time.ParseDuration("20s") @@ -108,7 +438,7 @@ func TestRecordMetrics_NoDriverName(t *testing.T) { func TestRecordMetrics_Negative(t *testing.T) { // Arrange - cmm := NewCSIMetricsManager( + cmm := NewCSIMetricsManagerForSidecar( "fake.csi.driver.io" /* driverName */) operationDuration, _ := time.ParseDuration("20s") @@ -146,7 +476,7 @@ func TestRecordMetrics_Negative(t *testing.T) { func TestStartMetricsEndPoint_Noop(t *testing.T) { // Arrange - cmm := NewCSIMetricsManager( + cmm := NewCSIMetricsManagerForSidecar( "fake.csi.driver.io" /* driverName */) operationDuration, _ := time.ParseDuration("20s")