Skip to content

Commit 6eeab72

Browse files
committed
add tpot to inference gateway exposed metrics
1 parent dc8c0d7 commit 6eeab72

File tree

1 file changed

+1
-0
lines changed

1 file changed

+1
-0
lines changed

pkg/epp/handlers/streamingserver.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -227,6 +227,7 @@ func (s *StreamingServer) Process(srv extProcPb.ExternalProcessor_ProcessServer)
227227
metrics.RecordResponseSizes(reqCtx.Model, reqCtx.ResolvedTargetModel, reqCtx.ResponseSize)
228228
metrics.RecordInputTokens(reqCtx.Model, reqCtx.ResolvedTargetModel, reqCtx.Usage.PromptTokens)
229229
metrics.RecordOutputTokens(reqCtx.Model, reqCtx.ResolvedTargetModel, reqCtx.Usage.CompletionTokens)
230+
metrics.RecordLatencyPerOutputToken(ctx, reqCtx.Model, reqCtx.ResolvedTargetModel, reqCtx.RequestReceivedTimestamp, reqCtx.ResponseCompleteTimestamp, reqCtx.Usage.CompletionTokens)
230231
}
231232
}
232233
}

0 commit comments

Comments
 (0)