Skip to content

Commit 1fe5162

Browse files
svenefftingeroboquat
svenefftinge
authored and
roboquat
committed
[usage] store more data for in usage entry
1 parent 52279b1 commit 1fe5162

File tree

8 files changed

+152
-208
lines changed

8 files changed

+152
-208
lines changed

‎components/gitpod-protocol/src/usage.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,7 @@ export interface WorkspaceInstanceUsageData {
117117
contextURL: string;
118118
startTime: string;
119119
endTime?: string;
120+
userId: string;
120121
userName: string;
121122
userAvatarURL: string;
122123
}

‎components/usage/pkg/apiv1/usage.go

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -434,11 +434,12 @@ func newUsageFromInstance(instance db.WorkspaceInstanceForUsage, pricer *Workspa
434434
WorkspaceId: instance.WorkspaceID,
435435
WorkspaceType: instance.Type,
436436
WorkspaceClass: instance.WorkspaceClass,
437-
ContextURL: "",
437+
ContextURL: instance.ContextURL,
438438
StartTime: startedTime,
439439
EndTime: endTime,
440-
UserName: "",
441-
UserAvatarURL: "",
440+
UserID: instance.UserID,
441+
UserName: instance.UserName,
442+
UserAvatarURL: instance.UserAvatarURL,
442443
})
443444
if err != nil {
444445
return db.Usage{}, fmt.Errorf("failed to serialize workspace instance metadata: %w", err)

‎components/usage/pkg/apiv1/usage_test.go

Lines changed: 6 additions & 185 deletions
Original file line numberDiff line numberDiff line change
@@ -7,12 +7,9 @@ package apiv1
77
import (
88
"context"
99
"database/sql"
10-
"reflect"
1110
"testing"
1211
"time"
1312

14-
"github.com/gitpod-io/gitpod/usage/pkg/contentservice"
15-
1613
"github.com/gitpod-io/gitpod/common-go/baseserver"
1714
v1 "github.com/gitpod-io/gitpod/usage-api/v1"
1815
"github.com/gitpod-io/gitpod/usage/pkg/db"
@@ -375,182 +372,6 @@ func TestInstanceToUsageRecords(t *testing.T) {
375372
}
376373
}
377374

378-
func TestReportGenerator_GenerateUsageReport(t *testing.T) {
379-
startOfMay := time.Date(2022, 05, 1, 0, 00, 00, 00, time.UTC)
380-
startOfJune := time.Date(2022, 06, 1, 0, 00, 00, 00, time.UTC)
381-
382-
teamID := uuid.New()
383-
scenarioRunTime := time.Date(2022, 05, 31, 23, 00, 00, 00, time.UTC)
384-
385-
instances := []db.WorkspaceInstance{
386-
// Ran throughout the reconcile period
387-
dbtest.NewWorkspaceInstance(t, db.WorkspaceInstance{
388-
ID: uuid.New(),
389-
UsageAttributionID: db.NewTeamAttributionID(teamID.String()),
390-
StartedTime: db.NewVarcharTime(time.Date(2022, 05, 1, 00, 01, 00, 00, time.UTC)),
391-
StoppingTime: db.NewVarcharTime(time.Date(2022, 06, 1, 1, 0, 0, 0, time.UTC)),
392-
}),
393-
// Still running
394-
dbtest.NewWorkspaceInstance(t, db.WorkspaceInstance{
395-
ID: uuid.New(),
396-
UsageAttributionID: db.NewTeamAttributionID(teamID.String()),
397-
StartedTime: db.NewVarcharTime(time.Date(2022, 05, 30, 00, 01, 00, 00, time.UTC)),
398-
}),
399-
// No creation time, invalid record, ignored
400-
dbtest.NewWorkspaceInstance(t, db.WorkspaceInstance{
401-
ID: uuid.New(),
402-
UsageAttributionID: db.NewTeamAttributionID(teamID.String()),
403-
StoppingTime: db.NewVarcharTime(time.Date(2022, 06, 1, 1, 0, 0, 0, time.UTC)),
404-
}),
405-
}
406-
407-
conn := dbtest.ConnectForTests(t)
408-
dbtest.CreateWorkspaceInstances(t, conn, instances...)
409-
410-
nowFunc := func() time.Time { return scenarioRunTime }
411-
generator := &ReportGenerator{
412-
nowFunc: nowFunc,
413-
conn: conn,
414-
pricer: DefaultWorkspacePricer,
415-
}
416-
417-
report, err := generator.GenerateUsageReport(context.Background(), startOfMay, startOfJune)
418-
require.NoError(t, err)
419-
420-
require.Equal(t, nowFunc(), report.GenerationTime)
421-
require.Equal(t, startOfMay, report.From)
422-
// require.Equal(t, startOfJune, report.To) TODO(gpl) This is not true anymore - does it really make sense to test for it?
423-
require.Len(t, report.InvalidSessions, 0)
424-
require.Len(t, report.UsageRecords, 2)
425-
}
426-
427-
func TestReportGenerator_GenerateUsageReportTable(t *testing.T) {
428-
teamID := uuid.New()
429-
instanceID := uuid.New()
430-
431-
Must := func(ti db.VarcharTime, err error) db.VarcharTime {
432-
if err != nil {
433-
t.Fatal(err)
434-
}
435-
return ti
436-
}
437-
Timestamp := func(timestampAsStr string) db.VarcharTime {
438-
return Must(db.NewVarcharTimeFromStr(timestampAsStr))
439-
}
440-
type Expectation struct {
441-
custom func(t *testing.T, report contentservice.UsageReport)
442-
usageRecords []db.WorkspaceInstanceUsage
443-
}
444-
445-
type TestCase struct {
446-
name string
447-
from time.Time
448-
to time.Time
449-
runtime time.Time
450-
instances []db.WorkspaceInstance
451-
expectation Expectation
452-
}
453-
tests := []TestCase{
454-
{
455-
name: "real example taken from DB: runtime _before_ instance.startedTime",
456-
from: time.Date(2022, 8, 1, 0, 00, 00, 00, time.UTC),
457-
to: time.Date(2022, 9, 1, 0, 00, 00, 00, time.UTC),
458-
runtime: Timestamp("2022-08-17T09:38:28Z").Time(),
459-
instances: []db.WorkspaceInstance{
460-
dbtest.NewWorkspaceInstance(t, db.WorkspaceInstance{
461-
ID: instanceID,
462-
UsageAttributionID: db.NewTeamAttributionID(teamID.String()),
463-
CreationTime: Timestamp("2022-08-17T09:40:47.316Z"),
464-
StartedTime: Timestamp("2022-08-17T09:40:53.115Z"),
465-
StoppingTime: Timestamp("2022-08-17T09:42:36.292Z"),
466-
StoppedTime: Timestamp("2022-08-17T09:43:04.874Z"),
467-
}),
468-
},
469-
expectation: Expectation{
470-
usageRecords: nil,
471-
// usageRecords: []db.WorkspaceInstanceUsage{
472-
// {
473-
// InstanceID: instanceID,
474-
// AttributionID: db.NewTeamAttributionID(teamID.String()),
475-
// StartedAt: Timestamp("2022-08-17T09:40:53.115Z").Time(),
476-
// StoppedAt: sql.NullTime{ Time: Timestamp("2022-08-17T09:43:04.874Z").Time(), Valid: true },
477-
// WorkspaceClass: "default",
478-
// CreditsUsed: 3.0,
479-
// },
480-
// },
481-
},
482-
},
483-
{
484-
name: "same as above, but with runtime _after_ startedTime",
485-
from: time.Date(2022, 8, 1, 0, 00, 00, 00, time.UTC),
486-
to: time.Date(2022, 9, 1, 0, 00, 00, 00, time.UTC),
487-
runtime: Timestamp("2022-08-17T09:41:00Z").Time(),
488-
instances: []db.WorkspaceInstance{
489-
dbtest.NewWorkspaceInstance(t, db.WorkspaceInstance{
490-
ID: instanceID,
491-
UsageAttributionID: db.NewTeamAttributionID(teamID.String()),
492-
CreationTime: Timestamp("2022-08-17T09:40:47.316Z"),
493-
StartedTime: Timestamp("2022-08-17T09:40:53.115Z"),
494-
StoppingTime: Timestamp("2022-08-17T09:42:36.292Z"),
495-
StoppedTime: Timestamp("2022-08-17T09:43:04.874Z"),
496-
}),
497-
},
498-
expectation: Expectation{
499-
usageRecords: []db.WorkspaceInstanceUsage{
500-
{
501-
InstanceID: instanceID,
502-
AttributionID: db.NewTeamAttributionID(teamID.String()),
503-
StartedAt: Timestamp("2022-08-17T09:40:53.115Z").Time(),
504-
StoppedAt: sql.NullTime{Time: Timestamp("2022-08-17T09:41:00Z").Time(), Valid: true},
505-
WorkspaceClass: "default",
506-
CreditsUsed: 0.019444444444444445,
507-
},
508-
},
509-
},
510-
},
511-
}
512-
513-
for _, test := range tests {
514-
t.Run(test.name, func(t *testing.T) {
515-
conn := dbtest.ConnectForTests(t)
516-
dbtest.CreateWorkspaceInstances(t, conn, test.instances...)
517-
518-
nowFunc := func() time.Time { return test.runtime }
519-
generator := &ReportGenerator{
520-
nowFunc: nowFunc,
521-
conn: conn,
522-
pricer: DefaultWorkspacePricer,
523-
}
524-
525-
report, err := generator.GenerateUsageReport(context.Background(), test.from, test.to)
526-
require.NoError(t, err)
527-
528-
require.Equal(t, test.runtime, report.GenerationTime)
529-
require.Equal(t, test.from, report.From)
530-
// require.Equal(t, test.to, report.To) TODO(gpl) This is not true anymore - does it really make sense to test for it?
531-
532-
// These invariants should always be true:
533-
// 1. No negative usage
534-
for _, rec := range report.UsageRecords {
535-
if rec.CreditsUsed < 0 {
536-
t.Error("Got report with negative credits!")
537-
}
538-
}
539-
540-
if !reflect.DeepEqual(test.expectation.usageRecords, report.UsageRecords) {
541-
t.Errorf("report.UsageRecords: expected %v but got %v", test.expectation.usageRecords, report.UsageRecords)
542-
}
543-
544-
// Custom expectations
545-
customTestFunction := test.expectation.custom
546-
if customTestFunction != nil {
547-
customTestFunction(t, report)
548-
require.NoError(t, err)
549-
}
550-
})
551-
}
552-
}
553-
554375
func TestUsageService_ReconcileUsageWithLedger(t *testing.T) {
555376
dbconn := dbtest.ConnectForTests(t)
556377
from := time.Date(2022, 05, 1, 0, 00, 00, 00, time.UTC)
@@ -673,11 +494,11 @@ func TestReconcileWithLedger(t *testing.T) {
673494
WorkspaceId: instance.WorkspaceID,
674495
WorkspaceType: instance.Type,
675496
WorkspaceClass: instance.WorkspaceClass,
676-
ContextURL: "",
497+
ContextURL: instance.ContextURL,
677498
StartTime: db.TimeToISO8601(instance.StartedTime.Time()),
678499
EndTime: "",
679-
UserName: "",
680-
UserAvatarURL: "",
500+
UserName: instance.UserName,
501+
UserAvatarURL: instance.UserAvatarURL,
681502
}))
682503
require.EqualValues(t, expectedUsage, inserts[0])
683504
})
@@ -731,11 +552,11 @@ func TestReconcileWithLedger(t *testing.T) {
731552
WorkspaceId: instance.WorkspaceID,
732553
WorkspaceType: instance.Type,
733554
WorkspaceClass: instance.WorkspaceClass,
734-
ContextURL: "",
555+
ContextURL: instance.ContextURL,
735556
StartTime: db.TimeToISO8601(instance.StartedTime.Time()),
736557
EndTime: "",
737-
UserName: "",
738-
UserAvatarURL: "",
558+
UserName: instance.UserName,
559+
UserAvatarURL: instance.UserAvatarURL,
739560
}))
740561
require.EqualValues(t, expectedUsage, updates[0])
741562
})

‎components/usage/pkg/db/dbtest/user.go

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
// Copyright (c) 2022 Gitpod GmbH. All rights reserved.
2+
// Licensed under the GNU Affero General Public License (AGPL).
3+
// See License-AGPL.txt in the project root for license information.
4+
5+
package dbtest
6+
7+
import (
8+
"testing"
9+
"time"
10+
11+
"github.com/gitpod-io/gitpod/usage/pkg/db"
12+
"github.com/google/uuid"
13+
"github.com/stretchr/testify/require"
14+
"gorm.io/gorm"
15+
)
16+
17+
type User struct {
18+
ID uuid.UUID `gorm:"primary_key;column:id;type:char;size:36;"`
19+
AvatarURL string `gorm:"column:avatarUrl;type:char;size:255;"`
20+
Name string `gorm:"column:name;type:char;size:255;"`
21+
FullName string `gorm:"column:fullName;type:char;size:255;"`
22+
CreationDate db.VarcharTime `gorm:"column:creationDate;type:varchar;size:255;"`
23+
24+
// user has more field but we don't care here as they are just used in tests.
25+
}
26+
27+
func (user *User) TableName() string {
28+
return "d_b_user"
29+
}
30+
31+
func NewUser(t *testing.T, user User) User {
32+
t.Helper()
33+
34+
result := User{
35+
ID: uuid.New(),
36+
AvatarURL: "https://avatars.githubusercontent.com/u/9071",
37+
Name: "HomerJSimpson",
38+
FullName: "Homer Simpson",
39+
CreationDate: db.NewVarcharTime(time.Now()),
40+
}
41+
42+
if user.ID != uuid.Nil {
43+
result.ID = user.ID
44+
}
45+
46+
if user.AvatarURL != "" {
47+
result.AvatarURL = user.AvatarURL
48+
}
49+
if user.Name != "" {
50+
result.Name = user.Name
51+
}
52+
if user.FullName != "" {
53+
result.FullName = user.FullName
54+
}
55+
56+
return result
57+
}
58+
59+
func CreatUser(t *testing.T, conn *gorm.DB, user ...User) []User {
60+
t.Helper()
61+
62+
var records []User
63+
var ids []uuid.UUID
64+
for _, u := range user {
65+
record := NewUser(t, u)
66+
records = append(records, record)
67+
ids = append(ids, record.ID)
68+
}
69+
70+
require.NoError(t, conn.CreateInBatches(&records, 1000).Error)
71+
72+
t.Cleanup(func() {
73+
require.NoError(t, conn.Where(ids).Delete(&User{}).Error)
74+
})
75+
76+
return records
77+
}

‎components/usage/pkg/db/dbtest/workspace_instance.go

Lines changed: 31 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
package dbtest
66

77
import (
8+
"context"
89
"database/sql"
910
"testing"
1011
"time"
@@ -23,7 +24,7 @@ func NewWorkspaceInstance(t *testing.T, instance db.WorkspaceInstance) db.Worksp
2324
t.Helper()
2425

2526
id := uuid.New()
26-
if instance.ID.ID() != 0 { // empty value
27+
if instance.ID != uuid.Nil {
2728
id = instance.ID
2829
}
2930

@@ -120,3 +121,32 @@ func CreateWorkspaceInstances(t *testing.T, conn *gorm.DB, instances ...db.Works
120121

121122
return records
122123
}
124+
125+
// ListWorkspaceInstancesInRange filters out instances by workspaceID to make tests robust and work only on their own data
126+
func ListWorkspaceInstancesInRange(t *testing.T, conn *gorm.DB, from, to time.Time, workspaceID string) []db.WorkspaceInstanceForUsage {
127+
all, err := db.ListWorkspaceInstancesInRange(context.Background(), conn, from, to)
128+
require.NoError(t, err)
129+
return filterByWorkspaceId(all, workspaceID)
130+
}
131+
132+
func FindStoppedWorkspaceInstancesInRange(t *testing.T, conn *gorm.DB, from, to time.Time, workspaceID string) []db.WorkspaceInstanceForUsage {
133+
all, err := db.FindStoppedWorkspaceInstancesInRange(context.Background(), conn, from, to)
134+
require.NoError(t, err)
135+
return filterByWorkspaceId(all, workspaceID)
136+
}
137+
138+
func FindRunningWorkspaceInstances(t *testing.T, conn *gorm.DB, workspaceID string) []db.WorkspaceInstanceForUsage {
139+
all, err := db.FindRunningWorkspaceInstances(context.Background(), conn)
140+
require.NoError(t, err)
141+
return filterByWorkspaceId(all, workspaceID)
142+
}
143+
144+
func filterByWorkspaceId(all []db.WorkspaceInstanceForUsage, workspaceID string) []db.WorkspaceInstanceForUsage {
145+
result := []db.WorkspaceInstanceForUsage{}
146+
for _, candidate := range all {
147+
if candidate.WorkspaceID == workspaceID {
148+
result = append(result, candidate)
149+
}
150+
}
151+
return result
152+
}

‎components/usage/pkg/db/usage.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,7 @@ type WorkspaceInstanceUsageData struct {
7777
ContextURL string `json:"contextURL"`
7878
StartTime string `json:"startTime"`
7979
EndTime string `json:"endTime"`
80+
UserID uuid.UUID `json:"userId"`
8081
UserName string `json:"userName"`
8182
UserAvatarURL string `json:"userAvatarURL"`
8283
}

0 commit comments

Comments
 (0)