From e6b29fcce7321c259ab38f5dba5751a28364c5ba Mon Sep 17 00:00:00 2001 From: Maya Barnea Date: Wed, 23 Apr 2025 10:23:53 +0300 Subject: [PATCH 1/4] Move filters and scorers registration to filter/scorer specific files --- pkg/epp/scheduling/config.go | 27 +++++++++ pkg/epp/scheduling/default_config.go | 31 +++++++++++ pkg/epp/scheduling/scheduler.go | 23 +++++--- pkg/epp/scheduling/scheduler_test.go | 83 ++++++++++++++-------------- 4 files changed, 116 insertions(+), 48 deletions(-) create mode 100644 pkg/epp/scheduling/config.go create mode 100644 pkg/epp/scheduling/default_config.go diff --git a/pkg/epp/scheduling/config.go b/pkg/epp/scheduling/config.go new file mode 100644 index 000000000..6c0f4be7b --- /dev/null +++ b/pkg/epp/scheduling/config.go @@ -0,0 +1,27 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package scheduling + +import "sigs.k8s.io/gateway-api-inference-extension/pkg/epp/scheduling/plugins" + +type SchedulerConfig struct { + preSchedulePlugins []plugins.PreSchedule + scorers []plugins.Scorer + filters []plugins.Filter + postSchedulePlugins []plugins.PostSchedule + picker plugins.Picker +} diff --git a/pkg/epp/scheduling/default_config.go b/pkg/epp/scheduling/default_config.go new file mode 100644 index 000000000..381770d07 --- /dev/null +++ b/pkg/epp/scheduling/default_config.go @@ -0,0 +1,31 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package scheduling + +import ( + "sigs.k8s.io/gateway-api-inference-extension/pkg/epp/scheduling/plugins" +) + +var defPlugin = &defaultPlugin{} + +var defaultConfig = &SchedulerConfig{ + preSchedulePlugins: []plugins.PreSchedule{}, + scorers: []plugins.Scorer{defPlugin}, + filters: []plugins.Filter{defPlugin}, + postSchedulePlugins: []plugins.PostSchedule{}, + picker: defPlugin, +} diff --git a/pkg/epp/scheduling/scheduler.go b/pkg/epp/scheduling/scheduler.go index beac5e6b8..74aa6043e 100644 --- a/pkg/epp/scheduling/scheduler.go +++ b/pkg/epp/scheduling/scheduler.go @@ -20,6 +20,7 @@ package scheduling import ( "context" "fmt" + "math/rand/v2" "time" "sigs.k8s.io/controller-runtime/pkg/log" @@ -68,16 +69,20 @@ var ( ) func NewScheduler(datastore Datastore) *Scheduler { - defaultPlugin := &defaultPlugin{} + return NewSchedulerWithConfig(datastore, defaultConfig) +} - return &Scheduler{ +func NewSchedulerWithConfig(datastore Datastore, config *SchedulerConfig) *Scheduler { + scheduler := &Scheduler{ datastore: datastore, - preSchedulePlugins: []plugins.PreSchedule{}, - scorers: []plugins.Scorer{}, - filters: []plugins.Filter{defaultPlugin}, - postSchedulePlugins: []plugins.PostSchedule{}, - picker: defaultPlugin, + preSchedulePlugins: config.preSchedulePlugins, + scorers: config.scorers, + filters: config.filters, + postSchedulePlugins: config.postSchedulePlugins, + picker: config.picker, } + + return scheduler } type Scheduler struct { @@ -199,3 +204,7 @@ func (p *defaultPlugin) Filter(ctx *types.SchedulingContext, pods []types.Pod) [ return sheddableRequestFilter.Filter(ctx, pods) } + +func (p *defaultPlugin) Score(ctx *types.SchedulingContext, pod types.Pod) float64 { + return rand.Float64() +} diff --git a/pkg/epp/scheduling/scheduler_test.go b/pkg/epp/scheduling/scheduler_test.go index cb729038e..89ef06fe0 100644 --- a/pkg/epp/scheduling/scheduler_test.go +++ b/pkg/epp/scheduling/scheduler_test.go @@ -220,9 +220,17 @@ func TestSchedule(t *testing.T) { }, } + schedConfig := &SchedulerConfig{ + preSchedulePlugins: []plugins.PreSchedule{}, + scorers: []plugins.Scorer{}, + filters: []plugins.Filter{defPlugin}, + postSchedulePlugins: []plugins.PostSchedule{}, + picker: defPlugin, + } + for _, test := range tests { t.Run(test.name, func(t *testing.T) { - scheduler := NewScheduler(&fakeDataStore{pods: test.input}) + scheduler := NewSchedulerWithConfig(&fakeDataStore{pods: test.input}, schedConfig) got, err := scheduler.Schedule(context.Background(), test.req) if test.err != (err != nil) { t.Errorf("Unexpected error, got %v, want %v", err, test.err) @@ -257,43 +265,43 @@ func TestSchedulePlugins(t *testing.T) { } tests := []struct { - name string - preSchedulePlugins []plugins.PreSchedule - filters []plugins.Filter - scorers []plugins.Scorer - postSchedulePlugins []plugins.PostSchedule - picker plugins.Picker - input []*backendmetrics.FakePodMetrics - wantTargetPod k8stypes.NamespacedName - targetPodScore float64 + name string + config SchedulerConfig + input []*backendmetrics.FakePodMetrics + wantTargetPod k8stypes.NamespacedName + targetPodScore float64 // Number of expected pods to score (after filter) numPodsToScore int err bool }{ { - name: "all plugins executed successfully", - preSchedulePlugins: []plugins.PreSchedule{tp1, tp2}, - filters: []plugins.Filter{tp1, tp2}, - scorers: []plugins.Scorer{tp1, tp2}, - postSchedulePlugins: []plugins.PostSchedule{tp1, tp2}, - picker: pickerPlugin, + name: "all plugins executed successfully", + config: SchedulerConfig{ + preSchedulePlugins: []plugins.PreSchedule{tp1, tp2}, + filters: []plugins.Filter{tp1, tp2}, + scorers: []plugins.Scorer{}, //tp1, tp2 + postSchedulePlugins: []plugins.PostSchedule{tp1, tp2}, + picker: pickerPlugin, + }, input: []*backendmetrics.FakePodMetrics{ {Pod: &backendmetrics.Pod{NamespacedName: k8stypes.NamespacedName{Name: "pod1"}}}, {Pod: &backendmetrics.Pod{NamespacedName: k8stypes.NamespacedName{Name: "pod2"}}}, {Pod: &backendmetrics.Pod{NamespacedName: k8stypes.NamespacedName{Name: "pod3"}}}, }, wantTargetPod: k8stypes.NamespacedName{Name: "pod1"}, - targetPodScore: 1.1, + targetPodScore: 0.0, //1.1, numPodsToScore: 2, err: false, }, { - name: "filter all", - preSchedulePlugins: []plugins.PreSchedule{tp1, tp2}, - filters: []plugins.Filter{tp1, tp_filterAll}, - scorers: []plugins.Scorer{tp1, tp2}, - postSchedulePlugins: []plugins.PostSchedule{tp1, tp2}, - picker: pickerPlugin, + name: "filter all", + config: SchedulerConfig{ + preSchedulePlugins: []plugins.PreSchedule{tp1, tp2}, + filters: []plugins.Filter{tp1, tp_filterAll}, + scorers: []plugins.Scorer{}, //tp1, tp2 + postSchedulePlugins: []plugins.PostSchedule{tp1, tp2}, + picker: pickerPlugin, + }, input: []*backendmetrics.FakePodMetrics{ {Pod: &backendmetrics.Pod{NamespacedName: k8stypes.NamespacedName{Name: "pod1"}}}, {Pod: &backendmetrics.Pod{NamespacedName: k8stypes.NamespacedName{Name: "pod2"}}}, @@ -307,29 +315,22 @@ func TestSchedulePlugins(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { // Reset all plugins before each new test case. - for _, plugin := range test.preSchedulePlugins { + for _, plugin := range test.config.preSchedulePlugins { plugin.(*TestPlugin).reset() } - for _, plugin := range test.postSchedulePlugins { + for _, plugin := range test.config.postSchedulePlugins { plugin.(*TestPlugin).reset() } - for _, plugin := range test.filters { + for _, plugin := range test.config.filters { plugin.(*TestPlugin).reset() } - for _, plugin := range test.scorers { + for _, plugin := range test.config.scorers { plugin.(*TestPlugin).reset() } - test.picker.(*TestPlugin).reset() + test.config.picker.(*TestPlugin).reset() // Initialize the scheduler - scheduler := &Scheduler{ - datastore: &fakeDataStore{pods: test.input}, - preSchedulePlugins: test.preSchedulePlugins, - filters: test.filters, - scorers: test.scorers, - postSchedulePlugins: test.postSchedulePlugins, - picker: test.picker, - } + scheduler := NewSchedulerWithConfig(&fakeDataStore{pods: test.input}, &test.config) req := &types.LLMRequest{Model: "test-model"} got, err := scheduler.Schedule(context.Background(), req) @@ -355,35 +356,35 @@ func TestSchedulePlugins(t *testing.T) { } // Validate plugin execution counts dynamically - for _, plugin := range test.preSchedulePlugins { + for _, plugin := range test.config.preSchedulePlugins { tp, _ := plugin.(*TestPlugin) if tp.PreScheduleCallCount != 1 { t.Errorf("Plugin %s PreSchedule() called %d times, expected 1", tp.NameRes, tp.PreScheduleCallCount) } } - for _, plugin := range test.filters { + for _, plugin := range test.config.filters { tp, _ := plugin.(*TestPlugin) if tp.FilterCallCount != 1 { t.Errorf("Plugin %s Filter() called %d times, expected 1", tp.NameRes, tp.FilterCallCount) } } - for _, plugin := range test.scorers { + for _, plugin := range test.config.scorers { tp, _ := plugin.(*TestPlugin) if tp.ScoreCallCount != test.numPodsToScore { t.Errorf("Plugin %s Score() called %d times, expected 1", tp.NameRes, tp.ScoreCallCount) } } - for _, plugin := range test.postSchedulePlugins { + for _, plugin := range test.config.postSchedulePlugins { tp, _ := plugin.(*TestPlugin) if tp.PostScheduleCallCount != 1 { t.Errorf("Plugin %s PostSchedule() called %d times, expected 1", tp.NameRes, tp.PostScheduleCallCount) } } - tp, _ := test.picker.(*TestPlugin) + tp, _ := test.config.picker.(*TestPlugin) if tp.PickCallCount != 1 { t.Errorf("Picker plugin %s Pick() called %d times, expected 1", tp.NameRes, tp.PickCallCount) } From b4c663064498afd9b864d835157a026e5a24c9df Mon Sep 17 00:00:00 2001 From: Maya Barnea Date: Thu, 24 Apr 2025 16:10:42 +0300 Subject: [PATCH 2/4] Default scheduler config contains empty list of scorers Signed-off-by: Maya Barnea --- pkg/epp/scheduling/default_config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/epp/scheduling/default_config.go b/pkg/epp/scheduling/default_config.go index 381770d07..e42f13179 100644 --- a/pkg/epp/scheduling/default_config.go +++ b/pkg/epp/scheduling/default_config.go @@ -24,7 +24,7 @@ var defPlugin = &defaultPlugin{} var defaultConfig = &SchedulerConfig{ preSchedulePlugins: []plugins.PreSchedule{}, - scorers: []plugins.Scorer{defPlugin}, + scorers: []plugins.Scorer{}, filters: []plugins.Filter{defPlugin}, postSchedulePlugins: []plugins.PostSchedule{}, picker: defPlugin, From 7f3b120cdf138d4da64fa957912b439b45933614 Mon Sep 17 00:00:00 2001 From: Maya Barnea Date: Thu, 24 Apr 2025 16:12:59 +0300 Subject: [PATCH 3/4] Default plugin is not a scorer any more Signed-off-by: Maya Barnea --- pkg/epp/scheduling/scheduler.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/pkg/epp/scheduling/scheduler.go b/pkg/epp/scheduling/scheduler.go index 74aa6043e..322f714f4 100644 --- a/pkg/epp/scheduling/scheduler.go +++ b/pkg/epp/scheduling/scheduler.go @@ -20,7 +20,6 @@ package scheduling import ( "context" "fmt" - "math/rand/v2" "time" "sigs.k8s.io/controller-runtime/pkg/log" @@ -204,7 +203,3 @@ func (p *defaultPlugin) Filter(ctx *types.SchedulingContext, pods []types.Pod) [ return sheddableRequestFilter.Filter(ctx, pods) } - -func (p *defaultPlugin) Score(ctx *types.SchedulingContext, pod types.Pod) float64 { - return rand.Float64() -} From 66c4cc22d691553429133163103bdbbd44624259 Mon Sep 17 00:00:00 2001 From: Maya Barnea Date: Thu, 24 Apr 2025 16:21:44 +0300 Subject: [PATCH 4/4] fix scheduler test + lint comments Signed-off-by: Maya Barnea --- pkg/epp/scheduling/scheduler_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/epp/scheduling/scheduler_test.go b/pkg/epp/scheduling/scheduler_test.go index 89ef06fe0..2fb26a865 100644 --- a/pkg/epp/scheduling/scheduler_test.go +++ b/pkg/epp/scheduling/scheduler_test.go @@ -279,7 +279,7 @@ func TestSchedulePlugins(t *testing.T) { config: SchedulerConfig{ preSchedulePlugins: []plugins.PreSchedule{tp1, tp2}, filters: []plugins.Filter{tp1, tp2}, - scorers: []plugins.Scorer{}, //tp1, tp2 + scorers: []plugins.Scorer{tp1, tp2}, postSchedulePlugins: []plugins.PostSchedule{tp1, tp2}, picker: pickerPlugin, }, @@ -289,7 +289,7 @@ func TestSchedulePlugins(t *testing.T) { {Pod: &backendmetrics.Pod{NamespacedName: k8stypes.NamespacedName{Name: "pod3"}}}, }, wantTargetPod: k8stypes.NamespacedName{Name: "pod1"}, - targetPodScore: 0.0, //1.1, + targetPodScore: 1.1, numPodsToScore: 2, err: false, }, @@ -298,7 +298,7 @@ func TestSchedulePlugins(t *testing.T) { config: SchedulerConfig{ preSchedulePlugins: []plugins.PreSchedule{tp1, tp2}, filters: []plugins.Filter{tp1, tp_filterAll}, - scorers: []plugins.Scorer{}, //tp1, tp2 + scorers: []plugins.Scorer{tp1, tp2}, postSchedulePlugins: []plugins.PostSchedule{tp1, tp2}, picker: pickerPlugin, },