Merge pull request neuralmagic#84 from shmuelk/dev-rebase-2025-04-29

Dev rebase 2025 04 29

Author: shmuelk

go.mod

@@ -17,12 +17,12 @@ require (
 	go.uber.org/zap v1.27.0
 	google.golang.org/grpc v1.71.1
 	google.golang.org/protobuf v1.36.6
-	k8s.io/api v0.32.3
-	k8s.io/apiextensions-apiserver v0.32.3
-	k8s.io/apimachinery v0.32.3
-	k8s.io/client-go v0.32.3
-	k8s.io/code-generator v0.32.3
-	k8s.io/component-base v0.32.3
+	k8s.io/api v0.32.4
+	k8s.io/apiextensions-apiserver v0.32.4
+	k8s.io/apimachinery v0.32.4
+	k8s.io/client-go v0.32.4
+	k8s.io/code-generator v0.32.4
+	k8s.io/component-base v0.32.4
 	k8s.io/utils v0.0.0-20241210054802-24370beab758
 	sigs.k8s.io/controller-runtime v0.20.4
 	sigs.k8s.io/structured-merge-diff/v4 v4.7.0
@@ -123,7 +123,7 @@ require (
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	k8s.io/apiserver v0.32.3 // indirect
+	k8s.io/apiserver v0.32.4 // indirect
 	k8s.io/gengo/v2 v2.0.0-20240911193312-2b36238f13e9 // indirect
 	k8s.io/klog/v2 v2.130.1 // indirect
 	k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f // indirect

go.sum

@@ -302,20 +302,20 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-k8s.io/api v0.32.3 h1:Hw7KqxRusq+6QSplE3NYG4MBxZw1BZnq4aP4cJVINls=
-k8s.io/api v0.32.3/go.mod h1:2wEDTXADtm/HA7CCMD8D8bK4yuBUptzaRhYcYEEYA3k=
-k8s.io/apiextensions-apiserver v0.32.3 h1:4D8vy+9GWerlErCwVIbcQjsWunF9SUGNu7O7hiQTyPY=
-k8s.io/apiextensions-apiserver v0.32.3/go.mod h1:8YwcvVRMVzw0r1Stc7XfGAzB/SIVLunqApySV5V7Dss=
-k8s.io/apimachinery v0.32.3 h1:JmDuDarhDmA/Li7j3aPrwhpNBA94Nvk5zLeOge9HH1U=
-k8s.io/apimachinery v0.32.3/go.mod h1:GpHVgxoKlTxClKcteaeuF1Ul/lDVb74KpZcxcmLDElE=
-k8s.io/apiserver v0.32.3 h1:kOw2KBuHOA+wetX1MkmrxgBr648ksz653j26ESuWNY8=
-k8s.io/apiserver v0.32.3/go.mod h1:q1x9B8E/WzShF49wh3ADOh6muSfpmFL0I2t+TG0Zdgc=
-k8s.io/client-go v0.32.3 h1:RKPVltzopkSgHS7aS98QdscAgtgah/+zmpAogooIqVU=
-k8s.io/client-go v0.32.3/go.mod h1:3v0+3k4IcT9bXTc4V2rt+d2ZPPG700Xy6Oi0Gdl2PaY=
-k8s.io/code-generator v0.32.3 h1:31p2TVzC9+hVdSkAFruAk3JY+iSfzrJ83Qij1yZutyw=
-k8s.io/code-generator v0.32.3/go.mod h1:+mbiYID5NLsBuqxjQTygKM/DAdKpAjvBzrJd64NU1G8=
-k8s.io/component-base v0.32.3 h1:98WJvvMs3QZ2LYHBzvltFSeJjEx7t5+8s71P7M74u8k=
-k8s.io/component-base v0.32.3/go.mod h1:LWi9cR+yPAv7cu2X9rZanTiFKB2kHA+JjmhkKjCZRpI=
+k8s.io/api v0.32.4 h1:kw8Y/G8E7EpNy7gjB8gJZl3KJkNz8HM2YHrZPtAZsF4=
+k8s.io/api v0.32.4/go.mod h1:5MYFvLvweRhyKylM3Es/6uh/5hGp0dg82vP34KifX4g=
+k8s.io/apiextensions-apiserver v0.32.4 h1:IA+CoR63UDOijR/vEpow6wQnX4V6iVpzazJBskHrpHE=
+k8s.io/apiextensions-apiserver v0.32.4/go.mod h1:Y06XO/b92H8ymOdG1HlA1submf7gIhbEDc3RjriqZOs=
+k8s.io/apimachinery v0.32.4 h1:8EEksaxA7nd7xWJkkwLDN4SvWS5ot9g6Z/VZb3ju25I=
+k8s.io/apimachinery v0.32.4/go.mod h1:GpHVgxoKlTxClKcteaeuF1Ul/lDVb74KpZcxcmLDElE=
+k8s.io/apiserver v0.32.4 h1:Yf7sd/y+GOQKH1Qf6wUeayZrYXe2SKZ17Bcq7VQM5HQ=
+k8s.io/apiserver v0.32.4/go.mod h1:JFUMNtE2M5yqLZpIsgCb06SkVSW1YcxW1oyLSTfjXR8=
+k8s.io/client-go v0.32.4 h1:zaGJS7xoYOYumoWIFXlcVrsiYioRPrXGO7dBfVC5R6M=
+k8s.io/client-go v0.32.4/go.mod h1:k0jftcyYnEtwlFW92xC7MTtFv5BNcZBr+zn9jPlT9Ic=
+k8s.io/code-generator v0.32.4 h1:d4dm/43RD6xhPBX22JgJw9JUpwTKzVR6tAxJD7pz83o=
+k8s.io/code-generator v0.32.4/go.mod h1:R0bKdIg1smtvsKvj9q7SxTeKq5X9ko6PuICCGt4yqxg=
+k8s.io/component-base v0.32.4 h1:HuF+2JVLbFS5GODLIfPCb1Td6b+G2HszJoArcWOSr5I=
+k8s.io/component-base v0.32.4/go.mod h1:10KloJEYw1keU/Xmjfy9TKJqUq7J2mYdiD1VDXoco4o=
 k8s.io/gengo/v2 v2.0.0-20240911193312-2b36238f13e9 h1:si3PfKm8dDYxgfbeA6orqrtLkvvIeH8UqffFJDl0bz4=
 k8s.io/gengo/v2 v2.0.0-20240911193312-2b36238f13e9/go.mod h1:EJykeLsmFC60UQbYJezXkEsG2FLrt0GPNkU5iK5GWxU=
 k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk=

pkg/epp/scheduling/plugins/picker/max-score.go

@@ -0,0 +1,49 @@
+package picker
+
+import (
+	"fmt"
+
+	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/scheduling/plugins"
+	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/scheduling/types"
+	logutil "sigs.k8s.io/gateway-api-inference-extension/pkg/epp/util/logging"
+)
+
+var _ plugins.Picker = &MaxScorePicker{}
+
+func NewMaxScorePicker() plugins.Picker {
+	return &MaxScorePicker{
+		random: &RandomPicker{},
+	}
+}
+
+// MaxScorePicker picks the pod with the maximum score from the list of candidates.
+type MaxScorePicker struct {
+	random *RandomPicker
+}
+
+// Name returns the name of the picker.
+func (p *MaxScorePicker) Name() string {
+	return "max_score"
+}
+
+// Pick selects the pod with the maximum score from the list of candidates.
+func (p *MaxScorePicker) Pick(ctx *types.SchedulingContext, scoredPods []*types.ScoredPod) *types.Result {
+	ctx.Logger.V(logutil.DEBUG).Info(fmt.Sprintf("Selecting a pod with the max score from %d candidates: %+v", len(scoredPods), scoredPods))
+
+	highestScorePods := []*types.ScoredPod{}
+	maxScore := -1.0 // pods min score is 0, putting value lower than 0 in order to find at least one pod as highest
+	for _, pod := range scoredPods {
+		if pod.Score > maxScore {
+			maxScore = pod.Score
+			highestScorePods = []*types.ScoredPod{pod}
+		} else if pod.Score == maxScore {
+			highestScorePods = append(highestScorePods, pod)
+		}
+	}
+
+	if len(highestScorePods) > 1 {
+		return p.random.Pick(ctx, highestScorePods) // pick randomly from the highest score pods
+	}
+
+	return &types.Result{TargetPod: highestScorePods[0]}
+}

pkg/epp/scheduling/plugins/picker/max_score_picker.go

@@ -30,12 +30,12 @@ var _ plugins.Picker = &RandomPicker{}
 // RandomPicker picks a random pod from the list of candidates.
 type RandomPicker struct{}
 
-func (rp *RandomPicker) Name() string {
+func (p *RandomPicker) Name() string {
 	return "random"
 }
 
-func (rp *RandomPicker) Pick(ctx *types.SchedulingContext, scoredPods []*types.ScoredPod) *types.Result {
+func (p *RandomPicker) Pick(ctx *types.SchedulingContext, scoredPods []*types.ScoredPod) *types.Result {
 	ctx.Logger.V(logutil.DEBUG).Info(fmt.Sprintf("Selecting a random pod from %d candidates: %+v", len(scoredPods), scoredPods))
 	i := rand.Intn(len(scoredPods))
-	return &types.Result{TargetPod: scoredPods[i].Pod}
+	return &types.Result{TargetPod: scoredPods[i]}
 }

pkg/epp/scheduling/plugins/picker/random_picker.go

@@ -93,17 +93,19 @@ func TestSchedule(t *testing.T) {
 				},
 			},
 			wantRes: &types.Result{
-				TargetPod: &types.PodMetrics{
-					Pod: &backendmetrics.Pod{NamespacedName: k8stypes.NamespacedName{Name: "pod2"}},
-					Metrics: &backendmetrics.Metrics{
-						WaitingQueueSize:    3,
-						KVCacheUsagePercent: 0.1,
-						MaxActiveModels:     2,
-						ActiveModels: map[string]int{
-							"foo":      1,
-							"critical": 1,
+				TargetPod: &types.ScoredPod{
+					Pod: &types.PodMetrics{
+						Pod: &backendmetrics.Pod{NamespacedName: k8stypes.NamespacedName{Name: "pod2"}},
+						Metrics: &backendmetrics.Metrics{
+							WaitingQueueSize:    3,
+							KVCacheUsagePercent: 0.1,
+							MaxActiveModels:     2,
+							ActiveModels: map[string]int{
+								"foo":      1,
+								"critical": 1,
+							},
+							WaitingModels: map[string]int{},
 						},
-						WaitingModels: map[string]int{},
 					},
 				},
 				MutatedHeaders: make(map[string]string),
@@ -155,17 +157,19 @@ func TestSchedule(t *testing.T) {
 				},
 			},
 			wantRes: &types.Result{
-				TargetPod: &types.PodMetrics{
-					Pod: &backendmetrics.Pod{NamespacedName: k8stypes.NamespacedName{Name: "pod1"}},
-					Metrics: &backendmetrics.Metrics{
-						WaitingQueueSize:    0,
-						KVCacheUsagePercent: 0.2,
-						MaxActiveModels:     2,
-						ActiveModels: map[string]int{
-							"foo": 1,
-							"bar": 1,
+				TargetPod: &types.ScoredPod{
+					Pod: &types.PodMetrics{
+						Pod: &backendmetrics.Pod{NamespacedName: k8stypes.NamespacedName{Name: "pod1"}},
+						Metrics: &backendmetrics.Metrics{
+							WaitingQueueSize:    0,
+							KVCacheUsagePercent: 0.2,
+							MaxActiveModels:     2,
+							ActiveModels: map[string]int{
+								"foo": 1,
+								"bar": 1,
+							},
+							WaitingModels: map[string]int{},
 						},
-						WaitingModels: map[string]int{},
 					},
 				},
 				MutatedHeaders: make(map[string]string),
@@ -525,6 +529,7 @@ func (tp *TestPlugin) Filter(ctx *types.SchedulingContext, pods []types.Pod) []t
 		tp.ReceivedRequestHeaders[key] = value
 	}
 	return findPods(ctx, tp.FilterRes...)
+
 }
 
 func (tp *TestPlugin) Score(ctx *types.SchedulingContext, pods []types.Pod) map[types.Pod]float64 {
@@ -574,7 +579,7 @@ func findPods(ctx *types.SchedulingContext, names ...k8stypes.NamespacedName) []
 func getPodScore(scoredPods []*types.ScoredPod, selectedPod types.Pod) float64 {
 	finalScore := 0.0
 	for _, scoredPod := range scoredPods {
-		if scoredPod.Pod.GetPod().NamespacedName.String() == selectedPod.GetPod().NamespacedName.String() {
+		if scoredPod.GetPod().NamespacedName.String() == selectedPod.GetPod().NamespacedName.String() {
 			finalScore = scoredPod.Score
 			break
 		}

pkg/epp/scheduling/scheduler_test.go

@@ -49,7 +49,7 @@ type Pod interface {
 }
 
 type ScoredPod struct {
-	Pod   Pod
+	Pod
 	Score float64
 }
 

pkg/epp/scheduling/types/types.go

@@ -5,26 +5,25 @@ import (
 	"strconv"
 
 	"github.com/go-logr/logr"
-	logutil "sigs.k8s.io/gateway-api-inference-extension/pkg/epp/util/logging"
 )
 
 // getEnvFloat gets a float64 from an environment variable with a default value
 func GetEnvFloat(key string, defaultVal float64, logger logr.Logger) float64 {
 	val, exists := os.LookupEnv(key)
 	if !exists {
-		logger.V(logutil.VERBOSE).Info("Environment variable not set, using default value",
+		logger.Info("Environment variable not set, using default value",
 			"key", key, "defaultValue", defaultVal)
 		return defaultVal
 	}
 
 	floatVal, err := strconv.ParseFloat(val, 64)
 	if err != nil {
-		logger.V(logutil.VERBOSE).Info("Failed to parse environment variable as float, using default value",
+		logger.Info("Failed to parse environment variable as float, using default value",
 			"key", key, "value", val, "error", err, "defaultValue", defaultVal)
 		return defaultVal
 	}
 
-	logger.V(logutil.VERBOSE).Info("Successfully loaded environment variable",
+	logger.Info("Successfully loaded environment variable",
 		"key", key, "value", floatVal)
 	return floatVal
 }
@@ -33,19 +32,30 @@ func GetEnvFloat(key string, defaultVal float64, logger logr.Logger) float64 {
 func GetEnvInt(key string, defaultVal int, logger logr.Logger) int {
 	val, exists := os.LookupEnv(key)
 	if !exists {
-		logger.V(logutil.VERBOSE).Info("Environment variable not set, using default value",
+		logger.Info("Environment variable not set, using default value",
 			"key", key, "defaultValue", defaultVal)
 		return defaultVal
 	}
 
 	intVal, err := strconv.Atoi(val)
 	if err != nil {
-		logger.V(logutil.VERBOSE).Info("Failed to parse environment variable as int, using default value",
+		logger.Info("Failed to parse environment variable as int, using default value",
 			"key", key, "value", val, "error", err, "defaultValue", defaultVal)
 		return defaultVal
 	}
 
-	logger.V(logutil.VERBOSE).Info("Successfully loaded environment variable",
+	logger.Info("Successfully loaded environment variable",
 		"key", key, "value", intVal)
 	return intVal
 }
+
+// GetEnvString gets a string from an environment variable with a default value
+func GetEnvString(key string, defaultVal string, logger logr.Logger) string {
+	val, exists := os.LookupEnv(key)
+	if !exists {
+		logger.Info("Environment variable not set, using default value",
+			"key", key, "defaultValue", defaultVal)
+		return defaultVal
+	}
+	return val
+}

pkg/epp/util/env/env.go

@@ -142,3 +142,64 @@ func TestGetEnvInt(t *testing.T) {
 		})
 	}
 }
+
+func TestGetEnvString(t *testing.T) {
+	logger := testr.New(t)
+
+	tests := []struct {
+		name       string
+		key        string
+		value      string
+		defaultVal string
+		expected   string
+		setup      func()
+		teardown   func()
+	}{
+		{
+			name:       "env variable exists and is valid",
+			key:        "TEST_STR",
+			value:      "123",
+			defaultVal: "default",
+			expected:   "123",
+			setup: func() {
+				os.Setenv("TEST_STR", "123")
+			},
+			teardown: func() {
+				os.Unsetenv("TEST_STR")
+			},
+		},
+		{
+			name:       "env variable does not exist",
+			key:        "TEST_STR_MISSING",
+			defaultVal: "default",
+			expected:   "default",
+			setup:      func() {},
+			teardown:   func() {},
+		},
+		{
+			name:       "env variable is empty string",
+			key:        "TEST_STR_EMPTY",
+			value:      "",
+			defaultVal: "default",
+			expected:   "",
+			setup: func() {
+				os.Setenv("TEST_STR_EMPTY", "")
+			},
+			teardown: func() {
+				os.Unsetenv("TEST_STR_EMPTY")
+			},
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			tc.setup()
+			defer tc.teardown()
+
+			result := GetEnvString(tc.key, tc.defaultVal, logger.V(logutil.VERBOSE))
+			if result != tc.expected {
+				t.Errorf("GetEnvString(%s, %s) = %s, expected %s", tc.key, tc.defaultVal, result, tc.expected)
+			}
+		})
+	}
+}