make ModelName immutable and fix model weight

Author: hzxuzhonghu

api/v1alpha2/inferencemodel_types.go

@@ -71,6 +71,7 @@ type InferenceModelSpec struct {
 	//
 	// +kubebuilder:validation:MaxLength=256
 	// +kubebuilder:validation:Required
+	// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="modelName is immutable"
 	ModelName string `json:"modelName"`
 
 	// Criticality defines how important it is to serve the model compared to other models referencing the same pool.
@@ -175,7 +176,7 @@ type TargetModel struct {
 	// Conversely weights are optional, so long as ALL targetModels do not specify a weight.
 	//
 	// +optional
-	// +kubebuilder:validation:Minimum=0
+	// +kubebuilder:validation:Minimum=1
 	// +kubebuilder:validation:Maximum=1000000
 	Weight *int32 `json:"weight,omitempty"`
 }

config/crd/bases/inference.networking.x-k8s.io_inferencemodels.yaml

@@ -82,6 +82,9 @@ spec:
                   an error will be returned specifying that no valid target model is found.
                 maxLength: 256
                 type: string
+                x-kubernetes-validations:
+                - message: modelName is immutable
+                  rule: self == oldSelf
               poolRef:
                 description: PoolRef is a reference to the inference pool, the pool
                   must exist in the same namespace.
@@ -143,7 +146,7 @@ spec:
                         Conversely weights are optional, so long as ALL targetModels do not specify a weight.
                       format: int32
                       maximum: 1000000
-                      minimum: 0
+                      minimum: 1
                       type: integer
                   required:
                   - name

pkg/epp/datastore/datastore.go

@@ -339,18 +339,25 @@ func stripLabelKeyAliasFromLabelMap(labels map[v1alpha2.LabelKey]v1alpha2.LabelV
 }
 
 func RandomWeightedDraw(logger logr.Logger, model *v1alpha2.InferenceModel, seed int64) string {
-	var weights int32
-
 	source := rand.NewSource(rand.Int63())
 	if seed > 0 {
 		source = rand.NewSource(seed)
 	}
 	r := rand.New(source)
+
+	// all the weight values are nil, then we should return random model name
+	if model.Spec.TargetModels[0].Weight == nil {
+		index := r.Int31n(int32(len(model.Spec.TargetModels)))
+		return model.Spec.TargetModels[index].Name
+	}
+
+	var weights int32
 	for _, model := range model.Spec.TargetModels {
 		weights += *model.Weight
 	}
 	logger.V(logutil.TRACE).Info("Weights for model computed", "model", model.Name, "weights", weights)
 	randomVal := r.Int31n(weights)
+	// TODO: optimize this without using loop
 	for _, model := range model.Spec.TargetModels {
 		if randomVal < *model.Weight {
 			return model.Name

pkg/epp/datastore/datastore_test.go

@@ -280,6 +280,25 @@ func TestRandomWeightedDraw(t *testing.T) {
 			},
 			want: "v1.1",
 		},
+		{
+			name: "weighted distribution with weight unset",
+			model: &v1alpha2.InferenceModel{
+				Spec: v1alpha2.InferenceModelSpec{
+					TargetModels: []v1alpha2.TargetModel{
+						{
+							Name: "canary",
+						},
+						{
+							Name: "v1.1",
+						},
+						{
+							Name: "v1",
+						},
+					},
+				},
+			},
+			want: "v1.1",
+		},
 	}
 	var seedVal int64 = 420
 	for _, test := range tests {