From e332cf423e3682be705d96251c7b95a1c438e68d Mon Sep 17 00:00:00 2001 From: Zhonghu Xu Date: Tue, 25 Feb 2025 15:28:21 +0800 Subject: [PATCH 1/4] Add event filter to the reconciler builder --- pkg/epp/controller/inferencemodel_reconciler.go | 11 ++++++----- pkg/epp/controller/inferencepool_reconciler.go | 11 ++++++----- pkg/epp/controller/pod_reconciler.go | 14 ++++++++++---- pkg/epp/server/runserver.go | 1 + 4 files changed, 23 insertions(+), 14 deletions(-) diff --git a/pkg/epp/controller/inferencemodel_reconciler.go b/pkg/epp/controller/inferencemodel_reconciler.go index 00358740c..294751122 100644 --- a/pkg/epp/controller/inferencemodel_reconciler.go +++ b/pkg/epp/controller/inferencemodel_reconciler.go @@ -27,6 +27,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/gateway-api-inference-extension/api/v1alpha2" "sigs.k8s.io/gateway-api-inference-extension/pkg/epp/datastore" logutil "sigs.k8s.io/gateway-api-inference-extension/pkg/epp/util/logging" @@ -41,10 +42,6 @@ type InferenceModelReconciler struct { } func (c *InferenceModelReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - if req.Namespace != c.PoolNamespacedName.Namespace { - return ctrl.Result{}, nil - } - logger := log.FromContext(ctx) loggerDefault := logger.V(logutil.DEFAULT) loggerDefault.Info("Reconciling InferenceModel", "name", req.NamespacedName) @@ -83,7 +80,11 @@ func (c *InferenceModelReconciler) updateDatastore(logger logr.Logger, infModel } func (c *InferenceModelReconciler) SetupWithManager(mgr ctrl.Manager) error { + // Filter inference model within same namespace as the pool + p := predicate.NewPredicateFuncs(func(object client.Object) bool { + return object.GetNamespace() == c.PoolNamespacedName.Namespace && object.GetName() == c.PoolNamespacedName.Name + }) return ctrl.NewControllerManagedBy(mgr). - For(&v1alpha2.InferenceModel{}). + For(&v1alpha2.InferenceModel{}).WithEventFilter(p). Complete(c) } diff --git a/pkg/epp/controller/inferencepool_reconciler.go b/pkg/epp/controller/inferencepool_reconciler.go index baf3332b5..d30af71df 100644 --- a/pkg/epp/controller/inferencepool_reconciler.go +++ b/pkg/epp/controller/inferencepool_reconciler.go @@ -27,6 +27,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/gateway-api-inference-extension/api/v1alpha2" "sigs.k8s.io/gateway-api-inference-extension/pkg/epp/datastore" logutil "sigs.k8s.io/gateway-api-inference-extension/pkg/epp/util/logging" @@ -44,10 +45,6 @@ type InferencePoolReconciler struct { } func (c *InferencePoolReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - if req.NamespacedName.Name != c.PoolNamespacedName.Name || req.NamespacedName.Namespace != c.PoolNamespacedName.Namespace { - return ctrl.Result{}, nil - } - logger := log.FromContext(ctx) loggerDefault := logger.V(logutil.DEFAULT) loggerDefault.Info("Reconciling InferencePool", "name", req.NamespacedName) @@ -90,7 +87,11 @@ func (c *InferencePoolReconciler) updateDatastore(ctx context.Context, newPool * } func (c *InferencePoolReconciler) SetupWithManager(mgr ctrl.Manager) error { + // Filter specific inference pool + p := predicate.NewPredicateFuncs(func(object client.Object) bool { + return object.GetNamespace() == c.PoolNamespacedName.Namespace && object.GetName() == c.PoolNamespacedName.Name + }) return ctrl.NewControllerManagedBy(mgr). - For(&v1alpha2.InferencePool{}). + For(&v1alpha2.InferencePool{}).WithEventFilter(p). Complete(c) } diff --git a/pkg/epp/controller/pod_reconciler.go b/pkg/epp/controller/pod_reconciler.go index 5b0c25c99..9a6c8d6f2 100644 --- a/pkg/epp/controller/pod_reconciler.go +++ b/pkg/epp/controller/pod_reconciler.go @@ -28,11 +28,15 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/gateway-api-inference-extension/pkg/epp/datastore" logutil "sigs.k8s.io/gateway-api-inference-extension/pkg/epp/util/logging" ) type PodReconciler struct { + // namespace of the InferencePool + // we donot support cross namespace pod selection + Namespace string client.Client Datastore datastore.Datastore Scheme *runtime.Scheme @@ -41,13 +45,11 @@ type PodReconciler struct { func (c *PodReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { logger := log.FromContext(ctx) - inferencePool, err := c.Datastore.PoolGet() + _, err := c.Datastore.PoolGet() if err != nil { logger.V(logutil.TRACE).Info("Skipping reconciling Pod because the InferencePool is not available yet", "error", err) // When the inferencePool is initialized it lists the appropriate pods and populates the datastore, so no need to requeue. return ctrl.Result{}, nil - } else if inferencePool.Namespace != req.Namespace { - return ctrl.Result{}, nil } logger.V(logutil.VERBOSE).Info("Pod being reconciled", "name", req.NamespacedName) @@ -67,8 +69,12 @@ func (c *PodReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.R } func (c *PodReconciler) SetupWithManager(mgr ctrl.Manager) error { + // Filter specific inference pool + p := predicate.NewPredicateFuncs(func(object client.Object) bool { + return object.GetNamespace() == c.Namespace + }) return ctrl.NewControllerManagedBy(mgr). - For(&corev1.Pod{}). + For(&corev1.Pod{}).WithEventFilter(p). Complete(c) } diff --git a/pkg/epp/server/runserver.go b/pkg/epp/server/runserver.go index 6e6b68b1a..0fd19dc63 100644 --- a/pkg/epp/server/runserver.go +++ b/pkg/epp/server/runserver.go @@ -114,6 +114,7 @@ func (r *ExtProcServerRunner) SetupWithManager(mgr ctrl.Manager) error { } if err := (&controller.PodReconciler{ + Namespace: r.PoolNamespace, Datastore: r.Datastore, Scheme: mgr.GetScheme(), Client: mgr.GetClient(), From 5409b840aee91a850778c730e800d50e014bdda0 Mon Sep 17 00:00:00 2001 From: Zhonghu Xu Date: Tue, 25 Feb 2025 16:09:24 +0800 Subject: [PATCH 2/4] Fix CI --- .../controller/inferencepool_reconciler_test.go | 14 +++----------- pkg/epp/test/utils.go | 7 +++++-- test/integration/hermetic_test.go | 1 + 3 files changed, 9 insertions(+), 13 deletions(-) diff --git a/pkg/epp/controller/inferencepool_reconciler_test.go b/pkg/epp/controller/inferencepool_reconciler_test.go index a96406f04..26b81d9a4 100644 --- a/pkg/epp/controller/inferencepool_reconciler_test.go +++ b/pkg/epp/controller/inferencepool_reconciler_test.go @@ -102,15 +102,7 @@ func TestReconcile_InferencePoolReconciler(t *testing.T) { t.Errorf("Unexpected diff (+got/-want): %s", diff) } - // Step 2: A reconcile on pool2 should not change anything. - if _, err := inferencePoolReconciler.Reconcile(ctx, ctrl.Request{NamespacedName: types.NamespacedName{Name: pool2.Name, Namespace: pool2.Namespace}}); err != nil { - t.Errorf("Unexpected InferencePool reconcile error: %v", err) - } - if diff := diffPool(datastore, pool1, []string{"pod1", "pod2"}); diff != "" { - t.Errorf("Unexpected diff (+got/-want): %s", diff) - } - - // Step 3: update the pool selector to include more pods + // Step 2: update the pool selector to include more pods newPool1 := &v1alpha2.InferencePool{} if err := fakeClient.Get(ctx, req.NamespacedName, newPool1); err != nil { t.Errorf("Unexpected pool get error: %v", err) @@ -127,7 +119,7 @@ func TestReconcile_InferencePoolReconciler(t *testing.T) { t.Errorf("Unexpected diff (+got/-want): %s", diff) } - // Step 4: update the pool port + // Step 3: update the pool port if err := fakeClient.Get(ctx, req.NamespacedName, newPool1); err != nil { t.Errorf("Unexpected pool get error: %v", err) } @@ -142,7 +134,7 @@ func TestReconcile_InferencePoolReconciler(t *testing.T) { t.Errorf("Unexpected diff (+got/-want): %s", diff) } - // Step 5: delete the pool to trigger a datastore clear + // Step 4: delete the pool to trigger a datastore clear if err := fakeClient.Get(ctx, req.NamespacedName, newPool1); err != nil { t.Errorf("Unexpected pool get error: %v", err) } diff --git a/pkg/epp/test/utils.go b/pkg/epp/test/utils.go index 6a75ed2ff..3f141e56f 100644 --- a/pkg/epp/test/utils.go +++ b/pkg/epp/test/utils.go @@ -116,8 +116,11 @@ func FakePodMetrics(index int, metrics datastore.Metrics) *datastore.PodMetrics address := fmt.Sprintf("address-%v", index) pod := datastore.PodMetrics{ Pod: datastore.Pod{ - NamespacedName: types.NamespacedName{Name: fmt.Sprintf("pod-%v", index)}, - Address: address, + NamespacedName: types.NamespacedName{ + Name: fmt.Sprintf("pod-%v", index), + Namespace: "default", + }, + Address: address, }, Metrics: metrics, } diff --git a/test/integration/hermetic_test.go b/test/integration/hermetic_test.go index 85c49913a..82b82b649 100644 --- a/test/integration/hermetic_test.go +++ b/test/integration/hermetic_test.go @@ -426,6 +426,7 @@ func BeforeSuit(t *testing.T) func() { serverRunner = runserver.NewDefaultExtProcServerRunner() // Adjust from defaults serverRunner.PoolName = "vllm-llama2-7b-pool" + serverRunner.PoolNamespace = "default" serverRunner.Datastore = datastore.NewDatastore() serverRunner.SecureServing = false From deeaac04235f3e95d9220df05c1ef51c2d817ae2 Mon Sep 17 00:00:00 2001 From: Zhonghu Xu Date: Tue, 25 Feb 2025 19:51:33 +0800 Subject: [PATCH 3/4] update --- test/integration/hermetic_test.go | 1 - test/testdata/client.yaml | 1 + test/testdata/envoy.yaml | 3 +++ test/testdata/model-secret.yaml | 1 + 4 files changed, 5 insertions(+), 1 deletion(-) diff --git a/test/integration/hermetic_test.go b/test/integration/hermetic_test.go index 82b82b649..85c49913a 100644 --- a/test/integration/hermetic_test.go +++ b/test/integration/hermetic_test.go @@ -426,7 +426,6 @@ func BeforeSuit(t *testing.T) func() { serverRunner = runserver.NewDefaultExtProcServerRunner() // Adjust from defaults serverRunner.PoolName = "vllm-llama2-7b-pool" - serverRunner.PoolNamespace = "default" serverRunner.Datastore = datastore.NewDatastore() serverRunner.SecureServing = false diff --git a/test/testdata/client.yaml b/test/testdata/client.yaml index a637de39f..0e7997c7a 100644 --- a/test/testdata/client.yaml +++ b/test/testdata/client.yaml @@ -5,6 +5,7 @@ metadata: labels: app: curl name: curl + namespace: default spec: containers: - command: diff --git a/test/testdata/envoy.yaml b/test/testdata/envoy.yaml index ffb8add78..6575c1d61 100644 --- a/test/testdata/envoy.yaml +++ b/test/testdata/envoy.yaml @@ -2,6 +2,7 @@ apiVersion: v1 kind: ConfigMap metadata: name: envoy + namespace: default labels: app: envoy data: @@ -203,6 +204,7 @@ apiVersion: apps/v1 kind: Deployment metadata: name: envoy + namespace: default labels: app: envoy spec: @@ -283,6 +285,7 @@ metadata: labels: app: envoy name: envoy + namespace: default spec: ports: - name: http-8081 diff --git a/test/testdata/model-secret.yaml b/test/testdata/model-secret.yaml index 01c2556c3..b54609faf 100644 --- a/test/testdata/model-secret.yaml +++ b/test/testdata/model-secret.yaml @@ -2,6 +2,7 @@ apiVersion: v1 kind: Secret metadata: name: hf-token + namespace: default labels: app: vllm stringData: From 58675ed87d5a1335ed4db9cc013b0ed9c7e75b15 Mon Sep 17 00:00:00 2001 From: Zhonghu Xu Date: Tue, 25 Feb 2025 20:15:13 +0800 Subject: [PATCH 4/4] update --- pkg/epp/controller/inferencemodel_reconciler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/epp/controller/inferencemodel_reconciler.go b/pkg/epp/controller/inferencemodel_reconciler.go index 294751122..98e0fc713 100644 --- a/pkg/epp/controller/inferencemodel_reconciler.go +++ b/pkg/epp/controller/inferencemodel_reconciler.go @@ -82,7 +82,7 @@ func (c *InferenceModelReconciler) updateDatastore(logger logr.Logger, infModel func (c *InferenceModelReconciler) SetupWithManager(mgr ctrl.Manager) error { // Filter inference model within same namespace as the pool p := predicate.NewPredicateFuncs(func(object client.Object) bool { - return object.GetNamespace() == c.PoolNamespacedName.Namespace && object.GetName() == c.PoolNamespacedName.Name + return object.GetNamespace() == c.PoolNamespacedName.Namespace }) return ctrl.NewControllerManagedBy(mgr). For(&v1alpha2.InferenceModel{}).WithEventFilter(p).