Merge branch 'kubernetes-sigs:main' into inferencepool-ref

Author: nicolexin

cmd/epp/main.go

@@ -30,6 +30,7 @@ import (
 	"go.uber.org/zap/zapcore"
 	"google.golang.org/grpc"
 	healthPb "google.golang.org/grpc/health/grpc_health_v1"
+	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/rest"
 	"k8s.io/component-base/metrics/legacyregistry"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -140,7 +141,11 @@ func run() error {
 		return err
 	}
 
-	mgr, err := runserver.NewDefaultManager(*poolNamespace, *poolName, cfg)
+	poolNamespacedName := types.NamespacedName{
+		Namespace: *poolNamespace,
+		Name:      *poolName,
+	}
+	mgr, err := runserver.NewDefaultManager(poolNamespacedName, cfg)
 	if err != nil {
 		setupLog.Error(err, "Failed to create controller manager")
 		return err

config/manifests/inferencepool-resources.yaml

@@ -4,7 +4,6 @@
 apiVersion: inference.networking.x-k8s.io/v1alpha2
 kind: InferencePool
 metadata:
-  labels:
   name: vllm-llama3-8b-instruct
 spec:
   targetPortNumber: 8000
@@ -54,6 +53,8 @@ spec:
         args:
         - -poolName
         - "vllm-llama3-8b-instruct"
+        - "-poolNamespace"
+        - "default"
         - -v
         - "4"
         - --zap-encoder

pkg/epp/controller/inferencepool_reconciler.go

@@ -21,7 +21,6 @@ import (
 	"reflect"
 
 	"k8s.io/apimachinery/pkg/api/errors"
-	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/tools/record"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -36,9 +35,8 @@ import (
 // will have the proper controller that will create/manage objects on behalf of the server pool.
 type InferencePoolReconciler struct {
 	client.Client
-	Record             record.EventRecorder
-	PoolNamespacedName types.NamespacedName
-	Datastore          datastore.Datastore
+	Record    record.EventRecorder
+	Datastore datastore.Datastore
 }
 
 func (c *InferencePoolReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {

pkg/epp/controller/inferencepool_reconciler_test.go

@@ -96,7 +96,7 @@ func TestInferencePoolReconciler(t *testing.T) {
 
 	pmf := backendmetrics.NewPodMetricsFactory(&backendmetrics.FakePodMetricsClient{}, time.Second)
 	datastore := datastore.NewDatastore(ctx, pmf)
-	inferencePoolReconciler := &InferencePoolReconciler{PoolNamespacedName: namespacedName, Client: fakeClient, Datastore: datastore}
+	inferencePoolReconciler := &InferencePoolReconciler{Client: fakeClient, Datastore: datastore}
 
 	// Step 1: Inception, only ready pods matching pool1 are added to the store.
 	if _, err := inferencePoolReconciler.Reconcile(ctx, req); err != nil {

pkg/epp/server/controller_manager.go

@@ -22,6 +22,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/fields"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/rest"
@@ -39,29 +40,29 @@ func init() {
 	utilruntime.Must(v1alpha2.Install(scheme))
 }
 
-// DefaultManagerOptions returns the default options used to create the manager.
-func DefaultManagerOptions(namespace, name string) ctrl.Options {
+// defaultManagerOptions returns the default options used to create the manager.
+func defaultManagerOptions(namespacedName types.NamespacedName) ctrl.Options {
 	return ctrl.Options{
 		Scheme: scheme,
 		Cache: cache.Options{
 			ByObject: map[client.Object]cache.ByObject{
 				&corev1.Pod{}: {
 					Namespaces: map[string]cache.Config{
-						namespace: {},
+						namespacedName.Namespace: {},
 					},
 				},
 				&v1alpha2.InferencePool{}: {
 					Namespaces: map[string]cache.Config{
-						namespace: {
+						namespacedName.Namespace: {
 							FieldSelector: fields.SelectorFromSet(fields.Set{
-								"metadata.name": name,
+								"metadata.name": namespacedName.Name,
 							}),
 						},
 					},
 				},
 				&v1alpha2.InferenceModel{}: {
 					Namespaces: map[string]cache.Config{
-						namespace: {},
+						namespacedName.Namespace: {},
 					},
 				},
 			},
@@ -70,8 +71,8 @@ func DefaultManagerOptions(namespace, name string) ctrl.Options {
 }
 
 // NewDefaultManager creates a new controller manager with default configuration.
-func NewDefaultManager(namespace, name string, restConfig *rest.Config) (ctrl.Manager, error) {
-	manager, err := ctrl.NewManager(restConfig, DefaultManagerOptions(namespace, name))
+func NewDefaultManager(namespacedName types.NamespacedName, restConfig *rest.Config) (ctrl.Manager, error) {
+	manager, err := ctrl.NewManager(restConfig, defaultManagerOptions(namespacedName))
 	if err != nil {
 		return nil, fmt.Errorf("failed to create controller manager: %v", err)
 	}

pkg/epp/server/runserver.go

@@ -87,11 +87,7 @@ func (r *ExtProcServerRunner) SetupWithManager(ctx context.Context, mgr ctrl.Man
 	if err := (&controller.InferencePoolReconciler{
 		Datastore: r.Datastore,
 		Client:    mgr.GetClient(),
-		PoolNamespacedName: types.NamespacedName{
-			Name:      r.PoolName,
-			Namespace: r.PoolNamespace,
-		},
-		Record: mgr.GetEventRecorderFor("InferencePool"),
+		Record:    mgr.GetEventRecorderFor("InferencePool"),
 	}).SetupWithManager(mgr); err != nil {
 		return fmt.Errorf("failed setting up InferencePoolReconciler: %w", err)
 	}

site-src/guides/index.md

@@ -119,9 +119,9 @@ This quickstart guide is intended for engineers familiar with k8s and model serv
 
       5. Given that the default connection timeout may be insufficient for most inference workloads, it is recommended to configure a timeout appropriate for your intended use case.
 
-      ```bash
-      kubectl apply -f https://github.com/kubernetes-sigs/gateway-api-inference-extension/raw/main/config/manifests/gateway/gke/gcp-backend-policy.yaml
-      ```
+         ```bash
+         kubectl apply -f https://github.com/kubernetes-sigs/gateway-api-inference-extension/raw/main/config/manifests/gateway/gke/gcp-backend-policy.yaml
+         ```
 
 === "Istio"
 
@@ -269,10 +269,10 @@ This quickstart guide is intended for engineers familiar with k8s and model serv
 
 ### Cleanup
 
-   The following cleanup assumes you would like to clean ALL resources that were created in this quickstart guide.  
+   The following instructions assume you would like to cleanup ALL resources that were created in this quickstart guide.
    Please be careful not to delete resources you'd like to keep.
 
-   1. Uninstall the Inference Pool
+   1. Uninstall the InferencePool, InferenceModel, and model server resources
 
       ```bash
       kubectl delete -f https://github.com/kubernetes-sigs/gateway-api-inference-extension/raw/main/config/manifests/inferencepool-resources.yaml --ignore-not-found
@@ -282,7 +282,7 @@ This quickstart guide is intended for engineers familiar with k8s and model serv
       kubectl delete secret hf-token --ignore-not-found
       ```
 
-   1. Uninstall the Gateway
+   1. Uninstall the Gateway API resources
 
       ```bash
       kubectl delete -f https://github.com/kubernetes-sigs/gateway-api-inference-extension/raw/main/config/manifests/gateway/gke/gateway.yaml --ignore-not-found
@@ -296,8 +296,40 @@ This quickstart guide is intended for engineers familiar with k8s and model serv
       kubectl delete -f https://github.com/kubernetes-sigs/gateway-api-inference-extension/raw/main/config/manifests/gateway/kgateway/httproute.yaml --ignore-not-found
       ```
 
-   1. Uninstall the CRDs
+   1. Uninstall the Gateway API Inference Extension CRDs
 
       ```bash
       kubectl delete -k https://github.com/kubernetes-sigs/gateway-api-inference-extension/config/crd --ignore-not-found
       ```
+
+   1. Choose one of the following options to cleanup the Inference Gateway.
+
+=== "GKE"
+
+      **TODO**
+
+=== "Istio"
+
+      **TODO**
+
+=== "Kgateway"
+
+      The following instructions assume you would like to cleanup ALL Kgateway resources that were created in this quickstart guide.
+
+      1. Uninstall Kgateway
+
+         ```bash
+         helm uninstall kgateway -n kgateway-system
+         ```
+
+      1. Uninstall the Kgateway CRDs.
+
+         ```bash
+         helm uninstall kgateway-crds -n kgateway-system
+         ```
+
+      1. Remove the Kgateway namespace.
+
+         ```bash
+         kubectl delete ns kgateway-system
+         ```

site-src/implementations/gateways.md

@@ -5,23 +5,25 @@ This project has several implementations that are planned or in progress:
 * [Envoy AI Gateway][1]
 * [Kgateway][2]
 * [Google Kubernetes Engine][3]
+* [Istio][4]
 
 [1]:#envoy-gateway
 [2]:#kgateway
 [3]:#google-kubernetes-engine
+[4]:#istio
 
 ## Envoy AI Gateway
 
 [Envoy AI Gateway][aigw-home] is an open source project built on top of 
-[Envoy][envoy-org] and [Envoy Gateway][aigw-gateway] to handle request traffic 
+[Envoy][envoy-org] and [Envoy Gateway][envoy-gateway] to handle request traffic 
 from application clients to GenAI services. The features and capabilities are outlined [here][aigw-capabilities]. Use the [quickstart][aigw-quickstart] to get Envoy AI Gateway running with Gateway API in a few simple steps.
 
 Progress towards supporting this project is tracked with a [GitHub
 Issue](https://github.com/envoyproxy/ai-gateway/issues/423).
 
-[aigw-home]:https://gateway.envoyproxy.io/
+[aigw-home]:https://aigateway.envoyproxy.io/
 [envoy-org]:https://github.com/envoyproxy
-[aigw-gateway]: https://gateway.envoyproxy.io/
+[envoy-gateway]: https://gateway.envoyproxy.io/
 [aigw-capabilities]:https://aigateway.envoyproxy.io/docs/capabilities/
 [aigw-quickstart]:https://aigateway.envoyproxy.io/docs/capabilities/gateway-api-inference-extension