Enabling full duplex streaming for request & response flows.

Author: kfswain

cmd/epp/main.go

@@ -110,6 +110,11 @@ func run() error {
 	flag.Parse()
 	initLogging(&opts)
 
+	useStreamingServer, err := strconv.ParseBool(os.Getenv("USE_STREAMING"))
+	if err != nil {
+		setupLog.Error(err, "Failed to parse env var USE_STREAMING, defaulting to false")
+	}
+
 	// Validate flags
 	if err := validateFlags(); err != nil {
 		setupLog.Error(err, "Failed to validate flags")
@@ -153,6 +158,7 @@ func run() error {
 		SecureServing:                            *secureServing,
 		CertPath:                                 *certPath,
 		Provider:                                 provider,
+		UseStreaming:                             useStreamingServer,
 	}
 	if err := serverRunner.SetupWithManager(ctx, mgr); err != nil {
 		setupLog.Error(err, "Failed to setup ext-proc controllers")

config/manifests/ext_proc.yaml

@@ -71,17 +71,20 @@ spec:
     spec:
       containers:
       - name: inference-gateway-ext-proc
-        image: us-east1-docker.pkg.dev/kfswain-gke-dev/test-repo/ext-proc:test-feb-32
+        image: us-east1-docker.pkg.dev/kfswain-gke-dev/test-repo/ext-proc:test-mar-35
         imagePullPolicy: Always
         args:
         - -poolName
         - "my-pool"
         - -v
-        - "3"
+        - "4"
         - -grpcPort
         - "9002"
         - -grpcHealthPort
         - "9003"
+        env:
+        - name: USE_STREAMING
+          value: "true"
         ports:
         - containerPort: 9002
         - containerPort: 9003

config/manifests/gateway/patch_policy.yaml

@@ -66,3 +66,23 @@ spec:
         op: add
         path: "/default_filter_chain/filters/0/typed_config/http_filters/0/typed_config/processing_mode/request_trailer_mode"
         value: SEND
+    # Uncomment to enable full duplex streaming
+    # - type: "type.googleapis.com/envoy.config.listener.v3.Listener"
+    #   name: "default/inference-gateway/llm-gw"
+    #   operation:
+    #     op: add
+    #     path: "/default_filter_chain/filters/0/typed_config/http_filters/0/typed_config/processing_mode/response_body_mode"
+    #     value: FULL_DUPLEX_STREAMED
+    # - type: "type.googleapis.com/envoy.config.listener.v3.Listener"
+    #   name: "default/inference-gateway/llm-gw"
+    #   operation:
+    #     op: replace
+    #     path: "/default_filter_chain/filters/0/typed_config/http_filters/0/typed_config/processing_mode/response_trailer_mode"
+    #     value: SEND
+    # - type: "type.googleapis.com/envoy.config.listener.v3.Listener"
+    #   name: "default/inference-gateway/llm-gw"
+    #   operation:
+    #     op: replace
+    #     path: "/default_filter_chain/filters/0/typed_config/http_filters/0/typed_config/processing_mode/response_header_mode"
+    #     value: SEND
+