embedding ready status into update so non-ready pods are deleted

Author: kfswain

pkg/ext-proc/backend/pod_reconciler.go

@@ -37,9 +37,6 @@ func (c *PodReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.R
 		return ctrl.Result{}, err
 	}
 
-	if !podIsReady(pod) {
-		return ctrl.Result{RequeueAfter: time.Second * 5}, nil
-	}
 	c.updateDatastore(pod, inferencePool)
 
 	return ctrl.Result{}, nil
@@ -56,7 +53,7 @@ func (c *PodReconciler) updateDatastore(k8sPod *corev1.Pod, inferencePool *v1alp
 		Name:    k8sPod.Name,
 		Address: k8sPod.Status.PodIP + ":" + strconv.Itoa(int(inferencePool.Spec.TargetPortNumber)),
 	}
-	if !c.Datastore.LabelsMatch(k8sPod.ObjectMeta.Labels) {
+	if !c.Datastore.LabelsMatch(k8sPod.ObjectMeta.Labels) || !podIsReady(k8sPod) {
 		c.Datastore.pods.Delete(pod)
 	} else {
 		c.Datastore.pods.Store(pod, true)

pkg/ext-proc/backend/pod_reconciler_test.go

@@ -43,9 +43,48 @@ func TestUpdateDatastore_EndpointSliceReconciler(t *testing.T) {
 						"some-key": "some-val",
 					},
 				},
+				Status: corev1.PodStatus{
+					Conditions: []corev1.PodCondition{
+						{
+							Type:   corev1.PodReady,
+							Status: corev1.ConditionTrue,
+						},
+					},
+				},
 			},
 			wantPods: []string{basePod1.Name, basePod2.Name, basePod3.Name},
 		},
+		{
+			name: "New pod, not ready, valid selector",
+			datastore: &K8sDatastore{
+				pods: populateMap(basePod1, basePod2),
+				inferencePool: &v1alpha1.InferencePool{
+					Spec: v1alpha1.InferencePoolSpec{
+						TargetPortNumber: int32(8000),
+						Selector: map[v1alpha1.LabelKey]v1alpha1.LabelValue{
+							"some-key": "some-val",
+						},
+					},
+				},
+			},
+			incomingPod: &corev1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "pod3",
+					Labels: map[string]string{
+						"some-key": "some-val",
+					},
+				},
+				Status: corev1.PodStatus{
+					Conditions: []corev1.PodCondition{
+						{
+							Type:   corev1.PodReady,
+							Status: corev1.ConditionFalse,
+						},
+					},
+				},
+			},
+			wantPods: []string{basePod1.Name, basePod2.Name},
+		},
 		{
 			name: "Remove pod that does not match selector",
 			datastore: &K8sDatastore{
@@ -66,6 +105,45 @@ func TestUpdateDatastore_EndpointSliceReconciler(t *testing.T) {
 						"some-wrong-key": "some-val",
 					},
 				},
+				Status: corev1.PodStatus{
+					Conditions: []corev1.PodCondition{
+						{
+							Type:   corev1.PodReady,
+							Status: corev1.ConditionTrue,
+						},
+					},
+				},
+			},
+			wantPods: []string{basePod2.Name},
+		},
+		{
+			name: "Remove pod that is not ready",
+			datastore: &K8sDatastore{
+				pods: populateMap(basePod1, basePod2),
+				inferencePool: &v1alpha1.InferencePool{
+					Spec: v1alpha1.InferencePoolSpec{
+						TargetPortNumber: int32(8000),
+						Selector: map[v1alpha1.LabelKey]v1alpha1.LabelValue{
+							"some-key": "some-val",
+						},
+					},
+				},
+			},
+			incomingPod: &corev1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "pod1",
+					Labels: map[string]string{
+						"some-wrong-key": "some-val",
+					},
+				},
+				Status: corev1.PodStatus{
+					Conditions: []corev1.PodCondition{
+						{
+							Type:   corev1.PodReady,
+							Status: corev1.ConditionFalse,
+						},
+					},
+				},
 			},
 			wantPods: []string{basePod2.Name},
 		},