Merge pull request #3895 from k8s-infra-cherrypick-robot/cherry-pick-3886-to-release-2.0

[release-2.0] [E2E] Fix AWS services creation flake in external infrastructure test

Author: k8s-ci-robot

test/e2e/shared/aws.go

@@ -165,36 +165,27 @@ func (i *AWSInfrastructure) AllocateAddress() AWSInfrastructure {
 		return *i
 	}
 
-	t := 0
-	addr, _ := GetAddress(i.Context, *aa.AllocationId)
-	for addr == nil && t < 180 {
-		time.Sleep(1 * time.Second)
+	var addr *ec2.Address
+	Eventually(func(gomega Gomega) {
 		addr, _ = GetAddress(i.Context, *aa.AllocationId)
-		t++
-	}
+	}, 2*time.Minute, 5*time.Second).Should(Succeed())
 	i.ElasticIP = addr
 	return *i
 }
 
 func (i *AWSInfrastructure) CreateNatGateway(ct string) AWSInfrastructure {
-	t := 0
-	s, serr := GetSubnetByName(i.Context, i.Spec.ClusterName+"-subnet-"+ct)
-	if serr != nil {
-		return *i
-	}
-	for s == nil && t < 180 {
-		time.Sleep(1 * time.Second)
+	var s *ec2.Subnet
+	Eventually(func(gomega Gomega) {
 		s, _ = GetSubnetByName(i.Context, i.Spec.ClusterName+"-subnet-"+ct)
-		t++
-	}
+	}, 2*time.Minute, 5*time.Second).Should(Succeed())
 	if s == nil {
 		return *i
 	}
 	ngwC, ngwce := CreateNatGateway(i.Context, i.Spec.ClusterName+"-nat", ct, *i.ElasticIP.AllocationId, *s.SubnetId)
 	if ngwce != nil {
 		return *i
 	}
-	if WaitForNatGatewayState(i.Context, *ngwC.NatGatewayId, 180, "available") {
+	if WaitForNatGatewayState(i.Context, *ngwC.NatGatewayId, "available") {
 		ngw, _ := GetNatGateway(i.Context, *ngwC.NatGatewayId)
 		i.NatGateway = ngw
 		i.State.NatGatewayState = ngw.State
@@ -238,6 +229,10 @@ func (i *AWSInfrastructure) GetRouteTable(rtID string) AWSInfrastructure {
 // routes to their respective gateway.
 func (i *AWSInfrastructure) CreateInfrastructure() AWSInfrastructure {
 	i.CreateVPC()
+	Eventually(func(gomega Gomega) bool {
+		return *i.RefreshVPCState().State.VpcState == "available"
+	}, 2*time.Minute, 5*time.Second).Should(BeTrue())
+
 	Byf("Created VPC - %s", *i.VPC.VpcId)
 	if i.VPC != nil {
 		i.CreatePublicSubnet()
@@ -248,12 +243,6 @@ func (i *AWSInfrastructure) CreateInfrastructure() AWSInfrastructure {
 		if i.State.PrivateSubnetID != nil {
 			Byf("Created Private Subnet - %s", *i.State.PrivateSubnetID)
 		}
-		for t := 0; t < 30; t++ {
-			if *i.RefreshVPCState().State.VpcState == "available" {
-				break
-			}
-			time.Sleep(1 * time.Second)
-		}
 		i.CreateInternetGateway()
 		if i.InternetGateway != nil {
 			Byf("Created Internet Gateway - %s", *i.InternetGateway.InternetGatewayId)
@@ -264,7 +253,7 @@ func (i *AWSInfrastructure) CreateInfrastructure() AWSInfrastructure {
 		Byf("Created Elastic IP - %s", *i.ElasticIP.AllocationId)
 		i.CreateNatGateway("public")
 		if i.NatGateway != nil && i.NatGateway.NatGatewayId != nil {
-			WaitForNatGatewayState(i.Context, *i.NatGateway.NatGatewayId, 180, "available")
+			WaitForNatGatewayState(i.Context, *i.NatGateway.NatGatewayId, "available")
 			Byf("Created NAT Gateway - %s", *i.NatGateway.NatGatewayId)
 		}
 	}
@@ -304,7 +293,7 @@ func (i *AWSInfrastructure) DeleteInfrastructure() {
 			Byf("Deleting orphaned instance: %s - %v", *instance.InstanceId, TerminateInstance(i.Context, *instance.InstanceId))
 		}
 	}
-	WaitForInstanceState(i.Context, i.Spec.ClusterName, 300, "terminated")
+	WaitForInstanceState(i.Context, i.Spec.ClusterName, "terminated")
 
 	loadbalancers, _ := ListLoadBalancers(i.Context, i.Spec.ClusterName)
 	for _, lb := range loadbalancers {
@@ -320,7 +309,7 @@ func (i *AWSInfrastructure) DeleteInfrastructure() {
 
 	if i.NatGateway != nil {
 		Byf("Deleting NAT Gateway - %s - %v", *i.NatGateway.NatGatewayId, DeleteNatGateway(i.Context, *i.NatGateway.NatGatewayId))
-		WaitForNatGatewayState(i.Context, *i.NatGateway.NatGatewayId, 180, "deleted")
+		WaitForNatGatewayState(i.Context, *i.NatGateway.NatGatewayId, "deleted")
 	}
 
 	if i.ElasticIP != nil {
@@ -1010,9 +999,8 @@ func ListClusterEC2Instances(e2eCtx *E2EContext, clusterName string) ([]*ec2.Ins
 	return instances, nil
 }
 
-func WaitForInstanceState(e2eCtx *E2EContext, clusterName string, timeout int, state string) bool {
-	t := 0
-	for t < timeout {
+func WaitForInstanceState(e2eCtx *E2EContext, clusterName string, state string) bool {
+	Eventually(func(gomega Gomega) bool {
 		st := map[string]int{
 			"pending":       0,
 			"running":       0,
@@ -1027,9 +1015,9 @@ func WaitForInstanceState(e2eCtx *E2EContext, clusterName string, timeout int, s
 		if st[state] == len(instances) || len(instances) == 0 {
 			return true
 		}
-		time.Sleep(1 * time.Second)
-		t++
-	}
+		return false
+	}, 5*time.Minute, 5*time.Second).Should(BeTrue())
+
 	return false
 }
 
@@ -1417,17 +1405,12 @@ func DeleteNatGateway(e2eCtx *E2EContext, gatewayID string) bool {
 	return true
 }
 
-func WaitForNatGatewayState(e2eCtx *E2EContext, gatewayID string, timeout int, state string) bool {
-	t := 0
-	for t < timeout {
+func WaitForNatGatewayState(e2eCtx *E2EContext, gatewayID string, state string) bool {
+	Eventually(func(gomega Gomega) bool {
 		gw, _ := GetNatGateway(e2eCtx, gatewayID)
 		gwState := *gw.State
-		if gwState == state {
-			return true
-		}
-		time.Sleep(1 * time.Second)
-		t++
-	}
+		return gwState == state
+	}, 3*time.Minute, 5*time.Second).Should(BeTrue())
 	return false
 }