Make network & VPC offering configurable

api/v1beta3/cloudstackfailuredomain_types.go

@@ -62,6 +62,13 @@ type Network struct {
 	// +optional
 	Netmask string `json:"netmask,omitempty"`
 
+	// Cloudstack Network Offering the cluster is built in.
+	// Default is "DefaultIsolatedNetworkOfferingWithSourceNatService" for
+	// isolated networks and "DefaultIsolatedNetworkOfferingForVpcNetworks"
+	// for VPC networks.
+	// +optional
+	Offering string `json:"offering,omitempty"`
+
 	// Cloudstack VPC the network belongs to.
 	// +optional
 	VPC *VPC `json:"vpc,omitempty"`
@@ -79,6 +86,11 @@ type VPC struct {
 	// CIDR for the VPC.
 	// +optional
 	CIDR string `json:"cidr,omitempty"`
+
+	// Cloudstack VPC Offering for the network.
+	// Default is "Default VPC offering"
+	// +optional
+	Offering string `json:"offering,omitempty"`
 }
 
 // CloudStackZoneSpec specifies a Zone's details.

api/v1beta3/cloudstackisolatednetwork_types.go

@@ -48,6 +48,13 @@ type CloudStackIsolatedNetworkSpec struct {
 	// +optional
 	Netmask string `json:"netmask,omitempty"`
 
+	// Offering for the network.
+	// Default is "DefaultIsolatedNetworkOfferingWithSourceNatService" for
+	// isolated networks and "DefaultIsolatedNetworkOfferingForVpcNetworks"
+	// for VPC networks.
+	// +optional
+	Offering string `json:"offering,omitempty"`
+
 	// VPC the network belongs to.
 	// +optional
 	VPC *VPC `json:"vpc,omitempty"`
@@ -67,12 +74,13 @@ type CloudStackIsolatedNetworkStatus struct {
 
 func (n *CloudStackIsolatedNetwork) Network() *Network {
 	return &Network{
-		Name:    n.Spec.Name,
-		Type:    "IsolatedNetwork",
-		ID:      n.Spec.ID,
-		Gateway: n.Spec.Gateway,
-		Netmask: n.Spec.Netmask,
-		VPC:     n.Spec.VPC,
+		Name:     n.Spec.Name,
+		Type:     "IsolatedNetwork",
+		ID:       n.Spec.ID,
+		Gateway:  n.Spec.Gateway,
+		Netmask:  n.Spec.Netmask,
+		VPC:      n.Spec.VPC,
+		Offering: n.Spec.Offering,
 	}
 }
 

config/crd/bases/infrastructure.cluster.x-k8s.io_cloudstackclusters.yaml

@@ -407,6 +407,12 @@ spec:
                               description: Cloudstack Network Netmask the cluster
                                 is built in.
                               type: string
+                            offering:
+                              description: Cloudstack Network Offering the cluster
+                                is built in. Default is "DefaultIsolatedNetworkOfferingWithSourceNatService"
+                                for isolated networks and "DefaultIsolatedNetworkOfferingForVpcNetworks"
+                                for VPC networks.
+                              type: string
                             type:
                               description: Cloudstack Network Type the cluster is
                                 built in.
@@ -423,6 +429,10 @@ spec:
                                 name:
                                   description: Cloudstack VPC Name of the network.
                                   type: string
+                                offering:
+                                  description: Cloudstack VPC Offering for the network.
+                                    Default is "Default VPC offering"
+                                  type: string
                               type: object
                           required:
                           - name

config/crd/bases/infrastructure.cluster.x-k8s.io_cloudstackfailuredomains.yaml

@@ -184,6 +184,12 @@ spec:
                         description: Cloudstack Network Netmask the cluster is built
                           in.
                         type: string
+                      offering:
+                        description: Cloudstack Network Offering the cluster is built
+                          in. Default is "DefaultIsolatedNetworkOfferingWithSourceNatService"
+                          for isolated networks and "DefaultIsolatedNetworkOfferingForVpcNetworks"
+                          for VPC networks.
+                        type: string
                       type:
                         description: Cloudstack Network Type the cluster is built
                           in.
@@ -200,6 +206,10 @@ spec:
                           name:
                             description: Cloudstack VPC Name of the network.
                             type: string
+                          offering:
+                            description: Cloudstack VPC Offering for the network.
+                              Default is "Default VPC offering"
+                            type: string
                         type: object
                     required:
                     - name

config/crd/bases/infrastructure.cluster.x-k8s.io_cloudstackisolatednetworks.yaml

@@ -204,6 +204,11 @@ spec:
               netmask:
                 description: Netmask for the network.
                 type: string
+              offering:
+                description: Offering for the network. Default is "DefaultIsolatedNetworkOfferingWithSourceNatService"
+                  for isolated networks and "DefaultIsolatedNetworkOfferingForVpcNetworks"
+                  for VPC networks.
+                type: string
               vpc:
                 description: VPC the network belongs to.
                 properties:
@@ -216,6 +221,10 @@ spec:
                   name:
                     description: Cloudstack VPC Name of the network.
                     type: string
+                  offering:
+                    description: Cloudstack VPC Offering for the network. Default
+                      is "Default VPC offering"
+                    type: string
                 type: object
             required:
             - controlPlaneEndpoint

controllers/utils/isolated_network.go

@@ -45,6 +45,7 @@ func (r *ReconciliationRunner) GenerateIsolatedNetwork(name string, fdNameFunc f
 		csIsoNet.Spec.ControlPlaneEndpoint.Port = r.CSCluster.Spec.ControlPlaneEndpoint.Port
 		csIsoNet.Spec.Gateway = network.Gateway
 		csIsoNet.Spec.Netmask = network.Netmask
+		csIsoNet.Spec.Offering = network.Offering
 
 		if network.VPC != nil {
 			csIsoNet.Spec.VPC = network.VPC

pkg/cloud/isolated_network.go

@@ -112,6 +112,10 @@ func (c *client) CreateIsolatedNetwork(fd *infrav1.CloudStackFailureDomain, isoN
 		offeringName = NetVPCOffering
 	}
 
+	if isoNet.Spec.Offering != "" {
+		offeringName = isoNet.Spec.Offering
+	}
+
 	// Get network offering ID.
 	offeringID, err := c.getOfferingID(offeringName)
 	if err != nil {
@@ -153,6 +157,19 @@ func (c *client) OpenFirewallRules(isoNet *infrav1.CloudStackIsolatedNetwork) (r
 	if isoNet.Spec.VPC != nil && isoNet.Spec.VPC.ID != "" {
 		return nil
 	}
+
+	// If network's egress policy is true, then we don't need to open the firewall rules for all protocols
+	network, count, err := c.cs.Network.GetNetworkByID(isoNet.Spec.ID, cloudstack.WithProject(c.user.Project.ID))
+	if err != nil {
+		return errors.Wrapf(err, "failed to get network by ID %s", isoNet.Spec.ID)
+	}
+	if count == 0 {
+		return errors.Errorf("no network found with ID %s", isoNet.Spec.ID)
+	}
+	if network.Egressdefaultpolicy {
+		return nil
+	}
+
 	protocols := []string{NetworkProtocolTCP, NetworkProtocolUDP, NetworkProtocolICMP}
 	for _, proto := range protocols {
 		p := c.cs.Firewall.NewCreateEgressFirewallRuleParams(isoNet.Spec.ID, proto)

pkg/cloud/isolated_network_test.go

@@ -84,6 +84,7 @@ var _ = ginkgo.Describe("Network", func() {
 					PublicIpAddresses: []*csapi.PublicIpAddress{{Id: dummies.PublicIPID, Ipaddress: "fakeIP"}}}, nil)
 			as.EXPECT().NewAssociateIpAddressParams().Return(&csapi.AssociateIpAddressParams{})
 			as.EXPECT().AssociateIpAddress(gomock.Any())
+			ns.EXPECT().GetNetworkByID(dummies.ISONet1.ID, gomock.Any()).Return(&csapi.Network{Egressdefaultpolicy: false}, 1, nil)
 			fs.EXPECT().NewCreateEgressFirewallRuleParams(dummies.ISONet1.ID, gomock.Any()).
 				DoAndReturn(func(_ string, protocol string) *csapi.CreateEgressFirewallRuleParams {
 					p := &csapi.CreateEgressFirewallRuleParams{}
@@ -138,6 +139,7 @@ var _ = ginkgo.Describe("Network", func() {
 	ginkgo.Context("for a closed firewall", func() {
 		ginkgo.It("OpenFirewallRule asks CloudStack to open the firewall", func() {
 			dummies.Zone1.Network = dummies.ISONet1
+			ns.EXPECT().GetNetworkByID(dummies.ISONet1.ID, gomock.Any()).Return(&csapi.Network{Egressdefaultpolicy: false}, 1, nil)
 			fs.EXPECT().NewCreateEgressFirewallRuleParams(dummies.ISONet1.ID, gomock.Any()).
 				DoAndReturn(func(_ string, protocol string) *csapi.CreateEgressFirewallRuleParams {
 					p := &csapi.CreateEgressFirewallRuleParams{}
@@ -165,6 +167,7 @@ var _ = ginkgo.Describe("Network", func() {
 		ginkgo.It("OpenFirewallRule asks CloudStack to open the firewall anyway, but doesn't fail", func() {
 			dummies.Zone1.Network = dummies.ISONet1
 
+			ns.EXPECT().GetNetworkByID(dummies.ISONet1.ID, gomock.Any()).Return(&csapi.Network{Egressdefaultpolicy: false}, 1, nil)
 			fs.EXPECT().NewCreateEgressFirewallRuleParams(dummies.ISONet1.ID, gomock.Any()).
 				DoAndReturn(func(_ string, protocol string) *csapi.CreateEgressFirewallRuleParams {
 					p := &csapi.CreateEgressFirewallRuleParams{}

pkg/cloud/network.go

@@ -65,6 +65,7 @@ func (c *client) ResolveNetwork(net *infrav1.Network) (retErr error) {
 		net.Type = netDetails.Type
 		net.Gateway = netDetails.Gateway
 		net.Netmask = netDetails.Netmask
+		net.Offering = netDetails.Networkofferingname
 		if netDetails.Vpcid != "" {
 			if net.VPC == nil {
 				net.VPC = &infrav1.VPC{}
@@ -88,6 +89,7 @@ func (c *client) ResolveNetwork(net *infrav1.Network) (retErr error) {
 	net.Type = netDetails.Type
 	net.Gateway = netDetails.Gateway
 	net.Netmask = netDetails.Netmask
+	net.Offering = netDetails.Networkofferingname
 	if netDetails.Vpcid != "" {
 		if net.VPC == nil {
 			net.VPC = &infrav1.VPC{}

pkg/cloud/vpc.go

@@ -40,8 +40,8 @@ type VPCIface interface {
 }
 
 // getVPCOfferingID fetches a vpc offering id.
-func (c *client) getVPCOfferingID() (string, error) {
-	offeringID, count, retErr := c.cs.VPC.GetVPCOfferingID(VPCOffering)
+func (c *client) getVPCOfferingID(offeringName string) (string, error) {
+	offeringID, count, retErr := c.cs.VPC.GetVPCOfferingID(offeringName)
 	if retErr != nil {
 		c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(retErr)
 		return "", retErr
@@ -70,6 +70,7 @@ func (c *client) ResolveVPC(vpc *infrav1.VPC) error {
 		}
 		vpc.Name = resp.Name
 		vpc.CIDR = resp.Cidr
+		vpc.Offering = resp.Vpcofferingname
 		return nil
 	}
 
@@ -84,6 +85,7 @@ func (c *client) ResolveVPC(vpc *infrav1.VPC) error {
 	}
 	vpc.ID = resp.Id
 	vpc.CIDR = resp.Cidr
+	vpc.Offering = resp.Vpcofferingname
 	return nil
 }
 
@@ -93,7 +95,12 @@ func (c *client) CreateVPC(fd *infrav1.CloudStackFailureDomain, vpc *infrav1.VPC
 		return errors.New("VPC name must be specified")
 	}
 
-	offeringID, err := c.getVPCOfferingID()
+	offeringName := VPCOffering
+	if vpc.Offering != "" {
+		offeringName = vpc.Offering
+	}
+
+	offeringID, err := c.getVPCOfferingID(offeringName)
 	if err != nil {
 		return err
 	}

test/e2e/common.go

@@ -62,12 +62,25 @@ const (
 	InvalidDomainName            = "CLOUDSTACK_INVALID_DOMAIN_NAME"
 	InvalidTemplateName          = "CLOUDSTACK_INVALID_TEMPLATE_NAME"
 	InvalidCPOfferingName        = "CLOUDSTACK_INVALID_CONTROL_PLANE_MACHINE_OFFERING"
+
+	CustomNetworkOfferingWithEgressPolicyName = "CLOUDSTACK_NETWORK_CUSTOM_OFFERING_EGRESS_POLICY_TRUE_NAME"
+	NetworkNameWithCustomOffering             = "CLOUDSTACK_NETWORK_NAME_WITH_CUSTOM_OFFERING"
+
+	VPCName                          = "CLOUDSTACK_VPC_NAME"
+	VPCNetworkName                   = "CLOUDSTACK_VPC_NETWORK_NAME"
+	CustomVPCOfferingName            = "CLOUDSTACK_CUSTOM_VPC_OFFERING_NAME"
+	CustomVPCNetworkOfferingName     = "CLOUDSTACK_CUSTOM_VPC_NETWORK_OFFERING_NAME"
+	VPCWithCustomOfferingName        = "CLOUDSTACK_VPC_NAME_WITH_CUSTOM_OFFERING"
+	VPCNetworkWithCustomOfferingName = "CLOUDSTACK_VPC_NETWORK_NAME_WITH_CUSTOM_OFFERING"
 )
 
 const (
 	ControlPlaneIndicator      = "-control-plane-"
 	MachineDeploymentIndicator = "-md-"
 	DataVolumePrefix           = "DATA-"
+	DefaultVPCOffering         = "Default VPC offering"
+	DefaultVPCNetworkOffering  = "DefaultIsolatedNetworkOfferingForVpcNetworks"
+	DefaultNetworkOffering     = "DefaultIsolatedNetworkOfferingWithSourceNatService"
 )
 
 type CommonSpecInput struct {
@@ -391,15 +404,11 @@ func CheckNetworkExists(client *cloudstack.CloudStackClient, networkName string)
 		}
 		return false, err
 	} else if count > 1 {
-		return false, fmt.Errorf("Expected 0-1 Network with name %s, but got %d.", networkName, count)
+		return false, fmt.Errorf("expected 0-1 network with name %s, but got %d", networkName, count)
 	}
 	return count == 1, nil
 }
 
-func CheckVPCExists(client *cloudstack.CloudStackClient, vpcName string) (bool, error) {
-	return CheckVPCExistsInProject(client, vpcName, "")
-}
-
 func CheckVPCExistsInProject(client *cloudstack.CloudStackClient, vpcName string, project string) (bool, error) {
 	p := client.VPC.NewListVPCsParams()
 	p.SetName(vpcName)
@@ -419,7 +428,7 @@ func CheckVPCExistsInProject(client *cloudstack.CloudStackClient, vpcName string
 		}
 		return false, err
 	} else if listResp.Count > 1 {
-		return false, fmt.Errorf("Expected 0-1 VPC with name %s, but got %d.", vpcName, listResp.Count)
+		return false, fmt.Errorf("expected 0-1 vpc with name %s, but got %d", vpcName, listResp.Count)
 	}
 	return listResp.Count == 1, nil
 }

test/e2e/config/cloudstack.yaml

@@ -89,6 +89,8 @@ providers:
       - sourcePath: "../data/infrastructure-cloudstack/v1beta3/cluster-template-second-cluster.yaml"
       - sourcePath: "../data/infrastructure-cloudstack/v1beta3/cluster-template-shared-network-kubevip.yaml"
       - sourcePath: "../data/infrastructure-cloudstack/v1beta3/cluster-template-vpc-network.yaml"
+      - sourcePath: "../data/infrastructure-cloudstack/v1beta3/cluster-template-custom-vpc-offering.yaml"
+      - sourcePath: "../data/infrastructure-cloudstack/v1beta3/cluster-template-network-with-custom-offering.yaml"
       - sourcePath: "../data/infrastructure-cloudstack/v1beta3/cluster-template-invalid-disk-offering.yaml"
       - sourcePath: "../data/infrastructure-cloudstack/v1beta3/cluster-template-invalid-disk-offering-size-for-non-customized.yaml"
       - sourcePath: "../data/infrastructure-cloudstack/v1beta3/cluster-template-invalid-disk-offering-size-for-customized.yaml"
@@ -129,10 +131,20 @@ variables:
   CLOUDSTACK_INVALID_ACCOUNT_NAME: accountXXXX
   CLOUDSTACK_DOMAIN_NAME: ROOT
   CLOUDSTACK_INVALID_DOMAIN_NAME: domainXXXX
+
   CLOUDSTACK_NETWORK_NAME: isolated-for-e2e-1
+  CLOUDSTACK_NETWORK_NAME_WITH_CUSTOM_OFFERING: isolated-for-e2e-1-with-custom-offering
+  CLOUDSTACK_NETWORK_CUSTOM_OFFERING_EGRESS_POLICY_TRUE_NAME: CustomNetworkOfferingWithEgressPolicyTrue
+
   CLOUDSTACK_VPC_NETWORK_NAME: vpc-isolated-for-e2e-1
   CLOUDSTACK_VPC_NAME: vpc-for-e2e-1
   CLOUDSTACK_VPC_CIDR: 10.10.0.0/16
+
+  CLOUDSTACK_VPC_NAME_WITH_CUSTOM_OFFERING: vpc-for-e2e-1-with-custom-offering
+  CLOUDSTACK_VPC_NETWORK_NAME_WITH_CUSTOM_OFFERING: vpc-isolated-for-e2e-1-with-custom-offering
+  CLOUDSTACK_CUSTOM_VPC_OFFERING_NAME: "CustomVPCOffering"
+  CLOUDSTACK_CUSTOM_VPC_NETWORK_OFFERING_NAME: "CustomVPCNetworkOffering"
+
   CLOUDSTACK_GATEWAY: 10.10.0.1
   CLOUDSTACK_NETMASK: 255.255.255.0
   CLOUDSTACK_NEW_NETWORK_NAME: isolated-for-e2e-new

test/e2e/data/infrastructure-cloudstack/v1beta3/cluster-template-custom-vpc-offering/cluster-with-custom-vpc-offering.yaml

@@ -0,0 +1,25 @@
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta3
+kind: CloudStackCluster
+metadata:
+  name: ${CLUSTER_NAME}
+spec:
+  failureDomains:
+  - name: ${CLOUDSTACK_FD1_NAME}
+    acsEndpoint:
+      name: ${CLOUDSTACK_FD1_SECRET_NAME}
+      namespace: default
+    zone:
+      name: ${CLOUDSTACK_ZONE_NAME}
+      network:
+        name: ${CLOUDSTACK_VPC_NETWORK_NAME_WITH_CUSTOM_OFFERING}
+        gateway: ${CLOUDSTACK_GATEWAY}
+        netmask: ${CLOUDSTACK_NETMASK}
+        offering: ${CLOUDSTACK_CUSTOM_VPC_NETWORK_OFFERING_NAME}
+        vpc:
+          name: ${CLOUDSTACK_VPC_NAME_WITH_CUSTOM_OFFERING}
+          cidr: ${CLOUDSTACK_VPC_CIDR}
+          offering: ${CLOUDSTACK_CUSTOM_VPC_OFFERING_NAME}
+  controlPlaneEndpoint:
+    host: ""
+    port: 6443

test/e2e/data/infrastructure-cloudstack/v1beta3/cluster-template-custom-vpc-offering/kustomization.yaml

@@ -0,0 +1,6 @@
+bases:
+  - ../bases/cluster-with-kcp.yaml
+  - ../bases/md.yaml
+
+patchesStrategicMerge:
+- ./cluster-with-custom-vpc-offering.yaml

test/e2e/data/infrastructure-cloudstack/v1beta3/cluster-template-network-with-custom-offering/cluster-with-network-with-custom-offering.yaml

@@ -0,0 +1,21 @@
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta3
+kind: CloudStackCluster
+metadata:
+  name: ${CLUSTER_NAME}
+spec:
+  failureDomains:
+  - name: ${CLOUDSTACK_FD1_NAME}
+    acsEndpoint:
+      name: ${CLOUDSTACK_FD1_SECRET_NAME}
+      namespace: default
+    zone:
+      name: ${CLOUDSTACK_ZONE_NAME}
+      network:
+        name: ${CLOUDSTACK_NETWORK_NAME_WITH_CUSTOM_OFFERING}
+        gateway: ${CLOUDSTACK_GATEWAY}
+        netmask: ${CLOUDSTACK_NETMASK}
+        offering: ${CLOUDSTACK_NETWORK_CUSTOM_OFFERING_EGRESS_POLICY_TRUE_NAME}
+  controlPlaneEndpoint:
+    host: ""
+    port: 6443

test/e2e/data/infrastructure-cloudstack/v1beta3/cluster-template-network-with-custom-offering/kustomization.yaml

@@ -0,0 +1,6 @@
+bases:
+  - ../bases/cluster-with-kcp.yaml
+  - ../bases/md.yaml
+
+patchesStrategicMerge:
+- ./cluster-with-network-with-custom-offering.yaml