diff --git a/api/v1alpha1/aws_node_types.go b/api/v1alpha1/aws_node_types.go index 74e46ee80..82009af18 100644 --- a/api/v1alpha1/aws_node_types.go +++ b/api/v1alpha1/aws_node_types.go @@ -19,6 +19,34 @@ type AWSNodeSpec struct { // If both AMI ID and AMI lookup arguments are provided then AMI ID takes precedence //+optional AMISpec *AMISpec `json:"ami,omitempty"` + + //+optional + AdditionalSecurityGroups AdditionalSecurityGroup `json:"additionalSecurityGroups,omitempty"` +} + +type AdditionalSecurityGroup []SecurityGroup + +type SecurityGroup struct { + // ID is the id of the security group + // +optional + ID *string `json:"id,omitempty"` +} + +func (AdditionalSecurityGroup) VariableSchema() clusterv1.VariableSchema { + return clusterv1.VariableSchema{ + OpenAPIV3Schema: clusterv1.JSONSchemaProps{ + Type: "array", + Items: &clusterv1.JSONSchemaProps{ + Type: "object", + Properties: map[string]clusterv1.JSONSchemaProps{ + "id": { + Type: "string", + Description: "Security group ID to add for the cluster Machines", + }, + }, + }, + }, + } } func (AWSNodeSpec) VariableSchema() clusterv1.VariableSchema { @@ -27,9 +55,10 @@ func (AWSNodeSpec) VariableSchema() clusterv1.VariableSchema { Description: "AWS Node configuration", Type: "object", Properties: map[string]clusterv1.JSONSchemaProps{ - "iamInstanceProfile": IAMInstanceProfile("").VariableSchema().OpenAPIV3Schema, - "instanceType": InstanceType("").VariableSchema().OpenAPIV3Schema, - "ami": AMISpec{}.VariableSchema().OpenAPIV3Schema, + "iamInstanceProfile": IAMInstanceProfile("").VariableSchema().OpenAPIV3Schema, + "instanceType": InstanceType("").VariableSchema().OpenAPIV3Schema, + "ami": AMISpec{}.VariableSchema().OpenAPIV3Schema, + "additionalSecurityGroups": AdditionalSecurityGroup{}.VariableSchema().OpenAPIV3Schema, }, }, } diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index fda1dbc5f..918c1397d 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -90,6 +90,13 @@ func (in *AWSNodeSpec) DeepCopyInto(out *AWSNodeSpec) { *out = new(AMISpec) (*in).DeepCopyInto(*out) } + if in.AdditionalSecurityGroups != nil { + in, out := &in.AdditionalSecurityGroups, &out.AdditionalSecurityGroups + *out = make(AdditionalSecurityGroup, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AWSNodeSpec. @@ -127,6 +134,27 @@ func (in *AWSSpec) DeepCopy() *AWSSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in AdditionalSecurityGroup) DeepCopyInto(out *AdditionalSecurityGroup) { + { + in := &in + *out = make(AdditionalSecurityGroup, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AdditionalSecurityGroup. +func (in AdditionalSecurityGroup) DeepCopy() AdditionalSecurityGroup { + if in == nil { + return nil + } + out := new(AdditionalSecurityGroup) + in.DeepCopyInto(out) + return *out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Addons) DeepCopyInto(out *Addons) { *out = *in @@ -607,6 +635,26 @@ func (in *ObjectMeta) DeepCopy() *ObjectMeta { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SecurityGroup) DeepCopyInto(out *SecurityGroup) { + *out = *in + if in.ID != nil { + in, out := &in.ID, &out.ID + *out = new(string) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecurityGroup. +func (in *SecurityGroup) DeepCopy() *SecurityGroup { + if in == nil { + return nil + } + out := new(SecurityGroup) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SubnetSpec) DeepCopyInto(out *SubnetSpec) { *out = *in diff --git a/charts/capi-runtime-extensions/templates/csi/aws-ebs/manifests/aws-ebs-csi-configmap.yaml b/charts/capi-runtime-extensions/templates/csi/aws-ebs/manifests/aws-ebs-csi-configmap.yaml index af5407e3a..99df34352 100644 --- a/charts/capi-runtime-extensions/templates/csi/aws-ebs/manifests/aws-ebs-csi-configmap.yaml +++ b/charts/capi-runtime-extensions/templates/csi/aws-ebs/manifests/aws-ebs-csi-configmap.yaml @@ -1630,7 +1630,7 @@ data: key: endpoint name: aws-meta optional: true - image: public.ecr.aws/ebs-csi-driver/aws-ebs-csi-driver:v1.23.1 + image: public.ecr.aws/ebs-csi-driver/aws-ebs-csi-driver:v1.23.2 imagePullPolicy: IfNotPresent livenessProbe: failureThreshold: 5 @@ -1675,7 +1675,7 @@ data: env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock - image: public.ecr.aws/eks-distro/kubernetes-csi/external-provisioner:v3.5.0-eks-1-28-4 + image: public.ecr.aws/eks-distro/kubernetes-csi/external-provisioner:v3.6.0-eks-1-28-7 imagePullPolicy: IfNotPresent name: csi-provisioner resources: @@ -1697,7 +1697,7 @@ data: env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock - image: public.ecr.aws/eks-distro/kubernetes-csi/external-attacher:v4.3.0-eks-1-28-4 + image: public.ecr.aws/eks-distro/kubernetes-csi/external-attacher:v4.4.0-eks-1-28-7 imagePullPolicy: IfNotPresent name: csi-attacher resources: @@ -1719,7 +1719,7 @@ data: env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock - image: public.ecr.aws/eks-distro/kubernetes-csi/external-snapshotter/csi-snapshotter:v6.2.2-eks-1-28-4 + image: public.ecr.aws/eks-distro/kubernetes-csi/external-snapshotter/csi-snapshotter:v6.3.0-eks-1-28-7 imagePullPolicy: IfNotPresent name: csi-snapshotter resources: @@ -1742,7 +1742,7 @@ data: env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock - image: public.ecr.aws/eks-distro/kubernetes-csi/external-resizer:v1.8.0-eks-1-28-4 + image: public.ecr.aws/eks-distro/kubernetes-csi/external-resizer:v1.9.0-eks-1-28-7 imagePullPolicy: IfNotPresent name: csi-resizer resources: @@ -1759,7 +1759,7 @@ data: name: socket-dir - args: - --csi-address=/csi/csi.sock - image: public.ecr.aws/eks-distro/kubernetes-csi/livenessprobe:v2.10.0-eks-1-28-4 + image: public.ecr.aws/eks-distro/kubernetes-csi/livenessprobe:v2.11.0-eks-1-28-7 imagePullPolicy: IfNotPresent name: liveness-probe resources: @@ -1891,6 +1891,14 @@ data: operator: NotIn values: - fargate + - key: node.kubernetes.io/instance-type + operator: NotIn + values: + - a1.medium + - a1.large + - a1.xlarge + - a1.2xlarge + - a1.4xlarge containers: - args: - node @@ -1904,7 +1912,7 @@ data: valueFrom: fieldRef: fieldPath: spec.nodeName - image: public.ecr.aws/ebs-csi-driver/aws-ebs-csi-driver:v1.23.1 + image: public.ecr.aws/ebs-csi-driver/aws-ebs-csi-driver:v1.23.2 imagePullPolicy: IfNotPresent lifecycle: preStop: @@ -1951,7 +1959,7 @@ data: value: /csi/csi.sock - name: DRIVER_REG_SOCK_PATH value: /var/lib/kubelet/plugins/ebs.csi.aws.com/csi.sock - image: public.ecr.aws/eks-distro/kubernetes-csi/node-driver-registrar:v2.8.0-eks-1-28-4 + image: public.ecr.aws/eks-distro/kubernetes-csi/node-driver-registrar:v2.9.0-eks-1-28-7 imagePullPolicy: IfNotPresent livenessProbe: exec: @@ -1981,7 +1989,7 @@ data: name: probe-dir - args: - --csi-address=/csi/csi.sock - image: public.ecr.aws/eks-distro/kubernetes-csi/livenessprobe:v2.10.0-eks-1-28-4 + image: public.ecr.aws/eks-distro/kubernetes-csi/livenessprobe:v2.11.0-eks-1-28-7 imagePullPolicy: IfNotPresent name: liveness-probe resources: diff --git a/docs/content/customization/aws/security-groups.md b/docs/content/customization/aws/security-groups.md new file mode 100644 index 000000000..6b6da19b5 --- /dev/null +++ b/docs/content/customization/aws/security-groups.md @@ -0,0 +1,74 @@ ++++ +title = "AWS Additional Security Group Spec" ++++ + +The AWS additional security group customization allows the user to specify security groups to the created machines. +The customization can be applied to both control plane and nodepool machines. +This customization will be available when the +[provider-specific cluster configuration patch]({{< ref "..">}}) is included in the `ClusterClass`. + +## Example + +To specify addiitonal security groups for all control plane and nodepools, use the following configuration: + +```yaml +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: +spec: + topology: + variables: + - name: clusterConfig + value: + controlPlane: + aws: + additionalSecurityGroups: + - id: "sg-0fcfece738d3211b8" + - name: workerConfig + value: + aws: + additionalSecurityGroups: + - id: "sg-0fcfece738d3211b8" +``` + +We can further customize individual MachineDeployments by using the overrides field with the following configuration: + +```yaml +spec: + topology: + # ... + workers: + machineDeployments: + - class: default-worker + name: md-0 + variables: + overrides: + - name: workerConfig + value: + aws: + additionalSecurityGroups: + - id: "sg-0fcfece738d3211b8" +``` + +Applying this configuration will result in the following value being set: + +- control-plane `AWSMachineTemplate`: + + - ```yaml + spec: + template: + spec: + additionalSecurityGroups: + - id: sg-0fcfece738d3211b8 + ``` + +- worker `AWSMachineTemplate`: + + - ```yaml + spec: + template: + spec: + additionalSecurityGroups: + - id: sg-0fcfece738d3211b8 + ``` diff --git a/pkg/handlers/aws/mutation/metapatch_handler.go b/pkg/handlers/aws/mutation/metapatch_handler.go index a7c2ffc4c..2785f69ab 100644 --- a/pkg/handlers/aws/mutation/metapatch_handler.go +++ b/pkg/handlers/aws/mutation/metapatch_handler.go @@ -14,6 +14,7 @@ import ( "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/aws/mutation/instancetype" "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/aws/mutation/network" "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/aws/mutation/region" + "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/aws/mutation/securitygroups" genericmutation "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation" ) @@ -27,6 +28,7 @@ func MetaPatchHandler(mgr manager.Manager) handlers.Named { iaminstanceprofile.NewControlPlanePatch(), instancetype.NewControlPlanePatch(), ami.NewControlPlanePatch(), + securitygroups.NewControlPlanePatch(), }, genericmutation.MetaMutators(mgr)..., ) @@ -43,6 +45,7 @@ func MetaWorkerPatchHandler() handlers.Named { iaminstanceprofile.NewWorkerPatch(), instancetype.NewWorkerPatch(), ami.NewWorkerPatch(), + securitygroups.NewWorkerPatch(), } return mutation.NewMetaGeneratePatchesHandler( diff --git a/pkg/handlers/aws/mutation/securitygroups/inject.go b/pkg/handlers/aws/mutation/securitygroups/inject.go new file mode 100644 index 000000000..ca2cbf76d --- /dev/null +++ b/pkg/handlers/aws/mutation/securitygroups/inject.go @@ -0,0 +1,99 @@ +// Copyright 2023 D2iQ, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package securitygroups + +import ( + "context" + + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/d2iq-labs/capi-runtime-extensions/api/v1alpha1" + "github.com/d2iq-labs/capi-runtime-extensions/common/pkg/capi/clustertopology/patches" + "github.com/d2iq-labs/capi-runtime-extensions/common/pkg/capi/clustertopology/variables" + capav1 "github.com/d2iq-labs/capi-runtime-extensions/common/pkg/external/sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2" +) + +const ( + // VariableName is the external patch variable name. + VariableName = "additionalSecurityGroups" +) + +type awsSecurityGroupSpecPatchHandler struct { + metaVariableName string + variableFieldPath []string + patchSelector clusterv1.PatchSelector +} + +func newAWSSecurityGroupSpecPatchHandler( + metaVariableName string, + variableFieldPath []string, + patchSelector clusterv1.PatchSelector, +) *awsSecurityGroupSpecPatchHandler { + return &awsSecurityGroupSpecPatchHandler{ + metaVariableName: metaVariableName, + variableFieldPath: variableFieldPath, + patchSelector: patchSelector, + } +} + +func (h *awsSecurityGroupSpecPatchHandler) Mutate( + ctx context.Context, + obj *unstructured.Unstructured, + vars map[string]apiextensionsv1.JSON, + holderRef runtimehooksv1.HolderReference, + _ client.ObjectKey, +) error { + log := ctrl.LoggerFrom(ctx).WithValues( + "holderRef", holderRef, + ) + additionalSecGroupVar, found, err := variables.Get[v1alpha1.AdditionalSecurityGroup]( + vars, + h.metaVariableName, + h.variableFieldPath..., + ) + if err != nil { + return err + } + if !found { + log.V(5). + Info("No additional security groups provided. Skipping.") + return nil + } + + log = log.WithValues( + "variableName", + h.metaVariableName, + "variableFieldPath", + h.variableFieldPath, + "variableValue", + additionalSecGroupVar, + ) + resourceRefs := make([]capav1.AWSResourceReference, 0, len(additionalSecGroupVar)) + for _, secGroup := range additionalSecGroupVar { + resourceRefs = append(resourceRefs, capav1.AWSResourceReference{ + ID: secGroup.ID, + }) + } + return patches.MutateIfApplicable( + obj, + vars, + &holderRef, + h.patchSelector, + log, + func(obj *capav1.AWSMachineTemplate) error { + log.WithValues( + "patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(), + "patchedObjectName", client.ObjectKeyFromObject(obj), + ).Info("setting additional security groups ") + + obj.Spec.Template.Spec.AdditionalSecurityGroups = resourceRefs + return nil + }, + ) +} diff --git a/pkg/handlers/aws/mutation/securitygroups/inject_control_plane.go b/pkg/handlers/aws/mutation/securitygroups/inject_control_plane.go new file mode 100644 index 000000000..8caa090ff --- /dev/null +++ b/pkg/handlers/aws/mutation/securitygroups/inject_control_plane.go @@ -0,0 +1,26 @@ +// Copyright 2023 D2iQ, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package securitygroups + +import ( + "github.com/d2iq-labs/capi-runtime-extensions/api/v1alpha1" + "github.com/d2iq-labs/capi-runtime-extensions/common/pkg/capi/clustertopology/patches/selectors" + capav1 "github.com/d2iq-labs/capi-runtime-extensions/common/pkg/external/sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2" + "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/clusterconfig" +) + +func NewControlPlanePatch() *awsSecurityGroupSpecPatchHandler { + return newAWSSecurityGroupSpecPatchHandler( + clusterconfig.MetaVariableName, + []string{ + clusterconfig.MetaControlPlaneConfigName, + v1alpha1.AWSVariableName, + VariableName, + }, + selectors.InfrastructureControlPlaneMachines( + capav1.GroupVersion.Version, + "AWSMachineTemplate", + ), + ) +} diff --git a/pkg/handlers/aws/mutation/securitygroups/inject_worker.go b/pkg/handlers/aws/mutation/securitygroups/inject_worker.go new file mode 100644 index 000000000..52a3b99c7 --- /dev/null +++ b/pkg/handlers/aws/mutation/securitygroups/inject_worker.go @@ -0,0 +1,24 @@ +// Copyright 2023 D2iQ, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +package securitygroups + +import ( + "github.com/d2iq-labs/capi-runtime-extensions/api/v1alpha1" + "github.com/d2iq-labs/capi-runtime-extensions/common/pkg/capi/clustertopology/patches/selectors" + capav1 "github.com/d2iq-labs/capi-runtime-extensions/common/pkg/external/sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2" + "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/workerconfig" +) + +func NewWorkerPatch() *awsSecurityGroupSpecPatchHandler { + return newAWSSecurityGroupSpecPatchHandler( + workerconfig.MetaVariableName, + []string{ + v1alpha1.AWSVariableName, + VariableName, + }, + selectors.InfrastructureWorkerMachineTemplates( + capav1.GroupVersion.Version, + "AWSMachineTemplate", + ), + ) +} diff --git a/pkg/handlers/aws/mutation/securitygroups/variables_test.go b/pkg/handlers/aws/mutation/securitygroups/variables_test.go new file mode 100644 index 000000000..de02a90a0 --- /dev/null +++ b/pkg/handlers/aws/mutation/securitygroups/variables_test.go @@ -0,0 +1,42 @@ +// Copyright 2023 D2iQ, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package securitygroups + +import ( + "testing" + + "k8s.io/utils/ptr" + + "github.com/d2iq-labs/capi-runtime-extensions/api/v1alpha1" + "github.com/d2iq-labs/capi-runtime-extensions/common/pkg/testutils/capitest" + awsclusterconfig "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/aws/clusterconfig" + "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/clusterconfig" +) + +func TestVariableValidation(t *testing.T) { + capitest.ValidateDiscoverVariables( + t, + clusterconfig.MetaVariableName, + ptr.To(v1alpha1.ClusterConfigSpec{AWS: &v1alpha1.AWSSpec{}}.VariableSchema()), + true, + awsclusterconfig.NewVariable, + capitest.VariableTestDef{ + Name: "Additional Security Group Specification", + Vals: v1alpha1.ClusterConfigSpec{ + ControlPlane: &v1alpha1.NodeConfigSpec{ + AWS: &v1alpha1.AWSNodeSpec{ + AdditionalSecurityGroups: v1alpha1.AdditionalSecurityGroup{ + { + ID: ptr.To("sg-1234"), + }, + { + ID: ptr.To("sg-0420"), + }, + }, + }, + }, + }, + }, + ) +} diff --git a/pkg/handlers/generic/lifecycle/cpi/aws/handler_test.go b/pkg/handlers/generic/lifecycle/cpi/aws/handler_test.go index c58f036bd..0fd559f5e 100644 --- a/pkg/handlers/generic/lifecycle/cpi/aws/handler_test.go +++ b/pkg/handlers/generic/lifecycle/cpi/aws/handler_test.go @@ -108,9 +108,17 @@ func Test_generateCPIConfigMapForCluster(t *testing.T) { test.startConfigMap, test.cluster, ) - cpiConfigMapExpectedName := fmt.Sprintf("%s-%s", test.startConfigMap.Name, test.cluster.Name) + cpiConfigMapExpectedName := fmt.Sprintf( + "%s-%s", + test.startConfigMap.Name, + test.cluster.Name, + ) if cm.Name != cpiConfigMapExpectedName { - t.Errorf("expected configmap name to be %s. got: %s", cpiConfigMapExpectedName, cm.Name) + t.Errorf( + "expected configmap name to be %s. got: %s", + cpiConfigMapExpectedName, + cm.Name, + ) } }) }