diff --git a/api/v1alpha1/clusterconfig_types.go b/api/v1alpha1/clusterconfig_types.go index eafbf04f3..09c0a9e44 100644 --- a/api/v1alpha1/clusterconfig_types.go +++ b/api/v1alpha1/clusterconfig_types.go @@ -200,6 +200,9 @@ type GenericClusterConfigSpec struct { // +kubebuilder:validation:Optional Users []User `json:"users,omitempty"` + + // +optional + EncryptionAtRest *EncryptionAtRest `json:"encryptionAtRest,omitempty"` } type Image struct { @@ -279,6 +282,28 @@ type User struct { Sudo string `json:"sudo,omitempty"` } +// EncryptionAtRest defines the configuration to enable encryption at REST +// This configuration is used by API server to encrypt data before storing it in ETCD. +// Currently the encryption only enabled for secrets and configmaps. +type EncryptionAtRest struct { + // Encryption providers + // +kubebuilder:default={{aescbc:{}}} + // +kubebuilder:validation:MaxItems=1 + // +kubebuilder:validation:Optional + Providers []EncryptionProviders `json:"providers,omitempty"` +} + +type EncryptionProviders struct { + // +kubebuilder:validation:Optional + AESCBC *AESConfiguration `json:"aescbc,omitempty"` + // +kubebuilder:validation:Optional + Secretbox *SecretboxConfiguration `json:"secretbox,omitempty"` +} + +type AESConfiguration struct{} + +type SecretboxConfiguration struct{} + func init() { SchemeBuilder.Register( &AWSClusterConfig{}, diff --git a/api/v1alpha1/crds/caren.nutanix.com_awsclusterconfigs.yaml b/api/v1alpha1/crds/caren.nutanix.com_awsclusterconfigs.yaml index dda2625fd..f943424f8 100644 --- a/api/v1alpha1/crds/caren.nutanix.com_awsclusterconfigs.yaml +++ b/api/v1alpha1/crds/caren.nutanix.com_awsclusterconfigs.yaml @@ -322,6 +322,26 @@ spec: type: string type: object type: object + encryptionAtRest: + description: |- + EncryptionAtRest defines the configuration to enable encryption at REST + This configuration is used by API server to encrypt data before storing it in ETCD. + Currently the encryption only enabled for secrets and configmaps. + properties: + providers: + default: + - aescbc: {} + description: Encryption providers + items: + properties: + aescbc: + type: object + secretbox: + type: object + type: object + maxItems: 1 + type: array + type: object etcd: properties: image: diff --git a/api/v1alpha1/crds/caren.nutanix.com_dockerclusterconfigs.yaml b/api/v1alpha1/crds/caren.nutanix.com_dockerclusterconfigs.yaml index 59132cade..db1c13406 100644 --- a/api/v1alpha1/crds/caren.nutanix.com_dockerclusterconfigs.yaml +++ b/api/v1alpha1/crds/caren.nutanix.com_dockerclusterconfigs.yaml @@ -239,6 +239,26 @@ spec: type: object docker: type: object + encryptionAtRest: + description: |- + EncryptionAtRest defines the configuration to enable encryption at REST + This configuration is used by API server to encrypt data before storing it in ETCD. + Currently the encryption only enabled for secrets and configmaps. + properties: + providers: + default: + - aescbc: {} + description: Encryption providers + items: + properties: + aescbc: + type: object + secretbox: + type: object + type: object + maxItems: 1 + type: array + type: object etcd: properties: image: diff --git a/api/v1alpha1/crds/caren.nutanix.com_genericclusterconfigs.yaml b/api/v1alpha1/crds/caren.nutanix.com_genericclusterconfigs.yaml index 483658d45..b97165cd3 100644 --- a/api/v1alpha1/crds/caren.nutanix.com_genericclusterconfigs.yaml +++ b/api/v1alpha1/crds/caren.nutanix.com_genericclusterconfigs.yaml @@ -233,6 +233,26 @@ spec: - provider type: object type: object + encryptionAtRest: + description: |- + EncryptionAtRest defines the configuration to enable encryption at REST + This configuration is used by API server to encrypt data before storing it in ETCD. + Currently the encryption only enabled for secrets and configmaps. + properties: + providers: + default: + - aescbc: {} + description: Encryption providers + items: + properties: + aescbc: + type: object + secretbox: + type: object + type: object + maxItems: 1 + type: array + type: object etcd: properties: image: diff --git a/api/v1alpha1/crds/caren.nutanix.com_nutanixclusterconfigs.yaml b/api/v1alpha1/crds/caren.nutanix.com_nutanixclusterconfigs.yaml index bf7596f1b..b0e33d455 100644 --- a/api/v1alpha1/crds/caren.nutanix.com_nutanixclusterconfigs.yaml +++ b/api/v1alpha1/crds/caren.nutanix.com_nutanixclusterconfigs.yaml @@ -410,6 +410,26 @@ spec: - machineDetails type: object type: object + encryptionAtRest: + description: |- + EncryptionAtRest defines the configuration to enable encryption at REST + This configuration is used by API server to encrypt data before storing it in ETCD. + Currently the encryption only enabled for secrets and configmaps. + properties: + providers: + default: + - aescbc: {} + description: Encryption providers + items: + properties: + aescbc: + type: object + secretbox: + type: object + type: object + maxItems: 1 + type: array + type: object etcd: properties: image: diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 21e6353a7..c2077f499 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -15,6 +15,21 @@ import ( runtime "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AESConfiguration) DeepCopyInto(out *AESConfiguration) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AESConfiguration. +func (in *AESConfiguration) DeepCopy() *AESConfiguration { + if in == nil { + return nil + } + out := new(AESConfiguration) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *AMILookup) DeepCopyInto(out *AMILookup) { *out = *in @@ -657,6 +672,53 @@ func (in *DockerSpec) DeepCopy() *DockerSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EncryptionAtRest) DeepCopyInto(out *EncryptionAtRest) { + *out = *in + if in.Providers != nil { + in, out := &in.Providers, &out.Providers + *out = make([]EncryptionProviders, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EncryptionAtRest. +func (in *EncryptionAtRest) DeepCopy() *EncryptionAtRest { + if in == nil { + return nil + } + out := new(EncryptionAtRest) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EncryptionProviders) DeepCopyInto(out *EncryptionProviders) { + *out = *in + if in.AESCBC != nil { + in, out := &in.AESCBC, &out.AESCBC + *out = new(AESConfiguration) + **out = **in + } + if in.Secretbox != nil { + in, out := &in.Secretbox, &out.Secretbox + *out = new(SecretboxConfiguration) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EncryptionProviders. +func (in *EncryptionProviders) DeepCopy() *EncryptionProviders { + if in == nil { + return nil + } + out := new(EncryptionProviders) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Etcd) DeepCopyInto(out *Etcd) { *out = *in @@ -750,6 +812,11 @@ func (in *GenericClusterConfigSpec) DeepCopyInto(out *GenericClusterConfigSpec) (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.EncryptionAtRest != nil { + in, out := &in.EncryptionAtRest, &out.EncryptionAtRest + *out = new(EncryptionAtRest) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GenericClusterConfigSpec. @@ -1151,6 +1218,21 @@ func (in *RegistryCredentials) DeepCopy() *RegistryCredentials { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SecretboxConfiguration) DeepCopyInto(out *SecretboxConfiguration) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecretboxConfiguration. +func (in *SecretboxConfiguration) DeepCopy() *SecretboxConfiguration { + if in == nil { + return nil + } + out := new(SecretboxConfiguration) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SecurityGroup) DeepCopyInto(out *SecurityGroup) { *out = *in diff --git a/common/pkg/k8s/client/create.go b/common/pkg/k8s/client/create.go new file mode 100644 index 000000000..c924a91a8 --- /dev/null +++ b/common/pkg/k8s/client/create.go @@ -0,0 +1,30 @@ +// Copyright 2024 Nutanix. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package client + +import ( + "context" + "fmt" + + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" +) + +func Create( + ctx context.Context, + c ctrlclient.Client, + obj ctrlclient.Object, + opts ...ctrlclient.CreateOption, +) error { + options := []ctrlclient.CreateOption{ctrlclient.FieldOwner(FieldOwner)} + options = append(options, opts...) + err := c.Create( + ctx, + obj, + options..., + ) + if err != nil { + return fmt.Errorf("create object failed: %w", err) + } + return nil +} diff --git a/pkg/handlers/generic/mutation/encryptionatrest/encryptionprovider_test.go b/pkg/handlers/generic/mutation/encryptionatrest/encryptionprovider_test.go new file mode 100644 index 000000000..2d5617176 --- /dev/null +++ b/pkg/handlers/generic/mutation/encryptionatrest/encryptionprovider_test.go @@ -0,0 +1,87 @@ +// Copyright 2024 Nutanix. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package encryptionatrest + +import ( + "encoding/base64" + "testing" + + "github.com/stretchr/testify/assert" + apiserverv1 "k8s.io/apiserver/pkg/apis/config/v1" + + carenv1 "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1" +) + +func Test_encryptionConfigForSecretsAndConfigMaps(t *testing.T) { + testcases := []struct { + name string + providers *carenv1.EncryptionProviders + wantErr error + want *apiserverv1.ResourceConfiguration + }{ + { + name: "encryption configuration using all providers", + providers: &carenv1.EncryptionProviders{ + AESCBC: &carenv1.AESConfiguration{}, + Secretbox: &carenv1.SecretboxConfiguration{}, + }, + wantErr: nil, + want: &apiserverv1.ResourceConfiguration{ + Resources: []string{"secrets", "configmaps"}, + Providers: []apiserverv1.ProviderConfiguration{ + { + AESCBC: &apiserverv1.AESConfiguration{ + Keys: []apiserverv1.Key{ + { + Name: "key1", + Secret: base64.StdEncoding.EncodeToString([]byte(testToken)), + }, + }, + }, + Secretbox: &apiserverv1.SecretboxConfiguration{ + Keys: []apiserverv1.Key{ + { + Name: "key1", + Secret: base64.StdEncoding.EncodeToString([]byte(testToken)), + }, + }, + }, + }, + }, + }, + }, + { + name: "encryption configuration using single provider", + providers: &carenv1.EncryptionProviders{ + AESCBC: &carenv1.AESConfiguration{}, + }, + wantErr: nil, + want: &apiserverv1.ResourceConfiguration{ + Resources: []string{"secrets", "configmaps"}, + Providers: []apiserverv1.ProviderConfiguration{ + { + AESCBC: &apiserverv1.AESConfiguration{ + Keys: []apiserverv1.Key{ + { + Name: "key1", + Secret: base64.StdEncoding.EncodeToString([]byte(testToken)), + }, + }, + }, + }, + }, + }, + }, + } + + for _, tt := range testcases { + t.Run(tt.name, func(t *testing.T) { + got, gErr := defaultEncryptionConfiguration( + tt.providers, + testTokenGenerator) + assert.Equal(t, tt.wantErr, gErr) + assert.Equal(t, tt.want, got) + }) + } +} diff --git a/pkg/handlers/generic/mutation/encryptionatrest/inject.go b/pkg/handlers/generic/mutation/encryptionatrest/inject.go new file mode 100644 index 000000000..912fedb1f --- /dev/null +++ b/pkg/handlers/generic/mutation/encryptionatrest/inject.go @@ -0,0 +1,309 @@ +// Copyright 2024 Nutanix. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package encryptionatrest + +import ( + "context" + "encoding/base64" + "fmt" + "strings" + + corev1 "k8s.io/api/core/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + apiserverv1 "k8s.io/apiserver/pkg/apis/config/v1" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + cabpkv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" + controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1" + runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1" + ctrl "sigs.k8s.io/controller-runtime" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/yaml" + + carenv1 "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/handlers/mutation" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/patches" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/patches/selectors" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/variables" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/utils" + k8sClientUtil "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/k8s/client" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/clusterconfig" +) + +const ( + // VariableName is the external patch variable name. + VariableName = "encryptionAtRest" + SecretKeyForEtcdEncryption = "config" + defaultEncryptionSecretNameTemplate = "%s-encryption-config" //nolint:gosec // Does not contain hard coded credentials. + encryptionConfigurationOnRemote = "/etc/kubernetes/pki/encryptionconfig.yaml" + apiServerEncryptionConfigArg = "encryption-provider-config" +) + +type encryptionPatchHandler struct { + client ctrlclient.Client + keyGenerator TokenGenerator + variableName string + variableFieldPath []string +} + +func NewPatch(client ctrlclient.Client, keyGenerator TokenGenerator) *encryptionPatchHandler { + return &encryptionPatchHandler{ + client: client, + keyGenerator: keyGenerator, + variableName: clusterconfig.MetaVariableName, + variableFieldPath: []string{VariableName}, + } +} + +func (h *encryptionPatchHandler) Mutate( + ctx context.Context, + obj *unstructured.Unstructured, + vars map[string]apiextensionsv1.JSON, + holderRef runtimehooksv1.HolderReference, + clusterKey ctrlclient.ObjectKey, + clusterGetter mutation.ClusterGetter, +) error { + log := ctrl.LoggerFrom(ctx, "holderRef", holderRef) + + encryptionVariable, err := variables.Get[carenv1.EncryptionAtRest]( + vars, + h.variableName, + h.variableFieldPath..., + ) + if err != nil { + if variables.IsNotFoundError(err) { + log.V(5).Info("encryption variable not defined") + return nil + } + return err + } + + log = log.WithValues( + "variableName", + h.variableName, + "variableFieldPath", + h.variableFieldPath, + "variableValue", + encryptionVariable, + ) + + return patches.MutateIfApplicable( + obj, vars, &holderRef, selectors.ControlPlane(), log, + func(obj *controlplanev1.KubeadmControlPlaneTemplate) error { + cluster, err := clusterGetter(ctx) + if err != nil { + log.Error(err, "failed to get cluster from encryption mutation handler") + return err + } + + found, err := h.defaultEncryptionSecretExists(ctx, cluster) + if err != nil { + log.WithValues( + "defaultEncryptionSecret", defaultEncryptionSecretName(cluster.Name), + ).Error(err, "failed to find default encryption configuration secret") + return err + } + + // we do not rotate or override the secret keys for encryption configuration + if !found { + encryptionConfig, err := h.generateEncryptionConfiguration( + encryptionVariable.Providers, + ) + if err != nil { + return err + } + if err := h.createEncryptionConfigurationSecret(ctx, encryptionConfig, cluster); err != nil { + return err + } + } + + log.WithValues( + "patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(), + "patchedObjectName", ctrlclient.ObjectKeyFromObject(obj), + ).Info("adding encryption configuration files and API server extra args in control plane kubeadm config spec") + + // Create kubadm config file for encryption config + obj.Spec.Template.Spec.KubeadmConfigSpec.Files = append( + obj.Spec.Template.Spec.KubeadmConfigSpec.Files, + generateEncryptionCredentialsFile(cluster)) + + // set APIServer args for encryption config + if obj.Spec.Template.Spec.KubeadmConfigSpec.ClusterConfiguration == nil { + obj.Spec.Template.Spec.KubeadmConfigSpec.ClusterConfiguration = &cabpkv1.ClusterConfiguration{} + } + apiServer := &obj.Spec.Template.Spec.KubeadmConfigSpec.ClusterConfiguration.APIServer + if apiServer.ExtraArgs == nil { + apiServer.ExtraArgs = make(map[string]string, 1) + } + apiServer.ExtraArgs[apiServerEncryptionConfigArg] = encryptionConfigurationOnRemote + + return nil + }) +} + +func generateEncryptionCredentialsFile(cluster *clusterv1.Cluster) cabpkv1.File { + secretName := defaultEncryptionSecretName(cluster.Name) + return cabpkv1.File{ + Path: encryptionConfigurationOnRemote, + ContentFrom: &cabpkv1.FileSource{ + Secret: cabpkv1.SecretFileSource{ + Name: secretName, + Key: SecretKeyForEtcdEncryption, + }, + }, + Permissions: "0640", + } +} + +func (h *encryptionPatchHandler) generateEncryptionConfiguration( + providers []carenv1.EncryptionProviders, +) (*apiserverv1.EncryptionConfiguration, error) { + resourceConfigs := []apiserverv1.ResourceConfiguration{} + for _, encProvider := range providers { + provider := encProvider + resourceConfig, err := defaultEncryptionConfiguration( + &provider, + h.keyGenerator, + ) + if err != nil { + return nil, err + } + resourceConfigs = append(resourceConfigs, *resourceConfig) + } + // We only support encryption for "secrets" and "configmaps" using "aescbc" provider. + + return &apiserverv1.EncryptionConfiguration{ + TypeMeta: metav1.TypeMeta{ + APIVersion: apiserverv1.SchemeGroupVersion.String(), + Kind: "EncryptionConfiguration", + }, + Resources: resourceConfigs, + }, nil +} + +func (h *encryptionPatchHandler) defaultEncryptionSecretExists( + ctx context.Context, + cluster *clusterv1.Cluster, +) (bool, error) { + secretName := defaultEncryptionSecretName(cluster.Name) + existingSecret := &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "Secret", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: secretName, + Namespace: cluster.Namespace, + }, + } + err := h.client.Get(ctx, ctrlclient.ObjectKeyFromObject(existingSecret), existingSecret) + if err != nil { + if errors.IsNotFound(err) { + return false, nil + } + return false, err + } + return true, nil +} + +func (h *encryptionPatchHandler) createEncryptionConfigurationSecret( + ctx context.Context, + encryptionConfig *apiserverv1.EncryptionConfiguration, + cluster *clusterv1.Cluster, +) error { + dataYaml, err := yaml.Marshal(encryptionConfig) + if err != nil { + return fmt.Errorf("unable to marshal encryption configuration to YAML: %w", err) + } + + secretData := map[string]string{ + SecretKeyForEtcdEncryption: strings.TrimSpace(string(dataYaml)), + } + secretName := defaultEncryptionSecretName(cluster.Name) + encryptionConfigSecret := &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "Secret", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: secretName, + Namespace: cluster.Namespace, + Labels: utils.NewLabels(utils.WithMove(), utils.WithClusterName(cluster.Name)), + }, + StringData: secretData, + Type: corev1.SecretTypeOpaque, + } + + if err = controllerutil.SetOwnerReference(cluster, encryptionConfigSecret, h.client.Scheme()); err != nil { + return fmt.Errorf( + "failed to set owner reference on encryption configuration secret: %w", + err, + ) + } + + // We only support creating encryption config in BeforeClusterCreate hook and ensure that the keys are immutable. + if err := k8sClientUtil.Create(ctx, h.client, encryptionConfigSecret); err != nil { + return fmt.Errorf("failed to create encryption configuration secret: %w", err) + } + return nil +} + +// We only support encryption for "secrets" and "configmaps". +func defaultEncryptionConfiguration( + providers *carenv1.EncryptionProviders, + secretGenerator TokenGenerator, +) (*apiserverv1.ResourceConfiguration, error) { + providerConfig := apiserverv1.ProviderConfiguration{} + // We only support "aescbc", "secretbox" for now. + // "aesgcm" is another AESConfiguration. "aesgcm" requires secret key rotation before 200k write calls. + // "aesgcm" should not be supported until secret key's rotation is implemented. + if providers.AESCBC != nil { + token, err := secretGenerator() + if err != nil { + return nil, fmt.Errorf( + "could not create random encryption token for aescbc provider: %w", + err, + ) + } + providerConfig.AESCBC = &apiserverv1.AESConfiguration{ + Keys: []apiserverv1.Key{ + { + Name: "key1", // we only support one key during cluster creation. + Secret: base64.StdEncoding.EncodeToString(token), + }, + }, + } + } + if providers.Secretbox != nil { + token, err := secretGenerator() + if err != nil { + return nil, fmt.Errorf( + "could not create random encryption token for secretbox provider: %w", + err, + ) + } + providerConfig.Secretbox = &apiserverv1.SecretboxConfiguration{ + Keys: []apiserverv1.Key{ + { + Name: "key1", // we only support one key during cluster creation. + Secret: base64.StdEncoding.EncodeToString(token), + }, + }, + } + } + + return &apiserverv1.ResourceConfiguration{ + Resources: []string{"secrets", "configmaps"}, + Providers: []apiserverv1.ProviderConfiguration{ + providerConfig, + }, + }, nil +} + +func defaultEncryptionSecretName(clusterName string) string { + return fmt.Sprintf(defaultEncryptionSecretNameTemplate, clusterName) +} diff --git a/pkg/handlers/generic/mutation/encryptionatrest/inject_test.go b/pkg/handlers/generic/mutation/encryptionatrest/inject_test.go new file mode 100644 index 000000000..89e0eb63b --- /dev/null +++ b/pkg/handlers/generic/mutation/encryptionatrest/inject_test.go @@ -0,0 +1,224 @@ +// Copyright 2024 Nutanix. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package encryptionatrest + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/onsi/gomega/format" + "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + + carenv1 "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/handlers/mutation" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/testutils/capitest" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/testutils/capitest/request" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/clusterconfig" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/test/helpers" +) + +const ( + testToken = "testAESConfigKey" //nolint:gosec // Does not contain hard coded credentials. + testEncryptionConfigSecretData = `apiVersion: apiserver.config.k8s.io/v1 +kind: EncryptionConfiguration +resources: +- providers: + - aescbc: + keys: + - name: key1 + secret: dGVzdEFFU0NvbmZpZ0tleQ== + resources: + - secrets + - configmaps` +) + +func testTokenGenerator() ([]byte, error) { + return []byte(testToken), nil +} + +func TestEncryptionConfigurationPatch(t *testing.T) { + RegisterFailHandler(Fail) + format.TruncatedDiff = false + RunSpecs(t, "Encryption configuration mutator suite") +} + +var _ = Describe("Generate Encryption configuration patches", func() { + clientScheme := runtime.NewScheme() + utilruntime.Must(clientgoscheme.AddToScheme(clientScheme)) + utilruntime.Must(clusterv1.AddToScheme(clientScheme)) + patchGenerator := func() mutation.GeneratePatches { + client, err := helpers.TestEnv.GetK8sClientWithScheme(clientScheme) + Expect(err).To(BeNil()) + return mutation.NewMetaGeneratePatchesHandler( + "", + client, + NewPatch(client, testTokenGenerator)).(mutation.GeneratePatches) + } + + encryptionVar := []runtimehooksv1.Variable{ + capitest.VariableWithValue( + clusterconfig.MetaVariableName, + carenv1.EncryptionAtRest{ + Providers: []carenv1.EncryptionProviders{ + { + AESCBC: &carenv1.AESConfiguration{}, + }, + }, + }, + VariableName, + ), + } + encryptionMatchers := []capitest.JSONPatchMatcher{ + { + Operation: "add", + Path: "/spec/template/spec/kubeadmConfigSpec/files", + ValueMatcher: ContainElements( + SatisfyAll( + HaveKeyWithValue( + "path", "/etc/kubernetes/pki/encryptionconfig.yaml", + ), + HaveKeyWithValue( + "permissions", "0640", + ), + HaveKeyWithValue( + "contentFrom", + map[string]interface{}{ + "secret": map[string]interface{}{ + "key": "config", + "name": defaultEncryptionSecretName(request.ClusterName), + }, + }, + ), + ), + ), + }, + { + Operation: "add", + Path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration", + ValueMatcher: HaveKeyWithValue( + "apiServer", + HaveKeyWithValue( + "extraArgs", + map[string]interface{}{ + "encryption-provider-config": "/etc/kubernetes/pki/encryptionconfig.yaml", + }, + ), + ), + }, + } + + // Create cluster before each test + BeforeEach(func(ctx SpecContext) { + client, err := helpers.TestEnv.GetK8sClientWithScheme(clientScheme) + Expect(err).To(BeNil()) + + Expect(client.Create( + ctx, + &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: request.ClusterName, + Namespace: metav1.NamespaceDefault, + }, + }, + )).To(BeNil()) + }) + + // Delete cluster after each test + AfterEach(func(ctx SpecContext) { + client, err := helpers.TestEnv.GetK8sClientWithScheme(clientScheme) + Expect(err).To(BeNil()) + + Expect(client.Delete( + ctx, + &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: request.ClusterName, + Namespace: metav1.NamespaceDefault, + }, + }, + )).To(BeNil()) + }) + + // Test that encryption config patch is generated without recreating the default encryption secret. + // The Mutate function must be ideompotent and always generate patch in success cases. + noOpEncryptionConfigDef := capitest.PatchTestDef{ + Name: "skip creating default encryption config secret if it already exists", + Vars: encryptionVar, + RequestItem: request.NewKubeadmControlPlaneTemplateRequestItem(""), + ExpectedPatchMatchers: encryptionMatchers, + } + + Context("Default encryption provider secret already exists", func() { + // encryption secret was created earlier + BeforeEach(func(ctx SpecContext) { + client, err := helpers.TestEnv.GetK8sClientWithScheme(clientScheme) + Expect(err).To(BeNil()) + + Expect(client.Create( + ctx, + testEncryptionSecretObj(), + )).To(BeNil()) + }) + // delete encryption configuration after the test + AfterEach(func(ctx SpecContext) { + client, err := helpers.TestEnv.GetK8sClientWithScheme(clientScheme) + Expect(err).To(BeNil()) + + Expect(client.Delete( + ctx, + testEncryptionSecretObj(), + )).To(BeNil()) + }) + It(noOpEncryptionConfigDef.Name, func(ctx SpecContext) { + capitest.AssertGeneratePatches(GinkgoT(), patchGenerator, &noOpEncryptionConfigDef) + }) + }) + + // Test that encryption configuration secret is generated and patched on kubeadmconfig spec. + patchEncryptionConfigDef := capitest.PatchTestDef{ + Name: "files added in KubeadmControlPlaneTemplate for Encryption Configuration", + Vars: encryptionVar, + RequestItem: request.NewKubeadmControlPlaneTemplateRequestItem(""), + ExpectedPatchMatchers: encryptionMatchers, + } + + Context("Default encryption configuration secret is created by the patch", func() { + It(patchEncryptionConfigDef.Name, func(ctx SpecContext) { + capitest.AssertGeneratePatches(GinkgoT(), patchGenerator, &patchEncryptionConfigDef) + + // assert secret containing Encryption configuration is generated + client, err := helpers.TestEnv.GetK8sClientWithScheme(clientScheme) + Expect(err).To(BeNil()) + + gotSecret := testEncryptionSecretObj() + err = client.Get( + ctx, + ctrlclient.ObjectKeyFromObject(gotSecret), + gotSecret) + Expect(err).To(BeNil()) + assert.Equal( + GinkgoT(), + testEncryptionConfigSecretData, + string(gotSecret.Data[SecretKeyForEtcdEncryption])) + }) + }) +}) + +func testEncryptionSecretObj() *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: defaultEncryptionSecretName(request.ClusterName), + Namespace: request.Namespace, + }, + } +} diff --git a/pkg/handlers/generic/mutation/encryptionatrest/tokengenerator.go b/pkg/handlers/generic/mutation/encryptionatrest/tokengenerator.go new file mode 100644 index 000000000..73951189d --- /dev/null +++ b/pkg/handlers/generic/mutation/encryptionatrest/tokengenerator.go @@ -0,0 +1,25 @@ +// Copyright 2024 Nutanix. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package encryptionatrest + +import "crypto/rand" + +const ( + tokenLength = 32 +) + +type TokenGenerator func() ([]byte, error) + +func RandomTokenGenerator() ([]byte, error) { + return createRandBytes(tokenLength) +} + +// createRandBytes returns a cryptographically secure slice of random bytes with a given size. +func createRandBytes(size uint32) ([]byte, error) { + b := make([]byte, size) + if _, err := rand.Read(b); err != nil { + return nil, err + } + return b, nil +} diff --git a/pkg/handlers/generic/mutation/handlers.go b/pkg/handlers/generic/mutation/handlers.go index e5d8f1965..9da8d1d73 100644 --- a/pkg/handlers/generic/mutation/handlers.go +++ b/pkg/handlers/generic/mutation/handlers.go @@ -12,6 +12,7 @@ import ( "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/containerdapplypatchesandrestart" "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/containerdmetrics" "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/containerdunprivilegedports" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/encryptionatrest" "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/etcd" "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/extraapiservercertsans" "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/httpproxy" @@ -35,6 +36,7 @@ func MetaMutators(mgr manager.Manager) []mutation.MetaMutator { users.NewPatch(), containerdmetrics.NewPatch(), containerdunprivilegedports.NewPatch(), + encryptionatrest.NewPatch(mgr.GetClient(), encryptionatrest.RandomTokenGenerator), // Some patches may have changed containerd configuration. // We write the configuration changes to disk, and must run a command