From e7efa7f0039ec8c2f6d4fb53fe64c5ea1a589e47 Mon Sep 17 00:00:00 2001 From: Shalin Patel Date: Tue, 7 May 2024 07:58:36 -0700 Subject: [PATCH 01/11] feat: mutation handler for EncryptionConfig --- api/v1alpha1/clusterconfig_types.go | 5 + .../encryption/encryptionprovider_test.go | 67 +++++ .../generic/mutation/encryption/inject.go | 270 ++++++++++++++++++ .../mutation/encryption/inject_test.go | 147 ++++++++++ .../mutation/encryption/tokengenerator.go | 25 ++ 5 files changed, 514 insertions(+) create mode 100644 pkg/handlers/generic/mutation/encryption/encryptionprovider_test.go create mode 100644 pkg/handlers/generic/mutation/encryption/inject.go create mode 100644 pkg/handlers/generic/mutation/encryption/inject_test.go create mode 100644 pkg/handlers/generic/mutation/encryption/tokengenerator.go diff --git a/api/v1alpha1/clusterconfig_types.go b/api/v1alpha1/clusterconfig_types.go index 09c0a9e44..9d2aafb41 100644 --- a/api/v1alpha1/clusterconfig_types.go +++ b/api/v1alpha1/clusterconfig_types.go @@ -12,6 +12,11 @@ import ( "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/variables" ) +const ( + AESCBC EncryptionProvider = "aescbc" + SecretBox EncryptionProvider = "secretbox" +) + var ( DefaultDockerCertSANs = []string{ "localhost", diff --git a/pkg/handlers/generic/mutation/encryption/encryptionprovider_test.go b/pkg/handlers/generic/mutation/encryption/encryptionprovider_test.go new file mode 100644 index 000000000..abd2c4eb8 --- /dev/null +++ b/pkg/handlers/generic/mutation/encryption/encryptionprovider_test.go @@ -0,0 +1,67 @@ +// Copyright 2024 Nutanix. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package encryption + +import ( + "encoding/base64" + "errors" + "testing" + + "github.com/stretchr/testify/assert" + apiserverv1 "k8s.io/apiserver/pkg/apis/config/v1" + + carenv1 "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1" +) + +func Test_encryptionConfigForSecretsAndConfigMaps(t *testing.T) { + testcases := []struct { + name string + providers []carenv1.EncryptionProvider + wantErr error + want *apiserverv1.ResourceConfiguration + }{ + { + name: "encryption configuration using aescbc and secretbox providers", + providers: []carenv1.EncryptionProvider{carenv1.AESCBC, carenv1.SecretBox}, + wantErr: nil, + want: &apiserverv1.ResourceConfiguration{ + Resources: []string{"secrets", "configmaps"}, + Providers: []apiserverv1.ProviderConfiguration{ + { + AESCBC: &apiserverv1.AESConfiguration{ + Keys: []apiserverv1.Key{ + { + Name: "key1", + Secret: base64.StdEncoding.EncodeToString([]byte(testToken)), + }, + }, + }, + Secretbox: &apiserverv1.SecretboxConfiguration{ + Keys: []apiserverv1.Key{ + { + Name: "key1", + Secret: base64.StdEncoding.EncodeToString([]byte(testToken)), + }, + }, + }, + }, + }, + }, + }, + { + name: "unsupported encryption provider", + providers: []carenv1.EncryptionProvider{carenv1.EncryptionProvider("kmsv2")}, + wantErr: errors.New("unknown encryption provider: kmsv2"), + want: nil, + }, + } + + for _, tt := range testcases { + t.Run(tt.name, func(t *testing.T) { + got, gErr := encryptionConfigForSecretsAndConfigMaps(tt.providers, testTokenGenerator) + assert.Equal(t, tt.wantErr, gErr) + assert.Equal(t, tt.want, got) + }) + } +} diff --git a/pkg/handlers/generic/mutation/encryption/inject.go b/pkg/handlers/generic/mutation/encryption/inject.go new file mode 100644 index 000000000..f41a3031d --- /dev/null +++ b/pkg/handlers/generic/mutation/encryption/inject.go @@ -0,0 +1,270 @@ +// Copyright 2024 Nutanix. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package encryption + +import ( + "context" + "encoding/base64" + "fmt" + "strings" + + corev1 "k8s.io/api/core/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + apiserverv1 "k8s.io/apiserver/pkg/apis/config/v1" + cabpkv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" + controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1" + runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1" + ctrl "sigs.k8s.io/controller-runtime" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/yaml" + + carenv1 "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/handlers/mutation" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/patches" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/patches/selectors" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/variables" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/utils" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/k8s/client" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/clusterconfig" +) + +const ( + // VariableName is the external patch variable name. + VariableName = "encryption" + SecretKeyForEtcdEncryption = "config" + defaultEncryptionSecretNameTemplate = "%s-encryption-config" //nolint:gosec // Does not contain hard coded credentials. + encryptionConfigurationOnRemote = "/etc/kubernetes/encryptionconfig.yaml" + apiServerEncryptionConfigArg = "encryption-provider-config" +) + +type Config struct { + Client ctrlclient.Client + AESSecretKeyGenerator TokenGenerator +} + +type encryptionPatchHandler struct { + config *Config + variableName string + variableFieldPath []string +} + +func NewPatch(config *Config) *encryptionPatchHandler { + return newEncryptionPatchHandler( + config, + clusterconfig.MetaVariableName, + VariableName) +} + +func newEncryptionPatchHandler( + config *Config, + variableName string, + variableFieldPath ...string, +) *encryptionPatchHandler { + return &encryptionPatchHandler{ + config: config, + variableName: variableName, + variableFieldPath: variableFieldPath, + } +} + +func (h *encryptionPatchHandler) Mutate( + ctx context.Context, + obj *unstructured.Unstructured, + vars map[string]apiextensionsv1.JSON, + holderRef runtimehooksv1.HolderReference, + clusterKey ctrlclient.ObjectKey, + _ mutation.ClusterGetter, +) error { + log := ctrl.LoggerFrom(ctx, "holderRef", holderRef) + + encryptionVariable, err := variables.Get[carenv1.Encryption]( + vars, + h.variableName, + h.variableFieldPath..., + ) + if err != nil { + if variables.IsNotFoundError(err) { + log.V(5).Info("encryption variable not defined") + return nil + } + return err + } + + log = log.WithValues( + "variableName", + h.variableName, + "variableFieldPath", + h.variableFieldPath, + "variableValue", + encryptionVariable, + ) + + return patches.MutateIfApplicable( + obj, vars, &holderRef, selectors.ControlPlane(), log, + func(obj *controlplanev1.KubeadmControlPlaneTemplate) error { + log.WithValues( + "patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(), + "patchedObjectName", ctrlclient.ObjectKeyFromObject(obj), + ).Info("setting encryption in control plane kubeadm config template") + encConfig, err := h.generateEncryptionConfiguration(encryptionVariable.Providers) + if err != nil { + return err + } + secretName, err := h.CreateEncryptionConfigurationSecret(ctx, encConfig, clusterKey) + if err != nil { + return err + } + // Create kubadm config file for encryption config + obj.Spec.Template.Spec.KubeadmConfigSpec.Files = append( + obj.Spec.Template.Spec.KubeadmConfigSpec.Files, + generateEncryptionCredentialsFile(secretName)) + + // set APIServer args for encryption config + if obj.Spec.Template.Spec.KubeadmConfigSpec.ClusterConfiguration == nil { + obj.Spec.Template.Spec.KubeadmConfigSpec.ClusterConfiguration = &cabpkv1.ClusterConfiguration{} + } + apiServer := &obj.Spec.Template.Spec.KubeadmConfigSpec.ClusterConfiguration.APIServer + if apiServer.ExtraArgs == nil { + apiServer.ExtraArgs = make(map[string]string, 1) + } + apiServer.ExtraArgs[apiServerEncryptionConfigArg] = encryptionConfigurationOnRemote + + return nil + }) +} + +func generateEncryptionCredentialsFile(secretName string) cabpkv1.File { + return cabpkv1.File{ + Path: encryptionConfigurationOnRemote, + ContentFrom: &cabpkv1.FileSource{ + Secret: cabpkv1.SecretFileSource{ + Name: secretName, + Key: SecretKeyForEtcdEncryption, + }, + }, + Permissions: "0600", + } +} + +func (h *encryptionPatchHandler) generateEncryptionConfiguration( + providers []carenv1.EncryptionProvider, +) (*apiserverv1.EncryptionConfiguration, error) { + // We only support encryption for "secrets" and "configmaps" using "aescbc" provider. + resourceConfig, err := encryptionConfigForSecretsAndConfigMaps( + providers, + h.config.AESSecretKeyGenerator, + ) + if err != nil { + return nil, err + } + return &apiserverv1.EncryptionConfiguration{ + TypeMeta: metav1.TypeMeta{ + APIVersion: apiserverv1.SchemeGroupVersion.String(), + Kind: "EncryptionConfiguration", + }, + Resources: []apiserverv1.ResourceConfiguration{ + *resourceConfig, + }, + }, nil +} + +func (h *encryptionPatchHandler) CreateEncryptionConfigurationSecret( + ctx context.Context, + encryptionConfig *apiserverv1.EncryptionConfiguration, + clusterKey ctrlclient.ObjectKey, +) (string, error) { + dataYaml, err := yaml.Marshal(encryptionConfig) + if err != nil { + return "", fmt.Errorf("unable to marshal encryption configuration to YAML: %w", err) + } + + secretData := map[string]string{ + // creating a Secret with newlines at the end fails + SecretKeyForEtcdEncryption: strings.TrimSpace(string(dataYaml)), + } + secretName := defaultEncryptionSecretName(clusterKey.Name) + encryptionConfigSecret := &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Secret", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: secretName, + Namespace: clusterKey.Namespace, + Labels: utils.NewLabels(utils.WithMove(), utils.WithClusterName(clusterKey.Name)), + }, + StringData: secretData, + Type: corev1.SecretTypeOpaque, + } + + // We only support creating encryption config in BeforeClusterCreate hook and ensure that the keys are immutable. + // Rotation of the keys can be implemented by cloning existing secret and adding new rotation key. + if err := client.Create(ctx, h.config.Client, encryptionConfigSecret); err != nil { + return "", fmt.Errorf("failed to create Encryption Configuration Secret: %w", err) + } + return secretName, nil +} + +// We only support encryption for "secrets" and "configmaps". +func encryptionConfigForSecretsAndConfigMaps( + providers []carenv1.EncryptionProvider, + secretGenerator TokenGenerator, +) (*apiserverv1.ResourceConfiguration, error) { + providerConfig := apiserverv1.ProviderConfiguration{} + for _, providerType := range providers { + // We only support "aescbc", "secretbox" for now. + // "aesgcm" is another AESConfiguration. "aesgcm" requires secret key rotation before 200k write calls. + // It should not be supported until secret key's rotation is implemented. + switch providerType { + case carenv1.AESCBC: + token, err := secretGenerator() + if err != nil { + return nil, fmt.Errorf( + "could not create random encryption token for aescbc provider: %w", + err, + ) + } + providerConfig.AESCBC = &apiserverv1.AESConfiguration{ + Keys: []apiserverv1.Key{ + { + Name: "key1", // we only support one key during cluster creation. + Secret: base64.StdEncoding.EncodeToString(token), + }, + }, + } + case carenv1.SecretBox: + token, err := secretGenerator() + if err != nil { + return nil, fmt.Errorf( + "could not create random encryption token for secretbox provider: %w", + err, + ) + } + providerConfig.Secretbox = &apiserverv1.SecretboxConfiguration{ + Keys: []apiserverv1.Key{ + { + Name: "key1", // we only support one key during cluster creation. + Secret: base64.StdEncoding.EncodeToString(token), + }, + }, + } + default: + // Schema validation should fail earlier upon providing name outside defined Enum + return nil, fmt.Errorf("unknown encryption provider: %s", providerType) + } + } + + return &apiserverv1.ResourceConfiguration{ + Resources: []string{"secrets", "configmaps"}, + Providers: []apiserverv1.ProviderConfiguration{ + providerConfig, + }, + }, nil +} + +func defaultEncryptionSecretName(clusterName string) string { + return fmt.Sprintf(defaultEncryptionSecretNameTemplate, clusterName) +} diff --git a/pkg/handlers/generic/mutation/encryption/inject_test.go b/pkg/handlers/generic/mutation/encryption/inject_test.go new file mode 100644 index 000000000..42ae5bc77 --- /dev/null +++ b/pkg/handlers/generic/mutation/encryption/inject_test.go @@ -0,0 +1,147 @@ +// Copyright 2024 Nutanix. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package encryption + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + + carenv1 "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/handlers/mutation" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/testutils/capitest" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/testutils/capitest/request" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/clusterconfig" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/test/helpers" +) + +const ( + testToken = "testAESConfigKey" //nolint:gosec // Does not contain hard coded credentials. + testEncryptionConfigSecretData = `apiVersion: apiserver.config.k8s.io/v1 +kind: EncryptionConfiguration +resources: +- providers: + - aescbc: + keys: + - name: key1 + secret: dGVzdEFFU0NvbmZpZ0tleQ== + secretbox: + keys: + - name: key1 + secret: dGVzdEFFU0NvbmZpZ0tleQ== + resources: + - secrets + - configmaps` +) + +func testTokenGenerator() ([]byte, error) { + return []byte(testToken), nil +} + +func TestEncryptionConfigurationPatch(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Encryption configuration mutator suite") +} + +var _ = Describe("Generate Encryption configuration patches", func() { + patchGenerator := func() mutation.GeneratePatches { + config := &Config{ + Client: helpers.TestEnv.Client, + AESSecretKeyGenerator: testTokenGenerator, + } + return mutation.NewMetaGeneratePatchesHandler( + "", + helpers.TestEnv.Client, + NewPatch(config)).(mutation.GeneratePatches) + } + + testDefs := []capitest.PatchTestDef{ + { + Name: "files added in KubeadmControlPlaneTemplate for Encryption Configuration", + Vars: []runtimehooksv1.Variable{ + capitest.VariableWithValue( + clusterconfig.MetaVariableName, + carenv1.Encryption{ + Providers: []carenv1.EncryptionProvider{ + carenv1.AESCBC, + carenv1.SecretBox, + }, + }, + VariableName, + ), + }, + RequestItem: request.NewKubeadmControlPlaneTemplateRequestItem(""), + ExpectedPatchMatchers: []capitest.JSONPatchMatcher{ + { + Operation: "add", + Path: "/spec/template/spec/kubeadmConfigSpec/files", + ValueMatcher: ContainElements( + SatisfyAll( + HaveKeyWithValue( + "path", "/etc/kubernetes/encryptionconfig.yaml", + ), + HaveKeyWithValue( + "permissions", "0600", + ), + HaveKeyWithValue( + "contentFrom", + map[string]interface{}{ + "secret": map[string]interface{}{ + "key": "config", + "name": defaultEncryptionSecretName(request.ClusterName), + }, + }, + ), + ), + ), + }, + { + Operation: "add", + Path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration", + ValueMatcher: HaveKeyWithValue( + "apiServer", + HaveKeyWithValue( + "extraArgs", + map[string]interface{}{ + "encryption-provider-config": "/etc/kubernetes/encryptionconfig.yaml", + }, + ), + ), + }, + }, + }, + } + // create test node for each case + for testIdx := range testDefs { + tt := testDefs[testIdx] + It(tt.Name, func(ctx SpecContext) { + capitest.AssertGeneratePatches(GinkgoT(), patchGenerator, &tt) + + // assert secret containing Encryption configuration is generated + gotSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: defaultEncryptionSecretName(request.ClusterName), + Namespace: request.Namespace, + }, + } + objName := ctrlclient.ObjectKeyFromObject(gotSecret) + client, err := helpers.TestEnv.GetK8sClient() + Expect(err).To(BeNil()) + + err = client.Get(ctx, objName, gotSecret) + Expect(err).To(BeNil()) + GinkgoWriter.Println(string(gotSecret.Data[SecretKeyForEtcdEncryption])) + assert.Equal( + GinkgoT(), + testEncryptionConfigSecretData, + string(gotSecret.Data[SecretKeyForEtcdEncryption])) + }) + } +}) diff --git a/pkg/handlers/generic/mutation/encryption/tokengenerator.go b/pkg/handlers/generic/mutation/encryption/tokengenerator.go new file mode 100644 index 000000000..ddd202e0a --- /dev/null +++ b/pkg/handlers/generic/mutation/encryption/tokengenerator.go @@ -0,0 +1,25 @@ +// Copyright 2024 Nutanix. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package encryption + +import "crypto/rand" + +const ( + tokenLength = 32 +) + +type TokenGenerator func() ([]byte, error) + +func RandomTokenGenerator() ([]byte, error) { + return createRandBytes(tokenLength) +} + +// createRandBytes returns a cryptographically secure slice of random bytes with a given size. +func createRandBytes(size uint32) ([]byte, error) { + b := make([]byte, size) + if _, err := rand.Read(b); err != nil { + return nil, err + } + return b, nil +} From b3885b5088d3194d65c5b5d9fc0064216f284523 Mon Sep 17 00:00:00 2001 From: Shalin Patel Date: Thu, 9 May 2024 15:32:33 -0700 Subject: [PATCH 02/11] fix: use object type for encryption providers --- api/v1alpha1/clusterconfig_types.go | 5 ----- api/v1alpha1/zz_generated.deepcopy.go | 25 +++++++++++++++++++++++++ 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/api/v1alpha1/clusterconfig_types.go b/api/v1alpha1/clusterconfig_types.go index 9d2aafb41..09c0a9e44 100644 --- a/api/v1alpha1/clusterconfig_types.go +++ b/api/v1alpha1/clusterconfig_types.go @@ -12,11 +12,6 @@ import ( "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/variables" ) -const ( - AESCBC EncryptionProvider = "aescbc" - SecretBox EncryptionProvider = "secretbox" -) - var ( DefaultDockerCertSANs = []string{ "localhost", diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 1fbf78849..e99d50b7d 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -751,6 +751,31 @@ func (in *EncryptionProviders) DeepCopy() *EncryptionProviders { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EncryptionProviders) DeepCopyInto(out *EncryptionProviders) { + *out = *in + if in.AESCBC != nil { + in, out := &in.AESCBC, &out.AESCBC + *out = new(AESConfiguration) + **out = **in + } + if in.Secretbox != nil { + in, out := &in.Secretbox, &out.Secretbox + *out = new(SecretboxConfiguration) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EncryptionProviders. +func (in *EncryptionProviders) DeepCopy() *EncryptionProviders { + if in == nil { + return nil + } + out := new(EncryptionProviders) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Etcd) DeepCopyInto(out *Etcd) { *out = *in From 43b88d7a21979638c6aee1a3de6be749bbd87f64 Mon Sep 17 00:00:00 2001 From: Shalin Patel Date: Thu, 9 May 2024 15:48:53 -0700 Subject: [PATCH 03/11] feat: mutation handler for encryption configuration --- .../encryption/encryptionprovider_test.go | 36 ++++++--- .../generic/mutation/encryption/inject.go | 79 +++++++++---------- .../mutation/encryption/inject_test.go | 6 +- 3 files changed, 66 insertions(+), 55 deletions(-) diff --git a/pkg/handlers/generic/mutation/encryption/encryptionprovider_test.go b/pkg/handlers/generic/mutation/encryption/encryptionprovider_test.go index abd2c4eb8..88de583ec 100644 --- a/pkg/handlers/generic/mutation/encryption/encryptionprovider_test.go +++ b/pkg/handlers/generic/mutation/encryption/encryptionprovider_test.go @@ -5,7 +5,6 @@ package encryption import ( "encoding/base64" - "errors" "testing" "github.com/stretchr/testify/assert" @@ -17,14 +16,17 @@ import ( func Test_encryptionConfigForSecretsAndConfigMaps(t *testing.T) { testcases := []struct { name string - providers []carenv1.EncryptionProvider + providers *carenv1.EncryptionProviders wantErr error want *apiserverv1.ResourceConfiguration }{ { - name: "encryption configuration using aescbc and secretbox providers", - providers: []carenv1.EncryptionProvider{carenv1.AESCBC, carenv1.SecretBox}, - wantErr: nil, + name: "encryption configuration using all providers", + providers: &carenv1.EncryptionProviders{ + AESCBC: &carenv1.AESConfiguration{}, + Secretbox: &carenv1.SecretboxConfiguration{}, + }, + wantErr: nil, want: &apiserverv1.ResourceConfiguration{ Resources: []string{"secrets", "configmaps"}, Providers: []apiserverv1.ProviderConfiguration{ @@ -50,10 +52,26 @@ func Test_encryptionConfigForSecretsAndConfigMaps(t *testing.T) { }, }, { - name: "unsupported encryption provider", - providers: []carenv1.EncryptionProvider{carenv1.EncryptionProvider("kmsv2")}, - wantErr: errors.New("unknown encryption provider: kmsv2"), - want: nil, + name: "encryption configuration using single provider", + providers: &carenv1.EncryptionProviders{ + AESCBC: &carenv1.AESConfiguration{}, + }, + wantErr: nil, + want: &apiserverv1.ResourceConfiguration{ + Resources: []string{"secrets", "configmaps"}, + Providers: []apiserverv1.ProviderConfiguration{ + { + AESCBC: &apiserverv1.AESConfiguration{ + Keys: []apiserverv1.Key{ + { + Name: "key1", + Secret: base64.StdEncoding.EncodeToString([]byte(testToken)), + }, + }, + }, + }, + }, + }, }, } diff --git a/pkg/handlers/generic/mutation/encryption/inject.go b/pkg/handlers/generic/mutation/encryption/inject.go index f41a3031d..737b3b0d2 100644 --- a/pkg/handlers/generic/mutation/encryption/inject.go +++ b/pkg/handlers/generic/mutation/encryption/inject.go @@ -150,7 +150,7 @@ func generateEncryptionCredentialsFile(secretName string) cabpkv1.File { } func (h *encryptionPatchHandler) generateEncryptionConfiguration( - providers []carenv1.EncryptionProvider, + providers *carenv1.EncryptionProviders, ) (*apiserverv1.EncryptionConfiguration, error) { // We only support encryption for "secrets" and "configmaps" using "aescbc" provider. resourceConfig, err := encryptionConfigForSecretsAndConfigMaps( @@ -182,7 +182,6 @@ func (h *encryptionPatchHandler) CreateEncryptionConfigurationSecret( } secretData := map[string]string{ - // creating a Secret with newlines at the end fails SecretKeyForEtcdEncryption: strings.TrimSpace(string(dataYaml)), } secretName := defaultEncryptionSecretName(clusterKey.Name) @@ -201,59 +200,53 @@ func (h *encryptionPatchHandler) CreateEncryptionConfigurationSecret( } // We only support creating encryption config in BeforeClusterCreate hook and ensure that the keys are immutable. - // Rotation of the keys can be implemented by cloning existing secret and adding new rotation key. if err := client.Create(ctx, h.config.Client, encryptionConfigSecret); err != nil { - return "", fmt.Errorf("failed to create Encryption Configuration Secret: %w", err) + return "", fmt.Errorf("failed to create encryption configuration secret: %w", err) } return secretName, nil } // We only support encryption for "secrets" and "configmaps". func encryptionConfigForSecretsAndConfigMaps( - providers []carenv1.EncryptionProvider, + providers *carenv1.EncryptionProviders, secretGenerator TokenGenerator, ) (*apiserverv1.ResourceConfiguration, error) { providerConfig := apiserverv1.ProviderConfiguration{} - for _, providerType := range providers { - // We only support "aescbc", "secretbox" for now. - // "aesgcm" is another AESConfiguration. "aesgcm" requires secret key rotation before 200k write calls. - // It should not be supported until secret key's rotation is implemented. - switch providerType { - case carenv1.AESCBC: - token, err := secretGenerator() - if err != nil { - return nil, fmt.Errorf( - "could not create random encryption token for aescbc provider: %w", - err, - ) - } - providerConfig.AESCBC = &apiserverv1.AESConfiguration{ - Keys: []apiserverv1.Key{ - { - Name: "key1", // we only support one key during cluster creation. - Secret: base64.StdEncoding.EncodeToString(token), - }, + // We only support "aescbc", "secretbox" for now. + // "aesgcm" is another AESConfiguration. "aesgcm" requires secret key rotation before 200k write calls. + // "aesgcm" should not be supported until secret key's rotation is implemented. + if providers.AESCBC != nil { + token, err := secretGenerator() + if err != nil { + return nil, fmt.Errorf( + "could not create random encryption token for aescbc provider: %w", + err, + ) + } + providerConfig.AESCBC = &apiserverv1.AESConfiguration{ + Keys: []apiserverv1.Key{ + { + Name: "key1", // we only support one key during cluster creation. + Secret: base64.StdEncoding.EncodeToString(token), }, - } - case carenv1.SecretBox: - token, err := secretGenerator() - if err != nil { - return nil, fmt.Errorf( - "could not create random encryption token for secretbox provider: %w", - err, - ) - } - providerConfig.Secretbox = &apiserverv1.SecretboxConfiguration{ - Keys: []apiserverv1.Key{ - { - Name: "key1", // we only support one key during cluster creation. - Secret: base64.StdEncoding.EncodeToString(token), - }, + }, + } + } + if providers.Secretbox != nil { + token, err := secretGenerator() + if err != nil { + return nil, fmt.Errorf( + "could not create random encryption token for secretbox provider: %w", + err, + ) + } + providerConfig.Secretbox = &apiserverv1.SecretboxConfiguration{ + Keys: []apiserverv1.Key{ + { + Name: "key1", // we only support one key during cluster creation. + Secret: base64.StdEncoding.EncodeToString(token), }, - } - default: - // Schema validation should fail earlier upon providing name outside defined Enum - return nil, fmt.Errorf("unknown encryption provider: %s", providerType) + }, } } diff --git a/pkg/handlers/generic/mutation/encryption/inject_test.go b/pkg/handlers/generic/mutation/encryption/inject_test.go index 42ae5bc77..62b47aa7f 100644 --- a/pkg/handlers/generic/mutation/encryption/inject_test.go +++ b/pkg/handlers/generic/mutation/encryption/inject_test.go @@ -69,9 +69,9 @@ var _ = Describe("Generate Encryption configuration patches", func() { capitest.VariableWithValue( clusterconfig.MetaVariableName, carenv1.Encryption{ - Providers: []carenv1.EncryptionProvider{ - carenv1.AESCBC, - carenv1.SecretBox, + Providers: &carenv1.EncryptionProviders{ + AESCBC: &carenv1.AESConfiguration{}, + Secretbox: &carenv1.SecretboxConfiguration{}, }, }, VariableName, From 2f43aefd2871c3ebbf48761f409fe4905f122ef1 Mon Sep 17 00:00:00 2001 From: Shalin Patel Date: Thu, 9 May 2024 18:47:28 -0700 Subject: [PATCH 04/11] fix: add owner reference for secret --- .../generic/mutation/encryption/inject.go | 124 +++++++--- .../mutation/encryption/inject_test.go | 226 ++++++++++++------ 2 files changed, 241 insertions(+), 109 deletions(-) diff --git a/pkg/handlers/generic/mutation/encryption/inject.go b/pkg/handlers/generic/mutation/encryption/inject.go index 737b3b0d2..6a2980440 100644 --- a/pkg/handlers/generic/mutation/encryption/inject.go +++ b/pkg/handlers/generic/mutation/encryption/inject.go @@ -11,14 +11,17 @@ import ( corev1 "k8s.io/api/core/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" apiserverv1 "k8s.io/apiserver/pkg/apis/config/v1" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" cabpkv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1" runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1" ctrl "sigs.k8s.io/controller-runtime" ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/yaml" carenv1 "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1" @@ -27,7 +30,7 @@ import ( "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/patches/selectors" "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/variables" "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/utils" - "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/k8s/client" + k8sClientUtil "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/k8s/client" "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/clusterconfig" ) @@ -40,33 +43,19 @@ const ( apiServerEncryptionConfigArg = "encryption-provider-config" ) -type Config struct { - Client ctrlclient.Client - AESSecretKeyGenerator TokenGenerator -} - type encryptionPatchHandler struct { - config *Config + client ctrlclient.Client + keyGenerator TokenGenerator variableName string variableFieldPath []string } -func NewPatch(config *Config) *encryptionPatchHandler { - return newEncryptionPatchHandler( - config, - clusterconfig.MetaVariableName, - VariableName) -} - -func newEncryptionPatchHandler( - config *Config, - variableName string, - variableFieldPath ...string, -) *encryptionPatchHandler { +func NewPatch(client ctrlclient.Client, keyGenerator TokenGenerator) *encryptionPatchHandler { return &encryptionPatchHandler{ - config: config, - variableName: variableName, - variableFieldPath: variableFieldPath, + client: client, + keyGenerator: keyGenerator, + variableName: clusterconfig.MetaVariableName, + variableFieldPath: []string{VariableName}, } } @@ -76,7 +65,7 @@ func (h *encryptionPatchHandler) Mutate( vars map[string]apiextensionsv1.JSON, holderRef runtimehooksv1.HolderReference, clusterKey ctrlclient.ObjectKey, - _ mutation.ClusterGetter, + clusterGetter mutation.ClusterGetter, ) error { log := ctrl.LoggerFrom(ctx, "holderRef", holderRef) @@ -102,25 +91,49 @@ func (h *encryptionPatchHandler) Mutate( encryptionVariable, ) + cluster, err := clusterGetter(ctx) + if err != nil { + log.Error(err, "failed to get cluster from encryption mutation handler") + return err + } + + found, err := h.DefaultEncryptionSecretExists(ctx, cluster) + if err != nil { + log.WithValues( + "defaultEncryptionSecret", defaultEncryptionSecretName(cluster.Name), + ).Error(err, "failed to find default encryption configuration secret") + return err + } + // we do not rotate or override the secret keys for encryption configuration + if found { + log.V(5).WithValues( + "defaultEncryptionSecret", defaultEncryptionSecretName(cluster.Name), + ).Info( + "skip generating encryption configuration. Default encryption configuration secret exists", + defaultEncryptionSecretName(cluster.Name)) + return nil + } + return patches.MutateIfApplicable( obj, vars, &holderRef, selectors.ControlPlane(), log, func(obj *controlplanev1.KubeadmControlPlaneTemplate) error { log.WithValues( "patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(), "patchedObjectName", ctrlclient.ObjectKeyFromObject(obj), - ).Info("setting encryption in control plane kubeadm config template") + ).Info("adding encryption configuration files and API server extra args in control plane kubeadm config spec") encConfig, err := h.generateEncryptionConfiguration(encryptionVariable.Providers) if err != nil { return err } - secretName, err := h.CreateEncryptionConfigurationSecret(ctx, encConfig, clusterKey) - if err != nil { + + if err := h.CreateEncryptionConfigurationSecret(ctx, encConfig, cluster); err != nil { return err } + // Create kubadm config file for encryption config obj.Spec.Template.Spec.KubeadmConfigSpec.Files = append( obj.Spec.Template.Spec.KubeadmConfigSpec.Files, - generateEncryptionCredentialsFile(secretName)) + generateEncryptionCredentialsFile(cluster)) // set APIServer args for encryption config if obj.Spec.Template.Spec.KubeadmConfigSpec.ClusterConfiguration == nil { @@ -136,7 +149,8 @@ func (h *encryptionPatchHandler) Mutate( }) } -func generateEncryptionCredentialsFile(secretName string) cabpkv1.File { +func generateEncryptionCredentialsFile(cluster *clusterv1.Cluster) cabpkv1.File { + secretName := defaultEncryptionSecretName(cluster.Name) return cabpkv1.File{ Path: encryptionConfigurationOnRemote, ContentFrom: &cabpkv1.FileSource{ @@ -155,7 +169,7 @@ func (h *encryptionPatchHandler) generateEncryptionConfiguration( // We only support encryption for "secrets" and "configmaps" using "aescbc" provider. resourceConfig, err := encryptionConfigForSecretsAndConfigMaps( providers, - h.config.AESSecretKeyGenerator, + h.keyGenerator, ) if err != nil { return nil, err @@ -171,39 +185,71 @@ func (h *encryptionPatchHandler) generateEncryptionConfiguration( }, nil } +func (h *encryptionPatchHandler) DefaultEncryptionSecretExists( + ctx context.Context, + cluster *clusterv1.Cluster, +) (bool, error) { + secretName := defaultEncryptionSecretName(cluster.Name) + existingSecret := &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "Secret", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: secretName, + Namespace: cluster.Namespace, + }, + } + err := h.client.Get(ctx, ctrlclient.ObjectKeyFromObject(existingSecret), existingSecret) + if err != nil { + if errors.IsNotFound(err) { + return false, nil + } + return false, err + } + return true, nil +} + func (h *encryptionPatchHandler) CreateEncryptionConfigurationSecret( ctx context.Context, encryptionConfig *apiserverv1.EncryptionConfiguration, - clusterKey ctrlclient.ObjectKey, -) (string, error) { + cluster *clusterv1.Cluster, +) error { dataYaml, err := yaml.Marshal(encryptionConfig) if err != nil { - return "", fmt.Errorf("unable to marshal encryption configuration to YAML: %w", err) + return fmt.Errorf("unable to marshal encryption configuration to YAML: %w", err) } secretData := map[string]string{ SecretKeyForEtcdEncryption: strings.TrimSpace(string(dataYaml)), } - secretName := defaultEncryptionSecretName(clusterKey.Name) + secretName := defaultEncryptionSecretName(cluster.Name) encryptionConfigSecret := &corev1.Secret{ TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", + APIVersion: corev1.SchemeGroupVersion.String(), Kind: "Secret", }, ObjectMeta: metav1.ObjectMeta{ Name: secretName, - Namespace: clusterKey.Namespace, - Labels: utils.NewLabels(utils.WithMove(), utils.WithClusterName(clusterKey.Name)), + Namespace: cluster.Namespace, + Labels: utils.NewLabels(utils.WithMove(), utils.WithClusterName(cluster.Name)), }, StringData: secretData, Type: corev1.SecretTypeOpaque, } + if err = controllerutil.SetOwnerReference(cluster, encryptionConfigSecret, h.client.Scheme()); err != nil { + return fmt.Errorf( + "failed to set owner reference on encryption configuration secret: %w", + err, + ) + } + // We only support creating encryption config in BeforeClusterCreate hook and ensure that the keys are immutable. - if err := client.Create(ctx, h.config.Client, encryptionConfigSecret); err != nil { - return "", fmt.Errorf("failed to create encryption configuration secret: %w", err) + if err := k8sClientUtil.Create(ctx, h.client, encryptionConfigSecret); err != nil { + return fmt.Errorf("failed to create encryption configuration secret: %w", err) } - return secretName, nil + return nil } // We only support encryption for "secrets" and "configmaps". diff --git a/pkg/handlers/generic/mutation/encryption/inject_test.go b/pkg/handlers/generic/mutation/encryption/inject_test.go index 62b47aa7f..01f164ec3 100644 --- a/pkg/handlers/generic/mutation/encryption/inject_test.go +++ b/pkg/handlers/generic/mutation/encryption/inject_test.go @@ -8,9 +8,14 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/onsi/gomega/format" "github.com/stretchr/testify/assert" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1" ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" @@ -47,101 +52,182 @@ func testTokenGenerator() ([]byte, error) { func TestEncryptionConfigurationPatch(t *testing.T) { RegisterFailHandler(Fail) + format.TruncatedDiff = false RunSpecs(t, "Encryption configuration mutator suite") } var _ = Describe("Generate Encryption configuration patches", func() { + clientScheme := runtime.NewScheme() + utilruntime.Must(clientgoscheme.AddToScheme(clientScheme)) + utilruntime.Must(clusterv1.AddToScheme(clientScheme)) patchGenerator := func() mutation.GeneratePatches { - config := &Config{ - Client: helpers.TestEnv.Client, - AESSecretKeyGenerator: testTokenGenerator, - } + client, err := helpers.TestEnv.GetK8sClientWithScheme(clientScheme) + Expect(err).To(BeNil()) return mutation.NewMetaGeneratePatchesHandler( "", - helpers.TestEnv.Client, - NewPatch(config)).(mutation.GeneratePatches) + client, + NewPatch(client, testTokenGenerator)).(mutation.GeneratePatches) } - testDefs := []capitest.PatchTestDef{ + encryptionVar := []runtimehooksv1.Variable{ + capitest.VariableWithValue( + clusterconfig.MetaVariableName, + carenv1.Encryption{ + Providers: &carenv1.EncryptionProviders{ + AESCBC: &carenv1.AESConfiguration{}, + Secretbox: &carenv1.SecretboxConfiguration{}, + }, + }, + VariableName, + ), + } + encryptionMatchers := []capitest.JSONPatchMatcher{ { - Name: "files added in KubeadmControlPlaneTemplate for Encryption Configuration", - Vars: []runtimehooksv1.Variable{ - capitest.VariableWithValue( - clusterconfig.MetaVariableName, - carenv1.Encryption{ - Providers: &carenv1.EncryptionProviders{ - AESCBC: &carenv1.AESConfiguration{}, - Secretbox: &carenv1.SecretboxConfiguration{}, + Operation: "add", + Path: "/spec/template/spec/kubeadmConfigSpec/files", + ValueMatcher: ContainElements( + SatisfyAll( + HaveKeyWithValue( + "path", "/etc/kubernetes/encryptionconfig.yaml", + ), + HaveKeyWithValue( + "permissions", "0600", + ), + HaveKeyWithValue( + "contentFrom", + map[string]interface{}{ + "secret": map[string]interface{}{ + "key": "config", + "name": defaultEncryptionSecretName(request.ClusterName), + }, }, + ), + ), + ), + }, + { + Operation: "add", + Path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration", + ValueMatcher: HaveKeyWithValue( + "apiServer", + HaveKeyWithValue( + "extraArgs", + map[string]interface{}{ + "encryption-provider-config": "/etc/kubernetes/encryptionconfig.yaml", }, - VariableName, ), - }, - RequestItem: request.NewKubeadmControlPlaneTemplateRequestItem(""), - ExpectedPatchMatchers: []capitest.JSONPatchMatcher{ - { - Operation: "add", - Path: "/spec/template/spec/kubeadmConfigSpec/files", - ValueMatcher: ContainElements( - SatisfyAll( - HaveKeyWithValue( - "path", "/etc/kubernetes/encryptionconfig.yaml", - ), - HaveKeyWithValue( - "permissions", "0600", - ), - HaveKeyWithValue( - "contentFrom", - map[string]interface{}{ - "secret": map[string]interface{}{ - "key": "config", - "name": defaultEncryptionSecretName(request.ClusterName), - }, - }, - ), - ), - ), + ), + }, + } + + // Create cluster before each test + BeforeEach(func(ctx SpecContext) { + client, err := helpers.TestEnv.GetK8sClientWithScheme(clientScheme) + Expect(err).To(BeNil()) + + Expect(client.Create( + ctx, + &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: request.ClusterName, + Namespace: metav1.NamespaceDefault, }, - { - Operation: "add", - Path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration", - ValueMatcher: HaveKeyWithValue( - "apiServer", - HaveKeyWithValue( - "extraArgs", - map[string]interface{}{ - "encryption-provider-config": "/etc/kubernetes/encryptionconfig.yaml", - }, - ), - ), + }, + )).To(BeNil()) + }) + + // Delete cluster after each test + AfterEach(func(ctx SpecContext) { + client, err := helpers.TestEnv.GetK8sClientWithScheme(clientScheme) + Expect(err).To(BeNil()) + + Expect(client.Delete( + ctx, + &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: request.ClusterName, + Namespace: metav1.NamespaceDefault, }, }, - }, + )).To(BeNil()) + }) + + // Test that encryption secret patch is skipped if it is already applied. + // noOpEncryptionConfigDef := capitest.PatchTestDef{ + // Name: "skip patching encryption config if default encryption config secret exists", + // Vars: encryptionVar, + // RequestItem: request.NewKubeadmControlPlaneTemplateRequestItem(""), + // ExpectedPatchMatchers: []capitest.JSONPatchMatcher{ + // { + // Operation: "", + // Path: "", + // ValueMatcher: Equal(nil), + // }, + // }, + // UnexpectedPatchMatchers: encryptionMatchers, + // } + + // Context("Default encryption provider secret already exists", func() { + // // encryption secret was created earlier + // BeforeEach(func(ctx SpecContext) { + // client, err := helpers.TestEnv.GetK8sClientWithScheme(clientScheme) + // Expect(err).To(BeNil()) + + // Expect(client.Create( + // ctx, + // testEncryptionSecretObj(), + // )).To(BeNil()) + // }) + // // delete encryption configuration after the test + // AfterEach(func(ctx SpecContext) { + // client, err := helpers.TestEnv.GetK8sClientWithScheme(clientScheme) + // Expect(err).To(BeNil()) + + // Expect(client.Delete( + // ctx, + // testEncryptionSecretObj(), + // )).To(BeNil()) + // }) + // It(noOpEncryptionConfigDef.Name, func(ctx SpecContext) { + // capitest.AssertGeneratePatches(GinkgoT(), patchGenerator, &noOpEncryptionConfigDef) + // }) + // }) + + // Test that encryption configuration secret is generated and patched on kubeadmconfig spec. + patchEncryptionConfigDef := capitest.PatchTestDef{ + Name: "files added in KubeadmControlPlaneTemplate for Encryption Configuration", + Vars: encryptionVar, + RequestItem: request.NewKubeadmControlPlaneTemplateRequestItem(""), + ExpectedPatchMatchers: encryptionMatchers, } - // create test node for each case - for testIdx := range testDefs { - tt := testDefs[testIdx] - It(tt.Name, func(ctx SpecContext) { - capitest.AssertGeneratePatches(GinkgoT(), patchGenerator, &tt) + + Context("Default encryption configuration secret is created by the patch", func() { + It(patchEncryptionConfigDef.Name, func(ctx SpecContext) { + capitest.AssertGeneratePatches(GinkgoT(), patchGenerator, &patchEncryptionConfigDef) // assert secret containing Encryption configuration is generated - gotSecret := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: defaultEncryptionSecretName(request.ClusterName), - Namespace: request.Namespace, - }, - } - objName := ctrlclient.ObjectKeyFromObject(gotSecret) - client, err := helpers.TestEnv.GetK8sClient() + client, err := helpers.TestEnv.GetK8sClientWithScheme(clientScheme) Expect(err).To(BeNil()) - err = client.Get(ctx, objName, gotSecret) + gotSecret := testEncryptionSecretObj() + err = client.Get( + ctx, + ctrlclient.ObjectKeyFromObject(gotSecret), + gotSecret) Expect(err).To(BeNil()) - GinkgoWriter.Println(string(gotSecret.Data[SecretKeyForEtcdEncryption])) assert.Equal( GinkgoT(), testEncryptionConfigSecretData, string(gotSecret.Data[SecretKeyForEtcdEncryption])) }) - } + }) }) + +func testEncryptionSecretObj() *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: defaultEncryptionSecretName(request.ClusterName), + Namespace: request.Namespace, + }, + } +} From 87b6457363c7986c358d0090aa12e0d3edd82083 Mon Sep 17 00:00:00 2001 From: Shalin Patel Date: Mon, 13 May 2024 14:10:28 -0700 Subject: [PATCH 05/11] fix: use array type for encryption providers --- .../encryption/encryptionprovider_test.go | 4 +- .../generic/mutation/encryption/inject.go | 86 +++++++++-------- .../mutation/encryption/inject_test.go | 93 +++++++++---------- 3 files changed, 87 insertions(+), 96 deletions(-) diff --git a/pkg/handlers/generic/mutation/encryption/encryptionprovider_test.go b/pkg/handlers/generic/mutation/encryption/encryptionprovider_test.go index 88de583ec..8b544f636 100644 --- a/pkg/handlers/generic/mutation/encryption/encryptionprovider_test.go +++ b/pkg/handlers/generic/mutation/encryption/encryptionprovider_test.go @@ -77,7 +77,9 @@ func Test_encryptionConfigForSecretsAndConfigMaps(t *testing.T) { for _, tt := range testcases { t.Run(tt.name, func(t *testing.T) { - got, gErr := encryptionConfigForSecretsAndConfigMaps(tt.providers, testTokenGenerator) + got, gErr := encryptionConfigForSecretsAndConfigMaps( + tt.providers, + testTokenGenerator) assert.Equal(t, tt.wantErr, gErr) assert.Equal(t, tt.want, got) }) diff --git a/pkg/handlers/generic/mutation/encryption/inject.go b/pkg/handlers/generic/mutation/encryption/inject.go index 6a2980440..20ff67a70 100644 --- a/pkg/handlers/generic/mutation/encryption/inject.go +++ b/pkg/handlers/generic/mutation/encryption/inject.go @@ -36,10 +36,10 @@ import ( const ( // VariableName is the external patch variable name. - VariableName = "encryption" + VariableName = "encryptionAtRest" SecretKeyForEtcdEncryption = "config" defaultEncryptionSecretNameTemplate = "%s-encryption-config" //nolint:gosec // Does not contain hard coded credentials. - encryptionConfigurationOnRemote = "/etc/kubernetes/encryptionconfig.yaml" + encryptionConfigurationOnRemote = "/etc/kubernetes/pki/encryptionconfig.yaml" apiServerEncryptionConfigArg = "encryption-provider-config" ) @@ -69,7 +69,7 @@ func (h *encryptionPatchHandler) Mutate( ) error { log := ctrl.LoggerFrom(ctx, "holderRef", holderRef) - encryptionVariable, err := variables.Get[carenv1.Encryption]( + encryptionVariable, err := variables.Get[carenv1.EncryptionAtRest]( vars, h.variableName, h.variableFieldPath..., @@ -91,45 +91,39 @@ func (h *encryptionPatchHandler) Mutate( encryptionVariable, ) - cluster, err := clusterGetter(ctx) - if err != nil { - log.Error(err, "failed to get cluster from encryption mutation handler") - return err - } - - found, err := h.DefaultEncryptionSecretExists(ctx, cluster) - if err != nil { - log.WithValues( - "defaultEncryptionSecret", defaultEncryptionSecretName(cluster.Name), - ).Error(err, "failed to find default encryption configuration secret") - return err - } - // we do not rotate or override the secret keys for encryption configuration - if found { - log.V(5).WithValues( - "defaultEncryptionSecret", defaultEncryptionSecretName(cluster.Name), - ).Info( - "skip generating encryption configuration. Default encryption configuration secret exists", - defaultEncryptionSecretName(cluster.Name)) - return nil - } - return patches.MutateIfApplicable( obj, vars, &holderRef, selectors.ControlPlane(), log, func(obj *controlplanev1.KubeadmControlPlaneTemplate) error { - log.WithValues( - "patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(), - "patchedObjectName", ctrlclient.ObjectKeyFromObject(obj), - ).Info("adding encryption configuration files and API server extra args in control plane kubeadm config spec") - encConfig, err := h.generateEncryptionConfiguration(encryptionVariable.Providers) + cluster, err := clusterGetter(ctx) if err != nil { + log.Error(err, "failed to get cluster from encryption mutation handler") return err } - if err := h.CreateEncryptionConfigurationSecret(ctx, encConfig, cluster); err != nil { + found, err := h.DefaultEncryptionSecretExists(ctx, cluster) + if err != nil { + log.WithValues( + "defaultEncryptionSecret", defaultEncryptionSecretName(cluster.Name), + ).Error(err, "failed to find default encryption configuration secret") return err } + // we do not rotate or override the secret keys for encryption configuration + if !found { + encConfig, err := h.generateEncryptionConfiguration(encryptionVariable.Providers) + if err != nil { + return err + } + if err := h.CreateEncryptionConfigurationSecret(ctx, encConfig, cluster); err != nil { + return err + } + } + + log.WithValues( + "patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(), + "patchedObjectName", ctrlclient.ObjectKeyFromObject(obj), + ).Info("adding encryption configuration files and API server extra args in control plane kubeadm config spec") + // Create kubadm config file for encryption config obj.Spec.Template.Spec.KubeadmConfigSpec.Files = append( obj.Spec.Template.Spec.KubeadmConfigSpec.Files, @@ -159,29 +153,33 @@ func generateEncryptionCredentialsFile(cluster *clusterv1.Cluster) cabpkv1.File Key: SecretKeyForEtcdEncryption, }, }, - Permissions: "0600", + Permissions: "0640", } } func (h *encryptionPatchHandler) generateEncryptionConfiguration( - providers *carenv1.EncryptionProviders, + providers []carenv1.EncryptionProviders, ) (*apiserverv1.EncryptionConfiguration, error) { - // We only support encryption for "secrets" and "configmaps" using "aescbc" provider. - resourceConfig, err := encryptionConfigForSecretsAndConfigMaps( - providers, - h.keyGenerator, - ) - if err != nil { - return nil, err + resourceConfigs := []apiserverv1.ResourceConfiguration{} + for _, encProvider := range providers { + provider := encProvider + resourceConfig, err := encryptionConfigForSecretsAndConfigMaps( + &provider, + h.keyGenerator, + ) + if err != nil { + return nil, err + } + resourceConfigs = append(resourceConfigs, *resourceConfig) } + // We only support encryption for "secrets" and "configmaps" using "aescbc" provider. + return &apiserverv1.EncryptionConfiguration{ TypeMeta: metav1.TypeMeta{ APIVersion: apiserverv1.SchemeGroupVersion.String(), Kind: "EncryptionConfiguration", }, - Resources: []apiserverv1.ResourceConfiguration{ - *resourceConfig, - }, + Resources: resourceConfigs, }, nil } diff --git a/pkg/handlers/generic/mutation/encryption/inject_test.go b/pkg/handlers/generic/mutation/encryption/inject_test.go index 01f164ec3..e406f14ac 100644 --- a/pkg/handlers/generic/mutation/encryption/inject_test.go +++ b/pkg/handlers/generic/mutation/encryption/inject_test.go @@ -37,10 +37,6 @@ resources: keys: - name: key1 secret: dGVzdEFFU0NvbmZpZ0tleQ== - secretbox: - keys: - - name: key1 - secret: dGVzdEFFU0NvbmZpZ0tleQ== resources: - secrets - configmaps` @@ -72,10 +68,11 @@ var _ = Describe("Generate Encryption configuration patches", func() { encryptionVar := []runtimehooksv1.Variable{ capitest.VariableWithValue( clusterconfig.MetaVariableName, - carenv1.Encryption{ - Providers: &carenv1.EncryptionProviders{ - AESCBC: &carenv1.AESConfiguration{}, - Secretbox: &carenv1.SecretboxConfiguration{}, + carenv1.EncryptionAtRest{ + Providers: []carenv1.EncryptionProviders{ + { + AESCBC: &carenv1.AESConfiguration{}, + }, }, }, VariableName, @@ -88,10 +85,10 @@ var _ = Describe("Generate Encryption configuration patches", func() { ValueMatcher: ContainElements( SatisfyAll( HaveKeyWithValue( - "path", "/etc/kubernetes/encryptionconfig.yaml", + "path", "/etc/kubernetes/pki/encryptionconfig.yaml", ), HaveKeyWithValue( - "permissions", "0600", + "permissions", "0640", ), HaveKeyWithValue( "contentFrom", @@ -113,7 +110,7 @@ var _ = Describe("Generate Encryption configuration patches", func() { HaveKeyWithValue( "extraArgs", map[string]interface{}{ - "encryption-provider-config": "/etc/kubernetes/encryptionconfig.yaml", + "encryption-provider-config": "/etc/kubernetes/pki/encryptionconfig.yaml", }, ), ), @@ -152,46 +149,40 @@ var _ = Describe("Generate Encryption configuration patches", func() { )).To(BeNil()) }) - // Test that encryption secret patch is skipped if it is already applied. - // noOpEncryptionConfigDef := capitest.PatchTestDef{ - // Name: "skip patching encryption config if default encryption config secret exists", - // Vars: encryptionVar, - // RequestItem: request.NewKubeadmControlPlaneTemplateRequestItem(""), - // ExpectedPatchMatchers: []capitest.JSONPatchMatcher{ - // { - // Operation: "", - // Path: "", - // ValueMatcher: Equal(nil), - // }, - // }, - // UnexpectedPatchMatchers: encryptionMatchers, - // } - - // Context("Default encryption provider secret already exists", func() { - // // encryption secret was created earlier - // BeforeEach(func(ctx SpecContext) { - // client, err := helpers.TestEnv.GetK8sClientWithScheme(clientScheme) - // Expect(err).To(BeNil()) - - // Expect(client.Create( - // ctx, - // testEncryptionSecretObj(), - // )).To(BeNil()) - // }) - // // delete encryption configuration after the test - // AfterEach(func(ctx SpecContext) { - // client, err := helpers.TestEnv.GetK8sClientWithScheme(clientScheme) - // Expect(err).To(BeNil()) - - // Expect(client.Delete( - // ctx, - // testEncryptionSecretObj(), - // )).To(BeNil()) - // }) - // It(noOpEncryptionConfigDef.Name, func(ctx SpecContext) { - // capitest.AssertGeneratePatches(GinkgoT(), patchGenerator, &noOpEncryptionConfigDef) - // }) - // }) + // Test that encryption config patch is generated without recreating the default encryption secret. + // The Mutate function must be ideompotent and always generate patch in success cases. + noOpEncryptionConfigDef := capitest.PatchTestDef{ + Name: "skip creating default encryption config secret if it already exists", + Vars: encryptionVar, + RequestItem: request.NewKubeadmControlPlaneTemplateRequestItem(""), + ExpectedPatchMatchers: encryptionMatchers, + } + + Context("Default encryption provider secret already exists", func() { + // encryption secret was created earlier + BeforeEach(func(ctx SpecContext) { + client, err := helpers.TestEnv.GetK8sClientWithScheme(clientScheme) + Expect(err).To(BeNil()) + + Expect(client.Create( + ctx, + testEncryptionSecretObj(), + )).To(BeNil()) + }) + // delete encryption configuration after the test + AfterEach(func(ctx SpecContext) { + client, err := helpers.TestEnv.GetK8sClientWithScheme(clientScheme) + Expect(err).To(BeNil()) + + Expect(client.Delete( + ctx, + testEncryptionSecretObj(), + )).To(BeNil()) + }) + It(noOpEncryptionConfigDef.Name, func(ctx SpecContext) { + capitest.AssertGeneratePatches(GinkgoT(), patchGenerator, &noOpEncryptionConfigDef) + }) + }) // Test that encryption configuration secret is generated and patched on kubeadmconfig spec. patchEncryptionConfigDef := capitest.PatchTestDef{ From 636aecdce4796cdd2d7f7d137865604a5fe7e44e Mon Sep 17 00:00:00 2001 From: Shalin Patel Date: Tue, 14 May 2024 11:16:27 -0700 Subject: [PATCH 06/11] fix: rename package to encryptionatrest --- .../encryption/encryptionprovider_test.go | 87 ----- .../generic/mutation/encryption/inject.go | 307 ------------------ .../mutation/encryption/inject_test.go | 224 ------------- .../mutation/encryption/tokengenerator.go | 25 -- 4 files changed, 643 deletions(-) delete mode 100644 pkg/handlers/generic/mutation/encryption/encryptionprovider_test.go delete mode 100644 pkg/handlers/generic/mutation/encryption/inject.go delete mode 100644 pkg/handlers/generic/mutation/encryption/inject_test.go delete mode 100644 pkg/handlers/generic/mutation/encryption/tokengenerator.go diff --git a/pkg/handlers/generic/mutation/encryption/encryptionprovider_test.go b/pkg/handlers/generic/mutation/encryption/encryptionprovider_test.go deleted file mode 100644 index 8b544f636..000000000 --- a/pkg/handlers/generic/mutation/encryption/encryptionprovider_test.go +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright 2024 Nutanix. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -package encryption - -import ( - "encoding/base64" - "testing" - - "github.com/stretchr/testify/assert" - apiserverv1 "k8s.io/apiserver/pkg/apis/config/v1" - - carenv1 "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1" -) - -func Test_encryptionConfigForSecretsAndConfigMaps(t *testing.T) { - testcases := []struct { - name string - providers *carenv1.EncryptionProviders - wantErr error - want *apiserverv1.ResourceConfiguration - }{ - { - name: "encryption configuration using all providers", - providers: &carenv1.EncryptionProviders{ - AESCBC: &carenv1.AESConfiguration{}, - Secretbox: &carenv1.SecretboxConfiguration{}, - }, - wantErr: nil, - want: &apiserverv1.ResourceConfiguration{ - Resources: []string{"secrets", "configmaps"}, - Providers: []apiserverv1.ProviderConfiguration{ - { - AESCBC: &apiserverv1.AESConfiguration{ - Keys: []apiserverv1.Key{ - { - Name: "key1", - Secret: base64.StdEncoding.EncodeToString([]byte(testToken)), - }, - }, - }, - Secretbox: &apiserverv1.SecretboxConfiguration{ - Keys: []apiserverv1.Key{ - { - Name: "key1", - Secret: base64.StdEncoding.EncodeToString([]byte(testToken)), - }, - }, - }, - }, - }, - }, - }, - { - name: "encryption configuration using single provider", - providers: &carenv1.EncryptionProviders{ - AESCBC: &carenv1.AESConfiguration{}, - }, - wantErr: nil, - want: &apiserverv1.ResourceConfiguration{ - Resources: []string{"secrets", "configmaps"}, - Providers: []apiserverv1.ProviderConfiguration{ - { - AESCBC: &apiserverv1.AESConfiguration{ - Keys: []apiserverv1.Key{ - { - Name: "key1", - Secret: base64.StdEncoding.EncodeToString([]byte(testToken)), - }, - }, - }, - }, - }, - }, - }, - } - - for _, tt := range testcases { - t.Run(tt.name, func(t *testing.T) { - got, gErr := encryptionConfigForSecretsAndConfigMaps( - tt.providers, - testTokenGenerator) - assert.Equal(t, tt.wantErr, gErr) - assert.Equal(t, tt.want, got) - }) - } -} diff --git a/pkg/handlers/generic/mutation/encryption/inject.go b/pkg/handlers/generic/mutation/encryption/inject.go deleted file mode 100644 index 20ff67a70..000000000 --- a/pkg/handlers/generic/mutation/encryption/inject.go +++ /dev/null @@ -1,307 +0,0 @@ -// Copyright 2024 Nutanix. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -package encryption - -import ( - "context" - "encoding/base64" - "fmt" - "strings" - - corev1 "k8s.io/api/core/v1" - apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - apiserverv1 "k8s.io/apiserver/pkg/apis/config/v1" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" - cabpkv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" - controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1" - runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1" - ctrl "sigs.k8s.io/controller-runtime" - ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - "sigs.k8s.io/yaml" - - carenv1 "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1" - "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/handlers/mutation" - "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/patches" - "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/patches/selectors" - "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/variables" - "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/utils" - k8sClientUtil "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/k8s/client" - "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/clusterconfig" -) - -const ( - // VariableName is the external patch variable name. - VariableName = "encryptionAtRest" - SecretKeyForEtcdEncryption = "config" - defaultEncryptionSecretNameTemplate = "%s-encryption-config" //nolint:gosec // Does not contain hard coded credentials. - encryptionConfigurationOnRemote = "/etc/kubernetes/pki/encryptionconfig.yaml" - apiServerEncryptionConfigArg = "encryption-provider-config" -) - -type encryptionPatchHandler struct { - client ctrlclient.Client - keyGenerator TokenGenerator - variableName string - variableFieldPath []string -} - -func NewPatch(client ctrlclient.Client, keyGenerator TokenGenerator) *encryptionPatchHandler { - return &encryptionPatchHandler{ - client: client, - keyGenerator: keyGenerator, - variableName: clusterconfig.MetaVariableName, - variableFieldPath: []string{VariableName}, - } -} - -func (h *encryptionPatchHandler) Mutate( - ctx context.Context, - obj *unstructured.Unstructured, - vars map[string]apiextensionsv1.JSON, - holderRef runtimehooksv1.HolderReference, - clusterKey ctrlclient.ObjectKey, - clusterGetter mutation.ClusterGetter, -) error { - log := ctrl.LoggerFrom(ctx, "holderRef", holderRef) - - encryptionVariable, err := variables.Get[carenv1.EncryptionAtRest]( - vars, - h.variableName, - h.variableFieldPath..., - ) - if err != nil { - if variables.IsNotFoundError(err) { - log.V(5).Info("encryption variable not defined") - return nil - } - return err - } - - log = log.WithValues( - "variableName", - h.variableName, - "variableFieldPath", - h.variableFieldPath, - "variableValue", - encryptionVariable, - ) - - return patches.MutateIfApplicable( - obj, vars, &holderRef, selectors.ControlPlane(), log, - func(obj *controlplanev1.KubeadmControlPlaneTemplate) error { - cluster, err := clusterGetter(ctx) - if err != nil { - log.Error(err, "failed to get cluster from encryption mutation handler") - return err - } - - found, err := h.DefaultEncryptionSecretExists(ctx, cluster) - if err != nil { - log.WithValues( - "defaultEncryptionSecret", defaultEncryptionSecretName(cluster.Name), - ).Error(err, "failed to find default encryption configuration secret") - return err - } - - // we do not rotate or override the secret keys for encryption configuration - if !found { - encConfig, err := h.generateEncryptionConfiguration(encryptionVariable.Providers) - if err != nil { - return err - } - if err := h.CreateEncryptionConfigurationSecret(ctx, encConfig, cluster); err != nil { - return err - } - } - - log.WithValues( - "patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(), - "patchedObjectName", ctrlclient.ObjectKeyFromObject(obj), - ).Info("adding encryption configuration files and API server extra args in control plane kubeadm config spec") - - // Create kubadm config file for encryption config - obj.Spec.Template.Spec.KubeadmConfigSpec.Files = append( - obj.Spec.Template.Spec.KubeadmConfigSpec.Files, - generateEncryptionCredentialsFile(cluster)) - - // set APIServer args for encryption config - if obj.Spec.Template.Spec.KubeadmConfigSpec.ClusterConfiguration == nil { - obj.Spec.Template.Spec.KubeadmConfigSpec.ClusterConfiguration = &cabpkv1.ClusterConfiguration{} - } - apiServer := &obj.Spec.Template.Spec.KubeadmConfigSpec.ClusterConfiguration.APIServer - if apiServer.ExtraArgs == nil { - apiServer.ExtraArgs = make(map[string]string, 1) - } - apiServer.ExtraArgs[apiServerEncryptionConfigArg] = encryptionConfigurationOnRemote - - return nil - }) -} - -func generateEncryptionCredentialsFile(cluster *clusterv1.Cluster) cabpkv1.File { - secretName := defaultEncryptionSecretName(cluster.Name) - return cabpkv1.File{ - Path: encryptionConfigurationOnRemote, - ContentFrom: &cabpkv1.FileSource{ - Secret: cabpkv1.SecretFileSource{ - Name: secretName, - Key: SecretKeyForEtcdEncryption, - }, - }, - Permissions: "0640", - } -} - -func (h *encryptionPatchHandler) generateEncryptionConfiguration( - providers []carenv1.EncryptionProviders, -) (*apiserverv1.EncryptionConfiguration, error) { - resourceConfigs := []apiserverv1.ResourceConfiguration{} - for _, encProvider := range providers { - provider := encProvider - resourceConfig, err := encryptionConfigForSecretsAndConfigMaps( - &provider, - h.keyGenerator, - ) - if err != nil { - return nil, err - } - resourceConfigs = append(resourceConfigs, *resourceConfig) - } - // We only support encryption for "secrets" and "configmaps" using "aescbc" provider. - - return &apiserverv1.EncryptionConfiguration{ - TypeMeta: metav1.TypeMeta{ - APIVersion: apiserverv1.SchemeGroupVersion.String(), - Kind: "EncryptionConfiguration", - }, - Resources: resourceConfigs, - }, nil -} - -func (h *encryptionPatchHandler) DefaultEncryptionSecretExists( - ctx context.Context, - cluster *clusterv1.Cluster, -) (bool, error) { - secretName := defaultEncryptionSecretName(cluster.Name) - existingSecret := &corev1.Secret{ - TypeMeta: metav1.TypeMeta{ - APIVersion: corev1.SchemeGroupVersion.String(), - Kind: "Secret", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: secretName, - Namespace: cluster.Namespace, - }, - } - err := h.client.Get(ctx, ctrlclient.ObjectKeyFromObject(existingSecret), existingSecret) - if err != nil { - if errors.IsNotFound(err) { - return false, nil - } - return false, err - } - return true, nil -} - -func (h *encryptionPatchHandler) CreateEncryptionConfigurationSecret( - ctx context.Context, - encryptionConfig *apiserverv1.EncryptionConfiguration, - cluster *clusterv1.Cluster, -) error { - dataYaml, err := yaml.Marshal(encryptionConfig) - if err != nil { - return fmt.Errorf("unable to marshal encryption configuration to YAML: %w", err) - } - - secretData := map[string]string{ - SecretKeyForEtcdEncryption: strings.TrimSpace(string(dataYaml)), - } - secretName := defaultEncryptionSecretName(cluster.Name) - encryptionConfigSecret := &corev1.Secret{ - TypeMeta: metav1.TypeMeta{ - APIVersion: corev1.SchemeGroupVersion.String(), - Kind: "Secret", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: secretName, - Namespace: cluster.Namespace, - Labels: utils.NewLabels(utils.WithMove(), utils.WithClusterName(cluster.Name)), - }, - StringData: secretData, - Type: corev1.SecretTypeOpaque, - } - - if err = controllerutil.SetOwnerReference(cluster, encryptionConfigSecret, h.client.Scheme()); err != nil { - return fmt.Errorf( - "failed to set owner reference on encryption configuration secret: %w", - err, - ) - } - - // We only support creating encryption config in BeforeClusterCreate hook and ensure that the keys are immutable. - if err := k8sClientUtil.Create(ctx, h.client, encryptionConfigSecret); err != nil { - return fmt.Errorf("failed to create encryption configuration secret: %w", err) - } - return nil -} - -// We only support encryption for "secrets" and "configmaps". -func encryptionConfigForSecretsAndConfigMaps( - providers *carenv1.EncryptionProviders, - secretGenerator TokenGenerator, -) (*apiserverv1.ResourceConfiguration, error) { - providerConfig := apiserverv1.ProviderConfiguration{} - // We only support "aescbc", "secretbox" for now. - // "aesgcm" is another AESConfiguration. "aesgcm" requires secret key rotation before 200k write calls. - // "aesgcm" should not be supported until secret key's rotation is implemented. - if providers.AESCBC != nil { - token, err := secretGenerator() - if err != nil { - return nil, fmt.Errorf( - "could not create random encryption token for aescbc provider: %w", - err, - ) - } - providerConfig.AESCBC = &apiserverv1.AESConfiguration{ - Keys: []apiserverv1.Key{ - { - Name: "key1", // we only support one key during cluster creation. - Secret: base64.StdEncoding.EncodeToString(token), - }, - }, - } - } - if providers.Secretbox != nil { - token, err := secretGenerator() - if err != nil { - return nil, fmt.Errorf( - "could not create random encryption token for secretbox provider: %w", - err, - ) - } - providerConfig.Secretbox = &apiserverv1.SecretboxConfiguration{ - Keys: []apiserverv1.Key{ - { - Name: "key1", // we only support one key during cluster creation. - Secret: base64.StdEncoding.EncodeToString(token), - }, - }, - } - } - - return &apiserverv1.ResourceConfiguration{ - Resources: []string{"secrets", "configmaps"}, - Providers: []apiserverv1.ProviderConfiguration{ - providerConfig, - }, - }, nil -} - -func defaultEncryptionSecretName(clusterName string) string { - return fmt.Sprintf(defaultEncryptionSecretNameTemplate, clusterName) -} diff --git a/pkg/handlers/generic/mutation/encryption/inject_test.go b/pkg/handlers/generic/mutation/encryption/inject_test.go deleted file mode 100644 index e406f14ac..000000000 --- a/pkg/handlers/generic/mutation/encryption/inject_test.go +++ /dev/null @@ -1,224 +0,0 @@ -// Copyright 2024 Nutanix. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -package encryption - -import ( - "testing" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - "github.com/onsi/gomega/format" - "github.com/stretchr/testify/assert" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" - clientgoscheme "k8s.io/client-go/kubernetes/scheme" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" - runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1" - ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" - - carenv1 "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1" - "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/handlers/mutation" - "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/testutils/capitest" - "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/testutils/capitest/request" - "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/clusterconfig" - "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/test/helpers" -) - -const ( - testToken = "testAESConfigKey" //nolint:gosec // Does not contain hard coded credentials. - testEncryptionConfigSecretData = `apiVersion: apiserver.config.k8s.io/v1 -kind: EncryptionConfiguration -resources: -- providers: - - aescbc: - keys: - - name: key1 - secret: dGVzdEFFU0NvbmZpZ0tleQ== - resources: - - secrets - - configmaps` -) - -func testTokenGenerator() ([]byte, error) { - return []byte(testToken), nil -} - -func TestEncryptionConfigurationPatch(t *testing.T) { - RegisterFailHandler(Fail) - format.TruncatedDiff = false - RunSpecs(t, "Encryption configuration mutator suite") -} - -var _ = Describe("Generate Encryption configuration patches", func() { - clientScheme := runtime.NewScheme() - utilruntime.Must(clientgoscheme.AddToScheme(clientScheme)) - utilruntime.Must(clusterv1.AddToScheme(clientScheme)) - patchGenerator := func() mutation.GeneratePatches { - client, err := helpers.TestEnv.GetK8sClientWithScheme(clientScheme) - Expect(err).To(BeNil()) - return mutation.NewMetaGeneratePatchesHandler( - "", - client, - NewPatch(client, testTokenGenerator)).(mutation.GeneratePatches) - } - - encryptionVar := []runtimehooksv1.Variable{ - capitest.VariableWithValue( - clusterconfig.MetaVariableName, - carenv1.EncryptionAtRest{ - Providers: []carenv1.EncryptionProviders{ - { - AESCBC: &carenv1.AESConfiguration{}, - }, - }, - }, - VariableName, - ), - } - encryptionMatchers := []capitest.JSONPatchMatcher{ - { - Operation: "add", - Path: "/spec/template/spec/kubeadmConfigSpec/files", - ValueMatcher: ContainElements( - SatisfyAll( - HaveKeyWithValue( - "path", "/etc/kubernetes/pki/encryptionconfig.yaml", - ), - HaveKeyWithValue( - "permissions", "0640", - ), - HaveKeyWithValue( - "contentFrom", - map[string]interface{}{ - "secret": map[string]interface{}{ - "key": "config", - "name": defaultEncryptionSecretName(request.ClusterName), - }, - }, - ), - ), - ), - }, - { - Operation: "add", - Path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration", - ValueMatcher: HaveKeyWithValue( - "apiServer", - HaveKeyWithValue( - "extraArgs", - map[string]interface{}{ - "encryption-provider-config": "/etc/kubernetes/pki/encryptionconfig.yaml", - }, - ), - ), - }, - } - - // Create cluster before each test - BeforeEach(func(ctx SpecContext) { - client, err := helpers.TestEnv.GetK8sClientWithScheme(clientScheme) - Expect(err).To(BeNil()) - - Expect(client.Create( - ctx, - &clusterv1.Cluster{ - ObjectMeta: metav1.ObjectMeta{ - Name: request.ClusterName, - Namespace: metav1.NamespaceDefault, - }, - }, - )).To(BeNil()) - }) - - // Delete cluster after each test - AfterEach(func(ctx SpecContext) { - client, err := helpers.TestEnv.GetK8sClientWithScheme(clientScheme) - Expect(err).To(BeNil()) - - Expect(client.Delete( - ctx, - &clusterv1.Cluster{ - ObjectMeta: metav1.ObjectMeta{ - Name: request.ClusterName, - Namespace: metav1.NamespaceDefault, - }, - }, - )).To(BeNil()) - }) - - // Test that encryption config patch is generated without recreating the default encryption secret. - // The Mutate function must be ideompotent and always generate patch in success cases. - noOpEncryptionConfigDef := capitest.PatchTestDef{ - Name: "skip creating default encryption config secret if it already exists", - Vars: encryptionVar, - RequestItem: request.NewKubeadmControlPlaneTemplateRequestItem(""), - ExpectedPatchMatchers: encryptionMatchers, - } - - Context("Default encryption provider secret already exists", func() { - // encryption secret was created earlier - BeforeEach(func(ctx SpecContext) { - client, err := helpers.TestEnv.GetK8sClientWithScheme(clientScheme) - Expect(err).To(BeNil()) - - Expect(client.Create( - ctx, - testEncryptionSecretObj(), - )).To(BeNil()) - }) - // delete encryption configuration after the test - AfterEach(func(ctx SpecContext) { - client, err := helpers.TestEnv.GetK8sClientWithScheme(clientScheme) - Expect(err).To(BeNil()) - - Expect(client.Delete( - ctx, - testEncryptionSecretObj(), - )).To(BeNil()) - }) - It(noOpEncryptionConfigDef.Name, func(ctx SpecContext) { - capitest.AssertGeneratePatches(GinkgoT(), patchGenerator, &noOpEncryptionConfigDef) - }) - }) - - // Test that encryption configuration secret is generated and patched on kubeadmconfig spec. - patchEncryptionConfigDef := capitest.PatchTestDef{ - Name: "files added in KubeadmControlPlaneTemplate for Encryption Configuration", - Vars: encryptionVar, - RequestItem: request.NewKubeadmControlPlaneTemplateRequestItem(""), - ExpectedPatchMatchers: encryptionMatchers, - } - - Context("Default encryption configuration secret is created by the patch", func() { - It(patchEncryptionConfigDef.Name, func(ctx SpecContext) { - capitest.AssertGeneratePatches(GinkgoT(), patchGenerator, &patchEncryptionConfigDef) - - // assert secret containing Encryption configuration is generated - client, err := helpers.TestEnv.GetK8sClientWithScheme(clientScheme) - Expect(err).To(BeNil()) - - gotSecret := testEncryptionSecretObj() - err = client.Get( - ctx, - ctrlclient.ObjectKeyFromObject(gotSecret), - gotSecret) - Expect(err).To(BeNil()) - assert.Equal( - GinkgoT(), - testEncryptionConfigSecretData, - string(gotSecret.Data[SecretKeyForEtcdEncryption])) - }) - }) -}) - -func testEncryptionSecretObj() *corev1.Secret { - return &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: defaultEncryptionSecretName(request.ClusterName), - Namespace: request.Namespace, - }, - } -} diff --git a/pkg/handlers/generic/mutation/encryption/tokengenerator.go b/pkg/handlers/generic/mutation/encryption/tokengenerator.go deleted file mode 100644 index ddd202e0a..000000000 --- a/pkg/handlers/generic/mutation/encryption/tokengenerator.go +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright 2024 Nutanix. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -package encryption - -import "crypto/rand" - -const ( - tokenLength = 32 -) - -type TokenGenerator func() ([]byte, error) - -func RandomTokenGenerator() ([]byte, error) { - return createRandBytes(tokenLength) -} - -// createRandBytes returns a cryptographically secure slice of random bytes with a given size. -func createRandBytes(size uint32) ([]byte, error) { - b := make([]byte, size) - if _, err := rand.Read(b); err != nil { - return nil, err - } - return b, nil -} From 45cd9fe1df8816690e68bc6139448d82ea0f2e4f Mon Sep 17 00:00:00 2001 From: Shalin Patel Date: Mon, 13 May 2024 20:52:14 -0700 Subject: [PATCH 07/11] docs: add encryptionAtRest config in capi-quick-start --- examples/capi-quick-start/aws-cluster-calico-crs.yaml | 3 +++ .../capi-quick-start/aws-cluster-calico-helm-addon.yaml | 3 +++ examples/capi-quick-start/aws-cluster-cilium-crs.yaml | 3 +++ .../capi-quick-start/aws-cluster-cilium-helm-addon.yaml | 3 +++ examples/capi-quick-start/docker-cluster-calico-crs.yaml | 3 +++ .../docker-cluster-calico-helm-addon.yaml | 3 +++ examples/capi-quick-start/docker-cluster-cilium-crs.yaml | 3 +++ .../docker-cluster-cilium-helm-addon.yaml | 3 +++ examples/capi-quick-start/nutanix-cluster-calico-crs.yaml | 3 +++ .../nutanix-cluster-calico-helm-addon.yaml | 3 +++ examples/capi-quick-start/nutanix-cluster-cilium-crs.yaml | 3 +++ .../nutanix-cluster-cilium-helm-addon.yaml | 3 +++ hack/examples/bases/aws/cluster/kustomization.yaml.tmpl | 3 +++ .../examples/bases/docker/cluster/kustomization.yaml.tmpl | 3 +++ .../bases/nutanix/cluster/kustomization.yaml.tmpl | 3 +++ hack/examples/patches/encryption.yaml | 8 ++++++++ 16 files changed, 53 insertions(+) create mode 100644 hack/examples/patches/encryption.yaml diff --git a/examples/capi-quick-start/aws-cluster-calico-crs.yaml b/examples/capi-quick-start/aws-cluster-calico-crs.yaml index 790b4a640..f67b253fb 100644 --- a/examples/capi-quick-start/aws-cluster-calico-crs.yaml +++ b/examples/capi-quick-start/aws-cluster-calico-crs.yaml @@ -47,6 +47,9 @@ spec: baseOS: ${AMI_LOOKUP_BASEOS} format: ${AMI_LOOKUP_FORMAT} org: "${AMI_LOOKUP_ORG}" + encryptionAtRest: + providers: + - aescbc: {} - name: workerConfig value: aws: diff --git a/examples/capi-quick-start/aws-cluster-calico-helm-addon.yaml b/examples/capi-quick-start/aws-cluster-calico-helm-addon.yaml index 52bf9648a..7d06d032c 100644 --- a/examples/capi-quick-start/aws-cluster-calico-helm-addon.yaml +++ b/examples/capi-quick-start/aws-cluster-calico-helm-addon.yaml @@ -47,6 +47,9 @@ spec: baseOS: ${AMI_LOOKUP_BASEOS} format: ${AMI_LOOKUP_FORMAT} org: "${AMI_LOOKUP_ORG}" + encryptionAtRest: + providers: + - aescbc: {} - name: workerConfig value: aws: diff --git a/examples/capi-quick-start/aws-cluster-cilium-crs.yaml b/examples/capi-quick-start/aws-cluster-cilium-crs.yaml index e9541e789..c0f4acff1 100644 --- a/examples/capi-quick-start/aws-cluster-cilium-crs.yaml +++ b/examples/capi-quick-start/aws-cluster-cilium-crs.yaml @@ -47,6 +47,9 @@ spec: baseOS: ${AMI_LOOKUP_BASEOS} format: ${AMI_LOOKUP_FORMAT} org: "${AMI_LOOKUP_ORG}" + encryptionAtRest: + providers: + - aescbc: {} - name: workerConfig value: aws: diff --git a/examples/capi-quick-start/aws-cluster-cilium-helm-addon.yaml b/examples/capi-quick-start/aws-cluster-cilium-helm-addon.yaml index e803994e8..fb6441e0f 100644 --- a/examples/capi-quick-start/aws-cluster-cilium-helm-addon.yaml +++ b/examples/capi-quick-start/aws-cluster-cilium-helm-addon.yaml @@ -47,6 +47,9 @@ spec: baseOS: ${AMI_LOOKUP_BASEOS} format: ${AMI_LOOKUP_FORMAT} org: "${AMI_LOOKUP_ORG}" + encryptionAtRest: + providers: + - aescbc: {} - name: workerConfig value: aws: diff --git a/examples/capi-quick-start/docker-cluster-calico-crs.yaml b/examples/capi-quick-start/docker-cluster-calico-crs.yaml index 69673293e..551aab200 100644 --- a/examples/capi-quick-start/docker-cluster-calico-crs.yaml +++ b/examples/capi-quick-start/docker-cluster-calico-crs.yaml @@ -29,6 +29,9 @@ spec: strategy: ClusterResourceSet nfd: strategy: ClusterResourceSet + encryptionAtRest: + providers: + - aescbc: {} - name: workerConfig value: {} version: ${KUBERNETES_VERSION} diff --git a/examples/capi-quick-start/docker-cluster-calico-helm-addon.yaml b/examples/capi-quick-start/docker-cluster-calico-helm-addon.yaml index 459f248a0..53c5cc4e7 100644 --- a/examples/capi-quick-start/docker-cluster-calico-helm-addon.yaml +++ b/examples/capi-quick-start/docker-cluster-calico-helm-addon.yaml @@ -29,6 +29,9 @@ spec: strategy: HelmAddon nfd: strategy: HelmAddon + encryptionAtRest: + providers: + - aescbc: {} - name: workerConfig value: {} version: ${KUBERNETES_VERSION} diff --git a/examples/capi-quick-start/docker-cluster-cilium-crs.yaml b/examples/capi-quick-start/docker-cluster-cilium-crs.yaml index 7409544d2..0688f562a 100644 --- a/examples/capi-quick-start/docker-cluster-cilium-crs.yaml +++ b/examples/capi-quick-start/docker-cluster-cilium-crs.yaml @@ -29,6 +29,9 @@ spec: strategy: ClusterResourceSet nfd: strategy: ClusterResourceSet + encryptionAtRest: + providers: + - aescbc: {} - name: workerConfig value: {} version: ${KUBERNETES_VERSION} diff --git a/examples/capi-quick-start/docker-cluster-cilium-helm-addon.yaml b/examples/capi-quick-start/docker-cluster-cilium-helm-addon.yaml index 8c2b7ef5e..1eda93a6e 100644 --- a/examples/capi-quick-start/docker-cluster-cilium-helm-addon.yaml +++ b/examples/capi-quick-start/docker-cluster-cilium-helm-addon.yaml @@ -29,6 +29,9 @@ spec: strategy: HelmAddon nfd: strategy: HelmAddon + encryptionAtRest: + providers: + - aescbc: {} - name: workerConfig value: {} version: ${KUBERNETES_VERSION} diff --git a/examples/capi-quick-start/nutanix-cluster-calico-crs.yaml b/examples/capi-quick-start/nutanix-cluster-calico-crs.yaml index 1680514c4..6d30f72db 100644 --- a/examples/capi-quick-start/nutanix-cluster-calico-crs.yaml +++ b/examples/capi-quick-start/nutanix-cluster-calico-crs.yaml @@ -105,6 +105,9 @@ spec: systemDiskSize: 40Gi vcpuSockets: 2 vcpusPerSocket: 1 + encryptionAtRest: + providers: + - aescbc: {} imageRegistries: - credentials: secretRef: diff --git a/examples/capi-quick-start/nutanix-cluster-calico-helm-addon.yaml b/examples/capi-quick-start/nutanix-cluster-calico-helm-addon.yaml index 3a68dfa6f..cad76de30 100644 --- a/examples/capi-quick-start/nutanix-cluster-calico-helm-addon.yaml +++ b/examples/capi-quick-start/nutanix-cluster-calico-helm-addon.yaml @@ -105,6 +105,9 @@ spec: systemDiskSize: 40Gi vcpuSockets: 2 vcpusPerSocket: 1 + encryptionAtRest: + providers: + - aescbc: {} imageRegistries: - credentials: secretRef: diff --git a/examples/capi-quick-start/nutanix-cluster-cilium-crs.yaml b/examples/capi-quick-start/nutanix-cluster-cilium-crs.yaml index f5e464ddb..fc8d480bc 100644 --- a/examples/capi-quick-start/nutanix-cluster-cilium-crs.yaml +++ b/examples/capi-quick-start/nutanix-cluster-cilium-crs.yaml @@ -105,6 +105,9 @@ spec: systemDiskSize: 40Gi vcpuSockets: 2 vcpusPerSocket: 1 + encryptionAtRest: + providers: + - aescbc: {} imageRegistries: - credentials: secretRef: diff --git a/examples/capi-quick-start/nutanix-cluster-cilium-helm-addon.yaml b/examples/capi-quick-start/nutanix-cluster-cilium-helm-addon.yaml index 5c1d2ce3b..0dd369e6c 100644 --- a/examples/capi-quick-start/nutanix-cluster-cilium-helm-addon.yaml +++ b/examples/capi-quick-start/nutanix-cluster-cilium-helm-addon.yaml @@ -105,6 +105,9 @@ spec: systemDiskSize: 40Gi vcpuSockets: 2 vcpusPerSocket: 1 + encryptionAtRest: + providers: + - aescbc: {} imageRegistries: - credentials: secretRef: diff --git a/hack/examples/bases/aws/cluster/kustomization.yaml.tmpl b/hack/examples/bases/aws/cluster/kustomization.yaml.tmpl index b90db5a0b..0e3f8436c 100644 --- a/hack/examples/bases/aws/cluster/kustomization.yaml.tmpl +++ b/hack/examples/bases/aws/cluster/kustomization.yaml.tmpl @@ -45,6 +45,9 @@ patches: - target: kind: Cluster path: ../../../patches/aws/config-var.yaml +- target: + kind: Cluster + path: ../../../patches/encryption.yaml # Delete the clusterclass-specific resources. - target: diff --git a/hack/examples/bases/docker/cluster/kustomization.yaml.tmpl b/hack/examples/bases/docker/cluster/kustomization.yaml.tmpl index 22bfa1a47..7a4e271eb 100644 --- a/hack/examples/bases/docker/cluster/kustomization.yaml.tmpl +++ b/hack/examples/bases/docker/cluster/kustomization.yaml.tmpl @@ -35,3 +35,6 @@ patches: - target: kind: Cluster path: ../../../patches/cluster-autoscaler.yaml +- target: + kind: Cluster + path: ../../../patches/encryption.yaml diff --git a/hack/examples/bases/nutanix/cluster/kustomization.yaml.tmpl b/hack/examples/bases/nutanix/cluster/kustomization.yaml.tmpl index 843bf5281..b03bee535 100644 --- a/hack/examples/bases/nutanix/cluster/kustomization.yaml.tmpl +++ b/hack/examples/bases/nutanix/cluster/kustomization.yaml.tmpl @@ -36,6 +36,9 @@ patches: - target: kind: Cluster path: ../../../patches/nutanix/initialize-variables.yaml +- target: + kind: Cluster + path: ../../../patches/encryption.yaml # Remove Additional Trust Bundle ConfigMap - target: diff --git a/hack/examples/patches/encryption.yaml b/hack/examples/patches/encryption.yaml new file mode 100644 index 000000000..b96c73a89 --- /dev/null +++ b/hack/examples/patches/encryption.yaml @@ -0,0 +1,8 @@ +# Copyright 2024 Nutanix. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +- op: "add" + path: "/spec/topology/variables/0/value/encryptionAtRest" + value: + providers: + - aescbc: {} From 75df09d3615301a14e173780167d8179abcf80ad Mon Sep 17 00:00:00 2001 From: Shalin Patel Date: Mon, 13 May 2024 20:52:54 -0700 Subject: [PATCH 08/11] docs: API docs for encryptionAtRest --- .../generic/encryption-at-rest.md | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 docs/content/customization/generic/encryption-at-rest.md diff --git a/docs/content/customization/generic/encryption-at-rest.md b/docs/content/customization/generic/encryption-at-rest.md new file mode 100644 index 000000000..d97bbf012 --- /dev/null +++ b/docs/content/customization/generic/encryption-at-rest.md @@ -0,0 +1,60 @@ ++++ +title = "Encryption At REST" ++++ + +`encryptionAtRest` variable enables encrypting kubernetes resources at REST using provided encryption provider. +When this variable is set, kuberntetes secrets and configmaps are encrypted before writing them at `etcd`. + +If the `encryptionAtRest` property is not specified, then +the customization will be skipped. The secrets and configmaps will not be stored as encrypted in `etcd`. + +We support following encryption providers + +- aescbc +- secretbox + +More information about encryption at REST: [Encrypting Confidential Data at Rest +](https://kubernetes.io/docs/tasks/administer-cluster/encrypt-data/) + +## Example + +To encrypt configmaps and secrets for using `aescbc` and `secretbox` encryption providers: + +```yaml +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: +spec: + topology: + variables: + - name: clusterConfig + value: + encryptionAtRest: + providers: + - aescbc: {} + - secretbox: {} +``` + +Applying this configuration will result in + +1. `-encryption-config` secret generated +1. following value being set: + +- `KubeadmControlPlaneTemplate`: + + - ```yaml + spec: + kubeadmConfigSpec: + clusterConfiguration: + apiServer: + extraArgs: + encryption-provider-config: /etc/kubernetes/pki/encryptionconfig.yaml + files: + - contentFrom: + secret: + key: config + name: my-cluster-encryption-config + path: /etc/kubernetes/pki/encryptionconfig.yaml + permissions: "0640" + ``` From eb7576b97509f2ecdc43eb9b444ba92e7ecb3abd Mon Sep 17 00:00:00 2001 From: Shalin Patel Date: Wed, 15 May 2024 09:44:14 -0700 Subject: [PATCH 09/11] docs: add detail about encryption keys --- api/v1alpha1/zz_generated.deepcopy.go | 25 ------------------- .../generic/encryption-at-rest.md | 24 +++++++++++------- 2 files changed, 15 insertions(+), 34 deletions(-) diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index e99d50b7d..1fbf78849 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -751,31 +751,6 @@ func (in *EncryptionProviders) DeepCopy() *EncryptionProviders { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *EncryptionProviders) DeepCopyInto(out *EncryptionProviders) { - *out = *in - if in.AESCBC != nil { - in, out := &in.AESCBC, &out.AESCBC - *out = new(AESConfiguration) - **out = **in - } - if in.Secretbox != nil { - in, out := &in.Secretbox, &out.Secretbox - *out = new(SecretboxConfiguration) - **out = **in - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EncryptionProviders. -func (in *EncryptionProviders) DeepCopy() *EncryptionProviders { - if in == nil { - return nil - } - out := new(EncryptionProviders) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Etcd) DeepCopyInto(out *Etcd) { *out = *in diff --git a/docs/content/customization/generic/encryption-at-rest.md b/docs/content/customization/generic/encryption-at-rest.md index d97bbf012..d89d9652b 100644 --- a/docs/content/customization/generic/encryption-at-rest.md +++ b/docs/content/customization/generic/encryption-at-rest.md @@ -3,10 +3,10 @@ title = "Encryption At REST" +++ `encryptionAtRest` variable enables encrypting kubernetes resources at REST using provided encryption provider. -When this variable is set, kuberntetes secrets and configmaps are encrypted before writing them at `etcd`. +When this variable is set, kuberntetes `secrets` and `configmap`s are encrypted before writing them at `etcd`. If the `encryptionAtRest` property is not specified, then -the customization will be skipped. The secrets and configmaps will not be stored as encrypted in `etcd`. +the customization will be skipped. The `secrets` and `configmaps` will not be stored as encrypted in `etcd`. We support following encryption providers @@ -18,7 +18,7 @@ More information about encryption at REST: [Encrypting Confidential Data at Rest ## Example -To encrypt configmaps and secrets for using `aescbc` and `secretbox` encryption providers: +To encrypt `configmaps` and `secrets` kubernetes resources using `aescbc` encryption provider: ```yaml apiVersion: cluster.x-k8s.io/v1beta1 @@ -33,17 +33,23 @@ spec: encryptionAtRest: providers: - aescbc: {} - - secretbox: {} ``` Applying this configuration will result in -1. `-encryption-config` secret generated -1. following value being set: +1. `-encryption-config` secret generated. + + A secret key for the encryption provider is generated and stored in `-encryption-config` secret. + The APIServer will be configured to use the secret key to encrypt `secrets` and + `configmaps` kubernetes resources before writing them to etcd. + When reading resources from `etcd`, encryption provider that matches the stored data attempts in order to decrypt the data. + We currently do not rotate the key once it generated. + +1. Configure APIServer with encryption configuration: - `KubeadmControlPlaneTemplate`: - - ```yaml + ```yaml spec: kubeadmConfigSpec: clusterConfiguration: @@ -54,7 +60,7 @@ Applying this configuration will result in - contentFrom: secret: key: config - name: my-cluster-encryption-config + name: -encryption-config path: /etc/kubernetes/pki/encryptionconfig.yaml permissions: "0640" - ``` + ``` From 05b4f8b128c71eaa316a537f7369ea9249ffbbfc Mon Sep 17 00:00:00 2001 From: Shalin Patel Date: Thu, 16 May 2024 18:00:18 -0700 Subject: [PATCH 10/11] fixup! docs: Apply suggestions from code review address review comments Co-authored-by: Dimitri Koshkin --- docs/content/customization/generic/encryption-at-rest.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/content/customization/generic/encryption-at-rest.md b/docs/content/customization/generic/encryption-at-rest.md index d89d9652b..71f6a47bf 100644 --- a/docs/content/customization/generic/encryption-at-rest.md +++ b/docs/content/customization/generic/encryption-at-rest.md @@ -1,8 +1,8 @@ +++ -title = "Encryption At REST" +title = "Encryption At Rest" +++ -`encryptionAtRest` variable enables encrypting kubernetes resources at REST using provided encryption provider. +`encryptionAtRest` variable enables encrypting kubernetes resources at rest using provided encryption provider. When this variable is set, kuberntetes `secrets` and `configmap`s are encrypted before writing them at `etcd`. If the `encryptionAtRest` property is not specified, then @@ -13,7 +13,7 @@ We support following encryption providers - aescbc - secretbox -More information about encryption at REST: [Encrypting Confidential Data at Rest +More information about encryption at-rest: [Encrypting Confidential Data at Rest ](https://kubernetes.io/docs/tasks/administer-cluster/encrypt-data/) ## Example @@ -43,7 +43,7 @@ Applying this configuration will result in The APIServer will be configured to use the secret key to encrypt `secrets` and `configmaps` kubernetes resources before writing them to etcd. When reading resources from `etcd`, encryption provider that matches the stored data attempts in order to decrypt the data. - We currently do not rotate the key once it generated. + CAREN currently does not rotate the key once it generated. 1. Configure APIServer with encryption configuration: From 89b59bbb4992d4943a336effa827a7cb2def6a85 Mon Sep 17 00:00:00 2001 From: Jimmi Dyson Date: Fri, 17 May 2024 09:17:07 +0100 Subject: [PATCH 11/11] fixup! test(e2e): Fix up ownership reference checks --- test/e2e/ownerreference_helpers.go | 4 ++-- test/e2e/quick_start_test.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/e2e/ownerreference_helpers.go b/test/e2e/ownerreference_helpers.go index 9e67d6208..b6915ac2a 100644 --- a/test/e2e/ownerreference_helpers.go +++ b/test/e2e/ownerreference_helpers.go @@ -167,8 +167,8 @@ var ( // https://github.com/kubernetes-sigs/cluster-api/tree/main/docs/book/src/reference/owner_references.md. KubernetesReferenceAssertions = map[string]func([]metav1.OwnerReference) error{ secretKind: func(owners []metav1.OwnerReference) error { - // TODO:deepakm-ntnx Currently pc-creds, pc-creds-for-csi, dockerhub-credentials - // and registry-creds have unexpected owners which needs more investigation + // TODO:deepakm-ntnx Currently pc-creds, pc-creds-for-csi, dockerhub-credentials, + // registry-creds, and encryption config secrets have unexpected owners which needs more investigation. return nil }, configMapKind: func(owners []metav1.OwnerReference) error { diff --git a/test/e2e/quick_start_test.go b/test/e2e/quick_start_test.go index 4dc061ca3..8f8bd0564 100644 --- a/test/e2e/quick_start_test.go +++ b/test/e2e/quick_start_test.go @@ -93,7 +93,7 @@ var _ = Describe("Quick start", Serial, func() { framework.DockerInfraOwnerReferenceAssertions, framework.KubeadmBootstrapOwnerReferenceAssertions, framework.KubeadmControlPlaneOwnerReferenceAssertions, - framework.KubernetesReferenceAssertions, + AWSInfraOwnerReferenceAssertions, NutanixInfraOwnerReferenceAssertions, AddonReferenceAssertions, KubernetesReferenceAssertions,