From 3fde31a71e7c9698956b586275987c67384ef9e8 Mon Sep 17 00:00:00 2001 From: Dimitri Koshkin Date: Thu, 21 Sep 2023 16:22:40 -0700 Subject: [PATCH 01/11] feat: add imageRegistryCredentials handler --- api/v1alpha1/clusterconfig_types.go | 39 +++ api/v1alpha1/zz_generated.deepcopy.go | 16 + .../templates/role.yaml | 2 + cmd/main.go | 2 + common/pkg/testutils/capitest/patches.go | 3 +- .../pkg/testutils/capitest/request/items.go | 18 + docs/content/addons/calico-cni.md | 2 +- docs/content/customization/_index.md | 3 + .../generic/image-registry-credentials.md | 44 +++ go.mod | 1 + go.sum | 2 + .../credential_provider_config_files.go | 231 +++++++++++++ .../credential_provider_config_files_test.go | 176 ++++++++++ .../credential_provider_insall_files.go | 63 ++++ .../credentialprovider/doc.go | 7 + .../credentialprovider/matcher.go | 71 ++++ .../credentialprovider/urls.go | 87 +++++ .../credentialprovider/urls_test.go | 107 ++++++ .../credentials_secret.go | 128 +++++++ .../mutation/imageregistrycredentials/doc.go | 5 + .../imageregistrycredentials/inject.go | 312 ++++++++++++++++++ .../imageregistrycredentials/inject_test.go | 189 +++++++++++ ...mic-credential-provider-config.yaml.gotmpl | 30 ++ ...age-credential-provider-config.yaml.gotmpl | 20 ++ ...all-kubelet-credential-providers.sh.gotmpl | 23 ++ .../static-credential-provider.json.gotmpl | 13 + .../imageregistrycredentials/variables.go | 51 +++ .../variables_test.go | 36 ++ 28 files changed, 1679 insertions(+), 2 deletions(-) create mode 100644 docs/content/customization/generic/image-registry-credentials.md create mode 100644 pkg/handlers/generic/mutation/imageregistrycredentials/credential_provider_config_files.go create mode 100644 pkg/handlers/generic/mutation/imageregistrycredentials/credential_provider_config_files_test.go create mode 100644 pkg/handlers/generic/mutation/imageregistrycredentials/credential_provider_insall_files.go create mode 100644 pkg/handlers/generic/mutation/imageregistrycredentials/credentialprovider/doc.go create mode 100644 pkg/handlers/generic/mutation/imageregistrycredentials/credentialprovider/matcher.go create mode 100644 pkg/handlers/generic/mutation/imageregistrycredentials/credentialprovider/urls.go create mode 100644 pkg/handlers/generic/mutation/imageregistrycredentials/credentialprovider/urls_test.go create mode 100644 pkg/handlers/generic/mutation/imageregistrycredentials/credentials_secret.go create mode 100644 pkg/handlers/generic/mutation/imageregistrycredentials/doc.go create mode 100644 pkg/handlers/generic/mutation/imageregistrycredentials/inject.go create mode 100644 pkg/handlers/generic/mutation/imageregistrycredentials/inject_test.go create mode 100644 pkg/handlers/generic/mutation/imageregistrycredentials/templates/dynamic-credential-provider-config.yaml.gotmpl create mode 100644 pkg/handlers/generic/mutation/imageregistrycredentials/templates/image-credential-provider-config.yaml.gotmpl create mode 100644 pkg/handlers/generic/mutation/imageregistrycredentials/templates/install-kubelet-credential-providers.sh.gotmpl create mode 100644 pkg/handlers/generic/mutation/imageregistrycredentials/templates/static-credential-provider.json.gotmpl create mode 100644 pkg/handlers/generic/mutation/imageregistrycredentials/variables.go create mode 100644 pkg/handlers/generic/mutation/imageregistrycredentials/variables_test.go diff --git a/api/v1alpha1/clusterconfig_types.go b/api/v1alpha1/clusterconfig_types.go index efcd8dcf4..1b384d16d 100644 --- a/api/v1alpha1/clusterconfig_types.go +++ b/api/v1alpha1/clusterconfig_types.go @@ -28,6 +28,10 @@ type GenericClusterConfig struct { // +optional ExtraAPIServerCertSANs ExtraAPIServerCertSANs `json:"extraAPIServerCertSANs,omitempty"` + // TODO: Add support for multiple registries. + // +optional + ImageRegistryCredentials ImageRegistryCredentials `json:"imageRegistryCredentials,omitempty"` + // +optional Addons *Addons `json:"addons,omitempty"` } @@ -46,6 +50,7 @@ func (GenericClusterConfig) VariableSchema() clusterv1.VariableSchema { "", ).VariableSchema(). OpenAPIV3Schema, + "imageRegistryCredentials": ImageRegistryCredentials{}.VariableSchema().OpenAPIV3Schema, }, }, } @@ -175,6 +180,40 @@ func (ExtraAPIServerCertSANs) VariableSchema() clusterv1.VariableSchema { } } +// ImageRegistryCredentials required for providing credentials for an image registry URL. +type ImageRegistryCredentials struct { + // Registry URL. + URL string `json:"url"` + + // Name of the Secret containing the registry credentials. + // The Secret should have keys 'username' and 'password'. + // This credentials Secret is not required for some registries, e.g. ECR. + // +optional + Secret string `json:"secret,omitempty"` +} + +func (ImageRegistryCredentials) VariableSchema() clusterv1.VariableSchema { + return clusterv1.VariableSchema{ + OpenAPIV3Schema: clusterv1.JSONSchemaProps{ + Description: "Extra Subject Alternative Names for the API Server signing cert", + Type: "object", + Properties: map[string]clusterv1.JSONSchemaProps{ + "url": { + Description: "Registry URL.", + Type: "string", + }, + "secret": { + Description: "Name of the Secret containing the registry credentials. " + + "The Secret should have keys 'username' and 'password'. " + + "This credentials Secret is not required for some registries, e.g. ECR.", + Type: "string", + }, + }, + Required: []string{"url"}, + }, + } +} + type Addons struct { // +optional CNI *CNI `json:"cni,omitempty"` diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 7554c215a..68e175118 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -207,6 +207,7 @@ func (in *GenericClusterConfig) DeepCopyInto(out *GenericClusterConfig) { *out = make(ExtraAPIServerCertSANs, len(*in)) copy(*out, *in) } + out.ImageRegistryCredentials = in.ImageRegistryCredentials if in.Addons != nil { in, out := &in.Addons, &out.Addons *out = new(Addons) @@ -259,6 +260,21 @@ func (in *Image) DeepCopy() *Image { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ImageRegistryCredentials) DeepCopyInto(out *ImageRegistryCredentials) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ImageRegistryCredentials. +func (in *ImageRegistryCredentials) DeepCopy() *ImageRegistryCredentials { + if in == nil { + return nil + } + out := new(ImageRegistryCredentials) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NFD) DeepCopyInto(out *NFD) { *out = *in diff --git a/charts/capi-runtime-extensions/templates/role.yaml b/charts/capi-runtime-extensions/templates/role.yaml index 5aa332ffb..8f5b5985c 100644 --- a/charts/capi-runtime-extensions/templates/role.yaml +++ b/charts/capi-runtime-extensions/templates/role.yaml @@ -23,8 +23,10 @@ rules: resources: - secrets verbs: + - create - get - list + - patch - watch - apiGroups: - addons.cluster.x-k8s.io diff --git a/cmd/main.go b/cmd/main.go index afe227e31..e02d2808c 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -39,6 +39,7 @@ import ( "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/etcd" "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/extraapiservercertsans" "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/httpproxy" + "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/imageregistrycredentials" "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/kubernetesimagerepository" ) @@ -135,6 +136,7 @@ func main() { extraapiservercertsans.NewMetaPatch(), httpproxy.NewMetaPatch(mgr.GetClient()), kubernetesimagerepository.NewMetaPatch(), + imageregistrycredentials.NewMetaPatch(mgr.GetClient()), } // awsMetaPatchHandlers combines all AWS patch and variable handlers under a single handler. diff --git a/common/pkg/testutils/capitest/patches.go b/common/pkg/testutils/capitest/patches.go index 64231043d..b7377947e 100644 --- a/common/pkg/testutils/capitest/patches.go +++ b/common/pkg/testutils/capitest/patches.go @@ -6,6 +6,7 @@ package capitest import ( "context" "encoding/json" + "fmt" "testing" "github.com/onsi/gomega" @@ -55,7 +56,7 @@ func ValidateGeneratePatches[T mutation.GeneratePatches]( } resp := &runtimehooksv1.GeneratePatchesResponse{} h.GeneratePatches(context.Background(), req, resp) - g.Expect(resp.Status).To(gomega.Equal(runtimehooksv1.ResponseStatusSuccess)) + g.Expect(resp.Status).To(gomega.Equal(runtimehooksv1.ResponseStatusSuccess), fmt.Sprintf("Message: %s", resp.Message)) if len(tt.ExpectedPatchMatchers) == 0 { g.Expect(resp.Items).To(gomega.BeEmpty()) diff --git a/common/pkg/testutils/capitest/request/items.go b/common/pkg/testutils/capitest/request/items.go index 334eaf753..a8619564b 100644 --- a/common/pkg/testutils/capitest/request/items.go +++ b/common/pkg/testutils/capitest/request/items.go @@ -4,6 +4,7 @@ package request import ( + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" @@ -16,6 +17,13 @@ import ( "github.com/d2iq-labs/capi-runtime-extensions/common/pkg/testutils/capitest/serializer" ) +const ( + ClusterName = "test-cluster" + KubeadmConfigTemplateRequestObjectName = "test-kubeadmconfigtemplate" + KubeadmControlPlaneTemplateRequestObjectName = "test-kubeadmcontrolplanetemplate" + Namespace = corev1.NamespaceDefault +) + // NewRequestItem returns a GeneratePatchesRequestItem with the given variables and object. func NewRequestItem( object runtime.Object, @@ -42,6 +50,10 @@ func NewKubeadmConfigTemplateRequestItem(uid types.UID) runtimehooksv1.GenerateP APIVersion: bootstrapv1.GroupVersion.String(), Kind: "KubeadmConfigTemplate", }, + ObjectMeta: metav1.ObjectMeta{ + Name: KubeadmConfigTemplateRequestObjectName, + Namespace: Namespace, + }, Spec: bootstrapv1.KubeadmConfigTemplateSpec{ Template: bootstrapv1.KubeadmConfigTemplateResource{ Spec: bootstrapv1.KubeadmConfigSpec{ @@ -67,10 +79,16 @@ func NewKubeadmControlPlaneTemplateRequestItem( APIVersion: controlplanev1.GroupVersion.String(), Kind: "KubeadmControlPlaneTemplate", }, + ObjectMeta: metav1.ObjectMeta{ + Name: KubeadmControlPlaneTemplateRequestObjectName, + Namespace: Namespace, + }, }, &runtimehooksv1.HolderReference{ Kind: "Cluster", FieldPath: "spec.controlPlaneRef", + Name: ClusterName, + Namespace: Namespace, }, uid, ) diff --git a/docs/content/addons/calico-cni.md b/docs/content/addons/calico-cni.md index 0728317ef..899cd2d9c 100644 --- a/docs/content/addons/calico-cni.md +++ b/docs/content/addons/calico-cni.md @@ -38,6 +38,6 @@ configure defaults specific for their environment rather than compiling the defa The Helm chart comes with default configurations for the Calico Installation CRS per supported provider, but overriding is possible. To do so, specify: -```bash +```shell --set-file handlers.CalicoCNI.defaultInstallationConfigMaps.DockerCluster.configMap.content= ``` diff --git a/docs/content/customization/_index.md b/docs/content/customization/_index.md index e03562baa..72588a43b 100644 --- a/docs/content/customization/_index.md +++ b/docs/content/customization/_index.md @@ -39,6 +39,9 @@ spec: additionalNo: - no-proxy-1.example.com - no-proxy-2.example.com + imageRegistryCredentials: + url: https://my-registry.io + secret: my-registry-credentials cni: provider: calico ``` diff --git a/docs/content/customization/generic/image-registry-credentials.md b/docs/content/customization/generic/image-registry-credentials.md new file mode 100644 index 000000000..a84f0fd71 --- /dev/null +++ b/docs/content/customization/generic/image-registry-credentials.md @@ -0,0 +1,44 @@ ++++ +title = "Image registry credentials" ++++ + +In some network environments it is necessary to use HTTP proxy to successfuly execute HTTP requests. +To configure Kubernetes components (`containerd`, `kubelet`) to use HTTP proxy use the `httpproxypatch` +external patch that will generate appropriate configuration for control plane and worker nodes. + +Add image registry credentials to all Nodes in the cluster. +When this handle is enabled, the handler will add `files` and `preKubeadmnCommands` with configurations for +[Kubelet image credential provider](https://kubernetes.io/docs/tasks/administer-cluster/kubelet-credential-provider/) +and [dynamic credential provider](https://github.com/mesosphere/dynamic-credential-provider). + +This customization will be available when the +[provider-specific cluster configuration patch]({{< ref "..">}}) is included in the `ClusterClass`. + +## Example + +If your registry requires static credentials, create a Kubernetes Secret with keys for `username` and `password`: + +```shell +kubectl create secret generic my-registry-credentials \ + --from-literal username=${REGISTRY_USERNAME} password=${REGISTRY_PASSWORD} +``` + +On the cluster resource then specify desired image registry credentials: + +```yaml +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: +spec: + topology: + variables: + - name: clusterConfig + value: + imageRegistryCredentials: + url: https://my-registry.io + secret: my-registry-credentials +``` + +Applying this configuration will result in new files and preKubeadmCommands +on the `KubeadmControlPlaneTemplate` and `KubeadmConfigTemplate`. diff --git a/go.mod b/go.mod index 00b515ba7..09f29246b 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( k8s.io/client-go v0.28.2 k8s.io/component-base v0.28.2 k8s.io/klog/v2 v2.100.1 + k8s.io/kubelet v0.28.2 k8s.io/utils v0.0.0-20230726121419-3b25d923346b sigs.k8s.io/cluster-api v1.5.2 sigs.k8s.io/controller-runtime v0.16.2 diff --git a/go.sum b/go.sum index 7db7b7d60..c1acf768e 100644 --- a/go.sum +++ b/go.sum @@ -512,6 +512,8 @@ k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg= k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 h1:LyMgNKD2P8Wn1iAwQU5OhxCKlKJy0sHc+PcDwFB24dQ= k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9/go.mod h1:wZK2AVp1uHCp4VamDVgBP2COHZjqD1T68Rf0CM3YjSM= +k8s.io/kubelet v0.28.2 h1:wqe5zKtVhNWwtdABU0mpcWVe8hc6VdVvs2kqQridZRw= +k8s.io/kubelet v0.28.2/go.mod h1:rvd0e7T5TjPcfZvy62P90XhFzp0IhPIOy+Pqy3Rtipo= k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= diff --git a/pkg/handlers/generic/mutation/imageregistrycredentials/credential_provider_config_files.go b/pkg/handlers/generic/mutation/imageregistrycredentials/credential_provider_config_files.go new file mode 100644 index 000000000..d9dec04de --- /dev/null +++ b/pkg/handlers/generic/mutation/imageregistrycredentials/credential_provider_config_files.go @@ -0,0 +1,231 @@ +// Copyright 2023 D2iQ, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package imageregistrycredentials + +import ( + "bytes" + _ "embed" + "fmt" + "net/url" + "path" + "text/template" + + credentialproviderv1alpha1 "k8s.io/kubelet/pkg/apis/credentialprovider/v1alpha1" + credentialproviderv1beta1 "k8s.io/kubelet/pkg/apis/credentialprovider/v1beta1" + cabpkv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" + + "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/imageregistrycredentials/credentialprovider" +) + +const ( + //nolint:gosec // Does not contain hard coded credentials. + kubeletStaticCredentialProviderCredentialsOnRemote = "/etc/kubernetes/static-image-credentials.json" + + //nolint:gosec // Does not contain hard coded credentials. + kubeletImageCredentialProviderConfigOnRemote = "/etc/kubernetes/image-credential-provider-config.yaml" + + //nolint:gosec // Does not contain hard coded credentials. + kubeletDynamicCredentialProviderConfigOnRemote = "/etc/kubernetes/dynamic-credential-provider-config.yaml" + + azureCloudConfigFilePath = "/etc/kubernetes/azure.json" + + //nolint:gosec // Does not contain hard coded credentials. + dynamicCredentialProviderImage = "ghcr.io/mesosphere/dynamic-credential-provider:v0.2.0" + + //nolint:gosec // Does not contain hard coded credentials. + credentialProviderTargetDir = "/etc/kubernetes/image-credential-provider/" +) + +var ( + //go:embed templates/dynamic-credential-provider-config.yaml.gotmpl + dynamicCredentialProviderConfigPatch []byte + + //go:embed templates/static-credential-provider.json.gotmpl + staticCredentialProviderConfigPatch []byte + + //go:embed templates/image-credential-provider-config.yaml.gotmpl + imageCredentialProviderConfigPatch []byte + + //go:embed templates/install-kubelet-credential-providers.sh.gotmpl + installKubeletCredentialProvidersScript []byte +) + +type imageRegistryCredentials struct { + URL string + Username string + Password string +} + +func (c imageRegistryCredentials) isCredentialsEmpty() bool { + return c.Username == "" && + c.Password == "" +} + +func templateFilesForImageCredentialProviderConfigs(credentials imageRegistryCredentials) ([]cabpkv1.File, error) { + var files []cabpkv1.File + + kubeletCredentialProviderConfigFile, err := templateKubeletCredentialProviderConfig(credentials) + if err != nil { + return nil, err + } + if kubeletCredentialProviderConfigFile != nil { + files = append(files, *kubeletCredentialProviderConfigFile) + } + + kubeletDynamicCredentialProviderConfigFile, err := templateDynamicCredentialProviderConfig(credentials) + if err != nil { + return nil, err + } + if kubeletDynamicCredentialProviderConfigFile != nil { + files = append(files, *kubeletDynamicCredentialProviderConfigFile) + } + + return files, nil +} + +func templateKubeletCredentialProviderConfig(credentials imageRegistryCredentials) (*cabpkv1.File, error) { + return templateCredentialProviderConfig( + credentials, + imageCredentialProviderConfigPatch, + kubeletImageCredentialProviderConfigOnRemote, + kubeletCredentialProvider, + ) +} + +func templateDynamicCredentialProviderConfig( + credentials imageRegistryCredentials, +) (*cabpkv1.File, error) { + return templateCredentialProviderConfig( + credentials, + dynamicCredentialProviderConfigPatch, + kubeletDynamicCredentialProviderConfigOnRemote, + dynamicCredentialProvider, + ) +} + +func templateCredentialProviderConfig( + credentials imageRegistryCredentials, + inputTemplate []byte, + filePath string, + providerFunc func( + hasStaticCredentials bool, + host string, + ) (providerBinary string, providerArgs []string, providerAPIVersion string, err error), +) (*cabpkv1.File, error) { + + mirrorURL, err := url.ParseRequestURI(credentials.URL) + if err != nil { + return nil, fmt.Errorf("failed parsing registry mirror: %w", err) + } + + t := template.New("") + t.Funcs(map[string]any{ + "urlsMatchStr": credentialprovider.URLsMatchStr, + }) + t, err = t.Parse(string(inputTemplate)) + if err != nil { + return nil, fmt.Errorf("failed to parse go template: %w", err) + } + + mirrorHostWithPath := mirrorURL.Host + if mirrorURL.Path != "" { + mirrorHostWithPath = path.Join(mirrorURL.Host, mirrorURL.Path) + } + + providerBinary, providerArgs, providerAPIVersion, err := providerFunc( + !credentials.isCredentialsEmpty(), + mirrorHostWithPath, + ) + if err != nil { + return nil, err + } + if providerBinary == "" { + return nil, nil + } + + templateInput := struct { + MirrorHost string + ProviderBinary string + ProviderArgs []string + ProviderAPIVersion string + }{ + MirrorHost: mirrorHostWithPath, + ProviderBinary: providerBinary, + ProviderArgs: providerArgs, + ProviderAPIVersion: providerAPIVersion, + } + + var b bytes.Buffer + err = t.Execute(&b, templateInput) + if err != nil { + return nil, fmt.Errorf("failed executing template: %w", err) + } + + return &cabpkv1.File{ + Path: filePath, + Content: b.String(), + Permissions: "0600", + }, nil +} + +func kubeletCredentialProvider(hasStaticCredentials bool, host string) ( + providerBinary string, providerArgs []string, providerAPIVersion string, err error, +) { + if needs, err := needCredentialProvider(hasStaticCredentials, host); !needs || err != nil { + return "", nil, "", err + } + return "dynamic-credential-provider", + []string{"get-credentials", "-c", kubeletDynamicCredentialProviderConfigOnRemote}, + credentialproviderv1beta1.SchemeGroupVersion.String(), + nil +} + +func dynamicCredentialProvider(hasStaticCredentials bool, host string) ( + providerBinary string, providerArgs []string, providerAPIVersion string, err error, +) { + if hasStaticCredentials { + return "static-credential-provider", + []string{kubeletStaticCredentialProviderCredentialsOnRemote}, + credentialproviderv1beta1.SchemeGroupVersion.String(), + nil + } + + if matches, err := credentialprovider.URLMatchesECR(host); matches || err != nil { + return "ecr-credential-provider", []string{"get-credentials"}, + credentialproviderv1alpha1.SchemeGroupVersion.String(), err + } + + if matches, err := credentialprovider.URLMatchesGCR(host); matches || err != nil { + return "gcr-credential-provider", []string{"get-credentials"}, + credentialproviderv1alpha1.SchemeGroupVersion.String(), err + } + + if matches, err := credentialprovider.URLMatchesACR(host); matches || err != nil { + return "acr-credential-provider", []string{ + azureCloudConfigFilePath, + }, credentialproviderv1alpha1.SchemeGroupVersion.String(), err + } + + return "", nil, "", nil +} + +func needCredentialProvider(hasStaticCredentials bool, host string) (bool, error) { + if hasStaticCredentials { + return true, nil + } + if matches, err := credentialprovider.URLMatchesECR(host); matches || err != nil { + //nolint:wrapcheck // No need to wrap this error, it has all context needed. + return matches, err + } + if matches, err := credentialprovider.URLMatchesGCR(host); matches || err != nil { + //nolint:wrapcheck // No need to wrap this error, it has all context needed. + return matches, err + } + if matches, err := credentialprovider.URLMatchesACR(host); matches || err != nil { + //nolint:wrapcheck // No need to wrap this error, it has all context needed. + return matches, err + } + + return false, nil +} diff --git a/pkg/handlers/generic/mutation/imageregistrycredentials/credential_provider_config_files_test.go b/pkg/handlers/generic/mutation/imageregistrycredentials/credential_provider_config_files_test.go new file mode 100644 index 000000000..3596dee6c --- /dev/null +++ b/pkg/handlers/generic/mutation/imageregistrycredentials/credential_provider_config_files_test.go @@ -0,0 +1,176 @@ +// Copyright 2023 D2iQ, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package imageregistrycredentials + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + cabpkv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" +) + +func Test_templateKubeletCredentialProviderConfig(t *testing.T) { + tests := []struct { + name string + credentials imageRegistryCredentials + want *cabpkv1.File + wantErr error + }{ + { + name: "ECR image registry", + credentials: imageRegistryCredentials{URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com"}, + want: &cabpkv1.File{ + Path: "/etc/kubernetes/image-credential-provider-config.yaml", + Owner: "", + Permissions: "0600", + Encoding: "", + Append: false, + Content: `apiVersion: kubelet.config.k8s.io/v1beta1 +kind: CredentialProviderConfig +providers: +- name: dynamic-credential-provider + args: + - get-credentials + - -c + - /etc/kubernetes/dynamic-credential-provider-config.yaml + matchImages: + - "123456789.dkr.ecr.us-east-1.amazonaws.com" + - "*" + - "*.*" + - "*.*.*" + - "*.*.*.*" + - "*.*.*.*.*" + - "*.*.*.*.*.*" + defaultCacheDuration: "0s" + apiVersion: credentialprovider.kubelet.k8s.io/v1beta1 +`, + }, + }, + { + name: "image registry with static credentials", + credentials: imageRegistryCredentials{ + URL: "https://myregistry.com", + Username: "myuser", + Password: "mypassword", + }, + want: &cabpkv1.File{ + Path: "/etc/kubernetes/image-credential-provider-config.yaml", + Owner: "", + Permissions: "0600", + Encoding: "", + Append: false, + Content: `apiVersion: kubelet.config.k8s.io/v1beta1 +kind: CredentialProviderConfig +providers: +- name: dynamic-credential-provider + args: + - get-credentials + - -c + - /etc/kubernetes/dynamic-credential-provider-config.yaml + matchImages: + - "myregistry.com" + - "*" + - "*.*" + - "*.*.*" + - "*.*.*.*" + - "*.*.*.*.*" + - "*.*.*.*.*.*" + defaultCacheDuration: "0s" + apiVersion: credentialprovider.kubelet.k8s.io/v1beta1 +`, + }, + }, + } + for idx := range tests { + tt := tests[idx] + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + file, err := templateKubeletCredentialProviderConfig(tt.credentials) + assert.ErrorIs(t, err, tt.wantErr) + assert.Equal(t, tt.want, file) + }) + } +} + +func Test_templateDynamicCredentialProviderConfig(t *testing.T) { + tests := []struct { + name string + credentials imageRegistryCredentials + want *cabpkv1.File + wantErr error + }{ + { + name: "ECR image registry", + credentials: imageRegistryCredentials{URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com"}, + want: &cabpkv1.File{ + Path: "/etc/kubernetes/dynamic-credential-provider-config.yaml", + Owner: "", + Permissions: "0600", + Encoding: "", + Append: false, + Content: `apiVersion: credentialprovider.d2iq.com/v1alpha1 +kind: DynamicCredentialProviderConfig +credentialProviderPluginBinDir: /etc/kubernetes/image-credential-provider/ +mirror: + endpoint: "123456789.dkr.ecr.us-east-1.amazonaws.com" + credentialsStrategy: MirrorCredentialsOnly +credentialProviders: + apiVersion: kubelet.config.k8s.io/v1beta1 + kind: CredentialProviderConfig + providers: + - name: ecr-credential-provider + args: + - get-credentials + matchImages: + - "123456789.dkr.ecr.us-east-1.amazonaws.com" + defaultCacheDuration: "0s" + apiVersion: credentialprovider.kubelet.k8s.io/v1alpha1 +`, + }, + }, + { + name: "image registry with static credentials", + credentials: imageRegistryCredentials{ + URL: "https://myregistry.com", + Username: "myuser", + Password: "mypassword", + }, + want: &cabpkv1.File{ + Path: "/etc/kubernetes/dynamic-credential-provider-config.yaml", + Owner: "", + Permissions: "0600", + Encoding: "", + Append: false, + Content: `apiVersion: credentialprovider.d2iq.com/v1alpha1 +kind: DynamicCredentialProviderConfig +credentialProviderPluginBinDir: /etc/kubernetes/image-credential-provider/ +mirror: + endpoint: "myregistry.com" + credentialsStrategy: MirrorCredentialsOnly +credentialProviders: + apiVersion: kubelet.config.k8s.io/v1beta1 + kind: CredentialProviderConfig + providers: + - name: static-credential-provider + args: + - /etc/kubernetes/static-image-credentials.json + matchImages: + - "myregistry.com" + defaultCacheDuration: "0s" + apiVersion: credentialprovider.kubelet.k8s.io/v1beta1 +`, + }, + }, + } + for idx := range tests { + tt := tests[idx] + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + file, err := templateDynamicCredentialProviderConfig(tt.credentials) + assert.ErrorIs(t, err, tt.wantErr) + assert.Equal(t, tt.want, file) + }) + } +} diff --git a/pkg/handlers/generic/mutation/imageregistrycredentials/credential_provider_insall_files.go b/pkg/handlers/generic/mutation/imageregistrycredentials/credential_provider_insall_files.go new file mode 100644 index 000000000..6692d0f16 --- /dev/null +++ b/pkg/handlers/generic/mutation/imageregistrycredentials/credential_provider_insall_files.go @@ -0,0 +1,63 @@ +// Copyright 2023 D2iQ, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package imageregistrycredentials + +import ( + "bytes" + "fmt" + "text/template" + + cabpkv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" +) + +const ( + //nolint:gosec // Does not contain hard coded credentials. + installKubeletCredentialProvidersScriptOnRemote = "/etc/konvoy/install-kubelet-credential-providers.sh" + + installKubeletCredentialProvidersScriptOnRemoteCommand = "/bin/bash " + installKubeletCredentialProvidersScriptOnRemote +) + +func templateFilesAndCommandsForInstallKubeletCredentialProviders() ([]cabpkv1.File, []string, error) { + var files []cabpkv1.File + var commands []string + + installKubeletCredentialProvidersScriptFile, installKubeletCredentialProvidersScriptCommand, err := + templateInstallKubeletCredentialProviders() + if err != nil { + return nil, nil, err + } + if installKubeletCredentialProvidersScriptFile != nil { + files = append(files, *installKubeletCredentialProvidersScriptFile) + commands = append(commands, installKubeletCredentialProvidersScriptCommand) + } + + return files, commands, nil +} + +func templateInstallKubeletCredentialProviders() (*cabpkv1.File, string, error) { + t, err := template.New("").Parse(string(installKubeletCredentialProvidersScript)) + if err != nil { + return nil, "", fmt.Errorf("failed to parse go template: %w", err) + } + + templateInput := struct { + DynamicCredentialProviderImage string + CredentialProviderTargetDir string + }{ + DynamicCredentialProviderImage: dynamicCredentialProviderImage, + CredentialProviderTargetDir: credentialProviderTargetDir, + } + + var b bytes.Buffer + err = t.Execute(&b, templateInput) + if err != nil { + return nil, "", fmt.Errorf("failed executing template: %w", err) + } + + return &cabpkv1.File{ + Path: installKubeletCredentialProvidersScriptOnRemote, + Content: b.String(), + Permissions: "0700", + }, installKubeletCredentialProvidersScriptOnRemoteCommand, nil +} diff --git a/pkg/handlers/generic/mutation/imageregistrycredentials/credentialprovider/doc.go b/pkg/handlers/generic/mutation/imageregistrycredentials/credentialprovider/doc.go new file mode 100644 index 000000000..dff284ce1 --- /dev/null +++ b/pkg/handlers/generic/mutation/imageregistrycredentials/credentialprovider/doc.go @@ -0,0 +1,7 @@ +// Copyright 2023 D2iQ, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Package credentialprovider includes Functions copied from +// https://github.com/kubernetes/kubernetes/blob/v1.26.1/pkg/credentialprovider/keyring.go#L160-L233. +// These functions have been copied over so as not to add k8s.io/kubernetes as a module dependency. +package credentialprovider diff --git a/pkg/handlers/generic/mutation/imageregistrycredentials/credentialprovider/matcher.go b/pkg/handlers/generic/mutation/imageregistrycredentials/credentialprovider/matcher.go new file mode 100644 index 000000000..667382dc3 --- /dev/null +++ b/pkg/handlers/generic/mutation/imageregistrycredentials/credentialprovider/matcher.go @@ -0,0 +1,71 @@ +// Copyright 2023 D2iQ, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package credentialprovider + +import ( + "regexp" +) + +//nolint:gochecknoglobals // Used here to more easily test URL globs. +var ( + ecrURLGlobs = []string{ + "*.dkr.ecr.*.amazonaws.com", + "*.dkr.ecr.*.amazonaws.cn", + "*.dkr.ecr-fips.*.amazonaws.com", + "*.dkr.ecr.us-iso-east-1.c2s.ic.gov", + "*.dkr.ecr.us-isob-east-1.sc2s.sgov.gov", + } + + gcrURLGlobs = []string{ + "container.cloud.google.com", + "gcr.io", + "*.gcr.io", + "*.pkg.dev", + } + + acrURLGlobs = []string{ + "*.azurecr.io", + "*.azurecr.cn", + "*.azurecr.de", + "*.azurecr.us", + "*.azurecr.*", + } +) + +func URLMatchesSupportedProvider(target string) (bool, error) { + urlGlobs := make([]string, 0, len(ecrURLGlobs)+len(gcrURLGlobs)+len(acrURLGlobs)) + urlGlobs = append(urlGlobs, ecrURLGlobs...) + urlGlobs = append(urlGlobs, gcrURLGlobs...) + urlGlobs = append(urlGlobs, acrURLGlobs...) + return URLMatchesOneOfGlobs(urlGlobs, target) +} + +func URLMatchesECR(target string) (bool, error) { + return URLMatchesOneOfGlobs(ecrURLGlobs, target) +} + +func URLMatchesGCR(target string) (bool, error) { + return URLMatchesOneOfGlobs(gcrURLGlobs, target) +} + +func URLMatchesACR(target string) (bool, error) { + return URLMatchesOneOfGlobs(acrURLGlobs, target) +} + +func URLMatchesOneOfGlobs(globs []string, target string) (bool, error) { + // Strip scheme from target if present. + target = regexp.MustCompile("^https?://").ReplaceAllString(target, "") + + for _, g := range globs { + matches, err := URLsMatchStr(g, target) + if err != nil { + return false, err + } + if matches { + return matches, nil + } + } + + return false, nil +} diff --git a/pkg/handlers/generic/mutation/imageregistrycredentials/credentialprovider/urls.go b/pkg/handlers/generic/mutation/imageregistrycredentials/credentialprovider/urls.go new file mode 100644 index 000000000..4b15f6c30 --- /dev/null +++ b/pkg/handlers/generic/mutation/imageregistrycredentials/credentialprovider/urls.go @@ -0,0 +1,87 @@ +// Copyright 2023 D2iQ, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package credentialprovider + +import ( + "fmt" + "net" + "net/url" + "path/filepath" + "strings" +) + +// ParseSchemelessURL parses a schemeless url and returns a url.URL +// url.Parse require a scheme, but ours don't have schemes. Adding a +// scheme to make url.Parse happy, then clear out the resulting scheme. +func ParseSchemelessURL(schemelessURL string) (*url.URL, error) { + parsed, err := url.Parse("https://" + schemelessURL) + if err != nil { + return nil, fmt.Errorf("invalid URL: %w", err) + } + // clear out the resulting scheme + parsed.Scheme = "" + return parsed, nil +} + +// SplitURL splits the host name into parts, as well as the port. +func SplitURL(u *url.URL) (parts []string, port string) { + host, port, err := net.SplitHostPort(u.Host) + if err != nil { + // could not parse port + host, port = u.Host, "" + } + return strings.Split(host, "."), port +} + +// URLsMatchStr is wrapper for URLsMatch, operating on strings instead of URLs. +func URLsMatchStr(glob, target string) (bool, error) { + globURL, err := ParseSchemelessURL(glob) + if err != nil { + return false, err + } + targetURL, err := ParseSchemelessURL(target) + if err != nil { + return false, err + } + return URLsMatch(globURL, targetURL) +} + +// URLsMatch checks whether the given target url matches the glob url, which may have +// glob wild cards in the host name. +// +// Examples: +// +// globURL=*.docker.io, targetURL=blah.docker.io => match +// globURL=*.docker.io, targetURL=not.right.io => no match +// +// Note that we don't support wildcards in ports and paths yet. +func URLsMatch(globURL, targetURL *url.URL) (bool, error) { + globURLParts, globPort := SplitURL(globURL) + targetURLParts, targetPort := SplitURL(targetURL) + if globPort != targetPort { + // port doesn't match + return false, nil + } + if len(globURLParts) != len(targetURLParts) { + // host name does not have the same number of parts + return false, nil + } + if !strings.HasPrefix(targetURL.Path, globURL.Path) { + // the path of the credential must be a prefix + return false, nil + } + for k, globURLPart := range globURLParts { + targetURLPart := targetURLParts[k] + matched, err := filepath.Match(globURLPart, targetURLPart) + if err != nil { + return false, fmt.Errorf("bad glob pattern: %w", err) + } + if !matched { + // glob mismatch for some part + return false, nil + } + } + // everything matches + return true, nil +} diff --git a/pkg/handlers/generic/mutation/imageregistrycredentials/credentialprovider/urls_test.go b/pkg/handlers/generic/mutation/imageregistrycredentials/credentialprovider/urls_test.go new file mode 100644 index 000000000..35e2f93f5 --- /dev/null +++ b/pkg/handlers/generic/mutation/imageregistrycredentials/credentialprovider/urls_test.go @@ -0,0 +1,107 @@ +// Copyright 2023 D2iQ, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package credentialprovider_test + +import ( + "testing" + + "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/imageregistrycredentials/credentialprovider" +) + +func TestURLsMatch(t *testing.T) { + tests := []struct { + globURL string + targetURL string + matchExpected bool + }{ + // match when there is no path component + { + globURL: "*.kubernetes.io", + targetURL: "prefix.kubernetes.io", + matchExpected: true, + }, + { + globURL: "prefix.*.io", + targetURL: "prefix.kubernetes.io", + matchExpected: true, + }, + { + globURL: "prefix.kubernetes.*", + targetURL: "prefix.kubernetes.io", + matchExpected: true, + }, + { + globURL: "*-good.kubernetes.io", + targetURL: "prefix-good.kubernetes.io", + matchExpected: true, + }, + // match with path components + { + globURL: "*.kubernetes.io/blah", + targetURL: "prefix.kubernetes.io/blah", + matchExpected: true, + }, + { + globURL: "prefix.*.io/foo", + targetURL: "prefix.kubernetes.io/foo/bar", + matchExpected: true, + }, + // match with path components and ports + { + globURL: "*.kubernetes.io:1111/blah", + targetURL: "prefix.kubernetes.io:1111/blah", + matchExpected: true, + }, + { + globURL: "prefix.*.io:1111/foo", + targetURL: "prefix.kubernetes.io:1111/foo/bar", + matchExpected: true, + }, + // no match when number of parts mismatch + { + globURL: "*.kubernetes.io", + targetURL: "kubernetes.io", + matchExpected: false, + }, + { + globURL: "*.*.kubernetes.io", + targetURL: "prefix.kubernetes.io", + matchExpected: false, + }, + { + globURL: "*.*.kubernetes.io", + targetURL: "kubernetes.io", + matchExpected: false, + }, + // no match when some parts mismatch + { + globURL: "kubernetes.io", + targetURL: "kubernetes.com", + matchExpected: false, + }, + { + globURL: "k*.io", + targetURL: "quay.io", + matchExpected: false, + }, + // no match when ports mismatch + { + globURL: "*.kubernetes.io:1234/blah", + targetURL: "prefix.kubernetes.io:1111/blah", + matchExpected: false, + }, + { + globURL: "prefix.*.io/foo", + targetURL: "prefix.kubernetes.io:1111/foo/bar", + matchExpected: false, + }, + } + for _, test := range tests { + matched, _ := credentialprovider.URLsMatchStr(test.globURL, test.targetURL) + if matched != test.matchExpected { + t.Errorf("Expected match result of %s and %s to be %t, but was %t", + test.globURL, test.targetURL, test.matchExpected, matched) + } + } +} diff --git a/pkg/handlers/generic/mutation/imageregistrycredentials/credentials_secret.go b/pkg/handlers/generic/mutation/imageregistrycredentials/credentials_secret.go new file mode 100644 index 000000000..83497c56c --- /dev/null +++ b/pkg/handlers/generic/mutation/imageregistrycredentials/credentials_secret.go @@ -0,0 +1,128 @@ +// Copyright 2023 D2iQ, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package imageregistrycredentials + +import ( + "bytes" + "fmt" + "net/url" + "strings" + "text/template" + + cabpkv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + clusterctlv1 "sigs.k8s.io/cluster-api/cmd/clusterctl/api/v1alpha3" +) + +const ( + secretKeyForStaticCredentialProviderConfig = "static-credential-provider" +) + +func generateCredentialsSecretFile(credentials imageRegistryCredentials, ownerName string) []cabpkv1.File { + if credentials.isCredentialsEmpty() { + return nil + } + return []cabpkv1.File{ + { + Path: kubeletStaticCredentialProviderCredentialsOnRemote, + ContentFrom: &cabpkv1.FileSource{ + Secret: cabpkv1.SecretFileSource{ + Name: credentialSecretName(ownerName), + Key: secretKeyForStaticCredentialProviderConfig, + }, + }, + Permissions: "0600", + }, + } +} + +// generateCredentialsSecret generates a Secret containing the credentials for the image registry mirror. +// The function needs the cluster name to add the required move and cluster name labels. +func generateCredentialsSecret( + credentials imageRegistryCredentials, clusterName, ownerName, namespace string, +) (*corev1.Secret, error) { + if credentials.isCredentialsEmpty() { + return nil, nil + } + + staticCredentialProviderSecretContents, err := kubeletStaticCredentialProviderSecretContents(credentials) + if err != nil { + return nil, err + } + secretData := map[string]string{ + secretKeyForStaticCredentialProviderConfig: staticCredentialProviderSecretContents, + } + + return &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Secret", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: credentialSecretName(ownerName), + Namespace: namespace, + Labels: newLabels(withMove(), withClusterName(clusterName)), + }, + StringData: secretData, + Type: corev1.SecretTypeOpaque, + }, nil +} + +func kubeletStaticCredentialProviderSecretContents(credentials imageRegistryCredentials) (string, error) { + mirrorURL, err := url.ParseRequestURI(credentials.URL) + if err != nil { + return "", fmt.Errorf("failed parsing registry mirror: %w", err) + } + + templateInput := struct { + MirrorHost string + Username string + Password string + }{ + MirrorHost: mirrorURL.Host, + Username: credentials.Username, + Password: credentials.Password, + } + t, err := template.New("").Parse(string(staticCredentialProviderConfigPatch)) + if err != nil { + return "", fmt.Errorf("failed to parse go template: %w", err) + } + + var b bytes.Buffer + err = t.Execute(&b, templateInput) + + if err != nil { + return "", fmt.Errorf("failed executing template: %w", err) + } + return strings.TrimSpace(b.String()), nil +} + +func credentialSecretName(ownerName string) string { + return fmt.Sprintf("%s-registry-credentials", ownerName) +} + +type labelFn func(labels map[string]string) + +func newLabels(fs ...labelFn) map[string]string { + labels := map[string]string{} + for _, f := range fs { + f(labels) + } + return labels +} + +func withClusterName(clusterName string) labelFn { + return func(labels map[string]string) { + labels[clusterv1.ClusterNameLabel] = clusterName + } +} + +func withMove() labelFn { + return func(labels map[string]string) { + labels[clusterctlv1.ClusterctlMoveLabel] = "" + } +} diff --git a/pkg/handlers/generic/mutation/imageregistrycredentials/doc.go b/pkg/handlers/generic/mutation/imageregistrycredentials/doc.go new file mode 100644 index 000000000..36d40c2ab --- /dev/null +++ b/pkg/handlers/generic/mutation/imageregistrycredentials/doc.go @@ -0,0 +1,5 @@ +// Copyright 2023 D2iQ, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// +kubebuilder:rbac:groups="",resources=secrets,verbs=watch;list;get;patch +package imageregistrycredentials diff --git a/pkg/handlers/generic/mutation/imageregistrycredentials/inject.go b/pkg/handlers/generic/mutation/imageregistrycredentials/inject.go new file mode 100644 index 000000000..b8baab1a5 --- /dev/null +++ b/pkg/handlers/generic/mutation/imageregistrycredentials/inject.go @@ -0,0 +1,312 @@ +// Copyright 2023 D2iQ, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package imageregistrycredentials + +import ( + "context" + _ "embed" + "fmt" + + corev1 "k8s.io/api/core/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer" + bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" + cabpkv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" + controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1" + runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1" + "sigs.k8s.io/cluster-api/exp/runtime/topologymutation" + ctrl "sigs.k8s.io/controller-runtime" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/d2iq-labs/capi-runtime-extensions/api/v1alpha1" + commonhandlers "github.com/d2iq-labs/capi-runtime-extensions/common/pkg/capi/clustertopology/handlers" + "github.com/d2iq-labs/capi-runtime-extensions/common/pkg/capi/clustertopology/handlers/mutation" + "github.com/d2iq-labs/capi-runtime-extensions/common/pkg/capi/clustertopology/patches" + "github.com/d2iq-labs/capi-runtime-extensions/common/pkg/capi/clustertopology/patches/selectors" + "github.com/d2iq-labs/capi-runtime-extensions/common/pkg/capi/clustertopology/variables" + "github.com/d2iq-labs/capi-runtime-extensions/common/pkg/k8s/client" + "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/clusterconfig" +) + +const ( + // HandlerNamePatch is the name of the inject handler. + HandlerNamePatch = "ImageRegistryCredentialsPatch" +) + +type imageRegistryCredentialsPatchHandler struct { + decoder runtime.Decoder + client ctrlclient.Client + + variableName string + variableFieldPath []string +} + +var ( + _ commonhandlers.Named = &imageRegistryCredentialsPatchHandler{} + _ mutation.GeneratePatches = &imageRegistryCredentialsPatchHandler{} + _ mutation.MetaMutater = &imageRegistryCredentialsPatchHandler{} +) + +func NewPatch( + cl ctrlclient.Client, +) *imageRegistryCredentialsPatchHandler { + return newImageRegistryCredentialsPatchHandler(cl, variableName) +} + +func NewMetaPatch( + cl ctrlclient.Client, +) *imageRegistryCredentialsPatchHandler { + return newImageRegistryCredentialsPatchHandler(cl, clusterconfig.MetaVariableName, variableName) +} + +func newImageRegistryCredentialsPatchHandler( + cl ctrlclient.Client, + variableName string, + variableFieldPath ...string, +) *imageRegistryCredentialsPatchHandler { + scheme := runtime.NewScheme() + _ = bootstrapv1.AddToScheme(scheme) + _ = controlplanev1.AddToScheme(scheme) + return &imageRegistryCredentialsPatchHandler{ + decoder: serializer.NewCodecFactory(scheme).UniversalDecoder( + controlplanev1.GroupVersion, + bootstrapv1.GroupVersion, + ), + client: cl, + variableName: variableName, + variableFieldPath: variableFieldPath, + } +} + +func (h *imageRegistryCredentialsPatchHandler) Name() string { + return HandlerNamePatch +} + +func (h *imageRegistryCredentialsPatchHandler) Mutate( + ctx context.Context, + obj runtime.Object, + vars map[string]apiextensionsv1.JSON, + holderRef runtimehooksv1.HolderReference, + clusterKey ctrlclient.ObjectKey, +) error { + log := ctrl.LoggerFrom(ctx).WithValues( + "holderRef", holderRef, + ) + + credentials, found, err := variables.Get[v1alpha1.ImageRegistryCredentials]( + vars, + h.variableName, + h.variableFieldPath..., + ) + if err != nil { + return err + } + if !found { + log.V(5).Info("Image Registry Credentials variable not defined") + return nil + } + + log = log.WithValues( + "variableName", + h.variableName, + "variableFieldPath", + h.variableFieldPath, + "variableValue", + credentials, + ) + + if err = patches.Generate( + obj, vars, &holderRef, selectors.ControlPlane(), log, + func(obj *controlplanev1.KubeadmControlPlaneTemplate) error { + registryWithOptionalCredentials, generateErr := + registryWithOptionalCredentialsFromImageRegistryCredentials(ctx, h.client, credentials, obj) + if generateErr != nil { + return generateErr + } + files, commands, generateErr := generateFilesAndCommands(registryWithOptionalCredentials, obj.GetName()) + if generateErr != nil { + return generateErr + } + + log.WithValues( + "patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(), + "patchedObjectName", ctrlclient.ObjectKeyFromObject(obj), + ).Info("adding files to control plane kubeadm config spec") + obj.Spec.Template.Spec.KubeadmConfigSpec.Files = append( + obj.Spec.Template.Spec.KubeadmConfigSpec.Files, + files..., + ) + + log.WithValues( + "patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(), + "patchedObjectName", ctrlclient.ObjectKeyFromObject(obj), + ).Info("adding PreKubeadmCommands to control plane kubeadm config spec") + obj.Spec.Template.Spec.KubeadmConfigSpec.PreKubeadmCommands = append( + obj.Spec.Template.Spec.KubeadmConfigSpec.PreKubeadmCommands, + commands..., + ) + + generateErr = createSecretIfNeeded(ctx, h.client, registryWithOptionalCredentials, obj, clusterKey) + if generateErr != nil { + return generateErr + } + + return nil + }); err != nil { + return err + } + + if err = patches.Generate( + obj, vars, &holderRef, selectors.AllWorkersSelector(), log, + func(obj *bootstrapv1.KubeadmConfigTemplate) error { + registryWithOptionalCredentials, generateErr := + registryWithOptionalCredentialsFromImageRegistryCredentials(ctx, h.client, credentials, obj) + if generateErr != nil { + return generateErr + } + files, commands, generateErr := generateFilesAndCommands(registryWithOptionalCredentials, obj.GetName()) + if generateErr != nil { + return generateErr + } + + log.WithValues( + "patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(), + "patchedObjectName", ctrlclient.ObjectKeyFromObject(obj), + ).Info("adding files to worker node kubeadm config template") + obj.Spec.Template.Spec.Files = append(obj.Spec.Template.Spec.Files, files...) + + log.WithValues( + "patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(), + "patchedObjectName", ctrlclient.ObjectKeyFromObject(obj), + ).Info("adding PreKubeadmCommands to worker node kubeadm config template") + obj.Spec.Template.Spec.PreKubeadmCommands = append(obj.Spec.Template.Spec.PreKubeadmCommands, commands...) + + generateErr = createSecretIfNeeded(ctx, h.client, registryWithOptionalCredentials, obj, clusterKey) + if generateErr != nil { + return generateErr + } + + return nil + }); err != nil { + return err + } + + return nil +} + +func (h *imageRegistryCredentialsPatchHandler) GeneratePatches( + ctx context.Context, + req *runtimehooksv1.GeneratePatchesRequest, + resp *runtimehooksv1.GeneratePatchesResponse, +) { + clusterKey := commonhandlers.ClusterKeyFromReq(req) + + topologymutation.WalkTemplates( + ctx, + h.decoder, + req, + resp, + func( + ctx context.Context, + obj runtime.Object, + vars map[string]apiextensionsv1.JSON, + holderRef runtimehooksv1.HolderReference, + ) error { + return h.Mutate(ctx, obj, vars, holderRef, clusterKey) + }, + ) +} + +func registryWithOptionalCredentialsFromImageRegistryCredentials( + ctx context.Context, + c ctrlclient.Client, + credentials v1alpha1.ImageRegistryCredentials, + obj ctrlclient.Object, +) (imageRegistryCredentials, error) { + registryWithOptionalCredentials := imageRegistryCredentials{ + URL: credentials.URL, + } + secret, err := secretForImageRegistryCredentials( + ctx, + c, + credentials, + obj.GetNamespace(), + ) + if err != nil { + return imageRegistryCredentials{}, fmt.Errorf( + "error getting secret %s/%s from Image Registry Credentials variable: %w", + obj.GetNamespace(), + credentials.Secret, + err, + ) + } + + if secret != nil { + registryWithOptionalCredentials.Username = string(secret.Data["username"]) + registryWithOptionalCredentials.Password = string(secret.Data["password"]) + } + + return registryWithOptionalCredentials, nil +} + +func generateFilesAndCommands( + registryWithOptionalCredentials imageRegistryCredentials, + objName string, +) ([]cabpkv1.File, []string, error) { + + files, commands, err := templateFilesAndCommandsForInstallKubeletCredentialProviders() + if err != nil { + return nil, nil, fmt.Errorf("error generating insall files and commands for Image Registry Credentials variable: %w", err) + } + imageCredentialProviderConfigFiles, err := templateFilesForImageCredentialProviderConfigs(registryWithOptionalCredentials) + if err != nil { + return nil, nil, fmt.Errorf("error generating files for Image Registry Credentials variable: %w", err) + } + files = append(files, imageCredentialProviderConfigFiles...) + files = append(files, generateCredentialsSecretFile(registryWithOptionalCredentials, objName)...) + + return files, commands, err +} + +func createSecretIfNeeded( + ctx context.Context, + c ctrlclient.Client, + registryWithOptionalCredentials imageRegistryCredentials, + obj ctrlclient.Object, + clusterKey ctrlclient.ObjectKey, +) error { + credentialsSecret, err := generateCredentialsSecret(registryWithOptionalCredentials, clusterKey.Name, obj.GetName(), obj.GetNamespace()) + if err != nil { + return fmt.Errorf("error generating credentials Secret for Image Registry Credentials variable: %w", err) + } + if credentialsSecret != nil { + if err := client.ServerSideApply(ctx, c, credentialsSecret); err != nil { + return fmt.Errorf("failed to apply Image Registry Credentials Secret: %w", err) + } + } + + return nil +} + +// secretForImageRegistryCredentials returns the Secret for the given ImageRegistryCredentials. +// Returns nil if the secret field is empty. +func secretForImageRegistryCredentials( + ctx context.Context, + c ctrlclient.Reader, + credentials v1alpha1.ImageRegistryCredentials, + namespace string, +) (*corev1.Secret, error) { + if credentials.Secret == "" { + return nil, nil + } + + key := ctrlclient.ObjectKey{ + Name: credentials.Secret, + Namespace: namespace, + } + secret := &corev1.Secret{} + err := c.Get(ctx, key, secret) + return secret, err +} diff --git a/pkg/handlers/generic/mutation/imageregistrycredentials/inject_test.go b/pkg/handlers/generic/mutation/imageregistrycredentials/inject_test.go new file mode 100644 index 000000000..5700849ba --- /dev/null +++ b/pkg/handlers/generic/mutation/imageregistrycredentials/inject_test.go @@ -0,0 +1,189 @@ +// Copyright 2023 D2iQ, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package imageregistrycredentials + +import ( + "testing" + + "k8s.io/apiserver/pkg/storage/names" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/onsi/gomega" + runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "github.com/d2iq-labs/capi-runtime-extensions/api/v1alpha1" + "github.com/d2iq-labs/capi-runtime-extensions/common/pkg/testutils/capitest" + "github.com/d2iq-labs/capi-runtime-extensions/common/pkg/testutils/capitest/request" +) + +const ( + validSecretName = "myregistry-credentials" +) + +func TestGeneratePatches(t *testing.T) { + capitest.ValidateGeneratePatches( + t, + func() *imageRegistryCredentialsPatchHandler { + // Server side apply does not work with the fake client, hack around it by pre-creating empty Secrets + // https://github.com/kubernetes-sigs/controller-runtime/issues/2341 + fakeClient := fake.NewClientBuilder().WithObjects( + newTestSecret(validSecretName, request.Namespace), + newEmptySecret( + credentialSecretName(request.KubeadmControlPlaneTemplateRequestObjectName), request.Namespace), + newEmptySecret( + credentialSecretName(request.KubeadmConfigTemplateRequestObjectName), request.Namespace), + ).Build() + return NewPatch(fakeClient) + }, + capitest.PatchTestDef{ + Name: "unset variable", + }, + capitest.PatchTestDef{ + Name: "files added in KubeadmControlPlaneTemplate for ECR without a Secret", + Vars: []runtimehooksv1.Variable{ + capitest.VariableWithValue( + variableName, + v1alpha1.ImageRegistryCredentials{ + URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com", + }, + ), + }, + RequestItem: request.NewKubeadmControlPlaneTemplateRequestItem(""), + ExpectedPatchMatchers: []capitest.JSONPatchMatcher{ + { + Operation: "add", + Path: "/spec/template/spec/kubeadmConfigSpec/files", + ValueMatcher: gomega.HaveLen(3), + }, + { + Operation: "add", + Path: "/spec/template/spec/kubeadmConfigSpec/preKubeadmCommands", + ValueMatcher: gomega.HaveLen(1), + }, + }, + }, + capitest.PatchTestDef{ + Name: "files added in KubeadmControlPlaneTemplate for registry with a Secret", + Vars: []runtimehooksv1.Variable{ + capitest.VariableWithValue( + variableName, + v1alpha1.ImageRegistryCredentials{ + URL: "https://my-registry.io", + Secret: validSecretName, + }, + ), + }, + RequestItem: request.NewKubeadmControlPlaneTemplateRequestItem(""), + ExpectedPatchMatchers: []capitest.JSONPatchMatcher{ + { + Operation: "add", + Path: "/spec/template/spec/kubeadmConfigSpec/files", + ValueMatcher: gomega.HaveLen(4), + }, + { + Operation: "add", + Path: "/spec/template/spec/kubeadmConfigSpec/preKubeadmCommands", + ValueMatcher: gomega.HaveLen(1), + }, + }, + }, + capitest.PatchTestDef{ + Name: "files added in KubeadmConfigTemplate for ECR without a Secret", + Vars: []runtimehooksv1.Variable{ + capitest.VariableWithValue( + variableName, + v1alpha1.ImageRegistryCredentials{ + URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com", + }, + ), + capitest.VariableWithValue( + "builtin", + map[string]any{ + "machineDeployment": map[string]any{ + "class": names.SimpleNameGenerator.GenerateName("worker-"), + }, + }, + ), + }, + RequestItem: request.NewKubeadmConfigTemplateRequestItem(""), + ExpectedPatchMatchers: []capitest.JSONPatchMatcher{ + { + Operation: "add", + Path: "/spec/template/spec/files", + ValueMatcher: gomega.HaveLen(3), + }, + { + Operation: "add", + Path: "/spec/template/spec/preKubeadmCommands", + ValueMatcher: gomega.HaveLen(1), + }, + }, + }, + capitest.PatchTestDef{ + Name: "files added in KubeadmConfigTemplate for registry with a Secret", + Vars: []runtimehooksv1.Variable{ + capitest.VariableWithValue( + variableName, + v1alpha1.ImageRegistryCredentials{ + URL: "https://my-registry.io", + Secret: validSecretName, + }, + ), + capitest.VariableWithValue( + "builtin", + map[string]any{ + "machineDeployment": map[string]any{ + "class": names.SimpleNameGenerator.GenerateName("worker-"), + }, + }, + ), + }, + RequestItem: request.NewKubeadmConfigTemplateRequestItem(""), + ExpectedPatchMatchers: []capitest.JSONPatchMatcher{ + { + Operation: "add", + Path: "/spec/template/spec/files", + ValueMatcher: gomega.HaveLen(4), + }, + { + Operation: "add", + Path: "/spec/template/spec/preKubeadmCommands", + ValueMatcher: gomega.HaveLen(1), + }, + }, + }, + ) +} + +func newTestSecret(name, namespace string) *corev1.Secret { + secretData := map[string][]byte{ + "username": []byte("myuser"), + "password": []byte("mypassword"), + } + return &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Secret", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Data: secretData, + Type: corev1.SecretTypeOpaque, + } +} + +func newEmptySecret(name, namespace string) *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Type: corev1.SecretTypeOpaque, + } +} diff --git a/pkg/handlers/generic/mutation/imageregistrycredentials/templates/dynamic-credential-provider-config.yaml.gotmpl b/pkg/handlers/generic/mutation/imageregistrycredentials/templates/dynamic-credential-provider-config.yaml.gotmpl new file mode 100644 index 000000000..8b30845f1 --- /dev/null +++ b/pkg/handlers/generic/mutation/imageregistrycredentials/templates/dynamic-credential-provider-config.yaml.gotmpl @@ -0,0 +1,30 @@ +apiVersion: credentialprovider.d2iq.com/v1alpha1 +kind: DynamicCredentialProviderConfig +credentialProviderPluginBinDir: /etc/kubernetes/image-credential-provider/ +{{- with .MirrorHost }} +{{- if and (ne . "registry-1.docker.io") (ne . "docker.io") }} +mirror: + endpoint: {{ printf "%q" . }} + credentialsStrategy: MirrorCredentialsOnly +{{- end }} +{{- end }} +credentialProviders: + apiVersion: kubelet.config.k8s.io/v1beta1 + kind: CredentialProviderConfig + providers: + - name: {{ .ProviderBinary }} + {{- with .ProviderArgs }} + args: + {{- range . }} + - {{ . }} + {{- end }} + {{- end }} + matchImages: + {{- with .MirrorHost }} + - {{ printf "%q" . }} + {{- if eq . "registry-1.docker.io" }} + - "docker.io" + {{- end }} + {{- end }} + defaultCacheDuration: "0s" + apiVersion: {{ .ProviderAPIVersion }} diff --git a/pkg/handlers/generic/mutation/imageregistrycredentials/templates/image-credential-provider-config.yaml.gotmpl b/pkg/handlers/generic/mutation/imageregistrycredentials/templates/image-credential-provider-config.yaml.gotmpl new file mode 100644 index 000000000..408557a66 --- /dev/null +++ b/pkg/handlers/generic/mutation/imageregistrycredentials/templates/image-credential-provider-config.yaml.gotmpl @@ -0,0 +1,20 @@ +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: CredentialProviderConfig +providers: +- name: {{ .ProviderBinary }} + {{- with .ProviderArgs }} + args: + {{- range . }} + - {{ . }} + {{- end }} + {{- end }} + matchImages: + - {{ printf "%q" .MirrorHost }} + - "*" + - "*.*" + - "*.*.*" + - "*.*.*.*" + - "*.*.*.*.*" + - "*.*.*.*.*.*" + defaultCacheDuration: "0s" + apiVersion: {{ .ProviderAPIVersion }} diff --git a/pkg/handlers/generic/mutation/imageregistrycredentials/templates/install-kubelet-credential-providers.sh.gotmpl b/pkg/handlers/generic/mutation/imageregistrycredentials/templates/install-kubelet-credential-providers.sh.gotmpl new file mode 100644 index 000000000..30eb73517 --- /dev/null +++ b/pkg/handlers/generic/mutation/imageregistrycredentials/templates/install-kubelet-credential-providers.sh.gotmpl @@ -0,0 +1,23 @@ +#!/bin/bash +set -euo pipefail +IFS=$'\n\t' + +declare -r CREDENTIAL_PROVIDER_IMAGE="{{ .DynamicCredentialProviderImage }}" + +if ! ctr --namespace k8s.io images check "name==${CREDENTIAL_PROVIDER_IMAGE}" | grep "${CREDENTIAL_PROVIDER_IMAGE}" >/dev/null; then + ctr --namespace k8s.io images pull "${CREDENTIAL_PROVIDER_IMAGE}" +fi + +cleanup() { + ctr images unmount "${tmp_ctr_mount_dir}" || true +} + +trap 'cleanup' EXIT + +readonly tmp_ctr_mount_dir="$(mktemp -d)" + +export CREDENTIAL_PROVIDER_SOURCE_DIR="${tmp_ctr_mount_dir}/opt/image-credential-provider/bin/" +export CREDENTIAL_PROVIDER_TARGET_DIR="{{ .CredentialProviderTargetDir }}" + +ctr --namespace k8s.io images mount "${CREDENTIAL_PROVIDER_IMAGE}" "${tmp_ctr_mount_dir}" +"${tmp_ctr_mount_dir}/opt/image-credential-provider/bin/dynamic-credential-provider" install diff --git a/pkg/handlers/generic/mutation/imageregistrycredentials/templates/static-credential-provider.json.gotmpl b/pkg/handlers/generic/mutation/imageregistrycredentials/templates/static-credential-provider.json.gotmpl new file mode 100644 index 000000000..8646d5202 --- /dev/null +++ b/pkg/handlers/generic/mutation/imageregistrycredentials/templates/static-credential-provider.json.gotmpl @@ -0,0 +1,13 @@ +{ + "kind":"CredentialProviderResponse", + "apiVersion":"credentialprovider.kubelet.k8s.io/v1beta1", + "cacheKeyType":"Image", + "cacheDuration":"0s", + "auth":{ + {{- if .MirrorHost }} + {{ printf "%q" .MirrorHost }}: {"username": {{ printf "%q" .Username }}, "password": {{ printf "%q" .Password }}}{{ if eq .MirrorHost "registry-1.docker.io" }}, + "docker.io": {"username": {{ printf "%q" .Username }}, "password": {{ printf "%q" .Password }}} + {{- end }} + {{- end }} + } +} diff --git a/pkg/handlers/generic/mutation/imageregistrycredentials/variables.go b/pkg/handlers/generic/mutation/imageregistrycredentials/variables.go new file mode 100644 index 000000000..5c5a3fbae --- /dev/null +++ b/pkg/handlers/generic/mutation/imageregistrycredentials/variables.go @@ -0,0 +1,51 @@ +// Copyright 2023 D2iQ, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package imageregistrycredentials + +import ( + "context" + + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1" + + "github.com/d2iq-labs/capi-runtime-extensions/api/v1alpha1" + "github.com/d2iq-labs/capi-runtime-extensions/common/pkg/capi/clustertopology/handlers" + "github.com/d2iq-labs/capi-runtime-extensions/common/pkg/capi/clustertopology/handlers/mutation" +) + +var ( + _ handlers.Named = &imageRegistryCredentialsVariableHandler{} + _ mutation.DiscoverVariables = &imageRegistryCredentialsVariableHandler{} +) + +const ( + // variableName is the external patch variable name. + variableName = "imageRegistryCredentials" + + // HandlerNameVariable is the name of the variable handler. + HandlerNameVariable = "ImageRegistryCredentialsVars" +) + +func NewVariable() *imageRegistryCredentialsVariableHandler { + return &imageRegistryCredentialsVariableHandler{} +} + +type imageRegistryCredentialsVariableHandler struct{} + +func (h *imageRegistryCredentialsVariableHandler) Name() string { + return HandlerNameVariable +} + +func (h *imageRegistryCredentialsVariableHandler) DiscoverVariables( + ctx context.Context, + _ *runtimehooksv1.DiscoverVariablesRequest, + resp *runtimehooksv1.DiscoverVariablesResponse, +) { + resp.Variables = append(resp.Variables, clusterv1.ClusterClassVariable{ + Name: variableName, + Required: false, + Schema: v1alpha1.ImageRegistryCredentials{}.VariableSchema(), + }) + resp.SetStatus(runtimehooksv1.ResponseStatusSuccess) +} diff --git a/pkg/handlers/generic/mutation/imageregistrycredentials/variables_test.go b/pkg/handlers/generic/mutation/imageregistrycredentials/variables_test.go new file mode 100644 index 000000000..dae6692ae --- /dev/null +++ b/pkg/handlers/generic/mutation/imageregistrycredentials/variables_test.go @@ -0,0 +1,36 @@ +// Copyright 2023 D2iQ, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package imageregistrycredentials + +import ( + "testing" + + "k8s.io/utils/ptr" + + "github.com/d2iq-labs/capi-runtime-extensions/api/v1alpha1" + "github.com/d2iq-labs/capi-runtime-extensions/common/pkg/testutils/capitest" +) + +func TestVariableValidation(t *testing.T) { + capitest.ValidateDiscoverVariables( + t, + variableName, + ptr.To(v1alpha1.ImageRegistryCredentials{}.VariableSchema()), + false, + NewVariable, + capitest.VariableTestDef{ + Name: "without a Secret", + Vals: v1alpha1.ImageRegistryCredentials{ + URL: "http://a.b.c.example.com", + }, + }, + capitest.VariableTestDef{ + Name: "with a Secret", + Vals: v1alpha1.ImageRegistryCredentials{ + URL: "http://a.b.c.example.com", + Secret: "a.b.c.example.com-creds", + }, + }, + ) +} From 70c37e482cefc3e155cfa5b117222e7a988bc611 Mon Sep 17 00:00:00 2001 From: Dimitri Koshkin Date: Mon, 25 Sep 2023 11:07:01 -0700 Subject: [PATCH 02/11] fix: use ObjectReference for secretRef in API --- api/v1alpha1/clusterconfig_types.go | 23 +++++++++++++++---- api/v1alpha1/zz_generated.deepcopy.go | 8 ++++++- docs/content/customization/_index.md | 3 ++- .../generic/image-registry-credentials.md | 3 ++- .../imageregistrycredentials/inject.go | 11 ++++++--- .../imageregistrycredentials/inject_test.go | 12 ++++++---- .../variables_test.go | 7 ++++-- 7 files changed, 50 insertions(+), 17 deletions(-) diff --git a/api/v1alpha1/clusterconfig_types.go b/api/v1alpha1/clusterconfig_types.go index 1b384d16d..1b019a506 100644 --- a/api/v1alpha1/clusterconfig_types.go +++ b/api/v1alpha1/clusterconfig_types.go @@ -4,6 +4,7 @@ package v1alpha1 import ( + corev1 "k8s.io/api/core/v1" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" "github.com/d2iq-labs/capi-runtime-extensions/common/pkg/capi/clustertopology/variables" @@ -185,11 +186,11 @@ type ImageRegistryCredentials struct { // Registry URL. URL string `json:"url"` - // Name of the Secret containing the registry credentials. + // The Secret containing the registry credentials. // The Secret should have keys 'username' and 'password'. // This credentials Secret is not required for some registries, e.g. ECR. // +optional - Secret string `json:"secret,omitempty"` + Secret *corev1.ObjectReference `json:"secretRef,omitempty"` } func (ImageRegistryCredentials) VariableSchema() clusterv1.VariableSchema { @@ -202,11 +203,23 @@ func (ImageRegistryCredentials) VariableSchema() clusterv1.VariableSchema { Description: "Registry URL.", Type: "string", }, - "secret": { - Description: "Name of the Secret containing the registry credentials. " + + "secretRef": { + Description: "The Secret containing the registry credentials. " + "The Secret should have keys 'username' and 'password'. " + "This credentials Secret is not required for some registries, e.g. ECR.", - Type: "string", + Type: "object", + Properties: map[string]clusterv1.JSONSchemaProps{ + "name": { + Description: "The name of the Secret containing the registry credentials.", + Type: "string", + }, + "namespace": { + Description: "The namespace of the Secret containing the registry credentials. " + + "Defaults to the namespace of the KubeadmControlPlaneTemplate and KubeadmConfigTemplate" + + " that reference this variable.", + Type: "string", + }, + }, }, }, Required: []string{"url"}, diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 68e175118..c09de75dd 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -8,6 +8,7 @@ package v1alpha1 import ( + v1 "k8s.io/api/core/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) @@ -207,7 +208,7 @@ func (in *GenericClusterConfig) DeepCopyInto(out *GenericClusterConfig) { *out = make(ExtraAPIServerCertSANs, len(*in)) copy(*out, *in) } - out.ImageRegistryCredentials = in.ImageRegistryCredentials + in.ImageRegistryCredentials.DeepCopyInto(&out.ImageRegistryCredentials) if in.Addons != nil { in, out := &in.Addons, &out.Addons *out = new(Addons) @@ -263,6 +264,11 @@ func (in *Image) DeepCopy() *Image { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ImageRegistryCredentials) DeepCopyInto(out *ImageRegistryCredentials) { *out = *in + if in.Secret != nil { + in, out := &in.Secret, &out.Secret + *out = new(v1.ObjectReference) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ImageRegistryCredentials. diff --git a/docs/content/customization/_index.md b/docs/content/customization/_index.md index 72588a43b..da7b309d6 100644 --- a/docs/content/customization/_index.md +++ b/docs/content/customization/_index.md @@ -41,7 +41,8 @@ spec: - no-proxy-2.example.com imageRegistryCredentials: url: https://my-registry.io - secret: my-registry-credentials + secretRef: + name: my-registry-credentials cni: provider: calico ``` diff --git a/docs/content/customization/generic/image-registry-credentials.md b/docs/content/customization/generic/image-registry-credentials.md index a84f0fd71..19190ba8b 100644 --- a/docs/content/customization/generic/image-registry-credentials.md +++ b/docs/content/customization/generic/image-registry-credentials.md @@ -37,7 +37,8 @@ spec: value: imageRegistryCredentials: url: https://my-registry.io - secret: my-registry-credentials + secretRef: + name: my-registry-credentials ``` Applying this configuration will result in new files and preKubeadmCommands diff --git a/pkg/handlers/generic/mutation/imageregistrycredentials/inject.go b/pkg/handlers/generic/mutation/imageregistrycredentials/inject.go index b8baab1a5..d4e4ee525 100644 --- a/pkg/handlers/generic/mutation/imageregistrycredentials/inject.go +++ b/pkg/handlers/generic/mutation/imageregistrycredentials/inject.go @@ -296,14 +296,19 @@ func secretForImageRegistryCredentials( ctx context.Context, c ctrlclient.Reader, credentials v1alpha1.ImageRegistryCredentials, - namespace string, + objectNamespace string, ) (*corev1.Secret, error) { - if credentials.Secret == "" { + if credentials.Secret == nil { return nil, nil } + namespace := objectNamespace + if credentials.Secret.Namespace != "" { + namespace = credentials.Secret.Namespace + } + key := ctrlclient.ObjectKey{ - Name: credentials.Secret, + Name: credentials.Secret.Name, Namespace: namespace, } secret := &corev1.Secret{} diff --git a/pkg/handlers/generic/mutation/imageregistrycredentials/inject_test.go b/pkg/handlers/generic/mutation/imageregistrycredentials/inject_test.go index 5700849ba..72dd46099 100644 --- a/pkg/handlers/generic/mutation/imageregistrycredentials/inject_test.go +++ b/pkg/handlers/generic/mutation/imageregistrycredentials/inject_test.go @@ -72,8 +72,10 @@ func TestGeneratePatches(t *testing.T) { capitest.VariableWithValue( variableName, v1alpha1.ImageRegistryCredentials{ - URL: "https://my-registry.io", - Secret: validSecretName, + URL: "https://my-registry.io", + Secret: &corev1.ObjectReference{ + Name: validSecretName, + }, }, ), }, @@ -129,8 +131,10 @@ func TestGeneratePatches(t *testing.T) { capitest.VariableWithValue( variableName, v1alpha1.ImageRegistryCredentials{ - URL: "https://my-registry.io", - Secret: validSecretName, + URL: "https://my-registry.io", + Secret: &corev1.ObjectReference{ + Name: validSecretName, + }, }, ), capitest.VariableWithValue( diff --git a/pkg/handlers/generic/mutation/imageregistrycredentials/variables_test.go b/pkg/handlers/generic/mutation/imageregistrycredentials/variables_test.go index dae6692ae..ce7743095 100644 --- a/pkg/handlers/generic/mutation/imageregistrycredentials/variables_test.go +++ b/pkg/handlers/generic/mutation/imageregistrycredentials/variables_test.go @@ -6,6 +6,7 @@ package imageregistrycredentials import ( "testing" + corev1 "k8s.io/api/core/v1" "k8s.io/utils/ptr" "github.com/d2iq-labs/capi-runtime-extensions/api/v1alpha1" @@ -28,8 +29,10 @@ func TestVariableValidation(t *testing.T) { capitest.VariableTestDef{ Name: "with a Secret", Vals: v1alpha1.ImageRegistryCredentials{ - URL: "http://a.b.c.example.com", - Secret: "a.b.c.example.com-creds", + URL: "http://a.b.c.example.com", + Secret: &corev1.ObjectReference{ + Name: "a.b.c.example.com-creds", + }, }, }, ) From 85a07d96aa380991a5f5df32480d7f7bb0658537 Mon Sep 17 00:00:00 2001 From: Dimitri Koshkin Date: Mon, 25 Sep 2023 13:19:26 -0700 Subject: [PATCH 03/11] fix: nest object under imageRegistries.credentials --- api/v1alpha1/clusterconfig_types.go | 105 ++++++++++++------ api/v1alpha1/zz_generated.deepcopy.go | 53 ++++++++- cmd/main.go | 2 +- docs/content/customization/_index.md | 9 +- ...try-credentials.md => image-registries.md} | 24 ++-- .../mutation/imageregistries/constants.go | 8 ++ .../credential_provider_config_files.go | 17 ++- .../credential_provider_config_files_test.go | 14 +-- .../credential_provider_insall_files.go | 2 +- .../credentials}/credentialprovider/doc.go | 0 .../credentialprovider/matcher.go | 0 .../credentials}/credentialprovider/urls.go | 0 .../credentialprovider/urls_test.go | 2 +- .../credentials}/credentials_secret.go | 8 +- .../credentials}/doc.go | 2 +- .../credentials}/inject.go | 54 +++++---- .../credentials}/inject_test.go | 28 +++-- ...mic-credential-provider-config.yaml.gotmpl | 0 ...age-credential-provider-config.yaml.gotmpl | 0 ...all-kubelet-credential-providers.sh.gotmpl | 0 .../static-credential-provider.json.gotmpl | 0 .../credentials}/variables.go | 4 +- .../credentials}/variables_test.go | 14 ++- 23 files changed, 225 insertions(+), 121 deletions(-) rename docs/content/customization/generic/{image-registry-credentials.md => image-registries.md} (50%) create mode 100644 pkg/handlers/generic/mutation/imageregistries/constants.go rename pkg/handlers/generic/mutation/{imageregistrycredentials => imageregistries/credentials}/credential_provider_config_files.go (93%) rename pkg/handlers/generic/mutation/{imageregistrycredentials => imageregistries/credentials}/credential_provider_config_files_test.go (91%) rename pkg/handlers/generic/mutation/{imageregistrycredentials => imageregistries/credentials}/credential_provider_insall_files.go (98%) rename pkg/handlers/generic/mutation/{imageregistrycredentials => imageregistries/credentials}/credentialprovider/doc.go (100%) rename pkg/handlers/generic/mutation/{imageregistrycredentials => imageregistries/credentials}/credentialprovider/matcher.go (100%) rename pkg/handlers/generic/mutation/{imageregistrycredentials => imageregistries/credentials}/credentialprovider/urls.go (100%) rename pkg/handlers/generic/mutation/{imageregistrycredentials => imageregistries/credentials}/credentialprovider/urls_test.go (97%) rename pkg/handlers/generic/mutation/{imageregistrycredentials => imageregistries/credentials}/credentials_secret.go (90%) rename pkg/handlers/generic/mutation/{imageregistrycredentials => imageregistries/credentials}/doc.go (83%) rename pkg/handlers/generic/mutation/{imageregistrycredentials => imageregistries/credentials}/inject.go (85%) rename pkg/handlers/generic/mutation/{imageregistrycredentials => imageregistries/credentials}/inject_test.go (88%) rename pkg/handlers/generic/mutation/{imageregistrycredentials => imageregistries/credentials}/templates/dynamic-credential-provider-config.yaml.gotmpl (100%) rename pkg/handlers/generic/mutation/{imageregistrycredentials => imageregistries/credentials}/templates/image-credential-provider-config.yaml.gotmpl (100%) rename pkg/handlers/generic/mutation/{imageregistrycredentials => imageregistries/credentials}/templates/install-kubelet-credential-providers.sh.gotmpl (100%) rename pkg/handlers/generic/mutation/{imageregistrycredentials => imageregistries/credentials}/templates/static-credential-provider.json.gotmpl (100%) rename pkg/handlers/generic/mutation/{imageregistrycredentials => imageregistries/credentials}/variables.go (95%) rename pkg/handlers/generic/mutation/{imageregistrycredentials => imageregistries/credentials}/variables_test.go (72%) diff --git a/api/v1alpha1/clusterconfig_types.go b/api/v1alpha1/clusterconfig_types.go index 1b019a506..a21686680 100644 --- a/api/v1alpha1/clusterconfig_types.go +++ b/api/v1alpha1/clusterconfig_types.go @@ -29,9 +29,8 @@ type GenericClusterConfig struct { // +optional ExtraAPIServerCertSANs ExtraAPIServerCertSANs `json:"extraAPIServerCertSANs,omitempty"` - // TODO: Add support for multiple registries. // +optional - ImageRegistryCredentials ImageRegistryCredentials `json:"imageRegistryCredentials,omitempty"` + ImageRegistries ImageRegistries `json:"imageRegistries,omitempty"` // +optional Addons *Addons `json:"addons,omitempty"` @@ -51,7 +50,7 @@ func (GenericClusterConfig) VariableSchema() clusterv1.VariableSchema { "", ).VariableSchema(). OpenAPIV3Schema, - "imageRegistryCredentials": ImageRegistryCredentials{}.VariableSchema().OpenAPIV3Schema, + "imageRegistries": ImageRegistries{}.VariableSchema().OpenAPIV3Schema, }, }, } @@ -181,49 +180,85 @@ func (ExtraAPIServerCertSANs) VariableSchema() clusterv1.VariableSchema { } } -// ImageRegistryCredentials required for providing credentials for an image registry URL. -type ImageRegistryCredentials struct { - // Registry URL. - URL string `json:"url"` - - // The Secret containing the registry credentials. - // The Secret should have keys 'username' and 'password'. - // This credentials Secret is not required for some registries, e.g. ECR. +type ImageRegistries struct { // +optional - Secret *corev1.ObjectReference `json:"secretRef,omitempty"` + ImageRegistryCredentials ImageRegistryCredentials `json:"credentials,omitempty"` } -func (ImageRegistryCredentials) VariableSchema() clusterv1.VariableSchema { +func (ImageRegistries) VariableSchema() clusterv1.VariableSchema { return clusterv1.VariableSchema{ OpenAPIV3Schema: clusterv1.JSONSchemaProps{ - Description: "Extra Subject Alternative Names for the API Server signing cert", + Description: "Configuration for image registries.", Type: "object", Properties: map[string]clusterv1.JSONSchemaProps{ - "url": { - Description: "Registry URL.", - Type: "string", - }, - "secretRef": { - Description: "The Secret containing the registry credentials. " + - "The Secret should have keys 'username' and 'password'. " + - "This credentials Secret is not required for some registries, e.g. ECR.", - Type: "object", - Properties: map[string]clusterv1.JSONSchemaProps{ - "name": { - Description: "The name of the Secret containing the registry credentials.", - Type: "string", - }, - "namespace": { - Description: "The namespace of the Secret containing the registry credentials. " + - "Defaults to the namespace of the KubeadmControlPlaneTemplate and KubeadmConfigTemplate" + - " that reference this variable.", - Type: "string", - }, + "credentials": imageRegistryCredentialsSchema, + }, + }, + } +} + +var ( + imageRegistryCredentialsSchema = clusterv1.JSONSchemaProps{ + Type: "array", + UniqueItems: true, + Items: &imageRegistryCredentialsResourceSchema, + } + + imageRegistryCredentialsResourceSchema = clusterv1.JSONSchemaProps{ + Description: "Image registry credentials to set up on all Nodes in the cluster. " + + "Enabling this will the Kubelets with https://kubernetes.io/docs/tasks/administer-cluster/kubelet-credential-provider/.", + Type: "object", + Properties: map[string]clusterv1.JSONSchemaProps{ + "url": { + Description: "Registry URL.", + Type: "string", + }, + "secretRef": { + Description: "The Secret containing the registry credentials. " + + "The Secret should have keys 'username' and 'password'. " + + "This credentials Secret is not required for some registries, e.g. ECR.", + Type: "object", + Properties: map[string]clusterv1.JSONSchemaProps{ + "name": { + Description: "The name of the Secret containing the registry credentials.", + Type: "string", + }, + "namespace": { + Description: "The namespace of the Secret containing the registry credentials. " + + "Defaults to the namespace of the KubeadmControlPlaneTemplate and KubeadmConfigTemplate" + + " that reference this variable.", + Type: "string", }, }, }, - Required: []string{"url"}, }, + Required: []string{"url"}, + } +) + +type ImageRegistryCredentials []ImageRegistryCredentialsResource + +func (ImageRegistryCredentials) VariableSchema() clusterv1.VariableSchema { + return clusterv1.VariableSchema{ + OpenAPIV3Schema: imageRegistryCredentialsSchema, + } +} + +// ImageRegistryCredentialsResource required for providing credentials for an image registry URL. +type ImageRegistryCredentialsResource struct { + // Registry URL. + URL string `json:"url"` + + // The Secret containing the registry credentials. + // The Secret should have keys 'username' and 'password'. + // This credentials Secret is not required for some registries, e.g. ECR. + // +optional + Secret *corev1.ObjectReference `json:"secretRef,omitempty"` +} + +func (ImageRegistryCredentialsResource) VariableSchema() clusterv1.VariableSchema { + return clusterv1.VariableSchema{ + OpenAPIV3Schema: imageRegistryCredentialsResourceSchema, } } diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index c09de75dd..0aacfa639 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -208,7 +208,7 @@ func (in *GenericClusterConfig) DeepCopyInto(out *GenericClusterConfig) { *out = make(ExtraAPIServerCertSANs, len(*in)) copy(*out, *in) } - in.ImageRegistryCredentials.DeepCopyInto(&out.ImageRegistryCredentials) + in.ImageRegistries.DeepCopyInto(&out.ImageRegistries) if in.Addons != nil { in, out := &in.Addons, &out.Addons *out = new(Addons) @@ -262,7 +262,50 @@ func (in *Image) DeepCopy() *Image { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ImageRegistryCredentials) DeepCopyInto(out *ImageRegistryCredentials) { +func (in *ImageRegistries) DeepCopyInto(out *ImageRegistries) { + *out = *in + if in.ImageRegistryCredentials != nil { + in, out := &in.ImageRegistryCredentials, &out.ImageRegistryCredentials + *out = make(ImageRegistryCredentials, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ImageRegistries. +func (in *ImageRegistries) DeepCopy() *ImageRegistries { + if in == nil { + return nil + } + out := new(ImageRegistries) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in ImageRegistryCredentials) DeepCopyInto(out *ImageRegistryCredentials) { + { + in := &in + *out = make(ImageRegistryCredentials, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ImageRegistryCredentials. +func (in ImageRegistryCredentials) DeepCopy() ImageRegistryCredentials { + if in == nil { + return nil + } + out := new(ImageRegistryCredentials) + in.DeepCopyInto(out) + return *out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ImageRegistryCredentialsResource) DeepCopyInto(out *ImageRegistryCredentialsResource) { *out = *in if in.Secret != nil { in, out := &in.Secret, &out.Secret @@ -271,12 +314,12 @@ func (in *ImageRegistryCredentials) DeepCopyInto(out *ImageRegistryCredentials) } } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ImageRegistryCredentials. -func (in *ImageRegistryCredentials) DeepCopy() *ImageRegistryCredentials { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ImageRegistryCredentialsResource. +func (in *ImageRegistryCredentialsResource) DeepCopy() *ImageRegistryCredentialsResource { if in == nil { return nil } - out := new(ImageRegistryCredentials) + out := new(ImageRegistryCredentialsResource) in.DeepCopyInto(out) return out } diff --git a/cmd/main.go b/cmd/main.go index e02d2808c..c8e7fe065 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -39,7 +39,7 @@ import ( "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/etcd" "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/extraapiservercertsans" "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/httpproxy" - "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/imageregistrycredentials" + imageregistrycredentials "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/imageregistries/credentials" "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/kubernetesimagerepository" ) diff --git a/docs/content/customization/_index.md b/docs/content/customization/_index.md index da7b309d6..cd730b334 100644 --- a/docs/content/customization/_index.md +++ b/docs/content/customization/_index.md @@ -39,10 +39,11 @@ spec: additionalNo: - no-proxy-1.example.com - no-proxy-2.example.com - imageRegistryCredentials: - url: https://my-registry.io - secretRef: - name: my-registry-credentials + imageRegistries: + credentials: + - url: https://my-registry.io + secretRef: + name: my-registry-credentials cni: provider: calico ``` diff --git a/docs/content/customization/generic/image-registry-credentials.md b/docs/content/customization/generic/image-registries.md similarity index 50% rename from docs/content/customization/generic/image-registry-credentials.md rename to docs/content/customization/generic/image-registries.md index 19190ba8b..ed7cecd70 100644 --- a/docs/content/customization/generic/image-registry-credentials.md +++ b/docs/content/customization/generic/image-registries.md @@ -1,15 +1,12 @@ +++ -title = "Image registry credentials" +title = "Image registries" +++ -In some network environments it is necessary to use HTTP proxy to successfuly execute HTTP requests. -To configure Kubernetes components (`containerd`, `kubelet`) to use HTTP proxy use the `httpproxypatch` -external patch that will generate appropriate configuration for control plane and worker nodes. +Add image registry configuration to all Nodes in the cluster. -Add image registry credentials to all Nodes in the cluster. -When this handle is enabled, the handler will add `files` and `preKubeadmnCommands` with configurations for +When the `credentials` variable is set, `files` and `preKubeadmnCommands` with configurations for [Kubelet image credential provider](https://kubernetes.io/docs/tasks/administer-cluster/kubelet-credential-provider/) -and [dynamic credential provider](https://github.com/mesosphere/dynamic-credential-provider). +and [dynamic credential provider](https://github.com/mesosphere/dynamic-credential-provider) will be added. This customization will be available when the [provider-specific cluster configuration patch]({{< ref "..">}}) is included in the `ClusterClass`. @@ -20,10 +17,10 @@ If your registry requires static credentials, create a Kubernetes Secret with ke ```shell kubectl create secret generic my-registry-credentials \ - --from-literal username=${REGISTRY_USERNAME} password=${REGISTRY_PASSWORD} + --from-literal username=${REGISTRY_USERNAME} --from-literal password=${REGISTRY_PASSWORD} ``` -On the cluster resource then specify desired image registry credentials: +To add image registry credentials, specify the following configuration: ```yaml apiVersion: cluster.x-k8s.io/v1beta1 @@ -35,10 +32,11 @@ spec: variables: - name: clusterConfig value: - imageRegistryCredentials: - url: https://my-registry.io - secretRef: - name: my-registry-credentials + imageRegistries: + credentials: + - url: https://my-registry.io + secretRef: + name: my-registry-credentials ``` Applying this configuration will result in new files and preKubeadmCommands diff --git a/pkg/handlers/generic/mutation/imageregistries/constants.go b/pkg/handlers/generic/mutation/imageregistries/constants.go new file mode 100644 index 000000000..f128c3bc6 --- /dev/null +++ b/pkg/handlers/generic/mutation/imageregistries/constants.go @@ -0,0 +1,8 @@ +// Copyright 2023 D2iQ, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package imageregistries + +const ( + VariableName = "imageRegistries" +) diff --git a/pkg/handlers/generic/mutation/imageregistrycredentials/credential_provider_config_files.go b/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files.go similarity index 93% rename from pkg/handlers/generic/mutation/imageregistrycredentials/credential_provider_config_files.go rename to pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files.go index d9dec04de..3603bceb4 100644 --- a/pkg/handlers/generic/mutation/imageregistrycredentials/credential_provider_config_files.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files.go @@ -1,7 +1,7 @@ // Copyright 2023 D2iQ, Inc. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -package imageregistrycredentials +package credentials import ( "bytes" @@ -15,7 +15,7 @@ import ( credentialproviderv1beta1 "k8s.io/kubelet/pkg/apis/credentialprovider/v1beta1" cabpkv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" - "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/imageregistrycredentials/credentialprovider" + "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/imageregistries/credentials/credentialprovider" ) const ( @@ -51,18 +51,18 @@ var ( installKubeletCredentialProvidersScript []byte ) -type imageRegistryCredentials struct { +type providerInput struct { URL string Username string Password string } -func (c imageRegistryCredentials) isCredentialsEmpty() bool { +func (c providerInput) isCredentialsEmpty() bool { return c.Username == "" && c.Password == "" } -func templateFilesForImageCredentialProviderConfigs(credentials imageRegistryCredentials) ([]cabpkv1.File, error) { +func templateFilesForImageCredentialProviderConfigs(credentials providerInput) ([]cabpkv1.File, error) { var files []cabpkv1.File kubeletCredentialProviderConfigFile, err := templateKubeletCredentialProviderConfig(credentials) @@ -84,7 +84,7 @@ func templateFilesForImageCredentialProviderConfigs(credentials imageRegistryCre return files, nil } -func templateKubeletCredentialProviderConfig(credentials imageRegistryCredentials) (*cabpkv1.File, error) { +func templateKubeletCredentialProviderConfig(credentials providerInput) (*cabpkv1.File, error) { return templateCredentialProviderConfig( credentials, imageCredentialProviderConfigPatch, @@ -94,7 +94,7 @@ func templateKubeletCredentialProviderConfig(credentials imageRegistryCredential } func templateDynamicCredentialProviderConfig( - credentials imageRegistryCredentials, + credentials providerInput, ) (*cabpkv1.File, error) { return templateCredentialProviderConfig( credentials, @@ -105,7 +105,7 @@ func templateDynamicCredentialProviderConfig( } func templateCredentialProviderConfig( - credentials imageRegistryCredentials, + credentials providerInput, inputTemplate []byte, filePath string, providerFunc func( @@ -113,7 +113,6 @@ func templateCredentialProviderConfig( host string, ) (providerBinary string, providerArgs []string, providerAPIVersion string, err error), ) (*cabpkv1.File, error) { - mirrorURL, err := url.ParseRequestURI(credentials.URL) if err != nil { return nil, fmt.Errorf("failed parsing registry mirror: %w", err) diff --git a/pkg/handlers/generic/mutation/imageregistrycredentials/credential_provider_config_files_test.go b/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files_test.go similarity index 91% rename from pkg/handlers/generic/mutation/imageregistrycredentials/credential_provider_config_files_test.go rename to pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files_test.go index 3596dee6c..e577f8c1e 100644 --- a/pkg/handlers/generic/mutation/imageregistrycredentials/credential_provider_config_files_test.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files_test.go @@ -1,7 +1,7 @@ // Copyright 2023 D2iQ, Inc. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -package imageregistrycredentials +package credentials import ( "testing" @@ -14,13 +14,13 @@ import ( func Test_templateKubeletCredentialProviderConfig(t *testing.T) { tests := []struct { name string - credentials imageRegistryCredentials + credentials providerInput want *cabpkv1.File wantErr error }{ { name: "ECR image registry", - credentials: imageRegistryCredentials{URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com"}, + credentials: providerInput{URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com"}, want: &cabpkv1.File{ Path: "/etc/kubernetes/image-credential-provider-config.yaml", Owner: "", @@ -50,7 +50,7 @@ providers: }, { name: "image registry with static credentials", - credentials: imageRegistryCredentials{ + credentials: providerInput{ URL: "https://myregistry.com", Username: "myuser", Password: "mypassword", @@ -97,13 +97,13 @@ providers: func Test_templateDynamicCredentialProviderConfig(t *testing.T) { tests := []struct { name string - credentials imageRegistryCredentials + credentials providerInput want *cabpkv1.File wantErr error }{ { name: "ECR image registry", - credentials: imageRegistryCredentials{URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com"}, + credentials: providerInput{URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com"}, want: &cabpkv1.File{ Path: "/etc/kubernetes/dynamic-credential-provider-config.yaml", Owner: "", @@ -132,7 +132,7 @@ credentialProviders: }, { name: "image registry with static credentials", - credentials: imageRegistryCredentials{ + credentials: providerInput{ URL: "https://myregistry.com", Username: "myuser", Password: "mypassword", diff --git a/pkg/handlers/generic/mutation/imageregistrycredentials/credential_provider_insall_files.go b/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_insall_files.go similarity index 98% rename from pkg/handlers/generic/mutation/imageregistrycredentials/credential_provider_insall_files.go rename to pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_insall_files.go index 6692d0f16..25f091d95 100644 --- a/pkg/handlers/generic/mutation/imageregistrycredentials/credential_provider_insall_files.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_insall_files.go @@ -1,7 +1,7 @@ // Copyright 2023 D2iQ, Inc. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -package imageregistrycredentials +package credentials import ( "bytes" diff --git a/pkg/handlers/generic/mutation/imageregistrycredentials/credentialprovider/doc.go b/pkg/handlers/generic/mutation/imageregistries/credentials/credentialprovider/doc.go similarity index 100% rename from pkg/handlers/generic/mutation/imageregistrycredentials/credentialprovider/doc.go rename to pkg/handlers/generic/mutation/imageregistries/credentials/credentialprovider/doc.go diff --git a/pkg/handlers/generic/mutation/imageregistrycredentials/credentialprovider/matcher.go b/pkg/handlers/generic/mutation/imageregistries/credentials/credentialprovider/matcher.go similarity index 100% rename from pkg/handlers/generic/mutation/imageregistrycredentials/credentialprovider/matcher.go rename to pkg/handlers/generic/mutation/imageregistries/credentials/credentialprovider/matcher.go diff --git a/pkg/handlers/generic/mutation/imageregistrycredentials/credentialprovider/urls.go b/pkg/handlers/generic/mutation/imageregistries/credentials/credentialprovider/urls.go similarity index 100% rename from pkg/handlers/generic/mutation/imageregistrycredentials/credentialprovider/urls.go rename to pkg/handlers/generic/mutation/imageregistries/credentials/credentialprovider/urls.go diff --git a/pkg/handlers/generic/mutation/imageregistrycredentials/credentialprovider/urls_test.go b/pkg/handlers/generic/mutation/imageregistries/credentials/credentialprovider/urls_test.go similarity index 97% rename from pkg/handlers/generic/mutation/imageregistrycredentials/credentialprovider/urls_test.go rename to pkg/handlers/generic/mutation/imageregistries/credentials/credentialprovider/urls_test.go index 35e2f93f5..a44280701 100644 --- a/pkg/handlers/generic/mutation/imageregistrycredentials/credentialprovider/urls_test.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/credentialprovider/urls_test.go @@ -6,7 +6,7 @@ package credentialprovider_test import ( "testing" - "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/imageregistrycredentials/credentialprovider" + "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/imageregistries/credentials/credentialprovider" ) func TestURLsMatch(t *testing.T) { diff --git a/pkg/handlers/generic/mutation/imageregistrycredentials/credentials_secret.go b/pkg/handlers/generic/mutation/imageregistries/credentials/credentials_secret.go similarity index 90% rename from pkg/handlers/generic/mutation/imageregistrycredentials/credentials_secret.go rename to pkg/handlers/generic/mutation/imageregistries/credentials/credentials_secret.go index 83497c56c..522342398 100644 --- a/pkg/handlers/generic/mutation/imageregistrycredentials/credentials_secret.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/credentials_secret.go @@ -1,7 +1,7 @@ // Copyright 2023 D2iQ, Inc. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -package imageregistrycredentials +package credentials import ( "bytes" @@ -22,7 +22,7 @@ const ( secretKeyForStaticCredentialProviderConfig = "static-credential-provider" ) -func generateCredentialsSecretFile(credentials imageRegistryCredentials, ownerName string) []cabpkv1.File { +func generateCredentialsSecretFile(credentials providerInput, ownerName string) []cabpkv1.File { if credentials.isCredentialsEmpty() { return nil } @@ -43,7 +43,7 @@ func generateCredentialsSecretFile(credentials imageRegistryCredentials, ownerNa // generateCredentialsSecret generates a Secret containing the credentials for the image registry mirror. // The function needs the cluster name to add the required move and cluster name labels. func generateCredentialsSecret( - credentials imageRegistryCredentials, clusterName, ownerName, namespace string, + credentials providerInput, clusterName, ownerName, namespace string, ) (*corev1.Secret, error) { if credentials.isCredentialsEmpty() { return nil, nil @@ -72,7 +72,7 @@ func generateCredentialsSecret( }, nil } -func kubeletStaticCredentialProviderSecretContents(credentials imageRegistryCredentials) (string, error) { +func kubeletStaticCredentialProviderSecretContents(credentials providerInput) (string, error) { mirrorURL, err := url.ParseRequestURI(credentials.URL) if err != nil { return "", fmt.Errorf("failed parsing registry mirror: %w", err) diff --git a/pkg/handlers/generic/mutation/imageregistrycredentials/doc.go b/pkg/handlers/generic/mutation/imageregistries/credentials/doc.go similarity index 83% rename from pkg/handlers/generic/mutation/imageregistrycredentials/doc.go rename to pkg/handlers/generic/mutation/imageregistries/credentials/doc.go index 36d40c2ab..bd924ddeb 100644 --- a/pkg/handlers/generic/mutation/imageregistrycredentials/doc.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/doc.go @@ -2,4 +2,4 @@ // SPDX-License-Identifier: Apache-2.0 // +kubebuilder:rbac:groups="",resources=secrets,verbs=watch;list;get;patch -package imageregistrycredentials +package credentials diff --git a/pkg/handlers/generic/mutation/imageregistrycredentials/inject.go b/pkg/handlers/generic/mutation/imageregistries/credentials/inject.go similarity index 85% rename from pkg/handlers/generic/mutation/imageregistrycredentials/inject.go rename to pkg/handlers/generic/mutation/imageregistries/credentials/inject.go index d4e4ee525..912426463 100644 --- a/pkg/handlers/generic/mutation/imageregistrycredentials/inject.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/inject.go @@ -1,7 +1,7 @@ // Copyright 2023 D2iQ, Inc. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -package imageregistrycredentials +package credentials import ( "context" @@ -28,6 +28,7 @@ import ( "github.com/d2iq-labs/capi-runtime-extensions/common/pkg/capi/clustertopology/variables" "github.com/d2iq-labs/capi-runtime-extensions/common/pkg/k8s/client" "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/clusterconfig" + "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/imageregistries" ) const ( @@ -35,7 +36,7 @@ const ( HandlerNamePatch = "ImageRegistryCredentialsPatch" ) -type imageRegistryCredentialsPatchHandler struct { +type imageRegistriesPatchHandler struct { decoder runtime.Decoder client ctrlclient.Client @@ -44,32 +45,32 @@ type imageRegistryCredentialsPatchHandler struct { } var ( - _ commonhandlers.Named = &imageRegistryCredentialsPatchHandler{} - _ mutation.GeneratePatches = &imageRegistryCredentialsPatchHandler{} - _ mutation.MetaMutater = &imageRegistryCredentialsPatchHandler{} + _ commonhandlers.Named = &imageRegistriesPatchHandler{} + _ mutation.GeneratePatches = &imageRegistriesPatchHandler{} + _ mutation.MetaMutater = &imageRegistriesPatchHandler{} ) func NewPatch( cl ctrlclient.Client, -) *imageRegistryCredentialsPatchHandler { - return newImageRegistryCredentialsPatchHandler(cl, variableName) +) *imageRegistriesPatchHandler { + return newImageRegistriesPatchHandler(cl, variableName) } func NewMetaPatch( cl ctrlclient.Client, -) *imageRegistryCredentialsPatchHandler { - return newImageRegistryCredentialsPatchHandler(cl, clusterconfig.MetaVariableName, variableName) +) *imageRegistriesPatchHandler { + return newImageRegistriesPatchHandler(cl, clusterconfig.MetaVariableName, imageregistries.VariableName, variableName) } -func newImageRegistryCredentialsPatchHandler( +func newImageRegistriesPatchHandler( cl ctrlclient.Client, variableName string, variableFieldPath ...string, -) *imageRegistryCredentialsPatchHandler { +) *imageRegistriesPatchHandler { scheme := runtime.NewScheme() _ = bootstrapv1.AddToScheme(scheme) _ = controlplanev1.AddToScheme(scheme) - return &imageRegistryCredentialsPatchHandler{ + return &imageRegistriesPatchHandler{ decoder: serializer.NewCodecFactory(scheme).UniversalDecoder( controlplanev1.GroupVersion, bootstrapv1.GroupVersion, @@ -80,11 +81,11 @@ func newImageRegistryCredentialsPatchHandler( } } -func (h *imageRegistryCredentialsPatchHandler) Name() string { +func (h *imageRegistriesPatchHandler) Name() string { return HandlerNamePatch } -func (h *imageRegistryCredentialsPatchHandler) Mutate( +func (h *imageRegistriesPatchHandler) Mutate( ctx context.Context, obj runtime.Object, vars map[string]apiextensionsv1.JSON, @@ -95,7 +96,7 @@ func (h *imageRegistryCredentialsPatchHandler) Mutate( "holderRef", holderRef, ) - credentials, found, err := variables.Get[v1alpha1.ImageRegistryCredentials]( + imageRegistryCredentials, found, err := variables.Get[v1alpha1.ImageRegistryCredentials]( vars, h.variableName, h.variableFieldPath..., @@ -108,6 +109,13 @@ func (h *imageRegistryCredentialsPatchHandler) Mutate( return nil } + // TODO: Add support for multiple registries. + if len(imageRegistryCredentials) > 1 { + return fmt.Errorf("multiple Image Registry Credentials are not supported at this time") + } + + credentials := imageRegistryCredentials[0] + log = log.WithValues( "variableName", h.variableName, @@ -196,7 +204,7 @@ func (h *imageRegistryCredentialsPatchHandler) Mutate( return nil } -func (h *imageRegistryCredentialsPatchHandler) GeneratePatches( +func (h *imageRegistriesPatchHandler) GeneratePatches( ctx context.Context, req *runtimehooksv1.GeneratePatchesRequest, resp *runtimehooksv1.GeneratePatchesResponse, @@ -222,10 +230,10 @@ func (h *imageRegistryCredentialsPatchHandler) GeneratePatches( func registryWithOptionalCredentialsFromImageRegistryCredentials( ctx context.Context, c ctrlclient.Client, - credentials v1alpha1.ImageRegistryCredentials, + credentials v1alpha1.ImageRegistryCredentialsResource, obj ctrlclient.Object, -) (imageRegistryCredentials, error) { - registryWithOptionalCredentials := imageRegistryCredentials{ +) (providerInput, error) { + registryWithOptionalCredentials := providerInput{ URL: credentials.URL, } secret, err := secretForImageRegistryCredentials( @@ -235,7 +243,7 @@ func registryWithOptionalCredentialsFromImageRegistryCredentials( obj.GetNamespace(), ) if err != nil { - return imageRegistryCredentials{}, fmt.Errorf( + return providerInput{}, fmt.Errorf( "error getting secret %s/%s from Image Registry Credentials variable: %w", obj.GetNamespace(), credentials.Secret, @@ -252,7 +260,7 @@ func registryWithOptionalCredentialsFromImageRegistryCredentials( } func generateFilesAndCommands( - registryWithOptionalCredentials imageRegistryCredentials, + registryWithOptionalCredentials providerInput, objName string, ) ([]cabpkv1.File, []string, error) { @@ -273,7 +281,7 @@ func generateFilesAndCommands( func createSecretIfNeeded( ctx context.Context, c ctrlclient.Client, - registryWithOptionalCredentials imageRegistryCredentials, + registryWithOptionalCredentials providerInput, obj ctrlclient.Object, clusterKey ctrlclient.ObjectKey, ) error { @@ -295,7 +303,7 @@ func createSecretIfNeeded( func secretForImageRegistryCredentials( ctx context.Context, c ctrlclient.Reader, - credentials v1alpha1.ImageRegistryCredentials, + credentials v1alpha1.ImageRegistryCredentialsResource, objectNamespace string, ) (*corev1.Secret, error) { if credentials.Secret == nil { diff --git a/pkg/handlers/generic/mutation/imageregistrycredentials/inject_test.go b/pkg/handlers/generic/mutation/imageregistries/credentials/inject_test.go similarity index 88% rename from pkg/handlers/generic/mutation/imageregistrycredentials/inject_test.go rename to pkg/handlers/generic/mutation/imageregistries/credentials/inject_test.go index 72dd46099..203f3b468 100644 --- a/pkg/handlers/generic/mutation/imageregistrycredentials/inject_test.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/inject_test.go @@ -1,7 +1,7 @@ // Copyright 2023 D2iQ, Inc. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -package imageregistrycredentials +package credentials import ( "testing" @@ -27,7 +27,7 @@ const ( func TestGeneratePatches(t *testing.T) { capitest.ValidateGeneratePatches( t, - func() *imageRegistryCredentialsPatchHandler { + func() *imageRegistriesPatchHandler { // Server side apply does not work with the fake client, hack around it by pre-creating empty Secrets // https://github.com/kubernetes-sigs/controller-runtime/issues/2341 fakeClient := fake.NewClientBuilder().WithObjects( @@ -48,7 +48,9 @@ func TestGeneratePatches(t *testing.T) { capitest.VariableWithValue( variableName, v1alpha1.ImageRegistryCredentials{ - URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com", + v1alpha1.ImageRegistryCredentialsResource{ + URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com", + }, }, ), }, @@ -72,9 +74,11 @@ func TestGeneratePatches(t *testing.T) { capitest.VariableWithValue( variableName, v1alpha1.ImageRegistryCredentials{ - URL: "https://my-registry.io", - Secret: &corev1.ObjectReference{ - Name: validSecretName, + v1alpha1.ImageRegistryCredentialsResource{ + URL: "https://my-registry.io", + Secret: &corev1.ObjectReference{ + Name: validSecretName, + }, }, }, ), @@ -99,7 +103,9 @@ func TestGeneratePatches(t *testing.T) { capitest.VariableWithValue( variableName, v1alpha1.ImageRegistryCredentials{ - URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com", + v1alpha1.ImageRegistryCredentialsResource{ + URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com", + }, }, ), capitest.VariableWithValue( @@ -131,9 +137,11 @@ func TestGeneratePatches(t *testing.T) { capitest.VariableWithValue( variableName, v1alpha1.ImageRegistryCredentials{ - URL: "https://my-registry.io", - Secret: &corev1.ObjectReference{ - Name: validSecretName, + v1alpha1.ImageRegistryCredentialsResource{ + URL: "https://my-registry.io", + Secret: &corev1.ObjectReference{ + Name: validSecretName, + }, }, }, ), diff --git a/pkg/handlers/generic/mutation/imageregistrycredentials/templates/dynamic-credential-provider-config.yaml.gotmpl b/pkg/handlers/generic/mutation/imageregistries/credentials/templates/dynamic-credential-provider-config.yaml.gotmpl similarity index 100% rename from pkg/handlers/generic/mutation/imageregistrycredentials/templates/dynamic-credential-provider-config.yaml.gotmpl rename to pkg/handlers/generic/mutation/imageregistries/credentials/templates/dynamic-credential-provider-config.yaml.gotmpl diff --git a/pkg/handlers/generic/mutation/imageregistrycredentials/templates/image-credential-provider-config.yaml.gotmpl b/pkg/handlers/generic/mutation/imageregistries/credentials/templates/image-credential-provider-config.yaml.gotmpl similarity index 100% rename from pkg/handlers/generic/mutation/imageregistrycredentials/templates/image-credential-provider-config.yaml.gotmpl rename to pkg/handlers/generic/mutation/imageregistries/credentials/templates/image-credential-provider-config.yaml.gotmpl diff --git a/pkg/handlers/generic/mutation/imageregistrycredentials/templates/install-kubelet-credential-providers.sh.gotmpl b/pkg/handlers/generic/mutation/imageregistries/credentials/templates/install-kubelet-credential-providers.sh.gotmpl similarity index 100% rename from pkg/handlers/generic/mutation/imageregistrycredentials/templates/install-kubelet-credential-providers.sh.gotmpl rename to pkg/handlers/generic/mutation/imageregistries/credentials/templates/install-kubelet-credential-providers.sh.gotmpl diff --git a/pkg/handlers/generic/mutation/imageregistrycredentials/templates/static-credential-provider.json.gotmpl b/pkg/handlers/generic/mutation/imageregistries/credentials/templates/static-credential-provider.json.gotmpl similarity index 100% rename from pkg/handlers/generic/mutation/imageregistrycredentials/templates/static-credential-provider.json.gotmpl rename to pkg/handlers/generic/mutation/imageregistries/credentials/templates/static-credential-provider.json.gotmpl diff --git a/pkg/handlers/generic/mutation/imageregistrycredentials/variables.go b/pkg/handlers/generic/mutation/imageregistries/credentials/variables.go similarity index 95% rename from pkg/handlers/generic/mutation/imageregistrycredentials/variables.go rename to pkg/handlers/generic/mutation/imageregistries/credentials/variables.go index 5c5a3fbae..e7eea0331 100644 --- a/pkg/handlers/generic/mutation/imageregistrycredentials/variables.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/variables.go @@ -1,7 +1,7 @@ // Copyright 2023 D2iQ, Inc. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -package imageregistrycredentials +package credentials import ( "context" @@ -21,7 +21,7 @@ var ( const ( // variableName is the external patch variable name. - variableName = "imageRegistryCredentials" + variableName = "credentials" // HandlerNameVariable is the name of the variable handler. HandlerNameVariable = "ImageRegistryCredentialsVars" diff --git a/pkg/handlers/generic/mutation/imageregistrycredentials/variables_test.go b/pkg/handlers/generic/mutation/imageregistries/credentials/variables_test.go similarity index 72% rename from pkg/handlers/generic/mutation/imageregistrycredentials/variables_test.go rename to pkg/handlers/generic/mutation/imageregistries/credentials/variables_test.go index ce7743095..238859961 100644 --- a/pkg/handlers/generic/mutation/imageregistrycredentials/variables_test.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/variables_test.go @@ -1,7 +1,7 @@ // Copyright 2023 D2iQ, Inc. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -package imageregistrycredentials +package credentials import ( "testing" @@ -23,15 +23,19 @@ func TestVariableValidation(t *testing.T) { capitest.VariableTestDef{ Name: "without a Secret", Vals: v1alpha1.ImageRegistryCredentials{ - URL: "http://a.b.c.example.com", + v1alpha1.ImageRegistryCredentialsResource{ + URL: "http://a.b.c.example.com", + }, }, }, capitest.VariableTestDef{ Name: "with a Secret", Vals: v1alpha1.ImageRegistryCredentials{ - URL: "http://a.b.c.example.com", - Secret: &corev1.ObjectReference{ - Name: "a.b.c.example.com-creds", + v1alpha1.ImageRegistryCredentialsResource{ + URL: "http://a.b.c.example.com", + Secret: &corev1.ObjectReference{ + Name: "a.b.c.example.com-creds", + }, }, }, }, From a08c789ce6ba5cd74ec36b5d1be7e4ab3781f7a3 Mon Sep 17 00:00:00 2001 From: Dimitri Koshkin Date: Wed, 27 Sep 2023 10:52:16 -0700 Subject: [PATCH 04/11] fix: use renamed function after rebase --- .../generic/mutation/imageregistries/credentials/inject.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/inject.go b/pkg/handlers/generic/mutation/imageregistries/credentials/inject.go index 912426463..2ff2bd31c 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/inject.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/inject.go @@ -167,7 +167,7 @@ func (h *imageRegistriesPatchHandler) Mutate( } if err = patches.Generate( - obj, vars, &holderRef, selectors.AllWorkersSelector(), log, + obj, vars, &holderRef, selectors.WorkersKubeadmConfigTemplateSelector(), log, func(obj *bootstrapv1.KubeadmConfigTemplate) error { registryWithOptionalCredentials, generateErr := registryWithOptionalCredentialsFromImageRegistryCredentials(ctx, h.client, credentials, obj) From 0ba51be057464d7aad142b0226a6da25f82b02e5 Mon Sep 17 00:00:00 2001 From: Dimitri Koshkin Date: Wed, 27 Sep 2023 10:57:02 -0700 Subject: [PATCH 05/11] fix: remove mirror configuration from configs This will be implemented in a different PR --- .../credential_provider_config_files.go | 22 +++++++++---------- .../credential_provider_config_files_test.go | 8 ------- .../credentials/credentials_secret.go | 18 +++++++-------- ...mic-credential-provider-config.yaml.gotmpl | 9 +------- ...ge-credential-provider-config.yaml.gotmpl} | 1 - .../static-credential-provider.json.gotmpl | 4 ++-- 6 files changed, 23 insertions(+), 39 deletions(-) rename pkg/handlers/generic/mutation/imageregistries/credentials/templates/{image-credential-provider-config.yaml.gotmpl => kubelet-image-credential-provider-config.yaml.gotmpl} (91%) diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files.go b/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files.go index 3603bceb4..e2e6645cb 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files.go @@ -44,8 +44,8 @@ var ( //go:embed templates/static-credential-provider.json.gotmpl staticCredentialProviderConfigPatch []byte - //go:embed templates/image-credential-provider-config.yaml.gotmpl - imageCredentialProviderConfigPatch []byte + //go:embed templates/kubelet-image-credential-provider-config.yaml.gotmpl + kubeletImageCredentialProviderConfigPatch []byte //go:embed templates/install-kubelet-credential-providers.sh.gotmpl installKubeletCredentialProvidersScript []byte @@ -87,7 +87,7 @@ func templateFilesForImageCredentialProviderConfigs(credentials providerInput) ( func templateKubeletCredentialProviderConfig(credentials providerInput) (*cabpkv1.File, error) { return templateCredentialProviderConfig( credentials, - imageCredentialProviderConfigPatch, + kubeletImageCredentialProviderConfigPatch, kubeletImageCredentialProviderConfigOnRemote, kubeletCredentialProvider, ) @@ -113,9 +113,9 @@ func templateCredentialProviderConfig( host string, ) (providerBinary string, providerArgs []string, providerAPIVersion string, err error), ) (*cabpkv1.File, error) { - mirrorURL, err := url.ParseRequestURI(credentials.URL) + registryURL, err := url.ParseRequestURI(credentials.URL) if err != nil { - return nil, fmt.Errorf("failed parsing registry mirror: %w", err) + return nil, fmt.Errorf("failed parsing registry URL: %w", err) } t := template.New("") @@ -127,14 +127,14 @@ func templateCredentialProviderConfig( return nil, fmt.Errorf("failed to parse go template: %w", err) } - mirrorHostWithPath := mirrorURL.Host - if mirrorURL.Path != "" { - mirrorHostWithPath = path.Join(mirrorURL.Host, mirrorURL.Path) + registryHostWithPath := registryURL.Host + if registryURL.Path != "" { + registryHostWithPath = path.Join(registryURL.Host, registryURL.Path) } providerBinary, providerArgs, providerAPIVersion, err := providerFunc( !credentials.isCredentialsEmpty(), - mirrorHostWithPath, + registryHostWithPath, ) if err != nil { return nil, err @@ -144,12 +144,12 @@ func templateCredentialProviderConfig( } templateInput := struct { - MirrorHost string + RegistryHost string ProviderBinary string ProviderArgs []string ProviderAPIVersion string }{ - MirrorHost: mirrorHostWithPath, + RegistryHost: registryHostWithPath, ProviderBinary: providerBinary, ProviderArgs: providerArgs, ProviderAPIVersion: providerAPIVersion, diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files_test.go b/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files_test.go index e577f8c1e..7b1f54c7e 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files_test.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files_test.go @@ -36,7 +36,6 @@ providers: - -c - /etc/kubernetes/dynamic-credential-provider-config.yaml matchImages: - - "123456789.dkr.ecr.us-east-1.amazonaws.com" - "*" - "*.*" - "*.*.*" @@ -70,7 +69,6 @@ providers: - -c - /etc/kubernetes/dynamic-credential-provider-config.yaml matchImages: - - "myregistry.com" - "*" - "*.*" - "*.*.*" @@ -113,9 +111,6 @@ func Test_templateDynamicCredentialProviderConfig(t *testing.T) { Content: `apiVersion: credentialprovider.d2iq.com/v1alpha1 kind: DynamicCredentialProviderConfig credentialProviderPluginBinDir: /etc/kubernetes/image-credential-provider/ -mirror: - endpoint: "123456789.dkr.ecr.us-east-1.amazonaws.com" - credentialsStrategy: MirrorCredentialsOnly credentialProviders: apiVersion: kubelet.config.k8s.io/v1beta1 kind: CredentialProviderConfig @@ -146,9 +141,6 @@ credentialProviders: Content: `apiVersion: credentialprovider.d2iq.com/v1alpha1 kind: DynamicCredentialProviderConfig credentialProviderPluginBinDir: /etc/kubernetes/image-credential-provider/ -mirror: - endpoint: "myregistry.com" - credentialsStrategy: MirrorCredentialsOnly credentialProviders: apiVersion: kubelet.config.k8s.io/v1beta1 kind: CredentialProviderConfig diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/credentials_secret.go b/pkg/handlers/generic/mutation/imageregistries/credentials/credentials_secret.go index 522342398..2f4d5f88e 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/credentials_secret.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/credentials_secret.go @@ -40,7 +40,7 @@ func generateCredentialsSecretFile(credentials providerInput, ownerName string) } } -// generateCredentialsSecret generates a Secret containing the credentials for the image registry mirror. +// generateCredentialsSecret generates a Secret containing the credentials for the image registry. // The function needs the cluster name to add the required move and cluster name labels. func generateCredentialsSecret( credentials providerInput, clusterName, ownerName, namespace string, @@ -73,19 +73,19 @@ func generateCredentialsSecret( } func kubeletStaticCredentialProviderSecretContents(credentials providerInput) (string, error) { - mirrorURL, err := url.ParseRequestURI(credentials.URL) + registryURL, err := url.ParseRequestURI(credentials.URL) if err != nil { - return "", fmt.Errorf("failed parsing registry mirror: %w", err) + return "", fmt.Errorf("failed parsing registry URL: %w", err) } templateInput := struct { - MirrorHost string - Username string - Password string + RegistryHost string + Username string + Password string }{ - MirrorHost: mirrorURL.Host, - Username: credentials.Username, - Password: credentials.Password, + RegistryHost: registryURL.Host, + Username: credentials.Username, + Password: credentials.Password, } t, err := template.New("").Parse(string(staticCredentialProviderConfigPatch)) if err != nil { diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/templates/dynamic-credential-provider-config.yaml.gotmpl b/pkg/handlers/generic/mutation/imageregistries/credentials/templates/dynamic-credential-provider-config.yaml.gotmpl index 8b30845f1..d476d4186 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/templates/dynamic-credential-provider-config.yaml.gotmpl +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/templates/dynamic-credential-provider-config.yaml.gotmpl @@ -1,13 +1,6 @@ apiVersion: credentialprovider.d2iq.com/v1alpha1 kind: DynamicCredentialProviderConfig credentialProviderPluginBinDir: /etc/kubernetes/image-credential-provider/ -{{- with .MirrorHost }} -{{- if and (ne . "registry-1.docker.io") (ne . "docker.io") }} -mirror: - endpoint: {{ printf "%q" . }} - credentialsStrategy: MirrorCredentialsOnly -{{- end }} -{{- end }} credentialProviders: apiVersion: kubelet.config.k8s.io/v1beta1 kind: CredentialProviderConfig @@ -20,7 +13,7 @@ credentialProviders: {{- end }} {{- end }} matchImages: - {{- with .MirrorHost }} + {{- with .RegistryHost }} - {{ printf "%q" . }} {{- if eq . "registry-1.docker.io" }} - "docker.io" diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/templates/image-credential-provider-config.yaml.gotmpl b/pkg/handlers/generic/mutation/imageregistries/credentials/templates/kubelet-image-credential-provider-config.yaml.gotmpl similarity index 91% rename from pkg/handlers/generic/mutation/imageregistries/credentials/templates/image-credential-provider-config.yaml.gotmpl rename to pkg/handlers/generic/mutation/imageregistries/credentials/templates/kubelet-image-credential-provider-config.yaml.gotmpl index 408557a66..57d62eec1 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/templates/image-credential-provider-config.yaml.gotmpl +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/templates/kubelet-image-credential-provider-config.yaml.gotmpl @@ -9,7 +9,6 @@ providers: {{- end }} {{- end }} matchImages: - - {{ printf "%q" .MirrorHost }} - "*" - "*.*" - "*.*.*" diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/templates/static-credential-provider.json.gotmpl b/pkg/handlers/generic/mutation/imageregistries/credentials/templates/static-credential-provider.json.gotmpl index 8646d5202..207ebf5d7 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/templates/static-credential-provider.json.gotmpl +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/templates/static-credential-provider.json.gotmpl @@ -4,8 +4,8 @@ "cacheKeyType":"Image", "cacheDuration":"0s", "auth":{ - {{- if .MirrorHost }} - {{ printf "%q" .MirrorHost }}: {"username": {{ printf "%q" .Username }}, "password": {{ printf "%q" .Password }}}{{ if eq .MirrorHost "registry-1.docker.io" }}, + {{- if .RegistryHost }} + {{ printf "%q" .RegistryHost }}: {"username": {{ printf "%q" .Username }}, "password": {{ printf "%q" .Password }}}{{ if eq .RegistryHost "registry-1.docker.io" }}, "docker.io": {"username": {{ printf "%q" .Username }}, "password": {{ printf "%q" .Password }}} {{- end }} {{- end }} From 7ff3d7af44dea48cb2700a9d308f8de5a558f349 Mon Sep 17 00:00:00 2001 From: Dimitri Koshkin Date: Wed, 27 Sep 2023 11:44:24 -0700 Subject: [PATCH 06/11] fix: use a better struct name --- .../credential_provider_config_files.go | 42 +++++++++---------- .../credential_provider_config_files_test.go | 34 +++++++-------- .../credential_provider_insall_files.go | 2 +- .../credentials/credentials_secret.go | 22 +++++----- .../imageregistries/credentials/inject.go | 12 +++--- .../credentials/inject_test.go | 2 +- .../imageregistries/credentials/variables.go | 2 +- 7 files changed, 58 insertions(+), 58 deletions(-) diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files.go b/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files.go index e2e6645cb..1ba8830fa 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files.go @@ -19,21 +19,21 @@ import ( ) const ( - //nolint:gosec // Does not contain hard coded credentials. - kubeletStaticCredentialProviderCredentialsOnRemote = "/etc/kubernetes/static-image-credentials.json" + //nolint:gosec // Does not contain hard coded config. + kubeletStaticCredentialProviderCredentialsOnRemote = "/etc/kubernetes/static-image-config.json" - //nolint:gosec // Does not contain hard coded credentials. + //nolint:gosec // Does not contain hard coded config. kubeletImageCredentialProviderConfigOnRemote = "/etc/kubernetes/image-credential-provider-config.yaml" - //nolint:gosec // Does not contain hard coded credentials. + //nolint:gosec // Does not contain hard coded config. kubeletDynamicCredentialProviderConfigOnRemote = "/etc/kubernetes/dynamic-credential-provider-config.yaml" azureCloudConfigFilePath = "/etc/kubernetes/azure.json" - //nolint:gosec // Does not contain hard coded credentials. + //nolint:gosec // Does not contain hard coded config. dynamicCredentialProviderImage = "ghcr.io/mesosphere/dynamic-credential-provider:v0.2.0" - //nolint:gosec // Does not contain hard coded credentials. + //nolint:gosec // Does not contain hard coded config. credentialProviderTargetDir = "/etc/kubernetes/image-credential-provider/" ) @@ -51,21 +51,21 @@ var ( installKubeletCredentialProvidersScript []byte ) -type providerInput struct { +type providerConfig struct { URL string Username string Password string } -func (c providerInput) isCredentialsEmpty() bool { +func (c providerConfig) isCredentialsEmpty() bool { return c.Username == "" && c.Password == "" } -func templateFilesForImageCredentialProviderConfigs(credentials providerInput) ([]cabpkv1.File, error) { +func templateFilesForImageCredentialProviderConfigs(config providerConfig) ([]cabpkv1.File, error) { var files []cabpkv1.File - kubeletCredentialProviderConfigFile, err := templateKubeletCredentialProviderConfig(credentials) + kubeletCredentialProviderConfigFile, err := templateKubeletCredentialProviderConfig(config) if err != nil { return nil, err } @@ -73,7 +73,7 @@ func templateFilesForImageCredentialProviderConfigs(credentials providerInput) ( files = append(files, *kubeletCredentialProviderConfigFile) } - kubeletDynamicCredentialProviderConfigFile, err := templateDynamicCredentialProviderConfig(credentials) + kubeletDynamicCredentialProviderConfigFile, err := templateDynamicCredentialProviderConfig(config) if err != nil { return nil, err } @@ -84,9 +84,9 @@ func templateFilesForImageCredentialProviderConfigs(credentials providerInput) ( return files, nil } -func templateKubeletCredentialProviderConfig(credentials providerInput) (*cabpkv1.File, error) { +func templateKubeletCredentialProviderConfig(config providerConfig) (*cabpkv1.File, error) { return templateCredentialProviderConfig( - credentials, + config, kubeletImageCredentialProviderConfigPatch, kubeletImageCredentialProviderConfigOnRemote, kubeletCredentialProvider, @@ -94,10 +94,10 @@ func templateKubeletCredentialProviderConfig(credentials providerInput) (*cabpkv } func templateDynamicCredentialProviderConfig( - credentials providerInput, + config providerConfig, ) (*cabpkv1.File, error) { return templateCredentialProviderConfig( - credentials, + config, dynamicCredentialProviderConfigPatch, kubeletDynamicCredentialProviderConfigOnRemote, dynamicCredentialProvider, @@ -105,7 +105,7 @@ func templateDynamicCredentialProviderConfig( } func templateCredentialProviderConfig( - credentials providerInput, + config providerConfig, inputTemplate []byte, filePath string, providerFunc func( @@ -113,7 +113,7 @@ func templateCredentialProviderConfig( host string, ) (providerBinary string, providerArgs []string, providerAPIVersion string, err error), ) (*cabpkv1.File, error) { - registryURL, err := url.ParseRequestURI(credentials.URL) + registryURL, err := url.ParseRequestURI(config.URL) if err != nil { return nil, fmt.Errorf("failed parsing registry URL: %w", err) } @@ -133,7 +133,7 @@ func templateCredentialProviderConfig( } providerBinary, providerArgs, providerAPIVersion, err := providerFunc( - !credentials.isCredentialsEmpty(), + !config.isCredentialsEmpty(), registryHostWithPath, ) if err != nil { @@ -175,7 +175,7 @@ func kubeletCredentialProvider(hasStaticCredentials bool, host string) ( return "", nil, "", err } return "dynamic-credential-provider", - []string{"get-credentials", "-c", kubeletDynamicCredentialProviderConfigOnRemote}, + []string{"get-config", "-c", kubeletDynamicCredentialProviderConfigOnRemote}, credentialproviderv1beta1.SchemeGroupVersion.String(), nil } @@ -191,12 +191,12 @@ func dynamicCredentialProvider(hasStaticCredentials bool, host string) ( } if matches, err := credentialprovider.URLMatchesECR(host); matches || err != nil { - return "ecr-credential-provider", []string{"get-credentials"}, + return "ecr-credential-provider", []string{"get-config"}, credentialproviderv1alpha1.SchemeGroupVersion.String(), err } if matches, err := credentialprovider.URLMatchesGCR(host); matches || err != nil { - return "gcr-credential-provider", []string{"get-credentials"}, + return "gcr-credential-provider", []string{"get-config"}, credentialproviderv1alpha1.SchemeGroupVersion.String(), err } diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files_test.go b/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files_test.go index 7b1f54c7e..add96be7d 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files_test.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files_test.go @@ -13,14 +13,14 @@ import ( func Test_templateKubeletCredentialProviderConfig(t *testing.T) { tests := []struct { - name string - credentials providerInput - want *cabpkv1.File - wantErr error + name string + config providerConfig + want *cabpkv1.File + wantErr error }{ { - name: "ECR image registry", - credentials: providerInput{URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com"}, + name: "ECR image registry", + config: providerConfig{URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com"}, want: &cabpkv1.File{ Path: "/etc/kubernetes/image-credential-provider-config.yaml", Owner: "", @@ -32,7 +32,7 @@ kind: CredentialProviderConfig providers: - name: dynamic-credential-provider args: - - get-credentials + - get-config - -c - /etc/kubernetes/dynamic-credential-provider-config.yaml matchImages: @@ -48,8 +48,8 @@ providers: }, }, { - name: "image registry with static credentials", - credentials: providerInput{ + name: "image registry with static config", + config: providerConfig{ URL: "https://myregistry.com", Username: "myuser", Password: "mypassword", @@ -65,7 +65,7 @@ kind: CredentialProviderConfig providers: - name: dynamic-credential-provider args: - - get-credentials + - get-config - -c - /etc/kubernetes/dynamic-credential-provider-config.yaml matchImages: @@ -85,7 +85,7 @@ providers: tt := tests[idx] t.Run(tt.name, func(t *testing.T) { t.Parallel() - file, err := templateKubeletCredentialProviderConfig(tt.credentials) + file, err := templateKubeletCredentialProviderConfig(tt.config) assert.ErrorIs(t, err, tt.wantErr) assert.Equal(t, tt.want, file) }) @@ -95,13 +95,13 @@ providers: func Test_templateDynamicCredentialProviderConfig(t *testing.T) { tests := []struct { name string - credentials providerInput + credentials providerConfig want *cabpkv1.File wantErr error }{ { name: "ECR image registry", - credentials: providerInput{URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com"}, + credentials: providerConfig{URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com"}, want: &cabpkv1.File{ Path: "/etc/kubernetes/dynamic-credential-provider-config.yaml", Owner: "", @@ -117,7 +117,7 @@ credentialProviders: providers: - name: ecr-credential-provider args: - - get-credentials + - get-config matchImages: - "123456789.dkr.ecr.us-east-1.amazonaws.com" defaultCacheDuration: "0s" @@ -126,8 +126,8 @@ credentialProviders: }, }, { - name: "image registry with static credentials", - credentials: providerInput{ + name: "image registry with static config", + credentials: providerConfig{ URL: "https://myregistry.com", Username: "myuser", Password: "mypassword", @@ -147,7 +147,7 @@ credentialProviders: providers: - name: static-credential-provider args: - - /etc/kubernetes/static-image-credentials.json + - /etc/kubernetes/static-image-config.json matchImages: - "myregistry.com" defaultCacheDuration: "0s" diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_insall_files.go b/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_insall_files.go index 25f091d95..8ec46f5b7 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_insall_files.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_insall_files.go @@ -12,7 +12,7 @@ import ( ) const ( - //nolint:gosec // Does not contain hard coded credentials. + //nolint:gosec // Does not contain hard coded config. installKubeletCredentialProvidersScriptOnRemote = "/etc/konvoy/install-kubelet-credential-providers.sh" installKubeletCredentialProvidersScriptOnRemoteCommand = "/bin/bash " + installKubeletCredentialProvidersScriptOnRemote diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/credentials_secret.go b/pkg/handlers/generic/mutation/imageregistries/credentials/credentials_secret.go index 2f4d5f88e..3c5c4878a 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/credentials_secret.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/credentials_secret.go @@ -22,8 +22,8 @@ const ( secretKeyForStaticCredentialProviderConfig = "static-credential-provider" ) -func generateCredentialsSecretFile(credentials providerInput, ownerName string) []cabpkv1.File { - if credentials.isCredentialsEmpty() { +func generateCredentialsSecretFile(config providerConfig, ownerName string) []cabpkv1.File { + if config.isCredentialsEmpty() { return nil } return []cabpkv1.File{ @@ -40,16 +40,16 @@ func generateCredentialsSecretFile(credentials providerInput, ownerName string) } } -// generateCredentialsSecret generates a Secret containing the credentials for the image registry. +// generateCredentialsSecret generates a Secret containing the config for the image registry. // The function needs the cluster name to add the required move and cluster name labels. func generateCredentialsSecret( - credentials providerInput, clusterName, ownerName, namespace string, + config providerConfig, clusterName, ownerName, namespace string, ) (*corev1.Secret, error) { - if credentials.isCredentialsEmpty() { + if config.isCredentialsEmpty() { return nil, nil } - staticCredentialProviderSecretContents, err := kubeletStaticCredentialProviderSecretContents(credentials) + staticCredentialProviderSecretContents, err := kubeletStaticCredentialProviderSecretContents(config) if err != nil { return nil, err } @@ -72,8 +72,8 @@ func generateCredentialsSecret( }, nil } -func kubeletStaticCredentialProviderSecretContents(credentials providerInput) (string, error) { - registryURL, err := url.ParseRequestURI(credentials.URL) +func kubeletStaticCredentialProviderSecretContents(config providerConfig) (string, error) { + registryURL, err := url.ParseRequestURI(config.URL) if err != nil { return "", fmt.Errorf("failed parsing registry URL: %w", err) } @@ -84,8 +84,8 @@ func kubeletStaticCredentialProviderSecretContents(credentials providerInput) (s Password string }{ RegistryHost: registryURL.Host, - Username: credentials.Username, - Password: credentials.Password, + Username: config.Username, + Password: config.Password, } t, err := template.New("").Parse(string(staticCredentialProviderConfigPatch)) if err != nil { @@ -102,7 +102,7 @@ func kubeletStaticCredentialProviderSecretContents(credentials providerInput) (s } func credentialSecretName(ownerName string) string { - return fmt.Sprintf("%s-registry-credentials", ownerName) + return fmt.Sprintf("%s-registry-config", ownerName) } type labelFn func(labels map[string]string) diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/inject.go b/pkg/handlers/generic/mutation/imageregistries/credentials/inject.go index 2ff2bd31c..3addf39bf 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/inject.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/inject.go @@ -232,8 +232,8 @@ func registryWithOptionalCredentialsFromImageRegistryCredentials( c ctrlclient.Client, credentials v1alpha1.ImageRegistryCredentialsResource, obj ctrlclient.Object, -) (providerInput, error) { - registryWithOptionalCredentials := providerInput{ +) (providerConfig, error) { + registryWithOptionalCredentials := providerConfig{ URL: credentials.URL, } secret, err := secretForImageRegistryCredentials( @@ -243,7 +243,7 @@ func registryWithOptionalCredentialsFromImageRegistryCredentials( obj.GetNamespace(), ) if err != nil { - return providerInput{}, fmt.Errorf( + return providerConfig{}, fmt.Errorf( "error getting secret %s/%s from Image Registry Credentials variable: %w", obj.GetNamespace(), credentials.Secret, @@ -260,7 +260,7 @@ func registryWithOptionalCredentialsFromImageRegistryCredentials( } func generateFilesAndCommands( - registryWithOptionalCredentials providerInput, + registryWithOptionalCredentials providerConfig, objName string, ) ([]cabpkv1.File, []string, error) { @@ -281,13 +281,13 @@ func generateFilesAndCommands( func createSecretIfNeeded( ctx context.Context, c ctrlclient.Client, - registryWithOptionalCredentials providerInput, + registryWithOptionalCredentials providerConfig, obj ctrlclient.Object, clusterKey ctrlclient.ObjectKey, ) error { credentialsSecret, err := generateCredentialsSecret(registryWithOptionalCredentials, clusterKey.Name, obj.GetName(), obj.GetNamespace()) if err != nil { - return fmt.Errorf("error generating credentials Secret for Image Registry Credentials variable: %w", err) + return fmt.Errorf("error generating config Secret for Image Registry Credentials variable: %w", err) } if credentialsSecret != nil { if err := client.ServerSideApply(ctx, c, credentialsSecret); err != nil { diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/inject_test.go b/pkg/handlers/generic/mutation/imageregistries/credentials/inject_test.go index 203f3b468..8d7e61e2c 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/inject_test.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/inject_test.go @@ -21,7 +21,7 @@ import ( ) const ( - validSecretName = "myregistry-credentials" + validSecretName = "myregistry-config" ) func TestGeneratePatches(t *testing.T) { diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/variables.go b/pkg/handlers/generic/mutation/imageregistries/credentials/variables.go index e7eea0331..73d298cfd 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/variables.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/variables.go @@ -21,7 +21,7 @@ var ( const ( // variableName is the external patch variable name. - variableName = "credentials" + variableName = "config" // HandlerNameVariable is the name of the variable handler. HandlerNameVariable = "ImageRegistryCredentialsVars" From fd5c35ffbe4560fffe8cce277644af0dd6a51b78 Mon Sep 17 00:00:00 2001 From: Dimitri Koshkin Date: Wed, 27 Sep 2023 13:10:03 -0700 Subject: [PATCH 07/11] fix: error on missing credentials --- common/pkg/testutils/capitest/patches.go | 7 +- .../credential_provider_config_files.go | 147 +++++++----------- .../credential_provider_config_files_test.go | 40 ++++- .../credential_provider_insall_files.go | 12 ++ .../credentials/credentials_secret.go | 6 + .../credentials/inject_test.go | 15 ++ 6 files changed, 138 insertions(+), 89 deletions(-) diff --git a/common/pkg/testutils/capitest/patches.go b/common/pkg/testutils/capitest/patches.go index b7377947e..b24416550 100644 --- a/common/pkg/testutils/capitest/patches.go +++ b/common/pkg/testutils/capitest/patches.go @@ -25,6 +25,7 @@ type PatchTestDef struct { Vars []runtimehooksv1.Variable RequestItem runtimehooksv1.GeneratePatchesRequestItem ExpectedPatchMatchers []JSONPatchMatcher + ExpectedFailure bool } type JSONPatchMatcher struct { @@ -56,7 +57,11 @@ func ValidateGeneratePatches[T mutation.GeneratePatches]( } resp := &runtimehooksv1.GeneratePatchesResponse{} h.GeneratePatches(context.Background(), req, resp) - g.Expect(resp.Status).To(gomega.Equal(runtimehooksv1.ResponseStatusSuccess), fmt.Sprintf("Message: %s", resp.Message)) + expectedStatus := runtimehooksv1.ResponseStatusSuccess + if tt.ExpectedFailure { + expectedStatus = runtimehooksv1.ResponseStatusFailure + } + g.Expect(resp.Status).To(gomega.Equal(expectedStatus), fmt.Sprintf("Message: %s", resp.Message)) if len(tt.ExpectedPatchMatchers) == 0 { g.Expect(resp.Items).To(gomega.BeEmpty()) diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files.go b/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files.go index 1ba8830fa..ab90f6d1e 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files.go @@ -6,6 +6,7 @@ package credentials import ( "bytes" _ "embed" + "errors" "fmt" "net/url" "path" @@ -29,26 +30,18 @@ const ( kubeletDynamicCredentialProviderConfigOnRemote = "/etc/kubernetes/dynamic-credential-provider-config.yaml" azureCloudConfigFilePath = "/etc/kubernetes/azure.json" - - //nolint:gosec // Does not contain hard coded config. - dynamicCredentialProviderImage = "ghcr.io/mesosphere/dynamic-credential-provider:v0.2.0" - - //nolint:gosec // Does not contain hard coded config. - credentialProviderTargetDir = "/etc/kubernetes/image-credential-provider/" ) var ( //go:embed templates/dynamic-credential-provider-config.yaml.gotmpl dynamicCredentialProviderConfigPatch []byte - //go:embed templates/static-credential-provider.json.gotmpl - staticCredentialProviderConfigPatch []byte - //go:embed templates/kubelet-image-credential-provider-config.yaml.gotmpl kubeletImageCredentialProviderConfigPatch []byte +) - //go:embed templates/install-kubelet-credential-providers.sh.gotmpl - installKubeletCredentialProvidersScript []byte +var ( + ErrCredentialsNotFound = errors.New("registry credentials not found") ) type providerConfig struct { @@ -65,7 +58,7 @@ func (c providerConfig) isCredentialsEmpty() bool { func templateFilesForImageCredentialProviderConfigs(config providerConfig) ([]cabpkv1.File, error) { var files []cabpkv1.File - kubeletCredentialProviderConfigFile, err := templateKubeletCredentialProviderConfig(config) + kubeletCredentialProviderConfigFile, err := templateKubeletCredentialProviderConfig() if err != nil { return nil, err } @@ -84,34 +77,33 @@ func templateFilesForImageCredentialProviderConfigs(config providerConfig) ([]ca return files, nil } -func templateKubeletCredentialProviderConfig(config providerConfig) (*cabpkv1.File, error) { - return templateCredentialProviderConfig( - config, - kubeletImageCredentialProviderConfigPatch, - kubeletImageCredentialProviderConfigOnRemote, - kubeletCredentialProvider, - ) -} +func templateKubeletCredentialProviderConfig() (*cabpkv1.File, error) { + t := template.New("") + t, err := t.Parse(string(kubeletImageCredentialProviderConfigPatch)) + if err != nil { + return nil, fmt.Errorf("failed to parse go template: %w", err) + } -func templateDynamicCredentialProviderConfig( - config providerConfig, -) (*cabpkv1.File, error) { - return templateCredentialProviderConfig( - config, - dynamicCredentialProviderConfigPatch, - kubeletDynamicCredentialProviderConfigOnRemote, - dynamicCredentialProvider, - ) + providerBinary, providerArgs, providerAPIVersion, err := kubeletCredentialProvider() + if err != nil { + return nil, err + } + + templateInput := struct { + ProviderBinary string + ProviderArgs []string + ProviderAPIVersion string + }{ + ProviderBinary: providerBinary, + ProviderArgs: providerArgs, + ProviderAPIVersion: providerAPIVersion, + } + + return fileFromTemplate(t, templateInput, kubeletImageCredentialProviderConfigOnRemote) } -func templateCredentialProviderConfig( +func templateDynamicCredentialProviderConfig( config providerConfig, - inputTemplate []byte, - filePath string, - providerFunc func( - hasStaticCredentials bool, - host string, - ) (providerBinary string, providerArgs []string, providerAPIVersion string, err error), ) (*cabpkv1.File, error) { registryURL, err := url.ParseRequestURI(config.URL) if err != nil { @@ -119,10 +111,7 @@ func templateCredentialProviderConfig( } t := template.New("") - t.Funcs(map[string]any{ - "urlsMatchStr": credentialprovider.URLsMatchStr, - }) - t, err = t.Parse(string(inputTemplate)) + t, err = t.Parse(string(dynamicCredentialProviderConfigPatch)) if err != nil { return nil, fmt.Errorf("failed to parse go template: %w", err) } @@ -132,15 +121,17 @@ func templateCredentialProviderConfig( registryHostWithPath = path.Join(registryURL.Host, registryURL.Path) } - providerBinary, providerArgs, providerAPIVersion, err := providerFunc( - !config.isCredentialsEmpty(), - registryHostWithPath, - ) + supportedProvider, err := credentialprovider.URLMatchesSupportedProvider(registryHostWithPath) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to check if registry matches a supporterd provider: %w", err) } - if providerBinary == "" { - return nil, nil + if config.isCredentialsEmpty() && !supportedProvider { + return nil, ErrCredentialsNotFound + } + + providerBinary, providerArgs, providerAPIVersion, err := dynamicCredentialProvider(registryHostWithPath) + if err != nil { + return nil, err } templateInput := struct { @@ -155,41 +146,21 @@ func templateCredentialProviderConfig( ProviderAPIVersion: providerAPIVersion, } - var b bytes.Buffer - err = t.Execute(&b, templateInput) - if err != nil { - return nil, fmt.Errorf("failed executing template: %w", err) - } - - return &cabpkv1.File{ - Path: filePath, - Content: b.String(), - Permissions: "0600", - }, nil + return fileFromTemplate(t, templateInput, kubeletDynamicCredentialProviderConfigOnRemote) } -func kubeletCredentialProvider(hasStaticCredentials bool, host string) ( +func kubeletCredentialProvider() ( providerBinary string, providerArgs []string, providerAPIVersion string, err error, ) { - if needs, err := needCredentialProvider(hasStaticCredentials, host); !needs || err != nil { - return "", nil, "", err - } return "dynamic-credential-provider", []string{"get-config", "-c", kubeletDynamicCredentialProviderConfigOnRemote}, credentialproviderv1beta1.SchemeGroupVersion.String(), nil } -func dynamicCredentialProvider(hasStaticCredentials bool, host string) ( +func dynamicCredentialProvider(host string) ( providerBinary string, providerArgs []string, providerAPIVersion string, err error, ) { - if hasStaticCredentials { - return "static-credential-provider", - []string{kubeletStaticCredentialProviderCredentialsOnRemote}, - credentialproviderv1beta1.SchemeGroupVersion.String(), - nil - } - if matches, err := credentialprovider.URLMatchesECR(host); matches || err != nil { return "ecr-credential-provider", []string{"get-config"}, credentialproviderv1alpha1.SchemeGroupVersion.String(), err @@ -206,25 +177,27 @@ func dynamicCredentialProvider(hasStaticCredentials bool, host string) ( }, credentialproviderv1alpha1.SchemeGroupVersion.String(), err } - return "", nil, "", nil + // if no supported provider was found, assume we are using the static credential provider + return "static-credential-provider", + []string{kubeletStaticCredentialProviderCredentialsOnRemote}, + credentialproviderv1beta1.SchemeGroupVersion.String(), + nil } -func needCredentialProvider(hasStaticCredentials bool, host string) (bool, error) { - if hasStaticCredentials { - return true, nil - } - if matches, err := credentialprovider.URLMatchesECR(host); matches || err != nil { - //nolint:wrapcheck // No need to wrap this error, it has all context needed. - return matches, err - } - if matches, err := credentialprovider.URLMatchesGCR(host); matches || err != nil { - //nolint:wrapcheck // No need to wrap this error, it has all context needed. - return matches, err - } - if matches, err := credentialprovider.URLMatchesACR(host); matches || err != nil { - //nolint:wrapcheck // No need to wrap this error, it has all context needed. - return matches, err +func fileFromTemplate( + t *template.Template, + templateInput any, + path string, +) (*cabpkv1.File, error) { + var b bytes.Buffer + err := t.Execute(&b, templateInput) + if err != nil { + return nil, fmt.Errorf("failed executing template: %w", err) } - return false, nil + return &cabpkv1.File{ + Path: path, + Content: b.String(), + Permissions: "0600", + }, nil } diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files_test.go b/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files_test.go index add96be7d..1540a08cd 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files_test.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files_test.go @@ -85,7 +85,7 @@ providers: tt := tests[idx] t.Run(tt.name, func(t *testing.T) { t.Parallel() - file, err := templateKubeletCredentialProviderConfig(tt.config) + file, err := templateKubeletCredentialProviderConfig() assert.ErrorIs(t, err, tt.wantErr) assert.Equal(t, tt.want, file) }) @@ -155,6 +155,44 @@ credentialProviders: `, }, }, + { + name: "docker.io registry with static config", + credentials: providerConfig{ + URL: "https://registry-1.docker.io", + Username: "myuser", + Password: "mypassword", + }, + want: &cabpkv1.File{ + Path: "/etc/kubernetes/dynamic-credential-provider-config.yaml", + Owner: "", + Permissions: "0600", + Encoding: "", + Append: false, + Content: `apiVersion: credentialprovider.d2iq.com/v1alpha1 +kind: DynamicCredentialProviderConfig +credentialProviderPluginBinDir: /etc/kubernetes/image-credential-provider/ +credentialProviders: + apiVersion: kubelet.config.k8s.io/v1beta1 + kind: CredentialProviderConfig + providers: + - name: static-credential-provider + args: + - /etc/kubernetes/static-image-config.json + matchImages: + - "registry-1.docker.io" + - "docker.io" + defaultCacheDuration: "0s" + apiVersion: credentialprovider.kubelet.k8s.io/v1beta1 +`, + }, + }, + { + name: "image registry with static config", + credentials: providerConfig{ + URL: "https://myregistry.com", + }, + wantErr: ErrCredentialsNotFound, + }, } for idx := range tests { tt := tests[idx] diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_insall_files.go b/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_insall_files.go index 8ec46f5b7..e3bc725d2 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_insall_files.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_insall_files.go @@ -5,6 +5,7 @@ package credentials import ( "bytes" + _ "embed" "fmt" "text/template" @@ -16,6 +17,17 @@ const ( installKubeletCredentialProvidersScriptOnRemote = "/etc/konvoy/install-kubelet-credential-providers.sh" installKubeletCredentialProvidersScriptOnRemoteCommand = "/bin/bash " + installKubeletCredentialProvidersScriptOnRemote + + //nolint:gosec // Does not contain hard coded config. + dynamicCredentialProviderImage = "ghcr.io/mesosphere/dynamic-credential-provider:v0.2.0" + + //nolint:gosec // Does not contain hard coded config. + credentialProviderTargetDir = "/etc/kubernetes/image-credential-provider/" +) + +var ( + //go:embed templates/install-kubelet-credential-providers.sh.gotmpl + installKubeletCredentialProvidersScript []byte ) func templateFilesAndCommandsForInstallKubeletCredentialProviders() ([]cabpkv1.File, []string, error) { diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/credentials_secret.go b/pkg/handlers/generic/mutation/imageregistries/credentials/credentials_secret.go index 3c5c4878a..e65220c5c 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/credentials_secret.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/credentials_secret.go @@ -5,6 +5,7 @@ package credentials import ( "bytes" + _ "embed" "fmt" "net/url" "strings" @@ -22,6 +23,11 @@ const ( secretKeyForStaticCredentialProviderConfig = "static-credential-provider" ) +var ( + //go:embed templates/static-credential-provider.json.gotmpl + staticCredentialProviderConfigPatch []byte +) + func generateCredentialsSecretFile(config providerConfig, ownerName string) []cabpkv1.File { if config.isCredentialsEmpty() { return nil diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/inject_test.go b/pkg/handlers/generic/mutation/imageregistries/credentials/inject_test.go index 8d7e61e2c..ef0760fec 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/inject_test.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/inject_test.go @@ -168,6 +168,21 @@ func TestGeneratePatches(t *testing.T) { }, }, }, + capitest.PatchTestDef{ + Name: "error for a registry with no credentials", + Vars: []runtimehooksv1.Variable{ + capitest.VariableWithValue( + variableName, + v1alpha1.ImageRegistryCredentials{ + v1alpha1.ImageRegistryCredentialsResource{ + URL: "https://my-registry.io", + }, + }, + ), + }, + RequestItem: request.NewKubeadmControlPlaneTemplateRequestItem(""), + ExpectedFailure: true, + }, ) } From bb851a4fac4ac05eea542833ddf2518497e05937 Mon Sep 17 00:00:00 2001 From: Dimitri Koshkin Date: Wed, 27 Sep 2023 14:28:49 -0700 Subject: [PATCH 08/11] fix: revert incorrect variable changes --- .../credential_provider_config_files.go | 14 +++++++------- .../credential_provider_config_files_test.go | 16 ++++++++-------- .../credential_provider_insall_files.go | 6 +++--- .../imageregistries/credentials/inject.go | 2 +- .../imageregistries/credentials/inject_test.go | 2 +- .../imageregistries/credentials/variables.go | 2 +- 6 files changed, 21 insertions(+), 21 deletions(-) diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files.go b/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files.go index ab90f6d1e..0ede07f34 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files.go @@ -20,13 +20,13 @@ import ( ) const ( - //nolint:gosec // Does not contain hard coded config. - kubeletStaticCredentialProviderCredentialsOnRemote = "/etc/kubernetes/static-image-config.json" + //nolint:gosec // Does not contain hard coded credentials. + kubeletStaticCredentialProviderCredentialsOnRemote = "/etc/kubernetes/static-image-credentials.json" - //nolint:gosec // Does not contain hard coded config. + //nolint:gosec // Does not contain hard coded credentials. kubeletImageCredentialProviderConfigOnRemote = "/etc/kubernetes/image-credential-provider-config.yaml" - //nolint:gosec // Does not contain hard coded config. + //nolint:gosec // Does not contain hard coded credentials. kubeletDynamicCredentialProviderConfigOnRemote = "/etc/kubernetes/dynamic-credential-provider-config.yaml" azureCloudConfigFilePath = "/etc/kubernetes/azure.json" @@ -153,7 +153,7 @@ func kubeletCredentialProvider() ( providerBinary string, providerArgs []string, providerAPIVersion string, err error, ) { return "dynamic-credential-provider", - []string{"get-config", "-c", kubeletDynamicCredentialProviderConfigOnRemote}, + []string{"get-credentials", "-c", kubeletDynamicCredentialProviderConfigOnRemote}, credentialproviderv1beta1.SchemeGroupVersion.String(), nil } @@ -162,12 +162,12 @@ func dynamicCredentialProvider(host string) ( providerBinary string, providerArgs []string, providerAPIVersion string, err error, ) { if matches, err := credentialprovider.URLMatchesECR(host); matches || err != nil { - return "ecr-credential-provider", []string{"get-config"}, + return "ecr-credential-provider", []string{"get-credentials"}, credentialproviderv1alpha1.SchemeGroupVersion.String(), err } if matches, err := credentialprovider.URLMatchesGCR(host); matches || err != nil { - return "gcr-credential-provider", []string{"get-config"}, + return "gcr-credential-provider", []string{"get-credentials"}, credentialproviderv1alpha1.SchemeGroupVersion.String(), err } diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files_test.go b/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files_test.go index 1540a08cd..e8b9d5dd0 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files_test.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files_test.go @@ -32,7 +32,7 @@ kind: CredentialProviderConfig providers: - name: dynamic-credential-provider args: - - get-config + - get-credentials - -c - /etc/kubernetes/dynamic-credential-provider-config.yaml matchImages: @@ -65,7 +65,7 @@ kind: CredentialProviderConfig providers: - name: dynamic-credential-provider args: - - get-config + - get-credentials - -c - /etc/kubernetes/dynamic-credential-provider-config.yaml matchImages: @@ -117,7 +117,7 @@ credentialProviders: providers: - name: ecr-credential-provider args: - - get-config + - get-credentials matchImages: - "123456789.dkr.ecr.us-east-1.amazonaws.com" defaultCacheDuration: "0s" @@ -126,7 +126,7 @@ credentialProviders: }, }, { - name: "image registry with static config", + name: "image registry with static credentials", credentials: providerConfig{ URL: "https://myregistry.com", Username: "myuser", @@ -147,7 +147,7 @@ credentialProviders: providers: - name: static-credential-provider args: - - /etc/kubernetes/static-image-config.json + - /etc/kubernetes/static-image-credentials.json matchImages: - "myregistry.com" defaultCacheDuration: "0s" @@ -156,7 +156,7 @@ credentialProviders: }, }, { - name: "docker.io registry with static config", + name: "docker.io registry with static credentials", credentials: providerConfig{ URL: "https://registry-1.docker.io", Username: "myuser", @@ -177,7 +177,7 @@ credentialProviders: providers: - name: static-credential-provider args: - - /etc/kubernetes/static-image-config.json + - /etc/kubernetes/static-image-credentials.json matchImages: - "registry-1.docker.io" - "docker.io" @@ -187,7 +187,7 @@ credentialProviders: }, }, { - name: "image registry with static config", + name: "error for a registry with no credentials", credentials: providerConfig{ URL: "https://myregistry.com", }, diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_insall_files.go b/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_insall_files.go index e3bc725d2..56b50e463 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_insall_files.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_insall_files.go @@ -13,15 +13,15 @@ import ( ) const ( - //nolint:gosec // Does not contain hard coded config. + //nolint:gosec // Does not contain hard coded credentials. installKubeletCredentialProvidersScriptOnRemote = "/etc/konvoy/install-kubelet-credential-providers.sh" installKubeletCredentialProvidersScriptOnRemoteCommand = "/bin/bash " + installKubeletCredentialProvidersScriptOnRemote - //nolint:gosec // Does not contain hard coded config. + //nolint:gosec // Does not contain hard coded credentials. dynamicCredentialProviderImage = "ghcr.io/mesosphere/dynamic-credential-provider:v0.2.0" - //nolint:gosec // Does not contain hard coded config. + //nolint:gosec // Does not contain hard coded credentials. credentialProviderTargetDir = "/etc/kubernetes/image-credential-provider/" ) diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/inject.go b/pkg/handlers/generic/mutation/imageregistries/credentials/inject.go index 3addf39bf..5d80655ca 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/inject.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/inject.go @@ -287,7 +287,7 @@ func createSecretIfNeeded( ) error { credentialsSecret, err := generateCredentialsSecret(registryWithOptionalCredentials, clusterKey.Name, obj.GetName(), obj.GetNamespace()) if err != nil { - return fmt.Errorf("error generating config Secret for Image Registry Credentials variable: %w", err) + return fmt.Errorf("error generating crdentials Secret for Image Registry Credentials variable: %w", err) } if credentialsSecret != nil { if err := client.ServerSideApply(ctx, c, credentialsSecret); err != nil { diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/inject_test.go b/pkg/handlers/generic/mutation/imageregistries/credentials/inject_test.go index ef0760fec..af5b50a19 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/inject_test.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/inject_test.go @@ -21,7 +21,7 @@ import ( ) const ( - validSecretName = "myregistry-config" + validSecretName = "myregistry-credentials" ) func TestGeneratePatches(t *testing.T) { diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/variables.go b/pkg/handlers/generic/mutation/imageregistries/credentials/variables.go index 73d298cfd..e7eea0331 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/variables.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/variables.go @@ -21,7 +21,7 @@ var ( const ( // variableName is the external patch variable name. - variableName = "config" + variableName = "credentials" // HandlerNameVariable is the name of the variable handler. HandlerNameVariable = "ImageRegistryCredentialsVars" From cb229e7758fc4afba1a02d2e2b2d276559719ef3 Mon Sep 17 00:00:00 2001 From: Dimitri Koshkin Date: Wed, 27 Sep 2023 15:37:12 -0700 Subject: [PATCH 09/11] fix: set kubeletExtraArgs --- .../pkg/testutils/capitest/request/items.go | 17 +++++++ .../credential_provider_kubelet_args.go | 11 +++++ .../imageregistries/credentials/inject.go | 31 ++++++++++++ .../credentials/inject_test.go | 48 +++++++++++++++++++ 4 files changed, 107 insertions(+) create mode 100644 pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_kubelet_args.go diff --git a/common/pkg/testutils/capitest/request/items.go b/common/pkg/testutils/capitest/request/items.go index a8619564b..0d191c07e 100644 --- a/common/pkg/testutils/capitest/request/items.go +++ b/common/pkg/testutils/capitest/request/items.go @@ -58,6 +58,9 @@ func NewKubeadmConfigTemplateRequestItem(uid types.UID) runtimehooksv1.GenerateP Template: bootstrapv1.KubeadmConfigTemplateResource{ Spec: bootstrapv1.KubeadmConfigSpec{ PostKubeadmCommands: []string{"initial-post-kubeadm"}, + JoinConfiguration: &bootstrapv1.JoinConfiguration{ + NodeRegistration: bootstrapv1.NodeRegistrationOptions{}, + }, }, }, }, @@ -83,6 +86,20 @@ func NewKubeadmControlPlaneTemplateRequestItem( Name: KubeadmControlPlaneTemplateRequestObjectName, Namespace: Namespace, }, + Spec: controlplanev1.KubeadmControlPlaneTemplateSpec{ + Template: controlplanev1.KubeadmControlPlaneTemplateResource{ + Spec: controlplanev1.KubeadmControlPlaneTemplateResourceSpec{ + KubeadmConfigSpec: bootstrapv1.KubeadmConfigSpec{ + InitConfiguration: &bootstrapv1.InitConfiguration{ + NodeRegistration: bootstrapv1.NodeRegistrationOptions{}, + }, + JoinConfiguration: &bootstrapv1.JoinConfiguration{ + NodeRegistration: bootstrapv1.NodeRegistrationOptions{}, + }, + }, + }, + }, + }, }, &runtimehooksv1.HolderReference{ Kind: "Cluster", diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_kubelet_args.go b/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_kubelet_args.go new file mode 100644 index 000000000..1c64694f1 --- /dev/null +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_kubelet_args.go @@ -0,0 +1,11 @@ +// Copyright 2023 D2iQ, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package credentials + +const () + +func addImageCredentialProviderArgs(args map[string]string) { + args["image-credential-provider-bin-dir"] = credentialProviderTargetDir + args["image-credential-provider-config"] = kubeletImageCredentialProviderConfigOnRemote +} diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/inject.go b/pkg/handlers/generic/mutation/imageregistries/credentials/inject.go index 5d80655ca..da84cd7b9 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/inject.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/inject.go @@ -161,6 +161,27 @@ func (h *imageRegistriesPatchHandler) Mutate( return generateErr } + initConfiguration := obj.Spec.Template.Spec.KubeadmConfigSpec.InitConfiguration + if initConfiguration == nil { + initConfiguration = &cabpkv1.InitConfiguration{} + } + obj.Spec.Template.Spec.KubeadmConfigSpec.InitConfiguration = initConfiguration + if initConfiguration.NodeRegistration.KubeletExtraArgs == nil { + initConfiguration.NodeRegistration.KubeletExtraArgs = map[string]string{} + } + addImageCredentialProviderArgs(initConfiguration.NodeRegistration.KubeletExtraArgs) + + joinConfiguration := obj.Spec.Template.Spec.KubeadmConfigSpec.JoinConfiguration + if joinConfiguration == nil { + joinConfiguration = &cabpkv1.JoinConfiguration{} + } + obj.Spec.Template.Spec.KubeadmConfigSpec.JoinConfiguration = joinConfiguration + if joinConfiguration.NodeRegistration.KubeletExtraArgs == nil { + joinConfiguration.NodeRegistration.KubeletExtraArgs = map[string]string{} + } + addImageCredentialProviderArgs(joinConfiguration.NodeRegistration.KubeletExtraArgs) + + fmt.Printf("%v", joinConfiguration) return nil }); err != nil { return err @@ -196,6 +217,16 @@ func (h *imageRegistriesPatchHandler) Mutate( return generateErr } + joinConfiguration := obj.Spec.Template.Spec.JoinConfiguration + if joinConfiguration == nil { + joinConfiguration = &cabpkv1.JoinConfiguration{} + } + obj.Spec.Template.Spec.JoinConfiguration = joinConfiguration + if joinConfiguration.NodeRegistration.KubeletExtraArgs == nil { + joinConfiguration.NodeRegistration.KubeletExtraArgs = map[string]string{} + } + addImageCredentialProviderArgs(joinConfiguration.NodeRegistration.KubeletExtraArgs) + return nil }); err != nil { return err diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/inject_test.go b/pkg/handlers/generic/mutation/imageregistries/credentials/inject_test.go index af5b50a19..48ed5c028 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/inject_test.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/inject_test.go @@ -66,6 +66,22 @@ func TestGeneratePatches(t *testing.T) { Path: "/spec/template/spec/kubeadmConfigSpec/preKubeadmCommands", ValueMatcher: gomega.HaveLen(1), }, + { + Operation: "add", + Path: "/spec/template/spec/kubeadmConfigSpec/initConfiguration/nodeRegistration/kubeletExtraArgs", + ValueMatcher: gomega.HaveKeyWithValue( + "image-credential-provider-bin-dir", + "/etc/kubernetes/image-credential-provider/", + ), + }, + { + Operation: "add", + Path: "/spec/template/spec/kubeadmConfigSpec/joinConfiguration/nodeRegistration/kubeletExtraArgs", + ValueMatcher: gomega.HaveKeyWithValue( + "image-credential-provider-config", + "/etc/kubernetes/image-credential-provider-config.yaml", + ), + }, }, }, capitest.PatchTestDef{ @@ -95,6 +111,22 @@ func TestGeneratePatches(t *testing.T) { Path: "/spec/template/spec/kubeadmConfigSpec/preKubeadmCommands", ValueMatcher: gomega.HaveLen(1), }, + { + Operation: "add", + Path: "/spec/template/spec/kubeadmConfigSpec/initConfiguration/nodeRegistration/kubeletExtraArgs", + ValueMatcher: gomega.HaveKeyWithValue( + "image-credential-provider-bin-dir", + "/etc/kubernetes/image-credential-provider/", + ), + }, + { + Operation: "add", + Path: "/spec/template/spec/kubeadmConfigSpec/joinConfiguration/nodeRegistration/kubeletExtraArgs", + ValueMatcher: gomega.HaveKeyWithValue( + "image-credential-provider-config", + "/etc/kubernetes/image-credential-provider-config.yaml", + ), + }, }, }, capitest.PatchTestDef{ @@ -129,6 +161,14 @@ func TestGeneratePatches(t *testing.T) { Path: "/spec/template/spec/preKubeadmCommands", ValueMatcher: gomega.HaveLen(1), }, + { + Operation: "add", + Path: "/spec/template/spec/joinConfiguration/nodeRegistration/kubeletExtraArgs", + ValueMatcher: gomega.HaveKeyWithValue( + "image-credential-provider-bin-dir", + "/etc/kubernetes/image-credential-provider/", + ), + }, }, }, capitest.PatchTestDef{ @@ -166,6 +206,14 @@ func TestGeneratePatches(t *testing.T) { Path: "/spec/template/spec/preKubeadmCommands", ValueMatcher: gomega.HaveLen(1), }, + { + Operation: "add", + Path: "/spec/template/spec/joinConfiguration/nodeRegistration/kubeletExtraArgs", + ValueMatcher: gomega.HaveKeyWithValue( + "image-credential-provider-bin-dir", + "/etc/kubernetes/image-credential-provider/", + ), + }, }, }, capitest.PatchTestDef{ From ff58ab5adb7ed19805d7ab47d6c519aac37f5f3d Mon Sep 17 00:00:00 2001 From: Jimmi Dyson Date: Thu, 28 Sep 2023 13:05:16 +0100 Subject: [PATCH 10/11] refactor: Embed variable schema --- api/v1alpha1/clusterconfig_types.go | 80 +++++++++---------- api/v1alpha1/zz_generated.deepcopy.go | 2 +- common/pkg/testutils/capitest/patches.go | 3 +- .../credential_provider_config_files.go | 28 +++---- .../credential_provider_config_files_test.go | 5 +- .../credential_provider_insall_files.go | 15 ++-- .../credential_provider_kubelet_args.go | 2 - .../credentials/credentials_secret.go | 15 ++-- .../imageregistries/credentials/inject.go | 63 ++++++++++----- .../credentials/inject_test.go | 18 +++-- 10 files changed, 124 insertions(+), 107 deletions(-) diff --git a/api/v1alpha1/clusterconfig_types.go b/api/v1alpha1/clusterconfig_types.go index a21686680..acf923fe1 100644 --- a/api/v1alpha1/clusterconfig_types.go +++ b/api/v1alpha1/clusterconfig_types.go @@ -191,56 +191,25 @@ func (ImageRegistries) VariableSchema() clusterv1.VariableSchema { Description: "Configuration for image registries.", Type: "object", Properties: map[string]clusterv1.JSONSchemaProps{ - "credentials": imageRegistryCredentialsSchema, + "credentials": ImageRegistryCredentials{}.VariableSchema().OpenAPIV3Schema, }, }, } } -var ( - imageRegistryCredentialsSchema = clusterv1.JSONSchemaProps{ - Type: "array", - UniqueItems: true, - Items: &imageRegistryCredentialsResourceSchema, - } - - imageRegistryCredentialsResourceSchema = clusterv1.JSONSchemaProps{ - Description: "Image registry credentials to set up on all Nodes in the cluster. " + - "Enabling this will the Kubelets with https://kubernetes.io/docs/tasks/administer-cluster/kubelet-credential-provider/.", - Type: "object", - Properties: map[string]clusterv1.JSONSchemaProps{ - "url": { - Description: "Registry URL.", - Type: "string", - }, - "secretRef": { - Description: "The Secret containing the registry credentials. " + - "The Secret should have keys 'username' and 'password'. " + - "This credentials Secret is not required for some registries, e.g. ECR.", - Type: "object", - Properties: map[string]clusterv1.JSONSchemaProps{ - "name": { - Description: "The name of the Secret containing the registry credentials.", - Type: "string", - }, - "namespace": { - Description: "The namespace of the Secret containing the registry credentials. " + - "Defaults to the namespace of the KubeadmControlPlaneTemplate and KubeadmConfigTemplate" + - " that reference this variable.", - Type: "string", - }, - }, - }, - }, - Required: []string{"url"}, - } -) - type ImageRegistryCredentials []ImageRegistryCredentialsResource func (ImageRegistryCredentials) VariableSchema() clusterv1.VariableSchema { + resourceSchema := ImageRegistryCredentialsResource{}.VariableSchema().OpenAPIV3Schema + return clusterv1.VariableSchema{ - OpenAPIV3Schema: imageRegistryCredentialsSchema, + OpenAPIV3Schema: clusterv1.JSONSchemaProps{ + Description: "Image registry credentials to set up on all Nodes in the cluster. " + + "Enabling this will configure the Kubelets with " + + "https://kubernetes.io/docs/tasks/administer-cluster/kubelet-credential-provider/.", + Type: "array", + Items: &resourceSchema, + }, } } @@ -258,7 +227,34 @@ type ImageRegistryCredentialsResource struct { func (ImageRegistryCredentialsResource) VariableSchema() clusterv1.VariableSchema { return clusterv1.VariableSchema{ - OpenAPIV3Schema: imageRegistryCredentialsResourceSchema, + OpenAPIV3Schema: clusterv1.JSONSchemaProps{ + Type: "object", + Properties: map[string]clusterv1.JSONSchemaProps{ + "url": { + Description: "Registry URL.", + Type: "string", + }, + "secretRef": { + Description: "The Secret containing the registry credentials. " + + "The Secret should have keys 'username' and 'password'. " + + "This credentials Secret is not required for some registries, e.g. ECR.", + Type: "object", + Properties: map[string]clusterv1.JSONSchemaProps{ + "name": { + Description: "The name of the Secret containing the registry credentials.", + Type: "string", + }, + "namespace": { + Description: "The namespace of the Secret containing the registry credentials. " + + "Defaults to the namespace of the KubeadmControlPlaneTemplate and KubeadmConfigTemplate" + + " that reference this variable.", + Type: "string", + }, + }, + }, + }, + Required: []string{"url"}, + }, } } diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 0aacfa639..360d2c64b 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -8,7 +8,7 @@ package v1alpha1 import ( - v1 "k8s.io/api/core/v1" + "k8s.io/api/core/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) diff --git a/common/pkg/testutils/capitest/patches.go b/common/pkg/testutils/capitest/patches.go index b24416550..1cc117192 100644 --- a/common/pkg/testutils/capitest/patches.go +++ b/common/pkg/testutils/capitest/patches.go @@ -61,7 +61,8 @@ func ValidateGeneratePatches[T mutation.GeneratePatches]( if tt.ExpectedFailure { expectedStatus = runtimehooksv1.ResponseStatusFailure } - g.Expect(resp.Status).To(gomega.Equal(expectedStatus), fmt.Sprintf("Message: %s", resp.Message)) + g.Expect(resp.Status). + To(gomega.Equal(expectedStatus), fmt.Sprintf("Message: %s", resp.Message)) if len(tt.ExpectedPatchMatchers) == 0 { g.Expect(resp.Items).To(gomega.BeEmpty()) diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files.go b/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files.go index 0ede07f34..32aa753f2 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files.go @@ -40,9 +40,7 @@ var ( kubeletImageCredentialProviderConfigPatch []byte ) -var ( - ErrCredentialsNotFound = errors.New("registry credentials not found") -) +var ErrCredentialsNotFound = errors.New("registry credentials not found") type providerConfig struct { URL string @@ -66,7 +64,9 @@ func templateFilesForImageCredentialProviderConfigs(config providerConfig) ([]ca files = append(files, *kubeletCredentialProviderConfigFile) } - kubeletDynamicCredentialProviderConfigFile, err := templateDynamicCredentialProviderConfig(config) + kubeletDynamicCredentialProviderConfigFile, err := templateDynamicCredentialProviderConfig( + config, + ) if err != nil { return nil, err } @@ -84,10 +84,7 @@ func templateKubeletCredentialProviderConfig() (*cabpkv1.File, error) { return nil, fmt.Errorf("failed to parse go template: %w", err) } - providerBinary, providerArgs, providerAPIVersion, err := kubeletCredentialProvider() - if err != nil { - return nil, err - } + providerBinary, providerArgs, providerAPIVersion := kubeletCredentialProvider() templateInput := struct { ProviderBinary string @@ -129,7 +126,9 @@ func templateDynamicCredentialProviderConfig( return nil, ErrCredentialsNotFound } - providerBinary, providerArgs, providerAPIVersion, err := dynamicCredentialProvider(registryHostWithPath) + providerBinary, providerArgs, providerAPIVersion, err := dynamicCredentialProvider( + registryHostWithPath, + ) if err != nil { return nil, err } @@ -149,13 +148,10 @@ func templateDynamicCredentialProviderConfig( return fileFromTemplate(t, templateInput, kubeletDynamicCredentialProviderConfigOnRemote) } -func kubeletCredentialProvider() ( - providerBinary string, providerArgs []string, providerAPIVersion string, err error, -) { +func kubeletCredentialProvider() (providerBinary string, providerArgs []string, providerAPIVersion string) { return "dynamic-credential-provider", []string{"get-credentials", "-c", kubeletDynamicCredentialProviderConfigOnRemote}, - credentialproviderv1beta1.SchemeGroupVersion.String(), - nil + credentialproviderv1beta1.SchemeGroupVersion.String() } func dynamicCredentialProvider(host string) ( @@ -187,7 +183,7 @@ func dynamicCredentialProvider(host string) ( func fileFromTemplate( t *template.Template, templateInput any, - path string, + fPath string, ) (*cabpkv1.File, error) { var b bytes.Buffer err := t.Execute(&b, templateInput) @@ -196,7 +192,7 @@ func fileFromTemplate( } return &cabpkv1.File{ - Path: path, + Path: fPath, Content: b.String(), Permissions: "0600", }, nil diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files_test.go b/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files_test.go index e8b9d5dd0..b678b8390 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files_test.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files_test.go @@ -7,11 +7,12 @@ import ( "testing" "github.com/stretchr/testify/assert" - cabpkv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" ) func Test_templateKubeletCredentialProviderConfig(t *testing.T) { + t.Parallel() + tests := []struct { name string config providerConfig @@ -93,6 +94,8 @@ providers: } func Test_templateDynamicCredentialProviderConfig(t *testing.T) { + t.Parallel() + tests := []struct { name string credentials providerConfig diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_insall_files.go b/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_insall_files.go index 56b50e463..9109110e0 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_insall_files.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_insall_files.go @@ -25,23 +25,20 @@ const ( credentialProviderTargetDir = "/etc/kubernetes/image-credential-provider/" ) -var ( - //go:embed templates/install-kubelet-credential-providers.sh.gotmpl - installKubeletCredentialProvidersScript []byte -) +//go:embed templates/install-kubelet-credential-providers.sh.gotmpl +var installKubeletCredentialProvidersScript []byte func templateFilesAndCommandsForInstallKubeletCredentialProviders() ([]cabpkv1.File, []string, error) { var files []cabpkv1.File var commands []string - installKubeletCredentialProvidersScriptFile, installKubeletCredentialProvidersScriptCommand, err := - templateInstallKubeletCredentialProviders() + installKCPScriptFile, installKCPScriptCommand, err := templateInstallKubeletCredentialProviders() if err != nil { return nil, nil, err } - if installKubeletCredentialProvidersScriptFile != nil { - files = append(files, *installKubeletCredentialProvidersScriptFile) - commands = append(commands, installKubeletCredentialProvidersScriptCommand) + if installKCPScriptFile != nil { + files = append(files, *installKCPScriptFile) + commands = append(commands, installKCPScriptCommand) } return files, commands, nil diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_kubelet_args.go b/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_kubelet_args.go index 1c64694f1..1053881c5 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_kubelet_args.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_kubelet_args.go @@ -3,8 +3,6 @@ package credentials -const () - func addImageCredentialProviderArgs(args map[string]string) { args["image-credential-provider-bin-dir"] = credentialProviderTargetDir args["image-credential-provider-config"] = kubeletImageCredentialProviderConfigOnRemote diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/credentials_secret.go b/pkg/handlers/generic/mutation/imageregistries/credentials/credentials_secret.go index e65220c5c..be7a3f566 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/credentials_secret.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/credentials_secret.go @@ -11,22 +11,19 @@ import ( "strings" "text/template" - cabpkv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" - corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + cabpkv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" clusterctlv1 "sigs.k8s.io/cluster-api/cmd/clusterctl/api/v1alpha3" ) const ( - secretKeyForStaticCredentialProviderConfig = "static-credential-provider" + secretKeyForStaticCredentialProviderConfig = "static-credential-provider" //nolint:gosec // Not a credential. ) -var ( - //go:embed templates/static-credential-provider.json.gotmpl - staticCredentialProviderConfigPatch []byte -) +//go:embed templates/static-credential-provider.json.gotmpl +var staticCredentialProviderConfigPatch []byte func generateCredentialsSecretFile(config providerConfig, ownerName string) []cabpkv1.File { if config.isCredentialsEmpty() { @@ -55,7 +52,9 @@ func generateCredentialsSecret( return nil, nil } - staticCredentialProviderSecretContents, err := kubeletStaticCredentialProviderSecretContents(config) + staticCredentialProviderSecretContents, err := kubeletStaticCredentialProviderSecretContents( + config, + ) if err != nil { return nil, err } diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/inject.go b/pkg/handlers/generic/mutation/imageregistries/credentials/inject.go index da84cd7b9..47d6e8be0 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/inject.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/inject.go @@ -13,7 +13,6 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer" bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" - cabpkv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1" runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1" "sigs.k8s.io/cluster-api/exp/runtime/topologymutation" @@ -59,7 +58,12 @@ func NewPatch( func NewMetaPatch( cl ctrlclient.Client, ) *imageRegistriesPatchHandler { - return newImageRegistriesPatchHandler(cl, clusterconfig.MetaVariableName, imageregistries.VariableName, variableName) + return newImageRegistriesPatchHandler( + cl, + clusterconfig.MetaVariableName, + imageregistries.VariableName, + variableName, + ) } func newImageRegistriesPatchHandler( @@ -125,11 +129,12 @@ func (h *imageRegistriesPatchHandler) Mutate( credentials, ) - if err = patches.Generate( + if err := patches.Generate( obj, vars, &holderRef, selectors.ControlPlane(), log, func(obj *controlplanev1.KubeadmControlPlaneTemplate) error { - registryWithOptionalCredentials, generateErr := - registryWithOptionalCredentialsFromImageRegistryCredentials(ctx, h.client, credentials, obj) + registryWithOptionalCredentials, generateErr := registryWithOptionalCredentialsFromImageRegistryCredentials( + ctx, h.client, credentials, obj, + ) if generateErr != nil { return generateErr } @@ -163,7 +168,7 @@ func (h *imageRegistriesPatchHandler) Mutate( initConfiguration := obj.Spec.Template.Spec.KubeadmConfigSpec.InitConfiguration if initConfiguration == nil { - initConfiguration = &cabpkv1.InitConfiguration{} + initConfiguration = &bootstrapv1.InitConfiguration{} } obj.Spec.Template.Spec.KubeadmConfigSpec.InitConfiguration = initConfiguration if initConfiguration.NodeRegistration.KubeletExtraArgs == nil { @@ -173,25 +178,24 @@ func (h *imageRegistriesPatchHandler) Mutate( joinConfiguration := obj.Spec.Template.Spec.KubeadmConfigSpec.JoinConfiguration if joinConfiguration == nil { - joinConfiguration = &cabpkv1.JoinConfiguration{} + joinConfiguration = &bootstrapv1.JoinConfiguration{} } obj.Spec.Template.Spec.KubeadmConfigSpec.JoinConfiguration = joinConfiguration if joinConfiguration.NodeRegistration.KubeletExtraArgs == nil { joinConfiguration.NodeRegistration.KubeletExtraArgs = map[string]string{} } addImageCredentialProviderArgs(joinConfiguration.NodeRegistration.KubeletExtraArgs) - - fmt.Printf("%v", joinConfiguration) return nil }); err != nil { return err } - if err = patches.Generate( + if err := patches.Generate( obj, vars, &holderRef, selectors.WorkersKubeadmConfigTemplateSelector(), log, func(obj *bootstrapv1.KubeadmConfigTemplate) error { - registryWithOptionalCredentials, generateErr := - registryWithOptionalCredentialsFromImageRegistryCredentials(ctx, h.client, credentials, obj) + registryWithOptionalCredentials, generateErr := registryWithOptionalCredentialsFromImageRegistryCredentials( + ctx, h.client, credentials, obj, + ) if generateErr != nil { return generateErr } @@ -219,7 +223,7 @@ func (h *imageRegistriesPatchHandler) Mutate( joinConfiguration := obj.Spec.Template.Spec.JoinConfiguration if joinConfiguration == nil { - joinConfiguration = &cabpkv1.JoinConfiguration{} + joinConfiguration = &bootstrapv1.JoinConfiguration{} } obj.Spec.Template.Spec.JoinConfiguration = joinConfiguration if joinConfiguration.NodeRegistration.KubeletExtraArgs == nil { @@ -293,18 +297,27 @@ func registryWithOptionalCredentialsFromImageRegistryCredentials( func generateFilesAndCommands( registryWithOptionalCredentials providerConfig, objName string, -) ([]cabpkv1.File, []string, error) { - +) ([]bootstrapv1.File, []string, error) { files, commands, err := templateFilesAndCommandsForInstallKubeletCredentialProviders() if err != nil { - return nil, nil, fmt.Errorf("error generating insall files and commands for Image Registry Credentials variable: %w", err) + return nil, nil, fmt.Errorf( + "error generating insall files and commands for Image Registry Credentials variable: %w", + err, + ) } - imageCredentialProviderConfigFiles, err := templateFilesForImageCredentialProviderConfigs(registryWithOptionalCredentials) + imageCredentialProviderConfigFiles, err := templateFilesForImageCredentialProviderConfigs( + registryWithOptionalCredentials, + ) if err != nil { - return nil, nil, fmt.Errorf("error generating files for Image Registry Credentials variable: %w", err) + return nil, nil, fmt.Errorf( + "error generating files for Image Registry Credentials variable: %w", + err, + ) } files = append(files, imageCredentialProviderConfigFiles...) - files = append(files, generateCredentialsSecretFile(registryWithOptionalCredentials, objName)...) + files = append( + files, + generateCredentialsSecretFile(registryWithOptionalCredentials, objName)...) return files, commands, err } @@ -316,9 +329,17 @@ func createSecretIfNeeded( obj ctrlclient.Object, clusterKey ctrlclient.ObjectKey, ) error { - credentialsSecret, err := generateCredentialsSecret(registryWithOptionalCredentials, clusterKey.Name, obj.GetName(), obj.GetNamespace()) + credentialsSecret, err := generateCredentialsSecret( + registryWithOptionalCredentials, + clusterKey.Name, + obj.GetName(), + obj.GetNamespace(), + ) if err != nil { - return fmt.Errorf("error generating crdentials Secret for Image Registry Credentials variable: %w", err) + return fmt.Errorf( + "error generating crdentials Secret for Image Registry Credentials variable: %w", + err, + ) } if credentialsSecret != nil { if err := client.ServerSideApply(ctx, c, credentialsSecret); err != nil { diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/inject_test.go b/pkg/handlers/generic/mutation/imageregistries/credentials/inject_test.go index 48ed5c028..214753b88 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/inject_test.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/inject_test.go @@ -6,12 +6,10 @@ package credentials import ( "testing" - "k8s.io/apiserver/pkg/storage/names" - + "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "github.com/onsi/gomega" + "k8s.io/apiserver/pkg/storage/names" runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1" "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -33,9 +31,17 @@ func TestGeneratePatches(t *testing.T) { fakeClient := fake.NewClientBuilder().WithObjects( newTestSecret(validSecretName, request.Namespace), newEmptySecret( - credentialSecretName(request.KubeadmControlPlaneTemplateRequestObjectName), request.Namespace), + credentialSecretName( + request.KubeadmControlPlaneTemplateRequestObjectName, + ), + request.Namespace, + ), newEmptySecret( - credentialSecretName(request.KubeadmConfigTemplateRequestObjectName), request.Namespace), + credentialSecretName( + request.KubeadmConfigTemplateRequestObjectName, + ), + request.Namespace, + ), ).Build() return NewPatch(fakeClient) }, From e1880d5dab12c11701c8a9ec5f680cb1ce11a3e0 Mon Sep 17 00:00:00 2001 From: Jimmi Dyson Date: Thu, 28 Sep 2023 15:51:57 +0100 Subject: [PATCH 11/11] build: Fix up RBAC role generation from annotations --- api/v1alpha1/doc.go | 4 ---- charts/capi-runtime-extensions/templates/role.yaml | 3 ++- make/go.mk | 3 +++ .../generic/mutation/imageregistries/credentials/doc.go | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/api/v1alpha1/doc.go b/api/v1alpha1/doc.go index 122e5b179..eda1e43d9 100644 --- a/api/v1alpha1/doc.go +++ b/api/v1alpha1/doc.go @@ -4,8 +4,4 @@ // Package v1alpha1 contains API Schema definitions for the CAPI extensions v1alpha1 API group // +kubebuilder:object:generate=true // +groupName=capiext.labs.d2iq.io -// -//go:generate -command CTRLGEN controller-gen paths="./..." -//go:generate CTRLGEN rbac:headerFile="../../hack/license-header.yaml.txt",roleName=capi-runtime-extensions-manager-role output:rbac:artifacts:config=../../charts/capi-runtime-extensions/templates -//go:generate CTRLGEN object:headerFile="../../hack/license-header.go.txt" output:object:artifacts:config=/dev/null package v1alpha1 diff --git a/charts/capi-runtime-extensions/templates/role.yaml b/charts/capi-runtime-extensions/templates/role.yaml index 8f5b5985c..3afe01406 100644 --- a/charts/capi-runtime-extensions/templates/role.yaml +++ b/charts/capi-runtime-extensions/templates/role.yaml @@ -4,7 +4,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: capi-runtime-extensions-manager-role + name: {{ include "chart.name" . }}-manager-role rules: - apiGroups: - "" @@ -27,6 +27,7 @@ rules: - get - list - patch + - update - watch - apiGroups: - addons.cluster.x-k8s.io diff --git a/make/go.mk b/make/go.mk index bff30e6ab..da07ed71b 100644 --- a/make/go.mk +++ b/make/go.mk @@ -177,6 +177,9 @@ go-fix.%: ; $(info $(M) go fixing $* module) go-generate: ## Runs go generate go-generate: ; $(info $(M) running go generate) go generate -x ./... + controller-gen paths="./..." rbac:headerFile="hack/license-header.yaml.txt",roleName=capi-runtime-extensions-manager-role output:rbac:artifacts:config=charts/capi-runtime-extensions/templates + sed --in-place 's/capi-runtime-extensions-manager-role/{{ include "chart.name" . }}-manager-role/' charts/capi-runtime-extensions/templates/role.yaml + controller-gen paths="./api/..." object:headerFile="hack/license-header.go.txt" output:object:artifacts:config=/dev/null $(MAKE) go-fix .PHONY: go-mod-upgrade diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/doc.go b/pkg/handlers/generic/mutation/imageregistries/credentials/doc.go index bd924ddeb..7bdc5bff0 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/doc.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/doc.go @@ -1,5 +1,5 @@ // Copyright 2023 D2iQ, Inc. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -// +kubebuilder:rbac:groups="",resources=secrets,verbs=watch;list;get;patch +// +kubebuilder:rbac:groups="",resources=secrets,verbs=watch;list;get;patch;create;update package credentials