diff --git a/api/v1alpha1/addon_types.go b/api/v1alpha1/addon_types.go index f2f6317cb..229195881 100644 --- a/api/v1alpha1/addon_types.go +++ b/api/v1alpha1/addon_types.go @@ -27,6 +27,8 @@ const ( ServiceLoadBalancerProviderMetalLB = "MetalLB" + RegistryProviderCNCFDistribution = "CNCF Distribution" + AddonStrategyClusterResourceSet AddonStrategy = "ClusterResourceSet" AddonStrategyHelmAddon AddonStrategy = "HelmAddon" @@ -100,6 +102,9 @@ type GenericAddons struct { // +kubebuilder:validation:Optional ServiceLoadBalancer *ServiceLoadBalancer `json:"serviceLoadBalancer,omitempty"` + + // +kubebuilder:validation:Optional + Registry *RegistryAddon `json:"registry,omitempty"` } type AddonStrategy string @@ -335,3 +340,10 @@ type AddressRange struct { // +kubebuilder:validation:Format=ipv4 End string `json:"end"` } + +type RegistryAddon struct { + // The OCI registry provider to deploy. + // +kubebuilder:default="CNCF Distribution" + // +kubebuilder:validation:Enum="CNCF Distribution" + Provider string `json:"provider"` +} diff --git a/api/v1alpha1/constants.go b/api/v1alpha1/constants.go index 613db6993..36bd05049 100644 --- a/api/v1alpha1/constants.go +++ b/api/v1alpha1/constants.go @@ -28,6 +28,8 @@ const ( ClusterAutoscalerVariableName = "clusterAutoscaler" // ServiceLoadBalancerVariableName is the Service LoadBalancer config patch variable name. ServiceLoadBalancerVariableName = "serviceLoadBalancer" + // RegistryAddonVariableName is the OCI registry config patch variable name. + RegistryAddonVariableName = "registry" // GlobalMirrorVariableName is the global image registry mirror patch variable name. GlobalMirrorVariableName = "globalImageRegistryMirror" diff --git a/api/v1alpha1/crds/caren.nutanix.com_awsclusterconfigs.yaml b/api/v1alpha1/crds/caren.nutanix.com_awsclusterconfigs.yaml index eb90f054c..e3452f9cb 100644 --- a/api/v1alpha1/crds/caren.nutanix.com_awsclusterconfigs.yaml +++ b/api/v1alpha1/crds/caren.nutanix.com_awsclusterconfigs.yaml @@ -234,6 +234,17 @@ spec: - HelmAddon type: string type: object + registry: + properties: + provider: + default: CNCF Distribution + description: The OCI registry provider to deploy. + enum: + - CNCF Distribution + type: string + required: + - provider + type: object serviceLoadBalancer: properties: configuration: diff --git a/api/v1alpha1/crds/caren.nutanix.com_dockerclusterconfigs.yaml b/api/v1alpha1/crds/caren.nutanix.com_dockerclusterconfigs.yaml index d752c324c..af175463a 100644 --- a/api/v1alpha1/crds/caren.nutanix.com_dockerclusterconfigs.yaml +++ b/api/v1alpha1/crds/caren.nutanix.com_dockerclusterconfigs.yaml @@ -243,6 +243,17 @@ spec: - HelmAddon type: string type: object + registry: + properties: + provider: + default: CNCF Distribution + description: The OCI registry provider to deploy. + enum: + - CNCF Distribution + type: string + required: + - provider + type: object serviceLoadBalancer: properties: configuration: diff --git a/api/v1alpha1/crds/caren.nutanix.com_nutanixclusterconfigs.yaml b/api/v1alpha1/crds/caren.nutanix.com_nutanixclusterconfigs.yaml index 6ae6e0386..786a82919 100644 --- a/api/v1alpha1/crds/caren.nutanix.com_nutanixclusterconfigs.yaml +++ b/api/v1alpha1/crds/caren.nutanix.com_nutanixclusterconfigs.yaml @@ -243,6 +243,17 @@ spec: - HelmAddon type: string type: object + registry: + properties: + provider: + default: CNCF Distribution + description: The OCI registry provider to deploy. + enum: + - CNCF Distribution + type: string + required: + - provider + type: object serviceLoadBalancer: properties: configuration: diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index d1876120a..4aa303710 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -1047,6 +1047,11 @@ func (in *GenericAddons) DeepCopyInto(out *GenericAddons) { *out = new(ServiceLoadBalancer) (*in).DeepCopyInto(*out) } + if in.Registry != nil { + in, out := &in.Registry, &out.Registry + *out = new(RegistryAddon) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GenericAddons. @@ -1717,6 +1722,21 @@ func (in *ObjectMeta) DeepCopy() *ObjectMeta { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RegistryAddon) DeepCopyInto(out *RegistryAddon) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RegistryAddon. +func (in *RegistryAddon) DeepCopy() *RegistryAddon { + if in == nil { + return nil + } + out := new(RegistryAddon) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *RegistryCredentials) DeepCopyInto(out *RegistryCredentials) { *out = *in diff --git a/charts/cluster-api-runtime-extensions-nutanix/README.md b/charts/cluster-api-runtime-extensions-nutanix/README.md index 54322ada9..6e2e84e97 100644 --- a/charts/cluster-api-runtime-extensions-nutanix/README.md +++ b/charts/cluster-api-runtime-extensions-nutanix/README.md @@ -85,6 +85,8 @@ A Helm chart for cluster-api-runtime-extensions-nutanix | hooks.nfd.crsStrategy.defaultInstallationConfigMap.name | string | `"node-feature-discovery"` | | | hooks.nfd.helmAddonStrategy.defaultValueTemplateConfigMap.create | bool | `true` | | | hooks.nfd.helmAddonStrategy.defaultValueTemplateConfigMap.name | string | `"default-nfd-helm-values-template"` | | +| hooks.registry.cncfDistribution.defaultValueTemplateConfigMap.create | bool | `true` | | +| hooks.registry.cncfDistribution.defaultValueTemplateConfigMap.name | string | `"default-cncf-distribution-registry-helm-values-template"` | | | hooks.serviceLoadBalancer.metalLB.defaultValueTemplateConfigMap.create | bool | `true` | | | hooks.serviceLoadBalancer.metalLB.defaultValueTemplateConfigMap.name | string | `"default-metallb-helm-values-template"` | | | hooks.virtualIP.kubeVip.defaultTemplateConfigMap.create | bool | `true` | | diff --git a/charts/cluster-api-runtime-extensions-nutanix/addons/registry/cncf-distribution/values-template.yaml b/charts/cluster-api-runtime-extensions-nutanix/addons/registry/cncf-distribution/values-template.yaml new file mode 100644 index 000000000..cb13ba698 --- /dev/null +++ b/charts/cluster-api-runtime-extensions-nutanix/addons/registry/cncf-distribution/values-template.yaml @@ -0,0 +1,12 @@ +replicaCount: 2 +persistence: + enabled: true + size: 50Gi +service: + type: ClusterIP + clusterIP: {{ .ServiceIP }} + port: 80 +statefulSet: + enabled: true + syncer: + interval: 2m diff --git a/charts/cluster-api-runtime-extensions-nutanix/templates/helm-config.yaml b/charts/cluster-api-runtime-extensions-nutanix/templates/helm-config.yaml index 3c5f1670c..77d78c82f 100644 --- a/charts/cluster-api-runtime-extensions-nutanix/templates/helm-config.yaml +++ b/charts/cluster-api-runtime-extensions-nutanix/templates/helm-config.yaml @@ -23,6 +23,10 @@ data: ChartName: cluster-autoscaler ChartVersion: 9.46.3 RepositoryURL: '{{ if .Values.helmRepository.enabled }}oci://helm-repository.{{ .Release.Namespace }}.svc/charts{{ else }}https://kubernetes.github.io/autoscaler{{ end }}' + cncf-distribution-registry: | + ChartName: docker-registry + ChartVersion: 2.3.1 + RepositoryURL: '{{ if .Values.helmRepository.enabled }}oci://helm-repository.{{ .Release.Namespace }}.svc/charts{{ else }}https://mesosphere.github.io/charts/staging/{{ end }}' cosi-controller: | ChartName: cosi ChartVersion: 0.0.1-alpha.5 diff --git a/charts/cluster-api-runtime-extensions-nutanix/templates/registry/distribution/helm-addon-installation.yaml b/charts/cluster-api-runtime-extensions-nutanix/templates/registry/distribution/helm-addon-installation.yaml new file mode 100644 index 000000000..40f1a9943 --- /dev/null +++ b/charts/cluster-api-runtime-extensions-nutanix/templates/registry/distribution/helm-addon-installation.yaml @@ -0,0 +1,12 @@ +# Copyright 2025 Nutanix. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +{{- if .Values.hooks.registry.cncfDistribution.defaultValueTemplateConfigMap.name }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: '{{ .Values.hooks.registry.cncfDistribution.defaultValueTemplateConfigMap.name }}' +data: + values.yaml: |- + {{- .Files.Get "addons/registry/cncf-distribution/values-template.yaml" | nindent 4 }} +{{- end -}} diff --git a/charts/cluster-api-runtime-extensions-nutanix/values.schema.json b/charts/cluster-api-runtime-extensions-nutanix/values.schema.json index 9a97cfddf..bbbce97c8 100644 --- a/charts/cluster-api-runtime-extensions-nutanix/values.schema.json +++ b/charts/cluster-api-runtime-extensions-nutanix/values.schema.json @@ -504,6 +504,27 @@ }, "type": "object" }, + "registry": { + "properties": { + "cncfDistribution": { + "properties": { + "defaultValueTemplateConfigMap": { + "properties": { + "create": { + "type": "boolean" + }, + "name": { + "type": "string" + } + }, + "type": "object" + } + }, + "type": "object" + } + }, + "type": "object" + }, "serviceLoadBalancer": { "properties": { "metalLB": { diff --git a/charts/cluster-api-runtime-extensions-nutanix/values.yaml b/charts/cluster-api-runtime-extensions-nutanix/values.yaml index 3c1c2789e..1d07ae27c 100644 --- a/charts/cluster-api-runtime-extensions-nutanix/values.yaml +++ b/charts/cluster-api-runtime-extensions-nutanix/values.yaml @@ -111,6 +111,11 @@ hooks: defaultValueTemplateConfigMap: create: true name: default-cosi-controller-helm-values-template + registry: + cncfDistribution: + defaultValueTemplateConfigMap: + create: true + name: default-cncf-distribution-registry-helm-values-template helmAddonsConfigMap: default-helm-addons-config diff --git a/docs/content/addons/registry.md b/docs/content/addons/registry.md new file mode 100644 index 000000000..0174a8ac8 --- /dev/null +++ b/docs/content/addons/registry.md @@ -0,0 +1,36 @@ ++++ +title = "Registry" +icon = "fa-solid fa-eye" ++++ + +By leveraging CAPI cluster lifecycle hooks, this handler deploys an OCI [Distribution] registry, +at the `AfterControlPlaneInitialized` phase and configures it as a mirror on the new cluster. +The registry will be deployed as a StatefulSet with a persistent volume claim for storage +and multiple replicas for high availability. +A sidecar container in each Pod running [Regsync] will periodically sync the OCI artifacts across all replicas. + +Deployment of this registry is opt-in via the [provider-specific cluster configuration]({{< ref ".." >}}). + +The hook will use the [Cluster API Add-on Provider for Helm] to deploy the registry resources. + +## Example + +To enable deployment of the registry on a cluster, specify the following values: + +```yaml +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: +spec: + topology: + variables: + - name: clusterConfig + value: + addons: + registry: {} +``` + +[Distribution]: https://github.com/distribution/distribution +[Cluster API Add-on Provider for Helm]: https://github.com/kubernetes-sigs/cluster-api-addon-provider-helm +[Regsync]: https://regclient.org/usage/regsync/ diff --git a/examples/capi-quick-start/docker-cluster-calico-crs.yaml b/examples/capi-quick-start/docker-cluster-calico-crs.yaml index aaad085ae..a871853d4 100644 --- a/examples/capi-quick-start/docker-cluster-calico-crs.yaml +++ b/examples/capi-quick-start/docker-cluster-calico-crs.yaml @@ -41,6 +41,7 @@ spec: strategy: ClusterResourceSet nfd: strategy: ClusterResourceSet + registry: {} serviceLoadBalancer: configuration: addressRanges: diff --git a/examples/capi-quick-start/docker-cluster-calico-helm-addon.yaml b/examples/capi-quick-start/docker-cluster-calico-helm-addon.yaml index 670912f40..a7ee08e8b 100644 --- a/examples/capi-quick-start/docker-cluster-calico-helm-addon.yaml +++ b/examples/capi-quick-start/docker-cluster-calico-helm-addon.yaml @@ -36,6 +36,7 @@ spec: default: {} snapshotController: {} nfd: {} + registry: {} serviceLoadBalancer: configuration: addressRanges: diff --git a/examples/capi-quick-start/docker-cluster-cilium-crs.yaml b/examples/capi-quick-start/docker-cluster-cilium-crs.yaml index a4eccd63c..8a9131ba4 100644 --- a/examples/capi-quick-start/docker-cluster-cilium-crs.yaml +++ b/examples/capi-quick-start/docker-cluster-cilium-crs.yaml @@ -41,6 +41,7 @@ spec: strategy: ClusterResourceSet nfd: strategy: ClusterResourceSet + registry: {} serviceLoadBalancer: configuration: addressRanges: diff --git a/examples/capi-quick-start/docker-cluster-cilium-helm-addon.yaml b/examples/capi-quick-start/docker-cluster-cilium-helm-addon.yaml index 8d9ff171e..9ae8310b4 100644 --- a/examples/capi-quick-start/docker-cluster-cilium-helm-addon.yaml +++ b/examples/capi-quick-start/docker-cluster-cilium-helm-addon.yaml @@ -36,6 +36,7 @@ spec: default: {} snapshotController: {} nfd: {} + registry: {} serviceLoadBalancer: configuration: addressRanges: diff --git a/hack/addons/helm-chart-bundler/repos.yaml b/hack/addons/helm-chart-bundler/repos.yaml index 8dabb881a..b3cd73091 100644 --- a/hack/addons/helm-chart-bundler/repos.yaml +++ b/hack/addons/helm-chart-bundler/repos.yaml @@ -31,6 +31,11 @@ repositories: charts: cosi: - 0.0.1-alpha.5 + docker-registry: + repoURL: https://mesosphere.github.io/charts/staging/ + charts: + docker-registry: + - 2.3.1 local-path-provisioner: repoURL: https://charts.containeroo.ch charts: diff --git a/hack/addons/kustomize/cncf-distribution-registry/kustomization.yaml.tmpl b/hack/addons/kustomize/cncf-distribution-registry/kustomization.yaml.tmpl new file mode 100644 index 000000000..87ea194ce --- /dev/null +++ b/hack/addons/kustomize/cncf-distribution-registry/kustomization.yaml.tmpl @@ -0,0 +1,27 @@ +# Copyright 2025 Nutanix. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# NOTE This file is used by the tool in `hack/tools/helm-cm` to add +# docker-registry chart metadata to the "helm-addons" ConfigMap. The tool takes +# a kustomization as input. We do not use this file with kustomize. + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +metadata: + name: registry-distribution + +sortOptions: + order: fifo + +helmCharts: +- name: docker-registry + repo: https://mesosphere.github.io/charts/staging/ + releaseName: cncf-distribution-registry + version: 2.3.1 + valuesFile: helm-values.yaml + includeCRDs: true + skipTests: true + namespace: registry-system + +namespace: registry-system diff --git a/hack/examples/bases/docker/cluster/kustomization.yaml.tmpl b/hack/examples/bases/docker/cluster/kustomization.yaml.tmpl index 92bc7ee96..5d715271a 100644 --- a/hack/examples/bases/docker/cluster/kustomization.yaml.tmpl +++ b/hack/examples/bases/docker/cluster/kustomization.yaml.tmpl @@ -40,7 +40,10 @@ patches: path: ../../../patches/docker/csi.yaml - target: kind: Cluster - path: ../../../patches/nutanix/cosi.yaml + path: ../../../patches/docker/cosi.yaml +- target: + kind: Cluster + path: ../../../patches/docker/registry.yaml - target: kind: Cluster path: ../../../patches/encryption.yaml diff --git a/hack/examples/patches/docker/registry.yaml b/hack/examples/patches/docker/registry.yaml new file mode 100644 index 000000000..bfda200ac --- /dev/null +++ b/hack/examples/patches/docker/registry.yaml @@ -0,0 +1,6 @@ +# Copyright 2025 Nutanix. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +- op: "add" + path: "/spec/topology/variables/0/value/addons/registry" + value: {} diff --git a/pkg/handlers/generic/lifecycle/config/cm.go b/pkg/handlers/generic/lifecycle/config/cm.go index e7be0c717..fe14dfd46 100644 --- a/pkg/handlers/generic/lifecycle/config/cm.go +++ b/pkg/handlers/generic/lifecycle/config/cm.go @@ -17,18 +17,19 @@ import ( type Component string const ( - Autoscaler Component = "cluster-autoscaler" - Tigera Component = "tigera-operator" - Cilium Component = "cilium" - NFD Component = "nfd" - NutanixStorageCSI Component = "nutanix-storage-csi" - SnapshotController Component = "snapshot-controller" - NutanixCCM Component = "nutanix-ccm" - MetalLB Component = "metallb" - LocalPathProvisionerCSI Component = "local-path-provisioner-csi" - AWSEBSCSI Component = "aws-ebs-csi" - AWSCCM Component = "aws-ccm" - COSIController Component = "cosi-controller" + Autoscaler Component = "cluster-autoscaler" + Tigera Component = "tigera-operator" + Cilium Component = "cilium" + NFD Component = "nfd" + NutanixStorageCSI Component = "nutanix-storage-csi" + SnapshotController Component = "snapshot-controller" + NutanixCCM Component = "nutanix-ccm" + MetalLB Component = "metallb" + LocalPathProvisionerCSI Component = "local-path-provisioner-csi" + AWSEBSCSI Component = "aws-ebs-csi" + AWSCCM Component = "aws-ccm" + COSIController Component = "cosi-controller" + CNCFDistributionRegistry Component = "cncf-distribution-registry" ) type HelmChartGetter struct { diff --git a/pkg/handlers/generic/lifecycle/handlers.go b/pkg/handlers/generic/lifecycle/handlers.go index 462eb1ac0..bd3034430 100644 --- a/pkg/handlers/generic/lifecycle/handlers.go +++ b/pkg/handlers/generic/lifecycle/handlers.go @@ -25,6 +25,8 @@ import ( nutanixcsi "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/csi/nutanix" "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/csi/snapshotcontroller" "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/nfd" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/registry" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/registry/cncfdistribution" "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/servicelbgc" "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/serviceloadbalancer" "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/serviceloadbalancer/metallb" @@ -45,6 +47,7 @@ type Handlers struct { localPathCSIConfig *localpath.Config snapshotControllerConfig *snapshotcontroller.Config cosiControllerConfig *cosi.ControllerConfig + distributionConfig *cncfdistribution.Config } func New( @@ -66,6 +69,7 @@ func New( localPathCSIConfig: localpath.NewConfig(globalOptions), snapshotControllerConfig: snapshotcontroller.NewConfig(globalOptions), cosiControllerConfig: cosi.NewControllerConfig(globalOptions), + distributionConfig: &cncfdistribution.Config{GlobalOptions: globalOptions}, } } @@ -100,6 +104,13 @@ func (h *Handlers) AllHandlers(mgr manager.Manager) []handlers.Named { helmChartInfoGetter, ), } + registryHandlers := map[string]registry.RegistryProvider{ + v1alpha1.RegistryProviderCNCFDistribution: cncfdistribution.New( + mgr.GetClient(), + h.distributionConfig, + helmChartInfoGetter, + ), + } serviceLoadBalancerHandlers := map[string]serviceloadbalancer.ServiceLoadBalancerProvider{ v1alpha1.ServiceLoadBalancerProviderMetalLB: metallb.New( mgr.GetClient(), @@ -117,6 +128,7 @@ func (h *Handlers) AllHandlers(mgr manager.Manager) []handlers.Named { snapshotcontroller.New(mgr.GetClient(), h.snapshotControllerConfig, helmChartInfoGetter), cosi.New(mgr.GetClient(), h.cosiControllerConfig, helmChartInfoGetter), servicelbgc.New(mgr.GetClient()), + registry.New(mgr.GetClient(), registryHandlers), // The order of the handlers in the list is important and are called consecutively. // The MetalLB provider may be configured to create a IPAddressPool on the remote cluster. // However, the MetalLB provider also has a webhook that validates IPAddressPool requests. @@ -218,4 +230,5 @@ func (h *Handlers) AddFlags(flagSet *pflag.FlagSet) { h.nutanixCCMConfig.AddFlags("ccm.nutanix", flagSet) h.metalLBConfig.AddFlags("metallb", flagSet) h.cosiControllerConfig.AddFlags("cosi.controller", flagSet) + h.distributionConfig.AddFlags("registry.cncf-distribution", flagSet) } diff --git a/pkg/handlers/generic/lifecycle/registry/cncfdistribution/handler.go b/pkg/handlers/generic/lifecycle/registry/cncfdistribution/handler.go new file mode 100644 index 000000000..a4080956f --- /dev/null +++ b/pkg/handlers/generic/lifecycle/registry/cncfdistribution/handler.go @@ -0,0 +1,121 @@ +// Copyright 2025 Nutanix. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package cncfdistribution + +import ( + "bytes" + "context" + "fmt" + "text/template" + + "github.com/go-logr/logr" + "github.com/spf13/pflag" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/addons" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/config" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/registry/utils" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/options" +) + +const ( + DefaultHelmReleaseName = "cncf-distribution-registry" + DefaultHelmReleaseNamespace = "registry-system" +) + +type Config struct { + *options.GlobalOptions + + defaultValuesTemplateConfigMapName string +} + +func (c *Config) AddFlags(prefix string, flags *pflag.FlagSet) { + flags.StringVar( + &c.defaultValuesTemplateConfigMapName, + prefix+".default-values-template-configmap-name", + "default-cncf-distribution-registry-helm-values-template", + "default values ConfigMap name", + ) +} + +type CNCFDistribution struct { + client ctrlclient.Client + config *Config + helmChartInfoGetter *config.HelmChartGetter +} + +func New( + c ctrlclient.Client, + cfg *Config, + helmChartInfoGetter *config.HelmChartGetter, +) *CNCFDistribution { + return &CNCFDistribution{ + client: c, + config: cfg, + helmChartInfoGetter: helmChartInfoGetter, + } +} + +func (n *CNCFDistribution) Apply( + ctx context.Context, + _ v1alpha1.RegistryAddon, + cluster *clusterv1.Cluster, + log logr.Logger, +) error { + log.Info("Applying CNCF Distribution registry installation") + + helmChartInfo, err := n.helmChartInfoGetter.For(ctx, log, config.CNCFDistributionRegistry) + if err != nil { + return fmt.Errorf("failed to get CNCF Distribution registry helm chart: %w", err) + } + + addonApplier := addons.NewHelmAddonApplier( + addons.NewHelmAddonConfig( + n.config.defaultValuesTemplateConfigMapName, + DefaultHelmReleaseNamespace, + DefaultHelmReleaseName, + ), + n.client, + helmChartInfo, + ).WithDefaultWaiter().WithValueTemplater(templateValues) + + if err := addonApplier.Apply(ctx, cluster, n.config.DefaultsNamespace(), log); err != nil { + return fmt.Errorf("failed to apply CNCF Distribution registry addon: %w", err) + } + + return nil +} + +func templateValues(cluster *clusterv1.Cluster, text string) (string, error) { + valuesTemplate, err := template.New("").Parse(text) + if err != nil { + return "", fmt.Errorf("failed to parse template: %w", err) + } + + serviceIP, err := utils.ServiceIPForCluster(cluster) + if err != nil { + return "", fmt.Errorf("error getting service IP for the CNCF distribution registry: %w", err) + } + + type input struct { + ServiceIP string + } + + templateInput := input{ + ServiceIP: serviceIP, + } + + var b bytes.Buffer + err = valuesTemplate.Execute(&b, templateInput) + if err != nil { + return "", fmt.Errorf( + "failed template values: %w", + err, + ) + } + + return b.String(), nil +} diff --git a/pkg/handlers/generic/lifecycle/registry/doc.go b/pkg/handlers/generic/lifecycle/registry/doc.go new file mode 100644 index 000000000..3eaf5f536 --- /dev/null +++ b/pkg/handlers/generic/lifecycle/registry/doc.go @@ -0,0 +1,6 @@ +// Copyright 2025 Nutanix. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Package registry provides a handler for managing a registry addon in clusters. +// The clusters will also be configured to use the registry as a Containerd mirror (done in a different handler). +package registry diff --git a/pkg/handlers/generic/lifecycle/registry/handler.go b/pkg/handlers/generic/lifecycle/registry/handler.go new file mode 100644 index 000000000..891e747d6 --- /dev/null +++ b/pkg/handlers/generic/lifecycle/registry/handler.go @@ -0,0 +1,165 @@ +// Copyright 2025 Nutanix. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package registry + +import ( + "context" + "fmt" + + "github.com/go-logr/logr" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1" + ctrl "sigs.k8s.io/controller-runtime" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1" + commonhandlers "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/handlers" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/handlers/lifecycle" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/variables" +) + +type RegistryProvider interface { + Apply( + ctx context.Context, + registryVar v1alpha1.RegistryAddon, + cluster *clusterv1.Cluster, + log logr.Logger, + ) error +} + +type RegistryHandler struct { + client ctrlclient.Client + variableName string + variablePath []string + ProviderHandler map[string]RegistryProvider +} + +var ( + _ commonhandlers.Named = &RegistryHandler{} + _ lifecycle.AfterControlPlaneInitialized = &RegistryHandler{} + _ lifecycle.BeforeClusterUpgrade = &RegistryHandler{} +) + +func New( + c ctrlclient.Client, + handlers map[string]RegistryProvider, +) *RegistryHandler { + return &RegistryHandler{ + client: c, + variableName: v1alpha1.ClusterConfigVariableName, + variablePath: []string{"addons", v1alpha1.RegistryAddonVariableName}, + ProviderHandler: handlers, + } +} + +func (r *RegistryHandler) Name() string { + return "RegistryHandler" +} + +func (r *RegistryHandler) AfterControlPlaneInitialized( + ctx context.Context, + req *runtimehooksv1.AfterControlPlaneInitializedRequest, + resp *runtimehooksv1.AfterControlPlaneInitializedResponse, +) { + commonResponse := &runtimehooksv1.CommonResponse{} + r.apply(ctx, &req.Cluster, commonResponse) + resp.Status = commonResponse.GetStatus() + resp.Message = commonResponse.GetMessage() +} + +func (r *RegistryHandler) BeforeClusterUpgrade( + ctx context.Context, + req *runtimehooksv1.BeforeClusterUpgradeRequest, + resp *runtimehooksv1.BeforeClusterUpgradeResponse, +) { + commonResponse := &runtimehooksv1.CommonResponse{} + r.apply(ctx, &req.Cluster, commonResponse) + resp.Status = commonResponse.GetStatus() + resp.Message = commonResponse.GetMessage() +} + +func (r *RegistryHandler) apply( + ctx context.Context, + cluster *clusterv1.Cluster, + resp *runtimehooksv1.CommonResponse, +) { + clusterKey := ctrlclient.ObjectKeyFromObject(cluster) + + log := ctrl.LoggerFrom(ctx).WithValues( + "cluster", + clusterKey, + ) + + varMap := variables.ClusterVariablesToVariablesMap(cluster.Spec.Topology.Variables) + registryVar, err := variables.Get[v1alpha1.RegistryAddon]( + varMap, + r.variableName, + r.variablePath...) + if err != nil { + if variables.IsNotFoundError(err) { + log.V(5). + Info( + "Skipping RegistryAddon, field is not specified", + "error", + err, + ) + return + } + log.Error( + err, + "failed to read RegistryAddon provider from cluster definition", + ) + resp.SetStatus(runtimehooksv1.ResponseStatusFailure) + resp.SetMessage( + fmt.Sprintf("failed to read RegistryAddon provider from cluster definition: %v", + err, + ), + ) + return + } + + handler, ok := r.ProviderHandler[registryVar.Provider] + if !ok { + err = fmt.Errorf("unknown RegistryAddon Provider") + log.Error(err, "provider", registryVar.Provider) + resp.SetStatus(runtimehooksv1.ResponseStatusFailure) + resp.SetMessage( + fmt.Sprintf("%s %s", err, registryVar.Provider), + ) + return + } + + log.Info(fmt.Sprintf("Deploying RegistryAddon provider %s", registryVar.Provider)) + err = handler.Apply( + ctx, + registryVar, + cluster, + log, + ) + if err != nil { + log.Error( + err, + fmt.Sprintf( + "failed to deploy RegistryAddon provider %s", + registryVar.Provider, + ), + ) + resp.SetStatus(runtimehooksv1.ResponseStatusFailure) + resp.SetMessage( + fmt.Sprintf( + "failed to deploy RegistryAddon provider: %v", + err, + ), + ) + return + } + + resp.SetStatus(runtimehooksv1.ResponseStatusSuccess) + resp.SetMessage( + fmt.Sprintf( + "Deployed RegistryAddon provider %s", + registryVar.Provider, + ), + ) +} diff --git a/pkg/handlers/generic/lifecycle/registry/utils/ip.go b/pkg/handlers/generic/lifecycle/registry/utils/ip.go new file mode 100644 index 000000000..8b46f34c2 --- /dev/null +++ b/pkg/handlers/generic/lifecycle/registry/utils/ip.go @@ -0,0 +1,47 @@ +// Copyright 2025 Nutanix. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package utils + +import ( + "errors" + "fmt" + + netutils "k8s.io/utils/net" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" +) + +const ipIndex = 20 + +func ServiceIPForCluster(cluster *clusterv1.Cluster) (string, error) { + var serviceCIDRBlocks []string + if cluster.Spec.ClusterNetwork != nil && cluster.Spec.ClusterNetwork.Services != nil { + serviceCIDRBlocks = cluster.Spec.ClusterNetwork.Services.CIDRBlocks + } + serviceIP, err := getServiceIP(serviceCIDRBlocks) + if err != nil { + return "", fmt.Errorf("error getting a service IP for a cluster: %w", err) + } + + return serviceIP, nil +} + +func getServiceIP(serviceSubnetStrings []string) (string, error) { + serviceSubnets, err := netutils.ParseCIDRs(serviceSubnetStrings) + if err != nil { + return "", fmt.Errorf("unable to parse service Subnets: %w", err) + } + if len(serviceSubnets) == 0 { + return "", errors.New("unexpected empty service Subnets") + } + + // Selects the 20th IP in service subnet CIDR range as the Service IP + serviceIP, err := netutils.GetIndexedIP(serviceSubnets[0], ipIndex) + if err != nil { + return "", fmt.Errorf( + "unable to get internal Kubernetes Service IP from the given service Subnets", + ) + } + + return serviceIP.String(), nil +} diff --git a/pkg/handlers/generic/lifecycle/registry/utils/ip_test.go b/pkg/handlers/generic/lifecycle/registry/utils/ip_test.go new file mode 100644 index 000000000..e23ca5b68 --- /dev/null +++ b/pkg/handlers/generic/lifecycle/registry/utils/ip_test.go @@ -0,0 +1,89 @@ +// Copyright 2025 Nutanix. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package utils + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" +) + +func Test_ServiceIPForCluster(t *testing.T) { + t.Parallel() + tests := []struct { + name string + cluster *clusterv1.Cluster + want string + wantErr error + }{ + { + name: "Cluster with nil service CIDR", + cluster: &clusterv1.Cluster{ + Spec: clusterv1.ClusterSpec{ + ClusterNetwork: &clusterv1.ClusterNetwork{}, + }, + }, + wantErr: errors.New("error getting a service IP for a cluster: unexpected empty service Subnets"), + }, + { + name: "Cluster with empty service CIDR slice", + cluster: &clusterv1.Cluster{ + Spec: clusterv1.ClusterSpec{ + ClusterNetwork: &clusterv1.ClusterNetwork{ + Services: &clusterv1.NetworkRanges{ + CIDRBlocks: []string{}, + }, + }, + }, + }, + wantErr: errors.New("error getting a service IP for a cluster: unexpected empty service Subnets"), + }, + { + name: "Cluster with a single service CIDR", + cluster: &clusterv1.Cluster{ + Spec: clusterv1.ClusterSpec{ + ClusterNetwork: &clusterv1.ClusterNetwork{ + Services: &clusterv1.NetworkRanges{ + CIDRBlocks: []string{ + "192.168.0.0/16", + }, + }, + }, + }, + }, + want: "192.168.0.20", + }, + { + name: "Cluster with a multiple service CIDRs", + cluster: &clusterv1.Cluster{ + Spec: clusterv1.ClusterSpec{ + ClusterNetwork: &clusterv1.ClusterNetwork{ + Services: &clusterv1.NetworkRanges{ + CIDRBlocks: []string{ + "192.168.0.0/16", + "10.96.0.0/12", + }, + }, + }, + }, + }, + want: "192.168.0.20", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + got, err := ServiceIPForCluster(tt.cluster) + if tt.wantErr != nil { + require.EqualError(t, err, tt.wantErr.Error()) + } else { + require.NoError(t, err) + } + assert.Equal(t, tt.want, got) + }) + } +} diff --git a/test/e2e/addon_helpers.go b/test/e2e/addon_helpers.go index 0ea61fd2e..31d9a54ae 100644 --- a/test/e2e/addon_helpers.go +++ b/test/e2e/addon_helpers.go @@ -21,6 +21,7 @@ type WaitForAddonsToBeReadyInWorkloadClusterInput struct { ClusterProxy framework.ClusterProxy DeploymentIntervals []interface{} DaemonSetIntervals []interface{} + StatefulSetIntervals []interface{} HelmReleaseIntervals []interface{} ClusterResourceSetIntervals []interface{} ResourceIntervals []interface{} @@ -119,4 +120,14 @@ func WaitForAddonsToBeReadyInWorkloadCluster( ResourceIntervals: input.ResourceIntervals, }, ) + + WaitForRegistryAddonToBeReadyInWorkloadCluster( + ctx, + WaitForRegistryAddonToBeReadyInWorkloadClusterInput{ + Registry: input.AddonsConfig.Registry, + WorkloadCluster: input.WorkloadCluster, + ClusterProxy: input.ClusterProxy, + StatefulSetIntervals: input.StatefulSetIntervals, + }, + ) } diff --git a/test/e2e/config/caren.yaml b/test/e2e/config/caren.yaml index 48933a1b4..66e59edd5 100644 --- a/test/e2e/config/caren.yaml +++ b/test/e2e/config/caren.yaml @@ -219,6 +219,7 @@ intervals: default/wait-nodes-ready: ["10m", "10s"] default/wait-deployment: ["10m", "10s"] default/wait-daemonset: [ "5m", "10s" ] + default/wait-statefulset: [ "10m", "10s" ] default/wait-clusterresourceset: [ "5m", "10s" ] default/wait-helmrelease: [ "5m", "10s" ] default/wait-resource: [ "5m", "10s" ] diff --git a/test/e2e/quick_start_test.go b/test/e2e/quick_start_test.go index 2bda81b16..f8c366c53 100644 --- a/test/e2e/quick_start_test.go +++ b/test/e2e/quick_start_test.go @@ -265,6 +265,10 @@ var _ = Describe("Quick start", func() { flavor, "wait-daemonset", ), + StatefulSetIntervals: testE2EConfig.GetIntervals( + flavor, + "wait-statefulset", + ), HelmReleaseIntervals: testE2EConfig.GetIntervals( flavor, "wait-helmrelease", diff --git a/test/e2e/registry.go b/test/e2e/registry.go new file mode 100644 index 000000000..d58298011 --- /dev/null +++ b/test/e2e/registry.go @@ -0,0 +1,58 @@ +//go:build e2e + +// Copyright 2024 Nutanix. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package e2e + +import ( + "context" + + appsv1 "k8s.io/api/apps/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + "sigs.k8s.io/cluster-api/test/framework" + + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1" +) + +type WaitForRegistryAddonToBeReadyInWorkloadClusterInput struct { + Registry *v1alpha1.RegistryAddon + WorkloadCluster *clusterv1.Cluster + ClusterProxy framework.ClusterProxy + StatefulSetIntervals []interface{} + HelmReleaseIntervals []interface{} +} + +func WaitForRegistryAddonToBeReadyInWorkloadCluster( + ctx context.Context, + input WaitForRegistryAddonToBeReadyInWorkloadClusterInput, //nolint:gocritic // This hugeParam is OK in tests. +) { + if input.Registry == nil { + return + } + + WaitForHelmReleaseProxyReadyForCluster( + ctx, + WaitForHelmReleaseProxyReadyForClusterInput{ + GetLister: input.ClusterProxy.GetClient(), + Cluster: input.WorkloadCluster, + HelmReleaseName: "cncf-distribution-registry", + }, + input.HelmReleaseIntervals..., + ) + + workloadClusterClient := input.ClusterProxy.GetWorkloadCluster( + ctx, input.WorkloadCluster.Namespace, input.WorkloadCluster.Name, + ).GetClient() + + WaitForStatefulSetsAvailable(ctx, WaitForStatefulSetAvailableInput{ + Getter: workloadClusterClient, + StatefulSet: &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cncf-distribution-registry-docker-registry", + Namespace: "registry-system", + }, + }, + }, input.StatefulSetIntervals...) +} diff --git a/test/e2e/self_hosted_test.go b/test/e2e/self_hosted_test.go index 3267b2b41..2b835c901 100644 --- a/test/e2e/self_hosted_test.go +++ b/test/e2e/self_hosted_test.go @@ -127,6 +127,10 @@ var _ = Describe("Self-hosted", Serial, func() { flavor, "wait-daemonset", ), + StatefulSetIntervals: e2eConfig.GetIntervals( + flavor, + "wait-statefulset", + ), HelmReleaseIntervals: e2eConfig.GetIntervals( flavor, "wait-helmrelease", diff --git a/test/e2e/statefulset_helpers.go b/test/e2e/statefulset_helpers.go new file mode 100644 index 000000000..1ed7344bd --- /dev/null +++ b/test/e2e/statefulset_helpers.go @@ -0,0 +1,44 @@ +//go:build e2e + +// Copyright 2024 Nutanix. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package e2e + +import ( + "context" + "time" + + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + capie2e "sigs.k8s.io/cluster-api/test/e2e" + "sigs.k8s.io/cluster-api/test/framework" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type WaitForStatefulSetAvailableInput struct { + Getter framework.Getter + StatefulSet *appsv1.StatefulSet +} + +// WaitForStatefulSetsAvailable waits until the Deployment has observedGeneration equal to generation and +// status.Available = True, that signals that all the desired replicas are in place. +func WaitForStatefulSetsAvailable( + ctx context.Context, input WaitForStatefulSetAvailableInput, intervals ...interface{}, +) { + start := time.Now() + key := client.ObjectKeyFromObject(input.StatefulSet) + capie2e.Byf("waiting for statefulset %s to be available", key) + Log("starting to wait for statefulset to become available") + Eventually(func() bool { + if err := input.Getter.Get(ctx, key, input.StatefulSet); err == nil { + if input.StatefulSet.Status.ObservedGeneration != input.StatefulSet.Generation { + return false + } + + return input.StatefulSet.Status.AvailableReplicas == input.StatefulSet.Status.Replicas + } + return false + }, intervals...).Should(BeTrue()) + Logf("StatefulSet %s is now available, took %v", key, time.Since(start)) +}