From 1170634ed95d3bd324500ada555262755dfb5666 Mon Sep 17 00:00:00 2001 From: Vinnie Marone Date: Sun, 2 Feb 2025 19:22:18 -0500 Subject: [PATCH 1/9] add support for secrets for backendtlspolicy --- .github/workflows/build.yml | 4 +- .../static/state/change_processor_test.go | 105 ++++++++++++++---- .../static/state/dataplane/configuration.go | 60 ++++++---- .../state/dataplane/configuration_test.go | 16 ++- .../static/state/graph/backend_tls_policy.go | 42 +++++-- .../state/graph/backend_tls_policy_test.go | 22 +++- .../static/state/graph/certificate_bundle.go | 66 +++++++++++ .../mode/static/state/graph/configmaps.go | 45 ++------ internal/mode/static/state/graph/graph.go | 3 +- .../mode/static/state/graph/graph_test.go | 18 ++- internal/mode/static/state/graph/secret.go | 24 +++- .../mode/static/state/graph/secret_test.go | 12 ++ 12 files changed, 314 insertions(+), 103 deletions(-) create mode 100644 internal/mode/static/state/graph/certificate_bundle.go diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b564018162..f1c8aba5d1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -103,8 +103,8 @@ jobs: with: context: ${{ inputs.tag != '' && 'git' || 'workflow' }} images: | - name=ghcr.io/nginxinc/nginx-gateway-fabric,enable=${{ inputs.image == 'ngf' && github.event_name != 'pull_request' }} - name=ghcr.io/nginxinc/nginx-gateway-fabric/nginx,enable=${{ inputs.image == 'nginx' && github.event_name != 'pull_request' }} + name=ghcr.io/${{ github.repository_owner }}/nginx-gateway-fabric,enable=${{ inputs.image == 'ngf' && github.event_name != 'pull_request' }} + name=ghcr.io/${{ github.repository_owner }}/nginx-gateway-fabric/nginx,enable=${{ inputs.image == 'nginx' && github.event_name != 'pull_request' }} name=docker-mgmt.nginx.com/nginx-gateway-fabric/nginx-plus,enable=${{ inputs.image == 'plus' && github.event_name != 'pull_request' }} name=us-docker.pkg.dev/${{ secrets.GCP_PROJECT_ID }}/nginx-gateway-fabric/nginx-plus,enable=${{ inputs.image == 'plus' && github.event_name != 'pull_request' }} name=localhost:5000/nginx-gateway-fabric/${{ inputs.image }} diff --git a/internal/mode/static/state/change_processor_test.go b/internal/mode/static/state/change_processor_test.go index 03d86a377b..c21e8d4912 100644 --- a/internal/mode/static/state/change_processor_test.go +++ b/internal/mode/static/state/change_processor_test.go @@ -422,6 +422,7 @@ var _ = Describe("ChangeProcessor", func() { var ( gcUpdated *v1.GatewayClass diffNsTLSSecret, sameNsTLSSecret *apiv1.Secret + diffNsTLSCert, sameNsTLSCert *graph.CertificateBundle hr1, hr1Updated, hr2 *v1.HTTPRoute gr1, gr1Updated, gr2 *v1.GRPCRoute tr1, tr1Updated, tr2 *v1alpha2.TLSRoute @@ -592,8 +593,19 @@ var _ = Describe("ChangeProcessor", func() { apiv1.TLSPrivateKeyKey: key, }, } + sameNsTLSCert = graph.NewCertificateBundle( + types.NamespacedName{Namespace: sameNsTLSSecret.Namespace, Name: sameNsTLSSecret.Name}, + "Secret", + &graph.Certificate{ + TLSCert: cert, + TLSPrivateKey: key, + }, + ) diffNsTLSSecret = &apiv1.Secret{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + }, ObjectMeta: metav1.ObjectMeta{ Name: "different-ns-tls-secret", Namespace: "cert-ns", @@ -605,6 +617,15 @@ var _ = Describe("ChangeProcessor", func() { }, } + diffNsTLSCert = graph.NewCertificateBundle( + types.NamespacedName{Namespace: diffNsTLSSecret.Namespace, Name: diffNsTLSSecret.Name}, + "Secret", + &graph.Certificate{ + TLSCert: cert, + TLSPrivateKey: key, + }, + ) + gw1 = createGateway( "gateway-1", createHTTPListener(), @@ -1155,6 +1176,14 @@ var _ = Describe("ChangeProcessor", func() { expGraph.ReferencedSecrets[client.ObjectKeyFromObject(diffNsTLSSecret)] = &graph.Secret{ Source: diffNsTLSSecret, + CertBundle: graph.NewCertificateBundle( + types.NamespacedName{Namespace: diffNsTLSSecret.Namespace, Name: diffNsTLSSecret.Name}, + "Secret", + &graph.Certificate{ + TLSCert: cert, + TLSPrivateKey: key, + }, + ), } expGraph.ReferencedServices = nil @@ -1189,6 +1218,14 @@ var _ = Describe("ChangeProcessor", func() { expGraph.ReferencedSecrets[client.ObjectKeyFromObject(diffNsTLSSecret)] = &graph.Secret{ Source: diffNsTLSSecret, + CertBundle: graph.NewCertificateBundle( + types.NamespacedName{Namespace: diffNsTLSSecret.Namespace, Name: diffNsTLSSecret.Name}, + "Secret", + &graph.Certificate{ + TLSCert: cert, + TLSPrivateKey: key, + }, + ), } processAndValidateGraph(expGraph) @@ -1209,6 +1246,14 @@ var _ = Describe("ChangeProcessor", func() { expGraph.ReferencedSecrets[client.ObjectKeyFromObject(diffNsTLSSecret)] = &graph.Secret{ Source: diffNsTLSSecret, + CertBundle: graph.NewCertificateBundle( + types.NamespacedName{Namespace: diffNsTLSSecret.Namespace, Name: diffNsTLSSecret.Name}, + "Secret", + &graph.Certificate{ + TLSCert: cert, + TLSPrivateKey: key, + }, + ), } processAndValidateGraph(expGraph) @@ -1219,7 +1264,8 @@ var _ = Describe("ChangeProcessor", func() { processor.CaptureUpsertChange(trServiceRefGrant) expGraph.ReferencedSecrets[client.ObjectKeyFromObject(diffNsTLSSecret)] = &graph.Secret{ - Source: diffNsTLSSecret, + Source: diffNsTLSSecret, + CertBundle: diffNsTLSCert, } processAndValidateGraph(expGraph) @@ -1230,7 +1276,8 @@ var _ = Describe("ChangeProcessor", func() { processor.CaptureUpsertChange(gatewayAPICRDUpdated) expGraph.ReferencedSecrets[client.ObjectKeyFromObject(diffNsTLSSecret)] = &graph.Secret{ - Source: diffNsTLSSecret, + Source: diffNsTLSSecret, + CertBundle: diffNsTLSCert, } expGraph.GatewayClass.Conditions = conditions.NewGatewayClassSupportedVersionBestEffort( @@ -1247,7 +1294,8 @@ var _ = Describe("ChangeProcessor", func() { processor.CaptureUpsertChange(gatewayAPICRDSameVersion) expGraph.ReferencedSecrets[client.ObjectKeyFromObject(diffNsTLSSecret)] = &graph.Secret{ - Source: diffNsTLSSecret, + Source: diffNsTLSSecret, + CertBundle: diffNsTLSCert, } expGraph.GatewayClass.Conditions = conditions.NewGatewayClassSupportedVersionBestEffort( @@ -1266,7 +1314,8 @@ var _ = Describe("ChangeProcessor", func() { processor.CaptureUpsertChange(gatewayAPICRD) expGraph.ReferencedSecrets[client.ObjectKeyFromObject(diffNsTLSSecret)] = &graph.Secret{ - Source: diffNsTLSSecret, + Source: diffNsTLSSecret, + CertBundle: diffNsTLSCert, } processAndValidateGraph(expGraph) @@ -1282,7 +1331,8 @@ var _ = Describe("ChangeProcessor", func() { listener80 := getListenerByName(expGraph.Gateway, httpListenerName) listener80.Routes[httpRouteKey1].Source.SetGeneration(hr1Updated.Generation) expGraph.ReferencedSecrets[client.ObjectKeyFromObject(diffNsTLSSecret)] = &graph.Secret{ - Source: diffNsTLSSecret, + Source: diffNsTLSSecret, + CertBundle: diffNsTLSCert, } processAndValidateGraph(expGraph) @@ -1299,7 +1349,8 @@ var _ = Describe("ChangeProcessor", func() { listener80 := getListenerByName(expGraph.Gateway, httpListenerName) listener80.Routes[grpcRouteKey1].Source.SetGeneration(gr1Updated.Generation) expGraph.ReferencedSecrets[client.ObjectKeyFromObject(diffNsTLSSecret)] = &graph.Secret{ - Source: diffNsTLSSecret, + Source: diffNsTLSSecret, + CertBundle: diffNsTLSCert, } processAndValidateGraph(expGraph) @@ -1313,7 +1364,8 @@ var _ = Describe("ChangeProcessor", func() { tlsListener.L4Routes[trKey1].Source.SetGeneration(tr1Updated.Generation) expGraph.ReferencedSecrets[client.ObjectKeyFromObject(diffNsTLSSecret)] = &graph.Secret{ - Source: diffNsTLSSecret, + Source: diffNsTLSSecret, + CertBundle: diffNsTLSCert, } processAndValidateGraph(expGraph) @@ -1325,7 +1377,8 @@ var _ = Describe("ChangeProcessor", func() { expGraph.Gateway.Source.Generation = gw1Updated.Generation expGraph.ReferencedSecrets[client.ObjectKeyFromObject(diffNsTLSSecret)] = &graph.Secret{ - Source: diffNsTLSSecret, + Source: diffNsTLSSecret, + CertBundle: diffNsTLSCert, } processAndValidateGraph(expGraph) @@ -1337,7 +1390,8 @@ var _ = Describe("ChangeProcessor", func() { expGraph.GatewayClass.Source.Generation = gcUpdated.Generation expGraph.ReferencedSecrets[client.ObjectKeyFromObject(diffNsTLSSecret)] = &graph.Secret{ - Source: diffNsTLSSecret, + Source: diffNsTLSSecret, + CertBundle: diffNsTLSCert, } processAndValidateGraph(expGraph) @@ -1348,7 +1402,8 @@ var _ = Describe("ChangeProcessor", func() { processor.CaptureUpsertChange(diffNsTLSSecret) expGraph.ReferencedSecrets[client.ObjectKeyFromObject(diffNsTLSSecret)] = &graph.Secret{ - Source: diffNsTLSSecret, + Source: diffNsTLSSecret, + CertBundle: diffNsTLSCert, } processAndValidateGraph(expGraph) @@ -1357,7 +1412,8 @@ var _ = Describe("ChangeProcessor", func() { When("no changes are captured", func() { It("returns nil graph", func() { expGraph.ReferencedSecrets[client.ObjectKeyFromObject(diffNsTLSSecret)] = &graph.Secret{ - Source: diffNsTLSSecret, + Source: diffNsTLSSecret, + CertBundle: diffNsTLSCert, } changed, graphCfg := processor.Process() @@ -1371,7 +1427,8 @@ var _ = Describe("ChangeProcessor", func() { processor.CaptureUpsertChange(sameNsTLSSecret) expGraph.ReferencedSecrets[client.ObjectKeyFromObject(diffNsTLSSecret)] = &graph.Secret{ - Source: diffNsTLSSecret, + Source: diffNsTLSSecret, + CertBundle: diffNsTLSCert, } changed, graphCfg := processor.Process() @@ -1388,7 +1445,8 @@ var _ = Describe("ChangeProcessor", func() { {Namespace: "test", Name: "gateway-2"}: gw2, } expGraph.ReferencedSecrets[client.ObjectKeyFromObject(diffNsTLSSecret)] = &graph.Secret{ - Source: diffNsTLSSecret, + Source: diffNsTLSSecret, + CertBundle: diffNsTLSCert, } processAndValidateGraph(expGraph) @@ -1411,7 +1469,8 @@ var _ = Describe("ChangeProcessor", func() { FailedCondition: staticConds.NewRouteNotAcceptedGatewayIgnored(), } expGraph.ReferencedSecrets[client.ObjectKeyFromObject(diffNsTLSSecret)] = &graph.Secret{ - Source: diffNsTLSSecret, + Source: diffNsTLSSecret, + CertBundle: diffNsTLSCert, } processAndValidateGraph(expGraph) @@ -1445,7 +1504,8 @@ var _ = Describe("ChangeProcessor", func() { } expGraph.ReferencedSecrets[client.ObjectKeyFromObject(diffNsTLSSecret)] = &graph.Secret{ - Source: diffNsTLSSecret, + Source: diffNsTLSSecret, + CertBundle: diffNsTLSCert, } processAndValidateGraph(expGraph) @@ -1485,7 +1545,8 @@ var _ = Describe("ChangeProcessor", func() { } expGraph.ReferencedSecrets[client.ObjectKeyFromObject(diffNsTLSSecret)] = &graph.Secret{ - Source: diffNsTLSSecret, + Source: diffNsTLSSecret, + CertBundle: diffNsTLSCert, } processAndValidateGraph(expGraph) @@ -1532,7 +1593,8 @@ var _ = Describe("ChangeProcessor", func() { sameNsTLSSecretRef := helpers.GetPointer(client.ObjectKeyFromObject(sameNsTLSSecret)) listener443.ResolvedSecret = sameNsTLSSecretRef expGraph.ReferencedSecrets[client.ObjectKeyFromObject(sameNsTLSSecret)] = &graph.Secret{ - Source: sameNsTLSSecret, + Source: sameNsTLSSecret, + CertBundle: sameNsTLSCert, } delete(expGraph.ReferencedServices, expRouteHR1.Spec.Rules[0].BackendRefs[0].SvcNsName) @@ -1583,7 +1645,8 @@ var _ = Describe("ChangeProcessor", func() { sameNsTLSSecretRef := helpers.GetPointer(client.ObjectKeyFromObject(sameNsTLSSecret)) listener443.ResolvedSecret = sameNsTLSSecretRef expGraph.ReferencedSecrets[client.ObjectKeyFromObject(sameNsTLSSecret)] = &graph.Secret{ - Source: sameNsTLSSecret, + Source: sameNsTLSSecret, + CertBundle: sameNsTLSCert, } delete(expGraph.ReferencedServices, expRouteHR1.Spec.Rules[0].BackendRefs[0].SvcNsName) @@ -1627,7 +1690,8 @@ var _ = Describe("ChangeProcessor", func() { sameNsTLSSecretRef := helpers.GetPointer(client.ObjectKeyFromObject(sameNsTLSSecret)) listener443.ResolvedSecret = sameNsTLSSecretRef expGraph.ReferencedSecrets[client.ObjectKeyFromObject(sameNsTLSSecret)] = &graph.Secret{ - Source: sameNsTLSSecret, + Source: sameNsTLSSecret, + CertBundle: sameNsTLSCert, } delete(expGraph.ReferencedServices, expRouteHR1.Spec.Rules[0].BackendRefs[0].SvcNsName) @@ -1668,7 +1732,8 @@ var _ = Describe("ChangeProcessor", func() { sameNsTLSSecretRef := helpers.GetPointer(client.ObjectKeyFromObject(sameNsTLSSecret)) listener443.ResolvedSecret = sameNsTLSSecretRef expGraph.ReferencedSecrets[client.ObjectKeyFromObject(sameNsTLSSecret)] = &graph.Secret{ - Source: sameNsTLSSecret, + Source: sameNsTLSSecret, + CertBundle: sameNsTLSCert, } expRouteHR1.Spec.Rules[0].BackendRefs[0].SvcNsName = types.NamespacedName{} diff --git a/internal/mode/static/state/dataplane/configuration.go b/internal/mode/static/state/dataplane/configuration.go index cafcbc0db8..a4b8d0683e 100644 --- a/internal/mode/static/state/dataplane/configuration.go +++ b/internal/mode/static/state/dataplane/configuration.go @@ -58,12 +58,14 @@ func BuildConfiguration( BackendGroups: backendGroups, SSLKeyPairs: buildSSLKeyPairs(g.ReferencedSecrets, g.Gateway.Listeners), Version: configVersion, - CertBundles: buildCertBundles(g.ReferencedCaCertConfigMaps, backendGroups), - Telemetry: buildTelemetry(g), - BaseHTTPConfig: baseHTTPConfig, - Logging: buildLogging(g), - MainSnippets: buildSnippetsForContext(g.SnippetsFilters, ngfAPI.NginxContextMain), - AuxiliarySecrets: buildAuxiliarySecrets(g.PlusSecrets), + CertBundles: buildCertBundles( + buildRefCertificateBundles(g.ReferencedSecrets, g.ReferencedCaCertConfigMaps), + backendGroups), + Telemetry: buildTelemetry(g), + BaseHTTPConfig: baseHTTPConfig, + Logging: buildLogging(g), + MainSnippets: buildSnippetsForContext(g.SnippetsFilters, ngfAPI.NginxContextMain), + AuxiliarySecrets: buildAuxiliarySecrets(g.PlusSecrets), } return config @@ -224,8 +226,28 @@ func buildSSLKeyPairs( return keyPairs } +func buildRefCertificateBundles( + secrets map[types.NamespacedName]*graph.Secret, + configMaps map[types.NamespacedName]*graph.CaCertConfigMap) []graph.CertificateBundle { + bundles := []graph.CertificateBundle{} + + for _, secret := range secrets { + if secret.CertBundle != nil { + bundles = append(bundles, *secret.CertBundle) + } + } + + for _, configMap := range configMaps { + if configMap.CertBundle != nil { + bundles = append(bundles, *configMap.CertBundle) + } + } + + return bundles +} + func buildCertBundles( - caCertConfigMaps map[types.NamespacedName]*graph.CaCertConfigMap, + refCertBundles []graph.CertificateBundle, backendGroups []BackendGroup, ) map[CertBundleID]CertBundle { bundles := make(map[CertBundleID]CertBundle) @@ -247,18 +269,16 @@ func buildCertBundles( } } - for cmName, cm := range caCertConfigMaps { - id := generateCertBundleID(cmName) + for _, bundle := range refCertBundles { + id := generateCertBundleID(bundle.Name) if _, exists := refByBG[id]; exists { - if cm.CACert != nil || len(cm.CACert) > 0 { - // the cert could be base64 encoded or plaintext - data := make([]byte, base64.StdEncoding.DecodedLen(len(cm.CACert))) - _, err := base64.StdEncoding.Decode(data, cm.CACert) - if err != nil { - data = cm.CACert - } - bundles[id] = data + // the cert could be base64 encoded or plaintext + data := make([]byte, base64.StdEncoding.DecodedLen(len(bundle.Cert.CACert))) + _, err := base64.StdEncoding.Decode(data, bundle.Cert.CACert) + if err != nil { + data = bundle.Cert.CACert } + bundles[id] = data } } @@ -766,11 +786,11 @@ func generateSSLKeyPairID(secret types.NamespacedName) SSLKeyPairID { return SSLKeyPairID(fmt.Sprintf("ssl_keypair_%s_%s", secret.Namespace, secret.Name)) } -// generateCertBundleID generates an ID for the certificate bundle based on the ConfigMap namespaced name. +// generateCertBundleID generates an ID for the certificate bundle based on the ConfigMap/Secret namespaced name. // It is guaranteed to be unique per unique namespaced name. // The ID is safe to use as a file name. -func generateCertBundleID(configMap types.NamespacedName) CertBundleID { - return CertBundleID(fmt.Sprintf("cert_bundle_%s_%s", configMap.Namespace, configMap.Name)) +func generateCertBundleID(caCertRef types.NamespacedName) CertBundleID { + return CertBundleID(fmt.Sprintf("cert_bundle_%s_%s", caCertRef.Namespace, caCertRef.Name)) } // buildTelemetry generates the Otel configuration. diff --git a/internal/mode/static/state/dataplane/configuration_test.go b/internal/mode/static/state/dataplane/configuration_test.go index 037bcd7d9d..fa31d97f5d 100644 --- a/internal/mode/static/state/dataplane/configuration_test.go +++ b/internal/mode/static/state/dataplane/configuration_test.go @@ -809,7 +809,13 @@ func TestBuildConfiguration(t *testing.T) { "ca.crt": "cert-1", }, }, - CACert: []byte("cert-1"), + CertBundle: graph.NewCertificateBundle( + types.NamespacedName{Namespace: "test", Name: "configmap-1"}, + "ConfigMap", + &graph.Certificate{ + CACert: []byte("cert-1"), + }, + ), }, {Namespace: "test", Name: "configmap-2"}: { Source: &apiv1.ConfigMap{ @@ -821,7 +827,13 @@ func TestBuildConfiguration(t *testing.T) { "ca.crt": []byte("cert-2"), }, }, - CACert: []byte("cert-2"), + CertBundle: graph.NewCertificateBundle( + types.NamespacedName{Namespace: "test", Name: "configmap-2"}, + "ConfigMap", + &graph.Certificate{ + CACert: []byte("cert-2"), + }, + ), }, } diff --git a/internal/mode/static/state/graph/backend_tls_policy.go b/internal/mode/static/state/graph/backend_tls_policy.go index 7ef8570331..46b98bb438 100644 --- a/internal/mode/static/state/graph/backend_tls_policy.go +++ b/internal/mode/static/state/graph/backend_tls_policy.go @@ -2,9 +2,11 @@ package graph import ( "fmt" + "slices" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/validation/field" + v1 "sigs.k8s.io/gateway-api/apis/v1" "sigs.k8s.io/gateway-api/apis/v1alpha3" "github.com/nginxinc/nginx-gateway-fabric/internal/framework/conditions" @@ -31,6 +33,7 @@ type BackendTLSPolicy struct { func processBackendTLSPolicies( backendTLSPolicies map[types.NamespacedName]*v1alpha3.BackendTLSPolicy, configMapResolver *configMapResolver, + secretResolver *secretResolver, ctlrName string, gateway *Gateway, ) map[types.NamespacedName]*BackendTLSPolicy { @@ -42,7 +45,7 @@ func processBackendTLSPolicies( for nsname, backendTLSPolicy := range backendTLSPolicies { var caCertRef types.NamespacedName - valid, ignored, conds := validateBackendTLSPolicy(backendTLSPolicy, configMapResolver, ctlrName) + valid, ignored, conds := validateBackendTLSPolicy(backendTLSPolicy, configMapResolver, secretResolver, ctlrName) if valid && !ignored && backendTLSPolicy.Spec.Validation.CACertificateRefs != nil { caCertRef = types.NamespacedName{ @@ -68,6 +71,7 @@ func processBackendTLSPolicies( func validateBackendTLSPolicy( backendTLSPolicy *v1alpha3.BackendTLSPolicy, configMapResolver *configMapResolver, + secretResolver *secretResolver, ctlrName string, ) (valid, ignored bool, conds []conditions.Condition) { valid = true @@ -93,7 +97,7 @@ func validateBackendTLSPolicy( conds = append(conds, staticConds.NewPolicyInvalid(msg)) case len(caCertRefs) > 0: - if err := validateBackendTLSCACertRef(backendTLSPolicy, configMapResolver); err != nil { + if err := validateBackendTLSCACertRef(backendTLSPolicy, configMapResolver, secretResolver); err != nil { valid = false conds = append(conds, staticConds.NewPolicyInvalid( fmt.Sprintf("invalid CACertificateRef: %s", err.Error()))) @@ -124,30 +128,44 @@ func validateBackendTLSHostname(btp *v1alpha3.BackendTLSPolicy) error { return nil } -func validateBackendTLSCACertRef(btp *v1alpha3.BackendTLSPolicy, configMapResolver *configMapResolver) error { +func validateBackendTLSCACertRef(btp *v1alpha3.BackendTLSPolicy, configMapResolver *configMapResolver, secretResolver *secretResolver) error { if len(btp.Spec.Validation.CACertificateRefs) != 1 { path := field.NewPath("tls.cacertrefs") valErr := field.TooMany(path, len(btp.Spec.Validation.CACertificateRefs), 1) return valErr } - if btp.Spec.Validation.CACertificateRefs[0].Kind != "ConfigMap" { + + selectedCertRef := btp.Spec.Validation.CACertificateRefs[0] + allowedCaCertKinds := []v1.Kind{"ConfigMap", "Secret"} + + if !slices.Contains(allowedCaCertKinds, selectedCertRef.Kind) { path := field.NewPath("tls.cacertrefs[0].kind") - valErr := field.NotSupported(path, btp.Spec.Validation.CACertificateRefs[0].Kind, []string{"ConfigMap"}) + valErr := field.NotSupported(path, btp.Spec.Validation.CACertificateRefs[0].Kind, allowedCaCertKinds) return valErr } - if btp.Spec.Validation.CACertificateRefs[0].Group != "" && - btp.Spec.Validation.CACertificateRefs[0].Group != "core" { + if selectedCertRef.Group != "" && + selectedCertRef.Group != "core" { path := field.NewPath("tls.cacertrefs[0].group") - valErr := field.NotSupported(path, btp.Spec.Validation.CACertificateRefs[0].Group, []string{"", "core"}) + valErr := field.NotSupported(path, selectedCertRef.Group, []string{"", "core"}) return valErr } nsName := types.NamespacedName{ Namespace: btp.Namespace, - Name: string(btp.Spec.Validation.CACertificateRefs[0].Name), + Name: string(selectedCertRef.Name), } - if err := configMapResolver.resolve(nsName); err != nil { - path := field.NewPath("tls.cacertrefs[0]") - return field.Invalid(path, btp.Spec.Validation.CACertificateRefs[0], err.Error()) + + if selectedCertRef.Kind == "ConfigMap" { + if err := configMapResolver.resolve(nsName); err != nil { + path := field.NewPath("tls.cacertrefs[0]") + return field.Invalid(path, selectedCertRef, err.Error()) + } + } else if selectedCertRef.Kind == "Secret" { + if err := secretResolver.resolve(nsName); err != nil { + path := field.NewPath("tls.cacertrefs[0]") + return field.Invalid(path, selectedCertRef, err.Error()) + } + } else { + return fmt.Errorf("`%s` invalid certificate reference supported", selectedCertRef.Kind) } return nil } diff --git a/internal/mode/static/state/graph/backend_tls_policy_test.go b/internal/mode/static/state/graph/backend_tls_policy_test.go index 791b8f8ffc..b3697ed4af 100644 --- a/internal/mode/static/state/graph/backend_tls_policy_test.go +++ b/internal/mode/static/state/graph/backend_tls_policy_test.go @@ -75,7 +75,7 @@ func TestProcessBackendTLSPoliciesEmpty(t *testing.T) { t.Parallel() g := NewWithT(t) - processed := processBackendTLSPolicies(test.backendTLSPolicies, nil, "test", test.gateway) + processed := processBackendTLSPolicies(test.backendTLSPolicies, nil, nil, "test", test.gateway) g.Expect(processed).To(Equal(test.expected)) }) @@ -404,13 +404,31 @@ func TestValidateBackendTLSPolicy(t *testing.T) { }, } + secretMaps := map[types.NamespacedName]*v1.Secret{ + {Namespace: "test", Name: "test-secret"}: { + ObjectMeta: metav1.ObjectMeta{ + Name: "test-secret", + Namespace: "test", + }, + // FILL IN + }, + {Namespace: "test", Name: "invalid-secret"}: { + ObjectMeta: metav1.ObjectMeta{ + Name: "invalid-secret", + Namespace: "test", + }, + }, + // FILL IN + } + configMapResolver := newConfigMapResolver(configMaps) + secretMapResolver := newSecretResolver(secretMaps) for _, test := range tests { t.Run(test.name, func(t *testing.T) { g := NewWithT(t) - valid, ignored, conds := validateBackendTLSPolicy(test.tlsPolicy, configMapResolver, "test") + valid, ignored, conds := validateBackendTLSPolicy(test.tlsPolicy, configMapResolver, secretMapResolver, "test") g.Expect(valid).To(Equal(test.isValid)) g.Expect(ignored).To(Equal(test.ignored)) diff --git a/internal/mode/static/state/graph/certificate_bundle.go b/internal/mode/static/state/graph/certificate_bundle.go new file mode 100644 index 0000000000..c3074ea065 --- /dev/null +++ b/internal/mode/static/state/graph/certificate_bundle.go @@ -0,0 +1,66 @@ +package graph + +import ( + "crypto/tls" + "crypto/x509" + "encoding/base64" + "encoding/pem" + "fmt" + + "k8s.io/apimachinery/pkg/types" + v1 "sigs.k8s.io/gateway-api/apis/v1" +) + +const CAKey = "ca.crt" + +type CertificateBundle struct { + Name types.NamespacedName + Kind v1.Kind + Cert *Certificate +} + +type Certificate struct { + TLSCert []byte + TLSPrivateKey []byte + CACert []byte +} + +func NewCertificateBundle(name types.NamespacedName, kind string, cert *Certificate) *CertificateBundle { + return &CertificateBundle{ + Name: name, + Kind: v1.Kind(kind), + Cert: cert, + } +} + +func validateTLS(tlsCert, tlsPrivateKey []byte) error { + _, err := tls.X509KeyPair(tlsCert, tlsPrivateKey) + if err != nil { + return fmt.Errorf("TLS secret is invalid: %w", err) + } + + return nil +} + +// validateCA validates the ca.crt entry in the Certificate. If it is valid, the function returns nil. +func validateCA(caData []byte) error { + data := make([]byte, base64.StdEncoding.DecodedLen(len(caData))) + _, err := base64.StdEncoding.Decode(data, caData) + if err != nil { + data = caData + } + block, _ := pem.Decode(data) + if block == nil { + return fmt.Errorf("the data field %s must hold a valid CERTIFICATE PEM block", CAKey) + } + if block.Type != "CERTIFICATE" { + return fmt.Errorf("the data field %s must hold a valid CERTIFICATE PEM block, but got '%s'", CAKey, block.Type) + } + + _, err = x509.ParseCertificate(block.Bytes) + if err != nil { + return fmt.Errorf("failed to validate certificate: %w", err) + } + + return nil +} diff --git a/internal/mode/static/state/graph/configmaps.go b/internal/mode/static/state/graph/configmaps.go index 23bec08c63..de914f9120 100644 --- a/internal/mode/static/state/graph/configmaps.go +++ b/internal/mode/static/state/graph/configmaps.go @@ -1,9 +1,6 @@ package graph import ( - "crypto/x509" - "encoding/base64" - "encoding/pem" "errors" "fmt" @@ -11,14 +8,11 @@ import ( "k8s.io/apimachinery/pkg/types" ) -const CAKey = "ca.crt" - // CaCertConfigMap represents a ConfigMap resource that holds CA Cert data. type CaCertConfigMap struct { // Source holds the actual ConfigMap resource. Can be nil if the ConfigMap does not exist. - Source *apiv1.ConfigMap - // CACert holds the actual CA Cert data. - CACert []byte + Source *apiv1.ConfigMap + CertBundle *CertificateBundle } type caCertConfigMapEntry struct { @@ -49,7 +43,7 @@ func (r *configMapResolver) resolve(nsname types.NamespacedName) error { cm, exist := r.clusterConfigMaps[nsname] var validationErr error - var caCert []byte + cert := &Certificate{} if !exist { validationErr = errors.New("ConfigMap does not exist") @@ -57,24 +51,24 @@ func (r *configMapResolver) resolve(nsname types.NamespacedName) error { if cm.Data != nil { if _, exists := cm.Data[CAKey]; exists { validationErr = validateCA([]byte(cm.Data[CAKey])) - caCert = []byte(cm.Data[CAKey]) + cert.CACert = []byte(cm.Data[CAKey]) } } if cm.BinaryData != nil { if _, exists := cm.BinaryData[CAKey]; exists { validationErr = validateCA(cm.BinaryData[CAKey]) - caCert = cm.BinaryData[CAKey] + cert.CACert = cm.BinaryData[CAKey] } } - if len(caCert) == 0 { + if len(cert.CACert) == 0 { validationErr = fmt.Errorf("ConfigMap does not have the data or binaryData field %v", CAKey) } } r.resolvedCaCertConfigMaps[nsname] = &caCertConfigMapEntry{ caCertConfigMap: CaCertConfigMap{ - Source: cm, - CACert: caCert, + Source: cm, + CertBundle: NewCertificateBundle(nsname, "ConfigMap", cert), }, err: validationErr, } @@ -97,26 +91,3 @@ func (r *configMapResolver) getResolvedConfigMaps() map[types.NamespacedName]*Ca return resolved } - -// validateCA validates the ca.crt entry in the ConfigMap. If it is valid, the function returns nil. -func validateCA(caData []byte) error { - data := make([]byte, base64.StdEncoding.DecodedLen(len(caData))) - _, err := base64.StdEncoding.Decode(data, caData) - if err != nil { - data = caData - } - block, _ := pem.Decode(data) - if block == nil { - return fmt.Errorf("the data field %s must hold a valid CERTIFICATE PEM block", CAKey) - } - if block.Type != "CERTIFICATE" { - return fmt.Errorf("the data field %s must hold a valid CERTIFICATE PEM block, but got '%s'", CAKey, block.Type) - } - - _, err = x509.ParseCertificate(block.Bytes) - if err != nil { - return fmt.Errorf("failed to validate certificate: %w", err) - } - - return nil -} diff --git a/internal/mode/static/state/graph/graph.go b/internal/mode/static/state/graph/graph.go index 0b56ec1018..a4466b7e44 100644 --- a/internal/mode/static/state/graph/graph.go +++ b/internal/mode/static/state/graph/graph.go @@ -59,7 +59,7 @@ type Graph struct { Routes map[RouteKey]*L7Route // L4Routes hold L4Route resources. L4Routes map[L4RouteKey]*L4Route - // ReferencedSecrets includes Secrets referenced by Gateway Listeners, including invalid ones. + // ReferencedSecrets includes Secrets referenced by Gateway Listeners or BackendTLSPolicies, including invalid ones. // It is different from the other maps, because it includes entries for Secrets that do not exist // in the cluster. We need such entries so that we can query the Graph to determine if a Secret is referenced // by the Gateway, including the case when the Secret is newly created. @@ -230,6 +230,7 @@ func BuildGraph( processedBackendTLSPolicies := processBackendTLSPolicies( state.BackendTLSPolicies, configMapResolver, + secretResolver, controllerName, gw, ) diff --git a/internal/mode/static/state/graph/graph_test.go b/internal/mode/static/state/graph/graph_test.go index f2b71d05f8..8bdc78e3d7 100644 --- a/internal/mode/static/state/graph/graph_test.go +++ b/internal/mode/static/state/graph/graph_test.go @@ -41,6 +41,9 @@ func TestBuildGraph(t *testing.T) { } cm := &v1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + Kind: "ConfigMap", + }, ObjectMeta: metav1.ObjectMeta{ Name: "configmap", Namespace: "service", @@ -338,6 +341,9 @@ func TestBuildGraph(t *testing.T) { } secret := &v1.Secret{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + }, ObjectMeta: metav1.ObjectMeta{ Namespace: testNs, Name: "secret", @@ -888,6 +894,10 @@ func TestBuildGraph(t *testing.T) { ReferencedSecrets: map[types.NamespacedName]*Secret{ client.ObjectKeyFromObject(secret): { Source: secret, + CertBundle: NewCertificateBundle(client.ObjectKeyFromObject(secret), "Secret", &Certificate{ + TLSCert: cert, + TLSPrivateKey: key, + }), }, }, ReferencedNamespaces: map[types.NamespacedName]*v1.Namespace{ @@ -900,7 +910,9 @@ func TestBuildGraph(t *testing.T) { ReferencedCaCertConfigMaps: map[types.NamespacedName]*CaCertConfigMap{ client.ObjectKeyFromObject(cm): { Source: cm, - CACert: []byte(caBlock), + CertBundle: NewCertificateBundle(client.ObjectKeyFromObject(cm), "ConfigMap", &Certificate{ + CACert: []byte(caBlock), + }), }, }, BackendTLSPolicies: map[types.NamespacedName]*BackendTLSPolicy{ @@ -1162,7 +1174,9 @@ func TestIsReferenced(t *testing.T) { ReferencedCaCertConfigMaps: map[types.NamespacedName]*CaCertConfigMap{ client.ObjectKeyFromObject(baseConfigMap): { Source: baseConfigMap, - CACert: []byte(caBlock), + CertBundle: NewCertificateBundle(client.ObjectKeyFromObject(baseConfigMap), "ConfigMap", &Certificate{ + CACert: []byte(caBlock), + }), }, }, } diff --git a/internal/mode/static/state/graph/secret.go b/internal/mode/static/state/graph/secret.go index c7d339aae4..6c1a26451e 100644 --- a/internal/mode/static/state/graph/secret.go +++ b/internal/mode/static/state/graph/secret.go @@ -1,7 +1,6 @@ package graph import ( - "crypto/tls" "errors" "fmt" @@ -13,6 +12,9 @@ import ( type Secret struct { // Source holds the actual Secret resource. Can be nil if the Secret does not exist. Source *apiv1.Secret + + // CertBundle holds actual certificate data. + CertBundle *CertificateBundle } type secretEntry struct { @@ -43,6 +45,7 @@ func (r *secretResolver) resolve(nsname types.NamespacedName) error { secret, exist := r.clusterSecrets[nsname] var validationErr error + var certBundle *CertificateBundle switch { case !exist: @@ -53,15 +56,26 @@ func (r *secretResolver) resolve(nsname types.NamespacedName) error { default: // A TLS Secret is guaranteed to have these data fields. - _, err := tls.X509KeyPair(secret.Data[apiv1.TLSCertKey], secret.Data[apiv1.TLSPrivateKeyKey]) - if err != nil { - validationErr = fmt.Errorf("TLS secret is invalid: %w", err) + cert := &Certificate{ + TLSCert: secret.Data[apiv1.TLSCertKey], + TLSPrivateKey: secret.Data[apiv1.TLSPrivateKeyKey], + } + + // Not always guaranteed to have a ca certificate in the secret. + if _, exists := secret.Data[CAKey]; exists { + cert.CACert = secret.Data[CAKey] + validationErr = validateCA(cert.CACert) } + + validationErr = validateTLS(cert.TLSCert, cert.TLSPrivateKey) + + certBundle = NewCertificateBundle(nsname, "Secret", cert) } r.resolvedSecrets[nsname] = &secretEntry{ Secret: Secret{ - Source: secret, + Source: secret, + CertBundle: certBundle, }, err: validationErr, } diff --git a/internal/mode/static/state/graph/secret_test.go b/internal/mode/static/state/graph/secret_test.go index 8c386b8528..1493d081b0 100644 --- a/internal/mode/static/state/graph/secret_test.go +++ b/internal/mode/static/state/graph/secret_test.go @@ -201,15 +201,27 @@ func TestSecretResolver(t *testing.T) { expectedResolved := map[types.NamespacedName]*Secret{ client.ObjectKeyFromObject(validSecret1): { Source: validSecret1, + CertBundle: NewCertificateBundle(client.ObjectKeyFromObject(validSecret1), "Secret", &Certificate{ + TLSCert: cert, + TLSPrivateKey: key, + }), }, client.ObjectKeyFromObject(invalidSecretType): { Source: invalidSecretType, }, client.ObjectKeyFromObject(invalidSecretCert): { Source: invalidSecretCert, + CertBundle: NewCertificateBundle(client.ObjectKeyFromObject(invalidSecretCert), "Secret", &Certificate{ + TLSCert: invalidCert, + TLSPrivateKey: key, + }), }, client.ObjectKeyFromObject(invalidSecretKey): { Source: invalidSecretKey, + CertBundle: NewCertificateBundle(client.ObjectKeyFromObject(invalidSecretKey), "Secret", &Certificate{ + TLSCert: cert, + TLSPrivateKey: invalidKey, + }), }, secretNotExistNsName: { Source: nil, From 95a170ed215b88f371d12013caded44ce9b522a1 Mon Sep 17 00:00:00 2001 From: Vinnie Marone Date: Mon, 10 Feb 2025 22:14:07 -0500 Subject: [PATCH 2/9] feedback changes, style --- internal/mode/static/state/dataplane/configuration.go | 6 ++++-- internal/mode/static/state/graph/backend_tls_policy.go | 9 +++++---- internal/mode/static/state/graph/certificate_bundle.go | 4 ++-- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/internal/mode/static/state/dataplane/configuration.go b/internal/mode/static/state/dataplane/configuration.go index c07ba6b6cd..6e67b8bb45 100644 --- a/internal/mode/static/state/dataplane/configuration.go +++ b/internal/mode/static/state/dataplane/configuration.go @@ -72,7 +72,8 @@ func BuildConfiguration( Version: configVersion, CertBundles: buildCertBundles( buildRefCertificateBundles(g.ReferencedSecrets, g.ReferencedCaCertConfigMaps), - backendGroups), + backendGroups, + ), Telemetry: buildTelemetry(g), BaseHTTPConfig: baseHTTPConfig, Logging: buildLogging(g), @@ -241,7 +242,8 @@ func buildSSLKeyPairs( func buildRefCertificateBundles( secrets map[types.NamespacedName]*graph.Secret, - configMaps map[types.NamespacedName]*graph.CaCertConfigMap) []graph.CertificateBundle { + configMaps map[types.NamespacedName]*graph.CaCertConfigMap, +) []graph.CertificateBundle { bundles := []graph.CertificateBundle{} for _, secret := range secrets { diff --git a/internal/mode/static/state/graph/backend_tls_policy.go b/internal/mode/static/state/graph/backend_tls_policy.go index 1d146bdfb2..9bbb692e24 100644 --- a/internal/mode/static/state/graph/backend_tls_policy.go +++ b/internal/mode/static/state/graph/backend_tls_policy.go @@ -154,18 +154,19 @@ func validateBackendTLSCACertRef(btp *v1alpha3.BackendTLSPolicy, configMapResolv Name: string(selectedCertRef.Name), } - if selectedCertRef.Kind == "ConfigMap" { + switch selectedCertRef.Kind { + case "ConfigMap": if err := configMapResolver.resolve(nsName); err != nil { path := field.NewPath("tls.cacertrefs[0]") return field.Invalid(path, selectedCertRef, err.Error()) } - } else if selectedCertRef.Kind == "Secret" { + case "Secret": if err := secretResolver.resolve(nsName); err != nil { path := field.NewPath("tls.cacertrefs[0]") return field.Invalid(path, selectedCertRef, err.Error()) } - } else { - return fmt.Errorf("`%s` invalid certificate reference supported", selectedCertRef.Kind) + default: + return fmt.Errorf("invalid certificate reference supported %q", selectedCertRef.Kind) } return nil } diff --git a/internal/mode/static/state/graph/certificate_bundle.go b/internal/mode/static/state/graph/certificate_bundle.go index c3074ea065..470c36a85d 100644 --- a/internal/mode/static/state/graph/certificate_bundle.go +++ b/internal/mode/static/state/graph/certificate_bundle.go @@ -51,10 +51,10 @@ func validateCA(caData []byte) error { } block, _ := pem.Decode(data) if block == nil { - return fmt.Errorf("the data field %s must hold a valid CERTIFICATE PEM block", CAKey) + return fmt.Errorf("the data field %q must hold a valid CERTIFICATE PEM block", CAKey) } if block.Type != "CERTIFICATE" { - return fmt.Errorf("the data field %s must hold a valid CERTIFICATE PEM block, but got '%s'", CAKey, block.Type) + return fmt.Errorf("the data field %q must hold a valid CERTIFICATE PEM block, but got %q", CAKey, block.Type) } _, err = x509.ParseCertificate(block.Bytes) From 7640e54710935fc86ab9de09d85bb9ec861cd57c Mon Sep 17 00:00:00 2001 From: Vinnie Marone Date: Wed, 12 Feb 2025 18:18:31 -0500 Subject: [PATCH 3/9] feedback changes and linter fixes --- internal/mode/static/state/graph/backend_tls_policy.go | 6 +++++- internal/mode/static/state/graph/certificate_bundle.go | 3 ++- internal/mode/static/state/graph/secret.go | 3 +-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/internal/mode/static/state/graph/backend_tls_policy.go b/internal/mode/static/state/graph/backend_tls_policy.go index 9bbb692e24..923fd0d6aa 100644 --- a/internal/mode/static/state/graph/backend_tls_policy.go +++ b/internal/mode/static/state/graph/backend_tls_policy.go @@ -128,7 +128,11 @@ func validateBackendTLSHostname(btp *v1alpha3.BackendTLSPolicy) error { return nil } -func validateBackendTLSCACertRef(btp *v1alpha3.BackendTLSPolicy, configMapResolver *configMapResolver, secretResolver *secretResolver) error { +func validateBackendTLSCACertRef( + btp *v1alpha3.BackendTLSPolicy, + configMapResolver *configMapResolver, + secretResolver *secretResolver, +) error { if len(btp.Spec.Validation.CACertificateRefs) != 1 { path := field.NewPath("tls.cacertrefs") valErr := field.TooMany(path, len(btp.Spec.Validation.CACertificateRefs), 1) diff --git a/internal/mode/static/state/graph/certificate_bundle.go b/internal/mode/static/state/graph/certificate_bundle.go index 470c36a85d..e12642ac38 100644 --- a/internal/mode/static/state/graph/certificate_bundle.go +++ b/internal/mode/static/state/graph/certificate_bundle.go @@ -14,9 +14,10 @@ import ( const CAKey = "ca.crt" type CertificateBundle struct { + Cert *Certificate + Name types.NamespacedName Kind v1.Kind - Cert *Certificate } type Certificate struct { diff --git a/internal/mode/static/state/graph/secret.go b/internal/mode/static/state/graph/secret.go index 6c1a26451e..c89b3ab228 100644 --- a/internal/mode/static/state/graph/secret.go +++ b/internal/mode/static/state/graph/secret.go @@ -60,6 +60,7 @@ func (r *secretResolver) resolve(nsname types.NamespacedName) error { TLSCert: secret.Data[apiv1.TLSCertKey], TLSPrivateKey: secret.Data[apiv1.TLSPrivateKeyKey], } + validationErr = validateTLS(cert.TLSCert, cert.TLSPrivateKey) // Not always guaranteed to have a ca certificate in the secret. if _, exists := secret.Data[CAKey]; exists { @@ -67,8 +68,6 @@ func (r *secretResolver) resolve(nsname types.NamespacedName) error { validationErr = validateCA(cert.CACert) } - validationErr = validateTLS(cert.TLSCert, cert.TLSPrivateKey) - certBundle = NewCertificateBundle(nsname, "Secret", cert) } From 23e6d75c0396ca604edb04df0c51222c2c323ff9 Mon Sep 17 00:00:00 2001 From: Vinnie Marone Date: Fri, 14 Feb 2025 15:11:30 -0500 Subject: [PATCH 4/9] add secret test --- .../state/graph/backend_tls_policy_test.go | 51 ++++++++++++++++--- 1 file changed, 43 insertions(+), 8 deletions(-) diff --git a/internal/mode/static/state/graph/backend_tls_policy_test.go b/internal/mode/static/state/graph/backend_tls_policy_test.go index bda8c3fb4a..cea42d64e9 100644 --- a/internal/mode/static/state/graph/backend_tls_policy_test.go +++ b/internal/mode/static/state/graph/backend_tls_policy_test.go @@ -83,6 +83,7 @@ func TestProcessBackendTLSPoliciesEmpty(t *testing.T) { } func TestValidateBackendTLSPolicy(t *testing.T) { + const testSecretName string = "test-secret" targetRefNormalCase := []v1alpha2.LocalPolicyTargetReferenceWithSectionName{ { LocalPolicyTargetReference: v1alpha2.LocalPolicyTargetReference{ @@ -100,6 +101,14 @@ func TestValidateBackendTLSPolicy(t *testing.T) { }, } + localObjectRefSecretNormalCase := []gatewayv1.LocalObjectReference{ + { + Kind: "Secret", + Name: gatewayv1.ObjectName(testSecretName), + Group: "", + }, + } + localObjectRefInvalidName := []gatewayv1.LocalObjectReference{ { Kind: "ConfigMap", @@ -196,6 +205,23 @@ func TestValidateBackendTLSPolicy(t *testing.T) { }, isValid: true, }, + { + name: "normal case with ca cert ref secrets", + tlsPolicy: &v1alpha3.BackendTLSPolicy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tls-policy", + Namespace: "test", + }, + Spec: v1alpha3.BackendTLSPolicySpec{ + TargetRefs: targetRefNormalCase, + Validation: v1alpha3.BackendTLSPolicyValidation{ + CACertificateRefs: localObjectRefSecretNormalCase, + Hostname: "foo.test.com", + }, + }, + }, + isValid: true, + }, { name: "normal case with ca cert refs and 16 ancestors including us", tlsPolicy: &v1alpha3.BackendTLSPolicy{ @@ -390,7 +416,7 @@ func TestValidateBackendTLSPolicy(t *testing.T) { Namespace: "test", }, Data: map[string]string{ - "ca.crt": caBlock, + CAKey: caBlock, }, }, {Namespace: "test", Name: "invalid"}: { @@ -399,26 +425,35 @@ func TestValidateBackendTLSPolicy(t *testing.T) { Namespace: "test", }, Data: map[string]string{ - "ca.crt": "invalid", + CAKey: "invalid", }, }, } secretMaps := map[types.NamespacedName]*v1.Secret{ - {Namespace: "test", Name: "test-secret"}: { + {Namespace: "test", Name: testSecretName}: { ObjectMeta: metav1.ObjectMeta{ - Name: "test-secret", + Name: testSecretName, Namespace: "test", }, - // FILL IN + Type: v1.SecretTypeTLS, + Data: map[string][]byte{ + v1.TLSCertKey: cert, + v1.TLSPrivateKeyKey: key, + CAKey: []byte(caBlock), + }, }, {Namespace: "test", Name: "invalid-secret"}: { ObjectMeta: metav1.ObjectMeta{ Name: "invalid-secret", Namespace: "test", }, + Data: map[string][]byte{ + v1.TLSCertKey: invalidCert, + v1.TLSPrivateKeyKey: invalidKey, + CAKey: []byte("invalid-cert"), + }, }, - // FILL IN } configMapResolver := newConfigMapResolver(configMaps) @@ -430,13 +465,13 @@ func TestValidateBackendTLSPolicy(t *testing.T) { valid, ignored, conds := validateBackendTLSPolicy(test.tlsPolicy, configMapResolver, secretMapResolver, "test") - g.Expect(valid).To(Equal(test.isValid)) - g.Expect(ignored).To(Equal(test.ignored)) if !test.isValid && !test.ignored { g.Expect(conds).To(HaveLen(1)) } else { g.Expect(conds).To(BeEmpty()) } + g.Expect(valid).To(Equal(test.isValid)) + g.Expect(ignored).To(Equal(test.ignored)) }) } } From 070abeda53540feb546be551d62d575cc19d4041 Mon Sep 17 00:00:00 2001 From: Vinnie Marone Date: Tue, 18 Feb 2025 21:20:57 -0500 Subject: [PATCH 5/9] implement more tests in secrets test, move validateca test to cert bundle, language feedback, comments --- .../static/state/dataplane/configuration.go | 7 +-- .../static/state/graph/backend_tls_policy.go | 2 +- .../static/state/graph/certificate_bundle.go | 14 ++++- .../state/graph/certificate_bundle_test.go | 55 ++++++++++++++++ .../static/state/graph/configmaps_test.go | 48 -------------- internal/mode/static/state/graph/secret.go | 3 + .../mode/static/state/graph/secret_test.go | 63 +++++++++++++++++-- 7 files changed, 132 insertions(+), 60 deletions(-) create mode 100644 internal/mode/static/state/graph/certificate_bundle_test.go diff --git a/internal/mode/static/state/dataplane/configuration.go b/internal/mode/static/state/dataplane/configuration.go index 6e67b8bb45..d3c86fcb3e 100644 --- a/internal/mode/static/state/dataplane/configuration.go +++ b/internal/mode/static/state/dataplane/configuration.go @@ -6,7 +6,6 @@ import ( "fmt" "sort" - apiv1 "k8s.io/api/core/v1" discoveryV1 "k8s.io/api/discovery/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -229,10 +228,10 @@ func buildSSLKeyPairs( id := generateSSLKeyPairID(*l.ResolvedSecret) secret := secrets[*l.ResolvedSecret] // The Data map keys are guaranteed to exist by the graph package. - // the Source field is guaranteed to be non-nil by the graph package. + // the CertBundle field is guaranteed to be non-nil by the graph package. keyPairs[id] = SSLKeyPair{ - Cert: secret.Source.Data[apiv1.TLSCertKey], - Key: secret.Source.Data[apiv1.TLSPrivateKeyKey], + Cert: secret.CertBundle.Cert.TLSCert, + Key: secret.CertBundle.Cert.TLSPrivateKey, } } } diff --git a/internal/mode/static/state/graph/backend_tls_policy.go b/internal/mode/static/state/graph/backend_tls_policy.go index 923fd0d6aa..4e00eecc56 100644 --- a/internal/mode/static/state/graph/backend_tls_policy.go +++ b/internal/mode/static/state/graph/backend_tls_policy.go @@ -170,7 +170,7 @@ func validateBackendTLSCACertRef( return field.Invalid(path, selectedCertRef, err.Error()) } default: - return fmt.Errorf("invalid certificate reference supported %q", selectedCertRef.Kind) + return fmt.Errorf("invalid certificate reference kind %q", selectedCertRef.Kind) } return nil } diff --git a/internal/mode/static/state/graph/certificate_bundle.go b/internal/mode/static/state/graph/certificate_bundle.go index e12642ac38..cdd3b07a94 100644 --- a/internal/mode/static/state/graph/certificate_bundle.go +++ b/internal/mode/static/state/graph/certificate_bundle.go @@ -11,8 +11,12 @@ import ( v1 "sigs.k8s.io/gateway-api/apis/v1" ) +// CAKey is the key that is stored in a secret or configmap to grab its data from. +// This follows the convention setup by kubernetes service account root ca +// key for optional root certificate authority const CAKey = "ca.crt" +// CertificateBundle is used to submit certificate data to nginx that is kubernetes aware type CertificateBundle struct { Cert *Certificate @@ -20,12 +24,17 @@ type CertificateBundle struct { Kind v1.Kind } +// Certificate houses the real certificate data that is sent to the configurator type Certificate struct { - TLSCert []byte + // TLSCert is the SSL certificate used to send to CA + TLSCert []byte + // TLSPrivateKey is the cryptographic key for encrpyting traffic during secure TLS TLSPrivateKey []byte - CACert []byte + // CACert is the root certificate authority + CACert []byte } +// NewCertificateBundle generates a kubernetes aware certificate that is used during the configurator for nginx func NewCertificateBundle(name types.NamespacedName, kind string, cert *Certificate) *CertificateBundle { return &CertificateBundle{ Name: name, @@ -34,6 +43,7 @@ func NewCertificateBundle(name types.NamespacedName, kind string, cert *Certific } } +// validateTLS ... func validateTLS(tlsCert, tlsPrivateKey []byte) error { _, err := tls.X509KeyPair(tlsCert, tlsPrivateKey) if err != nil { diff --git a/internal/mode/static/state/graph/certificate_bundle_test.go b/internal/mode/static/state/graph/certificate_bundle_test.go new file mode 100644 index 0000000000..1e3e207862 --- /dev/null +++ b/internal/mode/static/state/graph/certificate_bundle_test.go @@ -0,0 +1,55 @@ +package graph + +import ( + "encoding/base64" + "testing" + + . "github.com/onsi/gomega" +) + +func TestValidateCA(t *testing.T) { + t.Parallel() + base64Data := make([]byte, base64.StdEncoding.EncodedLen(len(caBlock))) + base64.StdEncoding.Encode(base64Data, []byte(caBlock)) + + tests := []struct { + name string + data []byte + errorExpected bool + }{ + { + name: "valid base64", + data: base64Data, + errorExpected: false, + }, + { + name: "valid plain text", + data: []byte(caBlock), + errorExpected: false, + }, + { + name: "invalid pem", + data: []byte("invalid"), + errorExpected: true, + }, + { + name: "invalid type", + data: []byte(caBlockInvalidType), + errorExpected: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + err := validateCA(test.data) + if test.errorExpected { + g.Expect(err).To(HaveOccurred()) + } else { + g.Expect(err).ToNot(HaveOccurred()) + } + }) + } +} diff --git a/internal/mode/static/state/graph/configmaps_test.go b/internal/mode/static/state/graph/configmaps_test.go index 852d7c8917..ca5c725a38 100644 --- a/internal/mode/static/state/graph/configmaps_test.go +++ b/internal/mode/static/state/graph/configmaps_test.go @@ -1,7 +1,6 @@ package graph import ( - "encoding/base64" "testing" . "github.com/onsi/gomega" @@ -87,53 +86,6 @@ UdxohGqleWFMQ3UNLOvc9Fk+q72ryg== ` ) -func TestValidateCA(t *testing.T) { - t.Parallel() - base64Data := make([]byte, base64.StdEncoding.EncodedLen(len(caBlock))) - base64.StdEncoding.Encode(base64Data, []byte(caBlock)) - - tests := []struct { - name string - data []byte - errorExpected bool - }{ - { - name: "valid base64", - data: base64Data, - errorExpected: false, - }, - { - name: "valid plain text", - data: []byte(caBlock), - errorExpected: false, - }, - { - name: "invalid pem", - data: []byte("invalid"), - errorExpected: true, - }, - { - name: "invalid type", - data: []byte(caBlockInvalidType), - errorExpected: true, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - t.Parallel() - g := NewWithT(t) - - err := validateCA(test.data) - if test.errorExpected { - g.Expect(err).To(HaveOccurred()) - } else { - g.Expect(err).ToNot(HaveOccurred()) - } - }) - } -} - func TestResolve(t *testing.T) { configMaps := map[types.NamespacedName]*v1.ConfigMap{ {Namespace: "test", Name: "configmap1"}: { diff --git a/internal/mode/static/state/graph/secret.go b/internal/mode/static/state/graph/secret.go index c89b3ab228..7515c514f7 100644 --- a/internal/mode/static/state/graph/secret.go +++ b/internal/mode/static/state/graph/secret.go @@ -63,6 +63,9 @@ func (r *secretResolver) resolve(nsname types.NamespacedName) error { validationErr = validateTLS(cert.TLSCert, cert.TLSPrivateKey) // Not always guaranteed to have a ca certificate in the secret. + // Cert-Manager puts this at ca.crt and thus this is statically placed like so. + // To follow the convention setup by kubernetes for a service account root ca + // for optional root certificate authority if _, exists := secret.Data[CAKey]; exists { cert.CACert = secret.Data[CAKey] validationErr = validateCA(cert.CACert) diff --git a/internal/mode/static/state/graph/secret_test.go b/internal/mode/static/state/graph/secret_test.go index 1493d081b0..cfcbbd2865 100644 --- a/internal/mode/static/state/graph/secret_test.go +++ b/internal/mode/static/state/graph/secret_test.go @@ -93,6 +93,19 @@ func TestSecretResolver(t *testing.T) { Type: apiv1.SecretTypeTLS, } + validSecret3 = &apiv1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "test", + Name: "secret-3", + }, + Data: map[string][]byte{ + apiv1.TLSCertKey: cert, + apiv1.TLSPrivateKeyKey: key, + CAKey: []byte(caBlock), + }, + Type: apiv1.SecretTypeTLS, + } + invalidSecretType = &apiv1.Secret{ ObjectMeta: metav1.ObjectMeta{ Namespace: "test", @@ -129,6 +142,19 @@ func TestSecretResolver(t *testing.T) { Type: apiv1.SecretTypeTLS, } + invalidSecretCaCert = &apiv1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "test", + Name: "invalid-ca-key", + }, + Data: map[string][]byte{ + apiv1.TLSCertKey: cert, + apiv1.TLSPrivateKeyKey: key, + CAKey: invalidCert, + }, + Type: apiv1.SecretTypeTLS, + } + secretNotExistNsName = types.NamespacedName{ Namespace: "test", Name: "not-exist", @@ -137,11 +163,13 @@ func TestSecretResolver(t *testing.T) { resolver := newSecretResolver( map[types.NamespacedName]*apiv1.Secret{ - client.ObjectKeyFromObject(validSecret1): validSecret1, - client.ObjectKeyFromObject(validSecret2): validSecret2, // we're not going to resolve it - client.ObjectKeyFromObject(invalidSecretType): invalidSecretType, - client.ObjectKeyFromObject(invalidSecretCert): invalidSecretCert, - client.ObjectKeyFromObject(invalidSecretKey): invalidSecretKey, + client.ObjectKeyFromObject(validSecret1): validSecret1, + client.ObjectKeyFromObject(validSecret2): validSecret2, // we're not going to resolve it + client.ObjectKeyFromObject(validSecret3): validSecret3, + client.ObjectKeyFromObject(invalidSecretType): invalidSecretType, + client.ObjectKeyFromObject(invalidSecretCert): invalidSecretCert, + client.ObjectKeyFromObject(invalidSecretKey): invalidSecretKey, + client.ObjectKeyFromObject(invalidSecretCaCert): invalidSecretCaCert, }) tests := []struct { @@ -157,6 +185,10 @@ func TestSecretResolver(t *testing.T) { name: "valid secret, again", nsname: client.ObjectKeyFromObject(validSecret1), }, + { + name: "valid secret, with ca certificate", + nsname: client.ObjectKeyFromObject(validSecret3), + }, { name: "doesn't exist", nsname: secretNotExistNsName, @@ -182,6 +214,11 @@ func TestSecretResolver(t *testing.T) { nsname: client.ObjectKeyFromObject(invalidSecretKey), expectedErrMsg: "TLS secret is invalid: tls: failed to parse private key", }, + { + name: "invalid secret ca cert", + nsname: client.ObjectKeyFromObject(invalidSecretCaCert), + expectedErrMsg: "failed to validate certificate: x509: malformed certificate", + }, } // Not running tests with t.Run(...) because the last one (getResolvedSecrets) depends on the execution of @@ -206,6 +243,14 @@ func TestSecretResolver(t *testing.T) { TLSPrivateKey: key, }), }, + client.ObjectKeyFromObject(validSecret3): { + Source: validSecret3, + CertBundle: NewCertificateBundle(client.ObjectKeyFromObject(validSecret3), "Secret", &Certificate{ + TLSCert: cert, + TLSPrivateKey: key, + CACert: []byte(caBlock), + }), + }, client.ObjectKeyFromObject(invalidSecretType): { Source: invalidSecretType, }, @@ -223,6 +268,14 @@ func TestSecretResolver(t *testing.T) { TLSPrivateKey: invalidKey, }), }, + client.ObjectKeyFromObject(invalidSecretCaCert): { + Source: invalidSecretCaCert, + CertBundle: NewCertificateBundle(client.ObjectKeyFromObject(invalidSecretCaCert), "Secret", &Certificate{ + TLSCert: cert, + TLSPrivateKey: key, + CACert: invalidCert, + }), + }, secretNotExistNsName: { Source: nil, }, From e0220ae626656360ede33be65af0d801fd8025c0 Mon Sep 17 00:00:00 2001 From: Vinnie Marone Date: Tue, 18 Feb 2025 21:30:47 -0500 Subject: [PATCH 6/9] modify tests for new cert bundle and comments --- .../static/state/dataplane/configuration_test.go | 16 ++++++++++++++++ .../static/state/graph/certificate_bundle.go | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/internal/mode/static/state/dataplane/configuration_test.go b/internal/mode/static/state/dataplane/configuration_test.go index e84f1e736e..d8219a1c9a 100644 --- a/internal/mode/static/state/dataplane/configuration_test.go +++ b/internal/mode/static/state/dataplane/configuration_test.go @@ -680,6 +680,14 @@ func TestBuildConfiguration(t *testing.T) { apiv1.TLSPrivateKeyKey: []byte("privateKey-1"), }, }, + CertBundle: graph.NewCertificateBundle( + secret1NsName, + "Secret", + &graph.Certificate{ + TLSCert: []byte("cert-1"), + TLSPrivateKey: []byte("privateKey-1"), + }, + ), } secret2NsName := types.NamespacedName{Namespace: "test", Name: "secret-2"} @@ -694,6 +702,14 @@ func TestBuildConfiguration(t *testing.T) { apiv1.TLSPrivateKeyKey: []byte("privateKey-2"), }, }, + CertBundle: graph.NewCertificateBundle( + secret2NsName, + "Secret", + &graph.Certificate{ + TLSCert: []byte("cert-2"), + TLSPrivateKey: []byte("privateKey-2"), + }, + ), } listener80 := v1.Listener{ diff --git a/internal/mode/static/state/graph/certificate_bundle.go b/internal/mode/static/state/graph/certificate_bundle.go index cdd3b07a94..be0c388fce 100644 --- a/internal/mode/static/state/graph/certificate_bundle.go +++ b/internal/mode/static/state/graph/certificate_bundle.go @@ -43,7 +43,7 @@ func NewCertificateBundle(name types.NamespacedName, kind string, cert *Certific } } -// validateTLS ... +// validateTLS checks to make sure a ssl certificate key pair is valid func validateTLS(tlsCert, tlsPrivateKey []byte) error { _, err := tls.X509KeyPair(tlsCert, tlsPrivateKey) if err != nil { From 3ffbbaf1230fbfe64361bfb1234bd68691d35964 Mon Sep 17 00:00:00 2001 From: Vinnie Marone Date: Wed, 19 Feb 2025 22:38:51 -0500 Subject: [PATCH 7/9] add more tests to certificate bundle test --- .../static/state/graph/certificate_bundle.go | 14 +++--- .../state/graph/certificate_bundle_test.go | 49 +++++++++++++++++++ .../mode/static/state/graph/secret_test.go | 4 +- 3 files changed, 57 insertions(+), 10 deletions(-) diff --git a/internal/mode/static/state/graph/certificate_bundle.go b/internal/mode/static/state/graph/certificate_bundle.go index be0c388fce..a641de719d 100644 --- a/internal/mode/static/state/graph/certificate_bundle.go +++ b/internal/mode/static/state/graph/certificate_bundle.go @@ -11,12 +11,10 @@ import ( v1 "sigs.k8s.io/gateway-api/apis/v1" ) -// CAKey is the key that is stored in a secret or configmap to grab its data from. -// This follows the convention setup by kubernetes service account root ca -// key for optional root certificate authority +// key for optional root certificate authority. const CAKey = "ca.crt" -// CertificateBundle is used to submit certificate data to nginx that is kubernetes aware +// CertificateBundle is used to submit certificate data to nginx that is kubernetes aware. type CertificateBundle struct { Cert *Certificate @@ -24,7 +22,7 @@ type CertificateBundle struct { Kind v1.Kind } -// Certificate houses the real certificate data that is sent to the configurator +// Certificate houses the real certificate data that is sent to the configurator. type Certificate struct { // TLSCert is the SSL certificate used to send to CA TLSCert []byte @@ -34,7 +32,7 @@ type Certificate struct { CACert []byte } -// NewCertificateBundle generates a kubernetes aware certificate that is used during the configurator for nginx +// NewCertificateBundle generates a kubernetes aware certificate that is used during the configurator for nginx. func NewCertificateBundle(name types.NamespacedName, kind string, cert *Certificate) *CertificateBundle { return &CertificateBundle{ Name: name, @@ -43,11 +41,11 @@ func NewCertificateBundle(name types.NamespacedName, kind string, cert *Certific } } -// validateTLS checks to make sure a ssl certificate key pair is valid +// validateTLS checks to make sure a ssl certificate key pair is valid. func validateTLS(tlsCert, tlsPrivateKey []byte) error { _, err := tls.X509KeyPair(tlsCert, tlsPrivateKey) if err != nil { - return fmt.Errorf("TLS secret is invalid: %w", err) + return fmt.Errorf("tls secret is invalid: %w", err) } return nil diff --git a/internal/mode/static/state/graph/certificate_bundle_test.go b/internal/mode/static/state/graph/certificate_bundle_test.go index 1e3e207862..bc26f19176 100644 --- a/internal/mode/static/state/graph/certificate_bundle_test.go +++ b/internal/mode/static/state/graph/certificate_bundle_test.go @@ -7,6 +7,55 @@ import ( . "github.com/onsi/gomega" ) +func TestValidateTLS(t *testing.T) { + t.Parallel() + tests := []struct { + expectedErr string + name string + tlsCert []byte + tlsPrivateKey []byte + }{ + { + name: "valid tls key pair", + tlsCert: cert, + tlsPrivateKey: key, + }, + { + name: "invalid tls cert valid key", + tlsCert: invalidCert, + tlsPrivateKey: key, + expectedErr: "tls secret is invalid: x509: malformed certificate", + }, + { + name: "invalid tls private key valid cert", + tlsCert: cert, + tlsPrivateKey: invalidKey, + expectedErr: "tls secret is invalid: tls: failed to parse private key", + }, + { + name: "invalid tls cert key pair", + tlsCert: invalidCert, + tlsPrivateKey: invalidKey, + expectedErr: "tls secret is invalid: x509: malformed certificate", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + g := NewWithT(t) + err := validateTLS(test.tlsCert, test.tlsPrivateKey) + if test.expectedErr != "" { + g.Expect(err).To(HaveOccurred()) + g.Expect(err).To(MatchError(test.expectedErr)) + } else { + g.Expect(err).ToNot(HaveOccurred()) + } + }) + } +} + func TestValidateCA(t *testing.T) { t.Parallel() base64Data := make([]byte, base64.StdEncoding.EncodedLen(len(caBlock))) diff --git a/internal/mode/static/state/graph/secret_test.go b/internal/mode/static/state/graph/secret_test.go index cfcbbd2865..3ef2fbeec3 100644 --- a/internal/mode/static/state/graph/secret_test.go +++ b/internal/mode/static/state/graph/secret_test.go @@ -207,12 +207,12 @@ func TestSecretResolver(t *testing.T) { { name: "invalid secret cert", nsname: client.ObjectKeyFromObject(invalidSecretCert), - expectedErrMsg: "TLS secret is invalid: x509: malformed certificate", + expectedErrMsg: "tls secret is invalid: x509: malformed certificate", }, { name: "invalid secret key", nsname: client.ObjectKeyFromObject(invalidSecretKey), - expectedErrMsg: "TLS secret is invalid: tls: failed to parse private key", + expectedErrMsg: "tls secret is invalid: tls: failed to parse private key", }, { name: "invalid secret ca cert", From f4b7af49cd8b5482f90651a979efc6a6f503d56c Mon Sep 17 00:00:00 2001 From: Vinnie Marone Date: Wed, 19 Feb 2025 22:41:21 -0500 Subject: [PATCH 8/9] grammar linter fixes --- internal/mode/static/state/graph/certificate_bundle.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/mode/static/state/graph/certificate_bundle.go b/internal/mode/static/state/graph/certificate_bundle.go index a641de719d..177538867c 100644 --- a/internal/mode/static/state/graph/certificate_bundle.go +++ b/internal/mode/static/state/graph/certificate_bundle.go @@ -24,11 +24,11 @@ type CertificateBundle struct { // Certificate houses the real certificate data that is sent to the configurator. type Certificate struct { - // TLSCert is the SSL certificate used to send to CA + // TLSCert is the SSL certificate used to send to CA. TLSCert []byte - // TLSPrivateKey is the cryptographic key for encrpyting traffic during secure TLS + // TLSPrivateKey is the cryptographic key for encrpyting traffic during secure TLS. TLSPrivateKey []byte - // CACert is the root certificate authority + // CACert is the root certificate authority. CACert []byte } From fe46ad4e7c7f32108c6d3e68fc1648a896afb852 Mon Sep 17 00:00:00 2001 From: Vinnie Marone Date: Wed, 19 Feb 2025 22:45:52 -0500 Subject: [PATCH 9/9] adjust comment --- internal/mode/static/state/graph/certificate_bundle.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/mode/static/state/graph/certificate_bundle.go b/internal/mode/static/state/graph/certificate_bundle.go index 177538867c..4567fad074 100644 --- a/internal/mode/static/state/graph/certificate_bundle.go +++ b/internal/mode/static/state/graph/certificate_bundle.go @@ -11,7 +11,7 @@ import ( v1 "sigs.k8s.io/gateway-api/apis/v1" ) -// key for optional root certificate authority. +// CAKey certificate key for optional root certificate authority. const CAKey = "ca.crt" // CertificateBundle is used to submit certificate data to nginx that is kubernetes aware.