diff --git a/charts/nginx-gateway-fabric/templates/deployment.yaml b/charts/nginx-gateway-fabric/templates/deployment.yaml index 0c9a6c06b7..5e4eafff89 100644 --- a/charts/nginx-gateway-fabric/templates/deployment.yaml +++ b/charts/nginx-gateway-fabric/templates/deployment.yaml @@ -36,6 +36,14 @@ spec: - --gatewayclass={{ .Values.nginxGateway.gatewayClassName }} - --config={{ include "nginx-gateway.config-name" . }} - --service={{ include "nginx-gateway.fullname" . }} + {{- if .Values.nginx.imagePullSecret }} + - --nginx-docker-secret={{ .Values.nginx.imagePullSecret }} + {{- end }} + {{- if .Values.nginx.imagePullSecrets }} + {{- range .Values.nginx.imagePullSecrets }} + - --nginx-docker-secret={{ . }} + {{- end }} + {{- end }} {{- if .Values.nginx.plus }} - --nginx-plus {{- if .Values.nginx.usage.secretName }} diff --git a/cmd/gateway/commands.go b/cmd/gateway/commands.go index 0a572c922e..5652a9f2dd 100644 --- a/cmd/gateway/commands.go +++ b/cmd/gateway/commands.go @@ -69,6 +69,7 @@ func createStaticModeCommand() *cobra.Command { leaderElectionLockNameFlag = "leader-election-lock-name" productTelemetryDisableFlag = "product-telemetry-disable" gwAPIExperimentalFlag = "gateway-api-experimental-features" + nginxDockerSecretFlag = "nginx-docker-secret" //nolint:gosec // not credentials usageReportSecretFlag = "usage-report-secret" usageReportEndpointFlag = "usage-report-endpoint" usageReportResolverFlag = "usage-report-resolver" @@ -120,7 +121,10 @@ func createStaticModeCommand() *cobra.Command { snippetsFilters bool - plus bool + plus bool + nginxDockerSecrets = stringSliceValidatingValue{ + validator: validateResourceName, + } usageReportSkipVerify bool usageReportSecretName = stringValidatingValue{ validator: validateResourceName, @@ -249,7 +253,8 @@ func createStaticModeCommand() *cobra.Command { Names: flagKeys, Values: flagValues, }, - SnippetsFilters: snippetsFilters, + SnippetsFilters: snippetsFilters, + NginxDockerSecretNames: nginxDockerSecrets.values, } if err := static.StartManager(conf); err != nil { @@ -378,6 +383,13 @@ func createStaticModeCommand() *cobra.Command { "Requires the Gateway APIs installed from the experimental channel.", ) + cmd.Flags().Var( + &nginxDockerSecrets, + nginxDockerSecretFlag, + "The name of the NGINX docker registry Secret(s). Must exist in the same namespace "+ + "that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway).", + ) + cmd.Flags().Var( &usageReportSecretName, usageReportSecretFlag, diff --git a/cmd/gateway/commands_test.go b/cmd/gateway/commands_test.go index b58fa3331b..e89a5a91dd 100644 --- a/cmd/gateway/commands_test.go +++ b/cmd/gateway/commands_test.go @@ -153,6 +153,8 @@ func TestStaticModeCmdFlagValidation(t *testing.T) { "--leader-election-lock-name=my-lock", "--leader-election-disable=false", "--nginx-plus", + "--nginx-docker-secret=secret1", + "--nginx-docker-secret=secret2", "--usage-report-secret=my-secret", "--usage-report-endpoint=example.com", "--usage-report-resolver=resolver.com", @@ -314,6 +316,31 @@ func TestStaticModeCmdFlagValidation(t *testing.T) { wantErr: true, expectedErrPrefix: `invalid argument "" for "--leader-election-disable" flag: strconv.ParseBool`, }, + { + name: "nginx-docker-secret is set to empty string", + args: []string{ + "--nginx-docker-secret=", + }, + wantErr: true, + expectedErrPrefix: `invalid argument "" for "--nginx-docker-secret" flag: must be set`, + }, + { + name: "nginx-docker-secret is invalid", + args: []string{ + "--nginx-docker-secret=!@#$", + }, + wantErr: true, + expectedErrPrefix: `invalid argument "!@#$" for "--nginx-docker-secret" flag: invalid format: `, + }, + { + name: "one nginx-docker-secret is invalid", + args: []string{ + "--nginx-docker-secret=valid", + "--nginx-docker-secret=!@#$", + }, + wantErr: true, + expectedErrPrefix: `invalid argument "!@#$" for "--nginx-docker-secret" flag: invalid format: `, + }, { name: "usage-report-secret is set to empty string", args: []string{ diff --git a/cmd/gateway/validating_types.go b/cmd/gateway/validating_types.go index 42d24782cb..1db3eab8dc 100644 --- a/cmd/gateway/validating_types.go +++ b/cmd/gateway/validating_types.go @@ -1,8 +1,11 @@ package main import ( + "bytes" + "encoding/csv" "fmt" "strconv" + "strings" "k8s.io/apimachinery/pkg/types" ) @@ -30,6 +33,53 @@ func (v *stringValidatingValue) Type() string { return "string" } +// stringSliceValidatingValue is a string slice flag value with custom validation logic. +// it implements the pflag.Value interface. +type stringSliceValidatingValue struct { + validator func(v string) error + values []string + changed bool +} + +func (v *stringSliceValidatingValue) String() string { + b := &bytes.Buffer{} + w := csv.NewWriter(b) + err := w.Write(v.values) + if err != nil { + return "" + } + + w.Flush() + str := strings.TrimSuffix(b.String(), "\n") + return "[" + str + "]" +} + +func (v *stringSliceValidatingValue) Set(param string) error { + if err := v.validator(param); err != nil { + return err + } + + stringReader := strings.NewReader(param) + csvReader := csv.NewReader(stringReader) + str, err := csvReader.Read() + if err != nil { + return err + } + + if !v.changed { + v.values = str + } else { + v.values = append(v.values, str...) + } + v.changed = true + + return nil +} + +func (v *stringSliceValidatingValue) Type() string { + return "stringSlice" +} + type intValidatingValue struct { validator func(v int) error value int diff --git a/deploy/experimental-nginx-plus/deploy.yaml b/deploy/experimental-nginx-plus/deploy.yaml index 88f5e771cd..9ac24a81da 100644 --- a/deploy/experimental-nginx-plus/deploy.yaml +++ b/deploy/experimental-nginx-plus/deploy.yaml @@ -209,6 +209,7 @@ spec: - --gatewayclass=nginx - --config=nginx-gateway-config - --service=nginx-gateway + - --nginx-docker-secret=nginx-plus-registry-secret - --nginx-plus - --usage-report-secret=nplus-license - --metrics-port=9113 diff --git a/deploy/nginx-plus/deploy.yaml b/deploy/nginx-plus/deploy.yaml index ca6be2dd91..6d6c1ca848 100644 --- a/deploy/nginx-plus/deploy.yaml +++ b/deploy/nginx-plus/deploy.yaml @@ -205,6 +205,7 @@ spec: - --gatewayclass=nginx - --config=nginx-gateway-config - --service=nginx-gateway + - --nginx-docker-secret=nginx-plus-registry-secret - --nginx-plus - --usage-report-secret=nplus-license - --metrics-port=9113 diff --git a/deploy/snippets-filters-nginx-plus/deploy.yaml b/deploy/snippets-filters-nginx-plus/deploy.yaml index f452442bef..88b9371440 100644 --- a/deploy/snippets-filters-nginx-plus/deploy.yaml +++ b/deploy/snippets-filters-nginx-plus/deploy.yaml @@ -207,6 +207,7 @@ spec: - --gatewayclass=nginx - --config=nginx-gateway-config - --service=nginx-gateway + - --nginx-docker-secret=nginx-plus-registry-secret - --nginx-plus - --usage-report-secret=nplus-license - --metrics-port=9113 diff --git a/internal/framework/controller/resource.go b/internal/framework/controller/resource.go index c238b64924..2fff439a50 100644 --- a/internal/framework/controller/resource.go +++ b/internal/framework/controller/resource.go @@ -4,6 +4,6 @@ import "fmt" // CreateNginxResourceName creates the base resource name for all nginx resources // created by the control plane. -func CreateNginxResourceName(gatewayName, gatewayClassName string) string { - return fmt.Sprintf("%s-%s", gatewayName, gatewayClassName) +func CreateNginxResourceName(prefix, suffix string) string { + return fmt.Sprintf("%s-%s", prefix, suffix) } diff --git a/internal/mode/static/config/config.go b/internal/mode/static/config/config.go index 19837a780a..d8556e19f2 100644 --- a/internal/mode/static/config/config.go +++ b/internal/mode/static/config/config.go @@ -32,6 +32,8 @@ type Config struct { ConfigName string // GatewayClassName is the name of the GatewayClass resource that the Gateway will use. GatewayClassName string + // NginxDockerSecretNames are the names of any Docker registry Secrets for the NGINX container. + NginxDockerSecretNames []string // LeaderElection contains the configuration for leader election. LeaderElection LeaderElectionConfig // ProductTelemetryConfig contains the configuration for collecting product telemetry. diff --git a/internal/mode/static/handler.go b/internal/mode/static/handler.go index 2ca73c7568..c22b182e8e 100644 --- a/internal/mode/static/handler.go +++ b/internal/mode/static/handler.go @@ -169,6 +169,12 @@ func (h *eventHandlerImpl) HandleEventBatch(ctx context.Context, logger logr.Log h.sendNginxConfig(ctx, logger, gr, changeType) } +// enable is called when the pod becomes leader to ensure the provisioner has +// the latest configuration. +func (h *eventHandlerImpl) enable(ctx context.Context) { + h.sendNginxConfig(ctx, h.cfg.logger, h.cfg.processor.GetLatestGraph(), state.ClusterStateChange) +} + func (h *eventHandlerImpl) sendNginxConfig( ctx context.Context, logger logr.Logger, @@ -176,7 +182,6 @@ func (h *eventHandlerImpl) sendNginxConfig( changeType state.ChangeType, ) { if gr == nil { - logger.Info("Handling events didn't result into NGINX configuration changes") return } @@ -246,13 +251,13 @@ func (h *eventHandlerImpl) processStateAndBuildConfig( h.setLatestConfiguration(&cfg) - deployment.Lock.Lock() + deployment.FileLock.Lock() if h.cfg.plus { configApplied = h.cfg.nginxUpdater.UpdateUpstreamServers(deployment, cfg) } else { configApplied = h.updateNginxConf(deployment, cfg) } - deployment.Lock.Unlock() + deployment.FileLock.Unlock() case state.ClusterStateChange: h.version++ cfg := dataplane.BuildConfiguration(ctx, gr, h.cfg.serviceResolver, h.version, h.cfg.plus) @@ -264,9 +269,9 @@ func (h *eventHandlerImpl) processStateAndBuildConfig( h.setLatestConfiguration(&cfg) - deployment.Lock.Lock() + deployment.FileLock.Lock() configApplied = h.updateNginxConf(deployment, cfg) - deployment.Lock.Unlock() + deployment.FileLock.Unlock() } return configApplied diff --git a/internal/mode/static/manager.go b/internal/mode/static/manager.go index 5eb31b3462..930be5d01e 100644 --- a/internal/mode/static/manager.go +++ b/internal/mode/static/manager.go @@ -201,13 +201,15 @@ func StartManager(cfg config.Config) error { ctx, mgr, provisioner.Config{ - DeploymentStore: nginxUpdater.NginxDeployments, - StatusQueue: statusQueue, - Logger: cfg.Logger.WithName("provisioner"), - EventRecorder: recorder, - GatewayPodConfig: cfg.GatewayPodConfig, - GCName: cfg.GatewayClassName, - Plus: cfg.Plus, + DeploymentStore: nginxUpdater.NginxDeployments, + StatusQueue: statusQueue, + Logger: cfg.Logger.WithName("provisioner"), + EventRecorder: recorder, + GatewayPodConfig: &cfg.GatewayPodConfig, + GCName: cfg.GatewayClassName, + Plus: cfg.Plus, + NginxDockerSecretNames: cfg.NginxDockerSecretNames, + PlusUsageConfig: &cfg.UsageReportConfig, }, ) if err != nil { @@ -265,6 +267,7 @@ func StartManager(cfg config.Config) error { if err = mgr.Add(runnables.NewCallFunctionsAfterBecameLeader([]func(context.Context){ groupStatusUpdater.Enable, nginxProvisioner.Enable, + eventHandler.enable, })); err != nil { return fmt.Errorf("cannot register functions that get called after Pod becomes leader: %w", err) } diff --git a/internal/mode/static/nginx/agent/agentfakes/fake_deployment_storer.go b/internal/mode/static/nginx/agent/agentfakes/fake_deployment_storer.go new file mode 100644 index 0000000000..af87b10a8e --- /dev/null +++ b/internal/mode/static/nginx/agent/agentfakes/fake_deployment_storer.go @@ -0,0 +1,230 @@ +// Code generated by counterfeiter. DO NOT EDIT. +package agentfakes + +import ( + "context" + "sync" + + "github.com/nginx/nginx-gateway-fabric/internal/mode/static/nginx/agent" + "k8s.io/apimachinery/pkg/types" +) + +type FakeDeploymentStorer struct { + GetStub func(types.NamespacedName) *agent.Deployment + getMutex sync.RWMutex + getArgsForCall []struct { + arg1 types.NamespacedName + } + getReturns struct { + result1 *agent.Deployment + } + getReturnsOnCall map[int]struct { + result1 *agent.Deployment + } + GetOrStoreStub func(context.Context, types.NamespacedName, chan struct{}) *agent.Deployment + getOrStoreMutex sync.RWMutex + getOrStoreArgsForCall []struct { + arg1 context.Context + arg2 types.NamespacedName + arg3 chan struct{} + } + getOrStoreReturns struct { + result1 *agent.Deployment + } + getOrStoreReturnsOnCall map[int]struct { + result1 *agent.Deployment + } + RemoveStub func(types.NamespacedName) + removeMutex sync.RWMutex + removeArgsForCall []struct { + arg1 types.NamespacedName + } + invocations map[string][][]interface{} + invocationsMutex sync.RWMutex +} + +func (fake *FakeDeploymentStorer) Get(arg1 types.NamespacedName) *agent.Deployment { + fake.getMutex.Lock() + ret, specificReturn := fake.getReturnsOnCall[len(fake.getArgsForCall)] + fake.getArgsForCall = append(fake.getArgsForCall, struct { + arg1 types.NamespacedName + }{arg1}) + stub := fake.GetStub + fakeReturns := fake.getReturns + fake.recordInvocation("Get", []interface{}{arg1}) + fake.getMutex.Unlock() + if stub != nil { + return stub(arg1) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeDeploymentStorer) GetCallCount() int { + fake.getMutex.RLock() + defer fake.getMutex.RUnlock() + return len(fake.getArgsForCall) +} + +func (fake *FakeDeploymentStorer) GetCalls(stub func(types.NamespacedName) *agent.Deployment) { + fake.getMutex.Lock() + defer fake.getMutex.Unlock() + fake.GetStub = stub +} + +func (fake *FakeDeploymentStorer) GetArgsForCall(i int) types.NamespacedName { + fake.getMutex.RLock() + defer fake.getMutex.RUnlock() + argsForCall := fake.getArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeDeploymentStorer) GetReturns(result1 *agent.Deployment) { + fake.getMutex.Lock() + defer fake.getMutex.Unlock() + fake.GetStub = nil + fake.getReturns = struct { + result1 *agent.Deployment + }{result1} +} + +func (fake *FakeDeploymentStorer) GetReturnsOnCall(i int, result1 *agent.Deployment) { + fake.getMutex.Lock() + defer fake.getMutex.Unlock() + fake.GetStub = nil + if fake.getReturnsOnCall == nil { + fake.getReturnsOnCall = make(map[int]struct { + result1 *agent.Deployment + }) + } + fake.getReturnsOnCall[i] = struct { + result1 *agent.Deployment + }{result1} +} + +func (fake *FakeDeploymentStorer) GetOrStore(arg1 context.Context, arg2 types.NamespacedName, arg3 chan struct{}) *agent.Deployment { + fake.getOrStoreMutex.Lock() + ret, specificReturn := fake.getOrStoreReturnsOnCall[len(fake.getOrStoreArgsForCall)] + fake.getOrStoreArgsForCall = append(fake.getOrStoreArgsForCall, struct { + arg1 context.Context + arg2 types.NamespacedName + arg3 chan struct{} + }{arg1, arg2, arg3}) + stub := fake.GetOrStoreStub + fakeReturns := fake.getOrStoreReturns + fake.recordInvocation("GetOrStore", []interface{}{arg1, arg2, arg3}) + fake.getOrStoreMutex.Unlock() + if stub != nil { + return stub(arg1, arg2, arg3) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeDeploymentStorer) GetOrStoreCallCount() int { + fake.getOrStoreMutex.RLock() + defer fake.getOrStoreMutex.RUnlock() + return len(fake.getOrStoreArgsForCall) +} + +func (fake *FakeDeploymentStorer) GetOrStoreCalls(stub func(context.Context, types.NamespacedName, chan struct{}) *agent.Deployment) { + fake.getOrStoreMutex.Lock() + defer fake.getOrStoreMutex.Unlock() + fake.GetOrStoreStub = stub +} + +func (fake *FakeDeploymentStorer) GetOrStoreArgsForCall(i int) (context.Context, types.NamespacedName, chan struct{}) { + fake.getOrStoreMutex.RLock() + defer fake.getOrStoreMutex.RUnlock() + argsForCall := fake.getOrStoreArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 +} + +func (fake *FakeDeploymentStorer) GetOrStoreReturns(result1 *agent.Deployment) { + fake.getOrStoreMutex.Lock() + defer fake.getOrStoreMutex.Unlock() + fake.GetOrStoreStub = nil + fake.getOrStoreReturns = struct { + result1 *agent.Deployment + }{result1} +} + +func (fake *FakeDeploymentStorer) GetOrStoreReturnsOnCall(i int, result1 *agent.Deployment) { + fake.getOrStoreMutex.Lock() + defer fake.getOrStoreMutex.Unlock() + fake.GetOrStoreStub = nil + if fake.getOrStoreReturnsOnCall == nil { + fake.getOrStoreReturnsOnCall = make(map[int]struct { + result1 *agent.Deployment + }) + } + fake.getOrStoreReturnsOnCall[i] = struct { + result1 *agent.Deployment + }{result1} +} + +func (fake *FakeDeploymentStorer) Remove(arg1 types.NamespacedName) { + fake.removeMutex.Lock() + fake.removeArgsForCall = append(fake.removeArgsForCall, struct { + arg1 types.NamespacedName + }{arg1}) + stub := fake.RemoveStub + fake.recordInvocation("Remove", []interface{}{arg1}) + fake.removeMutex.Unlock() + if stub != nil { + fake.RemoveStub(arg1) + } +} + +func (fake *FakeDeploymentStorer) RemoveCallCount() int { + fake.removeMutex.RLock() + defer fake.removeMutex.RUnlock() + return len(fake.removeArgsForCall) +} + +func (fake *FakeDeploymentStorer) RemoveCalls(stub func(types.NamespacedName)) { + fake.removeMutex.Lock() + defer fake.removeMutex.Unlock() + fake.RemoveStub = stub +} + +func (fake *FakeDeploymentStorer) RemoveArgsForCall(i int) types.NamespacedName { + fake.removeMutex.RLock() + defer fake.removeMutex.RUnlock() + argsForCall := fake.removeArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeDeploymentStorer) Invocations() map[string][][]interface{} { + fake.invocationsMutex.RLock() + defer fake.invocationsMutex.RUnlock() + fake.getMutex.RLock() + defer fake.getMutex.RUnlock() + fake.getOrStoreMutex.RLock() + defer fake.getOrStoreMutex.RUnlock() + fake.removeMutex.RLock() + defer fake.removeMutex.RUnlock() + copiedInvocations := map[string][][]interface{}{} + for key, value := range fake.invocations { + copiedInvocations[key] = value + } + return copiedInvocations +} + +func (fake *FakeDeploymentStorer) recordInvocation(key string, args []interface{}) { + fake.invocationsMutex.Lock() + defer fake.invocationsMutex.Unlock() + if fake.invocations == nil { + fake.invocations = map[string][][]interface{}{} + } + if fake.invocations[key] == nil { + fake.invocations[key] = [][]interface{}{} + } + fake.invocations[key] = append(fake.invocations[key], args) +} + +var _ agent.DeploymentStorer = new(FakeDeploymentStorer) diff --git a/internal/mode/static/nginx/agent/command.go b/internal/mode/static/nginx/agent/command.go index 31d96143a7..d5be137cd4 100644 --- a/internal/mode/static/nginx/agent/command.go +++ b/internal/mode/static/nginx/agent/command.go @@ -34,11 +34,10 @@ const connectionWaitTimeout = 30 * time.Second // commandService handles the connection and subscription to the data plane agent. type commandService struct { pb.CommandServiceServer - nginxDeployments *DeploymentStore - statusQueue *status.Queue - connTracker agentgrpc.ConnectionsTracker - k8sReader client.Reader - // TODO(sberman): all logs are at Info level right now. Adjust appropriately. + nginxDeployments *DeploymentStore + statusQueue *status.Queue + connTracker agentgrpc.ConnectionsTracker + k8sReader client.Reader logger logr.Logger connectionTimeout time.Duration } @@ -144,13 +143,9 @@ func (cs *commandService) Subscribe(in pb.CommandService_SubscribeServer) error go msgr.Run(ctx) // apply current config before starting event loop - deployment.Lock.RLock() if err := cs.setInitialConfig(ctx, deployment, conn, msgr); err != nil { - deployment.Lock.RUnlock() - return err } - deployment.Lock.RUnlock() // subscribe to the deployment broadcaster to get file updates broadcaster := deployment.GetBroadcaster() @@ -255,13 +250,15 @@ func (cs *commandService) waitForConnection( } // setInitialConfig gets the initial configuration for this connection and applies it. -// The caller MUST lock the deployment before calling this. func (cs *commandService) setInitialConfig( ctx context.Context, deployment *Deployment, conn *agentgrpc.Connection, msgr messenger.Messenger, ) error { + deployment.FileLock.Lock() + defer deployment.FileLock.Unlock() + fileOverviews, configVersion := deployment.GetFileOverviews() if err := msgr.Send(ctx, buildRequest(fileOverviews, conn.InstanceID, configVersion)); err != nil { cs.logAndSendErrorStatus(deployment, conn, err) @@ -420,7 +417,7 @@ func buildPlusAPIRequest(action *pb.NGINXPlusAction, instanceID string) *pb.Mana } func (cs *commandService) getPodOwner(podName string) (types.NamespacedName, error) { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() var pods v1.PodList @@ -451,12 +448,25 @@ func (cs *commandService) getPodOwner(podName string) (types.NamespacedName, err } var replicaSet appsv1.ReplicaSet - if err := cs.k8sReader.Get( + var replicaSetErr error + if err := wait.PollUntilContextCancel( ctx, - types.NamespacedName{Namespace: pod.Namespace, Name: podOwnerRefs[0].Name}, - &replicaSet, + 500*time.Millisecond, + true, /* poll immediately */ + func(ctx context.Context) (bool, error) { + if err := cs.k8sReader.Get( + ctx, + types.NamespacedName{Namespace: pod.Namespace, Name: podOwnerRefs[0].Name}, + &replicaSet, + ); err != nil { + replicaSetErr = err + return false, nil //nolint:nilerr // error is returned at the end + } + + return true, nil + }, ); err != nil { - return types.NamespacedName{}, fmt.Errorf("failed to get nginx Pod's ReplicaSet: %w", err) + return types.NamespacedName{}, fmt.Errorf("failed to get nginx Pod's ReplicaSet: %w", replicaSetErr) } replicaOwnerRefs := replicaSet.GetOwnerReferences() diff --git a/internal/mode/static/nginx/agent/deployment.go b/internal/mode/static/nginx/agent/deployment.go index bafdc6ad9e..5da82c7fd9 100644 --- a/internal/mode/static/nginx/agent/deployment.go +++ b/internal/mode/static/nginx/agent/deployment.go @@ -56,7 +56,8 @@ type Deployment struct { fileOverviews []*pb.File files []File - Lock sync.RWMutex + FileLock sync.RWMutex + errLock sync.RWMutex } // newDeployment returns a new Deployment object. @@ -72,56 +73,94 @@ func (d *Deployment) GetBroadcaster() broadcast.Broadcaster { return d.broadcaster } -// GetFileOverviews returns the current list of fileOverviews and configVersion for the deployment. -func (d *Deployment) GetFileOverviews() ([]*pb.File, string) { - d.Lock.RLock() - defer d.Lock.RUnlock() +// SetLatestConfigError sets the latest config apply error for the deployment. +func (d *Deployment) SetLatestConfigError(err error) { + d.errLock.Lock() + defer d.errLock.Unlock() - return d.fileOverviews, d.configVersion + d.latestConfigError = err } -// GetNGINXPlusActions returns the current NGINX Plus API Actions for the deployment. -func (d *Deployment) GetNGINXPlusActions() []*pb.NGINXPlusAction { - d.Lock.RLock() - defer d.Lock.RUnlock() +// SetLatestUpstreamError sets the latest upstream update error for the deployment. +func (d *Deployment) SetLatestUpstreamError(err error) { + d.errLock.Lock() + defer d.errLock.Unlock() - return d.nginxPlusActions + d.latestUpstreamError = err } // GetLatestConfigError gets the latest config apply error for the deployment. func (d *Deployment) GetLatestConfigError() error { - d.Lock.RLock() - defer d.Lock.RUnlock() + d.errLock.RLock() + defer d.errLock.RUnlock() return d.latestConfigError } // GetLatestUpstreamError gets the latest upstream update error for the deployment. func (d *Deployment) GetLatestUpstreamError() error { - d.Lock.RLock() - defer d.Lock.RUnlock() + d.errLock.RLock() + defer d.errLock.RUnlock() return d.latestUpstreamError } +// SetPodErrorStatus sets the error status of a Pod in this Deployment if applying the config failed. +func (d *Deployment) SetPodErrorStatus(pod string, err error) { + d.errLock.Lock() + defer d.errLock.Unlock() + + d.podStatuses[pod] = err +} + // RemovePodStatus deletes a pod from the pod status map. func (d *Deployment) RemovePodStatus(podName string) { - d.Lock.Lock() - defer d.Lock.Unlock() + d.errLock.Lock() + defer d.errLock.Unlock() delete(d.podStatuses, podName) } +// GetConfigurationStatus returns the current config status for this Deployment. It combines +// the most recent errors (if they exist) for all Pods in the Deployment into a single error. +func (d *Deployment) GetConfigurationStatus() error { + d.errLock.RLock() + defer d.errLock.RUnlock() + + errs := make([]error, 0, len(d.podStatuses)) + for _, err := range d.podStatuses { + errs = append(errs, err) + } + + if len(errs) == 1 { + return errs[0] + } + + return errors.Join(errs...) +} + /* The following functions for the Deployment object are UNLOCKED, meaning that they are unsafe. -Callers of these functions MUST ensure the lock is set before calling. +Callers of these functions MUST ensure the FileLock is set before calling. These functions are called as part of the ConfigApply or APIRequest processes. These entire processes are locked by the caller, hence why the functions themselves do not set the locks. */ +// GetFileOverviews returns the current list of fileOverviews and configVersion for the deployment. +// The deployment FileLock MUST already be locked before calling this function. +func (d *Deployment) GetFileOverviews() ([]*pb.File, string) { + return d.fileOverviews, d.configVersion +} + +// GetNGINXPlusActions returns the current NGINX Plus API Actions for the deployment. +// The deployment FileLock MUST already be locked before calling this function. +func (d *Deployment) GetNGINXPlusActions() []*pb.NGINXPlusAction { + return d.nginxPlusActions +} + // GetFile gets the requested file for the deployment and returns its contents. -// The deployment MUST already be locked before calling this function. +// The deployment FileLock MUST already be locked before calling this function. func (d *Deployment) GetFile(name, hash string) []byte { for _, file := range d.files { if name == file.Meta.GetName() && hash == file.Meta.GetHash() { @@ -133,7 +172,7 @@ func (d *Deployment) GetFile(name, hash string) []byte { } // SetFiles updates the nginx files and fileOverviews for the deployment and returns the message to send. -// The deployment MUST already be locked before calling this function. +// The deployment FileLock MUST already be locked before calling this function. func (d *Deployment) SetFiles(files []File) broadcast.NginxAgentMessage { d.files = files @@ -167,43 +206,18 @@ func (d *Deployment) SetFiles(files []File) broadcast.NginxAgentMessage { // SetNGINXPlusActions updates the deployment's latest NGINX Plus Actions to perform if using NGINX Plus. // Used by a Subscriber when it first connects. -// The deployment MUST already be locked before calling this function. +// The deployment FileLock MUST already be locked before calling this function. func (d *Deployment) SetNGINXPlusActions(actions []*pb.NGINXPlusAction) { d.nginxPlusActions = actions } -// SetPodErrorStatus sets the error status of a Pod in this Deployment if applying the config failed. -// The deployment MUST already be locked before calling this function. -func (d *Deployment) SetPodErrorStatus(pod string, err error) { - d.podStatuses[pod] = err -} - -// SetLatestConfigError sets the latest config apply error for the deployment. -// The deployment MUST already be locked before calling this function. -func (d *Deployment) SetLatestConfigError(err error) { - d.latestConfigError = err -} +//counterfeiter:generate . DeploymentStorer -// SetLatestUpstreamError sets the latest upstream update error for the deployment. -// The deployment MUST already be locked before calling this function. -func (d *Deployment) SetLatestUpstreamError(err error) { - d.latestUpstreamError = err -} - -// GetConfigurationStatus returns the current config status for this Deployment. It combines -// the most recent errors (if they exist) for all Pods in the Deployment into a single error. -// The deployment MUST already be locked before calling this function. -func (d *Deployment) GetConfigurationStatus() error { - errs := make([]error, 0, len(d.podStatuses)) - for _, err := range d.podStatuses { - errs = append(errs, err) - } - - if len(errs) == 1 { - return errs[0] - } - - return errors.Join(errs...) +// DeploymentStorer is an interface to store Deployments. +type DeploymentStorer interface { + Get(types.NamespacedName) *Deployment + GetOrStore(context.Context, types.NamespacedName, chan struct{}) *Deployment + Remove(types.NamespacedName) } // DeploymentStore holds a map of all Deployments. diff --git a/internal/mode/static/nginx/agent/file.go b/internal/mode/static/nginx/agent/file.go index 35f26b628c..fa604bc16b 100644 --- a/internal/mode/static/nginx/agent/file.go +++ b/internal/mode/static/nginx/agent/file.go @@ -25,8 +25,7 @@ type fileService struct { pb.FileServiceServer nginxDeployments *DeploymentStore connTracker agentgrpc.ConnectionsTracker - // TODO(sberman): all logs are at Info level right now. Adjust appropriately. - logger logr.Logger + logger logr.Logger } func newFileService( diff --git a/internal/mode/static/provisioner/eventloop.go b/internal/mode/static/provisioner/eventloop.go index c4ccc2b2e1..5f080156bd 100644 --- a/internal/mode/static/provisioner/eventloop.go +++ b/internal/mode/static/provisioner/eventloop.go @@ -80,6 +80,17 @@ func newEventLoop( ), }, }, + { + objectType: &corev1.Secret{}, + options: []controller.Option{ + controller.WithK8sPredicate( + k8spredicate.And( + k8spredicate.GenerationChangedPredicate{}, + nginxResourceLabelPredicate, + ), + ), + }, + }, } eventCh := make(chan interface{}) @@ -112,6 +123,7 @@ func newEventLoop( &corev1.ServiceList{}, &corev1.ServiceAccountList{}, &corev1.ConfigMapList{}, + &corev1.SecretList{}, }, ) diff --git a/internal/mode/static/provisioner/handler.go b/internal/mode/static/provisioner/handler.go index 405b670c18..5885373213 100644 --- a/internal/mode/static/provisioner/handler.go +++ b/internal/mode/static/provisioner/handler.go @@ -56,7 +56,7 @@ func (h *eventHandler) HandleEventBatch(ctx context.Context, logger logr.Logger, switch obj := e.Resource.(type) { case *gatewayv1.Gateway: h.store.updateGateway(obj) - case *appsv1.Deployment, *corev1.ServiceAccount, *corev1.ConfigMap: + case *appsv1.Deployment, *corev1.ServiceAccount, *corev1.ConfigMap, *corev1.Secret: objLabels := labels.Set(obj.GetLabels()) if h.labelSelector.Matches(objLabels) { gatewayName := objLabels.Get(controller.GatewayLabel) @@ -93,7 +93,7 @@ func (h *eventHandler) HandleEventBatch(ctx context.Context, logger logr.Logger, logger.Error(err, "error deprovisioning nginx resources") } h.store.deleteGateway(e.NamespacedName) - case *appsv1.Deployment, *corev1.Service, *corev1.ServiceAccount, *corev1.ConfigMap: + case *appsv1.Deployment, *corev1.Service, *corev1.ServiceAccount, *corev1.ConfigMap, *corev1.Secret: if err := h.reprovisionResources(ctx, e); err != nil { logger.Error(err, "error re-provisioning nginx resources") } @@ -129,9 +129,9 @@ func (h *eventHandler) updateOrDeleteResources( h.store.registerResourceInGatewayConfig(gatewayNSName, obj) - resourceName := controller.CreateNginxResourceName(gatewayNSName.Name, h.gcName) resources := h.store.getNginxResourcesForGateway(gatewayNSName) if resources.Gateway != nil { + resourceName := controller.CreateNginxResourceName(gatewayNSName.Name, h.gcName) if err := h.provisioner.provisionNginx( ctx, resourceName, diff --git a/internal/mode/static/provisioner/handler_test.go b/internal/mode/static/provisioner/handler_test.go new file mode 100644 index 0000000000..bc3aa61d08 --- /dev/null +++ b/internal/mode/static/provisioner/handler_test.go @@ -0,0 +1,174 @@ +package provisioner + +import ( + "context" + "testing" + + "github.com/go-logr/logr" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + + "github.com/nginx/nginx-gateway-fabric/internal/framework/controller" + "github.com/nginx/nginx-gateway-fabric/internal/framework/events" + "github.com/nginx/nginx-gateway-fabric/internal/mode/static/state/graph" + "github.com/nginx/nginx-gateway-fabric/internal/mode/static/status" +) + +func TestHandleEventBatch_Upsert(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + store := newStore(nil, "", "", "") + provisioner, fakeClient, _ := defaultNginxProvisioner() + provisioner.cfg.StatusQueue = status.NewQueue() + provisioner.cfg.Plus = false + provisioner.cfg.NginxDockerSecretNames = nil + + labelSelector := metav1.LabelSelector{ + MatchLabels: map[string]string{"app": "nginx"}, + } + gcName := "nginx" + + handler, err := newEventHandler(store, provisioner, labelSelector, gcName) + g.Expect(err).ToNot(HaveOccurred()) + + ctx := context.TODO() + logger := logr.Discard() + + gateway := &gatewayv1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gw", + Namespace: "default", + Labels: map[string]string{"app": "nginx"}, + }, + } + + deployment := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gw-nginx", + Namespace: "default", + Labels: map[string]string{"app": "nginx", controller.GatewayLabel: "gw"}, + }, + } + + service := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-service", + Namespace: "default", + Labels: map[string]string{"app": "nginx", controller.GatewayLabel: "test-gateway"}, + }, + } + + // Test handling Gateway + upsertEvent := &events.UpsertEvent{Resource: gateway} + + batch := events.EventBatch{upsertEvent} + handler.HandleEventBatch(ctx, logger, batch) + + g.Expect(store.getGateway(client.ObjectKeyFromObject(gateway))).To(Equal(gateway)) + + store.registerResourceInGatewayConfig( + client.ObjectKeyFromObject(gateway), + &graph.Gateway{Source: gateway, Valid: true}, + ) + + // Test handling Deployment + upsertEvent = &events.UpsertEvent{Resource: deployment} + batch = events.EventBatch{upsertEvent} + handler.HandleEventBatch(ctx, logger, batch) + + g.Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(deployment), &appsv1.Deployment{})).To(Succeed()) + + // Test handling Service + upsertEvent = &events.UpsertEvent{Resource: service} + batch = events.EventBatch{upsertEvent} + handler.HandleEventBatch(ctx, logger, batch) + + g.Expect(provisioner.cfg.StatusQueue.Dequeue(ctx)).ToNot(BeNil()) + + // remove Gateway from store and verify that Deployment UpsertEvent results in deletion of resource + store.deleteGateway(client.ObjectKeyFromObject(gateway)) + g.Expect(store.getGateway(client.ObjectKeyFromObject(gateway))).To(BeNil()) + + upsertEvent = &events.UpsertEvent{Resource: deployment} + batch = events.EventBatch{upsertEvent} + handler.HandleEventBatch(ctx, logger, batch) + + g.Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(deployment), &appsv1.Deployment{})).ToNot(Succeed()) + + // do the same thing but when provisioner is not leader. + // non-leader should not delete resources, but instead track them + g.Expect(fakeClient.Create(ctx, deployment)).To(Succeed()) + provisioner.leader = false + + upsertEvent = &events.UpsertEvent{Resource: deployment} + batch = events.EventBatch{upsertEvent} + handler.HandleEventBatch(ctx, logger, batch) + + g.Expect(provisioner.resourcesToDeleteOnStartup).To(HaveLen(1)) + g.Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(deployment), &appsv1.Deployment{})).To(Succeed()) +} + +func TestHandleEventBatch_Delete(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + store := newStore(nil, "", "", "") + provisioner, fakeClient, _ := defaultNginxProvisioner() + provisioner.cfg.StatusQueue = status.NewQueue() + provisioner.cfg.Plus = false + provisioner.cfg.NginxDockerSecretNames = nil + + labelSelector := metav1.LabelSelector{ + MatchLabels: map[string]string{"app": "nginx"}, + } + gcName := "nginx" + + handler, err := newEventHandler(store, provisioner, labelSelector, gcName) + g.Expect(err).ToNot(HaveOccurred()) + + ctx := context.TODO() + logger := logr.Discard() + + gateway := &gatewayv1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gw", + Namespace: "default", + Labels: map[string]string{"app": "nginx"}, + }, + } + + store.registerResourceInGatewayConfig( + client.ObjectKeyFromObject(gateway), + &graph.Gateway{Source: gateway, Valid: true}, + ) + + deployment := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gw-nginx", + Namespace: "default", + Labels: map[string]string{"app": "nginx", controller.GatewayLabel: "gw"}, + }, + } + + store.registerResourceInGatewayConfig(client.ObjectKeyFromObject(gateway), deployment) + + // if deployment is deleted, it should be re-created since Gateway still exists + deleteEvent := &events.DeleteEvent{Type: deployment, NamespacedName: client.ObjectKeyFromObject(deployment)} + batch := events.EventBatch{deleteEvent} + handler.HandleEventBatch(ctx, logger, batch) + + g.Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(deployment), &appsv1.Deployment{})).To(Succeed()) + + // delete Gateway + deleteEvent = &events.DeleteEvent{Type: gateway, NamespacedName: client.ObjectKeyFromObject(gateway)} + batch = events.EventBatch{deleteEvent} + handler.HandleEventBatch(ctx, logger, batch) + + g.Expect(store.getGateway(client.ObjectKeyFromObject(gateway))).To(BeNil()) + g.Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(deployment), &appsv1.Deployment{})).ToNot(Succeed()) +} diff --git a/internal/mode/static/provisioner/objects.go b/internal/mode/static/provisioner/objects.go index 6c3c6b5c01..dc73164a40 100644 --- a/internal/mode/static/provisioner/objects.go +++ b/internal/mode/static/provisioner/objects.go @@ -1,9 +1,12 @@ package provisioner import ( + "context" + "errors" "fmt" "maps" "strconv" + "time" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -39,12 +42,28 @@ func (p *NginxProvisioner) buildNginxResourceObjects( resourceName string, gateway *gatewayv1.Gateway, nProxyCfg *graph.EffectiveNginxProxy, -) []client.Object { - // TODO(sberman): handle nginx plus config - +) ([]client.Object, error) { ngxIncludesConfigMapName := controller.CreateNginxResourceName(resourceName, nginxIncludesConfigMapNameSuffix) ngxAgentConfigMapName := controller.CreateNginxResourceName(resourceName, nginxAgentConfigMapNameSuffix) + var jwtSecretName, caSecretName, clientSSLSecretName string + if p.cfg.Plus { + jwtSecretName = controller.CreateNginxResourceName(resourceName, p.cfg.PlusUsageConfig.SecretName) + if p.cfg.PlusUsageConfig.CASecretName != "" { + caSecretName = controller.CreateNginxResourceName(resourceName, p.cfg.PlusUsageConfig.CASecretName) + } + if p.cfg.PlusUsageConfig.ClientSSLSecretName != "" { + clientSSLSecretName = controller.CreateNginxResourceName(resourceName, p.cfg.PlusUsageConfig.ClientSSLSecretName) + } + } + + // map key is the new name, value is the original name + dockerSecretNames := make(map[string]string) + for _, name := range p.cfg.NginxDockerSecretNames { + newName := controller.CreateNginxResourceName(resourceName, name) + dockerSecretNames[newName] = name + } + selectorLabels := make(map[string]string) maps.Copy(selectorLabels, p.baseLabelSelector.MatchLabels) selectorLabels[controller.GatewayLabel] = gateway.GetName() @@ -72,11 +91,21 @@ func (p *NginxProvisioner) buildNginxResourceObjects( Annotations: annotations, } + secrets, err := p.buildNginxSecrets( + objectMeta, + dockerSecretNames, + jwtSecretName, + caSecretName, + clientSSLSecretName, + ) + configmaps := p.buildNginxConfigMaps( objectMeta, nProxyCfg, ngxIncludesConfigMapName, ngxAgentConfigMapName, + caSecretName != "", + clientSSLSecretName != "", ) serviceAccount := &corev1.ServiceAccount{ @@ -96,6 +125,10 @@ func (p *NginxProvisioner) buildNginxResourceObjects( ngxAgentConfigMapName, ports, selectorLabels, + dockerSecretNames, + jwtSecretName, + caSecretName, + clientSSLSecretName, ) // order to install resources: @@ -106,11 +139,114 @@ func (p *NginxProvisioner) buildNginxResourceObjects( // service // deployment/daemonset - objects := make([]client.Object, 0, len(configmaps)+3) + objects := make([]client.Object, 0, len(configmaps)+len(secrets)+3) + objects = append(objects, secrets...) objects = append(objects, configmaps...) objects = append(objects, serviceAccount, service, deployment) - return objects + return objects, err +} + +func (p *NginxProvisioner) buildNginxSecrets( + objectMeta metav1.ObjectMeta, + dockerSecretNames map[string]string, + jwtSecretName string, + caSecretName string, + clientSSLSecretName string, +) ([]client.Object, error) { + var secrets []client.Object + var errs []error + + for newName, origName := range dockerSecretNames { + newSecret, err := p.getAndUpdateSecret( + origName, + metav1.ObjectMeta{ + Name: newName, + Namespace: objectMeta.Namespace, + Labels: objectMeta.Labels, + Annotations: objectMeta.Annotations, + }, + ) + if err != nil { + errs = append(errs, err) + } else { + secrets = append(secrets, newSecret) + } + } + + if jwtSecretName != "" { + newSecret, err := p.getAndUpdateSecret( + p.cfg.PlusUsageConfig.SecretName, + metav1.ObjectMeta{ + Name: jwtSecretName, + Namespace: objectMeta.Namespace, + Labels: objectMeta.Labels, + Annotations: objectMeta.Annotations, + }, + ) + if err != nil { + errs = append(errs, err) + } else { + secrets = append(secrets, newSecret) + } + } + + if caSecretName != "" { + newSecret, err := p.getAndUpdateSecret( + p.cfg.PlusUsageConfig.CASecretName, + metav1.ObjectMeta{ + Name: caSecretName, + Namespace: objectMeta.Namespace, + Labels: objectMeta.Labels, + Annotations: objectMeta.Annotations, + }, + ) + if err != nil { + errs = append(errs, err) + } else { + secrets = append(secrets, newSecret) + } + } + + if clientSSLSecretName != "" { + newSecret, err := p.getAndUpdateSecret( + p.cfg.PlusUsageConfig.ClientSSLSecretName, + metav1.ObjectMeta{ + Name: clientSSLSecretName, + Namespace: objectMeta.Namespace, + Labels: objectMeta.Labels, + Annotations: objectMeta.Annotations, + }, + ) + if err != nil { + errs = append(errs, err) + } else { + secrets = append(secrets, newSecret) + } + } + + return secrets, errors.Join(errs...) +} + +func (p *NginxProvisioner) getAndUpdateSecret( + name string, + newObjectMeta metav1.ObjectMeta, +) (*corev1.Secret, error) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + key := types.NamespacedName{Namespace: p.cfg.GatewayPodConfig.Namespace, Name: name} + secret := &corev1.Secret{} + if err := p.k8sClient.Get(ctx, key, secret); err != nil { + return nil, fmt.Errorf("error getting secret: %w", err) + } + + newSecret := &corev1.Secret{ + ObjectMeta: newObjectMeta, + Data: secret.Data, + } + + return newSecret, nil } func (p *NginxProvisioner) buildNginxConfigMaps( @@ -118,6 +254,8 @@ func (p *NginxProvisioner) buildNginxConfigMaps( nProxyCfg *graph.EffectiveNginxProxy, ngxIncludesConfigMapName string, ngxAgentConfigMapName string, + caSecret bool, + clientSSLSecret bool, ) []client.Object { var logging *ngfAPIv1alpha2.NginxLogging if nProxyCfg != nil && nProxyCfg.Logging != nil { @@ -145,6 +283,17 @@ func (p *NginxProvisioner) buildNginxConfigMaps( }, } + if p.cfg.Plus { + mgmtFields := map[string]interface{}{ + "UsageEndpoint": p.cfg.PlusUsageConfig.Endpoint, + "SkipVerify": p.cfg.PlusUsageConfig.SkipVerify, + "UsageCASecret": caSecret, + "UsageClientSSLSecret": clientSSLSecret, + } + + bootstrapCM.Data["mgmt.conf"] = string(helpers.MustExecuteTemplate(mgmtTemplate, mgmtFields)) + } + metricsPort := config.DefaultNginxMetricsPort port, enableMetrics := graph.MetricsEnabledForNginxProxy(nProxyCfg) if port != nil { @@ -236,6 +385,10 @@ func (p *NginxProvisioner) buildNginxDeployment( ngxAgentConfigMapName string, ports map[int32]struct{}, selectorLabels map[string]string, + dockerSecretNames map[string]string, + jwtSecretName string, + caSecretName string, + clientSSLSecretName string, ) client.Object { podTemplateSpec := p.buildNginxPodTemplateSpec( objectMeta, @@ -243,6 +396,10 @@ func (p *NginxProvisioner) buildNginxDeployment( ngxIncludesConfigMapName, ngxAgentConfigMapName, ports, + dockerSecretNames, + jwtSecretName, + caSecretName, + clientSSLSecretName, ) var object client.Object @@ -271,15 +428,18 @@ func (p *NginxProvisioner) buildNginxDeployment( return object } +//nolint:gocyclo // will refactor at some point func (p *NginxProvisioner) buildNginxPodTemplateSpec( objectMeta metav1.ObjectMeta, nProxyCfg *graph.EffectiveNginxProxy, ngxIncludesConfigMapName string, ngxAgentConfigMapName string, ports map[int32]struct{}, + dockerSecretNames map[string]string, + jwtSecretName string, + caSecretName string, + clientSSLSecretName string, ) corev1.PodTemplateSpec { - // TODO(sberman): handle nginx plus; debug - containerPorts := make([]corev1.ContainerPort, 0, len(ports)) for port := range ports { containerPort := corev1.ContainerPort{ @@ -388,6 +548,7 @@ func (p *NginxProvisioner) buildNginxPodTemplateSpec( }, }, }, + ImagePullSecrets: []corev1.LocalObjectReference{}, ServiceAccountName: objectMeta.Name, Volumes: []corev1.Volume{ {Name: "nginx-agent", VolumeSource: emptyDirVolumeSource}, @@ -456,6 +617,76 @@ func (p *NginxProvisioner) buildNginxPodTemplateSpec( } } + for name := range dockerSecretNames { + ref := corev1.LocalObjectReference{Name: name} + spec.Spec.ImagePullSecrets = append(spec.Spec.ImagePullSecrets, ref) + } + + if p.cfg.Plus { + initCmd := spec.Spec.InitContainers[0].Command + initCmd = append(initCmd, + "--source", "/includes/mgmt.conf", "--destination", "/etc/nginx/main-includes", "--nginx-plus") + spec.Spec.InitContainers[0].Command = initCmd + + volumeMounts := spec.Spec.Containers[0].VolumeMounts + + volumeMounts = append(volumeMounts, corev1.VolumeMount{ + Name: "nginx-lib", + MountPath: "/var/lib/nginx/state", + }) + spec.Spec.Volumes = append(spec.Spec.Volumes, corev1.Volume{ + Name: "nginx-lib", + VolumeSource: emptyDirVolumeSource, + }) + + if jwtSecretName != "" { + volumeMounts = append(volumeMounts, corev1.VolumeMount{ + Name: "nginx-plus-license", + MountPath: "/etc/nginx/license.jwt", + SubPath: "license.jwt", + }) + spec.Spec.Volumes = append(spec.Spec.Volumes, corev1.Volume{ + Name: "nginx-plus-license", + VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: jwtSecretName}}, + }) + } + if caSecretName != "" || clientSSLSecretName != "" { + volumeMounts = append(volumeMounts, corev1.VolumeMount{ + Name: "nginx-plus-usage-certs", + MountPath: "/etc/nginx/certs-bootstrap/", + }) + + sources := []corev1.VolumeProjection{} + + if caSecretName != "" { + sources = append(sources, corev1.VolumeProjection{ + Secret: &corev1.SecretProjection{ + LocalObjectReference: corev1.LocalObjectReference{Name: caSecretName}, + }, + }) + } + + if clientSSLSecretName != "" { + sources = append(sources, corev1.VolumeProjection{ + Secret: &corev1.SecretProjection{ + LocalObjectReference: corev1.LocalObjectReference{Name: clientSSLSecretName}, + }, + }) + } + + spec.Spec.Volumes = append(spec.Spec.Volumes, corev1.Volume{ + Name: "nginx-plus-usage-certs", + VolumeSource: corev1.VolumeSource{ + Projected: &corev1.ProjectedVolumeSource{ + Sources: sources, + }, + }, + }) + } + + spec.Spec.Containers[0].VolumeMounts = volumeMounts + } + return spec } @@ -489,7 +720,18 @@ func (p *NginxProvisioner) buildImage(nProxyCfg *graph.EffectiveNginxProxy) (str return fmt.Sprintf("%s:%s", image, tag), pullPolicy } +// TODO(sberman): see about how this can be made more elegant. Maybe create some sort of Object factory +// that can better store/build all the objects we need, to reduce the amount of duplicate object lists that we +// have everywhere. func (p *NginxProvisioner) buildNginxResourceObjectsForDeletion(deploymentNSName types.NamespacedName) []client.Object { + // order to delete: + // deployment/daemonset + // service + // serviceaccount + // configmaps + // secrets + // scc (if openshift) + objectMeta := metav1.ObjectMeta{ Name: deploymentNSName.Name, Namespace: deploymentNSName.Namespace, @@ -517,13 +759,54 @@ func (p *NginxProvisioner) buildNginxResourceObjectsForDeletion(deploymentNSName }, } - // order to delete: - // deployment/daemonset - // service - // serviceaccount - // configmaps - // secrets - // scc (if openshift) + objects := []client.Object{deployment, service, serviceAccount, bootstrapCM, agentCM} + + for _, name := range p.cfg.NginxDockerSecretNames { + newName := controller.CreateNginxResourceName(deploymentNSName.Name, name) + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: newName, + Namespace: deploymentNSName.Namespace, + }, + } + objects = append(objects, secret) + } - return []client.Object{deployment, service, serviceAccount, bootstrapCM, agentCM} + var jwtSecretName, caSecretName, clientSSLSecretName string + if p.cfg.Plus { + if p.cfg.PlusUsageConfig.CASecretName != "" { + caSecretName = controller.CreateNginxResourceName(deploymentNSName.Name, p.cfg.PlusUsageConfig.CASecretName) + caSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: caSecretName, + Namespace: deploymentNSName.Namespace, + }, + } + objects = append(objects, caSecret) + } + if p.cfg.PlusUsageConfig.ClientSSLSecretName != "" { + clientSSLSecretName = controller.CreateNginxResourceName( + deploymentNSName.Name, + p.cfg.PlusUsageConfig.ClientSSLSecretName, + ) + clientSSLSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: clientSSLSecretName, + Namespace: deploymentNSName.Namespace, + }, + } + objects = append(objects, clientSSLSecret) + } + + jwtSecretName = controller.CreateNginxResourceName(deploymentNSName.Name, p.cfg.PlusUsageConfig.SecretName) + jwtSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: jwtSecretName, + Namespace: deploymentNSName.Namespace, + }, + } + objects = append(objects, jwtSecret) + } + + return objects } diff --git a/internal/mode/static/provisioner/objects_test.go b/internal/mode/static/provisioner/objects_test.go new file mode 100644 index 0000000000..27fba0d734 --- /dev/null +++ b/internal/mode/static/provisioner/objects_test.go @@ -0,0 +1,617 @@ +package provisioner + +import ( + "fmt" + "testing" + + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + + ngfAPIv1alpha2 "github.com/nginx/nginx-gateway-fabric/apis/v1alpha2" + "github.com/nginx/nginx-gateway-fabric/internal/framework/controller" + "github.com/nginx/nginx-gateway-fabric/internal/framework/helpers" + "github.com/nginx/nginx-gateway-fabric/internal/mode/static/config" + "github.com/nginx/nginx-gateway-fabric/internal/mode/static/state/graph" +) + +func TestBuildNginxResourceObjects(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + provisioner := &NginxProvisioner{ + cfg: Config{ + GatewayPodConfig: &config.GatewayPodConfig{ + Namespace: "default", + Version: "1.0.0", + Image: "ngf-image", + }, + }, + baseLabelSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "nginx", + }, + }, + } + + gateway := &gatewayv1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gw", + Namespace: "default", + }, + Spec: gatewayv1.GatewaySpec{ + Infrastructure: &gatewayv1.GatewayInfrastructure{ + Labels: map[gatewayv1.LabelKey]gatewayv1.LabelValue{ + "label": "value", + }, + Annotations: map[gatewayv1.AnnotationKey]gatewayv1.AnnotationValue{ + "annotation": "value", + }, + }, + Listeners: []gatewayv1.Listener{ + { + Port: 80, + }, + }, + }, + } + + expLabels := map[string]string{ + "label": "value", + "app": "nginx", + "gateway.networking.k8s.io/gateway-name": "gw", + "app.kubernetes.io/name": "gw-nginx", + } + expAnnotations := map[string]string{ + "annotation": "value", + } + + resourceName := "gw-nginx" + objects, err := provisioner.buildNginxResourceObjects(resourceName, gateway, &graph.EffectiveNginxProxy{}) + g.Expect(err).ToNot(HaveOccurred()) + + g.Expect(objects).To(HaveLen(5)) + + validateLabelsAndAnnotations := func(obj client.Object) { + g.Expect(obj.GetLabels()).To(Equal(expLabels)) + g.Expect(obj.GetAnnotations()).To(Equal(expAnnotations)) + } + + validateMeta := func(obj client.Object) { + g.Expect(obj.GetName()).To(Equal(resourceName)) + validateLabelsAndAnnotations(obj) + } + + cmObj := objects[0] + cm, ok := cmObj.(*corev1.ConfigMap) + g.Expect(ok).To(BeTrue()) + g.Expect(cm.GetName()).To(Equal(controller.CreateNginxResourceName(resourceName, nginxIncludesConfigMapNameSuffix))) + validateLabelsAndAnnotations(cm) + g.Expect(cm.Data).To(HaveKey("main.conf")) + g.Expect(cm.Data["main.conf"]).To(ContainSubstring("info")) + + cmObj = objects[1] + cm, ok = cmObj.(*corev1.ConfigMap) + g.Expect(ok).To(BeTrue()) + g.Expect(cm.GetName()).To(Equal(controller.CreateNginxResourceName(resourceName, nginxAgentConfigMapNameSuffix))) + validateLabelsAndAnnotations(cm) + g.Expect(cm.Data).To(HaveKey("nginx-agent.conf")) + g.Expect(cm.Data["nginx-agent.conf"]).To(ContainSubstring("command:")) + + svcAcctObj := objects[2] + svcAcct, ok := svcAcctObj.(*corev1.ServiceAccount) + g.Expect(ok).To(BeTrue()) + validateMeta(svcAcct) + + svcObj := objects[3] + svc, ok := svcObj.(*corev1.Service) + g.Expect(ok).To(BeTrue()) + validateMeta(svc) + g.Expect(svc.Spec.Type).To(Equal(defaultServiceType)) + g.Expect(svc.Spec.ExternalTrafficPolicy).To(Equal(defaultServicePolicy)) + g.Expect(svc.Spec.Ports).To(ContainElement(corev1.ServicePort{ + Port: 80, + Name: "port-80", + TargetPort: intstr.FromInt(80), + })) + + depObj := objects[4] + dep, ok := depObj.(*appsv1.Deployment) + g.Expect(ok).To(BeTrue()) + validateMeta(dep) + + template := dep.Spec.Template + g.Expect(template.GetAnnotations()).To(HaveKey("prometheus.io/scrape")) + g.Expect(template.Spec.Containers).To(HaveLen(1)) + container := template.Spec.Containers[0] + + g.Expect(container.Ports).To(ContainElement(corev1.ContainerPort{ + ContainerPort: config.DefaultNginxMetricsPort, + Name: "metrics", + })) + g.Expect(container.Ports).To(ContainElement(corev1.ContainerPort{ + ContainerPort: 80, + Name: "port-80", + })) + + g.Expect(container.Image).To(Equal(fmt.Sprintf("%s:1.0.0", defaultNginxImagePath))) + g.Expect(container.ImagePullPolicy).To(Equal(defaultImagePullPolicy)) + + g.Expect(template.Spec.InitContainers).To(HaveLen(1)) + initContainer := template.Spec.InitContainers[0] + + g.Expect(initContainer.Image).To(Equal("ngf-image")) + g.Expect(initContainer.ImagePullPolicy).To(Equal(defaultImagePullPolicy)) +} + +func TestBuildNginxResourceObjects_NginxProxyConfig(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + provisioner := &NginxProvisioner{ + cfg: Config{ + GatewayPodConfig: &config.GatewayPodConfig{ + Namespace: "default", + Version: "1.0.0", + }, + }, + baseLabelSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "nginx", + }, + }, + } + + gateway := &gatewayv1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gw", + Namespace: "default", + }, + } + + resourceName := "gw-nginx" + nProxyCfg := &graph.EffectiveNginxProxy{ + Logging: &ngfAPIv1alpha2.NginxLogging{ + ErrorLevel: helpers.GetPointer(ngfAPIv1alpha2.NginxLogLevelDebug), + AgentLevel: helpers.GetPointer(ngfAPIv1alpha2.AgentLogLevelDebug), + }, + Metrics: &ngfAPIv1alpha2.Metrics{ + Port: helpers.GetPointer[int32](8080), + }, + Kubernetes: &ngfAPIv1alpha2.KubernetesSpec{ + Service: &ngfAPIv1alpha2.ServiceSpec{ + ServiceType: helpers.GetPointer(ngfAPIv1alpha2.ServiceTypeNodePort), + ExternalTrafficPolicy: helpers.GetPointer(ngfAPIv1alpha2.ExternalTrafficPolicyCluster), + LoadBalancerIP: helpers.GetPointer("1.2.3.4"), + LoadBalancerSourceRanges: []string{"5.6.7.8"}, + }, + Deployment: &ngfAPIv1alpha2.DeploymentSpec{ + Replicas: helpers.GetPointer[int32](3), + Pod: ngfAPIv1alpha2.PodSpec{ + TerminationGracePeriodSeconds: helpers.GetPointer[int64](25), + }, + Container: ngfAPIv1alpha2.ContainerSpec{ + Image: &ngfAPIv1alpha2.Image{ + Repository: helpers.GetPointer("nginx-repo"), + Tag: helpers.GetPointer("1.1.1"), + PullPolicy: helpers.GetPointer(ngfAPIv1alpha2.PullAlways), + }, + Resources: &corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.Quantity{Format: "100m"}, + }, + }, + }, + }, + }, + } + + objects, err := provisioner.buildNginxResourceObjects(resourceName, gateway, nProxyCfg) + g.Expect(err).ToNot(HaveOccurred()) + + g.Expect(objects).To(HaveLen(5)) + + cmObj := objects[0] + cm, ok := cmObj.(*corev1.ConfigMap) + g.Expect(ok).To(BeTrue()) + g.Expect(cm.Data).To(HaveKey("main.conf")) + g.Expect(cm.Data["main.conf"]).To(ContainSubstring("debug")) + + cmObj = objects[1] + cm, ok = cmObj.(*corev1.ConfigMap) + g.Expect(ok).To(BeTrue()) + g.Expect(cm.Data["nginx-agent.conf"]).To(ContainSubstring("level: debug")) + g.Expect(cm.Data["nginx-agent.conf"]).To(ContainSubstring("port: 8080")) + + svcObj := objects[3] + svc, ok := svcObj.(*corev1.Service) + g.Expect(ok).To(BeTrue()) + g.Expect(svc.Spec.Type).To(Equal(corev1.ServiceTypeNodePort)) + g.Expect(svc.Spec.ExternalTrafficPolicy).To(Equal(corev1.ServiceExternalTrafficPolicyTypeCluster)) + g.Expect(svc.Spec.LoadBalancerIP).To(Equal("1.2.3.4")) + g.Expect(svc.Spec.LoadBalancerSourceRanges).To(Equal([]string{"5.6.7.8"})) + + depObj := objects[4] + dep, ok := depObj.(*appsv1.Deployment) + g.Expect(ok).To(BeTrue()) + + template := dep.Spec.Template + g.Expect(*template.Spec.TerminationGracePeriodSeconds).To(Equal(int64(25))) + + container := template.Spec.Containers[0] + + g.Expect(container.Ports).To(ContainElement(corev1.ContainerPort{ + ContainerPort: 8080, + Name: "metrics", + })) + + g.Expect(container.Image).To(Equal("nginx-repo:1.1.1")) + g.Expect(container.ImagePullPolicy).To(Equal(corev1.PullAlways)) + g.Expect(container.Resources.Limits).To(HaveKey(corev1.ResourceCPU)) + g.Expect(container.Resources.Limits[corev1.ResourceCPU].Format).To(Equal(resource.Format("100m"))) +} + +func TestBuildNginxResourceObjects_Plus(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + jwtSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: jwtTestSecretName, + Namespace: ngfNamespace, + }, + Data: map[string][]byte{"license.jwt": []byte("jwt")}, + } + caSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: caTestSecretName, + Namespace: ngfNamespace, + }, + Data: map[string][]byte{"ca.crt": []byte("ca")}, + } + clientSSLSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: clientTestSecretName, + Namespace: ngfNamespace, + }, + Data: map[string][]byte{"tls.crt": []byte("tls")}, + } + + fakeClient := fake.NewFakeClient(jwtSecret, caSecret, clientSSLSecret) + + provisioner := &NginxProvisioner{ + cfg: Config{ + GatewayPodConfig: &config.GatewayPodConfig{ + Namespace: ngfNamespace, + }, + Plus: true, + PlusUsageConfig: &config.UsageReportConfig{ + SecretName: jwtTestSecretName, + CASecretName: caTestSecretName, + ClientSSLSecretName: clientTestSecretName, + Endpoint: "test.com", + SkipVerify: true, + }, + }, + k8sClient: fakeClient, + baseLabelSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "nginx", + }, + }, + } + + gateway := &gatewayv1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gw", + Namespace: "default", + }, + Spec: gatewayv1.GatewaySpec{ + Infrastructure: &gatewayv1.GatewayInfrastructure{ + Labels: map[gatewayv1.LabelKey]gatewayv1.LabelValue{ + "label": "value", + }, + Annotations: map[gatewayv1.AnnotationKey]gatewayv1.AnnotationValue{ + "annotation": "value", + }, + }, + }, + } + + resourceName := "gw-nginx" + objects, err := provisioner.buildNginxResourceObjects(resourceName, gateway, &graph.EffectiveNginxProxy{}) + g.Expect(err).ToNot(HaveOccurred()) + + g.Expect(objects).To(HaveLen(8)) + + expLabels := map[string]string{ + "label": "value", + "app": "nginx", + "gateway.networking.k8s.io/gateway-name": "gw", + "app.kubernetes.io/name": "gw-nginx", + } + expAnnotations := map[string]string{ + "annotation": "value", + } + + secretObj := objects[0] + secret, ok := secretObj.(*corev1.Secret) + g.Expect(ok).To(BeTrue()) + g.Expect(secret.GetName()).To(Equal(controller.CreateNginxResourceName(resourceName, jwtTestSecretName))) + g.Expect(secret.GetLabels()).To(Equal(expLabels)) + g.Expect(secret.GetAnnotations()).To(Equal(expAnnotations)) + g.Expect(secret.Data).To(HaveKey("license.jwt")) + g.Expect(secret.Data["license.jwt"]).To(Equal([]byte("jwt"))) + + secretObj = objects[1] + secret, ok = secretObj.(*corev1.Secret) + g.Expect(ok).To(BeTrue()) + g.Expect(secret.GetName()).To(Equal(controller.CreateNginxResourceName(resourceName, caTestSecretName))) + g.Expect(secret.GetLabels()).To(Equal(expLabels)) + g.Expect(secret.GetAnnotations()).To(Equal(expAnnotations)) + g.Expect(secret.Data).To(HaveKey("ca.crt")) + g.Expect(secret.Data["ca.crt"]).To(Equal([]byte("ca"))) + + secretObj = objects[2] + secret, ok = secretObj.(*corev1.Secret) + g.Expect(ok).To(BeTrue()) + g.Expect(secret.GetName()).To(Equal(controller.CreateNginxResourceName(resourceName, clientTestSecretName))) + g.Expect(secret.GetLabels()).To(Equal(expLabels)) + g.Expect(secret.GetAnnotations()).To(Equal(expAnnotations)) + g.Expect(secret.Data).To(HaveKey("tls.crt")) + g.Expect(secret.Data["tls.crt"]).To(Equal([]byte("tls"))) + + cmObj := objects[3] + cm, ok := cmObj.(*corev1.ConfigMap) + g.Expect(ok).To(BeTrue()) + g.Expect(cm.Data).To(HaveKey("mgmt.conf")) + g.Expect(cm.Data["mgmt.conf"]).To(ContainSubstring("usage_report endpoint=test.com;")) + g.Expect(cm.Data["mgmt.conf"]).To(ContainSubstring("ssl_verify off;")) + g.Expect(cm.Data["mgmt.conf"]).To(ContainSubstring("ssl_trusted_certificate")) + g.Expect(cm.Data["mgmt.conf"]).To(ContainSubstring("ssl_certificate")) + g.Expect(cm.Data["mgmt.conf"]).To(ContainSubstring("ssl_certificate_key")) + + cmObj = objects[4] + cm, ok = cmObj.(*corev1.ConfigMap) + g.Expect(ok).To(BeTrue()) + g.Expect(cm.Data).To(HaveKey("nginx-agent.conf")) + g.Expect(cm.Data["nginx-agent.conf"]).To(ContainSubstring("api-action")) + + depObj := objects[7] + dep, ok := depObj.(*appsv1.Deployment) + g.Expect(ok).To(BeTrue()) + + template := dep.Spec.Template + container := template.Spec.Containers[0] + initContainer := template.Spec.InitContainers[0] + + g.Expect(initContainer.Command).To(ContainElement("/includes/mgmt.conf")) + g.Expect(container.VolumeMounts).To(ContainElement(corev1.VolumeMount{ + Name: "nginx-plus-license", + MountPath: "/etc/nginx/license.jwt", + SubPath: "license.jwt", + })) + g.Expect(container.VolumeMounts).To(ContainElement(corev1.VolumeMount{ + Name: "nginx-plus-usage-certs", + MountPath: "/etc/nginx/certs-bootstrap/", + })) +} + +func TestBuildNginxResourceObjects_DockerSecrets(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + dockerSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: dockerTestSecretName, + Namespace: ngfNamespace, + }, + Data: map[string][]byte{"data": []byte("docker")}, + } + fakeClient := fake.NewFakeClient(dockerSecret) + + provisioner := &NginxProvisioner{ + cfg: Config{ + GatewayPodConfig: &config.GatewayPodConfig{ + Namespace: ngfNamespace, + }, + NginxDockerSecretNames: []string{dockerTestSecretName}, + }, + k8sClient: fakeClient, + baseLabelSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "nginx", + }, + }, + } + + gateway := &gatewayv1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gw", + Namespace: "default", + }, + } + + resourceName := "gw-nginx" + objects, err := provisioner.buildNginxResourceObjects(resourceName, gateway, &graph.EffectiveNginxProxy{}) + g.Expect(err).ToNot(HaveOccurred()) + + g.Expect(objects).To(HaveLen(6)) + + expLabels := map[string]string{ + "app": "nginx", + "gateway.networking.k8s.io/gateway-name": "gw", + "app.kubernetes.io/name": "gw-nginx", + } + + secretObj := objects[0] + secret, ok := secretObj.(*corev1.Secret) + g.Expect(ok).To(BeTrue()) + g.Expect(secret.GetName()).To(Equal(controller.CreateNginxResourceName(resourceName, dockerTestSecretName))) + g.Expect(secret.GetLabels()).To(Equal(expLabels)) + + depObj := objects[5] + dep, ok := depObj.(*appsv1.Deployment) + g.Expect(ok).To(BeTrue()) + + g.Expect(dep.Spec.Template.Spec.ImagePullSecrets).To(ContainElement(corev1.LocalObjectReference{ + Name: controller.CreateNginxResourceName(resourceName, dockerTestSecretName), + })) +} + +func TestGetAndUpdateSecret_NotFound(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + fakeClient := fake.NewFakeClient() + + provisioner := &NginxProvisioner{ + cfg: Config{ + GatewayPodConfig: &config.GatewayPodConfig{ + Namespace: "default", + }, + }, + k8sClient: fakeClient, + } + + _, err := provisioner.getAndUpdateSecret("non-existent-secret", metav1.ObjectMeta{ + Name: "new-secret", + Namespace: "default", + }) + + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("error getting secret")) +} + +func TestBuildNginxResourceObjectsForDeletion(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + provisioner := &NginxProvisioner{} + + deploymentNSName := types.NamespacedName{ + Name: "gw-nginx", + Namespace: "default", + } + + objects := provisioner.buildNginxResourceObjectsForDeletion(deploymentNSName) + + g.Expect(objects).To(HaveLen(5)) + + validateMeta := func(obj client.Object, name string) { + g.Expect(obj.GetName()).To(Equal(name)) + g.Expect(obj.GetNamespace()).To(Equal(deploymentNSName.Namespace)) + } + + depObj := objects[0] + dep, ok := depObj.(*appsv1.Deployment) + g.Expect(ok).To(BeTrue()) + validateMeta(dep, deploymentNSName.Name) + + svcObj := objects[1] + svc, ok := svcObj.(*corev1.Service) + g.Expect(ok).To(BeTrue()) + validateMeta(svc, deploymentNSName.Name) + + svcAcctObj := objects[2] + svcAcct, ok := svcAcctObj.(*corev1.ServiceAccount) + g.Expect(ok).To(BeTrue()) + validateMeta(svcAcct, deploymentNSName.Name) + + cmObj := objects[3] + cm, ok := cmObj.(*corev1.ConfigMap) + g.Expect(ok).To(BeTrue()) + validateMeta(cm, controller.CreateNginxResourceName(deploymentNSName.Name, nginxIncludesConfigMapNameSuffix)) + + cmObj = objects[4] + cm, ok = cmObj.(*corev1.ConfigMap) + g.Expect(ok).To(BeTrue()) + validateMeta(cm, controller.CreateNginxResourceName(deploymentNSName.Name, nginxAgentConfigMapNameSuffix)) +} + +func TestBuildNginxResourceObjectsForDeletion_Plus(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + provisioner := &NginxProvisioner{ + cfg: Config{ + Plus: true, + PlusUsageConfig: &config.UsageReportConfig{ + SecretName: jwtTestSecretName, + CASecretName: caTestSecretName, + ClientSSLSecretName: clientTestSecretName, + }, + NginxDockerSecretNames: []string{dockerTestSecretName}, + }, + } + + deploymentNSName := types.NamespacedName{ + Name: "gw-nginx", + Namespace: "default", + } + + objects := provisioner.buildNginxResourceObjectsForDeletion(deploymentNSName) + + g.Expect(objects).To(HaveLen(9)) + + validateMeta := func(obj client.Object, name string) { + g.Expect(obj.GetName()).To(Equal(name)) + g.Expect(obj.GetNamespace()).To(Equal(deploymentNSName.Namespace)) + } + + depObj := objects[0] + dep, ok := depObj.(*appsv1.Deployment) + g.Expect(ok).To(BeTrue()) + validateMeta(dep, deploymentNSName.Name) + + svcObj := objects[1] + svc, ok := svcObj.(*corev1.Service) + g.Expect(ok).To(BeTrue()) + validateMeta(svc, deploymentNSName.Name) + + svcAcctObj := objects[2] + svcAcct, ok := svcAcctObj.(*corev1.ServiceAccount) + g.Expect(ok).To(BeTrue()) + validateMeta(svcAcct, deploymentNSName.Name) + + cmObj := objects[3] + cm, ok := cmObj.(*corev1.ConfigMap) + g.Expect(ok).To(BeTrue()) + validateMeta(cm, controller.CreateNginxResourceName(deploymentNSName.Name, nginxIncludesConfigMapNameSuffix)) + + cmObj = objects[4] + cm, ok = cmObj.(*corev1.ConfigMap) + g.Expect(ok).To(BeTrue()) + validateMeta(cm, controller.CreateNginxResourceName(deploymentNSName.Name, nginxAgentConfigMapNameSuffix)) + + secretObj := objects[5] + secret, ok := secretObj.(*corev1.Secret) + g.Expect(ok).To(BeTrue()) + validateMeta(secret, controller.CreateNginxResourceName( + deploymentNSName.Name, + provisioner.cfg.NginxDockerSecretNames[0], + )) + + secretObj = objects[6] + secret, ok = secretObj.(*corev1.Secret) + g.Expect(ok).To(BeTrue()) + validateMeta(secret, controller.CreateNginxResourceName( + deploymentNSName.Name, + provisioner.cfg.PlusUsageConfig.CASecretName, + )) + + secretObj = objects[7] + secret, ok = secretObj.(*corev1.Secret) + g.Expect(ok).To(BeTrue()) + validateMeta(secret, controller.CreateNginxResourceName( + deploymentNSName.Name, + provisioner.cfg.PlusUsageConfig.ClientSSLSecretName, + )) +} diff --git a/internal/mode/static/provisioner/provisioner.go b/internal/mode/static/provisioner/provisioner.go index a505cf90ab..643fb3c6ff 100644 --- a/internal/mode/static/provisioner/provisioner.go +++ b/internal/mode/static/provisioner/provisioner.go @@ -41,13 +41,17 @@ type Provisioner interface { // Config is the configuration for the Provisioner. type Config struct { - DeploymentStore *agent.DeploymentStore - StatusQueue *status.Queue - Logger logr.Logger - GatewayPodConfig config.GatewayPodConfig - EventRecorder record.EventRecorder - GCName string - Plus bool + GCName string + + DeploymentStore agent.DeploymentStorer + StatusQueue *status.Queue + GatewayPodConfig *config.GatewayPodConfig + PlusUsageConfig *config.UsageReportConfig + EventRecorder record.EventRecorder + Logger logr.Logger + NginxDockerSecretNames []string + + Plus bool } // NginxProvisioner handles provisioning nginx kubernetes resources. @@ -70,7 +74,13 @@ func NewNginxProvisioner( mgr manager.Manager, cfg Config, ) (*NginxProvisioner, *events.EventLoop, error) { - store := newStore() + var jwtSecretName, caSecretName, clientSSLSecretName string + if cfg.Plus && cfg.PlusUsageConfig != nil { + jwtSecretName = cfg.PlusUsageConfig.SecretName + caSecretName = cfg.PlusUsageConfig.CASecretName + clientSSLSecretName = cfg.PlusUsageConfig.ClientSSLSecretName + } + store := newStore(cfg.NginxDockerSecretNames, jwtSecretName, caSecretName, clientSSLSecretName) selector := metav1.LabelSelector{ MatchLabels: map[string]string{ @@ -150,7 +160,10 @@ func (p *NginxProvisioner) provisionNginx( return nil } - objects := p.buildNginxResourceObjects(resourceName, gateway, nProxyCfg) + objects, err := p.buildNginxResourceObjects(resourceName, gateway, nProxyCfg) + if err != nil { + return fmt.Errorf("error provisioning nginx resources :%w", err) + } p.cfg.Logger.Info( "Creating/Updating nginx resources", @@ -194,10 +207,6 @@ func (p *NginxProvisioner) provisionNginx( } cancel() - if res != controllerutil.OperationResultCreated && res != controllerutil.OperationResultUpdated { - continue - } - switch o := obj.(type) { case *appsv1.Deployment: deploymentObj = o @@ -211,6 +220,10 @@ func (p *NginxProvisioner) provisionNginx( } } + if res != controllerutil.OperationResultCreated && res != controllerutil.OperationResultUpdated { + continue + } + result := cases.Title(language.English, cases.Compact).String(string(res)) p.cfg.Logger.V(1).Info( fmt.Sprintf("%s nginx %s", result, obj.GetObjectKind().GroupVersionKind().Kind), @@ -259,7 +272,11 @@ func (p *NginxProvisioner) reprovisionNginx( if !p.isLeader() { return nil } - objects := p.buildNginxResourceObjects(resourceName, gateway, nProxyCfg) + + objects, err := p.buildNginxResourceObjects(resourceName, gateway, nProxyCfg) + if err != nil { + return fmt.Errorf("error provisioning nginx resources :%w", err) + } p.cfg.Logger.Info( "Re-creating nginx resources", @@ -287,36 +304,34 @@ func (p *NginxProvisioner) reprovisionNginx( } func (p *NginxProvisioner) deprovisionNginx(ctx context.Context, gatewayNSName types.NamespacedName) error { - if !p.isLeader() { - return nil - } - - p.cfg.Logger.Info( - "Removing nginx resources for Gateway", - "name", gatewayNSName.Name, - "namespace", gatewayNSName.Namespace, - ) - deploymentNSName := types.NamespacedName{ Name: controller.CreateNginxResourceName(gatewayNSName.Name, p.cfg.GCName), Namespace: gatewayNSName.Namespace, } - objects := p.buildNginxResourceObjectsForDeletion(deploymentNSName) + if p.isLeader() { + p.cfg.Logger.Info( + "Removing nginx resources for Gateway", + "name", gatewayNSName.Name, + "namespace", gatewayNSName.Namespace, + ) - createCtx, cancel := context.WithTimeout(ctx, 30*time.Second) - defer cancel() + objects := p.buildNginxResourceObjectsForDeletion(deploymentNSName) - for _, obj := range objects { - if err := p.k8sClient.Delete(createCtx, obj); err != nil && !apierrors.IsNotFound(err) { - p.cfg.EventRecorder.Eventf( - obj, - corev1.EventTypeWarning, - "DeleteFailed", - "Failed to delete nginx resource: %s", - err.Error(), - ) - return err + createCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + + for _, obj := range objects { + if err := p.k8sClient.Delete(createCtx, obj); err != nil && !apierrors.IsNotFound(err) { + p.cfg.EventRecorder.Eventf( + obj, + corev1.EventTypeWarning, + "DeleteFailed", + "Failed to delete nginx resource: %s", + err.Error(), + ) + return err + } } } @@ -335,6 +350,10 @@ func (p *NginxProvisioner) RegisterGateway( gateway *graph.Gateway, resourceName string, ) error { + if !p.isLeader() { + return nil + } + gatewayNSName := client.ObjectKeyFromObject(gateway.Source) if updated := p.store.registerResourceInGatewayConfig(gatewayNSName, gateway); !updated { return nil diff --git a/internal/mode/static/provisioner/provisioner_test.go b/internal/mode/static/provisioner/provisioner_test.go new file mode 100644 index 0000000000..2c611912d8 --- /dev/null +++ b/internal/mode/static/provisioner/provisioner_test.go @@ -0,0 +1,361 @@ +package provisioner + +import ( + "context" + "testing" + + "github.com/go-logr/logr" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/manager" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + + ngfAPIv1alpha2 "github.com/nginx/nginx-gateway-fabric/apis/v1alpha2" + "github.com/nginx/nginx-gateway-fabric/internal/framework/controller" + "github.com/nginx/nginx-gateway-fabric/internal/framework/helpers" + "github.com/nginx/nginx-gateway-fabric/internal/mode/static/config" + "github.com/nginx/nginx-gateway-fabric/internal/mode/static/nginx/agent/agentfakes" + "github.com/nginx/nginx-gateway-fabric/internal/mode/static/state/graph" +) + +var ( + jwtTestSecretName = "jwt-secret" + caTestSecretName = "ca-secret" + clientTestSecretName = "client-secret" + dockerTestSecretName = "docker-secret" + ngfNamespace = "nginx-gateway" +) + +func createScheme() *runtime.Scheme { + scheme := runtime.NewScheme() + + utilruntime.Must(gatewayv1.Install(scheme)) + utilruntime.Must(corev1.AddToScheme(scheme)) + utilruntime.Must(appsv1.AddToScheme(scheme)) + + return scheme +} + +func expectResourcesToExist(g *WithT, k8sClient client.Client, nsName types.NamespacedName, plus bool) { + g.Expect(k8sClient.Get(context.TODO(), nsName, &appsv1.Deployment{})).To(Succeed()) + + g.Expect(k8sClient.Get(context.TODO(), nsName, &corev1.Service{})).To(Succeed()) + + g.Expect(k8sClient.Get(context.TODO(), nsName, &corev1.ServiceAccount{})).To(Succeed()) + + boostrapCM := types.NamespacedName{ + Name: controller.CreateNginxResourceName(nsName.Name, nginxIncludesConfigMapNameSuffix), + Namespace: nsName.Namespace, + } + g.Expect(k8sClient.Get(context.TODO(), boostrapCM, &corev1.ConfigMap{})).To(Succeed()) + + agentCM := types.NamespacedName{ + Name: controller.CreateNginxResourceName(nsName.Name, nginxAgentConfigMapNameSuffix), + Namespace: nsName.Namespace, + } + g.Expect(k8sClient.Get(context.TODO(), agentCM, &corev1.ConfigMap{})).To(Succeed()) + + if !plus { + return + } + + jwtSecret := types.NamespacedName{ + Name: controller.CreateNginxResourceName(nsName.Name, jwtTestSecretName), + Namespace: nsName.Namespace, + } + g.Expect(k8sClient.Get(context.TODO(), jwtSecret, &corev1.Secret{})).To(Succeed()) + + caSecret := types.NamespacedName{ + Name: controller.CreateNginxResourceName(nsName.Name, caTestSecretName), + Namespace: nsName.Namespace, + } + g.Expect(k8sClient.Get(context.TODO(), caSecret, &corev1.Secret{})).To(Succeed()) + + clientSSLSecret := types.NamespacedName{ + Name: controller.CreateNginxResourceName(nsName.Name, clientTestSecretName), + Namespace: nsName.Namespace, + } + g.Expect(k8sClient.Get(context.TODO(), clientSSLSecret, &corev1.Secret{})).To(Succeed()) + + dockerSecret := types.NamespacedName{ + Name: controller.CreateNginxResourceName(nsName.Name, dockerTestSecretName), + Namespace: nsName.Namespace, + } + g.Expect(k8sClient.Get(context.TODO(), dockerSecret, &corev1.Secret{})).To(Succeed()) +} + +func expectResourcesToNotExist(g *WithT, k8sClient client.Client, nsName types.NamespacedName) { + g.Expect(k8sClient.Get(context.TODO(), nsName, &appsv1.Deployment{})).ToNot(Succeed()) + + g.Expect(k8sClient.Get(context.TODO(), nsName, &corev1.Service{})).ToNot(Succeed()) + + g.Expect(k8sClient.Get(context.TODO(), nsName, &corev1.ServiceAccount{})).ToNot(Succeed()) + + boostrapCM := types.NamespacedName{ + Name: controller.CreateNginxResourceName(nsName.Name, nginxIncludesConfigMapNameSuffix), + Namespace: nsName.Namespace, + } + g.Expect(k8sClient.Get(context.TODO(), boostrapCM, &corev1.ConfigMap{})).ToNot(Succeed()) + + agentCM := types.NamespacedName{ + Name: controller.CreateNginxResourceName(nsName.Name, nginxAgentConfigMapNameSuffix), + Namespace: nsName.Namespace, + } + g.Expect(k8sClient.Get(context.TODO(), agentCM, &corev1.ConfigMap{})).ToNot(Succeed()) + + jwtSecret := types.NamespacedName{ + Name: controller.CreateNginxResourceName(nsName.Name, jwtTestSecretName), + Namespace: nsName.Namespace, + } + g.Expect(k8sClient.Get(context.TODO(), jwtSecret, &corev1.Secret{})).ToNot(Succeed()) + + caSecret := types.NamespacedName{ + Name: controller.CreateNginxResourceName(nsName.Name, caTestSecretName), + Namespace: nsName.Namespace, + } + g.Expect(k8sClient.Get(context.TODO(), caSecret, &corev1.Secret{})).ToNot(Succeed()) + + clientSSLSecret := types.NamespacedName{ + Name: controller.CreateNginxResourceName(nsName.Name, clientTestSecretName), + Namespace: nsName.Namespace, + } + g.Expect(k8sClient.Get(context.TODO(), clientSSLSecret, &corev1.Secret{})).ToNot(Succeed()) + + dockerSecret := types.NamespacedName{ + Name: controller.CreateNginxResourceName(nsName.Name, dockerTestSecretName), + Namespace: nsName.Namespace, + } + g.Expect(k8sClient.Get(context.TODO(), dockerSecret, &corev1.Secret{})).ToNot(Succeed()) +} + +func defaultNginxProvisioner( + objects ...client.Object, +) (*NginxProvisioner, client.Client, *agentfakes.FakeDeploymentStorer) { + fakeClient := fake.NewClientBuilder().WithScheme(createScheme()).WithObjects(objects...).Build() + deploymentStore := &agentfakes.FakeDeploymentStorer{} + + return &NginxProvisioner{ + store: newStore([]string{"docker-secret"}, "jwt-secret", "ca-secret", "client-ssl-secret"), + k8sClient: fakeClient, + cfg: Config{ + DeploymentStore: deploymentStore, + GatewayPodConfig: &config.GatewayPodConfig{ + InstanceName: "test-instance", + Namespace: ngfNamespace, + }, + Logger: logr.Discard(), + EventRecorder: &record.FakeRecorder{}, + GCName: "nginx", + Plus: true, + PlusUsageConfig: &config.UsageReportConfig{ + SecretName: jwtTestSecretName, + CASecretName: caTestSecretName, + ClientSSLSecretName: clientTestSecretName, + }, + NginxDockerSecretNames: []string{dockerTestSecretName}, + }, + leader: true, + }, fakeClient, deploymentStore +} + +func TestNewNginxProvisioner(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + mgr, err := manager.New(&rest.Config{}, manager.Options{Scheme: createScheme()}) + g.Expect(err).ToNot(HaveOccurred()) + + cfg := Config{ + GCName: "test-gc", + GatewayPodConfig: &config.GatewayPodConfig{ + InstanceName: "test-instance", + }, + Logger: logr.Discard(), + } + + provisioner, eventLoop, err := NewNginxProvisioner(context.TODO(), mgr, cfg) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(provisioner).NotTo(BeNil()) + g.Expect(eventLoop).NotTo(BeNil()) + + labelSelector := metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app.kubernetes.io/managed-by": "test-instance-test-gc", + "app.kubernetes.io/instance": "test-instance", + }, + } + g.Expect(provisioner.baseLabelSelector).To(Equal(labelSelector)) +} + +func TestEnable(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + dep := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gw-nginx", + Namespace: "default", + }, + } + provisioner, fakeClient, _ := defaultNginxProvisioner(dep) + provisioner.setResourceToDelete(types.NamespacedName{Name: "gw", Namespace: "default"}) + provisioner.leader = false + + provisioner.Enable(context.TODO()) + g.Expect(provisioner.isLeader()).To(BeTrue()) + g.Expect(provisioner.resourcesToDeleteOnStartup).To(BeEmpty()) + expectResourcesToNotExist(g, fakeClient, types.NamespacedName{Name: "gw-nginx", Namespace: "default"}) +} + +func TestRegisterGateway(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + gateway := &graph.Gateway{ + Source: &gatewayv1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gw", + Namespace: "default", + }, + }, + Valid: true, + } + + objects := []client.Object{ + gateway.Source, + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: jwtTestSecretName, + Namespace: ngfNamespace, + }, + }, + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: caTestSecretName, + Namespace: ngfNamespace, + }, + }, + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: clientTestSecretName, + Namespace: ngfNamespace, + }, + }, + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: dockerTestSecretName, + Namespace: ngfNamespace, + }, + }, + } + + provisioner, fakeClient, deploymentStore := defaultNginxProvisioner(objects...) + + g.Expect(provisioner.RegisterGateway(context.TODO(), gateway, "gw-nginx")).To(Succeed()) + expectResourcesToExist(g, fakeClient, types.NamespacedName{Name: "gw-nginx", Namespace: "default"}, true) // plus + + // Call again, no updates so nothing should happen + g.Expect(provisioner.RegisterGateway(context.TODO(), gateway, "gw-nginx")).To(Succeed()) + expectResourcesToExist(g, fakeClient, types.NamespacedName{Name: "gw-nginx", Namespace: "default"}, true) // plus + + // Now set the Gateway to invalid, and expect a deprovision to occur + invalid := &graph.Gateway{ + Source: &gatewayv1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gw", + Namespace: "default", + }, + }, + Valid: false, + } + g.Expect(provisioner.RegisterGateway(context.TODO(), invalid, "gw-nginx")).To(Succeed()) + expectResourcesToNotExist(g, fakeClient, types.NamespacedName{Name: "gw-nginx", Namespace: "default"}) + + resources := provisioner.store.getNginxResourcesForGateway(types.NamespacedName{Name: "gw", Namespace: "default"}) + g.Expect(resources).To(BeNil()) + + g.Expect(deploymentStore.RemoveCallCount()).To(Equal(1)) +} + +func TestNonLeaderProvisioner(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + provisioner, fakeClient, deploymentStore := defaultNginxProvisioner() + provisioner.leader = false + nsName := types.NamespacedName{Name: "gw-nginx", Namespace: "default"} + + g.Expect(provisioner.RegisterGateway(context.TODO(), nil, "gw-nginx")).To(Succeed()) + expectResourcesToNotExist(g, fakeClient, nsName) + + g.Expect(provisioner.provisionNginx(context.TODO(), "gw-nginx", nil, nil)).To(Succeed()) + expectResourcesToNotExist(g, fakeClient, nsName) + + g.Expect(provisioner.reprovisionNginx(context.TODO(), "gw-nginx", nil, nil)).To(Succeed()) + expectResourcesToNotExist(g, fakeClient, nsName) + + g.Expect(provisioner.deprovisionNginx(context.TODO(), nsName)).To(Succeed()) + expectResourcesToNotExist(g, fakeClient, nsName) + g.Expect(deploymentStore.RemoveCallCount()).To(Equal(1)) +} + +func TestProvisionerRestartsDeployment(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + gateway := &graph.Gateway{ + Source: &gatewayv1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gw", + Namespace: "default", + }, + }, + Valid: true, + EffectiveNginxProxy: &graph.EffectiveNginxProxy{ + Logging: &ngfAPIv1alpha2.NginxLogging{ + AgentLevel: helpers.GetPointer(ngfAPIv1alpha2.AgentLogLevelDebug), + }, + }, + } + + // provision everything first + provisioner, fakeClient, _ := defaultNginxProvisioner(gateway.Source) + provisioner.cfg.Plus = false + provisioner.cfg.NginxDockerSecretNames = nil + + g.Expect(provisioner.RegisterGateway(context.TODO(), gateway, "gw-nginx")).To(Succeed()) + expectResourcesToExist(g, fakeClient, types.NamespacedName{Name: "gw-nginx", Namespace: "default"}, false) // not plus + + // update agent config + updatedConfig := &graph.Gateway{ + Source: &gatewayv1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gw", + Namespace: "default", + }, + }, + Valid: true, + EffectiveNginxProxy: &graph.EffectiveNginxProxy{ + Logging: &ngfAPIv1alpha2.NginxLogging{ + AgentLevel: helpers.GetPointer(ngfAPIv1alpha2.AgentLogLevelInfo), + }, + }, + } + g.Expect(provisioner.RegisterGateway(context.TODO(), updatedConfig, "gw-nginx")).To(Succeed()) + + // verify deployment was updated with the restart annotation + dep := &appsv1.Deployment{} + key := types.NamespacedName{Name: "gw-nginx", Namespace: "default"} + g.Expect(fakeClient.Get(context.TODO(), key, dep)).To(Succeed()) + + g.Expect(dep.Spec.Template.GetAnnotations()).To(HaveKey(controller.RestartedAnnotation)) +} diff --git a/internal/mode/static/provisioner/setter.go b/internal/mode/static/provisioner/setter.go index 4195fd6d2a..dfe42321bc 100644 --- a/internal/mode/static/provisioner/setter.go +++ b/internal/mode/static/provisioner/setter.go @@ -18,6 +18,8 @@ func objectSpecSetter(object client.Object) controllerutil.MutateFn { return func() error { return nil } case *corev1.ConfigMap: return configMapSpecSetter(obj, obj.Data) + case *corev1.Secret: + return secretSpecSetter(obj, obj.Data) } return nil @@ -43,3 +45,10 @@ func configMapSpecSetter(configMap *corev1.ConfigMap, data map[string]string) co return nil } } + +func secretSpecSetter(secret *corev1.Secret, data map[string][]byte) controllerutil.MutateFn { + return func() error { + secret.Data = data + return nil + } +} diff --git a/internal/mode/static/provisioner/store.go b/internal/mode/static/provisioner/store.go index bf78ee21c0..5a57d5cc99 100644 --- a/internal/mode/static/provisioner/store.go +++ b/internal/mode/static/provisioner/store.go @@ -7,6 +7,7 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" @@ -16,12 +17,16 @@ import ( // NginxResources are all of the NGINX resources deployed in relation to a Gateway. type NginxResources struct { - Gateway *graph.Gateway - Deployment *appsv1.Deployment - Service *corev1.Service - ServiceAccount *corev1.ServiceAccount - BootstrapConfigMap *corev1.ConfigMap - AgentConfigMap *corev1.ConfigMap + Gateway *graph.Gateway + Deployment metav1.ObjectMeta + Service metav1.ObjectMeta + ServiceAccount metav1.ObjectMeta + BootstrapConfigMap metav1.ObjectMeta + AgentConfigMap metav1.ObjectMeta + PlusJWTSecret metav1.ObjectMeta + PlusClientSSLSecret metav1.ObjectMeta + PlusCASecret metav1.ObjectMeta + DockerSecrets []metav1.ObjectMeta } // store stores the cluster state needed by the provisioner and allows to update it from the events. @@ -32,13 +37,33 @@ type store struct { // nginxResources is a map of Gateway NamespacedNames and their associated nginx resources. nginxResources map[types.NamespacedName]*NginxResources + dockerSecretNames map[string]struct{} + // NGINX Plus secrets + jwtSecretName string + caSecretName string + clientSSLSecretName string + lock sync.RWMutex } -func newStore() *store { +func newStore( + dockerSecretNames []string, + jwtSecretName, + caSecretName, + clientSSLSecretName string, +) *store { + dockerSecretNamesMap := make(map[string]struct{}) + for _, name := range dockerSecretNames { + dockerSecretNamesMap[name] = struct{}{} + } + return &store{ - gateways: make(map[types.NamespacedName]*gatewayv1.Gateway), - nginxResources: make(map[types.NamespacedName]*NginxResources), + gateways: make(map[types.NamespacedName]*gatewayv1.Gateway), + nginxResources: make(map[types.NamespacedName]*NginxResources), + dockerSecretNames: dockerSecretNamesMap, + jwtSecretName: jwtSecretName, + caSecretName: caSecretName, + clientSSLSecretName: clientSSLSecretName, } } @@ -85,48 +110,105 @@ func (s *store) registerResourceInGatewayConfig(gatewayNSName types.NamespacedNa case *appsv1.Deployment: if cfg, ok := s.nginxResources[gatewayNSName]; !ok { s.nginxResources[gatewayNSName] = &NginxResources{ - Deployment: obj, + Deployment: obj.ObjectMeta, } } else { - cfg.Deployment = obj + cfg.Deployment = obj.ObjectMeta } case *corev1.Service: if cfg, ok := s.nginxResources[gatewayNSName]; !ok { s.nginxResources[gatewayNSName] = &NginxResources{ - Service: obj, + Service: obj.ObjectMeta, } } else { - cfg.Service = obj + cfg.Service = obj.ObjectMeta } case *corev1.ServiceAccount: if cfg, ok := s.nginxResources[gatewayNSName]; !ok { s.nginxResources[gatewayNSName] = &NginxResources{ - ServiceAccount: obj, + ServiceAccount: obj.ObjectMeta, } } else { - cfg.ServiceAccount = obj + cfg.ServiceAccount = obj.ObjectMeta } case *corev1.ConfigMap: - if cfg, ok := s.nginxResources[gatewayNSName]; !ok { - if strings.HasSuffix(obj.GetName(), nginxIncludesConfigMapNameSuffix) { - s.nginxResources[gatewayNSName] = &NginxResources{ - BootstrapConfigMap: obj, - } - } else if strings.HasSuffix(obj.GetName(), nginxAgentConfigMapNameSuffix) { + s.registerConfigMapInGatewayConfig(obj, gatewayNSName) + case *corev1.Secret: + s.registerSecretInGatewayConfig(obj, gatewayNSName) + } + + return true +} + +func (s *store) registerConfigMapInGatewayConfig(obj *corev1.ConfigMap, gatewayNSName types.NamespacedName) { + if cfg, ok := s.nginxResources[gatewayNSName]; !ok { + if strings.HasSuffix(obj.GetName(), nginxIncludesConfigMapNameSuffix) { + s.nginxResources[gatewayNSName] = &NginxResources{ + BootstrapConfigMap: obj.ObjectMeta, + } + } else if strings.HasSuffix(obj.GetName(), nginxAgentConfigMapNameSuffix) { + s.nginxResources[gatewayNSName] = &NginxResources{ + AgentConfigMap: obj.ObjectMeta, + } + } + } else { + if strings.HasSuffix(obj.GetName(), nginxIncludesConfigMapNameSuffix) { + cfg.BootstrapConfigMap = obj.ObjectMeta + } else if strings.HasSuffix(obj.GetName(), nginxAgentConfigMapNameSuffix) { + cfg.AgentConfigMap = obj.ObjectMeta + } + } +} + +func (s *store) registerSecretInGatewayConfig(obj *corev1.Secret, gatewayNSName types.NamespacedName) { + hasSuffix := func(str, suffix string) bool { + return suffix != "" && strings.HasSuffix(str, suffix) + } + + if cfg, ok := s.nginxResources[gatewayNSName]; !ok { + switch { + case hasSuffix(obj.GetName(), s.jwtSecretName): + s.nginxResources[gatewayNSName] = &NginxResources{ + PlusJWTSecret: obj.ObjectMeta, + } + case hasSuffix(obj.GetName(), s.caSecretName): + s.nginxResources[gatewayNSName] = &NginxResources{ + PlusCASecret: obj.ObjectMeta, + } + case hasSuffix(obj.GetName(), s.clientSSLSecretName): + s.nginxResources[gatewayNSName] = &NginxResources{ + PlusClientSSLSecret: obj.ObjectMeta, + } + } + + for secret := range s.dockerSecretNames { + if hasSuffix(obj.GetName(), secret) { s.nginxResources[gatewayNSName] = &NginxResources{ - AgentConfigMap: obj, + DockerSecrets: []metav1.ObjectMeta{obj.ObjectMeta}, } + break } - } else { - if strings.HasSuffix(obj.GetName(), nginxIncludesConfigMapNameSuffix) { - cfg.BootstrapConfigMap = obj - } else if strings.HasSuffix(obj.GetName(), nginxAgentConfigMapNameSuffix) { - cfg.AgentConfigMap = obj + } + } else { + switch { + case hasSuffix(obj.GetName(), s.jwtSecretName): + cfg.PlusJWTSecret = obj.ObjectMeta + case hasSuffix(obj.GetName(), s.caSecretName): + cfg.PlusCASecret = obj.ObjectMeta + case hasSuffix(obj.GetName(), s.clientSSLSecretName): + cfg.PlusClientSSLSecret = obj.ObjectMeta + } + + for secret := range s.dockerSecretNames { + if hasSuffix(obj.GetName(), secret) { + if len(cfg.DockerSecrets) == 0 { + cfg.DockerSecrets = []metav1.ObjectMeta{obj.ObjectMeta} + } else { + cfg.DockerSecrets = append(cfg.DockerSecrets, obj.ObjectMeta) + } } } } - - return true } func gatewayChanged(original, updated *graph.Gateway) bool { @@ -159,34 +241,33 @@ func (s *store) deleteResourcesForGateway(nsName types.NamespacedName) { delete(s.nginxResources, nsName) } -//nolint:gocyclo // will refactor at some point func (s *store) gatewayExistsForResource(object client.Object, nsName types.NamespacedName) *graph.Gateway { s.lock.RLock() defer s.lock.RUnlock() - resourceMatches := func(obj client.Object) bool { - return obj.GetName() == nsName.Name && obj.GetNamespace() == nsName.Namespace - } - for _, resources := range s.nginxResources { switch object.(type) { case *appsv1.Deployment: - if resources.Deployment != nil && resourceMatches(resources.Deployment) { + if resourceMatches(resources.Deployment, nsName) { return resources.Gateway } case *corev1.Service: - if resources.Service != nil && resourceMatches(resources.Service) { + if resourceMatches(resources.Service, nsName) { return resources.Gateway } case *corev1.ServiceAccount: - if resources.ServiceAccount != nil && resourceMatches(resources.ServiceAccount) { + if resourceMatches(resources.ServiceAccount, nsName) { return resources.Gateway } case *corev1.ConfigMap: - if resources.BootstrapConfigMap != nil && resourceMatches(resources.BootstrapConfigMap) { + if resourceMatches(resources.BootstrapConfigMap, nsName) { + return resources.Gateway + } + if resourceMatches(resources.AgentConfigMap, nsName) { return resources.Gateway } - if resources.AgentConfigMap != nil && resourceMatches(resources.AgentConfigMap) { + case *corev1.Secret: + if secretResourceMatches(resources, nsName) { return resources.Gateway } } @@ -194,3 +275,25 @@ func (s *store) gatewayExistsForResource(object client.Object, nsName types.Name return nil } + +func secretResourceMatches(resources *NginxResources, nsName types.NamespacedName) bool { + for _, secret := range resources.DockerSecrets { + if resourceMatches(secret, nsName) { + return true + } + } + + if resourceMatches(resources.PlusJWTSecret, nsName) { + return true + } + + if resourceMatches(resources.PlusClientSSLSecret, nsName) { + return true + } + + return resourceMatches(resources.PlusCASecret, nsName) +} + +func resourceMatches(objMeta metav1.ObjectMeta, nsName types.NamespacedName) bool { + return objMeta.GetName() == nsName.Name && objMeta.GetNamespace() == nsName.Namespace +} diff --git a/internal/mode/static/provisioner/store_test.go b/internal/mode/static/provisioner/store_test.go new file mode 100644 index 0000000000..0358341f03 --- /dev/null +++ b/internal/mode/static/provisioner/store_test.go @@ -0,0 +1,505 @@ +package provisioner + +import ( + "fmt" + "testing" + + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + + ngfAPIv1alpha2 "github.com/nginx/nginx-gateway-fabric/apis/v1alpha2" + "github.com/nginx/nginx-gateway-fabric/internal/framework/controller" + "github.com/nginx/nginx-gateway-fabric/internal/framework/helpers" + "github.com/nginx/nginx-gateway-fabric/internal/mode/static/state/graph" +) + +func TestNewStore(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + store := newStore([]string{"docker-secret"}, "jwt-secret", "ca-secret", "client-ssl-secret") + + g.Expect(store).NotTo(BeNil()) + g.Expect(store.dockerSecretNames).To(HaveKey("docker-secret")) + g.Expect(store.jwtSecretName).To(Equal("jwt-secret")) + g.Expect(store.caSecretName).To(Equal("ca-secret")) + g.Expect(store.clientSSLSecretName).To(Equal("client-ssl-secret")) +} + +func TestUpdateGateway(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + store := newStore(nil, "", "", "") + gateway := &gatewayv1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-gateway", + Namespace: "default", + }, + } + nsName := client.ObjectKeyFromObject(gateway) + + store.updateGateway(gateway) + + g.Expect(store.gateways).To(HaveKey(nsName)) + g.Expect(store.getGateway(nsName)).To(Equal(gateway)) +} + +func TestDeleteGateway(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + store := newStore(nil, "", "", "") + nsName := types.NamespacedName{Name: "test-gateway", Namespace: "default"} + store.gateways[nsName] = &gatewayv1.Gateway{} + + store.deleteGateway(nsName) + + g.Expect(store.gateways).NotTo(HaveKey(nsName)) + g.Expect(store.getGateway(nsName)).To(BeNil()) +} + +func TestRegisterResourceInGatewayConfig(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + store := newStore([]string{"docker-secret"}, "jwt-secret", "ca-secret", "client-ssl-secret") + nsName := types.NamespacedName{Name: "test-gateway", Namespace: "default"} + + registerAndGetResources := func(obj interface{}) *NginxResources { + changed := store.registerResourceInGatewayConfig(nsName, obj) + g.Expect(changed).To(BeTrue(), fmt.Sprintf("failed: %T", obj)) + g.Expect(store.nginxResources).To(HaveKey(nsName), fmt.Sprintf("failed: %T", obj)) + + return store.getNginxResourcesForGateway(nsName) + } + + // Gateway, new config + gw := &graph.Gateway{} + resources := registerAndGetResources(gw) + g.Expect(resources.Gateway).To(Equal(gw)) + + // Gateway, updated config + gw = &graph.Gateway{ + Valid: true, + } + resources = registerAndGetResources(gw) + g.Expect(resources.Gateway).To(Equal(gw)) + + defaultMeta := metav1.ObjectMeta{ + Name: "test-resource", + Namespace: "default", + } + + // clear out resources before next test + store.deleteResourcesForGateway(nsName) + + // Deployment + dep := &appsv1.Deployment{ObjectMeta: defaultMeta} + resources = registerAndGetResources(dep) + g.Expect(resources.Deployment).To(Equal(defaultMeta)) + + // Deployment again, already exists + resources = registerAndGetResources(dep) + g.Expect(resources.Deployment).To(Equal(defaultMeta)) + + // clear out resources before next test + store.deleteResourcesForGateway(nsName) + + // Service + svc := &corev1.Service{ObjectMeta: defaultMeta} + resources = registerAndGetResources(svc) + g.Expect(resources.Service).To(Equal(defaultMeta)) + + // Service again, already exists + resources = registerAndGetResources(svc) + g.Expect(resources.Service).To(Equal(defaultMeta)) + + // clear out resources before next test + store.deleteResourcesForGateway(nsName) + + // ServiceAccount + svcAcct := &corev1.ServiceAccount{ObjectMeta: defaultMeta} + resources = registerAndGetResources(svcAcct) + g.Expect(resources.ServiceAccount).To(Equal(defaultMeta)) + + // ServiceAccount again, already exists + resources = registerAndGetResources(svcAcct) + g.Expect(resources.ServiceAccount).To(Equal(defaultMeta)) + + // clear out resources before next test + store.deleteResourcesForGateway(nsName) + + // ConfigMap + bootstrapCMMeta := metav1.ObjectMeta{ + Name: controller.CreateNginxResourceName(defaultMeta.Name, nginxIncludesConfigMapNameSuffix), + Namespace: defaultMeta.Namespace, + } + bootstrapCM := &corev1.ConfigMap{ObjectMeta: bootstrapCMMeta} + resources = registerAndGetResources(bootstrapCM) + g.Expect(resources.BootstrapConfigMap).To(Equal(bootstrapCMMeta)) + + // ConfigMap again, already exists + resources = registerAndGetResources(bootstrapCM) + g.Expect(resources.BootstrapConfigMap).To(Equal(bootstrapCMMeta)) + + // clear out resources before next test + store.deleteResourcesForGateway(nsName) + + // ConfigMap + agentCMMeta := metav1.ObjectMeta{ + Name: controller.CreateNginxResourceName(defaultMeta.Name, nginxAgentConfigMapNameSuffix), + Namespace: defaultMeta.Namespace, + } + agentCM := &corev1.ConfigMap{ObjectMeta: agentCMMeta} + resources = registerAndGetResources(agentCM) + g.Expect(resources.AgentConfigMap).To(Equal(agentCMMeta)) + + // ConfigMap again, already exists + resources = registerAndGetResources(agentCM) + g.Expect(resources.AgentConfigMap).To(Equal(agentCMMeta)) + + // clear out resources before next test + store.deleteResourcesForGateway(nsName) + + // Secret + jwtSecretMeta := metav1.ObjectMeta{ + Name: controller.CreateNginxResourceName(defaultMeta.Name, store.jwtSecretName), + Namespace: defaultMeta.Namespace, + } + jwtSecret := &corev1.Secret{ObjectMeta: jwtSecretMeta} + resources = registerAndGetResources(jwtSecret) + g.Expect(resources.PlusJWTSecret).To(Equal(jwtSecretMeta)) + + // Secret again, already exists + resources = registerAndGetResources(jwtSecret) + g.Expect(resources.PlusJWTSecret).To(Equal(jwtSecretMeta)) + + // clear out resources before next test + store.deleteResourcesForGateway(nsName) + + // Secret + caSecretMeta := metav1.ObjectMeta{ + Name: controller.CreateNginxResourceName(defaultMeta.Name, store.caSecretName), + Namespace: defaultMeta.Namespace, + } + caSecret := &corev1.Secret{ObjectMeta: caSecretMeta} + resources = registerAndGetResources(caSecret) + g.Expect(resources.PlusCASecret).To(Equal(caSecretMeta)) + + // Secret again, already exists + resources = registerAndGetResources(caSecret) + g.Expect(resources.PlusCASecret).To(Equal(caSecretMeta)) + + // clear out resources before next test + store.deleteResourcesForGateway(nsName) + + // Secret + clientSSLSecretMeta := metav1.ObjectMeta{ + Name: controller.CreateNginxResourceName(defaultMeta.Name, store.clientSSLSecretName), + Namespace: defaultMeta.Namespace, + } + clientSSLSecret := &corev1.Secret{ObjectMeta: clientSSLSecretMeta} + resources = registerAndGetResources(clientSSLSecret) + g.Expect(resources.PlusClientSSLSecret).To(Equal(clientSSLSecretMeta)) + + // Secret again, already exists + resources = registerAndGetResources(clientSSLSecret) + g.Expect(resources.PlusClientSSLSecret).To(Equal(clientSSLSecretMeta)) + + // clear out resources before next test + store.deleteResourcesForGateway(nsName) + + // Docker Secret + dockerSecretMeta := metav1.ObjectMeta{ + Name: controller.CreateNginxResourceName(defaultMeta.Name, "docker-secret"), + Namespace: defaultMeta.Namespace, + } + dockerSecret := &corev1.Secret{ObjectMeta: dockerSecretMeta} + resources = registerAndGetResources(dockerSecret) + g.Expect(resources.DockerSecrets).To(ContainElements(dockerSecretMeta)) + + // Docker Secret again, already exists + resources = registerAndGetResources(dockerSecret) + g.Expect(resources.DockerSecrets).To(ContainElement(dockerSecretMeta)) +} + +func TestGatewayChanged(t *testing.T) { + t.Parallel() + + tests := []struct { + original *graph.Gateway + updated *graph.Gateway + name string + changed bool + }{ + { + name: "nil gateway", + original: nil, + changed: true, + }, + { + name: "valid field changes", + original: &graph.Gateway{Valid: true}, + updated: &graph.Gateway{Valid: false}, + changed: true, + }, + { + name: "source changes", + original: &graph.Gateway{Source: &gatewayv1.Gateway{ + Spec: gatewayv1.GatewaySpec{ + Listeners: []gatewayv1.Listener{ + { + Port: 80, + }, + }, + }, + }}, + updated: &graph.Gateway{Source: &gatewayv1.Gateway{ + Spec: gatewayv1.GatewaySpec{ + Listeners: []gatewayv1.Listener{ + { + Port: 81, + }, + }, + }, + }}, + changed: true, + }, + { + name: "effective nginx proxy config changes", + original: &graph.Gateway{ + EffectiveNginxProxy: &graph.EffectiveNginxProxy{ + Kubernetes: &ngfAPIv1alpha2.KubernetesSpec{ + Deployment: &ngfAPIv1alpha2.DeploymentSpec{ + Replicas: helpers.GetPointer[int32](1), + }, + }, + }, + }, + updated: &graph.Gateway{ + EffectiveNginxProxy: &graph.EffectiveNginxProxy{ + Kubernetes: &ngfAPIv1alpha2.KubernetesSpec{ + Deployment: &ngfAPIv1alpha2.DeploymentSpec{ + Replicas: helpers.GetPointer[int32](2), + }, + }, + }, + }, + changed: true, + }, + { + name: "no changes", + original: &graph.Gateway{Source: &gatewayv1.Gateway{ + Spec: gatewayv1.GatewaySpec{ + Listeners: []gatewayv1.Listener{ + { + Port: 80, + }, + }, + }, + }}, + updated: &graph.Gateway{Source: &gatewayv1.Gateway{ + Spec: gatewayv1.GatewaySpec{ + Listeners: []gatewayv1.Listener{ + { + Port: 80, + }, + }, + }, + }}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + g.Expect(gatewayChanged(test.original, test.updated)).To(Equal(test.changed)) + }) + } +} + +func TestDeleteResourcesForGateway(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + store := newStore(nil, "", "", "") + nsName := types.NamespacedName{Name: "test-gateway", Namespace: "default"} + store.nginxResources[nsName] = &NginxResources{} + + store.deleteResourcesForGateway(nsName) + + g.Expect(store.nginxResources).NotTo(HaveKey(nsName)) +} + +func TestGatewayExistsForResource(t *testing.T) { + t.Parallel() + + store := newStore(nil, "", "", "") + gateway := &graph.Gateway{} + store.nginxResources[types.NamespacedName{Name: "test-gateway", Namespace: "default"}] = &NginxResources{ + Gateway: gateway, + Deployment: metav1.ObjectMeta{ + Name: "test-deployment", + Namespace: "default", + }, + Service: metav1.ObjectMeta{ + Name: "test-service", + Namespace: "default", + }, + ServiceAccount: metav1.ObjectMeta{ + Name: "test-serviceaccount", + Namespace: "default", + }, + BootstrapConfigMap: metav1.ObjectMeta{ + Name: "test-bootstrap-configmap", + Namespace: "default", + }, + AgentConfigMap: metav1.ObjectMeta{ + Name: "test-agent-configmap", + Namespace: "default", + }, + PlusJWTSecret: metav1.ObjectMeta{ + Name: "test-jwt-secret", + Namespace: "default", + }, + PlusCASecret: metav1.ObjectMeta{ + Name: "test-ca-secret", + Namespace: "default", + }, + PlusClientSSLSecret: metav1.ObjectMeta{ + Name: "test-client-ssl-secret", + Namespace: "default", + }, + DockerSecrets: []metav1.ObjectMeta{ + { + Name: "test-docker-secret", + Namespace: "default", + }, + }, + } + + tests := []struct { + expected *graph.Gateway + object client.Object + name string + }{ + { + name: "Deployment exists", + object: &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-deployment", + Namespace: "default", + }, + }, + expected: gateway, + }, + { + name: "Service exists", + object: &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-service", + Namespace: "default", + }, + }, + expected: gateway, + }, + { + name: "ServiceAccount exists", + object: &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-serviceaccount", + Namespace: "default", + }, + }, + expected: gateway, + }, + { + name: "Bootstrap ConfigMap exists", + object: &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-bootstrap-configmap", + Namespace: "default", + }, + }, + expected: gateway, + }, + { + name: "Agent ConfigMap exists", + object: &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-agent-configmap", + Namespace: "default", + }, + }, + expected: gateway, + }, + { + name: "JWT Secret exists", + object: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-jwt-secret", + Namespace: "default", + }, + }, + expected: gateway, + }, + { + name: "CA Secret exists", + object: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-ca-secret", + Namespace: "default", + }, + }, + expected: gateway, + }, + { + name: "Client SSL Secret exists", + object: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-client-ssl-secret", + Namespace: "default", + }, + }, + expected: gateway, + }, + { + name: "Docker Secret exists", + object: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-docker-secret", + Namespace: "default", + }, + }, + expected: gateway, + }, + { + name: "Resource does not exist", + object: &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "non-existent-service", + Namespace: "default", + }, + }, + expected: nil, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + result := store.gatewayExistsForResource(test.object, client.ObjectKeyFromObject(test.object)) + g.Expect(result).To(Equal(test.expected)) + }) + } +} diff --git a/internal/mode/static/provisioner/templates.go b/internal/mode/static/provisioner/templates.go index 0b4d1ca308..05e1b96623 100644 --- a/internal/mode/static/provisioner/templates.go +++ b/internal/mode/static/provisioner/templates.go @@ -3,31 +3,31 @@ package provisioner import gotemplate "text/template" var ( - mainTemplate = gotemplate.Must(gotemplate.New("main").Parse(mainTemplateText)) - // mgmtTemplate = gotemplate.Must(gotemplate.New("mgmt").Parse(mgmtTemplateText)). + mainTemplate = gotemplate.Must(gotemplate.New("main").Parse(mainTemplateText)) + mgmtTemplate = gotemplate.Must(gotemplate.New("mgmt").Parse(mgmtTemplateText)) agentTemplate = gotemplate.Must(gotemplate.New("agent").Parse(agentTemplateText)) ) const mainTemplateText = ` error_log stderr {{ .ErrorLevel }};` -// const mgmtTemplateText = `mgmt { -// {{- if .Values.nginx.usage.endpoint }} -// usage_report endpoint={{ .Values.nginx.usage.endpoint }}; -// {{- end }} -// {{- if .Values.nginx.usage.skipVerify }} -// ssl_verify off; -// {{- end }} -// {{- if .Values.nginx.usage.caSecretName }} -// ssl_trusted_certificate /etc/nginx/certs-bootstrap/ca.crt; -// {{- end }} -// {{- if .Values.nginx.usage.clientSSLSecretName }} -// ssl_certificate /etc/nginx/certs-bootstrap/tls.crt; -// ssl_certificate_key /etc/nginx/certs-bootstrap/tls.key; -// {{- end }} -// enforce_initial_report off; -// deployment_context /etc/nginx/main-includes/deployment_ctx.json; -// }` +const mgmtTemplateText = `mgmt { + {{- if .UsageEndpoint }} + usage_report endpoint={{ .UsageEndpoint }}; + {{- end }} + {{- if .SkipVerify }} + ssl_verify off; + {{- end }} + {{- if .UsageCASecret }} + ssl_trusted_certificate /etc/nginx/certs-bootstrap/ca.crt; + {{- end }} + {{- if .UsageClientSSLSecret }} + ssl_certificate /etc/nginx/certs-bootstrap/tls.crt; + ssl_certificate_key /etc/nginx/certs-bootstrap/tls.key; + {{- end }} + enforce_initial_report off; + deployment_context /etc/nginx/main-includes/deployment_ctx.json; +}` const agentTemplateText = `command: server: