adding new --exclude-resources-with-label manager CLI argument. Changed default exclusion by label mechanism to use this CLI argument instead of a hardcoded list of labels. Moved the default Rancher label exclusion to be a default CLI argument on the manager

Tested: made sure the current test for default exclusion fails after my
change, changed the test to reflect the changes, test now passes as
expected.
Executed e2e tests and they pass. Manually tested running the manager with the
new CLI argument and with that my labeled resource was skipped, then removed the
argument and it was propagated. Did the same with two different
annotations specified together as two CLI arguments, and resources that
had any of those labels were skipped as expected.

Author: dedri-github

cmd/manager/main.go

@@ -18,6 +18,7 @@ package main
 import (
 	"errors"
 	"flag"
+	"fmt"
 	"net/http"
 	"os"
 	"strings"
@@ -50,8 +51,9 @@ import (
 )
 
 var (
-	scheme   = runtime.NewScheme()
-	setupLog = zap.New().WithName("setup")
+	scheme                    = runtime.NewScheme()
+	setupLog                  = zap.New().WithName("setup")
+	excludeResourcesWithLabel = make(mapArg)
 )
 
 var (
@@ -156,10 +158,18 @@ func parseFlags() {
 	flag.BoolVar(&enableHRQ, "enable-hrq", false, "Enables hierarchical resource quotas")
 	flag.StringVar(&hncNamespace, "namespace", "hnc-system", "Namespace where hnc-manager and hnc resources deployed")
 	flag.DurationVar(&hrqSyncInterval, "hrq-sync-interval", 1*time.Minute, "Frequency to double-check that all HRQ usages are up-to-date (shouldn't be needed)")
+	flag.Var(excludeResourcesWithLabel, "exclude-objects-with-label", "An annotation specified as key=val that, if present, will cause HNC to skip objects that match this label. May be specified multiple times, with each instance specifying one label. See the user guide for more information.")
 	flag.Parse()
 
 	// Assign the array args to the configuration variables after the args are parsed.
 	config.UnpropagatedAnnotations = unpropagatedAnnotations
+
+	// Assign the exclusion label args to the configuration
+	for key, value := range excludeResourcesWithLabel {
+		setupLog.Info(fmt.Sprintf("Will exclude objects that match label %s=%s", key, value))
+		config.ExclusionByLabels = append(config.ExclusionByLabels, config.ExclusionByLabelsSpec{Key: key, Value: value})
+	}
+
 	config.SetNamespaces(includedNamespacesRegex, excludedNamespaces...)
 	if err := config.SetManagedMeta(managedNamespaceLabels, managedNamespaceAnnots); err != nil {
 		setupLog.Error(err, "Illegal flag values")
@@ -333,3 +343,29 @@ func (a *arrayArg) Set(val string) error {
 	*a = append(*a, val)
 	return nil
 }
+
+// mapArg is an arg that can be specified multiple times. All usages are
+// merged into this single map. Should be used by specifying `key=val`.
+type mapArg map[string]string
+
+func (m mapArg) String() string {
+	var items []string
+
+	for key, value := range m {
+		items = append(items, fmt.Sprintf("%s=\"%s\"", key, value))
+	}
+
+	return strings.Join(items, ", ")
+}
+
+func (m mapArg) Set(val string) error {
+	if !strings.Contains(val, "=") {
+		return errors.New("a map entry must be a key=val pair")
+	}
+	keyVal := strings.Split(val, "=")
+	if len(keyVal[0]) == 0 {
+		return errors.New("a map key must not be empty")
+	}
+	m[keyVal[0]] = keyVal[1]
+	return nil
+}

config/manager/manager.yaml

@@ -49,6 +49,7 @@ spec:
         - "--excluded-namespace=kube-public"
         - "--excluded-namespace=hnc-system"
         - "--excluded-namespace=kube-node-lease"
+        - "--exclude-objects-with-label=cattle.io/creator=norman"
         ports:
         - containerPort: 9443
           name: webhook-server

docs/user-guide/concepts.md

@@ -378,7 +378,10 @@ objects from being propagated by HNC.
   auto-created in new namespaces by Istio and Kubernetes respectively
 * Any objects with the label
   `cattle.io/creator:norman`, which are [inserted by Rancher to support
-  Projects](https://rancher.com/docs/rancher/v2.6/en/system-tools/#remove))
+  Projects](https://rancher.com/docs/rancher/v2.6/en/system-tools/#remove)).
+  Can be disabled by removing the default command line argument on the manager
+  `--exclude-objects-with-label=cattle.io/creator=norman`. Refer to [How-to:
+  Modify command-line arguments](how-to.md#modify-command-line-arguments) for more details.
 * *HNC v1.1+:* Secrets with type `helm.sh/release.v1`, which is auto-created in
   the namespaces where their respective Helm releases are deployed to.
 

docs/user-guide/how-to.md

@@ -956,3 +956,10 @@ Interesting parameters include:
   load on your metrics database (through increased metric cardinality) and also
   by increasing how carefully you need to guard your metrics against
   unauthorized viewers.
+* `--exclude-objects-with-label`: present by default with label
+  `cattle.io/creator=norman` in order to exclude objects created by Rancher
+  (refer to [Concepts: built in exceptions](concepts.md#built-in-exceptions)
+  for more information).
+  This argument may be specified multiple times, with each parameter representing
+  a pair of key and value that represent a label that should exclude an object from
+  propagation. Must have the `key=val` format.

internal/config/default_config.go

@@ -8,3 +8,14 @@ package config
 // This value is controlled by the --unpropagated-annotation command line, which may be set multiple
 // times.
 var UnpropagatedAnnotations []string
+
+// ExclusionByLabelsSpec specifies a label Key and Value which will cause an object to be excluded
+// from propagation if the object defines that label with this specific value.
+type ExclusionByLabelsSpec struct {
+	Key   string
+	Value string
+}
+
+// ExclusionByLabels is a configuration slice that contains all ExclusionByLabelSpec labels that should
+// cause objects to be ignored from propagation.
+var ExclusionByLabels []ExclusionByLabelsSpec

internal/objects/reconciler_test.go

@@ -446,15 +446,20 @@ var _ = Describe("Basic propagation", func() {
 		Eventually(HasObject(ctx, "secrets", barName, "helm-secret")).Should(BeFalse())
 	})
 
-	It("should not propagate builtin exclusions by labels", func() {
+	It("should not propagate objects excluded by labels", func() {
 		SetParent(ctx, barName, fooName)
+		config.ExclusionByLabels = append(config.ExclusionByLabels, config.ExclusionByLabelsSpec{Key: "cattle.io/creator", Value: "norman"})
+		config.ExclusionByLabels = append(config.ExclusionByLabels, config.ExclusionByLabelsSpec{Key: "ignore-label", Value: "label"})
+
 		MakeObjectWithLabels(ctx, "roles", fooName, "role-with-labels-blocked", map[string]string{"cattle.io/creator": "norman"})
+		MakeObjectWithLabels(ctx, "roles", fooName, "role-with-labels-blocked-2", map[string]string{"ignore-label": "label"})
 		MakeObjectWithLabels(ctx, "roles", fooName, "role-with-labels-wrong-value", map[string]string{"cattle.io/creator": "testme"})
 		MakeObjectWithLabels(ctx, "roles", fooName, "role-with-labels-something", map[string]string{"app": "hello-world"})
 		AddToHNCConfig(ctx, api.RBACGroup, api.RoleKind, api.Propagate)
 
 		// the first one should not propagate, everything else should
 		Consistently(HasObject(ctx, "roles", barName, "role-with-labels-blocked")).Should(BeFalse())
+		Consistently(HasObject(ctx, "roles", barName, "role-with-labels-blocked-2")).Should(BeFalse())
 		Eventually(HasObject(ctx, "roles", barName, "role-with-labels-wrong-value")).Should(BeTrue())
 		Eventually(HasObject(ctx, "roles", barName, "role-with-labels-something")).Should(BeTrue())
 

internal/selectors/selectors.go

@@ -13,6 +13,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/validation"
 
 	api "sigs.k8s.io/hierarchical-namespaces/api/v1alpha2"
+	"sigs.k8s.io/hierarchical-namespaces/internal/config"
 )
 
 // ShouldPropagate returns true if the given object should be propagated
@@ -197,19 +198,6 @@ func GetAllSelector(inst *unstructured.Unstructured) (bool, error) {
 // cmExclusionsByName are known (istio and kube-root) CA configmap which are excluded from propagation
 var cmExclusionsByName = []string{"istio-ca-root-cert", "kube-root-ca.crt"}
 
-// A label as a key, value pair, used to exclude resources matching this label (both key and value).
-type ExclusionByLabelsSpec struct {
-	Key   string
-	Value string
-}
-
-// ExclusionByLabelsSpec are known label key-value pairs which are excluded from propagation. Right
-// now only used to exclude resources created by Rancher, see "System Tools > Remove"
-// (https://rancher.com/docs/rancher/v2.6/en/system-tools/#remove)
-var exclusionByLabels = []ExclusionByLabelsSpec{
-	{Key: "cattle.io/creator", Value: "norman"},
-}
-
 // A annotation as a key, value pair, used to exclude resources matching this annotation
 type ExclusionByAnnotationsSpec struct {
 	Key   string
@@ -246,7 +234,7 @@ func isExcluded(inst *unstructured.Unstructured) (bool, error) {
 	}
 
 	// exclusion by labels
-	for _, res := range exclusionByLabels {
+	for _, res := range config.ExclusionByLabels {
 		gotLabelValue, ok := inst.GetLabels()[res.Key]
 		// check for presence has to be explicit, as empty label values are allowed and a
 		// nonexisting key in the `labels` map will also return an empty string ("") - potentially