Initial impl of managed labels/annotations

See kubernetes-retired#47. This is the initial implementation of managed labels and
annotations - that is, the ability to set a label (or annotation) in a
HierarchyConfiguration object, and have that label (...) propagated to
all descendants, similar to the way objects are propagated. As with
objects, only allowlisted labels are propagated, as defined by the
command line option '--managed-namespace-[labels|annotations]'.

Still to come: validator support, better conditions for conflicts,
better testing for external namespaces.

Tested: see new integ tests.

Author: adrianludwin

api/v1alpha2/hierarchy_types.go

@@ -124,6 +124,18 @@ type HierarchyConfigurationSpec struct {
 	// AllowCascadingDeletion indicates if the subnamespaces of this namespace are
 	// allowed to cascading delete.
 	AllowCascadingDeletion bool `json:"allowCascadingDeletion,omitempty"`
+
+	// Lables is a list of labels and values to apply to the current namespace and all of its
+	// descendants. All label keys must be specified on the command line by
+	// --managed-namespace-labels. A namespace cannot have a KVP that conflicts with one of its
+	// ancestors.
+	Labels []MetaKVP `json:"labels,omitempty"`
+
+	// Annotations is a list of annotations and values to apply to the current namespace and all of
+	// its descendants. All annotation keys must be specified on the command line by
+	// --managed-namespace-annotations. A namespace cannot have a KVP that conflicts with one of its
+	// ancestors.
+	Annotations []MetaKVP `json:"annotations,omitempty"`
 }
 
 // HierarchyStatus defines the observed state of Hierarchy
@@ -147,6 +159,15 @@ type HierarchyConfigurationList struct {
 	Items           []HierarchyConfiguration `json:"items"`
 }
 
+// MetaKVP represents a label or annotation
+type MetaKVP struct {
+	// Name is the name of the label or annotation.
+	Name string `json:"name"`
+
+	// Value is the value of the label or annotation.
+	Value string `json:"value"`
+}
+
 // metav1.Condition is introduced in k8s.io/apimachinery v0.20.0-alpha.1 and we
 // don't want to take a dependency on it yet, thus we copied the below struct from
 // https://github.com/kubernetes/apimachinery/blob/master/pkg/apis/meta/v1/types.go:

api/v1alpha2/zz_generated.deepcopy.go

@@ -65,6 +65,8 @@ var (
 	restartOnSecretRefresh  bool
 	unpropagatedAnnotations arrayArg
 	excludedNamespaces      arrayArg
+	managedNamespaceLables  arrayArg
+	managedNamespaceAnnots  arrayArg
 	includedNamespacesRegex string
 )
 
@@ -99,11 +101,29 @@ func main() {
 	flag.Var(&excludedNamespaces, "excluded-namespace", "A namespace that, if present, will be excluded from HNC management. May be specified multiple times, with each instance specifying one namespace. See the user guide for more information.")
 	flag.StringVar(&includedNamespacesRegex, "included-namespace-regex", ".*", "Namespace regular expression. Namespaces that match this regexp will be included and handle by HNC. As it is a regex, this parameter cannot be specified multiple times. Implicit wrapping of the expression \"^...$\" is done here")
 	flag.BoolVar(&restartOnSecretRefresh, "cert-restart-on-secret-refresh", false, "Kills the process when secrets are refreshed so that the pod can be restarted (secrets take up to 60s to be updated by running pods)")
+	flag.Var(&managedNamespaceLables, "managed-namespace-labels", "A list of labels on namespaces that are managed by HNC. These labels may only be set via the HierarchyConfiguration object. See the user guide for more information.")
+	flag.Var(&managedNamespaceAnnots, "managed-namespace-annotations", "A list of annotations on namespaces that are managed by HNC. These labels may only be set via the HierarchyConfiguration object. See the user guide for more details.")
 	flag.Parse()
+
 	// Assign the array args to the configuration variables after the args are parsed.
 	config.UnpropagatedAnnotations = unpropagatedAnnotations
-
 	config.SetNamespaces(includedNamespacesRegex, excludedNamespaces...)
+	config.ManagedNamespaceLabels = map[string]bool{}
+	for _, l := range managedNamespaceLables {
+		if strings.Contains(l, v1a2.MetaGroup) {
+			setupLog.Info("Error: cannot use --managed-namespace-labels to control labels in the " + v1a2.MetaGroup + " group")
+			os.Exit(1)
+		}
+		config.ManagedNamespaceLabels[l] = true
+	}
+	config.ManagedNamespaceAnnotations = map[string]bool{}
+	for _, a := range managedNamespaceAnnots {
+		if strings.Contains(a, v1a2.MetaGroup) {
+			setupLog.Info("Error: cannot use --managed-namespace-annotations to control annotations in the " + v1a2.MetaGroup + " group")
+			os.Exit(1)
+		}
+		config.ManagedNamespaceAnnotations[a] = true
+	}
 
 	// Enable OpenCensus exporters to export metrics
 	// to Stackdriver Monitoring.

cmd/manager/main.go

@@ -46,6 +46,44 @@ spec:
                 description: AllowCascadingDeletion indicates if the subnamespaces
                   of this namespace are allowed to cascading delete.
                 type: boolean
+              annotations:
+                description: Annotations is a list of annotations and values to apply
+                  to the current namespace and all of its descendants. All annotation
+                  keys must be specified on the command line by --managed-namespace-annotations.
+                  A namespace cannot have a KVP that conflicts with one of its ancestors.
+                items:
+                  description: MetaKVP represents a label or annotation
+                  properties:
+                    name:
+                      description: Name is the name of the label or annotation.
+                      type: string
+                    value:
+                      description: Value is the value of the label or annotation.
+                      type: string
+                  required:
+                  - name
+                  - value
+                  type: object
+                type: array
+              labels:
+                description: Lables is a list of labels and values to apply to the
+                  current namespace and all of its descendants. All label keys must
+                  be specified on the command line by --managed-namespace-labels.
+                  A namespace cannot have a KVP that conflicts with one of its ancestors.
+                items:
+                  description: MetaKVP represents a label or annotation
+                  properties:
+                    name:
+                      description: Name is the name of the label or annotation.
+                      type: string
+                    value:
+                      description: Value is the value of the label or annotation.
+                      type: string
+                  required:
+                  - name
+                  - value
+                  type: object
+                type: array
               parent:
                 description: Parent indicates the parent of this namespace, if any.
                 type: string

config/crd/bases/hnc.x-k8s.io_hierarchyconfigurations.yaml

@@ -1,10 +1,22 @@
 package config
 
-// UnpropgatedAnnotations is a list of annotations on objects that should _not_ be propagated by HNC.
-// Much like HNC itself, other systems (such as GKE Config Sync) use annotations to "claim" an
-// object - such as deleting objects it doesn't recognize. By removing these annotations on
-// propgated objects, HNC ensures that other systems won't attempt to claim the same object.
-//
-// This value is controlled by the --unpropagated-annotation command line, which may be set multiple
-// times.
-var UnpropagatedAnnotations []string
+var (
+	// UnpropgatedAnnotations is a list of annotations on objects that should _not_ be propagated by HNC.
+	// Much like HNC itself, other systems (such as GKE Config Sync) use annotations to "claim" an
+	// object - such as deleting objects it doesn't recognize. By removing these annotations on
+	// propgated objects, HNC ensures that other systems won't attempt to claim the same object.
+	//
+	// This value is controlled by the --unpropagated-annotation command line, which may be set multiple
+	// times.
+	UnpropagatedAnnotations []string
+
+	// ManagedNamespaceLabels is a set of labels whose values are controlled by the "labels" field in
+	// the HierarchyController CR. Any label in this list is removed from all managed namespaces
+	// unless specifically specified by the HC of the namespace or one of its ancestors.
+	ManagedNamespaceLabels map[string]bool
+
+	// ManagedNamespaceAnnotations is a set of annotations whose values are controlled by the "labels" field in
+	// the HierarchyController CR. Any label in this list is removed from all managed namespaces
+	// unless specifically specified by the HC of the namespace or one of its ancestors.
+	ManagedNamespaceAnnotations map[string]bool
+)

internal/config/default_config.go

@@ -32,6 +32,14 @@ type Namespace struct {
 	// and to store the tree labels of external namespaces.
 	labels map[string]string
 
+	// ManagedLabels are all managed labels explicitly set on this namespace (i.e., excluding anything
+	// set by ancestors).
+	ManagedLabels map[string]string
+
+	// ManagedAnnotations are all managed annotations explicitly set on this namespace (i.e.,
+	// excluding anything set by ancestors).
+	ManagedAnnotations map[string]string
+
 	// sourceObjects store the objects created by users, identified by GVK and name.
 	// It serves as the source of truth for object controllers to propagate objects.
 	sourceObjects objects

internal/forest/namespace.go

@@ -311,7 +311,8 @@ func (r *Reconciler) syncWithForest(log logr.Logger, nsInst *corev1.Namespace, i
 	initial := r.markExisting(log, ns)
 
 	// Sync labels and annotations, now that the structure's been updated.
-	nsCustomerLabelUpdated := r.syncTreeLabels(log, nsInst, ns)
+	updatedLabels := r.syncLabels(log, inst, nsInst, ns)
+	r.syncAnnotations(log, inst, nsInst, ns)
 
 	// Sync other spec and spec-like info
 	r.syncAnchors(log, ns, anms)
@@ -325,7 +326,7 @@ func (r *Reconciler) syncWithForest(log logr.Logger, nsInst *corev1.Namespace, i
 	r.syncConditions(log, inst, ns, wasHalted)
 
 	r.HNCConfigReconciler.Enqueue("namespace reconciled")
-	return initial || nsCustomerLabelUpdated
+	return initial || updatedLabels
 }
 
 // syncExternalNamespace sets external tree labels to the namespace in the forest
@@ -488,33 +489,60 @@ func (r *Reconciler) syncAnchors(log logr.Logger, ns *forest.Namespace, anms []s
 	}
 }
 
-// Sync namespace tree labels. Return true if the labels are updated.
-func (r *Reconciler) syncTreeLabels(log logr.Logger, nsInst *corev1.Namespace, ns *forest.Namespace) bool {
+// Sync namespace managed and tree labels. Return true if the labels are updated, so we know to
+// re-sync all objects that could be affected by exclusions.
+func (r *Reconciler) syncLabels(log logr.Logger, inst *api.HierarchyConfiguration, nsInst *corev1.Namespace, ns *forest.Namespace) bool {
+	// Get a list of all managed labels from the config object.
+	managed := map[string]string{}
+	for _, kvp := range inst.Spec.Labels {
+		managed[kvp.Name] = kvp.Value
+	}
+
+	// External namespaces can also have both managed and tree labels set directly. All other
+	// namespaces must have these removed now.
+	for k, v := range nsInst.Labels {
+		if config.ManagedNamespaceLabels[k] {
+			if ns.IsExternal() {
+				managed[k] = v
+			} else {
+				delete(nsInst.Labels, k)
+			}
+		}
+		if !ns.IsExternal() && strings.HasSuffix(k, api.LabelTreeDepthSuffix) {
+			delete(nsInst.Labels, k)
+		}
+	}
+
+	// Record all managed labels in the forest. If they've changed, we need to enqueue all descendants
+	// to propagate the changes.
+	if !reflect.DeepEqual(ns.ManagedLabels, managed) {
+		ns.ManagedLabels = managed
+		r.enqueueAffected(log, "managed labels have changed", ns.DescendantNames()...)
+	}
+
+	// For external namespaces, we're pretty much done since HNC mainly doesn't manage the metadata of
+	// external namespaces. Just make sure the zero-depth tree label is set if it wasn't already, and
+	// store all its labels so the tree labels can be retrieved later.
 	if ns.IsExternal() {
 		metadata.SetLabel(nsInst, nsInst.Name+api.LabelTreeDepthSuffix, "0")
-
-		// Set the labels so we can retrieve the external tree labels in the future if needed
 		ns.SetLabels(nsInst.Labels)
-		return false
-	}
 
-	// Remove all existing depth labels.
-	for k := range nsInst.Labels {
-		if strings.HasSuffix(k, api.LabelTreeDepthSuffix) {
-			delete(nsInst.Labels, k)
-		}
+		// There are no propagated objects in external namespaces so we don't need to notify the caller
+		// that any propagation-relevant labels have changed.
+		return false
 	}
 
-	// Look for all ancestors. Stop as soon as we find a namespaces that has a critical condition in
-	// the forest (note that AncestorHaltActivities is never included in the forest). This should handle orphans
-	// and cycles.
+	// Set all managed and tree labels, starting from the current namespace and going up through the
+	// hierarchy.
 	curNS := ns
 	depth := 0
 	for curNS != nil {
-		l := curNS.Name() + api.LabelTreeDepthSuffix
-		metadata.SetLabel(nsInst, l, strconv.Itoa(depth))
-		if curNS.IsHalted() {
-			break
+		// Set the tree label from this layer of hierarchy
+		metadata.SetLabel(nsInst, curNS.Name()+api.LabelTreeDepthSuffix, strconv.Itoa(depth))
+
+		// Add any managed labels. TODO: add conditions for conflicts.
+		for k, v := range curNS.ManagedLabels {
+			metadata.SetLabel(nsInst, k, v)
 		}
 
 		// If the root is an external namespace, add all its external tree labels too.
@@ -524,21 +552,83 @@ func (r *Reconciler) syncTreeLabels(log logr.Logger, nsInst *corev1.Namespace, n
 			for k, v := range curNS.GetTreeLabels() {
 				metadata.SetLabel(nsInst, k, strconv.Itoa(depth+v))
 			}
+
+			// Note that it's impossible to have an external namespace as a non-root (enforced elsewhere)
+			// so technically we don't need to break out of the loop here. But I find it cleaner.
 			break
 		}
 
+		// Stop if this namespace is halted, which could indicate a cycle or orphan.
+		if curNS.IsHalted() {
+			break
+		}
 		curNS = curNS.Parent()
 		depth++
 	}
-	// Update the labels in the forest so that we can quickly access the labels and
-	// compare if they match the given selector
+
+	// Update the labels in the forest so that they can be used in object propagation. If they've
+	// changed, return true so that all propagated objects in this namespace can be compared to its
+	// new labels.
 	if ns.SetLabels(nsInst.Labels) {
 		log.Info("Namespace managed and tree labels have been updated")
 		return true
 	}
 	return false
 }
 
+// Sync namespace managed annotations. This is mainly a simplified version of syncLabels with all
+// the tree stuff taken out.
+func (r *Reconciler) syncAnnotations(log logr.Logger, inst *api.HierarchyConfiguration, nsInst *corev1.Namespace, ns *forest.Namespace) {
+	// Get a list of all managed annotations from the config object.
+	managed := map[string]string{}
+	for _, kvp := range inst.Spec.Annotations {
+		managed[kvp.Name] = kvp.Value
+	}
+
+	// External namespaces can also have managed annotations set directly. All other namespaces must
+	// have these removed now.
+	for k, v := range nsInst.Annotations {
+		if config.ManagedNamespaceAnnotations[k] {
+			if ns.IsExternal() {
+				managed[k] = v
+			} else {
+				delete(nsInst.Labels, k)
+			}
+		}
+	}
+
+	// Record all managed annotations in the forest. If they've changed, we need to enqueue all
+	// descendants to propagate the changes.
+	if !reflect.DeepEqual(ns.ManagedAnnotations, managed) {
+		ns.ManagedAnnotations = managed
+		r.enqueueAffected(log, "managed annotations have changed", ns.DescendantNames()...)
+	}
+
+	// For external namespaces, we're done since HNC mainly doesn't manage the metadata of
+	// external namespaces.
+	if ns.IsExternal() {
+		return
+	}
+
+	// Set all managed annotations, starting from the current namespace and going up through the
+	// hierarchy.
+	curNS := ns
+	depth := 0
+	for curNS != nil {
+		// Add any managed annotations. TODO: add conditions for conflicts.
+		for k, v := range curNS.ManagedAnnotations {
+			metadata.SetAnnotation(nsInst, k, v)
+		}
+
+		// Stop if this namespace is halted, which could indicate a cycle or orphan.
+		if curNS.IsHalted() {
+			break
+		}
+		curNS = curNS.Parent()
+		depth++
+	}
+}
+
 func (r *Reconciler) syncConditions(log logr.Logger, inst *api.HierarchyConfiguration, ns *forest.Namespace, wasHalted bool) {
 	// If the halted status has changed, notify
 	if ns.IsHalted() != wasHalted {