Import HRQ into HNC (disabled by default)

This implementation of Hierarchical Resource Quotas (HRQs) is based on
the implementation from the GKE Hierarchy Controller.  It's not a
straight import since the GKE version was deployed as a separate
workload, but I've made minimal adaptations to get it to fit into HNC.

Tested: This has only been _very_ lightly tested in HNC so far. I've
imported all the integ tests and E2E tests, and they all pass, but I
certainly wouldn't recommend that anyone use it just yet (all existing
e2e tests pass too). The default manifest includes the HRQ CRD and
permissions but nothing else (notably, the HRQ webhooks), and one
feature - the periodic resync of HRQ usages - is not yet implemented
(I'll do that in a followup PR).

Author: adrianludwin

Makefile

@@ -173,7 +173,7 @@ manifests: controller-gen
 		${KUSTOMIZE} edit add resource ../config/crd
 	${KUSTOMIZE} build manifests/ -o manifests/crds.yaml
 	@cd manifests && \
-		for variant in default-cc default-cm nowebhooks-cc ha-webhooks-cc ; do \
+		for variant in default-cc default-cm nowebhooks-cc ha-webhooks-cc hrq ; do \
 			echo "Building $${variant} manifest"; \
 			rm kustomization.yaml; \
 			touch kustomization.yaml && \
@@ -242,9 +242,13 @@ deploy-ha: docker-push kubectl manifests
 	-kubectl -n hnc-system delete deployment --all
 	kubectl apply -f manifests/ha.yaml
 
-ha-deploy-watch-ha:
+deploy-watch-ha:
 	kubectl logs -n hnc-system --follow deployment/hnc-controller-manager-ha manager
 
+deploy-hrq: docker-push kubectl manifests
+	-kubectl -n hnc-system delete deployment --all
+	kubectl apply -f manifests/hrq.yaml
+
 # No need to delete the HA configuration here - everything "extra" that it
 # installs is in hnc-system, which gets deleted by the default manifest.
 undeploy: manifests

api/v1alpha2/hierarchicalresourcequota_types.go

@@ -0,0 +1,69 @@
+package v1alpha2
+
+import (
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+const (
+	// HRQLabelCleanup is added to resources created by HRQ (specifically the RQ
+	// singletons) for easier cleanup later by a selector.
+	HRQLabelCleanup = MetaGroup + "/hrq"
+
+	// NonPropagateAnnotation is added to RQ singletons so that they are not
+	// overwritten by ancestors.
+	NonPropagateAnnotation = AnnotationPropagatePrefix + "/none"
+
+	// EventCannotWriteResourceQuota is for events when the reconcilers cannot
+	// write ResourceQuota from an HRQ. Usually it means the HRQ has invalid
+	// resource quota types. The error message will point to the HRQ object.
+	EventCannotWriteResourceQuota string = "CannotWriteResourceQuota"
+)
+
+// HierarchicalResourceQuotaSpec defines the desired hard limits to enforce for
+// a namespace and descendant namespaces
+type HierarchicalResourceQuotaSpec struct {
+	// Hard is the set of desired hard limits for each named resource
+	// +optional
+	Hard corev1.ResourceList `json:"hard,omitempty"`
+}
+
+// HierarchicalResourceQuotaStatus defines the enforced hard limits and observed
+// use for a namespace and descendant namespaces
+type HierarchicalResourceQuotaStatus struct {
+	// Hard is the set of enforced hard limits for each named resource
+	// +optional
+	Hard corev1.ResourceList `json:"hard,omitempty"`
+	// Used is the current observed total usage of the resource in the namespace
+	// and its descendant namespaces.
+	// +optional
+	Used corev1.ResourceList `json:"used,omitempty"`
+}
+
+// +kubebuilder:object:root=true
+// +kubebuilder:resource:path=hierarchicalresourcequotas,shortName=hrq,scope=Namespaced
+
+// HierarchicalResourceQuota sets aggregate quota restrictions enforced for a
+// namespace and descendant namespaces
+type HierarchicalResourceQuota struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	// Spec defines the desired quota
+	Spec HierarchicalResourceQuotaSpec `json:"spec,omitempty"`
+	// Status defines the actual enforced quota and its current usage
+	Status HierarchicalResourceQuotaStatus `json:"status,omitempty"`
+}
+
+// +kubebuilder:object:root=true
+
+// HierarchicalResourceQuotaList contains a list of HierarchicalResourceQuota
+type HierarchicalResourceQuotaList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []HierarchicalResourceQuota `json:"items"`
+}
+
+func init() {
+	SchemeBuilder.Register(&HierarchicalResourceQuota{}, &HierarchicalResourceQuotaList{})
+}

api/v1alpha2/zz_generated.deepcopy.go

@@ -73,6 +73,7 @@ var (
 	managedNamespaceAnnots  arrayArg
 	includedNamespacesRegex string
 	webhooksOnly            bool
+	enableHRQ               bool
 )
 
 // init preloads some global vars before main() starts. Since this is the top-level module, I'm not
@@ -149,6 +150,7 @@ func parseFlags() {
 	flag.Var(&managedNamespaceLabels, "managed-namespace-label", "A regex indicating the labels on namespaces that are managed by HNC. These labels may only be set via the HierarchyConfiguration object. All regexes are implictly wrapped by \"^...$\". This argument can be specified multiple times. See the user guide for more information.")
 	flag.Var(&managedNamespaceAnnots, "managed-namespace-annotation", "A regex indicating the annotations on namespaces that are managed by HNC. These annotations may only be set via the HierarchyConfiguration object. All regexes are implictly wrapped by \"^...$\". This argument can be specified multiple times. See the user guide for more information.")
 	flag.BoolVar(&webhooksOnly, "webhooks-only", false, "Disables the controllers so HNC can be run in HA webhook mode")
+	flag.BoolVar(&enableHRQ, "enable-hrq", false, "Enables hierarchical resource quotas")
 	flag.Parse()
 
 	// Assign the array args to the configuration variables after the args are parsed.
@@ -303,6 +305,7 @@ func startControllers(mgr ctrl.Manager, certsReady chan struct{}) {
 		NoWebhooks:    noWebhooks,
 		MaxReconciles: maxReconciles,
 		ReadOnly:      webhooksOnly,
+		HRQ:           enableHRQ,
 	}
 	setup.Create(setupLog, mgr, f, opts)
 

cmd/manager/main.go

@@ -0,0 +1,85 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.8.0
+  creationTimestamp: null
+  name: hierarchicalresourcequotas.hnc.x-k8s.io
+spec:
+  group: hnc.x-k8s.io
+  names:
+    kind: HierarchicalResourceQuota
+    listKind: HierarchicalResourceQuotaList
+    plural: hierarchicalresourcequotas
+    shortNames:
+    - hrq
+    singular: hierarchicalresourcequota
+  scope: Namespaced
+  versions:
+  - name: v1alpha2
+    schema:
+      openAPIV3Schema:
+        description: HierarchicalResourceQuota sets aggregate quota restrictions enforced
+          for a namespace and descendant namespaces
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: Spec defines the desired quota
+            properties:
+              hard:
+                additionalProperties:
+                  anyOf:
+                  - type: integer
+                  - type: string
+                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                  x-kubernetes-int-or-string: true
+                description: Hard is the set of desired hard limits for each named
+                  resource
+                type: object
+            type: object
+          status:
+            description: Status defines the actual enforced quota and its current
+              usage
+            properties:
+              hard:
+                additionalProperties:
+                  anyOf:
+                  - type: integer
+                  - type: string
+                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                  x-kubernetes-int-or-string: true
+                description: Hard is the set of enforced hard limits for each named
+                  resource
+                type: object
+              used:
+                additionalProperties:
+                  anyOf:
+                  - type: integer
+                  - type: string
+                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                  x-kubernetes-int-or-string: true
+                description: Used is the current observed total usage of the resource
+                  in the namespace and its descendant namespaces.
+                type: object
+            type: object
+        type: object
+    served: true
+    storage: true
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []

config/crd/bases/hnc.x-k8s.io_hierarchicalresourcequotas.yaml

@@ -5,6 +5,7 @@ resources:
 - bases/hnc.x-k8s.io_hierarchyconfigurations.yaml
 - bases/hnc.x-k8s.io_hncconfigurations.yaml
 - bases/hnc.x-k8s.io_subnamespaceanchors.yaml
+- bases/hnc.x-k8s.io_hierarchicalresourcequotas.yaml
 # +kubebuilder:scaffold:crdkustomizeresource
 
 patchesStrategicMerge:
@@ -16,11 +17,6 @@ patchesStrategicMerge:
 # - patches/webhook_in_subnamespaceanchors.yaml
 # +kubebuilder:scaffold:crdkustomizewebhookpatch
 
-# [CERTMANAGER] To enable webhook, uncomment all the sections with [CERTMANAGER] prefix.
-# patches here are for enabling the CA injection for each CRD
-#- patches/cainjection_in_hierarchies.yaml
-# +kubebuilder:scaffold:crdkustomizecainjectionpatch
-
 # the following config is for teaching kustomize how to do kustomization for CRDs.
 configurations:
 - kustomizeconfig.yaml

config/crd/kustomization.yaml

@@ -5,6 +5,18 @@ metadata:
   creationTimestamp: null
   name: manager-role
 rules:
+- apiGroups:
+  - ""
+  resources:
+  - resourcequotas
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - '*'
   resources:
@@ -21,6 +33,26 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - hnc.x-k8s.io
+  resources:
+  - hierarchicalresourcequotas
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - hnc.x-k8s.io
+  resources:
+  - hierarchicalresourcequotas/status
+  verbs:
+  - get
+  - patch
+  - update
 - apiGroups:
   - hnc.x-k8s.io
   resources:

config/rbac/role.yaml

@@ -0,0 +1,2 @@
+This is identical to default_cc (and should probably use that as a base in the
+future) except that it enables hierarchical resource quotas (HRQ).