Import HRQ into HNC (disabled by default)

Makefile

@@ -173,7 +173,7 @@ manifests: controller-gen
 		${KUSTOMIZE} edit add resource ../config/crd
 	${KUSTOMIZE} build manifests/ -o manifests/crds.yaml
 	@cd manifests && \
-		for variant in default-cc default-cm nowebhooks-cc ha-webhooks-cc ; do \
+		for variant in default-cc default-cm nowebhooks-cc ha-webhooks-cc hrq ; do \
 			echo "Building $${variant} manifest"; \
 			rm kustomization.yaml; \
 			touch kustomization.yaml && \
@@ -242,9 +242,13 @@ deploy-ha: docker-push kubectl manifests
 	-kubectl -n hnc-system delete deployment --all
 	kubectl apply -f manifests/ha.yaml
 
-ha-deploy-watch-ha:
+deploy-watch-ha:
 	kubectl logs -n hnc-system --follow deployment/hnc-controller-manager-ha manager
 
+deploy-hrq: docker-push kubectl manifests
+	-kubectl -n hnc-system delete deployment --all
+	kubectl apply -f manifests/hrq.yaml
+
 # No need to delete the HA configuration here - everything "extra" that it
 # installs is in hnc-system, which gets deleted by the default manifest.
 undeploy: manifests

api/v1alpha2/hierarchicalresourcequota_types.go

@@ -0,0 +1,69 @@
+package v1alpha2
+
+import (
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+const (
+	// HRQLabelCleanup is added to resources created by HRQ (specifically the RQ
+	// singletons) for easier cleanup later by a selector.
+	HRQLabelCleanup = MetaGroup + "/hrq"
+
+	// NonPropagateAnnotation is added to RQ singletons so that they are not
+	// overwritten by ancestors.
+	NonPropagateAnnotation = AnnotationPropagatePrefix + "/none"
+
+	// EventCannotWriteResourceQuota is for events when the reconcilers cannot
+	// write ResourceQuota from an HRQ. Usually it means the HRQ has invalid
+	// resource quota types. The error message will point to the HRQ object.
+	EventCannotWriteResourceQuota string = "CannotWriteResourceQuota"
+)
+
+// HierarchicalResourceQuotaSpec defines the desired hard limits to enforce for
+// a namespace and descendant namespaces
+type HierarchicalResourceQuotaSpec struct {
+	// Hard is the set of desired hard limits for each named resource
+	// +optional
+	Hard corev1.ResourceList `json:"hard,omitempty"`
+}
+
+// HierarchicalResourceQuotaStatus defines the enforced hard limits and observed
+// use for a namespace and descendant namespaces
+type HierarchicalResourceQuotaStatus struct {
+	// Hard is the set of enforced hard limits for each named resource
+	// +optional
+	Hard corev1.ResourceList `json:"hard,omitempty"`
+	// Used is the current observed total usage of the resource in the namespace
+	// and its descendant namespaces.
+	// +optional
+	Used corev1.ResourceList `json:"used,omitempty"`
+}
+
+// +kubebuilder:object:root=true
+// +kubebuilder:resource:path=hierarchicalresourcequotas,shortName=hrq,scope=Namespaced
+
+// HierarchicalResourceQuota sets aggregate quota restrictions enforced for a
+// namespace and descendant namespaces
+type HierarchicalResourceQuota struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	// Spec defines the desired quota
+	Spec HierarchicalResourceQuotaSpec `json:"spec,omitempty"`
+	// Status defines the actual enforced quota and its current usage
+	Status HierarchicalResourceQuotaStatus `json:"status,omitempty"`
+}
+
+// +kubebuilder:object:root=true
+
+// HierarchicalResourceQuotaList contains a list of HierarchicalResourceQuota
+type HierarchicalResourceQuotaList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []HierarchicalResourceQuota `json:"items"`
+}
+
+func init() {
+	SchemeBuilder.Register(&HierarchicalResourceQuota{}, &HierarchicalResourceQuotaList{})
+}

cmd/manager/main.go

@@ -73,6 +73,7 @@ var (
 	managedNamespaceAnnots  arrayArg
 	includedNamespacesRegex string
 	webhooksOnly            bool
+	enableHRQ               bool
 )
 
 // init preloads some global vars before main() starts. Since this is the top-level module, I'm not
@@ -149,6 +150,7 @@ func parseFlags() {
 	flag.Var(&managedNamespaceLabels, "managed-namespace-label", "A regex indicating the labels on namespaces that are managed by HNC. These labels may only be set via the HierarchyConfiguration object. All regexes are implictly wrapped by \"^...$\". This argument can be specified multiple times. See the user guide for more information.")
 	flag.Var(&managedNamespaceAnnots, "managed-namespace-annotation", "A regex indicating the annotations on namespaces that are managed by HNC. These annotations may only be set via the HierarchyConfiguration object. All regexes are implictly wrapped by \"^...$\". This argument can be specified multiple times. See the user guide for more information.")
 	flag.BoolVar(&webhooksOnly, "webhooks-only", false, "Disables the controllers so HNC can be run in HA webhook mode")
+	flag.BoolVar(&enableHRQ, "enable-hrq", false, "Enables hierarchical resource quotas")
 	flag.Parse()
 
 	// Assign the array args to the configuration variables after the args are parsed.
@@ -303,6 +305,7 @@ func startControllers(mgr ctrl.Manager, certsReady chan struct{}) {
 		NoWebhooks:    noWebhooks,
 		MaxReconciles: maxReconciles,
 		ReadOnly:      webhooksOnly,
+		HRQ:           enableHRQ,
 	}
 	setup.Create(setupLog, mgr, f, opts)
 

config/crd/bases/hnc.x-k8s.io_hierarchicalresourcequotas.yaml

@@ -0,0 +1,85 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.8.0
+  creationTimestamp: null
+  name: hierarchicalresourcequotas.hnc.x-k8s.io
+spec:
+  group: hnc.x-k8s.io
+  names:
+    kind: HierarchicalResourceQuota
+    listKind: HierarchicalResourceQuotaList
+    plural: hierarchicalresourcequotas
+    shortNames:
+    - hrq
+    singular: hierarchicalresourcequota
+  scope: Namespaced
+  versions:
+  - name: v1alpha2
+    schema:
+      openAPIV3Schema:
+        description: HierarchicalResourceQuota sets aggregate quota restrictions enforced
+          for a namespace and descendant namespaces
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: Spec defines the desired quota
+            properties:
+              hard:
+                additionalProperties:
+                  anyOf:
+                  - type: integer
+                  - type: string
+                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                  x-kubernetes-int-or-string: true
+                description: Hard is the set of desired hard limits for each named
+                  resource
+                type: object
+            type: object
+          status:
+            description: Status defines the actual enforced quota and its current
+              usage
+            properties:
+              hard:
+                additionalProperties:
+                  anyOf:
+                  - type: integer
+                  - type: string
+                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                  x-kubernetes-int-or-string: true
+                description: Hard is the set of enforced hard limits for each named
+                  resource
+                type: object
+              used:
+                additionalProperties:
+                  anyOf:
+                  - type: integer
+                  - type: string
+                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                  x-kubernetes-int-or-string: true
+                description: Used is the current observed total usage of the resource
+                  in the namespace and its descendant namespaces.
+                type: object
+            type: object
+        type: object
+    served: true
+    storage: true
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []

config/crd/kustomization.yaml

@@ -5,6 +5,7 @@ resources:
 - bases/hnc.x-k8s.io_hierarchyconfigurations.yaml
 - bases/hnc.x-k8s.io_hncconfigurations.yaml
 - bases/hnc.x-k8s.io_subnamespaceanchors.yaml
+- bases/hnc.x-k8s.io_hierarchicalresourcequotas.yaml
 # +kubebuilder:scaffold:crdkustomizeresource
 
 patchesStrategicMerge:
@@ -16,11 +17,6 @@ patchesStrategicMerge:
 # - patches/webhook_in_subnamespaceanchors.yaml
 # +kubebuilder:scaffold:crdkustomizewebhookpatch
 
-# [CERTMANAGER] To enable webhook, uncomment all the sections with [CERTMANAGER] prefix.
-# patches here are for enabling the CA injection for each CRD
-#- patches/cainjection_in_hierarchies.yaml
-# +kubebuilder:scaffold:crdkustomizecainjectionpatch
-
 # the following config is for teaching kustomize how to do kustomization for CRDs.
 configurations:
 - kustomizeconfig.yaml

config/rbac/role.yaml

@@ -5,6 +5,18 @@ metadata:
   creationTimestamp: null
   name: manager-role
 rules:
+- apiGroups:
+  - ""
+  resources:
+  - resourcequotas
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - '*'
   resources:
@@ -21,6 +33,26 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - hnc.x-k8s.io
+  resources:
+  - hierarchicalresourcequotas
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - hnc.x-k8s.io
+  resources:
+  - hierarchicalresourcequotas/status
+  verbs:
+  - get
+  - patch
+  - update
 - apiGroups:
   - hnc.x-k8s.io
   resources:

config/variants/hrq/README

@@ -0,0 +1,2 @@
+This is identical to default_cc (and should probably use that as a base in the
+future) except that it enables hierarchical resource quotas (HRQ).