Add CoderdIneligiblePrebuilds alert

Signed-off-by: Danny Kopping <[email protected]>

Author: dannykopping

coder-observability/runbooks/coderd.md

@@ -76,3 +76,10 @@ Terraform plugin.
 Your Enterprise license is approaching or has exceeded the number of seats purchased.
 
 Please contact your Coder sales contact, or visit https://coder.com/contact/sales.
+
+## CoderdIneligiblePrebuilds
+
+Prebuilds only become eligible to be claimed by users once the workspace's agent is a) running and b) all of its startup
+scripts have completed.
+
+If a prebuilt workspace is not eligible, view its agent logs to diagnose the problem.

coder-observability/templates/configmap-prometheus-alerts.yaml

@@ -4,7 +4,7 @@ metadata:
   name: metrics-alerts
   namespace: {{ .Release.Namespace }}
 data:
-  {{- $service := dict "service" "coder" -}}
+  {{- $service := dict "service" "coderd" -}}
 
   {{- with .Values.global.coder.alerts.coderd }} {{/* start-section */}}
   coderd.yaml: |-
@@ -104,6 +104,27 @@ data:
   {{- end }}
   {{- end }}
 
+  {{- with .groups.IneligiblePrebuilds }}
+  {{- $group := . }}
+  {{- if .enabled }}
+    - name: Coderd Ineligible Prebuilds
+      rules:
+      {{ $alert := "CoderdIneligiblePrebuilds" }}
+      {{- range $severity, $threshold := .thresholds }}
+      - alert: {{ $alert }}
+        expr: max by (template_name, preset_name) (coderd_prebuilds_running - coderd_prebuilds_eligible) > 0
+        for: {{ $group.delay }}
+        annotations:
+          summary: >
+            {{ `{{ $value }}` }} prebuilt workspace(s) are currently ineligible for claiming for the "{{ `{{ $labels.template_name }}` }}" template and "{{ `{{ $labels.preset_name }}` }}" preset.
+            This usually indicates that the agent has not started correctly, or is still running its startup scripts after an extended period of time.
+        labels:
+          severity: {{ $severity }}
+          runbook_url: {{ template "runbook-url" (deepCopy $ | merge (dict "alert" $alert) $service) }}
+      {{- end }}
+  {{- end }}
+  {{- end }}
+
   {{- end }} {{/* end-section */}}
 
 

coder-observability/values.yaml

@@ -76,6 +76,11 @@ global:
               notify: 2
               warning: 5
               critical: 10
+          IneligiblePrebuilds:
+            enabled: true
+            delay: 10m
+            thresholds:
+              notify: 1
       provisionerd:
         groups:
           Replicas: