[ops] WebApp: Alerts on exessive RAM and CPU usage

Author: geropl

operations/observability/mixins/meta/rules/components/server/alerts.libsonnet

@@ -116,6 +116,38 @@
               description: 'Messagebus pod not running',
             },
           },
+          {
+            alert: 'WebAppServicesHighMemoryUsage',
+            // Reasoning: high rates of RAM consumption should only be temporary. Values based on past data (around 5-10 is constant)
+            expr: 'sum(rate(container_memory_working_set_bytes{container!="POD", node=~".*", pod=~"(server|ws-manager-bridge|usage)-.*"}[30m])) by (pod, node) > 10000000',
+            'for': '15m',
+            labels: {
+              // sent to the team internal channel until we fine tuned it
+              severity: 'warning',
+              team: 'webapp'
+            },
+            annotations: {
+              runbook_url: 'https://github.com/gitpod-io/runbooks/blob/main/runbooks/WebAppServicesHighMemoryUsage.md',
+              summary: 'WebApp services consume excessive amounts of memory. Investigation required.',
+              description: 'WebApp Services execcisve memory usage',
+            },
+          },
+          {
+            alert: 'WebAppServicesHighCPUUsage',
+            // Reasoning: high rates of CPU consumption should only be temporary.
+            expr: 'sum(rate(container_cpu_usage_seconds_total{container!="POD", node=~".*", pod=~"(content-service|dashboard|db|db-sync|messagebus|payment-endpoint|proxy|server|ws-manager-bridge|usage)-.*"}[5m])) by (pod, node) > 0.80',
+            'for': '10m',
+            labels: {
+              // sent to the team internal channel until we fine tuned it
+              severity: 'warning',
+              team: 'webapp'
+            },
+            annotations: {
+              runbook_url: 'https://github.com/gitpod-io/runbooks/blob/main/runbooks/WebAppServicesHighCPUUsage.md',
+              summary: 'WebApp services consume excessive amounts of CPU. Investigation required.',
+              description: 'WebApp Services execcisve CPU USAGE',
+            },
+          },
         ],
       },
     ],