From e5a4adb1c03ba1680a8c0af10bc85548b3acfdab Mon Sep 17 00:00:00 2001
From: Kenny Leung <kleung@chainguard.dev>
Date: Thu, 19 Dec 2024 13:19:44 -0800
Subject: [PATCH] alerts to catch 4xx type errors

Signed-off-by: Kenny Leung <kleung@chainguard.dev>
---
 modules/alerting/README.md    |   4 ++
 modules/alerting/main.tf      | 101 ++++++++++++++++++++++++++++++++++
 modules/alerting/variables.tf |  12 ++++
 3 files changed, 117 insertions(+)

diff --git a/modules/alerting/README.md b/modules/alerting/README.md
index 7afa89b8..7a016832 100644
--- a/modules/alerting/README.md
+++ b/modules/alerting/README.md
@@ -28,6 +28,8 @@ No modules.
 | [google_monitoring_alert_policy.cloud-run-scaling-failure](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/monitoring_alert_policy) | resource |
 | [google_monitoring_alert_policy.cloudrun_timeout](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/monitoring_alert_policy) | resource |
 | [google_monitoring_alert_policy.fatal](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/monitoring_alert_policy) | resource |
+| [google_monitoring_alert_policy.grpc_error_rate](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/monitoring_alert_policy) | resource |
+| [google_monitoring_alert_policy.http_error_rate](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/monitoring_alert_policy) | resource |
 | [google_monitoring_alert_policy.oom](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/monitoring_alert_policy) | resource |
 | [google_monitoring_alert_policy.panic](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/monitoring_alert_policy) | resource |
 | [google_monitoring_alert_policy.panic-stacktrace](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/monitoring_alert_policy) | resource |
@@ -48,6 +50,8 @@ No modules.
 | <a name="input_failure_rate_exclude_services"></a> [failure\_rate\_exclude\_services](#input\_failure\_rate\_exclude\_services) | List of service names to exclude from the 5xx failure rate alert | `list(string)` | `[]` | no |
 | <a name="input_failure_rate_ratio_threshold"></a> [failure\_rate\_ratio\_threshold](#input\_failure\_rate\_ratio\_threshold) | ratio threshold to alert for cloud run server failure rate. | `number` | `0.2` | no |
 | <a name="input_global_only_alerts"></a> [global\_only\_alerts](#input\_global\_only\_alerts) | only enable global alerts. when true, only create alerts that are global. | `bool` | `false` | no |
+| <a name="input_grpc_error_threshold"></a> [grpc\_error\_threshold](#input\_grpc\_error\_threshold) | threshold for grpc error. | `number` | `0.25` | no |
+| <a name="input_http_error_threshold"></a> [http\_error\_threshold](#input\_http\_error\_threshold) | threshold for http error. | `number` | `0.25` | no |
 | <a name="input_notification_channels"></a> [notification\_channels](#input\_notification\_channels) | List of notification channels to alert. | `list(string)` | `[]` | no |
 | <a name="input_notification_channels_email"></a> [notification\_channels\_email](#input\_notification\_channels\_email) | Email notification channel. | `list(string)` | `[]` | no |
 | <a name="input_notification_channels_pagerduty"></a> [notification\_channels\_pagerduty](#input\_notification\_channels\_pagerduty) | Email notification channel. | `list(string)` | `[]` | no |
diff --git a/modules/alerting/main.tf b/modules/alerting/main.tf
index 052dd5dc..f0cfdaeb 100644
--- a/modules/alerting/main.tf
+++ b/modules/alerting/main.tf
@@ -15,6 +15,7 @@ locals {
 locals {
   squad_log_filter = var.squad == "" ? "" : "labels.squad=\"${var.squad}\""
   name             = var.squad == "" ? "global" : var.squad
+  metric_filter    = var.squad == "" ? "" : "metric.labels.team=\"${var.squad}\""
 }
 
 locals {
@@ -916,3 +917,103 @@ resource "google_monitoring_alert_policy" "pinned" {
   enabled = "true"
   project = var.project_id
 }
+
+resource "google_monitoring_alert_policy" "http_error_rate" {
+  count = var.global_only_alerts ? 0 : 1
+
+  alert_strategy {
+    auto_close = "3600s" // 1 hour
+  }
+
+  combiner = "OR"
+
+  conditions {
+    condition_threshold {
+      aggregations {
+        alignment_period     = "60s"
+        cross_series_reducer = "REDUCE_MEAN"
+        per_series_aligner   = "ALIGN_RATE"
+        group_by_fields = [
+          "metric.label.team",
+          "metric.label.service_name",
+        ]
+      }
+
+      comparison = "COMPARISON_GT"
+      duration   = "300s"
+      # ignore registry service - valid 4xx use cases
+      # ignore prober - handled by prober alerts
+      # ignore 2xx and 3xx, only care 4xx and 5xx
+      filter = <<EOT
+        resource.type = "prometheus_target"
+        metric.type = "prometheus.googleapis.com/http_request_status_total/counter"
+        metric.labels.service_name != monitoring.regex.full_match(".*-registry"
+        metric.labels.service_name != monitoring.regex.full_match("prb-.*"
+        metric.labels.code != monitoring.regex.full_match("[23].."))"
+        ${local.metric_filter}
+      EOT
+
+      trigger {
+        count = "1"
+      }
+
+      threshold_value = var.http_error_threshold
+    }
+
+    display_name = "http error rate ${local.name}"
+  }
+  display_name = "http error rate ${local.name}"
+
+  notification_channels = length(var.notification_channels) != 0 ? var.notification_channels : local.slack
+
+  enabled = "true"
+  project = var.project_id
+}
+
+resource "google_monitoring_alert_policy" "grpc_error_rate" {
+  count = var.global_only_alerts ? 0 : 1
+
+  alert_strategy {
+    auto_close = "3600s" // 1 hour
+  }
+
+  combiner = "OR"
+
+  conditions {
+    condition_threshold {
+      aggregations {
+        alignment_period     = "60s"
+        cross_series_reducer = "REDUCE_MEAN"
+        per_series_aligner   = "ALIGN_RATE"
+        group_by_fields = [
+          "metric.label.team",
+          "metric.label.service_name",
+        ]
+      }
+
+      comparison = "COMPARISON_GT"
+      duration   = "300s"
+      # ignore OK and AlreadyExists code
+      filter = <<EOT
+        resource.type = "prometheus_target"
+        metric.type = "prometheus.googleapis.com/grpc_server_handled_total/counter"
+        metric.labels.grpc_code != monitoring.regex.full_match("OK|AlreadyExists"
+        ${local.metric_filter}
+      EOT
+
+      trigger {
+        count = "1"
+      }
+
+      threshold_value = var.grpc_error_threshold
+    }
+
+    display_name = "grpc error rate ${local.name}"
+  }
+  display_name = "grpc error rate ${local.name}"
+
+  notification_channels = length(var.notification_channels) != 0 ? var.notification_channels : local.slack
+
+  enabled = "true"
+  project = var.project_id
+}
diff --git a/modules/alerting/variables.tf b/modules/alerting/variables.tf
index ab024263..803d51f2 100644
--- a/modules/alerting/variables.tf
+++ b/modules/alerting/variables.tf
@@ -114,3 +114,15 @@ variable "global_only_alerts" {
   type        = bool
   default     = false
 }
+
+variable "http_error_threshold" {
+  description = "threshold for http error."
+  type        = number
+  default     = 0.25
+}
+
+variable "grpc_error_threshold" {
+  description = "threshold for grpc error."
+  type        = number
+  default     = 0.25
+}