Add alerts for high 5xx error rates

charts/app-of-apps/values-integration.yaml

@@ -93,11 +93,11 @@ ckanHelmValues:
       enabled: true
     appResources:
       limits:
-        memory: 1Gi
+        memory: 2Gi
         cpu: 1
       requests:
-        memory: 512Mi
-        cpu: 250m
+        memory: 1Gi
+        cpu: 500m
   fetch:
     appResources:
       limits:

charts/app-of-apps/values-production.yaml

@@ -107,11 +107,11 @@ ckanHelmValues:
       enabled: true
     appResources:
       limits:
-        memory: 1Gi
+        memory: 3Gi
         cpu: 1
       requests:
-        memory: 512Mi
-        cpu: 250m
+        memory: 1Gi
+        cpu: 500m
   fetch:
     replicaCount: 1
     appResources:

charts/app-of-apps/values-staging.yaml

@@ -102,11 +102,11 @@ ckanHelmValues:
       enabled: true
     appResources:
       limits:
-        memory: 1Gi
+        memory: 2Gi
         cpu: 1
       requests:
-        memory: 512Mi
-        cpu: 250m
+        memory: 1Gi
+        cpu: 500m
   fetch:
     replicaCount: 1
     appResources:

charts/ckan/templates/ckan/prometheusrule.yaml

@@ -0,0 +1,40 @@
+{{- if .Values.appEnabled }}
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: {{ .Release.Name }}-ckan
+  labels:
+    app: {{ .Release.Name }}-ckan
+    helm.sh/chart: govuk-dgu-charts
+    app.kubernetes.io/name: ckan
+    app.kubernetes.io/managed-by: Helm
+spec:
+  groups:
+  - name: ckan.rules
+    interval: 30s
+    rules:
+    - alert: Ckan5xxErrorRateHigh
+      expr: |
+        (sum(rate(http_requests_total{status=~"5.."}[5m])) by (job) /
+         sum(rate(http_requests_total[5m])) by (job)) * 100 > 20
+      for: 5m
+      labels:
+        severity: warning
+        service: ckan
+      annotations:
+        summary: "High 5xx error rate on CKAN"
+        description: "CKAN has a 5xx error rate of {{ $value | humanizePercentage }} (threshold: 20%) over the last 5 minutes"
+        runbook_url: "https://govuk-dgu-charts.publishing.service.gov.uk/runbooks/ckan-high-errors"
+    - alert: Ckan5xxErrorRateCritical
+      expr: |
+        (sum(rate(http_requests_total{status=~"5.."}[5m])) by (job) /
+         sum(rate(http_requests_total[5m])) by (job)) * 100 > 50
+      for: 2m
+      labels:
+        severity: critical
+        service: ckan
+      annotations:
+        summary: "Critical 5xx error rate on CKAN"
+        description: "CKAN has a critical 5xx error rate of {{ $value | humanizePercentage }} (threshold: 50%) over the last 5 minutes"
+        runbook_url: "https://govuk-dgu-charts.publishing.service.gov.uk/runbooks/ckan-critical-errors"
+{{- end }}

charts/datagovuk/templates/datagovuk/prometheusrule.yaml

@@ -0,0 +1,40 @@
+{{- if .Values.datagovuk.monitoring.enabled }}
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: {{ .Release.Name }}-datagovuk
+  labels:
+    app: {{ .Release.Name }}-datagovuk
+    helm.sh/chart: govuk-dgu-charts
+    app.kubernetes.io/name: datagovuk
+    app.kubernetes.io/managed-by: Helm
+spec:
+  groups:
+  - name: datagovuk-app.rules
+    interval: 30s
+    rules:
+    - alert: DatagovukApp5xxErrorRateHigh
+      expr: |
+        (sum(rate(http_requests_total{status=~"5.."}[5m])) by (job) /
+         sum(rate(http_requests_total[5m])) by (job)) * 100 > 20
+      for: 5m
+      labels:
+        severity: warning
+        service: datagovuk-app
+      annotations:
+        summary: "High 5xx error rate on datagovuk application"
+        description: "datagovuk-app has a 5xx error rate of {{ $value | humanizePercentage }} (threshold: 20%) over the last 5 minutes"
+        runbook_url: "https://govuk-dgu-charts.publishing.service.gov.uk/runbooks/datagovuk-app-high-errors"
+    - alert: DatagovukApp5xxErrorRateCritical
+      expr: |
+        (sum(rate(http_requests_total{status=~"5.."}[5m])) by (job) /
+         sum(rate(http_requests_total[5m])) by (job)) * 100 > 50
+      for: 2m
+      labels:
+        severity: critical
+        service: datagovuk-app
+      annotations:
+        summary: "Critical 5xx error rate on datagovuk application"
+        description: "datagovuk-app has a critical 5xx error rate of {{ $value | humanizePercentage }} (threshold: 50%) over the last 5 minutes"
+        runbook_url: "https://govuk-dgu-charts.publishing.service.gov.uk/runbooks/datagovuk-app-critical-errors"
+{{- end }}

charts/datagovuk/templates/find/prometheusrule.yaml

@@ -0,0 +1,40 @@
+{{- if .Values.monitoring.enabled }}
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: {{ .Release.Name }}-find
+  labels:
+    app: {{ .Release.Name }}-find
+    helm.sh/chart: govuk-dgu-charts
+    app.kubernetes.io/name: find
+    app.kubernetes.io/managed-by: Helm
+spec:
+  groups:
+  - name: datagovuk-find.rules
+    interval: 30s
+    rules:
+    - alert: DatagovukFind5xxErrorRateHigh
+      expr: |
+        (sum(rate(http_requests_total{status=~"5.."}[5m])) by (job) /
+         sum(rate(http_requests_total[5m])) by (job)) * 100 > 20
+      for: 5m
+      labels:
+        severity: warning
+        service: datagovuk-find
+      annotations:
+        summary: "High 5xx error rate on datagovuk-find"
+        description: "datagovuk-find has a 5xx error rate of {{ $value | humanizePercentage }} (threshold: 20%) over the last 5 minutes"
+        runbook_url: "https://govuk-dgu-charts.publishing.service.gov.uk/runbooks/datagovuk-find-high-errors"
+    - alert: DatagovukFind5xxErrorRateCritical
+      expr: |
+        (sum(rate(http_requests_total{status=~"5.."}[5m])) by (job) /
+         sum(rate(http_requests_total[5m])) by (job)) * 100 > 50
+      for: 2m
+      labels:
+        severity: critical
+        service: datagovuk-find
+      annotations:
+        summary: "Critical 5xx error rate on datagovuk-find"
+        description: "datagovuk-find has a critical 5xx error rate of {{ $value | humanizePercentage }} (threshold: 50%) over the last 5 minutes"
+        runbook_url: "https://govuk-dgu-charts.publishing.service.gov.uk/runbooks/datagovuk-find-critical-errors"
+{{- end }}