Alerta

ansible/host_vars/lovelace/prometheus.yml

@@ -116,6 +116,8 @@ prometheus_rules: |
           for: 15m
           labels:
             severity: warning
+            requires_ack: "true"
+            instance: '{{ $labels.instance }}/{{ $labels.name }}'
           annotations:
             summary: Node systemd unit {{ $labels.name }} has failed (instance {{ $labels.instance }})
 
@@ -125,7 +127,8 @@ prometheus_rules: |
         expr: pg_up == 0
         for: 0m
         labels:
-          severity: page
+          severity: critical
+          service: postgresql
         annotations:
           summary: "PostgreSQL is offline"
           description: "Postgres Exporter cannot connect to PostgreSQL."
@@ -134,7 +137,9 @@ prometheus_rules: |
         expr: (sum(pg_stat_activity_count) by (instance)) / on (instance) pg_settings_max_connections * 100 > 80
         for: 1m
         labels:
-          severity: page
+          severity: major
+          service: postgresql
+          requires_ack: "true"
         annotations:
           summary: PostgreSQL connections near max_connections setting
           description: "PostgreSQL instance is near the maximum connection limit, currently {{ $value }} connections"
@@ -143,7 +148,9 @@ prometheus_rules: |
         expr: increase(pg_stat_database_deadlocks{datname!~"template.*|postgres"}[1m]) > 3
         for: 1m
         labels:
-          severity: page
+          severity: major
+          service: postgresql
+          requires_ack: "true"
         annotations:
           summary: Too many deadlocked tables
           description: "PostgreSQL has dead-locks, value: {{ $value }}"
@@ -154,14 +161,15 @@ prometheus_rules: |
           expr: postfix_up != 1
           for: 5m
           labels:
-            severity: warning
+            severity: critical
           annotations:
             summary: Postfix is down (instance {{ $labels.instance }})
         - alert: postfix/smtp-temporary-errors
           expr: rate(postfix_smtpd_messages_rejected_total{code=~"^4.*"}[15m]) > 0
           for: 0m
           labels:
             severity: warning
+            requires_ack: "true"
           annotations:
             summary: Postfix is rejecting messages due to errors (instance {{ $labels.instance }})
             description: Postfix has seen code {{ $labels.code }} errors recently

ansible/roles/postgres/vars/main/main.yml

@@ -75,6 +75,10 @@ postgres_users:
     password: "{{ vault_postgres_user_passwords.rrdtool }}"
     roles:
 
+  - name: alerta
+    password: "{{ vault_postgres_user_passwords.alerta }}"
+    roles:
+
 postgres_hba_rules:
   #
   # Service HBA rules
@@ -151,6 +155,12 @@ postgres_hba_rules:
     address: all
     method: scram-sha-256
 
+  - conn_type: hostssl
+    database: alerta
+    user: alerta
+    address: all
+    method: scram-sha-256
+
   #
   # Backup service HBA rules
   #
@@ -247,6 +257,8 @@ postgres_databases:
     owner: polonium
   - name: thallium
     owner: thallium
+  - name: alerta
+    owner: alerta
 
 postgres_grants:
   - objs: api_nomination,api_infraction,api_user,api_nominationentry,pg_stat_statements,pg_stat_statements_info

dns/zones/pydis.wtf.zone/root.yaml

@@ -23,6 +23,14 @@ agents:
   type: CNAME
   value: lovelace.box.pydis.wtf.
 
+alerta:
+  octodns:
+    cloudflare:
+      proxied: true
+  ttl: 300
+  type: CNAME
+  value: nginx-gf.box.pydis.wtf.
+
 alertmanager:
   octodns:
     cloudflare:

kubernetes/namespaces/loki/loki/alert_rules.yml

@@ -39,7 +39,9 @@ data:
               )
             for: 5m
             labels:
-              severity: page
+              severity: minor
+              service: nginx-gateway
+              instance: '{{ $labels.host }}'
             annotations:
               summary: "High P99 latency on {{ $labels.host }}"
               description: >
@@ -71,7 +73,9 @@ data:
               )
             for: 2m
             labels:
-              severity: page
+              severity: informational
+              service: nginx-gateway
+              instance: '{{ $labels.host }}'
             annotations:
               summary: "Slow backend TTFB on {{ $labels.host }}"
               description: >
@@ -102,7 +106,9 @@ data:
               )
             for: 2m
             labels:
-              severity: page
+              severity: informational
+              service: nginx-gateway
+              instance: '{{ $labels.host }}'
             annotations:
               summary: "Slow upstream TCP connect time on {{ $labels.host }}"
               description: >
@@ -133,7 +139,10 @@ data:
               )
             for: 2m
             labels:
-              severity: low
+              severity: major
+              service: nginx-gateway
+              requires_ack: "true"
+              instance: '{{ $labels.host }}'
             annotations:
               summary: "Upstream queue saturated on {{ $labels.host }}"
               description: >
@@ -176,7 +185,10 @@ data:
               )
             for: 3m
             labels:
-              severity: page
+              severity: major
+              service: nginx-gateway
+              requires_ack: "true"
+              instance: '{{ $labels.host }}/{{ $labels.upstream_addr }}'
             annotations:
               summary: "Slow upstream pod {{ $labels.upstream_addr }} on {{ $labels.host }}"
               description: >
@@ -216,7 +228,10 @@ data:
               ) * 100 > 5
             for: 2m
             labels:
-              severity: page
+              severity: major
+              service: nginx-gateway
+              requires_ack: "true"
+              instance: '{{ $labels.host }}'
             annotations:
               summary: "High 5xx error rate on {{ $labels.host }}"
               description: "{{ $value | humanize }}% of requests are returning 5xx on {{ $labels.host }}."
@@ -260,7 +275,9 @@ data:
               )
             for: 5m
             labels:
-              severity: low
+              severity: minor
+              service: nginx-gateway
+              instance: '{{ $labels.host }}'
             annotations:
               summary: "High {{ $labels.status }} rate on {{ $labels.host }}"
               description: >
@@ -283,7 +300,10 @@ data:
               ) == 0
             for: 3m
             labels:
-              severity: page
+              severity: major
+              service: nginx-gateway
+              requires_ack: "true"
+              instance: '{{ $labels.host }}'
             annotations:
               summary: "No successful responses from {{ $labels.host }}"
               description: >
@@ -318,7 +338,10 @@ data:
               )
             for: 2m
             labels:
-              severity: page
+              severity: major
+              service: nginx-gateway
+              requires_ack: "true"
+              instance: '{{ $labels.host }}/{{ $labels.upstream_addr }}'
             annotations:
               summary: "Upstream errors masked by NGINX on {{ $labels.host }}"
               description: >
@@ -368,7 +391,9 @@ data:
               )
             for: 5m
             labels:
-              severity: page
+              severity: major
+              service: nginx-gateway
+              instance: '{{ $labels.host }}'
             annotations:
               summary: "High dropped connection rate on {{ $labels.host }}"
               description: >
@@ -402,7 +427,9 @@ data:
               )
             for: 2m
             labels:
-              severity: low
+              severity: informational
+              service: nginx-gateway
+              instance: '{{ $labels.host }}'
             annotations:
               summary: "Rate limiter rejecting requests on {{ $labels.host }}"
               description: >
@@ -432,7 +459,10 @@ data:
               )
             for: 2m
             labels:
-              severity: page
+              severity: major
+              service: nginx-gateway
+              requires_ack: "true"
+              instance: '{{ $labels.host }}'
             annotations:
               summary: "Unusually large average request size on {{ $labels.host }}"
               description: >
@@ -458,7 +488,10 @@ data:
               ) > 0
             for: 5m
             labels:
-              severity: page
+              severity: informational
+              service: nginx-gateway
+              requires_ack: "true"
+              instance: '{{ $labels.host }}'
             annotations:
               summary: "Legacy TLS {{ $labels.ssl_protocol }} in use on {{ $labels.host }}"
               description: >

kubernetes/namespaces/monitoring/alerta/README.md

@@ -0,0 +1,5 @@
+# Alerta
+
+This directory contains the manifests for deploying [Alerta](https://alerta.io/), our alert management service, in the `monitoring` namespace.
+
+Configuration is provided by the `alerta-config` and `alertad-conf-file` ConfigMaps and the `alerta-secrets` Secret.

kubernetes/namespaces/monitoring/alerta/config-file.yaml

@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: alertad-conf-file
+  namespace: monitoring
+data:
+  alertad.conf: |
+    ADMIN_ROLES = ["devops-sudo"]
+
+    # Alert expiry config
+    ALERT_TIMEOUT = 259200 # 3 days
+    DELETE_EXPIRED_AFTER = 1209600 # 14 days
+    DELETE_INFO_AFTER = 604800 # 7 days

kubernetes/namespaces/monitoring/alerta/configmap.yaml

@@ -0,0 +1,31 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: alerta-config
+  namespace: monitoring
+data:
+  # Auth configuration
+  AUTH_REQUIRED: "true"
+  AUTH_PROVIDER: "openid"
+
+  OIDC_ISSUER_URL: "https://id.pydis.wtf/realms/pydis"
+  OIDC_AUTH_URL: "https://id.pydis.wtf/realms/pydis/protocol/openid-connect/auth"
+  OIDC_LOGOUT_URL: "https://id.pydis.wtf/realms/pydis/protocol/openid-connect/logout"
+  OIDC_VERIFY_TOKEN: "true"
+
+  ALERTA_CONF_FILE: "/app/alerta.conf"
+
+  OIDC_ROLE_CLAIM: "roles"
+  ALLOWED_OIDC_ROLES: "devops"
+  ADMIN_ROLES: "devops-sudo"
+
+  GUEST_DEFAULT_SCOPES: ""
+
+  OAUTH2_CLIENT_ID: "alerta-ui"
+
+  # Service configuration
+  PLUGINS: "reject,heartbeat,blackout,acked_by,timeout,remote_ip"
+
+  # Docker image housekeeping config
+  HK_EXPIRED_DELETE_HRS: "336"
+  HK_INFO_DELETE_HRS: "168"

kubernetes/namespaces/monitoring/alerta/deployment.yaml

@@ -0,0 +1,52 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: alerta
+  namespace: monitoring
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app: alerta
+  template:
+    metadata:
+      labels:
+        app: alerta
+    spec:
+      containers:
+        - name: alerta
+          image: alerta/alerta-web:latest
+          ports:
+            - containerPort: 8080
+          envFrom:
+            - secretRef:
+                name: alerta-secrets
+            - configMapRef:
+                name: alerta-config
+          readinessProbe:
+            httpGet:
+              path: /api/management/gtg
+              port: 8080
+            initialDelaySeconds: 10
+            periodSeconds: 10
+          livenessProbe:
+            httpGet:
+              path: /_
+              port: 8080
+            initialDelaySeconds: 15
+            periodSeconds: 20
+          resources:
+            requests:
+              cpu: 100m
+              memory: 256Mi
+            limits:
+              cpu: 500m
+              memory: 512Mi
+          volumeMounts:
+            - name: alerta-config-volume
+              mountPath: /app/alertad.conf
+              subPath: alertad.conf
+      volumes:
+        - name: alerta-config-volume
+          configMap:
+            name: alertad-conf-file

kubernetes/namespaces/monitoring/alerta/service.yaml

@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: alerta
+  namespace: monitoring
+spec:
+  selector:
+    app: alerta
+  ports:
+    - port: 8080
+      targetPort: 8080