diff --git a/ansible/host_vars/lovelace/prometheus.yml b/ansible/host_vars/lovelace/prometheus.yml
index 674a1275..44f4beba 100644
--- a/ansible/host_vars/lovelace/prometheus.yml
+++ b/ansible/host_vars/lovelace/prometheus.yml
@@ -116,6 +116,8 @@ prometheus_rules: |
           for: 15m
           labels:
             severity: warning
+            requires_ack: "true"
+            instance: '{{ $labels.instance }}/{{ $labels.name }}'
           annotations:
             summary: Node systemd unit {{ $labels.name }} has failed (instance {{ $labels.instance }})
 
@@ -125,7 +127,8 @@ prometheus_rules: |
         expr: pg_up == 0
         for: 0m
         labels:
-          severity: page
+          severity: critical
+          service: postgresql
         annotations:
           summary: "PostgreSQL is offline"
           description: "Postgres Exporter cannot connect to PostgreSQL."
@@ -134,7 +137,9 @@ prometheus_rules: |
         expr: (sum(pg_stat_activity_count) by (instance)) / on (instance) pg_settings_max_connections * 100 > 80
         for: 1m
         labels:
-          severity: page
+          severity: major
+          service: postgresql
+          requires_ack: "true"
         annotations:
           summary: PostgreSQL connections near max_connections setting
           description: "PostgreSQL instance is near the maximum connection limit, currently {{ $value }} connections"
@@ -143,7 +148,9 @@ prometheus_rules: |
         expr: increase(pg_stat_database_deadlocks{datname!~"template.*|postgres"}[1m]) > 3
         for: 1m
         labels:
-          severity: page
+          severity: major
+          service: postgresql
+          requires_ack: "true"
         annotations:
           summary: Too many deadlocked tables
           description: "PostgreSQL has dead-locks, value: {{ $value }}"
@@ -154,7 +161,7 @@ prometheus_rules: |
           expr: postfix_up != 1
           for: 5m
           labels:
-            severity: warning
+            severity: critical
           annotations:
             summary: Postfix is down (instance {{ $labels.instance }})
         - alert: postfix/smtp-temporary-errors
@@ -162,6 +169,7 @@ prometheus_rules: |
           for: 0m
           labels:
             severity: warning
+            requires_ack: "true"
           annotations:
             summary: Postfix is rejecting messages due to errors (instance {{ $labels.instance }})
             description: Postfix has seen code {{ $labels.code }} errors recently
diff --git a/ansible/roles/postgres/vars/main/db_passwords.yml b/ansible/roles/postgres/vars/main/db_passwords.yml
index 324093a4..869da816 100644
--- a/ansible/roles/postgres/vars/main/db_passwords.yml
+++ b/ansible/roles/postgres/vars/main/db_passwords.yml
@@ -1,198 +1,201 @@
 $ANSIBLE_VAULT;1.1;AES256
-37373732323463623630666336653638383333616132663436363937643562383831623831646665
-6232363435306663356537383166346662366230326261350a363034623033633864376163376136
-34383239633465336134333832346330336336626636326331303430616266663836366336356166
-3361373239366637660a393736393533646439666666313666323132386664366330323637323365
-30343464396462636363623161643132363636643861323263363232633938303437636135393636
-38323532653465303466666163313164653432643337396334306562633231343237353537613139
-65633463363362343661366365316237346130656165633236653334613338616633386461323737
-38383337316130633038323065633539303834343336663936333264383663323335623666666230
-38613535643231343662343833653461633834396234613738636534653037663261616538623436
-61623833653937396634343839313537336135663361653036626333333761623432323666653438
-34363763646133313962376438333930333833626562313362343331623839383666313738353962
-36616465393537393837643533386331653433383430656631653934346563666561616464643337
-61373362623133313734383736633366393266303734343263323666396438313736386662633038
-32303363663437643762306466643538616365396432363961616435636637663265336332363961
-61663466383935376630356635356162636563623533343335656465643632393435343430306561
-30313162373337353037383566313963363464633839393536393530623261366464313537306530
-31316632303738663338366432323731333262373032656336363333663431646136373361386566
-62363761333063346634306435356338633739613036626162346330313237346134326430623239
-38643039306161623631323864653766393463316366613434346134396132376130343130313238
-66643739316262616130656630376235653262383266396231373036643361313462643535663734
-64323730346366633462613536646164366165643930396238616536663439653431646537363039
-30373263313439396163666631363731306461666431623339326136383466366333366438656364
-38346464626634653536356161653931626562303861383965396464326335663737393932333064
-65656535663564623332373635373834333733396234333162336666306362653864363962333261
-32613937633637623332363837346663373639633064383738653833663365353062306564323666
-63643430393266303765646439303161343937643131383438333934383732623166336432616634
-31653133393263653234383664623530326263306332343730333964663062323737643739323735
-37366231393437633731363164346439373136633633363064653838633030663363646336666532
-38663134383661626337343231646162396234396135363539623162613163343064343664643363
-65396437306334316234356363303937663232323538373330623432346539363739633534666339
-38656363356635636432383663656437663665343731326361353838336362386439396233613235
-34363938363564643066363439623163383339656636643465353036396338313334336430623735
-61396662396630326431333638343565333937343834393161623439373337666535626239323262
-66623634653966336232303536333532633034323166643439616539373265303366383962616232
-39386662326461646434373636356538306237386438306432373531363565376132646636356231
-30313930326663303235343735653238353037396366656634626637373264343563613130303032
-38323533623338363661393864643563663333396464393739356335646265323166363432393864
-61623933303261356565646437656166376238643531313935653938323535396636346433376162
-32373765303133666534313763306664363061366538633434303363613531363366396533363534
-34353133333238326630323839373035633366343739363334336665646530636436346237396431
-64356462623737383636376437306161356132653133383566636663626230646563323264383035
-39663832326333323962653130383738316364616336313338613635626531393864383335396130
-39353236373766653866383432373538363062323863643265393162393730366163303430373261
-36373438316366363235363431666637386264323962613731363365666536616562303238386637
-30633861366633613363373437326635656234343832623961623339326436323639363030346162
-64643165636135323830363931353530656534613335333632623061366463653661376334613730
-38376361363833393064643764316465633532623561303865366165333633663066623833633133
-36653431666565623238323666356136626637396136356662613330386230346632363266316135
-34643465376361613033323062376364373638383037313133303761383935613064613561313038
-61633066313332393538616165386264636435636334623763363266633331376332373235666238
-33633635386361323933653636666430363433363734613538373031323166313835353465323836
-36626433633632633961376332623830333862663731653663326464333534383934336535333537
-66623834616230633161663262376136636665306338646435656535366663656335643562616233
-37356330383565343365383438656338626565663063663438343532306135373838363564313162
-61653562613636376661356365653137633661623162313930346565366566336536363036656462
-64316366633233343566323763326636623635356263393166666532653732636261373739643566
-66636532363364393336323235333362313633346337653235383131656166616661623063636433
-34623833363139326233326566303436343738653434656439313536383937646632616138656266
-64656236663462656562333162353865663462353630333561383462633533613863393635613039
-38663633663235343931666163666332356561333833616433316436646435393432663266643335
-62373462653837396266313533393135306433313231376134636237343363343133386339316165
-64636361386261373334643239376137393165396333366362343230336335343835313764303635
-38336639393436666137323938366339373435666633386361373433373063643339643262613062
-37613438633930333866316334366537313433356636323437326163613430633231366430636138
-30636264343938363036373735383065376164333139313338316434656630616664656133616333
-35346339373265613164343863393131363065326435636664373062396435396261386664656336
-66363562353532333034636466666436376265353635333030633235393566386530623335386431
-65373566333139313561376464643635663135373437616164653931633035663932383932353431
-32336438343933346138313936633063333439343865393930336633373730646331656630623636
-39626433333264653339326237653935643763393262626264626362336464303330666561353930
-30306131396362323937646663633131303262326231656234663936373735373062313437376466
-35633333333039303830643839336237356132663738313362326462633763663066636336313032
-39313532643664333137323739643036633036333539663736663233303733663234376366633033
-63343738326564333561636165653839333131333137326332646664323639383232623761653163
-65646131346563373130393637613439663861393333303337383566636137316333363562623430
-63363636663263656464636265393339616164303736323030646531643665653562623730363736
-62656366343039313733396265633765366536383266643062643364666137333936373965303134
-36633939316433376333306338393464623836383033623962303930303465393435646562613436
-35656532613462326562306566343335333466303261353331393135313030633266346131383666
-36613166646266363634336332663537663863346666646161393030616639666133326638366636
-39393763613666343864393434373437383138646562366332353733646538613834316130363635
-36623663653766653433636230303436393433396262386236353139653938366334643138663034
-34396230393334306261663564343463623764306635613736396162346335323038383437303964
-66363464393238633830323730336539653062663962356536383832623130343837353231306439
-63646533613364613837353133623334396136376637373832333038346664613365313336326261
-62313937613938366563656632663265636538613438346531356538383464313739616231336564
-61343237363331623238636463313366376533653536356338623662633833626231356236613265
-39366536646336343336323533396264323062353963303732323832613533396332386434646134
-64356664666365633730313336626163653730333262316333613930303863626665616133346462
-35623964326131353161656636306630353939373462383663396430313335643832366533313630
-30313934333136356238333635313331346464353433393332663832336136653532653035363637
-36616166343231373166323466386131306233373232383561396466393863653762626261353164
-38363132666364396437363961303664623766383232313431663737323365653266623832313934
-38666132653962663665306237356262336237326137303033316239363134306438326230653137
-63653561323764383636643763663865663036366530636663373136383030303765333361323839
-36666164376136333761333264303437653534336430353034323464323833656135363464616232
-66336632623966383134303261623661386362363865373364313135616466316631333231623532
-62646130336435303534346437303936303436303764346539396662653266333035303936303338
-65333639393130303837303538633964656365653261653864633336323332323733643338353935
-31636331353436363134623038383536643363333330666662353638326533353532663835316364
-62346265633862646163613230346335316233353236303836636531383661343830366336633230
-35633537626130393433323761633232366364636138313537656430613338366561666462343233
-61333138303230616165383038393164373164623964326561356231633065616236626631633830
-30333233353761306431636434303034636436623465313562336231383434363965653635373536
-39393061306238333139363762363734323466376565646532636330313130376462353361623139
-65333838353062656436326362346261656331333365386336613965626638643335323638356662
-33343466363164663733313032376535633863353661323366636239313763663739626537336433
-36393162326161353431313462626534623164626362376661343636343261316637626462646634
-66353561396432643135376230666139653262396638663538373766303334316131346339316262
-31383237303339303737646533663132393439336633363132356132646231653835643735663034
-64343034396530396264613230653237336366666338383866366233306336373465653935306135
-32306330346430336263313564663330656335393761386335633836643563396265356339633734
-31396466383737623431623566323762363666316132396263623435663833623930646666366161
-39353538383263323163303964306634633265636465316461386237313138363935643263613666
-65343266303364303236306661343037323132323036653536306664363664306338383835383739
-37396334616161326161353533326130366132386463646132323631333733393861386633386535
-33326265356239656132353263633838346135353238303562633432346630306533313636623266
-39663932633631386133646663643338323666363963646565393536343366343130393232333463
-38643866663662666265356631303665303936663632356230633135643764326265333335343639
-35373836366235383233313731316238613136393764366336623466383032666163366261373363
-38366263316635656561383831303263333030383033616564663430383033643434386637626530
-38343236336266323536663133336164613238303833363962353132663365653034316564613966
-39373662343262633164353530636433363361323465336361393862366463393838353437346632
-30323633643730653761343661336239346636396163636136363135363232626236393935396163
-64363364626362323639613666386638363534376633386236663462303332646632636633353637
-61393263336537623062643861396537373237663264356130386133306132396265373634643737
-30616235333265386430663766636332633830366365636263346465656137376234383535343130
-34376130643433326164653330383462373237623238333664623964623335633164313239363936
-32373132613134643436333461333330303239313935663163373062393065376362313535646330
-61306362653562643263636463306237313837663837663636616230373737396561633532363334
-62363638643564343433363663303662383632326433366462616538396633633262376461363936
-35356438313530336563356234363265333864326536663738656462633237666631333766666435
-64383132653536313166666436653936613230336633326536343730333564636630626430333639
-63623234356662396466653332353930386666306235376263623638646438353865656631393462
-34346462356336303132326461613461653636323866623963396363313836663031656531663462
-64623030313263363634626563386662386636356566336637653564366163363839646534373433
-30666561663034663431306434626637336231383564643935636563363833346263383338663336
-62653663396635346463386231303165636436653466353437336265323966336266636231363338
-37633963346132303837343662393535643162383961643733356633613130313139626162303833
-62396438303561643165336661373835663665353738653739656462393163376132306339373930
-61393435316631653732653631363536353865643433303234346461326566376466666531653439
-34623966666465613030333437636164356136386239316662373064653635326134646535393935
-30383532653665643933343964353562623230383639303739313163343664343762633963653234
-30363839346465356666656634646365663438633034346333356336653432373032346233663765
-38303335353464343665306266303932353633643631346639353261653339383332623564343965
-39326531353939656231643063613534396436663536306333336232323538303436393939363334
-32346439323863306237363433613635613333663536626264313336663662663037616666366361
-30303734316264333031633034393165353561346135376432303765333261356235313561383336
-36643533313634306635323335326231306464666364393037373562353762653531636338626139
-35393937393466393961373836336365656430323365613534623139643231366465313032343564
-66303732303839643732333737643733333836353736363539313034336162353034633961346137
-65386231616331346466653230353037373265366565396162373261373034333934393739316162
-33363834376632356532663034343861653036633833366663613235346631386436633565316137
-32323466393037393065376330366231363563653765633462623364376132643935366638633266
-35376439373465643462373937396161616631333039306265633433353435373536623165393334
-39646636636333656565623030313338633031306139343633653130313364313037353835633037
-34666535323436353262336461336635636532616432663032323536323166363564623564633264
-38353731616264393030393462633939646263323638356463663734633337306234373639313062
-61316435383538383961376533396665363031373066663131613331316564636463646665346231
-34366263653738666361646665313735346431333262643765616137633039646438303832616362
-30363863656430633462393737386264326230636435393539393931666562323335333962343866
-34336164643134623465393164616439613530306566333636633164343166343336333234366536
-34663335303636636462386638346665306230363532353933316162393035613836623135373563
-64653133666132303534373630626661363435396633646634653865313864613361366165383465
-63333039646437303730623261303530633932626539363533623864326332306636396163303131
-35306434326165373638333137336663303238353234623137336165326133666339346563383033
-62633934636666316133643866393764383030343334373363393535363731303130623932663466
-64326236383130356632656635663134346164306330656432653461363730613866393164323035
-39346361643465653062336633663237613933613735623338636330616132363534363165376136
-63616533646666353565343438303435346433356463306535356561373261356333663838366530
-36396633633361303535663631623365363036663935646337656135396633366664663131623837
-32663437646665623761343562643534636632626133633638343231623130306534613233653136
-64383737653663646166623634383861323430623331346562396235356237303061303036633233
-37343264633537623166396231643737643764626366646139653166663431383636633765626537
-62306365623433633732313834323435366238653463663835663237666334353361356162643865
-32366162313062343261333362343732343137633764643331616662663264353131306135633930
-65383136376562643430306237303566376630656434653435313230633034316262613635363030
-39613433356437353937633637333230323839373265366664316131616130383330616232336265
-33636439663938363732383033376631633333306633656533306632303332646531643362306138
-38663433376564343338373838636333653461396436613639633535646638393032383765663031
-62393561303139336231323033333834623861313063313433643931333839333161373232623339
-35383766633362346335356335306564343764316130303235636661366630356534626339356233
-61373230376662386565303537663234373663346165326131663730306131303462326637613764
-37393063656161653238623762393135313732363136323364343639663266626638353564386564
-63643265336631636565613634643664343566373864313730393937613861353234313265386661
-62663565363031343938366339353238343933393630343334623734333830393163623330356535
-66333863376361393036343237323734386139373565323830653062383633306630363638323132
-33393233333462363031336531613031386434656232326536323233346335666366303030623633
-66323632333664643334343931363635636631643134623661663361386535396264636461353033
-32333866363536326162653130633132363065376337326439383630383565323530626139313165
-62623962613637313263623335346131623631656363323566613236373365313636326630303635
-64386261643561323265333564386163376331383562373337353032393436336561366265383037
-35386561646139326462313864323833323664363736306665616535333636663737656530336231
-35646136636130613466343738383332613763363463666631353637353863313330376137613233
-62633439376239316231353334343435616561306137333534363765396265363530303531303366
-36396562336261626562363963373232313439653936376238386232653638393239343237323865
-3664
+35663431363733646362366361363761343233373739663239313831313934663634326365313961
+6136336462653830343965626536653432346338356237300a326666303135663433646263623430
+36306436393837333130353932366334303333306638323266353135663364326361386338626331
+6565383064393964630a643365613634373063383139646237393139616663623036636266393832
+31343834653035313234383366316633663334633239313539376233356638326432656634343830
+33396562326635623731363038326135303164353636303038303463373834653832326432346263
+61386530613531393731633538353537386461373564613136656435623861303164393034376462
+32353030643161356434323062356662313266333838633731653663306531313563656330643935
+34303733323663376361393865366238303662666366626163663832613636303735393661353965
+61313438383330376537353139346361653631636431386434613164613631656639396536353264
+37636664303762396230393737353465353533393664303263353635636138383064376639376461
+39323763306536383738636161643736633532653362353635666265343736633263353866343330
+65313863313238333632326239333239633533656234363239613936636336633034333663396438
+35383465343664343034396432373166393164643434326235643430643438346235376334613162
+37343631613661666632646532663431613735616633633261393862363662633761363733333365
+63663163623830636262663134303839376131333765323638663462633666626139663962653065
+61656534656132396461653864366632313662306432336263366537333765313362303663623661
+33386266396134303333663037303235326666636230346436356139393263643831616331663966
+32356365626433376630393532653532336262643230346666343838353432313239323231643963
+33366138336263643962343736623438663762313365636535346332373537623262313933363631
+31323964303263623735313333313437666334386364346466653563326334636464663532343333
+66336563316461373666386465343465363735663532653561303036343332663362633036646139
+33323732386265393433323636353930336661626666303638636531393637643866386462376139
+39333063393765346231343938343433616537343031343931613130623135306664346338373964
+62616231653035303561323964646363373135663835346236336632653137623738353230613838
+36386662376535663866366332326366396138383138373665303766646563363831653733363137
+39636666303638333430666636393935306630656334313263636439636231646361376665353936
+64316237393532363738346334623436316233613466646662626461353535636565316239336562
+63656534373661653739366464356432623565323038343730306562373834633532633238373231
+64666633323461623633623861663164373761313832626238333432656530393139336362333432
+64376337636431393837626161656362316135656134353230616263336438326335666437383137
+63343864383138326537633063313236663262663237663432623065356539303264333063633233
+32633435656564646630623262613138666330386339346131666565316439306465313466336130
+34616666353732663032663263306638646163373665663064303165336266623165373532326533
+39663730343163303661303066396235306235313237363838623364656661623932376162316465
+37323865386130613062336537386161343132633232336363633366356264356531313333383336
+32663036306464323634393633313661626366653239626362663934366138386534643332633439
+39343861333331663030623337663531636434303865643665653963323964393733343961666231
+39346636316163363439613561666561656666633335663639613433323364386263633838373232
+37373736363939383433356138616238636263306530393131633136366436616234663336343862
+33393133323463353261643665386135373434623933393834326265383235396138383465653639
+65613134356562313939306330613031356261393163383466383134383134653039636236363165
+64343266303534333635393164346639343736383137643932653137666263343766356464346566
+66643139383831643131613031646537383166326436333030613364393363643538616439376662
+64666338653266386634613030613362373363363165653866643333353565666231376563316138
+30386661613134356466653332336439373931646361643137653335663038376463393530373035
+66326434373832383861323137356564633638616537393433623266623032353935626465323534
+30666133356164643333373666376562663762623437633462376235646333663837316632343263
+62636430343633373131396336353830633062313764613763326135396433636439666565383031
+61366664326663363061343731306564616330626563343666303761373631613332373837373261
+39373235363232323366356535386264376630303863653734376531336561353337353838623064
+39633163643234633366313035376337626364383236653036636465653765613230336464333538
+66303736383635376539343330373834396161323539663665663764653064356563636333666265
+34376531306363386634386463633736623739353662393164313538323730396531626566383461
+33313564343531326130656164613432636266643934653137333531336633333665646466363565
+38386237623033333132323964633937306136356661616332633462663466323833623162613237
+37393237363066633765326233386636623335336662376533356430616633313134353338303039
+38643937306334613966303866383733386264356233313866306634363838363039393337353963
+63636630363239363838643936303034393264623361653765633565633131333536323431303138
+30373137633630656366376562663161636431353864653339323161613539613734393735656639
+39613030316535363962333930303938633330383234353830306361333631383864383834373038
+65633538653961393539393133613534326138383738623166333835623865323130623034663131
+31316439393264383839386661396333636636643362383430613936376330383864316233356331
+30336334363265343435636531353535626131373836363935366234386634383662623837383530
+33616237386634643637393633336365373434303163373038313534653464633765343138383936
+30363432636438396166356163363466323663346264326465363834303561353961363431626631
+38393465343839396462353235616463323136323935326266336335333630633339316135343535
+32353237633461656631343331636361636364356461386339653163353631663762636438613031
+64353361303363376137313135653437643861326266363639346165316233363264366630633362
+35646265386135653139383638623335313135326539626266666235663235636564396239396237
+65656433343433306434306330353061653036336362383336636363303934626634666438663063
+36643139386166653134353637353530303038633165313630663730376333383837383361643465
+35613037623739356337336234363832633161303032383939643139623265346539383630373233
+63633634623438353564376463373330653037313965306636643834353935303334346664393066
+61386662353932343237383030323538643333303034623066346563333035626631323637333965
+37653436663536303932663930363630656435623063323261313330613661323736613937326430
+31336463306262643239363234323733323238386130623161376632386534383461353765303962
+37653535393737366535353664313366653064623639623131346633373731323463623039656466
+30396134336366663363343235323562643834363932353539353262623761616164383134313161
+39313031303337333537343165343462343936343533663636613534373766626335613764353032
+38393437396331383539303334336234313538376635356539383435666630616539316135336531
+66383837623635613538366331313134653264303761366262343836666337633261613135663630
+65656366653939316232653532396265333134313438646535636661656161626130323462356262
+34313838356366303236393763636236326266616565343237613765626264353234376638303362
+64383062323030376534316164626631633561623863396363393362393766656632393362366464
+65653739646234303334646663666565316232626462346363303039393734376163316132313161
+64363461633063613934663136663739653365376364643632663331383539653866336335336637
+66336432386637326639353131363665636537643030383138323333383666626538306339313532
+65393466363535363136623430633838616131353863373338326339323538316461613931383664
+39343337633065613464313834316362353636656136393838363537353065313334366134316435
+39333764353530313530396465633638643536373331343462303234393536373232633964633033
+63313462616338396166623632623064613431303165383231393363326539626231333866633437
+33386333616337623462303935346263336532366637333833633934303735363937323766323836
+36613061376237353864353433633334316233623536363334336539643433663064336130353630
+31373239303837643630303937386630363537616561343433643334653436643737313634343166
+36353230653430353437326534323834306530346232356365653661323533623433393334336434
+30663963663539613032633139356466306361663134323836626663313438666339646535616161
+65303366353966353231653463656236306136313666633230653237396432326361653730666235
+65336263646566663566353239306366656630663564343565633864363836353163333566643636
+39646337646630643436613662326265323264653731323236663932323132663333633030653032
+64663036323438636633356231663339613564646433383766663461396663343937303231623732
+66356361343238336164616464303033653932383265663839616434313163393164613161373838
+38313139376664643934633465303263643236313733353538363337356463313734656337333138
+31343766633733366239646137383437646361383534366634633864613861346532633465613330
+32626266313061636333343037663031373830653938613937313864323466656539366633323366
+38396561643832343231623763633236393733653739353235313738396165303335616162373866
+62383739306530663334376430366432366230393261316430316535363038326131643639373335
+34326432313164373033386563353864393766353135326465353664323462356431343636643962
+63643062306562306263323462353663396330656465656464666466646234323963376538356132
+39386261616264653838646334366532376338653735343631376439656534656534346461633265
+64356230303233303434373133623535633639373432656361633331306639356163626263393437
+64326633623965326664336561666163366166666631623734663130623337663331653561306464
+64653835366564363561346139303038653730353333356335646438336235663462373566373363
+30366161656465616232616639333363306166363635313134383564303364363031316539333665
+37336435616561356162343232663065633064663064353632633265626562663936323366343831
+63353039613833623234313437393039363835623332613734303433313261386231346333393861
+31353836343033356538316135393961633133343166323739396330316565356262343664323133
+35376538356464633462653835343233343731656563303563333866626333386236316136356136
+64613462636633303032613335393636316235313266393863363635636535313736346363366439
+33353137336432373732396239303539663437633032393764303965353534343733323230393163
+39373733383336326663303133356435333733313934316463323133666438616661316263623962
+35636264353939623638306562323466626264656235646663346133613934316431373637323664
+33623632386438363062356631633664323733363636336361613937626534383963356430383330
+64396264393265353837656432616465323930626130303637346433663436356436313161373565
+38653766666135616261643963336566623065346631393366373561386632386537646630616130
+33386133636564313739326334323933383861383036656266356361336534646166663437313864
+36346435633832306462376339323639326464336337323633386539386338353364353036616166
+63396162363062663534313535313933653034323931326333363930666363363831346537393762
+66343864633432383934396561626634353530393831376631613138616434656465363865636331
+35646661366438373837613239623738353765396532323266306436653235373466353232353132
+35666462613535343538613661393134313730356635626237393162366666663661313136396532
+35646262653737633264313035303337373332656532333435636462613061386633656266393934
+39383761666561643266313835393935666339616261326337326462656663326132343962323561
+32653930616331643663613632626234363333613031636361393962363637633637613864613533
+31326565313865653132626363336565643133616635646435343537383434303364346437663064
+62366466666536663935353534366566343335353431346130366433646136653133336365653733
+64656361386464363464623836653765333366363734373235633961636564656232376161353133
+31356436316561383531623033346435366633306139656666636361643930666137393062623333
+66383735383530373764366136396232646165643031343338313737376532656263346131623665
+39336338656331663131623237353265383366346266626665323637333662336438383530613138
+30643865353862656265336437303365366661613338613035633466653734623962313665316363
+37653432346432656233346565363631386430616331633563393834326231396162643438303865
+33623661313539613464653565373235613664363164623238323263356534343465646134383039
+38666539306630383333363039373739306363336262346538666133313336623863316663613436
+35663531643265386236316433386466626266656466346632666564633462343939316336626662
+32656464373164353839643431656563353535613162303332383534393061663365323934333039
+39366635613338323836363737633935623930323335396534643165613433393434343735636162
+63623337383330663637393662663032663530623038643033646530313637336538373663356665
+32636233343739363434363437643861343831373965363837663337313135643838396136396162
+62303835663835323066383761313736326262653864623234303666393231383034313063373761
+62653366336539663735303232313037303766326231343164613866306336353765383066363131
+35396232316231303962623965376461666434373064396364623564383835366132386563323930
+36323734653239613530313164316336366531653239336230343632636536396338373534353938
+61323561666234643664326338363035373663336338366265663936363062326236653262366130
+38643063333539303830393161326436343334653962306431353533343363303030376462343261
+66376332613035663535623939393532396236303437356633653436663961633463626337306262
+65646538636637343063343737323431653530333231633462656238373261363266633836356636
+66656533316336383761303739613536373031613530386535633661373666623663646232393435
+39363233613334613363363636376136313262323936363364363330616664626336376661646434
+31616533383335383939323238633230333564396535653931363934373062313464663566366236
+38653132336430323831336234366630333733663766626633323866626539623964636231383163
+61393233343236646538636463386661356465306135373631616533376534363965663032303034
+64313337303939363337363130623139346530383464343365313464666438356366613630333933
+31636264333363393632616161616662623535373631383938346466653137333530333134613438
+66616237663537383031356261373037616364643832323565633432653766356536393834323832
+39323765323938343737383962393235316561333932366563633365653362383936633931313934
+62383362303630373637636336643839383936613736663163393834333139326631313735353334
+34313835356238616537333635373666373864643264396430636534633737663363663932616264
+37363762613735313136653930323338613333306135313038653632306562326332313333383364
+32386338303163363236653133633361636166313133373530343565326566626336313362373733
+33396663303666626535316661353933353363356235663063613566396139646164666662643439
+64383737363165353434363136656635663133376235326163353761636464643334663132383536
+32313564626536346431633161376365366663616365316433343938316463343232613333316233
+64643162323333623337636165336238343135353065663331383066376139613131316636623131
+36373531383732323835366338373538613137393965323131613836666533633230306133626230
+38313564333338666632633066356331666364333661336566666634636238383635653039633363
+62393731303965643232356662363835376438396132313930393762313032366561623537336532
+61303735653362316137326362663836336661323765353331313065656262316337363835373332
+32343066376131316664303531653066333962343830636266303263353835316461643461653337
+62616339653230663737363534303731653331393562643731396364613663376235373636613163
+33363864636538633330633833383162643364333334346136646232336261343863316633316531
+30353735626632386132336335613431633331623032303338313038353637666265316430393837
+39366332393131393063363565633764343734353639656134306333653735346337383239323738
+64363735633334346230643162626135653662646334643737626630383564343762343365383037
+37303364396332626633663932353362373363613136353763393436376465626532613530366563
+63373731373334633835663263396433633831306137666330363236363730383036353539633831
+62386635386365646238373430353238326639336536666361626233373139366338333664613938
+38343061386335333934393164613433313632323164373234626165643162326564303661393963
+38383334636632636261663364363165386464346665623666373134383661353561653133316366
+32623962306132363033663638343236363762353432313265626366346635643733373835303630
+38323463366336353230636236653134656330376535613639653630323962613863313330373138
+34333466343938336432653337386662303930633464626336623338613739313537656236306463
+33636531663535623937376166373832613439353634346132663866376234633535626233323531
+63626534626539336532623939613131633232666161363936666161616531383466316166353732
+61333761373839303436326638313132663861656262666239626163396635616536396365336661
+62363565303266646362386163323832313835616133303239333935613164353331363535616664
+36663833363030666536393430633332373635663062666231613830373135656139636139383835
+30356138363632646434323836633038663839346235643337666564333061663430343137393036
+63636161316362623930653864363465366330623536623734353135623732376337353338363733
+66613962336634383264
diff --git a/ansible/roles/postgres/vars/main/main.yml b/ansible/roles/postgres/vars/main/main.yml
index 20d1c11c..4ec564c4 100644
--- a/ansible/roles/postgres/vars/main/main.yml
+++ b/ansible/roles/postgres/vars/main/main.yml
@@ -75,6 +75,10 @@ postgres_users:
     password: "{{ vault_postgres_user_passwords.rrdtool }}"
     roles:
 
+  - name: alerta
+    password: "{{ vault_postgres_user_passwords.alerta }}"
+    roles:
+
 postgres_hba_rules:
   #
   # Service HBA rules
@@ -151,6 +155,12 @@ postgres_hba_rules:
     address: all
     method: scram-sha-256
 
+  - conn_type: hostssl
+    database: alerta
+    user: alerta
+    address: all
+    method: scram-sha-256
+
   #
   # Backup service HBA rules
   #
@@ -247,6 +257,8 @@ postgres_databases:
     owner: polonium
   - name: thallium
     owner: thallium
+  - name: alerta
+    owner: alerta
 
 postgres_grants:
   - objs: api_nomination,api_infraction,api_user,api_nominationentry,pg_stat_statements,pg_stat_statements_info
diff --git a/dns/zones/pydis.wtf.zone/root.yaml b/dns/zones/pydis.wtf.zone/root.yaml
index 8e1325c3..7077e8a7 100644
--- a/dns/zones/pydis.wtf.zone/root.yaml
+++ b/dns/zones/pydis.wtf.zone/root.yaml
@@ -23,6 +23,14 @@ agents:
   type: CNAME
   value: lovelace.box.pydis.wtf.
 
+alerta:
+  octodns:
+    cloudflare:
+      proxied: true
+  ttl: 300
+  type: CNAME
+  value: nginx-gf.box.pydis.wtf.
+
 alertmanager:
   octodns:
     cloudflare:
diff --git a/kubernetes/namespaces/loki/loki/alert_rules.yml b/kubernetes/namespaces/loki/loki/alert_rules.yml
index 3b172806..59940437 100644
--- a/kubernetes/namespaces/loki/loki/alert_rules.yml
+++ b/kubernetes/namespaces/loki/loki/alert_rules.yml
@@ -39,7 +39,9 @@ data:
               )
             for: 5m
             labels:
-              severity: page
+              severity: minor
+              service: nginx-gateway
+              instance: '{{ $labels.host }}'
             annotations:
               summary: "High P99 latency on {{ $labels.host }}"
               description: >
@@ -71,7 +73,9 @@ data:
               )
             for: 2m
             labels:
-              severity: page
+              severity: informational
+              service: nginx-gateway
+              instance: '{{ $labels.host }}'
             annotations:
               summary: "Slow backend TTFB on {{ $labels.host }}"
               description: >
@@ -102,7 +106,9 @@ data:
               )
             for: 2m
             labels:
-              severity: page
+              severity: informational
+              service: nginx-gateway
+              instance: '{{ $labels.host }}'
             annotations:
               summary: "Slow upstream TCP connect time on {{ $labels.host }}"
               description: >
@@ -133,7 +139,10 @@ data:
               )
             for: 2m
             labels:
-              severity: low
+              severity: major
+              service: nginx-gateway
+              requires_ack: "true"
+              instance: '{{ $labels.host }}'
             annotations:
               summary: "Upstream queue saturated on {{ $labels.host }}"
               description: >
@@ -176,7 +185,10 @@ data:
               )
             for: 3m
             labels:
-              severity: page
+              severity: major
+              service: nginx-gateway
+              requires_ack: "true"
+              instance: '{{ $labels.host }}/{{ $labels.upstream_addr }}'
             annotations:
               summary: "Slow upstream pod {{ $labels.upstream_addr }} on {{ $labels.host }}"
               description: >
@@ -216,7 +228,10 @@ data:
               ) * 100 > 5
             for: 2m
             labels:
-              severity: page
+              severity: major
+              service: nginx-gateway
+              requires_ack: "true"
+              instance: '{{ $labels.host }}'
             annotations:
               summary: "High 5xx error rate on {{ $labels.host }}"
               description: "{{ $value | humanize }}% of requests are returning 5xx on {{ $labels.host }}."
@@ -260,7 +275,9 @@ data:
               )
             for: 5m
             labels:
-              severity: low
+              severity: minor
+              service: nginx-gateway
+              instance: '{{ $labels.host }}'
             annotations:
               summary: "High {{ $labels.status }} rate on {{ $labels.host }}"
               description: >
@@ -283,7 +300,10 @@ data:
               ) == 0
             for: 3m
             labels:
-              severity: page
+              severity: major
+              service: nginx-gateway
+              requires_ack: "true"
+              instance: '{{ $labels.host }}'
             annotations:
               summary: "No successful responses from {{ $labels.host }}"
               description: >
@@ -318,7 +338,10 @@ data:
               )
             for: 2m
             labels:
-              severity: page
+              severity: major
+              service: nginx-gateway
+              requires_ack: "true"
+              instance: '{{ $labels.host }}/{{ $labels.upstream_addr }}'
             annotations:
               summary: "Upstream errors masked by NGINX on {{ $labels.host }}"
               description: >
@@ -368,7 +391,9 @@ data:
               )
             for: 5m
             labels:
-              severity: page
+              severity: major
+              service: nginx-gateway
+              instance: '{{ $labels.host }}'
             annotations:
               summary: "High dropped connection rate on {{ $labels.host }}"
               description: >
@@ -402,7 +427,9 @@ data:
               )
             for: 2m
             labels:
-              severity: low
+              severity: informational
+              service: nginx-gateway
+              instance: '{{ $labels.host }}'
             annotations:
               summary: "Rate limiter rejecting requests on {{ $labels.host }}"
               description: >
@@ -432,7 +459,10 @@ data:
               )
             for: 2m
             labels:
-              severity: page
+              severity: major
+              service: nginx-gateway
+              requires_ack: "true"
+              instance: '{{ $labels.host }}'
             annotations:
               summary: "Unusually large average request size on {{ $labels.host }}"
               description: >
@@ -458,7 +488,10 @@ data:
               ) > 0
             for: 5m
             labels:
-              severity: page
+              severity: informational
+              service: nginx-gateway
+              requires_ack: "true"
+              instance: '{{ $labels.host }}'
             annotations:
               summary: "Legacy TLS {{ $labels.ssl_protocol }} in use on {{ $labels.host }}"
               description: >
diff --git a/kubernetes/namespaces/monitoring/alerta/README.md b/kubernetes/namespaces/monitoring/alerta/README.md
new file mode 100644
index 00000000..b1766de5
--- /dev/null
+++ b/kubernetes/namespaces/monitoring/alerta/README.md
@@ -0,0 +1,5 @@
+# Alerta
+
+This directory contains the manifests for deploying [Alerta](https://alerta.io/), our alert management service, in the `monitoring` namespace.
+
+Configuration is provided by the `alerta-config` and `alertad-conf-file` ConfigMaps and the `alerta-secrets` Secret.
diff --git a/kubernetes/namespaces/monitoring/alerta/config-file.yaml b/kubernetes/namespaces/monitoring/alerta/config-file.yaml
new file mode 100644
index 00000000..7749f82e
--- /dev/null
+++ b/kubernetes/namespaces/monitoring/alerta/config-file.yaml
@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: alertad-conf-file
+  namespace: monitoring
+data:
+  alertad.conf: |
+    ADMIN_ROLES = ["devops-sudo"]
+
+    # Alert expiry config
+    ALERT_TIMEOUT = 259200 # 3 days
+    DELETE_EXPIRED_AFTER = 1209600 # 14 days
+    DELETE_INFO_AFTER = 604800 # 7 days
diff --git a/kubernetes/namespaces/monitoring/alerta/configmap.yaml b/kubernetes/namespaces/monitoring/alerta/configmap.yaml
new file mode 100644
index 00000000..eed0bfc6
--- /dev/null
+++ b/kubernetes/namespaces/monitoring/alerta/configmap.yaml
@@ -0,0 +1,31 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: alerta-config
+  namespace: monitoring
+data:
+  # Auth configuration
+  AUTH_REQUIRED: "true"
+  AUTH_PROVIDER: "openid"
+
+  OIDC_ISSUER_URL: "https://id.pydis.wtf/realms/pydis"
+  OIDC_AUTH_URL: "https://id.pydis.wtf/realms/pydis/protocol/openid-connect/auth"
+  OIDC_LOGOUT_URL: "https://id.pydis.wtf/realms/pydis/protocol/openid-connect/logout"
+  OIDC_VERIFY_TOKEN: "true"
+
+  ALERTA_CONF_FILE: "/app/alerta.conf"
+
+  OIDC_ROLE_CLAIM: "roles"
+  ALLOWED_OIDC_ROLES: "devops"
+  ADMIN_ROLES: "devops-sudo"
+
+  GUEST_DEFAULT_SCOPES: ""
+
+  OAUTH2_CLIENT_ID: "alerta-ui"
+
+  # Service configuration
+  PLUGINS: "reject,heartbeat,blackout,acked_by,timeout,remote_ip"
+
+  # Docker image housekeeping config
+  HK_EXPIRED_DELETE_HRS: "336"
+  HK_INFO_DELETE_HRS: "168"
diff --git a/kubernetes/namespaces/monitoring/alerta/deployment.yaml b/kubernetes/namespaces/monitoring/alerta/deployment.yaml
new file mode 100644
index 00000000..23219a40
--- /dev/null
+++ b/kubernetes/namespaces/monitoring/alerta/deployment.yaml
@@ -0,0 +1,52 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: alerta
+  namespace: monitoring
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app: alerta
+  template:
+    metadata:
+      labels:
+        app: alerta
+    spec:
+      containers:
+        - name: alerta
+          image: alerta/alerta-web:latest
+          ports:
+            - containerPort: 8080
+          envFrom:
+            - secretRef:
+                name: alerta-secrets
+            - configMapRef:
+                name: alerta-config
+          readinessProbe:
+            httpGet:
+              path: /api/management/gtg
+              port: 8080
+            initialDelaySeconds: 10
+            periodSeconds: 10
+          livenessProbe:
+            httpGet:
+              path: /_
+              port: 8080
+            initialDelaySeconds: 15
+            periodSeconds: 20
+          resources:
+            requests:
+              cpu: 100m
+              memory: 256Mi
+            limits:
+              cpu: 500m
+              memory: 512Mi
+          volumeMounts:
+            - name: alerta-config-volume
+              mountPath: /app/alertad.conf
+              subPath: alertad.conf
+      volumes:
+        - name: alerta-config-volume
+          configMap:
+            name: alertad-conf-file
diff --git a/kubernetes/namespaces/monitoring/alerta/secrets.yaml b/kubernetes/namespaces/monitoring/alerta/secrets.yaml
new file mode 100644
index 00000000..84e3c751
Binary files /dev/null and b/kubernetes/namespaces/monitoring/alerta/secrets.yaml differ
diff --git a/kubernetes/namespaces/monitoring/alerta/service.yaml b/kubernetes/namespaces/monitoring/alerta/service.yaml
new file mode 100644
index 00000000..6d248e96
--- /dev/null
+++ b/kubernetes/namespaces/monitoring/alerta/service.yaml
@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: alerta
+  namespace: monitoring
+spec:
+  selector:
+    app: alerta
+  ports:
+    - port: 8080
+      targetPort: 8080
diff --git a/kubernetes/namespaces/monitoring/alerts/alertmanager.yaml b/kubernetes/namespaces/monitoring/alerts/alertmanager.yaml
index 82053afe..622ba317 100644
--- a/kubernetes/namespaces/monitoring/alerts/alertmanager.yaml
+++ b/kubernetes/namespaces/monitoring/alerts/alertmanager.yaml
@@ -7,19 +7,38 @@ route:
   group_interval: 1m
 
   routes:
+    - receiver: alerta-no-resolve
+      matchers:
+        - requires_ack="true"
+      continue: true
+
+    - receiver: alerta
+      matchers:
+        - requires_ack!="true"
+      continue: true
+
     - receiver: mod-team
       matchers:
         - additional_destination="mods"
 
     - receiver: low-priority
       matchers:
-        - severity="low"
+        - severity="informational"
 
     - receiver: devops-team
       continue: true
+
     - receiver: email
 
 receivers:
+  - name: alerta
+    webhook_configs:
+      - url_file: "/opt/pydis/alertmanager/webhooks/ALERTA_HOOK"
+        send_resolved: true
+  - name: alerta-no-resolve
+    webhook_configs:
+      - url_file: "/opt/pydis/alertmanager/webhooks/ALERTA_HOOK"
+        send_resolved: false
   - name: devops-team
     slack_configs:
       - api_url_file: "/opt/pydis/alertmanager/webhooks/DEVOPS_HOOK"
diff --git a/kubernetes/namespaces/monitoring/alerts/alertmanager/secrets.yaml b/kubernetes/namespaces/monitoring/alerts/alertmanager/secrets.yaml
index 0678f739..6492954e 100644
Binary files a/kubernetes/namespaces/monitoring/alerts/alertmanager/secrets.yaml and b/kubernetes/namespaces/monitoring/alerts/alertmanager/secrets.yaml differ
diff --git a/kubernetes/namespaces/monitoring/alerts/alerts.d/alertmanager.yaml b/kubernetes/namespaces/monitoring/alerts/alerts.d/alertmanager.yaml
index e1e98634..bc3dfe5a 100644
--- a/kubernetes/namespaces/monitoring/alerts/alerts.d/alertmanager.yaml
+++ b/kubernetes/namespaces/monitoring/alerts/alerts.d/alertmanager.yaml
@@ -6,7 +6,8 @@ groups:
     expr: alertmanager_cluster_failed_peers > 0
     for: 1m
     labels:
-      severity: page
+      severity: major
+      service: alertmanager
     annotations:
       summary: "An Alertmanager node is reporting failed peers"
       description: "AM {{ $labels.instance }} is reporting that {{ $value }} of it's peers is invalid."
@@ -15,7 +16,8 @@ groups:
     expr: alertmanager_cluster_health_score > 0
     for: 1m
     labels:
-      severity: page
+      severity: critical
+      service: alertmanager
     annotations:
       summary: "An AlertManagerNode is reporting an unhealthy cluster"
       description: "AM {{ $labels.instance }} is reporting that the cluster has a health score of {{ $value }} (where 0 is healthy.)"
diff --git a/kubernetes/namespaces/monitoring/alerts/alerts.d/certificates.yaml b/kubernetes/namespaces/monitoring/alerts/alerts.d/certificates.yaml
index ad0e9d51..d5acaf0f 100644
--- a/kubernetes/namespaces/monitoring/alerts/alerts.d/certificates.yaml
+++ b/kubernetes/namespaces/monitoring/alerts/alerts.d/certificates.yaml
@@ -8,7 +8,10 @@ groups:
     for: 0m
     labels:
       # This isn't critical, but if we have reached this point, renewal probably needs manual intervention.
-      severity: page
+      severity: critical
+      service: cert-manager
+      requires_ack: "true"
+      instance: '{{ $labels.name }}'
     annotations:
       summary: "Certificate is expiring in < 7 days"
       description: "The certificate named {{ $labels.name }} is due for expiry in {{ $value | humanize }} days."
diff --git a/kubernetes/namespaces/monitoring/alerts/alerts.d/coredns.yaml b/kubernetes/namespaces/monitoring/alerts/alerts.d/coredns.yaml
index 2cccdc34..6887852d 100644
--- a/kubernetes/namespaces/monitoring/alerts/alerts.d/coredns.yaml
+++ b/kubernetes/namespaces/monitoring/alerts/alerts.d/coredns.yaml
@@ -6,7 +6,8 @@ groups:
     expr: increase(coredns_panics_total[1m]) > 0
     for: 0m
     labels:
-      severity: page
+      severity: major
+      service: coredns
     annotations:
       summary: "CoreDNS is experiencing panic"
       description: "Number of CoreDNS panics encountered: {{ $value }}"
@@ -15,7 +16,8 @@ groups:
     expr: rate(coredns_cache_misses_total{}[10m]) / rate(coredns_cache_misses_total{}[10m] offset 10m) > 5.00
     for: 2h
     labels:
-      severity: low
+      severity: informational
+      service: coredns
     annotations:
       summary: "High CoreDNS cache misses in last 2 hours"
       description: "This can sometimes be an indication of networking troubles, currently {{ $value | humanizePercentage }} over last 2 hours."
diff --git a/kubernetes/namespaces/monitoring/alerts/alerts.d/cpu.yaml b/kubernetes/namespaces/monitoring/alerts/alerts.d/cpu.yaml
index 61862b7d..a3aa676f 100644
--- a/kubernetes/namespaces/monitoring/alerts/alerts.d/cpu.yaml
+++ b/kubernetes/namespaces/monitoring/alerts/alerts.d/cpu.yaml
@@ -6,7 +6,8 @@ groups:
     expr: rate(container_cpu_cfs_throttled_seconds_total{pod=~".+", container_name!="POD", image!=""}[5m]) > 1
     for: 5m
     labels:
-      severity: low
+      severity: informational
+      service: node-exporter
     annotations:
       summary: "Container {{ $labels.container_name }} in {{ $labels.pod }} high throttling "
       description: "{{ $labels.container_name }} inside {{ $labels.pod }} is at {{ $value }}"
@@ -15,7 +16,8 @@ groups:
     expr: 100 - (avg by (kubernetes_node) (irate(node_cpu_seconds_total{job="node-exporter",mode="idle"}[5m])) * 100) > 80
     for: 5m
     labels:
-      severity: page
+      severity: warning
+      service: node-exporter
     annotations:
       summary: "Node {{ $labels.kubernetes_node }} has CPU over 80% for last 5 minute"
       description: "CPU on {{ $labels.kubernetes_node }} is averaging {{ $value }}"
diff --git a/kubernetes/namespaces/monitoring/alerts/alerts.d/discord.yaml b/kubernetes/namespaces/monitoring/alerts/alerts.d/discord.yaml
index b36212c6..6769cfb6 100644
--- a/kubernetes/namespaces/monitoring/alerts/alerts.d/discord.yaml
+++ b/kubernetes/namespaces/monitoring/alerts/alerts.d/discord.yaml
@@ -6,8 +6,9 @@ groups:
     expr: deriv(bot_guild_total_members[30m]) * 3600 > 60
     for: 1m
     labels:
-      severity: page
+      severity: informational
       additional_destination: mods
+      service: discord
     annotations:
       summary: "High rate of guild member joins"
       description: "User join rate is {{ $value }} per hour, which may indicate a raid or other unusual activity."
@@ -17,7 +18,8 @@ groups:
       delta(bot_guild_total_members[5m]) > 50
     for: 1m
     labels:
-      severity: page
+      severity: informational
+      service: discord
     annotations:
       summary: "Sudden member spike detected"
       description: >
@@ -29,7 +31,8 @@ groups:
       deriv(bot_guild_total_members[15m]) * 60 > 10
     for: 10m
     labels:
-      severity: page
+      severity: informational
+      service: discord
     annotations:
       summary: "Abnormal sustained join rate"
       description: >
@@ -41,7 +44,8 @@ groups:
       delta(bot_guild_total_members[5m]) < -30
     for: 1m
     labels:
-      severity: page
+      severity: informational
+      service: discord
     annotations:
       summary: "Sudden member drop detected"
       description: >
@@ -53,7 +57,8 @@ groups:
       changes(bot_guild_total_members[10m]) > 20
     for: 5m
     labels:
-      severity: page
+      severity: informational
+      service: discord
     annotations:
       summary: "High member churn detected"
       description: >
diff --git a/kubernetes/namespaces/monitoring/alerts/alerts.d/django.yaml b/kubernetes/namespaces/monitoring/alerts/alerts.d/django.yaml
index 36fb36a3..e9856d5b 100644
--- a/kubernetes/namespaces/monitoring/alerts/alerts.d/django.yaml
+++ b/kubernetes/namespaces/monitoring/alerts/alerts.d/django.yaml
@@ -5,7 +5,9 @@ groups:
         expr: increase(django_http_responses_total_by_status_total{status=~"5.."}[5m]) > 0
         for: 5m
         labels:
-          severity: page
+          severity: critical
+          service: django
+          requires_ack: "true"
         annotations:
           summary: "Django is experiencing 5xx errors"
           description: "Django is experiencing 5xx errors on {{ $labels.namespace }}/{{ $labels.job }}"
@@ -14,7 +16,8 @@ groups:
         expr: histogram_quantile(0.95, rate(django_http_requests_latency_seconds_by_view_method_bucket{view!="api:github-artifacts", view!="api:github-webhook-filter", view!="home:home", view!="content:tag"}[5m])) > 1.0
         for: 15m
         labels:
-          severity: low
+          severity: informational
+          service: django
         annotations:
           summary: "Django route is experiencing high latency"
           description: "Django route {{ $labels.method }} {{ $labels.view }} has raised latency"
@@ -23,7 +26,9 @@ groups:
         expr: histogram_quantile(0.95, rate(django_http_requests_latency_seconds_by_view_method_bucket{view!="api:github-artifacts", view!="api:github-webhook-filter", view!="home:home", view!="content:tag"}[5m])) > 10.0
         for: 15m
         labels:
-          severity: page
+          severity: warning
+          service: django
+          requires_ack: "true"
         annotations:
           summary: "Django route is experiencing high latency"
           description: "Django route {{ $labels.method }} {{ $labels.view }} is experiencing high latency"
diff --git a/kubernetes/namespaces/monitoring/alerts/alerts.d/etcd.yaml b/kubernetes/namespaces/monitoring/alerts/alerts.d/etcd.yaml
index 1799e705..3bd62928 100644
--- a/kubernetes/namespaces/monitoring/alerts/alerts.d/etcd.yaml
+++ b/kubernetes/namespaces/monitoring/alerts/alerts.d/etcd.yaml
@@ -5,7 +5,8 @@ groups:
         expr: rate(etcd_request_error_total[5m]) > 0
         for: 5m
         labels:
-          severity: page
+          severity: major
+          service: etcd
         annotations:
           summary: "Etcd request errors are spiking"
           description: "Etcd request errors are spiking on {{ $labels.instance }} for {{ $labels.operation }}s to {{ $labels.type }}"
diff --git a/kubernetes/namespaces/monitoring/alerts/alerts.d/jobs.yaml b/kubernetes/namespaces/monitoring/alerts/alerts.d/jobs.yaml
index 740739a8..1e760f38 100644
--- a/kubernetes/namespaces/monitoring/alerts/alerts.d/jobs.yaml
+++ b/kubernetes/namespaces/monitoring/alerts/alerts.d/jobs.yaml
@@ -5,7 +5,9 @@ groups:
     expr: kube_cronjob_spec_suspend != 0
     for: 0m
     labels:
-      severity: low
+      severity: informational
+      service: kubernetes-jobs
+      instance: '{{ $labels.kubernetes_cronjob }}'
     annotations:
       summary: "Kubernetes CronJob suspended: {{ $labels.cronjob }}"
       description: "CronJob {{ $labels.kubernetes_namespace }}/{{ $labels.cronjob }} is suspended"
@@ -14,7 +16,10 @@ groups:
     expr: kube_job_status_failed > 0
     for: 0m
     labels:
-      severity: page
+      severity: major
+      service: kubernetes-jobs
+      requires_ack: "true"
+      instance: '{{ $labels.kubernetes_job }}'
     annotations:
       summary: "Kubernetes Job failed: {{ $labels.job_name }}"
       description: "Job {{$labels.kubernetes_namespacenamespace}}/{{$labels.job_name}} failed to complete"
diff --git a/kubernetes/namespaces/monitoring/alerts/alerts.d/memory.yaml b/kubernetes/namespaces/monitoring/alerts/alerts.d/memory.yaml
index b7de612f..8ba05a65 100644
--- a/kubernetes/namespaces/monitoring/alerts/alerts.d/memory.yaml
+++ b/kubernetes/namespaces/monitoring/alerts/alerts.d/memory.yaml
@@ -6,16 +6,22 @@ groups:
     expr: node_memory_Active_bytes / node_memory_MemTotal_bytes > 0.8
     for: 30s
     labels:
-      severity: page
+      severity: warning
+      service: kubernetes-nodes
+      requires_ack: "true"
+      instance: '{{ $labels.kubernetes_node }}'
     annotations:
       summary: "Node {{ $labels.kubernetes_node }} has RAM usage >80% for 5 minutes"
       description: 'RAM usage is currently {{ $value | humanizePercentage }} on {{ $labels.kubernetes_node }}'
 
   - alert: container/oom
-    expr: (kube_pod_container_status_restarts_total - kube_pod_container_status_restarts_total offset 10m >= 1) and ignoring (reason) min_over_time(kube_pod_container_status_last_terminated_reason{reason="OOMKilled"}[10m]) == 1
+    expr: (kube_pod_container_status_restarts_total - kube_pod_container_status_restarts_total offset 5m >= 1) and ignoring (reason) min_over_time(kube_pod_container_status_last_terminated_reason{reason="OOMKilled"}[5m]) == 1
     for: 0m
     labels:
-      severity: page
+      severity: critical
+      service: kubernetes-containers
+      requires_ack: "true"
+      instance: '{{ $labels.pod }}'
     annotations:
       summary: Kubernetes Container oom killer (instance {{ $labels.instance }})
-      description: "Container {{ $labels.container }} in pod {{ $labels.namespace }}/{{ $labels.pod }} has been OOMKilled {{ $value }} times in the last 10 minutes.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+      description: "Container {{ $labels.container }} in pod {{ $labels.namespace }}/{{ $labels.pod }} has been OOMKilled {{ $value }} times in the last 5 minutes.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
diff --git a/kubernetes/namespaces/monitoring/alerts/alerts.d/nodes.yaml b/kubernetes/namespaces/monitoring/alerts/alerts.d/nodes.yaml
index 6450af3d..f7aa3c98 100644
--- a/kubernetes/namespaces/monitoring/alerts/alerts.d/nodes.yaml
+++ b/kubernetes/namespaces/monitoring/alerts/alerts.d/nodes.yaml
@@ -6,7 +6,10 @@ groups:
     expr: kube_node_status_condition{condition="DiskPressure",status="true"} == 1
     for: 1m
     labels:
-      severity: page
+      severity: critical
+      service: kubernetes-nodes
+      requires_ack: "true"
+      instance: '{{ $labels.kubernetes_node }}'
     annotations:
       summary: Node {{ $labels.kubernetes_node }} is experiencing disk pressure
       description: "{{ $labels.kubernetes_node }} does not have adequate space to work with."
@@ -15,7 +18,10 @@ groups:
     expr: kube_node_status_condition{condition="MemoryPressure",status="true"} == 1
     for: 15s
     labels:
-      severity: page
+      severity: critical
+      service: kubernetes-nodes
+      requires_ack: "true"
+      instance: '{{ $labels.kubernetes_node }}'
     annotations:
       summary: Node {{ $labels.kubernetes_node }} is experiencing memory pressure
       description: "{{ $labels.kubernetes_node }} does not have adequate RAM to work with."
@@ -24,7 +30,10 @@ groups:
     expr: kube_node_status_condition{condition="NetworkUnavailable",status="true"} == 1
     for: 15s
     labels:
-      severity: page
+      severity: critical
+      service: kubernetes-nodes
+      requires_ack: "true"
+      instance: '{{ $labels.kubernetes_node }}'
     annotations:
       summary: Node {{ $labels.kubernetes_node }} is experiencing network problems
       description: "{{ $labels.kubernetes_node }} is experiencing trouble with inbound and outbound connections"
@@ -34,7 +43,10 @@ groups:
     expr: kube_node_status_condition{condition="PIDPressure",status="true"} == 1
     for: 15s
     labels:
-      severity: page
+      severity: critical
+      service: kubernetes-nodes
+      requires_ack: "true"
+      instance: '{{ $labels.kubernetes_node }}'
     annotations:
       summary: Node {{ $labels.kubernetes_node }} is experiencing PID exhaustion
       description: "{{ $labels.kubernetes_node }} does not have enough PIDs to work with."
@@ -43,7 +55,10 @@ groups:
     expr: kube_node_status_condition{condition="Ready",status="true"} == 0
     for: 1m
     labels:
-      severity: page
+      severity: critical
+      service: kubernetes-nodes
+      requires_ack: "true"
+      instance: '{{ $labels.kubernetes_node }}'
     annotations:
       summary: Kubernetes node ({{ $labels.kubernetes_node }} ) is marked as unready
       description: "Node {{ $labels.kubernetes_node }} has been unready for a long time"
@@ -52,7 +67,10 @@ groups:
     expr: kube_node_spec_unschedulable == 1
     for: 30m
     labels:
-      severity: low
+      severity: informational
+      service: kubernetes-nodes
+      requires_ack: "true"
+      instance: '{{ $labels.kubernetes_node }}'
     annotations:
       summary: Kubernetes node ({{ $labels.kubernetes_node }}) is cordoned
       description: "Node {{ $labels.kubernetes_node }} has been cordoned"
diff --git a/kubernetes/namespaces/monitoring/alerts/alerts.d/pods.yaml b/kubernetes/namespaces/monitoring/alerts/alerts.d/pods.yaml
index b03398a1..35b3a512 100644
--- a/kubernetes/namespaces/monitoring/alerts/alerts.d/pods.yaml
+++ b/kubernetes/namespaces/monitoring/alerts/alerts.d/pods.yaml
@@ -5,7 +5,10 @@ groups:
     expr: min_over_time(sum by (namespace, pod) (kube_pod_status_phase{phase=~"Pending|Unknown|Failed"})[3m:1m]) > 0
     for: 3m
     labels:
-      severity: page
+      severity: major
+      service: kubernetes-containers
+      requires_ack: "true"
+      instance: '{{ $labels.pod }}'
     annotations:
       summary: "Kubernetes Pod not healthy: {{ $labels.namespace }}/{{ $labels.pod }}"
       description: "Pod has been in a non-ready state for longer than 3 minutes."
@@ -14,7 +17,10 @@ groups:
     expr: increase(kube_pod_container_status_restarts_total[5m]) > 3
     for: 1m
     labels:
-      severity: warning
+      severity: major
+      service: kubernetes-containers
+      requires_ack: "true"
+      instance: '{{ $labels.pod }}'
     annotations:
       summary: "Kubernetes pod crash looping: {{ $labels.kubernetes_namespace }}/{{ $labels.pod }}"
       description: "Pod {{ $labels.pod }} is crash looping"
diff --git a/kubernetes/namespaces/monitoring/alerts/alerts.d/prometheus.yaml b/kubernetes/namespaces/monitoring/alerts/alerts.d/prometheus.yaml
index 22e16cf4..f805c41f 100644
--- a/kubernetes/namespaces/monitoring/alerts/alerts.d/prometheus.yaml
+++ b/kubernetes/namespaces/monitoring/alerts/alerts.d/prometheus.yaml
@@ -7,7 +7,9 @@ groups:
     expr: up == 0
     for: 5m
     labels:
-      severity: page
+      severity: critical
+      service: prometheus
+      instance: '{{ $labels.job }}/{{ $labels.instance }}'
     annotations:
       summary: "Instance {{ $labels.instance }} down"
       description: "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 5 minutes."
@@ -16,7 +18,8 @@ groups:
     expr: prometheus_config_last_reload_successful == 0
     for: 0m
     labels:
-      severity: page
+      severity: critical
+      service: prometheus
     annotations:
       summary: "Prometheus config reload in pod {{ $labels.kubernetes_pod_name }} has failed"
       description: "Prometheus instance {{ $labels.kubernetes_pod_name }} (`{{ $labels.instance }}`) has failed to reload its config."
diff --git a/kubernetes/namespaces/monitoring/alerts/alerts.d/redis.yaml b/kubernetes/namespaces/monitoring/alerts/alerts.d/redis.yaml
index 42ce4d15..f3f6df80 100644
--- a/kubernetes/namespaces/monitoring/alerts/alerts.d/redis.yaml
+++ b/kubernetes/namespaces/monitoring/alerts/alerts.d/redis.yaml
@@ -5,7 +5,8 @@ groups:
     expr: redis_up == 0
     for: 1m
     labels:
-      severity: page
+      severity: critical
+      service: redis
     annotations:
       summary: "Redis is offline"
       description: "Redis Exporter cannot connect to Redis."
@@ -14,7 +15,9 @@ groups:
     expr: redis_memory_used_bytes / redis_memory_max_bytes > 0.9
     for: 0m
     labels:
-      severity: page
+      severity: major
+      service: redis
+      requires_ack: "true"
     annotations:
       summary: "Redis is approaching it's memory limit"
       description: "Redis is currently using {{ $value | humanizePercentage }} of configured memory."
diff --git a/kubernetes/namespaces/monitoring/alerts/alerts.d/volumes.yaml b/kubernetes/namespaces/monitoring/alerts/alerts.d/volumes.yaml
index 410879d9..4e379581 100644
--- a/kubernetes/namespaces/monitoring/alerts/alerts.d/volumes.yaml
+++ b/kubernetes/namespaces/monitoring/alerts/alerts.d/volumes.yaml
@@ -5,7 +5,10 @@ groups:
     expr: kubelet_volume_stats_available_bytes{persistentvolumeclaim!="prometheus-storage"} / kubelet_volume_stats_capacity_bytes * 100 < 10
     for: 2m
     labels:
-      severity: page
+      severity: major
+      service: kubernetes-containers
+      requires_ack: "true"
+      instance: '{{ $labels.kubernetes_namespace }}/{{ $labels.persistentvolumeclaim }}'
     annotations:
       summary: Kubernetes Volume {{ $labels.kubernetes_namespace }}/{{ $labels.persistentvolumeclaim }} is running low on disk space
       description: "Volume is almost full (< 10% left)\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
diff --git a/kubernetes/namespaces/nginx-gateway/routes/service-routes/alerta-pydis-wtf.yaml b/kubernetes/namespaces/nginx-gateway/routes/service-routes/alerta-pydis-wtf.yaml
new file mode 100644
index 00000000..8829f538
--- /dev/null
+++ b/kubernetes/namespaces/nginx-gateway/routes/service-routes/alerta-pydis-wtf.yaml
@@ -0,0 +1,22 @@
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: alerta-pydis-wtf
+  namespace: nginx-gateway
+spec:
+  hostnames:
+    - alerta.pydis.wtf
+  parentRefs:
+    - name: nginx
+      namespace: nginx-gateway
+  rules:
+    - backendRefs:
+      - name: alerta
+        namespace: monitoring
+        port: 8080
+      filters:
+      - type: ExtensionRef
+        extensionRef:
+          group: gateway.nginx.org
+          kind: SnippetsFilter
+          name: enforce-mtls