diff --git a/README.md b/README.md index 61bdf788a7..409ac759a4 100644 --- a/README.md +++ b/README.md @@ -82,6 +82,7 @@ PGO runs containers with locked-down settings and provides Postgres credentials #### [Monitoring][monitoring] [Track the health of your PostgreSQL clusters][monitoring] using the open source [pgMonitor][] library. +With OpenTelemetry metrics enabled, pgBackRest-derived backup metrics, including `last_info_backup_error`, refresh every 10 minutes and may be stale by up to 10 minutes. Alerting should account for this delay. #### [Upgrade Management][update-postgres] diff --git a/internal/controller/postgrescluster/metrics_setup.sql b/internal/controller/postgrescluster/metrics_setup.sql index 2d282643ce..1def1c3cbc 100644 --- a/internal/controller/postgrescluster/metrics_setup.sql +++ b/internal/controller/postgrescluster/metrics_setup.sql @@ -30,6 +30,13 @@ CREATE TABLE monitor.pg_stat_statements_reset_info( reset_time timestamptz ); +DROP TABLE IF EXISTS monitor.pgbackrest_info_cache; +-- Table to cache pgBackRest info output and avoid frequent cloud egress. +CREATE TABLE monitor.pgbackrest_info_cache( + collected_at timestamptz DEFAULT now() NOT NULL, + data json NOT NULL +); + DROP FUNCTION IF EXISTS monitor.pg_stat_statements_reset_info(int); -- Function to reset pg_stat_statements periodically CREATE FUNCTION monitor.pg_stat_statements_reset_info(p_throttle_minutes integer DEFAULT 1440) @@ -89,6 +96,8 @@ DROP FUNCTION IF EXISTS get_pgbackrest_info(); --- get_pgbackrest_info is used by the OTel collector. --- get_pgbackrest_info is created as a function so that no ddl runs on a replica. --- In the query, the --stanza argument matches DefaultStanzaName, defined in internal/pgbackrest/config.go. +--- To match legacy postgres_exporter behavior, pgbackrest info output is refreshed +--- at most once every 10 minutes and cached in monitor.pgbackrest_info_cache. CREATE FUNCTION get_pgbackrest_info() RETURNS TABLE ( last_diff_backup BIGINT, @@ -102,6 +111,8 @@ RETURNS TABLE ( oldest_full_backup BIGINT, repo TEXT ) AS $$ +DECLARE + v_collected_timestamp timestamptz; BEGIN IF pg_is_in_recovery() THEN RETURN QUERY @@ -117,16 +128,45 @@ BEGIN 0::bigint AS oldest_full_backup, 'n/a' AS repo; ELSE - DROP TABLE IF EXISTS pgbackrest_info; - CREATE TEMPORARY TABLE pgbackrest_info (data json); - COPY pgbackrest_info (data) - FROM PROGRAM 'export LC_ALL=C && printf "\f" && pgbackrest info --log-level-console=info --log-level-stderr=warn --output=json --stanza=db && printf "\f"' - WITH (FORMAT csv, HEADER false, QUOTE E'\f'); + SELECT max(collected_at) + INTO v_collected_timestamp + FROM monitor.pgbackrest_info_cache; + + IF v_collected_timestamp IS NULL OR + CURRENT_TIMESTAMP - v_collected_timestamp > interval '10 minutes' THEN + IF pg_try_advisory_xact_lock( + hashtext('crunchydata.postgres-operator'), + hashtext('get_pgbackrest_info') + ) THEN + SELECT max(collected_at) + INTO v_collected_timestamp + FROM monitor.pgbackrest_info_cache; + + IF v_collected_timestamp IS NULL OR + CURRENT_TIMESTAMP - v_collected_timestamp > interval '10 minutes' THEN + CREATE TEMPORARY TABLE IF NOT EXISTS pgbackrest_info_tmp (data json) ON COMMIT DROP; + TRUNCATE TABLE pgbackrest_info_tmp; + COPY pgbackrest_info_tmp (data) + FROM PROGRAM 'export LC_ALL=C && printf "\f" && pgbackrest info --log-level-console=info --log-level-stderr=warn --output=json --stanza=db && printf "\f"' + WITH (FORMAT csv, HEADER false, QUOTE E'\f'); + + DELETE FROM monitor.pgbackrest_info_cache; + IF EXISTS (SELECT 1 FROM pgbackrest_info_tmp) THEN + INSERT INTO monitor.pgbackrest_info_cache(collected_at, data) + SELECT CURRENT_TIMESTAMP, data FROM pgbackrest_info_tmp; + ELSE + INSERT INTO monitor.pgbackrest_info_cache(collected_at, data) + VALUES (CURRENT_TIMESTAMP, '[]'::json); + END IF; + END IF; + END IF; + END IF; RETURN QUERY WITH all_backups (data) AS ( - SELECT jsonb_array_elements(to_jsonb(data)) FROM pgbackrest_info + SELECT jsonb_array_elements(to_jsonb(data)) + FROM monitor.pgbackrest_info_cache ), stanza_backups (stanza, backup) AS ( SELECT data->>'name', jsonb_array_elements(data->'backup') FROM all_backups diff --git a/internal/controller/postgrescluster/pgmonitor_test.go b/internal/controller/postgrescluster/pgmonitor_test.go index aed23ab0d8..c375166ec6 100644 --- a/internal/controller/postgrescluster/pgmonitor_test.go +++ b/internal/controller/postgrescluster/pgmonitor_test.go @@ -956,7 +956,7 @@ func TestReconcileExporterSqlSetup(t *testing.T) { ctx := feature.NewContext(ctx, gate) if tc.otelMetricsEnabled { - cluster.Spec.Instrumentation = instrumentationSpec + cluster.Spec.Instrumentation = instrumentationSpec.DeepCopy() } if tc.exporterEnabled { @@ -970,6 +970,14 @@ func TestReconcileExporterSqlSetup(t *testing.T) { assert.NilError(t, err) } assert.Equal(t, setup == "", tc.setupEmpty) + if tc.otelMetricsEnabled { + assert.Assert(t, strings.Contains(setup, "CREATE FUNCTION get_pgbackrest_info()")) + assert.Assert(t, !strings.Contains(setup, "__PGBACKREST_INFO_THROTTLE_MINUTES__")) + assert.Assert(t, !strings.Contains(setup, "p_throttle_minutes")) + assert.Assert(t, strings.Contains(setup, "interval '10 minutes'")) + assert.Assert(t, strings.Contains(setup, "pg_try_advisory_xact_lock(")) + assert.Assert(t, strings.Contains(setup, "VALUES (CURRENT_TIMESTAMP, '[]'::json)")) + } assert.Equal(t, len(recorder.Events), tc.expectedNumEvents) if tc.expectedNumEvents == 1 {