diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 91325fe..84f0b99 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -24,6 +24,38 @@ the per-chunk equality filters, `row_mask` records the delete and update marks, and `options` holds per-table settings. A storage id links a relation's physical file to its catalog rows. +## Crash safety + +A columnar relation's data lives in its main fork through the buffer manager and +WAL, and its metadata lives in WAL-logged heap catalogs, so a columnar table is +as crash-safe as any other PostgreSQL relation and the guarantee rests on core +rather than on this extension. No write path bypasses the buffer manager or +writes the file directly: data pages are dirtied and WAL-logged with +`log_newpage_buffer`, the initial metapage and reserved page are WAL-logged +before they are extended, and truncation is an `XLOG_SMGR_TRUNCATE` record. + +Two crash shapes follow from this. + +- The whole cluster crashes. On restart, WAL replay restores every committed + write and discards every uncommitted one, so a committed columnar table matches + a heap table written the same way, and an in-flight transaction leaves nothing + visible. `test/recovery.sh` asserts both against a heap mirror. +- One backend dies while the postmaster lives, which is the shape a fault in a + single session produces. The postmaster reinitializes shared memory, drops + every other session, and runs the same crash recovery. Committed columnar data + survives intact, and an uncommitted write in the dead backend leaves no visible + rows and no partial row group, for the same reason the cluster case does: the + surviving state is exactly what WAL records as committed. Other connected + sessions are dropped, not only the one that died. `test/native_backend_crash.sh` + asserts this against a heap mirror, killing one backend with `SIGSEGV` at more + than one point in the write path and once under a concurrent reader. + +What does not survive is process-local: an open transaction's uncommitted work, +and the connections of every session live at the moment of the crash. Every +committed row survives. No columnar-specific shared state is left broken for the +next backend, because the data and its catalog are both ordinary WAL-logged +storage. + ## Module map ### columnar_tableam.c diff --git a/test/native_backend_crash.sh b/test/native_backend_crash.sh new file mode 100755 index 0000000..9f46f43 --- /dev/null +++ b/test/native_backend_crash.sh @@ -0,0 +1,201 @@ +#!/usr/bin/env bash +# +# pgColumnar single-backend crash blast radius (#217). +# +# recovery.sh crashes the whole cluster and restarts it. This suite covers the +# other shape: one backend dies while the postmaster lives, which is the #210 +# case (a SIGSEGV takes down a single backend, the postmaster reinitializes +# shared memory and drops every other session). The guarantee under test is that +# committed columnar data is intact on the far side of that, and that an +# in-flight write leaves nothing, because columnar data goes through the buffer +# manager and WAL like any other relation. If that holds, a single backend death +# is no different from any other backend death in PostgreSQL. +# +# A heap mirror written the same way is the oracle: after the crash the columnar +# table must match it exactly. The kill is repeated at more than one point in the +# write path -- asserted from wait_event, not assumed from a timing delay. Because +# the write is uncommitted either way, "no rows survived" alone cannot tell a +# crash apart from a clean abort, so every scenario also opens a sentinel session +# before the kill and asserts the reinit dropped it: that is what proves a crash +# actually happened and the durability checks are not passing vacuously. +# +# Usage: test/native_backend_crash.sh [PG_CONFIG] +# Written fresh for pgColumnar. + +set -uo pipefail +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +pgc_setup "${1:-/usr/local/pg17/bin/pg_config}" + +# The postmaster must reinitialize a crashed backend rather than shut the cluster +# down; that is the default, set here so the suite does not depend on it. +psql_run "ALTER SYSTEM SET restart_after_crash = on;" +psql_run "SELECT pg_reload_conf();" >/dev/null + +# Heap mirror and columnar table, same committed baseline in both. +psql_run "DROP TABLE IF EXISTS h; DROP TABLE IF EXISTS c; + CREATE TABLE h (id int, v text, x float8); + CREATE TABLE c (id int, v text, x float8) USING pgcolumnar; + INSERT INTO h SELECT g, 'b'||g, g*2.5 FROM generate_series(1, 5000) g; + INSERT INTO c SELECT g, 'b'||g, g*2.5 FROM generate_series(1, 5000) g;" + +rowgroups() { + q "SELECT count(*) FROM pgcolumnar.row_group r + JOIN pgcolumnar.storage s ON s.storage_id = r.storage_id + WHERE s.relation_oid = 'c'::regclass;" +} +content() { pgc_set_hash "SELECT id, v, x FROM c"; } + +base_groups="$(rowgroups)" +base_hash="$(content)" +base_mirror="$(pgc_set_hash "SELECT id, v, x FROM h")" +pm_before="$(head -1 "$PGC_PGDATA/postmaster.pid" 2>/dev/null)" + +# Wait for the postmaster to finish reinitializing and accept connections again. +# Reinitialization briefly accepts a connection and then refuses the next while it +# finishes clearing the old backends, so a handful of quick successes can all land +# inside one transient "accepting" blip and race the recovery window. Require a +# continuous run of successes that spans a real stretch of wall-clock time instead: +# 12 in a row at 0.25s is a 3s uninterrupted window a reinit blip cannot fake, and +# it holds up under matrix contention where the blip is wider. +wait_up() { + local i ok=0 + for i in $(seq 1 1200); do + if [ "$(q 'SELECT 1' 2>/dev/null)" = "1" ]; then + ok=$((ok + 1)) + [ "$ok" -ge 12 ] && return 0 + else + ok=0 + fi + sleep 0.25 + done + return 1 +} + +# A backend that must NOT die of its own accord, opened before each crash so we +# can prove the postmaster actually reinitialized: a plain backend exit drops only +# itself, a crash-triggered reinit drops every other session too. If this session +# survives the kill, no reinit happened and the durability checks below would be +# passing vacuously (the insert is uncommitted either way). Sleeps 120s, longer +# than any single scenario, and is identified by that literal so its own pid is +# easy to find. +sentinel_dropped="" +open_sentinel() { + env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres \ + -d "$PGC_DB" -c "SELECT pg_sleep(120);" >/dev/null 2>&1 & + sentinel_bg=$! + # Generous budget: this connect races the rest of the matrix for the box, and + # a false "did not connect" here would fail the reinit assertion spuriously. + local i + for i in $(seq 1 300); do + [ -n "$(q "SELECT pid FROM pg_stat_activity + WHERE query LIKE '%pg_sleep(120)%' AND pid <> pg_backend_pid() LIMIT 1;")" ] && return 0 + sleep 0.1 + done + return 1 +} + +# Run an INSERT into c inside an explicit transaction held open by a trailing +# pg_sleep, so the write is never committed no matter how fast it runs; then +# SIGSEGV that one backend and wait for the postmaster to reinitialize. $1 rows +# sizes the write, $2 a marker that locates the backend in its query text, $3 the +# phase to kill in: "insert" lands the kill mid-write (the backend is still +# executing the INSERT), "sleep" lands it once every stripe has flushed and the +# backend is parked in the trailing pg_sleep, between the last flush and the +# commit that never comes. The phase is asserted, not assumed, from wait_event. +crash_one_backend() { + local rows="$1" marker="$2" phase="$3" bg pid i we src + open_sentinel || echo "-- sentinel did not connect" + env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres \ + -d "$PGC_DB" -c "BEGIN; INSERT INTO c /* $marker */ SELECT g, 'k'||g, g + FROM generate_series(1, $rows) g; SELECT pg_sleep(60);" >/dev/null 2>&1 & + bg=$! + pid="" + for i in $(seq 1 300); do + pid="$(q "SELECT pid FROM pg_stat_activity + WHERE query LIKE '%$marker%' AND state = 'active' + AND pid <> pg_backend_pid() LIMIT 1;")" + [ -n "$pid" ] && break + sleep 0.1 + done + if [ -z "$pid" ]; then + echo "-- could not locate the inserting backend" + kill "$bg" "$sentinel_bg" 2>/dev/null || true + return 1 + fi + # Drive to the requested phase. For "sleep" poll until the backend actually + # reaches pg_sleep (the insert has flushed and finished); for "insert" give it + # a moment to be well inside the write, then confirm it is not yet sleeping. + if [ "$phase" = sleep ]; then + for i in $(seq 1 600); do + [ "$(q "SELECT coalesce(wait_event,'') FROM pg_stat_activity WHERE pid = $pid;")" = PgSleep ] && break + sleep 0.1 + done + else + sleep 0.5 + fi + we="$(q "SELECT coalesce(wait_event,'') FROM pg_stat_activity WHERE pid = $pid;")" + if [ "$phase" = sleep ]; then + check "$marker: kill lands after the stripes flushed (backend in pg_sleep)" "$we" "PgSleep" + else + check "$marker: kill lands mid-write (backend not yet in pg_sleep)" \ + "$( [ "$we" != PgSleep ] && echo yes || echo no )" "yes" + fi + echo "-- SIGSEGV backend $pid ($marker, phase $phase, wait_event=${we:-})" + kill -SEGV "$pid" 2>/dev/null || true + kill "$bg" 2>/dev/null || true + wait "$bg" 2>/dev/null || true + wait_up || echo "-- cluster did not come back" + # The sentinel opened before the crash must have been dropped by the reinit; + # if it is still alive, no reinit happened and nothing below is a real test. + # Poll for the client to exit rather than wait on it: a reinit drops its + # connection and it exits within the recovery window (already elapsed in + # wait_up), while a session that was NOT dropped would otherwise block this + # wait for the full pg_sleep. No exit inside the window means not dropped. + sentinel_dropped=no + for i in $(seq 1 50); do + if ! kill -0 "$sentinel_bg" 2>/dev/null; then + wait "$sentinel_bg" 2>/dev/null || true + sentinel_dropped=yes + break + fi + sleep 0.1 + done + [ "$sentinel_dropped" = yes ] || { kill "$sentinel_bg" 2>/dev/null || true; wait "$sentinel_bg" 2>/dev/null || true; } +} + +assert_intact() { # label + check "$1: the crash reinitialized the cluster (a live session was dropped)" \ + "$sentinel_dropped" "yes" + check "$1: postmaster survived, same pid" \ + "$(head -1 "$PGC_PGDATA/postmaster.pid" 2>/dev/null)" "$pm_before" + check "$1: cluster accepts new connections" "$(q 'SELECT 1')" "1" + check "$1: committed content in c matches the heap mirror" "$(content)" "$base_hash" + check "$1: the mirror itself is unchanged" \ + "$(pgc_set_hash "SELECT id, v, x FROM h")" "$base_mirror" + check "$1: the killed insert left no visible rows" "$(q 'SELECT count(*) FROM c')" "5000" + check "$1: the killed insert left no partial row group" "$(rowgroups)" "$base_groups" +} + +# --- kill while a large insert is mid-write ---------------------------------- +crash_one_backend 3000000 crashA insert +assert_intact "mid-write" + +# --- kill after the stripes have flushed, before the commit that never comes - +crash_one_backend 100000 crashB sleep +assert_intact "post-flush" + +# --- kill with a concurrent reader holding a connection ---------------------- +# The reinit drops every session, not only the one that died. This scenario adds +# a second parked reader on top of the sentinel and asserts it too was dropped, +# so the "every session" claim is tested with more than one bystander present. +env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres \ + -d "$PGC_DB" -c "SELECT pg_sleep(90);" >/dev/null 2>&1 & +reader=$! +sleep 0.5 +crash_one_backend 3000000 crashC insert +wait "$reader" 2>/dev/null; reader_rc=$? +check "a second concurrent reader was also dropped by the crash" \ + "$( [ "$reader_rc" -ne 0 ] && echo yes || echo no )" "yes" +assert_intact "with-reader" + +pgc_summary diff --git a/test/run_all_versions.sh b/test/run_all_versions.sh index 3bf4ee4..07134a0 100755 --- a/test/run_all_versions.sh +++ b/test/run_all_versions.sh @@ -90,7 +90,7 @@ trap - EXIT SRCDIR="${PGC_RUN_SRCDIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}" SUITES=(harness_selftest smoke phase2 phase3 phase4 phase5 phase6 audit concurrency unique_conc \ - differential recovery fuzz fuzz_parquet hardening concurrent_diff parallel sorted_projection \ + differential recovery native_backend_crash fuzz fuzz_parquet hardening concurrent_diff parallel sorted_projection \ arrow_export parquet_export read_stream corruption \ generated_columns temporal arrow_import index_only projections arrow_nested parquet_import parquet_nested arrow_nested_import parquet_nested_import native_writer native_roundtrip native_encoding native_zonemap write_minmax_fastpath write_fsst_compressed encode_effort native_skip pushdown_report native_agg native_agg_deletes native_agg_addcolumn native_bloom bloom_setting native_vecskip native_index native_fetch_position native_dml alter_column_type native_ios native_projection native_cluster native_compact native_recluster native_reclaim native_ownership drop_cleanup native_reclaim_cycles native_reclaim_frag native_reclaim_reconcile native_gap native_truncate native_rewrite native_rewrite_conc rewrite_group_scan native_parquet_schema native_read_parquet native_parquet_fdw native_parquet_pushdown native_parquet_hardening native_parquet_stack native_parquet_units native_parquet_flba native_parquet_codecs native_parquet_projection native_parquet_multifile native_parquet_streaming native_parquet_partition native_cancel wal_envelope decode_interrupts import_exclusion import_deferred fk_referencing row_triggers native_lazy_slot native_fetch_cache native_fetch_interrupt analyze_stats analyze_reltuples native_fetch_projection isolation)