Assert the single-backend crash blast radius (#217)

docs/ARCHITECTURE.md

@@ -24,6 +24,38 @@ the per-chunk equality filters, `row_mask` records the delete and update marks,
 and `options` holds per-table settings. A storage id links a relation's physical
 file to its catalog rows.
 
+## Crash safety
+
+A columnar relation's data lives in its main fork through the buffer manager and
+WAL, and its metadata lives in WAL-logged heap catalogs, so a columnar table is
+as crash-safe as any other PostgreSQL relation and the guarantee rests on core
+rather than on this extension. No write path bypasses the buffer manager or
+writes the file directly: data pages are dirtied and WAL-logged with
+`log_newpage_buffer`, the initial metapage and reserved page are WAL-logged
+before they are extended, and truncation is an `XLOG_SMGR_TRUNCATE` record.
+
+Two crash shapes follow from this.
+
+- The whole cluster crashes. On restart, WAL replay restores every committed
+  write and discards every uncommitted one, so a committed columnar table matches
+  a heap table written the same way, and an in-flight transaction leaves nothing
+  visible. `test/recovery.sh` asserts both against a heap mirror.
+- One backend dies while the postmaster lives, which is the shape a fault in a
+  single session produces. The postmaster reinitializes shared memory, drops
+  every other session, and runs the same crash recovery. Committed columnar data
+  survives intact, and an uncommitted write in the dead backend leaves no visible
+  rows and no partial row group, for the same reason the cluster case does: the
+  surviving state is exactly what WAL records as committed. Other connected
+  sessions are dropped, not only the one that died. `test/native_backend_crash.sh`
+  asserts this against a heap mirror, killing one backend with `SIGSEGV` at more
+  than one point in the write path and once under a concurrent reader.
+
+What does not survive is process-local: an open transaction's uncommitted work,
+and the connections of every session live at the moment of the crash. Every
+committed row survives. No columnar-specific shared state is left broken for the
+next backend, because the data and its catalog are both ordinary WAL-logged
+storage.
+
 ## Module map
 
 ### columnar_tableam.c

test/native_backend_crash.sh

@@ -0,0 +1,201 @@
+#!/usr/bin/env bash
+#
+# pgColumnar single-backend crash blast radius (#217).
+#
+# recovery.sh crashes the whole cluster and restarts it. This suite covers the
+# other shape: one backend dies while the postmaster lives, which is the #210
+# case (a SIGSEGV takes down a single backend, the postmaster reinitializes
+# shared memory and drops every other session). The guarantee under test is that
+# committed columnar data is intact on the far side of that, and that an
+# in-flight write leaves nothing, because columnar data goes through the buffer
+# manager and WAL like any other relation. If that holds, a single backend death
+# is no different from any other backend death in PostgreSQL.
+#
+# A heap mirror written the same way is the oracle: after the crash the columnar
+# table must match it exactly. The kill is repeated at more than one point in the
+# write path -- asserted from wait_event, not assumed from a timing delay. Because
+# the write is uncommitted either way, "no rows survived" alone cannot tell a
+# crash apart from a clean abort, so every scenario also opens a sentinel session
+# before the kill and asserts the reinit dropped it: that is what proves a crash
+# actually happened and the durability checks are not passing vacuously.
+#
+# Usage:  test/native_backend_crash.sh [PG_CONFIG]
+# Written fresh for pgColumnar.
+
+set -uo pipefail
+. "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
+pgc_setup "${1:-/usr/local/pg17/bin/pg_config}"
+
+# The postmaster must reinitialize a crashed backend rather than shut the cluster
+# down; that is the default, set here so the suite does not depend on it.
+psql_run "ALTER SYSTEM SET restart_after_crash = on;"
+psql_run "SELECT pg_reload_conf();" >/dev/null
+
+# Heap mirror and columnar table, same committed baseline in both.
+psql_run "DROP TABLE IF EXISTS h; DROP TABLE IF EXISTS c;
+	CREATE TABLE h (id int, v text, x float8);
+	CREATE TABLE c (id int, v text, x float8) USING pgcolumnar;
+	INSERT INTO h SELECT g, 'b'||g, g*2.5 FROM generate_series(1, 5000) g;
+	INSERT INTO c SELECT g, 'b'||g, g*2.5 FROM generate_series(1, 5000) g;"
+
+rowgroups() {
+	q "SELECT count(*) FROM pgcolumnar.row_group r
+		JOIN pgcolumnar.storage s ON s.storage_id = r.storage_id
+		WHERE s.relation_oid = 'c'::regclass;"
+}
+content() { pgc_set_hash "SELECT id, v, x FROM c"; }
+
+base_groups="$(rowgroups)"
+base_hash="$(content)"
+base_mirror="$(pgc_set_hash "SELECT id, v, x FROM h")"
+pm_before="$(head -1 "$PGC_PGDATA/postmaster.pid" 2>/dev/null)"
+
+# Wait for the postmaster to finish reinitializing and accept connections again.
+# Reinitialization briefly accepts a connection and then refuses the next while it
+# finishes clearing the old backends, so a handful of quick successes can all land
+# inside one transient "accepting" blip and race the recovery window. Require a
+# continuous run of successes that spans a real stretch of wall-clock time instead:
+# 12 in a row at 0.25s is a 3s uninterrupted window a reinit blip cannot fake, and
+# it holds up under matrix contention where the blip is wider.
+wait_up() {
+	local i ok=0
+	for i in $(seq 1 1200); do
+		if [ "$(q 'SELECT 1' 2>/dev/null)" = "1" ]; then
+			ok=$((ok + 1))
+			[ "$ok" -ge 12 ] && return 0
+		else
+			ok=0
+		fi
+		sleep 0.25
+	done
+	return 1
+}
+
+# A backend that must NOT die of its own accord, opened before each crash so we
+# can prove the postmaster actually reinitialized: a plain backend exit drops only
+# itself, a crash-triggered reinit drops every other session too. If this session
+# survives the kill, no reinit happened and the durability checks below would be
+# passing vacuously (the insert is uncommitted either way).  Sleeps 120s, longer
+# than any single scenario, and is identified by that literal so its own pid is
+# easy to find.
+sentinel_dropped=""
+open_sentinel() {
+	env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres \
+		-d "$PGC_DB" -c "SELECT pg_sleep(120);" >/dev/null 2>&1 &
+	sentinel_bg=$!
+	# Generous budget: this connect races the rest of the matrix for the box, and
+	# a false "did not connect" here would fail the reinit assertion spuriously.
+	local i
+	for i in $(seq 1 300); do
+		[ -n "$(q "SELECT pid FROM pg_stat_activity
+			WHERE query LIKE '%pg_sleep(120)%' AND pid <> pg_backend_pid() LIMIT 1;")" ] && return 0
+		sleep 0.1
+	done
+	return 1
+}
+
+# Run an INSERT into c inside an explicit transaction held open by a trailing
+# pg_sleep, so the write is never committed no matter how fast it runs; then
+# SIGSEGV that one backend and wait for the postmaster to reinitialize. $1 rows
+# sizes the write, $2 a marker that locates the backend in its query text, $3 the
+# phase to kill in: "insert" lands the kill mid-write (the backend is still
+# executing the INSERT), "sleep" lands it once every stripe has flushed and the
+# backend is parked in the trailing pg_sleep, between the last flush and the
+# commit that never comes. The phase is asserted, not assumed, from wait_event.
+crash_one_backend() {
+	local rows="$1" marker="$2" phase="$3" bg pid i we src
+	open_sentinel || echo "-- sentinel did not connect"
+	env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres \
+		-d "$PGC_DB" -c "BEGIN; INSERT INTO c /* $marker */ SELECT g, 'k'||g, g
+			FROM generate_series(1, $rows) g; SELECT pg_sleep(60);" >/dev/null 2>&1 &
+	bg=$!
+	pid=""
+	for i in $(seq 1 300); do
+		pid="$(q "SELECT pid FROM pg_stat_activity
+			WHERE query LIKE '%$marker%' AND state = 'active'
+			  AND pid <> pg_backend_pid() LIMIT 1;")"
+		[ -n "$pid" ] && break
+		sleep 0.1
+	done
+	if [ -z "$pid" ]; then
+		echo "-- could not locate the inserting backend"
+		kill "$bg" "$sentinel_bg" 2>/dev/null || true
+		return 1
+	fi
+	# Drive to the requested phase. For "sleep" poll until the backend actually
+	# reaches pg_sleep (the insert has flushed and finished); for "insert" give it
+	# a moment to be well inside the write, then confirm it is not yet sleeping.
+	if [ "$phase" = sleep ]; then
+		for i in $(seq 1 600); do
+			[ "$(q "SELECT coalesce(wait_event,'') FROM pg_stat_activity WHERE pid = $pid;")" = PgSleep ] && break
+			sleep 0.1
+		done
+	else
+		sleep 0.5
+	fi
+	we="$(q "SELECT coalesce(wait_event,'') FROM pg_stat_activity WHERE pid = $pid;")"
+	if [ "$phase" = sleep ]; then
+		check "$marker: kill lands after the stripes flushed (backend in pg_sleep)" "$we" "PgSleep"
+	else
+		check "$marker: kill lands mid-write (backend not yet in pg_sleep)" \
+			"$( [ "$we" != PgSleep ] && echo yes || echo no )" "yes"
+	fi
+	echo "-- SIGSEGV backend $pid ($marker, phase $phase, wait_event=${we:-<null>})"
+	kill -SEGV "$pid" 2>/dev/null || true
+	kill "$bg" 2>/dev/null || true
+	wait "$bg" 2>/dev/null || true
+	wait_up || echo "-- cluster did not come back"
+	# The sentinel opened before the crash must have been dropped by the reinit;
+	# if it is still alive, no reinit happened and nothing below is a real test.
+	# Poll for the client to exit rather than wait on it: a reinit drops its
+	# connection and it exits within the recovery window (already elapsed in
+	# wait_up), while a session that was NOT dropped would otherwise block this
+	# wait for the full pg_sleep. No exit inside the window means not dropped.
+	sentinel_dropped=no
+	for i in $(seq 1 50); do
+		if ! kill -0 "$sentinel_bg" 2>/dev/null; then
+			wait "$sentinel_bg" 2>/dev/null || true
+			sentinel_dropped=yes
+			break
+		fi
+		sleep 0.1
+	done
+	[ "$sentinel_dropped" = yes ] || { kill "$sentinel_bg" 2>/dev/null || true; wait "$sentinel_bg" 2>/dev/null || true; }
+}
+
+assert_intact() {  # label
+	check "$1: the crash reinitialized the cluster (a live session was dropped)" \
+		"$sentinel_dropped" "yes"
+	check "$1: postmaster survived, same pid" \
+		"$(head -1 "$PGC_PGDATA/postmaster.pid" 2>/dev/null)" "$pm_before"
+	check "$1: cluster accepts new connections" "$(q 'SELECT 1')" "1"
+	check "$1: committed content in c matches the heap mirror" "$(content)" "$base_hash"
+	check "$1: the mirror itself is unchanged" \
+		"$(pgc_set_hash "SELECT id, v, x FROM h")" "$base_mirror"
+	check "$1: the killed insert left no visible rows" "$(q 'SELECT count(*) FROM c')" "5000"
+	check "$1: the killed insert left no partial row group" "$(rowgroups)" "$base_groups"
+}
+
+# --- kill while a large insert is mid-write ----------------------------------
+crash_one_backend 3000000 crashA insert
+assert_intact "mid-write"
+
+# --- kill after the stripes have flushed, before the commit that never comes -
+crash_one_backend 100000 crashB sleep
+assert_intact "post-flush"
+
+# --- kill with a concurrent reader holding a connection ----------------------
+# The reinit drops every session, not only the one that died. This scenario adds
+# a second parked reader on top of the sentinel and asserts it too was dropped,
+# so the "every session" claim is tested with more than one bystander present.
+env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres \
+	-d "$PGC_DB" -c "SELECT pg_sleep(90);" >/dev/null 2>&1 &
+reader=$!
+sleep 0.5
+crash_one_backend 3000000 crashC insert
+wait "$reader" 2>/dev/null; reader_rc=$?
+check "a second concurrent reader was also dropped by the crash" \
+	"$( [ "$reader_rc" -ne 0 ] && echo yes || echo no )" "yes"
+assert_intact "with-reader"
+
+pgc_summary

test/run_all_versions.sh

@@ -90,7 +90,7 @@ trap - EXIT
 
 SRCDIR="${PGC_RUN_SRCDIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}"
 SUITES=(harness_selftest smoke phase2 phase3 phase4 phase5 phase6 audit concurrency unique_conc \
-	differential recovery fuzz fuzz_parquet hardening concurrent_diff parallel sorted_projection \
+	differential recovery native_backend_crash fuzz fuzz_parquet hardening concurrent_diff parallel sorted_projection \
 	arrow_export parquet_export read_stream corruption \
 	generated_columns temporal arrow_import index_only projections arrow_nested parquet_import parquet_nested arrow_nested_import parquet_nested_import native_writer native_roundtrip native_encoding native_zonemap write_minmax_fastpath write_fsst_compressed encode_effort native_skip pushdown_report native_agg native_agg_deletes native_agg_addcolumn native_bloom bloom_setting native_vecskip native_index native_fetch_position native_dml alter_column_type native_ios native_projection native_cluster native_compact native_recluster native_reclaim native_ownership drop_cleanup native_reclaim_cycles native_reclaim_frag native_reclaim_reconcile native_gap native_truncate native_rewrite native_rewrite_conc rewrite_group_scan native_parquet_schema native_read_parquet native_parquet_fdw native_parquet_pushdown native_parquet_hardening native_parquet_stack native_parquet_units native_parquet_flba native_parquet_codecs native_parquet_projection native_parquet_multifile native_parquet_streaming native_parquet_partition native_cancel wal_envelope decode_interrupts import_exclusion import_deferred fk_referencing row_triggers native_lazy_slot native_fetch_cache native_fetch_interrupt analyze_stats analyze_reltuples native_fetch_projection isolation)