From 53791665185e9b7f3fe111c43c7c4778242bc36e Mon Sep 17 00:00:00 2001 From: Greg Burd Date: Mon, 13 Jul 2026 09:08:45 -0400 Subject: [PATCH 01/10] ci: fork upstream auto-sync Keeps gburd/postgres rebased hourly on postgres/postgres master with only .github/ changes on top (sync-upstream automatic + manual). Drops the bespoke Windows dependency-builder workflow: Windows is already built and tested in CI by upstream's pg-ci.yml (Visual Studio + MinGW meson jobs), so a separate dependency prebuild that nothing consumed was redundant. --- .github/.gitignore | 18 + .github/QUICKSTART.md | 378 +++++++++++++++++++++ .github/README.md | 315 +++++++++++++++++ .github/docs/pristine-master-policy.md | 225 ++++++++++++ .github/docs/sync-setup.md | 326 ++++++++++++++++++ .github/workflows/sync-upstream-manual.yml | 249 ++++++++++++++ .github/workflows/sync-upstream.yml | 256 ++++++++++++++ 7 files changed, 1767 insertions(+) create mode 100644 .github/.gitignore create mode 100644 .github/QUICKSTART.md create mode 100644 .github/README.md create mode 100644 .github/docs/pristine-master-policy.md create mode 100644 .github/docs/sync-setup.md create mode 100644 .github/workflows/sync-upstream-manual.yml create mode 100644 .github/workflows/sync-upstream.yml diff --git a/.github/.gitignore b/.github/.gitignore new file mode 100644 index 0000000000000..a447f99442861 --- /dev/null +++ b/.github/.gitignore @@ -0,0 +1,18 @@ +# Node modules +scripts/ai-review/node_modules/ +# Note: package-lock.json should be committed for reproducible CI/CD builds + +# Logs +scripts/ai-review/cost-log-*.json +scripts/ai-review/*.log + +# OS files +.DS_Store +Thumbs.db + +# Editor files +*.swp +*.swo +*~ +.vscode/ +.idea/ diff --git a/.github/QUICKSTART.md b/.github/QUICKSTART.md new file mode 100644 index 0000000000000..d22c4d562ab7d --- /dev/null +++ b/.github/QUICKSTART.md @@ -0,0 +1,378 @@ +# Quick Start Guide - PostgreSQL Mirror CI/CD + +**Goal:** Get your PostgreSQL mirror CI/CD system running in 15 minutes. + +--- + +## āœ… What's Been Implemented + +- **Phase 1: Automated Upstream Sync** - Daily sync from postgres/postgres āœ… +- **Phase 2: AI-Powered Code Review** - Claude-based PR reviews āœ… +- **Phase 3: Windows Builds** - Planned for weeks 4-6 šŸ“‹ + +--- + +## šŸš€ Setup Instructions + +### Step 1: Configure GitHub Actions Permissions (2 minutes) + +1. Go to: **Settings ā†’ Actions ā†’ General** +2. Scroll to: **Workflow permissions** +3. Select: **"Read and write permissions"** +4. Check: **"Allow GitHub Actions to create and approve pull requests"** +5. Click: **Save** + +āœ… This enables workflows to push commits and create issues. + +--- + +### Step 2: Set Up Upstream Sync (3 minutes) + +**Test manual sync first:** + +```bash +# Via GitHub Web UI: +# 1. Go to: Actions tab +# 2. Click: "Sync from Upstream (Manual)" +# 3. Click: "Run workflow" +# 4. Watch it run (should take ~2 minutes) + +# OR via GitHub CLI: +gh workflow run sync-upstream-manual.yml +gh run watch +``` + +**Verify sync worked:** + +```bash +git fetch origin +git log origin/master --oneline -5 + +# Compare with upstream: +# https://github.com/postgres/postgres/commits/master +``` + +**Enable automatic sync:** + +- Automatic sync runs daily at 00:00 UTC +- Already configured, no action needed +- Check: Actions ā†’ "Sync from Upstream (Automatic)" + +āœ… Your master branch will now stay synced automatically. + +--- + +### Step 3: Set Up AI Code Review (10 minutes) + +**Choose Your Provider:** + +You can use either **Anthropic API** (simpler) or **AWS Bedrock** (if you have AWS infrastructure). + +#### Option A: Anthropic API (Recommended for getting started) + +**A. Get Claude API Key:** + +1. Go to: https://console.anthropic.com/ +2. Sign up or log in +3. Navigate to: API Keys +4. Create new key +5. Copy the key (starts with `sk-ant-...`) + +**B. Add API Key to GitHub:** + +1. Go to: **Settings ā†’ Secrets and variables ā†’ Actions** +2. Click: **New repository secret** +3. Name: `ANTHROPIC_API_KEY` +4. Value: Paste your API key +5. Click: **Add secret** + +**C. Ensure config uses Anthropic:** + +Check `.github/scripts/ai-review/config.json` has: +```json +{ + "provider": "anthropic", + ... +} +``` + +#### Option B: AWS Bedrock (If you have AWS) + +See detailed guide: [.github/docs/bedrock-setup.md](.github/docs/bedrock-setup.md) + +**Quick steps:** +1. Enable Claude 3.5 Sonnet in AWS Bedrock console +2. Create IAM user with `bedrock:InvokeModel` permission +3. Add three secrets to GitHub: + - `AWS_ACCESS_KEY_ID` + - `AWS_SECRET_ACCESS_KEY` + - `AWS_REGION` (e.g., `us-east-1`) +4. Update `.github/scripts/ai-review/config.json`: +```json +{ + "provider": "bedrock", + "bedrock_model_id": "us.anthropic.claude-3-5-sonnet-20241022-v2:0", + "bedrock_region": "us-east-1", + ... +} +``` + +**Note:** Both providers have identical pricing ($0.003/1K input, $0.015/1K output tokens). + +--- + +**C. Install Dependencies:** + +```bash +cd .github/scripts/ai-review +npm install + +# Should install: +# - @anthropic-ai/sdk (for Anthropic API) +# - @aws-sdk/client-bedrock-runtime (for AWS Bedrock) +# - @actions/github +# - @actions/core +# - parse-diff +# - minimatch +``` + +**D. Test AI Review:** + +```bash +# Option 1: Create a test PR +git checkout -b test/ai-review +echo "// Test change" >> src/backend/utils/adt/int.c +git add . +git commit -m "Test: AI review" +git push origin test/ai-review +# Create PR via GitHub UI + +# Option 2: Manual trigger on existing PR +gh workflow run ai-code-review.yml -f pr_number= +``` + +āœ… AI will review the PR and post comments + summary. + +--- + +## šŸŽÆ Verify Everything Works + +### Check Sync Status + +```bash +# Check latest sync run +gh run list --workflow=sync-upstream.yml --limit 1 + +# View details +gh run view $(gh run list --workflow=sync-upstream.yml --limit 1 --json databaseId -q '.[0].databaseId') +``` + +**Expected:** āœ… Green checkmark, "Already up to date" or "Successfully synced X commits" + +### Check AI Review Status + +```bash +# Check latest AI review run +gh run list --workflow=ai-code-review.yml --limit 1 + +# View details +gh run view $(gh run list --workflow=ai-code-review.yml --limit 1 --json databaseId -q '.[0].databaseId') +``` + +**Expected:** āœ… Green checkmark, comments posted on PR + +--- + +## šŸ“Š Monitor Costs + +### GitHub Actions Minutes + +```bash +# View usage (requires admin access) +gh api /repos/gburd/postgres/actions/cache/usage + +# Expected monthly usage: +# - Sync: ~150 minutes (FREE - within 2,000 min limit) +# - AI Review: ~200 minutes (FREE - within limit) +``` + +### Claude API Costs + +**View per-PR cost:** +- Check AI review summary comment on PR +- Format: `Cost: $X.XX | Model: claude-3-5-sonnet` + +**Expected costs:** +- Small PR: $0.50 - $1.00 +- Medium PR: $1.00 - $3.00 +- Large PR: $3.00 - $7.50 +- **Monthly (20 PRs):** $35-50 + +**Download detailed logs:** +```bash +gh run list --workflow=ai-code-review.yml --limit 5 +gh run download -n ai-review-cost-log- +``` + +--- + +## šŸ”§ Configuration + +### Adjust Sync Schedule + +Edit `.github/workflows/sync-upstream.yml`: + +```yaml +on: + schedule: + # Current: Daily at 00:00 UTC + - cron: '0 0 * * *' + + # Options: + # Every 6 hours: '0 */6 * * *' + # Twice daily: '0 0,12 * * *' + # Weekdays only: '0 0 * * 1-5' +``` + +### Adjust AI Review Costs + +Edit `.github/scripts/ai-review/config.json`: + +```json +{ + "cost_limits": { + "max_per_pr_dollars": 15.0, // ā† Lower this to save money + "max_per_month_dollars": 200.0, // ā† Hard monthly cap + "alert_threshold_dollars": 150.0 + }, + + "max_file_size_lines": 5000, // ā† Skip files larger than this + + "skip_paths": [ + "*.png", "*.svg", // Already skipped + "vendor/**/*", // ā† Add more patterns here + "generated/**/*" + ] +} +``` + +### Adjust AI Review Prompts + +**Make AI reviews stricter or more lenient:** + +Edit files in `.github/scripts/ai-review/prompts/`: +- `c-code.md` - PostgreSQL C code review +- `sql.md` - SQL and regression tests +- `documentation.md` - Documentation review +- `build-system.md` - Makefile/Meson review + +--- + +## šŸ› Troubleshooting + +### Sync Not Working + +**Problem:** Workflow fails with "Permission denied" + +**Fix:** +- Check: Settings ā†’ Actions ā†’ Workflow permissions +- Ensure: "Read and write permissions" is selected + +--- + +### AI Review Not Posting Comments + +**Problem:** Workflow runs but no comments appear + +**Check:** +1. Is PR a draft? (Draft PRs are skipped to save costs) +2. Are there reviewable files? (Check workflow logs) +3. Is API key valid? (Settings ā†’ Secrets ā†’ ANTHROPIC_API_KEY) + +**Fix:** +- Mark PR as "Ready for review" if draft +- Check workflow logs: Actions ā†’ Latest run ā†’ View logs +- Verify API key at https://console.anthropic.com/ + +--- + +### High AI Review Costs + +**Problem:** Costs higher than expected + +**Check:** +- Download cost logs: `gh run download ` +- Look for large files being reviewed +- Check number of PR updates (each triggers review) + +**Fix:** +1. Add large files to `skip_paths` in config.json +2. Lower `max_tokens_per_request` (shorter reviews) +3. Use draft PRs for work-in-progress +4. Batch PR updates to reduce review frequency + +--- + +## šŸ“š Full Documentation + +- **Overview:** [.github/README.md](.github/README.md) +- **Sync Guide:** [.github/docs/sync-setup.md](.github/docs/sync-setup.md) +- **AI Review Guide:** [.github/docs/ai-review-guide.md](.github/docs/ai-review-guide.md) +- **Windows Builds:** [.github/docs/windows-builds.md](.github/docs/windows-builds.md) (planned) +- **Implementation Status:** [.github/IMPLEMENTATION_STATUS.md](.github/IMPLEMENTATION_STATUS.md) + +--- + +## āœØ What's Next? + +### Immediate +- āœ… **Monitor first automatic sync** (tonight at 00:00 UTC) +- āœ… **Test AI review on real PR** +- āœ… **Tune prompts** based on feedback + +### This Week +- Shadow mode testing for AI reviews (Week 1) +- Gather developer feedback +- Adjust configuration + +### Weeks 2-3 +- Enable full AI review mode +- Monitor costs and quality +- Iterate on prompts + +### Weeks 4-6 +- **Phase 3:** Implement Windows dependency builds +- Research winpgbuild approach +- Create build workflows +- Test artifact publishing + +--- + +## šŸŽ‰ Success Criteria + +You'll know everything is working when: + +āœ… **Sync:** +- Master branch matches postgres/postgres +- Daily sync runs show green checkmarks +- No open issues with label `sync-failure` + +āœ… **AI Review:** +- PRs receive inline comments + summary +- Feedback is relevant and actionable +- Costs stay under $50/month +- Developers find reviews helpful + +āœ… **Overall:** +- Automation saves 8-16 hours/month +- Issues caught earlier in development +- No manual sync needed + +--- + +**Need Help?** +- Check documentation: `.github/README.md` +- Check workflow logs: Actions ā†’ Failed run ā†’ View logs +- Create issue with workflow URL and error messages + +**Ready to go!** šŸš€ diff --git a/.github/README.md b/.github/README.md new file mode 100644 index 0000000000000..bdfcfe74ac4a4 --- /dev/null +++ b/.github/README.md @@ -0,0 +1,315 @@ +# PostgreSQL Mirror CI/CD System + +This directory contains the CI/CD infrastructure for the PostgreSQL personal mirror repository. + +## System Overview + +``` +ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” +ā”‚ PostgreSQL Mirror CI/CD ā”‚ +ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ + ā”‚ + ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”¼ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” + ā”‚ ā”‚ ā”‚ + [1] Sync [2] AI Review [3] Windows + Daily @ 00:00 On PR Events On Master Push + ā”‚ ā”‚ ā”‚ + ā–¼ ā–¼ ā–¼ + postgres/postgres Claude API Dependency Builds + ā”‚ ā”‚ ā”‚ + ā–¼ ā–¼ ā–¼ + github.com/gburd PR Comments Build Artifacts + /postgres/ + Labels (90-day retention) + master +``` + +## Components + +### 1. Automated Upstream Sync +**Status:** āœ“ Implemented +**Files:** `workflows/sync-upstream*.yml` + +Automatically syncs the `master` branch with upstream `postgres/postgres` daily. + +- **Frequency:** Daily at 00:00 UTC +- **Trigger:** Cron schedule + manual +- **Features:** + - Fast-forward merge (conflict-free) + - Automatic issue creation on conflicts + - Issue auto-closure on resolution +- **Cost:** Free (~150 min/month, well within free tier) + +**Documentation:** [docs/sync-setup.md](docs/sync-setup.md) + +### 2. AI-Powered Code Review +**Status:** āœ“ Implemented +**Files:** `workflows/ai-code-review.yml`, `scripts/ai-review/` + +Uses Claude API to provide PostgreSQL-aware code review on pull requests. + +- **Trigger:** PR opened/updated, ready for review +- **Features:** + - PostgreSQL-specific C code review + - SQL, documentation, build system review + - Inline comments on issues + - Automatic labeling (security, performance, etc.) + - Cost tracking and limits + - **Provider Options:** Anthropic API or AWS Bedrock +- **Cost:** $35-50/month (estimated) +- **Model:** Claude 3.5 Sonnet + +**Documentation:** [docs/ai-review-guide.md](docs/ai-review-guide.md) + +### 3. Windows Build Integration +**Status:** āœ… Implemented +**Files:** `workflows/windows-dependencies.yml`, `windows/`, `scripts/windows/` + +Builds PostgreSQL Windows dependencies for x64 Windows. + +- **Trigger:** Manual, manifest changes, weekly refresh +- **Features:** + - Core dependencies: OpenSSL, zlib, libxml2 + - Smart caching by version hash + - Dependency bundling + - Artifact publishing (90-day retention) + - PowerShell download helper + - **Cost optimization:** Skips builds for pristine commits (dev setup, .github/ only) +- **Cost:** ~$5-8/month (with caching and optimization) + +**Documentation:** [docs/windows-builds.md](docs/windows-builds.md) | [Usage](docs/windows-builds-usage.md) + +## Quick Start + +### Prerequisites + +1. **GitHub Actions enabled:** + - Settings ā†’ Actions ā†’ General ā†’ Allow all actions + +2. **Workflow permissions:** + - Settings ā†’ Actions ā†’ General ā†’ Workflow permissions + - Select: "Read and write permissions" + - Enable: "Allow GitHub Actions to create and approve pull requests" + +3. **Secrets configured:** + - **Option A - Anthropic API:** + - Settings ā†’ Secrets and variables ā†’ Actions + - Add: `ANTHROPIC_API_KEY` (get from https://console.anthropic.com/) + - **Option B - AWS Bedrock:** + - Add: `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_REGION` + - See: [docs/bedrock-setup.md](docs/bedrock-setup.md) + +### Using the Sync System + +**Manual sync:** +```bash +# Via GitHub UI: +# Actions ā†’ "Sync from Upstream (Manual)" ā†’ Run workflow + +# Via GitHub CLI: +gh workflow run sync-upstream-manual.yml +``` + +**Check sync status:** +```bash +# Latest sync run +gh run list --workflow=sync-upstream.yml --limit 1 + +# View details +gh run view +``` + +### Using AI Code Review + +AI reviews run automatically on PRs. To test manually: + +```bash +# Via GitHub UI: +# Actions ā†’ "AI Code Review" ā†’ Run workflow ā†’ Enter PR number + +# Via GitHub CLI: +gh workflow run ai-code-review.yml -f pr_number=123 +``` + +**Reviewing AI feedback:** +1. AI posts inline comments on specific lines +2. AI posts summary comment with overview +3. AI adds labels (security-concern, needs-tests, etc.) +4. Review and address feedback like human reviewer comments + +### Cost Monitoring + +**View AI review costs:** +```bash +# Download cost logs +gh run download -n ai-review-cost-log- +``` + +**Expected monthly costs (with optimizations):** +- Sync: $0 (free tier) +- AI Review: $30-45 (only on PRs, skips drafts) +- Windows Builds: $5-8 (caching + pristine commit skipping) +- **Total: $35-53/month** + +**Cost optimizations:** +- Windows builds skip "dev setup" and .github/-only commits +- AI review only runs on non-draft PRs +- Aggressive caching reduces build times by 80-90% +- See [Cost Optimization Guide](docs/cost-optimization.md) for details + +## Workflow Files + +### Sync Workflows +- `workflows/sync-upstream.yml` - Automatic daily sync +- `workflows/sync-upstream-manual.yml` - Manual testing sync + +### AI Review Workflows +- `workflows/ai-code-review.yml` - Automatic PR review + +### Windows Build Workflows +- `workflows/windows-dependencies.yml` - Dependency builds (TBD) + +## Configuration Files + +### AI Review Configuration +- `scripts/ai-review/config.json` - Cost limits, file patterns, labels +- `scripts/ai-review/prompts/*.md` - Review prompts by file type +- `scripts/ai-review/package.json` - Node.js dependencies + +### Windows Build Configuration +- `windows/manifest.json` - Dependency versions (TBD) + +## Branch Strategy + +### Master Branch: Mirror Only +- **Purpose:** Pristine copy of `postgres/postgres` +- **Rule:** Never commit directly to master +- **Sync:** Automatic via GitHub Actions +- **Protection:** Consider branch protection rules + +### Feature Branches: Development +- **Pattern:** `feature/*`, `dev/*`, `experiment/*` +- **Workflow:** + ```bash + git checkout master + git pull origin master + git checkout -b feature/my-feature + # ... make changes ... + git push origin feature/my-feature + # Create PR: feature/my-feature ā†’ master + ``` + +### Special Branches +- `recovery/*` - Temporary branches for sync conflict resolution +- Development remotes: commitfest, heikki, orioledb, zheap + +## Integration with Cirrus CI + +GitHub Actions and Cirrus CI run independently: + +- **Cirrus CI:** Comprehensive testing (Linux, FreeBSD, macOS, Windows) +- **GitHub Actions:** Sync, AI review, Windows dependency builds +- **No conflicts:** Both can run on same commits + +## Troubleshooting + +### Sync Issues + +**Problem:** Sync workflow failing +**Check:** Actions ā†’ "Sync from Upstream (Automatic)" ā†’ Latest run +**Fix:** See [docs/sync-setup.md](docs/sync-setup.md#sync-failure-recovery) + +### AI Review Issues + +**Problem:** AI review not running +**Check:** Is PR a draft? Draft PRs are skipped +**Fix:** Mark PR as ready for review + +**Problem:** AI review too expensive +**Check:** Cost logs in workflow artifacts +**Fix:** Adjust limits in `scripts/ai-review/config.json` + +### Workflow Permission Issues + +**Problem:** "Resource not accessible by integration" +**Check:** Settings ā†’ Actions ā†’ General ā†’ Workflow permissions +**Fix:** Enable "Read and write permissions" + +## Security + +### Secrets Management +- `ANTHROPIC_API_KEY`: Claude API key (required for AI review) +- `GITHUB_TOKEN`: Auto-generated, scoped to repository +- Never commit secrets to repository +- Rotate API keys quarterly + +### Permissions +- Workflows use minimum necessary permissions +- `contents: read` for code access +- `pull-requests: write` for comments +- `issues: write` for sync failure issues + +### Audit Trail +- All workflow runs logged (90-day retention) +- Cost tracking for AI reviews +- GitHub Actions audit log available + +## Support and Documentation + +### Detailed Documentation +- [Sync Setup Guide](docs/sync-setup.md) - Upstream sync system +- [AI Review Guide](docs/ai-review-guide.md) - AI code review system +- [Windows Builds Guide](docs/windows-builds.md) - Windows dependencies +- [Cost Optimization Guide](docs/cost-optimization.md) - Reducing CI/CD costs +- [Pristine Master Policy](docs/pristine-master-policy.md) - Master branch management + +### Reporting Issues + +Issues with CI/CD system: +1. Check workflow logs: Actions ā†’ Failed run ā†’ View logs +2. Search existing issues: label:automation +3. Create issue with workflow run URL and error messages + +### Modifying Workflows + +**Disabling a workflow:** +```bash +# Via GitHub UI: +# Actions ā†’ Select workflow ā†’ "..." ā†’ Disable workflow + +# Via git: +git mv .github/workflows/workflow-name.yml .github/workflows/workflow-name.yml.disabled +git commit -m "Disable workflow" +``` + +**Testing workflow changes:** +1. Create feature branch +2. Modify workflow file +3. Use `workflow_dispatch` trigger to test +4. Verify in Actions tab +5. Merge to master when working + +## Cost Summary + +| Component | Monthly Cost | Usage | Notes | +|-----------|-------------|-------|-------| +| Sync | $0 | ~150 min | Free tier: 2,000 min | +| AI Review | $30-45 | Variable | Claude API usage-based | +| Windows Builds | $5-8 | ~2,500 min | With caching + optimization | +| **Total** | **$35-53** | | After cost optimizations | + +**Comparison:** CodeRabbit (turnkey solution) = $99-499/month + +**Cost savings:** ~40-47% reduction through optimizations (see [Cost Optimization Guide](docs/cost-optimization.md)) + +## References + +- PostgreSQL: https://github.com/postgres/postgres +- GitHub Actions: https://docs.github.com/en/actions +- Claude API: https://docs.anthropic.com/ +- Cirrus CI: https://cirrus-ci.org/ +- winpgbuild: https://github.com/dpage/winpgbuild + +--- + +**Last Updated:** 2026-03-10 +**Maintained by:** PostgreSQL Mirror Automation diff --git a/.github/docs/pristine-master-policy.md b/.github/docs/pristine-master-policy.md new file mode 100644 index 0000000000000..9c0479d32df6a --- /dev/null +++ b/.github/docs/pristine-master-policy.md @@ -0,0 +1,225 @@ +# Pristine Master Policy + +## Overview + +The `master` branch in this mirror repository follows a "mostly pristine" policy, meaning it should closely mirror the upstream `postgres/postgres` repository with only specific exceptions allowed. + +## Allowed Commits on Master + +Master is considered "pristine" and the sync workflow will successfully merge upstream changes if local commits fall into these categories: + +### 1. āœ… CI/CD Configuration (`.github/` directory only) + +Commits that only modify files within the `.github/` directory are allowed. + +**Examples:** +- Adding GitHub Actions workflows +- Updating AI review configuration +- Modifying sync schedules +- Adding documentation in `.github/docs/` + +**Rationale:** CI/CD configuration is repository-specific and doesn't affect the PostgreSQL codebase itself. + +### 2. āœ… Development Environment Setup (commits named "dev setup ...") + +Commits with messages starting with "dev setup" (case-insensitive) are allowed, even if they modify files outside `.github/`. + +**Examples:** +- `dev setup v19` +- `Dev Setup: Add debugging configuration` +- `DEV SETUP - IDE and tooling` + +**Typical files in dev setup commits:** +- `.clang-format`, `.clangd` - Code formatting and LSP config +- `.envrc` - Directory environment variables (direnv) +- `.gdbinit` - Debugger configuration +- `.idea/`, `.vscode/` - IDE settings +- `flake.nix`, `shell.nix` - Nix development environment +- `pg-aliases.sh` - Personal shell aliases +- Other personal development tools + +**Rationale:** Development environment configuration is personal and doesn't affect the code or CI/CD. It's frequently updated as developers refine their workflow. + +### 3. āŒ Code Changes (NOT allowed) + +Any commits that: +- Modify PostgreSQL source code (`src/`, `contrib/`, etc.) +- Modify tests outside `.github/` +- Modify build system outside `.github/` +- Are not `.github/`-only AND don't start with "dev setup" + +**These will cause sync failures** and require manual resolution. + +## Branch Strategy + +### Master Branch +- **Purpose:** Mirror of upstream `postgres/postgres` + local CI/CD + dev environment +- **Updates:** Automatic hourly sync from upstream +- **Direct commits:** Only `.github/` changes or "dev setup" commits +- **All other work:** Use feature branches + +### Feature Branches +- **Purpose:** All PostgreSQL development work +- **Pattern:** `feature/*`, `dev/*`, `experiment/*` +- **Workflow:** + ```bash + git checkout master + git pull origin master + git checkout -b feature/my-feature + # Make changes... + git push origin feature/my-feature + # Create PR: feature/my-feature ā†’ master + ``` + +## Sync Workflow Behavior + +### Scenario 1: No Local Commits +``` +Upstream: A---B---C +Master: A---B---C +``` +**Result:** āœ… Already up to date (no action needed) + +### Scenario 2: Only .github/ Commits +``` +Upstream: A---B---C---D +Master: A---B---C---X (X modifies .github/ only) +``` +**Result:** āœ… Merge commit created +``` +Master: A---B---C---X---M + \ / + D---/ +``` + +### Scenario 3: Only "dev setup" Commits +``` +Upstream: A---B---C---D +Master: A---B---C---Y (Y is "dev setup v19") +``` +**Result:** āœ… Merge commit created +``` +Master: A---B---C---Y---M + \ / + D---/ +``` + +### Scenario 4: Mix of Allowed Commits +``` +Upstream: A---B---C---D +Master: A---B---C---X---Y (X=.github/, Y=dev setup) +``` +**Result:** āœ… Merge commit created + +### Scenario 5: Code Changes (Violation) +``` +Upstream: A---B---C---D +Master: A---B---C---Z (Z modifies src/backend/) +``` +**Result:** āŒ Sync fails, issue created + +**Recovery:** +1. Create feature branch from Z +2. Reset master to match upstream +3. Rebase feature branch +4. Create PR + +## Updating Dev Setup + +When you update your development environment: + +```bash +# Make changes to .clangd, flake.nix, etc. +git add .clangd flake.nix .vscode/ + +# Important: Start message with "dev setup" +git commit -m "dev setup v20: Update clangd config and add new aliases" + +git push origin master +``` + +The sync workflow will recognize this as a dev setup commit and preserve it during merges. + +**Naming convention:** +- āœ… `dev setup v20` +- āœ… `Dev setup: Update IDE config` +- āœ… `DEV SETUP - Add debugging tools` +- āŒ `Update development environment` (doesn't start with "dev setup") +- āŒ `dev environment changes` (doesn't start with "dev setup") + +## Sync Failure Recovery + +If sync fails because of non-allowed commits: + +### Check What's Wrong +```bash +git fetch origin +git fetch upstream https://github.com/postgres/postgres.git master + +# See which commits are problematic +git log upstream/master..origin/master --oneline + +# See which files were changed +git diff --name-only upstream/master...origin/master +``` + +### Option 1: Make Commit Acceptable + +If the commit should have been a "dev setup" commit: + +```bash +# Amend the commit message +git commit --amend -m "dev setup v21: Previous changes" +git push origin master --force-with-lease +``` + +### Option 2: Move to Feature Branch + +If the commit contains code changes: + +```bash +# Create feature branch +git checkout -b feature/recovery origin/master + +# Reset master to upstream +git checkout master +git reset --hard upstream/master +git push origin master --force + +# Your changes are safe in feature/recovery +git checkout feature/recovery +# Create PR when ready +``` + +## FAQ + +**Q: Why allow dev setup commits on master?** +A: Development environment configuration is personal, frequently updated, and doesn't affect the codebase or CI/CD. It's more convenient to keep it on master than manage separate branches. + +**Q: What if I forget to name it "dev setup"?** +A: Sync will fail. You can amend the commit message (see recovery above) or move the commit to a feature branch. + +**Q: Can I have both .github/ and dev setup changes in one commit?** +A: Yes! The sync workflow allows commits that modify .github/, or are named "dev setup", or both. + +**Q: What if upstream modifies the same files as my dev setup commit?** +A: The sync will attempt to merge automatically. If there are conflicts, you'll need to resolve them manually (rare, since upstream shouldn't touch personal dev files). + +**Q: Can I reorder commits on master?** +A: It's not recommended due to complexity. The sync workflow handles commits in any order as long as they follow the policy. + +## Monitoring + +**Check sync status:** +- Actions ā†’ "Sync from Upstream (Automatic)" +- Look for green āœ… on recent runs + +**Check for policy violations:** +- Open issues with label `sync-failure` +- These indicate commits that violated the pristine master policy + +## Related Documentation + +- [Sync Setup Guide](sync-setup.md) - Detailed sync workflow documentation +- [QUICKSTART](../QUICKSTART.md) - Quick setup guide +- [README](../README.md) - System overview diff --git a/.github/docs/sync-setup.md b/.github/docs/sync-setup.md new file mode 100644 index 0000000000000..1e12aeea3c5fc --- /dev/null +++ b/.github/docs/sync-setup.md @@ -0,0 +1,326 @@ +# Automated Upstream Sync Documentation + +## Overview + +This repository maintains a mirror of the official PostgreSQL repository at `postgres/postgres`. The sync system automatically keeps the `master` branch synchronized with upstream changes. + +## System Components + +### 1. Automatic Daily Sync +**File:** `.github/workflows/sync-upstream.yml` + +- **Trigger:** Daily at 00:00 UTC (cron schedule) +- **Purpose:** Automatically sync master branch without manual intervention +- **Process:** + 1. Fetches latest commits from `postgres/postgres` + 2. Fast-forward merges to local master (conflict-free) + 3. Pushes to `origin/master` + 4. Creates GitHub issue if conflicts detected + 5. Closes existing sync-failure issues on success + +### 2. Manual Sync Workflow +**File:** `.github/workflows/sync-upstream-manual.yml` + +- **Trigger:** Manual via Actions tab ā†’ "Sync from Upstream (Manual)" ā†’ Run workflow +- **Purpose:** Testing and on-demand syncs +- **Options:** + - `force_push`: Use `--force-with-lease` when pushing (default: true) + +## Branch Strategy + +### Critical Rule: Master is Pristine + +- **master branch:** Mirror only - pristine copy of `postgres/postgres` +- **All development:** Feature branches (e.g., `feature/hot-updates`, `experiment/zheap`) +- **Never commit directly to master** - this will cause sync failures + +### Feature Branch Workflow + +```bash +# Start new feature from latest master +git checkout master +git pull origin master +git checkout -b feature/my-feature + +# Work on feature +git commit -m "Add feature" + +# Keep feature updated with upstream +git checkout master +git pull origin master +git checkout feature/my-feature +git rebase master + +# Push feature branch +git push origin feature/my-feature + +# Create PR: feature/my-feature ā†’ master +``` + +## Sync Failure Recovery + +### Diagnosis + +If sync fails, you'll receive a GitHub issue with label `sync-failure`. Check what commits are on master but not upstream: + +```bash +# Clone or update your local repository +git fetch origin +git fetch upstream https://github.com/postgres/postgres.git master + +# View conflicting commits +git log upstream/master..origin/master --oneline + +# See detailed changes +git diff upstream/master...origin/master +``` + +### Recovery Option 1: Preserve Commits (Recommended) + +If the commits on master should be kept: + +```bash +# Create backup branch from current master +git checkout origin/master +git checkout -b recovery/master-backup-$(date +%Y%m%d) +git push origin recovery/master-backup-$(date +%Y%m%d) + +# Reset master to upstream +git checkout master +git reset --hard upstream/master +git push origin master --force + +# Create feature branch from backup +git checkout -b feature/recovered-work recovery/master-backup-$(date +%Y%m%d) + +# Optional: rebase onto new master +git rebase master + +# Push feature branch +git push origin feature/recovered-work + +# Create PR: feature/recovered-work ā†’ master +``` + +### Recovery Option 2: Discard Commits + +If the commits on master were mistakes or already merged upstream: + +```bash +git checkout master +git reset --hard upstream/master +git push origin master --force +``` + +### Verification + +After recovery, verify sync status: + +```bash +# Check that master matches upstream +git log origin/master --oneline -10 +git log upstream/master --oneline -10 + +# These should be identical + +# Or run manual sync workflow +# GitHub ā†’ Actions ā†’ "Sync from Upstream (Manual)" ā†’ Run workflow +``` + +The automatic sync will resume on next scheduled run (00:00 UTC daily). + +## Monitoring + +### Success Indicators + +- āœ“ GitHub Actions badge shows passing +- āœ“ No open issues with label `sync-failure` +- āœ“ `master` branch commit history matches `postgres/postgres` + +### Check Sync Status + +**Via GitHub UI:** +1. Go to: Actions ā†’ "Sync from Upstream (Automatic)" +2. Check latest run status + +**Via Git:** +```bash +git fetch origin +git fetch upstream https://github.com/postgres/postgres.git master +git log origin/master..upstream/master --oneline + +# No output = fully synced +# Commits listed = behind upstream (sync pending or failed) +``` + +**Via API:** +```bash +# Check latest workflow run +gh run list --workflow=sync-upstream.yml --limit 1 + +# View run details +gh run view +``` + +### Sync Lag + +Expected lag: <1 hour from upstream commit to mirror + +- Upstream commits at 12:30 UTC ā†’ Synced at next daily run (00:00 UTC next day) = ~11.5 hours max +- For faster sync: Manually trigger workflow after major upstream merges + +## Configuration + +### GitHub Actions Permissions + +Required settings (already configured): + +1. **Settings ā†’ Actions ā†’ General ā†’ Workflow permissions:** + - āœ“ "Read and write permissions" + - āœ“ "Allow GitHub Actions to create and approve pull requests" + +2. **Repository Settings ā†’ Branches:** + - Consider: Branch protection rule on `master` to prevent direct pushes + - Exception: Allow `github-actions[bot]` to push + +### Adjusting Sync Schedule + +Edit `.github/workflows/sync-upstream.yml`: + +```yaml +on: + schedule: + # Current: Daily at 00:00 UTC + - cron: '0 0 * * *' + + # Examples: + # Every 6 hours: '0 */6 * * *' + # Twice daily: '0 0,12 * * *' + # Weekdays only: '0 0 * * 1-5' +``` + +**Recommendation:** Keep daily schedule to balance freshness with API usage. + +## Troubleshooting + +### Issue: Workflow not running + +**Check:** +1. Actions tab ā†’ Check if workflow is disabled +2. Settings ā†’ Actions ā†’ Ensure workflows are enabled for repository + +**Fix:** +- Enable workflow: Actions ā†’ Select workflow ā†’ "Enable workflow" + +### Issue: Permission denied on push + +**Check:** +- Settings ā†’ Actions ā†’ General ā†’ Workflow permissions + +**Fix:** +- Set to "Read and write permissions" +- Enable "Allow GitHub Actions to create and approve pull requests" + +### Issue: Merge conflicts every sync + +**Root cause:** Commits being made directly to master + +**Fix:** +1. Review `.git/hooks/` for pre-commit hooks that might auto-commit +2. Check if any automation is committing to master +3. Enforce branch protection rules +4. Educate team members on feature branch workflow + +### Issue: Sync successful but CI fails + +**This is expected** if upstream introduced breaking changes or test failures. + +**Handling:** +- Upstream tests failures are upstream's responsibility +- Focus: Ensure mirror stays in sync +- Separate: Your feature branches should pass CI + +## Cost and Usage + +### GitHub Actions Minutes + +- **Sync workflow:** ~2-3 minutes per run +- **Frequency:** Daily = 60-90 minutes/month +- **Free tier:** 2,000 minutes/month (public repos: unlimited) +- **Cost:** $0 (well within limits) + +### Network Usage + +- Fetches only new commits (incremental) +- Typical: <10 MB per sync +- Total: <300 MB/month + +## Security Considerations + +### Secrets + +- Uses `GITHUB_TOKEN` (automatically provided, scoped to repository) +- No additional secrets required +- Token permissions: Minimum necessary (contents:write, issues:write) + +### Audit Trail + +All syncs are logged: +- GitHub Actions run history (90 days retention) +- Git reflog on server +- Issue creation/closure for failures + +## Integration with Other Workflows + +### Cirrus CI + +Cirrus CI tests trigger on pushes to master: +- Sync pushes ā†’ Cirrus CI runs tests on synced commits +- This validates upstream changes against your test matrix + +### AI Code Review + +AI review workflows trigger on PRs, not master pushes: +- Sync to master does NOT trigger AI reviews +- Feature branch PRs ā†’ master do trigger AI reviews + +### Windows Builds + +Windows dependency builds trigger on master pushes: +- Sync pushes ā†’ Windows builds run +- Ensures dependencies stay compatible with latest upstream + +## Support + +### Reporting Issues + +If sync consistently fails: + +1. Check open issues with label `sync-failure` +2. Review workflow logs: Actions ā†’ Failed run ā†’ View logs +3. Create issue with: + - Workflow run URL + - Error messages from logs + - Output of `git log upstream/master..origin/master` + +### Disabling Automatic Sync + +If needed (e.g., during major refactoring): + +```bash +# Disable via GitHub UI +# Actions ā†’ "Sync from Upstream (Automatic)" ā†’ "..." ā†’ Disable workflow + +# Or delete/rename the workflow file +git mv .github/workflows/sync-upstream.yml .github/workflows/sync-upstream.yml.disabled +git commit -m "Temporarily disable automatic sync" +git push +``` + +**Remember to re-enable** once work is complete. + +## References + +- Upstream repository: https://github.com/postgres/postgres +- GitHub Actions docs: https://docs.github.com/en/actions +- Git branching strategies: https://git-scm.com/book/en/v2/Git-Branching-Branching-Workflows diff --git a/.github/workflows/sync-upstream-manual.yml b/.github/workflows/sync-upstream-manual.yml new file mode 100644 index 0000000000000..362c119a128e7 --- /dev/null +++ b/.github/workflows/sync-upstream-manual.yml @@ -0,0 +1,249 @@ +name: Sync from Upstream (Manual) + +on: + workflow_dispatch: + inputs: + force_push: + description: 'Use --force-with-lease when pushing' + required: false + type: boolean + default: true + +jobs: + sync: + runs-on: ubuntu-latest + permissions: + contents: write + issues: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Configure Git + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + + - name: Add upstream remote + run: | + git remote add upstream https://github.com/postgres/postgres.git || true + git remote -v + + - name: Fetch upstream + run: | + echo "Fetching from upstream postgres/postgres..." + git fetch upstream master + echo "Current local master:" + git log origin/master --oneline -5 + echo "Upstream master:" + git log upstream/master --oneline -5 + + - name: Check for local commits + id: check_commits + run: | + git checkout master + LOCAL_COMMITS=$(git rev-list origin/master..upstream/master --count) + DIVERGED=$(git rev-list upstream/master..origin/master --count) + echo "commits_behind=$LOCAL_COMMITS" >> $GITHUB_OUTPUT + echo "commits_ahead=$DIVERGED" >> $GITHUB_OUTPUT + echo "Mirror is $DIVERGED commits ahead and $LOCAL_COMMITS commits behind upstream" + + if [ "$DIVERGED" -gt 0 ]; then + # Check commit messages for "dev setup" or "dev v" pattern + DEV_SETUP_COMMITS=$(git log --format=%s upstream/master...origin/master | grep -iE "^dev (setup|v[0-9])" | wc -l) + echo "dev_setup_commits=$DEV_SETUP_COMMITS" >> $GITHUB_OUTPUT + + # Check if diverged commits only touch .github/ directory + NON_GITHUB_CHANGES=$(git diff --name-only upstream/master...origin/master | grep -v "^\.github/" | wc -l) + echo "non_github_changes=$NON_GITHUB_CHANGES" >> $GITHUB_OUTPUT + + if [ "$NON_GITHUB_CHANGES" -eq 0 ]; then + echo "āœ“ All local commits are CI/CD configuration (.github/ only)" + elif [ "$DEV_SETUP_COMMITS" -gt 0 ]; then + echo "āœ“ Found $DEV_SETUP_COMMITS 'dev setup/version' commit(s)" + else + echo "āš ļø WARNING: Local commits modify files outside .github/ and are not 'dev setup/version' commits!" + git diff --name-only upstream/master...origin/master | grep -v "^\.github/" || true + fi + else + echo "non_github_changes=0" >> $GITHUB_OUTPUT + echo "dev_setup_commits=0" >> $GITHUB_OUTPUT + fi + + - name: Attempt merge + id: merge + run: | + COMMITS_AHEAD=${{ steps.check_commits.outputs.commits_ahead }} + COMMITS_BEHIND=${{ steps.check_commits.outputs.commits_behind }} + NON_GITHUB_CHANGES=${{ steps.check_commits.outputs.non_github_changes }} + DEV_SETUP_COMMITS=${{ steps.check_commits.outputs.dev_setup_commits }} + + # Check if there are problematic local commits + # Allow commits if: + # 1. Only .github/ changes (CI/CD config) + # 2. Has "dev setup/version" commits (personal development environment) + if [ "$COMMITS_AHEAD" -gt 0 ] && [ "$NON_GITHUB_CHANGES" -gt 0 ]; then + if [ "$DEV_SETUP_COMMITS" -eq 0 ]; then + echo "āŒ Local master has commits outside .github/ that are not 'dev setup/version' commits!" + echo "merge_status=conflict" >> $GITHUB_OUTPUT + exit 1 + else + echo "āœ“ Non-.github/ changes are from 'dev setup/version' commits - allowed" + fi + fi + + # Already up to date + if [ "$COMMITS_BEHIND" -eq 0 ]; then + echo "āœ“ Already up to date with upstream" + echo "merge_status=uptodate" >> $GITHUB_OUTPUT + exit 0 + fi + + # Try fast-forward first (clean case) + if [ "$COMMITS_AHEAD" -eq 0 ]; then + echo "Fast-forwarding to upstream (no local commits)..." + git merge --ff-only upstream/master + echo "merge_status=success" >> $GITHUB_OUTPUT + exit 0 + fi + + # Local commits exist (.github/ and/or dev setup/version) - rebase onto upstream + if [ "$DEV_SETUP_COMMITS" -gt 0 ]; then + echo "Rebasing local CI/CD and dev setup/version commits onto upstream..." + else + echo "Rebasing local CI/CD commits (.github/ only) onto upstream..." + fi + + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + + if git rebase upstream/master; then + echo "āœ“ Successfully rebased local commits onto upstream" + echo "merge_status=success" >> $GITHUB_OUTPUT + else + echo "āŒ Rebase conflict occurred" + echo "merge_status=conflict" >> $GITHUB_OUTPUT + + # Abort the failed rebase to clean up state + git rebase --abort + exit 1 + fi + continue-on-error: true + + - name: Push to origin + if: steps.merge.outputs.merge_status == 'success' + run: | + if [ "${{ inputs.force_push }}" == "true" ]; then + git push origin master --force-with-lease + else + git push origin master + fi + echo "āœ“ Successfully synced master with upstream" + + - name: Create issue on failure + if: steps.merge.outputs.merge_status == 'conflict' + uses: actions/github-script@v7 + with: + script: | + const title = 'šŸšØ Upstream Sync Failed - Manual Intervention Required'; + const body = `## Sync Failure Report + + The automated sync from \`postgres/postgres\` failed due to conflicting commits. + + **Details:** + - Local master has ${{ steps.check_commits.outputs.commits_ahead }} commit(s) not in upstream + - Upstream has ${{ steps.check_commits.outputs.commits_behind }} new commit(s) + - Non-.github/ changes: ${{ steps.check_commits.outputs.non_github_changes }} files + + **This indicates commits were made directly to master outside .github/**, which violates the pristine mirror policy. + + **Note:** Commits to .github/ (CI/CD configuration) are allowed and will be preserved during sync. + + ### Resolution Steps: + + 1. Identify the conflicting commits: + \`\`\`bash + git fetch origin + git fetch upstream https://github.com/postgres/postgres.git master + git log upstream/master..origin/master + \`\`\` + + 2. If these commits should be preserved: + - Create a feature branch: \`git checkout -b recovery/master-commits origin/master\` + - Reset master: \`git checkout master && git reset --hard upstream/master\` + - Push: \`git push origin master --force\` + - Cherry-pick or rebase the feature branch + + 3. If these commits should be discarded: + - Reset master: \`git checkout master && git reset --hard upstream/master\` + - Push: \`git push origin master --force\` + + 4. Close this issue once resolved + + **Workflow run:** ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + `; + + // Check if issue already exists + const issues = await github.rest.issues.listForRepo({ + owner: context.repo.owner, + repo: context.repo.repo, + state: 'open', + labels: 'sync-failure' + }); + + if (issues.data.length === 0) { + await github.rest.issues.create({ + owner: context.repo.owner, + repo: context.repo.repo, + title: title, + body: body, + labels: ['sync-failure', 'automation'] + }); + } + + - name: Close existing sync-failure issues + if: steps.merge.outputs.merge_status == 'success' + uses: actions/github-script@v7 + with: + script: | + const issues = await github.rest.issues.listForRepo({ + owner: context.repo.owner, + repo: context.repo.repo, + state: 'open', + labels: 'sync-failure' + }); + + for (const issue of issues.data) { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issue.number, + body: 'āœ“ Sync successful - closing this issue automatically.' + }); + + await github.rest.issues.update({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issue.number, + state: 'closed' + }); + } + + - name: Summary + if: always() + run: | + echo "### Sync Summary" >> $GITHUB_STEP_SUMMARY + echo "- **Status:** ${{ steps.merge.outputs.merge_status }}" >> $GITHUB_STEP_SUMMARY + echo "- **Commits behind:** ${{ steps.check_commits.outputs.commits_behind }}" >> $GITHUB_STEP_SUMMARY + echo "- **Commits ahead:** ${{ steps.check_commits.outputs.commits_ahead }}" >> $GITHUB_STEP_SUMMARY + if [ "${{ steps.merge.outputs.merge_status }}" == "success" ]; then + echo "- **Result:** āœ“ Successfully synced with upstream" >> $GITHUB_STEP_SUMMARY + elif [ "${{ steps.merge.outputs.merge_status }}" == "uptodate" ]; then + echo "- **Result:** āœ“ Already up to date" >> $GITHUB_STEP_SUMMARY + else + echo "- **Result:** āš ļø Sync failed - manual intervention required" >> $GITHUB_STEP_SUMMARY + fi diff --git a/.github/workflows/sync-upstream.yml b/.github/workflows/sync-upstream.yml new file mode 100644 index 0000000000000..b3a6466980b0d --- /dev/null +++ b/.github/workflows/sync-upstream.yml @@ -0,0 +1,256 @@ +name: Sync from Upstream (Automatic) + +on: + schedule: + # Run hourly every day + - cron: '0 * * * *' + workflow_dispatch: + +jobs: + sync: + runs-on: ubuntu-latest + permissions: + contents: write + issues: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Configure Git + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + + - name: Add upstream remote + run: | + git remote add upstream https://github.com/postgres/postgres.git || true + git remote -v + + - name: Fetch upstream + run: | + echo "Fetching from upstream postgres/postgres..." + git fetch upstream master + + - name: Check for local commits + id: check_commits + run: | + git checkout master + LOCAL_COMMITS=$(git rev-list origin/master..upstream/master --count) + DIVERGED=$(git rev-list upstream/master..origin/master --count) + echo "commits_behind=$LOCAL_COMMITS" >> $GITHUB_OUTPUT + echo "commits_ahead=$DIVERGED" >> $GITHUB_OUTPUT + + if [ "$LOCAL_COMMITS" -eq 0 ]; then + echo "āœ“ Already up to date with upstream" + else + echo "Mirror is $LOCAL_COMMITS commits behind upstream" + fi + + if [ "$DIVERGED" -gt 0 ]; then + echo "āš ļø Local master has $DIVERGED commits not in upstream" + + # Check commit messages for "dev setup" or "dev v" pattern + DEV_SETUP_COMMITS=$(git log --format=%s upstream/master..origin/master | grep -iE "^dev (setup|v[0-9])" | wc -l) + echo "dev_setup_commits=$DEV_SETUP_COMMITS" >> $GITHUB_OUTPUT + + # Check if diverged commits only touch .github/ directory + NON_GITHUB_CHANGES=$(git diff --name-only upstream/master...origin/master | grep -v "^\.github/" | wc -l) + echo "non_github_changes=$NON_GITHUB_CHANGES" >> $GITHUB_OUTPUT + + if [ "$NON_GITHUB_CHANGES" -eq 0 ]; then + echo "āœ“ All local commits are CI/CD configuration (.github/ only) - will merge" + elif [ "$DEV_SETUP_COMMITS" -gt 0 ]; then + echo "āœ“ Found $DEV_SETUP_COMMITS 'dev setup/version' commit(s)" + else + echo "āš ļø WARNING: Local commits modify files outside .github/ and are not 'dev setup/version' commits!" + git diff --name-only upstream/master...origin/master | grep -v "^\.github/" || true + echo "Non-dev commits:" + git log --format=" %h %s" upstream/master..origin/master | grep -ivE "^ [a-f0-9]* dev (setup|v[0-9])" || true + fi + else + echo "non_github_changes=0" >> $GITHUB_OUTPUT + echo "dev_setup_commits=0" >> $GITHUB_OUTPUT + fi + + - name: Attempt merge + id: merge + run: | + COMMITS_AHEAD=${{ steps.check_commits.outputs.commits_ahead }} + COMMITS_BEHIND=${{ steps.check_commits.outputs.commits_behind }} + NON_GITHUB_CHANGES=${{ steps.check_commits.outputs.non_github_changes }} + DEV_SETUP_COMMITS=${{ steps.check_commits.outputs.dev_setup_commits }} + + # Check if there are problematic local commits + # Allow commits if: + # 1. Only .github/ changes (CI/CD config) + # 2. Has "dev setup/version" commits (personal development environment) + if [ "$COMMITS_AHEAD" -gt 0 ] && [ "$NON_GITHUB_CHANGES" -gt 0 ]; then + if [ "$DEV_SETUP_COMMITS" -eq 0 ]; then + echo "āŒ Local master has commits outside .github/ that are not 'dev setup/version' commits!" + echo "merge_status=conflict" >> $GITHUB_OUTPUT + exit 1 + else + echo "āœ“ Non-.github/ changes are from 'dev setup/version' commits - allowed" + fi + fi + + # Already up to date + if [ "$COMMITS_BEHIND" -eq 0 ]; then + echo "āœ“ Already up to date with upstream" + echo "merge_status=uptodate" >> $GITHUB_OUTPUT + exit 0 + fi + + # Try fast-forward first (clean case) + if [ "$COMMITS_AHEAD" -eq 0 ]; then + echo "Fast-forwarding to upstream (no local commits)..." + git merge --ff-only upstream/master + echo "merge_status=success" >> $GITHUB_OUTPUT + exit 0 + fi + + # Local commits exist (.github/ and/or dev setup/version) - rebase onto upstream + if [ "$DEV_SETUP_COMMITS" -gt 0 ]; then + echo "Rebasing local CI/CD and dev setup/version commits onto upstream..." + else + echo "Rebasing local CI/CD commits (.github/ only) onto upstream..." + fi + + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + + if git rebase upstream/master; then + echo "āœ“ Successfully rebased local commits onto upstream" + echo "merge_status=success" >> $GITHUB_OUTPUT + else + echo "āŒ Rebase conflict occurred" + echo "merge_status=conflict" >> $GITHUB_OUTPUT + + # Abort the failed rebase to clean up state + git rebase --abort + exit 1 + fi + continue-on-error: true + + - name: Push to origin + if: steps.merge.outputs.merge_status == 'success' + run: | + git push origin master --force-with-lease + + COMMITS_SYNCED="${{ steps.check_commits.outputs.commits_behind }}" + echo "āœ“ Successfully synced $COMMITS_SYNCED commits from upstream" + + - name: Create issue on failure + if: steps.merge.outputs.merge_status == 'conflict' + uses: actions/github-script@v7 + with: + script: | + const title = 'šŸšØ Automated Upstream Sync Failed'; + const body = `## Automatic Sync Failure + + The daily sync from \`postgres/postgres\` failed. + + **Details:** + - Local master has ${{ steps.check_commits.outputs.commits_ahead }} commit(s) not in upstream + - Upstream has ${{ steps.check_commits.outputs.commits_behind }} new commit(s) + - Non-.github/ changes: ${{ steps.check_commits.outputs.non_github_changes }} files + - **Run date:** ${new Date().toISOString()} + + **Root cause:** Commits were made directly to master outside of .github/, which violates the pristine mirror policy. + + **Note:** Commits to .github/ (CI/CD configuration) are allowed and will be preserved during sync. + + ### Resolution Steps: + + 1. Review the conflicting commits: + \`\`\`bash + git log upstream/master..origin/master --oneline + \`\`\` + + 2. Determine if commits should be: + - **Preserved:** Create feature branch and reset master + - **Discarded:** Hard reset master to upstream + + 3. See [sync documentation](.github/docs/sync-setup.md) for detailed recovery procedures + + 4. Run manual sync workflow after resolution to verify + + **Workflow run:** ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + `; + + // Check if issue already exists + const issues = await github.rest.issues.listForRepo({ + owner: context.repo.owner, + repo: context.repo.repo, + state: 'open', + labels: 'sync-failure' + }); + + if (issues.data.length === 0) { + await github.rest.issues.create({ + owner: context.repo.owner, + repo: context.repo.repo, + title: title, + body: body, + labels: ['sync-failure', 'automation', 'urgent'] + }); + } else { + // Update existing issue + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issues.data[0].number, + body: `Sync failed again on ${new Date().toISOString()}\n\nWorkflow: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}` + }); + } + + - name: Close sync-failure issues + if: steps.merge.outputs.merge_status == 'success' + uses: actions/github-script@v7 + with: + script: | + const issues = await github.rest.issues.listForRepo({ + owner: context.repo.owner, + repo: context.repo.repo, + state: 'open', + labels: 'sync-failure' + }); + + for (const issue of issues.data) { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issue.number, + body: `āœ“ Automatic sync successful on ${new Date().toISOString()} - synced ${{ steps.check_commits.outputs.commits_behind }} commits.\n\nClosing issue automatically.` + }); + + await github.rest.issues.update({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issue.number, + state: 'closed' + }); + } + + - name: Summary + if: always() + run: | + echo "### Daily Sync Summary" >> $GITHUB_STEP_SUMMARY + echo "- **Date:** $(date -u)" >> $GITHUB_STEP_SUMMARY + echo "- **Status:** ${{ steps.merge.outputs.merge_status }}" >> $GITHUB_STEP_SUMMARY + echo "- **Commits synced:** ${{ steps.check_commits.outputs.commits_behind }}" >> $GITHUB_STEP_SUMMARY + + if [ "${{ steps.merge.outputs.merge_status }}" == "success" ]; then + echo "" >> $GITHUB_STEP_SUMMARY + echo "āœ“ Mirror successfully updated with upstream postgres/postgres" >> $GITHUB_STEP_SUMMARY + elif [ "${{ steps.merge.outputs.merge_status }}" == "uptodate" ]; then + echo "" >> $GITHUB_STEP_SUMMARY + echo "āœ“ Mirror already up to date" >> $GITHUB_STEP_SUMMARY + else + echo "" >> $GITHUB_STEP_SUMMARY + echo "āš ļø Sync failed - check created issue for details" >> $GITHUB_STEP_SUMMARY + fi From 5445b4d8a6d4fe33fe31f3c91f1abbebd9c67ffc Mon Sep 17 00:00:00 2001 From: Greg Burd Date: Mon, 13 Jul 2026 09:08:46 -0400 Subject: [PATCH 02/10] ci: AI/LLM PR review (OCR via Bedrock + Agora MCP history) The Open Code Review system: ocr-review and ocr-model-check workflows plus .github/ocr config (LiteLLM->Bedrock Claude Opus 4.8, rule.json, context.md, pg-history.py). --- .github/docs/ai-review-guide.md | 512 ++++++++++++++++++++++++++ .github/docs/bedrock-setup.md | 298 +++++++++++++++ .github/docs/cost-optimization.md | 219 +++++++++++ .github/ocr/context.md | 126 +++++++ .github/ocr/litellm.yaml | 41 +++ .github/ocr/pg-history.py | 225 +++++++++++ .github/ocr/rule.json | 65 ++++ .github/workflows/ocr-model-check.yml | 89 +++++ .github/workflows/ocr-review.yml | 427 +++++++++++++++++++++ 9 files changed, 2002 insertions(+) create mode 100644 .github/docs/ai-review-guide.md create mode 100644 .github/docs/bedrock-setup.md create mode 100644 .github/docs/cost-optimization.md create mode 100644 .github/ocr/context.md create mode 100644 .github/ocr/litellm.yaml create mode 100644 .github/ocr/pg-history.py create mode 100644 .github/ocr/rule.json create mode 100644 .github/workflows/ocr-model-check.yml create mode 100644 .github/workflows/ocr-review.yml diff --git a/.github/docs/ai-review-guide.md b/.github/docs/ai-review-guide.md new file mode 100644 index 0000000000000..eff0ed10cba4f --- /dev/null +++ b/.github/docs/ai-review-guide.md @@ -0,0 +1,512 @@ +# AI-Powered Code Review Guide + +## Overview + +This system uses Claude AI (Anthropic) to provide PostgreSQL-aware code reviews on pull requests. Reviews are similar in style to feedback from the PostgreSQL Hackers mailing list. + +## How It Works + +``` +PR Event (opened/updated) + ā†“ +GitHub Actions Workflow Starts + ā†“ +Fetch PR diff + metadata + ā†“ +Filter reviewable files (.c, .h, .sql, docs, Makefiles) + ā†“ +Route each file to appropriate review prompt + ā†“ +Send to Claude API with PostgreSQL context + ā†“ +Parse response for issues + ā†“ +Post inline comments + summary to PR + ā†“ +Add labels (security-concern, performance, etc.) +``` + +## Features + +### PostgreSQL-Specific Reviews + +**C Code Review:** +- Memory management (palloc/pfree, memory contexts) +- Concurrency (lock ordering, race conditions) +- Error handling (elog/ereport patterns) +- Performance (algorithm complexity, cache efficiency) +- Security (buffer overflows, SQL injection vectors) +- PostgreSQL conventions (naming, comments, style) + +**SQL Review:** +- PostgreSQL SQL dialect correctness +- Regression test patterns +- Performance (index usage, join strategy) +- Deterministic output for tests +- Edge case coverage + +**Documentation Review:** +- Technical accuracy +- SGML/DocBook format +- PostgreSQL style guide compliance +- Examples and cross-references + +**Build System Review:** +- Makefile correctness (GNU Make, PGXS) +- Meson build consistency +- Cross-platform portability +- VPATH build support + +### Automatic Labeling + +Reviews automatically add labels based on findings: + +- `security-concern` - Security issues, vulnerabilities +- `performance-concern` - Performance problems +- `needs-tests` - Missing test coverage +- `needs-docs` - Missing documentation +- `memory-management` - Memory leaks, context issues +- `concurrency-issue` - Deadlocks, race conditions + +### Cost Management + +- **Per-PR limit:** $15 (configurable) +- **Monthly limit:** $200 (configurable) +- **Alert threshold:** $150 +- **Skip draft PRs** to save costs +- **Skip large files** (>5000 lines) +- **Skip binary/generated files** + +## Setup + +### 1. Install Dependencies + +```bash +cd .github/scripts/ai-review +npm install +``` + +### 2. Configure API Key + +Get API key from: https://console.anthropic.com/ + +Add to repository secrets: +1. Settings ā†’ Secrets and variables ā†’ Actions +2. New repository secret +3. Name: `ANTHROPIC_API_KEY` +4. Value: Your API key +5. Add secret + +### 3. Enable Workflow + +The workflow is triggered automatically on PR events: +- PR opened +- PR synchronized (updated) +- PR reopened +- PR marked ready for review (draft ā†’ ready) + +**Draft PRs are skipped** to save costs. + +## Configuration + +### Main Configuration: `config.json` + +```json +{ + "model": "claude-3-5-sonnet-20241022", + "max_tokens_per_request": 4096, + "max_file_size_lines": 5000, + + "cost_limits": { + "max_per_pr_dollars": 15.0, + "max_per_month_dollars": 200.0, + "alert_threshold_dollars": 150.0 + }, + + "skip_paths": [ + "*.png", "*.jpg", "*.svg", + "src/test/regress/expected/*", + "*.po", "*.pot" + ], + + "auto_labels": { + "security-concern": ["security issue", "vulnerability"], + "performance-concern": ["inefficient", "O(nĀ²)"], + "needs-tests": ["missing test", "no test coverage"] + } +} +``` + +**Tunable parameters:** +- `max_tokens_per_request`: Response length (4096 = ~3000 words) +- `max_file_size_lines`: Skip files larger than this +- `cost_limits`: Adjust budget caps +- `skip_paths`: Add more patterns to skip +- `auto_labels`: Customize label keywords + +### Review Prompts + +Located in `.github/scripts/ai-review/prompts/`: + +- `c-code.md` - PostgreSQL C code review +- `sql.md` - SQL and regression test review +- `documentation.md` - Documentation review +- `build-system.md` - Makefile/Meson review + +**Customization:** Edit prompts to adjust review focus and style. + +## Usage + +### Automatic Reviews + +Reviews run automatically on PRs to `master` and `feature/**` branches. + +**Typical workflow:** +1. Create feature branch +2. Make changes +3. Push branch: `git push origin feature/my-feature` +4. Create PR +5. AI review runs automatically +6. Review AI feedback +7. Make updates if needed +8. Push updates ā†’ AI re-reviews + +### Manual Reviews + +Trigger manually via GitHub Actions: + +**Via UI:** +1. Actions ā†’ "AI Code Review" +2. Run workflow +3. Enter PR number +4. Run workflow + +**Via CLI:** +```bash +gh workflow run ai-code-review.yml -f pr_number=123 +``` + +### Interpreting Reviews + +**Inline comments:** +- Posted on specific lines of code +- Format: `**[Category]**` followed by description +- Categories: Memory, Security, Performance, etc. + +**Summary comment:** +- Posted at PR level +- Overview of files reviewed +- Issue count by category +- Cost information + +**Labels:** +- Automatically added based on findings +- Filter PRs by label to prioritize +- Remove label manually if false positive + +### Best Practices + +**Trust but verify:** +- AI reviews are helpful but not infallible +- False positives happen (~5% rate) +- Use judgment - AI doesn't have full context +- Especially verify: security and correctness issues + +**Iterative improvement:** +- AI learns from the prompts, not from feedback +- If AI consistently misses something, update prompts +- Share false positives/negatives to improve system + +**Cost consciousness:** +- Keep PRs focused (fewer files = lower cost) +- Use draft PRs for work-in-progress (AI skips drafts) +- Mark PR ready when you want AI review + +## Cost Tracking + +### View Costs + +**Per-PR cost:** +- Shown in AI review summary comment +- Format: `Cost: $X.XX | Model: claude-3-5-sonnet` + +**Monthly cost:** +- Download cost logs from workflow artifacts +- Aggregate to calculate monthly total + +**Download cost logs:** +```bash +# List recent runs +gh run list --workflow=ai-code-review.yml --limit 10 + +# Download artifact +gh run download -n ai-review-cost-log- +``` + +### Cost Estimation + +**Token costs (Claude 3.5 Sonnet):** +- Input: $0.003 per 1K tokens +- Output: $0.015 per 1K tokens + +**Typical costs:** +- Small PR (<500 lines, 5 files): $0.50-$1.00 +- Medium PR (500-2000 lines, 15 files): $1.00-$3.00 +- Large PR (2000-5000 lines, 30 files): $3.00-$7.50 + +**Expected monthly (20 PRs/month mixed sizes):** $35-50 + +### Budget Controls + +**Automatic limits:** +- Per-PR limit: Stops reviewing after $15 +- Monthly limit: Stops at $200 (requires manual override) +- Alert: Warning at $150 + +**Manual controls:** +- Disable workflow: Actions ā†’ AI Code Review ā†’ Disable +- Reduce `max_tokens_per_request` in config +- Add more patterns to `skip_paths` +- Increase `max_file_size_lines` threshold + +## Troubleshooting + +### Issue: No review posted + +**Possible causes:** +1. PR is draft (intentionally skipped) +2. No reviewable files (all binary or skipped patterns) +3. API key missing or invalid +4. Cost limit reached + +**Check:** +- Actions ā†’ "AI Code Review" ā†’ Latest run ā†’ View logs +- Look for: "Skipping draft PR" or "No reviewable files" +- Verify: `ANTHROPIC_API_KEY` secret exists + +### Issue: Review incomplete + +**Possible causes:** +1. PR cost limit reached ($15 default) +2. File too large (>5000 lines) +3. API rate limit hit + +**Check:** +- Review summary comment for "Reached PR cost limit" +- Workflow logs for "Skipping X - too large" + +**Fix:** +- Increase `max_per_pr_dollars` in config +- Increase `max_file_size_lines` (trade-off: higher cost) +- Split large PR into smaller PRs + +### Issue: False positives + +**Example:** AI flags correct code as problematic + +**Handling:** +1. Ignore the comment (human judgment overrides) +2. Reply to comment explaining why it's correct +3. If systematic: Update prompt to clarify + +**Note:** Some false positives are acceptable (5-10% rate) + +### Issue: Claude API errors + +**Error types:** +- `401 Unauthorized`: Invalid API key +- `429 Too Many Requests`: Rate limit +- `500 Internal Server Error`: Claude service issue + +**Check:** +- Workflow logs for error messages +- Claude status: https://status.anthropic.com/ + +**Fix:** +- Rotate API key if 401 +- Wait and retry if 429 or 500 +- Contact Anthropic support if persistent + +### Issue: High costs + +**Unexpected high costs:** +1. Check cost logs for large PRs +2. Review `skip_paths` - are large files being reviewed? +3. Check for repeated reviews (PR updated many times) + +**Optimization:** +- Add more skip patterns for generated files +- Lower `max_tokens_per_request` (shorter reviews) +- Increase `max_file_size_lines` to skip more files +- Batch PR updates to reduce review runs + +## Disabling AI Review + +### Temporarily disable + +**For one PR:** +- Convert to draft +- Or add `[skip ai]` to PR title (requires workflow modification) + +**For all PRs:** +```bash +# Via GitHub UI: +# Actions ā†’ "AI Code Review" ā†’ "..." ā†’ Disable workflow + +# Via git: +git mv .github/workflows/ai-code-review.yml \ + .github/workflows/ai-code-review.yml.disabled +git commit -m "Disable AI code review" +git push +``` + +### Permanently remove + +```bash +# Remove workflow +rm .github/workflows/ai-code-review.yml + +# Remove scripts +rm -rf .github/scripts/ai-review + +# Commit +git commit -am "Remove AI code review system" +git push +``` + +## Testing and Iteration + +### Shadow Mode (Week 1) + +Run reviews but don't post comments: + +1. Modify `review-pr.js`: + ```javascript + // Comment out posting functions + // await postInlineComments(...) + // await postSummaryComment(...) + ``` + +2. Reviews saved to workflow artifacts +3. Review quality offline +4. Tune prompts based on results + +### Comment Mode (Week 2) + +Post comments with `[AI Review]` prefix: + +1. Add prefix to comment body: + ```javascript + const body = `**[AI Review] [${issue.category}]**\n\n${issue.description}`; + ``` + +2. Gather feedback from developers +3. Adjust prompts and configuration + +### Full Mode (Week 3+) + +Remove prefix, enable all features: + +1. Remove `[AI Review]` prefix +2. Enable auto-labeling +3. Monitor quality and costs +4. Iterate on prompts as needed + +## Advanced Customization + +### Custom Review Prompts + +Add a new prompt for a file type: + +1. Create `.github/scripts/ai-review/prompts/my-type.md` +2. Write review guidelines (see existing prompts) +3. Update `config.json`: + ```json + "file_type_patterns": { + "my_type": ["*.ext", "special/*.files"] + } + ``` +4. Test with manual workflow trigger + +### Conditional Reviews + +Skip AI review for certain PRs: + +Modify `.github/workflows/ai-code-review.yml`: +```yaml +jobs: + ai-review: + if: | + github.event.pull_request.draft == false && + !contains(github.event.pull_request.title, '[skip ai]') && + !contains(github.event.pull_request.labels.*.name, 'no-ai-review') +``` + +### Cost Alerts + +Add cost alert notifications: + +1. Create workflow in `.github/workflows/cost-alert.yml` +2. Trigger: On schedule (weekly) +3. Aggregate cost logs +4. Post issue if over threshold + +## Security and Privacy + +### API Key Security + +- Store only in GitHub Secrets (encrypted at rest) +- Never commit to repository +- Never log in workflow output +- Rotate quarterly + +### Code Privacy + +- Code sent to Claude API (Anthropic) +- Anthropic does not train on API data +- API requests are not retained long-term +- See: https://www.anthropic.com/legal/privacy + +### Sensitive Code + +If reviewing sensitive/proprietary code: + +1. Review Anthropic's terms of service +2. Consider: Self-hosted alternative (future) +3. Or: Skip AI review for sensitive PRs (add label) + +## Support + +### Questions + +- Check this guide first +- Search GitHub issues: label:ai-review +- Check Claude API docs: https://docs.anthropic.com/ + +### Reporting Issues + +Create issue with: +- PR number +- Workflow run URL +- Error messages from logs +- Expected vs actual behavior + +### Improving Prompts + +Contributions welcome: +1. Identify systematic issue (false positive/negative) +2. Propose prompt modification +3. Test on sample PRs +4. Submit PR with updated prompt + +## References + +- Claude API: https://docs.anthropic.com/ +- Claude Models: https://www.anthropic.com/product +- PostgreSQL Hacker's Guide: https://wiki.postgresql.org/wiki/Developer_FAQ +- GitHub Actions: https://docs.github.com/en/actions + +--- + +**Version:** 1.0 +**Last Updated:** 2026-03-10 diff --git a/.github/docs/bedrock-setup.md b/.github/docs/bedrock-setup.md new file mode 100644 index 0000000000000..d8fbd898b51c6 --- /dev/null +++ b/.github/docs/bedrock-setup.md @@ -0,0 +1,298 @@ +# AWS Bedrock Setup for AI Code Review + +This guide explains how to use AWS Bedrock instead of the direct Anthropic API for AI code reviews. + +## Why Use Bedrock? + +- **AWS Credits:** Use existing AWS credits +- **Regional Availability:** Deploy in specific AWS regions +- **Compliance:** Meet specific compliance requirements +- **Integration:** Easier integration with AWS infrastructure +- **IAM Roles:** Use IAM roles instead of API keys when running on AWS + +## Prerequisites + +1. **AWS Account** with Bedrock access +2. **Bedrock Model Access** - Claude 3.5 Sonnet must be enabled +3. **IAM Permissions** for Bedrock API calls + +## Step 1: Enable Bedrock Model Access + +1. Log into AWS Console +2. Navigate to **Amazon Bedrock** +3. Go to **Model access** (left sidebar) +4. Click **Modify model access** +5. Find and enable: **Anthropic - Claude 3.5 Sonnet v2** +6. Click **Save changes** +7. Wait for status to show "Access granted" (~2-5 minutes) + +## Step 2: Create IAM User for GitHub Actions + +### Option A: IAM User with Access Keys (Recommended for GitHub Actions) + +1. Go to **IAM Console** +2. Click **Users** ā†’ **Create user** +3. Username: `github-actions-bedrock` +4. Click **Next** + +**Attach Policy:** +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "bedrock:InvokeModel" + ], + "Resource": [ + "arn:aws:bedrock:*::foundation-model/anthropic.claude-3-5-sonnet-*" + ] + } + ] +} +``` + +5. Click **Create policy** ā†’ **JSON** ā†’ Paste above +6. Name: `BedrockClaudeInvokeOnly` +7. Attach policy to user +8. Click **Create user** + +**Create Access Keys:** +1. Click on the created user +2. Go to **Security credentials** tab +3. Click **Create access key** +4. Select: **Third-party service** +5. Click **Next** ā†’ **Create access key** +6. **Download** or copy: + - Access key ID (starts with `AKIA...`) + - Secret access key (only shown once!) + +### Option B: IAM Role (For AWS-hosted runners) + +If running GitHub Actions on AWS (self-hosted runners): + +1. Create IAM Role with trust policy for your EC2/ECS/EKS +2. Attach same `BedrockClaudeInvokeOnly` policy +3. Assign role to your runner infrastructure +4. No access keys needed! + +## Step 3: Configure Repository + +### A. Add AWS Secrets to GitHub + +1. Go to: **Settings** ā†’ **Secrets and variables** ā†’ **Actions** +2. Click **New repository secret** for each: + +**Secret 1:** +- Name: `AWS_ACCESS_KEY_ID` +- Value: Your access key ID from Step 2 + +**Secret 2:** +- Name: `AWS_SECRET_ACCESS_KEY` +- Value: Your secret access key from Step 2 + +**Secret 3:** +- Name: `AWS_REGION` +- Value: Your Bedrock region (e.g., `us-east-1`) + +### B. Update Configuration + +Edit `.github/scripts/ai-review/config.json`: + +```json +{ + "provider": "bedrock", + "model": "claude-3-5-sonnet-20241022", + "bedrock_model_id": "us.anthropic.claude-3-5-sonnet-20241022-v2:0", + "bedrock_region": "us-east-1", + ... +} +``` + +**Available Bedrock Model IDs:** +- US: `us.anthropic.claude-3-5-sonnet-20241022-v2:0` +- EU: `eu.anthropic.claude-3-5-sonnet-20241022-v2:0` +- Asia Pacific: `apac.anthropic.claude-3-5-sonnet-20241022-v2:0` + +**Available Regions:** +- `us-east-1` (US East - N. Virginia) +- `us-west-2` (US West - Oregon) +- `eu-central-1` (Europe - Frankfurt) +- `eu-west-1` (Europe - Ireland) +- `eu-west-2` (Europe - London) +- `ap-southeast-1` (Asia Pacific - Singapore) +- `ap-southeast-2` (Asia Pacific - Sydney) +- `ap-northeast-1` (Asia Pacific - Tokyo) + +Check current availability: https://docs.aws.amazon.com/bedrock/latest/userguide/models-regions.html + +### C. Install Dependencies + +```bash +cd .github/scripts/ai-review +npm install +``` + +This will install the AWS SDK for Bedrock. + +## Step 4: Test Bedrock Integration + +```bash +# Create test PR +git checkout -b test/bedrock-review +echo "// Bedrock test" >> test.c +git add test.c +git commit -m "Test: Bedrock AI review" +git push origin test/bedrock-review +``` + +Then create PR via GitHub UI. Check: +1. **Actions** tab - workflow should run +2. **PR comments** - AI review should appear +3. **Workflow logs** - should show "Using AWS Bedrock as provider" + +## Cost Comparison + +### Bedrock Pricing (Claude 3.5 Sonnet - us-east-1) +- Input: $0.003 per 1K tokens +- Output: $0.015 per 1K tokens + +### Direct Anthropic API Pricing +- Input: $0.003 per 1K tokens +- Output: $0.015 per 1K tokens + +**Same price!** Choose based on infrastructure preference. + +## Troubleshooting + +### Error: "Access denied to model" + +**Check:** +1. Model access enabled in Bedrock console? +2. IAM policy includes correct model ARN? +3. Region matches between config and enabled models? + +**Fix:** +```bash +# Verify model access via AWS CLI +aws bedrock list-foundation-models --region us-east-1 --query 'modelSummaries[?contains(modelId, `claude-3-5-sonnet`)]' +``` + +### Error: "InvalidSignatureException" + +**Check:** +1. AWS_ACCESS_KEY_ID correct? +2. AWS_SECRET_ACCESS_KEY correct? +3. Secrets named exactly as shown? + +**Fix:** +- Re-create access keys +- Update GitHub secrets +- Ensure no extra spaces in secret values + +### Error: "ThrottlingException" + +**Cause:** Bedrock rate limits exceeded + +**Fix:** +1. Reduce `max_concurrent_requests` in config.json +2. Add delays between requests +3. Request quota increase via AWS Support + +### Error: "Model not found" + +**Check:** +1. `bedrock_model_id` matches your region +2. Using cross-region model ID (e.g., `us.anthropic...` in us-east-1) + +**Fix:** +Update `bedrock_model_id` in config.json to match your region: +- US regions: `us.anthropic.claude-3-5-sonnet-20241022-v2:0` +- EU regions: `eu.anthropic.claude-3-5-sonnet-20241022-v2:0` + +## Switching Between Providers + +### Switch to Bedrock + +Edit `.github/scripts/ai-review/config.json`: +```json +{ + "provider": "bedrock", + ... +} +``` + +### Switch to Direct Anthropic API + +Edit `.github/scripts/ai-review/config.json`: +```json +{ + "provider": "anthropic", + ... +} +``` + +No other changes needed! The code automatically detects the provider. + +## Advanced: Cross-Region Setup + +Deploy in multiple regions for redundancy: + +```json +{ + "provider": "bedrock", + "bedrock_regions": ["us-east-1", "us-west-2"], + "bedrock_failover": true +} +``` + +Then update `review-pr.js` to implement failover logic. + +## Security Best Practices + +1. **Least Privilege:** IAM user can only invoke Claude models +2. **Rotate Keys:** Rotate access keys quarterly +3. **Audit Logs:** Enable CloudTrail for Bedrock API calls +4. **Cost Alerts:** Set up AWS Budgets alerts +5. **Secrets:** Never commit AWS credentials to git + +## Monitoring + +### AWS CloudWatch + +Bedrock metrics available: +- `Invocations` - Number of API calls +- `InvocationLatency` - Response time +- `InvocationClientErrors` - 4xx errors +- `InvocationServerErrors` - 5xx errors + +### Cost Tracking + +```bash +# Check Bedrock costs (current month) +aws ce get-cost-and-usage \ + --time-period Start=2026-03-01,End=2026-03-31 \ + --granularity MONTHLY \ + --metrics BlendedCost \ + --filter file://filter.json + +# filter.json: +{ + "Dimensions": { + "Key": "SERVICE", + "Values": ["Amazon Bedrock"] + } +} +``` + +## References + +- AWS Bedrock Docs: https://docs.aws.amazon.com/bedrock/ +- Model Access: https://docs.aws.amazon.com/bedrock/latest/userguide/model-access.html +- Bedrock Pricing: https://aws.amazon.com/bedrock/pricing/ +- IAM Best Practices: https://docs.aws.amazon.com/IAM/latest/UserGuide/best-practices.html + +--- + +**Need help?** Check workflow logs in Actions tab or create an issue. diff --git a/.github/docs/cost-optimization.md b/.github/docs/cost-optimization.md new file mode 100644 index 0000000000000..bcfc1c47b3ed8 --- /dev/null +++ b/.github/docs/cost-optimization.md @@ -0,0 +1,219 @@ +# CI/CD Cost Optimization + +## Overview + +This document describes the cost optimization strategies used in the PostgreSQL mirror CI/CD system to minimize GitHub Actions minutes and API costs while maintaining full functionality. + +## Optimization Strategies + +### 1. Skip Builds for Pristine Commits + +**Problem:** "Dev setup" commits and .github/ configuration changes don't require expensive Windows dependency builds or comprehensive testing. + +**Solution:** The Windows Dependencies workflow includes a `check-changes` job that inspects recent commits and skips builds when all commits are: +- Messages starting with "dev setup" (case-insensitive), OR +- Only modifying files under `.github/` directory + +**Implementation:** See `.github/workflows/windows-dependencies.yml` lines 42-90 + +**Savings:** +- Avoids ~45 minutes of Windows runner time per push +- Windows runners cost 2x Linux minutes (1 minute = 2 billed minutes) +- Estimated savings: ~$8-12/month + +### 2. AI Review Only on Pull Requests + +**Problem:** AI code review is expensive and unnecessary for direct commits to master or pristine commits. + +**Solution:** The AI Code Review workflow only triggers on: +- `pull_request` events (opened, synchronized, reopened, ready_for_review) +- Manual `workflow_dispatch` for testing specific PRs +- Skips draft PRs automatically + +**Implementation:** See `.github/workflows/ai-code-review.yml` lines 3-17 + +**Savings:** +- No reviews on dev setup commits or CI/CD changes +- No reviews on draft PRs (saves ~$1-3 per draft) +- Estimated savings: ~$10-20/month + +### 3. Aggressive Caching + +**Windows Dependencies:** +- Cache key: `--win64-` +- Cache duration: GitHub's default (7 days unused, 10 GB limit) +- Cache hit rate: 80-90% for stable versions + +**Node.js Dependencies:** +- AI review scripts cache npm packages +- Cache key based on `package.json` hash +- Near 100% cache hit rate + +**Savings:** +- Reduces build time from 45 minutes to ~5 minutes on cache hit +- Estimated savings: ~$15-20/month + +### 4. Weekly Scheduled Builds + +**Problem:** GitHub Actions artifacts expire after 90 days, making cached dependencies stale. + +**Solution:** Windows Dependencies runs on a weekly schedule (Sunday 4 AM UTC) to refresh artifacts before expiration. + +**Cost:** +- Weekly builds: ~45 minutes/week Ɨ 4 weeks = 180 minutes/month +- Windows multiplier: 360 billed minutes +- Cost: ~$6/month (within budget) + +**Alternative considered:** Daily builds would cost ~$50/month (rejected) + +### 5. Sync Workflow Optimization + +**Automatic Sync:** +- Runs hourly to keep mirror current +- Very lightweight: ~2-3 minutes per run +- Cost: ~150 minutes/month = $0 (within free tier) + +**Manual Sync:** +- Only runs on explicit trigger +- Used for testing and recovery +- Cost: Negligible + +### 6. Smart Workflow Triggers + +**Path-based triggers:** +```yaml +push: + paths: + - '.github/windows/manifest.json' + - '.github/workflows/windows-dependencies.yml' +``` + +Only rebuild Windows dependencies when: +- Manifest versions change +- Workflow itself is updated +- Manual trigger or schedule + +**Branch-based triggers:** +- AI review only on PRs to master, feature/**, dev/** +- Sync only affects master branch + +## Cost Breakdown + +| Component | Monthly Cost | Notes | +|-----------|-------------|-------| +| GitHub Actions - Sync | $0 | ~150 min/month (free: 2,000 min) | +| GitHub Actions - AI Review | $0 | ~200 min/month (free: 2,000 min) | +| GitHub Actions - Windows | ~$5-8 | ~2,500 min/month with optimizations | +| Claude API (Bedrock) | $30-45 | Usage-based, ~15-20 PRs/month | +| **Total** | **~$35-53/month** | | + +**Before optimizations:** ~$75-100/month +**After optimizations:** ~$35-53/month +**Savings:** ~$40-47/month (40-47% reduction) + +## Monitoring Costs + +### GitHub Actions Usage + +Check usage in repository settings: +``` +Settings ā†’ Billing and plans ā†’ View usage +``` + +Or via CLI: +```bash +gh api repos/:owner/:repo/actions/billing/workflows --jq '.workflows' +``` + +### AWS Bedrock Usage + +Monitor Claude API costs in AWS Console: +``` +AWS Console ā†’ Bedrock ā†’ Usage ā†’ Invocation metrics +``` + +Or via cost logs in artifacts: +``` +.github/scripts/ai-review/cost-log-*.json +``` + +### Setting Alerts + +**GitHub Actions:** +- No built-in alerts +- Monitor via monthly email summaries +- Consider third-party monitoring (e.g., AWS Lambda + GitHub API) + +**AWS Bedrock:** +- Set CloudWatch billing alarms +- Recommended thresholds: + - Warning: $30/month + - Critical: $50/month +- Hard cap in code: $200/month (see `config.json`) + +## Future Optimizations + +### Potential Improvements + +1. **Conditional Testing on PRs** + - Only run full Cirrus CI suite if C code or SQL changes + - Skip for docs-only PRs + - Estimated savings: ~5-10% of testing costs + +2. **Incremental AI Review** + - On PR updates, only review changed files + - Current: Reviews entire PR on each update + - Estimated savings: ~20-30% of AI costs + +3. **Dependency Build Sampling** + - Build only changed dependencies instead of all + - Requires more sophisticated manifest diffing + - Estimated savings: ~30-40% of Windows build costs + +4. **Self-hosted Runners** + - Run Linux builds on own infrastructure + - Keep Windows runners on GitHub (licensing) + - Estimated savings: ~$10-15/month + - **Trade-off:** Maintenance overhead + +### Not Recommended + +1. **Reduce sync frequency** (hourly ā†’ daily) + - Savings: Negligible (~$0.50/month) + - Cost: Increased lag with upstream (unacceptable) + +2. **Skip Windows builds entirely** + - Savings: ~$8/month + - Cost: Lose reproducible dependency builds (defeats purpose) + +3. **Reduce AI review quality** (Claude Sonnet ā†’ Haiku) + - Savings: ~$20-25/month + - Cost: Significantly worse code review quality + +## Pristine Commit Policy + +The following commits are considered "pristine" and skip expensive builds: + +1. **Dev setup commits:** + - Message starts with "dev setup" (case-insensitive) + - Examples: "dev setup v19", "Dev Setup: Update IDE config" + - Contains: .clang-format, .idea/, .vscode/, flake.nix, etc. + +2. **CI/CD configuration commits:** + - Only modify files under `.github/` + - Examples: Workflow changes, script updates, documentation + +**Why this works:** +- Dev setup commits don't affect PostgreSQL code +- CI/CD commits are tested by running the workflows themselves +- Reduces unnecessary Windows builds by ~60-70% + +**Implementation:** See `pristine-master-policy.md` for details. + +## Questions? + +For more information: +- Pristine master policy: `.github/docs/pristine-master-policy.md` +- Sync setup: `.github/docs/sync-setup.md` +- AI review guide: `.github/docs/ai-review-guide.md` +- Windows builds: `.github/docs/windows-builds.md` diff --git a/.github/ocr/context.md b/.github/ocr/context.md new file mode 100644 index 0000000000000..c4a83b85b124e --- /dev/null +++ b/.github/ocr/context.md @@ -0,0 +1,126 @@ +# OCR review context ā€” PostgreSQL contribution standards + +You are reviewing a change to a **PostgreSQL** fork. Every PR here is destined to +become a patch posted to the **pgsql-hackers** mailing list and tracked in a +**commitfest**. Review with the combined rigor, taste, and attention to detail of +the PostgreSQL committers. This context applies to the *whole* change, on top of +the per-file rules. + +## Review discipline +- Be precise and blunt; lead with the most serious problem. No praise, no + validation of the author, no disclaimers ā€” accuracy is the only metric. +- Verify every claim against the actual diff. Confirm names, signatures, line + numbers, and APIs before asserting. Never invent behavior or cite code not in + the change. If unsure, say so, and tag each finding **high / moderate / low** + confidence. +- Judge the change on its merits regardless of how the PR frames it. A draft PR + is WIP: weight design/approach feedback over style nits. + +## Patch hygiene (top rejection reasons on -hackers) +1. **Minimal diff.** The fastest way to get a patch rejected is unrelated + changes: reformatting untouched lines, rewording unrelated comments, touching + code not required by the change. Flag any hunk not needed for the stated + purpose. After the patch, the code should read as if it had always been + written that way. +2. **Atomic, bisectable commits.** Each commit must build and pass tests on its + own ā€” a broken intermediate commit breaks `git bisect`, revert, and + cherry-pick. Flag a commit that only compiles once a later commit lands. + Prefer one focused patch, or a clearly-ordered series of + independently-committable pieces. +3. **Tests + docs are mandatory.** A user-visible change without regression/TAP + tests **and** documentation is WIP, not commit-ready. New behavior needs + tests that cover edge and error paths, not just the happy path. +4. **DRY / reuse.** Prefer existing infrastructure (`List` in `pg_list.h`, + `StringInfo`, `dynahash`/`simplehash`, `palloc`/`MemoryContext`, `foreach`) + over reinventing it. Flag copy-paste and speculative abstraction alike ā€” the + community wants minimal, targeted changes that fit the subsystem's existing + patterns. +5. **Whitespace.** No trailing whitespace; tabs (width 4) for C indentation; + `git diff --check` must be clean. Whitespace-only churn on untouched lines is + a defect. + +## Committer-owned files ā€” do NOT touch in a patch (flag if present) +These are the committer's job at push time; including them causes needless +merge conflicts and is a mistake: +- **`src/include/catalog/catversion.h`** ā€” the `CATALOG_VERSION_NO` bump is done + by the **committer** when pushing. A catversion bump in the PR is **wrong** ā€” + flag it. (This is the single most common author mistake in catalog patches.) +- **Release notes** (`doc/src/sgml/release-*.sgml`) and version strings + (`configure.ac` `AC_INIT` version, `meson.build` `version`, `PG_VERSION`). + +## Generated files ā€” never hand-edit; edit the source +Flag direct edits to generated output; point the author at the source instead: +- Catalog headers `src/include/catalog/*_d.h`, `postgres.bki`, `schemapg.h`, + `system_constraints.sql` ā†’ edit the `pg_*.dat` files. +- `src/backend/nodes/{copy,equal,out,read}funcs.c` and other + `gen_node_support.pl` output ā†’ annotate the `Node` struct in its header. +- `fmgroids.h`, `fmgrprotos.h`, `fmgrtab.c` ā†’ edit `pg_proc.dat`. +- `utils/errcodes.h` ā†’ `errcodes.txt`; wait-event headers ā†’ + `wait_event_names.txt`; `lwlocknames.h` ā†’ `lwlocknames.txt`. +- `configure` ā†’ `configure.ac`; `*.po` translations are handled separately; + generated Unicode tables come from their source scripts. + +## Portability is a hard gate +PostgreSQL runs on Linux, Windows (MSVC), macOS, the BSDs and Solaris, across +**x86_64, ARM64, RISC-V, PPC64, s390x**, both endiannesses and 32/64-bit. Any +change must be portable across all of them: +- No unaligned memory access; no dependence on `char` signedness, integer/pointer + width, endianness, or struct padding for on-disk/wire formats. +- Use `int16/int32/int64`, `Size`, and `INT64_FORMAT`/`UINT64_FORMAT` (never + `%ld` for `int64`). +- Atomics/barriers only via `port/atomics` (`pg_atomic_*`, `pg_read/write_barrier`). +- **Windows/MSVC:** any `extern` variable used from another module or an + extension needs `PGDLLIMPORT` in its header; no VLAs or compiler-specific + extensions beyond the tree's C99 baseline. + +## Backward compatibility ā€” the strongest constraint +Do not break SQL behavior, the libpq wire protocol, the logical-replication +protocol, dump/restore, `pg_upgrade`, or exported/`PGDLLIMPORT` APIs without +extraordinary justification. **ABI** matters for back-branches: changing the +size/layout of an exported struct or the signature of an exported function +breaks installed extensions. + +## Mailing-list context & etiquette +Because each PR becomes a pgsql-hackers email read by a busy, expert, opinionated +audience, also flag what reliably wastes reviewer time or draws rejection: +- A patch that **does more than one thing** or bundles unrelated cleanup ā€” split it. +- **Footguns**: easy-to-misuse APIs, silent data-loss/corruption hazards, unsafe + defaults ā€” name them explicitly. +- **Performance claims without a reproducible benchmark.** +- No reference to the **design discussion / prior -hackers thread** (Message-Id) + for a non-trivial change. +- **Do not bikeshed:** keep style nits proportionate and clearly separated from + substantive correctness findings. + +## Minimalism ā€” the "ponytail" discipline +The best code is the code you never wrote (YAGNI). Before accepting new code, +apply the ladder: (1) Does this need to exist at all? (2) Can existing +code/infrastructure already do it? (3) Is this the simplest thing that works? +Flag: speculative scaffolding and config for a path that isn't wired yet; dead +code and unused "flexibility" (fields, params, abstractions, options with no +caller); premature abstraction (a helper used exactly once); knobs/GUCs/flags +nobody asked for. Minimal, targeted changes that fit the existing patterns beat +clever or general-purpose ones. + +## Comment & identity accuracy +- Comments must describe what the code does **now**. Flag aspirational/ + future-tense comments for behavior that already shipped ("will be", "for now", + "not yet", "future", and stale "TODO/FIXME/XXX/HACK"); comments that drifted + from the code they sit above; and incomplete/trailing comments. Comments + explain **why**, not what. No commented-out code. +- **ASCII only** in source and diffs ā€” no smart quotes, em-dashes, or ellipsis + characters. + +## Commit & versioning discipline +- Conventional-commit style, imperative subject, one logical change per commit, + each commit building on its own. +- Do **not** bump version numbers or generated version stamps (including + `catversion.h`) ā€” that is the maintainer's job at commit/release time. + +Understand common list shorthand so your comments are precise and not +miscommunicated: WIP (work in progress), GUC (config variable), WAL, LSN, OID, +TOAST, FSM, TAM (table access method), RLS, DSM, 2PC, PITR, CIC (concurrent index +creation), SAOP, ABI/API, backpatch (apply to supported back-branches), HEAD +(master tip), catversion (catalog version), pgindent, buildfarm, cfbot, +`s/x/y/` (suggested text substitution), footgun, bikeshedding, POLA (principle of +least astonishment). diff --git a/.github/ocr/litellm.yaml b/.github/ocr/litellm.yaml new file mode 100644 index 0000000000000..e23cc4eee6fe2 --- /dev/null +++ b/.github/ocr/litellm.yaml @@ -0,0 +1,41 @@ +# LiteLLM proxy config ā€” bridges Open Code Review (OpenAI protocol) to AWS Bedrock. +# +# This proxy is NOT a hosted service. The ocr-review.yml workflow installs it +# (`pip install 'litellm[proxy]'`) and runs it as a background process bound to +# 127.0.0.1:4000 for the duration of a single GitHub Actions job, then it exits. +# +# Auth to Bedrock: LiteLLM uses boto3's default credential chain, which reads +# the temporary AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY / AWS_SESSION_TOKEN +# minted by the workflow's OIDC "Configure AWS credentials" step; region from +# AWS_REGION. + +model_list: + - model_name: ocr-bedrock + litellm_params: + # Set the repo variable OCR_BEDROCK_MODEL to an Opus inference-profile id + # your account has access to, e.g.: + # bedrock/converse/us.anthropic.claude-opus-4-8 + # The 'converse/' prefix uses Bedrock's Converse API, which is the most + # reliable path for Claude tool-use (what OCR relies on). + model: os.environ/OCR_BEDROCK_MODEL + aws_region_name: os.environ/AWS_REGION + + # "High effort" review. Claude Opus 4.8 on Bedrock uses *adaptive* thinking + # controlled by output_config.effort. Set it DIRECTLY here ā€” NOT via + # reasoning_effort, which LiteLLM still maps to the legacy + # thinking.type.enabled that Opus 4.8 rejects. LiteLLM forwards + # output_config into additionalModelRequestFields for Anthropic models; if + # the build doesn't recognize the effort param it is dropped with a warning + # (no error) and the model reviews at its default effort. + # Valid: low|medium|high|max|xhigh (auto-clamped to the model ceiling). + output_config: + effort: xhigh + max_tokens: 32000 + +litellm_settings: + drop_params: true # silently drop params a model doesn't support + modify_params: true # auto-fix minor request incompatibilities + request_timeout: 600 + +general_settings: + master_key: os.environ/LITELLM_MASTER_KEY diff --git a/.github/ocr/pg-history.py b/.github/ocr/pg-history.py new file mode 100644 index 0000000000000..5794f8a920bd7 --- /dev/null +++ b/.github/ocr/pg-history.py @@ -0,0 +1,225 @@ +#!/usr/bin/env python3 +""" +pg-history: tie a PR's changes to PostgreSQL git + pgsql-hackers email history. + +OCR (the code reviewer) cannot call MCP servers, so this is a separate agent: +it runs a Bedrock (Claude Opus) tool-use loop wired to the Agora MCP server at +https://pg.ddx.io/mcp, lets the model search the mailing-list archives / commit +history / commitfest data, and emits a Markdown summary linking the changes to +the relevant threads (https://pg.ddx.io/m/pgsql-hackers/). + +Env: + PG_HISTORY_MCP_URL MCP endpoint (default https://pg.ddx.io/mcp) + PG_HISTORY_MODEL Bedrock model id (e.g. us.anthropic.claude-opus-4-8) + AWS_REGION region (creds come from the OIDC step's env) + BASE_REF, HEAD_SHA PR base ref and head sha (for the git diff context) + GH_PR_TITLE PR title (optional, adds context) + PG_HISTORY_OUT output markdown path (default /tmp/pg-history.md) +Writes the markdown to PG_HISTORY_OUT; exits 0 even on soft failures (writes a note). +""" +import json, os, subprocess, sys, urllib.request + +MCP_URL = os.environ.get("PG_HISTORY_MCP_URL", "https://pg.ddx.io/mcp") +MODEL = os.environ.get("PG_HISTORY_MODEL", "us.anthropic.claude-opus-4-8").replace("bedrock/converse/", "").replace("bedrock/", "") +REGION = os.environ.get("AWS_REGION", "us-east-1") +BASE_REF = os.environ.get("BASE_REF", "") +HEAD_SHA = os.environ.get("HEAD_SHA", "") +PR_TITLE = os.environ.get("GH_PR_TITLE", "") +OUT = os.environ.get("PG_HISTORY_OUT", "/tmp/pg-history.md") +UA = "pg-history/0.1 (+github-actions)" + +# Curated subset of the 108 Agora tools ā€” the ones useful for connecting a +# change to its discussion/commit history. Intersected with what the server +# actually exposes, so unknown names are harmless. +TOOL_WHITELIST = { + "find_related_discussions", "find_similar_messages", "get_thread", + "discussion_links", "get_author_messages", "browse_by_date", + "blame_symbol", "check_upstream_status", "find_related", + "find_entries_for_thread", "find_entries_for_author", "get_commit", + "search", "hybrid_search", "get_callers", "get_callees", "find_pattern", +} +MAX_ROUNDS = 14 +TOOL_RESULT_CAP = 8000 # chars per tool result fed back to the model + + +def _mcp_post(body, sid=None): + headers = {"Content-Type": "application/json", + "Accept": "application/json, text/event-stream", "User-Agent": UA} + if sid: + headers["Mcp-Session-Id"] = sid + req = urllib.request.Request(MCP_URL, data=json.dumps(body).encode(), headers=headers, method="POST") + resp = urllib.request.urlopen(req, timeout=60) + sid_out = resp.headers.get("Mcp-Session-Id") + result = None + for line in resp.read().decode().splitlines(): + line = line.strip() + if line.startswith("data:"): + line = line[5:].strip() + if not line or line.startswith("event:"): + continue + try: + obj = json.loads(line) + except Exception: + continue + if isinstance(obj, dict) and ("result" in obj or "error" in obj): + result = obj + return result, sid_out + + +class MCP: + def __init__(self): + init, self.sid = _mcp_post({"jsonrpc": "2.0", "id": 1, "method": "initialize", + "params": {"protocolVersion": "2025-06-18", "capabilities": {}, + "clientInfo": {"name": "pg-history", "version": "0.1"}}}) + if not init or "result" not in init: + raise RuntimeError(f"MCP initialize failed: {init}") + try: + _mcp_post({"jsonrpc": "2.0", "method": "notifications/initialized", "params": {}}, self.sid) + except Exception: + pass + self._id = 1 + + def list_tools(self): + self._id += 1 + res, _ = _mcp_post({"jsonrpc": "2.0", "id": self._id, "method": "tools/list", "params": {}}, self.sid) + return (res or {}).get("result", {}).get("tools", []) + + def call(self, name, args): + self._id += 1 + res, _ = _mcp_post({"jsonrpc": "2.0", "id": self._id, "method": "tools/call", + "params": {"name": name, "arguments": args or {}}}, self.sid) + if not res: + return "(no response)" + if "error" in res: + return f"ERROR: {json.dumps(res['error'])[:500]}" + parts = [] + for c in res.get("result", {}).get("content", []): + if c.get("type") == "text": + parts.append(c["text"]) + return ("\n".join(parts) or "(empty)")[:TOOL_RESULT_CAP] + + +def git(*args): + try: + return subprocess.check_output(["git", *args], text=True, stderr=subprocess.DEVNULL).strip() + except Exception: + return "" + + +def pr_context(): + base = f"origin/{BASE_REF}" if BASE_REF else "" + rng = f"{base}..{HEAD_SHA}" if base and HEAD_SHA else HEAD_SHA + commits = git("log", "--no-merges", "--format=%h %s", f"{rng}") if rng else "" + stat = git("diff", "--stat", rng) if rng else "" + files = git("diff", "--name-only", rng) if rng else "" + return commits[:4000], stat[:3000], files[:2000] + + +SYSTEM = """You are a PostgreSQL community research assistant. Given a pull request's +commits and changed files, use the available tools (backed by the Agora index of +pgsql-hackers mail, commit history, and commitfest data) to connect the change to +its history. Your goal: + +- Find the mailing-list thread(s) and prior discussion behind this change. +- Identify related/superseded prior commits and any commitfest entry. +- Note relevant prior art, rejected approaches, or design rationale. + +Rules (voice & rigor): +- Be precise and blunt. No praise, no filler, no hedging, no disclaimers. Accuracy is + the only success metric ā€” not the author's approval. Lead with the most important finding. +- NEVER hallucinate. Verify every Message-ID, thread subject, commit hash, author name, + and date against an actual tool result before citing it. If a search returns nothing, + say so plainly ā€” do not guess or fabricate a plausible-looking link. +- Assess the change on its merits, independent of how the PR frames it. +- Tag any inferred (not tool-confirmed) linkage with an explicit confidence level: + high / moderate / low. +- Be decisive and efficient: a handful of targeted tool calls, not exhaustive search. +- Cite every mailing-list message as a Markdown link: [subject](https://pg.ddx.io/m/pgsql-hackers/MESSAGE_ID). +- If you find nothing relevant, say so in one line ā€” do not pad. + +When done, output ONLY Markdown (no preamble) with these sections, omitting any that are empty: +## šŸ§µ Related discussion +## šŸ”— Related commits / prior art +## šŸ“‹ Commitfest +## šŸ§­ Context for reviewers +Keep it tight (use bullets; link generously).""" + + +def to_toolspec(t): + schema = t.get("inputSchema") or {"type": "object", "properties": {}} + return {"toolSpec": {"name": t["name"], + "description": (t.get("description") or "")[:600], + "inputSchema": {"json": schema}}} + + +def main(): + commits, stat, files = pr_context() + if not commits and not files: + open(OUT, "w").write("") # nothing to do + print("No PR diff context; skipping.") + return + user = (f"PR title: {PR_TITLE}\n\n" if PR_TITLE else "") + \ + f"Commits:\n{commits or '(none)'}\n\nChanged files:\n{files or '(none)'}\n\nDiffstat:\n{stat or '(none)'}\n" + + try: + mcp = MCP() + tools = [to_toolspec(t) for t in mcp.list_tools() if t.get("name") in TOOL_WHITELIST] + except Exception as e: + open(OUT, "w").write(f"_pg-history: could not reach the Agora MCP server ({MCP_URL}): {e}_\n") + print(f"MCP unavailable: {e}") + return + if not tools: + open(OUT, "w").write("_pg-history: no usable MCP tools available._\n") + return + + import boto3 + from botocore.config import Config + + # botocore's default read timeout (60s) is too short for a multi-round + # (MAX_ROUNDS) tool-use loop against a large PR diff on a reasoning model; + # each converse() call alone can take several minutes. Bump it well past + # what a single round needs; connect_timeout stays short since a stuck + # TCP handshake is a different (and much cheaper to detect) failure mode. + brt = boto3.client("bedrock-runtime", region_name=REGION, + config=Config(read_timeout=900, connect_timeout=10)) + messages = [{"role": "user", "content": [{"text": user}]}] + final_text = "" + try: + for _ in range(MAX_ROUNDS): + resp = brt.converse( + modelId=MODEL, + system=[{"text": SYSTEM}], + messages=messages, + toolConfig={"tools": tools}, + inferenceConfig={"maxTokens": 4096}, + ) + out = resp["output"]["message"] + messages.append(out) + if resp.get("stopReason") == "tool_use": + results = [] + for blk in out["content"]: + tu = blk.get("toolUse") + if not tu: + continue + res_text = mcp.call(tu["name"], tu.get("input") or {}) + results.append({"toolResult": {"toolUseId": tu["toolUseId"], + "content": [{"text": res_text}]}}) + messages.append({"role": "user", "content": results}) + continue + final_text = "".join(b.get("text", "") for b in out["content"]).strip() + break + except Exception as e: + open(OUT, "w").write(f"_pg-history: Bedrock call failed: {e}_\n") + print(f"Bedrock error: {e}") + return + + if not final_text: + final_text = "_pg-history: no related history found._" + body = "## šŸ“œ Change history & discussion (Agora / pg.ddx.io)\n\n" + final_text + \ + "\n\nGenerated by pg-history via the Agora MCP server (pg.ddx.io).\n" + open(OUT, "w").write(body) + print(body) + + +if __name__ == "__main__": + main() diff --git a/.github/ocr/rule.json b/.github/ocr/rule.json new file mode 100644 index 0000000000000..60e13e73dcbe0 --- /dev/null +++ b/.github/ocr/rule.json @@ -0,0 +1,65 @@ +{ + "_comment": "OCR per-file review rules for PostgreSQL core + extensions. Cross-cutting contribution standards & mailing-list etiquette live in .github/ocr/context.md, passed via --background-file. OCR uses FIRST-MATCH-WINS in declaration order, so rules are ordered most-specific first. merge_system_rule:true keeps OCR's built-in fine-tuned checks (thread-safety, injection, NPE) alongside these PostgreSQL-specific rules.", + "rules": [ + { + "path": "src/test/**", + "merge_system_rule": true, + "rule": "REVIEW DISCIPLINE: Precise, blunt, verify against the diff, tag confidence, no praise. PostgreSQL tests. Coverage is mandatory for any behavioral change and must include edge cases (NULL, empty, boundary/overflow) and ERROR paths, not just the happy path. A test that still passes with the feature reverted is worthless ā€” confirm it actually exercises and would catch regressions in the new code. Regression (.sql/expected): deterministic, portable output ā€” ORDER BY where row order matters, no timing/plan-dependent output except intentional EXPLAIN, no absolute paths, locale-independent (C collation or explicit COLLATE), DROP objects the test creates; expected/ output must stay stable across platforms and under the parallel schedule. Concurrency/locking belongs in isolation tests (src/test/isolation, .spec + permutations). End-to-end/crash/replication/CLI behavior belongs in TAP tests (t/*.pl with PostgreSQL::Test::Cluster/Utils) ā€” no hardcoded ports/paths, no sleep as synchronization (use poll_query_until/wait_for), skip cleanly when prerequisites are missing, and clean up nodes." + }, + { + "path": "**/*.{c,h}", + "merge_system_rule": true, + "rule": "REVIEW DISCIPLINE: Precise, blunt, verify against the diff, tag confidence, no praise. PostgreSQL backend/frontend C ā€” review as pgsql-hackers committers do, in priority order.\n\n(1) CORRECTNESS (highest): Memory ā€” every palloc lives in the right MemoryContext; error paths via ereport/elog(ERROR) must not leak memory/buffers/locks/fds (rely on MemoryContext/ResourceOwner reset or PG_TRY/PG_FINALLY); no use-after-free; delete temp contexts. Concurrency ā€” consistent lock ordering (deadlock-free), correct lock levels, balanced LWLockAcquire/Release and START_/END_CRIT_SECTION, no TOCTOU, CHECK_FOR_INTERRUPTS in long loops, async-signal-safe signal handlers (volatile sig_atomic_t). WAL ā€” any change to shared on-disk state must be WAL-logged AND correctly replayed (redo path), crash- and replica-consistent. NULL/edge/overflow handling.\n\n(2) BACKWARD COMPATIBILITY / ABI: don't break behavior, dump/restore, pg_upgrade, libpq wire protocol, logical-replication protocol, or exported/PGDLLIMPORT'd APIs (struct size/layout, function signatures) without extraordinary justification.\n\n(3) CATALOG / GENERATED: new/changed catalog data goes in pg_*.dat, NOT the generated *_d.h/.bki. New Node types: ANNOTATE the struct in its header so gen_node_support.pl regenerates copy/equal/out/read ā€” do NOT hand-edit *funcs.c. New SQL-callable functions: add to pg_proc.dat with an OID from the 8000-9999 developer range (src/include/catalog/unused_oids; check duplicate_oids); committer renumbers at commit. DO NOT bump CATALOG_VERSION_NO in the patch ā€” flag any catversion.h change as a mistake (committer's job).\n\n(4) PERFORMANCE: no regression on hot paths; avoid O(n^2) where better is feasible; minimize work under contended locks; avoid needless palloc churn and large struct copies in hot paths.\n\n(5) SECURITY: bounded string ops (snprintf/strlcpy/strlcat ā€” never strcpy/strcat/sprintf); integer/size-overflow checks before allocation; never user input as a format string; privilege checks via pg_*_aclcheck; beware search_path and SECURITY DEFINER.\n\n(6) PORTABILITY (hard gate): no unaligned access; no dependence on char signedness, int/long/pointer width, endianness, or struct padding for on-disk/wire formats; use int16/int32/int64 + INT64_FORMAT/UINT64_FORMAT (never %ld for int64); align contended shared structs (pg_attribute_aligned/cache-line pad). Atomics/barriers only via port/atomics (pg_atomic_*, pg_read/write_barrier) ā€” never raw intrinsics or volatile-as-barrier. WINDOWS/MSVC: extern vars used cross-module/extension need PGDLLIMPORT; no VLAs or features beyond the C99 baseline the tree targets; use pg_pread/pg_pwrite. Applies across x86_64/ARM64/RISC-V/PPC64/s390x, big/little endian, 32/64-bit.\n\n(7) CONVENTIONS: errmsg starts lowercase, no trailing period, no embedded newlines; errdetail/errhint are complete capitalized sentences; correct ERRCODE_*; wrap user-facing text in _(); errmsg_plural for counts. Assert() only for can't-happen invariants (never user-reachable). Naming: snake_case with subsystem prefix (heap_insert) or CamelCase for major subsystems (ExecInitNode); ALL_CAPS macros. Must pgindent cleanly (tabs, width 4). Comments explain WHY not WHAT; no #ifdef 0 blocks, no commented-out code, no #ifdef fencing your feature. Reuse existing helpers (DRY)." + }, + { + "path": "**/*.dat", + "merge_system_rule": true, + "rule": "REVIEW DISCIPLINE: Precise, blunt, verify against the diff, tag confidence, no praise. PostgreSQL catalog data (pg_proc.dat, pg_type.dat, etc.) ā€” the SOURCE for generated headers. The generated *_d.h, postgres.bki, fmgroids.h, fmgrtab.c must NOT be hand-edited (they regenerate from these files). OIDs: use a value from the developer range 8000-9999 (src/include/catalog/unused_oids; verify with duplicate_oids); committer renumbers to a final contiguous block, so stay in-range and unique but don't over-optimize the exact number. Keep proc entries complete/consistent (prosrc, provolatile, proparallel, prorettype/proargtypes, matching description). DO NOT bump CATALOG_VERSION_NO / catversion.h ā€” committer's job at push time; flag any such change. New catalog columns/views need documentation in doc/src/sgml/catalogs.sgml." + }, + { + "path": "**/*.{sql,pgsql}", + "merge_system_rule": true, + "rule": "REVIEW DISCIPLINE: Precise, blunt, verify against the diff, tag confidence, no praise. PostgreSQL SQL. Valid PostgreSQL dialect (not MySQL/Oracle); correct types (bigint vs int, text vs varchar); sound transaction/isolation and CTE-materialization assumptions. SECURITY: flag SQL injection in dynamic SQL (require quote_identifier/quote_literal or format() with %I/%L), SECURITY DEFINER without a locked-down search_path, inappropriate RLS bypass. Prefer set-based over row-at-a-time/N+1. BACKWARD COMPATIBILITY (a top rejection reason): changing existing SQL behavior, the output of existing functions, or default GUCs needs extraordinary justification. New SQL-callable objects belong in pg_*.dat with OIDs from the 8000-9999 range, not in generated files. Minimal diff; add regression tests + docs." + }, + { + "path": "**/*.{pl,pm}", + "merge_system_rule": true, + "rule": "REVIEW DISCIPLINE: Precise, blunt, verify against the diff, tag confidence, no praise. PostgreSQL Perl (TAP tests and build/catalog tooling). Require 'use strict; use warnings;'. Must be perltidy-clean with the tree's src/tools/pgindent/perltidyrc and pass src/tools/perlcheck/pgperlcritic. Use the framework: PostgreSQL::Test::Cluster, PostgreSQL::Test::Utils, Test::More; no hardcoded ports/paths/PIDs; use safe_psql/poll_query_until, not sleep; skippable without optional prerequisites; clean up nodes. PORTABILITY: run on Windows (no fork-only constructs, use File::Spec, avoid unavailable signals) and the minimum supported Perl. Robustness: avoid two-arg open and string system()/qx with interpolated data (use list forms). Generator scripts (gen_node_support.pl, catalog Perl) must be deterministic and stay in sync with inputs; do not commit their generated output." + }, + { + "path": "**/*.py", + "merge_system_rule": true, + "rule": "REVIEW DISCIPLINE: Precise, blunt, verify against the diff, tag confidence, no praise. PostgreSQL Python (build/test tooling, oauth/pytest tests, src/tools). Follow surrounding style; keep imports to the standard library unless the dependency is already required by the tree (no surprise third-party deps in build/test tooling). PORTABILITY: support the project's minimum Python 3 and run on Windows and the BSDs (use os.path/pathlib, avoid POSIX-only calls and shell=True with interpolated input). Deterministic, self-cleaning tests; no hardcoded ports/paths; skip cleanly without prerequisites. For the Perl->pytest porting effort, confirm behavior parity with the TAP test replaced (same assertions/coverage), not a superficial translation. Minimal diff; match the tree's ruff/black config if present." + }, + { + "path": "**/*.{rs,toml}", + "merge_system_rule": true, + "rule": "REVIEW DISCIPLINE: Precise, blunt, verify against the diff, tag confidence, no praise. Rust PostgreSQL extension (pgrx) or Rust support crate. Not core C, but it runs inside/alongside the backend, so backend safety applies. SAFETY: in code reachable from an SQL call, a Rust panic aborts the Postgres process ā€” forbid unwrap()/expect()/panic!/unreachable!/todo! and index-panics on reachable paths; use Result and pgrx error reporting (error!/ereport!). Every `unsafe` block needs a comment justifying its invariant; scrutinize raw pointers and FFI across the pg_sys boundary. pgrx: honor #[pg_guard] on extern C fns (correct panic/longjmp handling); never hold Rust references across SPI or anything that can longjmp (skips Rust destructors -> leaks); respect MemoryContext lifetimes for palloc'd data; datum<->Rust conversions must handle NULL. Concurrency uses Postgres shmem/LWLocks (pgrx shmem API), not std::sync alone. Lints: must pass `cargo clippy --all-targets --all-features -- -D warnings` and `cargo fmt --check`; deny unwrap_used/expect_used/panic in libraries; thiserror (libs) / anyhow (bins). Justify every new dependency. Tests: #[pg_test] for in-backend behavior, #[test] for pure logic; cover error and NULL paths. Minimal, idiomatic diff." + }, + { + "path": "**/{configure.ac,*.m4,aclocal.m4}", + "merge_system_rule": true, + "rule": "REVIEW DISCIPLINE: Precise, blunt, verify against the diff, tag confidence, no praise. PostgreSQL Autoconf. Edit configure.ac / the m4 macros ā€” do NOT hand-edit generated 'configure' or pg_config.h.in in the same patch (regeneration is the committer's step; a patch that also rewrites generated configure output is suspect). Feature/header/function probes must be portable and not assume a specific OS/compiler. Every configure knob must be mirrored on the Meson side (meson_options.txt/meson.build) and documented. Minimal diff." + }, + { + "path": "**/{meson.build,meson_options.txt}", + "merge_system_rule": true, + "rule": "REVIEW DISCIPLINE: Precise, blunt, verify against the diff, tag confidence, no praise. PostgreSQL Meson build. Valid syntax; correct subdir()/dependency()/declare_dependency and install paths; new source files must be added here. CRITICAL: PostgreSQL maintains BOTH Meson and Autoconf/Make ā€” any new file, option, or feature check must be mirrored on the configure.ac/Makefile side so the two never drift (a file built by only one system is a common defect). New options need matching docs and sensible defaults. Minimal diff." + }, + { + "path": "**/{Makefile,GNUmakefile,*.mk}", + "merge_system_rule": true, + "rule": "REVIEW DISCIPLINE: Precise, blunt, verify against the diff, tag confidence, no praise. PostgreSQL Makefile (GNU Make). $(VAR) refs; correct .PHONY; accurate dependencies (no parallel -j races); $(MAKE) for recursion; VPATH/out-of-tree build support; no hardcoded paths (use standard PostgreSQL makefile vars and $(top_builddir)); clean/distclean/maintainer-clean must remove new artifacts; extensions use PGXS. Must stay in sync with meson.build. Minimal diff." + }, + { + "path": "doc/**/*.sgml", + "merge_system_rule": true, + "rule": "REVIEW DISCIPLINE: Precise, blunt, verify against the diff, tag confidence, no praise. PostgreSQL documentation (DocBook SGML). Technically accurate/complete (parameters, limitations, version/compat notes); correct tag usage/nesting (, , , , , /); working cross-references; spell it 'PostgreSQL' in prose; SQL keywords uppercase in examples. Coverage: a new GUC -> config.sgml (and postgresql.conf.sample); new/changed catalogs or views -> catalogs.sgml; new SQL syntax -> the matching ref/*.sgml; new functions -> func.sgml. Do NOT edit release-notes (release-*.sgml) ā€” written by the release team/committers; flag such edits. New user-facing behavior in this PR should ship with matching docs." + }, + { + "path": "**/*.md", + "merge_system_rule": true, + "rule": "REVIEW DISCIPLINE: Precise, blunt, verify against the diff, tag confidence, no praise. Markdown docs. Clear heading hierarchy; fenced code blocks with language hints; accurate instructions/prerequisites; consistent PostgreSQL terminology; no broken relative links or stale claims. Minimal diff." + } + ] +} diff --git a/.github/workflows/ocr-model-check.yml b/.github/workflows/ocr-model-check.yml new file mode 100644 index 0000000000000..10d250528cf7c --- /dev/null +++ b/.github/workflows/ocr-model-check.yml @@ -0,0 +1,89 @@ +# Checks AWS Bedrock weekly for a newer Claude Opus inference profile than the +# one OCR currently uses (vars.OCR_BEDROCK_MODEL) and, if found, opens/updates a +# single GitHub issue telling the maintainer to bump the variable. It does NOT +# change the model automatically: GITHUB_TOKEN cannot write Actions *variables* +# (that needs a PAT with admin), so this is a notify-only mechanism by design. +name: OCR model self-check + +on: + schedule: + - cron: '0 12 * * 1' # Mondays 12:00 UTC + workflow_dispatch: + +permissions: + id-token: write + contents: read + issues: write + +jobs: + check-model: + runs-on: ubuntu-latest + steps: + - name: Configure AWS credentials (OIDC) + uses: aws-actions/configure-aws-credentials@v6 + with: + role-to-assume: ${{ vars.AWS_ROLE_ARN }} + aws-region: ${{ vars.AWS_REGION }} + role-session-name: ocr-model-check-${{ github.run_id }} + + - name: Find newest Opus vs configured + id: check + env: + CURRENT: ${{ vars.OCR_BEDROCK_MODEL }} + AWS_REGION: ${{ vars.AWS_REGION }} + run: | + python3 - <<'PY' >> "$GITHUB_OUTPUT" + import os, re, subprocess, json + region = os.environ.get("AWS_REGION", "us-east-1") + current = os.environ.get("CURRENT", "") + out = subprocess.run( + ["aws", "bedrock", "list-inference-profiles", "--region", region, + "--query", "inferenceProfileSummaries[].inferenceProfileId", "--output", "json"], + capture_output=True, text=True) + ids = json.loads(out.stdout or "[]") + # Parse claude-opus-- from any profile id (prefix us./global. ok). + def ver(s): + m = re.search(r"claude-opus-(\d+)-(\d+)", s) + return (int(m.group(1)), int(m.group(2))) if m else None + opus = [(ver(i), i) for i in ids if ver(i) and i.startswith(("us.", "global."))] + if not opus: + print("newer=false"); raise SystemExit(0) + best_ver, best_id = max(opus, key=lambda x: x[0]) + cur = ver(current) + newer = (cur is None) or (best_ver > cur) + print(f"newer={'true' if newer else 'false'}") + print(f"best_id={best_id}") + print(f"best_ver={best_ver[0]}.{best_ver[1]}") + print(f"cur_ver={'unknown' if cur is None else f'{cur[0]}.{cur[1]}'}") + PY + + - name: Open/update issue if a newer model exists + if: steps.check.outputs.newer == 'true' + uses: actions/github-script@v9 + with: + script: | + const best = '${{ steps.check.outputs.best_id }}'; + const bestVer = '${{ steps.check.outputs.best_ver }}'; + const curVer = '${{ steps.check.outputs.cur_ver }}'; + const marker = ''; + const title = `OCR: newer Claude Opus available (${bestVer} > ${curVer})`; + const body = `${marker}\n` + + `A newer Claude Opus inference profile is available on Bedrock.\n\n` + + `- **Configured** (\`vars.OCR_BEDROCK_MODEL\`): Opus ${curVer}\n` + + `- **Newest on Bedrock**: \`${best}\` (Opus ${bestVer})\n\n` + + `To upgrade, set the repo variable:\n\n` + + '```\n' + + `gh variable set OCR_BEDROCK_MODEL -R ${context.repo.owner}/${context.repo.repo} \\\n` + + ` -b "bedrock/converse/${best}"\n` + + '```\n\n' + + `Also confirm the \`ocr-bedrock-ci\` IAM inline policy allows invoking the new model ` + + `(the resource is scoped to \`anthropic.claude-opus-*\`), then re-run OCR.\n\n` + + `_Automated by \`.github/workflows/ocr-model-check.yml\`; this issue is upserted._`; + const q = `repo:${context.repo.owner}/${context.repo.repo} in:body "${marker}" state:open`; + const found = await github.rest.search.issuesAndPullRequests({ q, per_page: 1 }); + if (found.data.total_count > 0) { + const n = found.data.items[0].number; + await github.rest.issues.update({ owner: context.repo.owner, repo: context.repo.repo, issue_number: n, title, body }); + } else { + await github.rest.issues.create({ owner: context.repo.owner, repo: context.repo.repo, title, body }); + } diff --git a/.github/workflows/ocr-review.yml b/.github/workflows/ocr-review.yml new file mode 100644 index 0000000000000..0828af429b57c --- /dev/null +++ b/.github/workflows/ocr-review.yml @@ -0,0 +1,427 @@ +# Open Code Review (OCR) ā€” AI PR review backed by AWS Bedrock via a LiteLLM proxy. +# +# Flow: +# PR opened/updated (incl. DRAFTS) ā”€ā” +# /open-code-review PR comment ā”€ā”¼ā”€ā–ŗ start LiteLLM (127.0.0.1:4000 ā†’ Bedrock) +# manual workflow_dispatch ā”€ā”˜ ā””ā–ŗ ocr review --format json +# ā””ā–ŗ post inline PR review comments +# +# Required (repo settings ā€” all repo *variables*, no secrets; auth is via GitHub OIDC): +# vars.AWS_ROLE_ARN - IAM role to assume via OIDC (granting bedrock:InvokeModel*) +# vars.AWS_REGION - e.g. us-east-1 +# vars.OCR_BEDROCK_MODEL - LiteLLM model string for the Opus inference profile, e.g. +# bedrock/converse/us.anthropic.claude-opus-4-8 +# +# No static AWS keys are stored. GITHUB_TOKEN (auto) posts the review comments. + +name: OCR AI Review + +on: + pull_request: + # Note: no draft filter ā€” drafts are reviewed too. + types: [opened, synchronize, reopened, ready_for_review] + issue_comment: + types: [created] + workflow_dispatch: + inputs: + pr_number: + description: 'PR number to review' + required: true + type: number + +# One review per PR; cancel superseded runs to save Bedrock spend. +concurrency: + group: ocr-review-${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }} + cancel-in-progress: true + +permissions: + id-token: write # required to mint the GitHub OIDC token for AWS role assumption + contents: read + pull-requests: write + +jobs: + ocr-review: + runs-on: ubuntu-latest + # PR events always; comment events only when the comment is on a PR and + # starts with the trigger keyword; manual dispatch always. + if: | + github.event_name == 'pull_request' || + github.event_name == 'workflow_dispatch' || + (github.event_name == 'issue_comment' && github.event.issue.pull_request && + (startsWith(github.event.comment.body, '/open-code-review') || + startsWith(github.event.comment.body, '@open-code-review'))) + + env: + # LiteLLM listens on localhost only; this key never leaves the runner. + LITELLM_MASTER_KEY: sk-ocr-ci-local + OCR_BEDROCK_MODEL: ${{ vars.OCR_BEDROCK_MODEL }} + # Region is a static var (safe at job level). AWS credentials are NOT set + # here ā€” they're minted by the OIDC "Configure AWS credentials" step below + # and exported to the environment for the LiteLLM/boto3 Bedrock calls. + AWS_REGION: ${{ vars.AWS_REGION }} + + steps: + - name: Resolve PR context + id: pr + uses: actions/github-script@v9 + with: + script: | + let prNumber; + if (context.eventName === 'pull_request') { + prNumber = context.payload.pull_request.number; + } else if (context.eventName === 'issue_comment') { + prNumber = context.issue.number; + } else { + prNumber = parseInt('${{ github.event.inputs.pr_number }}', 10); + } + const { data: pr } = await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: prNumber, + }); + const { data: repo } = await github.rest.repos.get({ + owner: context.repo.owner, + repo: context.repo.repo, + }); + core.setOutput('number', String(prNumber)); + core.setOutput('base_ref', pr.base.ref); + core.setOutput('head_ref', pr.head.ref); + core.setOutput('head_sha', pr.head.sha); + core.setOutput('default_branch', repo.default_branch); + core.setOutput('cross_repo', String(pr.head.repo.full_name !== pr.base.repo.full_name)); + + # NOTE: do NOT checkout the PR head. OCR reads the diff and file contents + # straight from git refs (git diff , git show :path, + # git grep ), so the working tree is irrelevant ā€” but our OCR config + # lives on the default branch, not on the PR branch. We check out the repo + # (default ref), fetch the base/head objects, and materialize the config + # from origin/. + - name: Checkout + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Prepare git refs and OCR config + env: + BASE_REF: ${{ steps.pr.outputs.base_ref }} + HEAD_REF: ${{ steps.pr.outputs.head_ref }} + HEAD_SHA: ${{ steps.pr.outputs.head_sha }} + DEFAULT_BRANCH: ${{ steps.pr.outputs.default_branch }} + run: | + git fetch --no-tags origin "+refs/heads/${DEFAULT_BRANCH}:refs/remotes/origin/${DEFAULT_BRANCH}" || true + git fetch --no-tags origin "+refs/heads/${BASE_REF}:refs/remotes/origin/${BASE_REF}" || true + git fetch --no-tags origin "+refs/heads/${HEAD_REF}:refs/remotes/origin/${HEAD_REF}" || true + git fetch --no-tags origin "${HEAD_SHA}" || true + + # OCR config lives on the default branch; materialize it independently + # of whatever ref is checked out. + mkdir -p "$RUNNER_TEMP/ocr" + git show "origin/${DEFAULT_BRANCH}:.github/ocr/litellm.yaml" > "$RUNNER_TEMP/ocr/litellm.yaml" + git show "origin/${DEFAULT_BRANCH}:.github/ocr/rule.json" > "$RUNNER_TEMP/ocr/rule.json" + git show "origin/${DEFAULT_BRANCH}:.github/ocr/context.md" > "$RUNNER_TEMP/ocr/context.md" + echo "Config materialized:"; ls -l "$RUNNER_TEMP/ocr" + + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: '3.12' + + - name: Setup Node.js + uses: actions/setup-node@v6 + with: + node-version: '20' + + - name: Install LiteLLM proxy + Open Code Review + run: | + python -m pip install --upgrade pip + # Pin LiteLLM to a main commit that supports Claude Opus 4.8 adaptive + # thinking (maps reasoning_effort -> output_config.effort, incl. xhigh). + # Not in any tagged release yet (PyPI latest 1.87.1 lacks the Opus + # normalizer). Bump this SHA once a release ships the feature. + pip install "litellm[proxy] @ git+https://github.com/BerriAI/litellm.git@5be0797d24a2f26eb2123e13788f90055a59d91d" + npm install -g @alibaba-group/open-code-review + + - name: Configure AWS credentials (OIDC) + uses: aws-actions/configure-aws-credentials@v6 + with: + role-to-assume: ${{ vars.AWS_ROLE_ARN }} + aws-region: ${{ vars.AWS_REGION }} + role-session-name: ocr-review-${{ github.run_id }} + + - name: Start LiteLLM proxy (Bedrock bridge) + run: | + if [ -z "$OCR_BEDROCK_MODEL" ]; then + echo "::error::vars.OCR_BEDROCK_MODEL is not set (e.g. bedrock/converse/us.anthropic.claude-opus-4-1-20250805-v1:0)" + exit 1 + fi + nohup litellm --config "$RUNNER_TEMP/ocr/litellm.yaml" --host 127.0.0.1 --port 4000 \ + > /tmp/litellm.log 2>&1 & + echo "Waiting for LiteLLM to become ready..." + for i in $(seq 1 60); do + if curl -sf http://127.0.0.1:4000/health/readiness >/dev/null; then + echo "LiteLLM ready."; exit 0 + fi + sleep 2 + done + echo "::error::LiteLLM did not become ready in time"; cat /tmp/litellm.log; exit 1 + + - name: Configure OCR + run: | + ocr config set llm.url http://127.0.0.1:4000/v1/chat/completions + ocr config set llm.auth_token "$LITELLM_MASTER_KEY" + ocr config set llm.model ocr-bedrock + ocr config set llm.use_anthropic false + ocr config set language English + + - name: Run OCR review + run: | + ocr review \ + --from "origin/${{ steps.pr.outputs.base_ref }}" \ + --to "${{ steps.pr.outputs.head_sha }}" \ + --rule "$RUNNER_TEMP/ocr/rule.json" \ + --background-file "$RUNNER_TEMP/ocr/context.md" \ + --concurrency 3 \ + --timeout 20 \ + --format json \ + > /tmp/ocr-result.json 2>/tmp/ocr-stderr.log || true + echo "----- OCR stdout -----"; cat /tmp/ocr-result.json || true + echo "----- OCR stderr -----"; cat /tmp/ocr-stderr.log || true + echo "----- LiteLLM log (tail) -----"; tail -n 50 /tmp/litellm.log || true + + - name: Post review to PR + uses: actions/github-script@v9 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const fs = require('fs'); + const prNumber = parseInt('${{ steps.pr.outputs.number }}', 10); + const commitSha = '${{ steps.pr.outputs.head_sha }}'; + + // Opus at high effort can emit dozens of findings. Posting them all + // one-by-one trips GitHub's SECONDARY rate limit (403 "content + // creation"), which is what made every run fail after the review + // was already generated. We (a) cap inline comments and overflow + // the rest into the summary, (b) prefer a single bulk createReview, + // and (c) throttle + back off with Retry-After on any fallback. + const MAX_INLINE = 25; + const sleep = (ms) => new Promise(r => setTimeout(r, ms)); + + async function withRetry(fn, label) { + for (let attempt = 1; attempt <= 5; attempt++) { + try { return await fn(); } + catch (e) { + const status = e.status || (e.response && e.response.status); + const h = (e.response && e.response.headers) || {}; + const isRate = status === 403 || status === 429; + if (!isRate || attempt === 5) throw e; + let waitMs = 0; + if (h['retry-after']) waitMs = parseInt(h['retry-after'], 10) * 1000; + else if (h['x-ratelimit-reset']) waitMs = parseInt(h['x-ratelimit-reset'], 10) * 1000 - Date.now(); + if (!waitMs || Number.isNaN(waitMs) || waitMs < 0) waitMs = 1000 * Math.pow(2, attempt); + waitMs = Math.min(waitMs, 60000) + 500; + core.warning(`${label}: rate-limited (status ${status}); waiting ${Math.round(waitMs / 1000)}s (attempt ${attempt}/5)`); + await sleep(waitMs); + } + } + } + + let result; + try { + result = JSON.parse(fs.readFileSync('/tmp/ocr-result.json', 'utf8')); + } catch (e) { + const stderr = (() => { try { return fs.readFileSync('/tmp/ocr-stderr.log', 'utf8').trim(); } catch { return ''; } })(); + await withRetry(() => github.rest.issues.createComment({ + owner: context.repo.owner, repo: context.repo.repo, issue_number: prNumber, + body: `āš ļø **OCR** could not produce a review.\n\n\`\`\`\n${(stderr || e.message).slice(0, 8000)}\n\`\`\``, + }), 'error-comment'); + return; + } + + const comments = result.comments || []; + const warnings = result.warnings || []; + + const formatComment = (c) => { + let body = c.content || ''; + if (c.suggestion_code && c.existing_code) { + body += '\n\n```suggestion\n' + c.suggestion_code + (c.suggestion_code.endsWith('\n') ? '' : '\n') + '```'; + } + return body; + }; + const formatMarkdown = (c) => { + let md = `### šŸ“„ \`${c.path}\``; + if (c.start_line && c.end_line) md += ` (L${c.start_line}-L${c.end_line})`; + md += '\n\n' + (c.content || ''); + if (c.suggestion_code && c.existing_code) { + md += '\n\n
šŸ’” Suggested change\n\n'; + md += '**Before:**\n```\n' + c.existing_code + '\n```\n\n**After:**\n```\n' + c.suggestion_code + '\n```\n\n
'; + } + return md; + }; + + if (comments.length === 0) { + await withRetry(() => github.rest.issues.createComment({ + owner: context.repo.owner, repo: context.repo.repo, issue_number: prNumber, + body: `āœ… **OCR**: ${result.message || 'No issues found.'}`, + }), 'no-issues-comment'); + return; + } + + const inlineAll = []; + const noLine = []; + for (const c of comments) { + const body = formatComment(c); + const hasLine = (c.start_line >= 1) || (c.end_line >= 1); + if (!hasLine) { noLine.push(c); continue; } + const rc = { path: c.path, body, side: 'RIGHT' }; + if (c.start_line >= 1 && c.end_line >= 1 && c.start_line !== c.end_line) { + rc.start_line = c.start_line; rc.line = c.end_line; rc.start_side = 'RIGHT'; + } else { + rc.line = c.end_line >= 1 ? c.end_line : c.start_line; + } + inlineAll.push({ rc, c }); + } + + const inline = inlineAll.slice(0, MAX_INLINE).map(x => x.rc); + const overflow = inlineAll.slice(MAX_INLINE).map(x => x.c); + + let summary = `šŸ” **OCR** found **${comments.length}** issue(s).`; + summary += `\n- ${inline.length} inline, ${noLine.length + overflow.length} in summary`; + if (overflow.length) summary += ` (inline capped at ${MAX_INLINE})`; + if (warnings.length) summary += `\n- āš ļø ${warnings.length} warning(s) during review`; + for (const c of noLine.concat(overflow)) summary += '\n\n---\n\n' + formatMarkdown(c); + + // Preferred path: ONE createReview carrying every inline comment. + try { + await withRetry(() => github.rest.pulls.createReview({ + owner: context.repo.owner, repo: context.repo.repo, pull_number: prNumber, + commit_id: commitSha, body: summary, event: 'COMMENT', comments: inline, + }), 'bulk-review'); + return; + } catch (e) { + core.warning(`bulk createReview failed (${e.status || '?'}: ${e.message}); falling back to throttled per-comment posting`); + } + + // Fallback: an invalid inline position (line not in the diff -> 422) + // rejects the whole bulk review. Post the summary, then each comment + // individually with a delay + backoff, skipping ones GitHub rejects. + let ok = 0; const failed = []; + try { + await withRetry(() => github.rest.pulls.createReview({ + owner: context.repo.owner, repo: context.repo.repo, pull_number: prNumber, + commit_id: commitSha, body: summary, event: 'COMMENT', + }), 'summary-review'); + } catch (err) { failed.push(`summary: ${err.message}`); } + + for (const rc of inline) { + try { + await withRetry(() => github.rest.pulls.createReviewComment({ + owner: context.repo.owner, repo: context.repo.repo, pull_number: prNumber, + commit_id: commitSha, path: rc.path, body: rc.body, + ...(rc.start_line ? { start_line: rc.start_line, start_side: rc.start_side } : {}), + line: rc.line, side: rc.side, + }), `comment ${rc.path}:${rc.line}`); + ok++; + } catch (inner) { + failed.push(`\`${rc.path}\` L${rc.line}: ${inner.message}`); + } + await sleep(1200); // stay under the secondary content-creation limit + } + + if (failed.length) { + await withRetry(() => github.rest.issues.createComment({ + owner: context.repo.owner, repo: context.repo.repo, issue_number: prNumber, + body: `šŸ“Š OCR posted ${ok}/${inline.length} inline comment(s).\n\n
${failed.length} could not be posted\n\n${failed.join('\n')}\n
`, + }), 'summary-failures'); + } + + # Companion job: OCR can't call MCP, so this separate agent ties the PR's + # changes to PostgreSQL git + pgsql-hackers history via the Agora MCP server + # (pg.ddx.io) and posts a single, upserted "history & discussion" comment. + pg-history: + runs-on: ubuntu-latest + if: | + github.event_name == 'pull_request' || + github.event_name == 'workflow_dispatch' || + (github.event_name == 'issue_comment' && github.event.issue.pull_request && + (startsWith(github.event.comment.body, '/open-code-review') || + startsWith(github.event.comment.body, '@open-code-review') || + startsWith(github.event.comment.body, '/pg-history'))) + steps: + - name: Resolve PR context + id: pr + uses: actions/github-script@v9 + with: + script: | + let prNumber; + if (context.eventName === 'pull_request') prNumber = context.payload.pull_request.number; + else if (context.eventName === 'issue_comment') prNumber = context.issue.number; + else prNumber = parseInt('${{ github.event.inputs.pr_number }}', 10); + const { data: pr } = await github.rest.pulls.get({ + owner: context.repo.owner, repo: context.repo.repo, pull_number: prNumber }); + core.setOutput('number', String(prNumber)); + core.setOutput('base_ref', pr.base.ref); + core.setOutput('head_sha', pr.head.sha); + core.setOutput('title', pr.title || ''); + + - name: Checkout + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Make base/head refs available + env: + BASE_REF: ${{ steps.pr.outputs.base_ref }} + HEAD_SHA: ${{ steps.pr.outputs.head_sha }} + run: | + git fetch --no-tags origin "+refs/heads/${BASE_REF}:refs/remotes/origin/${BASE_REF}" || true + git fetch --no-tags origin "${HEAD_SHA}" || true + + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: '3.12' + + - name: Configure AWS credentials (OIDC) + uses: aws-actions/configure-aws-credentials@v6 + with: + role-to-assume: ${{ vars.AWS_ROLE_ARN }} + aws-region: ${{ vars.AWS_REGION }} + role-session-name: pg-history-${{ github.run_id }} + + - name: Install deps + run: pip install boto3 + + - name: Run pg-history (Agora MCP) + env: + PG_HISTORY_MODEL: ${{ vars.OCR_BEDROCK_MODEL }} + AWS_REGION: ${{ vars.AWS_REGION }} + BASE_REF: ${{ steps.pr.outputs.base_ref }} + HEAD_SHA: ${{ steps.pr.outputs.head_sha }} + GH_PR_TITLE: ${{ steps.pr.outputs.title }} + PG_HISTORY_OUT: ${{ runner.temp }}/pg-history.md + run: | + python .github/ocr/pg-history.py || true + echo "----- output -----"; cat "${{ runner.temp }}/pg-history.md" 2>/dev/null || echo "(no output)" + + - name: Upsert PR comment + uses: actions/github-script@v9 + with: + script: | + const fs = require('fs'); + const path = process.env.RUNNER_TEMP + '/pg-history.md'; + let body = ''; + try { body = fs.readFileSync(path, 'utf8').trim(); } catch (e) {} + if (!body) { console.log('pg-history: empty output, nothing to post'); return; } + const prNumber = parseInt('${{ steps.pr.outputs.number }}', 10); + const marker = ''; + body = marker + '\n' + body; + const { data: comments } = await github.rest.issues.listComments({ + owner: context.repo.owner, repo: context.repo.repo, issue_number: prNumber, per_page: 100 }); + const mine = comments.find(c => c.user.type === 'Bot' && c.body && c.body.includes(marker)); + if (mine) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, repo: context.repo.repo, comment_id: mine.id, body }); + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, repo: context.repo.repo, issue_number: prNumber, body }); + } From c06506bf2070d47d504f66e9a443be170d897fa9 Mon Sep 17 00:00:00 2001 From: Greg Burd Date: Mon, 27 Jul 2026 11:28:11 -0400 Subject: [PATCH 03/10] ci: use SYNC_TOKEN (workflow scope) for upstream sync push The default GITHUB_TOKEN is blocked by GitHub from pushing commits that touch .github/workflows/, which broke the auto-sync push step once the fork carried its own workflow files. Use a PAT with repo+workflow scope (SYNC_TOKEN secret), falling back to GITHUB_TOKEN when unset. --- .github/workflows/sync-upstream-manual.yml | 5 ++++- .github/workflows/sync-upstream.yml | 6 +++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/sync-upstream-manual.yml b/.github/workflows/sync-upstream-manual.yml index 362c119a128e7..c89d7ee23f53a 100644 --- a/.github/workflows/sync-upstream-manual.yml +++ b/.github/workflows/sync-upstream-manual.yml @@ -21,7 +21,10 @@ jobs: uses: actions/checkout@v4 with: fetch-depth: 0 - token: ${{ secrets.GITHUB_TOKEN }} + # PAT with repo + workflow scope. The default GITHUB_TOKEN cannot push + # commits that touch .github/workflows/ (platform block). See SYNC_TOKEN + # repo secret. + token: ${{ secrets.SYNC_TOKEN || secrets.GITHUB_TOKEN }} - name: Configure Git run: | diff --git a/.github/workflows/sync-upstream.yml b/.github/workflows/sync-upstream.yml index b3a6466980b0d..a763dfa5f994f 100644 --- a/.github/workflows/sync-upstream.yml +++ b/.github/workflows/sync-upstream.yml @@ -18,7 +18,11 @@ jobs: uses: actions/checkout@v4 with: fetch-depth: 0 - token: ${{ secrets.GITHUB_TOKEN }} + # PAT with repo + workflow scope. The default GITHUB_TOKEN cannot push + # commits that touch .github/workflows/ (platform block). See SYNC_TOKEN + # repo secret. Falls back to GITHUB_TOKEN if unset (push will fail on + # workflow-file changes, but non-workflow syncs still work). + token: ${{ secrets.SYNC_TOKEN || secrets.GITHUB_TOKEN }} - name: Configure Git run: | From f72d10babb5a9e89231d4774f3d985ca7b5bc679 Mon Sep 17 00:00:00 2001 From: Greg Burd Date: Mon, 27 Jul 2026 12:06:36 -0400 Subject: [PATCH 04/10] ci: bump actions/checkout to v5 in sync workflows Silences the Node 20 deprecation warning on the auto-sync runs. --- .github/workflows/sync-upstream-manual.yml | 2 +- .github/workflows/sync-upstream.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/sync-upstream-manual.yml b/.github/workflows/sync-upstream-manual.yml index c89d7ee23f53a..3139e517f409f 100644 --- a/.github/workflows/sync-upstream-manual.yml +++ b/.github/workflows/sync-upstream-manual.yml @@ -18,7 +18,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: fetch-depth: 0 # PAT with repo + workflow scope. The default GITHUB_TOKEN cannot push diff --git a/.github/workflows/sync-upstream.yml b/.github/workflows/sync-upstream.yml index a763dfa5f994f..eca408a7a11c5 100644 --- a/.github/workflows/sync-upstream.yml +++ b/.github/workflows/sync-upstream.yml @@ -15,7 +15,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: fetch-depth: 0 # PAT with repo + workflow scope. The default GITHUB_TOKEN cannot push From 8db60a88b699f0f7ca1c1f75c0ab8d1f1cf888ce Mon Sep 17 00:00:00 2001 From: Greg Burd Date: Thu, 9 Jul 2026 16:19:10 -0400 Subject: [PATCH 05/10] storage: add SeqLock, a sequence-lock reader/writer primitive A seqlock protects read-mostly, rarely-written shared data with a single sequence counter (even = stable, odd = write in progress). Readers take no lock and pay no atomic read-modify-write and no StoreLoad fence on the common path: they read the counter, copy the data into local variables, re-read the counter, and retry if it changed. A writer -- serialized by the caller's own mutex -- bumps the counter odd, mutates the single copy in place, and bumps it even. The choice of a seqlock here follows from the access pattern of the data it protects: state that is read on nearly every operation but written only rarely, where readers must never block writers and a stale read is cheaply detectable and retryable. Three models were considered: - An LWLock shared acquire costs a CAS (an atomic read-modify-write) on a shared counter even for pure readers, so every reader contends on the same cache line; under a read-heavy load that shared-counter traffic is the bottleneck, and readers can still block behind a waiting writer. - A left-right scheme keeps two copies and lets readers proceed wait-free against a writer, but each read pays a sequentially-consistent fence to publish which copy it is reading, and it doubles the memory and the writer's work (the writer must update both copies). - A seqlock keeps a single copy and a single counter; readers issue only plain loads plus two acquire-ordered counter reads and a compiler barrier, with no per-read atomic and no full fence. The price is that a reader may observe a torn value mid-write and must retry, so it fits only data whose writes are short and infrequent enough that retries are rare. That is exactly this data's profile, so the seqlock's lower read-side cost is the right trade. Documented in the lmgr README and covered by a test_seqlock module that verifies the counter transitions, the read/retry handshake, and torn-read rejection. --- src/backend/storage/lmgr/README | 17 ++ src/include/storage/seqlock.h | 182 ++++++++++++++++++ src/test/modules/Makefile | 1 + src/test/modules/meson.build | 1 + src/test/modules/test_seqlock/Makefile | 23 +++ .../test_seqlock/expected/test_seqlock.out | 7 + src/test/modules/test_seqlock/meson.build | 33 ++++ .../modules/test_seqlock/sql/test_seqlock.sql | 2 + .../test_seqlock/test_seqlock--1.0.sql | 8 + src/test/modules/test_seqlock/test_seqlock.c | 141 ++++++++++++++ .../modules/test_seqlock/test_seqlock.control | 4 + src/tools/pgindent/typedefs.list | 1 + 12 files changed, 420 insertions(+) create mode 100644 src/include/storage/seqlock.h create mode 100644 src/test/modules/test_seqlock/Makefile create mode 100644 src/test/modules/test_seqlock/expected/test_seqlock.out create mode 100644 src/test/modules/test_seqlock/meson.build create mode 100644 src/test/modules/test_seqlock/sql/test_seqlock.sql create mode 100644 src/test/modules/test_seqlock/test_seqlock--1.0.sql create mode 100644 src/test/modules/test_seqlock/test_seqlock.c create mode 100644 src/test/modules/test_seqlock/test_seqlock.control diff --git a/src/backend/storage/lmgr/README b/src/backend/storage/lmgr/README index 45de0fd2bd6f0..637643c0cf51b 100644 --- a/src/backend/storage/lmgr/README +++ b/src/backend/storage/lmgr/README @@ -36,6 +36,23 @@ Regular locks should be used for all user-driven lock requests. * SIReadLock predicate locks. See separate README-SSI file for details. +In addition to those four interprocess lock types, there is a specialized +reader/writer primitive: + +* Sequence locks (seqlocks; see src/include/storage/seqlock.h). A seqlock +protects read-mostly, rarely-written shared data with a single sequence +counter (even = stable, odd = write in progress). Readers take no lock and +pay no atomic read-modify-write and no StoreLoad fence on the common path: +they read the counter, copy the data into local variables, re-read the +counter, and retry if it changed. A writer bumps the counter odd, mutates +the single copy in place, and bumps it even; writer-vs-writer exclusion is +the caller's responsibility (typically an LWLock already held for other +reasons). Readers may retry if a writer intervenes, so a reader is not +wait-free and can be starved by continuous writers -- seqlocks suit +read-mostly data with short, infrequent writes. Because the data has a +single copy that the writer mutates in place, a reader must not retain +pointers into it across the re-check. + Acquisition of either a spinlock or a lightweight lock causes query cancel and die() interrupts to be held off until all such locks are released. No such restriction exists for regular locks, however. Also diff --git a/src/include/storage/seqlock.h b/src/include/storage/seqlock.h new file mode 100644 index 0000000000000..be951249a627a --- /dev/null +++ b/src/include/storage/seqlock.h @@ -0,0 +1,182 @@ +/*------------------------------------------------------------------------- + * + * seqlock.h + * Sequence lock: a low-overhead reader/writer primitive for + * read-mostly, rarely-written shared data. + * + * A seqlock protects data with a single sequence counter (even = stable, + * odd = write in progress). Readers take no lock and pay no atomic + * read-modify-write and no StoreLoad fence on the common path: they read + * the counter, read the data into local variables, re-read the counter, + * and retry if it changed. A writer bumps the counter to odd, mutates the + * data in place, and bumps it back to even. + * + * Trade-offs vs LWLock and vs the left-right lock: + * - Reads acquire no lock: two relaxed counter loads plus read barriers, + * no CAS, no spinlock, no StoreLoad fence. This is cheaper than a + * left-right read (which requires a per-read SeqCst fence) and far + * cheaper than an LWLock shared acquire (a CAS on a shared counter). + * - Readers may RETRY if a writer intervenes, so a reader is not + * wait-free and can be starved by a continuous stream of writers. + * Seqlocks are therefore suited to read-mostly data with short, + * infrequent writes. + * - The protected data has a SINGLE copy (unlike left-right's two), so a + * reader must copy the fields it needs into locals inside the read + * section and only use them after a clean re-check; it must not retain + * pointers into the protected data across the re-check, because the + * writer mutates that data in place. + * - Writer serialization is the CALLER's responsibility (e.g. an LWLock + * or spinlock already held for other reasons). The seqlock itself is + * only the reader/writer coordination counter; it provides no mutual + * exclusion between writers. + * + * Why a new primitive rather than LWLock or bare atomics: + * - The motivating consumer is a read-mostly, heavily-concurrent shared + * hash whose readers vastly outnumber writers and read several words + * (a struct) that must be observed as one consistent snapshot. An + * LWLock shared-acquire is a CAS (read-modify-write) on a shared + * counter on every read; under many readers that shared cache line + * ping-pongs between cores and becomes the bottleneck, even though no + * reader mutates anything. The seqlock read path issues only two + * relaxed loads of the counter plus read barriers -- no atomic RMW, no + * contended cache line -- so read throughput scales with cores. + * - Plain atomics alone cannot give a multi-word consistent snapshot: an + * atomic per field lets a reader observe a torn mix of one writer's + * old and new values across fields. The seqlock's begin/retry frames + * the whole struct read as a single versioned transaction, so a reader + * either sees one writer's complete update or retries. That is the + * property neither an LWLock (correct but not scalable for pure reads) + * nor per-field atomics (scalable but not snapshot-consistent) provide + * on their own. + * + * Memory ordering: the writer's odd store is followed by a write barrier + * before the data mutation, and a write barrier precedes the even store; + * the reader issues a read barrier after the initial (even) counter load + * and before the re-read. No StoreLoad (full/SeqCst) fence is required + * because a reader never announces itself to the writer. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/storage/seqlock.h + * + *------------------------------------------------------------------------- + */ +#ifndef SEQLOCK_H +#define SEQLOCK_H + +#ifdef FRONTEND +#error "seqlock.h may not be included from frontend code" +#endif + +#include "port/atomics.h" +#include "storage/s_lock.h" + +/* + * A sequence lock. Embed one alongside the data it protects. The counter + * is even when the data is stable and odd while a writer is mutating it. + */ +typedef struct SeqLock +{ + pg_atomic_uint32 seq; +} SeqLock; + +/* + * Initialize a seqlock to the stable (even) state. Call once before any + * concurrent access. + */ +static inline void +SeqLockInit(SeqLock *lock) +{ + pg_atomic_init_u32(&lock->seq, 0); +} + +/* ---------------------------------------------------------------- + * Writer API + * + * The caller MUST serialize writers by other means (an LWLock or spinlock + * held across the whole write). SeqLockWriteBegin/End only publish the + * write window to readers; they do not exclude concurrent writers. + * ---------------------------------------------------------------- + */ + +/* + * Begin a write. Bumps the counter to odd so concurrent readers retry, + * then a write barrier so the subsequent data mutation is not reordered + * before the odd store becomes visible. + */ +static inline void +SeqLockWriteBegin(SeqLock *lock) +{ + uint32 s = pg_atomic_read_u32(&lock->seq); + + Assert((s & 1) == 0); + pg_atomic_write_u32(&lock->seq, s + 1); + pg_write_barrier(); +} + +/* + * End a write. A write barrier ensures the data mutation is visible + * before the counter returns to even (+2 total from the matching Begin). + */ +static inline void +SeqLockWriteEnd(SeqLock *lock) +{ + uint32 s = pg_atomic_read_u32(&lock->seq); + + Assert((s & 1) == 1); + pg_write_barrier(); + pg_atomic_write_u32(&lock->seq, s + 1); +} + +/* ---------------------------------------------------------------- + * Reader API + * + * Usage: + * uint32 seq; + * do { + * seq = SeqLockReadBegin(lock); + * ... read protected fields into LOCAL variables only ... + * } while (!SeqLockReadRetry(lock, seq)); + * ... act on the locals here (a consistent snapshot) ... + * + * A reader must NOT act on values read inside the loop, nor retain + * pointers into the protected data, until SeqLockReadRetry() has confirmed + * a consistent read (returned true). + * ---------------------------------------------------------------- + */ + +/* + * Begin a read. Spins while a writer is active (odd counter) and returns + * the even counter value observed. A read barrier orders the subsequent + * data reads after the counter load. + */ +static inline uint32 +SeqLockReadBegin(SeqLock *lock) +{ + uint32 s; + + for (;;) + { + s = pg_atomic_read_u32(&lock->seq); + if ((s & 1) == 0) + break; + SPIN_DELAY(); + } + pg_read_barrier(); + return s; +} + +/* + * Finish a read. Returns true if the read was consistent (no writer + * intervened since SeqLockReadBegin returned 'startseq'); false if the + * caller must discard its locals and retry. + */ +static inline bool +SeqLockReadRetry(SeqLock *lock, uint32 startseq) +{ + pg_read_barrier(); + return pg_atomic_read_u32(&lock->seq) == startseq; +} + +#endif /* SEQLOCK_H */ diff --git a/src/test/modules/Makefile b/src/test/modules/Makefile index 098bb8142ae76..797e8c7fb51a6 100644 --- a/src/test/modules/Makefile +++ b/src/test/modules/Makefile @@ -50,6 +50,7 @@ SUBDIRS = \ test_rls_hooks \ test_saslprep \ test_shmem \ + test_seqlock \ test_shm_mq \ test_slru \ test_tidstore \ diff --git a/src/test/modules/meson.build b/src/test/modules/meson.build index 4bca42bb3706a..297f38de63cdd 100644 --- a/src/test/modules/meson.build +++ b/src/test/modules/meson.build @@ -51,6 +51,7 @@ subdir('test_resowner') subdir('test_rls_hooks') subdir('test_saslprep') subdir('test_shmem') +subdir('test_seqlock') subdir('test_shm_mq') subdir('test_slru') subdir('test_tidstore') diff --git a/src/test/modules/test_seqlock/Makefile b/src/test/modules/test_seqlock/Makefile new file mode 100644 index 0000000000000..8bf2e3ffe7e30 --- /dev/null +++ b/src/test/modules/test_seqlock/Makefile @@ -0,0 +1,23 @@ +# src/test/modules/test_seqlock/Makefile + +MODULE_big = test_seqlock +OBJS = \ + $(WIN32RES) \ + test_seqlock.o +PGFILEDESC = "test_seqlock - test code for src/include/storage/seqlock.h" + +EXTENSION = test_seqlock +DATA = test_seqlock--1.0.sql + +REGRESS = test_seqlock + +ifdef USE_PGXS +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) +else +subdir = src/test/modules/test_seqlock +top_builddir = ../../../.. +include $(top_builddir)/src/Makefile.global +include $(top_srcdir)/contrib/contrib-global.mk +endif diff --git a/src/test/modules/test_seqlock/expected/test_seqlock.out b/src/test/modules/test_seqlock/expected/test_seqlock.out new file mode 100644 index 0000000000000..acf17ab1c740e --- /dev/null +++ b/src/test/modules/test_seqlock/expected/test_seqlock.out @@ -0,0 +1,7 @@ +CREATE EXTENSION test_seqlock; +SELECT test_seqlock(); + test_seqlock +-------------- + +(1 row) + diff --git a/src/test/modules/test_seqlock/meson.build b/src/test/modules/test_seqlock/meson.build new file mode 100644 index 0000000000000..c7683e8e008c8 --- /dev/null +++ b/src/test/modules/test_seqlock/meson.build @@ -0,0 +1,33 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +test_seqlock_sources = files( + 'test_seqlock.c', +) + +if host_system == 'windows' + test_seqlock_sources += rc_lib_gen.process(win32ver_rc, extra_args: [ + '--NAME', 'test_seqlock', + '--FILEDESC', 'test_seqlock - test code for src/include/storage/seqlock.h',]) +endif + +test_seqlock = shared_module('test_seqlock', + test_seqlock_sources, + kwargs: pg_test_mod_args, +) +test_install_libs += test_seqlock + +test_install_data += files( + 'test_seqlock.control', + 'test_seqlock--1.0.sql', +) + +tests += { + 'name': 'test_seqlock', + 'sd': meson.current_source_dir(), + 'bd': meson.current_build_dir(), + 'regress': { + 'sql': [ + 'test_seqlock', + ], + }, +} diff --git a/src/test/modules/test_seqlock/sql/test_seqlock.sql b/src/test/modules/test_seqlock/sql/test_seqlock.sql new file mode 100644 index 0000000000000..a445cf7317426 --- /dev/null +++ b/src/test/modules/test_seqlock/sql/test_seqlock.sql @@ -0,0 +1,2 @@ +CREATE EXTENSION test_seqlock; +SELECT test_seqlock(); diff --git a/src/test/modules/test_seqlock/test_seqlock--1.0.sql b/src/test/modules/test_seqlock/test_seqlock--1.0.sql new file mode 100644 index 0000000000000..575cec9559a83 --- /dev/null +++ b/src/test/modules/test_seqlock/test_seqlock--1.0.sql @@ -0,0 +1,8 @@ +/* src/test/modules/test_seqlock/test_seqlock--1.0.sql */ + +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION test_seqlock" to load this file. \quit + +CREATE FUNCTION test_seqlock() + RETURNS pg_catalog.void + AS 'MODULE_PATHNAME' LANGUAGE C; diff --git a/src/test/modules/test_seqlock/test_seqlock.c b/src/test/modules/test_seqlock/test_seqlock.c new file mode 100644 index 0000000000000..c00a02afca70a --- /dev/null +++ b/src/test/modules/test_seqlock/test_seqlock.c @@ -0,0 +1,141 @@ +/*-------------------------------------------------------------------------- + * + * test_seqlock.c + * Test module for the sequence lock (storage/seqlock.h). + * + * These are single-process unit tests of the seqlock protocol contract: + * the counter transitions, the read/retry handshake, and a simulated + * writer-interleaving that must force a reader retry. Cross-process + * stress is covered by the sLog tuple-tracking tests that exercise the + * seqlock under real concurrency. + * + * Copyright (c) 2026, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/test/modules/test_seqlock/test_seqlock.c + * + * ------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "fmgr.h" +#include "storage/seqlock.h" + +PG_MODULE_MAGIC; + +#define EXPECT_TRUE(expr) \ + do { \ + if (!(expr)) \ + elog(ERROR, "%s was unexpectedly false in file \"%s\" line %u", \ + #expr, __FILE__, __LINE__); \ + } while (0) + +#define EXPECT_EQ_U32(a, b) \ + do { \ + uint32 a_ = (a), b_ = (b); \ + if (a_ != b_) \ + elog(ERROR, "%u != %u (%s vs %s) in file \"%s\" line %u", \ + a_, b_, #a, #b, __FILE__, __LINE__); \ + } while (0) + +/* + * A tiny protected payload: two fields a writer keeps in a known invariant + * (b == a + 1) so a reader can detect a torn read. + */ +typedef struct GuardedData +{ + SeqLock lock; + uint64 a; + uint64 b; +} GuardedData; + +static void +guarded_write(GuardedData *g, uint64 v) +{ + SeqLockWriteBegin(&g->lock); + g->a = v; + g->b = v + 1; + SeqLockWriteEnd(&g->lock); +} + +/* Consistent read; loops until it observes a torn-free snapshot. */ +static void +guarded_read(GuardedData *g, uint64 *a_out, uint64 *b_out) +{ + uint32 seq; + + do + { + seq = SeqLockReadBegin(&g->lock); + *a_out = g->a; + *b_out = g->b; + } while (!SeqLockReadRetry(&g->lock, seq)); +} + +PG_FUNCTION_INFO_V1(test_seqlock); +Datum +test_seqlock(PG_FUNCTION_ARGS) +{ + GuardedData g; + uint64 a, + b; + uint32 s0, + s1; + + /* Init: counter even (stable). */ + SeqLockInit(&g.lock); + g.a = 0; + g.b = 1; + EXPECT_TRUE((pg_atomic_read_u32(&g.lock.seq) & 1) == 0); + + /* A begin/end pair advances the counter by 2 and leaves it even. */ + s0 = pg_atomic_read_u32(&g.lock.seq); + SeqLockWriteBegin(&g.lock); + EXPECT_TRUE((pg_atomic_read_u32(&g.lock.seq) & 1) == 1); /* odd mid-write */ + SeqLockWriteEnd(&g.lock); + s1 = pg_atomic_read_u32(&g.lock.seq); + EXPECT_EQ_U32(s1, s0 + 2); + EXPECT_TRUE((s1 & 1) == 0); + + /* A clean read (no interleaving writer) succeeds on the first try. */ + s0 = SeqLockReadBegin(&g.lock); + a = g.a; + b = g.b; + EXPECT_TRUE(SeqLockReadRetry(&g.lock, s0)); + EXPECT_TRUE(b == a + 1); + + /* Values a writer stored are visible to a subsequent consistent read. */ + guarded_write(&g, 42); + guarded_read(&g, &a, &b); + EXPECT_EQ_U32((uint32) a, 42); + EXPECT_EQ_U32((uint32) b, 43); + EXPECT_TRUE(b == a + 1); + + /* + * Simulate a writer interleaving between a reader's begin and retry: the + * retry must FAIL (return false), forcing the reader to loop. This is + * the core seqlock guarantee -- a snapshot spanning a write is rejected. + */ + s0 = SeqLockReadBegin(&g.lock); + guarded_write(&g, 100); /* writer completes a full cycle in the window */ + EXPECT_TRUE(!SeqLockReadRetry(&g.lock, s0)); + + /* + * After the retry-forced loop, the reader gets the invariant-holding + * pair. + */ + guarded_read(&g, &a, &b); + EXPECT_EQ_U32((uint32) a, 100); + EXPECT_TRUE(b == a + 1); + + /* Many begin/end cycles keep the counter even and monotonically rising. */ + { + uint32 before = pg_atomic_read_u32(&g.lock.seq); + + for (int i = 0; i < 1000; i++) + guarded_write(&g, (uint64) i); + EXPECT_EQ_U32(pg_atomic_read_u32(&g.lock.seq), before + 2000); + } + + PG_RETURN_VOID(); +} diff --git a/src/test/modules/test_seqlock/test_seqlock.control b/src/test/modules/test_seqlock/test_seqlock.control new file mode 100644 index 0000000000000..c919694696cae --- /dev/null +++ b/src/test/modules/test_seqlock/test_seqlock.control @@ -0,0 +1,4 @@ +comment = 'Test code for sequence lock' +default_version = '1.0' +module_pathname = '$libdir/test_seqlock' +relocatable = true diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 85d989f395d41..105269246cc3c 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2823,6 +2823,7 @@ Selectivity SelfJoinCandidate SemTPadded SemiAntiJoinFactors +SeqLock SeqScan SeqScanInstrumentation SeqScanState From 508deac53e16c39f91804e3a5979b5de1177ad83 Mon Sep 17 00:00:00 2001 From: Greg Burd Date: Tue, 23 Jun 2026 09:12:10 -0400 Subject: [PATCH 06/10] UNDO: add table-AM capability flags for cluster-wide UNDO Introduce two TableAmRoutine booleans and the begin_bulk_insert callback that the UNDO subsystem builds on, plus the RelationAmSupportsUndo() accessor index AMs use to gate UNDO record generation on the parent table. am_supports_undo marks an AM that registers an UNDO resource manager and emits UNDO records tagged with its own rmid; the UNDO core stays AM-agnostic and interprets the payload only through that RM's callbacks. am_inplace_update_keeps_tid marks an AM that updates in place and keeps the row's TID, so the executor can skip redundant index re-inserts for unchanged keys. The heap AM leaves both false. begin_bulk_insert is the start-of-bulk-DML counterpart to the existing finish_bulk_insert callback: an UNDO-supporting AM uses it to activate a batched UNDO write buffer for COPY and multi-row INSERT, coalescing per-row UNDO into larger records instead of one per tuple. It is null-checked in table_begin_bulk_insert(), so an AM that has no use for it (such as heap) simply leaves it unset. This commit only adds the routine fields and the accessor; no AM sets the flags or implements the callback yet. --- src/backend/access/table/tableam.c | 13 ++++++ src/include/access/tableam.h | 64 ++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+) diff --git a/src/backend/access/table/tableam.c b/src/backend/access/table/tableam.c index 68ff0966f1c57..37d0cfe45c283 100644 --- a/src/backend/access/table/tableam.c +++ b/src/backend/access/table/tableam.c @@ -823,3 +823,16 @@ table_block_relation_estimate_size(Relation rel, int32 *attr_widths, else *allvisfrac = (double) relallvisible / curpages; } + +/* + * RelationAmSupportsUndo + * Returns true if the relation's table AM declared UNDO support. + * Used by index AMs to gate UNDO record generation on the parent table. + */ +bool +RelationAmSupportsUndo(Relation rel) +{ + if (!rel->rd_tableam) + return false; + return rel->rd_tableam->am_supports_undo; +} diff --git a/src/include/access/tableam.h b/src/include/access/tableam.h index f2c36696bcad0..a4080509ddc54 100644 --- a/src/include/access/tableam.h +++ b/src/include/access/tableam.h @@ -323,6 +323,35 @@ typedef struct TableAmRoutine /* this must be set to T_TableAmRoutine */ NodeTag type; + /* + * am_supports_undo: true if this AM supports cluster-wide UNDO. + * + * An AM that sets this to true must: 1. Register an UNDO resource manager + * via RegisterUndoRmgr() (see src/include/access/undormgr.h) with an + * rm_undo callback that handles its own page format during rollback. 2. + * Write UNDO records tagged with its own urec_rmid so that undoapply.c + * dispatches to the correct apply handler. 3. Generate CLR (Compensation + * Log Records) in its rm_undo callback for crash-recovery idempotency. + * + * The UNDO infrastructure is AM-agnostic: UndoRecordHeader carries an + * opaque payload interpreted exclusively by the owning RM's callbacks. + * Each AM handles its own page format in its own rm_undo implementation. + * There is no requirement to use heap page layout. + * + * For UNDO record generation, AMs can either: (a) Use the shared Tier 2 + * buffer (UndoBufferAddRecord() / UndoBufferAddRecordParts() from + * undobuffer.h) to embed UNDO data into DML WAL records, or (b) Create a + * standalone UndoRecordSet for batched/deferred writes. + * + * How an AM decides whether UNDO is active for a given relation is + * AM-specific. The heap AM does not use UNDO (am_supports_undo = false). + * A future in-place-update AM will set am_supports_undo = true and + * register its own UNDO RM. + * + * See src/include/access/undormgr.h for the RM registration API and + * src/backend/access/undo/undoapply.c for the dispatch mechanism. + */ + bool am_supports_undo; /* ------------------------------------------------------------------------ * Slot related callbacks. @@ -599,6 +628,19 @@ typedef struct TableAmRoutine uint8 flags, TM_FailureData *tmfd); + /* + * Notify the AM that a bulk DML operation is about to begin. + * + * The AM can use this hint to pre-allocate resources, enable batched UNDO + * recording, or otherwise optimize for the expected workload. 'nrows' is + * the planner's estimate of the number of rows to be modified (0 means + * unknown). + * + * Optional callback. + */ + void (*begin_bulk_insert) (Relation rel, uint32 options, + int64 nrows); + /* * Perform operations necessary to complete insertions made via * tuple_insert and multi_insert with a BulkInsertState specified. In-tree @@ -1653,6 +1695,21 @@ table_tuple_lock(Relation rel, ItemPointer tid, Snapshot snapshot, flags, tmfd); } +/* + * Notify the AM that a bulk DML operation is about to begin. + * + * 'nrows' is the planner's row count estimate (0 = unknown). + * The AM may use this to pre-allocate UNDO buffers, enable batched + * recording, or other bulk-mode optimizations. + */ +static inline void +table_begin_bulk_insert(Relation rel, uint32 options, int64 nrows) +{ + /* optional callback */ + if (rel->rd_tableam && rel->rd_tableam->begin_bulk_insert) + rel->rd_tableam->begin_bulk_insert(rel, options, nrows); +} + /* * Perform operations necessary to complete insertions made via * tuple_insert and multi_insert with a BulkInsertState specified. @@ -2140,4 +2197,11 @@ extern const TableAmRoutine *GetTableAmRoutine(Oid amhandler); extern const TableAmRoutine *GetHeapamTableAmRoutine(void); +/* ---------------------------------------------------------------------------- + * Functions in tableam.c + * ---------------------------------------------------------------------------- + */ + +extern bool RelationAmSupportsUndo(Relation rel); + #endif /* TABLEAM_H */ From 80b242648871d65aa44f60e2b2e184d9dff708cd Mon Sep 17 00:00:00 2001 From: Greg Burd Date: Tue, 23 Jun 2026 09:12:12 -0400 Subject: [PATCH 07/10] UNDO: add cluster-wide UNDO-in-WAL engine with per-relation UNDO fork Add the AM-agnostic UNDO engine: the in-WAL UNDO record format and insertion path, the per-relation RELUNDO fork with its own resource manager, the shared sLog tuple-state map, the rollback apply driver and compensation-log generation, the discard horizon, and the background revert/undo workers. Register the UNDO, ATM, and RELUNDO resource managers and wire the subsystem into transaction start/commit/abort, two-phase commit, recovery, and process startup. The engine interprets UNDO payloads only through per-RM callbacks and the RelUndo*_hook function pointers (defined here, left NULL), so the core has no compile-time knowledge of any specific access method. The vacuum, pruning, reloptions, and executor integration points consume only the AM-agnostic interfaces. No UNDO-producing AM is registered yet: RegisterUndoRmgrs() initializes the dispatch table but registers no per-AM handlers. The index-AM apply handlers and the AM that sets am_supports_undo arrive in later commits. The design, record format, and recovery model (Constant-Time Recovery) are documented in src/backend/access/undo/README. This work derives substantially from the cluster-wide UNDO and zheap effort developed for PostgreSQL at EnterpriseDB; the in-WAL record format, discard horizon, and rollback/CLR machinery follow that design. Co-authored-by: Amit Kapila Co-authored-by: Dilip Kumar Co-authored-by: Thomas Munro Co-authored-by: Robert Haas Co-authored-by: Kuntal Ghosh Co-authored-by: Mahendra Singh Thalor Co-authored-by: Rafia Sabih Co-authored-by: Beena Emerson --- doc/src/sgml/filelist.sgml | 1 + doc/src/sgml/postgres.sgml | 1 + doc/src/sgml/undo.sgml | 738 +++++++++ examples/01-basic-undo-setup.sql | 42 + examples/02-undo-rollback.sql | 44 + examples/03-undo-subtransactions.sql | 45 + examples/05-undo-monitoring.sql | 38 + examples/README.md | 40 + src/backend/access/Makefile | 3 +- src/backend/access/meson.build | 1 + src/backend/access/rmgrdesc/Makefile | 3 + src/backend/access/rmgrdesc/atmdesc.c | 64 + src/backend/access/rmgrdesc/meson.build | 3 + src/backend/access/rmgrdesc/relundodesc.c | 140 ++ src/backend/access/rmgrdesc/undodesc.c | 209 +++ src/backend/access/transam/rmgr.c | 3 + src/backend/access/transam/twophase.c | 136 +- src/backend/access/transam/xact.c | 184 +++ src/backend/access/transam/xlog.c | 117 ++ src/backend/access/transam/xlogrecovery.c | 48 + src/backend/access/undo/Makefile | 39 + src/backend/access/undo/README | 1143 +++++++++++++ src/backend/access/undo/atm.c | 546 ++++++ .../access/undo/logical_revert_worker.c | 628 +++++++ src/backend/access/undo/meson.build | 26 + src/backend/access/undo/relundo.c | 1447 ++++++++++++++++ src/backend/access/undo/relundo_apply.c | 1057 ++++++++++++ src/backend/access/undo/relundo_discard.c | 536 ++++++ src/backend/access/undo/relundo_page.c | 360 ++++ src/backend/access/undo/relundo_recovery.c | 408 +++++ src/backend/access/undo/relundo_worker.c | 731 +++++++++ src/backend/access/undo/relundo_xlog.c | 760 +++++++++ src/backend/access/undo/slog.c | 805 +++++++++ src/backend/access/undo/undo.c | 278 ++++ src/backend/access/undo/undo_bufmgr.c | 250 +++ src/backend/access/undo/undo_xlog.c | 1461 +++++++++++++++++ src/backend/access/undo/undoapply.c | 326 ++++ src/backend/access/undo/undobuffer.c | 351 ++++ src/backend/access/undo/undoinsert.c | 166 ++ src/backend/access/undo/undolog.c | 534 ++++++ src/backend/access/undo/undorecord.c | 399 +++++ src/backend/access/undo/undormgr.c | 70 + src/backend/access/undo/undostats.c | 375 +++++ src/backend/access/undo/undoworker.c | 635 +++++++ src/backend/access/undo/xactundo.c | 1458 ++++++++++++++++ src/backend/commands/tablecmds.c | 1 + src/backend/executor/nodeModifyTable.c | 33 +- src/backend/postmaster/bgworker.c | 19 + src/backend/postmaster/postmaster.c | 9 + src/backend/storage/buffer/bufmgr.c | 135 ++ src/backend/tcop/utility.c | 1 + .../utils/activity/wait_event_names.txt | 7 + src/backend/utils/init/postinit.c | 4 + src/backend/utils/misc/guc_parameters.dat | 96 ++ src/backend/utils/misc/guc_tables.c | 4 + src/backend/utils/misc/postgresql.conf.sample | 14 + src/bin/pg_dump/pg_dump.c | 1 + src/bin/pg_waldump/relundodesc.c | 1 + src/bin/pg_waldump/rmgrdesc.c | 3 + src/bin/pg_waldump/t/001_basic.pl | 5 +- src/bin/pg_waldump/undodesc.c | 1 + src/common/relpath.c | 1 + src/include/access/atm.h | 64 + src/include/access/atm_xlog.h | 49 + src/include/access/heapam.h | 1 + src/include/access/heapam_xlog.h | 7 + src/include/access/logical_revert_worker.h | 43 + src/include/access/relundo.h | 697 ++++++++ src/include/access/relundo_worker.h | 89 + src/include/access/relundo_xlog.h | 164 ++ src/include/access/rmgrlist.h | 3 + src/include/access/slog.h | 103 ++ src/include/access/twophase.h | 3 + src/include/access/undo.h | 52 + src/include/access/undo_bufmgr.h | 297 ++++ src/include/access/undo_xlog.h | 332 ++++ src/include/access/undobuffer.h | 113 ++ src/include/access/undodefs.h | 56 + src/include/access/undolog.h | 197 +++ src/include/access/undorecord.h | 209 +++ src/include/access/undormgr.h | 118 ++ src/include/access/undormgrlist.h | 33 + src/include/access/undormgrs.h | 28 + src/include/access/undostats.h | 63 + src/include/access/undoworker.h | 66 + src/include/access/xact.h | 59 + src/include/access/xactundo.h | 117 ++ src/include/common/relpath.h | 5 +- src/include/storage/buf_internals.h | 14 + src/include/storage/bufmgr.h | 11 + src/include/storage/lwlocklist.h | 4 + src/include/storage/subsystemlist.h | 12 + src/test/regress/regress.c | 8 +- src/tools/pgindent/typedefs.list | 1 + 94 files changed, 19994 insertions(+), 8 deletions(-) create mode 100644 doc/src/sgml/undo.sgml create mode 100644 examples/01-basic-undo-setup.sql create mode 100644 examples/02-undo-rollback.sql create mode 100644 examples/03-undo-subtransactions.sql create mode 100644 examples/05-undo-monitoring.sql create mode 100644 examples/README.md create mode 100644 src/backend/access/rmgrdesc/atmdesc.c create mode 100644 src/backend/access/rmgrdesc/relundodesc.c create mode 100644 src/backend/access/rmgrdesc/undodesc.c create mode 100644 src/backend/access/undo/Makefile create mode 100644 src/backend/access/undo/README create mode 100644 src/backend/access/undo/atm.c create mode 100644 src/backend/access/undo/logical_revert_worker.c create mode 100644 src/backend/access/undo/meson.build create mode 100644 src/backend/access/undo/relundo.c create mode 100644 src/backend/access/undo/relundo_apply.c create mode 100644 src/backend/access/undo/relundo_discard.c create mode 100644 src/backend/access/undo/relundo_page.c create mode 100644 src/backend/access/undo/relundo_recovery.c create mode 100644 src/backend/access/undo/relundo_worker.c create mode 100644 src/backend/access/undo/relundo_xlog.c create mode 100644 src/backend/access/undo/slog.c create mode 100644 src/backend/access/undo/undo.c create mode 100644 src/backend/access/undo/undo_bufmgr.c create mode 100644 src/backend/access/undo/undo_xlog.c create mode 100644 src/backend/access/undo/undoapply.c create mode 100644 src/backend/access/undo/undobuffer.c create mode 100644 src/backend/access/undo/undoinsert.c create mode 100644 src/backend/access/undo/undolog.c create mode 100644 src/backend/access/undo/undorecord.c create mode 100644 src/backend/access/undo/undormgr.c create mode 100644 src/backend/access/undo/undostats.c create mode 100644 src/backend/access/undo/undoworker.c create mode 100644 src/backend/access/undo/xactundo.c create mode 120000 src/bin/pg_waldump/relundodesc.c create mode 120000 src/bin/pg_waldump/undodesc.c create mode 100644 src/include/access/atm.h create mode 100644 src/include/access/atm_xlog.h create mode 100644 src/include/access/logical_revert_worker.h create mode 100644 src/include/access/relundo.h create mode 100644 src/include/access/relundo_worker.h create mode 100644 src/include/access/relundo_xlog.h create mode 100644 src/include/access/slog.h create mode 100644 src/include/access/undo.h create mode 100644 src/include/access/undo_bufmgr.h create mode 100644 src/include/access/undo_xlog.h create mode 100644 src/include/access/undobuffer.h create mode 100644 src/include/access/undodefs.h create mode 100644 src/include/access/undolog.h create mode 100644 src/include/access/undorecord.h create mode 100644 src/include/access/undormgr.h create mode 100644 src/include/access/undormgrlist.h create mode 100644 src/include/access/undormgrs.h create mode 100644 src/include/access/undostats.h create mode 100644 src/include/access/undoworker.h create mode 100644 src/include/access/xactundo.h diff --git a/doc/src/sgml/filelist.sgml b/doc/src/sgml/filelist.sgml index 66ea8b988a18e..4dda4fd85ddf0 100644 --- a/doc/src/sgml/filelist.sgml +++ b/doc/src/sgml/filelist.sgml @@ -49,6 +49,7 @@ + diff --git a/doc/src/sgml/postgres.sgml b/doc/src/sgml/postgres.sgml index 2101442c90fcb..0940a557ffa2e 100644 --- a/doc/src/sgml/postgres.sgml +++ b/doc/src/sgml/postgres.sgml @@ -164,6 +164,7 @@ break is not needed in a wider output rendering. &high-availability; &monitoring; &wal; + &undo; &logical-replication; &jit; ®ress; diff --git a/doc/src/sgml/undo.sgml b/doc/src/sgml/undo.sgml new file mode 100644 index 0000000000000..bd11d3678599d --- /dev/null +++ b/doc/src/sgml/undo.sgml @@ -0,0 +1,738 @@ + + + + UNDO Logging + + + UNDO logging + + + + PostgreSQL provides an optional UNDO logging + system that records the inverse of data modifications to heap tables. + This enables two capabilities: transaction rollback using stored UNDO + records with full crash recovery and standby replay support, and + point-in-time recovery of pruned tuple data using the + pg_undorecover utility. + + + + The UNDO infrastructure must first be enabled cluster-wide by setting + the enable_undo GUC to on in + postgresql.conf (requires a server restart). + Individual tables then opt in via the enable_undo + storage parameter. When the server-level GUC is off (the default), + there is zero overhead on normal heap operations. + + + + The UNDO system uses a physical approach to + transaction rollback: rather than replaying high-level operations in + reverse, it restores the original page bytes directly. Each rollback + operation generates a WAL record (called a Compensation Log Record, or + CLR) that ensures correct replay on standbys and during crash recovery. + + + + Enabling UNDO Logging + + + First, enable the UNDO infrastructure at the server level in + postgresql.conf (requires restart): + + + +enable_undo = on + + + + Then enable UNDO logging on individual tables using the + enable_undo storage parameter: + + + +-- Enable at table creation +CREATE TABLE important_data ( + id serial PRIMARY KEY, + payload text +) WITH (enable_undo = on); + +-- Enable on an existing table +ALTER TABLE important_data SET (enable_undo = on); + +-- Disable UNDO logging +ALTER TABLE important_data SET (enable_undo = off); + + + + + Enabling or disabling enable_undo requires an + ACCESS EXCLUSIVE lock on the table. Plan for + a maintenance window if the table is under active use. + + + + + System catalogs cannot have UNDO enabled. Attempting to set + enable_undo = on on a system relation will + be silently ignored. + + + + + When to Use UNDO + + + Consider enabling UNDO logging when: + + + + + + You need to recover data that may be lost to aggressive vacuuming + or HOT pruning. UNDO records preserve pruned tuple versions in + a separate log, recoverable via pg_undorecover. + + + + + You want crash-safe rollback with full WAL integration for + critical tables, ensuring that aborted transactions are correctly + rolled back even after a crash or on streaming replication standbys. + + + + + You need an audit trail of old tuple versions for compliance + or forensic purposes. + + + + + + Do not enable UNDO logging on: + + + + + + High-throughput write-heavy tables where the additional I/O + overhead is unacceptable. + + + + + Temporary tables or tables with short-lived data that does not + need recovery protection. + + + + + + + Logged Operations + + + When UNDO is enabled on a table, the following operations generate + UNDO records: + + + + + INSERT + + + Records the block and offset of the newly inserted tuple along + with the ItemId state. On rollback, the inserted tuple is + physically removed from the page and the ItemId is restored to + its prior state. No full tuple payload is stored. + + + + + + DELETE + + + Records the full raw tuple data as it appears on the heap page. + On rollback, the original tuple bytes are restored to the page + via direct memory copy, and the ItemId is restored. + + + + + + UPDATE + + + Records the full raw data of the old tuple version before the + update. On rollback, the old tuple bytes are restored to their + original page location, and the new tuple is removed. + + + + + + Pruning (HOT cleanup and VACUUM) + + + Records full copies of tuples being marked as dead or unused + during page pruning. These records are not rolled back (pruning + is a maintenance operation, not a transactional data change) but + are preserved for point-in-time recovery via + pg_undorecover. + + + + + + + Each rollback operation generates a Compensation Log Record (CLR) in + the WAL stream. CLRs carry full page images, ensuring that the + rollback is correctly replayed on standbys and during crash recovery. + + + + + Crash Recovery and Replication + + + The UNDO system is fully integrated with PostgreSQL's WAL-based + crash recovery and streaming replication. + + + + When a transaction with UNDO records aborts, each UNDO application + generates a CLR (Compensation Log Record) WAL record. These CLRs + contain full page images of the restored heap pages, making them + self-contained and safe to replay. + + + + During crash recovery: + + + + + + The redo phase replays all WAL records forward, including any CLRs + that were generated before the crash. Pages are restored to their + post-rollback state. + + + + + For transactions that were aborting at crash time but had not + completed rollback, the recovery process walks the remaining UNDO + chain and generates new CLRs, using CLR pointers to skip + already-applied records. + + + + + + On streaming replication standbys, CLRs are replayed like any other + WAL record. The standby does not need access to the UNDO log data + itself, since the CLR WAL records are self-contained with full page + images. + + + + + Point-in-Time Recovery with pg_undorecover + + + The pg_undorecover utility reads UNDO log + files directly from the data directory and outputs recovered tuple data. + The server does not need to be running. + + + +# Show all UNDO records +pg_undorecover /path/to/pgdata + +# Filter by relation OID +pg_undorecover -r 16384 /path/to/pgdata + +# Filter by transaction ID and output as CSV +pg_undorecover -x 12345 -f csv /path/to/pgdata + +# Show only pruned records as JSON +pg_undorecover -t prune -f json /path/to/pgdata + +# Show statistics only +pg_undorecover -s -v /path/to/pgdata + + + + pg_undorecover options: + + + + + + + Filter records by relation OID. + + + + + + + Filter records by transaction ID. + + + + + + + + Filter by record type. Valid types: + insert, delete, + update, prune, + inplace. + + + + + + + + + Output format: text (default), + csv, or json. + + + + + + + + Show statistics summary only, without individual records. + + + + + + + Verbose mode with detailed scan progress. + + + + + + + Configuration Parameters + + + + enable_undo (boolean) + + + Master switch that enables the UNDO logging infrastructure. + This is a PGC_POSTMASTER parameter: changing it + requires a server restart. When off (the default), + the UNDO subsystem is completely dormant and tables cannot opt in. + When on, individual tables can enable UNDO via + the enable_undo storage parameter. + + + + + + undo_worker_naptime (integer) + + + Time in milliseconds between UNDO discard worker cycles. + The worker wakes periodically to check for UNDO records that + are no longer needed by any active transaction. + Default: 60000 (1 minute). + + + + + + undo_retention_time (integer) + + + Minimum time in milliseconds to retain UNDO records after + the creating transaction completes. Higher values allow + pg_undorecover to access older data + but consume more disk space. + Default: 3600000 (1 hour). + + + + + + + UNDO data is stored in the standard shared buffer pool alongside + heap and index pages. No dedicated UNDO buffer cache configuration + is needed. The shared buffer pool dynamically adapts to the UNDO + workload through its normal clock-sweep eviction policy. + + + + + UNDO Space Management + + + UNDO records are embedded directly in the WAL stream as + XLOG_UNDO_BATCH records (UNDO-in-WAL architecture). + There are no separate UNDO segment files or directories. This + eliminates a separate storage tier and leverages existing WAL + infrastructure for durability, replication, and archival. + + + + The UNDO discard worker background process advances the + undo_discard_horizon, allowing WAL segments + containing fully-discarded UNDO batches to be recycled. UNDO records + for unresolved (uncommitted/unaborted) transactions are never discarded. + The undo_retention_time controls how long committed + transaction UNDO records are retained beyond the visibility horizon. + + + + To monitor UNDO WAL retention: + + + +SELECT pg_size_pretty( + pg_wal_lsn_diff(pg_current_wal_lsn(), undo_discard_horizon) +) AS undo_wal_retained +FROM pg_stat_undo; + + + + If UNDO space is growing unexpectedly, check for: + + + + + + Long-running transactions that prevent discard. + + + + + A high undo_retention_time value. + + + + + The UNDO worker not running (check + pg_stat_activity for the + undo worker process). + + + + + + + Performance Impact + + + When UNDO is disabled (the default), there is no measurable + performance impact. When enabled on a table, expect: + + + + + + INSERT: Minimal overhead. A small header + record (~40 bytes) is written to the UNDO log recording the + ItemId state. + + + + + DELETE/UPDATE: Moderate overhead. The full + old tuple data is copied to the UNDO log as raw page bytes. + Cost scales with tuple size. + + + + + PRUNE: Overhead proportional to the number + of tuples being pruned. Records are batched for efficiency. + + + + + ABORT: Each UNDO record applied during + rollback generates a CLR WAL record with a full page image + (~8 KB). This increases abort latency by approximately 20-50% + compared to systems without CLR generation, but ensures crash + safety and correct standby replay. + + + + + + UNDO I/O is performed outside critical sections, so it does not + extend the time that buffer locks are held. + + + + + Monitoring + + + Monitor UNDO system health using: + + + + + + pg_stat_undo_logs: Per-log statistics + including size, discard progress, and oldest active transaction. + + + + + pg_waldump: Inspect CLR records in WAL. + CLR records appear as UNDO/APPLY_RECORD entries + and can be filtered with . + + + + + WAL retention due to UNDO batches (check pg_stat_undo). + + + + + pg_stat_activity: Verify the + undo worker background process is running. + + + + + + Key log messages to watch for (at DEBUG1 and above): + + + + + + "applying UNDO chain starting at ..." indicates + a transaction abort is applying its UNDO chain. + + + + + "UNDO rollback: relation %u no longer exists, skipping" + indicates an UNDO record was skipped because the target relation was + dropped before rollback completed. + + + + + + + Architecture Notes + + + The following notes describe the internal architecture for users + interested in the design rationale. + + + + Physical vs Logical UNDO + + + The UNDO system uses physical UNDO operations: + when rolling back a transaction, the original page bytes are restored + directly using memory copy operations. This contrasts with a + logical approach that would replay high-level + operations (like simple_heap_insert or + simple_heap_delete) in reverse. + + + + Advantages of physical UNDO: + + + + + + Crash Safety: Each UNDO application generates a + Compensation Log Record (CLR) in WAL, ensuring that rollback completes + correctly even after a system crash. + + + + + Standby Support: CLRs are replayed on physical + standbys just like forward-progress WAL records. Standbys see + identical heap state as the primary after an abort. + + + + + Determinism: Physical operations cannot fail due + to page-full conditions, TOAST complications, or index conflicts. + The operation is a direct memory copy with no side effects. + + + + + Simplicity: Direct memory copy operations are + simpler and faster than reconstructing logical operations, and have + no side effects (no index updates, no TOAST operations, no + statistics maintenance). + + + + + + Trade-offs: + + + + + + WAL Volume: CLRs with full page images (~8 KB + each) increase WAL generation significantly per abort compared to + PostgreSQL's default rollback mechanism + which generates no WAL. + + + + + Abort Latency: Approximately 20-50% overhead + compared to PostgreSQL's default rollback, + due to reading UNDO records, modifying pages, and writing CLRs. + + + + + + The design prioritizes correctness and crash safety over abort speed. + For workloads where transaction aborts are rare, the overhead is + negligible. + + + + + Compensation Log Records (CLRs) + + + A CLR is a WAL record generated each time an UNDO record is physically + applied to a heap page during rollback. CLRs serve three purposes: + + + + + + Crash recovery: If the server crashes during + rollback, the redo phase replays any CLRs that were already written, + restoring pages to their post-undo state. Rollback then continues + from where it left off, using CLR pointers in the UNDO records to + skip already-applied operations. + + + + + Standby replication: CLRs are streamed to + standbys like any other WAL record. The standby does not need + access to the UNDO log data itself, since CLRs are self-contained + with full page images. + + + + + Audit trail: CLRs provide a permanent record + in WAL of every rollback operation, viewable with + pg_waldump. + + + + + + Each CLR uses REGBUF_FORCE_IMAGE to store a + complete page image, making the CLR self-contained for recovery. + During redo, the page image is restored directly without needing + to re-read the UNDO record or re-apply the operation. + + + + + Buffer Pool Integration + + + UNDO log data is stored in the standard shared buffer pool alongside + heap and index pages. Each UNDO log is mapped to a virtual + RelFileLocator with a dedicated pseudo-database + OID (UNDO_DB_OID = 9), allowing the buffer manager + to handle UNDO data without any changes to the core + BufferTag structure. + + + + This design eliminates the need for a separate UNDO buffer cache, + reducing code complexity and allowing UNDO pages to participate in + the buffer manager's clock-sweep eviction and checkpoint mechanisms + automatically. No dedicated UNDO buffer cache configuration is needed; + the standard shared_buffers setting controls memory + available for all buffer types including UNDO. + + + + + Rollback Flow + + + When a transaction aborts, the rollback proceeds as follows: + + + + + + The transaction manager (xact.c) calls + ApplyUndoChain() with the first UNDO record + pointer for the aborting transaction. + + + + + For each UNDO record in the chain (walked backward): + + + + Read the UNDO record from the log. + + + Check the CLR pointer: if valid, this record was already + applied during a previous rollback attempt; skip it. + + + Open the target relation and read the target page into a + shared buffer with an exclusive lock. + + + Apply the physical modification (memcpy) within a critical + section. + + + Generate a CLR WAL record with a full page image. + + + Store the CLR's LSN back into the UNDO record's + urec_clr_ptr field to mark it as + applied. + + + + + + AtAbort_XactUndo() cleans up record sets and + resets per-transaction state. + + + + + + + diff --git a/examples/01-basic-undo-setup.sql b/examples/01-basic-undo-setup.sql new file mode 100644 index 0000000000000..82042081e4e9b --- /dev/null +++ b/examples/01-basic-undo-setup.sql @@ -0,0 +1,42 @@ +-- ============================================================================ +-- Example 1: Basic UNDO Setup and Monitoring +-- ============================================================================ +-- This example demonstrates: +-- 1. Creating a table that uses UNDO (via the recno access method) +-- 2. Performing modifications +-- 3. Monitoring UNDO activity + +-- STEP 1: Create a table using the recno AM (which supports UNDO) +-- No server-level configuration is needed; UNDO is always-on infrastructure. +CREATE TABLE customer_data ( + id serial PRIMARY KEY, + name text NOT NULL, + email text, + created_at timestamptz DEFAULT now() +) USING recno; + +-- STEP 2: Insert sample data +INSERT INTO customer_data (name, email) VALUES + ('Alice Smith', 'alice@example.com'), + ('Bob Johnson', 'bob@example.com'), + ('Charlie Brown', 'charlie@example.com'); + +-- STEP 3: Perform an update (in-place for recno) +UPDATE customer_data SET email = 'alice.smith@newdomain.com' WHERE name = 'Alice Smith'; + +-- STEP 4: Delete a row +DELETE FROM customer_data WHERE id = 2; + +-- STEP 5: Commit the transaction +COMMIT; + +-- STEP 6: Check UNDO log statistics +SELECT * FROM pg_stat_get_undo_logs(); + +-- STEP 7: Check UNDO buffer statistics +SELECT * FROM pg_stat_get_undo_buffers(); + +-- STEP 8: Verify the UNDO worker is running +SELECT pid, backend_type, state +FROM pg_stat_activity +WHERE backend_type = 'undo worker'; diff --git a/examples/02-undo-rollback.sql b/examples/02-undo-rollback.sql new file mode 100644 index 0000000000000..9af57664747e0 --- /dev/null +++ b/examples/02-undo-rollback.sql @@ -0,0 +1,44 @@ +-- ============================================================================ +-- Example 2: Transaction Rollback with UNDO +-- ============================================================================ +-- Demonstrates how UNDO records enable efficient transaction rollback + +-- Create a table using the recno AM (supports UNDO) +CREATE TABLE order_items ( + order_id int, + item_id int, + quantity int, + price numeric(10,2) +) USING recno; + +-- Begin transaction +BEGIN; + +-- Insert multiple rows +INSERT INTO order_items VALUES + (1001, 1, 5, 29.99), + (1001, 2, 3, 49.99), + (1001, 3, 1, 199.99); + +-- Perform updates +UPDATE order_items SET quantity = 10 WHERE item_id = 1; +UPDATE order_items SET price = 44.99 WHERE item_id = 2; + +-- Delete a row +DELETE FROM order_items WHERE item_id = 3; + +-- Check current state (before rollback) +SELECT * FROM order_items; +-- Should show: 2 rows (items 1 and 2, modified) + +-- Rollback the transaction +-- UNDO records will be applied automatically: +-- - item 3 re-inserted +-- - item 2 price restored to 49.99 +-- - item 1 quantity restored to 5 +-- - all 3 original inserts deleted +ROLLBACK; + +-- Verify all changes were rolled back +SELECT * FROM order_items; +-- Should show: 0 rows (everything rolled back via UNDO) diff --git a/examples/03-undo-subtransactions.sql b/examples/03-undo-subtransactions.sql new file mode 100644 index 0000000000000..22dac58d9d9aa --- /dev/null +++ b/examples/03-undo-subtransactions.sql @@ -0,0 +1,45 @@ +-- ============================================================================ +-- Example 3: Subtransactions (SAVEPOINTs) with UNDO +-- ============================================================================ + +CREATE TABLE account_ledger ( + account_id int, + amount numeric(10,2), + posted_at timestamptz DEFAULT now() +) USING recno; + +BEGIN; + +-- Parent transaction: Initial credit +INSERT INTO account_ledger VALUES (1001, 1000.00); + +SAVEPOINT sp1; + +-- Subtransaction 1: Debit attempt +INSERT INTO account_ledger VALUES (1001, -500.00); + +SAVEPOINT sp2; + +-- Subtransaction 2: Another debit +INSERT INTO account_ledger VALUES (1001, -300.00); + +-- Check balance +SELECT SUM(amount) FROM account_ledger WHERE account_id = 1001; +-- Shows: 200.00 + +-- Rollback to sp2 (undo the -300.00) +ROLLBACK TO sp2; + +-- Check balance after rollback +SELECT SUM(amount) FROM account_ledger WHERE account_id = 1001; +-- Shows: 500.00 + +-- Rollback to sp1 (undo the -500.00) +ROLLBACK TO sp1; + +-- Check balance after full rollback to sp1 +SELECT SUM(amount) FROM account_ledger WHERE account_id = 1001; +-- Shows: 1000.00 (only initial credit remains) + +-- Commit parent transaction +COMMIT; diff --git a/examples/05-undo-monitoring.sql b/examples/05-undo-monitoring.sql new file mode 100644 index 0000000000000..caf027a7eeb10 --- /dev/null +++ b/examples/05-undo-monitoring.sql @@ -0,0 +1,38 @@ +-- ============================================================================ +-- Example 5: Monitoring UNDO Subsystem +-- ============================================================================ + +-- View UNDO log statistics +SELECT * FROM pg_stat_get_undo_logs(); + +-- View UNDO buffer statistics +SELECT * FROM pg_stat_get_undo_buffers(); + +-- Force discard of UNDO records older than the retention horizon +-- (normally handled automatically by the UNDO worker) +SELECT pg_undo_force_discard(); + +-- List tables using an AM that supports UNDO (i.e., recno tables) +SELECT + n.nspname AS schema, + c.relname AS table, + am.amname AS access_method +FROM pg_class c +JOIN pg_namespace n ON c.relnamespace = n.oid +JOIN pg_am am ON c.relam = am.oid +WHERE am.amname = 'recno' +ORDER BY n.nspname, c.relname; + +-- Monitor UNDO worker activity +SELECT + pid, + backend_type, + state, + query_start, + state_change +FROM pg_stat_activity +WHERE backend_type = 'undo worker'; + +-- Check current UNDO retention settings +SHOW undo_retention_time; +SHOW undo_worker_naptime; diff --git a/examples/README.md b/examples/README.md new file mode 100644 index 0000000000000..096150dbd188d --- /dev/null +++ b/examples/README.md @@ -0,0 +1,40 @@ +# PostgreSQL UNDO Examples + +This directory contains practical examples demonstrating the UNDO subsystem +and transactional file operations (FILEOPS). + +## Prerequisites + +Tables opt into UNDO by using the `recno` access method: + + CREATE TABLE my_table (...) USING recno; + +UNDO is always-on infrastructure -- there is no GUC to enable or disable it +globally. Table access methods opt in via the `am_supports_undo` callback. + +Optional retention tuning (postgresql.conf): + + undo_retention_time = 3600000 # 1 hour in milliseconds + undo_worker_naptime = 60000 # 1 minute + +## Examples + +- **01-basic-undo-setup.sql**: Creating UNDO-enabled tables and monitoring +- **02-undo-rollback.sql**: Transaction rollback with UNDO records +- **03-undo-subtransactions.sql**: SAVEPOINT and subtransaction rollback +- **04-transactional-fileops.sql**: Crash-safe table creation/deletion +- **05-undo-monitoring.sql**: Monitoring UNDO subsystem usage + +## Running Examples + +```bash +psql -d testdb -f examples/01-basic-undo-setup.sql +psql -d testdb -f examples/02-undo-rollback.sql +... +``` + +## Notes + +- UNDO is always-on; tables opt in via `USING recno` +- FILEOPS (transactional file operations) is always-on for all tables +- System catalogs never use UNDO diff --git a/src/backend/access/Makefile b/src/backend/access/Makefile index e88d72ea0397d..2e4cc6a17e30b 100644 --- a/src/backend/access/Makefile +++ b/src/backend/access/Makefile @@ -22,6 +22,7 @@ SUBDIRS = \ sequence \ table \ tablesample \ - transam + transam \ + undo include $(top_srcdir)/src/backend/common.mk diff --git a/src/backend/access/meson.build b/src/backend/access/meson.build index 5fd18de74f92b..d569ac4e6e32a 100644 --- a/src/backend/access/meson.build +++ b/src/backend/access/meson.build @@ -14,3 +14,4 @@ subdir('spgist') subdir('table') subdir('tablesample') subdir('transam') +subdir('undo') diff --git a/src/backend/access/rmgrdesc/Makefile b/src/backend/access/rmgrdesc/Makefile index cd95eec37f148..3f94e17f281f3 100644 --- a/src/backend/access/rmgrdesc/Makefile +++ b/src/backend/access/rmgrdesc/Makefile @@ -9,6 +9,7 @@ top_builddir = ../../../.. include $(top_builddir)/src/Makefile.global OBJS = \ + atmdesc.o \ brindesc.o \ clogdesc.o \ committsdesc.o \ @@ -22,6 +23,7 @@ OBJS = \ mxactdesc.o \ nbtdesc.o \ relmapdesc.o \ + relundodesc.o \ replorigindesc.o \ rmgrdesc_utils.o \ seqdesc.o \ @@ -29,6 +31,7 @@ OBJS = \ spgdesc.o \ standbydesc.o \ tblspcdesc.o \ + undodesc.o \ xactdesc.o \ xlogdesc.o diff --git a/src/backend/access/rmgrdesc/atmdesc.c b/src/backend/access/rmgrdesc/atmdesc.c new file mode 100644 index 0000000000000..2864dfb7d2063 --- /dev/null +++ b/src/backend/access/rmgrdesc/atmdesc.c @@ -0,0 +1,64 @@ +/*------------------------------------------------------------------------- + * + * atmdesc.c + * rmgr descriptor routines for access/undo/atm.c + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/access/rmgrdesc/atmdesc.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/atm_xlog.h" + +void +atm_desc(StringInfo buf, XLogReaderState *record) +{ + char *data = XLogRecGetData(record); + uint8 info = XLogRecGetInfo(record) & ~XLR_INFO_MASK; + + switch (info) + { + case XLOG_ATM_ABORT: + { + xl_atm_abort *xlrec = (xl_atm_abort *) data; + + appendStringInfo(buf, + "xid %u, last_batch_lsn %X/%X, dboid %u", + xlrec->xid, + LSN_FORMAT_ARGS(xlrec->last_batch_lsn), + xlrec->dboid); + } + break; + + case XLOG_ATM_FORGET: + { + xl_atm_forget *xlrec = (xl_atm_forget *) data; + + appendStringInfo(buf, "xid %u", xlrec->xid); + } + break; + } +} + +const char * +atm_identify(uint8 info) +{ + const char *id = NULL; + + switch (info & ~XLR_INFO_MASK) + { + case XLOG_ATM_ABORT: + id = "ABORT"; + break; + case XLOG_ATM_FORGET: + id = "FORGET"; + break; + } + + return id; +} diff --git a/src/backend/access/rmgrdesc/meson.build b/src/backend/access/rmgrdesc/meson.build index d9000ccd9fd10..299cf2aeee201 100644 --- a/src/backend/access/rmgrdesc/meson.build +++ b/src/backend/access/rmgrdesc/meson.build @@ -2,6 +2,7 @@ # used by frontend programs like pg_waldump rmgr_desc_sources = files( + 'atmdesc.c', 'brindesc.c', 'clogdesc.c', 'committsdesc.c', @@ -15,6 +16,7 @@ rmgr_desc_sources = files( 'mxactdesc.c', 'nbtdesc.c', 'relmapdesc.c', + 'relundodesc.c', 'replorigindesc.c', 'rmgrdesc_utils.c', 'seqdesc.c', @@ -22,6 +24,7 @@ rmgr_desc_sources = files( 'spgdesc.c', 'standbydesc.c', 'tblspcdesc.c', + 'undodesc.c', 'xactdesc.c', 'xlogdesc.c', ) diff --git a/src/backend/access/rmgrdesc/relundodesc.c b/src/backend/access/rmgrdesc/relundodesc.c new file mode 100644 index 0000000000000..448d5765fed1e --- /dev/null +++ b/src/backend/access/rmgrdesc/relundodesc.c @@ -0,0 +1,140 @@ +/*------------------------------------------------------------------------- + * + * relundodesc.c + * rmgr descriptor routines for access/undo/relundo_xlog.c + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/access/rmgrdesc/relundodesc.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/relundo_xlog.h" + +/* + * relundo_desc - Describe a per-relation UNDO WAL record for pg_waldump + */ +void +relundo_desc(StringInfo buf, XLogReaderState *record) +{ + char *data = XLogRecGetData(record); + uint8 info = XLogRecGetInfo(record) & ~XLR_INFO_MASK; + + switch (info & ~XLOG_RELUNDO_INIT_PAGE) + { + case XLOG_RELUNDO_INIT: + { + xl_relundo_init *xlrec = (xl_relundo_init *) data; + + appendStringInfo(buf, "magic 0x%08X, version %u, counter %u", + xlrec->magic, xlrec->version, + xlrec->counter); + } + break; + + case XLOG_RELUNDO_INSERT: + { + xl_relundo_insert *xlrec = (xl_relundo_insert *) data; + const char *type_name; + + switch (xlrec->urec_type) + { + case 1: + type_name = "INSERT"; + break; + case 2: + type_name = "DELETE"; + break; + case 3: + type_name = "UPDATE"; + break; + case 4: + type_name = "TUPLE_LOCK"; + break; + default: + type_name = "UNKNOWN"; + break; + } + + appendStringInfo(buf, + "type %s, len %u, offset %u, new_pd_lower %u, max_xid %u", + type_name, xlrec->urec_len, + xlrec->page_offset, + xlrec->new_pd_lower, + xlrec->max_xid); + + if (info & XLOG_RELUNDO_INIT_PAGE) + appendStringInfoString(buf, " (init page)"); + } + break; + + case XLOG_RELUNDO_DISCARD: + { + xl_relundo_discard *xlrec = (xl_relundo_discard *) data; + + appendStringInfo(buf, + "slot %u, old_tail %u, new_tail %u, discard_xid %u, " + "npages_freed %u", + xlrec->slot, + xlrec->old_tail_blkno, + xlrec->new_tail_blkno, + xlrec->discard_xid, + xlrec->npages_freed); + } + break; + + case XLOG_RELUNDO_APPLY: + { + xl_relundo_apply *xlrec = (xl_relundo_apply *) data; + + appendStringInfo(buf, "urec_ptr %lu", + (unsigned long) xlrec->urec_ptr); + } + break; + + case XLOG_RELUNDO_TRUNCATE: + { + xl_relundo_truncate *xlrec = (xl_relundo_truncate *) data; + + appendStringInfo(buf, "new_nblocks %u", xlrec->new_nblocks); + } + break; + } +} + +/* + * relundo_identify - Identify a per-relation UNDO WAL record type + */ +const char * +relundo_identify(uint8 info) +{ + const char *id = NULL; + + switch (info & ~XLR_INFO_MASK) + { + case XLOG_RELUNDO_INIT: + id = "INIT"; + break; + case XLOG_RELUNDO_INSERT: + id = "INSERT"; + break; + case XLOG_RELUNDO_INSERT | XLOG_RELUNDO_INIT_PAGE: + id = "INSERT+INIT"; + break; + case XLOG_RELUNDO_DISCARD: + id = "DISCARD"; + break; + case XLOG_RELUNDO_TRUNCATE: + id = "TRUNCATE"; + break; + case XLOG_RELUNDO_APPLY: + id = "APPLY"; + break; + } + + return id; +} diff --git a/src/backend/access/rmgrdesc/undodesc.c b/src/backend/access/rmgrdesc/undodesc.c new file mode 100644 index 0000000000000..d4817684ad471 --- /dev/null +++ b/src/backend/access/rmgrdesc/undodesc.c @@ -0,0 +1,209 @@ +/*------------------------------------------------------------------------- + * + * undodesc.c + * rmgr descriptor routines for access/undo + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/access/rmgrdesc/undodesc.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/undo_xlog.h" +#include "access/xlogreader.h" + +/* + * undo_desc - Describe an UNDO WAL record for pg_waldump + * + * This function generates human-readable output for UNDO WAL records, + * used by pg_waldump and other debugging tools. + */ +void +undo_desc(StringInfo buf, XLogReaderState *record) +{ + char *rec = XLogRecGetData(record); + uint8 info = XLogRecGetInfo(record) & ~XLR_INFO_MASK; + + switch (info) + { + case XLOG_UNDO_ALLOCATE: + { + xl_undo_allocate *xlrec = (xl_undo_allocate *) rec; + + appendStringInfo(buf, "log %u, start %llu, len %u, xid %u", + xlrec->log_number, + (unsigned long long) xlrec->start_ptr, + xlrec->length, + xlrec->xid); + } + break; + + case XLOG_UNDO_DISCARD: + { + xl_undo_discard *xlrec = (xl_undo_discard *) rec; + + appendStringInfo(buf, "log %u, discard_ptr %llu, oldest_xid %u", + xlrec->log_number, + (unsigned long long) xlrec->discard_ptr, + xlrec->oldest_xid); + } + break; + + case XLOG_UNDO_EXTEND: + { + xl_undo_extend *xlrec = (xl_undo_extend *) rec; + + appendStringInfo(buf, "log %u, new_size %llu", + xlrec->log_number, + (unsigned long long) xlrec->new_size); + } + break; + + case XLOG_UNDO_APPLY_RECORD: + { + xl_undo_apply *xlrec = (xl_undo_apply *) rec; + const char *op_name; + + switch (xlrec->operation_type) + { + case 0x0001: + op_name = "INSERT"; + break; + case 0x0002: + op_name = "DELETE"; + break; + case 0x0003: + op_name = "UPDATE"; + break; + case 0x0004: + op_name = "PRUNE"; + break; + case 0x0005: + op_name = "INPLACE"; + break; + case 0x0006: + op_name = "HOT_UPDATE"; + break; + default: + op_name = "UNKNOWN"; + break; + } + + appendStringInfo(buf, + "undo apply %s: urec_ptr %llu, xid %u, " + "block %u, offset %u, clr_flags 0x%04x, " + "tuple_len %u", + op_name, + (unsigned long long) xlrec->urec_ptr, + xlrec->xid, + xlrec->target_block, + xlrec->target_offset, + xlrec->clr_flags, + xlrec->tuple_len); + } + break; + + case XLOG_UNDO_PAGE_WRITE: + { + xl_undo_page_write *xlrec = (xl_undo_page_write *) rec; + + appendStringInfo(buf, "page_offset %u, data_len %u", + xlrec->page_offset, + xlrec->data_len); + } + break; + + case XLOG_UNDO_BATCH: + { + xl_undo_batch *xlrec = (xl_undo_batch *) rec; + + appendStringInfo(buf, + "undo batch: xid %u, nrecords %u, " + "total_len %u, chain_prev %X/%X, " + "primary_reloid %u, persistence %d", + xlrec->xid, + xlrec->nrecords, + xlrec->total_len, + LSN_FORMAT_ARGS(xlrec->chain_prev), + xlrec->primary_reloid, + xlrec->persistence); + } + break; + + case XLOG_UNDO_ROTATE: + { + xl_undo_rotate *xlrec = (xl_undo_rotate *) rec; + const char *trigger_name; + + switch (xlrec->trigger) + { + case UNDO_ROTATE_CAPACITY: + trigger_name = "capacity"; + break; + case UNDO_ROTATE_CHECKPOINT: + trigger_name = "checkpoint"; + break; + case UNDO_ROTATE_PRESSURE: + trigger_name = "pressure"; + break; + case UNDO_ROTATE_MANUAL: + trigger_name = "manual"; + break; + default: + trigger_name = "unknown"; + break; + } + + appendStringInfo(buf, + "old_log %u, seal_ptr %llu, new_log %u, " + "trigger %s", + xlrec->old_log_number, + (unsigned long long) xlrec->old_seal_ptr, + xlrec->new_log_number, + trigger_name); + } + break; + } +} + +/* + * undo_identify - Identify an UNDO WAL record type + * + * Returns a string identifying the operation type for debugging output. + */ +const char * +undo_identify(uint8 info) +{ + const char *id = NULL; + + switch (info & ~XLR_INFO_MASK) + { + case XLOG_UNDO_ALLOCATE: + id = "ALLOCATE"; + break; + case XLOG_UNDO_DISCARD: + id = "DISCARD"; + break; + case XLOG_UNDO_EXTEND: + id = "EXTEND"; + break; + case XLOG_UNDO_APPLY_RECORD: + id = "APPLY_RECORD"; + break; + case XLOG_UNDO_ROTATE: + id = "ROTATE"; + break; + case XLOG_UNDO_PAGE_WRITE: + id = "PAGE_WRITE"; + break; + case XLOG_UNDO_BATCH: + id = "BATCH"; + break; + } + + return id; +} diff --git a/src/backend/access/transam/rmgr.c b/src/backend/access/transam/rmgr.c index 4fda03a3cfcc6..90712a7574229 100644 --- a/src/backend/access/transam/rmgr.c +++ b/src/backend/access/transam/rmgr.c @@ -40,6 +40,9 @@ #include "replication/origin.h" #include "storage/standby.h" #include "utils/relmapper.h" +#include "access/undo_xlog.h" +#include "access/atm.h" +#include "access/relundo_xlog.h" /* IWYU pragma: end_keep */ diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index fa3bc50ec483a..125b1c7960bcc 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -77,6 +77,8 @@ #include #include "access/commit_ts.h" +#include "access/atm.h" +#include "access/xactundo.h" #include "access/htup_details.h" #include "access/subtrans.h" #include "access/transam.h" @@ -162,6 +164,18 @@ typedef struct GlobalTransactionData */ XLogRecPtr prepare_start_lsn; /* XLOG offset of prepare record start */ XLogRecPtr prepare_end_lsn; /* XLOG offset of prepare record end */ + + /* + * Permanent-level UNDO chain-head LSN for this prepared xact, or + * InvalidXLogRecPtr if it generated no cluster-wide UNDO. Set at PREPARE + * (MarkAsPreparingGuts), at redo (PrepareRedoAdd), and when a stale 2PC + * file is recovered from disk. UndoGetOldestBatchLSN() scans this so the + * UNDO-batch WAL a still-prepared xact needs for ROLLBACK PREPARED is not + * recycled -- neither the per-backend retention slot (cleared when the + * preparing backend exits) nor the ATM (prepared xacts aren't in it) + * covers this case. + */ + XLogRecPtr undo_batch_lsn; FullTransactionId fxid; /* The GXACT full xid */ Oid owner; /* ID of user that executed the xact */ @@ -492,6 +506,7 @@ MarkAsPreparingGuts(GlobalTransaction gxact, FullTransactionId fxid, gxact->locking_backend = MyProcNumber; gxact->valid = false; gxact->inredo = false; + gxact->undo_batch_lsn = GetCurrentXactLastBatchLSN(UNDOPERSISTENCE_PERMANENT); strlcpy(gxact->gid, gid, GIDSIZE); /* @@ -978,8 +993,14 @@ TwoPhaseFilePath(char *path, FullTransactionId fxid) /* * Header for a 2PC state file + * + * TWOPHASE_MAGIC must be bumped whenever xl_xact_prepare changes layout. + * The struct gained last_batch_lsn[NUndoPersistenceLevels] (24 bytes) for + * UNDO chain tracking across 2PC boundaries, requiring this bump from + * 0x57F94534 to 0x57F94535 to prevent old servers from silently misreading + * the variable-length arrays that follow the fixed header at the wrong offsets. */ -#define TWOPHASE_MAGIC 0x57F94534 /* format identifier */ +#define TWOPHASE_MAGIC 0x57F94535 /* format identifier */ typedef xl_xact_prepare TwoPhaseFileHeader; @@ -1101,6 +1122,10 @@ StartPrepare(GlobalTransaction gxact) hdr.origin_lsn = InvalidXLogRecPtr; hdr.origin_timestamp = 0; + /* Save UNDO chain head LSNs so recovery can find UNDO records */ + for (int j = 0; j < NUndoPersistenceLevels; j++) + hdr.last_batch_lsn[j] = GetCurrentXactLastBatchLSN(j); + save_state_data(&hdr, sizeof(TwoPhaseFileHeader)); save_state_data(gxact->gid, hdr.gidlen); @@ -1498,6 +1523,47 @@ StandbyTransactionIdIsPrepared(TransactionId xid) return result; } +/* + * RecoveryTransactionIdIsPrepared + * Check if a transaction ID is in the in-memory prepared transaction list. + * + * This is used during crash recovery UNDO phase, before prepared transaction + * files exist on disk. It checks the in-memory TwoPhaseState that was + * reconstructed from WAL replay. + */ +bool +RecoveryTransactionIdIsPrepared(TransactionId xid) +{ + int i; + FullTransactionId fxid; + + Assert(TransactionIdIsValid(xid)); + + if (max_prepared_xacts <= 0) + return false; /* 2PC not enabled */ + + if (TwoPhaseState == NULL) + return false; /* 2PC not initialized yet */ + + fxid = AdjustToFullTransactionId(xid); + + LWLockAcquire(TwoPhaseStateLock, LW_SHARED); + + for (i = 0; i < TwoPhaseState->numPrepXacts; i++) + { + GlobalTransaction gxact = TwoPhaseState->prepXacts[i]; + + if (FullTransactionIdEquals(gxact->fxid, fxid)) + { + LWLockRelease(TwoPhaseStateLock); + return true; + } + } + + LWLockRelease(TwoPhaseStateLock); + return false; +} + /* * FinishPreparedTransaction: execute COMMIT PREPARED or ROLLBACK PREPARED */ @@ -1585,6 +1651,7 @@ FinishPreparedTransaction(const char *gid, bool isCommit) hdr->ninvalmsgs, invalmsgs, hdr->initfileinval, gid); else + { RecordTransactionAbortPrepared(xid, hdr->nsubxacts, children, hdr->nabortrels, abortrels, @@ -1592,6 +1659,31 @@ FinishPreparedTransaction(const char *gid, bool isCommit) abortstats, gid); + /* + * ROLLBACK PREPARED: hand any cluster-wide UNDO chain to the same + * async revert machinery an ordinary large-transaction abort uses. + * The permanent-level chain-head LSN was durably saved in the 2PC + * header at PREPARE; ATMAddAborted() records (xid, dboid, + * last_batch_lsn) in the sLog Aborted Transaction Map, and the + * logical revert worker walks the UNDO chain backwards from that LSN, + * restoring before-images. No apply code lives here. + * + * Only the permanent level is applied: TEMP undo is gone (the + * originating backend exited) and UNLOGGED forks are reset on crash; + * this matches the crash-recovery UNDO phase in undo_xlog.c. + */ + { + XLogRecPtr perm_lsn = + hdr->last_batch_lsn[UNDOPERSISTENCE_PERMANENT]; + + if (XLogRecPtrIsValid(perm_lsn) && + !ATMAddAborted(xid, hdr->database, perm_lsn)) + elog(WARNING, + "ATM full: could not record rolled-back prepared " + "transaction %u for UNDO", xid); + } + } + ProcArrayRemove(proc, latestXid); /* @@ -2145,6 +2237,14 @@ RecoverPreparedTransactions(void) hdr->prepared_at, hdr->owner, hdr->database); + /* + * MarkAsPreparingGuts read the live backend's last_batch_lsn (wrong + * xact during recovery); restore this xact's real value from the 2PC + * header so WAL retention pins its UNDO batch until ROLLBACK + * PREPARED. + */ + gxact->undo_batch_lsn = hdr->last_batch_lsn[UNDOPERSISTENCE_PERMANENT]; + /* recovered, so reset the flag for entries generated by redo */ gxact->inredo = false; @@ -2597,6 +2697,7 @@ PrepareRedoAdd(FullTransactionId fxid, char *buf, gxact->valid = false; gxact->ondisk = !XLogRecPtrIsValid(start_lsn); gxact->inredo = true; /* yes, added in redo */ + gxact->undo_batch_lsn = hdr->last_batch_lsn[UNDOPERSISTENCE_PERMANENT]; strlcpy(gxact->gid, gid, GIDSIZE); /* And insert it into the active array */ @@ -2878,3 +2979,36 @@ TwoPhaseGetOldestXidInCommit(void) return oldestRunningXid; } + +/* + * TwoPhaseGetOldestUndoBatchLSN + * Return the oldest permanent-level UNDO chain-head LSN across all + * prepared transactions, or InvalidXLogRecPtr if none generated UNDO. + * + * UndoGetOldestBatchLSN() folds this into the WAL-retention horizon so the + * UNDO-batch WAL a still-prepared xact needs for ROLLBACK PREPARED survives + * checkpoints and backend exit (see undo_batch_lsn in GlobalTransactionData). + */ +XLogRecPtr +TwoPhaseGetOldestUndoBatchLSN(void) +{ + XLogRecPtr oldest = InvalidXLogRecPtr; + + if (max_prepared_xacts <= 0) + return InvalidXLogRecPtr; + + LWLockAcquire(TwoPhaseStateLock, LW_SHARED); + + for (int i = 0; i < TwoPhaseState->numPrepXacts; i++) + { + XLogRecPtr lsn = TwoPhaseState->prepXacts[i]->undo_batch_lsn; + + if (XLogRecPtrIsValid(lsn) && + (!XLogRecPtrIsValid(oldest) || lsn < oldest)) + oldest = lsn; + } + + LWLockRelease(TwoPhaseStateLock); + + return oldest; +} diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c index 3a89149016fe6..1f4b12a2333ee 100644 --- a/src/backend/access/transam/xact.c +++ b/src/backend/access/transam/xact.c @@ -26,6 +26,12 @@ #include "access/subtrans.h" #include "access/transam.h" #include "access/twophase.h" +#include "access/relundo.h" +#include "access/relundo_worker.h" +#include "access/undo_xlog.h" +#include "access/undolog.h" +#include "access/undorecord.h" +#include "access/xactundo.h" #include "access/xact.h" #include "access/xlog.h" #include "access/xloginsert.h" @@ -217,6 +223,7 @@ typedef struct TransactionStateData bool parallelChildXact; /* is any parent transaction parallel? */ bool chain; /* start a new block after this one */ bool topXidLogged; /* for a subxact: is top-level XID logged? */ + uint64 undoRecPtr; /* most recent UNDO record in chain */ struct TransactionStateData *parent; /* back link to parent */ } TransactionStateData; @@ -316,6 +323,23 @@ typedef struct XactCallbackItem static XactCallbackItem *Xact_callbacks = NULL; +/* + * Table-AM hook for AtPrepare work that must run between StartPrepare() and + * EndPrepare() (see access/xact.h for why this can't be an ordinary + * XactCallback). NULL when no such AM is compiled in. + */ +void (*TableAMPrepare_hook) (void) = NULL; + +/* + * Pending-structural-file-operation hooks, mirroring smgr's + * own pending-deletes housekeeping ordering (see access/xact.h). NULL when + * no such subsystem is compiled in. + */ +void (*PendingPhysOpsDo_hook) (bool isCommit) = NULL; +void (*PendingPhysOpsPostPrepare_hook) (void) = NULL; +void (*PendingPhysOpsAtSubCommit_hook) (void) = NULL; +void (*PendingPhysOpsAtSubAbort_hook) (void) = NULL; + /* * List of add-on start- and end-of-subxact callbacks */ @@ -418,6 +442,37 @@ IsAbortedTransactionBlockState(void) } +/* + * EnterInlineUndoApplyState / LeaveInlineUndoApplyState + * + * AtAbort_XactUndo() may apply UNDO records synchronously in this backend + * while AbortTransaction() has already set s->state = TRANS_ABORT but has + * not yet torn down the relcache, locks, or resource owner. UNDO appliers + * (e.g. a table AM's undo-apply callback) need to open relations, which trips + * Assert(IsTransactionState()) because TRANS_ABORT is not TRANS_INPROGRESS. + * + * These helpers temporarily present TRANS_INPROGRESS for the duration of the + * inline apply. This is safe because all backing resources are still live at + * this point in the abort sequence; only the state enum has advanced. The + * caller MUST pair Enter/Leave (use PG_TRY/PG_FINALLY) so the real abort state + * is always restored before the rest of AbortTransaction() proceeds. + */ +int +EnterInlineUndoApplyState(void) +{ + TransactionState s = CurrentTransactionState; + int saved = (int) s->state; + + s->state = TRANS_INPROGRESS; + return saved; +} + +void +LeaveInlineUndoApplyState(int saved) +{ + CurrentTransactionState->state = (TransState) saved; +} + /* * GetTopTransactionId * @@ -1123,6 +1178,36 @@ IsInParallelMode(void) return s->parallelModeLevel != 0 || s->parallelChildXact; } +/* + * SetCurrentTransactionUndoRecPtr + * Set the most recent UNDO record pointer for the current transaction. + * + * Called from heap_insert/delete/update when they generate UNDO records. + * The pointer is used during abort to walk the UNDO chain and apply + * compensation operations. + */ +void +SetCurrentTransactionUndoRecPtr(uint64 undo_ptr) +{ + TransactionState s = CurrentTransactionState; + + s->undoRecPtr = undo_ptr; +} + +/* + * GetCurrentTransactionUndoRecPtr + * Get the most recent UNDO record pointer for the current transaction. + * + * Returns InvalidUndoRecPtr (0) if no UNDO records have been generated. + */ +uint64 +GetCurrentTransactionUndoRecPtr(void) +{ + TransactionState s = CurrentTransactionState; + + return s->undoRecPtr; +} + /* * CommandCounterIncrement */ @@ -2143,6 +2228,7 @@ StartTransaction(void) s->childXids = NULL; s->nChildXids = 0; s->maxChildXids = 0; + s->undoRecPtr = 0; /* no UNDO records yet */ /* * Once the current user ID and the security context flags are fetched, @@ -2449,6 +2535,9 @@ CommitTransaction(void) CallXactCallbacks(is_parallel_worker ? XACT_EVENT_PARALLEL_COMMIT : XACT_EVENT_COMMIT); + /* Clean up transaction undo state (free per-persistence record sets) */ + AtCommit_XactUndo(); + CurrentResourceOwner = NULL; ResourceOwnerRelease(TopTransactionResourceOwner, RESOURCE_RELEASE_BEFORE_LOCKS, @@ -2669,6 +2758,29 @@ PrepareTransaction(void) (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot PREPARE a transaction that has exported snapshots"))); + /* + * Don't allow PREPARE TRANSACTION if this transaction generated any UNDO + * -- cluster-wide (via InsertUndoRecord) or per-relation (via the + * per-relation UNDO fork). Neither has a working ROLLBACK PREPARED apply + * path yet: per-relation UNDO's before-images live in backend-private, + * CurTransactionContext-allocated state (XactUndo.relundo_list) that is + * gone the moment this function returns; cluster-wide UNDO's chain-head + * LSN is durably saved in the 2PC state file + * (xl_xact_prepare.last_batch_lsn) but nothing on the COMMIT PREPARED / + * ROLLBACK PREPARED path or crash-recovery path ever reads it back or + * calls into the UNDO apply machinery (confirmed: twophase.c's + * RecordTransactionCommitPrepared/RecordTransactionAbortPrepared never + * call AtAbort_XactUndo() or ATMAddAborted(), the only two entry points + * that trigger UNDO application). Silently proceeding would make + * ROLLBACK PREPARED a no-op for either kind of UNDO, corrupting data. + * Reject early, before StartPrepare() writes any 2PC state. + */ + if (XactUndoHasUnrecoverableUndo()) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot PREPARE a transaction that generated UNDO"), + errhint("Commit or roll back the transaction without PREPARE TRANSACTION."))); + /* Prevent cancel/die interrupt while cleaning up */ HOLD_INTERRUPTS(); @@ -2928,6 +3040,25 @@ AbortTransaction(void) TransStateAsString(s->state)); Assert(s->parent == NULL); + /* + * Discard the UNDO record pointer for this transaction. + * + * Physical UNDO application is NOT needed during standard transaction + * abort because PostgreSQL's MVCC-based heap already handles rollback + * through CLOG: the aborting transaction's xid is marked as aborted in + * CLOG, and subsequent visibility checks will ignore changes made by this + * transaction. INSERT tuples become invisible (eventually pruned), + * DELETE/UPDATE changes are ignored (old tuple versions remain visible). + * + * Physical UNDO application is intended for cases where the page has been + * modified in-place and the old state cannot be recovered through CLOG + * alone (e.g., in ZHeap-style in-place updates, or after pruning has + * removed old tuple versions). The UNDO records written during this + * transaction are preserved in the UNDO log for use by the undo worker, + * crash recovery, or future in-place update mechanisms. + */ + s->undoRecPtr = 0; + /* * set the current transaction state information appropriately during the * abort processing @@ -2963,6 +3094,9 @@ AbortTransaction(void) s->parallelModeLevel = 0; s->parallelChildXact = false; /* should be false already */ + /* Clean up transaction undo state (free per-persistence record sets) */ + AtAbort_XactUndo(); + /* * do abort processing */ @@ -3030,6 +3164,15 @@ AbortTransaction(void) ResourceOwnerRelease(TopTransactionResourceOwner, RESOURCE_RELEASE_AFTER_LOCKS, false, true); + + /* + * Wait for any pending synchronous per-relation UNDO worker to + * finish. Done after lock release so the worker can acquire its own + * lock on the target relation, making per-relation rollback + * synchronous from the client's point of view. + */ + WaitForPendingRelUndo(); + smgrDoPendingDeletes(false); AtEOXact_GUC(false, 1); @@ -6434,6 +6577,12 @@ xact_redo(XLogReaderState *record) ParseCommitRecord(XLogRecGetInfo(record), xlrec, &parsed); xact_redo_commit(&parsed, XLogRecGetXid(record), record->EndRecPtr, XLogRecGetOrigin(record)); + + /* + * Remove from UNDO recovery tracking ā€” committed, no rollback + * needed + */ + UndoRecoveryRemoveXid(XLogRecGetXid(record)); } else if (info == XLOG_XACT_COMMIT_PREPARED) { @@ -6448,6 +6597,9 @@ xact_redo(XLogReaderState *record) LWLockAcquire(TwoPhaseStateLock, LW_EXCLUSIVE); PrepareRedoRemove(parsed.twophase_xid, false); LWLockRelease(TwoPhaseStateLock); + + /* Remove from UNDO recovery tracking */ + UndoRecoveryRemoveXid(parsed.twophase_xid); } else if (info == XLOG_XACT_ABORT) { @@ -6457,6 +6609,13 @@ xact_redo(XLogReaderState *record) ParseAbortRecord(XLogRecGetInfo(record), xlrec, &parsed); xact_redo_abort(&parsed, XLogRecGetXid(record), record->EndRecPtr, XLogRecGetOrigin(record)); + + /* + * Remove from UNDO recovery tracking ā€” abort record present means + * the UNDO rollback was already completed (or will be handled by the + * abort record's own redo logic). + */ + UndoRecoveryRemoveXid(XLogRecGetXid(record)); } else if (info == XLOG_XACT_ABORT_PREPARED) { @@ -6471,12 +6630,22 @@ xact_redo(XLogReaderState *record) LWLockAcquire(TwoPhaseStateLock, LW_EXCLUSIVE); PrepareRedoRemove(parsed.twophase_xid, false); LWLockRelease(TwoPhaseStateLock); + + /* Remove from UNDO recovery tracking */ + UndoRecoveryRemoveXid(parsed.twophase_xid); } else if (info == XLOG_XACT_PREPARE) { + xl_xact_prepare *xlrec = (xl_xact_prepare *) XLogRecGetData(record); + /* * Store xid and start/end pointers of the WAL record in TwoPhaseState * gxact entry. + * + * NB: xl_xact_prepare includes last_batch_lsn[NUndoPersistenceLevels] + * for UNDO chain tracking across 2PC boundaries. This extended the + * on-disk struct by 24 bytes and required a XLOG_PAGE_MAGIC bump + * (0xD120 -> 0xD121) to prevent misinterpretation by older replicas. */ LWLockAcquire(TwoPhaseStateLock, LW_EXCLUSIVE); PrepareRedoAdd(InvalidFullTransactionId, @@ -6485,6 +6654,21 @@ xact_redo(XLogReaderState *record) record->EndRecPtr, XLogRecGetOrigin(record)); LWLockRelease(TwoPhaseStateLock); + + /* + * Restore UNDO recovery tracking for the prepared transaction. The + * UNDO chain LSNs were saved in the prepare record so that if the + * server crashes after PREPARE but before COMMIT/ROLLBACK PREPARED, + * recovery can still find and roll back UNDO records. + */ + for (int j = 0; j < NUndoPersistenceLevels; j++) + { + if (!XLogRecPtrIsInvalid(xlrec->last_batch_lsn[j])) + UndoRecoveryTrackBatch(xlrec->xid, + xlrec->last_batch_lsn[j], + InvalidXLogRecPtr, + (UndoPersistenceLevel) j); + } } else if (info == XLOG_XACT_ASSIGNMENT) { diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index b23d8bbbdad60..5d62099337901 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -46,6 +46,7 @@ #include #include +#include "access/atm.h" #include "access/clog.h" #include "access/commit_ts.h" #include "access/heaptoast.h" @@ -55,6 +56,8 @@ #include "access/timeline.h" #include "access/transam.h" #include "access/twophase.h" +#include "access/undolog.h" +#include "access/undo_xlog.h" #include "access/xact.h" #include "access/xlog_internal.h" #include "access/xlogarchive.h" @@ -6096,6 +6099,18 @@ StartupXLOG(void) */ restoreTwoPhaseData(); + /* + * Reload the Aborted Transaction Map from its checkpoint state file + * before the redo pass. The ATM is otherwise reconstructed only by + * replaying XLOG_ATM_ABORT / XLOG_ATM_FORGET, which redo cannot do for + * aborts whose records precede the checkpoint redo point. Reloading here + * -- before redo, exactly like restoreTwoPhaseData() above -- lets + * atm_redo's XLOG_ATM_FORGET replays correctly remove entries forgotten + * after the checkpoint, and XLOG_ATM_ABORT replays re-add (idempotently) + * entries aborted after it. + */ + ATMReloadFromCheckpoint(); + /* * When starting with crash recovery, reset pgstat data - it might not be * valid. Otherwise restore pgstat data. It's safe to do this here, @@ -6560,6 +6575,24 @@ StartupXLOG(void) if (performedWalRecovery) promoted = PerformRecoveryXLogAction(); + /* + * Finalize ATM state after recovery. WAL replay has reconstructed the + * Aborted Transaction Map via XLOG_ATM_ABORT and XLOG_ATM_FORGET redo + * handlers. Log a summary of entries that still need Logical Revert. + */ + if (performedWalRecovery) + ATMRecoveryFinalize(); + + /* + * Flush any deferred UNDO transactions to the ATM. During the UNDO + * phase, if syscache wasn't available, we deferred transaction + * processing. Now that recovery is complete and WAL writes are allowed + * (checkpoint/ end-of-recovery record was written above), we can add them + * to the ATM for asynchronous processing by the logical revert worker. + */ + if (performedWalRecovery) + FlushDeferredUndoXacts(); + /* * If any of the critical GUCs have changed, log them before we allow * backends to write WAL. @@ -7410,6 +7443,16 @@ CreateCheckPoint(int flags) VirtualTransactionId *vxids; int nvxids; int oldXLogAllowed = 0; + instr_time phase_start, + phase_end; + double syncpre_ms = 0, + delay_start_ms = 0, + delay_complete_ms = 0, + xlogflush_ms = 0, + ctlfile_ms = 0, + syncpost_ms = 0, + removewal_ms = 0, + truncsub_ms = 0; /* * An end-of-recovery checkpoint is really a shutdown checkpoint, just @@ -7440,7 +7483,11 @@ CreateCheckPoint(int flags) * smgr must not do anything that'd have to be undone if we decide no * checkpoint is needed. */ + INSTR_TIME_SET_CURRENT(phase_start); SyncPreCheckpoint(); + INSTR_TIME_SET_CURRENT(phase_end); + INSTR_TIME_SUBTRACT(phase_end, phase_start); + syncpre_ms = INSTR_TIME_GET_MILLISEC(phase_end); /* Run these points outside the critical section. */ INJECTION_POINT("create-checkpoint-initial", NULL); @@ -7692,6 +7739,7 @@ CreateCheckPoint(int flags) * clog and we will correctly flush the update below. So we cannot miss * any xacts we need to wait for. */ + INSTR_TIME_SET_CURRENT(phase_start); vxids = GetVirtualXIDsDelayingChkpt(&nvxids, DELAY_CHKPT_START); if (nvxids > 0) { @@ -7711,9 +7759,13 @@ CreateCheckPoint(int flags) DELAY_CHKPT_START)); } pfree(vxids); + INSTR_TIME_SET_CURRENT(phase_end); + INSTR_TIME_SUBTRACT(phase_end, phase_start); + delay_start_ms = INSTR_TIME_GET_MILLISEC(phase_end); CheckPointGuts(checkPoint.redo, flags); + INSTR_TIME_SET_CURRENT(phase_start); vxids = GetVirtualXIDsDelayingChkpt(&nvxids, DELAY_CHKPT_COMPLETE); if (nvxids > 0) { @@ -7728,6 +7780,9 @@ CreateCheckPoint(int flags) DELAY_CHKPT_COMPLETE)); } pfree(vxids); + INSTR_TIME_SET_CURRENT(phase_end); + INSTR_TIME_SUBTRACT(phase_end, phase_start); + delay_complete_ms = INSTR_TIME_GET_MILLISEC(phase_end); /* * Take a snapshot of running transactions and write this to WAL. This @@ -7751,7 +7806,11 @@ CreateCheckPoint(int flags) shutdown ? XLOG_CHECKPOINT_SHUTDOWN : XLOG_CHECKPOINT_ONLINE); + INSTR_TIME_SET_CURRENT(phase_start); XLogFlush(recptr); + INSTR_TIME_SET_CURRENT(phase_end); + INSTR_TIME_SUBTRACT(phase_end, phase_start); + xlogflush_ms = INSTR_TIME_GET_MILLISEC(phase_end); /* * We mustn't write any new WAL after a shutdown checkpoint, or it will be @@ -7785,6 +7844,7 @@ CreateCheckPoint(int flags) /* * Update the control file. */ + INSTR_TIME_SET_CURRENT(phase_start); LWLockAcquire(ControlFileLock, LW_EXCLUSIVE); if (shutdown) ControlFile->state = DB_SHUTDOWNED; @@ -7803,6 +7863,9 @@ CreateCheckPoint(int flags) UpdateControlFile(); LWLockRelease(ControlFileLock); + INSTR_TIME_SET_CURRENT(phase_end); + INSTR_TIME_SUBTRACT(phase_end, phase_start); + ctlfile_ms = INSTR_TIME_GET_MILLISEC(phase_end); /* * We are now done with critical updates; no need for system panic if we @@ -7832,7 +7895,11 @@ CreateCheckPoint(int flags) /* * Let smgr do post-checkpoint cleanup (eg, deleting old files). */ + INSTR_TIME_SET_CURRENT(phase_start); SyncPostCheckpoint(); + INSTR_TIME_SET_CURRENT(phase_end); + INSTR_TIME_SUBTRACT(phase_end, phase_start); + syncpost_ms = INSTR_TIME_GET_MILLISEC(phase_end); /* * Update the average distance between checkpoints if the prior checkpoint @@ -7861,8 +7928,12 @@ CreateCheckPoint(int flags) KeepLogSeg(recptr, &_logSegNo); } _logSegNo--; + INSTR_TIME_SET_CURRENT(phase_start); RemoveOldXlogFiles(_logSegNo, RedoRecPtr, recptr, checkPoint.ThisTimeLineID); + INSTR_TIME_SET_CURRENT(phase_end); + INSTR_TIME_SUBTRACT(phase_end, phase_start); + removewal_ms = INSTR_TIME_GET_MILLISEC(phase_end); /* * Make more log segments if needed. (Do this after recycling old log @@ -7878,8 +7949,24 @@ CreateCheckPoint(int flags) * in subtrans.c). During recovery, though, we mustn't do this because * StartupSUBTRANS hasn't been called yet. */ + INSTR_TIME_SET_CURRENT(phase_start); if (!RecoveryInProgress()) TruncateSUBTRANS(GetOldestTransactionIdConsideredRunning()); + INSTR_TIME_SET_CURRENT(phase_end); + INSTR_TIME_SUBTRACT(phase_end, phase_start); + truncsub_ms = INSTR_TIME_GET_MILLISEC(phase_end); + + /* Log phase breakdown for diagnosing slow checkpoints. */ + if (log_checkpoints) + ereport(LOG, + (errmsg("checkpoint phase breakdown: " + "SyncPre=%.3f s, DelayStart=%.3f s, DelayComplete=%.3f s, " + "XLogFlush=%.3f s, ControlFile=%.3f s, SyncPost=%.3f s, " + "RemoveWAL=%.3f s, TruncSub=%.3f s", + syncpre_ms / 1000.0, delay_start_ms / 1000.0, + delay_complete_ms / 1000.0, xlogflush_ms / 1000.0, + ctlfile_ms / 1000.0, syncpost_ms / 1000.0, + removewal_ms / 1000.0, truncsub_ms / 1000.0))); /* Real work is done; log and update stats. */ LogCheckpointEnd(false, flags); @@ -8051,6 +8138,9 @@ CheckPointGuts(XLogRecPtr checkPointRedo, int flags) CheckPointRelationMap(); CheckPointReplicationOrigin(); + /* Persist UNDO log discard pointers and log statistics */ + CheckPointUndoLog(); + /* Write out all dirty data in SLRUs and the main buffer pool */ TRACE_POSTGRESQL_BUFFER_CHECKPOINT_START(flags); CheckpointStats.ckpt_write_t = GetCurrentTimestamp(); @@ -8080,6 +8170,13 @@ CheckPointGuts(XLogRecPtr checkPointRedo, int flags) CheckPointSnapBuild(); CheckPointLogicalRewriteHeap(); CheckPointTwoPhase(checkPointRedo); + + /* + * Persist the Aborted Transaction Map so it survives a crash even when + * this checkpoint advances the redo pointer past an un-forgotten + * XLOG_ATM_ABORT record (see CheckPointATM / ATMReloadFromCheckpoint). + */ + CheckPointATM(); } /* @@ -8549,6 +8646,26 @@ KeepLogSeg(XLogRecPtr recptr, XLogSegNo *logSegNo) segno = unsummarized_segno; } + /* + * If UNDO-in-WAL is active, retain WAL segments that contain UNDO records + * still needed for rollback of in-progress transactions. + * + * Scan live per-backend UNDO batch LSN slots at every checkpoint rather + * than using the worker-updated cached horizon, to ensure WAL retention + * is accurate even when the UNDO worker lags. + */ + { + keep = UndoGetOldestBatchLSN(); + if (XLogRecPtrIsValid(keep)) + { + XLogSegNo undo_segno; + + XLByteToSeg(keep, undo_segno, wal_segment_size); + if (undo_segno < segno) + segno = undo_segno; + } + } + /* but, keep at least wal_keep_size if that's set */ if (wal_keep_size_mb > 0) { diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c index 5f3b065b8942e..3d5140ef990bb 100644 --- a/src/backend/access/transam/xlogrecovery.c +++ b/src/backend/access/transam/xlogrecovery.c @@ -30,8 +30,11 @@ #include #include +#include "access/relundo.h" #include "access/timeline.h" #include "access/transam.h" +#include "access/undo_xlog.h" +#include "access/undolog.h" #include "access/xact.h" #include "access/xlog_internal.h" #include "access/xlogarchive.h" @@ -1860,6 +1863,51 @@ PerformWalRecovery(void) (errmsg("last completed transaction was at log time %s", timestamptz_to_str(xtime)))); + /* + * ARIES-style undo phase: roll back incomplete transactions that + * wrote UNDO records (XLOG_UNDO_BATCH) but did not commit. + * + * During the redo phase above, UndoRecoveryTrackBatch() was called + * from the XLOG_UNDO_BATCH redo handler to record which transactions + * have UNDO data. UndoRecoveryRemoveXid() was called from the + * XLOG_XACT_COMMIT and XLOG_XACT_ABORT redo handlers to remove + * completed transactions. Any remaining entries represent incomplete + * transactions that need their UNDO chains walked for rollback. + * + * We check UndoRecoveryNeeded() to avoid overhead when no UNDO + * records were present in the WAL stream. + */ + if (UndoRecoveryNeeded()) + { + ereport(LOG, + (errmsg("starting undo phase for incomplete transactions"))); + PerformUndoRecovery(); + ereport(LOG, + (errmsg("undo phase complete"))); + } + + /* + * Reverse-apply per-relation UNDO (in-place before-images) for loser + * transactions. Like PerformUndoRecovery() above, this runs before + * WAL insertion is enabled, so it writes no WAL; durability is + * provided by the end-of-recovery checkpoint. + * + * Unlike the cluster-wide UNDO phase, this is driven by an + * end-of-redo scan of the UNDO forks on disk rather than redo-time + * tracking: a CHECKPOINT taken after an uncommitted in-place UPDATE + * advances the redo start past that UPDATE's WAL, so nothing is + * replayed for it even though its uncommitted value is durable. + * + * This block is reached only once redo has finished -- at crash + * recovery end or at standby promotion. A streaming hot standby + * never arrives here. At promotion StandbyMode is still set (it is + * cleared later in FinishWalRecovery), so gating on !StandbyMode + * would skip the scan for exactly the promoted-standby case that + * needs loser rollback. Run it unconditionally, matching + * PerformUndoRecovery() above. + */ + PerformRelUndoRecovery(); + InRedo = false; } else diff --git a/src/backend/access/undo/Makefile b/src/backend/access/undo/Makefile new file mode 100644 index 0000000000000..f7273f30a6a57 --- /dev/null +++ b/src/backend/access/undo/Makefile @@ -0,0 +1,39 @@ +#------------------------------------------------------------------------- +# +# Makefile-- +# Makefile for access/undo +# +# IDENTIFICATION +# src/backend/access/undo/Makefile +# +#------------------------------------------------------------------------- + +subdir = src/backend/access/undo +top_builddir = ../../../.. +include $(top_builddir)/src/Makefile.global + +OBJS = \ + atm.o \ + logical_revert_worker.o \ + relundo.o \ + relundo_apply.o \ + relundo_discard.o \ + relundo_page.o \ + relundo_recovery.o \ + relundo_worker.o \ + relundo_xlog.o \ + slog.o \ + undo.o \ + undo_bufmgr.o \ + undo_xlog.o \ + undoapply.o \ + undobuffer.o \ + undoinsert.o \ + undolog.o \ + undorecord.o \ + undormgr.o \ + undostats.o \ + undoworker.o \ + xactundo.o + +include $(top_srcdir)/src/backend/common.mk diff --git a/src/backend/access/undo/README b/src/backend/access/undo/README new file mode 100644 index 0000000000000..b0fb825d2ec51 --- /dev/null +++ b/src/backend/access/undo/README @@ -0,0 +1,1143 @@ +UNDO Log Management for PostgreSQL +=================================== + +This directory contains the implementation of the generic UNDO log system +for PostgreSQL, providing transactional UNDO logging for in-place-update +table operations, transactional rollback, and constant-time recovery (CTR). + +## 1. Architecture Overview + +The UNDO system adds a separate, append-only log that records the inverse +of each data modification. Every INSERT, DELETE, UPDATE, and PRUNE +operation on an UNDO-enabled table writes a record to the UNDO log +before (or just after, for INSERT) the actual modification. This +enables two key capabilities: + + 1. **Transaction rollback**: On ABORT, the UNDO chain is walked backward + and each operation is reversed (delete the inserted row, re-insert + the deleted row, etc.). + + 2. **Constant-time recovery (CTR)**: Committed work is durable + immediately through ordinary WAL replay, and physical rollback of + aborted transactions is applied by background workers rather than + synchronously at abort or crash-open time. ROLLBACK returns in O(1) + for the client while the physical reversal of the aborted + transaction's changes is deferred to background work. + +### UNDO Chain Model + +Each transaction that modifies an UNDO-enabled table builds a backward +chain of UNDO records: + + newest record --> ... --> oldest record + (currentUndoPtr) (firstUndoPtr) + +The chain is linked through the `urec_prev` field in each record header. +During rollback, the chain is traversed from `firstUndoPtr` forward +through the contiguous buffer written by UndoRecordSetInsert, then +follows `urec_prev` links to earlier batches. + +Subtransaction commit merges the child's chain into the parent. +Subtransaction abort applies the child's chain immediately. + +### Opt-In Model + +UNDO is always-on infrastructure. Table access methods opt in by +implementing the am_supports_undo callback. + +### Two UNDO Facilities: Cluster-Wide UndoLog vs. Per-Relation RelUndo + +The subsystem provides two distinct UNDO facilities, and a consumer chooses +the one whose locality and discard model fits its workload. They are not +redundant; each exists because the other is a poor fit for one class of +consumer. + + 1. **Cluster-wide UndoLog** (undolog.c, undorecord.c, undoapply.c): + one logical, append-only UNDO stream shared by all consumers, embedded + in the WAL (XLOG_UNDO_BATCH). Discard is governed globally by a single + cluster-wide horizon (undo_discard_horizon): a batch's WAL is retained + until every transaction that could still need it has resolved, then the + WAL segment is recycled normally. This suits **sparse, cross-object** + UNDO -- a transaction that touches a handful of unrelated objects (for + example filesystem operations, or an index AM logging a few structural + changes) writes one interleaved chain, and the single global horizon is + the right granularity because such operations are not concentrated on + any one relation. + + 2. **Per-relation RelUndo fork** (relundo.c, relundo_page.c, + relundo_apply.c, RELUNDO_FORKNUM): a dedicated UNDO fork living + alongside each relation's data, discarded per-relation on that + relation's own schedule (RelUndoVacuum / RelUndoMaybeVacuum). This + suits a **high-churn in-place-update table AM**, where UNDO volume is + dominated by one relation being updated repeatedly. Two properties + make the per-relation fork the right choice there: + + - **Locality.** Before-images for a relation's tuples live in that + relation's fork, next to the data they reverse, rather than + interleaved with every other consumer's records in one global + stream. Reverse-applying a chain, or reconstructing an older + version for a snapshot, reads one relation's fork, not the whole + cluster-wide log. + + - **Per-relation discard.** A hot table can reclaim its own UNDO the + instant its own oldest interesting snapshot advances, without being + pinned by an unrelated long-running transaction elsewhere in the + cluster. Under the single cluster-wide horizon, one long reader + anywhere would pin every consumer's UNDO; the per-relation fork + confines that back-pressure to the relation that actually has the + old snapshot. + + A cluster-wide consumer never touches the RelUndo fork, and an in-place + table AM never has to share the global horizon with sparse cross-object + UNDO. The two facilities share only the UNDO record header format and the + rollback-dispatch machinery (undormgr.c); their storage and discard paths + are independent. + + +## Recovery Model: Constant-Time Recovery (CTR), not ARIES + +This UNDO system implements Constant-Time Recovery (CTR), the model from +Antonopoulos et al., "Constant Time Recovery in Azure SQL Database" +(VLDB 2019) -- see "References" below. This is a deliberate design +choice, not an incomplete or informal implementation of ARIES's classic +analysis/redo/undo three-phase model with a backward log scan. The two +models differ in a way that matters operationally: under ARIES, +database-open after a crash waits for the undo phase (rolling back every +loser transaction) to finish; under CTR, rollback of already-committed +work is instantaneous from the client's perspective and physical +undo/rollback is deferred to background work that runs after (and +overlapping with) new transactions, not before them. + +The two phases are: + +**Redo (physical, synchronous, ordinary WAL replay):** Standard +PostgreSQL WAL replay applies all logged changes forward from the +checkpoint redo point, including XLOG_UNDO_BATCH records (which contain +UNDO payload data, not a rollback action) and any XLOG_UNDO_APPLY_RECORD +/ XLOG_RELUNDO_APPLY CLRs already generated before the crash. This is +exactly PostgreSQL's normal, single forward pass over WAL -- there is no +separate "analysis" pass building an in-memory dirty-page table the way +ARIES does; CLOG reconstruction during ordinary redo is what lets a +later step distinguish committed from aborted work. + +**Undo / rollback (logical, deferred, asynchronous):** Aborting a +transaction does NOT synchronously replay its UNDO chain the way ARIES's +undo phase does. Instead: + + - At ABORT time, xactundo.c's AtAbort_XactUndo() classifies the + transaction by UNDO volume (undo_instant_abort_threshold, default + 64KB). Small transactions apply their UNDO chain inline in the + aborting backend before ROLLBACK returns (still O(1) from the + client's perspective for typical row counts). Large transactions + take the "instant abort" path: ATMAddAborted() records the xid as + aborted in the shared-memory Aborted Transaction Map (ATM, atm.c) and + returns immediately -- ROLLBACK latency is O(1) regardless of how + much UNDO the transaction wrote, indistinguishable from a CLOG-only + abort. + - The ATM is sLog-backed (see access/slog.h): it is consulted by + an in-place-update AM's own visibility checks so that an aborted + transaction's writes are invisible to every reader immediately, before + any physical undo has run. + - A background Logical Revert worker (logical_revert_worker.c) scans + the ATM for entries not yet marked reverted and calls + ApplyUndoChainFromWAL() to physically walk and apply each one's UNDO + chain, generating a CLR (XLOG_UNDO_APPLY_RECORD) per page it + restores for crash-safety. This work is fully decoupled from + ROLLBACK's return to the client. + +This is the CTR guarantee stated precisely: physical redo is ordinary, +synchronous WAL replay (same cost and semantics as if UNDO did not +exist); logical undo of aborted work is asynchronous and does not block +transaction throughput, database open after crash, or any other +transaction's forward progress. It is not a defect that rollback +"doesn't happen immediately" for large transactions -- that deferral is +the entire point of the design. + +**Crash recovery's role:** PerformUndoRecovery() (undo_xlog.c), called +from the redo pass, does the CTR analog of ARIES's "redo incomplete +undo": each XLOG_UNDO_BATCH record is registered in an in-memory table +keyed by XID as it is replayed, and removed when a commit or abort +record for that XID is later replayed. After the forward redo pass +completes, whatever entries remain represent transactions that wrote +UNDO data but reached neither COMMIT nor ABORT before the crash -- +treated the same as any other in-flight-at-crash transaction, their UNDO +chains are walked and applied during this recovery pass (this is the one +place synchronous UNDO application happens on the recovery path itself, +because there is no live backend left to defer it to). Ordinary +already-aborted-before-the-crash transactions are handled by the ATM / +Logical Revert worker exactly as in normal runtime operation, not by a +special recovery-time undo phase. + +**Inter-transaction UNDO ordering:** Records are applied per-transaction in +newest-batch-first order within each transaction. No global LSN ordering across +concurrent aborted transactions is enforced. This is safe because PostgreSQL's +locking model prevents two concurrent transactions from holding conflicting physical +locks on the same tuple -- there can be no conflicting physical UNDO operations +between concurrent transactions. + +**Idempotency (crash-during-rollback safety):** Redo of a CLR is idempotent +through full-page-image + page-LSN comparison during ordinary WAL replay. +See "CLR idempotency" below. + +**TEMP and UNLOGGED skip:** During crash recovery, UNDOPERSISTENCE_TEMP records +are skipped (temporary tables do not survive server restart) and +UNDOPERSISTENCE_UNLOGGED records are skipped (unlogged table data forks are reset +to their empty init fork on crash recovery). This mirrors the behavior of the +standard heap AM for these persistence levels. + +## 2. UndoRecPtr Format + +UndoRecPtr is a 64-bit pointer encoding both log identity and position: + + Bits 63-40: Log number (24 bits = up to 16M logs) + Bits 39-0: Byte offset (40 bits = up to 1TB per log) + + #define MakeUndoRecPtr(logno, offset) (((uint64)(logno) << 40) | (uint64)(offset)) + #define UndoRecPtrGetLogNo(ptr) ((uint32)(((uint64)(ptr)) >> 40)) + #define UndoRecPtrGetOffset(ptr) (((uint64)(ptr)) & 0xFFFFFFFFFFULL) + +InvalidUndoRecPtr is defined as 0. Log number 0 is never allocated +(next_log_number starts at 1), so offset 0 in log 0 is always invalid. + +## 3. UNDO Record Format + +Every UNDO record starts with a fixed UndoRecordHeader (see undorecord.h). +The serialized size is given by SizeOfUndoRecordHeader: + + Offset Size Field Description + ------ ---- ----- ----------- + 0 1 urec_rmid UNDO resource manager ID (dispatches apply) + 1 1 urec_flags Generic flags (UNDO_INFO_HAS_PAYLOAD, etc.) + 2 2 urec_info RM-specific subtype and flags + 4 4 urec_len Total record length including header + payload + 8 4 urec_xid Transaction ID + 12 4 (padding) Alignment for 8-byte urec_prev + 16 8 urec_prev Previous UNDO record in chain (UndoRecPtr) + 24 4 urec_reloid Relation OID (InvalidOid if N/A) + 28 4 urec_payload_len Length of RM-specific payload that follows + +The header is AM-agnostic. The urec_rmid field identifies the resource manager +that owns the record. Block number, offset, and tuple data are part of the +RM-specific opaque payload, not the generic header. + +The header carries no per-record "already applied" marker. Idempotency of +rollback under crash-during-rollback is achieved through full-page-image + +page-LSN comparison during ordinary WAL redo; see "CLR idempotency" below for +the mechanism. + +### Record Types (example: an in-place-update table AM's resource manager) + +The engine treats urec_info record types as opaque and RM-specific: it does +not interpret them. During rollback it dispatches each record to the owning +resource manager's rm_undo callback (selected by urec_rmid), and that RM +defines and interprets its own record types in its own urec_info space. + +The types below are an example set that a consumer providing an +in-place-update table AM defines for its own resource manager. They belong +to that consumer, not to the engine, and the reversal semantics operate on +that consumer's page format (the engine is page-format-agnostic): + + UNDO_INSERT (0x0001) Marks an INSERT; no tuple payload needed. + Rollback: ItemId marked dead (indexed) or unused. + + UNDO_DELETE (0x0002) Stores the full old tuple. + Rollback: restore the old tuple bytes to the page. + + UNDO_UPDATE (0x0003) Stores the old tuple version. + Rollback: restore the old tuple bytes to the + original location. + + UNDO_PRUNE (0x0004) Stores a pruned tuple (LP_DEAD or LP_UNUSED). + Not rolled back; retained for diagnostics. + + UNDO_INPLACE (0x0005) Stores old data from in-place update. + Rollback: restore the old tuple bytes in place. + +Other resource managers (e.g., nbtree with UNDO_RMID_BTREE) define their own +record types in their own urec_info space. + +### Payload + +The payload is an opaque byte sequence whose interpretation is entirely +RM-specific. For the example in-place-update AM above, DELETE/UPDATE/PRUNE/INPLACE +payloads contain a small RM-specific header (block number, offset, tuple +length) followed by the consumer's raw tuple data. INSERT records have no +payload (urec_payload_len = 0). + +## 4. Storage Architecture (UNDO-in-WAL) + +UNDO records are embedded directly in the standard WAL stream as +XLOG_UNDO_BATCH records. There are NO separate UNDO segment files or +directories. Embedding UNDO in the WAL eliminates a separate storage +tier and leverages existing WAL infrastructure for durability, replication, +and archival. + +WAL retention of UNDO batches is governed by undo_discard_horizon, which +is the oldest XLogRecPtr still needed by either: + (a) an in-flight transaction that may abort (always retained), or + (b) the Logical Revert Worker's pending queue (ATM entries). + +UNDO records for unresolved (uncommitted/unaborted) transactions are +NEVER discarded regardless of any retention timer. + +## 5. Module Organization + +The undo subsystem is split into several modules with clean separation +of concerns, following the architecture of the EDB undo-record-set branch: + + undo.c - Central coordination: UndoShmemSize/UndoShmemInit + aggregates all subsystem shared memory needs. + UndoContext memory context management. + + undolog.c - UNDO log control structures and WAL batch coordination. + UndoLogControl/UndoLogSharedData structures. + + undorecord.c - UndoRecordSet and UndoRecordHeader: record format, + serialization, deserialization, and batch buffering. + + xactundo.c - Per-transaction undo management. Maintains up to 3 + UndoRecordSets per transaction (one per persistence + level: permanent, unlogged, temporary). Hooks into + xact.c via AtCommit/AtAbort_XactUndo. + + undoapply.c - Physical undo application during rollback. Walks the + undo chain backward and applies page-level restores + via memcpy. Generates CLRs for crash safety. + + undoinsert.c - Batch insertion of accumulated records into undo log. + + undo_xlog.c - WAL redo routines for the RM_UNDO_ID resource manager. + Handles CLR replay (XLOG_UNDO_APPLY_RECORD) using + full page images via XLogReadBufferForRedo. + + undo_bufmgr.c - Buffer management mapping undo logs into shared_buffers. + Virtual RelFileLocator: spcOid=1663, dbOid=9, + relNumber=log_number. + + undostats.c - Statistics and monitoring functions. + + undoworker.c - Background worker for undo record discard. + + undormgr.c - UNDO resource manager registry. RegisterUndoRmgr() + allows any AM to register an rm_undo callback keyed + by urec_rmid. undoapply.c dispatches to these callbacks. + + undobuffer.c - AM-agnostic Tier 2 UNDO write buffer. Accumulates + serialized UndoRecordHeaders in a per-backend buffer, + embedded into DML WAL records or flushed as standalone + XLOG_UNDO_BATCH records. Used by heapam and nbtree. + +### Key Types (from undodefs.h) + + UndoRecPtr - 64-bit pointer to an undo record + UndoPersistenceLevel - Enum: PERMANENT, UNLOGGED, TEMP + NUndoPersistenceLevels - 3 (array index bound) + UndoRecordSet - Opaque batch container for undo records + UndoRecordSetType - URST_TRANSACTION, URST_MULTI, URST_EPHEMERAL + UndoRecordSetChunkHeader - On-disk chunk header for multi-chunk sets + +### Initialization Flow + + ipci.c calls UndoShmemSize() and UndoShmemInit() from undo.c which + in turn calls each subsystem: + + UndoShmemSize() = UndoLogShmemSize() + + XactUndoShmemSize() + + UndoWorkerShmemSize() + + UndoShmemInit() -> UndoLogShmemInit() + -> XactUndoShmemInit() + -> UndoWorkerShmemInit() + + Per-backend initialization is done by InitializeUndo() which calls + InitializeXactUndo() and registers the exit callback. + +## 6. Shared Memory Structures (detail) + +### UndoLogSharedData + +Global control structure in shared memory: + + - logs[MAX_UNDO_LOGS] Array of UndoLogControl (one per active log) + - next_log_number Counter for allocating new log numbers + - allocation_lock LWLock protecting log allocation + +### UndoLogControl + +Per-log metadata (one per active log slot): + + - log_number Log file identity + - insert_ptr UndoRecPtr of next insertion position + - discard_ptr UndoRecPtr; data before this has been discarded + - oldest_xid Oldest transaction still referencing this log + - lock LWLock protecting concurrent access + - in_use Whether this slot is active + +### UNDO Buffer Manager (undo_bufmgr.c) + +UNDO log blocks are managed through PostgreSQL's standard shared_buffers +pool via undo_bufmgr.c. Each undo log is mapped to a virtual +RelFileLocator (spcOid=1663, dbOid=UNDO_DB_OID=9, relNumber=log_number) +and accessed via ReadBufferWithoutRelcache(). This provides: + + - Unified buffer management (no separate cache to tune) + - Automatic clock-sweep eviction via shared_buffers + - Built-in dirty buffer tracking and checkpoint support + - Standard buffer locking and pin semantics + +## 7. Physical UNDO Application (undoapply.c) + +The core design decision is **physical** UNDO application: during rollback, +stored tuple data is copied directly back to heap pages via memcpy, rather +than using logical operations (simple_heap_delete, simple_heap_insert). + +### Why Physical Over Logical + +Physical application stores the complete before-image and restores it with a +direct page copy, so it cannot fail: there is no executor path, no index +maintenance, no page-full or TOAST complication, and no visibility check to +fail during rollback. Logical application, which would reconstruct the +inverse operation through table-AM logic, can fail on exactly those +conditions and therefore is not used on the rollback path. + + Physical (used here): + - Stores: Complete tuple data (HeapTupleHeaderData + payload) + - Apply: Direct memcpy to restore exact page state + - Safety: Cannot fail (no page-full, no toast, no index conflicts) + - WAL: CLR with full page image (~8 KB per record) + + Logical (not used on the rollback path): + - Stores: Operation metadata (INSERT/DELETE/UPDATE type + TID) + - Apply: Reconstruct operation using table AM logic + - Safety: Can fail on page-full, toast complications, visibility checks + - WAL: Standard heap WAL records (~50-100 bytes per record) + +### Critical Section Pattern + +Each UNDO application follows this pattern (from ApplyOneUndoRecord): + + 1. Open relation with RowExclusiveLock + 2. ReadBuffer to get the target page + 3. LockBuffer(BUFFER_LOCK_EXCLUSIVE) + 4. START_CRIT_SECTION + 5. Physical modification (memcpy / ItemId manipulation) + 6. MarkBufferDirty + 7. Generate CLR via XLogInsert(RM_UNDO_ID, XLOG_UNDO_APPLY_RECORD) + with REGBUF_FORCE_IMAGE for full page image + 8. PageSetLSN(page, lsn) + 9. END_CRIT_SECTION + 10. UnlockReleaseBuffer + +There is no "write a CLR pointer back into the UNDO record" step: an UNDO +record is never updated after it is written. Step 8 (PageSetLSN) is what makes +step 9 onward idempotent -- see "CLR idempotency" below. + +Key principle: **UNDO record I/O (reading) occurs BEFORE the critical +section. Only the page modification and WAL write occur inside the +critical section.** + +### CLR idempotency + +The UNDO record header carries no per-record "already applied" marker. +Idempotency comes from full-page-image (FPI) + page-LSN comparison, +the same mechanism ordinary WAL redo uses everywhere else in PostgreSQL: + + 1. Every CLR (XLOG_UNDO_APPLY_RECORD) is written with + REGBUF_FORCE_IMAGE, so it always carries a full-page image of the + page it restores. + 2. After the physical page modification, PageSetLSN(page, lsn) sets + the page's LSN to the CLR's own LSN. + 3. If a crash happens and recovery replays this CLR again (because + redo starts from a checkpoint before the CLR), the standard + XLogReadBufferForRedo() call used by the CLR's own redo handler + compares the on-disk page's LSN to the record being replayed: if + the page's LSN is already >= the record's LSN, it returns + BLK_DONE (page already reflects this change or a later one) or + BLK_RESTORED (an FPI was applied) rather than BLK_NEEDS_REDO, and + no further modification happens. + 4. If instead a NEW rollback attempt (not a WAL replay) re-walks the + same UNDO chain -- e.g. ApplyUndoChainFromWAL() called again after + an interrupted first attempt -- there is no per-record "already + applied" flag to check at all; the record's rm_undo callback is + simply re-invoked. This is safe because the callbacks are + themselves idempotent: restoring the same before-image bytes twice + produces the same final page state (memcpy is idempotent), and any + second CLR this produces is itself subject to the same page-LSN + redo-skip logic in step 3. + +This prevents double-application and enables idempotent crash recovery +without ever needing to mutate an already-written UNDO record. + +## 8. WAL Integration + +### Resource Managers and the RM-ID Namespaces + +The UNDO subsystem introduces new WAL resource-manager IDs (in +access/rmgrlist.h) and, separately, a set of UNDO-record dispatch IDs (in +access/undormgr.h). These are two different namespaces and both are shared, +finite resources, so each new ID must be justified and, once shipped, kept +stable. + +**WAL resource-manager IDs (rmgrlist.h).** A WAL RM ID is stamped into +every WAL record and is limited to a one-byte field (RM_MAX_ID = 255, with +128..255 reserved for custom/extension RMs). A facility needs its own WAL +RM only when it emits WAL records with a redo/desc/identify behavior distinct +from every existing RM; sharing another RM's opcode space would force that +RM's redo routine to understand records it does not own. The UNDO stack adds +these, each because it replays a distinct on-disk change: + + RM_UNDO_ID "Undo" Cluster-wide UndoLog: batch allocation, discard, + log extension, and physical CLRs + (XLOG_UNDO_APPLY_RECORD). Its own RM because + none of these map onto any table/index RM's redo. + RM_ATM_ID "ATM" Aborted Transaction Map: XLOG_ATM_ABORT / + XLOG_ATM_FORGET, which reconstruct the + constant-time-recovery abort map during redo. + Distinct lifecycle from UndoLog batches, so a + distinct RM. + RM_RELUNDO_ID "RelUndo" Per-relation UNDO fork: metapage/page changes + and per-relation CLRs. Needs startup/cleanup and + mask callbacks the UndoLog RM does not, hence its + own RM. + + A consumer that writes UNDO through one of these facilities does NOT get + its own WAL RM for that purpose; it reuses the facility's RM. A consumer + adds a WAL RM only for WAL it emits itself that no existing RM can redo + (for example a table AM's own heap-change records). Those consumer WAL RMs + are added in the consumer's own commit, at the end of rmgrlist.h, so + existing IDs never shift (IDs are position-defined and WAL-durable). + +**UNDO-record dispatch IDs (undormgr.h, UNDO_RMID_*).** Independently of WAL +RM IDs, every UNDO record header carries a urec_rmid that selects which +registered rm_undo callback reverses it during rollback (see +access/undormgr.h and access/undormgrlist.h). This namespace is also finite +(MAX_UNDO_RMGRS = 256) and stamped into WAL-durable UNDO records, so IDs must +be unique and stable. The core header defines only UNDO_RMID_INVALID and the +built-in index-AM IDs; every other consumer defines its own UNDO_RMID_* +constant in its own header, in the same commit that registers it via +access/undormgrlist.h, so the core names no specific consumer. A new consumer +takes the next free value and never reuses a retired one. + +A resource manager is registered for UNDO-related WAL: + + RM_UNDO_ID (23) - UNDO log management operations + +### UNDO WAL Record Types + + XLOG_UNDO_ALLOCATE (0x00) Space allocated in UNDO log. + Fields: start_ptr, length, xid, log_number + + XLOG_UNDO_DISCARD (0x10) Discard pointer advanced. + Fields: discard_ptr, oldest_xid, log_number + + XLOG_UNDO_EXTEND (0x20) Log file extended. + Fields: log_number, new_size + + XLOG_UNDO_APPLY_RECORD (0x30) CLR: Physical UNDO applied to page. + Fields: urec_ptr, xid, target_locator, target_block, + target_offset, operation_type + Always includes REGBUF_FORCE_IMAGE (full page image). + +### WAL Replay + +During crash recovery: + + undo_redo() replays UNDO WAL records: + - ALLOCATE: Creates/updates log control structures, advances insert_ptr + - DISCARD: Updates discard_ptr and oldest_xid + - EXTEND: Extends the physical log file + - APPLY_RECORD: CLR -- restores full page image via XLogReadBufferForRedo. + Since CLRs use REGBUF_FORCE_IMAGE, the page is restored + directly from the WAL record without re-reading UNDO data. + +## 9. Recovery Process + +The UNDO system follows the Constant-Time Recovery (CTR) model described +above, not ARIES's analysis/redo/undo three-phase model: + + Redo: Ordinary, synchronous WAL replay forward (includes UNDO + allocations and any CLRs generated before the crash) -- + this is PostgreSQL's normal single forward pass, not a + distinct ARIES "redo phase" preceded by a separate analysis + pass. + Undo: Logical/deferred. Transactions that were in-flight (neither + committed nor aborted) at crash time are rolled back as part + of PerformUndoRecovery() during this recovery run (see + "Recovery Model" above). Transactions that had already + aborted before the crash are handled exactly as they are + during normal runtime operation -- via the ATM and the + background Logical Revert worker, not a recovery-time undo + phase. + +During normal operation, UNDO rollback for a small transaction is handled +in-process by ApplyUndoChainFromWAL() called from AtAbort_XactUndo() +(xactundo.c) on abort; for a large transaction it is deferred to the ATM +/ Logical Revert worker (see "Recovery Model" above) rather than run +synchronously in the aborting backend. + +During crash recovery, the UNDO log state is reconstructed by +redo (including replaying any CLRs generated before the crash), +and any transactions that were in progress at crash time will be +rolled back as part of normal recovery. + +### ApplyUndoChainFromWAL() -- Physical Application + +Walks the UNDO chain backward from the most recent WAL batch, +applying each record using +physical page modifications (memcpy, ItemId manipulation): + + INSERT -> ItemIdSetDead (if indexed) or ItemIdSetUnused + DELETE -> memcpy(page_htup, tuple_data, tuple_len) to restore old tuple + UPDATE -> memcpy(page_htup, tuple_data, tuple_len) to restore old version + PRUNE -> skipped (informational only) + INPLACE -> memcpy(page_htup, tuple_data, tuple_len) to restore old data + +For each applied record, a CLR is generated via XLogInsert with +REGBUF_FORCE_IMAGE. An UNDO record is never rewritten after it is applied -- +see "CLR idempotency" above for the mechanism actually in use +(full-page image + page-LSN comparison during redo). + +Physical application restores the exact before-image bytes with a direct +page copy, so it cannot fail during rollback: it does not go through the +executor path, does not maintain indexes, and does not run visibility +checks. This is why the rollback path stores and restores complete tuple +bytes rather than reconstructing the inverse operation logically. + +Error handling on the rollback path treats a concurrently dropped relation +as the one legitimate skip: if the relation an UNDO record targets has been +dropped, its storage is gone and its UNDO is moot, so a WARNING is emitted +and processing continues to the next record. This is not a general +tolerance for un-appliable records: a live relation whose before-image +cannot be restored during rollback is a corruption condition, not a +warn-and-continue case. + +### Crash During Rollback + +If a crash occurs during rollback: + + 1. Recovery replays WAL forward, including any CLRs already generated. + 2. Pages modified by already-applied UNDO records are restored via + the full page images in the CLRs, via the ordinary redo path's + page-LSN check (XLogReadBufferForRedo returning BLK_DONE / + BLK_RESTORED for a page already at or past the CLR's LSN). + 3. Any UNDO record whose page-level change was already durable is + therefore a no-op if its rm_undo callback runs again; this is decided + by the page LSN, not by any per-record applied marker. + 4. Remaining UNDO records are applied normally, generating new CLRs. + +Result: Rollback always completes, even after repeated crashes. + +## 10. UNDO Discard Worker + +The undoworker background process (undoworker.c) periodically scans +active transactions and advances discard pointers: + + 1. Queries ProcArray for the oldest active transaction + 2. Identifies UNDO records older than oldest_xid + 3. Advances discard_ptr (WAL-logged via XLOG_UNDO_DISCARD) + 4. Future: physically truncates/deletes reclaimed log files + +### GUC Parameters + + undo_worker_naptime Sleep interval between discard cycles (ms) + Default: 60000 (1 minute) + + undo_retention_time Minimum retention time for UNDO records (ms) + Default: 3600000 (1 hour) + +## 11. Performance Characteristics + +### Zero Overhead When Disabled + +For AMs that do not support UNDO, the only overhead is the +am_supports_undo check -- a single pointer dereference and comparison. +No UNDO allocations, writes, or locks are taken. + +### Overhead When Active + + INSERT: One UNDO record (header only, no payload). ~40 bytes + (SizeOfUndoRecordHeader). + DELETE: One UNDO record + full tuple copy. 40-byte header + t_len bytes. + UPDATE: One UNDO record + old tuple copy. 40-byte header + t_len bytes. + PRUNE: One UNDO record per pruned tuple. Batched via UndoRecordSet. + +UNDO I/O occurs outside critical sections to avoid holding buffer locks +during writes. For INSERT, UNDO is generated after END_CRIT_SECTION. +For DELETE/UPDATE/PRUNE, UNDO is generated before START_CRIT_SECTION. + +### Abort Overhead + + ABORT: Each UNDO record applied during rollback generates a CLR + WAL record with a full page image (~8 KB per record). + Abort latency increases approximately 20-50% compared to + PostgreSQL's default rollback, which generates no WAL. + WAL volume per abort increases significantly due to CLRs. + + RECOVERY: Checkpoint time increases 7-15% due to more dirty buffers. + Recovery time increases 10-20% due to CLR replay. + +Trade-off: Higher abort overhead in exchange for crash safety and +standby support. For workloads where aborts are rare, the overhead +is negligible. + +### Buffer Cache + +UNDO blocks share the standard shared_buffers pool with heap and index +data. No separate cache tuning is needed; the standard shared_buffers +setting controls memory available for all buffer types including UNDO. + +## 13. Monitoring and Troubleshooting + +### Monitoring Functions + + pg_stat_get_undo_logs() Per-log statistics (size, discard progress) + pg_stat_get_undo_buffers() Buffer hit/miss/eviction statistics + pg_undo_force_discard() Force discard of old UNDO records + +### Key Log Messages + + DEBUG1 "created UNDO log file: ..." + DEBUG1 "applying UNDO chain starting at ..." + DEBUG2 "transaction %u committed with UNDO chain starting at %llu" + DEBUG2 "UNDO log %u: discard pointer updated to offset %llu" + WARNING "UNDO rollback: relation %u no longer exists, skipping" + +### Common Issues + + "too many UNDO logs active" + The compile-time limit MAX_UNDO_LOGS (100) was reached. Each + concurrent writer to an UNDO-enabled table needs an active log. + + "UNDO log %u would exceed segment size" + The segment capacity threshold was reached. The UNDO log will + seal the current segment and rotate to a fresh one via + UndoLogSealAndRotate(). If rotation fails due to backpressure, + the transaction may block until space is reclaimed. + + Growing WAL retention from UNDO + Check that the UNDO worker is running (pg_stat_activity). + Verify undo_retention_time is not set too high. + Long-running transactions prevent discard. + +## 14. File Structure + +### Backend Implementation (src/backend/access/undo/) + + undo.c Central coordination, shared memory aggregation + undolog.c Core log file management, allocation, I/O, segment rotation + undorecord.c Record format, serialization, UndoRecordSet + undoinsert.c Batch insertion of accumulated records + undoapply.c Physical rollback: ApplyUndoChainFromWAL(), memcpy-based restore, CLRs + xactundo.c Per-transaction undo management, per-persistence-level sets + undo_xlog.c WAL redo routines, CLR replay, segment rotation WAL + undo_bufmgr.c shared_buffers integration, virtual RelFileLocator mapping + undoworker.c Background discard worker, rotation checks + undostats.c Statistics collection, segment state tracking + undormgr.c UNDO resource manager dispatch (RegisterUndoRmgr, per-AM callbacks) + undobuffer.c AM-agnostic Tier 2 UNDO write buffer (UndoBufferBegin/End/Flush) + +### Header Files (src/include/access/) + + undodefs.h Core type definitions (UndoRecPtr, UndoPersistenceLevel) + undo.h Central coordination API + undolog.h UndoLogControl, UndoLogSharedData, log management API + undorecord.h UndoRecordHeader, record types, UndoRecordSet + undo_xlog.h WAL record structures (xl_undo_allocate, xl_undo_apply, etc.) + xactundo.h Per-transaction undo API (PrepareXactUndoData, etc.) + undoworker.h Worker shared memory and GUC declarations + undo_bufmgr.h shared_buffers wrapper API for UNDO log blocks + undostats.h Statistics structures and functions + undormgr.h UNDO resource manager registration API (per-AM dispatch) + undobuffer.h AM-agnostic Tier 2 write buffer API + +### Modified Core Files + + src/backend/access/heap/heapam.c INSERT/DELETE/UPDATE UNDO logging, + RelationHasUndo() helper + src/backend/access/heap/heapam_handler.c begin/finish_bulk_insert -> UndoBuffer + src/backend/access/nbtree/nbtree_undo.c B-tree index UNDO RM (INSERT_LEAF etc.) + src/backend/access/heap/pruneheap.c PRUNE UNDO logging + src/backend/access/transam/xact.c Transaction UNDO chain tracking + src/backend/access/transam/rmgr.c Resource manager registration + src/backend/storage/ipc/ipci.c Shared memory initialization + src/include/access/rmgrlist.h RM_UNDO_ID + src/include/access/heapam.h RelationHasUndo() declaration + src/include/access/xact.h UNDO chain accessors + +## 15. Limitations and Future Work + +### Current Limitations + + - TOAST tables are ordinary heap relations and are not UNDO-protected; + wide-value rollback is handled by the base-table UNDO plus ordinary + VACUUM of the dead TOAST chunks (see Section 15's TOAST notes below) + - No delta compression for UPDATE records (full old tuple stored) + - ProcArray integration for oldest XID is simplified + - Reads use xmin/xmax + CLOG (heap-compatible MVCC); an in-place-update + AM's old versions come from the per-relation UNDO fork, not UNDO-based + MVCC + +### Implemented + + - Log rotation and segment lifecycle management + (see UndoLogSealAndRotate() in undolog.c) + - AM-agnostic UNDO write buffer for reduced per-row overhead in DML + (see UndoBufferBegin() in undobuffer.c) + +### Planned Future Work + + - Delta compression for UPDATE records + - Parallel UNDO application for faster rollback + - Online UNDO log compaction + +## Known Performance Gaps and Future Work + +**Large-transaction rollback complexity:** + +The UNDO system uses two rollback strategies depending on estimated UNDO +record size (controlled by ``undo_instant_abort_threshold``, default 64 KB): + +**Small transactions (UNDO < 64 KB, roughly < 600 rows for a 2-column table):** +Synchronous rollback -- the backend walks the UNDO chain and restores heap +tuples before ROLLBACK returns. Complexity is O(N) in modified rows. Each +row requires: + + - 1 WAL batch record read (amortized: N/``undo_batch_record_limit`` reads, + default 1000 records per batch) + - 1 heap buffer write (cache hit if table fits in ``shared_buffers``; + otherwise a cold I/O read + write) + - 1 CLR WAL record written per batch (~8 KB each) + +**Large transactions (UNDO >= 64 KB, roughly >= 600 rows):** +ATM instant abort -- the backend records the XID as aborted in the +shared-memory Aborted Transaction Map (ATM) and returns immediately. +User-visible ROLLBACK latency is O(1), indistinguishable from CLOG-only +rollback. The UNDO background worker (``undo worker``) applies the UNDO +chain asynchronously, restoring heap tuples without holding the committing +backend. + +This makes large-transaction rollback invisible to end users regardless of +transaction size. The O(N) heap restoration work happens in the background. + +**b8 benchmark results -- nuc** (FreeBSD 15, amd64, 8-core, 32 GB RAM, +shared_buffers=128MB, synchronous_commit=on, 2-iteration median, +PostgreSQL 19devel, 2026-05-05, with HEAP_UNDO_DELETE_VISIBILITY_ONLY):: + + DML execution time (baseline -> UNDO OFF -> UNDO ON): + + Rows INSERT (base->off->on) UPDATE (base->off->on) DELETE (base->off->on) + ------ ----------------------- ----------------------- ----------------------- + 10K 25ms -> 28ms -> 28ms 14ms -> 13ms -> 15ms 6ms -> 6ms -> 7ms + 100K 157ms-> 154ms -> 157ms 75ms -> 81ms -> 80ms 36ms -> 36ms -> 61ms (*) + + DML overhead at 10K rows (ON vs baseline): INSERT +10%, UPDATE +7%, DELETE +12%. + The DELETE overhead at 10K dropped from the pre-optimization baseline of +19% + to +12% with HEAP_UNDO_DELETE_VISIBILITY_ONLY, which writes 8 bytes per deleted + tuple (xmax + infomask + infomask2) instead of the full 160-560 byte + before-image, reducing DELETE UNDO WAL volume by ~93-98%. + + (*) The 100K DELETE measurement has CV=40% with 2 iterations -- too noisy to + interpret. The 10K result (CV=5%) is the reliable reference for DELETE overhead. + + Rollback latency (user-visible): + + Rows Baseline UNDO OFF UNDO ON Mechanism + -------- --------- --------- -------- --------------------------------- + 10,000 <1ms <1ms <1ms All: O(1) via CLOG / ATM + 100,000 <1ms <1ms <1ms ATM instant abort (>64KB threshold) + + ROLLBACK latency is O(1) for all transaction sizes. UNDO-based rollback + matches CLOG-only rollback latency due to the ATM instant-abort path. + Background restoration completes asynchronously with no client-visible + delay. Zero dead tuples remain after rollback completes (no VACUUM debt). + + pgbench TPS (standard OLTP, 30s runs, 2-iteration median): + + Scale Clients Baseline UNDO OFF UNDO ON OFF/Base ON/Base + ----- ------- --------- --------- --------- --------- --------- + 10 1 551 TPS 281 TPS 249 TPS 0.51x 0.45x (**) + 10 4 513 TPS 506 TPS 494 TPS 0.99x 0.96x + 10 8 805 TPS 796 TPS 770 TPS 0.99x 0.96x + 50 1 243 TPS 326 TPS 256 TPS 1.34x 1.05x (**) + 50 4 562 TPS 554 TPS 559 TPS 0.99x 0.99x + 50 8 1036 TPS 1042 TPS 1017 TPS 1.01x 0.98x + + (**) The c=1 results show high variance at both scales on FreeBSD; they are + dominated by measurement artifacts rather than PostgreSQL throughput. + The c=4 and c=8 results are representative: code-presence overhead (OFF vs + baseline) is 0-1%; full UNDO overhead (ON vs baseline) is 1-4%. + + Cold-WAL rollback (crash recovery): not measured in b8; would require + WAL read from disk. Expected O(N) at ~50 MB/s WAL read throughput for + large transactions. Crash recovery times scale with unrecovered UNDO + volume at server restart. + +Compared to baseline CLOG-only rollback: + + - Baseline rollback: O(1), <1ms regardless of transaction size + - UNDO ON rollback: O(1) user-visible via ATM, O(N) background work + - UNDO advantage: zero dead tuples after rollback; no VACUUM debt + - Net VACUUM savings: eliminates dead-tuple cleanup for rolled-back rows + +Tuning: increase ``undo_batch_size_kb`` (default 256 KB) and +``undo_batch_record_limit`` (default 1000 records) to reduce the number of +WAL reads during background UNDO application. Increase +``undo_instant_abort_threshold`` to force synchronous rollback for larger +transactions (trading user-visible rollback latency for faster background +cleanup); set to 0 to always use ATM (instant abort for all sizes). + +**TOAST:** An UNDO-based table AM's TOAST table is an ordinary heap +relation and is not enrolled in any UNDO log. UNDO does not apply to the +TOAST table itself. Rollback correctness for wide values comes the +ordinary way: rolling back the base-table UPDATE (via UNDO) restores the +base tuple, which references the OLD TOAST pointer. The NEW TOAST chunks +written during the update become dead heap tuples and are reclaimed by +ordinary VACUUM, exactly as heap-on-heap TOAST behaves. No UNDO is applied +to the TOAST chunks; the base tuple simply points at the old chunks again +once its UPDATE is reversed. + +**Autovacuum interaction:** UNDO-enabled tables have no dead tuples after +a rolled-back transaction -- the tuples are physically reversed, not merely +marked dead. As a result, ``pg_stat_user_tables.n_dead_tup`` stays near +zero for UNDO-enabled tables with active DML, and the standard autovacuum +dead-tuple trigger (``autovacuum_vacuum_threshold`` + +``autovacuum_vacuum_scale_factor``) will not fire. + +This does **not** mean autovacuum is unnecessary. UNDO-enabled tables still +accumulate update-chain bloat, need FSM updates, and benefit from hint-bit +setting and index bloat cleanup. To ensure timely vacuuming on UNDO-enabled +tables, either: + + * Set ``autovacuum_vacuum_threshold = 0`` at the table level so that + even zero dead tuples triggers autovacuum based on insert/update count: + + .. code-block:: sql + + CREATE TABLE my_table (...) USING + WITH (autovacuum_vacuum_threshold = 0, + autovacuum_vacuum_scale_factor = 0.05); + + -- or for an existing table: + ALTER TABLE my_table + SET (autovacuum_vacuum_threshold = 0, + autovacuum_vacuum_scale_factor = 0.05); + + * Or rely on the per-table ``autovacuum_vacuum_scale_factor`` (default + 0.2) which triggers on total modifications, including rolled-back ones + (``n_mod_since_analyze`` counts all modifications regardless of outcome). + +Operators deploying UNDO-enabled AMs on high-write tables should verify +autovacuum configuration and monitor ``pg_stat_user_tables`` for bloat. + +**Version reconstruction for an in-place-update AM:** For the heap AM, UNDO is +used only for physical rollback (restoring the pre-DML tuple state); heap +snapshot visibility uses the dead-tuple + CLOG mechanism. An in-place-update +AM can resolve visibility with xmin/xmax + CLOG exactly like heap -- that is +NOT UNDO-based MVCC and NOT an sLog before-image MVCC scheme. What differs is +physical: an in-place UPDATE overwrites the page, so the prior tuple version +is not on the page. A snapshot older than an in-place UPDATE reconstructs the +prior version by walking the relation's per-relation UNDO fork (RelUndo, +RELUNDO_FORKNUM), not the sLog. The sLog serves only the abort/self-visibility +window (constant-time abort and hiding an aborting xid's writes); the commit +oracle is CLOG. + +## 16. References + + Antonopoulos et al., "Constant Time Recovery in Azure SQL Database" + Proceedings of the VLDB Endowment, Vol. 12, No. 12, August 2019. + The recovery model implemented here: an aborting transaction is + recorded in a constant-time abort map so its writes are immediately + invisible, and a background worker performs the physical rollback + (Logical Revert). UNDO records are embedded in the WAL stream + rather than in a separate version store, sharing WAL retention + constraints while eliminating a separate storage tier. + +## 17. Known Limitations + +The current implementation has the following known limitations: + +### WAL Retention for UNDO +- UNDO batches in WAL are retained until undo_discard_horizon advances +- Horizon is gated by oldest in-flight transaction (never discards unresolved) +- Logical Revert Worker's pending ATM entries also pin WAL retention +- WAL segment recycling respects undo_discard_horizon automatically + +### WAL Level +UNDO-enabled tables require ``wal_level = replica`` or higher for +streaming standbys to receive CLR records written during UNDO application. +At ``wal_level = minimal``, single-row DML UNDO records are still written +and crash recovery works correctly, but standbys will not receive them. + +### Logical Decoding +``XLOG_UNDO_BATCH`` records use a custom resource manager. The UNDO RM's +``rm_decode`` callback is a no-op: logical decoding filters these records and +they do not appear as change events. Logical replication of UNDO-enabled +tables works correctly -- the logical decoder sees heap INSERT/UPDATE/DELETE +changes normally. + +### AM Compatibility +UNDO is always-on infrastructure. Only in-place-update table AMs benefit +from it; they opt in via the am_supports_undo callback. The default heap AM +sets am_supports_undo = false and does not use UNDO -- heap operates without +UNDO overhead and reclaims dead tuples through ordinary MVCC/VACUUM. + +### TOAST +TOAST tables are ordinary heap relations and are not UNDO-protected. An +UNDO-based table AM that stores wide values does so in a standard heap TOAST +table; that table writes normal heap WAL and is reclaimed by ordinary +MVCC/VACUUM. Rolling back an UPDATE that touched a wide value restores the +base tuple via UNDO, and the base tuple again references the OLD TOAST +pointer; the NEW TOAST chunks written during the update become dead heap +tuples reclaimed by VACUUM (see Section 15). UNDO is not applied to the +TOAST table itself. + +### Delta Compression +- UPDATE records store full old tuple, not delta +- Could be optimized similar to xl_heap_update PREFIX_FROM_OLD +- Impact: Higher UNDO write amplification on partial updates +- Mitigation: Use HOT updates when possible + +### Standby-Side Before-Image Regeneration (designed, rejected) +A proposed optimization would have standbys regenerate each in-place-update +before-image locally rather than receiving it in the per-relation UNDO +fork's WAL payload, trimming WAL volume on write-heavy workloads. It is +not implemented, and was rejected for two reasons: + +- It is not viable on the current write path. An in-place + UPDATE/DELETE overwrites the tuple and emits its data-change WAL record + BEFORE the relundo before-image record (RelUndoFinish). + Standby redo is strictly LSN-ordered, so by the time it replays the + before-image record the old tuple is already gone from the page -- there + is nothing to regenerate from. Enabling regeneration would require + reordering the two records across the write path, reintroducing the + cross-record ordering coupling the per-relation fork was built to avoid. +- The risk/reward is upside-down. The saving is one old-tuple image per + in-place UPDATE/DELETE (already delta-compressed on the CAS path), while + a regeneration defect would serve incorrect reads on a standby or break + rollback after promotion. Correctness on standbys is not worth trading + for a bounded WAL-volume saving. + +Revisit only if WAL volume from before-images becomes a measured +bottleneck AND the write path is reordered to log the before-image first, +behind a default-off switch that retains the payload path as fallback. + +### ProcArray Integration +- GetOldestActiveTransactionId() simplified for initial implementation +- Proper ProcArray scan for oldest XID needed for production +- Impact: Less aggressive UNDO discard than optimal + +### UNDO-Based MVCC (In Progress for in-place-update AMs) +- Heap AM does not use UNDO at all. +- In-place-update AM: may require UNDO-based read visibility (in-place + updates destroy the prior version). The sLog already captures + before-images at DML time; making these available in shared memory for + concurrent readers is the remaining work. + +### Platform Support +- Tested on: Linux (primary), FreeBSD, Windows, macOS +- Full platform matrix testing pending +- Extended file attributes (xattr) support varies by platform + +### Parallel UNDO Apply +- Transaction rollback runs sequentially in a single backend process +- Large aborts can be slow +- Future work: Parallel UNDO application for faster rollback + +## 18. Upgrade Guide + +### Prerequisites +- PostgreSQL 17+ (uses current rmgrlist.h structure) +- Sufficient WAL disk space (UNDO batches share WAL retention) + +### Enabling UNDO + +UNDO is **disabled by default** and must be enabled per-relation: + + -- Create a table using an AM that supports UNDO + CREATE TABLE important_data (id int, data text) USING ; + +### Monitoring UNDO + +UNDO batches share the WAL stream. Monitor WAL retention: + + SELECT slot_name, restart_lsn, confirmed_flush_lsn + FROM pg_replication_slots; + + -- Check UNDO log state: + SELECT * FROM pg_stat_get_undo_logs(); + +### Rollback Plan + +If issues arise: + +1. UNDO is integral to AMs that use it -- it cannot be + disabled per-table. To stop UNDO activity, convert the table to + a different AM (e.g., heap). + +2. Existing UNDO batches in WAL are retained until retention expires. + +3. Stop UNDO worker if needed: + SELECT pg_terminate_backend(pid) + FROM pg_stat_activity + WHERE backend_type = 'undo worker'; + +### Performance Tuning + +Recommended initial settings: + + # UNDO worker wakes every second + undo_worker_naptime = 1000 + + # Retain UNDO for 1 minute (adjust based on workload) + undo_retention_time = 60000 + + # Up to MAX_UNDO_LOGS (100) concurrent UNDO logs supported + + # WAL retention handles UNDO storage automatically + # Ensure max_wal_size accommodates UNDO batch retention + max_wal_size = 4GB + +Monitor and adjust based on: +- Long-running transaction frequency +- Update-heavy workload patterns +- Disk space availability + +### Bulk UNDO Hints (Implemented) + +The `begin_bulk_insert` table AM callback enables batched UNDO recording +for large DML operations (INSERT, UPDATE, DELETE with >1000 estimated rows). +Instead of per-row UndoLogAllocate + WAL insert + UndoLogWrite, records are +accumulated in a persistent UndoRecordSet and flushed in batches: + +- Flush threshold: 256KB or 1000 records (whichever comes first) +- Activated via: table_begin_bulk_insert() from ExecInitModifyTable +- Deactivated via: table_finish_bulk_insert() from ExecEndModifyTable +- Heap AM callbacks: heapam_begin_bulk_insert / heapam_finish_bulk_insert + +## Lock Ordering + +All UNDO LWLocks must be acquired in the order listed below to +prevent deadlocks. An agent holding lock N must never acquire lock M +where M < N. Standard PostgreSQL buffer-content locks and heavyweight +locks sit outside this hierarchy ā€” they must be acquired before any +UNDO LWLock. A consumer AM's own locks (for example an optional +per-tuple tracking structure, or an AM-private dirty-page map) extend this +hierarchy at higher-numbered levels and are documented with that consumer, +not here. + + Level Lock Holder(s) + ----- ---- --------- + 1 Buffer content locks All backends (standard PG) + 2 LWTRANCHE_UNDO_LOG Per-log lock (undolog.c:97) + (per-log instance) Protects log metadata and append + 3 LWTRANCHE_UNDO_LOG Allocation lock (undolog.c:105) + (allocation instance) Serializes log space allocation + 4 LWTRANCHE_UNDO_LOG Flush lock (undo_flush.c:61) + (flush instance) Protects batch flush state + 5 LWTRANCHE_UNDO_WORKER Revert worker state (logical_revert_worker.c:109) + Protects revert queue and worker state + +Rules: +- Never acquire a buffer content lock while holding any UNDO LWLock. +- The revert worker acquires locks 2ā†’5 in sequence during UNDO apply; + backends acquiring lock 5 (to enqueue work) must not already hold 2-4. diff --git a/src/backend/access/undo/atm.c b/src/backend/access/undo/atm.c new file mode 100644 index 0000000000000..85ec2e9152124 --- /dev/null +++ b/src/backend/access/undo/atm.c @@ -0,0 +1,546 @@ +/*------------------------------------------------------------------------- + * + * atm.c + * Aborted Transaction Map for CTR (Constant-Time Recovery) + * + * The ATM is a shared-memory data structure mapping TransactionId to UNDO + * chain metadata for aborted transactions. It enables: + * + * 1. Background Logical Revert: the Logical Revert worker scans the + * sLog for entries where revert_complete == false and applies their + * UNDO chains asynchronously. + * + * 2. Instant abort: at transaction abort time, the backend writes an + * ATM entry (sLog + WAL) instead of performing synchronous + * rollback, making ROLLBACK O(1). + * + * Implementation: All ATM functions are thin wrappers around the sLog + * (Secondary Log) hash tables defined in access/slog.h. The sLog + * provides O(1) lookups, replacing the old fixed-size linear array. + * + * WAL: ATMAddAborted() emits XLOG_ATM_ABORT; ATMForget() emits + * XLOG_ATM_FORGET. During recovery, atm_redo() replays these without + * re-emitting WAL. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/access/undo/atm.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include + +#include "access/atm.h" +#include "access/slog.h" +#include "access/xlog.h" +#include "access/xloginsert.h" +#include "access/xlogreader.h" +#include "access/xlogutils.h" +#include "common/file_utils.h" +#include "pgstat.h" +#include "port/pg_crc32c.h" +#include "storage/fd.h" +#include "storage/lwlock.h" +#include "storage/shmem.h" +#include "utils/wait_event.h" + +/* + * On-disk ATM state file. + * + * The Aborted Transaction Map is a shared-memory-only structure otherwise + * reconstructed only by replaying XLOG_ATM_ABORT / XLOG_ATM_FORGET during the + * redo pass. Because a checkpoint may advance the redo pointer PAST an + * un-forgotten XLOG_ATM_ABORT, redo alone can miss aborts and silently lose a + * guaranteed rollback. To close that window we persist the map at each + * checkpoint (CheckPointATM) and reload it at startup before redo + * (ATMReloadFromCheckpoint), exactly mirroring how CheckPointTwoPhase / + * restoreTwoPhaseData keep prepared-xact state that predates the redo point. + * + * The file is a single flat file in the data directory: a header, then a + * packed array of records, then a CRC over header+records. It is written to + * a temporary name and durably renamed into place, so a torn write never + * corrupts a previously good file. + */ +#define ATM_STATE_FILE "pg_undo_atm" +#define ATM_STATE_TMP_FILE "pg_undo_atm.tmp" +#define ATM_STATE_MAGIC 0x41544D31 /* "ATM1" */ + +typedef struct AtmStateHeader +{ + uint32 magic; + uint32 count; /* number of AtmStateRecord that follow */ +} AtmStateHeader; + +typedef struct AtmStateRecord +{ + TransactionId xid; + Oid reloid; + Oid dboid; + XLogRecPtr last_batch_lsn; + bool revert_complete; +} AtmStateRecord; + +/* Internal helpers that skip WAL emission (used during redo) */ +static bool ATMAddAbortedInternal(TransactionId xid, Oid dboid, Oid reloid, + XLogRecPtr last_batch_lsn); +static void ATMForgetInternal(TransactionId xid); + +/* + * ATMShmemSize + * Calculate shared memory space needed for the ATM. + * + * The ATM is now backed by sLog, which manages its own shared memory. + * ATM itself needs no additional shared memory. + */ +Size +ATMShmemSize(void) +{ + return 0; +} + +/* + * ATMShmemInit + * Initialize ATM shared memory (no-op, sLog handles it). + */ +void +ATMShmemInit(void) +{ + /* sLog initialization is done separately via SLogShmemInit() */ +} + +/* + * ATMGetLastBatchLSN + * Retrieve the WAL LSN of the last UNDO batch for an aborted transaction. + * + * Returns true if found, storing the LSN in *lsn_out. + */ +bool +ATMGetLastBatchLSN(TransactionId xid, XLogRecPtr *lsn_out) +{ + return SLogTxnLookupByXid(xid, lsn_out); +} + +/* + * ATMAddAbortedInternal + * Add an entry to the ATM without emitting WAL. + * + * Used during both normal operation (after WAL has been written by the + * caller) and during redo replay. + * + * Returns false if the sLog is full. + */ +static bool +ATMAddAbortedInternal(TransactionId xid, Oid dboid, Oid reloid, + XLogRecPtr last_batch_lsn) +{ + return SLogTxnInsert(xid, reloid, dboid, last_batch_lsn); +} + +/* + * ATMAddAborted + * Record an aborted transaction in the ATM with WAL logging. + * + * Called from the abort path. Returns false if the sLog is full, + * signaling the caller to fall back to synchronous rollback. + */ +bool +ATMAddAborted(TransactionId xid, Oid dboid, XLogRecPtr last_batch_lsn) +{ + xl_atm_abort xlrec; + + /* Write WAL first */ + xlrec.xid = xid; + xlrec.last_batch_lsn = last_batch_lsn; + xlrec.dboid = dboid; + xlrec.reloid = InvalidOid; + + XLogBeginInsert(); + XLogRegisterData((char *) &xlrec, SizeOfXlAtmAbort); + XLogInsert(RM_ATM_ID, XLOG_ATM_ABORT); + + /* Now update shared memory */ + return ATMAddAbortedInternal(xid, dboid, InvalidOid, last_batch_lsn); +} + +/* + * ATMForgetInternal + * Remove ATM entries for a transaction without emitting WAL. + */ +static void +ATMForgetInternal(TransactionId xid) +{ + SLogTxnRemoveByXid(xid); +} + +/* + * ATMForget + * Remove ATM entries after Logical Revert has completed. + * + * Emits a WAL record so that the removal survives recovery. + */ +void +ATMForget(TransactionId xid) +{ + xl_atm_forget xlrec; + + /* Write WAL first */ + xlrec.xid = xid; + + XLogBeginInsert(); + XLogRegisterData((char *) &xlrec, SizeOfXlAtmForget); + XLogInsert(RM_ATM_ID, XLOG_ATM_FORGET); + + /* Now update shared memory */ + ATMForgetInternal(xid); +} + +/* + * ATMMarkReverted + * Mark an ATM entry's revert as complete. + * + * The entry is kept in the ATM (for visibility checks) until ATMForget() + * is called after the Logical Revert worker confirms all effects are gone. + */ +void +ATMMarkReverted(TransactionId xid) +{ + SLogTxnMarkReverted(xid); +} + +/* + * ATMGetNextUnreverted + * Find the next ATM entry that hasn't been reverted yet. + * + * Used by the Logical Revert background worker to find work. + * + * Returns true if an unreverted entry was found, filling in the output + * parameters. + */ +bool +ATMGetNextUnreverted(TransactionId *xid_out, Oid *dboid_out, + XLogRecPtr *lsn_out) +{ + return SLogTxnGetNextUnreverted(xid_out, dboid_out, lsn_out); +} + +/* + * ATMCollectUnrevertedDatabases + * Collect the distinct database OIDs that have unreverted ATM entries. + * + * Returns the count; fills dboids[] up to max_dboids. Used by the logical + * revert launcher to spawn workers only for databases that have + * aborted-transaction UNDO to apply. + */ +int +ATMCollectUnrevertedDatabases(Oid *dboids, int max_dboids) +{ + return SLogTxnCollectUnrevertedDatabases(dboids, max_dboids); +} + +/* + * ATMGetOldestUnrevertedLSN + * Return the oldest last_batch_lsn across all unreverted ATM entries. + * + * Used by the WAL retention logic to prevent recycling WAL segments that + * still contain UNDO batches needed by the logical revert worker. + * Returns InvalidXLogRecPtr if no unreverted entries exist. + */ +XLogRecPtr +ATMGetOldestUnrevertedLSN(void) +{ + return SLogTxnGetOldestUnrevertedLSN(); +} + +/* + * ATMRecoveryFinalize + * Called at the end of recovery to log the ATM state. + * + * After WAL redo has reconstructed the ATM via sLog, this logs the number + * of unreverted entries so the DBA can see how much Logical Revert work + * remains, and the oldest unreverted last_batch_lsn -- the LSN that pins + * UNDO WAL against recycling (ATMGetOldestUnrevertedLSN -> + * UndoGetOldestBatchLSN -> KeepLogSeg) until the logical revert worker + * forgets the entry. Emitting the LSN here, before any worker runs, gives + * an observable proof that the retention floor survived the crash. + */ +void +ATMRecoveryFinalize(void) +{ + int total = 0; + int unreverted = 0; + + SLogRecoveryFinalize(&total, &unreverted); + + if (total > 0) + elog(LOG, "ATM recovery complete: %d entries, %d unreverted, " + "oldest unreverted LSN %X/%X", + total, unreverted, + LSN_FORMAT_ARGS(ATMGetOldestUnrevertedLSN())); +} + +/* + * atm_redo + * WAL redo handler for ATM resource manager. + */ +void +atm_redo(XLogReaderState *record) +{ + uint8 info = XLogRecGetInfo(record) & ~XLR_INFO_MASK; + + switch (info) + { + case XLOG_ATM_ABORT: + { + xl_atm_abort *xlrec = + (xl_atm_abort *) XLogRecGetData(record); + + ATMAddAbortedInternal(xlrec->xid, xlrec->dboid, + xlrec->reloid, xlrec->last_batch_lsn); + } + break; + + case XLOG_ATM_FORGET: + { + xl_atm_forget *xlrec = + (xl_atm_forget *) XLogRecGetData(record); + + ATMForgetInternal(xlrec->xid); + } + break; + + default: + elog(PANIC, "atm_redo: unknown op code %u", info); + break; + } +} + +/* + * CheckPointATM + * Durably persist the Aborted Transaction Map at a checkpoint. + * + * Writes every current ATM entry (xid, reloid, dboid, last_batch_lsn, + * revert_complete) to a single flat file, via a temporary file that is + * durably renamed into place, so the map survives a crash even when the + * checkpoint's redo pointer advances past the entries' XLOG_ATM_ABORT + * records. Called from CheckPointGuts(), outside any critical section + * (palloc and file I/O are therefore safe), alongside CheckPointTwoPhase. + * + * We snapshot ALL entries, not only those with an XLOG_ATM_ABORT preceding + * the redo point: an entry whose abort record follows the redo point will be + * re-added by redo, and ATMAddAbortedInternal is idempotent on (xid, reloid) + * (SLogTxnInsert leaves an existing entry untouched), so double-add across + * the checkpoint boundary converges to a single entry. Persisting the whole + * map is simpler than partitioning it by redo point and is cheap because the + * ATM only holds un-forgotten aborts. + */ +void +CheckPointATM(void) +{ + SLogTxnEntry *entries; + int count; + AtmStateHeader hdr; + pg_crc32c crc; + int fd; + int i; + + count = SLogTxnSnapshotForCheckpoint(&entries); + + /* + * Always (re)write the file, even when empty, so a stale file from a + * checkpoint that had entries is replaced by an authoritative empty one. + * An empty file (count == 0) records "no un-forgotten aborts as of this + * redo point". + */ + hdr.magic = ATM_STATE_MAGIC; + hdr.count = (uint32) count; + + INIT_CRC32C(crc); + COMP_CRC32C(crc, &hdr, sizeof(hdr)); + + fd = OpenTransientFile(ATM_STATE_TMP_FILE, + O_CREAT | O_TRUNC | O_WRONLY | PG_BINARY); + if (fd < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not create file \"%s\": %m", + ATM_STATE_TMP_FILE))); + + pgstat_report_wait_start(WAIT_EVENT_TWOPHASE_FILE_WRITE); + errno = 0; + if (write(fd, &hdr, sizeof(hdr)) != sizeof(hdr)) + { + if (errno == 0) + errno = ENOSPC; + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not write file \"%s\": %m", + ATM_STATE_TMP_FILE))); + } + + for (i = 0; i < count; i++) + { + AtmStateRecord rec; + + rec.xid = entries[i].xid; + rec.reloid = entries[i].reloid; + rec.dboid = entries[i].dboid; + rec.last_batch_lsn = entries[i].last_batch_lsn; + rec.revert_complete = entries[i].revert_complete; + + COMP_CRC32C(crc, &rec, sizeof(rec)); + + errno = 0; + if (write(fd, &rec, sizeof(rec)) != sizeof(rec)) + { + if (errno == 0) + errno = ENOSPC; + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not write file \"%s\": %m", + ATM_STATE_TMP_FILE))); + } + } + + FIN_CRC32C(crc); + errno = 0; + if (write(fd, &crc, sizeof(crc)) != sizeof(crc)) + { + if (errno == 0) + errno = ENOSPC; + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not write file \"%s\": %m", + ATM_STATE_TMP_FILE))); + } + pgstat_report_wait_end(); + + pgstat_report_wait_start(WAIT_EVENT_TWOPHASE_FILE_SYNC); + if (pg_fsync(fd) != 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not fsync file \"%s\": %m", + ATM_STATE_TMP_FILE))); + pgstat_report_wait_end(); + + if (CloseTransientFile(fd) != 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not close file \"%s\": %m", + ATM_STATE_TMP_FILE))); + + /* Atomically replace the live file, fsyncing both file and directory. */ + durable_rename(ATM_STATE_TMP_FILE, ATM_STATE_FILE, ERROR); + + if (entries != NULL) + pfree(entries); +} + +/* + * ATMReloadFromCheckpoint + * Reconstruct the ATM from the checkpoint state file, before redo. + * + * Called from StartupXLOG at the same point as restoreTwoPhaseData(), i.e. + * BEFORE the redo pass. Each persisted entry is re-inserted into the map + * without emitting WAL (ATMAddAbortedInternal), preserving its + * revert_complete flag. Reloading before redo is essential: an + * XLOG_ATM_FORGET replayed after the checkpoint must be able to remove an + * entry that WAS persisted, and an XLOG_ATM_ABORT replayed after the + * checkpoint re-adds idempotently on top of what we loaded. + * + * A missing file (fresh initdb, or a cluster that never checkpointed the + * ATM) is not an error: it means "no persisted entries". A file that fails + * its CRC or magic check is treated as absent with a warning; every entry it + * could have held is still WAL-durable via XLOG_ATM_ABORT, and any whose + * abort record precedes the redo point would then be lost -- but a corrupt + * ATM state file implies a torn write that durable_rename is designed to + * prevent, so this is a belt-and-suspenders path, not an expected one. + */ +void +ATMReloadFromCheckpoint(void) +{ + int fd; + AtmStateHeader hdr; + pg_crc32c crc, + file_crc; + AtmStateRecord *records = NULL; + uint32 i; + ssize_t nread; + + fd = OpenTransientFile(ATM_STATE_FILE, O_RDONLY | PG_BINARY); + if (fd < 0) + { + if (errno == ENOENT) + return; /* no persisted state -- nothing to reload */ + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not open file \"%s\": %m", ATM_STATE_FILE))); + } + + pgstat_report_wait_start(WAIT_EVENT_TWOPHASE_FILE_READ); + nread = read(fd, &hdr, sizeof(hdr)); + if (nread != sizeof(hdr) || hdr.magic != ATM_STATE_MAGIC) + { + pgstat_report_wait_end(); + CloseTransientFile(fd); + ereport(WARNING, + (errmsg("ignoring ATM state file \"%s\" with invalid header", + ATM_STATE_FILE))); + return; + } + + INIT_CRC32C(crc); + COMP_CRC32C(crc, &hdr, sizeof(hdr)); + + if (hdr.count > 0) + { + Size bytes = (Size) hdr.count * sizeof(AtmStateRecord); + + records = (AtmStateRecord *) palloc(bytes); + nread = read(fd, records, bytes); + if (nread != (ssize_t) bytes) + { + pgstat_report_wait_end(); + CloseTransientFile(fd); + pfree(records); + ereport(WARNING, + (errmsg("ignoring truncated ATM state file \"%s\"", + ATM_STATE_FILE))); + return; + } + COMP_CRC32C(crc, records, bytes); + } + + nread = read(fd, &file_crc, sizeof(file_crc)); + pgstat_report_wait_end(); + CloseTransientFile(fd); + + FIN_CRC32C(crc); + if (nread != sizeof(file_crc) || !EQ_CRC32C(crc, file_crc)) + { + if (records != NULL) + pfree(records); + ereport(WARNING, + (errmsg("ignoring ATM state file \"%s\" with bad checksum", + ATM_STATE_FILE))); + return; + } + + for (i = 0; i < hdr.count; i++) + { + ATMAddAbortedInternal(records[i].xid, records[i].dboid, + records[i].reloid, records[i].last_batch_lsn); + if (records[i].revert_complete) + ATMMarkReverted(records[i].xid); + } + + if (records != NULL) + pfree(records); + + if (hdr.count > 0) + elog(LOG, "ATM reloaded %u entries from checkpoint state file", + hdr.count); +} diff --git a/src/backend/access/undo/logical_revert_worker.c b/src/backend/access/undo/logical_revert_worker.c new file mode 100644 index 0000000000000..275a3a2c89202 --- /dev/null +++ b/src/backend/access/undo/logical_revert_worker.c @@ -0,0 +1,628 @@ +/*------------------------------------------------------------------------- + * + * logical_revert_worker.c + * Background worker for timer-driven Logical Revert via ATM scan + * + * This worker periodically scans the ATM (Aborted Transaction Map) for + * entries whose WAL-based UNDO chains have not yet been confirmed as applied. + * For each unreverted entry whose database matches the worker's connected + * database, the worker: + * + * 1. Applies the WAL-based UNDO chain via ApplyUndoChainFromWAL() + * (idempotent: CLR records prevent double-application) + * 2. Marks the ATM entry as reverted via ATMMarkReverted() + * 3. Emits XLOG_ATM_FORGET and removes the entry via ATMForget() + * + * Unlike event-driven UNDO worker variants (which process a shared memory work + * queue), this worker is timer-driven: it sleeps for logical_revert_naptime + * milliseconds between scan cycles. + * + * Shared memory: a single LogicalRevertState struct holds the LWLock + * protecting the running flag and a counter for assigning worker IDs. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/access/undo/logical_revert_worker.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include + +#include "access/atm.h" +#include "access/heapam.h" +#include "access/logical_revert_worker.h" +#include "access/table.h" +#include "access/undo_xlog.h" +#include "access/undorecord.h" +#include "access/xact.h" +#include "access/xlog.h" +#include "access/xlogdefs.h" +#include "catalog/pg_database.h" +#include "miscadmin.h" +#include "nodes/pg_list.h" +#include "pgstat.h" +#include "postmaster/bgworker.h" +#include "storage/ipc.h" +#include "storage/latch.h" +#include "storage/lwlock.h" +#include "storage/shmem.h" +#include "tcop/tcopprot.h" +#include "utils/guc.h" +#include "utils/injection_point.h" +#include "utils/memutils.h" + +/* GUC parameter: sleep time between ATM scans in milliseconds */ +int logical_revert_naptime = 1000; + +/* GUC parameter: max number of logical revert workers (0 = disabled) */ +int max_logical_revert_workers = 2; + +/* + * Upper bound on distinct databases the launcher tracks per scan when + * collecting those with unreverted ATM entries. Far more than the number of + * databases that realistically have in-flight aborted-transaction UNDO at + * once; extras (if ever) are picked up on the next scan. + */ +#define MAX_LOGICAL_REVERT_DATABASES 128 + +/* + * Shared memory state for the Logical Revert worker. + * + * Minimal: just a lock and a worker-id counter. The ATM itself is the + * "work queue" -- the worker reads it directly via ATMGetNextUnreverted(). + */ +typedef struct LogicalRevertState +{ + LWLock lock; + int next_worker_id; +} LogicalRevertState; + +static LogicalRevertState *RevertState = NULL; + +/* Flags set by signal handlers */ +static volatile sig_atomic_t got_SIGHUP = false; +static volatile sig_atomic_t got_SIGTERM = false; + +/* Signal handlers */ +static void logical_revert_sighup(SIGNAL_ARGS); +static void logical_revert_sigterm(SIGNAL_ARGS); +static void process_revert_entry(TransactionId xid, XLogRecPtr last_batch_lsn); +static List *get_revertable_database_list(MemoryContext resultcxt); + +/* + * LogicalRevertShmemSize + * Calculate shared memory space needed. + */ +Size +LogicalRevertShmemSize(void) +{ + return sizeof(LogicalRevertState); +} + +/* + * LogicalRevertShmemInit + * Allocate and initialize shared memory. + */ +void +LogicalRevertShmemInit(void) +{ + bool found; + + RevertState = (LogicalRevertState *) + ShmemInitStruct("Logical Revert Worker State", + sizeof(LogicalRevertState), + &found); + + if (!found) + { + LWLockInitialize(&RevertState->lock, LWTRANCHE_UNDO_WORKER); + RevertState->next_worker_id = 1; + } +} + +/* + * logical_revert_sighup + * SIGHUP signal handler -- reload configuration. + */ +static void +logical_revert_sighup(SIGNAL_ARGS) +{ + int save_errno = errno; + + got_SIGHUP = true; + SetLatch(MyLatch); + + errno = save_errno; +} + +/* + * logical_revert_sigterm + * SIGTERM signal handler -- request shutdown. + */ +static void +logical_revert_sigterm(SIGNAL_ARGS) +{ + int save_errno = errno; + + got_SIGTERM = true; + SetLatch(MyLatch); + + errno = save_errno; +} + +/* + * process_revert_entry + * Apply the WAL-based UNDO chain for a single ATM entry. + * + * Walks the UNDO chain from last_batch_lsn backward, applying each record. + * CLR records are written during application so that crash recovery is + * idempotent. Returns silently if last_batch_lsn is invalid (nothing to do). + */ +static void +process_revert_entry(TransactionId xid, XLogRecPtr last_batch_lsn) +{ + if (!XLogRecPtrIsValid(last_batch_lsn)) + return; /* nothing to apply */ + + ereport(DEBUG1, + (errmsg("logical revert: applying UNDO chain for xid %u " + "from LSN %X/%X", + xid, LSN_FORMAT_ARGS(last_batch_lsn)))); + + ApplyUndoChainFromWAL(last_batch_lsn); +} + +/* + * LogicalRevertWorkerMain + * Main entry point for the Logical Revert background worker. + * + * The worker connects to a specific database, then loops: scan the ATM + * for unreverted entries matching this database, apply them, mark done, + * forget. Sleep when idle. + */ +void +LogicalRevertWorkerMain(Datum main_arg) +{ + Oid dboid = DatumGetObjectId(main_arg); + int worker_id; + + /* Establish signal handlers */ + pqsignal(SIGHUP, logical_revert_sighup); + pqsignal(SIGTERM, logical_revert_sigterm); + + BackgroundWorkerUnblockSignals(); + + /* Connect to the target database */ + BackgroundWorkerInitializeConnectionByOid(dboid, InvalidOid, 0); + + /* Assign a worker ID */ + LWLockAcquire(&RevertState->lock, LW_EXCLUSIVE); + worker_id = RevertState->next_worker_id++; + LWLockRelease(&RevertState->lock); + + elog(LOG, "logical revert worker %d started for database %u", + worker_id, dboid); + + while (!got_SIGTERM) + { + TransactionId xid; + Oid entry_dboid; + XLogRecPtr last_batch_lsn; + int rc; + + /* + * Service interrupts and ProcSignalBarriers at the top of every + * iteration. Without this, a worker that is busy reverting back- + * to-back ATM entries via the `continue` path below never reaches the + * WaitLatch sleep, and ALTER DATABASE ... SET TABLESPACE / CREATE + * DATABASE ... STRATEGY=FILE_COPY will hang waiting for this backend + * to accept PROCSIGNAL_BARRIER_SMGRRELEASE. + */ + CHECK_FOR_INTERRUPTS(); + + /* Reload configuration on SIGHUP */ + if (got_SIGHUP) + { + got_SIGHUP = false; + ProcessConfigFile(PGC_SIGHUP); + } + + /* Scan ATM for the next unreverted entry */ + if (ATMGetNextUnreverted(&xid, &entry_dboid, &last_batch_lsn)) + { + /* + * ATMGetNextUnreverted returns entries for any database. Skip + * entries that belong to a different database. + */ + if (entry_dboid != MyDatabaseId) + goto sleep; + + StartTransactionCommand(); + + /* + * Test hook: let a recovery test freeze the worker here, after + * the ATM entry has been (re)discovered but before it is reverted + * and forgotten, so the test can observe the reconstructed ATM + * state. + */ + INJECTION_POINT("logical-revert-before-process", NULL); + + PG_TRY(); + { + process_revert_entry(xid, last_batch_lsn); + + /* + * Mark the ATM entry as reverted, then emit XLOG_ATM_FORGET + * and remove it from the ATM entirely. + */ + ATMMarkReverted(xid); + ATMForget(xid); + } + PG_CATCH(); + { + EmitErrorReport(); + FlushErrorState(); + + /* + * Reset the cached WAL reader -- it may hold stale segment + * state (invalid FD, partial read buffer) after the error. + * Without this, the next UndoReadBatchFromWAL call would + * reuse the corrupted reader and SIGSEGV. + */ + UndoResetBatchReader(); + + /* + * If the WAL segment holding this abort's UNDO batch is + * behind the redo pointer, it looks recycled -- which must + * never happen for an un-reverted abort. The retention chain + * pins that WAL against recycling for exactly as long as the + * ATM entry survives: + * + * ATMGetOldestUnrevertedLSN (atm.c) -> UndoGetOldestBatchLSN + * (undolog.c) -> KeepLogSeg (xlog.c) + * + * so KeepLogSeg cannot advance past last_batch_lsn until + * ATMForget removes the entry. Reaching this branch means + * that invariant was violated and UNDO WAL for an un-reverted + * abort is genuinely gone: the cluster's guaranteed-rollback + * claim is already broken. Silently marking the entry + * reverted would fake a rollback that never happened and + * leave stale tuple state behind, so fail loudly instead -- + * the integrity guarantee is cluster-wide, hence PANIC rather + * than continuing. + * + * The other-error branch is a transient failure (the WAL is + * still retained); log it and let the next iteration retry. + */ + { + XLogRecPtr redo = GetRedoRecPtr(); + + if (XLogRecPtrIsValid(last_batch_lsn) && + last_batch_lsn < redo) + { + elog(PANIC, "logical revert worker: UNDO WAL for " + "un-reverted xid %u at %X/%X was recycled " + "(redo at %X/%X); retention invariant violated " + "(ATMGetOldestUnrevertedLSN -> " + "UndoGetOldestBatchLSN -> KeepLogSeg must pin " + "this WAL until ATMForget) -- guaranteed " + "rollback can no longer be honored", + xid, LSN_FORMAT_ARGS(last_batch_lsn), + LSN_FORMAT_ARGS(redo)); + } + else + { + elog(LOG, "logical revert worker: failed to revert " + "xid %u, will retry", xid); + } + } + } + PG_END_TRY(); + + CommitTransactionCommand(); + + /* Immediately look for more work instead of sleeping */ + continue; + } + +sleep: + /* No work available, wait */ + rc = WaitLatch(MyLatch, + WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, + logical_revert_naptime, + PG_WAIT_EXTENSION); + + ResetLatch(MyLatch); + + if (rc & WL_POSTMASTER_DEATH) + proc_exit(1); + } + + elog(LOG, "logical revert worker %d shutting down", worker_id); + proc_exit(0); +} + +/* + * StartLogicalRevertWorker + * Launch a logical revert worker for the specified database. + * + * The worker uses a modest `bgw_restart_time` so that if it exits due + * to a transient error the postmaster auto-restarts it. That removes + * the need for the launcher to re-spawn workers on every scan. + */ +void +StartLogicalRevertWorker(Oid dboid) +{ + BackgroundWorker worker; + BackgroundWorkerHandle *handle; + + memset(&worker, 0, sizeof(BackgroundWorker)); + worker.bgw_flags = BGWORKER_SHMEM_ACCESS | + BGWORKER_BACKEND_DATABASE_CONNECTION | + BGWORKER_INTERRUPTIBLE; + worker.bgw_start_time = BgWorkerStart_RecoveryFinished; + worker.bgw_restart_time = 60; + sprintf(worker.bgw_library_name, "postgres"); + sprintf(worker.bgw_function_name, "LogicalRevertWorkerMain"); + snprintf(worker.bgw_name, BGW_MAXLEN, + "logical revert worker for database %u", dboid); + snprintf(worker.bgw_type, BGW_MAXLEN, "logical revert worker"); + worker.bgw_main_arg = ObjectIdGetDatum(dboid); + worker.bgw_notify_pid = MyProcPid; + + if (!RegisterDynamicBackgroundWorker(&worker, &handle)) + { + ereport(WARNING, + (errmsg("could not register logical revert worker for database %u", + dboid))); + } + else + { + elog(DEBUG1, "started logical revert worker for database %u", dboid); + } +} + +/* + * get_revertable_database_list + * Return a palloc'd List of OIDs of databases that should get a worker. + * + * Returns only databases that have at least one unreverted ATM entry: those + * are the only databases with aborted-transaction UNDO for a worker to apply. + * A connectable, non-template database with no unreverted entries gets no + * worker, so an idle cluster (the common case) launches nothing and no + * revert worker connects to -- or reports pgstat activity for -- a database + * that has no work. The returned list is allocated in resultcxt (a + * long-lived context) so it survives the transaction the scan runs in. + */ +static List * +get_revertable_database_list(MemoryContext resultcxt) +{ + List *dblist = NIL; + Relation pg_database; + TableScanDesc scan; + HeapTuple tup; + Oid work_dboids[MAX_LOGICAL_REVERT_DATABASES]; + int nwork; + int i; + + /* + * Fast path: if no database has an unreverted ATM entry, there is nothing + * to revert anywhere, so spawn no workers and open no database. + */ + nwork = ATMCollectUnrevertedDatabases(work_dboids, + MAX_LOGICAL_REVERT_DATABASES); + if (nwork == 0) + return NIL; + + StartTransactionCommand(); + + pg_database = table_open(DatabaseRelationId, AccessShareLock); + scan = table_beginscan_catalog(pg_database, 0, NULL); + while ((tup = heap_getnext(scan, ForwardScanDirection)) != NULL) + { + Form_pg_database db = (Form_pg_database) GETSTRUCT(tup); + MemoryContext oldcxt; + bool has_work = false; + + if (!db->datallowconn) + continue; + + /* + * Skip template databases. template1 / template0 are not expected to + * accumulate ATM entries, and holding a revert-worker connection to + * template1 would block subsequent CREATE DATABASE commands that use + * it as the source template. + */ + if (strncmp(NameStr(db->datname), "template", 8) == 0) + continue; + + /* Only databases with unreverted ATM entries need a worker. */ + for (i = 0; i < nwork; i++) + { + if (work_dboids[i] == db->oid) + { + has_work = true; + break; + } + } + if (!has_work) + continue; + + /* Append the OID in the caller's long-lived context. */ + oldcxt = MemoryContextSwitchTo(resultcxt); + dblist = lappend_oid(dblist, db->oid); + MemoryContextSwitchTo(oldcxt); + } + table_endscan(scan); + table_close(pg_database, AccessShareLock); + + CommitTransactionCommand(); + + return dblist; +} + +/* + * LogicalRevertLauncherMain + * Launcher background worker. + * + * The launcher re-scans pg_database on every wake-up and spawns a + * LogicalRevertWorker for any connectable, non-template database that does + * not already have one. This way databases created after server start get + * a worker without requiring a restart. Databases that disappear are + * dropped from the tracking set so their OIDs can be re-tracked if reused. + * + * We track the set of databases we have already spawned a worker for so we + * do not re-register on every wake-up: the per-db workers use + * `bgw_restart_time = 60` and the postmaster auto-restarts them if they + * exit, and the bgworker slot pool is small enough that eager re-spawning + * would exhaust it. + */ +void +LogicalRevertLauncherMain(Datum main_arg) +{ + MemoryContext launcher_cxt; + List *known_dbs = NIL; + + pqsignal(SIGHUP, logical_revert_sighup); + pqsignal(SIGTERM, logical_revert_sigterm); + BackgroundWorkerUnblockSignals(); + + /* + * The launcher does not connect to any database itself. It spawns per-db + * workers which do their own connection (via + * BackgroundWorkerInitializeConnectionByOid). For the pg_database scans + * we connect to postgres (not template1, because holding a connection to + * template1 would block CREATE DATABASE). + */ + BackgroundWorkerInitializeConnection("postgres", NULL, 0); + + /* + * Long-lived context for the set of databases we have already started a + * worker for. It must outlive the per-scan transactions, so it cannot + * live in a transaction-scoped context. + */ + launcher_cxt = AllocSetContextCreate(TopMemoryContext, + "Logical Revert Launcher", + ALLOCSET_SMALL_SIZES); + + elog(LOG, "logical revert launcher started"); + + while (!got_SIGTERM) + { + int rc; + List *current_dbs; + List *next_known = NIL; + MemoryContext oldcxt; + ListCell *lc; + + /* Service interrupts and ProcSignalBarriers at every iteration. */ + CHECK_FOR_INTERRUPTS(); + + if (got_SIGHUP) + { + got_SIGHUP = false; + ProcessConfigFile(PGC_SIGHUP); + } + + /* Re-scan pg_database to discover databases created or dropped. */ + current_dbs = get_revertable_database_list(launcher_cxt); + + oldcxt = MemoryContextSwitchTo(launcher_cxt); + foreach(lc, current_dbs) + { + Oid dboid = lfirst_oid(lc); + + /* + * Carry forward databases that already have a live worker. + */ + if (list_member_oid(known_dbs, dboid)) + { + next_known = lappend_oid(next_known, dboid); + continue; + } + + /* + * Start a worker for a newly seen database only while we stay at + * or below max_logical_revert_workers concurrently tracked + * per-database workers. The GUC is a hard ceiling on the number + * of live revert workers so the launcher never exhausts the + * shared background-worker slot pool. On a cluster with more + * connectable databases than workers, the databases past the + * ceiling are serviced once a tracked database is dropped (its + * slot frees up on the next scan); raise the GUC to cover all + * databases concurrently. + */ + if (list_length(next_known) >= max_logical_revert_workers) + continue; + + StartLogicalRevertWorker(dboid); + next_known = lappend_oid(next_known, dboid); + } + MemoryContextSwitchTo(oldcxt); + + /* + * Replace the tracked set with the databases that still exist. Any + * dropped database falls out here; its worker (if any) exits on its + * own when its database disappears, and forgetting the OID lets a + * reused OID be re-tracked on a later scan. + */ + list_free(known_dbs); + known_dbs = next_known; + list_free(current_dbs); + + /* + * Sleep until the next scan. A shorter interval than the original + * one-shot design so newly created databases pick up a revert worker + * promptly; the no-op fast path (every db already tracked) is cheap. + */ + rc = WaitLatch(MyLatch, + WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH, + 60000L, /* 1 minute */ + PG_WAIT_EXTENSION); + + if (rc & WL_POSTMASTER_DEATH) + proc_exit(1); + + ResetLatch(MyLatch); + } + + elog(LOG, "logical revert launcher shutting down"); + proc_exit(0); +} + +/* + * LogicalRevertLauncherRegister + * Register the logical revert launcher as a static background worker. + * + * Called from postmaster.c at startup, alongside ApplyLauncherRegister(). + * UNDO is always-on infrastructure; table AMs opt in via am_supports_undo. + */ +void +LogicalRevertLauncherRegister(void) +{ + BackgroundWorker bgw; + + /* Disabled during binary upgrade or when explicitly turned off. */ + if (IsBinaryUpgrade) + return; + if (max_logical_revert_workers <= 0) + return; + + memset(&bgw, 0, sizeof(bgw)); + bgw.bgw_flags = BGWORKER_SHMEM_ACCESS | + BGWORKER_BACKEND_DATABASE_CONNECTION; + bgw.bgw_start_time = BgWorkerStart_RecoveryFinished; + bgw.bgw_restart_time = 5; + snprintf(bgw.bgw_library_name, MAXPGPATH, "postgres"); + snprintf(bgw.bgw_function_name, BGW_MAXLEN, "LogicalRevertLauncherMain"); + snprintf(bgw.bgw_name, BGW_MAXLEN, "logical revert launcher"); + snprintf(bgw.bgw_type, BGW_MAXLEN, "logical revert launcher"); + bgw.bgw_notify_pid = 0; + bgw.bgw_main_arg = (Datum) 0; + + RegisterBackgroundWorker(&bgw); +} diff --git a/src/backend/access/undo/meson.build b/src/backend/access/undo/meson.build new file mode 100644 index 0000000000000..c78763a296b29 --- /dev/null +++ b/src/backend/access/undo/meson.build @@ -0,0 +1,26 @@ +# Copyright (c) 2022-2026, PostgreSQL Global Development Group + +backend_sources += files( + 'atm.c', + 'logical_revert_worker.c', + 'relundo.c', + 'relundo_apply.c', + 'relundo_discard.c', + 'relundo_page.c', + 'relundo_recovery.c', + 'relundo_worker.c', + 'relundo_xlog.c', + 'slog.c', + 'undo.c', + 'undo_bufmgr.c', + 'undo_xlog.c', + 'undoapply.c', + 'undobuffer.c', + 'undoinsert.c', + 'undolog.c', + 'undorecord.c', + 'undormgr.c', + 'undostats.c', + 'undoworker.c', + 'xactundo.c', +) diff --git a/src/backend/access/undo/relundo.c b/src/backend/access/undo/relundo.c new file mode 100644 index 0000000000000..f5eb3928a9a5c --- /dev/null +++ b/src/backend/access/undo/relundo.c @@ -0,0 +1,1447 @@ +/*------------------------------------------------------------------------- + * + * relundo.c + * Per-relation UNDO core implementation + * + * This file implements the main API for per-relation UNDO logging used by + * table access methods that need MVCC visibility via UNDO chain walking. + * + * The two-phase insert protocol works as follows: + * + * 1. RelUndoReserve() - Finds (or allocates) a page with enough space, + * pins and exclusively locks the buffer, advances pd_lower to reserve + * space, and returns an RelUndoRecPtr encoding the position. + * + * 2. Caller performs the DML operation. + * + * 3a. RelUndoFinish() - Writes the actual UNDO record into the reserved + * space, marks the buffer dirty, and releases it. + * 3b. RelUndoCancel() - Releases the buffer without writing; the reserved + * space becomes a hole (zero-filled). + * + * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/access/undo/relundo.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/relundo.h" +#include "access/relundo_xlog.h" +#include "access/xlog.h" +#include "access/xloginsert.h" +#include "access/xlogutils.h" +#include "catalog/storage.h" +#include "catalog/storage_xlog.h" +#include "common/relpath.h" +#include "miscadmin.h" +#include "port/atomics.h" +#include "storage/bufmgr.h" +#include "storage/bufpage.h" +#include "storage/procarray.h" +#include "storage/procnumber.h" +#include "storage/smgr.h" +#include "utils/timestamp.h" + +/* + * AM-neutral hook function pointers (declared in access/relundo.h). + * + * The owning in-place table AM installs these at subsystem init so the UNDO + * core can clear AM transient tuple flags and reclaim retained before-images + * without compile-time knowledge of any AM's tuple format. They remain NULL + * when no in-place AM is registered. + */ +void (*RelUndoClearTransientFlags_hook) (char *tuple_data) = NULL; +void (*RelUndoAbortCleanup_hook) (TransactionId xid) = NULL; +void (*RelUndoDiscardRetained_hook) (void) = NULL; + +/* + * Per-backend UNDO head page cache. + * + * Every RelUndoReserve() call currently acquires an EXCLUSIVE lock on the + * metapage to find the current head page. For insert-heavy workloads this + * is a severe contention point (100K inserts = 100K exclusive metapage locks). + * + * This hash-table cache remembers the current head page and its free space + * for recently-used relations. When the cached page has enough space, we + * skip the metapage entirely and go directly to the data page with an + * EXCLUSIVE lock. Cache misses and full pages fall back to the metapage path. + * + * The cache is implemented as an open-addressing hash table with linear + * probing and a clock-hand eviction policy. This gives O(1) average-case + * lookup and update, replacing the O(N) linear scan of the previous LRU list. + * + * Cache entries are invalidated when: + * - The cached page turns out to be full (optimistic approach) + * - A new page is allocated (the head page changes) + * - RelUndoInitRelation is called (the fork is recreated) + * + * Size: must be a power of two for the hash masking to work correctly. + * 64 slots covers workloads with up to ~48 concurrently-active relations + * before eviction starts (load factor ~0.75). Increasing to 128 or 256 + * is straightforward if profiling shows benefit. + * + * Probe chain correctness: we use a tombstone OID (RELUNDO_CACHE_TOMBSTONE) + * rather than InvalidOid for invalidated slots. Setting a slot to InvalidOid + * would create a gap that breaks the linear probe chain for entries that were + * inserted past that slot. A tombstone is skipped during lookup but does + * not terminate the probe, so all entries remain reachable. During insert, + * tombstone slots are reused (treated as free), which amortizes tombstone + * accumulation. The table is re-initialized on first use (all slots + * InvalidOid), so no "previous tombstone" confusion can arise at startup. + */ +#define RELUNDO_HEAD_CACHE_SIZE 64 /* must be a power of two */ +#define RELUNDO_HEAD_CACHE_MASK (RELUNDO_HEAD_CACHE_SIZE - 1) + +/* + * Sentinel OID used to mark a cache slot as "tombstone" (deleted but + * still part of a probe chain). Must not be a valid relation OID and + * must differ from InvalidOid (0). OID 1 is reserved by the system + * (pg_type), so it is safe to repurpose as a tombstone here; it will + * never appear as a user-relation OID passed to these cache functions. + */ +#define RELUNDO_CACHE_TOMBSTONE ((Oid) 1) + +typedef struct RelUndoHeadCacheEntry +{ + Oid relid; /* Relation OID; InvalidOid=empty, + * RELUNDO_CACHE_TOMBSTONE=deleted */ + BlockNumber head_blkno; /* Cached head page block number */ + Size free_space; /* Last-known free space on head page */ +} RelUndoHeadCacheEntry; + +static RelUndoHeadCacheEntry relundo_head_cache[RELUNDO_HEAD_CACHE_SIZE]; +static bool relundo_head_cache_init = false; + +/* Clock hand for round-robin eviction (avoids pure linear probing pile-up) */ +static int relundo_cache_evict_hand = 0; + +/* + * Per-backend pending metapage buffer. + * + * When RelUndoReserve() allocates a new UNDO page, the metapage is modified + * (new head_blkno) and must be included in the WAL record written by + * RelUndoFinish(). Previously, RelUndoReserve() released the metapage lock + * and RelUndoFinish() re-acquired it, creating an ABBA deadlock: + * + * Backend A: holds metapage ā†’ wants UNDO data page + * Backend B: holds UNDO data page ā†’ wants metapage + * + * Fix: keep the metapage locked through the Reserveā†’Finish cycle. + * RelUndoReserve() stores the locked metapage buffer here, and + * RelUndoFinish()/RelUndoFinishWithTuple() retrieves it. + * RelUndoCancel() releases it if the operation is aborted. + */ +static Buffer relundo_pending_metabuf = InvalidBuffer; + +/* + * relundo_oid_hash -- fast hash of a relation OID for cache slot selection. + * + * Uses Knuth's multiplicative hash (2654435761 is close to 2^32/phi). + * The result is masked to RELUNDO_HEAD_CACHE_SIZE slots. + */ +static inline int +relundo_oid_hash(Oid relid) +{ + return (int) (((uint32) relid * UINT32_C(2654435761)) >> (32 - 6)); +} + +/* + * relundo_my_slot -- pick this backend's UNDO head slot. + * + * Every committed append hashes to one of RELUNDO_NUM_HEADS independent head + * pages so concurrent writers spread across distinct tail-page content locks + * instead of serializing on one. A backend always resolves to the same slot + * (its ProcNumber modulo the slot count), which keeps the process-local head + * cache coherent: a given backend only ever touches its own slot's head page, + * so the single-entry-per-relation cache never aliases across slots. + * + * MyProcNumber is INVALID_PROC_NUMBER (-1) outside a normal backend (e.g. in + * the startup process during recovery, which never calls RelUndoReserve on the + * write path). Fold that to slot 0 defensively so the modulo is well defined. + */ +static inline int +relundo_my_slot(void) +{ + int pn = (int) MyProcNumber; + + if (pn < 0) + return 0; + return pn % RELUNDO_NUM_HEADS; +} + +/* + * relundo_head_cache_lookup -- find a cache entry for the given relation. + * + * Uses open-addressing with linear probing. Returns the cache entry pointer + * if found, NULL otherwise. Average cost: O(1) under low load. + * + * Tombstone slots (RELUNDO_CACHE_TOMBSTONE) are skipped but do not terminate + * the probe, preserving the correctness of chains established at insert time. + * Only a truly empty slot (InvalidOid) terminates the probe early. + */ +static RelUndoHeadCacheEntry * +relundo_head_cache_lookup(Oid relid) +{ + int start; + int i; + int probes; + + if (!relundo_head_cache_init) + { + int j; + + for (j = 0; j < RELUNDO_HEAD_CACHE_SIZE; j++) + relundo_head_cache[j].relid = InvalidOid; + relundo_head_cache_init = true; + } + + start = relundo_oid_hash(relid); + + /* + * Linear probe up to RELUNDO_HEAD_CACHE_SIZE slots. + * + * - Match found: return the entry. - Tombstone (RELUNDO_CACHE_TOMBSTONE): + * continue probing; the target entry may sit beyond this deleted slot. - + * Empty slot (InvalidOid): the entry is definitely not present (it would + * have been placed before the first empty slot at insert time), so + * terminate early. + */ + for (probes = 0; probes < RELUNDO_HEAD_CACHE_SIZE; probes++) + { + i = (start + probes) & RELUNDO_HEAD_CACHE_MASK; + + if (relundo_head_cache[i].relid == relid) + return &relundo_head_cache[i]; + + if (relundo_head_cache[i].relid == InvalidOid) + return NULL; /* definitely not present */ + + /* RELUNDO_CACHE_TOMBSTONE: keep probing */ + } + + return NULL; +} + +/* + * relundo_head_cache_update -- update or insert a cache entry. + * + * Finds the existing slot for relid (if any) or inserts into the first + * free slot in the probe sequence. If the probe sequence is full, evicts + * via the clock hand to avoid unbounded probing. + */ +static void +relundo_head_cache_update(Oid relid, BlockNumber head_blkno, Size free_space) +{ + int start; + int i; + int probes; + int free_slot = -1; + + if (!relundo_head_cache_init) + { + int j; + + for (j = 0; j < RELUNDO_HEAD_CACHE_SIZE; j++) + relundo_head_cache[j].relid = InvalidOid; + relundo_head_cache_init = true; + } + + start = relundo_oid_hash(relid); + + /* Probe for existing entry or first free/tombstone slot */ + for (probes = 0; probes < RELUNDO_HEAD_CACHE_SIZE; probes++) + { + i = (start + probes) & RELUNDO_HEAD_CACHE_MASK; + + if (relundo_head_cache[i].relid == relid) + { + /* Update in-place */ + relundo_head_cache[i].head_blkno = head_blkno; + relundo_head_cache[i].free_space = free_space; + return; + } + + /* + * A truly empty slot (InvalidOid) terminates the existing probe + * chain: the relid definitely doesn't exist beyond this point. Use + * the earliest tombstone found so far (if any) to compact the table, + * or this slot if no tombstone was seen. + */ + if (relundo_head_cache[i].relid == InvalidOid) + { + if (free_slot < 0) + free_slot = i; + break; + } + + /* + * Tombstone: record as candidate insertion point (reuses the slot to + * amortize tombstone accumulation) but keep probing in case relid + * exists beyond this slot. + */ + if (relundo_head_cache[i].relid == RELUNDO_CACHE_TOMBSTONE && + free_slot < 0) + free_slot = i; /* remember but keep probing for existing */ + } + + if (free_slot >= 0) + { + /* Insert into the free slot found during probing */ + relundo_head_cache[free_slot].relid = relid; + relundo_head_cache[free_slot].head_blkno = head_blkno; + relundo_head_cache[free_slot].free_space = free_space; + return; + } + + /* + * No free slot found in the probe sequence (table heavily loaded). Evict + * using the clock hand: advance to the next "real" entry (not empty, not + * tombstone) and overwrite it. This bounds the cost to O(1) amortized + * and avoids evicting a tombstone (which would leave a gap that looks + * like "nothing past here" during lookup). + */ + for (probes = 0; probes < RELUNDO_HEAD_CACHE_SIZE; probes++) + { + i = relundo_cache_evict_hand & RELUNDO_HEAD_CACHE_MASK; + relundo_cache_evict_hand = (i + 1) & RELUNDO_HEAD_CACHE_MASK; + + if (relundo_head_cache[i].relid != InvalidOid && + relundo_head_cache[i].relid != RELUNDO_CACHE_TOMBSTONE) + { + relundo_head_cache[i].relid = relid; + relundo_head_cache[i].head_blkno = head_blkno; + relundo_head_cache[i].free_space = free_space; + return; + } + } + + /* Should not reach here if init is correct, but handle gracefully */ + relundo_head_cache[start].relid = relid; + relundo_head_cache[start].head_blkno = head_blkno; + relundo_head_cache[start].free_space = free_space; +} + +/* + * relundo_head_cache_invalidate -- remove a cache entry for the given relation. + * + * Marks the slot with a tombstone (RELUNDO_CACHE_TOMBSTONE) rather than + * InvalidOid. This preserves the correctness of probe chains: other entries + * that were inserted past this slot via linear probing remain reachable. + * Tombstone slots are reused by relundo_head_cache_update, so they do not + * accumulate indefinitely. + */ +static void +relundo_head_cache_invalidate(Oid relid) +{ + int start; + int i; + int probes; + + if (!relundo_head_cache_init) + return; + + start = relundo_oid_hash(relid); + + for (probes = 0; probes < RELUNDO_HEAD_CACHE_SIZE; probes++) + { + i = (start + probes) & RELUNDO_HEAD_CACHE_MASK; + + if (relundo_head_cache[i].relid == relid) + { + /* Replace with tombstone, not InvalidOid, to preserve chains */ + relundo_head_cache[i].relid = RELUNDO_CACHE_TOMBSTONE; + return; + } + + if (relundo_head_cache[i].relid == InvalidOid) + return; /* not present; tombstones don't terminate + * probes */ + } +} + +/* + * RelUndoHeadCacheInvalidate -- public wrapper around the per-backend head + * page cache invalidation. + * + * RelUndoDiscard() (in relundo_discard.c) reclaims pages and may physically + * truncate the fork back to the metapage. The discarding backend's head page + * cache can still name a block that no longer exists on disk; the next reserve + * in that backend would then ReadBufferExtended() a truncated block and fault + * with "could not read blocks N..N: read only 0 of 8192 bytes". Discard must + * therefore drop the cache entry for the relation so the next reserve re-reads + * the metapage instead of trusting the stale block number. + */ +void +RelUndoHeadCacheInvalidate(Oid relid) +{ + relundo_head_cache_invalidate(relid); +} + +/* + * RelUndoReserve + * Reserve space for an UNDO record (Phase 1 of 2-phase insert) + * + * Finds a page with enough free space for record_size bytes (which must + * include the RelUndoRecordHeader). If the current head page doesn't have + * enough room, a new page is allocated and linked at the head. + * + * Returns an RelUndoRecPtr encoding (counter, blockno, offset). + * The buffer is returned pinned and exclusively locked via *undo_buffer. + */ +RelUndoRecPtr +RelUndoReserve(Relation rel, Size record_size, Buffer *undo_buffer) +{ + Buffer metabuf; + Page metapage; + RelUndoMetaPage meta; + Buffer databuf; + Page datapage; + RelUndoPageHeader datahdr; + BlockNumber blkno; + uint16 offset; + RelUndoRecPtr ptr; + RelUndoHeadCacheEntry *cache_entry; + int slot = relundo_my_slot(); + + /* + * Sanity check: record must fit on an empty data page. The usable space + * is the contents area minus our RelUndoPageHeaderData. + */ + { + Size max_record = BLCKSZ - MAXALIGN(SizeOfPageHeaderData) + - SizeOfRelUndoPageHeaderData; + + if (record_size > max_record) + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("UNDO record size %zu exceeds maximum %zu", + record_size, max_record))); + } + + /* + * Tier 1: per-backend head page cache. If we have a cached head page for + * this relation with enough free space, skip the metapage lock entirely + * and go directly to the data page. + * + * pd_lower is advanced (at the reserve: label below) with a plain + * non-atomic read-add-store, so it MUST be done under the buffer's + * EXCLUSIVE content lock. An earlier "lock-free" tier advanced pd_lower + * via an atomic CAS while holding no content lock; that was unsound + * because a buffer content lock and a lock-free atomic do not mutually + * exclude, so the non-atomic store on this path could straddle and + * clobber a concurrent CAS on the same hot head page, handing two + * reservers overlapping offsets and corrupting the resulting UNDO + * records. Every remaining reserve path advances pd_lower only under the + * buffer lock, so they are mutually exclusive. + */ + cache_entry = relundo_head_cache_lookup(RelationGetRelid(rel)); + if (cache_entry != NULL && + BlockNumberIsValid(cache_entry->head_blkno) && + cache_entry->free_space >= record_size && + cache_entry->head_blkno < + smgrnblocks(RelationGetSmgr(rel), RELUNDO_FORKNUM)) + { + /* + * The head cache is process-local, but a concurrent autovacuum worker + * (holding only ShareUpdateExclusiveLock, which does not exclude our + * RowExclusiveLock) can whole-chain-discard and physically truncate + * this fork. smgrtruncate broadcasts a smgr invalidation but does + * NOT touch any other backend's relundo_head_cache, so our cached + * head_blkno may now point past the shrunken EOF. Reading it would + * either fault on a nonexistent block or return a recycled/garbage + * page. The smgrnblocks bound in the guard above rejects that stale + * entry, and we fall through to the metapage path, which re-reads the + * authoritative head_blkno under lock. + */ + databuf = ReadBufferExtended(rel, RELUNDO_FORKNUM, + cache_entry->head_blkno, + RBM_NORMAL, NULL); + LockBuffer(databuf, BUFFER_LOCK_EXCLUSIVE); + datapage = BufferGetPage(databuf); + + /* Verify the page still has space (another backend may have used it) */ + if (relundo_get_free_space(datapage) >= record_size) + { + blkno = cache_entry->head_blkno; + + /* Update cached free space */ + cache_entry->free_space = relundo_get_free_space(datapage) - record_size; + + goto reserve; + } + + /* Cache was stale -- fall through to metapage path */ + UnlockReleaseBuffer(databuf); + relundo_head_cache_invalidate(RelationGetRelid(rel)); + } + + /* + * Tier 2: Shared-lock fast path. Read the metapage with SHARED lock to + * get the head page, then check the data page directly. Only if the data + * page is full do we re-acquire the metapage with EXCLUSIVE lock for new + * page allocation. This reduces contention when multiple backends are + * doing concurrent DML on the same relation. + */ + metabuf = relundo_get_metapage(rel, BUFFER_LOCK_SHARE); + metapage = BufferGetPage(metabuf); + meta = (RelUndoMetaPage) PageGetContents(metapage); + + elog(DEBUG1, "RelUndoReserve: record_size=%zu, slot=%d, head_blkno=%u", + record_size, slot, meta->head_blkno[slot]); + + if (BlockNumberIsValid(meta->head_blkno[slot])) + { + BlockNumber cached_head = meta->head_blkno[slot]; + + /* Release the shared lock before touching the data page */ + UnlockReleaseBuffer(metabuf); + + elog(DEBUG1, "RelUndoReserve: reading existing head page %u (shared-lock path)", + cached_head); + + databuf = ReadBufferExtended(rel, RELUNDO_FORKNUM, cached_head, + RBM_NORMAL, NULL); + LockBuffer(databuf, BUFFER_LOCK_EXCLUSIVE); + + datapage = BufferGetPage(databuf); + + elog(DEBUG1, "RelUndoReserve: free_space=%zu", + relundo_get_free_space(datapage)); + + if (relundo_get_free_space(datapage) >= record_size) + { + /* Enough space on current head page */ + blkno = cached_head; + + elog(DEBUG1, "RelUndoReserve: enough space, using block %u", blkno); + + /* Update the head page cache */ + relundo_head_cache_update(RelationGetRelid(rel), blkno, + relundo_get_free_space(datapage) - record_size); + + goto reserve; + } + + /* Not enough space; release this page, fall through to exclusive path */ + elog(DEBUG1, "RelUndoReserve: not enough space, need new page allocation"); + UnlockReleaseBuffer(databuf); + } + else + { + /* No head page yet -- release shared lock */ + UnlockReleaseBuffer(metabuf); + } + + /* + * Need EXCLUSIVE metapage lock for new page allocation. Re-read the + * metapage since another backend may have allocated a new page between + * our shared-lock release and now. + */ + metabuf = relundo_get_metapage(rel, BUFFER_LOCK_EXCLUSIVE); + metapage = BufferGetPage(metabuf); + meta = (RelUndoMetaPage) PageGetContents(metapage); + + /* Re-check: another backend may have added space while we waited */ + if (BlockNumberIsValid(meta->head_blkno[slot])) + { + databuf = ReadBufferExtended(rel, RELUNDO_FORKNUM, meta->head_blkno[slot], + RBM_NORMAL, NULL); + LockBuffer(databuf, BUFFER_LOCK_EXCLUSIVE); + datapage = BufferGetPage(databuf); + + if (relundo_get_free_space(datapage) >= record_size) + { + blkno = meta->head_blkno[slot]; + + relundo_head_cache_update(RelationGetRelid(rel), blkno, + relundo_get_free_space(datapage) - record_size); + + UnlockReleaseBuffer(metabuf); + goto reserve; + } + + UnlockReleaseBuffer(databuf); + } + + /* + * Need a new page. relundo_allocate_page handles free list / extend, + * links the new page as head, and marks both buffers dirty. + */ + blkno = relundo_allocate_page(rel, metabuf, slot, &databuf); + datapage = BufferGetPage(databuf); + + /* Update cache with the new head page */ + relundo_head_cache_update(RelationGetRelid(rel), blkno, + relundo_get_free_space(datapage) - record_size); + + /* + * Keep the metapage locked: RelUndoFinish() needs it for the WAL record. + * Store in per-backend variable to avoid changing the API. + */ + relundo_pending_metabuf = metabuf; + +reserve: + /* Reserve space by advancing pd_lower */ + elog(DEBUG1, "RelUndoReserve: at reserve label, block=%u", blkno); + + datahdr = (RelUndoPageHeader) PageGetContents(datapage); + + elog(DEBUG1, "RelUndoReserve: datahdr=%p, pd_lower=%u, pd_upper=%u, counter=%u", + datahdr, datahdr->pd_lower, datahdr->pd_upper, datahdr->counter); + + offset = datahdr->pd_lower; + datahdr->pd_lower += record_size; + + elog(DEBUG1, "RelUndoReserve: reserved offset=%u, new pd_lower=%u", + offset, datahdr->pd_lower); + + /* Build the UNDO pointer */ + ptr = MakeRelUndoRecPtr(datahdr->counter, blkno, offset); + + *undo_buffer = databuf; + return ptr; +} + +/* + * RelUndoStage + * Write an UNDO record onto its reserved page WITHOUT WAL logging. + * + * Performs every page mutation RelUndoFinish() does (header+payload memcpy, + * max_xid bump, MarkBufferDirty on the data page and, for a new page, the + * metapage) and builds the block-0 WAL data buffer, but does NOT open a + * critical section, XLogInsert, PageSetLSN, or release any buffer. The undo + * buffer (and metapage, if a new page was allocated) stay locked+pinned. + * + * The staged facts are returned in *result so the caller can either emit the + * standalone RM_RELUNDO_ID record (RelUndoFinish wrapper) or fold the same + * bytes into a combined record under a different resource manager (the + * caller's WAL-fold path). + */ +void +RelUndoStage(Relation rel, Buffer undo_buffer, RelUndoRecPtr ptr, + const RelUndoRecordHeader *header, const void *payload, + Size payload_size, RelUndoStageResult *result) +{ + Page page; + char *contents; + uint16 offset; + Size total_record_size; + char *record_data; + RelUndoPageHeader datahdr; + bool is_new_page; + Buffer metabuf = InvalidBuffer; + + page = BufferGetPage(undo_buffer); + contents = PageGetContents(page); + offset = RelUndoGetOffset(ptr); + datahdr = (RelUndoPageHeader) contents; + + /* + * Check if this is the first record on a newly allocated page. If the + * offset equals the header size, this is a new page. + */ + is_new_page = (offset == SizeOfRelUndoPageHeaderData); + + /* Calculate total UNDO record size */ + total_record_size = SizeOfRelUndoRecordHeader + payload_size; + + /* Write the header */ + memcpy(contents + offset, header, SizeOfRelUndoRecordHeader); + + /* Write the payload immediately after the header */ + if (payload_size > 0 && payload != NULL) + memcpy(contents + offset + SizeOfRelUndoRecordHeader, + payload, payload_size); + + /* + * Advance the page's max_xid watermark to cover this record. Done before + * WAL-logging so the new-page header copy and the xlrec both observe the + * updated value; redo restores max_xid from the xlrec. + */ + if (!TransactionIdIsValid(datahdr->max_xid) || + TransactionIdFollows(header->urec_xid, datahdr->max_xid)) + datahdr->max_xid = header->urec_xid; + + /* + * Mark the buffer dirty now, before any critical section. + * XLogRegisterBuffer requires the buffer to be dirty when called. + */ + MarkBufferDirty(undo_buffer); + + /* + * If this is a new page, adopt the metapage lock that RelUndoReserve left + * pending. It was modified during page allocation and must be included + * in whichever WAL record the caller emits. + */ + if (is_new_page) + { + Assert(BufferIsValid(relundo_pending_metabuf)); + metabuf = relundo_pending_metabuf; + relundo_pending_metabuf = InvalidBuffer; + + /* Mark metabuf dirty before WAL-logging (assertion requires it) */ + MarkBufferDirty(metabuf); + } + + /* + * Build the block-0 WAL data buffer. For a new page we prepend the + * RelUndoPageHeaderData so redo can reconstruct prev_blkno/counter. + */ + if (is_new_page) + { + Size wal_data_size = SizeOfRelUndoPageHeaderData + total_record_size; + + record_data = (char *) palloc(wal_data_size); + + /* Copy page header */ + memcpy(record_data, datahdr, SizeOfRelUndoPageHeaderData); + + /* Copy UNDO record after the page header */ + memcpy(record_data + SizeOfRelUndoPageHeaderData, + header, SizeOfRelUndoRecordHeader); + if (payload_size > 0 && payload != NULL) + memcpy(record_data + SizeOfRelUndoPageHeaderData + SizeOfRelUndoRecordHeader, + payload, payload_size); + + result->wal_record_size = wal_data_size; + } + else + { + /* Normal case: just the UNDO record */ + record_data = (char *) palloc(total_record_size); + memcpy(record_data, header, SizeOfRelUndoRecordHeader); + if (payload_size > 0 && payload != NULL) + memcpy(record_data + SizeOfRelUndoRecordHeader, payload, payload_size); + + result->wal_record_size = total_record_size; + } + + result->undo_buffer = undo_buffer; + result->metabuf = metabuf; + result->is_new_page = is_new_page; + result->urec_type = header->urec_type; + result->urec_len = header->urec_len; + result->page_offset = MAXALIGN(SizeOfPageHeaderData) + offset; + result->new_pd_lower = datahdr->pd_lower; + result->max_xid = datahdr->max_xid; + result->wal_record_data = record_data; +} + +/* + * RelUndoFinish + * Complete UNDO record insertion (Phase 2 of 2-phase insert) + * + * Writes the header and payload into the space reserved by RelUndoReserve(), + * WAL-logs the insertion as a standalone RM_RELUNDO_ID record, and releases the + * buffer(s). Built on RelUndoStage(): stage the page mutation, then emit the + * record and stamp the LSN. + */ +void +RelUndoFinish(Relation rel, Buffer undo_buffer, RelUndoRecPtr ptr, + const RelUndoRecordHeader *header, const void *payload, + Size payload_size) +{ + RelUndoStageResult staged; + Page page; + uint8 info; + + RelUndoStage(rel, undo_buffer, ptr, header, payload, payload_size, &staged); + + page = BufferGetPage(undo_buffer); + + /* WAL-log the insertion */ + START_CRIT_SECTION(); + + { + xl_relundo_insert xlrec; + + xlrec.urec_type = staged.urec_type; + xlrec.urec_len = staged.urec_len; + xlrec.page_offset = staged.page_offset; + xlrec.new_pd_lower = staged.new_pd_lower; + xlrec.max_xid = staged.max_xid; + + info = XLOG_RELUNDO_INSERT; + if (staged.is_new_page) + info |= XLOG_RELUNDO_INIT_PAGE; + + XLogBeginInsert(); + XLogRegisterData((char *) &xlrec, SizeOfRelundoInsert); + + /* + * Register the data page. We register the entire UNDO record (header + * + payload) as block data. + * + * For a new page, the block data also carries the + * RelUndoPageHeaderData so redo can reconstruct prev_blkno/counter; + * REGBUF_WILL_INIT tells redo it will initialize the page. + * + * For an existing page, do NOT pass REGBUF_STANDARD. RelUndo data + * pages keep the standard PageHeader.pd_lower pinned at the empty + * value and track their real used extent in the shadow + * RelUndoPageHeader inside the page contents area. REGBUF_STANDARD + * would treat [pd_lower, pd_upper) as a free "hole" and elide the + * entire contents from any full-page image, so a BLK_RESTORED redo + * would bring the page back zeroed -- losing the record bytes and the + * prev_blkno chain link. Logging the whole page (flag 0) keeps the + * FPI faithful. + */ + if (staged.is_new_page) + XLogRegisterBuffer(0, undo_buffer, REGBUF_WILL_INIT); + else + XLogRegisterBuffer(0, undo_buffer, 0); + + XLogRegisterBufData(0, staged.wal_record_data, staged.wal_record_size); + + /* + * When allocating a new page, the metapage was also updated + * (head_blkno). Register it as block 1 so the metapage state is + * preserved in WAL. Use REGBUF_STANDARD to get a full page image. + */ + if (staged.is_new_page) + XLogRegisterBuffer(1, staged.metabuf, REGBUF_STANDARD); + + { + XLogRecPtr recptr = XLogInsert(RM_RELUNDO_ID, info); + + /* + * Stamp the record LSN onto every page we dirtied and registered. + * Without this the buffer manager's WAL-before-data rule is + * broken: the checkpointer flushes WAL only up to a dirty + * buffer's page LSN before writing it, so a stale/zero LSN lets + * an UNDO page (including its prev_blkno chain links) reach disk + * ahead of the WAL that describes it, corrupting the chain on + * crash recovery. + */ + PageSetLSN(page, recptr); + if (staged.is_new_page) + PageSetLSN(BufferGetPage(staged.metabuf), recptr); + } + } + + END_CRIT_SECTION(); + + pfree(staged.wal_record_data); + + UnlockReleaseBuffer(undo_buffer); + + /* Release metapage if we locked it */ + if (BufferIsValid(staged.metabuf)) + UnlockReleaseBuffer(staged.metabuf); +} + +/* + * RelUndoFinishWithTuple + * Complete UNDO record insertion with tuple data (Phase 2 of 2-phase insert) + * + * Like RelUndoFinish(), but also writes tuple data after the payload. + * The total record layout on the UNDO page is: + * [RelUndoRecordHeader][payload][tuple_data] + * + * The header must have RELUNDO_INFO_HAS_TUPLE set and tuple_len filled in + * by the caller. + */ +void +RelUndoFinishWithTuple(Relation rel, Buffer undo_buffer, RelUndoRecPtr ptr, + const RelUndoRecordHeader *header, const void *payload, + Size payload_size, const char *tuple_data, + uint32 tuple_len) +{ + Page page; + char *contents; + uint16 offset; + Size total_record_size; + xl_relundo_insert xlrec; + char *record_data; + RelUndoPageHeader datahdr; + bool is_new_page; + uint8 info; + Buffer metabuf = InvalidBuffer; + + elog(DEBUG1, "RelUndoFinishWithTuple: starting, ptr=%lu, payload_size=%zu, tuple_len=%u", + (unsigned long) ptr, payload_size, tuple_len); + + page = BufferGetPage(undo_buffer); + contents = PageGetContents(page); + offset = RelUndoGetOffset(ptr); + datahdr = (RelUndoPageHeader) contents; + + is_new_page = (offset == SizeOfRelUndoPageHeaderData); + + /* Total UNDO record size includes header + payload + tuple data */ + total_record_size = SizeOfRelUndoRecordHeader + payload_size + tuple_len; + + /* Write the header */ + memcpy(contents + offset, header, SizeOfRelUndoRecordHeader); + + /* Write the payload immediately after the header */ + if (payload_size > 0 && payload != NULL) + memcpy(contents + offset + SizeOfRelUndoRecordHeader, + payload, payload_size); + + /* Write the tuple data after the payload */ + if (tuple_len > 0 && tuple_data != NULL) + memcpy(contents + offset + SizeOfRelUndoRecordHeader + payload_size, + tuple_data, tuple_len); + + /* Advance the page's max_xid watermark to cover this record. */ + if (!TransactionIdIsValid(datahdr->max_xid) || + TransactionIdFollows(header->urec_xid, datahdr->max_xid)) + datahdr->max_xid = header->urec_xid; + + MarkBufferDirty(undo_buffer); + + if (is_new_page) + { + Assert(BufferIsValid(relundo_pending_metabuf)); + metabuf = relundo_pending_metabuf; + relundo_pending_metabuf = InvalidBuffer; + + /* Mark metabuf dirty before WAL-logging (assertion requires it) */ + MarkBufferDirty(metabuf); + } + + /* + * Allocate WAL record data buffer before entering critical section. + */ + if (is_new_page) + { + Size wal_data_size = SizeOfRelUndoPageHeaderData + total_record_size; + + record_data = (char *) palloc(wal_data_size); + memcpy(record_data, datahdr, SizeOfRelUndoPageHeaderData); + memcpy(record_data + SizeOfRelUndoPageHeaderData, + header, SizeOfRelUndoRecordHeader); + if (payload_size > 0 && payload != NULL) + memcpy(record_data + SizeOfRelUndoPageHeaderData + SizeOfRelUndoRecordHeader, + payload, payload_size); + if (tuple_len > 0 && tuple_data != NULL) + memcpy(record_data + SizeOfRelUndoPageHeaderData + SizeOfRelUndoRecordHeader + payload_size, + tuple_data, tuple_len); + } + else + { + record_data = (char *) palloc(total_record_size); + memcpy(record_data, header, SizeOfRelUndoRecordHeader); + if (payload_size > 0 && payload != NULL) + memcpy(record_data + SizeOfRelUndoRecordHeader, payload, payload_size); + if (tuple_len > 0 && tuple_data != NULL) + memcpy(record_data + SizeOfRelUndoRecordHeader + payload_size, + tuple_data, tuple_len); + } + + /* WAL-log the insertion */ + START_CRIT_SECTION(); + + xlrec.urec_type = header->urec_type; + xlrec.urec_len = header->urec_len; + xlrec.page_offset = MAXALIGN(SizeOfPageHeaderData) + offset; + xlrec.new_pd_lower = datahdr->pd_lower; + xlrec.max_xid = datahdr->max_xid; + + info = XLOG_RELUNDO_INSERT; + if (is_new_page) + info |= XLOG_RELUNDO_INIT_PAGE; + + XLogBeginInsert(); + XLogRegisterData((char *) &xlrec, SizeOfRelundoInsert); + + if (is_new_page) + { + Size wal_data_size = SizeOfRelUndoPageHeaderData + total_record_size; + + XLogRegisterBuffer(0, undo_buffer, REGBUF_WILL_INIT); + XLogRegisterBufData(0, record_data, wal_data_size); + XLogRegisterBuffer(1, metabuf, REGBUF_STANDARD); + } + else + { + /* Full page image, no hole: see the REGBUF note in RelUndoFinish. */ + XLogRegisterBuffer(0, undo_buffer, 0); + XLogRegisterBufData(0, record_data, total_record_size); + } + + { + XLogRecPtr recptr = XLogInsert(RM_RELUNDO_ID, info); + + /* + * Stamp the record LSN onto every dirtied+registered page; see the + * WAL-before-data note in RelUndoFinish. + */ + PageSetLSN(page, recptr); + if (is_new_page) + PageSetLSN(BufferGetPage(metabuf), recptr); + } + + END_CRIT_SECTION(); + + pfree(record_data); + + UnlockReleaseBuffer(undo_buffer); + + if (BufferIsValid(metabuf)) + UnlockReleaseBuffer(metabuf); +} + +/* + * RelUndoCancel + * Cancel UNDO record reservation + * + * The reserved space is left as a zero-filled hole. Readers will see + * urec_type == 0 and skip it. The buffer is released. + */ +void +RelUndoCancel(Relation rel, Buffer undo_buffer, RelUndoRecPtr ptr) +{ + /* + * The space was already zeroed by relundo_init_page(). pd_lower has been + * advanced past it, so it's just a hole. Nothing to write. + */ + UnlockReleaseBuffer(undo_buffer); + + /* Release pending metapage buffer if RelUndoReserve allocated a new page */ + if (BufferIsValid(relundo_pending_metabuf)) + { + UnlockReleaseBuffer(relundo_pending_metabuf); + relundo_pending_metabuf = InvalidBuffer; + } +} + +/* + * relundo_fork_nblocks_fast + * + * Return the number of blocks in the UNDO fork with no per-call filesystem + * syscalls on the hot path. + * + * The old guard was smgrexists() + smgrnblocks() per call. smgrexists() -> + * mdexists() unconditionally mdclose()s the fork fd (to notice an unlink) and + * the next access reopens it; on the multiversion read path (per scanned + * tuple carrying a retained before-image) that FD open/close storm dominated + * CPU at high core count (dentry lockref contention). smgrnblocks_cached() + * does NOT help outside recovery -- it only returns a cached value when + * InRecovery -- so the previous version still hit smgrexists() every call. + * + * smgrnblocks() itself is cheap after the first call: it opens the fork once + * (mdopenfork) and leaves the fd cached in the SMgrRelation; it never closes + * it. So the fix is to drop smgrexists() from the hot path entirely and rely + * on smgrnblocks(). A valid RelUndoRecPtr can only have been produced by a + * writer that extended the fork, so the fork provably exists whenever a + * caller has a valid pointer into it; smgrnblocks() therefore never faults + * here. For extra safety against a fake/partial relcache entry whose fork is + * genuinely absent, we probe smgrexists() exactly ONCE per SMgrRelation and + * latch the positive result (a stat that then keeps the fd via the + * subsequent smgrnblocks open); if the fork is absent we return 0 and never + * cache, so a later-created fork is still picked up. + */ +static inline BlockNumber +relundo_fork_nblocks_fast(Relation rel) +{ + SMgrRelation smgr = RelationGetSmgr(rel); + + /* + * If the fork's fd is already open in this SMgrRelation, it exists and is + * open -- go straight to smgrnblocks (no stat, no close). + * md_num_open_segs is >0 once mdopenfork has run for this fork. + */ + if (smgr->md_num_open_segs[RELUNDO_FORKNUM] > 0) + return smgrnblocks(smgr, RELUNDO_FORKNUM); + + /* Cold: confirm the fork exists once, then open+size it (fd stays open). */ + if (!smgrexists(smgr, RELUNDO_FORKNUM)) + return 0; + return smgrnblocks(smgr, RELUNDO_FORKNUM); +} + +/* + * RelUndoReadRecord + * Read an UNDO record from the log + * + * Reads the header and payload from the location encoded in ptr. + * Returns false if the pointer is invalid or the record has been discarded. + * On success, *payload is palloc'd and must be pfree'd by the caller. + */ +bool +RelUndoReadRecord(Relation rel, RelUndoRecPtr ptr, RelUndoRecordHeader *header, + void **payload, Size *payload_size) +{ + BlockNumber blkno; + uint16 offset; + Buffer buf; + Page page; + char *contents; + Size psize; + + if (!RelUndoRecPtrIsValid(ptr)) + return false; + + blkno = RelUndoGetBlockNum(ptr); + offset = RelUndoGetOffset(ptr); + + /* + * Bounds-check the block against the UNDO fork size using the cached + * nblocks (no per-call filesystem syscalls; see + * relundo_fork_nblocks_fast). A zero result means the fork does not + * exist -- treat as out of range. The UNDO fork is always a standard + * BLCKSZ-paged smgr fork. + */ + if (blkno >= relundo_fork_nblocks_fast(rel)) + return false; + + buf = ReadBufferExtended(rel, RELUNDO_FORKNUM, blkno, RBM_NORMAL, NULL); + LockBuffer(buf, BUFFER_LOCK_SHARE); + + page = BufferGetPage(buf); + contents = PageGetContents(page); + + /* Validate that offset is within the written portion of the page */ + { + RelUndoPageHeader hdr = (RelUndoPageHeader) contents; + + if (offset < SizeOfRelUndoPageHeaderData || offset >= hdr->pd_lower) + { + UnlockReleaseBuffer(buf); + return false; + } + + /* + * ABA defense: reject a verptr whose embedded generation counter does + * not match the page's current counter. A free-list recycle bumps + * meta->counter (relundo_page.c) before re-initialising the page, so + * a stale verptr to a recycled blkno will see hdr->counter != its own + * counter and fail here. Returning false is the chain-end signal: + * the caller's version-reconstruction walk treats it as a best-effort + * terminator, which is the correct, safe answer (never silently + * reverse-apply a structurally valid but unrelated record). + */ + if (RelUndoGetCounter(ptr) != hdr->counter) + { + UnlockReleaseBuffer(buf); + return false; + } + } + + /* Copy the header */ + memcpy(header, contents + offset, SizeOfRelUndoRecordHeader); + + /* A zero urec_type means the slot was cancelled (hole) */ + if (header->urec_type == 0) + { + UnlockReleaseBuffer(buf); + return false; + } + + /* Calculate payload size and copy it */ + if (header->urec_len > SizeOfRelUndoRecordHeader) + { + psize = header->urec_len - SizeOfRelUndoRecordHeader; + *payload = palloc(psize); + memcpy(*payload, contents + offset + SizeOfRelUndoRecordHeader, psize); + *payload_size = psize; + } + else + { + *payload = NULL; + *payload_size = 0; + } + + UnlockReleaseBuffer(buf); + return true; +} + +/* + * RelUndoReadRecordHeader + * Read only the header of an UNDO record. + * + * Same discard/ABA/hole semantics as RelUndoReadRecord but skips the + * payload palloc+memcpy. Used by hot probes (e.g. the lost-update + * conflict probe) that need only urec_xid. + */ +bool +RelUndoReadRecordHeader(Relation rel, RelUndoRecPtr ptr, + RelUndoRecordHeader *header) +{ + BlockNumber blkno; + uint16 offset; + Buffer buf; + Page page; + char *contents; + + if (!RelUndoRecPtrIsValid(ptr)) + return false; + + blkno = RelUndoGetBlockNum(ptr); + offset = RelUndoGetOffset(ptr); + + /* Cached bounds check; see relundo_fork_nblocks_fast. */ + if (blkno >= relundo_fork_nblocks_fast(rel)) + return false; + + buf = ReadBufferExtended(rel, RELUNDO_FORKNUM, blkno, RBM_NORMAL, NULL); + LockBuffer(buf, BUFFER_LOCK_SHARE); + + page = BufferGetPage(buf); + contents = PageGetContents(page); + + { + RelUndoPageHeader hdr = (RelUndoPageHeader) contents; + + if (offset < SizeOfRelUndoPageHeaderData || offset >= hdr->pd_lower) + { + UnlockReleaseBuffer(buf); + return false; + } + + if (RelUndoGetCounter(ptr) != hdr->counter) + { + UnlockReleaseBuffer(buf); + return false; + } + } + + memcpy(header, contents + offset, SizeOfRelUndoRecordHeader); + + if (header->urec_type == 0) + { + UnlockReleaseBuffer(buf); + return false; + } + + UnlockReleaseBuffer(buf); + return true; +} + +/* + * RelUndoInitRelation + * Initialize per-relation UNDO for a new relation + * + * Creates the UNDO fork and writes the initial metapage (block 0). + * The chain starts empty (head_blkno = tail_blkno = InvalidBlockNumber). + * + * This function is idempotent: if the UNDO fork already exists (e.g., + * during TRUNCATE where the new relfilenumber may already have a fork + * from a prior operation, or during recovery replay), we truncate it + * back to zero blocks and reinitialize. + */ +void +RelUndoInitRelation(Relation rel) +{ + Buffer metabuf; + Page metapage; + RelUndoMetaPage meta; + SMgrRelation srel; + + /* Invalidate cached head page for this relation */ + relundo_head_cache_invalidate(RelationGetRelid(rel)); + + srel = RelationGetSmgr(rel); + + /* + * Create the physical fork file. Pass isRedo=true so that smgrcreate is + * idempotent -- if the file already exists (e.g., during TRUNCATE or + * recovery replay), it simply opens it rather than raising an error. + */ + smgrcreate(srel, RELUNDO_FORKNUM, true); + + /* + * WAL-log the fork creation for crash safety. + */ + if (!InRecovery) + log_smgrcreate(&rel->rd_locator, RELUNDO_FORKNUM); + + /* + * If the fork already has blocks (e.g., re-initialization during + * TRUNCATE), truncate it back to zero so we can reinitialize cleanly. + * This discards any stale UNDO data from the previous relfilenumber + * incarnation. + */ + if (smgrnblocks(srel, RELUNDO_FORKNUM) > 0) + { + ForkNumber forknum = RELUNDO_FORKNUM; + BlockNumber old_nblocks = smgrnblocks(srel, RELUNDO_FORKNUM); + BlockNumber new_nblocks = 0; + + smgrtruncate(srel, &forknum, 1, &old_nblocks, &new_nblocks); + } + + /* Allocate the metapage (block 0) */ + metabuf = ExtendBufferedRel(BMR_REL(rel), RELUNDO_FORKNUM, NULL, + EB_LOCK_FIRST); + + Assert(BufferGetBlockNumber(metabuf) == 0); + + metapage = BufferGetPage(metabuf); + + /* Initialize standard page header */ + PageInit(metapage, BLCKSZ, 0); + + /* Initialize the UNDO metapage fields */ + meta = (RelUndoMetaPage) PageGetContents(metapage); + meta->magic = RELUNDO_METAPAGE_MAGIC; + meta->version = RELUNDO_METAPAGE_VERSION; + meta->counter = 1; /* Start at 1 so 0 is clearly "no counter" */ + for (int s = 0; s < RELUNDO_NUM_HEADS; s++) + { + meta->head_blkno[s] = InvalidBlockNumber; + meta->tail_blkno[s] = InvalidBlockNumber; + } + meta->free_blkno = InvalidBlockNumber; + meta->total_records = 0; + meta->discarded_records = 0; + meta->system_alloc_watermark = InvalidBlockNumber; + + /* Include the meta struct in the recorded region of any FPI. */ + RelUndoMetaPageSetPdLower(metapage); + + MarkBufferDirty(metabuf); + + /* + * WAL-log the metapage initialization. This is critical for crash safety. + * If we crash after table creation but before the first INSERT, the + * metapage must be recoverable. + */ + if (!InRecovery) + { + xl_relundo_init xlrec; + XLogRecPtr recptr; + + xlrec.magic = RELUNDO_METAPAGE_MAGIC; + xlrec.version = RELUNDO_METAPAGE_VERSION; + xlrec.counter = 1; + + XLogBeginInsert(); + XLogRegisterData((char *) &xlrec, SizeOfRelundoInit); + XLogRegisterBuffer(0, metabuf, REGBUF_WILL_INIT | REGBUF_STANDARD); + + recptr = XLogInsert(RM_RELUNDO_ID, XLOG_RELUNDO_INIT); + + PageSetLSN(metapage, recptr); + } + + UnlockReleaseBuffer(metabuf); +} + +/* + * RelUndoDropRelation + * Drop per-relation UNDO when relation is dropped + * + * The UNDO fork is removed along with the relation's other forks by the + * storage manager. We just need to make sure we don't leave stale state. + */ +void +RelUndoDropRelation(Relation rel) +{ + SMgrRelation srel; + + srel = RelationGetSmgr(rel); + + /* + * If the UNDO fork doesn't exist, nothing to do. This handles the case + * where the relation never had per-relation UNDO enabled. + */ + if (!smgrexists(srel, RELUNDO_FORKNUM)) + return; + + /* + * The actual file removal happens as part of the relation's overall drop + * via smgrdounlinkall(). We don't need to explicitly drop the fork here + * because the storage manager handles all forks together. + * + * If in the future we need explicit fork removal, we could truncate and + * unlink here. + */ +} + +/* + * RelUndoVacuum + * Vacuum per-relation UNDO log + * + * Discards UNDO records whose owning transaction precedes oldest_xmin, the + * oldest XID for which any active transaction could still require a rollback + * before-image. Pages whose max_xid precedes oldest_xmin are reclaimed. + * + * RelUndoDiscard splices the reclaimed run directly onto the metapage's free + * list as a single bounded, fully WAL-logged operation, so the pages are + * immediately available for reuse by relundo_allocate_page and the fork stops + * growing across repeated VACUUM cycles. + */ +void +RelUndoVacuum(Relation rel, TransactionId oldest_xmin, bool nowait) +{ + /* If no UNDO fork exists, nothing to vacuum */ + if (!smgrexists(RelationGetSmgr(rel), RELUNDO_FORKNUM)) + return; + + /* A meaningless horizon would discard nothing; bail early. */ + if (!TransactionIdIsValid(oldest_xmin)) + return; + + RelUndoDiscard(rel, oldest_xmin, nowait); +} + +/* + * RELUNDO_MAYBE_VACUUM_MIN_BLOCKS + * Skip the throttled discard sweep below this fork size (in blocks). + * Chosen to make the common case (a quiescent or lightly-written + * table) resolve to a single relundo_fork_nblocks_fast() call with no + * buffer I/O, while still catching sustained-churn growth well before + * it reaches problematic size. + */ +#define RELUNDO_MAYBE_VACUUM_MIN_BLOCKS 64 + +/* + * RelUndoMaybeVacuum + * Throttled, self-clocking per-relation UNDO fork discard. + * + * RelUndoVacuum() (the function above) is normally invoked once per VACUUM, + * via the owning table AM's relation_vacuum callback. But an AM whose + * updates are in-place can correctly report near-zero dead tuples, + * so autovacuum's dead-tuple/insert-count thresholds may never fire even + * under sustained write churn -- and RelUndoVacuum is the ONLY code that + * discards the on-disk UNDO fork, so an AM that never gets vacuumed would + * grow its UNDO fork without bound. + * + * This is a cheap, throttled backstop an AM's DML path can call so fork + * discard runs on its own schedule, decoupled from VACUUM ever being + * triggered. Intended to be called from the AM's DML hot path after + * releasing all buffer/tuple locks -- RelUndoDiscard() takes the UNDO fork's + * own metapage lock, which must never be acquired while holding a data page + * lock (that would convoy every writer to a hot page behind the discard + * sweep). + * + * The cadence uses a per-backend static timestamp: each backend independently + * throttles to once every 5 seconds, so concurrent backends do redundant but + * bounded work rather than needing shared coordination. + */ +void +RelUndoMaybeVacuum(Relation rel) +{ + static TimestampTz relundo_last_vacuum = 0; + TimestampTz now_ts; + BlockNumber nblocks; + + now_ts = GetCurrentTimestamp(); + if (now_ts - relundo_last_vacuum < 5000000) /* 5 seconds */ + return; + + nblocks = relundo_fork_nblocks_fast(rel); + if (nblocks < RELUNDO_MAYBE_VACUUM_MIN_BLOCKS) + return; + + relundo_last_vacuum = now_ts; + + /* + * Hot-path discard MUST NOT block: this runs inline from the AM's update + * path for every qualifying update. Blocking on the per-relation UNDO + * metapage EXCLUSIVE lock here serializes ALL concurrent updates to the + * relation behind one lock (measured: the dominant cause of a TPROC-C + * throughput collapse and hours-long lock-wait pileups on hot tables like + * TPC-C district). Pass nowait=true so a contended metapage is simply + * skipped -- some other backend (or a later call) reclaims the space; + * UNDO discard is space reclamation, never per-update correctness. + */ + RelUndoVacuum(rel, GetOldestNonRemovableTransactionId(rel), true); +} diff --git a/src/backend/access/undo/relundo_apply.c b/src/backend/access/undo/relundo_apply.c new file mode 100644 index 0000000000000..aa3e0aacfa838 --- /dev/null +++ b/src/backend/access/undo/relundo_apply.c @@ -0,0 +1,1057 @@ +/*------------------------------------------------------------------------- + * + * relundo_apply.c + * Apply per-relation UNDO records for transaction rollback + * + * This module implements transaction rollback for per-relation UNDO. + * It walks the UNDO chain backwards and applies each operation to restore + * the database to its pre-transaction state. + * + * The rollback operations are: + * - INSERT: Mark inserted tuples as dead/unused + * - DELETE: Restore deleted tuple from UNDO record + * - UPDATE: Restore old tuple version from UNDO record + * - TUPLE_LOCK: Remove lock marker + * + * For crash safety, we write Compensation Log Records (CLRs) for each + * UNDO application. If we crash during rollback, the CLRs prevent + * double-application when recovery replays the UNDO chain. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/access/undo/relundo_apply.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/htup_details.h" +#include "access/relation.h" +#include "access/relundo.h" +#include "access/relundo_xlog.h" +#include "access/xloginsert.h" +#include "miscadmin.h" +#include "storage/buf.h" +#include "storage/bufmgr.h" +#include "storage/bufpage.h" +#include "utils/rel.h" + +/* + * Maximum number of distinct data pages a single UNDO-record apply may touch. + * A per-relation UNDO consumer emits INSERT (1 page), in-place UPDATE (1), + * out-of-place UPDATE/DELTA (2), and DELETE with ntids==1 (1). The cap leaves + * ample room below XLR_MAX_BLOCK_ID (32) for the fork page registered + * alongside. + */ +#define RELUNDO_APPLY_MAX_DATA_BUFS 8 + +/* + * Maximum number of consecutive UNDO records folded into one compensation log + * record (CLR). A bulk DELETE/UPDATE rollback emits one UNDO record per tuple, + * and long runs of those records target the same data page and live on the same + * UNDO-fork page (the chain is walked in reverse-insertion order). Folding such + * a run into a single CLR replaces N forced full-page images plus N XLogFlush + * calls with one of each. The cap bounds how long the batch holds the data + * page's exclusive buffer lock; each per-record apply is an in-memory + * memcpy/flag-set, so 128 is a small, bounded hold. + */ +#define RELUNDO_APPLY_MAX_BATCH 128 + +/* + * Maximum number of distinct UNDO-fork pages whose records a single CLR may + * cover. Each contributes one forced full-page image and one exclusive buffer + * lock held across the XLogInsert. Together with the single batched data page, + * the total registered blocks (1 + this) stays well under XLR_MAX_BLOCK_ID (32). + */ +#define RELUNDO_APPLY_MAX_FORK_BUFS 8 + +/* Forward declarations for internal functions */ +static Page RelUndoTrackPage(Relation rel, Buffer *touched, int *ntouched, + BlockNumber blkno); +static bool RelUndoRecordSingleDataPage(const RelUndoRecordHeader *header, + const void *payload, BlockNumber *blk); +static bool RelUndoForkTrack(BlockNumber *fork_blks, int *nfork, + BlockNumber blkno); +static void RelUndoApplyOneRecord(Relation rel, const RelUndoRecordHeader *header, + void *payload, RelUndoRecPtr current_ptr, + Buffer *touched, int *ntouched); +static void RelUndoApplyInsert(Relation rel, Page page, OffsetNumber offset); +static void RelUndoApplyUpdate(Relation rel, Page page, OffsetNumber offset, + char *tuple_data, uint32 tuple_len); +static void RelUndoApplyDelete(Relation rel, Page page, OffsetNumber offset, + char *tuple_data, uint32 tuple_len); +static void RelUndoApplyTupleLock(Relation rel, Page page, OffsetNumber offset); +static void RelUndoLogApplyCLR(Relation rel, const RelUndoRecPtr *urec_ptrs, + int nptrs, Buffer *touched, int ntouched); + +/* + * RelUndoApplyChain - Walk and apply per-relation UNDO chain for rollback + * + * This is the main entry point for transaction abort. We walk backwards + * through the UNDO chain starting from start_ptr, applying each operation + * until we reach an invalid pointer or the beginning of the chain. + * + * Each record type is handled self-contained: each case manages its own + * buffer acquisition, apply, dirty marking, and buffer release. + */ +void +RelUndoApplyChain(Relation rel, RelUndoRecPtr start_ptr) +{ + RelUndoRecPtr current_ptr = start_ptr; + RelUndoRecordHeader header; + void *payload = NULL; + Size payload_size; + + if (!RelUndoRecPtrIsValid(current_ptr)) + { + elog(DEBUG1, "RelUndoApplyChain: no valid UNDO pointer"); + return; + } + + elog(DEBUG1, "RelUndoApplyChain: starting rollback at %lu", + (unsigned long) current_ptr); + + /* + * Walk backwards through the chain, applying each record. + * + * Each record's physical restoration is applied to its data page(s) while + * holding their buffers exclusively locked, then a single redoable + * compensation log record (XLOG_RELUNDO_APPLY) logs full-page images of + * every restored page plus the UNDO-fork page(s) (carrying the + * CLR_APPLIED flag). The apply helpers may ereport(ERROR) on corruption, + * so they run BEFORE the critical section in RelUndoLogApplyCLR; an error + * there aborts the worker transaction and releases the buffer locks. + * + * Bulk DELETE/UPDATE rollback emits one UNDO record per tuple, and long + * runs of those records target the same data page (reverse-insertion + * order). We fold such a run into one CLR: we keep that one data page's + * buffer exclusively locked and apply each record's in-memory restoration + * onto it, accumulating the records' urec_ptrs, then emit a single CLR + * (one forced data-page image, one flush) covering the whole batch. A + * record that does not target exactly that single data page (out-of-place + * update, multi-TID delete, a different page) terminates the batch and is + * handled on its own with the unchanged singleton path. Folding never + * widens the simultaneous buffer-lock footprint beyond one data page. + */ + while (RelUndoRecPtrIsValid(current_ptr)) + { + Buffer touched[RELUNDO_APPLY_MAX_DATA_BUFS]; + int ntouched = 0; + RelUndoRecPtr batch_ptrs[RELUNDO_APPLY_MAX_BATCH]; + int nbatch = 0; + BlockNumber batch_blkno = InvalidBlockNumber; + BlockNumber fork_blks[RELUNDO_APPLY_MAX_FORK_BUFS]; + int nfork = 0; + int i; + + if (!RelUndoReadRecord(rel, current_ptr, &header, &payload, &payload_size)) + { + elog(WARNING, "RelUndoApplyChain: could not read UNDO record at %lu", + (unsigned long) current_ptr); + break; + } + + /* Skip already-applied records (CLR check for crash safety) */ + if (header.info_flags & RELUNDO_INFO_CLR_APPLIED) + { + elog(DEBUG1, "RelUndoApplyChain: skipping already-applied record at %lu", + (unsigned long) current_ptr); + current_ptr = header.urec_prevundorec; + if (payload) + { + pfree(payload); + payload = NULL; + } + continue; + } + + elog(DEBUG1, "RelUndoApplyChain: processing record type %d at %lu", + header.urec_type, (unsigned long) current_ptr); + + /* Apply this record; it pins+locks its data page(s) into touched[]. */ + RelUndoApplyOneRecord(rel, &header, payload, current_ptr, + touched, &ntouched); + batch_ptrs[nbatch++] = current_ptr; + (void) RelUndoRecordSingleDataPage(&header, payload, &batch_blkno); + (void) RelUndoForkTrack(fork_blks, &nfork, RelUndoGetBlockNum(current_ptr)); + + current_ptr = header.urec_prevundorec; + if (payload) + { + pfree(payload); + payload = NULL; + } + + /* + * Extend the batch with following records that restore the very same + * single data page. We only peek ahead while the current record was + * itself a single-page record (batch_blkno valid); a multi-page + * record leaves ntouched > 1 and ends the batch immediately. + */ + while (ntouched == 1 && + BlockNumberIsValid(batch_blkno) && + nbatch < RELUNDO_APPLY_MAX_BATCH && + RelUndoRecPtrIsValid(current_ptr)) + { + RelUndoRecordHeader peek_hdr; + void *peek_payload = NULL; + Size peek_size; + BlockNumber peek_blkno = InvalidBlockNumber; + + if (!RelUndoReadRecord(rel, current_ptr, &peek_hdr, + &peek_payload, &peek_size)) + break; + + if (peek_hdr.info_flags & RELUNDO_INFO_CLR_APPLIED) + { + /* Stop the batch; the skip is handled by the outer loop. */ + if (peek_payload) + pfree(peek_payload); + break; + } + + if (!RelUndoRecordSingleDataPage(&peek_hdr, peek_payload, &peek_blkno) || + peek_blkno != batch_blkno) + { + /* Not foldable: leave current_ptr for the outer loop. */ + if (peek_payload) + pfree(peek_payload); + break; + } + + /* + * The CLR will register one forced image per distinct fork page + * in the batch. If folding this record would exceed the + * fork-page cap (and its fork page is not already in the batch), + * stop here. + */ + if (nfork >= RELUNDO_APPLY_MAX_FORK_BUFS) + { + BlockNumber peek_fork = RelUndoGetBlockNum(current_ptr); + bool seen = false; + + for (i = 0; i < nfork; i++) + { + if (fork_blks[i] == peek_fork) + { + seen = true; + break; + } + } + if (!seen) + { + if (peek_payload) + pfree(peek_payload); + break; + } + } + + /* Foldable: apply onto the already-locked data page. */ + RelUndoApplyOneRecord(rel, &peek_hdr, peek_payload, current_ptr, + touched, &ntouched); + batch_ptrs[nbatch++] = current_ptr; + (void) RelUndoForkTrack(fork_blks, &nfork, + RelUndoGetBlockNum(current_ptr)); + + current_ptr = peek_hdr.urec_prevundorec; + if (peek_payload) + pfree(peek_payload); + } + + /* + * Log one redoable CLR carrying full-page images of every restored + * data page plus the fork page(s) for the batched records, then + * release the data buffers. For non-WAL relations there is nothing + * to log; just mark dirty and release. + */ + if (RelationNeedsWAL(rel)) + RelUndoLogApplyCLR(rel, batch_ptrs, nbatch, touched, ntouched); + else + { + for (i = 0; i < ntouched; i++) + { + MarkBufferDirty(touched[i]); + UnlockReleaseBuffer(touched[i]); + } + } + } + + elog(DEBUG1, "RelUndoApplyChain: rollback complete"); +} + +/* + * RelUndoApplyRecordForRecovery - Reverse-apply one UNDO record during crash + * recovery, without writing a compensation log record (CLR). + * + * Crash recovery rolls back loser transactions after the redo pass. At that + * point WAL insertion is not yet permitted (LocalSetXLogInsertAllowed() runs + * later in StartupXLOG), so this path mirrors the non-WAL branch of + * RelUndoApplyChain: it restores the before-image into the data page(s) in + * memory and marks them dirty, relying on the end-of-recovery checkpoint for + * durability. Re-application is harmless because redo always re-establishes + * the post-modification page state before this reverse-apply runs, so a crash + * mid-recovery simply replays redo and re-applies the before-image. + * + * Records already carrying RELUNDO_INFO_CLR_APPLIED (rolled back online before + * the crash) are skipped. The caller is responsible for driving this in + * newest-first order for each loser transaction's tracked record pointers. + * + * Idempotency across a second crash that brackets the end-of-recovery + * checkpoint: absolute records (full-tuple UPDATE/DELETE, INSERT) restore a + * fixed before-image, so re-applying them is a no-op. A DELTA record + * reconstructs the before-image relative to the live page tuple, which is NOT + * idempotent in isolation; the write path guarantees safety by emitting DELTA + * only when overwriting a COMMITTED version, so at most one DELTA exists per + * tid and every newer record for that tid is absolute. Newest-first replay + * therefore re-derives the DELTA's page anchor before it is reverse-applied, + * and a lone DELTA reverse-apply (overwrite-with-old-bytes) is idempotent. + */ +void +RelUndoApplyRecordForRecovery(Relation rel, RelUndoRecPtr ptr) +{ + RelUndoRecordHeader header; + void *payload = NULL; + Size payload_size; + Buffer touched[RELUNDO_APPLY_MAX_DATA_BUFS]; + int ntouched = 0; + int i; + + if (!RelUndoRecPtrIsValid(ptr)) + return; + + if (!RelUndoReadRecord(rel, ptr, &header, &payload, &payload_size)) + { + elog(WARNING, "RelUndoApplyRecordForRecovery: could not read UNDO record at %lu", + (unsigned long) ptr); + return; + } + + /* Already rolled back online before the crash: nothing to do. */ + if (header.info_flags & RELUNDO_INFO_CLR_APPLIED) + { + if (payload) + pfree(payload); + return; + } + + RelUndoApplyOneRecord(rel, &header, payload, ptr, touched, &ntouched); + + for (i = 0; i < ntouched; i++) + { + MarkBufferDirty(touched[i]); + UnlockReleaseBuffer(touched[i]); + } + + if (payload) + pfree(payload); +} + +/* + * RelUndoRecordSingleDataPage - Classify whether an UNDO record restores + * exactly one data page, and if so report that block number. + * + * Returns true and sets *blk when the record's restoration touches a single + * data page (in-place UPDATE, single-TID DELETE, INSERT, TUPLE_LOCK). Returns + * false for records that may touch two pages (out-of-place UPDATE where oldtid + * != newtid) or more than one TID (multi-TID DELETE); those are not eligible to + * be folded into a same-page batch. No buffers are touched. + */ +static bool +RelUndoRecordSingleDataPage(const RelUndoRecordHeader *header, + const void *payload, BlockNumber *blk) +{ + *blk = InvalidBlockNumber; + + switch (header->urec_type) + { + case RELUNDO_INSERT: + { + const RelUndoInsertPayload *p = payload; + + *blk = ItemPointerGetBlockNumber(&p->firsttid); + return true; + } + + case RELUNDO_DELETE: + { + const RelUndoDeletePayload *p = payload; + + if (p->ntids != 1) + return false; + *blk = ItemPointerGetBlockNumber(&p->tids[0]); + return true; + } + + case RELUNDO_UPDATE: + { + const RelUndoUpdatePayload *p = payload; + + if (!ItemPointerEquals(&p->oldtid, &p->newtid)) + return false; + *blk = ItemPointerGetBlockNumber(&p->oldtid); + return true; + } + + case RELUNDO_TUPLE_LOCK: + { + const RelUndoTupleLockPayload *p = payload; + + *blk = ItemPointerGetBlockNumber(&p->tid); + return true; + } + + default: + return false; + } +} + +/* + * RelUndoForkTrack - Add an UNDO-fork block to the batch's distinct-fork set. + * + * Returns true if blkno was newly added, false if it was already present or the + * set is full (caller has already guaranteed room via the cap check). + */ +static bool +RelUndoForkTrack(BlockNumber *fork_blks, int *nfork, BlockNumber blkno) +{ + int i; + + for (i = 0; i < *nfork; i++) + { + if (fork_blks[i] == blkno) + return false; + } + + Assert(*nfork < RELUNDO_APPLY_MAX_FORK_BUFS); + fork_blks[(*nfork)++] = blkno; + return true; +} + +/* + * RelUndoApplyOneRecord - Apply one UNDO record's physical restoration. + * + * Pins+locks the record's data page(s) into touched[] (via RelUndoTrackPage, + * which deduplicates against pages already locked for this batch) and mutates + * them in memory. Does NOT log or release buffers; the caller batches one or + * more applied records and emits a single CLR. May ereport(ERROR) on + * corruption, before any WAL is written. + */ +static void +RelUndoApplyOneRecord(Relation rel, const RelUndoRecordHeader *header, + void *payload, RelUndoRecPtr current_ptr, + Buffer *touched, int *ntouched) +{ + Page page; + BlockNumber target_blkno; + OffsetNumber target_offset; + int i; + + switch (header->urec_type) + { + case RELUNDO_INSERT: + { + RelUndoInsertPayload *ins_payload = (RelUndoInsertPayload *) payload; + + target_blkno = ItemPointerGetBlockNumber(&ins_payload->firsttid); + target_offset = ItemPointerGetOffsetNumber(&ins_payload->firsttid); + + page = RelUndoTrackPage(rel, touched, ntouched, target_blkno); + RelUndoApplyInsert(rel, page, target_offset); + break; + } + + case RELUNDO_DELETE: + { + RelUndoDeletePayload *del_payload = (RelUndoDeletePayload *) payload; + char *tuple_data_buf = NULL; + uint32 tlen = 0; + + RelUndoReadRecordWithTuple(rel, current_ptr, + &tuple_data_buf, &tlen); + + for (i = 0; i < del_payload->ntids; i++) + { + target_blkno = ItemPointerGetBlockNumber(&del_payload->tids[i]); + target_offset = ItemPointerGetOffsetNumber(&del_payload->tids[i]); + + page = RelUndoTrackPage(rel, touched, ntouched, target_blkno); + + if (tuple_data_buf && tlen > 0) + RelUndoApplyDelete(rel, page, target_offset, + tuple_data_buf, tlen); + } + + if (tuple_data_buf) + pfree(tuple_data_buf); + break; + } + + case RELUNDO_UPDATE: + { + RelUndoUpdatePayload *upd_payload = (RelUndoUpdatePayload *) payload; + char *tuple_data_buf = NULL; + uint32 tlen = 0; + + RelUndoReadRecordWithTuple(rel, current_ptr, + &tuple_data_buf, &tlen); + + /* Restore old tuple at the old location */ + target_blkno = ItemPointerGetBlockNumber(&upd_payload->oldtid); + target_offset = ItemPointerGetOffsetNumber(&upd_payload->oldtid); + + page = RelUndoTrackPage(rel, touched, ntouched, target_blkno); + + if (tuple_data_buf && tlen > 0) + RelUndoApplyUpdate(rel, page, target_offset, + tuple_data_buf, tlen); + + if (tuple_data_buf) + pfree(tuple_data_buf); + + /* + * Mark the new tuple version as unused, but only for + * out-of-place updates where oldtid != newtid. For in-place + * updates the old and new tuple share the same slot, so + * marking it unused would destroy the just-restored old + * tuple. + */ + if (!ItemPointerEquals(&upd_payload->oldtid, + &upd_payload->newtid)) + { + BlockNumber new_blkno; + OffsetNumber new_offset; + Page new_page; + + new_blkno = ItemPointerGetBlockNumber(&upd_payload->newtid); + new_offset = ItemPointerGetOffsetNumber(&upd_payload->newtid); + + new_page = RelUndoTrackPage(rel, touched, ntouched, new_blkno); + RelUndoApplyInsert(rel, new_page, new_offset); + } + break; + } + + case RELUNDO_TUPLE_LOCK: + { + RelUndoTupleLockPayload *lock_payload = (RelUndoTupleLockPayload *) payload; + + target_blkno = ItemPointerGetBlockNumber(&lock_payload->tid); + target_offset = ItemPointerGetOffsetNumber(&lock_payload->tid); + + page = RelUndoTrackPage(rel, touched, ntouched, target_blkno); + RelUndoApplyTupleLock(rel, page, target_offset); + break; + } + + default: + /* Release any tracked buffers before erroring out. */ + for (i = 0; i < *ntouched; i++) + UnlockReleaseBuffer(touched[i]); + *ntouched = 0; + elog(ERROR, "RelUndoApplyChain: unknown UNDO record type %d", + header->urec_type); + } +} + +/* + * RelUndoTrackPage - Read+pin+exclusive-lock a data page for the current + * apply, deduplicating against pages already locked for this record. + * + * Returns the page so the caller can mutate it. The buffer is added to the + * touched[] array (kept locked) the first time a block is requested; a repeat + * request for the same block returns the already-locked page. All tracked + * buffers are released by RelUndoLogApplyCLR after the CLR is logged. + */ +static Page +RelUndoTrackPage(Relation rel, Buffer *touched, int *ntouched, + BlockNumber blkno) +{ + Buffer buf; + int i; + + for (i = 0; i < *ntouched; i++) + { + if (BufferGetBlockNumber(touched[i]) == blkno) + return BufferGetPage(touched[i]); + } + + if (*ntouched >= RELUNDO_APPLY_MAX_DATA_BUFS) + { + /* Release locks before erroring so the worker can clean up. */ + for (i = 0; i < *ntouched; i++) + UnlockReleaseBuffer(touched[i]); + *ntouched = 0; + elog(ERROR, "RelUndoApplyChain: too many data pages (%d) for one UNDO record", + RELUNDO_APPLY_MAX_DATA_BUFS + 1); + } + + buf = ReadBuffer(rel, blkno); + LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE); + touched[*ntouched] = buf; + (*ntouched)++; + return BufferGetPage(buf); +} + +/* + * RelUndoApplyInsert - Undo an INSERT operation + * + * Mark the inserted tuple as dead/unused. For INSERT, we don't need the + * original tuple data - we just mark the slot as available. + */ +static void +RelUndoApplyInsert(Relation rel, Page page, OffsetNumber offset) +{ + ItemId lp; + + elog(DEBUG1, "RelUndoApplyInsert: page=%p, offset=%u", page, offset); + + /* Validate offset */ + if (offset == InvalidOffsetNumber || offset > PageGetMaxOffsetNumber(page)) + elog(ERROR, "RelUndoApplyInsert: invalid offset %u (max=%u)", + offset, PageGetMaxOffsetNumber(page)); + + elog(DEBUG1, "RelUndoApplyInsert: calling PageGetItemId"); + lp = PageGetItemId(page, offset); + + elog(DEBUG1, "RelUndoApplyInsert: got ItemId %p", lp); + + if (!ItemIdIsNormal(lp)) + elog(WARNING, "RelUndoApplyInsert: tuple at offset %u is not normal", offset); + + /* Mark the line pointer as unused (LP_UNUSED) */ + elog(DEBUG1, "RelUndoApplyInsert: calling ItemIdSetUnused"); + ItemIdSetUnused(lp); + + elog(DEBUG1, "RelUndoApplyInsert: marked tuple at offset %u as unused", offset); +} + +/* + * RelUndoApplyDelete - Undo a DELETE operation + * + * Restore the deleted tuple from the UNDO record. The tuple data is stored + * in the UNDO record and includes the full tuple (header + data). + */ +static void +RelUndoApplyDelete(Relation rel, Page page, OffsetNumber offset, + char *tuple_data, uint32 tuple_len) +{ + ItemId lp; + Size aligned_len; + + /* Validate inputs */ + if (tuple_data == NULL || tuple_len == 0) + elog(ERROR, "RelUndoApplyDelete: invalid tuple data"); + + if (offset == InvalidOffsetNumber || offset > PageGetMaxOffsetNumber(page)) + elog(ERROR, "RelUndoApplyDelete: invalid offset %u", offset); + + lp = PageGetItemId(page, offset); + + aligned_len = MAXALIGN(tuple_len); + + /* + * Restore the tuple data. We use memcpy to copy the complete tuple + * including the header. + */ + if (ItemIdIsUsed(lp)) + { + /* + * Tuple slot is still occupied -- the common case for an in-place AM + * whose DELETE is in place (it only flags the tuple deleted and + * leaves the full-length body on the page). Restoring is an in-place + * overwrite of the same-length slot, so it needs no free space; do + * not consult PageGetFreeSpace here or a full page would spuriously + * fail the rollback. + */ + if (ItemIdGetLength(lp) != tuple_len) + elog(ERROR, "RelUndoApplyDelete: tuple length mismatch"); + + memcpy(PageGetItem(page, lp), tuple_data, tuple_len); + } + else + { + /* Need to allocate a new slot -- this path consumes free space. */ + OffsetNumber new_offset; + + if (PageGetFreeSpace(page) < aligned_len) + elog(ERROR, "RelUndoApplyDelete: insufficient space on page to restore tuple"); + + new_offset = PageAddItem(page, tuple_data, tuple_len, + offset, false, false); + if (new_offset != offset) + elog(ERROR, "RelUndoApplyDelete: could not restore tuple at expected offset"); + } + + /* + * Clear transient flags. The restored tuple is the committed + * before-image of the DELETE, so it should not be marked deleted or + * uncommitted. + */ + if (RelUndoClearTransientFlags_hook) + { + lp = PageGetItemId(page, offset); + RelUndoClearTransientFlags_hook((char *) PageGetItem(page, lp)); + } + + elog(DEBUG2, "RelUndoApplyDelete: restored tuple at offset %u (%u bytes)", + offset, tuple_len); +} + +/* + * RelUndoApplyUpdate - Undo an UPDATE operation + * + * Restore the old tuple version from the UNDO record. The tuple data was + * stored in the UNDO record and includes the full tuple (header + data). + * + * For an in-place update, the old tuple was physically overwritten at + * the same offset. We restore it by copying the saved data back. + */ +static void +RelUndoApplyUpdate(Relation rel, Page page, OffsetNumber offset, + char *tuple_data, uint32 tuple_len) +{ + ItemId lp; + + /* Validate inputs */ + if (tuple_data == NULL || tuple_len == 0) + elog(ERROR, "RelUndoApplyUpdate: invalid tuple data"); + + if (offset == InvalidOffsetNumber || offset > PageGetMaxOffsetNumber(page)) + elog(ERROR, "RelUndoApplyUpdate: invalid offset %u", offset); + + lp = PageGetItemId(page, offset); + + if (!ItemIdIsNormal(lp)) + elog(ERROR, "RelUndoApplyUpdate: tuple at offset %u is not normal", offset); + + /* + * Restore the old tuple. Handle size differences between the new tuple + * (currently on page) and the old tuple (from UNDO). + */ + if (tuple_len <= ItemIdGetLength(lp)) + { + /* + * Old tuple is same size or smaller than the new one. Simply + * overwrite in place and adjust the length. + */ + memcpy(PageGetItem(page, lp), tuple_data, tuple_len); + if (tuple_len != ItemIdGetLength(lp)) + ItemIdSetNormal(lp, ItemIdGetOffset(lp), tuple_len); + } + else + { + /* + * Old tuple is larger than the new one. Delete the current item and + * re-add the old tuple at the same offset. + */ + OffsetNumber restored_offset; + + PageIndexTupleDelete(page, offset); + restored_offset = PageAddItem(page, tuple_data, + tuple_len, offset, false, false); + if (restored_offset == InvalidOffsetNumber) + { + /* + * Try without specifying a target offset. The page should have + * enough free space since we just removed the (smaller) new + * tuple. + */ + restored_offset = PageAddItem(page, tuple_data, + tuple_len, InvalidOffsetNumber, + false, false); + } + + if (restored_offset == InvalidOffsetNumber) + elog(ERROR, "RelUndoApplyUpdate: could not restore old tuple at offset %u (%u bytes)", + offset, tuple_len); + } + + /* + * Clear transient flags on the restored tuple. The UNDO record stores + * the before-image which was committed, so UNCOMMITTED should not be set. + * Clear it defensively in case lazy clearing hadn't run before the + * snapshot was taken, and also clear DELETED/UPDATED since the operation + * that set them is being rolled back. + */ + if (RelUndoClearTransientFlags_hook) + { + lp = PageGetItemId(page, offset); + RelUndoClearTransientFlags_hook((char *) PageGetItem(page, lp)); + } + + elog(DEBUG2, "RelUndoApplyUpdate: restored old tuple at offset %u (%u bytes)", + offset, tuple_len); +} + +/* + * RelUndoApplyTupleLock - Undo a tuple lock operation + * + * Remove the lock marker from the tuple by clearing the lock-related + * infomask bits and resetting xmax to InvalidTransactionId. + */ +static void +RelUndoApplyTupleLock(Relation rel, Page page, OffsetNumber offset) +{ + ItemId lp; + HeapTupleHeader htup; + + /* Validate offset */ + if (offset == InvalidOffsetNumber || offset > PageGetMaxOffsetNumber(page)) + elog(ERROR, "RelUndoApplyTupleLock: invalid offset %u", offset); + + lp = PageGetItemId(page, offset); + + if (!ItemIdIsNormal(lp)) + elog(ERROR, "RelUndoApplyTupleLock: tuple at offset %u is not normal", offset); + + htup = (HeapTupleHeader) PageGetItem(page, lp); + + /* Clear lock-related infomask bits */ + htup->t_infomask &= ~(HEAP_XMAX_LOCK_ONLY | + HEAP_XMAX_KEYSHR_LOCK | + HEAP_XMAX_SHR_LOCK | + HEAP_XMAX_EXCL_LOCK); + htup->t_infomask2 &= ~HEAP_KEYS_UPDATED; + + /* Reset xmax to invalid */ + HeapTupleHeaderSetXmax(htup, InvalidTransactionId); + + elog(DEBUG2, "RelUndoApplyTupleLock: cleared lock from tuple at offset %u", offset); +} + +/* + * RelUndoLogApplyCLR - Write a redoable Compensation Log Record for a batch + * + * Logs a single XLOG_RELUNDO_APPLY record covering one or more consecutive + * UNDO records (urec_ptrs[0..nptrs-1]) that were all applied to the same set of + * data pages (passed in touched[], already mutated and held exclusively + * locked). Each record's RELUNDO_INFO_CLR_APPLIED flag is set in place on its + * UNDO-fork page; the distinct fork pages are logged alongside the data pages. + * + * Because an in-place MVCC AM has no durable xmin, the rollback's + * physical page changes are NOT reconstructable from any forward WAL record; + * the forward record holds the *new* (aborted) value. We therefore force a + * full-page image of each restored page so crash redo reinstates the + * before-image. The CLR_APPLIED flag on each fork page makes a re-driven + * RelUndoApplyChain idempotent (it skips already-applied records), preventing + * double-application after a crash during rollback. + * + * relundo_redo_apply restores every registered block image and ignores the + * record body, so a batched CLR replays identically to a sequence of + * single-record CLRs -- only the WAL volume (one forced fork-page and data-page + * image instead of N) and the flush count (one instead of N) shrink. + * + * All data buffers in touched[] are released here after logging. + */ +static void +RelUndoLogApplyCLR(Relation rel, const RelUndoRecPtr *urec_ptrs, int nptrs, + Buffer *touched, int ntouched) +{ + xl_relundo_apply xlrec; + BlockNumber fork_blks[RELUNDO_APPLY_MAX_FORK_BUFS]; + Buffer fork_bufs[RELUNDO_APPLY_MAX_FORK_BUFS]; + int nfork = 0; + XLogRecPtr recptr; + uint8 block_id; + int i; + int j; + + Assert(nptrs >= 1); + Assert(ntouched >= 1); + + /* + * Collect the distinct fork pages this batch touches, in ascending block + * order so concurrent rollback appliers acquire fork-page locks in a + * consistent order (deadlock-free). Data pages were already locked by + * the apply path before any fork page, so the global order is + * data-then-fork. + */ + for (i = 0; i < nptrs; i++) + { + BlockNumber blk = RelUndoGetBlockNum(urec_ptrs[i]); + int pos; + + for (pos = 0; pos < nfork; pos++) + { + if (fork_blks[pos] == blk) + break; + } + if (pos < nfork) + continue; /* already collected */ + + Assert(nfork < RELUNDO_APPLY_MAX_FORK_BUFS); + /* insertion sort into ascending order */ + for (pos = nfork; pos > 0 && fork_blks[pos - 1] > blk; pos--) + fork_blks[pos] = fork_blks[pos - 1]; + fork_blks[pos] = blk; + nfork++; + } + + xlrec.urec_ptr = urec_ptrs[0]; + xlrec.target_reloc = rel->rd_locator; + + /* + * Read and exclusive-lock each distinct fork page (ascending), before the + * critical section since ReadBuffer may perform I/O. + */ + for (i = 0; i < nfork; i++) + { + fork_bufs[i] = ReadBufferExtended(rel, RELUNDO_FORKNUM, fork_blks[i], + RBM_NORMAL, NULL); + LockBuffer(fork_bufs[i], BUFFER_LOCK_EXCLUSIVE); + } + + START_CRIT_SECTION(); + + /* + * Set the CLR flags in place on every batched record's fork-page header. + */ + for (i = 0; i < nptrs; i++) + { + BlockNumber blk = RelUndoGetBlockNum(urec_ptrs[i]); + uint16 off = RelUndoGetOffset(urec_ptrs[i]); + char *contents; + RelUndoRecordHeader *rec_hdr; + + for (j = 0; j < nfork; j++) + { + if (fork_blks[j] == blk) + break; + } + Assert(j < nfork); + + contents = PageGetContents(BufferGetPage(fork_bufs[j])); + rec_hdr = (RelUndoRecordHeader *) (contents + off); + rec_hdr->info_flags |= RELUNDO_INFO_CLR_APPLIED; + } + + for (i = 0; i < ntouched; i++) + MarkBufferDirty(touched[i]); + for (i = 0; i < nfork; i++) + MarkBufferDirty(fork_bufs[i]); + + XLogBeginInsert(); + XLogRegisterData((char *) &xlrec, sizeof(xl_relundo_apply)); + + /* + * Force a full-page image of every restored data page then every fork + * page. The restoration is an arbitrary in-place rewrite, so only an FPI + * can reproduce it during redo. + */ + block_id = 0; + for (i = 0; i < ntouched; i++) + XLogRegisterBuffer(block_id++, touched[i], + REGBUF_STANDARD | REGBUF_FORCE_IMAGE); + for (i = 0; i < nfork; i++) + XLogRegisterBuffer(block_id++, fork_bufs[i], + REGBUF_STANDARD | REGBUF_FORCE_IMAGE); + + recptr = XLogInsert(RM_RELUNDO_ID, XLOG_RELUNDO_APPLY); + + for (i = 0; i < ntouched; i++) + PageSetLSN(BufferGetPage(touched[i]), recptr); + for (i = 0; i < nfork; i++) + PageSetLSN(BufferGetPage(fork_bufs[i]), recptr); + + END_CRIT_SECTION(); + + /* + * Force the CLR to durable storage before releasing the buffers. Unlike + * heap, an in-place MVCC AM has no durable xmin/clog, so recovery cannot + * re-drive the per-relation fork undo for a loser transaction: the + * before-image we just restored exists only in these (still dirty) + * buffers and in this CLR. If we crash before the CLR is flushed, redo + * replays only the forward (aborted) page change and the rollback is + * silently lost. Flushing here makes the compensation durable so crash + * redo reinstates the before-image from the forced full-page images. + * Batching amortizes this flush across every record folded into the CLR. + */ + XLogFlush(recptr); + + elog(DEBUG3, "RelUndoLogApplyCLR: CLR for %d UNDO record(s), %d data page(s), %d fork page(s)", + nptrs, ntouched, nfork); + + for (i = 0; i < nfork; i++) + UnlockReleaseBuffer(fork_bufs[i]); + for (i = 0; i < ntouched; i++) + UnlockReleaseBuffer(touched[i]); +} + +/* + * RelUndoReadRecordWithTuple - Read UNDO record including tuple data + * + * This is like RelUndoReadRecord but also reads the tuple data that follows + * the payload if RELUNDO_INFO_HAS_TUPLE is set. + */ +RelUndoRecordHeader * +RelUndoReadRecordWithTuple(Relation rel, RelUndoRecPtr ptr, + char **tuple_data_out, uint32 *tuple_len_out) +{ + RelUndoRecordHeader header_local; + RelUndoRecordHeader *header; + void *payload; + Size payload_size; + bool success; + + /* Initialize outputs */ + *tuple_data_out = NULL; + *tuple_len_out = 0; + + /* Read the basic record (header + payload, no tuple data) */ + success = RelUndoReadRecord(rel, ptr, &header_local, &payload, &payload_size); + if (!success) + return NULL; + + /* + * Allocate combined buffer for header + payload. Tuple data will be + * allocated separately if present. + */ + header = (RelUndoRecordHeader *) palloc(SizeOfRelUndoRecordHeader + payload_size); + memcpy(header, &header_local, SizeOfRelUndoRecordHeader); + memcpy((char *) header + SizeOfRelUndoRecordHeader, payload, payload_size); + + /* Free the payload allocated by RelUndoReadRecord */ + pfree(payload); + + /* + * If tuple data is present, extract it from the combined payload. + * + * RelUndoReadRecord reads (urec_len - SizeOfRelUndoRecordHeader) bytes as + * "payload", which includes both the actual payload and the tuple data. + * The tuple data occupies the last tuple_len bytes of that region. + */ + if ((header->info_flags & RELUNDO_INFO_HAS_TUPLE) && header->tuple_len > 0) + { + uint32 tlen = header->tuple_len; + Size actual_payload_size; + + /* + * payload_size from RelUndoReadRecord includes both the real payload + * and the tuple data. The actual payload is the first part. + */ + if (payload_size < tlen) + { + elog(WARNING, "RelUndoReadRecordWithTuple: tuple_len %u exceeds payload_size %zu", + tlen, payload_size); + return header; + } + + actual_payload_size = payload_size - tlen; + + /* + * Allocate and copy the tuple data from the tail of the combined + * buffer + */ + *tuple_data_out = (char *) palloc(tlen); + memcpy(*tuple_data_out, + (char *) header + SizeOfRelUndoRecordHeader + actual_payload_size, + tlen); + *tuple_len_out = tlen; + + elog(DEBUG2, "RelUndoReadRecordWithTuple: read %u bytes of tuple data", tlen); + } + + return header; +} diff --git a/src/backend/access/undo/relundo_discard.c b/src/backend/access/undo/relundo_discard.c new file mode 100644 index 0000000000000..0d92ac2ce2b6b --- /dev/null +++ b/src/backend/access/undo/relundo_discard.c @@ -0,0 +1,536 @@ +/*------------------------------------------------------------------------- + * + * relundo_discard.c + * Per-relation UNDO discard and space reclamation + * + * This file implements the counter-based discard logic for per-relation UNDO. + * During VACUUM, old UNDO records are discarded and their pages reclaimed + * to the free list for reuse. + * + * Discard walks the page chain from the tail (oldest) toward the head + * (newest). Each page's generation counter is compared against the + * oldest-visible cutoff using modular 16-bit arithmetic. If a page's + * counter precedes the cutoff, all records on that page are safe to + * discard and the page is moved to the free list. + * + * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/access/undo/relundo_discard.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/relundo.h" +#include "access/relundo_xlog.h" +#include "access/transam.h" +#include "access/xlog.h" +#include "access/xloginsert.h" +#include "common/relpath.h" +#include "miscadmin.h" +#include "storage/bufmgr.h" +#include "storage/bufpage.h" +#include "storage/lmgr.h" +#include "storage/proc.h" +#include "storage/smgr.h" +#include "utils/rel.h" + +static void RelUndoTruncateEmptyChain(Relation rel, Buffer metabuf); +static void RelUndoDiscardSlot(Relation rel, Buffer metabuf, int slot, + TransactionId oldest_xmin); + +/* + * relundo_page_is_discardable + * Check if every record on a page is older than the discard horizon. + * + * A page is discardable iff its max_xid (the largest urec_xid of any record + * on the page) precedes oldest_xmin. In that case no active transaction can + * still need any record on the page for rollback, so the page can be freed. + * + * An empty page (max_xid == InvalidTransactionId) carries no live records and + * is trivially discardable. + */ +static bool +relundo_page_is_discardable(Page page, TransactionId oldest_xmin) +{ + RelUndoPageHeader hdr; + + hdr = (RelUndoPageHeader) PageGetContents(page); + + if (!TransactionIdIsValid(hdr->max_xid)) + return true; + + return TransactionIdPrecedes(hdr->max_xid, oldest_xmin); +} + +/* + * RelUndoDiscard + * Discard old UNDO records and reclaim space across all head slots. + * + * Each of the RELUNDO_NUM_HEADS append chains is independently append-only, + * so discardability is monotonic tail->head within each slot. We walk and + * splice each slot's chain separately under the single metapage exclusive + * lock, then physically truncate the fork only if every slot ended empty. + */ +void +RelUndoDiscard(Relation rel, TransactionId oldest_xmin, bool nowait) +{ + Buffer metabuf; + Page metapage; + RelUndoMetaPage meta; + bool all_empty; + + /* + * Lock the metapage exclusively for the duration of discard. When nowait + * is set (the inline hot-path caller, RelUndoMaybeVacuum), acquire the + * lock CONDITIONALLY: if another backend holds it, skip this discard + * entirely rather than block the update. This keeps concurrent updates + * to the same relation from serializing behind the single per-relation + * UNDO metapage lock. Space reclamation is best-effort; a skipped + * discard is retried on a later update. + */ + if (nowait) + { + /* + * Pin the metapage (block 0) and take its content lock CONDITIONALLY. + * The UNDO fork is known to exist (RelUndoVacuum checked smgrexists), + * so block 0 is present and initialized. If another backend holds + * the lock, skip this discard rather than block the update. + */ + metabuf = ReadBufferExtended(rel, RELUNDO_FORKNUM, + RELUNDO_METAPAGE_BLKNO, RBM_NORMAL, NULL); + if (!ConditionalLockBuffer(metabuf)) + { + ReleaseBuffer(metabuf); + return; + } + } + else + metabuf = relundo_get_metapage(rel, BUFFER_LOCK_EXCLUSIVE); + metapage = BufferGetPage(metabuf); + meta = (RelUndoMetaPage) PageGetContents(metapage); + + /* + * A conditionally-locked metapage might (in a rare crash-recovery window) + * be uninitialized; the blocking relundo_get_metapage path reinitializes + * it, but the nowait path must not do WAL work, so just skip if invalid. + */ + if (nowait && meta->magic != RELUNDO_METAPAGE_MAGIC) + { + UnlockReleaseBuffer(metabuf); + return; + } + + for (int slot = 0; slot < RELUNDO_NUM_HEADS; slot++) + RelUndoDiscardSlot(rel, metabuf, slot, oldest_xmin); + + /* + * If every slot's chain is now empty, the free list holds every allocated + * data block -- the contiguous physical suffix [1 .. + * system_alloc_watermark] -- so the fork can be physically truncated back + * to the metapage. That is the only provably-safe truncate case, and we + * still hold the metapage exclusive lock (blocking any concurrent + * allocator). + */ + all_empty = true; + for (int slot = 0; slot < RELUNDO_NUM_HEADS; slot++) + { + if (BlockNumberIsValid(meta->tail_blkno[slot])) + { + all_empty = false; + break; + } + } + + if (all_empty) + RelUndoTruncateEmptyChain(rel, metabuf); + + /* + * Discard moved (and possibly truncated away) the pages this backend's + * head page cache may still name. Drop the cache entry so the next + * reserve re-reads the metapage instead of faulting on a stale block + * number that no longer exists on disk. + */ + RelUndoHeadCacheInvalidate(RelationGetRelid(rel)); + + UnlockReleaseBuffer(metabuf); +} + +/* + * RelUndoDiscardSlot + * Discard the tail run of one head slot's chain. + * + * Walks the slot's page chain from the head toward the tail. Any page whose + * max_xid precedes oldest_xmin holds only records that no active transaction + * can still need for rollback; such pages are unlinked from the data chain and + * spliced onto the shared free list in one WAL-logged operation. + * + * The chain is chronologically ordered (head newest, tail oldest), so the + * discardable pages form a contiguous run at the tail end. The caller holds + * metabuf pinned and exclusively locked. + */ +static void +RelUndoDiscardSlot(Relation rel, Buffer metabuf, int slot, + TransactionId oldest_xmin) +{ + Page metapage = BufferGetPage(metabuf); + RelUndoMetaPage meta = (RelUndoMetaPage) PageGetContents(metapage); + BlockNumber old_tail_blkno; + BlockNumber new_tail_blkno = InvalidBlockNumber; + BlockNumber run_head_blkno = InvalidBlockNumber; + BlockNumber current_blkno; + BlockNumber old_free_head; + uint32 npages_freed = 0; + Buffer runtail_buf; + Buffer newtail_buf = InvalidBuffer; + + old_tail_blkno = meta->tail_blkno[slot]; + old_free_head = meta->free_blkno; + + if (!BlockNumberIsValid(old_tail_blkno)) + { + /* Empty chain, nothing to discard */ + return; + } + + /* + * Pass 1 (read-only): walk from head toward tail following prev_blkno. A + * page is discardable iff its max_xid precedes oldest_xmin. The last + * (closest-to-tail) page that is NOT discardable becomes the new tail; + * every page below it forms a contiguous discardable run. run_head is + * the newest page in that run (the page just below the new tail, or the + * chain head if the whole chain is discardable). + * + * This relies on a precondition of the append-only fork: discardability + * is monotonic from tail (oldest) to head (newest). Records are appended + * in commit order, so a page's max_xid never decreases as the chain + * advances head-ward; once a page is non-discardable (its max_xid reaches + * oldest_xmin) every newer page above it is non-discardable too. Thus + * the discardable pages always form a single contiguous run at the tail, + * and keeping the closest-to-tail non-discardable page as the new tail + * never splices a still-live page onto the free list. + */ + current_blkno = meta->head_blkno[slot]; + while (BlockNumberIsValid(current_blkno) && current_blkno != RELUNDO_METAPAGE_BLKNO) + { + Buffer buf; + Page page; + RelUndoPageHeader hdr; + BlockNumber prev; + + buf = ReadBufferExtended(rel, RELUNDO_FORKNUM, current_blkno, + RBM_NORMAL, NULL); + LockBuffer(buf, BUFFER_LOCK_SHARE); + + page = BufferGetPage(buf); + hdr = (RelUndoPageHeader) PageGetContents(page); + prev = hdr->prev_blkno; + + if (!relundo_page_is_discardable(page, oldest_xmin)) + { + new_tail_blkno = current_blkno; + run_head_blkno = prev; + } + + UnlockReleaseBuffer(buf); + + /* + * Data pages are block >= 1; the metapage is block 0 and the caller + * already holds it EXCLUSIVE. A well-formed chain terminates with + * InvalidBlockNumber, never 0. Guard against a 0 (or meta-block) + * link so a malformed/uninitialized chain head cannot self-deadlock + * by re-locking the metapage. + */ + if (prev == RELUNDO_METAPAGE_BLKNO) + break; + current_blkno = prev; + } + + if (!BlockNumberIsValid(new_tail_blkno)) + { + /* Whole chain is discardable: the run starts at the chain head. */ + run_head_blkno = meta->head_blkno[slot]; + } + + if (!BlockNumberIsValid(run_head_blkno)) + { + /* Nothing below the new tail is discardable. */ + return; + } + + /* + * Pass 2 (read-only): count the pages in the discardable run, walking + * from run_head down to (and including) the old tail. + */ + current_blkno = run_head_blkno; + while (BlockNumberIsValid(current_blkno) && current_blkno != RELUNDO_METAPAGE_BLKNO) + { + Buffer buf; + Page page; + RelUndoPageHeader hdr; + BlockNumber prev; + + buf = ReadBufferExtended(rel, RELUNDO_FORKNUM, current_blkno, + RBM_NORMAL, NULL); + LockBuffer(buf, BUFFER_LOCK_SHARE); + page = BufferGetPage(buf); + hdr = (RelUndoPageHeader) PageGetContents(page); + prev = hdr->prev_blkno; + UnlockReleaseBuffer(buf); + npages_freed++; + if (prev == RELUNDO_METAPAGE_BLKNO) + break; + current_blkno = prev; + } + + Assert(npages_freed > 0); + + /* + * Splice the whole run directly onto the free list with a bounded, fully + * WAL-logged set of mutations. Both the free list and the discardable + * run are threaded through the same durable prev_blkno fields (logged at + * insert time), so the run's internal links are left untouched and only + * its boundaries change: + * + * - the run's tail (old chain tail) gets prev_blkno = old_free_head, + * appending the prior free list after the run; - the new live tail (if + * any) gets prev_blkno = InvalidBlockNumber, detaching the live chain + * from the run; - the metapage's tail and free-list head are updated. + * + * Folding reclamation into this single WAL-logged operation makes discard + * crash-safe end to end: there is no separate, unlogged deallocation step + * whose replay could leave the free list inconsistent with the metapage. + * + * Pin and exclusively lock the boundary data pages BEFORE the critical + * section so XLogRegisterBuffer sees them dirty and locked. + */ + runtail_buf = ReadBufferExtended(rel, RELUNDO_FORKNUM, old_tail_blkno, + RBM_NORMAL, NULL); + LockBuffer(runtail_buf, BUFFER_LOCK_EXCLUSIVE); + + if (BlockNumberIsValid(new_tail_blkno)) + { + newtail_buf = ReadBufferExtended(rel, RELUNDO_FORKNUM, new_tail_blkno, + RBM_NORMAL, NULL); + LockBuffer(newtail_buf, BUFFER_LOCK_EXCLUSIVE); + } + + /* Apply the in-memory mutations. */ + { + RelUndoPageHeader runtail_hdr; + + runtail_hdr = (RelUndoPageHeader) PageGetContents(BufferGetPage(runtail_buf)); + runtail_hdr->prev_blkno = old_free_head; + MarkBufferDirty(runtail_buf); + + if (BufferIsValid(newtail_buf)) + { + RelUndoPageHeader newtail_hdr; + + newtail_hdr = (RelUndoPageHeader) PageGetContents(BufferGetPage(newtail_buf)); + newtail_hdr->prev_blkno = InvalidBlockNumber; + MarkBufferDirty(newtail_buf); + + meta->tail_blkno[slot] = new_tail_blkno; + } + else + { + /* Whole chain discarded: this slot's data chain is now empty. */ + meta->head_blkno[slot] = InvalidBlockNumber; + meta->tail_blkno[slot] = InvalidBlockNumber; + } + + meta->free_blkno = run_head_blkno; + meta->discarded_records += npages_freed; /* approximate */ + MarkBufferDirty(metabuf); + } + + /* WAL-log the discard operation. */ + START_CRIT_SECTION(); + { + xl_relundo_discard xlrec; + XLogRecPtr lsn; + + xlrec.old_tail_blkno = old_tail_blkno; + xlrec.new_tail_blkno = meta->tail_blkno[slot]; + xlrec.free_head_blkno = run_head_blkno; + xlrec.old_free_head = old_free_head; + xlrec.discard_xid = oldest_xmin; + xlrec.npages_freed = npages_freed; + xlrec.slot = (uint16) slot; + + XLogBeginInsert(); + XLogRegisterData((char *) &xlrec, SizeOfRelundoDiscard); + + /* Block 0: metapage (tail + free-list head). */ + XLogRegisterBuffer(0, metabuf, REGBUF_STANDARD); + + /* + * Blocks 1 and 2 are data pages, which pin the standard PageHeader + * pd_lower at the empty value and track their real extent in the + * shadow RelUndoPageHeader. REGBUF_STANDARD would elide the whole + * contents as a free "hole", so a restored FPI would return a zeroed + * page and lose the prev_blkno chain links. Log the full page image + * (flag 0). + */ + + /* Block 1: run tail, whose prev_blkno now links to old_free_head. */ + XLogRegisterBuffer(1, runtail_buf, 0); + + /* Block 2: new live tail, whose prev_blkno is cleared (if present). */ + if (BufferIsValid(newtail_buf)) + XLogRegisterBuffer(2, newtail_buf, 0); + + lsn = XLogInsert(RM_RELUNDO_ID, XLOG_RELUNDO_DISCARD); + + /* + * Stamp the record LSN onto every page we dirtied and registered so + * the buffer manager cannot flush any of them to disk ahead of this + * WAL record (WAL-before-data). Otherwise a crash could leave a + * page's spliced prev_blkno link on disk without the matching WAL, + * corrupting the free-list / data-chain threading. + */ + PageSetLSN(metapage, lsn); + PageSetLSN(BufferGetPage(runtail_buf), lsn); + if (BufferIsValid(newtail_buf)) + PageSetLSN(BufferGetPage(newtail_buf), lsn); + } + END_CRIT_SECTION(); + + if (BufferIsValid(newtail_buf)) + UnlockReleaseBuffer(newtail_buf); + UnlockReleaseBuffer(runtail_buf); +} + +/* + * RelUndoTruncateEmptyChain + * Physically truncate an emptied UNDO fork back to the metapage. + * + * Precondition: the caller holds metabuf pinned and exclusively locked, and + * the data chain is empty (head_blkno == tail_blkno == InvalidBlockNumber) + * after a whole-chain discard. In that state the free list contains every + * data block ever allocated, i.e. the contiguous suffix [1 .. watermark], so + * the fork can be truncated to a single block (the metapage). + * + * Physical truncation drops buffers beyond the new EOF, so it must not run + * while a concurrent lock-free reserver holds a pin on (and is about to write + * to) one of those data pages. VACUUM holds only ShareUpdateExclusiveLock, + * which does not exclude the RowExclusiveLock that DML reservers hold, so we + * gate the truncate behind a *conditional* AccessExclusiveLock (mirroring + * lazy_truncate_heap()). If the lock is not immediately available, we skip + * the physical reclaim entirely: the pages remain on the free list and are + * safely recyclable on the next allocation, so nothing is lost but disk + * space until a later discard succeeds in acquiring the lock. + * + * When the lock is held, this mirrors the crash-safe WAL-logged truncate + * pattern in RelationTruncate(): set delayChkptFlags, enter the critical + * section, mutate the metapage and WAL-log it, XLogFlush, then smgrtruncate + * (which drops the now-defunct buffers), leave the critical section, and + * finally clear delayChkptFlags. Dirtying the metapage and setting the + * checkpoint-delay flags both happen inside the critical section, so a + * concurrent checkpoint can never observe a half-applied truncate. + */ +static void +RelUndoTruncateEmptyChain(Relation rel, Buffer metabuf) +{ + Page metapage = BufferGetPage(metabuf); + RelUndoMetaPage meta = (RelUndoMetaPage) PageGetContents(metapage); + SMgrRelation srel = RelationGetSmgr(rel); + ForkNumber forknum = RELUNDO_FORKNUM; + BlockNumber old_nblocks; + BlockNumber new_nblocks = 1; /* keep only the metapage (block 0) */ + + /* Caller guarantees every slot's chain was discarded. */ +#ifdef USE_ASSERT_CHECKING + for (int slot = 0; slot < RELUNDO_NUM_HEADS; slot++) + { + Assert(!BlockNumberIsValid(meta->head_blkno[slot])); + Assert(!BlockNumberIsValid(meta->tail_blkno[slot])); + } +#endif + + old_nblocks = smgrnblocks(srel, RELUNDO_FORKNUM); + + /* Nothing to reclaim if the fork is already just the metapage. */ + if (old_nblocks <= new_nblocks) + return; + + /* + * With the whole chain discarded, the free list holds every data block + * the metapage durably knows about -- the contiguous suffix [1 .. + * watermark]. The watermark is advanced in the same WAL-logged metapage + * mutation that threads a freshly extended block onto the chain, but + * ExtendBufferedRel grows the file before that record is flushed. A + * crash in that window leaves orphaned tail blocks (watermark < nblocks - + * 1) that belong to no chain and no free list. Truncating to the + * metapage drops them too, which is safe under the AccessExclusiveLock + * gate below since no reserver can hold a pin. The watermark can never + * exceed the physical EOF, so assert only that bound. It may also be + * invalid if the only blocks on disk are orphans from a torn extend whose + * metapage update never reached WAL. + */ + Assert(!BlockNumberIsValid(meta->system_alloc_watermark) || + meta->system_alloc_watermark <= old_nblocks - 1); + + /* + * Gate the physical truncate behind a conditional AccessExclusiveLock so + * we never drop a buffer that a concurrent lock-free reserver has pinned. + * If the lock is unavailable, leave the pages on the free list (still + * recyclable) and reclaim the disk on a future discard. + */ + if (!ConditionalLockRelation(rel, AccessExclusiveLock)) + return; + + Assert((MyProc->delayChkptFlags & + (DELAY_CHKPT_START | DELAY_CHKPT_COMPLETE)) == 0); + + START_CRIT_SECTION(); + { + xl_relundo_truncate xlrec; + XLogRecPtr lsn; + + MyProc->delayChkptFlags |= DELAY_CHKPT_START | DELAY_CHKPT_COMPLETE; + + /* + * Reset metapage to the empty-fork state: the data chain is already + * empty (head/tail cleared by the discard above) and the freed pages + * cease to exist. Reset all four pointers here too so the truncate + * WAL record is self-describing and its redo does not depend on the + * preceding discard record having been flushed. + */ + for (int slot = 0; slot < RELUNDO_NUM_HEADS; slot++) + { + meta->head_blkno[slot] = InvalidBlockNumber; + meta->tail_blkno[slot] = InvalidBlockNumber; + } + meta->free_blkno = InvalidBlockNumber; + meta->system_alloc_watermark = InvalidBlockNumber; + MarkBufferDirty(metabuf); + + xlrec.new_nblocks = new_nblocks; + + XLogBeginInsert(); + XLogRegisterData((char *) &xlrec, SizeOfRelundoTruncate); + + /* Block 0: metapage (free-list head + watermark reset). */ + XLogRegisterBuffer(0, metabuf, REGBUF_STANDARD); + + lsn = XLogInsert(RM_RELUNDO_ID, XLOG_RELUNDO_TRUNCATE); + PageSetLSN(metapage, lsn); + + /* + * Flush so the truncation cannot reach disk before its WAL record; + * smgrtruncate drops the to-be-removed buffers and shrinks the file. + */ + XLogFlush(lsn); + + smgrtruncate(srel, &forknum, 1, &old_nblocks, &new_nblocks); + } + END_CRIT_SECTION(); + + MyProc->delayChkptFlags &= ~(DELAY_CHKPT_START | DELAY_CHKPT_COMPLETE); + + UnlockRelation(rel, AccessExclusiveLock); +} diff --git a/src/backend/access/undo/relundo_page.c b/src/backend/access/undo/relundo_page.c new file mode 100644 index 0000000000000..32c49eb617408 --- /dev/null +++ b/src/backend/access/undo/relundo_page.c @@ -0,0 +1,360 @@ +/*------------------------------------------------------------------------- + * + * relundo_page.c + * Per-relation UNDO page management + * + * This file handles UNDO page allocation, metapage management, and chain + * traversal for per-relation UNDO logs. + * + * The UNDO fork layout is: + * Block 0: Metapage (standard PageHeaderData + RelUndoMetaPageData) + * Block 1+: Data pages (standard PageHeaderData + RelUndoPageHeaderData + records) + * + * Data pages grow from the bottom up: pd_lower advances as records are + * appended. All offsets in RelUndoPageHeaderData are relative to the + * start of the page contents area (after standard PageHeaderData). + * + * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/access/undo/relundo_page.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/relundo.h" +#include "common/relpath.h" +#include "storage/bufmgr.h" +#include "storage/bufpage.h" +#include "storage/smgr.h" + +/* + * relundo_get_metapage + * Read and pin the metapage for a relation's UNDO fork. + * + * The caller specifies the lock mode (BUFFER_LOCK_SHARE or + * BUFFER_LOCK_EXCLUSIVE). Returns a pinned and locked buffer. + * The caller must release the buffer when done. + */ +Buffer +relundo_get_metapage(Relation rel, int mode) +{ + Buffer buf; + Page page; + RelUndoMetaPage meta; + + /* + * If the RELUNDO fork has no blocks (e.g., after crash recovery where the + * fork was created but the metapage wasn't written), create and + * initialize the metapage now. + */ + if (smgrnblocks(RelationGetSmgr(rel), RELUNDO_FORKNUM) == 0) + { + if (mode == BUFFER_LOCK_EXCLUSIVE) + { + elog(LOG, "UNDO fork for relation \"%s\" has no blocks, initializing metapage", + RelationGetRelationName(rel)); + + buf = ExtendBufferedRel(BMR_REL(rel), RELUNDO_FORKNUM, NULL, + EB_LOCK_FIRST); + Assert(BufferGetBlockNumber(buf) == 0); + + page = BufferGetPage(buf); + PageInit(page, BLCKSZ, 0); + meta = (RelUndoMetaPage) PageGetContents(page); + meta->magic = RELUNDO_METAPAGE_MAGIC; + meta->version = RELUNDO_METAPAGE_VERSION; + meta->counter = 1; + for (int s = 0; s < RELUNDO_NUM_HEADS; s++) + { + meta->head_blkno[s] = InvalidBlockNumber; + meta->tail_blkno[s] = InvalidBlockNumber; + } + meta->free_blkno = InvalidBlockNumber; + meta->total_records = 0; + meta->discarded_records = 0; + meta->system_alloc_watermark = InvalidBlockNumber; + + /* Include the meta struct in the recorded region of any FPI. */ + RelUndoMetaPageSetPdLower(page); + + MarkBufferDirty(buf); + + /* Downgrade lock if caller only wants SHARE */ + if (mode == BUFFER_LOCK_SHARE) + { + LockBuffer(buf, BUFFER_LOCK_UNLOCK); + LockBuffer(buf, BUFFER_LOCK_SHARE); + } + + return buf; + } + else + { + ereport(ERROR, + (errcode(ERRCODE_INDEX_CORRUPTED), + errmsg("UNDO fork for relation \"%s\" has no blocks", + RelationGetRelationName(rel)))); + } + } + + buf = ReadBufferExtended(rel, RELUNDO_FORKNUM, 0, RBM_NORMAL, NULL); + LockBuffer(buf, mode); + + page = BufferGetPage(buf); + meta = (RelUndoMetaPage) PageGetContents(page); + + if (meta->magic != RELUNDO_METAPAGE_MAGIC) + { + /* + * The metapage magic is invalid. This can happen after crash + * recovery if the RELUNDO fork was created but the metapage + * initialization WAL record wasn't replayed (e.g., the crash occurred + * between smgrcreate and the metapage write). + * + * Reinitialize the metapage so subsequent UNDO operations can + * proceed. This is safe because an uninitialized metapage means no + * UNDO records were ever written, so there's nothing to lose. + */ + /* + * Upgrade to exclusive lock if needed for reinitialization. + */ + if (mode != BUFFER_LOCK_EXCLUSIVE) + { + LockBuffer(buf, BUFFER_LOCK_UNLOCK); + LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE); + page = BufferGetPage(buf); + meta = (RelUndoMetaPage) PageGetContents(page); + + /* Re-check after acquiring exclusive lock */ + if (meta->magic == RELUNDO_METAPAGE_MAGIC) + { + /* Another backend fixed it while we re-locked */ + LockBuffer(buf, BUFFER_LOCK_UNLOCK); + LockBuffer(buf, mode); + return buf; + } + } + + elog(LOG, "reinitializing corrupted UNDO metapage for relation \"%s\" " + "(found magic 0x%08X, expected 0x%08X)", + RelationGetRelationName(rel), meta->magic, + RELUNDO_METAPAGE_MAGIC); + + PageInit(page, BLCKSZ, 0); + meta = (RelUndoMetaPage) PageGetContents(page); + meta->magic = RELUNDO_METAPAGE_MAGIC; + meta->version = RELUNDO_METAPAGE_VERSION; + meta->counter = 1; + for (int s = 0; s < RELUNDO_NUM_HEADS; s++) + { + meta->head_blkno[s] = InvalidBlockNumber; + meta->tail_blkno[s] = InvalidBlockNumber; + } + meta->free_blkno = InvalidBlockNumber; + meta->total_records = 0; + meta->discarded_records = 0; + meta->system_alloc_watermark = InvalidBlockNumber; + + /* Include the meta struct in the recorded region of any FPI. */ + RelUndoMetaPageSetPdLower(page); + + MarkBufferDirty(buf); + + /* Downgrade back to requested lock mode */ + if (mode != BUFFER_LOCK_EXCLUSIVE) + { + LockBuffer(buf, BUFFER_LOCK_UNLOCK); + LockBuffer(buf, mode); + } + } + + if (meta->version != RELUNDO_METAPAGE_VERSION) + ereport(ERROR, + (errcode(ERRCODE_INDEX_CORRUPTED), + errmsg("unsupported UNDO metapage version %u in relation \"%s\" (expected %u)", + meta->version, RelationGetRelationName(rel), + RELUNDO_METAPAGE_VERSION))); + + return buf; +} + +/* + * relundo_allocate_page + * Allocate a new UNDO page and add it to the head of the slot's chain. + * + * The metapage buffer must be pinned and exclusively locked by the caller. + * Returns the new block number and the pinned/exclusively-locked buffer + * via *newbuf. The metapage is updated (head_blkno[slot]) and marked dirty. + * + * slot selects which of the RELUNDO_NUM_HEADS independent append chains the + * new page joins; the free list and the generation counter are shared across + * all slots (both are protected by the same metapage exclusive lock held here). + */ +BlockNumber +relundo_allocate_page(Relation rel, Buffer metabuf, int slot, Buffer *newbuf) +{ + Page metapage; + RelUndoMetaPage meta; + BlockNumber newblkno; + BlockNumber old_head; + Buffer buf; + Page page; + + Assert(slot >= 0 && slot < RELUNDO_NUM_HEADS); + + metapage = BufferGetPage(metabuf); + meta = (RelUndoMetaPage) PageGetContents(metapage); + + old_head = meta->head_blkno[slot]; + + /* Try the free list first */ + if (BlockNumberIsValid(meta->free_blkno)) + { + Buffer freebuf; + Page freepage; + RelUndoPageHeader freehdr; + + newblkno = meta->free_blkno; + + freebuf = ReadBufferExtended(rel, RELUNDO_FORKNUM, newblkno, + RBM_NORMAL, NULL); + LockBuffer(freebuf, BUFFER_LOCK_EXCLUSIVE); + + freepage = BufferGetPage(freebuf); + freehdr = (RelUndoPageHeader) PageGetContents(freepage); + + /* + * The free list is threaded through prev_blkno. Pop the head of the + * free list. + */ + meta->free_blkno = freehdr->prev_blkno; + + /* + * ABA defense for the WS-PVS2 reader: a recycled block reused at the + * same (blkno, offset) as a discarded prior record would otherwise be + * indistinguishable from that prior record to a stale verptr. Bump + * the generation counter so the recycled page's hdr->counter differs + * from any previous one at this blkno; RelUndoReadRecord validates + * (RelUndoGetCounter(ptr) == hdr->counter) to reject stale verptrs. + * + * Modular 16-bit increment, skipping 0 (reserved for "uninitialized" + * so a zeroed page never aliases a live counter). 16-bit wraparound + * (every 65535 recycles of the same fork) can produce a counter that + * matches a long-ago page header ā€” RelUndoReadRecord then returns + * false (chain-end / best-effort fallback), never returning wrong + * data. No further handling needed. + * + * Retention invariant: the oldest_xmin discard gate already protects + * every record the reader REVERSE-APPLIES (urec_xid >= oldest_xmin, + * thus non-discardable). This counter+validation only protects the + * single visibility-PROBE record the reader STOPS on, whose urec_xid + * may legitimately be < oldest_xmin and thus reside on a discardable + * (and reusable) page. + * + * Only the recycle branch needs the bump: a freshly extended block + * has never been the target of any verptr, so it has no ABA hazard. + * The counter need not be globally unique per page; uniqueness for + * the reader comes from a GIVEN blkno getting a different counter + * each time IT is recycled. + */ + meta->counter++; + if (meta->counter == 0) + meta->counter = 1; + + /* Re-initialize the page for use as a data page */ + relundo_init_page(freepage, old_head, meta->counter); + + MarkBufferDirty(freebuf); + buf = freebuf; + } + else + { + /* Extend the relation to get a new block */ + buf = ExtendBufferedRel(BMR_REL(rel), RELUNDO_FORKNUM, NULL, + EB_LOCK_FIRST); + newblkno = BufferGetBlockNumber(buf); + + page = BufferGetPage(buf); + relundo_init_page(page, old_head, meta->counter); + + MarkBufferDirty(buf); + } + + /* Update metapage: new head of this slot's chain */ + meta->head_blkno[slot] = newblkno; + + /* If this is the first data page in the slot, it's also the tail */ + if (!BlockNumberIsValid(old_head)) + meta->tail_blkno[slot] = newblkno; + + /* + * Track system allocation watermark. This records the highest block + * number allocated, enabling efficient reclamation of pages that were + * allocated by a system transaction but never used (because the user + * transaction aborted). + * + * Verified: The metapage is WAL-logged via REGBUF_STANDARD as block 1 in + * the caller's XLOG_RELUNDO_INSERT record (see RelUndoFinish). On crash + * recovery the FPI restores all metapage fields including + * system_alloc_watermark. + */ + if (!BlockNumberIsValid(meta->system_alloc_watermark) || + newblkno > meta->system_alloc_watermark) + meta->system_alloc_watermark = newblkno; + + MarkBufferDirty(metabuf); + + *newbuf = buf; + return newblkno; +} + +/* + * relundo_init_page + * Initialize a new UNDO data page. + * + * Uses standard PageInit for compatibility with the buffer manager's + * page verification, then sets up the RelUndoPageHeaderData in the + * contents area. + * + * pd_lower starts just after the UNDO page header; pd_upper is set to + * the full extent of the contents area. + */ +void +relundo_init_page(Page page, BlockNumber prev_blkno, uint16 counter) +{ + RelUndoPageHeader hdr; + + /* Initialize with standard page header (no special area) */ + PageInit(page, BLCKSZ, 0); + + /* Set up our UNDO-specific header in the page contents area */ + hdr = (RelUndoPageHeader) PageGetContents(page); + hdr->prev_blkno = prev_blkno; + hdr->max_xid = InvalidTransactionId; + hdr->counter = counter; + hdr->pd_lower = SizeOfRelUndoPageHeaderData; + hdr->pd_upper = BLCKSZ - MAXALIGN(SizeOfPageHeaderData); +} + +/* + * relundo_get_free_space + * Get amount of free space on an UNDO page. + * + * Returns the number of bytes available for new UNDO records. + * The offsets in the page header are relative to the contents area. + */ +Size +relundo_get_free_space(Page page) +{ + RelUndoPageHeader hdr; + + hdr = (RelUndoPageHeader) PageGetContents(page); + + if (hdr->pd_upper <= hdr->pd_lower) + return 0; + + return (Size) (hdr->pd_upper - hdr->pd_lower); +} diff --git a/src/backend/access/undo/relundo_recovery.c b/src/backend/access/undo/relundo_recovery.c new file mode 100644 index 0000000000000..fa731331903c7 --- /dev/null +++ b/src/backend/access/undo/relundo_recovery.c @@ -0,0 +1,408 @@ +/*------------------------------------------------------------------------- + * + * relundo_recovery.c + * Crash-recovery driver for per-relation UNDO (loser-transaction rollback) + * + * An in-place MVCC table access method overwrites the + * committed tuple bytes on the data page on UPDATE, and the only durable copy + * of the prior committed version is the before-image stored in the relation's + * UNDO fork. WAL redo faithfully re-establishes the page state as it was at + * crash time, including modifications made by transactions that never + * committed. + * Nothing in the redo pass reverses those uncommitted in-place changes, so an + * uncommitted new value would remain visible after restart -- a wrong-results + * bug. + * + * This module closes that gap with an end-of-recovery scan of the UNDO forks + * on disk. A track-during-redo approach cannot work: a CHECKPOINT taken after + * an uncommitted in-place UPDATE advances the redo start LSN past that UPDATE's + * WAL, so redo never replays it and a tracker would stay empty even though the + * uncommitted value is durable on the flushed data page. The before-image, + * however, is always durable in the UNDO fork. After redo finishes, CLOG has + * been fully reconstructed, so TransactionIdDidCommit() authoritatively + * separates committed (winner) from incomplete (loser) transactions. + * + * PerformRelUndoRecovery() enumerates every relundo fork on disk without + * catalog access (the ResetUnloggedRelations filesystem-walk pattern), opens + * each relation with a fake relcache entry, walks its UNDO page chain + * newest-first (metapage head_blkno then prev_blkno toward the tail; within a + * page, records are replayed high offset to low), and for every record whose + * creating transaction did not commit and is not a prepared transaction, calls + * RelUndoApplyRecordForRecovery() to restore the before-image in place. + * + * Newest-first order matters: a later in-place update's before-image is the + * earlier update's after-image, so the most recent record for a tuple must be + * reverse-applied first. Walking head->tail across pages and high->low within + * a page yields newest-first within every transaction; cross-transaction order + * is immaterial because each record restores an independent before-image. + * + * No compensation log record (CLR) is written here. PerformRelUndoRecovery() + * runs from PerformWalRecovery(), before StartupXLOG() enables WAL insertion + * for this backend, so WAL cannot be emitted. This mirrors the cluster-wide + * PerformUndoRecovery() path: durability is provided by the end-of-recovery + * checkpoint, and re-application after a mid-recovery crash is harmless because + * redo first re-establishes the post-modification page before the before-image + * is restored again. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/access/undo/relundo_recovery.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/relundo.h" +#include "access/transam.h" +#include "access/twophase.h" +#include "access/xlogutils.h" +#include "catalog/pg_tablespace_d.h" +#include "common/relpath.h" +#include "miscadmin.h" +#include "postgres_ext.h" +#include "storage/bufmgr.h" +#include "storage/bufpage.h" +#include "storage/fd.h" +#include "storage/reinit.h" +#include "storage/smgr.h" +#include "utils/memutils.h" + +static void RelUndoRecoveryScanTablespaceDir(const char *tsdirname, Oid spcoid); +static void RelUndoRecoveryScanDbspaceDir(const char *dbspacedirname, + Oid spcoid, Oid dboid); +static void RelUndoRecoveryScanOneFork(RelFileLocator rloc); +static int RelUndoRecoveryApplyPage(Relation rel, BlockNumber blkno); + +/* Running count of before-images restored across all forks. */ +static int relundo_recovery_applied = 0; + +/* + * PerformRelUndoRecovery - Reverse-apply loser transactions' before-images. + * + * Entry point called once at the end of WAL redo (crash recovery / PITR). + * Walks the data directory for per-relation UNDO forks and rolls back every + * incomplete transaction's in-place modifications. + */ +void +PerformRelUndoRecovery(void) +{ + char tblspc_path[MAXPGPATH + sizeof(PG_TBLSPC_DIR) + sizeof(TABLESPACE_VERSION_DIRECTORY)]; + DIR *spc_dir; + struct dirent *spc_de; + MemoryContext tmpctx, + oldctx; + + relundo_recovery_applied = 0; + + /* + * Use a private memory context so the directory-walk allocations are + * reclaimed in one shot regardless of how the scan exits. + */ + tmpctx = AllocSetContextCreate(CurrentMemoryContext, + "PerformRelUndoRecovery", + ALLOCSET_DEFAULT_SIZES); + oldctx = MemoryContextSwitchTo(tmpctx); + + /* Default tablespace lives under $PGDATA/base. */ + RelUndoRecoveryScanTablespaceDir("base", DEFAULTTABLESPACE_OID); + + /* Non-default tablespaces are symlinked under pg_tblspc. */ + spc_dir = AllocateDir(PG_TBLSPC_DIR); + while ((spc_de = ReadDir(spc_dir, PG_TBLSPC_DIR)) != NULL) + { + Oid spcoid; + + if (strcmp(spc_de->d_name, ".") == 0 || + strcmp(spc_de->d_name, "..") == 0) + continue; + + spcoid = atooid(spc_de->d_name); + if (!OidIsValid(spcoid)) + continue; + + snprintf(tblspc_path, sizeof(tblspc_path), "%s/%s/%s", + PG_TBLSPC_DIR, spc_de->d_name, TABLESPACE_VERSION_DIRECTORY); + RelUndoRecoveryScanTablespaceDir(tblspc_path, spcoid); + } + FreeDir(spc_dir); + + MemoryContextSwitchTo(oldctx); + MemoryContextDelete(tmpctx); + + if (relundo_recovery_applied > 0) + ereport(LOG, + (errmsg("per-relation UNDO recovery complete: %d before-image(s) restored", + relundo_recovery_applied))); +} + +/* + * RelUndoRecoveryScanTablespaceDir - Scan one tablespace's per-database dirs. + */ +static void +RelUndoRecoveryScanTablespaceDir(const char *tsdirname, Oid spcoid) +{ + DIR *ts_dir; + struct dirent *de; + char dbspace_path[MAXPGPATH * 2]; + + ts_dir = AllocateDir(tsdirname); + + /* + * A missing tablespace directory is not fatal: a crashed DROP TABLESPACE + * can leave a dangling pg_tblspc symlink. Mirror ResetUnloggedRelations + * and let it pass. + */ + if (ts_dir == NULL && errno == ENOENT) + return; + + while ((de = ReadDir(ts_dir, tsdirname)) != NULL) + { + Oid dboid; + + /* Per-database directories have purely numeric names. */ + if (strspn(de->d_name, "0123456789") != strlen(de->d_name)) + continue; + + dboid = atooid(de->d_name); + + snprintf(dbspace_path, sizeof(dbspace_path), "%s/%s", + tsdirname, de->d_name); + RelUndoRecoveryScanDbspaceDir(dbspace_path, spcoid, dboid); + } + + FreeDir(ts_dir); +} + +/* + * RelUndoRecoveryScanDbspaceDir - Scan one database dir for relundo forks. + * + * Only the first segment (segno 0) of each relundo fork is processed; the + * buffer manager transparently spans higher segments when reading the fork. + */ +static void +RelUndoRecoveryScanDbspaceDir(const char *dbspacedirname, Oid spcoid, Oid dboid) +{ + DIR *dbspace_dir; + struct dirent *de; + + dbspace_dir = AllocateDir(dbspacedirname); + if (dbspace_dir == NULL && errno == ENOENT) + return; + + while ((de = ReadDir(dbspace_dir, dbspacedirname)) != NULL) + { + RelFileNumber relnumber; + ForkNumber forknum; + unsigned segno; + RelFileLocator rloc; + + if (!parse_filename_for_nontemp_relation(de->d_name, &relnumber, + &forknum, &segno)) + continue; + + if (forknum != RELUNDO_FORKNUM || segno != 0) + continue; + + rloc.spcOid = spcoid; + rloc.dbOid = dboid; + rloc.relNumber = relnumber; + + RelUndoRecoveryScanOneFork(rloc); + } + + FreeDir(dbspace_dir); +} + +/* + * RelUndoRecoveryScanOneFork - Roll back losers recorded in one UNDO fork. + * + * Opens the relation with a fake relcache entry (no catalog access), reads the + * metapage to find the head of the page chain, and walks the chain newest-page + * first applying loser before-images. + */ +static void +RelUndoRecoveryScanOneFork(RelFileLocator rloc) +{ + Relation rel; + Buffer metabuf; + Page metapage; + RelUndoMetaPage meta; + BlockNumber head_blkno[RELUNDO_NUM_HEADS]; + BlockNumber nblocks; + + rel = CreateFakeRelcacheEntry(rloc); + + /* + * An empty or absent fork has nothing to roll back. The relation is a + * fake relcache entry whose rd_rel->relkind is zero, so the relkind + * dispatch in RelationGetNumberOfBlocksInFork() would trip an assertion; + * probe the underlying smgr directly instead, as XLOG replay does. + */ + if (!smgrexists(RelationGetSmgr(rel), RELUNDO_FORKNUM)) + { + FreeFakeRelcacheEntry(rel); + return; + } + + nblocks = smgrnblocks(RelationGetSmgr(rel), RELUNDO_FORKNUM); + if (nblocks == 0) + { + FreeFakeRelcacheEntry(rel); + return; + } + + /* Read the metapage (block 0) directly; do not auto-initialize it. */ + metabuf = ReadBufferExtended(rel, RELUNDO_FORKNUM, 0, RBM_NORMAL, NULL); + LockBuffer(metabuf, BUFFER_LOCK_SHARE); + metapage = BufferGetPage(metabuf); + meta = (RelUndoMetaPage) PageGetContents(metapage); + + if (meta->magic != RELUNDO_METAPAGE_MAGIC) + { + UnlockReleaseBuffer(metabuf); + FreeFakeRelcacheEntry(rel); + return; + } + + for (int slot = 0; slot < RELUNDO_NUM_HEADS; slot++) + head_blkno[slot] = meta->head_blkno[slot]; + UnlockReleaseBuffer(metabuf); + + /* + * Walk each of the RELUNDO_NUM_HEADS independent append chains from its + * head (newest) toward its tail (oldest). Only real data blocks (block + * >= 1, below the fork's block count) are valid chain links. Block 0 is + * the metapage: a zeroed data page carries a prev_blkno of 0 -- not + * InvalidBlockNumber -- and BlockNumberIsValid(0) is true, so an + * unguarded walk would read the metapage as a data page, interpret its + * magic as the next block number, and fault. Treat block 0 and any + * out-of-range block as the chain terminus. + * + * A corrupt prev_blkno could form a cycle. Each chain has at most + * nblocks links, so cap the walk at nblocks iterations and fail loudly on + * overrun rather than spinning forever during recovery. + */ + for (int slot = 0; slot < RELUNDO_NUM_HEADS; slot++) + { + BlockNumber blkno = head_blkno[slot]; + + for (BlockNumber steps = 0; + blkno != InvalidBlockNumber && blkno >= 1 && blkno < nblocks; + steps++) + { + if (steps >= nblocks) + ereport(ERROR, + (errcode(ERRCODE_DATA_CORRUPTED), + errmsg("per-relation UNDO page chain for relation %u/%u/%u exceeds %u pages; possible cycle from corrupt prev_blkno", + rloc.spcOid, rloc.dbOid, rloc.relNumber, nblocks))); + + blkno = RelUndoRecoveryApplyPage(rel, blkno); + } + } + + FreeFakeRelcacheEntry(rel); +} + +/* + * RelUndoRecoveryApplyPage - Reverse-apply loser records on one UNDO page. + * + * Reads the page's records (oldest-to-newest by ascending offset) under a + * share lock, then releases the lock and reverse-applies the loser records + * newest-first. The undo-page lock must be dropped before + * RelUndoApplyRecordForRecovery() runs, because that routine re-reads the same + * undo page and would otherwise self-deadlock on the buffer lock. + * + * Returns the previous page in the chain (toward the tail), or + * InvalidBlockNumber when this is the oldest page. + */ +static int +RelUndoRecoveryApplyPage(Relation rel, BlockNumber blkno) +{ + Buffer buf; + Page page; + char *contents; + RelUndoPageHeader hdr; + BlockNumber prev; + uint16 page_counter; + uint16 pd_lower; + uint16 offset; + uint16 *offsets; + TransactionId *xids; + int nrecs = 0; + int maxrecs; + int i; + + buf = ReadBufferExtended(rel, RELUNDO_FORKNUM, blkno, RBM_NORMAL, NULL); + LockBuffer(buf, BUFFER_LOCK_SHARE); + page = BufferGetPage(buf); + contents = PageGetContents(page); + hdr = (RelUndoPageHeader) contents; + + prev = hdr->prev_blkno; + page_counter = hdr->counter; + pd_lower = hdr->pd_lower; + + /* Upper bound on records: each record is at least a header in size. */ + maxrecs = (pd_lower > SizeOfRelUndoPageHeaderData) + ? (pd_lower - SizeOfRelUndoPageHeaderData) / SizeOfRelUndoRecordHeader + 1 + : 0; + + if (maxrecs == 0) + { + UnlockReleaseBuffer(buf); + return prev; + } + + offsets = (uint16 *) palloc(sizeof(uint16) * maxrecs); + xids = (TransactionId *) palloc(sizeof(TransactionId) * maxrecs); + + /* Collect record offsets and xids in insertion order. */ + offset = SizeOfRelUndoPageHeaderData; + while (offset < pd_lower && nrecs < maxrecs) + { + RelUndoRecordHeader rhdr; + + memcpy(&rhdr, contents + offset, SizeOfRelUndoRecordHeader); + + /* A zero-length or malformed record terminates the scan defensively. */ + if (rhdr.urec_len < SizeOfRelUndoRecordHeader) + break; + + /* urec_type 0 marks a cancelled hole; skip but keep striding. */ + if (rhdr.urec_type != 0) + { + offsets[nrecs] = offset; + xids[nrecs] = rhdr.urec_xid; + nrecs++; + } + + offset += rhdr.urec_len; + } + + UnlockReleaseBuffer(buf); + + /* Reverse-apply newest-first; skip winners and prepared transactions. */ + for (i = nrecs - 1; i >= 0; i--) + { + TransactionId xid = xids[i]; + + if (!TransactionIdIsNormal(xid)) + continue; + if (TransactionIdDidCommit(xid)) + continue; + if (RecoveryTransactionIdIsPrepared(xid)) + continue; + + RelUndoApplyRecordForRecovery(rel, + MakeRelUndoRecPtr(page_counter, blkno, + offsets[i])); + relundo_recovery_applied++; + } + + pfree(offsets); + pfree(xids); + + return prev; +} diff --git a/src/backend/access/undo/relundo_worker.c b/src/backend/access/undo/relundo_worker.c new file mode 100644 index 0000000000000..9112a47fbefdd --- /dev/null +++ b/src/backend/access/undo/relundo_worker.c @@ -0,0 +1,731 @@ +/*------------------------------------------------------------------------- + * + * relundo_worker.c + * Background worker for applying per-relation UNDO records asynchronously + * + * This module implements the async per-relation UNDO worker system that + * applies UNDO records for aborted transactions. Workers run in background + * processes to avoid blocking ROLLBACK commands with synchronous UNDO + * application. + * + * The system consists of: + * 1. A launcher process that manages the worker pool + * 2. Individual worker processes that apply UNDO chains + * 3. A shared memory work queue for coordinating pending work + * + * Architecture matches autovacuum: launcher spawns workers as needed, + * workers process work items, communicate via shared memory. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/access/undo/relundo_worker.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include +#include + +#include "access/heapam.h" +#include "access/htup_details.h" +#include "access/relundo_worker.h" +#include "access/xact.h" +#include "access/relundo.h" +#include "access/table.h" +#include "access/tableam.h" +#include "catalog/pg_class.h" +#include "common/relpath.h" +#include "miscadmin.h" +#include "pgstat.h" +#include "postmaster/bgworker.h" +#include "storage/bufmgr.h" +#include "storage/ipc.h" +#include "storage/latch.h" +#include "storage/lwlock.h" +#include "storage/shmem.h" +#include "storage/smgr.h" +#include "tcop/tcopprot.h" +#include "utils/fmgroids.h" +#include "utils/guc.h" +#include "utils/rel.h" +#include "utils/timestamp.h" + +/* GUC parameters */ +int max_relundo_workers = 3; +int relundo_worker_naptime = 5000; /* milliseconds */ + +/* Shared memory state */ +static RelUndoWorkQueue *WorkQueue = NULL; + +/* Flags set by signal handlers */ +static volatile sig_atomic_t got_SIGHUP = false; +static volatile sig_atomic_t got_SIGTERM = false; + +/* Forward declarations */ +static void relundo_worker_sighup(SIGNAL_ARGS); +static void relundo_worker_sigterm(SIGNAL_ARGS); +static void process_relundo_work_item(RelUndoWorkItem *item); + +/* + * RelUndoWorkerShmemSize + * Calculate shared memory space needed for per-relation UNDO workers + */ +Size +RelUndoWorkerShmemSize(void) +{ + Size size = 0; + + size = add_size(size, sizeof(RelUndoWorkQueue)); + return size; +} + +/* + * RelUndoWorkerShmemInit + * Allocate and initialize shared memory for per-relation UNDO workers + */ +void +RelUndoWorkerShmemInit(void) +{ + bool found; + + WorkQueue = (RelUndoWorkQueue *) + ShmemInitStruct("Per-Relation UNDO Work Queue", + sizeof(RelUndoWorkQueue), + &found); + + if (!found) + { + /* First time through, initialize the work queue */ + LWLockInitialize(&WorkQueue->lock, LWTRANCHE_UNDO_WORKER); + WorkQueue->num_items = 0; + WorkQueue->next_worker_id = 1; + memset(WorkQueue->items, 0, sizeof(WorkQueue->items)); + } +} + +/* + * RelUndoQueueAdd + * Add a new per-relation UNDO work item to the queue + * + * Called during transaction abort to queue UNDO application work for + * background workers. + */ +void +RelUndoQueueAdd(Oid dboid, Oid reloid, RelUndoRecPtr start_urec_ptr, + TransactionId xid) +{ + int i; + bool found_slot = false; + + LWLockAcquire(&WorkQueue->lock, LW_EXCLUSIVE); + + /* Check if we already have work for this relation */ + for (i = 0; i < WorkQueue->num_items; i++) + { + RelUndoWorkItem *item = &WorkQueue->items[i]; + + if (item->dboid == dboid && item->reloid == reloid) + { + /* Update existing entry with latest UNDO pointer */ + item->start_urec_ptr = start_urec_ptr; + item->xid = xid; + item->queued_at = GetCurrentTimestamp(); + found_slot = true; + break; + } + } + + if (!found_slot) + { + RelUndoWorkItem *item; + + /* Add new work item */ + if (WorkQueue->num_items >= MAX_UNDO_WORK_ITEMS) + { + LWLockRelease(&WorkQueue->lock); + ereport(WARNING, + (errmsg("Per-relation UNDO work queue is full, cannot queue work for relation %u", + reloid))); + return; + } + + item = &WorkQueue->items[WorkQueue->num_items]; + item->dboid = dboid; + item->reloid = reloid; + item->start_urec_ptr = start_urec_ptr; + item->xid = xid; + item->queued_at = GetCurrentTimestamp(); + item->in_progress = false; + item->worker_id = 0; + WorkQueue->num_items++; + } + + LWLockRelease(&WorkQueue->lock); + + elog(DEBUG1, "Queued per-relation UNDO work for database %u, relation %u (ptr=%lu)", + dboid, reloid, (unsigned long) start_urec_ptr); +} + +/* + * RelUndoQueueGetNext + * Get the next work item for a worker to process + * + * Returns true if work was found, false if queue is empty. + * Marks the item as in_progress to prevent other workers from taking it. + */ +bool +RelUndoQueueGetNext(RelUndoWorkItem *item_out, int worker_id) +{ + int i; + bool found = false; + + LWLockAcquire(&WorkQueue->lock, LW_EXCLUSIVE); + + for (i = 0; i < WorkQueue->num_items; i++) + { + RelUndoWorkItem *item = &WorkQueue->items[i]; + + if (!item->in_progress && item->dboid == MyDatabaseId) + { + /* Found work for this database */ + memcpy(item_out, item, sizeof(RelUndoWorkItem)); + item->in_progress = true; + item->worker_id = worker_id; + found = true; + break; + } + } + + LWLockRelease(&WorkQueue->lock); + + return found; +} + +/* + * RelUndoQueueMarkComplete + * Mark a work item as complete and remove it from the queue + */ +void +RelUndoQueueMarkComplete(Oid dboid, Oid reloid, int worker_id) +{ + int i, + j; + + LWLockAcquire(&WorkQueue->lock, LW_EXCLUSIVE); + + for (i = 0; i < WorkQueue->num_items; i++) + { + RelUndoWorkItem *item = &WorkQueue->items[i]; + + if (item->dboid == dboid && item->reloid == reloid && + item->worker_id == worker_id) + { + /* Found the item, remove it by shifting remaining items */ + for (j = i; j < WorkQueue->num_items - 1; j++) + { + memcpy(&WorkQueue->items[j], &WorkQueue->items[j + 1], + sizeof(RelUndoWorkItem)); + } + WorkQueue->num_items--; + break; + } + } + + LWLockRelease(&WorkQueue->lock); + + elog(DEBUG1, "Completed per-relation UNDO work for database %u, relation %u", + dboid, reloid); +} + +/* + * relundo_worker_sighup + * SIGHUP signal handler for per-relation UNDO worker + */ +static void +relundo_worker_sighup(SIGNAL_ARGS) +{ + int save_errno = errno; + + got_SIGHUP = true; + SetLatch(MyLatch); + + errno = save_errno; +} + +/* + * relundo_worker_sigterm + * SIGTERM signal handler for per-relation UNDO worker + */ +static void +relundo_worker_sigterm(SIGNAL_ARGS) +{ + int save_errno = errno; + + got_SIGTERM = true; + SetLatch(MyLatch); + + errno = save_errno; +} + +/* + * process_relundo_work_item + * Apply per-relation UNDO records for a single work item + */ +static void +process_relundo_work_item(RelUndoWorkItem *item) +{ + Relation rel; + + elog(LOG, "Per-relation UNDO worker processing: database %u, relation %u, UNDO ptr %lu", + item->dboid, item->reloid, (unsigned long) item->start_urec_ptr); + + /* + * Open the relation with RowExclusiveLock, the same lock level used by + * normal DML. The UNDO apply modifies individual tuples and does not + * need to block concurrent readers or writers at the relation level. + * + * Previously this used AccessExclusiveLock, which created lock convoys + * under high concurrency: the UNDO worker's exclusive lock request would + * queue behind active transactions, and all new transactions would queue + * behind the UNDO worker, causing a complete stall. + */ + PG_TRY(); + { + rel = table_open(item->reloid, RowExclusiveLock); + + /* Apply the UNDO chain */ + RelUndoApplyChain(rel, item->start_urec_ptr); + + /* + * Clean up any ABORTED sLog entries for this transaction. At abort + * time, sLog entries were marked ABORTED (not removed) so visibility + * checks could detect aborted-but-not-yet-undone inserts. Now that + * the tuples are physically restored, remove those entries. + */ + if (RelUndoAbortCleanup_hook) + RelUndoAbortCleanup_hook(item->xid); + + table_close(rel, RowExclusiveLock); + } + PG_CATCH(); + { + /* + * If relation was dropped or doesn't exist, that's OK - nothing to + * do. Just log it and move on. + */ + EmitErrorReport(); + FlushErrorState(); + + elog(LOG, "Per-relation UNDO worker: failed to process relation %u, skipping", + item->reloid); + } + PG_END_TRY(); +} + +/* + * RelUndoWorkerMain + * Main entry point for per-relation UNDO worker process + */ +void +RelUndoWorkerMain(Datum main_arg) +{ + Oid dboid = DatumGetObjectId(main_arg); + int worker_id; + + /* Establish signal handlers */ + pqsignal(SIGHUP, relundo_worker_sighup); + pqsignal(SIGTERM, relundo_worker_sigterm); + + /* We're now ready to receive signals */ + BackgroundWorkerUnblockSignals(); + + /* Connect to the specified database */ + BackgroundWorkerInitializeConnectionByOid(dboid, InvalidOid, 0); + + /* Get a worker ID */ + LWLockAcquire(&WorkQueue->lock, LW_EXCLUSIVE); + worker_id = WorkQueue->next_worker_id++; + LWLockRelease(&WorkQueue->lock); + + elog(LOG, "Per-relation UNDO worker %d started for database %u", worker_id, dboid); + + /* Main work loop */ + while (!got_SIGTERM) + { + RelUndoWorkItem item; + + /* Handle SIGHUP - reload configuration */ + if (got_SIGHUP) + { + got_SIGHUP = false; + ProcessConfigFile(PGC_SIGHUP); + } + + /* Check for UNDO chain application work */ + if (RelUndoQueueGetNext(&item, worker_id)) + { + /* Start a transaction for applying UNDO */ + StartTransactionCommand(); + + /* Process the work item */ + process_relundo_work_item(&item); + + /* Mark as complete */ + RelUndoQueueMarkComplete(item.dboid, item.reloid, worker_id); + + /* Commit the transaction */ + CommitTransactionCommand(); + } + else + { + /* + * No UNDO chain work available, so exit. The worker is + * registered with BGW_NEVER_RESTART so once the queue is drained + * it should not linger -- the aborting backend may be waiting for + * us via WaitForBackgroundWorkerShutdown(). + */ + break; /* exit the main loop */ + } + } + + elog(LOG, "Per-relation UNDO worker %d shutting down", worker_id); + proc_exit(0); +} + +/* + * Per-database worker tracking in the launcher. + * + * The launcher maintains a local array of per-database worker slots. + * Each slot records the database OID and the background worker handle + * returned by RegisterDynamicBackgroundWorker(). When a worker exits, + * the slot is freed for reuse. + */ +#define MAX_LAUNCHER_DB_SLOTS MAX_UNDO_WORK_ITEMS + +typedef struct LauncherDbSlot +{ + Oid dboid; /* Database OID, or InvalidOid if free */ + BackgroundWorkerHandle *handle; /* Worker handle (NULL if slot is free) */ + TimestampTz last_spawn_attempt; /* 0 = never attempted */ +} LauncherDbSlot; + +/* + * launcher_spawn_worker + * Spawn a per-relation UNDO worker for the given database. + * + * Returns true on success, false if RegisterDynamicBackgroundWorker fails + * (e.g., because the max_worker_processes limit was reached). + */ +static bool +launcher_spawn_worker(Oid dboid, BackgroundWorkerHandle **handle_out) +{ + BackgroundWorker worker; + + memset(&worker, 0, sizeof(BackgroundWorker)); + worker.bgw_flags = BGWORKER_SHMEM_ACCESS | + BGWORKER_BACKEND_DATABASE_CONNECTION; + worker.bgw_start_time = BgWorkerStart_RecoveryFinished; + worker.bgw_restart_time = BGW_NEVER_RESTART; + sprintf(worker.bgw_library_name, "postgres"); + sprintf(worker.bgw_function_name, "RelUndoWorkerMain"); + snprintf(worker.bgw_name, BGW_MAXLEN, + "per-relation undo worker for database %u", dboid); + snprintf(worker.bgw_type, BGW_MAXLEN, "per-relation undo worker"); + worker.bgw_main_arg = ObjectIdGetDatum(dboid); + worker.bgw_notify_pid = 0; /* launcher does not need SIGUSR1 */ + + if (!RegisterDynamicBackgroundWorker(&worker, handle_out)) + { + ereport(DEBUG1, + (errmsg("per-relation UNDO launcher: could not register worker for database %u", + dboid))); + return false; + } + + elog(DEBUG1, "per-relation UNDO launcher: spawned worker for database %u", + dboid); + return true; +} + +/* + * RelUndoLauncherMain + * Main entry point for per-relation UNDO launcher process + * + * The launcher periodically scans the shared work queue for databases + * that have pending UNDO work and spawns per-database worker processes + * as needed, up to max_relundo_workers total. + */ +void +RelUndoLauncherMain(Datum main_arg) +{ + LauncherDbSlot db_slots[MAX_LAUNCHER_DB_SLOTS]; + int nslots = 0; + + /* Establish signal handlers */ + pqsignal(SIGHUP, relundo_worker_sighup); + pqsignal(SIGTERM, relundo_worker_sigterm); + + /* We're now ready to receive signals */ + BackgroundWorkerUnblockSignals(); + + elog(LOG, "Per-relation UNDO launcher started"); + + memset(db_slots, 0, sizeof(db_slots)); + + /* Main monitoring loop */ + while (!got_SIGTERM) + { + int rc; + int active_count; + int i, + j; + Oid pending_dbs[MAX_UNDO_WORK_ITEMS]; + int npending = 0; + + /* Handle SIGHUP - reload configuration */ + if (got_SIGHUP) + { + got_SIGHUP = false; + ProcessConfigFile(PGC_SIGHUP); + } + + /* + * Step 1: Check existing worker handles to see which are still alive. + * Workers that have exited (BGWH_STOPPED) are freed. + */ + active_count = 0; + for (i = 0; i < nslots; i++) + { + if (db_slots[i].dboid == InvalidOid) + continue; + + if (db_slots[i].handle != NULL) + { + pid_t pid; + BgwHandleStatus status; + + status = GetBackgroundWorkerPid(db_slots[i].handle, &pid); + if (status == BGWH_STOPPED || status == BGWH_POSTMASTER_DIED) + { + /* + * Worker has exited. Reset any items that were marked + * in_progress for this database so they can be retried. + */ + LWLockAcquire(&WorkQueue->lock, LW_EXCLUSIVE); + for (j = 0; j < WorkQueue->num_items; j++) + { + if (WorkQueue->items[j].dboid == db_slots[i].dboid && + WorkQueue->items[j].in_progress) + WorkQueue->items[j].in_progress = false; + } + LWLockRelease(&WorkQueue->lock); + + { + Oid exited_dboid = db_slots[i].dboid; + + pfree(db_slots[i].handle); + db_slots[i].handle = NULL; + db_slots[i].dboid = InvalidOid; + + elog(DEBUG1, + "per-relation UNDO launcher: worker for database %u has exited", + exited_dboid); + } + continue; + } + + active_count++; + } + } + + /* Compact the slots array to remove freed entries */ + nslots = 0; + for (i = 0; i < MAX_LAUNCHER_DB_SLOTS; i++) + { + if (db_slots[i].dboid != InvalidOid) + nslots = i + 1; + } + + /* + * Step 2: Scan the work queue for databases with pending (not yet + * in_progress) work items that do not already have an active worker. + */ + LWLockAcquire(&WorkQueue->lock, LW_SHARED); + for (i = 0; i < WorkQueue->num_items; i++) + { + RelUndoWorkItem *item = &WorkQueue->items[i]; + bool has_worker; + bool already_listed; + + if (item->in_progress) + continue; /* someone is working on this */ + + /* Check if we already noted this database needs a worker */ + already_listed = false; + for (j = 0; j < npending; j++) + { + if (pending_dbs[j] == item->dboid) + { + already_listed = true; + break; + } + } + if (already_listed) + continue; + + /* Check if there is already an active worker for this database */ + has_worker = false; + for (j = 0; j < MAX_LAUNCHER_DB_SLOTS; j++) + { + if (db_slots[j].dboid == item->dboid && + db_slots[j].handle != NULL) + { + has_worker = true; + break; + } + } + + if (!has_worker) + { + if (npending < MAX_UNDO_WORK_ITEMS) + pending_dbs[npending++] = item->dboid; + } + } + LWLockRelease(&WorkQueue->lock); + + /* + * Step 3: Spawn workers for databases that need them, up to the + * max_relundo_workers limit. + */ + for (i = 0; i < npending; i++) + { + Oid dboid = pending_dbs[i]; + int free_slot = -1; + BackgroundWorkerHandle *handle; + + if (active_count >= max_relundo_workers) + break; /* at the worker limit */ + + /* Find a free slot in db_slots */ + for (j = 0; j < MAX_LAUNCHER_DB_SLOTS; j++) + { + if (db_slots[j].dboid == InvalidOid) + { + free_slot = j; + break; + } + } + if (free_slot < 0) + break; /* no free slots (shouldn't happen) */ + + /* + * Back off if we recently failed to spawn a worker for this slot. + * This prevents hammering RegisterDynamicBackgroundWorker() when + * worker slots are exhausted. + */ + if (db_slots[free_slot].last_spawn_attempt != 0) + { + TimestampTz now = GetCurrentTimestamp(); + + if (now - db_slots[free_slot].last_spawn_attempt < + (TimestampTz) relundo_worker_naptime * 4 * 1000) + continue; /* too soon, skip this database for now */ + } + + if (launcher_spawn_worker(dboid, &handle)) + { + db_slots[free_slot].dboid = dboid; + db_slots[free_slot].handle = handle; + db_slots[free_slot].last_spawn_attempt = 0; + if (free_slot >= nslots) + nslots = free_slot + 1; + active_count++; + } + else + { + db_slots[free_slot].last_spawn_attempt = GetCurrentTimestamp(); + } + } + + /* Sleep until next check or until woken */ + rc = WaitLatch(MyLatch, + WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, + relundo_worker_naptime * 2, + PG_WAIT_EXTENSION); + + ResetLatch(MyLatch); + + /* Emergency bailout if postmaster has died */ + if (rc & WL_POSTMASTER_DEATH) + proc_exit(1); + } + + elog(LOG, "Per-relation UNDO launcher shutting down"); + proc_exit(0); +} + +/* + * Saved background worker handle for the most recent synchronous UNDO + * worker. WaitForPendingRelUndo() uses this to block until the worker + * exits, making the "sync rollback" path truly synchronous. + */ +static BackgroundWorkerHandle *pending_undo_handle = NULL; + +/* + * StartRelUndoWorker + * Request a background worker for applying per-relation UNDO in a database + */ +void +StartRelUndoWorker(Oid dboid) +{ + BackgroundWorker worker; + BackgroundWorkerHandle *handle; + + memset(&worker, 0, sizeof(BackgroundWorker)); + worker.bgw_flags = BGWORKER_SHMEM_ACCESS | + BGWORKER_BACKEND_DATABASE_CONNECTION; + worker.bgw_start_time = BgWorkerStart_RecoveryFinished; + worker.bgw_restart_time = BGW_NEVER_RESTART; + sprintf(worker.bgw_library_name, "postgres"); + sprintf(worker.bgw_function_name, "RelUndoWorkerMain"); + snprintf(worker.bgw_name, BGW_MAXLEN, "per-relation undo worker for database %u", dboid); + snprintf(worker.bgw_type, BGW_MAXLEN, "per-relation undo worker"); + worker.bgw_main_arg = ObjectIdGetDatum(dboid); + worker.bgw_notify_pid = MyProcPid; + + if (!RegisterDynamicBackgroundWorker(&worker, &handle)) + { + ereport(WARNING, + (errmsg("could not register per-relation UNDO worker for database %u", dboid))); + } + else + { + elog(DEBUG1, "Started per-relation UNDO worker for database %u", dboid); + pending_undo_handle = handle; + } +} + +/* + * WaitForPendingRelUndo + * Block until the most recent synchronous UNDO worker exits. + * + * Called from AbortTransaction() AFTER locks have been released so the + * UNDO worker can acquire AccessExclusiveLock on the target relation. + * This makes the "sync rollback" path truly synchronous: the aborting + * backend does not return to the client until the UNDO is applied and + * the original tuple data is restored. + */ +void +WaitForPendingRelUndo(void) +{ + if (pending_undo_handle == NULL) + return; + + (void) WaitForBackgroundWorkerShutdown(pending_undo_handle); + + pfree(pending_undo_handle); + pending_undo_handle = NULL; +} diff --git a/src/backend/access/undo/relundo_xlog.c b/src/backend/access/undo/relundo_xlog.c new file mode 100644 index 0000000000000..d21fe0b7b1dd2 --- /dev/null +++ b/src/backend/access/undo/relundo_xlog.c @@ -0,0 +1,760 @@ +/*------------------------------------------------------------------------- + * + * relundo_xlog.c + * Per-relation UNDO resource manager WAL redo routines + * + * This module implements the WAL redo callback for the RM_RELUNDO_ID + * resource manager. It handles replay of: + * + * XLOG_RELUNDO_INIT - Replay metapage initialization + * XLOG_RELUNDO_INSERT - Replay UNDO record insertion into a data page + * XLOG_RELUNDO_DISCARD - Replay discard of old UNDO pages + * + * Redo Strategy + * ------------- + * INIT and DISCARD use full page images (FPI) via XLogInitBufferForRedo() + * or REGBUF_FORCE_IMAGE, so redo simply restores the page image. + * + * INSERT records may include FPIs on the first modification after a + * checkpoint. When no FPI is present (BLK_NEEDS_REDO), the redo + * function reconstructs the insertion by copying the UNDO record data + * into the page at the recorded offset and updating pd_lower. + * + * Async I/O Strategy + * ------------------ + * INSERT records may reference two blocks: block 0 (data page) and + * block 1 (metapage, when the head pointer was updated). To overlap + * the I/O for both blocks, we issue a PrefetchSharedBuffer() for + * block 1 before processing block 0. This allows the kernel or the + * AIO worker to start reading the metapage in parallel with the data + * page read, reducing overall latency during crash recovery. + * + * When io_method is WORKER or IO_URING, we also enter batch mode + * (pgaio_enter_batchmode) so that multiple I/O submissions can be + * coalesced into fewer system calls. The batch is exited after all + * blocks in the record have been processed. + * + * Parallel Redo Support + * --------------------- + * This resource manager supports parallel WAL replay for multi-core crash + * recovery via the startup, cleanup, and mask callbacks registered in + * rmgrlist.h. + * + * Page dependency rules for parallel redo: + * + * - Records that touch different pages can be replayed in parallel with + * no ordering constraints. + * + * - Within the same page, XLOG_RELUNDO_INIT (or INSERT with the + * XLOG_RELUNDO_INIT_PAGE flag) must be replayed before any subsequent + * XLOG_RELUNDO_INSERT on that page. The recovery manager enforces + * this automatically via the page LSN check in XLogReadBufferForRedo. + * + * - XLOG_RELUNDO_DISCARD only modifies the metapage (block 0). It is + * ordered relative to other metapage modifications by the page LSN. + * + * - The metapage (block 0) is a serialization point: INSERT records that + * update the head pointer and DISCARD records both touch the metapage, + * so they are serialized on that page by the buffer lock. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/access/undo/relundo_xlog.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/bufmask.h" +#include "access/relundo.h" +#include "access/relundo_xlog.h" +#include "access/xlogutils.h" +#include "miscadmin.h" +#include "storage/aio.h" +#include "storage/bufmgr.h" +#include "storage/bufpage.h" +#include "storage/smgr.h" + +/* + * relundo_redo_init - Replay metapage initialization + * + * The metapage is always logged with a full page image via + * XLogInitBufferForRedo, so we just need to initialize and restore it. + */ +static void +relundo_redo_init(XLogReaderState *record) +{ + XLogRecPtr lsn = record->EndRecPtr; + xl_relundo_init *xlrec = (xl_relundo_init *) XLogRecGetData(record); + Buffer buf; + Page page; + RelUndoMetaPageData *meta; + + /* Consistency checks on WAL record data */ + if (xlrec->magic != RELUNDO_METAPAGE_MAGIC) + elog(PANIC, "relundo_redo_init: invalid magic 0x%X (expected 0x%X)", + xlrec->magic, RELUNDO_METAPAGE_MAGIC); + + if (xlrec->version != RELUNDO_METAPAGE_VERSION) + elog(PANIC, "relundo_redo_init: invalid version %u (expected %u)", + xlrec->version, RELUNDO_METAPAGE_VERSION); + + /* + * Initial counter should be 1 for a freshly initialized metapage. + * RelUndoInitRelation sets counter to 1 so that 0 is clearly "no counter + * / uninitialized". Accept any small value as valid since the counter + * only increments from 1 and a freshly initialized metapage will have + * counter == 1. + */ + if (xlrec->counter > 1) + elog(PANIC, "relundo_redo_init: initial counter %u too large for init record", + xlrec->counter); + + buf = XLogInitBufferForRedo(record, 0); + page = BufferGetPage(buf); + + /* Initialize the metapage from scratch */ + PageInit(page, BLCKSZ, 0); + + meta = (RelUndoMetaPageData *) PageGetContents(page); + meta->magic = xlrec->magic; + meta->version = xlrec->version; + meta->counter = xlrec->counter; + for (int slot = 0; slot < RELUNDO_NUM_HEADS; slot++) + { + meta->head_blkno[slot] = InvalidBlockNumber; + meta->tail_blkno[slot] = InvalidBlockNumber; + } + meta->free_blkno = InvalidBlockNumber; + meta->total_records = 0; + meta->discarded_records = 0; + + /* Match the do-time metapage: cover the meta struct with pd_lower. */ + RelUndoMetaPageSetPdLower(page); + + PageSetLSN(page, lsn); + MarkBufferDirty(buf); + UnlockReleaseBuffer(buf); +} + +/* + * relundo_prefetch_block - Issue async prefetch for a WAL-referenced block + * + * If the WAL record references the given block_id and it has not already + * been prefetched by the XLogPrefetcher, initiate an async read via + * PrefetchSharedBuffer(). This is a no-op when USE_PREFETCH is not + * available or when the block is already in the buffer pool. + * + * Returns true if I/O was initiated, false otherwise (cache hit or no-op). + */ +static bool +relundo_prefetch_block(XLogReaderState *record, uint8 block_id) +{ +#ifdef USE_PREFETCH + RelFileLocator rlocator; + ForkNumber forknum; + BlockNumber blkno; + Buffer prefetch_buffer; + SMgrRelation smgr; + + if (!XLogRecGetBlockTagExtended(record, block_id, + &rlocator, &forknum, &blkno, + &prefetch_buffer)) + return false; + + /* If the XLogPrefetcher already cached a buffer hint, skip prefetch. */ + if (BufferIsValid(prefetch_buffer)) + return false; + + smgr = smgropen(rlocator, INVALID_PROC_NUMBER); + + /* + * Only prefetch if the relation fork exists and the block is within the + * current size. During recovery, relations may not yet have been + * extended to the referenced block. + */ + if (smgrexists(smgr, forknum)) + { + BlockNumber nblocks = smgrnblocks(smgr, forknum); + + if (blkno < nblocks) + { + PrefetchSharedBuffer(smgr, forknum, blkno); + return true; + } + } +#endif /* USE_PREFETCH */ + + return false; +} + +/* + * relundo_redo_insert - Replay UNDO record insertion + * + * When a full page image is present, it is restored automatically by + * XLogReadBufferForRedo (BLK_RESTORED). Otherwise (BLK_NEEDS_REDO), + * we copy the UNDO record data into the page at the recorded offset + * and update pd_lower. + * + * If the XLOG_RELUNDO_INIT_PAGE flag is set, the page is a newly + * allocated data page and must be initialized from scratch before + * inserting the record. + * + * Async I/O: When this record references both block 0 (data page) and + * block 1 (metapage), we prefetch block 1 before reading block 0. + * This allows the I/O for the metapage to proceed in parallel with + * the data page read and redo processing, reducing stall time. + */ +static void +relundo_redo_insert(XLogReaderState *record) +{ + XLogRecPtr lsn = record->EndRecPtr; + xl_relundo_insert *xlrec = (xl_relundo_insert *) XLogRecGetData(record); + Buffer buf; + XLogRedoAction action; + bool has_metapage = XLogRecHasBlockRef(record, 1); + bool use_batchmode; + + /* Consistency checks on WAL record data */ + if (xlrec->urec_len < SizeOfRelUndoRecordHeader) + elog(PANIC, "relundo_redo_insert: invalid record length %u (min %zu)", + xlrec->urec_len, SizeOfRelUndoRecordHeader); + + if (xlrec->page_offset > BLCKSZ - sizeof(RelUndoPageHeaderData)) + elog(PANIC, "relundo_redo_insert: invalid page offset %u", + xlrec->page_offset); + + if (xlrec->new_pd_lower > BLCKSZ) + elog(PANIC, "relundo_redo_insert: pd_lower %u exceeds page size", + xlrec->new_pd_lower); + + /* Cross-field check: record must fit within page */ + if ((uint32) xlrec->page_offset + (uint32) xlrec->urec_len > BLCKSZ) + elog(PANIC, "relundo_redo_insert: record extends past page end (offset %u + len %u > %u)", + xlrec->page_offset, xlrec->urec_len, (uint32) BLCKSZ); + + /* + * new_pd_lower must be at least as far as the start of the record we are + * inserting. page_offset is page-absolute + * (MAXALIGN(SizeOfPageHeaderData) + contents offset) while new_pd_lower + * is relative to the page contents area, so compare them in the same + * coordinate system by adding the standard page-header size to the + * contents-relative pd_lower. + */ + if (xlrec->new_pd_lower + MAXALIGN(SizeOfPageHeaderData) < xlrec->page_offset) + elog(PANIC, "relundo_redo_insert: new_pd_lower %u precedes page_offset %u", + xlrec->new_pd_lower, xlrec->page_offset); + + /* Validate record type is in valid range */ + if (xlrec->urec_type < RELUNDO_INSERT || xlrec->urec_type > RELUNDO_TUPLE_LOCK) + elog(PANIC, "relundo_redo_insert: invalid record type %u", xlrec->urec_type); + + /* + * Async I/O optimization: when the record touches both the data page + * (block 0) and the metapage (block 1), issue a prefetch for the metapage + * before we read block 0. This allows both I/Os to be in flight + * simultaneously. + * + * Enter batch mode so that the buffer manager can coalesce the I/O + * submissions when using io_method = worker or io_uring. Batch mode is + * only useful when we have multiple blocks to process; for single- block + * records the overhead is not worthwhile. + */ + use_batchmode = has_metapage && (io_method != IOMETHOD_SYNC); + + if (use_batchmode) + pgaio_enter_batchmode(); + + if (has_metapage) + relundo_prefetch_block(record, 1); + + if (XLogRecGetInfo(record) & XLOG_RELUNDO_INIT_PAGE) + { + /* New page: initialize from scratch, then apply insert */ + buf = XLogInitBufferForRedo(record, 0); + action = BLK_NEEDS_REDO; + } + else + { + action = XLogReadBufferForRedo(record, 0, &buf); + } + + if (action == BLK_NEEDS_REDO) + { + Page page = BufferGetPage(buf); + char *record_data; + Size record_len; + + record_data = XLogRecGetBlockData(record, 0, &record_len); + + if (record_data == NULL || record_len == 0) + elog(PANIC, "relundo_redo_insert: no block data for UNDO record"); + + /* Consistency check: verify data length is reasonable */ + if (record_len > BLCKSZ) + elog(PANIC, "relundo_redo_insert: block data too large (%zu bytes)", record_len); + + /* + * If the page was just initialized (INIT_PAGE flag), the block data + * contains both the RelUndoPageHeaderData and the UNDO record. + * Initialize the page structure first, then copy both. + */ + if (XLogRecGetInfo(record) & XLOG_RELUNDO_INIT_PAGE) + { + char *contents; + + /* INIT_PAGE data must include at least the page header */ + if (record_len < SizeOfRelUndoPageHeaderData) + elog(PANIC, "relundo_redo_insert: INIT_PAGE block data too small (%zu < %zu)", + record_len, SizeOfRelUndoPageHeaderData); + + /* Block data plus page header must fit in a page */ + if (record_len > BLCKSZ - MAXALIGN(SizeOfPageHeaderData)) + elog(PANIC, "relundo_redo_insert: INIT_PAGE block data too large (%zu bytes)", + record_len); + + PageInit(page, BLCKSZ, 0); + + /* + * The record_data contains: 1. RelUndoPageHeaderData + * (SizeOfRelUndoPageHeaderData bytes) 2. UNDO record (remaining + * bytes) + * + * Copy both to the page contents area. + */ + contents = PageGetContents(page); + memcpy(contents, record_data, record_len); + } + else + { + RelUndoPageHeader undohdr = (RelUndoPageHeader) PageGetContents(page); + + /* Consistency check: verify pd_lower is reasonable before update */ + if (undohdr->pd_lower > BLCKSZ) + elog(PANIC, "relundo_redo_insert: existing pd_lower %u exceeds page size", + undohdr->pd_lower); + + /* + * Normal case: page already exists, just copy the UNDO record to + * the specified offset. + */ + memcpy((char *) page + xlrec->page_offset, record_data, record_len); + + /* Update the page's free space pointer */ + undohdr->pd_lower = xlrec->new_pd_lower; + + /* Restore the page's max_xid discard watermark. */ + undohdr->max_xid = xlrec->max_xid; + + /* + * Post-condition check: verify pd_lower is reasonable after + * update. pd_lower is contents-relative while page_offset is + * page-absolute, so add the standard page-header size to pd_lower + * before comparing against the page-absolute end of the record. + */ + if (undohdr->pd_lower + MAXALIGN(SizeOfPageHeaderData) < xlrec->page_offset + record_len) + elog(PANIC, "relundo_redo_insert: pd_lower %u too small for offset %u + len %zu", + undohdr->pd_lower, xlrec->page_offset, record_len); + } + + PageSetLSN(page, lsn); + MarkBufferDirty(buf); + } + + if (BufferIsValid(buf)) + UnlockReleaseBuffer(buf); + + /* + * Block 1 (metapage) may also be present if the head pointer was updated. + * If so, restore its FPI. The prefetch issued above should have brought + * the page into cache (or at least started the I/O), so this read should + * complete quickly. + */ + if (has_metapage) + { + action = XLogReadBufferForRedo(record, 1, &buf); + /* Metapage is always logged with FPI, so BLK_RESTORED or BLK_DONE */ + if (BufferIsValid(buf)) + UnlockReleaseBuffer(buf); + } + + if (use_batchmode) + pgaio_exit_batchmode(); +} + +/* + * relundo_redo_discard - Replay UNDO page discard + * + * Discard splices a contiguous run of discardable pages off the tail of the + * data chain directly onto the metapage's free list. The run is already + * internally linked by durable prev_blkno fields, so only the run boundaries + * change. The record carries a bounded set of buffers: + * + * Block 0: the metapage (tail + free-list head) + * Block 1: the run's old-tail page (prev_blkno -> old free head) + * Block 2: the new live tail page (prev_blkno -> Invalid), if any + * + * The metapage is registered REGBUF_STANDARD (not a forced FPI), so on a + * BLK_NEEDS_REDO replay we re-apply the metapage mutations explicitly rather + * than relying on a restored image. + */ +static void +relundo_redo_discard(XLogReaderState *record) +{ + XLogRecPtr lsn = record->EndRecPtr; + Buffer buf; + XLogRedoAction action; + xl_relundo_discard *xlrec = (xl_relundo_discard *) XLogRecGetData(record); + bool whole_chain = !BlockNumberIsValid(xlrec->new_tail_blkno); + + /* + * Consistency check on WAL record data. A run is always at least one + * page; there is no upper bound, since a whole-chain discard of a large + * fork legitimately frees arbitrarily many pages. Do not impose an + * arbitrary ceiling here: do-time has no matching guard, so any cap we + * reject below an achievable run length would turn a logged record into + * one its own crash recovery refuses to replay (a PANIC loop). + */ + if (xlrec->npages_freed == 0) + elog(PANIC, "relundo_redo_discard: npages_freed is zero"); + + /* + * Block 0 is the metapage; the run's old tail is a real data page, so its + * block number must never be the metapage (block 0). + */ + if (xlrec->old_tail_blkno == 0) + elog(PANIC, "relundo_redo_discard: old_tail_blkno is metapage block 0"); + + /* + * new_tail_blkno is either a real data page (>= 1) or InvalidBlockNumber + * when the whole chain was discarded; it must never be the metapage. + */ + if (xlrec->new_tail_blkno == 0) + elog(PANIC, "relundo_redo_discard: new_tail_blkno is metapage block 0"); + + /* Block 0: metapage (tail + free-list head). */ + action = XLogReadBufferForRedo(record, 0, &buf); + + if (action == BLK_NEEDS_REDO) + { + Page page = BufferGetPage(buf); + RelUndoMetaPageData *meta; + + meta = (RelUndoMetaPageData *) PageGetContents(page); + + /* Post-condition checks on metapage */ + if (meta->magic != RELUNDO_METAPAGE_MAGIC) + elog(PANIC, "relundo_redo_discard: metapage has invalid magic 0x%X", + meta->magic); + + if (meta->counter > 65535) + elog(PANIC, "relundo_redo_discard: counter %u exceeds maximum", + meta->counter); + + /* Advance the live data chain tail (and head if it is now empty). */ + meta->tail_blkno[xlrec->slot] = xlrec->new_tail_blkno; + if (whole_chain) + meta->head_blkno[xlrec->slot] = InvalidBlockNumber; + + /* Splice the run directly onto the free list. */ + meta->free_blkno = xlrec->free_head_blkno; + meta->discarded_records += xlrec->npages_freed; + + PageSetLSN(page, lsn); + MarkBufferDirty(buf); + } + + if (BufferIsValid(buf)) + UnlockReleaseBuffer(buf); + + /* + * Block 1: the run's old-tail page, whose prev_blkno now links to the old + * free-list head (appending the prior free list after the run). + */ + action = XLogReadBufferForRedo(record, 1, &buf); + if (action == BLK_NEEDS_REDO) + { + Page page = BufferGetPage(buf); + RelUndoPageHeader hdr = (RelUndoPageHeader) PageGetContents(page); + + hdr->prev_blkno = xlrec->old_free_head; + PageSetLSN(page, lsn); + MarkBufferDirty(buf); + } + if (BufferIsValid(buf)) + UnlockReleaseBuffer(buf); + + /* + * Block 2: the new live tail page (present only when the chain is not + * fully discarded), whose prev_blkno is cleared to detach it from the + * run. + */ + if (!whole_chain && XLogRecHasBlockRef(record, 2)) + { + action = XLogReadBufferForRedo(record, 2, &buf); + if (action == BLK_NEEDS_REDO) + { + Page page = BufferGetPage(buf); + RelUndoPageHeader hdr = (RelUndoPageHeader) PageGetContents(page); + + hdr->prev_blkno = InvalidBlockNumber; + PageSetLSN(page, lsn); + MarkBufferDirty(buf); + } + if (BufferIsValid(buf)) + UnlockReleaseBuffer(buf); + } +} + +/* + * relundo_redo_truncate - Replay physical truncation of an emptied UNDO fork + * + * The do-time path (RelUndoTruncateEmptyChain) logs this only when a discard + * empties the whole data chain, at which point the free list -- and thus every + * allocated data block -- is the contiguous physical suffix [1 .. watermark]. + * Redo restores the metapage (block 0) with its free-list head and allocation + * watermark reset, then drops the truncated buffers and shrinks the fork file. + * + * The metapage is registered REGBUF_STANDARD, so on BLK_NEEDS_REDO we re-apply + * the field resets explicitly rather than relying on a restored image. + */ +static void +relundo_redo_truncate(XLogReaderState *record) +{ + XLogRecPtr lsn = record->EndRecPtr; + xl_relundo_truncate *xlrec = (xl_relundo_truncate *) XLogRecGetData(record); + Buffer buf; + XLogRedoAction action; + RelFileLocator rlocator; + ForkNumber forknum; + BlockNumber blkno; + SMgrRelation reln; + BlockNumber old_nblocks; + BlockNumber new_nblocks = xlrec->new_nblocks; + + /* The fork is always truncated back to just the metapage (block 0). */ + if (new_nblocks != 1) + elog(PANIC, "relundo_redo_truncate: unexpected new_nblocks %u", + new_nblocks); + + /* Block 0: metapage (free-list head + watermark reset). */ + action = XLogReadBufferForRedo(record, 0, &buf); + + if (action == BLK_NEEDS_REDO) + { + Page page = BufferGetPage(buf); + RelUndoMetaPageData *meta; + + meta = (RelUndoMetaPageData *) PageGetContents(page); + + if (meta->magic != RELUNDO_METAPAGE_MAGIC) + elog(PANIC, "relundo_redo_truncate: metapage has invalid magic 0x%X", + meta->magic); + + /* + * The discard that triggered this truncation already emptied the data + * chain; the freed pages no longer exist, so clear the free-list head + * and the allocation watermark too. + */ + for (int slot = 0; slot < RELUNDO_NUM_HEADS; slot++) + { + meta->head_blkno[slot] = InvalidBlockNumber; + meta->tail_blkno[slot] = InvalidBlockNumber; + } + meta->free_blkno = InvalidBlockNumber; + meta->system_alloc_watermark = InvalidBlockNumber; + + PageSetLSN(page, lsn); + MarkBufferDirty(buf); + } + + if (BufferIsValid(buf)) + UnlockReleaseBuffer(buf); + + /* + * Now physically truncate the fork. Resolve the relation from block 0's + * tag, force-create the fork if it is missing (it may have been dropped + * later in the WAL stream), and mirror smgr_redo's XLOG_SMGR_TRUNCATE + * arm: advance the minimum recovery point, then smgrtruncate (which drops + * the to-be-removed buffers via DropRelationBuffers). + */ + XLogRecGetBlockTag(record, 0, &rlocator, &forknum, &blkno); + Assert(forknum == RELUNDO_FORKNUM); + reln = smgropen(rlocator, INVALID_PROC_NUMBER); + smgrcreate(reln, forknum, true); + + XLogFlush(lsn); + + old_nblocks = smgrnblocks(reln, forknum); + if (old_nblocks > new_nblocks) + { + /* Tell xlogutils.c so cached relation sizes stay consistent. */ + XLogTruncateRelation(rlocator, forknum, new_nblocks); + + START_CRIT_SECTION(); + smgrtruncate(reln, &forknum, 1, &old_nblocks, &new_nblocks); + END_CRIT_SECTION(); + } +} + +/* + * relundo_redo_apply - Replay a rollback compensation log record (CLR) + * + * The CLR carries full-page images of every data page (main fork) restored + * during online abort plus the UNDO-fork page (carrying RELUNDO_INFO_CLR_APPLIED). + * Each referenced block was registered with REGBUF_FORCE_IMAGE, so redo simply + * restores the recorded images, reinstating the before-image rollback that + * would otherwise be lost when dirty buffers vanish on an immediate crash. + * + * Because every block is logged with a forced FPI, XLogReadBufferForRedo + * returns BLK_RESTORED for blocks needing redo and BLK_DONE for those already + * at or past the record LSN; in neither case is there extra work to do. + */ +static void +relundo_redo_apply(XLogReaderState *record) +{ + int block_id; + int max_block_id = XLogRecMaxBlockId(record); + + for (block_id = 0; block_id <= max_block_id; block_id++) + { + Buffer buf; + + if (!XLogRecHasBlockRef(record, block_id)) + continue; + + (void) XLogReadBufferForRedo(record, (uint8) block_id, &buf); + if (BufferIsValid(buf)) + UnlockReleaseBuffer(buf); + } +} + +/* + * relundo_redo - Main redo dispatch for RM_RELUNDO_ID + */ +void +relundo_redo(XLogReaderState *record) +{ + uint8 info = XLogRecGetInfo(record) & ~XLR_INFO_MASK; + + /* + * Strip XLOG_RELUNDO_INIT_PAGE flag for the switch; it only affects + * INSERT processing. + */ + switch (info & ~XLOG_RELUNDO_INIT_PAGE) + { + case XLOG_RELUNDO_INIT: + relundo_redo_init(record); + break; + + case XLOG_RELUNDO_INSERT: + relundo_redo_insert(record); + break; + + case XLOG_RELUNDO_DISCARD: + relundo_redo_discard(record); + break; + + case XLOG_RELUNDO_TRUNCATE: + relundo_redo_truncate(record); + break; + + case XLOG_RELUNDO_APPLY: + relundo_redo_apply(record); + break; + + default: + elog(PANIC, "relundo_redo: unknown op code %u", info); + } +} + +/* + * relundo_startup - Initialize per-backend state for parallel redo + * + * Called once per backend at the start of parallel WAL replay. + * We don't currently need any special per-backend state for per-relation UNDO, + * but this hook is required for parallel redo support. + */ +void +relundo_startup(void) +{ + /* + * No per-backend initialization needed currently. If we add backend-local + * caches or state in the future, initialize them here. + */ +} + +/* + * relundo_cleanup - Clean up per-backend state after parallel redo + * + * Called once per backend at the end of parallel WAL replay. + * Counterpart to relundo_startup(). + */ +void +relundo_cleanup(void) +{ + /* + * No per-backend cleanup needed currently. If relundo_startup() + * initializes any resources, release them here. + */ +} + +/* + * relundo_mask - Mask non-critical page fields for consistency checking + * + * During parallel redo, pages may be replayed in different order across + * backends. This function masks out fields that may differ but do not + * indicate corruption, so that page comparisons (e.g. by pg_waldump + * --check) avoid false positives. + * + * We use the standard mask_page_lsn_and_checksum() helper from bufmask.h, + * matching the convention used by heap, btree, and other resource managers. + * + * RelUndo pages do not use the standard line-pointer layout, so we cannot + * call mask_unused_space() (which operates on the standard PageHeader's + * pd_lower/pd_upper). Instead, for data pages we mask the free space + * tracked by the RelUndoPageHeader's own pd_lower and pd_upper fields + * within the contents area. + */ +void +relundo_mask(char *pagedata, BlockNumber blkno) +{ + Page page = (Page) pagedata; + + /* + * Mask LSN and checksum -- these may differ across parallel redo workers + * due to replay ordering. + */ + mask_page_lsn_and_checksum(page); + + if (blkno == 0) + { + /* + * Metapage: do not mask magic, version, counter, or block pointers. + * Those must match exactly for consistency. LSN and checksum are + * already masked above. + */ + } + else + { + /* + * Data page: mask unused space between the UNDO page header's + * pd_lower (next insertion point) and pd_upper (end of usable space). + * This region may contain stale data from prior page reuse and is not + * meaningful for consistency. + * + * The RelUndoPageHeader sits at the start of the page contents area + * (after the standard PageHeaderData). Its pd_lower and pd_upper are + * offsets relative to the contents area. + */ + RelUndoPageHeader undohdr = (RelUndoPageHeader) PageGetContents(page); + char *contents = (char *) PageGetContents(page); + int lower = undohdr->pd_lower; + int upper = undohdr->pd_upper; + + if (lower < upper) + memset(contents + lower, MASK_MARKER, upper - lower); + } +} diff --git a/src/backend/access/undo/slog.c b/src/backend/access/undo/slog.c new file mode 100644 index 0000000000000..8b5335225d6be --- /dev/null +++ b/src/backend/access/undo/slog.c @@ -0,0 +1,805 @@ +/*------------------------------------------------------------------------- + * + * slog.c + * Secondary Log (sLog) -- transaction Aborted Transaction Map (ATM) + * + * The sLog tracks aborted transactions in shared memory for the UNDO + * subsystem's Constant-Time Recovery. + * + * Transaction sLog (adaptive radix tree): + * - A shared-memory adaptive radix tree (radixtree.h, RT_SHMEM) keyed by + * (xid, reloid) packed into a uint64 stores full abort metadata. With + * xid in the high bits, ordered iteration groups all entries for a + * given xid contiguously, enabling efficient per-xid range operations. + * - The tree lives in the sLog's DSA area and grows on demand. + * - The radix tree is protected by a single LWLock (sLog modifications + * only occur on transaction abort, an uncommon path). + * + * An optional per-tuple flat-hash extension (see the tuple sLog, added by + * the consumer that needs bounded-recovery uncommitted-writer tracking) + * shares this file's shared-memory segment and initialization but is not + * required by the UNDO core. + * + * Locking: Transaction sLog uses LWTRANCHE_SLOG. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/access/undo/slog.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include +#ifdef WIN32 +#include +#endif + +#include "access/relundo.h" +#include "access/slog.h" +#include "access/transam.h" +#include "access/xact.h" +#include "common/hashfn.h" +#include "nodes/lockoptions.h" +#include "miscadmin.h" +#include "storage/lock.h" +#include "storage/lwlock.h" +#include "storage/off.h" +#include "storage/proc.h" +#include "storage/procarray.h" +#include "storage/shmem.h" +#include "storage/spin.h" +#include "storage/subsystems.h" +#include "utils/dsa.h" +#include "utils/memutils.h" +#include "utils/snapmgr.h" +#include "utils/timestamp.h" + +/* ---------------------------------------------------------------- + * Adaptive radix tree instantiation for the transaction sLog (ATM) + * + * The Aborted Transaction Map is keyed by a single uint64 formed from + * (xid, reloid): key = (xid << 32) | reloid. Both are uint32 and the + * two halves pack exactly. Because xid occupies the high 32 bits, the + * radix tree's ordered iteration groups all entries for one xid + * contiguously (xid-major, reloid-minor ascending) -- the same ordering + * property the old skip-list provided, which the per-xid range + * operations below rely on. + * + * The tree lives in the sLog's DSA area so it grows on demand rather + * than reserving a fixed shared-memory pool. All access is serialized externally by the + * existing txn_lock (LWLock): SET/DELETE under LW_EXCLUSIVE, FIND and + * iteration under LW_SHARED. We do not use the radix tree's own + * internal lock (RT_LOCK_*), since txn_lock already provides the needed + * serialization. + * ---------------------------------------------------------------- + */ + +/* + * slog_atm_value_t - value stored in the ATM radix tree. + * + * The key (xid, reloid) is implicit in the tree path, so only the data + * fields are stored here. + */ +typedef struct slog_atm_value_t +{ + XLogRecPtr last_batch_lsn; /* LSN of last UNDO batch for this xid */ + Oid dboid; + TimestampTz abort_time; + bool revert_complete; +} slog_atm_value_t; + +#define RT_PREFIX slog_atm +#define RT_SCOPE static pg_attribute_unused() +#define RT_DECLARE +#define RT_DEFINE +#define RT_VALUE_TYPE slog_atm_value_t +#define RT_SHMEM +#define RT_USE_DELETE +#include "lib/radixtree.h" + +/* ATM key encoding: xid in the high 32 bits, reloid in the low 32 bits. */ +static inline uint64 +slog_atm_key(TransactionId xid, Oid reloid) +{ + return ((uint64) xid << 32) | (uint64) reloid; +} + +static inline TransactionId +slog_atm_key_xid(uint64 key) +{ + return (TransactionId) (key >> 32); +} + +static inline Oid +slog_atm_key_reloid(uint64 key) +{ + return (Oid) (key & 0xFFFFFFFFu); +} + +/* + * Initial size for the sLog DSA area (backs the aborted-txn radix tree). + * Grows dynamically as needed up to slog_dsa_max_size_mb. + */ +#define SLOG_DSA_INIT_SIZE (512 * 1024) /* 512 KB */ +#define SLOG_DSA_MAX_SIZE_MB 256 /* default max: 256 MB */ + +/* ---------------------------------------------------------------- + * Shared state definition + * ---------------------------------------------------------------- + */ +typedef struct SLogSharedState +{ + /* Transaction ATM (adaptive radix tree in the DSA area below) */ + dsa_pointer atm_handle; /* RT handle; InvalidDsaPointer until init */ + LWLockPadded txn_lock; /* single LWLock serializing ATM access */ + + /* DSA area backing the aborted-txn radix tree */ + dsa_area *dsa_area; /* set during SLogShmemInit, NULL until then */ + char dsa_space[SLOG_DSA_INIT_SIZE]; +} SLogSharedState; + +/* GUC: maximum sLog DSA area size (in MB) */ +int slog_dsa_max_size_mb = SLOG_DSA_MAX_SIZE_MB; + +/* ---------------------------------------------------------------- + * Static variables + * ---------------------------------------------------------------- + */ +static SLogSharedState *SLogState = NULL; + +/* Per-backend DSA attachment (lazy, via SLogEnsureDsaAttached) */ +static dsa_area *slog_dsa_handle = NULL; + +/* Per-backend attached ATM radix tree (lazy, via slog_atm_tree) */ +static slog_atm_radix_tree *slog_atm_tree_local = NULL; + + +/* + * slog_atm_tree + * Return this backend's attached ATM radix tree, attaching lazily. + * + * Ensures the sLog DSA area is attached, then attaches to the shared + * radix tree via its handle. The attached wrapper is cached for the + * life of the backend (the control block lives in the pinned DSA area). + * The caller must hold txn_lock across any use of the returned tree. + */ +static slog_atm_radix_tree * +slog_atm_tree(void) +{ + MemoryContext oldcxt; + + if (slog_atm_tree_local != NULL) + return slog_atm_tree_local; + + SLogEnsureDsaAttached(); + if (slog_dsa_handle == NULL) + elog(PANIC, "sLog: DSA not attached for ATM radix tree"); + + /* + * Attach in TopMemoryContext so the wrapper persists across transactions, + * mirroring the DSA attach above. + */ + oldcxt = MemoryContextSwitchTo(TopMemoryContext); + slog_atm_tree_local = slog_atm_attach(slog_dsa_handle, + SLogState->atm_handle); + MemoryContextSwitchTo(oldcxt); + + return slog_atm_tree_local; +} + +/* ---------------------------------------------------------------- + * Shared memory sizing and initialization + * ---------------------------------------------------------------- + */ + +/* + * SLogShmemSize + * Calculate shared memory needed for the sLog. + * + * Note: The DSA initial region is embedded in SLogSharedState (dsa_space[]), + * so sizeof(SLogSharedState) already includes SLOG_DSA_INIT_SIZE. The ATM + * radix tree is allocated from that DSA area on demand, so it needs no fixed + * reservation here. + */ +Size +SLogShmemSize(void) +{ + return MAXALIGN(sizeof(SLogSharedState)); +} + +/* + * SLogShmemRequest + * Register shared memory needs for the sLog. + * + * Registers the shared state struct (which embeds the DSA initial region + * backing the aborted-transaction radix tree). + */ +void +SLogShmemRequest(void) +{ + /* Register the shared state structure */ + ShmemRequestStruct(.name = "Secondary Log State", + .size = sizeof(SLogSharedState), + .ptr = (void **) &SLogState, + ); +} + +/* + * SLogShmemInit + * Initialize sLog shared memory contents. + * + * Called during the init_fn phase. The framework has already allocated + * SLogState. We initialize the transaction ATM radix tree and its DSA area. + */ +void +SLogShmemInit(void) +{ + /* ATM radix tree is created below, after the DSA area exists. */ + SLogState->atm_handle = InvalidDsaPointer; + + /* ---- Initialize locks ---- */ + LWLockInitialize(&SLogState->txn_lock.lock, LWTRANCHE_SLOG); + + /* ---- Initialize DSA area (backs the aborted-txn radix tree) ---- */ + SLogState->dsa_area = dsa_create_in_place(SLogState->dsa_space, + SLOG_DSA_INIT_SIZE, + LWTRANCHE_SLOG, + 0); + dsa_pin(SLogState->dsa_area); + dsa_set_size_limit(SLogState->dsa_area, + (Size) slog_dsa_max_size_mb * 1024 * 1024); + + /* + * Create the ATM radix tree in the DSA area and record its handle. The + * transient tree wrapper returned here is discarded when init's memory + * context is reset; the control block lives in the pinned DSA area, and + * each backend re-attaches lazily via slog_atm_tree(). We reuse + * LWTRANCHE_SLOG for the tree's internal lock (unused -- txn_lock + * serializes all access -- but a valid tranche is required). + */ + { + slog_atm_radix_tree *tree; + + tree = slog_atm_create(SLogState->dsa_area, LWTRANCHE_SLOG); + SLogState->atm_handle = slog_atm_get_handle(tree); + } + + dsa_detach(SLogState->dsa_area); + SLogState->dsa_area = NULL; /* backends re-attach lazily */ +} + +/* + * SLogShmemRequest_cb / SLogShmemInit_cb + * ShmemCallbacks adapters for SLogShmemRequest()/SLogShmemInit(). + * + * The sLog is registered as its own PG_SHMEM_SUBSYSTEM entry (see + * storage/subsystemlist.h) rather than being sized/initialized from within + * the generic UNDO subsystem's callback, so the UNDO core has no + * compile-time or link-time dependency on any consumer. + * + * No .attach_fn is provided: SLogShmemInit() has no found-guard, so the + * framework re-attaches EXEC_BACKEND children to the already-initialized + * shared struct via the ShmemRequestStruct(.ptr=&SLogState) registration + * in SLogShmemRequest(). + */ +static void +SLogShmemRequest_cb(void *arg) +{ + SLogShmemRequest(); +} + +static void +SLogShmemInit_cb(void *arg) +{ + SLogShmemInit(); +} + +const ShmemCallbacks SLogShmemCallbacks = { + .request_fn = SLogShmemRequest_cb, + .init_fn = SLogShmemInit_cb, +}; + +/* ---------------------------------------------------------------- + * DSA lifecycle for the shared sLog DSA area + * ---------------------------------------------------------------- + */ + +/* + * SLogEnsureDsaAttached + * Lazy per-backend DSA attachment. + * + * Must be called before any DSA alloc/free/get_address operations. + * Safe to call multiple times (no-op after first attach). + */ +void +SLogEnsureDsaAttached(void) +{ + MemoryContext oldcxt; + + if (slog_dsa_handle != NULL) + return; /* already attached */ + + if (SLogState == NULL) + return; /* sLog not yet initialized */ + + /* + * Allocate in TopMemoryContext so the dsa_area struct persists across + * transactions. dsa_attach_in_place internally palloc's, so if we're in + * a transaction context, the handle would become a dangling pointer after + * transaction end. + */ + oldcxt = MemoryContextSwitchTo(TopMemoryContext); + slog_dsa_handle = dsa_attach_in_place(SLogState->dsa_space, NULL); + dsa_pin_mapping(slog_dsa_handle); + MemoryContextSwitchTo(oldcxt); +} + +/* ================================================================ + * Transaction sLog functions + * ================================================================ + */ + +/* + * SLogTxnInsert + * Insert an aborted transaction entry into the sLog. + * + * Creates an entry in the ATM radix tree. The tree grows on demand from + * the DSA area, so unlike the old fixed pool this never fails for lack of + * space; the return type is retained for API compatibility and is always + * true. + */ +bool +SLogTxnInsert(TransactionId xid, Oid reloid, Oid dboid, + XLogRecPtr last_batch_lsn) +{ + slog_atm_radix_tree *tree = slog_atm_tree(); + uint64 key = slog_atm_key(xid, reloid); + slog_atm_value_t value; + + value.last_batch_lsn = last_batch_lsn; + value.dboid = dboid; + value.abort_time = GetCurrentTimestamp(); + value.revert_complete = false; + + LWLockAcquire(&SLogState->txn_lock.lock, LW_EXCLUSIVE); + + /* + * RT_SET returns whether the key already existed. A duplicate insert is + * a no-op on the (xid, reloid) identity; the original entry's data fields + * would be overwritten, so preserve the old behavior of leaving an + * existing entry untouched by finding first. + */ + if (slog_atm_find(tree, key) == NULL) + (void) slog_atm_set(tree, key, &value); + + LWLockRelease(&SLogState->txn_lock.lock); + return true; +} + +/* + * SLogTxnLookup + * Look up a specific (xid, reloid) entry. + * + * Returns true if found, copying the entry into *entry_out. + */ +bool +SLogTxnLookup(TransactionId xid, Oid reloid, SLogTxnEntry *entry_out) +{ + slog_atm_radix_tree *tree = slog_atm_tree(); + uint64 key = slog_atm_key(xid, reloid); + slog_atm_value_t *found; + + LWLockAcquire(&SLogState->txn_lock.lock, LW_SHARED); + + found = slog_atm_find(tree, key); + + if (found != NULL && entry_out != NULL) + { + entry_out->xid = xid; + entry_out->reloid = reloid; + entry_out->last_batch_lsn = found->last_batch_lsn; + entry_out->dboid = found->dboid; + entry_out->abort_time = found->abort_time; + entry_out->revert_complete = found->revert_complete; + } + + LWLockRelease(&SLogState->txn_lock.lock); + + return (found != NULL); +} + +/* + * SLogTxnLookupByXid + * Find the UNDO chain for a given xid (any reloid). + * + * The radix tree has no seek-to-key API, so we iterate its entries in + * ascending key order (xid-major) and return the first one whose xid + * matches. Because keys are xid-major we can stop as soon as we pass + * the target xid. The ATM only holds un-reverted aborted transactions, + * so it is small and a full scan is acceptable on this cold path. + */ +bool +SLogTxnLookupByXid(TransactionId xid, XLogRecPtr *lsn_out) +{ + slog_atm_radix_tree *tree = slog_atm_tree(); + slog_atm_iter *iter; + slog_atm_value_t *val; + uint64 k; + bool result = false; + + LWLockAcquire(&SLogState->txn_lock.lock, LW_SHARED); + + iter = slog_atm_begin_iterate(tree); + while ((val = slog_atm_iterate_next(iter, &k)) != NULL) + { + TransactionId k_xid = slog_atm_key_xid(k); + + if (k_xid < xid) + continue; + if (k_xid > xid) + break; /* passed the target range */ + + if (lsn_out) + *lsn_out = val->last_batch_lsn; + result = true; + break; + } + slog_atm_end_iterate(iter); + + LWLockRelease(&SLogState->txn_lock.lock); + return result; +} + +/* + * SLogTxnRemove + * Remove a specific (xid, reloid) entry. + */ +void +SLogTxnRemove(TransactionId xid, Oid reloid) +{ + slog_atm_radix_tree *tree = slog_atm_tree(); + uint64 key = slog_atm_key(xid, reloid); + + LWLockAcquire(&SLogState->txn_lock.lock, LW_EXCLUSIVE); + + (void) slog_atm_delete(tree, key); + + LWLockRelease(&SLogState->txn_lock.lock); +} + +/* + * SLogTxnRemoveByXid + * Remove all sLog entries for a given transaction ID. + * + * Collects all keys for this xid into a local array during an ascending + * iteration (keys are contiguous thanks to the xid-major ordering), then + * deletes them after ending the iteration -- we do not delete while + * iterating. The ATM is small, so the full scan is acceptable. + */ +void +SLogTxnRemoveByXid(TransactionId xid) +{ + slog_atm_radix_tree *tree = slog_atm_tree(); + slog_atm_iter *iter; + slog_atm_value_t *val; + uint64 k; + uint64 *to_remove; + int nremove = 0; + int max_remove = 64; + int i; + + to_remove = (uint64 *) palloc(max_remove * sizeof(uint64)); + + LWLockAcquire(&SLogState->txn_lock.lock, LW_EXCLUSIVE); + + /* Collect all keys with matching xid */ + iter = slog_atm_begin_iterate(tree); + while ((val = slog_atm_iterate_next(iter, &k)) != NULL) + { + TransactionId k_xid = slog_atm_key_xid(k); + + (void) val; + if (k_xid < xid) + continue; + if (k_xid > xid) + break; + + if (nremove >= max_remove) + { + max_remove *= 2; + to_remove = (uint64 *) repalloc(to_remove, + max_remove * sizeof(uint64)); + } + to_remove[nremove++] = k; + } + slog_atm_end_iterate(iter); + + /* Remove collected keys */ + for (i = 0; i < nremove; i++) + (void) slog_atm_delete(tree, to_remove[i]); + + LWLockRelease(&SLogState->txn_lock.lock); + + pfree(to_remove); +} + +/* + * SLogTxnMarkReverted + * Mark all entries for a given xid as revert_complete. + * + * Iterates the tree in ascending key order and rewrites the value of each + * entry for this xid with revert_complete = true. Keys for one xid are + * contiguous, so we stop once we pass the target xid. We collect the + * matching (key, value) pairs first and RT_SET them after ending the + * iteration, so the tree is never mutated while an iterator is live. + */ +void +SLogTxnMarkReverted(TransactionId xid) +{ + slog_atm_radix_tree *tree = slog_atm_tree(); + slog_atm_iter *iter; + slog_atm_value_t *val; + uint64 k; + uint64 *keys; + slog_atm_value_t *vals; + int n = 0; + int max_n = 64; + int i; + + keys = (uint64 *) palloc(max_n * sizeof(uint64)); + vals = (slog_atm_value_t *) palloc(max_n * sizeof(slog_atm_value_t)); + + LWLockAcquire(&SLogState->txn_lock.lock, LW_EXCLUSIVE); + + /* + * Collect matching entries first, then RT_SET them after ending the + * iteration, to avoid mutating the tree while an iterator is live. + */ + iter = slog_atm_begin_iterate(tree); + while ((val = slog_atm_iterate_next(iter, &k)) != NULL) + { + TransactionId k_xid = slog_atm_key_xid(k); + + if (k_xid < xid) + continue; + if (k_xid > xid) + break; + + if (n >= max_n) + { + max_n *= 2; + keys = (uint64 *) repalloc(keys, max_n * sizeof(uint64)); + vals = (slog_atm_value_t *) repalloc(vals, + max_n * sizeof(slog_atm_value_t)); + } + keys[n] = k; + vals[n] = *val; + vals[n].revert_complete = true; + n++; + } + slog_atm_end_iterate(iter); + + for (i = 0; i < n; i++) + (void) slog_atm_set(tree, keys[i], &vals[i]); + + LWLockRelease(&SLogState->txn_lock.lock); + + pfree(keys); + pfree(vals); +} + +/* + * SLogTxnGetNextUnreverted + * Find an entry that hasn't been reverted yet. + * + * Iterates the tree in ascending key order (xid-major), returning the + * first entry with revert_complete == false. + */ +bool +SLogTxnGetNextUnreverted(TransactionId *xid_out, Oid *dboid_out, + XLogRecPtr *lsn_out) +{ + slog_atm_radix_tree *tree = slog_atm_tree(); + slog_atm_iter *iter; + slog_atm_value_t *val; + uint64 k; + bool result = false; + + LWLockAcquire(&SLogState->txn_lock.lock, LW_SHARED); + + iter = slog_atm_begin_iterate(tree); + while ((val = slog_atm_iterate_next(iter, &k)) != NULL) + { + if (!val->revert_complete) + { + *xid_out = slog_atm_key_xid(k); + *dboid_out = val->dboid; + *lsn_out = val->last_batch_lsn; + result = true; + break; + } + } + slog_atm_end_iterate(iter); + + LWLockRelease(&SLogState->txn_lock.lock); + return result; +} + +/* + * SLogTxnCollectUnrevertedDatabases + * Collect the distinct database OIDs that have at least one unreverted + * ATM entry. + * + * Fills dboids[] (caller-provided, capacity max_dboids) with the distinct + * OIDs and returns the count. Used by the logical revert launcher so it + * spawns a per-database worker only for databases that actually have + * aborted-transaction UNDO to apply, rather than for every database in the + * cluster. The set of databases with unreverted entries is tiny, so a + * linear dedup against the collected prefix is adequate. + */ +int +SLogTxnCollectUnrevertedDatabases(Oid *dboids, int max_dboids) +{ + slog_atm_radix_tree *tree = slog_atm_tree(); + slog_atm_iter *iter; + slog_atm_value_t *val; + uint64 k; + int n = 0; + + LWLockAcquire(&SLogState->txn_lock.lock, LW_SHARED); + + iter = slog_atm_begin_iterate(tree); + while ((val = slog_atm_iterate_next(iter, &k)) != NULL) + { + int i; + bool seen = false; + + if (val->revert_complete) + continue; + + for (i = 0; i < n; i++) + { + if (dboids[i] == val->dboid) + { + seen = true; + break; + } + } + if (!seen && n < max_dboids) + dboids[n++] = val->dboid; + } + slog_atm_end_iterate(iter); + + LWLockRelease(&SLogState->txn_lock.lock); + return n; +} + +/* + * SLogRecoveryFinalize + * Count entries after recovery for logging. + */ +void +SLogRecoveryFinalize(int *total_out, int *unreverted_out) +{ + slog_atm_radix_tree *tree = slog_atm_tree(); + slog_atm_iter *iter; + slog_atm_value_t *val; + uint64 k; + int total = 0; + int unreverted = 0; + + LWLockAcquire(&SLogState->txn_lock.lock, LW_SHARED); + + iter = slog_atm_begin_iterate(tree); + while ((val = slog_atm_iterate_next(iter, &k)) != NULL) + { + total++; + if (!val->revert_complete) + unreverted++; + } + slog_atm_end_iterate(iter); + + LWLockRelease(&SLogState->txn_lock.lock); + + if (total_out) + *total_out = total; + if (unreverted_out) + *unreverted_out = unreverted; +} + +/* + * SLogTxnGetOldestUnrevertedLSN + * Return the minimum last_batch_lsn across all unreverted entries. + * + * Used by the WAL retention logic to prevent recycling WAL segments that + * still contain UNDO batches needed by the logical revert worker. + * Returns InvalidXLogRecPtr if no unreverted entries exist. + */ +XLogRecPtr +SLogTxnGetOldestUnrevertedLSN(void) +{ + slog_atm_radix_tree *tree = slog_atm_tree(); + slog_atm_iter *iter; + slog_atm_value_t *val; + uint64 k; + XLogRecPtr oldest = InvalidXLogRecPtr; + + LWLockAcquire(&SLogState->txn_lock.lock, LW_SHARED); + + iter = slog_atm_begin_iterate(tree); + while ((val = slog_atm_iterate_next(iter, &k)) != NULL) + { + if (!val->revert_complete && + XLogRecPtrIsValid(val->last_batch_lsn)) + { + if (!XLogRecPtrIsValid(oldest) || + val->last_batch_lsn < oldest) + oldest = val->last_batch_lsn; + } + } + slog_atm_end_iterate(iter); + + LWLockRelease(&SLogState->txn_lock.lock); + return oldest; +} + +/* + * SLogTxnSnapshotForCheckpoint + * Copy every ATM entry into a palloc'd array for durable checkpointing. + * + * Returns the number of entries and stores a palloc'd array of SLogTxnEntry + * in *entries_out (NULL if there are none). The caller owns the array and + * must pfree it. Runs outside any critical section (CheckPointATM), so + * palloc is safe. + */ +int +SLogTxnSnapshotForCheckpoint(SLogTxnEntry **entries_out) +{ + slog_atm_radix_tree *tree = slog_atm_tree(); + slog_atm_iter *iter; + slog_atm_value_t *val; + uint64 k; + SLogTxnEntry *arr = NULL; + int count = 0; + int capacity = 64; + + *entries_out = NULL; + + arr = (SLogTxnEntry *) palloc(sizeof(SLogTxnEntry) * capacity); + + LWLockAcquire(&SLogState->txn_lock.lock, LW_SHARED); + + iter = slog_atm_begin_iterate(tree); + while ((val = slog_atm_iterate_next(iter, &k)) != NULL) + { + if (count >= capacity) + { + capacity *= 2; + arr = (SLogTxnEntry *) repalloc(arr, + sizeof(SLogTxnEntry) * capacity); + } + + arr[count].xid = slog_atm_key_xid(k); + arr[count].reloid = slog_atm_key_reloid(k); + arr[count].last_batch_lsn = val->last_batch_lsn; + arr[count].dboid = val->dboid; + arr[count].abort_time = val->abort_time; + arr[count].revert_complete = val->revert_complete; + count++; + } + slog_atm_end_iterate(iter); + + LWLockRelease(&SLogState->txn_lock.lock); + + if (count == 0) + { + pfree(arr); + return 0; + } + + *entries_out = arr; + return count; +} diff --git a/src/backend/access/undo/undo.c b/src/backend/access/undo/undo.c new file mode 100644 index 0000000000000..0cd7576465f9f --- /dev/null +++ b/src/backend/access/undo/undo.c @@ -0,0 +1,278 @@ +/*------------------------------------------------------------------------- + * + * undo.c + * Common undo layer coordination + * + * The undo subsystem consists of several logically separate subsystems + * that work together to achieve a common goal. The code in this file + * provides a limited amount of common infrastructure that can be used + * by all of those various subsystems, and helps coordinate activities + * such as shared memory initialization and startup/shutdown. + * + * This file has no compile-time or link-time dependency on any specific + * table or index access method. Shared memory for AM-specific tracking + * structures (such as an optional per-tuple tracking hash an AM may register) + * is registered as its own sibling entry in storage/subsystemlist.h, not + * sized or initialized here. Likewise, the + * set of UNDO resource managers to register comes from access/undormgrlist.h + * (mirroring the subsystemlist.h idiom): a new consumer adds itself to that + * list, and never edits this file. + * + * Shared memory initialization uses the PG_SHMEM_SUBSYSTEM pattern: + * UndoShmemCallbacks is registered in subsystemlist.h, and the framework + * calls UndoShmemRequest() and UndoShmemInit() at the appropriate times + * during postmaster startup. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/backend/access/undo/undo.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/atm.h" +#include "access/logical_revert_worker.h" +#include "access/relundo_worker.h" +#include "access/undo.h" +#include "access/undolog.h" +#include "access/undormgr.h" +#include "access/undormgrs.h" +#include "access/undoworker.h" +#include "access/xactundo.h" +#include "miscadmin.h" +#include "storage/ipc.h" +#include "storage/shmem.h" +#include "storage/subsystems.h" +#include "utils/memutils.h" + +/* + * UndoContext is a child of TopMemoryContext which is never reset. The only + * reason for having a separate context is to make it easier to spot leaks or + * excessive memory utilization related to undo operations. + */ +MemoryContext UndoContext = NULL; + +static void AtProcExit_Undo(int code, Datum arg); +static void UndoShmemRequest_internal(void *arg); +static void UndoShmemInit_internal(void *arg); +static void UndoShmemAttach_internal(void *arg); +static void RegisterUndoRmgrs(void); + +/* + * ShmemCallbacks for the UNDO subsystem. + * + * Registered via PG_SHMEM_SUBSYSTEM(UndoShmemCallbacks) in subsystemlist.h. + * + * init_fn initializes the contents of all UNDO shared memory structures. + */ +const ShmemCallbacks UndoShmemCallbacks = { + .request_fn = UndoShmemRequest_internal, + .init_fn = UndoShmemInit_internal, + .attach_fn = UndoShmemAttach_internal, +}; + +/* + * UndoShmemSize + * Figure out how much shared memory will be needed for undo. + * + * Each subsystem separately computes the space it requires, and we + * carefully add up those values here. AM-specific tracking structures + * (such as an optional per-tuple tracking hash) are sized by their own + * PG_SHMEM_SUBSYSTEM entry and are not included here. + */ +Size +UndoShmemSize(void) +{ + Size size; + + size = UndoLogShmemSize(); + size = add_size(size, XactUndoShmemSize()); + size = add_size(size, UndoWorkerShmemSize()); + size = add_size(size, RelUndoWorkerShmemSize()); + size = add_size(size, LogicalRevertShmemSize()); + size = add_size(size, ATMShmemSize()); + + return size; +} + +/* + * UndoShmemRequest_internal + * Register shared memory needs for UNDO subsystems. + * + * Called during the request_fn phase of postmaster startup, before shared + * memory is allocated. + */ +static void +UndoShmemRequest_internal(void *arg) +{ + /* + * Register the UNDO background worker. This must happen during the + * request_fn phase (before BackgroundWorkerShmemInit runs in the init_fn + * phase), because RegisterBackgroundWorker() cannot be called after + * BackgroundWorkerShmemInit(). + * + * Use a static flag to ensure we only register once. The request_fn + * callback is called again during postmaster reinitialize (after a child + * crash), and RegisterBackgroundWorker() would fail if called after the + * first shmem init. + */ + { + static bool undo_worker_registered = false; + + /* + * Only the postmaster can register a static background worker. In + * bootstrap and single-user mode (initdb) there is no postmaster, so + * skip registration; RegisterBackgroundWorker() would otherwise just + * emit a LOG and return without registering. + */ + if (!undo_worker_registered && + !IsUnderPostmaster && IsPostmasterEnvironment) + { + UndoWorkerRegister(); + undo_worker_registered = true; + } + } +} + +/* + * UndoShmemInit / UndoShmemInit_internal + * Initialize undo-related shared memory. + * + * Also, perform other initialization steps that need to be done very early. + * This is called once during postmaster startup via the ShmemCallbacks + * framework. + */ +static void +UndoShmemInit_internal(void *arg) +{ + UndoShmemInit(); +} + +void +UndoShmemInit(void) +{ + /* + * Initialize the undo memory context. If it already exists (crash restart + * via reset_shared()), reset it instead. + */ + if (UndoContext) + MemoryContextReset(UndoContext); + else + UndoContext = AllocSetContextCreate(TopMemoryContext, "Undo", + ALLOCSET_DEFAULT_SIZES); + + /* Now give various undo subsystems a chance to initialize. */ + UndoLogShmemInit(); + XactUndoShmemInit(); + UndoWorkerShmemInit(); + RelUndoWorkerShmemInit(); + LogicalRevertShmemInit(); + ATMShmemInit(); + + /* + * Initialize the UNDO resource manager dispatch table and register the + * built-in resource managers listed in access/undormgrlist.h. + */ + RegisterUndoRmgrs(); +} + +/* + * UndoShmemAttach_internal + * Re-establish per-process UNDO state in EXEC_BACKEND children. + * + * Under EXEC_BACKEND (Windows, or --enable-exec-backend builds) a child does + * not inherit the postmaster's address space, so module-scope state populated + * by UndoShmemInit() is not present and must be rebuilt here. Two distinct + * kinds of state need re-establishing: + * + * 1. The pointers into shared memory held by the legacy ShmemInitStruct-based + * sub-modules (UndoLogShared, UndoWorkerShmem, WorkQueue, RevertState). + * Re-calling each *ShmemInit() in a child is safe: + * ShmemInitStruct() self-attaches via AttachShmemIndexEntry() when + * IsUnderPostmaster, re-assigns the module global, and reports found=true so + * the one-time "if (!found)" initialization block is correctly skipped. + * + * 2. The UndoRmgrs[] dispatch table and each registered AM's relundo hook + * function pointers, both rebuilt by RegisterUndoRmgrs() (idempotent: + * InitUndoRmgrs() zeroes the table first). + * + * XactUndoShmemInit() and ATMShmemInit() are no-ops and so omitted here. Any + * AM-specific shared structure with the same "no found-guard" hazard + * handles its own EXEC_BACKEND re-attach via its own PG_SHMEM_SUBSYSTEM entry + * (see storage/subsystemlist.h); this function is only responsible for the + * generic UNDO sub-modules listed above. + */ +static void +UndoShmemAttach_internal(void *arg) +{ + UndoLogShmemInit(); + UndoWorkerShmemInit(); + RelUndoWorkerShmemInit(); + LogicalRevertShmemInit(); + + RegisterUndoRmgrs(); +} + +/* + * RegisterUndoRmgrs + * Initialize the UNDO resource manager dispatch table and register the + * built-in resource managers listed in access/undormgrlist.h. + * + * Called from both UndoShmemInit() (postmaster/standalone) and + * UndoShmemAttach_internal() (EXEC_BACKEND children). Idempotent: + * InitUndoRmgrs() zeroes the dispatch table before the *UndoRmgrInit() calls + * re-register, so RegisterUndoRmgr()'s double-registration guard is not + * tripped on a second invocation in the same process. + * + * This function has no compile-time knowledge of which resource managers + * exist: it just expands access/undormgrlist.h through the UNDO_RMGR_INIT + * macro. A new UNDO-writing consumer (a new index AM, table AM, or other + * subsystem) adds one line to that list; this function and the rest of + * undo.c are never modified. + */ +static void +RegisterUndoRmgrs(void) +{ + /* + * Initialize the UNDO resource manager dispatch table. + */ + InitUndoRmgrs(); + + /* + * Register every built-in resource manager listed in + * access/undormgrlist.h. Some *UndoRmgrInit() implementations also + * install their AM's relundo hooks (see access/relundo.h) as part of + * registration. + */ +#define UNDO_RMGR_INIT(initfunc) \ + initfunc(); +#include "access/undormgrlist.h" +#undef UNDO_RMGR_INIT +} + +/* + * InitializeUndo + * Per-backend initialization for the undo subsystem. + * + * Called once per backend from InitPostgres(). + */ +void +InitializeUndo(void) +{ + InitializeXactUndo(); + on_shmem_exit(AtProcExit_Undo, 0); +} + +/* + * AtProcExit_Undo + * Shut down undo subsystems in the correct order. + * + * Higher-level stuff should be shut down first. + */ +static void +AtProcExit_Undo(int code, Datum arg) +{ + AtProcExit_XactUndo(); +} diff --git a/src/backend/access/undo/undo_bufmgr.c b/src/backend/access/undo/undo_bufmgr.c new file mode 100644 index 0000000000000..1d35cde5596f1 --- /dev/null +++ b/src/backend/access/undo/undo_bufmgr.c @@ -0,0 +1,250 @@ +/*------------------------------------------------------------------------- + * + * undo_bufmgr.c + * UNDO log buffer manager integration with PostgreSQL's shared_buffers + * + * This module routes undo log I/O through PostgreSQL's standard + * shared buffer pool. The approach follows ZHeap's design where undo + * data is "accessed through the buffer pool ... similar to regular + * relation data" (ZHeap README, lines 30-40). + * + * Each undo log is mapped to a virtual RelFileLocator: + * + * spcOid = UNDO_DEFAULT_TABLESPACE_OID (pg_default, 1663) + * dbOid = UNDO_DB_OID (pseudo-database 9) + * relNumber = undo log number + * + * This virtual locator is used with ReadBufferWithoutRelcache() to + * read/write undo blocks through the shared buffer pool. The fork + * number MAIN_FORKNUM is used (following ZHeap's UndoLogForkNum + * convention), and undo buffers are distinguished from regular data + * by the UNDO_DB_OID in the BufferTag's dbOid field. + * + * Benefits: + * - Unified buffer management (no separate cache to tune) + * - Automatic clock-sweep eviction via shared_buffers + * - Built-in dirty buffer tracking and checkpoint support + * - WAL integration for crash safety + * - Standard buffer locking and pin semantics + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/access/undo/undo_bufmgr.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "storage/buf_internals.h" + +#include "access/undo_bufmgr.h" + + +/* ---------------------------------------------------------------- + * Buffer tag construction + * ---------------------------------------------------------------- + */ + +/* + * UndoMakeBufferTag + * Initialize a BufferTag for an undo log block. + * + * This constructs the BufferTag that the shared buffer manager uses + * to identify this undo block in its hash table. The tag encodes the + * virtual RelFileLocator (mapping log_number to a pseudo-relation) + * and UndoLogForkNum (MAIN_FORKNUM) as the fork number. + */ +void +UndoMakeBufferTag(BufferTag *tag, uint32 log_number, + BlockNumber block_number) +{ + RelFileLocator rlocator; + + UndoLogGetRelFileLocator(log_number, &rlocator); + InitBufferTag(tag, &rlocator, UndoLogForkNum, block_number); +} + + +/* ---------------------------------------------------------------- + * Buffer read/release API + * ---------------------------------------------------------------- + */ + +/* + * ReadUndoBuffer + * Read an undo log block into the shared buffer pool. + * + * Translates the undo log number and block number into a virtual + * RelFileLocator and calls ReadBufferWithoutRelcache() to obtain + * a shared buffer. + * + * The returned Buffer handle is pinned. The caller must release it + * via ReleaseUndoBuffer() (or UnlockReleaseUndoBuffer() if locked). + * + * For normal reads (RBM_NORMAL), the caller should lock the buffer + * after this call: + * + * buf = ReadUndoBuffer(logno, blkno, RBM_NORMAL); + * LockBuffer(buf, BUFFER_LOCK_SHARE); + * ... read data from BufferGetPage(buf) ... + * UnlockReleaseUndoBuffer(buf); + * + * For new page allocation (RBM_ZERO_AND_LOCK), the buffer is returned + * zero-filled and exclusively locked: + * + * buf = ReadUndoBuffer(logno, blkno, RBM_ZERO_AND_LOCK); + * ... initialize page contents ... + * MarkUndoBufferDirty(buf); + * UnlockReleaseUndoBuffer(buf); + */ +Buffer +ReadUndoBuffer(uint32 log_number, BlockNumber block_number, + ReadBufferMode mode) +{ + return ReadUndoBufferExtended(log_number, block_number, mode, NULL); +} + +/* + * ReadUndoBufferExtended + * Like ReadUndoBuffer but with explicit buffer access strategy. + * + * The strategy parameter can be used to control buffer pool usage when + * performing bulk undo log operations (e.g., sequential scan during + * discard, or recovery). Pass NULL for the default strategy. + * + * Undo logs are always permanent (they must survive crashes for + * recovery purposes), so we pass permanent=true to + * ReadBufferWithoutRelcache(). + */ +Buffer +ReadUndoBufferExtended(uint32 log_number, BlockNumber block_number, + ReadBufferMode mode, BufferAccessStrategy strategy) +{ + RelFileLocator rlocator; + + UndoLogGetRelFileLocator(log_number, &rlocator); + + return ReadBufferWithoutRelcache(rlocator, + UndoLogForkNum, + block_number, + mode, + strategy, + true); /* permanent */ +} + +/* + * ReleaseUndoBuffer + * Release a pinned undo buffer. + * + * The buffer must not be locked when this is called. + * This is a thin wrapper for API consistency; callers that hold + * a lock should use UnlockReleaseUndoBuffer() instead. + */ +void +ReleaseUndoBuffer(Buffer buffer) +{ + ReleaseBuffer(buffer); +} + +/* + * UnlockReleaseUndoBuffer + * Unlock and release an undo buffer in one call. + * + * Convenience function that combines UnlockReleaseBuffer() semantics + * for undo buffers. + */ +void +UnlockReleaseUndoBuffer(Buffer buffer) +{ + UnlockReleaseBuffer(buffer); +} + +/* + * MarkUndoBufferDirty + * Mark an undo buffer as needing write-back. + * + * The buffer must be exclusively locked when this is called. + * The dirty buffer will be written back during the next checkpoint + * or when evicted from the buffer pool. + */ +void +MarkUndoBufferDirty(Buffer buffer) +{ + MarkBufferDirty(buffer); +} + + +/* ---------------------------------------------------------------- + * Buffer invalidation + * ---------------------------------------------------------------- + */ + +/* + * InvalidateUndoBuffers + * Drop all shared buffers belonging to a given undo log. + * + * This is called when an undo log is fully discarded and no longer + * needed. All pages for the specified undo log number are removed + * from the shared buffer pool without being written back to disk, + * since the underlying undo log files are being removed. + * + * Uses DropRelationBuffers() which is the standard public API for + * dropping buffers belonging to a relation. We open an SMgrRelation + * for the virtual undo log locator and drop all buffers for the + * UndoLogForkNum fork starting from block 0. + * + * The caller must ensure that no other backend is concurrently + * accessing buffers for this undo log. + */ +void +InvalidateUndoBuffers(uint32 log_number) +{ + RelFileLocator rlocator; + SMgrRelation srel; + ForkNumber forknum = UndoLogForkNum; + BlockNumber firstDelBlock = 0; + + UndoLogGetRelFileLocator(log_number, &rlocator); + srel = smgropen(rlocator, INVALID_PROC_NUMBER); + + DropRelationBuffers(srel, &forknum, 1, &firstDelBlock); + + smgrclose(srel); +} + +/* + * InvalidateUndoBufferRange + * Drop shared buffers for a range of blocks in an undo log. + * + * This is called during undo log truncation when only a portion of + * the undo log is being discarded. Blocks starting from first_block + * onward are invalidated. + * + * Note: DropRelationBuffers drops all blocks >= firstDelBlock for the + * given fork, so we pass first_block as the starting block. The + * last_block parameter documents the intended range boundary but the + * buffer manager will drop any matching buffer with blockNum >= + * first_block. + * + * The caller must ensure that no other backend is concurrently + * accessing the buffers being invalidated. + */ +void +InvalidateUndoBufferRange(uint32 log_number, BlockNumber first_block, + BlockNumber last_block) +{ + RelFileLocator rlocator; + SMgrRelation srel; + ForkNumber forknum = UndoLogForkNum; + + Assert(first_block <= last_block); + + UndoLogGetRelFileLocator(log_number, &rlocator); + srel = smgropen(rlocator, INVALID_PROC_NUMBER); + + DropRelationBuffers(srel, &forknum, 1, &first_block); + + smgrclose(srel); +} diff --git a/src/backend/access/undo/undo_xlog.c b/src/backend/access/undo/undo_xlog.c new file mode 100644 index 0000000000000..6bd6942fddda8 --- /dev/null +++ b/src/backend/access/undo/undo_xlog.c @@ -0,0 +1,1461 @@ +/*------------------------------------------------------------------------- + * + * undo_xlog.c + * UNDO resource manager WAL redo routines + * + * This module implements the WAL redo callback for the RM_UNDO_ID resource + * manager. It handles replay of: + * + * XLOG_UNDO_ALLOCATE - Replay UNDO log space allocation + * XLOG_UNDO_DISCARD - Replay UNDO record discard + * XLOG_UNDO_EXTEND - Replay UNDO log file extension + * XLOG_UNDO_APPLY_RECORD - Replay CLR (Compensation Log Record) + * + * CLR Redo Strategy + * ----------------- + * CLRs for UNDO application use REGBUF_FORCE_IMAGE to store a full page + * image. During redo, XLogReadBufferForRedo() will restore the full page + * image automatically (returning BLK_RESTORED). No additional replay + * logic is needed because the page image already contains the result of + * the UNDO application. + * + * This is the same strategy used by ZHeap (log_zheap_undo_actions with + * REGBUF_FORCE_IMAGE) and is the simplest correct approach for crash + * recovery of UNDO operations. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/access/undo/undo_xlog.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/atm.h" +/* + * FIXME(reviewer-item-2): agnosticism breach. The core UNDO WAL layer + * should not know heap WAL record formats. This #include and the + * RM_HEAP_ID branches in UndoValidateBatchLSN() and + * UndoReadBatchFromWAL() below embed per-RM WAL-format knowledge (opcode -> + * payload offset, XLH_*_HAS_UNDO flag tests) directly here. The clean fix + * is an optional rmgr callback (e.g. rm_undo_batch_locate) that the owning + * rmgr implements and core calls if present. Deferred: adding an rmgr method + * changes the RmgrData struct (a PGDLLIMPORT, ABI for custom-rmgr + * extensions), the PG_RMGR macro arity, and every PG_RMGR line in + * rmgrlist.h (~25 RMs) plus the parallel pg_waldump table -- a tree-wide + * WAL-record-ABI change out of scope for this fix pass. + */ +#include "access/heapam_xlog.h" +#include "access/htup_details.h" +#include "access/twophase.h" +#include "access/undo_xlog.h" +#include "access/undolog.h" +#include "access/undorecord.h" +#include "access/undormgr.h" +#include "access/xlog.h" +#include "access/xlog_internal.h" +#include "access/xlogreader.h" +#include "access/xlogutils.h" +#include "miscadmin.h" +#include "storage/bufmgr.h" +#include "storage/bufpage.h" +#include "storage/itemid.h" +#include "utils/memutils.h" + +/* + * undo_redo - Replay an UNDO WAL record during crash recovery + * + * This function handles all UNDO resource manager WAL record types. + * For CLRs (XLOG_UNDO_APPLY_RECORD), the full page image is restored + * automatically by XLogReadBufferForRedo(), so no additional replay + * logic is needed. + */ +void +undo_redo(XLogReaderState *record) +{ + uint8 info = XLogRecGetInfo(record) & ~XLR_INFO_MASK; + + switch (info) + { + case XLOG_UNDO_ALLOCATE: + { + xl_undo_allocate *xlrec = (xl_undo_allocate *) XLogRecGetData(record); + + /* + * During recovery, update the UNDO log's insert pointer to + * reflect this allocation. This ensures that after crash + * recovery the UNDO log metadata is consistent. + * + * Note: UndoLogShared may not be initialized yet during early + * recovery. We guard against that. + */ + if (UndoLogShared != NULL) + { + UndoLogControl *log = NULL; + int i; + + /* + * Find the log control structure. O(MAX_UNDO_LOGS) scan: + * with MAX_UNDO_LOGS=64 this is acceptable at recovery + * time (called once per record). + */ + for (i = 0; i < MAX_UNDO_LOGS; i++) + { + if (UndoLogShared->logs[i].in_use && + UndoLogShared->logs[i].log_number == xlrec->log_number) + { + log = &UndoLogShared->logs[i]; + break; + } + } + + if (log == NULL) + { + /* Log doesn't exist yet, create it */ + for (i = 0; i < MAX_UNDO_LOGS; i++) + { + if (!UndoLogShared->logs[i].in_use) + { + log = &UndoLogShared->logs[i]; + log->log_number = xlrec->log_number; + pg_atomic_write_u64(&log->insert_ptr, xlrec->start_ptr); + log->discard_ptr = MakeUndoRecPtr(xlrec->log_number, 0); + log->oldest_xid = InvalidTransactionId; + log->in_use = true; + break; + } + } + } + + if (log != NULL) + { + /* + * Advance insert pointer past this allocation. Only + * move forward, never regress -- with coalesced WAL + * records from concurrent backends, a later record + * may cover a range already subsumed by an earlier + * one. + */ + UndoRecPtr new_end = xlrec->start_ptr + xlrec->length; + UndoRecPtr cur_ptr = pg_atomic_read_u64(&log->insert_ptr); + + if (new_end > cur_ptr) + pg_atomic_write_u64(&log->insert_ptr, new_end); + } + } + } + break; + + case XLOG_UNDO_DISCARD: + { + xl_undo_discard *xlrec = (xl_undo_discard *) XLogRecGetData(record); + + if (UndoLogShared != NULL) + { + int i; + + for (i = 0; i < MAX_UNDO_LOGS; i++) + { + if (UndoLogShared->logs[i].in_use && + UndoLogShared->logs[i].log_number == xlrec->log_number) + { + UndoLogShared->logs[i].discard_ptr = xlrec->discard_ptr; + UndoLogShared->logs[i].oldest_xid = xlrec->oldest_xid; + break; + } + } + } + } + break; + + case XLOG_UNDO_EXTEND: + { + xl_undo_extend *xlrec = (xl_undo_extend *) XLogRecGetData(record); + + /* + * Extend the UNDO log file to the specified size. The file + * will be created if it doesn't exist. + * + * With append-only I/O, the smgr-managed file is no longer + * used -- UNDO data is written directly to the segment file. + */ + ExtendUndoLogFile(xlrec->log_number, xlrec->new_size); + } + break; + + case XLOG_UNDO_APPLY_RECORD: + { + /* + * Physiological CLR redo: re-apply the exact page + * modification that was performed during UNDO application. + * + * If a full page image is present (BLK_RESTORED or + * UNDO_CLR_FULL_PAGE), the page is already correct. Otherwise + * (BLK_NEEDS_REDO), we replay the operation using the + * metadata and optional tuple data in the record. + */ + xl_undo_apply *xlrec; + Buffer buffer; + XLogRedoAction action; + + xlrec = (xl_undo_apply *) XLogRecGetData(record); + action = XLogReadBufferForRedo(record, 0, &buffer); + + switch (action) + { + case BLK_RESTORED: + /* Full page image applied -- nothing more to do */ + break; + + case BLK_DONE: + /* Page already up-to-date (LSN check) */ + break; + + case BLK_NEEDS_REDO: + { + Page page = BufferGetPage(buffer); + + if (xlrec->clr_flags & UNDO_CLR_LP_DEAD) + { + /* + * Mark the line pointer LP_DEAD, keeping its + * storage. Used only by nbtree/hash index + * INSERT undo, where btree binary search + * still reads the dead tuple's key until + * VACUUM physically removes it. Must match + * the forward-apply path (ItemIdMarkDead), + * which does not zero lp_off/lp_len. + */ + ItemId lp = PageGetItemId(page, + xlrec->target_offset); + + if (ItemIdIsNormal(lp)) + ItemIdMarkDead(lp); + } + else if (xlrec->clr_flags & UNDO_CLR_LP_UNUSED) + { + /* + * Mark the line pointer LP_UNUSED. Used for + * INSERT undo (no indexes). + */ + ItemId lp = PageGetItemId(page, + xlrec->target_offset); + + ItemIdSetUnused(lp); + PageSetHasFreeLinePointers(page); + } + else if (xlrec->clr_flags & UNDO_CLR_HAS_TUPLE) + { + /* + * Restore tuple data. Used for DELETE undo, + * full-tuple UPDATE undo, and INPLACE undo. + * The tuple data is in the buffer-specific + * data registered with block reference 0. + */ + ItemId lp = PageGetItemId(page, + xlrec->target_offset); + + if (ItemIdIsUsed(lp) && ItemIdHasStorage(lp) && + xlrec->tuple_len > 0) + { + HeapTupleHeader htup; + char *data; + Size datalen; + + data = XLogRecGetBlockData(record, 0, + &datalen); + Assert(data != NULL); + Assert(datalen >= xlrec->tuple_len); + + ItemIdSetNormal(lp, ItemIdGetOffset(lp), + xlrec->tuple_len); + htup = (HeapTupleHeader) PageGetItem(page, lp); + memcpy(htup, data, xlrec->tuple_len); + } + } + else if (xlrec->clr_flags & UNDO_CLR_HAS_DELTA) + { + /* + * Delta-encoded UPDATE redo. Reconstruct old + * tuple from current page contents + delta. + * The delta data (HeapUndoDeltaHeader + + * changed bytes) is in block data. + */ + ItemId lp = PageGetItemId(page, + xlrec->target_offset); + + if (ItemIdIsUsed(lp) && ItemIdHasStorage(lp)) + { + char *data; + Size datalen; + HeapTupleHeader cur_htup; + const char *cur_data; + Size cur_len; + uint16 prefix_len; + uint16 suffix_len; + uint32 changed_len; + uint32 old_tuple_len; + const char *changed_data; + char *restored; + Size hdr_size; + + data = XLogRecGetBlockData(record, 0, + &datalen); + Assert(data != NULL); + + /* + * The block data contains: - + * old_tuple_len (uint32) - prefix_len + * (uint16) - suffix_len (uint16) - + * changed_len (uint32) - changed bytes + * (changed_len) + */ + hdr_size = sizeof(uint32) + + 2 * sizeof(uint16) + sizeof(uint32); + + if (datalen < hdr_size) + ereport(ERROR, + (errmsg("invalid delta CLR at %X/%X: " + "block data too short (%zu bytes)", + LSN_FORMAT_ARGS(record->ReadRecPtr), + datalen))); + + memcpy(&old_tuple_len, data, sizeof(uint32)); + memcpy(&prefix_len, data + sizeof(uint32), + sizeof(uint16)); + memcpy(&suffix_len, + data + sizeof(uint32) + sizeof(uint16), + sizeof(uint16)); + memcpy(&changed_len, + data + sizeof(uint32) + 2 * sizeof(uint16), + sizeof(uint32)); + changed_data = data + hdr_size; + + cur_htup = (HeapTupleHeader) + PageGetItem(page, lp); + cur_data = (const char *) cur_htup; + cur_len = ItemIdGetLength(lp); + + /* + * Validate lengths before any pointer + * arithmetic: a corrupt CLR could + * otherwise cause a buffer underrun or + * overflow. + */ + if (prefix_len > cur_len || + suffix_len > cur_len || + prefix_len + suffix_len > cur_len || + (Size) (prefix_len + changed_len + suffix_len) != (Size) old_tuple_len || + datalen < hdr_size + changed_len) + ereport(ERROR, + (errmsg("invalid delta CLR at %X/%X: " + "prefix=%u suffix=%u changed=%u " + "old_len=%u cur_len=%zu", + LSN_FORMAT_ARGS(record->ReadRecPtr), + prefix_len, suffix_len, + changed_len, old_tuple_len, + cur_len))); + + restored = palloc(old_tuple_len); + + /* prefix from current tuple */ + if (prefix_len > 0) + memcpy(restored, cur_data, prefix_len); + + /* changed middle from CLR data */ + if (changed_len > 0) + memcpy(restored + prefix_len, + changed_data, changed_len); + + /* suffix from current tuple */ + if (suffix_len > 0) + memcpy(restored + prefix_len + changed_len, + cur_data + cur_len - suffix_len, + suffix_len); + + ItemIdSetNormal(lp, ItemIdGetOffset(lp), + old_tuple_len); + memcpy(cur_htup, restored, old_tuple_len); + pfree(restored); + } + } + else if (xlrec->clr_flags & UNDO_CLR_HAS_VISIBILITY) + { + /* + * Visibility-delta redo: restore only the + * three tuple-header fields changed by + * heap_delete(). The column data is unchanged + * on the page. + */ + char *vis_data; + Size vis_datalen; + xl_undo_apply_visibility vis_rec; + ItemId vlp; + HeapTupleHeader vhtup; + + vis_data = XLogRecGetBlockData(record, 0, + &vis_datalen); + Assert(vis_data != NULL); + Assert(vis_datalen >= SizeOfUndoApplyVisibility); + + memcpy(&vis_rec, vis_data, + SizeOfUndoApplyVisibility); + + vlp = PageGetItemId(page, + xlrec->target_offset); + if (ItemIdIsUsed(vlp) && ItemIdHasStorage(vlp)) + { + vhtup = (HeapTupleHeader) + PageGetItem(page, vlp); + HeapTupleHeaderSetXmax(vhtup, + vis_rec.old_xmax); + vhtup->t_infomask = + vis_rec.old_infomask; + vhtup->t_infomask2 = + vis_rec.old_infomask2; + } + } + else if (xlrec->clr_flags & UNDO_CLR_HOT_RESTORE) + { + /* + * HOT update rollback: restore old tuple's + * infomask and kill new tuple version. + */ + char *data; + Size datalen; + xl_undo_apply_hot hot_data; + ItemId old_lp; + HeapTupleHeader old_htup; + ItemId new_lp; + + data = XLogRecGetBlockData(record, 0, + &datalen); + Assert(data != NULL); + Assert(datalen >= SizeOfUndoApplyHot); + + memcpy(&hot_data, data, SizeOfUndoApplyHot); + + old_lp = PageGetItemId(page, + xlrec->target_offset); + if (ItemIdIsNormal(old_lp)) + { + old_htup = (HeapTupleHeader) + PageGetItem(page, old_lp); + old_htup->t_infomask = hot_data.old_infomask; + old_htup->t_infomask2 = hot_data.old_infomask2; + ItemPointerSet(&old_htup->t_ctid, + xlrec->target_block, + xlrec->target_offset); + } + + /* Kill the new tuple version */ + new_lp = PageGetItemId(page, + hot_data.new_offset); + if (ItemIdIsNormal(new_lp)) + ItemIdSetDead(new_lp); + } + + PageSetLSN(page, record->EndRecPtr); + MarkBufferDirty(buffer); + } + break; + + case BLK_NOTFOUND: + /* Block doesn't exist (truncated?) -- skip */ + break; + } + + if (BufferIsValid(buffer)) + UnlockReleaseBuffer(buffer); + } + break; + + case XLOG_UNDO_PAGE_WRITE: + + /* + * XLOG_UNDO_PAGE_WRITE is no longer emitted (append-only I/O + * architecture writes directly via pwrite, not through + * shared_buffers). We keep this case for backward compatibility + * with WAL from before the transition. Old records are simply + * ignored -- the UNDO data was already written to the segment + * file by the originating backend. + */ + break; + + case XLOG_UNDO_BATCH: + { + xl_undo_batch *xlrec = (xl_undo_batch *) XLogRecGetData(record); + + /* + * During recovery, track this batch for incomplete + * transaction detection. After redo completes, any + * transaction that wrote UNDO batches but did not commit will + * need its UNDO chain walked for rollback. + * + * The batch payload (serialized UNDO records) is part of the + * WAL record and can be re-read later via XLogReadRecord() + * during the undo phase. + */ + UndoRecoveryTrackBatch(xlrec->xid, record->ReadRecPtr, + xlrec->chain_prev, + xlrec->persistence); + + ereport(DEBUG2, + (errmsg("undo_redo: BATCH xid %u, nrecords %u, " + "total_len %u, chain_prev %X/%X", + xlrec->xid, xlrec->nrecords, + xlrec->total_len, + LSN_FORMAT_ARGS(xlrec->chain_prev)))); + } + break; + + case XLOG_UNDO_ROTATE: + { + xl_undo_rotate *xlrec = (xl_undo_rotate *) XLogRecGetData(record); + + /* + * Replay segment rotation: mark the old log SEALED and the + * new log ACTIVE. This reconstructs the lifecycle state so + * that after recovery the discard worker can clean up sealed + * logs properly. + */ + if (UndoLogShared != NULL) + { + int j; + + /* Seal the old log */ + if (xlrec->old_log_number != 0) + { + for (j = 0; j < MAX_UNDO_LOGS; j++) + { + UndoLogControl *old_log = &UndoLogShared->logs[j]; + + if (old_log->in_use && + old_log->log_number == xlrec->old_log_number) + { + old_log->state = UNDO_LOG_SEALED; + pg_atomic_write_u64(&old_log->seal_ptr, + xlrec->old_seal_ptr); + break; + } + } + } + + /* Activate the new log (find or create slot) */ + { + UndoLogControl *new_log = NULL; + int new_slot = -1; + + /* Check if it already exists (idempotent replay) */ + for (j = 0; j < MAX_UNDO_LOGS; j++) + { + if (UndoLogShared->logs[j].in_use && + UndoLogShared->logs[j].log_number == xlrec->new_log_number) + { + new_log = &UndoLogShared->logs[j]; + new_slot = j; + break; + } + } + + /* If not found, allocate a free slot */ + if (new_log == NULL) + { + for (j = 0; j < MAX_UNDO_LOGS; j++) + { + if (!UndoLogShared->logs[j].in_use) + { + new_log = &UndoLogShared->logs[j]; + new_slot = j; + new_log->log_number = xlrec->new_log_number; + pg_atomic_write_u64(&new_log->insert_ptr, + MakeUndoRecPtr(xlrec->new_log_number, 0)); + new_log->discard_ptr = MakeUndoRecPtr(xlrec->new_log_number, 0); + new_log->oldest_xid = InvalidTransactionId; + new_log->in_use = true; + break; + } + } + } + + if (new_log != NULL) + { + new_log->state = UNDO_LOG_ACTIVE; + pg_atomic_write_u64(&new_log->seal_ptr, InvalidUndoRecPtr); + pg_atomic_write_u32(&UndoLogShared->active_log_idx, + (uint32) new_slot); + } + } + } + } + break; + + default: + elog(PANIC, "undo_redo: unknown op code %u", info); + } +} + +/* ---------------------------------------------------------------- + * UNDO recovery tracking + * + * During WAL redo, we track which transactions wrote UNDO batches. + * When a commit/abort record is redone, the XID is removed. + * After redo completes, remaining entries represent incomplete + * transactions that need their UNDO chains walked. + * ---------------------------------------------------------------- + */ + +/* Hash table entry for tracking incomplete UNDO transactions */ +typedef struct UndoRecoveryEntry +{ + TransactionId xid; /* hash key */ + XLogRecPtr last_batch_lsn[NUndoPersistenceLevels]; /* chain heads per + * persistence level */ + char status; /* in use */ +} UndoRecoveryEntry; + +/* Simple dynamic array for recovery tracking (used during startup only) */ +static UndoRecoveryEntry *undo_recovery_entries = NULL; +static int undo_recovery_nentries = 0; +static int undo_recovery_capacity = 0; + +/* + * Safety cap to prevent OOM during recovery. If more than this many + * distinct in-flight XIDs are found in WAL at crash time, we stop + * tracking new ones and log a warning. The untracked transactions will + * need manual resolution (e.g. via pg_resetwal or targeted UNDO apply). + * + * 1 million entries Ɨ ~40 bytes each ā‰ˆ 40 MB, which is reasonable for + * a recovery-only allocation. + */ +#define UNDO_RECOVERY_MAX_ENTRIES 1048576 + +/* + * UndoRecoveryTrackBatch - Record an UNDO batch during WAL redo + * + * Called from the XLOG_UNDO_BATCH redo handler to track which + * transactions have UNDO data that may need rollback. + */ +void +UndoRecoveryTrackBatch(TransactionId xid, XLogRecPtr batch_lsn, + XLogRecPtr chain_prev, + UndoPersistenceLevel persistence) +{ + int i; + UndoRecoveryEntry *entry = NULL; + + if (!TransactionIdIsValid(xid)) + return; + + /* Find existing entry for this XID */ + for (i = 0; i < undo_recovery_nentries; i++) + { + if (undo_recovery_entries[i].xid == xid) + { + entry = &undo_recovery_entries[i]; + break; + } + } + + /* Create new entry if needed */ + if (entry == NULL) + { + /* Safety cap: refuse to track more XIDs to prevent OOM */ + if (undo_recovery_nentries >= UNDO_RECOVERY_MAX_ENTRIES) + { + static bool warned = false; + + if (!warned) + { + ereport(WARNING, + (errmsg("UNDO recovery: reached maximum tracked transaction limit (%d)", + UNDO_RECOVERY_MAX_ENTRIES), + errhint("Transactions beyond this limit will not be automatically rolled back. " + "Manual intervention may be required after recovery completes."))); + warned = true; + } + return; + } + + if (undo_recovery_nentries >= undo_recovery_capacity) + { + int new_capacity = (undo_recovery_capacity == 0) ? 64 : + undo_recovery_capacity * 2; + UndoRecoveryEntry *new_entries; + + /* Clamp doubling to not exceed the safety cap */ + if (new_capacity > UNDO_RECOVERY_MAX_ENTRIES) + new_capacity = UNDO_RECOVERY_MAX_ENTRIES; + + if (undo_recovery_entries == NULL) + { + new_entries = (UndoRecoveryEntry *) + palloc0(sizeof(UndoRecoveryEntry) * new_capacity); + } + else + { + new_entries = (UndoRecoveryEntry *) + repalloc(undo_recovery_entries, + sizeof(UndoRecoveryEntry) * new_capacity); + memset(&new_entries[undo_recovery_capacity], 0, + sizeof(UndoRecoveryEntry) * (new_capacity - undo_recovery_capacity)); + } + undo_recovery_entries = new_entries; + undo_recovery_capacity = new_capacity; + } + + entry = &undo_recovery_entries[undo_recovery_nentries++]; + entry->xid = xid; + for (i = 0; i < NUndoPersistenceLevels; i++) + entry->last_batch_lsn[i] = InvalidXLogRecPtr; + } + + /* Update the chain head for this persistence level */ + if (persistence < NUndoPersistenceLevels) + entry->last_batch_lsn[persistence] = batch_lsn; +} + +/* + * UndoRecoveryRemoveXid - Remove an XID from recovery tracking + * + * Called when a commit or abort record is redone during recovery. + * Committed transactions don't need UNDO rollback. Aborted transactions + * that were already fully rolled back (abort record present) also don't + * need further work. + */ +void +UndoRecoveryRemoveXid(TransactionId xid) +{ + int i; + + if (!TransactionIdIsValid(xid)) + return; + + for (i = 0; i < undo_recovery_nentries; i++) + { + if (undo_recovery_entries[i].xid == xid) + { + /* Mark as removed by zeroing XID */ + undo_recovery_entries[i].xid = InvalidTransactionId; + break; + } + } +} + +/* + * UndoRecoveryNeeded - Check if there are incomplete transactions needing UNDO + * + * Returns true if any tracked transactions remain after redo is complete. + */ +bool +UndoRecoveryNeeded(void) +{ + int i; + + for (i = 0; i < undo_recovery_nentries; i++) + { + if (TransactionIdIsValid(undo_recovery_entries[i].xid)) + return true; + } + + return false; +} + +/* + * DeferredUndoXact - Transaction deferred for async UNDO processing + * + * During crash recovery, if syscache isn't available, we skip UNDO application + * and defer the transaction for later processing by the logical revert worker. + */ +typedef struct DeferredUndoXact +{ + TransactionId xid; + Oid dboid; + XLogRecPtr last_batch_lsn; + struct DeferredUndoXact *next; +} DeferredUndoXact; + +static DeferredUndoXact *deferred_undo_xacts = NULL; + +/* + * PerformUndoRecovery - Walk and apply UNDO chains for incomplete transactions + * + * This is the ARIES-style undo phase, called after the redo loop completes. + * For each incomplete transaction that wrote UNDO batches, we walk the + * UNDO chain backward and apply each record via the RM dispatch table. + * + * CLRs are generated during this phase to ensure idempotency in case of + * a crash during the undo phase itself. + * + * If UNDO application is skipped (e.g., due to syscache not being available), + * the transaction is tracked for deferred processing after recovery completes. + */ +void +PerformUndoRecovery(void) +{ + int i, + j; + int total_xacts = 0; + int total_records = 0; + int pending_xacts = 0; + int deferred_xacts = 0; + + /* Count pending transactions for the opening log message. */ + for (i = 0; i < undo_recovery_nentries; i++) + { + if (TransactionIdIsValid(undo_recovery_entries[i].xid)) + pending_xacts++; + } + + if (pending_xacts > 0) + ereport(LOG, + (errmsg("UNDO recovery: %d incomplete transaction(s) to roll back", + pending_xacts))); + + for (i = 0; i < undo_recovery_nentries; i++) + { + UndoRecoveryEntry *entry = &undo_recovery_entries[i]; + bool any_skipped = false; + + if (!TransactionIdIsValid(entry->xid)) + continue; + + /* + * Skip prepared transactions. Prepared (2PC) transactions must remain + * in the prepared state after crash recovery, not be automatically + * rolled back. They will be explicitly committed or rolled back later + * via COMMIT PREPARED or ROLLBACK PREPARED. + * + * During recovery, RecoveryTransactionIdIsPrepared() checks the + * in-memory prepared transaction state reconstructed from WAL replay. + */ + if (RecoveryTransactionIdIsPrepared(entry->xid)) + { + ereport(LOG, + (errmsg("UNDO recovery: skipping prepared transaction %u " + "(will remain in prepared state)", + entry->xid))); + continue; + } + + total_xacts++; + + ereport(LOG, + (errmsg("UNDO recovery: rolling back transaction %u", + entry->xid))); + + /* + * Walk each persistence level's UNDO chain independently. This + * mirrors the normal abort path in AtAbort_XactUndo(). + * + * TEMP and UNLOGGED levels are skipped during crash recovery: - TEMP: + * temporary tables are destroyed on server restart, so there is + * nothing to roll back and the pages no longer exist. - UNLOGGED: + * unlogged table files are reset to empty on crash recovery + * (initfork), making any prior UNDO application wrong. + */ + for (j = 0; j < NUndoPersistenceLevels; j++) + { + XLogRecPtr batch_lsn = entry->last_batch_lsn[j]; + + if (j == UNDOPERSISTENCE_TEMP || j == UNDOPERSISTENCE_UNLOGGED) + continue; + + while (XLogRecPtrIsValid(batch_lsn)) + { + UndoBatchData *batch; + char *pos; + char *end; + + batch = UndoReadBatchFromWAL(batch_lsn); + if (batch == NULL) + { + /* + * A missing or unreadable UNDO batch during crash + * recovery. This can happen with fsync=off when WAL was + * not persisted before the crash, or when WAL segments + * were recycled before the UNDO chain was fully applied. + * + * Rather than PANIC (which makes the database permanently + * unrecoverable), skip this transaction's rollback. The + * affected tuples will retain their UNCOMMITTED flag and + * be invisible until VACUUM removes them. This is a + * bounded anomaly similar to the old hash-overflow + * degraded mode. + */ + ereport(WARNING, + (errmsg("UNDO recovery: could not read batch at %X/%X " + "for transaction %u; skipping rollback " + "(affected tuples will be cleaned by VACUUM)", + LSN_FORMAT_ARGS(batch_lsn), + entry->xid))); + break; /* skip remaining chain for this persistence + * level */ + } + + /* Walk records within this batch */ + pos = batch->payload; + end = pos + batch->payload_len; + + while (pos < end) + { + UndoRecordHeader header; + char *payload = NULL; + + if ((Size) (end - pos) < SizeOfUndoRecordHeader) + break; + + memcpy(&header, pos, SizeOfUndoRecordHeader); + + if (header.urec_len < SizeOfUndoRecordHeader || + (Size) (end - pos) < header.urec_len) + break; + + if (header.urec_payload_len > 0) + payload = pos + SizeOfUndoRecordHeader; + + /* + * Apply this UNDO record via the RM dispatch table. + * + * Idempotency note: UNDO records are immutable in WAL and + * carry no per-record applied marker; the CLR is a + * separate WAL record. Double-application is prevented + * by page LSN: each CLR bumps the heap page LSN to the + * CLR's EndRecPtr. When rm_undo reads the buffer, + * XLogReadBufferForRedo returns BLK_DONE or BLK_RESTORED + * for pages that were already restored by a CLR in Phase + * 1 redo, preventing re-application. + */ + { + const UndoRmgrData *rmgr = GetUndoRmgr(header.urec_rmid); + + if (rmgr != NULL) + { + UndoApplyResult result; + + result = rmgr->rm_undo(header.urec_rmid, + header.urec_info, + header.urec_xid, + header.urec_reloid, + payload, + header.urec_payload_len, + InvalidUndoRecPtr); + total_records++; + + /* + * If any UNDO record was skipped (e.g., due to + * syscache not being initialized), mark this + * transaction for deferred processing by the + * logical revert worker. + */ + if (result == UNDO_APPLY_SKIPPED) + any_skipped = true; + } + } + + pos += header.urec_len; + } + + /* Follow chain to previous batch */ + { + XLogRecPtr next_lsn = batch->header.chain_prev; + + /* + * Guard against circular or forward-pointing chains: + * chain_prev must be strictly older (smaller LSN) than + * the current batch or invalid (end of chain). A + * forward- pointing chain_prev would cause an infinite + * loop. + */ + if (XLogRecPtrIsValid(next_lsn) && next_lsn >= batch_lsn) + ereport(PANIC, + (errmsg("UNDO recovery: chain_prev %X/%X >= batch_lsn %X/%X " + "for transaction %u; corrupt UNDO chain", + LSN_FORMAT_ARGS(next_lsn), + LSN_FORMAT_ARGS(batch_lsn), + entry->xid))); + UndoFreeBatchData(batch); + batch_lsn = next_lsn; + } + } + } + + /* + * If any UNDO records were skipped (e.g., due to syscache not being + * initialized during early recovery), track this transaction for + * deferred processing. We cannot add it to the ATM yet because + * ATMAddAborted() writes WAL, which isn't allowed during recovery. + * + * Instead, we add it to an in-memory list that will be flushed to the + * ATM after recovery completes (when InRedo is set to false). + * + * Use the permanent persistence level's last_batch_lsn for tracking. + * TEMP and UNLOGGED are skipped during crash recovery anyway. + */ + if (any_skipped) + { + XLogRecPtr perm_lsn = entry->last_batch_lsn[UNDOPERSISTENCE_PERMANENT]; + + if (XLogRecPtrIsValid(perm_lsn)) + { + DeferredUndoXact *deferred = (DeferredUndoXact *) + palloc(sizeof(DeferredUndoXact)); + + deferred->xid = entry->xid; + deferred->dboid = MyDatabaseId; + deferred->last_batch_lsn = perm_lsn; + deferred->next = deferred_undo_xacts; + deferred_undo_xacts = deferred; + + deferred_xacts++; + ereport(LOG, + (errmsg("UNDO recovery: transaction %u deferred to " + "logical revert worker (syscache not ready)", + entry->xid))); + } + } + } + + if (total_xacts > 0) + { + if (deferred_xacts > 0) + ereport(LOG, + (errmsg("UNDO recovery complete: %d transactions processed, " + "%d records applied, %d transactions deferred to " + "logical revert worker", + total_xacts, total_records, deferred_xacts))); + else + ereport(LOG, + (errmsg("UNDO recovery complete: %d transactions rolled back, " + "%d records applied", + total_xacts, total_records))); + } + + /* Free tracking data */ + if (undo_recovery_entries != NULL) + { + pfree(undo_recovery_entries); + undo_recovery_entries = NULL; + } + undo_recovery_nentries = 0; + undo_recovery_capacity = 0; +} + +/* + * FlushDeferredUndoXacts - Add deferred transactions to the ATM + * + * Called after recovery completes (when InRedo is false) to add any + * transactions that were deferred during UNDO recovery to the Aborted + * Transaction Map (ATM). These transactions will be processed + * asynchronously by the logical revert worker. + * + * This must be called after InRedo is set to false because ATMAddAborted() + * writes WAL, which is not allowed during recovery. + */ +void +FlushDeferredUndoXacts(void) +{ + DeferredUndoXact *deferred; + int count = 0; + + if (deferred_undo_xacts == NULL) + return; + + ereport(LOG, + (errmsg("flushing deferred UNDO transactions to ATM"))); + + /* Walk the list and add each transaction to the ATM */ + while (deferred_undo_xacts != NULL) + { + deferred = deferred_undo_xacts; + deferred_undo_xacts = deferred->next; + + ATMAddAborted(deferred->xid, deferred->dboid, deferred->last_batch_lsn); + count++; + + pfree(deferred); + } + + if (count > 0) + ereport(LOG, + (errmsg("added %d deferred transaction(s) to ATM for async UNDO processing", + count))); +} + +/* ---------------------------------------------------------------- + * UNDO batch reading from WAL + * ---------------------------------------------------------------- + */ + +/* + * UndoReadBatchFromWAL - Read a single XLOG_UNDO_BATCH record from WAL + * + * Uses XLogReader to read the WAL record at the given LSN. + * Returns a palloc'd UndoBatchData containing the header and a copy + * of the payload. The caller must pfree via UndoFreeBatchData(). + * + * Returns NULL if the record cannot be read or is not an UNDO batch. + */ +/* + * Module-level cached XLogReader for UndoReadBatchFromWAL. + * Allocated once and reused across calls to avoid per-batch + * open/close overhead on WAL segment files during rollback. + */ +static XLogReaderState *undo_batch_reader = NULL; +static XLogReaderRoutine undo_batch_reader_routine = { + .page_read = read_local_xlog_page, + .segment_open = wal_segment_open, + .segment_close = wal_segment_close, +}; + +/* + * UndoValidateBatchLSN + * Quick check that the WAL record at batch_lsn is a valid UNDO source. + * + * Returns true if the record is RM_UNDO_ID (standalone batch) or RM_HEAP_ID + * with a HAS_UNDO flag. Returns false for any other unrecognized + * record type. Used by the inline UNDO path to avoid calling + * ApplyUndoChainFromWAL on a batch_lsn that points to the wrong record type. + * + * FIXME(reviewer-item-2): the RM_HEAP_ID branch below tests XLH_*_HAS_UNDO + * heap WAL flags -- per-RM WAL-format knowledge that belongs behind an rmgr + * callback, not in the AM-agnostic UNDO core. See the note at the + * heapam_xlog.h #include for why the extraction is deferred. + */ +bool +UndoValidateBatchLSN(XLogRecPtr batch_lsn) +{ + XLogRecord *record_hdr; + char *errormsg = NULL; + uint8 rmid; + MemoryContext old_ctx; + bool result = false; + + if (!XLogRecPtrIsValid(batch_lsn)) + return false; + + /* + * Perform the entire read under TopMemoryContext. The cached reader and, + * critically, its lazily-allocated decode_buffer (xlogreader.c) are + * palloc'd in CurrentMemoryContext. Callers such as the inline-abort + * path run with a transient context that is deleted right after; + * allocating the reader state there would leave the static + * undo_batch_reader (and its decode buffer) dangling, crashing the next + * abort. TopMemoryContext makes the cache truly persistent. + */ + old_ctx = MemoryContextSwitchTo(TopMemoryContext); + + if (undo_batch_reader == NULL) + { + undo_batch_reader = XLogReaderAllocate(wal_segment_size, NULL, + &undo_batch_reader_routine, NULL); + if (undo_batch_reader == NULL) + { + MemoryContextSwitchTo(old_ctx); + return false; + } + } + + XLogBeginRead(undo_batch_reader, batch_lsn); + record_hdr = XLogReadRecord(undo_batch_reader, &errormsg); + if (record_hdr == NULL) + { + MemoryContextSwitchTo(old_ctx); + return false; + } + + rmid = XLogRecGetRmid(undo_batch_reader); + + /* Standalone UNDO batch */ + if (rmid == RM_UNDO_ID) + result = true; + /* Heap record with embedded UNDO */ + else if (rmid == RM_HEAP_ID) + { + uint8 info = XLogRecGetInfo(undo_batch_reader) & XLOG_HEAP_OPMASK; + char *data = XLogRecGetData(undo_batch_reader); + + if (info == XLOG_HEAP_INSERT) + result = (((xl_heap_insert *) data)->flags & XLH_INSERT_HAS_UNDO) != 0; + else if (info == XLOG_HEAP_DELETE) + result = (((xl_heap_delete *) data)->flags & XLH_DELETE_HAS_UNDO) != 0; + else if (info == XLOG_HEAP_UPDATE || info == XLOG_HEAP_HOT_UPDATE) + result = (((xl_heap_update *) data)->flags & XLH_UPDATE_HAS_UNDO) != 0; + } + + /* Any other rmid is not a valid inline UNDO source */ + MemoryContextSwitchTo(old_ctx); + return result; +} + +UndoBatchData * +UndoReadBatchFromWAL(XLogRecPtr batch_lsn) +{ + XLogRecord *record_hdr; + char *errormsg = NULL; + UndoBatchData *result; + xl_undo_batch *xlrec; + char *record_data; + Size record_len; + Size payload_offset; + + if (!XLogRecPtrIsValid(batch_lsn)) + return NULL; + + /* + * Safety check: verify the WAL segment containing this LSN has not been + * recycled by a checkpoint. If the LSN is behind the current redo + * pointer and the segment file doesn't exist, reading would cause SIGBUS + * (signal 10: Bus error) or SIGSEGV. Return NULL gracefully instead. + * + * Compare against GetRedoRecPtr() ā€” if our target is well behind the + * redo pointer AND behind the last checkpoint's redo location, the + * segment may have been recycled. + */ + { + XLogRecPtr redo_ptr = GetRedoRecPtr(); + XLogSegNo target_segno; + char path[MAXPGPATH]; + + if (batch_lsn < redo_ptr) + { + XLByteToSeg(batch_lsn, target_segno, wal_segment_size); + XLogFilePath(path, GetWALInsertionTimeLine(), target_segno, + wal_segment_size); + + if (access(path, F_OK) != 0) + { + ereport(WARNING, + (errmsg("UNDO batch at %X/%X: WAL segment \"%s\" no longer " + "exists (recycled by checkpoint); skipping rollback", + LSN_FORMAT_ARGS(batch_lsn), path))); + return NULL; + } + } + } + + /* + * Allocate the reader and perform the read under TopMemoryContext. Both + * the cached reader and its lazily-allocated decode_buffer (xlogreader.c) + * are palloc'd in CurrentMemoryContext; if that is a transient caller + * context (e.g. the inline-abort context) it gets deleted, leaving the + * static undo_batch_reader and its decode buffer dangling and crashing + * the next abort. We switch back to the caller's context before + * allocating the returned UndoBatchData, which the caller owns. See the + * matching comment in UndoValidateBatchLSN. + */ + { + MemoryContext read_ctx = MemoryContextSwitchTo(TopMemoryContext); + + if (undo_batch_reader == NULL) + { + undo_batch_reader = XLogReaderAllocate(wal_segment_size, NULL, + &undo_batch_reader_routine, NULL); + if (undo_batch_reader == NULL) + { + MemoryContextSwitchTo(read_ctx); + ereport(WARNING, + (errmsg("could not allocate XLogReader for UNDO batch read"))); + return NULL; + } + } + + /* Position the reader at the target LSN, then read */ + XLogBeginRead(undo_batch_reader, batch_lsn); + record_hdr = XLogReadRecord(undo_batch_reader, &errormsg); + MemoryContextSwitchTo(read_ctx); + } + if (record_hdr == NULL) + { + if (errormsg) + ereport(WARNING, + (errmsg("could not read WAL record at %X/%X: %s", + LSN_FORMAT_ARGS(batch_lsn), errormsg))); + return NULL; + } + + /* + * Determine record format: either a standalone XLOG_UNDO_BATCH record + * (overflow path or legacy) or a heap WAL record with embedded UNDO + * (XLOG_HEAP_INSERT/DELETE/UPDATE with HAS_UNDO flag set). + * + * FIXME(reviewer-item-2): the RM_HEAP_ID branch derives the embedded + * xl_undo_batch payload offset from heap opcodes (SizeOfHeapInsert etc.) + * and tests XLH_*_HAS_UNDO flags -- heap WAL-format knowledge that should + * live behind an rmgr callback. Deferred; see the heapam_xlog.h + * #include. + */ + record_data = XLogRecGetData(undo_batch_reader); + record_len = XLogRecGetDataLen(undo_batch_reader); + + if (XLogRecGetRmid(undo_batch_reader) == RM_UNDO_ID && + (XLogRecGetInfo(undo_batch_reader) & ~XLR_INFO_MASK) == XLOG_UNDO_BATCH) + { + /* Standalone XLOG_UNDO_BATCH record (overflow / legacy path) */ + if (record_len < SizeOfUndoBatch) + { + ereport(WARNING, + (errmsg("UNDO batch record at %X/%X too short: %zu bytes", + LSN_FORMAT_ARGS(batch_lsn), record_len))); + return NULL; + } + + xlrec = (xl_undo_batch *) record_data; + payload_offset = SizeOfUndoBatch; + } + else if (XLogRecGetRmid(undo_batch_reader) == RM_HEAP_ID) + { + /* + * Heap WAL record with embedded UNDO payload. Determine the offset of + * the xl_undo_batch header from the opcode. + */ + uint8 info = XLogRecGetInfo(undo_batch_reader) & XLOG_HEAP_OPMASK; + + if (info == XLOG_HEAP_DELETE) + { + xl_heap_delete *del = (xl_heap_delete *) record_data; + + if (!(del->flags & XLH_DELETE_HAS_UNDO)) + { + ereport(WARNING, + (errmsg("heap DELETE record at %X/%X has no embedded UNDO", + LSN_FORMAT_ARGS(batch_lsn)))); + return NULL; + } + payload_offset = SizeOfHeapDelete; + } + else if (info == XLOG_HEAP_INSERT) + { + xl_heap_insert *ins = (xl_heap_insert *) record_data; + + if (!(ins->flags & XLH_INSERT_HAS_UNDO)) + { + ereport(WARNING, + (errmsg("heap INSERT record at %X/%X has no embedded UNDO", + LSN_FORMAT_ARGS(batch_lsn)))); + return NULL; + } + payload_offset = SizeOfHeapInsert; + } + else if (info == XLOG_HEAP_UPDATE || info == XLOG_HEAP_HOT_UPDATE) + { + xl_heap_update *upd = (xl_heap_update *) record_data; + + if (!(upd->flags & XLH_UPDATE_HAS_UNDO)) + { + ereport(WARNING, + (errmsg("heap UPDATE record at %X/%X has no embedded UNDO", + LSN_FORMAT_ARGS(batch_lsn)))); + return NULL; + } + payload_offset = SizeOfHeapUpdate; + } + else + { + ereport(WARNING, + (errmsg("unsupported heap opcode 0x%02x at %X/%X for UNDO read", + info, LSN_FORMAT_ARGS(batch_lsn)))); + return NULL; + } + + if (record_len < payload_offset + SizeOfUndoBatch) + { + ereport(WARNING, + (errmsg("heap record at %X/%X too short for embedded UNDO: %zu bytes", + LSN_FORMAT_ARGS(batch_lsn), record_len))); + return NULL; + } + + xlrec = (xl_undo_batch *) (record_data + payload_offset); + payload_offset += SizeOfUndoBatch; + } + else + { + ereport(WARNING, + (errmsg("WAL record at %X/%X is not an UNDO batch (rmid=%u, info=0x%02x)", + LSN_FORMAT_ARGS(batch_lsn), + XLogRecGetRmid(undo_batch_reader), + XLogRecGetInfo(undo_batch_reader) & ~XLR_INFO_MASK))); + return NULL; + } + + /* + * Validate that the claimed payload length fits within the WAL record. A + * mismatch here means the WAL segment was recycled and overwritten (the + * LSN now points to a different record), or the record is corrupt. + * Without this check, the memcpy below would read past the XLogReader's + * internal buffer, potentially accessing unmapped memory + * (SIGBUS/SIGSEGV). + */ + if (payload_offset + (Size) xlrec->total_len > record_len) + { + ereport(WARNING, + (errmsg("UNDO batch at %X/%X: payload length %u exceeds " + "record data (offset %zu, record_len %zu)", + LSN_FORMAT_ARGS(batch_lsn), + xlrec->total_len, payload_offset, record_len))); + return NULL; + } + + /* + * Allocate UndoBatchData. We use palloc (CurrentMemoryContext) because + * this structure is only needed until ApplyUndoChainFromWAL processes the + * batch. We intentionally do NOT pfree in UndoFreeBatchData() because + * calling pfree on BumpContext memory would ERROR. The memory will be + * reclaimed when the current memory context is reset. + */ + result = (UndoBatchData *) palloc(sizeof(UndoBatchData)); + memcpy(&result->header, xlrec, SizeOfUndoBatch); + result->payload_len = (Size) xlrec->total_len; + if (result->payload_len > 0) + { + result->payload = (char *) palloc(result->payload_len); + memcpy(result->payload, record_data + payload_offset, + result->payload_len); + } + else + { + result->payload = NULL; + } + + /* Do not free reader -- it is cached for reuse. */ + return result; +} + +/* + * UndoFreeBatchData - Release a UndoBatchData structure + * + * This is a no-op function. We don't actually pfree the batch or payload + * because they were allocated with palloc() from CurrentMemoryContext, which + * may be a BumpContext. Calling pfree on BumpContext memory would ERROR. + * The memory will be automatically reclaimed when the current memory context + * is reset (e.g., at end of query, transaction, or subtransaction). + * + * This function exists to maintain API compatibility and to serve as a + * clear marker in the code where batch data is no longer needed. + */ +void +UndoFreeBatchData(UndoBatchData *batch) +{ + /* Intentionally empty - memory reclaimed by context reset */ + (void) batch; +} + +/* + * UndoResetBatchReader - Free and NULL the cached WAL reader. + * + * Must be called after a PG_CATCH that could leave the static reader in + * an inconsistent state (stale segment FD, partial read buffer, etc.). + * The next call to UndoReadBatchFromWAL will reallocate a fresh reader. + */ +void +UndoResetBatchReader(void) +{ + if (undo_batch_reader != NULL) + { + XLogReaderFree(undo_batch_reader); + undo_batch_reader = NULL; + } +} diff --git a/src/backend/access/undo/undoapply.c b/src/backend/access/undo/undoapply.c new file mode 100644 index 0000000000000..7674adf2add33 --- /dev/null +++ b/src/backend/access/undo/undoapply.c @@ -0,0 +1,326 @@ +/*------------------------------------------------------------------------- + * + * undoapply.c + * Generic UNDO record application during transaction rollback + * + * When a transaction aborts, this module walks the UNDO chain backward + * from the most recent record to the first. For each record, it + * dispatches to the appropriate resource manager's rm_undo callback + * based on the urec_rmid field in the record header. + * + * This module is AM-agnostic: it contains no AM- or subsystem-specific + * code. All such UNDO application logic lives in the respective resource + * managers, each registered via RegisterUndoRmgr() (see access/undormgr.h). + * + * The dispatch pattern is analogous to WAL resource managers: each RM + * registers its callbacks via RegisterUndoRmgr(), and this module + * routes UNDO records to the correct handler. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/access/undo/undoapply.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/undolog.h" +#include "access/undorecord.h" +#include "access/undormgr.h" +#include "access/undo_xlog.h" +#include "miscadmin.h" +#include "utils/injection_point.h" +#include "utils/memutils.h" + +/* + * ApplyOneUndoRecord - Apply a single UNDO record via RM dispatch + * + * Checks the CLR pointer to avoid double-application, then dispatches + * to the appropriate resource manager's rm_undo callback. + * + * Returns true if successfully applied, false if skipped. + */ +static bool +ApplyOneUndoRecord(UndoRecordHeader *header, char *payload, + UndoRecPtr urec_ptr) +{ + const UndoRmgrData *rmgr; + UndoApplyResult result; + + /* + * Idempotency design note: + * + * UNDO records are immutable once written to WAL and carry no per-record + * applied marker. Double-application is prevented by page LSN instead: + * each CLR (XLOG_UNDO_APPLY_RECORD) written by rm_undo bumps the heap + * page LSN to the CLR's EndRecPtr. During crash recovery Phase 2, + * rm_undo reads the buffer via XLogReadBufferForRedo; if the page LSN >= + * CLR LSN (meaning the CLR was already replayed in Phase 1), the buffer + * read returns BLK_DONE or BLK_RESTORED and no modification is made. + * + * This function is therefore unconditionally correct to call for every + * UNDO record encountered during chain walking. + */ + + /* + * Look up the resource manager for this record. + */ + rmgr = GetUndoRmgr(header->urec_rmid); + if (rmgr == NULL) + { + ereport(WARNING, + (errmsg("UNDO rollback: unknown RM ID %u for record at %llu, skipping", + header->urec_rmid, + (unsigned long long) urec_ptr))); + return false; + } + + /* + * Dispatch to the RM's undo-apply callback. The callback is responsible + * for all AM-specific work: opening relations, locking buffers, modifying + * pages, generating CLRs, and releasing resources. + */ + result = rmgr->rm_undo(header->urec_rmid, + header->urec_info, + header->urec_xid, + header->urec_reloid, + payload, + header->urec_payload_len, + urec_ptr); + + if (result == UNDO_APPLY_SUCCESS) + { + ereport(DEBUG2, + (errmsg("UNDO rollback: applied %s record at %llu", + rmgr->rm_name, + (unsigned long long) urec_ptr))); + return true; + } + else if (result == UNDO_APPLY_SKIPPED) + { + ereport(DEBUG2, + (errmsg("UNDO rollback: skipped %s record at %llu", + rmgr->rm_name, + (unsigned long long) urec_ptr))); + return false; + } + else + { + ereport(WARNING, + (errmsg("UNDO rollback: error applying %s record at %llu", + rmgr->rm_name, + (unsigned long long) urec_ptr))); + return false; + } +} + +/* + * ApplyUndoChainFromWAL - Walk and apply an UNDO chain from WAL + * + * Reads XLOG_UNDO_BATCH WAL records via UndoReadBatchFromWAL() and + * iterates through the serialized records within each batch. + * + * The chain is walked backward via the xl_undo_batch.chain_prev LSN + * from the most recent batch to the first. Within each batch, records + * are applied in reverse order (newest to oldest) as required by + * ARIES-style rollback semantics. This is achieved by first scanning + * forward through the serialized records to collect their start offsets, + * then iterating the collected offsets in reverse to apply each record. + */ +bool +ApplyUndoChainFromWAL(XLogRecPtr last_batch_lsn) +{ + return ApplyUndoChainFromWALBounded(last_batch_lsn, InvalidXLogRecPtr); +} + +/* + * ApplyUndoChainFromWALBounded - Apply an UNDO chain, stopping at a boundary + * + * Identical to ApplyUndoChainFromWAL() except that the chain walk halts before + * processing any batch whose LSN is at or below stop_at_lsn. This applies only + * the batches strictly newer than stop_at_lsn, which is what subtransaction + * abort needs: stop_at_lsn is the parent's saved chain head, so the parent's + * (and earlier subtransactions') batches are left intact while the aborting + * subtransaction's batches are reverted. + * + * Pass InvalidXLogRecPtr as stop_at_lsn to walk the entire chain (top-level + * rollback / recovery semantics). + */ +bool +ApplyUndoChainFromWALBounded(XLogRecPtr last_batch_lsn, XLogRecPtr stop_at_lsn) +{ + XLogRecPtr batch_lsn; + int records_applied = 0; + int records_skipped = 0; + int batches_processed = 0; + + if (!XLogRecPtrIsValid(last_batch_lsn)) + return false; + + ereport(DEBUG1, + (errmsg("applying UNDO chain from WAL starting at %X/%X " + "(stop boundary %X/%X)", + LSN_FORMAT_ARGS(last_batch_lsn), + LSN_FORMAT_ARGS(stop_at_lsn)))); + + batch_lsn = last_batch_lsn; + + while (XLogRecPtrIsValid(batch_lsn)) + { + UndoBatchData *batch; + char *pos; + char *end; + + /* + * Bounded walk: stop once we reach a batch that belongs to the parent + * (or an earlier subtransaction). chain_prev LSNs decrease strictly + * as we walk backward, so the first batch at or below the boundary + * marks the start of the region we must preserve. + */ + if (XLogRecPtrIsValid(stop_at_lsn) && batch_lsn <= stop_at_lsn) + break; + + INJECTION_POINT("undo-apply-before-batch", NULL); + + batch = UndoReadBatchFromWAL(batch_lsn); + if (batch == NULL) + { + ereport(WARNING, + (errmsg("UNDO rollback: could not read batch at %X/%X, " + "stopping chain walk", + LSN_FORMAT_ARGS(batch_lsn)))); + break; + } + + batches_processed++; + + /* + * Walk through records within this batch in reverse order. + * + * ARIES requires that UNDO records within a batch be applied + * newest-first (reverse of serialization order). We first scan + * forward to collect pointers to each record start, then iterate the + * collected pointers in reverse to apply them. + */ + pos = batch->payload; + end = pos + batch->payload_len; + + { + char **record_starts; + int nrecords_in_batch = 0; + int max_records = 1024; + int i; + MemoryContext batchctx; + MemoryContext oldctx; + + /* + * record_starts must grow to hold EVERY record in the batch -- + * silently truncating rollback is a data-integrity bug (the + * dropped records are never applied, so a partially-applied + * in-place UPDATE/DELETE is left on the page). We therefore + * repalloc() to double the array whenever it fills. + * + * The caller's CurrentMemoryContext may be a BumpContext + * (executor abort path), which supports neither repalloc() nor + * pfree(). So we do all growth in a private AllocSet child + * context and delete it whole at the end of the batch -- + * context-agnostic, leak-free, and unbounded. + */ + batchctx = AllocSetContextCreate(CurrentMemoryContext, + "UNDO batch record_starts", + ALLOCSET_DEFAULT_SIZES); + oldctx = MemoryContextSwitchTo(batchctx); + + record_starts = (char **) palloc(max_records * sizeof(char *)); + + /* First pass: collect record start pointers by scanning forward */ + while (pos < end) + { + UndoRecordHeader hdr; + + if ((Size) (end - pos) < SizeOfUndoRecordHeader) + { + ereport(WARNING, + (errmsg("UNDO rollback: truncated record in batch at %X/%X", + LSN_FORMAT_ARGS(batch_lsn)))); + break; + } + + memcpy(&hdr, pos, SizeOfUndoRecordHeader); + + if (hdr.urec_len < SizeOfUndoRecordHeader || + (Size) (end - pos) < hdr.urec_len) + { + ereport(WARNING, + (errmsg("UNDO rollback: invalid record size %u in batch at %X/%X", + hdr.urec_len, LSN_FORMAT_ARGS(batch_lsn)))); + break; + } + + /* + * Grow the array (double it) rather than truncate the + * rollback + */ + if (nrecords_in_batch >= max_records) + { + max_records *= 2; + record_starts = (char **) repalloc(record_starts, + max_records * sizeof(char *)); + } + + record_starts[nrecords_in_batch++] = pos; + pos += hdr.urec_len; + } + + /* + * Second pass: apply records in reverse order (newest first). + * Even if there was a scan error, apply whatever records we + * successfully collected. + */ + for (i = nrecords_in_batch - 1; i >= 0; i--) + { + UndoRecordHeader header; + char *payload = NULL; + + memcpy(&header, record_starts[i], SizeOfUndoRecordHeader); + + if (header.urec_payload_len > 0) + payload = record_starts[i] + SizeOfUndoRecordHeader; + + if (ApplyOneUndoRecord(&header, payload, InvalidUndoRecPtr)) + records_applied++; + else + records_skipped++; + } + + MemoryContextSwitchTo(oldctx); + MemoryContextDelete(batchctx); + } + + INJECTION_POINT("undo-apply-after-batch", NULL); + + /* Follow chain to previous batch */ + batch_lsn = batch->header.chain_prev; + UndoFreeBatchData(batch); + } + + /* Report results */ + if (records_skipped > 0) + { + ereport(WARNING, + (errmsg("UNDO rollback from WAL: %d batches, %d records applied, " + "%d skipped", + batches_processed, records_applied, records_skipped))); + } + else + { + ereport(DEBUG1, + (errmsg("UNDO rollback from WAL complete: %d batches, " + "%d records applied", + batches_processed, records_applied))); + } + + return (batches_processed > 0); +} diff --git a/src/backend/access/undo/undobuffer.c b/src/backend/access/undo/undobuffer.c new file mode 100644 index 0000000000000..2c6b62c6ecc91 --- /dev/null +++ b/src/backend/access/undo/undobuffer.c @@ -0,0 +1,351 @@ +/*------------------------------------------------------------------------- + * + * undobuffer.c + * AM-agnostic Tier 2 UNDO write buffer + * + * This module implements a per-backend byte buffer that accumulates + * serialized UNDO records for the current DML operation. At WAL-write time, + * the buffer contents are embedded directly inside the AM's WAL record, + * eliminating a separate XLOG_UNDO_BATCH record for single-tuple operations. + * + * The buffer logic is entirely AM-agnostic: it serializes UndoRecordHeaders + * with opaque payloads, identified by urec_rmid for dispatch during rollback. + * Any access method (heap, nbtree, custom AMs) can use this buffer. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/access/undo/undobuffer.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/undobuffer.h" +#include "access/undolog.h" +#include "access/undorecord.h" +#include "access/undo_xlog.h" +#include "access/xactundo.h" +#include "access/xact.h" +#include "access/xlog.h" +#include "access/xloginsert.h" +#include "utils/memutils.h" +#include "utils/rel.h" + +/* + * Flush thresholds. Tunable via the undo_batch_size_kb and + * undo_batch_record_limit GUCs (src/backend/access/undo/undolog.c). + */ +#define UNDO_BUFFER_FLUSH_THRESHOLD (undo_batch_size_kb * 1024) +#define UNDO_BUFFER_FLUSH_RECORDS undo_batch_record_limit + +/* + * Per-backend Tier 2 UNDO buffer. Only one relation can be active at a time. + */ +typedef struct UndoTier2Buffer +{ + char *data; /* palloc'd: serialized + * UndoRecordHeader+payload */ + Size len; /* bytes currently used */ + Size capacity; /* allocated capacity */ + int nrecords; /* records in buffer */ + TransactionId xid; /* owning transaction */ + XLogRecPtr chain_prev; /* LSN of previous UNDO batch for chain + * linkage */ + Oid relid; /* OID of the relation with active buffer */ + bool active; +} UndoTier2Buffer; + +static UndoTier2Buffer undo_t2buf = +{ + .data = NULL, + .len = 0, + .capacity = 0, + .nrecords = 0, + .xid = InvalidTransactionId, + .chain_prev = InvalidXLogRecPtr, + .relid = InvalidOid, + .active = false, +}; + +/* + * UndoTier2EnsureCapacity - grow undo_t2buf to hold additional bytes + */ +static void +UndoTier2EnsureCapacity(Size additional) +{ + if (undo_t2buf.len + additional <= undo_t2buf.capacity) + return; /* already enough room */ + + if (undo_t2buf.capacity == 0) + { + undo_t2buf.capacity = Max(512, additional); + undo_t2buf.data = MemoryContextAlloc(TopMemoryContext, + undo_t2buf.capacity); + } + else + { + Size new_cap = undo_t2buf.capacity; + + while (new_cap < undo_t2buf.len + additional) + new_cap *= 2; + undo_t2buf.data = repalloc(undo_t2buf.data, new_cap); + undo_t2buf.capacity = new_cap; + } +} + +/* + * UndoTier2AddRecord - serialize one UNDO record into undo_t2buf + */ +static void +UndoTier2AddRecord(uint8 rmid, uint16 info, Oid reloid, + const char *payload, Size payload_len) +{ + Size record_size = SizeOfUndoRecordHeader + payload_len; + UndoRecordHeader *header; + char *dest; + + UndoTier2EnsureCapacity(record_size); + + dest = undo_t2buf.data + undo_t2buf.len; + header = (UndoRecordHeader *) dest; + memset(header, 0, SizeOfUndoRecordHeader); + header->urec_rmid = rmid; + header->urec_flags = UNDO_INFO_XID_VALID; + if (payload_len > 0) + header->urec_flags |= UNDO_INFO_HAS_PAYLOAD; + header->urec_info = info; + header->urec_len = (uint32) record_size; + header->urec_xid = undo_t2buf.xid; + header->urec_prev = (UndoRecPtr) undo_t2buf.chain_prev; + header->urec_reloid = reloid; + header->urec_payload_len = (uint32) payload_len; + + if (payload_len > 0 && payload != NULL) + memcpy(dest + SizeOfUndoRecordHeader, payload, payload_len); + + undo_t2buf.len += record_size; + undo_t2buf.nrecords++; +} + +/* + * UndoTier2AddRecordParts - like UndoTier2AddRecord but scatter-gather + */ +static void +UndoTier2AddRecordParts(uint8 rmid, uint16 info, Oid reloid, + const char *part1, Size part1_len, + const char *part2, Size part2_len) +{ + Size payload_len = part1_len + part2_len; + Size record_size = SizeOfUndoRecordHeader + payload_len; + UndoRecordHeader *header; + char *dest; + + UndoTier2EnsureCapacity(record_size); + + dest = undo_t2buf.data + undo_t2buf.len; + header = (UndoRecordHeader *) dest; + memset(header, 0, SizeOfUndoRecordHeader); + header->urec_rmid = rmid; + header->urec_flags = UNDO_INFO_XID_VALID; + if (payload_len > 0) + header->urec_flags |= UNDO_INFO_HAS_PAYLOAD; + header->urec_info = info; + header->urec_len = (uint32) record_size; + header->urec_xid = undo_t2buf.xid; + header->urec_prev = (UndoRecPtr) undo_t2buf.chain_prev; + header->urec_reloid = reloid; + header->urec_payload_len = (uint32) payload_len; + + dest += SizeOfUndoRecordHeader; + if (part1_len > 0 && part1 != NULL) + memcpy(dest, part1, part1_len); + if (part2_len > 0 && part2 != NULL) + memcpy(dest + part1_len, part2, part2_len); + + undo_t2buf.len += record_size; + undo_t2buf.nrecords++; +} + + +/* ----------------------------------------------------------------------- + * Public API + * ----------------------------------------------------------------------- + */ + +void +UndoBufferBegin(Relation rel, int64 nrows) +{ + /* Only one relation at a time can have an active buffer */ + if (undo_t2buf.active) + { + if (undo_t2buf.relid == RelationGetRelid(rel)) + return; /* already active for this relation */ + + /* Different relation -- flush and end the previous one */ + UndoBufferEnd(rel); + } + + undo_t2buf.xid = GetCurrentTransactionId(); + undo_t2buf.relid = RelationGetRelid(rel); + undo_t2buf.chain_prev = (XLogRecPtr) GetCurrentTransactionUndoRecPtr(); + undo_t2buf.len = 0; + undo_t2buf.nrecords = 0; + undo_t2buf.active = true; + /* undo_t2buf.data and capacity are preserved across activations */ + + ereport(DEBUG2, + (errmsg("UNDO tier2 buffer activated for relation %u, estimated %lld rows", + RelationGetRelid(rel), (long long) nrows))); +} + +void +UndoBufferEnd(Relation rel) +{ + if (!undo_t2buf.active) + return; + + /* Flush any remaining records via the overflow path */ + if (undo_t2buf.nrecords > 0) + UndoBufferFlush(); + + ereport(DEBUG2, + (errmsg("UNDO tier2 buffer deactivated for relation %u", + undo_t2buf.relid))); + + undo_t2buf.relid = InvalidOid; + undo_t2buf.len = 0; + undo_t2buf.nrecords = 0; + undo_t2buf.xid = InvalidTransactionId; + undo_t2buf.chain_prev = InvalidXLogRecPtr; + undo_t2buf.active = false; +} + +bool +UndoBufferIsActive(Relation rel) +{ + return undo_t2buf.active && + undo_t2buf.relid == RelationGetRelid(rel); +} + +void +UndoBufferAddRecord(Relation rel, uint8 rmid, uint16 info, + const char *payload, Size payload_len) +{ + Assert(undo_t2buf.active); + + UndoTier2AddRecord(rmid, info, RelationGetRelid(rel), + payload, payload_len); + + /* Overflow flush when thresholds are exceeded */ + if (undo_t2buf.len >= UNDO_BUFFER_FLUSH_THRESHOLD || + undo_t2buf.nrecords >= UNDO_BUFFER_FLUSH_RECORDS) + UndoBufferFlush(); +} + +void +UndoBufferAddRecordParts(Relation rel, uint8 rmid, uint16 info, + const char *part1, Size part1_len, + const char *part2, Size part2_len) +{ + Assert(undo_t2buf.active); + + UndoTier2AddRecordParts(rmid, info, RelationGetRelid(rel), + part1, part1_len, part2, part2_len); + + /* Overflow flush when thresholds are exceeded */ + if (undo_t2buf.len >= UNDO_BUFFER_FLUSH_THRESHOLD || + undo_t2buf.nrecords >= UNDO_BUFFER_FLUSH_RECORDS) + UndoBufferFlush(); +} + +bool +UndoBufferHasPendingData(void) +{ + return undo_t2buf.active && undo_t2buf.nrecords > 0; +} + +void +UndoBufferTakePayload(char **data_out, Size *len_out, int *nrecords_out, + XLogRecPtr *chain_prev_out) +{ + Assert(undo_t2buf.active); + Assert(undo_t2buf.nrecords > 0); + + *data_out = undo_t2buf.data; + *len_out = undo_t2buf.len; + *nrecords_out = undo_t2buf.nrecords; + *chain_prev_out = undo_t2buf.chain_prev; +} + +void +UndoBufferReset(XLogRecPtr embedded_lsn) +{ + /* Update chain head so the next batch links to this one */ + undo_t2buf.chain_prev = embedded_lsn; + undo_t2buf.len = 0; + undo_t2buf.nrecords = 0; + + /* + * Update the per-transaction undo pointer so that subsequent + * UndoBufferBegin calls (for different relations in the same transaction) + * pick up the correct chain_prev. Without this, multi-table transactions + * would break the UNDO chain. + */ + SetCurrentTransactionUndoRecPtr((UndoRecPtr) embedded_lsn); +} + +void +UndoBufferFlush(void) +{ + xl_undo_batch xlrec; + XLogRecPtr batch_lsn; + Oid primary_reloid = InvalidOid; + + if (!undo_t2buf.active || undo_t2buf.nrecords == 0) + return; + + /* Extract primary reloid from first record as an optimization hint */ + if (undo_t2buf.len >= SizeOfUndoRecordHeader) + { + UndoRecordHeader *first_hdr = (UndoRecordHeader *) undo_t2buf.data; + + primary_reloid = first_hdr->urec_reloid; + } + + /* Build the batch header */ + xlrec.xid = undo_t2buf.xid; + xlrec.chain_prev = undo_t2buf.chain_prev; + xlrec.nrecords = (uint32) undo_t2buf.nrecords; + xlrec.total_len = (uint32) undo_t2buf.len; + xlrec.primary_reloid = primary_reloid; + xlrec.persistence = UNDOPERSISTENCE_PERMANENT; + + XLogBeginInsert(); + XLogRegisterData((char *) &xlrec, SizeOfUndoBatch); + XLogRegisterData(undo_t2buf.data, undo_t2buf.len); + (void) XLogInsert(RM_UNDO_ID, XLOG_UNDO_BATCH); + + /* + * XLogInsert() returns the end+1 position; the rollback path re-reads the + * batch by its START LSN. ProcLastRecPtr is the start of the record we + * just inserted. See the matching comment in UndoRecordSetInsert(). + */ + batch_lsn = ProcLastRecPtr; + + /* Update chain tracking */ + undo_t2buf.chain_prev = batch_lsn; + UndoRegisterBatchLSN(batch_lsn); + XActUndoUpdateLastBatchLSN(batch_lsn, UNDOPERSISTENCE_PERMANENT); + SetCurrentTransactionUndoRecPtr((UndoRecPtr) batch_lsn); + + ereport(DEBUG2, + (errmsg("UNDO tier2 overflow flush: %d records, %zu bytes, lsn %X/%X", + undo_t2buf.nrecords, undo_t2buf.len, + LSN_FORMAT_ARGS(batch_lsn)))); + + /* Reset buffer for next batch */ + undo_t2buf.len = 0; + undo_t2buf.nrecords = 0; +} diff --git a/src/backend/access/undo/undoinsert.c b/src/backend/access/undo/undoinsert.c new file mode 100644 index 0000000000000..47ceb92877aea --- /dev/null +++ b/src/backend/access/undo/undoinsert.c @@ -0,0 +1,166 @@ +/*------------------------------------------------------------------------- + * + * undoinsert.c + * UNDO record batch insertion operations + * + * This file implements batch insertion of UNDO records into the WAL + * stream. Records are accumulated in an UndoRecordSet and then + * written as a single XLOG_UNDO_BATCH WAL record. + * + * UNDO-IN-WAL ARCHITECTURE + * ------------------------ + * All UNDO record data flows through the standard WAL pipeline: + * UndoRecordSetInsert() -> XLogBeginInsert() + * -> XLogRegisterData(batch_header) + * -> XLogRegisterData(uset->buffer, uset->buffer_size) + * -> XLogInsert(RM_UNDO_ID, XLOG_UNDO_BATCH) + * + * This eliminates the separate UNDO segment file I/O path (pwrite + + * fdatasync) and provides: + * - Replicas receive and can apply UNDO records + * - One durability path, one sync at commit + * - Unified crash recovery with explicit UNDO phase + * + * Coalescing: The existing UndoRecordSet mechanism batches records. + * This batch becomes one WAL record. A 1000-row INSERT produces ~1 + * WAL record containing 1000 UNDO records. + * + * Legacy support: UndoWalBatchFlush/Reset are kept as no-ops for + * callers that haven't been updated yet. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/access/undo/undoinsert.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/undolog.h" +#include "access/undorecord.h" +#include "access/undo_xlog.h" +#include "access/xloginsert.h" +#include "access/xlog.h" +#include "utils/injection_point.h" + +/* + * UndoWalBatchFlush - Legacy no-op + * + * With UNDO-in-WAL, there is no separate deferred WAL batch to flush. + * UNDO data is written directly to WAL in UndoRecordSetInsert(). + * This function is kept for callers that haven't been updated yet. + */ +void +UndoWalBatchFlush(void) +{ + /* No-op: UNDO data is now written directly to WAL */ +} + +/* + * UndoWalBatchReset - Legacy no-op + * + * With UNDO-in-WAL, there is no separate deferred WAL batch to reset. + */ +void +UndoWalBatchReset(void) +{ + /* No-op: UNDO data is now written directly to WAL */ +} + +/* + * UndoRecordSetInsert - Insert accumulated UNDO records into WAL + * + * This function writes all UNDO records in the set as a single + * XLOG_UNDO_BATCH WAL record. The batch payload is the serialized + * content of uset->buffer (concatenated UndoRecordHeader+payload). + * + * Returns the (legacy) UndoRecPtr for backward compatibility. + * The actual record location is the XLogRecPtr stored in + * uset->last_batch_lsn after this call. + */ +UndoRecPtr +UndoRecordSetInsert(UndoRecordSet *uset) +{ + xl_undo_batch xlrec; + XLogRecPtr batch_lsn; + Oid primary_reloid = InvalidOid; + + if (uset == NULL || uset->nrecords == 0) + return InvalidUndoRecPtr; + + /* + * Extract the primary relation OID from the first record in the batch as + * an optimization hint. Most batches contain records for a single + * relation. + */ + if (uset->buffer_size >= SizeOfUndoRecordHeader) + { + UndoRecordHeader *first_hdr = (UndoRecordHeader *) uset->buffer; + + primary_reloid = first_hdr->urec_reloid; + } + + /* Build the batch header */ + xlrec.xid = uset->xid; + xlrec.chain_prev = uset->last_batch_lsn; + xlrec.nrecords = (uint32) uset->nrecords; + xlrec.total_len = (uint32) uset->buffer_size; + xlrec.primary_reloid = primary_reloid; + xlrec.persistence = uset->persistence; + + /* + * Write the UNDO batch as a single WAL record. + * + * XLogRegisterData has no size limit on main data (tracked as uint64 in + * xloginsert.c), so even a 256KB batch is fine. The WAL insertion lock + * will be held for the duration of the record write, which is acceptable + * for batch sizes up to a few hundred KB. + */ + XLogBeginInsert(); + XLogRegisterData((char *) &xlrec, SizeOfUndoBatch); + XLogRegisterData(uset->buffer, uset->buffer_size); + + /* + * Use the CACHED variant: UndoRecordSetInsert() always runs inside the + * caller's critical section (it emits a WAL record), and + * InjectionPointRun() may palloc (dlopen the callback library, or a + * wait-mode callback allocating to suspend the backend), which is + * forbidden in a critical section. The matching INJECTION_POINT_LOAD is + * issued pre-crit in PrepareXactUndoData(); INJECTION_POINT_CACHED is + * palloc-free. + */ + INJECTION_POINT_CACHED("undo-batch-before-wal-insert", NULL); + + (void) XLogInsert(RM_UNDO_ID, XLOG_UNDO_BATCH); + + /* + * XLogInsert() returns the end+1 position of the record, but the rollback + * path must re-read the batch by its START LSN. ProcLastRecPtr holds the + * start of the record we just inserted; use it so UndoReadBatchFromWAL + * lands on this XLOG_UNDO_BATCH instead of the following record. + */ + batch_lsn = ProcLastRecPtr; + + INJECTION_POINT_CACHED("undo-batch-after-wal-insert", NULL); + + /* Update the record set's chain pointer for subsequent batches */ + uset->last_batch_lsn = batch_lsn; + + /* + * Register the batch LSN for WAL retention tracking. Only the first call + * per transaction takes effect (UndoRegisterBatchLSN is a no-op if the + * slot is already occupied), so this records the oldest batch for this + * transaction without additional bookkeeping. + */ + UndoRegisterBatchLSN(batch_lsn); + + /* + * For legacy compatibility, return a non-zero UndoRecPtr. The actual + * location is in uset->last_batch_lsn (XLogRecPtr). + */ + uset->prev_undo_ptr = (UndoRecPtr) batch_lsn; + + return (UndoRecPtr) batch_lsn; +} diff --git a/src/backend/access/undo/undolog.c b/src/backend/access/undo/undolog.c new file mode 100644 index 0000000000000..7af37e03c3c4b --- /dev/null +++ b/src/backend/access/undo/undolog.c @@ -0,0 +1,534 @@ +/*------------------------------------------------------------------------- + * + * undolog.c + * PostgreSQL UNDO log manager -- WAL-integrated version + * + * With UNDO-in-WAL, UNDO records are stored in the standard WAL stream + * as XLOG_UNDO_BATCH records. The separate base/undo/ segment files, + * direct pwrite()/pread() I/O path, and per-backend fd cache have been + * removed. This file retains: + * + * - GUC parameters (undo_retention_time, etc.) + * - Shared memory structures for UNDO state tracking + * - Discard pointer management (repurposed for WAL-based UNDO) + * - Checkpoint support (statistics logging) + * + * The previous functions (UndoLogAllocate, UndoLogWrite, UndoLogRead, + * UndoLogSync, UndoLogSealAndRotate, etc.) are removed. Callers now + * use UndoRecordSetInsert() which writes directly to WAL via XLogInsert. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/access/undo/undolog.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/atm.h" +#include "access/transam.h" +#include "access/twophase.h" +#include "access/undolog.h" +#include "access/xlog.h" +#include "miscadmin.h" +#include "storage/lwlock.h" +#include "storage/procnumber.h" +#include "storage/shmem.h" +#include "utils/guc.h" +#include "utils/timestamp.h" + +/* GUC parameters */ +int undo_retention_time = 60000; /* 60 seconds */ +int undo_worker_naptime = 10000; /* 10 seconds */ +int undo_buffer_size = 1024; /* 1MB in KB */ +int undo_max_wal_retention_size = 0; /* 0 = unlimited, in MB */ +int undo_batch_size_kb = 256; /* UNDO batch flush threshold in KB */ +int undo_batch_record_limit = 1000; /* UNDO batch flush threshold in + * records */ + +/* Shared memory pointer */ +UndoLogSharedData *UndoLogShared = NULL; + +/* + * UndoLogShmemSize + * Calculate shared memory size for UNDO log management + * + * The size includes the fixed UndoLogSharedData fields plus a per-backend + * array of pg_atomic_uint64 for first UNDO batch LSN tracking. + */ +Size +UndoLogShmemSize(void) +{ + Size size; + + /* Fixed struct size up to (but not including) the flexible array */ + size = offsetof(UndoLogSharedData, backend_undo_lsns); + + /* Per-backend first-batch LSN slots */ + size = add_size(size, mul_size(MaxBackends, sizeof(pg_atomic_uint64))); + + return size; +} + +/* + * UndoLogShmemInit + * Initialize shared memory for UNDO log management + */ +void +UndoLogShmemInit(void) +{ + bool found; + + UndoLogShared = (UndoLogSharedData *) + ShmemInitStruct("UNDO Log Control", UndoLogShmemSize(), &found); + + if (!found) + { + int i; + + /* Initialize all log control structures */ + for (i = 0; i < MAX_UNDO_LOGS; i++) + { + UndoLogControl *log = &UndoLogShared->logs[i]; + + log->log_number = 0; + pg_atomic_init_u64(&log->insert_ptr, InvalidUndoRecPtr); + log->discard_ptr = InvalidUndoRecPtr; + log->oldest_xid = InvalidTransactionId; + LWLockInitialize(&log->lock, LWTRANCHE_UNDO_LOG); + log->in_use = false; + log->state = UNDO_LOG_FREE; + pg_atomic_init_u64(&log->seal_ptr, InvalidUndoRecPtr); + log->sealed_time = 0; + } + + UndoLogShared->next_log_number = 1; + LWLockInitialize(&UndoLogShared->allocation_lock, LWTRANCHE_UNDO_LOG); + pg_atomic_init_u32(&UndoLogShared->active_log_idx, MAX_UNDO_LOGS); + pg_atomic_init_u64(&UndoLogShared->total_allocated, 0); + pg_atomic_init_u64(&UndoLogShared->total_discarded, 0); + pg_atomic_init_u64(&UndoLogShared->undo_discard_horizon, + InvalidXLogRecPtr); + + /* Initialize per-backend first UNDO batch LSN slots */ + for (i = 0; i < MaxBackends; i++) + pg_atomic_init_u64(&UndoLogShared->backend_undo_lsns[i], + InvalidXLogRecPtr); + } +} + +/* + * UndoLogDiscard + * Advance the UNDO discard horizon. + * + * With UNDO-in-WAL, discard means advancing the WAL retention horizon + * past which UNDO records are no longer needed for rollback. The + * background UNDO worker calls this after confirming all transactions + * older than oldest_needed have committed or had their UNDO applied. + */ +void +UndoLogDiscard(UndoRecPtr oldest_needed) +{ + int i; + + if (!UndoRecPtrIsValid(oldest_needed)) + return; + + for (i = 0; i < MAX_UNDO_LOGS; i++) + { + UndoLogControl *log = &UndoLogShared->logs[i]; + + if (!log->in_use) + continue; + + LWLockAcquire(&log->lock, LW_EXCLUSIVE); + + if (UndoRecPtrGetLogNo(oldest_needed) == log->log_number) + { + if (UndoRecPtrGetOffset(oldest_needed) > UndoRecPtrGetOffset(log->discard_ptr)) + { + log->discard_ptr = oldest_needed; + ereport(DEBUG2, + (errmsg("UNDO discard: log %u advanced to offset %llu", + log->log_number, + (unsigned long long) UndoRecPtrGetOffset(oldest_needed)))); + } + } + + LWLockRelease(&log->lock); + } +} + +/* + * UndoLogGetOldestDiscardPtr + * Get the oldest UNDO discard pointer across all active logs. + * + * Used to determine WAL retention requirements for UNDO. + */ +UndoRecPtr +UndoLogGetOldestDiscardPtr(void) +{ + UndoRecPtr oldest = InvalidUndoRecPtr; + int i; + + for (i = 0; i < MAX_UNDO_LOGS; i++) + { + UndoLogControl *log = &UndoLogShared->logs[i]; + + if (log->in_use) + { + if (!UndoRecPtrIsValid(oldest) || + log->discard_ptr < oldest) + oldest = log->discard_ptr; + } + } + + return oldest; +} + +/* + * CheckPointUndoLog + * Perform checkpoint processing for the UNDO log subsystem. + * + * With UNDO-in-WAL, there are no UNDO segment files to sync. + * This function logs statistics when log_checkpoints is enabled. + */ +void +CheckPointUndoLog(void) +{ + int active_logs = 0; + uint64 total_allocated = 0; + uint64 total_discarded = 0; + int i; + + if (UndoLogShared == NULL) + return; + + for (i = 0; i < MAX_UNDO_LOGS; i++) + { + UndoLogControl *log = &UndoLogShared->logs[i]; + + if (!log->in_use) + continue; + + active_logs++; + total_allocated += UndoRecPtrGetOffset(pg_atomic_read_u64(&log->insert_ptr)); + + LWLockAcquire(&log->lock, LW_SHARED); + total_discarded += UndoRecPtrGetOffset(log->discard_ptr); + LWLockRelease(&log->lock); + } + + if (log_checkpoints && active_logs > 0) + { + ereport(LOG, + (errmsg("UNDO checkpoint: %d active log(s), " + "%llu bytes allocated, %llu bytes discarded, " + "%llu bytes retained", + active_logs, + (unsigned long long) total_allocated, + (unsigned long long) total_discarded, + (unsigned long long) (total_allocated - total_discarded)))); + } +} + +/* + * UndoGetDiscardHorizon + * Return the current UNDO discard horizon LSN. + * + * WAL segments containing data at or after this LSN must be retained + * because they contain UNDO records that may still be needed for + * rollback of in-progress transactions. + * + * Returns InvalidXLogRecPtr if no UNDO data exists (UNDO not in use + * or all transactions have committed). + */ +XLogRecPtr +UndoGetDiscardHorizon(void) +{ + if (UndoLogShared == NULL) + return InvalidXLogRecPtr; + + return (XLogRecPtr) pg_atomic_read_u64(&UndoLogShared->undo_discard_horizon); +} + +/* + * UndoSetDiscardHorizon + * Advance the UNDO discard horizon to a new LSN. + * + * Called by the UNDO discard worker after confirming that all UNDO + * records before 'horizon' have been processed (transactions committed + * or rolled back, index pruning completed). + * + * The horizon only moves forward -- if the new value is older than + * the current horizon, the call is a no-op. + */ +void +UndoSetDiscardHorizon(XLogRecPtr horizon) +{ + uint64 old_horizon; + + if (UndoLogShared == NULL || !XLogRecPtrIsValid(horizon)) + return; + + /* Advance forward only */ + while (true) + { + old_horizon = pg_atomic_read_u64(&UndoLogShared->undo_discard_horizon); + + if (XLogRecPtrIsValid((XLogRecPtr) old_horizon) && + horizon <= (XLogRecPtr) old_horizon) + break; /* already at or past this point */ + + if (pg_atomic_compare_exchange_u64(&UndoLogShared->undo_discard_horizon, + &old_horizon, (uint64) horizon)) + break; + } +} + +/* + * UndoRegisterBatchLSN + * Register the first UNDO batch LSN for the current backend. + * + * Called from UndoRecordSetInsert() the first time a transaction writes + * UNDO data. Stores the LSN in the per-backend slot so that the UNDO + * discard worker can find the oldest in-flight UNDO batch and avoid + * recycling WAL segments that still contain needed UNDO data. + * + * Only the FIRST call per transaction takes effect (we want the oldest, + * i.e., smallest, LSN). Subsequent calls for the same transaction are + * no-ops because the slot is already occupied. + */ +void +UndoRegisterBatchLSN(XLogRecPtr batch_lsn) +{ + pg_atomic_uint64 *slot; + uint64 expected; + + if (UndoLogShared == NULL || !XLogRecPtrIsValid(batch_lsn)) + return; + if (MyProcNumber < 0 || MyProcNumber >= MaxBackends) + return; + + slot = &UndoLogShared->backend_undo_lsns[MyProcNumber]; + expected = InvalidXLogRecPtr; + + /* + * Only set if the slot is currently empty. This records the first + * (oldest) batch for this transaction; later batches have larger LSNs and + * should not overwrite the stored value. + */ + (void) pg_atomic_compare_exchange_u64(slot, &expected, (uint64) batch_lsn); +} + +/* + * UndoClearBatchLSN + * Clear the per-backend UNDO batch LSN registration. + * + * Called at transaction commit or abort to release the WAL retention + * hold that was established by UndoRegisterBatchLSN(). + */ +void +UndoClearBatchLSN(void) +{ + if (UndoLogShared == NULL) + return; + if (MyProcNumber < 0 || MyProcNumber >= MaxBackends) + return; + + pg_atomic_write_u64(&UndoLogShared->backend_undo_lsns[MyProcNumber], + (uint64) InvalidXLogRecPtr); +} + +/* + * UndoGetOldestBatchLSN + * Return the oldest UNDO batch LSN that must be retained in WAL. + * + * Considers both: + * 1. Per-backend slots (in-flight transactions with UNDO data) + * 2. ATM entries (aborted transactions awaiting Logical Revert) + * + * The ATM check is critical: once a transaction aborts, its per-backend + * slot is cleared by UndoClearBatchLSN(), but the logical revert worker + * still needs to read the UNDO batches from WAL. Without this check, + * checkpoints could recycle WAL segments containing needed UNDO data, + * causing the revert worker to crash (SIGBUS/SIGSEGV) or read garbage. + * + * Returns InvalidXLogRecPtr if no WAL retention is needed for UNDO. + */ +XLogRecPtr +UndoGetOldestBatchLSN(void) +{ + XLogRecPtr oldest = InvalidXLogRecPtr; + XLogRecPtr atm_oldest; + int i; + + if (UndoLogShared == NULL) + return InvalidXLogRecPtr; + + /* Check per-backend slots for in-flight transactions */ + for (i = 0; i < MaxBackends; i++) + { + XLogRecPtr lsn = (XLogRecPtr) + pg_atomic_read_u64(&UndoLogShared->backend_undo_lsns[i]); + + if (XLogRecPtrIsValid(lsn)) + { + if (!XLogRecPtrIsValid(oldest) || lsn < oldest) + oldest = lsn; + } + } + + /* + * Check ATM for aborted transactions whose UNDO chains haven't been + * applied yet. Their WAL segments must not be recycled. + */ + atm_oldest = ATMGetOldestUnrevertedLSN(); + if (XLogRecPtrIsValid(atm_oldest)) + { + if (!XLogRecPtrIsValid(oldest) || atm_oldest < oldest) + oldest = atm_oldest; + } + + /* + * Check prepared (2PC) transactions. A xact can sit PREPARED + * indefinitely; its UNDO-batch WAL must survive until ROLLBACK PREPARED + * reads it. Neither the per-backend slot (cleared when the preparing + * backend exits) nor the ATM (prepared xacts aren't in it) covers this. + */ + { + XLogRecPtr prep_oldest = TwoPhaseGetOldestUndoBatchLSN(); + + if (XLogRecPtrIsValid(prep_oldest) && + (!XLogRecPtrIsValid(oldest) || prep_oldest < oldest)) + oldest = prep_oldest; + } + + return oldest; +} + +/* + * Legacy no-op stubs + * + * UNDO-in-WAL has no per-log segment files, no fd cache, and no + * per-backend write-pointer tracking: UNDO data lives in the WAL stream + * and durability/recycling are handled by WAL flush and the discard + * worker. The operations below are therefore inherently nothing-to-do in + * this mode, but they still have live callers in the shared transaction + * and discard paths (e.g. UndoLogCloseFiles / UndoFlushResetMaxWritePtr + * from xactundo.c, UndoLogDeleteSegmentFile from the discard worker, + * ExtendUndoLogFile from the undo_xlog.c redo path). We keep them as + * no-ops so those callers stay uniform across both UNDO modes rather than + * sprinkling mode checks at every call site; the no-op is the correct + * behaviour here, not a placeholder awaiting future work. + */ + +void +UndoLogSync(void) +{ + /* No-op: WAL sync handles durability */ +} + +void +UndoLogCloseFiles(void) +{ + /* No-op: no fd cache with UNDO-in-WAL */ +} + +void +UndoFlushResetMaxWritePtr(void) +{ + /* No-op: no per-backend write pointer tracking with UNDO-in-WAL */ +} + +UndoRecPtr +UndoFlushGetMaxWritePtr(void) +{ + /* No-op: no per-backend write pointer tracking with UNDO-in-WAL */ + return InvalidUndoRecPtr; +} + +void +UndoLogSealAndRotate(uint8 trigger pg_attribute_unused()) +{ + /* No-op: no segment rotation with UNDO-in-WAL */ +} + +void +UndoLogDeleteSegmentFile(uint32 log_number pg_attribute_unused()) +{ + /* No-op: no segment files with UNDO-in-WAL */ +} + +bool +UndoLogTryPressureDiscard(void) +{ + /* No-op: no segment pressure with UNDO-in-WAL */ + return false; +} + +char * +UndoLogPath(uint32 log_number, char *path) +{ + /* Legacy: construct the path even though files no longer exist */ + snprintf(path, MAXPGPATH, "base/undo/%012u", log_number); + return path; +} + +void +ExtendUndoLogFile(uint32 log_number pg_attribute_unused(), + uint64 logical_end pg_attribute_unused()) +{ + /* No-op: no segment files with UNDO-in-WAL */ +} + +void +ExtendUndoLogSmgrFile(uint32 log_number pg_attribute_unused(), + uint64 logical_end pg_attribute_unused()) +{ + /* No-op: no smgr-managed UNDO files with UNDO-in-WAL */ +} + +UndoRecPtr +UndoLogGetInsertPtr(uint32 log_number) +{ + int i; + UndoRecPtr ptr = InvalidUndoRecPtr; + + for (i = 0; i < MAX_UNDO_LOGS; i++) + { + UndoLogControl *log = &UndoLogShared->logs[i]; + + if (log->in_use && log->log_number == log_number) + { + ptr = pg_atomic_read_u64(&log->insert_ptr); + break; + } + } + + return ptr; +} + +UndoRecPtr +UndoLogGetDiscardPtr(uint32 log_number) +{ + int i; + UndoRecPtr ptr = InvalidUndoRecPtr; + + for (i = 0; i < MAX_UNDO_LOGS; i++) + { + UndoLogControl *log = &UndoLogShared->logs[i]; + + if (log->in_use && log->log_number == log_number) + { + LWLockAcquire(&log->lock, LW_SHARED); + ptr = log->discard_ptr; + LWLockRelease(&log->lock); + break; + } + } + + return ptr; +} diff --git a/src/backend/access/undo/undorecord.c b/src/backend/access/undo/undorecord.c new file mode 100644 index 0000000000000..ab6c62fa74869 --- /dev/null +++ b/src/backend/access/undo/undorecord.c @@ -0,0 +1,399 @@ +/*------------------------------------------------------------------------- + * + * undorecord.c + * UNDO record assembly and serialization + * + * This file implements the AM-agnostic UNDO record format and provides + * functions for creating, serializing, and deserializing UNDO records. + * All AM-specific knowledge is kept out of this module; records carry + * opaque payloads whose interpretation is delegated to the owning RM. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/access/undo/undorecord.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/undorecord.h" +#include "utils/memutils.h" + +/* + * Per-backend recycled memory context for UndoRecordSet. + * + * Instead of creating and destroying a MemoryContext for every + * UndoRecordSet, we recycle one context across operations within a + * transaction. This avoids the overhead of repeated + * AllocSetContextCreate/MemoryContextDelete for high-frequency + * operations (e.g., 1000-row INSERTs). The cached context is cleaned + * up at transaction end by UndoRecordSetResetCache(). + */ +static MemoryContext UndoRecordReusableContext = NULL; + +/* + * UndoRecordGetPayloadSize - Calculate size needed for an UNDO record + * + * This includes the fixed header plus the RM-specific payload. + */ +Size +UndoRecordGetPayloadSize(Size payload_len) +{ + return SizeOfUndoRecordHeader + payload_len; +} + +/* + * UndoRecordSerialize - Serialize an UNDO record into a buffer + * + * The destination buffer must be large enough to hold the entire record. + * Use UndoRecordGetPayloadSize() to determine the required size. + */ +void +UndoRecordSerialize(char *dest, UndoRecordHeader *header, + const char *payload, Size payload_len) +{ + /* Copy header */ + memcpy(dest, header, SizeOfUndoRecordHeader); + + /* Copy payload if present */ + if (payload_len > 0 && payload != NULL) + { + memcpy(dest + SizeOfUndoRecordHeader, payload, payload_len); + } +} + +/* + * UndoRecordDeserialize - Deserialize an UNDO record from a buffer + * + * Reads the header and sets the payload pointer into the source buffer + * (zero-copy). Returns true on success, false on failure. + */ +bool +UndoRecordDeserialize(const char *src, UndoRecordHeader *header, + char **payload) +{ + if (src == NULL || header == NULL) + return false; + + /* Copy header */ + memcpy(header, src, SizeOfUndoRecordHeader); + + /* Set payload pointer if there is payload data */ + if (header->urec_payload_len > 0) + { + if (payload != NULL) + *payload = (char *) (src + SizeOfUndoRecordHeader); + } + else + { + if (payload != NULL) + *payload = NULL; + } + + return true; +} + +/* + * UndoRecordSetCreate - Create a new UNDO record set + * + * A record set accumulates multiple UNDO records before writing them + * to the UNDO log in a batch. This improves performance by reducing + * I/O operations. + */ +UndoRecordSet * +UndoRecordSetCreate(TransactionId xid, UndoRecPtr prev_undo_ptr) +{ + UndoRecordSet *uset; + MemoryContext mctx; + MemoryContext parent; + + /* + * Use TopTransactionContext as the parent so the record set survives + * across SPI statement boundaries. When called from PL/pgSQL DO blocks, + * CurrentMemoryContext is the executor's per-query context + * (es_query_cxt), which is destroyed in FreeExecutorState() after each + * SPI_execute call. Since xactundo.c stores the uset pointer in the + * static XactUndo.record_set[] and reuses it across multiple statements + * within a transaction, the context must outlive any single query. + * TopTransactionContext is ideal: it survives until transaction + * commit/abort, and AtAbort cleanup will free the uset via + * UndoRecordSetFree(). + */ + parent = TopTransactionContext; + + /* + * Reuse a previously recycled memory context if available. This avoids + * the overhead of AllocSetContextCreate/MemoryContextDelete for every + * UndoRecordSet within a transaction. MemoryContextReset clears all + * allocations but keeps the context's memory blocks for reuse. + */ + if (UndoRecordReusableContext != NULL) + { + mctx = UndoRecordReusableContext; + UndoRecordReusableContext = NULL; /* take ownership */ + MemoryContextReset(mctx); + MemoryContextSetParent(mctx, parent); + } + else + { + mctx = AllocSetContextCreate(parent, + "UNDO record set", + ALLOCSET_DEFAULT_SIZES); + } + + /* + * Allocate everything in the uset's memory context using direct + * MemoryContextAlloc to avoid MemoryContextSwitchTo overhead. + */ + uset = (UndoRecordSet *) MemoryContextAllocZero(mctx, sizeof(UndoRecordSet)); + uset->xid = xid; + uset->prev_undo_ptr = prev_undo_ptr; + uset->persistence = UNDOPERSISTENCE_PERMANENT; + uset->type = URST_TRANSACTION; + + /* + * Allocate initial buffer. 512 bytes is enough for a single UNDO record + * (48-byte header + typical heap payload). For bulk mode the buffer + * grows dynamically via UndoRecordEnsureCapacity. + */ + uset->buffer_capacity = 512; + uset->buffer = (char *) MemoryContextAlloc(mctx, uset->buffer_capacity); + uset->buffer_size = 0; + + uset->last_batch_lsn = InvalidXLogRecPtr; + uset->mctx = mctx; + + return uset; +} + +/* + * UndoRecordSetFree - Free an UNDO record set + * + * Recycles the memory context for later reuse if possible, otherwise + * destroys it. We keep at most one recycled context to bound memory. + */ +void +UndoRecordSetFree(UndoRecordSet *uset) +{ + MemoryContext mctx; + + if (uset == NULL || uset->mctx == NULL) + return; + + mctx = uset->mctx; + + if (UndoRecordReusableContext == NULL) + { + /* + * Recycle this context for the next UndoRecordSetCreate call. + * + * Re-parent to TopMemoryContext so the cached context is not + * destroyed if its original parent is cleaned up before + * UndoRecordSetResetCache() runs. This can happen when the UNDO + * record set was created inside an SPI execution context (e.g., DO $$ + * ... $$ blocks): SPI_finish() deletes its procCxt/execCxt, which + * would recursively destroy this child context, leaving + * UndoRecordReusableContext as a dangling pointer. + * UndoRecordSetCreate() will re-parent it to the caller's + * CurrentMemoryContext on reuse. + */ + MemoryContextSetParent(mctx, TopMemoryContext); + UndoRecordReusableContext = mctx; + } + else + { + /* Already have one recycled context; destroy this one */ + MemoryContextDelete(mctx); + } +} + +/* + * UndoRecordEnsureCapacity - Ensure the uset buffer can hold additional bytes + * + * Grows the buffer (using the uset's memory context) if needed. + * Avoids MemoryContextSwitchTo overhead by using MemoryContextAlloc directly. + */ +static void +UndoRecordEnsureCapacity(UndoRecordSet *uset, Size additional) +{ + if (uset->buffer_size + additional > uset->buffer_capacity) + { + Size new_capacity = uset->buffer_capacity * 2; + char *newbuf; + + while (new_capacity < uset->buffer_size + additional) + new_capacity *= 2; + + newbuf = (char *) MemoryContextAlloc(uset->mctx, new_capacity); + if (uset->buffer_size > 0) + memcpy(newbuf, uset->buffer, uset->buffer_size); + pfree(uset->buffer); + uset->buffer = newbuf; + uset->buffer_capacity = new_capacity; + } +} + +/* + * UndoRecordSetReset - Reset a record set for reuse + * + * Resets the buffer position and record count without freeing the memory + * context or reallocating the buffer. This is much cheaper than + * UndoRecordSetCreate/Free (~5 cycles vs ~300 cycles) because it avoids + * MemoryContextReset/AllocSetContextCreate overhead entirely. + * + * The prev_undo_ptr and other metadata are preserved so the record set + * can continue chaining records correctly across multiple insertions + * within the same transaction. + */ +void +UndoRecordSetReset(UndoRecordSet *uset) +{ + if (uset == NULL) + return; + + uset->buffer_size = 0; + uset->nrecords = 0; +} + +/* + * UndoRecordSetResetCache - Release the recycled memory context. + * + * Called at transaction end (commit or abort) to ensure the cached + * context does not outlive the transaction. + */ +void +UndoRecordSetResetCache(void) +{ + if (UndoRecordReusableContext != NULL) + { + MemoryContextDelete(UndoRecordReusableContext); + UndoRecordReusableContext = NULL; + } +} + +/* + * UndoRecordAddPayload - Add an UNDO record with opaque payload to the set + * + * This is the main API for adding UNDO records. The caller provides an + * RM ID, RM-specific info flags, a relation OID, and an opaque payload. + * The payload's interpretation is entirely RM-specific. + */ +void +UndoRecordAddPayload(UndoRecordSet *uset, + uint8 rmid, + uint16 info, + Oid reloid, + const char *payload, + Size payload_len) +{ + UndoRecordHeader *header; + Size record_size; + char *dest; + + if (uset == NULL) + elog(ERROR, "cannot add UNDO record to NULL set"); + + record_size = UndoRecordGetPayloadSize(payload_len); + + /* Expand buffer if needed (allocate in the uset's memory context) */ + UndoRecordEnsureCapacity(uset, record_size); + + /* + * Build the header directly in the buffer, avoiding a separate stack + * variable, memset, and memcpy. We zero the header in-place to avoid + * uninitialized padding bytes in the on-disk format. + */ + dest = uset->buffer + uset->buffer_size; + header = (UndoRecordHeader *) dest; + memset(header, 0, SizeOfUndoRecordHeader); + header->urec_rmid = rmid; + header->urec_flags = UNDO_INFO_XID_VALID; + if (payload_len > 0) + header->urec_flags |= UNDO_INFO_HAS_PAYLOAD; + header->urec_info = info; + header->urec_len = (uint32) record_size; + header->urec_xid = uset->xid; + header->urec_prev = uset->prev_undo_ptr; + header->urec_reloid = reloid; + header->urec_payload_len = (uint32) payload_len; + + /* Copy payload directly after header */ + if (payload_len > 0 && payload != NULL) + memcpy(dest + SizeOfUndoRecordHeader, payload, payload_len); + + uset->buffer_size += record_size; + uset->nrecords++; +} + +/* + * UndoRecordAddPayloadParts - Add an UNDO record with scatter-gather payload + * + * Like UndoRecordAddPayload, but takes the payload as two parts that are + * concatenated directly into the uset buffer. This avoids allocating an + * intermediate payload buffer when the caller has the data in separate + * pieces (e.g., a fixed header struct + variable-length tuple data). + */ +void +UndoRecordAddPayloadParts(UndoRecordSet *uset, + uint8 rmid, + uint16 info, + Oid reloid, + const char *part1, + Size part1_len, + const char *part2, + Size part2_len) +{ + UndoRecordHeader *header; + Size payload_len = part1_len + part2_len; + Size record_size; + char *dest; + + if (uset == NULL) + elog(ERROR, "cannot add UNDO record to NULL set"); + + record_size = UndoRecordGetPayloadSize(payload_len); + + UndoRecordEnsureCapacity(uset, record_size); + + /* Build header directly in the buffer */ + dest = uset->buffer + uset->buffer_size; + header = (UndoRecordHeader *) dest; + memset(header, 0, SizeOfUndoRecordHeader); + header->urec_rmid = rmid; + header->urec_flags = UNDO_INFO_XID_VALID; + if (payload_len > 0) + header->urec_flags |= UNDO_INFO_HAS_PAYLOAD; + header->urec_info = info; + header->urec_len = (uint32) record_size; + header->urec_xid = uset->xid; + header->urec_prev = uset->prev_undo_ptr; + header->urec_reloid = reloid; + header->urec_payload_len = (uint32) payload_len; + + /* Copy payload parts directly after header */ + dest += SizeOfUndoRecordHeader; + if (part1_len > 0 && part1 != NULL) + { + memcpy(dest, part1, part1_len); + dest += part1_len; + } + if (part2_len > 0 && part2 != NULL) + memcpy(dest, part2, part2_len); + + uset->buffer_size += record_size; + uset->nrecords++; +} + +/* + * UndoRecordSetGetSize - Get total size of all records in set + */ +Size +UndoRecordSetGetSize(UndoRecordSet *uset) +{ + if (uset == NULL) + return 0; + + return uset->buffer_size; +} diff --git a/src/backend/access/undo/undormgr.c b/src/backend/access/undo/undormgr.c new file mode 100644 index 0000000000000..851ef61ae8d4f --- /dev/null +++ b/src/backend/access/undo/undormgr.c @@ -0,0 +1,70 @@ +/*------------------------------------------------------------------------- + * + * undormgr.c + * UNDO resource manager registration and dispatch + * + * This module manages the registration table for UNDO resource managers. + * Each access method or subsystem that writes UNDO records registers + * its callbacks here. The generic UNDO infrastructure dispatches to + * the appropriate callback based on the urec_rmid in the record header. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/access/undo/undormgr.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/undormgr.h" + +/* Global registration table, indexed by RM ID */ +const UndoRmgrData *UndoRmgrs[MAX_UNDO_RMGRS]; + +/* + * RegisterUndoRmgr - Register an UNDO resource manager + * + * Called by each AM/subsystem during initialization to register its + * UNDO apply and describe callbacks. + */ +void +RegisterUndoRmgr(uint8 rmid, const UndoRmgrData *rmgr) +{ + if (rmid == UNDO_RMID_INVALID) + elog(ERROR, "cannot register UNDO RM with invalid ID 0"); + + if (UndoRmgrs[rmid] != NULL) + elog(ERROR, "UNDO RM ID %u already registered as \"%s\"", + rmid, UndoRmgrs[rmid]->rm_name); + + if (rmgr->rm_undo == NULL) + elog(ERROR, "UNDO RM \"%s\" must provide an rm_undo callback", + rmgr->rm_name ? rmgr->rm_name : "(null)"); + + UndoRmgrs[rmid] = rmgr; +} + +/* + * GetUndoRmgr - Look up an UNDO resource manager by ID + * + * Returns the registration entry, or NULL if not registered. + */ +const UndoRmgrData * +GetUndoRmgr(uint8 rmid) +{ + return UndoRmgrs[rmid]; +} + +/* + * InitUndoRmgrs - Initialize the UNDO resource manager table + * + * Called during postmaster startup. Individual RMs register themselves + * via RegisterUndoRmgr() during their initialization. + */ +void +InitUndoRmgrs(void) +{ + MemSet(UndoRmgrs, 0, sizeof(UndoRmgrs)); +} diff --git a/src/backend/access/undo/undostats.c b/src/backend/access/undo/undostats.c new file mode 100644 index 0000000000000..554252cbc57cc --- /dev/null +++ b/src/backend/access/undo/undostats.c @@ -0,0 +1,375 @@ +/*------------------------------------------------------------------------- + * + * undostats.c + * UNDO log statistics collection and reporting + * + * This module provides monitoring and observability for the UNDO + * subsystem, including: + * - Per-log statistics (insert/discard pointers, size, oldest xid, state) + * - Buffer cache statistics (hits, misses, evictions) + * - Aggregate counters (total records, bytes generated) + * - Force discard and rotation SQL function + * + * Statistics can be queried via SQL functions pg_stat_get_undo_logs() + * and pg_stat_get_undo_buffers(), registered in pg_proc.dat. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/access/undo/undostats.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/htup_details.h" +#include "access/undolog.h" +#include "access/undostats.h" +#include "access/undoworker.h" +#include "access/undo_xlog.h" +#include "catalog/pg_authid.h" +#include "fmgr.h" +#include "funcapi.h" +#include "miscadmin.h" +#include "storage/lwlock.h" +#include "utils/acl.h" +#include "utils/builtins.h" + +PG_FUNCTION_INFO_V1(pg_stat_get_undo_logs); +PG_FUNCTION_INFO_V1(pg_stat_get_undo_buffers); +PG_FUNCTION_INFO_V1(pg_undo_force_discard); + +/* + * UndoLogStateToString - Convert lifecycle state to display string + */ +static const char * +UndoLogStateToString(UndoLogState state) +{ + switch (state) + { + case UNDO_LOG_FREE: + return "free"; + case UNDO_LOG_ACTIVE: + return "active"; + case UNDO_LOG_SEALED: + return "sealed"; + case UNDO_LOG_DISCARDABLE: + return "discardable"; + } + return "unknown"; +} + +/* + * GetUndoLogStats - Get statistics for all active UNDO logs + * + * Fills the provided array with stats for each active log. + * Returns the number of active logs found. + */ +int +GetUndoLogStats(UndoLogStat *stats, int max_stats) +{ + int count = 0; + int i; + + if (UndoLogShared == NULL) + return 0; + + for (i = 0; i < MAX_UNDO_LOGS && count < max_stats; i++) + { + UndoLogControl *log = &UndoLogShared->logs[i]; + + if (!log->in_use) + continue; + + LWLockAcquire(&log->lock, LW_SHARED); + + stats[count].log_number = log->log_number; + stats[count].insert_ptr = pg_atomic_read_u64(&log->insert_ptr); + stats[count].discard_ptr = log->discard_ptr; + stats[count].oldest_xid = log->oldest_xid; + stats[count].state = log->state; + + /* Calculate size as difference between insert and discard offsets */ + stats[count].size_bytes = + UndoRecPtrGetOffset(stats[count].insert_ptr) - + UndoRecPtrGetOffset(log->discard_ptr); + + LWLockRelease(&log->lock); + + count++; + } + + return count; +} + +/* + * GetUndoBufferStats - Get UNDO buffer statistics + * + * With the shared_buffers integration, UNDO pages are managed by the + * standard buffer pool. Dedicated UNDO buffer statistics are no longer + * tracked separately. This function returns zeros for all counters. + * Use pg_buffercache to inspect UNDO pages in shared_buffers if needed. + */ +void +GetUndoBufferStats(UndoBufferStat *stats) +{ + stats->num_buffers = 0; + stats->cache_hits = 0; + stats->cache_misses = 0; + stats->cache_evictions = 0; + stats->cache_writes = 0; +} + +/* + * pg_stat_get_undo_logs - SQL-callable function returning UNDO log stats + * + * Returns a set of rows, one per active UNDO log, with columns: + * log_number, insert_offset, discard_offset, size_bytes, oldest_xid, state + */ +Datum +pg_stat_get_undo_logs(PG_FUNCTION_ARGS) +{ + FuncCallContext *funcctx; + UndoLogStat *stats; + + if (SRF_IS_FIRSTCALL()) + { + MemoryContext oldcxt; + TupleDesc tupdesc; + int nstats; + + funcctx = SRF_FIRSTCALL_INIT(); + oldcxt = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx); + + /* Build tuple descriptor with 6 columns (added state) */ + tupdesc = CreateTemplateTupleDesc(6); + TupleDescInitEntry(tupdesc, (AttrNumber) 1, "log_number", + INT4OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 2, "insert_offset", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 3, "discard_offset", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 4, "size_bytes", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 5, "oldest_xid", + XIDOID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 6, "state", + TEXTOID, -1, 0); + + funcctx->tuple_desc = BlessTupleDesc(tupdesc); + + /* Collect stats snapshot */ + stats = (UndoLogStat *) palloc(sizeof(UndoLogStat) * MAX_UNDO_LOGS); + nstats = GetUndoLogStats(stats, MAX_UNDO_LOGS); + + funcctx->user_fctx = stats; + funcctx->max_calls = nstats; + + MemoryContextSwitchTo(oldcxt); + } + + funcctx = SRF_PERCALL_SETUP(); + stats = (UndoLogStat *) funcctx->user_fctx; + + if (funcctx->call_cntr < funcctx->max_calls) + { + UndoLogStat *stat = &stats[funcctx->call_cntr]; + Datum values[6]; + bool nulls[6]; + HeapTuple tuple; + + MemSet(nulls, 0, sizeof(nulls)); + + values[0] = Int32GetDatum(stat->log_number); + values[1] = Int64GetDatum(UndoRecPtrGetOffset(stat->insert_ptr)); + values[2] = Int64GetDatum(UndoRecPtrGetOffset(stat->discard_ptr)); + values[3] = Int64GetDatum(stat->size_bytes); + values[4] = TransactionIdGetDatum(stat->oldest_xid); + values[5] = CStringGetTextDatum(UndoLogStateToString(stat->state)); + + tuple = heap_form_tuple(funcctx->tuple_desc, values, nulls); + + SRF_RETURN_NEXT(funcctx, HeapTupleGetDatum(tuple)); + } + + SRF_RETURN_DONE(funcctx); +} + +/* + * pg_stat_get_undo_buffers - SQL-callable function returning buffer stats + * + * Returns a single row with UNDO buffer cache statistics: + * num_buffers, cache_hits, cache_misses, cache_evictions, cache_writes, + * hit_ratio + */ +Datum +pg_stat_get_undo_buffers(PG_FUNCTION_ARGS) +{ + TupleDesc tupdesc; + Datum values[6]; + bool nulls[6]; + HeapTuple tuple; + UndoBufferStat stats; + + /* Build tuple descriptor */ + tupdesc = CreateTemplateTupleDesc(6); + TupleDescInitEntry(tupdesc, (AttrNumber) 1, "num_buffers", + INT4OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 2, "cache_hits", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 3, "cache_misses", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 4, "cache_evictions", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 5, "cache_writes", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 6, "hit_ratio", + FLOAT4OID, -1, 0); + + tupdesc = BlessTupleDesc(tupdesc); + + /* Get statistics */ + GetUndoBufferStats(&stats); + + MemSet(nulls, 0, sizeof(nulls)); + + values[0] = Int32GetDatum(stats.num_buffers); + values[1] = Int64GetDatum(stats.cache_hits); + values[2] = Int64GetDatum(stats.cache_misses); + values[3] = Int64GetDatum(stats.cache_evictions); + values[4] = Int64GetDatum(stats.cache_writes); + + /* Calculate hit ratio */ + { + uint64 total = stats.cache_hits + stats.cache_misses; + + if (total > 0) + values[5] = Float4GetDatum((float4) stats.cache_hits / total); + else + values[5] = Float4GetDatum(0.0); + } + + tuple = heap_form_tuple(tupdesc, values, nulls); + + PG_RETURN_DATUM(HeapTupleGetDatum(tuple)); +} + +/* + * pg_undo_force_discard - Force UNDO log discard and optional rotation + * + * SQL-callable function that performs immediate discard of reclaimable + * UNDO records and optionally rotates the active log segment. + * + * Arguments: + * force_rotate (bool) - If true, seal and rotate the active log first + * + * Returns the number of log segments freed (int4). + * + * Requires the pg_maintain role for access. + */ +Datum +pg_undo_force_discard(PG_FUNCTION_ARGS) +{ + bool force_rotate = PG_GETARG_BOOL(0); + int freed_count = 0; + TransactionId oldest_xid; + int i; + + /* Permission check: require pg_maintain role */ + if (!has_privs_of_role(GetUserId(), ROLE_PG_MAINTAIN)) + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("must be a member of pg_maintain to force UNDO discard"))); + + if (UndoLogShared == NULL) + ereport(ERROR, + (errmsg("UNDO subsystem is not initialized"))); + + /* Optional rotation */ + if (force_rotate) + UndoLogSealAndRotate(UNDO_ROTATE_MANUAL); + + /* Perform inline discard (same as discard worker Phase 1 + Phase 2) */ + oldest_xid = UndoWorkerGetOldestXid(); + if (!TransactionIdIsValid(oldest_xid)) + oldest_xid = ReadNextTransactionId(); + + /* Phase 1: advance discard pointers */ + for (i = 0; i < MAX_UNDO_LOGS; i++) + { + UndoLogControl *log = &UndoLogShared->logs[i]; + + if (!log->in_use) + continue; + + LWLockAcquire(&log->lock, LW_EXCLUSIVE); + + if (TransactionIdIsValid(log->oldest_xid) && + TransactionIdPrecedes(log->oldest_xid, oldest_xid)) + { + UndoRecPtr insert_ptr = pg_atomic_read_u64(&log->insert_ptr); + + if (UndoRecPtrGetOffset(insert_ptr) > + UndoRecPtrGetOffset(log->discard_ptr)) + { + log->discard_ptr = insert_ptr; + log->oldest_xid = oldest_xid; + } + } + + LWLockRelease(&log->lock); + } + + /* Phase 2: lifecycle transitions */ + for (i = 0; i < MAX_UNDO_LOGS; i++) + { + UndoLogControl *log = &UndoLogShared->logs[i]; + + if (!log->in_use) + continue; + + LWLockAcquire(&log->lock, LW_EXCLUSIVE); + + /* SEALED -> DISCARDABLE if fully discarded */ + if (log->state == UNDO_LOG_SEALED) + { + UndoRecPtr seal = pg_atomic_read_u64(&log->seal_ptr); + UndoRecPtr discard = log->discard_ptr; + + if (UndoRecPtrIsValid(seal) && + UndoRecPtrGetOffset(discard) >= UndoRecPtrGetOffset(seal)) + { + log->state = UNDO_LOG_DISCARDABLE; + } + } + + /* DISCARDABLE -> FREE: clean up */ + if (log->state == UNDO_LOG_DISCARDABLE) + { + uint32 log_number = log->log_number; + + log->in_use = false; + log->state = UNDO_LOG_FREE; + log->log_number = 0; + pg_atomic_write_u64(&log->insert_ptr, InvalidUndoRecPtr); + log->discard_ptr = InvalidUndoRecPtr; + log->oldest_xid = InvalidTransactionId; + pg_atomic_write_u64(&log->seal_ptr, InvalidUndoRecPtr); + log->sealed_time = 0; + + LWLockRelease(&log->lock); + + UndoLogDeleteSegmentFile(log_number); + freed_count++; + continue; + } + + LWLockRelease(&log->lock); + } + + /* Wake the background worker for any remaining work */ + WakeUndoDiscardWorker(); + + PG_RETURN_INT32(freed_count); +} diff --git a/src/backend/access/undo/undoworker.c b/src/backend/access/undo/undoworker.c new file mode 100644 index 0000000000000..f214def57dcf2 --- /dev/null +++ b/src/backend/access/undo/undoworker.c @@ -0,0 +1,635 @@ +/*------------------------------------------------------------------------- + * + * undoworker.c + * UNDO worker background process implementation + * + * The UNDO worker periodically discards old UNDO records that are no + * longer needed by any active transaction. This is essential for + * preventing unbounded growth of UNDO logs. + * + * The worker also advances the undo_discard_horizon, allowing WAL + * segments containing fully-discarded UNDO batches to be recycled. + * + * Design based on ZHeap's UNDO worker and PostgreSQL's autovacuum + * launcher patterns. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/access/undo/undoworker.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include +#include + +#include "access/relundo.h" +#include "access/undolog.h" +#include "access/undorecord.h" +#include "access/undormgr.h" +#include "access/undoworker.h" +#include "access/transam.h" +#include "access/xact.h" +#include "access/xlog.h" +#include "libpq/pqsignal.h" +#include "miscadmin.h" +#include "pgstat.h" +#include "postmaster/bgworker.h" +#include "postmaster/interrupt.h" +#include "storage/aio_subsys.h" +#include "storage/bufmgr.h" +#include "storage/fd.h" +#include "storage/ipc.h" +#include "storage/latch.h" +#include "storage/proc.h" +#include "storage/procarray.h" +#include "storage/procsignal.h" +#include "storage/smgr.h" +#include "tcop/tcopprot.h" +#include "utils/guc.h" +#include "utils/hsearch.h" +#include "utils/injection_point.h" +#include "utils/memutils.h" +#include "utils/resowner.h" +#include "utils/timeout.h" +#include "utils/timestamp.h" +#include "utils/wait_event.h" + +/* Shared memory state */ +static UndoWorkerShmemData *UndoWorkerShmem = NULL; + +/* Adaptive sleep: use shorter interval when sealed logs are pending */ +#define UNDO_WORKER_FAST_NAPTIME_MS 200 + +/* Forward declarations */ +static void undo_worker_sighup(SIGNAL_ARGS); +static void undo_worker_sigterm(SIGNAL_ARGS); +static void perform_undo_discard(void); + +/* + * UndoWorkerShmemSize - Calculate shared memory needed + */ +Size +UndoWorkerShmemSize(void) +{ + return sizeof(UndoWorkerShmemData); +} + +/* + * UndoWorkerShmemInit - Initialize shared memory + */ +void +UndoWorkerShmemInit(void) +{ + bool found; + + UndoWorkerShmem = (UndoWorkerShmemData *) + ShmemInitStruct("UNDO Worker Data", + UndoWorkerShmemSize(), + &found); + + if (!found) + { + LWLockInitialize(&UndoWorkerShmem->lock, + LWTRANCHE_UNDO_LOG); + + pg_atomic_init_u64(&UndoWorkerShmem->last_discard_time, 0); + UndoWorkerShmem->oldest_xid_checked = InvalidTransactionId; + UndoWorkerShmem->last_discard_ptr = InvalidUndoRecPtr; + UndoWorkerShmem->naptime_ms = undo_worker_naptime; + UndoWorkerShmem->shutdown_requested = false; + + /* Rotation coordination fields */ + UndoWorkerShmem->worker_proc = INVALID_PROC_NUMBER; + pg_atomic_init_u32(&UndoWorkerShmem->sealed_log_count, 0); + } +} + +/* + * undo_worker_sighup - SIGHUP handler + */ +static void +undo_worker_sighup(SIGNAL_ARGS) +{ + (void) postgres_signal_arg; /* unused */ + ConfigReloadPending = true; + SetLatch(MyLatch); +} + +/* + * undo_worker_sigterm - SIGTERM handler + */ +static void +undo_worker_sigterm(SIGNAL_ARGS) +{ + (void) postgres_signal_arg; /* unused */ + UndoWorkerShmem->shutdown_requested = true; + SetLatch(MyLatch); +} + +/* + * WakeUndoDiscardWorker + * Wake the UNDO discard worker via its latch. + * + * Follows the WAL writer wakeup pattern: read the worker's ProcNumber + * and set its latch to interrupt the WaitLatch sleep. Safe to call + * from any backend, including during allocation pressure. + */ +void +WakeUndoDiscardWorker(void) +{ + ProcNumber proc; + + if (UndoWorkerShmem == NULL) + return; + + proc = UndoWorkerShmem->worker_proc; + if (proc != INVALID_PROC_NUMBER) + SetLatch(&GetPGProcByNumber(proc)->procLatch); +} + +/* + * UndoWorkerGetOldestXid - Get oldest transaction still needing UNDO + * + * Returns the oldest transaction ID that is still active across all + * databases. Any UNDO records created by transactions older than this + * can be safely discarded, because those transactions have already + * committed or aborted and their UNDO is no longer needed. + * + * We use GetOldestActiveTransactionId() from procarray.c which properly + * acquires ProcArrayLock and scans all backends. We pass allDbs=true + * because UNDO logs are not per-database -- a single UNDO log may + * contain records for multiple databases. + * + * Returns InvalidTransactionId if there are no active transactions, + * meaning all UNDO records can potentially be discarded (subject to + * retention policy). + */ +TransactionId +UndoWorkerGetOldestXid(void) +{ + TransactionId oldest_xid; + + /* + * Don't attempt the scan during recovery -- the UNDO worker should not be + * running in that case, but guard defensively. + */ + if (RecoveryInProgress()) + return InvalidTransactionId; + + /* + * GetOldestActiveTransactionId scans ProcArray under ProcArrayLock + * (LW_SHARED) and returns the smallest XID among all active backends. We + * pass inCommitOnly=false (we want all active XIDs, not just those in + * commit critical section) and allDbs=true (UNDO spans all databases). + */ + oldest_xid = GetOldestActiveTransactionId(false, true); + + return oldest_xid; +} + +/* + * perform_undo_discard - Main discard logic + * + * Two-phase approach: + * Phase 1: Update discard pointers for all in-use logs based on + * the oldest active transaction ID. + * Phase 2: Scan SEALED/DISCARDABLE logs and manage lifecycle + * transitions: SEALED -> DISCARDABLE -> FREE. + */ +static void +perform_undo_discard(void) +{ + TransactionId oldest_xid; + UndoRecPtr oldest_undo_ptr; + TimestampTz current_time; + int i; + int freed_count = 0; + + /* Get oldest active transaction */ + oldest_xid = UndoWorkerGetOldestXid(); + + if (!TransactionIdIsValid(oldest_xid)) + { + /* No active transactions, can discard all UNDO */ + oldest_xid = ReadNextTransactionId(); + } + + current_time = GetCurrentTimestamp(); + + /* + * Scan per-backend UNDO batch LSN slots and clear any that belong to dead + * backends. A backend that was SIGKILLed (or otherwise exited without + * calling AtProcExit) will leave its slot occupied, which pins the WAL + * discard horizon indefinitely. We detect dead backends by checking + * ProcGlobal->allProcs[i].pid == 0, which indicates the slot is not in + * use by a live process (pid 0 also indicates prepared-xact dummy + * PGPROCs, but those do not write UNDO data). + */ + for (i = 0; i < MaxBackends; i++) + { + XLogRecPtr slot_lsn; + + slot_lsn = (XLogRecPtr) + pg_atomic_read_u64(&UndoLogShared->backend_undo_lsns[i]); + + if (!XLogRecPtrIsValid(slot_lsn)) + continue; + + if (GetPGProcByNumber(i)->pid == 0) + { + pg_atomic_write_u64(&UndoLogShared->backend_undo_lsns[i], + (uint64) InvalidXLogRecPtr); + ereport(DEBUG2, + (errmsg("UNDO worker: cleared stale batch LSN for dead backend slot %d", i))); + } + } + + /* + * Phase 1: For each UNDO log, determine what can be discarded. We need + * to respect the retention_time setting to allow point-in-time recovery. + */ + for (i = 0; i < MAX_UNDO_LOGS; i++) + { + UndoLogControl *log = &UndoLogShared->logs[i]; + + if (!log->in_use) + continue; + + /* + * Calculate the oldest UNDO pointer that must be retained. This is + * based on: 1. The oldest active transaction 2. The retention time + * setting + */ + LWLockAcquire(&log->lock, LW_SHARED); + + if (TransactionIdIsValid(log->oldest_xid) && + TransactionIdPrecedes(log->oldest_xid, oldest_xid)) + { + /* This log has UNDO that can be discarded */ + oldest_undo_ptr = pg_atomic_read_u64(&log->insert_ptr); + + LWLockRelease(&log->lock); + + /* Update discard pointer */ + UndoLogDiscard(oldest_undo_ptr); + + /* Update cumulative discard counter */ + pg_atomic_fetch_add_u64(&UndoLogShared->total_discarded, + UndoRecPtrGetOffset(oldest_undo_ptr)); + + ereport(DEBUG2, + (errmsg("UNDO worker: discarded log %u up to %llu", + log->log_number, + (unsigned long long) oldest_undo_ptr))); + } + else + { + LWLockRelease(&log->lock); + } + } + + /* + * Phase 2: Manage lifecycle transitions for SEALED and DISCARDABLE logs. + * + * SEALED logs whose discard_ptr >= seal_ptr have had all their records + * discarded and can transition to DISCARDABLE. DISCARDABLE logs can have + * their slot freed and segment file deleted. + */ + for (i = 0; i < MAX_UNDO_LOGS; i++) + { + UndoLogControl *log = &UndoLogShared->logs[i]; + + if (!log->in_use) + continue; + + LWLockAcquire(&log->lock, LW_EXCLUSIVE); + + if (log->state == UNDO_LOG_SEALED) + { + UndoRecPtr seal = pg_atomic_read_u64(&log->seal_ptr); + UndoRecPtr discard = log->discard_ptr; + + if (UndoRecPtrIsValid(seal) && + UndoRecPtrGetOffset(discard) >= UndoRecPtrGetOffset(seal)) + { + /* All records discarded -- transition to DISCARDABLE */ + log->state = UNDO_LOG_DISCARDABLE; + ereport(DEBUG1, + (errmsg("UNDO worker: log %u transitioned to DISCARDABLE", + log->log_number))); + } + } + + if (log->state == UNDO_LOG_DISCARDABLE) + { + uint32 log_number = log->log_number; + + /* Free the slot */ + log->in_use = false; + log->state = UNDO_LOG_FREE; + log->log_number = 0; + pg_atomic_write_u64(&log->insert_ptr, InvalidUndoRecPtr); + log->discard_ptr = InvalidUndoRecPtr; + log->oldest_xid = InvalidTransactionId; + pg_atomic_write_u64(&log->seal_ptr, InvalidUndoRecPtr); + log->sealed_time = 0; + + LWLockRelease(&log->lock); + + /* Delete the segment file outside the lock */ + UndoLogDeleteSegmentFile(log_number); + + /* Decrement sealed log count */ + pg_atomic_fetch_sub_u32(&UndoWorkerShmem->sealed_log_count, 1); + + freed_count++; + continue; + } + + LWLockRelease(&log->lock); + } + + if (freed_count > 0) + ereport(LOG, + (errmsg("UNDO worker: freed %d discardable log segment(s)", + freed_count))); + + /* + * Advance the WAL discard horizon so KeepLogSeg() can allow recycling of + * WAL segments no longer needed for UNDO rollback. + * + * UndoGetOldestBatchLSN() scans per-backend slots and returns the minimum + * first-batch LSN across all active transactions that have written UNDO + * data. WAL before this LSN cannot be recycled. + * + * If no backend has in-flight UNDO data the function returns + * InvalidXLogRecPtr, meaning there is no UNDO-imposed WAL retention + * requirement. We do not call UndoSetDiscardHorizon in that case because + * an invalid horizon is already the "no constraint" sentinel. + */ + { + XLogRecPtr new_horizon = UndoGetOldestBatchLSN(); + + if (XLogRecPtrIsValid(new_horizon)) + UndoSetDiscardHorizon(new_horizon); + + /* + * If undo_max_wal_retention_size is set, warn when the retained WAL + * distance between the current write position and the UNDO discard + * horizon exceeds the configured limit. This helps operators detect + * long-running transactions that prevent WAL recycling. + */ + if (undo_max_wal_retention_size > 0 && XLogRecPtrIsValid(new_horizon)) + { + XLogRecPtr write_ptr = GetXLogWriteRecPtr(); + + if (write_ptr > new_horizon) + { + uint64 retained_mb = (write_ptr - new_horizon) >> 20; + + if (retained_mb > (uint64) undo_max_wal_retention_size) + ereport(WARNING, + (errmsg("UNDO WAL retention (%lu MB) exceeds undo_max_wal_retention_size (%d MB)", + (unsigned long) retained_mb, undo_max_wal_retention_size), + errhint("Investigate long-running transactions or increase undo_max_wal_retention_size."))); + } + } + } + + /* Record this discard operation */ + LWLockAcquire(&UndoWorkerShmem->lock, LW_EXCLUSIVE); + pg_atomic_write_u64(&UndoWorkerShmem->last_discard_time, + (uint64) current_time); + UndoWorkerShmem->oldest_xid_checked = oldest_xid; + LWLockRelease(&UndoWorkerShmem->lock); +} + +/* + * UndoWorkerMain - Main loop for UNDO worker + * + * This is the entry point for the UNDO worker background process. + * It runs continuously, waking periodically to discard old UNDO. + * + * Uses adaptive sleep: when sealed logs are pending cleanup, the worker + * wakes more frequently (200ms) to process them promptly. Otherwise + * it uses the configured undo_worker_naptime. + */ +void +UndoWorkerMain(Datum main_arg) +{ + sigjmp_buf local_sigjmp_buf; + MemoryContext undo_worker_context; + + (void) main_arg; /* unused */ + + /* Establish signal handlers */ + pqsignal(SIGHUP, undo_worker_sighup); + pqsignal(SIGTERM, undo_worker_sigterm); + + /* We're now ready to receive signals */ + BackgroundWorkerUnblockSignals(); + + /* + * Connect with no specific database so the worker is a stats-reporting + * backend (visible in pg_stat_activity as 'undo worker') while not + * pinning any database against DROP. Discard operates on cluster-wide + * UNDO logs, so it needs no per-database catalog access. + */ + BackgroundWorkerInitializeConnection(NULL, NULL, 0); + + /* Register our ProcNumber for latch-based wakeup by other backends */ + UndoWorkerShmem->worker_proc = MyProcNumber; + + /* Initialize worker state */ + ereport(LOG, + (errmsg("UNDO worker started"))); + + /* + * Create a memory context for the worker. This will be reset after each + * iteration and during error recovery. + */ + undo_worker_context = AllocSetContextCreate(TopMemoryContext, + "UNDO Worker", + ALLOCSET_DEFAULT_SIZES); + MemoryContextSwitchTo(undo_worker_context); + + /* + * If an exception is encountered, processing resumes here. + * + * Unlike the autovacuum worker, this is a long-lived background process + * that must survive transient errors (e.g. an ERROR raised while + * discarding UNDO or cleaning up retained sLog before-images). Letting + * the error propagate would terminate the worker; the postmaster would + * restart it, but any in-flight discard progress would be abandoned and + * discard could stall. Instead we recover in place, mirroring the + * autovacuum launcher. + * + * We use sigsetjmp(..., 1) so the prevailing signal mask is restored on + * longjmp; signals other than SIGQUIT stay blocked until we exit. The + * HOLD_INTERRUPTS() call is still required because InterruptPending might + * already be set. + */ + if (sigsetjmp(local_sigjmp_buf, 1) != 0) + { + /* since not using PG_TRY, must reset error stack by hand */ + error_context_stack = NULL; + + /* Prevents interrupts while cleaning up */ + HOLD_INTERRUPTS(); + + /* Report the error to the server log */ + EmitErrorReport(); + + /* + * Abort the current transaction in order to recover, but only if one + * is actually in progress. perform_undo_discard() and the sLog + * cleanup operate on shared memory and do not normally open a + * transaction; guarding avoids a spurious "AbortCurrentTransaction + * when not in transaction" path. + */ + if (IsTransactionState()) + AbortCurrentTransaction(); + + /* Release any other resources we might still be holding. */ + LWLockReleaseAll(); + pgstat_report_wait_end(); + pgaio_error_cleanup(); + UnlockBuffers(); + if (AuxProcessResourceOwner) + ReleaseAuxProcessResources(false); + AtEOXact_Buffers(false); + AtEOXact_SMgr(); + AtEOXact_Files(false); + AtEOXact_HashTables(false); + + /* Return to the worker context and clear ErrorContext. */ + MemoryContextSwitchTo(undo_worker_context); + FlushErrorState(); + + /* Flush any leaked data in the worker context. */ + MemoryContextReset(undo_worker_context); + + /* Now we can allow interrupts again */ + RESUME_INTERRUPTS(); + + /* + * Sleep at least 1 second after any error. We don't want to be + * filling the error logs as fast as we can. + */ + pg_usleep(1000000L); + } + + /* We can now handle ereport(ERROR) */ + PG_exception_stack = &local_sigjmp_buf; + + /* + * Main loop: wake up periodically and discard old UNDO + */ + while (!UndoWorkerShmem->shutdown_requested) + { + int rc; + long naptime; + uint32 sealed_count; + + /* Process any pending configuration changes */ + if (ConfigReloadPending) + { + ConfigReloadPending = false; + ProcessConfigFile(PGC_SIGHUP); + + /* Update naptime from GUC */ + UndoWorkerShmem->naptime_ms = undo_worker_naptime; + } + + CHECK_FOR_INTERRUPTS(); + + INJECTION_POINT("undo-worker-before-discard", NULL); + + /* Perform UNDO discard */ + perform_undo_discard(); + + /* + * Clean up retained before-image entries that are no longer needed by + * any active snapshot. A registered AM hook performs the + * reclamation; the reclamation horizon is the xid horizon. + */ + if (RelUndoDiscardRetained_hook) + RelUndoDiscardRetained_hook(); + + INJECTION_POINT("undo-worker-after-discard", NULL); + + /* + * Adaptive sleep: use a shorter interval when sealed logs are pending + * cleanup, similar to the WAL writer's adaptive sleep. + */ + sealed_count = pg_atomic_read_u32(&UndoWorkerShmem->sealed_log_count); + if (sealed_count > 0) + naptime = UNDO_WORKER_FAST_NAPTIME_MS; + else + naptime = UndoWorkerShmem->naptime_ms; + + /* Sleep until next iteration, latch set, or signal */ + rc = WaitLatch(MyLatch, + WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, + naptime, + WAIT_EVENT_UNDO_WORKER_MAIN); + + ResetLatch(MyLatch); + + /* Emergency bailout if postmaster died */ + if (rc & WL_POSTMASTER_DEATH) + proc_exit(1); + } + + /* Clear our ProcNumber before exiting */ + UndoWorkerShmem->worker_proc = INVALID_PROC_NUMBER; + + /* Normal shutdown */ + ereport(LOG, + (errmsg("UNDO worker shutting down"))); + + proc_exit(0); +} + +/* + * UndoWorkerRegister - Register the UNDO worker at server start + * + * This is called from postmaster during server initialization. + */ +void +UndoWorkerRegister(void) +{ + BackgroundWorker worker; + + memset(&worker, 0, sizeof(BackgroundWorker)); + + worker.bgw_flags = BGWORKER_SHMEM_ACCESS | + BGWORKER_BACKEND_DATABASE_CONNECTION; + worker.bgw_start_time = BgWorkerStart_RecoveryFinished; + worker.bgw_restart_time = 10; /* Restart after 10 seconds if crashed */ + + sprintf(worker.bgw_library_name, "postgres"); + sprintf(worker.bgw_function_name, "UndoWorkerMain"); + snprintf(worker.bgw_name, BGW_MAXLEN, "undo worker"); + snprintf(worker.bgw_type, BGW_MAXLEN, "undo worker"); + + RegisterBackgroundWorker(&worker); +} + +/* + * UndoWorkerRequestShutdown - Request worker to shut down + */ +void +UndoWorkerRequestShutdown(void) +{ + if (UndoWorkerShmem != NULL) + { + LWLockAcquire(&UndoWorkerShmem->lock, LW_EXCLUSIVE); + UndoWorkerShmem->shutdown_requested = true; + LWLockRelease(&UndoWorkerShmem->lock); + } +} diff --git a/src/backend/access/undo/xactundo.c b/src/backend/access/undo/xactundo.c new file mode 100644 index 0000000000000..23f40f61aa341 --- /dev/null +++ b/src/backend/access/undo/xactundo.c @@ -0,0 +1,1458 @@ +/*------------------------------------------------------------------------- + * + * xactundo.c + * Management of undo record sets for transactions + * + * Undo records that need to be applied after a transaction or + * subtransaction abort should be inserted using the functions defined + * in this file; thus, every table or index access method that wants to + * use undo for post-abort cleanup should invoke these interfaces. + * + * The reason for this design is that we want to pack all of the undo + * records for a single transaction into one place, regardless of the + * AM which generated them. That way, we can apply the undo actions + * which pertain to that transaction in the correct order; namely, + * backwards as compared with the order in which the records were + * generated. + * + * We may use up to three undo record sets per transaction, one per + * persistence level (permanent, unlogged, temporary). We assume that + * it's OK to apply the undo records for each persistence level + * independently of the others. This is safe since the modifications + * must necessarily touch disjoint sets of pages. + * + * CROSS-RELATION ORDERING INVARIANT (important for TOAST correctness): + * + * All UNDO records for all relations touched by a single transaction are + * packed into the same UndoRecordSet, in strict WAL-LSN emission order. + * The newest-first application order during rollback guarantees correct + * restoration ordering for multi-object operations (e.g. a cluster-wide + * consumer that touches several relations in one transaction). + * + * PARALLEL RECOVERY: XLOG_UNDO_BATCH records are handled by the startup + * process via ApplyUndoChainFromWAL(); they are not dispatched to parallel + * workers because UNDO application requires coordinated per-transaction + * state. + * + * SUBTRANSACTION TRACKING: + * + * Subtransaction state is tracked using a dynamically-grown array allocated + * in TopMemoryContext. The array starts at INITIAL_SUBXACT_CAPACITY (64) + * slots and doubles when needed via repalloc(). The array persists across + * transactions within the same backend to avoid repeated allocation for + * steady-state workloads. + * + * Growth via repalloc() in TopMemoryContext during SubXactCallbacks is safe + * because it does not interact with pgstat's per-subtransaction tracking + * (the original corruption bug was caused by palloc of new per-subtransaction + * nodes in CurTransactionContext, not by growing an existing TopMemoryContext + * allocation). + * + * This design follows the EDB undo-record-set branch architecture + * (xactundo.c) adapted for the physical undo approach used here. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/backend/access/undo/xactundo.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/atm.h" +#include "access/relundo.h" +#include "access/relundo_worker.h" +#include "access/undo.h" +#include "access/undo_xlog.h" +#include "access/xlog.h" +#include "access/undolog.h" +#include "access/undorecord.h" +#include "access/xact.h" +#include "access/xactundo.h" +#include "access/xlogdefs.h" +#include "access/table.h" +#include "catalog/pg_class.h" +#include "miscadmin.h" +#include "storage/bufmgr.h" +#include "storage/ipc.h" +#include "utils/injection_point.h" +#include "storage/lmgr.h" +#include "utils/memutils.h" +#include "utils/rel.h" + +/* GUC: UNDO bytes threshold for instant abort via ATM */ +int undo_instant_abort_threshold = 65536; + +/* + * Initial capacity for the dynamically-grown subtransaction stack. + * Covers 99.9% of workloads without needing reallocation. The stack + * doubles when needed, so there is no artificial upper limit. + */ +#define INITIAL_SUBXACT_CAPACITY 64 + +/* Per-subtransaction backend-private undo state (array element). */ +typedef struct XactUndoSubTransactionState +{ + SubTransactionId nestingLevel; + UndoRecPtr start_location[NUndoPersistenceLevels]; + + /* + * Snapshot of the parent's last_batch_lsn at the time this subtransaction + * started. On subtransaction abort, after applying this subtransaction's + * UNDO chain, we restore XactUndo.last_batch_lsn to these saved values so + * the parent's subsequent abort (or further subtransactions) only covers + * the parent's own records and does not double-apply already-reversed + * batches. + */ + XLogRecPtr last_batch_lsn[NUndoPersistenceLevels]; +} XactUndoSubTransactionState; + +/* + * Per-relation UNDO registration. Any AM using the per-relation UNDO fork + * registers the head of its UNDO chain for each relation it modified. On + * abort, ApplyPerRelUndo() walks this list and either queues synchronous + * worker rollback or records an ATM instant-abort entry for asynchronous + * rollback. + */ +typedef struct PerRelUndoEntry +{ + Oid relid; + RelUndoRecPtr start_urec_ptr; + struct PerRelUndoEntry *next; +} PerRelUndoEntry; + +/* Backend-private undo state. */ +typedef struct XactUndoData +{ + bool has_undo; /* has this xact generated any undo? */ + int subxact_depth; /* 0 = top-level, 1+ = savepoints */ + int subxact_capacity; /* allocated slots in subxact_stack */ + + /* Dynamically-grown subtransaction stack (TopMemoryContext). */ + XactUndoSubTransactionState *subxact_stack; + + /* + * Per-persistence-level record sets. These are created lazily on first + * use and destroyed at transaction end. + */ + UndoRecordSet *record_set[NUndoPersistenceLevels]; + + /* Tracking for the most recent undo insertion per persistence level. */ + UndoRecPtr last_location[NUndoPersistenceLevels]; + + /* + * WAL-based UNDO chain heads. When UNDO records are routed through WAL + * via XLOG_UNDO_BATCH, this tracks the LSN of the most recent batch per + * persistence level. Used for rollback chain walking. + */ + XLogRecPtr last_batch_lsn[NUndoPersistenceLevels]; + + /* + * Per-relation UNDO chain heads, one entry per relation modified via the + * per-relation UNDO fork. Allocated in CurTransactionContext, so the + * list pointer is reset to NULL at every transaction end. + */ + PerRelUndoEntry *relundo_list; +} XactUndoData; + +static XactUndoData XactUndo; +static bool subxact_callback_registered = false; + +/* + * Compile-time guard: xl_xact_prepare.last_batch_lsn[3] must match + * NUndoPersistenceLevels. Both headers are available here; xact.h avoids + * including undodefs.h to keep its include footprint minimal. + */ +StaticAssertDecl(NUndoPersistenceLevels == 3, + "xl_xact_prepare.last_batch_lsn array size (3) must match NUndoPersistenceLevels"); + +static void ResetXactUndo(void); +static void ApplyPerRelUndo(void); +static void CollapseXactUndoSubTransactions(void); +static UndoPersistenceLevel GetUndoPersistenceLevel(char relpersistence); +static void EnsureSubxactStackCapacity(void); +static void XactUndo_SubXactCallback(SubXactEvent event, SubTransactionId mySubid, + SubTransactionId parentSubid, void *arg); + +/* Convenience macro: pointer to current subtransaction state. */ +#define CURRENT_SUBXACT() (&XactUndo.subxact_stack[XactUndo.subxact_depth]) + +/* + * XactUndoShmemSize + * How much shared memory do we need for transaction undo state? + * + * Currently no shared memory is needed -- all state is backend-private. + * This function exists for forward compatibility with the architecture + * where an UndoRequestManager will be added later. + */ +Size +XactUndoShmemSize(void) +{ + return 0; +} + +/* + * XactUndoShmemInit + * Initialize shared memory for transaction undo state. + * + * Currently a no-op; provided for the unified UndoShmemInit() pattern. + */ +void +XactUndoShmemInit(void) +{ + /* Nothing to do yet. */ +} + +/* + * InitializeXactUndo + * Per-backend initialization for transaction undo. + */ +void +InitializeXactUndo(void) +{ + /* Ensure the dynamic subxact stack is allocated */ + EnsureSubxactStackCapacity(); + + ResetXactUndo(); + + /* + * Register callback to track subtransaction lifecycle. Do this lazily on + * first transaction to ensure it's registered for the backend that will + * actually use UNDO. + */ + if (!subxact_callback_registered) + { + RegisterSubXactCallback(XactUndo_SubXactCallback, NULL); + subxact_callback_registered = true; + } +} + +/* + * GetUndoPersistenceLevel + * Map relation persistence character to UndoPersistenceLevel. + */ +static UndoPersistenceLevel +GetUndoPersistenceLevel(char relpersistence) +{ + switch (relpersistence) + { + case RELPERSISTENCE_PERMANENT: + return UNDOPERSISTENCE_PERMANENT; + case RELPERSISTENCE_UNLOGGED: + return UNDOPERSISTENCE_UNLOGGED; + case RELPERSISTENCE_TEMP: + return UNDOPERSISTENCE_TEMP; + default: + elog(ERROR, "unrecognized relpersistence: %c", relpersistence); + return UNDOPERSISTENCE_PERMANENT; /* keep compiler quiet */ + } +} + +/* + * PrepareXactUndoData + * Prepare to insert a transactional undo record. + * + * Finds or creates the appropriate per-persistence-level UndoRecordSet + * for the current transaction and adds the record to it. + * + * The API is AM-agnostic: callers pass an RM ID, RM-specific info, + * a relation OID, and an opaque payload. + * + * Returns the UndoRecPtr where the record will be inserted (or + * InvalidUndoRecPtr if undo is disabled). + */ +UndoRecPtr +PrepareXactUndoData(XactUndoContext *ctx, char persistence, + uint8 rmid, uint16 info, Oid reloid, + const char *payload, Size payload_len) +{ + int nestingLevel = GetCurrentTransactionNestLevel(); + UndoPersistenceLevel plevel = GetUndoPersistenceLevel(persistence); + TransactionId xid = GetCurrentTransactionId(); + UndoRecordSet *uset; + XactUndoSubTransactionState *cur; + UndoRecPtr *sub_start_location; + + /* Remember that we've done something undo-related. */ + XactUndo.has_undo = true; + + /* + * Load the UndoRecordSetInsert() injection points into this backend's + * local cache now, while we are guaranteed not to be inside a critical + * section (PrepareXactUndoData may itself palloc). UndoRecordSetInsert() + * runs inside the caller's crit section and can only fire these via the + * palloc-free INJECTION_POINT_CACHED variant. + */ + INJECTION_POINT_LOAD("undo-batch-before-wal-insert"); + INJECTION_POINT_LOAD("undo-batch-after-wal-insert"); + + /* + * If we've entered a subtransaction deeper than what's currently tracked, + * push a new entry onto the subxact_stack. This handles the case where + * PrepareXactUndoData is called for the first time in a subtransaction + * that was started before the SubXactCallback fired (e.g., if the + * callback hadn't been registered yet when the subtransaction began). + */ + cur = CURRENT_SUBXACT(); + if (nestingLevel > (int) cur->nestingLevel) + { + int i; + + XactUndo.subxact_depth++; + EnsureSubxactStackCapacity(); + + cur = CURRENT_SUBXACT(); + cur->nestingLevel = nestingLevel; + for (i = 0; i < NUndoPersistenceLevels; ++i) + { + cur->start_location[i] = InvalidUndoRecPtr; + cur->last_batch_lsn[i] = XactUndo.last_batch_lsn[i]; + } + } + + /* + * Make sure we have an UndoRecordSet of the appropriate type open for + * this persistence level. These record sets are always associated with + * the toplevel transaction, not a subtransaction, to avoid fragmentation. + */ + uset = XactUndo.record_set[plevel]; + if (uset == NULL) + { + uset = UndoRecordSetCreate(xid, GetCurrentTransactionUndoRecPtr()); + XactUndo.record_set[plevel] = uset; + } + + /* Remember persistence level for InsertXactUndoData. */ + ctx->plevel = plevel; + ctx->uset = uset; + + /* Add the record to the record set using generic payload API. */ + UndoRecordAddPayload(uset, rmid, info, reloid, payload, payload_len); + + /* + * If this is the first undo for this persistence level in this + * subtransaction, record the start location. The actual UndoRecPtr is not + * known until insertion, so we use a sentinel for now and the caller will + * update it after InsertXactUndoData. + */ + sub_start_location = &cur->start_location[plevel]; + if (!UndoRecPtrIsValid(*sub_start_location)) + *sub_start_location = (UndoRecPtr) 1; /* will be set properly */ + + return InvalidUndoRecPtr; /* actual ptr assigned during insert */ +} + +/* + * PrepareXactUndoDataParts + * Like PrepareXactUndoData, but with scatter-gather payload. + * + * Used when the payload is in two non-contiguous pieces (e.g., a fixed + * header struct followed by variable-length tuple data). Avoids the + * need to assemble an intermediate contiguous buffer. + */ +UndoRecPtr +PrepareXactUndoDataParts(XactUndoContext *ctx, char persistence, + uint8 rmid, uint16 info, Oid reloid, + const char *part1, Size part1_len, + const char *part2, Size part2_len) +{ + int nestingLevel = GetCurrentTransactionNestLevel(); + UndoPersistenceLevel plevel = GetUndoPersistenceLevel(persistence); + TransactionId xid = GetCurrentTransactionId(); + UndoRecordSet *uset; + XactUndoSubTransactionState *cur; + UndoRecPtr *sub_start_location; + + /* Remember that we've done something undo-related. */ + XactUndo.has_undo = true; + + /* + * If we've entered a subtransaction deeper than what's currently tracked, + * push a new entry onto the subxact_stack. + */ + cur = CURRENT_SUBXACT(); + if (nestingLevel > (int) cur->nestingLevel) + { + int i; + + XactUndo.subxact_depth++; + EnsureSubxactStackCapacity(); + + cur = CURRENT_SUBXACT(); + cur->nestingLevel = nestingLevel; + for (i = 0; i < NUndoPersistenceLevels; ++i) + { + cur->start_location[i] = InvalidUndoRecPtr; + cur->last_batch_lsn[i] = XactUndo.last_batch_lsn[i]; + } + } + + /* + * Make sure we have an UndoRecordSet of the appropriate type open for + * this persistence level. + */ + uset = XactUndo.record_set[plevel]; + if (uset == NULL) + { + uset = UndoRecordSetCreate(xid, GetCurrentTransactionUndoRecPtr()); + XactUndo.record_set[plevel] = uset; + } + + /* Remember persistence level for InsertXactUndoData. */ + ctx->plevel = plevel; + ctx->uset = uset; + + /* Add the record using scatter-gather payload API. */ + UndoRecordAddPayloadParts(uset, rmid, info, reloid, + part1, part1_len, part2, part2_len); + + /* + * If this is the first undo for this persistence level in this + * subtransaction, record the start location. + */ + sub_start_location = &cur->start_location[plevel]; + if (!UndoRecPtrIsValid(*sub_start_location)) + *sub_start_location = (UndoRecPtr) 1; /* will be set properly */ + + return InvalidUndoRecPtr; /* actual ptr assigned during insert */ +} + +/* + * InsertXactUndoData + * Insert the prepared undo data into the undo log. + * + * This performs the actual write of the accumulated records. + * Also updates the transaction-level undo record pointer (undoRecPtr + * in TransactionState) so that subsequent UNDO records chain correctly. + */ +void +InsertXactUndoData(XactUndoContext *ctx) +{ + UndoRecordSet *uset = ctx->uset; + UndoRecPtr ptr; + + Assert(uset != NULL); + + ptr = UndoRecordSetInsert(uset); + if (UndoRecPtrIsValid(ptr)) + { + XactUndoSubTransactionState *cur = CURRENT_SUBXACT(); + + XactUndo.last_location[ctx->plevel] = ptr; + + /* + * Track the WAL LSN of the most recent UNDO batch for this + * persistence level. This is used during rollback to walk the UNDO + * chain backward through WAL. + * + * The pointer must be a valid LSN; the caller must have produced an + * RM_UNDO_ID XLOG_UNDO_BATCH record via UndoRecordSetInsert. We do + * not validate the rmid here because UndoValidateBatchLSN reads from + * the on-disk WAL via read_local_xlog_page, which waits on flush and + * cannot be safely called from the WAL-insertion hot path. The + * rollback path at AtAbort_XactUndo validates before applying. + */ + Assert(XLogRecPtrIsValid(uset->last_batch_lsn)); + XactUndo.last_batch_lsn[ctx->plevel] = uset->last_batch_lsn; + + /* Fix up subtransaction start location if needed */ + if (cur->start_location[ctx->plevel] == (UndoRecPtr) 1) + cur->start_location[ctx->plevel] = ptr; + + /* + * Update the per-transaction undo pointer in TransactionState so that + * the next UndoRecordSetCreate (if called directly by heap AM or + * other subsystems) picks up the correct chain pointer. + */ + SetCurrentTransactionUndoRecPtr(ptr); + } + else + { + XactUndoSubTransactionState *cur = CURRENT_SUBXACT(); + + /* + * The batch was empty or elided, so no record pointer exists to fix + * up the sentinel with. Leaving the (UndoRecPtr) 1 sentinel in place + * would make UndoRecPtrIsValid() treat it as a real start location + * and leak it into subxact->parent merges and rollback chain walks; + * reset it so this level records no start location. + */ + if (cur->start_location[ctx->plevel] == (UndoRecPtr) 1) + cur->start_location[ctx->plevel] = InvalidUndoRecPtr; + } +} + +/* + * CleanupXactUndoInsertion + * Clean up after an undo insertion cycle. + * + * Resets the record set's buffer position and record count so it can + * accumulate more records. Does NOT free the record set -- that + * happens at transaction end (AtCommit_XactUndo / AtAbort_XactUndo). + * + * The record set's prev_undo_ptr is preserved across resets (it was + * updated by UndoRecordSetInsert), so subsequent records chain + * correctly through the undo log. + */ +void +CleanupXactUndoInsertion(XactUndoContext *ctx) +{ + if (ctx->uset != NULL) + UndoRecordSetReset(ctx->uset); +} + +/* + * GetCurrentXactUndoRecPtr + * Get the most recent undo record pointer for a persistence level. + */ +UndoRecPtr +GetCurrentXactUndoRecPtr(UndoPersistenceLevel plevel) +{ + return XactUndo.last_location[plevel]; +} + +/* + * GetCurrentXactLastBatchLSN + * Get the WAL LSN of the most recent UNDO batch for a persistence level. + * + * Used during transaction abort to start the WAL-based UNDO chain walk. + */ +XLogRecPtr +GetCurrentXactLastBatchLSN(UndoPersistenceLevel plevel) +{ + return XactUndo.last_batch_lsn[plevel]; +} + +/* + * XActUndoUpdateLastBatchLSN + * Record the LSN of an UNDO batch for the current transaction. + * + * Called from the heap DML code after writing an UNDO batch -- either + * embedded inside a heap WAL record (HAS_UNDO path) or as a standalone + * XLOG_UNDO_BATCH overflow record. Updates last_batch_lsn so that + * AtAbort_XactUndo() can find the head of the UNDO chain, and registers + * the batch LSN for WAL retention tracking on first call per transaction. + */ +void +XActUndoUpdateLastBatchLSN(XLogRecPtr lsn, UndoPersistenceLevel plevel) +{ + if (!XLogRecPtrIsValid(lsn) || plevel >= NUndoPersistenceLevels) + return; + + /* + * The pointer must be a valid LSN. Rmid validation is deferred to the + * rollback path; see comment in InsertXactUndoData for why we cannot call + * UndoValidateBatchLSN here. + */ + Assert(XLogRecPtrIsValid(lsn)); + + XactUndo.has_undo = true; + XactUndo.last_batch_lsn[plevel] = lsn; +} + +/* + * RegisterPerRelUndo + * Register (or refresh) the head of a relation's per-relation UNDO chain. + * + * Called by table AMs that use the per-relation UNDO fork after writing an + * UNDO record. The most recent pointer per relation is retained; rollback + * walks the chain backwards from it. Entries live in CurTransactionContext + * and are dropped automatically at transaction end. + */ +void +RegisterPerRelUndo(Oid relid, RelUndoRecPtr start_urec_ptr) +{ + PerRelUndoEntry *entry; + + /* Mark that we have UNDO so commit/abort cleanup runs. */ + XactUndo.has_undo = true; + + /* If already registered, advance the pointer to the latest record. */ + for (entry = XactUndo.relundo_list; entry != NULL; entry = entry->next) + { + if (entry->relid == relid) + { + entry->start_urec_ptr = start_urec_ptr; + return; + } + } + + entry = (PerRelUndoEntry *) MemoryContextAlloc(TopTransactionContext, + sizeof(PerRelUndoEntry)); + entry->relid = relid; + entry->start_urec_ptr = start_urec_ptr; + entry->next = XactUndo.relundo_list; + XactUndo.relundo_list = entry; +} + +/* + * GetPerRelUndoPtr + * Return the latest UNDO record pointer registered for a relation, or + * InvalidRelUndoRecPtr if none. Used to chain a new record's + * urec_prevundorec to the previous one. + */ +RelUndoRecPtr +GetPerRelUndoPtr(Oid relid) +{ + PerRelUndoEntry *entry; + + for (entry = XactUndo.relundo_list; entry != NULL; entry = entry->next) + { + if (entry->relid == relid) + return entry->start_urec_ptr; + } + + return InvalidRelUndoRecPtr; +} + +/* + * IteratePerRelUndo + * Invoke callback(relid, start_urec_ptr, arg) for each registered + * per-relation UNDO chain head. + * + * Used by an AM's PREPARE-time hook to serialize the per-relation UNDO chain + * heads into the 2PC state file so ROLLBACK PREPARED can restore + * before-images. + */ +void +IteratePerRelUndo(PerRelUndoIterCB callback, void *arg) +{ + PerRelUndoEntry *entry; + + for (entry = XactUndo.relundo_list; entry != NULL; entry = entry->next) + callback(entry->relid, entry->start_urec_ptr, arg); +} + +/* + * XactUndoHasUnrecoverableUndo + * Does the current transaction hold UNDO that has no working ROLLBACK + * PREPARED apply path yet? + * + * Both UNDO mechanisms are now recoverable across 2PC: + * + * - Cluster-wide UNDO (nbtree/hash and other cluster-wide consumers; + * last_batch_lsn): the permanent chain-head LSN is durably saved in + * xl_xact_prepare, its WAL is pinned while the xact stays prepared + * (undo_batch_lsn in twophase.c / UndoGetOldestBatchLSN), and + * FinishPreparedTransaction() feeds it to ATMAddAborted() on ROLLBACK + * PREPARED. + * + * - Per-relation UNDO (relundo_list): the owning AM's PREPARE-time hook + * serializes its per-relation UNDO chain heads via + * RegisterTwoPhaseRecord(). Its two-phase postabort handler replays the + * chain via RelUndoApplyChain() to restore in-place before-images; its + * postcommit handler discards it. + * + * Nothing remains that PREPARE must reject on UNDO grounds, so this returns + * false unconditionally. Kept as a single choke point (rather than deleting + * the call in PrepareTransaction) so any future UNDO mechanism that is not + * 2PC-safe has one obvious place to re-assert a guard. + */ +bool +XactUndoHasUnrecoverableUndo(void) +{ + return false; +} + +/* + * ApplyPerRelUndo + * Roll back all registered per-relation UNDO chains on abort. + * + * Each modified relation is queued for the background per-relation UNDO + * worker, which opens the relation and walks its UNDO chain backwards, + * restoring before-images in place. The aborting backend blocks on + * WaitForPendingRelUndo() (called from AbortTransaction after lock release) + * so rollback is synchronous from the client's point of view. + * + * Per-relation UNDO cannot be applied inline here: the backend is in + * TRANS_ABORT, where catalog access (relation_open) is unsafe. + */ +static void +ApplyPerRelUndo(void) +{ + PerRelUndoEntry *entry; + TransactionId xid = GetCurrentTransactionIdIfAny(); + bool all_applied = true; + + if (XactUndo.relundo_list == NULL) + return; + + /* + * Apply each relation's UNDO chain INLINE, in this backend, BEFORE the + * caller (AbortTransaction) reaches ProcArrayEndTransaction and releases + * the transaction lock that conflicting writers wait on. + * + * This ordering is mandatory for in-place MVCC: a second writer blocked + * in XactLockTableWait wakes the instant our XID leaves the proc array. + * If the before-image were restored asynchronously by the background + * worker (which runs only after lock release), the waiter would read our + * not-yet-reverted in-place value, write on top of it, commit, and then + * the worker would clobber the waiter's committed value with our stale + * before-image -- a lost update. Restoring synchronously here closes + * that window: the page already holds the pre-abort image when the waiter + * wakes. + * + * We are in TRANS_ABORT but all backing resources (relcache, locks, + * resource owner) are still live, so present TRANS_INPROGRESS for the + * duration of the apply (table_open asserts IsTransactionState()). The + * RowExclusiveLock we already hold from our own DML makes the re-open a + * no-op at the lock manager. + */ + for (entry = XactUndo.relundo_list; entry != NULL; entry = entry->next) + { + int saved_trans_state; + bool applied = false; + + saved_trans_state = EnterInlineUndoApplyState(); + PG_TRY(); + { + Relation rel = table_open(entry->relid, RowExclusiveLock); + + RelUndoApplyChain(rel, entry->start_urec_ptr); + table_close(rel, RowExclusiveLock); + applied = true; + } + PG_CATCH(); + { + /* + * A chain that errored mid-apply may still hold EXCLUSIVE content + * locks on the pages it had pinned into touched[] + * (RelUndoApplyChain releases them only on its normal CLR path). + * Release them here so a caught error cannot leak a held lock + * into the next iteration -- a subsequent entry that re-locks the + * same buffer would trip the BufferLockAcquire lockmode==UNLOCK + * assert. + */ + BufferLockReleaseAll(); + EmitErrorReport(); + FlushErrorState(); + applied = false; + } + PG_END_TRY(); + LeaveInlineUndoApplyState(saved_trans_state); + + if (!applied) + all_applied = false; + } + + if (all_applied) + { + /* + * Every page is physically restored. Remove the ABORTED sLog entries + * (kept until now so visibility checks treated the tuples as live). + */ + if (TransactionIdIsValid(xid) && RelUndoAbortCleanup_hook) + RelUndoAbortCleanup_hook(xid); + } + else + { + /* + * At least one relation could not be restored inline (dropped, + * error). Fall back to the background worker for those; queue every + * entry and let the worker's idempotent already-applied check skip + * the ones we already reverted. + */ + for (entry = XactUndo.relundo_list; entry != NULL; entry = entry->next) + RelUndoQueueAdd(MyDatabaseId, entry->relid, + entry->start_urec_ptr, xid); + + StartRelUndoWorker(MyDatabaseId); + } +} + +/* + * AtCommit_XactUndo + * Post-commit cleanup of the undo state. + * + * On commit, undo records are no longer needed for rollback. + * Free all record sets and reset state. + * + * UNDO pages are managed by shared_buffers and flushed by the + * checkpointer -- no per-commit fdatasync is needed. We only + * flush the deferred WAL allocation records so recovery can + * reconstruct the UNDO log insert pointer. + * + * NB: This code MUST NOT FAIL, since it is run as a post-commit step. + */ +void +AtCommit_XactUndo(void) +{ + int i; + + if (!XactUndo.has_undo) + { + /* Flush any deferred WAL even if has_undo is false */ + UndoWalBatchFlush(); + return; + } + + /* + * With UNDO-in-WAL, all UNDO data was already written to WAL via + * XLOG_UNDO_BATCH records during the transaction. The single XLogFlush() + * at commit time (in RecordTransactionCommit) ensures both the UNDO data + * and the commit record are durable. No separate fdatasync is needed. + * + * Legacy WAL batch flush is now a no-op but kept for safety. + */ + UndoWalBatchFlush(); + + /* + * Free all per-persistence-level record sets. + * + * We can safely call UndoRecordSetFree() during commit because we're in + * CurTransactionContext, not BumpContext (which is only used during + * abort). The record sets are allocated in CurTransactionContext and will + * be freed when that context is destroyed at transaction end. + */ + for (i = 0; i < NUndoPersistenceLevels; i++) + { + if (XactUndo.record_set[i] != NULL) + { + UndoRecordSetFree(XactUndo.record_set[i]); + XactUndo.record_set[i] = NULL; + } + } + + /* Release WAL retention hold acquired in UndoRecordSetInsert(). */ + UndoClearBatchLSN(); + + ResetXactUndo(); +} + +/* + * AtAbort_XactUndo + * Post-abort cleanup of the undo state. + * + * On abort, we need to apply the undo chain to roll back changes. + * The actual undo application is triggered by xact.c before calling + * this function. Here we apply per-relation UNDO and clean up the record sets. + * + * With append-only I/O, we sync UNDO files before UNDO replay so that + * if we crash during rollback, recovery can re-read the UNDO records + * from the segment file and continue the rollback. + */ +void +AtAbort_XactUndo(void) +{ + int i; + bool lsn_safely_held = false; /* true if inline UNDO or ATM + * holds LSN */ + + /* Always clean up the recycled context; see AtCommit_XactUndo. */ + UndoRecordSetResetCache(); + + if (!XactUndo.has_undo) + { + /* No UNDO data was written; nothing to do */ + UndoWalBatchReset(); + return; + } + + /* + * With UNDO-in-WAL, all UNDO data is already in the WAL stream. No + * separate sync is needed. The UNDO data can be read back from WAL for + * rollback via UndoReadBatchFromWAL(). + * + * For crash safety during abort: if we crash mid-rollback, the recovery + * undo phase will find this transaction's UNDO batches in WAL and + * complete the rollback. + */ + UndoWalBatchFlush(); /* no-op, kept for safety */ + + INJECTION_POINT("undo-xact-abort-before-apply", NULL); + + /* Collapse all subtransaction state. */ + CollapseXactUndoSubTransactions(); + + /* + * UNDO application strategy: inline for small transactions, deferred for + * large ones. Controlled by undo_instant_abort_threshold GUC. + * + * For small transactions (< threshold bytes of UNDO): apply UNDO + * synchronously in this backend. This avoids ATM pool accumulation and + * eliminates the dependency on the background logical revert worker. + * + * For large transactions (>= threshold): register in the ATM for deferred + * asynchronous rollback by the logical revert worker. + * + * The BumpContext issue (pfree crashes during abort) is avoided by: - + * Creating a temporary AllocSetContext for inline UNDO application - + * ApplyUndoChainFromWAL already avoids pfree on its allocations - + * Switching back to the abort context afterward + */ + { + XLogRecPtr perm_lsn = + XactUndo.last_batch_lsn[UNDOPERSISTENCE_PERMANENT]; + + if (XLogRecPtrIsValid(perm_lsn)) + { + Size total_undo_bytes = 0; + + /* Calculate total UNDO data size for threshold comparison */ + for (i = 0; i < NUndoPersistenceLevels; i++) + { + if (XactUndo.record_set[i] != NULL) + total_undo_bytes += UndoRecordSetGetSize( + XactUndo.record_set[i]); + } + + /* + * The UNDO batch records were inserted into the WAL buffers + * during this transaction but may not yet be flushed to disk. The + * inline apply path reads them back via the local XLog reader, + * which would otherwise busy-wait (pg_usleep) for the walwriter + * to flush past the batch LSN -- adding multi-second latency to + * every rollback. Flush WAL up to the end of the last record this + * backend wrote so the batch is immediately readable. We must not + * flush to GetXLogInsertRecPtr(): under concurrency that global + * insert position can point into the middle of a record another + * backend has reserved but not finished copying, so XLogFlush + * would fail with "xlog flush request is not satisfied". + * XactLastRecEnd is always a valid record boundary and, since + * this backend wrote the UNDO batch, is guaranteed to be at or + * past the batch end. This mirrors the commit-path flush in + * RecordTransactionCommit. + */ + XLogFlush(XactLastRecEnd); + + if (undo_instant_abort_threshold > 0 && + total_undo_bytes < (Size) undo_instant_abort_threshold) + { + /* + * Small transaction: apply UNDO inline. Use a dedicated + * AllocSetContext to avoid BumpContext pfree issues. + */ + MemoryContext undo_ctx; + MemoryContext old_ctx; + int saved_trans_state; + + undo_ctx = AllocSetContextCreate(TopMemoryContext, + "Inline UNDO Apply", + ALLOCSET_DEFAULT_SIZES); + old_ctx = MemoryContextSwitchTo(undo_ctx); + + { + bool undo_applied = false; + + /* + * Validate the batch LSN points to an actual UNDO record + * before attempting inline application. Stale LSNs from + * chain_prev tracking anomalies can point to non-UNDO WAL + * records, which would cause "not an UNDO batch" + * warnings. + */ + if (!UndoValidateBatchLSN(perm_lsn)) + { + elog(DEBUG1, "inline UNDO: last_batch_lsn %X/%X is not " + "a valid UNDO batch, deferring to ATM", + LSN_FORMAT_ARGS(perm_lsn)); + undo_applied = false; + goto inline_undo_done; + } + + /* + * AbortTransaction() has already advanced the transaction + * state to TRANS_ABORT, but the relcache, locks, and + * resource owner are all still live. UNDO appliers open + * relations, which asserts IsTransactionState(); + * temporarily present TRANS_INPROGRESS for the duration + * of the inline apply and always restore the real state + * afterward. + */ + saved_trans_state = EnterInlineUndoApplyState(); + PG_TRY(); + { + undo_applied = ApplyUndoChainFromWAL(perm_lsn); + } + PG_CATCH(); + { + /* + * If inline UNDO throws an error, fall back to ATM. + * Release any content locks the failed chain left + * held before unwinding. + */ + BufferLockReleaseAll(); + LeaveInlineUndoApplyState(saved_trans_state); + FlushErrorState(); + undo_applied = false; + } + PG_END_TRY(); + LeaveInlineUndoApplyState(saved_trans_state); + + MemoryContextSwitchTo(old_ctx); + MemoryContextDelete(undo_ctx); + + inline_undo_done: + if (!undo_applied) + { + /* + * Inline UNDO failed (WAL recycled, wrong record + * type, or chain walk aborted). Register in ATM for + * deferred processing by the revert worker. + */ + elog(DEBUG1, "inline UNDO failed for xid %u, " + "deferring to ATM", + GetCurrentTransactionId()); + + if (ATMAddAborted(GetCurrentTransactionId(), + MyDatabaseId, perm_lsn)) + lsn_safely_held = true; /* ATM holds the LSN */ + else + elog(WARNING, "ATM full: could not record aborted transaction %u", GetCurrentTransactionId()); + + + + + } + else + { + lsn_safely_held = true; /* UNDO fully applied, WAL can + * be recycled */ + ereport(DEBUG2, + (errmsg("inline UNDO applied for xid %u " + "(%zu bytes)", + GetCurrentTransactionId(), + total_undo_bytes))); + } + } + } + else + { + /* + * Large transaction or threshold=0: register in ATM for + * deferred rollback by the logical revert worker. + */ + if (ATMAddAborted(GetCurrentTransactionId(), + MyDatabaseId, perm_lsn)) + lsn_safely_held = true; + else + elog(WARNING, + "ATM full: could not record aborted transaction %u", + GetCurrentTransactionId()); + } + } + } + + INJECTION_POINT("undo-xact-abort-after-atm", NULL); + + /* + * Roll back per-relation UNDO chains. Queues each modified relation for + * the background per-relation UNDO worker; the aborting backend waits for + * completion in WaitForPendingRelUndo() after lock release. + */ + ApplyPerRelUndo(); + + /* Free all per-persistence-level record sets. */ + for (i = 0; i < NUndoPersistenceLevels; i++) + { + if (XactUndo.record_set[i] != NULL) + { + UndoRecordSetFree(XactUndo.record_set[i]); + XactUndo.record_set[i] = NULL; + } + } + + /* Close cached UNDO log fds. */ + UndoLogCloseFiles(); + + /* Reset per-backend write pointer tracking. */ + UndoFlushResetMaxWritePtr(); + + /* + * Release WAL retention hold ONLY if the LSN is safely held elsewhere: + * either inline UNDO completed (no WAL needed) or ATM registered the + * entry (revert worker will use ATM's copy of the LSN). + * + * If BOTH failed (inline UNDO failed AND ATM pool full), retain the + * per-backend slot to prevent checkpoint from recycling the WAL segment + * containing our UNDO data. The slot will be cleared at backend exit via + * AtCleanup_XactUndo. + */ + if (lsn_safely_held) + UndoClearBatchLSN(); + else + elog(DEBUG1, "retaining per-backend UNDO LSN slot (ATM and inline both failed)"); + + ResetXactUndo(); +} + +/* + * AtSubCommit_XactUndo + * Subtransaction commit: merge sub undo state into parent. + */ +void +AtSubCommit_XactUndo(int level) +{ + XactUndoSubTransactionState *cur; + XactUndoSubTransactionState *parent; + int i; + + if (XactUndo.subxact_depth <= 0) + return; + + cur = CURRENT_SUBXACT(); + if ((int) cur->nestingLevel != level) + return; + + parent = &XactUndo.subxact_stack[XactUndo.subxact_depth - 1]; + + /* + * Merge start locations into parent. + * + * Invariant: all UNDO records for this transaction, regardless of nesting + * level, are stored in a single chain per persistence level (one + * UndoRecordSet). start_location tracks the earliest record the + * subtransaction generated. Since records are strictly append-only, the + * parent's start location is always earlier than the subtransaction's if + * it exists. We only update the parent's start when it is not yet set + * (the parent wrote no UNDO before this subtransaction). + */ + for (i = 0; i < NUndoPersistenceLevels; i++) + { + if (UndoRecPtrIsValid(cur->start_location[i]) && + !UndoRecPtrIsValid(parent->start_location[i])) + { + parent->start_location[i] = cur->start_location[i]; + } + } + + XactUndo.subxact_depth--; +} + +/* + * AtSubAbort_XactUndo + * Subtransaction abort: apply undo for this sub-level, clean up. + * + * For per-relation UNDO, we apply the subtransaction's records synchronously + * by queuing work for the background UNDO worker. This ensures that tuples + * inserted/modified by the aborting subtransaction are physically restored + * before control returns to the caller. + * + * Any AM-private per-subtransaction tracking state is cleaned up by the + * owning AM's own SubXactCallback. + */ +void +AtSubAbort_XactUndo(int level) +{ + XactUndoSubTransactionState *cur; + XactUndoSubTransactionState *parent; + int i; + + if (XactUndo.subxact_depth <= 0) + return; + + cur = CURRENT_SUBXACT(); + if ((int) cur->nestingLevel != level) + return; + + parent = &XactUndo.subxact_stack[XactUndo.subxact_depth - 1]; + + /* + * Apply per-relation UNDO for records generated during this + * subtransaction. We iterate the record sets and apply records whose + * UndoRecPtr is at or after this subtransaction's start_location. + * + * For each persistence level where this subtransaction generated UNDO + * records, queue the work for the per-relation UNDO worker to apply them + * synchronously. The parent transaction's records (before the + * subtransaction's start_location) are preserved. + */ + for (i = 0; i < NUndoPersistenceLevels; i++) + { + UndoRecPtr sub_start = cur->start_location[i]; + + if (!UndoRecPtrIsValid(sub_start)) + continue; + + /* + * Eagerly roll back the cluster-wide UNDO this subtransaction wrote. + * + * If the subtransaction advanced XactUndo.last_batch_lsn[i] beyond + * the value saved at subtransaction start, it wrote UNDO batches that + * must now be reversed. We must apply them here rather than defer to + * the top-level ATM entry: a subtransaction can abort while its + * parent goes on to COMMIT, in which case no top-level ATM entry is + * ever created and the deferred UNDO would never run, leaking the + * aborted subtransaction's writes as if committed. + * + * Apply only the batches strictly newer than the parent's saved head + * (cur->last_batch_lsn[i]) via the bounded chain walk, so the + * parent's and earlier subtransactions' batches are preserved. + * Afterward restore last_batch_lsn[i] to the parent's value so a + * later parent abort walks only the parent's batches and does not + * double-apply ours. + * + * BumpContext safety: subtransaction abort may run under a + * BumpContext that does not support pfree(), so apply inside a + * dedicated AllocSetContext and present an in-progress transaction + * state for the relation opens, mirroring the inline path in + * AtAbort_XactUndo(). + */ + if (XLogRecPtrIsValid(XactUndo.last_batch_lsn[i]) && + XactUndo.last_batch_lsn[i] != cur->last_batch_lsn[i]) + { + XLogRecPtr sub_head = XactUndo.last_batch_lsn[i]; + XLogRecPtr parent_head = cur->last_batch_lsn[i]; + + /* + * Invariant: the current head must be strictly newer (larger) + * than the parent's saved head. If this fires, a sibling + * subtransaction improperly modified last_batch_lsn after its own + * abort, which would cause us to skip or double-apply batches. + */ + Assert(!XLogRecPtrIsValid(parent_head) || + sub_head > parent_head); + + /* + * Flush WAL so the subtransaction's batches are readable by the + * local XLog reader without busy-waiting on the walwriter. See + * the matching flush in AtAbort_XactUndo() for why XactLastRecEnd + * is the correct, always-valid record boundary to flush to. + */ + XLogFlush(XactLastRecEnd); + + if (UndoValidateBatchLSN(sub_head)) + { + MemoryContext undo_ctx; + MemoryContext old_ctx; + int saved_trans_state; + + undo_ctx = AllocSetContextCreate(TopMemoryContext, + "Subxact UNDO Apply", + ALLOCSET_DEFAULT_SIZES); + old_ctx = MemoryContextSwitchTo(undo_ctx); + + saved_trans_state = EnterInlineUndoApplyState(); + PG_TRY(); + { + ApplyUndoChainFromWALBounded(sub_head, parent_head); + } + PG_CATCH(); + { + /* + * Release any content locks the failed subxact UNDO chain + * left held before unwinding. + */ + BufferLockReleaseAll(); + LeaveInlineUndoApplyState(saved_trans_state); + FlushErrorState(); + } + PG_END_TRY(); + LeaveInlineUndoApplyState(saved_trans_state); + + MemoryContextSwitchTo(old_ctx); + MemoryContextDelete(undo_ctx); + } + else + elog(DEBUG1, "subxact UNDO: head %X/%X is not a valid UNDO " + "batch, skipping eager rollback", + LSN_FORMAT_ARGS(sub_head)); + + XactUndo.last_batch_lsn[i] = parent_head; + } + + /* + * Reset the last_location to what it was before this subtransaction, + * so that if the parent transaction continues and then aborts, only + * the parent's records are applied (the subtransaction's records have + * already been applied). + */ + if (UndoRecPtrIsValid(parent->start_location[i])) + XactUndo.last_location[i] = parent->start_location[i]; + } + + XactUndo.subxact_depth--; +} + +/* + * AtProcExit_XactUndo + * Process exit cleanup for transaction undo. + */ +void +AtProcExit_XactUndo(void) +{ + int i; + + /* Free any lingering record sets. */ + for (i = 0; i < NUndoPersistenceLevels; i++) + { + if (XactUndo.record_set[i] != NULL) + { + UndoRecordSetFree(XactUndo.record_set[i]); + XactUndo.record_set[i] = NULL; + } + } + + /* Close any cached UNDO log fds before process exit. */ + UndoLogCloseFiles(); + + /* Reset per-backend write pointer tracking. */ + UndoFlushResetMaxWritePtr(); + + /* Release any WAL retention hold (in case process exits mid-transaction). */ + UndoClearBatchLSN(); + + ResetXactUndo(); +} + +/* + * XactUndo_SubXactCallback + * Subtransaction callback to manage UNDO subtransaction state. + * + * This ensures the UNDO subsystem properly tracks all subtransactions, + * including those created by ROLLBACK TO SAVEPOINT. + * + * The subtransaction state is stored in a dynamically-grown array in + * TopMemoryContext. In the common case (depth < capacity), no allocation + * occurs. Growth via repalloc is safe here because it operates on a + * TopMemoryContext allocation, not a per-subtransaction context. + */ +static void +XactUndo_SubXactCallback(SubXactEvent event, SubTransactionId mySubid, + SubTransactionId parentSubid, void *arg) +{ + int i; + + /* These parameters are mandated by the callback signature. */ + (void) parentSubid; + (void) arg; + + switch (event) + { + case SUBXACT_EVENT_START_SUB: + + /* + * A new subtransaction is starting. Push an entry onto the + * dynamically-grown stack, extending it if needed. + */ + XactUndo.subxact_depth++; + EnsureSubxactStackCapacity(); + { + XactUndoSubTransactionState *s = CURRENT_SUBXACT(); + + s->nestingLevel = mySubid; + for (i = 0; i < NUndoPersistenceLevels; ++i) + { + s->start_location[i] = InvalidUndoRecPtr; + /* Save parent's last_batch_lsn for restore on abort */ + s->last_batch_lsn[i] = XactUndo.last_batch_lsn[i]; + } + } + break; + + case SUBXACT_EVENT_COMMIT_SUB: + + /* + * Subtransaction is committing. Merge its UNDO state into parent. + */ + AtSubCommit_XactUndo(mySubid); + break; + + case SUBXACT_EVENT_ABORT_SUB: + + /* + * Subtransaction is aborting. Apply UNDO and clean up. + */ + AtSubAbort_XactUndo(mySubid); + break; + + case SUBXACT_EVENT_PRE_COMMIT_SUB: + /* Nothing to do at pre-commit */ + break; + } +} + +/* + * EnsureSubxactStackCapacity + * Ensure the subxact_stack has room for the current depth. + * + * If the stack hasn't been allocated yet, allocate it with the initial + * capacity. If we've exceeded the current capacity, double it. + * The stack lives in TopMemoryContext so it persists across transactions + * within the same backend. + */ +static void +EnsureSubxactStackCapacity(void) +{ + if (XactUndo.subxact_stack == NULL) + { + /* First-time allocation */ + XactUndo.subxact_capacity = INITIAL_SUBXACT_CAPACITY; + XactUndo.subxact_stack = (XactUndoSubTransactionState *) + MemoryContextAllocZero(TopMemoryContext, + XactUndo.subxact_capacity * + sizeof(XactUndoSubTransactionState)); + } + else if (XactUndo.subxact_depth >= XactUndo.subxact_capacity) + { + int new_capacity = XactUndo.subxact_capacity * 2; + + XactUndo.subxact_stack = (XactUndoSubTransactionState *) + repalloc(XactUndo.subxact_stack, + new_capacity * sizeof(XactUndoSubTransactionState)); + /* Zero the newly-allocated portion */ + memset(&XactUndo.subxact_stack[XactUndo.subxact_capacity], 0, + (new_capacity - XactUndo.subxact_capacity) * + sizeof(XactUndoSubTransactionState)); + XactUndo.subxact_capacity = new_capacity; + } +} + +/* + * ResetXactUndo + * Reset all backend-private undo state for the next transaction. + */ +static void +ResetXactUndo(void) +{ + int i; + + XactUndo.has_undo = false; + XactUndo.subxact_depth = 0; + + /* + * The relundo_list entries live in CurTransactionContext and are freed + * automatically at transaction end; just drop the dangling head pointer. + */ + XactUndo.relundo_list = NULL; + + /* + * The subxact_stack allocation persists across transactions (it's in + * TopMemoryContext). We just reset the depth and initialize slot 0. + */ + if (XactUndo.subxact_stack != NULL) + { + XactUndo.subxact_stack[0].nestingLevel = 1; + for (i = 0; i < NUndoPersistenceLevels; i++) + { + XactUndo.subxact_stack[0].start_location[i] = InvalidUndoRecPtr; + XactUndo.subxact_stack[0].last_batch_lsn[i] = InvalidXLogRecPtr; + } + } + + for (i = 0; i < NUndoPersistenceLevels; i++) + { + XactUndo.record_set[i] = NULL; + XactUndo.last_location[i] = InvalidUndoRecPtr; + XactUndo.last_batch_lsn[i] = InvalidXLogRecPtr; + } +} + +/* + * CollapseXactUndoSubTransactions + * Collapse all subtransaction state into the top level. + */ +static void +CollapseXactUndoSubTransactions(void) +{ + while (XactUndo.subxact_depth > 0) + { + /* + * Merge current level into parent by calling AtSubCommit_XactUndo + * with the current level's nestingLevel. + */ + AtSubCommit_XactUndo( + XactUndo.subxact_stack[XactUndo.subxact_depth].nestingLevel); + } +} diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index 2fa534413eaca..ad687e56d1796 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -997,6 +997,7 @@ DefineRelation(CreateStmt *stmt, char relkind, Oid ownerId, break; default: (void) heap_reloptions(relkind, reloptions, true); + break; } if (stmt->ofTypename) diff --git a/src/backend/executor/nodeModifyTable.c b/src/backend/executor/nodeModifyTable.c index 9a1c0992bfea8..89eb64b98e318 100644 --- a/src/backend/executor/nodeModifyTable.c +++ b/src/backend/executor/nodeModifyTable.c @@ -5817,6 +5817,34 @@ ExecInitModifyTable(ModifyTable *node, EState *estate, int eflags) resultRelInfo->ri_BatchSize = 1; } + /* + * Signal the table AM about DML operations. + * + * Tell the AM that a DML operation is starting so it can enable + * optimizations like the UNDO write buffer. This is always done for + * INSERT, UPDATE, and DELETE operations regardless of estimated row count + * -- the UNDO write buffer overhead is negligible (one palloc of ~512 + * bytes, reused for the entire transaction) and the benefit of batched + * UNDO recording applies to operations of any size. + * + * We pass the subplan's row estimate (input rows to be modified) as a + * hint; the AM may use it for buffer pre-sizing. + */ + { + Cardinality estimated_rows = subplan->plan_rows; + + if (operation == CMD_INSERT || operation == CMD_UPDATE || + operation == CMD_DELETE) + { + for (i = 0; i < mtstate->mt_nrels; i++) + { + resultRelInfo = mtstate->resultRelInfo + i; + rel = resultRelInfo->ri_RelationDesc; + table_begin_bulk_insert(rel, 0, (int64) estimated_rows); + } + } + } + /* * Lastly, if this is not the primary (canSetTag) ModifyTable node, add it * to estate->es_auxmodifytables so that it will be run to completion by @@ -5847,13 +5875,16 @@ ExecEndModifyTable(ModifyTableState *node) int i; /* - * Allow any FDWs to shut down + * Allow any FDWs to shut down, and finalize bulk insert mode. */ for (i = 0; i < node->mt_nrels; i++) { int j; ResultRelInfo *resultRelInfo = node->resultRelInfo + i; + /* End bulk insert mode (flushes pending UNDO records) */ + table_finish_bulk_insert(resultRelInfo->ri_RelationDesc, 0); + if (!resultRelInfo->ri_usesFdwDirectModify && resultRelInfo->ri_FdwRoutine != NULL && resultRelInfo->ri_FdwRoutine->EndForeignModify != NULL) diff --git a/src/backend/postmaster/bgworker.c b/src/backend/postmaster/bgworker.c index 3da2a90417fc6..4b345e6fbf23b 100644 --- a/src/backend/postmaster/bgworker.c +++ b/src/backend/postmaster/bgworker.c @@ -13,6 +13,9 @@ #include "postgres.h" #include "access/parallel.h" +#include "access/logical_revert_worker.h" +#include "access/relundo_worker.h" +#include "access/undoworker.h" #include "commands/repack.h" #include "libpq/pqsignal.h" #include "miscadmin.h" @@ -166,6 +169,22 @@ static const struct { .fn_name = "DataChecksumsWorkerMain", .fn_addr = DataChecksumsWorkerMain + }, + { + .fn_name = "LogicalRevertWorkerMain", + .fn_addr = LogicalRevertWorkerMain + }, + { + .fn_name = "LogicalRevertLauncherMain", + .fn_addr = LogicalRevertLauncherMain + }, + { + .fn_name = "UndoWorkerMain", + .fn_addr = UndoWorkerMain + }, + { + .fn_name = "RelUndoWorkerMain", + .fn_addr = RelUndoWorkerMain } }; diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c index 90c7c4528e872..a76bc2c3c031b 100644 --- a/src/backend/postmaster/postmaster.c +++ b/src/backend/postmaster/postmaster.c @@ -89,6 +89,8 @@ #include #endif +#include "access/logical_revert_worker.h" +#include "access/undolog.h" #include "access/xlog.h" #include "access/xlog_internal.h" #include "access/xlogrecovery.h" @@ -925,6 +927,13 @@ PostmasterMain(int argc, char *argv[]) */ ApplyLauncherRegister(); + /* + * The Logical Revert Launcher wires up the async physical-undo-apply + * path. The launcher scans pg_database once at startup and spawns a + * per-database LogicalRevertWorker which drains the ATM. + */ + LogicalRevertLauncherRegister(); + /* * Register the shared memory needs of all core subsystems. */ diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c index 169829eb02070..dc6a01af58516 100644 --- a/src/backend/storage/buffer/bufmgr.c +++ b/src/backend/storage/buffer/bufmgr.c @@ -3218,6 +3218,68 @@ MarkBufferDirty(Buffer buffer) } } +/* + * MarkBufferDirtyShared + * + * Like MarkBufferDirty, but callable while holding only a SHARED content + * lock on the buffer. The BM_DIRTY bit is set atomically via CAS on the + * buffer header state word, which is safe regardless of lock mode. + * + * The caller MUST ensure that either: + * (a) a WAL record covering the modification has already been inserted + * (so the WAL flush in the checkpointer will find the record), or + * (b) full_page_writes will capture a consistent image (the modification + * is protected by a per-tuple lock that prevents torn pages). + * + * This is used by an in-place-update table AM's tuple-level CAS update path + * where per-tuple atomics protect individual tuple data under a shared page + * lock. + */ +void +MarkBufferDirtyShared(Buffer buffer) +{ + BufferDesc *bufHdr; + uint64 buf_state; + uint64 old_buf_state; + + if (!BufferIsValid(buffer)) + elog(ERROR, "bad buffer ID: %d", buffer); + + if (BufferIsLocal(buffer)) + { + MarkLocalBufferDirty(buffer); + return; + } + + bufHdr = GetBufferDescriptor(buffer - 1); + + Assert(BufferIsPinned(buffer)); + /* Caller holds at least BUFFER_LOCK_SHARE -- no exclusive assertion */ + + old_buf_state = pg_atomic_read_u64(&bufHdr->state); + for (;;) + { + if (old_buf_state & BM_LOCKED) + old_buf_state = WaitBufHdrUnlocked(bufHdr); + + buf_state = old_buf_state; + + Assert(BUF_STATE_GET_REFCOUNT(buf_state) > 0); + buf_state |= BM_DIRTY; + + if (pg_atomic_compare_exchange_u64(&bufHdr->state, &old_buf_state, + buf_state)) + break; + } + + if (!(old_buf_state & BM_DIRTY)) + { + pgBufferUsage.shared_blks_dirtied++; + if (VacuumCostActive) + VacuumCostBalance += VacuumCostPageDirty; + } +} + /* * ReleaseAndReadBuffer -- combine ReleaseBuffer() and ReadBuffer() * @@ -5899,6 +5961,79 @@ UnlockBuffers(void) } } +/* + * BufferLockReleaseAll -- release any buffer content locks still held by this + * backend. + * + * Normally, buffer content locks are released as part of resource-owner + * cleanup (see ResOwnerReleaseBuffer, which calls BufferLockUnlock on any + * entry whose data.lockmode != BUFFER_LOCK_UNLOCK). However, that cleanup + * runs in ResourceOwnerRelease(RESOURCE_RELEASE_LOCKS), which is *after* + * AtAbort_XactUndo() in AbortTransaction(). Inline UNDO application may + * itself take buffer content locks, and BufferLockAcquire asserts that no + * lock is already held. If a statement errors out while holding a share + * or exclusive content lock, that assert would trip on any re-lock during + * UNDO application. + * + * This helper releases such stragglers early, before inline UNDO runs. + * It mirrors the still-locked branch of ResOwnerReleaseBuffer (matching + * the HOLD_INTERRUPTS/BufferLockUnlock discipline). Callers must be in + * an error-recovery path -- releasing content locks under a live statement + * would violate correctness. + * + * The subsequent ResourceOwnerRelease pass at RESOURCE_RELEASE_LOCKS still + * runs, but ResOwnerReleaseBuffer only calls BufferLockUnlock when + * data.lockmode != BUFFER_LOCK_UNLOCK, so already-released buffers are + * skipped -- no double-unlock. + */ +void +BufferLockReleaseAll(void) +{ + PrivateRefCountEntry *res; + int i; + + /* Walk the small array first (fast path, covers the usual case). */ + for (i = 0; i < REFCOUNT_ARRAY_ENTRIES; i++) + { + Buffer buffer = PrivateRefCountArrayKeys[i]; + + if (buffer == InvalidBuffer) + continue; + + res = &PrivateRefCountArray[i]; + if (res->data.lockmode == BUFFER_LOCK_UNLOCK) + continue; + + /* Local buffers do not have content locks. */ + if (BufferIsLocal(buffer)) + continue; + + HOLD_INTERRUPTS(); /* matched by RESUME_INTERRUPTS in + * BufferLockUnlock */ + BufferLockUnlock(buffer, GetBufferDescriptor(buffer - 1)); + } + + /* Then the overflow hash, if any. */ + if (PrivateRefCountOverflowed) + { + refcount_iterator iter; + + refcount_start_iterate(PrivateRefCountHash, &iter); + while ((res = refcount_iterate(PrivateRefCountHash, &iter)) != NULL) + { + Buffer buffer = res->buffer; + + if (res->data.lockmode == BUFFER_LOCK_UNLOCK) + continue; + if (BufferIsLocal(buffer)) + continue; + + HOLD_INTERRUPTS(); + BufferLockUnlock(buffer, GetBufferDescriptor(buffer - 1)); + } + } +} + /* * Acquire the buffer content lock in the specified mode * diff --git a/src/backend/tcop/utility.c b/src/backend/tcop/utility.c index 73a56f1df1dc3..b093940cb0c34 100644 --- a/src/backend/tcop/utility.c +++ b/src/backend/tcop/utility.c @@ -1190,6 +1190,7 @@ ProcessUtilitySlow(ParseState *pstate, validnsps, true, false); + (void) heap_reloptions(RELKIND_TOASTVALUE, toast_options, true); diff --git a/src/backend/utils/activity/wait_event_names.txt b/src/backend/utils/activity/wait_event_names.txt index 256b3a3c02e01..80013fa70bb1b 100644 --- a/src/backend/utils/activity/wait_event_names.txt +++ b/src/backend/utils/activity/wait_event_names.txt @@ -69,6 +69,8 @@ WAL_RECEIVER_MAIN "Waiting in main loop of WAL receiver process." WAL_SENDER_MAIN "Waiting in main loop of WAL sender process." WAL_SUMMARIZER_WAL "Waiting in WAL summarizer for more WAL to be generated." WAL_WRITER_MAIN "Waiting in main loop of WAL writer process." +UNDO_FLUSH_MAIN "Waiting in main loop of UNDO flush writer process." +UNDO_WORKER_MAIN "Waiting in main loop of UNDO discard worker process." ABI_compatibility: @@ -167,6 +169,7 @@ WAL_RECEIVER_UPSTREAM_CATCHUP "Waiting for upstream server WAL flush position to WAL_RECEIVER_WAIT_START "Waiting for startup process to send initial data for streaming replication." WAL_SUMMARY_READY "Waiting for a new WAL summary to be generated." XACT_GROUP_UPDATE "Waiting for the group leader to update transaction status at transaction end." +UNDO_FLUSH_SYNC "Waiting for UNDO flush writer to sync UNDO data to disk." ABI_compatibility: @@ -418,6 +421,10 @@ XactSLRU "Waiting to access the transaction status SLRU cache." ParallelVacuumDSA "Waiting for parallel vacuum dynamic shared memory allocation." AioUringCompletion "Waiting for another process to complete IO via io_uring." ShmemIndex "Waiting to find or allocate space in shared memory." +UndoLog "Waiting to access or modify UNDO log metadata." +UndoWorker "Waiting to access or modify UNDO worker shared memory queue." +AbortedTxnMap "Waiting to access the Aborted Transaction Map." +SecondaryLog "Waiting to access the Secondary Log (sLog)." # No "ABI_compatibility" region here as WaitEventLWLock has its own C code. diff --git a/src/backend/utils/init/postinit.c b/src/backend/utils/init/postinit.c index 3d8c9bdebd559..993de1aa39752 100644 --- a/src/backend/utils/init/postinit.c +++ b/src/backend/utils/init/postinit.c @@ -21,6 +21,7 @@ #include "access/genam.h" #include "access/heapam.h" +#include "access/undo.h" #include "access/htup_details.h" #include "access/session.h" #include "access/tableam.h" @@ -856,6 +857,9 @@ InitPostgres(const char *in_dbname, Oid dboid, InitCatalogCache(); InitPlanCache(); + /* Initialize per-backend undo subsystem state */ + InitializeUndo(); + /* Initialize portal manager */ EnablePortalManager(); diff --git a/src/backend/utils/misc/guc_parameters.dat b/src/backend/utils/misc/guc_parameters.dat index e2b48ea69e923..0de04aec1cede 100644 --- a/src/backend/utils/misc/guc_parameters.dat +++ b/src/backend/utils/misc/guc_parameters.dat @@ -1951,6 +1951,16 @@ max => 'MAX_KILOBYTES', }, +{ name => 'logical_revert_naptime', type => 'int', context => 'PGC_SIGHUP', group => 'VACUUM_AUTOVACUUM', + short_desc => 'Time between ATM scan cycles in the logical revert worker.', + long_desc => 'The logical revert worker sleeps for this many milliseconds between scans of the ATM for unreverted aborted transactions.', + flags => 'GUC_UNIT_MS', + variable => 'logical_revert_naptime', + boot_val => '1000', + min => '100', + max => 'INT_MAX', +}, + { name => 'maintenance_io_concurrency', type => 'int', context => 'PGC_USERSET', group => 'RESOURCES_IO', short_desc => 'A variant of "effective_io_concurrency" that is used for maintenance work.', long_desc => '0 disables simultaneous requests.', @@ -2044,6 +2054,15 @@ max => 'MAX_BACKENDS', }, +{ name => 'max_logical_revert_workers', type => 'int', context => 'PGC_POSTMASTER', group => 'VACUUM_AUTOVACUUM', + short_desc => 'Maximum number of logical revert background workers.', + long_desc => 'Sets the maximum number of logical revert workers that apply UNDO chains for aborted transactions. Acts as a hard ceiling on the number of concurrently running per-database revert workers. Set to 0 to disable the logical revert launcher entirely.', + variable => 'max_logical_revert_workers', + boot_val => '2', + min => '0', + max => '64', +}, + { name => 'max_notify_queue_pages', type => 'int', context => 'PGC_POSTMASTER', group => 'RESOURCES_DISK', short_desc => 'Sets the maximum number of allocated pages for NOTIFY / LISTEN queue.', variable => 'max_notify_queue_pages', @@ -2755,6 +2774,15 @@ boot_val => '""', }, +{ name => 'slog_dsa_max_size_mb', type => 'int', context => 'PGC_POSTMASTER', group => 'WAL_SETTINGS', + short_desc => 'Maximum shared memory for the sLog DSA area.', + long_desc => 'Sets the maximum size (in MB) of the DSA area used by the sLog to back its aborted-transaction radix tree. When this limit is reached, new entries cannot be stored and a WARNING is emitted.', + variable => 'slog_dsa_max_size_mb', + boot_val => '256', + min => '1', + max => '16384', +}, + { name => 'ssl', type => 'bool', context => 'PGC_SIGHUP', group => 'CONN_AUTH_SSL', short_desc => 'Enables SSL connections.', variable => 'EnableSSL', @@ -3292,6 +3320,74 @@ boot_val => 'false', }, +{ name => 'undo_batch_record_limit', type => 'int', context => 'PGC_SIGHUP', group => 'RESOURCES_MEM', + short_desc => 'Sets the UNDO write buffer flush threshold in number of records.', + long_desc => 'When the UNDO write buffer accumulates this many records, it inserts a new XLOG_UNDO_BATCH WAL record. Larger values reduce the number of WAL reads during rollback of large transactions but hold the WAL insertion lock longer per batch.', + variable => 'undo_batch_record_limit', + boot_val => '1000', + min => '100', + max => '100000', +}, + +{ name => 'undo_batch_size_kb', type => 'int', context => 'PGC_SIGHUP', group => 'RESOURCES_MEM', + short_desc => 'Sets the UNDO write buffer flush threshold in kilobytes.', + long_desc => 'When the UNDO write buffer accumulates this many kilobytes, it inserts a new XLOG_UNDO_BATCH WAL record. (This does not flush WAL to disk; that is controlled by synchronous_commit.) Larger values reduce the number of WAL reads during rollback of large transactions but hold the WAL insertion lock longer per batch.', + flags => 'GUC_UNIT_KB', + variable => 'undo_batch_size_kb', + boot_val => '256', + min => '64', + max => '4096', +}, + +{ name => 'undo_buffer_size', type => 'int', context => 'PGC_POSTMASTER', group => 'RESOURCES_MEM', + short_desc => 'Sets the size of the UNDO buffer cache.', + long_desc => 'Size of the dedicated buffer cache for UNDO log pages, in kilobytes.', + flags => 'GUC_UNIT_KB', + variable => 'undo_buffer_size', + boot_val => '1024', + min => '128', + max => 'INT_MAX / 1024', +}, + +{ name => 'undo_instant_abort_threshold', type => 'int', context => 'PGC_USERSET', group => 'WAL_SETTINGS', + short_desc => 'Per-relation UNDO size threshold for ATM instant abort.', + long_desc => 'When estimated per-relation UNDO data for a transaction exceeds this many bytes, abort uses ATM instant abort (O(1) with asynchronous cleanup) instead of synchronous rollback. Set to 0 to always use ATM instant abort.', + variable => 'undo_instant_abort_threshold', + boot_val => '65536', + min => '0', + max => 'INT_MAX', +}, + +{ name => 'undo_max_wal_retention_size', type => 'int', context => 'PGC_SIGHUP', group => 'WAL_SETTINGS', + short_desc => 'Maximum WAL retained for UNDO rollback before warning.', + long_desc => 'If the WAL retained because of in-flight UNDO exceeds this size (in MB), a WARNING is logged. 0 disables the check.', + flags => 'GUC_UNIT_MB', + variable => 'undo_max_wal_retention_size', + boot_val => '0', + min => '0', + max => 'INT_MAX / 2', +}, + +{ name => 'undo_retention_time', type => 'int', context => 'PGC_SIGHUP', group => 'WAL_SETTINGS', + short_desc => 'Minimum time to retain UNDO records.', + long_desc => 'UNDO records will not be discarded until they are at least this old, in milliseconds.', + flags => 'GUC_UNIT_MS', + variable => 'undo_retention_time', + boot_val => '60000', + min => '0', + max => 'INT_MAX', +}, + +{ name => 'undo_worker_naptime', type => 'int', context => 'PGC_SIGHUP', group => 'VACUUM_AUTOVACUUM', + short_desc => 'Time to sleep between runs of the UNDO discard worker.', + long_desc => 'The UNDO discard worker wakes up periodically to discard old UNDO records.', + flags => 'GUC_UNIT_MS', + variable => 'undo_worker_naptime', + boot_val => '10000', + min => '1', + max => 'INT_MAX', +}, + { name => 'unix_socket_directories', type => 'string', context => 'PGC_POSTMASTER', group => 'CONN_AUTH_SETTINGS', short_desc => 'Sets the directories where Unix-domain sockets will be created.', flags => 'GUC_LIST_INPUT | GUC_LIST_QUOTE | GUC_SUPERUSER_ONLY', diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index 1ec460b6a8236..64c0eb8ee9148 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -31,9 +31,13 @@ #include "access/commit_ts.h" #include "access/gin.h" +#include "access/logical_revert_worker.h" +#include "access/slog.h" #include "access/slru.h" #include "access/toast_compression.h" #include "access/twophase.h" +#include "access/undolog.h" +#include "access/xactundo.h" #include "access/xlog_internal.h" #include "access/xlogprefetcher.h" #include "access/xlogrecovery.h" diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index 493f57409e11d..cd89ba3b85f71 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -913,6 +913,20 @@ # (change requires restart) #recovery_init_sync_method = fsync # fsync, syncfs (Linux 5.8+) +# - Cluster-wide UNDO (development and testing) - + +#undo_buffer_size = 1MB # memory buffer for UNDO log records + # (change requires restart) +#undo_instant_abort_threshold = 65536 # bytes; 0 = always use ATM instant abort +#undo_max_wal_retention_size = 0 # MB; 0 = unlimited UNDO WAL retention +#undo_retention_time = 300s # time to retain UNDO records +#undo_worker_naptime = 60s # time between UNDO discard worker runs +#undo_batch_size_kb = 256 # KB per UNDO batch flush (64-4096); reload +#undo_batch_record_limit = 1000 # records per UNDO batch flush (100-100000); reload +#slog_dsa_max_size_mb = 256 # MB; max sLog DSA area size (aborted-txn tree) +#max_logical_revert_workers = 2 # 0 disables the logical revert launcher +#logical_revert_naptime = 1s # time between ATM scan cycles + #------------------------------------------------------------------------------ # CONFIG FILE INCLUDES diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 4948e6d80c7ee..7380615a657f5 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -17475,6 +17475,7 @@ dumpTableSchema(Archive *fout, const TableInfo *tbinfo) if (nonemptyReloptions(tbinfo->reloptions)) { addcomma = true; + /* Emits all heap reloptions */ appendReloptionsArrayAH(q, tbinfo->reloptions, "", fout); } if (nonemptyReloptions(tbinfo->toast_reloptions)) diff --git a/src/bin/pg_waldump/relundodesc.c b/src/bin/pg_waldump/relundodesc.c new file mode 120000 index 0000000000000..90437665e3733 --- /dev/null +++ b/src/bin/pg_waldump/relundodesc.c @@ -0,0 +1 @@ +../../../src/backend/access/rmgrdesc/relundodesc.c \ No newline at end of file diff --git a/src/bin/pg_waldump/rmgrdesc.c b/src/bin/pg_waldump/rmgrdesc.c index 931ab8b979e23..fda1b2b47e93f 100644 --- a/src/bin/pg_waldump/rmgrdesc.c +++ b/src/bin/pg_waldump/rmgrdesc.c @@ -20,6 +20,9 @@ #include "access/nbtxlog.h" #include "access/rmgr.h" #include "access/spgxlog.h" +#include "access/atm_xlog.h" +#include "access/relundo_xlog.h" +#include "access/undo_xlog.h" #include "access/xact.h" #include "access/xlog_internal.h" #include "catalog/storage_xlog.h" diff --git a/src/bin/pg_waldump/t/001_basic.pl b/src/bin/pg_waldump/t/001_basic.pl index 53b2f016b8035..d1aafc2f37d56 100644 --- a/src/bin/pg_waldump/t/001_basic.pl +++ b/src/bin/pg_waldump/t/001_basic.pl @@ -80,7 +80,10 @@ ReplicationOrigin Generic LogicalMessage -XLOG2$/, +XLOG2 +Undo +ATM +RelUndo$/, 'rmgr list'); diff --git a/src/bin/pg_waldump/undodesc.c b/src/bin/pg_waldump/undodesc.c new file mode 120000 index 0000000000000..6bb50cf1d40f7 --- /dev/null +++ b/src/bin/pg_waldump/undodesc.c @@ -0,0 +1 @@ +../../../src/backend/access/rmgrdesc/undodesc.c \ No newline at end of file diff --git a/src/common/relpath.c b/src/common/relpath.c index 8fb3bed7873ab..32f12c5cdd8a2 100644 --- a/src/common/relpath.c +++ b/src/common/relpath.c @@ -35,6 +35,7 @@ const char *const forkNames[] = { [FSM_FORKNUM] = "fsm", [VISIBILITYMAP_FORKNUM] = "vm", [INIT_FORKNUM] = "init", + [RELUNDO_FORKNUM] = "relundo", }; StaticAssertDecl(lengthof(forkNames) == (MAX_FORKNUM + 1), diff --git a/src/include/access/atm.h b/src/include/access/atm.h new file mode 100644 index 0000000000000..fc0d382a752b0 --- /dev/null +++ b/src/include/access/atm.h @@ -0,0 +1,64 @@ +/*------------------------------------------------------------------------- + * + * atm.h + * Aborted Transaction Map for CTR (Constant-Time Recovery) + * + * The ATM is a shared-memory structure that tracks aborted transactions + * whose per-relation UNDO chains have not yet been applied (Logical + * Revert). It drives the background Logical Revert worker. + * + * The ATM is now backed by the sLog (Secondary Log) shared-memory hash + * tables defined in access/slog.h. All ATM functions are thin wrappers + * around sLog operations, preserving the existing API and WAL format. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/access/atm.h + * + *------------------------------------------------------------------------- + */ +#ifndef ATM_H +#define ATM_H + +#include "access/atm_xlog.h" +#include "access/transam.h" +#include "datatype/timestamp.h" +#include "storage/lwlock.h" + +/* Shared memory sizing and initialization */ +extern Size ATMShmemSize(void); +extern void ATMShmemInit(void); + +/* Core API */ +extern bool ATMGetLastBatchLSN(TransactionId xid, XLogRecPtr *lsn_out); +extern bool ATMAddAborted(TransactionId xid, Oid dboid, + XLogRecPtr last_batch_lsn); +extern void ATMForget(TransactionId xid); +extern void ATMMarkReverted(TransactionId xid); + +/* Iteration for Logical Revert worker */ +extern bool ATMGetNextUnreverted(TransactionId *xid_out, Oid *dboid_out, + XLogRecPtr *lsn_out); +extern int ATMCollectUnrevertedDatabases(Oid *dboids, int max_dboids); + +/* WAL retention: oldest batch LSN across unreverted entries */ +extern XLogRecPtr ATMGetOldestUnrevertedLSN(void); + +/* Recovery support */ +extern void ATMRecoveryFinalize(void); + +/* + * Crash-safety: persist the ATM at each checkpoint and reload it at startup + * before the redo pass. The ATM lives only in shared memory and is otherwise + * reconstructed solely by replaying XLOG_ATM_ABORT / XLOG_ATM_FORGET during + * redo; a checkpoint that advances the redo pointer past an un-forgotten + * XLOG_ATM_ABORT would make crash recovery miss that abort, silently losing a + * guaranteed rollback. CheckPointATM() durably snapshots the map; startup + * calls ATMReloadFromCheckpoint() before redo so atm_redo's XLOG_ATM_FORGET + * replays can correctly remove entries forgotten after the checkpoint. + */ +extern void CheckPointATM(void); +extern void ATMReloadFromCheckpoint(void); + +#endif /* ATM_H */ diff --git a/src/include/access/atm_xlog.h b/src/include/access/atm_xlog.h new file mode 100644 index 0000000000000..947512f5559c8 --- /dev/null +++ b/src/include/access/atm_xlog.h @@ -0,0 +1,49 @@ +/*------------------------------------------------------------------------- + * + * atm_xlog.h + * Aborted Transaction Map XLOG resource manager definitions + * + * This header is safe for inclusion from frontend code (e.g., pg_waldump). + * For the full ATM API, include "access/atm.h" instead. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/access/atm_xlog.h + * + *------------------------------------------------------------------------- + */ +#ifndef ATM_XLOG_H +#define ATM_XLOG_H + +#include "access/xlogreader.h" +#include "lib/stringinfo.h" + +/* WAL record types for RM_ATM_ID */ +#define XLOG_ATM_ABORT 0x00 +#define XLOG_ATM_FORGET 0x10 + +/* WAL record structures */ +typedef struct xl_atm_abort +{ + TransactionId xid; + XLogRecPtr last_batch_lsn; /* LSN of last UNDO batch for this xid */ + Oid dboid; + Oid reloid; /* InvalidOid (kept for struct layout) */ +} xl_atm_abort; + +#define SizeOfXlAtmAbort (offsetof(xl_atm_abort, reloid) + sizeof(Oid)) + +typedef struct xl_atm_forget +{ + TransactionId xid; +} xl_atm_forget; + +#define SizeOfXlAtmForget sizeof(xl_atm_forget) + +/* Resource manager functions */ +extern void atm_redo(XLogReaderState *record); +extern void atm_desc(StringInfo buf, XLogReaderState *record); +extern const char *atm_identify(uint8 info); + +#endif /* ATM_XLOG_H */ diff --git a/src/include/access/heapam.h b/src/include/access/heapam.h index 5176478c29583..3cf4050cddee9 100644 --- a/src/include/access/heapam.h +++ b/src/include/access/heapam.h @@ -544,4 +544,5 @@ heap_execute_freeze_tuple(HeapTupleHeader tuple, HeapTupleFreeze *frz) tuple->t_infomask2 = frz->t_infomask2; } + #endif /* HEAPAM_H */ diff --git a/src/include/access/heapam_xlog.h b/src/include/access/heapam_xlog.h index 3f79c389a90a7..4af279ea975ec 100644 --- a/src/include/access/heapam_xlog.h +++ b/src/include/access/heapam_xlog.h @@ -77,6 +77,8 @@ /* all_frozen_set always implies all_visible_set */ #define XLH_INSERT_ALL_FROZEN_SET (1<<5) +/* UNDO payload is embedded in this WAL record (bit 6 unused, confirmed by audit) */ +#define XLH_INSERT_HAS_UNDO (1<<6) /* * xl_heap_update flag values, 8 bits are available. @@ -90,6 +92,8 @@ #define XLH_UPDATE_CONTAINS_NEW_TUPLE (1<<4) #define XLH_UPDATE_PREFIX_FROM_OLD (1<<5) #define XLH_UPDATE_SUFFIX_FROM_OLD (1<<6) +/* UNDO payload is embedded in this WAL record (bit 7 unused, confirmed by audit) */ +#define XLH_UPDATE_HAS_UNDO (1<<7) /* convenience macro for checking whether any form of old tuple was logged */ #define XLH_UPDATE_CONTAINS_OLD \ @@ -107,6 +111,9 @@ /* See heap_delete() */ #define XLH_DELETE_NO_LOGICAL (1<<5) +/* UNDO payload is embedded in this WAL record (bit 6 unused, confirmed by audit) */ +#define XLH_DELETE_HAS_UNDO (1<<6) + /* convenience macro for checking whether any form of old tuple was logged */ #define XLH_DELETE_CONTAINS_OLD \ (XLH_DELETE_CONTAINS_OLD_TUPLE | XLH_DELETE_CONTAINS_OLD_KEY) diff --git a/src/include/access/logical_revert_worker.h b/src/include/access/logical_revert_worker.h new file mode 100644 index 0000000000000..43dc92c052911 --- /dev/null +++ b/src/include/access/logical_revert_worker.h @@ -0,0 +1,43 @@ +/*------------------------------------------------------------------------- + * + * logical_revert_worker.h + * Background worker for timer-driven Logical Revert via ATM scan + * + * The Logical Revert worker periodically scans the ATM (Aborted Transaction + * Map) for entries whose UNDO chains have not yet been applied, opens the + * target relation, applies the UNDO chain via the per-AM apply callback, + * marks the ATM entry as reverted, emits an XLOG_ATM_FORGET WAL record, and + * removes the entry from the ATM. + * + * This worker is timer-driven (periodic scan) rather than event-driven + * (queue-based). + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/access/logical_revert_worker.h + * + *------------------------------------------------------------------------- + */ +#ifndef LOGICAL_REVERT_WORKER_H +#define LOGICAL_REVERT_WORKER_H + +#include "postgres.h" + +/* Shared memory sizing and initialization */ +extern Size LogicalRevertShmemSize(void); +extern void LogicalRevertShmemInit(void); + +/* Worker entry points */ +extern void LogicalRevertWorkerMain(Datum main_arg); + +/* Launch a logical revert worker for a specific database */ +extern void StartLogicalRevertWorker(Oid dboid); +extern void LogicalRevertLauncherMain(Datum main_arg); +extern void LogicalRevertLauncherRegister(void); + +/* GUC parameters */ +extern int logical_revert_naptime; +extern int max_logical_revert_workers; + +#endif /* LOGICAL_REVERT_WORKER_H */ diff --git a/src/include/access/relundo.h b/src/include/access/relundo.h new file mode 100644 index 0000000000000..c7cdae3939de3 --- /dev/null +++ b/src/include/access/relundo.h @@ -0,0 +1,697 @@ +/*------------------------------------------------------------------------- + * + * relundo.h + * Per-relation UNDO for MVCC visibility determination + * + * This subsystem provides per-relation UNDO logging for table access methods + * that need to determine tuple visibility by walking UNDO chains. + * This is complementary to the existing cluster-wide UNDO system which is used + * for transaction rollback. + * + * ARCHITECTURE: + * ------------- + * Per-relation UNDO stores operation metadata (INSERT/DELETE/UPDATE/LOCK) within + * each relation's UNDO fork, enabling MVCC visibility checks via UNDO chain walking. + * Each UNDO record contains minimal metadata needed for visibility determination. + * + * This differs from cluster-wide UNDO which stores complete tuple data in shared + * log files for physical transaction rollback. The two systems coexist independently: + * + * Cluster-Wide UNDO (existing): Transaction rollback, crash recovery + * Per-Relation UNDO (this file): MVCC visibility determination + * + * UNDO POINTER FORMAT: + * ------------------- + * RelUndoRecPtr is a 64-bit pointer with three fields: + * Bits 0-15: Offset within page (16 bits, max 64KB pages) + * Bits 16-47: Block number (32 bits, max 4 billion blocks) + * Bits 48-63: Counter (16 bits, wraps every 65536 generations) + * + * The counter enables fast age comparison without reading UNDO pages. + * + * USAGE PATTERN: + * ------------- + * Table AMs that need per-relation UNDO follow this pattern: + * + * 1. RelUndoReserve() - Reserve space, pin buffer + * 2. Perform DML operation (may fail) + * 3. RelUndoFinish() - Write UNDO record, release buffer + * OR RelUndoCancel() - Release reservation on error + * + * Example: + * Buffer undo_buf; + * RelUndoRecPtr ptr = RelUndoReserve(rel, record_size, &undo_buf); + * + * // Perform DML (may error out safely) + * InsertTuple(rel, tid); + * + * // Commit UNDO record + * RelUndoFinish(rel, undo_buf, ptr, &header, payload, payload_size); + * + * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/access/relundo.h + * + *------------------------------------------------------------------------- + */ +#ifndef RELUNDO_H +#define RELUNDO_H + +#include "access/transam.h" +#include "access/xlogdefs.h" +#include "common/relpath.h" +#include "storage/block.h" +#include "storage/buf.h" +#include "storage/bufpage.h" +#include "storage/itemptr.h" +#include "storage/relfilelocator.h" +#include "utils/rel.h" +#include "utils/snapshot.h" + +/* + * RelUndoRecPtr: 64-bit pointer for per-relation UNDO records + * + * Layout: + * [63:48] Counter (16 bits) - Generation counter for age comparison + * [47:16] BlockNum (32 bits) - Block number in relation UNDO fork + * [15:0] Offset (16 bits) - Byte offset within page + */ +typedef uint64 RelUndoRecPtr; + +/* Invalid UNDO pointer constant */ +#define InvalidRelUndoRecPtr ((RelUndoRecPtr) 0) + +/* Check if pointer is valid */ +#define RelUndoRecPtrIsValid(ptr) \ + ((ptr) != InvalidRelUndoRecPtr) + +/* Extract counter field (bits 63:48) */ +#define RelUndoGetCounter(ptr) \ + ((uint16)(((ptr) >> 48) & 0xFFFF)) + +/* Extract block number field (bits 47:16) */ +#define RelUndoGetBlockNum(ptr) \ + ((BlockNumber)(((ptr) >> 16) & 0xFFFFFFFF)) + +/* Extract offset field (bits 15:0) */ +#define RelUndoGetOffset(ptr) \ + ((uint16)((ptr) & 0xFFFF)) + +/* Construct UNDO pointer from components */ +#define MakeRelUndoRecPtr(counter, blkno, offset) \ + ((((uint64)(counter)) << 48) | (((uint64)(blkno)) << 16) | ((uint64)(offset))) + +/* + * Per-relation UNDO record types + * + * These record the operations needed for MVCC visibility determination. + * Unlike cluster-wide UNDO (which stores complete tuples for rollback), + * per-relation UNDO stores only operation metadata. + */ +typedef enum RelUndoRecordType +{ + RELUNDO_INSERT = 1, /* Insertion record with TID range */ + RELUNDO_DELETE = 2, /* Deletion (batched up to 50 TIDs) */ + RELUNDO_UPDATE = 3, /* Update with old/new TID link */ + RELUNDO_TUPLE_LOCK = 4 /* SELECT FOR UPDATE/SHARE */ +} RelUndoRecordType; + +/* + * Common header for all per-relation UNDO records + * + * Every UNDO record starts with this fixed-size header, followed by + * type-specific payload data. + */ +typedef struct RelUndoRecordHeader +{ + uint16 urec_type; /* RelUndoRecordType */ + uint16 urec_len; /* Total length including header */ + TransactionId urec_xid; /* Creating transaction ID */ + RelUndoRecPtr urec_prevundorec; /* Previous record in chain */ + + /* Rollback support fields */ + uint16 info_flags; /* Information flags (see below) */ + uint16 tuple_len; /* Length of tuple data (0 if none) */ + /* Followed by type-specific payload + optional tuple data */ +} RelUndoRecordHeader; + +/* Size of the common UNDO record header */ +#define SizeOfRelUndoRecordHeader \ + sizeof(RelUndoRecordHeader) + +/* + * RelUndoRecordHeader info_flags values + * + * These flags indicate what additional data is stored with the UNDO record + * to support transaction rollback. + */ +#define RELUNDO_INFO_HAS_TUPLE 0x0001 /* Record contains complete tuple */ +#define RELUNDO_INFO_CLR_APPLIED 0x0004 /* CLR has been applied */ + +/* + * RELUNDO_INSERT payload + * + * Records insertion of a range of consecutive TIDs. + */ +typedef struct RelUndoInsertPayload +{ + ItemPointerData firsttid; /* First inserted TID */ + ItemPointerData endtid; /* Last inserted TID (inclusive) */ +} RelUndoInsertPayload; + +/* + * RELUNDO_DELETE payload + * + * Records deletion of up to 50 TIDs (batched for efficiency). + */ +#define RELUNDO_DELETE_MAX_TIDS 50 + +typedef struct RelUndoDeletePayload +{ + uint16 ntids; /* Number of TIDs in this record */ + ItemPointerData tids[RELUNDO_DELETE_MAX_TIDS]; +} RelUndoDeletePayload; + +/* + * RELUNDO_UPDATE payload + * + * Records update operation linking old and new tuple versions. + */ +typedef struct RelUndoUpdatePayload +{ + ItemPointerData oldtid; /* Old tuple TID */ + ItemPointerData newtid; /* New tuple TID */ + /* Optional: column bitmap for partial updates could be added here */ +} RelUndoUpdatePayload; + +/* + * RELUNDO_TUPLE_LOCK payload + * + * Records tuple lock (SELECT FOR UPDATE/SHARE). + */ +typedef struct RelUndoTupleLockPayload +{ + ItemPointerData tid; /* Locked tuple TID */ + uint16 lock_mode; /* LockTupleMode */ +} RelUndoTupleLockPayload; + +/* + * Per-relation UNDO metapage structure + * + * Stored at block 0 of the relation's UNDO fork. Tracks the head/tail + * of the UNDO page chain and the current generation counter. + * + * The metapage is the root of all per-relation UNDO state. It is read + * and updated during Reserve (to find the head page), Discard (to advance + * the tail), and Init (to set up an empty chain). All metapage modifications + * must be WAL-logged for crash safety. + * + * Memory layout is designed for 8-byte alignment of the 64-bit fields. + */ +/* + * Number of independent append points ("head slots") in a relation's UNDO + * fork. Every committed CAS UPDATE appends its before-image to the head page + * of one slot; a backend hashes to slot (MyProcNumber % RELUNDO_NUM_HEADS) so + * that concurrent writers contend on RELUNDO_NUM_HEADS distinct tail pages + * instead of one. + * + * Originally sized to match the common WAL's NUM_XLOGINSERT_LOCKS (8). + * Raised to 16 after Plan B EC2 benchmarking showed cached-tpcb throughput + * negative-scaling past ~64-96 concurrent writers on a 96-core host: at + * c=192, MyProcNumber % 8 puts ~24 backends on each slot's exclusive + * RelUndoReserve() buffer lock, cluster-wide (this is shared by every + * in-place-update table's writers, not scoped per relation). Doubling to 16 halves that + * per-slot contention (~12 backends/slot at c=192) at a bounded cost: + * RelUndoDiscard()/RelUndoDiscardSlot() walk all RELUNDO_NUM_HEADS slots + * under the metapage's exclusive lock on every VACUUM and on every + * RelUndoMaybeVacuum() throttled backstop call (relundo.c), so raising this + * further multiplies that discard-side cost without EC2-scale concurrency + * data to justify it; 16 was chosen as the largest change defensible from + * local (20-thread) validation alone. Revisit with real high-core-count A/B + * data before raising further. + * + * The per-txn rollback chain threads through urec_prevundorec and the reader + * keys off the physical (counter, blkno, offset) triple, so neither cares + * which slot a record landed on; striping is transparent to both. + * + * Changing this value changes the on-disk RelUndoMetaPageData layout + * (head_blkno[]/tail_blkno[] array size) -- see RELUNDO_METAPAGE_VERSION. + */ +#define RELUNDO_NUM_HEADS 16 + +typedef struct RelUndoMetaPageData +{ + uint32 magic; /* RELUNDO_METAPAGE_MAGIC: validates that + * block 0 is actually a metapage */ + uint16 version; /* Format version (currently 3); allows future + * on-disk format changes */ + uint16 counter; /* Current generation counter; incremented + * when starting a new batch of records. + * Embedded in RelUndoRecPtr for O(1) age + * comparison. Wraps at 65536. */ + BlockNumber head_blkno[RELUNDO_NUM_HEADS]; /* Newest UNDO page per slot + * (where new records are + * appended). + * InvalidBlockNumber if that + * slot's chain is empty. */ + BlockNumber tail_blkno[RELUNDO_NUM_HEADS]; /* Oldest UNDO page per slot + * (first to be discarded). + * InvalidBlockNumber if that + * slot's chain is empty. */ + BlockNumber free_blkno; /* Head of the free page list. Pages discarded + * by VACUUM are spliced here for reuse, + * avoiding fork extension. Shared across all + * slots. InvalidBlockNumber if no free pages. */ + uint64 total_records; /* Cumulative count of all UNDO records ever + * created (monotonically increasing) */ + uint64 discarded_records; /* Cumulative count of discarded records. + * (total - discarded) = live records. */ + BlockNumber system_alloc_watermark; /* High-water mark of system-allocated + * pages. Tracks the highest block + * number allocated via system + * transaction, enabling efficient + * reclamation of unused pages. */ +} RelUndoMetaPageData; + +typedef RelUndoMetaPageData *RelUndoMetaPage; + +/* Magic number for metapage validation */ +#define RELUNDO_METAPAGE_MAGIC 0x4F56554D /* "OVUM" */ + +/* + * Block number of the metapage within the UNDO fork. Data pages are block + * >= 1; a chain link is InvalidBlockNumber at the tail and must NEVER be this + * block. Chain walkers use it to avoid re-locking the metapage (which the + * caller typically already holds). + */ +#define RELUNDO_METAPAGE_BLKNO ((BlockNumber) 0) + +/* Current metapage format version */ +#define RELUNDO_METAPAGE_VERSION 4 + +/* + * Advance a freshly PageInit'd metapage's pd_lower to cover the + * RelUndoMetaPageData struct that lives in the page contents area. + * + * The metapage keeps all of its state in PageGetContents(page) rather than in + * line pointers, so a bare PageInit leaves pd_lower at the empty-page value and + * the entire meta struct sits inside the "hole" [pd_lower, pd_upper). A + * REGBUF_STANDARD full-page image elides that hole, so the FPI would carry a + * valid header and zeroed contents; a standby (or crash recovery) restoring it + * reconstructs a metapage with magic 0x0. Growing pd_lower past the struct + * makes the meta fields part of the FPI's recorded region. Mirrors nbtree's + * _bt_initmetapage. Call after PageInit and after populating the fields. + */ +static inline void +RelUndoMetaPageSetPdLower(Page page) +{ + ((PageHeader) page)->pd_lower = + ((char *) PageGetContents(page) + sizeof(RelUndoMetaPageData)) + - (char *) page; +} + +/* + * Per-relation UNDO data page header + * + * Each UNDO data page (block >= 1) starts with this header. + * Pages are linked in a singly-linked chain from head to tail via prev_blkno. + * + * Records are appended starting at pd_lower and grow toward pd_upper. + * Free space is [pd_lower, pd_upper). When pd_lower >= pd_upper, the page + * is full and a new page must be allocated. + * + * The max_xid field tracks the largest urec_xid of any record on the page. + * This drives page-granularity discard: a page is reclaimable once max_xid + * precedes the relation's oldest non-removable xid (oldest_xmin), since no + * active transaction can then need any record on the page for rollback. + * + * The counter field stamps the page with its generation at creation time. + * It is retained for record-pointer addressing (RelUndoRecPtr) but is no + * longer used for discard eligibility. + */ +typedef struct RelUndoPageHeaderData +{ + BlockNumber prev_blkno; /* Previous page in chain (toward tail). + * InvalidBlockNumber for the oldest page in + * the chain (the tail). */ + TransactionId max_xid; /* Largest urec_xid of any record on this + * page. InvalidTransactionId on an empty + * page. Used for discard eligibility checks. */ + uint16 counter; /* Generation counter at page creation. Used + * for record-pointer addressing. */ + uint16 pd_lower; /* Byte offset of next record insertion point + * (grows upward from header). */ + uint16 pd_upper; /* Byte offset of end of usable space + * (typically BLCKSZ). */ +} RelUndoPageHeaderData; + +typedef RelUndoPageHeaderData *RelUndoPageHeader; + +/* Size of UNDO page header */ +#define SizeOfRelUndoPageHeaderData (sizeof(RelUndoPageHeaderData)) + +/* Maximum free space in an UNDO data page */ +#define RelUndoPageMaxFreeSpace \ + (BLCKSZ - SizeOfRelUndoPageHeaderData) + +/* + * Internal page management functions (used by relundo.c and relundo_discard.c) + * ============================================================================= + */ + +/* Read and pin the metapage (block 0) of the UNDO fork */ +extern Buffer relundo_get_metapage(Relation rel, int mode); + +/* Allocate a new data page at the head of the given slot's chain */ +extern BlockNumber relundo_allocate_page(Relation rel, Buffer metabuf, + int slot, Buffer *newbuf); + +/* Initialize an UNDO data page */ +extern void relundo_init_page(Page page, BlockNumber prev_blkno, + uint16 counter); + +/* Get free space on an UNDO data page */ +extern Size relundo_get_free_space(Page page); + +/* + * Public API for table access methods + * ==================================== + */ + +/* + * RelUndoReserve - Reserve space for an UNDO record (Phase 1 of 2-phase insert) + * + * Reserves space in the relation's UNDO log and pins the buffer. The caller + * should then perform the DML operation, and finally call RelUndoFinish() to + * commit the UNDO record or RelUndoCancel() to release the reservation. + * + * Parameters: + * rel - Relation to insert UNDO record into + * record_size - Total size of UNDO record (header + payload) + * undo_buffer - (output) Buffer containing the reserved space + * + * Returns: + * RelUndoRecPtr pointing to the reserved space + * + * The returned buffer is pinned and locked (exclusive). Caller must eventually + * call RelUndoFinish() or RelUndoCancel(). + */ +extern RelUndoRecPtr RelUndoReserve(Relation rel, Size record_size, + Buffer *undo_buffer); + +/* + * RelUndoStageResult - facts produced by RelUndoStage() for a deferred WAL emit + * + * RelUndoStage() writes the UNDO record onto the reserved page and dirties the + * buffers but performs NO WAL insert. The caller then either lets RelUndoFinish + * emit the standalone RM_RELUNDO_ID record, or folds these staged bytes into a + * different resource manager's combined record (the caller's WAL-fold path). The undo and + * metapage buffers remain locked+pinned; the caller is responsible for + * PageSetLSN on them under its critical section and for releasing them. + */ +typedef struct RelUndoStageResult +{ + Buffer undo_buffer; /* reserved data-page buffer (still locked) */ + Buffer metabuf; /* metapage buffer if is_new_page, else + * Invalid */ + bool is_new_page; /* first record on a freshly allocated page */ + uint8 urec_type; /* header->urec_type (for the xlrec) */ + uint16 urec_len; /* header->urec_len (for the xlrec) */ + uint16 page_offset; /* page-absolute offset of the record */ + uint16 new_pd_lower; /* shadow pd_lower after the write */ + TransactionId max_xid; /* page max_xid watermark after the bump */ + char *wal_record_data; /* palloc'd block-0 data (caller pfrees) */ + Size wal_record_size; /* size of wal_record_data */ +} RelUndoStageResult; + +/* + * RelUndoStage - write an UNDO record onto its reserved page WITHOUT WAL. + * + * Performs every page mutation RelUndoFinish() does (header+payload memcpy, + * max_xid bump, MarkBufferDirty on the data page and, for a new page, the + * metapage) and builds the block-0 WAL data buffer, but does NOT open a + * critical section, XLogInsert, PageSetLSN, or release any buffer. The staged + * facts are returned in *result so the caller can emit the WAL record itself. + */ +extern void RelUndoStage(Relation rel, Buffer undo_buffer, RelUndoRecPtr ptr, + const RelUndoRecordHeader *header, + const void *payload, Size payload_size, + RelUndoStageResult *result); + +/* + * RelUndoFinish - Complete UNDO record insertion (Phase 2 of 2-phase insert) + * + * Writes the UNDO record to the previously reserved space and releases the buffer. + * This must be called after successful DML operation completion. + * + * Parameters: + * rel - Relation containing the UNDO log + * undo_buffer - Buffer from RelUndoReserve() (will be unlocked/unpinned) + * ptr - RelUndoRecPtr from RelUndoReserve() + * header - UNDO record header to write + * payload - UNDO record payload data + * payload_size - Size of payload data + * + * The buffer is marked dirty, WAL-logged, and released. + */ +extern void RelUndoFinish(Relation rel, Buffer undo_buffer, + RelUndoRecPtr ptr, + const RelUndoRecordHeader *header, + const void *payload, Size payload_size); + +/* + * RelUndoFinishWithTuple - Complete UNDO record insertion with tuple data + * + * Like RelUndoFinish(), but also writes tuple data after the payload for + * operations that need to store the complete tuple (DELETE, UPDATE). + * + * Parameters: + * rel - Relation containing the UNDO log + * undo_buffer - Buffer from RelUndoReserve() (will be unlocked/unpinned) + * ptr - RelUndoRecPtr from RelUndoReserve() + * header - UNDO record header (must have RELUNDO_INFO_HAS_TUPLE set) + * payload - UNDO record payload data + * payload_size - Size of payload data + * tuple_data - Complete tuple data to store + * tuple_len - Length of tuple data + * + * The record layout on the UNDO page is: + * [RelUndoRecordHeader][payload][tuple_data] + */ +extern void RelUndoFinishWithTuple(Relation rel, Buffer undo_buffer, + RelUndoRecPtr ptr, + const RelUndoRecordHeader *header, + const void *payload, Size payload_size, + const char *tuple_data, uint32 tuple_len); + +/* + * RelUndoCancel - Cancel UNDO record reservation + * + * Releases a reservation made by RelUndoReserve() without writing an UNDO record. + * Use this when the DML operation fails and needs to be rolled back. + * + * Parameters: + * rel - Relation containing the UNDO log + * undo_buffer - Buffer from RelUndoReserve() (will be unlocked/unpinned) + * ptr - RelUndoRecPtr from RelUndoReserve() + * + * The reserved space is left as a "hole" that can be skipped during chain walking. + */ +extern void RelUndoCancel(Relation rel, Buffer undo_buffer, RelUndoRecPtr ptr); + +/* + * RelUndoReadRecord - Read an UNDO record + * + * Reads an UNDO record at the specified pointer and returns the header and payload. + * + * Parameters: + * rel - Relation containing the UNDO log + * ptr - RelUndoRecPtr to read from + * header - (output) UNDO record header + * payload - (output) Allocated payload buffer (caller must pfree) + * payload_size - (output) Size of payload + * + * Returns: + * true if record was successfully read, false if pointer is invalid or + * record has been discarded + * + * If successful, *payload is allocated in CurrentMemoryContext and must be + * freed by the caller. + */ +extern bool RelUndoReadRecord(Relation rel, RelUndoRecPtr ptr, + RelUndoRecordHeader *header, + void **payload, Size *payload_size); + +/* + * RelUndoReadRecordHeader - Read only the header of an UNDO record. + * + * Header-only variant that skips the payload palloc. Used by hot probes + * that need only urec_xid (e.g. the lost-update conflict probe). Returns + * false with the same semantics as RelUndoReadRecord. + */ +extern bool RelUndoReadRecordHeader(Relation rel, RelUndoRecPtr ptr, + RelUndoRecordHeader *header); + +/* + * RelUndoDiscard - Discard old UNDO records + * + * Frees space occupied by UNDO records that no active transaction can still + * need for rollback. Called during VACUUM to reclaim space. + * + * Parameters: + * rel - Relation to discard UNDO from + * oldest_xmin - Oldest non-removable XID for this relation + * + * A page is discardable iff its max_xid precedes oldest_xmin, meaning every + * record on the page belongs to a transaction that has already committed or + * aborted and is older than any active snapshot. + */ +extern void RelUndoDiscard(Relation rel, TransactionId oldest_xmin, bool nowait); + +/* + * RelUndoHeadCacheInvalidate - drop the per-backend head page cache entry. + * + * Must be called by RelUndoDiscard() after it reclaims pages, since discard + * can physically truncate the fork and leave the cached head block number + * pointing past the new end of file. + */ +extern void RelUndoHeadCacheInvalidate(Oid relid); + +/* + * RelUndoInitRelation - Initialize per-relation UNDO for a new relation + * + * Creates the UNDO fork and initializes the metapage. Called during CREATE TABLE + * for table AMs that use per-relation UNDO. + * + * Parameters: + * rel - Relation to initialize + */ +extern void RelUndoInitRelation(Relation rel); + +/* + * RelUndoDropRelation - Drop per-relation UNDO when relation is dropped + * + * Removes the UNDO fork. Called during DROP TABLE for table AMs that use + * per-relation UNDO. + * + * Parameters: + * rel - Relation being dropped + */ +extern void RelUndoDropRelation(Relation rel); + +/* + * RelUndoVacuum - Vacuum per-relation UNDO log + * + * Performs maintenance on the UNDO log: discards old records, reclaims space, + * and updates statistics. Called during VACUUM. + * + * Parameters: + * rel - Relation to vacuum + * oldest_xmin - Oldest XID still visible to any transaction + */ +extern void RelUndoVacuum(Relation rel, TransactionId oldest_xmin, bool nowait); + +/* + * RelUndoMaybeVacuum - Throttled, self-clocking per-relation UNDO fork + * discard, decoupled from VACUUM. + * + * An in-place-update AM can correctly report near-zero dead tuples, + * so autovacuum's dead-tuple/insert-count thresholds may never fire even + * under sustained write churn, and RelUndoVacuum() above -- the only code + * that discards the on-disk UNDO fork -- is normally reached exclusively via + * VACUUM. Call this from the owning AM's DML hot path (after releasing all + * buffer/tuple locks -- see relundo.c for why) as a backstop so the fork + * gets discarded on its own schedule. A no-op most calls (throttled to once + * every 5 seconds per backend, and skipped entirely below a minimum fork + * size), so it is safe to call unconditionally on every DML operation. + * + * Parameters: + * rel - Relation whose UNDO fork may need discarding + */ +extern void RelUndoMaybeVacuum(Relation rel); + +/* + * ============================================================================= + * ROLLBACK API - Support for transaction abort via UNDO application + * ============================================================================= + */ + +/* + * RelUndoApplyChain - Walk and apply per-relation UNDO chain for rollback + * + * Walks backwards through the UNDO chain applying each operation to restore + * the database state. Called during transaction abort. + */ +extern void RelUndoApplyChain(Relation rel, RelUndoRecPtr start_ptr); + +/* + * RelUndoApplyRecordForRecovery - Reverse-apply one UNDO record during crash + * recovery, restoring the before-image in place without writing a CLR. + */ +extern void RelUndoApplyRecordForRecovery(Relation rel, RelUndoRecPtr ptr); + +/* Read UNDO record including tuple data for rollback */ +extern RelUndoRecordHeader *RelUndoReadRecordWithTuple(Relation rel, + RelUndoRecPtr ptr, + char **tuple_data_out, + uint32 *tuple_len_out); + +/* + * ============================================================================= + * CRASH-RECOVERY API - Reverse-apply per-relation UNDO for loser transactions + * ============================================================================= + * + * An in-place MVCC table AM overwrites the committed tuple + * bytes on UPDATE, and the only durable copy of the prior committed version + * is the before-image in the relation's UNDO fork. If a transaction does an + * in-place modification and the server crashes before it commits, redo + * re-establishes the uncommitted page state, so a recovery-time driver must + * reverse-apply the before-images of every incomplete (loser) transaction. + * + * After the redo pass finishes, CLOG is fully reconstructed, so + * PerformRelUndoRecovery() scans every per-relation UNDO fork on disk and, + * for each record whose creating transaction did not commit and is not a + * prepared transaction, reverse-applies its before-image. See + * relundo_recovery.c for why an end-of-recovery fork scan is required rather + * than tracking insertions during redo. + */ + +/* Scan UNDO forks and reverse-apply all loser transactions' before-images. */ +extern void PerformRelUndoRecovery(void); + +/* + * ============================================================================= + * AM-NEUTRAL HOOKS - decouple UNDO core from any specific table AM + * ============================================================================= + * + * The per-relation UNDO core (slog.c, relundo_apply.c, undoworker.c) is + * format-agnostic: it stores and restores opaque tuple bytes without knowing + * the in-place table AM's tuple layout. The operations that do require + * AM-specific knowledge are reached through these function pointers, which the + * owning table AM installs at subsystem init. When no in-place AM is + * registered the pointers stay NULL and the core degrades gracefully (no + * transient-flag clearing). + * + * RelUndoClearTransientFlags_hook: clear the AM's per-tuple transient state + * bits (uncommitted/deleted/updated) on a freshly restored before-image, so + * the restored committed version is not mistaken for an in-flight change. + * + * RelUndoAbortCleanup_hook: called after a rolled-back transaction's + * before-images have been physically restored (inline in xactundo.c, or by + * the background worker in relundo_worker.c), so the AM can drop any + * transient bookkeeping entries it kept ONLY for as long as the abort was + * pending (e.g. dirty-xid write-write-conflict markers). xid is the + * transaction that was rolled back. NULL is a valid no-op for an AM with + * no such transient state. + * + * RelUndoDiscardRetained_hook: called periodically by the UNDO discard worker + * (undoworker.c) so the AM can reclaim any retained-version bookkeeping + * (before-images, dirty markers) that has aged out. The reclamation + * horizon is the AM's own xid horizon. NULL is a valid no-op. + */ +extern void (*RelUndoClearTransientFlags_hook) (char *tuple_data); +extern void (*RelUndoAbortCleanup_hook) (TransactionId xid); +extern void (*RelUndoDiscardRetained_hook) (void); + +#endif /* RELUNDO_H */ diff --git a/src/include/access/relundo_worker.h b/src/include/access/relundo_worker.h new file mode 100644 index 0000000000000..aac33596681e1 --- /dev/null +++ b/src/include/access/relundo_worker.h @@ -0,0 +1,89 @@ +/*------------------------------------------------------------------------- + * + * relundo_worker.h + * Background worker for applying per-relation UNDO records asynchronously + * + * This module implements background workers that apply per-relation UNDO + * records for aborted transactions. The workers run asynchronously, similar + * to autovacuum, to avoid blocking ROLLBACK commands. + * + * Architecture: + * - Main launcher process manages worker pool + * - Individual workers process UNDO chains for specific databases + * - Shared memory queue tracks pending UNDO work + * - Workers coordinate to avoid duplicate work + * + * This follows the ZHeap architecture where UNDO application is deferred + * to background processes rather than being synchronous during ROLLBACK. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/access/relundo_worker.h + * + *------------------------------------------------------------------------- + */ +#ifndef RELUNDO_WORKER_H +#define RELUNDO_WORKER_H + +#include "postgres.h" +#include "access/relundo.h" +#include "datatype/timestamp.h" +#include "storage/lwlock.h" + +/* + * Shared memory structure for UNDO work queue + */ +/* + * MAX_UNDO_WORK_ITEMS limits the in-flight UNDO work queue. + * Keep moderate (64) so shmem stays small during bootstrap. + * Production workloads rarely exceed this with synchronous abort. + */ +#define MAX_UNDO_WORK_ITEMS 64 + +typedef struct RelUndoWorkItem +{ + Oid dboid; /* Database OID */ + Oid reloid; /* Relation OID */ + RelUndoRecPtr start_urec_ptr; /* First UNDO record to apply */ + TransactionId xid; /* Transaction that created the UNDO */ + TimestampTz queued_at; /* When this was queued */ + bool in_progress; /* Worker currently processing this */ + int worker_id; /* ID of worker processing (if in_progress) */ +} RelUndoWorkItem; + +typedef struct RelUndoWorkQueue +{ + LWLock lock; /* Protects the queue */ + int num_items; /* Number of pending items */ + int next_worker_id; /* For assigning worker IDs */ + RelUndoWorkItem items[MAX_UNDO_WORK_ITEMS]; +} RelUndoWorkQueue; + +/* + * Worker registration and lifecycle + */ +extern Size RelUndoWorkerShmemSize(void); +extern void RelUndoWorkerShmemInit(void); +extern void RelUndoLauncherMain(Datum main_arg); +extern void RelUndoWorkerMain(Datum main_arg); + +/* + * Work queue operations + */ +extern void RelUndoQueueAdd(Oid dboid, Oid reloid, RelUndoRecPtr start_urec_ptr, + TransactionId xid); +extern bool RelUndoQueueGetNext(RelUndoWorkItem *item_out, int worker_id); +extern void RelUndoQueueMarkComplete(Oid dboid, Oid reloid, int worker_id); + +/* + * Worker management + */ +extern void StartRelUndoWorker(Oid dboid); +extern void WaitForPendingRelUndo(void); + +/* GUC parameters */ +extern int max_relundo_workers; +extern int relundo_worker_naptime; + +#endif /* RELUNDO_WORKER_H */ diff --git a/src/include/access/relundo_xlog.h b/src/include/access/relundo_xlog.h new file mode 100644 index 0000000000000..5aedc1a53abcb --- /dev/null +++ b/src/include/access/relundo_xlog.h @@ -0,0 +1,164 @@ +/*------------------------------------------------------------------------- + * + * relundo_xlog.h + * Per-relation UNDO WAL record definitions + * + * This file contains the WAL record format definitions for per-relation + * UNDO operations. These records are logged by the RM_RELUNDO_ID resource + * manager. + * + * Record types: + * XLOG_RELUNDO_INIT - Metapage initialization + * XLOG_RELUNDO_INSERT - UNDO record insertion into a data page + * XLOG_RELUNDO_DISCARD - Discard old UNDO pages during VACUUM + * + * Per-relation UNDO stores operation metadata for MVCC visibility in + * each relation's UNDO fork. This is distinct from the cluster-wide + * UNDO system (RM_UNDO_ID) which handles transaction rollback. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/access/relundo_xlog.h + * + *------------------------------------------------------------------------- + */ +#ifndef RELUNDO_XLOG_H +#define RELUNDO_XLOG_H + +#include "postgres.h" + +#include "access/xlogreader.h" +#include "lib/stringinfo.h" +#include "storage/block.h" +#include "storage/relfilelocator.h" + +/* Forward declaration - full definition in relundo.h */ +typedef uint64 RelUndoRecPtr; + +/* + * WAL record types for per-relation UNDO operations + * + * The high 4 bits of the info byte encode the operation type, + * following PostgreSQL convention. + */ +#define XLOG_RELUNDO_INIT 0x00 /* Metapage initialization */ +#define XLOG_RELUNDO_INSERT 0x10 /* UNDO record insertion */ +#define XLOG_RELUNDO_DISCARD 0x20 /* Discard old UNDO pages */ +#define XLOG_RELUNDO_TRUNCATE 0x30 /* Physically truncate the fork */ +#define XLOG_RELUNDO_APPLY 0x40 /* Apply UNDO for rollback (CLR) */ + +/* + * Flag: set when the data page being inserted into is newly initialized + * (first tuple on the page). When set, redo will re-initialize the + * page from scratch before applying the insert. + */ +#define XLOG_RELUNDO_INIT_PAGE 0x80 + +/* + * xl_relundo_init - WAL record for metapage initialization + * + * Logged when RelUndoInitRelation() creates the UNDO fork and writes + * the initial metapage (block 0). + * + * Backup block 0: the metapage + */ +typedef struct xl_relundo_init +{ + uint32 magic; /* RELUNDO_METAPAGE_MAGIC */ + uint16 version; /* Format version */ + uint16 counter; /* Initial generation counter */ +} xl_relundo_init; + +#define SizeOfRelundoInit (offsetof(xl_relundo_init, counter) + sizeof(uint16)) + +/* + * xl_relundo_insert - WAL record for UNDO record insertion + * + * Logged when RelUndoFinish() writes an UNDO record to a data page. + * + * Backup block 0: the data page receiving the UNDO record + * Backup block 1: the metapage (if head_blkno was updated) + * + * The actual UNDO record data is stored as block data associated with + * backup block 0 (via XLogRegisterBufData). + */ +typedef struct xl_relundo_insert +{ + uint16 urec_type; /* RelUndoRecordType of the UNDO record */ + uint16 urec_len; /* Total length of UNDO record */ + uint16 page_offset; /* Byte offset within page where record starts */ + uint16 new_pd_lower; /* Updated pd_lower after insertion */ + TransactionId max_xid; /* Updated page max_xid watermark after insert */ +} xl_relundo_insert; + +#define SizeOfRelundoInsert (offsetof(xl_relundo_insert, max_xid) + sizeof(TransactionId)) + +/* + * xl_relundo_discard - WAL record for UNDO page discard + * + * Logged when RelUndoDiscard() reclaims a contiguous run of discardable + * pages from the tail of the data chain by splicing the whole run directly + * onto the metapage's free list. Only the run boundaries change, so the + * record covers a fixed set of buffers regardless of run length: + * + * Backup block 0: the metapage (tail + free-list head) + * Backup block 1: the run's old-tail page (prev_blkno -> old free head) + * Backup block 2: the new live tail page (prev_blkno -> Invalid) + */ +typedef struct xl_relundo_discard +{ + BlockNumber old_tail_blkno; /* Old chain tail (run's tail), block 1 */ + BlockNumber new_tail_blkno; /* New chain tail after discard */ + BlockNumber free_head_blkno; /* New free-list head (run's head) */ + BlockNumber old_free_head; /* Prior free-list head, written to block 1 */ + TransactionId discard_xid; /* oldest_xmin cutoff used for discard */ + uint32 npages_freed; /* Number of pages spliced onto free list */ + uint16 slot; /* Head slot whose chain was discarded */ +} xl_relundo_discard; + +#define SizeOfRelundoDiscard (offsetof(xl_relundo_discard, slot) + sizeof(uint16)) + +/* + * xl_relundo_truncate - WAL record for physical fork truncation + * + * Logged when RelUndoDiscard() empties the entire data chain. At that + * point the free list holds every allocated data block, i.e. the + * contiguous physical suffix [1 .. system_alloc_watermark], so the fork + * can be physically truncated back to just the metapage (block 0). Redo + * resets the metapage free-list/watermark fields and truncates the fork. + * + * Backup block 0: the metapage (free_blkno + watermark reset) + */ +typedef struct xl_relundo_truncate +{ + BlockNumber new_nblocks; /* New fork length in blocks (always 1) */ +} xl_relundo_truncate; + +#define SizeOfRelundoTruncate (offsetof(xl_relundo_truncate, new_nblocks) + sizeof(BlockNumber)) + +/* Resource manager functions */ +extern void relundo_redo(XLogReaderState *record); +extern void relundo_desc(StringInfo buf, XLogReaderState *record); +extern const char *relundo_identify(uint8 info); + +/* Parallel redo support */ +extern void relundo_startup(void); +extern void relundo_cleanup(void); +extern void relundo_mask(char *pagedata, BlockNumber blkno); + +/* + * XLOG_RELUNDO_APPLY - Compensation Log Record for UNDO application + * + * Records that we've applied an UNDO operation during transaction rollback. + * Prevents double-application if we crash during rollback. + */ +typedef struct xl_relundo_apply +{ + RelUndoRecPtr urec_ptr; /* UNDO record that was applied */ + RelFileLocator target_reloc; /* Target relation */ +} xl_relundo_apply; + +#define SizeOfRelUndoApply (offsetof(xl_relundo_apply, target_reloc) + sizeof(RelFileLocator)) + +#endif /* RELUNDO_XLOG_H */ diff --git a/src/include/access/rmgrlist.h b/src/include/access/rmgrlist.h index ae32ef16d67b6..b21f4fa30aa5f 100644 --- a/src/include/access/rmgrlist.h +++ b/src/include/access/rmgrlist.h @@ -48,3 +48,6 @@ PG_RMGR(RM_REPLORIGIN_ID, "ReplicationOrigin", replorigin_redo, replorigin_desc, PG_RMGR(RM_GENERIC_ID, "Generic", generic_redo, generic_desc, generic_identify, NULL, NULL, generic_mask, NULL) PG_RMGR(RM_LOGICALMSG_ID, "LogicalMessage", logicalmsg_redo, logicalmsg_desc, logicalmsg_identify, NULL, NULL, NULL, logicalmsg_decode) PG_RMGR(RM_XLOG2_ID, "XLOG2", xlog2_redo, xlog2_desc, xlog2_identify, NULL, NULL, NULL, xlog2_decode) +PG_RMGR(RM_UNDO_ID, "Undo", undo_redo, undo_desc, undo_identify, NULL, NULL, NULL, NULL) +PG_RMGR(RM_ATM_ID, "ATM", atm_redo, atm_desc, atm_identify, NULL, NULL, NULL, NULL) +PG_RMGR(RM_RELUNDO_ID, "RelUndo", relundo_redo, relundo_desc, relundo_identify, relundo_startup, relundo_cleanup, relundo_mask, NULL) diff --git a/src/include/access/slog.h b/src/include/access/slog.h new file mode 100644 index 0000000000000..6fe4d64c5fc5e --- /dev/null +++ b/src/include/access/slog.h @@ -0,0 +1,103 @@ +/*------------------------------------------------------------------------- + * + * slog.h + * Secondary Log (sLog) for shared-memory tracking + * + * The sLog provides a shared-memory Aborted Transaction Map (ATM) for the + * UNDO subsystem's Constant-Time Recovery: + * + * Transaction radix tree - Aborted transaction entries keyed by + * (xid, reloid) packed into a uint64, ordered for efficient xid-based + * range operations. Protected by a single LWLock (modifications are + * infrequent). + * + * An optional per-tuple flat-hash extension (the tuple sLog) adds + * bounded-recovery uncommitted-writer tracking for in-place-update table + * AMs. It shares this subsystem's shared-memory segment and initialization + * but is not required by the UNDO core; its API is declared where it is + * defined. + * + * WAL: Transaction sLog reuses existing RM_ATM_ID records. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/access/slog.h + * + *------------------------------------------------------------------------- + */ +#ifndef SLOG_H +#define SLOG_H + +#include "access/transam.h" +#include "access/xlogdefs.h" +#include "datatype/timestamp.h" +#include "storage/lwlock.h" +#include "utils/dsa.h" + +/* ---------------------------------------------------------------- + * Transaction sLog structures + * + * SLogTxnEntry is used as the public output type for lookups. + * Internally, the ATM radix tree stores only the data fields; the key + * (xid, reloid) is implicit in the tree path. + * ---------------------------------------------------------------- + */ + +/* + * SLogTxnEntry - Public output structure for transaction lookups. + * Callers receive copies of this via SLogTxnLookup(). + */ +typedef struct SLogTxnEntry +{ + TransactionId xid; + Oid reloid; + XLogRecPtr last_batch_lsn; /* LSN of last UNDO batch for this xid */ + Oid dboid; /* database OID */ + TimestampTz abort_time; /* when transaction aborted */ + bool revert_complete; /* has Logical Revert finished? */ +} SLogTxnEntry; + +/* ---------------------------------------------------------------- + * Shared state + * + * The transaction radix tree is allocated in shared memory; its internal + * structures are opaque to callers. The SLogSharedState is defined in + * slog.c. + * ---------------------------------------------------------------- + */ + +/* ---------------------------------------------------------------- + * API: Shared memory + * ---------------------------------------------------------------- + */ +extern Size SLogShmemSize(void); +extern void SLogShmemRequest(void); +extern void SLogShmemInit(void); + +/* ---------------------------------------------------------------- + * API: Transaction sLog + * ---------------------------------------------------------------- + */ +extern bool SLogTxnInsert(TransactionId xid, Oid reloid, Oid dboid, + XLogRecPtr last_batch_lsn); +extern bool SLogTxnLookup(TransactionId xid, Oid reloid, + SLogTxnEntry *entry_out); +extern bool SLogTxnLookupByXid(TransactionId xid, XLogRecPtr *lsn_out); +extern void SLogTxnRemove(TransactionId xid, Oid reloid); +extern void SLogTxnRemoveByXid(TransactionId xid); +extern void SLogTxnMarkReverted(TransactionId xid); +extern bool SLogTxnGetNextUnreverted(TransactionId *xid_out, Oid *dboid_out, + XLogRecPtr *lsn_out); +extern int SLogTxnCollectUnrevertedDatabases(Oid *dboids, int max_dboids); +extern XLogRecPtr SLogTxnGetOldestUnrevertedLSN(void); +extern int SLogTxnSnapshotForCheckpoint(SLogTxnEntry **entries_out); +extern void SLogRecoveryFinalize(int *total_out, int *unreverted_out); + +/* DSA lifecycle (shared-memory area backing the aborted-txn radix tree) */ +extern void SLogEnsureDsaAttached(void); + +/* GUC: maximum DSA size (in MB) */ +extern int slog_dsa_max_size_mb; + +#endif /* SLOG_H */ diff --git a/src/include/access/twophase.h b/src/include/access/twophase.h index 1d2ff42c9b72f..3714ea62b3385 100644 --- a/src/include/access/twophase.h +++ b/src/include/access/twophase.h @@ -48,6 +48,7 @@ extern GlobalTransaction MarkAsPreparing(FullTransactionId fxid, const char *gid extern void StartPrepare(GlobalTransaction gxact); extern void EndPrepare(GlobalTransaction gxact); extern bool StandbyTransactionIdIsPrepared(TransactionId xid); +extern bool RecoveryTransactionIdIsPrepared(TransactionId xid); extern TransactionId PrescanPreparedTransactions(TransactionId **xids_p, int *nxids_p); @@ -72,4 +73,6 @@ extern bool LookupGXactBySubid(Oid subid); extern TransactionId TwoPhaseGetOldestXidInCommit(void); +extern XLogRecPtr TwoPhaseGetOldestUndoBatchLSN(void); + #endif /* TWOPHASE_H */ diff --git a/src/include/access/undo.h b/src/include/access/undo.h new file mode 100644 index 0000000000000..d258c804e0151 --- /dev/null +++ b/src/include/access/undo.h @@ -0,0 +1,52 @@ +/*------------------------------------------------------------------------- + * + * undo.h + * Common undo layer interface + * + * The undo subsystem consists of several logically separate subsystems + * that work together: + * + * undolog.c - Undo log file management and space allocation + * undorecord.c - Record format, serialization, and UndoRecordSet + * xactundo.c - Per-transaction record set management + * undoapply.c - Physical undo application during rollback + * undoworker.c - Background discard worker + * undo_bufmgr.c - Buffer management via shared_buffers + * undo_xlog.c - WAL redo routines + * + * This header provides the unified entry points for shared memory + * initialization and startup/shutdown coordination across all undo + * subsystems. The design follows the EDB undo-record-set branch + * pattern where UndoShmemSize()/UndoShmemInit() aggregate the + * requirements of all subsystems. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/access/undo.h + * + *------------------------------------------------------------------------- + */ +#ifndef UNDO_H +#define UNDO_H + +#include "access/undodefs.h" +#include "utils/palloc.h" + +/* + * Unified shared memory initialization. + * + * UndoShmemSize() computes the total shared memory needed by all undo + * subsystems. UndoShmemInit() initializes all undo shared memory + * structures. These are called from ipci.c during postmaster startup. + */ +extern Size UndoShmemSize(void); +extern void UndoShmemInit(void); + +/* Per-backend initialization */ +extern void InitializeUndo(void); + +/* Memory context for undo-related allocations */ +extern MemoryContext UndoContext; + +#endif /* UNDO_H */ diff --git a/src/include/access/undo_bufmgr.h b/src/include/access/undo_bufmgr.h new file mode 100644 index 0000000000000..b0c3736b73122 --- /dev/null +++ b/src/include/access/undo_bufmgr.h @@ -0,0 +1,297 @@ +/*------------------------------------------------------------------------- + * + * undo_bufmgr.h + * UNDO log buffer management and file layout definitions + * + * UNDO-in-WAL architecture: + * + * - UNDO records are embedded in the WAL stream as XLOG_UNDO_BATCH + * records. There are no separate UNDO segment files. + * - Reads: UndoReadBatchFromWAL() reads UNDO batches from WAL via + * XLogReader (for rollback chain traversal). + * - Sync: WAL flush handles durability (standard XLogFlush path). + * - Retention: undo_discard_horizon prevents WAL recycling past + * oldest needed UNDO batch. + * + * This module retains virtual RelFileLocator mapping for: + * - Buffer invalidation during segment discard (InvalidateUndoBuffers) + * - Legacy backward compatibility + * + * Each undo log is mapped to a virtual relation: + * RelFileLocator = { + * spcOid = UNDO_DEFAULT_TABLESPACE_OID (pg_default, 1663) + * dbOid = UNDO_DB_OID (pseudo-database 9) + * relNumber = log_number (undo log number as RelFileNumber) + * } + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/access/undo_bufmgr.h + * + *------------------------------------------------------------------------- + */ +#ifndef UNDO_BUFMGR_H +#define UNDO_BUFMGR_H + +#include "storage/block.h" +#include "storage/buf.h" +#include "storage/bufmgr.h" +#include "storage/relfilelocator.h" + +/* + * Pseudo-database OID used for undo log relations in the buffer pool. + * This matches ZHeap's UndoLogDatabaseOid convention. This OID must not + * collide with any real database OID; value 9 is reserved for this purpose. + */ +#define UNDO_DB_OID 9 + +/* + * Default tablespace OID for undo log buffers. This matches the + * pg_default tablespace (OID 1663 from pg_tablespace.dat). + * Eventually per-tablespace undo logs may be supported, but for now + * all undo data uses the default tablespace. + */ +#define UNDO_DEFAULT_TABLESPACE_OID 1663 + +/* + * Fork number used for undo log buffers in the shared buffer pool. + * + * Following ZHeap's convention (UndoLogForkNum = MAIN_FORKNUM), we use + * MAIN_FORKNUM for undo log buffer operations. Undo buffers are + * distinguished from regular relation data by the UNDO_DB_OID in the + * dbOid field of the BufferTag, not by a special fork number. + * + * Using MAIN_FORKNUM is necessary because the smgr layer sizes internal + * arrays to MAX_FORKNUM+1 entries. A fork number beyond that range + * would cause out-of-bounds accesses in smgr_cached_nblocks[] and + * similar arrays. + */ +#define UndoLogForkNum MAIN_FORKNUM + +/* + * UNDO_FORKNUM is reserved for future use when the smgr layer is + * extended to support undo-specific file management (Task #5). + * It is defined in buf_internals.h as a constant but not currently + * used in buffer operations. + */ + + +/* ---------------------------------------------------------------- + * Undo log to RelFileLocator mapping + * ---------------------------------------------------------------- + */ + +/* + * UndoLogGetRelFileLocator + * Build a virtual RelFileLocator for an undo log number. + * + * This mapping allows the standard buffer manager to identify undo log + * blocks using its existing BufferTag infrastructure. The resulting + * RelFileLocator does not correspond to any entry in pg_class; it is + * purely a buffer-pool-internal identifier. + * + * Parameters: + * log_number - the undo log number (0..16M) + * rlocator - output RelFileLocator to populate + */ +static inline void +UndoLogGetRelFileLocator(uint32 log_number, RelFileLocator *rlocator) +{ + rlocator->spcOid = UNDO_DEFAULT_TABLESPACE_OID; + rlocator->dbOid = UNDO_DB_OID; + rlocator->relNumber = (RelFileNumber) log_number; +} + +/* + * IsUndoRelFileLocator + * Check whether a RelFileLocator refers to an undo log. + * + * This is useful for code that needs to distinguish undo log locators + * from regular relation locators (e.g., in smgr dispatch, checkpoint + * logic, or buffer tag inspection). + */ +static inline bool +IsUndoRelFileLocator(const RelFileLocator *rlocator) +{ + return (rlocator->dbOid == UNDO_DB_OID); +} + +/* + * UNDO file layout: append-only + * + * UNDO log files use an append-only layout with NO PageHeaderData overhead. + * The logical byte offset in UndoRecPtr maps directly to the physical file + * offset. This eliminates the overhead of page headers, pd_lower tracking, + * LSN management, and full-page images for UNDO data. + * + * UNDO data is written via pwrite() and read via pread(), bypassing + * shared_buffers entirely for the write path. For reads, hot data is + * served from the kernel page cache (no I/O), while cold data requires + * sequential I/O on the pre-allocated file. + * + * The buffer pool integration (ReadUndoBuffer etc.) is retained only for + * the buffer invalidation API used during segment discard. + */ + +/* + * UndoRecPtrGetFileOffset + * Compute the physical file offset for an undo log logical byte offset. + * + * With the append-only layout, the logical offset IS the file offset. + */ +#define UndoRecPtrGetFileOffset(offset) ((uint64) (offset)) + +/* + * Legacy page-layout macros (retained for undo_bufmgr.c invalidation API). + * + * These are used only by buffer invalidation during discard, not by the + * write/read paths. The "block number" is conceptual, mapping the + * contiguous byte stream to BLCKSZ-aligned regions. + */ +#define UNDO_USABLE_BYTES_PER_PAGE BLCKSZ + +#define UndoRecPtrGetBlockNum(offset) \ + ((BlockNumber) ((offset) / BLCKSZ)) + +#define UndoRecPtrGetPageOffset(offset) \ + ((uint32) ((offset) % BLCKSZ)) + +/* + * UndoLogicalToFileSize + * Compute the physical file size needed for a given logical byte count. + * + * With append-only layout, physical size equals logical size (no headers). + * We round up to BLCKSZ alignment for pre-allocation. + */ +#define UndoLogicalToFileSize(logical_size) \ + ((uint64) (((logical_size) + BLCKSZ - 1) / BLCKSZ) * BLCKSZ) + + +/* ---------------------------------------------------------------- + * Buffer read/release API + * ---------------------------------------------------------------- + */ + +/* + * ReadUndoBuffer + * Read an undo log block into the shared buffer pool. + * + * This is the primary entry point for reading undo data. It translates + * the undo log number and block number into a virtual RelFileLocator and + * calls ReadBufferWithoutRelcache() to obtain a shared buffer. + * + * The returned Buffer must be released with ReleaseUndoBuffer() when the + * caller is done. The caller may also need to lock the buffer (via + * LockBuffer) depending on the access pattern. + * + * Parameters: + * log_number - undo log number + * block_number - block within the undo log + * mode - RBM_NORMAL, RBM_ZERO_AND_LOCK, etc. + * + * Returns: a valid Buffer handle. + */ +extern Buffer ReadUndoBuffer(uint32 log_number, BlockNumber block_number, + ReadBufferMode mode); + +/* + * ReadUndoBufferExtended + * Like ReadUndoBuffer but with explicit strategy control. + * + * Allows the caller to specify a buffer access strategy (e.g., for + * sequential undo log scans during discard or recovery). + */ +extern Buffer ReadUndoBufferExtended(uint32 log_number, + BlockNumber block_number, + ReadBufferMode mode, + BufferAccessStrategy strategy); + +/* + * ReleaseUndoBuffer + * Release a previously read undo buffer. + * + * This is a thin wrapper around ReleaseBuffer() for API symmetry. + * If the buffer was locked, it must be unlocked first (or use + * UnlockReleaseUndoBuffer). + */ +extern void ReleaseUndoBuffer(Buffer buffer); + +/* + * UnlockReleaseUndoBuffer + * Unlock and release an undo buffer in one call. + */ +extern void UnlockReleaseUndoBuffer(Buffer buffer); + +/* + * MarkUndoBufferDirty + * Mark an undo buffer as dirty. + * + * This is a thin wrapper around MarkBufferDirty() for API consistency. + */ +extern void MarkUndoBufferDirty(Buffer buffer); + + +/* ---------------------------------------------------------------- + * Buffer tag construction (requires buf_internals.h) + * ---------------------------------------------------------------- + */ + +/* + * UndoMakeBufferTag + * Initialize a BufferTag for an undo log block. + * + * This constructs the BufferTag that the shared buffer manager will use + * to identify this undo block in its hash table. It uses the virtual + * RelFileLocator mapping and UndoLogForkNum. + * + * Callers must include storage/buf_internals.h before this header to + * make these declarations visible. + */ +#ifdef BUFMGR_INTERNALS_H +extern void UndoMakeBufferTag(BufferTag *tag, uint32 log_number, + BlockNumber block_number); + +/* + * IsUndoBufferTag + * Check whether a BufferTag refers to an undo log buffer. + * + * Undo buffers are identified by the UNDO_DB_OID in the dbOid field + * of the buffer tag. + */ +static inline bool +IsUndoBufferTag(const BufferTag *tag) +{ + return (tag->dbOid == UNDO_DB_OID); +} +#endif /* BUFMGR_INTERNALS_H */ + + +/* ---------------------------------------------------------------- + * Invalidation + * ---------------------------------------------------------------- + */ + +/* + * InvalidateUndoBuffers + * Drop all shared buffers for a given undo log. + * + * Called when an undo log is discarded to remove stale entries from + * the shared buffer pool. This is analogous to DropRelationBuffers() + * for regular relations. + */ +extern void InvalidateUndoBuffers(uint32 log_number); + +/* + * InvalidateUndoBufferRange + * Drop shared buffers for a range of blocks in an undo log. + * + * Called during undo log truncation/discard to invalidate only the + * blocks that are being reclaimed. Blocks starting from first_block + * onward are invalidated. + */ +extern void InvalidateUndoBufferRange(uint32 log_number, + BlockNumber first_block, + BlockNumber last_block); + +#endif /* UNDO_BUFMGR_H */ diff --git a/src/include/access/undo_xlog.h b/src/include/access/undo_xlog.h new file mode 100644 index 0000000000000..4070c4287ff6c --- /dev/null +++ b/src/include/access/undo_xlog.h @@ -0,0 +1,332 @@ +/*------------------------------------------------------------------------- + * + * undo_xlog.h + * UNDO resource manager WAL record definitions + * + * This file contains the WAL record format definitions for UNDO log + * operations. These records are logged by the RM_UNDO_ID resource manager. + * + * Record types: + * XLOG_UNDO_ALLOCATE - Log UNDO space allocation + * XLOG_UNDO_DISCARD - Log UNDO record discard + * XLOG_UNDO_EXTEND - Log UNDO log file extension + * XLOG_UNDO_APPLY_RECORD - CLR: Log physical UNDO application to a page + * + * The XLOG_UNDO_APPLY_RECORD type is a Compensation Log Record (CLR). + * CLRs record the fact that an UNDO operation was applied to a page + * during transaction rollback. This ensures crash safety: if we crash + * during rollback, the already-applied UNDO operations are preserved + * via WAL replay of the CLR's full page image. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/access/undo_xlog.h + * + *------------------------------------------------------------------------- + */ +#ifndef UNDO_XLOG_H +#define UNDO_XLOG_H + +#include "access/transam.h" +#include "access/xlogdefs.h" +#include "access/xlogreader.h" +#include "lib/stringinfo.h" +#include "storage/block.h" +#include "storage/off.h" +#include "storage/relfilelocator.h" + +/* + * UndoRecPtr type definition. We use undodefs.h which is lightweight + * and can be included in both frontend and backend code. If undodefs.h + * has already been included (via undolog.h or directly), this is a no-op. + */ +#include "access/undodefs.h" + +/* + * WAL record types for UNDO operations + * + * These are the info codes for UNDO WAL records. The low 4 bits are used + * for operation type, leaving the upper 4 bits for flags. + */ +#define XLOG_UNDO_ALLOCATE 0x00 /* Allocate UNDO log space + * (legacy) */ +#define XLOG_UNDO_DISCARD 0x10 /* Discard old UNDO records */ +#define XLOG_UNDO_EXTEND 0x20 /* Extend UNDO log file (legacy) */ +#define XLOG_UNDO_APPLY_RECORD 0x30 /* CLR: UNDO applied to page */ +#define XLOG_UNDO_ROTATE 0x40 /* Seal old log, activate new + * (legacy) */ +#define XLOG_UNDO_PAGE_WRITE 0x50 /* Write UNDO data to a page + * (legacy) */ +#define XLOG_UNDO_BATCH 0x60 /* Batched UNDO records in WAL */ + +/* + * xl_undo_allocate - WAL record for UNDO space allocation + * + * Logged when a backend allocates space in an UNDO log for writing + * UNDO records. This ensures crash recovery can reconstruct the + * insert pointer state. + */ +typedef struct xl_undo_allocate +{ + UndoRecPtr start_ptr; /* Starting position of allocation */ + uint32 length; /* Length of allocation in bytes */ + TransactionId xid; /* Transaction that allocated this space */ + uint32 log_number; /* Log number (extracted from start_ptr) */ +} xl_undo_allocate; + +#define SizeOfUndoAllocate (offsetof(xl_undo_allocate, log_number) + sizeof(uint32)) + +/* + * xl_undo_discard - WAL record for UNDO discard operation + * + * Logged when the UNDO worker discards old UNDO records that are no + * longer needed by any active transaction. This allows space to be + * reclaimed. + */ +typedef struct xl_undo_discard +{ + UndoRecPtr discard_ptr; /* New discard pointer (oldest still needed) */ + uint32 log_number; /* Which log is being discarded */ + TransactionId oldest_xid; /* Oldest XID still needing UNDO */ +} xl_undo_discard; + +#define SizeOfUndoDiscard (offsetof(xl_undo_discard, oldest_xid) + sizeof(TransactionId)) + +/* + * xl_undo_extend - WAL record for UNDO log file extension + * + * Logged when an UNDO log file is extended to accommodate more UNDO + * records. This ensures the file size is correctly restored during + * crash recovery. + */ +typedef struct xl_undo_extend +{ + uint32 log_number; /* Which log is being extended */ + uint64 new_size; /* New size of log file in bytes */ +} xl_undo_extend; + +#define SizeOfUndoExtend (offsetof(xl_undo_extend, new_size) + sizeof(uint64)) + +/* + * xl_undo_apply - CLR for physical UNDO application (physiological) + * + * This is a Compensation Log Record (CLR) generated when an UNDO record + * is physically applied to a heap or index page during transaction rollback. + * + * Physiological CLR approach: + * Instead of storing a full 8KB page image (REGBUF_FORCE_IMAGE), we log + * just the operation and its data. During redo, we re-apply the exact + * same page modification. This reduces WAL volume from ~8KB to + * ~100-500 bytes per CLR. + * + * For operations that only change LP state (INSERT undo, HOT_UPDATE kill), + * no additional data is needed -- the metadata in xl_undo_apply suffices. + * + * For operations that restore tuple data (DELETE/UPDATE/INPLACE undo), + * the tuple data follows the fixed header as registered buffer data. + * + * For full page image operations (DEDUP undo), REGBUF_FORCE_IMAGE is + * still used since the entire page is being replaced. + * + * CLR flags (in clr_flags): + * UNDO_CLR_HAS_TUPLE - Tuple data follows (for DELETE/UPDATE/INPLACE) + * UNDO_CLR_HAS_DELTA - Delta-encoded tuple data (for UPDATE) + * UNDO_CLR_LP_DEAD - Mark line pointer LP_DEAD (for INSERT undo) + * UNDO_CLR_LP_UNUSED - Mark line pointer LP_UNUSED (for INSERT undo) + * UNDO_CLR_FULL_PAGE - Full page image (fallback, DEDUP undo) + * UNDO_CLR_HOT_RESTORE - HOT update rollback (restore infomask + kill new) + */ + +/* CLR operation flags */ +#define UNDO_CLR_HAS_TUPLE 0x0001 /* Tuple data in buffer data */ +#define UNDO_CLR_HAS_DELTA 0x0002 /* Delta-encoded tuple restoration */ +#define UNDO_CLR_LP_DEAD 0x0004 /* Mark target LP_DEAD */ +#define UNDO_CLR_LP_UNUSED 0x0008 /* Mark target LP_UNUSED */ +#define UNDO_CLR_FULL_PAGE 0x0010 /* Full page image (DEDUP) */ +#define UNDO_CLR_HOT_RESTORE 0x0020 /* HOT update rollback */ +#define UNDO_CLR_HAS_VISIBILITY 0x0040 /* Visibility-delta (xmax+infomask) + * for DELETE */ + +typedef struct xl_undo_apply +{ + UndoRecPtr urec_ptr; /* UNDO record pointer that was applied */ + TransactionId xid; /* Transaction being rolled back */ + RelFileLocator target_locator; /* Target relation file locator */ + BlockNumber target_block; /* Target block number */ + OffsetNumber target_offset; /* Target item offset within page */ + uint16 operation_type; /* UNDO subtype (HEAP_UNDO_INSERT, etc.) */ + uint16 clr_flags; /* UNDO_CLR_* flags */ + uint32 tuple_len; /* Restored tuple length (0 if no tuple) */ +} xl_undo_apply; + +#define SizeOfUndoApply (offsetof(xl_undo_apply, tuple_len) + sizeof(uint32)) + +/* + * xl_undo_apply_hot - Additional data for HOT update CLR redo + * + * Follows xl_undo_apply when UNDO_CLR_HOT_RESTORE is set. + * Registered as additional XLogRegisterData after the main record. + */ +typedef struct xl_undo_apply_hot +{ + OffsetNumber new_offset; /* New (killed) tuple's offset */ + uint16 old_infomask; /* Restored infomask for old tuple */ + uint16 old_infomask2; /* Restored infomask2 for old tuple */ +} xl_undo_apply_hot; + +#define SizeOfUndoApplyHot (offsetof(xl_undo_apply_hot, old_infomask2) + sizeof(uint16)) + +/* + * xl_undo_apply_visibility - Additional data for DELETE visibility-delta CLR + * + * Follows xl_undo_apply when UNDO_CLR_HAS_VISIBILITY is set. + * Stores only the three header fields changed by DELETE, not the full tuple. + * This reduces DELETE UNDO WAL payload from ~160-560 bytes to 8 bytes. + */ +typedef struct xl_undo_apply_visibility +{ + TransactionId old_xmax; /* t_xmax before delete */ + uint16 old_infomask; /* t_infomask before delete */ + uint16 old_infomask2; /* t_infomask2 before delete */ +} xl_undo_apply_visibility; + +#define SizeOfUndoApplyVisibility \ + (offsetof(xl_undo_apply_visibility, old_infomask2) + sizeof(uint16)) + +/* + * xl_undo_page_write - WAL record for UNDO page data write + * + * Logged when UNDO data is written to a shared-buffer-managed page. + * The actual data follows the record header and is also registered + * via XLogRegisterBufData as buffer-specific data (block reference 0). + * + * During redo, the data is memcpy'd into the page at page_offset. + * If a full page image was stored (REGBUF_STANDARD enables FPI after + * checkpoints), XLogReadBufferForRedo restores it automatically and + * no additional replay is needed. + */ +typedef struct xl_undo_page_write +{ + uint32 page_offset; /* Offset within the page to write at */ + uint32 data_len; /* Length of data written */ +} xl_undo_page_write; + +#define SizeOfUndoPageWrite (offsetof(xl_undo_page_write, data_len) + sizeof(uint32)) + +/* + * Rotation trigger reasons for XLOG_UNDO_ROTATE records + */ +#define UNDO_ROTATE_CAPACITY 0x01 /* Rotated due to capacity threshold */ +#define UNDO_ROTATE_CHECKPOINT 0x02 /* Rotated at checkpoint boundary */ +#define UNDO_ROTATE_PRESSURE 0x03 /* Rotated under allocation pressure */ +#define UNDO_ROTATE_MANUAL 0x04 /* Rotated by pg_undo_force_discard() */ + +/* + * xl_undo_rotate - WAL record for UNDO log segment rotation + * + * Logged when the active UNDO log is sealed and a new one is activated. + * During recovery, the old log is marked SEALED and the new log is + * marked ACTIVE, restoring the correct lifecycle state. + */ +typedef struct xl_undo_rotate +{ + uint32 old_log_number; /* Log being sealed (0 if first log) */ + UndoRecPtr old_seal_ptr; /* Insert pointer at seal time */ + uint32 new_log_number; /* Newly activated log */ + uint8 trigger; /* UNDO_ROTATE_* reason */ +} xl_undo_rotate; + +#define SizeOfUndoRotate (offsetof(xl_undo_rotate, trigger) + sizeof(uint8)) + +/* + * xl_undo_batch - WAL record for batched UNDO data (XLOG_UNDO_BATCH) + * + * This record type replaces the old pwrite()-to-segment-file path. + * All UNDO records for a batch are serialized into a single WAL record. + * The batch payload contains concatenated UndoRecordHeader+payload pairs + * in their exact serialized format. + * + * The chain_prev field links this batch to the previous batch for the + * same transaction. During rollback, the UNDO chain is walked backward + * by reading WAL records at successive chain_prev LSNs. + * + * Coalescing: The existing UndoRecordSet mechanism batches records + * (flush at 256KB or 1000 records). This batch becomes one WAL record. + * A 1000-row INSERT produces ~1 WAL record containing 1000 UNDO records. + */ +typedef struct xl_undo_batch +{ + TransactionId xid; /* Owning transaction */ + XLogRecPtr chain_prev; /* LSN of previous batch for this xact + * (InvalidXLogRecPtr if first batch) */ + uint32 nrecords; /* Number of UNDO records in batch */ + uint32 total_len; /* Total bytes of serialized UNDO data */ + Oid primary_reloid; /* Relation OID (optimization for + * single-relation batches) */ + UndoPersistenceLevel persistence; /* Persistence level of this batch */ + /* Followed by total_len bytes of serialized UndoRecordHeader+payload */ +} xl_undo_batch; + +#define SizeOfUndoBatch (offsetof(xl_undo_batch, persistence) + sizeof(UndoPersistenceLevel)) + +/* + * xl_undo_chain_state - UNDO chain state for prepared transactions + * + * Saved in the two-phase state file during PREPARE TRANSACTION, so the + * UNDO chain can be restored during COMMIT/ROLLBACK PREPARED. + */ +typedef struct xl_undo_chain_state +{ + UndoRecPtr firstUndoPtr; /* First UNDO record in transaction chain */ + UndoRecPtr currentUndoPtr; /* Most recent UNDO record in chain */ +} xl_undo_chain_state; + +/* Function declarations for WAL operations */ +extern void undo_redo(XLogReaderState *record); +extern void undo_desc(StringInfo buf, XLogReaderState *record); +extern const char *undo_identify(uint8 info); + +/* Two-phase commit support */ +extern void undo_twophase_recover(FullTransactionId fxid, uint16 info, + void *recdata, uint32 len); +extern void undo_twophase_postcommit(FullTransactionId fxid, uint16 info, + void *recdata, uint32 len); +extern void undo_twophase_postabort(FullTransactionId fxid, uint16 info, + void *recdata, uint32 len); + +/* + * UNDO batch reading from WAL for rollback and recovery. + * + * UndoReadBatchFromWAL reads a single XLOG_UNDO_BATCH record at the + * given LSN and returns the header plus a pointer to the payload data. + * The caller must pfree the returned data when done. + */ +typedef struct UndoBatchData +{ + xl_undo_batch header; /* Batch header */ + char *payload; /* Serialized UNDO records (palloc'd) */ + Size payload_len; /* Length of payload */ +} UndoBatchData; + +extern bool UndoValidateBatchLSN(XLogRecPtr batch_lsn); +extern UndoBatchData *UndoReadBatchFromWAL(XLogRecPtr batch_lsn); +extern void UndoFreeBatchData(UndoBatchData *batch); +extern void UndoResetBatchReader(void); + +/* + * Recovery UNDO phase support. + * + * During WAL redo, XLOG_UNDO_BATCH records are tracked so that after + * redo completes, incomplete transactions can be identified and their + * UNDO chains walked for rollback. + */ +extern void UndoRecoveryTrackBatch(TransactionId xid, XLogRecPtr batch_lsn, + XLogRecPtr chain_prev, + UndoPersistenceLevel persistence); +extern void UndoRecoveryRemoveXid(TransactionId xid); +extern bool UndoRecoveryNeeded(void); +extern void PerformUndoRecovery(void); +extern void FlushDeferredUndoXacts(void); + +#endif /* UNDO_XLOG_H */ diff --git a/src/include/access/undobuffer.h b/src/include/access/undobuffer.h new file mode 100644 index 0000000000000..5579f8aaa8e4b --- /dev/null +++ b/src/include/access/undobuffer.h @@ -0,0 +1,113 @@ +/*------------------------------------------------------------------------- + * + * undobuffer.h + * AM-agnostic Tier 2 UNDO write buffer + * + * The Tier 2 buffer accumulates serialized UNDO records for the current DML + * operation in a per-backend byte buffer. At WAL-write time, the buffer + * contents are embedded directly inside the AM's DML WAL record via + * XLogRegisterData(), eliminating a separate XLOG_UNDO_BATCH record for + * single-tuple operations. + * + * If the buffer grows beyond the configured threshold before the DML WAL + * record is written, the overflow path flushes it as a standalone + * XLOG_UNDO_BATCH record (preserving bulk-operation semantics). + * + * The buffer is per-backend; only one relation can be active at a time. + * This matches the executor's single-ModifyTable-node pattern. + * + * Any access method (table or index) can use this buffer. The UNDO record + * header format (UndoRecordHeader) is AM-agnostic: each record carries an + * RM ID (urec_rmid) that identifies the resource manager responsible for + * interpreting and applying the record during rollback. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/access/undobuffer.h + * + *------------------------------------------------------------------------- + */ +#ifndef UNDOBUFFER_H +#define UNDOBUFFER_H + +#include "access/xlogdefs.h" +#include "utils/relcache.h" + +/* + * UndoBufferBegin - Activate the Tier 2 UNDO buffer for a relation. + * + * Only one relation can have an active buffer at a time. If the buffer is + * already active for a different relation, the previous buffer is flushed + * and deactivated before switching. + * + * 'nrows' is the planner's estimate (0 if unknown); reserved for future + * pre-sizing but not used currently. + */ +extern void UndoBufferBegin(Relation rel, int64 nrows); + +/* + * UndoBufferEnd - Deactivate the Tier 2 UNDO buffer. + * + * Any records accumulated since the last flush or WAL embedding will be + * flushed as an overflow batch. + */ +extern void UndoBufferEnd(Relation rel); + +/* + * UndoBufferAddRecord - Add an UNDO record to the Tier 2 buffer. + * + * Auto-flushes via the overflow path if size/count thresholds are exceeded. + */ +extern void UndoBufferAddRecord(Relation rel, uint8 rmid, uint16 info, + const char *payload, Size payload_len); + +/* + * UndoBufferAddRecordParts - Add an UNDO record with scatter-gather payload. + * + * Avoids an intermediate buffer for operations where the payload is a + * fixed header struct + variable-length data (e.g., index tuple). + */ +extern void UndoBufferAddRecordParts(Relation rel, uint8 rmid, uint16 info, + const char *part1, Size part1_len, + const char *part2, Size part2_len); + +/* + * UndoBufferFlush - Overflow flush: emit a standalone XLOG_UNDO_BATCH. + * + * Used when the buffer grows too large before the DML WAL record is written + * (bulk operations), or at UndoBufferEnd time. + */ +extern void UndoBufferFlush(void); + +/* + * UndoBufferIsActive - Check if the Tier 2 buffer is active for a relation. + */ +extern bool UndoBufferIsActive(Relation rel); + +/* + * UndoBufferHasPendingData - Return true if the buffer has records to embed. + */ +extern bool UndoBufferHasPendingData(void); + +/* + * UndoBufferTakePayload - Hand off buffer contents to the caller. + * + * Called from the DML WAL section before XLogInsert(). The caller embeds + * the returned data via XLogRegisterData() to carry UNDO inside the DML + * WAL record. After XLogInsert(), the caller must invoke UndoBufferReset() + * to release ownership and update chain tracking. + */ +extern void UndoBufferTakePayload(char **data_out, Size *len_out, + int *nrecords_out, + XLogRecPtr *chain_prev_out); + +/* + * UndoBufferReset - Reset after the DML WAL record has been written. + * + * Updates chain_prev to the LSN of the WAL record that embedded the UNDO, + * then clears len/nrecords so the buffer can accept new records. + */ +extern void UndoBufferReset(XLogRecPtr embedded_lsn); + +#endif /* UNDOBUFFER_H */ diff --git a/src/include/access/undodefs.h b/src/include/access/undodefs.h new file mode 100644 index 0000000000000..b21915bff1004 --- /dev/null +++ b/src/include/access/undodefs.h @@ -0,0 +1,56 @@ +/*------------------------------------------------------------------------- + * + * undodefs.h + * + * Basic definitions for PostgreSQL undo layer. These are separated into + * their own header file to avoid including more things than necessary + * into widely-used headers like xact.h. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/access/undodefs.h + * + *------------------------------------------------------------------------- + */ +#ifndef UNDODEFS_H +#define UNDODEFS_H + +/* The type used to identify an undo log and position within it. */ +typedef uint64 UndoRecPtr; + +/* The type used for undo record lengths. */ +typedef uint16 UndoRecordSize; + +/* Type for offsets within undo logs */ +typedef uint64 UndoLogOffset; + +/* Type for numbering undo logs. */ +typedef int UndoLogNumber; + +/* Special value for undo record pointer which indicates that it is invalid. */ +#define InvalidUndoRecPtr ((UndoRecPtr) 0) + +/* + * UndoRecPtrIsValid + * True iff undoRecPtr is valid. + */ +#define UndoRecPtrIsValid(undoRecPtr) \ + ((bool) ((UndoRecPtr) (undoRecPtr) != InvalidUndoRecPtr)) + +/* Persistence levels as small integers that can be used as array indexes. */ +typedef enum +{ + UNDOPERSISTENCE_PERMANENT = 0, + UNDOPERSISTENCE_UNLOGGED = 1, + UNDOPERSISTENCE_TEMP = 2 +} UndoPersistenceLevel; + +/* Number of supported persistence levels for undo. */ +#define NUndoPersistenceLevels 3 + +/* Opaque types. */ +struct UndoRecordSet; +typedef struct UndoRecordSet UndoRecordSet; + +#endif diff --git a/src/include/access/undolog.h b/src/include/access/undolog.h new file mode 100644 index 0000000000000..cabdc9dd6c816 --- /dev/null +++ b/src/include/access/undolog.h @@ -0,0 +1,197 @@ +/*------------------------------------------------------------------------- + * + * undolog.h + * PostgreSQL UNDO log manager -- WAL-integrated version + * + * With UNDO-in-WAL, UNDO records are stored in the standard WAL stream + * as XLOG_UNDO_BATCH records. The separate base/undo/ segment files + * and direct I/O path have been removed. This header retains: + * + * - UndoRecPtr encoding macros (still used for addressing) + * - Shared memory structures (UndoLogControl, UndoLogSharedData) + * - GUC parameter declarations + * - Functions for shmem init, discard, and checkpoint + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/access/undolog.h + * + *------------------------------------------------------------------------- + */ +#ifndef UNDOLOG_H +#define UNDOLOG_H + +#include "access/transam.h" +#include "access/undodefs.h" +#include "access/xlogdefs.h" +#include "datatype/timestamp.h" +#include "port/atomics.h" +#include "port/pg_crc32c.h" +#include "storage/lwlock.h" +#include "storage/shmem.h" + +/* + * UndoRecPtr: 64-bit pointer to UNDO record + * + * Format: + * Bits 0-39: Offset within log (40 bits = 1TB per log) + * Bits 40-63: Log number (24 bits = 16M logs) + * + * The actual UndoRecPtr typedef and InvalidUndoRecPtr are in undodefs.h + * to avoid circular include dependencies. + */ + +/* Extract log number and offset from UndoRecPtr */ +#define UndoRecPtrGetLogNo(ptr) ((uint32) (((uint64) (ptr)) >> 40)) +#define UndoRecPtrGetOffset(ptr) (((uint64) (ptr)) & 0xFFFFFFFFFFULL) + +/* Construct UndoRecPtr from log number and offset */ +#define MakeUndoRecPtr(logno, offset) \ + ((((uint64) (logno)) << 40) | ((uint64) (offset))) + +/* + * Legacy define -- no longer used (UNDO records are in WAL, not segment + * files). Retained for any code that still references it at compile time. + */ +#define UNDO_LOG_SEGMENT_SIZE (1024 * 1024 * 1024) + +/* Maximum number of concurrent UNDO logs */ +#define MAX_UNDO_LOGS 100 + +/* + * UndoLogState: Lifecycle state of an UNDO log slot + * + * With UNDO-in-WAL, the segment lifecycle is simplified -- these states + * are retained for shared memory structure compatibility but the + * ACTIVE->SEALED->DISCARDABLE rotation no longer occurs. + */ +typedef enum UndoLogState +{ + UNDO_LOG_FREE = 0, /* Slot available */ + UNDO_LOG_ACTIVE, /* Accepting writes */ + UNDO_LOG_SEALED, /* No more writes */ + UNDO_LOG_DISCARDABLE /* All records discarded */ +} UndoLogState; + +/* + * UndoLogControl: Shared memory control structure for one UNDO log + */ +typedef struct UndoLogControl +{ + uint32 log_number; /* Log number */ + pg_atomic_uint64 insert_ptr; /* Next insertion point (atomic) */ + UndoRecPtr discard_ptr; /* Can discard older than this */ + TransactionId oldest_xid; /* Oldest transaction needing this log */ + LWLock lock; /* Protects metadata (NOT insert_ptr) */ + bool in_use; /* Is this log slot active? */ + UndoLogState state; /* Current lifecycle state */ + pg_atomic_uint64 seal_ptr; /* insert_ptr frozen at seal time */ + TimestampTz sealed_time; /* When this log was sealed */ +} UndoLogControl; + +/* + * UndoLogSharedData: Shared memory for all UNDO logs + * + * Note: backend_undo_lsns is a flexible array member; the struct must be + * allocated with room for MaxBackends entries. Use UndoLogShmemSize() to + * get the correct allocation size. + */ +typedef struct UndoLogSharedData +{ + UndoLogControl logs[MAX_UNDO_LOGS]; + uint32 next_log_number; + LWLock allocation_lock; + pg_atomic_uint32 active_log_idx; + pg_atomic_uint64 total_allocated; + pg_atomic_uint64 total_discarded; + + /* + * UNDO discard horizon: the oldest XLogRecPtr of an XLOG_UNDO_BATCH + * record that is still needed for rollback or index pruning. WAL + * segments containing data at or after this LSN must be retained. Updated + * by the UNDO discard worker as transactions complete and their UNDO + * records are no longer needed. + */ + pg_atomic_uint64 undo_discard_horizon; + + /* + * Per-backend first UNDO batch LSN. + * + * Each active backend stores the XLogRecPtr of its first XLOG_UNDO_BATCH + * record here when it writes UNDO data for a transaction. Cleared at + * commit or abort. The UNDO discard worker scans this array to find the + * global minimum, which becomes the new undo_discard_horizon, preventing + * WAL recycling past the oldest in-flight UNDO batch. + * + * Indexed by MyProcNumber (0-based, range [0, MaxBackends)). + * + * Must be last field -- UndoLogShmemSize() uses + * offsetof(UndoLogSharedData, backend_undo_lsns). + */ + pg_atomic_uint64 backend_undo_lsns[FLEXIBLE_ARRAY_MEMBER]; +} UndoLogSharedData; + +StaticAssertDecl(sizeof(XLogRecPtr) == sizeof(uint64), + "XLogRecPtr must be 64 bits for UNDO per-backend atomic LSN slots to be correct"); + +/* Global shared memory pointer (set during startup) */ +extern UndoLogSharedData *UndoLogShared; + +/* GUC parameters */ +/* + * Note: UNDO records are embedded in WAL (no separate segment files). + * UNDO_LOG_SEGMENT_SIZE and MAX_UNDO_LOGS are legacy defines retained + * for compile-time compatibility. + */ +extern int undo_retention_time; +extern int undo_worker_naptime; +extern int undo_buffer_size; +extern int undo_max_wal_retention_size; +extern int undo_batch_size_kb; +extern int undo_batch_record_limit; + +/* + * Shared memory initialization + */ +extern Size UndoLogShmemSize(void); +extern void UndoLogShmemInit(void); + +/* + * Discard, retention, and checkpoint + */ +extern void UndoLogDiscard(UndoRecPtr oldest_needed); +extern UndoRecPtr UndoLogGetOldestDiscardPtr(void); +extern void CheckPointUndoLog(void); + +/* WAL retention for UNDO: get/set the discard horizon */ +extern XLogRecPtr UndoGetDiscardHorizon(void); +extern void UndoSetDiscardHorizon(XLogRecPtr horizon); + +/* Per-backend UNDO batch LSN registration for WAL retention */ +extern void UndoRegisterBatchLSN(XLogRecPtr batch_lsn); +extern void UndoClearBatchLSN(void); +extern XLogRecPtr UndoGetOldestBatchLSN(void); + +/* + * Utility functions + */ +extern UndoRecPtr UndoLogGetInsertPtr(uint32 log_number); +extern UndoRecPtr UndoLogGetDiscardPtr(uint32 log_number); +extern char *UndoLogPath(uint32 log_number, char *path); + +/* + * Legacy no-op stubs -- retained for callers not yet fully updated. + * These are all no-ops in the UNDO-in-WAL architecture. + */ +extern void UndoLogSync(void); +extern void UndoLogCloseFiles(void); +extern void ExtendUndoLogFile(uint32 log_number, uint64 new_size); +extern void ExtendUndoLogSmgrFile(uint32 log_number, uint64 logical_end); +extern UndoRecPtr UndoFlushGetMaxWritePtr(void); +extern void UndoFlushResetMaxWritePtr(void); +extern void UndoLogSealAndRotate(uint8 trigger); +extern void UndoLogDeleteSegmentFile(uint32 log_number); +extern bool UndoLogTryPressureDiscard(void); + +#endif /* UNDOLOG_H */ diff --git a/src/include/access/undorecord.h b/src/include/access/undorecord.h new file mode 100644 index 0000000000000..95d9bffe751b4 --- /dev/null +++ b/src/include/access/undorecord.h @@ -0,0 +1,209 @@ +/*------------------------------------------------------------------------- + * + * undorecord.h + * UNDO record format and insertion API + * + * This file defines the generic UNDO record format that can be used by + * any access method or subsystem. UNDO records are AM-agnostic: each + * record carries an RM ID (urec_rmid) that identifies the resource + * manager responsible for interpreting and applying the record. + * + * Design principles: + * - Physical: UNDO stores opaque payload data for direct restore + * - Generic: Usable by any AM or subsystem + * - Compact: Variable-length format to minimize space + * - Chained: Records form backward chains via urec_prev pointer + * - Batch-oriented: API encourages batching for performance + * - AM-agnostic: No AM-specific types in the generic header or API + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/access/undorecord.h + * + *------------------------------------------------------------------------- + */ +#ifndef UNDORECORD_H +#define UNDORECORD_H + +#include "access/undodefs.h" +#include "access/undolog.h" +#include "access/xlogdefs.h" +#include "storage/block.h" +#include "storage/itemptr.h" + +/* + * UNDO record info flags + * + * These flags provide additional metadata about the UNDO record. + * The lower byte is reserved for generic flags; the upper byte is + * available for RM-specific use. + */ +#define UNDO_INFO_HAS_PAYLOAD 0x01 /* Record contains opaque payload */ +#define UNDO_INFO_XID_VALID 0x08 /* urec_xid is valid */ + +/* + * UndoRecordHeader - Fixed header for all UNDO records + * + * Every UNDO record starts with this header, followed by an optional + * opaque payload whose interpretation is RM-specific. + * + * The urec_rmid field identifies which resource manager owns this record. + * The urec_info field carries RM-specific subtype/flags (e.g., an in-place + * update AM uses it to distinguish INSERT vs DELETE vs UPDATE). + * + * Size: 40 bytes (optimized for alignment) + */ +typedef struct UndoRecordHeader +{ + uint8 urec_rmid; /* UNDO RM ID */ + uint8 urec_flags; /* Generic flags (UNDO_INFO_*) */ + uint16 urec_info; /* RM-specific subtype and flags */ + uint32 urec_len; /* Total length including header + payload */ + + TransactionId urec_xid; /* Transaction that created this */ + UndoRecPtr urec_prev; /* Previous UNDO for same xact (chain) */ + + Oid urec_reloid; /* Relation OID (InvalidOid if N/A) */ + + /* + * Payload length: size of the RM-specific opaque data that follows the + * header. Interpretation is entirely RM-specific. + */ + uint32 urec_payload_len; + + /* Followed by variable-length RM-specific payload */ +} UndoRecordHeader; + +#define SizeOfUndoRecordHeader (offsetof(UndoRecordHeader, urec_payload_len) + sizeof(uint32)) + +/* + * Access macros for payload data following the header + * + * The payload immediately follows the fixed header in the serialized + * record. Its interpretation is entirely RM-specific. + */ +#define UndoRecGetPayload(header) \ + ((char *)(header) + SizeOfUndoRecordHeader) + +/* + * UndoRecordSetChunkHeader - Header at the start of each chunk. + * + * When an UndoRecordSet spans multiple undo logs (rare, since each log + * is up to 1TB), the data is organized into chunks, each with a header + * that records the chunk size and a back-pointer to the previous chunk. + * This design follows the EDB undo-record-set branch architecture. + */ +typedef struct UndoRecordSetChunkHeader +{ + UndoLogOffset size; + UndoRecPtr previous_chunk; + uint8 type; +} UndoRecordSetChunkHeader; + +#define SizeOfUndoRecordSetChunkHeader \ + (offsetof(UndoRecordSetChunkHeader, type) + sizeof(uint8)) + +/* + * Possible undo record set types. + */ +typedef enum UndoRecordSetType +{ + URST_INVALID = 0, /* Placeholder when there's no record set. */ + URST_TRANSACTION = 'T', /* Normal xact undo; apply on abort. */ + URST_MULTI = 'M', /* Informational undo. */ + URST_EPHEMERAL = 'E' /* Ephemeral data for testing purposes. */ +} UndoRecordSetType; + +/* + * UndoRecordSet - Batch container for UNDO records + * + * This structure accumulates multiple UNDO records before writing them + * to the UNDO log in a single operation. This improves performance by + * reducing the number of I/O operations and lock acquisitions. + * + * The records are serialized into a contiguous buffer that grows + * dynamically. The design follows the EDB undo-record-set branch + * architecture with chunk-based organization and per-persistence-level + * separation. + */ +typedef struct UndoRecordSet +{ + TransactionId xid; /* Transaction ID for all records */ + UndoRecPtr prev_undo_ptr; /* Previous UNDO pointer in chain (legacy) */ + UndoPersistenceLevel persistence; /* Persistence level of this set */ + UndoRecordSetType type; /* Record set type */ + + int nrecords; /* Number of records in set */ + + /* + * Dynamic buffer for serialized records. Grows as needed; no fixed + * maximum. This replaces the old fixed-capacity max_records array. + */ + char *buffer; /* Serialized record buffer */ + Size buffer_size; /* Current buffer size */ + Size buffer_capacity; /* Allocated buffer capacity */ + + /* + * WAL-based UNDO chain tracking. When UNDO records are written to WAL + * via XLOG_UNDO_BATCH, last_batch_lsn tracks the LSN of the most recent + * batch for this record set. This is used as the chain_prev link when + * the next batch is written. + */ + XLogRecPtr last_batch_lsn; /* LSN of last XLOG_UNDO_BATCH record */ + + MemoryContext mctx; /* Memory context for allocations */ +} UndoRecordSet; + +/* + * Public API for UNDO record management + */ + +/* Create/destroy/reset UNDO record sets */ +extern UndoRecordSet *UndoRecordSetCreate(TransactionId xid, + UndoRecPtr prev_undo_ptr); +extern void UndoRecordSetFree(UndoRecordSet *uset); +extern void UndoRecordSetReset(UndoRecordSet *uset); +extern void UndoRecordSetResetCache(void); + +/* Add records to a set - generic payload API */ +extern void UndoRecordAddPayload(UndoRecordSet *uset, + uint8 rmid, + uint16 info, + Oid reloid, + const char *payload, + Size payload_len); + +/* Add records with scatter-gather payload (avoids intermediate buffer) */ +extern void UndoRecordAddPayloadParts(UndoRecordSet *uset, + uint8 rmid, + uint16 info, + Oid reloid, + const char *part1, + Size part1_len, + const char *part2, + Size part2_len); + +/* Insert the accumulated records into UNDO log */ +extern UndoRecPtr UndoRecordSetInsert(UndoRecordSet *uset); + +/* WAL batch management for deferred UNDO allocation logging */ +extern void UndoWalBatchFlush(void); +extern void UndoWalBatchReset(void); + +/* Utility functions for record manipulation */ +extern Size UndoRecordGetPayloadSize(Size payload_len); +extern void UndoRecordSerialize(char *dest, UndoRecordHeader *header, + const char *payload, Size payload_len); +extern bool UndoRecordDeserialize(const char *src, UndoRecordHeader *header, + char **payload); + +/* Statistics and debugging */ +extern Size UndoRecordSetGetSize(UndoRecordSet *uset); + +/* UNDO application during rollback */ +extern bool ApplyUndoChainFromWAL(XLogRecPtr last_batch_lsn); +extern bool ApplyUndoChainFromWALBounded(XLogRecPtr last_batch_lsn, + XLogRecPtr stop_at_lsn); + +#endif /* UNDORECORD_H */ diff --git a/src/include/access/undormgr.h b/src/include/access/undormgr.h new file mode 100644 index 0000000000000..61e99c755d064 --- /dev/null +++ b/src/include/access/undormgr.h @@ -0,0 +1,118 @@ +/*------------------------------------------------------------------------- + * + * undormgr.h + * UNDO resource manager dispatch definitions + * + * This module provides a dispatch mechanism for UNDO record application, + * analogous to the WAL resource manager (rmgr) system. Each access method + * or subsystem that writes UNDO records registers an UndoRmgrData entry + * with callbacks for applying UNDO records and describing them for debugging. + * + * The generic UNDO infrastructure (undoapply.c) dispatches to the appropriate + * RM callback based on the urec_rmid field in the UNDO record header. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/access/undormgr.h + * + *------------------------------------------------------------------------- + */ +#ifndef UNDORMGR_H +#define UNDORMGR_H + +#include "postgres_ext.h" +#include "access/undodefs.h" +#include "access/xlogdefs.h" +#include "lib/stringinfo.h" + +/* + * UNDO Resource Manager IDs + * + * Each AM or subsystem that writes UNDO records is assigned a unique ID. + * This ID is stored in the urec_rmid field of every UNDO record header, + * enabling the generic UNDO infrastructure to dispatch to the correct + * apply callback during rollback. + * + * The core defines only UNDO_RMID_INVALID and the built-in index-AM IDs. + * Every other consumer defines its own UNDO_RMID_* constant in its own + * header (the same commit that registers it in access/undormgrlist.h), so + * this core header names no specific table AM or subsystem. The ID number + * space is a shared resource: IDs must be unique and stable across releases + * (they are stamped into WAL-durable UNDO records), so a new consumer takes + * the next free value in [1, MAX_UNDO_RMGRS) and never reuses one. + */ +#define UNDO_RMID_INVALID 0 +#define UNDO_RMID_NBTREE 1 +#define UNDO_RMID_HASH 3 +/* 2 and 4 are reserved by out-of-core consumers (see their own headers) */ + +#define MAX_UNDO_RMGRS 256 + +/* + * UndoApplyResult - Return value from undo apply callbacks + */ +typedef enum UndoApplyResult +{ + UNDO_APPLY_SUCCESS = 0, /* Successfully applied */ + UNDO_APPLY_SKIPPED, /* Skipped (e.g., relation dropped) */ + UNDO_APPLY_ERROR /* Error during application */ +} UndoApplyResult; + +/* + * UndoRmgrData - Resource manager registration entry + * + * Each UNDO RM provides: + * rm_name: Human-readable name for debugging/logging + * rm_undo: Apply one UNDO record (rollback callback) + * rm_desc: Describe an UNDO record for debugging output + * + * The rm_undo callback receives: + * - rmid: The RM ID (for verification) + * - info: RM-specific subtype/flags from urec_info + * - xid: Transaction being rolled back + * - reloid: Target relation OID (may be InvalidOid for non-relation ops) + * - payload: RM-specific opaque payload data + * - payload_len: Length of payload + * - urec_ptr: Position of this record in UNDO log (for CLR generation) + * + * The callback is responsible for: + * - Opening the relation (if applicable) + * - Locking and modifying the target page + * - Generating a CLR WAL record + * - Releasing all locks and buffers + */ +typedef UndoApplyResult (*UndoRmgrApplyFunc) (uint8 rmid, + uint16 info, + TransactionId xid, + Oid reloid, + const char *payload, + Size payload_len, + UndoRecPtr urec_ptr); + +typedef void (*UndoRmgrDescFunc) (StringInfo buf, + uint8 rmid, + uint16 info, + const char *payload, + Size payload_len); + +typedef struct UndoRmgrData +{ + const char *rm_name; /* Human-readable name */ + UndoRmgrApplyFunc rm_undo; /* Apply callback */ + UndoRmgrDescFunc rm_desc; /* Describe callback */ +} UndoRmgrData; + +/* Global registration table */ +extern const UndoRmgrData *UndoRmgrs[MAX_UNDO_RMGRS]; + +/* Registration function (called during _PG_init or startup) */ +extern void RegisterUndoRmgr(uint8 rmid, const UndoRmgrData *rmgr); + +/* Lookup function */ +extern const UndoRmgrData *GetUndoRmgr(uint8 rmid); + +/* Initialization */ +extern void InitUndoRmgrs(void); + +#endif /* UNDORMGR_H */ diff --git a/src/include/access/undormgrlist.h b/src/include/access/undormgrlist.h new file mode 100644 index 0000000000000..4ab60e2141881 --- /dev/null +++ b/src/include/access/undormgrlist.h @@ -0,0 +1,33 @@ +/*------------------------------------------------------------------------- + * + * undormgrlist.h + * + * List of *UndoRmgrInit() initialization calls for built-in UNDO resource + * managers. Kept in its own source file, mirroring the pattern used by + * storage/subsystemlist.h for ShmemCallbacks registration, so that the + * generic UNDO core (undo.c) can register every built-in resource manager + * without knowing any of their names at compile time: undo.c only expands + * this list through a macro it controls, and the extern declarations are + * generated the same way (see access/undormgrs.h). + * + * Each access method or subsystem that writes UNDO records adds itself here + * when it is compiled in. This file is edited by whichever commit + * introduces a new UNDO-writing consumer; undo.c itself is never touched to + * add a new consumer. + * + * UNDO_RMGR_INIT is defined by the caller depending on how the list is used. + * + * No built-in resource managers exist yet at this point in the UNDO + * subsystem's history; this file starts empty and grows as each consumer + * (an index AM, a table AM, or a subsystem that writes UNDO records) is + * compiled in and adds itself here. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/access/undormgrlist.h + * + *------------------------------------------------------------------------- + */ + +/* there is deliberately not an #ifndef UNDORMGRLIST_H here */ diff --git a/src/include/access/undormgrs.h b/src/include/access/undormgrs.h new file mode 100644 index 0000000000000..2a399f78dfab2 --- /dev/null +++ b/src/include/access/undormgrs.h @@ -0,0 +1,28 @@ +/*------------------------------------------------------------------------- + * + * undormgrs.h + * Provide extern declarations for all the built-in UNDO resource + * manager *Init() functions. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/access/undormgrs.h + * + *------------------------------------------------------------------------- + */ +#ifndef UNDORMGRS_H +#define UNDORMGRS_H + +/* + * Extern declarations of all the built-in *UndoRmgrInit() functions. + * + * The actual list is in undormgrlist.h, so that the same list can be used + * for other purposes (e.g. undo.c's RegisterUndoRmgrs()). + */ +#define UNDO_RMGR_INIT(initfunc) \ + extern void initfunc(void); +#include "access/undormgrlist.h" +#undef UNDO_RMGR_INIT + +#endif /* UNDORMGRS_H */ diff --git a/src/include/access/undostats.h b/src/include/access/undostats.h new file mode 100644 index 0000000000000..9a5a55e3956b3 --- /dev/null +++ b/src/include/access/undostats.h @@ -0,0 +1,63 @@ +/*------------------------------------------------------------------------- + * + * undostats.h + * UNDO log statistics collection and reporting + * + * Provides monitoring and observability for the UNDO subsystem, + * including per-log statistics and buffer cache statistics. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/access/undostats.h + * + *------------------------------------------------------------------------- + */ +#ifndef UNDOSTATS_H +#define UNDOSTATS_H + +#include "access/undolog.h" +#include "fmgr.h" + +/* + * UndoLogStat - Per-log statistics snapshot + * + * Point-in-time snapshot of a single UNDO log's state. + */ +typedef struct UndoLogStat +{ + uint32 log_number; /* UNDO log number */ + UndoRecPtr insert_ptr; /* Current insert pointer */ + UndoRecPtr discard_ptr; /* Current discard pointer */ + TransactionId oldest_xid; /* Oldest transaction in this log */ + uint64 size_bytes; /* Active size (insert - discard) */ + UndoLogState state; /* Current lifecycle state */ +} UndoLogStat; + +/* + * UndoBufferStat - UNDO buffer cache statistics + * + * Aggregate statistics from the UNDO buffer cache. + */ +typedef struct UndoBufferStat +{ + int num_buffers; /* Number of buffer slots */ + uint64 cache_hits; /* Total cache hits */ + uint64 cache_misses; /* Total cache misses */ + uint64 cache_evictions; /* Total evictions */ + uint64 cache_writes; /* Total dirty buffer writes */ +} UndoBufferStat; + +/* Functions for collecting statistics */ +extern int GetUndoLogStats(UndoLogStat *stats, int max_stats); +extern void GetUndoBufferStats(UndoBufferStat *stats); + +/* + * pg_undo_force_discard is declared via PG_FUNCTION_INFO_V1 in + * undostats.c. Do not redeclare it here: on Windows that emits a + * __declspec(dllimport) prototype that conflicts with the implicit + * dllexport from the V1 info macro. Catalog references go through + * pg_proc by name. + */ + +#endif /* UNDOSTATS_H */ diff --git a/src/include/access/undoworker.h b/src/include/access/undoworker.h new file mode 100644 index 0000000000000..9b2cb6069b122 --- /dev/null +++ b/src/include/access/undoworker.h @@ -0,0 +1,66 @@ +/*------------------------------------------------------------------------- + * + * undoworker.h + * UNDO worker background process + * + * The UNDO worker is a background process that periodically scans active + * transactions and discards UNDO records that are no longer needed. + * This reclaims space in UNDO logs. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/access/undoworker.h + * + *------------------------------------------------------------------------- + */ +#ifndef UNDOWORKER_H +#define UNDOWORKER_H + +#include "access/transam.h" +#include "access/undolog.h" +#include "fmgr.h" +#include "storage/lwlock.h" +#include "storage/procnumber.h" +#include "storage/shmem.h" + +/* + * UndoWorkerShmemData - Shared memory for UNDO worker coordination + * + * This structure tracks the state of UNDO discard operations and + * coordinates between the worker and other backends. + */ +typedef struct UndoWorkerShmemData +{ + LWLock lock; /* Protects this structure */ + + pg_atomic_uint64 last_discard_time; /* Last discard operation time */ + TransactionId oldest_xid_checked; /* Last XID used for discard */ + UndoRecPtr last_discard_ptr; /* Last UNDO pointer discarded */ + + int naptime_ms; /* Current sleep time in ms */ + bool shutdown_requested; /* Worker should exit */ + + /* Rotation coordination fields */ + ProcNumber worker_proc; /* For latch-based wakeup */ + pg_atomic_uint32 sealed_log_count; /* Number of SEALED logs pending */ +} UndoWorkerShmemData; + +/* GUC parameters */ +extern int undo_worker_naptime; +extern int undo_retention_time; + +/* Shared memory functions */ +extern Size UndoWorkerShmemSize(void); +extern void UndoWorkerShmemInit(void); + +/* Worker lifecycle functions */ +pg_noreturn extern void UndoWorkerMain(Datum main_arg); +extern void UndoWorkerRegister(void); + +/* Utility functions */ +extern TransactionId UndoWorkerGetOldestXid(void); +extern void UndoWorkerRequestShutdown(void); +extern void WakeUndoDiscardWorker(void); + +#endif /* UNDOWORKER_H */ diff --git a/src/include/access/xact.h b/src/include/access/xact.h index a8cbdf247c866..b10932c6344e4 100644 --- a/src/include/access/xact.h +++ b/src/include/access/xact.h @@ -138,6 +138,39 @@ typedef enum typedef void (*XactCallback) (XactEvent event, void *arg); +/* + * TableAMPrepare_hook: called from PrepareTransaction(), between + * StartPrepare() and EndPrepare(), where RegisterTwoPhaseRecord() is valid. + * + * This window is narrower than any XactCallback event: XACT_EVENT_PRE_PREPARE + * fires BEFORE StartPrepare() (before "the remaining actions cannot call any + * user-defined code" section begins), so a table AM that needs to register + * two-phase-commit records for tuples/rows it is tracking (e.g. an in-place + * MVCC AM saving per-tuple UNDO state so COMMIT PREPARED / ROLLBACK PREPARED + * can locate and finalize them) cannot do so from an ordinary XactCallback + * and needs this dedicated, narrower hook instead. NULL is a valid no-op + * for a build with no such AM compiled in. + */ +extern void (*TableAMPrepare_hook) (void); + +/* + * PendingPhysOps{Do,PostPrepare,AtSubCommit,AtSubAbort}_hook: mirror the + * matching core smgr{DoPendingDeletes,PostPrepare,AtSubCommit,AtSubAbort} + * calls in xact.c. A subsystem that manages pending structural filesystem + * operations needs the SAME commit/abort-sequencing position + * as smgr's own pending-deletes housekeeping -- e.g. + * PendingPhysOpsDo_hook(true) is placed immediately after + * smgrDoPendingDeletes(true), which core's own comment there documents as + * "best done after releasing relcache and buffer pins... this ordering is + * definitely critical during abort" -- a physical-file-cleanup ordering + * requirement no ordinary XactCallback/SubXactCallback event exposes. NULL + * is a valid no-op for a build with no such subsystem compiled in. + */ +extern void (*PendingPhysOpsDo_hook) (bool isCommit); +extern void (*PendingPhysOpsPostPrepare_hook) (void); +extern void (*PendingPhysOpsAtSubCommit_hook) (void); +extern void (*PendingPhysOpsAtSubAbort_hook) (void); + typedef enum { SUBXACT_EVENT_START_SUB, @@ -368,8 +401,28 @@ typedef struct xl_xact_prepare uint16 gidlen; /* length of the GID - GID follows the header */ XLogRecPtr origin_lsn; /* lsn of this record at origin node */ TimestampTz origin_timestamp; /* time of prepare at origin node */ + + /* + * UNDO chain head LSN per persistence level (3 == NUndoPersistenceLevels + * from undodefs.h; hardcoded here to keep xact.h free of UNDO headers). + * If NUndoPersistenceLevels changes, this array must be updated and both + * XLOG_PAGE_MAGIC (xlog_internal.h) and TWOPHASE_MAGIC (twophase.c) must + * be bumped. See StaticAssertDecl in xactundo.c for compile-time guard. + */ + XLogRecPtr last_batch_lsn[3]; } xl_xact_prepare; +#define SizeOfXactPrepare sizeof(xl_xact_prepare) + +/* + * Verify xl_xact_prepare contains the UNDO last_batch_lsn field. This struct + * is written into WAL as part of XLOG_XACT_PREPARE records, and into 2PC + * state files via TwoPhaseFileHeader. Any layout change requires bumping both + * XLOG_PAGE_MAGIC (xlog_internal.h) and TWOPHASE_MAGIC (twophase.c). + */ +StaticAssertDecl(offsetof(xl_xact_prepare, last_batch_lsn) > 0, + "xl_xact_prepare must contain last_batch_lsn for UNDO WAL compat"); + /* * Commit/Abort records in the above form are a bit verbose to parse, so * there's a deconstructed versions generated by ParseCommit/AbortRecord() for @@ -439,6 +492,8 @@ typedef struct xl_xact_parsed_abort */ extern bool IsTransactionState(void); extern bool IsAbortedTransactionBlockState(void); +extern int EnterInlineUndoApplyState(void); +extern void LeaveInlineUndoApplyState(int saved); extern TransactionId GetTopTransactionId(void); extern TransactionId GetTopTransactionIdIfAny(void); extern TransactionId GetCurrentTransactionId(void); @@ -535,4 +590,8 @@ extern void EnterParallelMode(void); extern void ExitParallelMode(void); extern bool IsInParallelMode(void); +/* UNDO chain management */ +extern void SetCurrentTransactionUndoRecPtr(uint64 undo_ptr); +extern uint64 GetCurrentTransactionUndoRecPtr(void); + #endif /* XACT_H */ diff --git a/src/include/access/xactundo.h b/src/include/access/xactundo.h new file mode 100644 index 0000000000000..b6a359678ac5a --- /dev/null +++ b/src/include/access/xactundo.h @@ -0,0 +1,117 @@ +/*------------------------------------------------------------------------- + * + * xactundo.h + * Transaction-level undo management + * + * This module manages per-transaction undo record sets. It maintains + * up to NUndoPersistenceLevels (3) record sets per transaction -- one + * for each persistence level (permanent, unlogged, temporary). This + * design follows the EDB undo-record-set branch architecture where + * undo records for different persistence levels are kept separate. + * + * Code that wants to write transactional undo should interface with + * these functions rather than manipulating UndoRecordSet directly. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/access/xactundo.h + * + *------------------------------------------------------------------------- + */ +#ifndef XACTUNDO_H +#define XACTUNDO_H + +#include "access/relundo.h" +#include "access/undodefs.h" +#include "access/undorecord.h" +#include "access/xlogdefs.h" + +/* + * XactUndoContext - Context for a single undo insertion within a transaction. + * + * Created by PrepareXactUndoData(), consumed by InsertXactUndoData() + * and cleaned up by CleanupXactUndoInsertion(). The plevel tracks which + * persistence-level record set this insertion belongs to. + */ +typedef struct XactUndoContext +{ + UndoPersistenceLevel plevel; + UndoRecordSet *uset; /* borrowed reference, do not free */ +} XactUndoContext; + +/* Shared memory initialization */ +extern Size XactUndoShmemSize(void); +extern void XactUndoShmemInit(void); + +/* Per-backend initialization */ +extern void InitializeXactUndo(void); + +/* + * Undo insertion API for any AM or subsystem. + * + * PrepareXactUndoData: Find or create the appropriate per-persistence-level + * UndoRecordSet for the current transaction and prepare it for a new + * record. Returns the UndoRecPtr where the record will be written. + * + * Parameters are AM-agnostic: the caller provides an RM ID, RM-specific + * info flags, a relation OID, and an opaque payload. + * + * InsertXactUndoData: Actually write the record data into the undo log. + * + * CleanupXactUndoInsertion: Release any resources held by the context. + */ +extern UndoRecPtr PrepareXactUndoData(XactUndoContext *ctx, + char persistence, + uint8 rmid, + uint16 info, + Oid reloid, + const char *payload, + Size payload_len); +extern UndoRecPtr PrepareXactUndoDataParts(XactUndoContext *ctx, + char persistence, + uint8 rmid, + uint16 info, + Oid reloid, + const char *part1, + Size part1_len, + const char *part2, + Size part2_len); +extern void InsertXactUndoData(XactUndoContext *ctx); +extern void CleanupXactUndoInsertion(XactUndoContext *ctx); + +/* Transaction lifecycle hooks */ +extern void AtCommit_XactUndo(void); +extern void AtAbort_XactUndo(void); +extern void AtSubCommit_XactUndo(int level); +extern void AtSubAbort_XactUndo(int level); +extern void AtProcExit_XactUndo(void); + +/* Per-relation UNDO chain registration (used by AMs on the per-relation fork) */ +extern void RegisterPerRelUndo(Oid relid, RelUndoRecPtr start_urec_ptr); +extern RelUndoRecPtr GetPerRelUndoPtr(Oid relid); + +/* Callback for IteratePerRelUndo: one call per registered chain head. */ +typedef void (*PerRelUndoIterCB) (Oid relid, RelUndoRecPtr start_urec_ptr, + void *arg); +extern void IteratePerRelUndo(PerRelUndoIterCB callback, void *arg); +extern bool XactUndoHasUnrecoverableUndo(void); + +/* Undo chain traversal for rollback */ +extern UndoRecPtr GetCurrentXactUndoRecPtr(UndoPersistenceLevel plevel); +extern XLogRecPtr GetCurrentXactLastBatchLSN(UndoPersistenceLevel plevel); +extern void XActUndoUpdateLastBatchLSN(XLogRecPtr lsn, + UndoPersistenceLevel plevel); + +/* + * GUC: UNDO bytes threshold for instant abort via ATM. + * + * Transactions with estimated UNDO bytes >= this threshold use ATM instant + * abort (deferred rollback via Logical Revert worker). Transactions below + * the threshold use synchronous rollback inline during transaction abort. + * + * A value of 0 means always use ATM instant abort regardless of size. + */ +extern int undo_instant_abort_threshold; + +#endif /* XACTUNDO_H */ diff --git a/src/include/common/relpath.h b/src/include/common/relpath.h index 9772125be7398..674a0dc58e1ac 100644 --- a/src/include/common/relpath.h +++ b/src/include/common/relpath.h @@ -60,6 +60,7 @@ typedef enum ForkNumber FSM_FORKNUM, VISIBILITYMAP_FORKNUM, INIT_FORKNUM, + RELUNDO_FORKNUM, /* * NOTE: if you add a new fork, change MAX_FORKNUM and possibly @@ -68,9 +69,9 @@ typedef enum ForkNumber */ } ForkNumber; -#define MAX_FORKNUM INIT_FORKNUM +#define MAX_FORKNUM RELUNDO_FORKNUM -#define FORKNAMECHARS 4 /* max chars for a fork name */ +#define FORKNAMECHARS 7 /* max chars for a fork name ("relundo") */ extern PGDLLIMPORT const char *const forkNames[]; diff --git a/src/include/storage/buf_internals.h b/src/include/storage/buf_internals.h index e4ff5619b79ca..a1dfded233ba1 100644 --- a/src/include/storage/buf_internals.h +++ b/src/include/storage/buf_internals.h @@ -146,6 +146,20 @@ StaticAssertDecl(MAX_BACKENDS_BITS <= (BUF_LOCK_BITS - 2), StaticAssertDecl(BM_MAX_USAGE_COUNT < (UINT64CONST(1) << BUF_USAGECOUNT_BITS), "BM_MAX_USAGE_COUNT doesn't fit in BUF_USAGECOUNT_BITS bits"); +/* + * Reserved fork number for UNDO log buffers. + * + * This constant is reserved for future use when the smgr layer is extended + * to support undo-specific file management. Currently, undo buffers use + * MAIN_FORKNUM (following ZHeap's UndoLogForkNum convention) because the + * smgr layer sizes internal arrays to MAX_FORKNUM+1. Undo buffers are + * distinguished from regular relation data by using a pseudo-database OID + * (UNDO_DB_OID = 9) in the BufferTag's dbOid field. + * + * See src/include/access/undo_bufmgr.h for the undo buffer manager API. + */ +#define UNDO_FORKNUM 5 + /* * Buffer tag identifies which disk block the buffer contains. * diff --git a/src/include/storage/bufmgr.h b/src/include/storage/bufmgr.h index 6837b35fc6d0b..6d9346aff58cf 100644 --- a/src/include/storage/bufmgr.h +++ b/src/include/storage/bufmgr.h @@ -259,6 +259,16 @@ extern bool BufferIsLockedByMe(Buffer buffer); extern bool BufferIsLockedByMeInMode(Buffer buffer, BufferLockMode mode); extern bool BufferIsDirty(Buffer buffer); extern void MarkBufferDirty(Buffer buffer); + +/* + * MarkBufferDirtyShared -- mark buffer dirty while holding only BUFFER_LOCK_SHARE. + * + * Safe ONLY when the page modification is performed via an atomic CAS and the + * buffer's dirty bit is set atomically (no exclusive content lock needed). + * Currently used by an in-place-update table AM's CAS-update path where the + * tuple's writer field is modified atomically under shared buffer lock. + */ +extern void MarkBufferDirtyShared(Buffer buffer); extern void IncrBufferRefCount(Buffer buffer); extern void CheckBufferIsPinnedOnce(Buffer buffer); extern Buffer ReleaseAndReadBuffer(Buffer buffer, Relation relation, @@ -321,6 +331,7 @@ extern bool BufferBeginSetHintBits(Buffer buffer); extern void BufferFinishSetHintBits(Buffer buffer, bool mark_dirty, bool buffer_std); extern void UnlockBuffers(void); +extern void BufferLockReleaseAll(void); extern void UnlockBuffer(Buffer buffer); extern void LockBufferInternal(Buffer buffer, BufferLockMode mode); diff --git a/src/include/storage/lwlocklist.h b/src/include/storage/lwlocklist.h index d7eb648bd2758..c9bb50f3001ce 100644 --- a/src/include/storage/lwlocklist.h +++ b/src/include/storage/lwlocklist.h @@ -140,3 +140,7 @@ PG_LWLOCKTRANCHE(XACT_SLRU, XactSLRU) PG_LWLOCKTRANCHE(PARALLEL_VACUUM_DSA, ParallelVacuumDSA) PG_LWLOCKTRANCHE(AIO_URING_COMPLETION, AioUringCompletion) PG_LWLOCKTRANCHE(SHMEM_INDEX, ShmemIndex) +PG_LWLOCKTRANCHE(UNDO_LOG, UndoLog) +PG_LWLOCKTRANCHE(UNDO_WORKER, UndoWorker) +PG_LWLOCKTRANCHE(ATM, AbortedTxnMap) +PG_LWLOCKTRANCHE(SLOG, SecondaryLog) diff --git a/src/include/storage/subsystemlist.h b/src/include/storage/subsystemlist.h index 9ad619080be22..f9c2ce9f05c60 100644 --- a/src/include/storage/subsystemlist.h +++ b/src/include/storage/subsystemlist.h @@ -88,3 +88,15 @@ PG_SHMEM_SUBSYSTEM(DataChecksumsShmemCallbacks) /* AIO subsystem. This delegates to the method-specific callbacks */ PG_SHMEM_SUBSYSTEM(AioShmemCallbacks) + +/* UNDO subsystem */ +PG_SHMEM_SUBSYSTEM(UndoShmemCallbacks) + +/* + * sLog: the UNDO subsystem's shared-memory Aborted Transaction Map (and an + * optional per-tuple tracking hash a consumer AM may extend it with). + * Registered as its own subsystem (not nested inside UndoShmemCallbacks) so + * the generic UNDO subsystem has no compile-time or link-time dependency on + * any consumer. See slog.c's SLogShmemCallbacks for details. + */ +PG_SHMEM_SUBSYSTEM(SLogShmemCallbacks) diff --git a/src/test/regress/regress.c b/src/test/regress/regress.c index 9801cdd1d8c3e..5c0f51d90bd56 100644 --- a/src/test/regress/regress.c +++ b/src/test/regress/regress.c @@ -1297,9 +1297,13 @@ test_relpath(PG_FUNCTION_ARGS) if ((int) ceil(log10(MAX_BACKENDS)) != PROCNUMBER_CHARS) elog(WARNING, "mismatch between MAX_BACKENDS and PROCNUMBER_CHARS"); - /* verify that the max-length relpath is generated ok */ + /* + * Verify that the max-length relpath is generated ok. Use the fork with + * the longest name (currently RELUNDO_FORKNUM, "relundo"), since that is + * what REL_PATH_STR_MAXLEN budgets FORKNAMECHARS for. + */ rpath = GetRelationPath(OID_MAX, OID_MAX, OID_MAX, MAX_BACKENDS - 1, - INIT_FORKNUM); + RELUNDO_FORKNUM); if (strlen(rpath.str) != REL_PATH_STR_MAXLEN) elog(WARNING, "maximum length relpath is if length %zu instead of %zu", diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 105269246cc3c..d0f21f3d7d907 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -4526,6 +4526,7 @@ xl_multixact_truncate xl_overwrite_contrecord xl_parameter_change xl_relmap_update +xl_relundo_truncate xl_replorigin_drop xl_replorigin_set xl_restore_point From ba29cd9f718b140ca64c7eff7a6f36775aa6e9d6 Mon Sep 17 00:00:00 2001 From: Greg Burd Date: Tue, 23 Jun 2026 09:12:13 -0400 Subject: [PATCH 08/10] UNDO: add nbtree and hash index UNDO apply handlers Add the UNDO resource-manager handlers for the nbtree and hash index AMs and register them from RegisterUndoRmgrs(). On rollback of an aborting transaction, the nbtree handler re-descends to the leaf entry by key and heap TID before marking it dead, so a committed entry that shifted onto the recorded slot under concurrent inserts or leaf splits is never killed; entries inside posting-list tuples are left for VACUUM. The hash handler reverses its own inserts analogously. Both are gated by RelationAmSupportsUndo() on the parent table, so they are inert until an UNDO-supporting table AM exists. --- src/backend/access/hash/Makefile | 1 + src/backend/access/hash/hash_undo.c | 279 +++++++++++ src/backend/access/hash/hashinsert.c | 10 + src/backend/access/hash/meson.build | 1 + src/backend/access/nbtree/Makefile | 1 + src/backend/access/nbtree/meson.build | 1 + src/backend/access/nbtree/nbtinsert.c | 15 + src/backend/access/nbtree/nbtree_undo.c | 591 ++++++++++++++++++++++++ src/include/access/hash.h | 5 + src/include/access/nbtree.h | 39 ++ src/include/access/undormgrlist.h | 4 + 11 files changed, 947 insertions(+) create mode 100644 src/backend/access/hash/hash_undo.c create mode 100644 src/backend/access/nbtree/nbtree_undo.c diff --git a/src/backend/access/hash/Makefile b/src/backend/access/hash/Makefile index 75bf36598246b..590c06c0e9976 100644 --- a/src/backend/access/hash/Makefile +++ b/src/backend/access/hash/Makefile @@ -14,6 +14,7 @@ include $(top_builddir)/src/Makefile.global OBJS = \ hash.o \ + hash_undo.o \ hash_xlog.o \ hashfunc.o \ hashinsert.o \ diff --git a/src/backend/access/hash/hash_undo.c b/src/backend/access/hash/hash_undo.c new file mode 100644 index 0000000000000..6fa2472ab6202 --- /dev/null +++ b/src/backend/access/hash/hash_undo.c @@ -0,0 +1,279 @@ +/*------------------------------------------------------------------------- + * + * hash_undo.c + * Hash index UNDO resource manager + * + * This module implements UNDO apply callbacks for the hash index AM. + * When a transaction aborts, provisionally inserted index entries are + * marked LP_DEAD so that VACUUM is not required to clean up after + * aborted transactions. + * + * Combined with heap UNDO and nbtree UNDO, hash UNDO provides a + * "zero-VACUUM" experience for aborted transactions: heap tuples and + * their index entries are cleaned up immediately during rollback. + * + * UNDO Subtypes: + * INSERT: Undo a hash index tuple insertion (mark entry LP_DEAD) + * + * All hooks are gated by RelationAmSupportsUndo(heapRel) -- hash UNDO + * is controlled by the parent table AM's am_supports_undo declaration. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/access/hash/hash_undo.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/hash.h" +#include "access/relation.h" +#include "access/undobuffer.h" +#include "access/undo_xlog.h" +#include "access/undorecord.h" +#include "access/undormgr.h" +#include "access/xact.h" +#include "access/xloginsert.h" +#include "access/xlogutils.h" +#include "miscadmin.h" +#include "storage/bufmgr.h" +#include "storage/bufpage.h" +#include "storage/itemid.h" +#include "utils/rel.h" +#include "utils/relcache.h" + +/* + * Hash UNDO subtypes (stored in urec_info) + */ +#define HASH_UNDO_INSERT 0x0001 /* bucket/overflow page tuple insertion */ + +/* + * HashUndoInsert - Payload for hash insert undo + */ +typedef struct HashUndoInsert +{ + Oid index_oid; /* OID of the hash index relation */ + BlockNumber blkno; /* Page where tuple was inserted */ + OffsetNumber offset; /* Offset of the inserted tuple */ +} HashUndoInsert; + +#define SizeOfHashUndoInsert \ + (offsetof(HashUndoInsert, offset) + sizeof(OffsetNumber)) + +/* Forward declarations */ +static UndoApplyResult hash_undo_apply(uint8 rmid, uint16 info, + TransactionId xid, Oid reloid, + const char *payload, Size payload_len, + UndoRecPtr urec_ptr); +static void hash_undo_desc(StringInfo buf, uint8 rmid, uint16 info, + const char *payload, Size payload_len); + +/* The hash UNDO RM registration entry */ +static const UndoRmgrData hash_undo_rmgr = { + .rm_name = "hash", + .rm_undo = hash_undo_apply, + .rm_desc = hash_undo_desc, +}; + +/* + * HashUndoRmgrInit - Register the hash UNDO resource manager + */ +void +HashUndoRmgrInit(void) +{ + RegisterUndoRmgr(UNDO_RMID_HASH, &hash_undo_rmgr); +} + +/* + * HashUndoLogInsert - Write UNDO record for a hash index tuple insertion + * + * Called from _hash_doinsert() after the insertion has been WAL-logged. + * This records enough information to mark the inserted entry LP_DEAD on abort. + */ +void +HashUndoLogInsert(Relation rel, Relation heapRel, Buffer buf, + OffsetNumber offset) +{ + TransactionId xid = GetCurrentTransactionId(); + HashUndoInsert hdr; + + hdr.index_oid = RelationGetRelid(rel); + hdr.blkno = BufferGetBlockNumber(buf); + hdr.offset = offset; + + /* + * When the heap has an active UNDO write buffer, piggyback on it to avoid + * a separate UndoLogAllocate + WAL insert + pwrite per index entry. + */ + if (UndoBufferIsActive(heapRel)) + { + UndoBufferAddRecordParts(heapRel, + UNDO_RMID_HASH, + HASH_UNDO_INSERT, + (const char *) &hdr, + SizeOfHashUndoInsert, + NULL, 0); + } + else + { + UndoRecordSet *uset; + + uset = UndoRecordSetCreate(xid, GetCurrentTransactionUndoRecPtr()); + UndoRecordAddPayloadParts(uset, + UNDO_RMID_HASH, + HASH_UNDO_INSERT, + RelationGetRelid(heapRel), + (const char *) &hdr, + SizeOfHashUndoInsert, + NULL, 0); + UndoRecordSetInsert(uset); + UndoRecordSetFree(uset); + } +} + +/* + * hash_undo_apply - Apply a single hash UNDO record + * + * This is the rm_undo callback for the hash RM. On abort, marks the + * inserted index entry as LP_DEAD. + */ +static UndoApplyResult +hash_undo_apply(uint8 rmid, uint16 info, TransactionId xid, Oid reloid, + const char *payload, Size payload_len, UndoRecPtr urec_ptr) +{ + Assert(rmid == UNDO_RMID_HASH); + + /* + * During crash recovery, syscache may not be initialized when + * PerformUndoRecovery() runs. Defer UNDO application until after the + * system is fully initialized (background worker will handle it). + */ + if (InRecovery) + { + ereport(DEBUG2, + (errmsg("hash UNDO: deferring transaction %u to logical revert worker " + "(in crash recovery, syscache not available)", + xid))); + return UNDO_APPLY_SKIPPED; + } + + switch (info) + { + case HASH_UNDO_INSERT: + { + HashUndoInsert hdr; + Relation indexrel; + Buffer buffer; + Page page; + + if (payload_len < SizeOfHashUndoInsert) + return UNDO_APPLY_ERROR; + + memcpy(&hdr, payload, SizeOfHashUndoInsert); + + /* + * Open the index directly using the OID stored in the UNDO + * payload. + */ + indexrel = try_relation_open(hdr.index_oid, RowExclusiveLock); + if (indexrel == NULL) + { + ereport(DEBUG2, + (errmsg("hash UNDO INSERT: index %u no longer exists", + hdr.index_oid))); + return UNDO_APPLY_SKIPPED; + } + + if (RelationGetNumberOfBlocks(indexrel) <= hdr.blkno) + { + ereport(DEBUG2, + (errmsg("hash UNDO INSERT: block %u beyond end of index %u", + hdr.blkno, hdr.index_oid))); + relation_close(indexrel, RowExclusiveLock); + return UNDO_APPLY_SKIPPED; + } + + buffer = ReadBuffer(indexrel, hdr.blkno); + LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); + page = BufferGetPage(buffer); + + if (hdr.offset <= PageGetMaxOffsetNumber(page)) + { + ItemId lp = PageGetItemId(page, hdr.offset); + + START_CRIT_SECTION(); + + if (ItemIdIsNormal(lp)) + ItemIdMarkDead(lp); + + MarkBufferDirty(buffer); + + /* Generate physiological CLR for crash recovery */ + if (RelationNeedsWAL(indexrel)) + { + XLogRecPtr clr_lsn; + xl_undo_apply xlrec; + + xlrec.urec_ptr = urec_ptr; + xlrec.xid = xid; + xlrec.target_locator = indexrel->rd_locator; + xlrec.target_block = hdr.blkno; + xlrec.target_offset = hdr.offset; + xlrec.operation_type = info; + xlrec.clr_flags = UNDO_CLR_LP_DEAD; + xlrec.tuple_len = 0; + + XLogBeginInsert(); + XLogRegisterData((char *) &xlrec, + SizeOfUndoApply); + XLogRegisterBuffer(0, buffer, + REGBUF_STANDARD); + clr_lsn = XLogInsert(RM_UNDO_ID, + XLOG_UNDO_APPLY_RECORD); + PageSetLSN(page, clr_lsn); + } + + END_CRIT_SECTION(); + } + + UnlockReleaseBuffer(buffer); + relation_close(indexrel, RowExclusiveLock); + return UNDO_APPLY_SUCCESS; + } + + default: + return UNDO_APPLY_SKIPPED; + } +} + +/* + * hash_undo_desc - Describe a hash UNDO record for debugging + */ +static void +hash_undo_desc(StringInfo buf, uint8 rmid, uint16 info, + const char *payload, Size payload_len) +{ + const char *opname; + + switch (info) + { + case HASH_UNDO_INSERT: + opname = "INSERT"; + break; + default: + opname = "UNKNOWN"; + break; + } + + appendStringInfo(buf, "hash %s", opname); + + if (payload_len >= sizeof(Oid) && info == HASH_UNDO_INSERT) + { + Oid index_oid; + + memcpy(&index_oid, payload, sizeof(Oid)); + appendStringInfo(buf, " index %u", index_oid); + } +} diff --git a/src/backend/access/hash/hashinsert.c b/src/backend/access/hash/hashinsert.c index 3395bbc13f825..4704e6a323de6 100644 --- a/src/backend/access/hash/hashinsert.c +++ b/src/backend/access/hash/hashinsert.c @@ -17,6 +17,8 @@ #include "access/hash.h" #include "access/hash_xlog.h" +#include "access/tableam.h" +#include "access/undobuffer.h" #include "access/xloginsert.h" #include "miscadmin.h" #include "storage/predicate.h" @@ -238,6 +240,14 @@ _hash_doinsert(Relation rel, IndexTuple itup, Relation heapRel, bool sorted) END_CRIT_SECTION(); + /* + * Write UNDO record for the insertion if the parent table AM supports + * UNDO. This must happen after WAL logging but while we still hold the + * buffer pin (needed for BufferGetBlockNumber). + */ + if (RelationAmSupportsUndo(heapRel) && UndoBufferIsActive(heapRel)) + HashUndoLogInsert(rel, heapRel, buf, itup_off); + /* drop lock on metapage, but keep pin */ LockBuffer(metabuf, BUFFER_LOCK_UNLOCK); diff --git a/src/backend/access/hash/meson.build b/src/backend/access/hash/meson.build index ad011b8f99ab6..ca2012be87bf7 100644 --- a/src/backend/access/hash/meson.build +++ b/src/backend/access/hash/meson.build @@ -2,6 +2,7 @@ backend_sources += files( 'hash.c', + 'hash_undo.c', 'hash_xlog.c', 'hashfunc.c', 'hashinsert.c', diff --git a/src/backend/access/nbtree/Makefile b/src/backend/access/nbtree/Makefile index 0daf640af96c7..492bcf578c112 100644 --- a/src/backend/access/nbtree/Makefile +++ b/src/backend/access/nbtree/Makefile @@ -20,6 +20,7 @@ OBJS = \ nbtpreprocesskeys.o \ nbtreadpage.o \ nbtree.o \ + nbtree_undo.o \ nbtsearch.o \ nbtsort.o \ nbtsplitloc.o \ diff --git a/src/backend/access/nbtree/meson.build b/src/backend/access/nbtree/meson.build index 812f067e7101c..f526ef6531729 100644 --- a/src/backend/access/nbtree/meson.build +++ b/src/backend/access/nbtree/meson.build @@ -8,6 +8,7 @@ backend_sources += files( 'nbtpreprocesskeys.c', 'nbtreadpage.c', 'nbtree.c', + 'nbtree_undo.c', 'nbtsearch.c', 'nbtsort.c', 'nbtsplitloc.c', diff --git a/src/backend/access/nbtree/nbtinsert.c b/src/backend/access/nbtree/nbtinsert.c index c8af97dd23dfb..128117d873adc 100644 --- a/src/backend/access/nbtree/nbtinsert.c +++ b/src/backend/access/nbtree/nbtinsert.c @@ -1420,6 +1420,21 @@ _bt_insertonpg(Relation rel, END_CRIT_SECTION(); + /* + * Write nbtree UNDO record for the insertion. This is done after the + * critical section (UNDO insertion involves I/O) but while we still + * hold the buffer lock. The UNDO record enables cleanup of this + * index entry if the transaction aborts. + * + * Only write UNDO if the parent table AM supports UNDO. The heaprel + * parameter is NULL during index builds and recovery. + */ + if (heaprel != NULL && RelationAmSupportsUndo(heaprel)) + { + NbtreeUndoLogInsert(rel, heaprel, buf, itup, + itemsz, newitemoff, isleaf); + } + /* Release subsidiary buffers */ if (BufferIsValid(metabuf)) _bt_relbuf(rel, metabuf); diff --git a/src/backend/access/nbtree/nbtree_undo.c b/src/backend/access/nbtree/nbtree_undo.c new file mode 100644 index 0000000000000..a9efff57b34c3 --- /dev/null +++ b/src/backend/access/nbtree/nbtree_undo.c @@ -0,0 +1,591 @@ +/*------------------------------------------------------------------------- + * + * nbtree_undo.c + * nbtree UNDO resource manager + * + * This module implements UNDO apply callbacks for the B-tree index AM. + * When a transaction aborts, provisionally inserted index entries are + * removed (or marked LP_DEAD) so that VACUUM is not required to clean + * up after aborted transactions. + * + * Combined with heap UNDO, nbtree UNDO provides a "zero-VACUUM" + * experience for aborted transactions: both heap tuples and their + * index entries are cleaned up immediately during rollback. + * + * UNDO Subtypes: + * INSERT_LEAF: Undo a leaf-page index tuple insertion + * INSERT_UPPER: Undo an internal-page downlink insertion + * INSERT_POST: Undo a posting list split + * DEDUP: Undo a deduplication pass (restore pre-dedup page) + * DELETE: Undo an ad-hoc deletion (re-insert deleted tuples) + * + * Structural operations (SPLIT, NEWROOT) and VACUUM operations are + * logged for completeness but their undo-apply is handled by falling + * back to per-entry LP_DEAD marking rather than reversing the + * structural change, since concurrent readers may have already + * observed the new structure. + * + * All hooks are guarded by RelationAmSupportsUndo(heaprel) -- nbtree + * UNDO is controlled by the parent table AM's am_supports_undo declaration. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/access/nbtree/nbtree_undo.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/relation.h" +#include "access/undobuffer.h" +#include "access/xact.h" +#include "access/nbtree.h" +#include "access/undo_xlog.h" +#include "access/undorecord.h" +#include "access/undormgr.h" +#include "access/xloginsert.h" +#include "access/xlogutils.h" +#include "miscadmin.h" +#include "storage/bufmgr.h" +#include "storage/bufpage.h" +#include "storage/itemid.h" +#include "utils/rel.h" +#include "utils/relcache.h" + +/* + * nbtree UNDO subtypes (stored in urec_info) + * + * These correspond to the WAL-logged nbtree operations. + */ +#define NBTREE_UNDO_INSERT_LEAF 0x0001 /* leaf tuple insertion */ +#define NBTREE_UNDO_INSERT_UPPER 0x0002 /* internal downlink insertion */ +#define NBTREE_UNDO_INSERT_POST 0x0004 /* posting list split */ +#define NBTREE_UNDO_DELETE 0x0005 /* ad-hoc tuple deletion */ +#define NBTREE_UNDO_SPLIT_L 0x0006 /* page split (new item on left) */ +#define NBTREE_UNDO_SPLIT_R 0x0007 /* page split (new item on right) */ +#define NBTREE_UNDO_NEWROOT 0x0008 /* new root creation */ +#define NBTREE_UNDO_DEDUP 0x0009 /* deduplication pass */ +#define NBTREE_UNDO_VACUUM 0x000A /* vacuum deletion (no-op undo) */ + +/* + * NbtreeUndoInsertLeaf - Payload for leaf insert undo + * + * index_oid allows direct index open during rollback, eliminating + * the O(N_indexes) scan through RelationGetIndexList(). + */ +typedef struct NbtreeUndoInsertLeaf +{ + Oid index_oid; /* OID of the index relation */ + BlockNumber blkno; /* Page where tuple was inserted */ + OffsetNumber offset; /* Offset of the inserted tuple */ + Size itup_sz; /* Size of the index tuple */ + /* Followed by the IndexTupleData */ +} NbtreeUndoInsertLeaf; + +#define SizeOfNbtreeUndoInsertLeaf offsetof(NbtreeUndoInsertLeaf, itup_sz) + sizeof(Size) + +/* + * NbtreeUndoInsertUpper - Payload for internal insert undo + */ +typedef struct NbtreeUndoInsertUpper +{ + Oid index_oid; /* OID of the index relation */ + BlockNumber blkno; /* Internal page */ + OffsetNumber offset; /* Offset of downlink */ + BlockNumber child_blkno; /* Child page whose downlink was added */ + Size itup_sz; /* Size of the downlink tuple */ + /* Followed by the IndexTupleData */ +} NbtreeUndoInsertUpper; + +#define SizeOfNbtreeUndoInsertUpper offsetof(NbtreeUndoInsertUpper, itup_sz) + sizeof(Size) + +/* + * NbtreeUndoDedup - Payload for dedup undo (full pre-dedup page image) + */ +typedef struct NbtreeUndoDedup +{ + Oid index_oid; /* OID of the index relation */ + BlockNumber blkno; /* Page that was deduplicated */ + uint16 page_len; /* Length of saved page image */ + /* Followed by the full page image (pre-dedup) */ +} NbtreeUndoDedup; + +#define SizeOfNbtreeUndoDedup offsetof(NbtreeUndoDedup, page_len) + sizeof(uint16) + +/* + * NbtreeUndoDelete - Payload for ad-hoc delete undo + */ +typedef struct NbtreeUndoDelete +{ + Oid index_oid; /* OID of the index relation */ + BlockNumber blkno; /* Page from which tuples were deleted */ + uint16 ndeleted; /* Number of deleted tuples */ + /* Followed by array of (OffsetNumber, IndexTupleData) pairs */ +} NbtreeUndoDelete; + +#define SizeOfNbtreeUndoDelete offsetof(NbtreeUndoDelete, ndeleted) + sizeof(uint16) + +/* Forward declarations */ +static UndoApplyResult nbtree_undo_apply(uint8 rmid, uint16 info, + TransactionId xid, Oid reloid, + const char *payload, Size payload_len, + UndoRecPtr urec_ptr); +static void nbtree_undo_desc(StringInfo buf, uint8 rmid, uint16 info, + const char *payload, Size payload_len); + +/* The nbtree UNDO RM registration entry */ +static const UndoRmgrData nbtree_undo_rmgr = { + .rm_name = "nbtree", + .rm_undo = nbtree_undo_apply, + .rm_desc = nbtree_undo_desc, +}; + +/* + * NbtreeUndoRmgrInit - Register the nbtree UNDO resource manager + */ +void +NbtreeUndoRmgrInit(void) +{ + RegisterUndoRmgr(UNDO_RMID_NBTREE, &nbtree_undo_rmgr); +} + +/* + * NbtreeUndoLogInsert - Write UNDO record for a leaf index tuple insertion + * + * Called from _bt_insertonpg() after the insertion has been WAL-logged. + * This records enough information to remove the inserted entry on abort. + */ +void +NbtreeUndoLogInsert(Relation rel, Relation heaprel, Buffer buf, + IndexTuple itup, Size itemsz, OffsetNumber offset, + bool isleaf) +{ + TransactionId xid = GetCurrentTransactionId(); + + if (isleaf) + { + NbtreeUndoInsertLeaf hdr; + + hdr.index_oid = RelationGetRelid(rel); + hdr.blkno = BufferGetBlockNumber(buf); + hdr.offset = offset; + hdr.itup_sz = itemsz; + + /* + * When the heap has an active UNDO write buffer, piggyback on it to + * avoid a separate UndoLogAllocate + WAL insert + pwrite per index + * entry. The UndoRecordSet accepts mixed RM IDs. + */ + if (UndoBufferIsActive(heaprel)) + { + UndoBufferAddRecordParts(heaprel, + UNDO_RMID_NBTREE, + NBTREE_UNDO_INSERT_LEAF, + (const char *) &hdr, + SizeOfNbtreeUndoInsertLeaf, + (const char *) itup, + itemsz); + } + else + { + UndoRecordSet *uset; + + uset = UndoRecordSetCreate(xid, GetCurrentTransactionUndoRecPtr()); + UndoRecordAddPayloadParts(uset, + UNDO_RMID_NBTREE, + NBTREE_UNDO_INSERT_LEAF, + RelationGetRelid(heaprel), + (const char *) &hdr, + SizeOfNbtreeUndoInsertLeaf, + (const char *) itup, + itemsz); + UndoRecordSetInsert(uset); + UndoRecordSetFree(uset); + } + } + else + { + NbtreeUndoInsertUpper upper_hdr; + + upper_hdr.index_oid = RelationGetRelid(rel); + upper_hdr.blkno = BufferGetBlockNumber(buf); + upper_hdr.offset = offset; + upper_hdr.child_blkno = BTreeTupleGetDownLink(itup); + upper_hdr.itup_sz = itemsz; + + if (UndoBufferIsActive(heaprel)) + { + UndoBufferAddRecordParts(heaprel, + UNDO_RMID_NBTREE, + NBTREE_UNDO_INSERT_UPPER, + (const char *) &upper_hdr, + SizeOfNbtreeUndoInsertUpper, + (const char *) itup, + itemsz); + } + else + { + UndoRecordSet *uset; + + uset = UndoRecordSetCreate(xid, GetCurrentTransactionUndoRecPtr()); + UndoRecordAddPayloadParts(uset, + UNDO_RMID_NBTREE, + NBTREE_UNDO_INSERT_UPPER, + RelationGetRelid(heaprel), + (const char *) &upper_hdr, + SizeOfNbtreeUndoInsertUpper, + (const char *) itup, + itemsz); + UndoRecordSetInsert(uset); + UndoRecordSetFree(uset); + } + } +} + +/* + * NbtreeUndoLogDedup - Write UNDO record before deduplication + * + * Called from _bt_dedup_pass() before the page is modified. + * Saves a full page image so dedup can be reversed on abort. + */ +void +NbtreeUndoLogDedup(Relation rel, Relation heaprel, Buffer buf) +{ + NbtreeUndoDedup hdr; + Page page = BufferGetPage(buf); + Size page_size = PageGetPageSize(page); + Size payload_size; + char *payload; + UndoRecordSet *uset; + TransactionId xid = GetCurrentTransactionId(); + + payload_size = SizeOfNbtreeUndoDedup + page_size; + payload = (char *) palloc(payload_size); + + hdr.index_oid = RelationGetRelid(rel); + hdr.blkno = BufferGetBlockNumber(buf); + hdr.page_len = (uint16) page_size; + memcpy(payload, &hdr, SizeOfNbtreeUndoDedup); + memcpy(payload + SizeOfNbtreeUndoDedup, page, page_size); + + uset = UndoRecordSetCreate(xid, GetCurrentTransactionUndoRecPtr()); + UndoRecordAddPayload(uset, UNDO_RMID_NBTREE, NBTREE_UNDO_DEDUP, + RelationGetRelid(heaprel), payload, payload_size); + UndoRecordSetInsert(uset); + UndoRecordSetFree(uset); + pfree(payload); +} + +/* + * nbtree_undo_apply - Apply a single nbtree UNDO record + * + * This is the rm_undo callback for the nbtree RM. + */ +static UndoApplyResult +nbtree_undo_apply(uint8 rmid, uint16 info, TransactionId xid, Oid reloid, + const char *payload, Size payload_len, UndoRecPtr urec_ptr) +{ + Assert(rmid == UNDO_RMID_NBTREE); + + /* + * During crash recovery, syscache may not be initialized yet when + * PerformUndoRecovery() runs. try_relation_open() requires syscache to + * check if the relation exists, so we must defer UNDO application until + * after the system is fully initialized. + * + * Check if we're in recovery mode (InRecovery flag is still set). During + * crash recovery, UNDO phase runs before syscache is initialized, so we + * skip UNDO application and rely on the logical revert worker to handle + * it asynchronously after startup completes. + * + * This transaction will be tracked in the ATM (Aborted Transaction Map) + * so the background worker can pick it up later. + * + * Note: InRecovery is only true during startup/recovery; it's false + * during normal operation and during normal transaction abort, so this + * check only affects crash recovery. + */ + if (InRecovery) + { + ereport(DEBUG2, + (errmsg("nbtree UNDO: deferring transaction %u to logical revert worker " + "(in crash recovery, syscache not available)", + xid))); + return UNDO_APPLY_SKIPPED; + } + + switch (info) + { + case NBTREE_UNDO_INSERT_LEAF: + { + NbtreeUndoInsertLeaf hdr; + Relation indexrel; + Buffer buffer; + Page page; + BTPageOpaque opaque; + + if (payload_len < SizeOfNbtreeUndoInsertLeaf) + return UNDO_APPLY_ERROR; + + memcpy(&hdr, payload, SizeOfNbtreeUndoInsertLeaf); + + /* + * Open the index directly using the OID stored in the UNDO + * payload. This avoids the O(N_indexes) scan through + * RelationGetIndexList(). + */ + indexrel = try_relation_open(hdr.index_oid, RowExclusiveLock); + if (indexrel == NULL) + { + ereport(DEBUG2, + (errmsg("nbtree UNDO INSERT_LEAF: index %u no longer exists", + hdr.index_oid))); + return UNDO_APPLY_SKIPPED; + } + + if (RelationGetNumberOfBlocks(indexrel) <= hdr.blkno) + { + ereport(DEBUG2, + (errmsg("nbtree UNDO INSERT_LEAF: block %u beyond end of index %u", + hdr.blkno, hdr.index_oid))); + relation_close(indexrel, RowExclusiveLock); + return UNDO_APPLY_SKIPPED; + } + + buffer = ReadBuffer(indexrel, hdr.blkno); + LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); + page = BufferGetPage(buffer); + opaque = BTPageGetOpaque(page); + + if (P_ISLEAF(opaque) && + hdr.offset <= PageGetMaxOffsetNumber(page)) + { + ItemId lp = PageGetItemId(page, hdr.offset); + + START_CRIT_SECTION(); + + if (ItemIdIsNormal(lp)) + ItemIdMarkDead(lp); + + MarkBufferDirty(buffer); + + /* Generate physiological CLR */ + if (RelationNeedsWAL(indexrel)) + { + XLogRecPtr clr_lsn; + xl_undo_apply xlrec; + + xlrec.urec_ptr = urec_ptr; + xlrec.xid = xid; + xlrec.target_locator = indexrel->rd_locator; + xlrec.target_block = hdr.blkno; + xlrec.target_offset = hdr.offset; + xlrec.operation_type = info; + xlrec.clr_flags = UNDO_CLR_LP_DEAD; + xlrec.tuple_len = 0; + + XLogBeginInsert(); + XLogRegisterData((char *) &xlrec, + SizeOfUndoApply); + XLogRegisterBuffer(0, buffer, + REGBUF_STANDARD); + clr_lsn = XLogInsert(RM_UNDO_ID, + XLOG_UNDO_APPLY_RECORD); + PageSetLSN(page, clr_lsn); + } + + END_CRIT_SECTION(); + } + + UnlockReleaseBuffer(buffer); + relation_close(indexrel, RowExclusiveLock); + return UNDO_APPLY_SUCCESS; + } + + case NBTREE_UNDO_INSERT_UPPER: + { + /* + * Undoing internal page insertions is complex and risky. The + * downlink is needed for tree navigation. Instead of removing + * it, we leave it in place. The child page (from a split that + * was part of the aborted transaction) will have its entries + * marked LP_DEAD by the leaf undo, and eventually the page + * will be recycled by VACUUM. + */ + return UNDO_APPLY_SKIPPED; + } + + case NBTREE_UNDO_DEDUP: + { + NbtreeUndoDedup hdr; + Relation indexrel; + Buffer buffer; + Page page; + + if (payload_len < SizeOfNbtreeUndoDedup) + return UNDO_APPLY_ERROR; + + memcpy(&hdr, payload, SizeOfNbtreeUndoDedup); + + /* + * Open the index directly using the OID stored in the UNDO + * payload. + */ + indexrel = try_relation_open(hdr.index_oid, RowExclusiveLock); + if (indexrel == NULL) + { + ereport(DEBUG2, + (errmsg("nbtree UNDO DEDUP: index %u no longer exists", + hdr.index_oid))); + return UNDO_APPLY_SKIPPED; + } + + if (RelationGetNumberOfBlocks(indexrel) <= hdr.blkno) + { + ereport(DEBUG2, + (errmsg("nbtree UNDO DEDUP: block %u beyond end of index %u", + hdr.blkno, hdr.index_oid))); + relation_close(indexrel, RowExclusiveLock); + return UNDO_APPLY_SKIPPED; + } + + buffer = ReadBuffer(indexrel, hdr.blkno); + LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); + page = BufferGetPage(buffer); + + START_CRIT_SECTION(); + + /* Restore the full pre-dedup page image */ + memcpy(page, + payload + SizeOfNbtreeUndoDedup, + hdr.page_len); + + MarkBufferDirty(buffer); + + if (RelationNeedsWAL(indexrel)) + { + XLogRecPtr clr_lsn; + xl_undo_apply xlrec; + + xlrec.urec_ptr = urec_ptr; + xlrec.xid = xid; + xlrec.target_locator = indexrel->rd_locator; + xlrec.target_block = hdr.blkno; + xlrec.target_offset = 0; + xlrec.operation_type = info; + xlrec.clr_flags = UNDO_CLR_FULL_PAGE; + xlrec.tuple_len = 0; + + XLogBeginInsert(); + XLogRegisterData((char *) &xlrec, + SizeOfUndoApply); + XLogRegisterBuffer(0, buffer, + REGBUF_FORCE_IMAGE | + REGBUF_STANDARD); + clr_lsn = XLogInsert(RM_UNDO_ID, + XLOG_UNDO_APPLY_RECORD); + PageSetLSN(page, clr_lsn); + } + + END_CRIT_SECTION(); + + UnlockReleaseBuffer(buffer); + relation_close(indexrel, RowExclusiveLock); + return UNDO_APPLY_SUCCESS; + } + + case NBTREE_UNDO_INSERT_POST: + case NBTREE_UNDO_SPLIT_L: + case NBTREE_UNDO_SPLIT_R: + case NBTREE_UNDO_NEWROOT: + + /* + * Structural operations: attempting to reverse a split is too + * dangerous due to concurrent readers. The individual leaf + * entries from the aborted transaction will be cleaned up by + * their own INSERT_LEAF undo records. Structural artifacts + * (empty pages from splits) will be recycled by VACUUM. + */ + return UNDO_APPLY_SKIPPED; + + case NBTREE_UNDO_DELETE: + + /* + * Ad-hoc deletion undo: re-insert the deleted tuples. This is + * complex since we need to find the correct insertion point. For + * now, skip and let the entries be re-created by the reverted + * heap operation. + */ + return UNDO_APPLY_SKIPPED; + + case NBTREE_UNDO_VACUUM: + /* VACUUM runs in its own transaction -- undo is always no-op */ + return UNDO_APPLY_SKIPPED; + + default: + ereport(WARNING, + (errmsg("nbtree UNDO: unknown subtype %u", info))); + return UNDO_APPLY_ERROR; + } +} + +/* + * nbtree_undo_desc - Describe an nbtree UNDO record for debugging + */ +static void +nbtree_undo_desc(StringInfo buf, uint8 rmid, uint16 info, + const char *payload, Size payload_len) +{ + const char *opname; + + switch (info) + { + case NBTREE_UNDO_INSERT_LEAF: + opname = "INSERT_LEAF"; + break; + case NBTREE_UNDO_INSERT_UPPER: + opname = "INSERT_UPPER"; + break; + case NBTREE_UNDO_INSERT_POST: + opname = "INSERT_POST"; + break; + case NBTREE_UNDO_DELETE: + opname = "DELETE"; + break; + case NBTREE_UNDO_SPLIT_L: + opname = "SPLIT_L"; + break; + case NBTREE_UNDO_SPLIT_R: + opname = "SPLIT_R"; + break; + case NBTREE_UNDO_NEWROOT: + opname = "NEWROOT"; + break; + case NBTREE_UNDO_DEDUP: + opname = "DEDUP"; + break; + case NBTREE_UNDO_VACUUM: + opname = "VACUUM"; + break; + default: + opname = "UNKNOWN"; + break; + } + + appendStringInfo(buf, "nbtree %s", opname); + + /* For types that have index_oid at the start of the payload, show it */ + if (payload_len >= sizeof(Oid) && + (info == NBTREE_UNDO_INSERT_LEAF || + info == NBTREE_UNDO_INSERT_UPPER || + info == NBTREE_UNDO_DEDUP || + info == NBTREE_UNDO_DELETE)) + { + Oid index_oid; + + memcpy(&index_oid, payload, sizeof(Oid)); + appendStringInfo(buf, " index %u", index_oid); + } +} diff --git a/src/include/access/hash.h b/src/include/access/hash.h index a8702f0e5ea13..ce8de4208ab7b 100644 --- a/src/include/access/hash.h +++ b/src/include/access/hash.h @@ -487,4 +487,9 @@ extern void hashbucketcleanup(Relation rel, Bucket cur_bucket, bool split_cleanup, IndexBulkDeleteCallback callback, void *callback_state); +/* hash_undo.c -- UNDO support */ +extern void HashUndoRmgrInit(void); +extern void HashUndoLogInsert(Relation rel, Relation heapRel, Buffer buf, + OffsetNumber offset); + #endif /* HASH_H */ diff --git a/src/include/access/nbtree.h b/src/include/access/nbtree.h index 3097e9bb1af9b..5ae836e96bc20 100644 --- a/src/include/access/nbtree.h +++ b/src/include/access/nbtree.h @@ -1331,4 +1331,43 @@ extern IndexBuildResult *btbuild(Relation heap, Relation index, struct IndexInfo *indexInfo); extern void _bt_parallel_build_main(dsm_segment *seg, shm_toc *toc); +/* + * nbtree UNDO support (nbtree_undo.c) + */ + +/* nbtree UNDO subtypes (stored in urec_info) */ +#define NBTREE_UNDO_INSERT_LEAF 0x0001 +#define NBTREE_UNDO_INSERT_UPPER 0x0002 +#define NBTREE_UNDO_INSERT_POST 0x0004 +#define NBTREE_UNDO_DELETE 0x0005 +#define NBTREE_UNDO_SPLIT_L 0x0006 +#define NBTREE_UNDO_SPLIT_R 0x0007 +#define NBTREE_UNDO_NEWROOT 0x0008 +#define NBTREE_UNDO_DEDUP 0x0009 +#define NBTREE_UNDO_VACUUM 0x000A + +/* + * NbtreeUndoInsertLeafHeader - Minimal payload header for INSERT_LEAF records + * + * This must match the first fields of the full NbtreeUndoInsertLeaf struct + * defined in nbtree_undo.c. Exposed here so the UNDO discard worker can + * extract (index_oid, blkno, offset) for targeted index pruning without + * depending on the full struct. + */ +typedef struct NbtreeUndoInsertLeafHeader +{ + Oid index_oid; /* OID of the index relation */ + BlockNumber blkno; /* Page where tuple was inserted */ + OffsetNumber offset; /* Offset of the inserted tuple */ +} NbtreeUndoInsertLeafHeader; + +#define SizeOfNbtreeUndoInsertLeafHeader \ + (offsetof(NbtreeUndoInsertLeafHeader, offset) + sizeof(OffsetNumber)) + +extern void NbtreeUndoRmgrInit(void); +extern void NbtreeUndoLogInsert(Relation rel, Relation heaprel, Buffer buf, + IndexTuple itup, Size itemsz, + OffsetNumber offset, bool isleaf); +extern void NbtreeUndoLogDedup(Relation rel, Relation heaprel, Buffer buf); + #endif /* NBTREE_H */ diff --git a/src/include/access/undormgrlist.h b/src/include/access/undormgrlist.h index 4ab60e2141881..f7d5ffb713071 100644 --- a/src/include/access/undormgrlist.h +++ b/src/include/access/undormgrlist.h @@ -31,3 +31,7 @@ */ /* there is deliberately not an #ifndef UNDORMGRLIST_H here */ + +/* built-in index AM UNDO resource managers */ +UNDO_RMGR_INIT(NbtreeUndoRmgrInit) +UNDO_RMGR_INIT(HashUndoRmgrInit) From 1465b4782fdbaa6f83ee4f0505ca80e652d9d557 Mon Sep 17 00:00:00 2001 From: Greg Burd Date: Mon, 27 Jul 2026 11:16:12 -0400 Subject: [PATCH 09/10] UNDO: add optional tuple sLog for bounded-recovery writer tracking Add the tuple sLog, an OPTIONAL extension of the sLog subsystem that gives in-place-MVCC table access methods bounded recovery and visibility cost. The tuple sLog is a flat, partitioned, seqlock-guarded hash of in-flight per-tuple operations keyed by (relid, tid). Because every hot tuple's uncommitted, aborted, and lock-holding writers are recorded in shared memory, recovery and cross-backend visibility can answer "is there an in-flight or aborted writer of this physical tuple?" in O(1) instead of scanning an unbounded UNDO chain. That O(1) answer is what bounds recovery time and per-read cost for an AM that updates tuples in place. Wait-free readers use the seqlock primitive (storage/seqlock.h): they take no lock and pay no atomic read-modify-write on the common path, retrying only if a writer intervened. Writers serialize per partition; the empty hash (the common case under committed-data OLTP) is skipped by a num_entries==0 fast path. The extension is OPTIONAL and consumer-agnostic: * The UNDO core and the transaction sLog (the Aborted Transaction Map in slog.c) do not require it. slog.c and slog_tuple.c share only the sLog shared-memory segment and its initialization; slog.c carries no flat-hash or partition knowledge. * An access method opts in by registering an SLogAmDescriptor (SLogRegisterAmDescriptor) once at startup. The descriptor carries only DATA -- resolved once at registration, never a per-op callback on the hot read/write path -- so opting in costs nothing on the tuple probe/insert fast path. Its live knob is before_image_max, the cap on the backend-local before-image an AM stashes for savepoint rollback. No consumer is named in the tuple sLog code, headers, or docs. --- src/backend/access/undo/Makefile | 2 + src/backend/access/undo/meson.build | 2 + src/backend/access/undo/slog.c | 31 +- src/backend/access/undo/slog_flathash.c | 796 +++++ src/backend/access/undo/slog_tuple.c | 2971 +++++++++++++++++ src/backend/utils/misc/guc_parameters.dat | 9 + src/backend/utils/misc/postgresql.conf.sample | 1 + src/include/access/slog.h | 290 ++ src/include/access/slog_flathash.h | 307 ++ src/include/access/slog_internal.h | 68 + 10 files changed, 4458 insertions(+), 19 deletions(-) create mode 100644 src/backend/access/undo/slog_flathash.c create mode 100644 src/backend/access/undo/slog_tuple.c create mode 100644 src/include/access/slog_flathash.h create mode 100644 src/include/access/slog_internal.h diff --git a/src/backend/access/undo/Makefile b/src/backend/access/undo/Makefile index f7273f30a6a57..3599eeed4cd5c 100644 --- a/src/backend/access/undo/Makefile +++ b/src/backend/access/undo/Makefile @@ -23,6 +23,8 @@ OBJS = \ relundo_worker.o \ relundo_xlog.o \ slog.o \ + slog_flathash.o \ + slog_tuple.o \ undo.o \ undo_bufmgr.o \ undo_xlog.o \ diff --git a/src/backend/access/undo/meson.build b/src/backend/access/undo/meson.build index c78763a296b29..8bdab6ef2da64 100644 --- a/src/backend/access/undo/meson.build +++ b/src/backend/access/undo/meson.build @@ -11,6 +11,8 @@ backend_sources += files( 'relundo_worker.c', 'relundo_xlog.c', 'slog.c', + 'slog_flathash.c', + 'slog_tuple.c', 'undo.c', 'undo_bufmgr.c', 'undo_xlog.c', diff --git a/src/backend/access/undo/slog.c b/src/backend/access/undo/slog.c index 8b5335225d6be..afc6a48f6e900 100644 --- a/src/backend/access/undo/slog.c +++ b/src/backend/access/undo/slog.c @@ -39,6 +39,7 @@ #include "access/relundo.h" #include "access/slog.h" +#include "access/slog_internal.h" #include "access/transam.h" #include "access/xact.h" #include "common/hashfn.h" @@ -121,26 +122,12 @@ slog_atm_key_reloid(uint64 key) /* * Initial size for the sLog DSA area (backs the aborted-txn radix tree). - * Grows dynamically as needed up to slog_dsa_max_size_mb. + * Grows dynamically as needed up to slog_dsa_max_size_mb. SLOG_DSA_INIT_SIZE + * and the shared-state struct are defined in access/slog_internal.h, shared + * with the optional tuple sLog (slog_tuple.c). */ -#define SLOG_DSA_INIT_SIZE (512 * 1024) /* 512 KB */ #define SLOG_DSA_MAX_SIZE_MB 256 /* default max: 256 MB */ -/* ---------------------------------------------------------------- - * Shared state definition - * ---------------------------------------------------------------- - */ -typedef struct SLogSharedState -{ - /* Transaction ATM (adaptive radix tree in the DSA area below) */ - dsa_pointer atm_handle; /* RT handle; InvalidDsaPointer until init */ - LWLockPadded txn_lock; /* single LWLock serializing ATM access */ - - /* DSA area backing the aborted-txn radix tree */ - dsa_area *dsa_area; /* set during SLogShmemInit, NULL until then */ - char dsa_space[SLOG_DSA_INIT_SIZE]; -} SLogSharedState; - /* GUC: maximum sLog DSA area size (in MB) */ int slog_dsa_max_size_mb = SLOG_DSA_MAX_SIZE_MB; @@ -148,7 +135,7 @@ int slog_dsa_max_size_mb = SLOG_DSA_MAX_SIZE_MB; * Static variables * ---------------------------------------------------------------- */ -static SLogSharedState *SLogState = NULL; +SLogSharedState *SLogState = NULL; /* Per-backend DSA attachment (lazy, via SLogEnsureDsaAttached) */ static dsa_area *slog_dsa_handle = NULL; @@ -207,7 +194,7 @@ slog_atm_tree(void) Size SLogShmemSize(void) { - return MAXALIGN(sizeof(SLogSharedState)); + return add_size(MAXALIGN(sizeof(SLogSharedState)), SLogTupleShmemSize()); } /* @@ -225,6 +212,9 @@ SLogShmemRequest(void) .size = sizeof(SLogSharedState), .ptr = (void **) &SLogState, ); + + /* Optional tuple sLog: register its flat-hash partition block. */ + SLogTupleShmemRequest(); } /* @@ -243,6 +233,9 @@ SLogShmemInit(void) /* ---- Initialize locks ---- */ LWLockInitialize(&SLogState->txn_lock.lock, LWTRANCHE_SLOG); + /* ---- Optional tuple sLog: allocate + init the flat-hash partitions ---- */ + SLogTupleShmemInit(); + /* ---- Initialize DSA area (backs the aborted-txn radix tree) ---- */ SLogState->dsa_area = dsa_create_in_place(SLogState->dsa_space, SLOG_DSA_INIT_SIZE, diff --git a/src/backend/access/undo/slog_flathash.c b/src/backend/access/undo/slog_flathash.c new file mode 100644 index 0000000000000..6896a1586eedc --- /dev/null +++ b/src/backend/access/undo/slog_flathash.c @@ -0,0 +1,796 @@ +/*------------------------------------------------------------------------- + * + * slog_flathash.c + * Seqlock-protected flat open-addressing hash for sLog tuple tracking. + * + * Implements the flat hash table operations (probe, insert, remove) and + * the apply handler that mutates the single hash copy. The hash uses + * linear probing with a power-of-2 capacity and tombstone markers. A + * per-partition seqlock (in SLogFlatPartition) provides retry-based + * consistent reads; the writer lock serializes mutations. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/access/undo/slog_flathash.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/slog_flathash.h" +#include "access/transam.h" +#include "common/hashfn.h" +#include "utils/dsa.h" + +/* + * Maximum probe distance before giving up. With load factor < 0.7 and + * power-of-2 sizing, typical probe chains are very short (< 5). + * We cap at 128 to bound worst-case scan time. + */ +#define SLOG_FLAT_MAX_PROBE 128 + +/* ---------------------------------------------------------------- + * Size computation + * ---------------------------------------------------------------- + */ + +/* + * SLogFlatHashDataSize + * Size of one copy of the flat hash data structure. + */ +Size +SLogFlatHashDataSize(int capacity) +{ + return offsetof(SLogFlatHash, buckets) + + (Size) capacity * sizeof(SLogFlatBucket); +} + +/* + * SLogFlatHashShmemSize + * Shared memory needed for one flat-hash copy (the seqlock guards a + * single copy in place; the sequence counter and writer lock live in + * the SLogFlatPartition struct, not here). + */ +Size +SLogFlatHashShmemSize(int capacity, int max_backends) +{ + (void) max_backends; /* retained for call-site compatibility */ + return MAXALIGN(SLogFlatHashDataSize(capacity)); +} + +/* + * SLogFlatHashPartitionedShmemSize + * Total shared memory needed for all partitions' flat-hash copies. + * + * Each partition gets capacity/N buckets in one copy. The per-partition + * seqlock counter and writer lock are embedded in SLogFlatPartition in the + * SLogSharedState, so only the hash copies are sized here. + */ +Size +SLogFlatHashPartitionedShmemSize(int total_capacity, int max_backends) +{ + int per_partition_capacity; + Size per_partition_size; + Size total; + + per_partition_capacity = total_capacity / SLogNumPartitions; + if (per_partition_capacity < 64) + per_partition_capacity = 64; + + per_partition_size = SLogFlatHashShmemSize(per_partition_capacity, + max_backends); + total = (Size) SLogNumPartitions * MAXALIGN(per_partition_size); + + return total; +} + +/* ---------------------------------------------------------------- + * Initialization + * ---------------------------------------------------------------- + */ + +/* + * SLogFlatHashInit + * Initialize a flat hash data block (one copy). + * + * Sets all buckets to EMPTY state. + */ +void +SLogFlatHashInit(void *data, int capacity) +{ + SLogFlatHash *ht = (SLogFlatHash *) data; + int i; + + ht->capacity = capacity; + ht->num_entries = 0; + ht->num_tombstones = 0; + ht->padding = 0; + + for (i = 0; i < capacity; i++) + { + ht->buckets[i].hash_val = SLOG_FLAT_EMPTY; + memset(&ht->buckets[i].key, 0, sizeof(SLogTupleKey)); + ht->buckets[i].padding = 0; + ht->buckets[i].entry.nops = 0; + memset(ht->buckets[i].entry.ops, 0, + sizeof(SLogTupleOp) * SLOG_MAX_TUPLE_OPS); + } +} + +/* ---------------------------------------------------------------- + * Hash function + * ---------------------------------------------------------------- + */ + +/* + * SLogFlatHashComputeHash + * Compute a 32-bit hash for an SLogTupleKey. + * + * The key must have been zeroed before population (for padding bytes). + * Returns a non-zero, non-TOMBSTONE value (adjusts if hash_bytes returns + * 0 or UINT32_MAX). + */ +uint32 +SLogFlatHashComputeHash(const SLogTupleKey *key) +{ + uint32 h; + + h = hash_bytes((const unsigned char *) key, sizeof(SLogTupleKey)); + + /* Ensure we never produce EMPTY or TOMBSTONE values */ + if (h == SLOG_FLAT_EMPTY) + h = 1; + else if (h == SLOG_FLAT_TOMBSTONE) + h = SLOG_FLAT_TOMBSTONE - 1; + + return h; +} + +/* ---------------------------------------------------------------- + * Probe operations + * ---------------------------------------------------------------- + */ + +/* + * SLogFlatHashProbe + * Look up a key in the flat hash. Returns bucket pointer if found, + * NULL if not present. + * + * Linear probing: start at hash_val % capacity, walk forward skipping + * tombstones, stop at EMPTY (not found) or matching key (found). + */ +SLogFlatBucket * +SLogFlatHashProbe(const SLogFlatHash *ht, const SLogTupleKey *key) +{ + uint32 h; + uint32 idx; + int probe; + + /* + * Fast path: an empty hash cannot contain the key, so skip the key hash + * computation and bucket walk entirely. This lookup sits on the calling + * AM's per-tuple visibility/conflict hot path and the sLog + * (uncommitted-writer tracking) is empty for the vast majority of lookups + * in a committed-data OLTP workload (measured ~4%% of AM CPU under a + * TPROC-C-style workload before this guard). Callers read the hash under + * the partition seqlock, so a concurrent writer's insert (which bumps + * num_entries) is caught by the seqlock retry. + */ + if (ht->num_entries == 0) + return NULL; + + h = SLogFlatHashComputeHash(key); + idx = h & (uint32) (ht->capacity - 1); + + for (probe = 0; probe < SLOG_FLAT_MAX_PROBE; probe++) + { + const SLogFlatBucket *bucket = &ht->buckets[idx]; + + if (bucket->hash_val == SLOG_FLAT_EMPTY) + return NULL; /* definitive miss */ + + if (bucket->hash_val != SLOG_FLAT_TOMBSTONE && + bucket->hash_val == h && + memcmp(&bucket->key, key, sizeof(SLogTupleKey)) == 0) + { + return (SLogFlatBucket *) bucket; /* found */ + } + + idx = (idx + 1) & (uint32) (ht->capacity - 1); + } + + return NULL; /* probe limit exceeded */ +} + +/* + * SLogFlatHashHasOpForXid + * Return true iff the entry for key holds an in-use op for xid. + * + * Used by SLogTupleInsert to confirm an op was actually stored, rather than + * silently dropped because the per-TID ops array was full. Probing the + * bucket alone is insufficient on a hot row, where the bucket pre-exists with + * other markers. + */ +bool +SLogFlatHashHasOpForXid(const SLogFlatHash *ht, const SLogTupleKey *key, + TransactionId xid) +{ + const SLogFlatBucket *bucket = SLogFlatHashProbe(ht, key); + const SLogTupleEntry *entry; + int i; + + if (bucket == NULL) + return false; + + entry = &bucket->entry; + for (i = 0; i < SLOG_MAX_TUPLE_OPS; i++) + { + if (entry->ops[i].in_use && + TransactionIdEquals(entry->ops[i].xid, xid)) + return true; + } + return false; +} + +/* + * SLogFlatHashProbeForInsert + * Find a slot for inserting a key. Returns the bucket to use. + * + * If the key already exists, returns that bucket (for update-in-place). + * Otherwise returns the first EMPTY or TOMBSTONE slot encountered. + * Returns NULL if probe limit exceeded without finding a slot. + */ +SLogFlatBucket * +SLogFlatHashProbeForInsert(SLogFlatHash *ht, const SLogTupleKey *key, + uint32 hash_val) +{ + uint32 idx; + int probe; + SLogFlatBucket *first_free = NULL; + + idx = hash_val & (uint32) (ht->capacity - 1); + + for (probe = 0; probe < SLOG_FLAT_MAX_PROBE; probe++) + { + SLogFlatBucket *bucket = &ht->buckets[idx]; + + if (bucket->hash_val == SLOG_FLAT_EMPTY) + { + /* Definitive miss ā€” use first_free if we found one, else this */ + return first_free ? first_free : bucket; + } + + if (bucket->hash_val == SLOG_FLAT_TOMBSTONE) + { + /* Remember first tombstone for potential reuse */ + if (first_free == NULL) + first_free = bucket; + } + else if (bucket->hash_val == hash_val && + memcmp(&bucket->key, key, sizeof(SLogTupleKey)) == 0) + { + /* Key already exists */ + return bucket; + } + + idx = (idx + 1) & (uint32) (ht->capacity - 1); + } + + /* Probe limit exceeded; return first_free if available */ + return first_free; +} + +/* ---------------------------------------------------------------- + * Apply handler (single-copy, under the seqlock writer) + * ---------------------------------------------------------------- + */ + +/* + * flat_hash_apply_insert + * Apply an INSERT operation: find/create entry, add op to slot. + */ +static void +flat_hash_apply_insert(SLogFlatHash *ht, const SLogFlatOp *op) +{ + uint32 h; + SLogFlatBucket *bucket; + SLogTupleEntry *entry; + int i; + bool existing; + + h = SLogFlatHashComputeHash(&op->key); + bucket = SLogFlatHashProbeForInsert(ht, &op->key, h); + + if (bucket == NULL) + { + elog(WARNING, "SLOG_LOST_OP table_full relid=%u xid=%u op=%d", + op->key.relid, op->tuple_op.xid, (int) op->tuple_op.op_type); + return; /* table full, operation lost */ + } + + /* Determine if this is an existing entry */ + existing = (bucket->hash_val != SLOG_FLAT_EMPTY && + bucket->hash_val != SLOG_FLAT_TOMBSTONE); + + if (!existing) + { + /* New entry */ + if (bucket->hash_val == SLOG_FLAT_TOMBSTONE) + ht->num_tombstones--; + + bucket->hash_val = h; + memcpy(&bucket->key, &op->key, sizeof(SLogTupleKey)); + bucket->entry.nops = 0; + memset(&bucket->entry.key, 0, sizeof(SLogTupleKey)); + memcpy(&bucket->entry.key, &op->key, sizeof(SLogTupleKey)); + memset(bucket->entry.ops, 0, sizeof(bucket->entry.ops)); + ht->num_entries++; + } + + entry = &bucket->entry; + + /* Check if this xid already has an op (overwrite) */ + for (i = 0; i < SLOG_MAX_TUPLE_OPS; i++) + { + if (entry->ops[i].in_use && + entry->ops[i].xid == op->tuple_op.xid) + { + /* Overwrite existing op for same xid */ + memcpy(&entry->ops[i], &op->tuple_op, sizeof(SLogTupleOp)); + return; + } + } + + /* Find a free slot */ + for (i = 0; i < SLOG_MAX_TUPLE_OPS; i++) + { + if (!entry->ops[i].in_use) + { + memcpy(&entry->ops[i], &op->tuple_op, sizeof(SLogTupleOp)); + entry->nops++; + return; + } + } + + /* + * No free slot ā€” reclaim the oldest horizon-eligible retained UPDATE + * marker. + * + * Hot rows (e.g. a TPC-C district) accumulate one retained UPDATE marker + * per committed transaction. Under WS-PVS3 committed UPDATE markers are + * removed at commit (flat_hash_apply_commit_xid), so in steady state the + * array only fills with in-progress writers or WS-PVS3 stragglers; this + * path drains the latter. + */ + { + int oldest_idx = -1; + TransactionId reclaim_xid_horizon = op->reclaim_xid_horizon; + + /* + * Take the first horizon-eligible UPDATE marker. A marker is + * reclaimable once its xid precedes the oldest active snapshot's xmin + * (reclaim_xid_horizon): at that point the xid's outcome is settled + * and visible to (or irrelevant to) every live snapshot, so no + * residual consumer can still need it (WS-PVS3 moved the write-write + * conflict probe and MVCC read to the durable UNDO fork chain). An + * in-progress xid can never precede this horizon, so this test alone + * never frees a marker a concurrent reader or writer still needs. + * Every marker that passes the horizon predicate is equally + * reclaimable, so we take the first one found. + * + * An INVALID reclaim_xid_horizon disables reclamation entirely: every + * marker is treated as non-reclaimable and we fall through to + * oldest_idx == -1. This is the fail-safe direction and is exactly + * what the caller relies on -- SLogTupleInsert passes + * InvalidTransactionId on the fast path (skipping the ProcArrayLock + * horizon scan) and only supplies a real, freshly-computed horizon on + * the full-array retry. A missing horizon must never make markers + * look reclaimable, or a live reader's / in-progress writer's op + * could be freed (lost update / MVCC corruption). + * + * The xid horizon is the authoritative, self-healing reclaim gate: a + * below-horizon marker is reclaimable whether or not it was ever + * committed in CLOG. + */ + for (i = 0; i < SLOG_MAX_TUPLE_OPS; i++) + { + if (!entry->ops[i].in_use) + continue; + if (entry->ops[i].op_type != SLOG_OP_UPDATE) + continue; + if (!TransactionIdIsValid(entry->ops[i].xid)) + continue; + if (!TransactionIdIsValid(reclaim_xid_horizon) || + !TransactionIdPrecedes(entry->ops[i].xid, reclaim_xid_horizon)) + continue; /* invalid horizon disables reclaim + * (fail-safe); else still visible-relevant to + * an active snapshot, or in-progress */ + oldest_idx = i; + break; + } + + if (oldest_idx >= 0) + { + /* Safe to reclaim ā€” no active snapshot needs this entry */ + entry->ops[oldest_idx].in_use = false; + entry->nops--; + + /* Now insert the new op in the freed slot */ + memcpy(&entry->ops[oldest_idx], &op->tuple_op, sizeof(SLogTupleOp)); + entry->nops++; + return; + } + } + + /* + * No reclaimable slot: every one of the SLOG_MAX_TUPLE_OPS slots holds an + * UPDATE whose xid is still at/above the reclaim horizon (in-progress or + * very recently committed) or a non-UPDATE marker. With the xid-horizon + * reclaim above this requires SLOG_MAX_TUPLE_OPS concurrent unsettled + * writers on a single TID, which is not reachable under normal load. The + * caller (SLogTupleInsert) detects the drop via SLogFlatHashHasOpForXid + * and falls back to local-only tracking + UNDO replay, so this is safe + * but worth surfacing. + */ + elog(WARNING, "SLOG_LOST_OP ops_full relid=%u xid=%u op=%d nops=%d horizon=%u", + op->key.relid, op->tuple_op.xid, (int) op->tuple_op.op_type, + entry->nops, op->reclaim_xid_horizon); +} + +/* + * flat_hash_apply_remove_xid + * Remove all ops for a given xid from an entry. + * Remove the entry entirely if nops reaches 0. + */ +static void +flat_hash_apply_remove_xid(SLogFlatHash *ht, const SLogFlatOp *op) +{ + SLogFlatBucket *bucket; + SLogTupleEntry *entry; + int i; + + bucket = SLogFlatHashProbe(ht, &op->key); + if (bucket == NULL) + return; + + entry = &bucket->entry; + + for (i = 0; i < SLOG_MAX_TUPLE_OPS; i++) + { + if (entry->ops[i].in_use && + entry->ops[i].xid == op->xid) + { + entry->ops[i].in_use = false; + entry->nops--; + } + } + + if (entry->nops == 0) + { + bucket->hash_val = SLOG_FLAT_TOMBSTONE; + ht->num_entries--; + ht->num_tombstones++; + } +} + +/* + * flat_hash_apply_remove_entry + * Remove an entire entry (tombstone it). + */ +static void +flat_hash_apply_remove_entry(SLogFlatHash *ht, const SLogFlatOp *op) +{ + SLogFlatBucket *bucket; + + bucket = SLogFlatHashProbe(ht, &op->key); + if (bucket == NULL) + return; + + bucket->hash_val = SLOG_FLAT_TOMBSTONE; + ht->num_entries--; + ht->num_tombstones++; +} + +/* + * flat_hash_apply_mark_aborted + * Mark all ops for a given xid as SLOG_OP_ABORTED. + */ +static void +flat_hash_apply_mark_aborted(SLogFlatHash *ht, const SLogFlatOp *op) +{ + SLogFlatBucket *bucket; + SLogTupleEntry *entry; + int i; + + bucket = SLogFlatHashProbe(ht, &op->key); + if (bucket == NULL) + return; + + entry = &bucket->entry; + + for (i = 0; i < SLOG_MAX_TUPLE_OPS; i++) + { + if (entry->ops[i].in_use && + entry->ops[i].xid == op->xid) + { + entry->ops[i].op_type = SLOG_OP_ABORTED; + } + } +} + +/* + * flat_hash_apply_update_op + * Update a specific op slot in place (e.g. re-parent a subxid). + */ +static void +flat_hash_apply_update_op(SLogFlatHash *ht, const SLogFlatOp *op) +{ + SLogFlatBucket *bucket; + SLogTupleEntry *entry; + int i; + + bucket = SLogFlatHashProbe(ht, &op->key); + if (bucket == NULL) + return; + + entry = &bucket->entry; + + for (i = 0; i < SLOG_MAX_TUPLE_OPS; i++) + { + if (entry->ops[i].in_use && + entry->ops[i].xid == op->xid) + { + /* + * Apply selective updates from the op. We use the tuple_op + * fields as the source of truth for what to update. + */ + if (op->subxid != InvalidTransactionId) + { + /* Re-parent subxid */ + if (entry->ops[i].subxid == op->tuple_op.subxid) + entry->ops[i].subxid = op->subxid; + } + break; + } + } +} + +/* + * flat_hash_apply_commit_xid + * Remove every op belonging to xid. + * + * WS-PVS3 Phase 2: committed UPDATE markers are no longer retained here. + * The write-write conflict probe now reads the head verptr on the tuple + * and resolves it in the durable UNDO fork (via the AM's version-reconstruction + * walk), and the shared before-image was dropped in Phase 1. With no consumer + * for a retained marker, an UPDATE at commit is treated like INSERT/DELETE/LOCK: + * removed. This drains bucket table_full pressure. + */ +static void +flat_hash_apply_commit_xid(SLogFlatHash *ht, const SLogFlatOp *op) +{ + SLogFlatBucket *bucket; + SLogTupleEntry *entry; + int i; + + bucket = SLogFlatHashProbe(ht, &op->key); + if (bucket == NULL) + return; + + entry = &bucket->entry; + + for (i = 0; i < SLOG_MAX_TUPLE_OPS; i++) + { + if (!entry->ops[i].in_use) + continue; + if (entry->ops[i].xid != op->xid) + continue; + + entry->ops[i].in_use = false; + entry->nops--; + } + + if (entry->nops == 0) + { + bucket->hash_val = SLOG_FLAT_TOMBSTONE; + ht->num_entries--; + ht->num_tombstones++; + } +} + +/* + * flat_hash_apply_cleanup_retained + * Remove retained committed UPDATE markers whose committing xid + * precedes the reclaim xid horizon (op->reclaim_xid_horizon). + * + * Gates on the xid horizon -- NOT an HLC threshold -- so a marker is + * reclaimed only when its committing xid precedes every live snapshot. + * The write-write conflict probe and MVCC read now use the durable UNDO + * fork chain, not this hash, so retained UPDATE markers here are dead + * bookkeeping under WS-PVS3; this sweep still applies for any leftover + * entries not removed at commit. + */ +static void +flat_hash_apply_cleanup_retained(SLogFlatHash *ht, const SLogFlatOp *op) +{ + SLogFlatBucket *bucket; + SLogTupleEntry *entry; + int i; + + bucket = SLogFlatHashProbe(ht, &op->key); + if (bucket == NULL) + return; + + entry = &bucket->entry; + + for (i = 0; i < SLOG_MAX_TUPLE_OPS; i++) + { + if (!entry->ops[i].in_use) + continue; + if (entry->ops[i].op_type != SLOG_OP_UPDATE) + continue; + if (!TransactionIdIsValid(entry->ops[i].xid)) + continue; + if (TransactionIdIsValid(op->reclaim_xid_horizon) && + !TransactionIdPrecedes(entry->ops[i].xid, + op->reclaim_xid_horizon)) + continue; + + /* + * Expired retained entry. Gated on the xid horizon alone -- a marker + * committed in CLOG or an aborted/in-flight one is drained the moment + * its xid precedes the horizon, so no slot is pinned forever. Must + * match the read-side eligibility scan in SLogTupleCleanupRetained. + */ + entry->ops[i].in_use = false; + entry->nops--; + } + + if (entry->nops == 0) + { + bucket->hash_val = SLOG_FLAT_TOMBSTONE; + ht->num_entries--; + ht->num_tombstones++; + } +} + +/* + * flat_hash_apply_create_aborted + * Create a new entry with an ABORTED op (for local-only INSERT abort). + */ +static void +flat_hash_apply_create_aborted(SLogFlatHash *ht, const SLogFlatOp *op) +{ + uint32 h; + SLogFlatBucket *bucket; + SLogTupleEntry *entry; + int i; + bool existing; + + h = SLogFlatHashComputeHash(&op->key); + bucket = SLogFlatHashProbeForInsert(ht, &op->key, h); + + if (bucket == NULL) + return; /* table full */ + + existing = (bucket->hash_val != SLOG_FLAT_EMPTY && + bucket->hash_val != SLOG_FLAT_TOMBSTONE); + + if (!existing) + { + if (bucket->hash_val == SLOG_FLAT_TOMBSTONE) + ht->num_tombstones--; + + bucket->hash_val = h; + memcpy(&bucket->key, &op->key, sizeof(SLogTupleKey)); + bucket->entry.nops = 0; + memset(&bucket->entry.key, 0, sizeof(SLogTupleKey)); + memcpy(&bucket->entry.key, &op->key, sizeof(SLogTupleKey)); + memset(bucket->entry.ops, 0, sizeof(bucket->entry.ops)); + ht->num_entries++; + } + + entry = &bucket->entry; + + /* Find a free slot for the ABORTED entry */ + for (i = 0; i < SLOG_MAX_TUPLE_OPS; i++) + { + if (!entry->ops[i].in_use) + { + entry->ops[i].xid = op->xid; + entry->ops[i].subxid = op->subxid; + entry->ops[i].op_type = SLOG_OP_ABORTED; + entry->ops[i].in_use = true; + entry->ops[i].commit_ts = 0; + entry->ops[i].spec_token = 0; + entry->ops[i].cid = InvalidCommandId; + entry->nops++; + return; + } + } + /* No free slot ā€” operation lost */ +} + +/* + * SLogFlatHashApply + * Apply one op to the flat hash. Dispatches to operation-specific handlers. + */ +void +SLogFlatHashApply(void *data, const void *operation, Size op_size) +{ + SLogFlatHash *ht = (SLogFlatHash *) data; + const SLogFlatOp *op = (const SLogFlatOp *) operation; + + Assert(op_size == sizeof(SLogFlatOp)); + + switch (op->kind) + { + case SLOG_FLAT_OP_INSERT: + flat_hash_apply_insert(ht, op); + break; + case SLOG_FLAT_OP_REMOVE_XID: + flat_hash_apply_remove_xid(ht, op); + break; + case SLOG_FLAT_OP_REMOVE_ENTRY: + flat_hash_apply_remove_entry(ht, op); + break; + case SLOG_FLAT_OP_MARK_ABORTED: + flat_hash_apply_mark_aborted(ht, op); + break; + case SLOG_FLAT_OP_UPDATE_OP: + flat_hash_apply_update_op(ht, op); + break; + case SLOG_FLAT_OP_COMMIT_XID: + flat_hash_apply_commit_xid(ht, op); + break; + case SLOG_FLAT_OP_CLEANUP_RETAINED: + flat_hash_apply_cleanup_retained(ht, op); + break; + case SLOG_FLAT_OP_CREATE_ABORTED: + flat_hash_apply_create_aborted(ht, op); + break; + } +} + +/* ---------------------------------------------------------------- + * Scan API + * ---------------------------------------------------------------- + */ + +/* + * SLogFlatHashScanInit + * Initialize a sequential scan over the flat hash. + */ +void +SLogFlatHashScanInit(SLogFlatHashScanState *state) +{ + state->current_index = 0; +} + +/* + * SLogFlatHashScanNext + * Return the next occupied bucket, or NULL when the scan is complete. + * + * Skips EMPTY and TOMBSTONE slots. The returned pointer is valid only + * within the current seqlock read section (or under the writer lock). + */ +const SLogFlatBucket * +SLogFlatHashScanNext(const SLogFlatHash *ht, SLogFlatHashScanState *state) +{ + while (state->current_index < ht->capacity) + { + const SLogFlatBucket *bucket = &ht->buckets[state->current_index]; + + state->current_index++; + + if (bucket->hash_val != SLOG_FLAT_EMPTY && + bucket->hash_val != SLOG_FLAT_TOMBSTONE) + { + return bucket; + } + } + + return NULL; +} diff --git a/src/backend/access/undo/slog_tuple.c b/src/backend/access/undo/slog_tuple.c new file mode 100644 index 0000000000000..19f3251e9f742 --- /dev/null +++ b/src/backend/access/undo/slog_tuple.c @@ -0,0 +1,2971 @@ +/*------------------------------------------------------------------------- + * + * slog_tuple.c + * Tuple sLog -- optional per-tuple UNDO tracking extension + * + * The tuple sLog is an OPTIONAL extension of the sLog subsystem (slog.c). + * It provides bounded-recovery-time uncommitted-writer tracking for in-place + * MVCC table access methods: a flat, partitioned, seqlock-guarded hash of + * in-flight per-tuple operations, keyed by (relid, tid). Because the + * uncommitted writers of every hot tuple are recorded in shared memory, + * recovery and cross-backend visibility need not scan an unbounded UNDO + * chain to decide whether a given physical tuple version is visible; the + * hash answers "is there an in-flight/aborted writer of this tuple?" in + * O(1), which is what bounds recovery and read cost. + * + * It is OPTIONAL: an access method opts in by registering an + * SLogAmDescriptor (SLogRegisterAmDescriptor), which supplies the AM's + * policy (e.g. the maximum before-image size it will stash). The UNDO core + * and the transaction sLog (the Aborted Transaction Map in slog.c) do not + * require this file; a build with no in-place-MVCC AM never calls into it. + * + * The flat hash and its shared-memory partitions are carved from the same + * shared segment as the transaction sLog (see slog.c's SLogShmemInit); the + * two facilities share only that segment and its initialization, never each + * other's data structures. + * + * WAL-free: entries are transient, removed at commit/abort. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/access/undo/slog_tuple.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include +#ifdef WIN32 +#include +#endif + +#include "access/relundo.h" +#include "access/slog.h" +#include "access/slog_flathash.h" +#include "access/slog_internal.h" +#include "access/transam.h" +#include "access/xact.h" +#include "common/hashfn.h" +#include "nodes/lockoptions.h" +#include "miscadmin.h" +#include "storage/lock.h" +#include "storage/lwlock.h" +#include "storage/off.h" +#include "storage/proc.h" +#include "storage/procarray.h" +#include "storage/shmem.h" +#include "storage/spin.h" +#include "utils/dsa.h" +#include "utils/memutils.h" +#include "utils/snapmgr.h" +#include "utils/timestamp.h" + + +/* + * slog_num_cpus - portable replacement for sysconf(_SC_NPROCESSORS_ONLN). + * + * Returns the number of online CPUs (or a sane positive fallback if the + * platform refuses to answer). Used to auto-size the sLog flat-hash + * partition count when the GUC is left at zero. + */ +static int +slog_num_cpus(void) +{ +#ifdef WIN32 + SYSTEM_INFO si; + + GetSystemInfo(&si); + if (si.dwNumberOfProcessors > 0) + return (int) si.dwNumberOfProcessors; + return 4; +#else + long n = sysconf(_SC_NPROCESSORS_ONLN); + + if (n <= 0) + return 4; + return (int) n; +#endif +} + + +/* + * Maximum number of abort ops applied to a partition between seqlock + * begin/end cycles in SLogTupleMarkAborted / SLogTupleCleanupRetained. + * + * The seqlock keeps its counter odd for the whole write cycle, so any + * concurrent wait-free reader spins until the counter turns even. A very + * large rollback (hundreds of thousands of tuples applied in one cycle) + * would pin readers spinning for the entire batch. Ending and re-beginning + * the seqlock every SLOG_ABORT_PUBLISH_BATCH ops bounds that odd-hold + * window; the partition writer_lock is held across the whole function, so + * dropping to even at a batch boundary leaves the partition consistent for + * any reader that observes it. + */ +#define SLOG_ABORT_PUBLISH_BATCH 24 + + +/* + * Wall-clock sampling period for the self-clocking throttles in + * SLogTupleInsert() and SLogTupleMaybeCleanupRetained(). GetCurrentTimestamp() + * is a clock_gettime() syscall that dominates the per-tuple hot path, so we + * read it once every SLOG_INSERT_CLOCK_PERIOD calls and reuse the cached + * horizon in between. Must be a power of two (callers mask with PERIOD - 1). + */ +#define SLOG_INSERT_CLOCK_PERIOD 256 + + +/* + * SLogTupleNumEntries + * Calculate the number of hash entries for the tuple sLog. + * + * Auto-sizing formula: MaxBackends * 1024, clamped to [4096, 4194304]. + */ +int +SLogTupleNumEntries(void) +{ + int n = MaxBackends * SLOG_TUPLE_PER_BACKEND_SLOTS; + + n = Max(n, SLOG_TUPLE_MIN_ENTRIES); + n = Min(n, SLOG_TUPLE_MAX_ENTRIES); + return n; +} + + + +/* + * GUC: slog_num_partitions ā€” number of flat hash partitions. + * 0 = auto (heuristic: 4 Ɨ NumCPUs, clamped [16..256], power of 2). + * Set at postmaster startup, immutable thereafter. + */ +int slog_num_partitions = 0; + +/* Runtime partition count (set once during SLogShmemInit, read everywhere) */ +int SLogNumPartitions = SLOG_FLAT_DEFAULT_PARTITIONS; + +/* + * Compute the effective partition count from the GUC value. + * Called once during SLogShmemInit. + */ +static int +SLogComputeNumPartitions(void) +{ + int n; + + if (slog_num_partitions > 0) + { + /* Explicit GUC value ā€” clamp and round to power of 2 */ + n = slog_num_partitions; + } + else + { + /* Auto-size: 4 x number of CPUs */ + int ncpus = slog_num_cpus(); + + n = ncpus * 4; + } + + /* Clamp */ + n = Max(n, SLOG_FLAT_MIN_PARTITIONS); + n = Min(n, SLOG_FLAT_MAX_PARTITIONS); + + /* Round up to next power of 2 (for fast modulo via bitmask) */ + { + int p = 1; + + while (p < n) + p <<= 1; + n = p; + } + + return n; +} + +/* + * Flat hash capacity: round up SLogTupleNumEntries to next power of 2, + * then divide by 0.7 (max load factor) to ensure probe chains stay short. + */ +static inline int +SLogFlatHashCapacity(void) +{ + int n = SLogTupleNumEntries(); + int cap; + + /* Target: num_entries / capacity <= 0.7, so capacity >= n / 0.7 */ + cap = (int) ((double) n / 0.7) + 1; + + /* Round up to next power of 2 */ + cap--; + cap |= cap >> 1; + cap |= cap >> 2; + cap |= cap >> 4; + cap |= cap >> 8; + cap |= cap >> 16; + cap++; + + /* Clamp to reasonable bounds */ + if (cap < 2048) + cap = 2048; + if (cap > 2 * 1048576) + cap = 2 * 1048576; + + return cap; +} + +/* ---------------------------------------------------------------- + * Static variables (tuple sLog) + * ---------------------------------------------------------------- + */ + +/* Pointer to ShmemAlloc'd flat-hash region, set by ShmemRequestStruct framework */ +static char *SLogFlatHashBlock = NULL; /* single allocation for all partition + * flat-hash copies */ + +/* Rate-limiting for sLog overflow warnings (per-backend) */ +static int slog_overflow_warning_count = 0; +static TimestampTz slog_overflow_last_warning = 0; + + +/* ---------------------------------------------------------------- + * Backend-private tracking for tuple sLog cleanup + * ---------------------------------------------------------------- + */ +typedef struct SLogTrackedKey +{ + SLogTupleKey key; + TransactionId xid; + TransactionId subxid; + bool local_only; /* no shared hash entry (INSERT-only) */ + SLogOpType op_type; /* DML type (for commit retention decisions) */ + + /* Before-image for savepoint rollback (NULL if not applicable) */ + char *before_image; /* palloc'd copy of tuple data, or NULL */ + int before_image_len; /* length of before_image data */ + uint16 before_flags; /* original t_flags before DML */ + uint64 before_commit_ts; /* original t_commit_ts before DML */ + + /* + * Physical relation locator captured at store time. Savepoint-abort + * restore runs in TRANS_ABORT state where relation_open's relcache lookup + * is unsafe, so we read the buffer via ReadBufferWithoutRelcache instead. + */ + RelFileLocator before_rlocator; + char before_relpersistence; + + struct SLogTrackedKey *next; +} SLogTrackedKey; + +static SLogTrackedKey *slog_tracked_keys = NULL; +static bool slog_has_shared_entries = false; /* any non-local_only entries? */ + +/* ---------------------------------------------------------------- + * TID encoding for backend-local INSERT tracking + * + * Each (blkno, offnum) pair maps to a dense 64-bit key used by the + * backend-local INSERT-tracking hash (see below): + * + * blkno * MaxOffsetNumber + (offnum - 1) + * + * Losslessness invariant: this key is in-memory and per-backend only -- + * never written to WAL or disk -- so the divisor need only be strictly + * greater than any real offnum a tracked page can hold. MaxOffsetNumber + * is the maximum number of line pointers any page can hold and is an + * AM-agnostic upper bound: offnum <= MaxOffsetNumber always holds, so the + * divisor never aliases two pages' TIDs onto the same key. A check at each + * encode site enforces this; an out-of-range offset is a corruption bug, + * not a recoverable condition, so it raises ERROR even in production builds. + * ---------------------------------------------------------------- + */ +#define SLOG_ENCODE_TID(blkno, offnum) \ + ((uint64) (blkno) * MaxOffsetNumber + (uint64) ((offnum) - 1)) +#define SLOG_DECODE_BLKNO(encoded) \ + ((BlockNumber) ((encoded) / MaxOffsetNumber)) +#define SLOG_DECODE_OFFNUM(encoded) \ + ((OffsetNumber) ((encoded) % MaxOffsetNumber + 1)) + +/* ---------------------------------------------------------------- + * Backend-local INSERT tracking (simplehash) + * + * Top-level local-only INSERTs are recorded in a single backend-local + * open-addressing hash keyed by (relid, encoded_tid). This replaces the + * former per-relid linked-list-of-sparsemaps, which forced an O(n) chunk + * scan on every per-tuple visibility probe. The hash lives in + * TopTransactionContext and is destroyed per transaction. No locks: it is + * strictly backend-private. + * ---------------------------------------------------------------- + */ +typedef struct SLogInsertTidKey +{ + Oid relid; + uint64 encoded_tid; /* SLOG_ENCODE_TID(blkno, offnum) */ +} SLogInsertTidKey; + +typedef struct SLogInsertTidEntry +{ + SLogInsertTidKey key; /* (relid, encoded_tid) */ + char status; /* required by simplehash */ +} SLogInsertTidEntry; + +static inline uint32 +slog_insert_tid_hash(Oid relid, uint64 encoded_tid) +{ + uint64 h = hash_combine64((uint64) hash_uint32((uint32) relid), + murmurhash64(encoded_tid)); + + return (uint32) (h ^ (h >> 32)); +} + +#define SH_PREFIX sloginsert +#define SH_ELEMENT_TYPE SLogInsertTidEntry +#define SH_KEY_TYPE SLogInsertTidKey +#define SH_KEY key +#define SH_HASH_KEY(tb, key) slog_insert_tid_hash((key).relid, (key).encoded_tid) +#define SH_EQUAL(tb, a, b) ((a).relid == (b).relid && (a).encoded_tid == (b).encoded_tid) +#define SH_SCOPE static inline +#define SH_DEFINE +#define SH_DECLARE +#include "lib/simplehash.h" + +static sloginsert_hash *slog_insert_tids = NULL; + +/* + * Lazily create the backend-local INSERT hash in TopTransactionContext. + * Returns the (now non-NULL) table. + */ +static inline sloginsert_hash * +slog_insert_tids_ensure(void) +{ + if (slog_insert_tids == NULL) + slog_insert_tids = sloginsert_create(TopTransactionContext, 256, NULL); + return slog_insert_tids; +} + +/* ---------------------------------------------------------------- + * Internal helpers + * ---------------------------------------------------------------- + */ + +/* + * Partition accessor helpers for the tuple sLog. + * + * These inline functions encapsulate the keyā†’partition routing so that + * callers don't need to repeat the pattern. + */ +static inline SLogFlatPartition * +SLogGetPartition(const SLogTupleKey *key) +{ + int part = SLogFlatHashPartitionIndex(key); + + return &SLogState->tuple_partitions[part]; +} + +static inline SLogFlatPartition * +SLogGetPartitionByIndex(int part) +{ + Assert(part >= 0 && part < SLogNumPartitions); + return &SLogState->tuple_partitions[part]; +} + +/* + * Convenience macros for partition-routed locking. + * + * Most functions have a local `key` variable of type SLogTupleKey and need + * to route to the correct partition. These macros minimize the diff from + * the old single-lock code. The `fp__` variable is defined in-scope by + * SLOG_PART_READ_BEGIN / SLOG_PART_WRITE_BEGIN. + */ +#define SLOG_PART_WRITER_LOCK(key_ptr) \ + (&SLogGetPartition(key_ptr)->writer_lock.lock) + +/* + * SLogPartApplyOne + * Single-op write helper: acquire writer_lock, one seqlock cycle, one + * apply, release. Covers the many write sites that apply exactly one + * SLogFlatOp to fp->hash. + */ +static inline void +SLogPartApplyOne(SLogFlatPartition *fp, const SLogFlatOp *op) +{ + LWLockAcquire(&fp->writer_lock.lock, LW_EXCLUSIVE); + SLogSeqWriteBegin(fp); + SLogFlatHashApply(fp->hash, op, sizeof(*op)); + SLogSeqWriteEnd(fp); + LWLockRelease(&fp->writer_lock.lock); +} + +/* ---------------------------------------------------------------- + * Per-AM descriptor (opt-in policy) + * + * An access method that wants tuple-sLog tracking registers a descriptor + * once, at startup (before shared memory is used). The descriptor carries + * only DATA -- policy resolved once, never a per-op callback on the hot + * read/write path -- so it costs nothing on the tuple probe/insert fast + * path. Its sole live knob today is before_image_max, the cap on the + * backend-local before-image the AM stashes for savepoint rollback. + * + * ponytail: data-descriptor only, resolved once at registration; NO per-op + * callback on the hot path. An opaque per-op payload and a pluggable key + * type are deferred until a second in-place-MVCC AM exists (YAGNI). + * ---------------------------------------------------------------- + */ +static SLogAmDescriptor slog_am_desc = { + .before_image_max = 0, /* 0 = no AM registered / before-images + * disabled */ +}; + +/* + * SLogRegisterAmDescriptor + * Record the opting-in AM's tuple-sLog policy. + * + * Called once at startup. Copies the caller's descriptor by value. + */ +void +SLogRegisterAmDescriptor(const SLogAmDescriptor *desc) +{ + Assert(desc != NULL); + slog_am_desc = *desc; +} + +/* ---------------------------------------------------------------- + * Tuple sLog shared-memory sizing and initialization + * + * Called from slog.c's SLogShmemSize/Request/Init so the tuple flat-hash + * partitions share the sLog's shared segment. Keeping the tuple-specific + * shmem code here (rather than in slog.c) keeps slog.c free of any + * flat-hash/partition knowledge. + * ---------------------------------------------------------------- + */ + +/* + * SLogTupleShmemSize + * Additional shared memory needed for the tuple flat-hash partitions. + */ +Size +SLogTupleShmemSize(void) +{ + return SLogFlatHashPartitionedShmemSize(SLogFlatHashCapacity(), MaxBackends); +} + +/* + * SLogTupleShmemRequest + * Register the tuple flat-hash partition block. + */ +void +SLogTupleShmemRequest(void) +{ + /* Compute partition count early so shmem sizing is correct */ + SLogNumPartitions = SLogComputeNumPartitions(); + + ShmemRequestStruct(.name = "sLog Flat Hash Partitions", + .size = SLogFlatHashPartitionedShmemSize( + SLogFlatHashCapacity(), MaxBackends), + .ptr = (void **) &SLogFlatHashBlock, + ); +} + +/* + * SLogTupleShmemInit + * Allocate and initialize the tuple flat-hash partitions. + * + * Invoked from SLogShmemInit() after the shared state struct exists. Runs + * unconditionally (whether or not an AM has opted in) so the two sLog files + * share one segment with no init-ordering dependency; an idle flat hash + * costs only its shared-memory footprint. + */ +void +SLogTupleShmemInit(void) +{ + int total_capacity; + int per_part_cap; + Size per_part_shmem_size; + char *block_ptr; + int part; + + /* ---- Compute and set partition count (once, globally) ---- */ + SLogNumPartitions = SLogComputeNumPartitions(); + SLogState->num_partitions = SLogNumPartitions; + + /* Allocate partition array in shared memory (after SLogState) */ + SLogState->tuple_partitions = (SLogFlatPartition *) + ShmemAlloc(sizeof(SLogFlatPartition) * SLogNumPartitions); + memset(SLogState->tuple_partitions, 0, + sizeof(SLogFlatPartition) * SLogNumPartitions); + + ereport(DEBUG1, + (errmsg("sLog: %d flat hash partitions (slog_num_partitions=%d, CPUs=%d)", + SLogNumPartitions, slog_num_partitions, + slog_num_cpus()))); + + /* ---- Initialize partitioned seqlock flat hashes ---- */ + total_capacity = SLogFlatHashCapacity(); + per_part_cap = total_capacity / SLogNumPartitions; + if (per_part_cap < 64) + per_part_cap = 64; + + per_part_shmem_size = MAXALIGN(SLogFlatHashShmemSize(per_part_cap, + MaxBackends)); + block_ptr = SLogFlatHashBlock; + + for (part = 0; part < SLogNumPartitions; part++) + { + SLogFlatPartition *fp = &SLogState->tuple_partitions[part]; + + /* Initialize per-partition writer lock */ + LWLockInitialize(&fp->writer_lock.lock, LWTRANCHE_SLOG); + + /* Seqlock counter starts even (stable, no writer in progress) */ + SeqLockInit(&fp->seqlock); + + /* Single flat-hash copy carved from the shared block */ + fp->hash = (SLogFlatHash *) block_ptr; + SLogFlatHashInit(fp->hash, per_part_cap); + + block_ptr += per_part_shmem_size; + } +} + +/* ================================================================ + * Tuple sLog functions + * ================================================================ + */ + +/* ---------------------------------------------------------------- + * Emergency eviction + * ---------------------------------------------------------------- + */ + +/* + * SLogTupleEvictCommitted -- evict entries for committed transactions. + * + * Called when the sLog flat hash is full. Scans each partition under its + * writer lock to collect evictable keys, then applies REMOVE_ENTRY ops. + * (This is a cold path, run only when a partition is full; holding the + * writer lock across the scan is simpler than a seqlock retry loop and the + * per-op TransactionIdIsInProgress/DidCommit probes must not run repeatedly.) + * + * Returns the number of entries evicted. + */ +static int +SLogTupleEvictCommitted(void) +{ + SLogTupleKey *keys_to_evict; + int nkeys = 0; + int max_evict = 1024; + int part; + + keys_to_evict = (SLogTupleKey *) + palloc(sizeof(SLogTupleKey) * max_evict); + + /* Phase 1: scan each partition under its writer lock to collect keys */ + for (part = 0; part < SLogNumPartitions && nkeys < max_evict; part++) + { + SLogFlatPartition *fp = SLogGetPartitionByIndex(part); + SLogFlatHashScanState scan; + const SLogFlatBucket *bucket; + + LWLockAcquire(&fp->writer_lock.lock, LW_EXCLUSIVE); + + SLogFlatHashScanInit(&scan); + while ((bucket = SLogFlatHashScanNext(fp->hash, &scan)) != NULL) + { + const SLogTupleEntry *entry = &bucket->entry; + bool all_committed = true; + bool has_any_op = false; + int i; + + for (i = 0; i < SLOG_MAX_TUPLE_OPS; i++) + { + if (!entry->ops[i].in_use) + continue; + has_any_op = true; + + if (TransactionIdIsInProgress(entry->ops[i].xid) || + !TransactionIdDidCommit(entry->ops[i].xid)) + { + all_committed = false; + break; + } + } + + if (has_any_op && all_committed) + { + keys_to_evict[nkeys++] = bucket->key; + if (nkeys >= max_evict) + break; + } + } + + LWLockRelease(&fp->writer_lock.lock); + } + + /* Phase 2: apply removals grouped by partition */ + if (nkeys > 0) + { + int i; + + for (part = 0; part < SLogNumPartitions; part++) + { + SLogFlatPartition *fp = SLogGetPartitionByIndex(part); + bool has_keys_for_part = false; + + /* Check if any keys belong to this partition */ + for (i = 0; i < nkeys; i++) + { + if (SLogFlatHashPartitionIndex(&keys_to_evict[i]) == part) + { + has_keys_for_part = true; + break; + } + } + if (!has_keys_for_part) + continue; + + LWLockAcquire(&fp->writer_lock.lock, LW_EXCLUSIVE); + SLogSeqWriteBegin(fp); + + for (i = 0; i < nkeys; i++) + { + SLogFlatOp flat_op; + + if (SLogFlatHashPartitionIndex(&keys_to_evict[i]) != part) + continue; + + memset(&flat_op, 0, sizeof(flat_op)); + flat_op.kind = SLOG_FLAT_OP_REMOVE_ENTRY; + flat_op.key = keys_to_evict[i]; + SLogFlatHashApply(fp->hash, &flat_op, sizeof(flat_op)); + } + + SLogSeqWriteEnd(fp); + LWLockRelease(&fp->writer_lock.lock); + } + } + + pfree(keys_to_evict); + + return nkeys; +} + +/* ---------------------------------------------------------------- + * Core Tuple sLog API + * ---------------------------------------------------------------- + */ + +/* + * SLogTupleInsert + * Record a tuple operation in the sLog. + * + * Inserts into the seqlock-protected flat hash (wait-free reads). + * Performs overflow handling before failing. + * + * Also adds the key to the backend-private tracking list for cleanup. + */ +bool +SLogTupleInsert(Oid relid, ItemPointer tid, TransactionId xid, + SLogOpType op_type, TransactionId subxid, + CommandId cid, TimestampTz commit_ts, + uint32 spec_token, LockTupleMode lock_mode) +{ + SLogTupleKey key; + SLogFlatOp flat_op; + SLogFlatPartition *fp; + int entries_before; + int entries_after; + + Assert(SLogState != NULL); + Assert(TransactionIdIsValid(xid)); + Assert(ItemPointerIsValid(tid)); + + /* Zero for deterministic hashing (ItemPointerData is 6 bytes) */ + memset(&key, 0, sizeof(key)); + key.relid = relid; + ItemPointerCopy(tid, &key.tid); + + fp = SLogGetPartition(&key); + + /* + * Retained UPDATE markers are never evicted by the on-overflow path -- + * SLogTupleEvictCommitted() leaves any op whose xid is in-progress or not + * yet committed in CLOG. They are cleaned up by + * SLogTupleCleanupRetained(), driven by the UNDO background worker or, + * when it is disabled, by the access method calling + * SLogTupleMaybeCleanupRetained() outside any buffer-locked section. This + * function never triggers that sweep itself (see note below). + * + * Per-TID ops array overflow (SLOG_MAX_TUPLE_OPS=32 slots all full of + * retained entries on hot rows) is handled by flat_hash_apply_insert() + * which reclaims the oldest retained entry when no free slot exists. + */ + + { + /* Build the flat hash op */ + memset(&flat_op, 0, sizeof(flat_op)); + flat_op.kind = SLOG_FLAT_OP_INSERT; + flat_op.key = key; + flat_op.xid = xid; + flat_op.subxid = subxid; + + /* + * The xid reclaim horizon is computed LAZILY, not on every insert. It + * is only consumed by flat_hash_apply_insert to reclaim a slot when a + * hot row's per-TID ops array is completely full; the common path + * (free slot, same-xid overwrite, or a slot freed by coalescing) + * never reads it. Computing it here would run a ProcArrayLock-shared + * xid-horizon scan on every CAS-update -- a measured hot-path cost -- + * for a value used only in the rare full-array case. + * + * So the fast path passes InvalidTransactionId, which now fail-safe + * DISABLES reclamation (see flat_hash_apply_insert): an invalid + * horizon frees NOTHING. This preserves the original safety + * invariant -- a stale or absent horizon must never cause a lost + * update -- in the most conservative direction possible. If the op + * is then dropped because the array was genuinely full, the caller + * (below) recomputes the real, authoritative horizon and retries; + * reclamation is thus attempted with a fresh horizon exactly when, + * and only when, it is actually needed. + */ + flat_op.reclaim_xid_horizon = InvalidTransactionId; + flat_op.tuple_op.xid = xid; + flat_op.tuple_op.subxid = subxid; + flat_op.tuple_op.op_type = op_type; + flat_op.tuple_op.cid = cid; + flat_op.tuple_op.commit_ts = commit_ts; + flat_op.tuple_op.spec_token = spec_token; + flat_op.tuple_op.lock_mode = lock_mode; + flat_op.tuple_op.in_use = true; + } + + /* Apply to flat hash via seqlock writer path (partition-local) */ + LWLockAcquire(&fp->writer_lock.lock, LW_EXCLUSIVE); + + /* Check capacity before insert (writer lock held: fp->hash is stable) */ + entries_before = fp->hash->num_entries; + + SLogSeqWriteBegin(fp); + SLogFlatHashApply(fp->hash, &flat_op, sizeof(flat_op)); + SLogSeqWriteEnd(fp); + + /* Check if insert succeeded */ + entries_after = fp->hash->num_entries; + + LWLockRelease(&fp->writer_lock.lock); + + /* + * Verify the op was actually stored. We must probe for THIS xid's op, + * not merely the bucket: on a hot row the bucket pre-exists (it holds + * other TIDs' / xids' markers), so num_entries is unchanged and the + * bucket is present even when flat_hash_apply_insert silently dropped our + * op because the per-TID ops array was full with nothing reclaimable. + * Testing only bucket presence (the old SLogFlatHashProbe != NULL) + * reports success for a dropped op, so no marker exists to stamp at + * PRE_COMMIT and the next concurrent writer clobbers this update -- a + * lost update. Probe the ops array for our xid instead. + */ + if (entries_after == entries_before) + { + bool op_stored; + + LWLockAcquire(&fp->writer_lock.lock, LW_EXCLUSIVE); + op_stored = SLogFlatHashHasOpForXid(fp->hash, &key, xid); + LWLockRelease(&fp->writer_lock.lock); + + if (!op_stored) + { + /* + * The op was dropped: the per-TID ops array was full and the fast + * path passed an invalid horizon, which disables reclamation. On + * a hot row the array is full of THIS relation's own + * below-horizon UPDATE markers, which the reclaim path can free + * WITHOUT the cross-partition SLogTupleEvictCommitted() sweep. So + * recompute the real, authoritative horizon now (the + * ProcArrayLock scan we skipped on the fast path) and retry: + * flat_hash_apply_insert can now reclaim a settled marker and + * store our op in place. + */ + TransactionId horizon = GetOldestNonRemovableTransactionId(NULL); + + flat_op.reclaim_xid_horizon = horizon; + + LWLockAcquire(&fp->writer_lock.lock, LW_EXCLUSIVE); + SLogSeqWriteBegin(fp); + SLogFlatHashApply(fp->hash, &flat_op, sizeof(flat_op)); + SLogSeqWriteEnd(fp); + op_stored = SLogFlatHashHasOpForXid(fp->hash, &key, xid); + LWLockRelease(&fp->writer_lock.lock); + } + + if (!op_stored) + { + /* Table or per-TID array was full ā€” try eviction */ + int evicted = SLogTupleEvictCommitted(); + + if (evicted > 0) + { + /* Retry */ + LWLockAcquire(&fp->writer_lock.lock, LW_EXCLUSIVE); + SLogSeqWriteBegin(fp); + SLogFlatHashApply(fp->hash, &flat_op, sizeof(flat_op)); + SLogSeqWriteEnd(fp); + op_stored = SLogFlatHashHasOpForXid(fp->hash, &key, xid); + LWLockRelease(&fp->writer_lock.lock); + } + + if (!op_stored) + { + slog_overflow_warning_count++; + { + TimestampTz now = GetCurrentTimestamp(); + + if (slog_overflow_warning_count == 1 || + TimestampDifferenceExceeds(slog_overflow_last_warning, + now, 1000)) + { + elog(WARNING, "sLog tuple hash partition full " + "(%d entries); %d overflow(s) this transaction " + "on rel %u (visibility relies on UNCOMMITTED " + "flag + UNDO replay)", + SLogTupleNumEntries() / SLogNumPartitions, + slog_overflow_warning_count, relid); + slog_overflow_last_warning = now; + } + } + + SLogTupleTrackLocalOnly(relid, tid, xid, subxid); + return false; + } + } + } + + SLogTupleTrackKey(key, xid, subxid, op_type); + return true; +} + +/* + * SLogTupleInsertRecovery + * Record a tuple operation during WAL replay (recovery-safe). + * + * This is a simplified variant of SLogTupleInsert() designed for use during + * WAL redo on hot standbys. Key differences: + * - Does NOT call SLogTupleTrackKey() (no backend-local tracking needed) + * - Returns false silently if the hash is full (instead of ERROR/PANIC) + * - No retries with pg_usleep (would delay WAL replay) + * + * Used by an AM's WAL redo path to register UNCOMMITTED tuples in the + * per-tuple sLog so that the AM's visibility check can correctly determine + * visibility on standbys (where the sLog is otherwise never populated). + */ +bool +SLogTupleInsertRecovery(Oid relid, ItemPointer tid, TransactionId xid, + SLogOpType op_type) +{ + SLogTupleKey key; + SLogFlatOp clear_op; + SLogFlatOp flat_op; + + if (SLogState == NULL) + return false; + + memset(&key, 0, sizeof(key)); + key.relid = relid; + ItemPointerCopy(tid, &key.tid); + + /* + * Clear any pre-existing entry at this exact TID before registering the + * new op. A physical INSERT always targets a free line pointer, so any + * sLog entry already present at this TID belongs to a dead prior occupant + * (deleted and VACUUM-recycled). On the primary the prior occupant's + * sLog entry is removed by its commit/abort xact callback, but the + * standby redo path only ever inserts -- it never removes -- so without + * this the stale entry survives. flat_hash_apply_insert() only + * overwrites a same-xid op, so a stale op from the prior occupant's + * (different) xid would persist and make the AM's visibility check trip + * on TransactionIdDidAbort()/IsInProgress() for that dead xid, wrongly + * hiding the freshly inserted live tuple. + * + * Tombstoning the entry is safe and self-healing: + * flat_hash_apply_insert() reuses tombstoned buckets, so the immediately + * following INSERT recreates a clean entry holding only this insert's op. + */ + memset(&clear_op, 0, sizeof(clear_op)); + clear_op.kind = SLOG_FLAT_OP_REMOVE_ENTRY; + clear_op.key = key; + + /* Apply to flat hash */ + memset(&flat_op, 0, sizeof(flat_op)); + flat_op.kind = SLOG_FLAT_OP_INSERT; + flat_op.key = key; + flat_op.xid = xid; + flat_op.tuple_op.xid = xid; + flat_op.tuple_op.subxid = InvalidTransactionId; + flat_op.tuple_op.op_type = op_type; + flat_op.tuple_op.cid = InvalidCommandId; + flat_op.tuple_op.commit_ts = 0; + flat_op.tuple_op.spec_token = 0; + flat_op.tuple_op.in_use = true; + + LWLockAcquire(SLOG_PART_WRITER_LOCK(&key), LW_EXCLUSIVE); + SLogSeqWriteBegin(SLogGetPartition(&key)); + SLogFlatHashApply(SLogGetPartition(&key)->hash, &clear_op, sizeof(clear_op)); + SLogFlatHashApply(SLogGetPartition(&key)->hash, &flat_op, sizeof(flat_op)); + SLogSeqWriteEnd(SLogGetPartition(&key)); + LWLockRelease(SLOG_PART_WRITER_LOCK(&key)); + + return true; +} + +/* + * SLogTupleLookup + * Look up a tuple's sLog entry (copy semantics). + * + * Returns true if found, copying the full entry into *entry_out. + * WAIT-FREE: uses the seqlock read-side retry loop. + */ +bool +SLogTupleLookup(Oid relid, ItemPointer tid, SLogTupleEntry *entry_out) +{ + SLogTupleKey key; + SLogFlatPartition *fp; + bool found; + uint32 slog_seq_; + + memset(&key, 0, sizeof(key)); + key.relid = relid; + ItemPointerCopy(tid, &key.tid); + + fp = SLogGetPartition(&key); + + SLOG_SEQ_READ_BEGIN(fp, slog_seq_) + { + const SLogFlatBucket *bucket = SLogFlatHashProbe(fp->hash, &key); + + found = (bucket != NULL); + if (found && entry_out) + memcpy(entry_out, &bucket->entry, sizeof(SLogTupleEntry)); + } + SLOG_SEQ_READ_END(fp, slog_seq_); + + return found; +} + +/* + * SLogTupleLookupFiltered + * Find sLog entries for a TID, optionally filtered by xid. + * + * WAIT-FREE: uses the seqlock read-side. If xid_filter is valid, returns + * only ops for that xid. If InvalidTransactionId, returns all active + * ops for this TID. + * + * Returns the number of ops written to ops_out. + */ +int +SLogTupleLookupFiltered(Oid relid, ItemPointer tid, + TransactionId xid_filter, + SLogTupleOp *ops_out, int max_ops) +{ + SLogTupleKey key; + SLogFlatPartition *fp; + int nfound = 0; + uint32 slog_seq_; + + memset(&key, 0, sizeof(key)); + key.relid = relid; + ItemPointerCopy(tid, &key.tid); + + fp = SLogGetPartition(&key); + + SLOG_SEQ_READ_BEGIN(fp, slog_seq_) + { + const SLogFlatBucket *bucket = SLogFlatHashProbe(fp->hash, &key); + + nfound = 0; /* reset: body may re-run on retry */ + if (bucket != NULL) + { + const SLogTupleEntry *entry = &bucket->entry; + int i; + + for (i = 0; i < SLOG_MAX_TUPLE_OPS && nfound < max_ops; i++) + { + if (!entry->ops[i].in_use) + continue; + + if (TransactionIdIsValid(xid_filter) && + !TransactionIdEquals(entry->ops[i].xid, xid_filter)) + continue; + + memcpy(&ops_out[nfound], &entry->ops[i], sizeof(SLogTupleOp)); + nfound++; + } + } + } + SLOG_SEQ_READ_END(fp, slog_seq_); + + return nfound; +} + +/* + * SLogTupleRemove + * Remove operations for a specific xid from a tuple entry. + * + * Uses the seqlock writer path with external (writer_lock) serialization. + */ +void +SLogTupleRemove(Oid relid, ItemPointer tid, TransactionId xid) +{ + SLogTupleKey key; + SLogFlatOp op; + + memset(&key, 0, sizeof(key)); + key.relid = relid; + ItemPointerCopy(tid, &key.tid); + + memset(&op, 0, sizeof(op)); + op.kind = SLOG_FLAT_OP_REMOVE_XID; + op.key = key; + op.xid = xid; + + SLogPartApplyOne(SLogGetPartition(&key), &op); +} + +/* + * Width of the touched-partition bitmap, in uint64 words. Sized to cover + * SLOG_FLAT_MAX_PARTITIONS so the bitmap is a fixed on-stack array regardless + * of the runtime partition count. + */ +#define SLOG_PART_BITMAP_WORDS ((SLOG_FLAT_MAX_PARTITIONS + 63) / 64) + +/* + * SLogCollectTrackedPartitions + * Build a bitmap of the partitions touched by xid's shared tracked keys. + * + * Walks slog_tracked_keys once, setting one bit per distinct partition that + * holds a non-local_only entry for xid. Callers then iterate only the set + * partitions instead of sweeping all SLogNumPartitions, turning the batch + * apply from O(num_partitions * tracked_keys) into + * O(touched_partitions * tracked_keys) -- optimal for the common single-row, + * single-partition transaction. + * + * Returns the number of distinct partitions marked. + */ +static int +SLogCollectTrackedPartitions(TransactionId xid, + uint64 *bitmap) +{ + SLogTrackedKey *tk; + int nparts = 0; + + memset(bitmap, 0, sizeof(uint64) * SLOG_PART_BITMAP_WORDS); + + for (tk = slog_tracked_keys; tk != NULL; tk = tk->next) + { + int part; + uint64 mask; + + if (!TransactionIdEquals(tk->xid, xid) || tk->local_only) + continue; + + part = SLogFlatHashPartitionIndex(&tk->key); + mask = UINT64CONST(1) << (part & 63); + if (!(bitmap[part >> 6] & mask)) + { + bitmap[part >> 6] |= mask; + nparts++; + } + } + + return nparts; +} + +/* + * SLogTupleRemoveByXid + * Remove all tuple sLog entries for a transaction. + * + * Uses the backend-local tracking list. Applies REMOVE_XID ops to the + * flat hash in a single writer batch. + */ +void +SLogTupleRemoveByXid(TransactionId xid) +{ + SLogTrackedKey *tk; + uint64 touched[SLOG_PART_BITMAP_WORDS] = {0}; + int part; + + if (SLogState == NULL) + return; + + /* + * Collect the partitions xid actually touches in one pass. Walking only + * those avoids an O(num_partitions * tracked_keys) sweep over every + * partition for what is usually a single-row transaction. + */ + if (SLogCollectTrackedPartitions(xid, touched) == 0) + return; + + /* Batch apply REMOVE_XID ops grouped by partition */ + for (part = 0; part < SLogNumPartitions; part++) + { + SLogFlatPartition *fp; + + if (!(touched[part >> 6] & (UINT64CONST(1) << (part & 63)))) + continue; + + fp = SLogGetPartitionByIndex(part); + + LWLockAcquire(&fp->writer_lock.lock, LW_EXCLUSIVE); + SLogSeqWriteBegin(fp); + + for (tk = slog_tracked_keys; tk != NULL; tk = tk->next) + { + SLogFlatOp flat_op; + + if (!TransactionIdEquals(tk->xid, xid) || tk->local_only) + continue; + if (SLogFlatHashPartitionIndex(&tk->key) != part) + continue; + + memset(&flat_op, 0, sizeof(flat_op)); + flat_op.kind = SLOG_FLAT_OP_REMOVE_XID; + flat_op.key = tk->key; + flat_op.xid = xid; + SLogFlatHashApply(fp->hash, &flat_op, sizeof(flat_op)); + } + + SLogSeqWriteEnd(fp); + LWLockRelease(&fp->writer_lock.lock); + } +} + +/* + * SLogTupleCommitByXid + * Handle commit for tuple sLog: remove ALL of the committing xid's ops + * (INSERT/DELETE/LOCK and UPDATE alike). + * + * WS-PVS3: committed-UPDATE markers are NO LONGER retained on the flat hash. + * Snapshot-isolation readers reconstruct the visible version by walking the + * durable UNDO fork chain (via the AM's version-reconstruction walk), so at + * commit every op of the xid is removed -- identical to INSERT/DELETE/LOCK -- + * which also drains bucket table_full pressure. (The backend-local + * before-image kept by SLogTupleStoreBeforeImage remains for + * intra-transaction savepoint rollback.) + * + * Uses the backend-local tracking list. Applies COMMIT_XID ops to the + * flat hash in batch. + */ +void +SLogTupleCommitByXid(TransactionId xid) +{ + SLogTrackedKey *tk; + uint64 touched[SLOG_PART_BITMAP_WORDS] = {0}; + + if (SLogState == NULL) + return; + + /* Fast path: INSERT-only transactions never touch the shared hash */ + if (!slog_has_shared_entries) + return; + + /* + * Collect the partitions xid touches in one pass; the COMMIT_XID batch + * below iterates only those rather than sweeping all SLogNumPartitions. + */ + if (SLogCollectTrackedPartitions(xid, touched) == 0) + return; + + /* + * WS-PVS3: committed-UPDATE cross-backend before-images are not published + * to shared DSA (Phase 1) and committed-UPDATE markers are no longer + * retained on the flat hash (Phase 2). Snapshot-isolation readers + * reconstruct the visible version by walking the durable UNDO fork chain + * (via the AM's version-reconstruction walk); the write-write conflict + * probe reads the head verptr on the on-page tuple and resolves it in the + * same fork. + * + * flat_hash_apply_commit_xid therefore removes every op of xid at commit + * -- identical to INSERT/DELETE/LOCK -- which drains bucket table_full. + * + * The local tk->before_image (palloc'd in TopTransactionContext by + * SLogTupleStoreBeforeImage) is still used for intra-transaction + * savepoint rollback via the AM's before-image restore and is unaffected. + */ + + /* Batch apply COMMIT_XID ops grouped by partition */ + { + int part; + + for (part = 0; part < SLogNumPartitions; part++) + { + SLogFlatPartition *fp; + + if (!(touched[part >> 6] & (UINT64CONST(1) << (part & 63)))) + continue; + + fp = SLogGetPartitionByIndex(part); + + LWLockAcquire(&fp->writer_lock.lock, LW_EXCLUSIVE); + SLogSeqWriteBegin(fp); + + for (tk = slog_tracked_keys; tk != NULL; tk = tk->next) + { + SLogFlatOp flat_op; + + if (!TransactionIdEquals(tk->xid, xid) || tk->local_only) + continue; + if (SLogFlatHashPartitionIndex(&tk->key) != part) + continue; + + memset(&flat_op, 0, sizeof(flat_op)); + flat_op.kind = SLOG_FLAT_OP_COMMIT_XID; + flat_op.key = tk->key; + flat_op.xid = xid; + SLogFlatHashApply(fp->hash, &flat_op, sizeof(flat_op)); + } + + SLogSeqWriteEnd(fp); + LWLockRelease(&fp->writer_lock.lock); + } + } +} + +/* + * SLogTupleRemoveByXidSingle + * Remove the sLog entry for a single tuple identified by (relid, tid, xid). + * + * Used by an AM's two-phase postcommit callback to clean up sLog entries + * one at a time (since the local tracking list is unavailable in the + * resolving backend). + */ +void +SLogTupleRemoveByXidSingle(Oid relid, ItemPointer tid, TransactionId xid) +{ + SLogTupleKey key; + SLogFlatOp flat_op; + + if (SLogState == NULL) + return; + + memset(&key, 0, sizeof(key)); + key.relid = relid; + ItemPointerCopy(tid, &key.tid); + + /* Apply to flat hash */ + memset(&flat_op, 0, sizeof(flat_op)); + flat_op.kind = SLOG_FLAT_OP_REMOVE_XID; + flat_op.key = key; + flat_op.xid = xid; + + SLogPartApplyOne(SLogGetPartition(&key), &flat_op); +} + +/* + * SLogTupleMarkAbortedSingle + * Mark the sLog entry for a single tuple as ABORTED. + * + * Used by an AM's two-phase postabort callback to mark sLog entries + * one at a time (since the local tracking list is unavailable in the + * resolving backend). Only operates on tuples that already have a shared + * sLog entry (DELETE/UPDATE operations). + */ +void +SLogTupleMarkAbortedSingle(Oid relid, ItemPointer tid, TransactionId xid) +{ + SLogTupleKey key; + SLogFlatOp flat_op; + + if (SLogState == NULL) + return; + + memset(&key, 0, sizeof(key)); + key.relid = relid; + ItemPointerCopy(tid, &key.tid); + + /* Apply to flat hash */ + memset(&flat_op, 0, sizeof(flat_op)); + flat_op.kind = SLOG_FLAT_OP_MARK_ABORTED; + flat_op.key = key; + flat_op.xid = xid; + + SLogPartApplyOne(SLogGetPartition(&key), &flat_op); +} + +/* + * SLogTupleRemoveBySubXid + * Handle subtransaction abort for tuple sLog. + * + * For entries that have a shared sLog entry, marks matching ops as ABORTED. + * For entries that only have local tracking (from SLogTupleTrackLocalOnly, + * used by INSERT), creates a shared ABORTED entry so visibility code can + * find it. + */ +void +SLogTupleRemoveBySubXid(TransactionId xid, TransactionId subxid) +{ + SLogTrackedKey *tk; + + if (SLogState == NULL) + return; + + for (tk = slog_tracked_keys; tk != NULL; tk = tk->next) + { + SLogFlatOp flat_op; + + if (!TransactionIdEquals(tk->xid, xid)) + continue; + if (tk->subxid != subxid) + continue; + + memset(&flat_op, 0, sizeof(flat_op)); + + if (tk->local_only) + { + /* + * INSERT elision: no shared entry exists yet. Create one with + * SLOG_OP_ABORTED so visibility code can find it. + */ + flat_op.kind = SLOG_FLAT_OP_CREATE_ABORTED; + flat_op.key = tk->key; + flat_op.xid = xid; + flat_op.subxid = subxid; + } + else + { + /* Shared entry exists -- mark matching ops ABORTED */ + flat_op.kind = SLOG_FLAT_OP_MARK_ABORTED; + flat_op.key = tk->key; + flat_op.xid = xid; + } + + LWLockAcquire(SLOG_PART_WRITER_LOCK(&tk->key), LW_EXCLUSIVE); + SLogSeqWriteBegin(SLogGetPartition(&tk->key)); + SLogFlatHashApply(SLogGetPartition(&tk->key)->hash, &flat_op, + sizeof(flat_op)); + SLogSeqWriteEnd(SLogGetPartition(&tk->key)); + LWLockRelease(SLOG_PART_WRITER_LOCK(&tk->key)); + + /* + * Mark the backend-local tracked key as aborted so the AM's + * commit-time flag clearing skips it. Without this, a local-only + * INSERT tracked key still reads as a live INSERT at top-level + * commit, and the AM's uncommitted-flag clearing would clear the + * tuple's UNCOMMITTED flag and stamp a commit marker -- resurrecting + * a tuple that the savepoint rollback was supposed to discard. + */ + tk->op_type = SLOG_OP_ABORTED; + } +} + +/* + * SLogTupleUpdateSubXid + * Re-parent ops on subtransaction commit. + * + * When a subtransaction commits, its entries' subxid is updated to the + * parent's subxid so they survive subtransaction commit but are cleaned + * up at top-level commit. + */ +void +SLogTupleUpdateSubXid(TransactionId xid, + TransactionId old_subxid, + TransactionId new_subxid) +{ + SLogTrackedKey *tk; + + if (SLogState == NULL) + return; + + for (tk = slog_tracked_keys; tk != NULL; tk = tk->next) + { + if (!TransactionIdEquals(tk->xid, xid)) + continue; + if (tk->subxid != old_subxid) + continue; + + /* Re-parent the local entry */ + tk->subxid = new_subxid; + + /* Also re-parent in the shared sLog if an entry exists */ + if (!tk->local_only) + { + SLogFlatOp flat_op; + + memset(&flat_op, 0, sizeof(flat_op)); + flat_op.kind = SLOG_FLAT_OP_UPDATE_OP; + flat_op.key = tk->key; + flat_op.xid = xid; + flat_op.subxid = new_subxid; + flat_op.tuple_op.subxid = old_subxid; + + LWLockAcquire(SLOG_PART_WRITER_LOCK(&tk->key), LW_EXCLUSIVE); + SLogSeqWriteBegin(SLogGetPartition(&tk->key)); + SLogFlatHashApply(SLogGetPartition(&tk->key)->hash, &flat_op, + sizeof(flat_op)); + SLogSeqWriteEnd(SLogGetPartition(&tk->key)); + LWLockRelease(SLOG_PART_WRITER_LOCK(&tk->key)); + } + } +} + +/* + * SLogTupleMarkAborted + * Mark all ops for a transaction as SLOG_OP_ABORTED. + * + * Called at transaction abort. Entries remain so visibility checks can + * distinguish "committed (no entry)" from "aborted (UNDO pending)". + * + * For local-only entries (INSERT elision), we CREATE a shared ABORTED + * entry at abort time. This is safe because: + * (a) Abort is uncommon (vast majority of transactions commit) + * (b) The UNDO worker removes both the sLog entry and the page tuple, + * bounding the lifetime of these entries + * (c) If the hash is full, we log a warning; the UNDO worker will + * eventually remove the tuple physically, resolving the anomaly + */ +void +SLogTupleMarkAborted(TransactionId xid) +{ + SLogTrackedKey *tk; + int part; + int ops_since_publish; + + if (SLogState == NULL) + return; + + /* + * Process all entries grouped by partition. For each partition, acquire + * its writer lock once, apply all relevant ops, then release. + */ + for (part = 0; part < SLogNumPartitions; part++) + { + SLogFlatPartition *fp = SLogGetPartitionByIndex(part); + bool has_entries = false; + + /* Quick check: any local-only INSERT entries at all? */ + if (slog_insert_tids != NULL && slog_insert_tids->members > 0) + has_entries = true; /* conservative; filtered per-entry below */ + + /* Check linked-list entries */ + if (!has_entries) + { + for (tk = slog_tracked_keys; tk != NULL; tk = tk->next) + { + if (!TransactionIdEquals(tk->xid, xid)) + continue; + if (SLogFlatHashPartitionIndex(&tk->key) == part) + { + has_entries = true; + break; + } + } + } + + if (!has_entries && + (slog_insert_tids == NULL || slog_insert_tids->members == 0)) + continue; + + LWLockAcquire(&fp->writer_lock.lock, LW_EXCLUSIVE); + + SLogSeqWriteBegin(fp); + + /* + * Bound the seqlock odd-hold window on a very large rollback + * (hundreds of thousands of tuples). While seq is odd every + * wait-free reader of this partition spins; ending and re-beginning + * the seqlock every SLOG_ABORT_PUBLISH_BATCH ops lets those readers + * make progress. Correctness across the boundary: each tuple's abort + * visibility is resolved independently and each seq cycle leaves + * fp->hash consistent, so a reader that observes the even counter at + * a boundary sees a valid partial state. + */ + ops_since_publish = 0; + + /* Process backend-local INSERT-hash entries for this partition */ + if (slog_insert_tids != NULL) + { + sloginsert_iterator it; + SLogInsertTidEntry *ie; + + sloginsert_start_iterate(slog_insert_tids, &it); + while ((ie = sloginsert_iterate(slog_insert_tids, &it)) != NULL) + { + SLogFlatOp flat_op; + SLogTupleKey smkey; + + memset(&smkey, 0, sizeof(smkey)); + smkey.relid = ie->key.relid; + ItemPointerSet(&smkey.tid, + SLOG_DECODE_BLKNO(ie->key.encoded_tid), + SLOG_DECODE_OFFNUM(ie->key.encoded_tid)); + + /* Only process if this key belongs to current partition */ + if (SLogFlatHashPartitionIndex(&smkey) == part) + { + memset(&flat_op, 0, sizeof(flat_op)); + flat_op.kind = SLOG_FLAT_OP_CREATE_ABORTED; + flat_op.key = smkey; + flat_op.xid = xid; + flat_op.subxid = InvalidTransactionId; + SLogFlatHashApply(fp->hash, &flat_op, sizeof(flat_op)); + + if (++ops_since_publish >= SLOG_ABORT_PUBLISH_BATCH) + { + SLogSeqWriteEnd(fp); + SLogSeqWriteBegin(fp); + ops_since_publish = 0; + } + } + } + } + + /* Process linked-list entries for this partition */ + for (tk = slog_tracked_keys; tk != NULL; tk = tk->next) + { + SLogFlatOp flat_op; + + if (!TransactionIdEquals(tk->xid, xid)) + continue; + if (SLogFlatHashPartitionIndex(&tk->key) != part) + continue; + + memset(&flat_op, 0, sizeof(flat_op)); + if (tk->local_only) + { + flat_op.kind = SLOG_FLAT_OP_CREATE_ABORTED; + flat_op.key = tk->key; + flat_op.xid = xid; + flat_op.subxid = tk->subxid; + } + else + { + flat_op.kind = SLOG_FLAT_OP_MARK_ABORTED; + flat_op.key = tk->key; + flat_op.xid = xid; + } + + SLogFlatHashApply(fp->hash, &flat_op, sizeof(flat_op)); + + if (++ops_since_publish >= SLOG_ABORT_PUBLISH_BATCH) + { + SLogSeqWriteEnd(fp); + SLogSeqWriteBegin(fp); + ops_since_publish = 0; + } + } + + SLogSeqWriteEnd(fp); + + LWLockRelease(&fp->writer_lock.lock); + } +} + +/* + * SLogTupleRemoveByXidGlobal + * Remove ALL ops for a transaction by scanning the shared flat hash. + * + * Unlike SLogTupleRemoveByXid, this does not use the backend-local tracking + * list (which doesn't exist in the UNDO worker process). Used by the UNDO + * worker to clean up ABORTED entries after UNDO has been applied. + */ +void +SLogTupleRemoveByXidGlobal(TransactionId xid) +{ + SLogTupleKey *collected_keys; + int max_keys; + int part; + + if (SLogState == NULL) + return; + + max_keys = SLogTupleNumEntries(); + if (max_keys <= 0) + return; + + collected_keys = (SLogTupleKey *) + palloc(sizeof(SLogTupleKey) * max_keys); + + /* + * Process each partition while holding its exclusive writer lock. The + * writer lock serializes all writers for the partition, so the single + * copy is stable while we scan it. We apply REMOVE_XID inside one + * seqlock cycle. + */ + for (part = 0; part < SLogNumPartitions; part++) + { + SLogFlatPartition *fp = SLogGetPartitionByIndex(part); + SLogFlatHashScanState scan; + const SLogFlatBucket *bucket; + int nkeys = 0; + int i; + + LWLockAcquire(&fp->writer_lock.lock, LW_EXCLUSIVE); + + /* Scan the single copy; stable because we hold the writer lock. */ + SLogFlatHashScanInit(&scan); + while ((bucket = SLogFlatHashScanNext(fp->hash, &scan)) != NULL) + { + const SLogTupleEntry *entry = &bucket->entry; + + for (i = 0; i < SLOG_MAX_TUPLE_OPS; i++) + { + if (entry->ops[i].in_use && + TransactionIdEquals(entry->ops[i].xid, xid)) + { + if (nkeys < max_keys) + collected_keys[nkeys++] = bucket->key; + break; + } + } + } + + /* Drop the slots in one seq cycle. */ + if (nkeys > 0) + { + SLogSeqWriteBegin(fp); + for (i = 0; i < nkeys; i++) + { + SLogFlatOp flat_op; + + memset(&flat_op, 0, sizeof(flat_op)); + flat_op.kind = SLOG_FLAT_OP_REMOVE_XID; + flat_op.key = collected_keys[i]; + flat_op.xid = xid; + SLogFlatHashApply(fp->hash, &flat_op, sizeof(flat_op)); + } + SLogSeqWriteEnd(fp); + } + + LWLockRelease(&fp->writer_lock.lock); + } + + pfree(collected_keys); +} + +/* + * SLogTupleIterateByTid + * Call a callback for each active operation on a tuple. + * + * WAIT-FREE: uses the seqlock read-side. The in-use ops are copied into a + * local array under the seqlock retry loop; the callback then runs after a + * consistent read, so it may have side effects and receives pointers into + * the local copy (valid for the duration of this call). + */ +void +SLogTupleIterateByTid(Oid relid, ItemPointer tid, + SLogTupleIterCallback callback, void *arg) +{ + SLogTupleKey key; + SLogFlatPartition *fp; + SLogTupleOp ops[SLOG_MAX_TUPLE_OPS]; + int nops = 0; + uint32 slog_seq_; + int i; + + memset(&key, 0, sizeof(key)); + key.relid = relid; + ItemPointerCopy(tid, &key.tid); + + fp = SLogGetPartition(&key); + + SLOG_SEQ_READ_BEGIN(fp, slog_seq_) + { + const SLogFlatBucket *bucket = SLogFlatHashProbe(fp->hash, &key); + + nops = 0; /* reset: body may re-run on retry */ + if (bucket != NULL) + { + const SLogTupleEntry *entry = &bucket->entry; + + for (i = 0; i < SLOG_MAX_TUPLE_OPS; i++) + if (entry->ops[i].in_use) + ops[nops++] = entry->ops[i]; + } + } + SLOG_SEQ_READ_END(fp, slog_seq_); + + /* Consistent read complete; run the (side-effecting) callback. */ + for (i = 0; i < nops; i++) + if (!callback(&ops[i], arg)) + break; +} + +/* ---------------------------------------------------------------- + * Convenience wrappers + * ---------------------------------------------------------------- + */ + +/* + * SLogTupleHasEntry -- quick probe: does ANY active entry exist for this TID? + * WAIT-FREE: uses the seqlock read-side. + */ +bool +SLogTupleHasEntry(Oid relid, ItemPointer tid) +{ + SLogTupleKey key; + SLogFlatPartition *fp; + bool has_entry = false; + uint32 slog_seq_; + + memset(&key, 0, sizeof(key)); + key.relid = relid; + ItemPointerCopy(tid, &key.tid); + + fp = SLogGetPartition(&key); + + SLOG_SEQ_READ_BEGIN(fp, slog_seq_) + { + const SLogFlatBucket *bucket = SLogFlatHashProbe(fp->hash, &key); + + has_entry = (bucket != NULL && bucket->entry.nops > 0); + } + SLOG_SEQ_READ_END(fp, slog_seq_); + + return has_entry; +} + +/* + * SLogTupleIsInsertedByMe -- check if current transaction inserted this tuple. + * + * Checks both the shared hash (normal case) and the backend-local tracking + * list (for local-only INSERTs that have no shared entry). + * + * Uses the top-level XID because SLogTupleTrackLocalOnly() always stores + * GetTopTransactionId(). This ensures correct results even when called + * from within a subtransaction (savepoint). + */ +bool +SLogTupleIsInsertedByMe(Oid relid, ItemPointer tid) +{ + SLogTupleOp op; + int nfound; + SLogTrackedKey *tk; + TransactionId myxid = GetTopTransactionIdIfAny(); + + if (!TransactionIdIsValid(myxid)) + return false; + + /* + * Special-marker TIDs (SpecTokenOffsetNumber 0xFFFE for speculative + * insertions, MovedPartitionsOffsetNumber 0xFFFD for cross-partition + * moves) can appear in a tuple's on-page t_ctid. All sLog entries are + * keyed by an on-page (block, offnum) with offnum <= MaxOffsetNumber, so + * this backend cannot have "inserted" the tuple under such a key. Bail + * out early to avoid tripping the encode-guard elog(ERROR) below. + */ + if (ItemPointerGetOffsetNumber(tid) > MaxOffsetNumber) + return false; + + /* Check shared hash first (normal case) */ + nfound = SLogTupleLookupFiltered(relid, tid, myxid, &op, 1); + if (nfound > 0 && op.op_type == SLOG_OP_INSERT) + return true; + + /* + * Check the backend-local INSERT hash (top-level local-only INSERTs). + */ + { + BlockNumber blkno = ItemPointerGetBlockNumber(tid); + OffsetNumber offnum = ItemPointerGetOffsetNumber(tid); + SLogInsertTidKey ikey; + + if (offnum > MaxOffsetNumber) + elog(ERROR, "offset %u exceeds page item limit %d in sLog TID encoding", + offnum, MaxOffsetNumber); + ikey.relid = relid; + ikey.encoded_tid = SLOG_ENCODE_TID(blkno, offnum); + if (slog_insert_tids != NULL && + sloginsert_lookup(slog_insert_tids, ikey) != NULL) + return true; + } + + /* + * Fall back to backend-local tracking list. This handles the overflow + * case where SLogTupleInsert returned false (hash full) but the INSERT + * was tracked locally via SLogTupleTrackLocalOnly or SLogTupleTrackKey. + * Also handles subtransaction local-only entries (which still use the + * linked list). + */ + for (tk = slog_tracked_keys; tk != NULL; tk = tk->next) + { + if (!TransactionIdEquals(tk->xid, myxid)) + continue; + if (tk->key.relid != relid) + continue; + if (ItemPointerEquals(&tk->key.tid, tid)) + return true; + } + + return false; +} + +/* + * SLogTupleIsDeletedByMe -- check if current transaction deleted this tuple. + */ +bool +SLogTupleIsDeletedByMe(Oid relid, ItemPointer tid) +{ + SLogTupleOp op; + int nfound; + TransactionId myxid = GetCurrentTransactionIdIfAny(); + + if (!TransactionIdIsValid(myxid)) + return false; + + nfound = SLogTupleLookupFiltered(relid, tid, myxid, &op, 1); + return (nfound > 0 && + (op.op_type == SLOG_OP_DELETE || + op.op_type == SLOG_OP_UPDATE)); +} + +/* + * SLogTupleGetDirtyXid -- for SNAPSHOT_DIRTY, get the xid of the in-progress + * transaction operating on this tuple. + * + * Returns the xid of the first in-progress INSERT or DELETE/UPDATE entry + * found (excluding our own), or InvalidTransactionId if none. + * + * WAIT-FREE: uses the seqlock read-side. The ops are snapshotted into a + * local array inside the retry loop; the (LWLock-taking) transaction-status + * checks then run after a consistent read, never inside the retry loop. + */ +TransactionId +SLogTupleGetDirtyXid(Oid relid, ItemPointer tid, bool *is_insert) +{ + SLogTupleKey key; + SLogFlatPartition *fp; + SLogTupleOp ops[SLOG_MAX_TUPLE_OPS]; + int nops = 0; + TransactionId result = InvalidTransactionId; + uint32 slog_seq_; + int i; + + memset(&key, 0, sizeof(key)); + key.relid = relid; + ItemPointerCopy(tid, &key.tid); + + fp = SLogGetPartition(&key); + + SLOG_SEQ_READ_BEGIN(fp, slog_seq_) + { + const SLogFlatBucket *bucket = SLogFlatHashProbe(fp->hash, &key); + + nops = 0; /* reset: body may re-run on retry */ + if (bucket != NULL) + { + const SLogTupleEntry *entry = &bucket->entry; + + for (i = 0; i < SLOG_MAX_TUPLE_OPS; i++) + if (entry->ops[i].in_use) + ops[nops++] = entry->ops[i]; + } + } + SLOG_SEQ_READ_END(fp, slog_seq_); + + /* Consistent snapshot taken; resolve status outside the retry loop. */ + for (i = 0; i < nops; i++) + { + TransactionId xid = ops[i].xid; + SLogOpType op = ops[i].op_type; + + if (TransactionIdIsCurrentTransactionId(xid)) + continue; + if (!TransactionIdIsInProgress(xid)) + continue; + + if (is_insert) + *is_insert = (op == SLOG_OP_INSERT); + result = xid; + break; + } + + return result; +} + +/* + * SLogTupleGetDirtyWriterXid -- like SLogTupleGetDirtyXid, but returns only + * the xid of an in-progress *writer* (INSERT/UPDATE/DELETE), ignoring + * lock-only markers (LOCK_SHARE/LOCK_EXCL) and aborted markers. + * + * Used at the write-wait decision in the UPDATE/DELETE paths. An updater + * already holds the heavyweight LOCKTAG_TUPLE lock (LockTupleNoKeyExclusive -> + * ExclusiveLock), which serializes against conflicting lockers via the + * standard lock matrix: a key-share locker (AccessShareLock) is compatible and + * does not block, while a share/exclusive locker conflicts and blocks the + * updater on the lock manager. XactLockTableWait must therefore fire only for + * an actual in-progress writer -- never for a pure locker. Waiting on a + * locker's xid here while that locker is queued behind us for the same tuple + * ExclusiveLock manufactures a deadlock cycle that heap avoids via + * HEAP_XMAX_IS_LOCKED_ONLY. + * + * WAIT-FREE: uses the seqlock read-side, identical to SLogTupleGetDirtyXid. + */ +TransactionId +SLogTupleGetDirtyWriterXid(Oid relid, ItemPointer tid, bool *is_insert) +{ + SLogTupleKey key; + SLogFlatPartition *fp; + SLogTupleOp ops[SLOG_MAX_TUPLE_OPS]; + int nops = 0; + TransactionId result = InvalidTransactionId; + uint32 slog_seq_; + int i; + + memset(&key, 0, sizeof(key)); + key.relid = relid; + ItemPointerCopy(tid, &key.tid); + + fp = SLogGetPartition(&key); + + SLOG_SEQ_READ_BEGIN(fp, slog_seq_) + { + const SLogFlatBucket *bucket = SLogFlatHashProbe(fp->hash, &key); + + nops = 0; /* reset: body may re-run on retry */ + if (bucket != NULL) + { + const SLogTupleEntry *entry = &bucket->entry; + + for (i = 0; i < SLOG_MAX_TUPLE_OPS; i++) + if (entry->ops[i].in_use) + ops[nops++] = entry->ops[i]; + } + } + SLOG_SEQ_READ_END(fp, slog_seq_); + + /* Consistent snapshot taken; resolve status outside the retry loop. */ + for (i = 0; i < nops; i++) + { + TransactionId xid; + SLogOpType op = ops[i].op_type; + + /* Only real writers block another writer. */ + if (op != SLOG_OP_INSERT && + op != SLOG_OP_UPDATE && + op != SLOG_OP_DELETE) + continue; + + xid = ops[i].xid; + + if (TransactionIdIsCurrentTransactionId(xid)) + continue; + if (!TransactionIdIsInProgress(xid)) + continue; + + if (is_insert) + *is_insert = (op == SLOG_OP_INSERT); + result = xid; + break; + } + + return result; +} + +/* + * slog_tuplock_to_lockmode -- map a LockTupleMode to its heavyweight LOCKMODE. + * + * The four tuple-lock strengths MUST map to four distinct LOCKMODEs so the + * real conflict matrix (DoLockModesConflict) can distinguish a compatible + * KeyShare FK locker from a conflicting Share/Exclusive locker. + */ +static LOCKMODE +slog_tuplock_to_lockmode(LockTupleMode mode) +{ + switch (mode) + { + case LockTupleKeyShare: + return AccessShareLock; + case LockTupleShare: + return RowShareLock; + case LockTupleNoKeyExclusive: + return ExclusiveLock; + case LockTupleExclusive: + return AccessExclusiveLock; + } + elog(ERROR, "invalid tuple lock mode: %d", (int) mode); + return NoLock; /* keep compiler quiet */ +} + +/* + * SLogTupleGetWriteConflictXid -- find an in-progress transaction whose marker + * conflicts with a writer (UPDATE/DELETE) acquiring tuple lock my_mode. + * + * Unlike SLogTupleGetDirtyWriterXid (which only ever reports writers and + * silently ignores lock-only markers), this also reports a *locker* whose + * recorded LockTupleMode conflicts with my_mode under the standard heavyweight + * matrix. That is required for correctness: a SELECT ... FOR UPDATE locker + * leaves only a LOCK_EXCL marker (no on-page writer state), and an updater that + * consults a writer-only probe sails past it and clobbers the row the locker is + * protecting. The four-way mapping keeps a KeyShare FK locker (AccessShareLock) + * compatible with a NoKeyExclusive UPDATE (ExclusiveLock) while making FOR SHARE + * (RowShareLock) and FOR UPDATE (AccessExclusiveLock) correctly block it. + * + * Writers take priority over lockers in the returned xid so the caller's + * is_insert handling (TM_Invisible for an in-progress INSERT) is preserved; a + * conflicting locker is returned only when no in-progress writer exists. + * + * Returns the conflicting xid, or InvalidTransactionId if none. *is_insert is + * set true only when the returned xid is an in-progress INSERT writer. + * + * WAIT-FREE: uses the seqlock read-side, identical to SLogTupleGetDirtyWriterXid. + */ +TransactionId +SLogTupleGetWriteConflictXid(Oid relid, ItemPointer tid, + LockTupleMode my_mode, bool *is_insert) +{ + SLogTupleKey key; + SLogFlatPartition *fp; + SLogTupleOp ops[SLOG_MAX_TUPLE_OPS]; + int nops = 0; + TransactionId writer_xid = InvalidTransactionId; + bool writer_is_insert = false; + TransactionId locker_xid = InvalidTransactionId; + LOCKMODE my_lockmode = slog_tuplock_to_lockmode(my_mode); + uint32 slog_seq_; + int i; + + memset(&key, 0, sizeof(key)); + key.relid = relid; + ItemPointerCopy(tid, &key.tid); + + fp = SLogGetPartition(&key); + + SLOG_SEQ_READ_BEGIN(fp, slog_seq_) + { + const SLogFlatBucket *bucket = SLogFlatHashProbe(fp->hash, &key); + + nops = 0; /* reset: body may re-run on retry */ + if (bucket != NULL) + { + const SLogTupleEntry *entry = &bucket->entry; + + for (i = 0; i < SLOG_MAX_TUPLE_OPS; i++) + if (entry->ops[i].in_use) + ops[nops++] = entry->ops[i]; + } + } + SLOG_SEQ_READ_END(fp, slog_seq_); + + /* Consistent snapshot taken; resolve status outside the retry loop. */ + for (i = 0; i < nops; i++) + { + TransactionId xid; + SLogOpType op = ops[i].op_type; + + xid = ops[i].xid; + + if (TransactionIdIsCurrentTransactionId(xid)) + continue; + if (!TransactionIdIsInProgress(xid)) + continue; + + if (op == SLOG_OP_INSERT || + op == SLOG_OP_UPDATE || + op == SLOG_OP_DELETE) + { + /* A real writer: highest priority, stop scanning. */ + writer_xid = xid; + writer_is_insert = (op == SLOG_OP_INSERT); + break; + } + + if (op == SLOG_OP_LOCK_SHARE || op == SLOG_OP_LOCK_EXCL) + { + LOCKMODE locker_lockmode = + slog_tuplock_to_lockmode(ops[i].lock_mode); + + if (DoLockModesConflict(my_lockmode, locker_lockmode)) + locker_xid = xid; /* candidate; keep seeking a writer */ + } + } + + if (TransactionIdIsValid(writer_xid)) + { + if (is_insert) + *is_insert = writer_is_insert; + return writer_xid; + } + + if (is_insert) + *is_insert = false; + return locker_xid; +} + +/* + * SLogTupleHasLockConflict -- check if any active lock entries on this TID + * conflict with the requested lock mode. + */ +bool +SLogTupleHasLockConflict(Oid relid, ItemPointer tid, + TransactionId my_xid, + SLogOpType requested_lock) +{ + SLogTupleOp ops[SLOG_MAX_TUPLE_OPS]; + int nfound; + int i; + + nfound = SLogTupleLookupFiltered(relid, tid, InvalidTransactionId, + ops, SLOG_MAX_TUPLE_OPS); + + for (i = 0; i < nfound; i++) + { + if (TransactionIdEquals(ops[i].xid, my_xid)) + continue; + if (!TransactionIdIsInProgress(ops[i].xid)) + continue; + + /* Only lock/mutating entries can conflict */ + if (ops[i].op_type != SLOG_OP_LOCK_SHARE && + ops[i].op_type != SLOG_OP_LOCK_EXCL && + ops[i].op_type != SLOG_OP_DELETE && + ops[i].op_type != SLOG_OP_UPDATE) + continue; + + /* + * Lock compatibility matrix: SHARE vs SHARE: compatible SHARE vs + * EXCL/DELETE/UPDATE: conflict EXCL vs anything: conflict + */ + if (requested_lock == SLOG_OP_LOCK_SHARE) + { + if (ops[i].op_type == SLOG_OP_LOCK_EXCL || + ops[i].op_type == SLOG_OP_DELETE || + ops[i].op_type == SLOG_OP_UPDATE) + return true; + } + else if (requested_lock == SLOG_OP_LOCK_EXCL) + { + return true; + } + } + + return false; +} + +/* + * SLogTupleGetLockConflictXid -- like SLogTupleHasLockConflict, but also + * returns the xid of the *conflicting* transaction (the one whose marker + * actually conflicts with requested_lock under the matrix above). + * + * The waiter must XactLockTableWait on the conflicting xid specifically. + * Waiting on the first in-progress xid found on the TID (as a broad + * SLogTupleGetDirtyXid probe would return) can pick a compatible peer -- e.g. + * another KeyShare locker -- which is not blocking us and may itself be queued + * behind us, manufacturing a spurious mutual-wait deadlock cycle. + * + * Returns true and sets *xid_out if a conflicting in-progress transaction + * exists; returns false otherwise (xid_out is set to InvalidTransactionId). + */ +bool +SLogTupleGetLockConflictXid(Oid relid, ItemPointer tid, + TransactionId my_xid, + SLogOpType requested_lock, + TransactionId *xid_out) +{ + SLogTupleOp ops[SLOG_MAX_TUPLE_OPS]; + int nfound; + int i; + + *xid_out = InvalidTransactionId; + + nfound = SLogTupleLookupFiltered(relid, tid, InvalidTransactionId, + ops, SLOG_MAX_TUPLE_OPS); + + for (i = 0; i < nfound; i++) + { + if (TransactionIdEquals(ops[i].xid, my_xid)) + continue; + if (!TransactionIdIsInProgress(ops[i].xid)) + continue; + + /* Only lock/mutating entries can conflict */ + if (ops[i].op_type != SLOG_OP_LOCK_SHARE && + ops[i].op_type != SLOG_OP_LOCK_EXCL && + ops[i].op_type != SLOG_OP_DELETE && + ops[i].op_type != SLOG_OP_UPDATE) + continue; + + /* Same matrix as SLogTupleHasLockConflict. */ + if (requested_lock == SLOG_OP_LOCK_SHARE) + { + if (ops[i].op_type == SLOG_OP_LOCK_EXCL || + ops[i].op_type == SLOG_OP_DELETE || + ops[i].op_type == SLOG_OP_UPDATE) + { + *xid_out = ops[i].xid; + return true; + } + } + else if (requested_lock == SLOG_OP_LOCK_EXCL) + { + *xid_out = ops[i].xid; + return true; + } + } + + return false; +} + +/* + * SLogTupleHasAbortedEntry -- check if any aborted sLog op exists for a TID. + */ +bool +SLogTupleHasAbortedEntry(Oid relid, ItemPointer tid) +{ + SLogTupleOp ops[SLOG_MAX_TUPLE_OPS]; + int nfound; + int i; + + nfound = SLogTupleLookupFiltered(relid, tid, InvalidTransactionId, + ops, SLOG_MAX_TUPLE_OPS); + + for (i = 0; i < nfound; i++) + { + /* Explicitly marked ABORTED */ + if (ops[i].op_type == SLOG_OP_ABORTED) + return true; + + /* Skip our own transaction's entries for CLOG fallback */ + if (TransactionIdIsCurrentTransactionId(ops[i].xid)) + continue; + + /* CLOG fallback: completed but did not commit => aborted */ + if (!TransactionIdIsInProgress(ops[i].xid) && + TransactionIdDidAbort(ops[i].xid)) + return true; + } + + return false; +} + +/* ---------------------------------------------------------------- + * Backend-private tracking for tuple sLog cleanup + * ---------------------------------------------------------------- + */ + +/* + * SLogTupleTrackKey + * Remember a tuple key for cleanup at commit/abort. + * + * Allocated in TopTransactionContext so it's automatically freed + * when the transaction ends. + */ +void +SLogTupleTrackKey(SLogTupleKey key, TransactionId xid, TransactionId subxid, + SLogOpType op_type) +{ + MemoryContext oldcxt; + SLogTrackedKey *tk; + + oldcxt = MemoryContextSwitchTo(TopTransactionContext); + + tk = (SLogTrackedKey *) palloc(sizeof(SLogTrackedKey)); + memcpy(&tk->key, &key, sizeof(SLogTupleKey)); + tk->xid = xid; + tk->subxid = subxid; + tk->local_only = false; + slog_has_shared_entries = true; + tk->op_type = op_type; + tk->before_image = NULL; + tk->before_image_len = 0; + tk->before_flags = 0; + tk->before_commit_ts = 0; + tk->next = slog_tracked_keys; + slog_tracked_keys = tk; + + MemoryContextSwitchTo(oldcxt); +} + +/* + * SLogTupleTrackLocalOnly + * Lightweight local-only tracking (INSERTs only). + * + * Records (relid, tid, xid, subxid) in the per-backend local list WITHOUT + * creating a shared sLog entry. On subtransaction abort, + * SLogTupleRemoveBySubXid will create a shared ABORTED entry for visibility. + * + * OOM optimization: When not inside a subtransaction, records the TID in a + * backend-local open-addressing hash (simplehash) keyed by (relid, + * encoded_tid) instead of a 136-byte linked-list node, giving O(1) insert + * and probe and bounded per-backend memory under bulk INSERT. Subtransaction + * entries still use the linked list because subtxn abort needs per-entry + * subxid filtering. + */ +void +SLogTupleTrackLocalOnly(Oid relid, ItemPointer tid, + TransactionId xid, TransactionId subxid) +{ + MemoryContext oldcxt; + + /* + * Fast path: top-level transaction with no savepoint ā†’ use the hash. + * The subxid is InvalidTransactionId in this case (top-level INSERTs + * always pass the top xid as both xid and subxid=Invalid). + */ + if (!IsSubTransaction()) + { + BlockNumber blkno = ItemPointerGetBlockNumber(tid); + OffsetNumber offnum = ItemPointerGetOffsetNumber(tid); + SLogInsertTidKey ikey; + bool found; + + if (offnum > MaxOffsetNumber) + elog(ERROR, "offset %u exceeds page item limit %d in sLog TID encoding", + offnum, MaxOffsetNumber); + ikey.relid = relid; + ikey.encoded_tid = SLOG_ENCODE_TID(blkno, offnum); + + /* + * Record the TID in the backend-local hash. Insert is amortized + * O(1); the table and its entries live in TopTransactionContext and + * are destroyed at transaction end. + */ + (void) sloginsert_insert(slog_insert_tids_ensure(), ikey, &found); + return; + } + + { + SLogTrackedKey *tk; + SLogTupleKey key; + + memset(&key, 0, sizeof(key)); + key.relid = relid; + ItemPointerCopy(tid, &key.tid); + + oldcxt = MemoryContextSwitchTo(TopTransactionContext); + + tk = (SLogTrackedKey *) palloc(sizeof(SLogTrackedKey)); + memcpy(&tk->key, &key, sizeof(SLogTupleKey)); + tk->xid = xid; + tk->subxid = subxid; + tk->local_only = true; + tk->op_type = SLOG_OP_INSERT; + tk->before_image = NULL; + tk->before_image_len = 0; + tk->before_flags = 0; + tk->before_commit_ts = 0; + tk->next = slog_tracked_keys; + slog_tracked_keys = tk; + + MemoryContextSwitchTo(oldcxt); + } +} + +/* + * SLogTupleUntrackLocalOnly + * Remove local-only INSERT tracking for (relid, tid). + * + * Counterpart to SLogTupleTrackLocalOnly. Used when a freshly inserted tuple + * must NOT be stamped with the inserting transaction's commit marker at commit + * time -- e.g. VACUUM FULL / CLUSTER copies a recently-dead tombstone into the + * new relation and rewrites it with its ORIGINAL delete timestamp. Leaving the + * INSERT tracked would let the AM's commit-time stamping clobber the tuple's + * commit metadata with the rewrite transaction's commit marker, resurrecting + * the deleted row for any reader whose snapshot predates that commit. + * + * Clears the backend-local INSERT-hash entry on the top-level fast path and + * also drops any matching linked-list node (the savepoint/fallback path). + */ +void +SLogTupleUntrackLocalOnly(Oid relid, ItemPointer tid) +{ + BlockNumber blkno = ItemPointerGetBlockNumber(tid); + OffsetNumber offnum = ItemPointerGetOffsetNumber(tid); + SLogTrackedKey **link; + + if (offnum > MaxOffsetNumber) + elog(ERROR, "offset %u exceeds page item limit %d in sLog TID encoding", + offnum, MaxOffsetNumber); + + /* Fast path: drop the backend-local INSERT-hash entry if present. */ + if (slog_insert_tids != NULL) + { + SLogInsertTidKey ikey; + + ikey.relid = relid; + ikey.encoded_tid = SLOG_ENCODE_TID(blkno, offnum); + (void) sloginsert_delete(slog_insert_tids, ikey); + } + + /* Fallback/subtxn path: unlink any matching local-only INSERT node. */ + link = &slog_tracked_keys; + while (*link != NULL) + { + SLogTrackedKey *tk = *link; + + if (tk->local_only && + tk->key.relid == relid && + ItemPointerGetBlockNumber(&tk->key.tid) == blkno && + ItemPointerGetOffsetNumber(&tk->key.tid) == offnum) + { + *link = tk->next; + pfree(tk); + continue; + } + link = &tk->next; + } +} + +/* + * SLogTupleStoreBeforeImage + * Attach a before-image to the most recent tracked key for the given + * (relid, tid, xid) combination. + * + * This is called during DELETE and UPDATE operations to stash the original + * tuple data before in-place modification. On subtransaction abort, the AM's + * before-image restore path uses this data to physically restore the tuple. + * + * The before-image is allocated in TopTransactionContext so it survives + * subtransaction rollback. Memory is freed when the tracked key list is + * reset at top-level transaction end. + * + * Size cap: if the tuple is larger than the registered AM's before_image_max + * (0 if no AM opted in), we skip storing the before-image. On savepoint + * rollback for such tuples, the tuple cannot be restored and the operation + * will raise an error. + */ +void +SLogTupleStoreBeforeImage(Oid relid, ItemPointer tid, TransactionId xid, + const char *data, int len, + uint16 flags, uint64 commit_ts, + RelFileLocator rlocator, char relpersistence) +{ + SLogTrackedKey *tk; + MemoryContext oldcxt; + + /* Enforce the opting-in AM's size cap */ + if (slog_am_desc.before_image_max == 0 || + len > (int) slog_am_desc.before_image_max) + return; + + /* Find the matching tracked key (most recently added = list head) */ + for (tk = slog_tracked_keys; tk != NULL; tk = tk->next) + { + if (!TransactionIdEquals(tk->xid, xid)) + continue; + if (tk->key.relid != relid) + continue; + if (!ItemPointerEquals(&tk->key.tid, tid)) + continue; + + /* Found it ā€” store local copy for savepoint rollback */ + oldcxt = MemoryContextSwitchTo(TopTransactionContext); + + tk->before_image = palloc(len); + memcpy(tk->before_image, data, len); + tk->before_image_len = len; + tk->before_flags = flags; + tk->before_commit_ts = commit_ts; + tk->before_rlocator = rlocator; + tk->before_relpersistence = relpersistence; + + MemoryContextSwitchTo(oldcxt); + + /* + * Only the local per-backend before-image is stored -- consumed by + * the AM's intra-transaction savepoint rollback. Cross-backend MVCC + * reads no longer need a shared before-image: WS-PVS3 walks the + * durable UNDO fork chain via the AM's version-reconstruction walk, + * and the write-write conflict probe resolves the head verptr in the + * same fork. + */ + + return; + } + + /* Tracked key not found ā€” this shouldn't happen, but is non-fatal */ + elog(WARNING, "SLogTupleStoreBeforeImage: no tracked key for rel %u tid (%u,%u) xid %u", + relid, ItemPointerGetBlockNumber(tid), + ItemPointerGetOffsetNumber(tid), xid); +} + +/* + * SLogTupleIterateTrackedKeysForSubXid + * Iterate over tracked keys matching a given xid AND subxid. + * + * This is used by the AM's before-image restore path to find entries that + * need physical restoration during savepoint rollback. + */ +void +SLogTupleIterateTrackedKeysForSubXid(TransactionId xid, + TransactionId subxid, + SLogTrackedKeyCallback callback, + void *arg) +{ + SLogTrackedKey *tk; + + for (tk = slog_tracked_keys; tk != NULL; tk = tk->next) + { + if (!TransactionIdEquals(tk->xid, xid)) + continue; + if (tk->subxid != subxid) + continue; + + if (!callback(&tk->key, tk->xid, tk->subxid, tk->local_only, arg)) + break; + } +} + +/* + * SLogTupleGetBeforeImage + * Retrieve the before-image for a specific tracked key. + * + * Returns true if a before-image is available, filling in the output params. + * Returns false if no before-image was stored (e.g., INSERT, or tuple was + * too large). + */ +bool +SLogTupleGetBeforeImage(Oid relid, ItemPointer tid, TransactionId xid, + TransactionId subxid, + char **data_out, int *len_out, + uint16 *flags_out, uint64 *commit_ts_out, + RelFileLocator *rlocator_out, char *relpersistence_out) +{ + SLogTrackedKey *tk; + + for (tk = slog_tracked_keys; tk != NULL; tk = tk->next) + { + if (!TransactionIdEquals(tk->xid, xid)) + continue; + if (tk->subxid != subxid) + continue; + if (tk->key.relid != relid) + continue; + if (!ItemPointerEquals(&tk->key.tid, tid)) + continue; + + if (tk->before_image == NULL) + return false; + + *data_out = tk->before_image; + *len_out = tk->before_image_len; + *flags_out = tk->before_flags; + *commit_ts_out = tk->before_commit_ts; + *rlocator_out = tk->before_rlocator; + *relpersistence_out = tk->before_relpersistence; + return true; + } + + return false; +} + +/* + * SLogTupleCleanupRetained + * Free retained sLog entries that are no longer visible to any + * active snapshot. + * + * WS-PVS3: committed UPDATE markers are no longer retained (commit_xid + * apply removes them), so under steady-state this routine primarily + * cleans up other retained state. Reclamation gates on the xid horizon + * (GetOldestNonRemovableTransactionId) so a marker is reclaimable only + * once its committing xid precedes the oldest active snapshot's xmin. + * + * Scans the flat hash under read-side, then applies CLEANUP_RETAINED ops. + * Walks every partition taking each writer lock LW_EXCLUSIVE, so callers MUST + * NOT hold a buffer content lock or other page-level critical section across + * this call. Driven by the UNDO background worker, or by + * SLogTupleMaybeCleanupRetained() when the worker is disabled. The + * reclamation decision is the xid horizon (GetOldestNonRemovableTransactionId) + * computed below. + */ +void +SLogTupleCleanupRetained(void) +{ + SLogTupleKey *collected_keys; + int max_keys = 256; + int part; + int i; + TransactionId reclaim_xid_horizon; + + if (SLogState == NULL) + return; + + /* + * Compute the xid horizon once for this pass. The read-side eligibility + * scan and the CLEANUP_RETAINED apply gate on the same horizon so the set + * of slots reclaimed here matches the set the apply drops. + */ + reclaim_xid_horizon = GetOldestNonRemovableTransactionId(NULL); + if (!TransactionIdIsValid(reclaim_xid_horizon)) + return; + + collected_keys = (SLogTupleKey *) + palloc(max_keys * sizeof(SLogTupleKey)); + + /* + * Process each partition independently while holding its exclusive writer + * lock. The writer lock serializes all writers (other cleanup runs and + * forward-path inserts) for the partition, so the single copy is stable + * to scan. We apply the CLEANUP_RETAINED ops inside seqlock cycles. + */ + for (part = 0; part < SLogNumPartitions; part++) + { + SLogFlatPartition *fp = SLogGetPartitionByIndex(part); + SLogFlatHashScanState scan; + const SLogFlatBucket *bucket; + int nkeys = 0; + + LWLockAcquire(&fp->writer_lock.lock, LW_EXCLUSIVE); + + /* + * Scan the single copy to collect expired keys. Stable because we + * hold the writer lock. + */ + SLogFlatHashScanInit(&scan); + while ((bucket = SLogFlatHashScanNext(fp->hash, &scan)) != NULL) + { + const SLogTupleEntry *entry = &bucket->entry; + bool has_expired = false; + + for (i = 0; i < SLOG_MAX_TUPLE_OPS; i++) + { + if (!entry->ops[i].in_use) + continue; + if (entry->ops[i].op_type != SLOG_OP_UPDATE) + continue; + if (!TransactionIdIsValid(entry->ops[i].xid)) + continue; + + /* + * Reclaim once the xid precedes the oldest active snapshot's + * xmin (xid authority -- see the function header). This is + * the authoritative, self-healing gate: a below-horizon + * marker is reclaimable whether or not it was ever committed + * in CLOG, and an in-progress xid can never precede the + * horizon. + */ + if (!TransactionIdPrecedes(entry->ops[i].xid, + reclaim_xid_horizon)) + continue; + + if (!has_expired) + { + if (nkeys >= max_keys) + { + max_keys *= 2; + collected_keys = (SLogTupleKey *) + repalloc(collected_keys, + max_keys * sizeof(SLogTupleKey)); + } + collected_keys[nkeys++] = bucket->key; + has_expired = true; + } + } + } + + /* Null the now-dangling pointers and drop the slots. */ + if (nkeys > 0) + { + int ops_since_publish = 0; + + SLogSeqWriteBegin(fp); + for (i = 0; i < nkeys; i++) + { + SLogFlatOp flat_op; + + memset(&flat_op, 0, sizeof(flat_op)); + flat_op.kind = SLOG_FLAT_OP_CLEANUP_RETAINED; + flat_op.key = collected_keys[i]; + flat_op.reclaim_xid_horizon = reclaim_xid_horizon; + SLogFlatHashApply(fp->hash, &flat_op, sizeof(flat_op)); + + /* + * Bound the seqlock odd-hold window so wait-free readers of + * this partition make progress. On a hot row many retained + * UPDATE markers expire at once, so nkeys can be large; a + * single uninterrupted odd window would spin every reader for + * its whole duration. Each key's reclamation is independent + * and each seq cycle leaves fp->hash consistent, so ending + * and re-beginning at a batch boundary is safe -- the + * partition LWLock still serializes us against other + * before-image freers. + */ + if (++ops_since_publish >= SLOG_ABORT_PUBLISH_BATCH) + { + SLogSeqWriteEnd(fp); + SLogSeqWriteBegin(fp); + ops_since_publish = 0; + } + } + SLogSeqWriteEnd(fp); + } + + LWLockRelease(&fp->writer_lock.lock); + } + + pfree(collected_keys); +} + +/* + * SLogTupleMaybeCleanupRetained + * Throttled, self-clocking entry point for retained-entry cleanup. + * + * Intended for access methods to call from their DML paths when the UNDO + * background worker is disabled (max_logical_revert_workers = 0), so the DSA + * before-image area does not fill up under sustained high-TPS load. Unlike + * SLogTupleCleanupRetained(), the heavy global sweep here runs at most once + * every 5 seconds per backend; the common call is a counter increment that + * returns immediately. + * + * CRITICAL: the caller MUST NOT hold any buffer content lock or other + * page-level critical section across this call. When the throttle fires this + * walks every sLog partition taking each writer lock LW_EXCLUSIVE; running it + * under a buffer lock would serialize all writers to a hot page behind the + * sweep (the c>=2 hot-row UPDATE convoy this throttle was introduced to avoid). + */ +void +SLogTupleMaybeCleanupRetained(void) +{ + static TimestampTz slog_last_cleanup = 0; + static uint32 slog_cleanup_clock = 0; + TimestampTz now_ts; + + if (SLogState == NULL) + return; + + /* + * Sample the wall clock only once every SLOG_INSERT_CLOCK_PERIOD calls. + * The throttle below already bounds the heavy sweep to once per 5s; this + * just keeps the common-case cost down to a counter increment so the + * post-unlock call on every UPDATE does not add a clock_gettime() syscall + * to the hot path (matters for the >=HEAP perf target). + */ + if ((slog_cleanup_clock++ & (SLOG_INSERT_CLOCK_PERIOD - 1)) != 0) + return; + + now_ts = GetCurrentTimestamp(); + + if (now_ts - slog_last_cleanup > 5000000) /* 5 seconds */ + { + slog_last_cleanup = now_ts; + SLogTupleCleanupRetained(); + } +} + +/* + * SLogTupleAnyTracked + * True iff the current backend has tracked any tuple key (INSERT, + * UPDATE, or DELETE) for the current transaction -- i.e. this backend + * touched at least one tracked tuple since the last + * SLogTupleResetTracking(). + * + * Used by the AM's xact callback to decide whether a transaction that + * touched no tracked tuples should skip writing a durable commit-map entry. + */ +bool +SLogTupleAnyTracked(void) +{ + return slog_tracked_keys != NULL || + (slog_insert_tids != NULL && slog_insert_tids->members > 0); +} + +/* + * SLogTupleResetTracking + * Clear the backend-private tracking list and reset overflow state. + * + * Also frees the backend-local INSERT hash. The table lives in + * TopTransactionContext, so it would be freed at transaction end anyway. + * Explicit cleanup here allows earlier memory reclaim and makes the state + * consistent for any subsequent operations within the same backend lifetime. + */ +void +SLogTupleResetTracking(void) +{ + /* Destroy the backend-local INSERT hash */ + if (slog_insert_tids != NULL) + { + sloginsert_destroy(slog_insert_tids); + slog_insert_tids = NULL; + } + + slog_tracked_keys = NULL; + slog_has_shared_entries = false; + slog_overflow_warning_count = 0; + slog_overflow_last_warning = 0; +} + +/* + * SLogTupleIterateTrackedKeys + * Iterate over tracked keys for a given xid. + * + * Calls the callback for each tracked key matching the given xid. + * If the callback returns false, iteration stops early. + * Used by AM-specific pre-commit callbacks that need to touch pages. + * + * Iterates both hash-based INSERT entries (top-level) and + * linked-list entries. + */ +void +SLogTupleIterateTrackedKeys(TransactionId xid, + SLogTrackedKeyCallback callback, + void *arg) +{ + SLogTrackedKey *tk; + + /* Iterate backend-local INSERT-hash entries (top-level local-only) */ + if (slog_insert_tids != NULL) + { + sloginsert_iterator it; + SLogInsertTidEntry *ie; + + sloginsert_start_iterate(slog_insert_tids, &it); + while ((ie = sloginsert_iterate(slog_insert_tids, &it)) != NULL) + { + SLogTupleKey key; + + memset(&key, 0, sizeof(key)); + key.relid = ie->key.relid; + ItemPointerSet(&key.tid, + SLOG_DECODE_BLKNO(ie->key.encoded_tid), + SLOG_DECODE_OFFNUM(ie->key.encoded_tid)); + + if (!callback(&key, xid, InvalidTransactionId, true, arg)) + return; + } + } + + /* Iterate linked-list entries */ + for (tk = slog_tracked_keys; tk != NULL; tk = tk->next) + { + if (!TransactionIdEquals(tk->xid, xid)) + continue; + + if (!callback(&tk->key, tk->xid, tk->subxid, tk->local_only, arg)) + break; + } +} + +/* + * SLogTupleIterateTrackedKeysExt + * Extended iteration that also passes before-image metadata. + * + * Used by commit-time callbacks that need the original t_commit_ts + * to restore it on in-place-updated tuples (preserving visibility + * for readers with older snapshots). + * + * Sparsemap entries are local-only INSERTs with no before-image, so + * before_commit_ts=0 and has_before_image=false for those entries. + */ +void +SLogTupleIterateTrackedKeysExt(TransactionId xid, + SLogTrackedKeyExtCallback callback, + void *arg) +{ + SLogTrackedKey *tk; + + /* Iterate backend-local INSERT-hash entries (top-level local-only) */ + if (slog_insert_tids != NULL) + { + sloginsert_iterator it; + SLogInsertTidEntry *ie; + + sloginsert_start_iterate(slog_insert_tids, &it); + while ((ie = sloginsert_iterate(slog_insert_tids, &it)) != NULL) + { + SLogTupleKey key; + + memset(&key, 0, sizeof(key)); + key.relid = ie->key.relid; + ItemPointerSet(&key.tid, + SLOG_DECODE_BLKNO(ie->key.encoded_tid), + SLOG_DECODE_OFFNUM(ie->key.encoded_tid)); + + if (!callback(&key, xid, InvalidTransactionId, true, + 0, false, arg)) + return; + } + } + + /* Iterate linked-list entries */ + for (tk = slog_tracked_keys; tk != NULL; tk = tk->next) + { + if (!TransactionIdEquals(tk->xid, xid)) + continue; + + if (!callback(&tk->key, tk->xid, tk->subxid, tk->local_only, + tk->before_commit_ts, + (tk->before_image != NULL), + arg)) + break; + } +} + +/* + * SLogTupleCollectTrackedKeys + * Collect all tracked keys for a given xid into a palloc'd array. + * + * Returns the number of collected keys. The caller can then sort the + * array for batch processing (e.g. by relid/blockno to amortize buffer + * I/O at commit time). + * + * The returned array is allocated in the current memory context; the caller + * is responsible for pfree(). + */ +int +SLogTupleCollectTrackedKeys(TransactionId xid, SLogTrackedKeyInfo **out_keys) +{ + SLogTrackedKey *tk; + int count = 0; + int capacity = 64; + SLogTrackedKeyInfo *arr; + + /* + * A single very large transaction can touch tens of millions of tracked + * keys; at 48 bytes each the array crosses MaxAllocSize (1 GB) past ~22M + * entries. Use the huge-allocation API so bulk loads do not fail at + * commit. This array is transient and pfree'd by the caller. + */ + arr = (SLogTrackedKeyInfo *) MemoryContextAllocHuge(CurrentMemoryContext, + sizeof(SLogTrackedKeyInfo) * (Size) capacity); + + /* Collect backend-local INSERT-hash entries (top-level local-only) */ + if (slog_insert_tids != NULL) + { + sloginsert_iterator it; + SLogInsertTidEntry *ie; + + sloginsert_start_iterate(slog_insert_tids, &it); + while ((ie = sloginsert_iterate(slog_insert_tids, &it)) != NULL) + { + if (count >= capacity) + { + capacity *= 2; + arr = (SLogTrackedKeyInfo *) + repalloc_huge(arr, sizeof(SLogTrackedKeyInfo) * (Size) capacity); + } + + memset(&arr[count].key, 0, sizeof(SLogTupleKey)); + arr[count].key.relid = ie->key.relid; + ItemPointerSet(&arr[count].key.tid, + SLOG_DECODE_BLKNO(ie->key.encoded_tid), + SLOG_DECODE_OFFNUM(ie->key.encoded_tid)); + arr[count].xid = xid; + arr[count].subxid = InvalidTransactionId; + arr[count].local_only = true; + arr[count].op_type = SLOG_OP_INSERT; + arr[count].before_commit_ts = 0; + arr[count].has_before_image = false; + count++; + } + } + + /* Collect linked-list entries */ + for (tk = slog_tracked_keys; tk != NULL; tk = tk->next) + { + if (!TransactionIdEquals(tk->xid, xid)) + continue; + + if (count >= capacity) + { + capacity *= 2; + arr = (SLogTrackedKeyInfo *) + repalloc_huge(arr, sizeof(SLogTrackedKeyInfo) * (Size) capacity); + } + + arr[count].key = tk->key; + arr[count].xid = tk->xid; + arr[count].subxid = tk->subxid; + arr[count].local_only = tk->local_only; + arr[count].op_type = tk->op_type; + arr[count].before_commit_ts = tk->before_commit_ts; + arr[count].has_before_image = (tk->before_image != NULL); + count++; + } + + *out_keys = arr; + return count; +} diff --git a/src/backend/utils/misc/guc_parameters.dat b/src/backend/utils/misc/guc_parameters.dat index 0de04aec1cede..ae437b6679f21 100644 --- a/src/backend/utils/misc/guc_parameters.dat +++ b/src/backend/utils/misc/guc_parameters.dat @@ -2783,6 +2783,15 @@ max => '16384', }, +{ name => 'slog_num_partitions', type => 'int', context => 'PGC_POSTMASTER', group => 'WAL_SETTINGS', + short_desc => 'Number of sLog flat hash partitions (0 = auto from CPU count).', + long_desc => 'Controls the number of partitions for the optional sLog tuple tracking hash. More partitions reduce writer lock contention. Set 0 for auto-sizing (4x CPUs, clamped 16-256, power of 2). Requires restart.', + variable => 'slog_num_partitions', + boot_val => '0', + min => '0', + max => '256', +}, + { name => 'ssl', type => 'bool', context => 'PGC_SIGHUP', group => 'CONN_AUTH_SSL', short_desc => 'Enables SSL connections.', variable => 'EnableSSL', diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index cd89ba3b85f71..40b9ab6c569de 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -924,6 +924,7 @@ #undo_batch_size_kb = 256 # KB per UNDO batch flush (64-4096); reload #undo_batch_record_limit = 1000 # records per UNDO batch flush (100-100000); reload #slog_dsa_max_size_mb = 256 # MB; max sLog DSA area size (aborted-txn tree) +#slog_num_partitions = 0 # 0 = auto (4x CPUs); requires restart #max_logical_revert_workers = 2 # 0 disables the logical revert launcher #logical_revert_naptime = 1s # time between ATM scan cycles diff --git a/src/include/access/slog.h b/src/include/access/slog.h index 6fe4d64c5fc5e..9de09a75b2d90 100644 --- a/src/include/access/slog.h +++ b/src/include/access/slog.h @@ -32,9 +32,60 @@ #include "access/transam.h" #include "access/xlogdefs.h" #include "datatype/timestamp.h" +#include "nodes/lockoptions.h" +#include "storage/itemptr.h" #include "storage/lwlock.h" +#include "storage/relfilelocator.h" #include "utils/dsa.h" +/* + * Maximum concurrent operations on a single TID. + * + * Under EPQ retry patterns with high concurrency, each backend that sees + * TM_Updated calls table_tuple_lock which adds a LOCK_EXCL sLog entry. + * These entries persist until the owning transaction commits, so with N + * concurrent backends hitting the same hot row, up to N-1 LOCK entries + * can coexist. Additionally, committed in-place UPDATE markers are retained + * (one per committing xid) until the oldest-snapshot horizon advances past + * them, so a hot row updated repeatedly while a long reader holds a snapshot + * accumulates markers on top of the live LOCK entries. 128 slots handles + * realistic OLTP concurrency plus retained-marker headroom; per-TID + * reclamation of the oldest retained marker kicks in before the array fills. + */ +#define SLOG_MAX_TUPLE_OPS 128 + +/* + * Tuple hash auto-sizing constants. + * + * All DML operations (INSERT/DELETE/UPDATE/LOCK) attempt to create shared + * hash entries. On overflow (hash full), the operation proceeds gracefully + * with local-only tracking rather than crashing. Auto-sizing formula: + * MaxBackends * SLOG_TUPLE_PER_BACKEND_SLOTS, clamped. + * + * The per-backend slot count must be large enough to accommodate OLTP + * workloads where UPDATE before-images are retained until eviction. + * With transactions touching ~25 rows, and retained entries from committed + * transactions accumulating between eviction passes, 1024 slots per backend + * provides adequate headroom. + */ +#define SLOG_TUPLE_PER_BACKEND_SLOTS 1024 +#define SLOG_TUPLE_MIN_ENTRIES 4096 +#define SLOG_TUPLE_MAX_ENTRIES 4194304 + +/* + * sLog entry types for tuple operations. + */ +typedef enum SLogOpType +{ + SLOG_ENTRY_ABORTED_TXN = 0, /* Transaction-level abort entry */ + SLOG_OP_INSERT = 1, + SLOG_OP_DELETE = 2, + SLOG_OP_UPDATE = 3, + SLOG_OP_LOCK_SHARE = 4, + SLOG_OP_LOCK_EXCL = 5, + SLOG_OP_ABORTED = 6, /* Tuple-level: op was aborted, UNDO pending */ +} SLogOpType; + /* ---------------------------------------------------------------- * Transaction sLog structures * @@ -58,6 +109,84 @@ typedef struct SLogTxnEntry bool revert_complete; /* has Logical Revert finished? */ } SLogTxnEntry; +/* ---------------------------------------------------------------- + * Tuple sLog structures + * ---------------------------------------------------------------- + */ + +/* + * SLogTupleKey - Hash key for the tuple flat hash. + * + * Note: ItemPointerData is 6 bytes. Always memset(&key, 0, sizeof(key)) + * before populating to ensure deterministic byte hashing. + */ +typedef struct SLogTupleKey +{ + Oid relid; + ItemPointerData tid; +} SLogTupleKey; + +/* + * SLogTupleOp - Single operation on a tuple. + * + * Uses an in_use slot-based model for O(1) removal without array compaction + * under the exclusive lock. + */ +typedef struct SLogTupleOp +{ + TransactionId xid; + TransactionId subxid; /* subtransaction ID, or InvalidTransactionId */ + SLogOpType op_type; + CommandId cid; + TimestampTz commit_ts; /* 0 if not yet committed */ + uint32 spec_token; /* speculative insertion token, or 0 */ + bool in_use; /* slot occupied? */ + + /* + * Precise tuple-lock strength for LOCK_SHARE/LOCK_EXCL markers. The + * SLOG_OP_LOCK_* op_type only distinguishes shared vs. exclusive intent; + * this field preserves the full four-way LockTupleMode so a concurrent + * updater can apply the real heavyweight conflict matrix (a KeyShare FK + * locker is compatible with a NoKeyExclusive UPDATE, but a + * Share/Exclusive locker conflicts and must block it). Ignored for + * non-lock ops. + */ + LockTupleMode lock_mode; +} SLogTupleOp; + +/* + * SLogTupleEntry - Value in the tuple flat hash. + */ +typedef struct SLogTupleEntry +{ + SLogTupleKey key; /* hash key */ + int nops; /* number of active operations */ + SLogTupleOp ops[SLOG_MAX_TUPLE_OPS]; +} SLogTupleEntry; + +/* + * SLogAmDescriptor - per-AM opt-in policy for the tuple sLog. + * + * An in-place-MVCC table AM that wants tuple tracking registers one of these + * once at startup via SLogRegisterAmDescriptor(). It carries only DATA + * resolved once at registration -- never a per-op callback -- so it adds no + * indirection to the tuple probe/insert hot path. The AM, not the generic + * tuple sLog, owns the policy values here. + */ +typedef struct SLogAmDescriptor +{ + /* + * Maximum size (bytes) of a before-image the AM will stash in the + * backend-local savepoint-rollback scratch. A larger tuple is not + * stored; the AM falls back to its durable UNDO fork for that version. 0 + * means the AM stores no before-images through the tuple sLog. + */ + uint32 before_image_max; +} SLogAmDescriptor; + +/* Callback type for tuple iteration */ +typedef bool (*SLogTupleIterCallback) (const SLogTupleOp *op, void *arg); + /* ---------------------------------------------------------------- * Shared state * @@ -100,4 +229,165 @@ extern void SLogEnsureDsaAttached(void); /* GUC: maximum DSA size (in MB) */ extern int slog_dsa_max_size_mb; +/* ---------------------------------------------------------------- + * API: Tuple sLog (optional per-tuple tracking extension, slog_tuple.c) + * ---------------------------------------------------------------- + */ + +/* Per-AM opt-in registration (call once at startup) */ +extern void SLogRegisterAmDescriptor(const SLogAmDescriptor *desc); + +/* Dynamic sizing */ +extern int SLogTupleNumEntries(void); + +/* Core operations */ +extern bool SLogTupleInsert(Oid relid, ItemPointer tid, TransactionId xid, + SLogOpType op_type, TransactionId subxid, + CommandId cid, TimestampTz commit_ts, + uint32 spec_token, LockTupleMode lock_mode); +extern bool SLogTupleInsertRecovery(Oid relid, ItemPointer tid, + TransactionId xid, SLogOpType op_type); +extern bool SLogTupleLookup(Oid relid, ItemPointer tid, + SLogTupleEntry *entry_out); +extern void SLogTupleRemove(Oid relid, ItemPointer tid, TransactionId xid); +extern void SLogTupleRemoveByXid(TransactionId xid); +extern void SLogTupleRemoveBySubXid(TransactionId xid, TransactionId subxid); +extern void SLogTupleIterateByTid(Oid relid, ItemPointer tid, + SLogTupleIterCallback callback, void *arg); + +/* Filtered lookup (xid_filter=InvalidTransactionId means all) */ +extern int SLogTupleLookupFiltered(Oid relid, ItemPointer tid, + TransactionId xid_filter, + SLogTupleOp *ops_out, int max_ops); + +/* Subtransaction re-parenting on subxact commit */ +extern void SLogTupleUpdateSubXid(TransactionId xid, + TransactionId old_subxid, + TransactionId new_subxid); + +/* Mark all ops for xid as SLOG_OP_ABORTED */ +extern void SLogTupleMarkAborted(TransactionId xid); + +/* Global removal for UNDO worker (no backend-local list) */ +extern void SLogTupleRemoveByXidGlobal(TransactionId xid); + +/* Lightweight local-only tracking (INSERTs only) */ +extern void SLogTupleTrackLocalOnly(Oid relid, ItemPointer tid, + TransactionId xid, TransactionId subxid); +extern void SLogTupleUntrackLocalOnly(Oid relid, ItemPointer tid); + +/* Convenience wrappers */ +extern bool SLogTupleHasEntry(Oid relid, ItemPointer tid); +extern bool SLogTupleIsInsertedByMe(Oid relid, ItemPointer tid); +extern bool SLogTupleIsDeletedByMe(Oid relid, ItemPointer tid); +extern TransactionId SLogTupleGetDirtyXid(Oid relid, ItemPointer tid, + bool *is_insert); +extern TransactionId SLogTupleGetDirtyWriterXid(Oid relid, ItemPointer tid, + bool *is_insert); +extern TransactionId SLogTupleGetWriteConflictXid(Oid relid, ItemPointer tid, + LockTupleMode my_mode, + bool *is_insert); +extern bool SLogTupleHasLockConflict(Oid relid, ItemPointer tid, + TransactionId my_xid, + SLogOpType requested_lock); +extern bool SLogTupleGetLockConflictXid(Oid relid, ItemPointer tid, + TransactionId my_xid, + SLogOpType requested_lock, + TransactionId *xid_out); +extern bool SLogTupleHasAbortedEntry(Oid relid, ItemPointer tid); + + +/* Backend-private tracking for cleanup at commit/abort */ +extern void SLogTupleTrackKey(SLogTupleKey key, TransactionId xid, + TransactionId subxid, SLogOpType op_type); +extern void SLogTupleResetTracking(void); +extern bool SLogTupleAnyTracked(void); + +/* + * SLogTrackedKeyInfo -- public snapshot of a tracked key for batch processing. + * + * Returned by SLogTupleCollectTrackedKeys() so that callers (e.g. an AM's + * commit-time stamping) can sort and batch-process tuples without knowledge + * of the internal tracked-key linked-list structure. + */ +typedef struct SLogTrackedKeyInfo +{ + SLogTupleKey key; + TransactionId xid; + TransactionId subxid; + bool local_only; + SLogOpType op_type; + uint64 before_commit_ts; + bool has_before_image; +} SLogTrackedKeyInfo; + +/* Collect tracked keys into a sortable array (for batch commit processing) */ +extern int SLogTupleCollectTrackedKeys(TransactionId xid, + SLogTrackedKeyInfo **out_keys); + +/* Iterate tracked keys (for AM-specific pre-commit callbacks) */ +typedef bool (*SLogTrackedKeyCallback) (const SLogTupleKey *key, + TransactionId xid, + TransactionId subxid, + bool local_only, + void *arg); +extern void SLogTupleIterateTrackedKeys(TransactionId xid, + SLogTrackedKeyCallback callback, + void *arg); + +/* Extended callback with before-image metadata (for commit-time processing) */ +typedef bool (*SLogTrackedKeyExtCallback) (const SLogTupleKey *key, + TransactionId xid, + TransactionId subxid, + bool local_only, + uint64 before_commit_ts, + bool has_before_image, + void *arg); +extern void SLogTupleIterateTrackedKeysExt(TransactionId xid, + SLogTrackedKeyExtCallback callback, + void *arg); + +/* Iterate tracked keys for a specific subtransaction (savepoint rollback) */ +extern void SLogTupleIterateTrackedKeysForSubXid(TransactionId xid, + TransactionId subxid, + SLogTrackedKeyCallback callback, + void *arg); + +/* Before-image storage for savepoint rollback */ +extern void SLogTupleStoreBeforeImage(Oid relid, ItemPointer tid, + TransactionId xid, + const char *data, int len, + uint16 flags, uint64 commit_ts, + RelFileLocator rlocator, + char relpersistence); +extern bool SLogTupleGetBeforeImage(Oid relid, ItemPointer tid, + TransactionId xid, TransactionId subxid, + char **data_out, int *len_out, + uint16 *flags_out, uint64 *commit_ts_out, + RelFileLocator *rlocator_out, + char *relpersistence_out); + +/* Commit retention: retain committed UPDATE entries with before-images */ +extern void SLogTupleCommitByXid(TransactionId xid); + +/* Per-tuple operations for two-phase commit resolution */ +extern void SLogTupleRemoveByXidSingle(Oid relid, ItemPointer tid, + TransactionId xid); +extern void SLogTupleMarkAbortedSingle(Oid relid, ItemPointer tid, + TransactionId xid); + +/* Cleanup retained entries when no longer needed by any snapshot */ +extern void SLogTupleCleanupRetained(void); + +/* + * Throttled cleanup trigger for access methods to call from DML paths OUTSIDE + * any buffer-locked critical section (drives cleanup when the UNDO worker is + * disabled). Cheap to call repeatedly; fires the global sweep at most once + * every few seconds per backend. + */ +extern void SLogTupleMaybeCleanupRetained(void); + +/* GUC: number of sLog flat hash partitions (0 = auto based on CPU count) */ +extern int slog_num_partitions; + #endif /* SLOG_H */ diff --git a/src/include/access/slog_flathash.h b/src/include/access/slog_flathash.h new file mode 100644 index 0000000000000..6c32afe0324b0 --- /dev/null +++ b/src/include/access/slog_flathash.h @@ -0,0 +1,307 @@ +/*------------------------------------------------------------------------- + * + * slog_flathash.h + * Seqlock-protected flat open-addressing hash for sLog tuple tracking. + * + * This provides wait-free read access to sLog tuple entries via a seqlock + * (sequence lock). The flat hash uses open addressing with linear probing + * and tombstone markers for deletions. + * + * Architecture: A SINGLE copy of the hash lives in shared memory, guarded + * by a per-partition sequence counter (SLogFlatPartition.seq). Readers + * acquire-load the counter, read data into local variables, then re-load + * the counter and retry if it changed -- no announce-store, no StoreLoad + * fence (unlike the left-right lock this replaces). Writers are already + * mutually excluded by the per-partition writer_lock LWLock, so a writer + * simply makes the counter odd (write in progress), mutates the single + * copy in place, then makes it even again (+2 total). A reader that + * observes an odd counter spins until it turns even. + * + * Scan semantics: SLogFlatHashScanInit/ScanNext iterate all occupied + * buckets linearly. Read-side scans must run inside a seqlock retry loop + * (SLOG_SEQ_READ_BEGIN/END) and may only copy data into locals; a writer + * holds writer_lock and mutates fp->hash directly. For write operations + * that need global scans (eviction, xid removal), the pattern is: scan + * fp->hash under the writer lock (stable, no reader can tear it), collect + * keys, then apply write ops. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/access/slog_flathash.h + * + *------------------------------------------------------------------------- + */ +#ifndef SLOG_FLATHASH_H +#define SLOG_FLATHASH_H + +#include "access/slog.h" +#include "port/atomics.h" +#include "storage/lwlock.h" +#include "storage/s_lock.h" +#include "storage/seqlock.h" + +/* + * Bucket states encoded via hash_val: + * 0 = empty (never used) + * UINT32_MAX = tombstone (deleted) + * anything else = occupied with that hash value + */ +#define SLOG_FLAT_EMPTY 0 +#define SLOG_FLAT_TOMBSTONE UINT32_MAX + +/* + * SLogFlatBucket - One bucket in the flat open-addressing hash table. + * + * Layout: hash_val marks the state, key is the lookup key, entry contains + * the full SLogTupleEntry data (ops array etc). + */ +typedef struct SLogFlatBucket +{ + uint32 hash_val; /* 0=empty, TOMBSTONE=deleted, else hash */ + SLogTupleKey key; /* (relid, tid) */ + uint16 padding; /* alignment padding */ + SLogTupleEntry entry; /* nops + ops[SLOG_MAX_TUPLE_OPS] */ +} SLogFlatBucket; + +/* + * SLogFlatHash - The flat hash table header, followed by buckets[]. + * + * A single copy lives in shared memory, guarded by the partition seqlock. + */ +typedef struct SLogFlatHash +{ + int32 capacity; /* number of buckets (power of 2) */ + int32 num_entries; /* current live entries */ + int32 num_tombstones; /* tombstone count (for load factor) */ + int32 padding; + SLogFlatBucket buckets[FLEXIBLE_ARRAY_MEMBER]; +} SLogFlatHash; + +/* + * Operation kinds for the flat-hash apply dispatcher (SLogFlatHashApply). + */ +typedef enum SLogFlatOpKind +{ + SLOG_FLAT_OP_INSERT, /* Insert/update a single op slot */ + SLOG_FLAT_OP_REMOVE_XID, /* Remove all ops for xid from entry */ + SLOG_FLAT_OP_REMOVE_ENTRY, /* Remove entire entry (tombstone it) */ + SLOG_FLAT_OP_MARK_ABORTED, /* Mark all ops for xid as ABORTED */ + SLOG_FLAT_OP_UPDATE_OP, /* Update a specific op slot in-place */ + SLOG_FLAT_OP_COMMIT_XID, /* Handle commit retention for an entry */ + SLOG_FLAT_OP_CLEANUP_RETAINED, /* Remove old retained entries */ + SLOG_FLAT_OP_CREATE_ABORTED, /* Create a new ABORTED entry (for + * local-only) */ +} SLogFlatOpKind; + +/* + * SLogFlatOp - A single operation to be applied to the flat hash. + * + * Passed to SLogFlatHashApply(), which mutates the single flat-hash copy. + */ +typedef struct SLogFlatOp +{ + SLogFlatOpKind kind; + SLogTupleKey key; /* which entry */ + TransactionId xid; /* target xid */ + TransactionId subxid; /* for subxid operations */ + TransactionId reclaim_xid_horizon; /* INSERT: oldest active snapshot + * xmin; a committed UPDATE marker may + * be reclaimed only if its xid + * precedes this (visible to all + * snapshots) */ + SLogTupleOp tuple_op; /* the op to insert/update (for INSERT) */ +} SLogFlatOp; + +/* ---------------------------------------------------------------- + * Partitioned flat hash: 32-way sharding to reduce writer lock contention. + * + * Each partition has its own single flat-hash copy guarded by a seqlock + * (wait-free reads) and its own writer lock. Key routing: + * hash(key) % NUM_PARTITIONS. This reduces writer lock contention + * proportionally to the number of partitions. + * + * The partition count is determined at startup by the slog_num_partitions + * GUC (default: 0 = auto-size based on CPU count). The heuristic targets + * 4Ɨ the number of CPUs, clamped to [16, 256], rounded to next power of 2. + * This ensures that at peak concurrency each CPU core has ~4 partitions to + * spread writes across, minimizing writer lock wait time. + * ---------------------------------------------------------------- + */ + +/* Default partition count used only before SLogShmemInit sets the real value */ +#define SLOG_FLAT_DEFAULT_PARTITIONS 32 +#define SLOG_FLAT_MIN_PARTITIONS 16 +#define SLOG_FLAT_MAX_PARTITIONS 256 + +/* + * SLogFlatPartition - Per-partition state. + * + * Each partition owns a slice of the total flat hash capacity. Reads are + * wait-free via the seqlock (seq); writes are serialized by writer_lock and + * mutate the single copy (*hash) in place. + */ +typedef struct SLogFlatPartition +{ + SLogFlatHash *hash; /* single flat-hash copy (in shmem) */ + SeqLock seqlock; /* retry-based consistent reads */ + LWLockPadded writer_lock; /* per-partition writer serialization */ +} SLogFlatPartition; + +/* + * Seqlock write-side helpers (thin wrappers over the generic SeqLock). + * + * The caller MUST already hold the partition writer_lock LW_EXCLUSIVE, which + * is the sole writer mutual-exclusion mechanism (the seqlock provides no + * writer exclusion of its own). Between _begin and _end the counter is odd, + * so any concurrent reader retries; the writer mutates fp->hash directly. + */ +static inline void +SLogSeqWriteBegin(SLogFlatPartition *fp) +{ + SeqLockWriteBegin(&fp->seqlock); +} + +static inline void +SLogSeqWriteEnd(SLogFlatPartition *fp) +{ + SeqLockWriteEnd(&fp->seqlock); +} + +/* + * Seqlock read-side helpers -- "copy into locals, then act after". + * + * Usage: + * uint32 slog_seq_; + * SLOG_SEQ_READ_BEGIN(fp, slog_seq_) + * { + * ... probe fp->hash and memcpy/read into LOCAL variables ONLY ... + * } + * SLOG_SEQ_READ_END(fp, slog_seq_); + * ... now act on the locals ... + * + * The body may run more than once, so it must have no side effects beyond + * writing caller-provided output locals (reset any accumulator at the top of + * the body). It must NOT retain pointers into fp->hash past the loop nor run + * callbacks with side effects; those run after SLOG_SEQ_READ_END confirms a + * consistent read. These macros wrap the generic SeqLock reader API. + */ +#define SLOG_SEQ_READ_BEGIN(fp, seqvar) \ + for (;;) \ + { \ + (seqvar) = SeqLockReadBegin(&(fp)->seqlock); + +#define SLOG_SEQ_READ_END(fp, seqvar) \ + if (SeqLockReadRetry(&(fp)->seqlock, (seqvar))) \ + break; \ + } + +/* + * Compute the shared memory size needed for the flat hash data + * (the single copy guarded by the seqlock). + */ +extern Size SLogFlatHashDataSize(int capacity); + +/* + * Compute the total shared memory needed for one partition's flat hash + * (the single copy; the seqlock and writer_lock are embedded in + * SLogFlatPartition). + */ +extern Size SLogFlatHashShmemSize(int capacity, int max_backends); + +/* + * Compute the total shared memory needed for all partitions. + */ +extern Size SLogFlatHashPartitionedShmemSize(int total_capacity, + int max_backends); + +/* + * Initialize the flat hash in a pre-allocated data block. + * Called during SLogShmemInit to set up the single copy. + */ +extern void SLogFlatHashInit(void *data, int capacity); + +/* + * Apply one SLogFlatOp to the (single) flat-hash copy. Called by writers + * holding the partition writer_lock, between SLogSeqWriteBegin/End. + */ +extern void SLogFlatHashApply(void *data, const void *operation, Size op_size); + +/* + * Hash computation for SLogTupleKey. + */ +extern uint32 SLogFlatHashComputeHash(const SLogTupleKey *key); + +/* + * Runtime partition count ā€” set during SLogShmemInit() based on + * the slog_num_partitions GUC. Declared in slog.c. + */ +extern int SLogNumPartitions; + +/* + * Compute which partition a key belongs to. + */ +static inline int +SLogFlatHashPartitionIndex(const SLogTupleKey *key) +{ + return (int) (SLogFlatHashComputeHash(key) % (uint32) SLogNumPartitions); +} + +/* + * Probe the flat hash for a key. Returns pointer to the bucket if found, + * NULL if not found. Only valid during a read-side or write-side critical + * section. + */ +extern SLogFlatBucket *SLogFlatHashProbe(const SLogFlatHash *ht, + const SLogTupleKey *key); + +/* + * Return true iff the entry for key holds an in-use op for xid. Detects a + * silently-dropped op (per-TID array full) where the bucket itself exists. + * Only valid during a read-side or write-side critical section. + */ +extern bool SLogFlatHashHasOpForXid(const SLogFlatHash *ht, + const SLogTupleKey *key, + TransactionId xid); + +/* + * Find a bucket for insertion (first empty or tombstone slot on probe chain). + * Returns NULL if the table is full (all slots on probe chain occupied). + * Only valid during write-side critical section. + */ +extern SLogFlatBucket *SLogFlatHashProbeForInsert(SLogFlatHash *ht, + const SLogTupleKey *key, + uint32 hash_val); + +/* + * Scan API for iterating all occupied buckets. + * + * Used by global-scan operations (eviction, xid removal, cleanup) that + * need to visit every entry. The scan iterates linearly over the bucket + * array, skipping EMPTY and TOMBSTONE slots. + * + * Usage pattern: + * SLogFlatHashScanState state; + * const SLogFlatBucket *bucket; + * + * SLogFlatHashScanInit(&state); + * while ((bucket = SLogFlatHashScanNext(ht, &state)) != NULL) + * { + * // process bucket->entry + * } + * + * The scan must be performed inside a seqlock read-side retry loop + * (SLOG_SEQ_READ_BEGIN/END, read into locals only) or by a writer holding + * writer_lock. For write operations, scan fp->hash under the writer lock + * (stable) to collect keys, then apply write ops. + */ +typedef struct SLogFlatHashScanState +{ + int32 current_index; +} SLogFlatHashScanState; + +extern void SLogFlatHashScanInit(SLogFlatHashScanState *state); +extern const SLogFlatBucket *SLogFlatHashScanNext(const SLogFlatHash *ht, + SLogFlatHashScanState *state); + +#endif /* SLOG_FLATHASH_H */ diff --git a/src/include/access/slog_internal.h b/src/include/access/slog_internal.h new file mode 100644 index 0000000000000..74d08db3b7a05 --- /dev/null +++ b/src/include/access/slog_internal.h @@ -0,0 +1,68 @@ +/*------------------------------------------------------------------------- + * + * slog_internal.h + * Shared-state definitions private to the sLog implementation + * + * The sLog is split across two translation units that share one + * shared-memory segment: slog.c (the always-present transaction Aborted + * Transaction Map) and slog_tuple.c (the optional per-tuple flat-hash + * tracking extension). This header exposes the shared state struct and the + * few globals both files touch, so it is deliberately NOT part of the public + * sLog API in access/slog.h. Only the two sLog .c files include it. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/access/slog_internal.h + * + *------------------------------------------------------------------------- + */ +#ifndef SLOG_INTERNAL_H +#define SLOG_INTERNAL_H + +#include "access/slog_flathash.h" +#include "storage/lwlock.h" +#include "utils/dsa.h" + +/* + * Initial size for the sLog DSA area (backs the aborted-txn radix tree). + * Grows dynamically as needed up to slog_dsa_max_size_mb. + */ +#define SLOG_DSA_INIT_SIZE (512 * 1024) /* 512 KB */ + +/* + * Shared state for the whole sLog subsystem. + * + * The transaction ATM fields are always used. The tuple flat-hash fields + * (tuple_partitions, num_partitions) are used only when the tuple sLog + * extension is compiled in and an access method has opted in; they are + * initialized unconditionally in SLogShmemInit() so the two files can share + * one segment without an init-ordering dependency. + */ +typedef struct SLogSharedState +{ + /* Transaction ATM (adaptive radix tree in the DSA area below) */ + dsa_pointer atm_handle; /* RT handle; InvalidDsaPointer until init */ + LWLockPadded txn_lock; /* single LWLock serializing ATM access */ + + /* + * Tuple flat hash: N-way partitioned for reduced writer contention. + * Partition count determined at startup by slog_num_partitions GUC. + */ + SLogFlatPartition *tuple_partitions; /* palloc'd array in shmem */ + int num_partitions; /* actual partition count */ + + /* DSA area backing the aborted-txn radix tree */ + dsa_area *dsa_area; /* set during SLogShmemInit, NULL until then */ + char dsa_space[SLOG_DSA_INIT_SIZE]; +} SLogSharedState; + +/* The single shared-state instance (defined in slog.c). */ +extern SLogSharedState *SLogState; + +/* Tuple-hash shmem sizing/init helpers (defined in slog_tuple.c). */ +extern Size SLogTupleShmemSize(void); +extern void SLogTupleShmemRequest(void); +extern void SLogTupleShmemInit(void); + +#endif /* SLOG_INTERNAL_H */ From efb5c31960b96909b6164fae585e629d1155b175 Mon Sep 17 00:00:00 2001 From: Greg Burd Date: Mon, 27 Jul 2026 11:25:00 -0400 Subject: [PATCH 10/10] FLUX: add an UNDO-based heap-replacement table access method FLUX is a faithful realization of what zheap was designed to be: a HEAP replacement that stores old row versions in UNDO instead of leaving dead tuples in the table, built entirely on today's index and executor APIs with no new index-AM contract. - In-place UPDATE for non-indexed-column changes (no index is touched); the old version goes to the per-relation UNDO fork and is reconstructed on demand for older snapshots. - Out-of-place (new-TID) UPDATE when an indexed column changes -- effectively DELETE old + INSERT new. The old index entries die with the old TID and are reclaimed by ordinary VACUUM. This is what lets FLUX use plain heap-TID secondary indexes with standard index maintenance and no new index-AM contract: idxscan, seqscan, and bitmapscan agree across key-changing UPDATEs, including the A->B->A case that a stable-TID model gets wrong. Keeping indexed-column UPDATEs in place instead -- avoiding the TID move and the re-insert into every index -- would require an index-AM contract that lets a scan recheck a possibly-stale key against the visible version; that is deliberately out of scope here and is called out as future work (see src/backend/access/flux/README). - Wide values use standard heap TOAST: the TOAST table is an ordinary heap relation (relation_needs_toast_table / relation_toast_am mirror heap; toast_helper externalizes oversized varlenas on write, VACUUM deletes external datums of dead tuples). UNDO does not apply to the TOAST table; rolling back the base-table UPDATE restores the base tuple's old TOAST pointer and the new chunks become dead heap tuples reclaimed by VACUUM. - Heap-shaped xmin/xmax MVCC, CLOG-authoritative, with the tuple sLog for uncommitted-writer tracking. - Crash safety via WAL (RM_FLUX_ID) plus per-relation UNDO recovery. Registration: pg_am 'flux' (FLUX_TABLE_AM_OID 9316, handler 9402), RM_FLUX_ID, UNDO_RMID_FLUX, TWOPHASE_RM_FLUX_ID, the FluxMvcc and FluxDirtyMap shared-memory subsystems, and the flux_lazy_uncommitted_clear GUC. FLUX is always compiled (USE_FLUX) because the catalog and rmgr registrations are unconditional. Adds src/test/regress/sql/flux.sql covering INSERT, UPDATE (both non-key and key-changing), DELETE, the idxscan==seqscan==bitmapscan cross-check under key-changing UPDATEs, amcheck (heapallindexed), ROLLBACK, and a >8KB TOAST round-trip. Design and differences from heap are documented in src/backend/access/flux/README. The UNDO storage model FLUX builds on derives from the cluster-wide UNDO and zheap effort developed for PostgreSQL at EnterpriseDB. Co-authored-by: Amit Kapila Co-authored-by: Dilip Kumar Co-authored-by: Kuntal Ghosh Co-authored-by: Mahendra Singh Thalor --- examples/01-basic-undo-setup.sql | 8 +- examples/02-undo-rollback.sql | 4 +- examples/03-undo-subtransactions.sql | 2 +- examples/05-undo-monitoring.sql | 4 +- examples/README.md | 6 +- meson.build | 11 + src/Makefile.global.in | 4 + src/backend/access/Makefile | 6 + src/backend/access/flux/Makefile | 31 + src/backend/access/flux/README | 157 + src/backend/access/flux/flux_dirtymap.c | 283 + src/backend/access/flux/flux_fsm.c | 170 + src/backend/access/flux/flux_handler.c | 4776 +++++++++++++ src/backend/access/flux/flux_lock.c | 357 + src/backend/access/flux/flux_mvcc.c | 1344 ++++ src/backend/access/flux/flux_operations.c | 7531 ++++++++++++++++++++ src/backend/access/flux/flux_pvs.c | 208 + src/backend/access/flux/flux_relundo.c | 100 + src/backend/access/flux/flux_slot.c | 796 +++ src/backend/access/flux/flux_stats.c | 286 + src/backend/access/flux/flux_tuple.c | 1134 +++ src/backend/access/flux/flux_undo.c | 440 ++ src/backend/access/flux/flux_vm.c | 643 ++ src/backend/access/flux/flux_xlog.c | 2761 +++++++ src/backend/access/flux/meson.build | 18 + src/backend/access/meson.build | 1 + src/backend/access/rmgrdesc/Makefile | 6 + src/backend/access/rmgrdesc/fluxdesc.c | 512 ++ src/backend/access/rmgrdesc/meson.build | 5 + src/backend/access/transam/rmgr.c | 3 + src/backend/access/transam/twophase_rmgr.c | 13 +- src/backend/replication/logical/decode.c | 462 ++ src/backend/utils/misc/guc_parameters.dat | 7 + src/backend/utils/misc/guc_tables.c | 1 + src/bin/pg_waldump/rmgrdesc.c | 1 + src/bin/pg_waldump/t/001_basic.pl | 3 +- src/include/access/flux.h | 910 +++ src/include/access/flux_dirtymap.h | 77 + src/include/access/flux_undo.h | 83 + src/include/access/flux_xlog.h | 523 ++ src/include/access/rmgrlist.h | 3 + src/include/access/twophase_rmgr.h | 3 +- src/include/access/undormgr.h | 2 +- src/include/access/undormgrlist.h | 3 + src/include/catalog/pg_am.dat | 3 + src/include/catalog/pg_proc.dat | 4 + src/include/storage/subsystemlist.h | 4 + src/test/regress/expected/create_am.out | 3 +- src/test/regress/expected/flux.out | 128 + src/test/regress/expected/psql.out | 12 +- src/test/regress/parallel_schedule | 1 + src/test/regress/sql/flux.sql | 68 + 52 files changed, 23897 insertions(+), 24 deletions(-) create mode 100644 src/backend/access/flux/Makefile create mode 100644 src/backend/access/flux/README create mode 100644 src/backend/access/flux/flux_dirtymap.c create mode 100644 src/backend/access/flux/flux_fsm.c create mode 100644 src/backend/access/flux/flux_handler.c create mode 100644 src/backend/access/flux/flux_lock.c create mode 100644 src/backend/access/flux/flux_mvcc.c create mode 100644 src/backend/access/flux/flux_operations.c create mode 100644 src/backend/access/flux/flux_pvs.c create mode 100644 src/backend/access/flux/flux_relundo.c create mode 100644 src/backend/access/flux/flux_slot.c create mode 100644 src/backend/access/flux/flux_stats.c create mode 100644 src/backend/access/flux/flux_tuple.c create mode 100644 src/backend/access/flux/flux_undo.c create mode 100644 src/backend/access/flux/flux_vm.c create mode 100644 src/backend/access/flux/flux_xlog.c create mode 100644 src/backend/access/flux/meson.build create mode 100644 src/backend/access/rmgrdesc/fluxdesc.c create mode 100644 src/include/access/flux.h create mode 100644 src/include/access/flux_dirtymap.h create mode 100644 src/include/access/flux_undo.h create mode 100644 src/include/access/flux_xlog.h create mode 100644 src/test/regress/expected/flux.out create mode 100644 src/test/regress/sql/flux.sql diff --git a/examples/01-basic-undo-setup.sql b/examples/01-basic-undo-setup.sql index 82042081e4e9b..0304e0358d140 100644 --- a/examples/01-basic-undo-setup.sql +++ b/examples/01-basic-undo-setup.sql @@ -2,18 +2,18 @@ -- Example 1: Basic UNDO Setup and Monitoring -- ============================================================================ -- This example demonstrates: --- 1. Creating a table that uses UNDO (via the recno access method) +-- 1. Creating a table that uses UNDO (via the flux access method) -- 2. Performing modifications -- 3. Monitoring UNDO activity --- STEP 1: Create a table using the recno AM (which supports UNDO) +-- STEP 1: Create a table using the flux AM (which supports UNDO) -- No server-level configuration is needed; UNDO is always-on infrastructure. CREATE TABLE customer_data ( id serial PRIMARY KEY, name text NOT NULL, email text, created_at timestamptz DEFAULT now() -) USING recno; +) USING flux; -- STEP 2: Insert sample data INSERT INTO customer_data (name, email) VALUES @@ -21,7 +21,7 @@ INSERT INTO customer_data (name, email) VALUES ('Bob Johnson', 'bob@example.com'), ('Charlie Brown', 'charlie@example.com'); --- STEP 3: Perform an update (in-place for recno) +-- STEP 3: Perform an update (in-place for flux) UPDATE customer_data SET email = 'alice.smith@newdomain.com' WHERE name = 'Alice Smith'; -- STEP 4: Delete a row diff --git a/examples/02-undo-rollback.sql b/examples/02-undo-rollback.sql index 9af57664747e0..c20afc0a84e1e 100644 --- a/examples/02-undo-rollback.sql +++ b/examples/02-undo-rollback.sql @@ -3,13 +3,13 @@ -- ============================================================================ -- Demonstrates how UNDO records enable efficient transaction rollback --- Create a table using the recno AM (supports UNDO) +-- Create a table using the flux AM (supports UNDO) CREATE TABLE order_items ( order_id int, item_id int, quantity int, price numeric(10,2) -) USING recno; +) USING flux; -- Begin transaction BEGIN; diff --git a/examples/03-undo-subtransactions.sql b/examples/03-undo-subtransactions.sql index 22dac58d9d9aa..358bc134b228e 100644 --- a/examples/03-undo-subtransactions.sql +++ b/examples/03-undo-subtransactions.sql @@ -6,7 +6,7 @@ CREATE TABLE account_ledger ( account_id int, amount numeric(10,2), posted_at timestamptz DEFAULT now() -) USING recno; +) USING flux; BEGIN; diff --git a/examples/05-undo-monitoring.sql b/examples/05-undo-monitoring.sql index caf027a7eeb10..51c1670357e7f 100644 --- a/examples/05-undo-monitoring.sql +++ b/examples/05-undo-monitoring.sql @@ -12,7 +12,7 @@ SELECT * FROM pg_stat_get_undo_buffers(); -- (normally handled automatically by the UNDO worker) SELECT pg_undo_force_discard(); --- List tables using an AM that supports UNDO (i.e., recno tables) +-- List tables using an AM that supports UNDO (i.e., flux tables) SELECT n.nspname AS schema, c.relname AS table, @@ -20,7 +20,7 @@ SELECT FROM pg_class c JOIN pg_namespace n ON c.relnamespace = n.oid JOIN pg_am am ON c.relam = am.oid -WHERE am.amname = 'recno' +WHERE am.amname = 'flux' ORDER BY n.nspname, c.relname; -- Monitor UNDO worker activity diff --git a/examples/README.md b/examples/README.md index 096150dbd188d..f10e7e3c8812e 100644 --- a/examples/README.md +++ b/examples/README.md @@ -5,9 +5,9 @@ and transactional file operations (FILEOPS). ## Prerequisites -Tables opt into UNDO by using the `recno` access method: +Tables opt into UNDO by using the `flux` access method: - CREATE TABLE my_table (...) USING recno; + CREATE TABLE my_table (...) USING flux; UNDO is always-on infrastructure -- there is no GUC to enable or disable it globally. Table access methods opt in via the `am_supports_undo` callback. @@ -35,6 +35,6 @@ psql -d testdb -f examples/02-undo-rollback.sql ## Notes -- UNDO is always-on; tables opt in via `USING recno` +- UNDO is always-on; tables opt in via `USING flux` - FILEOPS (transactional file operations) is always-on for all tables - System catalogs never use UNDO diff --git a/meson.build b/meson.build index f4cde2492423b..076616b5d2bf9 100644 --- a/meson.build +++ b/meson.build @@ -1585,6 +1585,17 @@ else endif +############################################################### +# Library: FLUX +############################################################### + +# FLUX table access method (UNDO-based heap replacement) is always built: +# guc_parameters.dat and pg_proc.dat reference flux symbols unconditionally, +# so a build with USE_FLUX unset fails to link. USE_FLUX is therefore always +# defined and the sources are always compiled. +flux = declare_dependency() +cdata.set('USE_FLUX', 1) + ############################################################### # Library: selinux diff --git a/src/Makefile.global.in b/src/Makefile.global.in index cef1ad7f87d98..c3443b7afd93e 100644 --- a/src/Makefile.global.in +++ b/src/Makefile.global.in @@ -256,6 +256,10 @@ PG_SYSROOT = @PG_SYSROOT@ override CPPFLAGS += $(ICU_CFLAGS) $(LIBNUMA_CFLAGS) $(LIBURING_CFLAGS) +# FLUX table access method (UNDO-based heap replacement) is always built +USE_FLUX = 1 +override CPPFLAGS += -DUSE_FLUX + ifdef PGXS override CPPFLAGS := -I$(includedir_server) -I$(includedir_internal) $(CPPFLAGS) else # not PGXS diff --git a/src/backend/access/Makefile b/src/backend/access/Makefile index 2e4cc6a17e30b..b28bdc98e01b4 100644 --- a/src/backend/access/Makefile +++ b/src/backend/access/Makefile @@ -25,4 +25,10 @@ SUBDIRS = \ transam \ undo +endif + +ifdef USE_FLUX +SUBDIRS += flux +endif + include $(top_srcdir)/src/backend/common.mk diff --git a/src/backend/access/flux/Makefile b/src/backend/access/flux/Makefile new file mode 100644 index 0000000000000..0443f539f9e24 --- /dev/null +++ b/src/backend/access/flux/Makefile @@ -0,0 +1,31 @@ +#------------------------------------------------------------------------- +# +# Makefile-- +# Makefile for access/flux +# +# IDENTIFICATION +# src/backend/access/flux/Makefile +# +#------------------------------------------------------------------------- + +subdir = src/backend/access/flux +top_builddir = ../../../.. +include $(top_builddir)/src/Makefile.global + +OBJS = \ + flux_dirtymap.o \ + flux_handler.o \ + flux_tuple.o \ + flux_mvcc.o \ + flux_stats.o \ + flux_xlog.o \ + flux_fsm.o \ + flux_operations.o \ + flux_lock.o \ + flux_slot.o \ + flux_vm.o \ + flux_undo.o \ + flux_relundo.o \ + flux_pvs.o + +include $(top_srcdir)/src/backend/common.mk \ No newline at end of file diff --git a/src/backend/access/flux/README b/src/backend/access/flux/README new file mode 100644 index 0000000000000..a6252dd54d759 --- /dev/null +++ b/src/backend/access/flux/README @@ -0,0 +1,157 @@ +FLUX +==== + +FLUX is a table access method that realizes what zheap was designed to be: a +HEAP replacement that stores old row versions in UNDO instead of leaving dead +tuples in the main relation, while working entirely on today's index and +executor APIs. FLUX adds no new index-AM contract. + +This README describes FLUX's design and how it differs from heap. For the +UNDO substrate FLUX relies on, see src/backend/access/undo/README (the +cluster-wide UNDO engine) and src/include/access/relundo.h (the per-relation +UNDO fork). The zheap origin design notes are in DESIGN. + + +What FLUX keeps from the zheap model +------------------------------------ + +* In-place UPDATE. When an UPDATE touches only non-indexed columns, the new + tuple version overwrites the old one in the same slot (same TID). The old + version is pushed to the relation's per-relation UNDO fork (RelUndo, + RELUNDO_FORKNUM) via the tuple header's version pointer (t_verptr). A + snapshot that cannot see the updater reconstructs the old version by walking + that UNDO chain (flux_pvs.c, FluxReconstructVisibleVersion). This is the + zheap win: an UPDATE that does not change any indexed column touches no + index at all. + +* UNDO-based rollback. ROLLBACK (and error/abort) replays the RelUndo chain + backward, restoring before-images in place (relundo_apply.c). There are no + dead tuples to vacuum away for a rolled-back change. + +* UNDO-based old-version reconstruction for MVCC. Older snapshots read the + pre-update image from the UNDO fork, not from a second on-page copy. + +* Crash safety. Every page modification and its UNDO record are WAL-logged + (resource manager RM_FLUX_ID, flux_xlog.c); recovery replays them and the + per-relation UNDO recovery restores any before-images for transactions that + were in flight at the crash. + +* sLog for uncommitted-writer tracking. FLUX shares the cluster's sLog + extension (slog.c, registered once as SLogShmemCallbacks) for self-visibility, + command-level (CID) checks, and write-write conflict serialization. The + authoritative commit oracle is CLOG; visibility is heap-shaped xmin/xmax + (see below). + + +How FLUX differs from heap +-------------------------- + +1. Old versions live in UNDO, not in the table. Heap keeps every old row + version in the relation until VACUUM removes it (bloat). FLUX keeps only + the newest version on the page and stores the old one in the UNDO fork, + which is discarded once no snapshot needs it. A non-in-place change's old + version and a DELETE's tombstone are reclaimable as soon as the transaction + commits and the version falls below the removable horizon. + +2. In-place for non-key UPDATEs; out-of-place for key-changing UPDATEs. + This is the central design decision and the reason FLUX needs no index-AM + change. See the next section. + +3. Wide values use standard heap TOAST. A FLUX relation gets an ordinary + heap TOAST table (relation_needs_toast_table mirrors heap; relation_toast_am + returns the heap AM). On INSERT/UPDATE, oversized varlena columns are + externalized through the AM-agnostic toast_helper routines (flux_toast_tuple + in flux_tuple.c) exactly as heap does; VACUUM deletes the external datums of + dead tuples (flux_toast_delete). FLUX has no on-page overflow mechanism of + its own. + +4. Heap-shaped xmin/xmax visibility. FLUX uses the "boring correct" + visibility model: each tuple carries t_xmin (inserter) and t_xmax + (deleter/updater), resolved against CLOG and the reader's snapshot exactly + like HeapTupleSatisfiesMVCC. + + +Secondary indexes and the key-changing UPDATE (stock table-AM design) +--------------------------------------------------------------------- + +Secondary indexes on a FLUX table are ordinary heap-TID nbtree (or any +standard index-AM) entries: plain 6-byte TIDs, exactly like heap. FLUX uses +only today's stock table-AM and index-AM interfaces -- no per-tuple +generation, no wider index identity, no executor stale-entry recheck +exactly like a heap table for indexing. + +The one hazard an in-place-with-stable-TID AM creates for a plain-TID index is +the "A -> B -> A" problem: if the tuple stays at one TID while an indexed +column changes, the index accumulates a stale (oldkey, TID) entry beside the +new (newkey, TID) entry, both pointing at the same live TID; an index scan +would then double-count or return rows under the wrong key. + +FLUX avoids the hazard the heap way: + + * An UPDATE that changes NO indexed column is done IN PLACE (same TID). The + index entries still point at the right key and TID, so no index + maintenance is needed. flux_tuple_update sets *update_indexes = TU_None + and the executor inserts no index entries. + + * An UPDATE that changes ANY indexed column is done OUT OF PLACE: FLUX stores + the new version at a NEW TID (DELETE old + INSERT new -- zheap's + non-in-place update, "essentially DELETE+INSERT"). It sets slot->tts_tid + to the new TID and *update_indexes = TU_All, so the executor inserts fresh + entries in every index at the new TID. The old TID's index entries now + point at a dead tuple and are reclaimed by ordinary VACUUM -- exactly like + heap. The old version remains reachable to older snapshots through the + UNDO before-image the DELETE recorded, and a ROLLBACK undoes both the + DELETE and the INSERT. + +Stock PostgreSQL no longer tells the AM which columns changed, so FLUX +self-computes it (flux_indexed_attr_changed), matching heap's internal +HeapDetermineColumnsInfo: it deforms the old on-page tuple, reads the new +values from the slot, and value-compares every attribute referenced by any +index (RelationGetIndexAttrBitmap HOT_BLOCKING + SUMMARIZED). If any indexed +attribute differs it takes the out-of-place path. FLUX never uses +TU_Summarizing: a key-changing UPDATE always moves the TID, so TU_All already +rebuilds summarizing indexes too. + +Consequence: idxscan, seqscan, and bitmapscan return identical results across +any sequence of key-changing UPDATEs (including A -> B -> A recurrences), +because the index is maintained by the standard heap path and never holds two +live entries for one TID under different keys. Verified by amcheck +(heapallindexed) and by the scan cross-check in the test suite. + + +Registration +------------ + +* pg_am: amname 'flux', amhandler flux_tableam_handler, amtype 't' + (FLUX_TABLE_AM_OID = 9316; pg_proc oid 9402). +* WAL resource manager: RM_FLUX_ID (rmgrlist.h), redo/desc/identify/mask in + flux_xlog.c / fluxdesc.c, logical decoding in decode.c (flux_decode). +* Cluster-wide UNDO resource manager: UNDO_RMID_FLUX (=5), FluxUndoRmgrInit + (undormgrlist.h), used so index inserts on a FLUX table can piggyback their + index-UNDO onto the table's active UNDO context. +* Two-phase commit: TWOPHASE_RM_FLUX_ID (=6), flux_twophase_* callbacks. +* Shared memory: FluxMvccShmemCallbacks and FluxDirtyMapShmemCallbacks + (subsystemlist.h); the sLog is the shared SLogShmemCallbacks. +* GUC: flux_lazy_uncommitted_clear. + +FLUX is always compiled (USE_FLUX) because the catalog and rmgr +lists reference its symbols unconditionally. + + +Source layout +------------- + + flux_handler.c table-AM routine table (flux_methods), scans, TOAST hooks + flux_operations.c INSERT / DELETE / UPDATE (in-place + out-of-place), VACUUM + flux_tuple.c tuple form/deform, flux_toast_tuple / flux_toast_delete + flux_mvcc.c heap-shaped xmin/xmax visibility state, shmem + flux_pvs.c prior-version reconstruction from the UNDO fork + flux_slot.c FLUX minimal tuple slot + flux_xlog.c WAL record types, redo, mask + flux_undo.c UNDO_RMID_FLUX resource manager (index-UNDO piggyback) + flux_relundo.c per-relation UNDO fork glue + flux_lock.c tuple locking (SELECT FOR UPDATE/SHARE) + flux_vm.c visibility map maintenance + flux_fsm.c free-space map integration + flux_dirtymap.c per-block dirty map (scan-path sLog bypass), shmem + flux_stats.c relation size / density estimation helpers diff --git a/src/backend/access/flux/flux_dirtymap.c b/src/backend/access/flux/flux_dirtymap.c new file mode 100644 index 0000000000000..b3dadd25824b4 --- /dev/null +++ b/src/backend/access/flux/flux_dirtymap.c @@ -0,0 +1,283 @@ +/*------------------------------------------------------------------------- + * + * flux_dirtymap.c + * Shared-memory dirty block map for the FLUX table access method. + * + * This module tracks which heap pages have ever carried an in-place + * modification whose before-image a scanner might still need from the sLog. + * The scan path uses it as a fast-path filter: if a page's bit is CLEAR, + * every tuple on it is plain-committed with no retained before-image, so the + * per-tuple sLog before-image probe can be skipped for the whole page. + * + * Implementation: + * - A partitioned, open-addressed hash set of 64-bit page keys + * (((uint64) relid << 32) | blkno) in a fixed shared-memory buffer. + * - Partitioned by hash into FLUX_DIRTYMAP_PARTITIONS independent tables, + * each with its own writer spinlock, so concurrent Mark calls on + * different pages do not contend a single global lock. + * - FluxDirtyMapCheck (the per-scanned-tuple hot path) is LOCK-FREE: the + * set is grow-only (a published key is never removed or moved), so a + * reader linear-probing the chain sees either the key or an empty + * terminator, never a torn or reverted slot. Slots are published with a + * single atomic store and read with a single atomic load. + * - FluxDirtyMapMark takes only the target partition's spinlock and does + * an O(1)-amortized open-addressed insert. + * - The map is GROW-ONLY: a key, once set, is never cleared during normal + * operation. See flux_dirtymap.h for the correctness rationale. + * - If a partition ever fills past its load-factor ceiling, a sticky "full" + * flag is latched for that partition and every check on it returns dirty + * (safe degradation to always-probe). + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/access/flux/flux_dirtymap.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/flux_dirtymap.h" +#include "port/atomics.h" +#include "storage/shmem.h" +#include "storage/spin.h" + +/* + * Partition count and per-partition capacity. + * + * Partitioning spreads Mark-path writer-lock contention across many locks. + * The Check path is lock-free regardless. Capacity is a power of two so the + * probe index is a cheap mask. 128 partitions x 8192 slots = ~1M slots x 8 + * bytes = 8 MB of shared memory; with the 0.75 load-factor ceiling that holds + * ~786K distinct in-place-modified pages before a partition latches full. + */ +#define FLUX_DIRTYMAP_PARTITIONS 128 +#define FLUX_DIRTYMAP_PART_SLOTS 8192 /* power of two */ +#define FLUX_DIRTYMAP_PART_MASK (FLUX_DIRTYMAP_PART_SLOTS - 1) +#define FLUX_DIRTYMAP_MAX_LOAD ((FLUX_DIRTYMAP_PART_SLOTS * 3) / 4) + +/* + * Empty-slot sentinel. A key of 0 would require relid 0, which is never a + * user relation, so 0 is safe to reserve as "empty". + */ +#define FLUX_DIRTYMAP_EMPTY_KEY UINT64CONST(0) + +/* + * Compose the collision-free 64-bit page key. relid is a 32-bit Oid and + * blkno a 32-bit BlockNumber, so the key is unique per (relid, blkno). + */ +#define FLUX_DIRTYMAP_KEY(relid, blkno) \ + (((uint64) (relid) << 32) | (uint64) (blkno)) + +/* + * One partition: a fixed open-addressed slot array of atomic 64-bit keys, + * a writer spinlock, a live-entry count, and a sticky full flag. + */ +typedef struct FluxDirtyMapPartition +{ + slock_t mutex; /* serializes Mark inserts into this partition */ + int nentries; /* live keys (writer-lock protected) */ + bool full; /* sticky: load ceiling hit -> check == dirty */ + pg_atomic_uint64 slots[FLUX_DIRTYMAP_PART_SLOTS]; +} FluxDirtyMapPartition; + +typedef struct FluxDirtyMapShared +{ + FluxDirtyMapPartition parts[FLUX_DIRTYMAP_PARTITIONS]; +} FluxDirtyMapShared; + +static FluxDirtyMapShared *DirtyMap = NULL; + +/* + * Hash a 64-bit key to a partition index and an initial probe slot. A + * multiplicative (Fibonacci) hash mixes the high and low words so that the + * sequential (relid, blkno) keys of a scan spread across partitions and + * slots rather than clustering. + */ +static inline uint64 +flux_dirtymap_mix(uint64 key) +{ + uint64 h = key * UINT64CONST(0x9E3779B97F4A7C15); + + return h ^ (h >> 32); +} + +static inline uint32 +flux_dirtymap_part(uint64 mixed) +{ + return (uint32) (mixed & (FLUX_DIRTYMAP_PARTITIONS - 1)); +} + +static inline uint32 +flux_dirtymap_slot0(uint64 mixed) +{ + return (uint32) ((mixed >> 7) & FLUX_DIRTYMAP_PART_MASK); +} + +/* ---------------------------------------------------------------- + * Shared memory initialization + * ---------------------------------------------------------------- + */ + +Size +FluxDirtyMapShmemSize(void) +{ + return MAXALIGN(sizeof(FluxDirtyMapShared)); +} + +static void +FluxDirtyMapShmemRequest(void *arg) +{ + ShmemRequestStruct(.name = "FLUX DirtyMap", + .size = FluxDirtyMapShmemSize(), + .ptr = (void **) &DirtyMap, + ); +} + +static void +FluxDirtyMapShmemInit_cb(void *arg) +{ + int p; + int s; + + Assert(DirtyMap != NULL); + + for (p = 0; p < FLUX_DIRTYMAP_PARTITIONS; p++) + { + FluxDirtyMapPartition *part = &DirtyMap->parts[p]; + + SpinLockInit(&part->mutex); + part->nentries = 0; + part->full = false; + for (s = 0; s < FLUX_DIRTYMAP_PART_SLOTS; s++) + pg_atomic_init_u64(&part->slots[s], FLUX_DIRTYMAP_EMPTY_KEY); + } +} + +void +FluxDirtyMapShmemInit(void) +{ + /* Initialization is handled by FluxDirtyMapShmemCallbacks */ +} + +const ShmemCallbacks FluxDirtyMapShmemCallbacks = { + .request_fn = FluxDirtyMapShmemRequest, + .init_fn = FluxDirtyMapShmemInit_cb, +}; + +/* ---------------------------------------------------------------- + * Mark and query + * ---------------------------------------------------------------- + */ + +/* + * FluxDirtyMapMark + * Record that (relid, blkno) carries a retained in-place modification. + * + * Must be called while the page's buffer is exclusively locked, before that + * lock is released, so a concurrent scanner always observes the published + * key. Idempotent. If the target partition is full, its sticky flag is + * latched so every subsequent check on it conservatively returns dirty. + */ +void +FluxDirtyMapMark(Oid relid, BlockNumber blkno) +{ + uint64 key = FLUX_DIRTYMAP_KEY(relid, blkno); + uint64 mixed = flux_dirtymap_mix(key); + FluxDirtyMapPartition *part = &DirtyMap->parts[flux_dirtymap_part(mixed)]; + uint32 slot = flux_dirtymap_slot0(mixed); + int probes; + + SpinLockAcquire(&part->mutex); + + if (part->full) + { + SpinLockRelease(&part->mutex); + return; + } + + for (probes = 0; probes < FLUX_DIRTYMAP_PART_SLOTS; probes++) + { + uint64 cur = pg_atomic_read_u64(&part->slots[slot]); + + if (cur == key) + { + /* already present -- idempotent */ + SpinLockRelease(&part->mutex); + return; + } + if (cur == FLUX_DIRTYMAP_EMPTY_KEY) + { + /* + * Publish the key with a single atomic store. A concurrent + * lock-free reader on the same chain sees either the old empty + * (and keeps probing / terminates) or the full key -- never a + * torn value. Grow-only means this slot never reverts. + */ + if (part->nentries >= FLUX_DIRTYMAP_MAX_LOAD) + { + part->full = true; + SpinLockRelease(&part->mutex); + return; + } + pg_atomic_write_u64(&part->slots[slot], key); + part->nentries++; + SpinLockRelease(&part->mutex); + return; + } + slot = (slot + 1) & FLUX_DIRTYMAP_PART_MASK; + } + + /* probe chain exhausted without an empty slot -> latch full */ + part->full = true; + SpinLockRelease(&part->mutex); +} + +/* + * FluxDirtyMapCheck + * Return true if the page's key is present (or its partition overflowed). + * + * LOCK-FREE. The set is grow-only, so a reader linear-probing the chain sees + * either the key (dirty), an empty terminator (clean), or the partition's + * sticky full flag (dirty). A concurrent Mark only publishes new keys with + * an atomic store; it never moves or clears a slot, so a reader can miss a + * key that is being inserted *concurrently* only if the insert has not yet + * completed -- and the mark-before-buffer-unlock ordering (see + * FluxDirtyMapMark's contract) guarantees any modification a scanner could + * observe was published before the scanner could reach the page. + * + * A false result means the page is provably clean and the scan path may skip + * the per-tuple sLog before-image probe for the whole page. + */ +bool +FluxDirtyMapCheck(Oid relid, BlockNumber blkno) +{ + uint64 key = FLUX_DIRTYMAP_KEY(relid, blkno); + uint64 mixed = flux_dirtymap_mix(key); + FluxDirtyMapPartition *part = &DirtyMap->parts[flux_dirtymap_part(mixed)]; + uint32 slot = flux_dirtymap_slot0(mixed); + int probes; + + /* + * The full flag is read without the spinlock. It is set-once (sticky) + * and only ever transitions false->true, so a stale-false read at worst + * costs one more lock-free probe pass, and a true read is permanent. + */ + if (part->full) + return true; + + for (probes = 0; probes < FLUX_DIRTYMAP_PART_SLOTS; probes++) + { + uint64 cur = pg_atomic_read_u64(&part->slots[slot]); + + if (cur == key) + return true; + if (cur == FLUX_DIRTYMAP_EMPTY_KEY) + return false; /* provably clean */ + slot = (slot + 1) & FLUX_DIRTYMAP_PART_MASK; + } + + /* full chain of non-matching keys: conservatively dirty */ + return true; +} diff --git a/src/backend/access/flux/flux_fsm.c b/src/backend/access/flux/flux_fsm.c new file mode 100644 index 0000000000000..2e4da494da331 --- /dev/null +++ b/src/backend/access/flux/flux_fsm.c @@ -0,0 +1,170 @@ +/*------------------------------------------------------------------------- + * + * flux_fsm.c + * FLUX free space management + * + * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/access/flux/flux_fsm.c + * + * NOTES + * This is a thin wrapper over PostgreSQL's standard free space map + * (src/backend/storage/freespace). It adds FLUX-specific relation + * extension (page initialization and WAL-logging of the new page) on + * top of the generic GetPageWithFreeSpace/RecordPageWithFreeSpace API. + * + * Page defragmentation is handled entirely by VACUUM via + * FluxPageDefragment() in flux_tuple.c; the FSM only tracks free + * space, it does not schedule or perform compaction. + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/flux.h" +#include "access/flux_xlog.h" +#include "storage/bufmgr.h" +#include "storage/freespace.h" +#include "utils/rel.h" +#include "miscadmin.h" + +/* + * FluxGetPageWithFreeSpace + * + * Find or create a page with at least 'needed' bytes of free space. + * + * First queries PostgreSQL's standard FSM (GetPageWithFreeSpace). If a + * page is returned, that block is used directly. + * + * If no suitable existing page is found, extends the relation by allocating + * a new page with ExtendBufferedRel(), initializes it with FluxInitPage(), + * WAL-logs the initialization, and returns the new block. + * + * Parameters: + * rel - open relation + * needed - minimum number of free bytes required + * + * Returns the block number of a page with sufficient free space. + */ +BlockNumber +FluxGetPageWithFreeSpace(Relation rel, Size needed) +{ + BlockNumber target_block; + Buffer buffer; + Page page; + Size free_space; + + /* + * Ask the FSM for a page with enough free space. Note: we do NOT verify + * the page by locking it here, because callers may already hold buffer + * locks (e.g., the update path holds the old tuple's page lock, and + * vacuum cross-page defrag holds the source page lock). Locking a page + * here would risk self-deadlock if the FSM returns a block that the + * caller already has locked. Callers are responsible for rechecking free + * space after they acquire their own lock on the returned page. + */ + target_block = GetPageWithFreeSpace(rel, needed); + + if (target_block != InvalidBlockNumber) + return target_block; + + /* + * No suitable page found -- extend the relation. + * + * Use the modern ExtendBufferedRel() API which properly handles + * concurrent extension by multiple backends. The old + * ReadBufferExtended(P_NEW) path had a race condition that caused + * BM_IO_IN_PROGRESS assertion failures under concurrency. + */ + buffer = ExtendBufferedRel(BMR_REL(rel), MAIN_FORKNUM, NULL, + EB_LOCK_FIRST); + target_block = BufferGetBlockNumber(buffer); + + page = BufferGetPage(buffer); + FluxInitPage(page, BufferGetPageSize(buffer)); + + START_CRIT_SECTION(); + + MarkBufferDirty(buffer); + + /* Log page initialization */ + if (RelationNeedsWAL(rel)) + { + uint64 init_commit_ts = FluxGetCommitTimestamp(); + FluxPageOpaque phdr; + XLogRecPtr recptr; + + /* + * Set the page's opaque data to match what the REDO handler will + * produce. This is essential for WAL consistency checking: the page + * image stored with the WAL record must match what REDO generates + * when replaying. + */ + phdr = FluxPageGetOpaque(page); + FluxPageSetCommitTs(phdr, init_commit_ts); + + recptr = FluxXLogInitPage(rel, buffer, 0, init_commit_ts); + + PageSetLSN(page, recptr); + } + + END_CRIT_SECTION(); + + /* Capture free space before releasing the buffer */ + free_space = PageGetFreeSpace(page); + + UnlockReleaseBuffer(buffer); + + /* Record the new page in FSM */ + FluxRecordFreeSpace(rel, target_block, free_space); + + /* + * Propagate the new FSM leaf value up through the FSM tree so that + * subsequent GetPageWithFreeSpace() calls can find it. Without this, the + * root of the FSM tree remains at zero and all searches fail. + */ + FreeSpaceMapVacuumRange(rel, target_block, target_block + 1); + + return target_block; +} + +/* + * FluxRecordFreeSpace + * + * Update the FSM with the actual free space for a page. + * + * Parameters: + * rel - open relation + * page - block number of the page + * freespace - actual free space in bytes on the page + */ +void +FluxRecordFreeSpace(Relation rel, BlockNumber page, Size freespace) +{ + RecordPageWithFreeSpace(rel, page, freespace); +} + +/* + * FluxVacuumFSM + * + * Update the FSM after a relation truncation. If the new block count is + * smaller than the old count, calls FreeSpaceMapPrepareTruncateRel() to + * remove FSM entries for the truncated pages. + * + * Parameters: + * rel - open relation + * new_nblocks - new number of blocks after truncation + */ +void +FluxVacuumFSM(Relation rel, BlockNumber new_nblocks) +{ + BlockNumber old_nblocks = RelationGetNumberOfBlocks(rel); + + if (new_nblocks < old_nblocks) + { + /* Truncated - update FSM */ + FreeSpaceMapPrepareTruncateRel(rel, new_nblocks); + } +} diff --git a/src/backend/access/flux/flux_handler.c b/src/backend/access/flux/flux_handler.c new file mode 100644 index 0000000000000..be1592af57686 --- /dev/null +++ b/src/backend/access/flux/flux_handler.c @@ -0,0 +1,4776 @@ +/*------------------------------------------------------------------------- + * + * flux_handler.c + * FLUX table access method handler + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/access/flux/flux_handler.c + * + * NOTES + * This file implements the FLUX table access method, which provides + * time-based MVCC with in-place updates, overflow pages for large + * attributes, compression, and advanced space management. + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/genam.h" +#include "access/heapam.h" +#include "access/heaptoast.h" +#include "access/detoast.h" +#include "access/flux.h" +#include "access/flux_dirtymap.h" +#include "access/slog.h" +#include "access/flux_xlog.h" +#include "access/relundo.h" +#include "access/tableam.h" +#include "access/undobuffer.h" +#include "access/tsmapi.h" +#include "access/multixact.h" +#include "access/xact.h" +#include "access/xloginsert.h" +#include "catalog/index.h" +#include "catalog/pg_am.h" +#include "catalog/storage.h" +#include "catalog/storage_xlog.h" +#include "commands/progress.h" +#include "executor/executor.h" +#include "nodes/execnodes.h" +#include "nodes/tidbitmap.h" +#include "utils/backend_progress.h" +#include "utils/builtins.h" +#include "utils/snapmgr.h" +#include "utils/timestamp.h" +#include "utils/tuplesort.h" +#include "miscadmin.h" +#include "storage/bufmgr.h" +#include "storage/lmgr.h" +#include "storage/predicate.h" +#include "storage/procarray.h" +#include "storage/read_stream.h" +#include "storage/smgr.h" +#include "utils/datum.h" +#include "utils/memutils.h" +#include "utils/rel.h" + +/* Forward declarations */ +static void flux_prepare_pagescan(FluxScanDesc scan, Buffer buffer); +static BlockNumber flux_scan_stream_read_next(ReadStream *stream, + void *callback_private_data, + void *per_buffer_data); +static BlockNumber flux_bitmap_stream_read_next(ReadStream *stream, + void *callback_private_data, + void *per_buffer_data); +static bool flux_scan_analyze_next_block(TableScanDesc scan, ReadStream *stream); +static bool flux_scan_analyze_next_tuple(TableScanDesc scan, + double *liverows, double *deadrows, + TupleTableSlot *slot); +static void flux_scan_set_tidrange(TableScanDesc sscan, ItemPointer mintid, + ItemPointer maxtid); +static bool flux_scan_getnextslot_tidrange(TableScanDesc sscan, ScanDirection direction, + TupleTableSlot *slot); +static bool flux_scan_bitmap_next_tuple(TableScanDesc scan, + TupleTableSlot *slot, + bool *recheck, + uint64 *lossy_pages, + uint64 *exact_pages); + +/* Include operations from other modules */ +extern void flux_tuple_insert(Relation relation, TupleTableSlot *slot, CommandId cid, + uint32 options, BulkInsertState bistate); +extern TM_Result flux_tuple_delete(Relation relation, ItemPointer tid, CommandId cid, + uint32 options, Snapshot snapshot, Snapshot crosscheck, + bool wait, TM_FailureData *tmfd); +extern TM_Result flux_tuple_update(Relation relation, ItemPointer otid, TupleTableSlot *slot, + CommandId cid, uint32 options, + Snapshot snapshot, Snapshot crosscheck, + bool wait, TM_FailureData *tmfd, + LockTupleMode *lockmode, TU_UpdateIndexes *update_indexes); +extern void flux_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples, + CommandId cid, uint32 options, BulkInsertState bistate); +extern void flux_relation_vacuum(Relation onerel, const VacuumParams *params, + BufferAccessStrategy bstrategy); + +/* + * Read stream callback for sequential scan prefetching. + * + * Returns the next block number to read ahead for the sequential scan. + * The read_stream infrastructure will prefetch these blocks asynchronously, + * reducing I/O wait time for cold data. + * + * Uses rs_prefetch_block (separate from rs_cblock) to track the prefetch + * position independently of the scan's current position. + */ +static BlockNumber +flux_scan_stream_read_next(ReadStream *stream, + void *callback_private_data, + void *per_buffer_data) +{ + FluxScanDesc scan = (FluxScanDesc) callback_private_data; + BlockNumber block; + + block = scan->rs_prefetch_block; + if (block >= scan->rs_nblocks) + return InvalidBlockNumber; + + scan->rs_prefetch_block = block + 1; + return block; +} + +/* + * Read stream callback for bitmap heap scans. + * + * Pulls the next block from the TBM iterator and hands it to the read + * stream so upcoming bitmap pages are prefetched asynchronously, matching + * the HEAP bitmapheap_stream_read_next() behaviour. The TBMIterateResult + * for each block is stashed in per_buffer_data so the consumer can read + * lossy/recheck flags and the exact tuple offsets without re-iterating. + */ +static BlockNumber +flux_bitmap_stream_read_next(ReadStream *stream, + void *callback_private_data, + void *per_buffer_data) +{ + FluxScanDesc scan = (FluxScanDesc) callback_private_data; + TableScanDesc sscan = &scan->rs_base; + TBMIterateResult *tbmres = per_buffer_data; + + for (;;) + { + CHECK_FOR_INTERRUPTS(); + + /* no more entries in the bitmap */ + if (!tbm_iterate(&sscan->st.rs_tbmiterator, tbmres)) + return InvalidBlockNumber; + + /* + * Ignore any claimed entries past what we think is the end of the + * relation. It may have been extended after the start of our scan. + * Skip this optimization under SERIALIZABLE, where all + * index-reachable tuples must be examined for conflict detection. + */ + if (!IsolationIsSerializable() && + tbmres->blockno >= scan->rs_nblocks) + continue; + + return tbmres->blockno; + } + + Assert(false); + return InvalidBlockNumber; +} + +/* + * ------------------------------------------------------------------------ + * Slot related callbacks for FLUX AM + * ------------------------------------------------------------------------ + */ + +/* + * Return slot implementation suitable for storing FLUX tuples + */ +static const TupleTableSlotOps * +flux_slot_callbacks(Relation relation) +{ + (void) relation; + return &TTSOpsFluxTuple; +} + +/* + * flux_begin_bulk_insert - Signal the start of a DML operation. + * + * Activates the Tier-2 UNDO write buffer for this relation. When active, + * per-tuple UNDO records are batched via UndoBufferAddRecord() and flushed + * in larger XLOG_UNDO_BATCH WAL records (overflow path), reducing per-row + * WAL overhead significantly for COPY and multi-insert. + * + * The overflow-flush path emits standalone XLOG_UNDO_BATCH records which + * the revert-worker's UndoReadBatchFromWAL() can walk for any AM. + */ +static void +flux_begin_bulk_insert(Relation rel, uint32 options, int64 nrows) +{ + (void) options; + + UndoBufferBegin(rel, nrows); +} + +/* + * flux_finish_bulk_insert - Complete a DML operation. + * + * Flushes any pending UNDO records and deactivates the write buffer. + */ +static void +flux_finish_bulk_insert(Relation rel, uint32 options) +{ + (void) options; + + UndoBufferEnd(rel); +} + +/* + * ------------------------------------------------------------------------ + * Table scan callbacks for FLUX AM + * ------------------------------------------------------------------------ + */ + +/* + * Start a scan of the FLUX relation + */ +static TableScanDesc +flux_scan_begin(Relation relation, Snapshot snapshot, + int nkeys, ScanKey key, + ParallelTableScanDesc pscan, + uint32 flags) +{ + FluxScanDesc scan; + + + scan = (FluxScanDesc) palloc0(sizeof(FluxScanDescData)); + + scan->rs_base.rs_rd = relation; + scan->rs_base.rs_snapshot = snapshot; + scan->rs_base.rs_nkeys = nkeys; + scan->rs_base.rs_key = key; + scan->rs_base.rs_flags = flags; + scan->rs_base.rs_parallel = pscan; + + scan->rs_cbuf = InvalidBuffer; + scan->rs_cblock = InvalidBlockNumber; + scan->rs_nblocks = RelationGetNumberOfBlocks(relation); + scan->rs_startblock = 0; + scan->rs_coffset = FirstOffsetNumber; + scan->rs_cindex = InvalidOffsetNumber; + scan->rs_inited = false; + scan->rs_ntuples = 0; + scan->rs_vistuples = NULL; + scan->rs_vm_buffer = InvalidBuffer; + scan->rs_vm_blockno = InvalidBlockNumber; + + /* Allocate parallel scan worker data if doing a parallel scan */ + if (pscan != NULL) + scan->rs_parallelworkerdata = palloc_object(ParallelBlockTableScanWorkerData); + else + scan->rs_parallelworkerdata = NULL; + + /* Set up MVCC bookkeeping timestamps (visibility uses xmin/xmax + CLOG) */ + scan->rs_snapshot_ts = GetCurrentTimestamp(); + scan->rs_xact_ts = GetCurrentTimestamp(); + + /* + * Initialize read stream for sequential prefetching (non-parallel only). + * The read stream uses the kernel readahead and our callback to prefetch + * upcoming pages, reducing I/O wait time for cold sequential scans. + * Parallel scans use their own block coordination, so skip the stream. + */ + scan->rs_prefetch_block = 0; + if (flags & SO_TYPE_BITMAPSCAN) + { + /* + * Bitmap scans drive the stream from the TBM iterator. The iterator + * is attached by the executor after beginscan, so the callback only + * runs once read_stream_next_buffer() is first called. Per-buffer + * data carries the TBMIterateResult for each prefetched block. + */ + scan->rs_read_stream = read_stream_begin_relation(READ_STREAM_DEFAULT, + NULL, /* bstrategy */ + relation, + MAIN_FORKNUM, + flux_bitmap_stream_read_next, + scan, + sizeof(TBMIterateResult)); + } + else if (pscan == NULL && scan->rs_nblocks > 0) + { + scan->rs_read_stream = read_stream_begin_relation(READ_STREAM_SEQUENTIAL | + READ_STREAM_USE_BATCHING, + NULL, /* bstrategy */ + relation, + MAIN_FORKNUM, + flux_scan_stream_read_next, + scan, + 0); + } + else + { + scan->rs_read_stream = NULL; + } + + return (TableScanDesc) scan; +} + +/* + * End the scan and release resources + */ +static void +flux_scan_end(TableScanDesc sscan) +{ + FluxScanDesc scan = (FluxScanDesc) sscan; + + /* + * FLUX does not compress attributes, so there is no compression + * dictionary to retrain at ANALYZE scan end. + */ + + /* End read stream before releasing buffers */ + if (scan->rs_read_stream != NULL) + { + read_stream_end(scan->rs_read_stream); + scan->rs_read_stream = NULL; + } + + /* Release buffer if held */ + if (BufferIsValid(scan->rs_cbuf)) + { + ReleaseBuffer(scan->rs_cbuf); + scan->rs_cbuf = InvalidBuffer; + } + + /* Release cached visibility map buffer */ + if (BufferIsValid(scan->rs_vm_buffer)) + { + ReleaseBuffer(scan->rs_vm_buffer); + scan->rs_vm_buffer = InvalidBuffer; + } + + if (scan->rs_vistuples) + pfree(scan->rs_vistuples); + + if (scan->rs_parallelworkerdata != NULL) + pfree(scan->rs_parallelworkerdata); + + /* + * Unregister the snapshot if this scan owns it (SO_TEMP_SNAPSHOT). + * Without this, catalog scans and parallel worker scans leak snapshot + * references, causing "resource was not closed" warnings. + */ + if (scan->rs_base.rs_flags & SO_TEMP_SNAPSHOT) + UnregisterSnapshot(scan->rs_base.rs_snapshot); + + pfree(scan); +} + +/* + * Restart a relation scan + */ +static void +flux_scan_rescan(TableScanDesc sscan, ScanKey key, + bool set_params, bool allow_strat, + bool allow_sync, bool allow_pagemode) +{ + FluxScanDesc scan = (FluxScanDesc) sscan; + + /* Release current buffer */ + if (BufferIsValid(scan->rs_cbuf)) + { + ReleaseBuffer(scan->rs_cbuf); + scan->rs_cbuf = InvalidBuffer; + } + + /* Release cached VM buffer on rescan (relation may have changed) */ + if (BufferIsValid(scan->rs_vm_buffer)) + { + ReleaseBuffer(scan->rs_vm_buffer); + scan->rs_vm_buffer = InvalidBuffer; + scan->rs_vm_blockno = InvalidBlockNumber; + } + + /* Reset read stream for rescan */ + if (scan->rs_read_stream != NULL) + { + read_stream_reset(scan->rs_read_stream); + scan->rs_prefetch_block = 0; + } + + /* Reset scan position to start of relation */ + scan->rs_cblock = InvalidBlockNumber; + scan->rs_nblocks = RelationGetNumberOfBlocks(sscan->rs_rd); + scan->rs_cindex = 0; + scan->rs_coffset = FirstOffsetNumber; + scan->rs_inited = false; + scan->rs_ntuples = 0; + + /* Update scan key if provided */ + scan->rs_base.rs_nkeys = key ? scan->rs_base.rs_nkeys : 0; + scan->rs_base.rs_key = key; +} + +/* + * Get the next block to scan. + * + * For serial scans, simply advances sequentially. For parallel scans, + * coordinates with other workers via table_block_parallelscan_nextpage() + * so that each block is scanned by exactly one worker. + * + * Returns the next block number, or InvalidBlockNumber when finished. + */ +static BlockNumber +flux_scan_nextblock(FluxScanDesc scan) +{ + BlockNumber nblocks = scan->rs_nblocks; + + if (nblocks == 0) + return InvalidBlockNumber; + + if (scan->rs_base.rs_parallel != NULL) + { + ParallelBlockTableScanDesc pbscan = + (ParallelBlockTableScanDesc) scan->rs_base.rs_parallel; + + /* Initialize parallel worker state on first call */ + if (!scan->rs_inited) + { + table_block_parallelscan_startblock_init(scan->rs_base.rs_rd, + scan->rs_parallelworkerdata, + pbscan, + scan->rs_startblock, + InvalidBlockNumber); + scan->rs_inited = true; + } + + return table_block_parallelscan_nextpage(scan->rs_base.rs_rd, + scan->rs_parallelworkerdata, + pbscan); + } + else + { + /* Serial scan: advance to next block sequentially */ + if (scan->rs_cblock == InvalidBlockNumber) + return 0; + + if (scan->rs_cblock + 1 >= nblocks) + return InvalidBlockNumber; + + return scan->rs_cblock + 1; + } +} + +/* + * flux_prepare_pagescan -- collect visible tuple offsets for page-mode scan + * + * Called once per page. Locks SHARE, checks visibility for all tuples, + * collects visible offsets into scan->rs_vistuples[], then unlocks. + * The buffer remains pinned via scan->rs_cbuf. + * + * This is the FLUX equivalent of heapgetpage(). By doing all visibility + * checks under a single SHARE lock acquisition per page, we avoid the + * overhead of ReadBuffer+LockBuffer+ReleaseBuffer per tuple that the + * original scan path had. + * + * The visibility map optimization (1D) is integrated here: if the page is + * marked all-visible, per-tuple visibility checks are skipped entirely. + */ +static void +flux_prepare_pagescan(FluxScanDesc scan, Buffer buffer) +{ + Page page; + OffsetNumber maxoff; + OffsetNumber offnum; + int ntup = 0; + bool all_visible; + + LockBuffer(buffer, BUFFER_LOCK_SHARE); + page = BufferGetPage(buffer); + + /* Skip new/empty pages */ + if (PageIsNew(page)) + { + scan->rs_ntuples = 0; + scan->rs_cindex = 0; + LockBuffer(buffer, BUFFER_LOCK_UNLOCK); + return; + } + + maxoff = PageGetMaxOffsetNumber(page); + + /* Allocate vistuples array if needed (shared with bitmap scan path) */ + if (scan->rs_vistuples == NULL) + { + scan->rs_vistuples = (OffsetNumber *) + MemoryContextAlloc(TopMemoryContext, + MaxOffsetNumber * sizeof(OffsetNumber)); + } + + /* + * Check visibility: first check the in-page PD_ALL_VISIBLE flag (zero + * cost since the page is already pinned and locked). Only fall through + * to the VM fork if the in-page flag is not set. Use the cached VM + * buffer to avoid per-page ReadBufferExtended overhead. + */ + if (PageIsAllVisible(page)) + all_visible = true; + else + all_visible = FluxVMCheckCached(scan->rs_base.rs_rd, scan->rs_cblock, + FLUX_VM_ALL_VISIBLE, + &scan->rs_vm_buffer, + &scan->rs_vm_blockno); + + for (offnum = FirstOffsetNumber; offnum <= maxoff; + offnum = OffsetNumberNext(offnum)) + { + ItemId itemid; + FluxTupleHeader *tuple_header; + + itemid = PageGetItemId(page, offnum); + if (!ItemIdIsNormal(itemid)) + continue; + + tuple_header = (FluxTupleHeader *) PageGetItem(page, itemid); + + /* Skip overflow records - they are not tuples */ + if (FluxIsOverflowRecordInline(tuple_header, ItemIdGetLength(itemid))) + continue; + + /* Skip speculative tuples not yet confirmed */ + if (tuple_header->t_flags & FLUX_TUPLE_SPECULATIVE) + continue; + + /* + * Check MVCC visibility. If the page is marked all-visible in the + * visibility map, skip the expensive per-tuple check. Otherwise, + * consult the commit timestamp and the sLog for in-progress + * transaction state. + * + * NOTE: Do NOT skip FLUX_TUPLE_DELETED tuples here. The DELETED flag + * is set physically at DELETE time, but the delete may be in-progress + * or aborted. The visibility function correctly consults the sLog to + * determine actual delete status. + */ + if (!all_visible && + scan->rs_base.rs_snapshot && + !FluxTupleVisibleToSnapshotDual(tuple_header, + scan->rs_base.rs_snapshot, + RelationGetRelid(scan->rs_base.rs_rd), + buffer)) + { + /* + * The on-page (newest) version is not visible to our snapshot. + * zheap read path: if this tuple was updated in place and still + * has a version chain in the UNDO fork, an OLDER version may be + * visible -- keep it as a candidate so flux_scan_getnextslot can + * reconstruct and serve the before-image. Only truly-invisible + * tuples (no history, or history exhausted) are dropped here. + */ + if ((tuple_header->t_flags & FLUX_TUPLE_UPDATED) && + IsMVCCSnapshot(scan->rs_base.rs_snapshot) && + RelUndoRecPtrIsValid(FluxTupleGetVersionPtr(tuple_header, + ItemIdGetLength(itemid)))) + { + scan->rs_vistuples[ntup++] = offnum; + continue; + } + + /* + * Tuple is not visible. If we're in a serializable transaction, + * check for rw-conflict out: a concurrent writer modified this + * tuple after our snapshot. + */ + if (IsolationIsSerializable()) + { + FluxCheckForSerializableConflictOut(scan->rs_base.rs_rd, + tuple_header, + buffer, + scan->rs_base.rs_snapshot); + } + continue; + } + + /* + * Tuple is visible. Acquire SIREAD predicate lock for SSI so that + * concurrent writers can detect rw-antidependencies on this tuple. + */ + if (IsolationIsSerializable()) + { + ItemPointerData item_tid; + + ItemPointerSet(&item_tid, BufferGetBlockNumber(buffer), offnum); + PredicateLockTID(scan->rs_base.rs_rd, &item_tid, + scan->rs_base.rs_snapshot, + InvalidTransactionId); + } + + scan->rs_vistuples[ntup++] = offnum; + } + + scan->rs_ntuples = ntup; + scan->rs_cindex = 0; + + LockBuffer(buffer, BUFFER_LOCK_UNLOCK); +} + +/* + * Get next tuple from scan (page-mode) + * + * Uses page-mode scanning: for each page, flux_prepare_pagescan() collects + * all visible tuple offsets under a single SHARE lock. This function then + * iterates through those offsets from the pinned-but-unlocked buffer. + * + * This eliminates the per-tuple ReadBuffer+LockBuffer+ReleaseBuffer overhead + * of the original implementation (50K rows = 50K buffer ops ā†’ 1 per page). + * + * For parallel scans, block assignment is coordinated via + * flux_scan_nextblock() which uses the parallel scan infrastructure. + */ +static bool +flux_scan_getnextslot(TableScanDesc sscan, ScanDirection direction, TupleTableSlot *slot) +{ + FluxScanDesc scan = (FluxScanDesc) sscan; + + /* If relation is empty, return false immediately */ + if (scan->rs_nblocks == 0) + { + ExecClearTuple(slot); + return false; + } + + for (;;) + { + Page page; + + /* + * Try to return the next visible tuple from the current page. The + * buffer is pinned but NOT locked -- this matches heap's page-mode + * pattern. FluxSlotStoreTuple acquires its own pin so the slot data + * remains valid after we move to the next page. + */ + while (scan->rs_cindex < scan->rs_ntuples) + { + OffsetNumber offnum; + ItemId itemid; + FluxTupleHeader *tuple_header; + + offnum = scan->rs_vistuples[scan->rs_cindex]; + scan->rs_cindex++; + + page = BufferGetPage(scan->rs_cbuf); + itemid = PageGetItemId(page, offnum); + + if (!ItemIdIsNormal(itemid)) + continue; + + tuple_header = (FluxTupleHeader *) PageGetItem(page, itemid); + + /* + * Before-image substitution for committed in-place UPDATEs (zheap + * read path). + * + * The on-page image is the NEWEST version, stamped with the + * updater's xmin. If that updater is not visible to our MVCC + * snapshot, we must serve the older version reconstructed from + * the per-relation UNDO fork (WS-PVS2): + * FluxReconstructVisibleVersion walks the t_verptr chain and + * stops at the version whose producing xid is visible + * (XidInMVCCSnapshot). Visibility is decided purely by the xid + * snapshot + CLOG, matching FluxTupleSatisfiesMVCC. + * + * Determine on-page visibility once; only reconstruct when the + * newest version is NOT visible to us. When it IS visible, serve + * on-page bytes directly. + */ + if ((tuple_header->t_flags & FLUX_TUPLE_UPDATED) && + scan->rs_base.rs_snapshot != NULL && + IsMVCCSnapshot(scan->rs_base.rs_snapshot) && + RelUndoRecPtrIsValid(FluxTupleGetVersionPtr(tuple_header, + ItemIdGetLength(itemid))) && + !FluxTupleVisibleToSnapshotDual(tuple_header, + scan->rs_base.rs_snapshot, + RelationGetRelid(scan->rs_base.rs_rd), + scan->rs_cbuf)) + { + char *bi_data = NULL; + int bi_len = 0; + ItemPointerData item_tid; + + ItemPointerSet(&item_tid, scan->rs_cblock, offnum); + + if (FluxReconstructVisibleVersion( + scan->rs_base.rs_rd, + &item_tid, + (const char *) tuple_header, + ItemIdGetLength(itemid), + scan->rs_base.rs_snapshot, + &bi_data, &bi_len)) + { + FluxTupleHeader *bi_tuple = (FluxTupleHeader *) bi_data; + + FluxSlotStoreMaterializedTuple(slot, bi_tuple, bi_len); + ItemPointerSet(&slot->tts_tid, scan->rs_cblock, offnum); + return true; + } + + /* + * No visible older version: the row is genuinely invisible to + * this snapshot (e.g. it was inserted-then-updated all after + * our snapshot). Skip it. + */ + continue; + } + + /* Normal: serve on-page data */ + FluxSlotStoreTuple(slot, tuple_header, + ItemIdGetLength(itemid), scan->rs_cbuf); + ItemPointerSet(&slot->tts_tid, scan->rs_cblock, offnum); + return true; + } + + /* + * Exhausted all visible tuples on the current page. Release the + * buffer pin and advance to the next block. + */ + if (BufferIsValid(scan->rs_cbuf)) + { + ReleaseBuffer(scan->rs_cbuf); + scan->rs_cbuf = InvalidBuffer; + } + + /* + * Get the next page. Use the read stream if available (prefetches + * upcoming pages for I/O efficiency), otherwise fall back to + * ReadBuffer for parallel scans. + */ + if (scan->rs_read_stream != NULL) + { + scan->rs_cbuf = read_stream_next_buffer(scan->rs_read_stream, NULL); + if (!BufferIsValid(scan->rs_cbuf)) + { + ExecClearTuple(slot); + return false; + } + scan->rs_cblock = BufferGetBlockNumber(scan->rs_cbuf); + } + else + { + BlockNumber block = flux_scan_nextblock(scan); + + if (!BlockNumberIsValid(block)) + { + ExecClearTuple(slot); + return false; + } + + scan->rs_cblock = block; + scan->rs_cbuf = ReadBuffer(scan->rs_base.rs_rd, scan->rs_cblock); + } + + /* + * Opportunistic cleanup: try to prune dead tuples before scanning. + * FluxPagePruneOpt() expects only a pin (no lock) and will + * non-blockingly try to get an exclusive lock. + */ + FluxPagePruneOpt(scan->rs_base.rs_rd, scan->rs_cbuf); + + /* + * Prepare the page scan: lock SHARE, collect all visible tuple + * offsets into rs_vistuples[], then unlock. The buffer stays pinned + * for efficient tuple access without re-locking. + */ + flux_prepare_pagescan(scan, scan->rs_cbuf); + } +} + +/* + * Set TID range for TID range scans + * + * This restricts the scan to only return tuples within the given TID range. + * Used by TID range scans (WHERE ctid >= ... AND ctid < ...). + * + * Following the heap AM pattern, we store the effective min/max TIDs in the + * base scan descriptor's st.tidrange fields and configure the scan start + * position accordingly. + */ +static void +flux_scan_set_tidrange(TableScanDesc sscan, ItemPointer mintid, + ItemPointer maxtid) +{ + FluxScanDesc scan = (FluxScanDesc) sscan; + BlockNumber nblocks; + ItemPointerData highestItem; + ItemPointerData lowestItem; + + nblocks = RelationGetNumberOfBlocks(sscan->rs_rd); + + /* + * For relations without any pages, we can simply leave the TID range + * unset. There will be no tuples to scan, therefore no tuples outside + * the given TID range. + */ + if (nblocks == 0) + return; + + /* + * Set up ItemPointers which point to the first and last possible tuples + * in the relation. + */ + ItemPointerSet(&highestItem, nblocks - 1, MaxOffsetNumber); + ItemPointerSet(&lowestItem, 0, FirstOffsetNumber); + + /* + * If the given maximum TID is below the highest possible TID in the + * relation, then restrict the range to that, otherwise we scan to the end + * of the relation. + */ + if (ItemPointerCompare(maxtid, &highestItem) < 0) + ItemPointerCopy(maxtid, &highestItem); + + /* + * If the given minimum TID is above the lowest possible TID in the + * relation, then restrict the range to only scan for TIDs above that. + */ + if (ItemPointerCompare(mintid, &lowestItem) > 0) + ItemPointerCopy(mintid, &lowestItem); + + /* + * Check for an empty range. + */ + if (ItemPointerCompare(&highestItem, &lowestItem) < 0) + { + /* Force an empty scan */ + scan->rs_cblock = nblocks; + scan->rs_coffset = FirstOffsetNumber; + ItemPointerSetInvalid(&sscan->st.tidrange.rs_mintid); + ItemPointerSetInvalid(&sscan->st.tidrange.rs_maxtid); + return; + } + + /* Set scan start position to the first block in range */ + scan->rs_cblock = ItemPointerGetBlockNumberNoCheck(&lowestItem); + scan->rs_coffset = FirstOffsetNumber; + + /* Store the effective TID range in the base scan descriptor */ + ItemPointerCopy(&lowestItem, &sscan->st.tidrange.rs_mintid); + ItemPointerCopy(&highestItem, &sscan->st.tidrange.rs_maxtid); +} + +/* + * Get next tuple within the TID range set by flux_scan_set_tidrange. + * + * This delegates to the regular flux_scan_getnextslot for tuple fetching + * and visibility, then filters to only return tuples within the TID range. + * This keeps visibility semantics consistent between regular and TID range + * scans. + */ +static bool +flux_scan_getnextslot_tidrange(TableScanDesc sscan, ScanDirection direction, + TupleTableSlot *slot) +{ + FluxScanDesc scan = (FluxScanDesc) sscan; + ItemPointer mintid = &sscan->st.tidrange.rs_mintid; + ItemPointer maxtid = &sscan->st.tidrange.rs_maxtid; + BlockNumber nblocks; + BlockNumber block; + BlockNumber maxblock; + Buffer buffer; + Page page; + OffsetNumber offnum; + OffsetNumber maxoff; + ItemId itemid; + FluxTupleHeader *tuple_header; + + /* If the range is invalid/empty, we're done */ + if (!ItemPointerIsValid(mintid) || !ItemPointerIsValid(maxtid)) + { + ExecClearTuple(slot); + return false; + } + + maxblock = ItemPointerGetBlockNumber(maxtid); + + /* Clear the slot */ + ExecClearTuple(slot); + + nblocks = RelationGetNumberOfBlocks(sscan->rs_rd); + if (nblocks == 0) + return false; + + /* Scan pages within the TID range */ + for (block = scan->rs_cblock; block <= maxblock && block < nblocks; block++) + { + buffer = ReadBuffer(sscan->rs_rd, block); + + /* + * Opportunistic cleanup on first visit to page, consistent with the + * regular scan path. + */ + if (scan->rs_coffset == FirstOffsetNumber) + FluxPagePruneOpt(sscan->rs_rd, buffer); + + LockBuffer(buffer, BUFFER_LOCK_SHARE); + page = BufferGetPage(buffer); + + /* Skip new/empty pages */ + if (PageIsNew(page)) + { + UnlockReleaseBuffer(buffer); + scan->rs_coffset = FirstOffsetNumber; + continue; + } + + maxoff = PageGetMaxOffsetNumber(page); + + for (offnum = scan->rs_coffset; offnum <= maxoff; offnum++) + { + ItemPointerData curtid; + + ItemPointerSet(&curtid, block, offnum); + + /* + * Filter by TID range. Skip tuples below mintid. + */ + if (ItemPointerCompare(&curtid, mintid) < 0) + continue; + + /* + * If we've passed maxtid, we're done scanning. + */ + if (ItemPointerCompare(&curtid, maxtid) > 0) + { + UnlockReleaseBuffer(buffer); + return false; + } + + itemid = PageGetItemId(page, offnum); + + if (!ItemIdIsNormal(itemid)) + continue; + + tuple_header = (FluxTupleHeader *) PageGetItem(page, itemid); + + /* Skip overflow records - they are not tuples */ + if (FluxIsOverflowRecordInline(tuple_header, + ItemIdGetLength(itemid))) + continue; + + /* Skip speculative tuples not yet confirmed */ + if (tuple_header->t_flags & FLUX_TUPLE_SPECULATIVE) + continue; + + /* Check MVCC visibility (handles DELETED via sLog) */ + if (sscan->rs_snapshot && + !FluxTupleVisibleToSnapshotDual(tuple_header, + sscan->rs_snapshot, + RelationGetRelid(sscan->rs_rd), + buffer)) + continue; + + /* + * Store the tuple into the slot with a buffer pin. Decompression + * happens lazily during deformation. + */ + FluxSlotStoreTuple(slot, tuple_header, + ItemIdGetLength(itemid), buffer); + ItemPointerSet(&slot->tts_tid, block, offnum); + /* Update scan position for next call */ + scan->rs_cblock = block; + scan->rs_coffset = offnum + 1; + + LockBuffer(buffer, BUFFER_LOCK_UNLOCK); + ReleaseBuffer(buffer); + return true; + } + + LockBuffer(buffer, BUFFER_LOCK_UNLOCK); + ReleaseBuffer(buffer); + scan->rs_coffset = FirstOffsetNumber; + } + + /* Reached end of assigned range */ + return false; +} + +/* + * Bitmap heap scan: fetch next tuple from bitmap + * + * This is the scan_bitmap_next_tuple callback. It iterates over TIDs from + * the TBM (TID bitmap) built by a BitmapIndexScan, fetches and checks + * visibility for each tuple, and returns visible ones in the slot. + * + * For each block indicated by the bitmap: + * - If the block is "lossy" (the bitmap lost per-tuple precision for this + * block), we check every tuple on the page. + * - If the block is "exact", we only check the specific offsets indicated. + * + * The scan descriptor was set up via table_beginscan_bm() and the TBM + * iterator is stored in scan->st.rs_tbmiterator. + */ +static bool +flux_scan_bitmap_next_tuple(TableScanDesc scan, + TupleTableSlot *slot, + bool *recheck, + uint64 *lossy_pages, + uint64 *exact_pages) +{ + FluxScanDesc rscan = (FluxScanDesc) scan; + + Assert(rscan->rs_read_stream); + + for (;;) + { + void *per_buffer_data; + TBMIterateResult *tbmres; + + /* + * If we have tuples remaining from a previously fetched page, try to + * return one. + */ + if (rscan->rs_ntuples > 0 && + rscan->rs_cindex < rscan->rs_ntuples) + { + OffsetNumber offnum; + Page page; + ItemId itemid; + FluxTupleHeader *tuple_hdr; + + offnum = rscan->rs_vistuples[rscan->rs_cindex]; + rscan->rs_cindex++; + + /* Re-read the page (we released the lock after visibility check) */ + if (!BufferIsValid(rscan->rs_cbuf)) + rscan->rs_cbuf = ReadBuffer(scan->rs_rd, rscan->rs_cblock); + + LockBuffer(rscan->rs_cbuf, BUFFER_LOCK_SHARE); + page = BufferGetPage(rscan->rs_cbuf); + + itemid = PageGetItemId(page, offnum); + if (!ItemIdIsNormal(itemid)) + { + LockBuffer(rscan->rs_cbuf, BUFFER_LOCK_UNLOCK); + continue; + } + + tuple_hdr = (FluxTupleHeader *) PageGetItem(page, itemid); + + /* + * NOTE: Do NOT skip FLUX_TUPLE_DELETED here. This tuple passed + * the visibility check during page scan preparation. The DELETED + * flag may be set but the delete could be in-progress or aborted. + */ + + /* + * FLUX updates in place keeping the same TID, so a changed + * indexed column leaves a stale (oldkey -> tid) secondary entry + * beside the new one. A bitmap scan keeps only TIDs and discards + * the index tuple, so it cannot compare the stored key against + * the live tuple. Force the executor to re-evaluate + * bitmapqualorig against the live tuple whenever we return a + * committed in-place UPDATE: an exact (non-lossy) bitmap would + * otherwise trust the stale index entry and return a row whose + * live key no longer matches the scan key. + */ + if (tuple_hdr->t_flags & FLUX_TUPLE_UPDATED) + *recheck = true; + + /* + * Before-image substitution for in-place UPDATEs (zheap read + * path), mirroring the sequential-scan and index-fetch paths. A + * snapshot that cannot see the updater's xmin must observe the + * before-image, not the on-page (new) data. Serve it from the + * per-relation UNDO fork (WS-PVS2); FluxReconstructVisibleVersion + * walks the t_verptr chain and stops at the version whose + * producing xid is visible. Visibility is ordinary heap-shaped + * xmin/xmax + CLOG + snapshot (FluxTupleSatisfiesMVCC). + */ + if ((tuple_hdr->t_flags & FLUX_TUPLE_UPDATED) && + scan->rs_snapshot != NULL && + IsMVCCSnapshot(scan->rs_snapshot) && + RelUndoRecPtrIsValid(FluxTupleGetVersionPtr(tuple_hdr, + ItemIdGetLength(itemid))) && + !FluxTupleVisibleToSnapshotDual(tuple_hdr, scan->rs_snapshot, + RelationGetRelid(scan->rs_rd), + rscan->rs_cbuf)) + { + char *bi_data = NULL; + int bi_len = 0; + ItemPointerData item_tid; + + ItemPointerSet(&item_tid, rscan->rs_cblock, offnum); + + if (FluxReconstructVisibleVersion( + scan->rs_rd, + &item_tid, + (const char *) tuple_hdr, + ItemIdGetLength(itemid), + scan->rs_snapshot, + &bi_data, &bi_len)) + { + FluxTupleHeader *bi_tuple = (FluxTupleHeader *) bi_data; + + FluxSlotStoreMaterializedTuple(slot, bi_tuple, bi_len); + slot->tts_tableOid = RelationGetRelid(scan->rs_rd); + ItemPointerSet(&slot->tts_tid, rscan->rs_cblock, offnum); + LockBuffer(rscan->rs_cbuf, BUFFER_LOCK_UNLOCK); + + return true; + } + + /* No visible older version: skip this tuple. */ + LockBuffer(rscan->rs_cbuf, BUFFER_LOCK_UNLOCK); + continue; + } + + /* + * Store the tuple with a buffer pin. The slot gets its own pin + * via FluxSlotStoreTuple so the data stays valid. + */ + FluxSlotStoreTuple(slot, tuple_hdr, + ItemIdGetLength(itemid), rscan->rs_cbuf); + slot->tts_tableOid = RelationGetRelid(scan->rs_rd); + ItemPointerSet(&slot->tts_tid, rscan->rs_cblock, offnum); + LockBuffer(rscan->rs_cbuf, BUFFER_LOCK_UNLOCK); + + return true; + } + + /* Release buffer from previous block */ + if (BufferIsValid(rscan->rs_cbuf)) + { + ReleaseBuffer(rscan->rs_cbuf); + rscan->rs_cbuf = InvalidBuffer; + } + + /* + * Advance to the next block in the bitmap. The read stream pulls + * blocks from the TBM iterator (via flux_bitmap_stream_read_next), + * prefetching upcoming bitmap pages, and hands back the matching + * TBMIterateResult in per_buffer_data. Out-of-range blocks were + * already filtered by the callback. + */ + rscan->rs_cbuf = read_stream_next_buffer(rscan->rs_read_stream, + &per_buffer_data); + + if (!BufferIsValid(rscan->rs_cbuf)) + return false; /* bitmap exhausted */ + + tbmres = per_buffer_data; + + Assert(BlockNumberIsValid(tbmres->blockno)); + Assert(BufferGetBlockNumber(rscan->rs_cbuf) == tbmres->blockno); + + *recheck = tbmres->recheck; + + rscan->rs_cblock = tbmres->blockno; + + LockBuffer(rscan->rs_cbuf, BUFFER_LOCK_SHARE); + { + Page page = BufferGetPage(rscan->rs_cbuf); + int ntup = 0; + OffsetNumber maxoff = PageGetMaxOffsetNumber(page); + + /* Allocate vistuples array if needed */ + if (rscan->rs_vistuples == NULL) + { + rscan->rs_vistuples = (OffsetNumber *) + MemoryContextAlloc(TopMemoryContext, + MaxOffsetNumber * sizeof(OffsetNumber)); + } + + if (!tbmres->lossy) + { + /* + * Exact page: only examine offsets listed in the bitmap. + */ + OffsetNumber offsets[TBM_MAX_TUPLES_PER_PAGE]; + int noffsets; + + noffsets = tbm_extract_page_tuple(tbmres, offsets, + TBM_MAX_TUPLES_PER_PAGE); + + for (int j = 0; j < noffsets; j++) + { + OffsetNumber offnum = offsets[j]; + ItemId itemid; + FluxTupleHeader *tuple_hdr; + + if (offnum < FirstOffsetNumber || offnum > maxoff) + continue; + + itemid = PageGetItemId(page, offnum); + if (!ItemIdIsNormal(itemid)) + continue; + + tuple_hdr = (FluxTupleHeader *) PageGetItem(page, itemid); + + /* Skip overflow records */ + if (FluxIsOverflowRecordInline(tuple_hdr, ItemIdGetLength(itemid))) + continue; + + /* Skip speculative tuples */ + if (tuple_hdr->t_flags & FLUX_TUPLE_SPECULATIVE) + continue; + + /* + * Check visibility. Keep an invisible-but-updated tuple + * that still has a version chain as a candidate so the + * tuple loop can reconstruct and serve the before-image + * (zheap read path). + */ + if (scan->rs_snapshot && + !FluxTupleVisibleToSnapshotDual(tuple_hdr, scan->rs_snapshot, + RelationGetRelid(scan->rs_rd), + rscan->rs_cbuf)) + { + if (!((tuple_hdr->t_flags & FLUX_TUPLE_UPDATED) && + IsMVCCSnapshot(scan->rs_snapshot) && + RelUndoRecPtrIsValid(FluxTupleGetVersionPtr(tuple_hdr, + ItemIdGetLength(itemid))))) + continue; + } + + rscan->rs_vistuples[ntup++] = offnum; + } + + (*exact_pages)++; + } + else + { + /* + * Lossy page: examine every tuple on the page. tbmres->lossy + * is true here. + */ + OffsetNumber offnum; + + for (offnum = FirstOffsetNumber; offnum <= maxoff; + offnum = OffsetNumberNext(offnum)) + { + ItemId itemid; + FluxTupleHeader *tuple_hdr; + + itemid = PageGetItemId(page, offnum); + if (!ItemIdIsNormal(itemid)) + continue; + + tuple_hdr = (FluxTupleHeader *) PageGetItem(page, itemid); + + /* Skip overflow records */ + if (FluxIsOverflowRecordInline(tuple_hdr, ItemIdGetLength(itemid))) + continue; + + /* Skip speculative tuples */ + if (tuple_hdr->t_flags & FLUX_TUPLE_SPECULATIVE) + continue; + + /* + * Check visibility. Keep an invisible-but-updated tuple + * that still has a version chain as a candidate so the + * tuple loop can reconstruct and serve the before-image + * (zheap read path). + */ + if (scan->rs_snapshot && + !FluxTupleVisibleToSnapshotDual(tuple_hdr, scan->rs_snapshot, + RelationGetRelid(scan->rs_rd), + rscan->rs_cbuf)) + { + if (!((tuple_hdr->t_flags & FLUX_TUPLE_UPDATED) && + IsMVCCSnapshot(scan->rs_snapshot) && + RelUndoRecPtrIsValid(FluxTupleGetVersionPtr(tuple_hdr, + ItemIdGetLength(itemid))))) + continue; + } + + rscan->rs_vistuples[ntup++] = offnum; + } + + (*lossy_pages)++; + } + + rscan->rs_ntuples = ntup; + rscan->rs_cindex = 0; + } + LockBuffer(rscan->rs_cbuf, BUFFER_LOCK_UNLOCK); + + /* Loop back to return the first visible tuple from this block */ + } +} + +/* + * ------------------------------------------------------------------------ + * Index scan callbacks for FLUX AM + * ------------------------------------------------------------------------ + */ + +static IndexFetchTableData * +flux_index_fetch_begin(Relation rel, uint32 flags) +{ + IndexFetchFluxData *scan = palloc0_object(IndexFetchFluxData); + + scan->base.rel = rel; + scan->buffer = InvalidBuffer; + + return &scan->base; +} + +static void +flux_index_fetch_reset(IndexFetchTableData *scan) +{ + IndexFetchFluxData *rscan = (IndexFetchFluxData *) scan; + + if (BufferIsValid(rscan->buffer)) + { + ReleaseBuffer(rscan->buffer); + rscan->buffer = InvalidBuffer; + } +} + +static void +flux_index_fetch_end(IndexFetchTableData *scan) +{ + IndexFetchFluxData *rscan = (IndexFetchFluxData *) scan; + + flux_index_fetch_reset(scan); + + pfree(rscan); +} + +/* + * Fetches, as part of an index scan, tuple at `tid` into `slot`, after doing + * a visibility test according to `snapshot`. If a tuple was found and passed + * the visibility test, returns true, false otherwise. Note that *tid may be + * modified when we return true (see later remarks on multiple row versions + * reachable via a single index entry). + * + * *call_again needs to be false on the first call to table_index_fetch_tuple() for + * a tid. If there potentially is another tuple matching the tid, *call_again + * will be set to true, signaling that table_index_fetch_tuple() should be called + * again for the same tid. + * + * *all_dead, if all_dead is not NULL, will be set to true by + * table_index_fetch_tuple() iff it is guaranteed that no backend needs to see + * that tuple. Index AMs can use that to avoid returning that tid in future + * searches. + * + * The difference between this function and table_tuple_fetch_row_version() + * is that this function returns the currently visible version of a row if + * the AM supports storing multiple row versions reachable via a single index + * entry. FLUX does in-place updates so there are no version chains to + * follow; this behaves identically to table_tuple_fetch_row_version(). + */ +static bool +flux_index_fetch_tuple(IndexFetchTableData *iftd, + ItemPointer tid, + Snapshot snapshot, + TupleTableSlot *tts, + bool *call_again, bool *all_dead) +{ + IndexFetchFluxData *scan = (IndexFetchFluxData *) iftd; + Relation rel = scan->base.rel; + BlockNumber blkno = ItemPointerGetBlockNumber(tid); + OffsetNumber offnum = ItemPointerGetOffsetNumber(tid); + Buffer buffer; + Page page; + ItemId itemid; + FluxTupleHeader *tuple_hdr; + bool visible = false; + + /* + * Initialize output parameters. all_dead is strictly PER-TID: it must + * reflect only whether *this* TID is dead to every snapshot, never a + * verdict carried over from a previously-fetched TID in the same scan. + * FLUX does in-place updates and never sets call_again, so each TID is + * fetched exactly once; a sticky scan-level all_dead would leak an + * earlier TID's dead verdict onto a later LIVE TID, making indexam.c set + * kill_prior_tuple and _bt_killitems mark the live entry LP_DEAD -- the + * silent-data-loss "kill LIVE entries" failure. So always start false + * and set true only in the truly-dead branch below. + */ + *call_again = false; + if (all_dead) + *all_dead = false; + + /* + * FLUX secondary indexes hold plain 6-byte heap-style TIDs and FLUX does + * OUT-OF-PLACE updates for any key-changing UPDATE (delete + insert at a + * new TID), so a changed indexed column never leaves a stale (oldkey -> + * tid) entry pointing at a live in-place tuple. There is therefore no + * in-place index staleness to recheck: this is a plain heap-style TID + * fetch that just checks visibility. + */ + + /* Clear the slot */ + ExecClearTuple(tts); + + /* + * Release any buffer from a previous index_fetch_tuple call. This + * prevents buffer leaks when scanning multiple tuples. + */ + if (BufferIsValid(scan->buffer)) + { + ReleaseBuffer(scan->buffer); + scan->buffer = InvalidBuffer; + } + + /* Read the page */ + buffer = ReadBuffer(rel, blkno); + LockBuffer(buffer, BUFFER_LOCK_SHARE); + + page = BufferGetPage(buffer); + + /* Validate offset */ + if (offnum < FirstOffsetNumber || offnum > PageGetMaxOffsetNumber(page)) + { + UnlockReleaseBuffer(buffer); + return false; + } + + /* Get the item */ + itemid = PageGetItemId(page, offnum); + + if (!ItemIdIsNormal(itemid)) + { + UnlockReleaseBuffer(buffer); + return false; + } + + /* Get tuple header */ + tuple_hdr = (FluxTupleHeader *) PageGetItem(page, itemid); + + /* Check visibility (heap-shaped xmin/xmax) */ + if (snapshot) + { + if (snapshot->snapshot_type == SNAPSHOT_DIRTY) + { + /* + * For SNAPSHOT_DIRTY (used by _bt_check_unique during ON + * CONFLICT), we must report the inserting/deleting xid through + * the snapshot struct so that SpeculativeInsertionWait can + * function correctly. This mirrors HeapTupleSatisfiesDirty() + * behaviour. + * + * Uses t_xmin for INSERT visibility (fast path) and a single + * batched sLog lookup for DELETE/UPDATE/LOCK state. + */ + SLogTupleOp slog_entries[SLOG_MAX_TUPLE_OPS]; + int slog_nfound = -1; /* lazy: -1 = not yet fetched */ + + snapshot->xmin = InvalidTransactionId; + snapshot->xmax = InvalidTransactionId; + snapshot->speculativeToken = 0; + + /* + * Check if this tuple is uncommitted (inserted by a still-running + * transaction). Use t_xmin for the fast path, falling back to + * sLog for speculative inserts. + */ + if (tuple_hdr->t_flags & FLUX_TUPLE_UNCOMMITTED) + { + TransactionId hint_xid = InvalidTransactionId; + + if (TransactionIdIsValid(hint_xid) && + !TransactionIdIsCurrentTransactionId(hint_xid)) + { + if (TransactionIdIsInProgress(hint_xid)) + { + /* + * Another transaction is inserting this tuple. Report + * xmin for SpeculativeInsertionWait. + */ + snapshot->xmin = hint_xid; + + if (tuple_hdr->t_flags & FLUX_TUPLE_SPECULATIVE) + { + /* Fetch sLog for speculative token */ + slog_nfound = SLogTupleLookupFiltered( + RelationGetRelid(rel), tid, + InvalidTransactionId, + slog_entries, SLOG_MAX_TUPLE_OPS); + { + int si; + + for (si = 0; si < slog_nfound; si++) + { + if (slog_entries[si].xid == hint_xid && + slog_entries[si].spec_token != 0) + { + snapshot->speculativeToken = + slog_entries[si].spec_token; + break; + } + } + } + } + + visible = true; + goto visibility_done; + } + else if (TransactionIdDidAbort(hint_xid)) + { + /* Inserter aborted -- invisible */ + visible = false; + goto visibility_done; + } + else + { + /* + * Inserter committed. Clear stale flag via + * BufferSetHintBits16 (handles lock upgrade). + */ + BufferSetHintBits16(&tuple_hdr->t_flags, + tuple_hdr->t_flags & ~FLUX_TUPLE_UNCOMMITTED, + buffer); + } + } + else if (TransactionIdIsValid(hint_xid) && + TransactionIdIsCurrentTransactionId(hint_xid)) + { + /* + * Our own insert. Check sLog for our own delete or for + * an ABORTED entry from savepoint rollback. + */ + slog_nfound = SLogTupleLookupFiltered( + RelationGetRelid(rel), tid, + InvalidTransactionId, + slog_entries, SLOG_MAX_TUPLE_OPS); + { + int si; + bool found_invisible = false; + + for (si = 0; si < slog_nfound; si++) + { + if (!TransactionIdEquals(slog_entries[si].xid, hint_xid)) + continue; + if (slog_entries[si].op_type == SLOG_OP_DELETE || + slog_entries[si].op_type == SLOG_OP_ABORTED) + { + found_invisible = true; + break; + } + } + + if (found_invisible) + { + visible = false; + goto visibility_done; + } + } + /* Our insert, not deleted/aborted by us -- fall through */ + } + else + { + /* + * Invalid hint_xid -- fall back to sLog lookup. This + * handles pre-upgrade tuples. + */ + slog_nfound = SLogTupleLookupFiltered( + RelationGetRelid(rel), tid, + InvalidTransactionId, + slog_entries, SLOG_MAX_TUPLE_OPS); + { + int si; + bool found_inserter = false; + + for (si = 0; si < slog_nfound; si++) + { + if (slog_entries[si].op_type == SLOG_OP_INSERT && + TransactionIdIsInProgress(slog_entries[si].xid)) + { + snapshot->xmin = slog_entries[si].xid; + found_inserter = true; + break; + } + if (slog_entries[si].op_type == SLOG_OP_ABORTED) + { + visible = false; + goto visibility_done; + } + } + + if (!found_inserter) + { + /* Stale flag, clear it */ + BufferSetHintBits16(&tuple_hdr->t_flags, + tuple_hdr->t_flags & ~FLUX_TUPLE_UNCOMMITTED, + buffer); + } + else + { + visible = true; + goto visibility_done; + } + } + } + } + + /* Inserting xact is committed (or ours). Check deletion. */ + if (tuple_hdr->t_flags & FLUX_TUPLE_DELETED) + { + /* Single batched sLog lookup for delete state */ + if (slog_nfound < 0) + slog_nfound = SLogTupleLookupFiltered( + RelationGetRelid(rel), tid, + InvalidTransactionId, + slog_entries, SLOG_MAX_TUPLE_OPS); + { + int si; + bool delete_aborted = false; + + for (si = 0; si < slog_nfound; si++) + { + if (TransactionIdIsCurrentTransactionId(slog_entries[si].xid) && + (slog_entries[si].op_type == SLOG_OP_DELETE || + slog_entries[si].op_type == SLOG_OP_UPDATE)) + { + /* We deleted it ourselves */ + visible = false; + goto visibility_done; + } + if (TransactionIdIsInProgress(slog_entries[si].xid) && + (slog_entries[si].op_type == SLOG_OP_DELETE || + slog_entries[si].op_type == SLOG_OP_UPDATE)) + { + /* Deleter still running */ + snapshot->xmax = slog_entries[si].xid; + visible = true; + goto visibility_done; + } + if (slog_entries[si].op_type == SLOG_OP_ABORTED) + { + /* + * Delete was rolled back (ROLLBACK TO SAVEPOINT + * or full abort with deferred UNDO). The UNDO + * worker has not yet cleared the DELETED flag. + */ + delete_aborted = true; + } + } + + if (delete_aborted) + { + /* + * All delete operations were aborted. Clear the + * stale DELETED flag via hint bits and treat the + * tuple as live. + */ + BufferSetHintBits16(&tuple_hdr->t_flags, + tuple_hdr->t_flags & ~FLUX_TUPLE_DELETED, + buffer); + /* Fall through ā€” tuple is still live */ + } + else if (slog_nfound == 0) + { + /* + * No sLog entries and DELETED flag is set. The UNDO + * worker always clears the DELETED flag before + * removing the sLog entry, so if we see DELETED with + * no sLog entries, the deletion committed and UNDO + * cleanup removed the entries afterward. + */ + Assert(!(tuple_hdr->t_flags & FLUX_TUPLE_DELETED) || + true); /* invariant: flag + no slog = + * committed */ + visible = false; + goto visibility_done; + } + else + { + /* + * sLog entries exist but none are in-progress, + * current, or aborted ā€” deletion committed. + */ + visible = false; + goto visibility_done; + } + } + } + + /* + * Check if this is the old version of an out-of-place update. The + * FLUX_TUPLE_UPDATED flag means this tuple has been superseded by + * a newer version at t_ctid. However, the updater may have + * aborted ā€” check sLog for ABORTED entries before declaring it + * invisible. + */ + if (tuple_hdr->t_flags & FLUX_TUPLE_UPDATED) + { + /* + * FLUX updates in place: an UPDATED tuple whose t_ctid still + * points at itself is the live current version, not a + * superseded old version. Only a genuine out-of-place move + * (cross-page defrag) repoints t_ctid elsewhere. Skip the + * supersession check for in-place updates so a retained + * committed UPDATE marker (kept for before-image serving) + * does not make the live row vanish from a dirty-snapshot + * probe -- e.g. the ON CONFLICT arbiter scan would otherwise + * miss the existing row after any prior UPDATE on it. This + * mirrors the MVCC path's handling in + * FluxTupleVisibleToSnapshotDual. + */ + if (ItemPointerEquals(&tuple_hdr->t_ctid, tid)) + { + /* In-place update: on-page tuple is live. */ + visible = true; + goto visibility_done; + } + + if (slog_nfound < 0) + slog_nfound = SLogTupleLookupFiltered( + RelationGetRelid(rel), tid, + InvalidTransactionId, + slog_entries, SLOG_MAX_TUPLE_OPS); + { + int si; + bool update_aborted = false; + + for (si = 0; si < slog_nfound; si++) + { + if (slog_entries[si].op_type == SLOG_OP_ABORTED) + { + update_aborted = true; + break; + } + if ((slog_entries[si].op_type == SLOG_OP_UPDATE) && + !TransactionIdIsCurrentTransactionId(slog_entries[si].xid) && + TransactionIdDidAbort(slog_entries[si].xid)) + { + update_aborted = true; + break; + } + } + + if (update_aborted) + { + /* + * Updater aborted. Clear stale UPDATED flag via + * hint-bits and treat as still-live tuple. + */ + BufferSetHintBits16(&tuple_hdr->t_flags, + tuple_hdr->t_flags & ~FLUX_TUPLE_UPDATED, + buffer); + /* Fall through to visible */ + } + else if (slog_nfound == 0) + { + /* + * No sLog entries but UPDATED flag is set. This + * means either: (a) the retained UPDATE entry was + * reclaimed by the per-TID oldest-entry eviction in + * flat_hash_apply_insert (hot row), or (b) the + * background worker cleaned up the entry. + * + * In both cases the update committed ā€” the tuple on + * page IS the current version. Clear the stale + * UPDATED flag and treat as visible (live tuple). + */ + BufferSetHintBits16(&tuple_hdr->t_flags, + tuple_hdr->t_flags & ~FLUX_TUPLE_UPDATED, + buffer); + /* Fall through to visible */ + } + else + { + /* + * Check if updater is current, in-progress, or + * committed + */ + bool updater_running = false; + + for (si = 0; si < slog_nfound; si++) + { + if (slog_entries[si].op_type != SLOG_OP_UPDATE) + continue; + + if (TransactionIdIsCurrentTransactionId(slog_entries[si].xid)) + { + /* + * We are the updater ā€” old version is dead. + * This mirrors the DELETED handling above. + */ + visible = false; + goto visibility_done; + } + + if (TransactionIdIsInProgress(slog_entries[si].xid)) + { + snapshot->xmax = slog_entries[si].xid; + updater_running = true; + break; + } + } + + if (!updater_running) + { + /* Updater committed ā€” old version is dead */ + visible = false; + goto visibility_done; + } + /* Updater still running ā€” tuple visible for now */ + } + } + } + + /* LOCKED tuples are still visible (lock != delete) */ + if ((tuple_hdr->t_flags & FLUX_TUPLE_LOCKED) && + !(tuple_hdr->t_flags & (FLUX_TUPLE_DELETED | FLUX_TUPLE_UPDATED))) + { + visible = true; + goto visibility_done; + } + + visible = true; + } + else + { + visible = FluxTupleVisibleToSnapshotDual(tuple_hdr, snapshot, + RelationGetRelid(rel), + buffer); + } + } + else + { + /* + * No snapshot means fetch unconditionally (e.g., system catalog + * scans, VACUUM FULL table rewrite). Only truly deleted tuples are + * invisible. UPDATED tuples are live (they contain the current + * version of the data after an in-place update). + */ + visible = !(tuple_hdr->t_flags & FLUX_TUPLE_DELETED); + } +visibility_done: + + if (!visible) + { + /* + * zheap read path (index fetch): the on-page (newest) version is not + * visible to our snapshot, but if this tuple was updated in place and + * still has a version chain in the UNDO fork, an OLDER version may be + * visible. Reconstruct it and serve the before-image, mirroring the + * sequential-scan path. This is what lets a REPEATABLE READ reader + * whose snapshot predates a committed UPDATE still find the row at + * its old indexed key. + */ + if ((tuple_hdr->t_flags & FLUX_TUPLE_UPDATED) && + snapshot != NULL && IsMVCCSnapshot(snapshot) && + RelUndoRecPtrIsValid(FluxTupleGetVersionPtr(tuple_hdr, + ItemIdGetLength(itemid)))) + { + char *bi_data = NULL; + int bi_len = 0; + + LockBuffer(buffer, BUFFER_LOCK_UNLOCK); + if (FluxReconstructVisibleVersion(rel, tid, + (const char *) tuple_hdr, + ItemIdGetLength(itemid), + snapshot, + &bi_data, &bi_len)) + { + FluxTupleHeader *bi_tuple = (FluxTupleHeader *) bi_data; + + FluxSlotStoreMaterializedTuple(tts, bi_tuple, bi_len); + tts->tts_tableOid = RelationGetRelid(rel); + ItemPointerCopy(tid, &tts->tts_tid); + ReleaseBuffer(buffer); + return true; + } + ReleaseBuffer(buffer); + return false; + } + + /* + * Set the all_dead hint only if the tuple is committed-deleted AND + * its deleter (t_xmax) precedes the oldest-xmin horizon, meaning no + * running or future snapshot can still see it. FluxTupleDeadToAll + * does the heap-shaped XID-horizon check (t_xmax committed and < + * oldest_xmin); this replaced the HLC-era FluxCanVacuumTimestamp, + * which compared the on-page word as a wall-clock timestamp -- WRONG + * post-pivot, since that word now packs the XID-based t_xmax. + * + * Setting all_dead prematurely would let the index AM remove the + * entry while concurrent transactions still need it. + */ + if (FluxTupleDeadToAll(tuple_hdr, FluxGetOldestXminHorizon(rel))) + { + if (all_dead) + *all_dead = true; + } + + /* + * Do NOT set call_again for UPDATED tuples. For out-of-place updates + * the executor inserts new index entries (TU_All), so the index scan + * will naturally find the new version through its own index entry. + * Setting call_again=true here with the same tid causes an infinite + * loop because the caller retries with the unchanged tid parameter. + */ + + UnlockReleaseBuffer(buffer); + return false; + } + + /* + * Unlock the buffer before materializing the slot. We keep the pin to + * ensure the page doesn't get evicted. We must unlock here because + * FluxTupleToSlotWithOverflow may need to fetch overflow data, and if + * that overflow is on the same page, it would try to lock an + * already-locked buffer causing an assertion failure. + */ + LockBuffer(buffer, BUFFER_LOCK_UNLOCK); + + /* + * Before-image substitution for in-place UPDATEs (zheap read path), + * mirroring the sequential-scan path (flux_getnextslot) and the + * fetch-by-TID path (flux_tuple_fetch_row_version). If the reader can + * see the updater's xmin the on-page value is correct and the callee + * returns false (serve on-page); if not, an older visible version is + * reconstructed from the per-relation UNDO fork (WS-PVS2) so a REPEATABLE + * READ / SERIALIZABLE reader whose snapshot cannot see the update + * observes the before-image through any index -- including one whose key + * the UPDATE never touched (e.g. a primary-key lookup). Visibility is + * ordinary heap-shaped xmin/xmax + CLOG + snapshot (XidInMVCCSnapshot in + * the callee). + * + * No index-key recheck is needed: FLUX does in-place updates only for + * NON-key columns (key changes go out of place to a new TID), so the + * before-image reached through any index entry has the same key as the + * stored entry that led here. + */ + if ((tuple_hdr->t_flags & FLUX_TUPLE_UPDATED) && + !(tuple_hdr->t_flags & FLUX_TUPLE_UNCOMMITTED) && + snapshot != NULL && IsMVCCSnapshot(snapshot) && + FluxDirtyMapCheck(RelationGetRelid(rel), + ItemPointerGetBlockNumber(tid))) + { + char *bi_data = NULL; + int bi_len = 0; + + if (FluxReconstructVisibleVersion(rel, tid, + (const char *) tuple_hdr, + ItemIdGetLength(itemid), + snapshot, + &bi_data, &bi_len)) + { + FluxTupleHeader *bi_tuple = (FluxTupleHeader *) bi_data; + + FluxSlotStoreMaterializedTuple(tts, bi_tuple, bi_len); + tts->tts_tableOid = RelationGetRelid(rel); + tts->tts_tid = *tid; + + scan->buffer = buffer; + return true; + } + } + + /* + * Tuple is visible. A committed-DELETED tuple can still be visible here: + * an RR/SERIALIZABLE reader whose snapshot predates the delete sees the + * row (FluxTupleVisibleToSnapshotDual already returned true because the + * deleter xid is not visible to this snapshot). + * FluxTupleToSlotWithOverflow refuses a DELETED tuple outright (it can't + * tell visible-to-this-snapshot from physically-dead), so serve the + * on-page image via FluxSlotStoreTuple -- the same virtual-slot store the + * sequential-scan path uses, which has no DELETED reject. This mirrors + * seqscan and fixes a lost-row/wrong-result under REPEATABLE READ via an + * index path. + */ + if (visible && (tuple_hdr->t_flags & FLUX_TUPLE_DELETED)) + { + FluxSlotStoreTuple(tts, tuple_hdr, ItemIdGetLength(itemid), buffer); + tts->tts_tableOid = RelationGetRelid(rel); + tts->tts_tid = *tid; + + /* + * Materialize while the page is still pinned so the returned varlena + * datums are copied into slot-owned memory and never dangle if the + * pinned buffer is released/reused before the caller consumes them. + */ + ExecMaterializeSlot(tts); + ReleaseBuffer(buffer); + return true; + } + + /* Tuple is visible - convert to slot (with overflow fetch) */ + if (FluxTupleToSlotWithOverflow(tuple_hdr, tts, rel)) + { + tts->tts_tableOid = RelationGetRelid(rel); + tts->tts_tid = *tid; + /* Slot is already marked valid by FluxTupleToSlotWithOverflow */ + } + else + { + ReleaseBuffer(buffer); + return false; + } + + /* + * Materialize while the page is still pinned: FluxTupleToSlotWithOverflow + * stored pointers into the buffer page for non-overflow columns. Copying + * them into slot-owned memory now means the returned datums do not depend + * on the buffer staying pinned, so a caller that reads them after the + * scan advances (releasing/reusing the buffer) cannot dangle. This + * matches the lifetime guarantee heap gives once a fetched slot is handed + * upward. + */ + ExecMaterializeSlot(tts); + ReleaseBuffer(buffer); + + return true; +} + +/* + * ------------------------------------------------------------------------ + * Tuple manipulation callbacks for FLUX AM + * ------------------------------------------------------------------------ + */ + +/* + * Fetch tuple at given TID + */ +static bool +flux_tuple_fetch_row_version(Relation relation, + ItemPointer tid, + Snapshot snapshot, + TupleTableSlot *slot) +{ + BlockNumber blkno = ItemPointerGetBlockNumber(tid); + OffsetNumber offnum = ItemPointerGetOffsetNumber(tid); + Buffer buffer; + Page page; + ItemId itemid; + FluxTupleHeader *tuple_hdr; + bool visible = false; + + /* Clear the slot */ + ExecClearTuple(slot); + + /* Read the page */ + buffer = ReadBuffer(relation, blkno); + LockBuffer(buffer, BUFFER_LOCK_SHARE); + + page = BufferGetPage(buffer); + + /* Validate offset */ + if (offnum < FirstOffsetNumber || offnum > PageGetMaxOffsetNumber(page)) + { + UnlockReleaseBuffer(buffer); + return false; + } + + /* Get the item */ + itemid = PageGetItemId(page, offnum); + + if (!ItemIdIsNormal(itemid)) + { + UnlockReleaseBuffer(buffer); + return false; + } + + /* Get tuple header */ + tuple_hdr = (FluxTupleHeader *) PageGetItem(page, itemid); + + /* + * Check visibility (heap-shaped xmin/xmax). Special case: SnapshotAny is + * used by DELETE RETURNING to fetch the just-deleted tuple, so we must + * allow deleted tuples in that case. + */ + if (snapshot && snapshot != SnapshotAny) + { + /* For normal snapshots, check visibility */ + visible = FluxTupleVisibleToSnapshotDual(tuple_hdr, snapshot, + RelationGetRelid(relation), + buffer); + + /* + * If the visibility function says invisible, the tuple is not + * visible. Do NOT additionally check FLUX_TUPLE_DELETED here ā€” the + * visibility function already consulted the sLog to determine if the + * delete is committed/in-progress/aborted. + */ + if (!visible) + { + UnlockReleaseBuffer(buffer); + return false; + } + } + else if (snapshot == SnapshotAny) + { + /* SnapshotAny: fetch any tuple, even if deleted (for RETURNING) */ + visible = true; + } + else + { + /* No snapshot means fetch unconditionally if not deleted */ + if (tuple_hdr->t_flags & FLUX_TUPLE_DELETED) + { + UnlockReleaseBuffer(buffer); + return false; + } + visible = true; + } + + if (!visible) + { + UnlockReleaseBuffer(buffer); + return false; + } + + /* + * Before-image substitution for committed in-place UPDATEs, mirroring the + * sequential-scan path. An index fetch that lands on a tuple updated in + * place by a transaction not visible to our MVCC snapshot must serve the + * before-image, not the on-page (new) value -- otherwise an UPDATE driven + * by this fetch (e.g. UPDATE ... WHERE pk = const) would recompute on top + * of a concurrent committed update and silently lose it. Serve from the + * per-relation UNDO fork (WS-PVS2). Visibility is ordinary heap-shaped + * xmin/xmax + CLOG + snapshot (XidInMVCCSnapshot in the callee): if the + * reader can see the updater the callee returns false and the on-page + * value is served. + */ + if ((tuple_hdr->t_flags & FLUX_TUPLE_UPDATED) && + !(tuple_hdr->t_flags & FLUX_TUPLE_UNCOMMITTED) && + snapshot != NULL && IsMVCCSnapshot(snapshot) && + FluxDirtyMapCheck(RelationGetRelid(relation), + ItemPointerGetBlockNumber(tid))) + { + char *bi_data = NULL; + int bi_len = 0; + + if (FluxReconstructVisibleVersion(relation, tid, + (const char *) tuple_hdr, + ItemIdGetLength(itemid), + snapshot, + &bi_data, &bi_len)) + { + FluxTupleHeader *bi_tuple = (FluxTupleHeader *) bi_data; + + FluxSlotStoreMaterializedTuple(slot, bi_tuple, bi_len); + slot->tts_tableOid = RelationGetRelid(relation); + slot->tts_tid = *tid; + LockBuffer(buffer, BUFFER_LOCK_UNLOCK); + ReleaseBuffer(buffer); + return true; + } + } + + /* Store tuple into slot with buffer pin for safe access */ + FluxSlotStoreTuple(slot, tuple_hdr, + ItemIdGetLength(itemid), buffer); + slot->tts_tableOid = RelationGetRelid(relation); + slot->tts_tid = *tid; + LockBuffer(buffer, BUFFER_LOCK_UNLOCK); + ReleaseBuffer(buffer); + + return true; +} + +/* + * Check if TID is valid for relation scan + */ +static bool +flux_tuple_tid_valid(TableScanDesc scan, ItemPointer tid) +{ + BlockNumber nblocks = RelationGetNumberOfBlocks(scan->rs_rd); + + return ItemPointerIsValid(tid) && + ItemPointerGetBlockNumber(tid) < nblocks; +} + +/* + * Get latest version of tuple (for updates) + */ +static void +flux_tuple_get_latest_tid(TableScanDesc scan, ItemPointer tid) +{ + /* FLUX uses in-place updates, so TID doesn't change */ + /* But we need to follow update chains if they exist */ + + BlockNumber block = ItemPointerGetBlockNumber(tid); + OffsetNumber offnum = ItemPointerGetOffsetNumber(tid); + Buffer buffer; + Page page; + ItemId itemid; + FluxTupleHeader *tuple_hdr; + + buffer = ReadBuffer(scan->rs_rd, block); + LockBuffer(buffer, BUFFER_LOCK_SHARE); + + page = BufferGetPage(buffer); + itemid = PageGetItemId(page, offnum); + + if (ItemIdIsNormal(itemid)) + { + tuple_hdr = (FluxTupleHeader *) PageGetItem(page, itemid); + + /* Follow update chain if needed */ + if (tuple_hdr->t_flags & FLUX_TUPLE_UPDATED) + { + ItemPointerCopy(&tuple_hdr->t_ctid, tid); + } + } + + LockBuffer(buffer, BUFFER_LOCK_UNLOCK); + ReleaseBuffer(buffer); +} + +/* + * Check if tuple satisfies snapshot + */ +static bool +flux_tuple_satisfies_snapshot(Relation rel, TupleTableSlot *slot, + Snapshot snapshot) +{ + Buffer buffer; + Page page; + ItemId itemid; + FluxTupleHeader *tuple_hdr; + BlockNumber blkno; + OffsetNumber offnum; + bool visible; + + /* + * Re-fetch the on-disk tuple header by TID so we can check the real + * commit timestamps and transaction status. The previous implementation + * used flux_tuple_from_slot() which fabricated a new tuple with current + * timestamps, making the visibility check meaningless. + */ + if (!ItemPointerIsValid(&slot->tts_tid)) + return false; + + blkno = ItemPointerGetBlockNumber(&slot->tts_tid); + offnum = ItemPointerGetOffsetNumber(&slot->tts_tid); + + buffer = ReadBuffer(rel, blkno); + LockBuffer(buffer, BUFFER_LOCK_SHARE); + page = BufferGetPage(buffer); + + if (offnum < FirstOffsetNumber || offnum > PageGetMaxOffsetNumber(page)) + { + UnlockReleaseBuffer(buffer); + return false; + } + + itemid = PageGetItemId(page, offnum); + if (!ItemIdIsNormal(itemid)) + { + UnlockReleaseBuffer(buffer); + return false; + } + + tuple_hdr = (FluxTupleHeader *) PageGetItem(page, itemid); + + /* + * For SNAPSHOT_DIRTY, we need to emulate HeapTupleSatisfiesDirty(): + * return the inserting xid and speculative token through the snapshot so + * that callers like _bt_check_unique / SpeculativeInsertionWait can + * properly wait for or detect speculative insertions. + * + * With the sLog migration, t_xmin/t_xmax no longer exist in the tuple + * header. We query the sLog for in-progress transaction state. + */ + if (snapshot->snapshot_type == SNAPSHOT_DIRTY) + { + ItemPointerData item_tid; + + snapshot->xmin = InvalidTransactionId; + snapshot->xmax = InvalidTransactionId; + snapshot->speculativeToken = 0; + + ItemPointerSet(&item_tid, blkno, offnum); + + /* + * Check if this tuple is uncommitted (inserted by a still-running + * transaction). + */ + if (tuple_hdr->t_flags & FLUX_TUPLE_UNCOMMITTED) + { + bool is_insert = false; + TransactionId dirty_xid; + + dirty_xid = SLogTupleGetDirtyXid(RelationGetRelid(rel), + &item_tid, &is_insert); + + if (TransactionIdIsValid(dirty_xid) && is_insert) + { + /* + * Another transaction is inserting this tuple. Report xmin + * for SpeculativeInsertionWait. + */ + snapshot->xmin = dirty_xid; + + if (tuple_hdr->t_flags & FLUX_TUPLE_SPECULATIVE) + { + snapshot->speculativeToken = + ItemPointerGetBlockNumber(&tuple_hdr->t_ctid); + } + + UnlockReleaseBuffer(buffer); + return true; /* visible for dirty snapshot purposes */ + } + else if (!TransactionIdIsValid(dirty_xid)) + { + /* + * No in-progress sLog entry found. Check for aborted insert + * with pending UNDO. + */ + if (SLogTupleHasAbortedEntry(RelationGetRelid(rel), + &item_tid)) + { + UnlockReleaseBuffer(buffer); + return false; + } + + /* + * The tuple's t_xmin is authoritative for the inserter; the + * sLog is consulted here for speculative-insert / command-id + * state on UNCOMMITTED tuples. + */ + { + SLogTupleOp my_entry; + int nfound; + TransactionId myxid = GetCurrentTransactionIdIfAny(); + + if (TransactionIdIsValid(myxid)) + { + nfound = SLogTupleLookupFiltered(RelationGetRelid(rel), + &item_tid, myxid, + &my_entry, 1); + if (nfound > 0 && + my_entry.op_type != SLOG_OP_DELETE) + { + /* Our own insert or in-place update */ + } + else if (nfound > 0) + { + /* Our own delete */ + UnlockReleaseBuffer(buffer); + return false; + } + else + { + /* + * Stale UNCOMMITTED flag. Clear and fall through. + */ + if (BufferIsValid(buffer)) + BufferSetHintBits16(&tuple_hdr->t_flags, + tuple_hdr->t_flags & ~FLUX_TUPLE_UNCOMMITTED, + buffer); + else + tuple_hdr->t_flags &= + ~FLUX_TUPLE_UNCOMMITTED; + } + } + else + { + /* + * No current transaction, stale flag. + */ + if (BufferIsValid(buffer)) + BufferSetHintBits16(&tuple_hdr->t_flags, + tuple_hdr->t_flags & ~FLUX_TUPLE_UNCOMMITTED, + buffer); + else + tuple_hdr->t_flags &= + ~FLUX_TUPLE_UNCOMMITTED; + } + } + } + } + + /* Tuple is committed (or ours). Check if deleted. */ + if (tuple_hdr->t_flags & FLUX_TUPLE_DELETED) + { + bool is_insert = false; + TransactionId del_xid; + + del_xid = SLogTupleGetDirtyXid(RelationGetRelid(rel), + &item_tid, &is_insert); + + if (TransactionIdIsValid(del_xid) && !is_insert) + { + /* Deleter still running */ + snapshot->xmax = del_xid; + UnlockReleaseBuffer(buffer); + return true; + } + else if (SLogTupleIsDeletedByMe(RelationGetRelid(rel), + &item_tid)) + { + /* We deleted it ourselves */ + UnlockReleaseBuffer(buffer); + return false; + } + else + { + /* + * Tuple is marked DELETED with no in-progress deleter -- + * deletion is committed. + */ + UnlockReleaseBuffer(buffer); + return false; + } + } + + /* + * Check if tuple has been superseded by an out-of-place update. For + * cross-page updates, t_ctid points to the new version's TID + * (different from this tuple's position). The old version is dead + * for index unique-check purposes. + * + * For in-place updates, t_ctid is self-referencing (points to the + * same TID as the tuple's own position). These tuples are live. + */ + if ((tuple_hdr->t_flags & FLUX_TUPLE_UPDATED) && + !ItemPointerEquals(&tuple_hdr->t_ctid, &item_tid)) + { + UnlockReleaseBuffer(buffer); + return false; + } + + /* Speculative but our own txn and not yet confirmed -- visible */ + UnlockReleaseBuffer(buffer); + return true; + } + + visible = FluxTupleVisibleToSnapshotDual(tuple_hdr, snapshot, + RelationGetRelid(rel), buffer); + + UnlockReleaseBuffer(buffer); + + return visible; +} + +/* + * Speculative tuple insertion for FLUX + * This is used for INSERT ... ON CONFLICT operations + */ +static void +flux_tuple_insert_speculative(Relation relation, TupleTableSlot *slot, + CommandId cid, uint32 options, + BulkInsertState bistate, uint32 specToken) +{ + FluxTuple tuple; + Buffer buf; + Page page; + Size tuple_size; + BlockNumber target_block; + OffsetNumber offnum; + FluxTupleHeader *tuple_hdr; + uint64 commit_ts; + FluxOverflowBuffers overflow_buffers; + int i; + + slot_getallattrs(slot); + + /* Create FLUX tuple from slot with overflow support */ + overflow_buffers.count = 0; + tuple = FluxFormTuple(RelationGetDescr(relation), + slot->tts_values, slot->tts_isnull, + relation, &overflow_buffers); + tuple_size = tuple->t_len; + + /* + * Get timestamp BEFORE entering critical section, as this may allocate + * memory. + */ + commit_ts = FluxGetCommitTimestamp(); + + /* + * Mark the tuple as uncommitted so that SNAPSHOT_DIRTY callers can detect + * in-progress insertions via the sLog. The sLog entry is registered + * after the tuple is placed on the page (below) so that the TID is valid. + */ + tuple->t_data->t_flags |= FLUX_TUPLE_UNCOMMITTED; + tuple->t_data->t_xmin = GetCurrentTransactionId(); /* subxid: heap-shaped, + * so savepoint rollback + * marks it aborted in + * CLOG */ + + /* + * Use the FSM to find a page with enough free space, or extend the + * relation with a properly initialized new page. + */ + target_block = FluxGetPageWithFreeSpace(relation, tuple_size); + if (target_block == InvalidBlockNumber) + { + /* Clean up overflow buffers before throwing error */ + for (i = 0; i < overflow_buffers.count; i++) + { + UnlockReleaseBuffer(overflow_buffers.buffers[i].buffer); + pfree(overflow_buffers.buffers[i].record_data); + } + FluxFreeTuple(tuple); + elog(ERROR, "FLUX failed to allocate page for speculative insertion"); + } + + buf = ReadBuffer(relation, target_block); + LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE); + page = BufferGetPage(buf); + + /* Verify page has sufficient space */ + if (PageGetFreeSpace(page) < tuple_size) + { + /* FSM was stale, update and retry */ + FluxRecordFreeSpace(relation, target_block, PageGetFreeSpace(page)); + UnlockReleaseBuffer(buf); + + target_block = FluxGetPageWithFreeSpace(relation, tuple_size); + if (target_block == InvalidBlockNumber) + { + /* Clean up overflow buffers before throwing error */ + for (i = 0; i < overflow_buffers.count; i++) + { + UnlockReleaseBuffer(overflow_buffers.buffers[i].buffer); + pfree(overflow_buffers.buffers[i].record_data); + } + FluxFreeTuple(tuple); + elog(ERROR, "FLUX failed to allocate page for speculative insertion after retry"); + } + + buf = ReadBuffer(relation, target_block); + LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE); + page = BufferGetPage(buf); + + if (PageGetFreeSpace(page) < tuple_size) + { + /* + * Both FSM attempts returned stale pages. Use P_NEW as final + * fallback ā€” extend the relation to get a guaranteed- empty + * page. Update FSM for accuracy on the bad page. + */ + FluxRecordFreeSpace(relation, BufferGetBlockNumber(buf), + PageGetFreeSpace(page)); + UnlockReleaseBuffer(buf); + + buf = ReadBuffer(relation, P_NEW); + LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE); + page = BufferGetPage(buf); + FluxInitPage(page, BufferGetPageSize(buf)); + } + } + + /* + * Try adding tuple before critical section. If the page is too full (FSM + * was optimistic about alignment/line-pointer overhead), extend the + * relation and use a fresh page. + */ + offnum = PageAddItem(page, tuple->t_data, tuple_size, + InvalidOffsetNumber, false, false); + if (offnum == InvalidOffsetNumber) + { + FluxRecordFreeSpace(relation, BufferGetBlockNumber(buf), + PageGetFreeSpace(page)); + UnlockReleaseBuffer(buf); + + buf = ReadBuffer(relation, P_NEW); + LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE); + page = BufferGetPage(buf); + FluxInitPage(page, BufferGetPageSize(buf)); + + offnum = PageAddItem(page, tuple->t_data, tuple_size, + InvalidOffsetNumber, false, false); + if (offnum == InvalidOffsetNumber) + elog(ERROR, "failed to add FLUX tuple to fresh page during " + "speculative insert (tuple_size=%zu)", tuple_size); + } + + /* NO EREPORT(ERROR) from here till changes are logged */ + START_CRIT_SECTION(); + + /* Mark as speculative insertion */ + tuple_hdr = (FluxTupleHeader *) PageGetItem(page, PageGetItemId(page, offnum)); + tuple_hdr->t_flags |= FLUX_TUPLE_SPECULATIVE; + tuple_hdr->t_commit_ts = 0; /* heap-shaped: t_xmax = Invalid */ + + /* + * Store the speculative insertion token in t_ctid, using the same + * encoding as heap: block number holds the token, offset is set to + * SpecTokenOffsetNumber so callers can distinguish a token from a real + * TID. + */ + ItemPointerSet(&tuple_hdr->t_ctid, specToken, SpecTokenOffsetNumber); + + /* Set TID in slot */ + ItemPointerSet(&slot->tts_tid, BufferGetBlockNumber(buf), offnum); + MarkBufferDirty(buf); + + /* WAL logging */ + if (RelationNeedsWAL(relation)) + { + XLogRecPtr recptr; + xl_flux_insert xlrec; + + xlrec.offnum = offnum; + xlrec.flags = FLUX_TUPLE_SPECULATIVE; + xlrec.tuple_len = (uint32) tuple_size; + xlrec.commit_ts = 0; /* heap-shaped: t_xmax = Invalid */ + + /* + * Force a full-page image and append the tuple body to the main data + * channel (matching FluxXLogInsert's layout, which flux_xlog_insert_ + * redo parses). The redo handler restores the tuple from the FPI and + * never reaches the PageAddItem path for these records; the body is + * carried so logical decoding and any future non-FPI replay see a + * well-formed record. Registering the body as block data instead + * (the historical behavior) left tuple_len uninitialized and made + * redo read past the record on a BLK_NEEDS_REDO replay. + */ + XLogBeginInsert(); + XLogRegisterBuffer(0, buf, REGBUF_STANDARD | REGBUF_FORCE_IMAGE); + XLogRegisterData((char *) &xlrec, sizeof(xl_flux_insert)); + XLogRegisterData((char *) tuple->t_data, tuple_size); + + /* Register overflow buffers if any */ + if (overflow_buffers.count > 0) + { + for (i = 0; i < overflow_buffers.count; i++) + { + FluxOverflowBuffer *ovb = &overflow_buffers.buffers[i]; + + /* Register the overflow buffer */ + XLogRegisterBuffer(i + 1, ovb->buffer, REGBUF_STANDARD); + + /* Register the overflow record data */ + XLogRegisterBufData(i + 1, ovb->record_data, ovb->record_len); + } + } + + recptr = XLogInsert(RM_FLUX_ID, XLOG_FLUX_INSERT); + PageSetLSN(page, recptr); + } + + END_CRIT_SECTION(); + + /* + * Register the speculative insertion in the sLog so that SNAPSHOT_DIRTY + * callers can find the inserting xid via SLogTupleGetDirtyXid(). + */ + FluxEnsureSLogCallbacks(); + SLogTupleInsert(RelationGetRelid(relation), &slot->tts_tid, + GetTopTransactionId(), SLOG_OP_INSERT, + GetCurrentSubTransactionId(), cid, commit_ts, + specToken, LockTupleNoKeyExclusive); + + /* Update FSM with remaining free space */ + FluxRecordFreeSpace(relation, BufferGetBlockNumber(buf), + PageGetFreeSpace(page)); + + UnlockReleaseBuffer(buf); + + /* Release overflow buffers, deduplicating shared buffers */ + for (i = 0; i < overflow_buffers.count; i++) + { + Buffer ovf_buf = overflow_buffers.buffers[i].buffer; + bool already_released = (ovf_buf == buf); + int j; + + for (j = 0; j < i && !already_released; j++) + { + if (overflow_buffers.buffers[j].buffer == ovf_buf) + already_released = true; + } + + if (!already_released) + UnlockReleaseBuffer(ovf_buf); + pfree(overflow_buffers.buffers[i].record_data); + } + + FluxFreeTuple(tuple); +} + +/* + * Complete speculative insertion for FLUX + */ +static void +flux_tuple_complete_speculative(Relation relation, TupleTableSlot *slot, + uint32 specToken, bool succeeded) +{ + ItemPointer tid = &slot->tts_tid; + Buffer buf; + Page page; + ItemId itemid; + FluxTupleHeader *tuple_hdr; + + buf = ReadBuffer(relation, ItemPointerGetBlockNumber(tid)); + LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE); + page = BufferGetPage(buf); + + itemid = PageGetItemId(page, ItemPointerGetOffsetNumber(tid)); + if (!ItemIdIsNormal(itemid)) + { + UnlockReleaseBuffer(buf); + return; + } + + tuple_hdr = (FluxTupleHeader *) PageGetItem(page, itemid); + + if (succeeded) + { + /* + * Speculative insertion succeeded. Clear the speculative flag and + * restore t_ctid to point to the tuple itself (removing the + * speculative token), mirroring heap_finish_speculative(). + * + * Dirty-read hole fix: this used to ALSO clear FLUX_TUPLE_UNCOMMITTED + * here, i.e. well before the inserting transaction actually commits. + * That made the row immediately visible to every concurrent snapshot + * the instant the speculative insert was confirmed (INSERT ... ON + * CONFLICT DO NOTHING/UPDATE), a straight dirty read -- HEAP never + * exposes an uncommitted speculative tuple this way. The speculative + * INSERT already registered a real shared sLog entry (SLOG_OP_INSERT, + * see flux_tuple_insert_speculative), so leaving + * FLUX_TUPLE_UNCOMMITTED set here is safe and correct: the existing + * commit-time machinery (FluxClearUncommittedFlags / + * flux_stamp_tuple_committed, driven by the tracked sLog key) clears + * it and clears the UNCOMMITTED hint at actual transaction commit, + * exactly like a plain INSERT. + */ + START_CRIT_SECTION(); + + tuple_hdr->t_flags &= ~FLUX_TUPLE_SPECULATIVE; + ItemPointerSet(&tuple_hdr->t_ctid, + ItemPointerGetBlockNumber(tid), + ItemPointerGetOffsetNumber(tid)); + + MarkBufferDirty(buf); + + /* WAL-log the confirmation */ + if (RelationNeedsWAL(relation)) + { + XLogRecPtr recptr; + xl_flux_insert xlrec; + + xlrec.offnum = ItemPointerGetOffsetNumber(tid); + xlrec.flags = 0; /* cleared SPECULATIVE */ + xlrec.tuple_len = 0; /* no body: confirm only flips flags */ + xlrec.commit_ts = tuple_hdr->t_commit_ts; + + /* + * Confirmation does not add a tuple; it clears the speculative + * flag on a tuple already present on the page. Force a full-page + * image so redo restores the (already-updated) page rather than + * re-adding the tuple via PageAddItem. tuple_len == 0 keeps the + * record body-less, which flux_xlog_insert_redo treats as + * "nothing to add" on the (unreachable, given the forced FPI) + * non-FPI path. + */ + XLogBeginInsert(); + XLogRegisterBuffer(0, buf, REGBUF_STANDARD | REGBUF_FORCE_IMAGE); + XLogRegisterData((char *) &xlrec, sizeof(xl_flux_insert)); + + recptr = XLogInsert(RM_FLUX_ID, XLOG_FLUX_INSERT); + PageSetLSN(page, recptr); + } + + END_CRIT_SECTION(); + } + else + { + TransactionId abort_xid; + + /* + * Speculative abort: mark the tuple deleted by our own XID (heap- + * shaped t_xmax). Since this transaction will abort, the tuple's + * xmin also resolves to aborted, so it is invisible either way; we + * stamp t_xmax for a clean tombstone. + */ + abort_xid = GetTopTransactionId(); + + START_CRIT_SECTION(); + + tuple_hdr->t_flags |= FLUX_TUPLE_DELETED; + FluxTupleSetXmax(tuple_hdr, abort_xid); + + MarkBufferDirty(buf); + + /* WAL log the speculative abort as a delete */ + if (RelationNeedsWAL(relation)) + { + XLogRecPtr recptr; + xl_flux_delete xlrec; + + xlrec.offnum = ItemPointerGetOffsetNumber(tid); + xlrec.flags = 0; + xlrec.tuple_len = ItemIdGetLength(itemid); + xlrec.commit_ts = (uint64) abort_xid; + + XLogBeginInsert(); + XLogRegisterData((char *) &xlrec, sizeof(xl_flux_delete)); + XLogRegisterBuffer(0, buf, REGBUF_STANDARD); + XLogRegisterBufData(0, (char *) tuple_hdr, ItemIdGetLength(itemid)); + + recptr = XLogInsert(RM_FLUX_ID, XLOG_FLUX_DELETE); + PageSetLSN(page, recptr); + } + + END_CRIT_SECTION(); + } + + UnlockReleaseBuffer(buf); +} + +/* + * Lock a tuple in FLUX table + */ +static TM_Result +flux_tuple_lock(Relation relation, ItemPointer tid, Snapshot snapshot, + TupleTableSlot *slot, CommandId cid, LockTupleMode mode, + LockWaitPolicy wait_policy, uint8 flags, + TM_FailureData *tmfd) +{ + Buffer buf; + Page page; + ItemId itemid; + FluxTupleHeader *tuple_hdr; + TM_Result result = TM_Ok; + TransactionId current_xid; + bool have_tuple_lock = false; + + /* + * Get transaction XID BEFORE entering critical section, as this may + * allocate memory. + */ + current_xid = GetTopTransactionId(); + +reacquire: + buf = ReadBuffer(relation, ItemPointerGetBlockNumber(tid)); + LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE); + page = BufferGetPage(buf); + + itemid = PageGetItemId(page, ItemPointerGetOffsetNumber(tid)); + if (!ItemIdIsNormal(itemid)) + { + UnlockReleaseBuffer(buf); + return TM_Invisible; + } + + tuple_hdr = (FluxTupleHeader *) PageGetItem(page, itemid); + + /* Check if tuple is deleted */ + if (tuple_hdr->t_flags & FLUX_TUPLE_DELETED) + { + if (tmfd) + { + tmfd->ctid = *tid; + tmfd->xmax = InvalidTransactionId; + tmfd->cmax = InvalidCommandId; + } + result = TM_Deleted; + goto out_unlock; + } + + /* + * Check visibility using timestamp-based MVCC and handle concurrent + * modifications. Same pattern as UPDATE/DELETE: distinguish truly + * invisible tuples from concurrent modifications. + */ + if (!FluxTupleVisibleToSnapshotDual(tuple_hdr, snapshot, + RelationGetRelid(relation), + buf)) + { + TransactionId dirty_xid; + bool is_insert_entry; + + /* + * Writer-only probe: this branch waits on an in-progress + * INSERT/UPDATE/DELETE writer. Lock-only markers (LOCK_SHARE/ + * LOCK_EXCL) are handled by the dedicated lock-conflict check below + * (SLogTupleHasLockConflict), which serializes lockers via the + * heavyweight LOCKTAG_TUPLE lock. Waiting-as-reader on a pure + * locker's xid here lets two lockers each XactLockTableWait on the + * other and deadlock. + */ + dirty_xid = SLogTupleGetDirtyWriterXid(RelationGetRelid(relation), + tid, &is_insert_entry); + + if (TransactionIdIsValid(dirty_xid) && is_insert_entry) + { + /* Another txn's in-progress INSERT - truly invisible */ + if (tmfd) + { + tmfd->ctid = *tid; + tmfd->xmax = dirty_xid; + tmfd->cmax = InvalidCommandId; + } + result = TM_Invisible; + goto out_unlock; + } + + if (TransactionIdIsValid(dirty_xid) && !is_insert_entry) + { + /* Another txn's in-progress UPDATE/DELETE - wait and retry */ + if (wait_policy == LockWaitBlock) + { + TransactionId wait_xid = dirty_xid; + + UnlockReleaseBuffer(buf); + if (!have_tuple_lock) + { + FluxLockTuple(relation, tid, mode, true, + &have_tuple_lock); + } + XactLockTableWait(wait_xid, relation, tid, XLTW_Lock); + goto reacquire; + } + else if (wait_policy == LockWaitError) + { + if (tmfd) + { + tmfd->ctid = *tid; + tmfd->xmax = dirty_xid; + tmfd->cmax = InvalidCommandId; + } + result = TM_WouldBlock; + goto out_unlock; + } + else /* LockWaitSkip */ + { + result = TM_WouldBlock; + goto out_unlock; + } + } + + /* + * No sLog entry - committed modification after our snapshot. + * + * If TUPLE_LOCK_FLAG_FIND_LAST_VERSION is set, the caller wants us to + * follow the update chain and lock the latest version. In FLUX with + * in-place updates, the current tuple IS the latest version (same + * TID), so fall through to lock it and set tmfd->traversed to trigger + * EPQ re-evaluation. + * + * Otherwise, report TM_Updated so the executor can handle it. + */ + if (flags & TUPLE_LOCK_FLAG_FIND_LAST_VERSION) + { + if (tmfd) + { + tmfd->ctid = *tid; + tmfd->xmax = InvalidTransactionId; + tmfd->cmax = InvalidCommandId; + tmfd->traversed = true; + } + /* Fall through to lock the current (latest) version */ + } + else + { + if (tmfd) + { + tmfd->ctid = *tid; + tmfd->xmax = InvalidTransactionId; + tmfd->cmax = InvalidCommandId; + tmfd->traversed = false; + } + result = TM_Updated; + goto out_unlock; + } + } + + /* + * Check for lock conflicts using the sLog. The sLog tracks all + * in-progress lock/delete/update operations, replacing the old + * t_xmax/MultiXact-based scheme. + */ + { + SLogOpType requested_lock; + TransactionId xwait = InvalidTransactionId; + + requested_lock = (mode == LockTupleKeyShare || + mode == LockTupleShare) + ? SLOG_OP_LOCK_SHARE : SLOG_OP_LOCK_EXCL; + + if (SLogTupleGetLockConflictXid(RelationGetRelid(relation), tid, + current_xid, requested_lock, + &xwait)) + { + /* There's a conflict - check wait policy */ + if (wait_policy == LockWaitError) + { + if (tmfd) + { + tmfd->ctid = *tid; + tmfd->xmax = InvalidTransactionId; + tmfd->cmax = InvalidCommandId; + } + result = TM_WouldBlock; + goto out_unlock; + } + else if (wait_policy == LockWaitSkip) + { + result = TM_WouldBlock; + goto out_unlock; + } + else /* LockWaitBlock */ + { + /* + * Wait on the specific conflicting transaction identified by + * SLogTupleGetLockConflictXid -- never on an arbitrary + * in-progress peer, which could be a compatible locker queued + * behind us and would form a spurious mutual-wait cycle. + */ + + /* Release buffer and wait */ + UnlockReleaseBuffer(buf); + + if (TransactionIdIsValid(xwait)) + { + /* Acquire tuple-level lock to wait */ + if (!have_tuple_lock) + { + FluxLockTuple(relation, tid, mode, true, + &have_tuple_lock); + } + + /* Wait for the conflicting transaction */ + XactLockTableWait(xwait, relation, tid, XLTW_Lock); + } + + /* Re-acquire buffer and retry */ + goto reacquire; + } + } + } + + /* + * Lock succeeded. Set traversed for FIND_LAST_VERSION callers. FLUX uses + * in-place updates, so the current tuple IS the latest version ā€” the + * update chain was trivially "followed." + */ + if (tmfd) + { + tmfd->traversed = (flags & TUPLE_LOCK_FLAG_FIND_LAST_VERSION) != 0; + tmfd->ctid = *tid; + tmfd->xmax = InvalidTransactionId; + tmfd->cmax = InvalidCommandId; + } + + tuple_hdr->t_flags |= FLUX_TUPLE_LOCKED; + + START_CRIT_SECTION(); + + MarkBufferDirty(buf); + + /* Log the lock operation */ + if (RelationNeedsWAL(relation)) + { + XLogRecPtr recptr; + xl_flux_lock xlrec; + + xlrec.offnum = ItemPointerGetOffsetNumber(tid); + xlrec.flags = 0; + xlrec.infomask = tuple_hdr->t_infomask; + xlrec.lock_mode = (uint8) mode; + + XLogBeginInsert(); + XLogRegisterData((char *) &xlrec, sizeof(xl_flux_lock)); + XLogRegisterBuffer(0, buf, REGBUF_STANDARD); + + recptr = XLogInsert(RM_FLUX_ID, XLOG_FLUX_LOCK); + PageSetLSN(page, recptr); + } + + END_CRIT_SECTION(); + + /* + * Populate the slot with the locked tuple's data. This must happen after + * END_CRIT_SECTION (since overflow fetch may do I/O and ereport). We must + * unlock the buffer (but keep the pin) before calling + * FluxTupleToSlotWithOverflow because it may fetch overflow data, and if + * that overflow is on the same page, it would try to lock an + * already-locked buffer causing an assertion failure. + * + * FK constraint triggers and other callers of table_tuple_lock() expect + * the slot to contain valid tuple data. + */ + if (slot && result == TM_Ok) + { + /* Unlock buffer but keep pin for slot materialization */ + LockBuffer(buf, BUFFER_LOCK_UNLOCK); + + /* + * Register the lock in the sLog AFTER releasing the buffer lock to + * avoid deadlocks with SLogTupleGetDirtyXid's slow path. + */ + { + SLogOpType lock_op; + + lock_op = (mode == LockTupleKeyShare || mode == LockTupleShare) + ? SLOG_OP_LOCK_SHARE : SLOG_OP_LOCK_EXCL; + + /* + * Record the precise LockTupleMode alongside the coarse lock_op + * so a concurrent updater can apply the real heavyweight conflict + * matrix: FOR KEY SHARE (AccessShareLock) stays compatible with a + * NoKeyExclusive UPDATE, while FOR SHARE/FOR UPDATE correctly + * block it. See SLogTupleGetWriteConflictXid. + */ + FluxEnsureSLogCallbacks(); + SLogTupleInsert(RelationGetRelid(relation), tid, + current_xid, lock_op, + GetCurrentSubTransactionId(), cid, 0, 0, mode); + + /* Mark this block dirty for the scan-path sLog bypass */ + FluxDirtyMapMark(RelationGetRelid(relation), + ItemPointerGetBlockNumber(tid)); + } + + if (!FluxTupleToSlotWithOverflow(tuple_hdr, slot, relation)) + { + /* + * Conversion failed (e.g., tuple was concurrently deleted). + * Return TM_Deleted rather than leaving the slot empty. + */ + ReleaseBuffer(buf); + if (have_tuple_lock) + FluxUnlockTuple(relation, tid, mode); + return TM_Deleted; + } + slot->tts_tid = *tid; + slot->tts_tableOid = RelationGetRelid(relation); + + /* + * FluxTupleToSlotWithOverflow leaves the slot virtual with + * tts_values[] pointing into the page buffer (and records no pin in + * the slot). Deep-copy those pass-by-reference datums into + * slot-owned memory BEFORE dropping the pin, so a returned varlena + * (e.g. numeric via UPDATE ... RETURNING / FK / EPQ locking) does not + * alias a buffer that is about to be unpinned and reused. This + * matches heap's slot discipline (ExecStorePinnedBufferHeapTuple + * keeps the pin in the slot). + */ + ExecMaterializeSlot(slot); + + /* Release the buffer pin now that slot is materialized */ + ReleaseBuffer(buf); + + /* Release tuple-level lock if we acquired it */ + if (have_tuple_lock) + FluxUnlockTuple(relation, tid, mode); + + return result; + } + +out_unlock: + UnlockReleaseBuffer(buf); + + /* Release tuple-level lock if we acquired it */ + if (have_tuple_lock) + FluxUnlockTuple(relation, tid, mode); + + return result; +} + +/* + * Nontransactional truncate for FLUX relation + * + * This is called for TRUNCATE operations. We use RelationTruncate which + * properly handles all forks (main, FSM, VM), WAL logging, shared buffer + * invalidation, and cache coherency. + */ +static void +flux_relation_nontransactional_truncate(Relation rel) +{ + RelationTruncate(rel, 0); +} + +/* + * Set new filelocator for FLUX relation + */ +static void +flux_relation_set_new_filelocator(Relation rel, + const RelFileLocator *newrlocator, + char persistence, + TransactionId *freezeXid, + MultiXactId *minmulti) +{ + SMgrRelation srel; + + /* Set freeze XID to current transaction minimum */ + *freezeXid = RecentXmin; + + /* Set minimum multixact ID */ + *minmulti = GetOldestMultiXactId(); + + /* Create the storage file (empty, no blocks yet) */ + srel = RelationCreateStorage(*newrlocator, persistence, true); + + /* WAL-log the file creation */ + if (persistence == RELPERSISTENCE_PERMANENT) + log_smgrcreate(newrlocator, MAIN_FORKNUM); + + /* + * Note: We do not initialize block 0 here. Block 0 will be created + * on-demand during the first scan or insert operation via + * FluxGetPageWithFreeSpace() or the scan's RBM_ZERO_AND_LOCK logic. + */ + + /* Set up init fork for unlogged tables if needed */ + if (persistence == RELPERSISTENCE_UNLOGGED) + { + Assert(rel->rd_rel->relkind == RELKIND_RELATION || + rel->rd_rel->relkind == RELKIND_TOASTVALUE); + smgrcreate(srel, INIT_FORKNUM, false); + log_smgrcreate(newrlocator, INIT_FORKNUM); + } + + smgrclose(srel); + + /* + * Initialize the per-relation UNDO fork for permanent and unlogged + * relations. This creates the UNDO fork file and writes the initial + * metapage so that subsequent INSERT/UPDATE/DELETE operations can reserve + * UNDO space via RelUndoReserve(). Without this, the smgrexists() guards + * in the DML paths skip all UNDO record emission. + * + * Temp tables are session-private and skip the fork. + * + * RelUndoInitRelation() targets RelationGetSmgr(rel) == rel->rd_locator, + * but during TRUNCATE the caller passes the new locator via newrlocator + * while rel->rd_locator still holds the old value (updated only after we + * return). Temporarily swap rd_locator to the new locator, init, then + * restore; the caller overwrites rd_locator regardless. + */ + if (persistence == RELPERSISTENCE_PERMANENT || + persistence == RELPERSISTENCE_UNLOGGED) + { + RelFileLocator saved_locator = rel->rd_locator; + + rel->rd_locator = *newrlocator; + RelationCloseSmgr(rel); + RelUndoInitRelation(rel); + RelationCloseSmgr(rel); + rel->rd_locator = saved_locator; + } +} + +/* + * Check whether table tuples referenced by index entries are dead. + * + * This is called by index AMs during index tuple deletion (both simple + * deletion during VACUUM and bottom-up deletion during retail inserts). + * The index AM passes a list of TIDs and we check each one's liveness. + * We set knowndeletable=true for entries whose table tuples are dead, + * allowing the index AM to remove its entries. + * + * IMPORTANT: This function must NEVER modify table data. It only reads + * tuple headers to check visibility status. + * + * Modeled on heap_index_delete_tuples() but simplified for FLUX's + * timestamp-based MVCC. + */ +static TransactionId +flux_index_delete_tuples(Relation rel, TM_IndexDeleteOp *delstate) +{ + TransactionId snapshotConflictHorizon = InvalidTransactionId; + BlockNumber blkno = InvalidBlockNumber; + Buffer buf = InvalidBuffer; + Page page = NULL; + OffsetNumber maxoff = InvalidOffsetNumber; + int finalndeltids = 0; + + Assert(delstate->ndeltids > 0); + + /* Iterate over deltids, determine which are deletable */ + for (int i = 0; i < delstate->ndeltids; i++) + { + TM_IndexDelete *ideltid = &delstate->deltids[i]; + TM_IndexStatus *istatus = delstate->status + ideltid->id; + ItemPointer htid = &ideltid->tid; + OffsetNumber offnum; + + /* + * Read buffer for this block if we haven't already. Avoid refetching + * if it's the same block as the previous entry. + */ + if (blkno == InvalidBlockNumber || + ItemPointerGetBlockNumber(htid) != blkno) + { + if (BufferIsValid(buf)) + UnlockReleaseBuffer(buf); + + blkno = ItemPointerGetBlockNumber(htid); + buf = ReadBuffer(rel, blkno); + LockBuffer(buf, BUFFER_LOCK_SHARE); + page = BufferGetPage(buf); + maxoff = PageGetMaxOffsetNumber(page); + } + + offnum = ItemPointerGetOffsetNumber(htid); + + /* Sanity check: offset must be valid */ + if (offnum < FirstOffsetNumber || offnum > maxoff) + { + /* + * Index entry points to invalid offset. Mark as deletable to + * clean up the corruption. + */ + istatus->knowndeletable = true; + finalndeltids = i + 1; + continue; + } + + /* Already known to be deletable by the index AM? */ + if (istatus->knowndeletable) + { + Assert(!delstate->bottomup && !istatus->promising); + finalndeltids = i + 1; + continue; + } + + { + ItemId lp = PageGetItemId(page, offnum); + + if (!ItemIdIsNormal(lp)) + { + /* + * LP_DEAD, LP_UNUSED, or LP_REDIRECT: the tuple is gone. The + * index entry can be removed. + */ + istatus->knowndeletable = true; + } + else + { + FluxTupleHeader *tuple_hdr; + + tuple_hdr = (FluxTupleHeader *) PageGetItem(page, lp); + + /* + * For FLUX, a tuple is vacuumable (and its index entry + * deletable) if it is deleted AND old enough that no snapshot + * can see it. + */ + if (tuple_hdr->t_flags & FLUX_TUPLE_DELETED) + { + if (FluxTupleDeadToAll(tuple_hdr, FluxGetOldestXminHorizon(rel))) + { + istatus->knowndeletable = true; + } + else + { + /* Recently dead -- cannot delete index entry yet */ + continue; + } + } + else + { + /* Live tuple -- cannot delete index entry */ + continue; + } + } + } + + /* Track progress for bottom-up deletion */ + if (delstate->bottomup && istatus->knowndeletable) + { + int actualfreespace = 0; + + actualfreespace += istatus->freespace; + if (actualfreespace >= delstate->bottomupfreespace) + { + /* Met the space target -- stop early */ + finalndeltids = i + 1; + break; + } + } + + finalndeltids = i + 1; + } + + if (BufferIsValid(buf)) + UnlockReleaseBuffer(buf); + + /* + * Shrink deltids array to exclude non-deletable entries at the end. + */ + Assert(finalndeltids > 0 || delstate->bottomup); + delstate->ndeltids = finalndeltids; + + return snapshotConflictHorizon; +} + +/* + * Copy data for FLUX relation (used by ALTER TABLE SET ACCESS METHOD, etc.) + * + * This performs a block-level copy of all storage forks from the old + * relation files to new ones. Since we copy directly without examining + * shared buffers, we must flush any dirty pages first. The old physical + * files are scheduled for deletion. + */ +static void +flux_relation_copy_data(Relation rel, const RelFileLocator *newrlocator) +{ + SMgrRelation dstrel; + + /* + * Since we copy the file directly without looking at the shared buffers, + * we'd better first flush out any pages of the source relation that are + * in shared buffers. We assume no new changes will be made while we are + * holding exclusive lock on the rel. + */ + FlushRelationBuffers(rel); + + /* + * Create and copy all forks of the relation, and schedule unlinking of + * old physical files. + * + * NOTE: any conflict in relfilenumber value will be caught in + * RelationCreateStorage(). + */ + dstrel = RelationCreateStorage(*newrlocator, rel->rd_rel->relpersistence, + true); + + /* Copy main fork */ + RelationCopyStorage(RelationGetSmgr(rel), dstrel, MAIN_FORKNUM, + rel->rd_rel->relpersistence); + + /* Copy any extra forks that exist (FSM, etc.) */ + for (ForkNumber forkNum = MAIN_FORKNUM + 1; + forkNum <= MAX_FORKNUM; forkNum++) + { + if (smgrexists(RelationGetSmgr(rel), forkNum)) + { + smgrcreate(dstrel, forkNum, false); + + /* + * WAL log creation if the relation is persistent, or this is the + * init fork of an unlogged relation. + */ + if (RelationIsPermanent(rel) || + (rel->rd_rel->relpersistence == RELPERSISTENCE_UNLOGGED && + forkNum == INIT_FORKNUM)) + log_smgrcreate(newrlocator, forkNum); + RelationCopyStorage(RelationGetSmgr(rel), dstrel, forkNum, + rel->rd_rel->relpersistence); + } + } + + /* Drop old relation storage, and close new one */ + RelationDropStorage(rel); + smgrclose(dstrel); +} + +/* + * Copy data for cluster operation + * + * This is called by CLUSTER and VACUUM FULL to copy tuples from the old + * table to the new one, optionally reordering by an index. We scan the + * old table using SnapshotAny and perform our own MVCC visibility checks + * to decide which tuples to keep, which to discard as dead, and which + * are recently dead. + * + * For FLUX, visibility is determined by the tuple's timestamp-based MVCC + * flags (deleted flag, commit timestamp, etc.) rather than heap-style xmin/xmax. + */ +static void +flux_relation_copy_for_cluster(Relation OldTable, Relation NewTable, + Relation OldIndex, bool use_sort, + TransactionId OldestXmin, + Snapshot snapshot, + TransactionId *xid_cutoff, + MultiXactId *multi_cutoff, + double *num_tuples, + double *tups_vacuumed, + double *tups_recently_dead) +{ + TableScanDesc tableScan; + IndexScanDesc indexScan; + TupleTableSlot *slot; + CommandId mycid = GetCurrentCommandId(true); + double live_tuples = 0; + double dead_tuples = 0; + double recent_dead = 0; + + /* Initialize return values */ + *xid_cutoff = InvalidTransactionId; + *multi_cutoff = InvalidMultiXactId; + *num_tuples = 0; + *tups_vacuumed = 0; + *tups_recently_dead = 0; + + /* + * Valid smgr_targblock implies something already wrote to the relation. + * This may be harmless, but this function hasn't planned for it. + */ + Assert(RelationGetTargetBlock(NewTable) == InvalidBlockNumber); + + /* + * Set up the scan. If we have an index and are not doing a sort, use an + * index scan to get tuples in index order. Otherwise do a sequential scan + * (and optionally sort afterward). + */ + if (OldIndex != NULL && !use_sort) + { + pgstat_progress_update_param(PROGRESS_REPACK_PHASE, + PROGRESS_REPACK_PHASE_INDEX_SCAN_HEAP); + + tableScan = NULL; + indexScan = index_beginscan(OldTable, OldIndex, SnapshotAny, NULL, + 0, 0, 0); + index_rescan(indexScan, NULL, 0, NULL, 0); + } + else + { + pgstat_progress_update_param(PROGRESS_REPACK_PHASE, + PROGRESS_REPACK_PHASE_SEQ_SCAN_HEAP); + + tableScan = table_beginscan(OldTable, SnapshotAny, 0, NULL, 0); + indexScan = NULL; + } + + slot = table_slot_create(OldTable, NULL); + + /* + * Scan through the old table. For each tuple, check visibility using + * FLUX's timestamp-based MVCC and either copy it to the new table or skip + * it. + */ + for (;;) + { + bool isdead = false; + bool is_tombstone = false; + uint64 delete_ts = 0; + + CHECK_FOR_INTERRUPTS(); + + if (indexScan != NULL) + { + if (!index_getnext_slot(indexScan, ForwardScanDirection, slot)) + break; + } + else + { + if (!table_scan_getnextslot(tableScan, ForwardScanDirection, slot)) + break; + } + + /* + * For FLUX, check tuple visibility using our page-level access. Read + * the tuple header from the page to check MVCC flags. + */ + { + Buffer buf; + Page page; + ItemId itemid; + FluxTupleHeader *tuple_hdr; + BlockNumber blkno = ItemPointerGetBlockNumber(&slot->tts_tid); + OffsetNumber offnum = ItemPointerGetOffsetNumber(&slot->tts_tid); + + buf = ReadBuffer(OldTable, blkno); + LockBuffer(buf, BUFFER_LOCK_SHARE); + page = BufferGetPage(buf); + + itemid = PageGetItemId(page, offnum); + if (!ItemIdIsNormal(itemid)) + { + /* Item pointer is dead or unused -- skip */ + UnlockReleaseBuffer(buf); + dead_tuples++; + continue; + } + + tuple_hdr = (FluxTupleHeader *) PageGetItem(page, itemid); + + if (tuple_hdr->t_flags & FLUX_TUPLE_UPDATED) + { + /* + * Distinguish cross-page (out-of-place) updates from in-place + * updates. Cross-page updates have t_ctid pointing to a + * different TID (the new version's location). In-place + * updates have t_ctid pointing to self (same TID). + * + * Only cross-page old versions are dead; in-place updated + * tuples contain the current data and are live. + */ + ItemPointerData self_tid; + + ItemPointerSet(&self_tid, blkno, offnum); + if (!ItemPointerEquals(&tuple_hdr->t_ctid, &self_tid)) + isdead = true; /* Cross-page: old version is dead */ + /* else: in-place update, tuple is live ā€” fall through */ + } + + if (!isdead && (tuple_hdr->t_flags & FLUX_TUPLE_DELETED)) + { + /* + * Tuple has been deleted. Check whether it's old enough to be + * truly dead vs recently dead (still needed for MVCC + * snapshots). XID-horizon gate (FluxTupleDeadToAll), not the + * removed HLC timestamp horizon: the on-page word now packs + * the XID-based t_xmax, not a wall-clock commit timestamp. + */ + if (FluxTupleDeadToAll(tuple_hdr, FluxGetOldestXminHorizon(OldTable))) + { + /* Definitely dead -- can discard */ + isdead = true; + } + else + { + /* + * Recently dead -- still needed by some snapshots. Copy + * it to the new relation as a tombstone, preserving its + * original delete timestamp so old-snapshot readers still + * see the pre-delete version (matching heap's + * RECENTLY_DEAD retention during cluster rewrite). + */ + recent_dead++; + isdead = false; + is_tombstone = true; + delete_ts = tuple_hdr->t_commit_ts; + } + } + + UnlockReleaseBuffer(buf); + } + + if (isdead) + { + dead_tuples++; + continue; + } + + /* Live or recently-dead tuple -- copy to new table */ + table_tuple_insert(NewTable, slot, mycid, 0, NULL); + + if (is_tombstone) + { + /* + * The row was copied as a fresh, live INSERT. Rewrite it in the + * new relation as a deleted tombstone carrying its original + * delete marker, and drop the INSERT's local sLog tracking so + * that commit-time bookkeeping does not clobber the tuple with + * this rewrite transaction's commit state (which would resurrect + * the row for readers whose snapshots predate the rewrite + * commit). + */ + Buffer nbuf; + Page npage; + ItemId nitemid; + FluxTupleHeader *ntuple_hdr; + FluxPageOpaque nphdr; + BlockNumber nblkno = ItemPointerGetBlockNumber(&slot->tts_tid); + OffsetNumber noffnum = ItemPointerGetOffsetNumber(&slot->tts_tid); + + SLogTupleUntrackLocalOnly(RelationGetRelid(NewTable), + &slot->tts_tid); + + nbuf = ReadBuffer(NewTable, nblkno); + LockBuffer(nbuf, BUFFER_LOCK_EXCLUSIVE); + npage = BufferGetPage(nbuf); + nitemid = PageGetItemId(npage, noffnum); + + START_CRIT_SECTION(); + + ntuple_hdr = (FluxTupleHeader *) PageGetItem(npage, nitemid); + ntuple_hdr->t_flags |= FLUX_TUPLE_DELETED; + ntuple_hdr->t_flags &= ~FLUX_TUPLE_UNCOMMITTED; + ntuple_hdr->t_commit_ts = delete_ts; + + nphdr = FluxPageGetOpaque(npage); + FluxPageSetCommitTs(nphdr, + Max(FluxPageGetCommitTs(nphdr), delete_ts)); + FluxPageSetFlag(nphdr, FLUX_PAGE_DEFRAG_NEEDED); + + MarkBufferDirty(nbuf); + + if (RelationNeedsWAL(NewTable)) + { + XLogRecPtr recptr; + xl_flux_delete xlrec; + + xlrec.offnum = noffnum; + xlrec.flags = 0; + xlrec.tuple_len = ItemIdGetLength(nitemid); + xlrec.commit_ts = delete_ts; + + XLogBeginInsert(); + XLogRegisterData((char *) &xlrec, sizeof(xl_flux_delete)); + XLogRegisterBuffer(0, nbuf, REGBUF_STANDARD); + + recptr = XLogInsert(RM_FLUX_ID, XLOG_FLUX_DELETE); + PageSetLSN(npage, recptr); + } + + END_CRIT_SECTION(); + + UnlockReleaseBuffer(nbuf); + } + + live_tuples++; + + pgstat_progress_update_param(PROGRESS_REPACK_HEAP_TUPLES_SCANNED, + live_tuples + dead_tuples + recent_dead); + } + + /* Clean up scan resources */ + if (indexScan != NULL) + index_endscan(indexScan); + if (tableScan != NULL) + table_endscan(tableScan); + + ExecDropSingleTupleTableSlot(slot); + + /* Return statistics to caller */ + *num_tuples = live_tuples; + *tups_vacuumed = dead_tuples; + *tups_recently_dead = recent_dead; +} + +/* + * Build range scan for index creation + * + * Scans the FLUX table and feeds tuples to the index AM's callback for + * index building. Handles partial indexes, expression indexes, uniqueness + * checking, concurrent index builds, and proper visibility classification. + * + * Modeled on heapam_index_build_range_scan(). + */ +static double +flux_index_build_range_scan(Relation tablerel, Relation indexrel, + IndexInfo *indexInfo, bool allow_sync, + bool anyvisible, bool progress, + BlockNumber start_blockno, BlockNumber numblocks, + IndexBuildCallback callback, void *callback_state, + TableScanDesc scan) +{ + double reltuples = 0; + bool checking_uniqueness PG_USED_FOR_ASSERTS_ONLY; + Datum values[INDEX_MAX_KEYS]; + bool isnull[INDEX_MAX_KEYS]; + ExprState *predicate; + TupleTableSlot *slot; + EState *estate; + ExprContext *econtext; + Snapshot snapshot; + bool need_unregister_snapshot = false; + + /* See whether we're verifying uniqueness/exclusion properties */ + checking_uniqueness = (indexInfo->ii_Unique || + indexInfo->ii_ExclusionOps != NULL); + + /* "Any visible" mode is not compatible with uniqueness checks */ + Assert(!(anyvisible && checking_uniqueness)); + + /* + * Need an EState for evaluation of index expressions and partial-index + * predicates. Also a slot to hold the current tuple. + */ + estate = CreateExecutorState(); + econtext = GetPerTupleExprContext(estate); + slot = table_slot_create(tablerel, NULL); + + /* Arrange for econtext's scan tuple to be the tuple under test */ + econtext->ecxt_scantuple = slot; + + /* Set up execution state for predicate, if any */ + predicate = ExecPrepareQual(indexInfo->ii_Predicate, estate); + + /* + * Prepare for scan. Normal index build uses SnapshotAny (we do our own + * visibility checks). Concurrent/bootstrap uses an MVCC snapshot. + */ + if (!scan) + { + if (IsBootstrapProcessingMode() || indexInfo->ii_Concurrent) + { + snapshot = RegisterSnapshot(GetTransactionSnapshot()); + need_unregister_snapshot = true; + } + else + snapshot = SnapshotAny; + + scan = table_beginscan_strat(tablerel, snapshot, 0, NULL, + true, allow_sync); + } + else + { + snapshot = scan->rs_snapshot; + } + + /* Scan all tuples in the base relation */ + while (table_scan_getnextslot(scan, ForwardScanDirection, slot)) + { + bool tupleIsAlive; + + CHECK_FOR_INTERRUPTS(); + + if (snapshot == SnapshotAny) + { + /* + * Classify the tuple using FLUX's timestamp-based MVCC by + * re-reading the tuple header from the page. + */ + Buffer buf; + Page page; + ItemId itemid; + FluxTupleHeader *tuple_hdr; + BlockNumber blkno = ItemPointerGetBlockNumber(&slot->tts_tid); + OffsetNumber offnum = ItemPointerGetOffsetNumber(&slot->tts_tid); + bool indexIt; + + buf = ReadBuffer(tablerel, blkno); + LockBuffer(buf, BUFFER_LOCK_SHARE); + page = BufferGetPage(buf); + + itemid = PageGetItemId(page, offnum); + if (!ItemIdIsNormal(itemid)) + { + UnlockReleaseBuffer(buf); + continue; + } + + tuple_hdr = (FluxTupleHeader *) PageGetItem(page, itemid); + + if (tuple_hdr->t_flags & FLUX_TUPLE_DELETED) + { + if (FluxTupleDeadToAll(tuple_hdr, FluxGetOldestXminHorizon(tablerel))) + { + /* Definitely dead -- skip */ + UnlockReleaseBuffer(buf); + continue; + } + else + { + /* Recently dead -- index for MVCC, don't count */ + indexIt = true; + tupleIsAlive = false; + } + } + else if (tuple_hdr->t_flags & FLUX_TUPLE_SPECULATIVE) + { + /* Speculative insertion not yet confirmed -- skip */ + UnlockReleaseBuffer(buf); + continue; + } + else + { + /* Live tuple -- index and count it */ + indexIt = true; + tupleIsAlive = true; + reltuples += 1; + } + + UnlockReleaseBuffer(buf); + + if (!indexIt) + continue; + } + else + { + /* MVCC snapshot already filtered for visibility */ + tupleIsAlive = true; + reltuples += 1; + } + + MemoryContextReset(econtext->ecxt_per_tuple_memory); + + /* In a partial index, discard tuples that don't satisfy predicate */ + if (predicate != NULL) + { + if (!ExecQual(predicate, econtext)) + continue; + } + + /* + * Extract all indexed attributes. This also evaluates any index + * expressions. + */ + FormIndexDatum(indexInfo, slot, estate, values, isnull); + + /* + * Call the AM's callback with the tuple's own TID. FLUX secondary + * indexes hold plain 6-byte heap-style TIDs (no RowID/gen suffix), so + * this matches the stock heap index-build contract exactly. + */ + callback(indexrel, &slot->tts_tid, + values, isnull, tupleIsAlive, callback_state); + } + + table_endscan(scan); + + if (need_unregister_snapshot) + UnregisterSnapshot(snapshot); + + ExecDropSingleTupleTableSlot(slot); + FreeExecutorState(estate); + + /* These may have been pointing to the now-gone estate */ + indexInfo->ii_ExpressionsState = NIL; + indexInfo->ii_PredicateState = NULL; + + return reltuples; +} + +/* + * Validate scan for index + */ +static void +flux_index_validate_scan(Relation tablerel, Relation indexrel, + IndexInfo *indexInfo, Snapshot snapshot, + ValidateIndexState *state) +{ + TableScanDesc scan; + TupleTableSlot *slot; + Datum values[INDEX_MAX_KEYS]; + bool isnull[INDEX_MAX_KEYS]; + ExprState *predicate; + EState *estate; + ExprContext *econtext; + ItemPointer indexcursor = NULL; + ItemPointerData decoded; + bool tuplesort_empty = false; + + /* + * Need an EState for evaluation of index expressions and partial-index + * predicates. + */ + estate = CreateExecutorState(); + econtext = GetPerTupleExprContext(estate); + slot = table_slot_create(tablerel, NULL); + econtext->ecxt_scantuple = slot; + + predicate = ExecPrepareQual(indexInfo->ii_Predicate, estate); + + /* + * Scan the table and the sorted output from tuplesort in parallel. For + * each table tuple, check if there's a matching index entry. Tuples that + * satisfy the predicate but have no index entry need to be inserted into + * the index. + */ + scan = table_beginscan_strat(tablerel, snapshot, 0, NULL, true, false); + + while (table_scan_getnextslot(scan, ForwardScanDirection, slot)) + { + CHECK_FOR_INTERRUPTS(); + + state->htups += 1; + + /* + * Skip tuples that don't satisfy the partial index predicate. + */ + if (predicate != NULL) + { + MemoryContextReset(econtext->ecxt_per_tuple_memory); + if (!ExecQual(predicate, econtext)) + continue; + } + + /* + * Advance the tuplesort cursor past any entries that are for TIDs + * earlier than the current table tuple. + */ + while (!tuplesort_empty && + (!indexcursor || + ItemPointerCompare(indexcursor, &slot->tts_tid) < 0)) + { + Datum ts_val; + bool ts_isnull; + + tuplesort_empty = !tuplesort_getdatum(state->tuplesort, + true, false, + &ts_val, &ts_isnull, + NULL); + Assert(tuplesort_empty || !ts_isnull); + if (!tuplesort_empty) + { + itemptr_decode(&decoded, DatumGetInt64(ts_val)); + indexcursor = &decoded; + } + else + { + indexcursor = NULL; + } + } + + /* + * If the sorted cursor TID matches the current table tuple, the index + * already has this entry. Otherwise, we need to add it. + */ + if (indexcursor != NULL && + ItemPointerCompare(indexcursor, &slot->tts_tid) == 0) + { + /* Already in the index -- skip */ + continue; + } + + MemoryContextReset(econtext->ecxt_per_tuple_memory); + + FormIndexDatum(indexInfo, slot, estate, values, isnull); + + /* + * Insert the missing index entry using the tuple's own TID. + */ + index_insert(indexrel, values, isnull, &slot->tts_tid, + tablerel, indexInfo->ii_Unique ? + UNIQUE_CHECK_YES : UNIQUE_CHECK_NO, + false, indexInfo); + + state->tups_inserted += 1; + } + + table_endscan(scan); + + ExecDropSingleTupleTableSlot(slot); + FreeExecutorState(estate); + + indexInfo->ii_ExpressionsState = NIL; + indexInfo->ii_PredicateState = NULL; +} + +/* + * Get relation size information + * + * Returns the on-disk size in bytes for the specified fork of the relation. + * This is used by pg_relation_size(), VACUUM, CLUSTER, and many other + * operations that need to know the physical storage footprint. + */ +/* + * Use table_block_relation_size() from tableam.c directly. FLUX uses + * standard BLCKSZ-width forks just like heap, so the generic + * implementation is correct and efficient (no smgrexists() overhead). + */ + +/* + * Check if relation needs a TOAST table + */ +static bool +flux_relation_needs_toast_table(Relation rel) +{ + /* + * FLUX uses standard heap TOAST for wide values (it has no on-page + * overflow mechanism). This mirrors heapam_relation_needs_toast_table: a + * TOAST table is needed iff there is a toastable attribute and the + * maximum tuple length could exceed TOAST_TUPLE_THRESHOLD. + */ + int32 data_length = 0; + bool maxlength_unknown = false; + bool has_toastable_attrs = false; + TupleDesc tupdesc = rel->rd_att; + int32 tuple_length; + int i; + + for (i = 0; i < tupdesc->natts; i++) + { + Form_pg_attribute att = TupleDescAttr(tupdesc, i); + + if (att->attisdropped) + continue; + if (att->attgenerated == ATTRIBUTE_GENERATED_VIRTUAL) + continue; + data_length = att_align_nominal(data_length, att->attalign); + if (att->attlen > 0) + { + data_length += att->attlen; + } + else + { + int32 maxlen = type_maximum_size(att->atttypid, + att->atttypmod); + + if (maxlen < 0) + maxlength_unknown = true; + else + data_length += maxlen; + if (att->attstorage != TYPSTORAGE_PLAIN) + has_toastable_attrs = true; + } + } + if (!has_toastable_attrs) + return false; /* nothing to toast? */ + if (maxlength_unknown) + return true; /* any unlimited-length attrs? */ + tuple_length = MAXALIGN(FLUX_TUPLE_OVERHEAD + + BITMAPLEN(tupdesc->natts)) + + MAXALIGN(data_length); + return (tuple_length > TOAST_TUPLE_THRESHOLD); +} + +/* + * TOAST tables for FLUX relations are ordinary heap relations (FLUX reuses + * the standard heap TOAST machinery). + */ +static Oid +flux_relation_toast_am(Relation rel) +{ + return HEAP_TABLE_AM_OID; +} + +/* + * Estimate relation size + * + * Provides the planner with estimates of the number of pages, tuples, + * and all-visible fraction for this relation. Uses the actual block count + * from storage and estimates tuple density from the first non-empty page. + */ +static void +flux_relation_estimate_size(Relation rel, int32 *attr_widths, + BlockNumber *pages, double *tuples, + double *allvisfrac) +{ + BlockNumber nblocks; + double tuple_count; + + /* Get actual block count from storage */ + nblocks = smgrnblocks(RelationGetSmgr(rel), MAIN_FORKNUM); + + *pages = Max(nblocks, 1); + + if (nblocks == 0) + { + *tuples = 0; + *allvisfrac = 0.0; + return; + } + + /* + * Estimate tuple count. If we have reltuples from pg_class, use that. + * Otherwise, sample the first block to estimate tuple density. + */ + if (rel->rd_rel->reltuples >= 0) + { + /* + * Scale reltuples by the ratio of current pages to relpages to + * account for growth or shrinkage since last ANALYZE. + */ + if (rel->rd_rel->relpages > 0) + tuple_count = rel->rd_rel->reltuples * + ((double) nblocks / (double) rel->rd_rel->relpages); + else + tuple_count = rel->rd_rel->reltuples; + } + else + { + /* + * No statistics available. Sample the first non-empty page to + * estimate tuple density. If we can't find one, fall back to a + * conservative estimate. + */ + double tuples_per_page = 0; + BlockNumber probe; + + for (probe = 0; probe < Min(nblocks, 10); probe++) + { + Buffer buf; + Page pg; + OffsetNumber maxoff; + OffsetNumber off; + int live = 0; + + buf = ReadBufferExtended(rel, MAIN_FORKNUM, probe, + RBM_NORMAL, NULL); + LockBuffer(buf, BUFFER_LOCK_SHARE); + pg = BufferGetPage(buf); + + if (PageIsNew(pg)) + { + UnlockReleaseBuffer(buf); + continue; + } + + maxoff = PageGetMaxOffsetNumber(pg); + for (off = FirstOffsetNumber; off <= maxoff; off++) + { + ItemId iid = PageGetItemId(pg, off); + + if (!ItemIdIsNormal(iid)) + continue; + + /* + * Skip overflow records -- they are not user tuples and + * should not inflate the density estimate. + */ + if (FluxIsOverflowRecordInline( + (FluxTupleHeader *) PageGetItem(pg, iid), + ItemIdGetLength(iid))) + continue; + + live++; + } + + UnlockReleaseBuffer(buf); + + if (live > 0) + { + tuples_per_page = (double) live; + break; + } + } + + /* Fallback if every sampled page was empty or new */ + if (tuples_per_page <= 0) + tuples_per_page = (BLCKSZ - FLUX_PAGE_OVERHEAD) / 100.0; + + tuple_count = tuples_per_page * nblocks; + } + + *tuples = Max(tuple_count, 0); + + /* + * Compute allvisfrac from pg_class.relallvisible, exactly as heap's + * table_block_relation_estimate_size() does. relallvisible is maintained + * by VACUUM and is an O(1) catalog read -- no per-plan Visibility Map + * scan. The previous implementation swept every VM page (plus a + * smgrexists()/smgrnblocks() probe of the VM fork) on every query plan, + * which showed up as ~9%% of CPU under high-concurrency pgbench. A stale + * catalog value is acceptable here: the planner tolerates an approximate + * allvisfrac, and heap relies on the same value. + */ + if (rel->rd_rel->relpages > 0) + { + double allvisible; + + allvisible = (double) rel->rd_rel->relallvisible / + (double) rel->rd_rel->relpages; + if (allvisible < 0.0) + allvisible = 0.0; + else if (allvisible > 1.0) + allvisible = 1.0; + *allvisfrac = allvisible; + } + else + *allvisfrac = 0.0; +} + +/* + * Sample scan: get next block for sampling (TABLESAMPLE support) + * + * Called by the TABLESAMPLE executor to prepare the next block for tuple + * extraction. The TSM (Table Sample Method) decides which block to visit + * via its NextSampleBlock callback, or, if that callback is NULL, we scan + * sequentially starting from rs_startblock and wrapping around. + * + * We read the selected block into a buffer (pinned, not locked -- locking + * is deferred to flux_scan_sample_next_tuple) and return true. Returns + * false when there are no more blocks to sample. + */ +static bool +flux_scan_sample_next_block(TableScanDesc scan, SampleScanState *scanstate) +{ + FluxScanDesc rscan = (FluxScanDesc) scan; + TsmRoutine *tsm = scanstate->tsmroutine; + BlockNumber blockno; + + /* Return false immediately if relation is empty */ + if (rscan->rs_nblocks == 0) + return false; + + /* Release previous buffer, if any */ + if (BufferIsValid(rscan->rs_cbuf)) + { + ReleaseBuffer(rscan->rs_cbuf); + rscan->rs_cbuf = InvalidBuffer; + } + + if (tsm->NextSampleBlock) + { + /* TSM tells us which block to visit next */ + blockno = tsm->NextSampleBlock(scanstate, rscan->rs_nblocks); + } + else + { + /* No NextSampleBlock callback -- scan sequentially */ + if (rscan->rs_cblock == InvalidBlockNumber) + { + Assert(!rscan->rs_inited); + blockno = rscan->rs_startblock; + } + else + { + Assert(rscan->rs_inited); + + blockno = rscan->rs_cblock + 1; + + if (blockno >= rscan->rs_nblocks) + { + /* Wrap to beginning of relation */ + blockno = 0; + } + + if (blockno == rscan->rs_startblock) + { + /* Completed full cycle -- done */ + blockno = InvalidBlockNumber; + } + } + } + + rscan->rs_cblock = blockno; + + if (!BlockNumberIsValid(blockno)) + { + rscan->rs_inited = false; + return false; + } + + Assert(rscan->rs_cblock < rscan->rs_nblocks); + + CHECK_FOR_INTERRUPTS(); + + /* Read the selected block -- comes back pinned but not locked */ + rscan->rs_cbuf = ReadBufferExtended(scan->rs_rd, MAIN_FORKNUM, + blockno, RBM_NORMAL, NULL); + + rscan->rs_inited = true; + return true; +} + +/* + * Sample scan: get next tuple from current block (TABLESAMPLE support) + * + * Called repeatedly for the block prepared by flux_scan_sample_next_block(). + * The TSM's NextSampleTuple callback decides which tuple offsets to examine. + * We lock the buffer, check the tuple at the selected offset for visibility, + * and either return it in the slot (true) or indicate end-of-page (false). + * + * Unlike the ANALYZE path which iterates all items sequentially, here the + * TSM picks specific offsets, and we loop until it returns InvalidOffsetNumber + * to signal that it is done with this block. + */ +static bool +flux_scan_sample_next_tuple(TableScanDesc scan, SampleScanState *scanstate, + TupleTableSlot *slot) +{ + FluxScanDesc rscan = (FluxScanDesc) scan; + TsmRoutine *tsm = scanstate->tsmroutine; + BlockNumber blockno = rscan->rs_cblock; + Page page; + OffsetNumber maxoffset; + + /* + * Lock the buffer for visibility checks. We hold the lock for the + * duration of this call and release before returning, matching the heap + * AM's non-pagemode pattern. + */ + LockBuffer(rscan->rs_cbuf, BUFFER_LOCK_SHARE); + + page = BufferGetPage(rscan->rs_cbuf); + maxoffset = PageGetMaxOffsetNumber(page); + + for (;;) + { + OffsetNumber tupoffset; + ItemId itemid; + FluxTupleHeader *tuple_hdr; + bool visible; + + CHECK_FOR_INTERRUPTS(); + + /* Ask the TSM which tuple to examine next on this page */ + tupoffset = tsm->NextSampleTuple(scanstate, blockno, maxoffset); + + if (OffsetNumberIsValid(tupoffset)) + { + /* Skip invalid item pointers */ + itemid = PageGetItemId(page, tupoffset); + if (!ItemIdIsNormal(itemid)) + continue; + + tuple_hdr = (FluxTupleHeader *) PageGetItem(page, itemid); + + /* Skip overflow records -- not user-visible tuples */ + if (FluxIsOverflowRecordInline(tuple_hdr, ItemIdGetLength(itemid))) + continue; + + /* + * Determine visibility. FLUX uses heap-shaped xmin/xmax MVCC via + * FluxTupleVisibleToSnapshotDual, which handles DELETED/UPDATED + * tuples via sLog consultation. + */ + if (tuple_hdr->t_flags & FLUX_TUPLE_SPECULATIVE) + visible = false; + else if (scan->rs_snapshot) + visible = FluxTupleVisibleToSnapshotDual(tuple_hdr, + scan->rs_snapshot, + RelationGetRelid(scan->rs_rd), + rscan->rs_cbuf); + else + visible = !(tuple_hdr->t_flags & FLUX_TUPLE_DELETED); + + if (!visible) + continue; + + /* + * Found a visible tuple. Store it into the slot with a buffer + * pin so the data stays valid after we unlock. + */ + FluxSlotStoreTuple(slot, tuple_hdr, + ItemIdGetLength(itemid), rscan->rs_cbuf); + slot->tts_tableOid = RelationGetRelid(scan->rs_rd); + ItemPointerSet(&slot->tts_tid, blockno, tupoffset); + LockBuffer(rscan->rs_cbuf, BUFFER_LOCK_UNLOCK); + + return true; + } + else + { + /* + * NextSampleTuple returned InvalidOffsetNumber -- done with this + * block. Unlock, clear the slot, and tell the caller to move on. + */ + LockBuffer(rscan->rs_cbuf, BUFFER_LOCK_UNLOCK); + ExecClearTuple(slot); + return false; + } + } + + /* unreachable */ + Assert(false); +} + +/* + * ------------------------------------------------------------------------ + * Main table AM routine structure for FLUX + * ------------------------------------------------------------------------ + */ +static const TableAmRoutine flux_methods = { + .type = T_TableAmRoutine, + + /* + * FLUX table DML records UNDO into the relation's own UNDO fork + * (RELUNDO_FORKNUM). flux_operations.c reserves space via + * RelUndoReserve() and emits RELUNDO_INSERT/DELETE/UPDATE/DELTA_UPDATE + * records; on abort RelUndoApplyChain() (relundo_apply.c) replays them + * synchronously in the aborting backend. This is the only UNDO path the + * table AM's own row operations use. + * + * The common-WAL UNDO path (UNDO_RMID_FLUX -> flux_undo.c, dispatched by + * undoapply.c) is the AM-agnostic mechanism shared with nbtree, hash, and + * FILEOPS. Indexes on a FLUX relation piggyback their UNDO records onto + * the table's active UNDO context (see NbtreeUndoLogInsert), so an index + * insert rolls back through the same chain as the row change. FLUX's own + * row DML does not emit UNDO_RMID_FLUX records. + * + * Upstream's am_supports_undo contract (see src/include/access/tableam.h) + * is AM-agnostic: each AM owns its page format and rollback path. No + * heap-page-layout constraint applies. + */ + .am_supports_undo = true, + + /* + * FLUX secondary indexes are plain 6-byte heap-TID nbtree entries with + * standard maintenance, like heap. A key-changing UPDATE is heap-like + * (new TID, old version to UNDO, old index entry dies and is VACUUMed); a + * non-key UPDATE is in place. The executor treats FLUX exactly like heap + * for indexing (the plain-TID index path; see indexam.c / + * execIndexing.c). + */ + + /* Use minimal tuple slot */ + .slot_callbacks = flux_slot_callbacks, + + /* Use minimal scan functions - just return empty results */ + .scan_begin = flux_scan_begin, + .scan_end = flux_scan_end, + .scan_rescan = flux_scan_rescan, + .scan_getnextslot = flux_scan_getnextslot, + + .scan_set_tidrange = flux_scan_set_tidrange, + .scan_getnextslot_tidrange = flux_scan_getnextslot_tidrange, + + .parallelscan_estimate = table_block_parallelscan_estimate, + .parallelscan_initialize = table_block_parallelscan_initialize, + .parallelscan_reinitialize = table_block_parallelscan_reinitialize, + + /* Use minimal index functions */ + .index_fetch_begin = flux_index_fetch_begin, + .index_fetch_reset = flux_index_fetch_reset, + .index_fetch_end = flux_index_fetch_end, + .index_fetch_tuple = flux_index_fetch_tuple, + + /* Use minimal tuple functions */ + .tuple_insert = flux_tuple_insert, + .tuple_insert_speculative = flux_tuple_insert_speculative, + .tuple_complete_speculative = flux_tuple_complete_speculative, + .multi_insert = flux_multi_insert, + .tuple_delete = flux_tuple_delete, + .tuple_update = flux_tuple_update, + .tuple_lock = flux_tuple_lock, + + /* UNDO write-buffer activation / deactivation */ + .begin_bulk_insert = flux_begin_bulk_insert, + .finish_bulk_insert = flux_finish_bulk_insert, + + .tuple_fetch_row_version = flux_tuple_fetch_row_version, + .tuple_get_latest_tid = flux_tuple_get_latest_tid, + .tuple_tid_valid = flux_tuple_tid_valid, + .tuple_satisfies_snapshot = flux_tuple_satisfies_snapshot, + .index_delete_tuples = flux_index_delete_tuples, + + /* Keep only essential relation functions */ + .relation_set_new_filelocator = flux_relation_set_new_filelocator, + .relation_nontransactional_truncate = flux_relation_nontransactional_truncate, + .relation_copy_data = flux_relation_copy_data, + .relation_copy_for_cluster = flux_relation_copy_for_cluster, + .relation_vacuum = flux_relation_vacuum, + .scan_analyze_next_block = flux_scan_analyze_next_block, + .scan_analyze_next_tuple = flux_scan_analyze_next_tuple, + .index_build_range_scan = flux_index_build_range_scan, + .index_validate_scan = flux_index_validate_scan, + + .relation_size = table_block_relation_size, + .relation_needs_toast_table = flux_relation_needs_toast_table, + .relation_toast_am = flux_relation_toast_am, + .relation_fetch_toast_slice = heap_fetch_toast_slice, + + .relation_estimate_size = flux_relation_estimate_size, + + .scan_bitmap_next_tuple = flux_scan_bitmap_next_tuple, + .scan_sample_next_block = flux_scan_sample_next_block, + .scan_sample_next_tuple = flux_scan_sample_next_tuple, + + /* + * FLUX secondary indexes hold plain 6-byte heap-style TID entries and are + * maintained exactly like heap indexes. A key-changing UPDATE stores the + * new version out of place (new TID) so the old (key, TID) index entry + * becomes dead and is reclaimed by VACUUM, never duplicated -- see + * flux_tuple_update(). + */ +}; + +/* + * Return the FLUX table AM routine + */ +const TableAmRoutine * +GetFluxTableAmRoutine(void) +{ + return &flux_methods; +} + +/* + * Handler function for FLUX table access method + */ +PG_FUNCTION_INFO_V1(flux_tableam_handler); + +Datum +flux_tableam_handler(PG_FUNCTION_ARGS) +{ + PG_RETURN_POINTER(&flux_methods); +} + +/* + * flux_analyze_accumulate_sample + * + * During an ANALYZE scan, capture the decompressed bytes of the relation's + * first varlena column from the just-materialized slot. These samples feed + * FluxMaybeRefreshDict() at scan end so it can train a candidate compression + * dictionary from data ANALYZE already sampled, with no extra table reads. + * + * Sample buffers are allocated lazily in the scan's own memory context and + * are bounded by fixed byte/count caps. Once a cap is hit we stop collecting + * but let the ANALYZE scan continue normally. + */ +static void +flux_analyze_accumulate_sample(FluxScanDesc rscan, TupleTableSlot *slot) +{ + /* + * FLUX does not compress attributes and has no compression dictionary, so + * ANALYZE does not accumulate a training corpus. + */ + (void) rscan; + (void) slot; +} + +/* + * ANALYZE support: select next block to sample + * + * Called by ANALYZE to prepare the next sampled block for tuple extraction. + * The ReadStream provides buffers for blocks selected by the BlockSampler + * in analyze.c -- we do not choose blocks ourselves. + * + * We acquire a buffer pin and shared lock here and hold them until + * flux_scan_analyze_next_tuple() has returned false for this block, + * preventing concurrent activity (e.g. pruning) from removing tuples + * out from under us. + */ +static bool +flux_scan_analyze_next_block(TableScanDesc scan, ReadStream *stream) +{ + FluxScanDesc rscan = (FluxScanDesc) scan; + + /* + * Get the next buffer from the read stream. The stream was set up by + * analyze.c with a BlockSampler callback, so it yields only the randomly + * selected sample blocks. The buffer comes back already pinned. + */ + rscan->rs_cbuf = read_stream_next_buffer(stream, NULL); + + if (!BufferIsValid(rscan->rs_cbuf)) + return false; + + /* + * Don't lock the buffer here; flux_scan_analyze_next_tuple() manages its + * own lock/unlock cycle so it can release the lock before returning a + * sampled tuple, allowing FluxFetchOverflowColumn() to safely lock the + * same buffer for overflow data on this page. + */ + + rscan->rs_cblock = BufferGetBlockNumber(rscan->rs_cbuf); + rscan->rs_cindex = FirstOffsetNumber; + + return true; +} + +/* + * ANALYZE support: get next tuple from current block + * + * Extracts tuples one at a time from the block prepared by + * flux_scan_analyze_next_block(). For each item pointer on the page we + * classify the tuple as live, dead, or not-a-tuple (overflow record, + * unused pointer) and update the caller's counters. + * + * When a live tuple suitable for sampling is found, it is materialized + * into the slot and we return true. When all items on the page have been + * examined, we release the buffer and return false. + * + * The buffer remains pinned and locked for the entire duration of tuple + * iteration on this block, matching the heap AM contract. + */ +static bool +flux_scan_analyze_next_tuple(TableScanDesc scan, + double *liverows, double *deadrows, + TupleTableSlot *slot) +{ + FluxScanDesc rscan = (FluxScanDesc) scan; + Page targpage; + OffsetNumber maxoffset; + + Assert(BufferIsValid(rscan->rs_cbuf)); + + /* + * Re-acquire the buffer content lock. We release it before returning a + * sampled tuple (see below) so that the caller can safely deform the + * tuple -- FluxFetchOverflowColumn() may need to lock the same buffer to + * read overflow data stored on the same page. + */ + LockBuffer(rscan->rs_cbuf, BUFFER_LOCK_SHARE); + + targpage = BufferGetPage(rscan->rs_cbuf); + maxoffset = PageGetMaxOffsetNumber(targpage); + + /* Inner loop over items on the selected page */ + for (; rscan->rs_cindex <= maxoffset; rscan->rs_cindex++) + { + ItemId itemid; + FluxTupleHeader *tuple_hdr; + bool sample_it = false; + RelUndoRecPtr proxy_verptr = InvalidRelUndoRecPtr; + + itemid = PageGetItemId(targpage, rscan->rs_cindex); + + /* + * Skip unused and dead line pointers. Dead line pointers are counted + * as dead rows because vacuum needs to reclaim them. + */ + if (!ItemIdIsNormal(itemid)) + { + if (ItemIdIsDead(itemid)) + *deadrows += 1; + continue; + } + + tuple_hdr = (FluxTupleHeader *) PageGetItem(targpage, itemid); + + /* Skip overflow records -- these are not user-visible tuples */ + if (FluxIsOverflowRecordInline(tuple_hdr, ItemIdGetLength(itemid))) + continue; + + /* + * Classify the tuple for ANALYZE purposes. FLUX uses timestamp-based + * MVCC rather than xmin/xmax, so we check the tuple flags and + * timestamps directly. + */ + if (tuple_hdr->t_flags & FLUX_TUPLE_DELETED) + { + /* Dead tuple (deleted) -- counted as dead for ANALYZE */ + *deadrows += 1; + } + else if (tuple_hdr->t_flags & FLUX_TUPLE_SPECULATIVE) + { + /* + * Speculative insertion not yet confirmed. Don't count it; if + * the inserter commits it will be picked up by a future ANALYZE. + */ + } + else + { + /* + * Tuple is live (or at least not deleted/speculative). Sample it + * for statistics. + */ + sample_it = true; + *liverows += 1; + + /* + * Synthetic dead-tuple proxy for the per-relation UNDO fork. An + * in-place FLUX UPDATE creates no genuine dead tuple, so the + * standard dead-tuple autovacuum trigger would never fire and the + * UNDO fork would grow unbounded. A live tuple that carries a + * version pointer has a prior committed image retained in the + * fork; if that record has not yet been discarded it is + * reclaimable work, so we count it as a dead-tuple proxy. The + * liveness probe touches a fork buffer, so it is deferred until + * after the content lock on the data page is released (below), + * matching the overflow-fetch discipline used for the dictionary + * sample. + */ + proxy_verptr = FluxTupleGetVersionPtr(tuple_hdr, + ItemIdGetLength(itemid)); + } + + if (sample_it) + { + /* + * Materialize the tuple into palloc'd memory rather than storing + * a buffer-pinned pointer. This is necessary because slot + * deformation may call FluxFetchOverflowColumn(), which acquires + * buffer locks on overflow pages. If the overflow data resides + * on the same page we are scanning, a buffer-pinned slot would + * cause a lock re-entry assertion failure in LockBuffer + * (bufmgr.c). + */ + Size tuple_size = ItemIdGetLength(itemid); + FluxTupleHeader *tuple_copy; + + tuple_copy = (FluxTupleHeader *) palloc(tuple_size); + memcpy(tuple_copy, tuple_hdr, tuple_size); + + FluxSlotStoreMaterializedTuple(slot, tuple_copy, tuple_size); + slot->tts_tableOid = RelationGetRelid(scan->rs_rd); + ItemPointerSet(&slot->tts_tid, rscan->rs_cblock, rscan->rs_cindex); + rscan->rs_cindex++; + + /* + * Release the content lock so the caller can safely deform the + * materialized tuple. The buffer pin is kept so the page stays + * in the buffer pool. We re-acquire the lock at the top of this + * function when called again. + */ + LockBuffer(rscan->rs_cbuf, BUFFER_LOCK_UNLOCK); + + /* + * Opportunistically feed the dictionary-refresh corpus from the + * same sampled tuple. Done after the unlock so any + * overflow-column fetch can lock this page safely. + */ + if (scan->rs_flags & SO_TYPE_ANALYZE) + flux_analyze_accumulate_sample(rscan, slot); + + /* + * Count the UNDO-fork version-chain proxy (see the live-tuple + * branch above). Probing the fork record must happen with no + * data page content lock held. RelUndoReadRecordHeader reads + * only the record header (no payload allocation) and returns + * false once the record has been discarded by VACUUM, so the + * proxy resets after the fork is reclaimed. + */ + if (RelUndoRecPtrIsValid(proxy_verptr)) + { + RelUndoRecordHeader urec_hdr; + + if (RelUndoReadRecordHeader(scan->rs_rd, proxy_verptr, + &urec_hdr)) + *deadrows += 1; + } + + return true; + } + } + + /* + * No more tuples on this page. Release the buffer pin and lock that were + * acquired in flux_scan_analyze_next_block(). + */ + UnlockReleaseBuffer(rscan->rs_cbuf); + rscan->rs_cbuf = InvalidBuffer; + + /* Prevent stale slot contents from holding a pin */ + ExecClearTuple(slot); + + return false; +} diff --git a/src/backend/access/flux/flux_lock.c b/src/backend/access/flux/flux_lock.c new file mode 100644 index 0000000000000..30ff6a21553cd --- /dev/null +++ b/src/backend/access/flux/flux_lock.c @@ -0,0 +1,357 @@ +/*------------------------------------------------------------------------- + * + * flux_lock.c + * FLUX locking mechanisms for concurrent access + * + * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/access/flux/flux_lock.c + * + * NOTES + * This implements proper locking for FLUX operations to ensure + * data consistency under concurrent access. Uses both buffer locks + * and tuple-level locks with deadlock detection. + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/flux.h" +#include "storage/bufmgr.h" +#include "storage/lmgr.h" +#include "storage/lwlock.h" +#include "storage/proc.h" +#include "utils/rel.h" +#include "miscadmin.h" +#include "access/tableam.h" + +/* + * FluxLockTuple + * + * Acquire a tuple-level lock on the specified tuple using PostgreSQL's + * standard LOCKTAG_TUPLE mechanism. The lock mode is converted from + * LockTupleMode to the corresponding LOCKMODE (ShareLock for read modes, + * ExclusiveLock for write modes). + * + * Parameters: + * rel - open relation containing the tuple + * tid - ItemPointer identifying the tuple (block + offset) + * mode - desired lock strength (LockTupleKeyShare through + * LockTupleExclusive) + * wait - if true, block until the lock is available; if false, + * return false immediately if the lock cannot be acquired + * have_tuple_lock - output: set to true if the lock was successfully acquired + * + * Returns true if the lock was acquired, false if 'wait' was false and the + * lock was not available. + * + * The caller is responsible for calling FluxUnlockTuple() to release the + * lock when done. + */ +bool +FluxLockTuple(Relation rel, ItemPointer tid, LockTupleMode mode, + bool wait, bool *have_tuple_lock) +{ + LOCKTAG tag; + LOCKMODE lockmode; + bool result; + + *have_tuple_lock = false; + + /* + * Convert tuple lock mode to standard lock mode using the same mapping as + * heap (tupleLockExtraInfo in heapam.c). The four modes MUST map to four + * distinct LOCKMODEs: collapsing KeyShare/Share or + * NoKeyExclusive/Exclusive makes an FK key-share lock conflict with a + * concurrent no-key UPDATE, manufacturing deadlocks that heap never + * suffers. + */ + switch (mode) + { + case LockTupleKeyShare: + lockmode = AccessShareLock; + break; + case LockTupleShare: + lockmode = RowShareLock; + break; + case LockTupleNoKeyExclusive: + lockmode = ExclusiveLock; + break; + case LockTupleExclusive: + lockmode = AccessExclusiveLock; + break; + default: + elog(ERROR, "invalid tuple lock mode: %d", mode); + } + + /* Set up lock tag for tuple */ + SET_LOCKTAG_TUPLE(tag, + rel->rd_locator.dbOid, + rel->rd_locator.relNumber, + ItemPointerGetBlockNumber(tid), + ItemPointerGetOffsetNumber(tid)); + + /* Acquire the lock */ + if (wait) + { + LockAcquire(&tag, lockmode, false, false); + result = true; + } + else + { + result = (LockAcquireExtended(&tag, lockmode, false, true, true, NULL, false) != LOCKACQUIRE_NOT_AVAIL); + } + + if (result) + *have_tuple_lock = true; + + return result; +} + +/* + * FluxUnlockTuple + * + * Release a tuple-level lock previously acquired by FluxLockTuple(). + * + * Parameters: + * rel - open relation containing the tuple + * tid - ItemPointer identifying the locked tuple + * mode - lock mode that was used when acquiring (must match) + */ +void +FluxUnlockTuple(Relation rel, ItemPointer tid, LockTupleMode mode) +{ + LOCKTAG tag; + LOCKMODE lockmode; + + /* + * Convert tuple lock mode to standard lock mode using the same mapping as + * heap (tupleLockExtraInfo in heapam.c). The four modes MUST map to four + * distinct LOCKMODEs: collapsing KeyShare/Share or + * NoKeyExclusive/Exclusive makes an FK key-share lock conflict with a + * concurrent no-key UPDATE, manufacturing deadlocks that heap never + * suffers. + */ + switch (mode) + { + case LockTupleKeyShare: + lockmode = AccessShareLock; + break; + case LockTupleShare: + lockmode = RowShareLock; + break; + case LockTupleNoKeyExclusive: + lockmode = ExclusiveLock; + break; + case LockTupleExclusive: + lockmode = AccessExclusiveLock; + break; + default: + elog(ERROR, "invalid tuple lock mode: %d", mode); + } + + /* Set up lock tag for tuple */ + SET_LOCKTAG_TUPLE(tag, + rel->rd_locator.dbOid, + rel->rd_locator.relNumber, + ItemPointerGetBlockNumber(tid), + ItemPointerGetOffsetNumber(tid)); + + /* Release the lock */ + LockRelease(&tag, lockmode, false); +} + +/* + * FluxLockPage + * + * Acquire a page-level lock using LOCKTAG_PAGE. This is used for operations + * that need exclusive access to an entire page's structure, such as + * defragmentation or cross-page tuple moves. + * + * Note: This is distinct from buffer-level locking (LockBuffer). Buffer + * locks protect the in-memory page image; page-level locks here protect + * the logical page structure across multiple buffer accesses. + * + * Parameters: + * rel - open relation containing the page + * blkno - block number to lock + * mode - lock mode (typically ShareLock or ExclusiveLock) + */ +void +FluxLockPage(Relation rel, BlockNumber blkno, LOCKMODE mode) +{ + LOCKTAG tag; + + /* Set up lock tag for page */ + SET_LOCKTAG_PAGE(tag, + rel->rd_locator.dbOid, + rel->rd_locator.relNumber, + blkno); + + /* Acquire the lock */ + LockAcquire(&tag, mode, false, false); +} + +/* + * FluxUnlockPage + * + * Release a page-level lock previously acquired by FluxLockPage(). + * + * Parameters: + * rel - open relation containing the page + * blkno - block number to unlock + * mode - lock mode that was used when acquiring (must match) + */ +void +FluxUnlockPage(Relation rel, BlockNumber blkno, LOCKMODE mode) +{ + LOCKTAG tag; + + /* Set up lock tag for page */ + SET_LOCKTAG_PAGE(tag, + rel->rd_locator.dbOid, + rel->rd_locator.relNumber, + blkno); + + /* Release the lock */ + LockRelease(&tag, mode, false); +} + + +/* + * FluxLockMultipleTuples + * + * Acquire tuple-level locks on multiple tuples in a consistent order to + * prevent deadlocks. The TIDs are sorted (using bubble sort, which is + * adequate since N is typically small) before acquiring locks, ensuring + * that all callers acquire locks in the same global order. + * + * If any lock acquisition fails (when wait=false), all previously acquired + * locks are released and the function returns false. + * + * Note: The tids array is sorted in-place, which modifies the caller's array. + * + * Parameters: + * rel - open relation containing the tuples + * tids - array of ItemPointerData identifying tuples to lock (sorted in-place) + * ntids - number of entries in tids array + * mode - desired lock strength for all tuples + * wait - if true, block until all locks are available + * + * Returns true if all locks were acquired, false if any could not be acquired. + */ +bool +FluxLockMultipleTuples(Relation rel, ItemPointerData *tids, int ntids, + LockTupleMode mode, bool wait) +{ + int i, + j; + bool all_locked = true; + bool *locked = palloc0(sizeof(bool) * ntids); + + /* Sort TIDs to ensure consistent lock ordering */ + for (i = 0; i < ntids - 1; i++) + { + for (j = i + 1; j < ntids; j++) + { + if (ItemPointerCompare(&tids[i], &tids[j]) > 0) + { + ItemPointerData temp = tids[i]; + + tids[i] = tids[j]; + tids[j] = temp; + } + } + } + + /* Acquire locks in sorted order */ + for (i = 0; i < ntids; i++) + { + bool have_lock; + + if (!FluxLockTuple(rel, &tids[i], mode, wait, &have_lock)) + { + all_locked = false; + break; + } + locked[i] = have_lock; + } + + /* If we failed to get all locks, release what we got */ + if (!all_locked) + { + for (j = 0; j < i; j++) + { + if (locked[j]) + FluxUnlockTuple(rel, &tids[j], mode); + } + } + + pfree(locked); + return all_locked; +} + +/* + * FluxLockRelationForDDL + * + * Acquire a relation-level lock for DDL operations (e.g., ALTER TABLE, + * DROP TABLE). Delegates to PostgreSQL's standard LockRelationOid(). + * + * Parameters: + * rel - open relation to lock + * lockmode - lock mode (typically AccessExclusiveLock for DDL) + */ +void +FluxLockRelationForDDL(Relation rel, LOCKMODE lockmode) +{ + /* Use standard relation locking */ + LockRelationOid(RelationGetRelid(rel), lockmode); +} + +/* + * FluxHoldsTupleLock + * + * Check whether the current transaction already holds a lock on the + * specified tuple at the given mode. This is useful for avoiding redundant + * lock acquisitions and for assertions in debug builds. + * + * Parameters: + * rel - open relation containing the tuple + * tid - ItemPointer identifying the tuple + * mode - lock mode to check for + * + * Returns true if the current transaction holds the specified lock. + */ +bool +FluxHoldsTupleLock(Relation rel, ItemPointer tid, LockTupleMode mode) +{ + LOCKTAG tag; + LOCKMODE lockmode; + + /* Convert tuple lock mode to standard lock mode */ + switch (mode) + { + case LockTupleKeyShare: + case LockTupleShare: + lockmode = ShareLock; + break; + case LockTupleNoKeyExclusive: + case LockTupleExclusive: + lockmode = ExclusiveLock; + break; + default: + return false; + } + + /* Set up lock tag for tuple */ + SET_LOCKTAG_TUPLE(tag, + rel->rd_locator.dbOid, + rel->rd_locator.relNumber, + ItemPointerGetBlockNumber(tid), + ItemPointerGetOffsetNumber(tid)); + + /* Check if we hold the lock */ + return LockHeldByMe(&tag, lockmode, false); +} diff --git a/src/backend/access/flux/flux_mvcc.c b/src/backend/access/flux/flux_mvcc.c new file mode 100644 index 0000000000000..620eba034d95f --- /dev/null +++ b/src/backend/access/flux/flux_mvcc.c @@ -0,0 +1,1344 @@ +/*------------------------------------------------------------------------- + * + * flux_mvcc.c + * FLUX heap-compatible xmin/xmax MVCC implementation + * + * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/access/flux/flux_mvcc.c + * + * NOTES + * FLUX uses ordinary heap-compatible xmin/xmax MVCC visibility, the same + * model as HeapTupleSatisfiesMVCC: a tuple is visible to a snapshot iff its + * inserter (t_xmin) is committed-and-visible-to-the-snapshot AND its + * deleter/updater (t_xmax) is invalid, not committed, or not visible. CLOG + * is the commit-status authority; the snapshot's xmin/xmax/xip decide + * visibility of committed XIDs; subtransaction aborts (ROLLBACK TO + * savepoint) are handled by stamping t_xmin/t_xmax with the current subxid + * so CLOG marks a rolled-back subxact's tuples aborted (plus a transient + * sLog ABORTED marker consulted for the still-in-progress window). + * + * In-place UPDATE keeps the NEWEST version on the page (new t_xmin) and + * pushes the pre-update image to the per-relation UNDO fork via t_verptr + * (zheap style). A snapshot that cannot see the updater's xmin reads the + * old version back from the fork with FluxReconstructVisibleVersion() + * (flux_pvs.c), which walks t_verptr and stops at the version whose + * producing xid is visible (XidInMVCCSnapshot). The single visibility + * function is FluxTupleSatisfiesMVCC(), reached via + * FluxTupleVisibleToSnapshotDual() from every read site. + * + * The pre-commit "dirty read" window closes for free: a reader resolves + * t_xmin against CLOG, which reports committed only at the durability point + * (RecordTransactionCommit flushes the commit record). A not-yet-committed + * (hence not-yet-durable) inserter is simply invisible -- no HLC, no + * timestamp rewind, no flux_cts durable map. + * + * The FLUX_TUPLE_UNCOMMITTED flag (0x0080) is set on insert/delete/update + * and cleared (as a hint bit) at commit. It no longer drives read + * visibility (CLOG does); it is retained for the sLog write-conflict and + * defrag paths. + * + * ISOLATION LEVEL SEMANTICS + * + * FLUX integrates with PostgreSQL's Serializable Snapshot Isolation + * (SSI) infrastructure in predicate.c. The scan path acquires SIREAD + * predicate locks via PredicateLockTID(), and the DML paths (INSERT, + * UPDATE, DELETE) call CheckForSerializableConflictIn() to detect + * rw-antidependencies. The FluxCheckForSerializableConflictOut() + * function handles the reverse direction (reader encounters a tuple + * written by a concurrent transaction) by looking up the writer's + * XID via the sLog and delegating to predicate.c. + * + * BEFORE-IMAGE SERVING: + * + * In-place UPDATEs destroy the pre-image on the page. Under WS-PVS3 the + * visible prior version is reconstructed by walking the durable UNDO fork + * chain (FluxReconstructVisibleVersion), not from any shared sLog DSA + * entry: readers whose snapshot cannot see the committing xid follow the + * on-page head verptr into the UNDO fork. This restores correct snapshot + * semantics for concurrent readers under REPEATABLE READ and SERIALIZABLE. + * + * CONCURRENCY: + * + * 1. Same-tuple write-write conflicts serialize correctly: the + * second writer blocks (via XactLockTableWait on the sLog dirty + * XID) until the first commits or aborts. + * + * 2. Write Skew (A5B) on disjoint tuples IS detected via predicate + * locking (SIREAD locks on tuples read + conflict-in checks on + * writes). + * + * In summary, FLUX's isolation guarantees are: + * - READ COMMITTED: Correct (no dirty reads, each statement gets + * fresh visibility via a per-statement snapshot) + * - REPEATABLE READ: Full Snapshot Isolation; concurrent committed + * UPDATEs are reconstructed from the UNDO fork per reader snapshot + * - SERIALIZABLE: Full SSI via predicate.c integration; write skew + * and phantom anomalies are prevented through predicate locking + * and rw-antidependency cycle detection + * + * References: + * - Berenson et al., "A Critique of ANSI SQL Isolation Levels" (1995) + * - Adya, "Weak Consistency: A Generalized Theory and Optimistic + * Implementations for Distributed Transactions" (2000) + * - Cahill et al., "Serializable Isolation for Snapshot Databases" (2009) + * - Ports & Grittner, "Serializable Snapshot Isolation in PostgreSQL" (2012) + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/atm.h" +#include "access/flux.h" +#include "access/slog.h" +#include "access/subtrans.h" +#include "access/transam.h" +#include "access/twophase.h" +#include "access/xact.h" +#include "utils/snapmgr.h" +#include "miscadmin.h" +#include "port/atomics.h" +#include "utils/memutils.h" +#include "storage/bufmgr.h" +#include "storage/ipc.h" +#include "storage/lwlock.h" +#include "storage/predicate.h" +#include "storage/proc.h" +#include "storage/procarray.h" +#include "storage/shmem.h" +#include "utils/guc.h" +#include "utils/timestamp.h" + +/* + * Total number of PGPROC slots, matching the allProcs array size in proc.c. + * This must cover regular backends, auxiliary procs, and prepared transactions + * since GetNumberFromPGProc() can return indices up to TotalProcs - 1. + */ +#define FLUX_TOTAL_PROCS \ + (MaxBackends + NUM_AUXILIARY_PROCS + max_prepared_xacts) + +/* + * Shared memory structures for MVCC + */ +typedef struct FluxMvccShmemData +{ + LWLock mvcc_lock; /* Protects serializable horizon only */ + uint64 oldest_active_ts; /* Cached oldest active transaction ts; + * backs FluxGetOldestActiveTimestamp(), + * which still feeds the defrag WAL record + * and the VM all-visible check. NOT a + * vacuum horizon. */ + uint64 serializable_horizon; /* Serializable isolation horizon */ + + pg_atomic_uint32 oldest_active_generation; /* Bumped when cache is + * invalidated */ + pg_atomic_uint32 active_xact_count; /* Number of active transactions + * (atomic) */ + + /* + * Per-backend active transaction start timestamps. Each backend slot + * stores the start timestamp of its current FLUX transaction, or 0 if + * idle. This array is indexed by pgprocno (the offset into + * ProcGlobal->allProcs) and is sized to FLUX_TOTAL_PROCS so that + * auxiliary procs and prepared transactions are covered. + * + * Each slot is written only by its owning backend and read by VACUUM, so + * no lock is needed ā€” just a compiler barrier via volatile access. + */ + int num_xact_slots; /* Number of slots (== FLUX_TOTAL_PROCS) */ + uint64 xact_start_ts_slots[FLEXIBLE_ARRAY_MEMBER]; + +} FluxMvccShmemData; + +static FluxMvccShmemData *FluxMvccShmem = NULL; + + +/* + * Per-transaction MVCC state. + * + * SSI (Serializable Snapshot Isolation) conflict detection is now delegated + * entirely to PostgreSQL's predicate.c infrastructure. FLUX integrates + * with it by calling PredicateLockTID() in the scan path and + * CheckForSerializableConflictIn/Out() in the DML paths. The private + * rw-conflict graph that was previously here has been removed. + * + * Commit visibility uses CLOG (heap-shaped xmin/xmax); the timestamp fields + * below are page-level bookkeeping only. + */ +struct FluxTransactionState +{ + uint64 xact_start_ts; /* Transaction start timestamp */ + uint64 xact_commit_ts; /* Transaction commit timestamp */ + + /* + * READ COMMITTED per-command read point. Under RC the visibility read + * point is "now", refreshed each command so a scan sees concurrent + * commits. Captured once per command (keyed by rc_read_point_cid) and + * reused for every tuple. + */ + CommandId rc_read_point_cid; + uint64 rc_read_point_xcc; /* snapXactCompletionCount at capture */ + bool is_serializable; /* Serializable isolation level */ + bool is_read_only; /* Transaction has not performed writes */ + + /* + * EPQ reconcile identity for the in-place write-write conflict path. + * + * When flux_tuple_update detects a committed concurrent UPDATE (head + * verptr's committer is invisible to snapshot) it returns TM_Updated and + * the executor runs EvalPlanQual: table_tuple_lock re-reads the latest + * on-page value (which, because FLUX updates in place, already reflects + * every commit so far) and the quals/SET expression recompute on top of + * it. But EPQ re-evaluates against the same statement snapshot (same xid + * xip / curcid), so a naive probe would re-detect the same committed + * marker and livelock. Heap escapes this because EPQ advances tupleid to + * a new tuple version with a clean xmax; FLUX has no new version to + * advance to. + * + * EPQ dedup: identity of the (verptr, xid) marker we bounced on to + * EvalPlanQual for this exact (relid, tid, cid). If the next probe on + * the same tuple in the same EPQ retry finds an unchanged head verptr and + * xid, we already reconciled it -- skip to prevent livelock. A + * strictly-newer committer stamps a NEW record with a fresh verptr, so + * dedup only suppresses re-firing on the identical marker. No temporal + * ordering is required: this is a physical identity check. + */ + Oid epq_reconcile_relid; + ItemPointerData epq_reconcile_tid; + CommandId epq_reconcile_cid; + RelUndoRecPtr epq_reconcile_verptr; + TransactionId epq_reconcile_xid; +}; + +/* Restart reasons */ +#define FLUX_RESTART_NONE 0 +#define FLUX_RESTART_UNCERTAINTY 1 +#define FLUX_RESTART_SERIALIZABLE 2 + +/* + * Per-backend static transaction state. Using a static struct avoids + * a palloc/pfree cycle per transaction. The struct is reset at the start + * of each transaction by FluxInitTransactionState(). + * + * MyFluxXactState points to &MyFluxXactStateData when a transaction is + * active, and is NULL between transactions. This preserves the existing + * NULL-check pattern throughout the codebase. + */ +static FluxTransactionState MyFluxXactStateData; +static FluxTransactionState *MyFluxXactState = NULL; + +/* + * GUC variables (flux_enable_serializable and flux_max_transactions removed; + * SSI is unconditionally provided via predicate.c) + */ + +/* + * Function prototypes + */ +static void FluxInitTransactionState(void); +static void FluxCleanupTransactionState(void); +static void FluxShmemExit(int code, Datum arg); + +/* FluxCheckUncommittedDelete removed -- logic inlined in visibility checks */ + + +/* + * Shared memory size calculation + * + * Per-transaction state (FluxTransactionState) is allocated in + * backend-local TopTransactionContext, NOT in shared memory, so it + * does not appear here. The only shared-memory array is the + * per-backend xact_start_ts_slots[], which scales naturally with + * FLUX_TOTAL_PROCS (and therefore MaxBackends). During bootstrap + * MaxBackends is ~4, keeping this allocation tiny. + */ +Size +FluxMvccShmemSize(void) +{ + Size size; + + /* + * Base struct (includes the flexible array header but not the array + * elements), plus one uint64 slot per PGPROC (regular backends, auxiliary + * procs, prepared transactions) for tracking active transaction start + * timestamps. + */ + size = offsetof(FluxMvccShmemData, xact_start_ts_slots); + size = add_size(size, mul_size(FLUX_TOTAL_PROCS, sizeof(uint64))); + + return size; +} + +/* + * Initialize shared memory for MVCC + */ +void +FluxMvccShmemInit(void) +{ + bool found; + + FluxMvccShmem = (FluxMvccShmemData *) + ShmemInitStruct("FLUX MVCC Data", + FluxMvccShmemSize(), + &found); + + if (!found) + { + int total_procs = FLUX_TOTAL_PROCS; + + /* Initialize shared memory */ + LWLockInitialize(&FluxMvccShmem->mvcc_lock, LWTRANCHE_BUFFER_MAPPING); + FluxMvccShmem->oldest_active_ts = 1; + pg_atomic_init_u32(&FluxMvccShmem->oldest_active_generation, 0); + FluxMvccShmem->serializable_horizon = 1; + pg_atomic_init_u32(&FluxMvccShmem->active_xact_count, 0); + + /* Initialize per-backend active timestamp slots to 0 (idle) */ + FluxMvccShmem->num_xact_slots = total_procs; + memset(FluxMvccShmem->xact_start_ts_slots, 0, + total_procs * sizeof(uint64)); + } + + /* Register cleanup function */ + on_shmem_exit(FluxShmemExit, 0); +} + +/* + * FluxGetDmlTimestamp -- return the transaction's start timestamp for DML + * page-level bookkeeping. + * + * Within a single transaction, all DML operations (INSERT, UPDATE, DELETE) + * stamp the page-level commit-ts word (FluxPageSetCommitTs) with the + * transaction's start timestamp. + * + * This value is NOT a visibility timestamp. Commit visibility comes from + * CLOG via heap-shaped xmin/xmax: while the transaction is in-flight the + * FLUX_TUPLE_UNCOMMITTED flag is set and self-visibility is determined by + * matching the inserter's XID in the sLog; after commit, t_xmin/t_xmax + CLOG + * decide visibility. Intra-transaction ordering (multiple DMLs in the same + * txn) is handled by CID (command ID) from the sLog entry. + */ +uint64 +FluxGetDmlTimestamp(void) +{ + /* + * Callers must have already called FluxGetTransactionTimestamp() or + * equivalent, which initializes the transaction state. We assert rather + * than lazily initializing, keeping this function as lean as possible on + * the hot path. + */ + Assert(MyFluxXactState != NULL); + + return MyFluxXactState->xact_start_ts; +} + +/* + * FluxGetCommitTimestamp + * + * Return a wall-clock timestamp (microseconds since the PostgreSQL epoch) + * used only for page-level commit-ts bookkeeping and the VM all-visible + * hint. Commit visibility comes from CLOG (heap-shaped xmin/xmax); nothing + * reads this value for a correctness/visibility DECISION, so it does not + * need strict cross-backend monotonicity. + * + * Historically this walked an atomic compare-and-swap loop over a single + * shared global_commit_ts counter to keep timestamps strictly increasing. + * That shared RMW was a hot-path serialization point (one cache line CAS'd + * ~2x per write txn) with no surviving consumer of strict monotonicity, so + * it has been replaced with a plain GetCurrentTimestamp() read: no shared + * counter, no CAS, no spin. Wall-clock microseconds are already + * monotonic-enough for a "min over active start-stamps" horizon and a + * ">= oldest" VM hint (both use non-strict comparisons). + */ +uint64 +FluxGetCommitTimestamp(void) +{ + if (FluxMvccShmem == NULL) + elog(ERROR, "FLUX MVCC not initialized"); + + return (uint64) GetCurrentTimestamp(); +} + +/* + * FluxGetTransactionTimestamp + * + * Return the start timestamp of the current transaction. Initializes + * per-transaction MVCC state on first call within a transaction. + * + * Returns the transaction's start timestamp (uint64). + */ +uint64 +FluxGetTransactionTimestamp(void) +{ + if (MyFluxXactState == NULL) + FluxInitTransactionState(); + + return MyFluxXactState->xact_start_ts; +} + +/* + * Subsystem callback wrappers for PG_SHMEM_SUBSYSTEM infrastructure + */ +static void +FluxMvccShmemRequest(void *arg) +{ + ShmemRequestStruct(.name = "FLUX MVCC Data", + .size = FluxMvccShmemSize(), + .ptr = (void **) &FluxMvccShmem); +} + +static void +FluxMvccShmemInit_cb(void *arg) +{ + int total_procs = FLUX_TOTAL_PROCS; + + /* FluxMvccShmem is already set by the ShmemRequestStruct .ptr mechanism */ + Assert(FluxMvccShmem != NULL); + + /* Initialize shared memory fields */ + LWLockInitialize(&FluxMvccShmem->mvcc_lock, LWTRANCHE_BUFFER_MAPPING); + FluxMvccShmem->oldest_active_ts = 1; + pg_atomic_init_u32(&FluxMvccShmem->oldest_active_generation, 0); + FluxMvccShmem->serializable_horizon = 1; + pg_atomic_init_u32(&FluxMvccShmem->active_xact_count, 0); + + /* Initialize per-backend active timestamp slots to 0 (idle) */ + FluxMvccShmem->num_xact_slots = total_procs; + memset(FluxMvccShmem->xact_start_ts_slots, 0, + total_procs * sizeof(uint64)); + + /* Register cleanup function */ + on_shmem_exit(FluxShmemExit, 0); +} + +const ShmemCallbacks FluxMvccShmemCallbacks = { + .request_fn = FluxMvccShmemRequest, + .init_fn = FluxMvccShmemInit_cb, +}; + +/* + * Initialize per-transaction MVCC state + * + * The start timestamp is a plain wall-clock value from + * FluxGetCommitTimestamp() held in the uint64 xact_start_ts field for + * per-backend slot tracking. + */ +/* + * Transaction callback for FLUX MVCC cleanup. + * + * This is registered once per backend via RegisterXactCallback. + * On transaction commit or abort, it calls FluxCommitTransaction() + * or FluxCleanupTransactionState() to reset MyFluxXactState, + * ensuring the next transaction in this backend gets a fresh start + * timestamp from FluxGetCommitTimestamp(). + */ +static bool flux_xact_callback_registered = false; + +static void +FluxXactCallback(XactEvent event, void *arg) +{ + switch (event) + { + case XACT_EVENT_COMMIT: + case XACT_EVENT_PARALLEL_COMMIT: + FluxCommitTransaction(); + break; + + case XACT_EVENT_ABORT: + case XACT_EVENT_PARALLEL_ABORT: + FluxCleanupTransactionState(); + break; + + case XACT_EVENT_PREPARE: + + /* + * At PREPARE, the transaction is still "in progress" for + * visibility purposes. We must NOT clear the shared-memory + * xact_start_ts slot or decrement active_xact_count -- doing so + * would allow VACUUM to advance the oldest-active horizon past + * this prepared transaction's start timestamp, risking premature + * tuple pruning between PREPARE and COMMIT PREPARED. + * + * AtPrepare_Flux() has already relocated the pin from this + * backend's proc slot onto the prepared xact's dummy-proc slot + * (FluxPrepareReassignSlot), so it survives this backend running + * new transactions or exiting; the backend that runs COMMIT/ + * ROLLBACK PREPARED clears the dummy slot via + * FluxResolvePreparedSlot() in the FLUX 2PC callbacks. Here we + * only drop the backend-local pointer so this backend can start + * fresh transactions. + */ + MyFluxXactState = NULL; + break; + + default: + /* Pre-commit, pre-prepare -- nothing to do */ + break; + } +} + +static void +FluxInitTransactionState(void) +{ + if (MyFluxXactState != NULL) + return; + + /* Register cleanup callback on first use in this backend */ + if (!flux_xact_callback_registered) + { + RegisterXactCallback(FluxXactCallback, NULL); + flux_xact_callback_registered = true; + } + + /* Use the static per-backend struct; zero it to start fresh */ + memset(&MyFluxXactStateData, 0, sizeof(FluxTransactionState)); + MyFluxXactState = &MyFluxXactStateData; + + /* + * Stamp the transaction start timestamp (monotonic wall-clock via + * FluxGetCommitTimestamp). This is a within-transaction bookkeeping + * value only; commit visibility comes from CLOG (heap-shaped xmin/xmax). + */ + MyFluxXactState->xact_start_ts = FluxGetCommitTimestamp(); + + MyFluxXactState->xact_commit_ts = 0; + + /* No RC per-command read point captured yet this transaction. */ + MyFluxXactState->rc_read_point_cid = InvalidCommandId; + MyFluxXactState->rc_read_point_xcc = 0; + MyFluxXactState->is_serializable = (XactIsoLevel == XACT_SERIALIZABLE); + MyFluxXactState->is_read_only = true; /* Until first write */ + + /* Register in shared memory for oldest-active-timestamp tracking */ + if (FluxMvccShmem != NULL) + { + int my_slot = MyProc ? (int) GetNumberFromPGProc(MyProc) : -1; + + /* + * Write our start timestamp into our per-backend slot. This is a + * single-writer/multi-reader pattern (only we write our slot, VACUUM + * reads it), so no lock is needed ā€” just a write barrier. + */ + if (my_slot >= 0 && my_slot < FluxMvccShmem->num_xact_slots) + { + pg_write_barrier(); + FluxMvccShmem->xact_start_ts_slots[my_slot] = + MyFluxXactState->xact_start_ts; + } + + pg_atomic_fetch_add_u32(&FluxMvccShmem->active_xact_count, 1); + + /* + * If our start timestamp is older than the cached oldest, invalidate + * the cache by bumping the generation counter. + */ + if (MyFluxXactState->xact_start_ts < FluxMvccShmem->oldest_active_ts) + pg_atomic_fetch_add_u32(&FluxMvccShmem->oldest_active_generation, 1); + } +} + +/* + * Cleanup per-transaction MVCC state + */ +static void +FluxCleanupTransactionState(void) +{ + if (MyFluxXactState == NULL) + return; + + /* + * Clear our slot in shared memory. No lock needed: each backend only + * writes its own slot, and the generation counter invalidates the cached + * oldest_active_ts when needed. + */ + if (FluxMvccShmem != NULL) + { + int my_slot = MyProc ? (int) GetNumberFromPGProc(MyProc) : -1; + uint64 my_ts = MyFluxXactState->xact_start_ts; + + /* Clear our per-backend slot */ + if (my_slot >= 0 && my_slot < FluxMvccShmem->num_xact_slots) + { + FluxMvccShmem->xact_start_ts_slots[my_slot] = 0; + pg_write_barrier(); + } + + pg_atomic_fetch_sub_u32(&FluxMvccShmem->active_xact_count, 1); + + /* + * Invalidate the cached oldest_active_ts if we might have been the + * oldest. Bump the generation counter so that + * FluxGetOldestActiveTimestamp() rescans on the next call. If no + * transactions remain, advance the cached value cheaply. + */ + if (pg_atomic_read_u32(&FluxMvccShmem->active_xact_count) == 0) + { + FluxMvccShmem->oldest_active_ts = (uint64) GetCurrentTimestamp(); + pg_atomic_fetch_add_u32(&FluxMvccShmem->oldest_active_generation, 1); + } + else if (my_ts == FluxMvccShmem->oldest_active_ts) + { + /* + * Only invalidate the cache when we were the actual oldest active + * transaction. If my_ts < oldest_active_ts, the cached value was + * already advanced past us by another backend's rescan, so our + * departure cannot change the oldest. Using strict equality + * instead of <= dramatically reduces invalidation frequency under + * high concurrency. + */ + pg_atomic_fetch_add_u32(&FluxMvccShmem->oldest_active_generation, 1); + } + } + + MyFluxXactState = NULL; +} + +/* + * FluxPrepareReassignSlot -- move this backend's oldest-active-timestamp slot + * to the prepared transaction's dummy-proc slot. + * + * Called from AtPrepare_Flux() in the preparing backend. A prepared xact is + * still "active" for visibility until COMMIT/ROLLBACK PREPARED, so its start + * timestamp must keep pinning the vacuum horizon -- but the preparing backend + * is about to become free to run new transactions (and may exit entirely). + * Keying the pin by the backend's own proc slot (as the in-progress path does) + * would orphan the pin the moment the backend exits, or let the resolving + * backend -- a different proc slot -- fail to find it. Relocate the pin to + * the dummy PGPROC slot the gxact owns for its whole prepared lifetime; + * FluxResolvePreparedSlot() clears exactly that slot when the xact resolves. + * + * active_xact_count is left unchanged: the transaction is still active, the + * pin simply moved slots. No-op if this backend never opened a FLUX xact. + */ +void +FluxPrepareReassignSlot(int dummy_slot) +{ + int my_slot; + uint64 my_ts; + + if (FluxMvccShmem == NULL || MyFluxXactState == NULL) + return; + + my_slot = MyProc ? (int) GetNumberFromPGProc(MyProc) : -1; + my_ts = MyFluxXactState->xact_start_ts; + + if (dummy_slot < 0 || dummy_slot >= FluxMvccShmem->num_xact_slots) + return; + + /* Publish the pin at the dummy slot, then release the backend slot. */ + FluxMvccShmem->xact_start_ts_slots[dummy_slot] = my_ts; + pg_write_barrier(); + if (my_slot >= 0 && my_slot < FluxMvccShmem->num_xact_slots) + FluxMvccShmem->xact_start_ts_slots[my_slot] = 0; + + /* Force a horizon rescan so both moves are observed. */ + pg_atomic_fetch_add_u32(&FluxMvccShmem->oldest_active_generation, 1); +} + +/* + * FluxResolvePreparedSlot -- clear a prepared xact's dummy-proc slot. + * + * Called from the resolving backend (flux_twophase_postcommit / + * flux_twophase_postabort) once COMMIT/ROLLBACK PREPARED finishes. Mirrors + * the tail of FluxCleanupTransactionState() but operates on the gxact's dummy + * slot rather than the resolver's own proc slot. Idempotent: clearing an + * already-zero slot only decrements the active count and bumps the generation + * when the slot actually held a pin, so it is safe to call once per 2PC record. + */ +void +FluxResolvePreparedSlot(int dummy_slot) +{ + uint64 slot_ts; + + if (FluxMvccShmem == NULL) + return; + if (dummy_slot < 0 || dummy_slot >= FluxMvccShmem->num_xact_slots) + return; + + slot_ts = FluxMvccShmem->xact_start_ts_slots[dummy_slot]; + if (slot_ts == 0) + return; /* already cleared -- nothing to do */ + + FluxMvccShmem->xact_start_ts_slots[dummy_slot] = 0; + pg_write_barrier(); + + pg_atomic_fetch_sub_u32(&FluxMvccShmem->active_xact_count, 1); + + if (pg_atomic_read_u32(&FluxMvccShmem->active_xact_count) == 0) + { + FluxMvccShmem->oldest_active_ts = (uint64) GetCurrentTimestamp(); + pg_atomic_fetch_add_u32(&FluxMvccShmem->oldest_active_generation, 1); + } + else if (slot_ts <= FluxMvccShmem->oldest_active_ts) + pg_atomic_fetch_add_u32(&FluxMvccShmem->oldest_active_generation, 1); +} + +/* + * SSI conflict detection is handled by PostgreSQL's predicate.c infrastructure + * via CheckForSerializableConflictIn/Out calls in the DML and scan paths. + * The FluxCheckSerializableConflict compatibility stub has been removed. + */ + +/* + * FluxCheckForSerializableConflictOut -- detect rw-conflicts where a + * serializable reader encounters a tuple written by a concurrent transaction. + * + * This is the FLUX equivalent of HeapCheckForSerializableConflictOut. + * It determines the XID of the concurrent writer via the sLog and delegates + * to the core CheckForSerializableConflictOut() in predicate.c. + * + * Called when a serializable transaction encounters a tuple that is not + * visible to our snapshot (concurrent insert or concurrent delete/update + * that made the tuple disappear). + */ +void +FluxCheckForSerializableConflictOut(Relation relation, + FluxTupleHeader *tuple, + Buffer buffer, + Snapshot snapshot) +{ + TransactionId xid; + bool is_insert; + + if (!CheckForSerializableConflictOutNeeded(relation, snapshot)) + return; + + /* + * Determine the writer's XID. For FLUX, the tuple header doesn't store + * XIDs ā€” we get them from the sLog. + */ + xid = SLogTupleGetDirtyXid(RelationGetRelid(relation), + &tuple->t_ctid, &is_insert); + + if (!TransactionIdIsValid(xid)) + { + /* + * No in-progress writer found. The writer already committed and its + * sLog entries were cleaned up. In this case, the conflicting + * transaction committed so long ago that it's no longer tracked. No + * conflict to report ā€” analogous to heap's HEAPTUPLE_DEAD case. + */ + return; + } + + /* Skip conflicts with our own transaction */ + if (TransactionIdIsCurrentTransactionId(xid)) + return; + + /* Get top-level XID for subtransaction support */ + xid = SubTransGetTopmostTransaction(xid); + + /* Skip if too old to be a concurrent transaction */ + if (TransactionIdPrecedes(xid, TransactionXmin)) + return; + + CheckForSerializableConflictOut(relation, xid, snapshot); +} + +/* + * Commit the current transaction and assign commit timestamp. + * + * The commit timestamp is a monotonic wall-clock value (FluxGetCommitTimestamp) + * used only for page-level bookkeeping; commit visibility comes from CLOG + * (heap-shaped xmin/xmax MVCC). + */ +void +FluxCommitTransaction(void) +{ + if (MyFluxXactState == NULL) + return; + + MyFluxXactState->xact_commit_ts = FluxGetCommitTimestamp(); + + /* Update serializable horizon (only for serializable transactions) */ + if (FluxMvccShmem != NULL && MyFluxXactState->is_serializable) + { + LWLockAcquire(&FluxMvccShmem->mvcc_lock, LW_EXCLUSIVE); + FluxMvccShmem->serializable_horizon = + Min(FluxMvccShmem->serializable_horizon, + MyFluxXactState->xact_commit_ts); + LWLockRelease(&FluxMvccShmem->mvcc_lock); + } + + FluxCleanupTransactionState(); +} + +/* + * Abort the current transaction + */ +void +FluxAbortTransaction(void) +{ + if (MyFluxXactState == NULL) + return; + + FluxCleanupTransactionState(); +} + +/* + * Get snapshot timestamp for reads + */ +uint64 +FluxGetSnapshotTimestamp(Snapshot snapshot) +{ + if (IsMVCCSnapshot(snapshot)) + { + if (MyFluxXactState == NULL) + FluxInitTransactionState(); + + /* + * REPEATABLE READ / SERIALIZABLE: return transaction-start timestamp + * for a consistent point-in-time snapshot across all statements. + */ + if (IsolationUsesXactSnapshot()) + return MyFluxXactState->xact_start_ts; + + /* + * READ COMMITTED: return current timestamp so each visibility check + * sees the latest committed state. + */ + return (uint64) FluxGetCommitTimestamp(); + } + else + { + /* SnapshotAny or other non-MVCC snapshots */ + return 0; + } +} + +/* + * Check if tuple is visible to the given snapshot + */ +bool +FluxTupleVisibleToSnapshot(FluxTupleHeader *tuple, Snapshot snapshot, + Oid relid, Buffer buffer) +{ + /* Heap-shaped: forward to the single xmin/xmax visibility entry point. */ + return FluxTupleVisibleToSnapshotDual(tuple, snapshot, relid, buffer); +} + +/* + * Invalidate the cached oldest active timestamp, forcing the next call + * to FluxGetOldestActiveTimestamp() to rescan all per-backend slots. + * + * Also callable from VACUUM or any code that needs to force a refresh. + */ +void +FluxUpdateOldestActiveTimestamp(void) +{ + if (FluxMvccShmem == NULL) + return; + + pg_atomic_fetch_add_u32(&FluxMvccShmem->oldest_active_generation, 1); +} + +/* + * Per-backend cache of the oldest-active-timestamp computation. + * Avoids rescanning all per-backend slots on every call; only rescans + * when the global generation counter has been bumped. + */ +static uint32 my_oldest_active_gen = 0; +static uint64 my_oldest_active_cached = 0; + +/* + * FluxGetOldestActiveTimestamp -- return the oldest active transaction's + * start timestamp. + * + * This is the FLUX analog of PostgreSQL's GetOldestNonRemovableTransactionId. + * VACUUM uses this to determine which deleted tuples can be safely removed: + * a deleted tuple whose commit timestamp is older than this value is no + * longer visible to any running transaction and can be reclaimed. + * + * If no transactions are active, returns the current global commit timestamp, + * meaning all committed deletions are eligible for cleanup. + * + * Uses a per-backend cache that is invalidated when the global generation + * counter changes. No LWLock acquisition needed in the common case. + */ +uint64 +FluxGetOldestActiveTimestamp(void) +{ + uint32 current_gen; + + if (FluxMvccShmem == NULL) + elog(ERROR, "FLUX MVCC not initialized"); + + /* Fast path: check if our cached value is still valid */ + current_gen = pg_atomic_read_u32(&FluxMvccShmem->oldest_active_generation); + if (current_gen == my_oldest_active_gen && my_oldest_active_cached != 0) + return my_oldest_active_cached; + + /* Slow path: rescan all per-backend slots (lockless) */ + { + uint64 oldest = 0; + int i; + + pg_read_barrier(); + + for (i = 0; i < FluxMvccShmem->num_xact_slots; i++) + { + uint64 ts = FluxMvccShmem->xact_start_ts_slots[i]; + + if (ts != 0 && (oldest == 0 || ts < oldest)) + oldest = ts; + } + + if (oldest == 0) + oldest = (uint64) GetCurrentTimestamp(); + + /* Update the shared cached value (benign race with other backends) */ + FluxMvccShmem->oldest_active_ts = oldest; + + /* Cache locally */ + my_oldest_active_cached = oldest; + my_oldest_active_gen = current_gen; + + return oldest; + } +} + +/* + * Get MVCC statistics + */ +void +FluxGetMvccStats(uint64 *current_ts, uint64 *oldest_ts, int *active_xacts) +{ + if (FluxMvccShmem == NULL) + { + *current_ts = 0; + *oldest_ts = 0; + *active_xacts = 0; + return; + } + + *current_ts = (uint64) GetCurrentTimestamp(); + *oldest_ts = FluxMvccShmem->oldest_active_ts; + *active_xacts = (int) pg_atomic_read_u32(&FluxMvccShmem->active_xact_count); +} + +/* + * Shared memory exit cleanup + */ +static void +FluxShmemExit(int code, Datum arg) +{ + FluxCleanupTransactionState(); +} + + +/* + * FluxTupleHasCommittedUpdateAfter -- fork-driven lost-update conflict probe. + * + * A FLUX in-place UPDATE stamps a trailing verptr on the new on-page image + * (WS-PVS1) pointing at the UNDO-fork record it just wrote (WS-PVS3 PVS). + * That verptr is the head of the tuple's version chain: the record it + * refers to describes the update that PRODUCED the current on-page image, + * so its urec_xid IS the last committer of this tuple. Older commits have + * already been absorbed into the on-page bytes we are about to overwrite; + * they are not lost-update candidates. A single-step probe is therefore + * sufficient. + * + * Conflict iff: + * 1) the head verptr resolves (RelUndoReadRecordHeader returns true), and + * 2) urec_xid is invisible to snapshot (XidInMVCCSnapshot returns true), + * and + * 3) urec_xid is not our own xid, and + * 4) urec_xid did commit (guarding against in-progress/aborted xids that + * the writer has yet to reach the wait path for). + * + * If RelUndoReadRecordHeader returns false the record was discarded -- + * WS-PVS4's oldest_xmin discard gate guarantees the committer xid then + * precedes every live snapshot's xmin, so it is visible to every reader and + * cannot be a lost-update candidate. Safe terminator. + * + * *out_head_verptr and *out_head_xid receive the observed (verptr, xid) + * identity so the caller can pass them to FluxEpqReconcileMark before + * returning TM_Updated, and to FluxEpqReconcileMatches to skip a + * previously-bounced marker. + */ +bool +FluxTupleHasCommittedUpdateAfter(Relation rel, + const FluxTupleHeader *tuple, + Size tuple_len, + Snapshot snapshot, + TransactionId exclude_xid, + RelUndoRecPtr *out_head_verptr, + TransactionId *out_head_xid, + bool *out_inprogress) +{ + RelUndoRecPtr head; + RelUndoRecordHeader hdr; + + if (out_head_verptr != NULL) + *out_head_verptr = InvalidRelUndoRecPtr; + if (out_head_xid != NULL) + *out_head_xid = InvalidTransactionId; + if (out_inprogress != NULL) + *out_inprogress = false; + + if (snapshot == NULL || !IsMVCCSnapshot(snapshot)) + return false; + + head = FluxTupleGetVersionPtr(tuple, tuple_len); + if (!RelUndoRecPtrIsValid(head)) + return false; /* never updated -- no committed conflict */ + + if (!RelUndoReadRecordHeader(rel, head, &hdr)) + return false; /* discarded -> visible to every snapshot */ + + if (out_head_verptr != NULL) + *out_head_verptr = head; + if (out_head_xid != NULL) + *out_head_xid = hdr.urec_xid; + + if (TransactionIdIsValid(exclude_xid) && + TransactionIdEquals(hdr.urec_xid, exclude_xid)) + return false; /* our own update, not a conflict */ + + if (!TransactionIdIsValid(hdr.urec_xid)) + return false; + + /* + * XidInMVCCSnapshot(xid, snap) returns true iff the xid is NOT visible to + * snap -- i.e. in-progress or later than the snapshot's xmax. A conflict + * requires that the head committer be invisible to the writer's snapshot + * AND actually committed (an in-progress or aborted xid is handled by the + * writer's dirty-xid / abort path, not here). + */ + if (!XidInMVCCSnapshot(hdr.urec_xid, snapshot)) + return false; + + /* + * The head committer is invisible to our snapshot and is not our own xid. + * Normally it must have committed to be a lost-update conflict. But + * there is a commit-visibility window: the committer's PRE_COMMIT + * callback removes its in-progress sLog marker and clears the on-page + * UNCOMMITTED flag BEFORE RecordTransactionCommit() marks CLOG and BEFORE + * ProcArrayEndTransaction() clears it from the running list. In that + * window TransactionIdDidCommit() is still false while + * TransactionIdIsInProgress() is still true, and the sLog marker the + * writer-wait path keys off is already gone. If we returned "no + * conflict" here, a concurrent updater would clobber the just-committed + * value with no 40001 / EPQ -- a lost update. Signal the caller to wait + * on the in-flight head_xid (heap semantics: see live modifier -> wait -> + * re-check CLOG) rather than treating not-yet-committed as no-conflict. + */ + if (!TransactionIdDidCommit(hdr.urec_xid)) + { + if (out_inprogress != NULL && + TransactionIdIsInProgress(hdr.urec_xid)) + *out_inprogress = true; + return false; + } + + return true; +} + +/* + * FluxEpqReconcileMatches -- true iff we already bounced this exact + * (relid, tid, cid, head_verptr, head_xid) marker to EvalPlanQual. + * + * A strictly-newer committer stamps a NEW UNDO record with a fresh + * (blkno, offset, counter) triple, so an unchanged head verptr and xid + * imply the observed marker is the identical one EPQ already reconciled. + * Physical identity, no temporal ordering. + */ +bool +FluxEpqReconcileMatches(Snapshot snapshot, Oid relid, ItemPointer tid, + RelUndoRecPtr head_verptr, TransactionId head_xid) +{ + return MyFluxXactState != NULL && + TransactionIdIsValid(MyFluxXactState->epq_reconcile_xid) && + MyFluxXactState->epq_reconcile_relid == relid && + MyFluxXactState->epq_reconcile_cid == snapshot->curcid && + ItemPointerEquals(&MyFluxXactState->epq_reconcile_tid, tid) && + MyFluxXactState->epq_reconcile_verptr == head_verptr && + TransactionIdEquals(MyFluxXactState->epq_reconcile_xid, head_xid); +} + +/* + * FluxEpqReconcileMark -- record the (verptr, xid) marker identity we are + * bouncing to EvalPlanQual so the retry does not re-report it. + */ +void +FluxEpqReconcileMark(Snapshot snapshot, Oid relid, ItemPointer tid, + RelUndoRecPtr head_verptr, TransactionId head_xid) +{ + if (MyFluxXactState == NULL) + FluxInitTransactionState(); + + MyFluxXactState->epq_reconcile_relid = relid; + ItemPointerCopy(tid, &MyFluxXactState->epq_reconcile_tid); + MyFluxXactState->epq_reconcile_cid = snapshot->curcid; + MyFluxXactState->epq_reconcile_verptr = head_verptr; + MyFluxXactState->epq_reconcile_xid = head_xid; +} + +/* + * FluxTupleSatisfiesMVCC -- heap-compatible xmin/xmax visibility check. + * + * This is the single visibility function for FLUX's heap-shaped MVCC model. + * It mirrors HeapTupleSatisfiesMVCC: a tuple is visible to `snapshot` iff its + * inserter (t_xmin) is committed-and-visible-to-the-snapshot AND its + * deleter/updater (t_xmax) is invalid, not committed, or not visible to the + * snapshot. CLOG is the authoritative commit-status oracle; the snapshot's + * xmin/xmax/xip decide visibility of committed XIDs. The sLog is consulted + * ONLY for the command-id (cid) of the current transaction's own uncommitted + * work, which FLUX stores there instead of in the tuple header. + * + * The pre-commit "dirty read" window closes for free here: a reader resolves + * t_xmin against CLOG, and CLOG marks a transaction committed only at the + * durability point (RecordTransactionCommit flushes the commit record before + * ProcArrayEndTransaction / TransactionIdDidCommit reports true for a + * crash-safe reader path). A not-yet-committed inserter -- which includes a + * not-yet-durable one -- is simply invisible. No HLC, no flux_cts durable + * map, no timestamp rewind. + * + * `curcid` is the reader's command id for MVCC snapshots (InvalidCommandId + * for SELF/ANY, which see all of the current transaction's work). + */ + +/* + * FluxSetHintBits -- cache a CLOG commit result on the tuple as a hint bit. + * + * Mirrors heap's SetHintBits: once TransactionIdDidCommit() has confirmed a + * tuple's xmin (or xmax) committed, record it in t_flags so subsequent + * visibility checks skip the CLOG SLRU lookup entirely. This is the + * optimization that keeps a hot-set scan off the CLOG buffer LWLock (the + * profiled cause of FLUX's post-HLC write-scaling regression: without it, + * every tuple examined by every scan did a TransactionIdDidCommit -> CLOG + * lookup, serializing on the CLOG buffer lock). Non-WAL hint write: the + * flag is reconstructible from CLOG, so a torn/lost hint is harmless. + */ +static inline void +FluxSetHintBits(FluxTupleHeader *tuple, Buffer buffer, uint16 hint) +{ + /* + * Cache the CLOG result as a hint bit, but ONLY when we hold the buffer + * content lock: BufferSetHintBits16 (like MarkBufferDirtyHint) errors on + * an unlocked buffer, and FluxTupleSatisfiesMVCC is reached from some + * paths (post-VACUUM reads, replication reads, detached-tuple checks) + * where the buffer is pinned but not lock-held. If we can't safely set + * the shared hint, just skip it -- the result is still correct, the CLOG + * lookup simply isn't cached on this call; the next scan holding the lock + * will cache it. Setting the in-memory bit without the lock would race a + * concurrent writer's t_flags update, so we do nothing rather than a bare + * |=. + */ + if (BufferIsValid(buffer) && BufferIsLockedByMe(buffer)) + BufferSetHintBits16(&tuple->t_flags, + tuple->t_flags | hint, buffer); +} + +static bool +FluxTupleSatisfiesMVCC(FluxTupleHeader *tuple, Snapshot snapshot, + Oid relid, CommandId curcid, Buffer buffer) +{ + TransactionId xmin; + TransactionId xmax; + + if (tuple == NULL) + return false; + + xmin = FluxTupleGetXmin(tuple); + xmax = FluxTupleGetXmax(tuple); + + /* + * SnapshotAny / non-MVCC "see everything": show every non-deleted tuple. + * A committed-and-visible xmax still hides it (the row is gone), but for + * SnapshotAny the caller wants raw existence, so only the DELETED flag + * matters. Callers that need SNAPSHOT_DIRTY semantics handle that + * separately in flux_handler.c before reaching here. + */ + if (!IsMVCCSnapshot(snapshot)) + return !(tuple->t_flags & FLUX_TUPLE_DELETED) || + !TransactionIdIsValid(xmax); + + /* ---- xmin side: is the inserter committed-and-visible? ---- */ + if (!TransactionIdIsValid(xmin)) + return false; /* no inserter -- cannot be visible */ + + if (TransactionIdIsCurrentTransactionId(xmin)) + { + /* + * Inserted by our own transaction. Visible unless: (a) it was + * inserted by a later command (curcid ordering), or (b) the + * subtransaction that inserted it was rolled back to a savepoint -- + * FLUX records that as an SLOG_OP_ABORTED entry on the tuple's TID + * (FLUX stamps t_xmin with the TOP xid, so TransactionIdIsCurrent is + * still true for a rolled-back subxact's tuple; the sLog ABORTED + * marker is the authoritative subxact-abort signal). Consult the + * sLog while the insert is still uncommitted; a committed self-insert + * from an earlier command is unconditionally visible. + */ + if (tuple->t_flags & FLUX_TUPLE_UNCOMMITTED) + { + SLogTupleOp e[SLOG_MAX_TUPLE_OPS]; + int n = SLogTupleLookupFiltered(relid, &tuple->t_ctid, + InvalidTransactionId, + e, SLOG_MAX_TUPLE_OPS); + int i; + + for (i = 0; i < n; i++) + { + /* Savepoint rollback marked this op aborted -> invisible. */ + if (e[i].op_type == SLOG_OP_ABORTED) + return false; + /* Our own later-command insert is not yet visible. */ + if (e[i].op_type == SLOG_OP_INSERT && + TransactionIdEquals(e[i].xid, xmin) && + curcid != InvalidCommandId && e[i].cid >= curcid) + return false; + } + } + /* our insert, visible so far -- fall through to xmax check */ + } + else if (XidInMVCCSnapshot(xmin, snapshot)) + return false; /* inserter not yet visible to snapshot */ + else if (tuple->t_flags & FLUX_TUPLE_XMIN_COMMITTED) + { + /* CLOG hint already set: inserter is committed, skip the CLOG lookup */ + } + else if (!TransactionIdDidCommit(xmin)) + return false; /* inserter aborted / in-flight-not-durable */ + else + { + /* + * Inserter committed. Cache it as a hint bit (like heap's + * HEAP_XMIN_COMMITTED) so subsequent visibility checks skip the CLOG + * SLRU lookup -- this is what keeps a hot-set scan off the CLOG + * buffer LWLock. Non-WAL hint write via MarkBufferDirtyHint. + */ + FluxSetHintBits(tuple, buffer, FLUX_TUPLE_XMIN_COMMITTED); + } + + /* Inserter is committed-and-visible. Now the xmax (deleter) side. */ + + if (!TransactionIdIsValid(xmax) || + !(tuple->t_flags & (FLUX_TUPLE_DELETED | FLUX_TUPLE_UPDATED))) + { + /* + * No deleter, or the tuple carries no delete/supersede marker: the + * on-page image is live for this snapshot. + * + * NOTE (in-place-UPDATE / zheap read path): when this tuple was + * updated in place, the on-page image is the NEWEST version and its + * xmin is the updater. If that updater is invisible to `snapshot`, + * the xmin check above already returned false, and the caller + * (flux_handler.c) reconstructs the older visible version from the + * UNDO fork via FluxReconstructVisibleVersion(). So an old snapshot + * never sees the wrong (too-new) bytes: it is either hidden here and + * reconstructed, or the updater is visible and the new image is + * correct. + */ + return true; + } + + /* There is a deleter/updater xmax; resolve it heap-style. */ + if (TransactionIdIsCurrentTransactionId(xmax)) + { + /* + * Deleted/superseded by our own transaction. Invisible to us unless + * the delete happened in a later command (then we still see the row). + * Consult the sLog for the delete cid while it is uncommitted. + */ + if (curcid != InvalidCommandId && + (tuple->t_flags & FLUX_TUPLE_UNCOMMITTED)) + { + SLogTupleOp e[SLOG_MAX_TUPLE_OPS]; + int n = SLogTupleLookupFiltered(relid, &tuple->t_ctid, + xmax, e, SLOG_MAX_TUPLE_OPS); + int i; + + for (i = 0; i < n; i++) + { + if ((e[i].op_type == SLOG_OP_DELETE || + e[i].op_type == SLOG_OP_UPDATE) && e[i].cid >= curcid) + return true; /* deleted after our scan started */ + } + } + return false; /* our own delete, visible to this command */ + } + + if (XidInMVCCSnapshot(xmax, snapshot)) + return true; /* deleter not yet visible -- row still here */ + + if (tuple->t_flags & FLUX_TUPLE_XMAX_COMMITTED) + return false; /* CLOG hint: deleter committed -- gone */ + + if (!TransactionIdDidCommit(xmax)) + return true; /* deleter aborted / in-flight -- row still + * here */ + + /* deleter committed-and-visible -- gone. Cache the CLOG result. */ + FluxSetHintBits(tuple, buffer, FLUX_TUPLE_XMAX_COMMITTED); + return false; +} + +/* + * FluxTupleVisibleToSnapshotDual -- primary visibility entry point. + * + * Formerly routed to HLC vs legacy timestamp visibility; FLUX now uses one + * heap-compatible xmin/xmax model, so this just forwards to + * FluxTupleSatisfiesMVCC. The name and signature are kept because ~35 call + * sites route through here. + */ +bool +FluxTupleVisibleToSnapshotDual(FluxTupleHeader *tuple, Snapshot snapshot, + Oid relid, Buffer buffer) +{ + /* + * Only apply CID filtering for MVCC snapshots. SNAPSHOT_SELF and + * SNAPSHOT_ANY must see all of the current transaction's work. + */ + return FluxTupleSatisfiesMVCC(tuple, snapshot, relid, + (snapshot != NULL && + snapshot->snapshot_type == SNAPSHOT_MVCC) + ? snapshot->curcid : InvalidCommandId, + buffer); +} + +/* + * FluxGetOldestXminHorizon -- XID retention horizon for VACUUM/prune. + * + * Heap-shaped replacement for the former HLC oldest_snapshot_hlc horizon. A + * committed-deleted tuple whose deleter (t_xmax) precedes this horizon is + * invisible to every current and future snapshot and can be physically + * removed. Uses the standard non-removable-transaction horizon so it tracks + * the cluster's oldest snapshot xmin exactly like heap VACUUM. + */ +TransactionId +FluxGetOldestXminHorizon(Relation rel) +{ + return GetOldestNonRemovableTransactionId(rel); +} + +/* + * FluxTupleDeadToAll -- true iff a committed-deleted tuple can be reclaimed. + * + * Heap-shaped: the tuple must carry a committed DELETE marker (DELETED set, + * UNCOMMITTED clear) whose deleter XID (t_xmax) is committed and older than + * the supplied oldest-xmin horizon, so no active or future snapshot can see + * it. Replaces the old `t_commit_ts < oldest_ts` timestamp comparison, which + * is meaningless now that the t_commit_ts word holds an XID, not a timestamp. + */ +bool +FluxTupleDeadToAll(FluxTupleHeader *tuple, TransactionId oldest_xmin) +{ + TransactionId xmax; + + if (!(tuple->t_flags & FLUX_TUPLE_DELETED)) + return false; + if (tuple->t_flags & FLUX_TUPLE_UNCOMMITTED) + return false; + + xmax = FluxTupleGetXmax(tuple); + if (!TransactionIdIsValid(xmax)) + return false; + if (!TransactionIdDidCommit(xmax)) + return false; /* delete aborted / in flight: keep */ + + return TransactionIdPrecedes(xmax, oldest_xmin); +} diff --git a/src/backend/access/flux/flux_operations.c b/src/backend/access/flux/flux_operations.c new file mode 100644 index 0000000000000..468e323429dac --- /dev/null +++ b/src/backend/access/flux/flux_operations.c @@ -0,0 +1,7531 @@ +/*------------------------------------------------------------------------- + * + * flux_operations.c + * FLUX table manipulation operations + * + * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/access/flux/flux_operations.c + * + * NOTES + * This implements the remaining table manipulation operations for + * FLUX storage manager including insert, update, delete, and + * various DDL operations. + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/genam.h" +#include "access/heapam.h" +#include "access/flux.h" +#include "access/flux_dirtymap.h" +#include "access/slog.h" +#include "access/twophase.h" +#include "access/twophase_rmgr.h" +#include "access/flux_undo.h" +#include "access/flux_xlog.h" +#include "access/tableam.h" +#include "access/undobuffer.h" +#include "access/xlog.h" +#include "access/tidstore.h" +#include "access/xact.h" +#include "access/xactundo.h" +#include "access/xloginsert.h" +#include "catalog/catalog.h" +#include "catalog/index.h" +#include "catalog/storage.h" +#include "commands/vacuum.h" +#include "executor/executor.h" +#include "pgstat.h" +#include "storage/bufmgr.h" +#include "storage/freespace.h" +#include "storage/latch.h" +#include "storage/read_stream.h" +#include "storage/lmgr.h" +#include "storage/predicate.h" +#include "storage/procarray.h" +#include "storage/smgr.h" +#include "utils/builtins.h" +#include "utils/datum.h" +#include "utils/injection_point.h" +#include "utils/rel.h" +#include "utils/timestamp.h" +#include "utils/wait_event.h" +#include "storage/bufpage.h" +#include "miscadmin.h" + +/* + * Maximum overflow pointers per tuple for VACUUM overflow cleanup. + * This limits memory usage during VACUUM and is conservative since most + * tuples won't have overflow data. 128 is sufficient for typical workloads. + */ +#define MAX_OVERFLOW_PTRS_PER_TUPLE 128 + +/* Function prototypes for locking */ +extern bool FluxLockTuple(Relation rel, ItemPointer tid, LockTupleMode mode, + bool wait, bool *have_tuple_lock); +extern void FluxUnlockTuple(Relation rel, ItemPointer tid, LockTupleMode mode); +extern void FluxLockPage(Relation rel, BlockNumber blkno, LOCKMODE mode); +extern void FluxUnlockPage(Relation rel, BlockNumber blkno, LOCKMODE mode); + +/* sLog transaction callback prototypes */ +static void FluxSLogXactCallback(XactEvent event, void *arg); +static void FluxSLogSubXactCallback(SubXactEvent event, + SubTransactionId mySubid, + SubTransactionId parentSubid, + void *arg); + + +/* + * In-place update statistics counters. + * + * These track the effectiveness of FLUX's in-place update optimization + * across the lifetime of the backend. They are exposed via + * FluxGetUpdateStats() for monitoring. + */ +static int64 flux_stat_in_place_updates = 0; +static int64 flux_stat_out_of_place_updates = 0; +static int64 flux_stat_defrag_triggered_updates = 0; + +/* Whether sLog transaction callbacks have been registered for this backend */ +static bool flux_slog_callbacks_registered = false; + +/* + * GUC: skip commit-time page re-visits (lazy clear of UNCOMMITTED flags). + * + * Default off. Clearing FLUX_TUPLE_UNCOMMITTED is a pure hint (commit + * visibility comes from CLOG via heap-shaped xmin/xmax); the lazy path clears + * the flag at the next visibility check, while the eager path visits each + * modified page at PRE_COMMIT so later readers can skip the sLog fast-path + * lookup. + */ +bool flux_lazy_uncommitted_clear = false; + +/* + * FluxGetUpdateStats - Return in-place update statistics + * + * Fills in the provided counters with the current backend-local statistics. + */ +void +FluxGetUpdateStats(int64 *in_place, int64 *out_of_place, int64 *defrag_triggered) +{ + if (in_place) + *in_place = flux_stat_in_place_updates; + if (out_of_place) + *out_of_place = flux_stat_out_of_place_updates; + if (defrag_triggered) + *defrag_triggered = flux_stat_defrag_triggered_updates; +} + + +/* + * FluxPagePruneOpt -- opportunistic dead-tuple cleanup on a page. + * + * This is the FLUX equivalent of heap_page_prune_opt(). It is called + * during normal DML operations (insert, update) and sequential scans + * when a page looks like it might benefit from cleanup. The goal is to + * reclaim space from deleted tuples without waiting for VACUUM. + * + * The caller must hold a pin on the buffer but must NOT hold a lock on it. + * We attempt a conditional (non-blocking) exclusive lock; if we cannot + * get it, we return immediately -- this is best-effort cleanup. + * + * Returns the number of tuples pruned. + */ +int +FluxPagePruneOpt(Relation relation, Buffer buffer) +{ + Page page; + FluxPageOpaque opaque; + OffsetNumber offnum; + OffsetNumber maxoff; + uint64 oldest_ts; + TransactionId oldest_xmin_prune; + int ndead = 0; + Size minfree; + + /* Cannot write WAL during recovery, so skip */ + if (RecoveryInProgress()) + return 0; + + page = BufferGetPage(buffer); + + /* Skip if page is not initialized */ + if (PageIsNew(page)) + return 0; + + /* + * Validate page header before accessing special space. During recovery or + * after crashes, pages may have invalid headers. Skip pruning if the page + * header looks corrupt. + */ + { + PageHeader phdr = (PageHeader) page; + + if (phdr->pd_special < SizeOfPageHeaderData || + phdr->pd_special > BLCKSZ) + return 0; + } + + /* + * Quick check without lock: does the page look like it needs pruning? The + * FLUX_PAGE_DEFRAG_NEEDED flag is set by delete and update operations + * when a tuple is marked as deleted. If no deletions have occurred on + * this page, there is nothing to clean up. + */ + opaque = FluxPageGetOpaque(page); + if (!(FluxPageGetFlags(opaque) & FLUX_PAGE_DEFRAG_NEEDED)) + return 0; + + /* + * Heuristic: only prune if the page's free space is below a threshold. + * This avoids spending cycles on pages that already have plenty of room. + * We use 10% of BLCKSZ as the minimum, matching heap's approach. Reading + * pd_lower/pd_upper without a lock is slightly racy but acceptable for a + * heuristic. + */ + minfree = BLCKSZ / 10; + if (PageGetFreeSpace(page) >= minfree && + !(FluxPageGetFlags(opaque) & FLUX_PAGE_FULL)) + return 0; + + /* + * Try to get an exclusive lock without blocking. If the page is busy, + * skip it -- we will get another chance later. + */ + if (!ConditionalLockBufferForCleanup(buffer)) + return 0; + + /* + * Re-check under lock: the page state may have changed while we were + * acquiring the lock (or another backend may have pruned it). + */ + page = BufferGetPage(buffer); + opaque = FluxPageGetOpaque(page); + if (!(FluxPageGetFlags(opaque) & FLUX_PAGE_DEFRAG_NEEDED)) + { + LockBuffer(buffer, BUFFER_LOCK_UNLOCK); + return 0; + } + + /* + * oldest_xmin_prune is the reclamation gate (XID horizon, + * FluxTupleDeadToAll below). oldest_ts is NOT a decision input here: it + * is carried only into the DEFRAG WAL record's cosmetic page commit-ts + * word (FluxXLogDefrag -> FluxPageSetCommitTs). + */ + oldest_ts = FluxGetOldestActiveTimestamp(); + oldest_xmin_prune = FluxGetOldestXminHorizon(relation); + maxoff = PageGetMaxOffsetNumber(page); + for (offnum = FirstOffsetNumber; offnum <= maxoff; offnum++) + { + ItemId itemid = PageGetItemId(page, offnum); + FluxTupleHeader *tuple_hdr; + + if (!ItemIdIsNormal(itemid)) + continue; + + /* Skip overflow records */ + if (FluxIsOverflowRecordInline(PageGetItem(page, itemid), + ItemIdGetLength(itemid))) + continue; + + tuple_hdr = (FluxTupleHeader *) PageGetItem(page, itemid); + + /* + * A deleted tuple can be pruned if: - UNCOMMITTED is NOT set (the + * inserting/deleting xact committed) - commit_ts is older than the + * oldest active snapshot + * + * If UNCOMMITTED is still set, the transaction is still in progress + * (or aborted but not yet cleaned up) -- skip it. + */ + if (FluxTupleDeadToAll(tuple_hdr, oldest_xmin_prune)) + { + ndead++; + } + } + + if (ndead == 0) + { + /* + * No reclaimable dead tuples. Clear the defrag flag so we don't + * recheck this page on every access until a new deletion occurs. + */ + FluxPageClearFlag(opaque, FLUX_PAGE_DEFRAG_NEEDED); + LockBuffer(buffer, BUFFER_LOCK_UNLOCK); + return 0; + } + + /* + * We have reclaimable dead tuples. Mark them LP_DEAD (reclaiming their + * storage) and defragment the page to compact free space. + * + * CRITICAL: opportunistic pruning must set LP_DEAD, never LP_UNUSED. A + * deleted FLUX tuple may still be referenced by index entries that only + * VACUUM (after index bulk-delete) is allowed to remove. LP_DEAD + * reclaims the tuple's storage while reserving its line pointer so the + * TID cannot be recycled by a later INSERT (PageAddItemExtended only + * reuses LP_UNUSED slots). Recycling a TID whose index entries still + * exist would let an index scan return the unrelated tuple later placed + * in that slot. Only VACUUM Phase III converts LP_DEAD -> LP_UNUSED, + * after Phase II has removed the corresponding index entries. This + * mirrors heap pruning, which sets LP_DEAD and defers LP_UNUSED to + * post-index-cleanup VACUUM. + */ + START_CRIT_SECTION(); + + maxoff = PageGetMaxOffsetNumber(page); + for (offnum = FirstOffsetNumber; offnum <= maxoff; offnum++) + { + ItemId itemid = PageGetItemId(page, offnum); + FluxTupleHeader *tuple_hdr; + + if (!ItemIdIsNormal(itemid)) + continue; + + /* Skip overflow records */ + if (FluxIsOverflowRecordInline(PageGetItem(page, itemid), + ItemIdGetLength(itemid))) + continue; + + tuple_hdr = (FluxTupleHeader *) PageGetItem(page, itemid); + + if (FluxTupleDeadToAll(tuple_hdr, oldest_xmin_prune)) + { + ItemIdSetDead(itemid); + } + } + + FluxPageDefragment(page); + + MarkBufferDirty(buffer); + + /* WAL-log the defragmentation using a proper defrag record */ + if (RelationNeedsWAL(relation)) + { + XLogRecPtr recptr; + + /* + * Use FluxXLogDefrag (not FluxXLogInitPage). INIT_PAGE with + * REGBUF_WILL_INIT would zero the page during redo, losing all live + * tuples. The defrag record uses REGBUF_STANDARD which stores a Full + * Page Image, preserving the page contents. + */ + recptr = FluxXLogDefrag(relation, buffer, NULL, 0, oldest_ts); + PageSetLSN(page, recptr); + } + + END_CRIT_SECTION(); + + /* Update FSM with the reclaimed free space */ + FluxRecordFreeSpace(relation, BufferGetBlockNumber(buffer), + PageGetFreeSpace(page)); + + LockBuffer(buffer, BUFFER_LOCK_UNLOCK); + + return true; +} + + +/* + * Insert a tuple into a FLUX table + */ +void +flux_tuple_insert(Relation relation, TupleTableSlot *slot, CommandId cid, + uint32 options, BulkInsertState bistate) +{ + FluxTuple flux_tuple; + Buffer buffer; + Page page; + OffsetNumber offnum; + BlockNumber target_block; + Size tuple_size; + ItemPointer tid = &slot->tts_tid; + uint64 current_ts; + FluxOverflowBuffers overflow_buffers; + FluxLogicalImage insert_logical_img; + Buffer undo_buffer = InvalidBuffer; + RelUndoRecPtr undo_ptr = InvalidRelUndoRecPtr; + Size saveFreeSpace; + int i; + Datum toast_values[MaxTupleAttributeNumber]; + bool toast_isnull[MaxTupleAttributeNumber]; + ToastAttrInfo toast_attr[MaxTupleAttributeNumber]; + ToastTupleContext ttc; + bool toasted = false; + + slot_getallattrs(slot); + + /* + * Get current timestamp for the page-level commit-ts bookkeeping word. + * + * IMPORTANT: Get transaction timestamp here, BEFORE entering critical + * section, because FluxGetTransactionTimestamp() may need to allocate + * memory to initialize transaction state. + * + * CRITICAL FIX: Use FluxGetTransactionTimestamp() for both current_ts and + * xact_ts to ensure consistency. The inserted tuple will be visible + * within the same transaction because the snapshot timestamp will match + * the tuple's commit timestamp. + */ + (void) FluxGetTransactionTimestamp(); + + current_ts = (uint64) FluxGetDmlTimestamp(); + + /* + * Create FLUX tuple from slot. Use the overflow-aware variant which will + * store large attributes (> FLUX_OVERFLOW_THRESHOLD) in overflow records + * on normal data pages, replacing them with compact inline overflow + * pointers. + * + * Overflow buffers are kept pinned for atomic WAL logging inside the + * critical section below. + */ + overflow_buffers.count = 0; + + /* + * FLUX has no on-page overflow: TOAST any wide varlena columns into the + * relation's standard heap TOAST table before forming the tuple, exactly + * like heap. On return toast_values[] holds the (possibly externalized) + * datums that FluxFormTuple stores verbatim. + */ + memcpy(toast_values, slot->tts_values, + RelationGetDescr(relation)->natts * sizeof(Datum)); + memcpy(toast_isnull, slot->tts_isnull, + RelationGetDescr(relation)->natts * sizeof(bool)); + flux_toast_tuple(relation, toast_values, toast_isnull, NULL, NULL, + &ttc, toast_attr, &toasted, 0); + + flux_tuple = FluxFormTuple(RelationGetDescr(relation), + toast_values, + toast_isnull, + relation, + &overflow_buffers); + + /* Set MVCC fields (heap-shaped xmin/xmax) */ + flux_tuple->t_data->t_commit_ts = 0; /* t_xmax = InvalidTransactionId */ + + /* + * Stamp the inserter XID as t_xmin. Visibility resolves t_xmin against + * CLOG + the reader's snapshot (FluxTupleSatisfiesMVCC), so an + * uncommitted insert is automatically invisible to other snapshots. The + * FLUX_TUPLE_UNCOMMITTED flag is kept for the sLog write-conflict and + * defrag paths, but it no longer drives read visibility. + */ + flux_tuple->t_data->t_flags |= FLUX_TUPLE_UNCOMMITTED; + flux_tuple->t_data->t_xmin = GetCurrentTransactionId(); /* subxid: heap-shaped, + * so savepoint rollback + * marks it aborted in + * CLOG */ + + /* + * WS-PVS1: the version-chain head lives in the fixed header field + * t_verptr (counted by FLUX_TUPLE_OVERHEAD), so every tuple carries it + * from birth with no on-page growth. Stamp InvalidRelUndoRecPtr here; + * the first UPDATE replaces it with a real chain head. Readers treat an + * invalid head as "no history" (flux_pvs.c), so a never-updated row is + * unaffected. Because the slot width never changes on UPDATE, the first + * UPDATE is always a same-length overwrite (Strategy 1 / CAS fast path). + */ + flux_tuple->t_data->t_flags |= FLUX_TUPLE_HAS_VERSION_PTR; + FluxTupleSetVersionPtr(flux_tuple->t_data, flux_tuple->t_len, + InvalidRelUndoRecPtr); + tuple_size = flux_tuple->t_len; + + /* Ensure relation storage exists */ + RelationGetSmgr(relation); + + /* + * Find a page with enough free space. + * + * Like heap (RelationGetBufferForTuple), we first try the page we last + * inserted into, cached per-relation in the relcache via + * RelationSetTargetBlock(). Only if we have no cached target do we ask + * the FSM. This avoids an fsm_search() + RecordPageWithFreeSpace() on + * every append: on an append-heavy table (e.g. pgbench_history) every + * backend otherwise pounds the same tail FSM pages, saturating the FSM + * buffer-header spinlock (LockBufHdr) at high concurrency. The cache is + * only a hint -- a stale or too-full target just falls through to the FSM + * below and to the have_page free-space recheck. + * + * The cached-target fast path is used only when the tuple has no overflow + * data: with overflow, FluxFormTupleWithOverflow has already chosen and + * locked pages and the target_block/overflow-buffer interplay below + * assumes the FSM was consulted, so that path stays on the FSM. + * + * Account for fill factor: reserve space for future in-place updates. + */ + saveFreeSpace = RelationGetTargetPageFreeSpace(relation, + FLUX_DEFAULT_FILLFACTOR); + { + target_block = InvalidBlockNumber; + if (overflow_buffers.count == 0) + target_block = RelationGetTargetBlock(relation); + + if (target_block == InvalidBlockNumber) + target_block = FluxGetPageWithFreeSpace(relation, + tuple_size + saveFreeSpace); + } + + if (target_block == InvalidBlockNumber) + { + /* Clean up overflow buffers before throwing error */ + for (i = 0; i < overflow_buffers.count; i++) + { + UnlockReleaseBuffer(overflow_buffers.buffers[i].buffer); + pfree(overflow_buffers.buffers[i].record_data); + } + elog(ERROR, "FLUX failed to allocate page for tuple insertion"); + } + + /* + * Pre-allocate WAL buffer space BEFORE acquiring the data buffer lock. + * XLogEnsureRecordSpace() may allocate memory, so it MUST be called + * outside the critical section. + * + * rdata slots needed: MAX_OVERFLOW_BUFFERS * 2 (header + data per + * overflow record) + 2 (xl_flux_insert header + tuple data) + */ + if (RelationNeedsWAL(relation)) + XLogEnsureRecordSpace(XLR_MAX_BLOCK_ID, 3 + MAX_OVERFLOW_BUFFERS * 2); + + /* + * Check if target_block is already locked in overflow_buffers from + * FluxFormTupleWithOverflow. If FSM returns the same block for both + * overflow storage and main tuple storage, we must reuse that buffer to + * avoid double-locking. + */ + buffer = InvalidBuffer; + for (i = 0; i < overflow_buffers.count; i++) + { + if (BufferGetBlockNumber(overflow_buffers.buffers[i].buffer) == target_block) + { + buffer = overflow_buffers.buffers[i].buffer; + break; + } + } + + /* + * Read and lock the target page only if we don't already have it locked + * from overflow processing. + */ + if (!BufferIsValid(buffer)) + { + buffer = ReadBuffer(relation, target_block); + LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); + } + page = BufferGetPage(buffer); + + /* + * Verify the page has room for the tuple PLUS the fillfactor reservation, + * so in-place UPDATE growth stays possible (heap applies the same + * targetFreeSpace = len + saveFreeSpace bar). A cached target block or a + * stale FSM answer that lacks the slack falls through to FSM retry / + * relation extension below. + */ + if (PageGetFreeSpace(page) < tuple_size + saveFreeSpace) + { + bool buffer_is_from_overflow = false; + + /* + * Check if this buffer is from overflow_buffers. If so, we must NOT + * unlock it for pruning, as overflow_buffers expects all its buffers + * to remain locked until the critical section. + */ + for (i = 0; i < overflow_buffers.count; i++) + { + if (overflow_buffers.buffers[i].buffer == buffer) + { + buffer_is_from_overflow = true; + break; + } + } + + /* + * Page doesn't have enough space. Try opportunistic pruning to + * reclaim space from dead tuples before falling back to the FSM. We + * must release our lock first since FluxPagePruneOpt() takes its own + * conditional lock. + * + * IMPORTANT: Skip pruning if buffer is from overflow_buffers, as we + * must keep those buffers locked. + */ + if (!buffer_is_from_overflow) + { + LockBuffer(buffer, BUFFER_LOCK_UNLOCK); + if (FluxPagePruneOpt(relation, buffer)) + { + /* Pruning freed space -- re-lock and check again */ + LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); + page = BufferGetPage(buffer); + if (PageGetFreeSpace(page) >= tuple_size + saveFreeSpace) + goto have_page; + /* Still not enough after pruning, fall through to FSM retry */ + } + + /* + * FSM information was stale or pruning didn't help. Update and + * retry. + */ + LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); + FluxRecordFreeSpace(relation, target_block, PageGetFreeSpace(page)); + UnlockReleaseBuffer(buffer); + } + else + { + /* + * Buffer is from overflow_buffers and can't be pruned or + * released. This means FSM returned an overflow page for the main + * tuple, which doesn't have enough space. This should be rare but + * can happen if overflow pages filled up during tuple formation. + * + * Update FSM for this page, then get a DIFFERENT page. We must + * retry until we find a page that's NOT in overflow_buffers. + */ + FluxRecordFreeSpace(relation, target_block, PageGetFreeSpace(page)); + } + + /* + * Retry with updated FSM, excluding blocks in overflow_buffers. Keep + * trying until we find a suitable page that we don't already have + * locked for overflow storage. + */ + for (;;) + { + target_block = FluxGetPageWithFreeSpace(relation, + tuple_size + saveFreeSpace); + if (target_block == InvalidBlockNumber) + { + /* Clean up overflow buffers before throwing error */ + for (i = 0; i < overflow_buffers.count; i++) + { + UnlockReleaseBuffer(overflow_buffers.buffers[i].buffer); + pfree(overflow_buffers.buffers[i].record_data); + } + elog(ERROR, "FLUX failed to allocate page for tuple insertion after retry"); + } + + /* + * Check if target_block is already locked in overflow_buffers. If + * so, skip it and try again - we need a different page. + */ + buffer = InvalidBuffer; + for (i = 0; i < overflow_buffers.count; i++) + { + if (BufferGetBlockNumber(overflow_buffers.buffers[i].buffer) == target_block) + { + /* + * This block is already used for overflow - mark FSM and + * retry + */ + FluxRecordFreeSpace(relation, target_block, 0); + buffer = InvalidBuffer; + break; + } + } + + /* + * If we found a block not in overflow_buffers, check if it has + * space + */ + if (i >= overflow_buffers.count) + { + buffer = ReadBuffer(relation, target_block); + LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); + page = BufferGetPage(buffer); + + /* + * Verify the page actually has enough space. If not, update + * FSM and retry. + */ + if (PageGetFreeSpace(page) >= tuple_size + saveFreeSpace) + { + /* Found a suitable page - exit retry loop */ + break; + } + else + { + /* FSM was wrong - update it and retry */ + FluxRecordFreeSpace(relation, target_block, PageGetFreeSpace(page)); + UnlockReleaseBuffer(buffer); + buffer = InvalidBuffer; + /* Continue outer loop to try again */ + } + } + } + } + +have_page: + + /* + * SSI: check for rw-conflict in. An INSERT may conflict with a + * concurrent serializable transaction that holds a relation-level or + * page-level predicate lock (e.g., from a range scan that would have + * included this new tuple). Pass NULL tid since the tuple doesn't exist + * yet ā€” only relation-level and page-level locks are checked. + */ + CheckForSerializableConflictIn(relation, NULL, BufferGetBlockNumber(buffer)); + + /* + * Ensure the current transaction has an XID assigned BEFORE entering the + * critical section. GetCurrentTransactionId() may call + * XactLockTableInsert() which acquires a lock and allocates memory -- + * both forbidden in a critical section. Most inserts already have an XID + * by now (assigned during the unique-index check), but inserting a NULL + * into a UNIQUE column skips that check, so the first assignment can + * otherwise land inside the crit section below. + * + * An assigned XID is also required for correctness: WAL records without + * an attached xid cannot be decoded into a logical replication stream + * (ReorderBuffer groups changes by xid and emits no commit record for + * InvalidTransactionId), and RecordTransactionCommit() would treat the + * transaction as read-only and skip the WAL flush. + */ + (void) GetCurrentTransactionId(); + + /* + * Final free-space check before entering the critical section. + * PageGetFreeSpace may have been optimistic (alignment, line pointer + * overhead). If the page can't actually fit the tuple, release it, + * update FSM, and extend the relation instead. This prevents the PANIC + * that would otherwise fire inside the critical section. + */ + offnum = FluxPageAddTuple(page, flux_tuple, tuple_size); + if (offnum == InvalidOffsetNumber) + { + /* Page too full despite FSM claim ā€” record actual free space */ + FluxRecordFreeSpace(relation, BufferGetBlockNumber(buffer), + PageGetFreeSpace(page)); + UnlockReleaseBuffer(buffer); + + /* Extend the relation to get a guaranteed-empty page */ + buffer = ReadBuffer(relation, P_NEW); + LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); + page = BufferGetPage(buffer); + FluxInitPage(page, BufferGetPageSize(buffer)); + + offnum = FluxPageAddTuple(page, flux_tuple, tuple_size); + if (offnum == InvalidOffsetNumber) + elog(ERROR, "failed to add FLUX tuple to new empty page (tuple_size=%zu)", + (Size) tuple_size); + } + + /* + * Prepare the heap-format logical-decoding image before entering the + * critical section (it calls palloc/heap_form_tuple, which are forbidden + * inside a crit section). No-op unless the relation is logically logged. + */ + FluxXLogPrepareLogicalImage(relation, flux_tuple, &insert_logical_img); + + /* + * Per-relation UNDO: reserve space for an INSERT UNDO record before the + * critical section, because RelUndoReserve() may extend the UNDO fork and + * error out. This is done after acquiring the data buffer lock to keep a + * consistent lock ordering (data buffer -> UNDO buffer) with the UPDATE + * and DELETE paths. + */ + if (smgrexists(RelationGetSmgr(relation), RELUNDO_FORKNUM)) + undo_ptr = RelUndoReserve(relation, + SizeOfRelUndoRecordHeader + + sizeof(RelUndoInsertPayload), + &undo_buffer); + + /* NO EREPORT(ERROR) from here till changes are logged */ + START_CRIT_SECTION(); + + /* Set the tuple's TID */ + ItemPointerSet(tid, BufferGetBlockNumber(buffer), offnum); + flux_tuple->t_self = *tid; + slot->tts_tableOid = RelationGetRelid(relation); + + /* + * Set the on-disk tuple's t_ctid to point to itself. This is needed for + * update chains and cross-page defragmentation, which check whether + * t_ctid == self to detect tuples that are not part of an update chain. + */ + { + ItemId inserted_itemid = PageGetItemId(page, offnum); + FluxTupleHeader *inserted_hdr = (FluxTupleHeader *) PageGetItem(page, inserted_itemid); + + ItemPointerSet(&inserted_hdr->t_ctid, BufferGetBlockNumber(buffer), offnum); + } + + + /* + * Update page opaque fields BEFORE WAL logging. When + * XLogRegisterBuffer() takes a Full Page Write (FPW), the page image must + * already contain the same opaque values that REDO will set during + * replay. Otherwise WAL consistency checking will detect a mismatch + * between the FPW and the page produced by REDO, causing a FATAL + * "inconsistent page found" error on the standby. + * + * This matches the fix applied to FluxXLogInitPage in flux_fsm.c. + */ + { + FluxPageOpaque phdr = FluxPageGetOpaque(page); + + FluxPageSetCommitTs(phdr, Max(FluxPageGetCommitTs(phdr), current_ts)); + } + + MarkBufferDirty(buffer); + + /* Log the insertion with all overflow buffers atomically */ + if (RelationNeedsWAL(relation)) + { + XLogRecPtr recptr = FluxXLogInsert(relation, buffer, offnum, + flux_tuple, current_ts, + &overflow_buffers, + &insert_logical_img, + false); + + PageSetLSN(page, recptr); + } + + END_CRIT_SECTION(); + + FluxXLogReleaseLogicalImage(&insert_logical_img); + + /* + * Release all overflow buffers and free their cached data. + * + * IMPORTANT: Due to spatial locality optimization, an overflow buffer + * might be the SAME as the main buffer or as another overflow buffer + * (when overflow data is placed on the same page). Skip releasing buffers + * that were already released. The main buffer is NOT released yet ā€” + * only overflow buffers that differ from it. + */ + for (i = 0; i < overflow_buffers.count; i++) + { + Buffer ovf_buf = overflow_buffers.buffers[i].buffer; + bool already_released = (ovf_buf == buffer); + int j; + + /* Check if this buffer was already released by a prior overflow entry */ + for (j = 0; j < i && !already_released; j++) + { + if (overflow_buffers.buffers[j].buffer == ovf_buf) + already_released = true; + } + + if (!already_released) + UnlockReleaseBuffer(ovf_buf); + pfree(overflow_buffers.buffers[i].record_data); + } + + /* + * Finish the per-relation UNDO record now that the insert is complete. + * Write the UNDO record with the inserted TID and register it with the + * transaction system so that rollback can find and apply it. + * + * IMPORTANT: This must happen BEFORE FluxVMUpdateForInsert to maintain + * consistent buffer lock ordering across forks. The UPDATE and DELETE + * paths already follow this ordering. + */ + + /* + * Per-relation UNDO: write the INSERT record now that the insert is + * complete and register it so rollback can find and reverse it. The + * record stores the inserted TID range; rollback marks the line pointer + * unused. + */ + if (RelUndoRecPtrIsValid(undo_ptr)) + { + RelUndoRecordHeader undo_hdr; + RelUndoInsertPayload undo_payload; + + undo_hdr.urec_type = RELUNDO_INSERT; + undo_hdr.urec_len = SizeOfRelUndoRecordHeader + + sizeof(RelUndoInsertPayload); + undo_hdr.urec_xid = GetCurrentTransactionId(); + undo_hdr.urec_prevundorec = GetPerRelUndoPtr(RelationGetRelid(relation)); + undo_hdr.info_flags = 0; + undo_hdr.tuple_len = 0; + + undo_payload.firsttid = *tid; + undo_payload.endtid = *tid; + + RelUndoFinish(relation, undo_buffer, undo_ptr, &undo_hdr, + &undo_payload, sizeof(RelUndoInsertPayload)); + + RegisterPerRelUndo(RelationGetRelid(relation), undo_ptr); + } + + /* + * Clear visibility map bits while buffer is still locked. This is + * usually a fast no-op for newly created tables (no VM fork yet). + */ + FluxVMUpdateForInsert(relation, flux_tuple->t_data, buffer); + + /* + * Save free space while we still have the buffer locked, then release the + * data buffer as soon as possible to reduce contention on hot pages. The + * remaining operations (FSM update, sLog registration) don't need the + * data buffer lock. + */ + { + Size saved_free_space = PageGetFreeSpace(page); + BlockNumber saved_blkno = BufferGetBlockNumber(buffer); + + UnlockReleaseBuffer(buffer); + + /* Update FSM with remaining free space on the page */ + FluxRecordFreeSpace(relation, saved_blkno, saved_free_space); + + /* + * Cache this block as the next insert target (heap's + * RelationSetTargetBlock pattern) so the following append skips the + * FSM search entirely. Only on the no-overflow path -- the overflow + * path deliberately stays on the FSM (see the target-block lookup + * above). If the page is now too full for the next tuple, the + * have_page free-space recheck falls back to the FSM. + */ + if (overflow_buffers.count == 0) + RelationSetTargetBlock(relation, saved_blkno); + } + + /* + * Lightweight subtransaction tracking for savepoint rollback. + * + * We do NOT create a full shared sLog entry here by default (that caused + * "out of shared memory" during bulk inserts with 100K+ rows). Instead, + * we record (tid, xid, subxid) in the per-backend local list only. If a + * savepoint is rolled back, SLogTupleRemoveBySubXid will find the + * matching local entries and create a shared sLog ABORTED entry at that + * time. + * + * Speculative inserts (ON CONFLICT) are handled by the separate + * flux_tuple_insert_speculative() function, which still registers full + * sLog entries for the speculative token. + */ + FluxEnsureSLogCallbacks(); + SLogTupleTrackLocalOnly(RelationGetRelid(relation), tid, + GetTopTransactionId(), + GetCurrentSubTransactionId()); + + FluxFreeTuple(flux_tuple); + + /* Release toasting temporaries now that the tuple is durably stored. */ + if (toasted) + flux_toast_cleanup(&ttc); + + pgstat_count_heap_insert(relation, 1); +} + +/* + * Delete a tuple from a FLUX table with proper tombstone marking + */ +TM_Result +flux_tuple_delete(Relation relation, ItemPointer tid, CommandId cid, + uint32 options, Snapshot snapshot, Snapshot crosscheck, + bool wait, TM_FailureData *tmfd) +{ + BlockNumber blkno; + OffsetNumber offnum; + Buffer buffer; + Page page; + ItemId itemid; + FluxTupleHeader *tuple_hdr; + uint64 current_ts; + bool have_tuple_lock; + FluxTuple old_tuple_for_delete_wal; + FluxLogicalImage delete_logical_img; + Buffer del_undo_buffer = InvalidBuffer; + RelUndoRecPtr del_undo_ptr = InvalidRelUndoRecPtr; + TransactionId del_hint_xid; + + /* Extract block and offset from TID */ + blkno = ItemPointerGetBlockNumber(tid); + offnum = ItemPointerGetOffsetNumber(tid); + + /* Validate TID range */ + if (blkno >= RelationGetNumberOfBlocks(relation)) + return TM_Invisible; + + /* Read the page containing the tuple */ + buffer = ReadBuffer(relation, blkno); + + /* + * Lock the buffer exclusively. The exclusive buffer lock is sufficient + * to prevent concurrent modifications ā€” heavyweight tuple locks + * (FluxLockTuple) are only needed for SELECT FOR UPDATE/SHARE, not for + * regular DML. This matches heap's approach for UPDATE/DELETE. + */ + have_tuple_lock = false; + LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); + page = BufferGetPage(buffer); + + /* Validate offset number */ + if (offnum < FirstOffsetNumber || offnum > PageGetMaxOffsetNumber(page)) + { + UnlockReleaseBuffer(buffer); + return TM_Invisible; + } + + /* Get the item */ + itemid = PageGetItemId(page, offnum); + if (!ItemIdIsNormal(itemid)) + { + UnlockReleaseBuffer(buffer); + return TM_Invisible; + } + + tuple_hdr = (FluxTupleHeader *) PageGetItem(page, itemid); + + /* + * Check if tuple is already deleted (tombstone exists). A DELETED flag + * can mean either a committed delete OR an in-progress delete by a + * concurrent transaction (which now also leaves UNCOMMITTED set). + * Distinguish the two: if another in-progress transaction owns the + * delete, wait for it and retry, matching heap's behavior where the + * second DELETE blocks behind the first. Only report TM_Deleted once the + * delete is genuinely committed (or ours). + */ + if (tuple_hdr->t_flags & FLUX_TUPLE_DELETED) + { + TransactionId del_xid = InvalidTransactionId; + bool del_is_insert = false; + + if (tuple_hdr->t_flags & FLUX_TUPLE_UNCOMMITTED) + del_xid = SLogTupleGetDirtyXid(RelationGetRelid(relation), + tid, &del_is_insert); + + if (wait && TransactionIdIsValid(del_xid) && + !TransactionIdIsCurrentTransactionId(del_xid) && + !del_is_insert) + { + TransactionId wait_xid = del_xid; + + UnlockReleaseBuffer(buffer); + XactLockTableWait(wait_xid, relation, tid, XLTW_Delete); + + /* Re-read after waking; the delete committed or aborted. */ + buffer = ReadBuffer(relation, blkno); + LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); + page = BufferGetPage(buffer); + + if (offnum < FirstOffsetNumber || + offnum > PageGetMaxOffsetNumber(page)) + { + UnlockReleaseBuffer(buffer); + return TM_Invisible; + } + itemid = PageGetItemId(page, offnum); + if (!ItemIdIsNormal(itemid)) + { + UnlockReleaseBuffer(buffer); + return TM_Invisible; + } + tuple_hdr = (FluxTupleHeader *) PageGetItem(page, itemid); + + /* + * If the delete committed, the tombstone remains: report it as + * deleted so the executor can re-evaluate via EPQ. If it + * aborted, the before-image was restored (DELETED cleared) and we + * fall through to perform our own delete. + */ + if (tuple_hdr->t_flags & FLUX_TUPLE_DELETED) + { + if (tmfd) + { + tmfd->ctid = *tid; + tmfd->xmax = wait_xid; + tmfd->cmax = InvalidCommandId; + tmfd->traversed = false; + } + UnlockReleaseBuffer(buffer); + return TM_Deleted; + } + } + else + { + if (tmfd) + { + tmfd->ctid = *tid; + tmfd->xmax = GetCurrentTransactionId(); + tmfd->cmax = InvalidCommandId; + tmfd->traversed = false; + } + UnlockReleaseBuffer(buffer); + return TM_Deleted; + } + } + + /* + * Handle LOCKED flag: same logic as the UPDATE path ā€” clear our own + * lock before proceeding with the delete. + */ + if (tuple_hdr->t_flags & FLUX_TUPLE_LOCKED) + { + SLogTupleOp lock_entry; + int nfound; + + nfound = SLogTupleLookupFiltered(RelationGetRelid(relation), tid, + GetCurrentTransactionId(), &lock_entry, 1); + if (nfound > 0 && + (lock_entry.op_type == SLOG_OP_LOCK_SHARE || + lock_entry.op_type == SLOG_OP_LOCK_EXCL)) + { + tuple_hdr->t_flags &= ~FLUX_TUPLE_LOCKED; + } + } + + /* + * Fast-path: clear stale UNCOMMITTED flag (same optimization as UPDATE). + * We hold the buffer lock exclusively, so this is safe. + */ + if ((tuple_hdr->t_flags & FLUX_TUPLE_UNCOMMITTED) && + !(tuple_hdr->t_flags & (FLUX_TUPLE_DELETED | FLUX_TUPLE_UPDATED))) + { + if (!SLogTupleHasEntry(RelationGetRelid(relation), tid)) + { + tuple_hdr->t_flags &= ~FLUX_TUPLE_UNCOMMITTED; + } + } + + /* + * Check tuple visibility against snapshot and handle concurrent + * modifications. Same logic as the UPDATE path: distinguish truly + * invisible tuples from concurrent modifications. + */ + if (snapshot) + { + bool visible; + + visible = FluxTupleVisibleToSnapshotDual(tuple_hdr, snapshot, + RelationGetRelid(relation), + buffer); + + if (!visible) + { + TransactionId dirty_xid; + bool is_insert_entry; + + /* + * Lock-free: SLogTupleGetDirtyXid reads the seqlock-guarded sLog + * flat hash with EBR. No need to release buffer lock. + */ + dirty_xid = SLogTupleGetDirtyXid(RelationGetRelid(relation), + tid, + &is_insert_entry); + + /* Check if tuple was deleted by another transaction */ + if (tuple_hdr->t_flags & FLUX_TUPLE_DELETED) + { + if (tmfd) + { + tmfd->ctid = *tid; + tmfd->xmax = TransactionIdIsValid(dirty_xid) ? + dirty_xid : GetCurrentTransactionId(); + tmfd->cmax = InvalidCommandId; + tmfd->traversed = false; + } + UnlockReleaseBuffer(buffer); + return TM_Deleted; + } + + /* + * Buffer lock was never released (wait-free sLog read), so the + * tuple cannot have changed. Proceed with dirty_xid. + */ + { + if (TransactionIdIsValid(dirty_xid) && is_insert_entry) + { + if (tmfd) + { + tmfd->ctid = *tid; + tmfd->xmax = dirty_xid; + tmfd->cmax = InvalidCommandId; + tmfd->traversed = false; + } + UnlockReleaseBuffer(buffer); + return TM_Invisible; + } + + if (TransactionIdIsValid(dirty_xid) && !is_insert_entry) + { + if (wait) + { + TransactionId wait_xid = dirty_xid; + + UnlockReleaseBuffer(buffer); + XactLockTableWait(wait_xid, relation, + tid, XLTW_Delete); + + buffer = ReadBuffer(relation, blkno); + LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); + page = BufferGetPage(buffer); + + if (offnum < FirstOffsetNumber || + offnum > PageGetMaxOffsetNumber(page)) + { + UnlockReleaseBuffer(buffer); + return TM_Invisible; + } + itemid = PageGetItemId(page, offnum); + if (!ItemIdIsNormal(itemid)) + { + UnlockReleaseBuffer(buffer); + return TM_Invisible; + } + tuple_hdr = (FluxTupleHeader *) + PageGetItem(page, itemid); + + if (tuple_hdr->t_flags & FLUX_TUPLE_DELETED) + { + if (tmfd) + { + tmfd->ctid = *tid; + tmfd->xmax = wait_xid; + tmfd->cmax = InvalidCommandId; + tmfd->traversed = false; + } + UnlockReleaseBuffer(buffer); + return TM_Deleted; + } + + visible = FluxTupleVisibleToSnapshotDual(tuple_hdr, snapshot, + RelationGetRelid(relation), + buffer); + + if (!visible) + { + /* + * Same EPQ livelock fix as the UPDATE path: check + * for our own LOCK entry before returning + * TM_Updated. + */ + TransactionId myxid_postw = + GetCurrentTransactionIdIfAny(); + + if (TransactionIdIsValid(myxid_postw)) + { + SLogTupleOp my_epw; + int my_nfound_postw; + + my_nfound_postw = SLogTupleLookupFiltered( + RelationGetRelid(relation), + tid, myxid_postw, + &my_epw, 1); + + if (my_nfound_postw > 0) + { + /* Own LOCK entry ā†’ proceed */ + } + else + { + if (tmfd) + { + tmfd->ctid = *tid; + tmfd->xmax = wait_xid; + tmfd->cmax = InvalidCommandId; + tmfd->traversed = false; + } + UnlockReleaseBuffer(buffer); + return TM_Updated; + } + } + else + { + if (tmfd) + { + tmfd->ctid = *tid; + tmfd->xmax = wait_xid; + tmfd->cmax = InvalidCommandId; + tmfd->traversed = false; + } + UnlockReleaseBuffer(buffer); + return TM_Updated; + } + } + } + else + { + if (tmfd) + { + tmfd->ctid = *tid; + tmfd->xmax = dirty_xid; + tmfd->cmax = InvalidCommandId; + tmfd->traversed = false; + } + UnlockReleaseBuffer(buffer); + return TM_WouldBlock; + } + } + else + { + /* + * No in-progress sLog entry for another txn. Same + * EPQ-loop fix as the UPDATE path: check if our + * transaction already has a sLog entry (from + * table_tuple_lock during EPQ). If so, fall through; + * otherwise trigger EPQ. + */ + TransactionId myxid_chk = + GetCurrentTransactionIdIfAny(); + + if (TransactionIdIsValid(myxid_chk)) + { + SLogTupleOp my_entry; + int my_nfound; + + my_nfound = SLogTupleLookupFiltered( + RelationGetRelid(relation), + tid, myxid_chk, &my_entry, 1); + if (my_nfound > 0) + { + /* EPQ already done; proceed. */ + } + else + { + if (tmfd) + { + tmfd->ctid = *tid; + tmfd->xmax = + InvalidTransactionId; + tmfd->cmax = InvalidCommandId; + tmfd->traversed = false; + } + UnlockReleaseBuffer(buffer); + return TM_Updated; + } + } + else + { + if (tmfd) + { + tmfd->ctid = *tid; + tmfd->xmax = InvalidTransactionId; + tmfd->cmax = InvalidCommandId; + tmfd->traversed = false; + } + UnlockReleaseBuffer(buffer); + return TM_Updated; + } + } + } + /* If now visible, fall through to perform the delete */ + } + } + + /* + * Same as the UPDATE path: even when visibility returned "true", check + * for in-progress modifications by another transaction. Block if found. + */ + if (tuple_hdr->t_flags & FLUX_TUPLE_UNCOMMITTED) + { + TransactionId dirty_xid; + bool is_insert_entry; + + /* Lock-free: no buffer unlock needed */ + dirty_xid = SLogTupleGetDirtyXid(RelationGetRelid(relation), + tid, &is_insert_entry); + + if (!TransactionIdIsValid(dirty_xid)) + { + /* + * No in-flight transaction is modifying this tuple. The + * UNCOMMITTED flag is stale (left over from a committed + * transaction whose cleanup callback already ran). Clear it + * opportunistically to prevent future visibility re-checks. + */ + tuple_hdr->t_flags &= ~FLUX_TUPLE_UNCOMMITTED; + MarkBufferDirty(buffer); + } + else if (TransactionIdIsValid(dirty_xid) && + !TransactionIdIsCurrentTransactionId(dirty_xid) && + !is_insert_entry) + { + if (wait) + { + TransactionId wait_xid = dirty_xid; + + UnlockReleaseBuffer(buffer); + XactLockTableWait(wait_xid, relation, tid, XLTW_Delete); + + buffer = ReadBuffer(relation, blkno); + LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); + page = BufferGetPage(buffer); + + if (offnum < FirstOffsetNumber || + offnum > PageGetMaxOffsetNumber(page)) + { + UnlockReleaseBuffer(buffer); + return TM_Invisible; + } + itemid = PageGetItemId(page, offnum); + if (!ItemIdIsNormal(itemid)) + { + UnlockReleaseBuffer(buffer); + return TM_Invisible; + } + tuple_hdr = (FluxTupleHeader *) + PageGetItem(page, itemid); + + if (tuple_hdr->t_flags & FLUX_TUPLE_DELETED) + { + if (tmfd) + { + tmfd->ctid = *tid; + tmfd->xmax = wait_xid; + tmfd->cmax = InvalidCommandId; + tmfd->traversed = false; + } + UnlockReleaseBuffer(buffer); + return TM_Deleted; + } + + if (tmfd) + { + tmfd->ctid = *tid; + tmfd->xmax = wait_xid; + tmfd->cmax = InvalidCommandId; + tmfd->traversed = false; + } + UnlockReleaseBuffer(buffer); + return TM_Updated; + } + else + { + if (tmfd) + { + tmfd->ctid = *tid; + tmfd->xmax = dirty_xid; + tmfd->cmax = InvalidCommandId; + tmfd->traversed = false; + } + UnlockReleaseBuffer(buffer); + return TM_WouldBlock; + } + } + } + + /* + * Authoritative write-write / lock-conflict gate (mirrors the UPDATE + * path's final gate). The three checks above use SLogTupleGetDirtyXid, + * which only detects in-progress WRITERS and is blind to lock-only + * markers -- so a SELECT ... FOR UPDATE/FOR SHARE locker (which leaves a + * SLOG_OP_LOCK_EXCL/LOCK_SHARE marker on a committed, non-deleted, + * non-UNCOMMITTED tuple) would sail straight through and DELETE the row + * the locker is protecting. Probe with SLogTupleGetWriteConflictXid, + * which additionally reports a locker whose recorded LockTupleMode + * conflicts with ours under the real heavyweight matrix. + * + * LockTupleMode: LockTupleExclusive (AccessExclusiveLock), matching heap + * DELETE ("we need the strongest one"). A DELETE destroys the key, so it + * must conflict with FOR KEY SHARE FK lockers too; unlike UPDATE (which + * uses NoKeyExclusive precisely to stay compatible with a KeyShare FK + * lock), waiting on a KeyShare locker here is CORRECT, not spurious -- + * the FK machinery guarantees that locker releases before the referenced + * row can go away. + * + * Self-wait / EPQ safety: SLogTupleGetWriteConflictXid skips our own xid + * internally, so XactLockTableWait never receives GetTopTransactionId() + * (no lmgr self-wait assertion). A lock this transaction took during EPQ + * re-evaluation is our own marker and is likewise skipped. Before + * sleeping we acquire the heavyweight LOCKTAG_TUPLE lock so two writers + * racing the same tuple queue FIFO instead of mutually XactLockTableWait + * deadlocking (matches heap_delete's "establish our priority"). + */ + { + TransactionId conflict_xid; + bool conflict_is_insert = false; + + conflict_xid = SLogTupleGetWriteConflictXid(RelationGetRelid(relation), + tid, LockTupleExclusive, + &conflict_is_insert); + + if (TransactionIdIsValid(conflict_xid) && + !TransactionIdIsCurrentTransactionId(conflict_xid) && + !conflict_is_insert) + { + if (wait) + { + TransactionId wait_xid = conflict_xid; + + UnlockReleaseBuffer(buffer); + if (!have_tuple_lock) + FluxLockTuple(relation, tid, LockTupleExclusive, + true, &have_tuple_lock); + XactLockTableWait(wait_xid, relation, tid, XLTW_Delete); + + /* Re-read and re-classify after the conflicter finished. */ + buffer = ReadBuffer(relation, blkno); + LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); + page = BufferGetPage(buffer); + + if (offnum < FirstOffsetNumber || + offnum > PageGetMaxOffsetNumber(page)) + { + UnlockReleaseBuffer(buffer); + if (have_tuple_lock) + FluxUnlockTuple(relation, tid, LockTupleExclusive); + return TM_Invisible; + } + itemid = PageGetItemId(page, offnum); + if (!ItemIdIsNormal(itemid)) + { + UnlockReleaseBuffer(buffer); + if (have_tuple_lock) + FluxUnlockTuple(relation, tid, LockTupleExclusive); + return TM_Invisible; + } + tuple_hdr = (FluxTupleHeader *) PageGetItem(page, itemid); + + /* + * If the conflicter was a writer that deleted the tuple, + * report TM_Deleted for EPQ. A locker leaves the tuple + * intact -- we hold the tuple lock and now own the FIFO slot, + * so fall through to perform the delete. We do NOT re-probe + * for further lockers: any locker that queued after us is + * behind us on LOCKTAG_TUPLE and will not be granted until we + * release, so re-waiting could livelock. + */ + if (tuple_hdr->t_flags & FLUX_TUPLE_DELETED) + { + if (tmfd) + { + tmfd->ctid = *tid; + tmfd->xmax = wait_xid; + tmfd->cmax = InvalidCommandId; + tmfd->traversed = false; + } + UnlockReleaseBuffer(buffer); + if (have_tuple_lock) + FluxUnlockTuple(relation, tid, LockTupleExclusive); + return TM_Deleted; + } + } + else + { + /* NOWAIT / SKIP LOCKED */ + if (tmfd) + { + tmfd->ctid = *tid; + tmfd->xmax = conflict_xid; + tmfd->cmax = InvalidCommandId; + tmfd->traversed = false; + } + UnlockReleaseBuffer(buffer); + return TM_WouldBlock; + } + } + } + + /* + * Get transaction timestamp BEFORE critical section. Use xact_ts as + * commit timestamp for within-transaction visibility. + */ + (void) FluxGetTransactionTimestamp(); + current_ts = (uint64) FluxGetDmlTimestamp(); + + /* + * Allocate old_tuple structure and save a copy of the old tuple data + * BEFORE entering the critical section. The tuple header will be + * modified below (DELETED flag, commit_ts, etc.), and the WAL record + * needs the unmodified before-image for UNDO support. + */ + { + uint32 del_old_len = ItemIdGetLength(itemid); + + old_tuple_for_delete_wal = palloc(sizeof(FluxTupleData)); + old_tuple_for_delete_wal->t_len = del_old_len; + old_tuple_for_delete_wal->t_data = (FluxTupleHeader *) palloc(del_old_len); + memcpy(old_tuple_for_delete_wal->t_data, tuple_hdr, del_old_len); + } + + /* + * Prepare the heap-format logical-decoding image of the deleted tuple + * before the critical section (palloc/heap_form_tuple are forbidden + * inside). No-op unless the relation is logically logged. + */ + FluxXLogPrepareLogicalImage(relation, old_tuple_for_delete_wal, + &delete_logical_img); + + /* + * SSI: check for rw-conflict in. If a concurrent serializable + * transaction read this tuple (holds a SIREAD lock on it), our delete + * creates an rw-antidependency that may form a dangerous structure. + */ + CheckForSerializableConflictIn(relation, tid, BufferGetBlockNumber(buffer)); + + /* + * Pre-allocate WAL buffer space BEFORE entering critical section. DELETE + * operations only need the main buffer (no overflow). + * + * CRITICAL: XLogEnsureRecordSpace() may allocate memory, so it MUST be + * called outside the critical section. + */ + if (RelationNeedsWAL(relation)) + XLogEnsureRecordSpace(0, 2); + + /* + * Per-relation UNDO: reserve space for a DELETE UNDO record with full + * tuple data so rollback can restore the deleted tuple. Must happen + * before the critical section since it may extend the UNDO fork. + */ + if (smgrexists(RelationGetSmgr(relation), RELUNDO_FORKNUM)) + del_undo_ptr = RelUndoReserve(relation, + SizeOfRelUndoRecordHeader + + sizeof(RelUndoDeletePayload) + + old_tuple_for_delete_wal->t_len, + &del_undo_buffer); + + /* + * Assign our top-level XID for the t_xmin stamp BEFORE the critical + * section: GetTopTransactionId() allocates a fresh XID on first use, + * which calls XactLockTableInsert -> LockAcquire -> palloc, forbidden + * inside a critical section. Precompute here and write only the bare + * field below, matching the surrounding "compute before crit, write + * inside" pattern (del_undo_ptr, XLogEnsureRecordSpace, logical image). + */ + del_hint_xid = GetCurrentTransactionId(); /* subxid: heap-shaped t_xmax */ + + /* Start critical section for WAL logging */ + START_CRIT_SECTION(); + + /* + * Mark tuple as deleted with tombstone - this is the key FLUX feature. + * Set UNCOMMITTED so concurrent writers/readers consult the sLog while + * the delete is in flight: a second DELETE or UPDATE on this TID must + * detect the in-progress delete (via SLogTupleGetDirtyXid) and block on + * it, rather than racing ahead. The flag is cleared at commit by + * flux_stamp_tuple_committed; VACUUM sees a clean committed delete via + * the xmax XID + CLOG. + * + * Heap-shaped: set t_xmax to the deleter XID (leaving t_xmin, the + * inserter, untouched). Visibility resolves t_xmax against CLOG + the + * reader's snapshot: an old snapshot that predates the delete's commit + * still sees the row; a snapshot after it does not. + */ + tuple_hdr->t_flags |= FLUX_TUPLE_DELETED; + tuple_hdr->t_flags |= FLUX_TUPLE_UNCOMMITTED; + /* Fresh xmax being stamped: drop any stale XMAX_COMMITTED CLOG hint. */ + tuple_hdr->t_flags &= ~FLUX_TUPLE_XMAX_COMMITTED; + FluxTupleSetXmax(tuple_hdr, del_hint_xid); + /* Keep the original t_ctid for potential update chains */ + ItemPointerCopy(tid, &tuple_hdr->t_ctid); + + /* Update page header to match what redo does */ + { + FluxPageOpaque phdr = FluxPageGetOpaque(page); + + FluxPageSetCommitTs(phdr, Max(FluxPageGetCommitTs(phdr), current_ts)); + FluxPageSetFlag(phdr, FLUX_PAGE_DEFRAG_NEEDED); + } + + MarkBufferDirty(buffer); + + /* WAL log the deletion using the pre-saved old tuple copy */ + if (RelationNeedsWAL(relation)) + { + XLogRecPtr recptr; + + recptr = FluxXLogDelete(relation, buffer, offnum, + old_tuple_for_delete_wal, + (uint64) del_hint_xid, + &delete_logical_img); + PageSetLSN(page, recptr); + } + + END_CRIT_SECTION(); + + FluxXLogReleaseLogicalImage(&delete_logical_img); + + /* + * Per-relation UNDO: write the DELETE record now that the delete is + * complete. The record stores the deleted TID plus the full old tuple so + * rollback can restore it in place. + */ + if (RelUndoRecPtrIsValid(del_undo_ptr)) + { + RelUndoRecordHeader del_undo_hdr; + RelUndoDeletePayload del_undo_payload; + + del_undo_hdr.urec_type = RELUNDO_DELETE; + del_undo_hdr.urec_len = (uint16) + (SizeOfRelUndoRecordHeader + sizeof(RelUndoDeletePayload) + + old_tuple_for_delete_wal->t_len); + del_undo_hdr.urec_xid = GetCurrentTransactionId(); + del_undo_hdr.urec_prevundorec = + GetPerRelUndoPtr(RelationGetRelid(relation)); + del_undo_hdr.info_flags = RELUNDO_INFO_HAS_TUPLE; + del_undo_hdr.tuple_len = (uint16) old_tuple_for_delete_wal->t_len; + + del_undo_payload.ntids = 1; + del_undo_payload.tids[0] = *tid; + + RelUndoFinishWithTuple(relation, del_undo_buffer, del_undo_ptr, + &del_undo_hdr, + &del_undo_payload, sizeof(RelUndoDeletePayload), + (const char *) old_tuple_for_delete_wal->t_data, + old_tuple_for_delete_wal->t_len); + + RegisterPerRelUndo(RelationGetRelid(relation), del_undo_ptr); + } + + /* + * Clear visibility map bits for this page since we've deleted a tuple. + * The page is no longer all-visible. + */ + FluxVMUpdateForDelete(relation, buffer); + + /* + * Clean up overflow chains if this tuple has overflow attributes. This + * must happen outside the critical section since it performs its own + * buffer I/O. We check the flag and capture the free space before + * releasing the buffer so we can read the tuple header and page state. + */ + { + bool has_overflow = (tuple_hdr->t_flags & FLUX_TUPLE_HAS_OVERFLOW) != 0; + Size free_space = PageGetFreeSpace(page); + + UnlockReleaseBuffer(buffer); + + /* + * Ensure xact/subxact callbacks are registered before any sLog + * operation. This is critical for savepoint rollback: without the + * SubXactCallback, ROLLBACK TO SAVEPOINT won't restore tuples. + */ + FluxEnsureSLogCallbacks(); + + /* + * Register the delete in the sLog AFTER releasing the buffer lock to + * avoid deadlocks with SLogTupleGetDirtyXid's slow path. + */ + SLogTupleInsert(RelationGetRelid(relation), tid, + GetTopTransactionId(), SLOG_OP_DELETE, + GetCurrentSubTransactionId(), cid, current_ts, 0, + LockTupleNoKeyExclusive); + + /* + * Store before-image for savepoint rollback. The tracked key was + * just created by SLogTupleInsert above. We stash the original tuple + * data (saved before the critical section) so that ROLLBACK TO + * SAVEPOINT can physically restore the tuple. + * + * For DELETE, the before-image captures the original flags and + * commit_ts so we can undo the DELETED flag and timestamp. + */ + SLogTupleStoreBeforeImage(RelationGetRelid(relation), tid, + GetTopTransactionId(), + (const char *) old_tuple_for_delete_wal->t_data, + old_tuple_for_delete_wal->t_len, + old_tuple_for_delete_wal->t_data->t_flags, + old_tuple_for_delete_wal->t_data->t_commit_ts, + relation->rd_locator, + relation->rd_rel->relpersistence); + + /* Free old_tuple copy now that before-image has been stored */ + FluxFreeTuple(old_tuple_for_delete_wal); + + /* Mark this block dirty for the scan-path sLog bypass */ + FluxDirtyMapMark(RelationGetRelid(relation), blkno); + + /* + * NOTE: We do NOT immediately clean up overflow chains here. + * Immediate cleanup was: 1. Buggy (collected wrong overflow pointers + * after modification) 2. Expensive on hot paths (extra buffer I/O + + * locking) 3. Complex to WAL-log correctly + * + * Instead, overflow cleanup is deferred to VACUUM. When VACUUM + * prunes deleted tuples, it will also reclaim orphaned overflow + * pages. + * + * Future enhancement: Log overflow block/offset in WAL DELETE record + * so UNDO log pruning can also clean up overflow chains. + */ + (void) has_overflow; /* Suppress unused variable warning */ + + /* Release tuple lock */ + if (have_tuple_lock) + FluxUnlockTuple(relation, tid, LockTupleExclusive); + + /* Update free space map - deleted tuple creates more free space */ + FluxRecordFreeSpace(relation, blkno, free_space); + } + + /* Return success - tuple was successfully marked as deleted */ + pgstat_count_heap_delete(relation); + return TM_Ok; +} + +/* + * Release a set of overflow buffers collected during a force-shrink update. + * + * FluxStoreOverflowColumn leaves each overflow page locked, dirty, and + * unlogged, handing the buffers back to the caller for atomic WAL logging on + * the success path. On any error path before that logging happens, the caller + * must release them here so no dirty unlogged page is pinned into transaction + * abort. Skips main_buffer (released separately by the caller) and any buffer + * that appears more than once (two overflow columns can land on one page). + */ +static void +flux_release_update_overflow_buffers(FluxOverflowBuffers *bufs, + Buffer main_buffer) +{ + int i; + + for (i = 0; i < bufs->count; i++) + { + Buffer ovf_buf = bufs->buffers[i].buffer; + bool already_released = (ovf_buf == main_buffer); + int j; + + for (j = 0; j < i && !already_released; j++) + { + if (bufs->buffers[j].buffer == ovf_buf) + already_released = true; + } + if (!already_released) + UnlockReleaseBuffer(ovf_buf); + pfree(bufs->buffers[i].record_data); + } + bufs->count = 0; +} + +/* + * Update a tuple in a FLUX table with versioning support + * + * FLUX_RELEASE_TUPLOCK releases the heavyweight tuple lock that competing + * updaters acquire before XactLockTableWait (see the wait sites below). The + * lock is held continuously from acquisition through the recheck/update so it + * serializes racing updaters into a FIFO queue (matching heap_update); it must + * be released on every function-exit path so it never leaks past commit. The + * have_tuple_lock guard makes the macro a no-op on paths that never acquired. + */ +#define FLUX_RELEASE_TUPLOCK() \ + do { \ + if (have_tuple_lock) \ + { \ + FluxUnlockTuple(relation, otid, LockTupleNoKeyExclusive); \ + have_tuple_lock = false; \ + } \ + } while (0) + +/* + * flux_indexed_attr_changed + * + * Self-compute whether any INDEXED attribute changed value between the old + * on-page tuple at 'otid' and the new row in 'slot'. Stock PostgreSQL no + * longer passes the modified-attrs set into tuple_update, so FLUX derives it + * here exactly like heap does internally (HeapDetermineColumnsInfo): deform + * the old on-page tuple, deform the new slot, and value-compare each indexed + * attribute with datum_image_eq. + * + * Returns true if any indexed attribute differs, forcing the OUT-OF-PLACE + * (delete + insert at new TID) update path so secondary indexes are rebuilt. + * Whole-row (attr 0) and non-tableOID system attributes in the indexed set + * are treated as changed (force out-of-place), matching heap's conservative + * direction. If the old tuple can't be read (gone/abnormal), we conservatively + * return true so index maintenance still happens. + */ +static bool +flux_indexed_attr_changed(Relation relation, ItemPointer otid, + TupleTableSlot *slot) +{ + Bitmapset *indexed; + Bitmapset *summarized; + TupleDesc tupdesc = RelationGetDescr(relation); + BlockNumber blkno = ItemPointerGetBlockNumber(otid); + OffsetNumber offnum = ItemPointerGetOffsetNumber(otid); + Buffer buffer; + Page page; + ItemId itemid; + FluxTupleData old_tup; + Datum old_values[MaxTupleAttributeNumber]; + bool old_isnull[MaxTupleAttributeNumber]; + bool changed = false; + int attidx = -1; + + /* + * The full set of attributes referenced by ANY index on the relation: + * HOT_BLOCKING (ordinary/key indexes) plus SUMMARIZED (BRIN and friends). + * Heap's HeapDetermineColumnsInfo uses the same bitmaps; taking their + * union means "any indexed column changed" and lets FLUX force the + * out-of-place TU_All path, which rebuilds every index (summarizing + * included) at the new TID -- so FLUX never needs TU_Summarizing. + */ + indexed = RelationGetIndexAttrBitmap(relation, INDEX_ATTR_BITMAP_HOT_BLOCKING); + summarized = RelationGetIndexAttrBitmap(relation, INDEX_ATTR_BITMAP_SUMMARIZED); + indexed = bms_add_members(indexed, summarized); + bms_free(summarized); + if (indexed == NULL) + return false; /* no indexed columns -> always in place */ + + /* New values live in the slot; make sure they are all deformed. */ + slot_getallattrs(slot); + + /* Read the old on-page tuple. */ + buffer = ReadBuffer(relation, blkno); + LockBuffer(buffer, BUFFER_LOCK_SHARE); + page = BufferGetPage(buffer); + if (offnum < FirstOffsetNumber || offnum > PageGetMaxOffsetNumber(page)) + { + UnlockReleaseBuffer(buffer); + bms_free(indexed); + return true; /* can't read old tuple -> be conservative */ + } + itemid = PageGetItemId(page, offnum); + if (!ItemIdIsNormal(itemid)) + { + UnlockReleaseBuffer(buffer); + bms_free(indexed); + return true; + } + + old_tup.t_len = ItemIdGetLength(itemid); + old_tup.t_data = (FluxTupleHeader *) PageGetItem(page, itemid); + ItemPointerSet(&old_tup.t_self, blkno, offnum); + + /* + * Deform only as far as the highest ordinary indexed column. Indexed + * columns are usually the leading attributes, so on a wide table this + * avoids deforming (walking + fetching) every trailing non-indexed column + * just to compare a few keys -- one of the two per-update overheads + * called out in the write-gap diagnosis. Whole-row (attrnum 0) and + * system (attrnum < 0) references are handled in the loop below without + * reading old_values[], so they do not raise the bound. + */ + { + int max_indexed_natts = 0; + int probe = -1; + + while ((probe = bms_next_member(indexed, probe)) >= 0) + { + AttrNumber probenum = probe + FirstLowInvalidHeapAttributeNumber; + + if (probenum > max_indexed_natts) + max_indexed_natts = probenum; + } + FluxDeformTupleUpTo(relation, &old_tup, tupdesc, old_values, old_isnull, + max_indexed_natts); + } + + while (!changed && (attidx = bms_next_member(indexed, attidx)) >= 0) + { + AttrNumber attrnum = attidx + FirstLowInvalidHeapAttributeNumber; + Form_pg_attribute att; + Datum old_val, + new_val; + bool old_null, + new_null; + + /* Whole-row reference: treat as changed (force all-index). */ + if (attrnum == 0) + { + changed = true; + break; + } + + /* + * System attributes other than tableOID cannot be compared + * meaningfully for an updated row: treat as changed. tableOID never + * changes. + */ + if (attrnum < 0) + { + if (attrnum != TableOidAttributeNumber) + changed = true; + continue; + } + + old_val = old_values[attrnum - 1]; + old_null = old_isnull[attrnum - 1]; + new_val = slot->tts_values[attrnum - 1]; + new_null = slot->tts_isnull[attrnum - 1]; + + /* Null-ness differs -> changed. */ + if (old_null != new_null) + { + changed = true; + break; + } + /* Both null -> unchanged. */ + if (new_null) + continue; + + att = TupleDescAttr(tupdesc, attrnum - 1); + if (!datum_image_eq(old_val, new_val, att->attbyval, att->attlen)) + { + changed = true; + break; + } + } + + UnlockReleaseBuffer(buffer); + bms_free(indexed); + return changed; +} + +TM_Result +flux_tuple_update(Relation relation, ItemPointer otid, TupleTableSlot *slot, + CommandId cid, uint32 options, + Snapshot snapshot, Snapshot crosscheck, + bool wait, TM_FailureData *tmfd, + LockTupleMode *lockmode, TU_UpdateIndexes *update_indexes) +{ + BlockNumber blkno; + OffsetNumber offnum; + Buffer buffer; + Page page; + ItemId itemid; + FluxTupleHeader *old_tuple_hdr; + FluxTuple new_tuple; + Size new_tuple_size; + uint64 current_ts; + bool old_has_overflow = false; + bool have_tuple_lock = false; + FluxTuple old_tuple_for_inplace_wal; + FluxLogicalImage update_old_img; + FluxLogicalImage update_new_img; + FluxOverflowBuffers update_overflow_buffers; + int upd_i; + uint64 defrag_oldest_ts = 0; + TransactionId defrag_oldest_xmin = InvalidTransactionId; + bool force_shrink_attempted = false; + Buffer upd_undo_buffer = InvalidBuffer; + RelUndoRecPtr upd_undo_ptr = InvalidRelUndoRecPtr; + Datum upd_toast_values[MaxTupleAttributeNumber]; + bool upd_toast_isnull[MaxTupleAttributeNumber]; + ToastAttrInfo upd_toast_attr[MaxTupleAttributeNumber]; + ToastTupleContext upd_ttc; + bool upd_toasted = false; + + /* Extract block and offset from old TID */ + blkno = ItemPointerGetBlockNumber(otid); + offnum = ItemPointerGetOffsetNumber(otid); + + /* Validate TID range */ + if (blkno >= RelationGetNumberOfBlocks(relation)) + return TM_Invisible; + + /* + * OUT-OF-PLACE (heap-like) UPDATE for index-key changes. + * + * FLUX does not implement the RowID/gen index contract: secondary indexes + * carry plain heap-style TID entries. If we kept the tuple in place + * while an indexed column changed, the executor would insert a new + * (newkey, TID) index entry while the old (oldkey, TID) entry still + * pointed at the same live TID -- the A->B->A duplicate bug, since there + * is no RowID/gen to disambiguate and no stale-entry recheck. + * + * Stock PostgreSQL no longer tells the AM which columns changed, so FLUX + * self-computes it (flux_indexed_attr_changed, matching heap's internal + * HeapDetermineColumnsInfo): when any indexed column changed, store the + * new version at a NEW TID (DELETE old + INSERT new, zheap's non-in-place + * update) and set *update_indexes = TU_All so the executor inserts fresh + * entries in every index at the new TID. The old tuple's index entries + * then die with its TID and are reclaimed by ordinary VACUUM, exactly + * like heap. Non-indexed-column UPDATEs fall through to the in-place + * path below and set TU_None (the zheap win: update without touching + * indexes). FLUX never uses TU_Summarizing: a key-changing update always + * moves the TID, so TU_All already covers summarizing indexes. + */ + if (flux_indexed_attr_changed(relation, otid, slot)) + { + TM_Result del_res; + + del_res = flux_tuple_delete(relation, otid, cid, options, + snapshot, crosscheck, wait, tmfd); + if (del_res != TM_Ok) + return del_res; + + /* + * Insert the new version. flux_tuple_insert sets slot->tts_tid to + * the new location and TOASTs wide columns. The old version remains + * reachable to older snapshots via the UNDO before-image the delete + * recorded; on ROLLBACK both the delete and the insert are undone. + */ + flux_tuple_insert(relation, slot, cid, options, NULL); + + if (update_indexes != NULL) + *update_indexes = TU_All; + if (lockmode != NULL) + *lockmode = LockTupleExclusive; + return TM_Ok; + } + + /* + * In-place UPDATE: no indexed column changed, so the row keeps its TID + * and no index maintenance is needed. Default the out-param to TU_None + * here so every TM_Ok exit of the in-place path below reports "no index + * update" (the caller does not pre-initialize *update_indexes; it reads + * it only on TM_Ok). + */ + if (update_indexes != NULL) + *update_indexes = TU_None; + + /* + * --------------------------------------------------------------- FAST + * PATH: Same-size CAS update under SHARE_EXCLUSIVE buffer lock. + * + * For simple same-size updates (e.g. balance += delta in TPC-B), we can + * avoid the fully exclusive buffer lock by using a share-exclusive lock + * combined with a single-attempt per-tuple CAS trylock (t_writer). This + * still allows concurrent readers while serializing writers on the same + * page. + * + * NOTE on t_writer contention (investigated, no change made): FluxTuple- + * WriterTryLock is a SINGLE compare-and-swap, not a spin loop. On + * failure the backend does not retry -- it drops SHARE_EXCLUSIVE and + * falls to the exclusive slow path below, which is a queued, sleeping + * lock. Moreover BUFFER_LOCK_SHARE_EXCLUSIVE already conflicts with + * itself (bufmgr.c BufferLockAttempt), so at most one CAS writer per + * buffer runs at a time; additional hot-row writers queue+sleep on the + * buffer content lock, not on t_writer. There is therefore no t_writer + * spin storm to convert to a queued lock: the hot-row path ALREADY + * degrades to fair queued locks (BUFFER_LOCK_SHARE_EXCLUSIVE, then + * LOCKTAG_TUPLE + XactLockTableWait in the slow path). The measured + * hot-row throughput decline (peak ~c8 then fall) is dominated by "Lock : + * tuple" heavyweight waits + per-update UNDO/sLog/WAL work under that + * lock -- the same serialization heap/zheap incur on a hot row -- NOT by + * spinning. Switching the fast path to BUFFER_LOCK_EXCLUSIVE would not + * change this (both modes serialize one writer per buffer) and would + * regress the scattered case by blocking concurrent plain-SHARE readers. + * + * FIXME(flux hot-row scaling): the useful lever is reducing per-update + * work under the serialized lock (fold UNDO/sLog/WAL work, shrink the + * critical section) or a group-update / delta-accumulation scheme, not a + * writer-lock mode change. Deferred: any such change is a correctness- + * sensitive redesign (lost-update + self-wait hazards) and must be + * benchmark-driven, not speculative. See TASK B analysis in the handoff. + * + * Eligibility requirements: - New tuple must be the same on-disk size as + * the old tuple - Old tuple must not have overflow data - Old tuple must + * not be deleted, locked, or uncommitted - No speculative insertion - + * Must be a simple UPDATE (not HOT-chain following) - Snapshot visibility + * must be trivially true (committed tuple) - Relation must need WAL (for + * crash safety) + * + * If any condition fails, we fall through to the exclusive-lock path. + * --------------------------------------------------------------- + */ + { + FluxTuple cas_new_tuple; + Size cas_new_size; + + /* Form the new tuple speculatively (no overflow handling) */ + slot_getallattrs(slot); + cas_new_tuple = FluxFormTuple(RelationGetDescr(relation), + slot->tts_values, + slot->tts_isnull, + NULL, /* no overflow */ + NULL); + + cas_new_size = cas_new_tuple->t_len; + + /* Attempt the CAS fast path */ + buffer = ReadBuffer(relation, blkno); + LockBuffer(buffer, BUFFER_LOCK_SHARE_EXCLUSIVE); + page = BufferGetPage(buffer); + + if (offnum >= FirstOffsetNumber && + offnum <= PageGetMaxOffsetNumber(page)) + { + itemid = PageGetItemId(page, offnum); + + if (ItemIdIsNormal(itemid) && + cas_new_size <= ItemIdGetLength(itemid)) + { + Size cas_target_size; + + old_tuple_hdr = (FluxTupleHeader *) PageGetItem(page, itemid); + + /* + * WS-PVS1: the version-chain head lives in the fixed header + * field t_verptr, so both the on-page tuple and the freshly + * formed new tuple carry it inside their header with no + * trailing growth. Same-column-width updates are therefore + * naturally the same length, so this stays on the CAS fast + * path with no first-time growth dispatch. + */ + cas_target_size = cas_new_size; + + /* + * Cheap in-page eligibility gate: committed, not deleted, not + * locked, no overflow, same size for direct memcpy. This is + * a pure buffer-domain check (no sLog probe). + * + * We deliberately do NOT probe the sLog for write-write + * conflicts here. The flag gate is not a reliable conflict + * signal -- a committed in-place UPDATE rewinds t_commit_ts + * and clears FLUX_TUPLE_UNCOMMITTED, and a reader/third + * writer may clear a still-live flag as "stale" -- so any + * pre-lock sLog probe would have to be repeated + * authoritatively after we own t_writer anyway (state can + * change between the probe and the trylock). The + * authoritative committed-update and in-progress-writer + * probes therefore run once, under the t_writer lock below; + * the pre-lock duplicates were redundant seqlock reads on the + * common no-conflict path. Taking the trylock speculatively + * is harmless: no WAL or page change happens until + * revalidation passes, and a lost race just releases t_writer + * and falls to the exclusive path. + */ + if (!(old_tuple_hdr->t_flags & (FLUX_TUPLE_DELETED | + FLUX_TUPLE_LOCKED | + FLUX_TUPLE_UNCOMMITTED | + FLUX_TUPLE_HAS_OVERFLOW | + FLUX_TUPLE_SPECULATIVE)) && + cas_target_size == ItemIdGetLength(itemid)) + { + uint32 expected = 0; + + if (FluxTupleWriterTryLock(old_tuple_hdr, &expected)) + { + /* + * We now own this tuple exclusively via t_writer, + * which -- not the pre-lock probe -- is the real + * serialization point. Between our pre-lock + * eligibility check and acquiring t_writer, a + * competing CAS writer may have completed its own + * in-place overwrite (it stamps + * FLUX_TUPLE_UNCOMMITTED into the new page image and + * resets on-page t_writer to 0, which lets our CAS + * succeed), or committed and left a conflict marker. + * Acting on the stale pre-lock decision would + * silently clobber that update (lost update). + * Re-validate the flags and the write-write probe + * under the lock; if the tuple is disqualified now, + * release t_writer and fall through to the exclusive + * path, which blocks on the in-progress writer or + * returns TM_Updated for EPQ. + */ + bool cas_revalidated; + + cas_revalidated = + !(old_tuple_hdr->t_flags & (FLUX_TUPLE_DELETED | + FLUX_TUPLE_LOCKED | + FLUX_TUPLE_UNCOMMITTED | + FLUX_TUPLE_HAS_OVERFLOW | + FLUX_TUPLE_SPECULATIVE)) && + cas_target_size == ItemIdGetLength(itemid); + if (cas_revalidated && + snapshot != NULL && IsMVCCSnapshot(snapshot)) + { + RelUndoRecPtr head_verptr; + TransactionId head_xid; + bool head_inprogress = false; + + if (FluxTupleHasCommittedUpdateAfter(relation, + old_tuple_hdr, + ItemIdGetLength(itemid), + snapshot, + GetCurrentTransactionIdIfAny(), + &head_verptr, + &head_xid, + &head_inprogress) && + !FluxEpqReconcileMatches(snapshot, + RelationGetRelid(relation), + otid, + head_verptr, + head_xid)) + cas_revalidated = false; + + /* + * Commit-window conflict: the head committer is + * between PRE_COMMIT (marker + UNCOMMITTED + * cleared) and CLOG commit. Its sLog marker is + * gone, so the dirty-xid probe below cannot see + * it. Abandon the CAS fast path and fall to the + * slow path, which waits on the in-flight writer; + * otherwise the just-committed update is lost. + */ + else if (head_inprogress) + cas_revalidated = false; + } + + /* + * Re-check for a concurrent in-progress writer (see + * above) + */ + if (cas_revalidated) + { + bool reval_is_insert = false; + TransactionId reval_dirty_xid = + SLogTupleGetDirtyXid(RelationGetRelid(relation), + otid, &reval_is_insert); + + if (TransactionIdIsValid(reval_dirty_xid) && + !reval_is_insert) + cas_revalidated = false; + } + + if (!cas_revalidated) + { + /* + * Lost the race: release and fall to exclusive + * path + */ + FluxTupleWriterUnlock(old_tuple_hdr); + } + else + { + /* + * Fairness gate: funnel the CAS writer through + * LOCKTAG_TUPLE so lockers (flux_tuple_lock) and + * slow-path writers already queued on the same + * tag get FIFO service. Without this, a stream + * of CAS writers can lap a locker sleeping in + * XactLockTableWait under FluxLockTuple, starving + * it past statement_timeout. Must be dontWait: + * t_writer is held here, and blocking on a + * heavyweight sleep under a spinlock would + * deadlock. If not granted, drop t_writer and + * fall through to the exclusive path, which takes + * LOCKTAG_TUPLE with wait=true and queues FIFO + * behind the waiter that beat us here. + */ + bool cas_have_tuple_lock = false; + + if (!FluxLockTuple(relation, otid, + LockTupleNoKeyExclusive, + false, &cas_have_tuple_lock)) + { + /* + * CANDIDATE A (surgical fell-through): a + * locker or updater is queued on + * LOCKTAG_TUPLE for this tid. Drop t_writer + * and the page lock, then BLOCK on + * LOCKTAG_TUPLE so this writer queues FIFO + * behind the existing waiter. Only after we + * acquire the tag (the locker ahead of us has + * been served) do we re-enter the update path + * via the slow route, which re-reads the page + * and re-classifies. We must release + * t_writer and the buffer content lock BEFORE + * the blocking acquire: holding either across + * a heavyweight sleep would deadlock. Set + * the OUTER have_tuple_lock so the slow + * path's dirty_xid-gated LockTuple sites + * (guarded by !have_tuple_lock) do not + * double-acquire and FLUX_RELEASE_TUPLOCK + * frees it on every exit. + */ + FluxTupleWriterUnlock(old_tuple_hdr); + LockBuffer(buffer, BUFFER_LOCK_UNLOCK); + ReleaseBuffer(buffer); + FluxFreeTuple(cas_new_tuple); + if (!have_tuple_lock) + FluxLockTuple(relation, otid, + LockTupleNoKeyExclusive, + true, &have_tuple_lock); + goto flux_update_slow_path; + } + { + /* + * We own this tuple exclusively under + * share-exclusive page lock. No other writer + * can modify it until we release t_writer. + * + * Compute timestamps outside critical section + * to avoid memory allocation issues. + */ + uint64 cas_current_ts; + uint16 cas_data_offset; + uint16 cas_data_len; + char *cas_old_bytes; + char *cas_new_bytes; + Size cas_tuple_len; + + (void) FluxGetTransactionTimestamp(); + cas_current_ts = (uint64) FluxGetDmlTimestamp(); + + /* Ensure we have an XID for WAL flush */ + (void) GetCurrentTransactionId(); + + /* + * Heap-shaped: the in-place image is a FRESH + * version. Stamp t_xmax = Invalid (this is + * the newest, live version) and t_xmin = the + * updater below. The pre-update image is + * preserved in the UNDO fork (the CAS path + * writes an undo record), so an older + * snapshot that cannot see the updater's xmin + * reads the before-image back via + * FluxReconstructVisible- Version. + * + * Torn-read note: the CAS path holds only + * BUFFER_LOCK_SHARE_EXCLUSIVE, so a + * concurrent SHARE reader can observe this + * image mid-overwrite. It is marked + * UNCOMMITTED and its xmin is the still- + * in-progress updater, so + * FluxTupleSatisfiesMVCC hides it + * (XidInMVCCSnapshot(updater) == true) and + * the reader reconstructs the prior version + * -- the row never blinks out. + */ + cas_new_tuple->t_data->t_commit_ts = 0; + cas_new_tuple->t_data->t_flags |= FLUX_TUPLE_UPDATED; + + /* + * Mark the in-place image UNCOMMITTED, + * mirroring the regular update path. A + * second writer that hits this row sees + * UNCOMMITTED in the CAS eligibility gate, + * bails to the regular path, and blocks on + * the in-progress writer via + * XactLockTableWait -- without this flag the + * second CAS would silently overwrite an + * uncommitted update (lost update / no + * write-write blocking). PRE_COMMIT clears + * the flag and stamps the real commit + * timestamp; readers self-heal a stale flag + * via the visibility path after a crash. + */ + cas_new_tuple->t_data->t_flags |= FLUX_TUPLE_UNCOMMITTED; + + /* + * Clear inherited CLOG hint bits: this image + * copied the old tuple's flags, but we are + * stamping a NEW t_xmin (the updater). A + * stale XMIN_COMMITTED from the old inserter + * would wrongly assert the new xmin is + * committed. + */ + cas_new_tuple->t_data->t_flags &= + ~(FLUX_TUPLE_XMIN_COMMITTED | FLUX_TUPLE_XMAX_COMMITTED); + cas_new_tuple->t_data->t_xmin = GetCurrentTransactionId(); /* subxid: heap-shaped, + * so savepoint rollback + * marks it aborted in + * CLOG */ + cas_new_tuple->t_data->t_writer = 0; /* clear in new image */ + ItemPointerSet(&cas_new_tuple->t_data->t_ctid, blkno, offnum); + + /* + * Carry the index-identity generation forward + * from the t_gen is reserved/unused in FLUX + * (FLUX uses plain heap-TID index identity, + * no RowID/gen scheme). Leave it at the + * palloc0 default of 0. + */ + cas_new_tuple->t_data->t_gen = 0; + + /* + * WS-PVS1: build the new-image buffer of + * cas_target_size (== cas_new_size; the + * version pointer is a header field, not a + * trailer). We reserve the per-rel UNDO + * record first so its RelUndoRecPtr can be + * stamped into the new image BEFORE both the + * diff scan (so the diff includes the + * version-pointer change) and the page + * memcpy. + * + * RelUndoReserve may extend the fork and + * ereport -- neither is safe inside the + * critical section. Lock order: data buffer + * then UNDO buffer. + */ + { + uint32 cas_old_len = ItemIdGetLength(itemid); + char *cas_old_copy = palloc(cas_old_len); + char *cas_full_image; + Buffer cas_undo_buffer = InvalidBuffer; + RelUndoRecPtr cas_undo_ptr = InvalidRelUndoRecPtr; + RelUndoRecPtr cas_prev_undo = InvalidRelUndoRecPtr; + Size cas_undo_reserve; + RelUndoStageResult cas_undo_staged; + bool cas_undo_staged_valid = false; + + memcpy(cas_old_copy, old_tuple_hdr, cas_old_len); + + /* + * Reserve worst-case UNDO size + * (full-tuple record) up front. We + * commit to either delta or full-tuple + * AFTER stamping the urec_ptr into the + * new image and computing the final diff. + * Over-reservation by a few bytes is + * benign -- the unused tail of the + * reserved space is wasted on that page, + * no worse than a SHRINK update. + */ + cas_undo_reserve = SizeOfRelUndoRecordHeader + + sizeof(RelUndoUpdatePayload) + cas_old_len; + if (smgrexists(RelationGetSmgr(relation), RELUNDO_FORKNUM)) + { + cas_prev_undo = + GetPerRelUndoPtr(RelationGetRelid(relation)); + cas_undo_ptr = RelUndoReserve(relation, + cas_undo_reserve, + &cas_undo_buffer); + } + + /* + * Assemble the on-page image: header+data + * from the freshly formed new tuple plus + * the trailing version-pointer. Every + * committed in-place UPDATE stamps the + * verptr, so the size check above forces + * this same-shape requirement; first-time + * stamping (item length grew from base to + * base+8) happens on the exclusive-lock + * path. The trailing field is part of + * the on-disk slot, so the diff scan and + * the page memcpy both see this final + * layout. Stamp the freshly reserved + * cas_undo_ptr (NEVER preserve the prior + * pointer): the new diff reconstructs + * this update's before-image, and the + * header's urec_prevundorec already + * chains to the prior diff -- the row + * always points at the head of the chain. + * If no UNDO fork exists (unlogged/temp), + * stamp InvalidRelUndoRecPtr to keep slot + * length stable; readers fall back to the + * on-page tuple when the head is invalid. + */ + cas_full_image = palloc(cas_target_size); + memcpy(cas_full_image, cas_new_tuple->t_data, cas_new_size); + { + FluxTupleHeader *cas_full_hdr = + (FluxTupleHeader *) cas_full_image; + + cas_full_hdr->t_flags |= FLUX_TUPLE_HAS_VERSION_PTR; + FluxTupleSetVersionPtr(cas_full_hdr, + cas_target_size, + cas_undo_ptr); + } + + /* + * Compute the diff region for WAL + * logging. We log only the bytes that + * actually changed within the unified + * image. Read the old side from + * cas_old_copy (already memcpy'd from the + * page, identical bytes) rather than the + * live locked page: the scan is pure CPU + * work with no I/O, and reading a + * palloc'd copy instead of old_tuple_hdr + * means this loop no longer needs the + * data page to stay locked -- + * cas_target_size == cas_old_len is + * guaranteed by the eligibility gate + * above (same-size CAS only), so + * cas_old_copy has exactly cas_tuple_len + * bytes available. + */ + cas_tuple_len = cas_target_size; + cas_old_bytes = cas_old_copy; + cas_new_bytes = cas_full_image; + + /* Find first differing byte */ + cas_data_offset = 0; + while (cas_data_offset < cas_tuple_len && + cas_old_bytes[cas_data_offset] == cas_new_bytes[cas_data_offset]) + cas_data_offset++; + + if (cas_data_offset < cas_tuple_len) + { + uint16 cas_end = (uint16) cas_tuple_len; + + /* Find last differing byte */ + while (cas_end > cas_data_offset && + cas_old_bytes[cas_end - 1] == cas_new_bytes[cas_end - 1]) + cas_end--; + + cas_data_len = cas_end - cas_data_offset; + } + else + { + /* + * No actual data change -- release + * and return OK. Cancel the UNDO + * reservation; we have not written + * anything to the page or the UNDO + * fork. + */ + if (RelUndoRecPtrIsValid(cas_undo_ptr)) + RelUndoCancel(relation, cas_undo_buffer, cas_undo_ptr); + pfree(cas_full_image); + pfree(cas_old_copy); + FluxTupleWriterUnlock(old_tuple_hdr); + LockBuffer(buffer, BUFFER_LOCK_UNLOCK); + ReleaseBuffer(buffer); + FluxFreeTuple(cas_new_tuple); + + /* Set output TID */ + ItemPointerSet(&slot->tts_tid, blkno, offnum); + slot->tts_tableOid = RelationGetRelid(relation); + if (update_indexes) + *update_indexes = TU_None; + pgstat_count_heap_update(relation, false, false); + FluxUnlockTuple(relation, otid, + LockTupleNoKeyExclusive); + return TM_Ok; + } + + /* SSI conflict check */ + CheckForSerializableConflictIn(relation, otid, + BufferGetBlockNumber(buffer)); + + /* + * FOLD variant: stage the UNDO + * before-image BEFORE the critical + * section so the combined WAL record can + * carry both the main-fork redo byte-diff + * and the undo before-image in one + * XLogInsert (halving per-UPDATE WAL + * insert count vs the standalone + * RM_RELUNDO record). + * + * RelUndoReserve() already returned the + * undo buffer exclusively locked+pinned, + * and (for a new page) left the metapage + * locked; both stay held through the crit + * section below, so one XLogInsert may + * legally register buffers from both + * forks. RelUndoStage() writes+dirties + * the undo page here (no WAL); the crit + * section emits the fold record and + * PageSetLSNs both pages. + * + * The tid the payloads reference is fully + * known now (blkno/offnum), so set it + * before staging. + */ + ItemPointerSet(&slot->tts_tid, blkno, offnum); + slot->tts_tableOid = RelationGetRelid(relation); + if (RelUndoRecPtrIsValid(cas_undo_ptr)) + { + RelUndoRecordHeader cas_undo_hdr; + char *cas_combined; + Size cas_payload_total; + RelUndoUpdatePayload cas_undo_payload; + + cas_undo_payload.oldtid = slot->tts_tid; + cas_undo_payload.newtid = slot->tts_tid; + cas_undo_hdr.urec_xid = GetCurrentTransactionId(); + cas_undo_hdr.urec_prevundorec = cas_prev_undo; + cas_undo_hdr.urec_type = RELUNDO_UPDATE; + + /* + * Full old-tuple before-image, + * always: FLUX matches ZHeap (whole + * old tuple to UNDO, no + * byte-diff/delta). + * [header][update-payload][old + * tuple]. + */ + cas_undo_hdr.urec_len = (uint16) + (SizeOfRelUndoRecordHeader + + sizeof(RelUndoUpdatePayload) + cas_old_len); + cas_undo_hdr.info_flags = RELUNDO_INFO_HAS_TUPLE; + cas_undo_hdr.tuple_len = (uint16) cas_old_len; + + cas_payload_total = sizeof(RelUndoUpdatePayload) + + cas_old_len; + cas_combined = palloc(cas_payload_total); + memcpy(cas_combined, &cas_undo_payload, + sizeof(RelUndoUpdatePayload)); + memcpy(cas_combined + sizeof(RelUndoUpdatePayload), + cas_old_copy, cas_old_len); + + RelUndoStage(relation, cas_undo_buffer, cas_undo_ptr, + &cas_undo_hdr, cas_combined, + cas_payload_total, &cas_undo_staged); + cas_undo_staged_valid = true; + pfree(cas_combined); + } + + /* Critical section: modify page + WAL */ + START_CRIT_SECTION(); + + memcpy(old_tuple_hdr, cas_full_image, cas_target_size); + + /* + * Update page-level commit timestamp + * atomically + */ + { + FluxPageOpaque cas_opaque = FluxPageGetOpaque(page); + uint64 cas_old_ts_flags; + uint64 cas_new_ts_flags; + uint64 cur_ts; + + do + { + cas_old_ts_flags = cas_opaque->pd_commit_ts_and_flags; + cur_ts = cas_old_ts_flags & FLUX_PAGE_TS_MASK; + + if (cas_current_ts <= cur_ts) + break; + cas_new_ts_flags = (cas_old_ts_flags & FLUX_PAGE_FLAG_MASK) | + (cas_current_ts & FLUX_PAGE_TS_MASK); + } while (!pg_atomic_compare_exchange_u64( + (pg_atomic_uint64 *) &cas_opaque->pd_commit_ts_and_flags, + &cas_old_ts_flags, cas_new_ts_flags)); + } + + MarkBufferDirtyShared(buffer); + + /* + * WAL log. When we staged an UNDO + * before-image, emit the single combined + * fold record carrying both the main-fork + * redo diff and the undo bytes; it + * PageSetLSNs both pages (and the + * metapage on a new undo page). Otherwise + * (no UNDO fork) emit the plain redo + * record. + */ + if (cas_undo_staged_valid) + { + FluxXLogCasUpdateUndo(relation, buffer, offnum, + cas_data_offset, cas_data_len, + cas_new_bytes + cas_data_offset, + 0, /* new version: t_xmax = + * Invalid */ + &cas_undo_staged); + } + else if (RelationNeedsWAL(relation)) + { + FluxXLogCasUpdate(relation, buffer, offnum, + cas_data_offset, cas_data_len, + cas_new_bytes + cas_data_offset, + 0); /* new version: t_xmax = + * Invalid */ + } + + END_CRIT_SECTION(); + + /* + * Release the staged UNDO buffers now + * that the fold record has logged them + * (mirrors the release RelUndoFinish + * would have done), and register the + * record for rollback discovery. + */ + if (cas_undo_staged_valid) + { + RegisterPerRelUndo(RelationGetRelid(relation), + cas_undo_ptr); + pfree(cas_undo_staged.wal_record_data); + UnlockReleaseBuffer(cas_undo_staged.undo_buffer); + if (BufferIsValid(cas_undo_staged.metabuf)) + UnlockReleaseBuffer(cas_undo_staged.metabuf); + } + + /* Release tuple-level CAS lock */ + FluxTupleWriterUnlock(old_tuple_hdr); + + /* + * sLog registration BEFORE buffer + * release. Eliminates the race window + * where another backend reads the + * modified tuple but finds no sLog entry + * (causing visibility failures at high + * concurrency). Safe: seqlock reads are + * wait-free, no deadlock with buffer + * lock. + */ + FluxEnsureSLogCallbacks(); + SLogTupleInsert(RelationGetRelid(relation), + &slot->tts_tid, + GetTopTransactionId(), + SLOG_OP_UPDATE, + GetCurrentSubTransactionId(), + cid, cas_current_ts, 0, + LockTupleNoKeyExclusive); + + /* Store before-image for rollback */ + SLogTupleStoreBeforeImage( + RelationGetRelid(relation), + &slot->tts_tid, + GetTopTransactionId(), + cas_old_copy, cas_old_len, + ((FluxTupleHeader *) cas_old_copy)->t_flags, + ((FluxTupleHeader *) cas_old_copy)->t_commit_ts, + relation->rd_locator, + relation->rd_rel->relpersistence); + + LockBuffer(buffer, BUFFER_LOCK_UNLOCK); + + /* + * No index maintenance for the CAS + * in-place path. A CAS update is + * same-size and only reaches here when no + * indexed column changed (key changes are + * routed out of place at the top of + * flux_tuple_update, TU_All). The TID is + * unchanged, so every (key, TID) entry + * still points at the live tuple; + * *update_indexes is set TU_None below. + */ + + pfree(cas_old_copy); + pfree(cas_full_image); + + /* + * Clear VM all-visible/all-frozen bits. + */ + FluxVMClear(relation, blkno, buffer, FLUX_VM_VALID_BITS); + + ReleaseBuffer(buffer); + + /* Track in-place update */ + flux_stat_in_place_updates++; + + /* + * Mark this block dirty for the scan-path + * sLog bypass + */ + FluxDirtyMapMark(RelationGetRelid(relation), blkno); + + FluxFreeTuple(cas_new_tuple); + + /* + * Drive retained-marker cleanup now that + * all buffer locks are released. + * Throttled internally; never runs the + * global sweep under a page lock (which + * would convoy all writers to this hot + * row). + */ + SLogTupleMaybeCleanupRetained(); + + /* + * Same reasoning for the per-relation + * UNDO fork: this in-place AM can + * correctly report near-zero dead tuples, + * so autovacuum may never launch and + * RelUndoVacuum() (VACUUM-only) may never + * run. RelUndoMaybeVacuum() is the + * throttled backstop. + */ + RelUndoMaybeVacuum(relation); + pgstat_count_heap_update(relation, false, false); + FluxUnlockTuple(relation, otid, + LockTupleNoKeyExclusive); + return TM_Ok; + } /* end block (tuple-lock granted) */ + } /* end else (revalidated) */ + } + } + /* CAS failed -- another writer has this tuple */ + } + /* Not eligible for CAS fast path */ + } + /* ItemId not normal or new tuple too large */ + } + /* Offset out of range */ + + /* Release shared lock, fall through to exclusive path */ + LockBuffer(buffer, BUFFER_LOCK_UNLOCK); + ReleaseBuffer(buffer); + FluxFreeTuple(cas_new_tuple); + } + + /* + * Lock the buffer exclusively. The exclusive buffer lock prevents + * concurrent modification while we hold it, but it is dropped while we + * XactLockTableWait on a conflicting in-progress writer (the wait sites + * below). To stop two backends that each see the other's in-progress + * write from mutually waiting and deadlocking, an updater first takes a + * heavyweight tuple lock (have_tuple_lock) that serializes them into a + * FIFO queue, held continuously through the recheck/update and released + * on every exit via FLUX_RELEASE_TUPLOCK(). This matches heap_update. + * + * CANDIDATE A: a CAS writer that found a locker queued on the tag jumps + * here (flux_update_slow_path) already holding have_tuple_lock, so do NOT + * reset it -- clobbering it would drop the lock we just queued for. The + * buffer was released before the goto, so the re-read below re-pins it; + * the label must precede ReadBuffer. + */ +flux_update_slow_path: + /* Read the page containing the old tuple */ + buffer = ReadBuffer(relation, blkno); + LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); + page = BufferGetPage(buffer); + + /* Validate offset number */ + if (offnum < FirstOffsetNumber || offnum > PageGetMaxOffsetNumber(page)) + { + UnlockReleaseBuffer(buffer); + return TM_Invisible; + } + + /* Get the old tuple */ + itemid = PageGetItemId(page, offnum); + if (!ItemIdIsNormal(itemid)) + { + UnlockReleaseBuffer(buffer); + return TM_Invisible; + } + + old_tuple_hdr = (FluxTupleHeader *) PageGetItem(page, itemid); + + /* Check if old tuple has overflow chains to clean up later */ + old_has_overflow = (old_tuple_hdr->t_flags & FLUX_TUPLE_HAS_OVERFLOW) != 0; + + /* + * Check if tuple is already deleted. As in the delete path, a DELETED + * flag may reflect an in-progress delete by a concurrent transaction + * (which also sets UNCOMMITTED). If so, wait for that transaction and + * retry rather than reporting TM_Deleted immediately -- this matches + * heap, where an UPDATE blocks behind a concurrent in-progress DELETE. + */ + if (old_tuple_hdr->t_flags & FLUX_TUPLE_DELETED) + { + TransactionId del_xid = InvalidTransactionId; + bool del_is_insert = false; + + if (old_tuple_hdr->t_flags & FLUX_TUPLE_UNCOMMITTED) + del_xid = SLogTupleGetWriteConflictXid(RelationGetRelid(relation), + otid, + LockTupleNoKeyExclusive, + &del_is_insert); + + if (wait && TransactionIdIsValid(del_xid) && + !TransactionIdIsCurrentTransactionId(del_xid) && + !del_is_insert) + { + TransactionId wait_xid = del_xid; + + /* + * Acquire a heavyweight tuple lock before sleeping on the + * conflicting xid. This serializes competing updaters of the + * same tuple into a FIFO queue; without it, two backends that + * each see the other's in-progress write would mutually + * XactLockTableWait and deadlock. Matches heap_update. + */ + UnlockReleaseBuffer(buffer); + if (!have_tuple_lock) + FluxLockTuple(relation, otid, LockTupleNoKeyExclusive, + true, &have_tuple_lock); + XactLockTableWait(wait_xid, relation, otid, XLTW_Update); + + buffer = ReadBuffer(relation, blkno); + LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); + page = BufferGetPage(buffer); + + if (offnum < FirstOffsetNumber || + offnum > PageGetMaxOffsetNumber(page)) + { + UnlockReleaseBuffer(buffer); + FLUX_RELEASE_TUPLOCK(); + return TM_Invisible; + } + itemid = PageGetItemId(page, offnum); + if (!ItemIdIsNormal(itemid)) + { + UnlockReleaseBuffer(buffer); + FLUX_RELEASE_TUPLOCK(); + return TM_Invisible; + } + old_tuple_hdr = (FluxTupleHeader *) PageGetItem(page, itemid); + + /* + * Delete committed -> tombstone persists, report TM_Deleted for + * EPQ. Delete aborted -> before-image restored (DELETED cleared), + * fall through to perform the update. + */ + if (old_tuple_hdr->t_flags & FLUX_TUPLE_DELETED) + { + if (tmfd) + { + tmfd->ctid = *otid; + tmfd->xmax = wait_xid; + tmfd->cmax = InvalidCommandId; + tmfd->traversed = false; + } + UnlockReleaseBuffer(buffer); + FLUX_RELEASE_TUPLOCK(); + return TM_Deleted; + } + } + else + { + if (tmfd) + { + tmfd->ctid = *otid; + tmfd->xmax = GetCurrentTransactionId(); + tmfd->cmax = InvalidCommandId; + tmfd->traversed = false; + } + UnlockReleaseBuffer(buffer); + FLUX_RELEASE_TUPLOCK(); + return TM_Deleted; + } + } + + /* + * Handle LOCKED flag: if this tuple is locked by the current transaction + * (FOR SHARE/FOR KEY SHARE/FOR UPDATE), the lock is compatible with + * UPDATE (self-lock). Clear the LOCKED flag since we're about to modify + * the tuple. The sLog LOCK entry will be overwritten by the UPDATE entry + * or cleaned up at commit. + */ + if (old_tuple_hdr->t_flags & FLUX_TUPLE_LOCKED) + { + SLogTupleOp lock_entry; + int nfound; + + nfound = SLogTupleLookupFiltered(RelationGetRelid(relation), otid, + GetCurrentTransactionId(), &lock_entry, 1); + if (nfound > 0 && + (lock_entry.op_type == SLOG_OP_LOCK_SHARE || + lock_entry.op_type == SLOG_OP_LOCK_EXCL)) + { + /* Our own lock - clear flag and proceed with update */ + old_tuple_hdr->t_flags &= ~FLUX_TUPLE_LOCKED; + } + + /* + * If it's another transaction's lock, the existing concurrency + * control handles waiting via SLogTupleGetDirtyXid. + */ + } + + /* + * Fast-path: if UNCOMMITTED is set but no sLog entry exists, the previous + * transaction committed and its sLog cleanup already ran. Clear the stale + * flag now while we hold the buffer lock exclusively. This avoids the + * expensive sLog lookup inside the visibility check for the common case + * of UPDATing a recently-committed tuple. + * + * Only do this for tuples that are NOT deleted/updated (those flags + * indicate the tuple is being superseded, which requires the full + * visibility check to determine if the delete/update committed). + */ + if ((old_tuple_hdr->t_flags & FLUX_TUPLE_UNCOMMITTED) && + !(old_tuple_hdr->t_flags & (FLUX_TUPLE_DELETED | FLUX_TUPLE_UPDATED))) + { + if (!SLogTupleHasEntry(RelationGetRelid(relation), otid)) + { + old_tuple_hdr->t_flags &= ~FLUX_TUPLE_UNCOMMITTED; + /* Page will be dirtied by our upcoming update anyway */ + } + } + + /* + * Check tuple visibility against snapshot and handle concurrent + * modifications. Unlike a simple scan visibility check, UPDATE must + * distinguish between: - Truly invisible (another txn's uncommitted + * insert) ā†’ TM_Invisible - Concurrent update committed after our + * snapshot ā†’ TM_Updated - In-progress modification by another txn ā†’ + * wait, retry + */ + if (snapshot) + { + bool visible; + + visible = FluxTupleVisibleToSnapshotDual(old_tuple_hdr, snapshot, + RelationGetRelid(relation), + buffer); + + if (!visible) + { + TransactionId dirty_xid; + bool is_insert_entry; + + /* + * Writer-only probe (seqlock read). We must wait only on an + * in-progress INSERT/UPDATE/DELETE writer, never on a pure + * lock-only marker: a KeyShare locker (AccessShareLock tuplock) + * is compatible with our NoKeyExclusive update (ExclusiveLock + * tuplock) in the standard tuple-lock matrix, so blocking on its + * xid here -- while it is queued behind us reporting a lock + * conflict -- forms a mutual XactLockTableWait cycle that heap + * avoids via HEAP_XMAX_IS_LOCKED_ONLY. Real lock conflicts + * (Share/Exclusive lockers) are still serialized by the + * heavyweight tuplock below. + */ + dirty_xid = SLogTupleGetWriteConflictXid(RelationGetRelid(relation), + otid, + LockTupleNoKeyExclusive, + &is_insert_entry); + + /* Check if tuple was deleted by another transaction */ + if (old_tuple_hdr->t_flags & FLUX_TUPLE_DELETED) + { + if (tmfd) + { + tmfd->ctid = *otid; + tmfd->xmax = TransactionIdIsValid(dirty_xid) ? + dirty_xid : GetCurrentTransactionId(); + tmfd->cmax = InvalidCommandId; + tmfd->traversed = false; + } + UnlockReleaseBuffer(buffer); + FLUX_RELEASE_TUPLOCK(); + return TM_Deleted; + } + + /* + * Buffer lock was never released (wait-free sLog read), so the + * tuple cannot have changed. Proceed with dirty_xid. + */ + { + if (TransactionIdIsValid(dirty_xid) && is_insert_entry) + { + /* + * Another txn's in-progress INSERT. The tuple truly + * doesn't exist in our snapshot. + */ + if (tmfd) + { + tmfd->ctid = *otid; + tmfd->xmax = dirty_xid; + tmfd->cmax = InvalidCommandId; + tmfd->traversed = false; + } + UnlockReleaseBuffer(buffer); + FLUX_RELEASE_TUPLOCK(); + return TM_Invisible; + } + + if (TransactionIdIsValid(dirty_xid) && !is_insert_entry) + { + /* + * Another txn's in-progress UPDATE/DELETE. Wait for it + * to finish and then retry (the tuple may be gone or + * changed). + */ + if (wait) + { + TransactionId wait_xid = dirty_xid; + + /* + * Serialize competing updaters via a heavyweight + * tuple lock before sleeping on the conflicting xid, + * so two backends racing on the same tuple queue + * instead of deadlocking. Matches heap_update. + */ + UnlockReleaseBuffer(buffer); + if (!have_tuple_lock) + FluxLockTuple(relation, otid, LockTupleNoKeyExclusive, + true, &have_tuple_lock); + XactLockTableWait(wait_xid, relation, + otid, XLTW_Update); + + /* Re-read the page and re-check after waking */ + buffer = ReadBuffer(relation, blkno); + LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); + page = BufferGetPage(buffer); + + if (offnum < FirstOffsetNumber || + offnum > PageGetMaxOffsetNumber(page)) + { + UnlockReleaseBuffer(buffer); + FLUX_RELEASE_TUPLOCK(); + return TM_Invisible; + } + itemid = PageGetItemId(page, offnum); + if (!ItemIdIsNormal(itemid)) + { + UnlockReleaseBuffer(buffer); + FLUX_RELEASE_TUPLOCK(); + return TM_Invisible; + } + old_tuple_hdr = (FluxTupleHeader *) + PageGetItem(page, itemid); + + /* If it got deleted while we waited, report that */ + if (old_tuple_hdr->t_flags & FLUX_TUPLE_DELETED) + { + if (tmfd) + { + tmfd->ctid = *otid; + tmfd->xmax = wait_xid; + tmfd->cmax = InvalidCommandId; + tmfd->traversed = false; + } + UnlockReleaseBuffer(buffer); + FLUX_RELEASE_TUPLOCK(); + return TM_Deleted; + } + + /* + * Re-check visibility. The tuple was modified by the + * now-committed txn; its commit_ts is now later than + * our snapshot -> TM_Updated so the executor can EPQ. + */ + visible = FluxTupleVisibleToSnapshotDual(old_tuple_hdr, snapshot, + RelationGetRelid(relation), + buffer); + + if (!visible) + { + /* + * Still not visible after the waited-on txn + * committed. Before returning TM_Updated (which + * triggers another EPQ cycle), check whether we + * already hold a LOCK entry from a previous EPQ + * iteration. If so, we've already re-evaluated + * the quals and should proceed with the update + * instead of looping forever. + * + * Without this check, the following livelock + * occurs with hot-row contention: + * + * 1. We return TM_Updated ā†’ executor EPQ 2. + * table_tuple_lock inserts LOCK_EXCL 3. Retry ā†’ + * another txn is in-progress ā†’ wait 4. Waited + * txn commits ā†’ still not visible 5. Return + * TM_Updated ā†’ goto 2 (infinite) + * + * Each iteration leaks per-query memory in the + * executor, eventually causing OOM. + */ + TransactionId myxid_postw = + GetCurrentTransactionIdIfAny(); + + if (TransactionIdIsValid(myxid_postw)) + { + SLogTupleOp my_entry_postw; + int my_nfound_postw; + + my_nfound_postw = SLogTupleLookupFiltered( + RelationGetRelid(relation), + otid, myxid_postw, + &my_entry_postw, 1); + + if (my_nfound_postw > 0) + { + /* + * Our LOCK entry from a prior EPQ cycle + * exists. Fall through to perform the + * update. + */ + } + else + { + if (tmfd) + { + tmfd->ctid = *otid; + tmfd->xmax = wait_xid; + tmfd->cmax = InvalidCommandId; + tmfd->traversed = false; + } + UnlockReleaseBuffer(buffer); + FLUX_RELEASE_TUPLOCK(); + return TM_Updated; + } + } + else + { + if (tmfd) + { + tmfd->ctid = *otid; + tmfd->xmax = wait_xid; + tmfd->cmax = InvalidCommandId; + tmfd->traversed = false; + } + UnlockReleaseBuffer(buffer); + FLUX_RELEASE_TUPLOCK(); + return TM_Updated; + } + } + /* Now visible ā€” fall through to perform the update */ + } + else + { + /* NOWAIT mode */ + if (tmfd) + { + tmfd->ctid = *otid; + tmfd->xmax = dirty_xid; + tmfd->cmax = InvalidCommandId; + tmfd->traversed = false; + } + UnlockReleaseBuffer(buffer); + FLUX_RELEASE_TUPLOCK(); + return TM_WouldBlock; + } + } + else + { + /* + * No in-progress sLog entry for another transaction. The + * modification has already committed. + * + * Check if our own transaction already has a sLog entry + * for this TID (e.g., LOCK_EXCL placed by + * table_tuple_lock during EvalPlanQual). If so, EPQ + * already re-evaluated the WHERE clause and we should + * proceed with the update. + * + * Without this, we return TM_Updated endlessly: FLUX's + * in-place updates mean the tuple's commit_ts permanently + * exceeds the statement snapshot, so the executor's EPQ + * retry loop never terminates. + */ + TransactionId myxid_chk = + GetCurrentTransactionIdIfAny(); + + if (TransactionIdIsValid(myxid_chk)) + { + SLogTupleOp my_entry; + int my_nfound; + + my_nfound = SLogTupleLookupFiltered( + RelationGetRelid(relation), + otid, myxid_chk, &my_entry, 1); + if (my_nfound > 0) + { + /* + * Our own sLog entry exists (LOCK from EPQ path). + * Fall through to perform the update. + */ + } + else + { + /* + * First encounter: trigger EPQ. + */ + if (tmfd) + { + tmfd->ctid = *otid; + tmfd->xmax = + InvalidTransactionId; + tmfd->cmax = InvalidCommandId; + tmfd->traversed = false; + } + UnlockReleaseBuffer(buffer); + FLUX_RELEASE_TUPLOCK(); + return TM_Updated; + } + } + else + { + if (tmfd) + { + tmfd->ctid = *otid; + tmfd->xmax = InvalidTransactionId; + tmfd->cmax = InvalidCommandId; + tmfd->traversed = false; + } + UnlockReleaseBuffer(buffer); + FLUX_RELEASE_TUPLOCK(); + return TM_Updated; + } + } + } + /* If now visible, fall through to perform the update */ + } + + /* + * Write-write conflict against an ALREADY-COMMITTED concurrent + * update. + * + * The visibility check above always returns "visible" for a committed + * in-place update, because at commit time t_commit_ts is rewound to + * the original insert timestamp (so mid-life readers still see the + * row). That rewind erases the "updated since you read it" signal + * heap keeps in xmax, so visibility alone can never detect a lost + * update. Instead, the on-page tuple's trailing verptr (WS-PVS1) + * points at the UNDO-fork record produced by the last committed + * update; if that committer is invisible to our snapshot, taking the + * in-place UPDATE now would silently lose it. Return TM_Updated so + * the executor re-evaluates via EvalPlanQual, matching heap. + * + * Converge like heap: bounce to EPQ once per DISTINCT committed + * update, not once per probe. FluxEpqReconcileMatches suppresses + * re-firing on the identical (verptr, xid) marker we already bounced + * on this statement; FluxEpqReconcileMark records the marker before + * we return. A strictly-newer committer stamps a fresh verptr, so + * identity dedup only suppresses the same marker and never masks a + * genuine new conflict. + */ + if (IsMVCCSnapshot(snapshot)) + { + RelUndoRecPtr head_verptr; + TransactionId head_xid; + bool head_inprogress = false; + + if (FluxTupleHasCommittedUpdateAfter(relation, + old_tuple_hdr, + ItemIdGetLength(itemid), + snapshot, + GetCurrentTransactionIdIfAny(), + &head_verptr, + &head_xid, + &head_inprogress) && + !FluxEpqReconcileMatches(snapshot, + RelationGetRelid(relation), + otid, head_verptr, head_xid)) + { + FluxEpqReconcileMark(snapshot, + RelationGetRelid(relation), otid, + head_verptr, head_xid); + if (tmfd) + { + tmfd->ctid = *otid; + tmfd->xmax = InvalidTransactionId; + tmfd->cmax = InvalidCommandId; + tmfd->traversed = false; + } + UnlockReleaseBuffer(buffer); + FLUX_RELEASE_TUPLOCK(); + return TM_Updated; + } + else if (head_inprogress) + { + /* + * Commit-window conflict (see + * FluxTupleHasCommittedUpdateAfter): the head committer + * cleared its sLog marker + UNCOMMITTED flag at PRE_COMMIT + * but has not yet reached CLOG. Neither this gate nor the + * sLog dirty-xid probe sees it, so we would clobber a + * just-committed update. Wait on the in-flight writer, then + * retry from the top of the slow path where CLOG now resolves + * it to a committed conflict -> TM_Updated -> EPQ. + * Heap-identical. + */ + if (wait) + { + TransactionId wait_xid = head_xid; + + /* + * Wait for the in-flight committer to reach CLOG or + * abort, then report TM_Updated so the executor + * re-fetches through EvalPlanQual. EPQ's SnapshotAny + * fetch reconstructs the correct visible version (the + * committed value if head_xid committed, or the restored + * before-image if it aborted) and re-projects the update + * on top. A blind update retry here would instead apply + * over the raw on-page bytes, which on abort are the + * not-yet-reverted value. + */ + UnlockReleaseBuffer(buffer); + if (!have_tuple_lock) + FluxLockTuple(relation, otid, LockTupleNoKeyExclusive, + true, &have_tuple_lock); + XactLockTableWait(wait_xid, relation, otid, XLTW_Update); + if (tmfd) + { + tmfd->ctid = *otid; + tmfd->xmax = InvalidTransactionId; + tmfd->cmax = InvalidCommandId; + tmfd->traversed = false; + } + FLUX_RELEASE_TUPLOCK(); + return TM_Updated; + } + } + } + } + + /* + * Even when visibility returned "true", the tuple may have an in-progress + * modification by another transaction. This happens when + * FluxTupleVisibleToSnapshotDual returns true for in-progress + * UPDATE/DELETE entries (to preserve tuple existence in scans). We must + * still detect the write-write conflict and block. + * + * The in-progress writer is detected authoritatively via the sLog + * (SLogTupleGetDirtyXid filters by TransactionIdIsInProgress), NOT via + * the on-page FLUX_TUPLE_UNCOMMITTED flag. The page flag and the sLog + * marker live in two domains (buffer locks vs. the sLog seqlock) that are + * not updated atomically, so a live writer's marker can exist while the + * page flag is transiently clear. Gating this wait on the flag (as we + * once did) let such a writer slip past, silently clobbering its + * in-progress update. Consult the sLog unconditionally; only use the flag + * to opportunistically clear stale state when no writer is present. + */ + { + TransactionId dirty_xid; + bool is_insert_entry; + + /* + * Lock-free. Probe for a transaction that conflicts with our + * NoKeyExclusive update under the real heavyweight tuple-lock matrix: + * an in-progress INSERT/UPDATE/DELETE writer always conflicts, and a + * lock-only marker conflicts iff its recorded LockTupleMode does (FOR + * UPDATE/FOR SHARE block; a KeyShare FK locker is compatible and does + * not). A pure writer-only probe would sail past a FOR UPDATE locker + * that left only a LOCK_EXCL marker and clobber the row it protects + * -- a correctness failure. We then acquire the same heavyweight + * LOCKTAG_TUPLE lock and XactLockTableWait on the conflicting xid, + * which serializes us into a FIFO queue rather than deadlocking. + */ + dirty_xid = SLogTupleGetWriteConflictXid(RelationGetRelid(relation), + otid, + LockTupleNoKeyExclusive, + &is_insert_entry); + + if (!TransactionIdIsValid(dirty_xid)) + { + /* + * No active writer. If the page still carries a stale + * UNCOMMITTED flag, clear it opportunistically. + */ + if (old_tuple_hdr->t_flags & FLUX_TUPLE_UNCOMMITTED) + { + old_tuple_hdr->t_flags &= ~FLUX_TUPLE_UNCOMMITTED; + MarkBufferDirty(buffer); + } + } + else if (TransactionIdIsValid(dirty_xid) && + !TransactionIdIsCurrentTransactionId(dirty_xid) && + !is_insert_entry) + { + /* + * Another transaction has an in-progress UPDATE/DELETE. Block + * until it finishes, then re-check. + */ + if (wait) + { + TransactionId wait_xid = dirty_xid; + + /* + * Serialize competing updaters via a heavyweight tuple lock + * before sleeping on the conflicting xid, so two backends + * racing on the same tuple queue instead of deadlocking. + * Matches heap_update. + */ + UnlockReleaseBuffer(buffer); + if (!have_tuple_lock) + FluxLockTuple(relation, otid, LockTupleNoKeyExclusive, + true, &have_tuple_lock); + XactLockTableWait(wait_xid, relation, otid, XLTW_Update); + + /* Re-read page after waking */ + buffer = ReadBuffer(relation, blkno); + LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); + page = BufferGetPage(buffer); + + if (offnum < FirstOffsetNumber || + offnum > PageGetMaxOffsetNumber(page)) + { + UnlockReleaseBuffer(buffer); + FLUX_RELEASE_TUPLOCK(); + return TM_Invisible; + } + itemid = PageGetItemId(page, offnum); + if (!ItemIdIsNormal(itemid)) + { + UnlockReleaseBuffer(buffer); + FLUX_RELEASE_TUPLOCK(); + return TM_Invisible; + } + old_tuple_hdr = (FluxTupleHeader *) + PageGetItem(page, itemid); + + /* If deleted while we waited, report that */ + if (old_tuple_hdr->t_flags & FLUX_TUPLE_DELETED) + { + if (tmfd) + { + tmfd->ctid = *otid; + tmfd->xmax = wait_xid; + tmfd->cmax = InvalidCommandId; + tmfd->traversed = false; + } + UnlockReleaseBuffer(buffer); + FLUX_RELEASE_TUPLOCK(); + return TM_Deleted; + } + + /* + * The waited-on txn finished (committed or aborted). Return + * TM_Updated to force the executor to re-evaluate via EPQ + * against the now-stable page. + * + * This is required even on ABORT, and is where FLUX differs + * from heap. FLUX updates in place, so a READ COMMITTED scan + * that ran while the other writer was in progress dirty-read + * that writer's uncommitted value as the base for the new + * tuple's expression (e.g. counter = counter + 10 computed + * off the in-flight value). Heap never sees this because its + * scan reads the prior committed version. By the time we get + * here: - COMMIT: the page holds the other writer's committed + * value; EPQ recomputes on top of it. - ABORT: + * ApplyPerRelUndo() has already restored the pre-update image + * inline (before the conflicting XID left the proc array), so + * the page holds the original committed value; EPQ recomputes + * on top of that. Either way EPQ re-reads the correct base + * and recomputes, matching heap's final result. + */ + if (tmfd) + { + tmfd->ctid = *otid; + tmfd->xmax = wait_xid; + tmfd->cmax = InvalidCommandId; + tmfd->traversed = false; + } + UnlockReleaseBuffer(buffer); + FLUX_RELEASE_TUPLOCK(); + return TM_Updated; + } + else + { + /* NOWAIT mode */ + if (tmfd) + { + tmfd->ctid = *otid; + tmfd->xmax = dirty_xid; + tmfd->cmax = InvalidCommandId; + tmfd->traversed = false; + } + UnlockReleaseBuffer(buffer); + FLUX_RELEASE_TUPLOCK(); + return TM_WouldBlock; + } + } + /* If dirty_xid is our own or invalid, proceed with update */ + } + + /* + * Final committed-update gate (authoritative). + * + * STAGE 2 above runs the committed-update detector once, before the + * in-progress-writer wait. That check races: while the head writer W is + * still in progress, the detector declines (W's undo record is not yet + * committed), so control falls through to the writer-wait stage. The + * buffer exclusive lock does NOT block W's commit -- W clears its + * in-progress sLog marker through the sLog seqlock domain, and because + * FLUX updates in place, the on-page value is already W's. If W commits + * in the window between STAGE 2's read and the writer-wait probe, the + * wait stage sees NO in-progress writer and would fall through to apply + * our update over W's now-committed value -- silently losing W's update. + * + * Re-run the detector here, after the wait stage has resolved, as the + * last action before the update proper. Every path that reaches the + * update passes through this point with the buffer exclusively locked, so + * a committer that landed anywhere upstream is caught. Reconcile dedup + * makes this converge exactly like STAGE 2: we bounce to EvalPlanQual + * once per distinct committed marker; a repeat of the identical (verptr, + * xid) marker we already reconciled falls through and applies. + */ + if (snapshot && IsMVCCSnapshot(snapshot)) + { + RelUndoRecPtr head_verptr; + TransactionId head_xid; + bool head_inprogress = false; + + if (FluxTupleHasCommittedUpdateAfter(relation, + old_tuple_hdr, + ItemIdGetLength(itemid), + snapshot, + GetCurrentTransactionIdIfAny(), + &head_verptr, + &head_xid, + &head_inprogress)) + { + if (!FluxEpqReconcileMatches(snapshot, + RelationGetRelid(relation), + otid, head_verptr, head_xid)) + { + FluxEpqReconcileMark(snapshot, + RelationGetRelid(relation), otid, + head_verptr, head_xid); + if (tmfd) + { + tmfd->ctid = *otid; + tmfd->xmax = InvalidTransactionId; + tmfd->cmax = InvalidCommandId; + tmfd->traversed = false; + } + UnlockReleaseBuffer(buffer); + FLUX_RELEASE_TUPLOCK(); + return TM_Updated; + } + } + else if (head_inprogress) + { + /* + * Commit-window conflict (see FluxTupleHasCommittedUpdateAfter): + * the head committer cleared its sLog marker + UNCOMMITTED flag + * at PRE_COMMIT but has not yet reached CLOG. Neither this gate + * nor the sLog dirty-xid probe sees it, so we would clobber a + * just-committed update. Wait on the in-flight writer, then + * retry from the top of the slow path where CLOG now resolves it + * to a committed conflict -> TM_Updated -> EPQ. Heap-identical. + */ + if (wait) + { + TransactionId wait_xid = head_xid; + + /* + * Wait for the in-flight committer to reach CLOG or abort, + * then report TM_Updated so the executor re-fetches through + * EvalPlanQual. EPQ's SnapshotAny fetch reconstructs the + * correct visible version (the committed value if head_xid + * committed, or the restored before-image if it aborted) and + * re-projects the update on top. A blind update retry here + * would instead apply over the raw on-page bytes, which on + * abort are the not-yet-reverted value. + */ + UnlockReleaseBuffer(buffer); + if (!have_tuple_lock) + FluxLockTuple(relation, otid, LockTupleNoKeyExclusive, + true, &have_tuple_lock); + XactLockTableWait(wait_xid, relation, otid, XLTW_Update); + if (tmfd) + { + tmfd->ctid = *otid; + tmfd->xmax = InvalidTransactionId; + tmfd->cmax = InvalidCommandId; + tmfd->traversed = false; + } + FLUX_RELEASE_TUPLOCK(); + return TM_Updated; + } + } + } + + /* + * Get transaction timestamp BEFORE critical section (initializes the + * per-transaction MVCC state FluxGetDmlTimestamp relies on). Commit + * visibility comes from CLOG (heap-shaped xmin/xmax MVCC). + */ + (void) FluxGetTransactionTimestamp(); + current_ts = (uint64) FluxGetDmlTimestamp(); + + /* + * Ensure the current transaction has an XID assigned BEFORE entering the + * critical section. GetCurrentTransactionId() may call + * XactLockTableInsert() which acquires a lock and allocates memory -- + * both forbidden in a critical section. + * + * Without an assigned XID, RecordTransactionCommit() considers the + * transaction read-only and skips the WAL flush, even though we write WAL + * records for the data change. This would cause the update to be lost on + * crash recovery. + */ + (void) GetCurrentTransactionId(); + + /* + * Form the new tuple from the slot. + * + * Fast path: If the old tuple is small (no overflow potential), keep the + * buffer locked and form the tuple without overflow handling. This + * avoids the expensive unlock/relock cycle and the re-validation that + * follows. + * + * Slow path: For large tuples or those with existing overflow data, + * release the buffer lock first. FluxFormTuple may call + * FluxStoreOverflowColumn which acquires buffer locks on overflow pages. + * If overflow data lands on the same page we're updating, that would + * cause a buffer lock re-entry assertion failure. + */ + update_overflow_buffers.count = 0; + { + bool buffer_unlocked = false; + + /* + * This in-place UPDATE path runs only when no indexed column changed + * (a key-changing UPDATE is routed to the out-of-place path), so the + * varlena columns are typically unchanged. FLUX has no on-page + * overflow: TOAST any oversized varlena into the relation's standard + * heap TOAST table before forming, exactly like the insert path, then + * form the tuple from the (possibly externalized) datums. + */ + slot_getallattrs(slot); + { + int upd_natts = RelationGetDescr(relation)->natts; + + memcpy(upd_toast_values, slot->tts_values, upd_natts * sizeof(Datum)); + memcpy(upd_toast_isnull, slot->tts_isnull, upd_natts * sizeof(bool)); + flux_toast_tuple(relation, upd_toast_values, upd_toast_isnull, + NULL, NULL, &upd_ttc, upd_toast_attr, + &upd_toasted, 0); + + new_tuple = FluxFormTuple(RelationGetDescr(relation), + upd_toast_values, + upd_toast_isnull, + NULL, /* FLUX has no on-page overflow */ + NULL); + } + + /* Set MVCC fields for new tuple (heap-shaped: fresh version) */ + new_tuple->t_data->t_commit_ts = 0; /* t_xmax = InvalidTransactionId */ + + /* + * The new on-page image is the NEWEST version; stamp its inserter + * (t_xmin) with our XID. Older snapshots that cannot see this XID + * read the pre-update image back from the UNDO fork via t_verptr. + */ + new_tuple->t_data->t_flags |= FLUX_TUPLE_UNCOMMITTED; + new_tuple->t_data->t_xmin = GetCurrentTransactionId(); /* subxid: heap-shaped, + * so savepoint rollback + * marks it aborted in + * CLOG */ + new_tuple_size = new_tuple->t_len; + + /* + * WS-PVS1: the version-chain head lives in the fixed header field + * t_verptr, so new_tuple already carries it inside its header with no + * trailing growth. The real urec_ptr is stamped in below after + * RelUndoReserve returns; until then t_verptr holds + * InvalidRelUndoRecPtr (consistent with the unlogged/temp path that + * never reserves). + */ + new_tuple->t_data->t_flags |= FLUX_TUPLE_HAS_VERSION_PTR; + FluxTupleSetVersionPtr(new_tuple->t_data, new_tuple_size, + InvalidRelUndoRecPtr); + + /* + * Pre-compute the oldest active timestamp before (re-)acquiring the + * buffer lock. This avoids an O(MaxBackends) shared-memory scan + * while holding a page-level exclusive lock. The value is used by + * the defrag estimation/execution path below. + */ + defrag_oldest_ts = FluxGetOldestActiveTimestamp(); + defrag_oldest_xmin = FluxGetOldestXminHorizon(relation); + + if (buffer_unlocked) + { + /* + * Re-acquire the buffer lock for the in-place update decision. + * Check if the main buffer is already locked as part of the + * overflow buffers to avoid double-lock assertion failure. + */ + bool buffer_already_locked = false; + + for (upd_i = 0; upd_i < update_overflow_buffers.count; upd_i++) + { + if (update_overflow_buffers.buffers[upd_i].buffer == buffer) + { + buffer_already_locked = true; + break; + } + } + + if (!buffer_already_locked) + LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); + + page = BufferGetPage(buffer); + + /* + * Re-validate the tuple after re-locking. Another backend may + * have reorganized the page while we didn't hold the lock. + */ + itemid = PageGetItemId(page, offnum); + if (!ItemIdIsNormal(itemid)) + { + UnlockReleaseBuffer(buffer); + FluxFreeTuple(new_tuple); + FLUX_RELEASE_TUPLOCK(); + return TM_Invisible; + } + old_tuple_hdr = (FluxTupleHeader *) PageGetItem(page, itemid); + } + } + + /* + * Determine if we can do an in-place update. + * + * In-place update is FLUX's primary advantage over heap: it avoids + * creating dead tuple versions and the associated index maintenance. We + * try several strategies in order of increasing cost: + * + * 1. Direct fit: new tuple fits within the old tuple's slot. 2. Page + * space fit: new tuple is larger but the extra bytes fit in the page's + * available free space. 3. Defrag fit: page defragmentation frees enough + * space for the new tuple to fit in-place. + * + */ +retry_fit: + if (new_tuple_size <= ItemIdGetLength(itemid)) + { + /* Strategy 1: new tuple fits within old tuple's slot */ + } + else if (new_tuple_size <= ItemIdGetLength(itemid) + PageGetFreeSpace(page)) + { + /* + * Strategy 2: new tuple is larger but the difference fits in the + * page's free space. We need to relocate the tuple data within the + * page, which PageRepairFragmentation can handle. + */ + } + else + { + /* + * Strategy 3: try page defragmentation to reclaim dead tuple space. + * If the page has the defrag-needed flag and defragmentation would + * free enough space, do it now. + */ + FluxPageOpaque upd_opaque = FluxPageGetOpaque(page); + Size potential_free; + + (void) upd_opaque; /* flag is only an optimization hint; see + * below */ + + { + /* + * Strategy 3: reclaim space occupied by dead-to-all superseded + * versions via defragmentation. + * + * We DO NOT gate this on FLUX_PAGE_DEFRAG_NEEDED. That flag is + * an optimization hint set by the pruning paths, but in-place + * UPDATEs accumulate dead-to-all versions on a hot page (e.g. + * TPC-C district) without reliably setting it. Trusting the flag + * as authoritative made a 2-byte tuple growth spuriously fail + * with "does not fit" on a page that was actually full of + * reclaimable dead versions (measured: ~34k such aborts/run, + * contigfree=0, maxoff 47-69 on a 500-row table). Always scan; + * only error if a full defrag genuinely cannot free enough space. + */ + /* + * Estimate how much space defragmentation could free by scanning + * for dead tuples. This is a quick scan without actually + * defragmenting yet. + */ + potential_free = PageGetFreeSpace(page); + { + OffsetNumber df_off; + OffsetNumber df_maxoff = PageGetMaxOffsetNumber(page); + + for (df_off = FirstOffsetNumber; df_off <= df_maxoff; df_off++) + { + ItemId df_itemid = PageGetItemId(page, df_off); + FluxTupleHeader *df_hdr; + + if (!ItemIdIsNormal(df_itemid)) + { + if (ItemIdIsDead(df_itemid)) + potential_free += ItemIdGetLength(df_itemid) + sizeof(ItemIdData); + continue; + } + + if (FluxIsOverflowRecordInline(PageGetItem(page, df_itemid), + ItemIdGetLength(df_itemid))) + continue; + + df_hdr = (FluxTupleHeader *) PageGetItem(page, df_itemid); + + if (FluxTupleDeadToAll(df_hdr, defrag_oldest_xmin)) + { + potential_free += ItemIdGetLength(df_itemid) + sizeof(ItemIdData); + } + } + } + + if (new_tuple_size <= ItemIdGetLength(itemid) + potential_free) + { + /* + * Defragmentation should free enough space. Do it now. We + * are already holding an exclusive lock on the buffer. First + * mark dead tuples LP_DEAD, then defragment. + * + * As in FluxPagePruneOpt, opportunistic pruning sets LP_DEAD + * (reclaiming storage) rather than LP_UNUSED. The deleted + * tuples may still have index entries; reserving the line + * pointer until VACUUM removes those entries prevents the TID + * from being recycled and returning wrong index-scan results. + * + * defrag_oldest_ts was pre-computed before acquiring the + * buffer lock to avoid shared-memory scans while holding + * page-level exclusive locks. + */ + START_CRIT_SECTION(); + { + OffsetNumber prune_off; + OffsetNumber prune_maxoff = PageGetMaxOffsetNumber(page); + + for (prune_off = FirstOffsetNumber; prune_off <= prune_maxoff; prune_off++) + { + ItemId prune_itemid = PageGetItemId(page, prune_off); + FluxTupleHeader *prune_hdr; + + if (!ItemIdIsNormal(prune_itemid)) + continue; + + if (FluxIsOverflowRecordInline(PageGetItem(page, prune_itemid), + ItemIdGetLength(prune_itemid))) + continue; + + prune_hdr = (FluxTupleHeader *) PageGetItem(page, prune_itemid); + + if (FluxTupleDeadToAll(prune_hdr, defrag_oldest_xmin)) + { + ItemIdSetDead(prune_itemid); + } + } + } + FluxPageDefragment(page); + MarkBufferDirty(buffer); + + if (RelationNeedsWAL(relation)) + { + XLogRecPtr df_lsn; + + df_lsn = FluxXLogDefrag(relation, buffer, NULL, 0, defrag_oldest_ts); + PageSetLSN(page, df_lsn); + } + END_CRIT_SECTION(); + + FluxRecordFreeSpace(relation, blkno, PageGetFreeSpace(page)); + + /* + * Re-fetch the item after defragmentation since line pointers + * may have been reorganized. The offset number should still + * be valid for surviving tuples. + */ + itemid = PageGetItemId(page, offnum); + old_tuple_hdr = (FluxTupleHeader *) PageGetItem(page, itemid); + + /* Check again if in-place update now fits */ + if (new_tuple_size <= ItemIdGetLength(itemid) + PageGetFreeSpace(page)) + { + flux_stat_defrag_triggered_updates++; + } + else + { + if (!force_shrink_attempted && + update_overflow_buffers.count == 0) + goto force_shrink_retry; + flux_release_update_overflow_buffers(&update_overflow_buffers, + buffer); + UnlockReleaseBuffer(buffer); + FluxFreeTuple(new_tuple); + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("updated flux tuple does not fit on page after defragmentation"), + errhint("FLUX tuples have stable TIDs and cannot move to another page. Lower the table's fillfactor to reserve room for in-place growth."))); + } + } + else + { + /* Defrag wouldn't free enough space */ + if (!force_shrink_attempted && + update_overflow_buffers.count == 0) + goto force_shrink_retry; + flux_release_update_overflow_buffers(&update_overflow_buffers, + buffer); + UnlockReleaseBuffer(buffer); + FluxFreeTuple(new_tuple); + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("updated flux tuple does not fit on page"), + errdetail("The updated row is larger than its current slot, all variable-length columns are already stored off-page, and the page has no free space."), + errhint("FLUX tuples have stable TIDs and cannot move to another page. Lower the table's fillfactor to reserve room for in-place growth."))); + } + } + } + + if (false) + { + /* + * Recovery path for an in-place-grown tuple that no longer fits in + * its slot, even after page defragmentation. FLUX TIDs are stable, + * so the tuple cannot move to another page; the only way to shrink it + * is to push variable-length columns off-page. Re-form the new tuple + * forcing every eligible varlena column into overflow with a zero + * inline prefix, which collapses the main tuple to its minimum + * footprint, then retry the in-place fit (it should now satisfy + * Strategy 1). + * + * This branch is only entered from the ERROR sites above, and only + * when the first form attempt took the no-overflow fast path (the + * main buffer is still locked and no overflow buffers were + * collected). + */ +force_shrink_retry: + { + bool relock_already_locked = false; + + FluxFreeTuple(new_tuple); + + /* Release the main buffer lock before re-forming with overflow. */ + LockBuffer(buffer, BUFFER_LOCK_UNLOCK); + + new_tuple = FluxFormTupleForceShrink(RelationGetDescr(relation), + slot->tts_values, + slot->tts_isnull, + relation, + &update_overflow_buffers); + + new_tuple->t_data->t_commit_ts = 0; /* t_xmax = + * InvalidTransactionId */ + new_tuple->t_data->t_flags |= FLUX_TUPLE_UNCOMMITTED; + new_tuple->t_data->t_xmin = GetCurrentTransactionId(); /* subxid: heap-shaped, + * so savepoint rollback + * marks it aborted in + * CLOG */ + new_tuple_size = new_tuple->t_len; + + /* + * WS-PVS1: stamp InvalidRelUndoRecPtr into t_verptr, mirroring + * the main exclusive path, regardless of whether we reached this + * branch via fit cascade or force-shrink retry. + */ + new_tuple->t_data->t_flags |= FLUX_TUPLE_HAS_VERSION_PTR; + FluxTupleSetVersionPtr(new_tuple->t_data, new_tuple_size, + InvalidRelUndoRecPtr); + + /* + * Re-acquire the main buffer lock. An overflow column may have + * landed on the page we are updating, in which case it is already + * locked as part of update_overflow_buffers. + */ + for (upd_i = 0; upd_i < update_overflow_buffers.count; upd_i++) + { + if (update_overflow_buffers.buffers[upd_i].buffer == buffer) + { + relock_already_locked = true; + break; + } + } + if (!relock_already_locked) + LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); + + page = BufferGetPage(buffer); + + /* Re-validate the line pointer after re-locking. */ + itemid = PageGetItemId(page, offnum); + if (!ItemIdIsNormal(itemid)) + { + flux_release_update_overflow_buffers(&update_overflow_buffers, + buffer); + UnlockReleaseBuffer(buffer); + FluxFreeTuple(new_tuple); + FLUX_RELEASE_TUPLOCK(); + return TM_Invisible; + } + old_tuple_hdr = (FluxTupleHeader *) PageGetItem(page, itemid); + + /* + * Re-validate visibility after the unlock/relock window. The fit + * cascade ran conflict detection before forming the tuple, but + * re-forming with overflow drops the buffer lock again. A + * concurrent committed delete that landed in that window must be + * reported as TM_Deleted rather than silently overwritten. + */ + if ((old_tuple_hdr->t_flags & FLUX_TUPLE_DELETED) && + !(old_tuple_hdr->t_flags & FLUX_TUPLE_UNCOMMITTED)) + { + flux_release_update_overflow_buffers(&update_overflow_buffers, + buffer); + UnlockReleaseBuffer(buffer); + FluxFreeTuple(new_tuple); + FLUX_RELEASE_TUPLOCK(); + return TM_Deleted; + } + + force_shrink_attempted = true; + goto retry_fit; + } + } + + /* + * Save a copy of the old tuple data BEFORE entering the critical section + * and BEFORE modifying the page. palloc is not allowed inside critical + * sections, and in-place updates overwrite the on-page data, so we must + * preserve the original tuple for WAL logging (the before-image). + * + * For small tuples (common case in pgbench-style OLTP), use stack buffers + * to avoid palloc overhead in the hot path. + */ + { + uint32 old_len = ItemIdGetLength(itemid); + char *old_copy; + + old_tuple_for_inplace_wal = palloc0(sizeof(FluxTupleData)); + old_copy = palloc(old_len); + + memcpy(old_copy, old_tuple_hdr, old_len); + old_tuple_for_inplace_wal->t_len = old_len; + old_tuple_for_inplace_wal->t_data = (FluxTupleHeader *) old_copy; + } + + /* + * NOTE: The early UNDO record (pre-modification) was removed to avoid + * double UNDO records. The deferred UNDO path below (upd_undo_ptr) + * handles UNDO recording for all in-place updates. The deferred approach + * is safe because the WAL record includes both old and new tuple data, + * enabling crash recovery regardless of UNDO write order. + */ + + + /* + * Pre-allocate WAL buffer space BEFORE entering critical section. We may + * need to register the main buffer plus overflow buffers. + * + * rdata slots needed for UPDATE: MAX_OVERFLOW_BUFFERS (data per overflow + * record, no separate header) + 3 (xl_flux_update header + old tuple data + * + new tuple data) + * + * CRITICAL: XLogEnsureRecordSpace() may allocate memory, so it MUST be + * called outside the critical section. + */ + if (RelationNeedsWAL(relation)) + XLogEnsureRecordSpace(XLR_MAX_BLOCK_ID, 4 + MAX_OVERFLOW_BUFFERS); + + /* + * Per-relation UNDO: reserve space for a full-tuple UPDATE record before + * the critical section. The record stores the old tuple so a top-level + * ROLLBACK can restore it in place (RelUndoApplyUpdate handles same-size, + * grow, and shrink). RelUndoReserve may extend the fork and ereport, so + * it must run outside the crit section. Lock order: data buffer then + * UNDO buffer. + * + * If a force-shrink retry pushed columns off-page, + * update_overflow_buffers holds buffers that are pinned, content-locked, + * and marked dirty but not yet WAL-logged (overflow WAL is deferred to + * the atomic record below). RelUndoReserve and + * CheckForSerializableConflictIn can both ereport here, before the + * critical section makes the change durable. Resource-owner cleanup + * would still release the pins and locks on abort, leaving only + * VACUUM-reclaimable dead overflow space, but release them explicitly so + * the abort path leaves no dirtied-yet-orphaned overflow pages behind. + */ + PG_TRY(); + { + if (smgrexists(RelationGetSmgr(relation), RELUNDO_FORKNUM)) + { + Size upd_undo_reserve; + + /* + * WS-PVS1: reserve a worst-case (full-tuple) record BEFORE + * computing the diff so we can stamp the returned RelUndoRecPtr + * into new_tuple's trailing version slot AHEAD of the diff scan + * and the page memcpy. The diff/full-tuple commit happens after + * stamping; over-reservation by a few bytes is wasted UNDO-fork + * space, not a correctness hazard. Mirrors the CAS fast path + * (flux_operations.c:2018-2037). + */ + upd_undo_reserve = SizeOfRelUndoRecordHeader + + sizeof(RelUndoUpdatePayload) + + old_tuple_for_inplace_wal->t_len; + + upd_undo_ptr = RelUndoReserve(relation, upd_undo_reserve, + &upd_undo_buffer); + + /* + * Stamp the freshly reserved urec_ptr (never preserve a prior + * pointer): the new diff/full-tuple record reconstructs THIS + * update's before-image, and the header's urec_prevundorec + * already chains to the prior diff -- the row's verptr always + * points at the head of the chain. + */ + FluxTupleSetVersionPtr(new_tuple->t_data, new_tuple_size, + upd_undo_ptr); + } + + /* + * WS-PVS1: widen-then-narrow fixup (now a verptr no-op; retained for + * the flag-removal commit). The version pointer lives in the fixed + * header field t_verptr, so FluxTupleGetVersionPtr/SetVersionPtr read + * and write it independently of the slot length -- widening the image + * to the old (larger) slot length no longer moves the pointer. The + * in-place overwrite path below keeps the old line-pointer length + * when the new image is smaller; deform's self-describing t_natts + * lets the trailing slack be ignored by ordinary readers. We still + * widen + zero-pad here so the memcpy below fills the whole slot (no + * stale trailing bytes). + * + * Done here, before the critical section, because repalloc()/pfree() + * are forbidden inside it. + */ + if (new_tuple_size <= ItemIdGetLength(itemid)) + { + Size slot_len = ItemIdGetLength(itemid); + + if (slot_len > new_tuple_size && + (new_tuple->t_data->t_flags & FLUX_TUPLE_HAS_VERSION_PTR)) + { + RelUndoRecPtr verptr = FluxTupleGetVersionPtr(new_tuple->t_data, + new_tuple_size); + + new_tuple->t_data = (FluxTupleHeader *) + repalloc(new_tuple->t_data, slot_len); + MemSet((char *) new_tuple->t_data + new_tuple_size, 0, + slot_len - new_tuple_size); + new_tuple->t_len = (uint32) slot_len; + new_tuple_size = slot_len; + FluxTupleSetVersionPtr(new_tuple->t_data, new_tuple_size, verptr); + } + } + + /* + * transaction read this tuple (holds a SIREAD lock on it), our update + * creates an rw-antidependency that may form a dangerous structure. + */ + CheckForSerializableConflictIn(relation, otid, BufferGetBlockNumber(buffer)); + + /* + * Always set UNCOMMITTED so that visibility checks consult the sLog. + * Even though the tuple position hasn't moved (in-place update), the + * DATA has changed and other transactions must see the old data until + * this update commits. The flag will be lazily cleared on the first + * visibility check after the updating transaction commits (since the + * sLog entry will have been removed at commit time). + * + * Also set FLUX_TUPLE_UPDATED to mark that this tuple has been + * updated in-place. After commit, this flag persists and indicates + * that the tuple's t_commit_ts reflects the original INSERT commit + * time (not the UPDATE commit time). This preserves visibility for + * readers whose snapshots predate the update. + */ + new_tuple->t_data->t_flags |= FLUX_TUPLE_UPDATED; + + /* + * t_gen is reserved/unused in FLUX (FLUX uses plain heap-TID index + * identity, no RowID/gen scheme). Leave it at the palloc0 default. + */ + new_tuple->t_data->t_gen = 0; + + /* + * Build the heap-format logical-decoding images (old and new) BEFORE + * the critical section, since heap_form_tuple()/palloc() are + * forbidden inside it. When the relation is not logically logged + * these leave data == NULL. These calls palloc and can ereport on + * OOM, so they stay inside the PG_TRY: a throw here must also release + * the force-shrink overflow buffers, exactly like the reservation + * above. + */ + FluxXLogPrepareLogicalImage(relation, old_tuple_for_inplace_wal, + &update_old_img); + FluxXLogPrepareLogicalImage(relation, new_tuple, &update_new_img); + } + PG_CATCH(); + { + flux_release_update_overflow_buffers(&update_overflow_buffers, buffer); + PG_RE_THROW(); + } + PG_END_TRY(); + + /* Start critical section for WAL logging */ + START_CRIT_SECTION(); + + /* + * Set t_ctid on the in-memory new tuple BEFORE copying to the page. This + * ensures the WAL record's new_tuple image includes the correct t_ctid, + * so redo produces a page identical to the primary. (We use blkno/offnum + * which is correct for both the "fits in existing slot" and + * "delete+re-add" strategies since we update offnum below if it changes.) + */ + ItemPointerSet(&new_tuple->t_data->t_ctid, blkno, offnum); + + if (new_tuple_size <= ItemIdGetLength(itemid)) + { + /* + * New tuple fits within the old tuple's allocated space. Overwrite + * directly -- safe because we don't exceed the existing allocation. + * When the kept slot is larger than the new image, new_tuple was + * already widened to the full slot length before the critical section + * (see the WS-PVS1 widen block above), so new_tuple_size == slot_len + * here and the memcpy fills the whole slot, keeping any trailing + * slack zeroed. The version pointer lives in the header field + * t_verptr, so it is copied by the memcpy regardless of slot width. + */ + memcpy(old_tuple_hdr, new_tuple->t_data, new_tuple_size); + } + else + { + /* + * New tuple is larger than the old one but fits on the page (Strategy + * 2 or 3). We cannot memcpy in place because that would overwrite + * adjacent data. Instead, remove the old line pointer entry, compact + * the page, and re-add the new tuple at the same offset. + * + * We use FluxPageIndexTupleDelete instead of the standard + * PageIndexTupleDelete because FLUX pages may contain LP_UNUSED items + * left by opportunistic defragmentation. PageIndexTupleDelete asserts + * all items are LP_NORMAL, which fails when LP_UNUSED items are + * present. FluxPageIndexTupleDelete skips LP_UNUSED items in the + * offset adjustment loop. + */ + FluxPageIndexTupleDelete(page, offnum); + + offnum = PageAddItem(page, new_tuple->t_data, + new_tuple_size, + offnum, false, false); + + if (offnum == InvalidOffsetNumber) + elog(PANIC, "failed to re-add FLUX tuple after delete for growing update"); + + /* Re-fetch itemid and header from the (same) location */ + itemid = PageGetItemId(page, offnum); + old_tuple_hdr = (FluxTupleHeader *) PageGetItem(page, itemid); + + ItemPointerSet(&new_tuple->t_data->t_ctid, blkno, offnum); + } + + /* Set new TID to same location */ + ItemPointerSet(&slot->tts_tid, blkno, offnum); + new_tuple->t_self = slot->tts_tid; + + /* + * t_ctid on the on-disk tuple is already correct from the memcpy or + * PageAddItem above, since we set it on new_tuple->t_data before copying. + */ + + /* Track in-place update success */ + flux_stat_in_place_updates++; + + slot->tts_tableOid = RelationGetRelid(relation); + + /* + * Update page opaque header to track the latest commit timestamp and + * current free space. This must happen before MarkBufferDirty and WAL + * logging so that full-page images capture the correct opaque state. The + * redo function performs the same updates so WAL consistency checking + * passes. + */ + { + FluxPageOpaque upd_phdr = FluxPageGetOpaque(page); + + FluxPageSetCommitTs(upd_phdr, Max(FluxPageGetCommitTs(upd_phdr), current_ts)); + } + + MarkBufferDirty(buffer); + + /* WAL log the update with all overflow buffers atomically */ + if (RelationNeedsWAL(relation)) + { + /* + * old_tuple_for_inplace_wal was populated with a copy of the old + * tuple data BEFORE we modified the page. Use its saved + * old_commit_ts for the WAL record so the before-image is correct. + */ + FluxXLogUpdate(relation, buffer, offnum, + old_tuple_for_inplace_wal, new_tuple, + old_tuple_for_inplace_wal->t_data->t_commit_ts, + 0, /* new version: t_xmax = Invalid (heap-shaped) */ + &update_overflow_buffers, + InvalidBuffer, + &update_old_img, &update_new_img); + } + + END_CRIT_SECTION(); + + FluxXLogReleaseLogicalImage(&update_old_img); + FluxXLogReleaseLogicalImage(&update_new_img); + + /* + * Release all overflow buffers first ā€” they were WAL-logged atomically + * above so they're safe to unlock now. + */ + for (upd_i = 0; upd_i < update_overflow_buffers.count; upd_i++) + { + Buffer ovf_buf = update_overflow_buffers.buffers[upd_i].buffer; + bool already_released = (ovf_buf == buffer); + int dup_j; + + for (dup_j = 0; dup_j < upd_i && !already_released; dup_j++) + { + if (update_overflow_buffers.buffers[dup_j].buffer == ovf_buf) + already_released = true; + } + + if (!already_released) + UnlockReleaseBuffer(ovf_buf); + pfree(update_overflow_buffers.buffers[upd_i].record_data); + } + + /* + * Clear VM bits and capture free space while we still hold the main + * buffer lock. Both need page access. + */ + FluxVMUpdateForUpdate(relation, buffer); + { + Size update_free_space = PageGetFreeSpace(page); + + /* + * sLog registration BEFORE buffer release ā€” eliminates the race + * window where another backend reads the modified tuple but finds no + * sLog entry. Safe: seqlock reads take no lock, no deadlock. + */ + FluxEnsureSLogCallbacks(); + SLogTupleInsert(RelationGetRelid(relation), &slot->tts_tid, + GetTopTransactionId(), SLOG_OP_UPDATE, + GetCurrentSubTransactionId(), cid, current_ts, 0, + LockTupleNoKeyExclusive); + SLogTupleStoreBeforeImage(RelationGetRelid(relation), &slot->tts_tid, + GetTopTransactionId(), + (const char *) old_tuple_for_inplace_wal->t_data, + old_tuple_for_inplace_wal->t_len, + old_tuple_for_inplace_wal->t_data->t_flags, + old_tuple_for_inplace_wal->t_data->t_commit_ts, + relation->rd_locator, + relation->rd_rel->relpersistence); + + /* Release the main buffer lock ā€” sLog registered, no race. */ + UnlockReleaseBuffer(buffer); + + FluxRecordFreeSpace(relation, blkno, update_free_space); + } + + /* + * Finish the per-relation UNDO record now that the buffer lock is + * released. This works from old_tuple_for_inplace_wal, a palloc'd copy + * taken before the page modification, so it does NOT need the page. + * + * Moving this out of the buffer-lock-held window significantly reduces + * contention at high concurrency (8+ clients on hot pages). + * + * Write a full-tuple RELUNDO_UPDATE record with old/new TID mapping and + * the old tuple data so that rollback can restore it. + */ + if (RelUndoRecPtrIsValid(upd_undo_ptr)) + { + RelUndoRecordHeader upd_undo_hdr; + RelUndoUpdatePayload upd_undo_payload; + char *upd_combined; + Size upd_payload_total; + + upd_undo_payload.oldtid = *otid; + upd_undo_payload.newtid = slot->tts_tid; + + upd_undo_hdr.urec_xid = GetCurrentTransactionId(); + upd_undo_hdr.urec_prevundorec = + GetPerRelUndoPtr(RelationGetRelid(relation)); + + /* + * Full-tuple UPDATE UNDO record: store the old tuple so rollback can + * restore it (RelUndoApplyUpdate handles same-size, grow, and + * shrink). FLUX matches ZHeap: the whole old tuple to UNDO, no + * byte-diff/delta. Layout written by RelUndoFinish: + * [header][payload][old tuple]. + */ + upd_undo_hdr.urec_type = RELUNDO_UPDATE; + upd_undo_hdr.urec_len = (uint16) + (SizeOfRelUndoRecordHeader + sizeof(RelUndoUpdatePayload) + + old_tuple_for_inplace_wal->t_len); + upd_undo_hdr.info_flags = RELUNDO_INFO_HAS_TUPLE; + upd_undo_hdr.tuple_len = (uint16) old_tuple_for_inplace_wal->t_len; + + upd_payload_total = sizeof(RelUndoUpdatePayload) + + old_tuple_for_inplace_wal->t_len; + upd_combined = palloc(upd_payload_total); + memcpy(upd_combined, &upd_undo_payload, sizeof(RelUndoUpdatePayload)); + memcpy(upd_combined + sizeof(RelUndoUpdatePayload), + old_tuple_for_inplace_wal->t_data, + old_tuple_for_inplace_wal->t_len); + + RelUndoFinish(relation, upd_undo_buffer, upd_undo_ptr, + &upd_undo_hdr, upd_combined, upd_payload_total); + RegisterPerRelUndo(RelationGetRelid(relation), upd_undo_ptr); + pfree(upd_combined); + } + + /* + * No index maintenance for this path. Key-changing UPDATEs are routed + * out of place at the top of flux_tuple_update (delete + insert at a new + * TID, *update_indexes = TU_All); this in-place path runs only for + * non-indexed changes, so the TID is unchanged and every secondary (key, + * TID) entry still points at the live tuple. *update_indexes was + * defaulted to TU_None at the top of the function, so no index re-insert + * happens. + */ + { + /* + * sLog registration was done above (before buffer release). The + * FluxEnsureSLogCallbacks + SLogTupleInsert + + * SLogTupleStoreBeforeImage calls were moved to eliminate the + * visibility race window at high concurrency. + */ + + /* Free old_tuple copy now that before-image has been stored */ + FluxFreeTuple(old_tuple_for_inplace_wal); + + /* Mark this block dirty for the scan-path sLog bypass */ + FluxDirtyMapMark(RelationGetRelid(relation), blkno); + + /* + * NOTE: We do NOT immediately clean up overflow chains here. + * Immediate cleanup was: 1. Buggy (collected wrong overflow pointers + * after in-place modification) 2. Expensive on hot paths (extra + * buffer I/O + locking during UPDATE) 3. Complex to WAL-log correctly + * + * Instead, overflow cleanup is deferred to VACUUM. When VACUUM + * prunes deleted tuples, it will also reclaim orphaned overflow + * pages. + * + * Future enhancement: Log overflow block/offset in WAL UPDATE record + * so UNDO log pruning can also clean up overflow chains. + */ + (void) old_has_overflow; /* Suppress unused variable warning */ + } + + FluxFreeTuple(new_tuple); + + FLUX_RELEASE_TUPLOCK(); + + /* Buffer + tuple locks released; safe to drive throttled cleanup. */ + SLogTupleMaybeCleanupRetained(); + RelUndoMaybeVacuum(relation); + pgstat_count_heap_update(relation, false, false); + return TM_Ok; +} + +#undef FLUX_RELEASE_TUPLOCK + + + +/* + * Multi-insert operation for bulk loading (batched page-at-a-time) + * + * Pre-forms all tuples, then inserts them page-at-a-time to minimize + * per-tuple overhead: one FSM lookup, one buffer lock, one WAL record, + * and one UNDO reservation per page batch instead of per tuple. + * + * Tuples that are too large for batch handling (need overflow) are + * inserted individually via the single-insert path. + */ +void +flux_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples, + CommandId cid, uint32 options, BulkInsertState bistate) +{ + FluxTuple *formed_tuples; + bool *needs_single_insert; + uint64 current_ts; + TransactionId my_xid; + int i; + int ndone; + + if (ntuples <= 0) + return; + + /* + * Get timestamps and XID outside the loop ā€” these are per-transaction + * cached values, but calling them once avoids function call overhead. + */ + (void) FluxGetTransactionTimestamp(); + current_ts = (uint64) FluxGetDmlTimestamp(); + my_xid = GetCurrentTransactionId(); /* subxid: heap-shaped xmin */ + /* Ensure relation storage exists */ + RelationGetSmgr(relation); + + /* + * Phase 1: Pre-form all tuples without overflow handling. Passing NULL + * for rel and overflow_buffers skips the overflow path, keeping the tuple + * inline. Tuples that exceed the page size will be detected below and + * routed to single-insert. + */ + formed_tuples = (FluxTuple *) palloc(ntuples * sizeof(FluxTuple)); + needs_single_insert = (bool *) palloc0(ntuples * sizeof(bool)); + + for (i = 0; i < ntuples; i++) + { + slot_getallattrs(slots[i]); + formed_tuples[i] = FluxFormTuple(RelationGetDescr(relation), + slots[i]->tts_values, + slots[i]->tts_isnull, + NULL, /* no overflow in batch */ + NULL); + + /* Set MVCC fields (heap-shaped xmin/xmax) */ + formed_tuples[i]->t_data->t_commit_ts = 0; /* t_xmax = Invalid */ + formed_tuples[i]->t_data->t_flags |= FLUX_TUPLE_UNCOMMITTED; + formed_tuples[i]->t_data->t_xmin = my_xid; + + /* + * WS-PVS1: reserve the trailing version-pointer slot at INSERT time + * so the first UPDATE is a same-length overwrite (see + * flux_tuple_insert for the rationale). Done before the size check + * below so oversize routing accounts for the base+8 on-page + * footprint. + */ + formed_tuples[i]->t_data->t_flags |= FLUX_TUPLE_HAS_VERSION_PTR; + FluxTupleSetVersionPtr(formed_tuples[i]->t_data, + formed_tuples[i]->t_len, + InvalidRelUndoRecPtr); + + /* Mark tuples too large for batch insert */ + if (formed_tuples[i]->t_len > FLUX_MAX_TUPLE_SIZE) + needs_single_insert[i] = true; + } + + /* + * Phase 2: Batch insert page-at-a-time. + * + * For each page: lock once, insert all fitting tuples, WAL-log once, + * unlock. This is much faster than per-tuple buffer operations. + * + * We register each inserted tuple in the backend-local tracked-key list + * (via SLogTupleTrackLocalOnly) so that FluxClearUncommittedFlags() can + * find and stamp them at commit time. Without this, COPY-inserted tuples + * retain FLUX_TUPLE_UNCOMMITTED permanently and are invisible to all + * snapshots. + */ + FluxEnsureSLogCallbacks(); + ndone = 0; + while (ndone < ntuples) + { + Buffer buffer; + Page page; + BlockNumber target_block; + int batch_start; + int batch_count; + int nfit; + Size saveFreeSpace; + Size avail; + OffsetNumber *offnums = NULL; + FluxLogicalImage *logical_imgs = NULL; + FluxLogicalImage combined_image = {NULL, 0}; + bool need_logical = RelationIsLogicallyLogged(relation); + + /* Skip tuples that need single-insert (overflow) */ + if (needs_single_insert[ndone]) + { + flux_tuple_insert(relation, slots[ndone], cid, options, bistate); + FluxFreeTuple(formed_tuples[ndone]); + ndone++; + continue; + } + + /* Find a page with space for at least one tuple (fill-factor aware) */ + saveFreeSpace = RelationGetTargetPageFreeSpace(relation, + FLUX_DEFAULT_FILLFACTOR); + target_block = FluxGetPageWithFreeSpace(relation, + formed_tuples[ndone]->t_len + saveFreeSpace); + if (target_block == InvalidBlockNumber) + { + /* Fall back to single insert */ + flux_tuple_insert(relation, slots[ndone], cid, options, bistate); + FluxFreeTuple(formed_tuples[ndone]); + ndone++; + continue; + } + + buffer = ReadBuffer(relation, target_block); + LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); + page = BufferGetPage(buffer); + + /* + * Determine how many tuples will fit on this page BEFORE entering the + * critical section. Logical-decoding images and the offnums array + * must be allocated outside the crit section, so we need the count up + * front. This mirrors heap_multi_insert(): walk the pending tuples, + * subtracting each tuple's (line-pointer + aligned body) cost from + * the page's free space until the next one would not fit, honouring + * the fill-factor reserve and stopping at the first overflow tuple. + */ + avail = PageGetFreeSpace(page); + + /* + * Honour the fill-factor reserve while PACKING, not just while + * choosing the target block. FLUX has stable TIDs and updates rows + * in place, so a page packed 100%% full leaves no room for later + * in-place growth and a growing UPDATE (e.g. an accumulating numeric) + * aborts with "does not fit". Reserve saveFreeSpace bytes here so + * bulk load (COPY / HammerDB) leaves the same headroom the + * single-insert path does. The "always take at least one tuple" rule + * below still guarantees forward progress even when a single tuple + * exceeds the post-reserve budget. + */ + if (avail > saveFreeSpace) + avail -= saveFreeSpace; + nfit = 0; + while (ndone + nfit < ntuples && + !needs_single_insert[ndone + nfit]) + { + Size need = sizeof(ItemIdData) + + MAXALIGN(formed_tuples[ndone + nfit]->t_len); + + /* Always take at least one tuple; stop when the next won't fit */ + if (nfit > 0 && need > avail) + break; + avail -= need; + nfit++; + } + Assert(nfit > 0); + + offnums = (OffsetNumber *) palloc(nfit * sizeof(OffsetNumber)); + if (need_logical) + { + Size total = 0; + char *p; + + logical_imgs = (FluxLogicalImage *) + palloc(nfit * sizeof(FluxLogicalImage)); + for (i = 0; i < nfit; i++) + FluxXLogPrepareLogicalImage(relation, + formed_tuples[ndone + i], + &logical_imgs[i]); + + /* + * Serialize every image into one contiguous blob, framed exactly + * as FluxXLogRegisterLogicalImage emits a single image: each + * frame is "[heap bytes][uint32 len]" in tuple order. The + * multi-insert emitter registers this blob as a single rdata + * chunk, keeping the registered-data slot count constant + * regardless of batch size (the old per-tuple registration + * overflowed XLR_NORMAL_RDATAS on dense pages). We size for all + * nfit images but register only the prefix covering the tuples + * actually inserted (see below), matching the ntuples trailing + * frames the redo/decode readers expect. + */ + for (i = 0; i < nfit; i++) + if (logical_imgs[i].data != NULL) + total += logical_imgs[i].len + sizeof(uint32); + if (total > 0) + { + combined_image.data = (char *) palloc(total); + p = combined_image.data; + for (i = 0; i < nfit; i++) + { + if (logical_imgs[i].data == NULL) + continue; + memcpy(p, logical_imgs[i].data, logical_imgs[i].len); + p += logical_imgs[i].len; + memcpy(p, &logical_imgs[i].len, sizeof(uint32)); + p += sizeof(uint32); + } + } + } + + /* + * Ensure the current transaction has an XID assigned BEFORE entering + * the critical section (GetCurrentTransactionId may lock and + * allocate, both forbidden in a crit section). An assigned XID is + * required for correctness: a multi-insert WAL record without an + * attached xid cannot be decoded into a logical replication stream + * (ReorderBuffer groups changes by xid and asserts the xid is valid), + * and a WAL-emitting transaction would otherwise be treated as + * read-only. This mirrors flux_tuple_insert() and + * heap_multi_insert(). + */ + if (RelationNeedsWAL(relation)) + (void) GetCurrentTransactionId(); + + /* Pre-allocate WAL space outside critical section */ + if (RelationNeedsWAL(relation)) + XLogEnsureRecordSpace(XLR_MAX_BLOCK_ID, 4); + + START_CRIT_SECTION(); + + batch_start = ndone; + batch_count = 0; + + /* Insert the tuples we sized for above */ + while (batch_count < nfit) + { + OffsetNumber offnum; + ItemId inserted_itemid; + FluxTupleHeader *inserted_hdr; + + offnum = FluxPageAddTuple(page, formed_tuples[ndone], + formed_tuples[ndone]->t_len); + if (offnum == InvalidOffsetNumber) + break; /* page is full, stop batching */ + + /* Set TID in slot */ + ItemPointerSet(&slots[ndone]->tts_tid, target_block, offnum); + formed_tuples[ndone]->t_self = slots[ndone]->tts_tid; + slots[ndone]->tts_tableOid = RelationGetRelid(relation); + /* Set t_ctid to self */ + inserted_itemid = PageGetItemId(page, offnum); + inserted_hdr = (FluxTupleHeader *) PageGetItem(page, inserted_itemid); + ItemPointerSet(&inserted_hdr->t_ctid, target_block, offnum); + + offnums[batch_count] = offnum; + batch_count++; + ndone++; + } + + if (batch_count > 0) + { + /* Update page opaque fields */ + FluxPageOpaque phdr = FluxPageGetOpaque(page); + + FluxPageSetCommitTs(phdr, Max(FluxPageGetCommitTs(phdr), current_ts)); + + MarkBufferDirty(buffer); + + /* + * WAL-log the whole batch in one record. Every tuple body is + * logged individually (no forced full-page image), so recovery is + * crash-safe even with full_page_writes=off, and one logical- + * decoding image per tuple is appended for logically-logged + * relations so COPY'd rows replicate. + */ + if (RelationNeedsWAL(relation)) + { + XLogRecPtr recptr; + FluxLogicalImage *emit_image = NULL; + + /* + * Register only the framed-image prefix covering the tuples + * we actually inserted. batch_count can be < nfit if the + * page filled mid-loop, and the redo/decode readers walk + * exactly ntuples (== batch_count) trailing frames, so a + * longer blob would misalign their backward walk. + */ + if (combined_image.data != NULL) + { + Size prefix = 0; + + for (i = 0; i < batch_count; i++) + if (logical_imgs[i].data != NULL) + prefix += logical_imgs[i].len + sizeof(uint32); + combined_image.len = (uint32) prefix; + if (prefix > 0) + emit_image = &combined_image; + } + + recptr = FluxXLogMultiInsert(relation, buffer, + offnums, + &formed_tuples[batch_start], + batch_count, current_ts, + emit_image); + PageSetLSN(page, recptr); + } + } + + END_CRIT_SECTION(); + + if (logical_imgs != NULL) + { + for (i = 0; i < nfit; i++) + FluxXLogReleaseLogicalImage(&logical_imgs[i]); + pfree(logical_imgs); + } + if (combined_image.data != NULL) + pfree(combined_image.data); + pfree(offnums); + + /* + * Register each inserted tuple in the backend-local tracked-key list. + * This is lightweight (no shared hash entry, no LWLock) ā€” just a + * palloc'd linked-list node per tuple. FluxClearUncommittedFlags() + * iterates this list at PRE_COMMIT to clear the UNCOMMITTED flag, + * making the tuples visible. + * + * We skip full sLog registration (shared hash) because COPY doesn't + * use SNAPSHOT_DIRTY or ON CONFLICT, and the local tracking is + * sufficient for commit-time processing. + */ + { + TransactionId xid = GetTopTransactionId(); + TransactionId subxid = GetCurrentSubTransactionId(); + + for (i = batch_start; i < batch_start + batch_count; i++) + { + SLogTupleTrackLocalOnly(RelationGetRelid(relation), + &slots[i]->tts_tid, + xid, subxid); + } + } + + /* Update FSM with remaining free space */ + FluxRecordFreeSpace(relation, target_block, PageGetFreeSpace(page)); + + /* Update visibility map */ + FluxVMUpdateForInsert(relation, formed_tuples[batch_start]->t_data, + buffer); + + UnlockReleaseBuffer(buffer); + + /* + * Count the batched inserts here rather than once for all ntuples at + * the end: tuples routed to the single-insert fallback are already + * counted by their own flux_tuple_insert call, so counting the whole + * ntuples at the tail would double-count them. + */ + pgstat_count_heap_insert(relation, batch_count); + + /* Free formed tuples in this batch */ + for (i = batch_start; i < batch_start + batch_count; i++) + FluxFreeTuple(formed_tuples[i]); + } + + pfree(formed_tuples); + pfree(needs_single_insert); +} + +/* + * FluxVacuumCrossPageDefrag - move live tuples from tail pages to front pages + * + * After single-page defragmentation (Phase III), pages near the end of the + * relation may still contain live tuples, preventing truncation. This + * function moves those tuples to pages near the front that have sufficient + * free space, thereby emptying tail pages so that Phase V can truncate them. + * + * Algorithm: + * 1. Scan backwards from the end of the relation to find source pages + * that have live tuples and could be emptied. + * 2. For each source page, find target pages near the front with enough + * free space (via the FLUX free space map). + * 3. Move each live tuple: copy to target page, insert new index entries + * pointing to the new TID, then mark the old line pointer unused. + * 4. WAL-log all modifications for crash safety. + * + * Locking protocol: + * We acquire an exclusive lock on the source page (higher block number). + * For each tuple move, we lock the target page exclusively while holding + * the source lock. Since we always hold the higher-numbered page first, + * this maintains a consistent lock ordering and avoids deadlocks. VACUUM + * also holds a heavyweight lock on the relation. + * + * We skip: + * - Pages with overflow records (complex linked structure) + * - Pages with tuples that have update chains (t_ctid != self) + * - Pages with deleted-but-not-yet-vacuumed tuples + * + * Returns the number of tuples moved. + */ +static int +FluxVacuumCrossPageDefrag(Relation rel, BlockNumber nblocks, + BlockNumber *empty_end_pages_p, + int nindexes, Relation *indrels, + BufferAccessStrategy bstrategy) +{ + BlockNumber src_blkno; + BlockNumber nonempty_limit; + int tuples_moved = 0; + int pages_emptied = 0; + EState *estate = NULL; + TupleTableSlot *slot = NULL; + IndexInfo **indexInfoArray = NULL; + + /* + * A deferred index-maintenance record: an off-page copy of a moved tuple + * plus its new TID. We must NOT call index_insert() while holding any + * table buffer content lock -- concurrent index bottom-up deletion locks + * the index leaf first and then the table page + * (flux_index_delete_tuples), so inserting into an index under the + * source/target page lock would form an AB-BA deadlock on buffer content + * locks, which the deadlock detector cannot see. Instead we accumulate + * the moves for a source page and replay them after releasing that page's + * lock. + */ + typedef struct FluxDefragMove + { + char *tupcopy; + Size tuplen; + ItemPointerData new_tid; + } FluxDefragMove; + + /* + * Cross-page defrag relocates a live tuple to a lower-numbered page, + * which changes its TID. FLUX's whole design guarantees stable TIDs + * (UPDATE is performed in place, keeping the same TID); its index + * integration and the UPDATE path (which ereport()s rather than move a + * tuple that no longer fits -- see "FLUX tuples have stable TIDs and + * cannot move to another page") both depend on that invariant. We can + * insert a new index entry for the new TID, but there is no cheap + * primitive to remove the OLD entry for the pre-move TID (btree reclaims + * dead entries only via ambulkdelete, not point-deletes). Leaving the + * stale entry behind and then truncating the vacated tail page orphans + * it: an index scan hitting that entry fails with "could not read blocks + * N..N". So when the relation has indexes we must not move tuples at all + * -- matching plain heap lazy VACUUM, which never relocates tuples + * precisely because it cannot cheaply update index pointers. The tail + * pages simply are not truncated this pass; a later VACUUM (or a REINDEX) + * can reclaim them. + * + * ponytail: skip-the-move when indexes exist. The optimal fix (relocate + * and keep every index consistent) needs a moved-from-TID bulk-delete + * pass per index; add that if tail reclamation on indexed FLUX tables + * ever measurably matters. + */ + if (nindexes > 0) + return 0; + + /* + * Compute the boundary: we only try to empty pages from position (nblocks + * - empty_end_pages - 1) backwards toward the "used" portion. If there + * are already empty_end_pages trailing, the first candidate source page + * is right before that run. + */ + if (*empty_end_pages_p >= nblocks) + return 0; + + nonempty_limit = nblocks - *empty_end_pages_p; + + /* Need at least a few pages to make defrag worthwhile */ + if (nonempty_limit <= 1) + return 0; + + /* + * Cross-page defragmentation is an internal heuristic whose page-by-page + * progress depends on FSM search order and tuple packing, both of which + * vary across platforms and BLCKSZ. Report it at DEBUG2 rather than INFO + * so that VACUUM VERBOSE output stays stable; the per-relation summary + * below (emitted at INFO) carries the user-meaningful result. + */ + ereport(DEBUG2, + (errmsg_internal("table \"%s\": starting cross-page defragmentation from block %u", + RelationGetRelationName(rel), + nonempty_limit - 1))); + + /* + * Create a single executor state and tuple slot, reused across all index + * insertions to avoid repeated allocation. Also pre-build IndexInfo for + * each index to avoid expensive catalog lookups inside the per-tuple + * loop. + */ + if (nindexes > 0) + { + estate = CreateExecutorState(); + slot = MakeSingleTupleTableSlot(RelationGetDescr(rel), + table_slot_callbacks(rel)); + GetPerTupleExprContext(estate)->ecxt_scantuple = slot; + + indexInfoArray = (IndexInfo **) palloc(nindexes * sizeof(IndexInfo *)); + for (int i = 0; i < nindexes; i++) + indexInfoArray[i] = BuildIndexInfo(indrels[i]); + } + + /* + * Scan backwards from the last candidate source page. Favor moving + * tuples from the highest-numbered pages first, as this maximizes the + * contiguous run of empty tail pages available for truncation. + */ + for (src_blkno = nonempty_limit - 1; + src_blkno != InvalidBlockNumber && src_blkno > 0; + src_blkno--) + { + Buffer src_buf; + Page src_page; + OffsetNumber maxoff; + OffsetNumber offnum; + bool page_emptied = true; + bool skip_page = false; + int ntuples_on_page = 0; + FluxDefragMove *deferred_moves = NULL; + int ndeferred = 0; + + CHECK_FOR_INTERRUPTS(); + + /* Read and exclusively lock the source page */ + src_buf = ReadBufferExtended(rel, MAIN_FORKNUM, src_blkno, + RBM_NORMAL, bstrategy); + LockBuffer(src_buf, BUFFER_LOCK_EXCLUSIVE); + src_page = BufferGetPage(src_buf); + + /* Skip new or empty pages */ + if (PageIsNew(src_page) || PageIsEmpty(src_page)) + { + UnlockReleaseBuffer(src_buf); + continue; + } + + /* + * First pass: check the page for suitability. + * + * We skip pages that have overflow records, deleted tuples that + * haven't been vacuumed yet, or tuples with update chains. + */ + maxoff = PageGetMaxOffsetNumber(src_page); + for (offnum = FirstOffsetNumber; offnum <= maxoff; + offnum = OffsetNumberNext(offnum)) + { + ItemId itemid = PageGetItemId(src_page, offnum); + FluxTupleHeader *tuple_hdr; + ItemPointerData self_tid; + + if (!ItemIdIsNormal(itemid)) + { + if (ItemIdIsDead(itemid)) + { + /* Dead items not yet cleaned -- skip page */ + skip_page = true; + break; + } + continue; /* LP_UNUSED slots are fine */ + } + + tuple_hdr = (FluxTupleHeader *) PageGetItem(src_page, itemid); + + /* Skip pages with overflow records -- too complex to relocate */ + if (FluxIsOverflowRecordInline(tuple_hdr, ItemIdGetLength(itemid))) + { + skip_page = true; + break; + } + + /* Skip pages with tuples that have overflow pointers */ + if (tuple_hdr->t_flags & FLUX_TUPLE_HAS_OVERFLOW) + { + skip_page = true; + break; + } + + /* Skip pages with deleted-but-not-yet-removed tuples */ + if (tuple_hdr->t_flags & FLUX_TUPLE_DELETED) + { + skip_page = true; + break; + } + + /* Skip pages with update chains: ctid must point to self */ + ItemPointerSet(&self_tid, src_blkno, offnum); + if (!ItemPointerIsValid(&tuple_hdr->t_ctid) || + !ItemPointerEquals(&tuple_hdr->t_ctid, &self_tid)) + { + skip_page = true; + break; + } + + ntuples_on_page++; + } + + if (skip_page || ntuples_on_page == 0) + { + UnlockReleaseBuffer(src_buf); + continue; + } + + /* + * Deferred index-maintenance list for this source page. Index + * inserts are replayed only after src_buf is unlocked (see + * FluxDefragMove). + */ + if (nindexes > 0) + { + deferred_moves = (FluxDefragMove *) + palloc(ntuples_on_page * sizeof(FluxDefragMove)); + ndeferred = 0; + } + + /* + * Second pass: move each live tuple to a target page near the front + * of the relation. + */ + for (offnum = FirstOffsetNumber; offnum <= maxoff; + offnum = OffsetNumberNext(offnum)) + { + ItemId src_itemid; + FluxTupleHeader *src_hdr; + Size tuple_len; + Buffer dst_buf; + Page dst_page; + BlockNumber dst_blkno; + OffsetNumber dst_offnum; + ItemPointerData new_tid; + + src_itemid = PageGetItemId(src_page, offnum); + if (!ItemIdIsNormal(src_itemid)) + continue; + + src_hdr = (FluxTupleHeader *) PageGetItem(src_page, src_itemid); + tuple_len = ItemIdGetLength(src_itemid); + + /* + * Find a target page with enough free space. Use the FSM + * directly (GetPageWithFreeSpace) instead of our wrapper + * FluxGetPageWithFreeSpace, because the wrapper reads and locks + * pages to verify free space, which would deadlock against the + * exclusive lock we already hold on the source page. We verify + * the actual free space below, after locking the target page. The + * target must be before the source block to be useful for + * truncation. + */ + dst_blkno = GetPageWithFreeSpace(rel, + tuple_len + sizeof(ItemIdData)); + if (dst_blkno == InvalidBlockNumber || dst_blkno >= src_blkno) + { + page_emptied = false; + continue; + } + + /* + * Lock the target page. Lock ordering is safe: we hold the + * higher-numbered source page lock already. + */ + dst_buf = ReadBufferExtended(rel, MAIN_FORKNUM, dst_blkno, + RBM_NORMAL, bstrategy); + LockBuffer(dst_buf, BUFFER_LOCK_EXCLUSIVE); + dst_page = BufferGetPage(dst_buf); + + /* Recheck free space -- FSM might be stale */ + if (PageGetFreeSpace(dst_page) < tuple_len + sizeof(ItemIdData)) + { + Size actual_free = PageGetFreeSpace(dst_page); + + UnlockReleaseBuffer(dst_buf); + + /* Update FSM with accurate info */ + FluxRecordFreeSpace(rel, dst_blkno, actual_free); + page_emptied = false; + continue; + } + + /* + * Perform the move in a critical section. Both pages are + * modified atomically from WAL's perspective. + */ + START_CRIT_SECTION(); + + /* Insert the tuple data into the target page */ + dst_offnum = PageAddItem(dst_page, src_hdr, tuple_len, + InvalidOffsetNumber, false, false); + + if (dst_offnum == InvalidOffsetNumber) + { + END_CRIT_SECTION(); + UnlockReleaseBuffer(dst_buf); + page_emptied = false; + continue; + } + + /* Update ctid in the new copy to point to itself */ + { + ItemId dst_itemid; + FluxTupleHeader *dst_hdr; + + dst_itemid = PageGetItemId(dst_page, dst_offnum); + dst_hdr = (FluxTupleHeader *) PageGetItem(dst_page, dst_itemid); + ItemPointerSet(&dst_hdr->t_ctid, dst_blkno, dst_offnum); + } + + ItemPointerSet(&new_tid, dst_blkno, dst_offnum); + + /* Mark the source line pointer as unused */ + ItemIdSetUnused(src_itemid); + + /* Mark both buffers dirty */ + MarkBufferDirty(dst_buf); + MarkBufferDirty(src_buf); + + /* WAL-log the cross-page move */ + if (RelationNeedsWAL(rel)) + { + XLogRecPtr recptr; + + /* + * Use the dedicated cross-page defrag record type. This logs + * both pages (with FPIs when needed) plus the tuple data for + * non-FPI replay. Block 0 = target, block 1 = source. + */ + recptr = FluxXLogCrossPageDefrag(rel, + dst_buf, dst_offnum, + src_buf, offnum, + src_hdr, (uint32) tuple_len); + PageSetLSN(dst_page, recptr); + PageSetLSN(src_page, recptr); + } + + END_CRIT_SECTION(); + + /* Update FSM for the target page */ + FluxRecordFreeSpace(rel, dst_blkno, + PageGetFreeSpace(dst_page)); + + /* + * If the relation has indexes, copy the moved tuple off-page now, + * while we still hold dst_buf, and defer the index insertion + * until after every table page lock for this source page is + * released. Calling index_insert() under a table content lock + * would invert the lock order used by concurrent index bottom-up + * deletion (flux_index_delete_tuples locks the index leaf, then + * the table page) and deadlock on buffer content locks. + */ + if (nindexes > 0) + { + ItemId moved_itemid = PageGetItemId(dst_page, dst_offnum); + FluxTupleHeader *moved_hdr = + (FluxTupleHeader *) PageGetItem(dst_page, moved_itemid); + FluxDefragMove *mv = &deferred_moves[ndeferred++]; + + mv->tuplen = ItemIdGetLength(moved_itemid); + mv->tupcopy = (char *) palloc(mv->tuplen); + memcpy(mv->tupcopy, moved_hdr, mv->tuplen); + mv->new_tid = new_tid; + } + + UnlockReleaseBuffer(dst_buf); + + tuples_moved++; + } + + /* Update FSM for the source page */ + FluxRecordFreeSpace(rel, src_blkno, + PageGetFreeSpace(src_page)); + + UnlockReleaseBuffer(src_buf); + + /* + * Now that no table buffer content lock is held, replay the deferred + * index insertions for the tuples moved off this source page. Working + * from off-page copies is safe: pages carrying overflow pointers were + * skipped in the first pass, so every copied tuple is self-contained. + */ + if (deferred_moves != NULL) + { + for (int m = 0; m < ndeferred; m++) + { + FluxDefragMove *mv = &deferred_moves[m]; + Datum values[INDEX_MAX_KEYS]; + bool isnull[INDEX_MAX_KEYS]; + + ExecClearTuple(slot); + FluxTupleToSlot((FluxTupleHeader *) mv->tupcopy, slot); + slot->tts_tid = mv->new_tid; + for (int i = 0; i < nindexes; i++) + { + FormIndexDatum(indexInfoArray[i], slot, estate, + values, isnull); + + /* + * Skip uniqueness check since we're relocating an + * existing tuple. + */ + index_insert(indrels[i], values, isnull, &mv->new_tid, + rel, UNIQUE_CHECK_NO, false, + indexInfoArray[i]); + + ResetPerTupleExprContext(estate); + } + + pfree(mv->tupcopy); + } + + pfree(deferred_moves); + deferred_moves = NULL; + ndeferred = 0; + } + + if (page_emptied) + { + pages_emptied++; + + /* + * Extend the trailing empty page count if this page is contiguous + * with the existing run. + */ + if (src_blkno == nblocks - *empty_end_pages_p - 1) + (*empty_end_pages_p)++; + } + else + { + /* + * Once we fail to empty a page, stop: pages below this one won't + * contribute to a contiguous run of trailing empties. + */ + break; + } + } + + /* Clean up executor state and pre-built index info */ + if (indexInfoArray != NULL) + { + for (int i = 0; i < nindexes; i++) + pfree(indexInfoArray[i]); + pfree(indexInfoArray); + } + if (slot != NULL) + ExecDropSingleTupleTableSlot(slot); + if (estate != NULL) + FreeExecutorState(estate); + + if (tuples_moved > 0) + ereport(DEBUG2, + (errmsg_internal("table \"%s\": cross-page defrag moved %d tuples, emptied %d pages", + RelationGetRelationName(rel), + tuples_moved, pages_emptied))); + + return tuples_moved; +} + +/* + * Per-stream state for the Phase I VACUUM scan callback. + */ +typedef struct FluxVacScanState +{ + Relation rel; + BlockNumber current_block; /* last block handed out (InvalidBlockNumber + * before the first call) */ + BlockNumber nblocks; + bool skip_all_frozen; /* honor the VM ALL_FROZEN skip? */ +} FluxVacScanState; + +/* + * Read stream callback for VACUUM Phase I. + * + * Hands the read stream the next block that actually needs scanning so the + * upcoming pages are prefetched under AIO, matching heapam's + * heap_vac_scan_next_block(). Pages marked ALL_FROZEN in the visibility map + * are skipped here (unless DISABLE_PAGE_SKIPPING was requested), so the + * stream never issues I/O for pages VACUUM would immediately discard. + */ +static BlockNumber +flux_vac_scan_next_block(ReadStream *stream, + void *callback_private_data, + void *per_buffer_data) +{ + FluxVacScanState *state = callback_private_data; + BlockNumber next_block = state->current_block + 1; + + for (; next_block < state->nblocks; next_block++) + { + CHECK_FOR_INTERRUPTS(); + + if (state->skip_all_frozen && + FluxVMCheck(state->rel, next_block, FLUX_VM_ALL_FROZEN)) + continue; + + state->current_block = next_block; + return next_block; + } + + state->current_block = state->nblocks; + return InvalidBlockNumber; +} + +/* Truncation lock-acquisition tuning, mirroring heap's lazy_truncate_heap. */ +#define FLUX_TRUNCATE_LOCK_WAIT_INTERVAL 50 /* ms */ +#define FLUX_TRUNCATE_LOCK_TIMEOUT 5000 /* ms */ + +/* + * FluxPageIsEmpty + * + * A FLUX page is truncatable-empty when it holds no line pointer in normal + * state. This deliberately treats a page whose line-pointer array is present + * but consists entirely of LP_UNUSED (and/or LP_DEAD) slots as empty, matching + * the emptiness definition used by cross-page defragmentation (page_emptied): + * defrag relocates every NORMAL tuple forward and leaves the source ItemIds + * LP_UNUSED without resetting pd_lower, so PageGetMaxOffsetNumber() still + * reports a nonzero count. Testing max-offset alone would wrongly classify + * such a defragmented tail page as non-empty and decline to truncate it, + * leaking the trailing pages defrag just emptied. + */ +static bool +FluxPageIsEmpty(Page page) +{ + OffsetNumber maxoff; + + if (PageIsNew(page)) + return true; + + maxoff = PageGetMaxOffsetNumber(page); + for (OffsetNumber offnum = FirstOffsetNumber; offnum <= maxoff; + offnum = OffsetNumberNext(offnum)) + { + ItemId itemid = PageGetItemId(page, offnum); + + if (ItemIdIsNormal(itemid)) + return false; + } + + return true; +} + +/* + * FluxCountNondeletablePages + * + * Scan backwards from the end of the relation and return the block number of + * the first (lowest-numbered) trailing page that is NOT empty, i.e. the number + * of blocks the relation should be truncated to. A page counts as empty when + * it has no line pointers in normal state. + * + * The caller MUST hold AccessExclusiveLock on the relation so that no other + * backend can add a tuple to a trailing page between this recount and the + * subsequent RelationTruncate(). This is the FLUX analogue of heap's + * count_nondeletable_pages() and is *necessary*, not optional: the forward + * scan that computed empty_end_pages ran under ShareUpdateExclusiveLock, so a + * concurrent INSERT could have populated a page that was empty at scan time. + */ +static BlockNumber +FluxCountNondeletablePages(Relation onerel) +{ + BlockNumber blkno = RelationGetNumberOfBlocks(onerel); + + while (blkno > 0) + { + Buffer buf; + Page page; + bool empty; + + CHECK_FOR_INTERRUPTS(); + + buf = ReadBuffer(onerel, blkno - 1); + LockBuffer(buf, BUFFER_LOCK_SHARE); + page = BufferGetPage(buf); + empty = FluxPageIsEmpty(page); + UnlockReleaseBuffer(buf); + + if (!empty) + break; + + blkno--; + } + + return blkno; +} + +/* + * FluxTruncateRelation + * + * Truncate trailing empty pages off a FLUX relation, mirroring heap's + * lazy_truncate_heap() locking protocol. We acquire AccessExclusiveLock + * conditionally (giving up rather than blocking or deadlocking against the + * lower-grade ShareUpdateExclusiveLock we already hold), re-verify under that + * lock that the trailing pages are still empty, then truncate and release. + * + * Doing the truncation under only ShareUpdateExclusiveLock (as the previous + * code did) races concurrent INSERT/extension: a backend could compute a + * target block, then find that block removed by our smgrtruncate, and + * ReadBuffer() it past EOF ("unexpected data beyond EOF in block N"), or worse + * silently lose a tuple written to a page we truncate away. + */ +static void +FluxTruncateRelation(Relation onerel, BlockNumber orig_nblocks, + BlockNumber desired_nblocks, bool verbose) +{ + BlockNumber new_nblocks; + int lock_retry = 0; + + /* + * Acquire AccessExclusiveLock, retrying with a bounded timeout. If a + * conflicting lock request arrives we give up truncating rather than + * block other backends or deadlock. + */ + while (!ConditionalLockRelation(onerel, AccessExclusiveLock)) + { + CHECK_FOR_INTERRUPTS(); + + if (++lock_retry > (FLUX_TRUNCATE_LOCK_TIMEOUT / + FLUX_TRUNCATE_LOCK_WAIT_INTERVAL)) + { + ereport(verbose ? INFO : DEBUG2, + (errmsg("\"%s\": stopping truncate due to conflicting lock request", + RelationGetRelationName(onerel)))); + return; + } + + (void) WaitLatch(MyLatch, + WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, + FLUX_TRUNCATE_LOCK_WAIT_INTERVAL, + WAIT_EVENT_VACUUM_TRUNCATE); + ResetLatch(MyLatch); + } + + /* + * Under the exclusive lock, re-check that the relation hasn't grown since + * the vacuum scan. If it has, the new pages presumably hold live tuples; + * give up. + */ + if (RelationGetNumberOfBlocks(onerel) != orig_nblocks) + { + UnlockRelation(onerel, AccessExclusiveLock); + return; + } + + /* + * Rescan the tail under the exclusive lock to confirm the pages we intend + * to drop are still empty. A concurrent INSERT under the lower-grade + * lock could have repopulated them after the vacuum scan. + */ + new_nblocks = FluxCountNondeletablePages(onerel); + if (new_nblocks < desired_nblocks) + new_nblocks = desired_nblocks; + + if (new_nblocks >= orig_nblocks) + { + /* Nothing to truncate after re-verification. */ + UnlockRelation(onerel, AccessExclusiveLock); + return; + } + + RelationTruncate(onerel, new_nblocks); + + /* + * Release the exclusive lock as soon as the truncation is done. Other + * backends process the smgr invalidation smgrtruncate sent out when they + * next acquire a lock on the relation. + */ + UnlockRelation(onerel, AccessExclusiveLock); + + if (verbose) + ereport(INFO, (errmsg("truncated \"%s\" from %u to %u pages", + RelationGetRelationName(onerel), + orig_nblocks, new_nblocks))); +} + +/* + * Vacuum a FLUX relation + * + * This performs garbage collection on a FLUX table in multiple phases: + * + * Phase I: Scan all pages, identify dead tuples, collect their TIDs + * Phase II: Remove dead index entries using the collected TIDs + * Phase III: Defragment data pages to reclaim space (must happen AFTER + * index cleanup to avoid dangling index pointers) + * Phase IV: Post-vacuum index cleanup (amvacuumcleanup) + * Phase IV-B: Cross-page defragmentation (move tail tuples to front pages) + * Phase V: Truncate trailing empty pages, update FSM + */ +void +flux_relation_vacuum(Relation onerel, const VacuumParams *params, + BufferAccessStrategy bstrategy) +{ + BlockNumber nblocks; + BlockNumber blkno; + Buffer buf; + Page page; + uint64 oldest_ts; + TransactionId oldest_xmin_vac; + int64 num_tuples = 0; + int64 dead_tuples = 0; + int64 live_tuples = 0; + int64 pages_vacuumed = 0; + BlockNumber empty_end_pages = 0; + bool verbose = (params->options & VACOPT_VERBOSE) != 0; + + /* Index cleanup state */ + Relation *indrels = NULL; + int nindexes = 0; + IndexBulkDeleteResult **indstats = NULL; + TidStore *dead_items = NULL; + VacDeadItemsInfo *dead_items_info = NULL; + bool do_index_cleanup; + + /* + * Initialize FLUX transaction state so that VACUUM's own start timestamp + * is registered in xact_start_ts_slots. Without this, + * FluxGetOldestActiveTimestamp() would see no active transactions and + * fall back to the current wall clock (GetCurrentTimestamp()), which for + * the VM all-visible hint would over-eagerly mark tuples all-visible. + * (The reclamation gate itself is the XID horizon below, not this ts.) + */ + (void) FluxGetTransactionTimestamp(); + + /* + * Get the oldest active transaction's start timestamp. Deleted tuples + * whose commit timestamp is older than this are no longer visible to any + * running transaction and can safely be removed. + * + * Previously this called FluxGetCommitTimestamp() which returns (and + * advances) the current wall-clock time. That was wrong: it made VACUUM + * consider almost all deleted tuples as reclaimable, even those still + * needed by long-running concurrent transactions. + */ + oldest_ts = FluxGetOldestActiveTimestamp(); + oldest_xmin_vac = FluxGetOldestXminHorizon(onerel); + nblocks = RelationGetNumberOfBlocks(onerel); + + if (verbose) + ereport(INFO, (errmsg("vacuuming \"%s\": scanning %u pages", + RelationGetRelationName(onerel), nblocks))); + + /* + * Open all indexes on the relation. We need RowExclusiveLock to prevent + * concurrent index modifications during vacuum. + */ + vac_open_indexes(onerel, RowExclusiveLock, &nindexes, &indrels); + do_index_cleanup = (nindexes > 0); + + /* + * Allocate TidStore for collecting dead tuple TIDs, and per-index stats + * array. We use maintenance_work_mem as the budget for the TidStore. + */ + if (do_index_cleanup) + { + dead_items_info = (VacDeadItemsInfo *) palloc0(sizeof(VacDeadItemsInfo)); + dead_items_info->max_bytes = (size_t) maintenance_work_mem * 1024; + dead_items_info->num_items = 0; + + dead_items = TidStoreCreateLocal(dead_items_info->max_bytes, true); + + indstats = (IndexBulkDeleteResult **) + palloc0(nindexes * sizeof(IndexBulkDeleteResult *)); + } + + /* + * ----------------------------------------------------------------------- + * Phase I: Scan all pages, identify dead tuples, collect TIDs + * + * We scan every page and classify each tuple as live, dead (removable), + * or recently dead (not yet removable). Dead tuple TIDs are recorded in + * the TidStore for later index cleanup. We do NOT defragment pages yet + * -- that must wait until after index entries pointing to dead tuples + * have been removed (Phase II), to avoid dangling index pointers. + * ----------------------------------------------------------------------- + */ + { + ReadStream *scan_stream; + FluxVacScanState scan_state; + + scan_state.rel = onerel; + scan_state.current_block = InvalidBlockNumber; + scan_state.nblocks = nblocks; + scan_state.skip_all_frozen = + !(params->options & VACOPT_DISABLE_PAGE_SKIPPING); + + scan_stream = read_stream_begin_relation(READ_STREAM_MAINTENANCE | + READ_STREAM_USE_BATCHING, + bstrategy, + onerel, + MAIN_FORKNUM, + flux_vac_scan_next_block, + &scan_state, + 0); + + while ((buf = read_stream_next_buffer(scan_stream, NULL)) != InvalidBuffer) + { + OffsetNumber offnum, + maxoffnum; + ItemId itemid; + OffsetNumber dead_offsets[MaxOffsetNumber]; + int ndead_on_page = 0; + + CHECK_FOR_INTERRUPTS(); + + blkno = BufferGetBlockNumber(buf); + LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE); + page = BufferGetPage(buf); + + /* Skip if page is new/uninitialized */ + if (PageIsNew(page)) + { + UnlockReleaseBuffer(buf); + continue; + } + + maxoffnum = PageGetMaxOffsetNumber(page); + + /* Scan all tuples on the page */ + for (offnum = FirstOffsetNumber; offnum <= maxoffnum; offnum++) + { + FluxTupleHeader *tuple_hdr; + + itemid = PageGetItemId(page, offnum); + + /* + * LP_DEAD line pointers were produced by opportunistic + * pruning (FluxPagePruneOpt / UPDATE defrag-fit), which + * reclaims a committed-deleted tuple's storage but reserves + * its TID so it is not recycled while index entries still + * reference it. VACUUM must record these TIDs for index + * cleanup (Phase II) before Phase III converts them to + * LP_UNUSED. Their storage is already gone, so there is no + * tuple header to inspect. + */ + if (ItemIdIsDead(itemid)) + { + if (do_index_cleanup) + { + dead_offsets[ndead_on_page++] = offnum; + dead_tuples++; + } + continue; + } + + /* Skip if not a normal tuple */ + if (!ItemIdIsNormal(itemid)) + continue; + + tuple_hdr = (FluxTupleHeader *) PageGetItem(page, itemid); + + /* Skip overflow records - they are managed by tuple lifecycle */ + if (FluxIsOverflowRecordInline(tuple_hdr, ItemIdGetLength(itemid))) + continue; + + num_tuples++; + + /* + * Check if tuple is deleted and old enough to be removed. + * + * With the sLog-based MVCC model, a deleted tuple can be + * vacuumed if: - FLUX_TUPLE_DELETED is set - + * FLUX_TUPLE_UNCOMMITTED is NOT set (committed delete) - + * commit_ts is older than the oldest active snapshot + * + * If UNCOMMITTED is still set, the deleting transaction is + * still in progress (or aborted but not yet cleaned up by the + * sLog callback). Skip it. + */ + if (tuple_hdr->t_flags & FLUX_TUPLE_DELETED) + { + if (tuple_hdr->t_flags & FLUX_TUPLE_UNCOMMITTED) + { + /* Transaction still in progress -- skip */ + live_tuples++; + } + else if (FluxTupleDeadToAll(tuple_hdr, oldest_xmin_vac)) + { + /* Tuple is dead and can be removed */ + dead_offsets[ndead_on_page++] = offnum; + dead_tuples++; + } + else + { + /* Recently dead -- not yet reclaimable but still dead */ + dead_tuples++; + } + } + else + { + /* Tuple is live */ + live_tuples++; + } + } + + /* + * Record dead tuple TIDs for this page in the TidStore. This is + * needed for index cleanup in Phase II. + */ + if (ndead_on_page > 0 && do_index_cleanup) + { + TidStoreSetBlockOffsets(dead_items, blkno, + dead_offsets, ndead_on_page); + dead_items_info->num_items += ndead_on_page; + } + + UnlockReleaseBuffer(buf); + } + + read_stream_end(scan_stream); + } + + /* + * ----------------------------------------------------------------------- + * Phase II: Index vacuum -- remove dead index entries + * + * For each index on the relation, call the index AM's bulk delete routine + * to remove entries pointing to dead tuples. This MUST happen before we + * defragment data pages (Phase III) to ensure no index entry points to a + * TID that has been recycled. + * ----------------------------------------------------------------------- + */ + if (do_index_cleanup && dead_items_info->num_items > 0) + { + int idx; + + if (verbose) + ereport(INFO, + (errmsg("vacuuming \"%s\": removing %lld dead index entries across %d indexes", + RelationGetRelationName(onerel), + (long long) dead_items_info->num_items, + nindexes))); + + for (idx = 0; idx < nindexes; idx++) + { + IndexVacuumInfo ivinfo; + + ivinfo.index = indrels[idx]; + ivinfo.heaprel = onerel; + ivinfo.analyze_only = false; + ivinfo.report_progress = false; + ivinfo.estimated_count = true; + ivinfo.message_level = verbose ? INFO : DEBUG2; + ivinfo.num_heap_tuples = (double) live_tuples; + ivinfo.strategy = bstrategy; + + indstats[idx] = vac_bulkdel_one_index(&ivinfo, indstats[idx], + dead_items, + dead_items_info); + } + } + + /* + * ----------------------------------------------------------------------- + * Phase III: Defragment data pages -- remove dead tuples from heap + * + * Now that index entries pointing to dead tuples have been removed, we + * can safely defragment data pages. This reclaims the space occupied by + * dead tuples and makes it available for reuse. + * ----------------------------------------------------------------------- + */ + { + ReadStream *stream; + BlockRangeReadStreamPrivate stream_private; + + stream_private.current_blocknum = 0; + stream_private.last_exclusive = nblocks; + + stream = read_stream_begin_relation(READ_STREAM_MAINTENANCE | + READ_STREAM_USE_BATCHING, + bstrategy, + onerel, + MAIN_FORKNUM, + block_range_read_stream_cb, + &stream_private, + 0); + + while ((buf = read_stream_next_buffer(stream, NULL)) != InvalidBuffer) + { + OffsetNumber offnum, + maxoffnum; + ItemId itemid; + bool page_has_dead_tuples = false; + bool page_modified = false; + + CHECK_FOR_INTERRUPTS(); + + blkno = BufferGetBlockNumber(buf); + LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE); + page = BufferGetPage(buf); + + if (PageIsNew(page)) + { + UnlockReleaseBuffer(buf); + continue; + } + + maxoffnum = PageGetMaxOffsetNumber(page); + + /* + * First pass: clean overflow chains for dead tuples. We must do + * this BEFORE defragmenting, because FluxPageDefragment removes + * dead item pointers and after that we can no longer identify + * which tuples had overflow data. We temporarily drop the + * exclusive lock since overflow chain deletion may need to read + * and lock other pages. + */ + for (offnum = FirstOffsetNumber; offnum <= maxoffnum; offnum++) + { + FluxTupleHeader *tuple_hdr; + + itemid = PageGetItemId(page, offnum); + + /* + * LP_DEAD line pointers were already pruned (storage + * reclaimed) by opportunistic pruning and their TIDs recorded + * in Phase I for index cleanup, which has now run. They + * carry no storage and no overflow chain to clean; just flag + * the page so the defragment pass below converts them to + * LP_UNUSED. + */ + if (ItemIdIsDead(itemid)) + { + page_has_dead_tuples = true; + continue; + } + + if (!ItemIdIsNormal(itemid)) + continue; + + tuple_hdr = (FluxTupleHeader *) PageGetItem(page, itemid); + if (FluxIsOverflowRecordInline(tuple_hdr, ItemIdGetLength(itemid))) + continue; + + if (FluxTupleDeadToAll(tuple_hdr, oldest_xmin_vac)) + { + page_has_dead_tuples = true; + + /* + * Delete any external TOAST datums referenced by this + * dead tuple before it is removed by defragmentation. + * FLUX has no on-page overflow chains; wide values live + * in the standard heap TOAST table and are reclaimed here + * exactly as heap VACUUM reclaims them. Only DELETED + * tuples are dead-to-all. + */ + if (tuple_hdr->t_infomask & FLUX_INFOMASK_HASEXTERNAL) + { + TupleDesc vtupdesc = RelationGetDescr(onerel); + Datum vvalues[MaxTupleAttributeNumber]; + bool visnull[MaxTupleAttributeNumber]; + FluxTupleData vtup; + + vtup.t_len = ItemIdGetLength(itemid); + vtup.t_data = tuple_hdr; + ItemPointerSet(&vtup.t_self, blkno, offnum); + FluxDeformTuple(onerel, &vtup, vtupdesc, vvalues, visnull); + + LockBuffer(buf, BUFFER_LOCK_UNLOCK); + flux_toast_delete(onerel, vvalues, visnull, false); + LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE); + page = BufferGetPage(buf); + } + } + } + + /* + * If page has dead tuples, defragment it to consolidate space. + */ + if (page_has_dead_tuples) + { + START_CRIT_SECTION(); + + /* + * Mark dead tuples as unused before defragmenting. The scan + * above already identified them; now set LP_UNUSED. Index + * cleanup (Phase II) has removed every index entry pointing + * at these TIDs, so it is now safe to free the line pointers + * for recycling -- both the still-materialized DELETED tuples + * and the already-pruned LP_DEAD placeholders. + */ + for (offnum = FirstOffsetNumber; offnum <= maxoffnum; offnum++) + { + FluxTupleHeader *vac_hdr; + + itemid = PageGetItemId(page, offnum); + + if (ItemIdIsDead(itemid)) + { + ItemIdSetUnused(itemid); + continue; + } + + if (!ItemIdIsNormal(itemid)) + continue; + + if (FluxIsOverflowRecordInline(PageGetItem(page, itemid), + ItemIdGetLength(itemid))) + continue; + + vac_hdr = (FluxTupleHeader *) PageGetItem(page, itemid); + if (FluxTupleDeadToAll(vac_hdr, oldest_xmin_vac)) + { + ItemIdSetUnused(itemid); + } + } + + FluxPageDefragment(page); + page_modified = true; + pages_vacuumed++; + + MarkBufferDirty(buf); + + if (RelationNeedsWAL(onerel)) + { + XLogRecPtr recptr; + + recptr = FluxXLogDefrag(onerel, buf, NULL, 0, oldest_ts); + PageSetLSN(page, recptr); + } + + END_CRIT_SECTION(); + } + + /* Update FSM with accurate free space information */ + if (page_modified || PageGetFreeSpace(page) > 0) + { + FluxRecordFreeSpace(onerel, blkno, PageGetFreeSpace(page)); + } + + /* + * Update the Visibility Map for this page. + * + * After defragmentation, check whether all remaining tuples on + * the page are visible to all transactions and/or frozen. If so, + * set the appropriate VM bits. This enables index-only scans to + * skip heap fetches and future VACUUMs to skip this page + * entirely. + */ + { + bool all_visible = true; + bool all_frozen = true; + OffsetNumber vm_offnum; + OffsetNumber vm_maxoff; + + vm_maxoff = PageGetMaxOffsetNumber(page); + + for (vm_offnum = FirstOffsetNumber; vm_offnum <= vm_maxoff; vm_offnum++) + { + ItemId vm_itemid; + FluxTupleHeader *vm_tuple_hdr; + + vm_itemid = PageGetItemId(page, vm_offnum); + if (!ItemIdIsNormal(vm_itemid)) + continue; + + /* Skip overflow records */ + if (FluxIsOverflowRecordInline(PageGetItem(page, vm_itemid), + ItemIdGetLength(vm_itemid))) + continue; + + vm_tuple_hdr = (FluxTupleHeader *) PageGetItem(page, vm_itemid); + + /* + * Dead tuples (deleted) that survived defrag are recently + * dead + */ + if (vm_tuple_hdr->t_flags & FLUX_TUPLE_DELETED) + { + all_visible = false; + all_frozen = false; + break; + } + + /* Speculative tuples are not visible to all */ + if (vm_tuple_hdr->t_flags & FLUX_TUPLE_SPECULATIVE) + { + all_visible = false; + all_frozen = false; + break; + } + + /* + * A live (not deleted/speculative/uncommitted -- checked + * above) tuple is all-visible AND all-frozen to every + * possible snapshot iff its inserting XID (t_xmin) has + * committed and precedes the oldest-xmin horizon, so no + * snapshot can fail to see it. (The former test compared + * the whole t_commit_ts word against a timestamp horizon + * -- meaningless in the heap-shaped model where that word + * holds an XID, not an HLC timestamp; mirrors + * FluxTupleDeadToAll's XID-horizon gate.) + */ + if ((vm_tuple_hdr->t_flags & FLUX_TUPLE_UNCOMMITTED) || + !TransactionIdIsValid(vm_tuple_hdr->t_xmin) || + !TransactionIdDidCommit(vm_tuple_hdr->t_xmin) || + !TransactionIdPrecedes(vm_tuple_hdr->t_xmin, oldest_xmin_vac)) + { + all_visible = false; + all_frozen = false; + break; + } + } + + /* Empty pages are trivially all-visible and all-frozen */ + if (vm_maxoff < FirstOffsetNumber) + { + all_visible = true; + all_frozen = true; + } + + FluxVMVacuumPage(onerel, buf, all_visible, all_frozen); + } + + /* Check if page is completely empty (for truncation) */ + if (PageGetMaxOffsetNumber(page) < FirstOffsetNumber) + { + if (blkno == nblocks - 1 - empty_end_pages) + empty_end_pages++; + } + else + { + empty_end_pages = 0; + } + + UnlockReleaseBuffer(buf); + } + + read_stream_end(stream); + } + + /* + * ----------------------------------------------------------------------- + * Phase IV: Index cleanup (amvacuumcleanup) + * + * Call each index AM's vacuum cleanup routine. This lets the index AM do + * any post-vacuum maintenance such as reclaiming empty pages, updating + * statistics, etc. This is called even if no dead tuples were found, + * since some index AMs use this to update internal metadata. + * ----------------------------------------------------------------------- + */ + if (do_index_cleanup) + { + int idx; + + for (idx = 0; idx < nindexes; idx++) + { + IndexVacuumInfo ivinfo; + + ivinfo.index = indrels[idx]; + ivinfo.heaprel = onerel; + ivinfo.analyze_only = false; + ivinfo.report_progress = false; + ivinfo.estimated_count = (nblocks > pages_vacuumed); + ivinfo.message_level = verbose ? INFO : DEBUG2; + ivinfo.num_heap_tuples = (double) live_tuples; + ivinfo.strategy = bstrategy; + + indstats[idx] = vac_cleanup_one_index(&ivinfo, indstats[idx]); + } + } + + /* + * ----------------------------------------------------------------------- + * Phase IV-B: Cross-page defragmentation + * + * Move live tuples from tail pages to front pages so that more trailing + * pages become empty and can be truncated in Phase V. This must run + * after index cleanup (Phase IV) so that stale index entries for + * previously-dead tuples have already been removed. Indexes are still + * open so we can insert new entries for relocated tuples. + * ----------------------------------------------------------------------- + */ + FluxVacuumCrossPageDefrag(onerel, nblocks, + &empty_end_pages, + nindexes, indrels, + bstrategy); + + /* + * ----------------------------------------------------------------------- + * Phase IV-C: Orphan overflow cleanup + * + * Run the two-pass orphan detection algorithm to find and remove overflow + * records that are not referenced by any live tuple. This catches + * overflow records that were orphaned by crashes, aborted transactions, + * or bugs in the eager cleanup path. + * ----------------------------------------------------------------------- + */ + /* FLUX has no on-page overflow records to vacuum (wide values use TOAST). */ + + /* + * ----------------------------------------------------------------------- + * Phase IV-D: per-relation UNDO fork discard + * + * Reclaim space in the relation's UNDO fork by discarding pages whose + * records are all older than the oldest transaction that could still need + * them for rollback. A fork page is discardable iff its max_xid (the + * largest urec_xid on the page) precedes the cluster-wide removable + * horizon, so no in-progress transaction can roll back into it. Run this + * before truncation so freed pages are returned to the fork's free list. + * ----------------------------------------------------------------------- + */ + RelUndoVacuum(onerel, GetOldestNonRemovableTransactionId(onerel), false); + + /* + * ----------------------------------------------------------------------- + * Phase V: Truncation and final cleanup + * ----------------------------------------------------------------------- + */ + + /* Truncate empty pages at the end of the relation */ + if (empty_end_pages > 0 && (params->options & VACOPT_DISABLE_PAGE_SKIPPING) == 0) + { + BlockNumber new_nblocks = nblocks - empty_end_pages; + + /* + * Truncate under AccessExclusiveLock with a tail re-verification, so + * we never remove a block a concurrent INSERT is targeting. The + * helper may truncate to fewer pages than requested (never more) or + * decline entirely if it can't get the lock or the tail is no longer + * empty. + */ + FluxTruncateRelation(onerel, nblocks, new_nblocks, verbose); + } + + /* Update FSM for the entire relation using the real post-truncate size */ + FluxVacuumFSM(onerel, RelationGetNumberOfBlocks(onerel)); + + /* Clean up index resources */ + if (dead_items != NULL) + TidStoreDestroy(dead_items); + if (dead_items_info != NULL) + pfree(dead_items_info); + if (indstats != NULL) + pfree(indstats); + vac_close_indexes(nindexes, indrels, RowExclusiveLock); + + /* Report statistics */ + if (verbose || params->options & VACOPT_VERBOSE) + { + ereport(INFO, + (errmsg("FLUX vacuum \"%s\": found %lld tuples (%lld live, %lld dead), " + "vacuumed %lld pages, truncated %u pages, " + "cleaned %d indexes", + RelationGetRelationName(onerel), + (long long) num_tuples, + (long long) live_tuples, + (long long) dead_tuples, + (long long) pages_vacuumed, + empty_end_pages, + nindexes))); + } +} + +/* ================================================================ + * sLog transaction callbacks for FLUX + * + * These handle FLUX-specific page operations (clearing UNCOMMITTED flags, + * marking aborted tuples as DELETED) at transaction boundaries. They call + * the generic SLogTuple* functions for shared-hash cleanup. + * ================================================================ + */ + +/* + * Two-phase commit record for FLUX. + * + * One record per tracked tuple is saved at PREPARE time via + * RegisterTwoPhaseRecord(). When COMMIT PREPARED fires, the postcommit + * callback uses these to locate and clear UNCOMMITTED flags. When + * ROLLBACK PREPARED fires, the postabort callback marks tuples as aborted. + */ +typedef struct FluxTwoPhaseRecord +{ + Oid relid; + ItemPointerData tid; + bool local_only; /* INSERT-only: no shared sLog entry */ + SLogOpType op_type; /* INSERT, DELETE, or UPDATE */ +} FluxTwoPhaseRecord; + +/* + * Two-phase info values distinguishing the record kinds saved under + * TWOPHASE_RM_FLUX_ID. info==FLUX_2PC_SLOG carries a FluxTwoPhaseRecord + * (per-tuple sLog/visibility state); info==FLUX_2PC_RELUNDO carries a + * FluxRelUndoTwoPhaseRecord (a per-relation UNDO chain head). + */ +#define FLUX_2PC_SLOG 0 +#define FLUX_2PC_RELUNDO 1 + +/* + * Two-phase commit record for a FLUX per-relation UNDO chain head. + * + * One record per relation touched by the prepared transaction. On ROLLBACK + * PREPARED, flux_twophase_postabort() replays this chain via + * RelUndoApplyChain() to restore in-place before-images -- the physical + * data-restore that the per-tuple sLog records (which only flip visibility + * flags) do not perform. + */ +typedef struct FluxRelUndoTwoPhaseRecord +{ + Oid relid; + RelUndoRecPtr start_urec_ptr; +} FluxRelUndoTwoPhaseRecord; + +/* + * FluxEnsureSLogCallbacks -- register xact/subxact callbacks once per backend. + */ +void +FluxEnsureSLogCallbacks(void) +{ + if (!flux_slog_callbacks_registered) + { + RegisterXactCallback(FluxSLogXactCallback, NULL); + RegisterSubXactCallback(FluxSLogSubXactCallback, NULL); + flux_slog_callbacks_registered = true; + } +} + +/* + * Callback for FluxProcessAbortedEntries: mark aborted INSERT tuples as + * DELETED and remove the ABORTED sLog entry. + * + * After marking DELETED on page, we must remove the shared ABORTED sLog entry. + * Otherwise, post-commit readers would find SLOG_OP_ABORTED and interpret it + * as "a delete was aborted" (tuple still alive), rather than "an INSERT was + * aborted" (tuple is dead). With the sLog entry removed, readers see + * DELETED + slog_nfound==0 ā†’ "deletion committed" ā†’ invisible. + */ +static bool +flux_process_aborted_cb(const SLogTupleKey *key, + TransactionId xid, TransactionId subxid, + bool local_only, void *arg) +{ + SLogTupleOp ops[SLOG_MAX_TUPLE_OPS]; + int nfound; + int i; + bool has_aborted = false; + + /* Check if this entry has an ABORTED op */ + nfound = SLogTupleLookupFiltered(key->relid, (ItemPointer) &key->tid, + xid, ops, SLOG_MAX_TUPLE_OPS); + for (i = 0; i < nfound; i++) + { + if (ops[i].op_type == SLOG_OP_ABORTED) + { + has_aborted = true; + break; + } + } + + if (has_aborted) + { + Buffer buf; + Page page; + ItemId itemid; + FluxTupleHeader *tuple_hdr; + OffsetNumber offnum; + Relation rel; + + rel = try_relation_open(key->relid, AccessShareLock); + if (rel == NULL) + return true; /* continue iteration */ + buf = ReadBuffer(rel, ItemPointerGetBlockNumber((ItemPointer) &key->tid)); + LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE); + page = BufferGetPage(buf); + offnum = ItemPointerGetOffsetNumber((ItemPointer) &key->tid); + + if (offnum <= PageGetMaxOffsetNumber(page)) + { + itemid = PageGetItemId(page, offnum); + if (ItemIdIsNormal(itemid)) + { + tuple_hdr = (FluxTupleHeader *) PageGetItem(page, itemid); + + if (tuple_hdr->t_flags & FLUX_TUPLE_UNCOMMITTED) + { + tuple_hdr->t_flags |= FLUX_TUPLE_DELETED; + tuple_hdr->t_flags &= ~FLUX_TUPLE_UNCOMMITTED; + MarkBufferDirty(buf); + } + } + } + + UnlockReleaseBuffer(buf); + relation_close(rel, AccessShareLock); + + /* + * Remove the ABORTED sLog entry now that the page is marked DELETED. + * This ensures post-commit readers see DELETED + no sLog entries, + * which the visibility function correctly interprets as "deletion + * committed" (tuple invisible). + */ + SLogTupleRemove(key->relid, (ItemPointer) &key->tid, xid); + } + + return true; /* continue iteration */ +} + +/* + * FluxProcessAbortedEntries -- at COMMIT, mark tuples from rolled-back + * subtransactions as DELETED on their pages. + */ +static void +FluxProcessAbortedEntries(TransactionId xid) +{ + SLogTupleIterateTrackedKeys(xid, flux_process_aborted_cb, NULL); +} + +/* ---------------------------------------------------------------- + * Batched commit-time UNCOMMITTED-flag clearing for + * FluxClearUncommittedFlags. + * + * Commit visibility comes from CLOG (heap-shaped xmin/xmax MVCC); commit + * does NOT stamp any timestamp. This path only clears the on-page + * FLUX_TUPLE_UNCOMMITTED hint flag so later readers skip the sLog + * fast-path lookup. t_xmin/t_xmax written by the DML stay untouched. + * + * The batched approach collects tracked keys, sorts by (relid, blockno), + * and processes them with minimal buffer I/O: + * - One try_relation_open() per distinct relid + * - One ReadBuffer() per distinct block + * - Local-only INSERTs skip the shared sLog lookup entirely + * ---------------------------------------------------------------- + */ +/* + * flux_cmp_tracked_key_by_block -- qsort comparator for batch commit stamping. + * + * Sorts by (relid, blockno, offnum) to enable sequential I/O: one + * try_relation_open per relation, one ReadBuffer per distinct block. + */ +static int +flux_cmp_tracked_key_by_block(const void *a, const void *b) +{ + const SLogTrackedKeyInfo *ka = (const SLogTrackedKeyInfo *) a; + const SLogTrackedKeyInfo *kb = (const SLogTrackedKeyInfo *) b; + + if (ka->key.relid < kb->key.relid) + return -1; + if (ka->key.relid > kb->key.relid) + return 1; + + { + BlockNumber ba = ItemPointerGetBlockNumber((ItemPointer) &ka->key.tid); + BlockNumber bb = ItemPointerGetBlockNumber((ItemPointer) &kb->key.tid); + + if (ba < bb) + return -1; + if (ba > bb) + return 1; + } + + { + OffsetNumber oa = ItemPointerGetOffsetNumber((ItemPointer) &ka->key.tid); + OffsetNumber ob = ItemPointerGetOffsetNumber((ItemPointer) &kb->key.tid); + + if (oa < ob) + return -1; + if (oa > ob) + return 1; + } + + return 0; +} + +/* + * flux_stamp_tuple_committed -- stamp a single tuple at commit time. + * + * Applies the appropriate timestamp and clears FLUX_TUPLE_UNCOMMITTED + * based on the tracked operation type. Called from the batched commit path + * with the buffer already locked exclusive. + * + * For local-only INSERTs (the common case for single-row INSERT), we skip + * the expensive SLogTupleLookupFiltered() call. A local-only INSERT can + * only be savepoint-aborted if SLogTupleRemoveBySubXid created a shared + * ABORTED entry ā€” in which case local_only would have been cleared on the + * tracked key. So if local_only is still true, this is a live INSERT. + */ +static void +flux_stamp_tuple_committed(Buffer buf, OffsetNumber offnum, + const SLogTrackedKeyInfo *tk, + TransactionId xid) +{ + Page page = BufferGetPage(buf); + ItemId itemid; + FluxTupleHeader *tuple_hdr; + SLogOpType found_op_type; + + if (offnum > PageGetMaxOffsetNumber(page)) + return; + itemid = PageGetItemId(page, offnum); + if (!ItemIdIsNormal(itemid)) + return; + + tuple_hdr = (FluxTupleHeader *) PageGetItem(page, itemid); + + /* + * Determine the effective operation type for this tuple. + * + * A tracked key whose op_type was flipped to SLOG_OP_ABORTED by + * SLogTupleRemoveBySubXid (savepoint rollback) must be skipped: its tuple + * is logically discarded and a shared ABORTED sLog entry already exists + * to enforce invisibility. Clearing UNCOMMITTED here would resurrect it. + * This check must precede the local_only fast path because savepoint- + * aborted INSERTs keep local_only == true. + * + * For local-only entries (INSERTs with no shared sLog entry), skip the + * shared hash lookup entirely ā€” this is the key optimization for INSERT + * workloads. + */ + if (tk->op_type == SLOG_OP_ABORTED) + { + found_op_type = SLOG_OP_ABORTED; + } + else if (tk->local_only) + { + /* Fast path: live local-only INSERT, no shared sLog lookup needed */ + found_op_type = SLOG_OP_INSERT; + } + else if (tk->op_type == SLOG_OP_INSERT || + tk->op_type == SLOG_OP_UPDATE || + tk->op_type == SLOG_OP_DELETE) + { + /* op_type was tracked correctly at insert time -- skip shared lookup */ + found_op_type = tk->op_type; + } + else + { + /* Fallback: unknown op_type, do the expensive lookup */ + SLogTupleOp ops[SLOG_MAX_TUPLE_OPS]; + int nfound; + int i; + + found_op_type = SLOG_ENTRY_ABORTED_TXN; + nfound = SLogTupleLookupFiltered(tk->key.relid, + (ItemPointer) &tk->key.tid, + xid, ops, SLOG_MAX_TUPLE_OPS); + for (i = 0; i < nfound; i++) + { + if (TransactionIdEquals(ops[i].xid, xid)) + { + found_op_type = ops[i].op_type; + break; + } + } + } + + /* + * Skip entries that were aborted by a savepoint rollback. + */ + if (found_op_type == SLOG_OP_ABORTED) + return; + + /* + * For SLOG_ENTRY_ABORTED_TXN with a non-local entry, this means no shared + * entry was found ā€” shouldn't happen for non-local, but treat as skip. + * For unrecognized op types, also skip. + */ + if (!tk->local_only && found_op_type == SLOG_ENTRY_ABORTED_TXN) + return; + if (found_op_type != SLOG_OP_INSERT && + found_op_type != SLOG_OP_UPDATE && + found_op_type != SLOG_OP_DELETE) + return; + + /* + * Clear the UNCOMMITTED flag (a hint bit) for INSERT/UPDATE/DELETE + * tuples. In the heap-shaped xmin/xmax model, commit does NOT stamp any + * timestamp: t_xmin (inserter) and t_xmax (deleter/updater, in the + * t_commit_ts word) were written by the DML and stay untouched. + * Overwriting t_commit_ts here would clobber t_xmax with a bogus value + * and corrupt visibility (a garbage low-32-bits would be read as an xmax + * XID). CLOG is the authoritative commit oracle; clearing the flag is a + * pure optimization so later readers skip the sLog fast-path lookup. + */ + if (tuple_hdr->t_flags & FLUX_TUPLE_UNCOMMITTED) + tuple_hdr->t_flags &= ~FLUX_TUPLE_UNCOMMITTED; +} + +/* + * flux_batch_clear_uncommitted -- batch-process tracked keys at commit time. + * + * Processes the pre-sorted array of tracked keys with sequential I/O: + * - One try_relation_open() per distinct relid + * - One ReadBuffer() per distinct block within each relation + * - All tuples on the same page are stamped while holding one buffer lock + * + * This replaces the per-tuple callback pattern which did O(n) ReadBuffer + * calls even when multiple tuples shared the same page. + */ +static void +flux_batch_clear_uncommitted(SLogTrackedKeyInfo *keys, int nkeys, + TransactionId xid) +{ + Oid cur_relid = InvalidOid; + BlockNumber cur_blkno = InvalidBlockNumber; + Relation rel = NULL; + Buffer buf = InvalidBuffer; + int i; + + /* + * Prefetch: issue advisory read-ahead for the first few distinct blocks. + * Since keys are sorted by (relid, blockno), we can scan ahead cheaply. + * This overlaps I/O with the kernel readahead path. + */ + if (nkeys > 0) + { + Oid pf_relid = keys[0].key.relid; + Relation pf_rel; + BlockNumber pf_prev = InvalidBlockNumber; + int pf_count = 0; + + pf_rel = try_relation_open(pf_relid, AccessShareLock); + if (pf_rel != NULL) + { + for (int j = 0; j < nkeys && pf_count < 8; j++) + { + BlockNumber pf_blk; + + /* Stop prefetching if we cross into a different relation */ + if (keys[j].key.relid != pf_relid) + break; + + pf_blk = ItemPointerGetBlockNumber((ItemPointer) &keys[j].key.tid); + if (pf_blk != pf_prev) + { + PrefetchBuffer(pf_rel, MAIN_FORKNUM, pf_blk); + pf_prev = pf_blk; + pf_count++; + } + } + relation_close(pf_rel, AccessShareLock); + } + } + + for (i = 0; i < nkeys; i++) + { + SLogTrackedKeyInfo *tk = &keys[i]; + Oid relid = tk->key.relid; + BlockNumber blkno = ItemPointerGetBlockNumber((ItemPointer) &tk->key.tid); + OffsetNumber offnum = ItemPointerGetOffsetNumber((ItemPointer) &tk->key.tid); + + /* Switch relation if needed (sorted order minimizes switches) */ + if (relid != cur_relid) + { + /* Mark outgoing page dirty (once per page, not per tuple) */ + if (BufferIsValid(buf)) + { + MarkBufferDirtyHint(buf, true); + UnlockReleaseBuffer(buf); + buf = InvalidBuffer; + } + if (rel != NULL) + { + relation_close(rel, AccessShareLock); + rel = NULL; + } + + rel = try_relation_open(relid, AccessShareLock); + if (rel == NULL) + { + cur_relid = relid; + cur_blkno = InvalidBlockNumber; + continue; + } + cur_relid = relid; + cur_blkno = InvalidBlockNumber; + } + + if (rel == NULL) + continue; + + /* + * Switch block if needed ā€” amortizes ReadBuffer across same-page + * tuples + */ + if (blkno != cur_blkno) + { + /* Mark outgoing page dirty (once per page, not per tuple) */ + if (BufferIsValid(buf)) + { + MarkBufferDirtyHint(buf, true); + UnlockReleaseBuffer(buf); + buf = InvalidBuffer; + } + + PG_TRY(); + { + buf = ReadBuffer(rel, blkno); + LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE); + } + PG_CATCH(); + { + /* + * If ReadBuffer fails (e.g., relation truncated + * concurrently), skip this block and continue with the next. + */ + buf = InvalidBuffer; + EmitErrorReport(); + FlushErrorState(); + cur_blkno = blkno; + continue; + } + PG_END_TRY(); + + cur_blkno = blkno; + } + + if (!BufferIsValid(buf)) + continue; + + /* Stamp this tuple */ + flux_stamp_tuple_committed(buf, offnum, tk, xid); + } + + /* Mark final page dirty and release */ + if (BufferIsValid(buf)) + { + MarkBufferDirtyHint(buf, true); + UnlockReleaseBuffer(buf); + } + if (rel != NULL) + relation_close(rel, AccessShareLock); +} + +/* + * FluxClearUncommittedFlags -- proactively clear FLUX_TUPLE_UNCOMMITTED on + * all tuples modified by the current transaction at PRE_COMMIT time. + * + * Uses a batched approach: collects all tracked keys into an array, sorts + * by (relid, blockno) for sequential I/O, then processes them with at most + * one ReadBuffer per distinct block and one try_relation_open per relation. + * + * For local-only INSERTs (the common single-row INSERT case), the expensive + * SLogTupleLookupFiltered() call is skipped entirely ā€” a local-only entry + * that hasn't been promoted to a shared ABORTED entry is guaranteed to be + * a live INSERT. + * + * Clearing the flag is a pure hint: commit visibility comes from CLOG + * (heap-shaped xmin/xmax MVCC), not from any stamped timestamp. + */ +static void +FluxClearUncommittedFlags(TransactionId xid) +{ + SLogTrackedKeyInfo *keys; + int nkeys; + + /* + * When lazy clear is enabled, skip the expensive batch page-visit loop. + * The UNCOMMITTED flags will be cleared lazily by readers via the + * visibility functions when they next access these tuples. + */ + if (flux_lazy_uncommitted_clear) + return; + + /* Collect tracked keys into a sortable array */ + nkeys = SLogTupleCollectTrackedKeys(xid, &keys); + if (nkeys == 0) + { + pfree(keys); + return; + } + + /* Sort by (relid, blockno, offnum) for sequential I/O */ + qsort(keys, nkeys, sizeof(SLogTrackedKeyInfo), flux_cmp_tracked_key_by_block); + + /* Batch-process: one ReadBuffer per distinct block */ + flux_batch_clear_uncommitted(keys, nkeys, xid); + + pfree(keys); +} + +/* + * FluxSLogXactCallback -- clean up sLog entries at transaction end. + */ +/* + * flux_register_twophase_cb -- callback for SLogTupleIterateTrackedKeys + * during PREPARE. Saves each tracked tuple as a two-phase record so that + * COMMIT PREPARED / ROLLBACK PREPARED can find them. + * + * For local-only entries (INSERTs), also creates a shared sLog entry so + * that other backends can see the transaction is in-progress and not treat + * the UNCOMMITTED flag as "stale committed." + */ +static bool +flux_register_twophase_cb(const SLogTupleKey *key, + TransactionId xid, TransactionId subxid, + bool local_only, void *arg) +{ + FluxTwoPhaseRecord rec; + + rec.relid = key->relid; + ItemPointerCopy(&key->tid, &rec.tid); + rec.local_only = local_only; + + /* + * Determine op_type: look up in shared sLog if not local-only. For + * local-only entries (INSERTs), we know it's SLOG_OP_INSERT. + */ + if (local_only) + { + rec.op_type = SLOG_OP_INSERT; + + /* + * Promote local-only INSERT to a shared sLog entry. This is critical + * for 2PC correctness: after PREPARE, the originating backend's local + * tracking is gone, but the tuple still has FLUX_TUPLE_UNCOMMITTED + * set. Without a shared sLog entry, other backends would see + * slog_nfound==0 and return invisible (correct), but COMMIT PREPARED + * needs the entry for its postcommit callback to locate and finalize + * the tuple. + * + * With the shared entry, other backends will also find the XID in + * sLog, call TransactionIdIsInProgress() ā†’ true (prepared XIDs are + * still "in progress"), and correctly hide the tuple. + */ + SLogTupleInsertRecovery(key->relid, (ItemPointer) &key->tid, + xid, SLOG_OP_INSERT); + } + else + { + SLogTupleOp ops[SLOG_MAX_TUPLE_OPS]; + int nfound; + int i; + + rec.op_type = SLOG_OP_INSERT; /* fallback */ + nfound = SLogTupleLookupFiltered(key->relid, (ItemPointer) &key->tid, + xid, ops, SLOG_MAX_TUPLE_OPS); + for (i = 0; i < nfound; i++) + { + if (TransactionIdEquals(ops[i].xid, xid)) + { + rec.op_type = ops[i].op_type; + break; + } + } + } + + RegisterTwoPhaseRecord(TWOPHASE_RM_FLUX_ID, FLUX_2PC_SLOG, + &rec, sizeof(FluxTwoPhaseRecord)); + return true; /* continue iteration */ +} + +/* + * flux_register_relundo_cb -- IteratePerRelUndo callback. + * + * Serializes one per-relation UNDO chain head into the 2PC state file so + * ROLLBACK PREPARED can restore in-place before-images. + */ +static void +flux_register_relundo_cb(Oid relid, RelUndoRecPtr start_urec_ptr, void *arg) +{ + FluxRelUndoTwoPhaseRecord rec; + + if (!RelUndoRecPtrIsValid(start_urec_ptr)) + return; + + rec.relid = relid; + rec.start_urec_ptr = start_urec_ptr; + RegisterTwoPhaseRecord(TWOPHASE_RM_FLUX_ID, FLUX_2PC_RELUNDO, + &rec, sizeof(FluxRelUndoTwoPhaseRecord)); +} + +/* + * AtPrepare_Flux -- register two-phase records for FLUX tuples. + * + * Called from PrepareTransaction() between StartPrepare() and EndPrepare(), + * where RegisterTwoPhaseRecord() is valid. Saves each tracked tuple so + * that COMMIT PREPARED / ROLLBACK PREPARED can locate and finalize them. + */ +void +AtPrepare_Flux(void) +{ + TransactionId xid = GetCurrentTransactionIdIfAny(); + + if (!TransactionIdIsValid(xid)) + return; + + SLogTupleIterateTrackedKeys(xid, flux_register_twophase_cb, NULL); + + /* + * Serialize the per-relation UNDO chain heads too. The sLog records + * above carry only per-tuple visibility state; the physical before-image + * of an in-place UPDATE lives in the per-relation UNDO chain and must be + * replayed by RelUndoApplyChain() on ROLLBACK PREPARED. + */ + IteratePerRelUndo(flux_register_relundo_cb, NULL); + + /* + * Relocate this xact's oldest-active-timestamp pin from this backend's + * proc slot onto the prepared xact's dummy-proc slot, so the pin survives + * this backend running new transactions or exiting entirely, and so the + * (possibly different) backend that later runs COMMIT/ROLLBACK PREPARED + * can find and clear it. MarkAsPreparing() has already assigned the + * dummy proc, so its slot number is available here. + */ + { + FullTransactionId prep_fxid = GetTopFullTransactionIdIfAny(); + + if (FullTransactionIdIsValid(prep_fxid)) + FluxPrepareReassignSlot(TwoPhaseGetDummyProcNumber(prep_fxid, false)); + } +} + +static void +FluxSLogXactCallback(XactEvent event, void *arg) +{ + switch (event) + { + case XACT_EVENT_PRE_COMMIT: + { + TransactionId xid = GetCurrentTransactionIdIfAny(); + + if (TransactionIdIsValid(xid)) + { + /* + * Ordering here is load-bearing for write-write + * correctness. Three steps, in this exact order: + * + * 1. FluxProcessAbortedEntries -- mark pages DELETED for + * subxact-aborted ops and remove their ABORTED sLog + * entries. MUST run before SLogTupleCommitByXid, whose + * COMMIT_XID apply removes every non-UPDATE op for the + * xid; running it first would erase the ABORTED marker + * this step needs and resurrect an aborted tuple. + * + * 2. SLogTupleCommitByXid -- finalize the xid's sLog ops. + * + * 3. FluxClearUncommittedFlags -- clear the on-page + * UNCOMMITTED hint flag. Commit visibility itself comes + * from CLOG (heap-shaped xmin/xmax MVCC); clearing the + * flag is a pure hint so later readers skip the sLog + * fast-path lookup. + * + * All three run at PRE_COMMIT, before the point of no + * return: DSA allocation must stay in an abort-legal + * phase to avoid a post-commit OOM->PANIC. + */ + FluxProcessAbortedEntries(xid); + SLogTupleCommitByXid(xid); + FluxClearUncommittedFlags(xid); + } + } + break; + + case XACT_EVENT_PRE_PREPARE: + { + TransactionId xid = GetCurrentTransactionIdIfAny(); + + if (TransactionIdIsValid(xid)) + { + /* + * At PREPARE, we must NOT clear UNCOMMITTED flags. The + * transaction is not yet committed and another backend + * might ROLLBACK PREPARED. + * + * We still need to process any subtransaction-aborted + * entries (mark them DELETED on page) since those are + * definitively aborted regardless of PREPARE outcome. + * + * Two-phase record registration happens in + * AtPrepare_Flux(), called from PrepareTransaction() + * after StartPrepare(). + */ + FluxProcessAbortedEntries(xid); + } + } + break; + + case XACT_EVENT_COMMIT: + case XACT_EVENT_PARALLEL_COMMIT: + { + /* + * The UNCOMMITTED-flag clear and before-image handling + * already happened at XACT_EVENT_PRE_COMMIT (see above). Here + * we only release backend-local tracking state, which is safe + * in the post-commit no-abort region. Commit visibility + * comes from CLOG (heap-shaped xmin/xmax MVCC); nothing + * further is written at COMMIT. + */ + SLogTupleResetTracking(); + } + break; + + case XACT_EVENT_PREPARE: + { + /* + * At PREPARE completion, do NOT remove sLog entries or + * decrement dirty map counters. The sLog entries must + * persist so that visibility checks can see the transaction + * is still in-progress (prepared). Only discard the + * backend-local tracking list since this backend is done. + * + * The two-phase records registered during PRE_PREPARE will be + * used by COMMIT PREPARED / ROLLBACK PREPARED to perform the + * actual cleanup. + */ + SLogTupleResetTracking(); + } + break; + + case XACT_EVENT_ABORT: + case XACT_EVENT_PARALLEL_ABORT: + { + TransactionId xid = GetCurrentTransactionIdIfAny(); + + if (TransactionIdIsValid(xid)) + { + /* + * Mark this transaction's shared sLog ops ABORTED. The + * DSA before-images are freed inside SLogTupleMarkAborted + * under the partition writer lock (single-owner), so we + * must not free them here: an unlocked free races the + * UNDO worker's SLogTupleRemoveByXidGlobal and corrupts + * the DSA heap. + */ + SLogTupleMarkAborted(xid); + } + + /* + * The dirty map is grow-only and carries no per-transaction + * tracking, so there is nothing to undo at ABORT. Leaving + * the page bit set is correct: the sLog entries are marked + * ABORTED (not removed), so a scanner must still probe the + * sLog to detect the aborted state and treat the tuples as + * still-live. + */ + SLogTupleResetTracking(); + } + break; + + default: + break; + } +} + +/* + * flux_restore_before_image_cb -- callback for FluxRestoreBeforeImages. + * + * For each tracked key with a before-image in the rolled-back subtransaction, + * physically restore the original tuple data on the page. + */ +static bool +flux_restore_before_image_cb(const SLogTupleKey *key, + TransactionId xid, TransactionId subxid, + bool local_only, void *arg) +{ + char *before_data; + int before_len; + uint16 before_flags; + uint64 before_commit_ts; + RelFileLocator before_rlocator; + char before_relpersistence; + + /* Check if this tracked key has a before-image */ + if (!SLogTupleGetBeforeImage(key->relid, (ItemPointer) &key->tid, + xid, subxid, + &before_data, &before_len, + &before_flags, &before_commit_ts, + &before_rlocator, &before_relpersistence)) + return true; /* No before-image (INSERT), continue */ + + /* + * Restore the physical tuple on the page. + * + * This callback fires from SUBXACT_EVENT_ABORT_SUB, by which point the + * subtransaction is already in TRANS_ABORT state (AbortSubTransaction + * sets the state before invoking subxact callbacks). That makes any + * relcache access -- relation_open and friends -- unsafe: it would trip + * the IsTransactionState() assertion in AssertCouldGetRelation. We + * therefore go straight to the buffer manager using the RelFileLocator + * captured at store time, which needs no relcache and works regardless of + * transaction state (including proc_exit teardown). + */ + { + Buffer buf = InvalidBuffer; + Page page; + ItemId itemid; + FluxTupleHeader *tuple_hdr; + OffsetNumber offnum; + bool permanent = (before_relpersistence == RELPERSISTENCE_PERMANENT); + + PG_TRY(); + { + buf = ReadBufferWithoutRelcache(before_rlocator, MAIN_FORKNUM, + ItemPointerGetBlockNumber((ItemPointer) &key->tid), + RBM_NORMAL, NULL, permanent); + LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE); + page = BufferGetPage(buf); + offnum = ItemPointerGetOffsetNumber((ItemPointer) &key->tid); + + if (offnum <= PageGetMaxOffsetNumber(page)) + { + itemid = PageGetItemId(page, offnum); + if (ItemIdIsNormal(itemid)) + { + tuple_hdr = (FluxTupleHeader *) PageGetItem(page, itemid); + + /* + * Restore the tuple to its pre-DML state. + * + * The before-image was the original occupant of this item + * slot. For Strategy 1 in-place updates (new tuple + * smaller than old), the physical page space at this + * offset hasn't been reclaimed or compacted within the + * same subtransaction, so writing before_len bytes is + * safe even when before_len > ItemIdGetLength. + */ + memcpy(tuple_hdr, before_data, before_len); + + /* Update item length if size changed */ + if (before_len != (int) ItemIdGetLength(itemid)) + ItemIdSetNormal(itemid, ItemIdGetOffset(itemid), + before_len); + + MarkBufferDirtyHint(buf, true); + } + } + + UnlockReleaseBuffer(buf); + buf = InvalidBuffer; + } + PG_CATCH(); + { + if (BufferIsValid(buf)) + UnlockReleaseBuffer(buf); + EmitErrorReport(); + FlushErrorState(); + } + PG_END_TRY(); + } + + return true; /* continue iteration */ +} + +/* + * FluxRestoreBeforeImages -- on savepoint rollback, restore physical tuples + * that were modified by the rolled-back subtransaction. + * + * Iterates tracked keys for the given (xid, subxid) and for each one that + * has a stashed before-image, reads the buffer and restores the tuple data. + * This must be called BEFORE SLogTupleRemoveBySubXid (which marks sLog + * entries as ABORTED) so that the tracked key list still has the subxid. + */ +static void +FluxRestoreBeforeImages(TransactionId xid, SubTransactionId subxid) +{ + SLogTupleIterateTrackedKeysForSubXid(xid, subxid, + flux_restore_before_image_cb, + NULL); +} + +/* + * FluxSLogSubXactCallback -- handle subtransaction events. + */ +static void +FluxSLogSubXactCallback(SubXactEvent event, + SubTransactionId mySubid, + SubTransactionId parentSubid, + void *arg) +{ + TransactionId xid; + + switch (event) + { + case SUBXACT_EVENT_ABORT_SUB: + xid = GetTopTransactionIdIfAny(); + if (TransactionIdIsValid(xid)) + { + /* + * Restore physical tuples from before-images FIRST, while the + * tracked key list still has entries for this subxid. Then + * mark sLog entries as ABORTED for visibility. + */ + FluxRestoreBeforeImages(xid, mySubid); + SLogTupleRemoveBySubXid(xid, mySubid); + } + + /* + * The dirty map is grow-only with no per-subtransaction tracking, + * so there is nothing to undo at subtransaction abort. Leaving + * the page bit set is correct: SLogTupleRemoveBySubXid marks + * entries SLOG_OP_ABORTED (does not remove them), so a scanner + * must still probe the sLog to detect the aborted state. + */ + break; + + case SUBXACT_EVENT_COMMIT_SUB: + xid = GetTopTransactionIdIfAny(); + if (TransactionIdIsValid(xid)) + SLogTupleUpdateSubXid(xid, mySubid, parentSubid); + break; + + default: + break; + } +} + +/* ================================================================ + * Two-phase commit callbacks for FLUX + * + * These are invoked by FinishPreparedTransaction() in the backend that + * runs COMMIT PREPARED or ROLLBACK PREPARED. They perform the tuple-level + * cleanup that would normally happen at XACT_EVENT_PRE_COMMIT / COMMIT + * or ABORT in the originating backend. + * ================================================================ + */ + +/* + * flux_twophase_postcommit -- called for each saved record when + * COMMIT PREPARED resolves a prepared transaction. + * + * Clears FLUX_TUPLE_UNCOMMITTED and removes + * shared sLog entries (for DELETE/UPDATE operations). + */ +void +flux_twophase_postcommit(FullTransactionId fxid, uint16 info, + void *recdata, uint32 len) +{ + FluxTwoPhaseRecord *rec = (FluxTwoPhaseRecord *) recdata; + TransactionId xid = XidFromFullTransactionId(fxid); + Buffer buf = InvalidBuffer; + Page page; + ItemId itemid; + FluxTupleHeader *tuple_hdr; + OffsetNumber offnum; + Relation rel; + + /* + * Release this prepared xact's oldest-active-timestamp pin (moved onto + * the dummy-proc slot at PREPARE). Idempotent, so running it for every + * record is harmless. Runs under TwoPhaseStateLock held by + * FinishPreparedTransaction(), so look the dummy proc up with lock_held. + */ + FluxResolvePreparedSlot(TwoPhaseGetDummyProcNumber(fxid, true)); + + /* + * Per-relation UNDO chain heads (FLUX_2PC_RELUNDO) are irrelevant on + * COMMIT PREPARED: the committed in-place data is already correct, the + * chain is simply discarded. Only the per-tuple sLog records need work. + */ + if (info == FLUX_2PC_RELUNDO) + return; + + Assert(len == sizeof(FluxTwoPhaseRecord)); + + /* + * Heap-shaped COMMIT PREPARED: nothing to stamp. t_xmin/t_xmax were + * written by the prepared DML and stay untouched; CLOG (updated by the + * 2PC machinery) makes the committed XID visible. We only clear the + * UNCOMMITTED hint-bit flag below. + */ + + rel = try_relation_open(rec->relid, AccessShareLock); + if (rel == NULL) + return; /* relation dropped before COMMIT PREPARED */ + + PG_TRY(); + { + buf = ReadBuffer(rel, ItemPointerGetBlockNumber(&rec->tid)); + LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE); + page = BufferGetPage(buf); + offnum = ItemPointerGetOffsetNumber(&rec->tid); + + if (offnum <= PageGetMaxOffsetNumber(page)) + { + itemid = PageGetItemId(page, offnum); + if (ItemIdIsNormal(itemid)) + { + tuple_hdr = (FluxTupleHeader *) PageGetItem(page, itemid); + + /* + * Clear UNCOMMITTED flag (hint bit); do NOT touch + * t_commit_ts. + */ + if (tuple_hdr->t_flags & FLUX_TUPLE_UNCOMMITTED) + tuple_hdr->t_flags &= ~FLUX_TUPLE_UNCOMMITTED; + + MarkBufferDirtyHint(buf, true); + } + } + + UnlockReleaseBuffer(buf); + buf = InvalidBuffer; + relation_close(rel, AccessShareLock); + rel = NULL; + } + PG_CATCH(); + { + if (BufferIsValid(buf)) + UnlockReleaseBuffer(buf); + if (rel != NULL) + relation_close(rel, AccessShareLock); + EmitErrorReport(); + FlushErrorState(); + } + PG_END_TRY(); + + /* + * Remove shared sLog entry for this tuple. At PREPARE time, we promoted + * local-only entries to shared (via SLogTupleInsertRecovery), so ALL + * entries now have a shared sLog entry that needs cleanup. + */ + SLogTupleRemoveByXidSingle(rec->relid, &rec->tid, xid); +} + +/* + * flux_twophase_postabort -- called for each saved record when + * ROLLBACK PREPARED resolves a prepared transaction. + * + * For INSERTs: marks the tuple as DELETED (the insert is rolled back). + * For DELETEs/UPDATEs: marks the sLog entry as ABORTED (the operation + * is undone, tuple remains/reverts to live). + */ +void +flux_twophase_postabort(FullTransactionId fxid, uint16 info, + void *recdata, uint32 len) +{ + FluxTwoPhaseRecord *rec = (FluxTwoPhaseRecord *) recdata; + TransactionId xid = XidFromFullTransactionId(fxid); + Buffer buf = InvalidBuffer; + Page page; + ItemId itemid; + FluxTupleHeader *tuple_hdr; + OffsetNumber offnum; + Relation rel; + + /* + * Release this prepared xact's oldest-active-timestamp pin (moved onto + * the dummy-proc slot at PREPARE). Idempotent and lock-held, as in + * flux_twophase_postcommit(); must precede the early returns below. + */ + FluxResolvePreparedSlot(TwoPhaseGetDummyProcNumber(fxid, true)); + + /* + * Per-relation UNDO chain head (FLUX_2PC_RELUNDO): replay the chain to + * restore in-place before-images. This is the physical data-restore for + * an aborted in-place UPDATE -- the per-tuple sLog records below only + * flip visibility flags. We run in the finishing backend's own live + * transaction (COMMIT/ROLLBACK PREPARED), so relation_open and the chain + * walk are legal here without the TRANS_ABORT juggling ApplyPerRelUndo + * needs. + */ + if (info == FLUX_2PC_RELUNDO) + { + FluxRelUndoTwoPhaseRecord *urec = + (FluxRelUndoTwoPhaseRecord *) recdata; + Relation urel; + + Assert(len == sizeof(FluxRelUndoTwoPhaseRecord)); + + urel = try_relation_open(urec->relid, RowExclusiveLock); + if (urel == NULL) + return; /* relation dropped */ + + PG_TRY(); + { + RelUndoApplyChain(urel, urec->start_urec_ptr); + } + PG_CATCH(); + { + BufferLockReleaseAll(); + EmitErrorReport(); + FlushErrorState(); + } + PG_END_TRY(); + + relation_close(urel, RowExclusiveLock); + return; + } + + Assert(len == sizeof(FluxTwoPhaseRecord)); + + rel = try_relation_open(rec->relid, AccessShareLock); + if (rel == NULL) + return; /* relation dropped */ + + PG_TRY(); + { + buf = ReadBuffer(rel, ItemPointerGetBlockNumber(&rec->tid)); + LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE); + page = BufferGetPage(buf); + offnum = ItemPointerGetOffsetNumber(&rec->tid); + + if (offnum <= PageGetMaxOffsetNumber(page)) + { + itemid = PageGetItemId(page, offnum); + if (ItemIdIsNormal(itemid)) + { + tuple_hdr = (FluxTupleHeader *) PageGetItem(page, itemid); + + if (rec->op_type == SLOG_OP_INSERT) + { + /* + * Aborted INSERT: keep UNCOMMITTED set. The shared sLog + * entry is marked ABORTED below, so the visibility code + * path at flux_mvcc.c:1008 will see UNCOMMITTED + + * SLOG_OP_ABORTED and goto not_visible. + * + * We do NOT clear UNCOMMITTED or set DELETED here, + * because that combination (committed-looking tuple with + * DELETED + ABORTED sLog) causes the deletion-check path + * to incorrectly reverse the deletion and make the tuple + * visible. + * + * The UNDO worker / VACUUM will eventually physically + * remove the dead tuple by recognizing the UNCOMMITTED + + * ABORTED pattern. + */ + } + else + { + /* + * Aborted DELETE/UPDATE: the tuple reverts to its pre-DML + * state. Clear any flags set by the aborted operation. + * For DELETE, remove the DELETED flag. For UPDATE, remove + * UPDATED flag and any UNCOMMITTED. + */ + if (rec->op_type == SLOG_OP_DELETE) + { + if (tuple_hdr->t_flags & FLUX_TUPLE_DELETED) + { + tuple_hdr->t_flags &= ~FLUX_TUPLE_DELETED; + MarkBufferDirtyHint(buf, true); + } + } + else if (rec->op_type == SLOG_OP_UPDATE) + { + if (tuple_hdr->t_flags & FLUX_TUPLE_UPDATED) + { + tuple_hdr->t_flags &= ~FLUX_TUPLE_UPDATED; + tuple_hdr->t_flags &= ~FLUX_TUPLE_UNCOMMITTED; + MarkBufferDirtyHint(buf, true); + } + } + } + } + } + + UnlockReleaseBuffer(buf); + buf = InvalidBuffer; + relation_close(rel, AccessShareLock); + rel = NULL; + } + PG_CATCH(); + { + if (BufferIsValid(buf)) + UnlockReleaseBuffer(buf); + if (rel != NULL) + relation_close(rel, AccessShareLock); + EmitErrorReport(); + FlushErrorState(); + } + PG_END_TRY(); + + /* + * Mark shared sLog entry as ABORTED. At PREPARE time, we promoted all + * local-only entries to shared, so every entry has a shared sLog entry. + * Marking it ABORTED ensures visibility code correctly hides the tuple + * until UNDO/VACUUM removes it. + */ + SLogTupleMarkAbortedSingle(rec->relid, &rec->tid, xid); +} + +/* + * flux_twophase_recover -- called during startup recovery for each + * saved FLUX record in a prepared transaction's state file. + * + * During recovery, we don't need to do anything special: the tuples + * are already in their prepared-but-uncommitted state on disk (with + * UNCOMMITTED flag set for INSERTs, or sLog entries for DELETE/UPDATE). + * The postcommit/postabort callbacks will handle cleanup when the + * prepared transaction is eventually resolved. + */ +void +flux_twophase_recover(FullTransactionId fxid, uint16 info, + void *recdata, uint32 len) +{ + /* Nothing to do during recovery -- state is already consistent on disk */ +} diff --git a/src/backend/access/flux/flux_pvs.c b/src/backend/access/flux/flux_pvs.c new file mode 100644 index 0000000000000..7657bf54114ad --- /dev/null +++ b/src/backend/access/flux/flux_pvs.c @@ -0,0 +1,208 @@ +/*------------------------------------------------------------------------- + * + * flux_pvs.c + * FLUX per-relation versioned storage (PVS) read path + * + * In-place UPDATEs on a FLUX tuple stamp a trailing RelUndoRecPtr (verptr) + * into the new on-page image (WS-PVS1). Each verptr points to the UNDO-fork + * record that describes the update which produced its host image; reversing + * the diff (or full-tuple) in that record yields the immediately prior + * committed image, whose own trailing verptr -- preserved verbatim through + * the reverse-apply -- continues the chain one further step back. + * + * FluxReconstructVisibleVersion walks that chain on behalf of an MVCC + * reader, stepping back one version at a time until it finds the image + * created by an xid that is VISIBLE to the reader's snapshot (i.e. one + * whose urec_xid is NOT in the in-progress set). The result is the + * before-image the reader should see in place of the on-page (newer) data. + * + * urec_prevundorec is NOT used here: that field threads a per-relation, + * per-transaction rollback LIFO, not a per-tuple version chain. The + * authoritative per-tuple chain is the verptr threaded through reconstructed + * images. + * + * Visibility per step is authoritative on the core MVCC snapshot: + * XidInMVCCSnapshot(urec_xid, snapshot) == true -> updater invisible, + * step back + * XidInMVCCSnapshot(urec_xid, snapshot) == false -> updater visible, + * serve current candidate + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/access/flux/flux_pvs.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/flux.h" +#include "access/relundo.h" +#include "utils/snapmgr.h" + +/* + * Safety bound on how deep the version chain we will walk. In a healthy + * system the chain is bounded by the number of UPDATEs visible to the oldest + * snapshot retaining UNDO records; this cap defends against a corrupted + * record that would otherwise loop forever. + */ +#define FLUX_PVS_MAX_CHAIN_DEPTH 10000 + +/* + * FluxReconstructVisibleVersion + * Walk the UNDO-fork version chain and reconstruct the tuple version + * that satisfies the reader's MVCC snapshot. + * + * Inputs: + * rel - target relation (used to read its UNDO fork) + * tid - on-page TID of the row (currently unused but reserved + * for future diagnostics) + * onpage_image - pointer to the on-page tuple bytes (read-only) + * onpage_len - length of onpage_image (ItemIdGetLength of the slot) + * snapshot - MVCC snapshot of the reader + * + * Outputs (only populated when the function returns true): + * out_data - palloc'd buffer holding the reconstructed image + * out_len - length of *out_data + * + * Returns true if a different-from-on-page version should be served and + * *out_data / *out_len have been populated. Returns false when the on-page + * value is what the reader should see (caller must NOT free *out_data in + * that case; it is left untouched). + * + * If the chain is incomplete (record discarded, malformed, or terminates + * before a visible version is found), the function returns the deepest + * reconstructed image it could build, mirroring the "best-effort" semantics + * of the legacy sLog before-image path. If no reconstruction was performed + * (i.e. the on-page image's own verptr is invalid or the very first record + * read failed), the function returns false and the caller serves on-page + * data unchanged. + */ +bool +FluxReconstructVisibleVersion(Relation rel, ItemPointer tid, + const char *onpage_image, Size onpage_len, + Snapshot snapshot, + char **out_data, int *out_len) +{ + const char *candidate = onpage_image; + Size candidate_len = onpage_len; + char *allocated = NULL; + int depth = 0; + + (void) tid; /* reserved for future diagnostics */ + + if (snapshot == NULL || onpage_image == NULL || onpage_len == 0) + return false; + + for (;;) + { + const FluxTupleHeader *hdr = (const FluxTupleHeader *) candidate; + RelUndoRecPtr verptr; + RelUndoRecordHeader urec_hdr; + void *payload = NULL; + Size payload_size = 0; + char *next_image = NULL; + Size next_len = 0; + bool stepped = false; + + if (depth++ > FLUX_PVS_MAX_CHAIN_DEPTH) + { + elog(WARNING, + "FLUX PVS: version chain at (%u,%u) of relation %u exceeds " + "depth cap %d; serving best-effort image", + ItemPointerGetBlockNumber(tid), + ItemPointerGetOffsetNumber(tid), + RelationGetRelid(rel), + FLUX_PVS_MAX_CHAIN_DEPTH); + break; + } + + verptr = FluxTupleGetVersionPtr(hdr, candidate_len); + if (!RelUndoRecPtrIsValid(verptr)) + break; /* no further history */ + + if (!RelUndoReadRecord(rel, verptr, &urec_hdr, &payload, &payload_size)) + break; /* discarded or unreadable */ + + /* + * urec_xid is the xid that produced the CURRENT candidate image. If + * it is visible to the reader, the candidate is what we should serve. + */ + if (!XidInMVCCSnapshot(urec_hdr.urec_xid, snapshot)) + { + if (payload != NULL) + pfree(payload); + break; + } + + /* + * Updater invisible to reader: reverse-apply the record to obtain the + * prior committed image and continue. + */ + switch (urec_hdr.urec_type) + { + case RELUNDO_UPDATE: + { + if (!(urec_hdr.info_flags & RELUNDO_INFO_HAS_TUPLE) || + urec_hdr.tuple_len == 0) + { + elog(WARNING, + "FLUX PVS: RELUNDO_UPDATE without tuple at %llu", + (unsigned long long) verptr); + break; + } + if (payload_size < urec_hdr.tuple_len) + { + elog(WARNING, + "FLUX PVS: RELUNDO_UPDATE payload (%zu) smaller " + "than tuple_len (%u) at %llu", + payload_size, urec_hdr.tuple_len, + (unsigned long long) verptr); + break; + } + + /* + * Layout written by RelUndoFinish for a full UPDATE + * record is [RelUndoUpdatePayload][old tuple bytes]; the + * old tuple occupies the trailing tuple_len bytes of the + * payload region returned by RelUndoReadRecord. + */ + next_len = urec_hdr.tuple_len; + next_image = (char *) palloc(next_len); + memcpy(next_image, + (const char *) payload + + (payload_size - next_len), + next_len); + stepped = true; + break; + } + + default: + elog(DEBUG2, + "FLUX PVS: cannot step past urec_type %u at %llu", + urec_hdr.urec_type, (unsigned long long) verptr); + break; + } + + if (payload != NULL) + pfree(payload); + + if (!stepped) + break; /* serve the best-effort candidate we have */ + + /* Replace the candidate with the reconstructed prior image. */ + if (allocated != NULL) + pfree(allocated); + allocated = next_image; + candidate = allocated; + candidate_len = next_len; + } + + if (allocated == NULL) + return false; /* no reconstruction; caller serves on-page */ + + *out_data = allocated; + *out_len = (int) candidate_len; + return true; +} diff --git a/src/backend/access/flux/flux_relundo.c b/src/backend/access/flux/flux_relundo.c new file mode 100644 index 0000000000000..d3bbef53c4d26 --- /dev/null +++ b/src/backend/access/flux/flux_relundo.c @@ -0,0 +1,100 @@ +/*------------------------------------------------------------------------- + * + * flux_relundo.c + * FLUX adapters for the AM-neutral per-relation UNDO hooks + * + * The per-relation UNDO core (src/backend/access/undo/relundo*.c) applies + * UNDO chains and cleans up retained before-images without any compile-time + * knowledge of the FLUX access method. Where it needs FLUX-specific + * behavior -- clearing FLUX transient tuple flags, reversing a FLUX + * byte-diff, or cleaning up + * FLUX's sLog bookkeeping after an abort/discard -- it calls through + * function pointers declared in access/relundo.h. + * + * This file provides the FLUX implementations of those hooks and installs + * them. FluxRelUndoInstallHooks() is invoked from FluxUndoRmgrInit() so + * the pointers are live before crash recovery replays any RELUNDO CLR. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/access/flux/flux_relundo.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/flux.h" +#include "access/flux_undo.h" +#include "access/relundo.h" +#include "access/slog.h" +#include "access/xact.h" + +/* + * FluxRelUndoClearTransientFlags + * Clear the transient flags on a restored FLUX tuple. + * + * The before-image restored during UNDO is the committed version, so the + * UNCOMMITTED/DELETED/UPDATED markers left by the rolled-back operation must + * be cleared. + */ +static void +FluxRelUndoClearTransientFlags(char *tuple_data) +{ + FluxTupleHeader *hdr = (FluxTupleHeader *) tuple_data; + + hdr->t_flags &= ~(FLUX_TUPLE_UNCOMMITTED | + FLUX_TUPLE_DELETED | + FLUX_TUPLE_UPDATED); +} + +/* + * FluxRelUndoAbortCleanup + * Drop sLog dirty-xid markers for a transaction whose before-images + * have just been physically restored by rollback. + * + * At abort time, sLog entries for this xid were marked ABORTED (not + * removed) so visibility checks could keep treating the tuples as live + * until the physical restore completed. Now that the restore is done, the + * markers can be dropped. Called by the AM-neutral UNDO core (inline from + * xactundo.c, or from the background worker in relundo_worker.c); FLUX is + * the only in-place AM that keeps this kind of transient bookkeeping, so the + * core reaches it through this hook rather than a compile-time dependency. + */ +static void +FluxRelUndoAbortCleanup(TransactionId xid) +{ + SLogTupleRemoveByXidGlobal(xid); +} + +/* + * FluxRelUndoDiscardRetained + * Reclaim sLog before-image entries no longer needed by any active + * snapshot. + * + * Called periodically by the UNDO discard worker (undoworker.c). The + * reclamation horizon is the xid horizon computed inside + * SLogTupleCleanupRetained(). + */ +static void +FluxRelUndoDiscardRetained(void) +{ + SLogTupleCleanupRetained(); +} + +/* + * FluxRelUndoInstallHooks + * Wire the FLUX implementations into the AM-neutral UNDO core. + * + * Called from FluxUndoRmgrInit() at postmaster startup, before crash + * recovery can replay any RELUNDO CLR that would invoke these hooks. + */ +void +FluxRelUndoInstallHooks(void) +{ + RelUndoClearTransientFlags_hook = FluxRelUndoClearTransientFlags; + RelUndoAbortCleanup_hook = FluxRelUndoAbortCleanup; + RelUndoDiscardRetained_hook = FluxRelUndoDiscardRetained; + TableAMPrepare_hook = AtPrepare_Flux; +} diff --git a/src/backend/access/flux/flux_slot.c b/src/backend/access/flux/flux_slot.c new file mode 100644 index 0000000000000..55242e92d6757 --- /dev/null +++ b/src/backend/access/flux/flux_slot.c @@ -0,0 +1,796 @@ +/*------------------------------------------------------------------------- + * + * flux_slot.c + * FLUX-specific TupleTableSlot implementation + * + * This implements custom TupleTableSlotOps for FLUX table access method. + * FLUX tuples use timestamps for MVCC instead of transaction IDs, and + * have a different on-disk format than heap tuples. This slot type handles + * the FLUX tuple format natively, avoiding unnecessary conversions + * through the heap tuple format. + * + * The slot can hold either: + * - A reference to a FLUX tuple in a buffer page (pinned buffer) + * - A materialized (palloc'd) copy of a FLUX tuple + * - Virtual data in tts_values/tts_isnull (after deforming or direct store) + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/access/flux/flux_slot.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/flux.h" +#include "access/slog.h" +#include "access/htup_details.h" +#include "access/tupdesc.h" +#include "access/xact.h" +#include "executor/tuptable.h" +#include "storage/bufmgr.h" +#include "utils/expandeddatum.h" +#include "utils/memutils.h" + +/* + * FluxTupleTableSlot - slot type for FLUX tuples + * + * This extends the base TupleTableSlot with FLUX-specific fields to + * hold a reference to a FLUX tuple either in a buffer or materialized + * in memory. + */ +typedef struct FluxTupleTableSlot +{ + TupleTableSlot base; + + /* Pointer to the FLUX tuple header (in buffer or materialized) */ + FluxTupleHeader *tuple; + + /* Length of the tuple data pointed to by 'tuple' */ + uint32 tuple_len; + + /* + * Values-only ("virtual") payload block. When copyslot deep-copies a + * source slot's datums instead of forming a physical tuple, the + * pass-by-reference values are packed into this single palloc'd block so + * that clear() can free them with one pfree (mirrors + * VirtualTupleTableSlot's ->data). NULL when the slot holds a physical + * tuple or has no pass-by-ref values. + */ + void *values_block; + + /* Deform state: offset into tuple data for lazy attribute extraction */ + uint32 off; + + /* + * If buffer is not InvalidBuffer, the slot holds a pin on this buffer and + * 'tuple' points into the buffer page. When the slot is cleared or + * materialized, the pin is released. + */ + Buffer buffer; +} FluxTupleTableSlot; + +/* Forward declarations */ +const TupleTableSlotOps TTSOpsFluxTuple; +static void tts_flux_deform(TupleTableSlot *slot, int natts); + + +/* + * Initialization - nothing special needed. + */ +static void +tts_flux_init(TupleTableSlot *slot) +{ + FluxTupleTableSlot *rslot = (FluxTupleTableSlot *) slot; + + rslot->tuple = NULL; + rslot->tuple_len = 0; + rslot->off = 0; + rslot->buffer = InvalidBuffer; + rslot->values_block = NULL; +} + +/* + * Destruction - release any resources. + */ +static void +tts_flux_release(TupleTableSlot *slot) +{ + FluxTupleTableSlot *rslot = (FluxTupleTableSlot *) slot; + + /* If we own a materialized tuple, free it */ + if (TTS_SHOULDFREE(slot) && rslot->tuple) + { + pfree(rslot->tuple); + rslot->tuple = NULL; + } + if (TTS_SHOULDFREE(slot) && rslot->values_block) + { + pfree(rslot->values_block); + rslot->values_block = NULL; + } + + /* Release buffer pin if held */ + if (BufferIsValid(rslot->buffer)) + { + ReleaseBuffer(rslot->buffer); + rslot->buffer = InvalidBuffer; + } +} + +/* + * Clear the slot contents. + * + * Free materialized tuple if owned, release buffer pin, and reset + * the slot to empty state. + */ +static void +tts_flux_clear(TupleTableSlot *slot) +{ + FluxTupleTableSlot *rslot = (FluxTupleTableSlot *) slot; + + /* + * Free materialized tuple data if we own it. A tuple residing in a buffer + * cannot be freed directly; only materialized copies can. + */ + if (TTS_SHOULDFREE(slot)) + { + Assert(!BufferIsValid(rslot->buffer)); + + if (rslot->tuple) + pfree(rslot->tuple); + if (rslot->values_block) + pfree(rslot->values_block); + + slot->tts_flags &= ~TTS_FLAG_SHOULDFREE; + } + + /* Release buffer pin if held */ + if (BufferIsValid(rslot->buffer)) + { + ReleaseBuffer(rslot->buffer); + rslot->buffer = InvalidBuffer; + } + + slot->tts_nvalid = 0; + slot->tts_flags |= TTS_FLAG_EMPTY; + ItemPointerSetInvalid(&slot->tts_tid); + rslot->tuple = NULL; + rslot->tuple_len = 0; + rslot->off = 0; + rslot->values_block = NULL; +} + +/* + * Deform FLUX tuple to extract attributes into tts_values/tts_isnull. + * + * This is the FLUX-native equivalent of slot_deform_heap_tuple. It reads + * the FLUX tuple format directly (bitmap + inline attribute data) rather + * than going through the heap tuple deforming path. + */ +static void +tts_flux_deform(TupleTableSlot *slot, int natts) +{ + FluxTupleTableSlot *rslot = (FluxTupleTableSlot *) slot; + TupleDesc tupdesc = slot->tts_tupleDescriptor; + FluxTupleHeader *header = rslot->tuple; + int attnum; + char *data_ptr; + uint8 *nulls_bitmap; + Size bitmap_len; + bool has_nulls; + bool tuple_has_compressed; + + Assert(header != NULL); + Assert(natts <= tupdesc->natts); + + /* Start from where we left off last time */ + attnum = slot->tts_nvalid; + if (attnum >= natts) + return; + + /* + * Use the tuple's actual natts for bitmap_len and data_ptr calculation, + * not the tupdesc's natts. After ALTER TABLE ADD COLUMN, old tuples may + * have fewer attributes than the current schema expects. + */ + { + int tuple_natts = header->t_natts; + + bitmap_len = BITMAPLEN(tuple_natts); + nulls_bitmap = (uint8 *) header->t_attrs_bitmap; + has_nulls = (header->t_infomask & FLUX_INFOMASK_HASNULL) != 0; + tuple_has_compressed = false; /* FLUX does not compress attributes */ + (void) tuple_has_compressed; + + /* + * If this is the first time deforming (attnum == 0), start from the + * beginning of the data area. Otherwise, resume from saved offset. + */ + if (attnum == 0) + data_ptr = (char *) header + FLUX_TUPLE_OVERHEAD + MAXALIGN(bitmap_len); + else + data_ptr = (char *) header + rslot->off; + + /* + * Limit deformation to the attributes physically present in the + * tuple. Attributes beyond tuple_natts were added by ALTER TABLE ADD + * COLUMN and will be filled with their defaults below. + */ + natts = Min(natts, tuple_natts); + } + + for (; attnum < natts; attnum++) + { + Form_pg_attribute att = TupleDescAttr(tupdesc, attnum); + + if (att->attisdropped) + { + slot->tts_values[attnum] = (Datum) 0; + slot->tts_isnull[attnum] = true; + continue; + } + + /* Check null bitmap */ + if (has_nulls && att_isnull(attnum, nulls_bitmap)) + { + slot->tts_values[attnum] = (Datum) 0; + slot->tts_isnull[attnum] = true; + continue; + } + + slot->tts_isnull[attnum] = false; + + if (att->attlen > 0) + { + /* Fixed-length attribute - align first */ + data_ptr = (char *) att_align_nominal(data_ptr, att->attalign); + slot->tts_values[attnum] = fetchatt(att, data_ptr); + data_ptr += att->attlen; + } + else if (att->attlen == -1) + { + Size attr_len; + + /* Variable-length attribute - align first */ + data_ptr = (char *) att_align_nominal(data_ptr, att->attalign); + attr_len = VARSIZE_ANY(data_ptr); + + /* + * FLUX stores varlena values verbatim (wide values are TOASTed + * through the standard heap TOAST path; FLUX has no on-page + * overflow and does not compress attributes). + */ + slot->tts_values[attnum] = PointerGetDatum(data_ptr); + data_ptr += attr_len; + } + else if (att->attlen == -2) + { + /* C string */ + data_ptr = (char *) att_align_nominal(data_ptr, att->attalign); + slot->tts_values[attnum] = CStringGetDatum(data_ptr); + data_ptr += strlen(data_ptr) + 1; + } + else + { + elog(ERROR, "unsupported attribute length: %d", att->attlen); + } + } + + /* Save deform state for incremental deforming */ + rslot->off = (uint32) (data_ptr - (char *) header); + slot->tts_nvalid = natts; +} + +/* + * Fill up first natts entries of tts_values and tts_isnull. + * + * If the slot has a FLUX tuple, deform it natively. If values were already + * stored directly (virtual-style), they are already present. + */ +static void +tts_flux_getsomeattrs(TupleTableSlot *slot, int natts) +{ + FluxTupleTableSlot *rslot = (FluxTupleTableSlot *) slot; + + Assert(!TTS_EMPTY(slot)); + + if (rslot->tuple != NULL) + { + /* Deform from the FLUX tuple */ + tts_flux_deform(slot, natts); + + /* + * If the tuple had fewer attributes than requested (e.g., after ALTER + * TABLE ADD COLUMN), fill in defaults for the missing ones. + */ + if (slot->tts_nvalid < natts) + { + slot_getmissingattrs(slot, slot->tts_nvalid, natts); + slot->tts_nvalid = natts; + } + } + else + { + /* + * No physical tuple - values were stored directly into tts_values + * (virtual-style). Fill missing attributes. + */ + slot_getmissingattrs(slot, slot->tts_nvalid, natts); + slot->tts_nvalid = natts; + } +} + +/* + * Return system attribute value for FLUX tuples. + * + * FLUX tuples have timestamps instead of XIDs, so most heap system columns + * are not directly applicable. We handle the subset that makes sense. + */ +static Datum +tts_flux_getsysattr(TupleTableSlot *slot, int attnum, bool *isnull) +{ + FluxTupleTableSlot *rslot = (FluxTupleTableSlot *) slot; + + Assert(!TTS_EMPTY(slot)); + + /* If no physical tuple, we cannot provide system attributes */ + if (!rslot->tuple) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot retrieve a system column in this context"))); + + /* + * Return the tuple's real MVCC system columns. FLUX uses heap-shaped + * xmin/xmax semantics: t_xmin is the inserter XID (always valid), and the + * xmax (deleter/updater XID) lives in the low 32 bits of t_commit_ts, + * accessed via FluxTupleGetXmax (0 == not deleted/updated). + */ + *isnull = false; + + switch (attnum) + { + case MinTransactionIdAttributeNumber: /* xmin */ + return TransactionIdGetDatum(rslot->tuple->t_xmin); + case MaxTransactionIdAttributeNumber: /* xmax */ + return TransactionIdGetDatum(FluxTupleGetXmax(rslot->tuple)); + case MinCommandIdAttributeNumber: /* cmin */ + case MaxCommandIdAttributeNumber: /* cmax */ + { + /* + * t_cid removed from FluxTupleHeader (saves 4 bytes). Look up + * the command ID from the sLog for in-progress operations; + * return InvalidCommandId if no sLog entry exists (committed + * tuple). + */ + SLogTupleOp slog_entry; + int nfound = SLogTupleLookupFiltered(slot->tts_tableOid, + &slot->tts_tid, + GetTopTransactionIdIfAny(), + &slog_entry, 1); + + return CommandIdGetDatum(nfound > 0 ? slog_entry.cid : InvalidCommandId); + } + default: + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("FLUX does not support system attribute %d", + attnum))); + return 0; /* silence compiler */ + } +} + +/* + * Check if the tuple was created by the current transaction. + * + * For FLUX, we consult the sLog to determine whether the current + * transaction inserted this tuple. This replaces the old t_xact_ts + * comparison that was removed in the sLog migration. + */ +static bool +tts_flux_is_current_xact_tuple(TupleTableSlot *slot) +{ + Assert(!TTS_EMPTY(slot)); + + if (!ItemPointerIsValid(&slot->tts_tid) || + slot->tts_tableOid == InvalidOid) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("don't have a storage tuple in this context"))); + + /* + * Ask the sLog whether the current transaction inserted this tuple. This + * is the FLUX equivalent of checking xmin == current xid. + */ + return SLogTupleIsInsertedByMe(slot->tts_tableOid, &slot->tts_tid); +} + +/* + * Pack a slot's pass-by-reference Datums into a single palloc'd block owned by + * the slot, leaving the slot in a values-only ("virtual") state (tuple == NULL). + * + * This mirrors tts_virtual_materialize: a two-pass scan that first sums the + * aligned size of every non-null pass-by-reference attribute, makes one + * allocation in the slot's memory context, then copies each Datum in and + * repoints tts_values at the copy. byval and null Datums are left untouched. + * + * We deliberately do NOT form a physical FLUX tuple here. Forming one would + * compress every varlena, and the immediate next consumer (flux_multi_insert / + * flux_tuple_insert via slot_getallattrs) would decompress it again before + * re-forming the on-page tuple. Deferring tuple formation to the insert path + * makes bulk COPY compress each value exactly once instead of three times. + * + * The caller must have already populated tts_values/tts_isnull and set + * tts_nvalid to the attribute count; those stay valid because getsomeattrs and + * getsysattr handle tuple == NULL by working from tts_values. + */ +static void +tts_flux_materialize_values(TupleTableSlot *slot) +{ + FluxTupleTableSlot *rslot = (FluxTupleTableSlot *) slot; + TupleDesc desc = slot->tts_tupleDescriptor; + Size sz = 0; + char *data; + + /* First pass: compute the aligned size of the owned block. */ + for (int natt = 0; natt < desc->natts; natt++) + { + CompactAttribute *att = TupleDescCompactAttr(desc, natt); + Datum val; + + if (att->attbyval || slot->tts_isnull[natt]) + continue; + + val = slot->tts_values[natt]; + + if (att->attlen == -1 && + VARATT_IS_EXTERNAL_EXPANDED(DatumGetPointer(val))) + { + sz = att_nominal_alignby(sz, att->attalignby); + sz += EOH_get_flat_size(DatumGetEOHP(val)); + } + else + { + sz = att_nominal_alignby(sz, att->attalignby); + sz = att_addlength_datum(sz, att->attlen, val); + } + } + + rslot->tuple = NULL; + rslot->tuple_len = 0; + rslot->off = 0; + + /* all data is byval / null: nothing to own */ + if (sz == 0) + return; + + rslot->values_block = data = MemoryContextAlloc(slot->tts_mcxt, sz); + slot->tts_flags |= TTS_FLAG_SHOULDFREE; + + /* Second pass: copy each pass-by-reference Datum and repoint tts_values. */ + for (int natt = 0; natt < desc->natts; natt++) + { + CompactAttribute *att = TupleDescCompactAttr(desc, natt); + Datum val; + + if (att->attbyval || slot->tts_isnull[natt]) + continue; + + val = slot->tts_values[natt]; + + if (att->attlen == -1 && + VARATT_IS_EXTERNAL_EXPANDED(DatumGetPointer(val))) + { + ExpandedObjectHeader *eoh = DatumGetEOHP(val); + Size data_length = EOH_get_flat_size(eoh); + + data = (char *) att_nominal_alignby(data, att->attalignby); + EOH_flatten_into(eoh, data, data_length); + + slot->tts_values[natt] = PointerGetDatum(data); + data += data_length; + } + else + { + Size data_length = 0; + + data = (char *) att_nominal_alignby(data, att->attalignby); + data_length = att_addlength_datum(data_length, att->attlen, val); + + memcpy(data, DatumGetPointer(val), data_length); + + slot->tts_values[natt] = PointerGetDatum(data); + data += data_length; + } + } +} + +/* + * Materialize the slot contents. + * + * After materialization, the slot's data is independent of any external + * storage (buffers, other memory contexts). If the slot references a + * tuple in a buffer, the tuple data is copied and the buffer pin released. + */ +static void +tts_flux_materialize(TupleTableSlot *slot) +{ + FluxTupleTableSlot *rslot = (FluxTupleTableSlot *) slot; + MemoryContext oldContext; + + Assert(!TTS_EMPTY(slot)); + + /* Already materialized */ + if (TTS_SHOULDFREE(slot)) + return; + + oldContext = MemoryContextSwitchTo(slot->tts_mcxt); + + if (rslot->tuple != NULL) + { + /* + * We have a physical FLUX tuple (in a buffer or external memory). + * Copy it into the slot's own memory context. + */ + FluxTupleHeader *newtuple; + + newtuple = (FluxTupleHeader *) palloc(rslot->tuple_len); + memcpy(newtuple, rslot->tuple, rslot->tuple_len); + rslot->tuple = newtuple; + + /* + * Reset deform state since tts_values entries may point into the old + * (buffer) tuple data that we're about to release. + */ + rslot->off = 0; + slot->tts_nvalid = 0; + } + else + { + /* + * Virtual tuple (values stored directly). Materialize by copying all + * pass-by-reference Datums into a single block in the slot's memory + * context and leaving the slot in a values-only state (tuple == + * NULL). We deliberately do NOT form a physical FLUX tuple: that + * would compress every varlena, and the insert path + * (flux_multi_insert via slot_getallattrs) would decompress and + * recompress it. Deferring tuple formation to insert compresses each + * value exactly once. + */ + tts_flux_materialize_values(slot); + + /* + * tts_flux_materialize_values sets SHOULDFREE itself when it owns a + * block; return early so we don't set SHOULDFREE unconditionally for + * a slot that owns nothing (all-byval/null). We still must release + * any buffer pin first. + */ + if (BufferIsValid(rslot->buffer)) + { + ReleaseBuffer(rslot->buffer); + rslot->buffer = InvalidBuffer; + } + MemoryContextSwitchTo(oldContext); + return; + } + + /* + * Release buffer pin if held. Do this after copying but before setting + * TTS_FLAG_SHOULDFREE to avoid a transient state where the slot owns a + * buffer and has SHOULDFREE set. + */ + if (BufferIsValid(rslot->buffer)) + { + ReleaseBuffer(rslot->buffer); + rslot->buffer = InvalidBuffer; + } + + slot->tts_flags |= TTS_FLAG_SHOULDFREE; + + MemoryContextSwitchTo(oldContext); +} + +/* + * Copy the contents of srcslot into dstslot. + * + * The destination must not depend on the source slot's memory after this + * returns. We satisfy that by deep-copying the source's attribute values + * into the destination's own memory context and leaving the destination in + * a values-only ("virtual") state -- rdst->tuple stays NULL. We deliberately + * do NOT form a physical FLUX tuple here: forming one would compress every + * varlena, and the immediate next consumer (flux_multi_insert / + * flux_tuple_insert via slot_getallattrs) would have to decompress it again + * before re-forming the on-page tuple. Deferring tuple formation to the + * insert path makes bulk COPY compress each value exactly once instead of + * three times (compress here, decompress there, recompress there). + * + * The values-only state is fully supported by the other slot ops: + * getsomeattrs and materialize both handle rslot->tuple == NULL by working + * from tts_values, and copy_heap_tuple/copy_minimal_tuple go through + * slot_getallattrs. + */ +static void +tts_flux_copyslot(TupleTableSlot *dstslot, TupleTableSlot *srcslot) +{ + TupleDesc desc = dstslot->tts_tupleDescriptor; + + tts_flux_clear(dstslot); + + slot_getallattrs(srcslot); + + /* Copy the datum pointers first; they still reference source memory. */ + for (int natt = 0; natt < desc->natts; natt++) + { + dstslot->tts_values[natt] = srcslot->tts_values[natt]; + dstslot->tts_isnull[natt] = srcslot->tts_isnull[natt]; + } + dstslot->tts_nvalid = desc->natts; + dstslot->tts_flags &= ~TTS_FLAG_EMPTY; + dstslot->tts_tid = srcslot->tts_tid; + + /* + * Deep-copy the pass-by-reference datums into a single owned block so the + * destination no longer depends on the source's memory (in COPY, the + * source is a per-row scratch slot that gets reset). This leaves the + * destination in a values-only state (tuple == NULL), deferring + * compression to the insert path. + */ + tts_flux_materialize_values(dstslot); +} + +/* + * Return a HeapTuple "owned" by the slot. + * + * Since FLUX tuples are not heap tuples, we must form one from the + * deformed values. The result is a palloc'd HeapTuple that the slot owns. + * + * This is needed by parts of the executor that require heap tuples + * (e.g., for index tuple formation, triggers, etc.). + */ +static HeapTuple +tts_flux_copy_heap_tuple(TupleTableSlot *slot) +{ + HeapTuple htup; + + Assert(!TTS_EMPTY(slot)); + + /* Ensure all attributes are deformed */ + slot_getallattrs(slot); + + htup = heap_form_tuple(slot->tts_tupleDescriptor, + slot->tts_values, + slot->tts_isnull); + + /* + * Propagate TID and table OID from the slot to the HeapTuple. ANALYZE's + * compare_rows() sorts sample tuples by t_self (TID), which + * heap_form_tuple leaves zeroed. Without this, the ItemPointerIsValid + * assertion in ItemPointerGetBlockNumber fires. + */ + htup->t_self = slot->tts_tid; + htup->t_tableOid = slot->tts_tableOid; + + return htup; +} + +/* + * Return a MinimalTuple copy allocated in the caller's memory context. + */ +static MinimalTuple +tts_flux_copy_minimal_tuple(TupleTableSlot *slot, Size extra) +{ + Assert(!TTS_EMPTY(slot)); + + /* Ensure all attributes are deformed */ + slot_getallattrs(slot); + + return heap_form_minimal_tuple(slot->tts_tupleDescriptor, + slot->tts_values, + slot->tts_isnull, + extra); +} + +/* + * The FLUX TupleTableSlotOps structure. + * + * FLUX slots do not "own" heap tuples or minimal tuples natively, so + * get_heap_tuple and get_minimal_tuple are NULL. The copy_ variants are + * provided to satisfy the executor's needs. + */ +const TupleTableSlotOps TTSOpsFluxTuple = { + .base_slot_size = sizeof(FluxTupleTableSlot), + .init = tts_flux_init, + .release = tts_flux_release, + .clear = tts_flux_clear, + .getsomeattrs = tts_flux_getsomeattrs, + .getsysattr = tts_flux_getsysattr, + .is_current_xact_tuple = tts_flux_is_current_xact_tuple, + .materialize = tts_flux_materialize, + .copyslot = tts_flux_copyslot, + + /* FLUX slots do not natively own heap or minimal tuples */ + .get_heap_tuple = NULL, + .get_minimal_tuple = NULL, + .copy_heap_tuple = tts_flux_copy_heap_tuple, + .copy_minimal_tuple = tts_flux_copy_minimal_tuple, +}; + + +/* + * Store a FLUX tuple from a buffer page into the slot. + * + * The tuple data remains in the buffer; a pin is acquired to keep the + * buffer valid for the lifetime of the slot reference. + * + * This is the primary way scan routines populate FLUX slots. + */ +void +FluxSlotStoreTuple(TupleTableSlot *slot, FluxTupleHeader *tuple, + uint32 tuple_len, Buffer buffer) +{ + FluxTupleTableSlot *rslot = (FluxTupleTableSlot *) slot; + + Assert(slot->tts_ops == &TTSOpsFluxTuple); + + /* + * Optimize for the common case during sequential scans: if the new tuple + * is on the same buffer as the previous one, skip the expensive + * ReleaseBuffer + IncrBufferRefCount cycle. This mirrors the + * optimization in heap's tts_buffer_heap_store_tuple(). + */ + if (rslot->buffer == buffer) + { + /* Same buffer ā€” just free any materialized data */ + if (unlikely(TTS_SHOULDFREE(slot))) + { + if (rslot->tuple) + pfree(rslot->tuple); + slot->tts_flags &= ~TTS_FLAG_SHOULDFREE; + } + } + else + { + /* Different buffer ā€” full clear (releases old pin) and acquire new */ + tts_flux_clear(slot); + rslot->buffer = buffer; + + if (BufferIsValid(buffer)) + IncrBufferRefCount(buffer); + } + + rslot->tuple = tuple; + rslot->tuple_len = tuple_len; + rslot->off = 0; + + slot->tts_flags &= ~TTS_FLAG_EMPTY; + slot->tts_nvalid = 0; +} + +/* + * Store a materialized (palloc'd) FLUX tuple into the slot. + * + * The slot takes ownership of the tuple data and will pfree it when + * cleared or released. + */ +void +FluxSlotStoreMaterializedTuple(TupleTableSlot *slot, + FluxTupleHeader *tuple, + uint32 tuple_len) +{ + FluxTupleTableSlot *rslot = (FluxTupleTableSlot *) slot; + + Assert(slot->tts_ops == &TTSOpsFluxTuple); + + tts_flux_clear(slot); + + rslot->tuple = tuple; + rslot->tuple_len = tuple_len; + rslot->off = 0; + rslot->buffer = InvalidBuffer; + + slot->tts_flags &= ~TTS_FLAG_EMPTY; + slot->tts_flags |= TTS_FLAG_SHOULDFREE; + slot->tts_nvalid = 0; +} diff --git a/src/backend/access/flux/flux_stats.c b/src/backend/access/flux/flux_stats.c new file mode 100644 index 0000000000000..92b55b3259ab6 --- /dev/null +++ b/src/backend/access/flux/flux_stats.c @@ -0,0 +1,286 @@ +/*------------------------------------------------------------------------- + * + * flux_stats.c + * FLUX-specific statistics collection for ANALYZE + * + * This module collects statistics that are unique to the FLUX storage + * format: compression ratios, overflow usage, space efficiency, and the + * distribution of the per-tuple commit-ts word. These statistics supplement + * the standard + * per-column statistics (MCV, histograms, NULL fractions, etc.) that + * PostgreSQL's ANALYZE framework collects automatically via the + * scan_analyze_next_block / scan_analyze_next_tuple callbacks. + * + * The collected statistics are logged at DEBUG1 level and made available + * through the FluxCollectRelationStats() interface so that the planner + * can incorporate FLUX-specific cost adjustments. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/access/flux/flux_stats.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/flux.h" +#include "miscadmin.h" +#include "storage/bufmgr.h" +#include "storage/smgr.h" +#include "utils/rel.h" + +/* + * FluxCollectRelationStats + * + * Scan the relation to collect FLUX-specific statistics. This performs + * a full sequential pass over every page, examining each item to measure + * compression ratios, overflow usage, tuple sizes, free space, and the + * distribution of the per-tuple commit-ts word. + * + * This is designed to be called during ANALYZE after the standard sampling + * is complete. It does its own full scan because the standard sampling + * only visits a random subset of blocks, which is fine for per-column + * statistics but insufficient for accurate relation-wide measurements + * like total overflow bytes or bloat factor. + * + * The caller must pass a zeroed FluxRelationStats struct. + */ +void +FluxCollectRelationStats(Relation rel, FluxRelationStats *stats) +{ + BlockNumber nblocks; + BlockNumber blkno; + int64 total_tuple_bytes = 0; + int64 total_compressed_tuples = 0; + int64 total_overflow_tuples = 0; + int64 total_overflow_chains = 0; + int64 total_live = 0; + int64 total_dead = 0; + double total_free_space = 0.0; + int64 total_uncompressed_size = 0; + int64 total_compressed_size = 0; + bool commit_ts_seen = false; + uint64 commit_ts_min = PG_UINT64_MAX; + uint64 commit_ts_max = 0; + + /* Initialize output */ + memset(stats, 0, sizeof(FluxRelationStats)); + + /* Get number of blocks */ + if (!smgrexists(RelationGetSmgr(rel), MAIN_FORKNUM)) + return; + + nblocks = smgrnblocks(RelationGetSmgr(rel), MAIN_FORKNUM); + stats->total_pages = nblocks; + + if (nblocks == 0) + return; + + /* + * Scan every page. We take only a shared lock on each page and release + * it before moving to the next, keeping contention low. + */ + for (blkno = 0; blkno < nblocks; blkno++) + { + Buffer buffer; + Page page; + OffsetNumber maxoff; + OffsetNumber offnum; + Size page_free; + + CHECK_FOR_INTERRUPTS(); + + buffer = ReadBufferExtended(rel, MAIN_FORKNUM, blkno, + RBM_NORMAL, NULL); + LockBuffer(buffer, BUFFER_LOCK_SHARE); + page = BufferGetPage(buffer); + + /* Skip uninitialized pages */ + if (PageIsNew(page)) + { + UnlockReleaseBuffer(buffer); + continue; + } + + maxoff = PageGetMaxOffsetNumber(page); + page_free = PageGetFreeSpace(page); + total_free_space += (double) page_free / (double) BLCKSZ; + + for (offnum = FirstOffsetNumber; offnum <= maxoff; offnum++) + { + ItemId itemid = PageGetItemId(page, offnum); + FluxTupleHeader *hdr; + Size item_len; + + if (!ItemIdIsNormal(itemid)) + { + if (ItemIdIsDead(itemid)) + total_dead++; + continue; + } + + item_len = ItemIdGetLength(itemid); + hdr = (FluxTupleHeader *) PageGetItem(page, itemid); + + /* Skip overflow records -- counted separately */ + if (FluxIsOverflowRecordInline(hdr, item_len)) + { + total_overflow_chains++; + stats->total_overflow_bytes += item_len; + continue; + } + + /* This is a real tuple */ + total_tuple_bytes += item_len; + + if (hdr->t_flags & FLUX_TUPLE_DELETED) + { + total_dead++; + continue; + } + + /* Live tuple */ + total_live++; + + /* Check compression */ + if (hdr->t_flags & FLUX_TUPLE_COMPRESSED) + { + total_compressed_tuples++; + + /* + * Estimate compression ratio from the compression header that + * follows the tuple header, if present. + */ + if (item_len > FLUX_TUPLE_OVERHEAD + sizeof(FluxCompressionHeader)) + { + FluxCompressionHeader *comp_hdr; + + comp_hdr = (FluxCompressionHeader *) + ((char *) hdr + FLUX_TUPLE_OVERHEAD); + total_uncompressed_size += comp_hdr->orig_size; + total_compressed_size += comp_hdr->comp_size; + } + } + + /* Check overflow */ + if (hdr->t_flags & FLUX_TUPLE_HAS_OVERFLOW) + total_overflow_tuples++; + + /* Track commit-ts word range (diagnostic) */ + if (hdr->t_commit_ts > 0) + { + commit_ts_seen = true; + if (hdr->t_commit_ts < commit_ts_min) + commit_ts_min = hdr->t_commit_ts; + if (hdr->t_commit_ts > commit_ts_max) + commit_ts_max = hdr->t_commit_ts; + } + } + + UnlockReleaseBuffer(buffer); + } + + /* Compute derived statistics */ + stats->total_live_tuples = total_live; + stats->total_dead_tuples = total_dead; + + if (total_live > 0) + { + stats->avg_tuple_size = (double) total_tuple_bytes / (double) total_live; + stats->pct_compressed = (double) total_compressed_tuples / (double) total_live; + stats->pct_overflow = (double) total_overflow_tuples / (double) total_live; + } + + if (total_compressed_size > 0 && total_uncompressed_size > 0) + stats->compression_ratio = (double) total_uncompressed_size / + (double) total_compressed_size; + else + stats->compression_ratio = 1.0; + + if (total_overflow_tuples > 0) + stats->avg_overflow_chain_len = (double) total_overflow_chains / + (double) total_overflow_tuples; + + if (nblocks > 0) + { + stats->avg_live_per_page = (double) total_live / (double) nblocks; + stats->free_space_frac = total_free_space / (double) nblocks; + } + + /* Bloat = total allocated space / actual live data */ + if (total_tuple_bytes > 0) + stats->bloat_factor = ((double) nblocks * BLCKSZ) / + (double) total_tuple_bytes; + else + stats->bloat_factor = 1.0; + + /* commit-ts word range */ + if (commit_ts_seen) + { + stats->commit_ts_stats_valid = true; + stats->commit_ts_min = commit_ts_min; + stats->commit_ts_max = commit_ts_max; + } +} + +/* + * FluxLogRelationStats + * + * Emit the collected FLUX statistics at the given log level (typically + * DEBUG1 during ANALYZE, or LOG for diagnostic purposes). Produces three + * separate ereport messages: + * 1. Page counts and live/dead tuple totals + * 2. Average tuple size, compression percentage/ratio, overflow stats + * 3. Average live tuples per page, free space fraction, bloat factor + * If commit-ts word statistics are valid, a fourth message shows the range. + * + * Parameters: + * rel - the relation whose statistics are being logged + * stats - the collected FluxRelationStats structure + * elevel - ereport log level (e.g., DEBUG1, LOG, WARNING) + */ +void +FluxLogRelationStats(Relation rel, const FluxRelationStats *stats, int elevel) +{ + ereport(elevel, + (errmsg("FLUX stats for \"%s\": " + "%lld pages, %lld live tuples, %lld dead tuples", + RelationGetRelationName(rel), + (long long) stats->total_pages, + (long long) stats->total_live_tuples, + (long long) stats->total_dead_tuples))); + + ereport(elevel, + (errmsg("FLUX stats for \"%s\": " + "avg tuple size %.1f bytes, " + "%.1f%% compressed (ratio %.2f), " + "%.1f%% overflow (avg chain %.1f)", + RelationGetRelationName(rel), + stats->avg_tuple_size, + stats->pct_compressed * 100.0, + stats->compression_ratio, + stats->pct_overflow * 100.0, + stats->avg_overflow_chain_len))); + + ereport(elevel, + (errmsg("FLUX stats for \"%s\": " + "avg %.1f live/page, " + "%.1f%% free space, " + "bloat factor %.2f", + RelationGetRelationName(rel), + stats->avg_live_per_page, + stats->free_space_frac * 100.0, + stats->bloat_factor))); + + if (stats->commit_ts_stats_valid) + { + ereport(elevel, + (errmsg("FLUX stats for \"%s\": " + "commit-ts word range [%llu .. %llu]", + RelationGetRelationName(rel), + (unsigned long long) stats->commit_ts_min, + (unsigned long long) stats->commit_ts_max))); + } +} diff --git a/src/backend/access/flux/flux_tuple.c b/src/backend/access/flux/flux_tuple.c new file mode 100644 index 0000000000000..f39bff905613c --- /dev/null +++ b/src/backend/access/flux/flux_tuple.c @@ -0,0 +1,1134 @@ +/*------------------------------------------------------------------------- + * + * flux_tuple.c + * FLUX tuple handling routines + * + * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/access/flux/flux_tuple.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/detoast.h" +#include "access/flux.h" +#include "access/flux_xlog.h" +#include "access/heapam.h" +#include "access/heaptoast.h" +#include "access/toast_helper.h" +#include "access/toast_internals.h" +#include "access/tupdesc.h" +#include "access/tupmacs.h" +#include "catalog/pg_type.h" +#include "common/hashfn.h" +#include "executor/tuptable.h" +#include "storage/bufpage.h" +#include "utils/datum.h" +#include "utils/lsyscache.h" +#include "utils/memutils.h" + +/* + * FluxComputeDataSize + * + * Calculate the total on-disk size needed to store a tuple with the given + * attributes. This includes the fixed-size FluxTupleHeader, the null + * bitmap, alignment padding, and all attribute data. + * + * Parameters: + * tupdesc - tuple descriptor defining the attributes + * values - array of Datum values for each attribute + * isnull - array of boolean null indicators + * + * Returns the total size in bytes, including header and alignment. + */ +Size +FluxComputeDataSize(TupleDesc tupdesc, Datum *values, bool *isnull) +{ + Size data_length = 0; + Size bitmap_len; + int i; + + Assert(tupdesc != NULL); + Assert(values != NULL); + Assert(isnull != NULL); + + /* Calculate null bitmap length */ + bitmap_len = BITMAPLEN(tupdesc->natts); + + /* Start with tuple header size */ + data_length = FLUX_TUPLE_OVERHEAD + MAXALIGN(bitmap_len); + + /* Add space for each attribute */ + for (i = 0; i < tupdesc->natts; i++) + { + Form_pg_attribute att = TupleDescAttr(tupdesc, i); + + if (att->attisdropped) + continue; + + if (!isnull[i]) + { + Size attr_len; + + /* Align attribute */ + data_length = att_align_nominal(data_length, att->attalign); + + if (att->attlen > 0) + { + /* Fixed-length attribute */ + attr_len = att->attlen; + } + else if (att->attlen == -1) + { + /* Variable-length attribute */ + attr_len = VARSIZE_ANY(DatumGetPointer(values[i])); + } + else if (att->attlen == -2) + { + /* C string */ + attr_len = strlen(DatumGetCString(values[i])) + 1; + } + else + { + elog(ERROR, "unsupported attribute length: %d", att->attlen); + } + + data_length += attr_len; + } + } + + return data_length; +} + +/* + * flux_toast_tuple + * + * TOAST the varlena columns of a to-be-stored FLUX tuple, exactly like heap. + * FLUX has no on-page overflow mechanism; wide values are pushed to the + * relation's standard heap TOAST table via the AM-agnostic toast_helper + * routines (toast_tuple_*). On return, values[]/isnull[] have had any + * externalized/compressed columns replaced so that the tuple formed from + * them fits within a FLUX page. + * + * oldvalues/oldisnull describe the previous tuple version on UPDATE (so + * unchanged external datums are reused and superseded ones scheduled for + * deletion); pass NULL for INSERT. + * + * The returned ToastTupleContext must be released with flux_toast_cleanup() + * after the caller has finished forming and storing the tuple. *changed is + * set true iff any column was actually toasted. + */ +void +flux_toast_tuple(Relation rel, Datum *values, bool *isnull, + Datum *oldvalues, bool *oldisnull, + ToastTupleContext *ttc, ToastAttrInfo *toast_attr, + bool *changed, uint32 options) +{ + TupleDesc tupleDesc = rel->rd_att; + int numAttrs = tupleDesc->natts; + Size hoff; + Size maxDataLen; + + options &= ~HEAP_INSERT_SPECULATIVE; + + ttc->ttc_rel = rel; + ttc->ttc_values = values; + ttc->ttc_isnull = isnull; + ttc->ttc_oldvalues = oldvalues; + ttc->ttc_oldisnull = oldisnull; + ttc->ttc_attr = toast_attr; + toast_tuple_init(ttc); + + /* + * Header overhead for a FLUX tuple: fixed header plus null bitmap when + * nulls are present. Convert to a data-size limit. FLUX targets the + * same TOAST_TUPLE_TARGET as heap so wide rows behave identically. + */ + hoff = FLUX_TUPLE_OVERHEAD; + if ((ttc->ttc_flags & TOAST_HAS_NULLS) != 0) + hoff += BITMAPLEN(numAttrs); + hoff = MAXALIGN(hoff); + maxDataLen = RelationGetToastTupleTarget(rel, TOAST_TUPLE_TARGET) - hoff; + + /* Round 1: compress EXTENDED, externalize very large EXTENDED/EXTERNAL */ + while (heap_compute_data_size(tupleDesc, values, isnull) > maxDataLen) + { + int biggest_attno; + + biggest_attno = toast_tuple_find_biggest_attribute(ttc, true, false); + if (biggest_attno < 0) + break; + if (TupleDescAttr(tupleDesc, biggest_attno)->attstorage == TYPSTORAGE_EXTENDED) + toast_tuple_try_compression(ttc, biggest_attno); + else + toast_attr[biggest_attno].tai_colflags |= TOASTCOL_INCOMPRESSIBLE; + if (toast_attr[biggest_attno].tai_size > maxDataLen && + rel->rd_rel->reltoastrelid != InvalidOid) + toast_tuple_externalize(ttc, biggest_attno, options); + } + + /* Round 2: externalize remaining inline EXTENDED/EXTERNAL */ + while (heap_compute_data_size(tupleDesc, values, isnull) > maxDataLen && + rel->rd_rel->reltoastrelid != InvalidOid) + { + int biggest_attno; + + biggest_attno = toast_tuple_find_biggest_attribute(ttc, false, false); + if (biggest_attno < 0) + break; + toast_tuple_externalize(ttc, biggest_attno, options); + } + + /* Round 3: compress MAIN */ + while (heap_compute_data_size(tupleDesc, values, isnull) > maxDataLen) + { + int biggest_attno; + + biggest_attno = toast_tuple_find_biggest_attribute(ttc, true, true); + if (biggest_attno < 0) + break; + toast_tuple_try_compression(ttc, biggest_attno); + } + + /* Round 4: externalize MAIN, at the larger MAIN target */ + maxDataLen = TOAST_TUPLE_TARGET_MAIN - hoff; + while (heap_compute_data_size(tupleDesc, values, isnull) > maxDataLen && + rel->rd_rel->reltoastrelid != InvalidOid) + { + int biggest_attno; + + biggest_attno = toast_tuple_find_biggest_attribute(ttc, false, true); + if (biggest_attno < 0) + break; + toast_tuple_externalize(ttc, biggest_attno, options); + } + + *changed = (ttc->ttc_flags & TOAST_NEEDS_CHANGE) != 0; +} + +/* + * flux_toast_cleanup + * + * Release toasting temporaries and delete any superseded external datums, + * mirroring toast_tuple_cleanup() as used by heap after the new tuple is + * durably stored. + */ +void +flux_toast_cleanup(ToastTupleContext *ttc) +{ + toast_tuple_cleanup(ttc); +} + +/* + * flux_toast_delete + * + * Delete the external TOAST datums referenced by a FLUX tuple that is being + * removed (DELETE, or the old version of an out-of-place UPDATE). + */ +void +flux_toast_delete(Relation rel, Datum *values, bool *isnull, + bool is_speculative) +{ + toast_delete_external(rel, values, isnull, is_speculative); +} + +/* + * FluxFormTuple + * + * Create a new FLUX tuple from the given attribute values and null indicators. + * Allocates memory for the FluxTupleData wrapper and the on-disk + * FluxTupleHeader + attribute data. + * + * When compression is enabled (flux_enable_compression GUC), variable-length + * attributes exceeding FLUX_MIN_COMPRESS_SIZE (32 bytes) are automatically + * compressed using the algorithm selected by FluxChooseCompressionType(). + * Compressed attributes are stored with a FluxCompressionHeader prefix and + * the tuple's FLUX_INFOMASK_COMPRESSED bit is set. + * + * When a relation is provided, large attributes exceeding FLUX_OVERFLOW_THRESHOLD + * are automatically stored in overflow pages. Overflow pointers are collected in + * overflow_buffers for atomic WAL logging by the caller. + * + * Parameters: + * tupdesc - tuple descriptor defining the schema + * values - array of Datum values for each attribute + * isnull - array of boolean null indicators + * rel - relation for overflow storage (NULL to disable overflow handling) + * overflow_buffers - output for overflow buffers (NULL if rel is NULL) + * + * Returns a palloc'd FluxTuple. The caller is responsible for freeing it + * with FluxFreeTuple() when done. + */ +static FluxTuple flux_form_tuple_internal(TupleDesc tupdesc, Datum *values, + bool *isnull, Relation rel, + FluxOverflowBuffers *overflow_buffers, + bool force_shrink, + const FluxOverflowPtr *old_ovptrs, + const bool *old_ovpresent); + +FluxTuple +FluxFormTuple(TupleDesc tupdesc, Datum *values, bool *isnull, + Relation rel, FluxOverflowBuffers *overflow_buffers) +{ + return flux_form_tuple_internal(tupdesc, values, isnull, rel, + overflow_buffers, false, NULL, NULL); +} + +/* + * FluxFormTupleUpdate + * + * Like FluxFormTuple, but for the in-place UPDATE path. old_ovptrs and + * old_ovpresent (indexed by attnum, natts entries) describe the OLD tuple's + * on-page overflow pointers, collected while its buffer was still locked. Any + * over-threshold varlena whose content hash matches the old pointer's stored + * hash (and byte-verifies equal) is COW-referenced against the existing + * overflow chain instead of being re-stored, avoiding needless disk growth and + * WAL. Pass NULL arrays to disable this (identical to FluxFormTuple). + */ +FluxTuple +FluxFormTupleUpdate(TupleDesc tupdesc, Datum *values, bool *isnull, + Relation rel, FluxOverflowBuffers *overflow_buffers, + const FluxOverflowPtr *old_ovptrs, + const bool *old_ovpresent) +{ + return flux_form_tuple_internal(tupdesc, values, isnull, rel, + overflow_buffers, false, + old_ovptrs, old_ovpresent); +} + +/* + * FluxFormTupleForceShrink + * + * Like FluxFormTuple, but forces every inline varlena attribute larger than + * an overflow pointer off-page with a zero inline prefix, regardless of the + * normal FLUX_OVERFLOW_THRESHOLD. This shrinks the main tuple to its minimum + * footprint (header + fixed columns + one overflow pointer per large varlena). + * + * Used as a last resort by the in-place UPDATE path: when an updated tuple has + * grown beyond the space available on its page and TID stability forbids + * relocating it, pushing its variable-length data off-page lets the main tuple + * fit back into (or near) its original slot. A relation and overflow_buffers + * are mandatory because every forced column is written to overflow pages. + */ +FluxTuple +FluxFormTupleForceShrink(TupleDesc tupdesc, Datum *values, bool *isnull, + Relation rel, FluxOverflowBuffers *overflow_buffers) +{ + Assert(rel != NULL); + Assert(overflow_buffers != NULL); + return flux_form_tuple_internal(tupdesc, values, isnull, rel, + overflow_buffers, true, NULL, NULL); +} + +static FluxTuple +flux_form_tuple_internal(TupleDesc tupdesc, Datum *values, bool *isnull, + Relation rel, FluxOverflowBuffers *overflow_buffers, + bool force_shrink, + const FluxOverflowPtr *old_ovptrs, + const bool *old_ovpresent) +{ + FluxTuple tuple; + FluxTupleHeader *header; + Size data_length; + Size tuple_length; + Size bitmap_len; + char *data_ptr; + uint8 *nulls_bitmap; + int i; + bool has_nulls = false; + bool has_varwidth = false; + bool has_external = false; + bool has_compressed = false; + bool has_overflow = false; + + /* + * Working arrays for compressed/overflowed values. We attempt compression + * and overflow first, then compute the final tuple size using the + * (possibly compressed/overflowed) attribute values. + * + * Use stack arrays for small tuples (common OLTP case) to avoid palloc. + */ +#define FLUX_FORM_STACK_ATTRS 16 + Datum *work_values; + bool *is_compressed; /* Track which attrs were compressed */ + bool *is_overflowed; /* Track which attrs were overflowed */ + Datum work_values_stack[FLUX_FORM_STACK_ATTRS]; + bool is_compressed_stack[FLUX_FORM_STACK_ATTRS]; + bool is_overflowed_stack[FLUX_FORM_STACK_ATTRS]; + + Assert(tupdesc != NULL); + Assert(values != NULL); + Assert(isnull != NULL); + + if (tupdesc->natts <= FLUX_FORM_STACK_ATTRS) + { + work_values = work_values_stack; + is_compressed = is_compressed_stack; + is_overflowed = is_overflowed_stack; + memset(is_compressed, 0, tupdesc->natts * sizeof(bool)); + memset(is_overflowed, 0, tupdesc->natts * sizeof(bool)); + } + else + { + work_values = (Datum *) palloc(tupdesc->natts * sizeof(Datum)); + is_compressed = (bool *) palloc0(tupdesc->natts * sizeof(bool)); + is_overflowed = (bool *) palloc0(tupdesc->natts * sizeof(bool)); + } + memcpy(work_values, values, tupdesc->natts * sizeof(Datum)); + + /* + * FLUX has no on-page overflow mechanism; wide varlena values are TOASTed + * by the caller (standard heap TOAST path) before reaching here. + * Likewise, FLUX does not compress attributes. work_values therefore + * mirrors values exactly. + */ + + /* + * Phase 2: Calculate total space needed + */ + data_length = FluxComputeDataSize(tupdesc, work_values, isnull); + tuple_length = data_length; + + /* Allocate tuple */ + tuple = (FluxTuple) palloc0(sizeof(FluxTupleData)); + tuple->t_len = tuple_length; + tuple->t_data = (FluxTupleHeader *) palloc0(tuple_length); + + /* Set up header */ + header = tuple->t_data; + header->t_natts = tupdesc->natts; + header->t_flags = 0; + header->t_commit_ts = 0; /* Will be set during insert */ + ItemPointerSetInvalid(&header->t_ctid); + header->t_infomask = 0; + + if (has_compressed) + { + header->t_flags |= FLUX_TUPLE_COMPRESSED; + header->t_infomask |= FLUX_INFOMASK_COMPRESSED; + } + (void) has_compressed; + + /* Set up null bitmap */ + bitmap_len = BITMAPLEN(tupdesc->natts); + nulls_bitmap = (uint8 *) header->t_attrs_bitmap; + data_ptr = (char *) header + FLUX_TUPLE_OVERHEAD + MAXALIGN(bitmap_len); + + /* + * Initialize null bitmap - PostgreSQL expects all bits set to 1 initially + * (all NOT NULL) + */ + memset(nulls_bitmap, 0xFF, bitmap_len); + + /* Set infomask bits */ + for (i = 0; i < tupdesc->natts; i++) + { + if (isnull[i]) + { + has_nulls = true; + /* Clear the bit for NULL attributes (bit=0 means NULL) */ + nulls_bitmap[i >> 3] &= ~(1 << (i & 0x07)); + } + else + { + Form_pg_attribute att = TupleDescAttr(tupdesc, i); + + if (att->attlen == -1 || att->attlen == -2) + has_varwidth = true; + + /* Check for external storage */ + if (att->attlen == -1 && VARATT_IS_EXTERNAL(DatumGetPointer(work_values[i]))) + has_external = true; + } + } + + if (has_nulls) + header->t_infomask |= FLUX_INFOMASK_HASNULL; + if (has_varwidth) + header->t_infomask |= FLUX_INFOMASK_HASVARWIDTH; + if (has_external) + header->t_infomask |= FLUX_INFOMASK_HASEXTERNAL; + if (has_overflow) + { + header->t_flags |= FLUX_TUPLE_HAS_OVERFLOW; + header->t_infomask |= FLUX_INFOMASK_HASOVERFLOW; + } + + /* + * Phase 3: Store attribute values (using compressed data where + * applicable) + */ + for (i = 0; i < tupdesc->natts; i++) + { + Form_pg_attribute att = TupleDescAttr(tupdesc, i); + + if (att->attisdropped || isnull[i]) + continue; + + /* Align attribute */ + data_ptr = (char *) att_align_nominal(data_ptr, att->attalign); + + if (att->attlen > 0) + { + /* + * Fixed-length attribute - never compressed. Must distinguish + * byval from by-reference fixed-length types (e.g., timetz is 12 + * bytes but passed by reference). + */ + if (att->attbyval) + store_att_byval(data_ptr, work_values[i], att->attlen); + else + memcpy(data_ptr, DatumGetPointer(work_values[i]), att->attlen); + data_ptr += att->attlen; + } + else if (att->attlen == -1) + { + /* Variable-length attribute (possibly compressed) */ + Size attr_len = VARSIZE_ANY(DatumGetPointer(work_values[i])); + + memcpy(data_ptr, DatumGetPointer(work_values[i]), attr_len); + data_ptr += attr_len; + } + else if (att->attlen == -2) + { + /* C string */ + Size attr_len = strlen(DatumGetCString(work_values[i])) + 1; + + memcpy(data_ptr, DatumGetCString(work_values[i]), attr_len); + data_ptr += attr_len; + } + } + + /* + * Free compressed and overflow datums that were allocated by + * FluxCompressAttribute and FluxStoreOverflowColumn + */ + for (i = 0; i < tupdesc->natts; i++) + { + if (is_compressed[i] || is_overflowed[i]) + pfree(DatumGetPointer(work_values[i])); + } + if (tupdesc->natts > FLUX_FORM_STACK_ATTRS) + { + pfree(work_values); + pfree(is_compressed); + pfree(is_overflowed); + } + + return tuple; +} + +/* + * FluxDeformTuple + * + * Extract attribute values and null indicators from a FLUX tuple into the + * provided arrays. This is the inverse of FluxFormTuple(). + * + * When the tuple has the FLUX_INFOMASK_COMPRESSED flag set, variable-length + * attributes may contain a FluxCompressionHeader prefix followed by + * compressed data. This function transparently decompresses such attributes + * so that callers always see the original uncompressed Datum values. + * + * Parameters: + * tuple - the FLUX tuple to deform + * tupdesc - tuple descriptor defining the schema + * values - output array of Datum values (must be pre-allocated) + * isnull - output array of boolean null indicators (must be pre-allocated) + */ +void +FluxDeformTuple(Relation rel, FluxTuple tuple, TupleDesc tupdesc, Datum *values, bool *isnull) +{ + FluxDeformTupleUpTo(rel, tuple, tupdesc, values, isnull, tupdesc->natts); +} + +/* + * FluxDeformTupleUpTo + * + * Like FluxDeformTuple(), but stops after extracting the first max_natts + * attributes. Because FLUX (like heap) stores variable-length attributes + * consecutively, deforming is inherently sequential: extracting attribute N + * still costs walking attributes 0..N. But a caller that only needs the + * low-numbered attributes (e.g. flux_indexed_attr_changed comparing indexed + * columns, which are typically the leading columns) can cap the walk at the + * highest attribute it will read instead of deforming every column of a wide + * tuple. Attributes at and beyond max_natts are returned as NULL/0 so a + * caller that accidentally reads past its bound gets a defined (not garbage) + * value; callers MUST NOT rely on those tail values. + */ +void +FluxDeformTupleUpTo(Relation rel, FluxTuple tuple, TupleDesc tupdesc, + Datum *values, bool *isnull, int max_natts) +{ + FluxTupleHeader *header; + uint8 *nulls_bitmap; + char *data_ptr; + Size bitmap_len; + int i; + bool tuple_has_compressed; + + Assert(tuple != NULL); + Assert(tupdesc != NULL); + Assert(values != NULL); + Assert(isnull != NULL); + + header = tuple->t_data; + + /* + * Use the tuple's actual natts for bitmap_len and data_ptr calculation. + * After ALTER TABLE ADD COLUMN, old tuples may have fewer attributes. + */ + { + int tuple_natts = header->t_natts; + int loop_natts = Min(tupdesc->natts, tuple_natts); + + /* Cap the deform at the caller's requested bound. */ + if (max_natts >= 0 && max_natts < loop_natts) + loop_natts = max_natts; + + bitmap_len = BITMAPLEN(tuple_natts); + nulls_bitmap = (uint8 *) header->t_attrs_bitmap; + data_ptr = (char *) header + FLUX_TUPLE_OVERHEAD + MAXALIGN(bitmap_len); + + tuple_has_compressed = false; /* FLUX does not compress attributes */ + (void) tuple_has_compressed; + + /* Extract each attribute present in the tuple */ + for (i = 0; i < loop_natts; i++) + { + Form_pg_attribute att = TupleDescAttr(tupdesc, i); + + if (att->attisdropped) + { + values[i] = (Datum) 0; + isnull[i] = true; + continue; + } + + /* + * Check null bitmap: bit=0 means NULL (bit cleared in + * FluxFormTuple) + */ + if (header->t_infomask & FLUX_INFOMASK_HASNULL && + att_isnull(i, nulls_bitmap)) + { + values[i] = (Datum) 0; + isnull[i] = true; + continue; + } + + isnull[i] = false; + + /* Align attribute */ + data_ptr = (char *) att_align_nominal(data_ptr, att->attalign); + + if (att->attlen > 0) + { + /* + * Fixed-length attribute - never compressed. Use actual + * attbyval flag (e.g., timetz is 12 bytes but by-ref). + */ + values[i] = fetch_att(data_ptr, att->attbyval, att->attlen); + data_ptr += att->attlen; + } + else if (att->attlen == -1) + { + /* Variable-length attribute (FLUX stores it verbatim) */ + Size attr_len = VARSIZE_ANY(data_ptr); + + values[i] = PointerGetDatum(data_ptr); + data_ptr += attr_len; + } + else if (att->attlen == -2) + { + /* C string - never compressed */ + values[i] = CStringGetDatum(data_ptr); + data_ptr += strlen(data_ptr) + 1; + } + else + { + elog(ERROR, "unsupported attribute length: %d", att->attlen); + } + } + + /* + * Fill missing attributes with defaults for columns added by ALTER + * TABLE ADD COLUMN after this tuple was stored. + */ + for (i = loop_natts; i < tupdesc->natts; i++) + { + values[i] = (Datum) 0; + isnull[i] = true; + } + } /* end of tuple_natts scope block */ +} + +/* + * FluxFreeTuple + * + * Free a FLUX tuple and its associated data. Safe to call with NULL. + * + * Parameters: + * tuple - the FluxTuple to free (may be NULL) + */ +void +FluxFreeTuple(FluxTuple tuple) +{ + if (tuple) + { + if (tuple->t_data) + pfree(tuple->t_data); + pfree(tuple); + } +} + +/* + * FluxInitPage + * + * Initialize a new FLUX page. Calls PostgreSQL's PageInit() with space + * reserved for FluxPageOpaqueData in the special area, then initializes + * the opaque data fields to their default values. + * + * Parameters: + * page - pointer to the page buffer + * pageSize - size of the page (typically BLCKSZ = 8192) + */ +void +FluxInitPage(Page page, Size pageSize) +{ + FluxPageOpaque phdr; + + PageInit(page, pageSize, sizeof(FluxPageOpaqueData)); + + phdr = FluxPageGetOpaque(page); + phdr->pd_commit_ts_and_flags = 0; +} + +/* + * FluxPageAddTuple + * + * Add a FLUX tuple to a page using PageAddItem(). Updates the page's + * opaque data (commit timestamp, free space) after successful insertion. + * + * Parameters: + * page - the page to add the tuple to (must be exclusively locked) + * tuple - the FLUX tuple to add + * tuple_size - size of the tuple data in bytes + * + * Returns the OffsetNumber where the tuple was placed, or + * InvalidOffsetNumber if the page does not have enough space. + */ +OffsetNumber +FluxPageAddTuple(Page page, FluxTuple tuple, Size tuple_size) +{ + FluxPageOpaque phdr; + OffsetNumber offnum; + + /* Try to add the tuple */ + offnum = PageAddItem(page, tuple->t_data, tuple_size, + InvalidOffsetNumber, false, false); + + if (offnum == InvalidOffsetNumber) + return InvalidOffsetNumber; + + /* Update page header */ + phdr = FluxPageGetOpaque(page); + + /* Mark page for defragmentation if fragmented */ + if (PageGetFreeSpace(page) >= tuple_size * 2 && + PageGetMaxOffsetNumber(page) > FirstOffsetNumber + 5) + { + FluxPageSetFlag(phdr, FLUX_PAGE_DEFRAG_NEEDED); + } + + return offnum; +} + +/* + * FluxPageUpdateTuple + * + * Attempt to update a tuple in place on a FLUX page. If the new tuple + * fits within the existing allocation (same size or smaller), the data is + * overwritten directly (in-place update). If the new tuple is larger but + * the page has enough total free space, the old tuple is removed and the + * new tuple is added at the same or a new offset. + * + * Parameters: + * page - the page containing the tuple (must be exclusively locked) + * offnum - offset number of the tuple to update + * new_tuple - the new tuple data + * old_commit_ts - commit timestamp of the old version (for WAL logging) + * new_commit_ts - commit timestamp for the new version + * + * Returns true if the update was performed on this page, false if the new + * tuple does not fit (caller must handle cross-page update). + */ +bool +FluxPageUpdateTuple(Page page, OffsetNumber offnum, FluxTuple new_tuple, + uint64 old_commit_ts, uint64 new_commit_ts) +{ + ItemId itemid; + FluxTupleHeader *old_tuple; + Size old_size, + new_size; + FluxPageOpaque phdr; + Size available_space; + OffsetNumber new_offnum; + + itemid = PageGetItemId(page, offnum); + if (!ItemIdIsNormal(itemid)) + return false; + + old_tuple = (FluxTupleHeader *) PageGetItem(page, itemid); + old_size = ItemIdGetLength(itemid); + new_size = new_tuple->t_len; + + /* Check if new tuple fits in same space */ + if (new_size <= old_size) + { + /* In-place update */ + memcpy(old_tuple, new_tuple->t_data, new_size); + if (new_size < old_size) + { + /* Update item length */ + ItemIdSetNormal(itemid, ItemIdGetOffset(itemid), new_size); + } + + /* Update page header */ + phdr = FluxPageGetOpaque(page); + FluxPageSetCommitTs(phdr, Max(FluxPageGetCommitTs(phdr), new_commit_ts)); + + return true; + } + + /* Need more space - check if available */ + available_space = PageGetFreeSpace(page) + old_size; + if (new_size <= available_space) + { + /* + * Remove old tuple and re-add the new (larger) one. + * + * We use FluxPageIndexTupleDelete instead of PageIndexTupleDelete + * because the page may contain LP_UNUSED items from defragmentation. + * PageIndexTupleDelete asserts all items are LP_NORMAL; + * FluxPageIndexTupleDelete skips LP_UNUSED items safely. + */ + FluxPageIndexTupleDelete(page, offnum); + + new_offnum = PageAddItem(page, new_tuple->t_data, + new_size, offnum, + false, false); + + if (new_offnum != InvalidOffsetNumber) + { + /* Update page header */ + phdr = FluxPageGetOpaque(page); + FluxPageSetCommitTs(phdr, Max(FluxPageGetCommitTs(phdr), new_commit_ts)); + return true; + } + } + + return false; /* Update failed - need new page */ +} + +/* + * Get number of live tuples on a FLUX page + */ +int +FluxPageGetLiveTuples(Page page, uint64 snapshot_ts) +{ + OffsetNumber maxoff = PageGetMaxOffsetNumber(page); + int live_tuples = 0; + OffsetNumber offnum; + + for (offnum = FirstOffsetNumber; offnum <= maxoff; offnum++) + { + ItemId itemid = PageGetItemId(page, offnum); + + if (ItemIdIsNormal(itemid)) + { + FluxTupleHeader *tuple = (FluxTupleHeader *) PageGetItem(page, itemid); + + /* + * Heap-shaped: "live" here means not carrying a committed delete + * marker. This helper is only an estimate (no current callers), + * so it counts non-DELETED tuples rather than doing a full + * snapshot visibility check. snapshot_ts is unused. + */ + (void) snapshot_ts; + if (!(tuple->t_flags & FLUX_TUPLE_DELETED)) + live_tuples++; + } + } + + return live_tuples; +} + +/* + * FluxPageDefragment + * + * Compact a FLUX page by calling PageRepairFragmentation() to consolidate + * free space. Updates the page opaque data with the new free space amount, + * increments the defrag counter, and clears the FLUX_PAGE_DEFRAG_NEEDED flag. + * + * Parameters: + * page - the page to defragment (must be exclusively locked) + */ +void +FluxPageDefragment(Page page) +{ + FluxPageOpaque phdr = FluxPageGetOpaque(page); + + /* Use standard PageRepairFragmentation */ + PageRepairFragmentation(page); + + /* Update page header */ + FluxPageClearFlag(phdr, FLUX_PAGE_DEFRAG_NEEDED); +} + +/* + * FluxPageIndexTupleDelete + * + * Like PageIndexTupleDelete, but tolerates LP_UNUSED items on the page. + * + * Standard PageIndexTupleDelete asserts that ALL line pointers have storage + * (ItemIdHasStorage). FLUX pages may contain LP_UNUSED items left behind + * by opportunistic defragmentation. This function skips LP_UNUSED items + * when adjusting offsets, preventing both assertion failures and data + * corruption (LP_UNUSED items have lp_off=0 and must not be adjusted). + */ +void +FluxPageIndexTupleDelete(Page page, OffsetNumber offnum) +{ + PageHeader phdr = (PageHeader) page; + char *addr; + ItemId tup; + Size size; + unsigned offset; + int nbytes; + int offidx; + int nline; + + if (phdr->pd_lower < SizeOfPageHeaderData || + phdr->pd_lower > phdr->pd_upper || + phdr->pd_upper > phdr->pd_special || + phdr->pd_special > BLCKSZ || + phdr->pd_special != MAXALIGN(phdr->pd_special)) + ereport(ERROR, + (errcode(ERRCODE_DATA_CORRUPTED), + errmsg("corrupted page pointers: lower = %u, upper = %u, special = %u", + phdr->pd_lower, phdr->pd_upper, phdr->pd_special))); + + nline = PageGetMaxOffsetNumber(page); + if ((int) offnum <= 0 || (int) offnum > nline) + elog(ERROR, "invalid index offnum: %u", offnum); + + offidx = offnum - 1; + + tup = PageGetItemId(page, offnum); + Assert(ItemIdHasStorage(tup)); + size = ItemIdGetLength(tup); + offset = ItemIdGetOffset(tup); + + if (offset < phdr->pd_upper || (offset + size) > phdr->pd_special || + offset != MAXALIGN(offset)) + ereport(ERROR, + (errcode(ERRCODE_DATA_CORRUPTED), + errmsg("corrupted line pointer: offset = %u, size = %zu", + offset, size))); + + size = MAXALIGN(size); + + /* Remove the line pointer entry by shifting subsequent entries down */ + nbytes = phdr->pd_lower - + ((char *) &phdr->pd_linp[offidx + 1] - (char *) phdr); + + if (nbytes > 0) + memmove(&(phdr->pd_linp[offidx]), + &(phdr->pd_linp[offidx + 1]), + nbytes); + + /* Shift tuple data forward to fill the gap */ + addr = (char *) page + phdr->pd_upper; + + if (offset > phdr->pd_upper) + memmove(addr + size, addr, offset - phdr->pd_upper); + + phdr->pd_upper += size; + phdr->pd_lower -= sizeof(ItemIdData); + + /* Adjust remaining line pointer offsets, skipping LP_UNUSED items */ + if (!PageIsEmpty(page)) + { + int i; + + nline--; + for (i = 1; i <= nline; i++) + { + ItemId ii = PageGetItemId(page, i); + + if (!ItemIdHasStorage(ii)) + continue; + if (ItemIdGetOffset(ii) <= offset) + ii->lp_off += size; + } + } +} + +/* + * Convert a FLUX tuple to a TupleTableSlot + * + * This is the primary retrieval path used during sequential scans. + * When the tuple has compressed attributes (FLUX_INFOMASK_COMPRESSED), + * they are transparently decompressed so the slot always contains + * uncompressed data visible to the executor. + * + * Overflow attributes (FLUX_INFOMASK_HASOVERFLOW) are returned as-is + * by this function since it has no Relation handle. Use + * FluxTupleToSlotWithOverflow() for transparent overflow fetching. + */ +bool +FluxTupleToSlot(FluxTupleHeader *tuple_header, TupleTableSlot *slot) +{ + return FluxTupleToSlotWithOverflow(tuple_header, slot, NULL); +} + +/* + * Convert a FLUX tuple to a TupleTableSlot with overflow support. + * + * When rel is non-NULL and the tuple has overflow attributes, they are + * transparently fetched from overflow records and the slot receives the + * complete original values. When rel is NULL, overflow pointers are + * returned as-is (same as FluxTupleToSlot). + */ +bool +FluxTupleToSlotWithOverflow(FluxTupleHeader *tuple_header, + TupleTableSlot *slot, Relation rel) +{ + TupleDesc tupdesc = slot->tts_tupleDescriptor; + char *data_ptr; + uint8 *nulls_bitmap; + int i; + Size bitmap_len; + bool tuple_has_compressed; + bool tuple_has_overflow; + + if (!tuple_header) + return false; + + /* Check if tuple is deleted */ + if (tuple_header->t_flags & FLUX_TUPLE_DELETED) + return false; + + /* Clear the slot first */ + ExecClearTuple(slot); + + /* + * Use the tuple's actual natts for bitmap_len and data_ptr calculation. + * After ALTER TABLE ADD COLUMN, old tuples may have fewer attributes than + * the current schema expects. + */ + { + int tuple_natts = tuple_header->t_natts; + int loop_natts = Min(tupdesc->natts, tuple_natts); + + bitmap_len = BITMAPLEN(tuple_natts); + + /* Set up pointers to data */ + nulls_bitmap = (uint8 *) tuple_header->t_attrs_bitmap; + data_ptr = (char *) tuple_header + FLUX_TUPLE_OVERHEAD + MAXALIGN(bitmap_len); + + tuple_has_compressed = false; /* FLUX does not compress attributes */ + tuple_has_overflow = false; /* FLUX uses TOAST, not on-page overflow */ + (void) tuple_has_compressed; + (void) tuple_has_overflow; + + /* Decode each attribute present in the tuple */ + for (i = 0; i < loop_natts; i++) + { + Form_pg_attribute att = TupleDescAttr(tupdesc, i); + bool is_null; + + if (att->attisdropped) + { + slot->tts_values[i] = (Datum) 0; + slot->tts_isnull[i] = true; + continue; + } + + /* Check if attribute is null */ + is_null = att_isnull(i, nulls_bitmap); + + if (is_null) + { + slot->tts_values[i] = (Datum) 0; + slot->tts_isnull[i] = true; + } + else + { + /* Extract the actual data */ + if (att->attlen == -1) + { + Size attr_len; + + /* + * Align to the start of this varlena, matching the form + * path (FluxFormTuple aligns data_ptr at the start of + * each attribute before writing). Without this the + * previous attribute's raw += attr_len advance can leave + * data_ptr unaligned, so VARSIZE_ANY reads the length + * from padding bytes -> garbage length -> pglz-corrupt / + * overrun. + */ + data_ptr = (char *) att_align_nominal(data_ptr, att->attalign); + attr_len = VARSIZE_ANY(data_ptr); + + /* + * FLUX stores varlena values verbatim (wide values are + * TOASTed by the standard heap TOAST path; no on-page + * overflow, no attribute compression). + */ + slot->tts_values[i] = PointerGetDatum(data_ptr); + data_ptr = (char *) att_align_nominal(data_ptr + attr_len, att->attalign); + } + else if (att->attlen > 0) + { + /* Fixed-length attribute - never compressed or overflow */ + data_ptr = (char *) att_align_nominal(data_ptr, att->attalign); + slot->tts_values[i] = fetchatt(att, data_ptr); + data_ptr += att->attlen; + } + else + { + /* This shouldn't happen */ + elog(ERROR, "unsupported attribute length: %d", att->attlen); + } + + slot->tts_isnull[i] = false; + } + } + + /* + * Fill missing attributes with defaults for columns added by ALTER + * TABLE ADD COLUMN after this tuple was stored. + */ + if (loop_natts < tupdesc->natts) + { + for (i = loop_natts; i < tupdesc->natts; i++) + { + slot->tts_values[i] = (Datum) 0; + slot->tts_isnull[i] = true; + } + slot->tts_nvalid = loop_natts; + slot_getmissingattrs(slot, loop_natts, tupdesc->natts); + } + } /* end of tuple_natts scope block */ + + /* Mark slot as valid */ + slot->tts_flags &= ~TTS_FLAG_EMPTY; + slot->tts_nvalid = tupdesc->natts; + + return true; +} diff --git a/src/backend/access/flux/flux_undo.c b/src/backend/access/flux/flux_undo.c new file mode 100644 index 0000000000000..90d66777c9839 --- /dev/null +++ b/src/backend/access/flux/flux_undo.c @@ -0,0 +1,440 @@ +/*------------------------------------------------------------------------- + * + * flux_undo.c + * FLUX UNDO resource manager + * + * FLUX writes one UNDO record per tuple INSERT, UPDATE and DELETE via + * the shared UNDO-in-WAL infrastructure. Records carry rmid + * UNDO_RMID_FLUX and an info subtype (FLUX_UNDO_INSERT / UPDATE / + * DELETE / DELTA_UPDATE); rollback is driven by undoapply.c which + * dispatches to flux_undo_apply() based on rmid. + * + * Visibility of aborted rows is handled independently of physical + * undo application: FLUX tuples carry a FLUX_TUPLE_UNCOMMITTED flag + * whose MVCC-visibility path consults the sLog, so an aborted + * transaction's tuples are invisible the moment the sLog entry + * transitions to ABORTED (see flux_slog.c's XACT_EVENT_ABORT handler). + * The physical page-mutation done here reclaims on-disk space so + * VACUUM does not have to touch every aborted row. + * + * Crash safety is provided by emitting an xl_undo_apply CLR record + * (XLOG_UNDO_APPLY_RECORD / RM_UNDO_ID) for every page modification. + * The CLR carries the new tuple image (or the LP-state change) and is + * replayed idempotently by the generic undo_xlog.c redo handler; + * FLUX does not need its own redo routine for the undo-apply path. + * + * The callback mirrors heapam_undo.c's control flow: + * + * 1. Defer while in crash recovery or inside a transaction's abort + * path (BumpContext makes relation_close/pfree unsafe); the + * logical-revert worker re-drives the record from a clean top- + * level memory context. + * 2. try_relation_open() the target; if the relation was dropped + * or truncated past the target block, return UNDO_APPLY_SKIPPED. + * 3. Dispatch on info to a page-modification branch, emit a CLR, + * mark the buffer dirty, release locks, close. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/backend/access/flux/flux_undo.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/heapam.h" +#include "access/flux.h" +#include "access/flux_undo.h" +#include "access/relation.h" +#include "access/table.h" +#include "access/undo_xlog.h" +#include "access/undormgr.h" +#include "access/xact.h" +#include "access/xlog.h" +#include "access/xloginsert.h" +#include "access/xlogrecovery.h" +#include "access/xlogutils.h" +#include "miscadmin.h" +#include "storage/bufmgr.h" +#include "utils/rel.h" +#include "utils/relcache.h" + + +static UndoApplyResult flux_undo_apply(uint8 rmid, uint16 info, + TransactionId xid, Oid reloid, + const char *payload, Size payload_len, + UndoRecPtr urec_ptr); +static void flux_undo_desc(StringInfo buf, uint8 rmid, uint16 info, + const char *payload, Size payload_len); + + +/* The FLUX UNDO RM registration entry */ +static const UndoRmgrData flux_undo_rmgr = { + .rm_name = "flux", + .rm_undo = flux_undo_apply, + .rm_desc = flux_undo_desc, +}; + + +/* + * FluxUndoRmgrInit + * Register the FLUX UNDO resource manager. + * + * Called from InitializeUndoSubsystem() at postmaster startup, alongside + * HeapUndoRmgrInit() and NbtreeUndoRmgrInit(). + */ +void +FluxUndoRmgrInit(void) +{ + RegisterUndoRmgr(UNDO_RMID_FLUX, &flux_undo_rmgr); + + /* + * Install the FLUX implementations of the AM-neutral per-relation UNDO + * hooks now, before crash recovery can replay a RELUNDO CLR that would + * dispatch through them. + */ + FluxRelUndoInstallHooks(); +} + + +/* + * emit_flux_undo_clr + * Emit an XLOG_UNDO_APPLY_RECORD CLR for the page modification + * just performed. Must be called inside the critical section, + * before END_CRIT_SECTION / UnlockReleaseBuffer. + * + * tuple_data is the image to replay into the target slot on redo + * (NULL for LP_UNUSED cases). tuple_len must match the on-page slot + * length that should be installed. + */ +static void +emit_flux_undo_clr(Relation rel, Buffer buffer, UndoRecPtr urec_ptr, + TransactionId xid, BlockNumber blkno, OffsetNumber offnum, + uint16 info, uint16 clr_flags, + const char *tuple_data, uint32 tuple_len) +{ + xl_undo_apply xlrec; + XLogRecPtr lsn; + + if (!RelationNeedsWAL(rel)) + { + /* + * Unlogged / temp relations need no CLR: they do not survive a crash, + * so replay idempotency is irrelevant. + */ + PageSetLSN(BufferGetPage(buffer), GetXLogInsertRecPtr()); + return; + } + + xlrec.urec_ptr = urec_ptr; + xlrec.xid = xid; + xlrec.target_locator = rel->rd_locator; + xlrec.target_block = blkno; + xlrec.target_offset = offnum; + xlrec.operation_type = info; + xlrec.clr_flags = clr_flags; + xlrec.tuple_len = tuple_len; + + XLogBeginInsert(); + XLogRegisterData((char *) &xlrec, SizeOfUndoApply); + XLogRegisterBuffer(0, buffer, REGBUF_STANDARD); + + if ((clr_flags & UNDO_CLR_HAS_TUPLE) && tuple_data != NULL && tuple_len > 0) + XLogRegisterBufData(0, tuple_data, tuple_len); + + lsn = XLogInsert(RM_UNDO_ID, XLOG_UNDO_APPLY_RECORD); + PageSetLSN(BufferGetPage(buffer), lsn); +} + + +/* + * apply_flux_undo_insert + * Undo an INSERT: mark the inserted tuple FLUX_TUPLE_DELETED so + * VACUUM can reclaim its space. The sLog-driven visibility path + * already hides the row from readers once the transaction is + * marked ABORTED; this routine exists purely for physical + * space-reclaim. + * + * We do not use UNDO_CLR_LP_DEAD / UNDO_CLR_LP_UNUSED because those + * drop the item entirely, whereas FLUX needs the tuple header to + * stay intact (the page's commit_ts, overflow pointers, and the + * DELETED bit itself are all read by VACUUM). + */ +static void +apply_flux_undo_insert(Relation rel, Buffer buffer, OffsetNumber offnum, + BlockNumber blkno, UndoRecPtr urec_ptr, + TransactionId xid) +{ + Page page = BufferGetPage(buffer); + ItemId lp; + FluxTupleHeader hdr; + Size len; + char *slot; + + lp = PageGetItemId(page, offnum); + if (!ItemIdIsNormal(lp)) + { + /* + * Already cleaned up (e.g. VACUUM ran between the abort and the + * logical-revert worker's pass). Nothing to do. + */ + return; + } + + len = ItemIdGetLength(lp); + slot = (char *) PageGetItem(page, lp); + + START_CRIT_SECTION(); + + /* + * Read, mutate, write back the tuple header in place. + * + * Set FLUX_TUPLE_DELETED so VACUUM can reclaim the space. Crucially, we + * must NOT clear FLUX_TUPLE_UNCOMMITTED: the inserting transaction never + * committed, so the tuple has to remain on the UNCOMMITTED visibility + * path (flux_mvcc.c), where a SLOG_OP_ABORTED entry resolves the row to + * not-visible. Clearing the flag would route the tuple to the + * post-commit deletion path, whose ABORTED handling resurrects aborted + * DELETEs and would therefore make this never-committed row visible. + */ + memcpy(&hdr, slot, sizeof(hdr)); + hdr.t_flags |= FLUX_TUPLE_DELETED; + memcpy(slot, &hdr, sizeof(hdr)); + + MarkBufferDirty(buffer); + + emit_flux_undo_clr(rel, buffer, urec_ptr, xid, blkno, offnum, + FLUX_UNDO_INSERT, UNDO_CLR_HAS_TUPLE, + slot, (uint32) len); + + END_CRIT_SECTION(); +} + + +/* + * apply_flux_undo_restore_tuple + * Shared helper for UPDATE, DELETE and DELTA_UPDATE undo: overwrite + * the current on-disk tuple with an in-memory before-image. The + * caller is responsible for preparing the before-image (direct + * copy for DELETE/UPDATE, reverse-diff reconstruction for + * DELTA_UPDATE). + * + * For DELETE undo the before-image already carries the pre-delete + * header, so the FLUX_TUPLE_DELETED bit will be cleared as a + * side-effect of the overwrite. + * + * If the before-image is larger than the current on-page slot, the + * undo is skipped (the slot was shrunk by a later in-place update and + * cannot be safely grown from here). The row will remain visible + * per sLog until VACUUM reclaims it. + */ +static bool +apply_flux_undo_restore_tuple(Relation rel, Buffer buffer, OffsetNumber offnum, + BlockNumber blkno, UndoRecPtr urec_ptr, + TransactionId xid, uint16 info, + const char *old_image, uint32 old_len) +{ + Page page = BufferGetPage(buffer); + ItemId lp; + char *slot; + + Assert(old_image != NULL && old_len > 0); + + lp = PageGetItemId(page, offnum); + if (!ItemIdIsNormal(lp)) + { + ereport(DEBUG2, + (errmsg_internal("FLUX UNDO: item (%u, %u) no longer normal, skipping", + blkno, offnum))); + return false; + } + + if (ItemIdGetLength(lp) < old_len) + { + ereport(WARNING, + (errmsg_internal("FLUX UNDO: current slot at (%u, %u) is smaller " + "than before-image (%u < %u); rollback skipped, " + "row left under MVCC retention until VACUUM", + blkno, offnum, + (unsigned) ItemIdGetLength(lp), + (unsigned) old_len))); + return false; + } + + slot = (char *) PageGetItem(page, lp); + + START_CRIT_SECTION(); + + memcpy(slot, old_image, old_len); + if (ItemIdGetLength(lp) != old_len) + ItemIdSetNormal(lp, ItemIdGetOffset(lp), old_len); + + MarkBufferDirty(buffer); + + emit_flux_undo_clr(rel, buffer, urec_ptr, xid, blkno, offnum, + info, UNDO_CLR_HAS_TUPLE, + old_image, old_len); + + END_CRIT_SECTION(); + return true; +} + + +/* + * flux_undo_apply + * Apply a single FLUX UNDO record. + * + * Dispatched from undoapply.c for records tagged UNDO_RMID_FLUX. + */ +static UndoApplyResult +flux_undo_apply(uint8 rmid, uint16 info, TransactionId xid, Oid reloid, + const char *payload, Size payload_len, UndoRecPtr urec_ptr) +{ + FluxUndoPayloadHeader hdr; + const char *image_bytes; + Size image_len; + Relation rel; + Buffer buffer; + BlockNumber blkno; + OffsetNumber offnum; + + Assert(rmid == UNDO_RMID_FLUX); + + /* + * Defer during crash recovery (syscache may not be initialised) or during + * an aborting transaction (BumpContext makes relation_close() and pfree() + * unsafe). The logical-revert worker will re-drive the record from a + * clean memory context. + */ + if (InRecovery || IsAbortedTransactionBlockState()) + { + ereport(DEBUG2, + (errmsg_internal("FLUX UNDO: deferring xid %u record at %llu " + "(in recovery or abort path)", + xid, + (unsigned long long) urec_ptr))); + return UNDO_APPLY_SKIPPED; + } + + /* Decode the common payload header */ + if (payload_len < SizeOfFluxUndoPayloadHeader) + { + ereport(WARNING, + (errmsg_internal("FLUX UNDO: payload too short (%zu bytes) " + "for record at %llu", + payload_len, + (unsigned long long) urec_ptr))); + return UNDO_APPLY_ERROR; + } + memcpy(&hdr, payload, SizeOfFluxUndoPayloadHeader); + image_bytes = payload + SizeOfFluxUndoPayloadHeader; + image_len = payload_len - SizeOfFluxUndoPayloadHeader; + blkno = ItemPointerGetBlockNumber(&hdr.tid); + offnum = ItemPointerGetOffsetNumber(&hdr.tid); + + /* Open the relation; skip if dropped */ + rel = try_relation_open(reloid, RowExclusiveLock); + if (rel == NULL) + { + ereport(DEBUG2, + (errmsg_internal("FLUX UNDO: relation %u no longer exists, " + "skipping record at %llu", + reloid, + (unsigned long long) urec_ptr))); + return UNDO_APPLY_SKIPPED; + } + + /* Skip if the target block was truncated away */ + if (RelationGetNumberOfBlocks(rel) <= blkno) + { + ereport(DEBUG2, + (errmsg_internal("FLUX UNDO: block %u beyond end of " + "relation %u, skipping", + blkno, reloid))); + relation_close(rel, RowExclusiveLock); + return UNDO_APPLY_SKIPPED; + } + + buffer = ReadBuffer(rel, blkno); + LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); + + switch (info) + { + case FLUX_UNDO_INSERT: + apply_flux_undo_insert(rel, buffer, offnum, blkno, + urec_ptr, xid); + break; + + case FLUX_UNDO_UPDATE: + case FLUX_UNDO_DELETE: + if (!(hdr.flags & FLUX_UNDO_FLAG_HAS_TUPLE) || image_len == 0) + { + ereport(WARNING, + (errmsg_internal("FLUX UNDO %s: missing before-image at %llu", + info == FLUX_UNDO_UPDATE ? "UPDATE" : "DELETE", + (unsigned long long) urec_ptr))); + break; + } + apply_flux_undo_restore_tuple(rel, buffer, offnum, blkno, + urec_ptr, xid, info, + image_bytes, (uint32) image_len); + break; + + default: + ereport(WARNING, + (errmsg_internal("FLUX UNDO: unknown subtype 0x%x at %llu", + info, (unsigned long long) urec_ptr))); + break; + } + + UnlockReleaseBuffer(buffer); + relation_close(rel, RowExclusiveLock); + return UNDO_APPLY_SUCCESS; +} + + +/* + * flux_undo_desc + * Describe a FLUX UNDO record for pg_waldump / debug logging. + */ +static void +flux_undo_desc(StringInfo buf, uint8 rmid, uint16 info, + const char *payload, Size payload_len) +{ + const char *subtype; + FluxUndoPayloadHeader hdr; + + switch (info) + { + case FLUX_UNDO_INSERT: + subtype = "INSERT"; + break; + case FLUX_UNDO_UPDATE: + subtype = "UPDATE"; + break; + case FLUX_UNDO_DELETE: + subtype = "DELETE"; + break; + default: + subtype = "UNKNOWN"; + break; + } + + if (payload_len >= SizeOfFluxUndoPayloadHeader) + { + memcpy(&hdr, payload, SizeOfFluxUndoPayloadHeader); + appendStringInfo(buf, + "%s tid=(%u,%u) tuple_len=%u flags=0x%x", + subtype, + ItemPointerGetBlockNumber(&hdr.tid), + ItemPointerGetOffsetNumber(&hdr.tid), + hdr.tuple_len, + hdr.flags); + } + else + { + appendStringInfo(buf, "%s (truncated payload, %zu bytes)", + subtype, payload_len); + } +} diff --git a/src/backend/access/flux/flux_vm.c b/src/backend/access/flux/flux_vm.c new file mode 100644 index 0000000000000..3093aee6082dd --- /dev/null +++ b/src/backend/access/flux/flux_vm.c @@ -0,0 +1,643 @@ +/*------------------------------------------------------------------------- + * + * flux_vm.c + * Visibility Map implementation for FLUX + * + * The Visibility Map (VM) tracks the visibility status of pages in a FLUX + * relation. It stores two bits per heap page: + * + * - ALL_VISIBLE: All tuples on the page are visible to all transactions + * - ALL_FROZEN: All tuples on the page are frozen (transaction IDs removed) + * + * The VM enables two critical optimizations: + * 1. Index-only scans can skip heap fetches for all-visible pages + * 2. VACUUM can skip pages that are already all-visible or all-frozen + * + * The VM is stored in a separate fork of the relation (VISIBILITYMAP_FORKNUM) + * and is WAL-logged for crash recovery. + * + * This implementation is based on the heap visibility map + * (src/backend/access/heap/visibilitymap.c) but adapted for FLUX's + * timestamp-based MVCC model. + * + * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/backend/access/flux/flux_vm.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/flux.h" +#include "access/flux_xlog.h" +#include "access/visibilitymapdefs.h" +#include "access/xlog.h" +#include "access/xloginsert.h" +#include "miscadmin.h" +#include "port/pg_bitutils.h" +#include "storage/bufmgr.h" +#include "storage/buf_internals.h" +#include "storage/lmgr.h" +#include "storage/smgr.h" +#include "utils/inval.h" +#include "utils/rel.h" + +/* + * Size of the bitmap on each visibility map page, in bytes. There's no + * extra headers, so the whole page minus the standard page header is + * used for the bitmap. + */ +#define MAPSIZE (BLCKSZ - MAXALIGN(SizeOfPageHeaderData)) + +/* Number of heap blocks we can represent in one VM page */ +#define HEAPBLOCKS_PER_PAGE (MAPSIZE * 4) + +/* Mapping macros */ +#define HEAPBLK_TO_MAPBLOCK(x) ((x) / HEAPBLOCKS_PER_PAGE) +#define HEAPBLK_TO_MAPBYTE(x) (((x) % HEAPBLOCKS_PER_PAGE) / 4) +#define HEAPBLK_TO_OFFSET(x) (((x) % HEAPBLOCKS_PER_PAGE) % 4) + +/* Bit manipulation - use FLUX-specific values that match PostgreSQL's VM bits */ + +/* Forward declaration */ +static Buffer flux_vm_extend(Relation rel, BlockNumber vm_nblocks); +static Buffer flux_vm_readbuf(Relation rel, BlockNumber blkno, bool extend); + +/* + * FluxVMInit - Initialize visibility map for a FLUX relation + * + * This is called when a FLUX table is created to ensure the VM fork exists. + */ +void +FluxVMInit(Relation rel) +{ + /* + * Create the visibility map fork if it doesn't exist. This happens + * automatically when we first try to extend it via flux_vm_extend(). + */ +} + +/* + * FluxVMSet - Set visibility map bits for a page + * + * Sets the specified bits for the given heap block. The heap buffer must + * be exclusively locked. The VM buffer will be pinned and locked as needed. + */ +void +FluxVMSet(Relation rel, BlockNumber heapBlk, Buffer heapBuf, uint8 flags) +{ + BlockNumber mapBlock; + uint32 mapByte; + uint8 mapOffset; + Page page; + uint8 *map; + Buffer vmBuf; + + Assert(BufferIsValid(heapBuf)); + /* Buffer should be exclusively locked */ + + /* Only set valid bits */ + flags &= FLUX_VM_VALID_BITS; + if (flags == 0) + return; + + /* Calculate the VM page and offset for this heap block */ + mapBlock = HEAPBLK_TO_MAPBLOCK(heapBlk); + mapByte = HEAPBLK_TO_MAPBYTE(heapBlk); + mapOffset = HEAPBLK_TO_OFFSET(heapBlk); + + /* + * Read or extend the visibility map buffer. flux_vm_readbuf() will + * create the VM fork if it doesn't exist yet. + */ + vmBuf = flux_vm_readbuf(rel, mapBlock, true); + LockBuffer(vmBuf, BUFFER_LOCK_EXCLUSIVE); + page = BufferGetPage(vmBuf); + + /* If the page is new, initialize it */ + if (PageIsNew(page)) + PageInit(page, BLCKSZ, 0); + + map = (uint8 *) PageGetContents(page); + + /* Set the bits for this heap block */ + map[mapByte] |= (flags << (mapOffset * 2)); + + MarkBufferDirty(vmBuf); + + /* XLOG stuff */ + if (RelationNeedsWAL(rel)) + { + xl_flux_vm_set xlrec; + XLogRecPtr recptr; + + xlrec.heapBlk = heapBlk; + xlrec.flags = flags; + + XLogBeginInsert(); + XLogRegisterData((char *) &xlrec, sizeof(xlrec)); + + /* + * Register the heap buffer with REGBUF_NO_IMAGE. We reference the + * heap page so that redo can update its LSN, but we do NOT need a + * full-page image of the heap page in this WAL record. The heap + * buffer may not be dirty (e.g., during VACUUM VM updates), so we + * must not let XLogInsert try to take an FPI of it -- that would trip + * the BufferIsDirty assertion. + */ + XLogRegisterBuffer(0, heapBuf, REGBUF_NO_IMAGE | REGBUF_NO_CHANGE); + XLogRegisterBuffer(1, vmBuf, REGBUF_STANDARD); + + recptr = XLogInsert(RM_FLUX_ID, XLOG_FLUX_VM_SET); + PageSetLSN(page, recptr); + } + + UnlockReleaseBuffer(vmBuf); +} + +/* + * FluxVMClear - Clear visibility map bits for a page + * + * Clears the specified bits for the given heap block. The heap buffer must + * be exclusively locked. + */ +void +FluxVMClear(Relation rel, BlockNumber heapBlk, Buffer heapBuf, uint8 flags) +{ + BlockNumber mapBlock; + uint32 mapByte; + uint8 mapOffset; + Page page; + uint8 *map; + Buffer vmBuf; + + Assert(BufferIsValid(heapBuf)); + /* Buffer should be exclusively locked */ + + /* Only clear valid bits */ + flags &= FLUX_VM_VALID_BITS; + if (flags == 0) + return; + + /* Calculate the VM page and offset for this heap block */ + mapBlock = HEAPBLK_TO_MAPBLOCK(heapBlk); + mapByte = HEAPBLK_TO_MAPBYTE(heapBlk); + mapOffset = HEAPBLK_TO_OFFSET(heapBlk); + + /* Check if the VM fork/page exists; if not, nothing to clear */ + if (!smgrexists(RelationGetSmgr(rel), VISIBILITYMAP_FORKNUM)) + return; + if (mapBlock >= RelationGetNumberOfBlocksInFork(rel, VISIBILITYMAP_FORKNUM)) + return; + + vmBuf = ReadBufferExtended(rel, VISIBILITYMAP_FORKNUM, mapBlock, + RBM_NORMAL, NULL); + LockBuffer(vmBuf, BUFFER_LOCK_EXCLUSIVE); + page = BufferGetPage(vmBuf); + map = (uint8 *) PageGetContents(page); + + /* + * Check if the requested bits are already clear. If so, skip the + * modification and WAL logging entirely. This is the common case after + * the first modification to a page since the last VACUUM, and avoids + * significant WAL amplification on hot pages. + */ + if ((map[mapByte] & (flags << (mapOffset * 2))) == 0) + { + UnlockReleaseBuffer(vmBuf); + return; + } + + /* Clear the bits for this heap block */ + map[mapByte] &= ~(flags << (mapOffset * 2)); + + MarkBufferDirty(vmBuf); + + /* XLOG stuff */ + if (RelationNeedsWAL(rel)) + { + xl_flux_vm_clear xlrec; + XLogRecPtr recptr; + + xlrec.heapBlk = heapBlk; + xlrec.flags = flags; + + XLogBeginInsert(); + XLogRegisterData((char *) &xlrec, sizeof(xlrec)); + + /* + * Register the heap buffer with REGBUF_NO_IMAGE for the same reason + * as in FluxVMSet: the heap buffer may not be dirty. + */ + XLogRegisterBuffer(0, heapBuf, REGBUF_NO_IMAGE | REGBUF_NO_CHANGE); + XLogRegisterBuffer(1, vmBuf, REGBUF_STANDARD); + + recptr = XLogInsert(RM_FLUX_ID, XLOG_FLUX_VM_CLEAR); + PageSetLSN(page, recptr); + } + + UnlockReleaseBuffer(vmBuf); +} + +/* + * FluxVMCheck - Check visibility map bits for a page + * + * Returns true if ALL the specified bits are set for the given heap block. + * This function does not require any locks and can be called from + * index-only scan paths. + */ +bool +FluxVMCheck(Relation rel, BlockNumber heapBlk, uint8 flags) +{ + BlockNumber mapBlock; + uint32 mapByte; + uint8 mapOffset; + Page page; + uint8 *map; + Buffer vmBuf; + bool result; + + /* Only check valid bits */ + flags &= FLUX_VM_VALID_BITS; + if (flags == 0) + return true; /* No bits to check */ + + /* Calculate the VM page and offset for this heap block */ + mapBlock = HEAPBLK_TO_MAPBLOCK(heapBlk); + mapByte = HEAPBLK_TO_MAPBYTE(heapBlk); + mapOffset = HEAPBLK_TO_OFFSET(heapBlk); + + /* If the VM fork/page doesn't exist, the bits can't be set */ + if (!smgrexists(RelationGetSmgr(rel), VISIBILITYMAP_FORKNUM)) + return false; + if (mapBlock >= RelationGetNumberOfBlocksInFork(rel, VISIBILITYMAP_FORKNUM)) + return false; + + vmBuf = ReadBufferExtended(rel, VISIBILITYMAP_FORKNUM, mapBlock, + RBM_NORMAL, NULL); + LockBuffer(vmBuf, BUFFER_LOCK_SHARE); + page = BufferGetPage(vmBuf); + map = (uint8 *) PageGetContents(page); + + /* Check if all requested bits are set */ + result = ((map[mapByte] >> (mapOffset * 2)) & flags) == flags; + + UnlockReleaseBuffer(vmBuf); + + return result; +} + +/* + * FluxVMCheckCached - Check visibility map bits with caller-managed buffer cache + * + * Like FluxVMCheck, but the caller provides pointers to a cached VM buffer + * and its block number. The VM buffer is kept pinned across calls; it is + * only released and re-read when the heap block maps to a different VM page. + * This eliminates per-page ReadBufferExtended + UnlockReleaseBuffer overhead + * for sequential scans (one VM page covers HEAPBLOCKS_PER_PAGE heap pages, + * typically ~32K pages with 8KB blocks). + * + * The caller must release the buffer when done (e.g., at scan end). + */ +bool +FluxVMCheckCached(Relation rel, BlockNumber heapBlk, uint8 flags, + Buffer *vmbuf, BlockNumber *vm_blockno) +{ + BlockNumber mapBlock; + uint32 mapByte; + uint8 mapOffset; + Page page; + uint8 *map; + bool result; + + /* Only check valid bits */ + flags &= FLUX_VM_VALID_BITS; + if (flags == 0) + return true; /* No bits to check */ + + /* Calculate the VM page and offset for this heap block */ + mapBlock = HEAPBLK_TO_MAPBLOCK(heapBlk); + mapByte = HEAPBLK_TO_MAPBYTE(heapBlk); + mapOffset = HEAPBLK_TO_OFFSET(heapBlk); + + /* If the VM fork doesn't exist, the bits can't be set */ + if (!smgrexists(RelationGetSmgr(rel), VISIBILITYMAP_FORKNUM)) + return false; + if (mapBlock >= RelationGetNumberOfBlocksInFork(rel, VISIBILITYMAP_FORKNUM)) + return false; + + /* + * Re-read the VM buffer only when the target VM page changes. Each VM + * page covers HEAPBLOCKS_PER_PAGE heap pages, so for sequential scans + * this avoids ~32K redundant buffer reads per VM page. + */ + if (!BufferIsValid(*vmbuf) || *vm_blockno != mapBlock) + { + if (BufferIsValid(*vmbuf)) + ReleaseBuffer(*vmbuf); + *vmbuf = ReadBufferExtended(rel, VISIBILITYMAP_FORKNUM, mapBlock, + RBM_NORMAL, NULL); + *vm_blockno = mapBlock; + } + + LockBuffer(*vmbuf, BUFFER_LOCK_SHARE); + page = BufferGetPage(*vmbuf); + map = (uint8 *) PageGetContents(page); + + /* Check if all requested bits are set */ + result = ((map[mapByte] >> (mapOffset * 2)) & flags) == flags; + + LockBuffer(*vmbuf, BUFFER_LOCK_UNLOCK); + + return result; +} + +/* + * FluxVMPinBuffer - Pin the visibility map buffer for a heap block + * + * This is used when we need to keep the VM buffer pinned across multiple + * operations. The caller is responsible for unpinning the buffer. + */ +void +FluxVMPinBuffer(Relation rel, BlockNumber heapBlk, Buffer *vmbuf) +{ + BlockNumber mapBlock; + + /* Calculate the VM page for this heap block */ + mapBlock = HEAPBLK_TO_MAPBLOCK(heapBlk); + + /* Pin the buffer if not already pinned */ + if (!BufferIsValid(*vmbuf) || BufferGetBlockNumber(*vmbuf) != mapBlock) + { + if (BufferIsValid(*vmbuf)) + ReleaseBuffer(*vmbuf); + *vmbuf = flux_vm_readbuf(rel, mapBlock, true); + } +} + +/* + * FluxVMExtend - Extend the visibility map to cover more heap blocks + * + * This is called when the heap relation is extended. + */ +void +FluxVMExtend(Relation rel, BlockNumber nheapblocks) +{ + BlockNumber newnblocks; + + /* Calculate how many VM blocks we need */ + newnblocks = (nheapblocks + HEAPBLOCKS_PER_PAGE - 1) / HEAPBLOCKS_PER_PAGE; + + /* Extend the VM fork if necessary, creating it if needed */ + if (newnblocks > 0) + { + Buffer buf; + + buf = flux_vm_extend(rel, newnblocks); + ReleaseBuffer(buf); + } +} + +/* + * FluxVMTruncate - Truncate the visibility map + * + * This is called when the heap relation is truncated. + */ +void +FluxVMTruncate(Relation rel, BlockNumber nheapblocks) +{ + BlockNumber newnblocks; + BlockNumber oldnblocks; + + /* Calculate how many VM blocks we need */ + /* If the VM fork doesn't exist, nothing to truncate */ + if (!smgrexists(RelationGetSmgr(rel), VISIBILITYMAP_FORKNUM)) + return; + + newnblocks = (nheapblocks + HEAPBLOCKS_PER_PAGE - 1) / HEAPBLOCKS_PER_PAGE; + oldnblocks = RelationGetNumberOfBlocksInFork(rel, VISIBILITYMAP_FORKNUM); + + if (newnblocks < oldnblocks) + { + /* + * Truncate the VM fork. We need to flush any dirty VM buffers first. + */ + ForkNumber forknum = VISIBILITYMAP_FORKNUM; + + FlushRelationBuffers(rel); + smgrtruncate(RelationGetSmgr(rel), &forknum, 1, &oldnblocks, &newnblocks); + } +} + +/* + * FluxVMGetPageSize - Get the size of a VM page + */ +Size +FluxVMGetPageSize(void) +{ + return MAPSIZE; +} + +/* + * FluxVMMapHeapToVM - Map a heap block number to VM block number + */ +BlockNumber +FluxVMMapHeapToVM(BlockNumber heapBlk) +{ + return HEAPBLK_TO_MAPBLOCK(heapBlk); +} + +/* + * flux_vm_extend - Extend the VM fork to at least vm_nblocks. + * + * Creates the VM fork if it doesn't exist yet. Returns a buffer for + * the last block of the extended fork (pinned but not locked). + */ +static Buffer +flux_vm_extend(Relation rel, BlockNumber vm_nblocks) +{ + Buffer buf; + + buf = ExtendBufferedRelTo(BMR_REL(rel), VISIBILITYMAP_FORKNUM, NULL, + EB_CREATE_FORK_IF_NEEDED | + EB_CLEAR_SIZE_CACHE, + vm_nblocks, + RBM_ZERO_ON_ERROR); + + /* + * Send a shared-inval message to force other backends to close any smgr + * references they may have for this rel, which we are about to change. + */ + CacheInvalidateSmgr(RelationGetSmgr(rel)->smgr_rlocator); + + return buf; +} + +/* + * flux_vm_readbuf - Read or extend the VM to get the page for blkno. + * + * If extend is true and the block doesn't exist, extends the fork + * (creating it if needed). Returns InvalidBuffer if extend is false + * and the block doesn't exist. Buffer is returned pinned but not locked. + */ +static Buffer +flux_vm_readbuf(Relation rel, BlockNumber blkno, bool extend) +{ + Buffer buf; + SMgrRelation reln = RelationGetSmgr(rel); + + /* + * Ensure we have the cached nblocks value for the VM fork. + */ + if (reln->smgr_cached_nblocks[VISIBILITYMAP_FORKNUM] == InvalidBlockNumber) + { + if (smgrexists(reln, VISIBILITYMAP_FORKNUM)) + smgrnblocks(reln, VISIBILITYMAP_FORKNUM); + else + reln->smgr_cached_nblocks[VISIBILITYMAP_FORKNUM] = 0; + } + + if (blkno >= reln->smgr_cached_nblocks[VISIBILITYMAP_FORKNUM]) + { + if (extend) + buf = flux_vm_extend(rel, blkno + 1); + else + return InvalidBuffer; + } + else + buf = ReadBufferExtended(rel, VISIBILITYMAP_FORKNUM, blkno, + RBM_ZERO_ON_ERROR, NULL); + + /* + * Initializing the page when needed is trickier than it looks, because of + * the possibility of multiple backends doing this concurrently, and our + * desire to not uselessly take the buffer lock in the normal path where + * the page is OK. For a page that's just been extended, this is not + * needed since it was already initialized by ExtendBufferedRelTo. + */ + if (PageIsNew(BufferGetPage(buf))) + { + LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE); + if (PageIsNew(BufferGetPage(buf))) + PageInit(BufferGetPage(buf), BLCKSZ, 0); + LockBuffer(buf, BUFFER_LOCK_UNLOCK); + } + + return buf; +} + +/* + * FluxVMUpdateForInsert - Update VM after inserting a tuple + * + * When we insert a tuple into a page, we may need to clear the all-visible + * and all-frozen bits if the new tuple is not immediately visible to all + * transactions. + */ +void +FluxVMUpdateForInsert(Relation rel, FluxTupleHeader *tuple, Buffer buffer) +{ + BlockNumber blkno = BufferGetBlockNumber(buffer); + + /* + * Check if the new tuple affects the page's visibility status. In FLUX's + * timestamp-based MVCC, a tuple is visible to all if its commit timestamp + * is older than the oldest active transaction. + * + * A future optimization could check if the new tuple is already visible + * to all transactions (e.g., a bulk load with old timestamps). For now, + * conservatively clear the bits on any insert. + */ + (void) tuple; /* reserved for future timestamp checking */ + + /* Clear in-page flag first (zero cost, no I/O) */ + PageClearAllVisible(BufferGetPage(buffer)); + + FluxVMClear(rel, blkno, buffer, FLUX_VM_VALID_BITS); +} + +/* + * FluxVMUpdateForUpdate - Update VM after updating a tuple + * + * Updates always clear the all-visible and all-frozen bits because they + * create a new tuple version that may not be immediately visible. + */ +void +FluxVMUpdateForUpdate(Relation rel, Buffer buffer) +{ + BlockNumber blkno = BufferGetBlockNumber(buffer); + + /* Clear in-page flag first (zero cost, no I/O) */ + PageClearAllVisible(BufferGetPage(buffer)); + + /* Clear both bits - the page now has a new tuple version */ + FluxVMClear(rel, blkno, buffer, FLUX_VM_VALID_BITS); +} + +/* + * FluxVMUpdateForDelete - Update VM after deleting a tuple + * + * Deletes clear the all-visible bit because the deleted tuple may still + * be visible to some transactions. + */ +void +FluxVMUpdateForDelete(Relation rel, Buffer buffer) +{ + BlockNumber blkno = BufferGetBlockNumber(buffer); + + /* Clear in-page flag first (zero cost, no I/O) */ + PageClearAllVisible(BufferGetPage(buffer)); + + /* + * Clear BOTH bits. ALL_FROZEN implies ALL_VISIBLE, so the frozen bit + * must never outlive the visible bit. Leaving ALL_FROZEN set on a page + * with a freshly deleted tuple would cause VACUUM Phase I to skip the + * page (the all-frozen fast path), so the deleted tuple's storage and -- + * critically -- its index entries would never be cleaned, eventually + * allowing TID recycling with stale index entries still present. + */ + FluxVMClear(rel, blkno, buffer, FLUX_VM_VALID_BITS); +} + +/* + * FluxVMVacuumPage - Update VM during VACUUM + * + * This is called by VACUUM after processing a page to set the appropriate + * visibility map bits based on the page's contents. + */ +void +FluxVMVacuumPage(Relation rel, Buffer buffer, bool all_visible, bool all_frozen) +{ + BlockNumber blkno = BufferGetBlockNumber(buffer); + uint8 flags = 0; + + if (all_visible) + flags |= FLUX_VM_ALL_VISIBLE; + if (all_frozen) + flags |= FLUX_VM_ALL_FROZEN; + + if (flags != 0) + FluxVMSet(rel, blkno, buffer, flags); + + /* + * Synchronize the in-page PD_ALL_VISIBLE flag with the VM. Use + * MarkBufferDirtyHint since losing this flag on crash is benign (just + * falls back to VM check on next scan; VACUUM will re-set it). + */ + if (all_visible) + { + if (!PageIsAllVisible(BufferGetPage(buffer))) + { + PageSetAllVisible(BufferGetPage(buffer)); + MarkBufferDirtyHint(buffer, true); + } + } + else + { + if (PageIsAllVisible(BufferGetPage(buffer))) + { + PageClearAllVisible(BufferGetPage(buffer)); + MarkBufferDirtyHint(buffer, true); + } + } +} diff --git a/src/backend/access/flux/flux_xlog.c b/src/backend/access/flux/flux_xlog.c new file mode 100644 index 0000000000000..9701eb96cf9df --- /dev/null +++ b/src/backend/access/flux/flux_xlog.c @@ -0,0 +1,2761 @@ +/*------------------------------------------------------------------------- + * + * flux_xlog.c + * FLUX WAL (Write-Ahead Logging) implementation + * + * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/access/flux/flux_xlog.c + * + * NOTES + * This implements WAL logging for FLUX operations, providing + * UNDO/REDO functionality for crash recovery. Unlike heap, + * FLUX uses in-place updates with before/after images. + * + * PANIC policy during redo + * ------------------------ + * The per-opcode redo helpers below use elog(PANIC, ...) for any + * invariant violation detected during WAL replay. This is + * deliberate: a mismatch between the WAL stream and the on-disk + * state (truncated overflow payload, failure to add a tuple the + * forward path just wrote, a page full the forward path just + * defragmented, an unknown opcode) is not a recoverable condition. + * Downgrading these sites to ERROR would promote silent divergence + * between the primary and a standby, or between the on-disk + * heap state and the WAL record that described it; PANIC forces + * a postmaster-wide restart and, in the standby case, marks the + * standby inconsistent. Each PANIC site is therefore guarded by + * logic that only fires on actually-corrupt input; fixing a PANIC + * that fires in practice is a correctness bug in the forward + * path, not a reason to soften the guard. + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/htup_details.h" +#include "access/flux.h" +#include "access/flux_xlog.h" +#include "access/bufmask.h" +#include "access/relundo.h" +#include "access/relundo_xlog.h" +#include "access/slog.h" +#include "access/xlog.h" +#include "access/xloginsert.h" +#include "access/xlogrecord.h" +#include "access/xlogutils.h" +#include "storage/buf_internals.h" +#include "storage/bufmgr.h" +#include "miscadmin.h" +#include "utils/rel.h" +#include "utils/timestamp.h" + +/* + * FluxXLogMaybeAppendLogicalTuple + * Append a heap-format image of `rtup` to the in-progress WAL record + * if `rel` is logically logged. Returns true and sets + * FLUX_WAL_LOGICAL_TUPLE in `*flags` if the image was appended. + * + * The heap image is what logical decoding consumes. Physical REDO + * reads the FLUX-format tuple that precedes this region. By writing + * both, we avoid the need for decode.c to call RelidByRelfilenumber / + * RelationIdGetRelation, which are unsafe before SetupHistoricSnapshot. + * + * The image is appended at the END of the main WAL data channel with + * the length trailing the bytes: + * + * ... [heap bytes] [uint32 heap_len] + * + * So the decoder can read heap_len from (end-4) and back up heap_len + * bytes to find the heap payload, regardless of what precedes it in + * the record (which may vary with compression / cross-page). + * + * For UPDATE we append two back-to-back trailers (old then new); see + * FluxXLogUpdate. + */ +void +FluxXLogPrepareLogicalImage(Relation rel, FluxTuple rtup, + FluxLogicalImage *img) +{ + TupleDesc tupdesc; + Datum *values; + bool *isnull; + HeapTuple heaptup; + + img->data = NULL; + img->len = 0; + + if (rel == NULL || rtup == NULL || !RelationIsLogicallyLogged(rel)) + return; + + tupdesc = RelationGetDescr(rel); + values = (Datum *) palloc(tupdesc->natts * sizeof(Datum)); + isnull = (bool *) palloc(tupdesc->natts * sizeof(bool)); + + FluxDeformTuple(rel, rtup, tupdesc, values, isnull); + heaptup = heap_form_tuple(tupdesc, values, isnull); + + /* + * Copy the heap tuple body into an image buffer that outlives the + * critical section. XLogRegisterData() only records pointers, so the + * bytes must remain valid until XLogInsert() reads them. + */ + img->len = (uint32) heaptup->t_len; + img->data = (char *) palloc(img->len); + memcpy(img->data, heaptup->t_data, img->len); + + heap_freetuple(heaptup); + pfree(values); + pfree(isnull); +} + +void +FluxXLogReleaseLogicalImage(FluxLogicalImage *img) +{ + if (img->data != NULL) + { + pfree(img->data); + img->data = NULL; + } + img->len = 0; +} + +/* + * FluxXLogRegisterLogicalImage + * Register a previously prepared heap-format image onto the in-progress + * WAL record. Allocation-free: safe to call inside a critical section. + * Appends "[heap bytes][uint32 heap_len]" to the main data channel and + * sets FLUX_WAL_LOGICAL_TUPLE in *flags. No-op when img is NULL or + * empty (relation not logically logged). + */ +static void +FluxXLogRegisterLogicalImage(FluxLogicalImage *img, uint16 *flags) +{ + if (img == NULL || img->data == NULL) + return; + + XLogRegisterData(img->data, img->len); + XLogRegisterData((char *) &img->len, sizeof(uint32)); + + *flags |= FLUX_WAL_LOGICAL_TUPLE; +} + +/* ---------------------------------------------------------------- + * WAL Record Logging Functions + * ---------------------------------------------------------------- + */ + +/* + * Log a tuple insert operation. + */ +XLogRecPtr +FluxXLogInsert(Relation rel, Buffer buffer, OffsetNumber offnum, + FluxTuple tuple, uint64 commit_ts, + FluxOverflowBuffers *overflow_buffers, + FluxLogicalImage *logical_img, + bool force_page_image) +{ + xl_flux_insert xlrec; + XLogRecPtr recptr; + Page page = BufferGetPage(buffer); + uint8 info = XLOG_FLUX_INSERT; + uint8 main_buf_flags = REGBUF_STANDARD; + int i; + xl_flux_overflow_write ovf_xlrecs[MAX_OVERFLOW_BUFFERS]; + + /* + * The multi-insert (COPY/bulk) path adds many tuples to one page but + * emits a single INSERT record describing only the first tuple. Under + * full_page_writes the page's initial touch captures the whole page, so + * redo restores every tuple. With full_page_writes off there is no such + * image, and the unlogged tuples vanish on crash recovery, leaving later + * CAS_UPDATE redo to PANIC on a missing item. Force a full-page image so + * the batch is crash-safe regardless of full_page_writes. + */ + if (force_page_image) + main_buf_flags |= REGBUF_FORCE_IMAGE; + + /* Fill in the insert record */ + xlrec.offnum = offnum; + xlrec.flags = 0; + xlrec.tuple_len = tuple->t_len; + xlrec.commit_ts = commit_ts; + + /* + * NOTE: XLogEnsureRecordSpace() has already been called by the caller + * (before entering the critical section) to pre-allocate space for the + * main buffer plus all overflow buffers. + */ + XLogBeginInsert(); + + /* Register buffer FIRST, before any data */ + XLogRegisterBuffer(0, buffer, main_buf_flags); + + /* + * Register all overflow buffers (buffers 1..N) for atomic WAL logging. + * This ensures the main tuple and all overflow records are restored + * together during crash recovery, preventing orphaned overflow pages. + * + * IMPORTANT: Due to spatial locality optimization, multiple overflow + * records may reside on the same page. We must register each unique + * buffer only once, but register data for all overflow records. + * + * For each overflow buffer, we need to include: 1. The offset where the + * record should be placed 2. The actual overflow record data + */ + if (overflow_buffers != NULL) + { + int registered_block_id = 1; /* Start after main buffer */ + Buffer registered_buffers[MAX_OVERFLOW_BUFFERS]; + int registered_buffer_ids[MAX_OVERFLOW_BUFFERS]; + int num_registered = 0; + + /* + * NOTE: ovf_xlrecs[] is declared at function scope (not here) so that + * the pointers registered via XLogRegisterBufData() remain valid + * until XLogInsert() is called after this block ends. + */ + + /* Ensure we don't exceed PostgreSQL's hard limit */ + if (overflow_buffers->count > XLR_MAX_BLOCK_ID) + elog(ERROR, "too many overflow records: %d (max %d)", + overflow_buffers->count, XLR_MAX_BLOCK_ID); + + for (i = 0; i < overflow_buffers->count; i++) + { + FluxOverflowBuffer *ovb = &overflow_buffers->buffers[i]; + int block_id = -1; + int j; + + /* + * First check if this overflow buffer is the SAME as the main + * buffer. This can happen when spatial locality places an + * overflow record on the same page as the main tuple. In this + * case, reuse block_id=0. + */ + if (ovb->buffer == buffer) + { + block_id = 0; + xlrec.flags |= FLUX_WAL_HAS_OVERFLOW_BLK0; + } + else + { + /* + * Check if this buffer was already registered among overflow + * buffers (spatial locality: multiple overflow records on + * same page). If so, reuse its block_id instead of + * registering again. + */ + for (j = 0; j < num_registered; j++) + { + if (registered_buffers[j] == ovb->buffer) + { + block_id = registered_buffer_ids[j]; + break; + } + } + + /* If not registered yet, register it now */ + if (block_id < 0) + { + block_id = registered_block_id++; + + /* + * Force a full-page image for overflow buffers. See + * FluxXLogUpdate for the rationale. + */ + XLogRegisterBuffer(block_id, ovb->buffer, + REGBUF_STANDARD | REGBUF_FORCE_IMAGE); + + /* Track this buffer so we don't register it again */ + registered_buffers[num_registered] = ovb->buffer; + registered_buffer_ids[num_registered] = block_id; + num_registered++; + } + } + + /* + * Create a proper xl_flux_overflow_write header with the offset. + * This tells the redo handler where to place the record. Each + * header is stored in a dedicated array slot so the pointer + * passed to XLogRegisterBufData remains valid until XLogInsert. + */ + ovf_xlrecs[i].offnum = ovb->offset; + ovf_xlrecs[i].flags = ovb->flags; + ovf_xlrecs[i].data_len = ovb->record_len; + ovf_xlrecs[i].commit_ts = commit_ts; + + /* Register the header first, then the data */ + XLogRegisterBufData(block_id, (char *) &ovf_xlrecs[i], sizeof(xl_flux_overflow_write)); + XLogRegisterBufData(block_id, ovb->record_data, ovb->record_len); + } + } + + /* Now register the main data */ + XLogRegisterData((char *) &xlrec, sizeof(xl_flux_insert)); + XLogRegisterData((char *) tuple->t_data, tuple->t_len); + + FluxXLogRegisterLogicalImage(logical_img, &xlrec.flags); + + recptr = XLogInsert(RM_FLUX_ID, info); + + /* Set LSN on main page */ + PageSetLSN(page, recptr); + + /* + * Set LSN on all overflow pages. Due to spatial locality, some buffers + * may appear multiple times in overflow_buffers. PageSetLSN is idempotent + * (setting the same LSN multiple times is safe), so we can just iterate + * through all entries without checking for duplicates. + */ + if (overflow_buffers != NULL) + { + for (i = 0; i < overflow_buffers->count; i++) + { + Page ovpage = BufferGetPage(overflow_buffers->buffers[i].buffer); + + PageSetLSN(ovpage, recptr); + } + } + + return recptr; +} + +/* + * Log a batched multi-tuple insert (COPY/bulk load) into one page. + * + * Unlike FluxXLogInsert, this records EVERY tuple body in the batch rather + * than relying on a forced full-page image, so the batch replays correctly + * regardless of full_page_writes. No overflow buffers are involved: tuples + * that need overflow are routed to the single-insert path by the caller. + * + * WAL main-data layout: + * [xl_flux_multi_insert header] + * ntuples * [xl_flux_multi_insert_tuple header][tuple t_data body] + * ntuples * [logical image bytes][uint32 len] (only when logically logged) + */ +XLogRecPtr +FluxXLogMultiInsert(Relation rel, Buffer buffer, + OffsetNumber *offnums, FluxTuple *tuples, + int ntuples, uint64 commit_ts, + FluxLogicalImage *logical_imgs) +{ + xl_flux_multi_insert xlrec; + + /* + * Serialize the whole per-tuple region into one scratch buffer and + * register it as a single rdata chunk. XLogRegisterData() is limited to + * XLR_NORMAL_RDATAS (20) slots, and XLogEnsureRecordSpace() cannot grow + * that inside a critical section, so registering two chunks per tuple + * would overflow the slot array once a page packs more than a handful of + * rows. Mirrors heap_multi_insert()'s single-scratch-buffer pattern; the + * on-WAL bytes are identical to per-tuple registration because redo and + * logical decoding read the region as one contiguous XLogRecGetData() + * stream. + * + * The region is bounded by BLCKSZ: every tuple consumed + * sizeof(ItemIdData) + MAXALIGN(t_len) on the page, so the sum of + * SizeOfFluxMultiInsertTuple + t_len across the batch is strictly less + * than one page. + */ + PGAlignedBlock scratch; + char *scratchptr = scratch.data; + XLogRecPtr recptr; + Page page = BufferGetPage(buffer); + uint8 info = XLOG_FLUX_MULTI_INSERT; + int i; + + Assert(ntuples > 0); + Assert(ntuples <= MaxOffsetNumber); + + xlrec.ntuples = (uint16) ntuples; + xlrec.flags = 0; + xlrec.commit_ts = commit_ts; + + for (i = 0; i < ntuples; i++) + { + xl_flux_multi_insert_tuple *tuphdr; + + /* + * SHORTALIGN each per-tuple header so its uint16 fields land on an + * even offset, mirroring heap_multi_insert(). Readers (redo and + * logical decode) advance with the identical SHORTALIGN, so the + * on-WAL stride stays byte-for-byte in lockstep. + */ + scratchptr = (char *) SHORTALIGN(scratchptr); + tuphdr = (xl_flux_multi_insert_tuple *) scratchptr; + + tuphdr->offnum = offnums[i]; + tuphdr->datalen = (uint16) tuples[i]->t_len; + scratchptr += SizeOfFluxMultiInsertTuple; + + memcpy(scratchptr, (char *) tuples[i]->t_data, tuples[i]->t_len); + scratchptr += tuples[i]->t_len; + } + Assert((scratchptr - scratch.data) < BLCKSZ); + + XLogBeginInsert(); + + /* + * Register the page WITHOUT forcing a full-page image. Crash safety + * comes from logging every tuple body above, not from an FPI. + */ + XLogRegisterBuffer(0, buffer, REGBUF_STANDARD); + + XLogRegisterData((char *) &xlrec, SizeOfFluxMultiInsert); + XLogRegisterData(scratch.data, (uint32) (scratchptr - scratch.data)); + + /* + * Append one logical-decoding image per tuple (when logically logged). + * The caller pre-serialized all images into one contiguous blob before + * entering the critical section (logical_imgs[0].data), so a single + * registration covers the whole region and keeps the rdata slot count + * constant. + */ + if (logical_imgs != NULL && logical_imgs[0].data != NULL) + { + XLogRegisterData(logical_imgs[0].data, logical_imgs[0].len); + xlrec.flags |= FLUX_WAL_LOGICAL_TUPLE; + } + + recptr = XLogInsert(RM_FLUX_ID, info); + + PageSetLSN(page, recptr); + + return recptr; +} + +/* + * Log a tuple update operation (in-place with before/after images) + */ +XLogRecPtr +FluxXLogUpdate(Relation rel, Buffer buffer, OffsetNumber offnum, + FluxTuple old_tuple, FluxTuple new_tuple, + uint64 old_commit_ts, uint64 new_commit_ts, + FluxOverflowBuffers *overflow_buffers, + Buffer new_buffer, + FluxLogicalImage *old_img, + FluxLogicalImage *new_img) +{ + xl_flux_update xlrec; + xl_flux_prefix_suffix ps; + XLogRecPtr recptr; + uint8 info = XLOG_FLUX_UPDATE_INPLACE; + int i; + bool is_cross_page = (BufferIsValid(new_buffer) && + new_buffer != buffer); + + /* Fill in the update record */ + xlrec.offnum = offnum; + xlrec.flags = 0; + xlrec.old_commit_ts = old_commit_ts; + xlrec.new_commit_ts = new_commit_ts; + xlrec.old_tuple_len = (uint16) old_tuple->t_len; + xlrec.new_tuple_len = (uint16) new_tuple->t_len; + xlrec.dst_block_id = 0; + memset(xlrec.pad, 0, sizeof(xlrec.pad)); + + if (is_cross_page) + xlrec.flags |= FLUX_WAL_CROSS_PAGE; + + /* + * NOTE: XLogEnsureRecordSpace() has already been called by the caller + * (before entering the critical section) to pre-allocate space for the + * main buffer plus all overflow buffers. + */ + XLogBeginInsert(); + + /* + * Register the source buffer (block 0). + * + * A same-page update rewrites the tuple in place at the same offset: redo + * overwrites the slot with the new tuple data (or patches the + * prefix/suffix diff), so no full-page image is required. An image is + * only needed when the new tuple is larger than the old one, because the + * write path then deletes and re-adds the line pointer (shifting page + * data), which redo cannot replay from the new tuple bytes alone. The + * predicate new_tuple->t_len > old_tuple->t_len is a safe superset of + * that grow case: the on-page slot is always at least old_tuple->t_len, + * so any update that outgrows the slot also outgrows old_tuple->t_len and + * forces the image. + * + * For cross-page updates the redo handler marks the old tuple UPDATED + * directly (see FLUX_WAL_CROSS_PAGE handling in flux_redo) and the new + * tuple is restored from the destination page's image, so the same + * grow-only image rule applies to block 0. + */ + { + uint8 buf_flags = REGBUF_STANDARD; + + if (new_tuple->t_len > old_tuple->t_len) + buf_flags |= REGBUF_FORCE_IMAGE; + + XLogRegisterBuffer(0, buffer, buf_flags); + } + + /* + * Register all overflow buffers (buffers 1..N) for atomic WAL logging. + * This ensures the main tuple UPDATE and all overflow records are + * restored together during crash recovery, preventing orphaned overflow + * pages. + * + * IMPORTANT: Due to spatial locality optimization, multiple overflow + * records may reside on the same page. We must register each unique + * buffer only once, but register data for all overflow records. + */ + { + int next_block_id = 1; /* Start after main buffer (block 0) */ + + if (overflow_buffers != NULL) + { + Buffer registered_buffers[MAX_OVERFLOW_BUFFERS]; + int registered_buffer_ids[MAX_OVERFLOW_BUFFERS]; + int num_registered = 0; + + /* Ensure we don't exceed PostgreSQL's hard limit */ + if (overflow_buffers->count > XLR_MAX_BLOCK_ID) + elog(ERROR, "too many overflow records: %d (max %d)", + overflow_buffers->count, XLR_MAX_BLOCK_ID); + + for (i = 0; i < overflow_buffers->count; i++) + { + FluxOverflowBuffer *ovb = &overflow_buffers->buffers[i]; + int block_id = -1; + int j; + + /* + * First check if this overflow buffer is the SAME as the main + * buffer. This can happen when spatial locality places an + * overflow record on the same page as the main tuple. In this + * case, reuse block_id=0. + */ + if (ovb->buffer == buffer) + { + block_id = 0; + xlrec.flags |= FLUX_WAL_HAS_OVERFLOW_BLK0; + } + else + { + /* + * Check if this buffer was already registered among + * overflow buffers (spatial locality: multiple overflow + * records on same page). If so, reuse its block_id + * instead of registering again. + */ + for (j = 0; j < num_registered; j++) + { + if (registered_buffers[j] == ovb->buffer) + { + block_id = registered_buffer_ids[j]; + break; + } + } + + /* If not registered yet, register it now */ + if (block_id < 0) + { + block_id = next_block_id++; + + /* + * Force a full-page image for overflow buffers. The + * redo handler for overflow pages reconstructs items + * using PageAddItem, but the page layout can differ + * from the primary when the page already contains + * items from prior operations (e.g., free space + * fragmentation, item alignment). Using + * REGBUF_FORCE_IMAGE guarantees the page is restored + * exactly as the primary had it. + */ + XLogRegisterBuffer((uint8) block_id, ovb->buffer, + REGBUF_STANDARD | REGBUF_FORCE_IMAGE); + + /* Track this buffer so we don't register it again */ + registered_buffers[num_registered] = ovb->buffer; + registered_buffer_ids[num_registered] = block_id; + num_registered++; + } + } + + /* Register overflow record data for this buffer */ + XLogRegisterBufData((uint8) block_id, ovb->record_data, + ovb->record_len); + } + } + + /* + * For cross-page out-of-place updates, register the destination + * buffer so both pages are crash-safe. Force a full-page image so + * redo simply restores the page without needing replay logic. + */ + if (is_cross_page) + { + xlrec.dst_block_id = (uint8) next_block_id; + XLogRegisterBuffer((uint8) next_block_id, new_buffer, + REGBUF_STANDARD | REGBUF_FORCE_IMAGE); + next_block_id++; + } + } + + /* Now register the main data */ + XLogRegisterData((char *) &xlrec, sizeof(xl_flux_update)); + + /* + * Log only new tuple for REDO. Old tuple data is stored exclusively in + * the shared UNDO log (UNDO_RMID_FLUX record written via + * UndoBufferAddRecordParts) and is not needed during WAL replay: + * + * - Same-size/shrinking updates: redo overwrites the slot in place using + * only the new tuple data. - Growing updates: REGBUF_FORCE_IMAGE is set + * above, so redo restores the page from a full-page image and never + * enters BLK_NEEDS_REDO. + * + * Prefix/suffix compression: For same-size in-place updates, we compute + * the common prefix and suffix between old and new tuple data. If the + * savings exceed sizeof(xl_flux_prefix_suffix) (4 bytes), we log only the + * changed bytes plus a small header. The redo handler reconstructs the + * full new tuple from the existing page data + diff. + * + * This is only safe for same-size updates without cross-page moves. + * Growing updates use REGBUF_FORCE_IMAGE and never enter BLK_NEEDS_REDO. + */ + if (!is_cross_page && + old_tuple->t_len == new_tuple->t_len && + new_tuple->t_len > 0) + { + char *oldp = (char *) old_tuple->t_data; + char *newp = (char *) new_tuple->t_data; + int len = new_tuple->t_len; + int difflen; + + /* Compute common prefix */ + for (ps.prefixlen = 0; ps.prefixlen < len; ps.prefixlen++) + if (oldp[ps.prefixlen] != newp[ps.prefixlen]) + break; + + /* Compute common suffix (don't overlap with prefix) */ + for (ps.suffixlen = 0; + ps.suffixlen < len - ps.prefixlen; + ps.suffixlen++) + if (oldp[len - 1 - ps.suffixlen] != newp[len - 1 - ps.suffixlen]) + break; + + difflen = len - ps.prefixlen - ps.suffixlen; + + /* + * Use compression only if the savings exceed the header overhead. The + * header is 4 bytes (two uint16s), so we need the prefix + suffix to + * save more than that. + */ + if (ps.prefixlen + ps.suffixlen > (int) sizeof(xl_flux_prefix_suffix) && + difflen >= 0) + { + xlrec.flags |= FLUX_WAL_PREFIX_SUFFIX; + XLogRegisterData((char *) &ps, sizeof(xl_flux_prefix_suffix)); + if (difflen > 0) + XLogRegisterData(newp + ps.prefixlen, difflen); + } + else + { + /* Not worth compressing, log full new tuple */ + XLogRegisterData((char *) new_tuple->t_data, new_tuple->t_len); + } + } + else + { + /* Cross-page or size-changing: log full new tuple */ + XLogRegisterData((char *) new_tuple->t_data, new_tuple->t_len); + } + + /* + * Append heap-format images of old + new tuples for logical decoding. + * Order: old first, then new. Flag is set uniformly on both or neither. + * The images were prepared by the caller before the critical section. + */ + { + uint16 tmpflag = 0; + + FluxXLogRegisterLogicalImage(old_img, &tmpflag); + FluxXLogRegisterLogicalImage(new_img, &tmpflag); + xlrec.flags |= tmpflag; + } + + recptr = XLogInsert(RM_FLUX_ID, info); + + return recptr; +} + +/* + * Log a tuple delete operation + */ +XLogRecPtr +FluxXLogDelete(Relation rel, Buffer buffer, OffsetNumber offnum, + FluxTuple tuple, uint64 commit_ts, + FluxLogicalImage *logical_img) +{ + xl_flux_delete xlrec; + XLogRecPtr recptr; + Page page = BufferGetPage(buffer); + uint8 info = XLOG_FLUX_DELETE; + + /* Fill in the delete record */ + xlrec.offnum = offnum; + xlrec.flags = 0; + xlrec.tuple_len = tuple->t_len; + xlrec.commit_ts = commit_ts; + + XLogBeginInsert(); + + /* Register buffer FIRST, before any data */ + XLogRegisterBuffer(0, buffer, REGBUF_STANDARD); + + /* + * Register delete header only -- old tuple data is stored exclusively in + * the UNDO fork. The redo handler only needs the offset and commit_ts to + * set FLUX_TUPLE_DELETED on the existing tuple. + */ + XLogRegisterData((char *) &xlrec, sizeof(xl_flux_delete)); + + /* + * Append heap-format image of the deleted tuple for logical decoding. + * DELETE's REDO path doesn't need the old tuple image on-page (it just + * flips a flag), so this region is strictly for the decode side. The + * image was prepared by the caller before the critical section. + */ + FluxXLogRegisterLogicalImage(logical_img, &xlrec.flags); + + recptr = XLogInsert(RM_FLUX_ID, info); + + PageSetLSN(page, recptr); + + return recptr; +} + +/* + * Log page defragmentation + */ +XLogRecPtr +FluxXLogDefrag(Relation rel, Buffer buffer, FluxOffsetMapping *mappings, + int nmappings, uint64 commit_ts) +{ + xl_flux_defrag xlrec; + XLogRecPtr recptr; + Page page = BufferGetPage(buffer); + uint8 info = XLOG_FLUX_DEFRAG; + + /* Fill in the defrag record */ + xlrec.ntuples = nmappings; + xlrec.commit_ts = commit_ts; + + XLogBeginInsert(); + XLogRegisterData((char *) &xlrec, sizeof(xl_flux_defrag)); + XLogRegisterData((char *) mappings, sizeof(FluxOffsetMapping) * nmappings); + + /* + * Force a full-page image. The caller may have removed dead tuples + * (ItemIdSetUnused) before compaction, and those removals are not encoded + * in the DEFRAG WAL record. Without an FPI the redo handler would call + * PageRepairFragmentation() on a page that still contains the dead + * tuples, producing a page inconsistent with the primary. + */ + XLogRegisterBuffer(0, buffer, REGBUF_STANDARD | REGBUF_FORCE_IMAGE); + + recptr = XLogInsert(RM_FLUX_ID, info); + + PageSetLSN(page, recptr); + + return recptr; +} + +/* + * Log overflow record write. + * + * The caller must already hold an exclusive lock on the buffer and have + * written the overflow record data to the page. We log either a new + * overflow record (header + data) or a link update (header only). + * + * buffer: already-locked buffer containing the overflow record + * offnum: offset of the overflow record on the page + * record_data: pointer to the record data to log (header, or header+data) + * record_len: length of data to log + * flags: FLUX_OVERFLOW_WAL_NEW_RECORD or FLUX_OVERFLOW_WAL_LINK_UPDATE + * commit_ts: commit timestamp + */ +XLogRecPtr +FluxXLogOverflowWrite(Relation rel, Buffer buffer, OffsetNumber offnum, + char *record_data, uint32 record_len, uint16 flags, + uint64 commit_ts) +{ + xl_flux_overflow_write xlrec; + XLogRecPtr recptr; + Page page = BufferGetPage(buffer); + uint8 info = XLOG_FLUX_OVERFLOW_WRITE; + + /* Fill in the overflow write record */ + xlrec.offnum = offnum; + xlrec.flags = flags; + xlrec.data_len = record_len; + xlrec.commit_ts = commit_ts; + + XLogBeginInsert(); + XLogRegisterData((char *) &xlrec, sizeof(xl_flux_overflow_write)); + XLogRegisterData(record_data, record_len); + XLogRegisterBuffer(0, buffer, REGBUF_STANDARD); + + recptr = XLogInsert(RM_FLUX_ID, info); + + PageSetLSN(page, recptr); + + return recptr; +} + +/* + * Log attribute compression + */ +XLogRecPtr +FluxXLogCompress(Relation rel, Buffer buffer, OffsetNumber offnum, + uint16 attr_num, FluxCompressionType comp_type, + uint8 comp_level, char *comp_data, + uint32 orig_size, uint32 comp_size, uint64 commit_ts) +{ + xl_flux_compress xlrec; + XLogRecPtr recptr; + Page page = BufferGetPage(buffer); + uint8 info = XLOG_FLUX_COMPRESS; + + /* Fill in the compress record */ + xlrec.offnum = offnum; + xlrec.attr_num = attr_num; + xlrec.comp_type = comp_type; + xlrec.comp_level = comp_level; + xlrec.orig_size = orig_size; + xlrec.comp_size = comp_size; + xlrec.commit_ts = commit_ts; + + XLogBeginInsert(); + XLogRegisterData((char *) &xlrec, sizeof(xl_flux_compress)); + XLogRegisterData(comp_data, comp_size); + XLogRegisterBuffer(0, buffer, REGBUF_STANDARD); + + recptr = XLogInsert(RM_FLUX_ID, info); + + PageSetLSN(page, recptr); + + return recptr; +} + +/* + * Log page initialization + */ +XLogRecPtr +FluxXLogInitPage(Relation rel, Buffer buffer, uint32 flags, uint64 commit_ts) +{ + xl_flux_init_page xlrec; + XLogRecPtr recptr; + Page page = BufferGetPage(buffer); + uint8 info = XLOG_FLUX_INIT_PAGE; + + /* Fill in the init page record */ + xlrec.flags = flags; + xlrec.commit_ts = commit_ts; + + XLogBeginInsert(); + + /* Register buffer FIRST, before any data */ + XLogRegisterBuffer(0, buffer, REGBUF_WILL_INIT | REGBUF_STANDARD); + + /* Now register the data */ + XLogRegisterData((char *) &xlrec, sizeof(xl_flux_init_page)); + + recptr = XLogInsert(RM_FLUX_ID, info); + + PageSetLSN(page, recptr); + + return recptr; +} + +/* + * Log a cross-page defragmentation move. + * + * This logs the move of a single tuple from a source page (block ref 1) + * to a target page (block ref 0). Both pages are registered so that + * full-page images will be taken if needed. The tuple data is included + * in the record so that recovery can replay the move even without FPIs. + */ +XLogRecPtr +FluxXLogCrossPageDefrag(Relation rel, + Buffer dst_buf, OffsetNumber dst_offnum, + Buffer src_buf, OffsetNumber src_offnum, + const void *tuple_data, uint32 tuple_len) +{ + xl_flux_cross_page_defrag xlrec; + XLogRecPtr recptr; + + xlrec.src_offnum = src_offnum; + xlrec.dst_offnum = dst_offnum; + xlrec.tuple_len = tuple_len; + + XLogBeginInsert(); + XLogRegisterData((char *) &xlrec, sizeof(xl_flux_cross_page_defrag)); + XLogRegisterData((char *) tuple_data, tuple_len); + XLogRegisterBuffer(0, dst_buf, REGBUF_STANDARD | REGBUF_FORCE_IMAGE); + XLogRegisterBuffer(1, src_buf, REGBUF_STANDARD); + + recptr = XLogInsert(RM_FLUX_ID, XLOG_FLUX_CROSS_PAGE_DEFRAG); + + return recptr; +} + +/* + * FluxXLogCasUpdate -- WAL record for same-size CAS in-place update. + * + * Logs only the changed byte range within the tuple. This is the minimal + * WAL record for the tuple-level CAS fast path where the entire tuple does + * not need to be logged (same size, only data bytes changed). + * + * The caller holds BUFFER_LOCK_SHARE_EXCLUSIVE and the per-tuple t_writer CAS + * lock. We do NOT force a full-page image because: + * (a) the modification is confined to a single tuple's data bytes, and + * (b) the redo handler is idempotent (memcpy of fixed-length data at + * a fixed offset within the tuple). + */ +XLogRecPtr +FluxXLogCasUpdate(Relation rel, Buffer buffer, OffsetNumber offnum, + uint16 data_offset, uint16 data_len, + const char *new_data, uint64 new_commit_ts) +{ + xl_flux_cas_update xlrec; + XLogRecPtr recptr; + Page page = BufferGetPage(buffer); + + xlrec.offnum = offnum; + xlrec.flags = 0; + xlrec.data_offset = data_offset; + xlrec.data_len = data_len; + xlrec.new_commit_ts = new_commit_ts; + + XLogBeginInsert(); + XLogRegisterData((char *) &xlrec, sizeof(xl_flux_cas_update)); + XLogRegisterData(new_data, data_len); + XLogRegisterBuffer(0, buffer, REGBUF_STANDARD); + + recptr = XLogInsert(RM_FLUX_ID, XLOG_FLUX_CAS_UPDATE); + PageSetLSN(page, recptr); + + return recptr; +} + +/* + * FluxXLogCasUpdateUndo -- FOLD variant of FluxXLogCasUpdate. + * + * Emits ONE combined WAL record carrying both the main-fork redo byte-diff + * (block 0, identical to FluxXLogCasUpdate) and the relundo-fork UNDO + * before-image (block 1, identical to what RelUndoFinish would have logged in + * a standalone RM_RELUNDO_ID record), plus the relundo metapage (block 2) when + * the UNDO record started a fresh relundo page. + * + * The caller must hold BOTH the main-fork buffer and the staged undo buffer + * (and the metapage buffer, if is_new_page) exclusively locked inside its + * critical section. RelUndoStage() has already written and dirtied the undo + * page; this function only logs the change and stamps all page LSNs. Returns + * the record LSN. + */ +XLogRecPtr +FluxXLogCasUpdateUndo(Relation rel, Buffer buffer, OffsetNumber offnum, + uint16 data_offset, uint16 data_len, + const char *new_data, uint64 new_commit_ts, + const struct RelUndoStageResult *undo) +{ + xl_flux_cas_update_undo xlrec; + XLogRecPtr recptr; + Page page = BufferGetPage(buffer); + + /* redo half (block 0) */ + xlrec.offnum = offnum; + xlrec.flags = 0; + xlrec.data_offset = data_offset; + xlrec.data_len = data_len; + xlrec.new_commit_ts = new_commit_ts; + + /* undo half (block 1) */ + xlrec.urec_type = undo->urec_type; + xlrec.is_new_page = undo->is_new_page ? 1 : 0; + xlrec.urec_len = undo->urec_len; + xlrec.page_offset = undo->page_offset; + xlrec.new_pd_lower = undo->new_pd_lower; + xlrec.max_xid = undo->max_xid; + + XLogBeginInsert(); + XLogRegisterData((char *) &xlrec, SizeOfFluxCasUpdateUndo); + XLogRegisterData(new_data, data_len); + + /* block 0: main-fork page, redo byte-diff */ + XLogRegisterBuffer(0, buffer, REGBUF_STANDARD); + + /* + * block 1: relundo data page. A freshly allocated page is registered + * WILL_INIT so redo reconstructs it from scratch (the redo handler keys + * off xlrec.is_new_page, not a record-level info bit, because the FLUX + * rmgr consumes the whole info upper-nibble as the opcode); the staged + * block data prepends the RelUndoPageHeaderData in that case + * (RelUndoStage built wal_record_data accordingly). An existing page + * registers the record bytes at page_offset with flag 0 to keep the FPI + * faithful (relundo data pages are non-standard). + */ + if (undo->is_new_page) + XLogRegisterBuffer(1, undo->undo_buffer, REGBUF_WILL_INIT); + else + XLogRegisterBuffer(1, undo->undo_buffer, 0); + + XLogRegisterBufData(1, undo->wal_record_data, undo->wal_record_size); + + /* block 2: relundo metapage, only when a new page was allocated */ + if (undo->is_new_page) + { + Assert(BufferIsValid(undo->metabuf)); + XLogRegisterBuffer(2, undo->metabuf, REGBUF_STANDARD); + } + + recptr = XLogInsert(RM_FLUX_ID, XLOG_FLUX_CAS_UPDATE_UNDO); + + /* stamp all touched pages */ + PageSetLSN(page, recptr); + PageSetLSN(BufferGetPage(undo->undo_buffer), recptr); + if (undo->is_new_page) + PageSetLSN(BufferGetPage(undo->metabuf), recptr); + + return recptr; +} + +/* + * FluxXLogWriteDict -- WAL-log a compression-dictionary fork page. + * + * The dictionary fork uses a non-standard page layout that the redo path + * cannot rebuild from a logical delta, so we register the page as a forced + * full-page image and the redo handler restores it verbatim. The caller + * must already hold the buffer locked and have dirtied it inside the same + * critical section. + */ +XLogRecPtr +FluxXLogWriteDict(Relation rel, Buffer buffer) +{ + xl_flux_write_dict xlrec; + XLogRecPtr recptr; + Page page = BufferGetPage(buffer); + + xlrec.blkno = BufferGetBlockNumber(buffer); + + XLogBeginInsert(); + XLogRegisterData((char *) &xlrec, sizeof(xl_flux_write_dict)); + XLogRegisterBuffer(0, buffer, REGBUF_FORCE_IMAGE); + + recptr = XLogInsert(RM_FLUX_ID, XLOG_FLUX_WRITE_DICT); + PageSetLSN(page, recptr); + + return recptr; +} + +/* + * REDO function for FLUX WAL records + */ +/* ---------------------------------------------------------------- + * Per-opcode REDO handlers. + * + * flux_redo() is the thin dispatcher; the real work for each + * XLOG_FLUX_* opcode lives in a dedicated static helper below. + * ---------------------------------------------------------------- + */ +static void flux_xlog_insert_redo(XLogReaderState *record); +static void flux_xlog_multi_insert_redo(XLogReaderState *record); +static void flux_xlog_update_inplace_redo(XLogReaderState *record); +static void flux_xlog_delete_redo(XLogReaderState *record); +static void flux_xlog_defrag_redo(XLogReaderState *record); +static void flux_xlog_overflow_write_redo(XLogReaderState *record); +static void flux_xlog_compress_redo(XLogReaderState *record); +static void flux_xlog_init_page_redo(XLogReaderState *record); +static void flux_xlog_cross_page_defrag_redo(XLogReaderState *record); +static void flux_xlog_vm_set_redo(XLogReaderState *record); +static void flux_xlog_vm_clear_redo(XLogReaderState *record); +static void flux_xlog_lock_redo(XLogReaderState *record); + +/* + * flux_xlog_insert_redo + * REDO handler for XLOG_FLUX_INSERT. + */ +static void +flux_xlog_insert_redo(XLogReaderState *record) +{ + RelFileLocator rlocator; + BlockNumber blkno; + Buffer buffer; + Page page; + + XLogRecGetBlockTag(record, 0, &rlocator, NULL, &blkno); + + { + xl_flux_insert *xlrec = (xl_flux_insert *) XLogRecGetData(record); + char *tuple_data = (char *) xlrec + sizeof(xl_flux_insert); + FluxTupleHeader *tuple_hdr = (FluxTupleHeader *) tuple_data; + XLogRedoAction action; + OffsetNumber final_offnum = InvalidOffsetNumber; + bool tuple_uncommitted = false; + + action = XLogReadBufferForRedo(record, 0, &buffer); + + /* + * For BLK_RESTORED (FPI), the page already has the tuple at + * xlrec->offnum + */ + if (action == BLK_RESTORED) + final_offnum = xlrec->offnum; + + if (action == BLK_NEEDS_REDO) + { + FluxPageOpaque phdr; + OffsetNumber inserted_offnum; + char *ovf_data; + Size ovf_len; + + page = BufferGetPage(buffer); + + /* + * XLogInitBufferForRedo does standard PageInit for new pages, but + * doesn't set up FLUX opaque space. Initialize it here if needed. + */ + if (PageIsNew(page)) + { + FluxInitPage(page, BufferGetPageSize(buffer)); + } + + /* + * CRITICAL: During normal operation, overflow records are + * inserted BEFORE the main tuple (via FluxStoreOverflowColumn + * called from FluxFormTuple, then FluxPageAddTuple for main). + * This means overflow records get lower offsets (1, 2, 3...) and + * the main tuple gets a higher offset (4, ...). + * + * We MUST replay in the same order. If there are overflow records + * on block_id=0 (same page as main tuple due to spatial + * locality), replay them FIRST before the main tuple. + */ + ovf_data = XLogRecGetBlockData(record, 0, &ovf_len); + if (ovf_data != NULL && ovf_len > 0 && + (xlrec->flags & FLUX_WAL_HAS_OVERFLOW_BLK0)) + { + char *ovf_ptr = ovf_data; + Size ovf_remaining = ovf_len; + + /* + * Block 0 has overflow data. Parse and replay all overflow + * records on this block before the main tuple. Each overflow + * record has format: [xl_flux_overflow_write header][actual + * record data] + */ + while (ovf_remaining > sizeof(xl_flux_overflow_write)) + { + xl_flux_overflow_write *ovf_xlrec = (xl_flux_overflow_write *) ovf_ptr; + char *actual_data = ovf_ptr + sizeof(xl_flux_overflow_write); + Size actual_len = ovf_xlrec->data_len; + OffsetNumber ovf_offnum; + + if (ovf_remaining < sizeof(xl_flux_overflow_write) + actual_len) + elog(PANIC, "FLUX INSERT redo: corrupt overflow data on block 0: " + "ovf_remaining=%zu, sizeof(hdr)=%zu, data_len=%u, " + "total_len=%zu, offnum=%u, flags=%u", + ovf_remaining, sizeof(xl_flux_overflow_write), + (unsigned) actual_len, ovf_len, + (unsigned) ovf_xlrec->offnum, + (unsigned) ovf_xlrec->flags); + + /* + * Use InvalidOffsetNumber to let PageAddItem choose the + * next available offset. This ensures sequential offsets + * matching the original insertion order. + */ + ovf_offnum = PageAddItem(page, actual_data, actual_len, + InvalidOffsetNumber, false, false); + if (ovf_offnum == InvalidOffsetNumber) + { + elog(WARNING, "FLUX INSERT redo: failed to add overflow " + "record on block %u; skipping redo", blkno); + goto insert_skip_tuple; + } + + /* Advance to next overflow record in the block data */ + ovf_ptr += sizeof(xl_flux_overflow_write) + actual_len; + ovf_remaining -= sizeof(xl_flux_overflow_write) + actual_len; + } + } + + /* + * Validate that the record actually carries a tuple body of the + * advertised length before dereferencing it. The speculative + * INSERT writers (INSERT ... ON CONFLICT confirm/abort) + * historically emitted body-less records and relied solely on a + * full-page image; such a record must never reach this non-FPI + * path, but if it does (or tuple_len is otherwise inconsistent + * with the main data), reading PageAddItem(page, tuple_hdr, + * tuple_len) would walk off the end of the WAL record and + * SIGSEGV. Treat a missing/short body as "nothing to replay + * here" rather than crashing recovery. + */ + if (xlrec->tuple_len == 0 || + XLogRecGetDataLen(record) < + sizeof(xl_flux_insert) + xlrec->tuple_len) + { + elog(WARNING, "FLUX INSERT redo: record on block %u lacks a " + "tuple body (data_len=%u, need=%zu); skipping tuple redo", + blkno, XLogRecGetDataLen(record), + sizeof(xl_flux_insert) + (Size) xlrec->tuple_len); + goto insert_skip_tuple; + } + + /* + * Now replay the main tuple. Use InvalidOffsetNumber to let + * PageAddItem choose the next sequential offset after any + * overflow records we just added. + */ + inserted_offnum = PageAddItem(page, tuple_hdr, xlrec->tuple_len, + InvalidOffsetNumber, false, false); + if (inserted_offnum == InvalidOffsetNumber) + { + /* + * PageAddItem can fail if the page was modified by a later + * operation (CLR from the UNDO subsystem, defrag, or prune) + * whose effects were checkpointed to disk before the crash. + * In that case this INSERT was already superseded and the + * page state is ahead of this WAL record. Advance the page + * LSN so recovery doesn't retry, and skip tuple setup. + * + * PANICing here would make the server permanently + * unrecoverable after certain crash sequences involving the + * logical revert worker. + */ + elog(WARNING, "FLUX INSERT redo: failed to add tuple on " + "block %u (page may have been modified by a later " + "operation); skipping redo", blkno); + goto insert_skip_tuple; + } + final_offnum = inserted_offnum; + + /* + * Fix the tuple's t_ctid to point to itself at the correct + * location. During normal operation, this is set in + * flux_tuple_insert after we know the final TID. During redo, we + * must fix it here. + * + * Defensive: validate the ItemId is LP_NORMAL *and* has storage + * before dereferencing via PageGetItem. PageGetItem asserts + * ItemIdHasStorage (lp_len != 0), which ItemIdIsNormal does not + * imply: PageAddItem can produce an LP_NORMAL line pointer with + * lp_len == 0 for a zero-length payload, and after crash recovery + * involving the UNDO revert worker the slot could be in an + * unexpected state. Skipping the t_ctid fixup for a zero-storage + * item is safe (there is no tuple body to point at). + */ + { + ItemId itemid = PageGetItemId(page, inserted_offnum); + + if (ItemIdIsNormal(itemid) && ItemIdHasStorage(itemid)) + { + FluxTupleHeader *inserted_hdr = + (FluxTupleHeader *) PageGetItem(page, itemid); + + /* blkno was already fetched at function entry */ + ItemPointerSet(&inserted_hdr->t_ctid, blkno, inserted_offnum); + } + } + + /* + * Update page header. CRITICAL: Must replicate the exact logic + * from FluxPageAddTuple() so the page matches the Full Page + * Write. FluxPageAddTuple sets the FLUX_PAGE_DEFRAG_NEEDED flag + * based on fragmentation heuristics, so we must do the same here. + */ + phdr = FluxPageGetOpaque(page); + FluxPageSetCommitTs(phdr, Max(FluxPageGetCommitTs(phdr), xlrec->commit_ts)); + + /* + * Mark page for defragmentation if fragmented. This matches the + * logic in FluxPageAddTuple() at flux_tuple.c:513-517. + */ + if (PageGetFreeSpace(page) >= xlrec->tuple_len * 2 && + PageGetMaxOffsetNumber(page) > FirstOffsetNumber + 5) + { + FluxPageSetFlag(phdr, FLUX_PAGE_DEFRAG_NEEDED); + } + + insert_skip_tuple: + PageSetLSN(page, record->EndRecPtr); + MarkBufferDirty(buffer); + } + + /* + * Capture the UNCOMMITTED flag from the replayed page tuple while we + * still hold the buffer. Reading it from the WAL main-data header is + * wrong for the speculative INSERT variants, which restore the tuple + * via a full-page image and carry no tuple body in main data. + */ + if (final_offnum != InvalidOffsetNumber && BufferIsValid(buffer)) + { + Page curpage = BufferGetPage(buffer); + + if (final_offnum <= PageGetMaxOffsetNumber(curpage)) + { + ItemId curiid = PageGetItemId(curpage, final_offnum); + + if (ItemIdIsNormal(curiid) && ItemIdHasStorage(curiid)) + { + FluxTupleHeader *curhdr = + (FluxTupleHeader *) PageGetItem(curpage, curiid); + + tuple_uncommitted = + (curhdr->t_flags & FLUX_TUPLE_UNCOMMITTED) != 0; + } + } + } + + if (BufferIsValid(buffer)) + UnlockReleaseBuffer(buffer); + + /* + * Register UNCOMMITTED tuples in the per-tuple sLog during WAL + * replay. On a hot standby, the sLog is never populated by normal + * INSERT operations (only the primary's transaction machinery does + * that). Without this, the visibility check sees slog_nfound==0 and + * incorrectly assumes the inserter committed, making aborted tuples + * visible until the CLR arrives from the logical revert worker. + * + * This entry is cleaned up lazily: for committed transactions, + * SLogTupleEvictCommitted() reclaims the slot when the hash fills. + * For aborted transactions, the CLR sets DELETED, making the sLog + * entry irrelevant for visibility. + */ + if (final_offnum != InvalidOffsetNumber && tuple_uncommitted) + { + TransactionId redo_xid = XLogRecGetXid(record); + + if (TransactionIdIsValid(redo_xid)) + { + ItemPointerData tid; + + ItemPointerSet(&tid, blkno, final_offnum); + SLogTupleInsertRecovery(rlocator.relNumber, &tid, + redo_xid, SLOG_OP_INSERT); + } + } + + /* + * Process overflow buffers on separate pages (buffers 1..N). Each + * overflow buffer contains an overflow record that was registered + * with XLogRegisterBufData during WAL logging. + * + * Note: Overflow records on block_id=0 were already handled above + * before the main tuple to preserve insertion order. + */ + for (int ovf_idx = 1; ovf_idx < XLR_MAX_BLOCK_ID; ovf_idx++) + { + Buffer ovf_buffer; + Page ovf_page; + XLogRedoAction ovf_action; + + if (!XLogRecHasBlockRef(record, ovf_idx)) + break; /* No more overflow buffers */ + + ovf_action = XLogReadBufferForRedo(record, (uint8) ovf_idx, &ovf_buffer); + if (ovf_action == BLK_NEEDS_REDO) + { + char *ovf_data; + Size ovf_len; + + ovf_page = BufferGetPage(ovf_buffer); + + /* Initialize as FLUX page if new */ + if (PageIsNew(ovf_page)) + { + FluxInitPage(ovf_page, BufferGetPageSize(ovf_buffer)); + } + + /* Get the overflow record data from WAL */ + ovf_data = XLogRecGetBlockData(record, (uint8) ovf_idx, &ovf_len); + if (ovf_data != NULL && ovf_len > 0) + { + char *ovf_ptr = ovf_data; + Size ovf_remaining = ovf_len; + + /* + * Parse and replay all overflow records on this block. + * Multiple overflow records may be on the same page due + * to spatial locality optimization. + */ + while (ovf_remaining > sizeof(xl_flux_overflow_write)) + { + xl_flux_overflow_write *ovf_xlrec = (xl_flux_overflow_write *) ovf_ptr; + char *actual_data = ovf_ptr + sizeof(xl_flux_overflow_write); + Size actual_len = ovf_xlrec->data_len; + OffsetNumber ovf_offnum; + + if (ovf_remaining < sizeof(xl_flux_overflow_write) + actual_len) + elog(PANIC, "FLUX INSERT redo: corrupt overflow data on block %u", + BufferGetBlockNumber(ovf_buffer)); + + /* + * Use the specific offset from WAL record. Overflow + * pointers reference these offsets. Use the specific + * offset from WAL record. Overflow pointers reference + * these offsets. + */ + ovf_offnum = PageAddItem(ovf_page, actual_data, actual_len, + ovf_xlrec->offnum, false, false); + if (ovf_offnum == InvalidOffsetNumber) + { + /* + * Overflow page may have been modified by a later + * operation that was checkpointed. Skip + * remaining overflow records on this page. + */ + elog(WARNING, "FLUX INSERT redo: failed to add " + "overflow record on block %u; skipping", + BufferGetBlockNumber(ovf_buffer)); + break; + } + + /* Advance to next overflow record */ + ovf_ptr += sizeof(xl_flux_overflow_write) + actual_len; + ovf_remaining -= sizeof(xl_flux_overflow_write) + actual_len; + } + } + + PageSetLSN(ovf_page, record->EndRecPtr); + MarkBufferDirty(ovf_buffer); + } + if (BufferIsValid(ovf_buffer)) + UnlockReleaseBuffer(ovf_buffer); + } + } +} + +/* + * flux_xlog_multi_insert_redo + * REDO handler for XLOG_FLUX_MULTI_INSERT. + * + * Replays a page-at-a-time batch insert. The WAL main data is: + * [xl_flux_multi_insert header] + * ntuples * { [xl_flux_multi_insert_tuple header][tuple t_data body] } + * ntuples * { [logical image bytes][uint32 len] } (only if logical) + * + * Unlike single INSERT, the batch path never spills to overflow (large + * tuples route to the single-insert path), so there is no overflow replay. + */ +static void +flux_xlog_multi_insert_redo(XLogReaderState *record) +{ + RelFileLocator rlocator; + BlockNumber blkno; + Buffer buffer; + Page page; + + XLogRecGetBlockTag(record, 0, &rlocator, NULL, &blkno); + + { + xl_flux_multi_insert *xlrec = (xl_flux_multi_insert *) XLogRecGetData(record); + int ntuples = xlrec->ntuples; + char *cursor = (char *) xlrec + SizeOfFluxMultiInsert; + XLogRedoAction action; + OffsetNumber *final_offnums; + bool *tuple_uncommitted; + int i; + + final_offnums = (OffsetNumber *) palloc0(ntuples * sizeof(OffsetNumber)); + tuple_uncommitted = (bool *) palloc0(ntuples * sizeof(bool)); + + action = XLogReadBufferForRedo(record, 0, &buffer); + + if (action == BLK_RESTORED) + { + /* + * The full-page image already carries every tuple; recover the + * offsets from the per-tuple WAL headers so we can still register + * uncommitted tuples in the sLog below. + */ + char *c = cursor; + + for (i = 0; i < ntuples; i++) + { + xl_flux_multi_insert_tuple *thdr; + + c = (char *) SHORTALIGN(c); + thdr = (xl_flux_multi_insert_tuple *) c; + + final_offnums[i] = thdr->offnum; + c += SizeOfFluxMultiInsertTuple + thdr->datalen; + } + } + + if (action == BLK_NEEDS_REDO) + { + FluxPageOpaque phdr; + Size max_body = 0; + + page = BufferGetPage(buffer); + + if (PageIsNew(page)) + FluxInitPage(page, BufferGetPageSize(buffer)); + + for (i = 0; i < ntuples; i++) + { + xl_flux_multi_insert_tuple *thdr; + char *body; + Size datalen; + OffsetNumber inserted_offnum; + + cursor = (char *) SHORTALIGN(cursor); + thdr = (xl_flux_multi_insert_tuple *) cursor; + body = cursor + SizeOfFluxMultiInsertTuple; + datalen = thdr->datalen; + + cursor = body + datalen; + + if (datalen == 0) + { + elog(WARNING, "FLUX MULTI_INSERT redo: zero-length tuple " + "%d on block %u; skipping", i, blkno); + continue; + } + + /* + * Use InvalidOffsetNumber so PageAddItem packs sequentially, + * matching the original batch insertion order. Skip on + * failure (the page may have been advanced past this record + * by a later checkpointed operation), exactly as the single + * INSERT redo path does. + */ + inserted_offnum = PageAddItem(page, body, datalen, + InvalidOffsetNumber, false, false); + if (inserted_offnum == InvalidOffsetNumber) + { + elog(WARNING, "FLUX MULTI_INSERT redo: failed to add " + "tuple %d on block %u (page may have been modified by " + "a later operation); skipping", i, blkno); + continue; + } + + final_offnums[i] = inserted_offnum; + if (datalen > max_body) + max_body = datalen; + + /* + * Fix the tuple's t_ctid to point at itself. Guard the + * PageGetItem dereference on ItemIdHasStorage as the single + * INSERT redo path does. + */ + { + ItemId itemid = PageGetItemId(page, inserted_offnum); + + if (ItemIdIsNormal(itemid) && ItemIdHasStorage(itemid)) + { + FluxTupleHeader *inserted_hdr = + (FluxTupleHeader *) PageGetItem(page, itemid); + + ItemPointerSet(&inserted_hdr->t_ctid, blkno, inserted_offnum); + } + } + } + + phdr = FluxPageGetOpaque(page); + FluxPageSetCommitTs(phdr, Max(FluxPageGetCommitTs(phdr), xlrec->commit_ts)); + + /* + * Mark the page for defragmentation if fragmented, matching + * FluxPageAddTuple(). Use the largest tuple body in the batch as + * the heuristic threshold. + */ + if (max_body > 0 && + PageGetFreeSpace(page) >= max_body * 2 && + PageGetMaxOffsetNumber(page) > FirstOffsetNumber + 5) + { + FluxPageSetFlag(phdr, FLUX_PAGE_DEFRAG_NEEDED); + } + + PageSetLSN(page, record->EndRecPtr); + MarkBufferDirty(buffer); + } + + /* + * Capture the UNCOMMITTED flag from each replayed tuple while we + * still hold the buffer (true for both BLK_NEEDS_REDO and + * BLK_RESTORED). + */ + if (BufferIsValid(buffer)) + { + Page curpage = BufferGetPage(buffer); + OffsetNumber maxoff = PageGetMaxOffsetNumber(curpage); + + for (i = 0; i < ntuples; i++) + { + OffsetNumber off = final_offnums[i]; + ItemId curiid; + + if (off == InvalidOffsetNumber || off > maxoff) + continue; + + curiid = PageGetItemId(curpage, off); + if (ItemIdIsNormal(curiid) && ItemIdHasStorage(curiid)) + { + FluxTupleHeader *curhdr = + (FluxTupleHeader *) PageGetItem(curpage, curiid); + + tuple_uncommitted[i] = + (curhdr->t_flags & FLUX_TUPLE_UNCOMMITTED) != 0; + } + } + } + + if (BufferIsValid(buffer)) + UnlockReleaseBuffer(buffer); + + /* + * Register UNCOMMITTED tuples in the per-tuple sLog during replay, so + * a hot standby sees aborted batch rows as invisible until the CLR + * arrives. Mirrors the single INSERT redo path. + */ + { + TransactionId redo_xid = XLogRecGetXid(record); + + if (TransactionIdIsValid(redo_xid)) + { + for (i = 0; i < ntuples; i++) + { + ItemPointerData tid; + + if (final_offnums[i] == InvalidOffsetNumber || + !tuple_uncommitted[i]) + continue; + + ItemPointerSet(&tid, blkno, final_offnums[i]); + SLogTupleInsertRecovery(rlocator.relNumber, &tid, + redo_xid, SLOG_OP_INSERT); + } + } + } + + pfree(final_offnums); + pfree(tuple_uncommitted); + } +} + +/* + * flux_xlog_update_inplace_redo + * REDO handler for XLOG_FLUX_UPDATE_INPLACE. + */ +static void +flux_xlog_update_inplace_redo(XLogReaderState *record) +{ + Buffer buffer; + Page page; + + { + xl_flux_update *xlrec = (xl_flux_update *) XLogRecGetData(record); + XLogRedoAction action; + + /* + * WAL record layout depends on FLUX_WAL_PREFIX_SUFFIX flag: + * + * Without prefix/suffix: [xl_flux_update][full new tuple data] With + * prefix/suffix: [xl_flux_update][xl_flux_prefix_suffix][diff + * bytes] + * + * Old tuple data is in the UNDO fork exclusively. + */ + char *after_header = (char *) xlrec + sizeof(xl_flux_update); + bool use_prefix_suffix = (xlrec->flags & FLUX_WAL_PREFIX_SUFFIX) != 0; + xl_flux_prefix_suffix ps_info = {0, 0}; + char *diff_data = NULL; + char *new_tuple_data = NULL; + FluxTupleHeader *new_tuple_hdr = NULL; + ItemId itemid; + FluxPageOpaque phdr; + + if (use_prefix_suffix) + { + memcpy(&ps_info, after_header, sizeof(xl_flux_prefix_suffix)); + diff_data = after_header + sizeof(xl_flux_prefix_suffix); + } + else + { + new_tuple_data = after_header; + new_tuple_hdr = (FluxTupleHeader *) new_tuple_data; + } + + action = XLogReadBufferForRedo(record, 0, &buffer); + + if (action == BLK_NEEDS_REDO) + { + char *blk0_ovf_data; + Size blk0_ovf_len; + + page = BufferGetPage(buffer); + + /* + * XLogInitBufferForRedo does standard PageInit for new pages, but + * doesn't set up FLUX opaque space. Initialize it here if needed. + */ + if (PageIsNew(page)) + { + FluxInitPage(page, BufferGetPageSize(buffer)); + } + + /* + * Process overflow records on block 0 BEFORE the main tuple, + * matching the original insertion order. During normal + * operation, overflow records stored on the same page as the main + * tuple (spatial locality) get lower offsets. We must replay + * them first so the main tuple ends up at the correct offset. + */ + blk0_ovf_data = XLogRecGetBlockData(record, 0, &blk0_ovf_len); + if (blk0_ovf_data != NULL && blk0_ovf_len > 0 && + (xlrec->flags & FLUX_WAL_HAS_OVERFLOW_BLK0)) + { + char *ovf_ptr = blk0_ovf_data; + Size ovf_remaining = blk0_ovf_len; + + while (ovf_remaining > sizeof(xl_flux_overflow_write)) + { + xl_flux_overflow_write *blk0_ovf_xlrec = + (xl_flux_overflow_write *) ovf_ptr; + char *actual_data = ovf_ptr + sizeof(xl_flux_overflow_write); + Size actual_len = blk0_ovf_xlrec->data_len; + OffsetNumber ovf_offnum; + + if (ovf_remaining < sizeof(xl_flux_overflow_write) + actual_len) + elog(PANIC, "FLUX UPDATE redo: corrupt overflow data on block 0"); + + ovf_offnum = PageAddItem(page, actual_data, actual_len, + InvalidOffsetNumber, false, false); + if (ovf_offnum == InvalidOffsetNumber) + elog(PANIC, "FLUX UPDATE redo: failed to add overflow record on block 0"); + + ovf_ptr += sizeof(xl_flux_overflow_write) + actual_len; + ovf_remaining -= sizeof(xl_flux_overflow_write) + actual_len; + } + } + + /* + * Apply the update. BLK_NEEDS_REDO is only returned when the + * page LSN < record LSN (no FPI). + * + * For cross-page out-of-place updates, the new tuple lives on the + * destination page (restored from its FPI). Here we just mark + * the old tuple as UPDATED so visibility checks filter it + * correctly. + * + * Same-page out-of-place updates always force an FPI (see + * FluxXLogUpdate), so they get BLK_RESTORED and never reach this + * code path. + */ + itemid = PageGetItemId(page, xlrec->offnum); + if (ItemIdIsNormal(itemid) && ItemIdHasStorage(itemid)) + { + FluxTupleHeader *existing_tuple = + (FluxTupleHeader *) PageGetItem(page, itemid); + + if (xlrec->flags & FLUX_WAL_CROSS_PAGE) + { + /* + * Cross-page out-of-place update: mark the old tuple as + * UPDATED. The new version is on the destination page + * restored from its FPI. + */ + existing_tuple->t_flags |= FLUX_TUPLE_UPDATED; + existing_tuple->t_flags &= ~FLUX_TUPLE_UNCOMMITTED; + existing_tuple->t_commit_ts = xlrec->new_commit_ts; + } + else + { + Size existing_len = ItemIdGetLength(itemid); + + if (use_prefix_suffix) + { + /* + * Prefix/suffix compressed update: reconstruct new + * tuple by patching the diff bytes into the existing + * tuple data on the page. + * + * The emitter computes prefix/suffix/difflen against + * new_tuple->t_len (a same-size update only), so redo + * must anchor on xlrec->new_tuple_len -- NOT the + * on-page slot length. A prior non-shrinking shrink + * update can leave the slot larger than the tuple's + * logical length (see flux_operations.c), so using + * existing_len here would over-copy past the diff + * bytes and corrupt the tuple's suffix. + */ + int difflen = (int) xlrec->new_tuple_len - + ps_info.prefixlen - ps_info.suffixlen; + + if (difflen < 0 || + ps_info.prefixlen + ps_info.suffixlen > xlrec->new_tuple_len) + elog(PANIC, "FLUX UPDATE REDO: invalid prefix/suffix " + "(prefix=%u, suffix=%u, tuple_len=%u)", + ps_info.prefixlen, ps_info.suffixlen, + xlrec->new_tuple_len); + + if (difflen > 0) + memcpy((char *) existing_tuple + ps_info.prefixlen, + diff_data, difflen); + } + else if (xlrec->new_tuple_len <= existing_len) + { + /* + * Full new tuple: overwrite in place. Do NOT shrink + * the line pointer; the write path keeps the original + * slot length so the undo before-image can be + * restored into it (see flux_operations.c). Redo + * must produce the identical slot length or WAL + * consistency checking + * (wal_consistency_checking=flux) will diverge. + */ + memcpy(existing_tuple, new_tuple_hdr, xlrec->new_tuple_len); + } + else + { + /* + * Should not happen: growing updates force FPI via + * REGBUF_FORCE_IMAGE, so BLK_NEEDS_REDO is never + * returned for them. + */ + elog(PANIC, "FLUX UPDATE REDO: new tuple (%u) larger " + "than existing slot (%zu) without FPI", + xlrec->new_tuple_len, existing_len); + } + } + } + else + { + elog(DEBUG1, "FLUX UPDATE REDO: ItemId at offnum=%u is not normal", xlrec->offnum); + } + + /* Update page header */ + phdr = FluxPageGetOpaque(page); + FluxPageSetCommitTs(phdr, Max(FluxPageGetCommitTs(phdr), xlrec->new_commit_ts)); + + PageSetLSN(page, record->EndRecPtr); + MarkBufferDirty(buffer); + } + if (BufferIsValid(buffer)) + UnlockReleaseBuffer(buffer); + + /* + * Process overflow buffers on separate pages (buffers 1..N) for + * UPDATE. Multiple overflow records may share a single block due to + * spatial locality, so we loop through all records within each + * block's data (matching INSERT redo). + */ + for (int ovf_idx = 1; ovf_idx < XLR_MAX_BLOCK_ID; ovf_idx++) + { + Buffer ovf_buffer; + Page ovf_page; + XLogRedoAction ovf_action; + + if (!XLogRecHasBlockRef(record, ovf_idx)) + break; /* No more overflow buffers */ + + ovf_action = XLogReadBufferForRedo(record, (uint8) ovf_idx, &ovf_buffer); + if (ovf_action == BLK_NEEDS_REDO) + { + char *ovf_data; + Size ovf_len; + + ovf_page = BufferGetPage(ovf_buffer); + + /* Initialize as FLUX page if new */ + if (PageIsNew(ovf_page)) + { + FluxInitPage(ovf_page, BufferGetPageSize(ovf_buffer)); + } + + /* Get the overflow record data from WAL */ + ovf_data = XLogRecGetBlockData(record, (uint8) ovf_idx, &ovf_len); + if (ovf_data != NULL && ovf_len > 0) + { + char *ovf_ptr = ovf_data; + Size ovf_remaining = ovf_len; + + /* + * Parse and replay all overflow records on this block. + * Multiple overflow records may share a page due to + * spatial locality. + */ + while (ovf_remaining > sizeof(xl_flux_overflow_write)) + { + xl_flux_overflow_write *ovf_xlrec2 = + (xl_flux_overflow_write *) ovf_ptr; + char *actual_data = ovf_ptr + sizeof(xl_flux_overflow_write); + Size actual_len = ovf_xlrec2->data_len; + OffsetNumber ovf_offnum; + + if (ovf_remaining < sizeof(xl_flux_overflow_write) + actual_len) + elog(PANIC, "FLUX UPDATE redo: corrupt overflow data on block %u", + BufferGetBlockNumber(ovf_buffer)); + + /* + * Use InvalidOffsetNumber to append sequentially, + * matching the original insertion order within this + * page. + */ + ovf_offnum = PageAddItem(ovf_page, actual_data, actual_len, + InvalidOffsetNumber, false, false); + if (ovf_offnum == InvalidOffsetNumber) + elog(PANIC, "FLUX UPDATE redo: failed to add overflow record on block %u", + BufferGetBlockNumber(ovf_buffer)); + + ovf_ptr += sizeof(xl_flux_overflow_write) + actual_len; + ovf_remaining -= sizeof(xl_flux_overflow_write) + actual_len; + } + } + + PageSetLSN(ovf_page, record->EndRecPtr); + MarkBufferDirty(ovf_buffer); + } + if (BufferIsValid(ovf_buffer)) + UnlockReleaseBuffer(ovf_buffer); + } + } +} + +/* + * flux_xlog_delete_redo + * REDO handler for XLOG_FLUX_DELETE. + */ +static void +flux_xlog_delete_redo(XLogReaderState *record) +{ + Buffer buffer; + Page page; + + { + xl_flux_delete *xlrec = (xl_flux_delete *) XLogRecGetData(record); + XLogRedoAction action; + ItemId itemid; + FluxPageOpaque phdr; + + /* + * WAL record contains only the delete header (offset + commit_ts). + * Old tuple data is stored exclusively in the UNDO fork for + * transaction rollback and is not needed here. + */ + + action = XLogReadBufferForRedo(record, 0, &buffer); + if (action == BLK_NEEDS_REDO) + { + page = BufferGetPage(buffer); + + /* + * XLogInitBufferForRedo does standard PageInit for new pages, but + * doesn't set up FLUX opaque space. Initialize it here if needed. + */ + if (PageIsNew(page)) + { + FluxInitPage(page, BufferGetPageSize(buffer)); + } + + /* REDO: Mark tuple as deleted */ + itemid = PageGetItemId(page, xlrec->offnum); + if (ItemIdIsNormal(itemid) && ItemIdHasStorage(itemid)) + { + FluxTupleHeader *tuple = + (FluxTupleHeader *) PageGetItem(page, itemid); + + tuple->t_flags |= FLUX_TUPLE_DELETED; + tuple->t_commit_ts = xlrec->commit_ts; + } + + /* Update page header */ + phdr = FluxPageGetOpaque(page); + FluxPageSetCommitTs(phdr, Max(FluxPageGetCommitTs(phdr), xlrec->commit_ts)); + FluxPageSetFlag(phdr, FLUX_PAGE_DEFRAG_NEEDED); + + PageSetLSN(page, record->EndRecPtr); + MarkBufferDirty(buffer); + } + if (BufferIsValid(buffer)) + UnlockReleaseBuffer(buffer); + } +} + +/* + * flux_xlog_defrag_redo + * REDO handler for XLOG_FLUX_DEFRAG. + */ +static void +flux_xlog_defrag_redo(XLogReaderState *record) +{ + Buffer buffer; + Page page; + + { + xl_flux_defrag *xlrec = (xl_flux_defrag *) XLogRecGetData(record); + XLogRedoAction action; + + FluxPageOpaque phdr; + + action = XLogReadBufferForRedo(record, 0, &buffer); + if (action == BLK_NEEDS_REDO) + { + page = BufferGetPage(buffer); + + /* + * XLogInitBufferForRedo does standard PageInit for new pages, but + * doesn't set up FLUX opaque space. Initialize it here if needed. + */ + if (PageIsNew(page)) + { + FluxInitPage(page, BufferGetPageSize(buffer)); + } + + /* Defragment the page */ + PageRepairFragmentation(page); + + /* Update page header */ + phdr = FluxPageGetOpaque(page); + FluxPageSetCommitTs(phdr, Max(FluxPageGetCommitTs(phdr), xlrec->commit_ts)); + FluxPageClearFlag(phdr, FLUX_PAGE_DEFRAG_NEEDED); + + PageSetLSN(page, record->EndRecPtr); + MarkBufferDirty(buffer); + } + if (BufferIsValid(buffer)) + UnlockReleaseBuffer(buffer); + } +} + +/* + * flux_xlog_overflow_write_redo + * REDO handler for XLOG_FLUX_OVERFLOW_WRITE. + */ +static void +flux_xlog_overflow_write_redo(XLogReaderState *record) +{ + Buffer buffer; + Page page; + + { + xl_flux_overflow_write *xlrec = + (xl_flux_overflow_write *) XLogRecGetData(record); + char *record_data = (char *) xlrec + sizeof(xl_flux_overflow_write); + XLogRedoAction action; + + action = XLogReadBufferForRedo(record, 0, &buffer); + if (action == BLK_NEEDS_REDO) + { + page = BufferGetPage(buffer); + + /* Initialize as normal FLUX page if needed */ + if (PageIsNew(page)) + { + FluxInitPage(page, BufferGetPageSize(buffer)); + } + + if (xlrec->flags & FLUX_OVERFLOW_WAL_LINK_UPDATE) + { + /* + * Link update: overwrite the existing overflow record header + * at the specified offset with updated chain pointers. + */ + ItemId itemid; + + itemid = PageGetItemId(page, xlrec->offnum); + if (ItemIdIsNormal(itemid) && ItemIdHasStorage(itemid)) + { + FluxOverflowRecordHeader *existing_hdr = + (FluxOverflowRecordHeader *) PageGetItem(page, itemid); + + memcpy(existing_hdr, record_data, + sizeof(FluxOverflowRecordHeader)); + } + } + else + { + /* + * New overflow record: the logged data is the complete record + * (FluxOverflowRecordHeader + chunk data). Add it to the page + * at the specified offset. + */ + OffsetNumber offnum; + + offnum = PageAddItem(page, record_data, xlrec->data_len, + xlrec->offnum, false, false); + if (offnum == InvalidOffsetNumber) + elog(ERROR, "failed to add overflow record to page during redo"); + } + + PageSetLSN(page, record->EndRecPtr); + MarkBufferDirty(buffer); + } + if (BufferIsValid(buffer)) + UnlockReleaseBuffer(buffer); + } +} + +/* + * flux_xlog_compress_redo + * REDO handler for XLOG_FLUX_COMPRESS. + */ +static void +flux_xlog_compress_redo(XLogReaderState *record) +{ + Buffer buffer; + Page page; + + { + xl_flux_compress *xlrec = (xl_flux_compress *) XLogRecGetData(record); + XLogRedoAction action; + + ItemId itemid; + FluxPageOpaque phdr; + + action = XLogReadBufferForRedo(record, 0, &buffer); + if (action == BLK_NEEDS_REDO) + { + page = BufferGetPage(buffer); + + /* + * XLogInitBufferForRedo does standard PageInit for new pages, but + * doesn't set up FLUX opaque space. Initialize it here if needed. + */ + if (PageIsNew(page)) + { + FluxInitPage(page, BufferGetPageSize(buffer)); + } + + /* Apply compression to the tuple attribute */ + itemid = PageGetItemId(page, xlrec->offnum); + if (ItemIdIsNormal(itemid) && ItemIdHasStorage(itemid)) + { + FluxTupleHeader *tuple = + (FluxTupleHeader *) PageGetItem(page, itemid); + + /* Mark tuple as compressed */ + tuple->t_flags |= FLUX_TUPLE_COMPRESSED; + tuple->t_infomask |= FLUX_INFOMASK_COMPRESSED; + tuple->t_commit_ts = xlrec->commit_ts; + } + + /* Update page header */ + phdr = FluxPageGetOpaque(page); + FluxPageSetCommitTs(phdr, Max(FluxPageGetCommitTs(phdr), xlrec->commit_ts)); + + PageSetLSN(page, record->EndRecPtr); + MarkBufferDirty(buffer); + } + if (BufferIsValid(buffer)) + UnlockReleaseBuffer(buffer); + } +} + +/* + * flux_xlog_init_page_redo + * REDO handler for XLOG_FLUX_INIT_PAGE. + */ +static void +flux_xlog_init_page_redo(XLogReaderState *record) +{ + Buffer buffer; + Page page; + + { + xl_flux_init_page *xlrec = (xl_flux_init_page *) XLogRecGetData(record); + XLogRedoAction action; + + action = XLogReadBufferForRedoExtended(record, 0, RBM_ZERO_AND_LOCK, false, &buffer); + if (action == BLK_NEEDS_REDO) + { + FluxPageOpaque phdr; + + page = BufferGetPage(buffer); + + /* Initialize page with FLUX opaque space */ + FluxInitPage(page, BufferGetPageSize(buffer)); + + /* Override commit_ts and flags from WAL record */ + phdr = FluxPageGetOpaque(page); + phdr->pd_commit_ts_and_flags = ((uint64) (xlrec->commit_ts) & FLUX_PAGE_TS_MASK) | (uint64) (xlrec->flags); + + PageSetLSN(page, record->EndRecPtr); + MarkBufferDirty(buffer); + } + if (BufferIsValid(buffer)) + UnlockReleaseBuffer(buffer); + } +} + +/* + * flux_xlog_cross_page_defrag_redo + * REDO handler for XLOG_FLUX_CROSS_PAGE_DEFRAG. + */ +static void +flux_xlog_cross_page_defrag_redo(XLogReaderState *record) +{ + Buffer buffer; + Page page; + + { + xl_flux_cross_page_defrag *xlrec = + (xl_flux_cross_page_defrag *) XLogRecGetData(record); + char *tuple_data = (char *) xlrec + + sizeof(xl_flux_cross_page_defrag); + XLogRedoAction dst_action; + XLogRedoAction src_action; + + /* + * Redo the target page (block 0): insert the moved tuple. + * XLogReadBufferForRedo will skip replay if FPI is present. + */ + dst_action = XLogReadBufferForRedo(record, 0, &buffer); + if (dst_action == BLK_NEEDS_REDO) + { + page = BufferGetPage(buffer); + + if (PageAddItem(page, tuple_data, xlrec->tuple_len, + xlrec->dst_offnum, false, false) + == InvalidOffsetNumber) + { + /* + * Defensive: with REGBUF_FORCE_IMAGE this path should be + * unreachable, but if it ever fires we must not PANIC ā€” + * skip the move and let the source page processing proceed. + */ + elog(DEBUG1, "flux cross-page defrag: insufficient space on target page during redo"); + if (BufferIsValid(buffer)) + UnlockReleaseBuffer(buffer); + goto process_source; + } + + /* Update ctid in the new copy to point to itself */ + { + ItemId dst_itemid; + FluxTupleHeader *dst_hdr; + BlockNumber dst_blkno; + + XLogRecGetBlockTag(record, 0, NULL, NULL, &dst_blkno); + dst_itemid = PageGetItemId(page, xlrec->dst_offnum); + if (ItemIdIsNormal(dst_itemid) && ItemIdHasStorage(dst_itemid)) + { + dst_hdr = (FluxTupleHeader *) PageGetItem(page, dst_itemid); + ItemPointerSet(&dst_hdr->t_ctid, dst_blkno, + xlrec->dst_offnum); + } + } + + PageSetLSN(page, record->EndRecPtr); + MarkBufferDirty(buffer); + } + if (BufferIsValid(buffer)) + UnlockReleaseBuffer(buffer); + + /* + * Redo the source page (block 1): mark the old slot unused. + */ +process_source: + src_action = XLogReadBufferForRedo(record, 1, &buffer); + if (src_action == BLK_NEEDS_REDO) + { + ItemId src_itemid; + + page = BufferGetPage(buffer); + src_itemid = PageGetItemId(page, xlrec->src_offnum); + ItemIdSetUnused(src_itemid); + + PageSetLSN(page, record->EndRecPtr); + MarkBufferDirty(buffer); + } + if (BufferIsValid(buffer)) + UnlockReleaseBuffer(buffer); + } +} + +/* + * flux_xlog_vm_set_redo + * REDO handler for XLOG_FLUX_VM_SET. + */ +static void +flux_xlog_vm_set_redo(XLogReaderState *record) +{ + { + xl_flux_vm_set *xlrec = (xl_flux_vm_set *) XLogRecGetData(record); + Buffer vmBuf; + Page vmPage; + uint32 mapByte; + uint8 mapOffset; + uint8 *map; + + /* + * Block 0 is the heap buffer, registered with REGBUF_NO_CHANGE. We + * don't need to redo it since the heap page is not modified by VM + * operations. + */ + + /* Redo VM buffer (block 1) */ + if (XLogReadBufferForRedo(record, 1, &vmBuf) == BLK_NEEDS_REDO) + { + vmPage = BufferGetPage(vmBuf); + + /* Calculate the VM byte and offset for this heap block */ + mapByte = (xlrec->heapBlk % ((BLCKSZ - MAXALIGN(SizeOfPageHeaderData)) * 4)) / 4; + mapOffset = (xlrec->heapBlk % ((BLCKSZ - MAXALIGN(SizeOfPageHeaderData)) * 4)) % 4; + + map = (uint8 *) PageGetContents(vmPage); + map[mapByte] |= (xlrec->flags << (mapOffset * 2)); + + PageSetLSN(vmPage, record->EndRecPtr); + MarkBufferDirty(vmBuf); + } + if (BufferIsValid(vmBuf)) + UnlockReleaseBuffer(vmBuf); + } +} + +/* + * flux_xlog_vm_clear_redo + * REDO handler for XLOG_FLUX_VM_CLEAR. + */ +static void +flux_xlog_vm_clear_redo(XLogReaderState *record) +{ + { + xl_flux_vm_clear *xlrec = (xl_flux_vm_clear *) XLogRecGetData(record); + Buffer vmBuf; + Page vmPage; + uint32 mapByte; + uint8 mapOffset; + uint8 *map; + + /* + * Block 0 is the heap buffer, registered with REGBUF_NO_CHANGE -- + * skip it. + */ + + /* Redo VM buffer (block 1) */ + if (XLogReadBufferForRedo(record, 1, &vmBuf) == BLK_NEEDS_REDO) + { + vmPage = BufferGetPage(vmBuf); + + /* Calculate the VM byte and offset for this heap block */ + mapByte = (xlrec->heapBlk % ((BLCKSZ - MAXALIGN(SizeOfPageHeaderData)) * 4)) / 4; + mapOffset = (xlrec->heapBlk % ((BLCKSZ - MAXALIGN(SizeOfPageHeaderData)) * 4)) % 4; + + map = (uint8 *) PageGetContents(vmPage); + map[mapByte] &= ~(xlrec->flags << (mapOffset * 2)); + + PageSetLSN(vmPage, record->EndRecPtr); + MarkBufferDirty(vmBuf); + } + if (BufferIsValid(vmBuf)) + UnlockReleaseBuffer(vmBuf); + } +} + +/* + * flux_xlog_lock_redo + * REDO handler for XLOG_FLUX_LOCK. + */ +static void +flux_xlog_lock_redo(XLogReaderState *record) +{ + Buffer buffer; + Page page; + + { + xl_flux_lock *xlrec = (xl_flux_lock *) XLogRecGetData(record); + XLogRedoAction action; + + action = XLogReadBufferForRedo(record, 0, &buffer); + if (action == BLK_NEEDS_REDO) + { + ItemId itemid; + + page = BufferGetPage(buffer); + + if (PageIsNew(page)) + { + FluxInitPage(page, BufferGetPageSize(buffer)); + } + + itemid = PageGetItemId(page, xlrec->offnum); + if (ItemIdIsNormal(itemid) && ItemIdHasStorage(itemid)) + { + FluxTupleHeader *tuple = + (FluxTupleHeader *) PageGetItem(page, itemid); + + /* Apply the lock state from the WAL record */ + tuple->t_infomask = xlrec->infomask; + tuple->t_flags |= FLUX_TUPLE_LOCKED; + } + + PageSetLSN(page, record->EndRecPtr); + MarkBufferDirty(buffer); + } + if (BufferIsValid(buffer)) + UnlockReleaseBuffer(buffer); + } +} + +/* + * flux_xlog_cas_update_redo + * REDO handler for XLOG_FLUX_CAS_UPDATE. + * + * Patches a contiguous byte range within a tuple on the page. The record + * carries only the changed bytes (data_offset..data_offset+data_len) and + * the new commit timestamp. Idempotent memcpy; safe for replay. + */ +static void +flux_xlog_cas_update_redo(XLogReaderState *record) +{ + xl_flux_cas_update *xlrec = (xl_flux_cas_update *) XLogRecGetData(record); + char *new_data = ((char *) xlrec) + sizeof(xl_flux_cas_update); + Buffer buffer; + + if (XLogReadBufferForRedo(record, 0, &buffer) == BLK_NEEDS_REDO) + { + Page page = BufferGetPage(buffer); + ItemId itemid; + FluxTupleHeader *tuple; + + itemid = PageGetItemId(page, xlrec->offnum); + if (!ItemIdIsNormal(itemid) || !ItemIdHasStorage(itemid)) + elog(PANIC, "FLUX CAS_UPDATE redo: invalid item at offset %u", + xlrec->offnum); + + tuple = (FluxTupleHeader *) PageGetItem(page, itemid); + + /* Patch the changed data bytes */ + memcpy(((char *) tuple) + xlrec->data_offset, new_data, xlrec->data_len); + + /* Update commit timestamp */ + tuple->t_commit_ts = xlrec->new_commit_ts; + + /* Ensure t_writer is cleared (crash may have left it non-zero) */ + tuple->t_writer = 0; + + PageSetLSN(page, record->EndRecPtr); + MarkBufferDirty(buffer); + } + if (BufferIsValid(buffer)) + UnlockReleaseBuffer(buffer); +} + +/* + * flux_xlog_cas_update_undo_redo + * REDO handler for XLOG_FLUX_CAS_UPDATE_UNDO (FOLD variant). + * + * Replays two page changes from one record: + * block 0 - main-fork page: patch the tuple data bytes, set commit ts, + * clear t_writer (identical to flux_xlog_cas_update_redo). + * block 1 - relundo-fork data page: write the UNDO before-image + * (identical to relundo_redo_insert; new pages honor WILL_INIT + * via xlrec->is_new_page, existing pages memcpy at page_offset). + * block 2 - relundo metapage: FPI restore, present only when is_new_page. + * + * Both page redos are idempotent memcpys; safe for replay. + */ +static void +flux_xlog_cas_update_undo_redo(XLogReaderState *record) +{ + XLogRecPtr lsn = record->EndRecPtr; + xl_flux_cas_update_undo *xlrec = + (xl_flux_cas_update_undo *) XLogRecGetData(record); + char *new_data = ((char *) xlrec) + SizeOfFluxCasUpdateUndo; + Buffer buffer; + XLogRedoAction action; + bool has_metapage = XLogRecHasBlockRef(record, 2); + + /* ---- block 0: main-fork redo byte-diff ---- */ + if (XLogReadBufferForRedo(record, 0, &buffer) == BLK_NEEDS_REDO) + { + Page page = BufferGetPage(buffer); + ItemId itemid; + FluxTupleHeader *tuple; + + itemid = PageGetItemId(page, xlrec->offnum); + if (!ItemIdIsNormal(itemid) || !ItemIdHasStorage(itemid)) + elog(PANIC, "FLUX CAS_UPDATE_UNDO redo: invalid item at offset %u", + xlrec->offnum); + + tuple = (FluxTupleHeader *) PageGetItem(page, itemid); + + memcpy(((char *) tuple) + xlrec->data_offset, new_data, xlrec->data_len); + tuple->t_commit_ts = xlrec->new_commit_ts; + tuple->t_writer = 0; + + PageSetLSN(page, lsn); + MarkBufferDirty(buffer); + } + if (BufferIsValid(buffer)) + UnlockReleaseBuffer(buffer); + + /* ---- block 1: relundo-fork UNDO before-image ---- */ + if (xlrec->urec_len < SizeOfRelUndoRecordHeader) + elog(PANIC, "CAS_UPDATE_UNDO redo: invalid urec_len %u (min %zu)", + xlrec->urec_len, SizeOfRelUndoRecordHeader); + if (xlrec->page_offset > BLCKSZ - sizeof(RelUndoPageHeaderData)) + elog(PANIC, "CAS_UPDATE_UNDO redo: invalid page offset %u", + xlrec->page_offset); + if (xlrec->new_pd_lower > BLCKSZ) + elog(PANIC, "CAS_UPDATE_UNDO redo: pd_lower %u exceeds page size", + xlrec->new_pd_lower); + if ((uint32) xlrec->page_offset + (uint32) xlrec->urec_len > BLCKSZ) + elog(PANIC, "CAS_UPDATE_UNDO redo: record extends past page end (offset %u + len %u > %u)", + xlrec->page_offset, xlrec->urec_len, (uint32) BLCKSZ); + if (xlrec->new_pd_lower + MAXALIGN(SizeOfPageHeaderData) < xlrec->page_offset) + elog(PANIC, "CAS_UPDATE_UNDO redo: new_pd_lower %u precedes page_offset %u", + xlrec->new_pd_lower, xlrec->page_offset); + if (xlrec->urec_type < RELUNDO_INSERT || xlrec->urec_type > RELUNDO_UPDATE) + elog(PANIC, "CAS_UPDATE_UNDO redo: invalid record type %u", xlrec->urec_type); + + if (xlrec->is_new_page) + { + buffer = XLogInitBufferForRedo(record, 1); + action = BLK_NEEDS_REDO; + } + else + action = XLogReadBufferForRedo(record, 1, &buffer); + + if (action == BLK_NEEDS_REDO) + { + Page page = BufferGetPage(buffer); + char *record_data; + Size record_len; + + record_data = XLogRecGetBlockData(record, 1, &record_len); + if (record_data == NULL || record_len == 0) + elog(PANIC, "CAS_UPDATE_UNDO redo: no block data for UNDO record"); + if (record_len > BLCKSZ) + elog(PANIC, "CAS_UPDATE_UNDO redo: block data too large (%zu bytes)", record_len); + + if (xlrec->is_new_page) + { + char *contents; + + if (record_len < SizeOfRelUndoPageHeaderData) + elog(PANIC, "CAS_UPDATE_UNDO redo: INIT_PAGE block data too small (%zu < %zu)", + record_len, SizeOfRelUndoPageHeaderData); + if (record_len > BLCKSZ - MAXALIGN(SizeOfPageHeaderData)) + elog(PANIC, "CAS_UPDATE_UNDO redo: INIT_PAGE block data too large (%zu bytes)", + record_len); + + PageInit(page, BLCKSZ, 0); + contents = PageGetContents(page); + memcpy(contents, record_data, record_len); + } + else + { + RelUndoPageHeader undohdr = (RelUndoPageHeader) PageGetContents(page); + + if (undohdr->pd_lower > BLCKSZ) + elog(PANIC, "CAS_UPDATE_UNDO redo: existing pd_lower %u exceeds page size", + undohdr->pd_lower); + + memcpy((char *) page + xlrec->page_offset, record_data, record_len); + undohdr->pd_lower = xlrec->new_pd_lower; + undohdr->max_xid = xlrec->max_xid; + + if (undohdr->pd_lower + MAXALIGN(SizeOfPageHeaderData) < xlrec->page_offset + record_len) + elog(PANIC, "CAS_UPDATE_UNDO redo: pd_lower %u too small for offset %u + len %zu", + undohdr->pd_lower, xlrec->page_offset, record_len); + } + + PageSetLSN(page, lsn); + MarkBufferDirty(buffer); + } + if (BufferIsValid(buffer)) + UnlockReleaseBuffer(buffer); + + /* ---- block 2: relundo metapage FPI ---- */ + if (has_metapage) + { + (void) XLogReadBufferForRedo(record, 2, &buffer); + if (BufferIsValid(buffer)) + UnlockReleaseBuffer(buffer); + } +} + +/* + * flux_xlog_write_dict_redo + * REDO handler for XLOG_FLUX_WRITE_DICT. + * + * The record always carries a forced full-page image of the dictionary-fork + * block, so XLogReadBufferForRedo restores the page directly and there is no + * delta to replay. We never reach BLK_NEEDS_REDO without the FPI present; + * the handler simply restores and releases the buffer. + */ +static void +flux_xlog_write_dict_redo(XLogReaderState *record) +{ + Buffer buffer; + + /* + * With REGBUF_FORCE_IMAGE the page is reconstructed from the FPI by + * XLogReadBufferForRedo, which returns BLK_RESTORED. BLK_NEEDS_REDO is + * not expected, but if it ever occurs there is nothing to apply. + */ + (void) XLogReadBufferForRedo(record, 0, &buffer); + if (BufferIsValid(buffer)) + UnlockReleaseBuffer(buffer); +} + +/* + * flux_redo + * Thin dispatcher for all XLOG_FLUX_* opcodes. Each case is + * delegated to a dedicated per-opcode static helper above. + */ +void +flux_redo(XLogReaderState *record) +{ + uint8 info = XLogRecGetInfo(record) & ~XLR_INFO_MASK; + + + switch (info) + { + case XLOG_FLUX_INSERT: + flux_xlog_insert_redo(record); + break; + + case XLOG_FLUX_UPDATE_INPLACE: + flux_xlog_update_inplace_redo(record); + break; + + case XLOG_FLUX_DELETE: + flux_xlog_delete_redo(record); + break; + + case XLOG_FLUX_DEFRAG: + flux_xlog_defrag_redo(record); + break; + + case XLOG_FLUX_OVERFLOW_WRITE: + flux_xlog_overflow_write_redo(record); + break; + + case XLOG_FLUX_COMPRESS: + flux_xlog_compress_redo(record); + break; + + case XLOG_FLUX_INIT_PAGE: + flux_xlog_init_page_redo(record); + break; + + case XLOG_FLUX_CROSS_PAGE_DEFRAG: + flux_xlog_cross_page_defrag_redo(record); + break; + + case XLOG_FLUX_VM_SET: + flux_xlog_vm_set_redo(record); + break; + + case XLOG_FLUX_VM_CLEAR: + flux_xlog_vm_clear_redo(record); + break; + + case XLOG_FLUX_LOCK: + flux_xlog_lock_redo(record); + break; + + case XLOG_FLUX_CAS_UPDATE: + flux_xlog_cas_update_redo(record); + break; + + case XLOG_FLUX_CAS_UPDATE_UNDO: + flux_xlog_cas_update_undo_redo(record); + break; + + case XLOG_FLUX_WRITE_DICT: + flux_xlog_write_dict_redo(record); + break; + + case XLOG_FLUX_MULTI_INSERT: + flux_xlog_multi_insert_redo(record); + break; + + default: + elog(PANIC, "flux_redo: unknown op code %u", info); + } +} + + +/* + * Mask function for FLUX pages (for consistency checking) + */ +void +flux_mask(char *page, BlockNumber blkno) +{ + Page flux_page = (Page) page; + FluxPageOpaque phdr; + bool is_overflow; + OffsetNumber offnum; + OffsetNumber maxoff; + + mask_page_lsn_and_checksum(flux_page); + + mask_page_hint_bits(flux_page); + mask_unused_space(flux_page); + + /* + * Dictionary-fork pages (XLOG_FLUX_WRITE_DICT full-page images) have no + * opaque special area and no line-pointer array; they are initialized + * with PageInit(page, BLCKSZ, 0). FluxPageGetOpaque would read 8 bytes + * past the page end and the line-pointer loop below would mis-mask dict + * payload, producing spurious "inconsistent pages" failures under + * wal_consistency_checking. Detect them by their zero special size and + * leave the (already LSN/checksum/hint-masked) page untouched. + */ + if (PageGetSpecialSize(flux_page) != MAXALIGN(sizeof(FluxPageOpaqueData))) + return; + + phdr = FluxPageGetOpaque(flux_page); + + /* Check page type before masking flags */ + is_overflow = (phdr->pd_commit_ts_and_flags & FLUX_PAGE_OVERFLOW) != 0; + + /* + * Mask the entire packed commit_ts_and_flags field. + * + * The timestamp uses Max(existing, new) during redo which can produce a + * different value if the page was concurrently modified. Heuristic flags + * (e.g., FLUX_PAGE_DEFRAG_NEEDED) may be set by redo but not by the + * original operation, or vice versa. + */ + phdr->pd_commit_ts_and_flags = 0; + + /* + * Overflow pages contain FluxOverflowRecordHeader items, not regular + * tuples. Their contents are fully determined by the WAL data, so no + * per-item masking is needed. + */ + if (is_overflow) + return; + + /* + * Mask tuple-level fields that function as hint bits and are not + * faithfully reproduced by WAL redo. The redo handlers only set the + * minimal fields needed for correctness (t_flags, t_commit_ts); + * transactional fields like infomask bits are set on the primary but not + * replayed. + */ + maxoff = PageGetMaxOffsetNumber(flux_page); + for (offnum = FirstOffsetNumber; offnum <= maxoff; offnum++) + { + ItemId itemid = PageGetItemId(flux_page, offnum); + FluxTupleHeader *tuple_hdr; + + if (!ItemIdIsNormal(itemid) || !ItemIdHasStorage(itemid)) + continue; + + tuple_hdr = (FluxTupleHeader *) PageGetItem(flux_page, itemid); + tuple_hdr->t_infomask = 0; + tuple_hdr->t_flags = 0; + tuple_hdr->t_commit_ts = 0; + tuple_hdr->t_writer = 0; /* transient CAS lock, not replayed */ + ItemPointerSetInvalid(&tuple_hdr->t_ctid); + } +} diff --git a/src/backend/access/flux/meson.build b/src/backend/access/flux/meson.build new file mode 100644 index 0000000000000..7f2cde188c524 --- /dev/null +++ b/src/backend/access/flux/meson.build @@ -0,0 +1,18 @@ +# Copyright (c) 2022-2025, PostgreSQL Global Development Group + +backend_sources += files( + 'flux_dirtymap.c', + 'flux_fsm.c', + 'flux_handler.c', + 'flux_operations.c', + 'flux_tuple.c', + 'flux_mvcc.c', + 'flux_stats.c', + 'flux_xlog.c', + 'flux_lock.c', + 'flux_slot.c', + 'flux_vm.c', + 'flux_undo.c', + 'flux_relundo.c', + 'flux_pvs.c', +) \ No newline at end of file diff --git a/src/backend/access/meson.build b/src/backend/access/meson.build index d569ac4e6e32a..ad3c1084395e3 100644 --- a/src/backend/access/meson.build +++ b/src/backend/access/meson.build @@ -7,6 +7,7 @@ subdir('gist') subdir('hash') subdir('heap') subdir('index') +subdir('flux') subdir('nbtree') subdir('rmgrdesc') subdir('sequence') diff --git a/src/backend/access/rmgrdesc/Makefile b/src/backend/access/rmgrdesc/Makefile index 3f94e17f281f3..d4ddaeada8010 100644 --- a/src/backend/access/rmgrdesc/Makefile +++ b/src/backend/access/rmgrdesc/Makefile @@ -35,4 +35,10 @@ OBJS = \ xactdesc.o \ xlogdesc.o +endif + +ifdef USE_FLUX +OBJS += fluxdesc.o +endif + include $(top_srcdir)/src/backend/common.mk diff --git a/src/backend/access/rmgrdesc/fluxdesc.c b/src/backend/access/rmgrdesc/fluxdesc.c new file mode 100644 index 0000000000000..52c9feadc9c46 --- /dev/null +++ b/src/backend/access/rmgrdesc/fluxdesc.c @@ -0,0 +1,512 @@ +/*------------------------------------------------------------------------- + * + * fluxdesc.c + * Resource manager descriptor for FLUX - frontend version + * + * This provides minimal desc/identify functions for frontend tools like pg_waldump. + * The full implementations are in flux_xlog.c for backend use. + * + * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/access/rmgrdesc/fluxdesc.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/xlog.h" +#include "access/xlog_internal.h" +#include "access/xlogreader.h" + +/* Function prototypes */ +extern void flux_desc(StringInfo buf, XLogReaderState *record); +extern const char *flux_identify(uint8 info); + +/* FLUX WAL record types - keep in sync with flux_xlog.h */ +#define XLOG_FLUX_INSERT 0x00 +#define XLOG_FLUX_UPDATE 0x10 +#define XLOG_FLUX_DELETE 0x20 +#define XLOG_FLUX_VACUUM 0x30 +#define XLOG_FLUX_OVERFLOW_WRITE 0x40 +#define XLOG_FLUX_COMPRESS 0x50 +#define XLOG_FLUX_INIT_PAGE 0x60 +#define XLOG_FLUX_CROSS_PAGE_DEFRAG 0x70 +#define XLOG_FLUX_VM_SET 0x80 +#define XLOG_FLUX_VM_CLEAR 0x90 +#define XLOG_FLUX_LOCK 0xA0 +#define XLOG_FLUX_CAS_UPDATE 0xB0 +#define XLOG_FLUX_WRITE_DICT 0xC0 +#define XLOG_FLUX_MULTI_INSERT 0xD0 +#define XLOG_FLUX_CAS_UPDATE_UNDO 0xE0 +#define XLOG_FLUX_OPMASK 0xF0 + +/* WAL record flags - keep in sync with flux_xlog.h */ +#define FLUX_WAL_CROSS_PAGE 0x0002 + +/* + * Frontend-safe copies of WAL record structures from flux_xlog.h. + * Duplicated here because fluxdesc.c is compiled with FRONTEND defined and + * we need to parse these records in pg_waldump without pulling in backend + * headers. + */ +typedef struct xl_flux_insert_fe +{ + uint16 offnum; + uint16 flags; + uint64 commit_ts; + uint64 xact_ts; +} xl_flux_insert_fe; + +typedef struct xl_flux_delete_fe +{ + uint16 offnum; + uint16 flags; + uint64 commit_ts; + uint64 xact_ts; +} xl_flux_delete_fe; + +typedef struct xl_flux_update_fe +{ + uint16 offnum; + uint16 flags; + uint64 old_commit_ts; + uint64 new_commit_ts; + uint64 xact_ts; +} xl_flux_update_fe; + +/* Mirrors backend xl_flux_cas_update_undo (flux_xlog.h) */ +typedef struct xl_flux_cas_update_undo_fe +{ + uint16 offnum; + uint16 flags; + uint16 data_offset; + uint16 data_len; + uint64 new_commit_ts; + uint8 urec_type; + uint8 is_new_page; + uint16 urec_len; + uint16 page_offset; + uint16 new_pd_lower; + uint32 max_xid; +} xl_flux_cas_update_undo_fe; + +typedef struct xl_flux_vacuum_fe +{ + uint32 ntuples; +} xl_flux_vacuum_fe; + +typedef struct xl_flux_compress_fe +{ + uint16 offnum; + uint16 attr_num; + uint8 comp_type; + uint8 comp_level; + uint32 orig_size; + uint32 comp_size; + uint64 commit_ts; +} xl_flux_compress_fe; + +typedef struct xl_flux_overflow_write_fe +{ + uint16 offnum; + uint16 flags; + uint32 data_len; + uint64 commit_ts; +} xl_flux_overflow_write_fe; + +typedef struct xl_flux_init_page_fe +{ + uint32 flags; + uint64 commit_ts; +} xl_flux_init_page_fe; + +typedef struct xl_flux_cross_page_defrag_fe +{ + uint16 src_offnum; + uint16 dst_offnum; + uint32 tuple_len; +} xl_flux_cross_page_defrag_fe; + +typedef struct xl_flux_vm_set_fe +{ + uint32 heapBlk; + uint8 flags; +} xl_flux_vm_set_fe; + +typedef struct xl_flux_vm_clear_fe +{ + uint32 heapBlk; + uint8 flags; +} xl_flux_vm_clear_fe; + +typedef struct xl_flux_lock_fe +{ + uint16 offnum; + uint16 flags; + uint32 xmax; + uint16 infomask; + uint16 infomask2; + uint8 lock_mode; +} xl_flux_lock_fe; + +typedef struct xl_flux_multi_insert_fe +{ + uint16 ntuples; + uint16 flags; + uint64 commit_ts; +} xl_flux_multi_insert_fe; + +/* + * Human-readable compression type names. + */ +static const char * +flux_comp_type_name(uint8 comp_type) +{ + switch (comp_type) + { + case 0: + return "NONE"; + case 1: + return "LZ4"; + case 2: + return "ZSTD"; + case 3: + return "DELTA"; + case 4: + return "DICTIONARY"; + default: + return "UNKNOWN"; + } +} + +void +flux_desc(StringInfo buf, XLogReaderState *record) +{ + uint8 info = XLogRecGetInfo(record) & ~XLR_INFO_MASK; + char *data = XLogRecGetData(record); + Size datalen = XLogRecGetDataLen(record); + uint16 flags = 0; + + switch (info & XLOG_FLUX_OPMASK) + { + case XLOG_FLUX_INSERT: + { + if (datalen >= sizeof(xl_flux_insert_fe)) + { + xl_flux_insert_fe xlrec; + + memcpy(&xlrec, data, sizeof(xl_flux_insert_fe)); + flags = xlrec.flags; + appendStringInfo(buf, "off: %u, flags: 0x%04X, " + "commit_ts: " UINT64_FORMAT ", " + "xact_ts: " UINT64_FORMAT, + xlrec.offnum, xlrec.flags, + xlrec.commit_ts, xlrec.xact_ts); + } + else + appendStringInfoString(buf, "insert (truncated)"); + } + break; + case XLOG_FLUX_DELETE: + { + if (datalen >= sizeof(xl_flux_delete_fe)) + { + xl_flux_delete_fe xlrec; + + memcpy(&xlrec, data, sizeof(xl_flux_delete_fe)); + flags = xlrec.flags; + appendStringInfo(buf, "off: %u, flags: 0x%04X, " + "commit_ts: " UINT64_FORMAT ", " + "xact_ts: " UINT64_FORMAT, + xlrec.offnum, xlrec.flags, + xlrec.commit_ts, xlrec.xact_ts); + } + else + appendStringInfoString(buf, "delete (truncated)"); + } + break; + case XLOG_FLUX_UPDATE: + { + if (datalen >= sizeof(xl_flux_update_fe)) + { + xl_flux_update_fe xlrec; + + memcpy(&xlrec, data, sizeof(xl_flux_update_fe)); + flags = xlrec.flags; + appendStringInfo(buf, "off: %u, flags: 0x%04X, " + "old_commit_ts: " UINT64_FORMAT ", " + "new_commit_ts: " UINT64_FORMAT ", " + "xact_ts: " UINT64_FORMAT, + xlrec.offnum, xlrec.flags, + xlrec.old_commit_ts, + xlrec.new_commit_ts, + xlrec.xact_ts); + if (flags & FLUX_WAL_CROSS_PAGE) + appendStringInfoString(buf, ", cross_page: true"); + } + else + appendStringInfoString(buf, "update (truncated)"); + } + break; + case XLOG_FLUX_VACUUM: + { + if (datalen >= sizeof(xl_flux_vacuum_fe)) + { + xl_flux_vacuum_fe xlrec; + + memcpy(&xlrec, data, sizeof(xl_flux_vacuum_fe)); + appendStringInfo(buf, "ntuples: %u", xlrec.ntuples); + } + else + appendStringInfoString(buf, "vacuum (truncated)"); + } + break; + case XLOG_FLUX_COMPRESS: + { + if (datalen >= sizeof(xl_flux_compress_fe)) + { + xl_flux_compress_fe xlrec; + + memcpy(&xlrec, data, sizeof(xl_flux_compress_fe)); + appendStringInfo(buf, "off: %u, attr: %u, " + "comp_type: %s, comp_level: %u, " + "orig_size: %u, comp_size: %u, " + "commit_ts: " UINT64_FORMAT, + xlrec.offnum, xlrec.attr_num, + flux_comp_type_name(xlrec.comp_type), + xlrec.comp_level, + xlrec.orig_size, xlrec.comp_size, + xlrec.commit_ts); + } + else + appendStringInfoString(buf, "compress (truncated)"); + } + break; + case XLOG_FLUX_OVERFLOW_WRITE: + { + if (datalen >= sizeof(xl_flux_overflow_write_fe)) + { + xl_flux_overflow_write_fe xlrec; + + memcpy(&xlrec, data, sizeof(xl_flux_overflow_write_fe)); + appendStringInfo(buf, "off: %u, flags: 0x%04X, " + "data_len: %u, " + "commit_ts: " UINT64_FORMAT, + xlrec.offnum, xlrec.flags, + xlrec.data_len, xlrec.commit_ts); + } + else + appendStringInfoString(buf, "overflow_write (truncated)"); + } + break; + case XLOG_FLUX_INIT_PAGE: + { + if (datalen >= sizeof(xl_flux_init_page_fe)) + { + xl_flux_init_page_fe xlrec; + + memcpy(&xlrec, data, sizeof(xl_flux_init_page_fe)); + appendStringInfo(buf, "flags: 0x%08X, " + "commit_ts: " UINT64_FORMAT, + xlrec.flags, xlrec.commit_ts); + } + else + appendStringInfoString(buf, "init_page (truncated)"); + } + break; + case XLOG_FLUX_CROSS_PAGE_DEFRAG: + { + if (datalen >= sizeof(xl_flux_cross_page_defrag_fe)) + { + xl_flux_cross_page_defrag_fe xlrec; + + memcpy(&xlrec, data, sizeof(xl_flux_cross_page_defrag_fe)); + appendStringInfo(buf, "src_off: %u, dst_off: %u, " + "tuple_len: %u", + xlrec.src_offnum, xlrec.dst_offnum, + xlrec.tuple_len); + } + else + appendStringInfoString(buf, "cross_page_defrag (truncated)"); + } + break; + case XLOG_FLUX_VM_SET: + { + if (datalen >= sizeof(xl_flux_vm_set_fe)) + { + xl_flux_vm_set_fe xlrec; + + memcpy(&xlrec, data, sizeof(xl_flux_vm_set_fe)); + appendStringInfo(buf, "heapBlk: %u, flags: 0x%02X", + xlrec.heapBlk, xlrec.flags); + } + else + appendStringInfoString(buf, "vm_set (truncated)"); + } + break; + case XLOG_FLUX_VM_CLEAR: + { + if (datalen >= sizeof(xl_flux_vm_clear_fe)) + { + xl_flux_vm_clear_fe xlrec; + + memcpy(&xlrec, data, sizeof(xl_flux_vm_clear_fe)); + appendStringInfo(buf, "heapBlk: %u, flags: 0x%02X", + xlrec.heapBlk, xlrec.flags); + } + else + appendStringInfoString(buf, "vm_clear (truncated)"); + } + break; + case XLOG_FLUX_LOCK: + { + if (datalen >= sizeof(xl_flux_lock_fe)) + { + xl_flux_lock_fe xlrec; + + memcpy(&xlrec, data, sizeof(xl_flux_lock_fe)); + appendStringInfo(buf, "off: %u, xmax: %u, " + "infomask: 0x%04X, infomask2: 0x%04X, " + "lock_mode: %u", + xlrec.offnum, xlrec.xmax, + xlrec.infomask, xlrec.infomask2, + xlrec.lock_mode); + } + else + appendStringInfoString(buf, "lock (truncated)"); + } + break; + case XLOG_FLUX_CAS_UPDATE: + { + if (datalen >= 14) /* minimum: + * offnum(2)+flags(2)+offset(2)+len(2)+ts(8) + * - 2 padding */ + { + uint16 offnum; + uint16 d_offset; + uint16 d_len; + + memcpy(&offnum, data, sizeof(uint16)); + memcpy(&d_offset, data + 4, sizeof(uint16)); + memcpy(&d_len, data + 6, sizeof(uint16)); + appendStringInfo(buf, "off: %u, data_offset: %u, data_len: %u", + offnum, d_offset, d_len); + } + else + appendStringInfoString(buf, "cas_update (truncated)"); + } + break; + case XLOG_FLUX_CAS_UPDATE_UNDO: + { + if (datalen >= (int) sizeof(xl_flux_cas_update_undo_fe)) + { + xl_flux_cas_update_undo_fe xlrec; + + memcpy(&xlrec, data, sizeof(xl_flux_cas_update_undo_fe)); + appendStringInfo(buf, + "off: %u, data_offset: %u, data_len: %u, " + "urec_type: %u, urec_len: %u, undo_off: %u, new_page: %u", + xlrec.offnum, xlrec.data_offset, xlrec.data_len, + xlrec.urec_type, xlrec.urec_len, xlrec.page_offset, + xlrec.is_new_page); + } + else + appendStringInfoString(buf, "cas_update_undo (truncated)"); + } + break; + case XLOG_FLUX_WRITE_DICT: + { + if (datalen >= sizeof(uint32)) + { + uint32 blkno; + + memcpy(&blkno, data, sizeof(uint32)); + appendStringInfo(buf, "blkno: %u (full-page image)", blkno); + } + else + appendStringInfoString(buf, "write_dict (truncated)"); + } + break; + case XLOG_FLUX_MULTI_INSERT: + { + if (datalen >= sizeof(xl_flux_multi_insert_fe)) + { + xl_flux_multi_insert_fe xlrec; + + memcpy(&xlrec, data, sizeof(xl_flux_multi_insert_fe)); + appendStringInfo(buf, "ntuples: %u, flags: 0x%04X, " + "commit_ts: " UINT64_FORMAT, + xlrec.ntuples, xlrec.flags, + xlrec.commit_ts); + } + else + appendStringInfoString(buf, "multi_insert (truncated)"); + } + break; + default: + appendStringInfoString(buf, "UNKNOWN"); + break; + } +} + +const char * +flux_identify(uint8 info) +{ + const char *id = NULL; + + switch (info & XLOG_FLUX_OPMASK) + { + case XLOG_FLUX_INSERT: + id = "INSERT"; + break; + case XLOG_FLUX_DELETE: + id = "DELETE"; + break; + case XLOG_FLUX_UPDATE: + id = "UPDATE"; + break; + case XLOG_FLUX_VACUUM: + id = "VACUUM"; + break; + case XLOG_FLUX_COMPRESS: + id = "COMPRESS"; + break; + case XLOG_FLUX_OVERFLOW_WRITE: + id = "OVERFLOW_WRITE"; + break; + case XLOG_FLUX_INIT_PAGE: + id = "INIT_PAGE"; + break; + case XLOG_FLUX_CROSS_PAGE_DEFRAG: + id = "CROSS_PAGE_DEFRAG"; + break; + case XLOG_FLUX_VM_SET: + id = "VM_SET"; + break; + case XLOG_FLUX_VM_CLEAR: + id = "VM_CLEAR"; + break; + case XLOG_FLUX_LOCK: + id = "LOCK"; + break; + case XLOG_FLUX_CAS_UPDATE: + id = "CAS_UPDATE"; + break; + case XLOG_FLUX_CAS_UPDATE_UNDO: + id = "CAS_UPDATE_UNDO"; + break; + case XLOG_FLUX_WRITE_DICT: + id = "WRITE_DICT"; + break; + case XLOG_FLUX_MULTI_INSERT: + id = "MULTI_INSERT"; + break; + default: + id = NULL; + break; + } + + return id; +} diff --git a/src/backend/access/rmgrdesc/meson.build b/src/backend/access/rmgrdesc/meson.build index 299cf2aeee201..d566ac8f1d993 100644 --- a/src/backend/access/rmgrdesc/meson.build +++ b/src/backend/access/rmgrdesc/meson.build @@ -29,4 +29,9 @@ rmgr_desc_sources = files( 'xlogdesc.c', ) + +# FLUX is always built; rmgrlist.h registers flux_desc/flux_identify +# unconditionally, so fluxdesc.c must always be compiled. +rmgr_desc_sources += files('fluxdesc.c') + backend_sources += rmgr_desc_sources diff --git a/src/backend/access/transam/rmgr.c b/src/backend/access/transam/rmgr.c index 90712a7574229..43bbb19a6af92 100644 --- a/src/backend/access/transam/rmgr.c +++ b/src/backend/access/transam/rmgr.c @@ -43,6 +43,9 @@ #include "access/undo_xlog.h" #include "access/atm.h" #include "access/relundo_xlog.h" +#ifdef USE_FLUX +#include "access/flux_xlog.h" +#endif /* IWYU pragma: end_keep */ diff --git a/src/backend/access/transam/twophase_rmgr.c b/src/backend/access/transam/twophase_rmgr.c index fae254c6e2364..f7ef856019c9f 100644 --- a/src/backend/access/transam/twophase_rmgr.c +++ b/src/backend/access/transam/twophase_rmgr.c @@ -15,6 +15,7 @@ #include "postgres.h" #include "access/multixact.h" +#include "access/flux.h" #include "access/twophase_rmgr.h" #include "pgstat.h" #include "storage/lock.h" @@ -27,7 +28,8 @@ const TwoPhaseCallback twophase_recover_callbacks[TWOPHASE_RM_MAX_ID + 1] = lock_twophase_recover, /* Lock */ NULL, /* pgstat */ multixact_twophase_recover, /* MultiXact */ - predicatelock_twophase_recover /* PredicateLock */ + predicatelock_twophase_recover, /* PredicateLock */ + flux_twophase_recover /* FLUX */ }; const TwoPhaseCallback twophase_postcommit_callbacks[TWOPHASE_RM_MAX_ID + 1] = @@ -36,7 +38,8 @@ const TwoPhaseCallback twophase_postcommit_callbacks[TWOPHASE_RM_MAX_ID + 1] = lock_twophase_postcommit, /* Lock */ pgstat_twophase_postcommit, /* pgstat */ multixact_twophase_postcommit, /* MultiXact */ - NULL /* PredicateLock */ + NULL, /* PredicateLock */ + flux_twophase_postcommit /* FLUX */ }; const TwoPhaseCallback twophase_postabort_callbacks[TWOPHASE_RM_MAX_ID + 1] = @@ -45,7 +48,8 @@ const TwoPhaseCallback twophase_postabort_callbacks[TWOPHASE_RM_MAX_ID + 1] = lock_twophase_postabort, /* Lock */ pgstat_twophase_postabort, /* pgstat */ multixact_twophase_postabort, /* MultiXact */ - NULL /* PredicateLock */ + NULL, /* PredicateLock */ + flux_twophase_postabort /* FLUX */ }; const TwoPhaseCallback twophase_standby_recover_callbacks[TWOPHASE_RM_MAX_ID + 1] = @@ -54,5 +58,6 @@ const TwoPhaseCallback twophase_standby_recover_callbacks[TWOPHASE_RM_MAX_ID + 1 lock_twophase_standby_recover, /* Lock */ NULL, /* pgstat */ NULL, /* MultiXact */ - NULL /* PredicateLock */ + NULL, /* PredicateLock */ + flux_twophase_recover /* FLUX */ }; diff --git a/src/backend/replication/logical/decode.c b/src/backend/replication/logical/decode.c index c944be4ac83c8..789a16b942a15 100644 --- a/src/backend/replication/logical/decode.c +++ b/src/backend/replication/logical/decode.c @@ -27,6 +27,7 @@ #include "postgres.h" #include "access/heapam_xlog.h" +#include "access/flux_xlog.h" #include "access/transam.h" #include "access/xact.h" #include "access/xlog_internal.h" @@ -68,6 +69,8 @@ static inline bool FilterPrepare(LogicalDecodingContext *ctx, static bool DecodeTXNNeedSkip(LogicalDecodingContext *ctx, XLogRecordBuffer *buf, Oid txn_dbid, ReplOriginId origin_id); +static inline bool FilterByOrigin(LogicalDecodingContext *ctx, + ReplOriginId origin_id); /* * Take every XLogReadRecord()ed record and perform the actions required to @@ -564,6 +567,465 @@ heap_decode(LogicalDecodingContext *ctx, XLogRecordBuffer *buf) } } +/* + * Parse XLOG_FLUX_INSERT from wal into proper tuplebufs. + * + * Inserts contain the new tuple in FLUX format. + */ +static void +DecodeFluxInsert(LogicalDecodingContext *ctx, XLogRecordBuffer *buf) +{ + Size datalen; + char *tupledata; + XLogReaderState *r = buf->record; + xl_flux_insert *xlrec; + ReorderBufferChange *change; + RelFileLocator target_locator; + + xlrec = (xl_flux_insert *) XLogRecGetData(r); + + /* only interested in our database */ + XLogRecGetBlockTag(r, 0, &target_locator, NULL, NULL); + if (target_locator.dbOid != ctx->slot->data.database) + return; + + /* output plugin doesn't look for this origin, no need to queue */ + if (FilterByOrigin(ctx, XLogRecGetOrigin(r))) + return; + + change = ReorderBufferAllocChange(ctx->reorder); + change->action = REORDER_BUFFER_CHANGE_INSERT; + change->origin_id = XLogRecGetOrigin(r); + + memcpy(&change->data.tp.rlocator, &target_locator, sizeof(RelFileLocator)); + + /* + * Get FLUX tuple data from WAL record. The tuple follows the + * xl_flux_insert structure. + */ + tupledata = (char *) xlrec + sizeof(xl_flux_insert); + datalen = xlrec->tuple_len; + + /* + * When FLUX_WAL_LOGICAL_TUPLE is set, the write path appended a + * heap-format image at the END of the main WAL data: ... [heap bytes] + * [uint32 heap_len] Read heap_len from the last 4 bytes, then back up + * heap_len more bytes to find the heap tuple payload. This is immune to + * whatever compression data precedes it. + */ + if (xlrec->flags & FLUX_WAL_LOGICAL_TUPLE) + { + Size full_len = XLogRecGetDataLen(r); + char *end = (char *) xlrec + full_len; + uint32 heap_len; + char *heap_data; + + memcpy(&heap_len, end - sizeof(uint32), sizeof(uint32)); + elog(DEBUG1, "FLUX DecodeInsert: full_len=%zu heap_len=%u action=INSERT lsn=%X/%X", + full_len, heap_len, + (uint32) (buf->origptr >> 32), (uint32) buf->origptr); + heap_data = end - sizeof(uint32) - heap_len; + + change->data.tp.newtuple = + ReorderBufferAllocTupleBuf(ctx->reorder, + heap_len - SizeofHeapTupleHeader); + change->data.tp.newtuple->t_len = heap_len; + ItemPointerSetInvalid(&change->data.tp.newtuple->t_self); + change->data.tp.newtuple->t_tableOid = InvalidOid; + memcpy(change->data.tp.newtuple->t_data, heap_data, heap_len); + } + else + { + /* + * Legacy path: no heap image in WAL (publisher compiled without or + * table not logically logged when the record was written). pgoutput + * will most likely reject this; initial-sync via COPY still works, so + * this is only a streaming-decoding concern. + */ + change->data.tp.newtuple = + ReorderBufferAllocTupleBuf(ctx->reorder, datalen); + memcpy(change->data.tp.newtuple->t_data, tupledata, datalen); + change->data.tp.newtuple->t_len = datalen; + ItemPointerSetInvalid(&change->data.tp.newtuple->t_self); + change->data.tp.newtuple->t_tableOid = InvalidOid; + } + + change->data.tp.clear_toast_afterwards = true; + + ReorderBufferQueueChange(ctx->reorder, XLogRecGetXid(r), buf->origptr, + change, false); +} + +/* + * Parse XLOG_FLUX_MULTI_INSERT from wal into proper tuplebufs. + * + * A batched insert carries N tuples on one page. The record layout is: + * + * xl_flux_multi_insert -- header + * N * { xl_flux_multi_insert_tuple | FLUX body }-- per-tuple region + * N * { heap body | uint32 len } -- when FLUX_WAL_LOGICAL_TUPLE + * + * We queue one REORDER_BUFFER_CHANGE_INSERT per tuple, preferring the + * trailing heap-format image when present (mirrors DecodeFluxInsert). The + * heap images are appended in tuple order, so the i-th image from the front + * of the trailing region corresponds to the i-th per-tuple entry. + */ +static void +DecodeFluxMultiInsert(LogicalDecodingContext *ctx, XLogRecordBuffer *buf) +{ + XLogReaderState *r = buf->record; + xl_flux_multi_insert *xlrec; + RelFileLocator target_locator; + char *data; + char *cursor; + Size total_len; + int ntuples; + int i; + bool has_logical; + char **image_data = NULL; + uint32 *image_len = NULL; + + xlrec = (xl_flux_multi_insert *) XLogRecGetData(r); + ntuples = xlrec->ntuples; + + /* only interested in our database */ + XLogRecGetBlockTag(r, 0, &target_locator, NULL, NULL); + if (target_locator.dbOid != ctx->slot->data.database) + return; + + /* output plugin doesn't look for this origin, no need to queue */ + if (FilterByOrigin(ctx, XLogRecGetOrigin(r))) + return; + + data = XLogRecGetData(r); + total_len = XLogRecGetDataLen(r); + has_logical = (xlrec->flags & FLUX_WAL_LOGICAL_TUPLE) != 0; + + /* + * When heap-format images are present, locate each one up front by + * walking the trailing image region from the record tail backward (each + * region is "[heap bytes][uint32 len]"). The images were appended in + * tuple order, so walking back collects them in reverse; we store them + * indexed by tuple so the queue loop can read them front-to-back. + */ + if (has_logical) + { + const char *end = data + total_len; + + image_data = (char **) palloc(ntuples * sizeof(char *)); + image_len = (uint32 *) palloc(ntuples * sizeof(uint32)); + + for (i = ntuples - 1; i >= 0; i--) + { + uint32 heap_len; + + memcpy(&heap_len, end - sizeof(uint32), sizeof(uint32)); + end -= sizeof(uint32); + end -= heap_len; + image_data[i] = (char *) end; + image_len[i] = heap_len; + } + } + + /* Walk the per-tuple region forward, queuing one INSERT per tuple */ + cursor = data + SizeOfFluxMultiInsert; + for (i = 0; i < ntuples; i++) + { + xl_flux_multi_insert_tuple *tuphdr; + char *tupledata; + ReorderBufferChange *change; + + cursor = (char *) SHORTALIGN(cursor); + tuphdr = (xl_flux_multi_insert_tuple *) cursor; + cursor += SizeOfFluxMultiInsertTuple; + tupledata = cursor; + cursor += tuphdr->datalen; + + change = ReorderBufferAllocChange(ctx->reorder); + change->action = REORDER_BUFFER_CHANGE_INSERT; + change->origin_id = XLogRecGetOrigin(r); + memcpy(&change->data.tp.rlocator, &target_locator, + sizeof(RelFileLocator)); + + if (has_logical) + { + uint32 heap_len = image_len[i]; + + change->data.tp.newtuple = + ReorderBufferAllocTupleBuf(ctx->reorder, + heap_len - SizeofHeapTupleHeader); + change->data.tp.newtuple->t_len = heap_len; + ItemPointerSetInvalid(&change->data.tp.newtuple->t_self); + change->data.tp.newtuple->t_tableOid = InvalidOid; + memcpy(change->data.tp.newtuple->t_data, image_data[i], heap_len); + } + else + { + /* + * Legacy path: no heap image (table not logically logged when the + * record was written). Queue the raw FLUX body; pgoutput will + * most likely reject it, but initial COPY sync still works. + */ + change->data.tp.newtuple = + ReorderBufferAllocTupleBuf(ctx->reorder, tuphdr->datalen); + memcpy(change->data.tp.newtuple->t_data, tupledata, + tuphdr->datalen); + change->data.tp.newtuple->t_len = tuphdr->datalen; + ItemPointerSetInvalid(&change->data.tp.newtuple->t_self); + change->data.tp.newtuple->t_tableOid = InvalidOid; + } + + change->data.tp.clear_toast_afterwards = (i == ntuples - 1); + + ReorderBufferQueueChange(ctx->reorder, XLogRecGetXid(r), buf->origptr, + change, false); + } + + if (has_logical) + { + pfree(image_data); + pfree(image_len); + } +} + +/* + * Parse XLOG_FLUX_UPDATE_INPLACE from wal into proper tuplebufs. + * + * Updates contain both the old and new tuple in FLUX format. + */ +static void +DecodeFluxUpdate(LogicalDecodingContext *ctx, XLogRecordBuffer *buf) +{ + XLogReaderState *r = buf->record; + xl_flux_update *xlrec; + ReorderBufferChange *change; + char *old_tuple_data; + char *new_tuple_data; + RelFileLocator target_locator; + + xlrec = (xl_flux_update *) XLogRecGetData(r); + + /* only interested in our database */ + XLogRecGetBlockTag(r, 0, &target_locator, NULL, NULL); + if (target_locator.dbOid != ctx->slot->data.database) + return; + + /* output plugin doesn't look for this origin, no need to queue */ + if (FilterByOrigin(ctx, XLogRecGetOrigin(r))) + return; + + change = ReorderBufferAllocChange(ctx->reorder); + change->action = REORDER_BUFFER_CHANGE_UPDATE; + change->origin_id = XLogRecGetOrigin(r); + memcpy(&change->data.tp.rlocator, &target_locator, sizeof(RelFileLocator)); + + /* + * Get old and new tuple data from WAL record. The structure is: + * xl_flux_update | old_tuple | new_tuple [| logical images] + */ + old_tuple_data = (char *) xlrec + sizeof(xl_flux_update); + new_tuple_data = old_tuple_data + xlrec->old_tuple_len; + + if (xlrec->flags & FLUX_WAL_LOGICAL_TUPLE) + { + /* + * Two heap-format images appended at the END of the record, each with + * trailing length: + * + * ... [old_heap bytes] [uint32 old_heap_len] [new_heap bytes] [uint32 + * new_heap_len] + * + * Walk backwards from end: read new_heap_len last, then old_heap_len + * before the new bytes + its length. + */ + Size full_len = XLogRecGetDataLen(r); + char *end = (char *) xlrec + full_len; + uint32 old_heap_len; + uint32 new_heap_len; + char *old_heap_data; + char *new_heap_data; + + memcpy(&new_heap_len, end - sizeof(uint32), sizeof(uint32)); + new_heap_data = end - sizeof(uint32) - new_heap_len; + memcpy(&old_heap_len, + new_heap_data - sizeof(uint32), sizeof(uint32)); + old_heap_data = new_heap_data - sizeof(uint32) - old_heap_len; + + change->data.tp.oldtuple = + ReorderBufferAllocTupleBuf(ctx->reorder, + old_heap_len - SizeofHeapTupleHeader); + change->data.tp.oldtuple->t_len = old_heap_len; + ItemPointerSetInvalid(&change->data.tp.oldtuple->t_self); + change->data.tp.oldtuple->t_tableOid = InvalidOid; + memcpy(change->data.tp.oldtuple->t_data, old_heap_data, old_heap_len); + + change->data.tp.newtuple = + ReorderBufferAllocTupleBuf(ctx->reorder, + new_heap_len - SizeofHeapTupleHeader); + change->data.tp.newtuple->t_len = new_heap_len; + ItemPointerSetInvalid(&change->data.tp.newtuple->t_self); + change->data.tp.newtuple->t_tableOid = InvalidOid; + memcpy(change->data.tp.newtuple->t_data, new_heap_data, new_heap_len); + } + else + { + /* Legacy: FLUX-format bytes (will confuse pgoutput) */ + change->data.tp.oldtuple = + ReorderBufferAllocTupleBuf(ctx->reorder, xlrec->old_tuple_len); + memcpy(change->data.tp.oldtuple->t_data, old_tuple_data, + xlrec->old_tuple_len); + change->data.tp.oldtuple->t_len = xlrec->old_tuple_len; + ItemPointerSetInvalid(&change->data.tp.oldtuple->t_self); + change->data.tp.oldtuple->t_tableOid = InvalidOid; + + change->data.tp.newtuple = + ReorderBufferAllocTupleBuf(ctx->reorder, xlrec->new_tuple_len); + memcpy(change->data.tp.newtuple->t_data, new_tuple_data, + xlrec->new_tuple_len); + change->data.tp.newtuple->t_len = xlrec->new_tuple_len; + ItemPointerSetInvalid(&change->data.tp.newtuple->t_self); + change->data.tp.newtuple->t_tableOid = InvalidOid; + } + + change->data.tp.clear_toast_afterwards = true; + + ReorderBufferQueueChange(ctx->reorder, XLogRecGetXid(r), buf->origptr, + change, false); +} + +/* + * Parse XLOG_FLUX_DELETE from wal into proper tuplebufs. + * + * Deletes contain the old tuple for UNDO purposes. + */ +static void +DecodeFluxDelete(LogicalDecodingContext *ctx, XLogRecordBuffer *buf) +{ + XLogReaderState *r = buf->record; + xl_flux_delete *xlrec; + ReorderBufferChange *change; + char *tupledata; + RelFileLocator target_locator; + + xlrec = (xl_flux_delete *) XLogRecGetData(r); + + /* only interested in our database */ + XLogRecGetBlockTag(r, 0, &target_locator, NULL, NULL); + if (target_locator.dbOid != ctx->slot->data.database) + return; + + /* output plugin doesn't look for this origin, no need to queue */ + if (FilterByOrigin(ctx, XLogRecGetOrigin(r))) + return; + + change = ReorderBufferAllocChange(ctx->reorder); + change->action = REORDER_BUFFER_CHANGE_DELETE; + change->origin_id = XLogRecGetOrigin(r); + memcpy(&change->data.tp.rlocator, &target_locator, sizeof(RelFileLocator)); + + /* + * Get old tuple data from WAL record. The tuple follows the + * xl_flux_delete structure. + */ + tupledata = (char *) xlrec + sizeof(xl_flux_delete); + + if (xlrec->flags & FLUX_WAL_LOGICAL_TUPLE) + { + /* + * Heap image is the last region of the WAL record: ... [heap bytes] + * [uint32 heap_len] + */ + Size full_len = XLogRecGetDataLen(r); + char *end = (char *) xlrec + full_len; + uint32 heap_len; + char *heap_data; + + memcpy(&heap_len, end - sizeof(uint32), sizeof(uint32)); + heap_data = end - sizeof(uint32) - heap_len; + + change->data.tp.oldtuple = + ReorderBufferAllocTupleBuf(ctx->reorder, + heap_len - SizeofHeapTupleHeader); + change->data.tp.oldtuple->t_len = heap_len; + ItemPointerSetInvalid(&change->data.tp.oldtuple->t_self); + change->data.tp.oldtuple->t_tableOid = InvalidOid; + memcpy(change->data.tp.oldtuple->t_data, heap_data, heap_len); + } + else + { + /* Legacy path */ + change->data.tp.oldtuple = + ReorderBufferAllocTupleBuf(ctx->reorder, xlrec->tuple_len); + memcpy(change->data.tp.oldtuple->t_data, tupledata, + xlrec->tuple_len); + change->data.tp.oldtuple->t_len = xlrec->tuple_len; + ItemPointerSetInvalid(&change->data.tp.oldtuple->t_self); + change->data.tp.oldtuple->t_tableOid = InvalidOid; + } + + change->data.tp.clear_toast_afterwards = true; + + ReorderBufferQueueChange(ctx->reorder, XLogRecGetXid(r), buf->origptr, + change, false); +} + + +/* + * Handle rmgr FLUX records for LogicalDecodingProcessRecord(). + * + * FLUX's WAL record layout and the XLOG_FLUX_* opcodes are defined in + * flux_xlog.h; the FLUX decode helpers above interpret them directly. + */ +void +flux_decode(LogicalDecodingContext *ctx, XLogRecordBuffer *buf) +{ + uint8 info = XLogRecGetInfo(buf->record) & XLOG_FLUX_OPMASK; + TransactionId xid = XLogRecGetXid(buf->record); + SnapBuild *builder = ctx->snapshot_builder; + + ReorderBufferProcessXid(ctx->reorder, xid, buf->origptr); + + if (SnapBuildCurrentState(builder) < SNAPBUILD_FULL_SNAPSHOT) + return; + + switch (info) + { + case XLOG_FLUX_INSERT: + if (SnapBuildProcessChange(builder, xid, buf->origptr) && + !ctx->fast_forward) + DecodeFluxInsert(ctx, buf); + break; + + case XLOG_FLUX_MULTI_INSERT: + if (SnapBuildProcessChange(builder, xid, buf->origptr) && + !ctx->fast_forward) + DecodeFluxMultiInsert(ctx, buf); + break; + + case XLOG_FLUX_UPDATE_INPLACE: + if (SnapBuildProcessChange(builder, xid, buf->origptr) && + !ctx->fast_forward) + DecodeFluxUpdate(ctx, buf); + break; + + case XLOG_FLUX_DELETE: + if (SnapBuildProcessChange(builder, xid, buf->origptr) && + !ctx->fast_forward) + DecodeFluxDelete(ctx, buf); + break; + + case XLOG_FLUX_DEFRAG: + case XLOG_FLUX_COMPRESS: + case XLOG_FLUX_OVERFLOW_WRITE: + case XLOG_FLUX_INIT_PAGE: + break; + + default: + elog(ERROR, "unexpected RM_FLUX_ID record type: %u", info); + break; + } +} + /* * Ask output plugin whether we want to skip this PREPARE and send * this transaction as a regular commit later. diff --git a/src/backend/utils/misc/guc_parameters.dat b/src/backend/utils/misc/guc_parameters.dat index ae437b6679f21..96914f01d7719 100644 --- a/src/backend/utils/misc/guc_parameters.dat +++ b/src/backend/utils/misc/guc_parameters.dat @@ -1105,6 +1105,13 @@ options => 'file_extend_method_options', }, +{ name => 'flux_lazy_uncommitted_clear', type => 'bool', context => 'PGC_USERSET', group => 'CLIENT_CONN_STATEMENT', + short_desc => 'Skip commit-time page re-visits for FLUX UNCOMMITTED flag clearing.', + flags => 'GUC_NOT_IN_SAMPLE', + variable => 'flux_lazy_uncommitted_clear', + boot_val => 'false', +}, + { name => 'from_collapse_limit', type => 'int', context => 'PGC_USERSET', group => 'QUERY_TUNING_OTHER', short_desc => 'Sets the FROM-list size beyond which subqueries are not collapsed.', long_desc => 'The planner will merge subqueries into upper queries if the resulting FROM list would have no more than this many items.', diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index 64c0eb8ee9148..2ae769b8fcc67 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -30,6 +30,7 @@ #endif #include "access/commit_ts.h" +#include "access/flux.h" #include "access/gin.h" #include "access/logical_revert_worker.h" #include "access/slog.h" diff --git a/src/bin/pg_waldump/rmgrdesc.c b/src/bin/pg_waldump/rmgrdesc.c index fda1b2b47e93f..9d4ec1b17b02a 100644 --- a/src/bin/pg_waldump/rmgrdesc.c +++ b/src/bin/pg_waldump/rmgrdesc.c @@ -17,6 +17,7 @@ #include "access/hash_xlog.h" #include "access/heapam_xlog.h" #include "access/multixact.h" +#include "access/flux_xlog.h" #include "access/nbtxlog.h" #include "access/rmgr.h" #include "access/spgxlog.h" diff --git a/src/bin/pg_waldump/t/001_basic.pl b/src/bin/pg_waldump/t/001_basic.pl index d1aafc2f37d56..07f65b4a38cda 100644 --- a/src/bin/pg_waldump/t/001_basic.pl +++ b/src/bin/pg_waldump/t/001_basic.pl @@ -83,7 +83,8 @@ XLOG2 Undo ATM -RelUndo$/, +RelUndo +FLUX$/, 'rmgr list'); diff --git a/src/include/access/flux.h b/src/include/access/flux.h new file mode 100644 index 0000000000000..76cced74147c2 --- /dev/null +++ b/src/include/access/flux.h @@ -0,0 +1,910 @@ +/*------------------------------------------------------------------------- + * + * flux.h + * FLUX table access method definitions + * + * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/access/flux.h + * + *------------------------------------------------------------------------- + */ +#ifndef FLUX_H +#define FLUX_H + +#include "postgres.h" + +#include "access/heapam.h" +#include "access/relundo.h" +#include "access/toast_helper.h" +#include "storage/shmem.h" +#include "access/relscan.h" +#include "access/sdir.h" +#include "access/tableam.h" +#include "access/xact.h" +#include "executor/tuptable.h" +#include "port/atomics.h" +#include "storage/buf.h" +#include "storage/bufpage.h" +#include "storage/procnumber.h" +#include "utils/rel.h" +#include "utils/snapshot.h" + +/* + * FLUX special space structure - stored in page special space (8 bytes) + * + * Packs the page-level commit timestamp and 3 flag bits into a single + * uint64. The top 3 bits (63-61) store page flags; the lower 61 bits + * store the commit timestamp (sufficient for 73,000+ years of microseconds). + * + * pd_free_space was removed -- use PageGetFreeSpace() directly (same as + * heap). pd_flags was removed -- flags are packed into the timestamp word. + */ +typedef struct FluxPageOpaqueData +{ + uint64 pd_commit_ts_and_flags; /* bits 63-61: flags, bits 60-0: ts */ +} FluxPageOpaqueData; + +typedef FluxPageOpaqueData *FluxPageOpaque; + +/* Page flags (stored in top 3 bits of pd_commit_ts_and_flags) */ +#define FLUX_PAGE_FLAG_SHIFT 61 +#define FLUX_PAGE_FLAG_MASK (UINT64CONST(0x7) << FLUX_PAGE_FLAG_SHIFT) +#define FLUX_PAGE_TS_MASK (~FLUX_PAGE_FLAG_MASK) + +#define FLUX_PAGE_OVERFLOW (UINT64CONST(1) << 61) +#define FLUX_PAGE_DEFRAG_NEEDED (UINT64CONST(1) << 62) +#define FLUX_PAGE_FULL (UINT64CONST(1) << 63) + +/* Accessor macros for page opaque */ +#define FluxPageGetOpaque(page) \ + ((FluxPageOpaque) PageGetSpecialPointer(page)) + +#define FluxPageGetCommitTs(opaque) \ + ((opaque)->pd_commit_ts_and_flags & FLUX_PAGE_TS_MASK) + +#define FluxPageSetCommitTs(opaque, ts) \ + ((opaque)->pd_commit_ts_and_flags = \ + ((opaque)->pd_commit_ts_and_flags & FLUX_PAGE_FLAG_MASK) | \ + ((uint64)(ts) & FLUX_PAGE_TS_MASK)) + +#define FluxPageGetFlags(opaque) \ + ((opaque)->pd_commit_ts_and_flags & FLUX_PAGE_FLAG_MASK) + +#define FluxPageSetFlag(opaque, flag) \ + ((opaque)->pd_commit_ts_and_flags |= (flag)) + +#define FluxPageClearFlag(opaque, flag) \ + ((opaque)->pd_commit_ts_and_flags &= ~(flag)) + +/* + * FLUX tuple header structure (v3 -- heap-compatible xmin/xmax MVCC) + * + * Visibility is ordinary heap-shaped xmin/xmax + CLOG + snapshot (the same + * model HeapTupleSatisfiesMVCC uses), NOT the former HLC/sLog timestamp + * scheme. A tuple is visible to a snapshot iff its inserter (t_xmin) is + * committed-and-visible-to-the-snapshot AND its deleter/updater (t_xmax) is + * either invalid, not committed, or not visible to the snapshot. + * + * - t_xmin: the inserting transaction's XID. Always valid once the tuple + * exists on the page (it rides through WAL redo verbatim in the logged + * tuple body, so it needs no separate redo reconstruction). + * - t_xmax: the deleting/updating transaction's XID, or InvalidTransactionId + * for a live never-superseded tuple. Physically stored in the low 32 + * bits of the t_commit_ts word (see FluxTupleGetXmax below) so the on-disk + * and WAL byte layout is unchanged from the HLC era; the DML paths and + * WAL now carry an XID in that slot instead of a timestamp. + * + * In-place UPDATE keeps the newest version on the page (new t_xmin) and pushes + * the pre-update image to the per-relation UNDO fork via t_verptr (zheap + * style). A snapshot that predates the update reads the old version back from + * the fork with FluxReconstructVisibleVersion(), which walks t_verptr and + * stops at the version whose producing xid is visible (XidInMVCCSnapshot). + * + * Transient operation state (who is inserting/deleting/locking concurrently) + * is still tracked in the sLog for write-write conflict serialization and SSI, + * but it is no longer consulted for read visibility -- CLOG is authoritative. + * + * t_writer: Per-tuple CAS writer lock for same-size updates under + * BUFFER_LOCK_SHARE. 0 = unlocked; non-zero = (MyProcNumber + 1) of + * the writer. Operated on via atomic CAS through FluxTupleWriter* + * macros below. It sits in the fixed header (see the field list and the + * 40-byte total documented on the struct below); it is never read on the + * visibility path, only CAS'd during an in-flight same-size update. + */ +typedef struct FluxTupleHeader +{ + uint64 t_commit_ts; /* 8B low 32 bits = t_xmax (deleter/updater + * XID); high 32 bits reserved. Accessed via + * FluxTupleGetXmax/SetXmax. Kept as a uint64 + * field so the on-disk/WAL byte layout is + * unchanged from the HLC era. */ + RelUndoRecPtr t_verptr; /* 8B UNDO-fork version-chain head (WS-PVS1) */ + uint32 t_writer; /* 4B Per-tuple CAS writer lock (0=free) */ + + /* + * t_gen: RESERVED, unused in FLUX. FLUX uses plain heap-TID index + * identity (no RowID/gen scheme): a key-changing UPDATE goes out-of-place + * (new TID) and maintains indexes the standard heap way, so no per-tuple + * generation is needed to disambiguate index entries. The field is + * retained (kept at 0) as reserved header space rather than changing the + * tuple geometry; a future slimming pass may remove it. + */ + uint32 t_gen; /* 4B reserved (unused in FLUX) */ + + /* + * t_xmin: the inserting transaction's XID (heap xmin semantics). Always + * valid once the tuple exists. Visibility resolves it against CLOG and + * the reader's snapshot exactly like HeapTupleSatisfiesMVCC, so no HLC + * timestamp or sLog lookup is needed on the common read path. The tuple + * body (including this field) is WAL-logged verbatim, so t_xmin survives + * redo without separate reconstruction. + * + * (Formerly t_xid_hint, which was "valid only while UNCOMMITTED"; the + * heap-shaped model makes it permanently authoritative as xmin.) + */ + TransactionId t_xmin; /* 4B Inserter XID (heap xmin) */ + + /* + * t_verptr is the head of this tuple's persistent version chain in the + * UNDO fork (WS-PVS1/2). It was formerly an unaligned 8-byte trailer at + * the end of the on-page item, located by item_len - 8 arithmetic and + * gated by FLUX_TUPLE_HAS_VERSION_PTR. Since every tuple now reserves it + * from birth (born-with-flag), the trailer was always present, so it is a + * plain aligned header field: no growth on first UPDATE, no packed-page + * failure, no trailing-byte arithmetic. InvalidRelUndoRecPtr means + * "never updated -- no history"; readers treat that as "on-page image is + * current". + * + * t_cid removed: command ID is now obtained from the sLog entry + * (FluxSLogEntry.cid) when FLUX_TUPLE_UNCOMMITTED is set. This saves 4 + * bytes per tuple. The sLog lookup is mandatory for uncommitted + * DELETE/UPDATE visibility anyway, so fetching the cid from there adds + * zero extra overhead. (INSERT visibility no longer needs the sLog at + * all in the common case -- see t_xmin above.) + */ + ItemPointerData t_ctid; /* 6B Current TID / update chain */ + uint16 t_natts; /* 2B Number of attributes */ + uint16 t_flags; /* 2B Tuple flags */ + uint8 t_infomask; /* 1B HASNULL, HASVARWIDTH, etc. */ + uint8 t_pad; /* 1B padding so t_attrs_bitmap begins at a + * MAXALIGN boundary equal to + * FLUX_TUPLE_OVERHEAD. Without this the null + * bitmap (anchored at t_attrs_bitmap) and the + * column data (anchored at + * FLUX_TUPLE_OVERHEAD) disagree, corrupting + * reads. The whole header is still MAXALIGN'd + * to 40 bytes (8+8+4+4+4+6+2+2+1+1 = 40), so + * no footprint change. */ + uint8 t_attrs_bitmap[FLEXIBLE_ARRAY_MEMBER]; +} FluxTupleHeader; + +/* + * Fixed size: 40 bytes raw (MAXALIGN'd to 40 bytes): 8 t_commit_ts + 8 t_verptr + * + 4 t_writer + 4 t_gen + 4 t_xmin + 6 t_ctid + 2 t_natts + 2 t_flags + * + 1 t_infomask + 1 t_pad = 40, then the FLEXIBLE_ARRAY_MEMBER null bitmap. + */ + +/* + * Per-tuple CAS writer lock accessor macros. + * + * t_writer is a plain uint32 on disk (initialized to 0 by palloc0/memset). + * At runtime we operate on it via pg_atomic_compare_exchange_u32 by casting + * its address to (pg_atomic_uint32 *). This is safe on all PostgreSQL + * platforms because pg_atomic_uint32 is { volatile uint32 value; } with + * identical size and alignment. + * + * FluxTupleWriterTryLock: CAS 0 -> (MyProcNumber+1). Returns true on success. + * FluxTupleWriterUnlock: Atomic write 0 (release). + * FluxTupleWriterIsLocked: Non-zero check (relaxed read). + */ +#define FluxTupleWriterTryLock(hdr, expected_ptr) \ + pg_atomic_compare_exchange_u32((pg_atomic_uint32 *) &(hdr)->t_writer, \ + (expected_ptr), (uint32)(MyProcNumber + 1)) + +#define FluxTupleWriterUnlock(hdr) \ + pg_atomic_write_u32((pg_atomic_uint32 *) &(hdr)->t_writer, 0) + +#define FluxTupleWriterIsLocked(hdr) \ + (pg_atomic_read_u32((pg_atomic_uint32 *) &(hdr)->t_writer) != 0) + +/* Tuple flags (uint16) */ +#define FLUX_TUPLE_COMPRESSED 0x0001 +#define FLUX_TUPLE_HAS_OVERFLOW 0x0002 +#define FLUX_TUPLE_DELETED 0x0004 +#define FLUX_TUPLE_UPDATED 0x0008 +#define FLUX_TUPLE_LOCKED 0x0010 +#define FLUX_TUPLE_SPECULATIVE 0x0020 +#define FLUX_TUPLE_UNCOMMITTED 0x0080 /* Inserted but not yet committed */ +#define FLUX_TUPLE_HAS_VERSION_PTR 0x0100 /* trailing RelUndoRecPtr + * (version-chain head) follows + * column data */ +#define FLUX_TUPLE_XMIN_COMMITTED 0x0200 /* t_xmin known-committed (CLOG + * hint, like heap + * HEAP_XMIN_COMMITTED) */ +#define FLUX_TUPLE_XMAX_COMMITTED 0x0400 /* t_xmax known-committed (CLOG + * hint, like heap + * HEAP_XMAX_COMMITTED) */ + +/* Tuple infomask bits (uint8 -- reduced from uint16) */ +#define FLUX_INFOMASK_HASNULL 0x01 +#define FLUX_INFOMASK_HASVARWIDTH 0x02 +#define FLUX_INFOMASK_HASEXTERNAL 0x04 +#define FLUX_INFOMASK_COMPRESSED 0x08 +#define FLUX_INFOMASK_HASOVERFLOW 0x10 + +/* + * FLUX tuple structure + */ +typedef struct FluxTupleData +{ + uint32 t_len; /* Length of tuple */ + ItemPointerData t_self; /* TID of this tuple */ + Oid t_tableOid; /* Table OID */ + FluxTupleHeader *t_data; /* Tuple header and data */ +} FluxTupleData; + +typedef FluxTupleData *FluxTuple; + +/* + * Column-level overflow + * + * When an individual column value is too large to store inline in the main + * tuple, it is stored as one or more "overflow records" on normal FLUX data + * pages. The main tuple stores a compact overflow pointer (FluxOverflowPtr) + * wrapped in a varlena, optionally preceded by an inline prefix of the + * original data for efficient prefix matching. + * + * Overflow records use a lightweight header (FluxOverflowRecordHeader) + * without MVCC fields -- they share the visibility of the parent tuple. + * Each overflow record holds a chunk of the column data and a continuation + * pointer to the next chunk (or InvalidBlockNumber if this is the last). + * + * This approach stores overflow data on regular pages that can also hold + * normal tuples, within the same relation -- no separate out-of-line + * relation is created. + */ + +/* + * Overflow pointer stored inline in the main tuple (wrapped as varlena). + * + * On-disk layout of an overflowed column in the main tuple: + * [varlena header][FluxOverflowPtr][inline_prefix_bytes...] + * + * The FLUX_OVERFLOW_PTR_MAGIC sentinel distinguishes this from a normal + * varlena value during deform. + */ +#define FLUX_OVERFLOW_PTR_MAGIC 0x52564F50 /* "RVOP" */ + +typedef struct FluxOverflowPtr +{ + uint32 ov_magic; /* FLUX_OVERFLOW_PTR_MAGIC */ + BlockNumber ov_first_block; /* First overflow record's page */ + OffsetNumber ov_first_offset; /* First overflow record's offset on page */ + uint16 ov_inline_prefix; /* Bytes of inline prefix stored after ptr */ + uint32 ov_total_length; /* Total uncompressed column data length */ + uint32 ov_content_hash; /* 32-bit content-hash prefilter for COW */ +} FluxOverflowPtr; + +/* + * ov_content_hash reclaims the two formerly-reserved uint16 fields + * (ov_padding, ov_flags), so the struct stays 20 bytes and + * FLUX_OVERFLOW_PTR_SIZE is unchanged. A wider (64-bit) hash would force + * struct growth that inflates FLUX_OVERFLOW_PTR_SIZE and defeats the + * force-shrink UPDATE recovery path, breaking in-place updates on full pages. + * + * The hash is only a cheap prefilter: every COW-reference candidate is + * byte-verified against the fetched old chain before it is accepted, so a + * 32-bit collision merely costs a wasted fetch and falls through to a normal + * re-store. It never produces an incorrect result. + */ + +/* Minimum varlena size for an overflow pointer (no inline prefix) */ +#define FLUX_OVERFLOW_PTR_SIZE (VARHDRSZ + sizeof(FluxOverflowPtr)) + +/* Default inline prefix size (configurable via GUC) */ +#define FLUX_OVERFLOW_DEFAULT_PREFIX 128 + +/* + * Check if a varlena datum is an overflow pointer. + * + * The check requires: correct size range, and magic value match. + */ +static inline bool +FluxIsOverflowPtr(const void *ptr) +{ + Size vsize; + const FluxOverflowPtr *ovp; + + if (ptr == NULL) + return false; + + vsize = VARSIZE_ANY_EXHDR(ptr); + if (vsize < sizeof(FluxOverflowPtr)) + return false; + + ovp = (const FluxOverflowPtr *) VARDATA_ANY(ptr); + return ovp->ov_magic == FLUX_OVERFLOW_PTR_MAGIC; +} + +/* + * Extract overflow pointer from a varlena datum. + */ +static inline const FluxOverflowPtr * +FluxGetOverflowPtr(const void *ptr) +{ + return (const FluxOverflowPtr *) VARDATA_ANY(ptr); +} + +/* + * Lightweight header for overflow records stored on normal data pages. + * + * Overflow records are stored via PageAddItem just like normal tuples, but + * they carry this minimal header instead of a full FluxTupleHeader. The + * ov_magic field lets us distinguish overflow records from normal tuples + * during page scans (e.g., sequential scan must skip these). + */ +#define FLUX_OVERFLOW_RECORD_MAGIC 0x524F5643 /* "ROVC" */ + +typedef struct FluxOverflowRecordHeader +{ + uint32 or_magic; /* FLUX_OVERFLOW_RECORD_MAGIC */ + uint32 or_data_len; /* Bytes of column data in this record */ + BlockNumber or_next_block; /* Next overflow record's page, or Invalid */ + OffsetNumber or_next_offset; /* Next overflow record's offset */ + uint16 or_flags; /* Flags (reserved) */ + /* Column data follows immediately after this header */ +} FluxOverflowRecordHeader; + +/* Maximum column data per overflow record */ +#define FLUX_OVERFLOW_RECORD_OVERHEAD MAXALIGN(sizeof(FluxOverflowRecordHeader)) +#define FLUX_OVERFLOW_MAX_CHUNK_SIZE \ + (FLUX_MAX_TUPLE_SIZE - FLUX_OVERFLOW_RECORD_OVERHEAD) + +/* + * Structure to track overflow buffers for atomic WAL logging. + * + * When creating overflow chains, we keep buffers pinned and collect them + * here so the caller can register them all in a single WAL record with + * the main tuple modification. This ensures atomicity during crash recovery. + */ +#define MAX_OVERFLOW_BUFFERS 32 + +typedef struct FluxOverflowBuffer +{ + Buffer buffer; /* Pinned buffer containing overflow record */ + OffsetNumber offset; /* Offset of overflow record on page */ + char *record_data; /* FluxOverflowRecordHeader + data */ + uint32 record_len; /* Total record length */ + uint16 flags; /* FLUX_OVERFLOW_WAL_NEW_RECORD or + * _LINK_UPDATE */ +} FluxOverflowBuffer; + +typedef struct FluxOverflowBuffers +{ + int count; /* Number of overflow buffers */ + FluxOverflowBuffer buffers[MAX_OVERFLOW_BUFFERS]; +} FluxOverflowBuffers; + +/* + * Compression types + */ +typedef enum FluxCompressionType +{ + FLUX_COMP_NONE, + FLUX_COMP_LZ4, + FLUX_COMP_ZSTD, + FLUX_COMP_DELTA, /* For numeric columns */ + FLUX_COMP_DICTIONARY /* For text columns */ +} FluxCompressionType; + +/* + * Values for the flux_compression_algorithm GUC. AUTO lets + * FluxChooseCompressionType() pick per attribute type; LZ4/ZSTD force that + * codec for compressible varlena attributes; NONE disables compression. + */ +typedef enum FluxCompressionAlgoGuc +{ + FLUX_COMP_ALGO_AUTO, + FLUX_COMP_ALGO_LZ4, + FLUX_COMP_ALGO_ZSTD, + FLUX_COMP_ALGO_OFF +} FluxCompressionAlgoGuc; + +typedef struct FluxCompressionHeader +{ + uint8 comp_type; + uint8 comp_level; + uint16 dict_id; /* trained-dict id, 0 = FLUX_DICT_INVALID_ID */ + uint32 orig_size; + uint32 comp_size; +} FluxCompressionHeader; + +/* + * FLUX timestamp word. + * + * A uint64 wall-clock timestamp (microseconds since the PG epoch) used for + * per-page commit-ts bookkeeping (FluxPageSetCommitTs). It is NOT a + * visibility timestamp: commit visibility comes from CLOG via heap-shaped + * xmin/xmax MVCC. + */ + +/* + * Tuple MVCC field accessors (heap-shaped xmin/xmax). + * + * t_xmax is physically the low 32 bits of the t_commit_ts word; the high 32 + * bits are reserved. This preserves the on-disk/WAL byte layout while giving + * the tuple a heap-compatible deleter/updater XID. InvalidTransactionId (0) + * means "live, never superseded". + */ +#define FluxTupleGetXmax(tup) ((TransactionId) ((tup)->t_commit_ts & 0xFFFFFFFFULL)) +#define FluxTupleSetXmax(tup, xid) \ + ((tup)->t_commit_ts = ((tup)->t_commit_ts & 0xFFFFFFFF00000000ULL) | \ + ((uint64) (TransactionId) (xid))) +#define FluxTupleGetXmin(tup) ((tup)->t_xmin) +#define FluxTupleSetXmin(tup, xid) ((tup)->t_xmin = (TransactionId) (xid)) + +/* + * Transaction state for uncertainty tracking. + * + * The full struct definition lives in flux_mvcc.c (private to that module). + * External code should use the opaque forward declaration below. + */ +typedef struct FluxTransactionState FluxTransactionState; + +/* + * Free space management + */ +typedef struct FluxFreeSpaceMap +{ + uint32 total_pages; + uint32 pages_with_space; + uint8 *fsm_data; /* Bitmap of page utilization */ + uint32 *defrag_queue; /* Pages needing defragmentation */ + uint32 defrag_queue_size; +} FluxFreeSpaceMap; + +/* Free space map levels */ +#define FLUX_FSM_FULL 0 +#define FLUX_FSM_75_PERCENT 1 +#define FLUX_FSM_50_PERCENT 2 +#define FLUX_FSM_25_PERCENT 3 +#define FLUX_FSM_EMPTY 4 + +/* + * Visibility Map support for FLUX + * + * The visibility map tracks two bits per page: + * - ALL_VISIBLE: all tuples on page are visible to all transactions + * - ALL_FROZEN: all tuples on page are frozen (no further VACUUM needed) + * + * This enables: + * - Index-only scans (can skip heap fetch if page is all-visible) + * - VACUUM optimization (can skip pages marked all-visible/frozen) + */ + +/* Visibility map bits */ +#define FLUX_VM_ALL_VISIBLE 0x01 /* All tuples visible to all xacts */ +#define FLUX_VM_ALL_FROZEN 0x02 /* All tuples frozen */ + +/* Combined flags for convenience */ +#define FLUX_VM_VALID_BITS (FLUX_VM_ALL_VISIBLE | FLUX_VM_ALL_FROZEN) + +/* Visibility map fork number (uses PostgreSQL's fork infrastructure) */ +#define FLUX_VM_FORKNUM VISIBILITYMAP_FORKNUM + +/* + * Scan descriptor for FLUX scans + */ +typedef struct FluxScanDescData +{ + TableScanDescData rs_base; /* Base scan descriptor */ + Buffer rs_cbuf; /* Current buffer */ + BlockNumber rs_cblock; /* Current block */ + BlockNumber rs_nblocks; /* Total blocks in relation (cached) */ + BlockNumber rs_startblock; /* Starting block for sample scans */ + OffsetNumber rs_cindex; /* Current offset in page */ + OffsetNumber rs_coffset; /* Current offset number */ + bool rs_inited; /* True after first block is fetched */ + int rs_ntuples; /* Number of tuples on current page */ + OffsetNumber *rs_vistuples; /* Offset numbers of visible tuples */ + uint64 rs_snapshot_ts; /* Snapshot timestamp */ + uint64 rs_xact_ts; /* Transaction timestamp */ + ParallelBlockTableScanWorkerData *rs_parallelworkerdata; /* Parallel scan worker + * state */ + struct ReadStream *rs_read_stream; /* Read stream for sequential + * prefetching */ + BlockNumber rs_prefetch_block; /* Next block for read stream callback */ + + /* Cached visibility map buffer to avoid per-page VM I/O */ + Buffer rs_vm_buffer; /* Pinned VM buffer (or InvalidBuffer) */ + BlockNumber rs_vm_blockno; /* VM block number for rs_vm_buffer */ + + /* + * ANALYZE dictionary-refresh sample accumulation. During an ANALYZE scan + * the decompressed bytes of the first varlena column are gathered here so + * FluxMaybeRefreshDict() can train a candidate dictionary at scan end. + * All fields stay zero/NULL on non-ANALYZE scans. + */ + char *rs_dict_samplebuf; /* Concatenated sample bytes */ + size_t *rs_dict_sizes; /* Per-sample lengths, in order */ + int rs_dict_nsamples; /* Number of accumulated samples */ + Size rs_dict_total; /* Total bytes in rs_dict_samplebuf */ + Size rs_dict_cap; /* Capacity of rs_dict_samplebuf */ + int rs_dict_maxsamples; /* Capacity of rs_dict_sizes */ + int16 rs_dict_attnum; /* 1-based varlena attr sampled, 0 = none */ +} FluxScanDescData; + +typedef FluxScanDescData *FluxScanDesc; + +/* + * Index fetch table data for FLUX + */ +typedef struct IndexFetchFluxData +{ + IndexFetchTableData base; /* AM independent part of the descriptor */ + + Buffer buffer; +} IndexFetchFluxData; + +/* + * Constants + */ +#define FLUX_PAGE_OVERHEAD (MAXALIGN(SizeOfPageHeaderData) + MAXALIGN(sizeof(FluxPageOpaqueData))) +#define FLUX_TUPLE_OVERHEAD (MAXALIGN(sizeof(FluxTupleHeader))) + +/* + * The null bitmap is anchored at t_attrs_bitmap and the column data is + * anchored at FLUX_TUPLE_OVERHEAD; the two MUST coincide or reads corrupt. + * t_pad keeps t_attrs_bitmap on the MAXALIGN boundary that equals the + * overhead. Enforce it so a future header change cannot silently break it. + */ +StaticAssertDecl(offsetof(FluxTupleHeader, t_attrs_bitmap) == FLUX_TUPLE_OVERHEAD, + "FLUX tuple null bitmap must begin at FLUX_TUPLE_OVERHEAD"); +#define FLUX_MAX_TUPLE_SIZE MAXALIGN_DOWN(BLCKSZ - FLUX_PAGE_OVERHEAD - sizeof(ItemIdData)) +#define FLUX_OVERFLOW_THRESHOLD (FLUX_MAX_TUPLE_SIZE / 4) + +/* + * Hard ceiling on the number of line pointers a FLUX page can hold. Unlike + * heap, FLUX pages mix full tuples with small overflow-continuation records, + * and FluxPageAddTuple deliberately omits PAI_IS_HEAP so PageAddItemExtended + * does not clamp offsets to MaxHeapTuplesPerPage. The smallest storable item + * is an overflow-record header, so the densest possible packing is bounded by + * that item size plus its line pointer. This is the FLUX-true analogue of + * MaxHeapTuplesPerPage and must be used for dense TID encoding so that every + * valid offset maps to a distinct index without aliasing into the next block. + */ +#define MaxFluxItemsPerPage \ + ((int) ((BLCKSZ - FLUX_PAGE_OVERHEAD) / \ + (FLUX_OVERFLOW_RECORD_OVERHEAD + sizeof(ItemIdData)))) + +/* + * Fill factor support. Unlike heap, FLUX has STABLE TIDs and updates rows + * IN PLACE -- a row that grows (e.g. an accumulating numeric like TPC-C + * w_ytd/d_ytd gaining a digit) cannot be relocated to another page the way + * heap moves a grown tuple. It must fit on its home page or the UPDATE fails + * with "does not fit". So FLUX must reserve per-page headroom by default; + * packing pages 100%% full (heap's default) guarantees that any in-place + * growth on a full page aborts. The default reserves ~10%% of each page, + * which comfortably covers digit-growth of numeric columns plus transient + * line-pointer bloat from concurrent updates to a hot page. Users can raise + * it with WITH (fillfactor=N) for append-mostly tables that never grow rows. + */ +#define FLUX_MIN_FILLFACTOR 10 +#define FLUX_DEFAULT_FILLFACTOR 90 + +/* Macros for tuple access */ +#define FluxTupleGetHeader(tuple) ((tuple)->t_data) +#define FluxTupleGetData(tuple) \ + ((char *) (tuple)->t_data + FLUX_TUPLE_OVERHEAD) + +/* + * Version-pointer accessors (WS-PVS1). + * + * The version-chain head now lives in the fixed header field t_verptr (it + * was formerly an unaligned 8-byte trailer located by item_len - 8). The + * item_len parameter is retained for source compatibility with existing + * call sites but is no longer used. + */ +static inline RelUndoRecPtr +FluxTupleGetVersionPtr(const FluxTupleHeader *hdr, Size item_len) +{ + (void) item_len; + return hdr->t_verptr; +} + +static inline void +FluxTupleSetVersionPtr(FluxTupleHeader *hdr, Size item_len, RelUndoRecPtr ptr) +{ + (void) item_len; + hdr->t_verptr = ptr; +} + +/* + * FluxReconstructVisibleVersion (WS-PVS2) + * + * Walk the per-tuple version chain in the UNDO fork to find the image + * the reader's MVCC snapshot should see in place of the on-page (newer) + * data. Returns true and populates *out_data / *out_len with a palloc'd + * reconstructed image when a step back was taken; returns false when the + * on-page image is what the reader should see. + * + * See src/backend/access/flux/flux_pvs.c for the algorithm. + */ +extern bool FluxReconstructVisibleVersion(Relation rel, ItemPointer tid, + const char *onpage_image, + Size onpage_len, + Snapshot snapshot, + char **out_data, int *out_len); + +/* Slot operations for FLUX tuples */ +extern PGDLLIMPORT const TupleTableSlotOps TTSOpsFluxTuple; +extern void FluxSlotStoreTuple(TupleTableSlot *slot, FluxTupleHeader *tuple, + uint32 tuple_len, Buffer buffer); +extern void FluxSlotStoreMaterializedTuple(TupleTableSlot *slot, + FluxTupleHeader *tuple, + uint32 tuple_len); + +#define TTS_IS_FLUXTUPLE(slot) ((slot)->tts_ops == &TTSOpsFluxTuple) + +/* Function prototypes */ +extern Size FluxComputeDataSize(TupleDesc tupdesc, Datum *values, bool *isnull); +extern void flux_toast_tuple(Relation rel, Datum *values, bool *isnull, + Datum *oldvalues, bool *oldisnull, + ToastTupleContext *ttc, ToastAttrInfo *toast_attr, + bool *changed, uint32 options); +extern void flux_toast_cleanup(ToastTupleContext *ttc); +extern void flux_toast_delete(Relation rel, Datum *values, bool *isnull, + bool is_speculative); +extern FluxTuple FluxFormTuple(TupleDesc tupdesc, Datum *values, bool *isnull, + Relation rel, FluxOverflowBuffers *overflow_buffers); +extern FluxTuple FluxFormTupleForceShrink(TupleDesc tupdesc, Datum *values, + bool *isnull, Relation rel, + FluxOverflowBuffers *overflow_buffers); +extern FluxTuple FluxFormTupleUpdate(TupleDesc tupdesc, Datum *values, + bool *isnull, Relation rel, + FluxOverflowBuffers *overflow_buffers, + const FluxOverflowPtr *old_ovptrs, + const bool *old_ovpresent); +extern void FluxDeformTuple(Relation rel, FluxTuple tuple, TupleDesc tupdesc, Datum *values, bool *isnull); +extern void FluxDeformTupleUpTo(Relation rel, FluxTuple tuple, TupleDesc tupdesc, Datum *values, bool *isnull, int max_natts); +extern void FluxFreeTuple(FluxTuple tuple); +extern bool FluxTupleToSlot(FluxTupleHeader *tuple_header, TupleTableSlot *slot); +extern bool FluxTupleToSlotWithOverflow(FluxTupleHeader *tuple_header, + TupleTableSlot *slot, Relation rel); + +/* Page management */ +extern void FluxInitPage(Page page, Size pageSize); +extern OffsetNumber FluxPageAddTuple(Page page, FluxTuple tuple, Size tuple_size); +extern bool FluxPageUpdateTuple(Page page, OffsetNumber offnum, FluxTuple new_tuple, + uint64 old_commit_ts, uint64 new_commit_ts); +extern int FluxPageGetLiveTuples(Page page, uint64 snapshot_ts); +extern void FluxPageDefragment(Page page); +extern void FluxPageIndexTupleDelete(Page page, OffsetNumber offnum); +extern int FluxPagePruneOpt(Relation rel, Buffer buffer); + +/* Overflow handling - column-level overflow */ +extern Datum FluxStoreOverflowColumn(Relation rel, Datum value, int attnum, + Size inline_prefix_size, + FluxOverflowBuffers *overflow_buffers); +extern Datum FluxFetchOverflowColumn(Relation rel, const void *overflow_varlena); +extern void FluxDeleteOverflowChain(Relation rel, BlockNumber first_block, + OffsetNumber first_offset); +extern int FluxCollectOverflowPtrs(FluxTupleHeader *tuple_hdr, + TupleDesc tupdesc, + BlockNumber *blocks, OffsetNumber *offsets, + int max_ptrs); +extern void FluxCollectOverflowPtrsByAttr(FluxTupleHeader *tuple_hdr, + TupleDesc tupdesc, + FluxOverflowPtr *out_ptrs, + bool *out_present, int natts); +extern void FluxDeleteTupleOverflows(Relation rel, FluxTupleHeader *tuple_hdr, + TupleDesc tupdesc); +extern bool FluxIsOverflowRecord(const void *item, Size item_len); + +/* + * Inline version of FluxIsOverflowRecord for hot scan paths. + * Checks whether an item is an overflow continuation record by testing + * the magic number in the header. + */ +static inline bool +FluxIsOverflowRecordInline(const void *item, Size item_len) +{ + if (item_len < sizeof(FluxOverflowRecordHeader)) + return false; + return ((const FluxOverflowRecordHeader *) item)->or_magic == + FLUX_OVERFLOW_RECORD_MAGIC; +} +extern void FluxGetOverflowStats(Relation rel, int64 *total_overflow_records, + int64 *total_overflow_bytes, int64 *avg_chain_length); + +/* Free space management */ +extern BlockNumber FluxGetPageWithFreeSpace(Relation rel, Size needed); +extern void FluxRecordFreeSpace(Relation rel, BlockNumber page, Size freespace); +extern void FluxVacuumFSM(Relation rel, BlockNumber new_nblocks); + +/* Visibility Map management */ +extern void FluxVMInit(Relation rel); +extern void FluxVMSet(Relation rel, BlockNumber heapBlk, Buffer heapBuf, uint8 flags); +extern void FluxVMClear(Relation rel, BlockNumber heapBlk, Buffer heapBuf, uint8 flags); +extern bool FluxVMCheck(Relation rel, BlockNumber heapBlk, uint8 flags); +extern bool FluxVMCheckCached(Relation rel, BlockNumber heapBlk, uint8 flags, + Buffer *vmbuf, BlockNumber *vm_blockno); +extern void FluxVMPinBuffer(Relation rel, BlockNumber heapBlk, Buffer *vmbuf); +extern void FluxVMExtend(Relation rel, BlockNumber nheapblocks); +extern void FluxVMTruncate(Relation rel, BlockNumber nheapblocks); +extern Size FluxVMGetPageSize(void); +extern BlockNumber FluxVMMapHeapToVM(BlockNumber heapBlk); +extern void FluxVMUpdateForInsert(Relation rel, FluxTupleHeader *tuple, Buffer buffer); +extern void FluxVMUpdateForUpdate(Relation rel, Buffer buffer); +extern void FluxVMUpdateForDelete(Relation rel, Buffer buffer); +extern void FluxVMVacuumPage(Relation rel, Buffer buffer, bool all_visible, bool all_frozen); + +/* MVCC functions */ +extern uint64 FluxGetCommitTimestamp(void); +extern uint64 FluxGetTransactionTimestamp(void); +extern uint64 FluxGetOldestActiveTimestamp(void); +extern Size FluxMvccShmemSize(void); +extern void FluxMvccShmemInit(void); +extern const ShmemCallbacks FluxMvccShmemCallbacks; +extern void FluxCommitTransaction(void); +extern void FluxAbortTransaction(void); +extern uint64 FluxGetSnapshotTimestamp(Snapshot snapshot); +extern bool FluxTupleVisibleToSnapshot(FluxTupleHeader *tuple, Snapshot snapshot, + Oid relid, Buffer buffer); +extern void FluxUpdateOldestActiveTimestamp(void); +extern void FluxPrepareReassignSlot(int dummy_slot); +extern void FluxResolvePreparedSlot(int dummy_slot); +extern void FluxGetMvccStats(uint64 *current_ts, uint64 *oldest_ts, int *active_xacts); + +/* SSI (Serializable Snapshot Isolation) via predicate.c integration */ +extern void FluxCheckForSerializableConflictOut(Relation relation, + FluxTupleHeader *tuple, + Buffer buffer, + Snapshot snapshot); + +/* MVCC timestamp helpers (page-level bookkeeping; visibility uses xmin/xmax) */ +extern uint64 FluxGetDmlTimestamp(void); + +/* + * WS-PVS3 lost-update conflict probe (fork-driven). Reads the head verptr + * from the on-page tuple, resolves it in the UNDO fork, and reports whether + * that head record's committer is concurrent-or-later to snapshot (and not + * exclude_xid). If it is, the caller returns TM_Updated to drive EPQ. EPQ + * dedup lives in FluxTransactionState: FluxEpqReconcileMatches skips a + * probe we already bounced on for this exact (relid, tid, cid, head verptr, + * head xid); FluxEpqReconcileMark stamps the identity we just bounced on. + */ +extern bool FluxTupleHasCommittedUpdateAfter(Relation rel, + const FluxTupleHeader *tuple, + Size tuple_len, + Snapshot snapshot, + TransactionId exclude_xid, + RelUndoRecPtr *out_head_verptr, + TransactionId *out_head_xid, + bool *out_inprogress); +extern bool FluxEpqReconcileMatches(Snapshot snapshot, Oid relid, + ItemPointer tid, + RelUndoRecPtr head_verptr, + TransactionId head_xid); +extern void FluxEpqReconcileMark(Snapshot snapshot, Oid relid, + ItemPointer tid, + RelUndoRecPtr head_verptr, + TransactionId head_xid); +extern bool FluxTupleVisibleToSnapshotDual(FluxTupleHeader *tuple, + Snapshot snapshot, + Oid relid, Buffer buffer); +extern TransactionId FluxGetOldestXminHorizon(Relation rel); +extern bool FluxTupleDeadToAll(FluxTupleHeader *tuple, + TransactionId oldest_xmin); + +/* + * MultiXact support has been removed. Concurrent tuple locking is now + * tracked via the sLog (flux_slog.c). + */ + +/* Dirty block map (lock-free sLog bypass) */ +extern Size FluxDirtyMapShmemSize(void); +extern void FluxDirtyMapShmemInit(void); +extern const ShmemCallbacks FluxDirtyMapShmemCallbacks; + +extern bool flux_lazy_uncommitted_clear; + + +/* Lock operations */ +extern bool FluxLockTuple(Relation rel, ItemPointer tid, LockTupleMode mode, + bool wait, bool *have_tuple_lock); +extern void FluxUnlockTuple(Relation rel, ItemPointer tid, LockTupleMode mode); +extern void FluxLockPage(Relation rel, BlockNumber blkno, LOCKMODE mode); +extern void FluxUnlockPage(Relation rel, BlockNumber blkno, LOCKMODE mode); +extern bool FluxLockMultipleTuples(Relation rel, ItemPointerData *tids, int ntids, + LockTupleMode mode, bool wait); +extern void FluxLockRelationForDDL(Relation rel, LOCKMODE lockmode); +extern bool FluxHoldsTupleLock(Relation rel, ItemPointer tid, LockTupleMode mode); + +/* Table operations */ +extern void flux_tuple_insert(Relation relation, TupleTableSlot *slot, CommandId cid, + uint32 options, BulkInsertState bistate); +extern TM_Result flux_tuple_delete(Relation relation, ItemPointer tid, CommandId cid, + uint32 options, Snapshot snapshot, Snapshot crosscheck, + bool wait, TM_FailureData *tmfd); +extern TM_Result flux_tuple_update(Relation relation, ItemPointer otid, TupleTableSlot *slot, + CommandId cid, uint32 options, + Snapshot snapshot, Snapshot crosscheck, + bool wait, TM_FailureData *tmfd, LockTupleMode *lockmode, + TU_UpdateIndexes *update_indexes); +extern void flux_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples, + CommandId cid, uint32 options, BulkInsertState bistate); +extern void flux_relation_vacuum(Relation onerel, const VacuumParams *params, + BufferAccessStrategy bstrategy); +extern const TableAmRoutine *GetFluxTableAmRoutine(void); + +/* + * flux_tableam_handler is declared via PG_FUNCTION_INFO_V1 in + * flux_handler.c. Do not redeclare it here: on Windows that emits a + * __declspec(dllimport) prototype that conflicts with the implicit + * dllexport from the V1 info macro. Catalog references go through + * pg_proc by name. + */ + +/* In-place update statistics */ +extern void FluxGetUpdateStats(int64 *in_place, int64 *out_of_place, + int64 *defrag_triggered); + +/* + * FLUX-specific ANALYZE statistics + * + * These statistics capture properties unique to the FLUX storage format + * and are collected during ANALYZE. They are stored in the relation's + * pg_class.reloptions and consumed by the planner to improve cost estimates. + */ +typedef struct FluxRelationStats +{ + /* Compression effectiveness */ + double compression_ratio; /* avg uncompressed/compressed size */ + double pct_compressed; /* fraction of tuples that are compressed */ + + /* Overflow usage */ + double pct_overflow; /* fraction of tuples with overflow attrs */ + double avg_overflow_chain_len; /* avg overflow records per overflow + * tuple */ + int64 total_overflow_bytes; /* total bytes in overflow records */ + + /* Space efficiency */ + double avg_tuple_size; /* average on-disk tuple size (bytes) */ + double avg_live_per_page; /* average live tuples per page */ + double free_space_frac; /* average fraction of free space per page */ + double bloat_factor; /* allocated space / live data ratio */ + + /* Page-level summary */ + int64 total_pages; /* total pages in relation */ + int64 total_live_tuples; /* total live tuples counted */ + int64 total_dead_tuples; /* total dead tuples counted */ + + /* Per-tuple commit-ts word distribution */ + bool commit_ts_stats_valid; /* true if commit-ts word fields are + * populated */ + uint64 commit_ts_min; /* min commit-ts word seen */ + uint64 commit_ts_max; /* max commit-ts word seen */ +} FluxRelationStats; + +/* ANALYZE statistics collection (flux_stats.c) */ +extern void FluxCollectRelationStats(Relation rel, FluxRelationStats *stats); +extern void FluxLogRelationStats(Relation rel, const FluxRelationStats *stats, + int elevel); + +/* sLog transaction callbacks (flux_operations.c) */ +extern void FluxEnsureSLogCallbacks(void); + +/* Two-phase commit support (flux_operations.c) */ +extern void AtPrepare_Flux(void); +extern void flux_twophase_postcommit(FullTransactionId fxid, uint16 info, + void *recdata, uint32 len); +extern void flux_twophase_postabort(FullTransactionId fxid, uint16 info, + void *recdata, uint32 len); +extern void flux_twophase_recover(FullTransactionId fxid, uint16 info, + void *recdata, uint32 len); + + +#endif /* FLUX_H */ diff --git a/src/include/access/flux_dirtymap.h b/src/include/access/flux_dirtymap.h new file mode 100644 index 0000000000000..1f33b297f79a9 --- /dev/null +++ b/src/include/access/flux_dirtymap.h @@ -0,0 +1,77 @@ +/*------------------------------------------------------------------------- + * + * flux_dirtymap.h + * Shared-memory dirty block map for the FLUX table access method. + * + * The dirty map tracks which heap pages have ever carried an in-place + * modification (in-place UPDATE or in-place DELETE) whose before-image a + * scanner might still need from the sLog. If a page's bit is CLEAR, every + * tuple on it is plain-committed with no retained before-image, and the scan + * path can skip the per-tuple sLog before-image probe for the whole page + * (the fast path). + * + * Implementation: a partitioned, open-addressed hash set of 64-bit page keys + * (((uint64) relid << 32) | blkno) in a fixed shared-memory buffer. The set + * is sharded into independent partitions by hash, each with its own writer + * spinlock; FluxDirtyMapCheck (the per-scanned-tuple hot path) is lock-free + * because the set is grow-only (a published key is never moved or cleared). + * Each (relid, blkno) page owns a distinct 64-bit key. A relation with no + * in-place modifications contributes no keys and costs no probes. + * + * Correctness invariant: + * - SET is MANDATORY. Every in-place modification must set the page's bit + * before the buffer lock that covers the modification is released, so a + * concurrent scanner on another backend always observes it. A scanner + * that pins-and-locks the page after the writer released the lock is + * guaranteed to see the set bit. + * - The map is GROW-ONLY during normal operation. A bit, once set, is not + * cleared by commit or abort. This is the safe direction: a stale set + * bit only costs an unnecessary sLog probe (which then returns the + * on-page value), whereas a wrongly-cleared bit while an old snapshot + * still needs the before-image would be a false negative -> WRONG + * RESULTS. Reclaiming bits safely requires rebuilding the clear set from + * the sLog's retained-entry set under the same xid horizon that gates + * before-image reclamation; that is a future optimization, not required + * for correctness. + * + * Overflow: the hash is fixed-size. If a partition ever fills past its + * load-factor ceiling, a sticky "full" flag is latched for that partition and + * every subsequent check on it returns dirty, degrading safely to the + * pre-fast-path behavior (always probe the sLog). + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/access/flux_dirtymap.h + * + *------------------------------------------------------------------------- + */ +#ifndef FLUX_DIRTYMAP_H +#define FLUX_DIRTYMAP_H + +#include "storage/block.h" +#include "storage/shmem.h" + +/* Shared memory sizing and initialization */ +extern Size FluxDirtyMapShmemSize(void); +extern void FluxDirtyMapShmemInit(void); +extern const ShmemCallbacks FluxDirtyMapShmemCallbacks; + +/* + * Mark a page's bit dirty (called from the INSERT/UPDATE/DELETE in-place + * paths while the page's buffer is exclusively locked). Idempotent; safe to + * call repeatedly for the same page. + */ +extern void FluxDirtyMapMark(Oid relid, BlockNumber blkno); + +/* + * Query: might the page carry a retained in-place modification? + * + * Returns true if the page's bit is set (or the map has overflowed), in which + * case the scan path must run the per-tuple sLog probe. Returns false only + * when the bit is provably clear, in which case the scan path may skip the + * probe for the whole page. + */ +extern bool FluxDirtyMapCheck(Oid relid, BlockNumber blkno); + +#endif /* FLUX_DIRTYMAP_H */ diff --git a/src/include/access/flux_undo.h b/src/include/access/flux_undo.h new file mode 100644 index 0000000000000..275d0c84827c2 --- /dev/null +++ b/src/include/access/flux_undo.h @@ -0,0 +1,83 @@ +/*------------------------------------------------------------------------- + * + * flux_undo.h + * Public interface for the FLUX UNDO resource manager + * + * FLUX participates in UNDO-in-WAL via its own UNDO resource manager + * (UNDO_RMID_FLUX). Records are written through the shared + * UndoBuffer* (access/undobuffer.h) / Xact-level UNDO APIs (access/xactundo.h); rollback is + * dispatched via undoapply.c to flux_undo_apply() based on the rmid + * stamped into each UNDO record. + * + * Visibility correctness for aborted transactions is handled by + * FLUX's sLog + FLUX_TUPLE_UNCOMMITTED flag, independently of + * physical UNDO application. The UNDO records written here drive + * the logical-revert worker's physical cleanup of aborted rows so + * VACUUM does not have to. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * + * src/include/access/flux_undo.h + * + *------------------------------------------------------------------------- + */ +#ifndef FLUX_UNDO_H +#define FLUX_UNDO_H + +#include "access/undodefs.h" +#include "access/undormgr.h" +#include "storage/itemptr.h" + +/* + * FLUX's UNDO resource-manager ID. Defined in FLUX's own header, not in the + * generic access/undormgr.h, so the UNDO core names no specific consumer. + * See access/undormgr.h for the shared, WAL-durable ID number space. + */ +#define UNDO_RMID_FLUX 5 + +/* + * FLUX UNDO subtypes. Values occupy the 16-bit urec_info field of + * the UNDO record header and are orthogonal to the FLUX WAL opcodes + * in flux_xlog.h. + */ +#define FLUX_UNDO_INSERT 0x0001 +#define FLUX_UNDO_UPDATE 0x0002 /* full-tuple before-image */ +#define FLUX_UNDO_DELETE 0x0003 /* restore deleted tuple */ + +/* + * Common fixed-length header for every FLUX UNDO payload. The + * variable-length tuple / diff image (if any) follows immediately + * after the header. + * + * The header is deliberately small and self-describing so the same + * struct can be passed as part1 in UndoBufferAddRecordParts() + * avoiding an intermediate palloc. + */ +typedef struct FluxUndoPayloadHeader +{ + ItemPointerData tid; /* target tuple id */ + uint32 tuple_len; /* length of trailing tuple/diff image */ + uint16 flags; /* future use: partial-tuple, index-flags */ + uint16 pad; +} FluxUndoPayloadHeader; + +#define SizeOfFluxUndoPayloadHeader (sizeof(FluxUndoPayloadHeader)) + +/* flags bits */ +#define FLUX_UNDO_FLAG_HAS_TUPLE 0x0001 +#define FLUX_UNDO_FLAG_PARTIAL_TUPLE 0x0002 + +/* + * Registration entry point, called once at postmaster startup from + * InitializeUndoSubsystem() alongside HeapUndoRmgrInit and friends. + */ +extern void FluxUndoRmgrInit(void); + +/* + * Install the FLUX implementations of the AM-neutral per-relation UNDO + * hooks (see access/relundo.h). Called from FluxUndoRmgrInit() so the + * pointers are live before crash recovery replays any RELUNDO CLR. + */ +extern void FluxRelUndoInstallHooks(void); + +#endif /* FLUX_UNDO_H */ diff --git a/src/include/access/flux_xlog.h b/src/include/access/flux_xlog.h new file mode 100644 index 0000000000000..d3a88d681bc0d --- /dev/null +++ b/src/include/access/flux_xlog.h @@ -0,0 +1,523 @@ +/*------------------------------------------------------------------------- + * + * flux_xlog.h + * FLUX table access method WAL definitions + * + * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/access/flux_xlog.h + * + *------------------------------------------------------------------------- + */ +#ifndef FLUX_XLOG_H +#define FLUX_XLOG_H + +#include "postgres.h" + +#include "access/xlogreader.h" +#include "lib/stringinfo.h" +#include "storage/buf.h" +#include "storage/off.h" + +/* Forward declarations */ +typedef struct FluxTupleData *FluxTuple; +typedef enum FluxCompressionType FluxCompressionType; +typedef struct RelationData *Relation; +typedef struct FluxOverflowBuffers FluxOverflowBuffers; +struct RelUndoStageResult; + +/* + * Heap-format tuple image for logical decoding. + * + * Logical decoding consumes a heap-format image of the FLUX tuple that the + * write path appends to the end of the main WAL data channel as + * "[heap bytes][uint32 heap_len]" (see decode.c). Forming that image calls + * heap_form_tuple()/palloc(), which are forbidden inside a WAL critical + * section. Callers therefore prepare the image with + * FluxXLogPrepareLogicalImage() BEFORE entering the critical section; the + * WAL functions only register the prebuilt bytes (an allocation-free + * operation) inside the section. The image must stay alive until after + * XLogInsert() returns; release it with FluxXLogReleaseLogicalImage() after + * END_CRIT_SECTION(). + * + * When the relation is not logically logged, prepare leaves data == NULL and + * the WAL functions skip the append. + */ +typedef struct FluxLogicalImage +{ + char *data; /* palloc'd heap tuple body, or NULL */ + uint32 len; /* heap tuple t_len (also written trailing) */ +} FluxLogicalImage; + +/* + * WAL record types for FLUX + */ +/* + * WAL record types for FLUX. + * + * Each opcode must be unique. The info byte uses bits 0-7 with + * XLR_INFO_MASK occupying the upper bits, so we have the lower + * nibble(s) available for opcodes. + */ +#define XLOG_FLUX_INSERT 0x00 +#define XLOG_FLUX_UPDATE_INPLACE 0x10 +#define XLOG_FLUX_DELETE 0x20 +#define XLOG_FLUX_DEFRAG 0x30 /* single-page defrag */ +#define XLOG_FLUX_OVERFLOW_WRITE 0x40 +#define XLOG_FLUX_COMPRESS 0x50 +#define XLOG_FLUX_INIT_PAGE 0x60 +#define XLOG_FLUX_CROSS_PAGE_DEFRAG 0x70 /* cross-page tuple move */ +#define XLOG_FLUX_VM_SET 0x80 /* Set visibility map bits */ +#define XLOG_FLUX_VM_CLEAR 0x90 /* Clear visibility map bits */ +#define XLOG_FLUX_LOCK 0xA0 /* Tuple lock */ +#define XLOG_FLUX_CAS_UPDATE 0xB0 /* Same-size CAS in-place update */ +#define XLOG_FLUX_WRITE_DICT 0xC0 /* Compression-dictionary fork + * write */ +#define XLOG_FLUX_MULTI_INSERT 0xD0 /* Batched multi-tuple insert */ +#define XLOG_FLUX_CAS_UPDATE_UNDO 0xE0 /* CAS update folded with UNDO + * before-image */ +#define XLOG_FLUX_OPMASK 0xF0 + +/* Aliases for backward compatibility / clarity */ +#define XLOG_FLUX_VACUUM XLOG_FLUX_DEFRAG +#define XLOG_FLUX_UPDATE XLOG_FLUX_UPDATE_INPLACE + +/* Flags for xl_flux_overflow_write */ +#define FLUX_OVERFLOW_WAL_NEW_RECORD 0x0000 /* New overflow record */ +#define FLUX_OVERFLOW_WAL_LINK_UPDATE 0x0001 /* Link update only */ + +/* + * Common WAL record flags. + * + * These appear in the 'flags' field of the DML WAL record structures + * (xl_flux_insert, xl_flux_update, xl_flux_delete). + * + * Note: bit 0x0001 is reserved (formerly FLUX_WAL_HAS_HLC) and unused. + */ +#define FLUX_WAL_CROSS_PAGE 0x0002 /* Cross-page out-of-place update */ +#define FLUX_WAL_HAS_OVERFLOW_BLK0 0x0004 /* Block 0 buf data has overflow + * records */ +#define FLUX_WAL_PREFIX_SUFFIX 0x0008 /* Update uses prefix/suffix + * compression */ +/* + * Heap-format tuple image is appended to the WAL record for the benefit of + * logical decoding. Set when RelationIsLogicallyLogged(rel) at WAL-emit + * time. The layout of the appended region is: + * + * uint32 logical_len -- bytes of the heap-tuple payload + * bytes[logical_len] HeapTuple t_data bytes + * + * For INSERT / DELETE the record contains exactly one heap-tuple payload. + * For UPDATE it contains two back-to-back payloads (old, then new). + */ +#define FLUX_WAL_LOGICAL_TUPLE 0x0010 + +#ifndef FRONTEND +/* + * WAL record data structures + */ +typedef struct xl_flux_insert +{ + OffsetNumber offnum; /* Offset number */ + uint16 flags; /* Flags (FLUX_WAL_* bits) */ + uint32 tuple_len; /* Length of tuple data that follows */ + uint64 commit_ts; /* Commit timestamp */ + /* Tuple data follows */ +} xl_flux_insert; + +typedef struct xl_flux_update +{ + OffsetNumber offnum; /* Offset number on source page (block 0) */ + uint16 flags; /* Flags (FLUX_WAL_* bits) */ + uint64 old_commit_ts; /* Old commit timestamp */ + uint64 new_commit_ts; /* New commit timestamp */ + uint16 old_tuple_len; /* Length of old tuple */ + uint16 new_tuple_len; /* Length of new tuple data that follows */ + uint8 dst_block_id; /* Block ID of destination page for cross-page + * updates (only valid when + * FLUX_WAL_CROSS_PAGE is set in flags) */ + uint8 pad[3]; /* Padding for alignment */ + /* New tuple data follows (old tuple data is in UNDO fork only) */ +} xl_flux_update; + +/* + * Prefix/suffix compression header for in-place updates. + * + * When FLUX_WAL_PREFIX_SUFFIX is set in xl_flux_update.flags, this header + * immediately follows the xl_flux_update struct and precedes the diff data. + * Only the changed bytes (between prefixlen and len-suffixlen) are logged. + * + * The redo handler reconstructs the full new tuple by: + * 1. Reading the existing tuple from the page (old data) + * 2. Keeping old[0..prefixlen-1] as-is + * 3. Copying the diff data into old[prefixlen..len-suffixlen-1] + * 4. Keeping old[len-suffixlen..len-1] as-is + */ +typedef struct xl_flux_prefix_suffix +{ + uint16 prefixlen; /* Bytes of common prefix */ + uint16 suffixlen; /* Bytes of common suffix */ +} xl_flux_prefix_suffix; + +typedef struct xl_flux_delete +{ + OffsetNumber offnum; /* Offset number */ + uint16 flags; /* Flags (FLUX_WAL_* bits) */ + uint32 tuple_len; /* Length of old tuple (for logical decoding) */ + uint64 commit_ts; /* Commit timestamp */ + /* Old tuple data is in UNDO fork only */ +} xl_flux_delete; + +/* + * WAL record for a batched multi-tuple insert (XLOG_FLUX_MULTI_INSERT). + * + * flux_multi_insert packs many tuples onto a single page in one critical + * section. Rather than emit one xl_flux_insert per tuple (and force a + * full-page image to stay crash-safe), we log every tuple body once in a + * single record modelled on heap's xl_heap_multi_insert. + * + * Layout of the record's main data: + * + * xl_flux_multi_insert -- header (ntuples, flags, ts) + * repeated ntuples times: + * xl_flux_multi_insert_tuple -- per-tuple header + * char body[datalen] -- tuple t_data bytes + * + * If flags & FLUX_WAL_LOGICAL_TUPLE, ntuples heap-format logical images + * follow (one per tuple) after the per-tuple region, each as [body][uint32 len]. + */ +typedef struct xl_flux_multi_insert +{ + uint16 ntuples; /* Number of tuples in this batch */ + uint16 flags; /* Flags (FLUX_WAL_* bits) */ + uint64 commit_ts; /* Shared commit timestamp */ + /* xl_flux_multi_insert_tuple entries follow */ +} xl_flux_multi_insert; + +#define SizeOfFluxMultiInsert sizeof(xl_flux_multi_insert) + +typedef struct xl_flux_multi_insert_tuple +{ + OffsetNumber offnum; /* Target offset on the page */ + uint16 datalen; /* Length of tuple t_data bytes that follow */ + /* char body[datalen] follows */ +} xl_flux_multi_insert_tuple; + +#define SizeOfFluxMultiInsertTuple sizeof(xl_flux_multi_insert_tuple) + +typedef struct xl_flux_lock +{ + OffsetNumber offnum; /* Offset number */ + uint16 flags; /* Flags */ + uint8 infomask; /* Infomask bits (uint8) */ + uint8 lock_mode; /* LockTupleMode */ +} xl_flux_lock; + +/* + * WAL record for same-size CAS in-place update (XLOG_FLUX_CAS_UPDATE). + * + * This is a lightweight record logged by the tuple-level CAS update fast + * path. Only the changed portion of the tuple data is logged (the region + * between data_offset and data_offset+data_len within the on-page tuple). + * The redo handler patches these bytes directly into the tuple on the page. + */ +typedef struct xl_flux_cas_update +{ + OffsetNumber offnum; /* Tuple offset on page */ + uint16 flags; /* FLUX_WAL_* bits */ + uint16 data_offset; /* Byte offset within tuple for patch start */ + uint16 data_len; /* Length of replacement data */ + uint64 new_commit_ts; /* New commit timestamp for the tuple */ + /* char data[data_len] follows */ +} xl_flux_cas_update; + +/* + * WAL record for a CAS in-place update folded with its UNDO before-image + * (XLOG_FLUX_CAS_UPDATE_UNDO). + * + * This is the FOLD variant of the CAS fast path: instead of emitting a + * separate RM_FLUX_ID CAS-update record and a separate RM_RELUNDO_ID + * before-image record, one record carries both. Block 0 is the main-fork + * page (same redo byte-diff as xl_flux_cas_update). Block 1 is the + * relundo-fork data page (the UNDO record bytes, replayed exactly like + * relundo_redo_insert). Block 2 is the relundo metapage, present only when + * the UNDO record started a fresh relundo page (is_new_page). + * + * The redo handler replays block 0 like flux_xlog_cas_update_redo, then + * block 1 like relundo_redo_insert (honoring the new-page INIT and the + * metapage FPI). + */ +typedef struct xl_flux_cas_update_undo +{ + /* --- redo half (block 0): mirrors xl_flux_cas_update --- */ + OffsetNumber offnum; /* Tuple offset on main-fork page */ + uint16 flags; /* FLUX_WAL_* bits */ + uint16 data_offset; /* Byte offset within tuple for patch start */ + uint16 data_len; /* Length of replacement data */ + uint64 new_commit_ts; /* New commit timestamp for the tuple */ + + /* --- undo half (block 1): mirrors xl_relundo_insert --- */ + uint8 urec_type; /* UNDO record type */ + uint8 is_new_page; /* first record on a freshly allocated page */ + uint16 urec_len; /* UNDO record length */ + uint16 page_offset; /* page-absolute offset of the UNDO record */ + uint16 new_pd_lower; /* shadow pd_lower after the UNDO write */ + TransactionId max_xid; /* undo-page max_xid watermark after the bump */ + /* char redo_data[data_len] follows (block 0 byte-diff) */ +} xl_flux_cas_update_undo; + +#define SizeOfFluxCasUpdateUndo sizeof(xl_flux_cas_update_undo) + +typedef struct xl_flux_defrag +{ + uint16 ntuples; /* Number of tuples moved */ + uint64 commit_ts; /* Commit timestamp */ + /* Array of offset mappings follows */ +} xl_flux_defrag; + +typedef struct xl_flux_overflow_write +{ + OffsetNumber offnum; /* Offset of overflow record on page */ + uint16 flags; /* Flags (0 = new record, 1 = link update) */ + uint32 data_len; /* Length of overflow data chunk */ + uint64 commit_ts; /* Commit timestamp */ + /* FluxOverflowRecordHeader + data follows for new records */ + /* FluxOverflowRecordHeader follows for link updates */ +} xl_flux_overflow_write; + +typedef struct xl_flux_compress +{ + OffsetNumber offnum; /* Offset number */ + uint16 attr_num; /* Attribute number */ + uint8 comp_type; /* Compression type */ + uint8 comp_level; /* Compression level */ + uint32 orig_size; /* Original size */ + uint32 comp_size; /* Compressed size */ + uint64 commit_ts; /* Commit timestamp */ + /* Compressed data follows */ +} xl_flux_compress; + +typedef struct xl_flux_vacuum +{ + uint32 ntuples; /* Number of removed tuples */ +} xl_flux_vacuum; + +/* + * Cross-page defragmentation: records moving a tuple from a source page + * (block ref 1) to a target page (block ref 0). The source line pointer + * is set LP_UNUSED and the tuple data is added to the target page. + * + * If full-page images are present, recovery simply restores both pages. + * Otherwise, recovery replays the move: adds the tuple to the target + * and marks the source slot unused. + */ +typedef struct xl_flux_cross_page_defrag +{ + OffsetNumber src_offnum; /* Source line pointer offset (on block 1) */ + OffsetNumber dst_offnum; /* Target line pointer offset (on block 0) */ + uint32 tuple_len; /* Length of moved tuple data */ + /* Tuple data follows */ +} xl_flux_cross_page_defrag; + +typedef struct xl_flux_init_page +{ + uint32 flags; /* Page flags */ + uint64 commit_ts; /* Initial commit timestamp */ +} xl_flux_init_page; + +/* + * Compression-dictionary fork write (XLOG_FLUX_WRITE_DICT). + * + * The dictionary fork (FLUX_DICT_FORKNUM) stores trained ZSTD/LZ4 dictionary + * blobs append-only. Its pages use a non-standard layout (the directory + * metapage keeps FluxDictMeta above pd_lower; data pages keep payload above + * pd_lower), so the redo path cannot reconstruct them from logical deltas. + * Instead each dirtied dict-fork page is logged as a full-page image with + * REGBUF_FORCE_IMAGE and the redo handler simply restores the registered + * block. No record-specific payload is required beyond the block image. + */ +typedef struct xl_flux_write_dict +{ + BlockNumber blkno; /* Dictionary-fork block being written */ +} xl_flux_write_dict; + +/* + * Visibility Map WAL records + */ +typedef struct xl_flux_vm_set +{ + BlockNumber heapBlk; /* Heap block number */ + uint8 flags; /* VM flags being set */ +} xl_flux_vm_set; + +typedef struct xl_flux_vm_clear +{ + BlockNumber heapBlk; /* Heap block number */ + uint8 flags; /* VM flags being cleared */ +} xl_flux_vm_clear; + +/* + * Offset mapping for defragmentation + */ +typedef struct FluxOffsetMapping +{ + OffsetNumber old_offnum; + OffsetNumber new_offnum; +} FluxOffsetMapping; +#else /* FRONTEND */ + +/* Frontend-safe versions of WAL record structures */ +typedef struct xl_flux_insert +{ + uint16 offnum; /* Offset number */ + uint16 flags; /* Flags */ + uint32 tuple_len; /* Length of tuple data */ + uint64 commit_ts; /* Commit timestamp */ +} xl_flux_insert; + +typedef struct xl_flux_delete +{ + uint16 offnum; /* Offset number */ + uint16 flags; /* Flags */ + uint32 tuple_len; /* Length of old tuple */ + uint64 commit_ts; /* Commit timestamp */ +} xl_flux_delete; + +typedef struct xl_flux_multi_insert +{ + uint16 ntuples; /* Number of tuples in this batch */ + uint16 flags; /* Flags */ + uint64 commit_ts; /* Shared commit timestamp */ +} xl_flux_multi_insert; + +typedef struct xl_flux_multi_insert_tuple +{ + uint16 offnum; /* Target offset on the page */ + uint16 datalen; /* Length of tuple data that follows */ +} xl_flux_multi_insert_tuple; + +typedef struct xl_flux_update +{ + uint16 offnum; /* Offset number */ + uint16 flags; /* Flags */ + uint64 old_commit_ts; /* Old commit timestamp */ + uint64 new_commit_ts; /* New commit timestamp */ + uint16 old_tuple_len; /* Length of old tuple */ + uint16 new_tuple_len; /* Length of new tuple */ +} xl_flux_update; + +typedef struct xl_flux_vacuum +{ + uint32 ntuples; /* Number of removed tuples */ +} xl_flux_vacuum; + +typedef struct xl_flux_compress +{ + uint16 offnum; /* Offset number */ + uint16 attr_num; /* Attribute number */ + uint8 comp_type; /* Compression type */ + uint8 comp_level; /* Compression level */ + uint32 orig_size; /* Original size */ + uint32 comp_size; /* Compressed size */ + uint64 commit_ts; /* Commit timestamp */ +} xl_flux_compress; + +#endif /* !FRONTEND */ + +/* + * Function prototypes + */ + +/* Frontend-safe function prototypes (pg_waldump, etc.) */ +extern void flux_desc(StringInfo buf, XLogReaderState *record); +extern const char *flux_identify(uint8 info); + +#ifndef FRONTEND +/* WAL replay and logging functions - backend only */ +extern void flux_redo(XLogReaderState *record); +extern void flux_mask(char *page, BlockNumber blkno); + +/* + * Prepare/release the heap-format logical-decoding image. Call prepare + * BEFORE START_CRIT_SECTION() and release AFTER END_CRIT_SECTION(). When rel + * is not logically logged, prepare sets img->data = NULL and the WAL + * functions emit no logical image. + */ +extern void FluxXLogPrepareLogicalImage(Relation rel, FluxTuple rtup, + FluxLogicalImage *img); +extern void FluxXLogReleaseLogicalImage(FluxLogicalImage *img); + +extern XLogRecPtr FluxXLogInsert(Relation rel, Buffer buffer, OffsetNumber offnum, + FluxTuple tuple, uint64 commit_ts, + FluxOverflowBuffers *overflow_buffers, + FluxLogicalImage *logical_img, + bool force_page_image); +extern XLogRecPtr FluxXLogUpdate(Relation rel, Buffer buffer, OffsetNumber offnum, + FluxTuple old_tuple, FluxTuple new_tuple, + uint64 old_commit_ts, uint64 new_commit_ts, + FluxOverflowBuffers *overflow_buffers, + Buffer new_buffer, + FluxLogicalImage *old_img, + FluxLogicalImage *new_img); +extern XLogRecPtr FluxXLogDelete(Relation rel, Buffer buffer, OffsetNumber offnum, + FluxTuple tuple, uint64 commit_ts, + FluxLogicalImage *logical_img); + +/* + * Log a batch of tuples inserted onto a single page by flux_multi_insert. + * Every tuple body is logged once (no forced full-page image), modelled on + * heap's xl_heap_multi_insert. offnums[i] is the on-page offset assigned to + * tuples[i]; logical_imgs may be NULL (no logical decoding) or an array of + * ntuples prepared images. + */ +extern XLogRecPtr FluxXLogMultiInsert(Relation rel, Buffer buffer, + OffsetNumber *offnums, FluxTuple *tuples, + int ntuples, uint64 commit_ts, + FluxLogicalImage *logical_imgs); + +extern XLogRecPtr FluxXLogDefrag(Relation rel, Buffer buffer, + FluxOffsetMapping *mappings, int nmappings, uint64 commit_ts); +extern XLogRecPtr FluxXLogOverflowWrite(Relation rel, Buffer buffer, + OffsetNumber offnum, char *record_data, + uint32 record_len, uint16 flags, + uint64 commit_ts); +extern XLogRecPtr FluxXLogCompress(Relation rel, Buffer buffer, OffsetNumber offnum, + uint16 attr_num, FluxCompressionType comp_type, + uint8 comp_level, char *comp_data, uint32 orig_size, uint32 comp_size, + uint64 commit_ts); +extern XLogRecPtr FluxXLogInitPage(Relation rel, Buffer buffer, uint32 flags, uint64 commit_ts); +extern XLogRecPtr FluxXLogCrossPageDefrag(Relation rel, + Buffer dst_buf, OffsetNumber dst_offnum, + Buffer src_buf, OffsetNumber src_offnum, + const void *tuple_data, uint32 tuple_len); +extern XLogRecPtr FluxXLogCasUpdate(Relation rel, Buffer buffer, + OffsetNumber offnum, + uint16 data_offset, uint16 data_len, + const char *new_data, + uint64 new_commit_ts); +extern XLogRecPtr FluxXLogCasUpdateUndo(Relation rel, Buffer buffer, + OffsetNumber offnum, + uint16 data_offset, uint16 data_len, + const char *new_data, + uint64 new_commit_ts, + const struct RelUndoStageResult *undo); + +/* + * Log a compression-dictionary fork page as a full-page image so crash + * recovery and physical replicas reproduce the append-only dict fork. The + * caller must hold the buffer locked and have already modified it inside a + * critical section; this registers the page with REGBUF_FORCE_IMAGE and sets + * the page LSN. + */ +extern XLogRecPtr FluxXLogWriteDict(Relation rel, Buffer buffer); + +/* + * Logical replication decode entry point for FLUX WAL records. + */ +struct LogicalDecodingContext; +struct XLogRecordBuffer; +extern void flux_decode(struct LogicalDecodingContext *ctx, + struct XLogRecordBuffer *buf); +#endif /* !FRONTEND */ +#endif /* FLUX_XLOG_H */ diff --git a/src/include/access/rmgrlist.h b/src/include/access/rmgrlist.h index b21f4fa30aa5f..3250764a4d247 100644 --- a/src/include/access/rmgrlist.h +++ b/src/include/access/rmgrlist.h @@ -51,3 +51,6 @@ PG_RMGR(RM_XLOG2_ID, "XLOG2", xlog2_redo, xlog2_desc, xlog2_identify, NULL, NULL PG_RMGR(RM_UNDO_ID, "Undo", undo_redo, undo_desc, undo_identify, NULL, NULL, NULL, NULL) PG_RMGR(RM_ATM_ID, "ATM", atm_redo, atm_desc, atm_identify, NULL, NULL, NULL, NULL) PG_RMGR(RM_RELUNDO_ID, "RelUndo", relundo_redo, relundo_desc, relundo_identify, relundo_startup, relundo_cleanup, relundo_mask, NULL) +#ifdef USE_FLUX +PG_RMGR(RM_FLUX_ID, "FLUX", flux_redo, flux_desc, flux_identify, NULL, NULL, flux_mask, flux_decode) +#endif diff --git a/src/include/access/twophase_rmgr.h b/src/include/access/twophase_rmgr.h index 8927f369c39b5..9560eca97bee5 100644 --- a/src/include/access/twophase_rmgr.h +++ b/src/include/access/twophase_rmgr.h @@ -28,7 +28,8 @@ typedef uint8 TwoPhaseRmgrId; #define TWOPHASE_RM_PGSTAT_ID 2 #define TWOPHASE_RM_MULTIXACT_ID 3 #define TWOPHASE_RM_PREDICATELOCK_ID 4 -#define TWOPHASE_RM_MAX_ID TWOPHASE_RM_PREDICATELOCK_ID +#define TWOPHASE_RM_FLUX_ID 5 +#define TWOPHASE_RM_MAX_ID TWOPHASE_RM_FLUX_ID extern PGDLLIMPORT const TwoPhaseCallback twophase_recover_callbacks[]; extern PGDLLIMPORT const TwoPhaseCallback twophase_postcommit_callbacks[]; diff --git a/src/include/access/undormgr.h b/src/include/access/undormgr.h index 61e99c755d064..8933798553668 100644 --- a/src/include/access/undormgr.h +++ b/src/include/access/undormgr.h @@ -45,7 +45,7 @@ #define UNDO_RMID_INVALID 0 #define UNDO_RMID_NBTREE 1 #define UNDO_RMID_HASH 3 -/* 2 and 4 are reserved by out-of-core consumers (see their own headers) */ +/* 2, 4, and 5 are reserved by out-of-core consumers (see their own headers) */ #define MAX_UNDO_RMGRS 256 diff --git a/src/include/access/undormgrlist.h b/src/include/access/undormgrlist.h index f7d5ffb713071..ec4eab8e1536f 100644 --- a/src/include/access/undormgrlist.h +++ b/src/include/access/undormgrlist.h @@ -35,3 +35,6 @@ /* built-in index AM UNDO resource managers */ UNDO_RMGR_INIT(NbtreeUndoRmgrInit) UNDO_RMGR_INIT(HashUndoRmgrInit) + +/* FLUX: UNDO-based heap-replacement table AM UNDO resource manager */ +UNDO_RMGR_INIT(FluxUndoRmgrInit) diff --git a/src/include/catalog/pg_am.dat b/src/include/catalog/pg_am.dat index 46d361047fe67..708786dce6119 100644 --- a/src/include/catalog/pg_am.dat +++ b/src/include/catalog/pg_am.dat @@ -33,5 +33,8 @@ { oid => '3580', oid_symbol => 'BRIN_AM_OID', descr => 'block range index (BRIN) access method', amname => 'brin', amhandler => 'brinhandler', amtype => 'i' }, +{ oid => '9316', oid_symbol => 'FLUX_TABLE_AM_OID', + descr => 'flux table access method', + amname => 'flux', amhandler => 'flux_tableam_handler', amtype => 't' }, ] diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index f8a021987b5e5..d82ab28e4125b 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -912,6 +912,10 @@ proname => 'heap_tableam_handler', provolatile => 'v', prorettype => 'table_am_handler', proargtypes => 'internal', prosrc => 'heap_tableam_handler' }, +{ oid => '9402', descr => 'flux table access method handler', + proname => 'flux_tableam_handler', provolatile => 'v', + prorettype => 'table_am_handler', proargtypes => 'internal', + prosrc => 'flux_tableam_handler' }, # Index access method handlers { oid => '330', descr => 'btree index access method handler', diff --git a/src/include/storage/subsystemlist.h b/src/include/storage/subsystemlist.h index f9c2ce9f05c60..9a9937823f99f 100644 --- a/src/include/storage/subsystemlist.h +++ b/src/include/storage/subsystemlist.h @@ -100,3 +100,7 @@ PG_SHMEM_SUBSYSTEM(UndoShmemCallbacks) * any consumer. See slog.c's SLogShmemCallbacks for details. */ PG_SHMEM_SUBSYSTEM(SLogShmemCallbacks) + +/* FLUX table access method subsystems */ +PG_SHMEM_SUBSYSTEM(FluxMvccShmemCallbacks) +PG_SHMEM_SUBSYSTEM(FluxDirtyMapShmemCallbacks) diff --git a/src/test/regress/expected/create_am.out b/src/test/regress/expected/create_am.out index c1a951572512c..4018dc0bab665 100644 --- a/src/test/regress/expected/create_am.out +++ b/src/test/regress/expected/create_am.out @@ -131,9 +131,10 @@ ERROR: function bthandler must return type table_am_handler SELECT amname, amhandler, amtype FROM pg_am where amtype = 't' ORDER BY 1, 2; amname | amhandler | amtype --------+----------------------+-------- + flux | flux_tableam_handler | t heap | heap_tableam_handler | t heap2 | heap_tableam_handler | t -(2 rows) +(3 rows) -- First create tables employing the new AM using USING -- plain CREATE TABLE diff --git a/src/test/regress/expected/flux.out b/src/test/regress/expected/flux.out new file mode 100644 index 0000000000000..a0587da2db695 --- /dev/null +++ b/src/test/regress/expected/flux.out @@ -0,0 +1,128 @@ +-- +-- FLUX table access method: core functional + index-integrity tests. +-- +-- The critical property is that +-- index scans, sequential scans, and bitmap scans agree after key-changing +-- UPDATEs, including A -> B -> A recurrences. FLUX achieves this by doing a +-- non-in-place (new-TID) UPDATE whenever an indexed column changes, so +-- secondary indexes are maintained by the standard heap-TID path. +-- +CREATE TABLE flux_basic (id int, k int, v text) USING flux; +INSERT INTO flux_basic SELECT g, g, 'v' || g FROM generate_series(1, 20) g; +CREATE INDEX flux_basic_k_idx ON flux_basic (k); +-- non-key UPDATE (in place): TID and index unchanged +UPDATE flux_basic SET v = 'updated' WHERE id = 5; +SELECT id, k, v FROM flux_basic WHERE id = 5; + id | k | v +----+---+--------- + 5 | 5 | updated +(1 row) + +-- key-changing UPDATEs (out of place): old index entry dies, new one inserted +UPDATE flux_basic SET k = 105 WHERE id = 5; +UPDATE flux_basic SET k = 106 WHERE id = 6; +-- A -> B -> A recurrence on an indexed key +UPDATE flux_basic SET k = 999 WHERE id = 7; +UPDATE flux_basic SET k = 7 WHERE id = 7; +DELETE FROM flux_basic WHERE id = 20; +-- The gate: idxscan == seqscan == bitmapscan. +SET enable_seqscan = on; SET enable_indexscan = off; SET enable_bitmapscan = off; +SELECT count(*) AS seq_count, sum(k) AS seq_sumk FROM flux_basic; + seq_count | seq_sumk +-----------+---------- + 19 | 390 +(1 row) + +SET enable_seqscan = off; SET enable_indexscan = on; SET enable_bitmapscan = off; +SELECT count(*) AS idx_count, sum(k) AS idx_sumk FROM flux_basic WHERE k > -2147483648; + idx_count | idx_sumk +-----------+---------- + 19 | 390 +(1 row) + +SELECT count(*) AS idx_k_eq7 FROM flux_basic WHERE k = 7; -- 1, not 2 + idx_k_eq7 +----------- + 1 +(1 row) + +SELECT count(*) AS idx_k_eq5 FROM flux_basic WHERE k = 5; -- 0 (moved to 105) + idx_k_eq5 +----------- + 0 +(1 row) + +SELECT count(*) AS idx_k_eq105 FROM flux_basic WHERE k = 105; -- 1 + idx_k_eq105 +------------- + 1 +(1 row) + +SET enable_seqscan = off; SET enable_indexscan = off; SET enable_bitmapscan = on; +SELECT count(*) AS bmp_count, sum(k) AS bmp_sumk FROM flux_basic WHERE k > -2147483648; + bmp_count | bmp_sumk +-----------+---------- + 19 | 390 +(1 row) + +RESET enable_seqscan; RESET enable_indexscan; RESET enable_bitmapscan; +-- VACUUM then amcheck (heapallindexed): no error means the index is consistent +-- with the heap after key churn. +VACUUM flux_basic; +CREATE EXTENSION IF NOT EXISTS amcheck; +SELECT bt_index_check('flux_basic_k_idx'::regclass, true); + bt_index_check +---------------- + +(1 row) + +-- ROLLBACK restores the old value (non-key) and old key + index entry (key). +BEGIN; +UPDATE flux_basic SET v = 'should_not_persist' WHERE id = 10; +ROLLBACK; +SELECT v FROM flux_basic WHERE id = 10; + v +----- + v10 +(1 row) + +BEGIN; +UPDATE flux_basic SET k = 5000 WHERE id = 11; +ROLLBACK; +SELECT k FROM flux_basic WHERE id = 11; + k +---- + 11 +(1 row) + +SET enable_seqscan = off; SET enable_indexscan = on; SET enable_bitmapscan = off; +SELECT count(*) AS idx_k_eq11 FROM flux_basic WHERE k = 11; -- 1 + idx_k_eq11 +------------ + 1 +(1 row) + +SELECT count(*) AS idx_k_eq5000 FROM flux_basic WHERE k = 5000; -- 0 + idx_k_eq5000 +-------------- + 0 +(1 row) + +RESET enable_seqscan; RESET enable_indexscan; RESET enable_bitmapscan; +-- TOAST: a >8KB value round-trips exactly and the relation gets a TOAST table. +CREATE TABLE flux_toast (id int, big text) USING flux; +INSERT INTO flux_toast VALUES (1, repeat('X', 100000)); +SELECT id, length(big) AS len, (big = repeat('X', 100000)) AS exact FROM flux_toast; + id | len | exact +----+--------+------- + 1 | 100000 | t +(1 row) + +SELECT reltoastrelid <> 0 AS has_toast_table FROM pg_class WHERE relname = 'flux_toast'; + has_toast_table +----------------- + t +(1 row) + +DROP TABLE flux_basic; +DROP TABLE flux_toast; diff --git a/src/test/regress/expected/psql.out b/src/test/regress/expected/psql.out index 42635a56a06c3..249fa21fe64dd 100644 --- a/src/test/regress/expected/psql.out +++ b/src/test/regress/expected/psql.out @@ -5191,13 +5191,14 @@ List of access methods --------+------- brin | Index btree | Index + flux | Table gin | Index gist | Index hash | Index heap | Table heap2 | Table spgist | Index -(8 rows) +(9 rows) \dA * List of access methods @@ -5205,13 +5206,14 @@ List of access methods --------+------- brin | Index btree | Index + flux | Table gin | Index gist | Index hash | Index heap | Table heap2 | Table spgist | Index -(8 rows) +(9 rows) \dA h* List of access methods @@ -5241,13 +5243,14 @@ List of access methods --------+-------+----------------------+---------------------------------------- brin | Index | brinhandler | block range index (BRIN) access method btree | Index | bthandler | b-tree index access method + flux | Table | flux_tableam_handler | flux table access method gin | Index | ginhandler | GIN index access method gist | Index | gisthandler | GiST index access method hash | Index | hashhandler | hash index access method heap | Table | heap_tableam_handler | heap table access method heap2 | Table | heap_tableam_handler | spgist | Index | spghandler | SP-GiST index access method -(8 rows) +(9 rows) \dA+ * List of access methods @@ -5255,13 +5258,14 @@ List of access methods --------+-------+----------------------+---------------------------------------- brin | Index | brinhandler | block range index (BRIN) access method btree | Index | bthandler | b-tree index access method + flux | Table | flux_tableam_handler | flux table access method gin | Index | ginhandler | GIN index access method gist | Index | gisthandler | GiST index access method hash | Index | hashhandler | hash index access method heap | Table | heap_tableam_handler | heap table access method heap2 | Table | heap_tableam_handler | spgist | Index | spghandler | SP-GiST index access method -(8 rows) +(9 rows) \dA+ h* List of access methods diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule index 8fa0a6c47fb30..21882a6d6aea0 100644 --- a/src/test/regress/parallel_schedule +++ b/src/test/regress/parallel_schedule @@ -72,6 +72,7 @@ test: brin gin gist spgist privileges init_privs security_label collate matview # Additional BRIN tests # ---------- test: brin_bloom brin_multi +test: flux # ---------- # Another group of parallel tests diff --git a/src/test/regress/sql/flux.sql b/src/test/regress/sql/flux.sql new file mode 100644 index 0000000000000..5c642448c1de7 --- /dev/null +++ b/src/test/regress/sql/flux.sql @@ -0,0 +1,68 @@ +-- +-- FLUX table access method: core functional + index-integrity tests. +-- +-- The critical property is that +-- index scans, sequential scans, and bitmap scans agree after key-changing +-- UPDATEs, including A -> B -> A recurrences. FLUX achieves this by doing a +-- non-in-place (new-TID) UPDATE whenever an indexed column changes, so +-- secondary indexes are maintained by the standard heap-TID path. +-- + +CREATE TABLE flux_basic (id int, k int, v text) USING flux; +INSERT INTO flux_basic SELECT g, g, 'v' || g FROM generate_series(1, 20) g; +CREATE INDEX flux_basic_k_idx ON flux_basic (k); + +-- non-key UPDATE (in place): TID and index unchanged +UPDATE flux_basic SET v = 'updated' WHERE id = 5; +SELECT id, k, v FROM flux_basic WHERE id = 5; + +-- key-changing UPDATEs (out of place): old index entry dies, new one inserted +UPDATE flux_basic SET k = 105 WHERE id = 5; +UPDATE flux_basic SET k = 106 WHERE id = 6; +-- A -> B -> A recurrence on an indexed key +UPDATE flux_basic SET k = 999 WHERE id = 7; +UPDATE flux_basic SET k = 7 WHERE id = 7; + +DELETE FROM flux_basic WHERE id = 20; + +-- The gate: idxscan == seqscan == bitmapscan. +SET enable_seqscan = on; SET enable_indexscan = off; SET enable_bitmapscan = off; +SELECT count(*) AS seq_count, sum(k) AS seq_sumk FROM flux_basic; +SET enable_seqscan = off; SET enable_indexscan = on; SET enable_bitmapscan = off; +SELECT count(*) AS idx_count, sum(k) AS idx_sumk FROM flux_basic WHERE k > -2147483648; +SELECT count(*) AS idx_k_eq7 FROM flux_basic WHERE k = 7; -- 1, not 2 +SELECT count(*) AS idx_k_eq5 FROM flux_basic WHERE k = 5; -- 0 (moved to 105) +SELECT count(*) AS idx_k_eq105 FROM flux_basic WHERE k = 105; -- 1 +SET enable_seqscan = off; SET enable_indexscan = off; SET enable_bitmapscan = on; +SELECT count(*) AS bmp_count, sum(k) AS bmp_sumk FROM flux_basic WHERE k > -2147483648; +RESET enable_seqscan; RESET enable_indexscan; RESET enable_bitmapscan; + +-- VACUUM then amcheck (heapallindexed): no error means the index is consistent +-- with the heap after key churn. +VACUUM flux_basic; +CREATE EXTENSION IF NOT EXISTS amcheck; +SELECT bt_index_check('flux_basic_k_idx'::regclass, true); + +-- ROLLBACK restores the old value (non-key) and old key + index entry (key). +BEGIN; +UPDATE flux_basic SET v = 'should_not_persist' WHERE id = 10; +ROLLBACK; +SELECT v FROM flux_basic WHERE id = 10; + +BEGIN; +UPDATE flux_basic SET k = 5000 WHERE id = 11; +ROLLBACK; +SELECT k FROM flux_basic WHERE id = 11; +SET enable_seqscan = off; SET enable_indexscan = on; SET enable_bitmapscan = off; +SELECT count(*) AS idx_k_eq11 FROM flux_basic WHERE k = 11; -- 1 +SELECT count(*) AS idx_k_eq5000 FROM flux_basic WHERE k = 5000; -- 0 +RESET enable_seqscan; RESET enable_indexscan; RESET enable_bitmapscan; + +-- TOAST: a >8KB value round-trips exactly and the relation gets a TOAST table. +CREATE TABLE flux_toast (id int, big text) USING flux; +INSERT INTO flux_toast VALUES (1, repeat('X', 100000)); +SELECT id, length(big) AS len, (big = repeat('X', 100000)) AS exact FROM flux_toast; +SELECT reltoastrelid <> 0 AS has_toast_table FROM pg_class WHERE relname = 'flux_toast'; + +DROP TABLE flux_basic; +DROP TABLE flux_toast;