fix: cache the parse cache kapi 1.2 writes, not retired paths

.github/workflows/test.yml

@@ -13,6 +13,11 @@ on:
     - cron: "17 5 * * *"
   workflow_dispatch:
 
+env:
+  # The state cache jobs need kapi 1.2, the first release that keeps its state
+  # under .kapi/work/.
+  STATE_CACHE_KAPI_VERSION: 1.2.0-rc32
+
 jobs:
   test-latest:
     name: "Latest (${{ matrix.os }})"
@@ -130,6 +135,131 @@ jobs:
         shell: bash
         run: kapi version
 
+  # The state cache round trip on a kapi 1.2 project. This job runs the fixture
+  # and the action saves the cache at job end; the next job restores it. Each
+  # job compares its run in the cached project directory with a cold run of a
+  # pristine copy, and the comparison covers every command's exit code and
+  # output and every file the project holds afterwards.
+  test-state-cache-save:
+    name: "State cache save (${{ matrix.os }})"
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        # kapi 1.2 prereleases publish no Windows CLI build.
+        os: [ubuntu-latest, macos-latest]
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Setup kapi
+        uses: ./
+        with:
+          token: ${{ secrets.NEOKAPI_GITHUB_TOKEN }}
+          version: ${{ env.STATE_CACHE_KAPI_VERSION }}
+          plugins: ""
+          project-dir: test/fixture
+
+      - name: Run the fixture and a cold copy
+        shell: bash
+        run: |
+          test/state-cache/cold-copy.sh "${RUNNER_TEMP}/cold"
+          test/state-cache/run.sh test/fixture "${RUNNER_TEMP}/results/cached"
+          test/state-cache/run.sh "${RUNNER_TEMP}/cold/test/fixture" "${RUNNER_TEMP}/results/cold"
+          test/state-cache/compare.sh "${RUNNER_TEMP}/results/cold" "${RUNNER_TEMP}/results/cached"
+
+      # The restore job needs something to restore; this fails first if kapi
+      # stops writing the path the action caches.
+      - name: Require the parse cache kapi writes
+        shell: bash
+        run: |
+          test -s test/fixture/.kapi/work/cache/docs/index.db
+          cp test/fixture/.kapi/work/store.db "${RUNNER_TEMP}/results/store.db"
+
+      - uses: actions/upload-artifact@v7
+        with:
+          name: state-cache-save-${{ matrix.os }}
+          path: ${{ runner.temp }}/results
+
+  test-state-cache-restore:
+    name: "State cache restore (${{ matrix.os }})"
+    needs: test-state-cache-save
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Setup kapi
+        uses: ./
+        with:
+          token: ${{ secrets.NEOKAPI_GITHUB_TOKEN }}
+          version: ${{ env.STATE_CACHE_KAPI_VERSION }}
+          plugins: ""
+          project-dir: test/fixture
+
+      - name: Require the restored parse cache
+        shell: bash
+        run: |
+          docs=test/fixture/.kapi/work/cache/docs
+          test -s "${docs}/index.db"
+          ls "${docs}"/parts-*.log > /dev/null
+          echo "Restored $(find "${docs}" -type f | wc -l | tr -d ' ') files into ${docs}"
+          cp -R "${docs}" "${RUNNER_TEMP}/restored-docs"
+
+      - uses: actions/download-artifact@v8
+        with:
+          name: state-cache-save-${{ matrix.os }}
+          path: ${{ runner.temp }}/saved
+
+      - name: A warm run matches a cold run
+        shell: bash
+        run: |
+          test/state-cache/cold-copy.sh "${RUNNER_TEMP}/cold"
+          test/state-cache/run.sh test/fixture "${RUNNER_TEMP}/results/warm"
+          test/state-cache/run.sh "${RUNNER_TEMP}/cold/test/fixture" "${RUNNER_TEMP}/results/cold"
+          test -s "${RUNNER_TEMP}/results/warm/docs-before.txt"
+          test/state-cache/compare.sh "${RUNNER_TEMP}/results/cold" "${RUNNER_TEMP}/results/warm"
+          test/state-cache/compare.sh "${RUNNER_TEMP}/saved/cold" "${RUNNER_TEMP}/results/warm"
+
+      - name: A cache saved before a source change does not hide it
+        shell: bash
+        run: |
+          git clean -fdxq test/fixture
+          git checkout -- test/fixture
+          mkdir -p test/fixture/.kapi/work/cache
+          cp -R "${RUNNER_TEMP}/restored-docs" test/fixture/.kapi/work/cache/docs
+          test/state-cache/change-source.sh test/fixture
+          test/state-cache/cold-copy.sh "${RUNNER_TEMP}/cold-changed"
+          test/state-cache/change-source.sh "${RUNNER_TEMP}/cold-changed/test/fixture"
+          test/state-cache/run.sh test/fixture "${RUNNER_TEMP}/results/warm-changed"
+          test/state-cache/run.sh "${RUNNER_TEMP}/cold-changed/test/fixture" "${RUNNER_TEMP}/results/cold-changed"
+          test/state-cache/compare.sh "${RUNNER_TEMP}/results/cold-changed" "${RUNNER_TEMP}/results/warm-changed"
+
+      # The negative control: .kapi/work/store.db holds stored targets and the
+      # unit working set, so restoring it has to change a result. If the
+      # comparison reports no difference here, it cannot be trusted above.
+      - name: The comparison catches a restored store
+        shell: bash
+        run: |
+          git clean -fdxq test/fixture
+          git checkout -- test/fixture
+          mkdir -p test/fixture/.kapi/work/cache
+          cp -R "${RUNNER_TEMP}/restored-docs" test/fixture/.kapi/work/cache/docs
+          cp "${RUNNER_TEMP}/saved/store.db" test/fixture/.kapi/work/store.db
+          test/state-cache/run.sh test/fixture "${RUNNER_TEMP}/results/warm-store"
+          if test/state-cache/compare.sh "${RUNNER_TEMP}/results/cold" "${RUNNER_TEMP}/results/warm-store"; then
+            echo "::error::Restoring .kapi/work/store.db changed no result, so the comparison cannot detect restored state."
+            exit 1
+          fi
+
+      - uses: actions/upload-artifact@v7
+        if: always()
+        with:
+          name: state-cache-restore-${{ matrix.os }}
+          path: ${{ runner.temp }}/results
+
   test-cache:
     name: "Cache hit (${{ matrix.os }})"
     runs-on: ${{ matrix.os }}

README.md

@@ -24,7 +24,7 @@ steps:
       version: "1.1.0"
 ```
 
-### Run a localization pipeline
+### Run a flow
 
 ```yaml
 steps:
@@ -42,12 +42,15 @@ needs credentials:
 steps:
   - uses: neokapi/setup-kapi@v1
     with:
+      version: "1.2.0-rc32"
       auth-token: ${{ secrets.BOWRAIN_AUTH_TOKEN }}
       server: https://your.bowrain.server
 
   - run: kapi up
 ```
 
+The newest stable release, which `latest` installs, has no `kapi up`, so this example pins the prerelease that has it.
+
 `kapi up` runs the kapi loop: with the bowrain plugin installed and a `server:` block in the recipe, it pushes, catches up on the server, and pulls the produced targets back. To run it and commit the results, pair this with [`kapi-action`](https://github.com/neokapi/kapi-action).
 
 ## Inputs
@@ -59,8 +62,8 @@ steps:
 | `plugins` | Newline- or comma-separated plugin refs to install, as the registry names them (`bowrain`, `okapi-bridge`; a `kapi-` prefix is stripped). Pass `''` to install nothing | `bowrain` | No |
 | `auth-token` | Bowrain server JWT, exported as `BOWRAIN_AUTH_TOKEN` | — | No |
 | `server` | Bowrain server URL, exported as `BOWRAIN_SERVER_URL` | — | No |
-| `cache-tm` | Restore/persist the project translation memory across runs via the job cache (out of git). Runs only when a `kapi.yaml` recipe (or legacy `*.kapi`) is present. Set `false` to disable | `true` | No |
-| `project-dir` | Directory holding the `kapi.yaml` project for the TM cache | `.` | No |
+| `cache-tm` | Carry kapi's parse cache (`.kapi/work/cache/docs`) between runs with the job cache; see [Project parse cache](#project-parse-cache). Runs only when a `kapi.yaml` recipe (or legacy `*.kapi`) is present. Set `false` to disable | `true` | No |
+| `project-dir` | Directory holding the `kapi.yaml` project whose parse cache is carried between runs | `.` | No |
 
 ## Outputs
 
@@ -102,12 +105,27 @@ image.
 4. **Add to PATH** — makes `kapi` available to all subsequent steps.
 5. **Configure auth** (optional) — exports `BOWRAIN_AUTH_TOKEN`/`BOWRAIN_SERVER_URL` when `auth-token` is set.
 6. **Install plugins** — installs each ref in `plugins` (default: `bowrain`) via `kapi plugins install`, cached keyed on the plugin set + OS + arch. Refs use the registry names; a `kapi-` binary prefix is stripped (`kapi-bowrain` → `bowrain`).
-7. **Restore project TM cache** (when a `kapi.yaml` recipe (or legacy `*.kapi`) is present) — restores the latest translation memory for the branch from the job cache and, via a run-unique key, saves the grown TM back at job end. The TM is **derived state kept out of git**: it accumulates leverage across runs without being committed, and a cold cache simply rebuilds from the committed translations. No commits, no locking (per-branch, last-write-wins). Disable with `cache-tm: false`.
+7. **Restore the project parse cache** (when a `kapi.yaml` recipe, or legacy `*.kapi`, is present): restores `.kapi/work/cache/docs` from the job cache, and saves it again at job end under a key unique to the job and run attempt. See [Project parse cache](#project-parse-cache). Disable with `cache-tm: false`.
 
 ## Caching
 
 The binary is cached keyed on version + OS + arch; plugins are cached keyed on the plugin set + OS + arch. Both skip their download step on a cache hit.
 
+### Project parse cache
+
+kapi keeps the state it derives out of git, under `.kapi/work/` (the project's `.kapi/.gitignore` ignores `work/` and `filters.local.json`). With `cache-tm` on and a recipe in `project-dir`, the action restores one directory of it, `.kapi/work/cache/docs`, before your steps run, and `actions/cache` saves it again when the job ends. kapi records there how it parsed each source and target file, so a later run can replay a file instead of parsing it again.
+
+A restored parse cache does not change any result. kapi keys each entry by the file's path and content hash, the parse configuration, the recipe and the kapi build, and parses the file again when any of them differs. The test workflow checks this on every change: with the cache restored, `kapi status`, `kapi check`, `kapi check --ship` and `kapi up` must match a cold run exactly, in exit codes, output and every file written, and they must still match after the source changes.
+
+Everything else under `.kapi/` stays out of the cache:
+
+- The content memory, terms, voice profile and unit-state record are committed under `.kapi/`, so the checkout already holds them, and kapi builds its local store from them.
+- `.kapi/work/store.db` also holds stored targets and staged review decisions. Restored from an earlier run, it reports targets the checkout does not hold, and `kapi status`, `kapi check --ship` and `kapi up` report differently than they would on a cold run.
+- `.kapi/work/cache/extractions/` holds `kapi extract` batches for `kapi merge`, and `.kapi/work/cache/redaction/` and `.kapi/work/vault/` hold withheld original values.
+- `.kapi/work/cache/sync-cache.json` and `.kapi/work/cache/refs.json` hold server sync state, including a claim token.
+
+The cache key holds the runner OS, the resolved kapi version and the ref, so a new kapi version starts from an empty cache. How much time the cache saves depends on the formats: container formats such as DOCX replay faster than they parse, while JSON and Markdown parse about as fast as they replay.
+
 ## Platform support
 
 | Runner OS | Architectures |

action.yml

@@ -32,16 +32,20 @@ inputs:
     default: ""
   cache-tm:
     description: >-
-      Restore and persist the project translation memory across CI runs via the
-      job cache. The TM is derived state kept out of git (AD-009): the latest
-      branch TM is restored at setup, and the grown TM is saved back at job end
-      under a run-unique key, so leverage compounds without committing anything.
-      A cold cache is fine — kapi rebuilds the TM from the committed translations.
-      Runs only when a kapi.yaml recipe (or legacy *.kapi) is present. Set 'false' to disable.
+      Carry kapi's parse cache (.kapi/work/cache/docs) between CI runs with the
+      job cache. kapi keeps the state it derives out of git, under .kapi/work/.
+      The content memory, terms and unit-state record are committed under
+      .kapi/, and kapi builds its local store from them in a fresh checkout, so
+      they need no cache. Only the parse cache is restored: each entry is keyed
+      by the file's content, the parse configuration, the recipe and the kapi
+      build, and kapi parses a file again when any of them differs. The store
+      and the rest of .kapi/work/ are never restored, because restoring them
+      changes what kapi status, kapi check and kapi up report. Runs only when a
+      kapi.yaml recipe (or legacy *.kapi) is present. Set 'false' to disable.
     required: false
     default: "true"
   project-dir:
-    description: "Directory holding the kapi.yaml project (recipe + state) for the TM cache. Default: repository root."
+    description: "Directory holding the kapi.yaml project whose parse cache is carried between runs. Default: repository root."
     required: false
     default: "."
 
@@ -186,9 +190,9 @@ runs:
           kapi plugins install ${args[@]+"${args[@]}"} "${plugin}"
         done <<< "${PLUGINS}"
 
-    # 11. Detect a kapi project — a committed kapi.yaml recipe (or legacy *.kapi). The TM cache only
-    #     runs where there is a project TM to persist (the .kapi/ state dir is
-    #     gitignored, so it may not exist yet; the recipe is the committed signal).
+    # 11. Detect a kapi project: a committed kapi.yaml recipe (or legacy *.kapi).
+    #     The parse cache is carried only for a project. The recipe is the
+    #     signal because .kapi/work/ is gitignored and absent from a checkout.
     - name: Detect kapi project
       id: detect-project
       if: inputs.cache-tm != 'false'
@@ -199,25 +203,37 @@ runs:
         if ls "${PROJECT_DIR}"/kapi.yaml >/dev/null 2>&1 || ls "${PROJECT_DIR}"/*.kapi >/dev/null 2>&1; then
           echo "found=true" >> "$GITHUB_OUTPUT"
         else
-          echo "found=false (no kapi.yaml recipe (or legacy *.kapi) in ${PROJECT_DIR}); skipping TM cache" >&2
+          echo "found=false (no kapi.yaml recipe (or legacy *.kapi) in ${PROJECT_DIR}); skipping the parse cache" >&2
           echo "found=false" >> "$GITHUB_OUTPUT"
         fi
 
-    # 12. Restore/persist the project TM across runs via the job cache. The TM is
-    #     derived state kept out of git (AD-009): restore the latest branch TM at
-    #     setup, and — because the key is run-unique — actions/cache saves the
-    #     grown TM at job end via its own post step. No git writes, no locking
-    #     (additive + rebuildable, last-write-wins); a cold cache rebuilds from
-    #     committed content. Covers the current .kapi/tm.db and the .kapi/cache/
-    #     derived stores.
-    - name: Restore project TM cache
+    # 12. Restore kapi's parse cache, and save it at job end. The key is unique
+    #     to the job and run attempt, so actions/cache saves through its own
+    #     post step; the restore takes the newest cache for the same kapi
+    #     version, preferring the same ref.
+    #
+    #     Only .kapi/work/cache/docs is carried. kapi keys each entry by the
+    #     file's absolute path and content hash, the parse configuration, the
+    #     recipe and the kapi build, and parses again on any mismatch, so a
+    #     restored entry is either valid for the checkout or ignored. The rest
+    #     of .kapi/work/ stays out of the cache:
+    #       - store.db holds stored targets and the unit working set; restored,
+    #         it changes what kapi status, kapi check --ship and kapi up report.
+    #       - cache/extractions holds kapi extract batches that kapi merge reads.
+    #       - cache/redaction and vault/ hold withheld original values.
+    #       - cache/sync-cache.json and cache/refs.json hold server sync state,
+    #         including a claim token.
+    #     The content memory, terms and unit-state record are committed under
+    #     .kapi/ and come with the checkout.
+    #
+    #     The kapi version is part of the key because the parse cache keeps the
+    #     entries every build wrote; a new version starts from an empty cache.
+    - name: Restore kapi parse cache
       if: inputs.cache-tm != 'false' && steps.detect-project.outputs.found == 'true'
       uses: actions/cache@v5
       with:
-        path: |
-          ${{ inputs.project-dir }}/.kapi/tm.db
-          ${{ inputs.project-dir }}/.kapi/cache
-        key: kapi-tm-${{ runner.os }}-${{ github.ref_name }}-${{ github.run_id }}
+        path: ${{ inputs.project-dir }}/.kapi/work/cache/docs
+        key: kapi-parse-${{ runner.os }}-${{ steps.resolve.outputs.version }}-${{ github.ref_name }}-${{ github.job }}-${{ github.run_id }}-${{ github.run_attempt }}
         restore-keys: |
-          kapi-tm-${{ runner.os }}-${{ github.ref_name }}-
-          kapi-tm-${{ runner.os }}-
+          kapi-parse-${{ runner.os }}-${{ steps.resolve.outputs.version }}-${{ github.ref_name }}-
+          kapi-parse-${{ runner.os }}-${{ steps.resolve.outputs.version }}-

test/fixture/.kapi/.gitignore

@@ -0,0 +1,2 @@
+work/
+filters.local.json

test/fixture/docs/en/intro.md

@@ -0,0 +1,5 @@
+# Getting started
+
+This guide shows how to install the tool and run it the first time.
+
+Run the installer, then open a terminal and type the command.

test/fixture/kapi.yaml

@@ -0,0 +1,20 @@
+# A small kapi 1.2 project for the state cache test. The pseudo flow needs no
+# provider credentials, so kapi up runs the same on every runner.
+version: v1
+name: cache-fixture
+defaults:
+  source_language: en
+  target_languages:
+    - fr
+  flow: pseudo
+collections:
+  - path: "locales/en.json"
+    format: json
+    target: "locales/{lang}.json"
+  - path: "docs/en/*.md"
+    format: markdown
+    target: "docs/{lang}/*.md"
+flows:
+  pseudo:
+    steps:
+      - tool: pseudo-translate

test/fixture/locales/en.json

@@ -0,0 +1,6 @@
+{
+  "greeting": "Hello, {name}!",
+  "farewell": "Goodbye  and good luck.",
+  "cta": "Start your free trial today",
+  "items": "You have {count} items in your cart"
+}

test/state-cache/change-source.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+# Edits the fixture's JSON source the way a later commit would: one string
+# changes, a doubled space is fixed and a string is added. The Markdown source
+# is left alone, so a restored parse cache still holds a valid entry for it.
+#
+# usage: change-source.sh <project-dir>
+set -euo pipefail
+cat > "$1/locales/en.json" << 'JSON'
+{
+  "greeting": "Welcome back, {name}!",
+  "farewell": "Goodbye and good luck.",
+  "cta": "Start your free trial today",
+  "items": "You have {count} items in your cart",
+  "empty": "Your cart is empty"
+}
+JSON

test/state-cache/cold-copy.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+# Extracts the committed fixture into a fresh directory, so a cold run sees
+# exactly what a checkout holds and no state from any earlier run.
+#
+# usage: cold-copy.sh <dest>   (the project lands in <dest>/test/fixture)
+set -euo pipefail
+rm -rf "$1"
+mkdir -p "$1"
+git archive HEAD test/fixture | tar -x -C "$1"

test/state-cache/compare.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+# Compares two records written by run.sh: every command's exit code and
+# normalized output, and the files the project held afterwards. Prints each
+# difference and exits 1 when there is any.
+#
+# usage: compare.sh <record-a> <record-b>
+set -uo pipefail
+a=$1
+b=$2
+status=0
+for name in status-before check-before ship-before up status-after check-after ship-after; do
+  for ext in rc json; do
+    diff -u "$a/$name.$ext" "$b/$name.$ext" || status=1
+  done
+done
+diff -u "$a/tree.txt" "$b/tree.txt" || status=1
+if [ "$status" -eq 0 ]; then
+  echo "Identical: $a and $b"
+fi
+exit "$status"