diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..a2a9536 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +packages/core/hosted export-ignore diff --git a/.github/workflows/ci-rust.yml b/.github/workflows/ci-rust.yml index 9594a45..c8d9849 100644 --- a/.github/workflows/ci-rust.yml +++ b/.github/workflows/ci-rust.yml @@ -108,6 +108,9 @@ jobs: - name: Checkout uses: actions/checkout@v4 + - name: Validate CLI version bump policy + run: bash scripts/ci/validate-cli-version-bump.sh + - name: Install Rust toolchain uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 with: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 525b9a4..409842b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,6 +40,24 @@ jobs: - name: Run repo hygiene run: node scripts/ci/repo-hygiene.mjs + workflow-lint: + name: workflow-lint + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + contents: read + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + persist-credentials: false + # actionlint parses the shell inside every `run:` block. The repo's other + # workflow checks match text and cannot: a stray `fi` made + # mirror-cli-r2.yml's first step a bash syntax error and sat on `dev` + # undetected for two days (ATO-1934). + - name: Lint workflows + run: bash scripts/ci/lint-workflows.sh + package-metadata: name: package-metadata runs-on: ubuntu-24.04 @@ -71,13 +89,15 @@ jobs: - name: Setup pnpm run: | corepack enable - corepack prepare pnpm@${PNPM_VERSION} --activate + corepack prepare "pnpm@${PNPM_VERSION}" --activate - name: Install dependencies run: pnpm install --frozen-lockfile --ignore-scripts - name: Run release-policy guardrails run: node scripts/ci/release-policy.mjs - name: Run release-policy unit tests run: node --test scripts/ci/__tests__/release-policy.test.mjs + - name: Test public release input wiring + run: node --test scripts/ci/__tests__/public-release-inputs.test.mjs # Every surface that can write a verbatim memory must be able to stamp # content_class. The 0.2.0 release fixed Hermes for Core's raw-content # policy and missed OpenClaw, whose published plugin could not perform a @@ -114,11 +134,14 @@ jobs: - name: Setup pnpm run: | corepack enable - corepack prepare pnpm@${PNPM_VERSION} --activate + corepack prepare "pnpm@${PNPM_VERSION}" --activate - name: Install dependencies run: pnpm install --frozen-lockfile - name: Run affected build, typecheck, lint, and self-contained tests run: node scripts/ci/run-root-script.mjs ci:affected + - name: Run private hosted Core compatibility tests + if: github.repository == 'atomicstrata/atomicmemory-internal' + run: bash packages/core/hosted/__tests__/run.sh code-health: name: code-health @@ -136,7 +159,7 @@ jobs: - name: Setup pnpm run: | corepack enable - corepack prepare pnpm@${PNPM_VERSION} --activate + corepack prepare "pnpm@${PNPM_VERSION}" --activate - name: Install dependencies run: pnpm install --frozen-lockfile - name: Run code health checks @@ -158,7 +181,7 @@ jobs: - name: Setup pnpm run: | corepack enable - corepack prepare pnpm@${PNPM_VERSION} --activate + corepack prepare "pnpm@${PNPM_VERSION}" --activate - name: Install dependencies run: pnpm install --frozen-lockfile - name: Run package dry-runs through turbo @@ -180,7 +203,7 @@ jobs: - name: Setup pnpm run: | corepack enable - corepack prepare pnpm@${PNPM_VERSION} --activate + corepack prepare "pnpm@${PNPM_VERSION}" --activate - name: Install dependencies run: pnpm install --frozen-lockfile - name: Run docs contract @@ -210,7 +233,7 @@ jobs: - name: Setup pnpm run: | corepack enable - corepack prepare pnpm@${PNPM_VERSION} --activate + corepack prepare "pnpm@${PNPM_VERSION}" --activate - name: Install dependencies run: pnpm install --frozen-lockfile - name: Run public integration smoke @@ -273,13 +296,20 @@ jobs: - name: Setup pnpm run: | corepack enable - corepack prepare pnpm@${PNPM_VERSION} --activate + corepack prepare "pnpm@${PNPM_VERSION}" --activate - name: Install dependencies run: pnpm install --frozen-lockfile --ignore-scripts - name: Run security compliance contract tests run: | pnpm run test:security-compliance pnpm run test:release-cli-version + pnpm run test:cli-version-bump pnpm run test:mirror-cli-r2 pnpm run test:install-cli + pnpm run test:install-cli-internal + pnpm run test:cli-install-smoke + pnpm run test:cli-install-smoke-reporter + pnpm run test:reconcile-internal-release + pnpm run test:release-policy + pnpm run test:guards pnpm run security-compliance diff --git a/.github/workflows/cli-install-smoke.yml b/.github/workflows/cli-install-smoke.yml new file mode 100644 index 0000000..431e201 --- /dev/null +++ b/.github/workflows/cli-install-smoke.yml @@ -0,0 +1,140 @@ +name: CLI Fresh Install Smoke + +# Installs the published `am` from cli-internal-latest (or a dispatch-selected +# tag such as cli-canary-latest) onto a clean runner and +# proves it works — the artifact-level counterpart to core-docker-smoke. +# +# The fixture tests (scripts/__tests__/install-cli*.test.sh) drive the installer +# with a fake gh and a fake am, so they prove installer logic and nothing about +# the release. internal-cli-release.yml's own "Native smoke" untars the binary +# on the machine that just built it, bypassing both installers. Neither notices +# if cli-internal-latest is deleted, if its tarballs and version.json disagree, +# or if the shipped binary cannot start without a Rust toolchain present. +# +# One job per published target, so a broken tarball is attributed to its +# platform rather than to "the release". +# +# Repo guard: this file is mirrored into the public repo, and the release it +# installs from is private. Jobs run ONLY on atomicstrata/atomicmemory-internal. + +on: + schedule: + # Daily. The release only changes on pushes to main, so this is watching for + # rot — a deleted release, a missing asset, a runner image that stops + # satisfying the binary — not for churn. + - cron: "20 6 * * *" + workflow_dispatch: + inputs: + tag: + description: "Release tag to install (default: cli-internal-latest; use cli-canary-latest for canary)" + required: false + default: cli-internal-latest + type: string + +permissions: + contents: read + +defaults: + run: + shell: bash + +jobs: + install-smoke: + name: install-smoke ${{ matrix.target }} + if: github.repository == 'atomicstrata/atomicmemory-internal' + runs-on: ${{ matrix.runner }} + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + include: + - target: aarch64-apple-darwin + runner: macos-15 + - target: x86_64-apple-darwin + runner: macos-15-intel + - target: x86_64-unknown-linux-gnu + runner: ubuntu-24.04 + - target: aarch64-unknown-linux-gnu + runner: ubuntu-24.04-arm + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + persist-credentials: false + + # GITHUB_TOKEN with contents:read can read this repository's own release + # assets, so no PAT is involved. + - name: Fresh install smoke + env: + GH_TOKEN: ${{ github.token }} + AM_INTERNAL_TAG: ${{ inputs.tag || 'cli-internal-latest' }} + run: bash scripts/cli-install-smoke.sh + + report: + name: report scheduled status + if: always() && github.repository == 'atomicstrata/atomicmemory-internal' + needs: install-smoke + runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: + contents: read + issues: write + steps: + # A nightly nobody is told about is not a gate. Opens one self-clearing + # issue on failure and closes it on the next green run. + # + # Gated on `schedule` so a manual dispatch — which an engineer is already + # watching, and which may target an arbitrary tag — can neither open nor + # close the nightly's issue. `cancelled` is skipped because a cancelled + # run is not evidence either way. continue-on-error keeps a reporter fault + # from failing a run that actually passed, which would invert the signal + # this exists to protect. + - name: Open or close the failure issue + if: github.event_name == 'schedule' && needs.install-smoke.result != 'cancelled' + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + SMOKE_RESULT: ${{ needs.install-smoke.result }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + LABEL: cli-install-smoke + run: | + set -euo pipefail + # Ask about this label rather than listing and grepping: a repo with + # more labels than the page size would look like it has none, and the + # create would then fail on a label that already exists. + if ! gh label list --search "$LABEL" --json name --jq '.[].name' | grep -qx "$LABEL"; then + gh label create "$LABEL" --color B60205 \ + --description "Nightly fresh-install smoke for the published am CLI" + fi + existing="$(gh issue list --label "$LABEL" --state open \ + --limit 1 --json number --jq '.[0].number // empty')" + + if [ "$SMOKE_RESULT" = "success" ]; then + if [ -n "$existing" ]; then + gh issue close "$existing" \ + --comment "Fresh-install smoke is green again: ${RUN_URL}" + echo "closed #${existing}" + else + echo "green, nothing open" + fi + exit 0 + fi + + body="Nightly fresh-install smoke for \`cli-internal-latest\` reported \`${SMOKE_RESULT}\`. + + Run: ${RUN_URL} + + Each job installs the published \`am\` into a throwaway \$HOME on a clean + runner. A failure means the release as published does not install, not + that a test is flaky — the job list names the affected target. + + This issue closes itself on the next green scheduled run." + if [ -n "$existing" ]; then + gh issue comment "$existing" --body "Still failing: ${RUN_URL}" + echo "commented on #${existing}" + else + gh issue create --label "$LABEL" \ + --title "Nightly CLI fresh-install smoke is failing" \ + --body "$body" + fi diff --git a/.github/workflows/cli-public-install-smoke.yml b/.github/workflows/cli-public-install-smoke.yml new file mode 100644 index 0000000..0839753 --- /dev/null +++ b/.github/workflows/cli-public-install-smoke.yml @@ -0,0 +1,281 @@ +name: CLI Public Install Smoke + +# Installs `am` the way a user does — `curl https://get.atomicstrata.ai/install.sh | sh` +# — onto a clean runner, and proves the result works (ATO-1863). +# +# Its own lane rather than a matrix leg of cli-install-smoke.yml, because the +# two watch different things and fail for different reasons. The internal lane +# reads a private release through `gh` and says nothing about the mirror: it +# stays green while get.atomicstrata.ai serves a 404, a stale version.json, or +# tarballs whose checksums no longer match. That is the path every customer +# takes, and today its breakage would be reported by one of them. +# +# It also reaches two things the internal channel structurally cannot: +# +# * the R2 mirror serves a working install.sh and a version.json that agrees +# with the tarballs beside it +# * attestation verification actually succeeds — the `public-provenance` job +# runs `gh attestation verify` against the published tarball, rather than +# leaving it to the installer's default `auto`, which skips silently on a +# runner where it cannot verify and would let a release with no provenance +# pass as if it had been checked +# +# Needs no repo auth for the install itself, which is the point — if this lane +# ever needs a token to install, users need one too, and that is the bug. +# +# Repo guard: this file is mirrored into the public repo. Jobs run ONLY on +# atomicstrata/atomicmemory-internal so the mirror does not run it twice. + +on: + schedule: + # 06:50 UTC, half an hour after the internal lane. The mirror only changes + # on a release, so this watches for rot — an expired cert, a broken R2 + # sync, a deleted object — rather than for churn. + - cron: "50 6 * * *" + workflow_dispatch: + inputs: + base_url: + # A choice, not free text. `resolve-release` fetches this URL from a + # repo runner before either smoke job applies the script's allowlist, so + # a free-text value made the resolver fetch and parse an arbitrary host + # and publish the result as a job output — and the run then failed + # anyway, because both downstream jobs refuse anything but the pinned + # mirror. The input promised a freedom the lane does not have. + # + # Add a line here and in cli-install-smoke.sh's allowlist together when + # there is a second mirror. + description: "Public mirror to install from" + required: false + default: https://get.atomicstrata.ai + type: choice + options: + - https://get.atomicstrata.ai + +permissions: + contents: read + +defaults: + run: + shell: bash + +jobs: + # Resolved once, then handed to both lanes. They run on separate runners and + # each fetch from the mirror themselves, so without a shared expectation a + # release landing mid-run leaves one installing X and the other attesting + # X+1 — both green, and nothing verified what was installed. + # + # No credential here: it reads a public file. + resolve-release: + name: resolve release version + if: github.repository == 'atomicstrata/atomicmemory-internal' + runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: + contents: read + outputs: + version: ${{ steps.v.outputs.version }} + steps: + - name: Read version.json from the mirror + id: v + env: + BASE: ${{ inputs.base_url || 'https://get.atomicstrata.ai' }} + run: | + set -euo pipefail + # Retried and time-boxed. This is the fan-in for both lanes: a single + # transient blip here skips them both, and a skip is reported. + version="$(curl -fsSL --proto '=https' --tlsv1.2 \ + --max-time 30 --retry 3 --retry-connrefused "${BASE}/version.json" \ + | sed -n 's/.*"version"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' | head -n1)" + if [[ ! "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "::error::mirror version.json has no X.Y.Z version (got '${version}')" + exit 1 + fi + echo "version=${version}" >> "$GITHUB_OUTPUT" + echo "resolved ${version}" >&2 + + public-install-smoke: + needs: resolve-release + name: public-install-smoke ${{ matrix.target }} + if: github.repository == 'atomicstrata/atomicmemory-internal' + runs-on: ${{ matrix.runner }} + timeout-minutes: 15 + strategy: + # One job per published target: a tarball broken for one platform is + # attributed to that platform rather than to "the release". + fail-fast: false + matrix: + include: + - target: aarch64-apple-darwin + runner: macos-15 + - target: x86_64-apple-darwin + runner: macos-15-intel + - target: x86_64-unknown-linux-gnu + runner: ubuntu-24.04 + - target: aarch64-unknown-linux-gnu + runner: ubuntu-24.04-arm + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + persist-credentials: false + + # No GH_TOKEN. This step runs install.sh, which was downloaded from a + # mirror and is not authenticated at the point it executes — so a + # compromised mirror would be handed a private-repo credential before any + # integrity check could matter. It is also the honest customer path: a + # user installing from get.atomicstrata.ai has no repo token. + # + # Provenance is verified in the separate job below, which holds the token + # and executes nothing it downloaded. + - name: Public install smoke (customer path, no credential) + env: + AM_SMOKE_MODE: install + AM_SMOKE_CHANNEL: public + AM_PUBLIC_BASE_URL: ${{ inputs.base_url || 'https://get.atomicstrata.ai' }} + AM_SMOKE_EXPECT_VERSION: ${{ needs.resolve-release.outputs.version }} + AM_SMOKE_TARGET: ${{ matrix.target }} + run: bash scripts/cli-install-smoke.sh + + # Separate job, not a later step: a step in the job above would put the token + # in the same environment the downloaded installer ran in, and on Linux a + # child can read its parent's /proc entry — so `env -u` alone is not a + # boundary. A different job is. + public-provenance: + needs: resolve-release + name: public-provenance ${{ matrix.target }} + if: github.repository == 'atomicstrata/atomicmemory-internal' + runs-on: ${{ matrix.runner }} + timeout-minutes: 15 + permissions: + contents: read + strategy: + fail-fast: false + matrix: + include: + - target: aarch64-apple-darwin + runner: macos-15 + - target: x86_64-apple-darwin + runner: macos-15-intel + - target: x86_64-unknown-linux-gnu + runner: ubuntu-24.04 + - target: aarch64-unknown-linux-gnu + runner: ubuntu-24.04-arm + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + persist-credentials: false + + # Downloads the published tarball and checks who signed it, from the + # checked-in script. Nothing fetched from the network is executed here, + # which is the property that lets this job hold a credential at all. + - name: Verify release provenance + env: + AM_SMOKE_MODE: provenance + AM_SMOKE_CHANNEL: public + AM_PUBLIC_BASE_URL: ${{ inputs.base_url || 'https://get.atomicstrata.ai' }} + AM_SMOKE_EXPECT_VERSION: ${{ needs.resolve-release.outputs.version }} + AM_SMOKE_TARGET: ${{ matrix.target }} + GH_TOKEN: ${{ github.token }} + run: bash scripts/cli-install-smoke.sh + + report: + name: report scheduled status + if: always() && github.repository == 'atomicstrata/atomicmemory-internal' + needs: [resolve-release, public-install-smoke, public-provenance] + runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: + contents: read + issues: write + steps: + # A nightly nobody is told about is not a gate. Same shape as the internal + # lane: one self-clearing issue, its own label so the two cannot close + # each other's. + # + # Gated on `schedule` so a dispatch — which an engineer is watching, and + # which may point at an arbitrary mirror — neither opens nor closes it. + # + # Every job in `needs` is graded. Grading only the install half would let + # a provenance failure — the half that actually checks signatures — pass + # the night in silence, which is the failure this lane exists to catch. + - name: Open or close the failure issue + if: >- + github.event_name == 'schedule' + && needs.resolve-release.result != 'cancelled' + && needs.public-install-smoke.result != 'cancelled' + && needs.public-provenance.result != 'cancelled' + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + RESOLVE_RESULT: ${{ needs.resolve-release.result }} + INSTALL_RESULT: ${{ needs.public-install-smoke.result }} + PROVENANCE_RESULT: ${{ needs.public-provenance.result }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + LABEL: cli-public-install-smoke + run: | + set -euo pipefail + if ! gh label list --search "$LABEL" --json name --jq '.[].name' | grep -qx "$LABEL"; then + gh label create "$LABEL" --color B60205 \ + --description "Nightly fresh-install smoke for the PUBLIC am CLI channel" + fi + existing="$(gh issue list --label "$LABEL" --state open \ + --limit 1 --json number --jq '.[0].number // empty')" + + if [ "$RESOLVE_RESULT" = "success" ] \ + && [ "$INSTALL_RESULT" = "success" ] \ + && [ "$PROVENANCE_RESULT" = "success" ]; then + if [ -n "$existing" ]; then + gh issue close "$existing" \ + --comment "Public install smoke is green again: ${RUN_URL}" + echo "closed #${existing}" + else + echo "green, nothing open" + fi + exit 0 + fi + + if [ "$RESOLVE_RESULT" != "success" ]; then + DIAGNOSIS="**Neither smoke lane ran.** \`resolve-release\` did not + succeed, so both lanes were skipped and nothing about the mirror's + install path or its signatures was tested by this run. Check the + resolver job first: it reads ${RUN_URL} for \`version.json\`. This is a + fan-in failure, not evidence about the customer path." + else + DIAGNOSIS="A failure here means the path users take is broken — the + mirror is not serving a working install.sh, version.json disagrees with + the tarballs beside it, a checksum does not verify, or the published + tarball is not signed by the release workflow. It is not a flaky test, + and it is customer-visible." + fi + + body="Nightly fresh-install smoke for the **public** channel is not green. + + | lane | result | + | --- | --- | + | resolve release version | \`${RESOLVE_RESULT}\` | + | public-install-smoke | \`${INSTALL_RESULT}\` | + | public-provenance | \`${PROVENANCE_RESULT}\` | + + Run: ${RUN_URL} + + \`public-install-smoke\` runs the install command from the README + against https://get.atomicstrata.ai on a clean runner, with no repo + credential and no attestation check — deliberately the path a user + without \`gh\` is on. Provenance is checked separately by + \`public-provenance\`, which holds the token and executes nothing it + downloaded. + + ${DIAGNOSIS} + + The job list names the affected target. This issue closes itself on the + next green scheduled run." + if [ -n "$existing" ]; then + gh issue comment "$existing" --body "Still failing: ${RUN_URL}" + echo "commented on #${existing}" + else + gh issue create --label "$LABEL" \ + --title "Nightly PUBLIC CLI install smoke is failing" \ + --body "$body" + fi diff --git a/.github/workflows/core-deno-compile-macos.yml b/.github/workflows/core-deno-compile-macos.yml new file mode 100644 index 0000000..3df5065 --- /dev/null +++ b/.github/workflows/core-deno-compile-macos.yml @@ -0,0 +1,62 @@ +name: Core Deno compile (macOS) + +on: + pull_request: + paths: + - 'packages/core/**' + - '.github/workflows/core-deno-compile-macos.yml' + push: + branches: + - main + - feat/core-deno-lean-binary + paths: + - 'packages/core/**' + - '.github/workflows/core-deno-compile-macos.yml' + workflow_dispatch: + +permissions: + contents: read + +jobs: + deno-compile-darwin-arm64: + name: deno-compile-darwin-arm64 + runs-on: macos-14 + timeout-minutes: 45 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version: "22" + + - name: Setup pnpm + run: | + corepack enable + corepack prepare pnpm@9.15.4 --activate + + - name: Install Deno + uses: denoland/setup-deno@22d081ff2d3a40755e97629de92e3bcbfa7cf2ed # v2.0.5 + with: + deno-version: v2.9.4 + + - name: Install dependencies + run: pnpm install --frozen-lockfile --filter @atomicmemory/core + + - name: Compile Core binary + working-directory: packages/core + run: ./scripts/deno-compile-core.sh + + - name: Smoke help + working-directory: packages/core + # --help is a compile smoke only. Storage upload/read/delete is + # scripts/macos-compiled-storage-smoke.sh against a reachable database. + run: ./dist-bin/atomicmemory-core-darwin-arm64 --help + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: atomicmemory-core-darwin-arm64 + path: packages/core/dist-bin/atomicmemory-core-darwin-arm64 + if-no-files-found: error diff --git a/.github/workflows/core-ecr-dev-staging.yml b/.github/workflows/core-ecr-dev-staging.yml new file mode 100644 index 0000000..0d37d2f --- /dev/null +++ b/.github/workflows/core-ecr-dev-staging.yml @@ -0,0 +1,405 @@ +name: Core ECR Dev/Staging + +# Internal Dev/Staging lane. On each new @atomicmemory/core version on `dev` +# (or an operator dispatch), builds packages/core/Dockerfile for linux/amd64, +# pushes to the enterprise ECR repository, updates SSM core_image params, and +# rolls the Dev/Staging ECS services to an immutable tag. +# +# This is NOT the public GHCR release pipeline. Official +# ghcr.io/atomicstrata/atomicmemory-core images stay on publish-core-docker.yml. +# Private GHCR test builds stay on internal-core-docker-image.yml. +# +# Naming contract with release-policy: scripts/ci/release-policy.mjs forbids +# workflow_dispatch on every `.github/workflows/publish-*.yml`. This file is +# deliberately named OUTSIDE that prefix so operators can dispatch rebuilds +# and selective deploys. It is an enumerated image publisher pinned to the +# ECR enterprise repository (see release-policy.mjs). +# +# Repo guard: mirrored into the public export, so jobs run ONLY on +# atomicstrata/atomicmemory-internal. Keep this file free of secrets. + +on: + push: + branches: + - dev + paths: + - "packages/core/package.json" + - ".github/workflows/core-ecr-dev-staging.yml" + workflow_dispatch: + inputs: + ref: + description: "Branch, tag, or SHA of atomicmemory-internal to build (default: current SHA / dispatch ref)" + required: false + default: "" + type: string + deploy_dev: + description: "Roll atomicmemory-dev-cluster / atomicmemory-core after push" + required: false + default: true + type: boolean + deploy_staging: + description: "Roll atomicmemory-staging-cluster / atomicmemory-core after push" + required: false + default: true + type: boolean + force: + description: "Rebuild and retag even when the VERSION tag already exists in ECR" + required: false + default: false + type: boolean + +permissions: + contents: read + id-token: write + +concurrency: + group: core-ecr-dev-staging-${{ inputs.ref || github.ref_name }} + cancel-in-progress: false + +defaults: + run: + shell: bash + +env: + IMAGE_NAME: 636941960505.dkr.ecr.us-east-1.amazonaws.com/atomicmemory-core-enterprise + ECR_REPOSITORY: atomicmemory-core-enterprise + AWS_REGION_DEFAULT: us-east-1 + PLATFORM: linux/amd64 + SOURCE_LABEL: atomicmemory-internal + # Defaults match the companion infra SSM params; override via repo vars if paths differ. + SSM_CORE_IMAGE_DEV_DEFAULT: /am-cloud/dev/core_image + SSM_CORE_IMAGE_STAGING_DEFAULT: /am-cloud/staging/core_image + ECS_CLUSTER_DEV: atomicmemory-dev-cluster + ECS_CLUSTER_STAGING: atomicmemory-staging-cluster + ECS_SERVICE: atomicmemory-core + +jobs: + publish: + name: build, push ECR, roll Dev/Staging + if: github.repository == 'atomicstrata/atomicmemory-internal' + runs-on: ubuntu-24.04 + timeout-minutes: 60 + steps: + - name: Checkout requested ref + uses: actions/checkout@v4 + with: + # Empty dispatch ref falls back to the triggering SHA. + ref: ${{ inputs.ref != '' && inputs.ref || github.sha }} + fetch-depth: 0 + + - name: Decide whether to publish + id: gate + env: + EVENT_NAME: ${{ github.event_name }} + EVENT_BEFORE: ${{ github.event.before || '' }} + # Boolean inputs must not use `&& x || true` — that collapses false to true. + FORCE_INPUT: ${{ github.event_name == 'workflow_dispatch' && (inputs.force && 'true' || 'false') || 'false' }} + DEPLOY_DEV_INPUT: ${{ github.event_name == 'workflow_dispatch' && (inputs.deploy_dev && 'true' || 'false') || 'true' }} + DEPLOY_STAGING_INPUT: ${{ github.event_name == 'workflow_dispatch' && (inputs.deploy_staging && 'true' || 'false') || 'true' }} + run: | + set -euo pipefail + + sha="$(git rev-parse HEAD)" + short_sha="${sha:0:7}" + version="$(node -p "require('./packages/core/package.json').version")" + if [[ ! "${version}" =~ ^[0-9]+\.[0-9]+\.[0-9]+([-+][0-9A-Za-z.-]+)?$ ]]; then + echo "::error::packages/core/package.json version '${version}' is not a usable image tag." + exit 1 + fi + + should_run=true + reason="workflow_dispatch" + + if [[ "${EVENT_NAME}" == "push" ]]; then + reason="push_without_prior_version" + if [[ -n "${EVENT_BEFORE}" && "${EVENT_BEFORE}" =~ ^[0-9a-f]{40}$ && "${EVENT_BEFORE}" != "0000000000000000000000000000000000000000" ]]; then + if git cat-file -e "${EVENT_BEFORE}:packages/core/package.json" 2>/dev/null; then + prev_version="$(git show "${EVENT_BEFORE}:packages/core/package.json" | node -p "JSON.parse(require('fs').readFileSync(0,'utf8')).version")" + if [[ "${prev_version}" == "${version}" ]]; then + should_run=false + reason="version_unchanged:${version}" + else + reason="version_changed:${prev_version}->${version}" + fi + fi + fi + fi + + force=false + if [[ "${FORCE_INPUT}" == "true" ]]; then + force=true + should_run=true + reason="force_rebuild" + fi + + deploy_dev=true + deploy_staging=true + if [[ "${EVENT_NAME}" == "workflow_dispatch" ]]; then + [[ "${DEPLOY_DEV_INPUT}" == "true" ]] || deploy_dev=false + [[ "${DEPLOY_STAGING_INPUT}" == "true" ]] || deploy_staging=false + fi + + { + echo "should_run=${should_run}" + echo "force=${force}" + echo "reason=${reason}" + echo "sha=${sha}" + echo "short_sha=${short_sha}" + echo "version=${version}" + echo "deploy_dev=${deploy_dev}" + echo "deploy_staging=${deploy_staging}" + echo "image_sha_tag=sha-${short_sha}" + echo "image_version_tag=${version}" + } >>"${GITHUB_OUTPUT}" + + echo "Gate: should_run=${should_run} reason=${reason} version=${version} sha=${short_sha}" + echo "Deploy flags: dev=${deploy_dev} staging=${deploy_staging} force=${force}" + + - name: Skip when Core version did not change + if: steps.gate.outputs.should_run != 'true' + run: | + set -euo pipefail + echo "No @atomicmemory/core version change on this push; skipping ECR publish and ECS roll." + + - name: Free runner disk space + if: steps.gate.outputs.should_run == 'true' + run: | + set -euo pipefail + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL /usr/local/share/boost + sudo docker image prune --all --force >/dev/null + df -h / + + - name: Resolve AWS settings + id: aws + if: steps.gate.outputs.should_run == 'true' + env: + AWS_REGION_VAR: ${{ vars.AWS_REGION }} + ROLE_SHARED: ${{ vars.AWS_CORE_ECR_PUBLISH_ROLE_ARN }} + ROLE_DEV: ${{ vars.AWS_CORE_ECR_PUBLISH_ROLE_ARN_DEV }} + ROLE_STAGING: ${{ vars.AWS_CORE_ECR_PUBLISH_ROLE_ARN_STAGING }} + SSM_DEV_VAR: ${{ vars.AWS_CORE_IMAGE_SSM_PARAM_DEV }} + SSM_STAGING_VAR: ${{ vars.AWS_CORE_IMAGE_SSM_PARAM_STAGING }} + SSM_DEV_DEFAULT: ${{ env.SSM_CORE_IMAGE_DEV_DEFAULT }} + SSM_STAGING_DEFAULT: ${{ env.SSM_CORE_IMAGE_STAGING_DEFAULT }} + AWS_REGION_DEFAULT: ${{ env.AWS_REGION_DEFAULT }} + run: | + set -euo pipefail + + region="${AWS_REGION_VAR:-${AWS_REGION_DEFAULT}}" + role_publish="${ROLE_SHARED:-}" + role_dev="${ROLE_DEV:-${role_publish}}" + role_staging="${ROLE_STAGING:-${role_publish}}" + ssm_dev="${SSM_DEV_VAR:-${SSM_DEV_DEFAULT}}" + ssm_staging="${SSM_STAGING_VAR:-${SSM_STAGING_DEFAULT}}" + + if [[ -z "${role_dev}" ]]; then + echo "::error::Set repository variable AWS_CORE_ECR_PUBLISH_ROLE_ARN or AWS_CORE_ECR_PUBLISH_ROLE_ARN_DEV (GitHub OIDC role for ECR/ECS/SSM)." + exit 1 + fi + if [[ -z "${role_staging}" ]]; then + echo "::error::Set repository variable AWS_CORE_ECR_PUBLISH_ROLE_ARN or AWS_CORE_ECR_PUBLISH_ROLE_ARN_STAGING." + exit 1 + fi + + { + echo "region=${region}" + echo "role_dev=${role_dev}" + echo "role_staging=${role_staging}" + echo "ssm_dev=${ssm_dev}" + echo "ssm_staging=${ssm_staging}" + } >>"${GITHUB_OUTPUT}" + + - name: Configure AWS credentials (publish / Dev) + if: steps.gate.outputs.should_run == 'true' + uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a + with: + role-to-assume: ${{ steps.aws.outputs.role_dev }} + aws-region: ${{ steps.aws.outputs.region }} + + - name: Login to Amazon ECR + if: steps.gate.outputs.should_run == 'true' + id: ecr + uses: aws-actions/amazon-ecr-login@03f1aad4c6c7ffd436567f42f9384779290529bd + + - name: Check existing VERSION tag + id: existing + if: steps.gate.outputs.should_run == 'true' + env: + VERSION: ${{ steps.gate.outputs.version }} + FORCE: ${{ steps.gate.outputs.force }} + ECR_REPOSITORY: ${{ env.ECR_REPOSITORY }} + run: | + set -euo pipefail + + exists=false + if aws ecr describe-images \ + --repository-name "${ECR_REPOSITORY}" \ + --image-ids "imageTag=${VERSION}" \ + >/dev/null 2>&1; then + exists=true + fi + + skip_build=false + if [[ "${exists}" == "true" && "${FORCE}" != "true" ]]; then + skip_build=true + echo "VERSION tag ${VERSION} already exists in ECR and force=false; will reuse it for ECS roll." + fi + + { + echo "version_tag_exists=${exists}" + echo "skip_build=${skip_build}" + } >>"${GITHUB_OUTPUT}" + + - name: Set up Docker Buildx + if: steps.gate.outputs.should_run == 'true' && steps.existing.outputs.skip_build != 'true' + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f + + - name: Build and verify linux/amd64 image + if: steps.gate.outputs.should_run == 'true' && steps.existing.outputs.skip_build != 'true' + env: + PLATFORM: ${{ env.PLATFORM }} + SHORT_SHA: ${{ steps.gate.outputs.short_sha }} + CORE_VERSION: ${{ steps.gate.outputs.version }} + run: | + set -euo pipefail + + verify_suffix="verify-${SHORT_SHA}-linux-amd64" + + docker buildx build \ + --platform "${PLATFORM}" \ + --file packages/core/Dockerfile \ + --target hosted-runtime \ + --load \ + --tag "${IMAGE_NAME}:${verify_suffix}" \ + . + + image_version="$(docker run --rm --platform "${PLATFORM}" --entrypoint node "${IMAGE_NAME}:${verify_suffix}" -p "require('./package.json').version")" + if [[ "${image_version}" != "${CORE_VERSION}" ]]; then + echo "::error::image package version is ${image_version}, expected ${CORE_VERSION}" + exit 1 + fi + echo "Verified @atomicmemory/core ${image_version} on ${PLATFORM}" + docker image rm --force "${IMAGE_NAME}:${verify_suffix}" >/dev/null + + - name: Build and push ECR tags + if: steps.gate.outputs.should_run == 'true' && steps.existing.outputs.skip_build != 'true' + env: + PLATFORM: ${{ env.PLATFORM }} + GIT_SHA: ${{ steps.gate.outputs.sha }} + SHORT_SHA: ${{ steps.gate.outputs.short_sha }} + CORE_VERSION: ${{ steps.gate.outputs.version }} + SOURCE_LABEL: ${{ env.SOURCE_LABEL }} + run: | + set -euo pipefail + + docker buildx build \ + --platform "${PLATFORM}" \ + --file packages/core/Dockerfile \ + --target hosted-runtime \ + --label "org.opencontainers.image.source=${SOURCE_LABEL}" \ + --label "org.opencontainers.image.revision=${GIT_SHA}" \ + --label "org.opencontainers.image.version=${CORE_VERSION}" \ + --label "org.opencontainers.image.title=@atomicmemory/core (Dev/Staging ECR)" \ + --tag "${IMAGE_NAME}:${CORE_VERSION}" \ + --tag "${IMAGE_NAME}:sha-${SHORT_SHA}" \ + --tag "${IMAGE_NAME}:dev" \ + --push \ + . + + - name: Resolve immutable image URI for ECS + id: image + if: steps.gate.outputs.should_run == 'true' + env: + SHORT_SHA: ${{ steps.gate.outputs.short_sha }} + CORE_VERSION: ${{ steps.gate.outputs.version }} + SKIP_BUILD: ${{ steps.existing.outputs.skip_build }} + run: | + set -euo pipefail + + # Prefer sha-* when we just built this commit; when reusing an + # existing VERSION tag without rebuild, pin ECS to VERSION. + if [[ "${SKIP_BUILD}" == "true" ]]; then + tag="${CORE_VERSION}" + else + tag="sha-${SHORT_SHA}" + fi + uri="${IMAGE_NAME}:${tag}" + { + echo "tag=${tag}" + echo "uri=${uri}" + } >>"${GITHUB_OUTPUT}" + echo "ECS will track ${uri}" + + - name: Roll Dev ECS + if: steps.gate.outputs.should_run == 'true' && steps.gate.outputs.deploy_dev == 'true' + env: + IMAGE_URI: ${{ steps.image.outputs.uri }} + SSM_PARAM: ${{ steps.aws.outputs.ssm_dev }} + CLUSTER: ${{ env.ECS_CLUSTER_DEV }} + SERVICE: ${{ env.ECS_SERVICE }} + IMAGE_REPO: ${{ env.IMAGE_NAME }} + run: | + set -euo pipefail + scripts/ci/roll-core-ecs-image.sh \ + "${IMAGE_URI}" \ + "${IMAGE_REPO}" \ + "${SSM_PARAM}" \ + "${CLUSTER}" \ + "${SERVICE}" + + - name: Configure AWS credentials (Staging) + if: steps.gate.outputs.should_run == 'true' && steps.gate.outputs.deploy_staging == 'true' && steps.aws.outputs.role_staging != steps.aws.outputs.role_dev + uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a + with: + role-to-assume: ${{ steps.aws.outputs.role_staging }} + aws-region: ${{ steps.aws.outputs.region }} + + - name: Roll Staging ECS + if: steps.gate.outputs.should_run == 'true' && steps.gate.outputs.deploy_staging == 'true' + env: + IMAGE_URI: ${{ steps.image.outputs.uri }} + SSM_PARAM: ${{ steps.aws.outputs.ssm_staging }} + CLUSTER: ${{ env.ECS_CLUSTER_STAGING }} + SERVICE: ${{ env.ECS_SERVICE }} + IMAGE_REPO: ${{ env.IMAGE_NAME }} + run: | + set -euo pipefail + scripts/ci/roll-core-ecs-image.sh \ + "${IMAGE_URI}" \ + "${IMAGE_REPO}" \ + "${SSM_PARAM}" \ + "${CLUSTER}" \ + "${SERVICE}" + + - name: Summary + if: always() && steps.gate.outcome == 'success' + env: + SHOULD_RUN: ${{ steps.gate.outputs.should_run }} + REASON: ${{ steps.gate.outputs.reason }} + VERSION: ${{ steps.gate.outputs.version }} + SHORT_SHA: ${{ steps.gate.outputs.short_sha }} + GIT_SHA: ${{ steps.gate.outputs.sha }} + IMAGE_URI: ${{ steps.image.outputs.uri || '' }} + DEPLOY_DEV: ${{ steps.gate.outputs.deploy_dev }} + DEPLOY_STAGING: ${{ steps.gate.outputs.deploy_staging }} + SKIP_BUILD: ${{ steps.existing.outputs.skip_build || '' }} + run: | + set -euo pipefail + + { + echo "## Core ECR Dev/Staging" + echo "" + echo "- should_run: \`${SHOULD_RUN}\` (${REASON})" + echo "- package version: \`${VERSION}\`" + echo "- revision: \`${GIT_SHA}\`" + if [[ "${SHOULD_RUN}" == "true" ]]; then + echo "- skip_build: \`${SKIP_BUILD}\`" + echo "- ECS image URI: \`${IMAGE_URI}\`" + echo "- tags (when built):" + echo " - \`${IMAGE_NAME}:${VERSION}\`" + echo " - \`${IMAGE_NAME}:sha-${SHORT_SHA}\`" + echo " - \`${IMAGE_NAME}:dev\`" + echo "- deploy_dev: \`${DEPLOY_DEV}\`" + echo "- deploy_staging: \`${DEPLOY_STAGING}\`" + fi + } >>"${GITHUB_STEP_SUMMARY}" diff --git a/.github/workflows/internal-cli-release.yml b/.github/workflows/internal-cli-release.yml index f2aca8d..9712eb8 100644 --- a/.github/workflows/internal-cli-release.yml +++ b/.github/workflows/internal-cli-release.yml @@ -1,8 +1,14 @@ name: Internal CLI Release -# Internal-only convenience lane. Builds prebuilt `am` binaries from +# Internal-only convenience lanes. Builds prebuilt `am` binaries from # atomicmemory-internal and publishes them to PRIVATE GitHub Releases for eng -# testing (tags cli-internal-*). +# testing: +# +# main → cli-internal- + floating cli-internal-latest (AM_BUILD_ENV=internal) +# dev → cli-canary- + floating cli-canary-latest (AM_BUILD_ENV=canary) +# +# Channel is derived from the build ref (push branch or workflow_dispatch `ref`). +# Do not pass a separate channel input — auto-from-ref avoids footguns. # # This is NOT the public release pipeline. Official cli-v* Releases + R2 # mirroring are owned exclusively by release-cli.yml / mirror-cli-r2.yml on @@ -20,13 +26,18 @@ on: workflow_dispatch: inputs: ref: - description: "Branch, tag, or SHA of atomicmemory-internal to build" + description: >- + Branch, tag, or SHA of atomicmemory-internal to build. Use `main` for + the internal lane (cli-internal-* / cli-internal-latest) or `dev` for + the canary lane (cli-canary-* / cli-canary-latest). Other refs publish + an immutable cli-internal- only (no floating alias). required: false default: main type: string push: branches: - main + - dev paths: - "crates/**" - "Cargo.toml" @@ -39,11 +50,21 @@ on: permissions: contents: read -# Serialize the whole lane so floating cli-internal-latest has a single writer. -# Branch/SHA dispatches still publish immutable cli-internal- tags; only -# main refreshes the floating alias (see publish job). +# Separate writers per floating alias so canary and internal cannot race. +# Arbitrary SHA dispatches get their own group (no floating refresh). concurrency: - group: internal-cli-release + group: >- + ${{ + ( + (github.event_name == 'push' && github.ref_name == 'dev') || + (github.event_name == 'workflow_dispatch' && inputs.ref == 'dev') + ) && 'internal-cli-release-canary' || + ( + (github.event_name == 'push' && github.ref_name == 'main') || + (github.event_name == 'workflow_dispatch' && (inputs.ref || 'main') == 'main') + ) && 'internal-cli-release-main' || + format('internal-cli-release-other-{0}', github.run_id) + }} cancel-in-progress: false defaults: @@ -61,6 +82,9 @@ jobs: short_sha: ${{ steps.meta.outputs.short_sha }} version: ${{ steps.meta.outputs.version }} tag: ${{ steps.meta.outputs.tag }} + channel: ${{ steps.meta.outputs.channel }} + build_env: ${{ steps.meta.outputs.build_env }} + floating_tag: ${{ steps.meta.outputs.floating_tag }} refresh_latest: ${{ steps.meta.outputs.refresh_latest }} steps: - name: Checkout requested ref @@ -69,12 +93,13 @@ jobs: ref: ${{ inputs.ref || github.sha }} persist-credentials: false - - name: Resolve immutable SHA and version + - name: Resolve immutable SHA, channel, and version id: meta env: # Pass through env — never interpolate untrusted ref text into the script body. EVENT_NAME: ${{ github.event_name }} INPUT_REF: ${{ inputs.ref }} + PUSH_REF_NAME: ${{ github.ref_name }} run: | set -euo pipefail ver="$(awk '/^\[workspace\.package\]/{found=1; next} found && /^version = /{gsub(/[" ]/,"",$3); print $3; exit}' Cargo.toml)" @@ -83,29 +108,105 @@ jobs: exit 1 fi sha="$(git rev-parse HEAD)" - tag="cli-internal-${sha}" + + # Auto channel from ref: main → internal, dev → canary. + # Floating aliases refresh only for those exact named refs. + channel="" + refresh_latest=false + if [ "$EVENT_NAME" = "push" ]; then + case "$PUSH_REF_NAME" in + main) + channel=internal + refresh_latest=true + ;; + dev) + channel=canary + refresh_latest=true + ;; + *) + echo "::error::unexpected push branch '${PUSH_REF_NAME}' (expected main or dev)" + exit 1 + ;; + esac + else + case "${INPUT_REF:-main}" in + main) + channel=internal + refresh_latest=true + ;; + dev) + channel=canary + refresh_latest=true + ;; + *) + # Arbitrary SHA/branch: immutable internal tag only (legacy contract). + channel=internal + refresh_latest=false + ;; + esac + fi + + if [ "$channel" = "canary" ]; then + tag_prefix="cli-canary" + floating_tag="cli-canary-latest" + build_env="canary" + else + tag_prefix="cli-internal" + floating_tag="cli-internal-latest" + build_env="internal" + fi + tag="${tag_prefix}-${sha}" + + # Fail closed: refuse public namespace and cross-channel tag mistakes. case "$tag" in cli-v*) echo "::error::refusing public cli-v tag namespace" exit 1 ;; esac - # Floating alias is only rewritten from main so branch builds cannot - # race to become "latest". Compare quoted env values only. - refresh_latest=false - if [ "$EVENT_NAME" = "push" ]; then - refresh_latest=true - elif [ "${INPUT_REF:-main}" = "main" ]; then - refresh_latest=true + if [ "$channel" = "canary" ]; then + case "$tag" in + cli-internal-*) + echo "::error::canary builds must use cli-canary-, not cli-internal-*" + exit 1 + ;; + esac + if [ "$floating_tag" != "cli-canary-latest" ]; then + echo "::error::canary floating tag must be cli-canary-latest (got ${floating_tag})" + exit 1 + fi + if [ "$refresh_latest" = "true" ] && [ "$build_env" != "canary" ]; then + echo "::error::canary refresh requires AM_BUILD_ENV=canary" + exit 1 + fi + else + case "$tag" in + cli-canary-*) + echo "::error::internal builds must use cli-internal-, not cli-canary-*" + exit 1 + ;; + esac + if [ "$floating_tag" != "cli-internal-latest" ]; then + echo "::error::internal floating tag must be cli-internal-latest (got ${floating_tag})" + exit 1 + fi + if [ "$refresh_latest" = "true" ] && [ "$build_env" != "internal" ]; then + echo "::error::internal refresh requires AM_BUILD_ENV=internal" + exit 1 + fi fi + { printf 'version=%s\n' "$ver" printf 'sha=%s\n' "$sha" printf 'short_sha=%s\n' "${sha:0:7}" printf 'tag=%s\n' "$tag" + printf 'channel=%s\n' "$channel" + printf 'build_env=%s\n' "$build_env" + printf 'floating_tag=%s\n' "$floating_tag" printf 'refresh_latest=%s\n' "$refresh_latest" } >>"$GITHUB_OUTPUT" - echo "Resolved internal version ${ver} @ ${sha} (refresh_latest=${refresh_latest})" + echo "Resolved ${channel} version ${ver} @ ${sha} (tag=${tag}, refresh_latest=${refresh_latest})" build: name: build ${{ matrix.target }} @@ -141,9 +242,12 @@ jobs: - name: Cache Cargo uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 with: - key: internal-cli-${{ matrix.target }} + key: internal-cli-${{ needs.resolve.outputs.channel }}-${{ matrix.target }} - name: Build + env: + AM_GIT_SHA: ${{ needs.resolve.outputs.sha }} + AM_BUILD_ENV: ${{ needs.resolve.outputs.build_env }} run: cargo build --release -p atomicmemory --bin am --target ${{ matrix.target }} --locked - name: Package @@ -163,12 +267,19 @@ jobs: printf '%s\n' "$members" | grep -qx am printf '%s\n' "$members" | grep -qx LICENSE printf '%s\n' "$members" | grep -qx README.md - ! printf '%s\n' "$members" | grep -qx atomicmemory + # `! cmd` is exempt from errexit, so the negated form passed whether or + # not the file was there — the one assertion here that could never fail + # (SC2251). The three above abort correctly because they are not negated. + if printf '%s\n' "$members" | grep -qx atomicmemory; then + echo "::error::tarball contains an 'atomicmemory' member; expected only am, LICENSE, README.md" + exit 1 + fi ls -l dist - name: Native smoke env: VERSION: ${{ needs.resolve.outputs.version }} + BUILD_ENV: ${{ needs.resolve.outputs.build_env }} run: | set -euo pipefail ver="$VERSION" @@ -179,9 +290,19 @@ jobs: tar -xzf "$tarball" -C "$work" chmod +x "$work/am" got="$("$work/am" --version)" - expected="am ${ver}" - if [ "$got" != "$expected" ]; then - echo "::error::version mismatch: expected '${expected}', got '${got}'" + got_ver="$(printf '%s' "$got" | sed -n 's/.*"version"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' | head -n1)" + got_sha="$(printf '%s' "$got" | sed -n 's/.*"gitSha"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' | head -n1)" + got_env="$(printf '%s' "$got" | sed -n 's/.*"env"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' | head -n1)" + if [ "$got_ver" != "$ver" ]; then + echo "::error::version mismatch: expected version '${ver}', got '${got}'" + exit 1 + fi + if [ -z "$got_sha" ]; then + echo "::error::CI-stamped build must report non-empty gitSha; got '${got}'" + exit 1 + fi + if [ "$got_env" != "$BUILD_ENV" ]; then + echo "::error::env mismatch: expected '${BUILD_ENV}', got '${got}'" exit 1 fi "$work/am" --help | head -n1 | grep -qi atomicmemory @@ -189,13 +310,13 @@ jobs: - name: Upload tarball uses: actions/upload-artifact@v4 with: - name: tarball-${{ matrix.target }} + name: tarball-${{ needs.resolve.outputs.channel }}-${{ matrix.target }} path: dist/*.tar.gz if-no-files-found: error retention-days: 7 publish: - name: publish internal GitHub Release + name: publish ${{ needs.resolve.outputs.channel }} GitHub Release if: github.repository == 'atomicstrata/atomicmemory-internal' needs: [resolve, build] runs-on: ubuntu-24.04 @@ -215,7 +336,7 @@ jobs: uses: actions/download-artifact@v4 with: path: artifacts - pattern: tarball-* + pattern: tarball-${{ needs.resolve.outputs.channel }}-* merge-multiple: true - name: Assemble release assets @@ -223,6 +344,8 @@ jobs: VERSION: ${{ needs.resolve.outputs.version }} TAG: ${{ needs.resolve.outputs.tag }} SHA: ${{ needs.resolve.outputs.sha }} + CHANNEL: ${{ needs.resolve.outputs.channel }} + BUILD_ENV: ${{ needs.resolve.outputs.build_env }} run: | set -euo pipefail ver="$VERSION" @@ -230,9 +353,10 @@ jobs: cp artifacts/*.tar.gz dist/ cp scripts/install-cli.sh dist/install-cli.sh cp scripts/install-cli-internal.sh dist/install.sh - printf '{"version":"%s","tag":"%s","git_sha":"%s","channel":"internal"}\n' \ - "$ver" "$TAG" "$SHA" >dist/version.json - ( cd dist && sha256sum *.tar.gz >SHA256SUMS ) + # env + channel both carry the lane so testers can tell canary from internal. + printf '{"surface":"cli","version":"%s","gitSha":"%s","env":"%s","tag":"%s","channel":"%s"}\n' \ + "$ver" "$SHA" "$BUILD_ENV" "$TAG" "$CHANNEL" >dist/version.json + ( cd dist && sha256sum -- *.tar.gz >SHA256SUMS ) echo "==== SHA256SUMS ====" cat dist/SHA256SUMS echo "==== version.json ====" @@ -251,41 +375,104 @@ jobs: # verifies target SHA + name manifest, then swaps dist/ for the # immutable release's actual bytes (rebuilt tarballs are not # byte-reproducible) so the floating alias upload can never - # publish content different from cli-internal-. + # publish content different from the immutable tag. run: scripts/ci/reconcile-internal-release.sh - - name: Create immutable internal release + - name: Create immutable release if: steps.reconcile.outputs.release_exists != 'true' env: TAG: ${{ needs.resolve.outputs.tag }} VERSION: ${{ needs.resolve.outputs.version }} SHA: ${{ needs.resolve.outputs.sha }} SHORT_SHA: ${{ needs.resolve.outputs.short_sha }} + CHANNEL: ${{ needs.resolve.outputs.channel }} + FLOATING_TAG: ${{ needs.resolve.outputs.floating_tag }} GH_TOKEN: ${{ github.token }} run: | set -euo pipefail + case "$CHANNEL" in + canary) + case "$TAG" in + cli-canary-*) ;; + *) + echo "::error::canary create refused unexpected tag '${TAG}'" + exit 1 + ;; + esac + ;; + internal) + case "$TAG" in + cli-internal-*) ;; + *) + echo "::error::internal create refused unexpected tag '${TAG}'" + exit 1 + ;; + esac + ;; + *) + echo "::error::unknown channel '${CHANNEL}'" + exit 1 + ;; + esac gh release create "$TAG" dist/* \ --repo "$GH_REPO" \ --target "${SHA}" \ - --title "am ${VERSION} (internal ${SHORT_SHA})" \ - --notes "Internal eng-team build of am ${VERSION} at ${SHA}. Not a public release. Install: tmp=\"\$(mktemp -d)\" && gh release download cli-internal-latest --repo atomicstrata/atomicmemory-internal --pattern install.sh --dir \"\$tmp\" && sh \"\$tmp/install.sh\"" \ + --title "am ${VERSION} (${CHANNEL} ${SHORT_SHA})" \ + --notes "${CHANNEL} eng-team build of am ${VERSION} at ${SHA}. Not a public release. Install: tmp=\"\$(mktemp -d)\" && AM_INTERNAL_TAG=${FLOATING_TAG} gh release download ${FLOATING_TAG} --repo atomicstrata/atomicmemory-internal --pattern install.sh --dir \"\$tmp\" && AM_INTERNAL_TAG=${FLOATING_TAG} sh \"\$tmp/install.sh\"" \ --latest=false - - name: Refresh floating cli-internal-latest + - name: Refresh floating channel alias if: needs.resolve.outputs.refresh_latest == 'true' env: VERSION: ${{ needs.resolve.outputs.version }} SHA: ${{ needs.resolve.outputs.sha }} SHORT_SHA: ${{ needs.resolve.outputs.short_sha }} + CHANNEL: ${{ needs.resolve.outputs.channel }} + FLOATING_TAG: ${{ needs.resolve.outputs.floating_tag }} + IMMUTABLE_TAG: ${{ needs.resolve.outputs.tag }} GH_TOKEN: ${{ github.token }} run: | set -euo pipefail - if gh release view cli-internal-latest --repo "$GH_REPO" >/dev/null 2>&1; then - gh release delete cli-internal-latest --repo "$GH_REPO" --yes --cleanup-tag + # Fail closed: never let a canary job touch cli-internal-latest or + # an internal job touch cli-canary-latest. + case "$CHANNEL" in + canary) + if [ "$FLOATING_TAG" != "cli-canary-latest" ]; then + echo "::error::canary lane refused to refresh '${FLOATING_TAG}'" + exit 1 + fi + case "$IMMUTABLE_TAG" in + cli-canary-*) ;; + *) + echo "::error::canary refresh refused unexpected immutable tag '${IMMUTABLE_TAG}'" + exit 1 + ;; + esac + ;; + internal) + if [ "$FLOATING_TAG" != "cli-internal-latest" ]; then + echo "::error::internal lane refused to refresh '${FLOATING_TAG}'" + exit 1 + fi + case "$IMMUTABLE_TAG" in + cli-internal-*) ;; + *) + echo "::error::internal refresh refused unexpected immutable tag '${IMMUTABLE_TAG}'" + exit 1 + ;; + esac + ;; + *) + echo "::error::unknown channel '${CHANNEL}'" + exit 1 + ;; + esac + if gh release view "$FLOATING_TAG" --repo "$GH_REPO" >/dev/null 2>&1; then + gh release delete "$FLOATING_TAG" --repo "$GH_REPO" --yes --cleanup-tag fi - gh release create cli-internal-latest dist/* \ + gh release create "$FLOATING_TAG" dist/* \ --repo "$GH_REPO" \ --target "${SHA}" \ - --title "am ${VERSION} (internal latest → ${SHORT_SHA})" \ - --notes "Floating internal eng-team channel for am ${VERSION} @ ${SHA}. Not a public release." \ + --title "am ${VERSION} (${CHANNEL} latest → ${SHORT_SHA})" \ + --notes "Floating ${CHANNEL} eng-team channel for am ${VERSION} @ ${SHA} (from ${IMMUTABLE_TAG}). Not a public release." \ --latest=false diff --git a/.github/workflows/mirror-cli-r2.yml b/.github/workflows/mirror-cli-r2.yml index b571feb..1c31cf0 100644 --- a/.github/workflows/mirror-cli-r2.yml +++ b/.github/workflows/mirror-cli-r2.yml @@ -35,15 +35,24 @@ jobs: INPUT_VERSION: ${{ github.event_name == 'workflow_dispatch' && inputs.version || '' }} PAYLOAD_VERSION: ${{ github.event.client_payload.version || '' }} PAYLOAD_TAG: ${{ github.event.client_payload.tag || '' }} + PAYLOAD_GIT_SHA: ${{ github.event.client_payload.gitSha || '' }} + GH_TOKEN: ${{ github.token }} run: | set -euo pipefail if [ "$EVENT_NAME" = "repository_dispatch" ]; then tag="$PAYLOAD_TAG" ver="$PAYLOAD_VERSION" + git_sha="$PAYLOAD_GIT_SHA" else ver="$INPUT_VERSION" ver="${ver#v}" tag="cli-v${ver}" + # Peel the public release tag to a commit SHA for version.json. + git_sha="$(gh api "repos/atomicstrata/atomicmemory/git/ref/tags/${tag}" --jq .object.sha)" + obj_type="$(gh api "repos/atomicstrata/atomicmemory/git/ref/tags/${tag}" --jq .object.type)" + if [ "$obj_type" = "tag" ]; then + git_sha="$(gh api "repos/atomicstrata/atomicmemory/git/tags/${git_sha}" --jq .object.sha)" + fi fi if ! printf '%s' "$ver" | grep -Eq '^[0-9]+\.[0-9]+\.[0-9]+$'; then echo "::error::invalid version: ${ver} (expected X.Y.Z)" @@ -53,9 +62,14 @@ jobs: echo "::error::tag/version mismatch: tag=${tag} version=${ver}" exit 1 fi + if [ "$EVENT_NAME" = "repository_dispatch" ] && [ -z "$git_sha" ]; then + echo "::error::cli-release-published payload missing gitSha" + exit 1 + fi { printf 'tag=%s\n' "$tag" printf 'version=%s\n' "$ver" + printf 'git_sha=%s\n' "$git_sha" } >>"$GITHUB_OUTPUT" - name: Download public Release assets @@ -94,9 +108,15 @@ jobs: env: VERSION: ${{ steps.rel.outputs.version }} TAG: ${{ steps.rel.outputs.tag }} + GIT_SHA: ${{ steps.rel.outputs.git_sha }} run: | set -euo pipefail - printf '{"version":"%s","tag":"%s"}\n' "$VERSION" "$TAG" > version.json + if [ -z "$GIT_SHA" ]; then + echo "::error::gitSha is required to render version.json for the CLI contract" + exit 1 + fi + printf '{"surface":"cli","version":"%s","gitSha":"%s","env":"production","tag":"%s"}\n' \ + "$VERSION" "$GIT_SHA" "$TAG" > version.json - name: Upload versioned assets to Cloudflare R2 env: @@ -165,9 +185,14 @@ jobs: AM_BASE_URL="${base}" AM_VERSION="${ver}" \ sh dist/install.sh --bin-dir "$HOME/.am/bin" --no-modify-path got="$("$HOME/.am/bin/am" --version)" - expected="am ${ver}" - if [ "$got" != "$expected" ]; then - echo "::error::version mismatch: expected '${expected}', got '${got}'" + got_ver="$(printf '%s' "$got" | sed -n 's/.*"version"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' | head -n1)" + got_sha="$(printf '%s' "$got" | sed -n 's/.*"gitSha"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' | head -n1)" + if [ "$got_ver" != "$ver" ]; then + echo "::error::version mismatch: expected version '${ver}', got '${got}'" + exit 1 + fi + if [ -z "$got_sha" ]; then + echo "::error::mirrored build must report non-empty gitSha; got '${got}'" exit 1 fi "$HOME/.am/bin/am" --help | head -n1 | grep -qi atomicmemory diff --git a/.github/workflows/publish-core-docker.yml b/.github/workflows/publish-core-docker.yml index 5422805..f8dd930 100644 --- a/.github/workflows/publish-core-docker.yml +++ b/.github/workflows/publish-core-docker.yml @@ -5,6 +5,10 @@ on: types: - core-npm-published workflow_call: + secrets: + PUBLIC_ARTIFACT_SIGNATURES: + description: "Private newline-separated patterns used to validate public artifacts." + required: true inputs: core_version: description: "Published @atomicmemory/core version to build and tag." @@ -266,6 +270,29 @@ jobs: exit 1 fi + - name: Check public package and image for private identifiers + if: steps.package.outputs.should_publish == 'true' && steps.package.outputs.retag_latest_only != 'true' && steps.package.outputs.retag_aliases_only != 'true' + env: + PUBLIC_ARTIFACT_SIGNATURES: ${{ secrets.PUBLIC_ARTIFACT_SIGNATURES }} + PACKAGE_TARBALL: ${{ steps.package.outputs.tarball }} + IMAGE_REFERENCE: ${{ env.IMAGE_NAME }}:${{ steps.package.outputs.version }} + run: | + set -euo pipefail + tarball_path="${RUNNER_TEMP}/atomicmemory-core-package.tgz" + curl --fail --location --retry 3 --output "${tarball_path}" "${PACKAGE_TARBALL}" + if [[ -z "${PUBLIC_ARTIFACT_SIGNATURES//[[:space:]]/}" ]]; then + echo "::error::PUBLIC_ARTIFACT_SIGNATURES must be configured for Core releases." + exit 64 + fi + umask 077 + signature_file="$(mktemp "${RUNNER_TEMP}/public-artifact-signatures.XXXXXX")" + trap 'rm -f "$signature_file"' EXIT + printf '%s\n' "${PUBLIC_ARTIFACT_SIGNATURES}" | sed '/^[[:space:]]*$/d' > "${signature_file}" + release-source/packages/core/scripts/check-public-artifacts.sh \ + "${tarball_path}" \ + "${IMAGE_REFERENCE}" \ + "${signature_file}" + - name: Smoke test local release image if: steps.package.outputs.should_publish == 'true' && steps.package.outputs.retag_latest_only != 'true' && steps.package.outputs.retag_aliases_only != 'true' env: @@ -469,7 +496,7 @@ jobs: verify_tag() { local image_ref="$1" - for attempt in {1..12}; do + for _ in {1..12}; do if assert_platform "${image_ref}" linux amd64 && assert_platform "${image_ref}" linux arm64; then echo "${image_ref} includes linux/amd64 and linux/arm64." return 0 diff --git a/.github/workflows/publish-packages.yml b/.github/workflows/publish-packages.yml index a50a328..9740e96 100644 --- a/.github/workflows/publish-packages.yml +++ b/.github/workflows/publish-packages.yml @@ -75,14 +75,10 @@ jobs: echo "manifest_json<>"${GITHUB_OUTPUT}" - { echo "public_sha=${public_sha}" echo "publish=${publish}" echo "core_selected=${core_selected}" echo "core_version=${core_version}" - } >>"${GITHUB_OUTPUT}" - { echo "selected_summary< "${signature_file}" + package_dir="${RUNNER_TEMP}/atomicmemory-core-package" + image_ref="atomicmemory-core:public-leak-check" + mkdir -p "${package_dir}" + (cd packages/core && npm pack --ignore-scripts --pack-destination "${package_dir}") + package_tarball="$(find "${package_dir}" -maxdepth 1 -name '*.tgz' -print -quit)" + test -n "${package_tarball}" + docker build --file packages/core/Dockerfile --tag "${image_ref}" . + packages/core/scripts/check-public-artifacts.sh \ + "${package_tarball}" \ + "${image_ref}" \ + "${signature_file}" - name: per-selected-package npm pack --dry-run run: | set -euo pipefail @@ -183,7 +204,7 @@ jobs: - name: Setup pnpm run: | corepack enable - corepack prepare pnpm@${PNPM_VERSION} --activate + corepack prepare "pnpm@${PNPM_VERSION}" --activate - name: Require npm >= 11.5.1 for Trusted Publishing run: | set -euo pipefail @@ -232,7 +253,7 @@ jobs: - name: Verify each selected npm package is visible and pinned to public_sha run: | set -euo pipefail - while IFS=$'\t' read -r name version path registry; do + while IFS=$'\t' read -r name version _path registry; do if [[ "${registry}" != "npm" ]]; then continue fi @@ -255,6 +276,8 @@ jobs: needs: [manifest, verify-npm] if: needs.manifest.outputs.publish == 'true' && needs.manifest.outputs.core_selected == 'true' uses: ./.github/workflows/publish-core-docker.yml + secrets: + PUBLIC_ARTIFACT_SIGNATURES: ${{ secrets.PUBLIC_ARTIFACT_SIGNATURES }} permissions: contents: read packages: write diff --git a/.github/workflows/release-cli.yml b/.github/workflows/release-cli.yml index 26e6e9c..22f51fc 100644 --- a/.github/workflows/release-cli.yml +++ b/.github/workflows/release-cli.yml @@ -121,6 +121,14 @@ jobs: exit 1 fi + - name: Validate public version bump + env: + VERSION: ${{ steps.ver.outputs.version }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -euo pipefail + PROPOSED_VERSION="$VERSION" RELEASE_MODE=1 bash scripts/ci/validate-cli-version-bump.sh + - name: Install Rust toolchain uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 with: @@ -133,6 +141,9 @@ jobs: key: release-${{ matrix.target }} - name: Build + env: + AM_GIT_SHA: ${{ github.sha }} + AM_BUILD_ENV: production run: cargo build --release -p atomicmemory --bin am --target ${{ matrix.target }} --locked - name: Package @@ -152,7 +163,13 @@ jobs: printf '%s\n' "$members" | grep -qx am printf '%s\n' "$members" | grep -qx LICENSE printf '%s\n' "$members" | grep -qx README.md - ! printf '%s\n' "$members" | grep -qx atomicmemory + # `! cmd` is exempt from errexit, so the negated form passed whether or + # not the file was there — the one assertion here that could never fail + # (SC2251). The three above abort correctly because they are not negated. + if printf '%s\n' "$members" | grep -qx atomicmemory; then + echo "::error::tarball contains an 'atomicmemory' member; expected only am, LICENSE, README.md" + exit 1 + fi ls -l dist - name: Native smoke @@ -168,9 +185,14 @@ jobs: tar -xzf "$tarball" -C "$work" chmod +x "$work/am" got="$("$work/am" --version)" - expected="am ${ver}" - if [ "$got" != "$expected" ]; then - echo "::error::version mismatch: expected '${expected}', got '${got}'" + got_ver="$(printf '%s' "$got" | sed -n 's/.*"version"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' | head -n1)" + got_sha="$(printf '%s' "$got" | sed -n 's/.*"gitSha"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' | head -n1)" + if [ "$got_ver" != "$ver" ]; then + echo "::error::version mismatch: expected version '${ver}', got '${got}'" + exit 1 + fi + if [ -z "$got_sha" ]; then + echo "::error::CI-stamped build must report non-empty gitSha; got '${got}'" exit 1 fi "$work/am" --help | head -n1 | grep -qi atomicmemory @@ -242,11 +264,10 @@ jobs: VERSION: ${{ steps.ver.outputs.version }} run: | set -euo pipefail - ver="$VERSION" mkdir -p dist cp artifacts/*.tar.gz dist/ cp scripts/install-cli.sh dist/install.sh - ( cd dist && sha256sum *.tar.gz > SHA256SUMS ) + ( cd dist && sha256sum -- *.tar.gz > SHA256SUMS ) echo "==== SHA256SUMS ====" cat dist/SHA256SUMS @@ -284,12 +305,14 @@ jobs: env: VERSION: ${{ steps.ver.outputs.version }} TAG: ${{ steps.ver.outputs.tag }} + GIT_SHA: ${{ github.sha }} run: | set -euo pipefail payload="$(jq -n \ --arg ver "$VERSION" \ --arg tag "$TAG" \ - '{event_type:"cli-release-published", client_payload:{version:$ver, tag:$tag}}')" + --arg sha "$GIT_SHA" \ + '{event_type:"cli-release-published", client_payload:{version:$ver, tag:$tag, gitSha:$sha}}')" curl -fsSL -X POST \ -H "Authorization: token ${MIRROR_DISPATCH_TOKEN}" \ -H "Accept: application/vnd.github+json" \ diff --git a/.gitignore b/.gitignore index 0eed08b..0e10720 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,8 @@ node_modules/ dist/ build/ lib/ +!packages/core/scripts/lib/ +!packages/core/scripts/lib/** /target/ *.tsbuildinfo diff --git a/CHANGELOG.md b/CHANGELOG.md index 46e33a2..f0c0379 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,8 +40,8 @@ or publish pending. - CLI (`am`) public install channel via GitHub Releases and `get.atomicstrata.ai`, with install and verification steps in [`crates/cli/README.md`](crates/cli/README.md). -- `am integrate` for global host MCP install into Cursor, Claude Code, and - Codex (`list`, `detect`, `install`, `update`, `doctor`, `uninstall`). See +- `am integrate` for global host MCP install into Cursor, Claude Code, Codex, + and OpenCode (`list`, `detect`, `install`, `update`, `doctor`, `uninstall`). See [`crates/cli/README.md`](crates/cli/README.md). - MCP `memory_ingest` reserved-metadata preflight and agent-facing schema guidance in `@atomicmemory/mcp-server` 0.1.5. See @@ -55,6 +55,10 @@ or publish pending. - Core OpenAI chat parameter selection and retry mitigations for reasoning and token-limit SKUs (no public API change). See [`packages/core/CHANGELOG.md`](packages/core/CHANGELOG.md). +- `@atomicmemory/core` 1.2.2: harden `extractFacts` JSON parse against trailing + prose after a successful completion (ATO-2185), and keep hosted migration + compatibility in private deployment tooling. See + [`packages/core/CHANGELOG.md`](packages/core/CHANGELOG.md). ### Changed diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index be24833..ad20eec 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -44,6 +44,44 @@ These checks always read current repository state. Some are explicit `cache: false` Turbo tasks; others are direct root scripts that bypass Turbo's cache. +### CLI fresh-install smoke + +Installs the published `am` from the internal channel onto a machine that has +never had it, and proves the result works — the artifact-level counterpart to +`core-docker-smoke`. Needs an authenticated `gh` with access to +`atomicstrata/atomicmemory-internal`; nothing else. + +```bash +gh auth login # once +pnpm run smoke:cli-install +AM_INTERNAL_TAG=cli-internal- pnpm run smoke:cli-install # pin a build +AM_INTERNAL_TAG=cli-canary-latest pnpm run smoke:cli-install # floating canary (dev) +AM_SMOKE_KEEP=1 pnpm run smoke:cli-install # keep the sandbox to inspect +``` + +Install canary without the smoke harness: + +```bash +tmp="$(mktemp -d)" && \ +AM_INTERNAL_TAG=cli-canary-latest gh release download cli-canary-latest \ + --repo atomicstrata/atomicmemory-internal \ + --pattern install.sh \ + --dir "$tmp" \ + && AM_INTERNAL_TAG=cli-canary-latest sh "$tmp/install.sh" +``` + +`$HOME` and the install directory are throwaway, so it will not disturb an `am` +you already have installed — and it cannot be fooled by one either, because +every check addresses the newly installed binary by absolute path. `PATH` is +deliberately left as it is: the one check that does consult it, that sourcing +`~/.atomicmemory/env` makes the new install win, is a stronger check when a +competing `am` is present. +`scripts/cli-install-smoke.sh` documents what each check catches and why the +existing fixture tests do not cover it. It also runs daily in CI, one job per +published target — see `.github/workflows/cli-install-smoke.yml`. A scheduled +failure opens one self-clearing issue labelled `cli-install-smoke`; a manual +dispatch never touches it. + CI uses thin `ci:*` aliases that wrap the same Turbo tasks: ```bash diff --git a/Cargo.lock b/Cargo.lock index fde2f2c..f330820 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -164,6 +164,7 @@ dependencies = [ "fs4", "hex", "indicatif", + "jsonc-parser", "open", "rand", "regex", @@ -191,9 +192,9 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] name = "aws-lc-rs" -version = "1.17.3" +version = "1.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00bdb5da18dac48ca2cc7cd4a98e533e8635a58e2361d13a1a4ee3888e0d72f1" +checksum = "b281d307588d634de920874890732659e2e7672f72b5e10e81badc1a8a83621e" dependencies = [ "aws-lc-sys", "zeroize", @@ -201,9 +202,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.43.0" +version = "0.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43103168cc76fe62678a375e722fc9cb3a0146159ac5828bc4f0dfd755c2224c" +checksum = "9bff6c3b54fad79a2e60b8102caf565819711497c1f5f092f49508e2f5c31b27" dependencies = [ "cc", "cmake", @@ -595,7 +596,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -1178,6 +1179,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "jsonc-parser" +version = "0.33.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ff5a48f48971be8e762a6ff955725a0802b6e46c441057992da5a673db9fd3a" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "lazy_static" version = "1.5.0" @@ -1479,7 +1490,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -1683,14 +1694,14 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] name = "rustls" -version = "0.23.42" +version = "0.23.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c54fcab019b409d04215d3a17cb438fd7fbf192ee61461f20f4fe18704bc138" +checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" dependencies = [ "aws-lc-rs", "once_cell", @@ -1740,7 +1751,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -1751,9 +1762,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" [[package]] name = "rustls-webpki" -version = "0.103.13" +version = "0.103.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" dependencies = [ "aws-lc-rs", "ring", @@ -2046,7 +2057,7 @@ dependencies = [ "getrandom 0.4.3", "once_cell", "rustix 1.1.4", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -2564,7 +2575,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 4aa62f8..fff7c7f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,6 +34,7 @@ reqwest = { version = "0.13.3", default-features = false, features = [ ] } serde = { version = "1.0.228", features = ["derive"] } serde_json = { version = "1.0.150", features = ["preserve_order"] } +jsonc-parser = { version = "0.33.2", features = ["cst", "serde", "serde_json"] } validator = { version = "0.20.0", features = ["derive"] } regex = "1.12.3" uuid = { version = "1.23.1", features = ["v7", "serde"] } @@ -62,7 +63,7 @@ indicatif = "0.18" wiremock = "0.6.5" clap = { version = "4", features = ["derive", "env"] } fs4 = { version = "0.12.0", features = ["sync"] } -am-core-types = { path = "crates/core-types", version = "0.2.0" } -am-cloud-types = { path = "crates/cloud-types", version = "0.2.0" } -am-cloud-client = { path = "crates/cloud-client", version = "0.2.0" } -atomicmemory = { path = "crates/cli", version = "0.2.0" } +am-core-types = { path = "crates/core-types", version = "0.2.1" } +am-cloud-types = { path = "crates/cloud-types", version = "0.2.1" } +am-cloud-client = { path = "crates/cloud-client", version = "0.2.1" } +atomicmemory = { path = "crates/cli", version = "0.2.1" } diff --git a/README.md b/README.md index 22e1918..5cbf4fe 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,7 @@ am memory search "seat preference" Connect the active profile to an agent host when you are ready: ```bash -am integrate --yes --host cursor # or claude-code / codex +am integrate --yes --host cursor # or claude-code / codex / opencode ``` `am integrate` writes the host's user-level MCP configuration. It does not @@ -169,7 +169,7 @@ Both initialization paths leave an active profile that the published MCP server can use. Configure a supported host with: ```bash -am integrate --yes --host cursor # or claude-code / codex +am integrate --yes --host cursor # or claude-code / codex / opencode ``` Codex and Cursor marketplace plugin packages remain **coming soon** in the diff --git a/ROADMAP.md b/ROADMAP.md index 7c0e3ac..772ef54 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -52,7 +52,7 @@ auditable. - Keep the MCP server stable for memory capture, retrieval, search, and context packaging workflows. - **Shipped:** `am integrate` installs the published `@atomicmemory/mcp-server` - into global host MCP config for Cursor, Claude Code, and Codex (`list`, + into global host MCP config for Cursor, Claude Code, Codex, and OpenCode (`list`, `detect`, `install`, `update`, `doctor`, `uninstall`). Global user config only in v1. - **Next:** project-scoped host configs (for example repo-local `.cursor/mcp.json`), diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index 1b0a753..67ccf93 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -37,6 +37,7 @@ regex.workspace = true reqwest.workspace = true serde.workspace = true serde_json.workspace = true +jsonc-parser.workspace = true sha2.workspace = true hex.workspace = true thiserror.workspace = true diff --git a/crates/cli/README.md b/crates/cli/README.md index d51f77f..8fd89b0 100644 --- a/crates/cli/README.md +++ b/crates/cli/README.md @@ -6,6 +6,8 @@ management, Connected Local linking, and memory operations. Phase 2 ships prebuilt **`am`** binaries. End users install with one command; contributors can still build from source. + +Machine-readable `--version` JSON and latest-version discovery are documented in [`VERSION.md`](./VERSION.md) (ATO-1844). ```bash curl --proto '=https' --tlsv1.2 -fsSL https://get.atomicstrata.ai/install.sh | sh -s -- --init ``` @@ -39,7 +41,7 @@ am --help ``` This is the **CLI** (`am`): auth, org/project/key, connect, instance, -memory, migrate, doctor, integrate (MCP), and lifecycle hooks. +slm, memory, migrate, doctor, integrate (MCP), and lifecycle hooks. Consolidation of the npm `@atomicmemory/cli` package into `am` is **in progress**: `am` covers Cloud, memory, MCP integration, and lifecycle hooks, @@ -103,8 +105,55 @@ automation must be explicit and provide `OPENAI_API_KEY` when starting Core: ```bash am init --yes --project OPENAI_API_KEY=sk-... am init --local --yes +am init --local --slm --yes +``` + +`am init --local --slm` is the one-shot Connected Local SLM path: it creates a +local profile, installs `am-slm` if needed, pulls Qwen + Nomic + `am-slm-core` +(~1.7GB; `--yes` confirms), starts host Metal SLM, and starts Core Docker +without an OpenAI key. Apple Silicon macOS only. + +### Provider switching and recovery + +Use `--provider openai|slm` with `am init --local`, `am connect --project`, or +`am instance start`. `--slm` remains shorthand for `--provider slm`; combining +them is an error. Omitting both reuses the project's saved provider; new +installations use OpenAI. + +```bash +am --profile local instance start --provider slm --yes +am --profile local instance start --provider openai +am --profile local doctor --smoke ``` +Each Cloud origin, project, and provider has separate persistent data and Core +credentials. Switching prepares the target provider before replacing the one +active managed Core container. Switching back reopens its existing dataset; +memories are not copied or re-embedded. Recognized legacy storage is retained; +unidentified volumes are preserved without being assigned to a provider. + +Connected Local supports HTTP loopback addresses such as +`http://127.0.0.1:17352`. Core retains its internal port; the CLI uses the +configured host port for binding and health checks. Unsupported managed +addresses fail before startup. + +Model downloads report file names, bytes, totals when available, and cache +hits. After interruption, rerun the same command: completed cache entries stay +in place and readiness is checked again. Quiet mode suppresses progress; JSON +stdout contains machine-readable results. + +Init reports verification as `passed`, `failed`, `deliberately_skipped`, or +`not_run`. Attempted verification failures exit nonzero even when configuration +was saved. Follow the receipt's profile-specific recovery command. SLM +verification exercises full extraction and retrieval, then removes its test +memories. Cloud membership or scope errors must be corrected before retrying; +the CLI does not rotate credentials in response to those errors. + +`am instance remove` preserves datasets. `am instance remove --purge-data --yes` +deletes only the selected dataset after ownership checks. +`am instance start --provider slm --slm-reset-data --yes` explicitly resets the +selected SLM dataset. Neither command is needed for ordinary provider switching. + With no Cloud projects, interactive init opens onboarding and polls every two seconds for up to ten minutes, then resumes after project creation. Without a TTY or under `--yes`, it prints the onboarding URL and exact `am init` recovery @@ -141,6 +190,8 @@ shell export is ignored for init-managed profiles unless you set am auth login am link local --name local --local-url http://127.0.0.1:17350 am instance start +# SLM instead of OpenAI (Apple Silicon): +# am instance start --slm --yes ``` `am instance start` auto-provisions a Cloud `amc_` key when needed and injects a @@ -212,7 +263,8 @@ am --base-url https://api.staging.example.com \ | `memory` | Ingest (`--mode text\|messages\|verbatim`), search, **package**, list, get, delete | | `hooks` | Lifecycle hooks for Codex and Claude Code (complements `integrate` MCP) | | `connect`, `instance`, `link` | Connected Local + Docker Core | -| `integrate` | Install AtomicMemory MCP into Cursor, Claude Code, and Codex | +| `slm` | Host Metal SLM runtime (`am slm install|start|models pull`); `am init --local --slm` | +| `integrate` | Install AtomicMemory MCP into Cursor, Claude Code, Codex, and OpenCode | | `trace`, `usage`, `overview` | Observability | | `migrate` | Export/import local Core memories | | `doctor`, `health` | Diagnostics | @@ -227,7 +279,7 @@ v1): ```bash am integrate detect -am integrate --yes --global --host cursor --host claude-code +am integrate --yes --global --host cursor --host claude-code --host opencode am integrate doctor am integrate uninstall --host cursor ``` @@ -235,6 +287,10 @@ am integrate uninstall --host cursor Installs set `ATOMICMEMORY_SCOPE_LOCK=true` in the generated MCP server env and pin `@atomicmemory/mcp-server@0.1.5`. Project-scoped configs (for example `.cursor/mcp.json` in a repo) are not supported yet — use global install only. +OpenCode uses its V2 global `mcp.servers.atomicmemory` configuration under +`$XDG_CONFIG_HOME/opencode` or `~/.config/opencode`. +An OpenCode project-level `opencode.json` or `opencode.jsonc` takes precedence +over this global entry; project-scoped OpenCode installs are not supported yet. `--dry-run` prints planned writes without mutating host files. In non-interactive sessions, pass `--yes` and/or explicit `--host` before mutating configs. Interactive wizard progress and next-step hints go to stderr; human install diff --git a/crates/cli/VERSION.md b/crates/cli/VERSION.md new file mode 100644 index 0000000..a97fc1b --- /dev/null +++ b/crates/cli/VERSION.md @@ -0,0 +1,90 @@ +# CLI version contract (ATO-1844) + +Machine-readable version identity for the `am` CLI. The upgrade gate +([ATO-1843](https://linear.app/atomic-strata/issue/ATO-1843)) depends on this +shape; it does **not** live in this document’s scope (no throttle / force-upgrade +here). + +## `am --version` / `am -V` + +Prints one JSON object on stdout (exit 0), no `am ` prefix: + +```json +{"surface":"cli","version":"0.2.0","gitSha":null,"env":"dev"} +``` + +| Field | Meaning | +| --- | --- | +| `surface` | Always `"cli"` for this binary. | +| `version` | Crate / workspace semver (`CARGO_PKG_VERSION`). Same number as git tags `cli-v*`. Do not invent a second version. | +| `gitSha` | Commit SHA stamped at CI via `AM_GIT_SHA`. JSON `null` when unset (local/`cargo build`). Never a placeholder like `"unknown"`. | +| `env` | Build channel stamped via `AM_BUILD_ENV` (`dev` default; `production` / `internal` / `canary` in release lanes). | + +Shipped (CI) builds must have a non-empty `gitSha`. Local and unstamped +developer builds intentionally leave it `null`. + +## Latest-version discovery + +Clients (and ATO-1843) discover the newest published CLI from the install +mirror: + +```text +GET https://get.atomicstrata.ai/version.json +``` + +Internal eng builds use the floating internal release’s `version.json` (same +object shape). Document body: + +```json +{ + "surface": "cli", + "version": "0.2.0", + "gitSha": "abc…", + "env": "production", + "tag": "cli-v0.2.0" +} +``` + +`tag` is installer metadata; parsers for the upgrade gate may ignore it. +`version` / `gitSha` / `env` / `surface` match `--version`. + +Rust helper (no enforcement): `crate::version::fetch_latest_version(base_url)` in the CLI crate. + +## CI stamp + +Release workflows set compile-time env when building `am`: + +- `AM_GIT_SHA=` +- `AM_BUILD_ENV=production` (public), `internal` (main eng lane), or `canary` (dev eng lane) + +Override discovery base in tests with any HTTPS origin that serves `version.json`. + +## Public release bump rule + +Public releases use git tags `cli-vX.Y.Z` on the public product repository. The +workspace semver in root `Cargo.toml` must match the tag exactly. The next +public release must be an **adjacent** semver increment over the highest +existing `cli-v*` tag: + +| Last public | Allowed next | +| --- | --- | +| `A.B.C` | `A.B.(C+1)` (patch) | +| `A.B.C` | `A.(B+1).0` (minor) | +| `A.B.C` | `(A+1).0.0` (major) | + +Jumps (`0.2.0` → `0.2.5`), downgrades, and re-tagging the same version are +rejected. Internal/canary lanes may ship multiple builds at the same workspace +semver; only the public tag is monotonic. + +Enforcement: `scripts/ci/validate-cli-version-bump.sh` runs in +`release-cli.yml` (hard gate on tag push) and `ci-rust.yml` (PR early gate). +The four workspace dependency pins (`am-core-types`, `am-cloud-types`, +`am-cloud-client`, `atomicmemory`) and their `Cargo.lock` entries must stay +lockstep with `[workspace.package].version`. + +To cut a bump, do not edit those fields by hand. Refresh pins and the lockfile +together: + +```bash +pnpm run bump:cli-version -- 0.2.2 +``` diff --git a/crates/cli/clippy.toml b/crates/cli/clippy.toml new file mode 100644 index 0000000..aaa04b2 --- /dev/null +++ b/crates/cli/clippy.toml @@ -0,0 +1,4 @@ +disallowed-methods = [ + { path = "reqwest::Client::builder", reason = "Cloud and OAuth HTTP clients must use crate::auth::http" }, + { path = "reqwest::Client::new", reason = "Cloud and OAuth HTTP clients must use crate::auth::http" }, +] diff --git a/crates/cli/docs/DEMO-SLM.md b/crates/cli/docs/DEMO-SLM.md new file mode 100644 index 0000000..e518f5e --- /dev/null +++ b/crates/cli/docs/DEMO-SLM.md @@ -0,0 +1,91 @@ +# `am slm` demo (Apple Silicon / M5) — ATO-1936 + +**Apple Silicon macOS only.** No Polar. No `OPENAI_API_KEY` for SLM mode. + +Public contract: `https://get.atomicstrata.ai/am-slm/version.json` (**0.1.1**). + +## One-shot Connected Local (cold start) + +From a machine with no prior SLM install, this bootstraps the binary, pulls +Qwen + Nomic + `am-slm-core` (~1.7GB, requires `--yes` or a prompt), starts +host Metal SLM on `:8080`, and starts Core Docker with the SLM overlay: + +```bash +am init --local --slm --yes +``` + +## SLM-only path (install → status → start) + +```bash +# From repo root (workspace package name is `atomicmemory`, binary `am`): +cargo build --release -p atomicmemory +export AM=./target/release/am + +$AM slm install +$AM -o json slm status + +$AM slm start --yes +curl -fsS http://127.0.0.1:8080/health +curl -fsS http://127.0.0.1:8080/v1/models +# expect ids: am-slm-core + nomic-embed-text + +$AM slm stop +``` + +`am slm start` installs the runtime if missing and pulls the start-complete +model set when the cache is empty (`--yes` or a prompt). To pull without +starting: + +```bash +$AM slm models pull --yes # Qwen + Nomic + am-slm-core; never silent +$AM slm models status +``` + +Fail-closed on non-Apple-Silicon, checksum mismatch, or unreachable R2. + +## Connected Local Core (`--slm`) + +Starts host Metal SLM first, then Core with openai-compatible env at +`http://host.docker.internal:8080/v1` (`LLM_MODEL=am-slm-core`, +`EMBEDDING_MODEL=nomic-embed-text`, `EMBEDDING_DIMENSIONS=768`). No OpenAI key. + + +Default Connected Local extract contract (required for Playground Full ingest +against today's published Core image): + +- Host `am-slm serve`: `AM_SLM_CORE_JSON_SCHEMA=1` only (full extraction grammar) +- **Not** set: `AM_SLM_CORE_COMPACT_SCHEMA` or Core `EXTRACTION_PROMPT_VARIANT=compact` + (current `ghcr.io/atomicstrata/atomicmemory-core:latest` has no compact prompt — + compact deferred until that image ships) + +`am slm start` and `am instance start --slm` set `AM_SLM_CORE_JSON_SCHEMA=1` +automatically. If a CLI-managed SLM is already running without that flag, start +restarts it. An external (non-managed) listener without the flag fails loudly — +stop it and re-run managed start, or export `AM_SLM_CORE_JSON_SCHEMA=1` yourself. + +After pulling this branch onto a machine that already had `am-slm` running, +restart the host SLM once so the JSON-schema env applies (Core recreate alone +is not enough; a prior compact-schema process must be restarted): + +```bash +$AM slm stop && $AM slm start --yes + +# Existing local profile: +$AM instance start --slm --yes +# Migrating from an OpenAI / 1536-dim Core volume (explicit wipe only): +# $AM instance start --slm --slm-reset-data --yes +``` + +If an old Core is already up, `--slm` recreates the managed container so the +SLM overlay env applies; `--slm-reset-data` stops/removes Core first, then +wipes `atomic-memory-data` / `atomic-memory-state` volumes. + +Core default URL: `:17350` (`am instance status`). + +## Doctor / uninstall + +```bash +$AM doctor +$AM slm uninstall # keeps model cache +# $AM slm uninstall --purge-models +``` diff --git a/crates/cli/src/auth/device_login.rs b/crates/cli/src/auth/device_login.rs index fa2d8ef..567801a 100644 --- a/crates/cli/src/auth/device_login.rs +++ b/crates/cli/src/auth/device_login.rs @@ -7,6 +7,7 @@ use anyhow::{Context, Result, bail}; use reqwest::Url; use tokio::time::sleep; +use crate::auth::http; use crate::auth::login_feedback::LoginFeedback; use crate::auth::setup::setup_default_project; use crate::config::{OAuthTokens, load_config, store_oauth, store_profile_base_url}; @@ -32,9 +33,7 @@ pub async fn run_device_login( let feedback = LoginFeedback::detect(opts.verbose, opts.quiet); let step_id = progress_step.unwrap_or("identity"); let base = Url::parse(&opts.base_url).context("parse cloud base_url")?; - let http = reqwest::Client::builder() - .timeout(Duration::from_secs(30)) - .build()?; + let http = http::client()?; let authorize_url = base .join("api/oauth/device/authorize") @@ -144,8 +143,104 @@ pub async fn run_device_login( #[cfg(test)] mod tests { + use std::sync::{Arc, Mutex}; + + use axum::extract::State; + use axum::http::{HeaderMap, StatusCode, header}; + use axum::response::{IntoResponse, Response}; + use axum::routing::post; + use axum::{Json, Router}; + use super::*; + type RecordedUserAgent = (String, Option); + + #[derive(Clone, Default)] + struct SeenUserAgents(Arc>>); + + impl SeenUserAgents { + fn record(&self, path: &str, headers: &HeaderMap) { + let user_agent = headers + .get(header::USER_AGENT) + .and_then(|value| value.to_str().ok()) + .map(str::to_string); + self.0.lock().unwrap().push((path.into(), user_agent)); + } + } + + fn has_cli_user_agent(headers: &HeaderMap) -> bool { + headers + .get(header::USER_AGENT) + .is_some_and(|value| value == crate::auth::http::USER_AGENT) + } + + async fn authorize(State(seen): State, headers: HeaderMap) -> Response { + seen.record("authorize", &headers); + if !has_cli_user_agent(&headers) { + return StatusCode::FORBIDDEN.into_response(); + } + Json(serde_json::json!({ + "device_code": "device-code", + "user_code": "user-code", + "verification_uri": "https://example.com/activate", + "verification_uri_complete": "https://example.com/activate?code=user-code", + "expires_in": 600, + "interval": 1 + })) + .into_response() + } + + async fn token(State(seen): State, headers: HeaderMap) -> Response { + seen.record("token", &headers); + if !has_cli_user_agent(&headers) { + return StatusCode::FORBIDDEN.into_response(); + } + ( + StatusCode::BAD_REQUEST, + Json(serde_json::json!({"error": "expired_token"})), + ) + .into_response() + } + + #[tokio::test] + async fn device_requests_send_versioned_cli_user_agent() { + let seen = SeenUserAgents::default(); + let app = Router::new() + .route("/api/oauth/device/authorize", post(authorize)) + .route("/api/oauth/device/token", post(token)) + .with_state(seen.clone()); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + + let error = run_device_login( + DeviceLoginOptions { + profile: "test".into(), + base_url: format!("http://{address}"), + client_id: None, + quiet: true, + verbose: false, + }, + None, + None, + ) + .await + .expect_err("expired fixture must stop the polling loop"); + server.abort(); + + assert!(error.to_string().contains("device code expired")); + assert_eq!( + *seen.0.lock().unwrap(), + vec![ + ( + "authorize".into(), + Some(crate::auth::http::USER_AGENT.into()) + ), + ("token".into(), Some(crate::auth::http::USER_AGENT.into())), + ] + ); + } + #[test] fn device_login_feedback_is_concise_on_tty() { let fb = LoginFeedback::for_test(false, false, true); diff --git a/crates/cli/src/auth/doctor.rs b/crates/cli/src/auth/doctor.rs index 9a4a705..6bc0d2d 100644 --- a/crates/cli/src/auth/doctor.rs +++ b/crates/cli/src/auth/doctor.rs @@ -3,26 +3,18 @@ use std::time::Duration; use anyhow::{Context, Result, bail}; -use reqwest::Client; use serde::Serialize; use crate::auth::clerk_oauth::{resolve_oauth_pair, resolve_public_client_id}; +use crate::auth::http; use crate::auth::token::discover_metadata; use crate::config::{ DEFAULT_OAUTH_CALLBACK_PORT, ensure_config_initialized, load_config, resolve_profile, }; use crate::environment::is_production_api_url; -const OAUTH_HTTP_TIMEOUT: Duration = Duration::from_secs(30); const API_HEALTH_TIMEOUT: Duration = Duration::from_secs(10); -fn oauth_http_client() -> Result { - Client::builder() - .timeout(OAUTH_HTTP_TIMEOUT) - .build() - .context("build oauth http client") -} - /// Optional overrides for dev / custom Clerk instances (e.g. `auth login --issuer --client-id`). #[derive(Debug, Clone, Default)] pub struct DoctorOverrides { @@ -190,7 +182,7 @@ async fn probe_clerk_public_client( client_id: &str, redirect_uri: &str, ) -> Result> { - let client = oauth_http_client()?; + let client = http::client()?; let body: serde_json::Value = client .post(token_endpoint) .form(&[ @@ -230,7 +222,7 @@ fn classify_clerk_probe(error: Option<&str>) -> Option { async fn probe_api_health(base_url: &str) -> Result { let url = format!("{}/healthz", base_url.trim_end_matches('/')); - let client = oauth_http_client()?; + let client = http::client()?; let status = client .get(&url) .timeout(API_HEALTH_TIMEOUT) @@ -257,9 +249,56 @@ pub async fn require_login_ready( #[cfg(test)] mod tests { + use std::sync::{Arc, Mutex}; + + use axum::Router; + use axum::extract::State; + use axum::http::{HeaderMap, StatusCode, header}; + use axum::routing::get; + use super::*; use crate::config::DEFAULT_CLOUD_URL; + #[tokio::test] + async fn api_health_probe_sends_versioned_cli_user_agent() { + let seen_user_agent = Arc::new(Mutex::new(None::)); + let app = Router::new() + .route( + "/healthz", + get( + |State(seen): State>>>, headers: HeaderMap| async move { + let user_agent = headers + .get(header::USER_AGENT) + .and_then(|value| value.to_str().ok()) + .map(str::to_string); + *seen.lock().unwrap() = user_agent.clone(); + if user_agent.as_deref() + == Some(crate::auth::http::USER_AGENT) + { + StatusCode::OK + } else { + StatusCode::FORBIDDEN + } + }, + ), + ) + .with_state(seen_user_agent.clone()); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + + let healthy = probe_api_health(&format!("http://{address}")) + .await + .unwrap(); + server.abort(); + + assert!(healthy); + assert_eq!( + seen_user_agent.lock().unwrap().as_deref(), + Some(crate::auth::http::USER_AGENT) + ); + } + #[test] fn report_ok_requires_clerk_client() { let ok = DoctorReport { diff --git a/crates/cli/src/auth/ensure_org.rs b/crates/cli/src/auth/ensure_org.rs index d709c2a..665d010 100644 --- a/crates/cli/src/auth/ensure_org.rs +++ b/crates/cli/src/auth/ensure_org.rs @@ -126,6 +126,9 @@ fn map_org_error(err: CloudClientError) -> anyhow::Error { CloudClientError::Auth => anyhow::anyhow!( "authentication failed — run `am auth login` or `am init` to refresh your session" ), + CloudClientError::Forbidden { code } => anyhow::anyhow!( + "Cloud forbade this org request ({code}). Refreshing login does not change your org role." + ), other => anyhow::anyhow!("{other}"), } } @@ -151,4 +154,14 @@ mod tests { let picked = pick_org(&orgs, true).await.unwrap().unwrap(); assert_eq!(picked.slug, "solo"); } + + #[test] + fn forbidden_org_error_does_not_recommend_login() { + let err = map_org_error(CloudClientError::Forbidden { + code: "forbidden".into(), + }); + let msg = err.to_string(); + assert!(msg.contains("does not change your org role")); + assert!(!msg.contains("am auth login")); + } } diff --git a/crates/cli/src/auth/http.rs b/crates/cli/src/auth/http.rs new file mode 100644 index 0000000..191c50c --- /dev/null +++ b/crates/cli/src/auth/http.rs @@ -0,0 +1,125 @@ +//! Shared HTTP client construction for CLI authentication requests. + +use std::time::Duration; + +use anyhow::{Context, Result}; + +const AUTH_HTTP_TIMEOUT: Duration = Duration::from_secs(30); +pub(crate) const USER_AGENT: &str = concat!("am/", env!("CARGO_PKG_VERSION")); + +/// Build an authentication client identified by the CLI name and version. +/// +/// The production Cloud edge rejects requests without a user agent, so all +/// authentication requests must use this constructor rather than a raw client. +pub(crate) fn client() -> Result { + client_with_timeout(AUTH_HTTP_TIMEOUT).context("build authentication HTTP client") +} + +/// Build an identified authentication client with a caller-specific timeout. +pub(crate) fn client_with_timeout(timeout: Duration) -> reqwest::Result { + reqwest::Client::builder() + .user_agent(USER_AGENT) + .timeout(timeout) + .build() +} + +#[cfg(test)] +mod tests { + const AUTH_SOURCES: &[(&str, &str)] = &[ + ("auth_wait.rs", include_str!("auth_wait.rs")), + ("claims.rs", include_str!("claims.rs")), + ("clerk_oauth.rs", include_str!("clerk_oauth.rs")), + ("device_login.rs", include_str!("device_login.rs")), + ("doctor.rs", include_str!("doctor.rs")), + ("ensure_org.rs", include_str!("ensure_org.rs")), + ("login.rs", include_str!("login.rs")), + ("login_feedback.rs", include_str!("login_feedback.rs")), + ("mod.rs", include_str!("mod.rs")), + ("origin.rs", include_str!("origin.rs")), + ("pkce.rs", include_str!("pkce.rs")), + ("setup.rs", include_str!("setup.rs")), + ("token.rs", include_str!("token.rs")), + ("token_login.rs", include_str!("token_login.rs")), + ]; + + fn invokes_constructor(compact: &str, type_name: &str) -> bool { + ["builder", "new"].iter().any(|method| { + let needle = format!("{type_name}::{method}("); + compact.match_indices(&needle).any(|(offset, _)| { + compact[..offset] + .chars() + .next_back() + .is_none_or(|prefix| !prefix.is_alphanumeric() && prefix != '_') + }) + }) + } + + fn constructs_raw_client(source: &str) -> bool { + let compact: String = source + .chars() + .filter(|character| !character.is_whitespace()) + .collect(); + if invokes_constructor(&compact, "Client") { + return true; + } + + compact.split(';').any(|statement| { + if !statement.starts_with("usereqwest::") { + return false; + } + statement.find("Clientas").is_some_and(|offset| { + let alias = statement[offset + "Clientas".len()..] + .chars() + .take_while(|character| character.is_alphanumeric() || *character == '_') + .collect::(); + !alias.is_empty() && invokes_constructor(&compact, &alias) + }) + }) + } + + #[test] + fn cloud_auth_callers_cannot_bypass_the_shared_http_client() { + let declared_modules: Vec<&str> = include_str!("mod.rs") + .lines() + .filter_map(|line| { + line.trim() + .strip_prefix("pub mod ") + .and_then(|module| module.strip_suffix(';')) + }) + .filter(|module| *module != "http") + .collect(); + let scanned_modules: Vec<&str> = AUTH_SOURCES + .iter() + .filter_map(|(path, _)| path.strip_suffix(".rs")) + .filter(|module| *module != "mod") + .collect(); + + assert_eq!( + scanned_modules, declared_modules, + "every auth module must be included in the raw-client source scan", + ); + + for (path, source) in AUTH_SOURCES.iter().copied().chain([( + "commands/connect.rs", + include_str!("../commands/connect.rs"), + )]) { + assert!( + !constructs_raw_client(source), + "{path} must use auth::http rather than construct a raw reqwest client", + ); + } + } + + #[test] + fn raw_client_scan_rejects_spaced_rust_syntax() { + assert!(constructs_raw_client("reqwest::Client :: builder ()")); + assert!(constructs_raw_client("Client :: new ()")); + assert!(constructs_raw_client( + "use reqwest::Client as HttpClient; HttpClient :: new()" + )); + assert!(constructs_raw_client( + "use reqwest::{Client as C}; C :: builder()" + )); + assert!(!constructs_raw_client("DashboardClient :: new ()")); + } +} diff --git a/crates/cli/src/auth/mod.rs b/crates/cli/src/auth/mod.rs index aa74f4c..9097906 100644 --- a/crates/cli/src/auth/mod.rs +++ b/crates/cli/src/auth/mod.rs @@ -1,11 +1,15 @@ //! Authentication: OAuth login, device flow, token storage, and diagnostics. +#![deny(clippy::disallowed_methods)] + pub mod auth_wait; pub mod claims; pub mod clerk_oauth; pub mod device_login; pub mod doctor; pub mod ensure_org; +#[allow(clippy::disallowed_methods)] +pub mod http; pub mod login; pub mod login_feedback; pub mod origin; diff --git a/crates/cli/src/auth/setup.rs b/crates/cli/src/auth/setup.rs index de2fe55..1bc1269 100644 --- a/crates/cli/src/auth/setup.rs +++ b/crates/cli/src/auth/setup.rs @@ -47,6 +47,9 @@ pub async fn setup_default_project( am_cloud_client::CloudClientError::Auth => { anyhow::anyhow!("list projects: {e}\n{}", missing_org_login_hint()) } + am_cloud_client::CloudClientError::Forbidden { code } => anyhow::anyhow!( + "list projects: Cloud forbade this request ({code}). Refreshing login does not change your org role." + ), other => anyhow::anyhow!("list projects: {other}"), })?; let Some(project) = pick_project(&projects, interactive)? else { diff --git a/crates/cli/src/auth/token.rs b/crates/cli/src/auth/token.rs index f076e85..934082a 100644 --- a/crates/cli/src/auth/token.rs +++ b/crates/cli/src/auth/token.rs @@ -1,29 +1,18 @@ //! OAuth token refresh and bearer resolution. -use std::time::Duration; - use anyhow::{Context, Result, anyhow, bail}; use chrono::Utc; -use reqwest::Client; use reqwest::Url; use serde::Deserialize; use crate::auth::claims::decode_id_token; use crate::auth::clerk_oauth::resolve_oauth_pair; +use crate::auth::http; use crate::auth::origin::check_token_origin; use crate::config::{ ConfigFile, CredentialsFile, OAuthTokens, load_config, load_credentials, update_credentials, }; -const OAUTH_HTTP_TIMEOUT: Duration = Duration::from_secs(30); - -fn oauth_http_client() -> Result { - Client::builder() - .timeout(OAUTH_HTTP_TIMEOUT) - .build() - .context("build oauth http client") -} - #[derive(Debug, Deserialize)] pub struct OAuthMetadata { pub authorization_endpoint: String, @@ -41,7 +30,7 @@ struct TokenResponse { pub async fn discover_metadata(issuer: &str) -> Result { let base = issuer.trim_end_matches('/'); let url = format!("{base}/.well-known/oauth-authorization-server"); - let client = oauth_http_client()?; + let client = http::client()?; let meta: OAuthMetadata = client .get(&url) .send() @@ -62,7 +51,7 @@ pub async fn exchange_code( redirect_uri: &str, verifier: &str, ) -> Result { - let client = oauth_http_client()?; + let client = http::client()?; let resp: TokenResponse = client .post(token_endpoint) .form(&[ @@ -88,7 +77,7 @@ pub async fn refresh_tokens( client_id: &str, refresh_token: &str, ) -> Result { - let client = oauth_http_client()?; + let client = http::client()?; let resp: TokenResponse = client .post(token_endpoint) .form(&[ diff --git a/crates/cli/src/cli.rs b/crates/cli/src/cli.rs index 0478fd3..9b3b206 100644 --- a/crates/cli/src/cli.rs +++ b/crates/cli/src/cli.rs @@ -4,7 +4,7 @@ use clap::{Parser, Subcommand, ValueEnum}; use crate::commands::{ auth, config_cmd, connect, doctor_cmd, hooks, init, instance, integrate, key, link, memory, - migrate, org, project, trace, usage, + migrate, org, project, slm, trace, usage, }; use crate::environment::Environment; @@ -19,7 +19,10 @@ pub enum OutputFormat { #[derive(Debug, Parser)] #[command( name = "am", - version, + // Pure JSON `--version` is handled in `main` (clap would prefix `am `). + // Contract: crates/cli/VERSION.md (ATO-1844). + disable_version_flag = true, + after_help = "Version:\n am --version Print machine-readable version JSON (crates/cli/VERSION.md)", about = "AtomicMemory CLI", long_about = "Manage hosted AtomicMemory Cloud instances, link local deployments, and run memory operations." )] @@ -138,10 +141,13 @@ pub enum Command { /// Start/stop the local Core Docker container #[command(subcommand)] Instance(instance::InstanceCommand), + /// Install and manage the host Metal SLM runtime (Connected Local) + #[command(subcommand)] + Slm(slm::SlmCommand), /// Export/import local memories to Cloud #[command(subcommand)] Migrate(migrate::MigrateCommand), - /// Install AtomicMemory MCP into agent hosts (Cursor, Claude Code, Codex) + /// Install AtomicMemory MCP into agent hosts (Cursor, Claude Code, Codex, OpenCode) Integrate(integrate::IntegrateOptions), /// Lifecycle hooks for Codex and Claude Code (complements `am integrate` MCP) #[command(subcommand)] @@ -165,6 +171,7 @@ pub fn command_path(command: &Command) -> String { Command::Link(_) => "link".into(), Command::Connect(_) => "connect".into(), Command::Instance(_) => "instance".into(), + Command::Slm(_) => "slm".into(), Command::Migrate(_) => "migrate".into(), Command::Integrate(_) => "integrate".into(), Command::Hooks(cmd) => format!("hooks {}", hooks::command_label(cmd)), @@ -193,4 +200,21 @@ mod tests { global.agent = true; assert!(!global.allow_prompts(false)); } + #[test] + fn provider_selection_is_available_on_all_onboarding_commands() { + for args in [ + vec!["am", "init", "--local", "--provider", "slm", "--yes"], + vec![ + "am", + "connect", + "--project", + "proj_test", + "--provider", + "openai", + ], + vec!["am", "instance", "start", "--provider", "openai"], + ] { + assert!(Cli::try_parse_from(args.clone()).is_ok(), "{args:?}"); + } + } } diff --git a/crates/cli/src/commands/client.rs b/crates/cli/src/commands/client.rs index 67b2be4..347f833 100644 --- a/crates/cli/src/commands/client.rs +++ b/crates/cli/src/commands/client.rs @@ -53,17 +53,23 @@ pub fn emit_cloud_export_warning_if_needed( stored_kind: crate::config::ProfileKind, profile: &ResolvedProfile, ) { + let config = load_config().ok(); + let stored_profile_base_url = config + .as_ref() + .and_then(|file| file.profiles.get(&profile.name)) + .and_then(|entry| entry.base_url.as_deref()); if let Some(warning) = local_profile_cloud_export_warning( stored_kind, global.base_url.as_deref(), &profile.base_url, std::env::var(ENV_API_KEY).ok().as_deref(), &profile.memory_base_url, + stored_profile_base_url, ) { message(!global.quiet, &warning); } - if let (Ok(config), Ok(creds)) = (load_config(), load_credentials()) { + if let (Some(config), Ok(creds)) = (config, load_credentials()) { let api_key_ref = config .profiles .get(&profile.name) @@ -172,11 +178,13 @@ pub(crate) async fn memory_client_for_profile(profile: &ResolvedProfile) -> Resu // Prefer the managed container's persisted CORE_API_KEY over a Cloud-minted // JWT. Core rejects JWT for smoke / some local namespaces; reading the key // from state keeps ingest/search working without a shell override. - if let Some(core_key) = - crate::instance::read_managed_core_api_key(&profile.name, &profile.memory_base_url) - .await - { - return MemoryClient::new(base, core_key).context("create core memory client"); + if let Some(core_key) = crate::instance::read_managed_core_api_key(profile).await? { + let managed_base = + crate::instance::address::ManagedAddress::parse(&profile.memory_base_url)? + .url() + .parse()?; + return MemoryClient::new(managed_base, core_key) + .context("create core memory client"); } // Pinned: cloud_api_key_client would resolve the active profile // again, so a "pinned" caller silently minted a token for whatever diff --git a/crates/cli/src/commands/cloud_api_key.rs b/crates/cli/src/commands/cloud_api_key.rs index 6ba8e51..2ada29e 100644 --- a/crates/cli/src/commands/cloud_api_key.rs +++ b/crates/cli/src/commands/cloud_api_key.rs @@ -140,6 +140,22 @@ pub fn should_rotate_after_probe(err: &CloudClientError) -> bool { matches!(err, CloudClientError::Auth) } +// Both onboarding entry points must use the same fail-closed reuse gate. +fn stored_key_probe_allows_reuse( + result: Result<(), CloudClientError>, + resolved_project: Option<&str>, + requested_project: &str, +) -> Result { + match result { + Ok(()) => Ok(resolved_project == Some(requested_project)), + Err(err) if should_rotate_after_probe(&err) => Ok(false), + Err(err) => Err(err).context( + "verify Cloud API key (POST /v1/local/token): stored key preserved; \ + verification failed, so setup cannot continue", + ), + } +} + pub(crate) fn is_api_key_quota_exceeded(err: &CloudClientError) -> bool { match err { CloudClientError::Status { code, body } => { @@ -160,29 +176,20 @@ pub async fn ensure_connected_local_cloud_api_key( global: &GlobalOptions, profile: &ResolvedProfile, ) -> Result<(String, ProvisionOutcome)> { + let project_id = require_project_id(profile, None)?; if let Ok(key) = require_api_key(profile) && is_cloud_api_key(&key) { - match probe_cloud_api_key_mint(&profile.base_url, &key).await { - Ok(()) => return Ok((key, ProvisionOutcome::Reused)), - Err(err) if should_rotate_after_probe(&err) => { - info!( - profile = %profile.name, - "stored Cloud API key rejected; rotating or creating connected-local-runtime" - ); - } - Err(err) => { - tracing::warn!( - error = %err, - base_url = %profile.base_url, - "could not verify Cloud API key against tier; continuing with stored key" - ); - return Ok((key, ProvisionOutcome::Reused)); - } + if stored_key_probe_allows_reuse( + probe_cloud_api_key_mint(&profile.base_url, &key).await, + profile.project_id.as_deref(), + &project_id, + )? { + return Ok((key, ProvisionOutcome::Reused)); } + info!("stored Cloud API key rejected; provisioning this installation's credential"); } - let project_id = require_project_id(profile, None)?; let (mint_profile, client) = dashboard_client(global).await?; if !same_origin(&mint_profile.base_url, &profile.base_url) { bail!( @@ -204,7 +211,9 @@ pub async fn ensure_connected_local_cloud_api_key( |secret| store_api_key(&profile.name, secret, &profile.base_url, &project_id), ) .await?; - probe_cloud_api_key_mint(&profile.base_url, &secret).await?; + probe_cloud_api_key_mint(&profile.base_url, &secret) + .await + .context("verify newly provisioned Cloud API key (POST /v1/local/token)")?; if let Some(msg) = outcome.operator_message() { message(!global.quiet, &msg); } @@ -219,37 +228,15 @@ pub async fn ensure_connected_local_cloud_api_key_stored( project_id: &str, ) -> Result { if let Ok((resolved, client)) = cloud_api_key_client(global).await { - // The client resolves the ACTIVE profile, which need not be the profile - // whose project this call is provisioning for. A key that mints happily - // for the active profile's project is still the wrong key for this one, - // so reuse requires the projects to agree. Selection already refuses a - // key whose stored project does not match its own profile; this closes - // the remaining gap between "the resolved profile" and "the requested - // project". - let same_project = resolved.project_id.as_deref() == Some(project_id); - match client.mint_local_token().await { - Ok(_) if same_project => return Ok(ProvisionOutcome::Reused), - Ok(_) => { - info!( - profile = %profile_name, - "stored Cloud API key belongs to a different project; provisioning one for this project" - ); - } - Err(err) if should_rotate_after_probe(&err) => { - info!( - profile = %profile_name, - "stored Cloud API key rejected; rotating or creating connected-local-runtime" - ); - } - Err(err) => { - tracing::warn!( - error = %err, - profile = %profile_name, - "Cloud API key probe failed; preserving stored key (not rotating)" - ); - return Ok(ProvisionOutcome::Reused); - } + // The active profile can differ from the project being provisioned. + if stored_key_probe_allows_reuse( + client.mint_local_token().await.map(|_| ()), + resolved.project_id.as_deref(), + project_id, + )? { + return Ok(ProvisionOutcome::Reused); } + info!("stored Cloud API key cannot be reused for the requested project"); } let (profile, client) = dashboard_client(global).await?; @@ -350,7 +337,23 @@ where Then re-run init — the CLI will rotate only this installation's '{key_name}' key when present." )) } - Err(err) => Err(err).context(format!("create Cloud API key '{key_name}' on {api_origin}")), + Err(err) => { + let context = cloud_key_create_context(&err, key_name, api_origin); + Err(err).context(context) + } + } +} + +fn cloud_key_create_context(err: &CloudClientError, key_name: &str, api_origin: &str) -> String { + match err { + CloudClientError::Auth => format!( + "create Cloud API key '{key_name}' on {api_origin}: authentication failed — run `am auth login` and retry" + ), + CloudClientError::Forbidden { code } => format!( + "create Cloud API key '{key_name}' on {api_origin}: Cloud rejected key creation \ + (403; {code}). Check project access and permissions before retrying." + ), + _ => format!("create Cloud API key '{key_name}' on {api_origin}"), } } @@ -645,8 +648,43 @@ mod tests { } #[test] - fn should_rotate_after_probe_only_on_auth() { + fn create_key_auth_error_tells_operator_to_login() { + let msg = cloud_key_create_context( + &CloudClientError::Auth, + TEST_LOCAL_KEY_NAME, + "https://api.atomicstrata.ai/", + ); + assert!(msg.contains("am auth login")); + assert!(msg.contains(TEST_LOCAL_KEY_NAME)); + let other = cloud_key_create_context( + &CloudClientError::Timeout, + TEST_LOCAL_KEY_NAME, + "https://api.atomicstrata.ai/", + ); + assert!(!other.contains("am auth login")); + } + + #[test] + fn create_key_forbidden_preserves_reason_without_guessing_membership() { + let msg = cloud_key_create_context( + &CloudClientError::Forbidden { + code: "forbidden".into(), + }, + TEST_LOCAL_KEY_NAME, + "https://api.atomicstrata.ai/", + ); + assert!(msg.contains("forbidden")); + assert!(msg.contains(TEST_LOCAL_KEY_NAME)); + assert!(!msg.contains("need org admin")); + assert!(!msg.contains("--token")); + } + + #[test] + fn should_rotate_after_probe_only_on_invalid_credentials() { assert!(should_rotate_after_probe(&CloudClientError::Auth)); + assert!(!should_rotate_after_probe(&CloudClientError::Forbidden { + code: "forbidden".into(), + })); assert!(!should_rotate_after_probe(&CloudClientError::Timeout)); assert!(!should_rotate_after_probe(&CloudClientError::Network( "dns".into() @@ -657,6 +695,99 @@ mod tests { })); } + #[tokio::test] + async fn reuse_requires_a_selected_project_before_probing() { + let profile = ResolvedProfile { + name: "probe-test".into(), + base_url: "not a URL".into(), + kind: crate::config::ProfileKind::Cloud, + project_id: None, + memory_base_url: "not a URL".into(), + api_key: Some("amc_stored_secret".into()), + oauth: None, + }; + let error = ensure_connected_local_cloud_api_key(&GlobalOptions::default(), &profile) + .await + .expect_err("reuse needs a project binding"); + assert!(format!("{error:#}").contains("missing project")); + } + + #[tokio::test] + async fn inconclusive_probe_fails_closed_and_preserves_the_stored_key() { + let app = axum::Router::new().route( + "/v1/local/token", + axum::routing::post(|| async { + ( + axum::http::StatusCode::UNPROCESSABLE_ENTITY, + "expected an object", + ) + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + let profile = ResolvedProfile { + name: "probe-test".into(), + base_url: format!("http://{address}"), + kind: crate::config::ProfileKind::Cloud, + project_id: Some("proj_test".into()), + memory_base_url: format!("http://{address}"), + api_key: Some("amc_stored_secret".into()), + oauth: None, + }; + let result = + ensure_connected_local_cloud_api_key(&GlobalOptions::default(), &profile).await; + server.abort(); + let error = result.expect_err("an inconclusive mint must never be called verified reuse"); + assert!(format!("{error:#}").contains("stored key preserved")); + assert_eq!(profile.api_key.as_deref(), Some("amc_stored_secret")); + } + + #[test] + fn reuse_gate_rejects_uncertain_probes_even_for_a_different_project() { + for project in [Some("proj_test"), Some("proj_other"), None] { + for error in [ + CloudClientError::NoActiveOrganization, + CloudClientError::Forbidden { + code: "membership_required".into(), + }, + CloudClientError::Forbidden { + code: "insufficient_scope".into(), + }, + CloudClientError::Forbidden { + code: "forbidden".into(), + }, + CloudClientError::Status { + code: 422, + body: "validation failed".into(), + }, + CloudClientError::Status { + code: 429, + body: "rate limited".into(), + }, + CloudClientError::Status { + code: 503, + body: "unavailable".into(), + }, + CloudClientError::Network("offline".into()), + CloudClientError::Timeout, + ] { + assert!(stored_key_probe_allows_reuse(Err(error), project, "proj_test").is_err()); + } + } + assert!(stored_key_probe_allows_reuse(Ok(()), Some("proj_test"), "proj_test").unwrap()); + assert!(!stored_key_probe_allows_reuse(Ok(()), Some("proj_other"), "proj_test").unwrap()); + assert!(!stored_key_probe_allows_reuse(Ok(()), None, "proj_test").unwrap()); + assert!( + !stored_key_probe_allows_reuse( + Err(CloudClientError::Auth), + Some("proj_test"), + "proj_test" + ) + .unwrap() + ); + } + #[test] fn quota_exceeded_detector_matches_cloud_body() { let err = CloudClientError::Status { diff --git a/crates/cli/src/commands/connect.rs b/crates/cli/src/commands/connect.rs index d318c2f..c239119 100644 --- a/crates/cli/src/commands/connect.rs +++ b/crates/cli/src/commands/connect.rs @@ -24,6 +24,7 @@ use crate::instance::{ use crate::output::{emit, message}; const RECENT_TRACE_WINDOW: Duration = Duration::from_secs(15 * 60); +const JWKS_HTTP_TIMEOUT: Duration = Duration::from_secs(10); /// Which environment block `connect env` emits. #[derive(Debug, Clone, Copy, Default, ValueEnum, PartialEq, Eq)] @@ -46,6 +47,15 @@ pub struct ConnectOptions { /// Authenticate via OAuth device flow instead of browser login #[arg(long)] pub device: bool, + /// Select the Connected Local provider. + #[arg(long, value_enum, conflicts_with = "slm", requires = "project")] + pub provider: Option, + /// Use Connected Local SLM without an OpenAI key. + #[arg(long, requires = "project")] + pub slm: bool, + /// Accept defaults and model download without prompting. + #[arg(long)] + pub yes: bool, /// Skip starting Core when using `--project` #[arg(long)] pub no_instance: bool, @@ -97,7 +107,10 @@ pub async fn run(opts: ConnectOptions, global: &GlobalOptions) -> Result<()> { skip_verify: opts.skip_verify, replace: opts.replace, instance_image: None, - interactive: !global.quiet, + interactive: global.allow_prompts(opts.yes), + slm: opts.slm, + provider: opts.provider, + slm_pull_yes: opts.yes, }; return run_connect_project(&project, opts.device, &connect_opts, global).await; } @@ -215,12 +228,11 @@ async fn resolve_client_key_for_env( profile: &crate::config::ResolvedProfile, _show_secrets: bool, ) -> Result { - let docker = RealDockerRunner::new(); - let state_key = read_managed_core_key(&docker, &profile.name, &profile.memory_base_url).await?; - if let Some(key) = state_key { + if let Some(key) = resolve_core_api_key() { return Ok(key); } - if let Some(key) = resolve_core_api_key() { + let docker = RealDockerRunner::new(); + if let Some(key) = read_managed_core_key(&docker, profile).await? { return Ok(key); } bail!( @@ -230,10 +242,9 @@ async fn resolve_client_key_for_env( async fn read_managed_core_key( docker: &dyn DockerRunner, - profile_name: &str, - destination_url: &str, + profile: &crate::config::ResolvedProfile, ) -> Result> { - read_managed_core_api_key_with(docker, profile_name, destination_url).await + read_managed_core_api_key_with(docker, profile).await } async fn run_doctor(global: &GlobalOptions) -> Result<()> { @@ -437,12 +448,7 @@ async fn check_jwks_reachable(cloud_base_url: &str) -> DoctorCheck { }; } }; - let client = match reqwest::Client::builder() - // Cloud/CDN returns 403 when User-Agent is missing (bare reqwest default). - .user_agent(concat!("am/", env!("CARGO_PKG_VERSION"))) - .timeout(Duration::from_secs(10)) - .build() - { + let client = match crate::auth::http::client_with_timeout(JWKS_HTTP_TIMEOUT) { Ok(client) => client, Err(err) => { return DoctorCheck { @@ -513,9 +519,7 @@ async fn check_local_client_auth( docker: &dyn DockerRunner, quiet: bool, ) -> DoctorCheck { - let state_key = read_managed_core_key(docker, &profile.name, &profile.memory_base_url) - .await - .unwrap_or(None); + let state_key = read_managed_core_key(docker, profile).await.unwrap_or(None); let info = resolve_local_clients(&profile.memory_base_url, state_key.as_deref(), !quiet); match info.provenance { diff --git a/crates/cli/src/commands/connect_project.rs b/crates/cli/src/commands/connect_project.rs index b88651b..d599996 100644 --- a/crates/cli/src/commands/connect_project.rs +++ b/crates/cli/src/commands/connect_project.rs @@ -25,14 +25,19 @@ use crate::instance::docker::{ DockerRunner, RealDockerRunner, ensure_docker_available_with_preflight, }; use crate::instance::managed_core_needs_env_sync; -use crate::onboarding_runtime::{default_runtime_wait, wait_runtime_online_with_progress}; +use crate::onboarding_runtime::{ + default_runtime_wait, managed_core_instance_id, wait_runtime_online_with_progress, +}; use crate::output::message; use crate::progress::{ProgressReporter, progress_for}; use crate::telemetry::{ ActivationContext, ActivationEvent, InitStep, capture_activation, capture_email_hash, capture_step_failure, }; -use crate::verification::receipt::{InitReceiptInput, build_init_receipt, print_init_receipt}; +use crate::verification::receipt::{ + InitReceiptInput, VerificationAttempt, VerificationStatus, build_init_receipt, + print_init_receipt, +}; use crate::verification::smoke::{SmokeOptions, SmokeTelemetry, run_memory_smoke}; #[derive(Debug, Clone)] @@ -43,6 +48,11 @@ pub struct ConnectProjectOptions { pub instance_image: Option, /// When false, stdin prompts (Docker preflight retry, etc.) are skipped. pub interactive: bool, + /// Start Core with Connected Local SLM (host Metal runtime). + pub slm: bool, + pub provider: Option, + /// Confirm SLM model download without prompting (`am init --yes`). + pub slm_pull_yes: bool, } impl Default for ConnectProjectOptions { @@ -53,6 +63,9 @@ impl Default for ConnectProjectOptions { replace: false, instance_image: None, interactive: true, + slm: false, + provider: None, + slm_pull_yes: false, } } } @@ -77,7 +90,8 @@ async fn run_with_progress( global: &GlobalOptions, progress: &mut dyn ProgressReporter, ) -> Result<()> { - let ctx = authenticate_and_bootstrap_org(use_device, global, progress).await?; + let ctx = + authenticate_and_bootstrap_org(use_device, opts.interactive, global, progress).await?; let mut cloud_global = global.clone(); cloud_global.profile = Some(ctx.cloud_profile.clone()); cloud_global.base_url = Some(ctx.cloud_api_url.clone()); @@ -120,11 +134,21 @@ pub async fn connect_local_project( ); } + let mut effective_opts = opts.clone(); + let opts = &mut effective_opts; progress.start_step("project", "Link local project"); let local_url = project .local_url .clone() .unwrap_or_else(|| "http://127.0.0.1:17350".to_string()); + if !opts.no_instance { + crate::instance::preflight_managed( + Some(&local_url), + crate::instance::storage::Provider::requested(opts.provider, opts.slm)?, + opts.interactive, + ) + .await?; + } let (profile_name, profile_relinked) = match ensure_local_profile(&project, &cloud_profile, cloud_api_url.as_str(), &local_url) { Ok(v) => v, @@ -154,6 +178,31 @@ pub async fn connect_local_project( local_global.profile = Some(profile_name.clone()); local_global.base_url = Some(cloud_api_url.clone()); + let profile = resolve_profile( + Some(&profile_name), + Some(&cloud_api_url), + global.environment, + )?; + let local_url = if opts.no_instance { + profile.memory_base_url.clone() + } else { + crate::instance::address::ManagedAddress::parse(&profile.memory_base_url)?.url() + }; + if !opts.no_instance { + let observed = RealDockerRunner::new() + .inspect(crate::instance::DEFAULT_CONTAINER_NAME) + .await?; + let provider = crate::instance::storage::RuntimeStore::open()?.provider( + &profile, + crate::instance::storage::Provider::requested(opts.provider, opts.slm)?, + observed.as_ref(), + )?; + crate::instance::storage::preflight_storage(&profile, provider, opts.replace).await?; + opts.provider = Some(provider); + opts.slm = false; + crate::instance::preflight_managed(Some(&local_url), Some(provider), opts.interactive) + .await?; + } progress.start_step("credential", "Cloud API key"); let cloud_key_outcome = match ensure_connected_local_cloud_api_key_stored( &local_global, @@ -235,8 +284,10 @@ pub struct OnboardingContext { pub signed_in_as: Option, } +/// Establish an OAuth session and organization within the command prompt policy. pub async fn authenticate_and_bootstrap_org( use_device: bool, + allow_prompts: bool, global: &GlobalOptions, progress: &mut dyn ProgressReporter, ) -> Result { @@ -262,10 +313,10 @@ pub async fn authenticate_and_bootstrap_org( &cloud_profile, cloud_api_url.as_str(), use_device, + allow_prompts, global, progress, &mut actx, - global.no_telemetry, ) .await { @@ -293,7 +344,7 @@ pub async fn authenticate_and_bootstrap_org( let org = match ensure_org_context( &cloud_profile, None, - !global.quiet, + allow_prompts, Some(cloud_api_url.as_str()), EnsureOrgOptions { skip_default_project: true, @@ -609,7 +660,7 @@ async fn start_core_with_env_sync( progress.resume_after_input(); } if let Err(err) = docker_result { - progress.fail("runtime", Some(&err.to_string())); + progress.fail("runtime", Some("Core setup failed; see recovery below")); capture_step_failure(InitStep::Docker, &err, Some(actx.props()), no_telemetry); return Err(err); } @@ -628,11 +679,6 @@ async fn start_core_with_env_sync( ) .await?; let running = core_reachable(local_global).await; - if running && !needs_env_sync && !env_sync.cloud_key_changed { - progress.succeed("runtime", Some("already running")); - return Ok(true); - } - if (needs_env_sync || env_sync.cloud_key_changed) && running { progress.tick("runtime", "recreating with Cloud trace sync"); } else if !running { @@ -644,6 +690,10 @@ async fn start_core_with_env_sync( InstanceCommand::Start { image: opts.instance_image.clone(), openai_api_key: None, + slm: opts.slm, + provider: opts.provider, + slm_reset_data: false, + yes: opts.slm_pull_yes, // Operator authority ONLY. The internal recreate requirement is // passed separately below: `replace` is read downstream as consent // to force-remove a container this CLI does not manage, so a @@ -682,7 +732,7 @@ async fn start_core_with_env_sync( Ok(started) } Err(err) => { - progress.fail("runtime", Some(&err.to_string())); + progress.fail("runtime", Some("Core setup failed; see recovery below")); capture_step_failure(InitStep::CoreStart, &err, Some(actx.props()), no_telemetry); Err(err) } @@ -706,13 +756,22 @@ async fn finish_onboarding(input: FinishOnboardingInput<'_>) -> Result<()> { } = input; let cloud_connection_online = if !opts.no_instance && core_healthy { progress.start_step("heartbeat", "Wait for Cloud runtime online"); - let online = wait_runtime_online_with_progress( - local_global, - &project.id, - default_runtime_wait(), - Some(progress), - ) - .await; + let online = match managed_core_instance_id(local_global).await { + Ok(instance_id) => { + wait_runtime_online_with_progress( + local_global, + &project.id, + &instance_id, + default_runtime_wait(), + Some(progress), + ) + .await + } + Err(error) => { + progress.tick("heartbeat", &error.to_string()); + false + } + }; if online { capture_activation( ActivationEvent::HeartbeatReceived, @@ -727,7 +786,10 @@ async fn finish_onboarding(input: FinishOnboardingInput<'_>) -> Result<()> { Some(actx.props()), global.no_telemetry, ); - progress.warn("heartbeat", Some("timed out")); + progress.warn( + "heartbeat", + Some("this Core runtime is not verified online"), + ); } online } else { @@ -739,9 +801,14 @@ async fn finish_onboarding(input: FinishOnboardingInput<'_>) -> Result<()> { props: Some(actx.props()), }; - let smoke = if !opts.no_instance && !opts.skip_verify && core_healthy { + let verification = if !opts.no_instance && !opts.skip_verify && core_healthy { progress.start_step("smoke", "Memory pipeline smoke"); - match run_memory_smoke(local_global, SmokeOptions::default(), Some(smoke_telemetry)).await { + let smoke_options = if opts.provider == Some(crate::instance::storage::Provider::Slm) { + SmokeOptions::full_extraction() + } else { + SmokeOptions::default() + }; + match run_memory_smoke(local_global, smoke_options, Some(smoke_telemetry)).await { Ok(result) => { if result.verified { capture_activation( @@ -753,7 +820,7 @@ async fn finish_onboarding(input: FinishOnboardingInput<'_>) -> Result<()> { } else { progress.warn("smoke", Some("not verified")); } - Some(result) + VerificationAttempt::Passed(result) } Err(err) => { capture_step_failure( @@ -762,8 +829,8 @@ async fn finish_onboarding(input: FinishOnboardingInput<'_>) -> Result<()> { Some(actx.props()), global.no_telemetry, ); - progress.warn("smoke", Some(&format!("skipped: {err:#}"))); - None + progress.fail("smoke", Some("verification failed")); + VerificationAttempt::Failed(format!("{err:#}")) } } } else { @@ -778,7 +845,11 @@ async fn finish_onboarding(input: FinishOnboardingInput<'_>) -> Result<()> { }), ); } - None + if opts.skip_verify || opts.no_instance { + VerificationAttempt::DeliberatelySkipped + } else { + VerificationAttempt::NotRun + } }; progress.start_step("receipt", "Init receipt"); @@ -794,7 +865,8 @@ async fn finish_onboarding(input: FinishOnboardingInput<'_>) -> Result<()> { no_instance: opts.no_instance, cloud_connection_online, credential_ready, - smoke, + profile_name: local_global.profile.as_deref().unwrap_or("local"), + verification, }); progress.succeed( "receipt", @@ -806,6 +878,24 @@ async fn finish_onboarding(input: FinishOnboardingInput<'_>) -> Result<()> { ); print_init_receipt(&receipt, global); + if !receipt.credential_ready { + bail!( + "Cloud credential is not verified. Retry: {}", + receipt.next_command + ); + } + if receipt.verification_status == VerificationStatus::Failed { + bail!( + "Memory verification failed. Retry: {}", + receipt.next_command + ); + } + if !opts.no_instance && !cloud_connection_online { + bail!( + "Cloud runtime connection is not verified. Retry: {}", + receipt.next_command + ); + } Ok(()) } @@ -813,10 +903,10 @@ async fn ensure_authenticated( cloud_profile: &str, cloud_api_url: &str, use_device: bool, + allow_prompts: bool, global: &GlobalOptions, progress: &mut dyn ProgressReporter, actx: &mut ActivationContext, - no_telemetry: bool, ) -> Result<()> { if valid_bearer_token(cloud_profile, cloud_api_url) .await @@ -826,6 +916,18 @@ async fn ensure_authenticated( return Ok(()); } + if !crate::commands::init::may_run_init_login( + allow_prompts, + use_device, + io::stdin().is_terminal(), + ) { + bail!( + "sign-in required — run `am auth login --token ` first, \ + or `am init --device` to sign in with a device code. Browser sign-in \ + needs an interactive terminal, and --yes never opens one." + ); + } + if use_device { progress.tick("identity", "device login"); run_device_login( @@ -865,7 +967,7 @@ async fn ensure_authenticated( capture_activation( ActivationEvent::LoginCompleted, Some(actx.props()), - no_telemetry, + global.no_telemetry, ); progress.succeed("identity", Some("signed in")); Ok(()) @@ -911,6 +1013,45 @@ mod tests { } } + #[tokio::test] + async fn incomplete_credentials_never_return_onboarding_success() { + let global = GlobalOptions { + quiet: true, + no_telemetry: true, + ..Default::default() + }; + let mut progress = progress_for(&global); + let mut actx = ActivationContext::local(); + let org = Organization { + id: "org_a".into(), + clerk_org_id: "clerk_a".into(), + name: "org".into(), + slug: "org".into(), + created_at: Utc::now(), + }; + let project = sample_project("org_a", "local"); + let opts = ConnectProjectOptions { + no_instance: true, + ..Default::default() + }; + let result = finish_onboarding(FinishOnboardingInput { + local_global: &global, + project: &project, + org: &org, + local_url: "http://127.0.0.1:17350", + cloud_api_url: "https://api.atomicmemory.ai", + signed_in_as: None, + core_healthy: false, + credential_ready: false, + opts: &opts, + actx: &mut actx, + global: &global, + progress: progress.as_mut(), + }) + .await; + assert!(result.unwrap_err().to_string().contains("credential")); + } + #[test] fn project_id_prefix_detects_proj_ids() { assert!("proj_abc".starts_with("proj_")); diff --git a/crates/cli/src/commands/doctor_cmd.rs b/crates/cli/src/commands/doctor_cmd.rs index d283e75..9a1532a 100644 --- a/crates/cli/src/commands/doctor_cmd.rs +++ b/crates/cli/src/commands/doctor_cmd.rs @@ -12,7 +12,7 @@ use crate::progress::progress_for; use crate::telemetry::{ ActivationContext, ActivationEvent, InitStep, capture_activation, capture_step_failure, }; -use crate::verification::smoke::{SmokeOptions, SmokeTelemetry, run_memory_smoke}; +use crate::verification::smoke::{SmokeTelemetry, run_memory_smoke}; #[derive(Debug, Args)] #[command(about = "Run onboarding health checks (auth, connect wiring, optional smoke)")] @@ -61,6 +61,9 @@ async fn run_with_progress( ConnectOptions { project: None, device: false, + provider: None, + slm: false, + yes: false, no_instance: false, skip_verify: false, replace: false, @@ -84,6 +87,58 @@ async fn run_with_progress( ); } + // Soft SLM diagnostics (never fail the doctor solely for unsupported arch). + progress.start_step("slm", "Connected Local SLM"); + match crate::slm::current_target() { + None => progress.warn( + "slm", + Some( + "unsupported platform — Apple Silicon macOS only until Linux/Intel artifacts exist", + ), + ), + Some(_) => { + let paths = match crate::slm::default_slm_paths() { + Ok(p) => p, + Err(err) => { + progress.fail("slm", Some(&err.to_string())); + return Err(err); + } + }; + if !paths.binary().is_file() { + progress.warn( + "slm", + Some("runtime not installed — run `am slm install` for Connected Local SLM"), + ); + } else { + let port = paths + .read_state() + .ok() + .and_then(|s| s.port) + .unwrap_or(crate::slm::DEFAULT_SLM_PORT); + let endpoint = format!("http://127.0.0.1:{port}"); + // Port collision soft check + if std::net::TcpListener::bind(("127.0.0.1", port)).is_err() { + // Something is listening — probe health + let client = reqwest::Client::new(); + match crate::slm::check_ready(&client, &endpoint).await { + Ok(models) if models.ready() => { + progress.succeed("slm", Some(&format!("runtime healthy on :{port}"))); + } + Ok(_) => progress.warn( + "slm", + Some("listener on SLM port but required models not advertised"), + ), + Err(err) => progress + .warn("slm", Some(&format!("port {port} busy / unhealthy: {err}"))), + } + } else { + let hint = slm_idle_hint(&paths).await; + progress.warn("slm", Some(&hint)); + } + } + } + } + if opts.smoke { progress.start_step("smoke", "Memory pipeline smoke"); let mut actx = ActivationContext::local(); @@ -92,7 +147,8 @@ async fn run_with_progress( no_telemetry: global.no_telemetry, props: Some(actx.props()), }; - match run_memory_smoke(global, SmokeOptions::default(), Some(smoke_telemetry)).await { + let smoke_options = crate::instance::smoke_options(&profile).await?; + match run_memory_smoke(global, smoke_options, Some(smoke_telemetry)).await { Ok(smoke) => { capture_activation( ActivationEvent::FirstRetrievalCompleted, @@ -123,3 +179,61 @@ async fn run_with_progress( Ok(()) } + +async fn slm_idle_hint(paths: &crate::slm::SlmPaths) -> String { + match crate::slm::status_models_json(paths).await { + Err(err) => format!( + "am-slm models status failed: {err} — run `am slm install` (a pull cannot repair this)" + ), + Ok(status) => { + if crate::slm::models_cache_ready_with_disk(&status, &crate::slm::slm_hf_home(paths)) { + "installed but not running — `am slm start`".into() + } else { + "models not cached — run `am slm models pull --yes` then `am slm start`".into() + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + #[cfg(unix)] + async fn idle_hint_surfaces_status_probe_error() { + use std::fs; + use std::os::unix::fs::PermissionsExt; + + let dir = tempfile::tempdir().unwrap(); + let paths = crate::slm::SlmPaths::at(dir.path().to_path_buf()); + paths.ensure().unwrap(); + fs::write(paths.binary(), "#!/bin/sh\necho probe-failed >&2\nexit 7\n").unwrap(); + fs::set_permissions(paths.binary(), fs::Permissions::from_mode(0o755)).unwrap(); + let hint = slm_idle_hint(&paths).await; + assert!(hint.contains("models status failed"), "{hint}"); + assert!(hint.contains("am slm install"), "{hint}"); + assert!(!hint.contains("models pull --yes"), "{hint}"); + } + + #[tokio::test] + #[cfg(unix)] + async fn idle_hint_pull_only_after_successful_empty_inspection() { + use std::fs; + use std::os::unix::fs::PermissionsExt; + + let dir = tempfile::tempdir().unwrap(); + let paths = crate::slm::SlmPaths::at(dir.path().to_path_buf()); + paths.ensure().unwrap(); + let incomplete = include_str!("../../tests/fixtures/am-slm-models-status-incomplete.json"); + fs::write( + paths.binary(), + format!("#!/bin/sh\nprintf '%s\\n' '{incomplete}'\n"), + ) + .unwrap(); + fs::set_permissions(paths.binary(), fs::Permissions::from_mode(0o755)).unwrap(); + let hint = slm_idle_hint(&paths).await; + assert!(hint.contains("models pull --yes"), "{hint}"); + assert!(!hint.contains("am slm install"), "{hint}"); + } +} diff --git a/crates/cli/src/commands/init.rs b/crates/cli/src/commands/init.rs index f0ed669..367d248 100644 --- a/crates/cli/src/commands/init.rs +++ b/crates/cli/src/commands/init.rs @@ -80,6 +80,12 @@ pub struct InitOptions { /// Skip memory pipeline smoke verification at the end #[arg(long)] pub skip_verify: bool, + /// Start Connected Local with host Metal SLM (no OpenAI key) + #[arg(long)] + pub slm: bool, + /// Select the local memory provider; omitted reuses the saved provider. + #[arg(long, value_enum, conflicts_with = "slm")] + pub provider: Option, } pub async fn run(opts: InitOptions, global: &GlobalOptions) -> Result<()> { @@ -96,6 +102,14 @@ async fn run_with_progress( ) -> Result<()> { ensure_config_initialized()?; let interactive = global.allow_prompts(opts.yes); + if opts.local && !opts.no_instance { + crate::instance::preflight_managed( + opts.local_url.as_deref(), + crate::instance::storage::Provider::requested(opts.provider, opts.slm)?, + interactive, + ) + .await?; + } if opts.project.is_none() && !opts.local { validate_local_only_options(&opts, InitActivationPath::HostedCloud)?; @@ -189,6 +203,9 @@ async fn run_with_progress( replace: opts.replace, instance_image: opts.image.clone(), interactive, + slm: opts.slm, + provider: opts.provider, + slm_pull_yes: opts.yes, }; let mut onboarding_ctx = OnboardingContext { @@ -228,7 +245,10 @@ async fn run_with_progress( ensure_local_project_for_connect(&project)?; onboarding_ctx.actx.mode = ActivationContext::local().mode; if !opts.no_instance && !global.quiet { - announce_connected_local_prerequisites(interactive); + announce_connected_local_prerequisites( + interactive, + opts.slm || opts.provider == Some(crate::instance::storage::Provider::Slm), + ); } return connect_local_project( project, @@ -273,7 +293,10 @@ async fn run_with_progress( .map_err(|e| anyhow::anyhow!("{e}"))?; if !opts.no_instance && !global.quiet { - announce_connected_local_prerequisites(interactive); + announce_connected_local_prerequisites( + interactive, + opts.slm || opts.provider == Some(crate::instance::storage::Provider::Slm), + ); } let cloud_siblings = cloud_projects(&all_projects); @@ -334,6 +357,8 @@ fn validate_local_only_options(opts: &InitOptions, mode: InitActivationPath) -> } let has_local_only_option = opts.no_instance + || opts.slm + || opts.provider.is_some() || opts.skip_verify || opts.replace || opts.image.is_some() @@ -406,18 +431,23 @@ async fn resolve_init_project( } /// Printed once before Docker / OpenAI work so operators know what to prepare. -fn announce_connected_local_prerequisites(interactive: bool) { - if interactive { - eprintln!( - "\nConnected Local needs Docker running and an OpenAI API key.\n\ +fn announce_connected_local_prerequisites(interactive: bool, slm: bool) { + eprint!("{}", connected_local_prerequisites(interactive, slm)); +} + +fn connected_local_prerequisites(interactive: bool, slm: bool) -> &'static str { + if slm { + return "\nConnected Local SLM needs Docker running and ~1.7GB of models (`--yes` or a prompt).\n\ • Docker: https://docs.docker.com/desktop/\n\ - • OpenAI: have your key ready — `am init` prompts with hidden input.\n" - ); + • No OPENAI_API_KEY. Cloud login is still required for the runtime API key.\n"; + } + if interactive { + "\nConnected Local needs Docker running and an OpenAI API key.\n\ + • Docker: https://docs.docker.com/desktop/\n\ + • OpenAI: have your key ready — `am init` prompts with hidden input.\n" } else { - eprintln!( - "\nConnected Local needs Docker running and OPENAI_API_KEY set in the environment.\n\ - • Docker: https://docs.docker.com/desktop/\n" - ); + "\nConnected Local needs Docker running and OPENAI_API_KEY set in the environment.\n\ + • Docker: https://docs.docker.com/desktop/\n" } } @@ -466,7 +496,11 @@ struct InitAuthInput<'a> { /// the full timeout and then failed. `--yes` was already fail-closed; the same /// reasoning applies whenever there is no terminal. The device flow prints a /// code to enter elsewhere, so it stays allowed on explicit opt-in. -fn may_run_init_login(allow_prompts: bool, use_device: bool, stdin_is_tty: bool) -> bool { +pub(super) fn may_run_init_login( + allow_prompts: bool, + use_device: bool, + stdin_is_tty: bool, +) -> bool { allow_prompts && (use_device || stdin_is_tty) } @@ -723,6 +757,30 @@ mod tests { assert!(Cli::try_parse_from(["am", "init", "--cloud", "--local"]).is_err()); } + #[test] + fn init_local_slm_yes_parses() { + let cli = Cli::try_parse_from(["am", "init", "--local", "--slm", "--yes"]).unwrap(); + match cli.command { + crate::cli::Command::Init(InitOptions { + local, slm, yes, .. + }) => { + assert!(local && slm && yes); + } + _ => panic!("expected init --local --slm --yes"), + } + } + + #[test] + fn slm_prerequisites_omit_openai_and_name_model_download() { + let slm = connected_local_prerequisites(true, true); + assert!(slm.contains("1.7GB")); + assert!(slm.contains("No OPENAI_API_KEY")); + assert!(!slm.contains("have your key ready")); + let openai = connected_local_prerequisites(true, false); + assert!(openai.contains("OpenAI API key")); + assert!(!openai.contains("1.7GB")); + } + #[test] fn init_mode_defaults_cloud_and_infers_project_kind() { assert_eq!( diff --git a/crates/cli/src/commands/init/hosted_cloud.rs b/crates/cli/src/commands/init/hosted_cloud.rs index 8148392..8b85830 100644 --- a/crates/cli/src/commands/init/hosted_cloud.rs +++ b/crates/cli/src/commands/init/hosted_cloud.rs @@ -142,7 +142,7 @@ async fn provision_hosted_cloud_key( }) { match probe.health(api_origin, &stored.secret).await { Ok(()) => return Ok(HostedCredentialOutcome::Reused), - Err(CloudClientError::Auth) => {} + Err(CloudClientError::Auth | CloudClientError::Forbidden { .. }) => {} Err(err) => { return Err(anyhow::anyhow!( "could not verify the stored Hosted Cloud credential: {err}" diff --git a/crates/cli/src/commands/instance.rs b/crates/cli/src/commands/instance.rs index a53b178..aabb03b 100644 --- a/crates/cli/src/commands/instance.rs +++ b/crates/cli/src/commands/instance.rs @@ -10,7 +10,7 @@ use reqwest::Url; use serde::Serialize; use crate::cli::GlobalOptions; -use crate::commands::client::{memory_client, resolve_ctx}; +use crate::commands::client::resolve_ctx; use crate::commands::cloud_api_key::{ProvisionOutcome, ensure_connected_local_cloud_api_key}; use crate::commands::connect::next_step_after_instance_start; use crate::commands::local_clients::{render_local_clients_card, resolve_local_clients}; @@ -20,14 +20,15 @@ use crate::config::{ resolve_openai_api_key_with_source, store_openai_api_key, }; use crate::environment::{CoreImageInput, resolve_core_image}; +use crate::instance::address::ManagedAddress; use crate::instance::docker::{ - ContainerInspect, ContainerState, DEFAULT_BIND_HOST, DEFAULT_HOST_PORT, DockerRunner, - InstanceEnv, RealDockerRunner, default_instance_config, managed_core_local_url, tail_lines, + ContainerInspect, ContainerState, DEFAULT_BIND_HOST, DockerRunner, InstanceEnv, + RealDockerRunner, default_instance_config, tail_lines, }; +use crate::instance::storage::{Provider, RuntimeStore}; use crate::instance::{ DEFAULT_CONTAINER_NAME, DEFAULT_WAIT_SECS, HEALTH_POLL_INTERVAL_SECS, MAX_FAILURE_LOG_LINES, - VOLUME_DATA, VOLUME_STATE, managed_core_cloud_env_mismatch, managed_core_profile_mismatch, - resolve_instance_core_api_key, + managed_core_cloud_env_mismatch, managed_core_profile_mismatch, }; use crate::output::{emit, message}; use crate::progress::{ProgressReporter, progress_for}; @@ -43,6 +44,15 @@ pub enum InstanceCommand { /// OpenAI API key for Core (overrides env and stored profile secret) #[arg(long, env = "OPENAI_API_KEY")] openai_api_key: Option, + /// Use Connected Local SLM (host Metal runtime) instead of OpenAI + #[arg(long)] + slm: bool, + /// Select the local provider; omitted reuses the saved provider. + #[arg(long, value_enum, conflicts_with = "slm")] + provider: Option, + /// Reset only the selected SLM dataset (requires --yes) + #[arg(long)] + slm_reset_data: bool, /// Recreate an existing CLI-managed container #[arg(long)] replace: bool, @@ -52,6 +62,9 @@ pub enum InstanceCommand { /// Show raw `CORE_API_KEY` in output (default: redacted) #[arg(long)] show_secrets: bool, + /// Confirm downloads or an explicit SLM reset without prompting + #[arg(long)] + yes: bool, }, /// Stop the managed Core container Stop, @@ -90,9 +103,13 @@ pub async fn run(cmd: InstanceCommand, global: &GlobalOptions) -> Result<()> { InstanceCommand::Start { image, openai_api_key, + slm, + provider, + slm_reset_data, replace, wait_secs, show_secrets, + yes, } => { let mut progress = progress_for(global); let result = run_start( @@ -101,6 +118,9 @@ pub async fn run(cmd: InstanceCommand, global: &GlobalOptions) -> Result<()> { StartOptions { image, openai_api_key, + slm, + provider, + slm_reset_data, replace, sync_managed: false, wait_secs, @@ -108,7 +128,8 @@ pub async fn run(cmd: InstanceCommand, global: &GlobalOptions) -> Result<()> { brief_output: false, progress: Some(progress.as_mut()), brief_progress_id: None, - allow_prompts: global.allow_prompts(false), + allow_prompts: global.allow_prompts(yes), + slm_pull_yes: yes, }, ) .await @@ -147,9 +168,13 @@ pub(crate) async fn run_start_brief<'a>( let InstanceCommand::Start { image, openai_api_key, + slm, + provider, + slm_reset_data, replace, wait_secs, show_secrets, + yes, } = cmd else { anyhow::bail!("run_start_brief expects InstanceCommand::Start"); @@ -161,6 +186,9 @@ pub(crate) async fn run_start_brief<'a>( StartOptions { image, openai_api_key, + slm, + provider, + slm_reset_data, replace, sync_managed, wait_secs, @@ -169,6 +197,7 @@ pub(crate) async fn run_start_brief<'a>( progress, brief_progress_id, allow_prompts, + slm_pull_yes: yes, }, ) .await @@ -178,7 +207,7 @@ async fn ensure_local_profile(global: &GlobalOptions) -> Result bool { + plan.recreate_managed || slm +} + fn confirm_replace_foreign_container( container_name: &str, replace_flag: bool, @@ -397,37 +435,10 @@ fn build_instance_env( atomicmemory_api_url: profile.base_url.clone(), cloud_jwks_url: jwks, core_api_key: Some(core_api_key), + slm: false, }) } -async fn ensure_core_key_override_allowed( - docker: &dyn DockerRunner, - container_name: &str, - shell_override: Option<&str>, - replace: bool, -) -> Result<()> { - let Some(shell_key) = shell_override.filter(|k| !k.is_empty()) else { - return Ok(()); - }; - let inspect = docker.inspect(container_name).await?; - let Some(inspect) = inspect else { - return Ok(()); - }; - if !inspect.managed_by_cli || !inspect.state.is_running() { - return Ok(()); - } - if let Some(persisted) = docker.read_core_api_key(container_name).await? - && persisted != shell_key - && !replace - { - bail!( - "CORE_API_KEY override differs from the running container's persisted key.\n\ - Recreate with override: CORE_API_KEY= am instance start --replace" - ); - } - Ok(()) -} - fn format_auth_chain_diag(cloud_base_url: &str, detail: &str) -> String { format!( "Core HTTP is up (401 on unauthenticated health) — verifying auth chain ({detail}). Cloud tier={cloud_base_url}" @@ -435,7 +446,7 @@ fn format_auth_chain_diag(cloud_base_url: &str, detail: &str) -> String { } async fn core_health_probe( - global: &GlobalOptions, + _global: &GlobalOptions, docker: &dyn DockerRunner, container_name: &str, local_url: &Url, @@ -456,8 +467,9 @@ async fn core_health_probe( .context("create inspect core memory client")?; client.health().await.context("inspect env core health")?; } else { - let (_p, client) = memory_client(global).await?; - client.health().await.context("memory client core health")?; + bail!( + "managed Core has no persisted credential; rerun instance start for the selected profile" + ); } Ok(()) } @@ -469,6 +481,7 @@ struct CoreHealthWaitContext<'a> { /// Cloud base URL for the auth-chain DIAGNOSTIC only. Never probed and /// never sent a credential; the probe URL is derived below. cloud_base_url_for_diag: &'a str, + address: &'a ManagedAddress, timeout: Duration, emit_plain_ticks: bool, bootstrap_core_key: Option<&'a str>, @@ -509,22 +522,15 @@ async fn wait_for_core_health( docker, container_name, cloud_base_url_for_diag, + address, timeout, emit_plain_ticks, bootstrap_core_key, brief_parent, } = ctx; - // Probe what we PUBLISHED, never what the profile claims. This function - // sends the bootstrap Core key as a bearer to whatever URL it probes, and - // `profile.memory_base_url` derives from the Cloud API's - // `project.local_url` - so parsing it here handed the key to any host a - // project record named, on every default `am instance start`, bypassing - // the container-label guard entirely (that guard runs on the read path, - // not on this probe). No profile parameter, so it cannot come back. - let local_url = managed_core_local_url() - .parse::() - .context("parse derived local_url for health check")?; - let host_port = DEFAULT_HOST_PORT; + // The caller supplies the address used to publish this container, never a remote profile target. + let local_url = Url::parse(&address.url())?; + let host_port = address.port(); let host = DEFAULT_BIND_HOST; let deadline = tokio::time::Instant::now() + timeout; @@ -664,6 +670,11 @@ struct StartOptions<'a> { sync_managed: bool, image: Option, openai_api_key: Option, + /// Connected Local SLM mode (host Metal runtime; no OpenAI key required). + slm: bool, + provider: Option, + /// Delete Core data volumes before SLM start when embedding dims conflict. + slm_reset_data: bool, replace: bool, wait_secs: u64, show_secrets: bool, @@ -673,6 +684,38 @@ struct StartOptions<'a> { brief_progress_id: Option<&'a str>, // When false, OpenAI key stdin prompts are skipped (`am init --yes`). allow_prompts: bool, + /// Confirm the ~1.7GB SLM model download without prompting. + slm_pull_yes: bool, +} + +async fn bootstrap_host_slm( + slm_paths: &crate::slm::SlmPaths, + confirm: crate::slm::BootstrapConfirm, + progress: &mut Option<&mut dyn ProgressReporter>, + brief_id: Option<&str>, +) -> Result<()> { + let client = reqwest::Client::builder() + .timeout(Duration::from_secs(30)) + .build() + .context("build SLM HTTP client")?; + let should_pause = confirm.allow_prompt && !confirm.yes; + if should_pause { + progress_pause(progress); + } + let result = crate::slm::bootstrap_managed_slm( + &client, + slm_paths, + confirm, + crate::slm::DEFAULT_SLM_PORT, + Duration::from_secs(90), + |step| progress_tick(progress, brief_id, "credentials", step), + ) + .await; + if should_pause { + progress_resume(progress); + } + result.context("start managed am-slm before Core")?; + Ok(()) } /// Routes instance-start progress to nested steps or init parent ticks. @@ -799,6 +842,29 @@ fn existing_container_blocks_start( } async fn run_start( + global: &GlobalOptions, + docker: &dyn DockerRunner, + opts: StartOptions<'_>, +) -> Result { + let provider = Provider::requested(opts.provider, opts.slm)?; + let profile = ensure_local_profile(global).await?; + let name = profile.name.as_str(); + let retry = provider + .map(|p| crate::instance::storage::provider_command(name, p)) + .unwrap_or_else(|| { + format!( + "am --profile '{}' instance start", + name.replace('\'', "'\\''") + ) + }); + run_start_inner(global, docker, opts) + .await + .with_context(|| { + format!("Core setup did not finish; retained datasets are preserved. Retry: {retry}") + }) +} + +async fn run_start_inner( global: &GlobalOptions, docker: &dyn DockerRunner, mut opts: StartOptions<'_>, @@ -820,10 +886,37 @@ async fn run_start( config_core_image: config_file.core_image.as_deref(), }) .value; - let config = default_instance_config(&profile.name, &resolved_image); + let address = ManagedAddress::parse(&profile.memory_base_url)?; + let mut config = default_instance_config(&profile.name, &resolved_image); + config.host_port = address.port(); + config.local_url = address.url(); + config.project_id = profile.project_id.clone(); let expected_jwks = jwks_url(&profile.base_url)?; + let mut registry = RuntimeStore::open()?; let existing = docker.inspect(&config.container_name).await?; + let requested = Provider::requested(opts.provider, opts.slm)?; + let provider = registry.provider(&profile, requested, existing.as_ref())?; + let volumes = registry.select(&profile, provider, existing.as_ref())?; + config.provider = provider; + config.storage = volumes; + let use_slm = provider == Provider::Slm; + if opts.slm_reset_data && (!use_slm || !opts.slm_pull_yes) { + bail!( + "--slm-reset-data requires the SLM provider and --yes; ordinary provider switching preserves all data" + ); + } + if use_slm && crate::slm::current_target().is_none() { + bail!("Connected Local SLM requires Apple Silicon macOS"); + } + let binding_matches = existing + .as_ref() + .is_some_and(|i| i.local_url.as_deref() == Some(config.local_url.as_str())); + if !binding_matches { + let listener = std::net::TcpListener::bind((DEFAULT_BIND_HOST, config.host_port)) + .context("managed Core port is occupied; choose a free --local-url port and retry")?; + drop(listener); + } let mut confirmed_managed_recreate = false; if let Some(inspect) = &existing && existing_container_blocks_start( @@ -856,39 +949,72 @@ async fn run_start( ); } - let missing_key = - needs_interactive_openai_key(&profile.name, &opts.openai_api_key, opts.allow_prompts); - let pause_for_openai = may_prompt_openai_key(opts.allow_prompts) && missing_key; - if pause_for_openai { - progress_pause(&mut opts.progress); + // Resolve foreign ownership before model preparation or any removal. + if existing.as_ref().is_some_and(|i| !i.managed_by_cli) && !opts.replace { + bail!( + "container 'atomic-memory' is not CLI-managed; use --replace to authorize replacement (volumes are preserved)" + ); + } + let live_legacy = crate::instance::storage::owned_storage(&profile, existing.as_ref()) + .is_some_and(|storage| storage.provider == provider && storage.volumes == config.storage); + // Capture the outgoing legacy pair before switching to another provider, while + // its container still proves ownership. The registry commit precedes removal. + if let Some(outgoing) = crate::instance::storage::owned_storage(&profile, existing.as_ref()) { + let mut outgoing_config = config.clone(); + outgoing_config.provider = outgoing.provider; + outgoing_config.storage = outgoing.volumes.clone(); + let identity = docker + .validate_storage( + &outgoing_config, + registry.identity(&profile, outgoing.provider)?, + true, + ) + .await?; + registry.record_identity(&profile, outgoing.provider, &outgoing.volumes, identity)?; + } + docker + .validate_storage(&config, registry.identity(&profile, provider)?, live_legacy) + .await?; + + let slm_paths = crate::slm::default_slm_paths()?; + let openai_key = if use_slm { progress_tick( &mut opts.progress, opts.brief_progress_id, "credentials", - "OpenAI API key required below", + if opts.slm { + "Connected Local SLM (no OpenAI key)" + } else { + "preserving Connected Local SLM (768-dim volumes)" + }, ); - } - progress_tick( - &mut opts.progress, - opts.brief_progress_id, - "credentials", - "validating OpenAI key", - ); - let openai_key = - ensure_openai_api_key(&profile.name, opts.openai_api_key, opts.allow_prompts).await; - if pause_for_openai { - progress_resume(&mut opts.progress); - } - let openai_key = openai_key?; - - let shell_override = resolve_core_api_key(); - ensure_core_key_override_allowed( - docker, - &config.container_name, - shell_override.as_deref(), - opts.replace, - ) - .await?; + String::new() + } else { + let missing_key = + needs_interactive_openai_key(&profile.name, &opts.openai_api_key, opts.allow_prompts); + let pause_for_openai = may_prompt_openai_key(opts.allow_prompts) && missing_key; + if pause_for_openai { + progress_pause(&mut opts.progress); + progress_tick( + &mut opts.progress, + opts.brief_progress_id, + "credentials", + "OpenAI API key required below", + ); + } + progress_tick( + &mut opts.progress, + opts.brief_progress_id, + "credentials", + "validating OpenAI key", + ); + let openai_key = + ensure_openai_api_key(&profile.name, opts.openai_api_key, opts.allow_prompts).await; + if pause_for_openai { + progress_resume(&mut opts.progress); + } + openai_key? + }; progress_tick( &mut opts.progress, @@ -915,9 +1041,83 @@ async fn run_start( opts.sync_managed || cloud_key_outcome.requires_container_sync() || credentials_drifted, confirmed_managed_recreate, ); - let needs_recreate = plan.recreate_managed; - let core_api_key = resolve_instance_core_api_key(docker, false).await?; - let env = build_instance_env(&profile, &api_key, &openai_key, core_api_key.clone())?; + let dataset_matches = existing + .as_ref() + .and_then(|i| i.storage.as_ref()) + .is_some_and(|s| s.provider == provider && s.volumes == config.storage); + let needs_recreate = needs_managed_recreate( + plan, + !dataset_matches || !binding_matches || opts.slm_reset_data, + ); + if use_slm { + bootstrap_host_slm( + &slm_paths, + crate::slm::BootstrapConfirm { + yes: opts.slm_pull_yes, + allow_prompt: may_prompt_for_input(opts.allow_prompts), + }, + &mut opts.progress, + opts.brief_progress_id, + ) + .await?; + } + let identity = docker + .prepare_storage(&config, registry.identity(&profile, provider)?, live_legacy) + .await?; + registry.record_identity(&profile, provider, &config.storage, identity)?; + let core_api_key = crate::instance::credentials::resolve_dataset_key( + docker, + &config, + opts.slm_reset_data, + resolve_core_api_key(), + opts.replace, + ) + .await?; + let mut env = build_instance_env(&profile, &api_key, &openai_key, core_api_key.clone())?; + if use_slm { + crate::slm::apply_slm_overlay(&mut env); + } + // Commit the target before the swap; interruption resumes this selection, never an implicit fallback. + registry.save()?; + message( + !global.quiet && global.output != crate::cli::OutputFormat::Json, + &format!( + "Using {} at {}. Data: {}. State: {}. Other provider data is preserved.", + provider.as_str(), + config.local_url, + config.storage.data, + config.storage.state + ), + ); + if let Some(previous) = existing.as_ref().and_then(|i| i.storage.as_ref()) + && previous.provider != provider + { + message( + !global.quiet && global.output != crate::cli::OutputFormat::Json, + &format!( + "Switch back: {}", + crate::instance::storage::provider_command(&profile.name, previous.provider) + ), + ); + } + if opts.slm_reset_data { + docker + .validate_storage(&config, registry.identity(&profile, provider)?, live_legacy) + .await?; + if let Some(current) = &existing + && current + .storage + .as_ref() + .is_some_and(|s| s.volumes == config.storage) + { + docker.rm_force(&config.container_name).await?; + } + docker.volume_rm(&config.storage.data).await?; + docker.volume_rm(&config.storage.state).await?; + let identity = docker.prepare_storage(&config, None, false).await?; + registry.record_identity(&profile, provider, &config.storage, identity)?; + registry.save()?; + } progress_succeed( &mut opts.progress, opts.brief_progress_id, @@ -977,37 +1177,12 @@ async fn run_start( } } Some(inspect) if inspect.state.is_running() && !needs_recreate => { - if opts.brief_output { - if progress_is_active(&opts.progress) { - progress_succeed( - &mut opts.progress, - opts.brief_progress_id, - "container", - Some("already running"), - ); - } else { - message( - !global.quiet, - &format!("Core already running at {}", profile.memory_base_url), - ); - } - return Ok(true); - } - let report = - instance_status_report(&profile, Some(inspect), docker, global, opts.show_secrets) - .await?; - emit_instance_report(global, &report, opts.show_secrets)?; - if progress_is_active(&opts.progress) { - progress_succeed( - &mut opts.progress, - opts.brief_progress_id, - "container", - Some("already running"), - ); - } else { - message(!global.quiet, "Instance already running."); - } - return Ok(true); + progress_succeed( + &mut opts.progress, + opts.brief_progress_id, + "container", + Some("already running"), + ); } Some(_) if needs_recreate => { docker.rm_force(&config.container_name).await?; @@ -1067,6 +1242,7 @@ async fn run_start( docker, container_name: &config.container_name, cloud_base_url_for_diag: &profile.base_url, + address: &address, timeout: Duration::from_secs(opts.wait_secs), emit_plain_ticks, bootstrap_core_key: Some(core_api_key.as_str()), @@ -1124,11 +1300,48 @@ async fn run_start( Ok(true) } +fn ensure_selected_instance( + profile: &crate::config::ResolvedProfile, + inspect: &ContainerInspect, +) -> Result<()> { + if !inspect.managed_by_cli + || managed_core_profile_mismatch(inspect, &profile.name) + || managed_core_cloud_env_mismatch( + inspect, + &profile.base_url, + &jwks_url(&profile.base_url)?, + ) + { + bail!( + "the active Core belongs to another profile or installation; select its profile before changing it" + ); + } + match inspect + .storage + .as_ref() + .and_then(|s| s.project_id.as_deref()) + { + Some(id) if profile.project_id.as_deref() == Some(id) => {} + Some(_) => bail!("active Core belongs to another project"), + None => { + let key = crate::config::require_api_key(profile)?; + if inspect.atomicmemory_api_key.as_deref() != Some(key.as_str()) { + bail!( + "legacy Core project ownership cannot be proven with the selected profile's credential" + ); + } + } + } + Ok(()) +} + async fn run_stop(global: &GlobalOptions, docker: &dyn DockerRunner) -> Result<()> { + let _registry = RuntimeStore::open()?; let profile = ensure_local_profile(global).await?; docker.version().await?; let name = DEFAULT_CONTAINER_NAME; if let Some(inspect) = docker.inspect(name).await? { + ensure_selected_instance(&profile, &inspect)?; if !inspect.managed_by_cli { bail!("container '{name}' is not managed by `am instance`"); } @@ -1156,7 +1369,22 @@ async fn run_restart( let profile = ensure_local_profile(global).await?; docker.version().await?; let name = DEFAULT_CONTAINER_NAME; + let _registry = RuntimeStore::open()?; let inspect = docker.inspect(name).await?; + let current = inspect + .as_ref() + .ok_or_else(|| anyhow::anyhow!("no managed instance; run am instance start"))?; + ensure_selected_instance(&profile, current)?; + let configured_url = if current.state.is_running() { + current.local_url.clone() + } else { + docker.configured_local_url(name).await? + }; + let expected_address = ManagedAddress::parse(configured_url.as_deref().ok_or_else(|| { + anyhow::anyhow!( + "container has no trusted loopback binding; recreate with am instance start" + ) + })?)?; match inspect { Some(i) if i.managed_by_cli => { docker.stop(name).await?; @@ -1166,6 +1394,19 @@ async fn run_restart( Some(_) => bail!("container '{name}' is not managed by `am instance`"), None => bail!("no managed instance '{name}' — run `am instance start`"), } + let restarted = docker + .inspect(name) + .await? + .ok_or_else(|| anyhow::anyhow!("Core container disappeared after restart"))?; + ensure_selected_instance(&profile, &restarted)?; + let address = ManagedAddress::parse(restarted.local_url.as_deref().ok_or_else(|| { + anyhow::anyhow!( + "restarted Core has no published loopback port; run am instance start to recover" + ) + })?)?; + if address != expected_address { + bail!("Core publication changed during restart; refusing to send credentials"); + } if wait_secs > 0 { let mut progress: Option<&mut dyn ProgressReporter> = None; wait_for_core_health( @@ -1174,6 +1415,7 @@ async fn run_restart( docker, container_name: name, cloud_base_url_for_diag: &profile.base_url, + address: &address, timeout: Duration::from_secs(wait_secs), emit_plain_ticks: !global.quiet, bootstrap_core_key: None, @@ -1239,29 +1481,34 @@ async fn run_remove( purge_data: bool, yes: bool, ) -> Result<()> { + // Authorization and selected-dataset resolution precede every destructive call. + if purge_data { + validate_purge_confirmed(yes)?; + } let profile = ensure_local_profile(global).await?; + let mut registry = RuntimeStore::open()?; docker.version().await?; let name = DEFAULT_CONTAINER_NAME; - - if let Some(inspect) = docker.inspect(name).await? - && !inspect.managed_by_cli - { - bail!("container '{name}' is not managed by `am instance`"); - } - - docker.rm_force(name).await?; - message(!global.quiet, "Container removed."); - + let current = docker.inspect(name).await?; + let volumes = remove_selected_dataset( + &profile, + docker, + &mut registry, + current.as_ref(), + purge_data, + ) + .await?; if purge_data { - validate_purge_confirmed(yes)?; - docker.volume_rm(VOLUME_DATA).await?; - docker.volume_rm(VOLUME_STATE).await?; - message(!global.quiet, "Named volumes removed."); + message( + !global.quiet, + "Selected provider dataset removed; other datasets preserved.", + ); } else { message( !global.quiet, &format!( - "Data volumes preserved ({VOLUME_DATA}, {VOLUME_STATE}). CORE_API_KEY persists in {VOLUME_STATE} until `--purge-data --yes`." + "Container removed. Data preserved in {} and {}.", + volumes.data, volumes.state ), ); } @@ -1270,6 +1517,63 @@ async fn run_remove( emit_instance_report(global, &report, false) } +async fn remove_selected_dataset( + profile: &crate::config::ResolvedProfile, + docker: &dyn DockerRunner, + registry: &mut RuntimeStore, + current: Option<&ContainerInspect>, + purge_data: bool, +) -> Result { + let name = DEFAULT_CONTAINER_NAME; + if let Some(inspect) = current { + ensure_selected_instance(profile, inspect)?; + } + let selected = registry.selected_dataset(profile)?; + let (provider, volumes) = match selected { + Some(selected) => selected, + None => { + let observed = + crate::instance::storage::owned_storage(profile, current).ok_or_else(|| { + anyhow::anyhow!( + "no proven dataset for this profile; no container or volumes were removed" + ) + })?; + let provider = observed.provider; + (provider, registry.select(profile, provider, current)?) + } + }; + let live_legacy = crate::instance::storage::owned_storage(profile, current) + .is_some_and(|storage| storage.provider == provider && storage.volumes == volumes); + if let Some(inspect) = current { + if !inspect + .storage + .as_ref() + .is_some_and(|s| s.provider == provider && s.volumes == volumes) + { + bail!( + "active container does not match the selected dataset; run am instance start before removing it" + ); + } + } + let mut config = default_instance_config(&profile.name, ""); + config.provider = provider; + config.storage = volumes.clone(); + let identity = docker + .validate_storage(&config, registry.identity(profile, provider)?, live_legacy) + .await?; + registry.record_identity(profile, provider, &config.storage, identity)?; + // Keep adopted identity durable before removing its only live ownership witness. + registry.save()?; + if current.is_some() { + docker.rm_force(name).await?; + } + if purge_data { + docker.volume_rm(&volumes.data).await?; + docker.volume_rm(&volumes.state).await?; + } + Ok(volumes) +} + fn validate_purge_confirmed(yes: bool) -> Result<()> { if !yes { bail!("refusing to delete volumes — pass both --purge-data and --yes"); @@ -1293,6 +1597,7 @@ struct ContainerStatus { image: String, managed_by_cli: bool, profile_label: Option, + storage: Option, } fn emit_instance_report( @@ -1301,7 +1606,7 @@ fn emit_instance_report( show_secrets: bool, ) -> Result<()> { emit(global.output, report, global.quiet)?; - if !global.quiet { + if !global.quiet && global.output != crate::cli::OutputFormat::Json { message( true, &render_local_clients_card(&report.local_clients, show_secrets), @@ -1312,14 +1617,14 @@ fn emit_instance_report( async fn read_profile_core_key( docker: &dyn DockerRunner, - profile_name: &str, + profile: &crate::config::ResolvedProfile, inspect: Option<&ContainerInspect>, ) -> Option { let inspect = inspect?; if !inspect.managed_by_cli || !inspect.state.is_running() { return None; } - if inspect.profile_label.as_deref() != Some(profile_name) { + if ensure_selected_instance(profile, inspect).is_err() || inspect.local_url.is_none() { return None; } docker @@ -1333,7 +1638,7 @@ async fn instance_status_report( profile: &crate::config::ResolvedProfile, inspect: Option<&ContainerInspect>, docker: &dyn DockerRunner, - global: &GlobalOptions, + _global: &GlobalOptions, show_secrets: bool, ) -> Result { let container = inspect.map(|i| ContainerStatus { @@ -1341,31 +1646,38 @@ async fn instance_status_report( image: i.image.clone(), managed_by_cli: i.managed_by_cli, profile_label: i.profile_label.clone(), + storage: i.storage.clone(), }); - let core_health = match memory_client(global).await { - Ok((_p, client)) => match client.health().await { - Ok(_) => Some("ok".into()), - Err(e) => Some(format!("error: {e}")), - }, - Err(e) => Some(format!("unavailable: {e}")), + let state_key = read_profile_core_key(docker, profile, inspect).await; + let local_url = inspect + .filter(|i| ensure_selected_instance(profile, i).is_ok()) + .and_then(|i| i.local_url.clone()) + .unwrap_or_else(|| profile.memory_base_url.clone()); + let core_health = match (&state_key, ManagedAddress::parse(&local_url)) { + (Some(key), Ok(address)) => { + let client = + am_cloud_client::MemoryClient::new(Url::parse(&address.url())?, key.clone())?; + Some(match client.health().await { + Ok(_) => "ok".into(), + Err(err) => format!("error: {err}"), + }) + } + _ => Some("unavailable: no matching running managed Core".into()), }; - - let state_key = read_profile_core_key(docker, &profile.name, inspect).await; // Raw secrets depend on --show-secrets and nothing else. A `reveal_on_start` // override meant an ordinary `am instance start` printed the persisted // CORE_API_KEY, a usable bearer token, into terminals and captured logs // while the flag advertised that secrets were redacted without it. let reveal = show_secrets; - let local_clients = - resolve_local_clients(&profile.memory_base_url, state_key.as_deref(), reveal); + let local_clients = resolve_local_clients(&local_url, state_key.as_deref(), reveal); Ok(InstanceStatusReport { profile: profile.name.clone(), container_name: DEFAULT_CONTAINER_NAME.to_string(), container, core_health, - local_url: profile.memory_base_url.clone(), + local_url, local_clients, }) } @@ -1452,6 +1764,24 @@ mod tests { assert!(!plan.may_replace_foreign); } + #[test] + fn slm_forces_recreate_even_when_container_already_running() { + // No --replace, no credential sync, no profile mismatch — but --slm. + let plan = ReplacementPlan::resolve(false, false, false); + assert!( + !plan.recreate_managed, + "baseline plan alone must not recreate" + ); + assert!( + needs_managed_recreate(plan, true), + "--slm must force recreate so overlay env is applied via docker.run" + ); + assert!( + !needs_managed_recreate(plan, false), + "without --slm, a quiet plan must keep the already-running early return" + ); + } + /// The startup health probe must authenticate against the URL we /// PUBLISHED, never one the profile supplies. /// @@ -1496,7 +1826,7 @@ mod tests { "the health path must not read the profile's local URL; it sends a bearer key", ); assert!( - body.contains("managed_core_local_url()"), + body.contains("address.url()"), "the probe URL must come from the published binding", ); } @@ -1540,6 +1870,7 @@ mod tests { atomicmemory_api_url: "https://api.dev.example.com".into(), cloud_jwks_url: "https://api.dev.example.com/.well-known/atomic-core/jwks.json".into(), core_api_key: Some("generated-core-key".into()), + slm: false, } } @@ -1552,9 +1883,13 @@ mod tests { crate::cli::Command::Instance(InstanceCommand::Start { image, openai_api_key, + slm: false, + provider: None, + slm_reset_data: false, replace, wait_secs, show_secrets, + yes: false, }) => { assert!(image.is_none()); assert!(openai_api_key.is_none()); @@ -1566,6 +1901,20 @@ mod tests { } } + #[test] + fn instance_start_slm_yes_parses() { + use crate::cli::Cli; + use clap::Parser; + let cli = Cli::try_parse_from(["am", "instance", "start", "--slm", "--yes"]).unwrap(); + match cli.command { + crate::cli::Command::Instance(InstanceCommand::Start { slm, yes, .. }) => { + assert!(slm); + assert!(yes); + } + _ => panic!("expected instance start --slm --yes"), + } + } + #[test] fn instance_status_accepts_show_secrets() { use crate::cli::Cli; @@ -1759,6 +2108,7 @@ mod tests { fn managed_inspect(state: ContainerState, profile: &str) -> ContainerInspect { ContainerInspect { + storage: None, name: "atomic-memory".into(), image: "ghcr.io/atomicstrata/atomicmemory-core:latest".into(), state, @@ -1772,6 +2122,25 @@ mod tests { } } + #[test] + fn legacy_lifecycle_gate_requires_matching_project_credential() { + let profile = crate::config::ResolvedProfile { + name: "local-test".into(), + base_url: "https://api.atomicstrata.ai".into(), + kind: crate::config::ProfileKind::Cloud, + project_id: Some("new-project".into()), + memory_base_url: "http://127.0.0.1:17350".into(), + api_key: Some("amc_new_project".into()), + oauth: None, + }; + let mut inspect = managed_inspect(ContainerState::Running, &profile.name); + inspect.cloud_jwks_url = Some(jwks_url(&profile.base_url).unwrap()); + inspect.atomicmemory_api_key = Some("amc_previous_project".into()); + assert!(ensure_selected_instance(&profile, &inspect).is_err()); + inspect.atomicmemory_api_key = profile.api_key.clone(); + assert!(ensure_selected_instance(&profile, &inspect).is_ok()); + } + #[test] fn cloud_key_rotated_outcome_forces_replace_sync() { assert!( @@ -2007,3 +2376,7 @@ mod tests { assert_eq!(reporter.input_events, vec!["pause", "resume"]); } } + +#[cfg(all(test, unix))] +#[path = "instance_removal_tests.rs"] +mod removal_tests; diff --git a/crates/cli/src/commands/instance_removal_tests.rs b/crates/cli/src/commands/instance_removal_tests.rs new file mode 100644 index 0000000..b16945a --- /dev/null +++ b/crates/cli/src/commands/instance_removal_tests.rs @@ -0,0 +1,266 @@ +//! Removal regressions exercising real Docker argv against an isolated fake executable. + +use super::*; +use crate::config::ResolvedProfile; +use crate::instance::storage::{ObservedStorage, StorageIdentity, StorageVolumes}; +use std::os::unix::fs::PermissionsExt; + +struct Fixture { + directory: tempfile::TempDir, + docker: RealDockerRunner, + registry: RuntimeStore, + profile: ResolvedProfile, +} + +impl Fixture { + fn new(created_at: &str) -> Self { + let directory = tempfile::tempdir().unwrap(); + let binary = directory.path().join("fake-docker"); + let log = directory.path().join("calls"); + let json = serde_json::json!([{"Labels": null, "CreatedAt":created_at}]); + let script = format!( + "#!/bin/sh\nprintf '%s\\n' \"$*\" >> '{}'\nif [ \"$1 $2\" = 'volume inspect' ]; then printf '%s\\n' '{}'; fi\n", + log.display(), + json + ); + let staging = directory.path().join("fake-docker.staging"); + std::fs::write(&staging, script).unwrap(); + std::fs::set_permissions(&staging, std::fs::Permissions::from_mode(0o700)).unwrap(); + std::fs::rename(&staging, &binary).unwrap(); + let registry = RuntimeStore::at(&directory.path().join("runtimes.json")).unwrap(); + Self { + directory, + registry, + docker: RealDockerRunner { + docker_bin: binary.to_string_lossy().into(), + }, + profile: ResolvedProfile { + name: "local-test".into(), + base_url: "https://api.atomicstrata.ai".into(), + kind: ProfileKind::Local, + project_id: Some("project-one".into()), + memory_base_url: "http://127.0.0.1:17350".into(), + api_key: Some("amc_owner".into()), + oauth: None, + }, + } + } + + fn calls(&self) -> String { + match std::fs::read_to_string(self.directory.path().join("calls")) { + Ok(calls) => calls, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => String::new(), + Err(err) => panic!("read fake Docker calls: {err}"), + } + } + + fn legacy_container(&self) -> ContainerInspect { + ContainerInspect { + name: DEFAULT_CONTAINER_NAME.into(), + image: "fixture".into(), + state: ContainerState::Running, + managed_by_cli: true, + profile_label: Some(self.profile.name.clone()), + atomicmemory_api_url: Some(self.profile.base_url.clone()), + cloud_jwks_url: Some(jwks_url(&self.profile.base_url).unwrap()), + core_api_key: None, + atomicmemory_api_key: self.profile.api_key.clone(), + local_url: Some(self.profile.memory_base_url.clone()), + storage: Some(ObservedStorage { + provider: Provider::Openai, + volumes: StorageVolumes::legacy(), + project_id: None, + }), + } + } + + fn adopt_legacy(&mut self, created_at: &str) { + self.registry + .select( + &self.profile, + Provider::Openai, + Some(&self.legacy_container()), + ) + .unwrap(); + let identity = StorageIdentity { + data: Some(created_at.into()), + state: Some(created_at.into()), + }; + self.registry + .record_identity( + &self.profile, + Provider::Openai, + &StorageVolumes::legacy(), + identity, + ) + .unwrap(); + self.registry.save().unwrap(); + } +} + +#[tokio::test] +async fn purge_does_not_invent_a_dataset_for_unregistered_profile() { + let mut fixture = Fixture::new("foreign"); + let result = remove_selected_dataset( + &fixture.profile, + &fixture.docker, + &mut fixture.registry, + None, + true, + ) + .await; + assert!( + result + .unwrap_err() + .to_string() + .contains("no proven dataset") + ); + assert!(fixture.calls().is_empty()); +} + +#[tokio::test] +async fn purge_refuses_foreign_volume_before_any_destructive_docker_command() { + let mut fixture = Fixture::new("foreign"); + fixture + .registry + .select(&fixture.profile, Provider::Openai, None) + .unwrap(); + let result = remove_selected_dataset( + &fixture.profile, + &fixture.docker, + &mut fixture.registry, + None, + true, + ) + .await; + assert!(result.unwrap_err().to_string().contains("ownership")); + let calls = fixture.calls(); + assert_eq!(calls.lines().count(), 2); + assert!( + calls + .lines() + .all(|line| line.starts_with("volume inspect ")) + ); +} + +#[tokio::test] +async fn purge_rejects_legacy_names_recreated_after_adoption() { + let mut fixture = Fixture::new("replacement"); + fixture.adopt_legacy("original"); + let result = remove_selected_dataset( + &fixture.profile, + &fixture.docker, + &mut fixture.registry, + None, + true, + ) + .await; + assert!( + result + .unwrap_err() + .to_string() + .contains("legacy creation identity") + ); + assert!(!fixture.calls().contains("volume rm")); +} + +#[tokio::test] +async fn purge_accepts_retired_legacy_pair_with_durable_creation_identity() { + let mut fixture = Fixture::new("original"); + fixture.adopt_legacy("original"); + let removed = remove_selected_dataset( + &fixture.profile, + &fixture.docker, + &mut fixture.registry, + None, + true, + ) + .await + .unwrap(); + assert_eq!(removed, StorageVolumes::legacy()); + assert_eq!( + fixture + .calls() + .lines() + .filter(|line| line.starts_with("volume rm ")) + .count(), + 2 + ); +} + +#[tokio::test] +async fn legacy_project_mismatch_is_rejected_before_docker_mutation() { + let mut fixture = Fixture::new("original"); + let mut current = fixture.legacy_container(); + current.atomicmemory_api_key = Some("amc_other_project".into()); + let result = remove_selected_dataset( + &fixture.profile, + &fixture.docker, + &mut fixture.registry, + Some(¤t), + true, + ) + .await; + assert!(result.unwrap_err().to_string().contains("ownership")); + assert!(fixture.calls().is_empty()); +} + +#[tokio::test] +async fn preflight_saves_only_outgoing_legacy_proof_before_cloud_key_rotation() { + let mut fixture = Fixture::new("original"); + let binary = fixture.directory.path().join("fake-docker"); + let log = fixture.directory.path().join("calls"); + let volumes = StorageVolumes::legacy(); + let container = serde_json::json!([{ + "State":{"Status":"running"}, + "Config":{"Image":"fixture", "Labels":{ + "ai.atomicstrata.managed-by":"am-cli", "ai.atomicstrata.profile":"local-test" + }, "Env":["OPENAI_API_KEY=test", "ATOMICMEMORY_API_URL=https://api.atomicstrata.ai", "ATOMICMEMORY_API_KEY=amc_owner"]}, + "Mounts":[ + {"Type":"volume", "Name":volumes.data, "Destination":"/var/lib/atomicmemory/postgres"}, + {"Type":"volume", "Name":volumes.state, "Destination":"/var/lib/atomicmemory/state"} + ] + }]); + let script = format!( + "#!/bin/sh\nprintf '%s\\n' \"$*\" >> '{}'\nif [ \"$1\" = inspect ]; then printf '%s\\n' '{}'; exit 0; fi\nif [ \"$3\" = '{}' ] || [ \"$3\" = '{}' ]; then printf '%s\\n' '[{{\"Labels\":null,\"CreatedAt\":\"original\"}}]'; exit 0; fi\nprintf '%s\\n' 'No such volume' >&2\nexit 1\n", + log.display(), + container, + volumes.data, + volumes.state + ); + let staging = fixture.directory.path().join("fake-docker.staging"); + std::fs::write(&staging, script).unwrap(); + std::fs::set_permissions(&staging, std::fs::Permissions::from_mode(0o700)).unwrap(); + std::fs::rename(&staging, &binary).unwrap(); + crate::instance::storage::preflight_storage_with( + &fixture.profile, + Provider::Slm, + false, + &fixture.docker, + &mut fixture.registry, + ) + .await + .unwrap(); + assert!( + fixture + .calls() + .lines() + .all(|line| line.starts_with("inspect ") || line.starts_with("volume inspect ")) + ); + drop(fixture.registry); + fixture.profile.api_key = Some("amc_rotated".into()); + let registry = RuntimeStore::at(&fixture.directory.path().join("runtimes.json")).unwrap(); + assert_eq!( + registry.selected_dataset(&fixture.profile).unwrap(), + Some((Provider::Openai, StorageVolumes::legacy())) + ); + assert_eq!( + registry + .identity(&fixture.profile, Provider::Openai) + .unwrap() + .unwrap() + .data + .as_deref(), + Some("original") + ); +} diff --git a/crates/cli/src/commands/integrate.rs b/crates/cli/src/commands/integrate.rs index 6b8a395..0adb775 100644 --- a/crates/cli/src/commands/integrate.rs +++ b/crates/cli/src/commands/integrate.rs @@ -1,6 +1,9 @@ //! `am integrate` — detect, install, update, doctor, and uninstall host MCP configs. +use std::env; +use std::ffi::OsString; use std::io::{self, IsTerminal}; +use std::path::PathBuf; use anyhow::{Result, bail}; use clap::{Args, Subcommand, ValueEnum}; @@ -8,11 +11,12 @@ use serde::Serialize; use crate::cli::{GlobalOptions, OutputFormat}; use crate::integrate::install::{InstallOptions, default_cwd}; +use crate::integrate::path_util::home_dir; use crate::integrate::spec::preflight_install_runtime; use crate::integrate::state::list_owned_status; use crate::integrate::{ - DetectReport, DoctorReport, DoctorStatus, Host, InstallAction, InstallReport, InstallScope, - PROJECT_SCOPE_UNSUPPORTED, all_hosts, detect_hosts, detected_hosts, doctor_hosts, + DetectReport, DoctorReport, DoctorStatus, Host, HostConfigPaths, InstallAction, InstallReport, + InstallScope, PROJECT_SCOPE_UNSUPPORTED, all_hosts, detect_hosts, detected_hosts, doctor_hosts, install_hosts, parse_host, resolve_credentials, select_hosts_interactive, uninstall_hosts, }; use crate::output::{emit, message}; @@ -20,7 +24,7 @@ use crate::progress::{ProgressReporter, progress_for}; #[derive(Debug, Args)] pub struct IntegrateOptions { - /// Target host (repeatable): cursor, claude-code, codex + /// Target host (repeatable): cursor, claude-code, codex, opencode #[arg(long = "host", value_enum, global = true)] pub hosts: Vec, @@ -54,6 +58,8 @@ pub enum HostArg { #[value(name = "claude-code")] ClaudeCode, Codex, + #[value(name = "opencode")] + OpenCode, } impl From for Host { @@ -62,6 +68,7 @@ impl From for Host { HostArg::Cursor => Host::Cursor, HostArg::ClaudeCode => Host::ClaudeCode, HostArg::Codex => Host::Codex, + HostArg::OpenCode => Host::OpenCode, } } } @@ -113,9 +120,10 @@ struct OwnedInstallRow { pub async fn run(opts: IntegrateOptions, global: &GlobalOptions) -> Result<()> { ensure_global_scope(&opts)?; let cwd = default_cwd()?; + let config_paths = resolve_host_config_paths()?; let scope = InstallScope::Global; let detect = if needs_host_detection(&opts) { - detect_hosts(&cwd) + detect_hosts(&cwd, &config_paths) } else { DetectReport { cwd: cwd.display().to_string(), @@ -124,56 +132,89 @@ pub async fn run(opts: IntegrateOptions, global: &GlobalOptions) -> Result<()> { }; match opts.command { - Some(IntegrateCommand::List) => return run_list(global, &detect).await, + Some(IntegrateCommand::List) => return run_list(global, &detect, &config_paths).await, Some(IntegrateCommand::Detect) => { let detect = filter_detect_report(&detect, &explicit_hosts(&opts)); return run_detect(global, &detect).await; } Some(IntegrateCommand::Doctor { ref positional }) => { - return run_doctor(global, &opts, &cwd, scope, &detect, positional).await; - } - Some(IntegrateCommand::Uninstall { ref positional }) => { - return run_uninstall(global, &opts, &cwd, scope, &detect, positional).await; - } - Some(IntegrateCommand::Install { ref positional }) => { - return run_install( + return run_doctor( global, &opts, &cwd, + &config_paths, scope, &detect, positional, - InstallAction::Install, ) .await; } - Some(IntegrateCommand::Update { ref positional }) => { - return run_install( + Some(IntegrateCommand::Uninstall { ref positional }) => { + return run_uninstall( global, &opts, &cwd, + &config_paths, scope, &detect, positional, - InstallAction::Update, ) .await; } + Some(IntegrateCommand::Install { ref positional }) => { + return run_install(InstallRunParams { + global, + opts: &opts, + cwd: &cwd, + config_paths: &config_paths, + scope, + detect: &detect, + positional, + action: InstallAction::Install, + }) + .await; + } + Some(IntegrateCommand::Update { ref positional }) => { + return run_install(InstallRunParams { + global, + opts: &opts, + cwd: &cwd, + config_paths: &config_paths, + scope, + detect: &detect, + positional, + action: InstallAction::Update, + }) + .await; + } None => { - return run_install( + return run_install(InstallRunParams { global, - &opts, - &cwd, + opts: &opts, + cwd: &cwd, + config_paths: &config_paths, scope, - &detect, - &[], - InstallAction::Install, - ) + detect: &detect, + positional: &[], + action: InstallAction::Install, + }) .await; } } } +fn resolve_host_config_paths() -> Result { + let xdg_config_home = valid_xdg_config_home(env::var_os("XDG_CONFIG_HOME")); + Ok(HostConfigPaths::new(home_dir()?, xdg_config_home)) +} + +fn valid_xdg_config_home(value: Option) -> Option { + value + .filter(|value| !value.is_empty()) + .map(PathBuf::from) + .filter(|path| path.is_absolute()) +} + fn needs_host_detection(opts: &IntegrateOptions) -> bool { opts.command.is_none() || matches!( @@ -208,8 +249,12 @@ fn ensure_global_scope(opts: &IntegrateOptions) -> Result<()> { Ok(()) } -async fn run_list(global: &GlobalOptions, detect: &DetectReport) -> Result<()> { - let installs = list_owned_status(&all_hosts())?; +async fn run_list( + global: &GlobalOptions, + detect: &DetectReport, + config_paths: &HostConfigPaths, +) -> Result<()> { + let installs = list_owned_status(&all_hosts(), config_paths)?; let report = ListReport { supported: all_hosts().iter().map(|h| h.id()).collect(), detect: detect.clone(), @@ -246,25 +291,8 @@ async fn run_detect(global: &GlobalOptions, detect: &DetectReport) -> Result<()> Ok(()) } -async fn run_install( - global: &GlobalOptions, - opts: &IntegrateOptions, - cwd: &std::path::Path, - scope: InstallScope, - detect: &DetectReport, - positional: &[String], - action: InstallAction, -) -> Result<()> { - let mut progress = progress_for(global); - let params = InstallRunParams { - global, - opts, - cwd, - scope, - detect, - positional, - action, - }; +async fn run_install(params: InstallRunParams<'_>) -> Result<()> { + let mut progress = progress_for(params.global); let result = run_install_with_progress(params, progress.as_mut()).await; progress.finish(); result @@ -274,6 +302,7 @@ struct InstallRunParams<'a> { global: &'a GlobalOptions, opts: &'a IntegrateOptions, cwd: &'a std::path::Path, + config_paths: &'a HostConfigPaths, scope: InstallScope, detect: &'a DetectReport, positional: &'a [String], @@ -301,7 +330,13 @@ async fn run_install_with_progress( if will_prompt { progress.pause_for_input(); } - let hosts = resolve_hosts(params.opts, params.detect, params.positional, true)?; + let hosts = resolve_hosts( + params.opts, + params.detect, + params.config_paths, + params.positional, + true, + )?; if will_prompt { progress.resume_after_input(); } @@ -320,6 +355,7 @@ async fn run_install_with_progress( hosts: &hosts, scope: params.scope, cwd: params.cwd, + config_paths: params.config_paths, creds: &creds, force: params.opts.force, dry_run: params.opts.dry_run, @@ -334,6 +370,7 @@ async fn run_doctor( global: &GlobalOptions, opts: &IntegrateOptions, cwd: &std::path::Path, + config_paths: &HostConfigPaths, scope: InstallScope, detect: &DetectReport, positional: &[String], @@ -346,8 +383,8 @@ async fn run_doctor( "profile unavailable — running structural checks only (npx, config, ownership)", ); } - let hosts = resolve_hosts(opts, detect, positional, false)?; - let report = doctor_hosts(&hosts, scope, cwd, creds); + let hosts = resolve_hosts(opts, detect, config_paths, positional, false)?; + let report = doctor_hosts(&hosts, scope, cwd, config_paths, creds); emit(global.output, &report, global.quiet)?; print_doctor_summary(global, &report); if report.entries.is_empty() { @@ -364,12 +401,13 @@ async fn run_uninstall( global: &GlobalOptions, opts: &IntegrateOptions, cwd: &std::path::Path, + config_paths: &HostConfigPaths, scope: InstallScope, detect: &DetectReport, positional: &[String], ) -> Result<()> { - let hosts = resolve_hosts(opts, detect, positional, true)?; - let report = uninstall_hosts(&hosts, scope, cwd, opts.force, opts.dry_run)?; + let hosts = resolve_hosts(opts, detect, config_paths, positional, true)?; + let report = uninstall_hosts(&hosts, scope, cwd, config_paths, opts.force, opts.dry_run)?; emit(global.output, &report, global.quiet)?; for row in &report.results { if row.changed { @@ -388,6 +426,7 @@ async fn run_uninstall( fn resolve_hosts( opts: &IntegrateOptions, detect: &DetectReport, + config_paths: &HostConfigPaths, positional: &[String], require_automation_guard: bool, ) -> Result> { @@ -402,7 +441,7 @@ fn resolve_hosts( bail!("non-interactive session requires --yes and/or explicit --host"); } let detected = if detect.hosts.is_empty() { - detected_hosts(&detect_hosts(default_cwd()?.as_path())) + detected_hosts(&detect_hosts(default_cwd()?.as_path(), config_paths)) } else { detected_hosts(detect) }; @@ -609,6 +648,22 @@ mod parser_tests { assert_eq!(opts.hosts.len(), 2); } + #[test] + fn opencode_host_flag_parses() { + let opts = parse(&["am", "integrate", "--yes", "--global", "--host", "opencode"]); + assert_eq!(opts.hosts.len(), 1); + } + + #[test] + fn relative_xdg_config_home_is_ignored() { + assert_eq!(valid_xdg_config_home(Some("config".into())), None); + let absolute = std::env::temp_dir().join("config"); + assert_eq!( + valid_xdg_config_home(Some(absolute.clone().into_os_string())), + Some(absolute) + ); + } + #[test] fn hidden_project_flag_parses() { let opts = parse(&["am", "integrate", "install", "--project"]); diff --git a/crates/cli/src/commands/mod.rs b/crates/cli/src/commands/mod.rs index adc1c98..da1f01f 100644 --- a/crates/cli/src/commands/mod.rs +++ b/crates/cli/src/commands/mod.rs @@ -4,6 +4,7 @@ pub mod auth; pub mod client; pub mod cloud_api_key; pub mod config_cmd; +#[deny(clippy::disallowed_methods)] pub mod connect; pub mod connect_project; pub mod doctor_cmd; @@ -19,5 +20,6 @@ pub mod memory; pub mod migrate; pub mod org; pub mod project; +pub mod slm; pub mod trace; pub mod usage; diff --git a/crates/cli/src/commands/slm.rs b/crates/cli/src/commands/slm.rs new file mode 100644 index 0000000..62660fe --- /dev/null +++ b/crates/cli/src/commands/slm.rs @@ -0,0 +1,274 @@ +//! `am slm` — install/manage the host Metal SLM runtime (ATO-1936). + +use std::io::{self, IsTerminal}; +use std::time::Duration; + +use anyhow::{Context, Result, bail}; +use clap::Subcommand; +use serde::Serialize; + +use crate::cli::GlobalOptions; +use crate::output::{emit, message}; +use crate::progress::progress_for; +use crate::slm::{ + BootstrapConfirm, DEFAULT_SLM_PORT, RequiredModels, bootstrap_managed_slm, check_ready, + collect_status, confirm_models_pull, default_slm_paths, install_or_update, + manifest_url_from_env, pull_models, read_log_tail, status_models_json, stop_runtime, uninstall, +}; + +const READY_WAIT: Duration = Duration::from_secs(90); + +#[derive(Debug, Subcommand)] +pub enum SlmCommand { + /// Download and install the managed `am-slm` binary from the public manifest + Install, + /// Update the managed binary when a newer manifest version is published + Update, + /// Show managed runtime / model / health status (stable JSON) + Status, + /// Start `am-slm serve` and wait until `/health` + required models are ready + Start { + /// Listen port (default 8080) + #[arg(long, default_value_t = DEFAULT_SLM_PORT)] + port: u16, + /// Confirm SLM model download (~1.7GB) without prompting + #[arg(long)] + yes: bool, + }, + /// Stop the CLI-managed `am-slm` process + Stop, + /// Tail the managed runtime log + Logs { + /// Max bytes to print from the end of the log + #[arg(long, default_value_t = 16_384)] + bytes: usize, + }, + /// Remove the managed binary (and optionally model cache) + Uninstall { + /// Also delete the downloaded model cache (~1.7GB) + #[arg(long)] + purge_models: bool, + /// Confirm destructive uninstall + #[arg(long)] + yes: bool, + }, + /// Model cache lifecycle + #[command(subcommand)] + Models(SlmModelsCommand), +} + +#[derive(Debug, Subcommand)] +pub enum SlmModelsCommand { + /// Download model weights (~1.7GB; requires `--yes` or interactive confirmation) + Pull { + /// Pull a specific adapter (default: Qwen + Nomic + am-slm-core) + #[arg(long)] + adapter: Option, + /// Confirm the large download without prompting + #[arg(long)] + yes: bool, + }, + /// Show model cache status (`am-slm models status --json`) + Status, +} + +pub async fn run(cmd: SlmCommand, global: &GlobalOptions) -> Result<()> { + let client = reqwest::Client::builder() + .timeout(Duration::from_secs(120)) + .build() + .context("build HTTP client")?; + let paths = default_slm_paths()?; + let manifest_url = manifest_url_from_env(); + + match cmd { + SlmCommand::Install | SlmCommand::Update => { + let outcome = install_or_update(&client, &paths, &manifest_url).await?; + emit(global.output, &outcome, global.quiet)?; + if !global.quiet { + message( + true, + &format!( + "am-slm {} installed at {}{}", + outcome.version, + outcome.binary_path.display(), + if outcome.replaced_existing { + " (replaced existing)" + } else { + "" + } + ), + ); + message( + true, + "Next: am slm models pull --yes && am slm start (or am init --local --slm)", + ); + } + } + SlmCommand::Status => { + let endpoint = format!( + "http://127.0.0.1:{}", + paths + .read_state() + .ok() + .and_then(|s| s.port) + .unwrap_or(DEFAULT_SLM_PORT) + ); + let models = check_ready(&client, &endpoint).await.ok(); + let health_ok = models + .as_ref() + .map(|m: &RequiredModels| m.ready()) + .unwrap_or(false); + let status = collect_status(&client, &paths, None, models, health_ok).await?; + emit(global.output, &status, global.quiet)?; + } + SlmCommand::Start { port, yes } => { + let allow_prompt = global.allow_prompts(yes) && io::stdin().is_terminal(); + let quiet = global.quiet || global.output == crate::cli::OutputFormat::Json; + let outcome = bootstrap_managed_slm( + &client, + &paths, + BootstrapConfirm { yes, allow_prompt }, + port, + READY_WAIT, + |step| { + if !quiet { + message(true, step); + } + }, + ) + .await?; + emit(global.output, &outcome, global.quiet)?; + if !global.quiet { + message( + true, + &format!( + "am-slm ready on {} (pid {}, already_running={})", + outcome.endpoint, outcome.pid, outcome.already_running + ), + ); + } + } + SlmCommand::Stop => { + let outcome = stop_runtime(&paths).await?; + emit(global.output, &outcome, global.quiet)?; + if !global.quiet { + message(true, &outcome.message); + } + } + SlmCommand::Logs { bytes } => { + let tail = read_log_tail(&paths, bytes)?; + if global.output == crate::cli::OutputFormat::Json { + #[derive(Serialize)] + struct LogsOut<'a> { + log: &'a str, + } + emit(global.output, &LogsOut { log: &tail }, global.quiet)?; + } else { + print!("{tail}"); + } + } + SlmCommand::Uninstall { purge_models, yes } => { + if !yes { + bail!("refusing to uninstall without `--yes`"); + } + let _ = stop_runtime(&paths).await; + uninstall(&paths, purge_models)?; + #[derive(Serialize)] + struct UninstallOut { + uninstalled: bool, + purged_models: bool, + } + emit( + global.output, + &UninstallOut { + uninstalled: true, + purged_models: purge_models, + }, + global.quiet, + )?; + if !global.quiet { + message( + true, + if purge_models { + "am-slm uninstalled (models purged)" + } else { + "am-slm uninstalled (model cache retained)" + }, + ); + } + } + SlmCommand::Models(models_cmd) => match models_cmd { + SlmModelsCommand::Pull { adapter, yes } => { + let allow_prompt = global.allow_prompts(yes) && io::stdin().is_terminal(); + if !confirm_models_pull(yes, allow_prompt)? { + bail!("model download cancelled"); + } + let mut progress = progress_for(global); + progress.start_step("models", "Download SLM models"); + let result = pull_models(&paths, adapter.as_deref(), |detail| { + progress.tick("models", detail) + }) + .await; + match &result { + Ok(_) => progress.succeed("models", Some("download complete")), + Err(_) => progress.fail( + "models", + Some("cache retained; retry with am slm models pull --yes"), + ), + } + progress.finish(); + let outcome = result?; + emit(global.output, &outcome, global.quiet)?; + if !global.quiet { + message(true, &outcome.message); + } + } + SlmModelsCommand::Status => { + let value = status_models_json(&paths).await?; + emit(global.output, &value, global.quiet)?; + } + }, + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use clap::Parser; + + #[derive(Debug, Parser)] + struct Probe { + #[command(subcommand)] + cmd: SlmCommand, + } + + #[test] + fn parses_install_and_models_pull_yes() { + let p = Probe::try_parse_from(["slm", "install"]).unwrap(); + assert!(matches!(p.cmd, SlmCommand::Install)); + let p = Probe::try_parse_from(["slm", "models", "pull", "--yes"]).unwrap(); + match p.cmd { + SlmCommand::Models(SlmModelsCommand::Pull { yes, .. }) => assert!(yes), + other => panic!("unexpected {other:?}"), + } + } + + #[test] + fn parses_start_yes_and_adapter_pull() { + let p = Probe::try_parse_from(["slm", "start", "--yes"]).unwrap(); + match p.cmd { + SlmCommand::Start { yes, .. } => assert!(yes), + other => panic!("unexpected {other:?}"), + } + let p = + Probe::try_parse_from(["slm", "models", "pull", "--adapter", "am-slm-core"]).unwrap(); + match p.cmd { + SlmCommand::Models(SlmModelsCommand::Pull { adapter, yes }) => { + assert_eq!(adapter.as_deref(), Some("am-slm-core")); + assert!(!yes); + } + other => panic!("unexpected {other:?}"), + } + } +} diff --git a/crates/cli/src/config.rs b/crates/cli/src/config.rs index d4f6d35..9e7d770 100644 --- a/crates/cli/src/config.rs +++ b/crates/cli/src/config.rs @@ -9,7 +9,7 @@ use anyhow::{Context, Result, anyhow, bail}; use fs4::fs_std::FileExt; use serde::{Deserialize, Deserializer, Serialize}; -use crate::auth::origin::check_api_key_origin; +use crate::auth::origin::{check_api_key_origin, same_origin}; use crate::environment::{BaseUrlInput, Environment, is_remote_cloud_api_url, resolve_base_url}; pub use crate::environment::ENV_CORE_IMAGE; @@ -839,6 +839,7 @@ pub fn local_profile_cloud_export_warning( resolved_base_url: &str, env_api_key: Option<&str>, local_memory_url: &str, + stored_profile_base_url: Option<&str>, ) -> Option { if stored_kind != ProfileKind::Local { return None; @@ -847,6 +848,9 @@ pub fn local_profile_cloud_export_warning( if !is_remote_cloud_api_url(resolved_base_url) { return None; } + if stored_profile_base_url.is_some_and(|stored| same_origin(stored, resolved_base_url)) { + return None; + } if env_api_key.is_some_and(is_cloud_api_key) { return None; } @@ -2395,12 +2399,42 @@ api_key_ref = "hosted-cloud-proj_a" Environment::PROD_BASE_URL, None, "http://127.0.0.1:17350", + None, ) .expect("expected warning"); assert!(warning.contains("active profile is Local")); assert!(warning.contains("127.0.0.1:17350")); } + #[test] + fn local_profile_cloud_export_warning_suppressed_when_pin_matches_stored_cloud() { + assert!( + super::local_profile_cloud_export_warning( + ProfileKind::Local, + Some(Environment::PROD_BASE_URL), + Environment::PROD_BASE_URL, + None, + "http://127.0.0.1:17350", + Some(Environment::PROD_BASE_URL), + ) + .is_none() + ); + } + + #[test] + fn local_profile_cloud_export_warning_when_pin_differs_from_stored_cloud() { + let warning = super::local_profile_cloud_export_warning( + ProfileKind::Local, + Some("https://api.dest.example/"), + "https://api.dest.example/", + None, + "http://127.0.0.1:17350", + Some(Environment::PROD_BASE_URL), + ) + .expect("expected warning"); + assert!(warning.contains("active profile is Local")); + } + #[test] fn local_profile_cloud_export_warning_is_suppressed_with_exported_cloud_key() { assert!( @@ -2410,6 +2444,7 @@ api_key_ref = "hosted-cloud-proj_a" Environment::PROD_BASE_URL, Some("amc_dashboard_key"), "http://127.0.0.1:17350", + None, ) .is_none() ); diff --git a/crates/cli/src/instance/address.rs b/crates/cli/src/instance/address.rs new file mode 100644 index 0000000..5fa8b40 --- /dev/null +++ b/crates/cli/src/instance/address.rs @@ -0,0 +1,72 @@ +//! Validated loopback addresses for the managed Docker binding. + +use anyhow::{Result, bail}; +use url::Url; + +/// A managed endpoint, derived from a loopback host and validated port. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ManagedAddress { + port: u16, +} + +impl ManagedAddress { + /// Accept only HTTP numeric IPv4 loopback or localhost, without URL extras. + pub fn parse(value: &str) -> Result { + let url = Url::parse(value)?; + if url.scheme() != "http" + || !matches!(url.host_str(), Some("127.0.0.1" | "localhost")) + || !url.username().is_empty() + || url.password().is_some() + || url.query().is_some() + || url.fragment().is_some() + || !matches!(url.path(), "" | "/") + { + bail!( + "managed Core requires http://127.0.0.1: (or localhost); use --no-instance for an externally managed Core" + ); + } + let port = url + .port_or_known_default() + .filter(|port| *port > 0) + .ok_or_else(|| anyhow::anyhow!("managed Core requires a nonzero TCP port"))?; + Ok(Self { port }) + } + + /// The host-side Docker port. + pub fn port(&self) -> u16 { + self.port + } + + /// Canonical URL used for published labels and authenticated probes. + pub fn url(&self) -> String { + format!("http://127.0.0.1:{}", self.port) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn preserves_custom_port_and_normalizes_localhost() { + let address = ManagedAddress::parse("http://localhost:17352/").unwrap(); + assert_eq!(address.port(), 17352); + assert_eq!(address.url(), "http://127.0.0.1:17352"); + } + + #[test] + fn refuses_remote_or_ambiguous_managed_addresses() { + for value in [ + "https://127.0.0.1:17352", + "http://example.com:17352", + "http://127.0.0.1:0", + "http://user@localhost:17352", + "http://localhost:17352/path", + "http://localhost:17352?x=1", + "http://localhost:17352#x", + "http://localhost.example.com:17352", + ] { + assert!(ManagedAddress::parse(value).is_err(), "{value}"); + } + } +} diff --git a/crates/cli/src/instance/credentials.rs b/crates/cli/src/instance/credentials.rs new file mode 100644 index 0000000..cf52770 --- /dev/null +++ b/crates/cli/src/instance/credentials.rs @@ -0,0 +1,63 @@ +//! Resolve a Core credential only from the selected provider's state volume. + +use super::docker::{DockerRunner, InstanceConfig}; +use anyhow::{Result, bail}; + +/// Preserve the selected dataset's bearer unless an explicit replacement or reset was requested. +pub async fn resolve_dataset_key( + docker: &dyn DockerRunner, + config: &InstanceConfig, + reset: bool, + explicit: Option, + replace: bool, +) -> Result { + let saved = docker + .read_volume_core_api_key(&config.storage.state, &config.image) + .await?; + select_key(saved, reset, explicit, replace) +} + +fn select_key( + saved: Option, + reset: bool, + explicit: Option, + replace: bool, +) -> Result { + if let (Some(saved), Some(explicit)) = (&saved, &explicit) + && saved != explicit + && !replace + { + bail!( + "CORE_API_KEY differs from the selected dataset's persisted key; use --replace to authorize updating it" + ); + } + Ok(explicit + .or(if reset { None } else { saved }) + .unwrap_or_else(super::generate_core_api_key)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn credentials_are_reused_until_an_explicit_reset() { + let saved = Some("provider-secret".into()); + assert_eq!( + select_key(saved.clone(), false, None, false).unwrap(), + "provider-secret" + ); + let reset = select_key(saved, true, None, false).unwrap(); + assert_eq!(reset.len(), 64); + assert_ne!(reset, "provider-secret"); + } + + #[test] + fn conflicting_override_requires_explicit_replacement() { + assert!(select_key(Some("saved".into()), false, Some("new".into()), false).is_err()); + assert_eq!( + select_key(Some("saved".into()), false, Some("new".into()), true).unwrap(), + "new" + ); + } +} diff --git a/crates/cli/src/instance/docker.rs b/crates/cli/src/instance/docker.rs index 125c425..68b6b98 100644 --- a/crates/cli/src/instance/docker.rs +++ b/crates/cli/src/instance/docker.rs @@ -11,12 +11,12 @@ use serde::{Deserialize, Serialize}; use tokio::process::Command; use tracing::instrument; -use super::{ - DEFAULT_CONTAINER_NAME, LOCAL_URL_LABEL, MANAGED_BY_LABEL, PROFILE_LABEL_PREFIX, VOLUME_DATA, - VOLUME_STATE, -}; +use super::{DEFAULT_CONTAINER_NAME, LOCAL_URL_LABEL, MANAGED_BY_LABEL, PROFILE_LABEL_PREFIX}; use crate::environment::{cloud_tier_from_api_url, image_has_registry}; +mod core_instance_id; +mod storage_ops; + /// Runtime configuration for a managed Core container. #[derive(Debug, Clone, PartialEq, Eq)] pub struct InstanceConfig { @@ -26,6 +26,9 @@ pub struct InstanceConfig { pub profile_name: String, /// Published local Core URL baked into container labels for origin binding. pub local_url: String, + pub storage: super::storage::StorageVolumes, + pub provider: super::storage::Provider, + pub project_id: Option, } /// Environment variables forwarded to `docker run` (values via child env, not argv). @@ -37,6 +40,8 @@ pub struct InstanceEnv { pub cloud_jwks_url: String, /// Explicit operator override forwarded as `CORE_API_KEY` (omit for Core auto-generation). pub core_api_key: Option, + /// When true, Core uses host Metal SLM (openai-compatible) instead of OpenAI. + pub slm: bool, } /// Map a Cloud API base URL to Core's `CLOUD_ENV` tier label. @@ -54,16 +59,22 @@ pub(crate) fn inspect_stderr_means_missing(stderr: &str) -> bool { impl InstanceEnv { /// Env var names passed to Docker — values live in the child process environment. pub fn docker_env_names(&self) -> Vec<&'static str> { - let mut names = vec![ - "OPENAI_API_KEY", + let mut names = Vec::new(); + if self.slm { + names.extend(crate::slm::slm_docker_env_names()); + } else { + names.push("OPENAI_API_KEY"); + } + names.extend([ "ATOMICMEMORY_API_KEY", + "ALLOWED_ORIGINS", "ATOMICMEMORY_API_URL", "CLOUD_TRACE_SYNC_ENABLED", "CLOUD_JWKS_URL", "CLOUD_ENV", "CLOUD_JWT_ISSUER", "CLOUD_JWT_AUDIENCE", - ]; + ]); if self.core_api_key.is_some() { names.push("CORE_API_KEY"); } @@ -78,11 +89,25 @@ impl InstanceEnv { .trim_end_matches('/') .to_string(); let mut env = HashMap::new(); - env.insert("OPENAI_API_KEY".into(), self.openai_api_key.clone()); + if self.slm { + for (k, v) in crate::slm::slm_child_env_entries() { + env.insert(k, v); + } + } else { + env.insert("OPENAI_API_KEY".into(), self.openai_api_key.clone()); + } env.insert( "ATOMICMEMORY_API_KEY".into(), self.atomicmemory_api_key.clone(), ); + // Connected Local publishes Core only on loopback, so the CLI can + // provide the explicit CORS contract the image requires for Cloud sync. + env.insert( + "ALLOWED_ORIGINS".into(), + format!( + "http://localhost:{DEFAULT_HOST_PORT},http://{DEFAULT_BIND_HOST}:{DEFAULT_HOST_PORT}" + ), + ); env.insert("ATOMICMEMORY_API_URL".into(), api_url.clone()); env.insert("CLOUD_TRACE_SYNC_ENABLED".into(), "true".into()); env.insert("CLOUD_JWKS_URL".into(), self.cloud_jwks_url.clone()); @@ -147,6 +172,7 @@ pub struct ContainerInspect { pub atomicmemory_api_key: Option, /// Local Core URL label from `docker run` (`ai.atomicstrata.local-url`). pub local_url: Option, + pub storage: Option, } struct InspectedEnv { @@ -209,6 +235,9 @@ pub fn default_instance_config(profile_name: &str, image: &str) -> InstanceConfi host_port: DEFAULT_HOST_PORT, profile_name: profile_name.to_string(), local_url: managed_core_local_url(), + storage: super::storage::StorageVolumes::legacy(), + provider: super::storage::Provider::Openai, + project_id: None, } } @@ -228,7 +257,7 @@ pub fn managed_core_local_url() -> String { pub fn build_run_argv(config: &InstanceConfig, env: &InstanceEnv) -> Vec { let bind = format!( "{DEFAULT_BIND_HOST}:{}:{}", - config.host_port, config.host_port + config.host_port, DEFAULT_HOST_PORT ); let mut argv = vec!["run".into(), "-d".into()]; if image_has_registry(&config.image) { @@ -243,9 +272,9 @@ pub fn build_run_argv(config: &InstanceConfig, env: &InstanceEnv) -> Vec "-p".into(), bind, "-v".into(), - format!("{VOLUME_DATA}:/var/lib/atomicmemory/postgres"), + format!("{}:/var/lib/atomicmemory/postgres", config.storage.data), "-v".into(), - format!("{VOLUME_STATE}:/var/lib/atomicmemory/state"), + format!("{}:/var/lib/atomicmemory/state", config.storage.state), "--label".into(), MANAGED_BY_LABEL.into(), "--label".into(), @@ -253,10 +282,21 @@ pub fn build_run_argv(config: &InstanceConfig, env: &InstanceEnv) -> Vec "--label".into(), format!("{LOCAL_URL_LABEL}={}", config.local_url), ]); + if let Some(project_id) = &config.project_id { + argv.extend([ + "--label".into(), + format!("ai.atomicstrata.project-id={project_id}"), + ]); + } for name in env.docker_env_names() { argv.push("--env".into()); argv.push(name.into()); } + if env.slm { + for arg in crate::slm::SLM_ADD_HOST_ARGS { + argv.push((*arg).into()); + } + } argv.push(config.image.clone()); argv } @@ -288,24 +328,54 @@ pub async fn ensure_docker_available_with_preflight( docker: &dyn DockerRunner, interactive: bool, ) -> Result<()> { - if docker.version().await.is_ok() { - return Ok(()); - } - eprintln!("\n{}\n", docker_install_links()); - if interactive && io::stdin().is_terminal() { - eprint!("Press Enter after Docker is installed and running (Ctrl+C to abort)… "); - io::stderr().flush().ok(); - let mut line = String::new(); - io::stdin() - .read_line(&mut line) - .context("read docker preflight confirmation")?; + let error = match docker.version().await { + Ok(()) => return Ok(()), + Err(error) => error, + }; + if !interactive || !io::stdin().is_terminal() { + return Err(error); } + eprintln!("{error}"); + eprint!("Press Enter after Docker is ready (Ctrl+C to abort)… "); + io::stderr().flush()?; + let mut line = String::new(); + io::stdin() + .read_line(&mut line) + .context("read Docker readiness confirmation")?; + ensure_docker_available(docker).await } #[async_trait::async_trait] pub trait DockerRunner: Send + Sync { async fn version(&self) -> Result<()>; + /// Ensure the selected dataset exists and is owned by this installation. + async fn prepare_storage( + &self, + _config: &InstanceConfig, + _recorded: Option<&super::storage::StorageIdentity>, + _live_legacy: bool, + ) -> Result { + bail!("dataset preparation is not supported by this Docker runner") + } + /// Revalidate existing ownership without creating or deleting any volume. + async fn validate_storage( + &self, + _config: &InstanceConfig, + _recorded: Option<&super::storage::StorageIdentity>, + _live_legacy: bool, + ) -> Result { + bail!("dataset validation is not supported by this Docker runner") + } + /// Read the selected provider state without starting its database. + async fn read_volume_core_api_key(&self, _name: &str, _image: &str) -> Result> { + bail!("volume key retrieval is not supported by this Docker runner") + } + + /// Configured loopback publication for validating a stopped container before restart. + async fn configured_local_url(&self, _name: &str) -> Result> { + Ok(None) + } async fn inspect(&self, name: &str) -> Result>; async fn run(&self, config: &InstanceConfig, env: &InstanceEnv) -> Result; async fn start(&self, name: &str) -> Result<()>; @@ -314,6 +384,10 @@ pub trait DockerRunner: Send + Sync { async fn logs_tail(&self, name: &str, tail: u32) -> Result; async fn logs_follow(&self, name: &str, tail: u32) -> Result<()>; async fn volume_rm(&self, name: &str) -> Result<()>; + /// Read the persisted runtime identity after the caller verifies container ownership. + async fn read_core_instance_id(&self, _name: &str) -> Result> { + bail!("runtime identity retrieval is not supported by this Docker runner") + } /// Read Core's persisted local client key from the state volume (secrets not in argv). async fn read_core_api_key(&self, name: &str) -> Result>; } @@ -337,6 +411,7 @@ impl RealDockerRunner { ) -> Result<(i32, String, String)> { let mut cmd = Command::new(&self.docker_bin); cmd.args(args); + cmd.kill_on_drop(true); cmd.stdout(Stdio::piped()); cmd.stderr(Stdio::piped()); if let Some(env) = child_env { @@ -357,6 +432,7 @@ impl RealDockerRunner { async fn exec_inherit(&self, args: &[&str]) -> Result { let output = Command::new(&self.docker_bin) .args(args) + .kill_on_drop(true) .stdout(Stdio::inherit()) .stderr(Stdio::inherit()) .output() @@ -374,15 +450,62 @@ impl Default for RealDockerRunner { #[async_trait::async_trait] impl DockerRunner for RealDockerRunner { + async fn configured_local_url(&self, name: &str) -> Result> { + let (code, stdout, _) = self + .exec_capture(&["inspect", "--type", "container", name], None) + .await?; + if code != 0 { + bail!("could not inspect configured Core port; no restart was attempted"); + } + let entries: Vec = + serde_json::from_str(&stdout).context("parse configured Core binding")?; + let Some(value) = entries + .first() + .and_then(|entry| entry.pointer("/HostConfig/PortBindings/17350~1tcp")) + else { + return Ok(None); + }; + if value.is_null() { + return Ok(None); + } + let bindings: Vec = + serde_json::from_value(value.clone()).context("parse configured Core port")?; + Ok(loopback_binding(&bindings)) + } + async fn prepare_storage( + &self, + config: &InstanceConfig, + recorded: Option<&super::storage::StorageIdentity>, + live_legacy: bool, + ) -> Result { + storage_ops::ensure(self, config, recorded, live_legacy).await + } + async fn validate_storage( + &self, + config: &InstanceConfig, + recorded: Option<&super::storage::StorageIdentity>, + live_legacy: bool, + ) -> Result { + storage_ops::validate(self, config, recorded, live_legacy).await + } + async fn read_volume_core_api_key(&self, name: &str, image: &str) -> Result> { + storage_ops::read_key(self, name, image).await + } + #[instrument(skip(self))] async fn version(&self) -> Result<()> { - let (code, _, stderr) = self.exec_capture(&["version"], None).await?; - if code != 0 { - bail!( - "docker is not available or the daemon is not running\n{stderr}\n\ - {install_links}", - install_links = docker_install_links() - ); + let output = Command::new(&self.docker_bin).arg("version").output().await; + match output { + Err(err) if err.kind() == io::ErrorKind::NotFound => { + bail!("Docker CLI is not installed. {}", docker_install_links()) + } + Err(err) => { + return Err(err).context("could not run Docker CLI; check installation and retry"); + } + Ok(output) if !output.status.success() => bail!( + "Docker is installed, but its daemon is unavailable. Start Docker, confirm with `docker version`, then rerun the setup command." + ), + Ok(_) => {} } Ok(()) } @@ -396,8 +519,7 @@ impl DockerRunner for RealDockerRunner { if inspect_stderr_means_missing(&stderr) { return Ok(None); } - tracing::warn!(%stderr, "docker inspect failed"); - return Ok(None); + bail!("Docker container inspection failed; check the daemon and retry"); } let entries: Vec = serde_json::from_str(stdout.trim()).context("parse docker inspect JSON")?; @@ -405,13 +527,14 @@ impl DockerRunner for RealDockerRunner { .into_iter() .next() .ok_or_else(|| anyhow::anyhow!("docker inspect returned empty array for '{name}'"))?; - let labels = entry.config.labels.unwrap_or_default(); + let labels = entry.config.labels.clone().unwrap_or_default(); let managed = labels .get("ai.atomicstrata.managed-by") .map(|v| v == "am-cli") .unwrap_or(false); let profile_label = labels.get("ai.atomicstrata.profile").cloned(); - let local_url = labels.get(LOCAL_URL_LABEL).cloned(); + let local_url = inspected_binding(&entry); + let storage = inspected_storage(&entry, &labels); let env = env_from_docker_inspect(entry.config.env.as_deref()); Ok(Some(ContainerInspect { name: name.to_string(), @@ -424,6 +547,7 @@ impl DockerRunner for RealDockerRunner { core_api_key: env.core_api_key, atomicmemory_api_key: env.atomicmemory_api_key, local_url, + storage, })) } @@ -515,12 +639,16 @@ impl DockerRunner for RealDockerRunner { #[instrument(skip(self), fields(volume = name))] async fn volume_rm(&self, name: &str) -> Result<()> { let (code, _, stderr) = self.exec_capture(&["volume", "rm", name], None).await?; - if code != 0 && !stderr.contains("No such volume") { + if code != 0 && !stderr.to_ascii_lowercase().contains("no such volume") { bail!("docker volume rm failed: {stderr}"); } Ok(()) } + async fn read_core_instance_id(&self, name: &str) -> Result> { + core_instance_id::read(&self.docker_bin, name).await + } + #[instrument(skip(self), fields(container = name))] async fn read_core_api_key(&self, name: &str) -> Result> { let (code, stdout, stderr) = self @@ -533,8 +661,7 @@ impl DockerRunner for RealDockerRunner { { return Ok(None); } - tracing::warn!(%stderr, "docker exec read core-api-key failed"); - return Ok(None); + bail!("could not read the managed Core credential; check Docker and retry"); } let key = stdout.trim().to_string(); if key.is_empty() { @@ -551,6 +678,92 @@ struct InspectEntry { state: InspectState, #[serde(default)] config: InspectConfig, + #[serde(default)] + mounts: Vec, + #[serde(default)] + network_settings: InspectNetwork, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "PascalCase")] +struct InspectMount { + name: Option, + destination: String, + #[serde(rename = "Type")] + kind: String, +} +#[derive(Debug, Default, Deserialize)] +#[serde(rename_all = "PascalCase")] +struct InspectNetwork { + #[serde(default)] + ports: HashMap>>, +} +#[derive(Debug, Deserialize)] +#[serde(rename_all = "PascalCase")] +struct PortBinding { + host_ip: String, + host_port: String, +} + +fn inspected_binding(entry: &InspectEntry) -> Option { + let bindings = entry.network_settings.ports.get("17350/tcp")?.as_ref()?; + loopback_binding(bindings) +} + +fn loopback_binding(bindings: &[PortBinding]) -> Option { + if bindings.len() != 1 || bindings[0].host_ip != DEFAULT_BIND_HOST { + return None; + } + let port = bindings[0] + .host_port + .parse::() + .ok() + .filter(|port| *port > 0)?; + Some(format!("http://127.0.0.1:{port}")) +} + +fn inspected_storage( + entry: &InspectEntry, + labels: &HashMap, +) -> Option { + use super::storage::{ObservedStorage, Provider, StorageVolumes}; + let env: HashMap<&str, &str> = entry + .config + .env + .as_ref()? + .iter() + .filter_map(|e| e.split_once('=')) + .collect(); + let provider = if env.get("EMBEDDING_PROVIDER") == Some(&"openai-compatible") + && env.get("EMBEDDING_DIMENSIONS") == Some(&"768") + && env.get("EMBEDDING_MODEL") == Some(&crate::slm::SLM_EMBED_MODEL) + && env.get("LLM_MODEL") == Some(&crate::slm::SLM_CHAT_MODEL) + { + Provider::Slm + } else if env.contains_key("OPENAI_API_KEY") + && matches!(env.get("EMBEDDING_PROVIDER"), None | Some(&"openai")) + && matches!(env.get("EMBEDDING_DIMENSIONS"), None | Some(&"1536")) + { + Provider::Openai + } else { + return None; + }; + let named = |destination: &str| { + entry + .mounts + .iter() + .find(|m| m.kind == "volume" && m.destination == destination)? + .name + .clone() + }; + Some(ObservedStorage { + provider, + volumes: StorageVolumes { + data: named("/var/lib/atomicmemory/postgres")?, + state: named("/var/lib/atomicmemory/state")?, + }, + project_id: labels.get("ai.atomicstrata.project-id").cloned(), + }) } #[derive(Debug, Deserialize, Default)] @@ -634,6 +847,7 @@ mod tests { atomicmemory_api_url: "https://api.atomicstrata.ai".into(), cloud_jwks_url: "https://api.atomicstrata.ai/.well-known/jwks.json".into(), core_api_key: None, + slm: false, }; let argv = build_run_argv(&config, &env); let bind = format!("{DEFAULT_BIND_HOST}:{DEFAULT_HOST_PORT}:{DEFAULT_HOST_PORT}"); @@ -657,6 +871,74 @@ mod tests { const DEV_IMAGE: &str = "ghcr.io/atomicstrata/atomicmemory-core:test"; const PROD_IMAGE: &str = Environment::PROD_CORE_IMAGE; + #[cfg(unix)] + fn cli_env_passes_connected_local_entrypoint(slm: bool) { + use std::fs; + use std::os::unix::fs::PermissionsExt; + use std::process::Command; + use std::time::{SystemTime, UNIX_EPOCH}; + + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("system clock after epoch") + .as_nanos(); + let work_dir = std::env::temp_dir().join(format!( + "atomicmemory-cli-entrypoint-{}-{nonce}", + std::process::id() + )); + let bin_dir = work_dir.join("bin"); + fs::create_dir_all(&bin_dir).expect("create fixture bin directory"); + let gosu = bin_dir.join("gosu"); + fs::write(&gosu, "#!/usr/bin/env bash\nshift\nexec \"$@\"\n").expect("write fixture gosu"); + fs::set_permissions(&gosu, fs::Permissions::from_mode(0o755)) + .expect("make fixture gosu executable"); + + let mut env = test_env_for_port(); + if slm { + crate::slm::apply_slm_overlay(&mut env); + } + let child_env = env.as_child_env(); + let entrypoint = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../packages/core/scripts/docker-entrypoint.sh"); + let inherited_path = std::env::var("PATH").expect("PATH must be set"); + let output = { + let mut command = Command::new("bash"); + command + .arg(entrypoint) + .arg("true") + .env_clear() + .env("PATH", format!("{}:{inherited_path}", bin_dir.display())) + .env("CORE_STATE_DIR", work_dir.join("state")) + .env("DATABASE_URL", "postgresql://fixture.example/atomicmemory") + .env("ATOMICMEMORY_RUN_MIGRATIONS_ON_STARTUP", "false"); + for name in env.docker_env_names() { + command.env( + name, + child_env + .get(name) + .expect("every Docker env name must have a generated value"), + ); + } + command.output().expect("run actual Core entrypoint") + }; + let _ = fs::remove_dir_all(&work_dir); + + assert!( + output.status.success(), + "CLI-generated {} environment must pass the actual entrypoint:\nstdout:\n{}\nstderr:\n{}", + if slm { "SLM" } else { "OpenAI" }, + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + } + + #[cfg(unix)] + #[test] + fn cli_generated_openai_and_slm_envs_pass_connected_local_entrypoint() { + cli_env_passes_connected_local_entrypoint(false); + cli_env_passes_connected_local_entrypoint(true); + } + #[test] fn env_from_docker_inspect_parses_cloud_urls() { let env = vec![ @@ -720,6 +1002,7 @@ mod tests { cloud_jwks_url: "https://api.staging.example.com/.well-known/atomic-core/jwks.json" .into(), core_api_key: None, + slm: false, }; let child = env.as_child_env(); assert_eq!(child.get("CLOUD_ENV").map(String::as_str), Some("custom")); @@ -779,6 +1062,7 @@ mod tests { atomicmemory_api_url: String::new(), cloud_jwks_url: String::new(), core_api_key: None, + slm: false, }; let argv = build_run_argv(&config, &env); assert!(argv.contains(&"run".to_string())); @@ -823,6 +1107,7 @@ mod tests { atomicmemory_api_url: String::new(), cloud_jwks_url: String::new(), core_api_key: None, + slm: false, }; let argv = build_run_argv(&config, &env); assert!(argv.contains(&"--pull".to_string())); @@ -838,6 +1123,7 @@ mod tests { atomicmemory_api_url: "https://api.dev.example.com".into(), cloud_jwks_url: "https://api.dev.example.com/jwks.json".into(), core_api_key: Some("core-secret".into()), + slm: false, }; let argv = build_run_argv(&config, &env); let joined = argv.join(" "); @@ -855,6 +1141,7 @@ mod tests { atomicmemory_api_url: String::new(), cloud_jwks_url: String::new(), core_api_key: None, + slm: false, }; let argv = build_run_argv(&config, &env); assert_eq!(argv.last().map(String::as_str), Some("my/core:v2")); @@ -868,6 +1155,7 @@ mod tests { atomicmemory_api_url: "https://api.dev.example.com".into(), cloud_jwks_url: "https://api.dev.example.com/.well-known/atomic-core/jwks.json".into(), core_api_key: None, + slm: false, }; let child = env.as_child_env(); assert_eq!( @@ -971,4 +1259,32 @@ mod tests { let runner = RealDockerRunner::new(); runner.version().await.expect("docker version"); } + #[test] + fn custom_host_port_maps_to_the_fixed_core_port() { + let mut config = default_instance_config("local", DEV_IMAGE); + config.host_port = 17352; + config.local_url = "http://127.0.0.1:17352".into(); + let args = build_run_argv(&config, &test_env_for_port()); + assert!( + args.windows(2) + .any(|pair| pair == ["-p", "127.0.0.1:17352:17350"]) + ); + } + + fn test_env_for_port() -> InstanceEnv { + InstanceEnv { + openai_api_key: "test".into(), + atomicmemory_api_key: "amc_test".into(), + atomicmemory_api_url: "https://api.example.com".into(), + cloud_jwks_url: "https://api.example.com/jwks".into(), + core_api_key: None, + slm: false, + } + } } + +#[cfg(all(test, unix))] +mod recovery_tests; + +#[cfg(test)] +mod live_storage_tests; diff --git a/crates/cli/src/instance/docker/core_instance_id.rs b/crates/cli/src/instance/docker/core_instance_id.rs new file mode 100644 index 0000000..bbce299 --- /dev/null +++ b/crates/cli/src/instance/docker/core_instance_id.rs @@ -0,0 +1,64 @@ +//! Read the persisted runtime identity with bounded Docker output and lifetime. + +use std::process::Stdio; +use std::time::Duration; + +use anyhow::{Context, Result, bail}; +use tokio::io::AsyncReadExt; +use tokio::process::Command; + +const INSTANCE_ID_PATH: &str = "/var/lib/atomicmemory/state/core-instance-id"; +const MAX_INSTANCE_ID_BYTES: usize = 2048; +const MAX_INSTANCE_ID_CHARS: usize = 512; +const ID_READ_TIMEOUT: Duration = Duration::from_secs(10); + +pub(super) async fn read(binary: &str, container: &str) -> Result> { + let mut child = Command::new(binary) + .args(["exec", container, "cat", INSTANCE_ID_PATH]) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .kill_on_drop(true) + .spawn() + .context("read managed Core runtime identity")?; + let stdout = child + .stdout + .take() + .context("capture Core runtime identity")?; + let operation = async { + let mut bytes = Vec::new(); + stdout + .take((MAX_INSTANCE_ID_BYTES + 1) as u64) + .read_to_end(&mut bytes) + .await?; + if bytes.len() > MAX_INSTANCE_ID_BYTES { + bail!("Core runtime identity exceeds its size limit"); + } + let status = child.wait().await?; + if !status.success() { + bail!( + "cannot read persisted Core runtime identity (docker exit {status}); restart this profile's instance and retry" + ); + } + let raw = String::from_utf8(bytes).context("invalid Core runtime identity encoding")?; + let id = raw.trim(); + if id.is_empty() { + return Ok(None); + } + if id.chars().count() > MAX_INSTANCE_ID_CHARS || id.chars().any(char::is_control) { + bail!("invalid persisted Core runtime identity"); + } + Ok(Some(id.to_string())) + }; + let result = tokio::time::timeout(ID_READ_TIMEOUT, operation) + .await + .context("Core runtime identity read timed out") + .and_then(|result| result); + if result.is_err() { + child + .kill() + .await + .context("stop Core runtime identity read")?; + } + result +} diff --git a/crates/cli/src/instance/docker/live_storage_tests.rs b/crates/cli/src/instance/docker/live_storage_tests.rs new file mode 100644 index 0000000..a2add99 --- /dev/null +++ b/crates/cli/src/instance/docker/live_storage_tests.rs @@ -0,0 +1,60 @@ +//! Opt-in Docker integration for retained provider data and credential volumes. + +use super::*; +use crate::config::{ProfileKind, ResolvedProfile}; +use crate::instance::storage::{Provider, RuntimeStore}; + +#[tokio::test] +#[ignore = "requires a Docker daemon and debian:bookworm-slim image"] +async fn live_provider_switch_preserves_data_and_state() { + let docker = RealDockerRunner::new(); + docker.version().await.unwrap(); + let directory = tempfile::tempdir().unwrap(); + let mut registry = RuntimeStore::at(&directory.path().join("runtimes.json")).unwrap(); + let profile = ResolvedProfile { + name: "provider-test".into(), + base_url: "https://cloud.example.test".into(), + kind: ProfileKind::Local, + project_id: Some(uuid::Uuid::now_v7().to_string()), + memory_base_url: "http://127.0.0.1:17352".into(), + api_key: None, + oauth: None, + }; + let mut datasets = Vec::new(); + for provider in [Provider::Openai, Provider::Slm] { + let mut config = default_instance_config(&profile.name, "debian:bookworm-slim"); + config.provider = provider; + config.storage = registry.select(&profile, provider, None).unwrap(); + datasets.push(config); + } + let result: Result<()> = async { + for index in [0, 1, 0, 1, 0] { + let config = &datasets[index]; + docker.prepare_storage(config, None, false).await?; + let data = format!("type=volume,src={},dst=/data", config.storage.data); + let state = format!("type=volume,src={},dst=/state", config.storage.state); + let expected = format!("{}-retained-memory", config.provider.as_str()); + let (code, _, _) = docker.exec_capture(&[ + "run", "--rm", "--network", "none", "--mount", &data, "--mount", &state, + &config.image, "sh", "-eu", "-c", + "if test -f /data/memory; then test \"$(cat /data/memory)\" = \"$1\"; else printf %s \"$1\" > /data/memory; fi; if ! test -f /state/core-api-key; then printf %s \"$1-key\" > /state/core-api-key; fi", + "provider-test", &expected, + ], None).await?; + anyhow::ensure!(code == 0, "provider data changed after container replacement"); + let key = crate::instance::credentials::resolve_dataset_key(&docker, config, false, None, false).await?; + anyhow::ensure!(key == format!("{expected}-key"), "selected provider credential changed"); + registry.select(&profile, config.provider, None)?; + registry.save()?; + } + // Purging one selected pair must leave the other provider's credential readable. + docker.volume_rm(&datasets[1].storage.data).await?; + docker.volume_rm(&datasets[1].storage.state).await?; + anyhow::ensure!(docker.read_volume_core_api_key(&datasets[0].storage.state, &datasets[0].image).await?.as_deref() == Some("openai-retained-memory-key")); + Ok(()) + }.await; + for config in datasets { + docker.volume_rm(&config.storage.data).await.unwrap(); + docker.volume_rm(&config.storage.state).await.unwrap(); + } + result.unwrap(); +} diff --git a/crates/cli/src/instance/docker/recovery_tests.rs b/crates/cli/src/instance/docker/recovery_tests.rs new file mode 100644 index 0000000..b86d135 --- /dev/null +++ b/crates/cli/src/instance/docker/recovery_tests.rs @@ -0,0 +1,257 @@ +//! Exercise managed Docker inspection and credential recovery through actual subprocess boundaries. + +use super::{DockerRunner, RealDockerRunner, default_instance_config}; +use crate::instance::credentials::resolve_dataset_key; +use crate::instance::storage::{Provider, StorageVolumes}; +use serde_json::{Value, json}; +use std::os::unix::fs::PermissionsExt; + +fn fake_docker(script: &str) -> (tempfile::TempDir, RealDockerRunner) { + let directory = tempfile::tempdir().unwrap(); + let executable = directory.path().join("docker"); + // Write via rename so spawn never hits ETXTBSY on a still-open writer FD. + let staging = directory.path().join("docker.staging"); + std::fs::write(&staging, format!("#!/bin/sh\nset -eu\n{script}\n")).unwrap(); + std::fs::set_permissions(&staging, std::fs::Permissions::from_mode(0o700)).unwrap(); + std::fs::rename(&staging, &executable).unwrap(); + let docker = RealDockerRunner { + docker_bin: executable.to_string_lossy().into_owned(), + }; + (directory, docker) +} + +fn inspect_fixture() -> Value { + json!([{ + "State": { "Status": "running" }, + "Config": { + "Image": "example/core:fixture", + "Labels": { + "ai.atomicstrata.managed-by": "am-cli", + "ai.atomicstrata.profile": "test-profile", + "ai.atomicstrata.project-id": "project-one", + "ai.atomicstrata.local-url": "http://127.0.0.1:17350" + }, + "Env": ["OPENAI_API_KEY=fixture-openai-key", "EMBEDDING_DIMENSIONS=1536"] + }, + "Mounts": [ + { "Type": "volume", "Name": "provider-data", "Destination": "/var/lib/atomicmemory/postgres" }, + { "Type": "volume", "Name": "provider-state", "Destination": "/var/lib/atomicmemory/state" } + ], + "NetworkSettings": { "Ports": { "17350/tcp": [{ "HostIp": "127.0.0.1", "HostPort": "17352" }] } } + }]) +} + +async fn inspect_fixture_with(fixture: Value) -> super::ContainerInspect { + let (directory, docker) = fake_docker("cat \"$(dirname \"$0\")/inspect.json\""); + std::fs::write(directory.path().join("inspect.json"), fixture.to_string()).unwrap(); + docker.inspect("atomic-memory").await.unwrap().unwrap() +} + +#[tokio::test] +async fn actual_inspect_binding_wins_over_a_misleading_url_label() { + let inspected = inspect_fixture_with(inspect_fixture()).await; + assert_eq!( + inspected.local_url.as_deref(), + Some("http://127.0.0.1:17352") + ); + let storage = inspected.storage.unwrap(); + assert_eq!(storage.provider, Provider::Openai); + assert_eq!(storage.project_id.as_deref(), Some("project-one")); + assert_eq!( + storage.volumes, + StorageVolumes { + data: "provider-data".into(), + state: "provider-state".into() + } + ); +} + +#[tokio::test] +async fn wildcard_or_ambiguous_bindings_never_prove_a_safe_destination() { + for bindings in [ + json!([{ "HostIp": "0.0.0.0", "HostPort": "17352" }]), + json!([{ "HostIp": "::", "HostPort": "17352" }]), + json!([{ "HostIp": "127.0.0.1", "HostPort": "0" }]), + json!([{ "HostIp": "127.0.0.1", "HostPort": "17352" }, { "HostIp": "0.0.0.0", "HostPort": "17352" }]), + Value::Null, + ] { + let mut fixture = inspect_fixture(); + fixture[0]["NetworkSettings"]["Ports"]["17350/tcp"] = bindings; + assert!(inspect_fixture_with(fixture).await.local_url.is_none()); + } +} + +#[tokio::test] +async fn unknown_embedding_configurations_never_prove_provider_storage() { + for env in [ + json!(["OPENAI_API_KEY=fixture", "EMBEDDING_PROVIDER=custom"]), + json!(["OPENAI_API_KEY=fixture", "EMBEDDING_DIMENSIONS=768"]), + json!([]), + json!([ + "EMBEDDING_PROVIDER=openai-compatible", + "EMBEDDING_DIMENSIONS=768" + ]), + ] { + let mut fixture = inspect_fixture(); + fixture[0]["Config"]["Env"] = env; + assert!(inspect_fixture_with(fixture).await.storage.is_none()); + } +} + +#[tokio::test] +async fn provider_proof_requires_both_named_docker_volumes() { + for index in [0, 1] { + let mut fixture = inspect_fixture(); + fixture[0]["Mounts"][index]["Type"] = json!("bind"); + assert!(inspect_fixture_with(fixture).await.storage.is_none()); + } + let mut fixture = inspect_fixture(); + fixture[0]["Mounts"][1]["Name"] = Value::Null; + assert!(inspect_fixture_with(fixture).await.storage.is_none()); +} + +#[tokio::test] +async fn slm_storage_is_identified_from_observed_models_and_mounts() { + let mut fixture = inspect_fixture(); + fixture[0]["Config"]["Env"] = json!([ + "EMBEDDING_PROVIDER=openai-compatible", + "EMBEDDING_DIMENSIONS=768", + format!("EMBEDDING_MODEL={}", crate::slm::SLM_EMBED_MODEL), + format!("LLM_MODEL={}", crate::slm::SLM_CHAT_MODEL), + ]); + let inspected = inspect_fixture_with(fixture).await; + assert_eq!(inspected.storage.unwrap().provider, Provider::Slm); +} + +#[tokio::test] +async fn credential_resolution_reads_only_the_target_state_volume() { + let (directory, docker) = fake_docker( + r#" + printf '%s\n' "$@" >> "$(dirname "$0")/calls" + case "$1" in + volume) printf '[{}]' ;; + run) + case "$*" in + *src=slm-state,dst=/state,readonly*) printf 'saved-slm-key\n' ;; + *) printf 'outgoing-openai-key\n' ;; + esac ;; + *) exit 91 ;; + esac + "#, + ); + let mut config = default_instance_config("test-profile", "example/core:fixture"); + config.provider = Provider::Slm; + config.storage = StorageVolumes { + data: "slm-data".into(), + state: "slm-state".into(), + }; + let key = resolve_dataset_key(&docker, &config, false, None, false) + .await + .unwrap(); + assert_eq!(key, "saved-slm-key"); + let calls = std::fs::read_to_string(directory.path().join("calls")).unwrap(); + assert!(calls.contains("type=volume,src=slm-state,dst=/state,readonly")); + assert!(calls.contains("--network\nnone\n")); + for forbidden in ["atomic-memory\n", "openai-state", "saved-slm-key", "exec\n"] { + assert!( + !calls.contains(forbidden), + "unexpected argument {forbidden}" + ); + } +} + +#[tokio::test] +async fn target_state_read_failure_does_not_generate_a_replacement_key() { + let (_directory, docker) = + fake_docker("case \"$1\" in volume) printf '[{}]' ;; run) exit 1 ;; esac"); + let config = default_instance_config("test-profile", "example/core:fixture"); + let error = resolve_dataset_key(&docker, &config, false, None, false) + .await + .unwrap_err(); + assert!( + error + .to_string() + .contains("could not read selected provider's Core key") + ); +} + +#[tokio::test] +async fn missing_docker_and_unavailable_daemon_have_distinct_recovery_messages() { + let directory = tempfile::tempdir().unwrap(); + let missing = RealDockerRunner { + docker_bin: directory + .path() + .join("missing-docker") + .to_string_lossy() + .into_owned(), + }; + assert!( + missing + .version() + .await + .unwrap_err() + .to_string() + .contains("not installed") + ); + let (_directory, unavailable) = fake_docker("printf 'daemon private diagnostic' >&2; exit 1"); + let message = unavailable.version().await.unwrap_err().to_string(); + assert!(message.contains("daemon is unavailable")); + assert!(!message.contains("not installed")); + assert!(!message.contains("private diagnostic")); +} + +#[tokio::test] +async fn inspect_distinguishes_missing_container_from_unavailable_daemon() { + let (_directory, missing) = + fake_docker("printf 'Error: No such container: atomic-memory' >&2; exit 1"); + assert!(missing.inspect("atomic-memory").await.unwrap().is_none()); + let (_directory, unavailable) = + fake_docker("printf 'Cannot connect to Docker daemon' >&2; exit 1"); + assert!(unavailable.inspect("atomic-memory").await.is_err()); +} + +#[tokio::test] +async fn stopped_container_binding_is_available_only_for_restart_preparation() { + let mut fixture = inspect_fixture(); + fixture[0]["State"]["Status"] = json!("exited"); + fixture[0]["NetworkSettings"]["Ports"] = json!({}); + fixture[0]["HostConfig"] = + json!({"PortBindings": {"17350/tcp": [{"HostIp": "127.0.0.1", "HostPort": "17352"}]}}); + let (directory, docker) = fake_docker("cat \"$(dirname \"$0\")/inspect.json\""); + std::fs::write(directory.path().join("inspect.json"), fixture.to_string()).unwrap(); + assert!( + docker + .inspect("atomic-memory") + .await + .unwrap() + .unwrap() + .local_url + .is_none() + ); + assert_eq!( + docker + .configured_local_url("atomic-memory") + .await + .unwrap() + .as_deref(), + Some("http://127.0.0.1:17352") + ); + fixture[0]["HostConfig"]["PortBindings"]["17350/tcp"][0]["HostIp"] = json!("0.0.0.0"); + std::fs::write(directory.path().join("inspect.json"), fixture.to_string()).unwrap(); + assert!( + docker + .configured_local_url("atomic-memory") + .await + .unwrap() + .is_none() + ); +} + +#[tokio::test] +async fn repeated_purge_accepts_missing_volume_but_not_other_daemon_errors() { + let (_directory, docker) = + fake_docker("echo 'Error response from daemon: get selected: no such volume' >&2; exit 1"); + docker.volume_rm("selected").await.unwrap(); + let (_directory, docker) = fake_docker("echo 'permission denied' >&2; exit 1"); + assert!(docker.volume_rm("selected").await.is_err()); +} diff --git a/crates/cli/src/instance/docker/storage_ops.rs b/crates/cli/src/instance/docker/storage_ops.rs new file mode 100644 index 0000000..38e1508 --- /dev/null +++ b/crates/cli/src/instance/docker/storage_ops.rs @@ -0,0 +1,260 @@ +//! Docker volume ownership checks and read-only provider credential retrieval. + +use super::{InstanceConfig, RealDockerRunner}; +use crate::instance::storage::{StorageIdentity, StorageVolumes}; +use anyhow::{Context, Result, bail}; +use serde::Deserialize; +use std::collections::HashMap; + +#[derive(Deserialize)] +#[serde(rename_all = "PascalCase")] +struct Volume { + #[serde(default)] + labels: Option>, + #[serde(default)] + created_at: Option, +} + +async fn inspect(docker: &RealDockerRunner, name: &str) -> Result> { + let (code, stdout, stderr) = docker + .exec_capture(&["volume", "inspect", name], None) + .await?; + if code != 0 { + if stderr.to_ascii_lowercase().contains("no such volume") { + return Ok(None); + } + bail!("could not inspect Core volume; check Docker and retry"); + } + let volumes: Vec = + serde_json::from_str(&stdout).context("parse Docker volume metadata")?; + volumes + .into_iter() + .next() + .map(Some) + .ok_or_else(|| anyhow::anyhow!("Docker returned no volume metadata")) +} + +pub(super) async fn exists(docker: &RealDockerRunner, name: &str) -> Result { + Ok(inspect(docker, name).await?.is_some()) +} + +/// Check both existing volumes before any create, attach, or destructive operation. +pub(super) async fn validate( + docker: &RealDockerRunner, + config: &InstanceConfig, + recorded: Option<&StorageIdentity>, + live_legacy: bool, +) -> Result { + let data = inspect(docker, &config.storage.data).await?; + let state = inspect(docker, &config.storage.state).await?; + for (name, volume, expected) in [ + ( + &config.storage.data, + &data, + recorded.and_then(|i| i.data.as_deref()), + ), + ( + &config.storage.state, + &state, + recorded.and_then(|i| i.state.as_deref()), + ), + ] { + if let Some(volume) = volume { + validate_volume(config, name, volume, expected, live_legacy)?; + } + } + Ok(StorageIdentity { + data: data.and_then(|volume| volume.created_at), + state: state.and_then(|volume| volume.created_at), + }) +} + +pub(super) async fn ensure( + docker: &RealDockerRunner, + config: &InstanceConfig, + recorded: Option<&StorageIdentity>, + live_legacy: bool, +) -> Result { + let identity = validate(docker, config, recorded, live_legacy).await?; + let dataset = format!("{}:{}", config.storage.data, config.provider.as_str()); + for name in [&config.storage.data, &config.storage.state] { + if inspect(docker, name).await?.is_some() { + continue; + } + let label = format!("ai.atomicstrata.dataset={dataset}"); + let (code, _, _) = docker + .exec_capture( + &[ + "volume", + "create", + "--label", + "ai.atomicstrata.managed-by=am-cli", + "--label", + &label, + name, + ], + None, + ) + .await?; + if code != 0 { + bail!("could not prepare provider volume '{name}'; existing data was preserved"); + } + } + validate(docker, config, Some(&identity), live_legacy).await +} + +pub(super) async fn read_key( + docker: &RealDockerRunner, + name: &str, + image: &str, +) -> Result> { + if !exists(docker, name).await? { + return Ok(None); + } + let mount = format!("type=volume,src={name},dst=/state,readonly"); + let (code, stdout, _) = docker + .exec_capture( + &[ + "run", + "--rm", + "--network", + "none", + "--mount", + &mount, + "--entrypoint", + "sh", + image, + "-c", + "if test -f /state/core-api-key; then cat /state/core-api-key; fi", + ], + None, + ) + .await?; + if code != 0 { + bail!( + "could not read selected provider's Core key; dataset preserved, retry when Docker is ready" + ); + } + let key = stdout.trim(); + Ok((!key.is_empty()).then(|| key.to_string())) +} + +fn validate_volume( + config: &InstanceConfig, + name: &str, + volume: &Volume, + recorded: Option<&str>, + live_legacy: bool, +) -> Result<()> { + let dataset = format!("{}:{}", config.storage.data, config.provider.as_str()); + if volume.labels.as_ref().is_some_and(|labels| { + labels.get("ai.atomicstrata.dataset") == Some(&dataset) + && labels.get("ai.atomicstrata.managed-by").map(String::as_str) == Some("am-cli") + }) { + return Ok(()); + } + if config.storage == StorageVolumes::legacy() { + let created = volume + .created_at + .as_deref() + .filter(|value| !value.is_empty()); + if created.is_some() && (live_legacy || recorded.is_some() && created == recorded) { + return Ok(()); + } + } + bail!( + "volume '{name}' lacks matching managed dataset ownership or legacy creation identity; preserved without attaching or deleting it" + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn config(legacy: bool) -> InstanceConfig { + let mut config = crate::instance::docker::default_instance_config("local", "core-test"); + if !legacy { + config.storage = crate::instance::storage::StorageVolumes { + data: "known-data".into(), + state: "known-state".into(), + }; + } + config + } + + fn volume(created_at: &str) -> Volume { + Volume { + labels: None, + created_at: Some(created_at.into()), + } + } + + #[test] + fn foreign_named_volume_is_rejected_before_removal() { + assert!( + validate_volume( + &config(false), + "known-data", + &volume("foreign"), + None, + false + ) + .is_err() + ); + } + + #[test] + fn retired_legacy_volume_requires_the_recorded_creation_identity() { + let config = config(true); + assert!( + validate_volume( + &config, + &config.storage.data, + &volume("original"), + Some("original"), + false + ) + .is_ok() + ); + assert!( + validate_volume( + &config, + &config.storage.data, + &volume("replacement"), + Some("original"), + false + ) + .is_err() + ); + assert!( + validate_volume( + &config, + &config.storage.data, + &volume("original"), + None, + false + ) + .is_err() + ); + } + + #[test] + fn live_legacy_adoption_requires_a_creation_timestamp() { + let config = config(true); + assert!( + validate_volume( + &config, + &config.storage.data, + &volume("original"), + None, + true + ) + .is_ok() + ); + let missing = Volume { + labels: None, + created_at: None, + }; + assert!(validate_volume(&config, &config.storage.data, &missing, None, true).is_err()); + } +} diff --git a/crates/cli/src/instance/mod.rs b/crates/cli/src/instance/mod.rs index 3580cd2..c0cdb84 100644 --- a/crates/cli/src/instance/mod.rs +++ b/crates/cli/src/instance/mod.rs @@ -3,10 +3,10 @@ use anyhow::Result; use rand::Rng; -use crate::auth::origin::same_origin; -use crate::config::resolve_core_api_key; - +pub mod address; +pub mod credentials; pub mod docker; +pub mod storage; pub use docker::{ContainerInspect, DockerRunner, RealDockerRunner}; @@ -48,31 +48,6 @@ pub fn generate_core_api_key() -> String { hex::encode(bytes) } -/// Resolve the `CORE_API_KEY` to inject on `docker run` and use for health checks. -/// -/// Precedence: shell override → persisted state file → container env → generate. -/// A new key is minted only when `purge_data` is true or no persisted/env key exists -/// (`--replace` alone must not rotate the local Core bearer). -pub async fn resolve_instance_core_api_key( - docker: &dyn DockerRunner, - purge_data: bool, -) -> Result { - if let Some(key) = resolve_core_api_key() { - return Ok(key); - } - if !purge_data { - if let Some(key) = docker.read_core_api_key(DEFAULT_CONTAINER_NAME).await? { - return Ok(key); - } - if let Some(inspect) = docker.inspect(DEFAULT_CONTAINER_NAME).await? - && let Some(key) = inspect.core_api_key - { - return Ok(key); - } - } - Ok(generate_core_api_key()) -} - /// True when a CLI-managed container was started for a different local profile. pub fn managed_core_profile_mismatch(inspect: &ContainerInspect, profile_name: &str) -> bool { inspect.managed_by_cli && inspect.profile_label.as_deref() != Some(profile_name) @@ -127,38 +102,91 @@ pub async fn managed_core_needs_env_sync( || managed_core_cloud_env_mismatch(&inspect, expected_api_url, expected_jwks_url)) } -/// Read `CORE_API_KEY` from the CLI-managed Core container when it matches `profile_name`. +/// Read a managed Core key only for the resolved project's active dataset. pub async fn read_managed_core_api_key( - profile_name: &str, - destination_url: &str, -) -> Option { - let docker = RealDockerRunner::new(); - read_managed_core_api_key_with(&docker, profile_name, destination_url) - .await - .ok() - .flatten() + profile: &crate::config::ResolvedProfile, +) -> Result> { + read_managed_core_api_key_with(&RealDockerRunner::new(), profile).await } +/// Resolve the managed credential through the shared Cloud and dataset ownership gate. pub async fn read_managed_core_api_key_with( docker: &dyn DockerRunner, - profile_name: &str, - destination_url: &str, + profile: &crate::config::ResolvedProfile, ) -> Result> { - let inspect = docker.inspect(DEFAULT_CONTAINER_NAME).await?; + read_managed_core_api_key_using(docker, profile, storage::RuntimeStore::open).await +} + +async fn read_managed_core_api_key_using( + docker: &dyn DockerRunner, + profile: &crate::config::ResolvedProfile, + load_registry: F, +) -> Result> +where + F: FnOnce() -> Result, +{ + // External Core destinations retain their explicit-key / Cloud JWT path. + let Ok(destination) = address::ManagedAddress::parse(&profile.memory_base_url) else { + return Ok(None); + }; + // Keep inspection and key retrieval in the same lifecycle transaction. + let registry = load_registry()?; + let inspect = match docker.inspect(DEFAULT_CONTAINER_NAME).await { + Ok(inspect) => inspect, + Err(error) + if error.chain().any(|cause| { + cause + .downcast_ref::() + .is_some_and(|io| io.kind() == std::io::ErrorKind::NotFound) + }) => + { + return Ok(None); + } + Err(error) => return Err(error), + }; let Some(inspect) = inspect else { return Ok(None); }; if !inspect.managed_by_cli || !inspect.state.is_running() { return Ok(None); } - if inspect.profile_label.as_deref() != Some(profile_name) { + let bound = inspect + .local_url + .as_deref() + .and_then(|url| address::ManagedAddress::parse(url).ok()); + if bound != Some(destination) { return Ok(None); } - let Some(ref local_url) = inspect.local_url else { - return Ok(None); - }; - if !same_origin(local_url, destination_url) { - return Ok(None); + if !storage::matches_profile_context(profile, &inspect) { + anyhow::bail!( + "managed Core belongs to a different Cloud project or origin; run `am instance start --replace` before using this profile" + ); + } + let observed = inspect.storage.as_ref().ok_or_else(|| { + anyhow::anyhow!( + "managed Core provider and mounts could not be verified; administrator key withheld" + ) + })?; + if let Some((provider, volumes)) = registry.selected_dataset(profile)? + && (provider != observed.provider || volumes != observed.volumes) + { + anyhow::bail!( + "managed Core is still running a different dataset from the saved selection; finish `am instance start` before using this profile" + ); + } + let identity = registry.identity(profile, observed.provider)?; + if let Some(identity) = identity { + let mut config = docker::default_instance_config(&profile.name, &inspect.image); + config.provider = observed.provider; + config.storage = observed.volumes.clone(); + config.project_id = profile.project_id.clone(); + docker + .validate_storage(&config, Some(identity), false) + .await?; + } else if storage::owned_storage(profile, Some(&inspect)).is_none() { + anyhow::bail!( + "managed Core project ownership could not be verified; administrator key withheld" + ); } if let Some(key) = docker.read_core_api_key(DEFAULT_CONTAINER_NAME).await? { return Ok(Some(key)); @@ -174,6 +202,7 @@ mod tests { fn inspect_with_profile(profile: Option<&str>) -> ContainerInspect { ContainerInspect { + storage: None, name: DEFAULT_CONTAINER_NAME.into(), image: "test".into(), state: ContainerState::Running, @@ -245,9 +274,13 @@ mod tests { )); } + #[derive(Default)] struct StubDocker { state_key: Option, inspect: Option, + observed_identity: Option, + inspect_lock_path: Option, + read_calls: std::sync::atomic::AtomicUsize, } #[async_trait::async_trait] @@ -257,6 +290,12 @@ mod tests { } async fn inspect(&self, _name: &str) -> Result> { + if let Some(path) = &self.inspect_lock_path { + assert!( + storage::RuntimeStore::at(path).is_err(), + "inspection must happen while the lifecycle lock is held" + ); + } Ok(self.inspect.clone()) } @@ -293,8 +332,53 @@ mod tests { } async fn read_core_api_key(&self, _name: &str) -> Result> { + self.read_calls + .fetch_add(1, std::sync::atomic::Ordering::SeqCst); Ok(self.state_key.clone()) } + async fn validate_storage( + &self, + config: &docker::InstanceConfig, + recorded: Option<&storage::StorageIdentity>, + live_legacy: bool, + ) -> Result { + assert!( + !live_legacy, + "rotated credentials require durable ownership proof" + ); + assert_eq!(config.storage, storage::StorageVolumes::legacy()); + let actual = self + .observed_identity + .as_ref() + .ok_or_else(|| anyhow::anyhow!("missing identity fixture"))?; + if recorded != Some(actual) { + bail!("legacy volume creation identity changed"); + } + Ok(actual.clone()) + } + } + + fn key_profile(destination: &str) -> crate::config::ResolvedProfile { + crate::config::ResolvedProfile { + name: "default".into(), + base_url: "https://api.dev.example.com".into(), + project_id: Some("project-one".into()), + kind: crate::config::ProfileKind::Local, + memory_base_url: destination.into(), + api_key: Some("amc_current".into()), + oauth: None, + } + } + + async fn read_test_key( + docker: &dyn DockerRunner, + profile: &crate::config::ResolvedProfile, + ) -> Result> { + let directory = tempfile::tempdir().unwrap(); + read_managed_core_api_key_using(docker, profile, || { + storage::RuntimeStore::at(&directory.path().join("runtimes.json")) + }) + .await } fn managed_inspect_with_key( @@ -303,6 +387,11 @@ mod tests { core_api_key: Option<&str>, ) -> ContainerInspect { ContainerInspect { + storage: Some(storage::ObservedStorage { + provider: storage::Provider::Openai, + volumes: storage::StorageVolumes::legacy(), + project_id: Some("project-one".into()), + }), name: DEFAULT_CONTAINER_NAME.into(), image: "test".into(), state: ContainerState::Running, @@ -319,24 +408,39 @@ mod tests { } #[tokio::test] - async fn resolve_reuses_persisted_key_without_purge() { - let docker = StubDocker { - state_key: Some("persisted-core-key".into()), - inspect: None, - }; - let key = resolve_instance_core_api_key(&docker, false).await.unwrap(); - assert_eq!(key, "persisted-core-key"); - } - - #[tokio::test] - async fn resolve_generates_when_purge_data_even_if_persisted() { - let docker = StubDocker { - state_key: Some("persisted-core-key".into()), - inspect: None, - }; - let key = resolve_instance_core_api_key(&docker, true).await.unwrap(); - assert_ne!(key, "persisted-core-key"); - assert_eq!(key.len(), 64); + async fn read_managed_key_rejects_same_label_and_port_for_foreign_cloud_context() { + for foreign_origin in [false, true] { + let mut inspect = managed_inspect_with_key( + "default", + "http://127.0.0.1:17350", + Some("outgoing-admin"), + ); + inspect.storage = Some(storage::ObservedStorage { + provider: storage::Provider::Openai, + volumes: storage::StorageVolumes::legacy(), + project_id: Some( + if foreign_origin { + "project-one" + } else { + "project-other" + } + .into(), + ), + }); + if foreign_origin { + inspect.atomicmemory_api_url = Some("https://foreign.example.test".into()); + } + let docker = StubDocker { + state_key: Some("outgoing-admin".into()), + inspect: Some(inspect), + ..Default::default() + }; + let result = read_test_key(&docker, &key_profile("http://127.0.0.1:17350")).await; + assert!( + result.is_err(), + "foreign dataset leaked its administrator key" + ); + } } #[tokio::test] @@ -348,16 +452,16 @@ mod tests { "http://127.0.0.1:17350", None, )), + ..Default::default() }; - let key = read_managed_core_api_key_with(&docker, "default", "http://127.0.0.1:17350") + let key = read_test_key(&docker, &key_profile("http://127.0.0.1:17350")) .await .unwrap(); assert_eq!(key.as_deref(), Some("core-from-state")); - let mismatched = - read_managed_core_api_key_with(&docker, "default", "http://127.0.0.1:9999") - .await - .unwrap(); + let mismatched = read_test_key(&docker, &key_profile("http://127.0.0.1:9999")) + .await + .unwrap(); assert!(mismatched.is_none()); } @@ -368,10 +472,177 @@ mod tests { let docker = StubDocker { state_key: Some("core-from-state".into()), inspect: Some(inspect), + ..Default::default() }; - let key = read_managed_core_api_key_with(&docker, "default", "http://127.0.0.1:17350") + let key = read_test_key(&docker, &key_profile("http://127.0.0.1:17350")) .await .unwrap(); assert!(key.is_none()); } + #[tokio::test] + async fn read_managed_key_rejects_outgoing_provider_or_mapping_after_failed_switch() { + for requested in [storage::Provider::Slm, storage::Provider::Openai] { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("runtimes.json"); + let profile = key_profile("http://127.0.0.1:17350"); + let mut registry = storage::RuntimeStore::at(&path).unwrap(); + registry.select(&profile, requested, None).unwrap(); + registry.save().unwrap(); + drop(registry); + let docker = StubDocker { + state_key: Some("outgoing-admin".into()), + inspect: Some(managed_inspect_with_key( + "default", + &profile.memory_base_url, + Some("outgoing-env-admin"), + )), + ..Default::default() + }; + let result = read_managed_core_api_key_using(&docker, &profile, || { + storage::RuntimeStore::at(&path) + }) + .await; + assert!( + result.is_err(), + "saved selection mismatch must withhold both state and env keys" + ); + assert_eq!( + docker.read_calls.load(std::sync::atomic::Ordering::SeqCst), + 0 + ); + } + } + + #[tokio::test] + async fn read_managed_key_requires_unchanged_legacy_identity_after_cloud_key_rotation() { + for recreated in [false, true] { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("runtimes.json"); + let mut profile = key_profile("http://127.0.0.1:17350"); + let mut inspect = managed_inspect_with_key("default", &profile.memory_base_url, None); + inspect.storage.as_mut().unwrap().project_id = None; + inspect.atomicmemory_api_key = profile.api_key.clone(); + let identity = storage::StorageIdentity { + data: Some("created-data".into()), + state: Some("created-state".into()), + }; + let mut registry = storage::RuntimeStore::at(&path).unwrap(); + let volumes = registry + .select(&profile, storage::Provider::Openai, Some(&inspect)) + .unwrap(); + registry + .record_identity( + &profile, + storage::Provider::Openai, + &volumes, + identity.clone(), + ) + .unwrap(); + registry.save().unwrap(); + drop(registry); + profile.api_key = Some("amc_rotated".into()); + let mut actual = identity; + if recreated { + actual.state = Some("different-creation".into()); + } + let docker = StubDocker { + state_key: Some("retained-admin".into()), + inspect: Some(inspect), + observed_identity: Some(actual), + ..Default::default() + }; + let result = read_managed_core_api_key_using(&docker, &profile, || { + storage::RuntimeStore::at(&path) + }) + .await; + if recreated { + assert!(result.is_err()); + } else { + assert_eq!(result.unwrap().as_deref(), Some("retained-admin")); + } + assert_eq!( + docker.read_calls.load(std::sync::atomic::Ordering::SeqCst), + usize::from(!recreated) + ); + } + } + + #[tokio::test] + async fn read_managed_key_holds_lifecycle_lock_before_inspecting_container() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("runtimes.json"); + let docker = StubDocker { + inspect_lock_path: Some(path.clone()), + ..Default::default() + }; + let profile = key_profile("http://127.0.0.1:17350"); + let result = + read_managed_core_api_key_using(&docker, &profile, || storage::RuntimeStore::at(&path)) + .await + .unwrap(); + assert!(result.is_none()); + } + + #[tokio::test] + async fn read_managed_key_keeps_external_and_missing_docker_paths_available() { + let docker = StubDocker::default(); + let profile = key_profile("https://external-core.example.test"); + let result = read_managed_core_api_key_using(&docker, &profile, || { + panic!("external Core must not load managed state") + }) + .await + .unwrap(); + assert!(result.is_none()); + let directory = tempfile::tempdir().unwrap(); + let missing = RealDockerRunner { + docker_bin: directory + .path() + .join("missing-docker") + .to_string_lossy() + .into_owned(), + }; + let profile = key_profile("http://127.0.0.1:17350"); + let result = read_managed_core_api_key_using(&missing, &profile, || { + storage::RuntimeStore::at(&directory.path().join("runtimes.json")) + }) + .await + .unwrap(); + assert!(result.is_none()); + } } + +/// Validate known prerequisites before provisioning Cloud resources or downloading models. +pub async fn preflight_managed( + local_url: Option<&str>, + provider: Option, + interactive: bool, +) -> Result<()> { + if let Some(url) = local_url { + address::ManagedAddress::parse(url)?; + } + if provider == Some(storage::Provider::Slm) && crate::slm::current_target().is_none() { + anyhow::bail!("Connected Local SLM requires Apple Silicon macOS"); + } + docker::ensure_docker_available_with_preflight(&RealDockerRunner::new(), interactive).await +} + +/// Pick the shared smoke mode from the selected provider, including saved selections. +pub async fn smoke_options( + profile: &crate::config::ResolvedProfile, +) -> Result { + if profile.kind != crate::config::ProfileKind::Local { + return Ok(Default::default()); + } + let observed = RealDockerRunner::new() + .inspect(DEFAULT_CONTAINER_NAME) + .await?; + let provider = storage::RuntimeStore::open()?.provider(profile, None, observed.as_ref())?; + Ok(if provider == storage::Provider::Slm { + crate::verification::smoke::SmokeOptions::full_extraction() + } else { + Default::default() + }) +} + +#[cfg(test)] +mod storage_tests; diff --git a/crates/cli/src/instance/storage.rs b/crates/cli/src/instance/storage.rs new file mode 100644 index 0000000..52939bc --- /dev/null +++ b/crates/cli/src/instance/storage.rs @@ -0,0 +1,415 @@ +//! Persistent per-project provider datasets and managed lifecycle serialization. + +use super::docker::ContainerInspect; +use crate::config::{ResolvedProfile, require_api_key, require_project_id}; +use anyhow::{Context, Result, bail}; +use clap::ValueEnum; +use fs4::fs_std::FileExt; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::collections::BTreeMap; +use std::fs::{self, File, OpenOptions}; +use std::path::{Path, PathBuf}; + +/// Supported local provider families. Each owns a separate data/state pair. +#[derive( + Debug, Clone, Copy, Default, PartialEq, Eq, PartialOrd, Ord, ValueEnum, Serialize, Deserialize, +)] +#[serde(rename_all = "snake_case")] +pub enum Provider { + #[default] + Openai, + Slm, +} +impl Provider { + /// Stable CLI and storage identifier. + pub fn as_str(self) -> &'static str { + match self { + Self::Openai => "openai", + Self::Slm => "slm", + } + } + /// Resolve compatibility shorthand without introducing a second precedence rule. + pub fn requested(provider: Option, slm: bool) -> Result> { + if provider.is_some() && slm { + bail!("--provider and --slm cannot be combined"); + } + Ok(provider.or(slm.then_some(Self::Slm))) + } +} + +/// Explicit Docker volume pair; neither name is inferred during deletion. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct StorageVolumes { + pub data: String, + pub state: String, +} +impl StorageVolumes { + /// Names used by older CLI versions, adopted only with ownership evidence. + pub fn legacy() -> Self { + Self { + data: super::VOLUME_DATA.into(), + state: super::VOLUME_STATE.into(), + } + } + fn validate(&self) -> Result<()> { + for name in [&self.data, &self.state] { + if name.is_empty() + || !name + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b"_.-".contains(&b)) + { + bail!("invalid managed volume name"); + } + } + if self.data == self.state { + bail!("data and state volumes must be distinct"); + } + Ok(()) + } +} + +/// Docker creation identities recorded when adopting an unlabeled legacy pair. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct StorageIdentity { + pub data: Option, + pub state: Option, +} + +/// Provider and mounted named volumes observed from Docker, not a global marker. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ObservedStorage { + pub provider: Provider, + pub volumes: StorageVolumes, + pub project_id: Option, +} + +/// Per-project remembered provider and retained dataset mappings. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +struct ProjectStorage { + selected: Provider, + datasets: BTreeMap, + #[serde(default)] + identities: BTreeMap, +} + +/// Exclusive flock guard. Unlock explicitly on drop so the next opener does not +/// observe a stale EAGAIN under high thread churn (close-only release can lag). +struct LifecycleLock(File); + +impl Drop for LifecycleLock { + fn drop(&mut self) { + let _ = FileExt::unlock(&self.0); + } +} + +/// A locked registry. Hold it across a lifecycle operation to prevent interleaved switches. +pub struct RuntimeStore { + path: PathBuf, + _lock: LifecycleLock, + projects: BTreeMap, +} +impl RuntimeStore { + /// Open the user registry and refuse overlapping lifecycle commands. + pub fn open() -> Result { + Self::at(&crate::config::config_dir()?.join("runtimes.json")) + } + /// Open an explicit registry path (also used by isolated tests). + pub fn at(path: &Path) -> Result { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; + } + let lock = OpenOptions::new() + .create(true) + .truncate(false) + .write(true) + .open(path.with_extension("lock"))?; + lock.try_lock_exclusive().context( + "another managed Core lifecycle command is running; retry after it finishes", + )?; + let lock = LifecycleLock(lock); + let projects: BTreeMap = match fs::read(path) { + Ok(bytes) => serde_json::from_slice(&bytes).context( + "invalid runtime storage registry; existing volumes have been preserved", + )?, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => BTreeMap::new(), + Err(err) => return Err(err.into()), + }; + let mut assigned = std::collections::BTreeSet::new(); + for project in projects.values() { + for volumes in project.datasets.values() { + volumes.validate()?; + for name in [&volumes.data, &volumes.state] { + if !assigned.insert(name.clone()) { + bail!( + "runtime registry maps a volume to multiple datasets; data preserved" + ); + } + } + } + } + Ok(Self { + path: path.to_path_buf(), + _lock: lock, + projects, + }) + } + + /// Resolve the saved provider without opening or mutating any Docker data. + pub fn provider( + &self, + profile: &ResolvedProfile, + requested: Option, + observed: Option<&ContainerInspect>, + ) -> Result { + let key = storage_key(profile)?; + Ok(requested + .or(self.projects.get(&key).map(|p| p.selected)) + .or_else(|| owned_storage(profile, observed).map(|s| s.provider)) + .unwrap_or_default()) + } + + /// Persistable adoption evidence without changing an existing provider selection. + pub fn adopt_observed( + &mut self, + profile: &ResolvedProfile, + observed: Option<&ContainerInspect>, + ) -> Result<()> { + let Some(storage) = owned_storage(profile, observed) else { + return Ok(()); + }; + let key = storage_key(profile)?; + storage.volumes.validate()?; + for (owner, project) in &self.projects { + for (existing_provider, volumes) in &project.datasets { + let overlaps = [&volumes.data, &volumes.state] + .iter() + .any(|name| **name == storage.volumes.data || **name == storage.volumes.state); + if overlaps && (owner != &key || *existing_provider != storage.provider) { + bail!( + "observed volumes already belong to another dataset; preserved without reassignment" + ); + } + } + } + let project = self.projects.entry(key).or_insert_with(|| ProjectStorage { + selected: storage.provider, + ..ProjectStorage::default() + }); + if let Some(volumes) = project.datasets.get(&storage.provider) { + if volumes != &storage.volumes { + bail!( + "observed volumes do not match the recorded dataset; preserved without reassignment" + ); + } + } else { + project + .datasets + .insert(storage.provider, storage.volumes.clone()); + } + Ok(()) + } + + /// Remember proven ownership and select an isolated target dataset. + pub fn select( + &mut self, + profile: &ResolvedProfile, + provider: Provider, + observed: Option<&ContainerInspect>, + ) -> Result { + self.adopt_observed(profile, observed)?; + let key = storage_key(profile)?; + let project = self.projects.entry(key.clone()).or_default(); + let volumes = project + .datasets + .entry(provider) + .or_insert_with(|| StorageVolumes { + data: format!("atomic-memory-{key}-{}-data", provider.as_str()), + state: format!("atomic-memory-{key}-{}-state", provider.as_str()), + }) + .clone(); + volumes.validate()?; + project.selected = provider; + Ok(volumes) + } + + /// Read an existing selected dataset without inventing a claim to volume names. + pub fn selected_dataset( + &self, + profile: &ResolvedProfile, + ) -> Result> { + let key = storage_key(profile)?; + Ok(self.projects.get(&key).and_then(|project| { + project + .datasets + .get(&project.selected) + .map(|volumes| (project.selected, volumes.clone())) + })) + } + + /// Read durable creation identities for a previously validated dataset. + pub fn identity( + &self, + profile: &ResolvedProfile, + provider: Provider, + ) -> Result> { + let key = storage_key(profile)?; + Ok(self + .projects + .get(&key) + .and_then(|project| project.identities.get(&provider))) + } + + /// Record current Docker creation identities after ownership validation. + pub fn record_identity( + &mut self, + profile: &ResolvedProfile, + provider: Provider, + volumes: &StorageVolumes, + identity: StorageIdentity, + ) -> Result<()> { + let key = storage_key(profile)?; + let project = self + .projects + .get_mut(&key) + .ok_or_else(|| anyhow::anyhow!("dataset was not selected"))?; + if project.datasets.get(&provider) != Some(volumes) { + bail!( + "observed volumes do not match the recorded dataset; preserved without reassignment" + ); + } + project.identities.insert(provider, identity); + Ok(()) + } + + /// Commit a prepared selection before changing the active container, so retries resume it. + pub fn save(&self) -> Result<()> { + let temp = self.path.with_extension("json.tmp"); + let mut options = OpenOptions::new(); + options.write(true).create(true).truncate(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + let mut file = options.open(&temp)?; + use std::io::Write; + file.write_all(&serde_json::to_vec_pretty(&self.projects)?)?; + file.sync_all()?; + fs::rename(temp, &self.path).context("save runtime storage selection") + } +} + +fn storage_key(profile: &ResolvedProfile) -> Result { + let project = require_project_id(profile, None)?; + let origin = url::Url::parse(&profile.base_url)? + .origin() + .ascii_serialization(); + let bytes = serde_json::to_vec(&(origin, project))?; + Ok(hex::encode(Sha256::digest(bytes))[..24].to_string()) +} + +/// Check the managed installation, profile, Cloud origin, and any explicit project label. +pub(crate) fn matches_profile_context( + profile: &ResolvedProfile, + observed: &ContainerInspect, +) -> bool { + if !observed.managed_by_cli + || observed.profile_label.as_deref() != Some(&profile.name) + || !observed + .atomicmemory_api_url + .as_deref() + .is_some_and(|url| crate::auth::origin::same_origin(url, &profile.base_url)) + { + return false; + } + observed + .storage + .as_ref() + .and_then(|storage| storage.project_id.as_deref()) + .is_none_or(|id| profile.project_id.as_deref() == Some(id)) +} + +/// Return mounted storage whose profile and project ownership are directly proven. +pub(crate) fn owned_storage<'a>( + profile: &ResolvedProfile, + observed: Option<&'a ContainerInspect>, +) -> Option<&'a ObservedStorage> { + let observed = observed?; + if !matches_profile_context(profile, observed) { + return None; + } + let storage = observed.storage.as_ref()?; + let project_matches = storage.project_id.is_some() + || require_api_key(profile) + .ok() + .is_some_and(|key| observed.atomicmemory_api_key.as_deref() == Some(key.as_str())); + project_matches.then_some(storage) +} + +/// Validate storage before credentials change, preserving proven outgoing legacy identity. +/// Target selection remains in memory until the caller completes runtime preparation. +pub async fn preflight_storage( + profile: &ResolvedProfile, + provider: Provider, + replace: bool, +) -> Result<()> { + let docker = super::docker::RealDockerRunner::new(); + let mut registry = RuntimeStore::open()?; + preflight_storage_with(profile, provider, replace, &docker, &mut registry).await +} + +pub(crate) async fn preflight_storage_with( + profile: &ResolvedProfile, + provider: Provider, + replace: bool, + docker: &dyn super::docker::DockerRunner, + registry: &mut RuntimeStore, +) -> Result<()> { + use super::docker::default_instance_config; + let observed = docker.inspect(super::DEFAULT_CONTAINER_NAME).await?; + if observed + .as_ref() + .is_some_and(|container| !container.managed_by_cli) + && !replace + { + bail!( + "container 'atomic-memory' is not CLI-managed; use --replace to authorize replacement (volumes are preserved)" + ); + } + if let Some(outgoing) = owned_storage(profile, observed.as_ref()) { + let mut config = default_instance_config(&profile.name, ""); + config.provider = outgoing.provider; + config.storage = outgoing.volumes.clone(); + let identity = docker + .validate_storage( + &config, + registry.identity(profile, outgoing.provider)?, + true, + ) + .await?; + registry.adopt_observed(profile, observed.as_ref())?; + registry.record_identity(profile, outgoing.provider, &outgoing.volumes, identity)?; + // The old Cloud key may be rotated next. Preserve its ownership proof now, + // while leaving the previous selected provider intact until preparation succeeds. + registry.save()?; + } + let mut config = default_instance_config(&profile.name, ""); + config.provider = provider; + config.storage = registry.select(profile, provider, observed.as_ref())?; + let live_legacy = owned_storage(profile, observed.as_ref()) + .is_some_and(|storage| storage.provider == provider && storage.volumes == config.storage); + docker + .validate_storage(&config, registry.identity(profile, provider)?, live_legacy) + .await?; + Ok(()) +} + +/// Runnable provider switch command, including a shell-safe profile argument. +pub fn provider_command(profile: &str, provider: Provider) -> String { + format!( + "am --profile '{}' instance start --provider {}", + profile.replace('\'', "'\\''"), + provider.as_str() + ) +} diff --git a/crates/cli/src/instance/storage_tests.rs b/crates/cli/src/instance/storage_tests.rs new file mode 100644 index 0000000..4ae9407 --- /dev/null +++ b/crates/cli/src/instance/storage_tests.rs @@ -0,0 +1,337 @@ +//! Regression coverage for durable provider datasets, ownership, and lifecycle locks. + +use crate::config::{ProfileKind, ResolvedProfile}; +use crate::instance::docker::{ContainerInspect, ContainerState}; +use crate::instance::storage::{ObservedStorage, Provider, RuntimeStore, StorageVolumes}; + +fn profile(project: &str, origin: &str) -> ResolvedProfile { + ResolvedProfile { + name: "local-test".into(), + base_url: origin.into(), + kind: ProfileKind::Cloud, + project_id: Some(project.into()), + memory_base_url: "http://127.0.0.1:18800".into(), + api_key: Some("amc_test_credential".into()), + oauth: None, + } +} + +fn legacy(profile: &ResolvedProfile, provider: Provider) -> ContainerInspect { + ContainerInspect { + name: "atomic-memory".into(), + image: "example/core:fixture".into(), + state: ContainerState::Running, + managed_by_cli: true, + profile_label: Some(profile.name.clone()), + atomicmemory_api_url: Some(profile.base_url.clone()), + cloud_jwks_url: None, + core_api_key: None, + atomicmemory_api_key: profile.api_key.clone(), + local_url: Some(profile.memory_base_url.clone()), + storage: Some(ObservedStorage { + provider, + volumes: StorageVolumes::legacy(), + project_id: profile.project_id.clone(), + }), + } +} + +#[test] +fn provider_switch_round_trip_reopens_the_original_datasets() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("runtimes.json"); + let profile = profile("project-one", "https://cloud.example.test"); + let mut store = RuntimeStore::at(&path).unwrap(); + let openai = store.select(&profile, Provider::Openai, None).unwrap(); + store.save().unwrap(); + drop(store); + + let mut store = RuntimeStore::at(&path).unwrap(); + let slm = store.select(&profile, Provider::Slm, None).unwrap(); + assert_ne!(openai.data, slm.data); + assert_ne!(openai.state, slm.state); + store.save().unwrap(); + drop(store); + + let mut store = RuntimeStore::at(&path).unwrap(); + assert_eq!(store.provider(&profile, None, None).unwrap(), Provider::Slm); + assert_eq!( + store.select(&profile, Provider::Openai, None).unwrap(), + openai + ); + store.save().unwrap(); + drop(store); + let mut store = RuntimeStore::at(&path).unwrap(); + assert_eq!( + store.provider(&profile, None, None).unwrap(), + Provider::Openai + ); + assert_eq!(store.select(&profile, Provider::Slm, None).unwrap(), slm); +} + +#[test] +fn storage_is_isolated_by_project_and_cloud_origin() { + let directory = tempfile::tempdir().unwrap(); + let mut store = RuntimeStore::at(&directory.path().join("runtimes.json")).unwrap(); + let first = profile("project-one", "https://cloud.example.test"); + let second = profile("project-two", "https://cloud.example.test"); + let foreign = profile("project-one", "https://other.example.test"); + let first_volumes = store.select(&first, Provider::Slm, None).unwrap(); + for other in [&second, &foreign] { + assert_eq!(store.provider(other, None, None).unwrap(), Provider::Openai); + let volumes = store.select(other, Provider::Slm, None).unwrap(); + assert_ne!(first_volumes.data, volumes.data); + assert_ne!(first_volumes.state, volumes.state); + } + let mut alias = profile("project-one", "https://cloud.example.test:443/api/"); + alias.name = "another-profile".into(); + assert_eq!( + store.select(&alias, Provider::Slm, None).unwrap(), + first_volumes + ); +} + +#[test] +fn proven_legacy_dataset_survives_a_provider_switch() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("runtimes.json"); + let profile = profile("project-one", "https://cloud.example.test"); + let observed = legacy(&profile, Provider::Openai); + let mut store = RuntimeStore::at(&path).unwrap(); + let slm = store + .select(&profile, Provider::Slm, Some(&observed)) + .unwrap(); + assert_ne!(slm, StorageVolumes::legacy()); + store.save().unwrap(); + drop(store); + let mut store = RuntimeStore::at(&path).unwrap(); + assert_eq!( + store.select(&profile, Provider::Openai, None).unwrap(), + StorageVolumes::legacy() + ); + assert_eq!(store.select(&profile, Provider::Slm, None).unwrap(), slm); +} + +#[test] +fn unknown_or_foreign_observations_never_adopt_legacy_volumes() { + let profile = profile("project-one", "https://cloud.example.test"); + for variation in 0..7 { + let directory = tempfile::tempdir().unwrap(); + let mut store = RuntimeStore::at(&directory.path().join("runtimes.json")).unwrap(); + let mut observed = legacy(&profile, Provider::Slm); + match variation { + 0 => observed.managed_by_cli = false, + 1 => observed.profile_label = Some("someone-else".into()), + 2 => observed.atomicmemory_api_url = Some("https://foreign.example.test".into()), + 3 => observed.atomicmemory_api_url = None, + 4 => observed.storage.as_mut().unwrap().project_id = Some("foreign-project".into()), + 5 => observed.storage = None, + 6 => observed.profile_label = None, + _ => unreachable!(), + } + assert_eq!( + store.provider(&profile, None, Some(&observed)).unwrap(), + Provider::Openai + ); + let selected = store + .select(&profile, Provider::Slm, Some(&observed)) + .unwrap(); + assert_ne!( + selected, + StorageVolumes::legacy(), + "adopted unproven variant {variation}" + ); + } +} + +#[test] +fn legacy_without_project_label_requires_matching_cloud_credential() { + let profile = profile("project-one", "https://cloud.example.test"); + for matches in [false, true] { + let directory = tempfile::tempdir().unwrap(); + let mut store = RuntimeStore::at(&directory.path().join("runtimes.json")).unwrap(); + let mut observed = legacy(&profile, Provider::Slm); + observed.storage.as_mut().unwrap().project_id = None; + if !matches { + observed.atomicmemory_api_key = Some("amc_foreign_credential".into()); + } + let selected = store + .select(&profile, Provider::Slm, Some(&observed)) + .unwrap(); + assert_eq!(selected == StorageVolumes::legacy(), matches); + } +} + +#[test] +fn corrupt_registry_is_rejected_without_overwriting_it() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("runtimes.json"); + let bytes = b"{broken-registry"; + std::fs::write(&path, bytes).unwrap(); + let error = RuntimeStore::at(&path) + .err() + .expect("corrupt registry must fail closed"); + assert!( + error + .to_string() + .contains("invalid runtime storage registry") + ); + assert_eq!(std::fs::read(&path).unwrap(), bytes); +} + +#[test] +fn lifecycle_lock_rejects_overlapping_operations_and_releases_on_drop() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("runtimes.json"); + let store = RuntimeStore::at(&path).unwrap(); + let error = RuntimeStore::at(&path) + .err() + .expect("second lifecycle operation must not race"); + assert!( + error + .to_string() + .contains("another managed Core lifecycle command") + ); + drop(store); + assert!(RuntimeStore::at(&path).is_ok()); +} + +#[test] +fn invalid_observed_volume_names_are_never_persisted() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("runtimes.json"); + let profile = profile("project-one", "https://cloud.example.test"); + let mut observed = legacy(&profile, Provider::Openai); + observed.storage.as_mut().unwrap().volumes.data = "../../foreign-data".into(); + let mut store = RuntimeStore::at(&path).unwrap(); + assert!( + store + .select(&profile, Provider::Slm, Some(&observed)) + .is_err() + ); + assert!(!path.exists()); +} + +#[test] +fn a_legacy_dataset_already_claimed_by_another_project_cannot_be_adopted() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("runtimes.json"); + let first = profile("project-one", "https://cloud.example.test"); + let second = profile("project-two", "https://cloud.example.test"); + let mut store = RuntimeStore::at(&path).unwrap(); + let volumes = store + .select( + &first, + Provider::Openai, + Some(&legacy(&first, Provider::Openai)), + ) + .unwrap(); + store.save().unwrap(); + drop(store); + let mut store = RuntimeStore::at(&path).unwrap(); + let result = store.select( + &second, + Provider::Openai, + Some(&legacy(&second, Provider::Openai)), + ); + assert!( + result.is_err(), + "one dataset must never belong to two projects" + ); + assert_eq!( + store.select(&first, Provider::Openai, None).unwrap(), + volumes + ); +} + +#[test] +fn selected_dataset_lookup_never_claims_unseen_names() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("runtimes.json"); + let profile = profile("unseen-project", "https://cloud.example.test"); + let store = RuntimeStore::at(&path).unwrap(); + assert!(store.selected_dataset(&profile).unwrap().is_none()); + assert!(!path.exists()); +} + +#[test] +fn adopted_legacy_identity_survives_switching_and_reopening_registry() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("runtimes.json"); + let profile = profile("legacy-project", "https://cloud.example.test"); + let mut store = RuntimeStore::at(&path).unwrap(); + store + .select( + &profile, + Provider::Slm, + Some(&legacy(&profile, Provider::Openai)), + ) + .unwrap(); + let identity = crate::instance::storage::StorageIdentity { + data: Some("2026-01-01T00:00:00Z".into()), + state: Some("2026-01-01T00:00:01Z".into()), + }; + store + .record_identity( + &profile, + Provider::Openai, + &StorageVolumes::legacy(), + identity.clone(), + ) + .unwrap(); + store.save().unwrap(); + drop(store); + let mut store = RuntimeStore::at(&path).unwrap(); + assert_eq!( + store.identity(&profile, Provider::Openai).unwrap(), + Some(&identity) + ); + assert_eq!( + store.select(&profile, Provider::Openai, None).unwrap(), + StorageVolumes::legacy() + ); +} + +#[test] +fn precredential_adoption_preserves_previous_selection_across_key_rotation() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("runtimes.json"); + let mut profile = profile("legacy-project", "https://cloud.example.test"); + let mut observed = legacy(&profile, Provider::Slm); + observed.storage.as_mut().unwrap().project_id = None; + let mut store = RuntimeStore::at(&path).unwrap(); + store.adopt_observed(&profile, Some(&observed)).unwrap(); + let identity = crate::instance::storage::StorageIdentity { + data: Some("original-data".into()), + state: Some("original-state".into()), + }; + store + .record_identity( + &profile, + Provider::Slm, + &StorageVolumes::legacy(), + identity.clone(), + ) + .unwrap(); + store.save().unwrap(); + store + .select(&profile, Provider::Openai, Some(&observed)) + .unwrap(); + drop(store); + profile.api_key = Some("amc_rotated_key".into()); + let mut store = RuntimeStore::at(&path).unwrap(); + assert_eq!( + store.provider(&profile, None, Some(&observed)).unwrap(), + Provider::Slm + ); + assert_eq!( + store + .select(&profile, Provider::Slm, Some(&observed)) + .unwrap(), + StorageVolumes::legacy() + ); + assert_eq!( + store.identity(&profile, Provider::Slm).unwrap(), + Some(&identity) + ); +} diff --git a/crates/cli/src/integrate/detect.rs b/crates/cli/src/integrate/detect.rs index 16f0d78..873c27d 100644 --- a/crates/cli/src/integrate/detect.rs +++ b/crates/cli/src/integrate/detect.rs @@ -4,8 +4,8 @@ use std::path::Path; use serde::Serialize; -use crate::integrate::host::{Host, InstallScope, all_hosts}; -use crate::integrate::path_util::{binary_on_path, home_dir}; +use crate::integrate::host::{Host, HostConfigPaths, all_hosts}; +use crate::integrate::path_util::binary_on_path; #[derive(Debug, Clone, Serialize)] pub struct HostDetectEntry { @@ -20,54 +20,47 @@ pub struct DetectReport { pub hosts: Vec, } -pub fn detect_hosts(cwd: &Path) -> DetectReport { +pub fn detect_hosts(cwd: &Path, config_paths: &HostConfigPaths) -> DetectReport { DetectReport { cwd: cwd.display().to_string(), hosts: all_hosts() .into_iter() - .filter_map(|host| detect_one(host, cwd).ok()) + .map(|host| detect_one(host, config_paths)) .collect(), } } -fn detect_one(host: Host, _cwd: &Path) -> anyhow::Result { +fn detect_one(host: Host, config_paths: &HostConfigPaths) -> HostDetectEntry { let mut signals = Vec::new(); - if binary_on_path(host_binary(host)) { - signals.push(format!("binary `{}` on PATH", host_binary(host))); - } - if host - .config_path(InstallScope::Global, Path::new("."))? - .exists() + if let Some(binary) = host_binaries(host) + .iter() + .copied() + .find(|binary| binary_on_path(binary)) { + signals.push(format!("binary `{binary}` on PATH")); + } + if config_paths.global_config_path(host).exists() { signals.push("global config exists".into()); } - if host_support_dir_exists(host)? { + if config_paths.support_path_exists(host) { signals.push("support directory exists".into()); } - Ok(HostDetectEntry { + HostDetectEntry { host, detected: !signals.is_empty(), signals, - }) + } } -fn host_binary(host: Host) -> &'static str { +fn host_binaries(host: Host) -> &'static [&'static str] { match host { - Host::Cursor => "cursor-agent", - Host::ClaudeCode => "claude", - Host::Codex => "codex", + Host::Cursor => &["cursor-agent"], + Host::ClaudeCode => &["claude"], + Host::Codex => &["codex"], + Host::OpenCode => &["opencode2", "opencode"], } } -fn host_support_dir_exists(host: Host) -> anyhow::Result { - let home = home_dir()?; - Ok(match host { - Host::Cursor => home.join(".cursor").is_dir(), - Host::ClaudeCode => home.join(".claude").is_dir() || home.join(".claude.json").exists(), - Host::Codex => home.join(".codex").is_dir(), - }) -} - pub fn detected_hosts(report: &DetectReport) -> Vec { report .hosts @@ -85,7 +78,9 @@ mod tests { #[test] fn detect_report_lists_all_hosts() { let cwd = env::current_dir().unwrap(); - let report = detect_hosts(&cwd); - assert_eq!(report.hosts.len(), 3); + let home = tempfile::tempdir().unwrap(); + let paths = HostConfigPaths::new(home.path().into(), None); + let report = detect_hosts(&cwd, &paths); + assert_eq!(report.hosts.len(), 4); } } diff --git a/crates/cli/src/integrate/doctor.rs b/crates/cli/src/integrate/doctor.rs index 89a931d..121aa18 100644 --- a/crates/cli/src/integrate/doctor.rs +++ b/crates/cli/src/integrate/doctor.rs @@ -7,11 +7,13 @@ use serde::Serialize; use crate::integrate::codex_edit::{current_codex_entry, read_codex_document}; use crate::integrate::fingerprint::{fingerprint_json, fingerprint_toml}; -use crate::integrate::host::{Host, InstallScope}; +use crate::integrate::host::{Host, HostConfigPaths, InstallScope}; use crate::integrate::path_util::binary_on_path; use crate::integrate::spec::{IntegrateCredentials, codex_mcp_table, json_mcp_server}; -use crate::integrate::state::load_record; -use crate::integrate::write::{current_json_entry, read_json_file}; +use crate::integrate::state::{load_record_for_host, managed_config_path, validated_config_path}; +use crate::integrate::write::{ + current_json_entry, ensure_single_opencode_entry, read_host_json_file, +}; #[derive(Debug, Clone, Serialize, PartialEq, Eq)] #[serde(rename_all = "snake_case")] @@ -42,14 +44,15 @@ pub fn doctor_hosts( hosts: &[Host], scope: InstallScope, cwd: &Path, + config_paths: &HostConfigPaths, creds: Option<&IntegrateCredentials>, ) -> DoctorReport { let mut entries = Vec::new(); for &host in hosts { - match doctor_one(host, scope, cwd, creds) { + match doctor_one(host, scope, cwd, config_paths, creds) { Ok(Some(entry)) => entries.push(entry), Ok(None) => {} - Err(err) => entries.push(failed_doctor_entry(host, scope, cwd, err)), + Err(err) => entries.push(failed_doctor_entry(host, scope, cwd, config_paths, err)), } } DoctorReport { entries } @@ -59,10 +62,10 @@ fn failed_doctor_entry( host: Host, scope: InstallScope, cwd: &Path, + config_paths: &HostConfigPaths, err: anyhow::Error, ) -> HostDoctorEntry { - let path = host - .config_path(scope, cwd) + let path = managed_config_path(config_paths, host, scope, cwd) .map(|p| p.display().to_string()) .unwrap_or_default(); HostDoctorEntry { @@ -78,10 +81,14 @@ fn doctor_one( host: Host, scope: InstallScope, cwd: &Path, + config_paths: &HostConfigPaths, creds: Option<&IntegrateCredentials>, ) -> Result> { - let path = host.config_path(scope, cwd)?; + let path = validated_config_path(config_paths, host, scope, cwd)?; let path_str = path.display().to_string(); + if host == Host::OpenCode { + ensure_single_opencode_entry(&path)?; + } if !binary_on_path("npx") { return Ok(Some(HostDoctorEntry { host, @@ -102,8 +109,8 @@ fn doctor_one( } match host { - Host::Cursor | Host::ClaudeCode => { - let existing = read_json_file(&path); + Host::Cursor | Host::ClaudeCode | Host::OpenCode => { + let existing = read_host_json_file(&path, host); match existing { Ok(doc) => Ok(Some(compare_json(host, scope, &path_str, &doc, creds)?)), Err(err) => Ok(Some(HostDoctorEntry { @@ -136,13 +143,15 @@ fn compare_json( creds: Option<&IntegrateCredentials>, ) -> Result { let expected = creds.map(|c| json_mcp_server(c, host)); - let current = current_json_entry(existing); + let current = current_json_entry(existing, host); let (status, detail) = match current { None => ( DoctorStatus::Missing, Some("no atomicmemory server entry".into()), ), Some(entry) => diagnose_entry( + host, + scope, path, &entry, expected.as_ref(), @@ -175,6 +184,8 @@ fn compare_codex( Some("no atomicmemory server entry".into()), ), Some(entry) => diagnose_entry( + host, + scope, path, &serde_json::to_value(&entry).unwrap_or_default(), expected_json.as_ref(), @@ -192,23 +203,25 @@ fn compare_codex( } fn diagnose_entry( + host: Host, + scope: InstallScope, path: &str, entry: &serde_json::Value, expected: Option<&serde_json::Value>, current_fp: String, profile_name: Option<&str>, ) -> Result<(DoctorStatus, Option)> { - if entry - .get("env") - .and_then(|e| e.get("ATOMICMEMORY_SCOPE_LOCK")) - != Some(&serde_json::Value::String("true".into())) - { - return Ok(( - DoctorStatus::Drift, - Some("missing ATOMICMEMORY_SCOPE_LOCK=true — run `am integrate update`".into()), - )); - } - let record = load_record(Path::new(path))?; + let record = load_record_for_host(Path::new(path), host, scope)?; + diagnose_entry_with_record(entry, expected, current_fp, profile_name, record.as_ref()) +} + +fn diagnose_entry_with_record( + entry: &serde_json::Value, + expected: Option<&serde_json::Value>, + current_fp: String, + profile_name: Option<&str>, + record: Option<&crate::config::IntegrationRecord>, +) -> Result<(DoctorStatus, Option)> { if record.is_none() { let suffix = profile_name .map(|name| format!(" ({name})")) @@ -220,6 +233,17 @@ fn diagnose_entry( )), )); } + if entry + .get("env") + .or_else(|| entry.get("environment")) + .and_then(|e| e.get("ATOMICMEMORY_SCOPE_LOCK")) + != Some(&serde_json::Value::String("true".into())) + { + return Ok(( + DoctorStatus::Drift, + Some("missing ATOMICMEMORY_SCOPE_LOCK=true — run `am integrate update`".into()), + )); + } let Some(record) = record else { unreachable!() }; if record.entry_fingerprint != current_fp { return Ok(( @@ -241,6 +265,7 @@ fn diagnose_entry( #[cfg(test)] mod tests { use super::*; + use crate::config::{IntegrationRecord, ProfileKind}; #[test] fn missing_entry_reports_missing() { @@ -254,4 +279,97 @@ mod tests { .unwrap(); assert_eq!(entry.status, DoctorStatus::Missing); } + + #[test] + fn opencode_valid_unowned_entry_reports_unowned_not_drift() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("opencode.json"); + let doc = serde_json::json!({ + "mcp": { + "servers": { + "atomicmemory": { + "type": "local", + "command": ["npx", "server"], + "environment": { "ATOMICMEMORY_SCOPE_LOCK": "true" } + } + } + } + }); + + let entry = compare_json( + Host::OpenCode, + InstallScope::Global, + path.to_str().unwrap(), + &doc, + None, + ) + .unwrap(); + + assert_eq!(entry.status, DoctorStatus::Unowned); + } + + #[test] + fn opencode_conflicting_entry_without_scope_lock_reports_unowned() { + let entry = serde_json::json!({ + "type": "remote", + "url": "https://example.com/mcp" + }); + + let diagnosis = diagnose_entry_with_record( + &entry, + None, + fingerprint_json(&entry).unwrap(), + Some("local"), + None, + ) + .unwrap(); + + assert_eq!(diagnosis.0, DoctorStatus::Unowned); + } + + #[test] + fn opencode_doctor_classifies_healthy_and_drifted_entries() { + let creds = IntegrateCredentials { + api_url: "http://127.0.0.1:17350".into(), + api_key: "key".into(), + scope_user: "user".into(), + scope_namespace: None, + profile_name: "local".into(), + profile_kind: ProfileKind::Local, + }; + let entry = json_mcp_server(&creds, Host::OpenCode); + let fingerprint = fingerprint_json(&entry).unwrap(); + let record = IntegrationRecord { + host: "opencode".into(), + scope: "global".into(), + config_path: "/tmp/opencode.json".into(), + profile: "local".into(), + installed_at: "2026-09-17T00:00:00Z".into(), + entry_fingerprint: fingerprint.clone(), + prior_entry: None, + }; + + let healthy = diagnose_entry_with_record( + &entry, + Some(&entry), + fingerprint.clone(), + Some("local"), + Some(&record), + ) + .unwrap(); + assert_eq!(healthy.0, DoctorStatus::Ok); + + let mut changed = entry.clone(); + changed["environment"]["ATOMICMEMORY_API_URL"] = + serde_json::Value::String("https://changed.example.com".into()); + let drift = diagnose_entry_with_record( + &changed, + Some(&entry), + fingerprint_json(&changed).unwrap(), + Some("local"), + Some(&record), + ) + .unwrap(); + assert_eq!(drift.0, DoctorStatus::Drift); + } } diff --git a/crates/cli/src/integrate/host.rs b/crates/cli/src/integrate/host.rs index c769f58..9936ed7 100644 --- a/crates/cli/src/integrate/host.rs +++ b/crates/cli/src/integrate/host.rs @@ -6,8 +6,6 @@ use anyhow::{Result, bail}; use clap::ValueEnum; use serde::Serialize; -use crate::integrate::path_util::home_dir; - pub const MCP_SERVER_NAME: &str = "atomicmemory"; pub const PROJECT_SCOPE_UNSUPPORTED: &str = "project-scoped host installs are not supported yet — use global install (omit --project)"; @@ -20,6 +18,9 @@ pub enum Host { Cursor, ClaudeCode, Codex, + #[serde(rename = "opencode")] + #[value(name = "opencode")] + OpenCode, } impl Host { @@ -28,6 +29,7 @@ impl Host { Host::Cursor => "cursor", Host::ClaudeCode => "claude-code", Host::Codex => "codex", + Host::OpenCode => "opencode", } } @@ -40,19 +42,73 @@ impl Host { Host::Cursor => "Cursor", Host::ClaudeCode => "Claude Code", Host::Codex => "Codex", + Host::OpenCode => "OpenCode", } } +} + +/// Resolved process-boundary roots used to locate host configuration files. +#[derive(Debug, Clone)] +pub struct HostConfigPaths { + home: PathBuf, + config_home: PathBuf, +} - pub fn config_path(self, scope: InstallScope, _cwd: &Path) -> Result { +impl HostConfigPaths { + /// Build host paths from home and an optional process-resolved XDG config root. + pub fn new(home: PathBuf, xdg_config_home: Option) -> Self { + let config_home = xdg_config_home.unwrap_or_else(|| home.join(".config")); + Self { home, config_home } + } + + /// Return the configuration file used by a host at the requested scope. + pub fn config_path(&self, host: Host, scope: InstallScope, _cwd: &Path) -> Result { if scope == InstallScope::Project { bail!("{PROJECT_SCOPE_UNSUPPORTED}"); } - let home = home_dir()?; - Ok(match self { - Host::Cursor => home.join(".cursor/mcp.json"), - Host::ClaudeCode => home.join(".claude.json"), - Host::Codex => home.join(".codex/config.toml"), - }) + Ok(self.global_config_path(host)) + } + + /// Return the global configuration file for a host. + pub fn global_config_path(&self, host: Host) -> PathBuf { + match host { + Host::Cursor => self.home.join(".cursor/mcp.json"), + Host::ClaudeCode => self.home.join(".claude.json"), + Host::Codex => self.home.join(".codex/config.toml"), + Host::OpenCode => opencode_config_path(&self.config_home), + } + } + + /// Return every OpenCode V2 global config file in merge order. + pub fn opencode_global_config_paths(&self) -> [PathBuf; 3] { + let dir = self.config_home.join("opencode"); + [ + dir.join("config.json"), + dir.join("opencode.json"), + dir.join("opencode.jsonc"), + ] + } + + /// Return whether a host-specific support directory or file exists. + pub fn support_path_exists(&self, host: Host) -> bool { + match host { + Host::Cursor => self.home.join(".cursor").is_dir(), + Host::ClaudeCode => { + self.home.join(".claude").is_dir() || self.home.join(".claude.json").exists() + } + Host::Codex => self.home.join(".codex").is_dir(), + Host::OpenCode => self.config_home.join("opencode").is_dir(), + } + } +} + +fn opencode_config_path(config_home: &Path) -> PathBuf { + let dir = config_home.join("opencode"); + let jsonc = dir.join("opencode.jsonc"); + if jsonc.exists() { + jsonc + } else { + dir.join("opencode.json") } } @@ -73,8 +129,8 @@ impl InstallScope { } } -pub fn all_hosts() -> [Host; 3] { - [Host::Cursor, Host::ClaudeCode, Host::Codex] +pub fn all_hosts() -> [Host; 4] { + [Host::Cursor, Host::ClaudeCode, Host::Codex, Host::OpenCode] } pub fn parse_host(raw: &str) -> Result { @@ -82,19 +138,58 @@ pub fn parse_host(raw: &str) -> Result { "cursor" => Ok(Host::Cursor), "claude-code" | "claude_code" | "claude" => Ok(Host::ClaudeCode), "codex" => Ok(Host::Codex), - other => bail!("unknown host {other:?} — expected cursor, claude-code, or codex"), + "opencode" => Ok(Host::OpenCode), + other => bail!("unknown host {other:?} — expected cursor, claude-code, codex, or opencode"), } } #[cfg(test)] mod tests { use super::*; + use std::fs; #[test] fn project_scope_is_refused() { - let err = Host::Cursor - .config_path(InstallScope::Project, Path::new("/tmp")) + let paths = HostConfigPaths::new(PathBuf::from("/home/test"), None); + let err = paths + .config_path(Host::Cursor, InstallScope::Project, Path::new("/tmp")) .unwrap_err(); assert!(err.to_string().contains("not supported")); } + + #[test] + fn opencode_path_uses_default_config_home() { + let home = tempfile::tempdir().unwrap(); + let paths = HostConfigPaths::new(home.path().to_path_buf(), None); + assert_eq!( + paths + .config_path(Host::OpenCode, InstallScope::Global, Path::new(".")) + .unwrap(), + home.path().join(".config/opencode/opencode.json") + ); + } + + #[test] + fn opencode_path_respects_xdg_and_existing_jsonc() { + let xdg = tempfile::tempdir().unwrap(); + let dir = xdg.path().join("opencode"); + fs::create_dir_all(&dir).unwrap(); + fs::write(dir.join("opencode.jsonc"), "{}\n").unwrap(); + + let paths = HostConfigPaths::new(PathBuf::from("/home/test"), Some(xdg.path().into())); + assert_eq!( + paths + .config_path(Host::OpenCode, InstallScope::Global, Path::new(".")) + .unwrap(), + dir.join("opencode.jsonc") + ); + } + + #[test] + fn opencode_serializes_with_public_host_id() { + assert_eq!( + serde_json::to_string(&Host::OpenCode).unwrap(), + "\"opencode\"" + ); + } } diff --git a/crates/cli/src/integrate/install.rs b/crates/cli/src/integrate/install.rs index 9a84d37..dfa1d19 100644 --- a/crates/cli/src/integrate/install.rs +++ b/crates/cli/src/integrate/install.rs @@ -5,23 +5,24 @@ use std::path::Path; use anyhow::{Context, Result, bail}; use serde::Serialize; -use serde_json::Value; use crate::integrate::codex_edit::{ merge_codex_mcp, read_codex_document, remove_or_restore_codex_mcp, serialize_codex_entry, write_codex_document, }; -use crate::integrate::host::{Host, InstallScope}; +use crate::integrate::host::{Host, HostConfigPaths, InstallScope}; use crate::integrate::spec::{ IntegrateCredentials, codex_mcp_table, json_mcp_server, preflight_install_runtime, }; use crate::integrate::state::{ - assert_install_allowed, assert_uninstall_allowed, clear_install, clear_stale_record_if_needed, - fingerprint_json_entry, fingerprint_toml_entry, record_install, + assert_install_allowed, assert_uninstall_allowed, clear_install, clear_install_for_host, + clear_stale_record_if_needed, fingerprint_json_entry, fingerprint_toml_entry, + load_record_for_host, managed_config_path, record_install, validated_config_path, }; use crate::integrate::write::{ - backup_host_config, current_json_entry, merge_json_mcp, read_json_file, - remove_or_restore_json_mcp, restore_host_config, write_secure_file, + backup_host_config, current_json_entry, ensure_single_opencode_entry, merge_json_mcp, + opencode_entry_count, read_host_json_file, remove_or_restore_json_mcp, render_json_host_config, + restore_host_config, write_secure_file, }; #[derive(Debug, Clone, Copy, Serialize, PartialEq, Eq)] @@ -55,6 +56,7 @@ pub struct InstallOptions<'a> { pub hosts: &'a [Host], pub scope: InstallScope, pub cwd: &'a Path, + pub config_paths: &'a HostConfigPaths, pub creds: &'a IntegrateCredentials, pub force: bool, pub dry_run: bool, @@ -76,7 +78,7 @@ pub fn install_hosts(opts: &InstallOptions<'_>) -> Result { } Err(err) => { partial_failure = true; - let path = host.config_path(opts.scope, opts.cwd); + let path = managed_config_path(opts.config_paths, *host, opts.scope, opts.cwd); results.push(HostInstallResult { host: *host, scope: opts.scope, @@ -95,8 +97,9 @@ pub fn install_hosts(opts: &InstallOptions<'_>) -> Result { results.push(HostInstallResult { host: *host, scope: opts.scope, - path: host - .config_path(opts.scope, opts.cwd) + path: opts + .config_paths + .config_path(*host, opts.scope, opts.cwd) .map(|p| p.display().to_string()) .unwrap_or_default(), action: opts.action, @@ -126,39 +129,59 @@ struct HostPlan { path: std::path::PathBuf, changed: bool, adopt_only: bool, - merged_json: Option, + rendered_json: Option, codex_doc: Option, new_fingerprint: String, prior_entry: Option, } fn plan_host(host: Host, opts: &InstallOptions<'_>) -> Result { - let path = host.config_path(opts.scope, opts.cwd)?; + let path = validated_config_path(opts.config_paths, host, opts.scope, opts.cwd)?; match host { - Host::Cursor | Host::ClaudeCode => plan_json_host(host, opts, &path), + Host::Cursor | Host::ClaudeCode | Host::OpenCode => plan_json_host(host, opts, &path), Host::Codex => plan_codex_host(host, opts, &path), } } fn plan_json_host(host: Host, opts: &InstallOptions<'_>, path: &Path) -> Result { - let existing = read_json_file(path)?; - let current = current_json_entry(&existing); + if host == Host::OpenCode { + ensure_single_opencode_entry(path)?; + } + let existing = read_host_json_file(path, host)?; + let current = current_json_entry(&existing, host); let current_fp = current.as_ref().map(fingerprint_json_entry).transpose()?; - let owned = is_owned(path, current_fp.as_deref())?; - assert_install_allowed(path, current_fp.as_deref(), opts.force)?; + let record = load_record_for_host(path, host, opts.scope)?; + let owned = record + .as_ref() + .zip(current_fp.as_deref()) + .is_some_and(|(record, current)| record.entry_fingerprint == current); + assert_install_allowed(host, opts.scope, path, current_fp.as_deref(), opts.force)?; let entry = json_mcp_server(opts.creds, host); let effective_force = opts.force || owned; - let (merged, changed) = merge_json_mcp(&existing, &entry, effective_force)?; + let (merged, changed) = merge_json_mcp(&existing, &entry, host, effective_force)?; let new_fp = fingerprint_json_entry(&entry)?; - let adopt_only = !changed && opts.force && !owned && current.is_some(); + let record_moved = record + .as_ref() + .is_some_and(|record| Path::new(&record.config_path) != path); + let adopt_only = + !changed && ((opts.force && !owned && current.is_some()) || (owned && record_moved)); + let rendered_json = changed + .then(|| render_json_host_config(path, host, &merged)) + .transpose()?; let prior_entry = if changed { if owned { - crate::integrate::state::load_record(path)?.and_then(|r| r.prior_entry) + record + .as_ref() + .and_then(|record| record.prior_entry.clone()) } else { current.as_ref().and_then(|v| serde_json::to_string(v).ok()) } } else if adopt_only { - current.as_ref().and_then(|v| serde_json::to_string(v).ok()) + if owned { + record.and_then(|record| record.prior_entry) + } else { + current.as_ref().and_then(|v| serde_json::to_string(v).ok()) + } } else { None }; @@ -167,7 +190,7 @@ fn plan_json_host(host: Host, opts: &InstallOptions<'_>, path: &Path) -> Result< path: path.to_path_buf(), changed, adopt_only, - merged_json: Some(merged), + rendered_json, codex_doc: None, new_fingerprint: new_fp, prior_entry, @@ -179,7 +202,7 @@ fn plan_codex_host(host: Host, opts: &InstallOptions<'_>, path: &Path) -> Result let current = crate::integrate::codex_edit::current_codex_entry(&doc); let current_fp = current.as_ref().map(fingerprint_toml_entry).transpose()?; let owned = is_owned(path, current_fp.as_deref())?; - assert_install_allowed(path, current_fp.as_deref(), opts.force)?; + assert_install_allowed(host, opts.scope, path, current_fp.as_deref(), opts.force)?; let entry = codex_mcp_table(opts.creds, host); let effective_force = opts.force || owned; let changed = merge_codex_mcp(&mut doc, entry.clone(), effective_force)?; @@ -201,7 +224,7 @@ fn plan_codex_host(host: Host, opts: &InstallOptions<'_>, path: &Path) -> Result path: path.to_path_buf(), changed, adopt_only, - merged_json: None, + rendered_json: None, codex_doc: Some(doc), new_fingerprint: new_fp, prior_entry, @@ -263,9 +286,8 @@ fn execute_plan(plan: HostPlan, opts: &InstallOptions<'_>) -> Result Result<()> { - if let Some(merged) = &plan.merged_json { - let rendered = serde_json::to_string_pretty(merged).context("serialize JSON")?; - write_secure_file(&plan.path, &format!("{rendered}\n"))?; + if let Some(rendered) = &plan.rendered_json { + write_secure_file(&plan.path, rendered)?; } else if let Some(doc) = &plan.codex_doc { write_codex_document(&plan.path, doc)?; } @@ -328,13 +350,14 @@ pub fn uninstall_hosts( hosts: &[Host], scope: InstallScope, cwd: &Path, + config_paths: &HostConfigPaths, force: bool, dry_run: bool, ) -> Result { let mut results = Vec::new(); let mut partial_failure = false; for host in hosts { - match uninstall_host(*host, scope, cwd, force, dry_run) { + match uninstall_host(*host, scope, cwd, config_paths, force, dry_run) { Ok(row) => { if row.error.is_some() { partial_failure = true; @@ -405,10 +428,11 @@ fn uninstall_host( host: Host, scope: InstallScope, cwd: &Path, + config_paths: &HostConfigPaths, force: bool, dry_run: bool, ) -> Result { - let path = host.config_path(scope, cwd)?; + let path = validated_config_path(config_paths, host, scope, cwd)?; if !path.exists() { return Ok(uninstall_row( host, @@ -418,7 +442,9 @@ fn uninstall_host( )); } match host { - Host::Cursor | Host::ClaudeCode => uninstall_json_host(host, scope, &path, force, dry_run), + Host::Cursor | Host::ClaudeCode | Host::OpenCode => { + uninstall_json_host(host, scope, &path, force, dry_run) + } Host::Codex => uninstall_codex_host(host, scope, &path, force, dry_run), } } @@ -430,10 +456,12 @@ fn uninstall_json_host( force: bool, dry_run: bool, ) -> Result { - let existing = read_json_file(path)?; - let current = current_json_entry(&existing); + let existing = read_host_json_file(path, host)?; + let current = current_json_entry(&existing, host); + let hidden_opencode_entries = + host == Host::OpenCode && current.is_none() && opencode_entry_count(path)? > 0; let current_fp = current.as_ref().map(fingerprint_json_entry).transpose()?; - if current.is_none() { + if current.is_none() && !hidden_opencode_entries { return Ok(uninstall_row( host, scope, @@ -441,7 +469,17 @@ fn uninstall_json_host( stale_uninstall_outcome(path, dry_run, None)?, )); } - let restore = assert_uninstall_allowed(path, current_fp.as_deref(), force)?; + let restore = if hidden_opencode_entries { + if !force { + bail!( + "OpenCode config contains a hidden duplicate `{}` entry — pass --force to remove every duplicate", + crate::integrate::host::MCP_SERVER_NAME + ); + } + None + } else { + assert_uninstall_allowed(host, scope, path, current_fp.as_deref(), force)? + }; if dry_run { return Ok(uninstall_row( host, @@ -458,7 +496,8 @@ fn uninstall_json_host( } let backup = backup_host_config(path)?; let backup_path = backup.as_deref(); - let (merged, removed) = remove_or_restore_json_mcp(&existing, restore.as_deref())?; + let (merged, removed) = remove_or_restore_json_mcp(&existing, host, restore.as_deref())?; + let removed = removed || hidden_opencode_entries; if !removed { return Ok(uninstall_row( host, @@ -473,9 +512,9 @@ fn uninstall_json_host( }, )); } - let rendered = serde_json::to_string_pretty(&merged).context("serialize JSON")?; - write_secure_file(path, &format!("{rendered}\n"))?; - if let Err(err) = clear_install(path) { + let rendered = render_json_host_config(path, host, &merged)?; + write_secure_file(path, &rendered)?; + if let Err(err) = clear_install_for_host(path, host, scope) { restore_host_config(path, backup_path)?; return Ok(uninstall_row( host, @@ -522,7 +561,7 @@ fn uninstall_codex_host( stale_uninstall_outcome(path, dry_run, None)?, )); } - let restore = assert_uninstall_allowed(path, current_fp.as_deref(), force)?; + let restore = assert_uninstall_allowed(host, scope, path, current_fp.as_deref(), force)?; if dry_run { return Ok(uninstall_row( host, @@ -700,7 +739,7 @@ pub fn select_hosts_interactive( bail!("non-interactive session requires --yes and/or explicit --host"); } if yes && detected.is_empty() { - bail!("no hosts detected — pass --host cursor|claude-code|codex"); + bail!("no hosts detected — pass --host cursor|claude-code|codex|opencode"); } if detected.is_empty() { eprintln!("No hosts auto-detected. Select hosts to configure:"); @@ -724,6 +763,7 @@ mod tests { use crate::config::ProfileKind; use crate::integrate::all_hosts; use crate::integrate::spec::json_mcp_server; + use serde_json::Value; use serde_json::json; #[test] @@ -842,7 +882,7 @@ mod tests { }; let entry = json_mcp_server(&creds, Host::Cursor); let existing = json!({ "mcpServers": { "atomicmemory": entry.clone() } }); - let (_, changed) = merge_json_mcp(&existing, &entry, false).unwrap(); + let (_, changed) = merge_json_mcp(&existing, &entry, Host::Cursor, false).unwrap(); assert!(!changed); } @@ -851,7 +891,7 @@ mod tests { let prior = json!({ "command": "legacy" }); let existing = json!({ "mcpServers": { "atomicmemory": { "command": "npx" } } }); let (merged, changed) = - remove_or_restore_json_mcp(&existing, Some(&prior.to_string())).unwrap(); + remove_or_restore_json_mcp(&existing, Host::Cursor, Some(&prior.to_string())).unwrap(); assert!(changed); assert_eq!(merged["mcpServers"]["atomicmemory"], prior); } @@ -873,10 +913,12 @@ mod tests { std::fs::write(&path, serde_json::to_string_pretty(&existing).unwrap()).unwrap(); let hosts = [Host::Cursor]; + let config_paths = HostConfigPaths::new(dir.path().into(), None); let opts = InstallOptions { hosts: &hosts, scope: InstallScope::Global, cwd: dir.path(), + config_paths: &config_paths, creds: &creds, force: true, dry_run: false, diff --git a/crates/cli/src/integrate/mod.rs b/crates/cli/src/integrate/mod.rs index 03d048f..7769cdd 100644 --- a/crates/cli/src/integrate/mod.rs +++ b/crates/cli/src/integrate/mod.rs @@ -13,7 +13,9 @@ pub mod write; pub use detect::{DetectReport, detect_hosts, detected_hosts}; pub use doctor::{DoctorReport, DoctorStatus, doctor_hosts}; -pub use host::{Host, InstallScope, PROJECT_SCOPE_UNSUPPORTED, all_hosts, parse_host}; +pub use host::{ + Host, HostConfigPaths, InstallScope, PROJECT_SCOPE_UNSUPPORTED, all_hosts, parse_host, +}; pub use install::{ InstallAction, InstallReport, install_hosts, select_hosts_interactive, uninstall_hosts, }; diff --git a/crates/cli/src/integrate/spec.rs b/crates/cli/src/integrate/spec.rs index a7e2030..927e866 100644 --- a/crates/cli/src/integrate/spec.rs +++ b/crates/cli/src/integrate/spec.rs @@ -44,7 +44,7 @@ pub async fn resolve_credentials(global: &GlobalOptions) -> Result { let url = profile.memory_base_url.clone(); - let key = resolve_local_core_key(&profile.name, &url).await?; + let key = resolve_local_core_key(&profile).await?; (url, key) } ProfileKind::Cloud => { @@ -68,11 +68,11 @@ pub fn preflight_install_runtime() -> Result<()> { require_npx() } -async fn resolve_local_core_key(profile_name: &str, local_url: &str) -> Result { +async fn resolve_local_core_key(profile: &crate::config::ResolvedProfile) -> Result { if let Some(key) = resolve_core_api_key() { return Ok(key); } - if let Some(key) = crate::instance::read_managed_core_api_key(profile_name, local_url).await { + if let Some(key) = crate::instance::read_managed_core_api_key(profile).await? { return Ok(key); } bail!( @@ -113,11 +113,21 @@ fn push_scope_env(env: &mut Map, creds: &IntegrateCredentials, ho } } -/// JSON MCP server entry for Cursor / Claude Code. +/// JSON MCP server entry for Cursor, Claude Code, or OpenCode. pub fn json_mcp_server(creds: &IntegrateCredentials, host: Host) -> Value { let mut env = Map::new(); push_scope_env(&mut env, creds, host); let (command, args) = launcher_command(); + if host == Host::OpenCode { + let mut command = vec![Value::String(command)]; + command.extend(args.into_iter().map(Value::String)); + return json!({ + "type": "local", + "command": command, + "environment": Value::Object(env), + "codemode": false, + }); + } json!({ "type": "stdio", "command": command, @@ -219,6 +229,50 @@ mod tests { ); } + #[test] + fn opencode_mcp_uses_v2_local_server_shape() { + let creds = IntegrateCredentials { + api_url: "http://127.0.0.1:17350".into(), + api_key: "local-dev-key".into(), + scope_user: "pip".into(), + scope_namespace: Some("proj".into()), + profile_name: "local".into(), + profile_kind: ProfileKind::Local, + }; + + let entry = json_mcp_server(&creds, Host::OpenCode); + + assert_eq!(entry["type"], "local"); + #[cfg(windows)] + assert_eq!( + entry["command"], + json!([ + "cmd", + "/c", + "npx", + "-y", + "--package", + MCP_SERVER_PACKAGE, + "atomicmemory-mcp" + ]) + ); + #[cfg(not(windows))] + assert_eq!( + entry["command"], + json!([ + "npx", + "-y", + "--package", + MCP_SERVER_PACKAGE, + "atomicmemory-mcp" + ]) + ); + assert_eq!(entry["environment"]["ATOMICMEMORY_SCOPE_AGENT"], "opencode"); + assert_eq!(entry["codemode"], false); + assert!(entry.get("args").is_none()); + assert!(entry.get("env").is_none()); + } + #[cfg(windows)] #[test] fn windows_launcher_uses_cmd() { diff --git a/crates/cli/src/integrate/state.rs b/crates/cli/src/integrate/state.rs index 6b33bfd..5ea73b9 100644 --- a/crates/cli/src/integrate/state.rs +++ b/crates/cli/src/integrate/state.rs @@ -5,13 +5,16 @@ use chrono::Utc; use anyhow::{Result, bail}; use serde::Serialize; -use crate::config::{ConfigStore, IntegrationRecord}; +use crate::config::{ConfigFile, ConfigStore, IntegrationRecord}; use crate::integrate::codex_edit::{current_codex_entry, read_codex_document}; use crate::integrate::fingerprint::{fingerprint_json, fingerprint_toml}; -use crate::integrate::host::{Host, InstallScope}; +use crate::integrate::host::{Host, HostConfigPaths, InstallScope}; use crate::integrate::path_util::canonical_path; -use crate::integrate::write::{current_json_entry, read_json_file}; -use std::path::Path; +use crate::integrate::write::{ + current_json_entry, ensure_no_opencode_sibling_entry, ensure_single_opencode_entry, + read_host_json_file, +}; +use std::path::{Path, PathBuf}; #[derive(Debug, Clone, Serialize, PartialEq, Eq)] pub struct StaleRecordCleanup { @@ -45,6 +48,42 @@ pub fn load_record(config_path: &Path) -> Result> { load_record_in(&ConfigStore::production()?, config_path) } +fn matching_record_in( + cfg: &ConfigFile, + config_path: &Path, + host: Host, + scope: InstallScope, +) -> Result> { + let key = record_key(config_path)?; + if let Some(record) = cfg.integrations.get(&key) { + return Ok(Some(record.clone())); + } + if host != Host::OpenCode { + return Ok(None); + } + let mut relocated = cfg.integrations.values().filter(|record| { + let recorded_path = Path::new(&record.config_path); + record.host == host.id() + && record.scope == scope.id() + && recorded_path.parent() == config_path.parent() + && !recorded_path.exists() + }); + let record = relocated.next().cloned(); + if relocated.next().is_some() { + bail!("multiple stale OpenCode ownership records found"); + } + Ok(record) +} + +pub fn load_record_for_host( + config_path: &Path, + host: Host, + scope: InstallScope, +) -> Result> { + let cfg = ConfigStore::production()?.load()?; + matching_record_in(&cfg, config_path, host, scope) +} + pub(crate) fn record_install_in( store: &ConfigStore, host: Host, @@ -65,6 +104,10 @@ pub(crate) fn record_install_in( prior_entry, }; store.update(|cfg| { + if host == Host::OpenCode { + cfg.integrations + .retain(|_, existing| existing.host != host.id() || existing.scope != scope.id()); + } cfg.integrations.insert(key, record); Ok(()) }) @@ -101,6 +144,26 @@ pub fn clear_install(config_path: &Path) -> Result<()> { clear_install_in(&ConfigStore::production()?, config_path) } +pub fn clear_install_for_host(config_path: &Path, host: Host, scope: InstallScope) -> Result<()> { + let key = record_key(config_path)?; + ConfigStore::production()?.update(|cfg| { + cfg.integrations.retain(|record_key, record| { + if record_key == &key { + return false; + } + if host != Host::OpenCode { + return true; + } + let recorded_path = Path::new(&record.config_path); + !(record.host == host.id() + && record.scope == scope.id() + && recorded_path.parent() == config_path.parent() + && !recorded_path.exists()) + }); + Ok(()) + }) +} + pub(crate) fn clear_stale_record_if_needed_in( store: &ConfigStore, config_path: &Path, @@ -132,19 +195,104 @@ pub fn clear_stale_record_if_needed( clear_stale_record_if_needed_in(&ConfigStore::production()?, config_path, dry_run) } -pub fn list_owned_status(hosts: &[Host]) -> Result> { +/// Resolve a host config path, retaining the OpenCode file recorded at install time. +pub fn managed_config_path( + config_paths: &HostConfigPaths, + host: Host, + scope: InstallScope, + cwd: &Path, +) -> Result { + let cfg = ConfigStore::production()?.load()?; + managed_config_path_in(config_paths, host, scope, cwd, &cfg) +} + +fn managed_config_path_in( + config_paths: &HostConfigPaths, + host: Host, + scope: InstallScope, + cwd: &Path, + cfg: &ConfigFile, +) -> Result { + let default_path = config_paths.config_path(host, scope, cwd)?; + if host != Host::OpenCode { + return Ok(default_path); + } + let candidates = config_paths.opencode_global_config_paths(); + let mut owned_paths = cfg + .integrations + .values() + .filter(|record| record.host == host.id() && record.scope == scope.id()) + .map(|record| PathBuf::from(&record.config_path)) + .filter(|path| candidates.contains(path) && path.exists()); + let Some(path) = owned_paths.next() else { + return Ok(default_path); + }; + if owned_paths.next().is_some() { + bail!("multiple owned OpenCode config paths found — remove the stale ownership record"); + } + Ok(path) +} + +/// Resolve a host path and reject an OpenCode sibling that defines the same server. +pub fn validated_config_path( + config_paths: &HostConfigPaths, + host: Host, + scope: InstallScope, + cwd: &Path, +) -> Result { + let path = managed_config_path(config_paths, host, scope, cwd)?; + if host == Host::OpenCode { + ensure_no_opencode_sibling_entry(config_paths, &path)?; + } + Ok(path) +} + +pub fn list_owned_status( + hosts: &[Host], + config_paths: &HostConfigPaths, +) -> Result> { let cfg = ConfigStore::production()?.load()?; let mut out = Vec::new(); for host in hosts { - let path = host.config_path(InstallScope::Global, Path::new("."))?; + let path = match managed_config_path_in( + config_paths, + *host, + InstallScope::Global, + Path::new("."), + &cfg, + ) { + Ok(path) => path, + Err(_) => { + let path = config_paths.config_path(*host, InstallScope::Global, Path::new("."))?; + let record = cfg.integrations.values().find(|record| { + record.host == host.id() && record.scope == InstallScope::Global.id() + }); + out.push(OwnedInstallStatus { + host: *host, + config_path: record_key(&path)?, + owned: record.is_some(), + fingerprint_match: false, + profile: record.map(|record| record.profile.clone()), + }); + continue; + } + }; let key = record_key(&path)?; - let record = cfg.integrations.get(&key); - let fingerprint_match = match (record, host) { - (Some(record), Host::Cursor | Host::ClaudeCode) => read_json_file(&path) - .ok() - .and_then(|doc| current_json_entry(&doc)) - .and_then(|entry| fingerprint_json_entry(&entry).ok()) - .is_some_and(|fp| fp == record.entry_fingerprint), + let record = matching_record_in(&cfg, &path, *host, InstallScope::Global)?; + let fingerprint_match = match (record.as_ref(), host) { + (Some(record), Host::Cursor | Host::ClaudeCode | Host::OpenCode) => { + let sibling_safe = *host != Host::OpenCode + || ensure_no_opencode_sibling_entry(config_paths, &path).is_ok(); + let target_unambiguous = + *host != Host::OpenCode || ensure_single_opencode_entry(&path).is_ok(); + sibling_safe + && target_unambiguous + && read_host_json_file(&path, *host) + .ok() + .and_then(|doc| current_json_entry(&doc, *host)) + .and_then(|entry| fingerprint_json_entry(&entry).ok()) + .is_some_and(|fp| fp == record.entry_fingerprint) + } (Some(record), Host::Codex) => read_codex_document(&path) .ok() .and_then(|doc| current_codex_entry(&doc)) @@ -157,18 +305,20 @@ pub fn list_owned_status(hosts: &[Host]) -> Result> { config_path: key, owned: record.is_some(), fingerprint_match, - profile: record.map(|r| r.profile.clone()), + profile: record.map(|r| r.profile), }); } Ok(out) } pub fn assert_install_allowed( + host: Host, + scope: InstallScope, config_path: &Path, current_fingerprint: Option<&str>, force: bool, ) -> Result> { - let record = load_record(config_path)?; + let record = load_record_for_host(config_path, host, scope)?; let Some(current) = current_fingerprint else { return Ok(None); }; @@ -194,11 +344,13 @@ pub fn assert_install_allowed( } pub fn assert_uninstall_allowed( + host: Host, + scope: InstallScope, config_path: &Path, current_fingerprint: Option<&str>, force: bool, ) -> Result> { - let record = load_record(config_path)?; + let record = load_record_for_host(config_path, host, scope)?; let Some(record) = record else { if force { return Ok(None); @@ -257,7 +409,14 @@ mod tests { fn install_refuses_unowned_without_force() { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("mcp.json"); - let err = assert_install_allowed(&path, Some("deadbeef"), false).unwrap_err(); + let err = assert_install_allowed( + Host::Cursor, + InstallScope::Global, + &path, + Some("deadbeef"), + false, + ) + .unwrap_err(); assert!(err.to_string().contains("not installed by `am integrate`")); } @@ -265,7 +424,14 @@ mod tests { fn uninstall_refuses_drift_without_force() { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("mcp.json"); - let err = assert_uninstall_allowed(&path, Some("deadbeef"), false).unwrap_err(); + let err = assert_uninstall_allowed( + Host::Cursor, + InstallScope::Global, + &path, + Some("deadbeef"), + false, + ) + .unwrap_err(); assert!(err.to_string().contains("no owned")); } diff --git a/crates/cli/src/integrate/write.rs b/crates/cli/src/integrate/write.rs index 370ee78..aebdc49 100644 --- a/crates/cli/src/integrate/write.rs +++ b/crates/cli/src/integrate/write.rs @@ -7,10 +7,12 @@ use std::sync::atomic::{AtomicU64, Ordering}; use std::time::{SystemTime, UNIX_EPOCH}; use anyhow::{Context, Result, bail}; +use jsonc_parser::ParseOptions; +use jsonc_parser::cst::{CstInputValue, CstObject, CstObjectProp, CstRootNode}; use serde_json::Value; use crate::config::config_dir; -use crate::integrate::host::MCP_SERVER_NAME; +use crate::integrate::host::{Host, MCP_SERVER_NAME}; use crate::integrate::path_util::home_dir; static TMP_COUNTER: AtomicU64 = AtomicU64::new(0); @@ -262,7 +264,28 @@ fn guard_write_target(link: &Path, real_target: &Path) -> Result<()> { Ok(()) } +/// Read a JSON host configuration, returning an empty object for a missing or empty file. pub fn read_json_file(path: &Path) -> Result { + read_json_document(path, "JSON", |raw| Ok(serde_json::from_str(raw)?)) +} + +/// Read a host configuration using OpenCode's JSONC support when required. +pub fn read_host_json_file(path: &Path, host: Host) -> Result { + if host != Host::OpenCode { + return read_json_file(path); + } + read_json_document(path, "JSONC", |raw| { + let root = parse_jsonc_root(raw)?; + root.to_serde_value() + .context("JSONC config does not contain a value") + }) +} + +fn read_json_document( + path: &Path, + format: &str, + parse: impl FnOnce(&str) -> Result, +) -> Result { if !path.exists() { return Ok(Value::Object(Default::default())); } @@ -270,8 +293,7 @@ pub fn read_json_file(path: &Path) -> Result { if raw.trim().is_empty() { return Ok(Value::Object(Default::default())); } - let value: Value = - serde_json::from_str(&raw).with_context(|| format!("parse JSON {}", path.display()))?; + let value = parse(&raw).with_context(|| format!("parse {format} {}", path.display()))?; if !value.is_object() { bail!( "{} root must be a JSON object — refusing to overwrite", @@ -281,19 +303,207 @@ pub fn read_json_file(path: &Path) -> Result { Ok(value) } -pub fn current_json_entry(existing: &Value) -> Option { - existing - .get("mcpServers") - .and_then(|s| s.get(MCP_SERVER_NAME)) - .cloned() +fn parse_jsonc_root(raw: &str) -> Result { + CstRootNode::parse(raw, &ParseOptions::default()).map_err(anyhow::Error::msg) +} + +fn named_properties(object: &CstObject, name: &str) -> Vec { + object + .properties() + .into_iter() + .filter(|property| property.decoded_name().as_deref() == Some(name)) + .collect() +} + +fn opencode_server_objects(root: &CstObject) -> Result> { + let mut server_objects = Vec::new(); + for mcp_property in named_properties(root, "mcp") { + let mcp = mcp_property + .value() + .and_then(|value| value.as_object()) + .context("mcp must be a JSON object")?; + for servers_property in named_properties(&mcp, "servers") { + let servers = servers_property + .value() + .and_then(|value| value.as_object()) + .context("mcp.servers must be a JSON object")?; + server_objects.push(servers); + } + } + Ok(server_objects) +} + +fn opencode_entry_properties(root: &CstObject) -> Result> { + let mut entries = Vec::new(); + for servers in opencode_server_objects(root)? { + entries.extend(named_properties(&servers, MCP_SERVER_NAME)); + } + Ok(entries) +} + +fn ensure_unambiguous_opencode_containers(root: &CstObject) -> Result<()> { + let mcp_properties = named_properties(root, "mcp"); + if mcp_properties.len() > 1 { + bail!("OpenCode config contains duplicate `mcp` properties"); + } + let Some(mcp) = mcp_properties.first() else { + return Ok(()); + }; + let mcp = mcp + .value() + .and_then(|value| value.as_object()) + .context("mcp must be a JSON object")?; + if named_properties(&mcp, "servers").len() > 1 { + bail!("OpenCode config contains duplicate `mcp.servers` properties"); + } + Ok(()) +} + +/// Count managed OpenCode properties across duplicate parent containers. +pub fn opencode_entry_count(path: &Path) -> Result { + if !path.exists() { + return Ok(0); + } + let raw = fs::read_to_string(path).with_context(|| format!("read {}", path.display()))?; + if raw.trim().is_empty() { + return Ok(0); + } + let root = parse_jsonc_root(&raw).with_context(|| format!("parse JSONC {}", path.display()))?; + let object = root + .object_value() + .with_context(|| format!("{} root must be a JSON object", path.display()))?; + opencode_entry_properties(&object).map(|entries| entries.len()) +} + +/// Refuse an ambiguous OpenCode target with duplicate managed server properties. +pub fn ensure_single_opencode_entry(path: &Path) -> Result<()> { + if !path.exists() { + return Ok(()); + } + let raw = fs::read_to_string(path).with_context(|| format!("read {}", path.display()))?; + if raw.trim().is_empty() { + return Ok(()); + } + let root = parse_jsonc_root(&raw).with_context(|| format!("parse JSONC {}", path.display()))?; + let object = root + .object_value() + .with_context(|| format!("{} root must be a JSON object", path.display()))?; + ensure_unambiguous_opencode_containers(&object)?; + if opencode_entry_properties(&object)?.len() > 1 { + bail!( + "{} contains duplicate `mcp.servers.{MCP_SERVER_NAME}` entries — remove duplicates before retrying", + path.display() + ); + } + Ok(()) +} + +/// Fail closed when another OpenCode global config file also defines the managed server. +pub fn ensure_no_opencode_sibling_entry( + config_paths: &crate::integrate::host::HostConfigPaths, + target: &Path, +) -> Result<()> { + for sibling in config_paths.opencode_global_config_paths() { + if sibling == target || !sibling.exists() { + continue; + } + if opencode_entry_count(&sibling)? > 0 { + bail!( + "OpenCode also defines `mcp.servers.{MCP_SERVER_NAME}` in {} — remove the sibling entry before retrying", + sibling.display() + ); + } + } + Ok(()) +} + +fn cst_input(value: &Value) -> CstInputValue { + match value { + Value::Null => CstInputValue::Null, + Value::Bool(value) => CstInputValue::Bool(*value), + Value::Number(value) => CstInputValue::Number(value.to_string()), + Value::String(value) => CstInputValue::String(value.clone()), + Value::Array(values) => CstInputValue::Array(values.iter().map(cst_input).collect()), + Value::Object(values) => CstInputValue::Object( + values + .iter() + .map(|(key, value)| (key.clone(), cst_input(value))) + .collect(), + ), + } } +/// Render a merged JSON host config, preserving OpenCode JSONC trivia and formatting. +pub fn render_json_host_config(path: &Path, host: Host, merged: &Value) -> Result { + if host != Host::OpenCode { + let rendered = serde_json::to_string_pretty(merged).context("serialize JSON")?; + return Ok(format!("{rendered}\n")); + } + let raw = if path.exists() { + fs::read_to_string(path).with_context(|| format!("read {}", path.display()))? + } else { + "{}".into() + }; + let root = parse_jsonc_root(if raw.trim().is_empty() { "{}" } else { &raw })?; + let root_object = root + .object_value() + .context("OpenCode config root must be a JSON object")?; + apply_opencode_entry(&root_object, current_json_entry(merged, host))?; + let mut rendered = root.to_string(); + if raw.ends_with('\n') && !rendered.ends_with('\n') { + rendered.push('\n'); + } + Ok(rendered) +} + +fn apply_opencode_entry(root: &jsonc_parser::cst::CstObject, entry: Option) -> Result<()> { + if let Some(entry) = entry { + ensure_unambiguous_opencode_containers(root)?; + let current_entries = opencode_entry_properties(root)?; + if current_entries.len() > 1 { + bail!( + "OpenCode config contains duplicate `mcp.servers.{MCP_SERVER_NAME}` entries — remove duplicates before retrying" + ); + } + if let Some(current) = current_entries.first() { + current.set_value(cst_input(&entry)); + return Ok(()); + } + let mcp = root + .object_value_or_create("mcp") + .context("mcp must be a JSON object")?; + let servers = mcp + .object_value_or_create("servers") + .context("mcp.servers must be a JSON object")?; + servers.append(MCP_SERVER_NAME, cst_input(&entry)); + return Ok(()); + } + for current in opencode_entry_properties(root)? { + current.remove(); + } + // Keep empty containers: comments attached inside `mcp` or `servers` are + // user configuration and must survive uninstall. + Ok(()) +} + +/// Return the managed MCP server entry for a JSON-based host. +pub fn current_json_entry(existing: &Value, host: Host) -> Option { + let servers = if host == Host::OpenCode { + existing.get("mcp").and_then(|mcp| mcp.get("servers")) + } else { + existing.get("mcpServers") + }; + servers.and_then(|s| s.get(MCP_SERVER_NAME)).cloned() +} + +/// Merge the managed MCP server entry while preserving unrelated host settings. pub fn merge_json_mcp( existing: &Value, server_entry: &Value, + host: Host, force: bool, ) -> Result<(Value, bool)> { - let current = current_json_entry(existing); + let current = current_json_entry(existing, host); if let Some(current) = ¤t { if current == server_entry { return Ok((existing.clone(), false)); @@ -306,30 +516,54 @@ pub fn merge_json_mcp( let obj = root .as_object_mut() .context("host config root must be a JSON object")?; - let servers = obj - .entry("mcpServers") - .or_insert_with(|| Value::Object(Default::default())); + let servers = if host == Host::OpenCode { + let mcp = obj + .entry("mcp") + .or_insert_with(|| Value::Object(Default::default())) + .as_object_mut() + .context("mcp must be a JSON object")?; + mcp.entry("servers") + .or_insert_with(|| Value::Object(Default::default())) + } else { + obj.entry("mcpServers") + .or_insert_with(|| Value::Object(Default::default())) + }; let map = servers .as_object_mut() - .context("mcpServers must be a JSON object")?; + .context("MCP servers must be a JSON object")?; map.insert(MCP_SERVER_NAME.to_string(), server_entry.clone()); Ok((root, true)) } +/// Remove the managed MCP entry or restore the entry replaced during installation. pub fn remove_or_restore_json_mcp( existing: &Value, + host: Host, restore_entry: Option<&str>, ) -> Result<(Value, bool)> { let mut root = existing.clone(); let (changed, remove_empty_servers) = { - let Some(obj) = root.as_object_mut() else { - return Ok((root, false)); - }; - let Some(servers) = obj.get_mut("mcpServers") else { - return Ok((root, false)); - }; - let Some(map) = servers.as_object_mut() else { - return Ok((root, false)); + let obj = root + .as_object_mut() + .context("host config root must be a JSON object")?; + let map = if host == Host::OpenCode { + let Some(mcp) = obj.get_mut("mcp") else { + return Ok((root, false)); + }; + let mcp = mcp.as_object_mut().context("mcp must be a JSON object")?; + let Some(servers) = mcp.get_mut("servers") else { + return Ok((root, false)); + }; + servers + .as_object_mut() + .context("mcp.servers must be a JSON object")? + } else { + let Some(servers) = obj.get_mut("mcpServers") else { + return Ok((root, false)); + }; + servers + .as_object_mut() + .context("mcpServers must be a JSON object")? }; if let Some(raw) = restore_entry { let entry: Value = serde_json::from_str(raw).context("parse stored prior MCP entry")?; @@ -342,7 +576,16 @@ pub fn remove_or_restore_json_mcp( }; if remove_empty_servers { if let Some(obj) = root.as_object_mut() { - obj.remove("mcpServers"); + if host == Host::OpenCode { + if let Some(mcp) = obj.get_mut("mcp").and_then(Value::as_object_mut) { + mcp.remove("servers"); + if mcp.is_empty() { + obj.remove("mcp"); + } + } + } else { + obj.remove("mcpServers"); + } } } Ok((root, changed)) @@ -357,7 +600,45 @@ mod tests { fn merge_refuses_conflicting_entry_without_force() { let existing = json!({ "mcpServers": { "atomicmemory": { "command": "old" } } }); let entry = json!({ "command": "npx" }); - assert!(merge_json_mcp(&existing, &entry, false).is_err()); + assert!(merge_json_mcp(&existing, &entry, Host::Cursor, false).is_err()); + } + + #[test] + fn merge_opencode_preserves_unrelated_config_and_is_idempotent() { + let existing = json!({ + "$schema": "https://opencode.ai/config.json", + "model": "example/model", + "mcp": { + "timeout": { "startup": 45_000 }, + "servers": { "other": { "type": "remote", "url": "https://example.com/mcp" } } + } + }); + let entry = json!({ "type": "local", "command": ["npx", "server"] }); + + let (merged, changed) = merge_json_mcp(&existing, &entry, Host::OpenCode, false).unwrap(); + assert!(changed); + assert_eq!(merged["model"], "example/model"); + assert_eq!(merged["mcp"]["timeout"]["startup"], 45_000); + assert_eq!(merged["mcp"]["servers"]["other"]["type"], "remote"); + assert_eq!(merged["mcp"]["servers"]["atomicmemory"], entry); + + let (again, changed) = merge_json_mcp(&merged, &entry, Host::OpenCode, false).unwrap(); + assert!(!changed); + assert_eq!(again, merged); + } + + #[test] + fn merge_opencode_requires_force_for_conflicting_entry() { + let existing = json!({ + "mcp": { "servers": { "atomicmemory": { "type": "remote", "url": "https://example.com" } } } + }); + let entry = json!({ "type": "local", "command": ["npx", "server"] }); + + assert!(merge_json_mcp(&existing, &entry, Host::OpenCode, false).is_err()); + let (merged, changed) = merge_json_mcp(&existing, &entry, Host::OpenCode, true).unwrap(); + + assert!(changed); + assert_eq!(merged["mcp"]["servers"]["atomicmemory"], entry); } #[test] @@ -368,6 +649,108 @@ mod tests { assert!(read_json_file(&path).is_err()); } + #[test] + fn reads_opencode_jsonc_comments_and_trailing_commas() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("opencode.jsonc"); + fs::write( + &path, + r#"{ + // Keep the user's model setting. + "model": "example/model", + "note": "https://example.com/a,b//c/*d*/", + "mcp": { + "servers": {}, + }, + }"#, + ) + .unwrap(); + + let config = read_host_json_file(&path, Host::OpenCode).unwrap(); + + assert_eq!(config["model"], "example/model"); + assert_eq!(config["note"], "https://example.com/a,b//c/*d*/"); + assert!(config["mcp"]["servers"].is_object()); + + let entry = json!({ "type": "local", "command": ["npx", "server"] }); + let (merged, _) = merge_json_mcp(&config, &entry, Host::OpenCode, false).unwrap(); + let rendered = render_json_host_config(&path, Host::OpenCode, &merged).unwrap(); + assert!(rendered.contains("// Keep the user's model setting.")); + assert!(rendered.contains("https://example.com/a,b//c/*d*/")); + assert!(rendered.contains("\"atomicmemory\"")); + assert!(rendered.contains(",\n },")); + } + + #[test] + fn opencode_install_rejects_duplicate_managed_properties() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("opencode.jsonc"); + fs::write( + &path, + r#"{ + "mcp": { "servers": { + "atomicmemory": { "type": "remote", "url": "https://first.example" }, + "atomicmemory": { "type": "remote", "url": "https://second.example" }, + } }, +}"#, + ) + .unwrap(); + + let error = ensure_single_opencode_entry(&path).unwrap_err(); + assert!(error.to_string().contains("duplicate")); + } + + #[test] + fn opencode_uninstall_removes_all_duplicate_managed_properties() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("opencode.jsonc"); + fs::write( + &path, + r#"{ + "mcp": { "servers": { + "atomicmemory": { "type": "remote", "url": "https://first.example" }, + "atomicmemory": { "type": "remote", "url": "https://second.example" }, + } }, +}"#, + ) + .unwrap(); + let existing = read_host_json_file(&path, Host::OpenCode).unwrap(); + let (merged, changed) = + remove_or_restore_json_mcp(&existing, Host::OpenCode, None).unwrap(); + + assert!(changed); + let rendered = render_json_host_config(&path, Host::OpenCode, &merged).unwrap(); + assert_eq!(rendered.matches("\"atomicmemory\"").count(), 0); + } + + #[test] + fn opencode_uninstall_preserves_comments_in_empty_mcp_containers() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("opencode.jsonc"); + fs::write( + &path, + r#"{ + "mcp": { + // Keep this MCP-level note. + "servers": { + // Keep this server-level note. + "atomicmemory": { "type": "local", "command": ["npx", "server"] }, + }, + }, +}"#, + ) + .unwrap(); + let existing = read_host_json_file(&path, Host::OpenCode).unwrap(); + let (merged, changed) = + remove_or_restore_json_mcp(&existing, Host::OpenCode, None).unwrap(); + + assert!(changed); + let rendered = render_json_host_config(&path, Host::OpenCode, &merged).unwrap(); + assert!(rendered.contains("// Keep this MCP-level note.")); + assert!(rendered.contains("// Keep this server-level note.")); + assert!(rendered.contains("\"servers\"")); + } + #[test] fn atomic_write_uses_unique_temp_and_rename() { let dir = tempfile::tempdir().unwrap(); @@ -503,8 +886,44 @@ mod tests { #[test] fn remove_without_restore_drops_empty_mcp_servers_object() { let existing = json!({ "mcpServers": { "atomicmemory": { "command": "npx" } } }); - let (merged, changed) = remove_or_restore_json_mcp(&existing, None).unwrap(); + let (merged, changed) = remove_or_restore_json_mcp(&existing, Host::Cursor, None).unwrap(); assert!(changed); assert!(merged.get("mcpServers").is_none()); } + + #[test] + fn opencode_remove_and_restore_preserve_unrelated_settings() { + let prior = json!({ "type": "remote", "url": "https://example.com" }); + let existing = json!({ + "model": "example/model", + "mcp": { + "timeout": { "startup": 45_000 }, + "servers": { + "other": { "type": "remote", "url": "https://other.example.com" }, + "atomicmemory": { "type": "local", "command": ["npx", "server"] } + } + } + }); + + let (restored, changed) = + remove_or_restore_json_mcp(&existing, Host::OpenCode, Some(&prior.to_string())) + .unwrap(); + assert!(changed); + assert_eq!(restored["mcp"]["servers"]["atomicmemory"], prior); + assert_eq!(restored["mcp"]["servers"]["other"]["type"], "remote"); + assert_eq!(restored["model"], "example/model"); + + let (removed, changed) = + remove_or_restore_json_mcp(&existing, Host::OpenCode, None).unwrap(); + assert!(changed); + assert!(removed["mcp"]["servers"].get("atomicmemory").is_none()); + assert_eq!(removed["mcp"]["timeout"]["startup"], 45_000); + } + + #[test] + fn opencode_remove_refuses_malformed_mcp_container() { + let existing = json!({ "mcp": [] }); + let err = remove_or_restore_json_mcp(&existing, Host::OpenCode, None).unwrap_err(); + assert!(err.to_string().contains("mcp must be a JSON object")); + } } diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs index 3d05ddf..2941849 100644 --- a/crates/cli/src/main.rs +++ b/crates/cli/src/main.rs @@ -1,5 +1,7 @@ //! AtomicMemory CLI — hosted tenancy + memory operations. +#![allow(clippy::disallowed_methods)] + mod agent_sanitize; mod argv_output; mod auth; @@ -14,9 +16,11 @@ mod integrate; mod onboarding_runtime; mod output; mod progress; +mod slm; mod telemetry; mod validation; mod verification; +mod version; use anyhow::Result; use clap::Parser; @@ -33,10 +37,34 @@ fn default_log_level(verbose: u8) -> &'static str { } } +/// True when argv requests top-level `--version` / `-V` (before `--`). +fn argv_requests_version(argv: &[String]) -> bool { + let mut seen_double_dash = false; + for arg in argv.iter().skip(1) { + if seen_double_dash { + continue; + } + if arg == "--" { + seen_double_dash = true; + continue; + } + if arg == "--version" || arg == "-V" || arg.starts_with("--version=") { + return true; + } + } + false +} + #[tokio::main] async fn main() { let started_at = std::time::Instant::now(); let argv: Vec = std::env::args().collect(); + // Print pure JSON (no `am ` prefix). Clap's built-in version flag always + // prefixes the bin name, which breaks the machine-readable contract. + if argv_requests_version(&argv) { + println!("{}", version::clap_version_json()); + return; + } let argv_agent = argv_output::detect_argv_agent(&argv); let cli = match Cli::try_parse() { @@ -89,7 +117,13 @@ async fn main() { std::process::exit(2); } - let result = run(cli).await; + let result = tokio::select! { + result = run(cli) => result, + signal = tokio::signal::ctrl_c() => match signal { + Ok(()) => Err(anyhow::anyhow!("command cancelled; rerun the command to resume using retained datasets and downloaded cache")), + Err(error) => Err(anyhow::anyhow!("could not listen for cancellation: {error}")), + }, + }; telemetry::flush_telemetry().await; if let Err(err) = result { if agent_output { @@ -123,6 +157,7 @@ async fn run(cli: Cli) -> Result<()> { Command::Link(cmd) => commands::link::run(cmd, &cli.global).await, Command::Connect(cmd) => commands::connect::run(cmd, &cli.global).await, Command::Instance(cmd) => commands::instance::run(cmd, &cli.global).await, + Command::Slm(cmd) => commands::slm::run(cmd, &cli.global).await, Command::Migrate(cmd) => commands::migrate::run(cmd, &cli.global).await, Command::Integrate(opts) => commands::integrate::run(opts, &cli.global).await, Command::Hooks(cmd) => commands::hooks::run(cmd, &cli.global).await, @@ -132,8 +167,6 @@ async fn run(cli: Cli) -> Result<()> { #[cfg(test)] mod tests { use super::*; - use clap::CommandFactory; - #[test] fn verbose_flag_raises_the_default_log_level() { assert_eq!(default_log_level(0), "warn"); @@ -143,18 +176,24 @@ mod tests { } #[test] - fn version_output_uses_am_identity() { - let mut cmd = cli::Cli::command(); - cmd.set_bin_name("am"); - let output = cmd.render_version().to_string(); - assert!( - output.starts_with("am "), - "expected version banner to start with 'am ', got: {output}" - ); - assert!( - output.contains(env!("CARGO_PKG_VERSION")), - "expected version banner to include workspace version" - ); + fn version_output_is_machine_readable_json_contract() { + assert!(argv_requests_version(&["am".into(), "--version".into()])); + assert!(argv_requests_version(&["am".into(), "-V".into()])); + assert!(!argv_requests_version(&[ + "am".into(), + "--".into(), + "--version".into() + ])); + let line = version::clap_version_json(); + let info: version::VersionInfo = serde_json::from_str(line) + .unwrap_or_else(|e| panic!("--version must be JSON: {e}; got: {line}")); + assert_eq!(info.surface, "cli"); + assert_eq!(info.version, env!("CARGO_PKG_VERSION")); + assert!(!info.env.is_empty()); + // Unstamped local builds must not invent a commit SHA. + if option_env!("AM_GIT_SHA").is_none() { + assert_eq!(info.git_sha, None, "unstamped gitSha must be null/None"); + } } #[test] diff --git a/crates/cli/src/onboarding_runtime.rs b/crates/cli/src/onboarding_runtime.rs index 54fb782..2dd2c87 100644 --- a/crates/cli/src/onboarding_runtime.rs +++ b/crates/cli/src/onboarding_runtime.rs @@ -2,7 +2,10 @@ use std::time::Duration; -use am_cloud_types::RuntimePresence; +use crate::instance::docker::{DockerRunner, RealDockerRunner}; +use crate::instance::{DEFAULT_CONTAINER_NAME, address::ManagedAddress, storage::owned_storage}; +use am_cloud_types::{RuntimePresence, RuntimeSummary}; +use anyhow::{Context, Result, bail}; use crate::cli::GlobalOptions; use crate::commands::client::dashboard_client; @@ -10,49 +13,281 @@ use crate::progress::ProgressReporter; const DEFAULT_WAIT: Duration = Duration::from_secs(30); const POLL_INTERVAL: Duration = Duration::from_secs(2); +const IDENTITY_INSPECT_TIMEOUT: Duration = Duration::from_secs(10); #[allow(dead_code)] pub async fn wait_runtime_online( global: &GlobalOptions, project_id: &str, + instance_id: &str, timeout: Duration, ) -> bool { - wait_runtime_online_with_progress(global, project_id, timeout, None).await + wait_runtime_online_with_progress(global, project_id, instance_id, timeout, None).await } pub async fn wait_runtime_online_with_progress( global: &GlobalOptions, project_id: &str, + instance_id: &str, timeout: Duration, mut progress: Option<&mut dyn ProgressReporter>, ) -> bool { + if instance_id.is_empty() { + return false; + } let started = tokio::time::Instant::now(); - let deadline = started + timeout; - while tokio::time::Instant::now() < deadline { - if runtime_online_now(global, project_id).await { - return true; - } - let elapsed = started.elapsed().as_secs(); - if let Some(p) = progress.as_deref_mut() { - p.tick("heartbeat", &format!("{elapsed}s/{}s", timeout.as_secs())); + tokio::time::timeout(timeout, async { + loop { + if runtime_online_now(global, project_id, instance_id).await { + return true; + } + if let Some(p) = progress.as_deref_mut() { + p.tick( + "heartbeat", + &format!("{}s/{}s", started.elapsed().as_secs(), timeout.as_secs()), + ); + } + tokio::time::sleep(POLL_INTERVAL).await; } - tokio::time::sleep(POLL_INTERVAL).await; - } - runtime_online_now(global, project_id).await + }) + .await + .unwrap_or(false) } -pub async fn runtime_online_now(global: &GlobalOptions, project_id: &str) -> bool { +pub async fn runtime_online_now( + global: &GlobalOptions, + project_id: &str, + instance_id: &str, +) -> bool { let Ok((_profile, dash)) = dashboard_client(global).await else { return false; }; let Ok(runtimes) = dash.list_runtimes(project_id).await else { return false; }; - runtimes - .iter() - .any(|r| r.presence == RuntimePresence::Online) + matching_runtime_online(&runtimes, project_id, instance_id) +} + +fn matching_runtime_online( + runtimes: &[RuntimeSummary], + project_id: &str, + instance_id: &str, +) -> bool { + !instance_id.is_empty() + && runtimes.iter().any(|runtime| { + runtime.project_id == project_id + && runtime.core_instance_id == instance_id + && runtime.presence == RuntimePresence::Online + && runtime.revoked_at.is_none() + }) +} + +/// Resolve the exact local Core identity only after checking managed ownership and binding. +pub async fn managed_core_instance_id(global: &GlobalOptions) -> Result { + let profile = crate::config::resolve_profile( + global.profile.as_deref(), + global.base_url.as_deref(), + global.environment, + )?; + // Keep CLI lifecycle commands from replacing the container between inspection and read. + let _registry = crate::instance::storage::RuntimeStore::open()?; + managed_core_instance_id_with(&RealDockerRunner::new(), &profile).await +} + +async fn managed_core_instance_id_with( + docker: &dyn DockerRunner, + profile: &crate::config::ResolvedProfile, +) -> Result { + let expected = ManagedAddress::parse(&profile.memory_base_url)?; + let inspected = tokio::time::timeout( + IDENTITY_INSPECT_TIMEOUT, + docker.inspect(DEFAULT_CONTAINER_NAME), + ) + .await + .context("inspect Core runtime identity timed out")?? + .context("managed Core is missing; cloud connection cannot be verified")?; + let bound = inspected + .local_url + .as_deref() + .and_then(|url| ManagedAddress::parse(url).ok()); + if !inspected.state.is_running() + || owned_storage(profile, Some(&inspected)).is_none() + || bound != Some(expected) + { + bail!( + "Core runtime ownership or local binding could not be verified; refusing to read its identity" + ); + } + docker + .read_core_instance_id(DEFAULT_CONTAINER_NAME) + .await? + .filter(|id| !id.is_empty()) + .context("persisted Core runtime identity is missing; cloud connection cannot be verified") } pub fn default_runtime_wait() -> Duration { DEFAULT_WAIT } + +#[cfg(test)] +mod tests { + use super::*; + + fn runtime() -> RuntimeSummary { + RuntimeSummary { + id: "registry-row".into(), + project_id: "project-one".into(), + core_instance_id: "this-machine".into(), + name: None, + runtime_type: "core".into(), + presence: RuntimePresence::Online, + capabilities: Vec::new(), + core_version: None, + connector_version: None, + last_heartbeat_at: Some(chrono::Utc::now()), + revoked_at: None, + created_at: chrono::Utc::now(), + updated_at: chrono::Utc::now(), + } + } + + #[test] + fn another_online_machine_cannot_activate_this_installation() { + assert!(!matching_runtime_online( + &[runtime()], + "project-one", + "different-machine" + )); + } + + #[test] + fn matching_online_instance_requires_project_and_unrevoked_state() { + let expected = runtime(); + assert!(matching_runtime_online( + std::slice::from_ref(&expected), + "project-one", + "this-machine" + )); + assert!(!matching_runtime_online( + std::slice::from_ref(&expected), + "different-project", + "this-machine" + )); + let mut revoked = expected.clone(); + revoked.revoked_at = Some(chrono::Utc::now()); + assert!(!matching_runtime_online( + &[revoked], + "project-one", + "this-machine" + )); + let mut offline = expected; + offline.presence = RuntimePresence::Offline; + assert!(!matching_runtime_online( + &[offline], + "project-one", + "this-machine" + )); + } +} + +#[cfg(all(test, unix))] +mod ownership_tests { + use super::*; + use serde_json::{Value, json}; + use std::os::unix::fs::PermissionsExt; + + fn profile() -> crate::config::ResolvedProfile { + crate::config::ResolvedProfile { + name: "local-test".into(), + base_url: "https://api.example.test".into(), + kind: crate::config::ProfileKind::Local, + project_id: Some("project-one".into()), + memory_base_url: "http://127.0.0.1:17352".into(), + api_key: None, + oauth: None, + } + } + + fn inspect() -> Value { + json!([{ + "State":{"Status":"running"}, + "Config":{"Image":"example/core:test","Labels":{ + "ai.atomicstrata.managed-by":"am-cli", "ai.atomicstrata.profile":"local-test", + "ai.atomicstrata.project-id":"project-one" + },"Env":["OPENAI_API_KEY=fixture-key","EMBEDDING_DIMENSIONS=1536","ATOMICMEMORY_API_URL=https://api.example.test"]}, + "Mounts":[ + {"Type":"volume","Name":"data-test","Destination":"/var/lib/atomicmemory/postgres"}, + {"Type":"volume","Name":"state-test","Destination":"/var/lib/atomicmemory/state"} + ], + "NetworkSettings":{"Ports":{"17350/tcp":[{"HostIp":"127.0.0.1","HostPort":"17352"}]}} + }]) + } + + fn fake_docker(inspected: &Value, id: &str) -> (tempfile::TempDir, RealDockerRunner) { + let dir = tempfile::tempdir().unwrap(); + std::fs::write(dir.path().join("inspect.json"), inspected.to_string()).unwrap(); + std::fs::write(dir.path().join("identity"), id).unwrap(); + let executable = dir.path().join("docker"); + let script = r#"#!/bin/sh +script_dir=$(dirname "$0") +case "$1" in + inspect) cat "$script_dir/inspect.json";; + exec) + test "$2" = atomic-memory && test "$3" = cat && test "$4" = /var/lib/atomicmemory/state/core-instance-id || exit 9 + touch "$script_dir/read-called" + cat "$script_dir/identity";; + *) exit 8;; +esac +"#; + std::fs::write(&executable, script).unwrap(); + std::fs::set_permissions(&executable, std::fs::Permissions::from_mode(0o700)).unwrap(); + let docker = RealDockerRunner { + docker_bin: executable.to_string_lossy().into_owned(), + }; + (dir, docker) + } + + #[tokio::test] + async fn reads_only_the_owned_running_instances_persisted_identity() { + let (dir, docker) = fake_docker(&inspect(), "this-machine\n"); + assert_eq!( + managed_core_instance_id_with(&docker, &profile()) + .await + .unwrap(), + "this-machine" + ); + assert!(dir.path().join("read-called").exists()); + } + + #[tokio::test] + async fn identity_is_never_read_from_wrong_owner_project_or_binding() { + let mut wrong_profile = inspect(); + wrong_profile[0]["Config"]["Labels"]["ai.atomicstrata.profile"] = json!("another-profile"); + let mut wrong_project = inspect(); + wrong_project[0]["Config"]["Labels"]["ai.atomicstrata.project-id"] = + json!("another-project"); + let mut wrong_binding = inspect(); + wrong_binding[0]["NetworkSettings"]["Ports"]["17350/tcp"][0]["HostPort"] = json!("18888"); + for inspected in [wrong_profile, wrong_project, wrong_binding] { + let (dir, docker) = fake_docker(&inspected, "other-machine"); + assert!( + managed_core_instance_id_with(&docker, &profile()) + .await + .is_err() + ); + assert!(!dir.path().join("read-called").exists()); + } + } + + #[tokio::test] + async fn missing_or_oversized_identity_cannot_verify_cloud_connection() { + for id in [String::new(), "x".repeat(4096)] { + let (_dir, docker) = fake_docker(&inspect(), &id); + assert!( + managed_core_instance_id_with(&docker, &profile()) + .await + .is_err() + ); + } + } +} diff --git a/crates/cli/src/progress.rs b/crates/cli/src/progress.rs index 943172e..390112c 100644 --- a/crates/cli/src/progress.rs +++ b/crates/cli/src/progress.rs @@ -78,6 +78,7 @@ struct PlainSteps { step_n: usize, active: HashMap, /// Captured lines for tests (also mirrored to stderr when not capturing-only). + #[cfg(test)] lines: Vec, write_stderr: bool, } @@ -87,6 +88,7 @@ impl PlainSteps { Self { step_n: 0, active: HashMap::new(), + #[cfg(test)] lines: Vec::new(), write_stderr: true, } @@ -104,6 +106,7 @@ impl PlainSteps { if self.write_stderr { let _ = writeln!(io::stderr(), "{line}"); } + #[cfg(test)] self.lines.push(line); } } diff --git a/crates/cli/src/slm/bootstrap.rs b/crates/cli/src/slm/bootstrap.rs new file mode 100644 index 0000000..f00493f --- /dev/null +++ b/crates/cli/src/slm/bootstrap.rs @@ -0,0 +1,364 @@ +//! Cold-start bootstrap: install → confirmed pull → serve (ATO-1936 follow-up). + +use std::time::Duration; + +use anyhow::{Context, Result, bail}; + +use super::cache::slm_hf_home; +use super::confirm_models_pull; +use super::install::install_or_update; +use super::manifest_url_from_env; +use super::models::{pull_models, status_models_json}; +use super::models_cache_ready_with_disk; +use super::paths::SlmPaths; +use super::process::{StartOutcome, preflight_managed_start, start_runtime}; + +/// Confirmation policy for the ~1.7GB model download. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct BootstrapConfirm { + pub yes: bool, + pub allow_prompt: bool, +} + +/// Whether the managed `am-slm` binary is missing and must be installed. +pub fn runtime_missing(paths: &SlmPaths) -> bool { + !paths.binary().is_file() +} + +/// Disk cache is start-complete (Qwen + Nomic + `am-slm-core`). +pub async fn disk_cache_ready(paths: &SlmPaths) -> Result { + if runtime_missing(paths) { + return Ok(false); + } + Ok(models_cache_ready_with_disk( + &status_models_json(paths).await?, + &slm_hf_home(paths), + )) +} + +/// Install if needed, pull if cache is incomplete, then start the runtime. +/// +/// Reuses a healthy managed listener and rejects a busy port before any +/// download confirmation or model pull. +pub async fn bootstrap_managed_slm( + client: &reqwest::Client, + paths: &SlmPaths, + confirm: BootstrapConfirm, + port: u16, + wait: Duration, + mut report: impl FnMut(&str), +) -> Result { + ensure_runtime_installed(client, paths, &mut report).await?; + if let Some(outcome) = preflight_managed_start(client, paths, port, wait).await? { + return Ok(outcome); + } + ensure_models_for_start(paths, confirm, &mut report).await?; + report("starting host Metal SLM"); + start_runtime(client, paths, port, wait).await +} + +async fn ensure_runtime_installed( + client: &reqwest::Client, + paths: &SlmPaths, + report: &mut impl FnMut(&str), +) -> Result<()> { + if !runtime_missing(paths) { + return Ok(()); + } + report("installing am-slm"); + install_or_update(client, paths, &manifest_url_from_env()) + .await + .context("install am-slm before start")?; + Ok(()) +} + +async fn ensure_models_for_start( + paths: &SlmPaths, + confirm: BootstrapConfirm, + report: &mut impl FnMut(&str), +) -> Result<()> { + if disk_cache_ready(paths).await? { + return Ok(()); + } + if !confirm_models_pull(confirm.yes, confirm.allow_prompt)? { + bail!("model download cancelled"); + } + report("downloading models (~1.7GB)"); + pull_models(paths, None, report).await?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs; + use tempfile::tempdir; + + #[test] + fn missing_binary_needs_install() { + let dir = tempdir().unwrap(); + let paths = SlmPaths::at(dir.path().to_path_buf()); + paths.ensure().unwrap(); + assert!(runtime_missing(&paths)); + fs::write(paths.binary(), b"x").unwrap(); + assert!(!runtime_missing(&paths)); + } + + #[test] + fn confirm_yes_skips_prompt_for_bootstrap() { + assert!(confirm_models_pull(true, false).unwrap()); + let err = confirm_models_pull(false, false).unwrap_err(); + assert!(err.to_string().contains("--yes")); + } + + #[tokio::test] + #[cfg(unix)] + async fn ready_cache_skips_pull() { + let (paths, log, _dir) = fake_runtime(COMPLETE); + let mut steps = Vec::new(); + ensure_models_for_start( + &paths, + BootstrapConfirm { + yes: false, + allow_prompt: false, + }, + &mut |step| steps.push(step.to_string()), + ) + .await + .unwrap(); + assert!(steps.is_empty()); + assert!(!log.exists() || fs::read_to_string(&log).unwrap().is_empty()); + } + + #[tokio::test] + #[cfg(unix)] + async fn incomplete_cache_without_yes_refuses() { + let (paths, log, _dir) = fake_runtime(INCOMPLETE); + let err = ensure_models_for_start( + &paths, + BootstrapConfirm { + yes: false, + allow_prompt: false, + }, + &mut |_| {}, + ) + .await + .unwrap_err(); + assert!(err.to_string().contains("--yes")); + assert!(!log.exists()); + } + + #[tokio::test] + #[cfg(unix)] + async fn yes_pulls_when_cache_incomplete() { + let (paths, log, _dir) = fake_runtime(INCOMPLETE_THEN_READY); + let mut steps = Vec::new(); + ensure_models_for_start( + &paths, + BootstrapConfirm { + yes: true, + allow_prompt: false, + }, + &mut |step| steps.push(step.to_string()), + ) + .await + .unwrap(); + assert_eq!(steps, ["downloading models (~1.7GB)"]); + let recorded = fs::read_to_string(&log).unwrap(); + assert!(recorded.contains("models pull")); + assert!(recorded.contains("--adapter am-slm-core")); + } + + #[cfg(unix)] + const COMPLETE: &str = include_str!("../../tests/fixtures/am-slm-models-status.json"); + #[cfg(unix)] + const INCOMPLETE: &str = + include_str!("../../tests/fixtures/am-slm-models-status-incomplete.json"); + #[cfg(unix)] + const INCOMPLETE_THEN_READY: &str = "__toggle__"; + + #[cfg(unix)] + fn fake_runtime(mode: &str) -> (SlmPaths, std::path::PathBuf, tempfile::TempDir) { + use std::os::unix::fs::PermissionsExt; + + let dir = tempfile::tempdir().unwrap(); + let paths = SlmPaths::at(dir.path().to_path_buf()); + paths.ensure().unwrap(); + let log = dir.path().join("pull.log"); + let script = fake_runtime_script(mode, &log, dir.path().join("status-state")); + fs::write(paths.binary(), script).unwrap(); + fs::set_permissions(paths.binary(), fs::Permissions::from_mode(0o755)).unwrap(); + (paths, log, dir) + } + + #[tokio::test] + #[cfg(unix)] + async fn reuses_healthy_listener_without_download_confirm() { + use crate::slm::paths::SlmStateFile; + use crate::slm::process_identity::capture_process_identity; + use std::os::unix::fs::PermissionsExt; + + let (mut http, port) = spawn_ready_http(); + let pid = http.id(); + let identity = capture_process_identity(pid).expect("http identity"); + let (paths, log, _dir) = fake_runtime(INCOMPLETE); + paths + .write_state(&SlmStateFile { + managed_process: true, + json_schema_contract: true, + pid: Some(pid), + port: Some(port), + process_start_stamp: Some(identity.start_stamp), + ..SlmStateFile::default() + }) + .unwrap(); + fs::write(paths.pid_path(), format!("{pid}\n")).unwrap(); + fs::set_permissions(paths.binary(), fs::Permissions::from_mode(0o755)).unwrap(); + + let client = reqwest::Client::builder() + .timeout(Duration::from_secs(2)) + .build() + .unwrap(); + let outcome = bootstrap_managed_slm( + &client, + &paths, + BootstrapConfirm { + yes: false, + allow_prompt: false, + }, + port, + Duration::from_secs(2), + |_| {}, + ) + .await + .expect("reuse healthy listener"); + assert!(outcome.already_running); + assert_eq!(outcome.pid, pid); + assert!(!log.exists() || fs::read_to_string(&log).unwrap().is_empty()); + + http.kill().ok(); + let _ = http.wait(); + } + + #[tokio::test] + #[cfg(unix)] + async fn rejects_busy_port_before_model_pull() { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let port = listener.local_addr().unwrap().port(); + let (paths, log, _dir) = fake_runtime(INCOMPLETE); + let client = reqwest::Client::builder() + .timeout(Duration::from_secs(2)) + .build() + .unwrap(); + let err = bootstrap_managed_slm( + &client, + &paths, + BootstrapConfirm { + yes: true, + allow_prompt: false, + }, + port, + Duration::from_secs(1), + |_| {}, + ) + .await + .expect_err("busy port must fail before pull"); + assert!( + err.to_string().contains("already in use"), + "expected port conflict, got: {err}" + ); + assert!(!log.exists() || fs::read_to_string(&log).unwrap().is_empty()); + drop(listener); + } + + #[cfg(unix)] + fn spawn_ready_http() -> (std::process::Child, u16) { + use std::io::{BufRead, BufReader}; + use std::process::{Command, Stdio}; + + let dir = tempfile::tempdir().unwrap(); + let script_path = dir.path().join("ready.py"); + let script = r#" +from http.server import BaseHTTPRequestHandler, HTTPServer +import json + +class H(BaseHTTPRequestHandler): + def do_GET(self): + if self.path.endswith('/health'): + self.send_response(200) + self.end_headers() + self.wfile.write(b'ok') + return + if self.path.endswith('/v1/models'): + body = json.dumps({"data":[{"id":"__CHAT__"},{"id":"__EMBED__"}]}).encode() + self.send_response(200) + self.send_header('Content-Type', 'application/json') + self.end_headers() + self.wfile.write(body) + return + self.send_response(404) + self.end_headers() + def log_message(self, *args): + pass + +server = HTTPServer(('127.0.0.1', 0), H) +print(server.server_address[1], flush=True) +server.serve_forever() +"# + .replace("__CHAT__", crate::slm::SLM_CHAT_MODEL) + .replace("__EMBED__", crate::slm::SLM_EMBED_MODEL); + fs::write(&script_path, script).unwrap(); + std::mem::forget(dir); + let mut child = Command::new("python3") + .arg(&script_path) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .spawn() + .expect("spawn python http"); + let stdout = child.stdout.take().expect("http stdout"); + let mut line = String::new(); + BufReader::new(stdout) + .read_line(&mut line) + .expect("read http port"); + let port: u16 = line.trim().parse().expect("http port"); + (child, port) + } + + #[cfg(unix)] + fn fake_runtime_script(mode: &str, log: &std::path::Path, state: std::path::PathBuf) -> String { + let complete = COMPLETE.replace('\'', "'\\''"); + let incomplete = INCOMPLETE.replace('\'', "'\\''"); + if mode == INCOMPLETE_THEN_READY { + format!( + "#!/bin/sh\n\ + if [ \"$1\" = models ] && [ \"$2\" = pull ]; then\n\ + echo \"$*\" >> '{log}'\n\ + echo pulled > '{state}'\n\ + exit 0\n\ + fi\n\ + if [ \"$1\" = models ] && [ \"$2\" = status ]; then\n\ + if [ -f '{state}' ]; then printf '%s\\n' '{complete}'; else printf '%s\\n' '{incomplete}'; fi\n\ + exit 0\n\ + fi\n\ + exit 1\n", + log = log.display(), + state = state.display(), + ) + } else { + format!( + "#!/bin/sh\n\ + if [ \"$1\" = models ] && [ \"$2\" = pull ]; then\n\ + echo \"$*\" >> '{log}'\n\ + exit 0\n\ + fi\n\ + if [ \"$1\" = models ] && [ \"$2\" = status ]; then\n\ + printf '%s\\n' '{status}'\n\ + exit 0\n\ + fi\n\ + exit 1\n", + log = log.display(), + status = mode.replace('\'', "'\\''"), + ) + } + } +} diff --git a/crates/cli/src/slm/cache.rs b/crates/cli/src/slm/cache.rs new file mode 100644 index 0000000..77c8bc4 --- /dev/null +++ b/crates/cli/src/slm/cache.rs @@ -0,0 +1,180 @@ +//! CLI-owned model cache directories for published am-slm (HF_HOME + AM_SLM_CACHE). + +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; + +use anyhow::{Context, Result}; + +use super::paths::SlmPaths; +use super::{SLM_CHAT_MODEL, SLM_EMBED_MODEL}; + +/// Hugging Face / catalog IDs emitted by published am-slm 0.1.1 `models status --json`. +pub(crate) const QWEN_BASE_IDS: &[&str] = &["Qwen/Qwen3-0.6B", "qwen3-0.6b"]; +pub(crate) const NOMIC_BASE_IDS: &[&str] = &[ + "nomic-ai/nomic-embed-text-v1.5", + "nomic-embed-text-v1.5", + SLM_EMBED_MODEL, +]; +const CORE_ADAPTER_IDS: &[&str] = &[SLM_CHAT_MODEL]; + +const GAP_QWEN: &str = "Qwen/Qwen3-0.6B"; +const GAP_NOMIC: &str = "nomic-ai/nomic-embed-text-v1.5"; +const GAP_ADAPTER: &str = "am-slm-core adapter"; + +const HF_HOME_DIR: &str = "hf-home"; +const ADAPTER_CACHE_DIR: &str = "am-slm-cache"; + +/// Per-artifact disk cache flags from `am-slm models status --json`. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct ModelsCacheFlags { + pub qwen: bool, + pub nomic: bool, + pub am_slm_core: bool, +} + +impl ModelsCacheFlags { + pub fn ready(self) -> bool { + self.qwen && self.nomic && self.am_slm_core + } + + pub fn gaps(self) -> Vec<&'static str> { + let mut gaps = Vec::new(); + if !self.qwen { + gaps.push(GAP_QWEN); + } + if !self.nomic { + gaps.push(GAP_NOMIC); + } + if !self.am_slm_core { + gaps.push(GAP_ADAPTER); + } + gaps + } +} + +/// Hugging Face weights cache root owned by the CLI (published runtime reads HF_HOME). +pub fn slm_hf_home(paths: &SlmPaths) -> PathBuf { + paths.root().join(HF_HOME_DIR) +} + +/// Merged adapter cache root (published runtime reads AM_SLM_CACHE). +pub fn slm_adapter_cache(paths: &SlmPaths) -> PathBuf { + paths.root().join(ADAPTER_CACHE_DIR) +} + +/// Directories deleted by `uninstall --purge-models`. +pub fn managed_model_cache_dirs(paths: &SlmPaths) -> [PathBuf; 2] { + [slm_hf_home(paths), slm_adapter_cache(paths)] +} + +/// Apply the cache contract understood by published am-slm 0.1.1+. +pub fn apply_slm_cache_env(cmd: &mut Command, paths: &SlmPaths) { + cmd.env("HF_HOME", slm_hf_home(paths)); + cmd.env("AM_SLM_CACHE", slm_adapter_cache(paths)); + cmd.env_remove("AM_SLM_MODELS_DIR"); +} + +pub fn ensure_cache_roots(paths: &SlmPaths) -> Result<()> { + fs::create_dir_all(slm_hf_home(paths)).context("create HF_HOME cache directory")?; + fs::create_dir_all(slm_adapter_cache(paths)).context("create AM_SLM_CACHE directory")?; + Ok(()) +} + +pub fn purge_managed_model_cache(paths: &SlmPaths) -> Result<()> { + for dir in managed_model_cache_dirs(paths) { + if dir.exists() { + fs::remove_dir_all(&dir) + .with_context(|| format!("remove managed model cache {}", dir.display()))?; + } + } + Ok(()) +} + +/// Parse published `am-slm models status --json` (`models`/`adapters` arrays). +/// +/// Required: Qwen + Nomic base weights and the `am-slm-core` adapter, each +/// with `ready: true`. Top-level `ready` is ignored — the 0.1.1 runtime does +/// not emit it, and a lone flag must not skip the ID checks. +pub fn models_cache_flags(status: &serde_json::Value) -> ModelsCacheFlags { + let models = status_entries(status, "models"); + let adapters = status_entries(status, "adapters"); + ModelsCacheFlags { + qwen: entry_ready(models, QWEN_BASE_IDS), + nomic: entry_ready(models, NOMIC_BASE_IDS), + am_slm_core: entry_ready(adapters, CORE_ADAPTER_IDS), + } +} + +fn status_entries<'a>(status: &'a serde_json::Value, key: &str) -> &'a [serde_json::Value] { + status + .get(key) + .and_then(|value| value.as_array()) + .map(Vec::as_slice) + .unwrap_or(&[]) +} + +fn entry_ready(entries: &[serde_json::Value], ids: &[&str]) -> bool { + entries.iter().any(|entry| { + let id = entry + .get("id") + .and_then(|value| value.as_str()) + .unwrap_or(""); + ids.contains(&id) && entry.get("ready").and_then(|value| value.as_bool()) == Some(true) + }) +} + +/// Contract test helper: published runtime reports HF_HOME as cache_dir. +#[cfg(test)] +fn cache_dir_matches_hf_home(status: &serde_json::Value, hf_home: &Path) -> bool { + status + .get("cache_dir") + .and_then(|v| v.as_str()) + .is_some_and(|dir| Path::new(dir) == hf_home) +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::tempdir; + + #[test] + fn empty_status_is_not_ready() { + assert!(!models_cache_flags(&serde_json::json!({})).ready()); + } + + #[test] + fn top_level_ready_flag_does_not_skip_id_checks() { + assert!(!models_cache_flags(&serde_json::json!({"ready": true})).ready()); + } + + #[test] + fn published_array_schema_requires_qwen_nomic_and_core() { + let complete = include_str!("../../tests/fixtures/am-slm-models-status.json"); + let incomplete = include_str!("../../tests/fixtures/am-slm-models-status-incomplete.json"); + assert!(models_cache_flags(&serde_json::from_str(complete).unwrap()).ready()); + assert!(!models_cache_flags(&serde_json::from_str(incomplete).unwrap()).ready()); + } + + #[test] + fn gaps_name_qwen_nomic_and_adapter_independently() { + let incomplete = include_str!("../../tests/fixtures/am-slm-models-status-incomplete.json"); + let gaps = models_cache_flags(&serde_json::from_str(incomplete).unwrap()).gaps(); + assert_eq!(gaps, ["Qwen/Qwen3-0.6B", "am-slm-core adapter"]); + let complete = include_str!("../../tests/fixtures/am-slm-models-status.json"); + assert!( + models_cache_flags(&serde_json::from_str(complete).unwrap()) + .gaps() + .is_empty() + ); + } + + #[test] + fn cache_dir_follows_hf_home_fixture() { + let dir = tempdir().unwrap(); + let paths = SlmPaths::at(dir.path().to_path_buf()); + let hf_home = slm_hf_home(&paths); + let status = serde_json::json!({"cache_dir": hf_home.display().to_string()}); + assert!(cache_dir_matches_hf_home(&status, &hf_home)); + } +} diff --git a/crates/cli/src/slm/cache_probe.rs b/crates/cli/src/slm/cache_probe.rs new file mode 100644 index 0000000..f7da8b4 --- /dev/null +++ b/crates/cli/src/slm/cache_probe.rs @@ -0,0 +1,228 @@ +//! On-disk HF snapshot probe when `am-slm models status` lies about base weights (ATO-1944). +//! +//! Accepts `snapshots//` without a `refs` entry. A different +//! snapshot is not a substitute when the runtime pins a revision. + +use std::fs; +use std::path::{Path, PathBuf}; + +use super::cache::{ModelsCacheFlags, NOMIC_BASE_IDS, QWEN_BASE_IDS, models_cache_flags}; + +const DEFAULT_BASE_FILES: &[&str] = &["config.json", "tokenizer.json", "model.safetensors"]; + +/// JSON flags plus a disk override for **base** models only. Adapter stays JSON. +pub fn models_cache_flags_with_disk( + status: &serde_json::Value, + hf_home: &Path, +) -> ModelsCacheFlags { + let mut flags = models_cache_flags(status); + if !flags.qwen { + flags.qwen = base_snapshot_present(status, hf_home, QWEN_BASE_IDS); + } + if !flags.nomic { + flags.nomic = base_snapshot_present(status, hf_home, NOMIC_BASE_IDS); + } + flags +} + +pub fn models_cache_ready_with_disk(status: &serde_json::Value, hf_home: &Path) -> bool { + models_cache_flags_with_disk(status, hf_home).ready() +} + +fn hub_root(status: &serde_json::Value, hf_home: &Path) -> PathBuf { + status + .get("cache_dir") + .and_then(|value| value.as_str()) + .map(PathBuf::from) + .unwrap_or_else(|| hf_home.join("hub")) +} + +fn base_snapshot_present(status: &serde_json::Value, hf_home: &Path, ids: &[&str]) -> bool { + let Some(entry) = find_entry(status, "models", ids) else { + return false; + }; + let files = required_files(entry); + let revision = entry + .get("revision") + .and_then(|value| value.as_str()) + .filter(|value| !value.is_empty()); + let repo_id = entry + .get("id") + .and_then(|value| value.as_str()) + .unwrap_or(""); + snapshot_has_files(&hub_root(status, hf_home), repo_id, revision, &files) +} + +fn find_entry<'a>( + status: &'a serde_json::Value, + key: &str, + ids: &[&str], +) -> Option<&'a serde_json::Value> { + status.get(key)?.as_array()?.iter().find(|entry| { + let id = entry + .get("id") + .and_then(|value| value.as_str()) + .unwrap_or(""); + ids.contains(&id) + }) +} + +fn required_files(entry: &serde_json::Value) -> Vec { + let missing = entry + .get("missing") + .and_then(|value| value.as_array()) + .map(|items| { + items + .iter() + .filter_map(|item| item.as_str()) + .filter(|name| !name.is_empty() && *name != "weights") + .map(str::to_string) + .collect::>() + }) + .unwrap_or_default(); + if missing.is_empty() { + return DEFAULT_BASE_FILES + .iter() + .map(|name| (*name).to_string()) + .collect(); + } + missing +} + +fn snapshot_has_files(hub: &Path, repo_id: &str, revision: Option<&str>, files: &[String]) -> bool { + if repo_id.is_empty() || files.is_empty() { + return false; + } + let snapshots = hub.join(hf_hub_dirname(repo_id)).join("snapshots"); + match revision { + Some(revision) => files_present(&snapshots.join(revision), files), + None => any_snapshot_has_files(&snapshots, files), + } +} + +fn hf_hub_dirname(repo_id: &str) -> String { + format!("models--{}", repo_id.replace('/', "--")) +} + +fn any_snapshot_has_files(snapshots: &Path, files: &[String]) -> bool { + let Ok(entries) = fs::read_dir(snapshots) else { + return false; + }; + entries.flatten().any(|entry| { + let path = entry.path(); + path.is_dir() && files_present(&path, files) + }) +} + +fn files_present(dir: &Path, files: &[String]) -> bool { + files.iter().all(|name| dir.join(name).is_file()) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs; + use tempfile::tempdir; + + const QWEN: &str = "Qwen/Qwen3-0.6B"; + const NOMIC: &str = "nomic-ai/nomic-embed-text-v1.5"; + const REV: &str = "c1899de289a04d12100db370d81485cdf75e47ca"; + + fn ethan_status(cache_dir: &Path) -> serde_json::Value { + serde_json::json!({ + "cache_dir": cache_dir.display().to_string(), + "models": [ + { + "id": QWEN, + "revision": REV, + "ready": false, + "missing": ["config.json", "tokenizer.json", "model.safetensors"] + }, + { + "id": NOMIC, + "revision": "e9b6763023c676ca8431644204f50c2b100d9aab", + "ready": false, + "missing": ["config.json", "tokenizer.json", "model.safetensors"] + } + ], + "adapters": [{ "id": "am-slm-core", "ready": true, "missing": [] }] + }) + } + + fn write_base(hub: &Path, repo: &str, revision: &str) { + let dir = hub + .join(hf_hub_dirname(repo)) + .join("snapshots") + .join(revision); + fs::create_dir_all(&dir).unwrap(); + for name in DEFAULT_BASE_FILES { + fs::write(dir.join(name), b"x").unwrap(); + } + } + + #[test] + fn empty_hub_does_not_override_json() { + let dir = tempdir().unwrap(); + let hub = dir.path().join("hub"); + fs::create_dir_all(&hub).unwrap(); + let flags = models_cache_flags_with_disk(ðan_status(&hub), dir.path()); + assert!(!flags.qwen && !flags.nomic); + assert!(flags.am_slm_core); + assert!(!flags.ready()); + } + + #[test] + fn snapshots_at_named_revision_make_bases_ready() { + let dir = tempdir().unwrap(); + let hub = dir.path().join("hub"); + write_base(&hub, QWEN, REV); + write_base(&hub, NOMIC, "e9b6763023c676ca8431644204f50c2b100d9aab"); + let flags = models_cache_flags_with_disk(ðan_status(&hub), dir.path()); + assert!(flags.qwen && flags.nomic && flags.am_slm_core); + assert!(flags.ready()); + } + + #[test] + fn mismatched_revision_is_not_ready() { + let dir = tempdir().unwrap(); + let hub = dir.path().join("hub"); + write_base(&hub, QWEN, "other-pin"); + write_base(&hub, NOMIC, "other-pin"); + let flags = models_cache_flags_with_disk(ðan_status(&hub), dir.path()); + assert!(!flags.qwen && !flags.nomic); + assert!(!flags.ready()); + } + + #[test] + fn unpinned_status_accepts_any_complete_snapshot() { + let dir = tempdir().unwrap(); + let hub = dir.path().join("hub"); + write_base(&hub, QWEN, "unpinned-snap"); + write_base(&hub, NOMIC, "unpinned-snap"); + let mut status = ethan_status(&hub); + status["models"][0] + .as_object_mut() + .unwrap() + .remove("revision"); + status["models"][1] + .as_object_mut() + .unwrap() + .remove("revision"); + let flags = models_cache_flags_with_disk(&status, dir.path()); + assert!(flags.ready()); + } + + #[test] + fn adapter_false_is_not_flipped_by_base_snapshots() { + let dir = tempdir().unwrap(); + let hub = dir.path().join("hub"); + write_base(&hub, QWEN, REV); + write_base(&hub, NOMIC, "e9b6763023c676ca8431644204f50c2b100d9aab"); + let mut status = ethan_status(&hub); + status["adapters"][0]["ready"] = serde_json::json!(false); + let flags = models_cache_flags_with_disk(&status, dir.path()); + assert!(flags.qwen && flags.nomic); + assert!(!flags.am_slm_core); + assert!(!flags.ready()); + } +} diff --git a/crates/cli/src/slm/core_env.rs b/crates/cli/src/slm/core_env.rs new file mode 100644 index 0000000..c4b5643 --- /dev/null +++ b/crates/cli/src/slm/core_env.rs @@ -0,0 +1,108 @@ +//! Provider environment overlay for Connected Local SLM; dataset ownership lives in instance/storage. + +use super::{SLM_CHAT_MODEL, SLM_EMBED_DIMENSIONS, SLM_EMBED_MODEL, SLM_HOST_DOCKER_BASE}; +use crate::instance::docker::InstanceEnv; + +/// Extra `docker run` argv so Core can reach the host Metal runtime. +pub const SLM_ADD_HOST_ARGS: &[&str] = &["--add-host", "host.docker.internal:host-gateway"]; + +/// Apply the Connected Local SLM provider overlay onto a Core `InstanceEnv`. +/// +/// Points openai-compatible LLM + embedding endpoints at the host `am-slm` +/// runtime and clears any requirement for `OPENAI_API_KEY`. +pub fn apply_slm_overlay(env: &mut InstanceEnv) { + env.slm = true; + env.openai_api_key.clear(); +} + +/// Env var names added when `InstanceEnv.slm` is true. +pub fn slm_docker_env_names() -> Vec<&'static str> { + vec![ + "LLM_PROVIDER", + "LLM_API_URL", + "LLM_API_KEY", + "LLM_MODEL", + "EMBEDDING_PROVIDER", + "EMBEDDING_API_URL", + "EMBEDDING_API_KEY", + "EMBEDDING_MODEL", + "EMBEDDING_DIMENSIONS", + // TODO(ATO-1936): EXTRACTION_PROMPT_VARIANT when Core image ships compact. + ] +} + +/// Child-process env map entries for the SLM overlay. +pub fn slm_child_env_entries() -> Vec<(String, String)> { + vec![ + ("LLM_PROVIDER".into(), "openai-compatible".into()), + ("LLM_API_URL".into(), SLM_HOST_DOCKER_BASE.into()), + ("LLM_API_KEY".into(), "local".into()), + ("LLM_MODEL".into(), SLM_CHAT_MODEL.into()), + ("EMBEDDING_PROVIDER".into(), "openai-compatible".into()), + ("EMBEDDING_API_URL".into(), SLM_HOST_DOCKER_BASE.into()), + ("EMBEDDING_API_KEY".into(), "local".into()), + ("EMBEDDING_MODEL".into(), SLM_EMBED_MODEL.into()), + ( + "EMBEDDING_DIMENSIONS".into(), + SLM_EMBED_DIMENSIONS.to_string(), + ), + // Do not set EXTRACTION_PROMPT_VARIANT=compact: published Core image + // only has full EXTRACTION_PROMPT — env is a no-op and pairing with + // AM_SLM_CORE_COMPACT_SCHEMA mismatches. TODO(ATO-1936): restore when + // Core image catches up. + ] +} + +#[cfg(test)] +mod tests { + use super::*; + fn sample_env() -> InstanceEnv { + InstanceEnv { + openai_api_key: "sk-test".into(), + atomicmemory_api_key: "amc_test".into(), + atomicmemory_api_url: "https://api.dev.example.com".into(), + cloud_jwks_url: "https://api.dev.example.com/jwks.json".into(), + core_api_key: None, + slm: false, + } + } + + #[test] + fn overlay_clears_openai_and_sets_slm_flag() { + let mut env = sample_env(); + apply_slm_overlay(&mut env); + assert!(env.slm); + assert!(env.openai_api_key.is_empty()); + assert!(slm_docker_env_names().contains(&"LLM_PROVIDER")); + assert!( + slm_child_env_entries() + .iter() + .any(|(k, v)| k == "EMBEDDING_DIMENSIONS" && v == "768") + ); + assert!(!slm_docker_env_names().contains(&"EXTRACTION_PROMPT_VARIANT")); + assert!( + slm_child_env_entries() + .iter() + .all(|(k, _)| k != "EXTRACTION_PROMPT_VARIANT"), + "compact prompt deferred until Core image supports EXTRACTION_PROMPT_VARIANT" + ); + } + + #[test] + fn overlay_omits_compact_extraction_prompt_until_core_image() { + assert!( + slm_child_env_entries() + .iter() + .all(|(k, _)| k != "EXTRACTION_PROMPT_VARIANT") + ); + assert!(!slm_docker_env_names().contains(&"EXTRACTION_PROMPT_VARIANT")); + } + + #[test] + fn add_host_args_are_host_gateway() { + assert_eq!( + SLM_ADD_HOST_ARGS, + &["--add-host", "host.docker.internal:host-gateway"] + ); + } +} diff --git a/crates/cli/src/slm/health.rs b/crates/cli/src/slm/health.rs new file mode 100644 index 0000000..2e38d25 --- /dev/null +++ b/crates/cli/src/slm/health.rs @@ -0,0 +1,112 @@ +//! Readiness probes for managed `am-slm` (`/health` + `/v1/models`). + +use std::time::{Duration, Instant}; + +use anyhow::{Context, Result, bail}; +use serde::Deserialize; + +use super::{SLM_CHAT_MODEL, SLM_EMBED_MODEL}; + +/// Required model advertisement for Connected Local SLM. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct RequiredModels { + pub chat: bool, + pub embed: bool, +} + +impl RequiredModels { + pub fn ready(self) -> bool { + self.chat && self.embed + } +} + +#[derive(Debug, Deserialize)] +struct ModelsResponse { + data: Vec, +} + +#[derive(Debug, Deserialize)] +struct ModelEntry { + id: String, +} + +/// Probe `/health` and `/v1/models` once. +pub async fn check_ready(client: &reqwest::Client, endpoint: &str) -> Result { + let health_url = format!("{}/health", endpoint.trim_end_matches('/')); + let health = client + .get(&health_url) + .send() + .await + .with_context(|| format!("GET {health_url}"))?; + if !health.status().is_success() { + bail!("am-slm /health returned HTTP {}", health.status()); + } + + let models_url = format!("{}/v1/models", endpoint.trim_end_matches('/')); + let response = client + .get(&models_url) + .send() + .await + .with_context(|| format!("GET {models_url}"))?; + if !response.status().is_success() { + bail!("am-slm /v1/models returned HTTP {}", response.status()); + } + let body: ModelsResponse = response + .json() + .await + .context("parse am-slm /v1/models JSON")?; + let ids: Vec<&str> = body.data.iter().map(|m| m.id.as_str()).collect(); + Ok(RequiredModels { + chat: ids.contains(&SLM_CHAT_MODEL), + embed: ids.contains(&SLM_EMBED_MODEL), + }) +} + +/// Poll until both required models are advertised, or `timeout` elapses. +pub async fn wait_until_ready( + client: &reqwest::Client, + endpoint: &str, + timeout: Duration, +) -> Result<()> { + let started = Instant::now(); + let mut last_err: Option = None; + while started.elapsed() < timeout { + match check_ready(client, endpoint).await { + Ok(models) if models.ready() => return Ok(()), + Ok(models) => { + last_err = Some(anyhow::anyhow!( + "am-slm /v1/models missing required ids (chat={} embed={} need {SLM_CHAT_MODEL} + {SLM_EMBED_MODEL})", + models.chat, + models.embed + )); + } + Err(err) => last_err = Some(err), + } + tokio::time::sleep(Duration::from_millis(250)).await; + } + Err(last_err.unwrap_or_else(|| anyhow::anyhow!("am-slm readiness timed out after {timeout:?}"))) + .context("wait for am-slm /health + /v1/models") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn required_models_ready_requires_both() { + assert!( + !RequiredModels { + chat: true, + embed: false + } + .ready() + ); + assert!( + RequiredModels { + chat: true, + embed: true + } + .ready() + ); + } +} diff --git a/crates/cli/src/slm/install.rs b/crates/cli/src/slm/install.rs new file mode 100644 index 0000000..4fdb78c --- /dev/null +++ b/crates/cli/src/slm/install.rs @@ -0,0 +1,282 @@ +//! Download, verify, and atomically install the managed `am-slm` binary. + +use std::fs::{self, File}; +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::process::Command; + +use anyhow::{Context, Result, bail}; +use serde::Serialize; +use sha2::{Digest, Sha256}; + +use super::manifest::{ + ManifestArtifact, VersionManifest, current_target, fetch_manifest, select_artifact, +}; +use super::paths::{SlmPaths, SlmStateFile}; + +/// Result of `install` / `update`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct InstallOutcome { + pub version: String, + pub tag: String, + pub binary_path: PathBuf, + pub replaced_existing: bool, + pub artifact_sha256: String, +} + +/// Install or update from the public manifest (fail closed on target/checksum). +pub async fn install_or_update( + client: &reqwest::Client, + paths: &SlmPaths, + manifest_url: &str, +) -> Result { + let target = current_target().ok_or_else(|| { + anyhow::anyhow!( + "am-slm is not published for this platform ({os}-{arch}); Apple Silicon macOS only", + os = std::env::consts::OS, + arch = std::env::consts::ARCH + ) + })?; + let manifest = fetch_manifest(client, manifest_url).await?; + let artifact = select_artifact(&manifest, target)?.clone(); + install_artifact(client, paths, &manifest, &artifact).await +} + +/// Install a previously selected artifact (tests may inject local file URLs). +pub async fn install_artifact( + client: &reqwest::Client, + paths: &SlmPaths, + manifest: &VersionManifest, + artifact: &ManifestArtifact, +) -> Result { + paths.ensure()?; + let staging = paths.staging_dir(); + let _ = fs::remove_dir_all(&staging); + fs::create_dir_all(&staging).context("recreate am-slm staging dir")?; + + let tarball = staging.join(&artifact.filename); + download_verified(client, &artifact.url, artifact.sha256.as_str(), &tarball).await?; + + let extract_dir = staging.join("extract"); + fs::create_dir_all(&extract_dir).context("create extract dir")?; + extract_tarball(&tarball, &extract_dir)?; + + let staged_bin = find_am_slm_binary(&extract_dir)?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let mut perms = fs::metadata(&staged_bin) + .context("stat staged am-slm")? + .permissions(); + perms.set_mode(0o755); + fs::set_permissions(&staged_bin, perms).context("chmod staged am-slm")?; + } + + let dest = paths.binary(); + let replaced_existing = dest.exists(); + let dest_tmp = paths.bin_dir().join("am-slm.new"); + fs::copy(&staged_bin, &dest_tmp).context("copy staged am-slm into bin")?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let mut perms = fs::metadata(&dest_tmp)?.permissions(); + perms.set_mode(0o755); + fs::set_permissions(&dest_tmp, perms)?; + } + fs::rename(&dest_tmp, &dest).context("atomic replace am-slm binary")?; + let _ = fs::remove_dir_all(&staging); + + let mut state = paths.read_state().unwrap_or_default(); + state.installed_version = Some(manifest.version.clone()); + state.installed_tag = Some(manifest.tag.clone()); + state.source_sha = Some(manifest.source_sha.clone()); + state.artifact_sha256 = Some(artifact.sha256.clone()); + state.binary_path = Some(dest.display().to_string()); + state.log_path = Some(paths.log_path().display().to_string()); + paths.write_state(&state)?; + + Ok(InstallOutcome { + version: manifest.version.clone(), + tag: manifest.tag.clone(), + binary_path: dest, + replaced_existing, + artifact_sha256: artifact.sha256.clone(), + }) +} + +async fn download_verified( + client: &reqwest::Client, + url: &str, + expected_sha256: &str, + dest: &Path, +) -> Result<()> { + let response = client + .get(url) + .send() + .await + .with_context(|| format!("download am-slm artifact from {url}"))?; + if !response.status().is_success() { + bail!( + "am-slm artifact unreachable at {url} (HTTP {}) — fail closed", + response.status() + ); + } + let bytes = response + .bytes() + .await + .context("read am-slm artifact bytes")?; + let digest = hex::encode(Sha256::digest(&bytes)); + if !digest.eq_ignore_ascii_case(expected_sha256) { + bail!( + "am-slm artifact checksum mismatch (expected {expected_sha256}, got {digest}) — refusing install" + ); + } + let mut file = File::create(dest).context("create am-slm tarball staging file")?; + file.write_all(&bytes).context("write am-slm tarball")?; + let _ = file.flush(); + Ok(()) +} + +fn extract_tarball(tarball: &Path, dest_dir: &Path) -> Result<()> { + let status = Command::new("tar") + .args([ + "-xzf", + tarball + .to_str() + .ok_or_else(|| anyhow::anyhow!("am-slm tarball path is not UTF-8"))?, + "-C", + dest_dir + .to_str() + .ok_or_else(|| anyhow::anyhow!("am-slm extract path is not UTF-8"))?, + ]) + .status() + .context("spawn tar to extract am-slm artifact")?; + if !status.success() { + bail!("tar failed extracting am-slm artifact (exit {status})"); + } + Ok(()) +} + +fn find_am_slm_binary(extract_dir: &Path) -> Result { + let direct = extract_dir.join("am-slm"); + if direct.is_file() { + return Ok(direct); + } + for entry in fs::read_dir(extract_dir).context("list am-slm extract dir")? { + let entry = entry?; + let candidate = entry.path().join("am-slm"); + if candidate.is_file() { + return Ok(candidate); + } + } + bail!( + "am-slm binary missing from extracted artifact under {}", + extract_dir.display() + ); +} + +/// SHA-256 hex of a local file (tests / doctor). +#[cfg_attr(not(test), allow(dead_code))] +pub fn file_sha256(path: &Path) -> Result { + let bytes = fs::read(path).with_context(|| format!("read {}", path.display()))?; + Ok(hex::encode(Sha256::digest(&bytes))) +} + +/// Remove the managed binary and state (models retained unless `purge_models`). +pub fn uninstall(paths: &SlmPaths, purge_models: bool) -> Result<()> { + let preserved_dims = paths + .read_state() + .ok() + .and_then(|state| state.embedding_dimensions); + + if paths.binary().exists() { + fs::remove_file(paths.binary()).context("remove am-slm binary")?; + } + let _ = fs::remove_file(paths.pid_path()); + let _ = fs::remove_file(paths.state_path()); + let _ = fs::remove_file(paths.log_path()); + let _ = fs::remove_dir_all(paths.staging_dir()); + if purge_models { + super::cache::purge_managed_model_cache(paths)?; + } + // Clear install fields but keep embedding_dimensions marker when Core data remains. + let state = SlmStateFile { + embedding_dimensions: preserved_dims, + ..SlmStateFile::default() + }; + let _ = paths.write_state(&state); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::slm::SLM_EMBED_DIMENSIONS; + use crate::slm::manifest::parse_manifest; + use sha2::{Digest, Sha256}; + use tempfile::tempdir; + + #[test] + fn checksum_mismatch_surface() { + let dir = tempdir().unwrap(); + let payload = b"not-a-real-tarball"; + let good = hex::encode(Sha256::digest(payload)); + let tarball = dir.path().join("a.tgz"); + fs::write(&tarball, payload).unwrap(); + assert_eq!(file_sha256(&tarball).unwrap(), good); + assert_ne!(good, "00".repeat(32)); + } + + #[test] + fn find_binary_nested_or_flat() { + let dir = tempdir().unwrap(); + let flat = dir.path().join("flat"); + fs::create_dir_all(&flat).unwrap(); + fs::write(flat.join("am-slm"), b"x").unwrap(); + assert_eq!(find_am_slm_binary(&flat).unwrap(), flat.join("am-slm")); + + let nested_root = dir.path().join("nested"); + let nested = nested_root.join("am-slm-0.1.1"); + fs::create_dir_all(&nested).unwrap(); + fs::write(nested.join("am-slm"), b"x").unwrap(); + assert_eq!( + find_am_slm_binary(&nested_root).unwrap(), + nested.join("am-slm") + ); + } + + #[test] + fn manifest_fixture_parses_for_install_wiring() { + let body = include_str!("../../tests/fixtures/am-slm-version.json"); + let m = parse_manifest(body).unwrap(); + assert_eq!(m.version, "0.1.1"); + } + + #[test] + fn uninstall_preserves_embedding_dimensions_without_purge() { + let dir = tempdir().unwrap(); + let paths = SlmPaths::at(dir.path().to_path_buf()); + let mut state = paths.read_state().unwrap(); + state.embedding_dimensions = Some(SLM_EMBED_DIMENSIONS); + paths.write_state(&state).unwrap(); + uninstall(&paths, false).unwrap(); + let state = paths.read_state().unwrap(); + assert_eq!(state.embedding_dimensions, Some(SLM_EMBED_DIMENSIONS)); + } + + #[test] + fn uninstall_preserves_embedding_dimensions_when_purging_models() { + let dir = tempdir().unwrap(); + let paths = SlmPaths::at(dir.path().to_path_buf()); + let mut state = paths.read_state().unwrap(); + state.embedding_dimensions = Some(SLM_EMBED_DIMENSIONS); + paths.write_state(&state).unwrap(); + uninstall(&paths, true).unwrap(); + let state = paths.read_state().unwrap(); + assert_eq!( + state.embedding_dimensions, + Some(SLM_EMBED_DIMENSIONS), + "model-cache purge must not drop the volume/provider marker" + ); + } +} diff --git a/crates/cli/src/slm/manifest.rs b/crates/cli/src/slm/manifest.rs new file mode 100644 index 0000000..22c571b --- /dev/null +++ b/crates/cli/src/slm/manifest.rs @@ -0,0 +1,197 @@ +//! Public `am-slm` version.json (schema_version 1) — parse + target select. + +use anyhow::{Context, Result, bail}; +use serde::{Deserialize, Serialize}; + +/// Default public manifest URL (ATO-1935 / ATO-1936 contract). +pub const DEFAULT_MANIFEST_URL: &str = "https://get.atomicstrata.ai/am-slm/version.json"; + +/// Failures specific to manifest consumption (stable operator messages). +#[derive(Debug)] +pub enum ManifestError { + UnsupportedTarget { found: String }, + UnsupportedSchema { found: u32 }, + MissingArtifact { target: String }, +} + +impl std::fmt::Display for ManifestError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::UnsupportedTarget { found } => write!( + f, + "am-slm is not published for this platform ({found}); Apple Silicon macOS (aarch64-apple-darwin) only until a Linux/Intel artifact exists" + ), + Self::UnsupportedSchema { found } => write!( + f, + "am-slm manifest schema_version {found} is unsupported (expected 1)" + ), + Self::MissingArtifact { target } => { + write!(f, "am-slm manifest has no artifact for target {target}") + } + } + } +} + +impl std::error::Error for ManifestError {} + +/// Top-level `version.json` document. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct VersionManifest { + pub schema_version: u32, + pub version: String, + pub tag: String, + pub source_sha: String, + pub supported_targets: Vec, + pub artifacts: Vec, + pub min_disk_bytes: MinDiskBytes, + pub models_catalog_url: String, + pub runtime_api_compat: String, + #[serde(default)] + pub license_id: Option, + #[serde(default)] + pub third_party_notices_file: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct ManifestArtifact { + pub target: String, + pub filename: String, + pub url: String, + pub sha256: String, + pub size_bytes: u64, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct MinDiskBytes { + pub binary_only: u64, + pub with_models: u64, +} + +/// Host target triple the managed runtime publishes for, when supported. +pub fn current_target() -> Option<&'static str> { + if cfg!(all(target_os = "macos", target_arch = "aarch64")) { + Some("aarch64-apple-darwin") + } else { + None + } +} + +/// Human platform label for status / doctor. +pub fn current_platform_label() -> String { + format!("{}-{}", std::env::consts::OS, std::env::consts::ARCH) +} + +/// Parse and validate a manifest body (fail closed on schema drift). +pub fn parse_manifest(body: &str) -> Result { + let manifest: VersionManifest = + serde_json::from_str(body).context("parse am-slm version.json")?; + if manifest.schema_version != 1 { + bail!(ManifestError::UnsupportedSchema { + found: manifest.schema_version + }); + } + if manifest.version.trim().is_empty() { + bail!("am-slm manifest version must be non-empty"); + } + if manifest.artifacts.is_empty() { + bail!("am-slm manifest artifacts must be non-empty"); + } + if manifest.runtime_api_compat.trim().is_empty() { + bail!("am-slm manifest runtime_api_compat must be non-empty"); + } + for artifact in &manifest.artifacts { + validate_artifact(artifact)?; + } + Ok(manifest) +} + +fn validate_artifact(artifact: &ManifestArtifact) -> Result<()> { + if artifact.target.trim().is_empty() { + bail!("am-slm artifact target must be non-empty"); + } + if artifact.url.trim().is_empty() { + bail!("am-slm artifact url must be non-empty"); + } + if artifact.sha256.len() != 64 || !artifact.sha256.chars().all(|c| c.is_ascii_hexdigit()) { + bail!( + "am-slm artifact sha256 must be 64 hex chars, got {:?}", + artifact.sha256 + ); + } + if artifact.size_bytes == 0 { + bail!("am-slm artifact size_bytes must be > 0"); + } + Ok(()) +} + +/// Select the artifact for `target`, fail closed when unsupported/missing. +pub fn select_artifact<'a>( + manifest: &'a VersionManifest, + target: &str, +) -> Result<&'a ManifestArtifact> { + if !manifest.supported_targets.iter().any(|t| t == target) { + bail!(ManifestError::UnsupportedTarget { + found: target.to_string() + }); + } + manifest + .artifacts + .iter() + .find(|a| a.target == target) + .ok_or_else(|| { + anyhow::Error::new(ManifestError::MissingArtifact { + target: target.to_string(), + }) + }) +} + +/// Fetch + parse the public (or override) manifest URL. +pub async fn fetch_manifest(client: &reqwest::Client, url: &str) -> Result { + let response = client + .get(url) + .send() + .await + .with_context(|| format!("fetch am-slm manifest from {url}"))?; + if !response.status().is_success() { + bail!( + "am-slm manifest unreachable at {url} (HTTP {}) — check network / R2", + response.status() + ); + } + let body = response.text().await.context("read am-slm manifest body")?; + parse_manifest(&body) +} + +#[cfg(test)] +mod tests { + use super::*; + + const FIXTURE: &str = include_str!("../../tests/fixtures/am-slm-version.json"); + + #[test] + fn parses_public_0_1_1_contract() { + let m = parse_manifest(FIXTURE).unwrap(); + assert_eq!(m.version, "0.1.1"); + assert_eq!(m.runtime_api_compat, "am-slm-openai-v1"); + let art = select_artifact(&m, "aarch64-apple-darwin").unwrap(); + assert_eq!(art.sha256.len(), 64); + assert_eq!(art.size_bytes, 6_374_708); + } + + #[test] + fn rejects_unsupported_target() { + let m = parse_manifest(FIXTURE).unwrap(); + let err = select_artifact(&m, "x86_64-unknown-linux-gnu").unwrap_err(); + assert!(err.to_string().contains("Apple Silicon")); + } + + #[test] + fn rejects_bad_schema_and_sha() { + assert!(parse_manifest(r#"{"schema_version":2,"version":"1","tag":"t","source_sha":"s","supported_targets":[],"artifacts":[],"min_disk_bytes":{"binary_only":1,"with_models":1},"models_catalog_url":"u","runtime_api_compat":"c"}"#).is_err()); + let mut bad = FIXTURE.to_string(); + if let Some(art) = parse_manifest(FIXTURE).unwrap().artifacts.first() { + bad = bad.replace(&art.sha256, "deadbeef"); + } + assert!(parse_manifest(&bad).is_err()); + } +} diff --git a/crates/cli/src/slm/mod.rs b/crates/cli/src/slm/mod.rs new file mode 100644 index 0000000..4de3ffc --- /dev/null +++ b/crates/cli/src/slm/mod.rs @@ -0,0 +1,69 @@ +//! Managed `am-slm` runtime for Connected Local SLM (ATO-1936). +//! +//! Public contract: `https://get.atomicstrata.ai/am-slm/version.json` (0.1.1). +//! Apple Silicon macOS only until Linux/Intel artifacts exist. + +#![allow(unused_imports)] // re-exports form the public SLM API surface +pub mod bootstrap; +pub mod cache; +pub mod cache_probe; +pub mod core_env; +pub mod health; +pub mod install; +pub mod manifest; +pub mod models; +pub mod paths; +pub mod process; +pub mod process_identity; +pub mod status; + +pub use bootstrap::{BootstrapConfirm, bootstrap_managed_slm, disk_cache_ready, runtime_missing}; +pub use cache::{ + apply_slm_cache_env, ensure_cache_roots, managed_model_cache_dirs, models_cache_flags, + purge_managed_model_cache, slm_adapter_cache, slm_hf_home, +}; +pub use cache_probe::{models_cache_flags_with_disk, models_cache_ready_with_disk}; +pub use core_env::{ + SLM_ADD_HOST_ARGS, apply_slm_overlay, slm_child_env_entries, slm_docker_env_names, +}; +pub use health::{RequiredModels, check_ready, wait_until_ready}; +pub use install::{InstallOutcome, install_or_update, uninstall}; +pub use manifest::{ + DEFAULT_MANIFEST_URL, ManifestArtifact, VersionManifest, current_platform_label, + current_target, fetch_manifest, parse_manifest, select_artifact, +}; +pub use models::{ + ModelsPullOutcome, confirm_models_pull, ensure_models_cached, pull_models, status_models_json, +}; +pub use paths::{SlmPaths, SlmStateFile, default_slm_paths}; +pub use process::{ + SLM_CORE_JSON_SCHEMA_ENV, StartOutcome, StopOutcome, pid_alive, preflight_managed_start, + read_log_tail, slm_serve_env_entries, start_runtime, stop_runtime, +}; +pub use status::{SlmStatusJson, collect_status}; + +/// Default OpenAI-compatible listen port for managed `am-slm serve`. +pub const DEFAULT_SLM_PORT: u16 = 8080; + +/// Chat model id when the core adapter is loaded. +pub const SLM_CHAT_MODEL: &str = "am-slm-core"; + +/// Embedding model id Core should call (no query/document suffix). +pub const SLM_EMBED_MODEL: &str = "nomic-embed-text"; + +/// Nomic embedding dimensionality for Connected Local SLM. +pub const SLM_EMBED_DIMENSIONS: u32 = 768; + +/// Runtime API compatibility token from the public manifest. +pub const RUNTIME_API_COMPAT: &str = "am-slm-openai-v1"; + +/// Base URL Core containers use to reach the host Metal runtime. +pub const SLM_HOST_DOCKER_BASE: &str = "http://host.docker.internal:8080/v1"; + +/// Env override for the manifest URL (tests / mirrors). +pub const ENV_MANIFEST_URL: &str = "AM_SLM_MANIFEST_URL"; + +/// Resolve the manifest URL (env override or public default). +pub fn manifest_url_from_env() -> String { + std::env::var(ENV_MANIFEST_URL).unwrap_or_else(|_| DEFAULT_MANIFEST_URL.to_string()) +} diff --git a/crates/cli/src/slm/models.rs b/crates/cli/src/slm/models.rs new file mode 100644 index 0000000..252b523 --- /dev/null +++ b/crates/cli/src/slm/models.rs @@ -0,0 +1,343 @@ +//! Wrap `am-slm models pull|status` (never silent ~1.7GB downloads). + +use std::io::{self, Write}; +use std::process::Command; + +use anyhow::{Context, Result, bail}; +use serde::Serialize; + +mod progress; +mod status; + +use super::SLM_CHAT_MODEL; +use super::cache::{apply_slm_cache_env, ensure_cache_roots, slm_hf_home}; +use super::cache_probe::models_cache_flags_with_disk; +use super::paths::SlmPaths; + +/// Outcome of an explicit models pull. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct ModelsPullOutcome { + pub ok: bool, + pub message: String, +} + +/// Confirm a large model download when `--yes` is absent. +pub fn confirm_models_pull(yes: bool, allow_prompt: bool) -> Result { + if yes { + return Ok(true); + } + if !allow_prompt { + bail!( + "model download is ~1.7GB and requires confirmation — re-run with `--yes` (non-interactive) or without `--quiet`/`-o json`" + ); + } + eprint!("Download am-slm models (~1.7GB)? [y/N] "); + let _ = io::stderr().flush(); + let mut line = String::new(); + io::stdin() + .read_line(&mut line) + .context("read models pull confirmation")?; + Ok(matches!( + line.trim().to_ascii_lowercase().as_str(), + "y" | "yes" + )) +} + +/// Run `am-slm models pull` (optionally scoped to an adapter). +/// +/// No `--adapter` pulls the start-complete set: Qwen + Nomic bases, then +/// `am-slm-core`, then verifies disk cache readiness. +pub async fn pull_models( + paths: &SlmPaths, + adapter: Option<&str>, + mut report: impl FnMut(&str), +) -> Result { + require_slm_binary(paths)?; + ensure_cache_roots(paths)?; + if let Some(adapter) = adapter { + run_models_pull(paths, Some(adapter), &mut report).await?; + return Ok(completed_pull()); + } + run_models_pull(paths, None, &mut report).await?; + run_models_pull(paths, Some(SLM_CHAT_MODEL), &mut report).await?; + let status = status_models_json(paths).await.map_err(|error| { + anyhow::anyhow!( + "{error:#}. Downloaded files were retained. Retry with `am slm models pull --yes`" + ) + })?; + verify_cache_status(paths, &status)?; + Ok(completed_pull()) +} + +fn completed_pull() -> ModelsPullOutcome { + ModelsPullOutcome { + ok: true, + message: "models pull completed".into(), + } +} + +fn require_slm_binary(paths: &SlmPaths) -> Result<()> { + let binary = paths.binary(); + if binary.is_file() { + return Ok(()); + } + bail!( + "am-slm binary not installed at {} — run `am slm install` first", + binary.display() + ) +} + +async fn run_models_pull( + paths: &SlmPaths, + adapter: Option<&str>, + report: &mut impl FnMut(&str), +) -> Result<()> { + let mut cmd = Command::new(paths.binary()); + cmd.args(["models", "pull", "--json"]); + if let Some(adapter) = adapter { + cmd.args(["--adapter", adapter]); + } + apply_slm_cache_env(&mut cmd, paths); + progress::run(cmd, report).await.map_err(|error| anyhow::anyhow!( + "{error:#}. Cached and partial downloads were retained. Check your connection and available disk space, then retry with `am slm models pull --yes{}`", + adapter.map(|value| format!(" --adapter {value}")).unwrap_or_default() + )) +} + +/// Read model status asynchronously with bounded output and a deadline. +pub async fn status_models_json(paths: &SlmPaths) -> Result { + require_slm_binary(paths)?; + ensure_cache_roots(paths)?; + let mut command = Command::new(paths.binary()); + command.args(["models", "status", "--json"]); + apply_slm_cache_env(&mut command, paths); + status::capture(command).await +} + +/// Fail closed when required model weights/adapters are not cached locally. +pub async fn ensure_models_cached(paths: &SlmPaths) -> Result<()> { + verify_cache_status(paths, &status_models_json(paths).await?) +} + +fn verify_cache_status(paths: &SlmPaths, status: &serde_json::Value) -> Result<()> { + let flags = models_cache_flags_with_disk(status, &slm_hf_home(paths)); + if flags.ready() { + return Ok(()); + } + bail!("{}", cache_miss_message(&flags)) +} + +fn cache_miss_message(flags: &super::cache::ModelsCacheFlags) -> String { + format!( + "am-slm models are not cached locally (~1.7GB). missing: {}.\n\ + Download with `am slm models pull --yes`", + flags.gaps().join(", ") + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::slm::cache::models_cache_flags; + use std::fs; + + #[test] + fn yes_skips_prompt() { + assert!(confirm_models_pull(true, false).unwrap()); + } + + #[test] + fn noninteractive_without_yes_refuses() { + let err = confirm_models_pull(false, false).unwrap_err(); + assert!(err.to_string().contains("--yes")); + assert!(err.to_string().contains("1.7GB")); + } + + #[test] + fn fixture_models_status_is_ready() { + let body = include_str!("../../tests/fixtures/am-slm-models-status.json"); + let json: serde_json::Value = serde_json::from_str(body).unwrap(); + assert!(models_cache_flags(&json).ready()); + } + + #[test] + fn incomplete_published_status_is_not_ready() { + let body = include_str!("../../tests/fixtures/am-slm-models-status-incomplete.json"); + let json: serde_json::Value = serde_json::from_str(body).unwrap(); + assert!(!models_cache_flags(&json).ready()); + } + + #[tokio::test] + #[cfg(unix)] + async fn ensure_models_cached_refuses_empty_cache() { + use std::fs; + use std::os::unix::fs::PermissionsExt; + + let dir = tempfile::tempdir().unwrap(); + let paths = SlmPaths::at(dir.path().to_path_buf()); + paths.ensure().unwrap(); + let incomplete = include_str!("../../tests/fixtures/am-slm-models-status-incomplete.json"); + fs::write( + paths.binary(), + format!("#!/bin/sh\nprintf '%s\\n' '{incomplete}'\n"), + ) + .unwrap(); + std::fs::set_permissions(paths.binary(), std::fs::Permissions::from_mode(0o755)).unwrap(); + let err = ensure_models_cached(&paths).await.unwrap_err(); + let msg = err.to_string(); + assert!(msg.contains("models pull --yes")); + assert!(msg.contains("missing: Qwen/Qwen3-0.6B, am-slm-core adapter")); + assert!(!msg.contains("instance start")); + } + + #[tokio::test] + #[cfg(unix)] + async fn default_pull_runs_bases_then_core_adapter() { + let (paths, log, _dir) = fake_slm_with_status(COMPLETE_STATUS); + pull_models(&paths, None, |_| {}).await.unwrap(); + let recorded = fs::read_to_string(&log).unwrap(); + assert_eq!( + recorded.lines().collect::>(), + [ + "models pull --json", + "models pull --json --adapter am-slm-core" + ] + ); + } + + #[tokio::test] + #[cfg(unix)] + async fn default_pull_fails_when_status_stays_incomplete() { + let (paths, log, _dir) = fake_slm_with_status(INCOMPLETE_STATUS); + let err = pull_models(&paths, None, |_| {}).await.unwrap_err(); + assert!(err.to_string().contains("missing: Qwen/Qwen3-0.6B")); + assert!(fs::read_to_string(&log).unwrap().contains("models pull")); + } + + #[tokio::test] + #[cfg(unix)] + async fn default_pull_accepts_r2_snapshots_when_status_json_lies() { + let (paths, log, _dir) = fake_slm_with_status(&ato_1944_status_json()); + write_r2_snapshots(&slm_hf_home(&paths).join("hub")); + pull_models(&paths, None, |_| {}).await.unwrap(); + let recorded = fs::read_to_string(&log).unwrap(); + assert!(recorded.contains("models pull")); + assert!(recorded.contains("--adapter am-slm-core")); + } + + #[tokio::test] + #[cfg(unix)] + async fn adapter_pull_does_not_require_full_cache() { + let (paths, log, _dir) = fake_slm_with_status(INCOMPLETE_STATUS); + let outcome = pull_models(&paths, Some("am-slm-core"), |_| {}) + .await + .unwrap(); + assert!(outcome.ok); + assert_eq!( + fs::read_to_string(&log).unwrap().trim(), + "models pull --json --adapter am-slm-core" + ); + } + + #[tokio::test] + #[cfg(unix)] + async fn pull_does_not_inherit_child_output_and_requests_json() { + let (paths, log, _dir) = fake_slm_with_status(COMPLETE_STATUS); + pull_models(&paths, None, |_| {}).await.unwrap(); + assert!( + fs::read_to_string(log) + .unwrap() + .lines() + .all(|line| line.contains("--json")) + ); + } + + #[tokio::test] + #[cfg(unix)] + async fn pull_rejects_unbounded_status_output() { + let status = format!("{}{}", COMPLETE_STATUS, " ".repeat(128 * 1024)); + let (paths, _, _dir) = fake_slm_with_status(&status); + let error = pull_models(&paths, None, |_| {}).await.unwrap_err(); + assert!(format!("{error:#}").contains("status output exceeds")); + } + + #[cfg(unix)] + const COMPLETE_STATUS: &str = include_str!("../../tests/fixtures/am-slm-models-status.json"); + #[cfg(unix)] + const INCOMPLETE_STATUS: &str = + include_str!("../../tests/fixtures/am-slm-models-status-incomplete.json"); + + #[cfg(unix)] + fn ato_1944_status_json() -> String { + serde_json::json!({ + "models": [ + { + "id": "Qwen/Qwen3-0.6B", + "revision": "c1899de289a04d12100db370d81485cdf75e47ca", + "ready": false, + "missing": ["config.json", "tokenizer.json", "model.safetensors"] + }, + { + "id": "nomic-ai/nomic-embed-text-v1.5", + "revision": "e9b6763023c676ca8431644204f50c2b100d9aab", + "ready": false, + "missing": ["config.json", "tokenizer.json", "model.safetensors"] + } + ], + "adapters": [{ "id": "am-slm-core", "ready": true, "missing": [] }] + }) + .to_string() + } + + #[cfg(unix)] + fn write_r2_snapshots(hub: &std::path::Path) { + for (repo, rev) in [ + ( + "Qwen/Qwen3-0.6B", + "c1899de289a04d12100db370d81485cdf75e47ca", + ), + ( + "nomic-ai/nomic-embed-text-v1.5", + "e9b6763023c676ca8431644204f50c2b100d9aab", + ), + ] { + let dir = hub + .join(format!("models--{}", repo.replace('/', "--"))) + .join("snapshots") + .join(rev); + fs::create_dir_all(&dir).unwrap(); + for name in ["config.json", "tokenizer.json", "model.safetensors"] { + fs::write(dir.join(name), b"x").unwrap(); + } + } + } + + #[cfg(unix)] + fn fake_slm_with_status( + status_json: &str, + ) -> (SlmPaths, std::path::PathBuf, tempfile::TempDir) { + use std::os::unix::fs::PermissionsExt; + + let dir = tempfile::tempdir().unwrap(); + let paths = SlmPaths::at(dir.path().to_path_buf()); + paths.ensure().unwrap(); + let log = dir.path().join("pull.log"); + let script = format!( + "#!/bin/sh\n\ + if [ \"$1\" = models ] && [ \"$2\" = pull ]; then\n\ + echo \"$*\" >> '{log}'\n\ + exit 0\n\ + fi\n\ + if [ \"$1\" = models ] && [ \"$2\" = status ]; then\n\ + printf '%s\\n' '{status}'\n\ + exit 0\n\ + fi\n\ + exit 1\n", + log = log.display(), + status = status_json.replace('\'', "'\\''"), + ); + fs::write(paths.binary(), script).unwrap(); + fs::set_permissions(paths.binary(), fs::Permissions::from_mode(0o755)).unwrap(); + (paths, log, dir) + } +} diff --git a/crates/cli/src/slm/models/progress.rs b/crates/cli/src/slm/models/progress.rs new file mode 100644 index 0000000..2697d9a --- /dev/null +++ b/crates/cli/src/slm/models/progress.rs @@ -0,0 +1,252 @@ +//! Bounded runtime download event reader and human progress summaries. + +use std::collections::HashMap; +use std::future::Future; +use std::process::{Command, Stdio}; +use std::time::Duration; + +use anyhow::{Context, Result, bail}; +use serde::Deserialize; +use tokio::io::{AsyncBufRead, AsyncBufReadExt, BufReader}; + +const MAX_EVENT_BYTES: usize = 16 * 1024; +const MAX_FILES: usize = 256; +const REPORT_BYTES: u64 = 8 * 1024 * 1024; +const DOWNLOAD_TIMEOUT: Duration = Duration::from_secs(2 * 60 * 60); + +#[derive(Deserialize)] +#[serde(tag = "event", rename_all = "snake_case")] +enum Event { + ModelStart { + model_id: String, + }, + ModelDone { + model_id: String, + }, + FileCached { + model_id: String, + file: String, + }, + FileStart { + model_id: String, + file: String, + bytes_total: u64, + }, + FileProgress { + model_id: String, + file: String, + bytes: u64, + }, + FileDone { + model_id: String, + file: String, + }, + BundleStart { + models: usize, + }, + BundleDone, +} + +#[derive(Default)] +struct FileProgress { + bytes: u64, + total: u64, + reported: u64, +} + +#[derive(Default)] +struct DownloadProgress { + files: HashMap<(String, String), FileProgress>, +} + +impl DownloadProgress { + fn apply(&mut self, event: Event) -> Result> { + let message = match event { + Event::BundleStart { models } => format!("downloading {models} model(s)"), + Event::BundleDone => "model bundle downloaded".into(), + Event::ModelStart { model_id } => format!("downloading {}", label(&model_id)), + Event::ModelDone { model_id } => format!("{} downloaded", label(&model_id)), + Event::FileCached { model_id, file } => { + format!("{} / {} — cached", label(&model_id), label(&file)) + } + Event::FileStart { + model_id, + file, + bytes_total, + } => { + if self.files.len() >= MAX_FILES { + bail!("too many model download files in runtime progress"); + } + let detail = file_detail(&model_id, &file, 0, bytes_total); + self.files.insert( + (model_id, file), + FileProgress { + total: bytes_total, + ..Default::default() + }, + ); + detail + } + Event::FileProgress { + model_id, + file, + bytes, + } => { + let Some(progress) = self.files.get_mut(&(model_id.clone(), file.clone())) else { + // Some mirrors report no file size: show bytes without inventing a total. + if self.files.len() >= MAX_FILES { + bail!("too many model download files in runtime progress"); + } + self.files.insert( + (model_id.clone(), file.clone()), + FileProgress { + bytes, + reported: bytes, + total: 0, + }, + ); + return Ok(Some(file_detail(&model_id, &file, bytes, 0))); + }; + // hf-hub reports byte deltas, including bytes reused from partial downloads. + progress.bytes = progress.bytes.saturating_add(bytes); + if progress.bytes.saturating_sub(progress.reported) < REPORT_BYTES { + return Ok(None); + } + progress.reported = progress.bytes; + file_detail(&model_id, &file, progress.bytes, progress.total) + } + Event::FileDone { model_id, file } => { + match self.files.remove(&(model_id.clone(), file.clone())) { + Some(progress) => format!( + "{} — complete", + file_detail(&model_id, &file, progress.bytes, progress.total) + ), + None => format!("{} / {} — complete", label(&model_id), label(&file)), + } + } + }; + Ok(Some(message)) + } +} + +fn label(value: &str) -> String { + value + .chars() + .filter(|ch| !ch.is_control()) + .take(160) + .collect() +} + +fn human_bytes(bytes: u64) -> String { + const MIB: u64 = 1024 * 1024; + if bytes >= MIB { + format!("{:.1} MiB", bytes as f64 / MIB as f64) + } else { + format!("{bytes} B") + } +} + +fn file_detail(model: &str, file: &str, bytes: u64, total: u64) -> String { + let size = if total > 0 { + format!("{} / {}", human_bytes(bytes), human_bytes(total)) + } else { + human_bytes(bytes) + }; + format!("{} / {} — {size}", label(model), label(file)) +} + +/// Read a complete event without allowing a malformed runtime line to grow memory. +async fn read_event_line( + reader: &mut (impl AsyncBufRead + Unpin), + line: &mut Vec, +) -> Result { + line.clear(); + loop { + let available = reader + .fill_buf() + .await + .context("read model download progress")?; + if available.is_empty() { + return Ok(!line.is_empty()); + } + let end = available + .iter() + .position(|byte| *byte == b'\n') + .map(|index| index + 1); + let consumed = end.unwrap_or(available.len()); + if line.len() + consumed > MAX_EVENT_BYTES { + bail!("model download progress event exceeds {MAX_EVENT_BYTES} bytes"); + } + line.extend_from_slice(&available[..consumed]); + reader.consume(consumed); + if end.is_some() { + return Ok(true); + } + } +} + +/// Stream progress with cancellation and a deadline; never inherit runtime output. +pub(super) async fn run(command: Command, report: &mut impl FnMut(&str)) -> Result<()> { + run_until_cancel(command, report, async { + tokio::signal::ctrl_c() + .await + .context("listen for download cancellation") + }) + .await +} + +async fn run_until_cancel( + command: Command, + report: &mut impl FnMut(&str), + cancel: impl Future>, +) -> Result<()> { + let mut command = tokio::process::Command::from(command); + let mut child = command + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .kill_on_drop(true) + .spawn() + .context("spawn am-slm models pull")?; + let stderr = child + .stderr + .take() + .context("capture model download progress")?; + let mut reader = BufReader::new(stderr); + let mut progress = DownloadProgress::default(); + let mut last_detail = String::new(); + let operation = async { + let mut line = Vec::new(); + while read_event_line(&mut reader, &mut line).await? { + // Runtime diagnostics may share stderr; only the structured protocol is displayed. + if let Ok(event) = serde_json::from_slice::(&line) + && let Some(detail) = progress.apply(event)? + { + report(&detail); + last_detail = detail; + } + } + let status = child.wait().await.context("wait for model download")?; + if !status.success() { + bail!("am-slm models pull failed (exit {status})"); + } + Ok(()) + }; + let result = tokio::select! { + result = operation => result, + result = cancel => result.and_then(|()| Err(anyhow::anyhow!("model download cancelled"))), + () = tokio::time::sleep(DOWNLOAD_TIMEOUT) => Err(anyhow::anyhow!("model download timed out after 2 hours")), + }; + if let Err(error) = result { + // Explicitly reap on handled errors; kill_on_drop also covers cancellation of this future. + child.kill().await.context("stop failed model download")?; + if last_detail.is_empty() { + return Err(error); + } + return Err(error.context(format!("download interrupted at {last_detail}"))); + } + Ok(()) +} + +#[cfg(test)] +mod tests; diff --git a/crates/cli/src/slm/models/progress/tests.rs b/crates/cli/src/slm/models/progress/tests.rs new file mode 100644 index 0000000..ae3ade3 --- /dev/null +++ b/crates/cli/src/slm/models/progress/tests.rs @@ -0,0 +1,211 @@ +//! Deterministic runtime event and subprocess contract tests. + +use super::*; + +#[test] +fn accumulates_byte_deltas_and_reports_known_totals() { + let mut state = DownloadProgress::default(); + state + .apply(Event::FileStart { + model_id: "qwen".into(), + file: "weights".into(), + bytes_total: 33554432, + }) + .unwrap(); + state + .apply(Event::FileProgress { + model_id: "qwen".into(), + file: "weights".into(), + bytes: 8388608, + }) + .unwrap(); + let detail = state + .apply(Event::FileProgress { + model_id: "qwen".into(), + file: "weights".into(), + bytes: 8388608, + }) + .unwrap() + .unwrap(); + assert!(detail.contains("16.0 MiB / 32.0 MiB")); + assert!(!detail.contains('%')); +} + +#[test] +fn unknown_size_and_cache_hits_do_not_invent_a_total() { + let mut state = DownloadProgress::default(); + let detail = state + .apply(Event::FileProgress { + model_id: "qwen".into(), + file: "weights".into(), + bytes: 42, + }) + .unwrap() + .unwrap(); + assert!(detail.ends_with("42 B")); + let cached = state + .apply(Event::FileCached { + model_id: "qwen".into(), + file: "config".into(), + }) + .unwrap() + .unwrap(); + assert!(cached.ends_with("cached")); + assert!(!cached.contains('%')); +} + +#[test] +fn bounds_tracked_files_and_rate_of_byte_updates() { + let mut state = DownloadProgress::default(); + for i in 0..MAX_FILES { + state + .apply(Event::FileStart { + model_id: "qwen".into(), + file: i.to_string(), + bytes_total: 100, + }) + .unwrap(); + } + assert!( + state + .apply(Event::FileStart { + model_id: "qwen".into(), + file: "overflow".into(), + bytes_total: 100 + }) + .is_err() + ); + for _ in 0..1000 { + assert!( + state + .apply(Event::FileProgress { + model_id: "qwen".into(), + file: "0".into(), + bytes: 1 + }) + .unwrap() + .is_none() + ); + } +} + +#[tokio::test] +async fn rejects_oversize_stderr_without_reading_unbounded_lines() { + let bytes = vec![b'x'; MAX_EVENT_BYTES + 1]; + let mut reader = BufReader::new(bytes.as_slice()); + let mut line = Vec::new(); + let error = read_event_line(&mut reader, &mut line).await.unwrap_err(); + assert!(error.to_string().contains("exceeds")); + assert!(line.len() <= MAX_EVENT_BYTES); +} + +#[cfg(unix)] +fn script(body: &str) -> Command { + let mut command = Command::new("/bin/sh"); + command.args(["-c", body]); + command +} + +#[tokio::test] +#[cfg(unix)] +async fn subprocess_only_forwards_structured_stderr_and_drains_stdout() { + let command = script( + r#" + printf '%s\n' '{"event":"model_start","model_id":"qwen","revision":"main"}' >&2 + printf '%s\n' '{"event":"file_cached","model_id":"qwen","file":"config","path":"private"}' >&2 + printf '%s\n' 'diagnostic with private data' >&2 + dd if=/dev/zero bs=1024 count=128 2>/dev/null + printf '%s\n' '{"event":"model_done","model_id":"qwen"}' >&2 + "#, + ); + let mut reports = Vec::new(); + run_until_cancel( + command, + &mut |detail| reports.push(detail.to_string()), + std::future::pending(), + ) + .await + .unwrap(); + assert_eq!( + reports, + [ + "downloading qwen", + "qwen / config — cached", + "qwen downloaded" + ] + ); +} + +#[tokio::test] +#[cfg(unix)] +async fn subprocess_failure_keeps_model_context() { + let command = + script(r#"printf '%s\n' '{"event":"model_start","model_id":"qwen"}' >&2; exit 7"#); + let error = run_until_cancel(command, &mut |_| {}, std::future::pending()) + .await + .unwrap_err(); + let detail = format!("{error:#}"); + assert!(detail.contains("qwen")); + assert!(detail.contains('7')); +} + +#[tokio::test] +#[cfg(unix)] +async fn cancellation_kills_and_reaps_the_child() { + let command = script( + r#"printf '{"event":"model_start","model_id":"%s"}\n' "$$" >&2; while :; do :; done"#, + ); + let (tx, rx) = tokio::sync::oneshot::channel(); + let mut tx = Some(tx); + let mut pid = None; + // The child is started before the first event; cancellation follows that event, without sleeps. + let error = run_until_cancel( + command, + &mut |detail| { + pid = Some( + detail + .strip_prefix("downloading ") + .unwrap() + .parse::() + .unwrap(), + ); + if let Some(tx) = tx.take() { + tx.send(()).unwrap(); + } + }, + async { rx.await.context("cancel test") }, + ) + .await + .unwrap_err(); + assert!(!crate::slm::process::pid_alive(pid.unwrap())); + assert!(error.to_string().contains("download interrupted")); + assert!(format!("{error:#}").contains("cancelled")); +} + +#[tokio::test] +#[cfg(unix)] +async fn download_deadline_kills_a_child_that_stops_reporting() { + let command = script( + r#"printf '{"event":"model_start","model_id":"%s"}\n' "$$" >&2; while :; do :; done"#, + ); + let mut pid = None; + let error = run_until_cancel( + command, + &mut |detail| { + pid = Some( + detail + .strip_prefix("downloading ") + .unwrap() + .parse::() + .unwrap(), + ); + // Advance only after the child is known to be running, without a wall-clock sleep. + tokio::time::pause(); + }, + std::future::pending(), + ) + .await + .unwrap_err(); + assert!(format!("{error:#}").contains("timed out")); + assert!(!crate::slm::process::pid_alive(pid.unwrap())); +} diff --git a/crates/cli/src/slm/models/status.rs b/crates/cli/src/slm/models/status.rs new file mode 100644 index 0000000..89862db --- /dev/null +++ b/crates/cli/src/slm/models/status.rs @@ -0,0 +1,86 @@ +//! Bounded asynchronous cache-status capture for the model download lifecycle. + +use std::process::{Command, Stdio}; +use std::time::Duration; + +use anyhow::{Context, Result, bail}; +use tokio::io::{AsyncRead, AsyncReadExt}; + +const MAX_STATUS_BYTES: usize = 64 * 1024; +const MAX_DIAGNOSTIC_BYTES: usize = 8 * 1024; +const STATUS_TIMEOUT: Duration = Duration::from_secs(30); + +/// Capture the small runtime model catalog without blocking the async executor. +pub(super) async fn capture(command: Command) -> Result { + let mut command = tokio::process::Command::from(command); + let mut child = command + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .kill_on_drop(true) + .spawn() + .context("spawn am-slm models status")?; + let stdout = child.stdout.take().context("capture model status")?; + let stderr = child + .stderr + .take() + .context("capture model status diagnostics")?; + let operation = async { + let (bytes, diagnostics) = tokio::try_join!( + read_bounded(stdout, MAX_STATUS_BYTES, "model status output"), + read_bounded(stderr, MAX_DIAGNOSTIC_BYTES, "model status diagnostics"), + )?; + let status = child.wait().await.context("wait for model status")?; + if !status.success() { + let diagnostics: String = String::from_utf8_lossy(&diagnostics) + .chars() + .filter(|ch| !ch.is_control() || *ch == '\n') + .collect(); + let diagnostics = am_cloud_client::redact::redact_secrets(&diagnostics); + bail!( + "am-slm models status failed (exit {status}): {}", + diagnostics.trim() + ); + } + serde_json::from_slice(&bytes).context("parse am-slm models status --json") + }; + let result = tokio::select! { + result = operation => result, + signal = tokio::signal::ctrl_c() => signal.context("listen for status cancellation").and_then(|()| Err(anyhow::anyhow!("model status cancelled"))), + () = tokio::time::sleep(STATUS_TIMEOUT) => Err(anyhow::anyhow!("model status timed out after 30 seconds")), + }; + if result.is_err() { + child.kill().await.context("stop failed model status")?; + } + result +} + +async fn read_bounded( + stream: impl AsyncRead + Unpin, + limit: usize, + label: &str, +) -> Result> { + let mut bytes = Vec::new(); + stream + .take((limit + 1) as u64) + .read_to_end(&mut bytes) + .await + .with_context(|| format!("read {label}"))?; + if bytes.len() > limit { + bail!("{label} exceeds {limit} bytes"); + } + Ok(bytes) +} + +#[cfg(all(test, unix))] +mod tests { + use super::*; + + #[tokio::test(start_paused = true)] + async fn status_deadline_stops_a_runtime_that_never_exits() { + let mut command = Command::new("/bin/sh"); + command.args(["-c", "while :; do :; done"]); + let error = capture(command).await.unwrap_err(); + assert!(error.to_string().contains("timed out")); + } +} diff --git a/crates/cli/src/slm/paths.rs b/crates/cli/src/slm/paths.rs new file mode 100644 index 0000000..d06cbb5 --- /dev/null +++ b/crates/cli/src/slm/paths.rs @@ -0,0 +1,130 @@ +//! Managed filesystem layout for the CLI-owned `am-slm` runtime. + +use std::fs; +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result, anyhow}; +use directories::ProjectDirs; +use serde::{Deserialize, Serialize}; + +/// On-disk state written by install/start/stop. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)] +pub struct SlmStateFile { + pub installed_version: Option, + pub installed_tag: Option, + pub source_sha: Option, + pub artifact_sha256: Option, + /// True when the CLI owns the process (pidfile + stop/uninstall). + pub managed_process: bool, + pub pid: Option, + pub port: Option, + /// Start stamp (`ps lstart`) for the managed PID — detects PID reuse. + #[serde(default)] + pub process_start_stamp: Option, + pub binary_path: Option, + pub log_path: Option, + /// Embedding dimensionality last used for Connected Local SLM Core volumes. + pub embedding_dimensions: Option, + /// True when managed `am-slm serve` was started with Connected Local + /// full-extract JSON-schema host flag (`AM_SLM_CORE_JSON_SCHEMA=1`). + /// Missing/false means a pre-contract process that `--slm` / + /// `am slm start` should restart. Compact schema is deferred until the + /// published Core image honors `EXTRACTION_PROMPT_VARIANT`. + #[serde(default)] + pub json_schema_contract: bool, +} + +/// Resolved paths under the AtomicMemory data directory. +#[derive(Debug, Clone)] +pub struct SlmPaths { + root: PathBuf, +} + +impl SlmPaths { + pub fn at(root: PathBuf) -> Self { + Self { root } + } + + /// Managed runtime root directory (data-local `slm/`). + #[allow(dead_code)] // public accessor for callers / future doctor paths + pub fn root(&self) -> &Path { + &self.root + } + + pub fn bin_dir(&self) -> PathBuf { + self.root.join("bin") + } + + pub fn binary(&self) -> PathBuf { + self.bin_dir().join("am-slm") + } + + pub fn state_path(&self) -> PathBuf { + self.root.join("state.json") + } + + pub fn pid_path(&self) -> PathBuf { + self.root.join("am-slm.pid") + } + + pub fn log_path(&self) -> PathBuf { + self.root.join("am-slm.log") + } + + pub fn staging_dir(&self) -> PathBuf { + self.root.join("staging") + } + + pub fn ensure(&self) -> Result<()> { + fs::create_dir_all(self.bin_dir()).context("create am-slm bin directory")?; + fs::create_dir_all(self.staging_dir()).context("create am-slm staging directory")?; + Ok(()) + } + + pub fn read_state(&self) -> Result { + let path = self.state_path(); + if !path.exists() { + return Ok(SlmStateFile::default()); + } + let raw = fs::read_to_string(&path).context("read am-slm state.json")?; + serde_json::from_str(&raw).context("parse am-slm state.json") + } + + pub fn write_state(&self, state: &SlmStateFile) -> Result<()> { + self.ensure()?; + let raw = serde_json::to_string_pretty(state).context("serialize am-slm state")?; + let tmp = self.root.join("state.json.tmp"); + fs::write(&tmp, raw.as_bytes()).context("write am-slm state tmp")?; + fs::rename(&tmp, self.state_path()).context("replace am-slm state.json")?; + Ok(()) + } +} + +/// Default managed root: `{data_local}/slm`. +pub fn default_slm_paths() -> Result { + let dirs = ProjectDirs::from("ai", "atomicstrata", "atomicmemory") + .ok_or_else(|| anyhow!("cannot resolve AtomicMemory data directory"))?; + Ok(SlmPaths::at(dirs.data_local_dir().join("slm"))) +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::tempdir; + + #[test] + fn state_round_trip() { + let dir = tempdir().unwrap(); + let paths = SlmPaths::at(dir.path().to_path_buf()); + let state = SlmStateFile { + installed_version: Some("0.1.1".into()), + managed_process: true, + pid: Some(42), + port: Some(8080), + embedding_dimensions: Some(768), + ..SlmStateFile::default() + }; + paths.write_state(&state).unwrap(); + assert_eq!(paths.read_state().unwrap(), state); + } +} diff --git a/crates/cli/src/slm/process.rs b/crates/cli/src/slm/process.rs new file mode 100644 index 0000000..938e55a --- /dev/null +++ b/crates/cli/src/slm/process.rs @@ -0,0 +1,964 @@ +//! Process supervision for managed `am-slm serve` (pidfile, start/stop, stale PID). + +use std::fs; +use std::net::TcpListener; +use std::process::{Command, Stdio}; +use std::time::Duration; + +use anyhow::{Context, Result, bail}; +use serde::Serialize; + +use super::SLM_CHAT_MODEL; +use super::cache::{apply_slm_cache_env, ensure_cache_roots}; +use super::health::{RequiredModels, wait_until_ready}; +use super::models::ensure_models_cached; +use super::paths::{SlmPaths, SlmStateFile}; +use super::process_identity::{ + ProcessIdentity, capture_process_identity, port_owned_by_pid, process_identity_matches, +}; + +/// Outcome of a managed start attempt. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct StartOutcome { + pub pid: u32, + pub port: u16, + pub endpoint: String, + pub already_running: bool, +} + +/// Outcome of a managed stop attempt. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct StopOutcome { + pub stopped: bool, + pub message: String, +} + +/// Host env required for Core Full ingest against `am-slm-core` (json_object). +/// +/// `AM_SLM_CORE_JSON_SCHEMA=1` installs an llguidance grammar on OpenAI +/// `response_format: json_object` matching Core's default (full) extraction +/// prompt. Do **not** set `AM_SLM_CORE_COMPACT_SCHEMA` until the published +/// Core image honors `EXTRACTION_PROMPT_VARIANT=compact` — compact grammar +/// + full prompt yields `compact_prompt_contract_mismatch` and empty memories. +/// +/// TODO(ATO-1936): when Core image ships compact prompt support, restore +/// `AM_SLM_CORE_COMPACT_SCHEMA=1` here and `EXTRACTION_PROMPT_VARIANT=compact` +/// in `core_env` (and optionally fail loud on compact-schema + full-prompt). +pub const SLM_CORE_JSON_SCHEMA_ENV: &str = "AM_SLM_CORE_JSON_SCHEMA"; + +/// Env entries applied to every managed `am-slm serve` spawn. +pub fn slm_serve_env_entries() -> Vec<(String, String)> { + vec![(SLM_CORE_JSON_SCHEMA_ENV.into(), "1".into())] +} + +/// Published am-slm 0.1.1 honors these; leftover manual-setup values must not +/// leak into a managed child (wrong adapter, or `/v1/*` auth that `check_ready` lacks). +const FORBIDDEN_MANAGED_SERVE_ENV: &[&str] = &["AM_SLM_ADAPTER_PATH", "AM_SLM_API_TOKENS"]; + +fn external_slm_missing_contract_message(port: u16) -> String { + format!( + "am-slm is already listening on port {port} but is not CLI-managed with the Connected Local JSON-schema contract.\nManaged starts export {SLM_CORE_JSON_SCHEMA_ENV}=1 so Core Full ingest (`json_object`) returns a memories array (full extraction grammar; compact deferred until Core image supports EXTRACTION_PROMPT_VARIANT).\nStop the external process, then re-run `am slm start` or `am instance start --slm`, or restart it yourself with that env var set." + ) +} + +/// Env entries and cache contract applied to every managed `am-slm serve` spawn. +pub fn configure_serve_command(cmd: &mut Command, paths: &SlmPaths) -> Result<()> { + configure_serve_command_with_inherited(cmd, paths, &inherited_forbidden_serve_env()) +} + +fn configure_serve_command_with_inherited( + cmd: &mut Command, + paths: &SlmPaths, + inherited_conflicts: &[&str], +) -> Result<()> { + refuse_inherited_serve_overrides(inherited_conflicts)?; + for (key, value) in slm_serve_env_entries() { + cmd.env(key, value); + } + cmd.env_remove("AM_SLM_CORE_COMPACT_SCHEMA"); + for key in FORBIDDEN_MANAGED_SERVE_ENV { + cmd.env_remove(*key); + } + ensure_cache_roots(paths)?; + apply_slm_cache_env(cmd, paths); + Ok(()) +} + +fn inherited_forbidden_serve_env() -> Vec<&'static str> { + inherited_forbidden_serve_env_from(|key| std::env::var_os(key)) +} + +fn inherited_forbidden_serve_env_from(lookup: F) -> Vec<&'static str> +where + F: Fn(&str) -> Option, +{ + FORBIDDEN_MANAGED_SERVE_ENV + .iter() + .copied() + .filter(|key| lookup(key).is_some_and(|value| !value.is_empty())) + .collect() +} + +fn refuse_inherited_serve_overrides(conflicts: &[&str]) -> Result<()> { + if conflicts.is_empty() { + return Ok(()); + } + let listed = conflicts.join(" and "); + bail!( + "refusing managed am-slm start: inherited {listed} override the locked Connected Local runtime.\n\ + Unset {listed} and re-run `am slm start` or `am instance start --slm`." + ) +} + +/// Ownership-aware reuse / port conflict check. Does not inspect the disk cache. +pub async fn preflight_managed_start( + client: &reqwest::Client, + paths: &SlmPaths, + port: u16, + wait: Duration, +) -> Result> { + let binary = paths.binary(); + if !binary.is_file() { + bail!( + "am-slm binary not installed at {} — run `am slm install` first", + binary.display() + ); + } + + let endpoint = format!("http://127.0.0.1:{port}"); + let state = paths.read_state().unwrap_or_default(); + let has_json_schema_contract = state.json_schema_contract; + + if let Ok(models) = probe_models(client, &endpoint).await + && models.chat + && models.embed + { + if !state.managed_process { + bail!("{}", external_slm_missing_contract_message(port)); + } + if has_json_schema_contract { + if let Some(pid) = managed_pid(&state, paths) + && trusted_managed_listener(&state, pid, port) + { + return Ok(Some(StartOutcome { + pid, + port, + endpoint, + already_running: true, + })); + } + clear_stale_managed_state(paths).await?; + bail!("{}", external_slm_missing_contract_message(port)); + } + if let Some(pid) = managed_pid(&state, paths) { + if recorded_identity_matches(&state, pid) { + let _ = stop_runtime(paths).await; + } else { + clear_stale_managed_state(paths).await?; + } + } + } else if let Some(pid) = managed_pid(&state, paths) + && process_alive(pid) + { + if !state.managed_process { + bail!("{}", external_slm_missing_contract_message(port)); + } + if has_json_schema_contract && recorded_identity_matches(&state, pid) { + wait_until_ready(client, &endpoint, wait).await?; + if !trusted_managed_listener(&state, pid, port) { + bail!("{}", external_slm_missing_contract_message(port)); + } + return Ok(Some(StartOutcome { + pid, + port, + endpoint, + already_running: true, + })); + } + if recorded_identity_matches(&state, pid) { + let _ = stop_runtime(paths).await; + } else { + clear_stale_managed_state(paths).await?; + } + } + + if port_in_use(port) { + bail!( + "port {port} is already in use by a non-ready process — stop it or choose another port" + ); + } + Ok(None) +} + +/// Start managed `am-slm serve` (idempotent when healthy on the same port). +pub async fn start_runtime( + client: &reqwest::Client, + paths: &SlmPaths, + port: u16, + wait: Duration, +) -> Result { + if let Some(outcome) = preflight_managed_start(client, paths, port, wait).await? { + return Ok(outcome); + } + ensure_models_cached(paths).await?; + spawn_managed_serve(client, paths, port, wait).await +} + +async fn spawn_managed_serve( + client: &reqwest::Client, + paths: &SlmPaths, + port: u16, + wait: Duration, +) -> Result { + let binary = paths.binary(); + let endpoint = format!("http://127.0.0.1:{port}"); + paths.ensure()?; + let log_file = fs::OpenOptions::new() + .create(true) + .append(true) + .open(paths.log_path()) + .context("open am-slm.log")?; + let log_err = log_file.try_clone().context("clone am-slm.log handle")?; + + let mut cmd = Command::new(&binary); + cmd.args([ + "serve", + "--host", + "127.0.0.1", + "--port", + &port.to_string(), + "--model", + SLM_CHAT_MODEL, + ]) + .stdin(Stdio::null()) + .stdout(Stdio::from(log_file)) + .stderr(Stdio::from(log_err)); + configure_serve_command(&mut cmd, paths)?; + let mut child = cmd + .spawn() + .with_context(|| format!("spawn {}", binary.display()))?; + + let pid = child.id(); + write_pidfile(paths, pid)?; + + let mut state = paths.read_state()?; + state.managed_process = true; + state.json_schema_contract = true; + state.pid = Some(pid); + state.port = Some(port); + state.process_start_stamp = capture_process_identity(pid).map(|id| id.start_stamp); + state.binary_path = Some(binary.display().to_string()); + state.log_path = Some(paths.log_path().display().to_string()); + paths.write_state(&state)?; + + match wait_until_ready(client, &endpoint, wait).await { + Ok(()) => { + let state = paths.read_state().unwrap_or_default(); + if !trusted_managed_listener(&state, pid, port) { + let _ = child.kill(); + let _ = child.wait(); + let _ = stop_runtime(paths).await; + bail!("{}", external_slm_missing_contract_message(port)); + } + Ok(StartOutcome { + pid, + port, + endpoint, + already_running: false, + }) + } + Err(err) => { + let _ = child.kill(); + let _ = child.wait(); + let _ = stop_runtime(paths).await; + Err(err).context("am-slm serve started but failed readiness (/health + /v1/models)") + } + } +} + +/// Stop only a CLI-managed process (never kill an external listener blindly). +pub async fn stop_runtime(paths: &SlmPaths) -> Result { + let mut state = paths.read_state()?; + let Some(pid) = state.pid.or_else(|| read_pidfile(paths)) else { + clear_managed_process_files(paths, &mut state); + paths.write_state(&state)?; + return Ok(StopOutcome { + stopped: false, + message: "no managed am-slm process recorded".into(), + }); + }; + + if !state.managed_process { + return Ok(StopOutcome { + stopped: false, + message: "am-slm process is not CLI-managed — refusing to stop".into(), + }); + } + + if !process_alive(pid) { + clear_managed_process_files(paths, &mut state); + paths.write_state(&state)?; + return Ok(StopOutcome { + stopped: false, + message: format!("managed pid {pid} already exited (cleared stale pidfile)"), + }); + } + + if !recorded_identity_matches(&state, pid) { + clear_managed_process_files(paths, &mut state); + paths.write_state(&state)?; + return Ok(StopOutcome { + stopped: false, + message: format!( + "refusing to signal pid {pid}: recorded am-slm start identity is missing or no longer matches (cleared stale ownership). Stop any leftover listener yourself, then re-run `am slm start`" + ), + }); + } + + terminate_pid(pid)?; + tokio::time::sleep(Duration::from_millis(400)).await; + // Recheck start identity before SIGKILL: the PID can be recycled in the + // wait window. Existence alone is not ownership. + if should_force_kill(&state, pid) { + force_kill_pid(pid)?; + } + + clear_managed_process_files(paths, &mut state); + paths.write_state(&state)?; + + Ok(StopOutcome { + stopped: true, + message: format!("stopped managed am-slm pid {pid}"), + }) +} + +fn managed_pid(state: &SlmStateFile, paths: &SlmPaths) -> Option { + state.pid.or_else(|| read_pidfile(paths)) +} + +fn recorded_identity(state: &SlmStateFile, pid: u32) -> Option { + state + .process_start_stamp + .as_ref() + .map(|start_stamp| ProcessIdentity { + pid, + start_stamp: start_stamp.clone(), + }) +} + +fn recorded_identity_matches(state: &SlmStateFile, pid: u32) -> bool { + match recorded_identity(state, pid) { + Some(expected) => process_identity_matches(pid, &expected), + None => false, + } +} + +fn should_force_kill(state: &SlmStateFile, pid: u32) -> bool { + process_alive(pid) && recorded_identity_matches(state, pid) +} + +fn trusted_managed_listener(state: &SlmStateFile, pid: u32, port: u16) -> bool { + recorded_identity_matches(state, pid) && port > 0 && port_owned_by_pid(port, pid) +} + +async fn clear_stale_managed_state(paths: &SlmPaths) -> Result<()> { + let mut state = paths.read_state()?; + clear_managed_process_files(paths, &mut state); + paths.write_state(&state) +} + +fn clear_managed_process_files(paths: &SlmPaths, state: &mut SlmStateFile) { + clear_pidfile(paths); + state.managed_process = false; + state.json_schema_contract = false; + state.pid = None; + state.process_start_stamp = None; +} + +async fn probe_models(client: &reqwest::Client, endpoint: &str) -> Result { + super::health::check_ready(client, endpoint).await +} + +fn port_in_use(port: u16) -> bool { + TcpListener::bind(("127.0.0.1", port)).is_err() +} + +fn read_pidfile(paths: &SlmPaths) -> Option { + let raw = fs::read_to_string(paths.pid_path()).ok()?; + raw.trim().parse().ok() +} + +fn write_pidfile(paths: &SlmPaths, pid: u32) -> Result<()> { + paths.ensure()?; + fs::write(paths.pid_path(), format!("{pid}\n")).context("write am-slm pidfile") +} + +fn clear_pidfile(paths: &SlmPaths) { + let _ = fs::remove_file(paths.pid_path()); +} + +pub fn pid_alive(pid: u32) -> bool { + process_alive(pid) +} + +#[cfg(unix)] +fn process_alive(pid: u32) -> bool { + // `/bin/kill -0` — existence check without linking libc. + Command::new("/bin/kill") + .args(["-0", &pid.to_string()]) + .status() + .map(|s| s.success()) + .unwrap_or(false) +} + +#[cfg(not(unix))] +fn process_alive(_pid: u32) -> bool { + // Managed SLM process control is Apple Silicon / Unix-only. + false +} + +#[cfg(unix)] +fn terminate_pid(pid: u32) -> Result<()> { + let status = Command::new("/bin/kill") + .args(["-TERM", &pid.to_string()]) + .status() + .with_context(|| format!("send SIGTERM to am-slm pid {pid}"))?; + if !status.success() && process_alive(pid) { + bail!("failed to SIGTERM am-slm pid {pid}"); + } + Ok(()) +} + +#[cfg(not(unix))] +fn terminate_pid(_pid: u32) -> Result<()> { + bail!("managed am-slm process control is Apple Silicon only") +} + +#[cfg(unix)] +fn force_kill_pid(pid: u32) -> Result<()> { + let _ = Command::new("/bin/kill") + .args(["-KILL", &pid.to_string()]) + .status(); + Ok(()) +} + +#[cfg(not(unix))] +fn force_kill_pid(_pid: u32) -> Result<()> { + bail!("managed am-slm process control is Apple Silicon only") +} + +/// Tail the managed log file (best-effort). +pub fn read_log_tail(paths: &SlmPaths, max_bytes: usize) -> Result { + let path = paths.log_path(); + if !path.exists() { + return Ok(String::new()); + } + let data = fs::read(&path).context("read am-slm.log")?; + if data.len() <= max_bytes { + return Ok(String::from_utf8_lossy(&data).into_owned()); + } + Ok(String::from_utf8_lossy(&data[data.len() - max_bytes..]).into_owned()) +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::tempdir; + + #[test] + fn pidfile_round_trip() { + let dir = tempdir().unwrap(); + let paths = SlmPaths::at(dir.path().to_path_buf()); + write_pidfile(&paths, 12345).unwrap(); + assert_eq!(read_pidfile(&paths), Some(12345)); + clear_pidfile(&paths); + assert_eq!(read_pidfile(&paths), None); + } + + #[test] + fn port_free_on_ephemeral() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let port = listener.local_addr().unwrap().port(); + assert!(port_in_use(port)); + drop(listener); + // Another process can briefly race the freed ephemeral port in CI. + let free = (0..20).any(|_| { + if !port_in_use(port) { + return true; + } + std::thread::sleep(Duration::from_millis(10)); + false + }); + assert!( + free, + "ephemeral port {port} stayed busy after listener drop" + ); + } + + #[test] + #[cfg(unix)] + fn process_alive_current_pid() { + let pid = std::process::id(); + assert!(process_alive(pid)); + assert!(!process_alive(1_000_000_001)); + } + + #[test] + #[cfg(unix)] + fn force_kill_requires_live_recorded_identity() { + let pid = std::process::id(); + let identity = capture_process_identity(pid).expect("current process identity"); + let matching = SlmStateFile { + process_start_stamp: Some(identity.start_stamp), + ..SlmStateFile::default() + }; + assert!(should_force_kill(&matching, pid)); + let stale = SlmStateFile { + process_start_stamp: Some("stale-start-stamp".into()), + ..SlmStateFile::default() + }; + assert!(!should_force_kill(&stale, pid)); + assert!(!should_force_kill(&SlmStateFile::default(), pid)); + assert!(!should_force_kill(&matching, 1_000_000_001)); + } + + #[test] + #[cfg(not(unix))] + fn process_alive_is_false_on_non_unix() { + assert!(!process_alive(std::process::id())); + assert!(terminate_pid(1).is_err()); + assert!(force_kill_pid(1).is_err()); + } + + #[test] + fn serve_env_includes_json_schema_only() { + let entries = slm_serve_env_entries(); + assert!( + entries + .iter() + .any(|(k, v)| k == SLM_CORE_JSON_SCHEMA_ENV && v == "1") + ); + assert!( + entries + .iter() + .all(|(k, _)| k != "AM_SLM_CORE_COMPACT_SCHEMA"), + "compact schema deferred until Core image ships EXTRACTION_PROMPT_VARIANT" + ); + assert_eq!(entries.len(), 1); + } + + #[test] + fn external_missing_contract_message_names_json_schema() { + let msg = external_slm_missing_contract_message(8080); + assert!(msg.contains(SLM_CORE_JSON_SCHEMA_ENV)); + assert!(!msg.contains("AM_SLM_CORE_COMPACT_SCHEMA")); + assert!(msg.contains("8080")); + } + + #[test] + #[cfg(unix)] + fn serve_command_clears_inherited_compact_schema() { + use std::os::unix::fs::PermissionsExt; + + let dir = tempdir().unwrap(); + let slm_root = dir.path().join("slm"); + let paths = SlmPaths::at(slm_root); + paths.ensure().unwrap(); + fs::write( + paths.binary(), + "#!/bin/sh\nprintf '%s\\n' \"compact=${AM_SLM_CORE_COMPACT_SCHEMA-}\" \"json=${AM_SLM_CORE_JSON_SCHEMA-}\"\n", + ) + .unwrap(); + fs::set_permissions(paths.binary(), fs::Permissions::from_mode(0o755)).unwrap(); + + let mut probe = Command::new(paths.binary()); + probe.env("AM_SLM_CORE_COMPACT_SCHEMA", "1"); + configure_serve_command(&mut probe, &paths).unwrap(); + let serve_output = probe.output().unwrap(); + let serve_stdout = String::from_utf8_lossy(&serve_output.stdout); + assert!(serve_stdout.contains("json=1"), "got: {serve_stdout}"); + assert!( + !serve_stdout.contains("compact=1"), + "configure_serve_command must strip compact schema, got: {serve_stdout}" + ); + } + + #[test] + fn inherited_lookup_detects_adapter_and_tokens() { + let found = inherited_forbidden_serve_env_from(|key| match key { + "AM_SLM_ADAPTER_PATH" => Some("/tmp/unrelated-adapter".into()), + "AM_SLM_API_TOKENS" => Some("tok_leftover".into()), + _ => None, + }); + assert_eq!(found, vec!["AM_SLM_ADAPTER_PATH", "AM_SLM_API_TOKENS"]); + } + + #[test] + fn inherited_lookup_ignores_empty_overrides() { + let found = inherited_forbidden_serve_env_from(|key| match key { + "AM_SLM_ADAPTER_PATH" => Some(String::new().into()), + "AM_SLM_API_TOKENS" => None, + _ => None, + }); + assert!(found.is_empty()); + refuse_inherited_serve_overrides(&[]).unwrap(); + } + + #[test] + fn inherited_adapter_and_tokens_are_refused() { + let adapter = refuse_inherited_serve_overrides(&["AM_SLM_ADAPTER_PATH"]).unwrap_err(); + assert!(adapter.to_string().contains("AM_SLM_ADAPTER_PATH")); + assert!(adapter.to_string().contains("Unset")); + let tokens = refuse_inherited_serve_overrides(&["AM_SLM_API_TOKENS"]).unwrap_err(); + assert!(tokens.to_string().contains("AM_SLM_API_TOKENS")); + let both = refuse_inherited_serve_overrides(&["AM_SLM_ADAPTER_PATH", "AM_SLM_API_TOKENS"]) + .unwrap_err(); + assert!( + both.to_string() + .contains("AM_SLM_ADAPTER_PATH and AM_SLM_API_TOKENS") + ); + } + + #[test] + fn configure_serve_rejects_inherited_parent_overrides() { + let dir = tempdir().unwrap(); + let paths = SlmPaths::at(dir.path().to_path_buf()); + paths.ensure().unwrap(); + let err = configure_serve_command_with_inherited( + &mut Command::new(paths.binary()), + &paths, + &["AM_SLM_ADAPTER_PATH"], + ) + .expect_err("parent adapter path must fail closed"); + assert!(err.to_string().contains("AM_SLM_ADAPTER_PATH")); + let err = configure_serve_command_with_inherited( + &mut Command::new(paths.binary()), + &paths, + &["AM_SLM_API_TOKENS"], + ) + .expect_err("parent API tokens must fail closed"); + assert!(err.to_string().contains("AM_SLM_API_TOKENS")); + } + + #[test] + #[cfg(unix)] + fn serve_command_strips_adapter_and_tokens_from_child() { + use std::os::unix::fs::PermissionsExt; + + let dir = tempdir().unwrap(); + let paths = SlmPaths::at(dir.path().join("slm")); + paths.ensure().unwrap(); + fs::write( + paths.binary(), + "#!/bin/sh\nprintf '%s\\n' \"adapter=${AM_SLM_ADAPTER_PATH-}\" \"tokens=${AM_SLM_API_TOKENS-}\"\n", + ) + .unwrap(); + fs::set_permissions(paths.binary(), fs::Permissions::from_mode(0o755)).unwrap(); + + let mut probe = Command::new(paths.binary()); + probe.env("AM_SLM_ADAPTER_PATH", "/tmp/unrelated-adapter"); + probe.env("AM_SLM_API_TOKENS", "tok_leftover"); + configure_serve_command_with_inherited(&mut probe, &paths, &[]).unwrap(); + let output = probe.output().unwrap(); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!( + !stdout.contains("adapter=/tmp/unrelated-adapter"), + "child must not inherit adapter path, got: {stdout}" + ); + assert!( + !stdout.contains("tokens=tok_leftover"), + "child must not inherit API tokens, got: {stdout}" + ); + } + + #[tokio::test] + #[cfg(unix)] + async fn stop_refuses_unrelated_process_for_stale_identity() { + let mut child = Command::new("sleep") + .arg("120") + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("spawn sleep"); + let pid = child.id(); + + let dir = tempdir().unwrap(); + let paths = SlmPaths::at(dir.path().to_path_buf()); + paths + .write_state(&SlmStateFile { + managed_process: true, + json_schema_contract: true, + pid: Some(pid), + port: Some(8080), + process_start_stamp: Some("stale-start-stamp".into()), + ..SlmStateFile::default() + }) + .unwrap(); + write_pidfile(&paths, pid).unwrap(); + + let outcome = stop_runtime(&paths).await.unwrap(); + assert!(!outcome.stopped); + assert!(outcome.message.contains("refusing to signal")); + assert!(super::pid_alive(pid)); + + child.kill().ok(); + let _ = child.wait(); + } + + #[tokio::test] + #[cfg(unix)] + async fn stop_refuses_missing_start_stamp_even_when_pid_owns_port() { + let (mut child, port) = spawn_python_listener(); + let pid = child.id(); + + let dir = tempdir().unwrap(); + let paths = SlmPaths::at(dir.path().to_path_buf()); + paths + .write_state(&SlmStateFile { + managed_process: true, + json_schema_contract: true, + pid: Some(pid), + port: Some(port), + process_start_stamp: None, + ..SlmStateFile::default() + }) + .unwrap(); + write_pidfile(&paths, pid).unwrap(); + + let outcome = stop_runtime(&paths).await.unwrap(); + assert!(!outcome.stopped); + assert!(outcome.message.contains("refusing to signal")); + assert!( + super::pid_alive(pid), + "legacy state must not kill the listener" + ); + + child.kill().ok(); + let _ = child.wait(); + } + + #[tokio::test] + #[cfg(unix)] + async fn stop_terminates_identified_child_before_listener() { + let mut child = Command::new("sleep") + .arg("120") + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("spawn sleep"); + let pid = child.id(); + let identity = capture_process_identity(pid).expect("sleep identity"); + + let dir = tempdir().unwrap(); + let paths = SlmPaths::at(dir.path().to_path_buf()); + paths + .write_state(&SlmStateFile { + managed_process: true, + json_schema_contract: true, + pid: Some(pid), + port: Some(8080), + process_start_stamp: Some(identity.start_stamp), + ..SlmStateFile::default() + }) + .unwrap(); + write_pidfile(&paths, pid).unwrap(); + + let outcome = stop_runtime(&paths).await.unwrap(); + assert!(outcome.stopped, "{}", outcome.message); + let _ = child.wait(); + assert!(!super::pid_alive(pid)); + } + + #[tokio::test] + #[cfg(unix)] + async fn start_kills_child_when_readiness_fails() { + use std::os::unix::fs::PermissionsExt; + + let dir = tempdir().unwrap(); + let paths = SlmPaths::at(dir.path().to_path_buf()); + paths.ensure().unwrap(); + let marker = dir.path().join("child.pid"); + let ready = include_str!("../../tests/fixtures/am-slm-models-status.json"); + let script = format!( + "#!/bin/sh\n\ + if [ \"$1\" = models ]; then\n\ + printf '%s\\n' '{ready}'\n\ + exit 0\n\ + fi\n\ + printf '%s\\n' \"$$\" > '{marker}'\n\ + exec sleep 120\n", + ready = ready.replace('\'', "'\\''"), + marker = marker.display() + ); + fs::write(paths.binary(), script).unwrap(); + fs::set_permissions(paths.binary(), fs::Permissions::from_mode(0o755)).unwrap(); + + let port = { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + listener.local_addr().unwrap().port() + }; + let client = reqwest::Client::builder() + .timeout(Duration::from_secs(2)) + .build() + .unwrap(); + let err = start_runtime(&client, &paths, port, Duration::from_millis(400)) + .await + .expect_err("readiness must fail"); + assert!(err.to_string().contains("readiness")); + + let child_pid: u32 = fs::read_to_string(&marker) + .unwrap_or_else(|_| panic!("child pid marker missing: {err}")) + .trim() + .parse() + .unwrap(); + assert!( + !super::pid_alive(child_pid), + "failed start must terminate the spawned child" + ); + } + + #[tokio::test] + #[cfg(unix)] + async fn waiting_start_refuses_foreign_listener_that_becomes_ready() { + use std::os::unix::fs::PermissionsExt; + + let (mut http, port) = spawn_python_http_ready_after_503(); + let mut child = Command::new("sleep") + .arg("120") + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("spawn sleep"); + let pid = child.id(); + let identity = capture_process_identity(pid).expect("sleep identity"); + + let dir = tempdir().unwrap(); + let paths = SlmPaths::at(dir.path().to_path_buf()); + paths.ensure().unwrap(); + fs::write(paths.binary(), "#!/bin/sh\nexit 1\n").unwrap(); + fs::set_permissions(paths.binary(), fs::Permissions::from_mode(0o755)).unwrap(); + paths + .write_state(&SlmStateFile { + managed_process: true, + json_schema_contract: true, + pid: Some(pid), + port: Some(port), + process_start_stamp: Some(identity.start_stamp), + ..SlmStateFile::default() + }) + .unwrap(); + write_pidfile(&paths, pid).unwrap(); + + let client = reqwest::Client::builder() + .timeout(Duration::from_secs(2)) + .build() + .unwrap(); + let err = start_runtime(&client, &paths, port, Duration::from_secs(3)) + .await + .expect_err("foreign listener must not be adopted"); + assert!( + err.to_string().contains("not CLI-managed"), + "expected foreign-listener refusal, got: {err}" + ); + assert!( + super::pid_alive(pid), + "identified child is not the listener" + ); + + child.kill().ok(); + let _ = child.wait(); + http.kill().ok(); + let _ = http.wait(); + } + + #[cfg(unix)] + fn spawn_python_listener() -> (std::process::Child, u16) { + use std::io::{BufRead, BufReader}; + + let mut child = Command::new("python3") + .args([ + "-c", + "import socket, sys, time\n\ + s = socket.socket(); s.bind(('127.0.0.1', 0)); s.listen(1)\n\ + sys.stdout.write(str(s.getsockname()[1]) + '\\n'); sys.stdout.flush()\n\ + time.sleep(120)\n", + ]) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .spawn() + .expect("spawn python listener"); + let stdout = child.stdout.take().expect("listener stdout"); + let mut line = String::new(); + BufReader::new(stdout) + .read_line(&mut line) + .expect("read listener port"); + let port: u16 = line.trim().parse().expect("listener port"); + (child, port) + } + + #[cfg(unix)] + fn spawn_python_http_ready_after_503() -> (std::process::Child, u16) { + use std::io::{BufRead, BufReader}; + + let dir = tempdir().unwrap(); + let script_path = dir.path().join("ready_after_503.py"); + let script = r#" +from http.server import BaseHTTPRequestHandler, HTTPServer +import json + +class H(BaseHTTPRequestHandler): + hits = 0 + def do_GET(self): + H.hits += 1 + if H.hits == 1: + self.send_response(503) + self.end_headers() + return + if self.path.endswith('/health'): + self.send_response(200) + self.end_headers() + self.wfile.write(b'ok') + return + if self.path.endswith('/v1/models'): + body = json.dumps({"data":[{"id":"__CHAT__"},{"id":"__EMBED__"}]}).encode() + self.send_response(200) + self.send_header('Content-Type', 'application/json') + self.end_headers() + self.wfile.write(body) + return + self.send_response(404) + self.end_headers() + def log_message(self, *args): + pass + +server = HTTPServer(('127.0.0.1', 0), H) +print(server.server_address[1], flush=True) +server.serve_forever() +"# + .replace("__CHAT__", crate::slm::SLM_CHAT_MODEL) + .replace("__EMBED__", crate::slm::SLM_EMBED_MODEL); + fs::write(&script_path, script).unwrap(); + // Keep the temp dir alive for the child by leaking it for the test process. + std::mem::forget(dir); + let mut child = Command::new("python3") + .arg(&script_path) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .expect("spawn python http"); + let stdout = child.stdout.take().expect("http stdout"); + let mut line = String::new(); + BufReader::new(stdout) + .read_line(&mut line) + .expect("read http port"); + let port: u16 = line.trim().parse().unwrap_or_else(|_| { + let stderr = child.stderr.take().map(|mut err| { + let mut buf = String::new(); + let _ = std::io::Read::read_to_string(&mut err, &mut buf); + buf + }); + panic!("http port {line:?}; stderr={stderr:?}"); + }); + (child, port) + } +} diff --git a/crates/cli/src/slm/process_identity.rs b/crates/cli/src/slm/process_identity.rs new file mode 100644 index 0000000..23c3db8 --- /dev/null +++ b/crates/cli/src/slm/process_identity.rs @@ -0,0 +1,109 @@ +//! Verify a recorded managed PID still refers to the same process before signalling it. + +use std::process::Command; + +/// Stable identity for a process at spawn time (survives PID reuse checks). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ProcessIdentity { + pub pid: u32, + /// Platform-specific start stamp (`lstart` on Unix). + pub start_stamp: String, +} + +/// Capture the current start identity for `pid`, if the process exists. +pub fn capture_process_identity(pid: u32) -> Option { + read_start_stamp(pid).map(|start_stamp| ProcessIdentity { pid, start_stamp }) +} + +/// True when `pid` is alive and its start stamp still matches `expected`. +pub fn process_identity_matches(pid: u32, expected: &ProcessIdentity) -> bool { + if pid != expected.pid { + return false; + } + match read_start_stamp(pid) { + Some(stamp) => stamp == expected.start_stamp, + None => false, + } +} + +/// PID listening on `127.0.0.1:port`, if any. +pub fn listener_pid_on_port(port: u16) -> Option { + listener_pid_on_port_impl(port) +} + +pub fn port_owned_by_pid(port: u16, pid: u32) -> bool { + listener_pid_on_port(port).is_some_and(|listener| listener == pid) +} + +#[cfg(unix)] +fn read_start_stamp(pid: u32) -> Option { + let output = Command::new("ps") + .args(["-p", &pid.to_string(), "-o", "lstart="]) + .output() + .ok()?; + if !output.status.success() { + return None; + } + let stamp = String::from_utf8_lossy(&output.stdout).trim().to_string(); + if stamp.is_empty() { None } else { Some(stamp) } +} + +#[cfg(not(unix))] +fn read_start_stamp(_pid: u32) -> Option { + None +} + +#[cfg(unix)] +fn listener_pid_on_port_impl(port: u16) -> Option { + let output = Command::new("lsof") + .args(["-nP", &format!("-iTCP:{port}"), "-sTCP:LISTEN", "-t"]) + .output() + .ok()?; + if !output.status.success() { + return None; + } + String::from_utf8_lossy(&output.stdout) + .lines() + .next() + .and_then(|line| line.trim().parse().ok()) +} + +#[cfg(not(unix))] +fn listener_pid_on_port_impl(_port: u16) -> Option { + None +} + +#[cfg(test)] +mod tests { + use super::*; + use std::process::{Command, Stdio}; + use std::time::Duration; + + #[test] + #[cfg(unix)] + fn capture_and_match_current_process() { + let pid = std::process::id(); + let identity = capture_process_identity(pid).expect("current process identity"); + assert!(process_identity_matches(pid, &identity)); + } + + #[test] + #[cfg(unix)] + fn stale_pid_does_not_match_after_process_exits() { + let mut child = Command::new("sleep") + .arg("30") + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("spawn sleep"); + let pid = child.id(); + let identity = capture_process_identity(pid).expect("sleep identity"); + child.kill().ok(); + let _ = child.wait(); + std::thread::sleep(Duration::from_millis(50)); + assert!( + !process_identity_matches(pid, &identity), + "exited pid must not still match its old start stamp" + ); + } +} diff --git a/crates/cli/src/slm/status.rs b/crates/cli/src/slm/status.rs new file mode 100644 index 0000000..e2e5b44 --- /dev/null +++ b/crates/cli/src/slm/status.rs @@ -0,0 +1,313 @@ +//! Stable JSON status document for `am slm status`. + +use serde::Serialize; + +use super::DEFAULT_SLM_PORT; +use super::RUNTIME_API_COMPAT; +use super::cache::slm_hf_home; +use super::cache_probe::models_cache_flags_with_disk; +use super::health::RequiredModels; +use super::manifest::{VersionManifest, current_platform_label, current_target, fetch_manifest}; +use super::manifest_url_from_env; +use super::models::status_models_json; +use super::paths::SlmPaths; +use super::process::pid_alive; + +/// Stable automation-facing status payload. +#[derive(Debug, Clone, Serialize, PartialEq)] +pub struct SlmStatusJson { + pub supported: bool, + pub platform: String, + pub target: Option, + pub installed: bool, + pub installed_version: Option, + pub manifest_version: Option, + pub update_available: bool, + pub process: ProcessStatus, + pub health: HealthStatus, + pub models: ModelsStatus, + pub cache: CacheStatus, + pub disk: DiskStatus, + pub endpoint: String, + pub runtime_api_compat: String, + pub managed: bool, +} + +#[derive(Debug, Clone, Serialize, PartialEq)] +pub struct ProcessStatus { + pub managed: bool, + pub pid: Option, + pub alive: bool, + pub port: u16, +} + +#[derive(Debug, Clone, Serialize, PartialEq)] +pub struct HealthStatus { + pub ok: bool, + pub endpoint: String, +} + +#[derive(Debug, Clone, Serialize, PartialEq)] +pub struct ModelsStatus { + pub am_slm_core: bool, + pub nomic_embed_text: bool, + pub ready: bool, +} + +/// Disk cache readiness (independent of a live `/v1/models` probe). +#[derive(Debug, Clone, Serialize, PartialEq, Default)] +pub struct CacheStatus { + pub qwen: bool, + pub nomic: bool, + pub am_slm_core: bool, + pub ready: bool, + /// Set when `am-slm models status --json` cannot be executed or parsed. + /// `ready: false` without `error` means a successful inspection found gaps. + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, +} + +#[derive(Debug, Clone, Serialize, PartialEq)] +pub struct DiskStatus { + pub binary_only_bytes: u64, + pub with_models_bytes: u64, +} + +/// Collect status using optional live health/manifest probes. +pub async fn collect_status( + client: &reqwest::Client, + paths: &SlmPaths, + manifest: Option<&VersionManifest>, + models: Option, + health_ok: bool, +) -> anyhow::Result { + let target = current_target().map(str::to_string); + let supported = target.is_some(); + let state = paths.read_state().unwrap_or_default(); + let installed = paths.binary().is_file(); + let port = state.port.unwrap_or(DEFAULT_SLM_PORT); + let endpoint_base = format!("http://127.0.0.1:{port}"); + let pid = state.pid; + let alive = pid.map(pid_alive).unwrap_or(false); + + let (manifest_version, disk, runtime_compat) = if let Some(m) = manifest { + ( + Some(m.version.clone()), + DiskStatus { + binary_only_bytes: m.min_disk_bytes.binary_only, + with_models_bytes: m.min_disk_bytes.with_models, + }, + m.runtime_api_compat.clone(), + ) + } else { + match fetch_manifest(client, &manifest_url_from_env()).await { + Ok(m) => ( + Some(m.version.clone()), + DiskStatus { + binary_only_bytes: m.min_disk_bytes.binary_only, + with_models_bytes: m.min_disk_bytes.with_models, + }, + m.runtime_api_compat.clone(), + ), + Err(_) => ( + None, + DiskStatus { + binary_only_bytes: 0, + with_models_bytes: 0, + }, + RUNTIME_API_COMPAT.to_string(), + ), + } + }; + + let installed_version = state.installed_version.clone(); + let update_available = match (&installed_version, &manifest_version) { + (Some(local), Some(remote)) => local != remote, + _ => false, + }; + + let models = models.unwrap_or_default(); + Ok(SlmStatusJson { + supported, + platform: current_platform_label(), + target, + installed, + installed_version, + manifest_version, + update_available, + process: ProcessStatus { + managed: state.managed_process, + pid, + alive, + port, + }, + health: HealthStatus { + ok: health_ok, + endpoint: endpoint_base.clone(), + }, + models: ModelsStatus { + am_slm_core: models.chat, + nomic_embed_text: models.embed, + ready: models.ready(), + }, + cache: disk_cache_status(paths, installed).await, + disk, + endpoint: format!("{endpoint_base}/v1"), + runtime_api_compat: runtime_compat, + managed: state.managed_process, + }) +} + +async fn disk_cache_status(paths: &SlmPaths, installed: bool) -> CacheStatus { + if !installed { + return CacheStatus::default(); + } + match status_models_json(paths).await { + Ok(status) => { + let flags = models_cache_flags_with_disk(&status, &slm_hf_home(paths)); + CacheStatus { + qwen: flags.qwen, + nomic: flags.nomic, + am_slm_core: flags.am_slm_core, + ready: flags.ready(), + error: None, + } + } + Err(err) => CacheStatus { + error: Some(format!("{err:#}")), + ..CacheStatus::default() + }, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::slm::manifest::parse_manifest; + use tempfile::tempdir; + + #[tokio::test] + async fn status_json_shape_from_fixture() { + let body = include_str!("../../tests/fixtures/am-slm-version.json"); + let manifest = parse_manifest(body).unwrap(); + let dir = tempdir().unwrap(); + let paths = SlmPaths::at(dir.path().to_path_buf()); + let client = reqwest::Client::new(); + let status = collect_status( + &client, + &paths, + Some(&manifest), + Some(RequiredModels { + chat: true, + embed: true, + }), + true, + ) + .await + .unwrap(); + let value = serde_json::to_value(&status).unwrap(); + for key in [ + "supported", + "platform", + "target", + "installed", + "installed_version", + "manifest_version", + "update_available", + "process", + "health", + "models", + "cache", + "disk", + "endpoint", + "runtime_api_compat", + "managed", + ] { + assert!(value.get(key).is_some(), "missing {key}"); + } + assert_eq!(status.manifest_version.as_deref(), Some("0.1.1")); + assert_eq!(status.runtime_api_compat, "am-slm-openai-v1"); + assert!(status.models.ready); + assert!(!status.cache.ready); + assert!(status.cache.error.is_none()); + assert_eq!(status.disk.binary_only_bytes, 52_428_800); + } + + #[tokio::test] + #[cfg(unix)] + async fn cache_ready_when_process_is_down() { + use std::fs; + use std::os::unix::fs::PermissionsExt; + + let body = include_str!("../../tests/fixtures/am-slm-version.json"); + let manifest = parse_manifest(body).unwrap(); + let dir = tempdir().unwrap(); + let paths = SlmPaths::at(dir.path().to_path_buf()); + paths.ensure().unwrap(); + let ready = include_str!("../../tests/fixtures/am-slm-models-status.json"); + fs::write( + paths.binary(), + format!("#!/bin/sh\nprintf '%s\\n' '{ready}'\n"), + ) + .unwrap(); + fs::set_permissions(paths.binary(), fs::Permissions::from_mode(0o755)).unwrap(); + let client = reqwest::Client::new(); + let status = collect_status(&client, &paths, Some(&manifest), None, false) + .await + .unwrap(); + assert!(status.cache.ready); + assert!(status.cache.qwen && status.cache.nomic && status.cache.am_slm_core); + assert!(status.cache.error.is_none()); + assert!(!status.models.ready); + assert!(!status.health.ok); + } + + #[tokio::test] + #[cfg(unix)] + async fn cache_error_on_nonzero_status_exit() { + let status = collect_with_binary("#!/bin/sh\necho broken-status >&2\nexit 7\n").await; + assert!(!status.cache.ready); + let error = status.cache.error.expect("inspection error"); + assert!( + error.contains("7") && error.contains("broken-status"), + "{error}" + ); + } + + #[tokio::test] + #[cfg(unix)] + async fn cache_errors_preserve_diagnostics_without_credentials() { + let status = collect_with_binary("#!/bin/sh\necho 'broken-status access_token=private-token user@example.com' >&2\nexit 7\n").await; + let error = status.cache.error.unwrap(); + assert!(error.contains("broken-status")); + assert!(!error.contains("private-token")); + assert!(!error.contains("user@example.com")); + } + + #[tokio::test] + #[cfg(unix)] + async fn cache_error_on_malformed_status_json() { + let status = collect_with_binary("#!/bin/sh\nprintf 'not-json\\n'\nexit 0\n").await; + assert!(!status.cache.ready); + let error = status.cache.error.expect("parse error"); + assert!(error.contains("parse am-slm models status"), "{error}"); + } + + #[cfg(unix)] + async fn collect_with_binary(script: &str) -> SlmStatusJson { + use std::fs; + use std::os::unix::fs::PermissionsExt; + + let body = include_str!("../../tests/fixtures/am-slm-version.json"); + let manifest = parse_manifest(body).unwrap(); + let dir = tempdir().unwrap(); + let paths = SlmPaths::at(dir.path().to_path_buf()); + paths.ensure().unwrap(); + fs::write(paths.binary(), script).unwrap(); + fs::set_permissions(paths.binary(), fs::Permissions::from_mode(0o755)).unwrap(); + let client = reqwest::Client::new(); + collect_status(&client, &paths, Some(&manifest), None, false) + .await + .unwrap() + } +} diff --git a/crates/cli/src/verification/receipt.rs b/crates/cli/src/verification/receipt.rs index 76e5745..11fb5ae 100644 --- a/crates/cli/src/verification/receipt.rs +++ b/crates/cli/src/verification/receipt.rs @@ -6,6 +6,34 @@ use crate::cli::{GlobalOptions, OutputFormat}; use crate::environment::dashboard_project_url; use crate::verification::smoke::SmokeResult; +/// The observed outcome of the verification step, including intentional skips. +#[derive(Debug, Clone)] +pub enum VerificationAttempt { + /// The smoke completed and returned its evidence. + Passed(SmokeResult), + /// An attempted verification failed; retain its actionable error. + Failed(String), + /// The user explicitly requested skipping verification or the instance. + DeliberatelySkipped, + /// Prerequisites prevented the verification from running. + NotRun, +} + +/// Stable machine-readable verification states. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum VerificationStatus { + /// The complete smoke passed. + Passed, + /// An attempted smoke failed. + Failed, + /// The user explicitly skipped verification. + DeliberatelySkipped, + /// Verification did not run because prerequisites were unavailable. + NotRun, +} + +/// Completed onboarding state and the next action for the selected profile. #[derive(Debug, Clone, Serialize)] pub struct InitReceipt { pub identity_ready: bool, @@ -27,6 +55,9 @@ pub struct InitReceipt { pub cloud_connection_online: bool, pub memory_pipeline_verified: bool, pub verification_skipped: bool, + pub verification_status: VerificationStatus, + #[serde(skip_serializing_if = "Option::is_none")] + pub verification_error: Option, #[serde(skip_serializing_if = "Option::is_none")] pub dashboard_url: Option, pub next_command: String, @@ -34,37 +65,59 @@ pub struct InitReceipt { pub smoke: Option, } +/// Explicit observations supplied by the onboarding command boundary. pub struct InitReceiptInput<'a> { pub signed_in_as: Option<&'a str>, pub org_name: &'a str, pub org_id: &'a str, pub project_name: &'a str, pub project_id: &'a str, + pub profile_name: &'a str, pub local_url: &'a str, pub api_base_url: &'a str, pub core_healthy: bool, pub no_instance: bool, pub cloud_connection_online: bool, pub credential_ready: bool, - pub smoke: Option, + pub verification: VerificationAttempt, } +/// Build a receipt whose activation requires every operational prerequisite. pub fn build_init_receipt(input: InitReceiptInput<'_>) -> InitReceipt { - let runtime_ready = input.core_healthy || input.no_instance; - let pipeline_verified = input.smoke.as_ref().is_some_and(|s| s.verified); - let verification_skipped = input.smoke.is_none(); - let verified = pipeline_verified; - let dashboard_url = dashboard_project_url(input.api_base_url, input.project_id); + let (verification_status, verification_error, smoke) = match input.verification { + VerificationAttempt::Passed(smoke) if smoke.verified => { + (VerificationStatus::Passed, None, Some(smoke)) + } + VerificationAttempt::Passed(smoke) => ( + VerificationStatus::Failed, + Some("Memory smoke returned an unverified result".into()), + Some(smoke), + ), + VerificationAttempt::Failed(error) => (VerificationStatus::Failed, Some(error), None), + VerificationAttempt::DeliberatelySkipped => { + (VerificationStatus::DeliberatelySkipped, None, None) + } + VerificationAttempt::NotRun => (VerificationStatus::NotRun, None, None), + }; + let verified = verification_status == VerificationStatus::Passed; + let activated = + input.credential_ready && input.core_healthy && input.cloud_connection_online && verified; + let profile = shell_argument(input.profile_name); + let next_command = if activated { + format!("am --profile {profile} memory ingest \"My preferred editor is Zed\"") + } else { + format!("am --profile {profile} doctor --smoke") + }; InitReceipt { identity_ready: true, workspace_ready: true, project_ready: true, credential_ready: input.credential_ready, - runtime_ready, + runtime_ready: input.core_healthy, linked: input.cloud_connection_online, verified, - activated: verified, + activated, signed_in_as: input.signed_in_as.map(str::to_string), workspace_name: input.org_name.to_string(), workspace_id: input.org_id.to_string(), @@ -74,14 +127,29 @@ pub fn build_init_receipt(input: InitReceiptInput<'_>) -> InitReceipt { core_running: input.core_healthy, core_skipped: input.no_instance, cloud_connection_online: input.cloud_connection_online, - memory_pipeline_verified: pipeline_verified, - verification_skipped, - dashboard_url, - next_command: "am memory ingest \"My preferred editor is Zed\"".into(), - smoke: input.smoke, + memory_pipeline_verified: verified, + verification_skipped: verification_status == VerificationStatus::DeliberatelySkipped, + verification_status, + verification_error, + dashboard_url: dashboard_project_url(input.api_base_url, input.project_id), + next_command, + smoke, + } +} + +pub(super) fn shell_argument(value: &str) -> String { + if !value.is_empty() + && value + .chars() + .all(|c| c.is_ascii_alphanumeric() || "_-./".contains(c)) + { + value.into() + } else { + format!("'{}'", value.replace('\'', "'\\''")) } } +/// Render the receipt as JSON or an actionable human-readable footer. pub fn print_init_receipt(receipt: &InitReceipt, global: &GlobalOptions) { if global.output == OutputFormat::Json { if let Ok(json) = serde_json::to_string_pretty(receipt) { @@ -89,30 +157,29 @@ pub fn print_init_receipt(receipt: &InitReceipt, global: &GlobalOptions) { } return; } - if global.quiet { return; } - - // Progressive wizard/plain already printed per-step outcomes — footer only. - if receipt.core_skipped { - println!( - "Hint: am instance start or am connect --project {}", - receipt.project_name - ); - } else if !receipt.core_running { - println!("Hint: run `am instance status` if ingest fails"); - } - if !receipt.core_skipped && !receipt.cloud_connection_online { - println!("Hint: Cloud connection pending — run `am connect doctor`"); + match receipt.verification_status { + VerificationStatus::Failed => { + if let Some(error) = &receipt.verification_error { + println!("Memory pipeline verification failed: {error}"); + } + } + VerificationStatus::DeliberatelySkipped => { + println!("Memory pipeline verification was deliberately skipped."); + } + VerificationStatus::NotRun => { + println!("Memory pipeline verification has not run; prerequisites are incomplete."); + } + VerificationStatus::Passed => {} } - if !receipt.core_skipped && !receipt.memory_pipeline_verified && !receipt.verification_skipped { - println!("Hint: Memory pipeline not verified — run `am doctor --smoke`"); + if !receipt.core_running { + println!("Core is not running; start the selected profile's instance."); } - if receipt.verification_skipped { - println!("Hint: Memory pipeline verification was skipped — run `am doctor --smoke`"); + if !receipt.cloud_connection_online { + println!("Cloud connection is not online; onboarding remains incomplete."); } - println!(); if let Some(url) = &receipt.dashboard_url { println!("Dashboard: {url}"); @@ -124,49 +191,126 @@ pub fn print_init_receipt(receipt: &InitReceipt, global: &GlobalOptions) { mod tests { use super::*; - #[test] - fn human_receipt_is_footer_only() { - let receipt = build_init_receipt(InitReceiptInput { - signed_in_as: Some("user@example.com"), + fn input() -> InitReceiptInput<'static> { + InitReceiptInput { + signed_in_as: None, org_name: "Personal", org_id: "org_1", project_name: "local", project_id: "proj_1", + profile_name: "connected-local", local_url: "http://127.0.0.1:17350", api_base_url: "https://api.atomicstrata.ai", core_healthy: true, no_instance: false, cloud_connection_online: true, credential_ready: true, - smoke: None, - }); - assert!(receipt.verification_skipped); - assert!(!receipt.memory_pipeline_verified); + verification: VerificationAttempt::Passed(SmokeResult { + verified: true, + mode: crate::verification::smoke::SmokeMode::Quick, + facts_extracted: 0, + ingest_trace_id: None, + memory_ids_cleaned: vec![], + marker: "test".into(), + }), + } + } + + #[test] + fn activation_requires_credentials_core_and_heartbeat() { + assert!(build_init_receipt(input()).activated); + for missing in 0..3 { + let mut input = input(); + match missing { + 0 => input.credential_ready = false, + 1 => input.core_healthy = false, + _ => input.cloud_connection_online = false, + } + let receipt = build_init_receipt(input); + assert!(!receipt.activated); + assert_eq!( + receipt.next_command, + "am --profile connected-local doctor --smoke" + ); + } + } + + #[test] + fn absence_of_attempt_is_not_deliberate_skip() { + let mut input = input(); + input.verification = VerificationAttempt::NotRun; + let receipt = build_init_receipt(input); + assert!(!receipt.verification_skipped); + assert_eq!(receipt.verification_status, VerificationStatus::NotRun); + } + + #[test] + fn explicit_skip_is_distinct_from_failure_and_not_run() { + for (attempt, expected, skipped) in [ + ( + VerificationAttempt::DeliberatelySkipped, + "deliberately_skipped", + true, + ), + (VerificationAttempt::NotRun, "not_run", false), + ( + VerificationAttempt::Failed("extraction failed".into()), + "failed", + false, + ), + ] { + let mut input = input(); + input.verification = attempt; + let receipt = build_init_receipt(input); + let json = serde_json::to_value(&receipt).unwrap(); + assert_eq!(json["verification_status"], expected); + assert_eq!(receipt.verification_skipped, skipped); + assert!(!receipt.activated); + assert!(!receipt.next_command.contains("memory ingest")); + if expected == "failed" { + assert_eq!(json["verification_error"], "extraction failed"); + } + } + } + + #[test] + fn skipped_instance_is_not_runtime_ready() { + let mut input = input(); + input.core_healthy = false; + input.no_instance = true; + input.verification = VerificationAttempt::DeliberatelySkipped; + assert!(!build_init_receipt(input).runtime_ready); + } + + #[test] + fn completed_receipt_links_dashboard_and_uses_selected_profile() { + let receipt = build_init_receipt(input()); assert!( receipt .dashboard_url .as_ref() - .is_some_and(|url| url.contains("/overview")) + .unwrap() + .contains("/overview") + ); + assert_eq!(receipt.verification_status, VerificationStatus::Passed); + assert!( + receipt + .next_command + .starts_with("am --profile connected-local memory ingest") ); - assert!(receipt.next_command.contains("am memory ingest")); } #[test] - fn custom_api_base_url_omits_dashboard_link() { - let receipt = build_init_receipt(InitReceiptInput { - signed_in_as: None, - org_name: "Personal", - org_id: "org_1", - project_name: "local", - project_id: "proj_1", - local_url: "http://127.0.0.1:17350", - api_base_url: "https://custom.example.com", - core_healthy: true, - no_instance: false, - cloud_connection_online: true, - credential_ready: true, - smoke: None, - }); + fn recovery_quotes_profile_names_and_custom_api_omits_dashboard() { + let mut input = input(); + input.profile_name = "local 'project'"; + input.api_base_url = "https://custom.example.com"; + input.verification = VerificationAttempt::NotRun; + let receipt = build_init_receipt(input); assert!(receipt.dashboard_url.is_none()); + assert_eq!( + receipt.next_command, + "am --profile 'local '\\''project'\\''' doctor --smoke" + ); } } diff --git a/crates/cli/src/verification/smoke.rs b/crates/cli/src/verification/smoke.rs index b1c1ed7..f6796a7 100644 --- a/crates/cli/src/verification/smoke.rs +++ b/crates/cli/src/verification/smoke.rs @@ -1,4 +1,4 @@ -//! Ephemeral ingest → search → delete round-trip for onboarding verification. +//! Ephemeral extraction/ingest → embedding search → cleanup for onboarding verification. use std::time::Duration; @@ -6,23 +6,22 @@ use am_cloud_client::MemoryClient; use am_core_types::{CoreIngestRequest, CoreMemoryQuery, CoreSearchRequest}; use anyhow::{Context, Result, bail}; use serde::Serialize; +use tokio::time::Instant; use crate::cli::GlobalOptions; use crate::commands::client::{memory_client_for_profile, resolve_ctx}; +use crate::config::ProfileKind; use crate::telemetry::{ActivationEvent, capture_activation}; use crate::validation::with_operation_recovery; +/// Dedicated local-admin namespace for ephemeral CLI verification. pub const SMOKE_USER_ID: &str = "am-cli-smoke"; +/// Source tag used to isolate smoke retrieval. pub const SMOKE_SOURCE_SITE: &str = "am-cli-smoke"; -/// Backoff between search attempts while waiting for the ingested marker. -/// -/// Ingest and search are separate calls, and indexing is not guaranteed to be -/// synchronous, so a single immediate search can miss a memory that is about to -/// become retrievable. Bounded retry keeps the check honest — it still fails -/// when retrieval is genuinely broken — without failing onboarding on ordinary -/// indexing lag. Total added wait is under four seconds, and the entire retry -/// loop shares one `SmokeOptions::timeout` deadline. +const FULL_SMOKE_TIMEOUT: Duration = Duration::from_secs(120); +const CLEANUP_RESERVE: Duration = Duration::from_secs(15); +/// Bounded indexing-lag retries share the ingest/retrieval operation deadline. const SEARCH_RETRY_DELAYS: [Duration; 4] = [ Duration::from_millis(250), Duration::from_millis(500), @@ -30,23 +29,50 @@ const SEARCH_RETRY_DELAYS: [Duration; 4] = [ Duration::from_millis(2000), ]; +/// Successful pipeline evidence, emitted only after every cleanup succeeds. #[derive(Debug, Clone, Serialize)] pub struct SmokeResult { pub verified: bool, + pub mode: SmokeMode, + pub facts_extracted: i32, pub ingest_trace_id: Option, pub memory_ids_cleaned: Vec, pub marker: String, } +/// Pipeline exercised by an onboarding smoke. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum SmokeMode { + /// Existing OpenAI-compatible verbatim ingest and embedding retrieval. + Quick, + /// SLM extraction followed by embedding retrieval. + FullExtraction, +} + +/// Deadline and pipeline selected by the command boundary. #[derive(Debug, Clone, Copy)] pub struct SmokeOptions { + /// Total operation budget, including the reserved cleanup window. pub timeout: Duration, + pub mode: SmokeMode, } impl Default for SmokeOptions { fn default() -> Self { Self { timeout: Duration::from_secs(45), + mode: SmokeMode::Quick, + } + } +} + +impl SmokeOptions { + /// Full extraction with a bounded budget for local model inference. + pub fn full_extraction() -> Self { + Self { + timeout: FULL_SMOKE_TIMEOUT, + mode: SmokeMode::FullExtraction, } } } @@ -58,7 +84,7 @@ pub struct SmokeTelemetry { pub props: Option>, } -/// Create a tagged ephemeral memory, retrieve it, then delete all residue. +/// Create a tagged ephemeral memory, retrieve it, then delete all known residue. pub async fn run_memory_smoke( global: &GlobalOptions, opts: SmokeOptions, @@ -67,16 +93,34 @@ pub async fn run_memory_smoke( let profile = resolve_ctx(global) .await .context("resolve profile for smoke test")?; - // Building the client is where a missing or withheld credential surfaces. - // Returning that straight through `?` gave it only generic context, so the - // profile-aware playbook this function installs below never applied to the - // most likely failure. - let client = memory_client_for_profile(&profile) - .await - .map_err(|err| with_operation_recovery(err, "Memory smoke client", profile.kind))?; + let client = memory_client_for_profile(&profile).await.map_err(|err| { + smoke_recovery( + err, + "Memory smoke client", + profile.kind, + &profile.name, + opts.mode, + ) + })?; run_memory_smoke_with_client(client, opts, telemetry) .await - .map_err(|err| with_operation_recovery(err, "Memory smoke", profile.kind)) + .map_err(|err| smoke_recovery(err, "Memory smoke", profile.kind, &profile.name, opts.mode)) +} + +fn smoke_recovery( + err: anyhow::Error, + operation: &str, + kind: ProfileKind, + profile: &str, + mode: SmokeMode, +) -> anyhow::Error { + if mode == SmokeMode::Quick { + return with_operation_recovery(err, operation, kind); + } + let profile = super::receipt::shell_argument(profile); + anyhow::anyhow!( + "{operation} failed: {err:#}\n\nCheck the selected profile's SLM extraction and embedding services, then run `am --profile {profile} doctor --smoke`." + ) } async fn run_memory_smoke_with_client( @@ -84,13 +128,21 @@ async fn run_memory_smoke_with_client( opts: SmokeOptions, telemetry: Option, ) -> Result { - let marker = format!("am-cli-smoke-{}", uuid_like_marker()); - - let ingest_req = smoke_ingest_request(&marker); - - let ingest = tokio::time::timeout(opts.timeout, client.ingest_quick(&ingest_req)) - .await - .context("smoke ingest timed out")??; + let deadline = Instant::now() + opts.timeout; + // Reserve cleanup time inside the single overall budget so failed or timed-out + // retrieval still gets a bounded chance to remove all known ingested records. + let pipeline_deadline = deadline - CLEANUP_RESERVE.min(opts.timeout / 4); + let client = client.with_timeout(opts.timeout)?; + let marker = format!("am-cli-smoke-{}", uuid::Uuid::now_v7()); + let ingest_req = smoke_ingest_request(&marker, opts.mode); + let ingest = tokio::time::timeout_at(pipeline_deadline, async { + match opts.mode { + SmokeMode::Quick => client.ingest_quick(&ingest_req).await, + SmokeMode::FullExtraction => client.ingest(&ingest_req).await, + } + }) + .await + .context("smoke ingest timed out; storage may have occurred without a response")??; if let Some(tel) = telemetry.as_ref() { capture_activation( @@ -99,15 +151,59 @@ async fn run_memory_smoke_with_client( tel.no_telemetry, ); } - - let mut memory_ids = ingest.stored_memory_ids.clone(); - if memory_ids.is_empty() && !ingest.updated_memory_ids.is_empty() { - memory_ids = ingest.updated_memory_ids.clone(); + let mut memory_ids = Vec::new(); + for id in ingest + .stored_memory_ids + .iter() + .chain(&ingest.updated_memory_ids) + { + if !memory_ids.contains(id) { + memory_ids.push(id.clone()); + } } + let retrieval = if opts.mode == SmokeMode::FullExtraction && ingest.facts_extracted <= 0 { + Err(anyhow::anyhow!( + "smoke extraction failed — no facts extracted" + )) + } else if memory_ids.is_empty() { + Err(anyhow::anyhow!( + "smoke ingest returned no memory IDs; extraction and cleanup cannot be verified" + )) + } else { + tokio::time::timeout_at( + pipeline_deadline, + retrieve_marker(&client, &marker, &memory_ids, opts.mode), + ) + .await + .map_err(|_| anyhow::anyhow!("smoke search timed out")) + .and_then(|result| result) + }; + let cleanup = cleanup_memories(&client, &memory_ids, deadline).await; + let cleaned = match (retrieval, cleanup) { + (Ok(()), Ok(cleaned)) => cleaned, + (Err(retrieval), Ok(_)) => return Err(retrieval), + (Ok(()), Err(cleanup)) => return Err(cleanup), + (Err(retrieval), Err(cleanup)) => bail!("{retrieval:#}; {cleanup:#}"), + }; + Ok(SmokeResult { + verified: true, + mode: opts.mode, + facts_extracted: ingest.facts_extracted, + ingest_trace_id: ingest.ingest_trace_id, + memory_ids_cleaned: cleaned, + marker, + }) +} +async fn retrieve_marker( + client: &MemoryClient, + marker: &str, + memory_ids: &[String], + mode: SmokeMode, +) -> Result<()> { let search_req = CoreSearchRequest { user_id: SMOKE_USER_ID.into(), - query: marker.clone(), + query: marker.into(), limit: Some(5), threshold: None, token_budget: None, @@ -122,90 +218,75 @@ async fn run_memory_smoke_with_client( namespace_scope: None, config_override: None, }; - - // One overall deadline for the whole retry loop. A per-attempt timeout - // would let a slow-but-alive backend consume timeout × attempts (minutes) - // where a single attempt used to fail at `opts.timeout`; the backoff - // schedule exists for indexing lag, not for a degraded backend. - let retrieval: Result = tokio::time::timeout(opts.timeout, async { - let mut attempt = 0usize; - loop { - let search = client.search_fast(&search_req).await?; - - if search - .memories - .iter() - .any(|hit| hit.memory.content.contains(&marker)) - { - return Ok(true); - } - - let Some(delay) = SEARCH_RETRY_DELAYS.get(attempt) else { - return Ok(false); - }; - tokio::time::sleep(*delay).await; - attempt += 1; + let mut attempt = 0usize; + loop { + // Fast search still embeds the query; it omits only optional LLM stages. + let search = client.search_fast(&search_req).await?; + if search.memories.iter().any(|hit| { + memory_ids.contains(&hit.memory.id) + && (mode == SmokeMode::FullExtraction || hit.memory.content.contains(marker)) + }) { + return Ok(()); } - }) - .await - .map_err(|_| anyhow::anyhow!("smoke search timed out")) - .and_then(|result| result); + let Some(delay) = SEARCH_RETRY_DELAYS.get(attempt) else { + bail!("smoke verification failed — ingested memory not retrieved"); + }; + tokio::time::sleep(*delay).await; + attempt += 1; + } +} - // Clean up before reporting the outcome. The ingested memory exists - // whether or not retrieval worked, so returning the verification error - // first would leave the smoke marker behind in the user's Core. +async fn cleanup_memories( + client: &MemoryClient, + memory_ids: &[String], + deadline: Instant, +) -> Result> { let query = CoreMemoryQuery { user_id: SMOKE_USER_ID.into(), workspace_id: None, agent_id: None, }; - let mut cleaned = Vec::new(); - for id in &memory_ids { - if client.delete_memory(id, &query).await.is_ok() { - cleaned.push(id.clone()); + let mut failures = Vec::new(); + for id in memory_ids { + match tokio::time::timeout_at(deadline, client.delete_memory(id, &query)).await { + Ok(Ok(result)) if result.deleted => cleaned.push(id.clone()), + Ok(Ok(_)) => failures.push(format!("{id}: deletion was not confirmed")), + Ok(Err(err)) => failures.push(format!("{id}: {err}")), + Err(_) => failures.push(format!("{id}: cleanup timed out")), } } - - let found = retrieval?; - if !found { - bail!("smoke verification failed — memory not retrieved (search returned no marker match)"); + if !failures.is_empty() { + bail!( + "smoke cleanup failed; remaining memory IDs: {}", + failures.join("; ") + ); } - - Ok(SmokeResult { - verified: found, - ingest_trace_id: ingest.ingest_trace_id, - memory_ids_cleaned: cleaned, - marker, - }) + Ok(cleaned) } -fn smoke_ingest_request(marker: &str) -> CoreIngestRequest { +fn smoke_ingest_request(marker: &str, mode: SmokeMode) -> CoreIngestRequest { + let quick = mode == SmokeMode::Quick; CoreIngestRequest { user_id: SMOKE_USER_ID.into(), source_site: SMOKE_SOURCE_SITE.into(), - conversation: format!("CLI onboarding smoke marker: {marker}"), + conversation: if quick { + format!("CLI onboarding smoke marker: {marker}") + } else { + format!("My preferred project codename is {marker}. Please remember this preference.") + }, agent_id: None, workspace_id: None, session_id: Some(SMOKE_USER_ID.into()), source_url: None, metadata: None, - skip_extraction: Some(true), - content_class: Some("summary".into()), + skip_extraction: Some(quick), + content_class: quick.then(|| "summary".into()), visibility: None, config_override: None, } } -fn uuid_like_marker() -> String { - use std::time::{SystemTime, UNIX_EPOCH}; - let nanos = SystemTime::now() - .duration_since(UNIX_EPOCH) - .map(|d| d.as_nanos()) - .unwrap_or(0); - format!("{nanos:x}") -} - #[cfg(test)] mod tests { use super::*; @@ -219,43 +300,28 @@ mod tests { #[test] fn smoke_ingest_request_stamps_verbatim_content_class() { - let req = smoke_ingest_request("marker-abc"); + let req = smoke_ingest_request("marker-abc", SmokeMode::Quick); assert_eq!(req.skip_extraction, Some(true)); assert_eq!(req.content_class.as_deref(), Some("summary")); assert!(req.conversation.contains("marker-abc")); } #[test] - fn smoke_wires_recovery_into_client_construction() { - // The formatter test below passes even if run_memory_smoke never calls - // it. Pin the wiring: the client-construction path must carry recovery - // text, which is where a missing credential actually fails. - let src = include_str!("smoke.rs"); - let body = src - .split("pub async fn run_memory_smoke(") - .nth(1) - .expect("run_memory_smoke present"); - let body = &body[..body.find("\nasync fn ").unwrap_or(body.len())]; - assert!( - body.contains("memory_client_for_profile"), - "client must be built from the resolved profile" - ); - assert_eq!( - body.matches("with_operation_recovery").count(), - 2, - "both client construction and the smoke run must install recovery" - ); - } - - #[test] - fn smoke_recovery_uses_client_profile_kind_without_re_resolve() { - let err = with_operation_recovery( - anyhow::anyhow!("http 401 unauthorized"), + fn full_extraction_recovery_uses_selected_profile_without_openai_advice() { + let err = smoke_recovery( + anyhow::anyhow!("authentication failed"), "Memory smoke", ProfileKind::Local, + "local project", + SmokeMode::FullExtraction, ); let msg = err.to_string(); - assert!(!msg.contains("am init --project")); - assert!(msg.contains("am instance")); + assert!(msg.contains("am --profile 'local project' doctor --smoke")); + assert!(msg.contains("SLM")); + assert!(!msg.contains("OPENAI_API_KEY")); } } + +#[cfg(test)] +#[path = "smoke_http_tests.rs"] +mod http_tests; diff --git a/crates/cli/src/verification/smoke_http_tests.rs b/crates/cli/src/verification/smoke_http_tests.rs new file mode 100644 index 0000000..0083f1a --- /dev/null +++ b/crates/cli/src/verification/smoke_http_tests.rs @@ -0,0 +1,249 @@ +//! HTTP contract tests for smoke extraction, retrieval, and residue cleanup. + +use super::*; +use axum::{ + Json, Router, + extract::State, + http::{Method, StatusCode, Uri}, + routing::any, +}; +use serde_json::{Value, json}; +use std::sync::{Arc, Mutex}; + +#[derive(Clone, Default)] +struct Scenario { + requests: Arc>>, + marker: Arc>, + cleanup_fails: bool, + search_fails: bool, + no_facts: bool, + duplicate_stored: bool, + paraphrased: bool, + search_pending: bool, + search_started: Arc, +} + +async fn handle( + State(scenario): State, + method: Method, + uri: Uri, + body: Option>, +) -> (StatusCode, Json) { + let body = body.map(|v| v.0).unwrap_or(Value::Null); + scenario + .requests + .lock() + .unwrap() + .push((method.clone(), uri.to_string(), body.clone())); + if uri.path().contains("ingest") { + *scenario.marker.lock().unwrap() = body["conversation"].as_str().unwrap().into(); + return ( + StatusCode::OK, + Json(json!({ + "episode_id":"episode", "facts_extracted": if scenario.no_facts {0} else {1}, + "stored_memory_ids": if scenario.duplicate_stored {vec!["stored", "updated", "stored"]} else {vec!["stored"]}, "updated_memory_ids":["updated", "stored"] + })), + ); + } + if method == Method::DELETE { + return ( + StatusCode::OK, + Json(json!({"deleted": !scenario.cleanup_fails})), + ); + } + if scenario.search_pending { + scenario.search_started.notify_one(); + std::future::pending::<()>().await; + } + if scenario.search_fails { + return ( + StatusCode::BAD_REQUEST, + Json(json!({"error":"search failed"})), + ); + } + ( + StatusCode::OK, + Json(json!({"memories":[{ + "id":"stored", "content": if scenario.paraphrased { "Remember the preferred project codename".into() } else {scenario.marker.lock().unwrap().clone()} + }]})), + ) +} + +async fn run(scenario: Scenario, opts: SmokeOptions) -> Result { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let router = Router::new().fallback(any(handle)).with_state(scenario); + let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); + let client = + MemoryClient::new(format!("http://{addr}").parse().unwrap(), "local-admin").unwrap(); + let result = run_memory_smoke_with_client(client, opts, None).await; + server.abort(); + result +} + +#[tokio::test] +async fn cleanup_includes_stored_and_updated_ids_without_duplicates() { + let scenario = Scenario::default(); + let result = run(scenario, SmokeOptions::default()).await.unwrap(); + assert_eq!(result.memory_ids_cleaned, ["stored", "updated"]); +} + +#[tokio::test] +async fn cleanup_rejection_fails_verification() { + let scenario = Scenario { + cleanup_fails: true, + ..Scenario::default() + }; + let result = run(scenario.clone(), SmokeOptions::default()).await; + assert!( + result.is_err(), + "failed cleanup must not yield verified=true" + ); + assert_eq!( + scenario + .requests + .lock() + .unwrap() + .iter() + .filter(|(m, _, _)| *m == Method::DELETE) + .count(), + 2 + ); +} + +#[tokio::test] +async fn retrieval_failure_still_cleans_all_ingested_ids() { + let scenario = Scenario { + search_fails: true, + ..Scenario::default() + }; + assert!( + run(scenario.clone(), SmokeOptions::default()) + .await + .is_err() + ); + assert_eq!( + scenario + .requests + .lock() + .unwrap() + .iter() + .filter(|(m, _, _)| *m == Method::DELETE) + .count(), + 2 + ); +} + +fn full_options() -> SmokeOptions { + SmokeOptions { + mode: SmokeMode::FullExtraction, + ..SmokeOptions::default() + } +} + +#[tokio::test] +async fn full_mode_exercises_extraction_before_embedding_retrieval() { + let scenario = Scenario::default(); + assert!( + run(scenario.clone(), full_options()) + .await + .unwrap() + .verified + ); + let requests = scenario.requests.lock().unwrap(); + assert_eq!(requests[0].1, "/v1/memories/ingest"); + assert_eq!(requests[0].2["skip_extraction"], false); + assert!(requests[0].2.get("content_class").is_none()); + assert_eq!(requests[1].1, "/v1/memories/search/fast"); + assert_eq!(requests[0].2["user_id"], "am-cli-smoke"); +} + +#[tokio::test] +async fn full_mode_rejects_zero_extracted_facts_and_cleans_residue() { + let scenario = Scenario { + no_facts: true, + ..Scenario::default() + }; + let result = run(scenario.clone(), full_options()).await; + assert!( + result.is_err(), + "successful HTTP response alone must not prove extraction" + ); + assert_eq!( + scenario + .requests + .lock() + .unwrap() + .iter() + .filter(|(m, _, _)| *m == Method::DELETE) + .count(), + 2 + ); +} + +#[tokio::test] +async fn nonadjacent_stored_duplicates_are_cleaned_only_once() { + let scenario = Scenario { + duplicate_stored: true, + ..Scenario::default() + }; + let result = run(scenario, SmokeOptions::default()).await.unwrap(); + assert_eq!(result.memory_ids_cleaned, ["stored", "updated"]); +} + +#[tokio::test] +async fn extraction_can_paraphrase_without_losing_retrieval_identity() { + let scenario = Scenario { + paraphrased: true, + ..Scenario::default() + }; + let result = run(scenario, full_options()).await.unwrap(); + assert!(result.verified); + assert_eq!(result.mode, SmokeMode::FullExtraction); + assert_eq!(result.facts_extracted, 1); +} + +#[tokio::test] +async fn combined_search_and_cleanup_failures_preserve_both_reasons() { + let scenario = Scenario { + search_fails: true, + cleanup_fails: true, + ..Scenario::default() + }; + let error = run(scenario, full_options()).await.unwrap_err().to_string(); + assert!(error.contains("search failed")); + assert!(error.contains("cleanup failed")); +} + +#[tokio::test] +async fn search_timeout_reserves_time_to_clean_known_memories() { + let scenario = Scenario { + search_pending: true, + ..Scenario::default() + }; + let opts = SmokeOptions { + timeout: Duration::from_secs(4), + ..full_options() + }; + let mut smoke = tokio::spawn(run(scenario.clone(), opts)); + tokio::select! { + () = scenario.search_started.notified() => {}, + result = &mut smoke => panic!("smoke ended before search: {result:?}"), + } + tokio::time::pause(); + tokio::time::advance(Duration::from_secs(3)).await; + tokio::time::resume(); + let error = smoke.await.unwrap().unwrap_err().to_string(); + assert!(error.contains("search timed out")); + assert!(!error.contains("cleanup failed")); + assert_eq!( + scenario + .requests + .lock() + .unwrap() + .iter() + .filter(|(m, _, _)| *m == Method::DELETE) + .count(), + 2 + ); +} diff --git a/crates/cli/src/version.rs b/crates/cli/src/version.rs new file mode 100644 index 0000000..d45f455 --- /dev/null +++ b/crates/cli/src/version.rs @@ -0,0 +1,293 @@ +//! CLI version identity and latest-version discovery (ATO-1844). +//! +//! `--version` prints the machine-readable contract documented in +//! `crates/cli/VERSION.md`. Latest published metadata is fetched from the +//! install mirror's `version.json` for the future upgrade gate (ATO-1843); +//! this module does not enforce upgrades. + +use std::sync::OnceLock; +use std::time::Duration; + +use anyhow::{Context, Result, bail}; +use serde::{Deserialize, Serialize}; + +/// Surface id for this binary in the cross-product version contract. +pub const SURFACE: &str = "cli"; + +/// Default public mirror that publishes latest CLI `version.json`. +pub const DEFAULT_LATEST_VERSION_BASE_URL: &str = "https://get.atomicstrata.ai"; + +/// Compile-time crate/workspace semver (`[workspace.package].version`). +pub const CRATE_VERSION: &str = env!("CARGO_PKG_VERSION"); + +/// Env var CI sets to the commit SHA when building release binaries. +#[allow(dead_code)] // referenced by stamp_env_vars / release docs +pub const GIT_SHA_ENV: &str = "AM_GIT_SHA"; + +/// Env var CI sets to the build channel (`production`, `internal`, `canary`, …). +#[allow(dead_code)] // referenced by stamp_env_vars / release docs +pub const BUILD_ENV_ENV: &str = "AM_BUILD_ENV"; + +/// Local/dev default when `AM_BUILD_ENV` is unset at compile time. +pub const DEFAULT_BUILD_ENV: &str = "dev"; + +/// Compile-time stamp env var names (for release docs / CI). +#[allow(dead_code)] // release docs / ATO-1843 +pub const fn stamp_env_vars() -> (&'static str, &'static str) { + (GIT_SHA_ENV, BUILD_ENV_ENV) +} + +const LATEST_FETCH_TIMEOUT: Duration = Duration::from_secs(10); + +/// Machine-readable identity printed by `am --version` / `am -V`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct VersionInfo { + pub surface: String, + pub version: String, + /// Commit SHA stamped at CI. JSON `null` when unset (local/dev builds). + #[serde(default)] + pub git_sha: Option, + pub env: String, +} + +/// Latest published CLI metadata from `{base}/version.json`. +/// +/// Extra fields (`tag`, …) are tolerated so installers can keep advertising +/// them without breaking the upgrade-gate parser. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct LatestVersionInfo { + pub surface: String, + pub version: String, + #[serde(default)] + pub git_sha: Option, + pub env: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tag: Option, +} + +impl VersionInfo { + /// Identity for this running binary. + pub fn current() -> Self { + Self { + surface: SURFACE.to_string(), + version: CRATE_VERSION.to_string(), + git_sha: compile_time_git_sha(), + env: compile_time_build_env().to_string(), + } + } + + /// Single-line JSON for clap's `--version` printer. + pub fn to_json_line(&self) -> String { + serde_json::to_string(self).expect("VersionInfo serializes") + } +} + +/// Clap `--version` string (JSON object, one line). Cached for `'static`. +pub fn clap_version_json() -> &'static str { + static JSON: OnceLock = OnceLock::new(); + JSON.get_or_init(|| VersionInfo::current().to_json_line()) +} + +/// Resolve `AM_GIT_SHA` at compile time. Empty/whitespace → `None` (never fake). +pub fn compile_time_git_sha() -> Option { + normalize_git_sha(option_env!("AM_GIT_SHA")) +} + +/// Resolve `AM_BUILD_ENV` at compile time; default `dev` when unset. +pub fn compile_time_build_env() -> &'static str { + match option_env!("AM_BUILD_ENV") { + Some(value) if !value.trim().is_empty() => value, + _ => DEFAULT_BUILD_ENV, + } +} + +/// Treat missing/blank/`unknown` as unset so local builds do not pretend a SHA. +pub fn normalize_git_sha(raw: Option<&str>) -> Option { + let value = raw?.trim(); + if value.is_empty() || value.eq_ignore_ascii_case("unknown") { + return None; + } + Some(value.to_string()) +} + +/// URL for the latest-version discovery document. +pub fn latest_version_url(base_url: &str) -> String { + format!("{}/version.json", base_url.trim().trim_end_matches('/')) +} + +/// Default public discovery URL for the latest CLI version document. +#[allow(dead_code)] // ATO-1843 discovery entrypoint +pub fn default_latest_version_url() -> String { + latest_version_url(DEFAULT_LATEST_VERSION_BASE_URL) +} + +/// Fetch and parse latest CLI version metadata (no upgrade enforcement). +#[allow(dead_code)] // ATO-1843 calls this; no enforcement in this ticket +pub async fn fetch_latest_version(base_url: &str) -> Result { + let url = latest_version_url(base_url); + let client = reqwest::Client::builder() + .user_agent(concat!("am/", env!("CARGO_PKG_VERSION"))) + .timeout(LATEST_FETCH_TIMEOUT) + .build() + .context("build HTTP client for latest version discovery")?; + let response = client + .get(&url) + .send() + .await + .with_context(|| format!("fetch latest version from {url}"))?; + if !response.status().is_success() { + bail!( + "latest version discovery returned HTTP {}", + response.status() + ); + } + let body = response + .text() + .await + .context("read latest version response body")?; + parse_latest_version_json(&body) +} + +/// Parse a `version.json` body into [`LatestVersionInfo`]. +pub fn parse_latest_version_json(body: &str) -> Result { + let info: LatestVersionInfo = + serde_json::from_str(body).context("parse version.json as LatestVersionInfo")?; + validate_latest_version(&info)?; + Ok(info) +} + +fn validate_latest_version(info: &LatestVersionInfo) -> Result<()> { + if info.surface != SURFACE { + bail!( + "version.json surface must be \"{SURFACE}\", got {:?}", + info.surface + ); + } + if info.version.trim().is_empty() { + bail!("version.json version must be non-empty"); + } + if info.env.trim().is_empty() { + bail!("version.json env must be non-empty"); + } + // Mirror the binary contract: never accept a faked placeholder SHA. + if let Some(sha) = info.git_sha.as_deref() { + if normalize_git_sha(Some(sha)).is_none() { + bail!("version.json gitSha is empty/unknown; omit the field or set a real SHA"); + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn current_uses_crate_semver_and_cli_surface() { + let info = VersionInfo::current(); + assert_eq!(info.surface, "cli"); + assert_eq!(info.version, CRATE_VERSION); + assert!(!info.version.is_empty()); + assert_eq!(info.env, compile_time_build_env()); + } + + #[test] + fn current_json_shape_is_stable() { + let value: serde_json::Value = + serde_json::from_str(&VersionInfo::current().to_json_line()).expect("json"); + assert_eq!(value["surface"], "cli"); + assert_eq!(value["version"], CRATE_VERSION); + assert!(value.get("env").and_then(|v| v.as_str()).is_some()); + assert!( + value.get("gitSha").is_some(), + "gitSha key must always be present, got {value}" + ); + // Local/dev CI without AM_GIT_SHA must not invent a SHA string. + if option_env!("AM_GIT_SHA").is_none() { + assert!( + value["gitSha"].is_null(), + "unset AM_GIT_SHA must serialize as null, got {value}" + ); + } + } + + #[test] + fn normalize_git_sha_rejects_empty_and_unknown() { + assert_eq!(normalize_git_sha(None), None); + assert_eq!(normalize_git_sha(Some("")), None); + assert_eq!(normalize_git_sha(Some(" ")), None); + assert_eq!(normalize_git_sha(Some("unknown")), None); + assert_eq!(normalize_git_sha(Some("UNKNOWN")), None); + assert_eq!( + normalize_git_sha(Some("abc123def")), + Some("abc123def".into()) + ); + } + + #[test] + fn clap_version_json_is_parseable_contract() { + let line = clap_version_json(); + let info: VersionInfo = serde_json::from_str(line).expect("clap version json"); + assert_eq!(info.surface, SURFACE); + assert_eq!(info.version, CRATE_VERSION); + } + + #[test] + fn stamp_env_vars_match_compile_time_names() { + let (git, env) = stamp_env_vars(); + assert_eq!(git, "AM_GIT_SHA"); + assert_eq!(env, "AM_BUILD_ENV"); + assert_eq!( + default_latest_version_url(), + "https://get.atomicstrata.ai/version.json" + ); + } + + #[test] + fn latest_version_url_joins_version_json() { + assert_eq!( + latest_version_url("https://get.atomicstrata.ai"), + "https://get.atomicstrata.ai/version.json" + ); + assert_eq!( + latest_version_url("https://get.atomicstrata.ai/"), + "https://get.atomicstrata.ai/version.json" + ); + } + + #[test] + fn parse_latest_version_json_accepts_contract_plus_tag() { + let body = r#"{"surface":"cli","version":"0.2.0","gitSha":"deadbeef","env":"production","tag":"cli-v0.2.0"}"#; + let info = parse_latest_version_json(body).expect("parse"); + assert_eq!(info.surface, "cli"); + assert_eq!(info.version, "0.2.0"); + assert_eq!(info.git_sha.as_deref(), Some("deadbeef")); + assert_eq!(info.env, "production"); + assert_eq!(info.tag.as_deref(), Some("cli-v0.2.0")); + } + + #[test] + fn parse_latest_version_json_allows_missing_git_sha() { + let body = r#"{"surface":"cli","version":"0.2.0","env":"production"}"#; + let info = parse_latest_version_json(body).expect("parse"); + assert_eq!(info.git_sha, None); + } + + #[test] + fn parse_latest_version_json_rejects_unknown_git_sha() { + let body = r#"{"surface":"cli","version":"0.2.0","gitSha":"unknown","env":"production"}"#; + let err = parse_latest_version_json(body).expect_err("unknown sha"); + assert!( + err.to_string().contains("gitSha") || err.to_string().contains("unknown"), + "{err:#}" + ); + } + + #[test] + fn parse_latest_version_json_rejects_wrong_surface() { + let body = r#"{"surface":"api","version":"0.2.0","env":"production"}"#; + assert!(parse_latest_version_json(body).is_err()); + } +} diff --git a/crates/cli/tests/connect_noninteractive.rs b/crates/cli/tests/connect_noninteractive.rs new file mode 100644 index 0000000..7c20eb6 --- /dev/null +++ b/crates/cli/tests/connect_noninteractive.rs @@ -0,0 +1,31 @@ +//! Public Connected Local auth gates must honor machine-output and prompt flags. + +#![cfg(unix)] + +use std::process::Command; + +#[test] +fn connect_requires_existing_auth_when_prompts_are_disabled() { + for (global_flags, flags) in [ + (vec![], vec!["--yes"]), + (vec!["-o", "json"], vec![]), + (vec!["--quiet"], vec![]), + (vec![], vec!["--yes", "--device"]), + ] { + let home = tempfile::tempdir().unwrap(); + let output = Command::new(env!("CARGO_BIN_EXE_am")) + .env("HOME", home.path()) + .env("XDG_DATA_HOME", home.path().join("data")) + .env("XDG_CONFIG_HOME", home.path().join("config")) + .args(["--no-telemetry", "--base-url", "http://127.0.0.1:9"]) + .args(&global_flags) + .args(["connect", "--project", "proj_test"]) + .args(&flags) + .output() + .unwrap(); + assert!(!output.status.success()); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(stderr.contains("sign-in required"), "{flags:?}: {stderr}"); + assert!(output.stdout.is_empty(), "unexpected login output"); + } +} diff --git a/crates/cli/tests/fixtures/am-slm-models-status-incomplete.json b/crates/cli/tests/fixtures/am-slm-models-status-incomplete.json new file mode 100644 index 0000000..03eb264 --- /dev/null +++ b/crates/cli/tests/fixtures/am-slm-models-status-incomplete.json @@ -0,0 +1,22 @@ +{ + "cache_dir": "/tmp/am-slm-fixture/hf-home", + "models": [ + { + "id": "Qwen/Qwen3-0.6B", + "ready": false, + "missing": ["weights"] + }, + { + "id": "nomic-ai/nomic-embed-text-v1.5", + "ready": true, + "missing": [] + } + ], + "adapters": [ + { + "id": "am-slm-core", + "ready": false, + "missing": ["adapter"] + } + ] +} diff --git a/crates/cli/tests/fixtures/am-slm-models-status.json b/crates/cli/tests/fixtures/am-slm-models-status.json new file mode 100644 index 0000000..0911c29 --- /dev/null +++ b/crates/cli/tests/fixtures/am-slm-models-status.json @@ -0,0 +1,22 @@ +{ + "cache_dir": "/tmp/am-slm-fixture/hf-home", + "models": [ + { + "id": "Qwen/Qwen3-0.6B", + "ready": true, + "missing": [] + }, + { + "id": "nomic-ai/nomic-embed-text-v1.5", + "ready": true, + "missing": [] + } + ], + "adapters": [ + { + "id": "am-slm-core", + "ready": true, + "missing": [] + } + ] +} diff --git a/crates/cli/tests/fixtures/am-slm-version.json b/crates/cli/tests/fixtures/am-slm-version.json new file mode 100644 index 0000000..f9baa38 --- /dev/null +++ b/crates/cli/tests/fixtures/am-slm-version.json @@ -0,0 +1,26 @@ +{ + "schema_version": 1, + "version": "0.1.1", + "tag": "am-slm-v0.1.1", + "source_sha": "774d5a03dd067cddc3b91000cc4473562ed3f7d0", + "supported_targets": [ + "aarch64-apple-darwin" + ], + "artifacts": [ + { + "target": "aarch64-apple-darwin", + "filename": "am-slm-0.1.1-aarch64-apple-darwin.tar.gz", + "url": "https://get.atomicstrata.ai/am-slm/v0.1.1/am-slm-0.1.1-aarch64-apple-darwin.tar.gz", + "sha256": "fdd26cd4a4d6952065f4c5941728a80b0dccbe91e88510381e76651f5b90b37d", + "size_bytes": 6374708 + } + ], + "min_disk_bytes": { + "binary_only": 52428800, + "with_models": 2684354560 + }, + "models_catalog_url": "https://get.atomicstrata.ai/am-slm/models/index.json", + "runtime_api_compat": "am-slm-openai-v1", + "license_id": "Apache-2.0", + "third_party_notices_file": "THIRD_PARTY_NOTICES.md" +} diff --git a/crates/cli/tests/integrate_opencode.rs b/crates/cli/tests/integrate_opencode.rs new file mode 100644 index 0000000..033f89c --- /dev/null +++ b/crates/cli/tests/integrate_opencode.rs @@ -0,0 +1,355 @@ +//! Public OpenCode integration lifecycle against isolated user configuration. + +#![cfg(unix)] + +use std::fs; +use std::os::unix::fs::PermissionsExt; +use std::path::PathBuf; +use std::process::{Command, Output}; + +use serde_json::{Value, json}; + +struct Fixture { + home: tempfile::TempDir, + config_home: PathBuf, + config_path: PathBuf, + bin_dir: PathBuf, +} + +impl Fixture { + fn new(config: &Value) -> Self { + let contents = format!("{}\n", serde_json::to_string_pretty(config).unwrap()); + Self::with_config("opencode.json", &contents) + } + + fn new_jsonc(config: &str) -> Self { + Self::with_config("opencode.jsonc", config) + } + + fn with_config(file_name: &str, contents: &str) -> Self { + let home = tempfile::tempdir().unwrap(); + let config_home = home.path().join("config"); + let opencode_dir = config_home.join("opencode"); + let config_path = opencode_dir.join(file_name); + let bin_dir = home.path().join("bin"); + fs::create_dir_all(&opencode_dir).unwrap(); + fs::create_dir_all(&bin_dir).unwrap(); + fs::write(&config_path, contents).unwrap(); + let npx = bin_dir.join("npx"); + fs::write(&npx, "#!/bin/sh\nexit 0\n").unwrap(); + fs::set_permissions(npx, fs::Permissions::from_mode(0o755)).unwrap(); + Self { + home, + config_home, + config_path, + bin_dir, + } + } + + fn run(&self, args: &[&str]) -> Output { + Command::new(env!("CARGO_BIN_EXE_am")) + .env("HOME", self.home.path()) + .env("XDG_CONFIG_HOME", &self.config_home) + .env("XDG_DATA_HOME", self.home.path().join("data")) + .env("PATH", &self.bin_dir) + .env("ATOMICMEMORY_API_KEY", "amc_test") + .env("USER", "test-user") + .args(["--no-telemetry", "-o", "json", "integrate"]) + .args(args) + .output() + .unwrap() + } + + fn run_success(&self, args: &[&str]) -> Output { + let output = self.run(args); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + output + } + + fn config(&self) -> Value { + jsonc_parser::parse_to_serde_value(&self.raw_config(), &Default::default()).unwrap() + } + + fn raw_config(&self) -> String { + fs::read_to_string(&self.config_path).unwrap() + } +} + +#[test] +fn opencode_install_update_doctor_and_uninstall_lifecycle() { + let fixture = Fixture::new(&json!({ + "model": "example/model", + "mcp": { "servers": { "other": { "type": "remote", "url": "https://example.com" } } } + })); + + fixture.run_success(&["--yes", "--global", "--host", "opencode"]); + let installed = fixture.config(); + assert_eq!(installed["model"], "example/model"); + assert_eq!(installed["mcp"]["servers"]["atomicmemory"]["type"], "local"); + assert_eq!( + installed["mcp"]["servers"]["atomicmemory"]["codemode"], + false + ); + let command = installed["mcp"]["servers"]["atomicmemory"]["command"] + .as_array() + .unwrap(); + assert_eq!(command[0], "npx"); + assert!( + command + .iter() + .any(|part| part == "@atomicmemory/mcp-server@0.1.5") + ); + assert!(command.iter().any(|part| part == "atomicmemory-mcp")); + assert_eq!(installed["mcp"]["servers"]["other"]["type"], "remote"); + + fixture.run_success(&["--yes", "--global", "--host", "opencode"]); + assert_eq!(fixture.config(), installed); + fixture.run_success(&["update", "--host", "opencode"]); + fixture.run_success(&["update", "--host", "opencode"]); + assert_eq!(fixture.config(), installed); + fixture.run_success(&["doctor", "--host", "opencode"]); + + fixture.run_success(&["uninstall", "--host", "opencode"]); + let removed = fixture.config(); + assert!(removed["mcp"]["servers"].get("atomicmemory").is_none()); + assert_eq!(removed["mcp"]["servers"]["other"]["type"], "remote"); + assert_eq!(removed["model"], "example/model"); +} + +#[test] +fn opencode_force_install_restores_prior_entry_on_uninstall() { + let prior = json!({ "type": "remote", "url": "https://prior.example.com/mcp" }); + let fixture = Fixture::new(&json!({ + "mcp": { "servers": { "atomicmemory": prior.clone() } } + })); + + let refused = fixture.run(&["--yes", "--host", "opencode"]); + assert!(!refused.status.success()); + assert_eq!(fixture.config()["mcp"]["servers"]["atomicmemory"], prior); + + fixture.run_success(&["--yes", "--host", "opencode", "--force"]); + assert_eq!( + fixture.config()["mcp"]["servers"]["atomicmemory"]["type"], + "local" + ); + + fixture.run_success(&["uninstall", "--host", "opencode"]); + assert_eq!(fixture.config()["mcp"]["servers"]["atomicmemory"], prior); +} + +#[test] +fn opencode_jsonc_comments_survive_install_and_uninstall() { + let fixture = Fixture::new_jsonc( + r#"{ + // Keep this explanation with the user's model. + "model": "example/model", + "mcp": { + "servers": { + "other": { "type": "remote", "url": "https://example.com/mcp" }, + }, + }, +} +"#, + ); + + fixture.run_success(&["--yes", "--host", "opencode"]); + assert!( + fixture + .raw_config() + .contains("// Keep this explanation with the user's model.") + ); + assert_eq!( + fixture.config()["mcp"]["servers"]["atomicmemory"]["type"], + "local" + ); + assert!(!fixture.config_path.with_file_name("opencode.json").exists()); + + fixture.run_success(&["uninstall", "--host", "opencode"]); + assert!( + fixture + .raw_config() + .contains("// Keep this explanation with the user's model.") + ); + let removed = fixture.config(); + assert!(removed["mcp"]["servers"].get("atomicmemory").is_none()); + assert_eq!(removed["mcp"]["servers"]["other"]["type"], "remote"); +} + +#[test] +fn opencode_refuses_managed_entry_in_sibling_global_file() { + let fixture = Fixture::new(&json!({ "model": "example/model" })); + let sibling = fixture.config_path.with_file_name("config.json"); + fs::write( + &sibling, + serde_json::to_string_pretty(&json!({ + "mcp": { "servers": { "atomicmemory": { + "type": "remote", "url": "https://sibling.example.com/mcp" + } } } + })) + .unwrap(), + ) + .unwrap(); + let before = fixture.raw_config(); + + let output = fixture.run(&["--yes", "--host", "opencode", "--force"]); + + assert!(!output.status.success()); + assert_eq!(fixture.raw_config(), before); + assert!(String::from_utf8_lossy(&output.stdout).contains("config.json")); +} + +#[test] +fn opencode_operations_stay_on_recorded_install_file() { + let fixture = Fixture::new(&json!({ "model": "example/model" })); + fixture.run_success(&["--yes", "--host", "opencode"]); + let jsonc = fixture.config_path.with_file_name("opencode.jsonc"); + fs::write(&jsonc, "{\n // User-created later.\n}\n").unwrap(); + + fixture.run_success(&["update", "--host", "opencode"]); + fixture.run_success(&["uninstall", "--host", "opencode"]); + + assert!( + fixture.config()["mcp"]["servers"] + .get("atomicmemory") + .is_none() + ); + assert_eq!( + fs::read_to_string(jsonc).unwrap(), + "{\n // User-created later.\n}\n" + ); +} + +#[test] +fn opencode_operations_follow_recorded_file_rename() { + let fixture = Fixture::new(&json!({ "model": "example/model" })); + fixture.run_success(&["--yes", "--host", "opencode"]); + let renamed = fixture.config_path.with_file_name("opencode.jsonc"); + fs::rename(&fixture.config_path, &renamed).unwrap(); + + fixture.run_success(&["doctor", "--host", "opencode"]); + fixture.run_success(&["update", "--host", "opencode"]); + fixture.run_success(&["uninstall", "--host", "opencode"]); + + let raw = fs::read_to_string(renamed).unwrap(); + let config: Value = jsonc_parser::parse_to_serde_value(&raw, &Default::default()).unwrap(); + assert!(config["mcp"]["servers"].get("atomicmemory").is_none()); + assert_eq!(config["model"], "example/model"); +} + +#[test] +fn opencode_uninstall_follows_recorded_file_rename_without_update() { + let fixture = Fixture::new(&json!({ "model": "example/model" })); + fixture.run_success(&["--yes", "--host", "opencode"]); + let renamed = fixture.config_path.with_file_name("opencode.jsonc"); + fs::rename(&fixture.config_path, &renamed).unwrap(); + + fixture.run_success(&["uninstall", "--host", "opencode"]); + + let raw = fs::read_to_string(renamed).unwrap(); + let config: Value = jsonc_parser::parse_to_serde_value(&raw, &Default::default()).unwrap(); + assert!(config["mcp"]["servers"].get("atomicmemory").is_none()); + assert_eq!(config["model"], "example/model"); +} + +#[test] +fn opencode_duplicate_entries_fail_install_and_force_uninstall_removes_all() { + let fixture = Fixture::new_jsonc( + r#"{ + "mcp": { "servers": { + "atomicmemory": { "type": "remote", "url": "https://first.example" }, + "atomicmemory": { "type": "remote", "url": "https://second.example" }, + } }, +} +"#, + ); + + let install = fixture.run(&["--yes", "--host", "opencode", "--force"]); + assert!(!install.status.success()); + fixture.run_success(&["uninstall", "--host", "opencode", "--force"]); + assert_eq!(fixture.raw_config().matches("\"atomicmemory\"").count(), 0); +} + +#[test] +fn opencode_force_uninstall_removes_entry_hidden_by_duplicate_parent() { + let fixture = Fixture::new_jsonc( + r#"{ + "mcp": { "servers": { + "atomicmemory": { "type": "remote", "url": "https://hidden.example" }, + } }, + "mcp": { "servers": {} }, +} +"#, + ); + + let refused = fixture.run(&["uninstall", "--host", "opencode"]); + assert!(!refused.status.success()); + fixture.run_success(&["uninstall", "--host", "opencode", "--force"]); + assert_eq!(fixture.raw_config().matches("\"atomicmemory\"").count(), 0); +} + +#[test] +fn integrate_list_survives_unreadable_owned_opencode_config() { + let fixture = Fixture::new(&json!({})); + fixture.run_success(&["--yes", "--host", "opencode"]); + fs::write(&fixture.config_path, "{ invalid jsonc").unwrap(); + + let output = fixture.run_success(&["list"]); + let report: Value = serde_json::from_slice(&output.stdout).unwrap(); + let opencode = report["installs"] + .as_array() + .unwrap() + .iter() + .find(|row| row["host"] == "opencode") + .unwrap(); + assert_eq!(opencode["owned"], true); + assert_eq!(opencode["fingerprint_match"], false); +} + +#[test] +fn opencode2_binary_is_detected_without_config() { + let home = tempfile::tempdir().unwrap(); + let config_home = home.path().join("config"); + let bin_dir = home.path().join("bin"); + fs::create_dir_all(&bin_dir).unwrap(); + let opencode2 = bin_dir.join("opencode2"); + fs::write(&opencode2, "#!/bin/sh\nexit 0\n").unwrap(); + fs::set_permissions(opencode2, fs::Permissions::from_mode(0o755)).unwrap(); + + let output = Command::new(env!("CARGO_BIN_EXE_am")) + .env("HOME", home.path()) + .env("XDG_CONFIG_HOME", &config_home) + .env("XDG_DATA_HOME", home.path().join("data")) + .env("PATH", &bin_dir) + .args([ + "--no-telemetry", + "-o", + "json", + "integrate", + "detect", + "--host", + "opencode", + ]) + .output() + .unwrap(); + + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + let report: Value = serde_json::from_slice(&output.stdout).unwrap(); + assert_eq!(report["hosts"][0]["detected"], true); + assert!( + report["hosts"][0]["signals"] + .as_array() + .unwrap() + .iter() + .any(|signal| signal + .as_str() + .is_some_and(|value| value.contains("opencode2"))) + ); +} diff --git a/crates/cli/tests/slm_models_progress.rs b/crates/cli/tests/slm_models_progress.rs new file mode 100644 index 0000000..f17495b --- /dev/null +++ b/crates/cli/tests/slm_models_progress.rs @@ -0,0 +1,116 @@ +//! Public model-pull output and cache recovery contracts with a fake runtime. + +#![cfg(unix)] + +use std::fs; +use std::os::unix::fs::PermissionsExt; +use std::path::PathBuf; +use std::process::{Command, Output}; + +struct Fixture { + home: tempfile::TempDir, + root: PathBuf, +} + +impl Fixture { + fn new(fail: bool) -> Self { + let home = tempfile::tempdir().unwrap(); + let root = if cfg!(target_os = "macos") { + home.path() + .join("Library/Application Support/ai.atomicstrata.atomicmemory/slm") + } else { + home.path().join("data/atomicmemory/slm") + }; + fs::create_dir_all(root.join("bin")).unwrap(); + let status = include_str!("fixtures/am-slm-models-status.json"); + let script = format!( + r#"#!/bin/sh +if [ "$2" = status ]; then + printf '%s\n' '{status}' + exit 0 +fi +if [ "$1" != models ] || [ "$2" != pull ] || [ "$3" != --json ]; then exit 19; fi +printf '%s\n' '{{"event":"model_start","model_id":"qwen"}}' >&2 +printf '%s\n' '{{"event":"file_cached","model_id":"qwen","file":"config.json","path":"private-cache-path"}}' >&2 +printf '%s\n' '{{"event":"file_start","model_id":"qwen","file":"weights","bytes_total":33554432}}' >&2 +printf '%s\n' '{{"event":"file_progress","model_id":"qwen","file":"weights","bytes":8388608}}' >&2 +printf '%s\n' 'child stdout must never leak' +printf '%s\n' 'partial bytes' > "$HF_HOME/weights.partial" +exit {exit} +"#, + exit = if fail { 7 } else { 0 }, + ); + let binary = root.join("bin/am-slm"); + fs::write(&binary, script).unwrap(); + fs::set_permissions(binary, fs::Permissions::from_mode(0o755)).unwrap(); + Self { home, root } + } + + fn pull(&self, flags: &[&str]) -> Output { + Command::new(env!("CARGO_BIN_EXE_am")) + .env("HOME", self.home.path()) + .env("XDG_DATA_HOME", self.home.path().join("data")) + .env("XDG_CONFIG_HOME", self.home.path().join("config")) + .env("AM_NO_TELEMETRY", "1") + .arg("--no-telemetry") + .args(flags) + .args(["slm", "models", "pull", "--yes"]) + .output() + .unwrap() + } +} + +#[test] +fn json_pull_is_one_valid_result_without_child_stdout() { + let fixture = Fixture::new(false); + let output = fixture.pull(&["-o", "json"]); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + let value: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap(); + assert_eq!(value["ok"], true); + assert!(!String::from_utf8_lossy(&output.stderr).contains("file_progress")); +} + +#[test] +fn quiet_pull_suppresses_child_and_reporter_output() { + let fixture = Fixture::new(false); + let output = fixture.pull(&["--quiet"]); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + assert!(output.stdout.is_empty()); + assert!(output.stderr.is_empty()); +} + +#[test] +fn plain_pull_reports_known_bytes_and_cache_hits() { + let fixture = Fixture::new(false); + let output = fixture.pull(&[]); + assert!(output.status.success()); + let stderr = String::from_utf8(output.stderr).unwrap(); + assert!(stderr.contains("qwen / config.json — cached")); + assert!(stderr.contains("8.0 MiB / 32.0 MiB")); + assert!(!stderr.contains("private-cache-path")); + assert!(!stderr.contains('\u{1b}')); +} + +#[test] +fn failed_pull_keeps_partial_cache_and_gives_retry() { + let fixture = Fixture::new(true); + let output = fixture.pull(&[]); + assert!(!output.status.success()); + assert_eq!( + fs::read_to_string(fixture.root.join("hf-home/weights.partial")).unwrap(), + "partial bytes\n" + ); + let stderr = String::from_utf8(output.stderr).unwrap(); + assert!(stderr.contains("am slm models pull --yes")); + assert!(stderr.contains("retained")); + assert!(stderr.contains("qwen")); + assert!(!String::from_utf8_lossy(&output.stdout).contains("child stdout")); +} diff --git a/crates/cloud-client/src/client.rs b/crates/cloud-client/src/client.rs index 20fae97..a1f8e18 100644 --- a/crates/cloud-client/src/client.rs +++ b/crates/cloud-client/src/client.rs @@ -258,6 +258,13 @@ impl MemoryClient { }) } + /// Set an explicit request timeout for workloads such as local model extraction. + pub fn with_timeout(self, timeout: std::time::Duration) -> Result { + Ok(Self { + transport: self.transport.with_timeout(timeout)?, + }) + } + pub fn base_url(&self) -> &Url { self.transport.base_url() } diff --git a/crates/cloud-client/src/error.rs b/crates/cloud-client/src/error.rs index 9d455ec..d2cb31e 100644 --- a/crates/cloud-client/src/error.rs +++ b/crates/cloud-client/src/error.rs @@ -16,11 +16,14 @@ pub enum CloudClientError { #[error("invalid path `{path}`: {message}")] InvalidPath { path: String, message: String }, - #[error("authentication failed (401/403)")] + #[error("authentication failed (401)")] Auth, + #[error("forbidden (403; {code})")] + Forbidden { code: String }, + #[error( - "session has no active organization — run `am auth login` with an org selected, or `am init`" + "session has no active organization (403; no_active_organization) — run `am auth login` with an org selected, or `am init`" )] NoActiveOrganization, @@ -38,27 +41,46 @@ pub enum CloudClientError { } impl CloudClientError { + /// Classify a JSON server error while redacting its diagnostic excerpt. pub fn from_status(code: u16, body: serde_json::Value) -> Self { + Self::from_response_body(code, &body.to_string()) + } + + /// Classify JSON or plain-text HTTP failures without discarding server diagnostics. + pub fn from_response_body(code: u16, raw: &str) -> Self { + let body = serde_json::from_str::(raw).ok(); let error_code = body - .get("error") + .as_ref() + .and_then(|body| body.get("error")) .and_then(|e| e.get("code")) .and_then(|c| c.as_str()); if code == 403 && error_code == Some("no_active_organization") { return Self::NoActiveOrganization; } - if code == 401 || code == 403 { + if code == 401 { return Self::Auth; } + if code == 403 { + return Self::Forbidden { + code: crate::redact::error_excerpt(error_code.unwrap_or("forbidden")), + }; + } + let excerpt = crate::redact::error_excerpt(raw); Self::Status { code, - body: crate::redact::redact_secrets(&body.to_string()), + body: match error_code { + Some(error_code) => { + format!("{}: {excerpt}", crate::redact::error_excerpt(error_code)) + } + None => excerpt, + }, } } /// Exit code category for CLI scripting (see `am` README). pub fn exit_code(&self) -> i32 { match self { - Self::Auth | Self::NoActiveOrganization => 2, + Self::Auth | Self::Forbidden { .. } | Self::NoActiveOrganization => 2, Self::Timeout | Self::Network(_) => 3, Self::Status { .. } => 4, Self::InvalidBaseUrl(_) @@ -75,6 +97,17 @@ mod tests { use super::*; use serde_json::json; + #[test] + fn status_preserves_code_even_when_the_excerpt_is_truncated() { + let body = format!( + r#"{{"detail":"{}","error":{{"code":"validation_error"}}}}"#, + "x".repeat(2000) + ); + let error = CloudClientError::from_response_body(422, &body); + assert!(error.to_string().contains("validation_error")); + assert!(error.to_string().len() < 1500); + } + #[test] fn from_status_maps_no_active_organization() { let err = CloudClientError::from_status( @@ -82,11 +115,24 @@ mod tests { json!({"error": {"code": "no_active_organization", "message": "session has no active organization"}}), ); assert!(matches!(err, CloudClientError::NoActiveOrganization)); + assert!(err.to_string().contains("403")); + assert!(err.to_string().contains("no_active_organization")); } #[test] - fn from_status_maps_generic_forbidden_to_auth() { + fn from_status_maps_generic_forbidden() { let err = CloudClientError::from_status(403, json!({"error": {"code": "forbidden"}})); + assert!(matches!( + err, + CloudClientError::Forbidden { ref code } if code == "forbidden" + )); + assert!(err.to_string().contains("forbidden")); + assert!(err.to_string().contains("403")); + } + + #[test] + fn from_status_maps_401_to_auth() { + let err = CloudClientError::from_status(401, json!({"error": {"code": "unauthorized"}})); assert!(matches!(err, CloudClientError::Auth)); } } diff --git a/crates/cloud-client/src/redact.rs b/crates/cloud-client/src/redact.rs index a9d210e..e33f720 100644 --- a/crates/cloud-client/src/redact.rs +++ b/crates/cloud-client/src/redact.rs @@ -18,12 +18,25 @@ fn find_prefix_ignore_ascii_case(haystack: &str, prefix: &str, from: usize) -> O }) } -/// Strip bearer tokens and `amc_*` keys from a string for safe logging/display. +/// Redact bearer tokens, API keys, JWTs, secret assignments, and email addresses. pub fn redact_secrets(input: &str) -> String { + match serde_json::from_str::(input) { + Ok(mut value) => { + redact_fields(&mut value); + redact_text(&value.to_string()) + } + Err(_) if input.trim_start().starts_with(['{', '[']) => { + "malformed or truncated JSON error response".into() + } + Err(_) => redact_text(input), + } +} + +fn redact_text(input: &str) -> String { let mut out = input.to_string(); // Prefixes are ASCII, so matching is case-insensitive over ASCII only and // every match index lands on a char boundary. - for prefix in ["Bearer ", "amc_"] { + for prefix in ["Bearer ", "amc_", "eyJ"] { let mut search_from = 0; while let Some(idx) = find_prefix_ignore_ascii_case(&out, prefix, search_from) { let token_start = idx + prefix.len(); @@ -37,13 +50,163 @@ pub fn redact_secrets(input: &str) -> String { search_from = token_start + REDACTED.len(); } } - out + redact_assignments(&mut out); + out.split_inclusive(char::is_whitespace) + .map(|word| { + if word.contains('@') { + format!("{REDACTED}{}", &word[word.trim_end().len()..]) + } else { + word.to_string() + } + }) + .collect() +} + +fn redact_assignments(out: &mut String) { + for name in [ + "access_token", + "refresh_token", + "api_key", + "secret", + "password", + "authorization", + "cookie", + ] { + let mut from = 0; + while let Some(index) = find_prefix_ignore_ascii_case(out, name, from) { + let after_name = index + name.len(); + from = after_name; + let tail = out[after_name..].trim_start_matches(['\"', '\'', ' ']); + if !tail.starts_with([':', '=']) { + continue; + } + let value = tail[1..].trim_start(); + let quote = value.chars().next().filter(|c| matches!(c, '\"' | '\'')); + let value = if quote.is_some() { &value[1..] } else { value }; + let start = out.len() - value.len(); + let end = value + .find(|c: char| match quote { + Some(quote) => c == quote, + None => c.is_whitespace() || matches!(c, '\"' | '\'' | ',' | '}' | ';'), + }) + .map_or(out.len(), |length| start + length); + out.replace_range(start..end, REDACTED); + from = start + REDACTED.len(); + } + } +} + +/// Bound user-visible server diagnostics and redact structured secret fields. +pub(crate) fn error_excerpt(input: &str) -> String { + const MAX_ERROR_CHARS: usize = 1024; + let redacted = redact_secrets(input); + let mut excerpt: String = redacted.chars().take(MAX_ERROR_CHARS).collect(); + if redacted.chars().count() > MAX_ERROR_CHARS { + excerpt.push_str(" … [truncated]"); + } + excerpt +} + +fn redact_fields(value: &mut serde_json::Value) { + match value { + serde_json::Value::Object(fields) => { + for (name, value) in fields { + let name = name + .chars() + .filter(|ch| !matches!(ch, '_' | '-')) + .collect::() + .to_ascii_lowercase(); + if [ + "apikey", + "token", + "secret", + "password", + "authorization", + "cookie", + "email", + ] + .iter() + .any(|part| name.contains(part)) + { + *value = serde_json::Value::String(REDACTED.into()); + } else { + redact_fields(value); + } + } + } + serde_json::Value::Array(items) => items.iter_mut().for_each(redact_fields), + _ => {} + } } #[cfg(test)] mod tests { use super::*; + #[test] + fn public_redactor_protects_structured_subprocess_diagnostics() { + let input = r#"{"error":{"api\u004bey":["private-key",{"value":"another-key"}]},"message":"failed"}"#; + let redacted = redact_secrets(input); + assert!(!redacted.contains("private-key"), "{redacted}"); + assert!(!redacted.contains("another-key"), "{redacted}"); + let value: serde_json::Value = serde_json::from_str(&redacted).unwrap(); + assert_eq!(value["error"]["apiKey"], REDACTED); + assert_eq!(value["message"], "failed"); + } + + #[test] + fn redacts_api_key_fields_before_rendering_nested_values() { + let input = r#"{ + "message":"quota exceeded", + "apiKey":"camel-secret", + "nested":[ + {"API_KEY":["array-secret",{"value":"nested-secret"}]}, + {"api-key":{"value":"object-secret"}}, + {"a_p-i_K-e_y":123456789}, + {"api\u004bey":true} + ] + }"#; + let excerpt = error_excerpt(input); + for secret in [ + "camel-secret", + "array-secret", + "nested-secret", + "object-secret", + "123456789", + ] { + assert!(!excerpt.contains(secret), "leaked {secret}: {excerpt}"); + } + let value: serde_json::Value = serde_json::from_str(&excerpt).unwrap(); + assert_eq!(value["message"], "quota exceeded"); + assert_eq!(value["apiKey"], REDACTED); + for (index, key) in ["API_KEY", "api-key", "a_p-i_K-e_y", "apiKey"] + .into_iter() + .enumerate() + { + assert_eq!(value["nested"][index][key], REDACTED); + } + } + + #[test] + fn redacts_secrets_in_truncated_json_and_plain_text() { + for input in [ + r#"{"access_token":"private-token", "incomplete":"#, + "upstream ACCESS_TOKEN = private-token", + "upstream password = \"private-token multi-word-secret\" rejected", + "rejected eyJhbGciOiJIUzI1NiJ9.payload.signature for person@example.test", + ] { + let excerpt = error_excerpt(input); + for secret in [ + "private-token", + "multi-word-secret", + "payload.signature", + "person@example.test", + ] { + assert!(!excerpt.contains(secret), "leaked {secret}: {excerpt}"); + } + } + } + #[test] fn redacts_bearer_token() { let s = redact_secrets("Authorization: Bearer eyJhbGciOiJIUz"); diff --git a/crates/cloud-client/src/transport.rs b/crates/cloud-client/src/transport.rs index 034b399..83af09f 100644 --- a/crates/cloud-client/src/transport.rs +++ b/crates/cloud-client/src/transport.rs @@ -11,6 +11,7 @@ use crate::redact::redact_secrets; const USER_AGENT: &str = concat!("am-cloud-client/", env!("CARGO_PKG_VERSION")); const DEFAULT_TIMEOUT: Duration = Duration::from_secs(30); +const MAX_ERROR_BODY_BYTES: usize = 8192; #[derive(Clone)] pub struct HttpTransport { @@ -33,6 +34,15 @@ impl HttpTransport { }) } + /// Set an explicit request deadline for operations with a larger work budget. + pub fn with_timeout(mut self, timeout: Duration) -> Result { + self.http = reqwest::Client::builder() + .user_agent(USER_AGENT) + .timeout(timeout) + .build()?; + Ok(self) + } + pub fn base_url(&self) -> &Url { &self.base_url } @@ -136,28 +146,50 @@ impl HttpTransport { if e.is_timeout() { CloudClientError::Timeout } else { - CloudClientError::Network(redact_secrets(&e.to_string())) + CloudClientError::Network(redact_secrets(&e.without_url().to_string())) } })?; let status = resp.status(); - let bytes = resp.bytes().await?; let elapsed_ms = started.elapsed().as_millis() as u64; tracing::Span::current().record("status", status.as_u16()); tracing::Span::current().record("latency_ms", elapsed_ms); if !status.is_success() { - let body: serde_json::Value = - serde_json::from_slice(&bytes).unwrap_or(serde_json::Value::Null); + // Server bodies can contain arbitrary PII. Keep them out of tracing; + // expose only a bounded, redacted excerpt in the returned error. debug!( status = status.as_u16(), latency_ms = elapsed_ms, - body = %redact_secrets(&body.to_string()), "cloud request failed" ); - return Err(CloudClientError::from_status(status.as_u16(), body)); + let body = + read_error_body(resp) + .await + .map_err(|error| CloudClientError::Status { + code: status.as_u16(), + body: format!( + "{method} {}: could not read error response: {error}", + redact_secrets(path) + ), + })?; + let credential = self.auth_header.strip_prefix("Bearer ").unwrap_or(""); + let body = if credential.is_empty() { + body + } else { + body.replace(credential, "") + }; + let error = CloudClientError::from_response_body(status.as_u16(), &body); + return Err(match error { + CloudClientError::Status { code, body } => CloudClientError::Status { + code, + body: format!("{method} {}: {body}", redact_secrets(path)), + }, + error => error, + }); } + let bytes = resp.bytes().await?; let value: serde_json::Value = if bytes.is_empty() { serde_json::Value::Null } else { @@ -174,6 +206,26 @@ impl HttpTransport { } } +async fn read_error_body(mut response: reqwest::Response) -> Result { + let mut bytes = Vec::new(); + while let Some(chunk) = response.chunk().await.map_err(|error| { + if error.is_timeout() { + CloudClientError::Timeout + } else { + CloudClientError::Network(redact_secrets(&error.without_url().to_string())) + } + })? { + let remaining = MAX_ERROR_BODY_BYTES - bytes.len(); + bytes.extend_from_slice(&chunk[..chunk.len().min(remaining)]); + if bytes.len() == MAX_ERROR_BODY_BYTES { + let mut body = String::from_utf8_lossy(&bytes).into_owned(); + body.push_str(" … [truncated]"); + return Ok(body); + } + } + Ok(String::from_utf8_lossy(&bytes).into_owned()) +} + fn normalize_base(mut url: Url) -> Url { if !url.path().ends_with('/') { let mut p = url.path().to_string(); diff --git a/crates/cloud-client/tests/client.rs b/crates/cloud-client/tests/client.rs index 296d33e..7c6401e 100644 --- a/crates/cloud-client/tests/client.rs +++ b/crates/cloud-client/tests/client.rs @@ -214,3 +214,67 @@ async fn delete_project_still_fails_on_error_status() { "discarding the body must not swallow a failing status", ); } + +#[tokio::test] +async fn mint_preserves_non_json_failure_context() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/v1/local/token")) + .respond_with( + ResponseTemplate::new(422) + .set_body_raw("Unprocessable Entity: expected an object", "text/plain"), + ) + .mount(&server) + .await; + let client = MemoryClient::new(Url::parse(&server.uri()).unwrap(), "amc_test_key").unwrap(); + let error = client.mint_local_token().await.unwrap_err(); + let display = error.to_string(); + for context in ["422", "POST", "v1/local/token", "expected an object"] { + assert!(display.contains(context), "missing {context}: {display}"); + } +} + +#[tokio::test] +async fn mint_bounds_and_redacts_non_json_failure() { + let server = MockServer::start().await; + let body = format!( + "upstream failure Bearer jwt-secret amc_key-secret {}", + "界".repeat(20_000) + ); + Mock::given(method("POST")) + .and(path("/v1/local/token")) + .respond_with(ResponseTemplate::new(502).set_body_raw(body, "text/html")) + .mount(&server) + .await; + let client = MemoryClient::new(Url::parse(&server.uri()).unwrap(), "amc_test_key").unwrap(); + let display = client.mint_local_token().await.unwrap_err().to_string(); + assert!(display.contains("upstream failure")); + assert!(!display.contains("jwt-secret")); + assert!(!display.contains("key-secret")); + assert!(display.len() < 5_000, "error excerpt must be bounded"); + assert!(display.contains("truncated")); +} + +#[tokio::test] +async fn mint_redacts_raw_credential_echo_and_json_secret_fields() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/v1/local/token")) + .respond_with(ResponseTemplate::new(422).set_body_json(serde_json::json!({ + "error": { "code": "validation_error", "message": "echo opaque-credential" }, + "access_token": "private-response-token", "email": "person@example.test" + }))) + .mount(&server) + .await; + let client = + MemoryClient::new(Url::parse(&server.uri()).unwrap(), "opaque-credential").unwrap(); + let display = client.mint_local_token().await.unwrap_err().to_string(); + assert!(display.contains("validation_error")); + for private in [ + "opaque-credential", + "private-response-token", + "person@example.test", + ] { + assert!(!display.contains(private), "leaked {private}: {display}"); + } +} diff --git a/package.json b/package.json index d59446e..e61407c 100644 --- a/package.json +++ b/package.json @@ -30,15 +30,22 @@ "public-integration-smoke": "turbo run public-integration-smoke", "repo-hygiene": "node scripts/ci/repo-hygiene.mjs", "security-compliance": "node scripts/security/security-compliance.mjs", + "lint:workflows": "bash scripts/ci/lint-workflows.sh", "pack-dry-run": "node scripts/ci/pack-dry-run.mjs", "release-policy": "node scripts/ci/release-policy.mjs", "test:guards": "node --test scripts/guards/__tests__/*.test.mjs", "test:install-cli": "bash scripts/__tests__/install-cli.test.sh", "test:install-cli-internal": "bash scripts/__tests__/install-cli-internal.test.sh", + "test:cli-install-smoke": "bash scripts/__tests__/cli-install-smoke.test.sh", + "test:cli-install-smoke-reporter": "bash scripts/__tests__/cli-install-smoke-reporter.test.sh", + "smoke:cli-install": "bash scripts/cli-install-smoke.sh", "test:reconcile-internal-release": "bash scripts/__tests__/reconcile-internal-release.test.sh", "test:release-cli-version": "bash scripts/__tests__/release-cli-version.test.sh", + "test:cli-version-bump": "bash scripts/__tests__/validate-cli-version-bump.test.sh", + "bump:cli-version": "bash scripts/ci/bump-cli-workspace-version.sh", "test:security-compliance": "node --test scripts/ci/__tests__/security-compliance.test.mjs", "test:release-policy": "node --test scripts/ci/__tests__/release-policy.test.mjs", + "test:roll-core-ecs-image": "node --test scripts/ci/__tests__/roll-core-ecs-*.test.mjs", "check:plugin-versions": "node scripts/version-families.mjs plugin --check", "check:adapter-versions": "node scripts/version-families.mjs adapter --check", "check:tool-versions": "node scripts/version-families.mjs tool --check", diff --git a/packages/core/.env.example b/packages/core/.env.example index 7c1310c..2408d8e 100644 --- a/packages/core/.env.example +++ b/packages/core/.env.example @@ -30,34 +30,32 @@ OPENAI_API_KEY= # Optional: verify Cloud-issued RS256 JWTs for connected-local console access. # JWKS_URL + ISSUER + AUDIENCE must be set together; when unset, only # CORE_API_KEY is accepted. -# CLOUD_JWKS_URL=https://api.dev.atomicstrata.ai/.well-known/atomic-core/jwks.json -# CLOUD_JWT_ISSUER=https://api.dev.atomicstrata.ai +# CLOUD_JWKS_URL=https://api.example.com/.well-known/atomic-core/jwks.json +# CLOUD_JWT_ISSUER=https://api.example.com # CLOUD_JWT_AUDIENCE=atomicmemory-core # Optional project binding. When set, tokens for other projects are rejected. # When omitted (single-key local), Core trusts the token's own project_id. # CLOUD_PROJECT_ID=proj_your_connected_local_project -# Temporary until Cloud mint includes memory_user_id (dev PoC uses user "default"): +# Temporary compatibility setting when the issuer omits memory_user_id: # CLOUD_JWT_LEGACY_DEFAULT_MEMORY_USER_ID=default # Optional — when false (default), CORE_API_KEY is rejected once JWT verify is enabled. # CLOUD_JWT_STATIC_KEY_FALLBACK=false # --- Connected-local (Docker entrypoint defaults) --- -# Minimal self-hosted + Cloud dev console: set only these in docker run. +# Operators must supply the Cloud API and console origins for their deployment. # OPENAI_API_KEY=sk-... # ATOMICMEMORY_API_KEY=amc_live_... +# ATOMICMEMORY_API_URL=https://api.example.com +# ALLOWED_ORIGINS=https://memory.example.com # Mount atomic-memory-state (or equivalent) so the entrypoint can persist # an auto-generated CORE_API_KEY at /var/lib/atomicmemory/state/core-api-key. # Retrieve after start: docker exec atomic-memory cat /var/lib/atomicmemory/state/core-api-key # Optional override: CORE_API_KEY=your-local-secret (skips auto-generation) -# Setting ATOMICMEMORY_API_KEY turns connected-local on; the Docker entrypoint -# then applies tier defaults when unset (CLOUD_ENV=dev): -# ATOMICMEMORY_API_URL, CLOUD_TRACE_SYNC_ENABLED=true, -# CLOUD_JWKS_URL, CLOUD_JWT_ISSUER, CLOUD_JWT_AUDIENCE=atomicmemory-core, -# ALLOWED_ORIGINS, CLOUD_JWT_LEGACY_DEFAULT_MEMORY_USER_ID=default, -# CLOUD_JWT_STATIC_KEY_FALLBACK=true +# Setting ATOMICMEMORY_API_KEY enables connected-local. The entrypoint derives +# CLOUD_JWKS_URL, CLOUD_JWT_ISSUER, CLOUD_JWT_AUDIENCE=atomicmemory-core, and +# CLOUD_JWT_STATIC_KEY_FALLBACK=true from the operator-supplied API URL. # CLOUD_PROJECT_ID is optional: set it to pin one project; when omitted Core -# trusts the Cloud-minted token's own project_id. Override tier with -# CLOUD_ENV=staging|production +# trusts the Cloud-minted token's own project_id. # Trusted-proxy identity guard (Radar C4). The shared CORE_API_KEY above # authenticates the *caller process*, not the end user; `user_id` is @@ -89,6 +87,10 @@ STORAGE_KEY_HMAC_SECRET=000102030405060708090a0b0c0d0e0f101112131415161718191a1b # --- Server --- PORT=17350 +# Optional bind address. Unset keeps Node's all-interfaces default (hosted/Docker). +# The macOS embedded launcher sets 127.0.0.1. Do not use HOST — it collides +# with the system hostname on many platforms. +# LISTEN_HOST=127.0.0.1 # Required deployment posture for storage-policy gates. Use `local` for # laptop/docker-compose development, `staging` for pre-prod, and `production` # for hardened hosted deployments. @@ -162,6 +164,35 @@ EMBEDDING_DIMENSIONS=1536 # For fully local/no-provider-key development, pair this with a non-OpenAI # embedding provider such as EMBEDDING_PROVIDER=transformers. +# --- Extraction / AUDN (SLM latency) --- +# Default `full` keeps the complete extraction contract. `compact` is a +# reduced-capability opt-in for local SLM *extraction* only. It records +# user-stated facts only and omits: +# - assistant recommendations / advice +# - contact-info, named-entity, and date guarantees from the full prompt +# - short-input retention rules +# - headline, keywords, entities, relations, and importance in model JSON +# (headline/keywords are derived from fact text after decode) +# AUDN mutation policy (ADD/UPDATE/SUPERSEDE/DELETE/NOOP/CLARIFY) stays the +# full prompt in both variants. Wire format is a separate axis from the +# prompt variant: set AUDN_JSON_SCHEMA=true when the LLM provider honors +# client json_schema (am-local-slm + AM_SLM_CORE_JSON_SCHEMA=1). Leave +# false (default) for Groq and other OpenAI-compatible hosts that reject +# strict Structured Outputs. Pair compact extraction with +# AM_SLM_CORE_COMPACT_SCHEMA=1. Invalid values fail startup. +# EXTRACTION_PROMPT_VARIANT=full +# EXTRACTION_PROMPT_VARIANT=compact +# Decode caps are whole safe positive integers (default 4096 / 2048; +# max 8192 / 4096). Not overridable via request config_override. +# EXTRACTION_MAX_TOKENS=4096 +# AUDN_MAX_TOKENS=2048 +# false keeps OpenAI-compatible requests on response_format=json_object. +# AUDN_JSON_SCHEMA=false +# Integer env vars parsed by Core (adaptive limits, CLOUD_TRACE_SYNC_*, +# RAW_UPLOAD_MAX_BYTES, decode caps) must be whole decimals such as `128`. +# Prefixes that parseInt used to truncate (`128junk`, `1.5`, `1e3`) now +# fail startup. + # --- Runtime config mutation (dev/test only) --- # Opt-in gate for PUT /memories/config. Leave unset in production — the # route returns 410 Gone unless this is true. diff --git a/packages/core/.fallowrc.json b/packages/core/.fallowrc.json index 85e9be2..cb0fd7b 100644 --- a/packages/core/.fallowrc.json +++ b/packages/core/.fallowrc.json @@ -1,5 +1,9 @@ { "$schema": "https://fallow.tools/schema.json", + "entry": [ + "hosted/migration-history-overlay.mjs", + "hosted/__tests__/migration-history-overlay.test.mjs" + ], "ignorePatterns": [ "**/one-offs/**", "scripts/**" @@ -17,5 +21,13 @@ "unused-class-members": "off", "duplicate-exports": "off" }, - "duplicates": {} + "duplicates": { + "ignore": [ + "**/__tests__/**", + "**/*.test.ts", + "**/*.test.mjs", + "tests/**", + "scripts/**" + ] + } } diff --git a/packages/core/.gitignore b/packages/core/.gitignore index bd8fc76..e6c965e 100644 --- a/packages/core/.gitignore +++ b/packages/core/.gitignore @@ -1,5 +1,8 @@ # Build outputs dist/ +dist-bin/ +.deno-compile-staging/ +deno.lock *.tsbuildinfo # Dependencies diff --git a/packages/core/CHANGELOG.md b/packages/core/CHANGELOG.md index 906a2ba..b5c775d 100644 --- a/packages/core/CHANGELOG.md +++ b/packages/core/CHANGELOG.md @@ -6,6 +6,43 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ## [Unreleased] +## [1.2.2] - 2026-09-20 + +### Fixed + +- Extraction JSON parsing now uses the same first-object extraction path as + AUDN, so trailing prose or a second JSON block after a successful model + completion no longer fails `JSON.parse` (ATO-2185). Truncated-JSON repair + (including a single complete entry) is retained, with one JSON-only retry + nudge on substantial unparseable content. +- The private hosted deployment image archives its retired migration history + before public Core starts. The public package continues to ship only + `0001` through `0005_cloud_trace_outbox`. + +### Changed + +- `EXTRACTION_PROMPT_VARIANT=compact` is a reduced-capability **extraction** + opt-in. AUDN mutation policy stays the full UPDATE/DELETE/CLARIFY prompt. + AUDN wire format is independent of the prompt variant: `AUDN_JSON_SCHEMA=true` + sends OpenAI-strict `json_schema` (`core_audn`) for am-local-slm; + the default `false` keeps `response_format: json_object` so Groq and other + OpenAI-compatible providers that lack Structured Outputs keep working. +- `parsePositiveIntEnv` / `parseBoundedPositiveIntEnv` now require a whole + decimal integer in the safe-integer range. Values such as `100abc`, `1.5`, + `1e3`, or overflow that used to `parseInt`-truncate now fail startup. This + applies to adaptive retrieval limits, all `CLOUD_TRACE_SYNC_*` intervals, + `RAW_UPLOAD_MAX_BYTES`, and the extraction/AUDN decode caps. +- Full-path extraction: an omitted `keywords` field is derived from the fact + text, and non-string keyword entries are dropped. Explicit `keywords: []` + stays empty. +- `extractionMaxTokens`, `audnMaxTokens`, and `audnJsonSchema` are not + request-overridable via `config_override` (LLM calls and cache identity + still read the process singleton). Those keys are reported on + `X-Atomicmem-Ignored-Override-Keys`. `X-Atomicmem-Config-Override-Applied` + and `X-Atomicmem-Config-Override-Keys` reflect only the keys that were + actually applied. `X-Atomicmem-Unknown-Override-Keys` stays reserved for + keys that do not match a current `RuntimeConfig` field. + ## [1.2.1] - 2026-08-07 ### Fixed diff --git a/packages/core/Dockerfile b/packages/core/Dockerfile index f53a365..7e72917 100644 --- a/packages/core/Dockerfile +++ b/packages/core/Dockerfile @@ -3,12 +3,9 @@ # 1) `turbo prune --docker` produces an `out/` tree with only @atomicmemory/core # and its workspace dependencies (manifests + lockfile in `out/json`, # source in `out/full`). -# 2) Installer stage installs prod deps from the pruned lockfile — this layer -# is cached as long as manifests and lockfile don't change. -# 3) Builder stage runs `pnpm deploy --prod` to produce a self-contained -# `/deploy` tree (flat node_modules + package.json). -# 4) Runtime image is pgvector + Node + the deployed package + TypeScript -# source for tsx. +# 2) Builder installs the pruned workspace, compiles Core, then produces a +# self-contained production dependency tree in `/deploy`. +# 3) Runtime copies only compiled production files, never the source/tests. FROM node:22-slim AS node-base @@ -23,27 +20,23 @@ COPY . . RUN pnpm dlx turbo@2.9.14 prune @atomicmemory/core --docker # --------------------------------------------------------------------------- -# Stage 2 — installer: production deps only +# Stage 2 — builder: compile Core and deploy production dependencies # --------------------------------------------------------------------------- -FROM node:22-slim AS installer +FROM node:22-slim AS builder ENV CI=1 HUSKY=0 RUN corepack enable WORKDIR /repo COPY --from=pruner /repo/out/json/ ./ COPY --from=pruner /repo/out/pnpm-lock.yaml ./pnpm-lock.yaml -RUN pnpm install --frozen-lockfile --filter @atomicmemory/core --prod --ignore-scripts - -# --------------------------------------------------------------------------- -# Stage 3 — builder: add full source and deploy a flat tree to /deploy -# --------------------------------------------------------------------------- -FROM installer AS builder +RUN pnpm install --frozen-lockfile --filter @atomicmemory/core --ignore-scripts COPY --from=pruner /repo/out/full/ ./ -RUN pnpm deploy --filter @atomicmemory/core --prod /deploy +RUN pnpm --filter @atomicmemory/core build \ + && pnpm deploy --filter @atomicmemory/core --prod /deploy # --------------------------------------------------------------------------- -# Stage 4 — runtime: pgvector + Node + deployed package +# Stage 3 — runtime base: pgvector + Node + compiled Core # --------------------------------------------------------------------------- -FROM pgvector/pgvector:pg17 +FROM pgvector/pgvector:pg17 AS runtime-base WORKDIR /app @@ -60,12 +53,11 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ COPY --from=builder /deploy/node_modules ./node_modules COPY --from=builder /deploy/package.json ./package.json -# Source / scripts / tsconfig — runtime uses tsx against src/, not prebuilt dist/. -COPY packages/core/src ./src +# Compile output excludes test files by tsconfig.build.json. +COPY --from=builder /repo/packages/core/dist ./dist COPY packages/core/scripts/docker-entrypoint.sh ./scripts/docker-entrypoint.sh -COPY packages/core/tsconfig.json ./tsconfig.json # The committed OpenAPI spec — loaded eagerly at startup by src/app/openapi-spec.ts -# (`../../openapi.json` -> /app/openapi.json) and served at GET /openapi.json. +# (`../../openapi.json` from dist/app -> /app/openapi.json) and served at GET /openapi.json. # Without this the container crashes on boot with ENOENT. COPY packages/core/openapi.json ./openapi.json COPY packages/core/openapi.yaml ./openapi.yaml @@ -90,5 +82,15 @@ ENV EMBEDDED_POSTGRES_DB=atomicmemory EXPOSE 17350 +# ECR-only stage: retain private hosted compatibility outside public npm/GHCR. +FROM runtime-base AS hosted-runtime +COPY packages/core/hosted/migration-history-overlay.mjs /app/private/migration-history-overlay.mjs +COPY packages/core/hosted/docker-entrypoint.sh /app/private/docker-entrypoint.sh +RUN chmod +x /app/private/docker-entrypoint.sh +ENTRYPOINT ["/app/private/docker-entrypoint.sh"] +CMD ["node", "dist/server.js"] + +# Public default stage: no private overlay, source tree, or test tree. +FROM runtime-base AS public-runtime ENTRYPOINT ["/app/scripts/docker-entrypoint.sh"] -CMD ["./node_modules/.bin/tsx", "src/server.ts"] +CMD ["node", "dist/server.js"] diff --git a/packages/core/README.md b/packages/core/README.md index 87cda2d..8d49ad7 100644 --- a/packages/core/README.md +++ b/packages/core/README.md @@ -74,6 +74,11 @@ present in GHCR with both required platforms, checks out the package `gitHead`, builds `packages/core/Dockerfile`, smoke-tests the local `linux/amd64` image, and then pushes the matching multi-platform GHCR tags. +Internal Dev/Staging ECS does **not** use that public GHCR image. On +`atomicmemory-internal`, version bumps of `@atomicmemory/core` on `dev` build +the same Dockerfile into the enterprise ECR repository and roll Dev/Staging +ECS. See [docs/ecr-dev-staging.md](docs/ecr-dev-staging.md). + Local Docker defaults use `Authorization: Bearer local-dev-key`, OpenAI embeddings at 1536 dimensions, and `RAW_STORAGE_DEPLOYMENT_ENV=local`. The quickstart binds to `127.0.0.1` so that default key is only exposed locally. @@ -295,19 +300,21 @@ curl -X POST http://localhost:17350/v1/memories/search \ }' ``` -Responses from requests carrying an override emit four observability headers: +Responses from requests carrying an override emit these observability headers: | Header | Emitted when | Value | |--------|--------------|-------| -| `X-Atomicmem-Config-Override-Applied` | Override present | `true` | +| `X-Atomicmem-Config-Override-Applied` | Override present | `true` if at least one key was applied, otherwise `false` | | `X-Atomicmem-Effective-Config-Hash` | Override present | `sha256:` of the merged config | -| `X-Atomicmem-Config-Override-Keys` | Override present | Comma-joined sorted override keys | +| `X-Atomicmem-Config-Override-Keys` | At least one key was applied | Comma-joined sorted **applied** keys | +| `X-Atomicmem-Ignored-Override-Keys` | One or more submitted keys are current `RuntimeConfig` fields that cannot take effect on this request (decode caps, `audnJsonSchema`) | Comma-joined sorted ignored keys | | `X-Atomicmem-Unknown-Override-Keys` | One or more keys don't match a current `RuntimeConfig` field | Comma-joined sorted unknown keys | The schema is permissive — unknown keys don't 400. They ride through on the -effective config and surface via the fourth header plus a server-side warning -log, so callers catch typos without gating new runtime fields behind a schema -release. +effective config and surface via `X-Atomicmem-Unknown-Override-Keys` plus a +server-side warning log, so callers catch typos without gating new runtime +fields behind a schema release. Non-overridable known fields are listed on +`X-Atomicmem-Ignored-Override-Keys` and are not counted as applied. ## Environment Variables diff --git a/packages/core/docs/darwin-pgvector-pack.md b/packages/core/docs/darwin-pgvector-pack.md new file mode 100644 index 0000000..5cf7214 --- /dev/null +++ b/packages/core/docs/darwin-pgvector-pack.md @@ -0,0 +1,65 @@ +# Darwin Postgres 17 + pgvector pack + +Build a **relocatable** Postgres/pgvector tree for macOS DMG embedding. Matches Docker image `pgvector/pgvector:pg17` major version (PostgreSQL 17). + +## Output + +```text +dist-bin/atomicmemory-postgres-pgvector-darwin-arm64.tar.gz + postgres/ + bin/ + lib/ + share/ +``` + +Gitignored — never commit the tarball or `dist-bin/postgres/`. + +## Prerequisites (build machine) + +- macOS arm64 (Apple Silicon) for the default artifact name +- Xcode command line tools +- Optional dev path: Homebrew `postgresql@17` + `pgvector` for local assembly only + +## Build + +```bash +cd packages/core + +# Option A: assemble from a prepared source tree (recommended for CI) +export POSTGRES_SOURCE_PREFIX=/path/to/relocateable/postgres +./scripts/package-darwin-pgvector.sh + +# Option B: dev-only — copy from Homebrew prefix (NOT for redistribution) +export POSTGRES_SOURCE_PREFIX="$(brew --prefix postgresql@17)" +export PGVECTOR_LIB_DIR="$(brew --prefix pgvector)/lib" +./scripts/package-darwin-pgvector.sh +``` + +## Smoke test + +```bash +tar -xzf dist-bin/atomicmemory-postgres-pgvector-darwin-arm64.tar.gz -C /tmp/am-pg-test +export PATH="/tmp/am-pg-test/postgres/bin:$PATH" +export DYLD_LIBRARY_PATH="/tmp/am-pg-test/postgres/lib" +printf 'smoke-password\n' > /tmp/am-pg-pw +initdb -D /tmp/am-pg-data --username=atomicmemory \ + --auth-local=scram-sha-256 --auth-host=scram-sha-256 --pwfile=/tmp/am-pg-pw +printf "\nlisten_addresses = '127.0.0.1'\nport = 54329\n" >> /tmp/am-pg-data/postgresql.conf +pg_ctl -D /tmp/am-pg-data -w start +PGPASSWORD=smoke-password psql -h 127.0.0.1 -p 54329 -U atomicmemory -d postgres -c "CREATE DATABASE atomicmemory" +PGPASSWORD=smoke-password psql -h 127.0.0.1 -p 54329 -U atomicmemory -d atomicmemory -c "CREATE EXTENSION vector" +pg_ctl -D /tmp/am-pg-data -w stop +``` + +## Version pins + +| Component | Pin | +|-----------|-----| +| PostgreSQL | 17.x (match Docker `pgvector/pgvector:pg17`) | +| pgvector | Latest compatible with PG17 | + +Record exact versions used in release notes when publishing DMG artifacts. + +## Production note + +Homebrew copies are for **local development smoke only**. Production DMG builds should use a CI-produced relocatable pack with `install_name_tool` adjustments for `@loader_path` — extend `package-darwin-pgvector.sh` as that pipeline matures. diff --git a/packages/core/docs/db/migrations.md b/packages/core/docs/db/migrations.md index cbab723..9e24ca4 100644 --- a/packages/core/docs/db/migrations.md +++ b/packages/core/docs/db/migrations.md @@ -15,6 +15,7 @@ src/db/ 0001_baseline.sql Frozen Phase-1 schema; never edited after shipment. 0002_.sql First post-baseline migration. … + 0005_cloud_trace_outbox.sql Shipped public name; never renumber. migration-api.ts migrate() / migrationStatus() entry points. migrate.ts CLI shim used by `npm run migrate`. ``` @@ -28,6 +29,29 @@ order against an empty database, is the schema. To get a current schema dump locally, replay the files against an empty database and run `pg_dump --schema-only`. +## Hosted / enterprise history bridge + +Public migration names and SQL remain immutable. Fresh Core installations apply +only the public migrations, including `0005_cloud_trace_outbox.sql`; enterprise +DDL and placeholder migrations are not part of the package. + +Some existing hosted databases carry a separate migration lineage after the +shared public predecessors. Under the migration advisory lock, the runner +recognizes the explicitly enumerated hosted sequence and validates both its +order and the public predecessor sequence. Only that recognized history bypasses +`node-pg-migrate`'s index-based order check. Unknown, missing, duplicate, or +reordered entries in that lineage fail closed. + +The bridge preserves all existing history rows and applies the original +idempotent outbox SQL under its public name. This also supports a recognized +hosted history that already recorded the outbox under the short-lived `0014` +name. It never resets a database or installs enterprise-only schema objects. + +Regression tests cover the shipped public upgrade with existing outbox data, +fresh installs, hosted histories, repeated migration calls, and rejected +histories. The migration DAG check compares shipped names and bytes against +`main` using the repository's actual `packages/core/src/db/migrations` path. + ## Inspecting state Two tables hold migration state, on purpose: diff --git a/packages/core/docs/deno-compile-spike.md b/packages/core/docs/deno-compile-spike.md new file mode 100644 index 0000000..0efcdee --- /dev/null +++ b/packages/core/docs/deno-compile-spike.md @@ -0,0 +1,141 @@ +# Deno compile spike — Core API (external Postgres) + +Experimental packaging path for compiling AtomicMemory Core into a single +executable. **Node + Docker remain the canonical distribution.** This spike +validates whether Deno `compile` can ship the API process against an external +Postgres/pgvector instance. + +## Quick start + +```bash +brew install deno +pnpm install --filter @atomicmemory/core +cd packages/core +./scripts/deno-compile-core.sh + +docker compose up postgres -d +export DATABASE_URL='postgresql://atomicmemory:atomicmemory@localhost:5433/atomicmemory' +export RAW_STORAGE_DEPLOYMENT_ENV=local +export CORE_API_KEY='test-core-api-key' +export STORAGE_KEY_HMAC_SECRET='000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f' +export EMBEDDING_DIMENSIONS=1024 +export EMBEDDING_PROVIDER=openai +export OPENAI_API_KEY='sk-...' +./dist-bin/atomicmemory-core migrate +./dist-bin/atomicmemory-core start +curl http://localhost:17350/health +``` + +## Spike results (2026-07-31) + +| Item | Result | +|------|--------| +| Deno version | 2.9.4 (Homebrew, macOS arm64) | +| Entry point | lean staging `dist/bin.js` (after `pnpm run build`) | +| Binary size | **~116 MB** (lean staging + `--node-modules-dir=manual`) | +| Cold start (`--help`) | ~24 ms | +| `migrate` | Pass | +| `start` + `GET /health` | Pass (HTTP 200) | +| `GET /openapi.json` | Pass | +| Runtime Deno required | No | + +Environment profile: remote OpenAI embeddings (`EMBEDDING_PROVIDER=openai`), +non-Filecoin storage. Local `@huggingface/transformers` and Filecoin/viem are +**not** in the lean binary. + +### Size progression + +| Approach | Size | Notes | +|----------|------|-------| +| `--node-modules-dir=auto` (full monorepo) | ~774 MB | Works; embeds transformers, viem, Filecoin, vitest, typescript | +| `--bundle --exclude-unused-npm` | ~447 MB | Still pulls heavy reachable deps | +| `--bundle --minify` | ~441 MB | Negligible further gain | +| Lean staging (drop heavy deps) + manual | **~116 MB** | Current default in `deno-compile-core.sh` | + +## Go / no-go + +**Go for experimental local distribution** of the Core API process when: + +- Postgres/pgvector is external. +- Remote embedding/LLM providers are configured (OpenAI-compatible). +- Filecoin / local transformers are not required. +- ~116 MB binary size is acceptable. + +**No-go for replacing Docker/npm today** because: + +- Lean profile intentionally omits Filecoin + local transformers. +- No CI artifact or release channel yet. +- Node + Docker remain the supported paths. + +## Compile strategy (current) + +`scripts/deno-compile-core.sh`: + +1. `pnpm run build` → `dist/` +2. Stage a lean `package.json` without: + - `@huggingface/transformers` + - `@filoz/synapse-core`, `@filoz/synapse-sdk`, `viem`, `multiformats` + - `tsx` (runtime uses compiled `dist/`) + - all `optionalDependencies` +3. Pin kept dependencies to exact `pnpm list --depth 0 --prod` versions +4. `pnpm install --prod --ignore-scripts --ignore-workspace` in + `.deno-compile-staging/` with `node-linker=hoisted` (not `pnpm deploy`; + deploy copies the full isolated `.pnpm` store and Deno compile times out) +5. `deno compile --no-check --node-modules-dir=manual` with `--allow-write` and + `--include` for `openapi.json`, `openapi.yaml`, `package.json`, + `dist/db/migrations` + +### Why not `--node-modules-dir=auto`? + +Against the monorepo pnpm tree it embeds the full dependency graph (~774 MB). +Against the hoisted lean tree, `--manual` is enough and much smaller. + +### Why not `--bundle` alone? + +esbuild still follows string-literal dynamic imports into +`@huggingface/transformers` and Filecoin providers, so those packages stay +in the embed unless they are removed from the staging dependency set. + +### Entrypoint note + +`src/bin.ts` treats Deno `import.meta.main` and any basename starting with +`atomicmemory-core` as the CLI entrypoint (needed for compiled binaries). + +## Embedded assets + +| Asset | Consumer | +|-------|----------| +| `openapi.json` | `dist/app/openapi-spec.js` | +| `openapi.yaml` | optional parity | +| `package.json` | migration schema fingerprint | +| `dist/db/migrations/*.sql` | `dist/db/migration-schema.js` | + +## Follow-ups + +1. Further shrink with `--bundle` **after** lean staging (opaque dynamic imports + or stubs for Filecoin JS still present in `dist/`). +2. Optional “full” profile binary that re-adds transformers/Filecoin. +3. Cross-compile (`--target`) and CI artifact once size/profile are stable. + +## macOS DMG consumer + +For the AtomicMemory Mac app embedded runtime, see: + +- [macos-embedded-runtime.md](./macos-embedded-runtime.md) — env contract, ports, artifact names +- [darwin-pgvector-pack.md](./darwin-pgvector-pack.md) — Postgres pack build +- `scripts/macos-embedded-entrypoint.sh` — lifecycle entrypoint +- am-app [ADR 0002](https://github.com/atomicstrata/am-app/blob/main/docs/decisions/0002-embedded-local-runtime.md) + +CI produces `atomicmemory-core-darwin-arm64` on macOS runners via `.github/workflows/core-deno-compile-macos.yml`. + +## Explicit non-goals (unchanged) + +- Postgres/pgvector inside the binary +- Docker embedded-Postgres parity +- Replacing `@atomicmemory/core` npm `bin` or Dockerfile + +## Rollback + +Remove `scripts/deno-compile-core.sh`, this doc, and `dist-bin/` / +`.deno-compile-staging/` (gitignored). Revert `src/bin.ts` entrypoint tweak +if unused. No production surfaces change. diff --git a/packages/core/docs/ecr-dev-staging.md b/packages/core/docs/ecr-dev-staging.md new file mode 100644 index 0000000..27d8b71 --- /dev/null +++ b/packages/core/docs/ecr-dev-staging.md @@ -0,0 +1,76 @@ +# Core Dev/Staging ECR images + +Dev and Staging ECS run Core from the private enterprise ECR repository, not +from public GHCR. Images are built from `atomicmemory-internal` whenever +`@atomicmemory/core`'s version on `dev` changes. + +## Pipeline + +| Lane | Workflow | Registry | Trigger | +| --- | --- | --- | --- | +| Public release | `publish-core-docker.yml` | `ghcr.io/atomicstrata/atomicmemory-core` | npm publish (`repository_dispatch`) | +| Internal GHCR test | `internal-core-docker-image.yml` | `ghcr.io/atomicstrata/atomicmemory-core-internal` | `main` paths / dispatch | +| **Dev/Staging ECS** | `core-ecr-dev-staging.yml` | `…/atomicmemory-core-enterprise` | `dev` version bump / dispatch | + +`core-ecr-dev-staging.yml` is named outside the `publish-*.yml` prefix on +purpose: release-policy forbids `workflow_dispatch` on that prefix, and this +lane must stay operator-dispatchable. + +## Tags + +Each successful build pushes (linux/amd64, Fargate): + +- `${VERSION}` — from `packages/core/package.json` (e.g. `1.2.2`) +- `sha-${GITHUB_SHA:0:7}` — immutable per commit (preferred for ECS) +- `dev` — floating tip of the last successful Dev/Staging publish + +OCI labels: `org.opencontainers.image.revision`, `.version`, +`.source=atomicmemory-internal`. + +Example: + +```text +636941960505.dkr.ecr.us-east-1.amazonaws.com/atomicmemory-core-enterprise:1.2.2 +636941960505.dkr.ecr.us-east-1.amazonaws.com/atomicmemory-core-enterprise:sha-a1b2c3d +636941960505.dkr.ecr.us-east-1.amazonaws.com/atomicmemory-core-enterprise:dev +``` + +ECS task definitions are updated to the immutable `sha-…` URI (or `${VERSION}` +when reusing an existing version tag without a rebuild). + +## After push + +1. Write SSM String parameters (defaults; override with repo vars if infra differs): + - `/am-cloud/dev/core_image` + - `/am-cloud/staging/core_image` +2. Register a new task-definition revision with the new image URI for any + container already pointing at `atomicmemory-core-enterprise`. +3. `update-service --force-new-deployment` on: + - `atomicmemory-dev-cluster` / `atomicmemory-core` + - `atomicmemory-staging-cluster` / `atomicmemory-core` + +## Required GitHub Actions variables + +Companion infra (OIDC role + SSM params) must exist before the first real run. +Do not put role ARNs or account secrets in this repository — configure them as +repository **variables** on `atomicstrata/atomicmemory-internal`: + +| Variable | Required | Purpose | +| --- | --- | --- | +| `AWS_CORE_ECR_PUBLISH_ROLE_ARN` | one of shared / per-env | Shared GitHub OIDC role that can push ECR, write SSM, register ECS task defs | +| `AWS_CORE_ECR_PUBLISH_ROLE_ARN_DEV` | optional | Dev-only role; falls back to the shared ARN | +| `AWS_CORE_ECR_PUBLISH_ROLE_ARN_STAGING` | optional | Staging-only role; falls back to the shared ARN | +| `AWS_REGION` | optional | Defaults to `us-east-1` | +| `AWS_CORE_IMAGE_SSM_PARAM_DEV` | optional | Defaults to `/am-cloud/dev/core_image` | +| `AWS_CORE_IMAGE_SSM_PARAM_STAGING` | optional | Defaults to `/am-cloud/staging/core_image` | + +## Operator dispatch + +Actions → **Core ECR Dev/Staging** → Run workflow: + +- `ref` — branch/tag/SHA to build (empty = triggering SHA) +- `deploy_dev` / `deploy_staging` — default true; set false for image-only +- `force` — rebuild even when the `${VERSION}` tag already exists in ECR + +Dry-run once OIDC is wired: dispatch with `deploy_dev=false` and +`deploy_staging=false` to push tags without rolling ECS. diff --git a/packages/core/docs/macos-embedded-runtime.md b/packages/core/docs/macos-embedded-runtime.md new file mode 100644 index 0000000..3c1d551 --- /dev/null +++ b/packages/core/docs/macos-embedded-runtime.md @@ -0,0 +1,106 @@ +# macOS embedded runtime contract + +**Status:** Experimental — consumed by `am-app` DMG packaging (ADR 0002) +**Engine PR:** [atomicstrata/atomicmemory-internal#81](https://github.com/atomicstrata/atomicmemory-internal/pull/81) +**am-app spec:** [am-app embedded DMG spec](https://github.com/atomicstrata/am-app/blob/main/docs/superpowers/specs/2026-08-07-embedded-macos-dmg-runtime.md) + +## Overview + +The macOS DMG ships three runtime pieces under `AtomicMemory.app/Contents/Resources/Runtime/`: + +| Artifact | Role | +|----------|------| +| `atomicmemory-core` | Deno-compiled lean Core API binary (~116 MB) | +| `postgres/` | Relocatable Postgres 17 + pgvector tree | +| `macos-embedded-entrypoint.sh` | Lifecycle script (initdb → migrate → start) | +| `lib/` | Sourced launcher helpers (Postgres + lifecycle) | + +Data persists under `~/Library/Application Support/AtomicMemory/`. Node, Deno, and Docker are **not** required at runtime. + +## Artifact names (CI / release) + +| Name | Platform | +|------|----------| +| `atomicmemory-core-darwin-arm64` | macOS Apple Silicon | +| `atomicmemory-postgres-pgvector-darwin-arm64.tar.gz` | macOS Apple Silicon Postgres pack | + +## Ports and binding + +| Service | Default | Binding | +|---------|---------|---------| +| Core HTTP | `17350` | `127.0.0.1` only | +| Embedded Postgres | `54329` | `127.0.0.1` only (avoids clash with local 5432/5433) | + +## Environment variables + +| Variable | Required | Default | Notes | +|----------|----------|---------|-------| +| `RUNTIME_ROOT` | No | Entrypoint script directory | Bundle `Resources/Runtime` | +| `STATE_ROOT` | No | `~/Library/Application Support/AtomicMemory` | Data + logs | +| `DATABASE_URL` | No | `embedded` | Use `embedded` or external `postgresql://...` | +| `CORE_API_KEY` | No | Generated + persisted | Also readable from Keychain by app supervisor | +| `STORAGE_KEY_HMAC_SECRET` | No | Local dev default | Set explicitly for production-like installs | +| `RAW_STORAGE_DEPLOYMENT_ENV` | No | `local` | | +| `EMBEDDING_PROVIDER` | No | `openai` | Lean profile | +| `EMBEDDING_DIMENSIONS` | No | `1536` | | +| `OPENAI_API_KEY` | **Yes** (lean) | — | Remote embeddings required in lean binary | +| `EMBEDDED_POSTGRES_PORT` | No | `54329` | | +| `EMBEDDED_POSTGRES_USER` | No | `atomicmemory` | | +| `EMBEDDED_POSTGRES_DB` | No | `atomicmemory` | | +| `PORT` | No | `17350` | Core listen port | +| `LISTEN_HOST` | No | `127.0.0.1` (launcher) | Core bind address; hosted/Docker leave unset | + +## Directory layout + +```text +Runtime/ + atomicmemory-core + macos-embedded-entrypoint.sh + lib/ macos-embedded-postgres.sh macos-embedded-lifecycle.sh + postgres/ + bin/ postgres pg_ctl initdb psql … + lib/ + share/ + +~/Library/Application Support/AtomicMemory/ + postgres/ # PG data (PG_VERSION, …) + postgres-run/ # Unix socket directory + state/ # core-api-key, postgres-password, runtime.pid + logs/ # postgres.log, core.log +``` + +## Lifecycle + +```bash +export RUNTIME_ROOT=/path/to/Runtime +export STATE_ROOT="$HOME/Library/Application Support/AtomicMemory" +export OPENAI_API_KEY=sk-... +"$RUNTIME_ROOT/macos-embedded-entrypoint.sh" start +``` + +1. Validate runtime layout (Core binary + Postgres bin + `lib/`) +2. Resolve/generate `CORE_API_KEY`, Postgres SCRAM password, and local secrets +3. If `DATABASE_URL=embedded`: `initdb` (empty/valid cluster only) → conf-file `pg_ctl start` → `CREATE EXTENSION vector` +4. `atomicmemory-core migrate` +5. `atomicmemory-core start` bound to `LISTEN_HOST` (foreground; supervisor may wrap this) +6. On EXIT / SIGTERM / SIGINT: stop Core, then Postgres +7. `stop` signals the owning launcher/Core from `state/runtime.pid` after identity checks + +Fail closed: any step failure exits non-zero. A nonempty data directory without `PG_VERSION` is not deleted. + +## Lean profile limitations + +Same as [deno-compile-spike.md](./deno-compile-spike.md): + +- No local `@huggingface/transformers` +- No Filecoin / viem stack +- Remote OpenAI-compatible embeddings required + +## License / redistribution + +Shipping Postgres and pgvector inside a DMG requires compliance with their respective licenses (PostgreSQL License, pgvector license). Document attribution in the app About / legal notices. Do not redistribute Homebrew-built binaries in production DMGs without a dedicated relocatable build — use `scripts/package-darwin-pgvector.sh` output. + +## Related docs + +- [deno-compile-spike.md](./deno-compile-spike.md) — Deno lean binary +- [darwin-pgvector-pack.md](./darwin-pgvector-pack.md) — Postgres pack build diff --git a/packages/core/openapi.json b/packages/core/openapi.json index 635994c..66505dd 100644 --- a/packages/core/openapi.json +++ b/packages/core/openapi.json @@ -10423,7 +10423,7 @@ } ] }, - "description": "Optional per-request overlay on RuntimeConfig. Keys correspond to RuntimeConfig field names; values must be primitives (boolean / number / string / null). Unknown keys are accepted but surfaced via the X-Atomicmem-Unknown-Override-Keys response header and a server-side warning log — they do not cause a 400. Scope: just this request — no server mutation.", + "description": "Optional per-request overlay on RuntimeConfig. Keys correspond to RuntimeConfig field names; values must be primitives (boolean / number / string / null). Unknown keys are accepted but surfaced via X-Atomicmem-Unknown-Override-Keys. Known non-overridable fields (decode caps, audnJsonSchema) are listed on X-Atomicmem-Ignored-Override-Keys and are not counted as applied. Scope: just this request — no server mutation.", "type": "object" }, "content_class": { @@ -10744,7 +10744,7 @@ } ] }, - "description": "Optional per-request overlay on RuntimeConfig. Keys correspond to RuntimeConfig field names; values must be primitives (boolean / number / string / null). Unknown keys are accepted but surfaced via the X-Atomicmem-Unknown-Override-Keys response header and a server-side warning log — they do not cause a 400. Scope: just this request — no server mutation.", + "description": "Optional per-request overlay on RuntimeConfig. Keys correspond to RuntimeConfig field names; values must be primitives (boolean / number / string / null). Unknown keys are accepted but surfaced via X-Atomicmem-Unknown-Override-Keys. Known non-overridable fields (decode caps, audnJsonSchema) are listed on X-Atomicmem-Ignored-Override-Keys and are not counted as applied. Scope: just this request — no server mutation.", "type": "object" }, "content_class": { @@ -13021,7 +13021,7 @@ } ] }, - "description": "Optional per-request overlay on RuntimeConfig. Keys correspond to RuntimeConfig field names; values must be primitives (boolean / number / string / null). Unknown keys are accepted but surfaced via the X-Atomicmem-Unknown-Override-Keys response header and a server-side warning log — they do not cause a 400. Scope: just this request — no server mutation.", + "description": "Optional per-request overlay on RuntimeConfig. Keys correspond to RuntimeConfig field names; values must be primitives (boolean / number / string / null). Unknown keys are accepted but surfaced via X-Atomicmem-Unknown-Override-Keys. Known non-overridable fields (decode caps, audnJsonSchema) are listed on X-Atomicmem-Ignored-Override-Keys and are not counted as applied. Scope: just this request — no server mutation.", "type": "object" }, "limit": { @@ -13844,7 +13844,7 @@ } ] }, - "description": "Optional per-request overlay on RuntimeConfig. Keys correspond to RuntimeConfig field names; values must be primitives (boolean / number / string / null). Unknown keys are accepted but surfaced via the X-Atomicmem-Unknown-Override-Keys response header and a server-side warning log — they do not cause a 400. Scope: just this request — no server mutation.", + "description": "Optional per-request overlay on RuntimeConfig. Keys correspond to RuntimeConfig field names; values must be primitives (boolean / number / string / null). Unknown keys are accepted but surfaced via X-Atomicmem-Unknown-Override-Keys. Known non-overridable fields (decode caps, audnJsonSchema) are listed on X-Atomicmem-Ignored-Override-Keys and are not counted as applied. Scope: just this request — no server mutation.", "type": "object" }, "limit": { diff --git a/packages/core/openapi.yaml b/packages/core/openapi.yaml index 516ad05..02390df 100644 --- a/packages/core/openapi.yaml +++ b/packages/core/openapi.yaml @@ -7311,7 +7311,7 @@ paths: - type: number - type: string - type: "null" - description: "Optional per-request overlay on RuntimeConfig. Keys correspond to RuntimeConfig field names; values must be primitives (boolean / number / string / null). Unknown keys are accepted but surfaced via the X-Atomicmem-Unknown-Override-Keys response header and a server-side warning log — they do not cause a 400. Scope: just this request — no server mutation." + description: "Optional per-request overlay on RuntimeConfig. Keys correspond to RuntimeConfig field names; values must be primitives (boolean / number / string / null). Unknown keys are accepted but surfaced via X-Atomicmem-Unknown-Override-Keys. Known non-overridable fields (decode caps, audnJsonSchema) are listed on X-Atomicmem-Ignored-Override-Keys and are not counted as applied. Scope: just this request — no server mutation." type: object content_class: description: "Optional sensitivity class of the supplied content: 'summary' (distilled, hosted-safe), 'redacted' (sensitive spans removed by the caller), or 'raw' (verbatim prompt/response/diff/source). When the deployment runs RAW_CONTENT_POLICY=reject, a verbatim write of 'raw' content — or content with no content_class at all (treated as unknown/raw) — is rejected with 422 raw_content_rejected; on extraction paths the raw transcript is instead withheld from the stored audit episode." @@ -7535,7 +7535,7 @@ paths: - type: number - type: string - type: "null" - description: "Optional per-request overlay on RuntimeConfig. Keys correspond to RuntimeConfig field names; values must be primitives (boolean / number / string / null). Unknown keys are accepted but surfaced via the X-Atomicmem-Unknown-Override-Keys response header and a server-side warning log — they do not cause a 400. Scope: just this request — no server mutation." + description: "Optional per-request overlay on RuntimeConfig. Keys correspond to RuntimeConfig field names; values must be primitives (boolean / number / string / null). Unknown keys are accepted but surfaced via X-Atomicmem-Unknown-Override-Keys. Known non-overridable fields (decode caps, audnJsonSchema) are listed on X-Atomicmem-Ignored-Override-Keys and are not counted as applied. Scope: just this request — no server mutation." type: object content_class: description: "Optional sensitivity class of the supplied content: 'summary' (distilled, hosted-safe), 'redacted' (sensitive spans removed by the caller), or 'raw' (verbatim prompt/response/diff/source). When the deployment runs RAW_CONTENT_POLICY=reject, a verbatim write of 'raw' content — or content with no content_class at all (treated as unknown/raw) — is rejected with 422 raw_content_rejected; on extraction paths the raw transcript is instead withheld from the stored audit episode." @@ -9120,7 +9120,7 @@ paths: - type: number - type: string - type: "null" - description: "Optional per-request overlay on RuntimeConfig. Keys correspond to RuntimeConfig field names; values must be primitives (boolean / number / string / null). Unknown keys are accepted but surfaced via the X-Atomicmem-Unknown-Override-Keys response header and a server-side warning log — they do not cause a 400. Scope: just this request — no server mutation." + description: "Optional per-request overlay on RuntimeConfig. Keys correspond to RuntimeConfig field names; values must be primitives (boolean / number / string / null). Unknown keys are accepted but surfaced via X-Atomicmem-Unknown-Override-Keys. Known non-overridable fields (decode caps, audnJsonSchema) are listed on X-Atomicmem-Ignored-Override-Keys and are not counted as applied. Scope: just this request — no server mutation." type: object limit: maximum: 100 @@ -9693,7 +9693,7 @@ paths: - type: number - type: string - type: "null" - description: "Optional per-request overlay on RuntimeConfig. Keys correspond to RuntimeConfig field names; values must be primitives (boolean / number / string / null). Unknown keys are accepted but surfaced via the X-Atomicmem-Unknown-Override-Keys response header and a server-side warning log — they do not cause a 400. Scope: just this request — no server mutation." + description: "Optional per-request overlay on RuntimeConfig. Keys correspond to RuntimeConfig field names; values must be primitives (boolean / number / string / null). Unknown keys are accepted but surfaced via X-Atomicmem-Unknown-Override-Keys. Known non-overridable fields (decode caps, audnJsonSchema) are listed on X-Atomicmem-Ignored-Override-Keys and are not counted as applied. Scope: just this request — no server mutation." type: object limit: maximum: 100 diff --git a/packages/core/package.json b/packages/core/package.json index 8c64de8..39f29c0 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -1,6 +1,6 @@ { "name": "@atomicmemory/core", - "version": "1.2.1", + "version": "1.2.2", "description": "Open-source memory engine for AI applications — semantic retrieval, AUDN mutation, and contradiction-safe claim versioning.", "type": "module", "license": "Apache-2.0", @@ -65,7 +65,7 @@ "openapi.yaml" ], "scripts": { - "code-health": "fallow --no-cache --format compact --fail-on-issues", + "code-health": "fallow audit --base=${FALLOW_BASE_REF:-origin/main} --no-cache --format compact --fail-on-issues", "dev": "dotenv -e .env -- tsx watch src/server.ts", "start": "dotenv -e .env -- tsx src/server.ts", "typecheck": "tsc --noEmit", @@ -74,9 +74,10 @@ "generate:openapi": "tsx scripts/generate-openapi.ts", "check:openapi": "pnpm run generate:openapi && git diff --exit-code openapi.yaml openapi.json", "prepublishOnly": "node ../../scripts/guards/guard-npm-publish.mjs && npm run generate:openapi && npm run build", - "test": "dotenv -e .env.test -- vitest run --reporter verbose", + "test": "dotenv -e .env.test -- vitest run --reporter verbose && pnpm run test:scripts", "test:watch": "dotenv -e .env.test -- vitest --reporter verbose", - "test:deployment": "vitest run src/__tests__/deployment-config.test.ts --reporter verbose", + "test:scripts": "bash scripts/__tests__/run.sh", + "test:deployment": "vitest run src/__tests__/macos-embedded-config.test.ts src/app/__tests__/listen-host.test.ts --reporter verbose && pnpm run test:scripts", "test:docker-smoke": "./scripts/docker-smoke-test.sh", "test:schema": "tests/schema/run-schemathesis.sh", "migrate": "dotenv -e .env -- tsx src/db/migrate.ts", diff --git a/packages/core/scripts/__tests__/helpers.sh b/packages/core/scripts/__tests__/helpers.sh new file mode 100644 index 0000000..f679ab2 --- /dev/null +++ b/packages/core/scripts/__tests__/helpers.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +# Shared fixtures for macOS embedded launcher tests. + +SCRIPT_TESTS_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +CORE_SCRIPTS_DIR="$(cd "$SCRIPT_TESTS_DIR/.." && pwd)" +ENTRYPOINT="$CORE_SCRIPTS_DIR/macos-embedded-entrypoint.sh" + +assert_eq() { + local name="$1" expected="$2" actual="$3" + if [[ "$expected" != "$actual" ]]; then + printf 'FAIL %s: expected %q got %q\n' "$name" "$expected" "$actual" >&2 + exit 1 + fi + printf 'PASS %s\n' "$name" +} + +assert_file() { + local name="$1" path="$2" + if [[ ! -e "$path" ]]; then + printf 'FAIL %s: missing %s\n' "$name" "$path" >&2 + exit 1 + fi + printf 'PASS %s\n' "$name" +} + +assert_contains() { + local name="$1" needle="$2" haystack="$3" + if [[ "$haystack" != *"$needle"* ]]; then + printf 'FAIL %s: missing %q\n' "$name" "$needle" >&2 + exit 1 + fi + printf 'PASS %s\n' "$name" +} + +assert_not_contains() { + local name="$1" needle="$2" haystack="$3" + if [[ "$haystack" == *"$needle"* ]]; then + printf 'FAIL %s: unexpectedly found %q\n' "$name" "$needle" >&2 + exit 1 + fi + printf 'PASS %s\n' "$name" +} + +install_stubs() { + local bin="$RUNTIME_ROOT/postgres/bin" + mkdir -p "$bin" + cp "$SCRIPT_TESTS_DIR/stubs/initdb" "$bin/initdb" + cp "$SCRIPT_TESTS_DIR/stubs/pg_ctl" "$bin/pg_ctl" + cp "$SCRIPT_TESTS_DIR/stubs/psql" "$bin/psql" + cp "$SCRIPT_TESTS_DIR/stubs/postgres" "$bin/postgres" + cp "$SCRIPT_TESTS_DIR/stubs/atomicmemory-core" "$RUNTIME_ROOT/atomicmemory-core" + chmod +x "$bin/initdb" "$bin/pg_ctl" "$bin/psql" "$bin/postgres" "$RUNTIME_ROOT/atomicmemory-core" +} + +make_workspace() { + AM_WORK="$(mktemp -d "${TMPDIR:-/tmp}/am-embed.XXXXXX")" + export AM_WORK + export STATE_ROOT="$AM_WORK/Application Support/AtomicMemory" + export RUNTIME_ROOT="$AM_WORK/Runtime" + export AM_STUB_DIR="$AM_WORK/stub-logs" + mkdir -p "$RUNTIME_ROOT/postgres/bin" "$AM_STUB_DIR" "$STATE_ROOT" + export AM_STUB_INITDB_ARGS="$AM_STUB_DIR/initdb.args" + export AM_STUB_PGCTL_LOG="$AM_STUB_DIR/pgctl.log" + export AM_STUB_PGCTL_STOPS="$AM_STUB_DIR/pgctl.stops" + export AM_STUB_PGCTL_STARTS="$AM_STUB_DIR/pgctl.starts" + : > "$AM_STUB_PGCTL_LOG" + : > "$AM_STUB_PGCTL_STOPS" + : > "$AM_STUB_PGCTL_STARTS" + install_stubs +} + +run_entry() { + OPENAI_API_KEY="${OPENAI_API_KEY:-test-openai}" \ + RUNTIME_ROOT="$RUNTIME_ROOT" \ + STATE_ROOT="$STATE_ROOT" \ + "$ENTRYPOINT" "$@" +} + +wait_for_file() { + local path="$1" + local i + for i in $(seq 1 80); do + if [[ -f "$path" ]]; then + return 0 + fi + sleep 0.05 + done + printf 'FAIL timeout waiting for %s\n' "$path" >&2 + exit 1 +} + +cleanup_work() { + if [[ -n "${AM_WORK:-}" && -d "$AM_WORK" ]]; then + rm -rf "$AM_WORK" + fi +} diff --git a/packages/core/scripts/__tests__/macos-embedded-cleanup.sh b/packages/core/scripts/__tests__/macos-embedded-cleanup.sh new file mode 100755 index 0000000..821a428 --- /dev/null +++ b/packages/core/scripts/__tests__/macos-embedded-cleanup.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# EXIT cleanup must stop owned Postgres after Core or migrate exits. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +# shellcheck source=helpers.sh +source "$SCRIPT_DIR/helpers.sh" +trap cleanup_work EXIT + +make_workspace +run_entry migrate +assert_eq "migrate stops postgres once" "1" "$(wc -l < "$AM_STUB_PGCTL_STOPS" | tr -d ' ')" + +: > "$AM_STUB_PGCTL_STOPS" +export AM_CORE_STUB_EXIT=23 +set +e +run_entry start >/dev/null 2>&1 +status=$? +set -e +assert_eq "start preserves Core exit" "23" "$status" +assert_eq "nonzero Core stops postgres" "1" "$(wc -l < "$AM_STUB_PGCTL_STOPS" | tr -d ' ')" + +: > "$AM_STUB_PGCTL_STARTS" +: > "$AM_STUB_PGCTL_STOPS" +unset AM_CORE_STUB_EXIT +export AM_CORE_STUB_EXIT=0 +run_entry start +assert_eq "restart can start postgres" "1" "$(wc -l < "$AM_STUB_PGCTL_STARTS" | tr -d ' ')" diff --git a/packages/core/scripts/__tests__/macos-embedded-preserve-data.sh b/packages/core/scripts/__tests__/macos-embedded-preserve-data.sh new file mode 100755 index 0000000..4e3f2a9 --- /dev/null +++ b/packages/core/scripts/__tests__/macos-embedded-preserve-data.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# Nonempty non-cluster data dirs must be preserved. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +# shellcheck source=helpers.sh +source "$SCRIPT_DIR/helpers.sh" +trap cleanup_work EXIT + +make_workspace +mkdir -p "$STATE_ROOT/postgres" +printf 'keep\n' > "$STATE_ROOT/postgres/important.txt" + +set +e +run_entry migrate >/tmp/am-preserve.out 2>&1 +status=$? +set -e + +assert_eq "nonzero for unrecognized data dir" "1" "$status" +assert_file "important.txt preserved" "$STATE_ROOT/postgres/important.txt" +assert_eq "important.txt contents" "keep" "$(tr -d '\n' < "$STATE_ROOT/postgres/important.txt")" +assert_contains "actionable error" "missing PG_VERSION" "$(cat /tmp/am-preserve.out)" +rm -f /tmp/am-preserve.out diff --git a/packages/core/scripts/__tests__/macos-embedded-runtime-root.sh b/packages/core/scripts/__tests__/macos-embedded-runtime-root.sh new file mode 100755 index 0000000..e9b3e8b --- /dev/null +++ b/packages/core/scripts/__tests__/macos-embedded-runtime-root.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# Default RUNTIME_ROOT is the script directory (bundle Runtime/). +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +# shellcheck source=helpers.sh +source "$SCRIPT_DIR/helpers.sh" +trap cleanup_work EXIT + +make_workspace +cp "$ENTRYPOINT" "$RUNTIME_ROOT/macos-embedded-entrypoint.sh" +cp -R "$CORE_SCRIPTS_DIR/lib" "$RUNTIME_ROOT/lib" +chmod +x "$RUNTIME_ROOT/macos-embedded-entrypoint.sh" + +unset RUNTIME_ROOT +set +e +OPENAI_API_KEY=test-openai STATE_ROOT="$STATE_ROOT" \ + "$AM_WORK/Runtime/macos-embedded-entrypoint.sh" migrate >/tmp/am-runtime-root.out 2>&1 +status=$? +set -e +assert_eq "unset RUNTIME_ROOT migrate" "0" "$status" +assert_not_contains "found bundled Core" "missing Core binary" "$(cat /tmp/am-runtime-root.out)" +rm -f /tmp/am-runtime-root.out diff --git a/packages/core/scripts/__tests__/macos-embedded-scram.sh b/packages/core/scripts/__tests__/macos-embedded-scram.sh new file mode 100755 index 0000000..8600f7a --- /dev/null +++ b/packages/core/scripts/__tests__/macos-embedded-scram.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# Embedded initdb must use SCRAM and persist a private password. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +# shellcheck source=helpers.sh +source "$SCRIPT_DIR/helpers.sh" +trap cleanup_work EXIT + +make_workspace +run_entry migrate + +initdb_args="$(cat "$AM_STUB_INITDB_ARGS")" +assert_contains "scram local" "--auth-local=scram-sha-256" "$initdb_args" +assert_contains "scram host" "--auth-host=scram-sha-256" "$initdb_args" +assert_not_contains "no local trust" "--auth-local=trust" "$initdb_args" +assert_not_contains "no host trust" "--auth-host=trust" "$initdb_args" + +pw_file="$STATE_ROOT/state/postgres-password" +assert_file "password persisted" "$pw_file" +assert_eq "password file mode" "600" "$(stat -c '%a' "$pw_file" 2>/dev/null || stat -f '%OLp' "$pw_file")" + +hba="$(cat "$STATE_ROOT/postgres/pg_hba.conf")" +assert_contains "scram hba" "scram-sha-256" "$hba" +assert_not_contains "no trust leftover" "trust" "$hba" + +set +e +"$RUNTIME_ROOT/postgres/bin/psql" -w -d postgres >/dev/null 2>&1 +psql_status=$? +set -e +assert_eq "passwordless psql rejected" "2" "$psql_status" diff --git a/packages/core/scripts/__tests__/macos-embedded-spaces.sh b/packages/core/scripts/__tests__/macos-embedded-spaces.sh new file mode 100755 index 0000000..0fe8446 --- /dev/null +++ b/packages/core/scripts/__tests__/macos-embedded-spaces.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Conf-file pg_ctl start must keep Application Support socket paths intact. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +# shellcheck source=helpers.sh +source "$SCRIPT_DIR/helpers.sh" +trap cleanup_work EXIT + +make_workspace +run_entry migrate + +conf="$STATE_ROOT/postgres/conf.d/atomicmemory.conf" +assert_file "quoted socket conf" "$conf" +assert_contains "socket path with spaces" \ + "Application Support/AtomicMemory/postgres-run" "$(cat "$conf")" + +pgctl_log="$(cat "$AM_STUB_PGCTL_LOG")" +assert_not_contains "no -o path interpolation" "-o" "$pgctl_log" +assert_not_contains "no split Support fragment" $'Support/state\n' "$pgctl_log" diff --git a/packages/core/scripts/__tests__/macos-embedded-stop.sh b/packages/core/scripts/__tests__/macos-embedded-stop.sh new file mode 100755 index 0000000..d8deb81 --- /dev/null +++ b/packages/core/scripts/__tests__/macos-embedded-stop.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# stop must terminate a running Core process via the PID file. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +# shellcheck source=helpers.sh +source "$SCRIPT_DIR/helpers.sh" +trap cleanup_work EXIT + +make_workspace +export AM_CORE_STUB_SLEEP=1 +run_entry start >/dev/null 2>&1 & +launcher_pid=$! +wait_for_file "$STATE_ROOT/state/runtime.pid" + +core_pid="$(awk -F= '/^core_pid=/{print $2}' "$STATE_ROOT/state/runtime.pid")" +assert_eq "core still running before stop" "0" "$(kill -0 "$core_pid" && echo 0 || echo 1)" + +run_entry stop +set +e +kill -0 "$core_pid" 2>/dev/null +core_alive=$? +kill -0 "$launcher_pid" 2>/dev/null +launcher_alive=$? +set -e +assert_eq "core terminated" "1" "$core_alive" +assert_eq "launcher terminated" "1" "$launcher_alive" +assert_eq "postgres stopped" "1" "$(wc -l < "$AM_STUB_PGCTL_STOPS" | tr -d ' ')" diff --git a/packages/core/scripts/__tests__/package-darwin-pgvector.sh b/packages/core/scripts/__tests__/package-darwin-pgvector.sh new file mode 100755 index 0000000..74438b0 --- /dev/null +++ b/packages/core/scripts/__tests__/package-darwin-pgvector.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# Darwin pack validation and archive-name contract. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +PACK="$SCRIPT_DIR/../package-darwin-pgvector.sh" + +assert_eq() { + local name="$1" expected="$2" actual="$3" + if [[ "$expected" != "$actual" ]]; then + printf 'FAIL %s: expected %q got %q\n' "$name" "$expected" "$actual" >&2 + exit 1 + fi + printf 'PASS %s\n' "$name" +} + +work="$(mktemp -d "${TMPDIR:-/tmp}/am-pgpack.XXXXXX")" +trap 'rm -rf "$work"' EXIT +export AM_PACKAGE_ALLOW_NON_DARWIN=1 +export ARCH=arm64 + +empty="$work/empty" +mkdir -p "$empty/bin" "$empty/lib" "$empty/share" +set +e +POSTGRES_SOURCE_PREFIX="$empty" OUTPUT_DIR="$work/out-empty" "$PACK" >/dev/null 2>&1 +empty_status=$? +set -e +assert_eq "empty prefix fails" "1" "$empty_status" + +src="$work/src" +mkdir -p "$src/bin" "$src/lib" "$src/share/extension" +printf 'x\n' > "$src/bin/postgres" +printf 'x\n' > "$src/bin/pg_ctl" +printf 'x\n' > "$src/bin/initdb" +printf 'x\n' > "$src/bin/psql" +printf 'x\n' > "$src/lib/vector.dylib" +printf 'x\n' > "$src/share/extension/vector.control" +printf 'x\n' > "$src/share/extension/vector--0.8.0.sql" + +POSTGRES_SOURCE_PREFIX="$src" OUTPUT_DIR="$work/out" "$PACK" >/dev/null +tarball="$work/out/atomicmemory-postgres-pgvector-darwin-arm64.tar.gz" +if [[ ! -f "$tarball" ]]; then + printf 'FAIL missing %s\n' "$tarball" >&2 + exit 1 +fi +assert_eq "archive name" "atomicmemory-postgres-pgvector-darwin-arm64.tar.gz" "$(basename "$tarball")" + +rm -f "$src/share/extension/vector.control" +set +e +POSTGRES_SOURCE_PREFIX="$src" OUTPUT_DIR="$work/out-novec" "$PACK" >/dev/null 2>&1 +novec=$? +set -e +assert_eq "missing vector.control fails" "1" "$novec" diff --git a/packages/core/scripts/__tests__/public-artifact-leak-check.sh b/packages/core/scripts/__tests__/public-artifact-leak-check.sh new file mode 100755 index 0000000..01eda92 --- /dev/null +++ b/packages/core/scripts/__tests__/public-artifact-leak-check.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash +# Regression tests for the public package and final-image leak gate. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +CHECKER="$SCRIPT_DIR/../check-public-artifacts.sh" +WORK_DIR="$(mktemp -d "${TMPDIR:-/tmp}/am-public-artifacts.XXXXXX")" +PATTERN_FILE="$WORK_DIR/private-signatures.txt" +trap 'rm -rf "$WORK_DIR"' EXIT + +assert_fails() { + if "$@"; then + printf 'expected command to fail: %q\n' "$*" >&2 + exit 1 + fi +} + +make_archive() { + local directory="$1" + local output="$2" + tar -C "$directory" -czf "$output" . +} + +make_image_archive() { + local directory="$1" + local output="$2" + tar -C "$directory" -cf "$output" . +} + +make_fake_docker() { + mkdir -p "$WORK_DIR/bin" + cat > "$WORK_DIR/bin/docker" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +case "$1" in + create) printf '%s\n' fixture-container ;; + export) cat "$FAKE_IMAGE_TAR" ;; + rm) ;; + *) exit 1 ;; +esac +EOF + chmod +x "$WORK_DIR/bin/docker" +} + +mkdir -p \ + "$WORK_DIR/package" \ + "$WORK_DIR/image/app/dist" \ + "$WORK_DIR/image/app/scripts" \ + "$WORK_DIR/image/app/node_modules/@aws-sdk/client-s3" +printf 'safe package\n' > "$WORK_DIR/package/README.md" +printf 'safe image\n' > "$WORK_DIR/image/app/dist/server.js" +printf 'safe entrypoint\n' > "$WORK_DIR/image/app/scripts/docker-entrypoint.sh" +printf '{}\n' > "$WORK_DIR/image/app/openapi.json" +printf 'openapi: 3.1.0\n' > "$WORK_DIR/image/app/openapi.yaml" +printf '{"name":"@atomicmemory/core"}\n' > "$WORK_DIR/image/app/package.json" +# This is public documentation from a pinned production dependency. The gate +# must inspect our shipped application files without treating this generic term +# as enterprise-only content. +printf 'Configure a customer-cmk for S3 encryption.\n' \ + > "$WORK_DIR/image/app/node_modules/@aws-sdk/client-s3/README.md" +make_archive "$WORK_DIR/package" "$WORK_DIR/package.tgz" +make_image_archive "$WORK_DIR/image" "$WORK_DIR/image.tar" +make_fake_docker +printf '%s\n' \ + 'private-fixture-marker' \ + 'private-test-endpoint.invalid' \ + > "$PATTERN_FILE" + +PATH="$WORK_DIR/bin:$PATH" FAKE_IMAGE_TAR="$WORK_DIR/image.tar" \ + "$CHECKER" "$WORK_DIR/package.tgz" fixture:image "$PATTERN_FILE" + +printf 'private-fixture-marker\n' > "$WORK_DIR/package/README.md" +make_archive "$WORK_DIR/package" "$WORK_DIR/package.tgz" +assert_fails env PATH="$WORK_DIR/bin:$PATH" FAKE_IMAGE_TAR="$WORK_DIR/image.tar" \ + "$CHECKER" "$WORK_DIR/package.tgz" fixture:image "$PATTERN_FILE" + +printf 'safe package\n' > "$WORK_DIR/package/README.md" +printf 'private-fixture-marker\n' > "$WORK_DIR/image/app/dist/server.js" +make_archive "$WORK_DIR/package" "$WORK_DIR/package.tgz" +make_image_archive "$WORK_DIR/image" "$WORK_DIR/image.tar" +assert_fails env PATH="$WORK_DIR/bin:$PATH" FAKE_IMAGE_TAR="$WORK_DIR/image.tar" \ + "$CHECKER" "$WORK_DIR/package.tgz" fixture:image "$PATTERN_FILE" + +printf 'https://private-test-endpoint.invalid\n' > "$WORK_DIR/image/app/dist/server.js" +make_archive "$WORK_DIR/package" "$WORK_DIR/package.tgz" +make_image_archive "$WORK_DIR/image" "$WORK_DIR/image.tar" +assert_fails env PATH="$WORK_DIR/bin:$PATH" FAKE_IMAGE_TAR="$WORK_DIR/image.tar" \ + "$CHECKER" "$WORK_DIR/package.tgz" fixture:image "$PATTERN_FILE" diff --git a/packages/core/scripts/__tests__/public-source-export.sh b/packages/core/scripts/__tests__/public-source-export.sh new file mode 100644 index 0000000..d765dbe --- /dev/null +++ b/packages/core/scripts/__tests__/public-source-export.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# Verifies the public git archive omits the ECR-only hosted compatibility overlay. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +if ! REPO_ROOT="$(git -C "$SCRIPT_DIR/../../../.." rev-parse --show-toplevel 2>/dev/null)"; then + printf '%s\n' 'SKIP public source export audit outside a Git checkout' + exit 0 +fi + +ARCHIVE_DIR="$(mktemp -d "${TMPDIR:-/tmp}/am-public-source.XXXXXX")" +trap 'rm -rf "$ARCHIVE_DIR"' EXIT +ARCHIVE_PATHS="$(git -C "$REPO_ROOT" archive --format=tar --worktree-attributes HEAD | tar -tf -)" +ARCHIVE_PATHS_WITH_BOUNDARIES=$'\n'"$ARCHIVE_PATHS"$'\n' + +if [[ "$ARCHIVE_PATHS_WITH_BOUNDARIES" == *$'\npackages/core/hosted/'* ]]; then + printf '%s\n' 'private hosted overlay is present in the public source archive' >&2 + exit 1 +fi + +if [[ "$ARCHIVE_PATHS_WITH_BOUNDARIES" != *$'\npackages/core/Dockerfile\n'* ]]; then + printf '%s\n' 'public source archive unexpectedly omits packages/core/Dockerfile' >&2 + exit 1 +fi + +for detector_path in \ + packages/core/scripts/check-public-artifacts.sh \ + packages/core/scripts/__tests__/public-artifact-leak-check.sh +do + if [[ "$ARCHIVE_PATHS_WITH_BOUNDARIES" != *$'\n'"$detector_path"$'\n'* ]]; then + printf 'public source archive unexpectedly omits detector fixture: %s\n' "$detector_path" >&2 + exit 1 + fi +done + +git -C "$REPO_ROOT" archive --format=tar --worktree-attributes HEAD | tar -x -C "$ARCHIVE_DIR" +( + cd "$ARCHIVE_DIR/packages/core" + bash scripts/__tests__/run.sh +) diff --git a/packages/core/scripts/__tests__/run.sh b/packages/core/scripts/__tests__/run.sh new file mode 100755 index 0000000..79e7980 --- /dev/null +++ b/packages/core/scripts/__tests__/run.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# Runner for Core script fixture tests (Linux-CI safe). +set -euo pipefail + +DIR="$(cd "$(dirname "$0")" && pwd)" +status=0 + +for test in \ + "$DIR"/macos-embedded-spaces.sh \ + "$DIR"/macos-embedded-preserve-data.sh \ + "$DIR"/macos-embedded-scram.sh \ + "$DIR"/macos-embedded-cleanup.sh \ + "$DIR"/macos-embedded-stop.sh \ + "$DIR"/macos-embedded-runtime-root.sh \ + "$DIR"/public-artifact-leak-check.sh \ + "$DIR"/public-source-export.sh \ + "$DIR"/package-darwin-pgvector.sh +do + printf 'RUN %s\n' "$(basename "$test")" + if ! bash "$test"; then + status=1 + fi +done + +printf 'RUN write-lean-staging-package.test.mjs\n' +if ! node --test "$DIR/write-lean-staging-package.test.mjs"; then + status=1 +fi + +exit "$status" diff --git a/packages/core/scripts/__tests__/stubs/atomicmemory-core b/packages/core/scripts/__tests__/stubs/atomicmemory-core new file mode 100755 index 0000000..347c4e0 --- /dev/null +++ b/packages/core/scripts/__tests__/stubs/atomicmemory-core @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# Stub Core binary for launcher lifecycle tests. +set -euo pipefail + +log_path="${AM_CORE_STUB_LOG:-/tmp/am-core.stub.log}" +printf '%s\n' "${1:-}" >> "$log_path" +if [[ "${1:-}" == "migrate" ]]; then + exit "${AM_CORE_MIGRATE_EXIT:-0}" +fi +if [[ "${1:-}" == "start" ]]; then + if [[ "${AM_CORE_STUB_SLEEP:-}" == "1" ]]; then + sleep 60 + exit 0 + fi + exit "${AM_CORE_STUB_EXIT:-0}" +fi +exit 0 diff --git a/packages/core/scripts/__tests__/stubs/initdb b/packages/core/scripts/__tests__/stubs/initdb new file mode 100755 index 0000000..d9d6832 --- /dev/null +++ b/packages/core/scripts/__tests__/stubs/initdb @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# Records initdb argv and creates a minimal cluster layout for launcher tests. +set -euo pipefail + +: > "${AM_STUB_INITDB_ARGS:-/tmp/am-initdb.args}" +for arg in "$@"; do + printf 'ARG %s\n' "$arg" >> "${AM_STUB_INITDB_ARGS:-/tmp/am-initdb.args}" +done +data="" +prev="" +for arg in "$@"; do + if [[ "$prev" == "-D" ]]; then + data="$arg" + fi + if [[ "$arg" == --auth-local=trust || "$arg" == --auth-host=trust ]]; then + echo "initdb stub: trust auth is forbidden" >&2 + exit 1 + fi + prev="$arg" +done +if [[ -z "$data" ]]; then + echo "initdb stub: missing -D" >&2 + exit 1 +fi +mkdir -p "$data/conf.d" +printf '17\n' > "$data/PG_VERSION" +printf "# stub postgresql.conf\n" > "$data/postgresql.conf" +printf "local all all scram-sha-256\n" > "$data/pg_hba.conf" diff --git a/packages/core/scripts/__tests__/stubs/pg_ctl b/packages/core/scripts/__tests__/stubs/pg_ctl new file mode 100755 index 0000000..1e9959d --- /dev/null +++ b/packages/core/scripts/__tests__/stubs/pg_ctl @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# Records pg_ctl argv and start/stop/reload counts for launcher tests. +set -euo pipefail + +for arg in "$@"; do + printf 'ARG %s\n' "$arg" >> "${AM_STUB_PGCTL_LOG:-/tmp/am-pgctl.log}" +done +printf '%s\n' '---' >> "${AM_STUB_PGCTL_LOG:-/tmp/am-pgctl.log}" + +data="" +cmd="" +prev="" +for arg in "$@"; do + if [[ "$prev" == "-D" ]]; then + data="$arg" + fi + case "$arg" in + start|stop|reload) cmd="$arg" ;; + esac + prev="$arg" +done + +case "$cmd" in + start) + printf 'start\n' >> "${AM_STUB_PGCTL_STARTS:-/tmp/am-pgctl.starts}" + if [[ -n "$data" ]]; then + printf '1\n' > "$data/postmaster.pid" + fi + ;; + stop) + printf 'stop\n' >> "${AM_STUB_PGCTL_STOPS:-/tmp/am-pgctl.stops}" + if [[ -n "$data" ]]; then + rm -f "$data/postmaster.pid" + fi + ;; + reload) + printf 'reload\n' >> "${AM_STUB_PGCTL_RELOADS:-/tmp/am-pgctl.reloads}" + ;; +esac diff --git a/packages/core/scripts/__tests__/stubs/postgres b/packages/core/scripts/__tests__/stubs/postgres new file mode 100755 index 0000000..c6f0ea7 --- /dev/null +++ b/packages/core/scripts/__tests__/stubs/postgres @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +# Stub postgres executable so require_runtime_layout succeeds. +exit 0 diff --git a/packages/core/scripts/__tests__/stubs/psql b/packages/core/scripts/__tests__/stubs/psql new file mode 100755 index 0000000..5d52f9c --- /dev/null +++ b/packages/core/scripts/__tests__/stubs/psql @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# Stub psql: reject passwordless (-w) connections and succeed otherwise. +set -euo pipefail + +for arg in "$@"; do + if [[ "$arg" == "-w" ]]; then + echo "psql: fe_sendauth: no password supplied" >&2 + exit 2 + fi +done +for arg in "$@"; do + if [[ "$arg" == *"pg_database"* ]]; then + printf '1\n' + exit 0 + fi +done +exit 0 diff --git a/packages/core/scripts/__tests__/write-lean-staging-package.test.mjs b/packages/core/scripts/__tests__/write-lean-staging-package.test.mjs new file mode 100644 index 0000000..c0a8494 --- /dev/null +++ b/packages/core/scripts/__tests__/write-lean-staging-package.test.mjs @@ -0,0 +1,94 @@ +/** + * Unit tests for the lean Deno-compile staging helper. + */ + +import { mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { + LEAN_DROP_DEPENDENCIES, + buildLeanPackage, + lockVersionsFromPnpmList, + pinExactVersions, + pruneDeployedTree, + verifyStagingVersions, + writePinnedLeanPackageFile, +} from '../lib/write-lean-staging-package.mjs'; + +describe('write-lean-staging-package', () => { + it('drops the lean-profile packages from dependencies', () => { + const lean = buildLeanPackage({ + name: '@atomicmemory/core', + version: '1.2.1', + dependencies: { + express: '5.1.0', + '@huggingface/transformers': '3.8.1', + viem: '2.0.0', + tsx: '4.0.0', + }, + }); + assert.deepEqual(Object.keys(lean.dependencies), ['express']); + for (const name of LEAN_DROP_DEPENDENCIES) { + assert.equal(lean.dependencies[name], undefined); + } + }); + + it('pins kept dependencies to exact lockfile versions', () => { + const staging = mkdtempSync(join(tmpdir(), 'am-lean-pin-')); + try { + const srcPath = join(staging, 'source.json'); + const lockPath = join(staging, 'lock.json'); + const destPath = join(staging, 'package.json'); + writeFileSync(srcPath, JSON.stringify({ + name: '@atomicmemory/core', + version: '1.2.1', + dependencies: { express: '^5.1.0', viem: '^2.0.0' }, + })); + writeFileSync(lockPath, JSON.stringify({ + dependencies: { + express: { version: '5.1.0(zod@4.0.0)' }, + viem: { version: '2.0.0' }, + }, + })); + const lean = writePinnedLeanPackageFile(srcPath, lockPath, destPath); + assert.deepEqual(lean.dependencies, { express: '5.1.0' }); + assert.deepEqual( + pinExactVersions({ dependencies: { express: '^5.1.0' } }, { express: '5.1.0' }).dependencies, + { express: '5.1.0' }, + ); + } finally { + rmSync(staging, { recursive: true, force: true }); + } + }); + + it('fails closed when a kept dependency has no lock version', () => { + assert.throws( + () => pinExactVersions({ dependencies: { express: '^5.1.0' } }, {}), + /missing lockfile version for express/, + ); + }); + + it('prunes dropped node_modules and verifies lock versions', () => { + const staging = mkdtempSync(join(tmpdir(), 'am-lean-')); + try { + mkdirSync(join(staging, 'node_modules', 'express'), { recursive: true }); + mkdirSync(join(staging, 'node_modules', 'viem'), { recursive: true }); + writeFileSync(join(staging, 'package.json'), JSON.stringify({ + name: '@atomicmemory/core', + version: '1.2.1', + dependencies: { express: '^5.1.0', viem: '^2.0.0' }, + })); + writeFileSync(join(staging, 'node_modules', 'express', 'package.json'), '{"version":"5.1.0"}\n'); + writeFileSync(join(staging, 'node_modules', 'viem', 'package.json'), '{"version":"2.0.0"}\n'); + pruneDeployedTree(staging); + const pkg = JSON.parse(readFileSync(join(staging, 'package.json'), 'utf8')); + assert.equal(pkg.dependencies.viem, undefined); + const versions = lockVersionsFromPnpmList({ dependencies: { express: { version: '5.1.0' } } }); + verifyStagingVersions(staging, versions); + } finally { + rmSync(staging, { recursive: true, force: true }); + } + }); +}); diff --git a/packages/core/scripts/check-public-artifacts.sh b/packages/core/scripts/check-public-artifacts.sh new file mode 100755 index 0000000..d669164 --- /dev/null +++ b/packages/core/scripts/check-public-artifacts.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +# Fails a release when private hosted identifiers enter public Core artifacts. +set -euo pipefail + +if [ "$#" -ne 3 ]; then + printf 'usage: %s \n' "$0" >&2 + exit 64 +fi + +PACKAGE_TARBALL="$1" +IMAGE_REFERENCE="$2" +FORBIDDEN_PATTERN_FILE="$3" +WORK_DIR="$(mktemp -d "${TMPDIR:-/tmp}/am-public-artifacts.XXXXXX")" +PACKAGE_DIR="$WORK_DIR/package" +IMAGE_DIR="$WORK_DIR/image" +CONTAINER_ID="" + +if [ ! -s "$FORBIDDEN_PATTERN_FILE" ]; then + printf 'forbidden pattern file is missing or empty: %s\n' "$FORBIDDEN_PATTERN_FILE" >&2 + exit 64 +fi + +cleanup() { + if [ -n "$CONTAINER_ID" ]; then + docker rm -f "$CONTAINER_ID" >/dev/null 2>&1 || true + fi + rm -rf "$WORK_DIR" +} +trap cleanup EXIT + +scan_artifact() { + local label="$1" + shift + local artifact_path + local scan_status + local matches + for artifact_path in "$@"; do + if [ ! -e "$artifact_path" ]; then + printf 'expected artifact path missing from %s: %s\n' "$label" "$artifact_path" >&2 + exit 1 + fi + done + if matches="$(rg --files-with-matches --hidden --no-ignore -a -i \ + -f "$FORBIDDEN_PATTERN_FILE" "$@")"; then + : + else + scan_status=$? + if [ "$scan_status" -ne 1 ]; then + printf 'unable to scan %s with forbidden pattern file: %s\n' \ + "$label" "$FORBIDDEN_PATTERN_FILE" >&2 + exit "$scan_status" + fi + matches="" + fi + if [ -n "$matches" ]; then + printf 'private identifier found in %s:\n%s\n' "$label" "$matches" >&2 + exit 1 + fi +} + +mkdir -p "$PACKAGE_DIR" "$IMAGE_DIR" +tar -xzf "$PACKAGE_TARBALL" -C "$PACKAGE_DIR" +scan_artifact 'package tarball' "$PACKAGE_DIR" + +CONTAINER_ID="$(docker create "$IMAGE_REFERENCE")" +docker export "$CONTAINER_ID" | tar -x -C "$IMAGE_DIR" +scan_artifact 'final image application inputs' \ + "$IMAGE_DIR/app/dist" \ + "$IMAGE_DIR/app/scripts" \ + "$IMAGE_DIR/app/openapi.json" \ + "$IMAGE_DIR/app/openapi.yaml" \ + "$IMAGE_DIR/app/package.json" + +printf 'public artifact leak check passed\n' diff --git a/packages/core/scripts/deno-compile-core.sh b/packages/core/scripts/deno-compile-core.sh new file mode 100755 index 0000000..007e10f --- /dev/null +++ b/packages/core/scripts/deno-compile-core.sh @@ -0,0 +1,130 @@ +#!/usr/bin/env bash +# Experimental Deno compile for AtomicMemory Core (API-only, external Postgres). +# +# Builds a lean single executable at dist-bin/atomicmemory-core that runs +# `atomicmemory-core start` and `atomicmemory-core migrate` without Node or +# Deno installed at runtime. Postgres/pgvector must be provided separately. +# +# Size strategy: write a lockfile-pinned lean package.json, then +# `pnpm install --prod` with a hoisted linker *outside* the workspace graph +# so Deno never embeds the isolated `.pnpm` store (transformers, viem, …). +# +# Prerequisites: +# brew install deno +# pnpm install (from repo root or packages/core) +# +# Usage: +# ./scripts/deno-compile-core.sh +# OUTPUT=./dist-bin/my-core ./scripts/deno-compile-core.sh +# +# See docs/deno-compile-spike.md for findings and limitations. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +REPO_ROOT="$(cd "$PROJECT_DIR/../.." && pwd)" +OUTPUT="${OUTPUT:-$PROJECT_DIR/dist-bin/atomicmemory-core}" +STAGING="${STAGING:-$PROJECT_DIR/.deno-compile-staging}" +LEAN_HELPER="$SCRIPT_DIR/lib/write-lean-staging-package.mjs" + +if ! command -v deno >/dev/null 2>&1; then + echo "deno-compile-core: deno not found. Install with: brew install deno" >&2 + exit 1 +fi +if ! command -v pnpm >/dev/null 2>&1; then + echo "deno-compile-core: pnpm not found (needed for lockfile-faithful install)" >&2 + exit 1 +fi + +write_hoisted_npmrc() { + cat > "$1/.npmrc" <<'EOF' +node-linker=hoisted +shamefully-hoist=true +package-import-method=copy +ignore-workspace=true +EOF +} + +# Deno --node-modules-dir=manual walks node_modules/.pnpm. Hoisted+copy +# leaves real packages at the root, so the store can be removed. +strip_isolated_store() { + local nm="$1/node_modules" + if [[ -e "$nm/@huggingface/transformers" || -e "$nm/viem" ]]; then + echo "deno-compile-core: lean tree still contains a dropped package" >&2 + exit 1 + fi + if [[ -L "$nm/express" && "$(readlink "$nm/express")" == *".pnpm"* ]]; then + echo "deno-compile-core: hoisted packages still symlink into .pnpm" >&2 + exit 1 + fi + rm -rf "$nm/.pnpm" +} + +echo "[deno-compile-core] deno $(deno --version | head -1)" +echo "[deno-compile-core] building TypeScript to dist/ ..." +(cd "$PROJECT_DIR" && pnpm run build) + +if [[ ! -f "$PROJECT_DIR/dist/bin.js" ]]; then + echo "deno-compile-core: dist/bin.js missing after build" >&2 + exit 1 +fi + +echo "[deno-compile-core] writing lockfile-pinned lean package to $STAGING ..." +rm -rf "$STAGING" +mkdir -p "$STAGING" +LOCK_JSON="$STAGING/.pnpm-list.json" +(cd "$REPO_ROOT" && pnpm list --filter @atomicmemory/core --depth 0 --prod --json) > "$LOCK_JSON" +node "$LEAN_HELPER" write-pinned "$PROJECT_DIR/package.json" "$LOCK_JSON" "$STAGING/package.json" +write_hoisted_npmrc "$STAGING" +printf 'packages:\n - "."\n' > "$STAGING/pnpm-workspace.yaml" +cp -R "$PROJECT_DIR/dist" "$STAGING/dist" +cp "$PROJECT_DIR/openapi.json" "$PROJECT_DIR/openapi.yaml" "$STAGING/" + +echo "[deno-compile-core] installing hoisted prod tree in $STAGING ..." +( + cd "$STAGING" + pnpm install --prod --ignore-scripts \ + --ignore-workspace \ + --config.node-linker=hoisted \ + --config.package-import-method=copy +) +node "$LEAN_HELPER" verify "$STAGING" "$LOCK_JSON" +rm -f "$LOCK_JSON" +strip_isolated_store "$STAGING" + +mkdir -p "$(dirname "$OUTPUT")" + +# --node-modules-dir=manual against the hoisted lean tree. `pnpm deploy` +# left an isolated `.pnpm` store (~900 packages) that Deno never finished +# embedding inside the 45-minute CI budget. +# --allow-write is required for local_fs uploads, state, caches, and logs. +# --no-check: Deno rejects import.meta.dirname typing in openapi-spec. +echo "[deno-compile-core] compiling to $OUTPUT ..." +( + cd "$STAGING" + deno compile \ + --no-check \ + --node-modules-dir=manual \ + --allow-net \ + --allow-env \ + --allow-read \ + --allow-write \ + --allow-sys \ + --allow-ffi \ + --include=openapi.json \ + --include=openapi.yaml \ + --include=package.json \ + --include=dist/db/migrations \ + --output="$OUTPUT" \ + dist/bin.js +) + +ls -lh "$OUTPUT" +if [[ "$(uname -s)" == "Darwin" ]]; then + ARCH_NAME="darwin-$(uname -m)" + PLATFORM_OUTPUT="$(dirname "$OUTPUT")/atomicmemory-core-${ARCH_NAME}" + cp "$OUTPUT" "$PLATFORM_OUTPUT" + ls -lh "$PLATFORM_OUTPUT" +fi +echo "[deno-compile-core] done. See docs/deno-compile-spike.md for runtime notes." diff --git a/packages/core/scripts/docker-entrypoint.sh b/packages/core/scripts/docker-entrypoint.sh index 1c4a566..7d1d3c6 100644 --- a/packages/core/scripts/docker-entrypoint.sh +++ b/packages/core/scripts/docker-entrypoint.sh @@ -68,34 +68,8 @@ resolve_core_api_key() { log "CORE_API_KEY generated and persisted to $CORE_API_KEY_FILE" } -cloud_tier_api_url() { - case "${1:-dev}" in - dev) printf '%s' 'https://api.dev.atomicstrata.ai' ;; - staging) printf '%s' 'https://api.staging.atomicstrata.ai' ;; - production|prod) printf '%s' 'https://api.atomicstrata.ai' ;; - *) - log "Unknown CLOUD_ENV: ${1}" - exit 1 - ;; - esac -} - -cloud_tier_memory_origin() { - case "${1:-dev}" in - dev) printf '%s' 'https://memory.dev.atomicstrata.ai' ;; - staging) printf '%s' 'https://memory.staging.atomicstrata.ai' ;; - production|prod) printf '%s' 'https://memory.atomicstrata.ai' ;; - *) - log "Unknown CLOUD_ENV: ${1}" - exit 1 - ;; - esac -} - -# When running self-hosted Core for connected-local, apply tier defaults so -# operators only pass OPENAI_API_KEY + ATOMICMEMORY_API_KEY. The presence of -# ATOMICMEMORY_API_KEY is the single switch that turns connected-local on; -# CLOUD_PROJECT_ID is optional (Core trusts the token's project_id when unset). +# When running self-hosted Core for connected-local, derive related settings +# from URLs supplied by the operator. Public images never select a hosted tier. apply_connected_local_defaults() { if is_hosted_deployment_env "${RAW_STORAGE_DEPLOYMENT_ENV:-local}"; then return @@ -105,24 +79,23 @@ apply_connected_local_defaults() { return fi - local tier="${CLOUD_ENV:-dev}" - local api_url memory_origin - - api_url="$(cloud_tier_api_url "$tier")" - memory_origin="$(cloud_tier_memory_origin "$tier")" + if [ -z "${ATOMICMEMORY_API_URL:-}" ]; then + log "ATOMICMEMORY_API_URL is required when ATOMICMEMORY_API_KEY is set" + exit 1 + fi + if [ -z "${ALLOWED_ORIGINS:-}" ]; then + log "ALLOWED_ORIGINS is required when ATOMICMEMORY_API_KEY is set" + exit 1 + fi + local api_url="${ATOMICMEMORY_API_URL%/}" export CLOUD_TRACE_SYNC_ENABLED="${CLOUD_TRACE_SYNC_ENABLED:-true}" - export ATOMICMEMORY_API_URL="${ATOMICMEMORY_API_URL:-$api_url}" - + export ATOMICMEMORY_API_URL="$api_url" export CLOUD_JWKS_URL="${CLOUD_JWKS_URL:-${api_url}/.well-known/atomic-core/jwks.json}" export CLOUD_JWT_ISSUER="${CLOUD_JWT_ISSUER:-$api_url}" export CLOUD_JWT_AUDIENCE="${CLOUD_JWT_AUDIENCE:-atomicmemory-core}" - export ALLOWED_ORIGINS="${ALLOWED_ORIGINS:-$memory_origin}" export CLOUD_JWT_STATIC_KEY_FALLBACK="${CLOUD_JWT_STATIC_KEY_FALLBACK:-true}" - if [ "$tier" = "dev" ]; then - export CLOUD_JWT_LEGACY_DEFAULT_MEMORY_USER_ID="${CLOUD_JWT_LEGACY_DEFAULT_MEMORY_USER_ID:-default}" - fi - log "Connected-local defaults applied (CLOUD_ENV=$tier, api=$ATOMICMEMORY_API_URL)" + log "Connected-local defaults applied (api=$ATOMICMEMORY_API_URL)" } stop_postgres() { @@ -235,7 +208,7 @@ run_migrations() { fi log "Running migrations..." - gosu appuser ./node_modules/.bin/tsx src/db/migrate.ts "${migrate_args[@]}" + gosu appuser node dist/db/migrate.js "${migrate_args[@]}" } configure_local_defaults diff --git a/packages/core/scripts/lib/macos-embedded-lifecycle.sh b/packages/core/scripts/lib/macos-embedded-lifecycle.sh new file mode 100644 index 0000000..43fc313 --- /dev/null +++ b/packages/core/scripts/lib/macos-embedded-lifecycle.sh @@ -0,0 +1,147 @@ +#!/usr/bin/env bash +# Process-lifecycle helpers for the macOS embedded launcher. +# +# Installs a shared EXIT cleanup that stops only processes this invocation +# started, and implements `stop` via a validated PID file so a second +# invocation can terminate Core and Postgres. + +pid_is_numeric() { + [[ "$1" =~ ^[0-9]+$ ]] +} + +pid_args_match() { + local pid="$1" + local needle="$2" + local args + args="$(ps -p "$pid" -o args= 2>/dev/null || true)" + [[ -n "$args" && "$args" == *"$needle"* ]] +} + +write_runtime_pid() { + mkdir -p "$CORE_STATE_DIR" + { + printf 'launcher_pid=%s\n' "$$" + printf 'core_pid=%s\n' "$APP_PID" + printf 'core_bin=%s\n' "$CORE_BIN" + printf 'port=%s\n' "${PORT:-}" + } > "$RUNTIME_PID_FILE" +} + +read_runtime_pid() { + LAUNCHER_PID="" + SAVED_CORE_PID="" + SAVED_CORE_BIN="" + SAVED_PORT="" + [[ -f "$RUNTIME_PID_FILE" ]] || return 1 + while IFS='=' read -r key value; do + case "$key" in + launcher_pid) LAUNCHER_PID="$value" ;; + core_pid) SAVED_CORE_PID="$value" ;; + core_bin) SAVED_CORE_BIN="$value" ;; + port) SAVED_PORT="$value" ;; + esac + done < "$RUNTIME_PID_FILE" +} + +stop_owned_pid() { + local pid="$1" + local needle="$2" + if ! pid_is_numeric "$pid"; then + return 1 + fi + if ! kill -0 "$pid" 2>/dev/null; then + return 0 + fi + if ! pid_args_match "$pid" "$needle"; then + return 1 + fi + kill -TERM "$pid" 2>/dev/null || true + wait "$pid" 2>/dev/null || true + local i + for i in 1 2 3 4 5 6 7 8 9 10; do + kill -0 "$pid" 2>/dev/null || return 0 + sleep 0.05 + done + kill -KILL "$pid" 2>/dev/null || true + return 0 +} + +on_exit() { + local status=$? + if [[ "${CLEANING_UP:-false}" = "true" ]]; then + return 0 + fi + CLEANING_UP=true + if [[ -n "${APP_PID:-}" ]] && kill -0 "$APP_PID" 2>/dev/null; then + kill -TERM "$APP_PID" 2>/dev/null || true + wait "$APP_PID" 2>/dev/null || true + fi + stop_postgres + rm -f "${RUNTIME_PID_FILE:-}" + exit "$status" +} + +on_signal() { + log "Shutting down..." + exit 0 +} + +install_lifecycle_traps() { + CLEANING_UP=false + trap on_exit EXIT + trap on_signal SIGTERM SIGINT +} + +stop_recorded_core() { + if ! pid_is_numeric "${SAVED_CORE_PID:-}"; then + return 0 + fi + if ! kill -0 "$SAVED_CORE_PID" 2>/dev/null; then + return 0 + fi + if stop_owned_pid "$SAVED_CORE_PID" "atomicmemory-core"; then + return 0 + fi + log "refusing to signal unrecognized Core pid $SAVED_CORE_PID" + return 1 +} + +stop_recorded_launcher() { + if ! pid_is_numeric "${LAUNCHER_PID:-}"; then + return 0 + fi + if [[ "$LAUNCHER_PID" = "$$" ]]; then + return 0 + fi + if ! kill -0 "$LAUNCHER_PID" 2>/dev/null; then + return 0 + fi + if stop_owned_pid "$LAUNCHER_PID" "macos-embedded-entrypoint"; then + return 0 + fi + log "refusing to signal unrecognized launcher pid $LAUNCHER_PID" + return 1 +} + +stop_postmaster_if_present() { + if [[ -f "$EMBEDDED_POSTGRES_DATA_DIR/postmaster.pid" ]]; then + POSTGRES_STARTED=true + stop_postgres + fi +} + +cmd_stop() { + trap - EXIT + local failed=false + if read_runtime_pid; then + stop_recorded_core || failed=true + stop_recorded_launcher || failed=true + rm -f "$RUNTIME_PID_FILE" + fi + stop_postmaster_if_present + if [[ "$failed" = "true" ]]; then + log "stop failed: process identity could not be validated" + exit 1 + fi + log "Stopped" +} diff --git a/packages/core/scripts/lib/macos-embedded-postgres.sh b/packages/core/scripts/lib/macos-embedded-postgres.sh new file mode 100644 index 0000000..2db04fe --- /dev/null +++ b/packages/core/scripts/lib/macos-embedded-postgres.sh @@ -0,0 +1,196 @@ +#!/usr/bin/env bash +# Embedded Postgres helpers for the macOS DMG launcher. +# +# Owns initdb safety, SCRAM auth, quoted conf (paths may contain spaces), +# and DATABASE_URL construction. Sourced by macos-embedded-entrypoint.sh. + +urlencode_component() { + local raw="$1" + local out="" c hex i + for ((i = 0; i < ${#raw}; i++)); do + c="${raw:i:1}" + case "$c" in + [a-zA-Z0-9.~_-]) out+="$c" ;; + *) + printf -v hex '%%%02X' "'$c" + out+="$hex" + ;; + esac + done + printf '%s' "$out" +} + +pg_conf_quote() { + local value="$1" + value="${value//\'/\'\'}" + printf "'%s'" "$value" +} + +persist_postgres_password() { + local password="$1" + mkdir -p "$CORE_STATE_DIR" + printf '%s\n' "$password" > "$POSTGRES_PASSWORD_FILE" + chmod 600 "$POSTGRES_PASSWORD_FILE" +} + +resolve_postgres_password() { + if [[ -n "${EMBEDDED_POSTGRES_PASSWORD:-}" ]]; then + persist_postgres_password "$EMBEDDED_POSTGRES_PASSWORD" + return + fi + if [[ -s "$POSTGRES_PASSWORD_FILE" ]]; then + EMBEDDED_POSTGRES_PASSWORD="$(tr -d '[:space:]' < "$POSTGRES_PASSWORD_FILE")" + if [[ -n "$EMBEDDED_POSTGRES_PASSWORD" ]]; then + return + fi + fi + EMBEDDED_POSTGRES_PASSWORD="$(openssl rand -hex 32)" + persist_postgres_password "$EMBEDDED_POSTGRES_PASSWORD" +} + +write_embedded_postgres_conf() { + local pgconf="$EMBEDDED_POSTGRES_DATA_DIR/postgresql.conf" + local confdir="$EMBEDDED_POSTGRES_DATA_DIR/conf.d" + mkdir -p "$confdir" + if [[ -f "$pgconf" ]] && ! grep -q "^include_dir = 'conf.d'" "$pgconf"; then + printf "\ninclude_dir = 'conf.d'\n" >> "$pgconf" + fi + cat > "$confdir/atomicmemory.conf" < "$EMBEDDED_POSTGRES_DATA_DIR/pg_hba.conf" <<'EOF' +# AtomicMemory embedded cluster — SCRAM only. +local all all scram-sha-256 +host all all 127.0.0.1/32 scram-sha-256 +host all all ::1/128 scram-sha-256 +EOF +} + +cluster_uses_trust() { + local hba="$EMBEDDED_POSTGRES_DATA_DIR/pg_hba.conf" + [[ -f "$hba" ]] && grep -Eq '^[[:space:]]*(local|host)[[:space:]].*[[:space:]]trust([[:space:]]|$)' "$hba" +} + +data_dir_is_nonempty() { + local dir="$1" + [[ -d "$dir" ]] && [[ -n "$(ls -A "$dir" 2>/dev/null)" ]] +} + +refuse_unrecognized_data_dir() { + log "refusing to initialize $EMBEDDED_POSTGRES_DATA_DIR: directory is nonempty and is not a Postgres cluster (missing PG_VERSION)" + exit 1 +} + +init_new_cluster() { + mkdir -p "$EMBEDDED_POSTGRES_DATA_DIR" + chmod 700 "$EMBEDDED_POSTGRES_DATA_DIR" + log "Initializing embedded Postgres at $EMBEDDED_POSTGRES_DATA_DIR" + LC_ALL="$LC_ALL" LANG="$LANG" "$PG_BIN/initdb" \ + -D "$EMBEDDED_POSTGRES_DATA_DIR" \ + --username="$EMBEDDED_POSTGRES_USER" \ + --locale="$LC_ALL" \ + --encoding=UTF8 \ + --auth-local=scram-sha-256 \ + --auth-host=scram-sha-256 \ + --pwfile="$POSTGRES_PASSWORD_FILE" + write_embedded_postgres_conf + write_embedded_pg_hba +} + +prepare_embedded_data_dir() { + if [[ -s "$EMBEDDED_POSTGRES_DATA_DIR/PG_VERSION" ]]; then + log "Using existing embedded Postgres data at $EMBEDDED_POSTGRES_DATA_DIR" + write_embedded_postgres_conf + return + fi + if data_dir_is_nonempty "$EMBEDDED_POSTGRES_DATA_DIR"; then + refuse_unrecognized_data_dir + fi + init_new_cluster +} + +run_psql() { + PGPASSWORD="$EMBEDDED_POSTGRES_PASSWORD" "$PG_BIN/psql" \ + -h "$EMBEDDED_POSTGRES_RUN_DIR" \ + -p "$EMBEDDED_POSTGRES_PORT" \ + -U "$EMBEDDED_POSTGRES_USER" \ + "$@" +} + +stop_postgres() { + if [[ "${POSTGRES_STARTED:-false}" != "true" ]]; then + return 0 + fi + log "Stopping embedded Postgres..." + "$PG_BIN/pg_ctl" -D "$EMBEDDED_POSTGRES_DATA_DIR" -m fast -w stop >/dev/null 2>&1 || true + POSTGRES_STARTED=false +} + +start_postgres_process() { + log "Starting embedded Postgres on 127.0.0.1:$EMBEDDED_POSTGRES_PORT ..." + "$PG_BIN/pg_ctl" \ + -D "$EMBEDDED_POSTGRES_DATA_DIR" \ + -l "$LOG_DIR/postgres.log" \ + -w \ + start +} + +set_role_password() { + local escaped="${EMBEDDED_POSTGRES_PASSWORD//\'/\'\'}" + run_psql -d postgres -v ON_ERROR_STOP=1 \ + -c "ALTER ROLE \"$EMBEDDED_POSTGRES_USER\" PASSWORD '$escaped'" \ + >/dev/null +} + +reload_postgres() { + "$PG_BIN/pg_ctl" -D "$EMBEDDED_POSTGRES_DATA_DIR" reload >/dev/null +} + +ensure_embedded_database() { + if run_psql -d postgres -tAc "SELECT 1 FROM pg_database WHERE datname = '$EMBEDDED_POSTGRES_DB'" | grep -qx 1; then + return + fi + log "Creating database $EMBEDDED_POSTGRES_DB" + run_psql -d postgres -v ON_ERROR_STOP=1 \ + -c "CREATE DATABASE \"$EMBEDDED_POSTGRES_DB\"" >/dev/null +} + +ensure_vector_extension() { + log "Ensuring pgvector extension" + run_psql -d "$EMBEDDED_POSTGRES_DB" -v ON_ERROR_STOP=1 \ + -c "CREATE EXTENSION IF NOT EXISTS vector" >/dev/null +} + +export_embedded_database_url() { + local encoded + encoded="$(urlencode_component "$EMBEDDED_POSTGRES_PASSWORD")" + export DATABASE_URL="postgresql://${EMBEDDED_POSTGRES_USER}:${encoded}@127.0.0.1:${EMBEDDED_POSTGRES_PORT}/${EMBEDDED_POSTGRES_DB}" +} + +lock_down_cluster_auth() { + set_role_password + write_embedded_pg_hba + reload_postgres +} + +start_embedded_postgres() { + configure_locale + mkdir -p "$EMBEDDED_POSTGRES_RUN_DIR" "$LOG_DIR" + export PATH="$PG_BIN:$PATH" + export DYLD_LIBRARY_PATH="${PG_ROOT}/lib:${DYLD_LIBRARY_PATH:-}" + resolve_postgres_password + export PGPASSWORD="$EMBEDDED_POSTGRES_PASSWORD" + prepare_embedded_data_dir + start_postgres_process + POSTGRES_STARTED=true + lock_down_cluster_auth + ensure_embedded_database + ensure_vector_extension + export_embedded_database_url +} diff --git a/packages/core/scripts/lib/write-lean-staging-package.mjs b/packages/core/scripts/lib/write-lean-staging-package.mjs new file mode 100644 index 0000000..7f62923 --- /dev/null +++ b/packages/core/scripts/lib/write-lean-staging-package.mjs @@ -0,0 +1,214 @@ +/** + * Lean Deno-compile staging helpers. + * + * Writes a production package.json without the local-transformers and + * Filecoin/viem stacks, pins kept dependencies to `pnpm list` lock + * versions, and verifies the hoisted install against those versions. + */ + +import { readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +/** Packages dropped from the lean binary profile. */ +export const LEAN_DROP_DEPENDENCIES = Object.freeze([ + '@huggingface/transformers', + '@filoz/synapse-core', + '@filoz/synapse-sdk', + 'viem', + 'multiformats', + 'tsx', +]); + +const dropSet = new Set(LEAN_DROP_DEPENDENCIES); + +/** + * Filters a dependency map to the lean runtime set. + * + * @param {Record} dependencies + * @returns {Record} + */ +export function filterLeanDependencies(dependencies) { + return Object.fromEntries( + Object.entries(dependencies).filter(([name]) => !dropSet.has(name)), + ); +} + +/** + * Builds the lean package.json object from a full Core manifest. + * + * @param {{ name: string, version: string, dependencies?: Record }} src + */ +export function buildLeanPackage(src) { + return { + name: src.name, + version: src.version, + type: 'module', + main: './dist/index.js', + bin: { 'atomicmemory-core': './dist/bin.js' }, + dependencies: filterLeanDependencies(src.dependencies ?? {}), + }; +} + +/** + * Rewrites the deployed package.json and deletes dropped node_modules trees. + * + * @param {string} stagingDir + */ +export function pruneDeployedTree(stagingDir) { + const pkgPath = join(stagingDir, 'package.json'); + const src = JSON.parse(readFileSync(pkgPath, 'utf8')); + const lean = buildLeanPackage(src); + writeFileSync(pkgPath, `${JSON.stringify(lean, null, 2)}\n`); + for (const name of LEAN_DROP_DEPENDENCIES) { + rmSync(join(stagingDir, 'node_modules', name), { recursive: true, force: true }); + } + return lean; +} + +/** + * Reads `pnpm list --json --depth 0` output into a name → version map. + * + * @param {unknown} listJson + * @returns {Record} + */ +export function lockVersionsFromPnpmList(listJson) { + const root = Array.isArray(listJson) ? listJson[0] : listJson; + if (!root || typeof root !== 'object') { + throw new Error('pnpm list JSON did not contain a package object'); + } + const deps = /** @type {{ dependencies?: Record }} */ ( + root + ).dependencies ?? {}; + /** @type {Record} */ + const versions = {}; + for (const [name, info] of Object.entries(deps)) { + const version = typeof info === 'string' ? info : info?.version; + if (typeof version === 'string' && version.length > 0) { + versions[name] = normalizeListedVersion(version); + } + } + return versions; +} + +/** + * Pins lean dependencies to exact lockfile versions. + * + * @param {{ dependencies?: Record }} pkg + * @param {Record} versions + */ +export function pinExactVersions(pkg, versions) { + const dependencies = { ...pkg.dependencies }; + for (const name of Object.keys(dependencies)) { + const version = versions[name]; + if (typeof version !== 'string' || version.length === 0) { + throw new Error(`lean staging: missing lockfile version for ${name}`); + } + dependencies[name] = version; + } + return { ...pkg, dependencies }; +} + +/** + * @param {string} srcPath + * @param {string} lockJsonPath + * @param {string} destPath + */ +export function writePinnedLeanPackageFile(srcPath, lockJsonPath, destPath) { + const src = JSON.parse(readFileSync(srcPath, 'utf8')); + const listJson = JSON.parse(readFileSync(lockJsonPath, 'utf8')); + const lean = pinExactVersions(buildLeanPackage(src), lockVersionsFromPnpmList(listJson)); + writeFileSync(destPath, `${JSON.stringify(lean, null, 2)}\n`); + return lean; +} + +/** + * pnpm list may append peer suffixes such as `1.2.3(zod@4.0.0)`. + * + * @param {string} version + */ +function normalizeListedVersion(version) { + const paren = version.indexOf('('); + return paren === -1 ? version : version.slice(0, paren); +} + +/** + * Fails if a kept staging dependency does not match the lockfile version. + * + * @param {string} stagingDir + * @param {Record} lockVersions + */ +export function verifyStagingVersions(stagingDir, lockVersions) { + const pkg = JSON.parse(readFileSync(join(stagingDir, 'package.json'), 'utf8')); + const dependencies = pkg.dependencies ?? {}; + for (const name of Object.keys(dependencies)) { + const installed = readInstalledVersion(stagingDir, name); + const expected = lockVersions[name]; + if (!expected) { + throw new Error(`lean staging: ${name} is not in the pnpm lock resolution set`); + } + if (installed !== expected) { + throw new Error(`lean staging: ${name} is ${installed}, lockfile has ${expected}`); + } + } +} + +/** + * @param {string} stagingDir + * @param {string} name + */ +function readInstalledVersion(stagingDir, name) { + const installedPath = join(stagingDir, 'node_modules', name, 'package.json'); + try { + const installed = JSON.parse(readFileSync(installedPath, 'utf8')); + if (typeof installed.version === 'string') { + return installed.version; + } + } catch { + throw new Error(`lean staging: missing installed package ${name}`); + } + throw new Error(`lean staging: ${name} has no version`); +} + +/** + * @param {string} stagingDir + * @param {string} destPath + */ +export function writeLeanPackageFile(srcPath, destPath) { + const src = JSON.parse(readFileSync(srcPath, 'utf8')); + const lean = buildLeanPackage(src); + writeFileSync(destPath, `${JSON.stringify(lean, null, 2)}\n`); + return lean; +} + +function main(argv) { + const [command, ...rest] = argv; + if (command === 'write') { + const [srcPath, destPath] = rest; + const lean = writeLeanPackageFile(srcPath, destPath); + process.stdout.write(`lean deps: ${Object.keys(lean.dependencies).sort().join(', ')}\n`); + return; + } + if (command === 'write-pinned') { + const [srcPath, lockJsonPath, destPath] = rest; + const lean = writePinnedLeanPackageFile(srcPath, lockJsonPath, destPath); + process.stdout.write(`lean pinned deps: ${Object.keys(lean.dependencies).sort().join(', ')}\n`); + return; + } + if (command === 'prune') { + pruneDeployedTree(rest[0]); + return; + } + if (command === 'verify') { + const [stagingDir, lockJsonPath] = rest; + const listJson = JSON.parse(readFileSync(lockJsonPath, 'utf8')); + verifyStagingVersions(stagingDir, lockVersionsFromPnpmList(listJson)); + return; + } + throw new Error('usage: write-lean-staging-package.mjs write|write-pinned|prune|verify ...'); +} + +const thisFile = fileURLToPath(import.meta.url); +if (process.argv[1] && resolve(process.argv[1]) === thisFile) { + main(process.argv.slice(2)); +} diff --git a/packages/core/scripts/macos-compiled-storage-smoke.sh b/packages/core/scripts/macos-compiled-storage-smoke.sh new file mode 100755 index 0000000..bd4a85a --- /dev/null +++ b/packages/core/scripts/macos-compiled-storage-smoke.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +# Manual/macOS smoke: compiled-binary upload/read/delete against a reachable DB. +# +# Requires a compiled Core binary and DATABASE_URL. Not part of Linux CI. +# +# export DATABASE_URL='postgresql://...' +# export CORE_API_KEY='...' +# export STORAGE_KEY_HMAC_SECRET='000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f' +# export RAW_STORAGE_MODE=managed_blob +# export RAW_STORAGE_PROVIDER=local_fs +# export RAW_STORAGE_LOCAL_FS_ROOT=/tmp/am-raw +# export RAW_STORAGE_PREFIX=local/core +# ./scripts/macos-compiled-storage-smoke.sh ./dist-bin/atomicmemory-core + +set -euo pipefail + +CORE_BIN="${1:-}" +if [[ -z "$CORE_BIN" || ! -x "$CORE_BIN" ]]; then + echo "usage: macos-compiled-storage-smoke.sh /path/to/atomicmemory-core" >&2 + exit 1 +fi +if [[ -z "${DATABASE_URL:-}" || -z "${CORE_API_KEY:-}" ]]; then + echo "DATABASE_URL and CORE_API_KEY are required" >&2 + exit 1 +fi + +export RAW_STORAGE_DEPLOYMENT_ENV="${RAW_STORAGE_DEPLOYMENT_ENV:-local}" +export RAW_STORAGE_MODE="${RAW_STORAGE_MODE:-managed_blob}" +export RAW_STORAGE_PROVIDER="${RAW_STORAGE_PROVIDER:-local_fs}" +export RAW_STORAGE_LOCAL_FS_ROOT="${RAW_STORAGE_LOCAL_FS_ROOT:-/tmp/am-raw-smoke}" +export RAW_STORAGE_PREFIX="${RAW_STORAGE_PREFIX:-local/core}" +export PORT="${PORT:-17359}" +export LISTEN_HOST="${LISTEN_HOST:-127.0.0.1}" +mkdir -p "$RAW_STORAGE_LOCAL_FS_ROOT" + +"$CORE_BIN" migrate +"$CORE_BIN" start & +core_pid=$! +cleanup() { + kill -TERM "$core_pid" 2>/dev/null || true + wait "$core_pid" 2>/dev/null || true +} +trap cleanup EXIT + +base="http://${LISTEN_HOST}:${PORT}" +for _ in $(seq 1 40); do + if curl -sf "$base/health" >/dev/null; then + break + fi + sleep 0.1 +done + +doc_id="$(python3 - <<'PY' +import uuid +print(uuid.uuid4()) +PY +)" +# Limits first so a missing write grant fails before PUT. +curl -sf -H "Authorization: Bearer ${CORE_API_KEY}" "$base/v1/documents/limits" >/dev/null +printf 'smoke-body' > /tmp/am-smoke-body.bin +# PUT/GET/DELETE the raw object. Route paths match the public documents API. +curl -sf -X PUT \ + -H "Authorization: Bearer ${CORE_API_KEY}" \ + -H "Content-Type: application/octet-stream" \ + --data-binary @/tmp/am-smoke-body.bin \ + "$base/v1/documents/${doc_id}/raw" >/dev/null +curl -sf -H "Authorization: Bearer ${CORE_API_KEY}" \ + "$base/v1/documents/${doc_id}/raw" >/tmp/am-smoke-got.bin +cmp /tmp/am-smoke-body.bin /tmp/am-smoke-got.bin +curl -sf -X DELETE -H "Authorization: Bearer ${CORE_API_KEY}" \ + "$base/v1/documents/${doc_id}/raw" >/dev/null +echo "compiled storage smoke ok" diff --git a/packages/core/scripts/macos-embedded-entrypoint.sh b/packages/core/scripts/macos-embedded-entrypoint.sh new file mode 100755 index 0000000..e9c3a35 --- /dev/null +++ b/packages/core/scripts/macos-embedded-entrypoint.sh @@ -0,0 +1,204 @@ +#!/usr/bin/env bash +# macOS embedded runtime entrypoint for AtomicMemory DMG packaging. +# +# Lifecycle parity with scripts/docker-entrypoint.sh without gosu/docker. +# Intended to be invoked by AtomicMemory.app CoreRuntimeSupervisor or manually: +# +# export RUNTIME_ROOT=/path/to/Resources/Runtime +# export STATE_ROOT="$HOME/Library/Application Support/AtomicMemory" +# export CORE_API_KEY=... +# export OPENAI_API_KEY=... +# ./macos-embedded-entrypoint.sh start +# +# Commands: +# start — init embedded Postgres (if needed), migrate, start Core (foreground) +# migrate — run migrations only (Postgres must be reachable) +# stop — stop Core and embedded Postgres via validated PID file +# help — usage +# +# Bundle layout must include this script, lib/, atomicmemory-core, and postgres/. +# See docs/macos-embedded-runtime.md + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +# shellcheck source=lib/macos-embedded-postgres.sh +source "$SCRIPT_DIR/lib/macos-embedded-postgres.sh" +# shellcheck source=lib/macos-embedded-lifecycle.sh +source "$SCRIPT_DIR/lib/macos-embedded-lifecycle.sh" + +RUNTIME_ROOT="${RUNTIME_ROOT:-$SCRIPT_DIR}" +STATE_ROOT="${STATE_ROOT:-$HOME/Library/Application Support/AtomicMemory}" + +CORE_BIN="${RUNTIME_ROOT}/atomicmemory-core" +PG_ROOT="${RUNTIME_ROOT}/postgres" +PG_BIN="${PG_ROOT}/bin" + +EMBEDDED_POSTGRES_DATA_DIR="${EMBEDDED_POSTGRES_DATA_DIR:-$STATE_ROOT/postgres}" +EMBEDDED_POSTGRES_RUN_DIR="${EMBEDDED_POSTGRES_RUN_DIR:-$STATE_ROOT/postgres-run}" +EMBEDDED_POSTGRES_PORT="${EMBEDDED_POSTGRES_PORT:-54329}" +EMBEDDED_POSTGRES_USER="${EMBEDDED_POSTGRES_USER:-atomicmemory}" +EMBEDDED_POSTGRES_DB="${EMBEDDED_POSTGRES_DB:-atomicmemory}" + +CORE_STATE_DIR="${CORE_STATE_DIR:-$STATE_ROOT/state}" +CORE_API_KEY_FILE="$CORE_STATE_DIR/core-api-key" +POSTGRES_PASSWORD_FILE="$CORE_STATE_DIR/postgres-password" +RUNTIME_PID_FILE="$CORE_STATE_DIR/runtime.pid" +LOCAL_STORAGE_KEY_HMAC_SECRET="000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f" +LOG_DIR="${LOG_DIR:-$STATE_ROOT/logs}" + +APP_PID="" +POSTGRES_STARTED=false +CLEANING_UP=false + +log() { + printf '[macos-embedded] %s\n' "$*" >&2 +} + +configure_locale() { + # GUI-launched apps often inherit an empty/minimal locale; initdb requires a valid one. + export LANG="${LANG:-en_US.UTF-8}" + export LC_ALL="${LC_ALL:-en_US.UTF-8}" +} + +usage() { + cat < + +Environment: + RUNTIME_ROOT Bundle Resources/Runtime (default: this script's directory) + STATE_ROOT Application Support root (default: ~/Library/Application Support/AtomicMemory) + DATABASE_URL "embedded" or postgresql://... (default: embedded) + CORE_API_KEY API key (generated/persisted under STATE_ROOT/state if unset) + STORAGE_KEY_HMAC_SECRET HMAC secret (local default if unset) + OPENAI_API_KEY Required for lean embedding profile + LISTEN_HOST Core bind address (default: 127.0.0.1) + EMBEDDED_POSTGRES_PORT Loopback port (default: 54329) +EOF +} + +require_runtime_layout() { + if [[ ! -x "$CORE_BIN" ]]; then + log "missing Core binary: $CORE_BIN" + exit 1 + fi + if [[ ! -x "$PG_BIN/postgres" ]]; then + log "missing Postgres binary: $PG_BIN/postgres" + exit 1 + fi +} + +persist_core_api_key() { + local key="$1" + mkdir -p "$CORE_STATE_DIR" + printf '%s\n' "$key" > "$CORE_API_KEY_FILE" + chmod 600 "$CORE_API_KEY_FILE" +} + +resolve_core_api_key() { + if [[ -n "${CORE_API_KEY:-}" ]]; then + persist_core_api_key "$CORE_API_KEY" + export CORE_API_KEY + log "CORE_API_KEY from environment (persisted)" + return + fi + if [[ -s "$CORE_API_KEY_FILE" ]]; then + CORE_API_KEY="$(tr -d '[:space:]' < "$CORE_API_KEY_FILE")" + if [[ -n "$CORE_API_KEY" ]]; then + export CORE_API_KEY + log "CORE_API_KEY loaded from $CORE_API_KEY_FILE" + return + fi + fi + CORE_API_KEY="$(openssl rand -hex 32)" + persist_core_api_key "$CORE_API_KEY" + export CORE_API_KEY + log "CORE_API_KEY generated and persisted" +} + +reject_unsupported_storage() { + if [[ "${RAW_STORAGE_MODE:-pointer_only}" = "managed_blob" \ + && "${RAW_STORAGE_PROVIDER:-local_fs}" != "local_fs" ]]; then + log "lean profile requires RAW_STORAGE_PROVIDER=local_fs when RAW_STORAGE_MODE=managed_blob" + exit 1 + fi +} + +configure_local_defaults() { + export RAW_STORAGE_DEPLOYMENT_ENV="${RAW_STORAGE_DEPLOYMENT_ENV:-local}" + export PORT="${PORT:-17350}" + export LISTEN_HOST="${LISTEN_HOST:-127.0.0.1}" + reject_unsupported_storage + resolve_core_api_key + if [[ -z "${STORAGE_KEY_HMAC_SECRET:-}" ]]; then + export STORAGE_KEY_HMAC_SECRET="$LOCAL_STORAGE_KEY_HMAC_SECRET" + log "STORAGE_KEY_HMAC_SECRET not set; using local default" + fi + export EMBEDDING_PROVIDER="${EMBEDDING_PROVIDER:-openai}" + export EMBEDDING_DIMENSIONS="${EMBEDDING_DIMENSIONS:-1536}" + if [[ -z "${OPENAI_API_KEY:-}" ]]; then + log "OPENAI_API_KEY is required for lean embedding profile" + exit 1 + fi +} + +run_migrations() { + log "Running migrations..." + "$CORE_BIN" migrate +} + +start_core_foreground() { + log "Starting AtomicMemory Core on ${LISTEN_HOST}:${PORT} ..." + "$CORE_BIN" start >>"$LOG_DIR/core.log" 2>&1 & + APP_PID="$!" + write_runtime_pid + wait "$APP_PID" +} + +prepare_database() { + if [[ "${DATABASE_URL:-embedded}" = "embedded" ]]; then + start_embedded_postgres + return + fi + log "Using external DATABASE_URL" +} + +cmd_migrate() { + require_runtime_layout + configure_local_defaults + prepare_database + run_migrations +} + +cmd_start() { + require_runtime_layout + configure_local_defaults + mkdir -p "$LOG_DIR" + prepare_database + run_migrations + start_core_foreground +} + +main() { + configure_locale + local cmd="${1:-help}" + case "$cmd" in + start) + install_lifecycle_traps + cmd_start + ;; + migrate) + install_lifecycle_traps + cmd_migrate + ;; + stop) cmd_stop ;; + help|-h|--help) usage ;; + *) + log "unknown command: $cmd" + usage + exit 1 + ;; + esac +} + +main "$@" diff --git a/packages/core/scripts/package-darwin-pgvector.sh b/packages/core/scripts/package-darwin-pgvector.sh new file mode 100755 index 0000000..8ae6a35 --- /dev/null +++ b/packages/core/scripts/package-darwin-pgvector.sh @@ -0,0 +1,123 @@ +#!/usr/bin/env bash +# Assemble a relocatable Postgres 17 + pgvector tree for macOS DMG embedding. +# +# Writes: +# dist-bin/postgres/ (staging tree) +# dist-bin/atomicmemory-postgres-pgvector-darwin-arm64.tar.gz +# +# Prerequisites: +# POSTGRES_SOURCE_PREFIX — root containing bin/, lib/, share/ (e.g. prepared PG17 build) +# PGVECTOR_LIB_DIR (optional) — directory with vector.dylib for PG17 +# +# See docs/darwin-pgvector-pack.md + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +OUTPUT_DIR="${OUTPUT_DIR:-$PROJECT_DIR/dist-bin}" +STAGING="$OUTPUT_DIR/postgres" +ARCH="${ARCH:-$(uname -m)}" +TARBALL="$OUTPUT_DIR/atomicmemory-postgres-pgvector-darwin-${ARCH}.tar.gz" + +log() { + printf '[package-darwin-pgvector] %s\n' "$*" >&2 +} + +require_darwin() { + if [[ "${AM_PACKAGE_ALLOW_NON_DARWIN:-}" = "1" ]]; then + return 0 + fi + if [[ "$(uname -s)" != "Darwin" ]]; then + log "this script must run on macOS" + exit 1 + fi +} + +require_source_tree() { + if [[ -z "${POSTGRES_SOURCE_PREFIX:-}" ]]; then + log "POSTGRES_SOURCE_PREFIX is required (see docs/darwin-pgvector-pack.md)" + exit 1 + fi + local sub + for sub in bin lib share; do + if [[ ! -d "$POSTGRES_SOURCE_PREFIX/$sub" ]]; then + log "missing $POSTGRES_SOURCE_PREFIX/$sub" + exit 1 + fi + done +} + +require_file() { + if [[ ! -e "$1" ]]; then + log "missing required file: $1" + exit 1 + fi +} + +copy_pgvector_extras() { + if [[ -z "${PGVECTOR_LIB_DIR:-}" ]]; then + return 0 + fi + if [[ ! -d "$PGVECTOR_LIB_DIR" ]]; then + log "PGVECTOR_LIB_DIR is not a directory: $PGVECTOR_LIB_DIR" + exit 1 + fi + log "copying pgvector libs from $PGVECTOR_LIB_DIR" + mkdir -p "$STAGING/lib" + cp -R "$PGVECTOR_LIB_DIR/." "$STAGING/lib/" + if [[ -n "${PGVECTOR_SHARE_DIR:-}" ]]; then + if [[ ! -d "$PGVECTOR_SHARE_DIR" ]]; then + log "PGVECTOR_SHARE_DIR is not a directory: $PGVECTOR_SHARE_DIR" + exit 1 + fi + mkdir -p "$STAGING/share/extension" + cp -R "$PGVECTOR_SHARE_DIR/." "$STAGING/share/extension/" + fi +} + +validate_runtime_binaries() { + local exe + for exe in postgres pg_ctl initdb psql; do + require_file "$STAGING/bin/$exe" + done +} + +validate_pgvector_files() { + require_file "$STAGING/share/extension/vector.control" + if ! compgen -G "$STAGING/share/extension/vector--*.sql" > /dev/null; then + log "missing pgvector SQL files under $STAGING/share/extension" + exit 1 + fi + if [[ ! -e "$STAGING/lib/vector.dylib" && ! -e "$STAGING/lib/postgresql/vector.dylib" ]]; then + log "missing pgvector library (vector.dylib)" + exit 1 + fi +} + +stage_postgres_tree() { + log "staging from $POSTGRES_SOURCE_PREFIX -> $STAGING" + rm -rf "$STAGING" + mkdir -p "$STAGING" + cp -R "$POSTGRES_SOURCE_PREFIX/bin" "$STAGING/" + cp -R "$POSTGRES_SOURCE_PREFIX/lib" "$STAGING/" + cp -R "$POSTGRES_SOURCE_PREFIX/share" "$STAGING/" + copy_pgvector_extras + chmod -R u+w "$STAGING/bin" + chmod +x "$STAGING/bin/"* +} + +write_tarball() { + log "creating $TARBALL" + mkdir -p "$OUTPUT_DIR" + tar -czf "$TARBALL" -C "$OUTPUT_DIR" postgres + ls -lh "$TARBALL" + log "done. Smoke test per docs/darwin-pgvector-pack.md" +} + +require_darwin +require_source_tree +stage_postgres_tree +validate_runtime_binaries +validate_pgvector_files +write_tarball diff --git a/packages/core/src/__tests__/config-env.test.ts b/packages/core/src/__tests__/config-env.test.ts index a7bcc2d..09a4a19 100644 --- a/packages/core/src/__tests__/config-env.test.ts +++ b/packages/core/src/__tests__/config-env.test.ts @@ -21,6 +21,11 @@ const trackedEnvNames = [ 'ANTHROPIC_API_KEY', 'CODEX_AUTH_PATH', 'CODEX_HOME', + 'EXTRACTION_PROMPT_VARIANT', + 'EXTRACTION_MAX_TOKENS', + 'AUDN_MAX_TOKENS', + 'AUDN_JSON_SCHEMA', + 'LISTEN_HOST', ] as const; const originalEnv = Object.fromEntries( trackedEnvNames.map((name) => [name, process.env[name]]), @@ -126,4 +131,89 @@ describe('config env loading', () => { 'CORE_TEST_SCOPE_ALLOW_PATTERN must be a valid JavaScript regular expression', ); }); + + it('defaults EXTRACTION_PROMPT_VARIANT to full', async () => { + delete process.env.EXTRACTION_PROMPT_VARIANT; + vi.resetModules(); + const { config } = await import('../config.js'); + expect(config.extractionPromptVariant).toBe('full'); + }); + + it('accepts compact EXTRACTION_PROMPT_VARIANT exactly', async () => { + process.env.EXTRACTION_PROMPT_VARIANT = 'compact'; + vi.resetModules(); + const { config } = await import('../config.js'); + expect(config.extractionPromptVariant).toBe('compact'); + }); + + it.each(['Compact', 'compact ', 'comapct'])( + 'rejects invalid EXTRACTION_PROMPT_VARIANT value %s', + async (value) => { + process.env.EXTRACTION_PROMPT_VARIANT = value; + vi.resetModules(); + await expect(import('../config.js')).rejects.toThrow( + "EXTRACTION_PROMPT_VARIANT must be 'full' or 'compact'", + ); + }, + ); + + it('loads bounded decode token caps from the environment', async () => { + process.env.EXTRACTION_MAX_TOKENS = '768'; + process.env.AUDN_MAX_TOKENS = '128'; + vi.resetModules(); + const { config } = await import('../config.js'); + expect(config.extractionMaxTokens).toBe(768); + expect(config.audnMaxTokens).toBe(128); + }); + + it.each(['oops', '0', '-1', '128junk', '1.5', '1e3'])( + 'rejects invalid AUDN_MAX_TOKENS value %s', + async (value) => { + process.env.AUDN_MAX_TOKENS = value; + vi.resetModules(); + await expect(import('../config.js')).rejects.toThrow('AUDN_MAX_TOKENS must be a positive integer'); + }, + ); + + it('rejects AUDN_MAX_TOKENS above the documented upper bound', async () => { + process.env.AUDN_MAX_TOKENS = '999999999'; + vi.resetModules(); + await expect(import('../config.js')).rejects.toThrow('AUDN_MAX_TOKENS must be at most 4096'); + }); + + it('defaults AUDN_JSON_SCHEMA to false', async () => { + delete process.env.AUDN_JSON_SCHEMA; + vi.resetModules(); + const { config } = await import('../config.js'); + expect(config.audnJsonSchema).toBe(false); + }); + + it('accepts AUDN_JSON_SCHEMA=true exactly', async () => { + process.env.AUDN_JSON_SCHEMA = 'true'; + vi.resetModules(); + const { config } = await import('../config.js'); + expect(config.audnJsonSchema).toBe(true); + }); + + it('leaves listenHost unset when LISTEN_HOST is absent', async () => { + delete process.env.LISTEN_HOST; + vi.resetModules(); + const { config } = await import('../config.js'); + expect(config.listenHost).toBeUndefined(); + }); + + it('loads LISTEN_HOST from the environment', async () => { + process.env.LISTEN_HOST = '127.0.0.1'; + vi.resetModules(); + const { config } = await import('../config.js'); + expect(config.listenHost).toBe('127.0.0.1'); + }); + + it.each(['TRUE', '1', 'yes'])('rejects invalid AUDN_JSON_SCHEMA value %s', async (value) => { + process.env.AUDN_JSON_SCHEMA = value; + vi.resetModules(); + await expect(import('../config.js')).rejects.toThrow( + "AUDN_JSON_SCHEMA must be 'true' or 'false'", + ); + }); }); diff --git a/packages/core/src/__tests__/deployment-config.test.ts b/packages/core/src/__tests__/deployment-config.test.ts index 26ece68..34db825 100644 --- a/packages/core/src/__tests__/deployment-config.test.ts +++ b/packages/core/src/__tests__/deployment-config.test.ts @@ -148,6 +148,18 @@ describe('deployment configuration', () => { expect(envExample).toContain('VOYAGE_DOCUMENT_MODEL'); expect(envExample).toContain('VOYAGE_QUERY_MODEL'); }); + + it('documents compact extraction as a reduced-capability opt-in', () => { + const envExample = readEnvExample(); + expect(envExample).toContain('EXTRACTION_PROMPT_VARIANT'); + expect(envExample).toContain('EXTRACTION_MAX_TOKENS'); + expect(envExample).toContain('AUDN_MAX_TOKENS'); + expect(envExample).toContain('assistant recommendations'); + expect(envExample).toContain('contact-info'); + expect(envExample).toContain('short-input'); + expect(envExample).toContain('AUDN mutation policy'); + expect(envExample).toContain('json_object'); + }); }); describe('Dockerfile', () => { @@ -191,7 +203,13 @@ describe('deployment configuration', () => { expect(entrypoint).toContain('persist_core_api_key'); expect(entrypoint).toContain('is_hosted_deployment_env'); expect(entrypoint).toContain('apply_connected_local_defaults'); - expect(entrypoint).toContain('CLOUD_ENV'); + expect(entrypoint).toContain('ATOMICMEMORY_API_URL is required'); + expect(entrypoint).toContain('ALLOWED_ORIGINS is required'); + expect(entrypoint).not.toContain('CLOUD_ENV'); + expect(entrypoint).not.toContain('api.dev.atomicstrata.ai'); + expect(entrypoint).not.toContain('api.staging.atomicstrata.ai'); + expect(entrypoint).not.toContain('memory.dev.atomicstrata.ai'); + expect(entrypoint).not.toContain('memory.staging.atomicstrata.ai'); expect(entrypoint).toContain('LOCAL_DOCKER_STORAGE_KEY_HMAC_SECRET='); expect(entrypoint).toContain('production|staging'); expect(entrypoint).toContain('CORE_API_KEY is required'); @@ -217,9 +235,12 @@ describe('deployment configuration', () => { expect(dockerfile).toContain('USER'); }); - it('copies tsconfig.json for tsx runtime', () => { + it('runs compiled distribution without copying source or tests', () => { const dockerfile = readDockerfile(); - expect(dockerfile).toContain('tsconfig.json'); + expect(dockerfile).toContain('COPY --from=builder /repo/packages/core/dist ./dist'); + expect(dockerfile).toContain('CMD ["node", "dist/server.js"]'); + expect(dockerfile).not.toContain('COPY packages/core/src ./src'); + expect(dockerfile).not.toContain('./node_modules/.bin/tsx'); }); }); }); diff --git a/packages/core/src/__tests__/macos-embedded-config.test.ts b/packages/core/src/__tests__/macos-embedded-config.test.ts new file mode 100644 index 0000000..9c3e7c2 --- /dev/null +++ b/packages/core/src/__tests__/macos-embedded-config.test.ts @@ -0,0 +1,53 @@ +/** + * Static contracts for the macOS embedded runtime and Deno compile path. + */ + +import { readFileSync } from 'node:fs'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { describe, expect, it } from 'vitest'; + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '../..'); + +function read(rel: string): string { + return readFileSync(resolve(root, rel), 'utf8'); +} + +describe('macos embedded runtime contracts', () => { + it('launcher defaults RUNTIME_ROOT to the script directory and binds loopback', () => { + const entry = read('scripts/macos-embedded-entrypoint.sh'); + expect(entry).toContain('RUNTIME_ROOT="${RUNTIME_ROOT:-$SCRIPT_DIR}"'); + expect(entry).toContain('LISTEN_HOST="${LISTEN_HOST:-127.0.0.1}"'); + expect(entry).not.toContain('$(cd "$SCRIPT_DIR/.." && pwd)'); + }); + + it('embedded Postgres uses SCRAM and refuses unrecognized data dirs', () => { + const pg = read('scripts/lib/macos-embedded-postgres.sh'); + expect(pg).toContain('--auth-local=scram-sha-256'); + expect(pg).toContain('--auth-host=scram-sha-256'); + expect(pg).toContain('refusing to initialize'); + expect(pg).not.toContain('--auth-local=trust'); + expect(pg).not.toContain('rm -rf "$EMBEDDED_POSTGRES_DATA_DIR"'); + }); + + it('compile path is lockfile-faithful and grants write access', () => { + const compile = read('scripts/deno-compile-core.sh'); + expect(compile).toContain('write-pinned'); + expect(compile).toContain('pnpm list --filter @atomicmemory/core --depth 0 --prod --json'); + expect(compile).toContain('pnpm install --prod --ignore-scripts'); + expect(compile).toContain('--ignore-workspace'); + expect(compile).toContain('--config.node-linker=hoisted'); + expect(compile).toContain('strip_isolated_store'); + expect(compile).toContain('--allow-write'); + expect(compile).not.toContain('pnpm --filter @atomicmemory/core deploy --prod'); + expect(compile).not.toMatch(/^\s*npm install/m); + }); + + it('postgres pack validates pgvector and emits a darwin- archive name', () => { + const pack = read('scripts/package-darwin-pgvector.sh'); + expect(pack).toContain('atomicmemory-postgres-pgvector-darwin-${ARCH}.tar.gz'); + expect(pack).toContain('vector.control'); + expect(pack).toContain('vector.dylib'); + expect(pack).not.toContain('2>/dev/null || true'); + }); +}); diff --git a/packages/core/src/__tests__/memory-route-config-override.test.ts b/packages/core/src/__tests__/memory-route-config-override.test.ts index 8cfd599..1a63297 100644 --- a/packages/core/src/__tests__/memory-route-config-override.test.ts +++ b/packages/core/src/__tests__/memory-route-config-override.test.ts @@ -6,8 +6,8 @@ * 1. Absent override → no `X-Atomicmem-Config-Override-*` headers * (zero-cost path) and the service receives the startup config * (effectiveConfig undefined). - * 2. Present override → all three headers emitted - * (applied=true, hash=sha256:, keys=sorted csv). + * 2. Present override → Applied/Hash/Keys reflect the applied subset; + * ignored known fields use `X-Atomicmem-Ignored-Override-Keys`. * 3. Search routes forward `effectiveConfig` via the scopedSearch * options bag; ingest routes forward it through the named input. * 4. Unknown override keys do NOT 400 (the schema is permissive so @@ -250,6 +250,43 @@ describe('POST /memories/* — per-request config_override', () => { warnSpy.mockRestore(); }); + it('decode-cap overrides are reported unused and not applied', async () => { + const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}); + const res = await postSearchWithConfigOverride(booted, { + hybridSearchEnabled: true, + extractionMaxTokens: 128, + audnMaxTokens: 64, + }); + expect(res.status).toBe(200); + expect(res.headers.get('X-Atomicmem-Config-Override-Applied')).toBe('true'); + expect(res.headers.get('X-Atomicmem-Config-Override-Keys')).toBe('hybridSearchEnabled'); + expect(res.headers.get('X-Atomicmem-Ignored-Override-Keys')) + .toBe('audnMaxTokens,extractionMaxTokens'); + expect(res.headers.get('X-Atomicmem-Unknown-Override-Keys')).toBeNull(); + const call = scopedSearch.mock.calls[0]!; + const options = call[2] as { effectiveConfig?: { extractionMaxTokens: number; audnMaxTokens: number } }; + expect(options.effectiveConfig?.extractionMaxTokens).toBe(config.extractionMaxTokens); + expect(options.effectiveConfig?.audnMaxTokens).toBe(config.audnMaxTokens); + warnSpy.mockRestore(); + }); + + it('caps-only override reports Applied false and does not invent unknown keys', async () => { + const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}); + const res = await postSearchWithConfigOverride(booted, { + extractionMaxTokens: 128, + audnMaxTokens: 64, + }); + expect(res.status).toBe(200); + expect(res.headers.get('X-Atomicmem-Config-Override-Applied')).toBe('false'); + expect(res.headers.get('X-Atomicmem-Config-Override-Keys')).toBeNull(); + expect(res.headers.get('X-Atomicmem-Ignored-Override-Keys')) + .toBe('audnMaxTokens,extractionMaxTokens'); + expect(res.headers.get('X-Atomicmem-Unknown-Override-Keys')).toBeNull(); + const options = scopedSearch.mock.calls[0]![2] as { effectiveConfig?: RuntimeConfig }; + expect(options.effectiveConfig).toBeUndefined(); + warnSpy.mockRestore(); + }); + it('all-known keys → no X-Atomicmem-Unknown-Override-Keys header', async () => { const res = await postSearchWithConfigOverride(booted, { hybridSearchEnabled: true }); expect(res.status).toBe(200); diff --git a/packages/core/src/app/__tests__/listen-host.test.ts b/packages/core/src/app/__tests__/listen-host.test.ts new file mode 100644 index 0000000..87c71ae --- /dev/null +++ b/packages/core/src/app/__tests__/listen-host.test.ts @@ -0,0 +1,34 @@ +/** + * Bind-address helper tests. Localhost HTTP is not enough to detect wildcard + * listeners — these assertions inspect the constructed options and the + * listening socket address. + */ + +import express from 'express'; +import type { AddressInfo } from 'node:net'; +import { describe, expect, it } from 'vitest'; +import { buildListenOptions, formatListenUrl } from '../listen-options.js'; + +describe('listen host', () => { + it('omits host when LISTEN_HOST is unset', () => { + expect(buildListenOptions(17350)).toEqual({ port: 17350 }); + expect(formatListenUrl(17350)).toBe('http://localhost:17350'); + }); + + it('includes an explicit host when set', () => { + expect(buildListenOptions(17350, '127.0.0.1')).toEqual({ + port: 17350, + host: '127.0.0.1', + }); + expect(formatListenUrl(17350, '127.0.0.1')).toBe('http://127.0.0.1:17350'); + }); + + it('binds the listening socket to 127.0.0.1 when host is set', async () => { + const app = express(); + const server = app.listen(buildListenOptions(0, '127.0.0.1')); + await new Promise((resolve) => server.once('listening', () => resolve())); + const addr = server.address() as AddressInfo; + expect(addr.address).toBe('127.0.0.1'); + await new Promise((resolve) => server.close(() => resolve())); + }); +}); diff --git a/packages/core/src/app/listen-options.ts b/packages/core/src/app/listen-options.ts new file mode 100644 index 0000000..31c34af --- /dev/null +++ b/packages/core/src/app/listen-options.ts @@ -0,0 +1,30 @@ +/** + * Builds Express listen options so Core can bind a host when LISTEN_HOST is set. + * + * Hosted/Docker leave LISTEN_HOST unset and keep Node's all-interfaces default. + * The macOS embedded launcher sets LISTEN_HOST=127.0.0.1. + */ + +export interface CoreListenOptions { + port: number; + host?: string; +} + +/** + * Returns listen options for `app.listen`. Omits `host` when unset so the + * process keeps current hosted bind behavior. + */ +export function buildListenOptions(port: number, listenHost?: string): CoreListenOptions { + if (listenHost && listenHost.length > 0) { + return { port, host: listenHost }; + } + return { port }; +} + +/** + * Formats the boot log URL. Uses the bound host when present. + */ +export function formatListenUrl(port: number, listenHost?: string): string { + const host = listenHost && listenHost.length > 0 ? listenHost : 'localhost'; + return `http://${host}:${port}`; +} diff --git a/packages/core/src/bin.ts b/packages/core/src/bin.ts index 20c697c..9d07f4e 100644 --- a/packages/core/src/bin.ts +++ b/packages/core/src/bin.ts @@ -7,6 +7,7 @@ * migration entry points used by source and Docker workflows. */ +import { basename } from 'node:path'; import { fileURLToPath } from 'node:url'; const LOCAL_PROFILE_DEFAULTS = { @@ -144,8 +145,14 @@ function applyProfile(profile: ProfileName | null): void { } function isEntrypoint(): boolean { + // Deno `compile` sets import.meta.main; Node npm bin ends with atomicmemory-core. + const meta = import.meta as ImportMeta & { main?: boolean }; + if (meta.main === true) return true; const invokedPath = process.argv[1] ?? ''; - return invokedPath === fileURLToPath(import.meta.url) || invokedPath.endsWith('/atomicmemory-core'); + if (!invokedPath) return false; + if (invokedPath === fileURLToPath(import.meta.url)) return true; + const name = basename(invokedPath); + return name === 'atomicmemory-core' || name.startsWith('atomicmemory-core'); } if (isEntrypoint()) { diff --git a/packages/core/src/cloud/__tests__/env.test.ts b/packages/core/src/cloud/__tests__/env.test.ts new file mode 100644 index 0000000..8d544a9 --- /dev/null +++ b/packages/core/src/cloud/__tests__/env.test.ts @@ -0,0 +1,59 @@ +/** + * Fail-closed parsers for decode-cap and related integer env vars. + */ + +import { afterEach, describe, expect, it } from 'vitest'; +import { parseBoundedPositiveIntEnv, parsePositiveIntEnv } from '../env.js'; + +const NAME = 'AUDN_MAX_TOKENS'; +const UNBOUNDED_NAME = 'RAW_UPLOAD_MAX_BYTES'; + +afterEach(() => { + delete process.env[NAME]; + delete process.env[UNBOUNDED_NAME]; +}); + +describe('parseBoundedPositiveIntEnv', () => { + it('accepts a whole decimal integer', () => { + process.env[NAME] = '128'; + expect(parseBoundedPositiveIntEnv(NAME, 2048, 4096)).toBe(128); + }); + + it.each(['128junk', '1.5', '1e3', '0', '-1', 'oops', '08'])( + 'rejects malformed value %s', + (value) => { + process.env[NAME] = value; + expect(() => parseBoundedPositiveIntEnv(NAME, 2048, 4096)).toThrow( + 'AUDN_MAX_TOKENS must be a positive integer', + ); + }, + ); + + it('rejects values above the documented upper bound', () => { + process.env[NAME] = '4097'; + expect(() => parseBoundedPositiveIntEnv(NAME, 2048, 4096)).toThrow( + 'AUDN_MAX_TOKENS must be at most 4096', + ); + }); +}); + +describe('parsePositiveIntEnv', () => { + it('accepts Number.MAX_SAFE_INTEGER', () => { + process.env[UNBOUNDED_NAME] = String(Number.MAX_SAFE_INTEGER); + expect(parsePositiveIntEnv(UNBOUNDED_NAME, 1)).toBe(Number.MAX_SAFE_INTEGER); + }); + + it('rejects overflow that Number() would coerce to Infinity', () => { + process.env[UNBOUNDED_NAME] = '9'.repeat(400); + expect(() => parsePositiveIntEnv(UNBOUNDED_NAME, 1)).toThrow( + 'RAW_UPLOAD_MAX_BYTES must be a positive integer', + ); + }); + + it('rejects Number.MAX_SAFE_INTEGER + 1', () => { + process.env[UNBOUNDED_NAME] = String(Number.MAX_SAFE_INTEGER + 1); + expect(() => parsePositiveIntEnv(UNBOUNDED_NAME, 1)).toThrow( + 'RAW_UPLOAD_MAX_BYTES must be a positive integer', + ); + }); +}); diff --git a/packages/core/src/cloud/env.ts b/packages/core/src/cloud/env.ts index 8d074ae..d5a947d 100644 --- a/packages/core/src/cloud/env.ts +++ b/packages/core/src/cloud/env.ts @@ -6,12 +6,37 @@ export function optionalEnv(name: string): string | undefined { return process.env[name] || undefined; } +/** Whole decimal integer only — rejects parseInt prefixes like `128junk` / `1.5` / `1e3`. */ +const STRICT_POSITIVE_INT = /^[1-9]\d*$/; + +function parseStrictPositiveInt(name: string, raw: string): number { + if (!STRICT_POSITIVE_INT.test(raw)) { + throw new Error(`${name} must be a positive integer`); + } + const parsed = Number(raw); + if (!Number.isSafeInteger(parsed) || parsed <= 0) { + throw new Error(`${name} must be a positive integer`); + } + return parsed; +} + export function parsePositiveIntEnv(name: string, fallback: number): number { const raw = optionalEnv(name); if (!raw) return fallback; - const parsed = Number.parseInt(raw, 10); - if (!Number.isFinite(parsed) || parsed <= 0) { - throw new Error(`${name} must be a positive integer`); + return parseStrictPositiveInt(name, raw); +} + +/** Positive integer with an upper bound; invalid or out-of-range values fail closed. */ +export function parseBoundedPositiveIntEnv( + name: string, + fallback: number, + max: number, +): number { + const raw = optionalEnv(name); + if (!raw) return fallback; + const parsed = parseStrictPositiveInt(name, raw); + if (parsed > max) { + throw new Error(`${name} must be at most ${max} (got ${parsed})`); } return parsed; } diff --git a/packages/core/src/config.ts b/packages/core/src/config.ts index b176428..bbc8c6b 100644 --- a/packages/core/src/config.ts +++ b/packages/core/src/config.ts @@ -24,7 +24,9 @@ import { } from './storage/providers/filecoin/config.js'; import { parseCloudJwtConfig } from './cloud/jwt-config.js'; import { parseCloudTraceSyncConfig } from './cloud/trace-sync-config.js'; -import { optionalEnv, parsePositiveIntEnv, parseStrictBoolEnv } from './cloud/env.js'; +import { optionalEnv, parseBoundedPositiveIntEnv, parsePositiveIntEnv, parseStrictBoolEnv } from './cloud/env.js'; +import type { ExtractionPromptVariant } from './services/extraction-prompt-variant.js'; +import { parseExtractionPromptVariant } from './services/extraction-prompt-variant.js'; import type { CloudJwtConfig, CloudTraceSyncConfig } from './cloud/types.js'; export type { CloudJwtConfig, CloudTraceSyncConfig } from './cloud/types.js'; @@ -126,6 +128,11 @@ export interface RuntimeConfig { */ storageKeyHmacSecret: string; port: number; + /** + * Optional bind address. Unset keeps Node's all-interfaces default + * (hosted/Docker). The macOS launcher sets `127.0.0.1`. + */ + listenHost?: string; retrievalProfile: RetrievalProfileName; retrievalProfileSettings: RetrievalProfile; maxSearchResults: number; @@ -191,6 +198,18 @@ export interface RuntimeConfig { consensusExtractionEnabled: boolean; consensusExtractionRuns: number; observationDateExtractionEnabled: boolean; + /** `compact` selects shorter extraction/AUDN prompts for SLM latency tuning (reduced capability). */ + extractionPromptVariant: ExtractionPromptVariant; + /** Decode cap for extraction LLM calls (default 4096, max 8192). */ + extractionMaxTokens: number; + /** Decode cap for AUDN LLM calls (default 2048, max 4096). */ + audnMaxTokens: number; + /** + * When true, AUDN sends OpenAI-strict `json_schema` (`core_audn`). + * Required for am-local-slm + `AM_SLM_CORE_JSON_SCHEMA=1` so json_object + * is not rewritten to the extraction grammar. Leave false for Groq. + */ + audnJsonSchema: boolean; quotedEntityExtractionEnabled: boolean; entropyGateEnabled: boolean; entropyGateThreshold: number; @@ -1294,6 +1313,7 @@ export const config: RuntimeConfig = { coreTestScopeAllowPattern: parseRegexEnv('CORE_TEST_SCOPE_ALLOW_PATTERN'), storageKeyHmacSecret: parseStorageKeyHmacSecret(requireEnv('STORAGE_KEY_HMAC_SECRET')), port: parseInt(process.env.PORT ?? '17350', 10), + listenHost: optionalEnv('LISTEN_HOST')?.trim() || undefined, retrievalProfile, retrievalProfileSettings, maxSearchResults: retrievalProfileSettings.maxSearchResults, @@ -1368,6 +1388,10 @@ export const config: RuntimeConfig = { consensusExtractionEnabled: (optionalEnv('CONSENSUS_EXTRACTION_ENABLED') ?? 'false') === 'true', consensusExtractionRuns: parseInt(optionalEnv('CONSENSUS_EXTRACTION_RUNS') ?? '3', 10), observationDateExtractionEnabled: (optionalEnv('OBSERVATION_DATE_EXTRACTION_ENABLED') ?? 'false') === 'true', + extractionPromptVariant: parseExtractionPromptVariant(optionalEnv('EXTRACTION_PROMPT_VARIANT')), + extractionMaxTokens: parseBoundedPositiveIntEnv('EXTRACTION_MAX_TOKENS', 4096, 8192), + audnMaxTokens: parseBoundedPositiveIntEnv('AUDN_MAX_TOKENS', 2048, 4096), + audnJsonSchema: parseStrictBoolEnv('AUDN_JSON_SCHEMA', false), quotedEntityExtractionEnabled: (optionalEnv('QUOTED_ENTITY_EXTRACTION_ENABLED') ?? 'false') === 'true', entropyGateEnabled: (optionalEnv('ENTROPY_GATE_ENABLED') ?? 'false') === 'true', entropyGateThreshold: parseFloat(optionalEnv('ENTROPY_GATE_THRESHOLD') ?? '0.35'), @@ -1574,6 +1598,7 @@ export const SUPPORTED_RUNTIME_CONFIG_FIELDS = [ 'databaseUrl', 'openaiApiKey', 'coreApiKey', 'trustedProxyMode', 'coreAdminApiKey', 'coreTestScopeAllowPattern', 'storageKeyHmacSecret', 'port', + 'listenHost', // Provider / model selection (startup config) 'embeddingProvider', 'embeddingModel', 'embeddingDimensions', 'embeddingApiUrl', 'embeddingApiKey', @@ -1628,7 +1653,9 @@ export const INTERNAL_POLICY_CONFIG_FIELDS = [ 'chunkedExtractionEnabled', 'chunkedExtractionFallbackEnabled', 'chunkSizeTurns', 'chunkOverlapTurns', 'consensusExtractionEnabled', 'consensusExtractionRuns', - 'observationDateExtractionEnabled', 'quotedEntityExtractionEnabled', + 'observationDateExtractionEnabled', 'extractionPromptVariant', + 'extractionMaxTokens', 'audnMaxTokens', 'audnJsonSchema', + 'quotedEntityExtractionEnabled', 'entropyGateEnabled', 'entropyGateThreshold', 'entropyGateAlpha', // Affinity clustering 'affinityClusteringThreshold', 'affinityClusteringMinSize', diff --git a/packages/core/src/db/__tests__/cutover-scenarios.test.ts b/packages/core/src/db/__tests__/cutover-scenarios.test.ts index 5619677..b344dd8 100644 --- a/packages/core/src/db/__tests__/cutover-scenarios.test.ts +++ b/packages/core/src/db/__tests__/cutover-scenarios.test.ts @@ -36,6 +36,7 @@ import { applyLegacySchemaAndSeed, expectSeededForeignKeysResolvable, expectSeededRowsPreservedAcrossMigrate, + projectSnapshotsToBeforeColumns, } from './migration-preservation-assertions.js'; import { pgmigrationsRows, @@ -73,7 +74,7 @@ describe('Phase 2 — Scenario B: v1.0.x install upgraded to Phase 2', () => { await expectOnlyBaselineStamped(); }); - it('does not modify any existing legacy table, column, index, or constraint', async () => { + it('preserves every legacy table, column, index, and foreign key across migrate', async () => { await applyLegacySchemaAndSeed(pool); const before = await structuralSnapshotExcludingBookkeeping(pool); @@ -166,39 +167,45 @@ describe('Phase 2 — Scenario C: Phase 1 install upgraded to Phase 2', () => { await migrate({ pool }); const after = await snapshotAllSeededTables(pool); - expect(after).toEqual(before); + expect(projectSnapshotsToBeforeColumns(before, after)).toEqual(before); }); }); /** - * Assert that two structural snapshots describe the same set of tables, - * columns, indexes, check constraints, and foreign keys. Uses - * `toStrictEqual` against the helper output rather than per-field - * iteration because the helper already canonicalizes ordering. + * Assert that legacy structure survives Phase 2 migrate. Post-baseline + * migrations may add tables/columns and may widen CHECK constraints; they + * must not remove or rewrite legacy columns, indexes, or foreign keys. */ function assertStructuralEqual( before: StructuralSnapshot, after: StructuralSnapshot, ): void { - // Post-baseline migrations may add new tables; verify only that existing - // tables, indexes, and constraints are unchanged — not removed or altered. for (const t of before.tables) { - expect(after.tables).toContainEqual(t); + const afterTable = after.tables.find((candidate) => candidate.name === t.name); + expect(afterTable, `legacy table ${t.name} missing after migrate`).toBeDefined(); + for (const column of t.columns) { + expect( + afterTable?.columns, + `legacy column ${t.name}.${column.column} missing or altered`, + ).toContainEqual(column); + } } for (const idx of before.indexes) { expect(after.indexes).toContainEqual(idx); } - expect(after.checkConstraints).toEqual(before.checkConstraints); - expect(after.foreignKeys).toEqual(before.foreignKeys); + for (const fk of before.foreignKeys) { + expect(after.foreignKeys).toContainEqual(fk); + } } async function expectOnlyBaselineStamped(): Promise { const rows = await pgmigrationsRows(pool); // Baseline is always stamped (without re-running against legacy data). - // Post-baseline migrations (e.g. 0002_entity_settings) run normally on - // legacy installs since those tables did not exist yet. Update the count - // and last-name here when new migration files ship. - expect(rows.length).toBe(2); + // Post-baseline migrations run normally on legacy installs since those + // tables did not exist yet. Update the count and last-name here when new + // migration files ship. + expect(rows.length).toBe(6); expect(rows[0].name).toBe(BASELINE_MIGRATION_NAME); expect(rows[1].name).toBe('0002_entity_settings'); + expect(rows.at(-1)?.name).toBe('0005_cloud_trace_outbox'); } diff --git a/packages/core/src/db/__tests__/dag-sanity.test.ts b/packages/core/src/db/__tests__/dag-sanity.test.ts index 65b4824..62052a9 100644 --- a/packages/core/src/db/__tests__/dag-sanity.test.ts +++ b/packages/core/src/db/__tests__/dag-sanity.test.ts @@ -1,42 +1,13 @@ /** - * Phase 2 — DAG sanity for the migration directory. - * - * Per docs/ops/db/phase-2-versioned-migrations.md § "DAG sanity tests" and - * the risks checklist (mistakenly editing or deleting an already-shipped - * migration file): - * - * - **SQL-only**: Phase 2 ships only SQL migrations. The build copy step, - * the runtime hash manifest, and the fail-closed loader all assume - * `.sql`. Allowing `.js`/`.ts` files in the directory here would let a - * PR commit one that never gets packaged or hashed; the regex - * deliberately rejects them so the contract holds at PR review time. - * - **Monotonic**: every file in `src/db/migrations/` follows the - * `_.sql` convention, prefixes are strictly increasing - * across files, and there are no gaps in the sequence. Catches the - * "two PRs both picked 0007 and the lexical sort silently runs one - * before the other" failure mode at PR review time. - * - **No-rewrite**: the diff against the base branch never `D`eletes or - * `R`enames a migration file. Once a migration is on `main`, it is - * frozen. This is the machine-checkable answer to the - * "mistakenly editing the baseline post-shipment" risk that turns - * Scenario B/C into a silent corruption path. - * - * The monotonic check is fully local; the no-rewrite check needs git and - * a reachable base branch. The base branch lookup tries `origin/main` - * first (CI's normal default) and falls back to local `main`. If neither - * resolves the no-rewrite assertion is skipped with `it.skip` semantics - * surfaced via an explicit `expect.fail`-style message: the test does - * not pass silently, but it does not falsely fail when run in a - * detached environment that lacks a remote. - * - * Runtime dependency: the monotonic test asserts the migrations directory - * exists. Until the Phase 2 runtime lands and ships `0001_baseline.sql` - * the directory is absent and this test fails with a clear assertion - * pointing at the missing directory. + * Migration DAG contract: SQL-only files, contiguous numeric slots, and immutable + * shipped names/bytes. The two already-shipped 0002 migrations are grandfathered; + * renumbering either would break existing databases. The base comparison uses + * the monorepo path and checks working-tree bytes as well as committed changes. */ import { spawnSync } from 'node:child_process'; -import { existsSync, readdirSync } from 'node:fs'; +import { existsSync, readdirSync, readFileSync } from 'node:fs'; +import { resolve } from 'node:path'; import { describe, expect, it } from 'vitest'; import { MIGRATIONS_DIR } from './phase2-cutover-helpers.js'; @@ -74,14 +45,16 @@ describe('Phase 2 — migration file DAG (monotonic)', () => { expect(offending).toEqual([]); }); - it('every migration file has a unique numeric prefix', () => { + it('has no duplicate numeric prefix beyond the shipped 0002 pair', () => { const parsed = parseMigrationFiles(listMigrationFiles()); const prefixes = parsed.map((entry) => entry.prefix); - expect(prefixes.length).toBe(new Set(prefixes).size); + const duplicates = parsed.filter((entry, index) => prefixes.indexOf(entry.prefix) !== index); + // Both 0002 names shipped before this guard; renumbering either breaks upgrades. + expect(duplicates.map((entry) => entry.file)).toEqual(['0002_memories_external_id_index.sql']); }); it('prefixes are strictly increasing with no gaps starting at 1', () => { - const parsed = parseMigrationFiles(listMigrationFiles()); + const parsed = [...new Map(parseMigrationFiles(listMigrationFiles()).map((entry) => [entry.prefix, entry])).values()]; expect(parsed.length).toBeGreaterThan(0); for (let i = 0; i < parsed.length; i += 1) { expect(parsed[i].prefix).toBe(i + 1); @@ -94,83 +67,38 @@ describe('Phase 2 — migration file DAG (monotonic)', () => { }); }); -/** - * Files we consider "framework-managed" Phase 2 migrations: the 4-digit - * prefix convention `0001_…`, `0002_…`. The legacy 8-digit - * timestamped files (`20260512_…`) that used to live in the same - * directory were provenance-only and were explicitly moved to - * `docs/db/changelog/` as part of the Phase 2 cutover cleanup (see - * Phase 2 plan §Cleanup). Deletions/renames of those legacy files are - * expected and are NOT a no-rewrite violation. - */ -const FRAMEWORK_FILE_REGEX = /^src\/db\/migrations\/0\d{3}_[a-z0-9](?:[a-z0-9_-]*[a-z0-9])?\.sql$/; +/** Public migrations are immutable once present on the base branch. */ +const MIGRATIONS_PATH = 'packages/core/src/db/migrations'; +const REPO_ROOT = resolve(MIGRATIONS_DIR, '../../../../..'); describe('Phase 2 — migration file DAG (no-rewrite vs base branch)', () => { - it('never deletes or renames a framework-managed migration file', () => { + it('preserves every shipped migration name and its bytes', () => { const base = resolveBaseBranch(); - if (!base) { - // Surface as a failed assertion rather than a silent pass. CI has - // `origin/main`; a local dev environment without it gets an - // explicit reason rather than a passing test that didn't run. - expect.fail( - 'no-rewrite check requires `origin/main` or local `main` to be ' + - 'reachable. None resolved; cannot diff migration files against the base branch.', - ); - return; + if (!base) throw new Error('Migration no-rewrite check requires origin/main or main'); + const files = git(['ls-tree', '-r', '--name-only', base, '--', MIGRATIONS_PATH]) + .trim().split('\n').filter((file) => /\/\d{4}_[^/]+\.sql$/.test(file)); + expect(files.length, 'The base path must resolve real migration files').toBeGreaterThan(0); + for (const file of files) { + const path = resolve(REPO_ROOT, file); + expect(existsSync(path), `Shipped migration removed or renamed: ${file}`).toBe(true); + expect(readFileSync(path, 'utf8'), `Shipped migration changed: ${file}`) + .toBe(git(['show', `${base}:${file}`])); } - const diff = gitDiffNameStatus(base, 'src/db/migrations'); - const forbidden = diff.filter( - (entry) => - (entry.status === 'D' || entry.status === 'R') && - FRAMEWORK_FILE_REGEX.test(entry.file), - ); - expect(forbidden).toEqual([]); }); }); function resolveBaseBranch(): string | null { - for (const candidate of ['origin/feat/db-migration-phase1', 'feat/db-migration-phase1', 'origin/main', 'main']) { + for (const candidate of ['origin/main', 'main']) { const probe = spawnSync('git', ['rev-parse', '--verify', candidate], { - encoding: 'utf-8', + cwd: REPO_ROOT, encoding: 'utf-8', }); if (probe.status === 0) return candidate; } return null; } -interface DiffEntry { - readonly status: string; - readonly file: string; -} - -function gitDiffNameStatus(base: string, pathFilter: string): DiffEntry[] { - const result = spawnSync( - 'git', - ['diff', '--name-status', `${base}...HEAD`, '--', pathFilter], - { encoding: 'utf-8' }, - ); - if (result.status !== 0) { - throw new Error( - `git diff against ${base} failed (status=${result.status}): ${result.stderr}`, - ); - } - return parseDiffOutput(result.stdout); -} - -function parseDiffOutput(stdout: string): DiffEntry[] { - return stdout - .split('\n') - .map((line) => line.trim()) - .filter((line) => line.length > 0) - .map(parseDiffLine); -} - -function parseDiffLine(line: string): DiffEntry { - // Examples: "A\tsrc/db/migrations/0002_foo.sql" - // "D\tsrc/db/migrations/0001_baseline.sql" - // "R100\tsrc/db/migrations/0001_baseline.sql\tsrc/db/migrations/0001_renamed.sql" - const [statusToken, ...paths] = line.split('\t'); - // Status may be "R100" (rename with similarity %); keep just the first char. - const status = statusToken.charAt(0); - return { status, file: paths[0] ?? '' }; +function git(args: string[]): string { + const result = spawnSync('git', args, { cwd: REPO_ROOT, encoding: 'utf-8' }); + if (result.status !== 0) throw new Error(`Migration history check failed: ${result.stderr}`); + return result.stdout; } diff --git a/packages/core/src/db/__tests__/fixtures/shipped-cloud-trace-outbox.sql b/packages/core/src/db/__tests__/fixtures/shipped-cloud-trace-outbox.sql new file mode 100644 index 0000000..df2676f --- /dev/null +++ b/packages/core/src/db/__tests__/fixtures/shipped-cloud-trace-outbox.sql @@ -0,0 +1,25 @@ +-- Durable outbox for connected-local Cloud trace upload. + +CREATE TABLE IF NOT EXISTS cloud_trace_outbox ( + event_id UUID PRIMARY KEY, + schema_version INTEGER NOT NULL, + payload JSONB NOT NULL, + delivery_state TEXT NOT NULL CHECK ( + delivery_state IN ('pending', 'claimed', 'sent', 'dead_letter') + ), + attempt_count INTEGER NOT NULL DEFAULT 0, + next_attempt_at TIMESTAMPTZ NOT NULL DEFAULT now(), + last_error_code TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + claimed_at TIMESTAMPTZ, + sent_at TIMESTAMPTZ, + dead_letter_at TIMESTAMPTZ +); + +CREATE INDEX IF NOT EXISTS cloud_trace_outbox_pending_idx + ON cloud_trace_outbox (delivery_state, next_attempt_at) + WHERE delivery_state IN ('pending', 'claimed'); + +CREATE INDEX IF NOT EXISTS cloud_trace_outbox_sent_retention_idx + ON cloud_trace_outbox (sent_at) + WHERE delivery_state = 'sent'; diff --git a/packages/core/src/db/__tests__/migration-api.test.ts b/packages/core/src/db/__tests__/migration-api.test.ts index 49fe968..8896e23 100644 --- a/packages/core/src/db/__tests__/migration-api.test.ts +++ b/packages/core/src/db/__tests__/migration-api.test.ts @@ -148,8 +148,8 @@ function expectOlderDbStatus( } // Update these two constants whenever a new migration file is added. -const CURRENT_MIGRATION_COUNT = 2; -const CURRENT_LATEST_MIGRATION = '0002_entity_settings'; +const CURRENT_MIGRATION_COUNT = 6; +const CURRENT_LATEST_MIGRATION = '0005_cloud_trace_outbox'; function expectBaselineMigrationCurrent(status: MigrationStatus): void { expect(status.appliedMigrationCount).toBe(CURRENT_MIGRATION_COUNT); diff --git a/packages/core/src/db/__tests__/migration-preservation-assertions.ts b/packages/core/src/db/__tests__/migration-preservation-assertions.ts index 63336fe..79b57fa 100644 --- a/packages/core/src/db/__tests__/migration-preservation-assertions.ts +++ b/packages/core/src/db/__tests__/migration-preservation-assertions.ts @@ -28,7 +28,34 @@ export async function expectSeededRowsPreservedAcrossMigrate( const before = await snapshotAllSeededTables(pool); await migrate({ pool }); const after = await snapshotAllSeededTables(pool); - expect(after).toEqual(before); + expect(projectSnapshotsToBeforeColumns(before, after)).toEqual(before); +} + +/** + * Post-baseline migrations may add nullable columns. Preservation compares + * only the columns that existed before migrate so additive DDL does not look + * like row mutation. + */ +export function projectSnapshotsToBeforeColumns( + before: Awaited>, + after: Awaited>, +): typeof before { + const projected: Record[] }> = {}; + for (const [tableName, beforeSnap] of Object.entries(before)) { + const afterSnap = after[tableName as keyof typeof after]; + const beforeKeys = beforeSnap.rows[0] ? Object.keys(beforeSnap.rows[0]) : []; + projected[tableName] = { + table: beforeSnap.table, + rows: afterSnap.rows.map((row) => { + const narrowed: Record = {}; + for (const key of beforeKeys) { + narrowed[key] = row[key]; + } + return narrowed; + }), + }; + } + return projected as typeof before; } export async function expectSeededForeignKeysResolvable( diff --git a/packages/core/src/db/migration-api.ts b/packages/core/src/db/migration-api.ts index 6f3b73b..1e06042 100644 --- a/packages/core/src/db/migration-api.ts +++ b/packages/core/src/db/migration-api.ts @@ -322,7 +322,9 @@ async function runBaselineMigration(client: PoolClient): Promise { * misbuilt package mark a DB as migrated without ever applying DDL; that * was the audit finding this guards against. */ -async function runFrameworkMigrationsToHead(client: PoolClient): Promise { +async function runFrameworkMigrationsToHead( + client: PoolClient, +): Promise { await runMigrationRunner(client); } @@ -330,12 +332,10 @@ async function runMigrationRunner( client: PoolClient, opts: { file?: string; fake?: boolean } = {}, ): Promise { - // Side-effect call: validates the shipped migration set before letting the - // framework touch the database. Throws on missing dir / no .sql files / - // missing 0001_baseline.sql / empty file. Return value intentionally - // discarded — node-pg-migrate enumerates the directory itself. + // Revalidate shipped files before the framework touches migration state. listMigrationFilenames(); await runMigrations({ + checkOrder: true, dbClient: client, dir: MIGRATIONS_DIR, migrationsTable: PGMIGRATIONS_TABLE, diff --git a/packages/core/src/routes/memories.ts b/packages/core/src/routes/memories.ts index 125dc45..f4ca9e8 100644 --- a/packages/core/src/routes/memories.ts +++ b/packages/core/src/routes/memories.ts @@ -26,8 +26,8 @@ import { MemoryService, type RetrievalResult } from '../services/memory-service. import type { MemoryScope, MemoryServiceDeps, RetrievalObservability } from '../services/memory-service-types.js'; import { applyConfigOverride, + classifyOverrideKeys, hashEffectiveConfig, - summarizeOverrideKeys, } from '../services/retrieval-config-overlay.js'; import { formatIngestResponse, @@ -961,16 +961,16 @@ function toMemoryScope( * Overlay a validated body-level config_override onto the startup * singleton and emit the observability response headers. Returns the * EffectiveConfig to hand to MemoryService (or undefined when no - * override was present — the zero-cost no-headers path). + * override was present, or when every submitted key was ignored). * - * Headers emitted when an override is applied: - * X-Atomicmem-Config-Override-Applied: true + * Headers emitted when an override object is present: + * X-Atomicmem-Config-Override-Applied: true|false (true only if a key applied) * X-Atomicmem-Effective-Config-Hash: sha256: - * X-Atomicmem-Config-Override-Keys: comma-joined sorted key list + * X-Atomicmem-Config-Override-Keys: applied keys only * - * Additional header, emitted only when one or more override keys do - * not correspond to a known RuntimeConfig field on this build: - * X-Atomicmem-Unknown-Override-Keys: comma-joined sorted key list + * Additional headers: + * X-Atomicmem-Ignored-Override-Keys: known fields that cannot take effect + * X-Atomicmem-Unknown-Override-Keys: keys that do not match RuntimeConfig * * Unknown keys are NOT rejected — the permissive schema is deliberate * so adding a new RuntimeConfig field in a future release doesn't @@ -983,23 +983,34 @@ function applyRequestConfigOverride( override: Partial | undefined, ): MemoryServiceDeps['config'] | undefined { if (!override || Object.keys(override).length === 0) return undefined; + const classified = classifyOverrideKeys(override, new Set(Object.keys(baseConfig))); const effective = applyConfigOverride(baseConfig, override); - res.setHeader('X-Atomicmem-Config-Override-Applied', 'true'); - res.setHeader('X-Atomicmem-Effective-Config-Hash', hashEffectiveConfig(effective)); - res.setHeader('X-Atomicmem-Config-Override-Keys', summarizeOverrideKeys(override)); - - const knownKeys = new Set(Object.keys(baseConfig)); - const unknownKeys = Object.keys(override) - .filter((k) => !knownKeys.has(k)) - .sort(); - if (unknownKeys.length > 0) { - res.setHeader('X-Atomicmem-Unknown-Override-Keys', unknownKeys.join(',')); + emitConfigOverrideHeaders(res, classified, effective); + if (classified.unknown.length > 0) { console.warn( - `[config_override] request carried ${unknownKeys.length} unknown key(s): ${unknownKeys.join(', ')} — carried through on effective config but nothing currently reads them`, + `[config_override] request carried ${classified.unknown.length} unknown key(s): ${classified.unknown.join(', ')} — carried through on effective config but nothing currently reads them`, ); } + return classified.applied.length > 0 ? effective : undefined; +} - return effective; +function emitConfigOverrideHeaders( + res: Response, + classified: { applied: string[]; ignored: string[]; unknown: string[] }, + effective: RuntimeConfig, +): void { + const applied = classified.applied.length > 0; + res.setHeader('X-Atomicmem-Config-Override-Applied', applied ? 'true' : 'false'); + res.setHeader('X-Atomicmem-Effective-Config-Hash', hashEffectiveConfig(effective)); + if (applied) { + res.setHeader('X-Atomicmem-Config-Override-Keys', classified.applied.join(',')); + } + if (classified.ignored.length > 0) { + res.setHeader('X-Atomicmem-Ignored-Override-Keys', classified.ignored.join(',')); + } + if (classified.unknown.length > 0) { + res.setHeader('X-Atomicmem-Unknown-Override-Keys', classified.unknown.join(',')); + } } function buildRetrievalObservability(result: RetrievalResult): RetrievalObservability | undefined { diff --git a/packages/core/src/schemas/memories.ts b/packages/core/src/schemas/memories.ts index 1cf5c24..894c04b 100644 --- a/packages/core/src/schemas/memories.ts +++ b/packages/core/src/schemas/memories.ts @@ -246,6 +246,10 @@ const RetrievalModeField = z * unmatched keys and logs a warning. This catches typos without * rejecting a request that would otherwise be valid once the field * lands in a future release. + * - Known fields that cannot take effect on this request (decode caps, + * `audnJsonSchema`) are listed on `X-Atomicmem-Ignored-Override-Keys`. + * `X-Atomicmem-Config-Override-Applied` and + * `X-Atomicmem-Config-Override-Keys` reflect only the applied subset. * - If you want a typed, IDE-autocompleted experience, import * `RuntimeConfig` from `src/config.ts` and type your override as * `Partial` on the caller side. @@ -260,7 +264,7 @@ export const ConfigOverrideSchema = z ) .openapi({ description: - 'Optional per-request overlay on RuntimeConfig. Keys correspond to RuntimeConfig field names; values must be primitives (boolean / number / string / null). Unknown keys are accepted but surfaced via the X-Atomicmem-Unknown-Override-Keys response header and a server-side warning log — they do not cause a 400. Scope: just this request — no server mutation.', + 'Optional per-request overlay on RuntimeConfig. Keys correspond to RuntimeConfig field names; values must be primitives (boolean / number / string / null). Unknown keys are accepted but surfaced via X-Atomicmem-Unknown-Override-Keys. Known non-overridable fields (decode caps, audnJsonSchema) are listed on X-Atomicmem-Ignored-Override-Keys and are not counted as applied. Scope: just this request — no server mutation.', }); /** diff --git a/packages/core/src/server.ts b/packages/core/src/server.ts index 1becfbc..69ba341 100644 --- a/packages/core/src/server.ts +++ b/packages/core/src/server.ts @@ -17,6 +17,7 @@ import { createApp } from './app/create-app.js'; import { checkEmbeddingDimensions } from './app/startup-checks.js'; import { startDeferredAudnScheduler, type DeferredAudnScheduler } from './services/deferred-audn-scheduler.js'; import { startCloudTraceSync, stopCloudTraceSync } from './services/cloud-trace-sync.js'; +import { buildListenOptions, formatListenUrl } from './app/listen-options.js'; // Process-lifecycle signal handlers reference `runtime` via a closure // captured AFTER `bootstrap()` resolves — wired below. Reconciler @@ -53,8 +54,9 @@ async function bootstrap(): Promise { console.log('[startup] Cloud trace sync uploader started'); } - app.listen(runtime.config.port, () => { - console.log(`AtomicMemory Core running on http://localhost:${runtime!.config.port}`); + const listenOptions = buildListenOptions(runtime.config.port, runtime.config.listenHost); + app.listen(listenOptions, () => { + console.log(`AtomicMemory Core running on ${formatListenUrl(runtime!.config.port, runtime!.config.listenHost)}`); }); // Drain the deferred-AUDN queue in the background so ingest stays fast while diff --git a/packages/core/src/services/__tests__/audn-call-site-enumeration.test.ts b/packages/core/src/services/__tests__/audn-call-site-enumeration.test.ts new file mode 100644 index 0000000..d945652 --- /dev/null +++ b/packages/core/src/services/__tests__/audn-call-site-enumeration.test.ts @@ -0,0 +1,89 @@ +/** + * Ensures every production resolveAUDN / cachedResolveAUDN call passes promptVariant. + */ + +import { describe, expect, it } from 'vitest'; +import { readFileSync, readdirSync } from 'node:fs'; +import { dirname, extname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const SERVICES = resolve(__dirname, '..'); +const CALL_RE = /\b(cachedResolveAUDN|resolveAUDN)\(/g; + +function collectTsFiles(dir: string): string[] { + const results: string[] = []; + for (const entry of readdirSync(dir, { withFileTypes: true })) { + const full = resolve(dir, entry.name); + if (entry.isDirectory()) { + if (entry.name === '__tests__' || entry.name === 'node_modules') continue; + results.push(...collectTsFiles(full)); + } else if (entry.isFile() && extname(entry.name) === '.ts') { + results.push(full); + } + } + return results; +} + +function stripComments(source: string): string { + return source + .replace(/\/\*[\s\S]*?\*\//g, ' ') + .replace(/\/\/.*$/gm, ' '); +} + +function countTopLevelCommas(callSnippet: string): number { + const open = callSnippet.indexOf('('); + let depth = 0; + let commas = 0; + for (const char of callSnippet.slice(open)) { + if (char === '(') depth += 1; + if (char === ')') depth -= 1; + if (char === ',' && depth === 1) commas += 1; + } + return commas; +} + +function isAudnFunctionDefinition(prefix: string): boolean { + return /function\s+\w*$/.test(prefix) || prefix.includes('export async function'); +} + +function extractBalancedCall(source: string, startIndex: number): string { + let depth = 0; + for (let end = startIndex; end < source.length; end++) { + const char = source[end]!; + if (char === '(') depth += 1; + if (char === ')') { + depth -= 1; + if (depth === 0) return source.slice(startIndex, end + 1); + } + } + return source.slice(startIndex); +} + +function findUnderSpecifiedAudnCalls(source: string): string[] { + const offenders: string[] = []; + for (const match of source.matchAll(CALL_RE)) { + const index = match.index ?? 0; + const prefix = source.slice(Math.max(0, index - 40), index); + if (isAudnFunctionDefinition(prefix)) continue; + + const snippet = extractBalancedCall(source, index); + if (countTopLevelCommas(snippet) < 2) { + offenders.push(snippet.replace(/\s+/g, ' ').trim()); + } + } + return offenders; +} + +describe('AUDN call-site enumeration', () => { + it('requires promptVariant on every non-test resolveAUDN / cachedResolveAUDN call', () => { + const offenders: string[] = []; + for (const filePath of collectTsFiles(SERVICES)) { + const source = stripComments(readFileSync(filePath, 'utf-8')); + for (const snippet of findUnderSpecifiedAudnCalls(source)) { + offenders.push(`${filePath}: ${snippet}`); + } + } + expect(offenders).toEqual([]); + }); +}); diff --git a/packages/core/src/services/__tests__/audn-compact-schema.test.ts b/packages/core/src/services/__tests__/audn-compact-schema.test.ts new file mode 100644 index 0000000..f7adeb7 --- /dev/null +++ b/packages/core/src/services/__tests__/audn-compact-schema.test.ts @@ -0,0 +1,155 @@ +/** + * AUDN must use an explicit JSON schema so am-local-slm does not route it + * through the extraction-only core_extraction_schema grammar. + */ + +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +vi.mock('../llm.js', () => ({ llm: { chat: vi.fn() } })); +vi.mock('../../config.js', () => ({ + config: { audnMaxTokens: 128, extractionMaxTokens: 768, audnJsonSchema: false }, +})); + +const { llm } = await import('../llm.js'); +const { config } = await import('../../config.js'); +const { resolveAUDN, defaultDecision } = await import('../extraction.js'); +const { AUDN_SCHEMA_NAME, audnChatOptions, audnJsonChatOptions } = await import('../extraction-json-schema.js'); +const { openAIResponseFormat } = await import('../openai-chat-params.js'); + +const mockLlmChat = vi.mocked(llm.chat); +const TARGET_ID = '11111111-1111-4111-8111-111111111111'; + +beforeEach(() => { + mockLlmChat.mockReset(); + (config as { audnJsonSchema: boolean }).audnJsonSchema = false; +}); + +describe('AUDN OpenAI / am-local-slm contract', () => { + it('lists every property in required so OpenAI strict mode accepts the schema', () => { + const schema = audnJsonChatOptions(128).jsonSchema?.schema as { + properties: Record; + required: string[]; + }; + const propertyNames = Object.keys(schema.properties); + expect(schema.required).toEqual(propertyNames); + expect(propertyNames).toEqual([ + 'action', + 'target_memory_id', + 'updated_content', + 'clarification_note', + 'contradiction_confidence', + ]); + }); + + it('selects json_schema from the capability flag, not the prompt variant', () => { + expect(audnChatOptions(true, 128).jsonSchema?.name).toBe(AUDN_SCHEMA_NAME); + expect(audnChatOptions(false, 128).jsonSchema).toBeUndefined(); + }); + + it('emits json_schema so am-local-slm forwards the client grammar', () => { + const options = audnJsonChatOptions(128); + expect(openAIResponseFormat(true, options.jsonSchema)).toEqual({ + response_format: { + type: 'json_schema', + json_schema: { + name: AUDN_SCHEMA_NAME, + strict: true, + schema: options.jsonSchema?.schema, + }, + }, + }); + }); +}); + +describe('resolveAUDN compact runtime contract', () => { + it('sends core_audn json_schema on the full prompt when the provider flag is on', async () => { + (config as { audnJsonSchema: boolean }).audnJsonSchema = true; + mockLlmChat.mockResolvedValueOnce(JSON.stringify({ + action: 'NOOP', + target_memory_id: TARGET_ID, + updated_content: null, + clarification_note: null, + contradiction_confidence: null, + })); + + await resolveAUDN('User likes Vite.', [{ + id: TARGET_ID, + content: 'User likes Vite.', + similarity: 0.99, + }], 'full'); + + expect(mockLlmChat).toHaveBeenCalledWith( + expect.any(Array), + expect.objectContaining({ + jsonSchema: expect.objectContaining({ name: AUDN_SCHEMA_NAME, strict: true }), + }), + ); + }); + + it('keeps compact+Groq on json_object when the provider flag is off', async () => { + mockLlmChat.mockResolvedValueOnce(JSON.stringify({ + action: 'NOOP', + target_memory_id: TARGET_ID, + updated_content: null, + clarification_note: null, + contradiction_confidence: null, + })); + + await resolveAUDN('User likes Vite.', [{ + id: TARGET_ID, + content: 'User likes Vite.', + similarity: 0.99, + }], 'compact'); + + expect(mockLlmChat).toHaveBeenCalledWith( + expect.any(Array), + { temperature: 0, jsonMode: true, maxTokens: 128 }, + ); + }); + + it('requests the core_audn schema and returns a non-ADD decision', async () => { + (config as { audnJsonSchema: boolean }).audnJsonSchema = true; + mockLlmChat.mockResolvedValueOnce(JSON.stringify({ + action: 'NOOP', + target_memory_id: TARGET_ID, + updated_content: null, + clarification_note: null, + contradiction_confidence: null, + })); + + const decision = await resolveAUDN('User likes Vite.', [{ + id: TARGET_ID, + content: 'User likes Vite.', + similarity: 0.99, + }], 'compact'); + + expect(decision.action).toBe('NOOP'); + expect(mockLlmChat).toHaveBeenCalledWith( + expect.arrayContaining([ + expect.objectContaining({ + role: 'system', + content: expect.stringContaining('ACTIONS:'), + }), + ]), + expect.objectContaining({ + jsonMode: true, + maxTokens: 128, + jsonSchema: expect.objectContaining({ name: AUDN_SCHEMA_NAME, strict: true }), + }), + ); + }); + + it('does not silently default to ADD when json_object would return extraction shape', async () => { + mockLlmChat.mockResolvedValueOnce(JSON.stringify({ + memories: [{ fact: 'User likes Vite.', type: 'preference' }], + })); + + const decision = await resolveAUDN('User likes Vite.', [{ + id: TARGET_ID, + content: 'User likes Vite.', + similarity: 0.99, + }], 'compact'); + + expect(decision).toEqual(defaultDecision()); + }); +}); diff --git a/packages/core/src/services/__tests__/audn-workspace-scope-fence.test.ts b/packages/core/src/services/__tests__/audn-workspace-scope-fence.test.ts index 97dbfb9..b33f674 100644 --- a/packages/core/src/services/__tests__/audn-workspace-scope-fence.test.ts +++ b/packages/core/src/services/__tests__/audn-workspace-scope-fence.test.ts @@ -47,7 +47,10 @@ const baseFact = { function makeDeps() { mockStoreMemory.mockResolvedValue('clarify-mem-1'); return { - config: { entityGraphEnabled: false, lessonsEnabled: false, fastAudnEnabled: false, fastAudnDuplicateThreshold: 0.95 }, + config: { + entityGraphEnabled: false, lessonsEnabled: false, fastAudnEnabled: false, + fastAudnDuplicateThreshold: 0.95, extractionPromptVariant: 'full', + }, stores: { memory: { storeMemory: mockStoreMemory, @@ -108,7 +111,7 @@ describe('AUDN workspace scope fences', () => { makeTraceContext(baseFact, undefined, []), ); - expect(mockCachedResolveAUDN).toHaveBeenCalledWith(baseFact.fact, []); + expect(mockCachedResolveAUDN).toHaveBeenCalledWith(baseFact.fact, [], 'full'); expect(mockStoreCanonicalFact).toHaveBeenCalledTimes(1); expect(result).toMatchObject({ outcome: 'stored', memoryId: 'new-memory-1' }); }); diff --git a/packages/core/src/services/__tests__/compact-extraction-policy.test.ts b/packages/core/src/services/__tests__/compact-extraction-policy.test.ts new file mode 100644 index 0000000..972c231 --- /dev/null +++ b/packages/core/src/services/__tests__/compact-extraction-policy.test.ts @@ -0,0 +1,56 @@ +/** + * Documents compact extraction as a reduced-capability opt-in. + */ + +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +vi.mock('../llm.js', () => ({ llm: { chat: vi.fn() } })); +vi.mock('../fact-normalization.js', () => ({ + normalizeExtractedFacts: (facts: unknown[]) => facts, +})); +vi.mock('../../config.js', () => ({ + config: { extractionMaxTokens: 768, audnMaxTokens: 128 }, +})); + +const { llm } = await import('../llm.js'); +const { EXTRACTION_PROMPT_COMPACT, extractFacts } = await import('../extraction.js'); + +const mockLlmChat = vi.mocked(llm.chat); + +beforeEach(() => { + mockLlmChat.mockReset(); +}); + +describe('compact extraction product contract', () => { + it('documents reduced capability on the compact prompt', () => { + expect(EXTRACTION_PROMPT_COMPACT).toContain("Never record the assistant's commentary"); + expect(EXTRACTION_PROMPT_COMPACT).not.toContain('DO extract specific factual content from assistant responses'); + expect(EXTRACTION_PROMPT_COMPACT).not.toContain('CONTACT INFO'); + }); + + it('derives keywords when compact sanitization empties an invalid array', async () => { + const fact = 'User prefers PostgreSQL over MongoDB.'; + mockLlmChat.mockResolvedValueOnce( + JSON.stringify({ memories: [{ fact, keywords: [null, '', 42] }] }), + ); + const facts = await extractFacts('User: I prefer PostgreSQL over MongoDB.', { + promptVariant: 'compact', + }); + expect(facts[0]?.keywords).toEqual(['PostgreSQL', 'MongoDB']); + }); + + it('does not extract assistant recommendations under compact mode', async () => { + mockLlmChat.mockResolvedValueOnce(JSON.stringify({ memories: [] })); + const facts = await extractFacts( + 'Assistant: You should migrate to PostgreSQL for production.\nUser: Thanks.', + { promptVariant: 'compact' }, + ); + expect(facts).toEqual([]); + expect(mockLlmChat).toHaveBeenCalledWith( + expect.arrayContaining([ + expect.objectContaining({ role: 'system', content: EXTRACTION_PROMPT_COMPACT }), + ]), + expect.any(Object), + ); + }); +}); diff --git a/packages/core/src/services/__tests__/consensus-extraction-runtime-config.test.ts b/packages/core/src/services/__tests__/consensus-extraction-runtime-config.test.ts index 829a5b6..ea5c903 100644 --- a/packages/core/src/services/__tests__/consensus-extraction-runtime-config.test.ts +++ b/packages/core/src/services/__tests__/consensus-extraction-runtime-config.test.ts @@ -53,11 +53,12 @@ describe('consensusExtractFacts runtime config', () => { extractionCacheEnabled: false, observationDateExtractionEnabled: true, quotedEntityExtractionEnabled: false, + extractionPromptVariant: 'full', }); expect(mockChunkedExtractFacts).toHaveBeenCalledWith( 'User: I commute 45 minutes.', - { observationDateExtractionEnabled: true }, + { observationDateExtractionEnabled: true, promptVariant: 'full' }, { chunkSizeTurns: 8, chunkOverlapTurns: 2, extractionCacheEnabled: false }, ); expect(mockCachedExtractFacts).not.toHaveBeenCalled(); @@ -82,14 +83,16 @@ describe('consensusExtractFacts runtime config', () => { extractionCacheEnabled: true, observationDateExtractionEnabled: false, quotedEntityExtractionEnabled: false, + extractionPromptVariant: 'full', }); expect(mockCachedExtractFacts).toHaveBeenCalledWith(longConversation, { observationDateExtractionEnabled: false, + promptVariant: 'full', }); expect(mockChunkedExtractFacts).toHaveBeenCalledWith( longConversation, - { observationDateExtractionEnabled: false }, + { observationDateExtractionEnabled: false, promptVariant: 'full' }, { chunkSizeTurns: 2, chunkOverlapTurns: 1, extractionCacheEnabled: true }, ); }); @@ -108,6 +111,7 @@ describe('consensusExtractFacts runtime config', () => { extractionCacheEnabled: true, observationDateExtractionEnabled: false, quotedEntityExtractionEnabled: false, + extractionPromptVariant: 'full', }); expect(mockChunkedExtractFacts).not.toHaveBeenCalled(); @@ -127,10 +131,12 @@ describe('consensusExtractFacts runtime config', () => { extractionCacheEnabled: false, observationDateExtractionEnabled: false, quotedEntityExtractionEnabled: false, + extractionPromptVariant: 'full', }); expect(mockExtractFacts).toHaveBeenCalledWith('User: I prefer Rust', { observationDateExtractionEnabled: false, + promptVariant: 'full', }); expect(mockCachedExtractFacts).not.toHaveBeenCalled(); }); diff --git a/packages/core/src/services/__tests__/deferred-audn-reconcile.test.ts b/packages/core/src/services/__tests__/deferred-audn-reconcile.test.ts new file mode 100644 index 0000000..33401e5 --- /dev/null +++ b/packages/core/src/services/__tests__/deferred-audn-reconcile.test.ts @@ -0,0 +1,77 @@ +/** + * Deferred AUDN reconciliation passes the configured prompt variant through AUDN. + */ + +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const mockCachedResolveAUDN = vi.fn(); +const mockClearDeferredFlag = vi.fn(); +const mockGetMemory = vi.fn(); +const mockSoftDeleteMemory = vi.fn(); + +vi.mock('../../config.js', () => ({ + config: { + deferredAudnEnabled: true, + deferredAudnBatchSize: 20, + deferredAudnConcurrency: 1, + auditLoggingEnabled: false, + extractionPromptVariant: 'compact', + }, +})); + +vi.mock('../extraction-cache.js', () => ({ + cachedResolveAUDN: (...args: unknown[]) => mockCachedResolveAUDN(...args), +})); + +vi.mock('../../db/repository-deferred-audn.js', () => ({ + findDeferredMemories: vi.fn(), + findAllDeferredMemories: vi.fn(), + clearDeferredFlag: (...args: unknown[]) => mockClearDeferredFlag(...args), + countDeferredMemories: vi.fn(), + markMemoryDeferred: vi.fn(), +})); + +vi.mock('../embedding.js', () => ({ + embedText: vi.fn(async () => [0.1, 0.2]), +})); + +const { reconcileUser } = await import('../deferred-audn.js'); +const { findDeferredMemories } = await import('../../db/repository-deferred-audn.js'); + +beforeEach(() => { + vi.clearAllMocks(); + mockCachedResolveAUDN.mockResolvedValue({ + action: 'ADD', + targetMemoryId: null, + updatedContent: null, + clarificationNote: null, + contradictionConfidence: null, + }); + mockGetMemory.mockResolvedValue({ id: 'target', importance: 0.5, deleted_at: null }); +}); + +describe('deferred AUDN reconciliation', () => { + it('passes config.extractionPromptVariant into cachedResolveAUDN', async () => { + vi.mocked(findDeferredMemories).mockResolvedValueOnce([{ + id: 'mem-1', + userId: 'user-1', + content: 'User prefers Vite.', + createdAt: new Date('2026-01-01T00:00:00Z'), + candidates: [{ id: 'mem-2', content: 'User likes Vite.', similarity: 0.88 }], + }]); + + const repo = { + getMemory: mockGetMemory, + softDeleteMemory: mockSoftDeleteMemory, + updateMemoryContent: vi.fn(), + }; + + await reconcileUser({} as never, repo as never, 'user-1', 1); + + expect(mockCachedResolveAUDN).toHaveBeenCalledWith( + 'User prefers Vite.', + expect.any(Array), + 'compact', + ); + }); +}); diff --git a/packages/core/src/services/__tests__/extraction-cache.test.ts b/packages/core/src/services/__tests__/extraction-cache.test.ts index 5e020ec..0b894ac 100644 --- a/packages/core/src/services/__tests__/extraction-cache.test.ts +++ b/packages/core/src/services/__tests__/extraction-cache.test.ts @@ -17,6 +17,9 @@ vi.mock('../../config.js', () => ({ config: { extractionCacheEnabled: true, extractionCacheDir: TEST_CACHE_DIR, + extractionMaxTokens: 4096, + audnMaxTokens: 2048, + audnJsonSchema: false, }, })); @@ -28,7 +31,7 @@ vi.mock('../extraction.js', () => ({ resolveAUDN: (...args: unknown[]) => mockResolveAUDN(...args), })); -const { cachedExtractFacts, cachedResolveAUDN } = await import('../extraction-cache.js'); +const { cachedExtractFacts, cachedResolveAUDN, EXTRACTION_CACHE_CONTRACT } = await import('../extraction-cache.js'); beforeEach(() => { mkdirSync(TEST_CACHE_DIR, { recursive: true }); @@ -105,6 +108,20 @@ describe('cachedExtractFacts', () => { const tmpFiles = files.filter((f) => f.endsWith('.tmp')); expect(tmpFiles).toHaveLength(0); }); + + it('misses when the effective extraction token cap changes', async () => { + mockExtractFacts.mockResolvedValue(SAMPLE_FACTS); + const { config } = await import('../../config.js'); + const original = config.extractionMaxTokens; + + await cachedExtractFacts('cap check'); + (config as { extractionMaxTokens: number }).extractionMaxTokens = 128; + await cachedExtractFacts('cap check'); + (config as { extractionMaxTokens: number }).extractionMaxTokens = original; + + expect(mockExtractFacts).toHaveBeenCalledTimes(2); + expect(EXTRACTION_CACHE_CONTRACT).toBe('v3'); + }); }); describe('cachedResolveAUDN', () => { @@ -123,18 +140,18 @@ describe('cachedResolveAUDN', () => { it('calls resolveAUDN on cache miss', async () => { mockResolveAUDN.mockResolvedValueOnce(SAMPLE_DECISION); - const result = await cachedResolveAUDN('new fact', EXISTING_MEMORIES); + const result = await cachedResolveAUDN('new fact', EXISTING_MEMORIES, 'full'); expect(mockResolveAUDN).toHaveBeenCalledOnce(); - expect(mockResolveAUDN).toHaveBeenCalledWith('new fact', EXISTING_MEMORIES); + expect(mockResolveAUDN).toHaveBeenCalledWith('new fact', EXISTING_MEMORIES, 'full'); expect(result).toEqual(SAMPLE_DECISION); }); it('returns cached result on hit', async () => { mockResolveAUDN.mockResolvedValueOnce(SAMPLE_DECISION); - const first = await cachedResolveAUDN('fact A', EXISTING_MEMORIES); - const second = await cachedResolveAUDN('fact A', EXISTING_MEMORIES); + const first = await cachedResolveAUDN('fact A', EXISTING_MEMORIES, 'full'); + const second = await cachedResolveAUDN('fact A', EXISTING_MEMORIES, 'full'); expect(mockResolveAUDN).toHaveBeenCalledOnce(); expect(second).toEqual(first); @@ -146,8 +163,8 @@ describe('cachedResolveAUDN', () => { { id: 'mem-2', content: 'User likes Python', similarity: 0.75 }, ]; - await cachedResolveAUDN('same fact', EXISTING_MEMORIES); - await cachedResolveAUDN('same fact', differentMemories); + await cachedResolveAUDN('same fact', EXISTING_MEMORIES, 'full'); + await cachedResolveAUDN('same fact', differentMemories, 'full'); expect(mockResolveAUDN).toHaveBeenCalledTimes(2); }); @@ -155,12 +172,38 @@ describe('cachedResolveAUDN', () => { it('writes audn-prefixed cache files', async () => { mockResolveAUDN.mockResolvedValueOnce(SAMPLE_DECISION); - await cachedResolveAUDN('audn test', EXISTING_MEMORIES); + await cachedResolveAUDN('audn test', EXISTING_MEMORIES, 'full'); const files = readdirSync(TEST_CACHE_DIR).filter((f) => f.startsWith('audn-')); expect(files).toHaveLength(1); expect(files[0]).toMatch(/^audn-[a-f0-9]{16}\.json$/); }); + + it('misses when the effective AUDN token cap changes', async () => { + mockResolveAUDN.mockResolvedValue(SAMPLE_DECISION); + const { config } = await import('../../config.js'); + const original = config.audnMaxTokens; + + await cachedResolveAUDN('cap audn', EXISTING_MEMORIES, 'compact'); + (config as { audnMaxTokens: number }).audnMaxTokens = 128; + await cachedResolveAUDN('cap audn', EXISTING_MEMORIES, 'compact'); + (config as { audnMaxTokens: number }).audnMaxTokens = original; + + expect(mockResolveAUDN).toHaveBeenCalledTimes(2); + }); + + it('misses when the AUDN json_schema capability flag changes', async () => { + mockResolveAUDN.mockResolvedValue(SAMPLE_DECISION); + const { config } = await import('../../config.js'); + const typed = config as { audnJsonSchema: boolean }; + + await cachedResolveAUDN('schema audn', EXISTING_MEMORIES, 'full'); + typed.audnJsonSchema = true; + await cachedResolveAUDN('schema audn', EXISTING_MEMORIES, 'full'); + typed.audnJsonSchema = false; + + expect(mockResolveAUDN).toHaveBeenCalledTimes(2); + }); }); describe('cache disabled', () => { diff --git a/packages/core/src/services/__tests__/extraction-keywords.test.ts b/packages/core/src/services/__tests__/extraction-keywords.test.ts new file mode 100644 index 0000000..23d97ec --- /dev/null +++ b/packages/core/src/services/__tests__/extraction-keywords.test.ts @@ -0,0 +1,28 @@ +/** + * Unicode-safe keyword derivation for compact extraction backfill. + */ + +import { describe, expect, it } from 'vitest'; +import { deriveKeywordsFromFact } from '../extraction-keywords.js'; + +describe('deriveKeywordsFromFact — Unicode', () => { + it('keeps accented Latin place names intact', () => { + const keywords = deriveKeywordsFromFact('User lives in Zürich.'); + expect(keywords).toContain('Zürich'); + expect(keywords).not.toContain('lives'); + expect(keywords).not.toContain('rich'); + }); + + it('keeps multi-word accented proper nouns together', () => { + const keywords = deriveKeywordsFromFact('El usuario vive en São Paulo.'); + expect(keywords).toContain('São Paulo'); + expect(keywords).not.toContain('El'); + expect(keywords).not.toContain('Paulo'); + }); + + it('derives non-Latin content words when patterns are sparse', () => { + const keywords = deriveKeywordsFromFact('用户住在北京。'); + expect(keywords.length).toBeGreaterThan(0); + expect(keywords.some((term) => term.includes('北京') || term.includes('用户'))).toBe(true); + }); +}); diff --git a/packages/core/src/services/__tests__/extraction-parse.test.ts b/packages/core/src/services/__tests__/extraction-parse.test.ts new file mode 100644 index 0000000..7b0d02a --- /dev/null +++ b/packages/core/src/services/__tests__/extraction-parse.test.ts @@ -0,0 +1,128 @@ +/** + * Unit tests for extractFacts JSON parse hardening (ATO-2185). + * Covers trailing prose, fenced JSON, concatenated objects, truncated repair, + * and valid empty memories — without changing quickExtractFacts. + */ + +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +vi.mock('../llm.js', () => ({ llm: { chat: vi.fn() } })); +vi.mock('../../config.js', () => ({ + config: { extractionMaxTokens: 4096, audnMaxTokens: 2048 }, +})); +vi.mock('../fact-normalization.js', () => ({ + normalizeExtractedFacts: (facts: unknown[]) => facts, +})); +vi.mock('../supplemental-extraction.js', () => ({ + mergeSupplementalFacts: (facts: unknown[]) => facts, +})); + +const { llm } = await import('../llm.js'); +const { + extractFacts, + getExtractParseFailureCount, + resetExtractParseFailureCount, +} = await import('../extraction.js'); + +const mockLlmChat = vi.mocked(llm.chat); + +const MEMORY_FACT = { + fact: 'User prefers Vite over Webpack.', + type: 'preference', + headline: 'Prefers Vite', + importance: 0.7, + keywords: ['Vite', 'Webpack'], + entities: [], + relations: [], +}; + +const VALID_MEMORIES = JSON.stringify({ memories: [MEMORY_FACT] }); + +beforeEach(() => { + mockLlmChat.mockReset(); + resetExtractParseFailureCount(); +}); + +describe('extractFacts — JSON parse hardening (ATO-2185)', () => { + it('parses JSON followed by trailing prose', async () => { + mockLlmChat.mockResolvedValueOnce( + `${VALID_MEMORIES}\n\nI extracted one preference about Vite.`, + ); + const facts = await extractFacts('User: I prefer Vite over Webpack.'); + expect(facts).toHaveLength(1); + expect(facts[0]?.fact).toContain('Vite'); + expect(mockLlmChat).toHaveBeenCalledTimes(1); + }); + + it('parses fenced JSON with trailing text after the fence', async () => { + mockLlmChat.mockResolvedValueOnce( + `\`\`\`json\n${VALID_MEMORIES}\n\`\`\`\nThanks, here is the extraction.`, + ); + const facts = await extractFacts('User: I prefer Vite over Webpack.'); + expect(facts).toHaveLength(1); + expect(facts[0]?.fact).toContain('Vite'); + }); + + it('uses the first object when two JSON objects are concatenated', async () => { + const second = JSON.stringify({ + memories: [{ fact: 'User prefers Parcel exclusively.', type: 'preference' }], + }); + mockLlmChat.mockResolvedValueOnce(`${VALID_MEMORIES}\n${second}`); + const facts = await extractFacts('User: I prefer Vite over Webpack.'); + expect(facts).toHaveLength(1); + expect(facts[0]?.fact).toContain('Vite'); + expect(facts[0]?.fact).not.toContain('Parcel'); + }); + + it('repairs truncated memories arrays and keeps complete entries', async () => { + const truncated = + '{"memories":[{"fact":"User prefers Vite over Webpack.","type":"preference"},{"fact":"User uses React'; + mockLlmChat.mockResolvedValueOnce(truncated); + const facts = await extractFacts('User: I prefer Vite over Webpack.'); + expect(facts.length).toBeGreaterThanOrEqual(1); + expect(facts[0]?.fact).toContain('Vite'); + }); + + it('treats an empty memories array as valid success without retry', async () => { + mockLlmChat.mockResolvedValueOnce(JSON.stringify({ memories: [] })); + const facts = await extractFacts('User: hello'); + expect(facts).toEqual([]); + expect(mockLlmChat).toHaveBeenCalledTimes(1); + expect(getExtractParseFailureCount()).toBe(0); + }); + + it('retries once with a JSON-only nudge when substantial content will not parse', async () => { + mockLlmChat + .mockResolvedValueOnce('Sure!\n\nHere is nonsense {not-json at all} trailing prose filler.') + .mockResolvedValueOnce(VALID_MEMORIES); + + const facts = await extractFacts('User: I prefer Vite over Webpack.'); + expect(facts).toHaveLength(1); + expect(mockLlmChat).toHaveBeenCalledTimes(2); + const retrySystem = mockLlmChat.mock.calls[1]?.[0]?.[0]?.content as string; + expect(retrySystem).toContain('ONLY the JSON object'); + expect(getExtractParseFailureCount()).toBe(0); + }); + + it('logs a parse failure metric when retry still cannot parse', async () => { + const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined); + const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => undefined); + mockLlmChat + .mockResolvedValueOnce('Sure!\n\nHere is nonsense {not-json at all} trailing prose filler.') + .mockResolvedValueOnce('Still broken {also not json} with more prose here.'); + + const facts = await extractFacts('User: I prefer Vite over Webpack.'); + expect(facts).toEqual([]); + expect(mockLlmChat).toHaveBeenCalledTimes(2); + expect(getExtractParseFailureCount()).toBe(1); + expect(errorSpy).toHaveBeenCalledWith( + expect.stringContaining('[extractFacts] parse failed after retry'), + ); + expect(JSON.stringify(errorSpy.mock.calls)).not.toContain('Still broken'); + expect(JSON.stringify(errorSpy.mock.calls)).not.toContain('not-json'); + expect(JSON.stringify(warnSpy.mock.calls)).not.toContain('not-json'); + expect(JSON.stringify(warnSpy.mock.calls)).not.toContain('also not json'); + warnSpy.mockRestore(); + errorSpy.mockRestore(); + }); +}); diff --git a/packages/core/src/services/__tests__/extraction.test.ts b/packages/core/src/services/__tests__/extraction.test.ts index a56f356..62f09ed 100644 --- a/packages/core/src/services/__tests__/extraction.test.ts +++ b/packages/core/src/services/__tests__/extraction.test.ts @@ -8,6 +8,9 @@ import { beforeEach, describe, it, expect, vi } from 'vitest'; /** Mock llm and fact-normalization to avoid config.ts env var requirements. */ vi.mock('../llm.js', () => ({ llm: { chat: vi.fn() } })); +vi.mock('../../config.js', () => ({ + config: { extractionMaxTokens: 4096, audnMaxTokens: 2048 }, +})); vi.mock('../fact-normalization.js', () => ({ normalizeExtractedFacts: (facts: unknown[]) => facts, })); @@ -20,14 +23,26 @@ const { normalizeConfidence, inferConflictConfidence, generateFallbackHeadline, + deriveKeywordsFromFact, normalizeExtractedEntities, normalizeExtractedRelations, EXTRACTION_PROMPT, + EXTRACTION_PROMPT_COMPACT, + resolveAudnPrompt, + resolveExtractionPrompt, + extractFacts, } = await import('../extraction.js'); - type AUDNAction = Awaited>; const mockLlmChat = vi.mocked(llm.chat); +const ADD_AUDN_LLM_PAYLOAD = JSON.stringify({ + action: 'ADD', + target_memory_id: null, + updated_content: null, + clarification_note: null, + contradiction_confidence: null, +}); + beforeEach(() => { mockLlmChat.mockReset(); }); @@ -99,7 +114,7 @@ describe('resolveAUDN', () => { id: '11111111-1111-4111-8111-111111111111', content: 'User likes Vite.', similarity: 0.99, - }]); + }], 'full'); expect(decision.action).toBe('NOOP'); expect(decision.targetMemoryId).toBe('11111111-1111-4111-8111-111111111111'); @@ -116,27 +131,36 @@ describe('resolveAUDN', () => { } This is final.`); - const decision = await resolveAUDN('User uses Supabase.', []); + const decision = await resolveAUDN('User uses Supabase.', [], 'full'); expect(decision).toEqual(defaultDecision()); }); - it('requests a larger AUDN output budget for Anthropic-compatible models', async () => { - mockLlmChat.mockResolvedValueOnce(JSON.stringify({ - action: 'ADD', - target_memory_id: null, - updated_content: null, - clarification_note: null, - contradiction_confidence: null, - })); + it('keeps json_object on the default full path so Groq-compatible providers stay valid', async () => { + mockLlmChat.mockResolvedValueOnce(ADD_AUDN_LLM_PAYLOAD); - await resolveAUDN('User uses Tailwind.', []); + await resolveAUDN('User uses Tailwind.', [], 'full'); - expect(mockLlmChat).toHaveBeenLastCalledWith(expect.any(Array), expect.objectContaining({ + expect(mockLlmChat).toHaveBeenLastCalledWith(expect.any(Array), { + temperature: 0, jsonMode: true, maxTokens: 2048, - temperature: 0, - })); + }); + }); + + it('uses config.audnMaxTokens for the decode cap', async () => { + const { config } = await import('../../config.js'); + const original = config.audnMaxTokens; + (config as { audnMaxTokens: number }).audnMaxTokens = 128; + try { + mockLlmChat.mockResolvedValueOnce(ADD_AUDN_LLM_PAYLOAD); + await resolveAUDN('User uses Tailwind.', [], 'full'); + expect(mockLlmChat).toHaveBeenLastCalledWith(expect.any(Array), expect.objectContaining({ + maxTokens: 128, + })); + } finally { + (config as { audnMaxTokens: number }).audnMaxTokens = original; + } }); }); @@ -361,6 +385,24 @@ describe('EXTRACTION_PROMPT — assistant-turn extraction directives', () => { expect(EXTRACTION_PROMPT).toContain('DO extract specific factual content from assistant responses'); }); + it('defaults resolveExtractionPrompt to the full prompt', () => { + expect(resolveExtractionPrompt(undefined)).toBe(EXTRACTION_PROMPT); + expect(resolveExtractionPrompt('full')).toBe(EXTRACTION_PROMPT); + }); + + it('selects compact extraction prompt when configured', () => { + expect(resolveExtractionPrompt('compact')).toBe(EXTRACTION_PROMPT_COMPACT); + expect(EXTRACTION_PROMPT_COMPACT.length).toBeLessThan(EXTRACTION_PROMPT.length / 2); + }); + + it('keeps the full AUDN mutation policy under compact extraction', () => { + expect(resolveAudnPrompt('compact')).toBe(resolveAudnPrompt('full')); + expect(resolveAudnPrompt('compact')).toContain('ACTIONS:'); + expect(resolveAudnPrompt('compact')).toContain('clarification_note'); + expect(resolveAudnPrompt('compact')).toContain('contradiction_confidence'); + expect(resolveAudnPrompt('compact')).toContain('If you are unsure whether to SUPERSEDE or CLARIFY'); + }); + it('instructs to skip generic assistant chatter', () => { expect(EXTRACTION_PROMPT).toContain('Skip generic assistant chatter'); }); @@ -370,3 +412,116 @@ describe('EXTRACTION_PROMPT — assistant-turn extraction directives', () => { expect(EXTRACTION_PROMPT).not.toContain('extract only user-provided info'); }); }); + +describe('extractFacts — compact SLM envelope', () => { + it('accepts fact-only memories and backfills headline and keywords on compact', async () => { + const fact = 'User prefers PostgreSQL over MongoDB for all production databases.'; + mockLlmChat.mockResolvedValue(JSON.stringify({ memories: [{ fact }] })); + const facts = await extractFacts( + 'User: I prefer PostgreSQL over MongoDB for all production databases.', + { promptVariant: 'compact' }, + ); + const match = facts.find((f) => f.fact.includes('PostgreSQL')); + expect(match).toBeDefined(); + // normalizeRawFact: missing headline → generateFallbackHeadline(fact), + // missing keywords → derived from the fact text so keyword search has something + // to match on the compact path (it used to be left empty). + expect(match?.headline).toBe(generateFallbackHeadline(fact)); + expect(match?.keywords).toEqual(['PostgreSQL', 'MongoDB']); + expect(match?.importance).toBe(0.5); + expect(match?.type).toBe('knowledge'); + }); + + it('keeps model-provided keywords instead of deriving them', async () => { + const fact = 'User prefers PostgreSQL over MongoDB.'; + mockLlmChat.mockResolvedValue( + JSON.stringify({ memories: [{ fact, keywords: ['production databases'] }] }), + ); + const facts = await extractFacts('User: I prefer PostgreSQL over MongoDB.', { promptVariant: 'compact' }); + expect(facts[0]?.keywords).toEqual(['production databases']); + }); + + it('keeps explicit empty keywords on the full path', async () => { + const fact = 'User prefers PostgreSQL over MongoDB.'; + mockLlmChat.mockResolvedValue(JSON.stringify({ memories: [{ fact, keywords: [] }] })); + const facts = await extractFacts('User: I prefer PostgreSQL over MongoDB.', { promptVariant: 'full' }); + expect(facts[0]?.keywords).toEqual([]); + }); + + it('sanitizes untrusted keyword arrays before accepting them', async () => { + const fact = 'User prefers PostgreSQL over MongoDB.'; + mockLlmChat.mockResolvedValue( + JSON.stringify({ memories: [{ fact, keywords: [null, '', ' PostgreSQL '] }] }), + ); + const facts = await extractFacts('User: I prefer PostgreSQL over MongoDB.', { promptVariant: 'full' }); + expect(facts[0]?.keywords).toEqual(['PostgreSQL']); + }); +}); + +describe('EXTRACTION_PROMPT_COMPACT — contract agreement with the compact grammar', () => { + // The am-slm runtime's compact llguidance grammar (AM_SLM_CORE_COMPACT_SCHEMA=1) + // allows only `fact` and `type`. A prompt asking for more fields than the grammar + // permits made the model pad the array with duplicate facts instead. + it('asks for exactly the fields the compact grammar allows', () => { + expect(EXTRACTION_PROMPT_COMPACT).toContain( + '{"memories":[{"fact":"...","type":"preference|project|knowledge|person|plan"}]}', + ); + }); + + it('does not request fields the compact grammar forbids', () => { + for (const forbidden of ['"headline"', '"keywords"', '"entities"', '"relations"', '"importance"']) { + expect(EXTRACTION_PROMPT_COMPACT).not.toContain(forbidden); + } + }); + + it('states the anti-padding rules the grammar cannot express', () => { + expect(EXTRACTION_PROMPT_COMPACT).toContain('One fact per distinct claim'); + expect(EXTRACTION_PROMPT_COMPACT).toContain('Emit few facts'); + expect(EXTRACTION_PROMPT_COMPACT).toContain("Never record the assistant's commentary"); + }); +}); + +describe('deriveKeywordsFromFact', () => { + it('extracts proper nouns and technical identifiers', () => { + expect(deriveKeywordsFromFact('User prefers PostgreSQL over MongoDB in production.')).toEqual([ + 'PostgreSQL', + 'MongoDB', + ]); + }); + + it('keeps whole dates rather than splitting them', () => { + expect( + deriveKeywordsFromFact("User's sprint deadline is April 5, 2026, replacing March 29, 2026."), + ).toEqual(['April 5, 2026', 'March 29, 2026']); + }); + + it('does not emit sub-spans of a multi-word proper noun', () => { + const keywords = deriveKeywordsFromFact('User earned a degree from UC Berkeley.'); + expect(keywords).toContain('UC Berkeley'); + expect(keywords).not.toContain('Berkeley'); + }); + + it('drops the possessive subject rather than emitting it', () => { + // "User's" must not survive as a keyword; "manager" comes from the content-word + // fallback, which fires because the fact yields only one proper noun. + const keywords = deriveKeywordsFromFact("User's manager is Sarah."); + expect(keywords).toContain('Sarah'); + expect(keywords).not.toContain("User's"); + expect(keywords).not.toContain('User'); + }); + + it('falls back to content words when a fact has no proper noun', () => { + // Without this, keyword search has nothing at all to match on such facts. + expect(deriveKeywordsFromFact('User is vegetarian.')).toEqual(['vegetarian']); + }); + + it('returns nothing for empty input', () => { + expect(deriveKeywordsFromFact('')).toEqual([]); + expect(deriveKeywordsFromFact(' ')).toEqual([]); + }); + + it('caps the keyword list', () => { + const fact = 'User visited Lisbon, Nairobi, Osaka, Bogota, Helsinki, Karachi, Toronto, Warsaw, Quito, Dakar.'; + expect(deriveKeywordsFromFact(fact).length).toBeLessThanOrEqual(8); + }); +}); diff --git a/packages/core/src/services/__tests__/openai-chat-params.test.ts b/packages/core/src/services/__tests__/openai-chat-params.test.ts index 1ba4237..8b0dea0 100644 --- a/packages/core/src/services/__tests__/openai-chat-params.test.ts +++ b/packages/core/src/services/__tests__/openai-chat-params.test.ts @@ -15,6 +15,7 @@ import { OPENAI_CHAT_MAX_ATTEMPTS, openAIChatTokenLimit, openAIReasoningParams, + openAIResponseFormat, openAISamplingParams, prefersMaxCompletionTokens, reasoningEffortForModel, @@ -370,3 +371,28 @@ describe('assertOpenAIChatCompletionsModel', () => { } }); }); + +describe('openAIResponseFormat', () => { + it('prefers json_schema over json_object when both are requested', () => { + expect(openAIResponseFormat(true, { + name: 'core_audn', + strict: true, + schema: { type: 'object' }, + })).toEqual({ + response_format: { + type: 'json_schema', + json_schema: { + name: 'core_audn', + strict: true, + schema: { type: 'object' }, + }, + }, + }); + }); + + it('falls back to json_object when only jsonMode is set', () => { + expect(openAIResponseFormat(true, undefined)).toEqual({ + response_format: { type: 'json_object' }, + }); + }); +}); diff --git a/packages/core/src/services/__tests__/retrieval-config-overlay.test.ts b/packages/core/src/services/__tests__/retrieval-config-overlay.test.ts index a2a2787..8f7b484 100644 --- a/packages/core/src/services/__tests__/retrieval-config-overlay.test.ts +++ b/packages/core/src/services/__tests__/retrieval-config-overlay.test.ts @@ -11,6 +11,7 @@ import { describe, expect, it } from 'vitest'; import { applyConfigOverride, + classifyOverrideKeys, hashEffectiveConfig, summarizeOverrideKeys, } from '../retrieval-config-overlay.js'; @@ -51,6 +52,24 @@ describe('applyConfigOverride', () => { applyConfigOverride(base, { hybridSearchEnabled: true }); expect(base.hybridSearchEnabled).toBe(false); }); + + it('does not apply decode-cap overrides that LLM calls read from the singleton', () => { + const base = makeConfig({ + extractionMaxTokens: 4096, + audnMaxTokens: 2048, + audnJsonSchema: false, + }); + const result = applyConfigOverride(base, { + extractionMaxTokens: 128, + audnMaxTokens: 64, + audnJsonSchema: true, + hybridSearchEnabled: true, + }); + expect(result.extractionMaxTokens).toBe(4096); + expect(result.audnMaxTokens).toBe(2048); + expect(result.audnJsonSchema).toBe(false); + expect(result.hybridSearchEnabled).toBe(true); + }); }); describe('hashEffectiveConfig', () => { @@ -72,6 +91,30 @@ describe('hashEffectiveConfig', () => { }); }); +describe('classifyOverrideKeys', () => { + const known = new Set(['hybridSearchEnabled', 'extractionMaxTokens', 'audnMaxTokens']); + + it('reports denylisted RuntimeConfig fields as ignored, not unknown', () => { + expect(classifyOverrideKeys({ extractionMaxTokens: 128, audnMaxTokens: 64 }, known)).toEqual({ + applied: [], + ignored: ['audnMaxTokens', 'extractionMaxTokens'], + unknown: [], + }); + }); + + it('keeps applied keys separate from ignored and unknown', () => { + expect(classifyOverrideKeys({ + hybridSearchEnabled: true, + extractionMaxTokens: 128, + futureFieldX: true, + } as Partial, known)).toEqual({ + applied: ['futureFieldX', 'hybridSearchEnabled'], + ignored: ['extractionMaxTokens'], + unknown: ['futureFieldX'], + }); + }); +}); + describe('summarizeOverrideKeys', () => { it('returns an empty string for an empty override', () => { expect(summarizeOverrideKeys({})).toBe(''); diff --git a/packages/core/src/services/consensus-extraction.ts b/packages/core/src/services/consensus-extraction.ts index 1bdc4c7..aa05449 100644 --- a/packages/core/src/services/consensus-extraction.ts +++ b/packages/core/src/services/consensus-extraction.ts @@ -11,6 +11,7 @@ * N× extraction API calls. */ +import type { ExtractionPromptVariant } from './extraction-prompt-variant.js'; import { extractFacts, type ExtractedFact } from './extraction.js'; import { cachedExtractFacts } from './extraction-cache.js'; import { chunkedExtractFacts } from './chunked-extraction.js'; @@ -34,6 +35,7 @@ export interface ConsensusExtractionConfig { chunkOverlapTurns: number; extractionCacheEnabled: boolean; observationDateExtractionEnabled: boolean; + extractionPromptVariant: ExtractionPromptVariant; quotedEntityExtractionEnabled: boolean; } @@ -86,7 +88,7 @@ function applyOptionalQuotedEntityExtraction( /** Run extractFacts() N times to get independent LLM samples. */ async function runMultipleExtractions( conversationText: string, - config: Pick, + config: Pick, ): Promise { const allRunFacts: ExtractedFact[][] = []; const options = buildExtractionOptions(config); @@ -97,10 +99,11 @@ async function runMultipleExtractions( } function buildExtractionOptions( - config: Pick, + config: Pick, ) { return { observationDateExtractionEnabled: config.observationDateExtractionEnabled, + promptVariant: config.extractionPromptVariant, }; } diff --git a/packages/core/src/services/deferred-audn.ts b/packages/core/src/services/deferred-audn.ts index 2231ee9..ad09250 100644 --- a/packages/core/src/services/deferred-audn.ts +++ b/packages/core/src/services/deferred-audn.ts @@ -162,7 +162,7 @@ async function reconcileSingleMemory( } const decision = applyClarificationOverrides( - await cachedResolveAUDN(memory.content, candidates), + await cachedResolveAUDN(memory.content, candidates, config.extractionPromptVariant), memory.content, candidates, [], diff --git a/packages/core/src/services/extraction-cache.ts b/packages/core/src/services/extraction-cache.ts index 36fd2e5..6b099fb 100644 --- a/packages/core/src/services/extraction-cache.ts +++ b/packages/core/src/services/extraction-cache.ts @@ -17,6 +17,10 @@ import { type ExtractedFact, type ExistingMemory, } from './extraction.js'; +import type { ExtractionPromptVariant } from './extraction-prompt-variant.js'; + +/** Bump when cache identity inputs change (prompt contract, token caps, normalizers). */ +export const EXTRACTION_CACHE_CONTRACT = 'v3'; function hashInput(parts: string[]): string { return createHash('sha256').update(parts.join('\0')).digest('hex').slice(0, 16); @@ -45,7 +49,12 @@ export async function cachedExtractFacts( ): Promise { if (!config.extractionCacheEnabled) return extractFacts(conversationText, options); - const key = `extract-${hashInput([conversationText, JSON.stringify(options)])}`; + const key = `extract-${hashInput([ + conversationText, + JSON.stringify(options), + EXTRACTION_CACHE_CONTRACT, + String(config.extractionMaxTokens), + ])}`; const filePath = cacheFilePath(key); const cached = readCache(filePath); if (cached) return cached; @@ -58,16 +67,26 @@ export async function cachedExtractFacts( export async function cachedResolveAUDN( newFact: string, existingMemories: ExistingMemory[], + promptVariant: ExtractionPromptVariant, ): Promise { - if (!config.extractionCacheEnabled) return resolveAUDN(newFact, existingMemories); + if (!config.extractionCacheEnabled) { + return resolveAUDN(newFact, existingMemories, promptVariant); + } const memoriesKey = JSON.stringify(existingMemories.map((m) => ({ id: m.id, content: m.content, similarity: m.similarity }))); - const key = `audn-${hashInput([newFact, memoriesKey])}`; + const key = `audn-${hashInput([ + newFact, + memoriesKey, + promptVariant, + EXTRACTION_CACHE_CONTRACT, + String(config.audnMaxTokens), + String(config.audnJsonSchema), + ])}`; const filePath = cacheFilePath(key); const cached = readCache(filePath); if (cached) return cached; - const result = await resolveAUDN(newFact, existingMemories); + const result = await resolveAUDN(newFact, existingMemories, promptVariant); writeCache(filePath, result); return result; } diff --git a/packages/core/src/services/extraction-json-schema.ts b/packages/core/src/services/extraction-json-schema.ts new file mode 100644 index 0000000..999c08b --- /dev/null +++ b/packages/core/src/services/extraction-json-schema.ts @@ -0,0 +1,61 @@ +/** + * Structured JSON schemas for extraction/AUDN LLM calls. + * AUDN uses an explicit schema so am-local-slm does not route it through + * the extraction-only `core_extraction_schema()` grammar. + */ + +import type { ChatOptions } from './llm.js'; + +export const AUDN_SCHEMA_NAME = 'core_audn'; + +/** Matches the AUDN parser fields in extraction.ts. */ +const AUDN_RESPONSE_SCHEMA: Record = { + type: 'object', + additionalProperties: false, + properties: { + action: { + type: 'string', + enum: ['ADD', 'UPDATE', 'SUPERSEDE', 'DELETE', 'NOOP', 'CLARIFY'], + }, + target_memory_id: { type: ['string', 'null'] }, + updated_content: { type: ['string', 'null'] }, + clarification_note: { type: ['string', 'null'] }, + contradiction_confidence: { type: ['number', 'null'] }, + }, + // OpenAI strict json_schema requires every property in `required`; + // nullable types represent optional values. + required: [ + 'action', + 'target_memory_id', + 'updated_content', + 'clarification_note', + 'contradiction_confidence', + ], +}; + +/** Chat options that request the AUDN grammar instead of generic json_object. */ +export function audnJsonChatOptions(maxTokens: number): Pick { + return { + jsonMode: true, + maxTokens, + jsonSchema: { + name: AUDN_SCHEMA_NAME, + strict: true, + schema: AUDN_RESPONSE_SCHEMA, + }, + }; +} + +/** + * Prompt variant and wire format are independent axes. Send json_schema + * only when the provider can honor a client grammar (am-local-slm with + * AM_SLM_CORE_JSON_SCHEMA=1). Groq and other OpenAI-compatible hosts that + * reject strict Structured Outputs stay on json_object. + */ +export function audnChatOptions( + useJsonSchema: boolean, + maxTokens: number, +): Pick { + if (useJsonSchema) return audnJsonChatOptions(maxTokens); + return { jsonMode: true, maxTokens }; +} diff --git a/packages/core/src/services/extraction-keywords.ts b/packages/core/src/services/extraction-keywords.ts new file mode 100644 index 0000000..83b06fe --- /dev/null +++ b/packages/core/src/services/extraction-keywords.ts @@ -0,0 +1,97 @@ +/** + * Keyword derivation for compact extraction backfill. + * Uses Unicode letter/number classes so international facts retain retrievable terms. + */ + +const KEYWORDS_MAX = 8; +const CONTENT_WORD_MIN_LENGTH_ASCII = 4; +const CONTENT_WORD_MIN_LENGTH_UNICODE = 2; + +const KEYWORD_STOPWORDS = new Set([ + 'user', 'users', 'assistant', 'the', 'a', 'an', 'this', 'that', 'it', 'they', + 'is', 'are', 'was', 'were', 'be', 'been', 'being', 'am', 'has', 'have', 'had', + 'does', 'did', 'do', 'will', 'would', 'can', 'could', 'should', 'may', 'might', + 'and', 'or', 'but', 'for', 'with', 'from', 'into', 'about', 'over', 'under', + 'of', 'in', 'on', 'at', 'to', 'as', 'by', 'not', 'no', 'than', 'then', 'also', + 'instead', 'earlier', 'now', 'still', 'always', 'never', 'their', 'there', + 'which', 'when', 'where', 'while', 'because', 'these', 'those', 'some', 'any', + 'all', 'both', 'each', 'more', 'most', 'other', 'such', 'only', 'very', + 'lives', 'live', 'vive', 'usuario', + 'el', 'la', 'los', 'las', 'en', 'de', +]); + +/** Title-case or all-caps word runs, including accented Latin (Zürich, São Paulo). */ +const PROPER_NOUN_RUN = /\b[\p{Lu}][\p{L}\p{M}'’-]*(?:\s+[\p{Lu}][\p{L}\p{M}'’-]*)*/gu; +/** Mixed-case technical tokens: PostgreSQL, Qwen3, GPT-4. */ +const TECHNICAL_TOKEN = /\b(?=[\p{L}\p{N}_-]*[\p{Lu}])(?=[\p{L}\p{N}_-]*[\p{Ll}\p{N}])[\p{L}\p{N}_-]+\b|\b[\p{Ll}]+\d[\p{L}\p{N}_-]*/gu; +const DATE_LIKE = + /\b(?:\d{4}-\d{2}-\d{2}|(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[a-z]*\s+\d{1,2},?\s*\d{4}?|\b(?:19|20)\d{2})\b/g; + +interface KeywordAccumulator { + seen: Set; + terms: string[]; +} + +function normalizeKeywordTerm(raw: string): string { + return raw.trim().replace(/[.,;:]+$/, '').replace(/['’]s$/iu, ''); +} + +function addKeyword(acc: KeywordAccumulator, raw: string): void { + const term = normalizeKeywordTerm(raw); + if (term.length < 2) return; + const key = term.toLowerCase(); + if (KEYWORD_STOPWORDS.has(key) || acc.seen.has(key)) return; + if (acc.terms.some((existing) => existing.toLowerCase().includes(key))) return; + acc.seen.add(key); + acc.terms.push(term); +} + +function trimLeadingStopwords(run: string[]): string[] { + let start = 0; + while (start < run.length - 1 && KEYWORD_STOPWORDS.has(run[start].toLowerCase())) start += 1; + return run.slice(start); +} + +function collectPatternKeywords(fact: string, acc: KeywordAccumulator): void { + for (const pattern of [DATE_LIKE, PROPER_NOUN_RUN, TECHNICAL_TOKEN]) { + for (const match of fact.matchAll(pattern)) { + addKeyword(acc, trimLeadingStopwords(match[0].split(/\s+/)).join(' ')); + if (acc.terms.length >= KEYWORDS_MAX) return; + } + } +} + +function isMostlyAscii(text: string): boolean { + return !/[^\x00-\x7F]/.test(text); +} + +function contentWordMinLength(word: string): number { + return isMostlyAscii(word) ? CONTENT_WORD_MIN_LENGTH_ASCII : CONTENT_WORD_MIN_LENGTH_UNICODE; +} + +function collectContentWordKeywords(fact: string, acc: KeywordAccumulator): void { + for (const word of fact.split(/[^\p{L}\p{N}'’-]+/u)) { + if (word.length >= contentWordMinLength(word)) addKeyword(acc, word); + if (acc.terms.length >= KEYWORDS_MAX) return; + } +} + +const MIN_PATTERN_KEYWORDS = 2; + +/** Derive keyword-search terms from fact text when the model omits them. */ +export function deriveKeywordsFromFact(fact: string): string[] { + if (!fact.trim()) return []; + const acc: KeywordAccumulator = { seen: new Set(), terms: [] }; + collectPatternKeywords(fact, acc); + if (acc.terms.length < MIN_PATTERN_KEYWORDS) collectContentWordKeywords(fact, acc); + return acc.terms; +} + +/** Keep trimmed non-empty strings from a model-provided keyword array. */ +export function sanitizeKeywords(raw: unknown): string[] { + if (!Array.isArray(raw)) return []; + return raw + .filter((entry): entry is string => typeof entry === 'string') + .map((entry) => entry.trim()) + .filter((entry) => entry.length > 0); +} diff --git a/packages/core/src/services/extraction-prompt-variant.ts b/packages/core/src/services/extraction-prompt-variant.ts new file mode 100644 index 0000000..a65c66b --- /dev/null +++ b/packages/core/src/services/extraction-prompt-variant.ts @@ -0,0 +1,17 @@ +/** + * Closed prompt-variant type for extraction and AUDN. + * Parsed once at startup from EXTRACTION_PROMPT_VARIANT. + */ + +export type ExtractionPromptVariant = 'full' | 'compact'; + +const VALID_VARIANTS: ReadonlySet = new Set(['full', 'compact']); + +/** Fail-closed parse: only exact `full` or `compact`; unset defaults to `full`. */ +export function parseExtractionPromptVariant(raw: string | undefined): ExtractionPromptVariant { + if (raw === undefined || raw === '') return 'full'; + if (VALID_VARIANTS.has(raw as ExtractionPromptVariant)) { + return raw as ExtractionPromptVariant; + } + throw new Error(`EXTRACTION_PROMPT_VARIANT must be 'full' or 'compact' (got '${raw}')`); +} diff --git a/packages/core/src/services/extraction.ts b/packages/core/src/services/extraction.ts index 7e9ebcf..85a7940 100644 --- a/packages/core/src/services/extraction.ts +++ b/packages/core/src/services/extraction.ts @@ -7,6 +7,7 @@ */ import { llm } from './llm.js'; +import { config } from '../config.js'; import { withCostStage } from './cost-telemetry.js'; import { timed, timedSync } from './timing.js'; import { normalizeExtractedFacts } from './fact-normalization.js'; @@ -18,9 +19,12 @@ import { type ExtractionOptions, } from './observation-date-extraction.js'; import { filterMetaFacts } from './meta-fact-filter.js'; +import { audnChatOptions } from './extraction-json-schema.js'; +import { deriveKeywordsFromFact, sanitizeKeywords } from './extraction-keywords.js'; +import type { ExtractionPromptVariant } from './extraction-prompt-variant.js'; -const EXTRACTION_MAX_TOKENS = 4096; -const AUDN_MAX_TOKENS = 2048; +export type { ExtractionPromptVariant } from './extraction-prompt-variant.js'; +export { deriveKeywordsFromFact } from './extraction-keywords.js'; export type { ExtractionOptions }; @@ -112,7 +116,12 @@ function repairTruncatedJson(raw: string): string | null { if (isValidJson(candidate)) return candidate; const repaired = closeAtLastCompleteArrayEntry(candidate); - return repaired && isValidJson(repaired) ? repaired : null; + if (repaired && isValidJson(repaired)) return repaired; + + // One complete array entry then mid-entry truncation: slice ends at `}` so + // there is no `},` marker — close the open `[` / `{` containers directly. + const closed = closeOpenJsonContainers(candidate); + return isValidJson(closed) ? closed : null; } function removeTrailingJsonCommas(value: string): string { @@ -309,39 +318,161 @@ OUTPUT FORMAT (JSON): If no extractable facts exist, return: {"memories": []}`; +/** + * Short prompt for latency-tuned SLM paths (default-off via EXTRACTION_PROMPT_VARIANT). + * + * **Reduced capability:** records user-stated facts only. Unlike the full prompt, compact + * does not retain assistant recommendations, contact/entity/date guarantees, or short-input + * rules. Opt in knowingly via EXTRACTION_PROMPT_VARIANT=compact. + * + * The stated contract here must match the `am-slm` runtime's compact llguidance + * grammar (`AM_SLM_CORE_COMPACT_SCHEMA=1`), which allows only `fact` and `type` and + * forbids `headline`/`keywords`/`entities`/`relations`. An earlier revision asked for + * five fields against that two-field grammar; the model tried to spend tokens on an + * envelope the grammar refused, and the mass spilled into duplicate array entries + * (77 facts extracted vs 46 from the cloud ceiling on the same 24 conversations). + * Keep prompt, grammar and LoRA training targets stating one identical contract. + * + * `headline` and `keywords` are backfilled from the fact text in normalizeRawFact, + * so omitting them costs no retrieval quality: measured on 593 teacher-labelled + * memories, 94% of model-chosen keywords were already literal substrings of the fact. + */ +export const EXTRACTION_PROMPT_COMPACT = `You extract atomic, retrievable user facts from the conversation below. + +Return JSON in exactly this shape, with no other keys: +{"memories":[{"fact":"...","type":"preference|project|knowledge|person|plan"}]} + +RULES: +- One fact per distinct claim. Never restate, rephrase, or split a claim you already emitted. +- Emit few facts when the conversation contains few. Fewer, sharper facts beat more. +- Terse third-person single clause, under 160 characters. No trailing rationale. +- Record what the USER said or is. Never record the assistant's commentary or advice. +- Do not open with a date that the rest of the fact already states. +- One technology/tool/framework per fact. +- Preserve supersessions ("replacing X", "instead of Y"). +- Skip filler, pleasantries, and meta-observations about the chat. +- If nothing durable is present, return {"memories": []}.`; + +export function resolveExtractionPrompt(variant: ExtractionPromptVariant | undefined): string { + return variant === 'compact' ? EXTRACTION_PROMPT_COMPACT : EXTRACTION_PROMPT; +} + +/** Nudge used on a single retry when the first completion was non-empty but unparseable. */ +const EXTRACTION_JSON_ONLY_NUDGE = + 'Respond with ONLY the JSON object matching the schema. No prose, no markdown fences, no explanation.'; + +/** + * Raw LLM text long enough that a failed parse is unexpected noise rather than + * an empty completion. Matches the ATO-2185 "after JSON at position 22" class. + */ +const SUBSTANTIAL_EXTRACTION_RAW_MIN_CHARS = 20; + +/** Process-lifetime counter for extract parse failures after retry (ATO-2185). */ +let extractParseFailureCount = 0; + +/** Operator-facing count of extract completions that stayed unparseable after retry. */ +export function getExtractParseFailureCount(): number { + return extractParseFailureCount; +} + +/** Reset parse-failure counter between tests. */ +export function resetExtractParseFailureCount(): void { + extractParseFailureCount = 0; +} + export async function extractFacts( conversationText: string, options: ExtractionOptions = {}, ): Promise { - const content = await timed('ingest.extract.llm', () => withCostStage('extract', () => llm.chat( + const systemPrompt = resolveExtractionPrompt(options.promptVariant); + const userMessage = buildExtractionUserMessage(conversationText, options); + const content = await callExtractionLlm(systemPrompt, userMessage); + if (!content) return []; + + const rawFacts = await parseExtractionWithRetry(content, systemPrompt, userMessage); + if (!rawFacts) return []; + + return timedSync('ingest.extract.post-process', () => + postProcessExtractedFacts(rawFacts, conversationText, options), + ); +} + +async function callExtractionLlm(systemPrompt: string, userMessage: string): Promise { + return timed('ingest.extract.llm', () => withCostStage('extract', () => llm.chat( [ - { role: 'system', content: EXTRACTION_PROMPT }, - { role: 'user', content: buildExtractionUserMessage(conversationText, options) }, + { role: 'system', content: systemPrompt }, + { role: 'user', content: userMessage }, ], - { temperature: 0, jsonMode: true, maxTokens: EXTRACTION_MAX_TOKENS }, + { temperature: 0, jsonMode: true, maxTokens: config.extractionMaxTokens }, ))); +} - if (!content) return []; +/** + * Parse extraction JSON; on substantial but unparseable content, retry once with a + * JSON-only system nudge. Empty `{"memories":[]}` is valid and does not retry. + */ +async function parseExtractionWithRetry( + content: string, + systemPrompt: string, + userMessage: string, +): Promise<(RawExtractedFact | LeafFact)[] | null> { + const first = timedSync('ingest.extract.parse', () => parseExtractionResponse(content)); + if (first !== null) return first; + if (!isSubstantialExtractionContent(content)) return null; + + const retryContent = await retryExtractionForJsonOnly(systemPrompt, userMessage); + const retried = timedSync('ingest.extract.parse-retry', () => + parseExtractionResponse(retryContent ?? ''), + ); + if (retried !== null) return retried; - const rawFacts = timedSync('ingest.extract.parse', () => parseExtractionResponse(content)); - if (!rawFacts) return []; + recordExtractParseFailure(); + return null; +} + +function isSubstantialExtractionContent(content: string): boolean { + return content.trim().length >= SUBSTANTIAL_EXTRACTION_RAW_MIN_CHARS && content.includes('{'); +} + +async function retryExtractionForJsonOnly( + systemPrompt: string, + userMessage: string, +): Promise { + return timed('ingest.extract.llm-retry', () => withCostStage('extract', () => llm.chat( + [ + { role: 'system', content: `${systemPrompt}\n\n${EXTRACTION_JSON_ONLY_NUDGE}` }, + { role: 'user', content: userMessage }, + ], + { temperature: 0, jsonMode: true, maxTokens: config.extractionMaxTokens }, + ))); +} - return timedSync('ingest.extract.post-process', () => { - const normalized: ExtractedFact[] = rawFacts.map((m) => normalizeRawFact(m)); - const anchoredFacts = applyObservationDateAnchors(normalized, conversationText, options); - const baseFacts = enrichExtractedFacts(normalizeExtractedFacts(anchoredFacts)); - const merged = mergeSupplementalFacts(baseFacts, conversationText); - // Drop extraction-style meta-facts that describe the conversation - // itself rather than recording a durable user fact. These poison - // the embedding pool downstream. The filter is on by default; - // operators can disable for incident response via - // ATOMICMEMORY_META_FACT_FILTER=off. See - // src/services/meta-fact-filter.ts for rationale + AlignBench v0 - // (the AlignBench v0 results) for evidence. - // Drops are logged structured ("[meta-fact-filter] dropped …") and - // counted via getMetaFactDropStats() for operator monitoring. - return filterMetaFacts(merged, { source: 'extract' }); - }); +function recordExtractParseFailure(): void { + extractParseFailureCount += 1; + console.error( + '[extractFacts] parse failed after retry; returning empty.', + ); +} + +function postProcessExtractedFacts( + rawFacts: (RawExtractedFact | LeafFact)[], + conversationText: string, + options: ExtractionOptions, +): ExtractedFact[] { + const normalized: ExtractedFact[] = rawFacts.map((m) => normalizeRawFact(m, options.promptVariant)); + const anchoredFacts = applyObservationDateAnchors(normalized, conversationText, options); + const baseFacts = enrichExtractedFacts(normalizeExtractedFacts(anchoredFacts)); + const merged = mergeSupplementalFacts(baseFacts, conversationText); + // Drop extraction-style meta-facts that describe the conversation + // itself rather than recording a durable user fact. These poison + // the embedding pool downstream. The filter is on by default; + // operators can disable for incident response via + // ATOMICMEMORY_META_FACT_FILTER=off. See + // src/services/meta-fact-filter.ts for rationale + AlignBench v0 + // (the AlignBench v0 results) for evidence. + // Drops are logged structured ("[meta-fact-filter] dropped …") and + // counted via getMetaFactDropStats() for operator monitoring. + return filterMetaFacts(merged, { source: 'extract' }); } type RawExtractedFact = ExtractedFact & { @@ -352,9 +483,14 @@ type RawExtractedFact = ExtractedFact & { relations?: ExtractedRelation[]; }; -/** Parse and validate LLM extraction response, returning raw facts or null on failure. */ +/** + * Parse and validate LLM extraction response, returning raw facts or null on failure. + * Uses extractFirstJsonObject (same as AUDN) so trailing prose / second JSON blocks + * do not trip JSON.parse — the ATO-2185 "non-whitespace after JSON at position N" mode. + * Incomplete first objects still fall through to truncated-JSON repair. + */ function parseExtractionResponse(content: string): (RawExtractedFact | LeafFact)[] | null { - const cleanedContent = stripJsonFences(content); + const cleanedContent = extractFirstJsonObject(content); const parsed = parseJsonWithRepair(cleanedContent); if (!parsed) return null; return resolveFactArray(parsed, content); @@ -365,19 +501,19 @@ function parseJsonWithRepair( cleanedContent: string, ): Record | null { try { - return JSON.parse(cleanedContent); - } catch (err) { - console.warn(`[extractFacts] JSON parse failed (${(err as Error).message}); attempting repair`); + return JSON.parse(cleanedContent) as Record; + } catch { + console.warn('[extractFacts] JSON parse failed; attempting repair'); } const repaired = repairTruncatedJson(cleanedContent); if (!repaired) { - console.warn('[extractFacts] No valid JSON found; returning empty. Raw:', cleanedContent.slice(0, 300)); + console.warn('[extractFacts] No valid JSON found; returning empty.'); return null; } try { - return JSON.parse(repaired); + return JSON.parse(repaired) as Record; } catch { - console.warn('[extractFacts] JSON repair failed; returning empty. Raw:', cleanedContent.slice(0, 300)); + console.warn('[extractFacts] JSON repair failed; returning empty.'); return null; } } @@ -404,19 +540,64 @@ function resolveFactArray( } /** Normalize a single raw extracted fact into the canonical ExtractedFact shape. */ -function normalizeRawFact(m: RawExtractedFact | LeafFact): ExtractedFact { +const VALID_FACT_TYPES = new Set([ + 'preference', + 'project', + 'knowledge', + 'person', + 'plan', +]); + +const DEFAULT_IMPORTANCE = 0.5; + +function coerceImportance(raw: unknown): number { + const value = Number(raw); + return Number.isFinite(value) ? Math.max(0, Math.min(1, value)) : DEFAULT_IMPORTANCE; +} + +function coerceFactType(raw: unknown): ExtractedFact['type'] { + const candidate = typeof raw === 'string' ? raw.toLowerCase() : ''; + return VALID_FACT_TYPES.has(candidate as ExtractedFact['type']) + ? (candidate as ExtractedFact['type']) + : 'knowledge'; +} + +/** + * Backfill the retrieval envelope the compact SLM contract omits. + * + * Both fields are derived from the fact text rather than requested from the model, so + * the compact grammar can forbid them without degrading keyword search. + */ +function coerceKeywords( + raw: unknown, + fact: string, + promptVariant: ExtractionPromptVariant | undefined, +): string[] { + const sanitized = sanitizeKeywords(raw); + if (sanitized.length > 0) return sanitized; + if (promptVariant === 'compact' || raw === undefined) { + return deriveKeywordsFromFact(fact); + } + return []; +} + +function coerceHeadline(raw: unknown, fact: string): string { + return typeof raw === 'string' && raw.trim() ? raw.trim() : generateFallbackHeadline(fact); +} + +function normalizeRawFact( + m: RawExtractedFact | LeafFact, + promptVariant: ExtractionPromptVariant | undefined, +): ExtractedFact { const rawEntry = m as RawExtractedFact; const fact = rawEntry.fact ?? rawEntry.statement ?? ''; - const rawImportance = Number(m.importance); - const importance = Number.isFinite(rawImportance) ? Math.max(0, Math.min(1, rawImportance)) : 0.5; - const VALID_TYPES = new Set(['preference', 'project', 'knowledge', 'person', 'plan']); - const rawType = typeof m.type === 'string' ? m.type.toLowerCase() : ''; + const keywordsPresent = Object.prototype.hasOwnProperty.call(rawEntry, 'keywords'); return { fact, - importance, - type: VALID_TYPES.has(rawType as ExtractedFact['type']) ? rawType as ExtractedFact['type'] : 'knowledge', - keywords: Array.isArray(m.keywords) ? m.keywords : [], - headline: typeof m.headline === 'string' && m.headline.trim() ? m.headline.trim() : generateFallbackHeadline(fact), + importance: coerceImportance(m.importance), + type: coerceFactType(m.type), + keywords: coerceKeywords(keywordsPresent ? rawEntry.keywords : undefined, fact, promptVariant), + headline: coerceHeadline(m.headline, fact), entities: normalizeExtractedEntities(m.entities), relations: normalizeExtractedRelations(m.relations), }; @@ -550,9 +731,14 @@ OUTPUT FORMAT (JSON): Return only the JSON object. Do not wrap it in markdown fences. Do not explain your reasoning. `; +export function resolveAudnPrompt(_variant: ExtractionPromptVariant): string { + return AUDN_PROMPT; +} + export async function resolveAUDN( newFact: string, existingMemories: ExistingMemory[], + promptVariant: ExtractionPromptVariant, ): Promise { const memoriesBlock = existingMemories .map((m) => `[ID: ${m.id}] (similarity: ${m.similarity.toFixed(2)}) ${m.content}`) @@ -560,10 +746,10 @@ export async function resolveAUDN( const content = await llm.chat( [ - { role: 'system', content: AUDN_PROMPT }, + { role: 'system', content: resolveAudnPrompt(promptVariant) }, { role: 'user', content: `NEW FACT: ${newFact}\n\nEXISTING MEMORIES:\n${memoriesBlock}` }, ], - { temperature: 0, jsonMode: true, maxTokens: AUDN_MAX_TOKENS }, + { temperature: 0, ...audnChatOptions(config.audnJsonSchema, config.audnMaxTokens) }, ); if (!content) { diff --git a/packages/core/src/services/llm.ts b/packages/core/src/services/llm.ts index ec54303..17b0adb 100644 --- a/packages/core/src/services/llm.ts +++ b/packages/core/src/services/llm.ts @@ -12,6 +12,7 @@ import { OPENAI_CHAT_MAX_ATTEMPTS, openAIChatTokenLimit, openAIReasoningParams, + openAIResponseFormat, openAISamplingParams, tryApplyOpenAIRetry, type OpenAIRetryState, @@ -87,6 +88,11 @@ export interface ChatOptions { temperature?: number; maxTokens?: number; jsonMode?: boolean; + jsonSchema?: { + name: string; + strict?: boolean; + schema: Record; + }; seed?: number; } @@ -217,7 +223,7 @@ class OpenAICompatibleLLM implements LLMProvider { ...openAISamplingParams(this.model, options.temperature, effectiveSeed), ...openAIChatTokenLimit(this.model, options.maxTokens, forceMaxCompletionTokens), ...openAIReasoningParams(this.model, forceMaxCompletionTokens), - ...(options.jsonMode ? { response_format: { type: 'json_object' as const } } : {}), + ...openAIResponseFormat(options.jsonMode, options.jsonSchema), }); const started = performance.now(); diff --git a/packages/core/src/services/memory-audn.ts b/packages/core/src/services/memory-audn.ts index c36e8f4..d007c6b 100644 --- a/packages/core/src/services/memory-audn.ts +++ b/packages/core/src/services/memory-audn.ts @@ -96,7 +96,11 @@ export async function resolveAndExecuteAudn( return resolveAndExecuteTbc(deps, ctx, filteredCandidates, candidateIds, supersededTargets, requireTraceContext(traceContext)); } - const rawDecision = await timed('ingest.fact.audn', () => cachedResolveAUDN(fact.fact, filteredCandidates)); + const rawDecision = await timed('ingest.fact.audn', () => cachedResolveAUDN( + fact.fact, + filteredCandidates, + deps.config.extractionPromptVariant, + )); let decision = applyClarificationOverrides(rawDecision, fact.fact, filteredCandidates, fact.keywords, fact.type); if (deps.config.entityGraphEnabled && deps.stores.entity) { decision = await applyEntityScopedDedup(deps, decision, userId, fact.entities); diff --git a/packages/core/src/services/memory-service-types.ts b/packages/core/src/services/memory-service-types.ts index 01e097b..ca4bee6 100644 --- a/packages/core/src/services/memory-service-types.ts +++ b/packages/core/src/services/memory-service-types.ts @@ -6,6 +6,7 @@ import { type TrustScore } from './trust-scoring.js'; import { type ExtractedEntity, type ExtractedRelation } from './extraction.js'; import { type MemoryNetwork } from './memory-network.js'; import type { AUDNAction } from './extraction.js'; +import type { ExtractionPromptVariant } from './extraction-prompt-variant.js'; import type { BeliefOperator } from './typed-belief-calculus.js'; import { type ClaimSlotInput } from '../db/claim-repository.js'; @@ -387,6 +388,7 @@ export interface IngestRuntimeConfig { consensusExtractionRuns: number; extractionCacheEnabled: boolean; observationDateExtractionEnabled: boolean; + extractionPromptVariant: ExtractionPromptVariant; quotedEntityExtractionEnabled: boolean; entityGraphEnabled: boolean; entropyGateAlpha: number; diff --git a/packages/core/src/services/observation-date-extraction.ts b/packages/core/src/services/observation-date-extraction.ts index 7529b2d..e9607d3 100644 --- a/packages/core/src/services/observation-date-extraction.ts +++ b/packages/core/src/services/observation-date-extraction.ts @@ -6,6 +6,7 @@ */ import type { ExtractedFact } from './extraction.js'; +import type { ExtractionPromptVariant } from './extraction-prompt-variant.js'; import { annotateRelativeTemporalText, extractRelativeTemporalAnchors, @@ -14,6 +15,8 @@ import { extractSessionTimestamp, parseSessionDate } from './session-date.js'; export interface ExtractionOptions { observationDateExtractionEnabled?: boolean; + /** `compact` selects shorter SLM-oriented prompts (default-off, reduced capability). */ + promptVariant?: ExtractionPromptVariant; } export function buildExtractionUserMessage( diff --git a/packages/core/src/services/openai-chat-params.ts b/packages/core/src/services/openai-chat-params.ts index a418fbc..1ed1f85 100644 --- a/packages/core/src/services/openai-chat-params.ts +++ b/packages/core/src/services/openai-chat-params.ts @@ -8,6 +8,40 @@ /** Chat Completions reasoning_effort values we intentionally set. */ export type ReasoningEffort = 'none' | 'minimal' | 'low'; +export interface OpenAIJsonSchemaRequest { + name: string; + strict?: boolean; + schema: Record; +} + +/** Prefer client json_schema over generic json_object when both are requested. */ +export function openAIResponseFormat( + jsonMode: boolean | undefined, + jsonSchema: OpenAIJsonSchemaRequest | undefined, +): { response_format: { type: 'json_object' } } | { + response_format: { + type: 'json_schema'; + json_schema: { name: string; strict: boolean; schema: Record }; + }; +} | Record { + if (jsonSchema) { + return { + response_format: { + type: 'json_schema', + json_schema: { + name: jsonSchema.name, + strict: jsonSchema.strict ?? true, + schema: jsonSchema.schema, + }, + }, + }; + } + if (jsonMode) { + return { response_format: { type: 'json_object' } }; + } + return {}; +} + /** Model name with any provider prefix (e.g. `openai/`) stripped, lowercased. */ function bareModelName(model: string): string { const normalized = model.toLowerCase(); diff --git a/packages/core/src/services/retrieval-config-overlay.ts b/packages/core/src/services/retrieval-config-overlay.ts index 872c197..dd41f96 100644 --- a/packages/core/src/services/retrieval-config-overlay.ts +++ b/packages/core/src/services/retrieval-config-overlay.ts @@ -16,9 +16,10 @@ * response header so callers can link traces to a canonical config * fingerprint. * - * 3. `summarizeOverrideKeys` — comma-separated list of top-level keys - * present in the override object, for the - * `X-Atomicmem-Config-Override-Keys` header. + * 3. `classifyOverrideKeys` — applied / ignored / unknown split for + * `X-Atomicmem-Config-Override-Keys`, + * `X-Atomicmem-Ignored-Override-Keys`, and + * `X-Atomicmem-Unknown-Override-Keys`. * * Public contract: request overrides are validated before this helper sees * them, so the merge step remains intentionally shallow and deterministic. @@ -27,12 +28,48 @@ import { createHash } from 'node:crypto'; import type { RuntimeConfig } from '../config.js'; +/** + * Decode caps and AUDN wire format are read from the process singleton. + * Accepting them on config_override would report "applied" while doing nothing. + */ +const REQUEST_SINGLETON_ONLY_CONFIG_KEYS = new Set([ + 'extractionMaxTokens', + 'audnMaxTokens', + 'audnJsonSchema', +]); + +/** Applied / ignored / unknown classification for override response headers. */ +export function classifyOverrideKeys( + override: Partial, + knownKeys: ReadonlySet, +): { applied: string[]; ignored: string[]; unknown: string[] } { + const ignored = nonOverridableOverrideKeys(override); + const ignoredSet = new Set(ignored); + const submitted = Object.keys(override); + return { + applied: submitted.filter((key) => !ignoredSet.has(key)).sort(), + ignored, + unknown: submitted.filter((key) => !knownKeys.has(key)).sort(), + }; +} + +/** Override keys that cannot take effect on the current request path. */ +function nonOverridableOverrideKeys(override: Partial): string[] { + return Object.keys(override) + .filter((key) => REQUEST_SINGLETON_ONLY_CONFIG_KEYS.has(key)) + .sort(); +} + /** Merge a validated override on top of the startup runtime config. */ export function applyConfigOverride( base: RuntimeConfig, override: Partial, ): RuntimeConfig { - return { ...base, ...override }; + const applied = { ...override }; + for (const key of REQUEST_SINGLETON_ONLY_CONFIG_KEYS) { + delete applied[key as keyof RuntimeConfig]; + } + return { ...base, ...applied }; } /** diff --git a/scripts/__tests__/cli-install-smoke-reporter.test.sh b/scripts/__tests__/cli-install-smoke-reporter.test.sh new file mode 100755 index 0000000..58d414b --- /dev/null +++ b/scripts/__tests__/cli-install-smoke-reporter.test.sh @@ -0,0 +1,144 @@ +#!/usr/bin/env bash +# Tests the failure reporter embedded in .github/workflows/cli-install-smoke.yml. +# +# The reporter decides, unattended, whether an issue is opened or closed. Its +# failure modes are quiet and backwards — closing a live issue, or reporting a +# failure that did not happen — and nobody reviews it until it has already +# misled someone. So it runs here against a stub gh. +# +# The step body and its literal env are read out of the workflow, so editing +# the workflow cannot leave this test passing against logic it no longer has. +# +# Run: bash scripts/__tests__/cli-install-smoke-reporter.test.sh + +set -uo pipefail + +ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +WORKFLOW="$ROOT/.github/workflows/cli-install-smoke.yml" +TMP="$(mktemp -d "${TMPDIR:-/tmp}/cli-smoke-reporter.XXXXXX")" +trap 'rm -rf "$TMP"' EXIT INT TERM + +passed=0 +failed=0 +reporter_rc=0 + +assert() { + local name="$1" + shift + if "$@"; then + echo " PASS: $name" + passed=$((passed + 1)) + else + echo " FAIL: $name" + failed=$((failed + 1)) + fi +} + +node -e ' +const {parse} = require("yaml"); +const fs = require("fs"); +const wf = parse(fs.readFileSync(process.argv[1], "utf8")); +const step = wf.jobs.report.steps.find((s) => typeof s.run === "string"); +if (!step) throw new Error("no run step found in the report job"); +fs.writeFileSync(process.argv[2], step.run); +// Values GitHub interpolates at run time are supplied per scenario; the +// literal ones (LABEL) belong to the workflow and are taken from it. +const literal = Object.entries(step.env || {}) + .filter(([, v]) => typeof v === "string" && !v.includes("${{")) + .map(([k, v]) => `export ${k}=${JSON.stringify(v)}`) + .join("\n"); +if (!literal) throw new Error("expected at least one literal env value (LABEL)"); +fs.writeFileSync(process.argv[3], literal + "\n"); +' "$WORKFLOW" "$TMP/reporter.sh" "$TMP/reporter-env.sh" + +[ -s "$TMP/reporter.sh" ] || { echo "FAIL: extracted an empty reporter body"; exit 1; } +# shellcheck source=/dev/null +source "$TMP/reporter-env.sh" + +mkdir -p "$TMP/bin" +cat >"$TMP/bin/gh" <<'GH_EOF' +#!/usr/bin/env bash +# Stub gh: logs the command it was asked to run and answers from FAKE_* env. +printf '%s\n' "$*" >>"$GH_LOG" +case "$1 $2" in + "label list") printf '%s\n' ${FAKE_LABELS:-} ;; + "issue list") printf '%s' "${FAKE_OPEN_ISSUE:-}" ;; +esac +exit 0 +GH_EOF +chmod +x "$TMP/bin/gh" + +run_reporter() { + local result="$1" open_issue="$2" labels="$3" + : >"$TMP/gh.log" + reporter_rc=0 + PATH="$TMP/bin:$PATH" \ + GH_LOG="$TMP/gh.log" \ + FAKE_OPEN_ISSUE="$open_issue" \ + FAKE_LABELS="$labels" \ + GH_TOKEN=stub GH_REPO=owner/repo LABEL="$LABEL" \ + SMOKE_RESULT="$result" \ + RUN_URL="https://example.invalid/run/1" \ + bash "$TMP/reporter.sh" >"$TMP/out.txt" 2>&1 || reporter_rc=$? +} + +# Without this, every "did not do X" assertion also passes when the reporter +# crashed before doing anything — which is how this test first ran. +ran_clean() { + [[ $reporter_rc -eq 0 ]] && return 0 + echo " (reporter exited $reporter_rc: $(head -1 "$TMP/out.txt"))" + return 1 +} + +# Anchored: an issue body mentioning "closes itself" must not read as a call to +# `gh issue close`. Assert on the command invoked, never on prose inside it. +invoked() { grep -q "^$1" "$TMP/gh.log"; } +not_invoked() { ! grep -q "^$1" "$TMP/gh.log"; } + +echo "--- green runs ---" + +run_reporter success "" "$LABEL" +assert "ran cleanly (green, none open)" ran_clean +assert "green with nothing open opens nothing" not_invoked "issue create" +assert "green with nothing open closes nothing" not_invoked "issue close" + +run_reporter success 42 "$LABEL" +assert "ran cleanly (green, one open)" ran_clean +assert "green closes the open failure issue" invoked "issue close 42" +assert "green never opens an issue" not_invoked "issue create" + +echo "--- failing runs ---" + +run_reporter failure "" "$LABEL" +assert "ran cleanly (failing, none open)" ran_clean +assert "first failure opens an issue" invoked "issue create" +assert "first failure closes nothing" not_invoked "issue close" + +run_reporter failure 42 "$LABEL" +assert "ran cleanly (failing, one open)" ran_clean +assert "repeat failure comments instead of duplicating" invoked "issue comment 42" +assert "repeat failure opens no second issue" not_invoked "issue create" + +# A job that never ran reports something other than "success"; anything that is +# not success has to be reported, or a smoke that failed to start reads as green. +run_reporter skipped "" "$LABEL" +assert "ran cleanly (skipped)" ran_clean +assert "a skipped smoke is reported, not read as green" invoked "issue create" + +echo "--- label bootstrap ---" + +run_reporter failure "" "some-other-label" +assert "ran cleanly (label absent)" ran_clean +assert "creates the label when the repo lacks it" invoked "label create" + +run_reporter failure "" "$LABEL some-other-label" +assert "ran cleanly (label present)" ran_clean +assert "does not recreate an existing label" not_invoked "label create" + +echo "" +if [[ $failed -eq 0 ]]; then + echo "ALL PASSED: $passed/$((passed + failed))" +else + echo "FAILED: $failed/$((passed + failed))" + exit 1 +fi diff --git a/scripts/__tests__/cli-install-smoke.test.sh b/scripts/__tests__/cli-install-smoke.test.sh new file mode 100755 index 0000000..85da8a7 --- /dev/null +++ b/scripts/__tests__/cli-install-smoke.test.sh @@ -0,0 +1,393 @@ +#!/usr/bin/env bash +# Guard tests for scripts/cli-install-smoke.sh. +# +# The smoke itself needs the network and an authenticated gh, so it cannot run +# as a unit test. What can be pinned here is the part that decides whether the +# smoke is measuring anything: the version expectation and the PATH sandbox. +# Both are the kind of guard that passes vacuously when it breaks — a missing +# version.json would compare "" to "" and a leftover `am` on PATH would let a +# stale binary answer every probe. +# +# Run: bash scripts/__tests__/cli-install-smoke.test.sh + +set -uo pipefail + +ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +TMP="$(mktemp -d "${TMPDIR:-/tmp}/cli-install-smoke-test.XXXXXX")" +trap 'rm -rf "$TMP"' EXIT INT TERM + +passed=0 +failed=0 + +# `!` is a shell keyword, not a command, so it cannot be passed to assert. +fails() { ! "$@"; } + +assert() { + local name="$1" + shift + if "$@"; then + echo " PASS: $name" + passed=$((passed + 1)) + else + echo " FAIL: $name" + failed=$((failed + 1)) + fi +} + +# shellcheck source=../cli-install-smoke.sh +source "$ROOT/scripts/cli-install-smoke.sh" + +write_version_json() { + printf '%s' "$2" >"$TMP/$1" + printf '%s' "$TMP/$1" +} + +# read_expected_version exits on bad input, so each case runs in a subshell and +# is judged on its exit status and captured output. +version_case() { + local file="$1" + ( read_expected_version "$file" ) 2>&1 +} +version_rc() { + ( read_expected_version "$1" ) >/dev/null 2>&1 +} + +echo "--- channel selection (ATO-1863) ---" + +# The public channel is the one a user actually runs, and it must need no repo +# auth — that is the whole reason it is a separate lane. If `gh` crept back into +# its requirements the lane would fail for a reason no user could ever hit. +# Every input pinned, including the mode. Leaving AM_SMOKE_MODE ambient meant +# `AM_SMOKE_MODE=provenance npm run test:cli-install-smoke` failed three of +# these — the test read the environment it was run in rather than the case it +# names. +gh_case() { + ( AM_SMOKE_MODE=install AM_SMOKE_CHANNEL="$1" AM_VERIFY_ATTESTATION="$2" needs_gh ) +} + +assert "internal needs gh — it downloads through the API" \ + gh_case internal auto +assert "public does not need gh by default" \ + fails gh_case public auto +# These two used to assert the opposite: that demanding attestation made the +# public lane acquire a token. That was the design ATO-1863's review rejected — +# the token was acquired so the *downloaded installer* could verify itself, +# which both hands a credential to unauthenticated code and asks the artifact +# to vouch for itself. Provenance moved to its own mode, so the public install +# path now never needs `gh` whatever attestation is set to. +assert "public still does not need gh when attestation is demanded" \ + fails gh_case public 1 +assert "nor for the spelled-out form" \ + fails gh_case public required + +# An unrecognised channel must stop rather than fall through to whichever branch +# happens to be first. Silently smoking the internal channel while the operator +# believes they tested the public mirror is the failure this whole ticket is +# about: a green that means nothing. +channel_rc() { + ( AM_SMOKE_CHANNEL="$1" WORK="$TMP" download_release_assets ) >/dev/null 2>&1 +} +assert "an unknown channel is refused, not silently treated as internal" \ + fails channel_rc publik + +echo "--- read_expected_version ---" + +good="$(write_version_json good.json '{"version":"0.2.1","channel":"internal"}')" +assert "reads the version out of a well-formed version.json" \ + test "$(version_case "$good")" = "0.2.1" + +empty="$(write_version_json empty.json '')" +assert "rejects an empty version.json instead of expecting the empty string" \ + fails version_rc "$empty" + +blank="$(write_version_json blank.json '{"version":"","channel":"internal"}')" +assert "rejects a blank version" fails version_rc "$blank" + +nonsemver="$(write_version_json tag.json '{"version":"cli-internal-latest"}')" +assert "rejects a non X.Y.Z version" fails version_rc "$nonsemver" + +missing="$(write_version_json other.json '{"channel":"internal"}')" +assert "rejects a version.json with no version key" fails version_rc "$missing" + +echo "--- am_version_output_matches ---" + +assert "accepts machine-readable JSON version output" \ + am_version_output_matches '{"surface":"cli","version":"0.2.1","gitSha":null,"env":"dev"}' 0.2.1 +assert "accepts legacy am X.Y.Z banner during transition" \ + am_version_output_matches 'am 0.2.1' 0.2.1 +assert "rejects JSON with the wrong version" \ + fails am_version_output_matches '{"surface":"cli","version":"0.2.0","gitSha":null,"env":"dev"}' 0.2.1 +assert "rejects a lookalike version that would match an unescaped regex" \ + fails am_version_output_matches '{"surface":"cli","version":"0x2x1"}' 0.2.1 +assert "rejects malformed version JSON" \ + fails am_version_output_matches '{"surface":"cli","version":0.2.1' 0.2.1 +assert "rejects a nested version field without a top-level version" \ + fails am_version_output_matches '{"surface":"cli","nested":{"version":"0.2.1"}}' 0.2.1 +assert "rejects a version value containing an embedded newline" \ + fails am_version_output_matches '{"surface":"cli","version":"0.2.1\nwrong"}' 0.2.1 +assert "rejects a surface value that smuggles the version after a newline" \ + fails am_version_output_matches '{"surface":"cli\n0.2.1","version":"9.9.9"}' 0.2.1 + +echo "--- PATH is left alone ---" + +# An earlier version built a sandbox PATH with every directory holding an `am` +# dropped, to reproduce a machine that had never installed the CLI. It also +# dropped gh, curl or tar whenever the operator had installed `am` into the same +# prefix — the normal case for Homebrew and /usr/local/bin — and the smoke then +# failed inside the downloaded installer with "need GitHub CLI (gh) on PATH". +# CI could not catch it, because runners have no `am` to collide with. +# +# The probes address the binary by absolute path, so they never needed it. +assert "the harness never assigns to PATH" \ + fails grep -qE '^[[:space:]]*(export[[:space:]]+)?PATH=' "$ROOT/scripts/cli-install-smoke.sh" + +# It still has to *read* PATH for the activation check, which is the one probe +# that legitimately depends on it. +assert "it still reads PATH for the activation check" \ + grep -q 'PATH="\$outer_path"' "$ROOT/scripts/cli-install-smoke.sh" + +# Every am invocation is absolute. A bare `am` would be answered by whatever the +# operator has installed, which is the thing the sandbox was there to prevent. +assert "every am probe addresses the install dir directly" \ + fails grep -qE '(assert_ok|quietly)[^\n]*[^/"]\bam\b (--|config)' "$ROOT/scripts/cli-install-smoke.sh" + +echo "--- the public mirror is pinned ---" + +# This script downloads install.sh from AM_PUBLIC_BASE_URL and pipes it to `sh`, +# so a free-text origin is remote code execution on the runner for anyone who +# can dispatch the workflow. Prefix matching is not enough: the first case below +# is a hostname that *starts with* the real one. +# Asserts the *guard* refused, not merely that the run failed. Checking only +# for a non-zero exit passes for the wrong reason: an un-allowlisted host also +# fails later when curl cannot fetch install.sh from it, so a prefix-matching +# guard that wrongly admits `get.atomicstrata.ai.evil.com` would still look +# green here. The message is the only signal that distinguishes them. +refuses_mirror() { + # Captured, not piped. Under `set -o pipefail` a `script | grep -q` pipeline + # returns the script's exit 2 even when grep matches, so the assertion would + # fail on a guard that worked perfectly. + local out + out="$(AM_SMOKE_MODE=install AM_SMOKE_CHANNEL=public AM_PUBLIC_BASE_URL="$1" \ + bash "$ROOT/scripts/cli-install-smoke.sh" 2>&1 || true)" + [[ "$out" == *"Refusing to install from $1"* ]] +} + +for bad in \ + "https://get.atomicstrata.ai.evil.com" \ + "https://get.atomicstrata.ai@evil.com" \ + "http://get.atomicstrata.ai" \ + "https://evil.com"; do + assert "refuses mirror ${bad}" refuses_mirror "$bad" +done + +# The installer reads AM_BASE_URL; this script downloads its inputs from +# AM_PUBLIC_BASE_URL. Without the bridge the public lane verifies the candidate +# version.json and installs the production tarball — green, and meaningless. +assert "exports AM_BASE_URL for the public channel" \ + grep -q 'export AM_BASE_URL="\$AM_PUBLIC_BASE_URL"' "$ROOT/scripts/cli-install-smoke.sh" + +echo "--- the installer never holds a GitHub token ---" + +# The public installer is downloaded from a mirror and executed before anything +# has authenticated it. If it holds a private-repo token, a compromised mirror +# exfiltrates the credential and every later integrity check is too late. +needs_gh_for() { + ( AM_SMOKE_MODE="$1" AM_SMOKE_CHANNEL="$2" + AM_VERIFY_ATTESTATION="${3:-auto}" + needs_gh ) +} + +assert "a public install does not ask for a token" \ + fails needs_gh_for install public +assert "a public install does not ask for one even with attestation demanded" \ + fails needs_gh_for install public 1 +assert "the internal install still may (it downloads through gh)" \ + needs_gh_for install internal +assert "provenance mode does need one" \ + needs_gh_for provenance public + +# Defence in depth for a developer running this locally with `gh` logged in: +# the job boundary is the real control, but the child should not inherit a +# credential that happens to be in the parent's environment. +# +# Asserted on the built argv, not by grepping the source. The grep this +# replaced was satisfied by any line containing the literal — a comment would +# have done — and it went stale silently the moment the invocation was made +# conditional. +install_cmd_for() { + ( AM_SMOKE_CHANNEL="$1" set_install_cmd /release/install.sh + printf '%s' "${INSTALL_CMD[*]}" ) +} + +assert "a public install runs the installer with both token vars stripped" \ + test "$(install_cmd_for public)" = \ + "env -u GH_TOKEN -u GITHUB_TOKEN sh /release/install.sh" + +# The internal installer IS scripts/install-cli-internal.sh, and it reaches the +# private repo with `gh release download`. Stripping the credential there — on +# top of a sandbox HOME that already hides ~/.config/gh — left gh with nothing +# and broke the internal nightly on all four targets. This is that regression +# written down. +assert "an internal install keeps its credential" \ + test "$(install_cmd_for internal)" = "sh /release/install.sh" + +# Delegating attestation to the downloaded installer is what required giving it +# a token in the first place — and an installer verifying its own provenance +# proves nothing if it is the thing that was replaced. Anchored to a real +# assignment so a mention in a comment cannot satisfy it. +assert "attestation is not delegated to the downloaded installer" \ + grep -qE '^[[:space:]]*export AM_VERIFY_ATTESTATION=0[[:space:]]*$' \ + "$ROOT/scripts/cli-install-smoke.sh" + +# A verifier that lets the artifact name its own signer verifies nothing. +assert "the attestation identity is pinned in this file" \ + grep -qE '^[[:space:]]*AM_ATTESTATION_REPO="atomicstrata/atomicmemory"' \ + "$ROOT/scripts/cli-install-smoke.sh" +assert "the signer workflow is pinned too" \ + grep -qE '^[[:space:]]*AM_ATTESTATION_WORKFLOW="atomicstrata/atomicmemory/\.github/workflows/release-cli\.yml"' \ + "$ROOT/scripts/cli-install-smoke.sh" + +# Judged on the guard's own message and exit status, not on "the script failed". +# The previous form ran the whole script and accepted any non-zero exit, so +# deleting the mode guard outright still passed — the run simply died later at +# require_commands, and on a machine with `gh` authenticated it would have +# performed a real internal download first. +# Every input pinned, so an exported variable in the caller's environment +# cannot change which case is under test. +# `${2-internal}` not `${2:-internal}`: the colon form substitutes for an empty +# value too, so the "empty channel" case silently became the "internal" case and +# the assertion tested nothing. +validate_rc() { + ( AM_SMOKE_MODE="$1" AM_SMOKE_CHANNEL="${2-internal}" + AM_PUBLIC_BASE_URL="${3-https://get.atomicstrata.ai}" + validate_config ) >/dev/null 2>&1 +} +validate_msg() { + ( AM_SMOKE_MODE="$1" AM_SMOKE_CHANNEL="${2-internal}" + AM_PUBLIC_BASE_URL="${3-https://get.atomicstrata.ai}" + validate_config ) 2>&1 +} +# validate_config normalises in place, so the normalised value is read back from +# the same subshell. Do NOT re-source the script to test this: under +# `bash -c 'source "$0"' script.sh`, BASH_SOURCE[0] equals $0, so the main block +# runs and the "unit test" performs a real download. +normalised_base() { + ( AM_SMOKE_MODE=install AM_SMOKE_CHANNEL=internal AM_PUBLIC_BASE_URL="$1" + validate_config >/dev/null 2>&1 + printf '%s' "$AM_PUBLIC_BASE_URL" ) +} + +assert "a known mode is accepted" validate_rc install +assert "provenance is a known mode" validate_rc provenance public +assert "an unknown mode is refused" fails validate_rc bogus +assert "and it says which value it rejected" \ + grep -q "got 'bogus'" <<<"$(validate_msg bogus)" + +# needs_gh treated `publik`, `PUBLIC` and "" as "a public install", dropping the +# gh requirement and the AM_BASE_URL bridge, then failing much later for an +# unrelated reason. Enumerated at entry now, like the mode. +assert "an unknown channel is refused at the validator" \ + fails validate_rc install publik +assert "channel matching is exact, not case-insensitive" \ + fails validate_rc install PUBLIC +# An empty value reaching validate_config is refused. (Set in the environment it +# never gets this far — line 62 defaults it to internal before anything reads +# it — but the enumeration must not depend on that happening first.) +assert "an empty channel is refused" fails validate_rc install "" + +# provenance always attests the PUBLIC mirror's tarball, so pairing it with the +# internal channel downloaded internal metadata and then reported a pass for an +# artifact it never named. Internal builds carry no attestations at all. +assert "provenance against the internal channel is refused" \ + fails validate_rc provenance internal + +assert "an unknown mirror is refused by the same validator" \ + fails validate_rc install internal https://evil.example +assert "a trailing slash is normalised, not rejected" \ + test "$(normalised_base https://get.atomicstrata.ai/)" = "https://get.atomicstrata.ai" +assert "the bare form is left alone" \ + test "$(normalised_base https://get.atomicstrata.ai)" = "https://get.atomicstrata.ai" + +echo "--- the two lanes must agree on the release ---" + +# install and provenance run as separate jobs on separate runners, each +# fetching from the mirror. A release landing between them would have one +# installing X while the other attests X+1, both green. The workflow resolves +# the version once; this is the check that makes a disagreement fail. +version_agreement() { + local tmp; tmp="$(mktemp -d)" + printf '{"version":"%s"}' "$1" > "$tmp/version.json" + ( AM_SMOKE_EXPECT_VERSION="$2" read_expected_version "$tmp/version.json" ) >/dev/null 2>&1 + local rc=$? + rm -rf "$tmp" + return $rc +} + +assert "accepts the version the workflow resolved" \ + version_agreement 1.2.3 1.2.3 +assert "accepts any version when nothing was pinned" \ + version_agreement 1.2.3 "" +assert "refuses a release that moved mid-run" \ + fails version_agreement 1.2.4 1.2.3 + +# The first provenance implementation built the tarball URL without the +# `/cli/v` segment and 404'd on every platform. Every test passed, +# because they all scanned source and none fetched anything. This pins the +# layout install-cli.sh actually publishes to. +assert "the provenance URL keeps the cli/v path segment" \ + grep -q 'url="\${AM_PUBLIC_BASE_URL}/cli/v\${version}/\${tarball}"' \ + "$ROOT/scripts/cli-install-smoke.sh" + +echo "--- the reporter grades every lane it depends on ---" + +# ATO-1863 first shipped a reporter that listed `public-provenance` in `needs` +# and graded only `public-install-smoke`: a signature failure would have closed +# the issue and passed the night in silence, on the half of the lane that is the +# reason it exists. Enumerated rather than grepped for the three job names, so a +# lane added later cannot be left ungraded the same way. +reporter_check() { + python3 "$ROOT/scripts/__tests__/reporter-grades-every-lane.py" "$@" +} +# Same check with the diagnostics muted, for the case that is meant to fail — +# so its stderr does not read as a suite failure in the CI log. +reporter_check_quiet() { + reporter_check "$@" 2>/dev/null +} + +assert "every job in a reporter's needs is graded, in both lanes" \ + reporter_check "$ROOT/.github/workflows/cli-public-install-smoke.yml" \ + "$ROOT/.github/workflows/cli-install-smoke.yml" + +# The check above passes on a file it fails to parse, so prove it can still say +# no. Reintroduces the original defect on a copy. +ungraded="$TMP/ungraded-reporter.yml" +sed "/PROVENANCE_RESULT: /d" \ + "$ROOT/.github/workflows/cli-public-install-smoke.yml" >"$ungraded" +assert "an ungraded lane is reported, not passed over" \ + fails reporter_check_quiet "$ungraded" + +# Presence of the variable is not the property — being *required* is. The first +# version of the checker only looked for `"$VAR" = "success"` somewhere in the +# block, so flipping the reporter's `&&` to `||` passed: any one green lane +# closed the issue, and a release with no valid attestation went unreported. +any_lane="$TMP/any-lane-reporter.yml" +sed 's|&& \[ "$INSTALL_RESULT"|\|\| [ "$INSTALL_RESULT"|; s|&& \[ "$PROVENANCE_RESULT"|\|\| [ "$PROVENANCE_RESULT"|' \ + "$ROOT/.github/workflows/cli-public-install-smoke.yml" >"$any_lane" +assert "lanes joined with || are rejected, not just counted" \ + fails reporter_check_quiet "$any_lane" + +# The checker used to return "no problems" whenever its regex missed the job, +# so a workflow with every grading line deleted passed as long as the block was +# unrecognisable. Not finding the reporter must be loud. +no_reporter="$TMP/no-reporter.yml" +sed -n '1,120p' "$ROOT/.github/workflows/cli-public-install-smoke.yml" >"$no_reporter" +assert "a workflow with no reporter fails instead of passing vacuously" \ + fails reporter_check_quiet "$no_reporter" + +echo "" +if [[ $failed -eq 0 ]]; then + echo "ALL PASSED: $passed/$((passed + failed))" +else + echo "FAILED: $failed/$((passed + failed))" + exit 1 +fi diff --git a/scripts/__tests__/install-cli-internal.test.sh b/scripts/__tests__/install-cli-internal.test.sh index 09bed20..9b80404 100755 --- a/scripts/__tests__/install-cli-internal.test.sh +++ b/scripts/__tests__/install-cli-internal.test.sh @@ -40,7 +40,7 @@ create_fake_am() { cat >"$path" <SHA256SUMS fi ) - printf '{"version":"%s","tag":"cli-internal-latest","git_sha":"deadbeef","channel":"internal"}\n' \ + printf '{"surface":"cli","version":"%s","gitSha":"deadbeef","env":"internal","tag":"cli-internal-latest"}\n' \ "$ver" >"${release_dir}/version.json" cp "$ROOT/scripts/install-cli.sh" "${release_dir}/install-cli.sh" cp "$ROOT/scripts/install-cli-internal.sh" "${release_dir}/install.sh" @@ -96,7 +96,7 @@ if [ "\$1" = "release" ] && [ "\$2" = "download" ]; then [ -n "\$dir" ] || exit 1 [ "\$repo" = "atomicstrata/atomicmemory-internal" ] || exit 1 case "\$tag" in - cli-internal-latest|cli-internal-deadbeef) ;; + cli-internal-latest|cli-internal-deadbeef|cli-canary-latest|cli-canary-deadbeef) ;; *) echo "unknown tag \$tag" >&2; exit 1 ;; esac cp "${release_dir}"/* "\$dir/" @@ -158,7 +158,7 @@ main() { else assert "install from floating internal tag" true fi - if [ -x "${bin_dir}/am" ] && [ "$("${bin_dir}/am" --version)" = "am ${ver}" ]; then + if [ -x "${bin_dir}/am" ] && printf \'%s\' "$("${bin_dir}/am" --version)" | grep -q "\"version\":\"${ver}\""; then assert "installed binary reports expected version" true else assert "installed binary reports expected version" false @@ -170,6 +170,21 @@ main() { assert "refuses public cli-v tag" true fi + local canary_bin="${FIXTURE_ROOT}/bin-canary" + mkdir -p "$canary_bin" + if ! AM_INTERNAL_REPO=atomicstrata/atomicmemory-internal \ + AM_INTERNAL_TAG=cli-canary-latest \ + sh "$INSTALLER" --bin-dir "$canary_bin" --no-modify-path; then + assert "install from floating canary tag" false + else + assert "install from floating canary tag" true + fi + if [ -x "${canary_bin}/am" ] && printf \'%s\' "$("${canary_bin}/am" --version)" | grep -q "\"version\":\"${ver}\""; then + assert "canary install reports expected version" true + else + assert "canary install reports expected version" false + fi + # Hostile sibling beside the downloaded wrapper must never win over the # authenticated release asset (trust boundary for private GitHub Releases). local wrap_dir="${FIXTURE_ROOT}/wrap" @@ -198,7 +213,7 @@ HOSTILE else assert "hostile sibling install-cli.sh was not executed" true fi - if [ -x "${wrap_bin}/am" ] && [ "$("${wrap_bin}/am" --version)" = "am ${ver}" ]; then + if [ -x "${wrap_bin}/am" ] && printf \'%s\' "$("${wrap_bin}/am" --version)" | grep -q "\"version\":\"${ver}\""; then assert "install still uses release install-cli.sh asset" true else assert "install still uses release install-cli.sh asset" false diff --git a/scripts/__tests__/install-cli.test.sh b/scripts/__tests__/install-cli.test.sh index 4b71e3a..b44cf5f 100755 --- a/scripts/__tests__/install-cli.test.sh +++ b/scripts/__tests__/install-cli.test.sh @@ -41,7 +41,7 @@ create_fake_am() { cat >"$path" <"${FIXTURE_ROOT}/version.json" + printf '{"surface":"cli","version":"%s","gitSha":"testdeadbeef","env":"production","tag":"cli-v%s"}\n' \ + "$ver" "$ver" >"${FIXTURE_ROOT}/version.json" local port port="$(python3 - <<'PY' import socket @@ -182,13 +183,13 @@ last_line="$(tail -n 1 "$INSTALLER")" printf '\nCase: successful install verifies identity and version\n' if run_install --version 0.2.0 --bin-dir "$BIN_DIR" --no-modify-path >/dev/null; then got="$("$BIN_DIR/am" --version)" - if [ "$got" = "am 0.2.0" ]; then - assert "install succeeds and binary reports am 0.2.0" true + if printf '%s' "$got" | grep -q '"version"[[:space:]]*:[[:space:]]*"0.2.0"'; then + assert "install succeeds and binary reports version 0.2.0" true else - assert "install succeeds and binary reports am 0.2.0" false + assert "install succeeds and binary reports version 0.2.0" false fi else - assert "install succeeds and binary reports am 0.2.0" false + assert "install succeeds and binary reports version 0.2.0" false fi printf '\nCase: invalid version is rejected\n' @@ -269,7 +270,8 @@ if run_install --version 0.2.0 --bin-dir "$upgrade_dir" --no-modify-path >/dev/n esac if [ -x "$upgrade_dir/am" ]; then got="$("$upgrade_dir/am" --version)" - [ "$got" = "am 0.2.0" ] && assert "failed upgrade preserves working am" true \ + printf '%s' "$got" | grep -q '"version"[[:space:]]*:[[:space:]]*"0.2.0"' \ + && assert "failed upgrade preserves working am" true \ || assert "failed upgrade preserves working am" false else assert "failed upgrade preserves working am" false diff --git a/scripts/__tests__/reconcile-internal-release.test.sh b/scripts/__tests__/reconcile-internal-release.test.sh index 9bc0a07..bcdb0b6 100755 --- a/scripts/__tests__/reconcile-internal-release.test.sh +++ b/scripts/__tests__/reconcile-internal-release.test.sh @@ -69,7 +69,7 @@ seed_release() { sha256sums_for_dir "$dir" >"${dir}/SHA256SUMS" printf 'install-cli\n' >"${dir}/install-cli.sh" printf 'install\n' >"${dir}/install.sh" - printf '{"version":"%s","tag":"cli-internal-abc","git_sha":"abc","channel":"internal"}\n' \ + printf '{"surface":"cli","version":"%s","gitSha":"abc","env":"internal","tag":"cli-internal-abc"}\n' \ "$ver" >"${dir}/version.json" } @@ -88,7 +88,7 @@ seed_local_dist_same_names() { sha256sums_for_dir "$dir" >"${dir}/SHA256SUMS" printf 'install-cli\n' >"${dir}/install-cli.sh" printf 'install\n' >"${dir}/install.sh" - printf '{"version":"%s","tag":"cli-internal-abc","git_sha":"abc","channel":"internal"}\n' \ + printf '{"surface":"cli","version":"%s","gitSha":"abc","env":"internal","tag":"cli-internal-abc"}\n' \ "$ver" >"${dir}/version.json" } diff --git a/scripts/__tests__/reporter-grades-every-lane.py b/scripts/__tests__/reporter-grades-every-lane.py new file mode 100755 index 0000000..dbc9609 --- /dev/null +++ b/scripts/__tests__/reporter-grades-every-lane.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python3 +"""Assert a nightly workflow's reporter grades every job it depends on. + +Both CLI install-smoke lanes end in a `report` job that opens or closes a +GitHub issue. Its verdict is only as good as the set of upstream results it +reads: ATO-1863 shipped a reporter that listed `public-provenance` in `needs` +but graded only `public-install-smoke`, so a provenance failure — the half of +the lane that checks signatures — would have passed the night in silence. + +Grepping for three literal job names would not have caught that, and would not +catch the next job someone adds. This enumerates `needs` and requires each +entry to be (a) covered by the cancelled guard, (b) bound to an env var, and +(c) compared against "success". A new lane that is not graded fails here. + +Regex rather than a YAML parser so the test suite keeps its only dependencies +as bash and python3; the shapes it matches are pinned by the tests below it. + +Usage: + python3 scripts/__tests__/reporter-grades-every-lane.py ... +Exits 0 when every reporter grades every lane, 1 otherwise. +""" + +import re +import sys + + +def report_job(text: str) -> str: + """Return the reporter job block, or "" when none is found. + + Matches any job whose body opens the failure issue, rather than the literal + name `report:` — renaming the job used to make this return "" and, with the + old contract, silently pass a workflow with every grading line deleted. + """ + for match in re.finditer(r"^ [A-Za-z][\w-]*:\n(?:(?: .*)?\n)*", text, re.MULTILINE): + if "needs." in match.group(0) and "issue" in match.group(0): + return match.group(0) + return "" + + +def declared_needs(block: str) -> list[str]: + """Job names from `needs: a`, `needs: [a, b]`, or a `- a` block list.""" + inline = re.search(r"^ needs:\s*\[([^\]]*)\]", block, re.MULTILINE) + if inline: + return [name.strip() for name in inline.group(1).split(",") if name.strip()] + scalar = re.search(r"^ needs:\s*([A-Za-z][\w-]*)\s*$", block, re.MULTILINE) + if scalar: + return [scalar.group(1)] + listed = re.search(r"^ needs:\s*\n((?:\s+-\s*[\w-]+\n)+)", block, re.MULTILINE) + if listed: + return re.findall(r"-\s*([\w-]+)", listed.group(1)) + return [] + + +def success_condition(block: str) -> str: + """The `if …; then` that decides the green path, line continuations joined. + + Selected by content, not position: the reporter's first `if` is the label + bootstrap, so taking the first match graded the wrong statement. + """ + joined = re.sub(r"\\\n\s*", " ", block) + for match in re.finditer(r"^\s*if\s+(.*?);\s*then\s*$", joined, re.MULTILINE): + if '= "success"' in match.group(1): + return match.group(1) + return "" + + +def check(path: str) -> list[str]: + text = open(path, encoding="utf-8").read().replace("\r\n", "\n") + if not text.endswith("\n"): + text += "\n" + block = report_job(text) + if not block: + # Failing to locate the reporter is a failure, not a pass. This function + # used to return [] here, so a renamed job disabled the guard silently. + return [f"{path}: no reporter job found — cannot verify anything"] + + needs = declared_needs(block) + if not needs: + return [f"{path}: report job declares no `needs` — nothing is graded"] + + # VAR: ${{ needs..result }} + bound = dict( + (job, var) + for var, job in re.findall( + r"^\s+([A-Z][A-Z0-9_]*):\s*\$\{\{\s*needs\.([\w-]+)\.result\s*\}\}", + block, + re.MULTILINE, + ) + ) + + problems = [] + for job in needs: + if f"needs.{job}.result != 'cancelled'" not in block: + problems.append(f"{path}: `{job}` is in needs but not in the cancelled guard") + if job not in bound: + problems.append(f"{path}: `{job}` is in needs but bound to no env var") + continue + if f'"${bound[job]}" = "success"' not in block: + problems.append( + f"{path}: `{job}` -> ${bound[job]} is never compared against \"success\"" + ) + + for job in bound: + if job not in needs: + problems.append(f"{path}: grades `{job}`, which is not in needs") + + # Presence is not enough: `A || B || C` mentions every variable and is + # green whenever any single lane passed — the exact silent-green this file + # exists to prevent, and the mutation the first version of it missed. + condition = success_condition(block) + if not condition: + problems.append(f"{path}: no `if …; then` success condition found") + else: + for operator in ("||", " -o "): + if operator in condition: + problems.append( + f"{path}: success condition joins lanes with `{operator.strip()}`" + " — every lane must be required" + ) + for job, var in bound.items(): + if f'"${var}"' not in condition: + problems.append( + f"{path}: `{job}` -> ${var} is not part of the success condition" + ) + return problems + + +def main(paths: list[str]) -> int: + if not paths: + print("usage: reporter-grades-every-lane.py ...", file=sys.stderr) + return 2 + problems = [problem for path in paths for problem in check(path)] + for problem in problems: + print(f" {problem}", file=sys.stderr) + return 1 if problems else 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/scripts/__tests__/validate-cli-version-bump.test.sh b/scripts/__tests__/validate-cli-version-bump.test.sh new file mode 100755 index 0000000..6cfed12 --- /dev/null +++ b/scripts/__tests__/validate-cli-version-bump.test.sh @@ -0,0 +1,283 @@ +#!/usr/bin/env bash +# +# Contract tests for scripts/ci/validate-cli-version-bump.sh and release-cli wiring. + +set -uo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +VALIDATOR="$ROOT/scripts/ci/validate-cli-version-bump.sh" +BUMPER="$ROOT/scripts/ci/bump-cli-workspace-version.sh" +WORKFLOW="$ROOT/.github/workflows/release-cli.yml" +CI_RUST="$ROOT/.github/workflows/ci-rust.yml" + +PASS_COUNT=0 +FAIL_COUNT=0 +TMP_DIR="" + +cleanup() { + if [ -n "$TMP_DIR" ] && [ -d "$TMP_DIR" ]; then + rm -rf "$TMP_DIR" + fi +} +trap cleanup EXIT + +assert() { + local name="$1" + local condition="$2" + if [ "$condition" = "true" ]; then + printf ' ✓ %s\n' "$name" + PASS_COUNT=$((PASS_COUNT + 1)) + else + printf ' ✗ %s\n' "$name" >&2 + FAIL_COUNT=$((FAIL_COUNT + 1)) + fi +} + +write_cargo_toml() { + local version="$1" + local pin_drift="${2:-}" + local core_pin="$version" + if [ "$pin_drift" = "drift" ]; then + core_pin="9.9.9" + fi + cat >"$TMP_DIR/Cargo.toml" <"$TMP_DIR/Cargo.lock" </dev/null 2>&1 + local status=$? + set -e + if [ "$status" -eq "$expected_exit" ]; then + return 0 + fi + return 1 +} + +printf '\nvalidate-cli-version-bump contract tests\n' + +TMP_DIR="$(mktemp -d)" + +printf '\nCase: adjacent bumps from 0.2.0\n' +write_cargo_toml "0.2.1" +run_validator 0 CARGO_TOML="$TMP_DIR/Cargo.toml" LAST_PUBLIC_VERSION=0.2.0 \ + && assert "patch 0.2.0 -> 0.2.1 passes" true || assert "patch 0.2.0 -> 0.2.1 passes" false + +write_cargo_toml "0.3.0" +run_validator 0 CARGO_TOML="$TMP_DIR/Cargo.toml" LAST_PUBLIC_VERSION=0.2.0 \ + && assert "minor 0.2.0 -> 0.3.0 passes" true || assert "minor 0.2.0 -> 0.3.0 passes" false + +write_cargo_toml "1.0.0" +run_validator 0 CARGO_TOML="$TMP_DIR/Cargo.toml" LAST_PUBLIC_VERSION=0.2.0 \ + && assert "major 0.2.0 -> 1.0.0 passes" true || assert "major 0.2.0 -> 1.0.0 passes" false + +printf '\nCase: invalid bumps\n' +write_cargo_toml "0.2.5" +run_validator 1 CARGO_TOML="$TMP_DIR/Cargo.toml" LAST_PUBLIC_VERSION=0.2.0 \ + && assert "rejects jump 0.2.0 -> 0.2.5" true || assert "rejects jump 0.2.0 -> 0.2.5" false + +write_cargo_toml "0.2.0" +run_validator 1 CARGO_TOML="$TMP_DIR/Cargo.toml" LAST_PUBLIC_VERSION=0.2.1 \ + && assert "rejects downgrade 0.2.1 -> 0.2.0" true || assert "rejects downgrade 0.2.1 -> 0.2.0" false + +write_cargo_toml "0.2.0" +run_validator 1 CARGO_TOML="$TMP_DIR/Cargo.toml" LAST_PUBLIC_VERSION=0.2.0 RELEASE_MODE=1 \ + && assert "release mode rejects unchanged 0.2.0" true || assert "release mode rejects unchanged 0.2.0" false + +write_cargo_toml "0.2.0" +run_validator 0 CARGO_TOML="$TMP_DIR/Cargo.toml" LAST_PUBLIC_VERSION=0.2.0 \ + && assert "PR mode allows unchanged 0.2.0" true || assert "PR mode allows unchanged 0.2.0" false + +printf '\nCase: workspace pin lockstep\n' +write_cargo_toml "0.2.1" "drift" +run_validator 1 CARGO_TOML="$TMP_DIR/Cargo.toml" LAST_PUBLIC_VERSION=0.2.0 \ + && assert "rejects pin drift" true || assert "rejects pin drift" false + +printf '\nCase: Cargo.lock lockstep\n' +write_cargo_toml "0.2.1" +write_cargo_lock "0.2.1" +run_validator 0 CARGO_TOML="$TMP_DIR/Cargo.toml" LAST_PUBLIC_VERSION=0.2.0 \ + && assert "aligned lockfile passes" true || assert "aligned lockfile passes" false + +write_cargo_toml "0.2.1" +write_cargo_lock "0.2.0" +run_validator 1 CARGO_TOML="$TMP_DIR/Cargo.toml" LAST_PUBLIC_VERSION=0.2.0 \ + && assert "rejects stale Cargo.lock" true || assert "rejects stale Cargo.lock" false + +rm -f "$TMP_DIR/Cargo.lock" +write_cargo_toml "0.2.1" +run_validator 1 CARGO_TOML="$TMP_DIR/Cargo.toml" CARGO_LOCK="$TMP_DIR/missing.lock" LAST_PUBLIC_VERSION=0.2.0 \ + && assert "rejects missing explicit Cargo.lock" true || assert "rejects missing explicit Cargo.lock" false + +printf '\nCase: bump helper rewrites pins\n' +rm -f "$TMP_DIR/Cargo.lock" +write_cargo_toml "0.2.0" +SKIP_LOCKFILE=1 SKIP_VALIDATE=1 CARGO_TOML="$TMP_DIR/Cargo.toml" bash "$BUMPER" 0.2.2 +grep -q 'version = "0.2.2"' "$TMP_DIR/Cargo.toml" \ + && grep -q 'am-core-types = { path = "crates/core-types", version = "0.2.2" }' "$TMP_DIR/Cargo.toml" \ + && grep -q 'atomicmemory = { path = "crates/cli", version = "0.2.2" }' "$TMP_DIR/Cargo.toml" \ + && assert "bumper updates workspace version and pins" true \ + || assert "bumper updates workspace version and pins" false + +printf '\nCase: first public release\n' +rm -f "$TMP_DIR/Cargo.lock" +write_cargo_toml "0.2.0" +run_validator 0 CARGO_TOML="$TMP_DIR/Cargo.toml" LAST_PUBLIC_VERSION="" \ + && assert "first release allows any X.Y.Z" true || assert "first release allows any X.Y.Z" false + +printf '\nCase: triggering tag present on remote (release-cli state)\n' +PUBLIC_TAG_LINES=$'abc123\trefs/tags/cli-v0.2.1\nabc124\trefs/tags/cli-v0.2.2' +write_cargo_toml "0.2.2" +write_cargo_lock "0.2.2" +run_validator 0 CARGO_TOML="$TMP_DIR/Cargo.toml" CARGO_LOCK="$TMP_DIR/Cargo.lock" \ + PUBLIC_CLI_TAGS="$PUBLIC_TAG_LINES" PROPOSED_VERSION=0.2.2 RELEASE_MODE=1 \ + && assert "release mode accepts adjacent bump when triggering tag is on remote" true \ + || assert "release mode accepts adjacent bump when triggering tag is on remote" false + +write_cargo_toml "0.2.1" +write_cargo_lock "0.2.1" +run_validator 1 CARGO_TOML="$TMP_DIR/Cargo.toml" CARGO_LOCK="$TMP_DIR/Cargo.lock" \ + PUBLIC_CLI_TAGS="$PUBLIC_TAG_LINES" PROPOSED_VERSION=0.2.1 RELEASE_MODE=1 \ + && assert "release mode rejects re-tagging an already published version" true \ + || assert "release mode rejects re-tagging an already published version" false + +write_cargo_toml "0.2.0" +write_cargo_lock "0.2.0" +run_validator 0 CARGO_TOML="$TMP_DIR/Cargo.toml" CARGO_LOCK="$TMP_DIR/Cargo.lock" \ + PUBLIC_CLI_TAGS=$'abc123\trefs/tags/cli-v0.2.0' PROPOSED_VERSION=0.2.0 \ + && assert "PR mode allows unchanged version when only that tag exists" true \ + || assert "PR mode allows unchanged version when only that tag exists" false + +write_cargo_toml "0.2.0" +write_cargo_lock "0.2.0" +run_validator 0 CARGO_TOML="$TMP_DIR/Cargo.toml" CARGO_LOCK="$TMP_DIR/Cargo.lock" \ + PUBLIC_CLI_TAGS=$'abc123\trefs/tags/cli-v0.2.0' PUBLIC_CLI_RELEASES="" \ + PROPOSED_VERSION=0.2.0 RELEASE_MODE=1 \ + && assert "release mode allows first tag when no GitHub Release exists yet" true \ + || assert "release mode allows first tag when no GitHub Release exists yet" false + +write_cargo_toml "0.2.2" +write_cargo_lock "0.2.2" +run_validator 1 CARGO_TOML="$TMP_DIR/Cargo.toml" CARGO_LOCK="$TMP_DIR/Cargo.lock" \ + PUBLIC_CLI_TAGS="$PUBLIC_TAG_LINES" PUBLIC_CLI_RELEASES=$'0.2.1\n0.2.2' \ + PROPOSED_VERSION=0.2.2 RELEASE_MODE=1 \ + && assert "release mode rejects reusing an already-published latest version" true \ + || assert "release mode rejects reusing an already-published latest version" false + +write_cargo_toml "0.2.2" +write_cargo_lock "0.2.2" +run_validator 0 CARGO_TOML="$TMP_DIR/Cargo.toml" CARGO_LOCK="$TMP_DIR/Cargo.lock" \ + PUBLIC_CLI_TAGS="$PUBLIC_TAG_LINES" PUBLIC_CLI_RELEASES=$'0.2.1\n0.2.2' \ + PROPOSED_VERSION=0.2.2 \ + && assert "PR mode allows unchanged latest published version when an older tag exists" true \ + || assert "PR mode allows unchanged latest published version when an older tag exists" false + +printf '\nCase: live gh release lookup (no PUBLIC_CLI_TAGS fixture)\n' +GH_STUB_DIR="$TMP_DIR/gh-stub" +mkdir -p "$GH_STUB_DIR" +write_gh_stub() { + local exit_code="$1" + local stderr_msg="$2" + cat >"$GH_STUB_DIR/gh" <&2 +exit ${exit_code} +EOF + chmod +x "$GH_STUB_DIR/gh" +} + +write_cargo_toml "0.2.2" +write_cargo_lock "0.2.2" +write_gh_stub 4 "gh: authentication required" +run_validator 1 CARGO_TOML="$TMP_DIR/Cargo.toml" CARGO_LOCK="$TMP_DIR/Cargo.lock" \ + PATH="$GH_STUB_DIR:$PATH" LAST_PUBLIC_VERSION=0.2.1 \ + PROPOSED_VERSION=0.2.2 RELEASE_MODE=1 \ + && assert "release mode fails closed when gh release lookup is unauthenticated" true \ + || assert "release mode fails closed when gh release lookup is unauthenticated" false + +write_gh_stub 1 "release not found" +run_validator 0 CARGO_TOML="$TMP_DIR/Cargo.toml" CARGO_LOCK="$TMP_DIR/Cargo.lock" \ + PATH="$GH_STUB_DIR:$PATH" LAST_PUBLIC_VERSION=0.2.1 \ + PROPOSED_VERSION=0.2.2 RELEASE_MODE=1 \ + && assert "release mode allows adjacent bump when gh confirms release is absent" true \ + || assert "release mode allows adjacent bump when gh confirms release is absent" false + +write_gh_stub 0 "" +run_validator 1 CARGO_TOML="$TMP_DIR/Cargo.toml" CARGO_LOCK="$TMP_DIR/Cargo.lock" \ + PATH="$GH_STUB_DIR:$PATH" LAST_PUBLIC_VERSION=0.2.1 \ + PROPOSED_VERSION=0.2.2 RELEASE_MODE=1 \ + && assert "release mode rejects when gh confirms the latest release already exists" true \ + || assert "release mode rejects when gh confirms the latest release already exists" false + +printf '\nCase: bump helper accepts pnpm forwarded -- separator\n' +rm -f "$TMP_DIR/Cargo.lock" +write_cargo_toml "0.2.0" +SKIP_LOCKFILE=1 SKIP_VALIDATE=1 CARGO_TOML="$TMP_DIR/Cargo.toml" pnpm run bump:cli-version -- 0.2.2 >/dev/null 2>&1 \ + && grep -q 'version = "0.2.2"' "$TMP_DIR/Cargo.toml" \ + && assert "pnpm bump:cli-version -- 0.2.2 rewrites workspace version" true \ + || assert "pnpm bump:cli-version -- 0.2.2 rewrites workspace version" false + +printf '\nCase: workflow wiring\n' +grep -q 'Validate public version bump' "$WORKFLOW" \ + && assert "release-cli invokes bump validator" true \ + || assert "release-cli invokes bump validator" false +grep -q 'validate-cli-version-bump.sh' "$WORKFLOW" \ + && assert "release-cli references validator script" true \ + || assert "release-cli references validator script" false +grep -q 'RELEASE_MODE=1' "$WORKFLOW" \ + && assert "release-cli sets RELEASE_MODE" true \ + || assert "release-cli sets RELEASE_MODE" false +grep -q 'GH_TOKEN:' "$WORKFLOW" \ + && assert "release-cli supplies GH_TOKEN for release lookup" true \ + || assert "release-cli supplies GH_TOKEN for release lookup" false +grep -q 'Validate CLI version bump policy' "$CI_RUST" \ + && assert "ci-rust invokes bump validator on PRs" true \ + || assert "ci-rust invokes bump validator on PRs" false +grep -q 'bump:cli-version' "$ROOT/package.json" \ + && assert "package.json exposes bump:cli-version" true \ + || assert "package.json exposes bump:cli-version" false + +printf '\nResults: %s passed, %s failed\n' "$PASS_COUNT" "$FAIL_COUNT" +if [ "$FAIL_COUNT" -ne 0 ]; then + exit 1 +fi diff --git a/scripts/ci/__tests__/public-release-inputs.test.mjs b/scripts/ci/__tests__/public-release-inputs.test.mjs new file mode 100644 index 0000000..055f308 --- /dev/null +++ b/scripts/ci/__tests__/public-release-inputs.test.mjs @@ -0,0 +1,59 @@ +/** Verify mirrored workflows consume private inputs without private source files. */ +import assert from 'node:assert/strict'; +import { mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { spawnSync } from 'node:child_process'; +import { test } from 'node:test'; +import { parse } from 'yaml'; + +const workflow = (name) => parse(readFileSync(new URL(`../../../.github/workflows/${name}.yml`, import.meta.url), 'utf8')); +const secretExpression = '${{ secrets.PUBLIC_ARTIFACT_SIGNATURES }}'; +const packages = workflow('publish-packages'); +const docker = workflow('publish-core-docker'); + +test('hosted tests run only in the internal repository', () => { + const step = workflow('ci').jobs['affected-build-test'].steps.find((s) => s.run?.includes('hosted/__tests__/run.sh')); + assert.equal(step.if, "github.repository == 'atomicstrata/atomicmemory-internal'"); +}); + +test('reusable Docker publisher receives the required signature secret', () => { + assert.equal(docker.on.workflow_call.secrets.PUBLIC_ARTIFACT_SIGNATURES.required, true); + assert.equal(packages.jobs['publish-core-docker'].secrets.PUBLIC_ARTIFACT_SIGNATURES, secretExpression); +}); + +// Run the actual workflow shell blocks in a source fixture with no hosted/ tree. +// Stubs replace only network/build operations; the signature handoff is real. +for (const [name, steps, prefix] of [ + ['packages', packages.jobs.preflight.steps, ''], + ['docker', docker.jobs.publish.steps, 'release-source/'], +]) { + const step = steps.find((s) => s.name.includes('private identifiers')); + test(`${name}: secret reaches scanner, missing secret refuses release`, () => { + assert.equal(step.env.PUBLIC_ARTIFACT_SIGNATURES, secretExpression); + const dir = mkdtempSync(join(tmpdir(), 'am-release-input-')); + try { + const scripts = join(dir, prefix, 'packages/core/scripts'); + mkdirSync(scripts, { recursive: true }); + writeFileSync(join(scripts, 'check-public-artifacts.sh'), '#!/bin/bash\nset -eu\ncmp "$3" "$EXPECTED_SIGNATURES"\nprintf reached > "$RUNNER_TEMP/scanned"\n', { mode: 0o755 }); + const expected = join(dir, 'expected'); + writeFileSync(expected, 'synthetic-private-marker\n'); + const stubs = 'npm() { touch "$RUNNER_TEMP/atomicmemory-core-package/test.tgz"; }; docker() { :; }; curl() { :; };\n'; + const env = { ...process.env, RUNNER_TEMP: dir, PACKAGE_TARBALL: 'fixture', IMAGE_REFERENCE: 'fixture', EXPECTED_SIGNATURES: expected }; + const run = (value) => spawnSync('bash', ['-c', stubs + step.run], { cwd: dir, env: { ...env, PUBLIC_ARTIFACT_SIGNATURES: value }, encoding: 'utf8' }); + for (const value of ['synthetic-private-marker', '\nsynthetic-private-marker\n \n']) { + const success = run(value); + assert.equal(success.status, 0, success.stderr); + assert.equal(readFileSync(join(dir, 'scanned'), 'utf8'), 'reached'); + rmSync(join(dir, 'scanned')); + } + for (const value of ['', ' \n ']) { + const failure = run(value); + assert.equal(failure.status, 64, failure.stderr); + assert.match(failure.stdout, /must be configured/); + } + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +} diff --git a/scripts/ci/__tests__/release-policy.test.mjs b/scripts/ci/__tests__/release-policy.test.mjs index 46475f1..97cacf3 100644 --- a/scripts/ci/__tests__/release-policy.test.mjs +++ b/scripts/ci/__tests__/release-policy.test.mjs @@ -92,7 +92,7 @@ test("workflow without image pushes is not held to image-publisher policy", () = test("non-enumerated workflow pushing images is rejected", () => { const failures = checkImagePublisherText("jobs:\n x:\n steps:\n - run: docker buildx build --push .\n", ".github/workflows/nightly.yml"); - assert.ok(failures.some((failure) => /neither a publish-\*\.yml release lane nor the enumerated internal image publisher/.test(failure))); + assert.ok(failures.some((failure) => /neither a publish-\*\.yml release lane nor an enumerated operator image publisher/.test(failure))); }); test("internal image publisher without the repository guard is rejected", () => { @@ -144,6 +144,54 @@ test("valid internal image publisher fixture passes", () => { assert.deepEqual(checkImagePublisherText(VALID_INTERNAL_IMAGE_YAML, ".github/workflows/internal-core-docker-image.yml"), []); }); +const VALID_ECR_IMAGE_YAML = [ + "env:", + " IMAGE_NAME: 636941960505.dkr.ecr.us-east-1.amazonaws.com/atomicmemory-core-enterprise", + "jobs:", + " publish:", + " if: github.repository == 'atomicstrata/atomicmemory-internal'", + " steps:", + ' - run: docker buildx build --tag "${IMAGE_NAME}:sha-abc1234" --push .', + "", +].join("\n"); + +test("valid ECR Dev/Staging image publisher fixture passes", () => { + assert.deepEqual(checkImagePublisherText(VALID_ECR_IMAGE_YAML, ".github/workflows/core-ecr-dev-staging.yml"), []); +}); + +test("ECR image publisher pinned to the wrong image name is rejected", () => { + const yaml = VALID_ECR_IMAGE_YAML.replace( + "636941960505.dkr.ecr.us-east-1.amazonaws.com/atomicmemory-core-enterprise", + "ghcr.io/atomicstrata/atomicmemory-core-internal", + ); + const failures = checkImagePublisherText(yaml, ".github/workflows/core-ecr-dev-staging.yml"); + assert.ok(failures.some((failure) => /assign env IMAGE_NAME exactly once/.test(failure))); +}); + +test("ECR image publisher without the repository guard is rejected", () => { + const yaml = VALID_ECR_IMAGE_YAML.replace(/^.*github\.repository.*\n/m, ""); + const failures = checkImagePublisherText(yaml, ".github/workflows/core-ecr-dev-staging.yml"); + assert.ok(failures.some((failure) => /repository guard|github\.repository/.test(failure))); +}); + +test("ECR image publisher using a non-allowlisted action is rejected", () => { + const yaml = VALID_ECR_IMAGE_YAML + " - uses: actions/setup-node@v4\n"; + const failures = checkImagePublisherText(yaml, ".github/workflows/core-ecr-dev-staging.yml"); + assert.ok(failures.some((failure) => /action allowlist/.test(failure))); +}); + +test("ECR image publisher may use aws-actions OIDC and ECR login", () => { + const yaml = VALID_ECR_IMAGE_YAML.replace( + " steps:", + [ + " steps:", + " - uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a", + " - uses: aws-actions/amazon-ecr-login@03f1aad4c6c7ffd436567f42f9384779290529bd", + ].join("\n"), + ); + assert.deepEqual(checkImagePublisherText(yaml, ".github/workflows/core-ecr-dev-staging.yml"), []); +}); + test("workflow pushing via --output=type=registry is treated as a publisher", () => { const failures = checkImagePublisherText("jobs:\n x:\n steps:\n - run: docker buildx build --output=type=registry,name=ghcr.io/x/y .\n", ".github/workflows/nightly.yml"); assert.ok(failures.some((failure) => /neither a publish-\*\.yml release lane/.test(failure))); diff --git a/scripts/ci/__tests__/roll-core-ecs-image.test.mjs b/scripts/ci/__tests__/roll-core-ecs-image.test.mjs new file mode 100644 index 0000000..c9fc282 --- /dev/null +++ b/scripts/ci/__tests__/roll-core-ecs-image.test.mjs @@ -0,0 +1,107 @@ +/** + * Coverage for scripts/ci/rewrite-ecs-task-def-image.py. + */ + +import assert from "node:assert/strict"; +import { mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { spawnSync } from "node:child_process"; +import { test } from "node:test"; + +const SCRIPT = "scripts/ci/rewrite-ecs-task-def-image.py"; +const REPO = "636941960505.dkr.ecr.us-east-1.amazonaws.com/atomicmemory-core-enterprise"; +const NEW_URI = `${REPO}:sha-abcdef0`; + +/** Top-level RegisterTaskDefinitionRequest members (botocore ecs 2014-11-13). */ +const REGISTER_TASK_DEFINITION_KEYS = new Set([ + "containerDefinitions", + "cpu", + "enableFaultInjection", + "ephemeralStorage", + "executionRoleArn", + "family", + "inferenceAccelerators", + "ipcMode", + "memory", + "networkMode", + "pidMode", + "placementConstraints", + "proxyConfiguration", + "requiresCompatibilities", + "runtimePlatform", + "tags", + "taskRoleArn", + "volumes", +]); + +/** Response-only TaskDefinition members that must never reach register. */ +const DESCRIBE_RESPONSE_ONLY_KEYS = [ + "compatibilities", + "deleteRequestedAt", + "deregisteredAt", + "registeredAt", + "registeredBy", + "requiresAttributes", + "revision", + "status", + "taskDefinitionArn", +]; + +function runRewrite(taskDef) { + const dir = mkdtempSync(join(tmpdir(), "roll-core-ecs-")); + const src = join(dir, "in.json"); + const dst = join(dir, "out.json"); + writeFileSync(src, JSON.stringify(taskDef)); + const result = spawnSync("python3", [SCRIPT, NEW_URI, REPO, src, dst], { encoding: "utf8" }); + return { result, dst }; +} + +test("rewrites matching enterprise core image and keeps only register keys", () => { + const responseOnly = Object.fromEntries( + DESCRIBE_RESPONSE_ONLY_KEYS.map((key) => [key, key === "revision" ? 9 : `value-${key}`]), + ); + const { result, dst } = runRewrite({ + ...responseOnly, + family: "atomicmemory-core", + taskRoleArn: "arn:aws:iam::123:role/task", + executionRoleArn: "arn:aws:iam::123:role/exec", + networkMode: "awsvpc", + requiresCompatibilities: ["FARGATE"], + cpu: "1024", + memory: "2048", + runtimePlatform: { cpuArchitecture: "X86_64", operatingSystemFamily: "LINUX" }, + containerDefinitions: [ + { name: "sidecar", image: "public.ecr.aws/x/y:1" }, + { name: "atomicmemory-core", image: `${REPO}:c7b25cf` }, + ], + volumes: [], + // Future AWS response junk must also be dropped by the allowlist. + unexpectedAwsField: "must-not-survive", + }); + assert.equal(result.status, 0, result.stderr); + const out = JSON.parse(readFileSync(dst, "utf8")); + assert.equal(out.containerDefinitions[1].image, NEW_URI); + assert.equal(out.containerDefinitions[0].image, "public.ecr.aws/x/y:1"); + assert.equal(out.family, "atomicmemory-core"); + assert.equal(out.cpu, "1024"); + assert.equal(out.unexpectedAwsField, undefined); + for (const key of DESCRIBE_RESPONSE_ONLY_KEYS) { + assert.equal(out[key], undefined, `response-only key ${key} must be stripped`); + } + for (const key of Object.keys(out)) { + assert.ok( + REGISTER_TASK_DEFINITION_KEYS.has(key), + `output key ${key} is not a RegisterTaskDefinitionRequest member`, + ); + } +}); + +test("fails closed when no container matches the enterprise repo", () => { + const { result } = runRewrite({ + family: "atomicmemory-core", + containerDefinitions: [{ name: "api", image: "ghcr.io/atomicstrata/atomicmemory-core:1.2.1" }], + }); + assert.notEqual(result.status, 0); + assert.match(`${result.stderr}${result.stdout}`, /No container image matching/); +}); diff --git a/scripts/ci/__tests__/roll-core-ecs-order.test.mjs b/scripts/ci/__tests__/roll-core-ecs-order.test.mjs new file mode 100644 index 0000000..84bd6ae --- /dev/null +++ b/scripts/ci/__tests__/roll-core-ecs-order.test.mjs @@ -0,0 +1,49 @@ +/** Exercise deployment failure paths without contacting AWS. */ +import assert from 'node:assert/strict'; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { spawnSync } from 'node:child_process'; +import { test } from 'node:test'; + +function roll(t, failAt) { + const dir = mkdtempSync(join(tmpdir(), 'am-ecs-order-')); + t.after(() => rmSync(dir, { recursive: true, force: true })); + const log = join(dir, 'calls'); + writeFileSync(join(dir, 'aws'), `#!/usr/bin/env node +const fs = require('node:fs'); +const operation = process.argv.slice(2, 4).join(' '); +fs.appendFileSync(process.env.AM_TEST_LOG, operation + '\\n'); +if (operation === process.env.AM_TEST_FAIL) process.exit(1); +if (operation === 'ecs describe-services') console.log('old-task'); +if (operation === 'ecs describe-task-definition') { + console.log(JSON.stringify({family:'test',containerDefinitions:[{name:'core',image:'example/core:old'}]})); +} +if (operation === 'ecs register-task-definition') console.log('new-task'); +`, { mode: 0o755 }); + const result = spawnSync('bash', [ + 'scripts/ci/roll-core-ecs-image.sh', 'example/core:new', 'example/core', + '/test/image', 'test-cluster', 'test-service', + ], { + encoding: 'utf8', + env: { ...process.env, PATH: `${dir}:${process.env.PATH}`, AM_TEST_LOG: log, AM_TEST_FAIL: failAt }, + }); + return { result, calls: readFileSync(log, 'utf8').trim().split('\n') }; +} + +for (const operation of ['ecs describe-services', 'ecs register-task-definition', 'ecs update-service']) { + test(`does not advance SSM when ${operation} fails`, (t) => { + const { result, calls } = roll(t, operation); + assert.notEqual(result.status, 0); + assert.ok(!calls.includes('ssm put-parameter')); + }); +} + +test('records SSM only after ECS accepts the new task definition', (t) => { + const { result, calls } = roll(t, ''); + assert.equal(result.status, 0, result.stderr); + assert.deepEqual(calls, [ + 'ecs describe-services', 'ecs describe-task-definition', + 'ecs register-task-definition', 'ecs update-service', 'ssm put-parameter', + ]); +}); diff --git a/scripts/ci/__tests__/security-compliance.test.mjs b/scripts/ci/__tests__/security-compliance.test.mjs index eb169d0..0cff634 100644 --- a/scripts/ci/__tests__/security-compliance.test.mjs +++ b/scripts/ci/__tests__/security-compliance.test.mjs @@ -15,6 +15,7 @@ import { readFileSync } from "node:fs"; import { test } from "node:test"; import { validateMirrorCliPromotionGuard, + validateReleaseCliVersionBumpGuard, validateWorkflowPermissions, } from "../../security/security-compliance.mjs"; @@ -22,6 +23,9 @@ const WORKFLOW = ".github/workflows/release-cli.yml"; const INTERNAL_WORKFLOW = ".github/workflows/internal-cli-release.yml"; const MIRROR_WORKFLOW = ".github/workflows/mirror-cli-r2.yml"; const NON_EXEMPT_WORKFLOW = ".github/workflows/ci.yml"; +const CLI_INSTALL_SMOKE_WORKFLOW = ".github/workflows/cli-install-smoke.yml"; +const CLI_PUBLIC_INSTALL_SMOKE_WORKFLOW = + ".github/workflows/cli-public-install-smoke.yml"; function readWorkflowText() { return readFileSync(WORKFLOW, "utf8"); @@ -35,6 +39,10 @@ function readInternalWorkflowText() { return readFileSync(INTERNAL_WORKFLOW, "utf8"); } +function readCliInstallSmokeWorkflowText() { + return readFileSync(CLI_INSTALL_SMOKE_WORKFLOW, "utf8"); +} + test("release-cli keeps publish-only write permissions", () => { const failures = validateWorkflowPermissions(WORKFLOW, readWorkflowText()); assert.deepEqual(failures, []); @@ -236,3 +244,146 @@ test("mirror-cli fails when the monotonic promotion guard is removed", () => { const failures = validateMirrorCliPromotionGuard(MIRROR_WORKFLOW, mutated); assert.ok(failures.some((failure) => /must compare requested version against current version\.json/.test(failure))); }); + +test("release-cli keeps the public version bump guard", () => { + const failures = validateReleaseCliVersionBumpGuard(WORKFLOW, readWorkflowText()); + assert.deepEqual(failures, []); +}); + +test("release-cli fails when the version bump guard is removed", () => { + const mutated = readWorkflowText().replace( + /\n\s+- name: Validate public version bump[\s\S]*?validate-cli-version-bump\.sh\n/, + "\n", + ); + const failures = validateReleaseCliVersionBumpGuard(WORKFLOW, mutated); + assert.ok( + failures.some((failure) => /must invoke scripts\/ci\/validate-cli-version-bump\.sh/.test(failure)), + ); +}); + +// cli-install-smoke is the one entry in the allow-table that is not a release +// lane. Its reporter needs issues: write to file a nightly failure, so these +// pin that the exemption stays exactly that wide and no wider. +test("cli-install-smoke keeps issue reporting as its only write scope", () => { + const failures = validateWorkflowPermissions( + CLI_INSTALL_SMOKE_WORKFLOW, + readCliInstallSmokeWorkflowText(), + ); + assert.deepEqual(failures, []); +}); + +test("cli-install-smoke fails when the report job gains contents write", () => { + const mutated = readCliInstallSmokeWorkflowText().replace( + " contents: read\n issues: write", + " contents: write\n issues: write", + ); + const failures = validateWorkflowPermissions(CLI_INSTALL_SMOKE_WORKFLOW, mutated); + assert.ok(failures.some((failure) => /job report/.test(failure))); +}); + +test("cli-install-smoke fails when the install job gains write permissions", () => { + const doc = readCliInstallSmokeWorkflowText(); + const anchor = " install-smoke:\n"; + const mutated = doc.replace(anchor, anchor + " permissions:\n contents: write\n"); + assert.notEqual(mutated, doc); + const failures = validateWorkflowPermissions(CLI_INSTALL_SMOKE_WORKFLOW, mutated); + assert.ok( + failures.some((failure) => /job install-smoke must not request write permissions/.test(failure)), + ); +}); + +test("cli-install-smoke fails when workflow-level write permissions appear", () => { + const mutated = readCliInstallSmokeWorkflowText().replace( + "permissions:\n contents: read", + "permissions: write-all", + ); + const failures = validateWorkflowPermissions(CLI_INSTALL_SMOKE_WORKFLOW, mutated); + assert.ok( + failures.some((failure) => /workflow permissions must be exactly contents: read/.test(failure)), + ); +}); + +// The public smoke is the second non-release writer, and it earned its entry by +// failing this check first: an unlisted workflow must be read-only, so adding +// the lane without adding the table row is caught rather than assumed. These +// pin that its exemption is exactly as wide as the internal one (ATO-1863). +function readCliPublicInstallSmokeWorkflowText() { + return readFileSync(CLI_PUBLIC_INSTALL_SMOKE_WORKFLOW, "utf8"); +} + +test("cli-public-install-smoke keeps issue reporting as its only write scope", () => { + const failures = validateWorkflowPermissions( + CLI_PUBLIC_INSTALL_SMOKE_WORKFLOW, + readCliPublicInstallSmokeWorkflowText(), + ); + assert.deepEqual(failures, []); +}); + +test("cli-public-install-smoke fails when the report job gains contents write", () => { + const mutated = readCliPublicInstallSmokeWorkflowText().replace( + " contents: read\n issues: write", + " contents: write\n issues: write", + ); + const failures = validateWorkflowPermissions( + CLI_PUBLIC_INSTALL_SMOKE_WORKFLOW, + mutated, + ); + assert.ok(failures.some((failure) => /job report/.test(failure))); +}); + +test("cli-public-install-smoke fails when the smoke job gains write permissions", () => { + const doc = readCliPublicInstallSmokeWorkflowText(); + const anchor = " public-install-smoke:\n"; + const mutated = doc.replace( + anchor, + anchor + " permissions:\n contents: write\n", + ); + assert.notEqual(mutated, doc); + const failures = validateWorkflowPermissions( + CLI_PUBLIC_INSTALL_SMOKE_WORKFLOW, + mutated, + ); + assert.ok( + failures.some((failure) => + /job public-install-smoke must not request write permissions/.test(failure), + ), + ); +}); + +test("cli-public-install-smoke fails when workflow-level write permissions appear", () => { + const mutated = readCliPublicInstallSmokeWorkflowText().replace( + "permissions:\n contents: read", + "permissions: write-all", + ); + const failures = validateWorkflowPermissions( + CLI_PUBLIC_INSTALL_SMOKE_WORKFLOW, + mutated, + ); + assert.ok( + failures.some((failure) => + /workflow permissions must be exactly contents: read/.test(failure), + ), + ); +}); + +const CORE_ECR_WORKFLOW = ".github/workflows/core-ecr-dev-staging.yml"; + +test("core-ecr-dev-staging keeps OIDC publish permissions only", () => { + const text = readFileSync(CORE_ECR_WORKFLOW, "utf8"); + const failures = validateWorkflowPermissions(CORE_ECR_WORKFLOW, text); + assert.deepEqual(failures, []); +}); + +test("core-ecr-dev-staging fails when packages:write is added", () => { + const text = readFileSync(CORE_ECR_WORKFLOW, "utf8"); + const mutated = text.replace( + "permissions:\n contents: read\n id-token: write", + "permissions:\n contents: read\n id-token: write\n packages: write", + ); + const failures = validateWorkflowPermissions(CORE_ECR_WORKFLOW, mutated); + assert.ok( + failures.some((failure) => + /workflow permissions must be exactly contents: read, id-token: write/.test(failure), + ), + ); +}); diff --git a/scripts/ci/bump-cli-workspace-version.sh b/scripts/ci/bump-cli-workspace-version.sh new file mode 100755 index 0000000..18b15ab --- /dev/null +++ b/scripts/ci/bump-cli-workspace-version.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +# Bump the Rust CLI workspace semver and refresh Cargo.lock in one step. +# +# Updates [workspace.package].version, the four workspace.dependencies pins, +# and regenerates Cargo.lock so ci-rust --locked does not fail after a bump. +# +# Usage: +# pnpm run bump:cli-version -- 0.2.2 +# bash scripts/ci/bump-cli-workspace-version.sh 0.2.2 +# +# Optional env: +# CARGO_TOML root Cargo.toml (default: repo Cargo.toml) +# SKIP_LOCKFILE when set, skip cargo generate-lockfile (tests) +# SKIP_VALIDATE when set, skip the adjacent-bump validator (tests) +set -euo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$HERE/../.." && pwd)" +CARGO_TOML="${CARGO_TOML:-$REPO_ROOT/Cargo.toml}" +VERSION_RE='^[0-9]+\.[0-9]+\.[0-9]+$' +PIN_CRATES=(am-core-types am-cloud-types am-cloud-client atomicmemory) + +fail() { + printf '::error::%s\n' "$*" >&2 + exit 1 +} + +rewrite_cargo_toml() { + local ver="$1" tmp + tmp="$(mktemp)" + awk -v ver="$ver" ' + BEGIN { + pins["am-core-types"] = 1 + pins["am-cloud-types"] = 1 + pins["am-cloud-client"] = 1 + pins["atomicmemory"] = 1 + } + /^\[workspace\.package\]/ { in_pkg = 1 } + in_pkg && /^\[/ && $0 != "[workspace.package]" { in_pkg = 0 } + in_pkg && /^version = / { + print "version = \"" ver "\"" + next + } + { + for (crate in pins) { + if (index($0, crate " = { path = ") == 1) { + sub(/version = "[^"]+"/, "version = \"" ver "\"") + print + next + } + } + print + } + ' "$CARGO_TOML" >"$tmp" + mv "$tmp" "$CARGO_TOML" +} + +refresh_lockfile() { + local toml_dir + toml_dir="$(cd "$(dirname "$CARGO_TOML")" && pwd)" + if ! (cd "$toml_dir" && cargo generate-lockfile); then + fail "cargo generate-lockfile failed after rewriting ${CARGO_TOML}" + fi +} + +main() { + local ver="${1:-}" + if [ "${1:-}" = "--" ]; then + shift + ver="${1:-}" + fi + if [ -z "$ver" ]; then + fail "usage: bump-cli-workspace-version.sh X.Y.Z" + fi + if ! printf '%s' "$ver" | grep -Eq "$VERSION_RE"; then + fail "version '${ver}' is not X.Y.Z" + fi + if [ ! -f "$CARGO_TOML" ]; then + fail "Cargo.toml not found at ${CARGO_TOML}" + fi + rewrite_cargo_toml "$ver" + echo "updated ${CARGO_TOML} to ${ver} (workspace + ${PIN_CRATES[*]})" + if [ -z "${SKIP_LOCKFILE:-}" ]; then + refresh_lockfile + echo "refreshed Cargo.lock for workspace ${ver}" + fi + if [ -z "${SKIP_VALIDATE:-}" ]; then + CARGO_TOML="$CARGO_TOML" bash "$HERE/validate-cli-version-bump.sh" + fi +} + +main "$@" diff --git a/scripts/ci/lint-workflows.sh b/scripts/ci/lint-workflows.sh new file mode 100755 index 0000000..a8eca0c --- /dev/null +++ b/scripts/ci/lint-workflows.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +# Lint every GitHub Actions workflow with actionlint, including the shell inside +# each `run:` block. +# +# Why this exists: on 2026-09-01 a stray `fi` landed in mirror-cli-r2.yml's first +# step (ATO-1934). That step is a bash syntax error — it cannot execute a single +# line — and it sat on `dev` undetected, because the repo's four workflow +# checkers (test:mirror-cli-r2, test:release-cli-version, security-compliance, +# reporter-grades-every-lane) all match workflow *text* and none of them parse +# the embedded shell. actionlint does, via shellcheck, and reports it as an +# error. It would have failed the commit that introduced it. +# +# Pinned by version and verified by checksum rather than installed from a +# floating action: this runs on every push, and an unpinned tool in CI is a +# supply-chain edge we control cheaply. +# +# Usage: +# bash scripts/ci/lint-workflows.sh # download the pinned binary +# AM_ACTIONLINT=$(command -v actionlint) \ +# bash scripts/ci/lint-workflows.sh # use an already-installed one +set -euo pipefail + +ACTIONLINT_VERSION="1.7.12" + +ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +cd "$ROOT" + +# An operator-supplied binary skips the download, but never the version check — +# a different actionlint reports a different set of findings, and a lane that +# silently grades against an unknown version is not a gate. +if [[ -n "${AM_ACTIONLINT:-}" ]]; then + installed="$("$AM_ACTIONLINT" --version | head -n1)" + if [[ "$installed" != "$ACTIONLINT_VERSION" ]]; then + echo "AM_ACTIONLINT is actionlint ${installed}, expected ${ACTIONLINT_VERSION}" >&2 + exit 1 + fi + exec "$AM_ACTIONLINT" -color +fi + +# Platform and its checksum resolved together, in a `case` rather than an +# associative array: `declare -A` needs bash 4 and macOS ships 3.2, where it +# fails as an unbound variable instead of a syntax error. +case "$(uname -s)/$(uname -m)" in + Linux/x86_64) + platform="linux_amd64" + expected="8aca8db96f1b94770f1b0d72b6dddcb1ebb8123cb3712530b08cc387b349a3d8" + ;; + Darwin/arm64) + platform="darwin_arm64" + expected="aba9ced2dee8d27fecca3dc7feb1a7f9a52caefa1eb46f3271ea66b6e0e6953f" + ;; + *) + echo "no pinned actionlint for $(uname -s)/$(uname -m)." >&2 + echo "Install it yourself and re-run with AM_ACTIONLINT=\$(command -v actionlint)." >&2 + exit 1 + ;; +esac + +work="$(mktemp -d)" +trap 'rm -rf "$work"' EXIT INT TERM + +tarball="actionlint_${ACTIONLINT_VERSION}_${platform}.tar.gz" +url="https://github.com/rhysd/actionlint/releases/download/v${ACTIONLINT_VERSION}/${tarball}" +curl -fsSL --proto '=https' --tlsv1.2 --max-time 120 --retry 3 "$url" -o "$work/$tarball" + +if command -v sha256sum >/dev/null 2>&1; then + actual="$(sha256sum "$work/$tarball" | cut -d' ' -f1)" +else + actual="$(shasum -a 256 "$work/$tarball" | cut -d' ' -f1)" +fi +if [[ "$actual" != "$expected" ]]; then + echo "checksum mismatch for ${tarball}" >&2 + echo " expected ${expected}" >&2 + echo " actual ${actual}" >&2 + exit 1 +fi + +tar -C "$work" -xzf "$work/$tarball" actionlint +exec "$work/actionlint" -color diff --git a/scripts/ci/reconcile-internal-release.sh b/scripts/ci/reconcile-internal-release.sh index b12ed7e..0e6b06a 100755 --- a/scripts/ci/reconcile-internal-release.sh +++ b/scripts/ci/reconcile-internal-release.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # Reconcile a rerun of the Internal CLI Release workflow against the -# existing immutable cli-internal- release, if any. +# existing immutable cli-internal- / cli-canary- release, if any. # # Called from .github/workflows/internal-cli-release.yml. Because # rebuilt tarballs are NOT byte-reproducible (tar embeds mtimes), a @@ -20,11 +20,11 @@ # downloaded tarballs, and replace $DIST_DIR contents with the # downloaded bytes. Downstream steps (floating alias refresh) # therefore upload the immutable release's exact bytes, so the -# floating alias can never diverge from cli-internal- for +# floating alias can never diverge from the immutable tag for # the same SHA. # # Required env: -# TAG immutable release tag (cli-internal-) +# TAG immutable release tag (cli-internal- or cli-canary-) # SHA expected target commit SHA # GH_REPO owner/name of the repository # GH_TOKEN implicit; passed through to gh @@ -139,7 +139,7 @@ for f in "$DIST_DIR"/*; do done # Swap DIST_DIR for the immutable release's bytes so downstream steps -# (Refresh floating cli-internal-latest) upload identical content, +# (Refresh floating channel alias) upload identical content, # never divergent rebuilds. rm -rf "$DIST_DIR" mv "$reconciled" "$DIST_DIR" diff --git a/scripts/ci/release-policy.mjs b/scripts/ci/release-policy.mjs index bd0fc43..da55033 100644 --- a/scripts/ci/release-policy.mjs +++ b/scripts/ci/release-policy.mjs @@ -15,12 +15,13 @@ * - Container-image publishers are explicitly enumerated: any workflow that * pushes images (docker push / buildx --push / output exporters / * imagetools create / docker/build-push-action) must be either a - * publish-*.yml release lane (covered by the invariants above) or the - * enumerated internal operator publisher. That publisher's workflow YAML - * is parsed structurally and must guard every job on the - * atomicmemory-internal repository, assign IMAGE_NAME exactly once at the - * workflow level (pinned to the private internal package), and push only - * via buildx --push with every --tag deriving from that pin. + * publish-*.yml release lane (covered by the invariants above) or one of + * the enumerated operator publishers (private GHCR internal image, or + * Dev/Staging ECR). Each enumerated publisher's workflow YAML is parsed + * structurally and must guard every job on the atomicmemory-internal + * repository, assign IMAGE_NAME exactly once at the workflow level + * (pinned to that publisher's registry path), and push only via + * buildx --push with every --tag deriving from that pin. */ import { readFileSync, readdirSync } from "node:fs"; @@ -35,7 +36,10 @@ const CODEOWNERS_FILE = ".github/CODEOWNERS"; const GUARD_REL_PATH = "scripts/guards/guard-npm-publish.mjs"; const PUBLISH_WORKFLOW_FILENAME_PREFIX = "publish-"; const INTERNAL_IMAGE_WORKFLOW_FILENAME = "internal-core-docker-image.yml"; +const ECR_DEV_STAGING_WORKFLOW_FILENAME = "core-ecr-dev-staging.yml"; const INTERNAL_IMAGE_NAME = "ghcr.io/atomicstrata/atomicmemory-core-internal"; +const ECR_CORE_IMAGE_NAME = + "636941960505.dkr.ecr.us-east-1.amazonaws.com/atomicmemory-core-enterprise"; // The job-level `if` must equal this exactly (whitespace-normalized): a // compound condition (e.g. `A || B`) could satisfy a substring match while // still running in the mirrored public repository. @@ -47,16 +51,47 @@ const OUTPUT_EXPORTER_RE = /(^|\s)(-o|--output)[=\s][^\n]*\b(type=registry|push= const OTHER_PUSH_SINKS_RE = /\b(docker\s+image\s+push|docker\s+compose\s+push|docker-compose\s+push|podman\s+(image\s+)?push|buildah\s+push|skopeo\s+(copy|sync)|crane\s+(push|cp|copy)|oras\s+push)\b/; // Reusable-workflow refs that publish; a job-level `uses:` of one of these // from outside the audited release lane would launder a publish. -const PUBLISHING_WORKFLOW_REF_RE = /(^|\/)(publish-[^/@\s]*\.ya?ml|internal-core-docker-image\.yml)(@|$)/i; +const PUBLISHING_WORKFLOW_REF_RE = + /(^|\/)(publish-[^/@\s]*\.ya?ml|internal-core-docker-image\.yml|core-ecr-dev-staging\.yml)(@|$)/i; const EXPRESSION_MARKER = "$" + "{{"; const COMPOSITE_SCAN_SKIP_DIRS = new Set(["node_modules", ".git", "dist", "build", ".turbo", ".worktrees"]); -// Actions the enumerated internal publisher may use; anything else (any -// case) is a policy failure so a new action is an explicit policy change. -const INTERNAL_ALLOWED_ACTIONS = [ - "actions/checkout@", - "docker/setup-qemu-action@", - "docker/setup-buildx-action@", -]; +// Enumerated operator image publishers (outside publish-*.yml). Each pin +// IMAGE_NAME and an action allowlist; adding a publisher is an explicit +// policy change. +const ENUMERATED_IMAGE_PUBLISHERS = new Map([ + [ + INTERNAL_IMAGE_WORKFLOW_FILENAME, + { + imageName: INTERNAL_IMAGE_NAME, + allowedActions: [ + "actions/checkout@", + "docker/setup-qemu-action@", + "docker/setup-buildx-action@", + ], + }, + ], + [ + ECR_DEV_STAGING_WORKFLOW_FILENAME, + { + imageName: ECR_CORE_IMAGE_NAME, + allowedActions: [ + "actions/checkout@", + "docker/setup-buildx-action@", + "aws-actions/configure-aws-credentials@", + "aws-actions/amazon-ecr-login@", + ], + }, + ], +]); + +function enumeratedPublisherBasename(filename) { + const basename = filename.split("/").pop() ?? ""; + return ENUMERATED_IMAGE_PUBLISHERS.has(basename) ? basename : null; +} + +function enumeratedPublisherNames() { + return [...ENUMERATED_IMAGE_PUBLISHERS.keys()].join(", "); +} const PUBLISHED_PACKAGE_PATHS = [ "packages/core/package.json", "packages/sdk/package.json", @@ -134,7 +169,7 @@ export function checkCompositeActionText(rawText, filename) { (step) => typeof step?.uses === "string" && step.uses.toLowerCase().startsWith("docker/build-push-action"), ); if (pushesContainerImages(shellText) || usesPushAction) { - return [`${filename}: composite actions must not push container images; only publish-*.yml release lanes and ${INTERNAL_IMAGE_WORKFLOW_FILENAME} may publish.`]; + return [`${filename}: composite actions must not push container images; only publish-*.yml release lanes and enumerated operator publishers (${enumeratedPublisherNames()}) may publish.`]; } return []; } @@ -184,6 +219,11 @@ export function checkImagePublisherText(rawText, filename) { return [`${filename}: workflow YAML is not a mapping, so image-publisher policy cannot validate it.`]; } + const publisherBasename = enumeratedPublisherBasename(filename); + const publisher = publisherBasename + ? ENUMERATED_IMAGE_PUBLISHERS.get(publisherBasename) + : undefined; + const jobs = Object.entries(doc.jobs ?? {}); const runScripts = []; const usedActions = []; @@ -198,17 +238,17 @@ export function checkImagePublisherText(rawText, filename) { const failuresEarly = []; // A job-level `uses:` of a publishing reusable workflow is itself a - // publish path: only publish-*.yml release lanes may call one, and the - // enumerated internal publisher may not delegate to reusable workflows + // publish path: only publish-*.yml release lanes may call one, and + // enumerated operator publishers may not delegate to reusable workflows // at all. for (const { jobName, ref } of jobLevelUses) { - if (filename.endsWith(INTERNAL_IMAGE_WORKFLOW_FILENAME)) { - failuresEarly.push(`${filename}: job '${jobName}' must not call a reusable workflow (uses: ${ref}); the internal image publisher defines its own steps only.`); + if (publisher) { + failuresEarly.push(`${filename}: job '${jobName}' must not call a reusable workflow (uses: ${ref}); the enumerated image publisher defines its own steps only.`); } else if (PUBLISHING_WORKFLOW_REF_RE.test(ref)) { failuresEarly.push(`${filename}: job '${jobName}' calls publishing reusable workflow ${ref}; only publish-*.yml release lanes may do that.`); } } - if (failuresEarly.length > 0 && !filename.endsWith(INTERNAL_IMAGE_WORKFLOW_FILENAME)) return failuresEarly; + if (failuresEarly.length > 0 && !publisher) return failuresEarly; // Docker accepts compact short-option forms (-tVALUE, -oVALUE); split them // so the sink and destination scans below see the canonical spaced form. const shellText = runScripts.join("\n").replace(/(^|[\s"'])-([to])(?=[^\s=])/gm, "$1-$2 "); @@ -216,9 +256,9 @@ export function checkImagePublisherText(rawText, filename) { const usesPushAction = usedActions.some((action) => action.toLowerCase().startsWith("docker/build-push-action")); if (!pushesContainerImages(shellText) && !usesPushAction) return failuresEarly; - if (!filename.endsWith(INTERNAL_IMAGE_WORKFLOW_FILENAME)) { + if (!publisher) { return [ - `${filename}: pushes container images but is neither a ${PUBLISH_WORKFLOW_FILENAME_PREFIX}*.yml release lane nor the enumerated internal image publisher (${INTERNAL_IMAGE_WORKFLOW_FILENAME}).`, + `${filename}: pushes container images but is neither a ${PUBLISH_WORKFLOW_FILENAME_PREFIX}*.yml release lane nor an enumerated operator image publisher (${enumeratedPublisherNames()}).`, ]; } @@ -233,13 +273,13 @@ export function checkImagePublisherText(rawText, filename) { } } // (2) Exactly one IMAGE_NAME env assignment may exist, at the workflow - // level, pinned to the internal package. The parser resolves quoted - // keys, flow mappings, and anchor/alias tricks before we count. + // level, pinned to this publisher's registry path. The parser resolves + // quoted keys, flow mappings, and anchor/alias tricks before we count. const assignments = collectImageNameAssignments(doc, jobs); const pin = assignments.length === 1 ? assignments[0] : undefined; - if (!pin || pin.where !== "workflow env" || pin.value !== INTERNAL_IMAGE_NAME) { + if (!pin || pin.where !== "workflow env" || pin.value !== publisher.imageName) { const found = assignments.map((a) => `${a.where}=${a.value}`).join(", ") || "none"; - failures.push(`${filename}: must assign env IMAGE_NAME exactly once, at the workflow level, pinned to ${INTERNAL_IMAGE_NAME} (found: ${found}).`); + failures.push(`${filename}: must assign env IMAGE_NAME exactly once, at the workflow level, pinned to ${publisher.imageName} (found: ${found}).`); } // (3) No shell-side reassignment (IMAGE_NAME=... in run blocks or // GITHUB_ENV writes). @@ -275,8 +315,8 @@ export function checkImagePublisherText(rawText, filename) { // (7) Only enumerated actions may be used, so no third-party or local // composite action can push on this workflow's behalf. for (const action of usedActions) { - if (!INTERNAL_ALLOWED_ACTIONS.some((allowed) => action.toLowerCase().startsWith(allowed))) { - failures.push(`${filename}: uses: ${action} is not in the internal image publisher's action allowlist (${INTERNAL_ALLOWED_ACTIONS.join(", ")}).`); + if (!publisher.allowedActions.some((allowed) => action.toLowerCase().startsWith(allowed))) { + failures.push(`${filename}: uses: ${action} is not in the enumerated image publisher's action allowlist (${publisher.allowedActions.join(", ")}).`); } } return failures; @@ -413,6 +453,7 @@ function checkCodeownersCovers(root) { return [ ...checkCodeownersText(text, PUBLISH_WORKFLOW), ...checkCodeownersText(text, `.github/workflows/${INTERNAL_IMAGE_WORKFLOW_FILENAME}`), + ...checkCodeownersText(text, `.github/workflows/${ECR_DEV_STAGING_WORKFLOW_FILENAME}`), ]; } diff --git a/scripts/ci/rewrite-ecs-task-def-image.py b/scripts/ci/rewrite-ecs-task-def-image.py new file mode 100755 index 0000000..6d310d4 --- /dev/null +++ b/scripts/ci/rewrite-ecs-task-def-image.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +"""Rewrite an ECS task-definition JSON to pin matching containers to a new image. + +Fail closed: exits non-zero when no container image matches image_repo:*. +Keeps only RegisterTaskDefinition request keys so describe-task-definition +response metadata (taskDefinitionArn, revision, deregisteredAt, …) cannot +reach `aws ecs register-task-definition --cli-input-json`. +""" + +from __future__ import annotations + +import json +import sys + +# Top-level members of RegisterTaskDefinitionRequest (botocore ecs 2014-11-13). +# Prefer an allowlist over popping known read-only keys: AWS adds response-only +# fields (e.g. deregisteredAt, deleteRequestedAt) that break ParamValidation. +REGISTER_TASK_DEFINITION_KEYS = frozenset( + { + "containerDefinitions", + "cpu", + "enableFaultInjection", + "ephemeralStorage", + "executionRoleArn", + "family", + "inferenceAccelerators", + "ipcMode", + "memory", + "networkMode", + "pidMode", + "placementConstraints", + "proxyConfiguration", + "requiresCompatibilities", + "runtimePlatform", + "tags", + "taskRoleArn", + "volumes", + } +) + + +def rewrite_task_definition(task_def: dict, image_uri: str, image_repo: str) -> dict: + replaced = 0 + for container in task_def.get("containerDefinitions", []): + image = container.get("image") or "" + if image == image_uri or image.startswith(f"{image_repo}:"): + container["image"] = image_uri + replaced += 1 + if replaced == 0: + raise SystemExit( + f"No container image matching {image_repo}:* in task definition; refuse to guess." + ) + print(f"Updated {replaced} container image(s) to {image_uri}") + return {key: task_def[key] for key in task_def if key in REGISTER_TASK_DEFINITION_KEYS} + + +def main(argv: list[str]) -> int: + if len(argv) != 4: + print( + "usage: rewrite-ecs-task-def-image.py ", + file=sys.stderr, + ) + return 2 + image_uri, image_repo, src, dst = argv + with open(src, encoding="utf-8") as handle: + task_def = json.load(handle) + rewritten = rewrite_task_definition(task_def, image_uri, image_repo) + with open(dst, "w", encoding="utf-8") as handle: + json.dump(rewritten, handle) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/ci/roll-core-ecs-image.sh b/scripts/ci/roll-core-ecs-image.sh new file mode 100755 index 0000000..4151134 --- /dev/null +++ b/scripts/ci/roll-core-ecs-image.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# Roll one Core ECS service to a new image URI and record it in SSM. +# +# Usage: +# roll-core-ecs-image.sh +# +# Fail closed: refuses to update if no container image matches image-repo:*. +set -euo pipefail + +if [[ "$#" -ne 5 ]]; then + echo "usage: $0 " >&2 + exit 2 +fi + +image_uri="$1" +image_repo="$2" +ssm_param="$3" +cluster="$4" +service="$5" + +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +task_def_arn="$(aws ecs describe-services \ + --cluster "${cluster}" \ + --services "${service}" \ + --query 'services[0].taskDefinition' \ + --output text)" +if [[ -z "${task_def_arn}" || "${task_def_arn}" == "None" ]]; then + echo "::error::No task definition found for ${cluster}/${service}" >&2 + exit 1 +fi + +tmp_dir="$(mktemp -d)" +trap 'rm -rf "${tmp_dir}"' EXIT + +aws ecs describe-task-definition \ + --task-definition "${task_def_arn}" \ + --query 'taskDefinition' \ + > "${tmp_dir}/task-def.json" + +python3 "${here}/rewrite-ecs-task-def-image.py" \ + "${image_uri}" \ + "${image_repo}" \ + "${tmp_dir}/task-def.json" \ + "${tmp_dir}/task-def-new.json" + +new_arn="$(aws ecs register-task-definition \ + --cli-input-json "file://${tmp_dir}/task-def-new.json" \ + --query 'taskDefinition.taskDefinitionArn' \ + --output text)" + +aws ecs update-service \ + --cluster "${cluster}" \ + --service "${service}" \ + --task-definition "${new_arn}" \ + --force-new-deployment \ + >/dev/null + +aws ssm put-parameter \ + --name "${ssm_param}" \ + --type String \ + --value "${image_uri}" \ + --overwrite + +echo "ECS updated: ${cluster}/${service} -> ${new_arn} (${image_uri})" +echo "SSM ${ssm_param}=${image_uri}" diff --git a/scripts/ci/validate-cli-version-bump.sh b/scripts/ci/validate-cli-version-bump.sh new file mode 100755 index 0000000..eaaba66 --- /dev/null +++ b/scripts/ci/validate-cli-version-bump.sh @@ -0,0 +1,303 @@ +#!/usr/bin/env bash +# Validate public `am` CLI semver bumps against the last published cli-v* tag. +# +# Single chokepoint for release-cli.yml (hard gate) and ci-rust.yml (PR gate). +# Adjacent bumps only: patch +1, minor +1 with patch 0, or major +1 with +# minor/patch 0. Workspace dependency pins must match [workspace.package].version. +# +# Optional env: +# CARGO_TOML path to root Cargo.toml (default: Cargo.toml) +# CARGO_LOCK path to Cargo.lock (default: sibling of CARGO_TOML) +# PROPOSED_VERSION version under test (default: workspace.package.version) +# LAST_PUBLIC_VERSION override last public semver (tests / offline) +# PUBLIC_CLI_TAGS ls-remote-shaped tag lines for tests (skips git ls-remote) +# PUBLIC_CLI_RELEASES published GitHub Release versions for tests (skips gh) +# PUBLIC_REPO git remote for cli-v* tags (default: public product repo) +# RELEASE_MODE when set, fail if proposed equals last public (tag push) +set -euo pipefail + +CARGO_TOML="${CARGO_TOML:-Cargo.toml}" +PUBLIC_REPO="${PUBLIC_REPO:-https://github.com/atomicstrata/atomicmemory.git}" +VERSION_RE='^[0-9]+\.[0-9]+\.[0-9]+$' +PIN_CRATES=(am-core-types am-cloud-types am-cloud-client atomicmemory) + +fail() { + printf '::error::%s\n' "$*" >&2 + exit 1 +} + +read_workspace_version() { + local ver + ver="$(awk '/^\[workspace\.package\]/{found=1; next} found && /^version = /{ + gsub(/[" ]/,"",$3); print $3; exit + }' "$CARGO_TOML")" + if [ -z "$ver" ]; then + fail "could not read [workspace.package].version from ${CARGO_TOML}" + fi + if ! printf '%s' "$ver" | grep -Eq "$VERSION_RE"; then + fail "workspace version '${ver}' is not X.Y.Z" + fi + printf '%s' "$ver" +} + +read_pin_version() { + local crate="$1" + sed -n "s/^${crate} = { path = .* version = \"\\([^\"]*\\)\".*/\\1/p" "$CARGO_TOML" +} + +assert_workspace_pins() { + local workspace_ver="$1" + local crate pin + for crate in "${PIN_CRATES[@]}"; do + pin="$(read_pin_version "$crate")" + if [ -z "$pin" ]; then + fail "could not read workspace.dependencies pin for ${crate} in ${CARGO_TOML}" + fi + if [ "$pin" != "$workspace_ver" ]; then + fail "workspace pin ${crate}=${pin} does not match [workspace.package].version=${workspace_ver}" + fi + done +} + +resolve_cargo_lock() { + if [ -n "${CARGO_LOCK:-}" ]; then + printf '%s' "$CARGO_LOCK" + return + fi + local sibling + sibling="$(dirname "$CARGO_TOML")/Cargo.lock" + if [ -f "$sibling" ]; then + printf '%s' "$sibling" + fi +} + +read_lock_version() { + local crate="$1" lock="$2" + awk -v crate="$crate" ' + function flush() { + if (done) return + if (pending != "" && has_source == 0) { + print pending + done = 1 + exit + } + pending = "" + has_source = 0 + } + /^\[\[package\]\]/ { flush() } + /^name = / { gsub(/"/, "", $3); name = $3 } + /^version = / { + gsub(/"/, "", $3) + if (name == crate) pending = $3 + } + /^source = / { has_source = 1 } + END { flush() } + ' "$lock" +} + +assert_lockfile_versions() { + local workspace_ver="$1" + local lock crate lock_ver + lock="$(resolve_cargo_lock || true)" + if [ -z "$lock" ]; then + return 0 + fi + if [ ! -f "$lock" ]; then + fail "Cargo.lock not found at ${lock}; run: pnpm run bump:cli-version -- ${workspace_ver}" + fi + for crate in "${PIN_CRATES[@]}"; do + lock_ver="$(read_lock_version "$crate" "$lock")" + if [ -z "$lock_ver" ]; then + fail "could not read ${crate} version from ${lock}" + fi + if [ "$lock_ver" != "$workspace_ver" ]; then + fail "${crate} in ${lock} is ${lock_ver}, expected ${workspace_ver}; run: pnpm run bump:cli-version -- ${workspace_ver}" + fi + done +} + +semver_gt() { + local left="$1" right="$2" + local l_major l_minor l_patch r_major r_minor r_patch + IFS=. read -r l_major l_minor l_patch <<<"$left" + IFS=. read -r r_major r_minor r_patch <<<"$right" + if ((10#$l_major > 10#$r_major)); then return 0; fi + if ((10#$l_major < 10#$r_major)); then return 1; fi + if ((10#$l_minor > 10#$r_minor)); then return 0; fi + if ((10#$l_minor < 10#$r_minor)); then return 1; fi + ((10#$l_patch > 10#$r_patch)) +} + +allowed_next_versions() { + local base="$1" + local major minor patch + IFS=. read -r major minor patch <<<"$base" + printf '%s.%s.%s\n' "$major" "$minor" "$((patch + 1))" + printf '%s.%s.0\n' "$major" "$((minor + 1))" + printf '%s.0.0\n' "$((major + 1))" +} + +is_adjacent_bump() { + local from="$1" to="$2" candidate + while IFS= read -r candidate; do + if [ "$candidate" = "$to" ]; then + return 0 + fi + done < <(allowed_next_versions "$from") + return 1 +} + +fetch_public_cli_tag_lines() { + if [ "${PUBLIC_CLI_TAGS+set}" = "set" ]; then + printf '%s' "$PUBLIC_CLI_TAGS" + return 0 + fi + git ls-remote --tags "$PUBLIC_REPO" 'refs/tags/cli-v*' 2>/dev/null +} + +parse_tag_line_version() { + local tag="$1" + local ver="${tag##*/cli-v}" + ver="${ver%%\^*}" + if printf '%s' "$ver" | grep -Eq "$VERSION_RE"; then + printf '%s' "$ver" + fi +} + +public_release_repo() { + local repo="${PUBLIC_REPO#https://github.com/}" + repo="${repo%.git}" + printf '%s' "$repo" +} + +# A git tag is not a publication. release-cli creates the GitHub Release after +# this gate, so the triggering tag always exists while the release may not. +# gh exit 0 = published; a confirmed 404 = absent; any other failure is closed. +published_release_exists() { + local proposed="$1" line ver + if [ "${PUBLIC_CLI_TAGS+set}" = "set" ] || [ "${PUBLIC_CLI_RELEASES+set}" = "set" ]; then + while IFS= read -r line; do + [ -n "$line" ] || continue + ver="${line#cli-v}" + if [ "$ver" = "$proposed" ]; then + return 0 + fi + done <<<"${PUBLIC_CLI_RELEASES:-}" + return 1 + fi + if [ -z "${RELEASE_MODE:-}" ]; then + return 1 + fi + lookup_published_release "$proposed" +} + +lookup_published_release() { + local proposed="$1" repo status + repo="$(public_release_repo)" + if ! command -v gh >/dev/null 2>&1; then + fail "cannot determine whether cli-v${proposed} is already published (gh not available)" + fi + local err + err="$(mktemp)" + set +e + gh release view "cli-v${proposed}" --repo "$repo" >/dev/null 2>"$err" + status=$? + set -e + if [ "$status" -eq 0 ]; then + rm -f "$err" + return 0 + fi + if [ "$status" -eq 1 ] && grep -qiE 'not found|HTTP[[:space:]]*404' "$err"; then + rm -f "$err" + return 1 + fi + rm -f "$err" + fail "cannot determine whether cli-v${proposed} is already published (gh exited ${status})" +} + +# Previous published baseline: max cli-v* semver excluding the release under test. +# The triggering tag is already on the remote when release-cli runs, so counting +# it as "last public" rejects valid adjacent releases (0.2.1 -> 0.2.2). +resolve_last_public_version() { + local exclude_ver="${1:-}" + + if [ "${LAST_PUBLIC_VERSION+set}" = "set" ]; then + printf '%s' "$LAST_PUBLIC_VERSION" + return + fi + + local tags tag ver best="" + if ! tags="$(fetch_public_cli_tag_lines)"; then + fail "could not list public cli-v* tags from ${PUBLIC_REPO}" + fi + while IFS= read -r tag; do + [ -n "$tag" ] || continue + ver="$(parse_tag_line_version "$tag")" + [ -n "$ver" ] || continue + if [ -n "$exclude_ver" ] && [ "$ver" = "$exclude_ver" ]; then + continue + fi + if [ -z "$best" ] || semver_gt "$ver" "$best"; then + best="$ver" + fi + done <<<"$tags" + if [ -n "$best" ]; then + printf '%s' "$best" + fi +} + +validate_bump() { + local proposed="$1" last="$2" + if [ -z "$last" ]; then + if published_release_exists "$proposed"; then + if [ -n "${RELEASE_MODE:-}" ]; then + fail "refusing release ${proposed}: cli-v${proposed} is already published" + fi + echo "ok: workspace version ${proposed} matches last public release (no bump yet)" + return 0 + fi + echo "ok: first public release (${proposed}); no prior cli-v* baseline" + return 0 + fi + if [ "$proposed" = "$last" ]; then + if [ -n "${RELEASE_MODE:-}" ]; then + fail "refusing release ${proposed}: identical to last public cli-v${last}" + fi + echo "ok: workspace version ${proposed} matches last public release (no bump yet)" + return 0 + fi + if ! semver_gt "$proposed" "$last"; then + fail "refusing ${proposed}: lower than last public cli-v${last}" + fi + if ! is_adjacent_bump "$last" "$proposed"; then + local allowed + allowed="$(allowed_next_versions "$last" | tr '\n' ',' | sed 's/,$//')" + fail "refusing ${proposed}: not an adjacent bump from last public ${last} (allowed: ${allowed})" + fi + if published_release_exists "$proposed"; then + if [ -n "${RELEASE_MODE:-}" ]; then + fail "refusing release ${proposed}: cli-v${proposed} is already published" + fi + echo "ok: workspace version ${proposed} matches last public release (no bump yet)" + return 0 + fi + echo "ok: ${proposed} is a valid adjacent bump from last public ${last}" +} + +main() { + local workspace_ver proposed last + workspace_ver="$(read_workspace_version)" + proposed="${PROPOSED_VERSION:-$workspace_ver}" + if ! printf '%s' "$proposed" | grep -Eq "$VERSION_RE"; then + fail "proposed version '${proposed}' is not X.Y.Z" + fi + if [ "$proposed" != "$workspace_ver" ] && [ -n "${RELEASE_MODE:-}" ]; then + fail "proposed version ${proposed} does not match workspace version ${workspace_ver}" + fi + assert_workspace_pins "$workspace_ver" + assert_lockfile_versions "$workspace_ver" + last="$(resolve_last_public_version "$proposed" || true)" + validate_bump "$proposed" "$last" +} + +main "$@" diff --git a/scripts/cli-install-smoke.sh b/scripts/cli-install-smoke.sh new file mode 100755 index 0000000..3d8bd49 --- /dev/null +++ b/scripts/cli-install-smoke.sh @@ -0,0 +1,603 @@ +#!/usr/bin/env bash +# Fresh-install smoke test for the published `am` CLI. +# +# Installs a published channel the way someone with a brand-new laptop would, +# into a throwaway $HOME, and proves the result is a working CLI: +# +# 1. Build a sandbox $HOME and an empty install dir +# 2. Fetch install.sh + version.json from the channel under test +# 3. Run the installer into that dir +# 4. Assert the installed binary reports version.json's version +# 5. Assert it identifies itself and runs a real offline subcommand +# 6. Assert PATH activation puts *this* binary first on PATH +# +# Every probe addresses the binary by absolute path, so an `am` the operator +# already has installed cannot answer for the one under test. The activation +# check is the only one that consults PATH, and a competing `am` makes that +# check stronger rather than weaker: it has to lose to the env file. +# +# Why this exists when scripts/__tests__/install-cli*.test.sh already pass: +# those are fixture tests. They drive the installer with a fake `gh` and a fake +# `am` shell script, so they prove the installer's *logic* and nothing about +# the artifact. They stay green if cli-internal-latest is deleted, if its +# tarballs and its version.json disagree, or if the real binary cannot start on +# a machine without a Rust toolchain. The release workflow's own "Native smoke" +# is closer but runs on the build runner, in the build job, minutes after +# cargo built it — the least clean machine available, and it untars directly +# without touching either installer. +# +# Two channels, because they fail differently and only one of them is what a +# user runs (ATO-1863). +# +# internal cli-internal-latest, via `gh release download`. Needs repo auth, +# so anyone on the team can run it, and it says nothing about the +# public mirror. +# public https://get.atomicstrata.ai, via curl — the exact command in the +# README. Needs no auth at all, and is the one whose breakage a +# customer finds before we do. It also reaches things the internal +# channel structurally cannot: that the mirror serves a working +# install.sh, and that its version.json agrees with the tarballs +# sitting beside it. +# +# Everything after acquisition is identical, which is why this is one script +# with a switch rather than two scripts that drift. +# +# Usage: +# ./scripts/cli-install-smoke.sh # internal +# AM_SMOKE_CHANNEL=public ./scripts/cli-install-smoke.sh # public mirror +# AM_INTERNAL_TAG=cli-internal- ./scripts/cli-install-smoke.sh # pin a build +# AM_INTERNAL_TAG=cli-canary-latest ./scripts/cli-install-smoke.sh # floating canary +# AM_SMOKE_MODE=provenance AM_SMOKE_CHANNEL=public ./scripts/... # verify signatures +# AM_SMOKE_KEEP=1 ./scripts/cli-install-smoke.sh # keep the sandbox +# +# AM_VERIFY_ATTESTATION is NOT an input here. `install` mode forces it to 0 for +# the installer (a downloaded script verifying its own provenance is theatre); +# signatures are checked by `provenance` mode, which calls `gh attestation +# verify` on the tarball directly. +# +# Requires: curl, tar, sha256sum or shasum. `gh` only for the internal channel +# and for `provenance` mode. + +set -euo pipefail + +AM_SMOKE_CHANNEL="${AM_SMOKE_CHANNEL:-internal}" + +# What this invocation is allowed to do, and therefore what credential it may +# hold. The two must not run in one process (ATO-1863 review). +# +# install the customer path. Executes install.sh, which we downloaded +# from a mirror and do not control. Must never hold a GitHub +# token: that script runs before anything has authenticated it, +# so a compromised mirror could exfiltrate the credential and +# any later checksum or attestation check would be too late to +# matter. It is the one step whose whole point is that a user +# needs no repo auth. +# +# provenance verifies the published tarball with `gh attestation verify`, +# from this checked-in script. Holds the token, executes nothing +# it downloaded. +# +# Splitting them is what makes the credential unreachable rather than merely +# unused: `env -u` would still leave the token readable through the parent's +# /proc entry on Linux, so the boundary has to be the job, not the child. +AM_SMOKE_MODE="${AM_SMOKE_MODE:-install}" +AM_PUBLIC_BASE_URL="${AM_PUBLIC_BASE_URL:-https://get.atomicstrata.ai}" +# In a function, not at top level: scripts/__tests__/cli-install-smoke.test.sh +# sources this file, so a top-level `exit 2` here killed the test process. Any +# engineer or CI job with AM_SMOKE_MODE or AM_PUBLIC_BASE_URL exported turned +# `npm run test:cli-install-smoke` into a one-line refusal with 0 of its tests +# run — a gate silently reduced to nothing by an unrelated environment variable. +validate_config() { + case "$AM_SMOKE_MODE" in + install|provenance) ;; + *) + echo "AM_SMOKE_MODE must be 'install' or 'provenance', got '${AM_SMOKE_MODE}'" >&2 + return 2 + ;; + esac + + # Enumerated at the same chokepoint as the mode. Without this, `publik`, + # `PUBLIC` and the empty string were all classified as "a public install" by + # needs_gh — dropping the gh requirement and skipping the AM_BASE_URL bridge, + # then dying much later with an unrelated message. download_release_assets + # has its own arm for this, but that is the point of consequence, not the + # point of entry. + case "$AM_SMOKE_CHANNEL" in + internal|public) ;; + *) + echo "AM_SMOKE_CHANNEL must be 'internal' or 'public', got '${AM_SMOKE_CHANNEL}'" >&2 + return 2 + ;; + esac + + # provenance_main downloads release metadata for AM_SMOKE_CHANNEL but always + # attests ${AM_PUBLIC_BASE_URL}/cli/v/. With the default + # channel that means pulling internal metadata over gh and then verifying the + # PUBLIC mirror's tarball, reporting a pass for an artifact it never named — + # and internal builds carry no attestations at all (release-cli.yml is the + # only workflow with `attestations: write`). + if [[ "$AM_SMOKE_MODE" == "provenance" && "$AM_SMOKE_CHANNEL" != "public" ]]; then + echo "AM_SMOKE_MODE=provenance requires AM_SMOKE_CHANNEL=public;" \ + "got '${AM_SMOKE_CHANNEL}'. Only the public mirror publishes attestations." >&2 + return 2 + fi + + # Normalised before it is compared or concatenated. The allowlist accepted the + # trailing-slash form, and every consumer builds "${BASE}/path" — so that form + # produced "//install.sh", which the mirror 404s. Verified live: /install.sh + # is 200, //install.sh is 404. Accepting a value that cannot work is worse + # than rejecting it, because the failure looks like a mirror outage. + AM_PUBLIC_BASE_URL="${AM_PUBLIC_BASE_URL%/}" + + # Enumerated, because this script downloads install.sh from that origin and + # pipes it to `sh`. A free-text override is remote code execution on the + # runner for anyone who can dispatch the workflow, and "only maintainers can + # dispatch" is a policy that lives somewhere else and can change without this + # file noticing. Exact hosts, not prefixes: `https://get.atomicstrata.ai*` + # also matches `https://get.atomicstrata.ai.evil.com`. + # + # There is exactly one public mirror today. Add a second line here when there + # is a second mirror — that is the intended way to extend this, not removing + # it. + case "${AM_PUBLIC_BASE_URL}" in + https://get.atomicstrata.ai) ;; + *) + echo "Refusing to install from ${AM_PUBLIC_BASE_URL}: not an allowed mirror." >&2 + echo "This script pipes the downloaded install.sh to sh; the origin is pinned." >&2 + return 2 + ;; + esac +} +AM_INTERNAL_REPO="${AM_INTERNAL_REPO:-atomicstrata/atomicmemory-internal}" +AM_INTERNAL_TAG="${AM_INTERNAL_TAG:-cli-internal-latest}" + +# Who must have signed the release, and from which workflow. Pinned here rather +# than read from the release: an attestation check that trusts the artifact to +# say who signed it verifies nothing. +AM_ATTESTATION_REPO="atomicstrata/atomicmemory" +AM_ATTESTATION_WORKFLOW="atomicstrata/atomicmemory/.github/workflows/release-cli.yml" +AM_SMOKE_KEEP="${AM_SMOKE_KEEP:-0}" +# The installer picks its rc file from $SHELL. Pinning it keeps the PATH +# assertions deterministic on runners, where SHELL is often unset. +SANDBOX_SHELL="/bin/bash" +SEMVER_RE='^[0-9]+\.[0-9]+\.[0-9]+$' + +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' + +passed=0 +failed=0 +total=0 +WORK="" + +# All three write to stderr. read_expected_version and detect_target are called +# inside `$( )`, so anything they put on stdout is captured into the variable +# instead of shown: a version mismatch or an unsupported arch used to exit 1 +# having printed zero bytes anywhere the operator could see, leaving a red job +# with an empty log. Same defect class as the swallowed `call()` exit in the +# api live-smoke. +log() { echo -e "${GREEN}[cli-smoke]${NC} $*" >&2; } +warn() { echo -e "${YELLOW}[cli-smoke]${NC} $*" >&2; } +fail() { echo -e "${RED}[FAIL]${NC} $*" >&2; } + +# Discards the command's own output. Redirecting the assert_ok call instead +# would also discard its PASS/FAIL line, leaving a failure with no name. +quietly() { "$@" >/dev/null 2>&1; } + +# Same, but keeps stderr. For commands whose diagnostics are the only thing +# distinguishing one failure from another — `gh attestation verify` reports +# whether the digest has no attestation, the signer workflow disagrees, or the +# source ref is wrong, and all three look identical without it. Mirrors +# install-cli.sh:417, which redirects stdout only. +quietly_stdout() { "$@" >/dev/null; } + +# Runs the command directly rather than eval'ing a string, so values +# containing spaces or quotes cannot change what gets tested. +assert_ok() { + local name="$1" + shift + total=$((total + 1)) + if "$@"; then + log " PASS: $name" + passed=$((passed + 1)) + else + fail " FAIL: $name" + failed=$((failed + 1)) + fi +} + +cleanup() { + if [[ -n "$WORK" && -d "$WORK" ]]; then + if [[ "$AM_SMOKE_KEEP" == "1" ]]; then + warn "keeping sandbox: $WORK" + else + rm -rf "$WORK" + fi + fi +} +trap cleanup EXIT INT TERM + +# `gh` is required only where it is actually used: the internal channel's +# download, and attestation verification on either. Demanding it unconditionally +# would make the public lane — whose whole point is that a user needs no repo +# auth — fail for a reason no user could ever hit. +needs_gh() { + # Provenance is the mode that exists to use it. + [[ "$AM_SMOKE_MODE" == "provenance" ]] && return 0 + # The internal channel downloads its release through gh; that is our own + # private artifact and the call is made by this script, not by anything it + # fetched. + [[ "$AM_SMOKE_CHANNEL" == "internal" ]] && return 0 + # A public install never needs it, and must not have it — see AM_SMOKE_MODE. + # AM_SMOKE_MODE is validated to install|provenance above, so this is total: + # every remaining case is a public install. + return 1 +} + +# The workflow runs one job per published target and names each job after it, +# but the job derives its target from `uname`, not from the matrix. Nothing tied +# the two together: GitHub has re-pointed runner labels before (macos-latest +# flipped Intel to arm64), so the leg named x86_64-apple-darwin could quietly +# install and attest the aarch64 tarball, report green, and leave the x86_64 +# artifact never exercised again — invisibly, because the job name still said +# otherwise. Same silent-green class as the missing /cli/v segment. +# +# No-op when AM_SMOKE_TARGET is unset, so a local run needs no ceremony. +assert_target_matches_matrix() { + [[ -n "${AM_SMOKE_TARGET:-}" ]] || return 0 + local detected + detected="$(detect_target)" + assert_ok "runner matches the matrix target (${AM_SMOKE_TARGET})" \ + test "$detected" = "$AM_SMOKE_TARGET" +} + +# Sets INSTALL_CMD to the argv that runs the downloaded installer. +# +# Public only: the installer arrives from a mirror and is not authenticated +# when it runs, so a locally-run smoke on a developer machine with `gh` logged +# in must not hand it a credential. +# +# The internal channel is the opposite case and must NOT be stripped. Its +# install.sh *is* our own scripts/install-cli-internal.sh +# (internal-cli-release.yml: `cp scripts/install-cli-internal.sh +# dist/install.sh`) and it reaches the private repo with `gh release download`. +# HOME is already the sandbox, so ~/.config/gh is invisible; stripping both vars +# on top of that left gh with no credential at all and the installer died with +# "are you authenticated?". Applying the strip to both channels broke the +# internal nightly on every target. +# +# Both branches are non-empty on purpose: `"${arr[@]}"` on an empty array is an +# unbound-variable error under `set -u` in bash 3.2, which the macOS runners +# ship. +set_install_cmd() { + if [[ "$AM_SMOKE_CHANNEL" == "public" ]]; then + INSTALL_CMD=(env -u GH_TOKEN -u GITHUB_TOKEN sh "$1") + else + INSTALL_CMD=(sh "$1") + fi +} + +require_commands() { + local cmd + local required=(curl tar python3) + needs_gh && required+=(gh) + for cmd in "${required[@]}"; do + command -v "$cmd" >/dev/null 2>&1 || { fail "required command not found: $cmd"; exit 1; } + done + command -v sha256sum >/dev/null 2>&1 || command -v shasum >/dev/null 2>&1 \ + || { fail "need sha256sum or shasum on PATH"; exit 1; } +} + +# The installer runs under a sandbox $HOME, which would hide ~/.config/gh and +# break `gh release download`. Hand it an explicit token instead of relocating +# the operator's gh config into the sandbox. +resolve_gh_token() { + if [[ -n "${GH_TOKEN:-}" ]]; then return 0; fi + if [[ -n "${GITHUB_TOKEN:-}" ]]; then + GH_TOKEN="$GITHUB_TOKEN" + export GH_TOKEN + return 0 + fi + GH_TOKEN="$(gh auth token 2>/dev/null || true)" + [[ -n "$GH_TOKEN" ]] || { fail "gh is not authenticated; run: gh auth login"; exit 1; } + export GH_TOKEN +} + +# install-cli.sh rejects install dirs outside [A-Za-z0-9._/-]; catch a hostile +# TMPDIR here so the failure names the cause instead of surfacing as a +# confusing installer error. +make_sandbox() { + WORK="$(mktemp -d "${TMPDIR:-/tmp}/am-cli-smoke.XXXXXX")" || { fail "mktemp failed"; exit 1; } + case "$WORK" in + /*) ;; + *) fail "sandbox is not an absolute path: $WORK"; exit 1 ;; + esac + case "$WORK" in + *[!A-Za-z0-9._/-]*) + fail "sandbox path has characters the installer rejects (set TMPDIR to a plain path): $WORK" + exit 1 + ;; + esac + mkdir -p "$WORK/home" "$WORK/bin" "$WORK/release" +} + +# Fetch over https with the same flags the README's one-liner uses, so a TLS or +# redirect problem that would break a real install breaks this too. `-f` is what +# turns a 404 page into a non-zero exit instead of a file full of HTML that the +# next step would cheerfully parse. +fetch_public_asset() { + local name="$1" + curl --proto '=https' --tlsv1.2 -fsSL --max-time 60 \ + "${AM_PUBLIC_BASE_URL}/${name}" -o "$WORK/release/${name}" \ + || { fail "could not fetch ${name} from ${AM_PUBLIC_BASE_URL}"; exit 1; } +} + +download_release_assets() { + case "$AM_SMOKE_CHANNEL" in + internal) + log "Downloading ${AM_INTERNAL_TAG} metadata from ${AM_INTERNAL_REPO}" + gh release download "$AM_INTERNAL_TAG" \ + --repo "$AM_INTERNAL_REPO" \ + --dir "$WORK/release" \ + --pattern install.sh \ + --pattern version.json \ + || { fail "gh release download failed for ${AM_INTERNAL_TAG}"; exit 1; } + ;; + public) + log "Fetching install.sh and version.json from ${AM_PUBLIC_BASE_URL}" + fetch_public_asset install.sh + fetch_public_asset version.json + ;; + *) + fail "AM_SMOKE_CHANNEL must be 'internal' or 'public', got '${AM_SMOKE_CHANNEL}'" + exit 1 + ;; + esac + [[ -f "$WORK/release/install.sh" ]] || { fail "install.sh missing from the ${AM_SMOKE_CHANNEL} channel"; exit 1; } + [[ -f "$WORK/release/version.json" ]] || { fail "version.json missing from the ${AM_SMOKE_CHANNEL} channel"; exit 1; } +} + +# Read the release's own version and prove it is a real version before using it +# as an expectation. Without this an empty version.json would make the compare +# "" = "" and the whole smoke would pass while asserting nothing. +read_expected_version() { + local version_json="$1" + local version + version="$(sed -n 's/.*"version"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' \ + "$version_json" | head -n1)" + if [[ ! "$version" =~ $SEMVER_RE ]]; then + fail "version.json does not carry an X.Y.Z version (got: '${version}')" + exit 1 + fi + + # The install and provenance lanes are separate jobs, each fetching from the + # mirror on its own runner. Without this they can silently disagree: a release + # landing between them means one installs X while the other attests X+1, and + # both report green. The workflow resolves the version once and hands the same + # value to both, so a mismatch here means the artifact under test is not the + # artifact that was verified — which has to be a failure, not a shrug. + # + # It does not make the two downloads identical. A mirror serving different + # bytes for the same version to different runners would still pass both; that + # would need a shared artifact rather than a shared version, and is out of + # scope for a nightly against our own CDN. + if [[ -n "${AM_SMOKE_EXPECT_VERSION:-}" && "$version" != "$AM_SMOKE_EXPECT_VERSION" ]]; then + fail "release moved mid-run: expected ${AM_SMOKE_EXPECT_VERSION}, mirror now serves ${version}" + exit 1 + fi + + printf '%s' "$version" +} + +# Accept the machine-readable JSON contract (`am --version`) and the legacy +# `am X.Y.Z` banner during the one-release transition. Version comparison is +# literal string equality after parsing top-level JSON fields — dots in +# X.Y.Z must not become regex wildcards. +am_version_output_matches() { + local reported="$1" + local expected="$2" + if [[ "$reported" == "am ${expected}" ]]; then + return 0 + fi + printf '%s' "$reported" | python3 -c ' +import json, sys +expected = sys.argv[1] +try: + data = json.loads(sys.stdin.read()) +except Exception: + sys.exit(1) +if not isinstance(data, dict): + sys.exit(1) +surface = data.get("surface") +version = data.get("version") +if surface == "cli" and version == expected: + sys.exit(0) +sys.exit(1) +' "$expected" +} + +# Same rules install-cli.sh uses, so the artifact verified is the artifact a +# user on this machine would install. Copied rather than sourced: install-cli.sh +# is the file under test on the public channel, and deriving the expectation +# from the thing being tested is how a check ends up agreeing with a compromise. +detect_target() { + local os arch os_part arch_part + os="$(uname -s)"; arch="$(uname -m)" + case "$os" in + Linux) os_part="unknown-linux-gnu" ;; + Darwin) os_part="apple-darwin" ;; + *) fail "unsupported OS for provenance: $os"; exit 1 ;; + esac + case "$arch" in + x86_64|amd64) arch_part="x86_64" ;; + arm64|aarch64) arch_part="aarch64" ;; + *) fail "unsupported arch for provenance: $arch"; exit 1 ;; + esac + printf '%s-%s' "$arch_part" "$os_part" +} + +# Downloads the tarball and checks who signed it. Nothing here executes +# anything that came off the network — that is the property that lets this hold +# the token. +provenance_main() { + require_commands + resolve_gh_token + make_sandbox + download_release_assets + + local version target tarball url + version="$(read_expected_version "$WORK/release/version.json")" + assert_target_matches_matrix + target="$(detect_target)" + tarball="am-${version}-${target}.tar.gz" + # Layout comes from install-cli.sh: REL_URL="${AM_BASE_URL}/cli/v${AM_VERSION}". + # The first version of this omitted the path segment and 404'd on every + # platform — invisible to the tests, which only ever scanned source. + url="${AM_PUBLIC_BASE_URL}/cli/v${version}/${tarball}" + + log "Verifying provenance of ${tarball}" + curl -fsSL --proto '=https' --tlsv1.2 "$url" -o "$WORK/release/${tarball}" \ + || { fail "could not fetch ${tarball} from ${AM_PUBLIC_BASE_URL}"; exit 1; } + + # quietly_stdout, not quietly: this is the one command the provenance job + # exists to run, and swallowing its stderr left a failure as a bare FAIL line. + assert_ok "gh attestation verify ${tarball}" \ + quietly_stdout gh attestation verify "$WORK/release/${tarball}" \ + --repo "${AM_ATTESTATION_REPO}" \ + --signer-workflow "${AM_ATTESTATION_WORKFLOW}" \ + --source-ref "refs/tags/cli-v${version}" + + report "provenance of am ${version}" +} + +main() { + require_commands + needs_gh && resolve_gh_token + make_sandbox + download_release_assets + + local expected_version + expected_version="$(read_expected_version "$WORK/release/version.json")" + assert_target_matches_matrix + log "Release ${AM_INTERNAL_TAG} declares am ${expected_version}" + + local sandbox_home="$WORK/home" + local install_dir="$WORK/bin" + # Kept so the activation check can start from the real PATH rather than one + # the installer has already modified. + local outer_path="$PATH" + + # A fresh $HOME is what makes this a *fresh install*: no ~/.atomicmemory, no + # rc files, nothing the CLI has seen before. PATH is deliberately left alone — + # an earlier version of this dropped every directory containing an `am`, which + # takes gh, curl or tar with it whenever the operator installed the CLI into + # the same prefix. That is the normal case for Homebrew and /usr/local/bin, + # and CI never sees it because runners have no `am` to collide with. + export HOME="$sandbox_home" + export SHELL="$SANDBOX_SHELL" + export AM_INTERNAL_REPO AM_INTERNAL_TAG + # Attestation is NOT delegated to the downloaded installer any more. Letting + # it verify itself required handing it a token, which is the thing that must + # not happen — and a compromised installer verifying its own provenance is + # theatre regardless. The `provenance` mode checks the same artifact from + # this script instead. `0` also matches what a real user without `gh` gets, + # which is the path most of them are on. + export AM_VERIFY_ATTESTATION=0 + # install-cli.sh fetches tarballs from AM_BASE_URL, not AM_PUBLIC_BASE_URL. + # Without this the public lane asserts --version against the *candidate* + # version.json it just downloaded and then installs the *production* tarball — + # the two agree on the scheduled default, so it would only ever be wrong on a + # dispatch, which is exactly where nobody would look. + [[ "$AM_SMOKE_CHANNEL" == "public" ]] && export AM_BASE_URL="$AM_PUBLIC_BASE_URL" + # And pin the version. install-cli.sh re-resolves ${AM_BASE_URL}/version.json + # when AM_VERSION is empty (install-cli.sh:646), so without this the smoke + # asserts `am --version` against the version IT fetched while the installer + # independently fetched again seconds later. A release publishing in that + # window reds the job and files "the path users take is broken" when nothing + # is — and the run never proves it installed the version resolve-release + # pinned and public-provenance attested. mirror-cli-r2.yml:186 does the same. + export AM_VERSION="$expected_version" + + log "Test: the sandbox starts clean" + assert_ok "install dir starts empty" test ! -e "${install_dir}/am" + assert_ok "sandbox HOME starts without ~/.atomicmemory" test ! -e "${sandbox_home}/.atomicmemory" + + log "Installing into ${install_dir} with HOME=${sandbox_home}" + # Public only. Belt to the job-level braces: the public installer arrives from + # a mirror and is not authenticated when it runs, so a locally-run smoke on a + # developer machine with `gh` logged in must not hand it a credential. + # + # The internal channel is the opposite case and must NOT be stripped. Its + # install.sh *is* our own scripts/install-cli-internal.sh + # (internal-cli-release.yml: `cp scripts/install-cli-internal.sh + # dist/install.sh`), and it reaches the private repo with `gh release + # download`. HOME is already the sandbox, so ~/.config/gh is invisible too — + # stripping both vars on top of that leaves gh with no credential at all and + # the installer dies with "are you authenticated?". Applying the strip to + # both channels broke the internal nightly on every target. + set_install_cmd "$WORK/release/install.sh" + if ! "${INSTALL_CMD[@]}" --bin-dir "$install_dir"; then + fail "installer exited non-zero" + exit 1 + fi + + log "Test: installed artifact" + assert_ok "am landed in the requested bin dir" test -x "${install_dir}/am" + + local reported + reported="$("${install_dir}/am" --version 2>/dev/null || true)" + assert_ok "am --version reports the release's version (${expected_version})" \ + am_version_output_matches "$reported" "$expected_version" + + local help_first_line + help_first_line="$("${install_dir}/am" --help 2>/dev/null | head -n1 || true)" + assert_ok "am --help exits 0" quietly "${install_dir}/am" --help + assert_ok "am --help identifies AtomicMemory" \ + grep -qi atomicmemory <<<"$help_first_line" + + # --version and --help are answered by the arg parser. This runs a real + # command path — config load, environment resolution, output — on a $HOME + # with no prior state, which is the first thing a new install ever does. + log "Test: a real subcommand runs on an unconfigured HOME" + assert_ok "am config env show succeeds with no existing config" \ + quietly "${install_dir}/am" config env show + + log "Test: PATH activation" + local env_file="${sandbox_home}/.atomicmemory/env" + assert_ok "installer wrote ~/.atomicmemory/env" test -f "$env_file" + local resolved + resolved="$(PATH="$outer_path" sh -c '. "$1" >/dev/null 2>&1; command -v am' _ "$env_file" || true)" + assert_ok "sourcing the env file puts the installed am first on PATH" \ + test "$resolved" = "${install_dir}/am" + + report "am ${expected_version}" +} + +# Shared by both modes so neither can drift into reporting a pass it did not +# earn. Exits non-zero on any failure — a smoke that returns 0 with failures +# recorded is worse than no smoke. +report() { + local subject="$1" + echo "" + log "=========================================" + if [[ $failed -eq 0 ]]; then + log " ALL PASSED: $passed/$total checks (${subject}, $(uname -s)/$(uname -m))" + else + fail " FAILED: $failed/$total checks" + fi + log "=========================================" + [[ $failed -eq 0 ]] || exit 1 +} + +# Sourcing this file (scripts/__tests__/cli-install-smoke.test.sh) must not run +# the smoke — the tests exercise the guard functions on their own. +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + validate_config || exit $? + if [[ "$AM_SMOKE_MODE" == "provenance" ]]; then + provenance_main "$@" + else + main "$@" + fi +fi diff --git a/scripts/install-cli-internal.sh b/scripts/install-cli-internal.sh index 8095e21..1c7abcd 100755 --- a/scripts/install-cli-internal.sh +++ b/scripts/install-cli-internal.sh @@ -12,8 +12,19 @@ # --dir "$tmp" \ # && sh "$tmp/install.sh" # +# Canary (dev tip) channel: +# tmp="$(mktemp -d)" && \ +# AM_INTERNAL_TAG=cli-canary-latest gh release download cli-canary-latest \ +# --repo atomicstrata/atomicmemory-internal \ +# --pattern install.sh \ +# --dir "$tmp" \ +# && AM_INTERNAL_TAG=cli-canary-latest sh "$tmp/install.sh" +# # Optional: +# AM_INTERNAL_TAG=cli-internal-latest floating main/internal (default) +# AM_INTERNAL_TAG=cli-canary-latest floating dev/canary # AM_INTERNAL_TAG=cli-internal- pin a specific internal release +# AM_INTERNAL_TAG=cli-canary- pin a specific canary release # AM_INTERNAL_REPO=owner/repo override source repo (tests) # AM_INSTALL_DIR / --bin-dir same as scripts/install-cli.sh set -eu @@ -36,7 +47,7 @@ have tar || err "need tar on PATH" case "$AM_INTERNAL_TAG" in cli-v* | v[0-9]* | [0-9]*.[0-9]*.[0-9]*) - err "refusing public release tag '${AM_INTERNAL_TAG}'; use cli-internal-latest or cli-internal-" + err "refusing public release tag '${AM_INTERNAL_TAG}'; use cli-internal-latest, cli-canary-latest, or cli-*-" ;; esac @@ -44,7 +55,7 @@ TMP="$(mktemp -d "${TMPDIR:-/tmp}/am-internal.XXXXXX")" || err "mktemp failed" cleanup() { rm -rf "$TMP"; } trap cleanup EXIT INT TERM -info "info: downloading internal release ${AM_INTERNAL_TAG} from ${AM_INTERNAL_REPO}" +info "info: downloading release ${AM_INTERNAL_TAG} from ${AM_INTERNAL_REPO}" gh release download "$AM_INTERNAL_TAG" \ --repo "$AM_INTERNAL_REPO" \ --dir "$TMP" \ @@ -80,5 +91,5 @@ export AM_BASE_URL="file://${TMP}/mirror" export AM_VERIFY_ATTESTATION=0 export AM_VERSION -info "info: installing am ${AM_VERSION} from internal channel (${AM_INTERNAL_TAG})" +info "info: installing am ${AM_VERSION} from private channel (${AM_INTERNAL_TAG})" sh "$installer" "$@" diff --git a/scripts/install-cli.sh b/scripts/install-cli.sh index f9fcd31..13ce7a0 100755 --- a/scripts/install-cli.sh +++ b/scripts/install-cli.sh @@ -43,6 +43,11 @@ is_our_am() { ver="$("$cmd" --version 2>/dev/null || true)" case "$ver" in am\ [0-9]* | atomicmemory\ [0-9]*) return 0 ;; + \{*) + # ATO-1844 machine-readable contract + printf '%s' "$ver" | grep -q '"surface"[[:space:]]*:[[:space:]]*"cli"' || return 1 + return 0 + ;; esac return 1 } @@ -103,8 +108,16 @@ assert_am_version() { bin="$1" expected_ver="$2" got="$("$bin" --version 2>/dev/null || true)" - expected="am ${expected_ver}" - [ "$got" = "$expected" ] + case "$got" in + \{*) + got_ver="$(printf '%s' "$got" | sed -n 's/.*"version"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' | head -n1)" + [ "$got_ver" = "$expected_ver" ] + ;; + *) + # Legacy banner during transitional installs + [ "$got" = "am ${expected_ver}" ] + ;; + esac } validate_version_string() { diff --git a/scripts/security/security-compliance.mjs b/scripts/security/security-compliance.mjs index 76ffffa..db557a2 100644 --- a/scripts/security/security-compliance.mjs +++ b/scripts/security/security-compliance.mjs @@ -23,7 +23,12 @@ const MIRROR_CLI_WORKFLOW = ".github/workflows/mirror-cli-r2.yml"; const PUBLISH_PACKAGES_WORKFLOW = ".github/workflows/publish-packages.yml"; const PUBLISH_CORE_DOCKER_WORKFLOW = ".github/workflows/publish-core-docker.yml"; const INTERNAL_CORE_DOCKER_WORKFLOW = ".github/workflows/internal-core-docker-image.yml"; +const CORE_ECR_DEV_STAGING_WORKFLOW = ".github/workflows/core-ecr-dev-staging.yml"; +const CLI_INSTALL_SMOKE_WORKFLOW = ".github/workflows/cli-install-smoke.yml"; +const CLI_PUBLIC_INSTALL_SMOKE_WORKFLOW = + ".github/workflows/cli-public-install-smoke.yml"; const RELEASE_PUBLISH_JOB = "publish"; +const CLI_INSTALL_SMOKE_REPORT_JOB = "report"; const READ_ONLY_WORKFLOW_PERMISSIONS = { contents: "read" }; const RELEASE_PUBLISH_PERMISSIONS = { contents: "write", @@ -42,6 +47,18 @@ const GHCR_PUBLISH_PERMISSIONS = { packages: "write", }; const DOCKER_PUBLISH_WORKFLOW_PERMISSIONS = GHCR_PUBLISH_PERMISSIONS; +// ECR Dev/Staging publisher: OIDC to AWS only (no GHCR packages:write). +const ECR_OIDC_PUBLISH_PERMISSIONS = { + contents: "read", + "id-token": "write", +}; +// The first non-release writer in this table. It files no artifact and +// touches no registry: it opens and closes one issue so a scheduled smoke +// failure reaches a person. Narrowest scope that can do that. +const NIGHTLY_REPORT_PERMISSIONS = { + contents: "read", + issues: "write", +}; // Single data-driven allow-table for every workflow that legitimately holds // write scopes. Anything not listed here must be read-only at both the @@ -91,6 +108,32 @@ const RELEASE_LANE_ALLOW_TABLE = new Map([ jobs: {}, }, ], + [ + CORE_ECR_DEV_STAGING_WORKFLOW, + { + workflow: ECR_OIDC_PUBLISH_PERMISSIONS, + jobs: {}, + }, + ], + [ + CLI_INSTALL_SMOKE_WORKFLOW, + { + workflow: READ_ONLY_WORKFLOW_PERMISSIONS, + jobs: { [CLI_INSTALL_SMOKE_REPORT_JOB]: NIGHTLY_REPORT_PERMISSIONS }, + }, + ], + // Same shape as the internal smoke's reporter and for the same reason: it + // opens and closes one issue so a scheduled failure reaches a person. The + // public lane needs its own entry rather than sharing one, because the table + // is keyed by file and an unlisted workflow is required to be read-only — + // which is exactly how this entry came to be written (ATO-1863). + [ + CLI_PUBLIC_INSTALL_SMOKE_WORKFLOW, + { + workflow: READ_ONLY_WORKFLOW_PERMISSIONS, + jobs: { [CLI_INSTALL_SMOKE_REPORT_JOB]: NIGHTLY_REPORT_PERMISSIONS }, + }, + ], ]); function checkSecrets() { @@ -120,6 +163,7 @@ function validateWorkflowFile(filePath) { return [ ...validateWorkflowPermissions(filePath, text), ...validateMirrorCliPromotionGuard(filePath, text), + ...validateReleaseCliVersionBumpGuard(filePath, text), ...validateWorkflowActions(filePath, lines), ]; } @@ -261,6 +305,30 @@ export function validateMirrorCliPromotionGuard(filePath, source) { return [`${filePath}: must compare requested version against current version.json before promoting latest`]; } +/** + * release-cli.yml must validate adjacent public semver bumps before building + * release artifacts, so a mistagged cli-v* cannot bypass Cargo.toml alignment. + */ +export function validateReleaseCliVersionBumpGuard(filePath, source) { + if (filePath !== RELEASE_CLI_WORKFLOW) { + return []; + } + + const text = workflowSourceText(source); + const required = [ + /Validate public version bump/, + /scripts\/ci\/validate-cli-version-bump\.sh/, + /RELEASE_MODE=1/, + /PROPOSED_VERSION="\$VERSION"/, + ]; + + if (required.every((pattern) => pattern.test(text))) { + return []; + } + + return [`${filePath}: must invoke scripts/ci/validate-cli-version-bump.sh with RELEASE_MODE before building`]; +} + function validateWorkflowActions(filePath, lines) { return lines.flatMap((line, index) => { const match = line.match(ACTION_REF_PATTERN);