diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 0000000..473b6eb --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,3 @@ +self-hosted-runner: + labels: + - openadapt-capture-qualified diff --git a/.github/workflows/production-qualification.yml b/.github/workflows/production-qualification.yml new file mode 100644 index 0000000..b977cf4 --- /dev/null +++ b/.github/workflows/production-qualification.yml @@ -0,0 +1,379 @@ +name: Production qualification + +on: + workflow_dispatch: + inputs: + candidate_sha: + description: "Exact 40-character main commit SHA to qualify" + type: string + required: true + +concurrency: + group: capture-production-qualification-${{ github.sha }} + cancel-in-progress: false + +permissions: + contents: read + +jobs: + build-candidate: + name: Build candidate distributions + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Checkout the dispatched commit + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + + - name: Require the exact protected-main candidate + shell: bash + env: + CANDIDATE_SHA: ${{ inputs.candidate_sha }} + run: | + set -euo pipefail + if [[ ! "${CANDIDATE_SHA}" =~ ^[0-9a-f]{40}$ ]]; then + echo "candidate_sha must be a lowercase 40-character Git commit SHA" + exit 1 + fi + if [ "${GITHUB_REF}" != "refs/heads/main" ]; then + echo "Production qualification must be dispatched on protected main." + exit 1 + fi + if [ "${CANDIDATE_SHA}" != "${GITHUB_SHA}" ]; then + echo "Input candidate ${CANDIDATE_SHA} differs from dispatched commit ${GITHUB_SHA}." + exit 1 + fi + if [ "$(git rev-parse HEAD)" != "${GITHUB_SHA}" ]; then + echo "Checkout does not match the dispatched commit." + exit 1 + fi + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + + - name: Install exact uv + uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 + with: + version: "0.11.29" + + - name: Build the candidate once + shell: bash + run: | + set -euo pipefail + uv build --wheel --sdist + python scripts/verify_distribution.py dist/* + python scripts/check_source_boundary.py --require-dist + ( + cd dist + sha256sum -- *.whl *.tar.gz > SHA256SUMS + ) + + - name: Upload the exact candidate + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: capture-candidate-${{ github.sha }} + path: dist/ + if-no-files-found: error + retention-days: 7 + + clean-wheel: + name: Clean candidate wheel (${{ matrix.os }}) + needs: build-candidate + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + runs-on: ${{ matrix.os }} + timeout-minutes: 20 + steps: + - name: Checkout the exact candidate source + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + + - name: Download the exact candidate + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: capture-candidate-${{ github.sha }} + path: dist + + - name: Run the clean install and uninstall lifecycle + shell: bash + run: | + set -euo pipefail + python scripts/candidate_lifecycle.py \ + --dist dist \ + --manifest dist/SHA256SUMS \ + --candidate-sha "${GITHUB_SHA}" \ + --output "evidence/clean-${{ matrix.os }}.json" + + - name: Upload clean-machine evidence + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: capture-clean-${{ matrix.os }}-${{ github.sha }} + path: evidence/ + if-no-files-found: error + retention-days: 14 + + interactive-linux: + name: Interactive qualification (Linux X64) + needs: build-candidate + environment: production-qualification + runs-on: [self-hosted, Linux, X64, openadapt-capture-qualified] + timeout-minutes: 35 + steps: + - name: Checkout the exact candidate source + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + + - name: Install exact uv + uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 + with: + version: "0.11.29" + + - name: Download the exact candidate + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: capture-candidate-${{ github.sha }} + path: dist + + - name: Install the exact wheel into an isolated environment + shell: bash + run: | + set -euo pipefail + qualification_root="${RUNNER_TEMP}/capture-qualification-${GITHUB_RUN_ID}-${GITHUB_JOB}" + qualification_python="${qualification_root}/bin/python" + uv venv --clear --python 3.12 "${qualification_root}" + uv pip install --python "${qualification_python}" \ + dist/*.whl pytest==9.1.1 pytest-timeout==2.4.0 pynput==1.8.2 + echo "QUALIFICATION_PYTHON=${qualification_python}" >> "${GITHUB_ENV}" + + - name: Require the reviewed external video tools + shell: bash + run: | + set -euo pipefail + mkdir -p evidence + command -v ffmpeg + command -v ffprobe + ffmpeg -version > evidence/ffmpeg-version.txt + ffprobe -version > evidence/ffprobe-version.txt + + - name: Require a stable multiple-monitor desktop + shell: bash + run: | + set -euo pipefail + "${QUALIFICATION_PYTHON}" scripts/check_display_topology.py \ + --minimum-monitors 2 \ + --output evidence/display-topology.json + + - name: Run the complete live recorder qualification + shell: bash + env: + OPENADAPT_CAPTURE_PRODUCTION_QUALIFICATION: "1" + run: | + set -euo pipefail + cd "${RUNNER_TEMP}" + "${QUALIFICATION_PYTHON}" -m pytest \ + "${GITHUB_WORKSPACE}/tests/test_performance.py" \ + -m slow -v --timeout=300 --import-mode=importlib \ + --junitxml="${GITHUB_WORKSPACE}/evidence/interactive-linux.xml" + + - name: Reject skipped or incomplete qualification tests + shell: bash + run: | + set -euo pipefail + python scripts/check_junit_no_skips.py evidence/interactive-linux.xml + + - name: Upload interactive qualification evidence + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: capture-interactive-linux-${{ github.sha }} + path: evidence/ + if-no-files-found: error + retention-days: 30 + + interactive-macos: + name: Interactive qualification (macOS ARM64) + needs: build-candidate + environment: production-qualification + runs-on: [self-hosted, macOS, ARM64, openadapt-capture-qualified] + timeout-minutes: 35 + steps: + - name: Checkout the exact candidate source + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + + - name: Install exact uv + uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 + with: + version: "0.11.29" + + - name: Download the exact candidate + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: capture-candidate-${{ github.sha }} + path: dist + + - name: Install the exact wheel into an isolated environment + shell: bash + run: | + set -euo pipefail + qualification_root="${RUNNER_TEMP}/capture-qualification-${GITHUB_RUN_ID}-${GITHUB_JOB}" + qualification_python="${qualification_root}/bin/python" + uv venv --clear --python 3.12 "${qualification_root}" + uv pip install --python "${qualification_python}" \ + dist/*.whl pytest==9.1.1 pytest-timeout==2.4.0 pynput==1.8.2 + echo "QUALIFICATION_PYTHON=${qualification_python}" >> "${GITHUB_ENV}" + + - name: Require the reviewed external video tools + shell: bash + run: | + set -euo pipefail + mkdir -p evidence + command -v ffmpeg + command -v ffprobe + ffmpeg -version > evidence/ffmpeg-version.txt + ffprobe -version > evidence/ffprobe-version.txt + + - name: Require a stable multiple-monitor desktop + shell: bash + run: | + set -euo pipefail + "${QUALIFICATION_PYTHON}" scripts/check_display_topology.py \ + --minimum-monitors 2 \ + --output evidence/display-topology.json + + - name: Run the complete live recorder and window qualification + shell: bash + env: + OPENADAPT_CAPTURE_PRODUCTION_QUALIFICATION: "1" + run: | + set -euo pipefail + cd "${RUNNER_TEMP}" + "${QUALIFICATION_PYTHON}" -m pytest \ + "${GITHUB_WORKSPACE}/tests/test_performance.py" \ + "${GITHUB_WORKSPACE}/tests/test_window_capture.py" \ + -m slow -v --timeout=300 --import-mode=importlib \ + --junitxml="${GITHUB_WORKSPACE}/evidence/interactive-macos.xml" + + - name: Reject skipped or incomplete qualification tests + shell: bash + run: | + set -euo pipefail + python scripts/check_junit_no_skips.py evidence/interactive-macos.xml + + - name: Upload interactive qualification evidence + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: capture-interactive-macos-${{ github.sha }} + path: evidence/ + if-no-files-found: error + retention-days: 30 + + interactive-windows: + name: Interactive qualification (Windows X64) + needs: build-candidate + environment: production-qualification + runs-on: [self-hosted, Windows, X64, openadapt-capture-qualified] + timeout-minutes: 35 + steps: + - name: Checkout the exact candidate source + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + + - name: Install exact uv + uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 + with: + version: "0.11.29" + + - name: Download the exact candidate + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: capture-candidate-${{ github.sha }} + path: dist + + - name: Install the exact wheel into an isolated environment + shell: pwsh + run: | + $ErrorActionPreference = "Stop" + $qualificationRoot = Join-Path $env:RUNNER_TEMP "capture-qualification-$env:GITHUB_RUN_ID-$env:GITHUB_JOB" + $qualificationPython = Join-Path $qualificationRoot "Scripts\python.exe" + uv venv --clear --python 3.12 $qualificationRoot + $wheel = (Get-ChildItem "dist\*.whl" -File -ErrorAction Stop).FullName + if ($wheel.Count -ne 1) { throw "Expected exactly one candidate wheel." } + uv pip install --python $qualificationPython $wheel ` + "pytest==9.1.1" "pytest-timeout==2.4.0" "pynput==1.8.2" + "QUALIFICATION_PYTHON=$qualificationPython" | Out-File ` + -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + + - name: Require the reviewed external video tools + shell: pwsh + run: | + $ErrorActionPreference = "Stop" + New-Item -ItemType Directory -Force -Path evidence | Out-Null + Get-Command ffmpeg -ErrorAction Stop | Out-Null + Get-Command ffprobe -ErrorAction Stop | Out-Null + ffmpeg -version | Out-File evidence/ffmpeg-version.txt -Encoding utf8 + ffprobe -version | Out-File evidence/ffprobe-version.txt -Encoding utf8 + + - name: Require a stable multiple-monitor desktop + shell: pwsh + run: | + $ErrorActionPreference = "Stop" + & $env:QUALIFICATION_PYTHON scripts/check_display_topology.py ` + --minimum-monitors 2 ` + --output evidence/display-topology.json + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + + - name: Run the complete live recorder and window qualification + shell: pwsh + env: + OPENADAPT_CAPTURE_PRODUCTION_QUALIFICATION: "1" + run: | + $ErrorActionPreference = "Stop" + Push-Location $env:RUNNER_TEMP + try { + & $env:QUALIFICATION_PYTHON -m pytest ` + "$env:GITHUB_WORKSPACE/tests/test_performance.py" ` + "$env:GITHUB_WORKSPACE/tests/test_window_capture.py" ` + -m slow -v --timeout=300 --import-mode=importlib ` + "--junitxml=$env:GITHUB_WORKSPACE/evidence/interactive-windows.xml" + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + } finally { + Pop-Location + } + + - name: Reject skipped or incomplete qualification tests + shell: pwsh + run: | + $ErrorActionPreference = "Stop" + python scripts/check_junit_no_skips.py evidence/interactive-windows.xml + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + + - name: Upload interactive qualification evidence + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: capture-interactive-windows-${{ github.sha }} + path: evidence/ + if-no-files-found: error + retention-days: 30 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6af5c36..05e6902 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -55,50 +55,17 @@ jobs: echo "skip=true" >> "${GITHUB_OUTPUT}" fi - - name: Wait for exact-head Tests workflow + - name: Wait for exact-head test and production qualification evidence if: steps.check_skip.outputs.skip != 'true' env: GH_TOKEN: ${{ github.token }} shell: bash run: | set -euo pipefail - deadline=$((SECONDS + 2700)) - while [ "${SECONDS}" -lt "${deadline}" ]; do - response="$( - gh api --method GET \ - "repos/${GITHUB_REPOSITORY}/actions/workflows/test.yml/runs" \ - --raw-field head_sha="${GITHUB_SHA}" \ - --raw-field event=push \ - --raw-field per_page=20 - )" - status="$( - printf '%s' "${response}" | - jq -r --arg sha "${GITHUB_SHA}" \ - '[.workflow_runs[] - | select(.head_sha == $sha and .event == "push")] - | sort_by(.created_at) | last | .status // "missing"' - )" - conclusion="$( - printf '%s' "${response}" | - jq -r --arg sha "${GITHUB_SHA}" \ - '[.workflow_runs[] - | select(.head_sha == $sha and .event == "push")] - | sort_by(.created_at) | last | .conclusion // "pending"' - )" - echo "test.yml for ${GITHUB_SHA}: ${status}/${conclusion}" - if [ "${status}" = "completed" ] && [ "${conclusion}" != "success" ]; then - echo "Refusing to publish because test.yml concluded ${conclusion}." - exit 1 - fi - if [ "${status}" = "completed" ] && [ "${conclusion}" = "success" ]; then - break - fi - sleep 10 - done - if [ "${status}" != "completed" ] || [ "${conclusion}" != "success" ]; then - echo "Refusing to publish: exact-head test.yml did not succeed within 45 minutes." - exit 1 - fi + python scripts/check_release_ci.py \ + --repository "${GITHUB_REPOSITORY}" \ + --sha "${GITHUB_SHA}" \ + --timeout-seconds 2700 - name: Require dispatched head to remain current protected main if: steps.check_skip.outputs.skip != 'true' diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 934dd3d..71534c2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -162,6 +162,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v7 + with: + fetch-depth: 0 - name: Install uv uses: astral-sh/setup-uv@v7 @@ -171,6 +173,9 @@ jobs: - name: Set up Python run: uv python install 3.12 + - name: Verify maintained changelog + run: python scripts/check_changelog.py + - name: Build wheel and source distribution run: uv build diff --git a/CHANGELOG.md b/CHANGELOG.md index f165d33..d42b23c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,232 @@ # CHANGELOG + + + +## v1.2.2 (2026-07-28) + +_This release is published under the MIT License._ + +### Bug Fixes + +- Stop fabricating display metrics, window data, and UIA evidence + ([#61](https://github.com/OpenAdaptAI/openadapt-capture/pull/61), + [`e9e2d82`](https://github.com/OpenAdaptAI/openadapt-capture/commit/e9e2d8247e4cd56c7fc9aa250046852f0233912a)) + +### Build System + +- Keep repository clutter out of source archive + ([#59](https://github.com/OpenAdaptAI/openadapt-capture/pull/59), + [`0137ecb`](https://github.com/OpenAdaptAI/openadapt-capture/commit/0137ecb1572c20b95ac379a423bc200120271a26)) + +### Chores + +- Gitignore `.private/` + ([#60](https://github.com/OpenAdaptAI/openadapt-capture/pull/60), + [`c0ee3e8`](https://github.com/OpenAdaptAI/openadapt-capture/commit/c0ee3e86be97f47e164c323d2017c9688aa51d19)) + +### Continuous Integration + +- Detect unreleased work and silently skipped publishes + ([#57](https://github.com/OpenAdaptAI/openadapt-capture/pull/57), + [`fc549ed`](https://github.com/OpenAdaptAI/openadapt-capture/commit/fc549ed47b3185bc804e725fa2972cd7b2673134)) +- Simplify and harden release health + ([#58](https://github.com/OpenAdaptAI/openadapt-capture/pull/58), + [`191a352`](https://github.com/OpenAdaptAI/openadapt-capture/commit/191a3523d2a4bff5564377aaf5de72d25b2d84e8)) + +**Detailed Changes**: [v1.2.1...v1.2.2](https://github.com/OpenAdaptAI/openadapt-capture/compare/v1.2.1...v1.2.2) + + +## v1.2.1 (2026-07-27) + +_This release is published under the MIT License._ + +### Bug Fixes + +- **audio**: Make narration capture on-device only and fail closed + ([#51](https://github.com/OpenAdaptAI/openadapt-capture/pull/51), + [`27ccd26`](https://github.com/OpenAdaptAI/openadapt-capture/commit/27ccd2625ddb6d76f81d4d53c52793366aafdddb)) + +### Chores + +- **deps**: Update release and CodeQL actions + ([#56](https://github.com/OpenAdaptAI/openadapt-capture/pull/56), + [`22ca9cf`](https://github.com/OpenAdaptAI/openadapt-capture/commit/22ca9cfb36174e28dd18a5b62500a51ad4e34be6)) + +### Continuous Integration + +- Run capture platform integration on exact `main` + ([#50](https://github.com/OpenAdaptAI/openadapt-capture/pull/50), + [`f54346c`](https://github.com/OpenAdaptAI/openadapt-capture/commit/f54346c4f40b018e302d1edef8082829251d6b7f)) + +### Documentation + +- Note the PyAV/GPL packaging boundary for the `transcribe-fast` extra + ([#51](https://github.com/OpenAdaptAI/openadapt-capture/pull/51), + [`27ccd26`](https://github.com/OpenAdaptAI/openadapt-capture/commit/27ccd2625ddb6d76f81d4d53c52793366aafdddb)) + +**Detailed Changes**: [v1.2.0...v1.2.1](https://github.com/OpenAdaptAI/openadapt-capture/compare/v1.2.0...v1.2.1) + + +## v1.2.0 (2026-07-26) + +_This release is published under the MIT License._ + +### Bug Fixes + +- Bound native structural evidence + ([#48](https://github.com/OpenAdaptAI/openadapt-capture/pull/48), + [`1547f83`](https://github.com/OpenAdaptAI/openadapt-capture/commit/1547f83b01d2e087dcc0f41462e5be8d5b521b82)) + +### Features + +- Preserve keyboard shortcuts in Capture events + ([#49](https://github.com/OpenAdaptAI/openadapt-capture/pull/49), + [`3720cea`](https://github.com/OpenAdaptAI/openadapt-capture/commit/3720cea5dad937100c98211213330302cc367664)) + +**Detailed Changes**: [v1.1.1...v1.2.0](https://github.com/OpenAdaptAI/openadapt-capture/compare/v1.1.1...v1.2.0) + + +## v1.1.1 (2026-07-25) + +_This release is published under the MIT License._ + +### Bug Fixes + +- Preserve observer setup failure at readiness timeout + ([#47](https://github.com/OpenAdaptAI/openadapt-capture/pull/47), + [`1b2cbd3`](https://github.com/OpenAdaptAI/openadapt-capture/commit/1b2cbd38227ea67fd2ab9e618123e091039d7f89)) +- Preserve readiness timeout on startup cancellation + ([#47](https://github.com/OpenAdaptAI/openadapt-capture/pull/47), + [`1b2cbd3`](https://github.com/OpenAdaptAI/openadapt-capture/commit/1b2cbd38227ea67fd2ab9e618123e091039d7f89)) +- Preserve setup failure at observer timeout + ([#47](https://github.com/OpenAdaptAI/openadapt-capture/pull/47), + [`1b2cbd3`](https://github.com/OpenAdaptAI/openadapt-capture/commit/1b2cbd38227ea67fd2ab9e618123e091039d7f89)) + +**Detailed Changes**: [v1.1.0...v1.1.1](https://github.com/OpenAdaptAI/openadapt-capture/compare/v1.1.0...v1.1.1) + + +## v1.1.0 (2026-07-25) + +_This release is published under the MIT License._ + +### Bug Fixes + +- Enumerate UIA candidates with supported filters + ([#45](https://github.com/OpenAdaptAI/openadapt-capture/pull/45), + [`ba08f0b`](https://github.com/OpenAdaptAI/openadapt-capture/commit/ba08f0b6da322378c165b699129487d41d91334f)) +- Make Windows UIA capture use real APIs + ([#45](https://github.com/OpenAdaptAI/openadapt-capture/pull/45), + [`ba08f0b`](https://github.com/OpenAdaptAI/openadapt-capture/commit/ba08f0b6da322378c165b699129487d41d91334f)) + +### Chores + +- Remove the generated viewer artifact + ([#44](https://github.com/OpenAdaptAI/openadapt-capture/pull/44), + [`031b378`](https://github.com/OpenAdaptAI/openadapt-capture/commit/031b37880f3f09ce91578130cc1a8c4f9dc03075)) + +### Features + +- Capture Windows UIA structural evidence + ([#45](https://github.com/OpenAdaptAI/openadapt-capture/pull/45), + [`ba08f0b`](https://github.com/OpenAdaptAI/openadapt-capture/commit/ba08f0b6da322378c165b699129487d41d91334f)) + +### Testing + +- Exercise real Windows UIA observation + ([#45](https://github.com/OpenAdaptAI/openadapt-capture/pull/45), + [`ba08f0b`](https://github.com/OpenAdaptAI/openadapt-capture/commit/ba08f0b6da322378c165b699129487d41d91334f)) + +**Detailed Changes**: [v1.0.4...v1.1.0](https://github.com/OpenAdaptAI/openadapt-capture/compare/v1.0.4...v1.1.0) + + +## v1.0.4 (2026-07-24) + +_This release is published under the MIT License._ + +### Bug Fixes + +- Report FFmpeg input worker startup failure + ([#43](https://github.com/OpenAdaptAI/openadapt-capture/pull/43), + [`c11969b`](https://github.com/OpenAdaptAI/openadapt-capture/commit/c11969b7efd2eaaf9981d853fbe62d2bdaddd953)) + +**Detailed Changes**: [v1.0.3...v1.0.4](https://github.com/OpenAdaptAI/openadapt-capture/compare/v1.0.3...v1.0.4) + + +## v1.0.3 (2026-07-23) + +_This release is published under the MIT License._ + +### Bug Fixes + +- Isolate video codec runtime from capture + ([#41](https://github.com/OpenAdaptAI/openadapt-capture/pull/41), + [`dd84934`](https://github.com/OpenAdaptAI/openadapt-capture/commit/dd849344e5b1687b78fc85242912635a08930b4a)) +- Propagate video writer contracts and failures + ([#41](https://github.com/OpenAdaptAI/openadapt-capture/pull/41), + [`dd84934`](https://github.com/OpenAdaptAI/openadapt-capture/commit/dd849344e5b1687b78fc85242912635a08930b4a)) + +**Detailed Changes**: [v1.0.2...v1.0.3](https://github.com/OpenAdaptAI/openadapt-capture/compare/v1.0.2...v1.0.3) + + +## v1.0.2 (2026-07-23) + +_This release is published under the MIT License._ + +### Bug Fixes + +- Replace copyleft input dependencies + ([`f0af5a8`](https://github.com/OpenAdaptAI/openadapt-capture/commit/f0af5a8b13ccc7cdb02e8309f0b0f8ff9fece914)) + +**Detailed Changes**: [v1.0.1...v1.0.2](https://github.com/OpenAdaptAI/openadapt-capture/compare/v1.0.1...v1.0.2) + + +## v1.0.1 (2026-07-23) + +_This release is published under the MIT License._ + +### Bug Fixes + +- Dispatch docs updates to canonical repository + ([#38](https://github.com/OpenAdaptAI/openadapt-capture/pull/38), + [`b65d3ad`](https://github.com/OpenAdaptAI/openadapt-capture/commit/b65d3adc280cc98c9855cdc31f8bf1accde38e17)) + +### Documentation + +- Refresh README to shared OpenAdapt house style + ([#37](https://github.com/OpenAdaptAI/openadapt-capture/pull/37), + [`da30acd`](https://github.com/OpenAdaptAI/openadapt-capture/commit/da30acddd7ac52617d45ec09945046e84f0295de)) + +**Detailed Changes**: [v1.0.0...v1.0.1](https://github.com/OpenAdaptAI/openadapt-capture/compare/v1.0.0...v1.0.1) + + +## v1.0.0 (2026-07-21) + +_This release is published under the MIT License._ + +### Chores + +- Add security CI for CodeQL, Gitleaks, dependency review, and Dependabot + ([#31](https://github.com/OpenAdaptAI/openadapt-capture/pull/31), + [`ea786c4`](https://github.com/OpenAdaptAI/openadapt-capture/commit/ea786c45e550ff29bb2ce5caaf67811120506fdb)) +- **deps**: Bump `actions/checkout` from 4 to 7 + ([#32](https://github.com/OpenAdaptAI/openadapt-capture/pull/32), + [`3b5d694`](https://github.com/OpenAdaptAI/openadapt-capture/commit/3b5d694567fffd883094423e40a3be3e92d27b00)) +- **deps**: Bump `actions/dependency-review-action` from 4.9.0 to 5.0.0 + ([#33](https://github.com/OpenAdaptAI/openadapt-capture/pull/33), + [`100f8cf`](https://github.com/OpenAdaptAI/openadapt-capture/commit/100f8cf13701317a1070714ad8198b09027223cd)) +- **deps**: Bump `astral-sh/setup-uv` from 4 to 7 + ([#35](https://github.com/OpenAdaptAI/openadapt-capture/pull/35), + [`2829be5`](https://github.com/OpenAdaptAI/openadapt-capture/commit/2829be5ccc99fb8060f297e04c24e6d7ada09eba)) +- **deps**: Bump `peter-evans/repository-dispatch` from 3 to 4 + ([#36](https://github.com/OpenAdaptAI/openadapt-capture/pull/36), + [`52c6a4f`](https://github.com/OpenAdaptAI/openadapt-capture/commit/52c6a4f62448598e549827086a9461b6bec86fba)) +- **deps**: Bump `python-semantic-release/python-semantic-release` + ([#34](https://github.com/OpenAdaptAI/openadapt-capture/pull/34), + [`4b1f42b`](https://github.com/OpenAdaptAI/openadapt-capture/commit/4b1f42b9eb879cc045244c738d04613adc4878e3)) + +**Detailed Changes**: [v0.6.0...v1.0.0](https://github.com/OpenAdaptAI/openadapt-capture/compare/v0.6.0...v1.0.0) + ## v0.6.0 (2026-07-18) diff --git a/README.md b/README.md index a0ea812..fb2c06c 100644 --- a/README.md +++ b/README.md @@ -68,14 +68,23 @@ Documentation for the whole stack lives at | Windows, macOS, and Linux demonstrations | `openadapt-capture` records native input and action-gated screen video; Windows can also retain action-time UI Automation evidence. `openadapt-flow` converts the session into compiler input. | | RDP and Citrix/VDI demonstrations | `openadapt-capture` records the selected client window in its own pixel space. The remote application remains externally black-box, and `openadapt-flow` converts the session into compiler input. | | Browser demonstrations | `openadapt-flow` uses its Playwright recorder. It can launch Chromium or attach to one existing signed-in local Chromium tab. It does not require this package. | -| Chrome extension in this repository | Prototype alternate acquisition transport. It is not the supported recorder and must not perform direct replay. | - -The browser path stays inside `openadapt-flow` because the compiler needs -ordered before/after frames, page state, secret-field redaction, and one bound -event schema. Flow now supports an existing authenticated browser session -through its local-loopback CDP attach mode. The extension can become another -acquisition transport after it emits that same evidence contract. It must not -create a second compiler format or bypass governed replay. +| Chrome extension in this repository | Repository-only prototype. Its bridge and legacy direct replay are excluded from the wheel and source archive. It is not the supported recorder. | + +The supported browser path stays inside `openadapt-flow`. Playwright owns the +browser context and can bind DOM identity, field geometry, ordered before/after +frames, and source-time secret redaction to one recording contract. A Chrome +extension cannot guarantee that contract across browser profiles, extension +permissions, browser-internal pages, and process or tab disconnects. + +The extension remains useful as a research observer and as a possible future +source of optional DOM evidence. It should become a supported auxiliary +observer only after it emits the shared event schema, has a fail-closed +connection and permission contract, redacts secret fields before persistence, +and passes the same compiler qualification as the Playwright path. It should +not replace the Playwright recorder merely to make the package layout uniform, +create a second compiler format, or bypass governed replay. Flow supports an +existing authenticated browser session through its local-loopback CDP attach +mode. ## Use it with OpenAdapt @@ -182,12 +191,11 @@ retain window-scoped pixels and coordinates for Flow's remote visual compiler. ## Window-scoped recording -**Status: implemented and unit-proven on all CI platforms; live-validated -end to end on macOS (frames, translated coordinates, bounds timeline, and -video verified against a real window on a real display). Windows uses a -Win32 + `mss` region grab and is exercised by the same unit suite; its live -smoke test awaits an interactive Windows desktop. Not yet validated against -a Parallels/Citrix client window specifically.** +**Status: implemented, with display-free unit coverage on every +supported operating system.** The production release gate also requires live +window capture, input injection, movement, resize, video verification, and no +skipped tests on interactive macOS and Windows runners. A customer RDP or +Citrix deployment still requires task- and environment-specific qualification. By default the recorder captures the full screen. Window-scoped mode records ONE window in that window's own pixel space. This is the mode built for @@ -224,21 +232,47 @@ In this mode: exact inverse of the replay mapping). Input outside the window records out-of-range coordinates rather than being silently clamped. - **The window scoping is persisted**: the recording's config JSON carries the - target, resolved window, initial bounds, scale, and viewport + target, resolved window, initial bounds, fixed output viewport, current + source viewport, scale-to-fit mapping, and content rectangle (`CaptureSession.window_capture`), and the window is re-resolved every frame with bounds changes recorded as window events, a bounds timeline converters can use to be exact even when the window moves. +- **Window movement and resize are supported.** The first frame fixes the + encoded video size. Later source frames scale to fit and letterbox into that + viewport. Input uses the exact current bounds and content rectangle. No frame + is silently skipped because the window changed size or moved to a display + with a different scale. - **Fail-loud guarantees:** recording refuses to start if the window cannot be resolved and captured; input arriving before the first frame is discarded with a warning instead of being recorded in the wrong coordinate space; a - mid-recording window *resize* skips unencodable video frames loudly - (screenshots and the bounds timeline stay exact), so avoid resizing the - target during a demonstration. + lost window, capture failure, or unexpected output-frame size fails the + session instead of producing complete-looking media with an evidence gap. Note for converters: window-mode coordinates are already in captured-frame pixels (`coordinate_space == "window_pixels"`); do not rescale them by `pixel_ratio`. +## Multiple monitors + +Full-screen mode records the complete virtual desktop reported by MSS, not +only the primary monitor. Capture stores its origin, fixed viewport, monitor +count, and privacy-safe monitor rectangles as +`CaptureSession.desktop_capture`. Global input is translated into this exact +combined-frame coordinate space. This includes a secondary monitor whose +native coordinates have a negative origin. + +The release qualification requires at least two real monitors on each +interactive operating-system runner. It checks the topology and then runs the +native screen and input tests. Downstream converters must not apply the legacy +display-ratio scale when +`coordinate_space == "virtual_desktop_pixels"`. + +The monitor topology is fixed for one recording. Connecting, disconnecting, +rotating, or changing the resolution or scale of a display changes the encoded +frame contract. Capture fails the session if that occurs. Start a new recording +after a display-topology change. This boundary does not restrict movement or +resize of a recorded window across an unchanged monitor layout. + ## Data and privacy boundary A raw capture can contain everything visible on screen and everything typed, @@ -286,13 +320,16 @@ boundary. `openadapt-flow` still refuses desktop `--secret` authoring until its source-time field-redaction contract can prove that sensitive values were not retained. Review the desktop guide before recording sensitive workflows. -The Chrome extension prototype can observe pages across its configured host -permissions and can emit DOM text and keyboard events to a local WebSocket. -It does not yet provide source-time secret exclusion, authenticated +The repository-only Chrome extension prototype can observe pages across its +configured host permissions. Its development bridge can emit DOM text and +keyboard events to an unauthenticated local WebSocket and contains legacy +direct DOM replay. These files are excluded from the package wheel and source +archive. The production Capture API does not export the bridge, and the former +`browser_events=True` opt-in fails before it binds a listener. The prototype +does not provide source-time secret exclusion, authenticated profile/tab/document/session binding, acknowledged ordered delivery, or exact -frame-to-event evidence. Its direct DOM replay does not use Flow's identity, -policy, fresh-frame, and effect checks. Treat it as development code. Do not -deploy it in a sensitive browser profile. +frame-to-event evidence. Treat it as development code. Do not deploy it in a +sensitive browser profile. Use Flow's supported attach recorder when an existing SSO or 2FA browser session is required. See the @@ -308,10 +345,13 @@ session is required. See the text, named keys, modifier chords, and scrolling. It rejects unsupported input such as middle clicks, non-left-button drags, malformed shortcuts, and unmapped keys instead of silently compiling an incomplete workflow. -- Browser-extension installation, security hardening, and compiler integration - are not part of the current product path. Promotion requires the shared Flow - schema, source-time secret exclusion, authenticated and sequenced delivery, - exact frame binding, compiler integration, and removal of direct replay. +- Display hot-plug, rotation, resolution changes, and scale changes require a + new recording because one media stream has one fixed virtual-desktop viewport. +- Browser-extension installation, bridge code, and direct replay are not part + of the published Capture artifacts or supported browser path. Promotion + requires the shared Flow schema, source-time secret exclusion, authenticated + and sequenced delivery, exact frame binding, compiler integration, and + removal of direct replay. See the organization-wide [repository lifecycle registry](https://github.com/OpenAdaptAI/.github/blob/main/REPOSITORY_LIFECYCLE.md) diff --git a/chrome_extension/README.md b/chrome_extension/README.md index 1d0e59b..6793af3 100644 --- a/chrome_extension/README.md +++ b/chrome_extension/README.md @@ -16,6 +16,11 @@ The extension can collect DOM events and visible HTML and send them to the Capture WebSocket bridge on `localhost:8765`. It also contains legacy direct DOM replay code. +The extension and bridge are repository-only development files. The published +wheel and source archive exclude the bridge. The package API does not export +it, and the package does not install its WebSocket dependency. The former +browser-event recording opt-in fails before it binds a listener. + Do not use it in a sensitive browser profile. The current implementation does not provide these supported-path controls: diff --git a/docs/DESIGN.md b/docs/DESIGN.md index 383284d..18b4fdc 100644 --- a/docs/DESIGN.md +++ b/docs/DESIGN.md @@ -1,302 +1,214 @@ -# openadapt-capture Design +# OpenAdapt Capture design -> **Historical design record.** This file retains early goals and proposed -> formats. It is not the current package contract. The current recorder -> writes `recording.db` plus verified time-aligned MP4 media, uses native input -> observers on Windows, macOS, and Linux, retains action-time Windows UIA -> evidence, and supports window-scoped capture on Windows and macOS. See -> [`README.md`](../README.md) and the public API for the current behavior. Items -> such as `capture.db`, chunked continuous capture, and structural observers on -> every operating system remain historical proposals where the code differs. +## Product role + +`openadapt-capture` is the canonical native recorder for OpenAdapt. It records +screen media, native input, timing, window geometry, and optional action-time UI +structure into one local session. `openadapt-flow` consumes the session, applies +the compiler and qualification contracts, and owns governed replay. -## Problem Statement +The package lifecycle is **Experimental**. A successful unit test or a runnable package +does not by itself make a recording path production-qualified. A release must +also pass the exact-commit clean-install and interactive native qualification +described below. -We need a platform-agnostic representation of GUI interactions that: +Capture is local-first. It does not upload a recording. A raw session can +contain screen text, typed secrets, accessibility text, and optional narration. +It remains inside the approved local boundary unless a separate explicit +operation exports it. -1. Can be captured from any OS (macOS, Windows, Linux) -2. Can be scrubbed for privacy (via `openadapt-privacy`) -3. Can be replayed for automation -4. Can be used for ML training (via `openadapt-ml`) -5. Works as a generic library that any system can adapt to +## Supported recording paths -## Design Goals +| Path | Capture contract | +| --- | --- | +| Full virtual desktop | Capture the combined MSS desktop and translate global input into that exact pixel space. | +| One native window | Re-resolve and capture the selected window on each frame; translate input into a fixed encoded viewport. | +| RDP or Citrix client window | Use the native window path. Treat the remote application as pixels; local accessibility APIs do not cross the remote boundary. | +| Browser | `openadapt-flow` owns the supported Playwright recorder. The Chrome extension and bridge in this repository are source-only development prototypes and are excluded from Capture release artifacts. | + +Capture supports native input observation on macOS, Windows, and X11 Linux. +Windows can also retain UI Automation evidence at action time. The structural +schema permits another injected provider, but the package does not currently +ship macOS Accessibility or Linux AT-SPI structural observers. -`openadapt-capture` is designed for **production use** - it should run uninterrupted for days without hindering the user. +## Session pipeline -### Key Features +One recording has these stages: -1. **Production-ready** - Designed for continuous operation without degradation -2. **Process-isolated video** - The MIT package stages frames and invokes a - separately provisioned FFmpeg executable; it does not bundle or link codec - libraries -3. **Low resource footprint** - Non-blocking capture that doesn't slow down user's work -4. **Chunked media** - Video/audio split into manageable segments for long captures -5. **Audio capture** - Built-in audio recording with Whisper transcription -6. **Privacy-first** - Designed to integrate with `openadapt-privacy` for scrubbing -7. **Multi-process architecture** - Optimized queues for high-throughput event handling +1. Resolve and verify all required local dependencies before input listeners + start. This includes a real encode-and-decode probe when video is enabled. +2. Resolve the initial native-window or virtual-desktop coordinate scope. +3. Create the per-capture SQLite database and media staging path. +4. Observe native input and screen frames on separate workers. +5. Put all observed events onto one timestamped processing queue. +6. Associate actionable input with the preceding screen observation and + optional structural observation. +7. Stream RGB frames to a separately provisioned FFmpeg process. +8. Close, verify, and atomically promote the MP4. Retain an incomplete partial + file on an encoder failure and never report it as complete media. +9. Post-process raw input into the public action view. -### Production Requirements +A worker failure stops the session and propagates through the recording +boundary. A frame whose size violates the fixed stream contract is an error. It +is not silently skipped. -For continuous capture over days/weeks: +## Coordinate contracts -1. **Memory bounded** - Stream events to disk, don't accumulate in RAM -2. **Chunked video** - Split into segments (e.g., 10 min each) to avoid giant files -3. **Graceful recovery** - Handle crashes, resume without data loss -4. **Minimal CPU** - Capture shouldn't impact user's work -5. **Disk management** - Configurable retention, auto-cleanup of old captures +### Full virtual desktop -## Scope Decision: Accessibility Data +MSS monitor zero is the bounding rectangle of all active monitors. Native input +uses global desktop coordinates. Capture stores: -OpenAdapt currently captures **accessibility tree data** (element state, UI hierarchy) via platform-specific APIs: -- macOS: `ApplicationServices` / `AXUIElement` -- Windows: `UIAutomation` -- Linux: `AT-SPI` +- `coordinate_space = "virtual_desktop_pixels"` +- the combined desktop origin and viewport +- the physical monitor count +- privacy-safe physical monitor rectangles -### Recommendation +For each input point, Capture subtracts the combined desktop origin. This maps +negative-origin secondary monitors into the exact captured frame without a +fabricated per-monitor scale. A converter must not apply the legacy +`pixel_ratio` scale to this coordinate space. -**Start vision-only, add accessibility as optional layer later.** +The desktop topology is fixed for one recording. Display hot-plug, rotation, +resolution, or scale changes alter the encoded frame contract and fail the +session. Multiple monitors are supported when the topology stays unchanged. -The core capture should be: -- Input events (mouse, keyboard, scroll) -- Screen frames (video) -- Audio (optional) +### Native window -Accessibility data can be added as an optional enrichment step, not a core requirement. +The first successful frame fixes the encoded viewport. Capture then re-resolves +the selected window for each frame. It retains the current bounds, source +viewport, source-to-output scale, letterbox content rectangle, and change +timeline. -### Window Events +When the window moves, resizes, or moves between displays with different +scales, Capture scales the complete source frame to fit the fixed encoded +viewport and adds letterboxing as required. It maps input with the same current +bounds and content rectangle. It does not discard a frame because the source +window changed size. -**Decision:** Exclude window change events from core capture. +Input outside the selected window remains out of range. Capture does not clamp +it into a valid-looking target coordinate. -Without accessibility data, window focus/bounds changes have limited value: -- We already have screenshots showing window state -- Window metadata without accessibility tree is just title + bounds -- If needed, can be added as optional stream later - -## Terminology - -| Concept | Term | Rationale | -|---------|------|-----------| -| Container | **Capture** | Avoids "Recording" (implies audio/video), "Session" (overloaded) | -| Atomic unit | **Event** | Standard term across systems | -| Event sequence | **Stream** | Time-ordered events of one type | -| Multi-capture | **Sequence** | Optional, for workflows | - -## Event Types - -### Raw Events (captured) - -Record primitive events, combine them in post-processing: - -```python -# Mouse events -"mouse.move" # x, y -"mouse.down" # x, y, button -"mouse.up" # x, y, button -"mouse.scroll" # x, y, dx, dy - -# Keyboard events -"key.down" # key, key_char, modifiers -"key.up" # key, key_char, modifiers - -# Screen events -"screen.frame" # reference to video timestamp or image path - -# Audio events (optional) -"audio.chunk" # reference to audio file + timestamp range -``` - -### Derived Events (post-processing) - -Combine raw events into higher-level actions (see OpenAdapt's `events.py`): - -```python -# Derived from mouse.down + mouse.up -"mouse.click" # single click -"mouse.doubleclick" # two clicks within threshold -"mouse.drag" # down + move + up (TODO: add to OpenAdapt) - -# Derived from key.down + key.up sequences -"key.type" # sequence of characters typed -"key.shortcut" # modifier + key combination -``` - -### Event Processing Pipeline - -Based on OpenAdapt's `events.py`: - -1. `remove_invalid_keyboard_events` - Filter invalid key codes -2. `remove_redundant_mouse_move_events` - Remove moves that don't change position -3. `merge_consecutive_keyboard_events` - Combine key sequences into "type" events -4. `merge_consecutive_mouse_move_events` - Reduce move event density -5. `merge_consecutive_mouse_scroll_events` - Combine scroll events -6. `merge_consecutive_mouse_click_events` - Detect single/double clicks - -**TODO:** Add `mouse.drag` detection (currently missing from OpenAdapt). - -## Proposed Abstraction - -### Event Schema - -```python -@dataclass -class Event: - timestamp: float # Unix timestamp (seconds, float for sub-ms) - type: str # Event type identifier - data: dict # Event-specific payload -``` - -### Stream Schema - -```python -@dataclass -class Stream: - id: str - type: str # "action" | "screen" | "audio" - events: list[Event] -``` - -**Note:** Using "action" not "input" - clearer terminology. - -### Capture Schema - -```python -@dataclass -class Capture: - id: str - started_at: float - ended_at: float | None - platform: str # "darwin" | "win32" | "linux" - screen_dimensions: tuple[int, int] # For coordinate normalization - streams: dict[str, Stream] - metadata: dict # task_description, etc. -``` - -## Media Handling - -### Video Encoding - -Capture keeps its video-first behavior without importing PyAV. During a -recording it streams in-memory RGB frames directly into an externally -provisioned FFmpeg process. Missing integer PTS slots reuse the preceding RGB -frame, producing a deterministic constant-rate stream without depending on -wall-clock arrival time. A compact ignored MP4 UUID box maps logical capture -timestamps to their decoded-frame indexes, preserving the existing -nearest-frame contract without an image sidecar. Finalization closes the -bounded process, appends that metadata, verifies the output by decoding a PNG -frame, and atomically promotes it. No intermediate screenshot sequence or -`ffconcat` manifest is written. Failure retains only an explicitly incomplete -partial MP4 and never reports it as complete. A sibling or explicitly -configured `ffprobe` supports legacy nearest-frame extraction and metadata -inspection without linking codec libraries into Capture. The real preflight -exercises raw-video input through a pipe, the selected encoder, MP4, PNG -through `image2pipe`, and the `select` filter before any input listener starts. - -```python -codec = None # Probe platform encoders, then portable mpeg4 fallback -pix_fmt = None # Selected with the verified codec -crf = 0 # Lossless (adjustable for size vs quality) -preset = "veryslow" # Maximum compression -fps = 24 # Configurable -``` - -### Storage: SQLite vs Filesystem - -OpenAdapt uses SQLite for events. Benchmarks show it's faster than filesystem for: -- Many small writes (events) -- Querying by timestamp -- Atomic transactions - -**Recommendation:** SQLite for events, filesystem for media (video, audio). - -``` -capture_abc123/ -├── capture.db # SQLite: events, metadata -├── screen/ -│ └── video.mp4 # Or chunked: video_001.mp4, video_002.mp4, ... -└── audio/ - └── audio.flac # Compressed audio -``` - -### Screenshots vs Video - -**Decision:** Default to video mode. - -- More storage-efficient than permanent individual screenshots -- Exact timestamp alignment is preserved by deterministic PTS-gap filling and - logical timestamps embedded in the MP4 -- Screenshots can be extracted from video when needed -- Individual screenshots remain optional for debugging frame alignment - -## Audio Handling - -Based on OpenAdapt's implementation: - -1. **Capture:** `sounddevice.InputStream` at 16kHz mono -2. **Storage:** FLAC compression (lossless, ~50% size reduction) -3. **Transcription:** Whisper with word-level timestamps -4. **Schema:** - ```python - AudioInfo: - flac_data: bytes - transcribed_text: str - sample_rate: int - words_with_timestamps: list[dict] # [{"word": "hello", "start": 0.5, "end": 0.8}] - ``` - -Transcription stored separately from audio stream events, linked by timestamp. - -## Coordinate Handling - -**Decision:** Store absolute pixels, include screen dimensions in metadata. - -```python -Capture: - screen_dimensions: (1920, 1080) - -Event (mouse.click): - data: {x: 500, y: 300, button: "left"} -``` - -Normalization can happen at read time if needed: -```python -normalized_x = event.data["x"] / capture.screen_dimensions[0] -``` - -## Privacy Integration - -Scrubbing operates at the Capture level: - -```python -from openadapt_privacy import PresidioScrubbingProvider - -def scrub_capture(capture: Capture, scrubber: ScrubbingProvider) -> Capture: - """Return a new Capture with PII removed.""" - # Scrub metadata (task_description, etc.) - # Scrub any text in key events - # Scrub video frames - # Scrub audio transcription - ... -``` - -## Open Questions - -1. **Drag detection:** How to detect drag events from mouse.down → move → mouse.up sequences? - - Time threshold? - - Distance threshold? - - Review other implementations - -2. **Video chunking:** What segment duration for long captures? - - 10 minutes? 1 hour? - - Based on file size or time? - -3. **SQLite schema:** Match OpenAdapt's schema for compatibility, or start fresh? - -## Next Steps - -1. Define exact event schemas (Pydantic models) -2. Implement SQLite storage for events -3. Maintain external-process video capture without bundling codec binaries -4. Port event processing from OpenAdapt's `events.py` -5. Add drag detection -6. Integrate with `openadapt-privacy` -7. Add audio capture +## Native input + +Capture records these primitive event classes: + +- mouse move +- mouse button press and release +- mouse scroll +- key press and release + +Platform observers have one ordered callback contract. They identify injected +events when the operating system provides that information. Capture can exclude +its own injected qualification events from a normal session. It refuses an +incomplete observer startup instead of reporting partial coverage as complete. + +The post-processing layer merges primitive events into higher-level actions. +The compiler remains responsible for refusing action forms that its selected +backend cannot replay safely. + +## Structural observations + +Structural observations are optional evidence beside an action. The versioned +schema can retain: + +- provider and query type +- element role, name, AutomationId, class, framework, bounds, and patterns +- process and top-level window identity +- bounded ancestry +- exact candidate cardinality and its matching fields + +The package currently creates a Windows UIA observer. A missing optional field +stays missing. Capture does not infer an accessibility value from a screenshot, +coordinate, or neighboring control. Provider text has strict length and depth +bounds. A transient provider failure omits the optional observation without +corrupting the screen and input evidence. + +UIA describes the local accessibility tree. It does not describe controls +inside an RDP or Citrix pixel stream. + +## Video and frame timing + +Capture does not import, link, download, or bundle FFmpeg. It invokes an +explicitly configured, Desktop-provisioned, or user-provisioned executable +through a process boundary. Preflight verifies the required raw-video input, +selected encoder, MP4 muxing, PNG encode/decode, `image2pipe`, and `select` +filter before recording starts. + +The writer emits a deterministic constant-rate stream. It reuses the preceding +RGB frame for a missing integer PTS slot. A compact MP4 metadata box binds +logical capture timestamps to decoded frame indexes. Final verification decodes +a real frame from the staged artifact before the file is promoted. + +Capture uses SQLite for events and the filesystem for media. The current media +contract is one MP4 per recording. The package does not claim video chunking, +automatic retention, or crash resume. + +## Audio boundary + +Microphone narration is off by default. When enabled, Capture requires an +installed on-device transcription backend before it opens the microphone. It +does not use a remote fallback. It discards the waveform after transcription +unless the operator explicitly enables waveform retention. + +Transcript text is unsanitized. A retained waveform is biometric data. Neither +is safe for automatic egress. + +## Browser boundary + +The supported browser recorder remains Playwright-native in `openadapt-flow`. +It needs one owner for the browser context, DOM identity, field geometry, +ordered before/after frames, page state, and source-time secret redaction. + +The Chrome extension can supply useful DOM observations, but it does not yet +provide this complete contract. It can become a supported auxiliary observer +after it has: + +1. a shared versioned event schema; +2. source-time secret redaction; +3. explicit permission and browser-profile boundaries; +4. fail-closed reconnect and tab-lifecycle behavior; +5. exact frame and event binding; and +6. end-to-end compiler qualification. + +It should not replace the Playwright recorder only to consolidate packages. +The stronger ownership and redaction boundary is more important than package +uniformity. + +The extension and its unauthenticated development bridge are repository-only. +Wheel and source archives exclude the bridge and its legacy direct replay. The +production package keeps only the passive browser-event schemas needed to read +old local captures. + +## Release qualification + +The production release workflow is manual and binds evidence to one exact +commit. It must: + +- build and validate one wheel and sdist; +- install and uninstall that exact wheel in a clean environment on Linux, + macOS, and Windows; +- run interactive native qualification on labeled Linux, macOS, and Windows + hosts with real display and input permissions; +- require at least two real monitors on each interactive host; +- run the complete slow native capture tests with no skip; +- verify live window movement and resize where the operating system supports + window-scoped capture; and +- retain machine-readable test and topology evidence. + +The release workflow accepts only a successful, complete job set for its exact +commit. Missing, stale, skipped, partial, or failed evidence blocks publication. + +## Known boundaries + +- A visible logged-in desktop session and operating-system permissions are + required. +- Windows window capture uses a screen-region grab and requires an unobstructed + target window. +- A display-topology change requires a new recording. +- Browser extension capture is a repository-only development prototype. Its + bridge and direct replay are not in release artifacts. It is not the + supported browser recorder. +- A raw capture is sensitive and has no automatic safe-for-egress derivative. +- Customer RDP and Citrix environments require their own task-specific + qualification. diff --git a/openadapt_capture/__init__.py b/openadapt_capture/__init__.py index 58db2dc..97365a2 100644 --- a/openadapt_capture/__init__.py +++ b/openadapt_capture/__init__.py @@ -11,6 +11,26 @@ __version__ = "0+unknown" # High-level APIs (primary interface) +# Passive browser-event schemas remain public so existing local captures can +# still be inspected. The repository-only Chrome-extension bridge is not part +# of the production package or API. Supported browser recording is owned by +# openadapt-flow's Playwright launch and attach paths. +from openadapt_capture.browser_events import ( + BoundingBox, + BrowserClickEvent, + BrowserEvent, + BrowserEventType, + BrowserFocusEvent, + BrowserInputEvent, + BrowserKeyEvent, + BrowserNavigationEvent, + BrowserScrollEvent, + DOMSnapshot, + ElementState, + NavigationType, + SemanticElementRef, + VisibleElement, +) from openadapt_capture.capture import Action, Capture, CaptureSession # Frame comparison utilities @@ -111,34 +131,6 @@ translate_point, ) -# Browser events and bridge (optional - requires websockets) -try: - from openadapt_capture.browser_bridge import ( - BrowserBridge, - BrowserEventRecord, - BrowserMode, - run_browser_bridge, - ) - from openadapt_capture.browser_events import ( - BoundingBox, - BrowserClickEvent, - BrowserEvent, - BrowserEventType, - BrowserFocusEvent, - BrowserInputEvent, - BrowserKeyEvent, - BrowserNavigationEvent, - BrowserScrollEvent, - DOMSnapshot, - ElementState, - NavigationType, - SemanticElementRef, - VisibleElement, - ) - _BROWSER_BRIDGE_AVAILABLE = True -except ImportError: - _BROWSER_BRIDGE_AVAILABLE = False - __all__ = [ # Version "__version__", @@ -221,13 +213,7 @@ # Visualization "create_demo", "create_html", - # Browser bridge (optional) - "_BROWSER_BRIDGE_AVAILABLE", - "BrowserBridge", - "BrowserMode", - "BrowserEventRecord", - "run_browser_bridge", - # Browser events + # Passive browser events (legacy capture reads) "BrowserEventType", "BrowserEvent", "BrowserClickEvent", diff --git a/openadapt_capture/capture.py b/openadapt_capture/capture.py index 30bf146..75574b3 100644 --- a/openadapt_capture/capture.py +++ b/openadapt_capture/capture.py @@ -647,6 +647,21 @@ def window_capture(self) -> dict | None: return config.get("capture_window") return None + @property + def desktop_capture(self) -> dict | None: + """Combined-monitor geometry for a full-screen recording, else None. + + New full-screen sessions translate native global input into MSS monitor + zero, the captured virtual-desktop frame. The metadata retains its + origin, viewport, monitor count, and privacy-safe monitor rectangles so + converters must not apply the legacy display-ratio scale again. + """ + + config = getattr(self._recording, "config", None) + if isinstance(config, dict): + return config.get("capture_desktop") + return None + @property def audio_start_time(self) -> float | None: """Start timestamp of the audio recording, or None if unavailable.""" diff --git a/openadapt_capture/cli.py b/openadapt_capture/cli.py index 3490d6e..861129b 100644 --- a/openadapt_capture/cli.py +++ b/openadapt_capture/cli.py @@ -34,15 +34,22 @@ def record( unless RECORD_AUDIO_RETAIN_WAVEFORM is explicitly enabled. Requires an on-device transcription backend to be installed. images: Also save screenshots as PNGs (default: False). - browser_events: Enable the development Chrome extension prototype - (default: False). It is not the supported browser recorder. Use - openadapt-flow browser launch or attach mode for a compile-ready - recording. The prototype connects to localhost:8765. + browser_events: Removed legacy opt-in. If true, the command fails + before it starts a recorder. Use openadapt-flow Playwright launch + or attach recording instead. send_profile: Send profiling data via wormhole after recording (default: False). window_owner: Owner-app substring for window-scoped recording — capture ONE window in its own pixel space (e.g. --window-owner Parallels). window_title: Title substring to disambiguate the target window. """ + if browser_events: + print( + "The Chrome-extension WebSocket prototype is not part of the " + "supported openadapt-capture runtime." + ) + print("Use openadapt-flow Playwright launch or attach recording instead.") + raise SystemExit(2) + import time from openadapt_capture.recorder import Recorder @@ -86,11 +93,6 @@ def record( print(f"Recording to: {output_dir}") if window: print(f"Window-scoped: owner={window_owner!r} title={window_title!r}") - if browser_events: - print("PROTOTYPE: Capture Chrome-extension events are not a supported path.") - print("Use openadapt-flow browser launch or attach mode for real workflows.") - print("Browser event capture enabled (WebSocket on localhost:8765)") - print("Make sure the openadapt-capture Chrome extension is installed.") print("Press Ctrl+C or type stop sequence to stop recording...") print() @@ -100,7 +102,6 @@ def record( capture_video=video, capture_audio=audio, capture_images=images, - capture_browser_events=browser_events, send_profile=send_profile, window=window, ) as recorder: diff --git a/openadapt_capture/config.py b/openadapt_capture/config.py index f505825..3881d77 100644 --- a/openadapt_capture/config.py +++ b/openadapt_capture/config.py @@ -78,11 +78,6 @@ class Settings(BaseSettings): # Performance plotting PLOT_PERFORMANCE: bool = True - # Browser Events Record (extension) configurations - BROWSER_WEBSOCKET_SERVER_IP: str = "localhost" - BROWSER_WEBSOCKET_PORT: int = 8765 - BROWSER_WEBSOCKET_MAX_SIZE: int = 2**22 # 4MB - # Database DB_ECHO: bool = False diff --git a/openadapt_capture/desktop_capture.py b/openadapt_capture/desktop_capture.py new file mode 100644 index 0000000..dbfdeec --- /dev/null +++ b/openadapt_capture/desktop_capture.py @@ -0,0 +1,176 @@ +"""Virtual-desktop coordinate contract for full-screen recordings.""" + +from __future__ import annotations + +import threading +import time +from dataclasses import dataclass +from dataclasses import field as dataclass_field +from numbers import Integral +from typing import Any, Callable, Iterable, Mapping + + +class DesktopCaptureError(RuntimeError): + """The virtual desktop geometry is absent or malformed.""" + + +def _integer_geometry(monitor: Mapping[str, Any]) -> dict[str, int]: + geometry: dict[str, int] = {} + for field in ("left", "top", "width", "height"): + value = monitor.get(field) + if isinstance(value, bool) or not isinstance(value, Integral): + raise DesktopCaptureError(f"virtual desktop {field} must be an integer") + parsed = int(value) + if field in {"width", "height"} and parsed <= 0: + raise DesktopCaptureError(f"virtual desktop {field} must be positive") + geometry[field] = parsed + return geometry + + +@dataclass(frozen=True) +class DesktopCaptureScope: + """Map global native input into the combined MSS frame coordinate space. + + MSS monitor zero is the bounding rectangle of all active monitors. Native + input coordinates use the same global desktop origin on the supported + platforms. Subtracting the combined rectangle's left/top therefore makes + negative-origin and secondary-monitor input line up with the captured + frame without a fabricated per-monitor scale. + """ + + left: int + top: int + width: int + height: int + monitors: tuple[dict[str, int], ...] + _topology_reader: Callable[[], Iterable[Mapping[str, Any]]] | None = dataclass_field( + default=None, + compare=False, + repr=False, + ) + _topology_lock: threading.Lock = dataclass_field( + default_factory=threading.Lock, + compare=False, + repr=False, + ) + _last_topology_check: list[float] = dataclass_field( + default_factory=lambda: [0.0], + compare=False, + repr=False, + ) + + @classmethod + def from_monitors( + cls, + monitors: Iterable[Mapping[str, Any]], + *, + topology_reader: Callable[[], Iterable[Mapping[str, Any]]] | None = None, + ) -> "DesktopCaptureScope": + values = list(monitors) + if len(values) < 2: + raise DesktopCaptureError( + "MSS did not report a combined desktop and a physical monitor" + ) + combined = _integer_geometry(values[0]) + physical = tuple(_integer_geometry(monitor) for monitor in values[1:]) + combined_right = combined["left"] + combined["width"] + combined_bottom = combined["top"] + combined["height"] + if any( + monitor["left"] < combined["left"] + or monitor["top"] < combined["top"] + or monitor["left"] + monitor["width"] > combined_right + or monitor["top"] + monitor["height"] > combined_bottom + for monitor in physical + ): + raise DesktopCaptureError( + "a physical monitor falls outside the combined virtual desktop" + ) + physical_bounds = ( + min(monitor["left"] for monitor in physical), + min(monitor["top"] for monitor in physical), + max(monitor["left"] + monitor["width"] for monitor in physical), + max(monitor["top"] + monitor["height"] for monitor in physical), + ) + if physical_bounds != ( + combined["left"], + combined["top"], + combined_right, + combined_bottom, + ): + raise DesktopCaptureError("physical monitors do not span the combined virtual desktop") + return cls( + left=combined["left"], + top=combined["top"], + width=combined["width"], + height=combined["height"], + monitors=physical, + _topology_reader=topology_reader, + ) + + @classmethod + def current(cls) -> "DesktopCaptureScope": + """Read and retain a live-check contract for the combined desktop.""" + + return cls.from_monitors( + _read_current_monitors(), + topology_reader=_read_current_monitors, + ) + + def assert_current(self, *, force: bool = False) -> None: + """Reject any display topology change after recording starts. + + A topology can change its origin or monitor layout without changing the + combined frame size. A size-only video check cannot detect that case, + and stale translation would bind input to the wrong pixels. + """ + + if self._topology_reader is None: + return + with self._topology_lock: + now = time.monotonic() + if not force and now - self._last_topology_check[0] < 0.05: + return + current = type(self).from_monitors(self._topology_reader()) + if current != self: + raise DesktopCaptureError( + "virtual desktop topology changed during recording; " + f"expected {self.snapshot()}, got {current.snapshot()}" + ) + self._last_topology_check[0] = now + + def translate(self, x: float, y: float) -> tuple[float, float]: + """Translate global input to combined-frame pixels.""" + + self.assert_current() + return (x - self.left, y - self.top) + + def snapshot(self) -> dict[str, Any]: + """Return privacy-safe topology metadata retained with the session.""" + + return { + "coordinate_space": "virtual_desktop_pixels", + "origin": [self.left, self.top], + "viewport": [self.width, self.height], + "monitor_count": len(self.monitors), + "monitors": [ + [ + monitor["left"], + monitor["top"], + monitor["width"], + monitor["height"], + ] + for monitor in self.monitors + ], + } + + +def _read_current_monitors() -> list[Mapping[str, Any]]: + """Read fresh MSS topology without reusing its cached monitor inventory.""" + + # Lazy import preserves the headless-import contract. MSS caches its + # ``monitors`` property for the lifetime of an instance, so a process-local + # screenshot handle cannot detect a hot-plug or same-size rearrangement. + import mss + + with mss.mss() as capture: + return list(capture.monitors) diff --git a/openadapt_capture/recorder.py b/openadapt_capture/recorder.py index 3719c58..02ad1eb 100644 --- a/openadapt_capture/recorder.py +++ b/openadapt_capture/recorder.py @@ -20,7 +20,6 @@ """ import io -import json import multiprocessing import os import queue @@ -44,6 +43,7 @@ from openadapt_capture.config import config from openadapt_capture.db import create_db, crud, get_session_for_path from openadapt_capture.db.models import ActionEvent, Recording +from openadapt_capture.desktop_capture import DesktopCaptureScope from openadapt_capture.extensions import synchronized_queue as sq from openadapt_capture.input_observer import ( ObservedInput, @@ -61,28 +61,16 @@ observe_structural_action, ) from openadapt_capture.window_capture import ( - WindowCaptureError, WindowCaptureScope, build_window_scope, ) +CoordinateScope = WindowCaptureScope | DesktopCaptureScope + try: import soundfile - import websockets.sync.server except ImportError: soundfile = None - websockets = None - -def set_browser_mode( - mode: str, websocket: "websockets.sync.server.ServerConnection" -) -> None: - """Send a message to the browser extension to set the mode.""" - logger.info(f"{type(websocket)=}") - VALID_MODES = ("idle", "record", "replay") - assert mode in VALID_MODES, f"{mode=} not in {VALID_MODES=}" - message = json.dumps({"type": "SET_MODE", "mode": mode}) - logger.info(f"sending {message=}") - websocket.send(message) def _send_profiling_via_wormhole(profile_path: str, timeout: int = 60) -> None: @@ -119,6 +107,11 @@ def _send_profiling_via_wormhole(profile_path: str, timeout: int = 60) -> None: EVENT_TYPES = ("screen", "action", "window", "browser") LOG_LEVEL = "INFO" +BROWSER_RECORDING_GUIDANCE = ( + "The Chrome-extension WebSocket prototype is not part of the supported " + "openadapt-capture runtime. Use openadapt-flow Playwright launch or attach " + "recording instead." +) class _ScreenTimingStats: @@ -170,7 +163,6 @@ def __bool__(self): PRE_READY_TASK_JOIN_TIMEOUT_SECONDS = 2.0 stop_sequence_detected = False -ws_server_instance = None def _run_task_fail_loud( @@ -207,11 +199,7 @@ def _wait_for_tasks_started( logger.info("Recording startup cancelled before all tasks were ready") return False - waiting_for = [ - name - for name, event in task_started_events.items() - if not event.is_set() - ] + waiting_for = [name for name, event in task_started_events.items() if not event.is_set()] if not waiting_for: return True @@ -221,17 +209,12 @@ def _wait_for_tasks_started( if name in task_by_name and not task_by_name[name].is_alive() ] if stopped_before_ready: - logger.error( - "Recording tasks exited before readiness: " - f"{stopped_before_ready}" - ) + logger.error(f"Recording tasks exited before readiness: {stopped_before_ready}") terminate_processing.set() return False logger.info(f"Waiting for tasks to start: {waiting_for}") - logger.info( - f"Started tasks: {expected_starts - len(waiting_for)}/{expected_starts}" - ) + logger.info(f"Started tasks: {expected_starts - len(waiting_for)}/{expected_starts}") terminate_processing.wait(STARTUP_WAIT_POLL_SECONDS) @@ -263,9 +246,7 @@ def _join_tasks( # Python cannot forcibly stop threads; recorder-owned threads are daemons # and all receive terminate_processing before this helper is called. if isinstance(task, multiprocessing.process.BaseProcess): - logger.warning( - f"terminating {task_name!r} after pre-ready shutdown timeout" - ) + logger.warning(f"terminating {task_name!r} after pre-ready shutdown timeout") task.terminate() task.join(timeout=0.5) @@ -282,13 +263,11 @@ def _raise_for_failed_processes(task_by_name: dict[str, Any]) -> None: failures = { name: task.exitcode for name, task in task_by_name.items() - if isinstance(task, multiprocessing.process.BaseProcess) - and task.exitcode not in (None, 0) + if isinstance(task, multiprocessing.process.BaseProcess) and task.exitcode not in (None, 0) } if failures: detail = ", ".join( - f"{name} (exit code {exitcode})" - for name, exitcode in sorted(failures.items()) + f"{name} (exit code {exitcode})" for name, exitcode in sorted(failures.items()) ) raise RuntimeError(f"Recording child process failed: {detail}") @@ -716,14 +695,12 @@ def video_pre_callback( else: # TODO XXX replace with utils.get_monitor_dims() once fixed monitor_width, monitor_height = utils.take_screenshot().size - video_container, video_stream, video_start_timestamp = ( - video.initialize_video_writer( - video_file_path, - monitor_width, - monitor_height, - timeout_seconds=timeout_seconds, - preflight_provision=provision, - ) + video_container, video_stream, video_start_timestamp = video.initialize_video_writer( + video_file_path, + monitor_width, + monitor_height, + timeout_seconds=timeout_seconds, + preflight_provision=provision, ) crud.update_video_start_time(db, recording, video_start_timestamp) return { @@ -791,24 +768,13 @@ def write_video_event( screenshot_timestamp = event.timestamp stream_size = (video_stream.width, video_stream.height) if (screenshot_image.width, screenshot_image.height) != stream_size: - # A frame whose size differs from the stream (e.g. the target window - # of a window-scoped recording was resized mid-recording) cannot be - # encoded into this stream. Skip it LOUDLY: screenshots and the - # bounds timeline still record the change exactly. - logger.warning( - f"Skipping video frame {screenshot_image.size} != stream " - f"{stream_size} (window resized mid-recording?)" + # Window-scoped capture normalizes every resize into its initial fixed + # viewport. A mismatch now means a producer violated the stream + # contract. Stop instead of emitting a complete-looking video with a + # silent evidence gap. + raise ValueError( + f"video frame {screenshot_image.size} differs from fixed stream {stream_size}" ) - perf_q.put((event.type, event.timestamp, utils.get_timestamp())) - return { - **kwargs, - **{ - "video_container": video_container, - "video_stream": video_stream, - "video_start_timestamp": video_start_timestamp, - "last_pts": last_pts, - }, - } force_key_frame = last_pts == 0 # ensure that the first frame is available (otherwise occasionally it is not) # TODO: why isn't force_key_frame sufficient? @@ -841,7 +807,7 @@ def write_video_event( def trigger_action_event( event_q: queue.Queue, action_event_args: dict[str, Any], - window_scope: WindowCaptureScope | None = None, + coordinate_scope: CoordinateScope | None = None, timestamp: float | None = None, structural_observer: StructuralObserver | None = None, ) -> None: @@ -850,10 +816,10 @@ def trigger_action_event( Args: event_q: The event queue to add the action event to. action_event_args: A dictionary containing the arguments for the action event. - window_scope: When set (window-scoped capture), global mouse - coordinates are translated into the target window's pixel space - before being recorded, so recorded coordinates match the captured - frames directly. + coordinate_scope: When set, global mouse coordinates are translated + into the exact captured-frame pixel space. A window scope tracks + the target window. A desktop scope subtracts the combined virtual + desktop origin, including negative-origin secondary monitors. timestamp: Native event-receipt time. Defaults to the current recording clock only for legacy/direct callers. structural_observer: Optional accessibility observer. Evidence is @@ -886,14 +852,11 @@ def trigger_action_event( else: element_state = {} action_event_args["element_state"] = element_state - if window_scope is not None: - try: - wx, wy = window_scope.translate(x, y) - except WindowCaptureError as exc: - # No frame captured yet: recording a global coordinate in a - # window-scoped session would silently mix coordinate spaces. - logger.warning(f"Discarding input before first window frame: {exc}") - return + if coordinate_scope is not None: + # The recorder captures and validates its first scoped frame before + # starting input. A translation failure after that boundary means + # evidence is incomplete and must terminate the session. + wx, wy = coordinate_scope.translate(x, y) action_event_args["mouse_x"] = wx action_event_args["mouse_y"] = wy event_q.put( @@ -907,7 +870,7 @@ def trigger_action_event( def on_move( event_q: queue.Queue, - window_scope: WindowCaptureScope | None, + coordinate_scope: CoordinateScope | None, x: float, y: float, injected: bool = False, @@ -917,7 +880,7 @@ def on_move( Args: event_q: The event queue to add the 'move' event to. - window_scope: Optional window scope for coordinate translation. + coordinate_scope: Optional captured-frame coordinate translator. x: The x-coordinate of the mouse. y: The y-coordinate of the mouse. injected: Whether the event was injected or not. @@ -930,14 +893,14 @@ def on_move( trigger_action_event( event_q, {"name": "move", "mouse_x": x, "mouse_y": y}, - window_scope, + coordinate_scope, timestamp, ) def on_click( event_q: queue.Queue, - window_scope: WindowCaptureScope | None, + coordinate_scope: CoordinateScope | None, x: float, y: float, button: str, @@ -950,7 +913,7 @@ def on_click( Args: event_q: The event queue to add the 'click' event to. - window_scope: Optional window scope for coordinate translation. + coordinate_scope: Optional captured-frame coordinate translator. x: The x-coordinate of the mouse. y: The y-coordinate of the mouse. button: The mouse button. @@ -971,7 +934,7 @@ def on_click( "mouse_button_name": button, "mouse_pressed": pressed, }, - window_scope, + coordinate_scope, timestamp, structural_observer if pressed else None, ) @@ -979,7 +942,7 @@ def on_click( def on_scroll( event_q: queue.Queue, - window_scope: WindowCaptureScope | None, + coordinate_scope: CoordinateScope | None, x: float, y: float, dx: float, @@ -992,7 +955,7 @@ def on_scroll( Args: event_q: The event queue to add the 'scroll' event to. - window_scope: Optional window scope for coordinate translation. + coordinate_scope: Optional captured-frame coordinate translator. x: The x-coordinate of the mouse. y: The y-coordinate of the mouse. dx: The horizontal scroll amount. @@ -1013,7 +976,7 @@ def on_scroll( "mouse_dx": dx, "mouse_dy": dy, }, - window_scope, + coordinate_scope, timestamp, structural_observer, ) @@ -1056,6 +1019,7 @@ def read_screen_events( started_event: threading.Event, _screen_timing: _ScreenTimingStats | None = None, window_scope: WindowCaptureScope | None = None, + desktop_scope: DesktopCaptureScope | None = None, ) -> None: """Read screen events and add them to the event queue. @@ -1075,7 +1039,11 @@ def read_screen_events( started_event: Event to set once started. _screen_timing: If provided, record (screenshot_dur, total_dur) per iteration. window_scope: Optional window scope for window-pixel-space capture. + desktop_scope: Full-screen virtual-desktop contract. It verifies the + monitor topology before and after each captured frame. """ + if window_scope is not None and desktop_scope is not None: + raise ValueError("screen reader cannot use both window and desktop scopes") utils.set_start_time(recording.timestamp) fps = config.SCREEN_CAPTURE_FPS @@ -1087,15 +1055,10 @@ def read_screen_events( while not terminate_processing.is_set(): t_start = time.perf_counter() if window_scope is not None: - try: - screenshot, window_changed = window_scope.capture_frame() - except WindowCaptureError as exc: - # Loud + recoverable: the window may be mid-move/minimized. - # No frame is queued (never fall back to full-screen pixels - # in a window-scoped recording), and we retry. - logger.warning(f"Window capture failed (retrying): {exc}") - time.sleep(0.5) - continue + # Any failed capture terminates the session. Retrying would omit a + # frame while input continues and could produce complete-looking + # evidence with a missing interval. + screenshot, window_changed = window_scope.capture_frame() if window_changed or not announced_window: event_q.put( Event( @@ -1105,6 +1068,13 @@ def read_screen_events( ) ) announced_window = True + elif desktop_scope is not None: + # A monitor can move or change scale while the combined frame keeps + # the same dimensions. Check both sides of the grab so neither the + # frame nor later input uses stale origin or monitor geometry. + desktop_scope.assert_current(force=True) + screenshot = utils.take_screenshot() + desktop_scope.assert_current(force=True) else: screenshot = utils.take_screenshot() t_screenshot = time.perf_counter() @@ -1296,6 +1266,7 @@ def create_recording( task_description: str, capture_dir: str, window_capture_info: dict | None = None, + desktop_capture_info: dict | None = None, ) -> tuple[Recording, str]: """Create a new recording entry in the per-capture database. @@ -1306,10 +1277,14 @@ def create_recording( (``WindowCaptureScope.snapshot()``) persisted in the recording's config JSON under ``capture_window`` so converters know the session's coordinates are in window-pixel space. + desktop_capture_info: Combined-monitor geometry persisted under + ``capture_desktop`` for full-screen recordings. Returns: tuple of (Recording object, db_path). """ + if window_capture_info is not None and desktop_capture_info is not None: + raise ValueError("a recording cannot declare both window and desktop capture scopes") os.makedirs(capture_dir, exist_ok=True) db_path = os.path.join(capture_dir, "recording.db") @@ -1337,8 +1312,13 @@ def create_recording( "platform": sys.platform, "task_description": task_description, } + capture_config: dict[str, Any] = {} if window_capture_info is not None: - recording_data["config"] = {"capture_window": window_capture_info} + capture_config["capture_window"] = window_capture_info + if desktop_capture_info is not None: + capture_config["capture_desktop"] = desktop_capture_info + if capture_config: + recording_data["config"] = capture_config engine, Session = create_db(db_path) session = Session() recording = crud.insert_recording(session, recording_data) @@ -1351,7 +1331,7 @@ def read_input_events( terminate_processing: multiprocessing.Event, recording: Recording, started_event: threading.Event, - window_scope: WindowCaptureScope | None = None, + coordinate_scope: CoordinateScope | None = None, structural_observer: StructuralObserver | None = None, ) -> None: """Read globally ordered keyboard and mouse events from one native observer.""" @@ -1362,7 +1342,7 @@ def on_observed(event: ObservedInput) -> None: if isinstance(event, ObservedMouseMove): on_move( event_q, - window_scope, + coordinate_scope, event.x, event.y, event.injected, @@ -1372,7 +1352,7 @@ def on_observed(event: ObservedInput) -> None: if isinstance(event, ObservedMouseButton): on_click( event_q, - window_scope, + coordinate_scope, event.x, event.y, event.button, @@ -1385,7 +1365,7 @@ def on_observed(event: ObservedInput) -> None: if isinstance(event, ObservedMouseScroll): on_scroll( event_q, - window_scope, + coordinate_scope, event.x, event.y, event.dx, @@ -1415,9 +1395,7 @@ def on_observed(event: ObservedInput) -> None: if candidate == expected: stop_sequence_indices[index] += 1 else: - stop_sequence_indices[index] = ( - 1 if candidate == sequence[0].lower() else 0 - ) + stop_sequence_indices[index] = 1 if candidate == sequence[0].lower() else 0 if stop_sequence_indices[index] == len(sequence): stop_sequence_indices[index] = 0 logger.info("Stop sequence entered! Stopping recording now.") @@ -1507,9 +1485,7 @@ def audio_callback( audio_frames.append(indata.copy()) # open InputStream and start recording while ActionEvents are recorded - audio_stream = sounddevice.InputStream( - callback=audio_callback, samplerate=16000, channels=1 - ) + audio_stream = sounddevice.InputStream(callback=audio_callback, samplerate=16000, channels=1) logger.info("Audio recording started.") start_timestamp = utils.get_timestamp() audio_stream.start() @@ -1533,9 +1509,7 @@ def audio_callback( "or permission may have been denied. Storing an empty transcript." ) session = get_session_for_path(db_path) - crud.insert_audio_info( - session, b"", "", recording, start_timestamp, sample_rate, [] - ) + crud.insert_audio_info(session, b"", "", recording, start_timestamp, sample_rate, []) return # Concatenate into one Numpy array @@ -1549,9 +1523,9 @@ def audio_callback( # NOTE: the transcript is deliberately NOT logged. Narration can contain # names, dates of birth, and diagnoses; logging it would copy that into # terminal scrollback and any configured log sink. - logger.info("Transcription complete ({} characters).".format( - len(result_info.get("text") or "") - )) + logger.info( + "Transcription complete ({} characters).".format(len(result_info.get("text") or "")) + ) # empty word_list if the user didn't say anything word_list = [] @@ -1614,105 +1588,6 @@ def _transcribe_on_device(audio: "np.ndarray", backend: str) -> dict: return recorder._transcribe_openai_whisper(audio, "base", True) -@logger.catch -@utils.trace(logger) -def read_browser_events( - websocket: "websockets.sync.server.ServerConnection", - event_q: queue.Queue, - terminate_processing: Event, - recording: Recording, -) -> None: - """Read browser events and add them to the event queue. - - Params: - websocket: The websocket object. - event_q: A queue for adding browser events. - terminate_processing: An event to signal the termination of the process. - recording: The recording object. - - Returns: - None - """ - utils.set_start_time(recording.timestamp) - - # set the browser mode - set_browser_mode("record", websocket) - - logger.info("Starting Reading Browser Events ...") - - while not terminate_processing.is_set(): - try: - message = websocket.recv(0.01) - except TimeoutError: - continue - timestamp = utils.get_timestamp() - data = json.loads(message) - event_q.put( - Event( - timestamp, - "browser", - {"message": data}, - ) - ) - - set_browser_mode("idle", websocket) - - -@logger.catch -@utils.trace(logger) -def run_browser_event_server( - event_q: queue.Queue, - terminate_processing: Event, - recording: Recording, - started_event: threading.Event, -) -> None: - """Run the browser event server. - - Params: - event_q: A queue for adding browser events. - terminate_processing: An event to signal the termination of the process. - recording: The recording object. - started_event: Event to set once started. - - Returns: - None - """ - global ws_server_instance - - # Function to run the server in a separate thread - def run_server() -> None: - global ws_server_instance - with websockets.sync.server.serve( - lambda ws: read_browser_events( - ws, - event_q, - terminate_processing, - recording, - ), - config.BROWSER_WEBSOCKET_SERVER_IP, - config.BROWSER_WEBSOCKET_PORT, - max_size=config.BROWSER_WEBSOCKET_MAX_SIZE, - ) as server: - ws_server_instance = server - logger.info("WebSocket server started") - started_event.set() - server.serve_forever() - - # Start the server in a separate thread - server_thread = threading.Thread(target=run_server) - server_thread.start() - - # Wait for a termination signal - terminate_processing.wait() - logger.info("Termination signal received, shutting down server") - - if ws_server_instance: - ws_server_instance.shutdown() - - # Ensure the server thread is terminated cleanly - server_thread.join() - - @logger.catch(reraise=True) @utils.trace(logger) def record( @@ -1738,7 +1613,7 @@ def record( window_title: str | None = None, structural_observer: StructuralObserver | None = None, ) -> None: - """Record Screenshots/ActionEvents/WindowEvents/BrowserEvents. + """Record native screenshots, action events, and window events. Args: task_description: A text description of the task to be recorded. @@ -1756,6 +1631,12 @@ def record( omitted, the platform factory follows ``RECORD_STRUCTURAL_OBSERVATIONS``. """ + if config.RECORD_BROWSER_EVENTS: + # Fail before encoder checks, display access, database creation, or any + # listener bind. The source-only extension bridge has no governed + # replay, authentication, or source-time secret exclusion contract. + raise RuntimeError(BROWSER_RECORDING_GUIDANCE) + assert config.RECORD_VIDEO or config.RECORD_IMAGES, ( config.RECORD_VIDEO, config.RECORD_IMAGES, @@ -1795,12 +1676,20 @@ def record( window_title or config.RECORD_WINDOW_TITLE, ) initial_window_frame = None + desktop_scope = None if window_scope is not None: initial_window_frame, _ = window_scope.capture_frame() logger.info( f"window-scoped capture resolved: {window_scope.snapshot()} " f"initial frame {initial_window_frame.size}" ) + else: + # MSS monitor zero is the exact combined frame read by + # ``utils.take_screenshot``. Retain its origin and translate native + # input into that same pixel space so secondary monitors with negative + # global coordinates remain aligned with the video. + desktop_scope = DesktopCaptureScope.current() + logger.info(f"virtual desktop capture resolved: {desktop_scope.snapshot()}") if structural_observer is None: structural_observer = create_structural_observer( @@ -1812,9 +1701,8 @@ def record( recording, db_path = create_recording( task_description, capture_dir, - window_capture_info=( - window_scope.snapshot() if window_scope is not None else None - ), + window_capture_info=(window_scope.snapshot() if window_scope is not None else None), + desktop_capture_info=(desktop_scope.snapshot() if desktop_scope is not None else None), ) recording_timestamp = recording.timestamp @@ -1847,9 +1735,7 @@ def record( event_q, terminate_processing, recording, - task_started_events.setdefault( - "window_event_reader", threading.Event() - ), + task_started_events.setdefault("window_event_reader", threading.Event()), ), terminate_processing, task_errors, @@ -1858,32 +1744,23 @@ def record( window_event_reader.start() task_by_name["window_event_reader"] = window_event_reader - if config.RECORD_BROWSER_EVENTS: - browser_event_reader = threading.Thread( - target=run_browser_event_server, - daemon=True, - args=( + screen_event_reader = threading.Thread( + target=_run_task_fail_loud, + daemon=True, + args=( + "screen_event_reader", + read_screen_events, + ( event_q, terminate_processing, recording, - task_started_events.setdefault( - "browser_event_reader", threading.Event() - ), + task_started_events.setdefault("screen_event_reader", threading.Event()), + _screen_timing, + window_scope, + desktop_scope, ), - ) - browser_event_reader.start() - task_by_name["browser_event_reader"] = browser_event_reader - - screen_event_reader = threading.Thread( - target=read_screen_events, - daemon=True, - args=( - event_q, terminate_processing, - recording, - task_started_events.setdefault("screen_event_reader", threading.Event()), - _screen_timing, - window_scope, + task_errors, ), ) screen_event_reader.start() @@ -1894,7 +1771,7 @@ def record( terminate_processing, recording, task_started_events.setdefault("input_event_reader", threading.Event()), - window_scope, + window_scope or desktop_scope, structural_observer, ) input_event_reader = threading.Thread( @@ -1957,34 +1834,12 @@ def record( recording, db_path, terminate_processing, - task_started_events.setdefault( - "screen_event_writer", multiprocessing.Event() - ), + task_started_events.setdefault("screen_event_writer", multiprocessing.Event()), ), ) screen_event_writer.start() task_by_name["screen_event_writer"] = screen_event_writer - if config.RECORD_BROWSER_EVENTS: - browser_event_writer = multiprocessing.Process( - target=write_events, - args=( - "browser", - write_browser_event, - browser_write_q, - num_browser_events, - perf_q, - recording, - db_path, - terminate_processing, - task_started_events.setdefault( - "browser_event_writer", multiprocessing.Event() - ), - ), - ) - browser_event_writer.start() - task_by_name["browser_event_writer"] = browser_event_writer - action_event_writer = multiprocessing.Process( target=utils.WrapStdout(write_events), args=( @@ -1996,9 +1851,7 @@ def record( recording, db_path, terminate_processing, - task_started_events.setdefault( - "action_event_writer", multiprocessing.Event() - ), + task_started_events.setdefault("action_event_writer", multiprocessing.Event()), ), ) action_event_writer.start() @@ -2016,9 +1869,7 @@ def record( recording, db_path, terminate_processing, - task_started_events.setdefault( - "window_event_writer", multiprocessing.Event() - ), + task_started_events.setdefault("window_event_writer", multiprocessing.Event()), ), ) window_event_writer.start() @@ -2043,9 +1894,7 @@ def record( # Window-scoped frames are the window's pixels, not the # monitor's: size the stream from the initial frame. frame_size=( - initial_window_frame.size - if initial_window_frame is not None - else None + initial_window_frame.size if initial_window_frame is not None else None ), provision=video_provision, timeout_seconds=config.VIDEO_FFMPEG_TIMEOUT_SECONDS, @@ -2063,9 +1912,7 @@ def record( recording, db_path, terminate_processing, - task_started_events.setdefault( - "audio_event_writer", multiprocessing.Event() - ), + task_started_events.setdefault("audio_event_writer", multiprocessing.Event()), ), ) audio_recorder.start() @@ -2079,9 +1926,7 @@ def record( recording, db_path, terminate_perf_event, - task_started_events.setdefault( - "perf_stats_writer", multiprocessing.Event() - ), + task_started_events.setdefault("perf_stats_writer", multiprocessing.Event()), ), ) perf_stats_writer.start() @@ -2120,9 +1965,7 @@ def record( if startup_ready: for _ in range(5): logger.info("*" * 40) - logger.info( - "All readers and writers have started. Waiting for input events..." - ) + logger.info("All readers and writers have started. Waiting for input events...") if status_pipe: status_pipe.send({"type": "record.started"}) @@ -2143,19 +1986,15 @@ def record( collect_stats(performance_snapshots) log_memory_usage(_tracker, performance_snapshots) - pre_ready_timeout = ( - None if startup_ready else PRE_READY_TASK_JOIN_TIMEOUT_SECONDS - ) + pre_ready_timeout = None if startup_ready else PRE_READY_TASK_JOIN_TIMEOUT_SECONDS _join_tasks( task_by_name, [ "window_event_reader", - "browser_event_reader", "screen_event_reader", "input_event_reader", "event_processor", "screen_event_writer", - "browser_event_writer", "action_event_writer", "window_event_writer", "video_writer", @@ -2179,13 +2018,19 @@ def record( add_exception_note(task_error, f"recording task {task_name!r} failed") raise task_error _raise_for_failed_processes(task_by_name) + if desktop_scope is not None: + # Close the interval between the last captured frame and operator stop. + # A topology change in that interval still invalidates the session. + desktop_scope.assert_current(force=True) if config.PLOT_PERFORMANCE and startup_ready: from openadapt_capture import plotting session = get_session_for_path(db_path) plotting.plot_performance( - session, recording, save_dir=capture_dir, + session, + recording, + save_dir=capture_dir, ) logger.info(f"Saved {recording_timestamp=}") @@ -2231,6 +2076,7 @@ def record( _profile_path = os.path.join(capture_dir, "profiling.json") try: import json as _json + with open(_profile_path, "w") as _f: _json.dump(_profile_data, _f, indent=2) logger.info(f"Profiling saved to {_profile_path}") @@ -2245,13 +2091,17 @@ def record( print(f" {k}: {v} events ({rate:.1f}/s)") if _screen_timing: st = _profile_data["screen_timing"] - print(f" screenshot: avg={st['screenshot_avg_ms']}ms " - f"max={st['screenshot_max_ms']}ms " - f"min={st['screenshot_min_ms']}ms") - print(f"Config: WINDOW_DATA={config.RECORD_WINDOW_DATA} " - f"VIDEO={config.RECORD_VIDEO} " - f"PLOT_PERF={config.PLOT_PERFORMANCE} " - f"FPS={config.SCREEN_CAPTURE_FPS}") + print( + f" screenshot: avg={st['screenshot_avg_ms']}ms " + f"max={st['screenshot_max_ms']}ms " + f"min={st['screenshot_min_ms']}ms" + ) + print( + f"Config: WINDOW_DATA={config.RECORD_WINDOW_DATA} " + f"VIDEO={config.RECORD_VIDEO} " + f"PLOT_PERF={config.PLOT_PERFORMANCE} " + f"FPS={config.SCREEN_CAPTURE_FPS}" + ) print("=========================\n") # Auto-send profiling via wormhole if requested @@ -2329,6 +2179,11 @@ def __init__( self.task_description = task_description self._send_profile = send_profile + if capture_browser_events: + # Preserve a clear error for callers of the former keyword while + # preventing the unsupported server from reaching record(). + raise ValueError(BROWSER_RECORDING_GUIDANCE) + # Validate the window spec up front (loud, before any thread starts). window_target = WindowTarget.from_spec(window) @@ -2430,7 +2285,8 @@ def _run_record(self) -> None: def __enter__(self) -> "Recorder": # Start status drain thread self._status_thread = threading.Thread( - target=self._drain_status_pipe, daemon=True, + target=self._drain_status_pipe, + daemon=True, ) self._status_thread.start() @@ -2449,8 +2305,7 @@ def __exit__(self, exc_type, exc_val, exc_tb) -> None: if self._worker_error is not None: if exc_val is not None: add_exception_note( - exc_val, - f"the recorder worker also failed: {self._worker_error!r}" + exc_val, f"the recorder worker also failed: {self._worker_error!r}" ) else: raise self._worker_error diff --git a/openadapt_capture/window_capture.py b/openadapt_capture/window_capture.py index 4588dfc..a4c5f1e 100644 --- a/openadapt_capture/window_capture.py +++ b/openadapt_capture/window_capture.py @@ -88,13 +88,10 @@ def from_spec(cls, spec: "WindowTarget | dict | None") -> "WindowTarget | None": unknown = set(spec) - {"owner", "title"} if unknown: raise ValueError( - f"unknown window spec keys {sorted(unknown)}; " - "expected {'owner', 'title'}" + f"unknown window spec keys {sorted(unknown)}; expected {{'owner', 'title'}}" ) return cls(owner=spec.get("owner"), title=spec.get("title")) - raise TypeError( - f"window spec must be a dict or WindowTarget, got {type(spec).__name__}" - ) + raise TypeError(f"window spec must be a dict or WindowTarget, got {type(spec).__name__}") @dataclass(frozen=True) @@ -164,7 +161,13 @@ def __init__( self._lock = threading.Lock() self._window: TargetWindow | None = None self._scale: float | None = None + self._scale_x: float | None = None + self._scale_y: float | None = None self._viewport: tuple[int, int] | None = None + self._source_viewport: tuple[int, int] | None = None + self._content_rect: tuple[int, int, int, int] | None = None + self._fit_scale: float | None = None + self._bound_window_id: int | None = None # Window of the last CAPTURED frame (not merely resolved): the # bounds-timeline 'changed' flag compares frame to frame, so a bare # resolve() (e.g. a pre-flight existence check) never suppresses the @@ -172,7 +175,7 @@ def __init__( self._frame_window: TargetWindow | None = None def resolve(self) -> TargetWindow: - """Resolve the target window now, updating shared bounds. + """Resolve the target window without changing captured-frame geometry. Raises: WindowCaptureError: If no matching window is on screen. @@ -184,8 +187,6 @@ def resolve(self) -> TargetWindow: f"title {self.target.title!r}; is the target application " "running with a visible window?" ) - with self._lock: - self._window = win return win def capture_frame(self) -> tuple["Image.Image", bool]: @@ -193,6 +194,10 @@ def capture_frame(self) -> tuple["Image.Image", bool]: Re-resolves the window first so bounds/scale can never disagree with the frame just captured (mirrors flow's ``screenshot()`` contract). + The first successful frame fixes the recording viewport. If the window + later resizes, the complete new frame is scaled to fit that viewport + and letterboxed. This preserves one encodable video stream without + discarding resize frames or mixing coordinate spaces. Returns: (PIL.Image in the window's pixel space, @@ -203,17 +208,54 @@ def capture_frame(self) -> tuple["Image.Image", bool]: Raises: WindowCaptureError: If the window is gone or capture fails. """ - prev = self._frame_window + with self._lock: + prev = self._frame_window + bound_window_id = self._bound_window_id win = self.resolve() - image = self._capturer(win) - if image.width <= 0 or image.height <= 0: + if bound_window_id is not None and win.window_id != bound_window_id: + raise WindowCaptureError( + "the resolved target changed window identity during recording: " + f"expected {bound_window_id}, got {win.window_id}" + ) + source_image = self._capturer(win) + if source_image.width <= 0 or source_image.height <= 0: raise WindowCaptureError("window capture returned an empty frame") - bounds_w = win.bounds[2] or float(image.width) - scale = (image.width / bounds_w) if bounds_w else 1.0 + source_viewport = (source_image.width, source_image.height) + output_viewport = self._viewport or source_viewport + output_width, output_height = output_viewport + fit_scale = min( + output_width / source_image.width, + output_height / source_image.height, + ) + fitted_width = max(1, min(output_width, round(source_image.width * fit_scale))) + fitted_height = max(1, min(output_height, round(source_image.height * fit_scale))) + offset_x = (output_width - fitted_width) // 2 + offset_y = (output_height - fitted_height) // 2 + if source_viewport == output_viewport: + image = source_image + else: + from PIL import Image + + resized = source_image.resize((fitted_width, fitted_height), Image.Resampling.LANCZOS) + image = Image.new("RGB", output_viewport, color=(0, 0, 0)) + image.paste(resized, (offset_x, offset_y)) + bounds_w = win.bounds[2] or float(source_image.width) + bounds_h = win.bounds[3] or float(source_image.height) + scale_x = fitted_width / bounds_w + scale_y = fitted_height / bounds_h with self._lock: self._window = win - self._scale = scale - self._viewport = (image.width, image.height) + # ``scale`` is the historical scalar field. Keep it as the x-axis + # value for old readers. Current readers can use both exact axes. + # Integer resize rounding can make the axes differ slightly. + self._scale = scale_x + self._scale_x = scale_x + self._scale_y = scale_y + self._viewport = output_viewport + self._source_viewport = source_viewport + self._content_rect = (offset_x, offset_y, fitted_width, fitted_height) + self._fit_scale = fit_scale + self._bound_window_id = win.window_id self._frame_window = win changed = ( prev is None @@ -236,13 +278,18 @@ def translate(self, x: float, y: float) -> tuple[float, float]: """ with self._lock: window = self._window - scale = self._scale - if window is None or scale is None: + scale_x = self._scale_x + scale_y = self._scale_y + content_rect = self._content_rect + if window is None or scale_x is None or scale_y is None or content_rect is None: raise WindowCaptureError( "translate() called before the first captured frame; " "capture_frame() must succeed before input can be scoped" ) - return translate_point(x, y, window.bounds, scale) + return ( + (x - window.bounds[0]) * scale_x + content_rect[0], + (y - window.bounds[1]) * scale_y + content_rect[1], + ) def window_event_data(self) -> dict: """Bounds-timeline entry for the WindowEvent table. @@ -255,7 +302,12 @@ def window_event_data(self) -> dict: with self._lock: window = self._window scale = self._scale + scale_x = self._scale_x + scale_y = self._scale_y viewport = self._viewport + source_viewport = self._source_viewport + content_rect = self._content_rect + fit_scale = self._fit_scale if window is None: raise WindowCaptureError("no resolved window; call capture_frame() first") x, y, w, h = window.bounds @@ -272,7 +324,12 @@ def window_event_data(self) -> dict: "pid": window.pid, "bounds": [x, y, w, h], "scale": scale, + "scale_x": scale_x, + "scale_y": scale_y, "viewport": list(viewport) if viewport else None, + "source_viewport": (list(source_viewport) if source_viewport else None), + "content_rect": list(content_rect) if content_rect else None, + "fit_scale": fit_scale, "on_screen": window.on_screen, }, } @@ -287,7 +344,12 @@ def snapshot(self) -> dict: with self._lock: window = self._window scale = self._scale + scale_x = self._scale_x + scale_y = self._scale_y viewport = self._viewport + source_viewport = self._source_viewport + content_rect = self._content_rect + fit_scale = self._fit_scale data: dict = { "target": {"owner": self.target.owner, "title": self.target.title}, "coordinate_space": "window_pixels", @@ -301,7 +363,12 @@ def snapshot(self) -> dict: "pid": window.pid, "initial_bounds": list(window.bounds), "scale": scale, + "scale_x": scale_x, + "scale_y": scale_y, "viewport": list(viewport) if viewport else None, + "source_viewport": (list(source_viewport) if source_viewport else None), + "content_rect": list(content_rect) if content_rect else None, + "fit_scale": fit_scale, } ) return data @@ -319,8 +386,7 @@ def resolve_window(target: WindowTarget) -> TargetWindow | None: if sys.platform == "win32": return _resolve_window_windows(target) raise WindowCaptureError( - f"window-scoped capture is not supported on {sys.platform} " - "(supported: darwin, win32)" + f"window-scoped capture is not supported on {sys.platform} (supported: darwin, win32)" ) @@ -331,8 +397,7 @@ def capture_window(window: TargetWindow) -> "Image.Image": if sys.platform == "win32": return _capture_window_windows(window) raise WindowCaptureError( - f"window-scoped capture is not supported on {sys.platform} " - "(supported: darwin, win32)" + f"window-scoped capture is not supported on {sys.platform} (supported: darwin, win32)" ) @@ -348,9 +413,7 @@ def _resolve_window_macos(target: WindowTarget) -> TargetWindow | None: owner_l = target.owner.lower() if target.owner else None title_l = target.title.lower() if target.title else None - wins = Quartz.CGWindowListCopyWindowInfo( - Quartz.kCGWindowListOptionAll, Quartz.kCGNullWindowID - ) + wins = Quartz.CGWindowListCopyWindowInfo(Quartz.kCGWindowListOptionAll, Quartz.kCGNullWindowID) best: TargetWindow | None = None best_area = -1.0 for w in wins or []: @@ -535,9 +598,7 @@ def _capture_window_windows(window: TargetWindow) -> "Image.Image": return Image.frombytes("RGB", sct_img.size, sct_img.bgra, "raw", "BGRX") -def build_window_scope( - owner: str | None, title: str | None -) -> WindowCaptureScope | None: +def build_window_scope(owner: str | None, title: str | None) -> WindowCaptureScope | None: """Build a :class:`WindowCaptureScope` when a target is configured. Central place the recorder uses to turn (possibly-empty) config values diff --git a/pyproject.toml b/pyproject.toml index 2f43d95..8c5387a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,7 +29,6 @@ dependencies = [ "soundfile>=0.13.1", # Note: openai-whisper moved to optional [transcribe] extra due to Python version conflicts "pydantic-settings>=2.12.0", - "websockets>=12.0", # Recording-pipeline dependencies (carried forward from OpenAdapt record.py) "sqlalchemy>=2.0.0", "alembic>=1.0.0", @@ -98,6 +97,9 @@ build-backend = "hatchling.build" [tool.hatch.build.targets.wheel] packages = ["openadapt_capture"] +exclude = [ + "openadapt_capture/browser_bridge.py", +] [tool.hatch.build.targets.sdist] ignore-vcs = true @@ -108,6 +110,7 @@ only-include = [ "docs", ] exclude = [ + "/openadapt_capture/browser_bridge.py", "/docs/images/demo.gif", "/docs/whisper-integration-plan.md", ] @@ -132,6 +135,14 @@ markers = [ version_toml = ["pyproject.toml:project.version"] commit_message = "chore: release {version}" +[tool.semantic_release.changelog] +mode = "update" +insertion_flag = "" + +[tool.semantic_release.changelog.default_templates] +changelog_file = "CHANGELOG.md" +output_format = "md" + [tool.semantic_release.branches.main] match = "main" @@ -149,4 +160,7 @@ dev = [ # Test-only synthetic input driver; excluded from package metadata/runtime. "pynput>=1.7.6", "pytest>=9.0.2", + # Repository-only Chrome-extension prototype tests. The bridge and this + # dependency are not part of wheel or source-distribution runtime. + "websockets>=12.0", ] diff --git a/scripts/candidate_lifecycle.py b/scripts/candidate_lifecycle.py new file mode 100644 index 0000000..10aaa9f --- /dev/null +++ b/scripts/candidate_lifecycle.py @@ -0,0 +1,214 @@ +#!/usr/bin/env python3 +"""Install, inspect, and remove one exact Capture candidate wheel.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import subprocess +import sys +import tempfile +import venv +from pathlib import Path + + +class CandidateLifecycleError(RuntimeError): + """The candidate artifact or its clean install lifecycle is invalid.""" + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def verify_manifest(dist_dir: Path, manifest_path: Path) -> dict[str, str]: + """Verify that the manifest accounts for exactly one wheel and one sdist.""" + + expected: dict[str, str] = {} + for line_number, raw_line in enumerate( + manifest_path.read_text(encoding="utf-8").splitlines(), start=1 + ): + line = raw_line.strip() + if not line: + continue + parts = line.split() + if len(parts) != 2 or len(parts[0]) != 64: + raise CandidateLifecycleError( + f"invalid SHA256 manifest line {line_number}: {raw_line!r}" + ) + digest, name = parts + name = name.removeprefix("*") + if Path(name).name != name or any(char not in "0123456789abcdef" for char in digest): + raise CandidateLifecycleError( + f"unsafe SHA256 manifest line {line_number}: {raw_line!r}" + ) + if name in expected: + raise CandidateLifecycleError(f"duplicate SHA256 manifest entry: {name}") + expected[name] = digest + + archives = sorted( + path for path in dist_dir.iterdir() if path.suffix == ".whl" or path.name.endswith(".tar.gz") + ) + names = {path.name for path in archives} + if len([path for path in archives if path.suffix == ".whl"]) != 1: + raise CandidateLifecycleError("the candidate must contain exactly one wheel") + if len([path for path in archives if path.name.endswith(".tar.gz")]) != 1: + raise CandidateLifecycleError("the candidate must contain exactly one source distribution") + if names != set(expected): + raise CandidateLifecycleError( + "the SHA256 manifest and candidate archives differ: " + f"manifest={sorted(expected)}, archives={sorted(names)}" + ) + for archive in archives: + actual = _sha256(archive) + if actual != expected[archive.name]: + raise CandidateLifecycleError( + f"SHA256 mismatch for {archive.name}: expected {expected[archive.name]}, got {actual}" + ) + return expected + + +def _venv_python(environment: Path) -> Path: + if os.name == "nt": + return environment / "Scripts" / "python.exe" + return environment / "bin" / "python" + + +def _clean_environment() -> dict[str, str]: + environment = os.environ.copy() + environment.pop("PYTHONPATH", None) + environment["PYTHONNOUSERSITE"] = "1" + environment["PIP_DISABLE_PIP_VERSION_CHECK"] = "1" + return environment + + +def run_lifecycle(wheel: Path, output: Path, *, candidate_sha: str) -> dict[str, object]: + """Run a network-resolved clean install and uninstall lifecycle.""" + + with tempfile.TemporaryDirectory(prefix="openadapt-capture-candidate-") as temporary: + root = Path(temporary) + environment_dir = root / "venv" + venv.EnvBuilder(with_pip=True, clear=True).create(environment_dir) + python = _venv_python(environment_dir) + environment = _clean_environment() + + subprocess.run( + [ + str(python), + "-m", + "pip", + "install", + "--no-input", + str(wheel.resolve()), + ], + cwd=root, + env=environment, + check=True, + ) + inspection = subprocess.run( + [ + str(python), + "-c", + ( + "import json; " + "from importlib.metadata import distribution; " + "from openadapt_capture import CaptureSession, Recorder; " + "from openadapt_capture.cli import main; " + "dist=distribution('openadapt-capture'); " + "eps=[ep for ep in dist.entry_points " + "if ep.group == 'console_scripts' and ep.name == 'capture']; " + "assert len(eps) == 1 and eps[0].value == 'openadapt_capture.cli:main'; " + "print(json.dumps({'version': dist.version, " + "'capture_session': CaptureSession.__name__, " + "'recorder': Recorder.__name__, 'cli': main.__name__}, sort_keys=True))" + ), + ], + cwd=root, + env=environment, + check=True, + capture_output=True, + text=True, + ) + inspected = json.loads(inspection.stdout) + subprocess.run( + [ + str(python), + "-m", + "openadapt_capture.cli", + "--help", + ], + cwd=root, + env=environment, + check=True, + capture_output=True, + text=True, + ) + subprocess.run( + [str(python), "-m", "pip", "uninstall", "--yes", "openadapt-capture"], + cwd=root, + env=environment, + check=True, + ) + removed = subprocess.run( + [ + str(python), + "-c", + ( + "import importlib.util, sys; " + "sys.exit(0 if importlib.util.find_spec('openadapt_capture') is None else 1)" + ), + ], + cwd=root, + env=environment, + check=False, + ) + if removed.returncode != 0: + raise CandidateLifecycleError("openadapt_capture remained importable after uninstall") + + evidence: dict[str, object] = { + "schema_version": 1, + "candidate_sha": candidate_sha, + "wheel": wheel.name, + "wheel_sha256": _sha256(wheel), + "platform": sys.platform, + "python": sys.version.split()[0], + "installed_version": inspected["version"], + "imports": { + "CaptureSession": inspected["capture_session"], + "Recorder": inspected["recorder"], + }, + "cli_entry_point": "openadapt_capture.cli:main", + "uninstall_verified": True, + } + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(json.dumps(evidence, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return evidence + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--dist", type=Path, required=True) + parser.add_argument("--manifest", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--candidate-sha", required=True) + args = parser.parse_args() + if len(args.candidate_sha) != 40 or any( + char not in "0123456789abcdef" for char in args.candidate_sha + ): + raise SystemExit("--candidate-sha must be a lowercase 40-character Git commit SHA") + + verify_manifest(args.dist, args.manifest) + wheels = list(args.dist.glob("*.whl")) + evidence = run_lifecycle( + wheels[0], args.output, candidate_sha=args.candidate_sha + ) + print(json.dumps(evidence, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/scripts/check_changelog.py b/scripts/check_changelog.py new file mode 100644 index 0000000..c4781a0 --- /dev/null +++ b/scripts/check_changelog.py @@ -0,0 +1,216 @@ +#!/usr/bin/env python3 +"""Fail closed when release metadata and the maintained changelog disagree.""" + +from __future__ import annotations + +import argparse +import re +import subprocess +from dataclasses import dataclass +from datetime import date +from pathlib import Path + +try: + import tomllib +except ModuleNotFoundError: # Python 3.10 test matrix. + import tomli as tomllib + +REPOSITORY = "OpenAdaptAI/openadapt-capture" +INSERTION_FLAG = "" +RELEASE_HEADING = re.compile( + r"^## v(?P\d+\.\d+\.\d+) \((?P\d{4}-\d{2}-\d{2})\)$", + re.MULTILINE, +) +STABLE_TAG = re.compile(r"^v(?P\d+\.\d+\.\d+)$") + + +class ChangelogContractError(ValueError): + """The changelog cannot prove the package's release history.""" + + +@dataclass(frozen=True) +class Release: + version: str + version_key: tuple[int, int, int] + released_on: date + body: str + + +def _version_key(version: str) -> tuple[int, int, int]: + try: + major, minor, patch = version.split(".") + return int(major), int(minor), int(patch) + except (TypeError, ValueError) as exc: + raise ChangelogContractError(f"invalid stable version: {version!r}") from exc + + +def parse_releases(changelog: str) -> list[Release]: + """Parse all stable release sections and reject malformed release headings.""" + heading_lines = [line for line in changelog.splitlines() if line.startswith("## v")] + matches = list(RELEASE_HEADING.finditer(changelog)) + if len(matches) != len(heading_lines): + valid_lines = {match.group(0) for match in matches} + malformed = [line for line in heading_lines if line not in valid_lines] + raise ChangelogContractError( + f"malformed release heading(s): {', '.join(repr(line) for line in malformed)}" + ) + if not matches: + raise ChangelogContractError("the changelog has no stable release sections") + + releases: list[Release] = [] + for index, match in enumerate(matches): + body_end = matches[index + 1].start() if index + 1 < len(matches) else len(changelog) + version = match.group("version") + try: + released_on = date.fromisoformat(match.group("date")) + except ValueError as exc: + raise ChangelogContractError( + f"v{version} has an invalid release date: {match.group('date')!r}" + ) from exc + releases.append( + Release( + version=version, + version_key=_version_key(version), + released_on=released_on, + body=changelog[match.end() : body_end], + ) + ) + return releases + + +def _semantic_release_config(pyproject: str) -> tuple[str, dict[str, object]]: + try: + document = tomllib.loads(pyproject) + project = document["project"] + semantic_release = document["tool"]["semantic_release"] + except (tomllib.TOMLDecodeError, KeyError, TypeError) as exc: + raise ChangelogContractError( + "pyproject.toml has no valid project and semantic-release configuration" + ) from exc + version = project.get("version") + if not isinstance(version, str): + raise ChangelogContractError("project.version must be a string") + return version, semantic_release + + +def validate_documents(changelog: str, pyproject: str) -> list[Release]: + """Validate the files that are available both in Git and in the source archive.""" + project_version, semantic_release = _semantic_release_config(pyproject) + releases = parse_releases(changelog) + versions = [release.version for release in releases] + if len(set(versions)) != len(versions): + raise ChangelogContractError("the changelog contains a duplicate release version") + if [release.version_key for release in releases] != sorted( + (release.version_key for release in releases), reverse=True + ): + raise ChangelogContractError("release sections are not in descending version order") + if releases[0].version != project_version: + raise ChangelogContractError( + "the newest changelog release does not match project.version: " + f"v{releases[0].version} != v{project_version}" + ) + + if changelog.count(INSERTION_FLAG) != 1: + raise ChangelogContractError( + f"the changelog must contain exactly one {INSERTION_FLAG!r} insertion flag" + ) + if changelog.index(INSERTION_FLAG) > changelog.index("## v"): + raise ChangelogContractError("the changelog insertion flag must precede every release") + + version_toml = semantic_release.get("version_toml") + if not isinstance(version_toml, list) or "pyproject.toml:project.version" not in version_toml: + raise ChangelogContractError( + "semantic-release must update pyproject.toml:project.version" + ) + changelog_config = semantic_release.get("changelog") + if not isinstance(changelog_config, dict): + raise ChangelogContractError("semantic-release changelog configuration is missing") + if changelog_config.get("mode") != "update": + raise ChangelogContractError("semantic-release changelog mode must be 'update'") + if changelog_config.get("insertion_flag") != INSERTION_FLAG: + raise ChangelogContractError("semantic-release must use the maintained insertion flag") + templates = changelog_config.get("default_templates") + if not isinstance(templates, dict): + raise ChangelogContractError("semantic-release default changelog templates are missing") + if templates.get("changelog_file") != "CHANGELOG.md": + raise ChangelogContractError("semantic-release must update CHANGELOG.md") + if templates.get("output_format") != "md": + raise ChangelogContractError("semantic-release changelog output must be Markdown") + + # v1.0.0 is the first release that was absent from the maintained file. + # Require generated release content and an exact adjacent-tag comparison + # for every backfilled and future 1.x-or-later release. + for index, release in enumerate(releases): + if release.version_key < (1, 0, 0): + continue + if "\n### " not in release.body or not re.search(r"(?m)^- ", release.body): + raise ChangelogContractError(f"v{release.version} has no categorized release notes") + if index + 1 >= len(releases): + raise ChangelogContractError( + f"v{release.version} has no prior release for its comparison link" + ) + previous = releases[index + 1].version + comparison = ( + f"https://github.com/{REPOSITORY}/compare/v{previous}...v{release.version}" + ) + if comparison not in release.body: + raise ChangelogContractError( + f"v{release.version} is missing its exact adjacent-tag comparison link" + ) + return releases + + +def validate_git_tags(releases: list[Release], repository: Path) -> None: + """Require every stable Git tag to have exactly one changelog section.""" + try: + result = subprocess.run( + ["git", "tag", "--list", "v*"], + cwd=repository, + check=True, + capture_output=True, + text=True, + ) + except (OSError, subprocess.CalledProcessError) as exc: + raise ChangelogContractError("cannot read the repository's release tags") from exc + tags = { + match.group("version") + for line in result.stdout.splitlines() + if (match := STABLE_TAG.fullmatch(line)) is not None + } + if not tags: + raise ChangelogContractError( + "no stable Git tags are available; fetch the complete tag history" + ) + documented = {release.version for release in releases} + missing = sorted(tags - documented, key=_version_key) + if missing: + raise ChangelogContractError( + "stable Git tag(s) are absent from CHANGELOG.md: " + + ", ".join(f"v{version}" for version in missing) + ) + untagged = sorted(documented - tags, key=_version_key) + if untagged: + raise ChangelogContractError( + "CHANGELOG.md contains untagged stable release(s): " + + ", ".join(f"v{version}" for version in untagged) + ) + + +def check_repository(repository: Path) -> list[Release]: + changelog = (repository / "CHANGELOG.md").read_text(encoding="utf-8") + pyproject = (repository / "pyproject.toml").read_text(encoding="utf-8") + releases = validate_documents(changelog, pyproject) + validate_git_tags(releases, repository) + return releases + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--repository", type=Path, default=Path(__file__).resolve().parents[1]) + args = parser.parse_args() + releases = check_repository(args.repository.resolve()) + print(f"verified CHANGELOG.md through v{releases[0].version}") + + +if __name__ == "__main__": + main() diff --git a/scripts/check_display_topology.py b/scripts/check_display_topology.py new file mode 100644 index 0000000..0ed5869 --- /dev/null +++ b/scripts/check_display_topology.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +"""Prove a stable multi-monitor topology on an interactive qualification rig.""" + +from __future__ import annotations + +import argparse +import json +import time +from pathlib import Path +from typing import Any, Callable + +from openadapt_capture.desktop_capture import DesktopCaptureScope + + +class DisplayTopologyError(RuntimeError): + """The qualification display topology does not meet the contract.""" + + +def qualify_topology( + read_snapshot: Callable[[], dict[str, Any]], + *, + minimum_monitors: int, + samples: int, + interval_seconds: float, +) -> dict[str, Any]: + if minimum_monitors < 1: + raise DisplayTopologyError("minimum_monitors must be positive") + if samples < 2: + raise DisplayTopologyError("samples must be at least two") + + snapshots: list[dict[str, Any]] = [] + for index in range(samples): + snapshot = read_snapshot() + if snapshot.get("coordinate_space") != "virtual_desktop_pixels": + raise DisplayTopologyError("the coordinate space is not virtual_desktop_pixels") + monitor_count = snapshot.get("monitor_count") + monitors = snapshot.get("monitors") + if ( + isinstance(monitor_count, bool) + or not isinstance(monitor_count, int) + or monitor_count < minimum_monitors + ): + raise DisplayTopologyError( + f"the rig has {monitor_count!r} monitors; at least {minimum_monitors} are required" + ) + if not isinstance(monitors, list) or len(monitors) != monitor_count: + raise DisplayTopologyError("the monitor inventory does not match monitor_count") + snapshots.append(snapshot) + if index + 1 < samples: + time.sleep(interval_seconds) + + if any(snapshot != snapshots[0] for snapshot in snapshots[1:]): + raise DisplayTopologyError("the display topology changed during qualification") + return { + "schema_version": 1, + "samples": samples, + "stable": True, + "topology": snapshots[0], + } + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--minimum-monitors", type=int, default=2) + parser.add_argument("--samples", type=int, default=3) + parser.add_argument("--interval-seconds", type=float, default=1.0) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + + evidence = qualify_topology( + lambda: DesktopCaptureScope.current().snapshot(), + minimum_monitors=args.minimum_monitors, + samples=args.samples, + interval_seconds=args.interval_seconds, + ) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(evidence, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps(evidence, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/scripts/check_junit_no_skips.py b/scripts/check_junit_no_skips.py new file mode 100644 index 0000000..da4ee7c --- /dev/null +++ b/scripts/check_junit_no_skips.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +"""Reject an empty, skipped, failed, or errored qualification JUnit report.""" + +from __future__ import annotations + +import argparse +import xml.etree.ElementTree as ET +from pathlib import Path + + +class JUnitQualificationError(RuntimeError): + """A qualification result is incomplete.""" + + +def check_reports(paths: list[Path]) -> dict[str, int]: + if not paths: + raise JUnitQualificationError("no JUnit report was supplied") + totals = {"tests": 0, "skipped": 0, "failures": 0, "errors": 0} + for path in paths: + try: + root = ET.parse(path).getroot() + except (OSError, ET.ParseError) as exc: + raise JUnitQualificationError(f"cannot read JUnit report {path}: {exc}") from exc + cases = root.findall(".//testcase") + totals["tests"] += len(cases) + totals["skipped"] += sum(case.find("skipped") is not None for case in cases) + totals["failures"] += sum(case.find("failure") is not None for case in cases) + totals["errors"] += sum(case.find("error") is not None for case in cases) + if totals["tests"] == 0: + raise JUnitQualificationError("the qualification ran zero tests") + if totals["skipped"]: + raise JUnitQualificationError( + f"the qualification skipped {totals['skipped']} of {totals['tests']} tests" + ) + if totals["failures"] or totals["errors"]: + raise JUnitQualificationError( + "the qualification contains " + f"{totals['failures']} failures and {totals['errors']} errors" + ) + return totals + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("reports", nargs="+", type=Path) + args = parser.parse_args() + totals = check_reports(args.reports) + print( + "qualification JUnit is complete: " + f"{totals['tests']} tests, no skips, no failures, no errors" + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/check_release_ci.py b/scripts/check_release_ci.py new file mode 100644 index 0000000..2338bdd --- /dev/null +++ b/scripts/check_release_ci.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python3 +"""Require exact-commit test and production-qualification evidence.""" + +from __future__ import annotations + +import argparse +import json +import os +import time +import urllib.error +import urllib.parse +import urllib.request +from dataclasses import dataclass +from typing import Any, Callable + +EXPECTED_QUALIFICATION_JOBS = frozenset( + { + "Build candidate distributions", + "Clean candidate wheel (ubuntu-latest)", + "Clean candidate wheel (macos-latest)", + "Clean candidate wheel (windows-latest)", + "Interactive qualification (Linux X64)", + "Interactive qualification (macOS ARM64)", + "Interactive qualification (Windows X64)", + } +) +ACTIVE_STATES = frozenset({"queued", "in_progress", "waiting", "pending", "requested"}) + + +class ReleaseEvidenceError(RuntimeError): + """The exact release candidate does not have complete successful evidence.""" + + +class EvidencePending(RuntimeError): + """The exact release candidate can still obtain the required evidence.""" + + +@dataclass(frozen=True) +class WorkflowRequirement: + file_name: str + event: str + + +REQUIREMENTS = ( + WorkflowRequirement("test.yml", "push"), + WorkflowRequirement("production-qualification.yml", "workflow_dispatch"), +) + + +def _github_get(url: str, token: str) -> dict[str, Any]: + request = urllib.request.Request( + url, + headers={ + "Accept": "application/vnd.github+json", + "Authorization": f"Bearer {token}", + "X-GitHub-Api-Version": "2022-11-28", + "User-Agent": "openadapt-capture-release-evidence/1", + }, + ) + try: + with urllib.request.urlopen(request, timeout=30) as response: + return json.loads(response.read(4 * 1024 * 1024)) + except (urllib.error.URLError, OSError, ValueError, TypeError) as exc: + raise EvidencePending(f"GitHub evidence query failed: {exc}") from exc + + +def select_exact_run( + runs: list[dict[str, Any]], *, sha: str, event: str +) -> dict[str, Any]: + exact = [ + run + for run in runs + if run.get("head_sha") == sha and run.get("event") == event + ] + if not exact: + raise EvidencePending(f"no {event} workflow run exists for exact commit {sha}") + exact.sort(key=lambda run: (run.get("created_at") or "", int(run.get("id") or 0))) + run = exact[-1] + status = run.get("status") + conclusion = run.get("conclusion") + if status in ACTIVE_STATES or status != "completed": + raise EvidencePending( + f"workflow run {run.get('id')} for {sha} is {status}/{conclusion}" + ) + if conclusion != "success": + raise ReleaseEvidenceError( + f"workflow run {run.get('id')} for {sha} concluded {conclusion}" + ) + return run + + +def validate_qualification_jobs(jobs: list[dict[str, Any]]) -> None: + names = [str(job.get("name")) for job in jobs] + duplicates = sorted({name for name in names if names.count(name) > 1}) + if duplicates: + raise ReleaseEvidenceError( + f"production qualification has duplicate jobs: {duplicates}" + ) + actual = set(names) + if actual != EXPECTED_QUALIFICATION_JOBS: + missing = sorted(EXPECTED_QUALIFICATION_JOBS - actual) + unexpected = sorted(actual - EXPECTED_QUALIFICATION_JOBS) + raise ReleaseEvidenceError( + "production qualification job set differs from the release contract: " + f"missing={missing}, unexpected={unexpected}" + ) + incomplete = [ + f"{job.get('name')}={job.get('status')}/{job.get('conclusion')}" + for job in jobs + if job.get("status") != "completed" or job.get("conclusion") != "success" + ] + if incomplete: + raise ReleaseEvidenceError( + "production qualification has incomplete jobs: " + ", ".join(incomplete) + ) + + +def check_once( + *, + repository: str, + sha: str, + token: str, + get_json: Callable[[str, str], dict[str, Any]] = _github_get, +) -> dict[str, int]: + base = f"https://api.github.com/repos/{repository}" + selected: dict[str, dict[str, Any]] = {} + for requirement in REQUIREMENTS: + query = urllib.parse.urlencode( + {"head_sha": sha, "event": requirement.event, "per_page": "100"} + ) + payload = get_json( + f"{base}/actions/workflows/{requirement.file_name}/runs?{query}", token + ) + runs = payload.get("workflow_runs") + if not isinstance(runs, list): + raise EvidencePending( + f"GitHub returned no workflow_runs list for {requirement.file_name}" + ) + selected[requirement.file_name] = select_exact_run( + runs, sha=sha, event=requirement.event + ) + + qualification = selected["production-qualification.yml"] + jobs_payload = get_json( + f"{base}/actions/runs/{qualification['id']}/jobs?filter=latest&per_page=100", + token, + ) + jobs = jobs_payload.get("jobs") + if not isinstance(jobs, list): + raise EvidencePending("GitHub returned no production qualification jobs list") + validate_qualification_jobs(jobs) + return {name: int(run["id"]) for name, run in selected.items()} + + +def wait_for_evidence( + *, repository: str, sha: str, token: str, timeout_seconds: int, interval_seconds: int +) -> dict[str, int]: + deadline = time.monotonic() + timeout_seconds + while True: + try: + return check_once(repository=repository, sha=sha, token=token) + except EvidencePending as exc: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise ReleaseEvidenceError( + f"exact-commit release evidence did not complete within {timeout_seconds}s: {exc}" + ) from exc + print(f"waiting for exact-commit evidence: {exc}", flush=True) + time.sleep(min(interval_seconds, remaining)) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--repository", required=True) + parser.add_argument("--sha", required=True) + parser.add_argument("--timeout-seconds", type=int, default=2700) + parser.add_argument("--interval-seconds", type=int, default=10) + args = parser.parse_args() + if len(args.sha) != 40 or any(char not in "0123456789abcdef" for char in args.sha): + raise SystemExit("--sha must be a lowercase 40-character Git commit SHA") + token = os.environ.get("GH_TOKEN") or os.environ.get("GITHUB_TOKEN") + if not token: + raise SystemExit("GH_TOKEN or GITHUB_TOKEN is required") + evidence = wait_for_evidence( + repository=args.repository, + sha=args.sha, + token=token, + timeout_seconds=args.timeout_seconds, + interval_seconds=args.interval_seconds, + ) + print(f"exact-commit release evidence is complete: {json.dumps(evidence, sort_keys=True)}") + + +if __name__ == "__main__": + main() diff --git a/scripts/verify_distribution.py b/scripts/verify_distribution.py index ebf8e03..970112a 100644 --- a/scripts/verify_distribution.py +++ b/scripts/verify_distribution.py @@ -8,8 +8,13 @@ import zipfile from pathlib import Path -FORBIDDEN_DEPENDENCIES = ("oa-atomacos", "pynput") -FORBIDDEN_SOURCE_TOKENS = ("oa_atomacos", "pynput") +if __package__: + from .check_changelog import validate_documents +else: # Direct execution: python scripts/verify_distribution.py + from check_changelog import validate_documents + +FORBIDDEN_DEPENDENCIES = ("oa-atomacos", "pynput", "websockets") +FORBIDDEN_SOURCE_TOKENS = ("oa_atomacos", "pynput", "EXECUTE_ACTION") FORBIDDEN_ARCHIVE_PATHS = ( ".env.example", ".github/", @@ -19,6 +24,7 @@ "docs/whisper-integration-plan.md", "scripts/", "tests/", + "openadapt_capture/browser_bridge.py", ) @@ -26,17 +32,14 @@ def _archive_files(path: Path) -> dict[str, bytes]: if path.suffix == ".whl": with zipfile.ZipFile(path) as archive: return { - name: archive.read(name) - for name in archive.namelist() - if not name.endswith("/") + name: archive.read(name) for name in archive.namelist() if not name.endswith("/") } if path.name.endswith(".tar.gz"): with tarfile.open(path, "r:gz") as archive: return { member.name: extracted.read() for member in archive.getmembers() - if member.isfile() - and (extracted := archive.extractfile(member)) is not None + if member.isfile() and (extracted := archive.extractfile(member)) is not None } raise ValueError(f"Unsupported distribution archive: {path}") @@ -51,7 +54,8 @@ def _relative_archive_name(name: str) -> str: def verify_distribution(path: Path) -> None: files = _archive_files(path) - relative_names = {_relative_archive_name(name) for name in files} + relative_files = {_relative_archive_name(name): content for name, content in files.items()} + relative_names = set(relative_files) assert any(Path(name).name == "LICENSE" for name in files), ( f"{path}: MIT LICENSE file is missing" ) @@ -64,6 +68,7 @@ def verify_distribution(path: Path) -> None: if path.name.endswith(".tar.gz"): required_source_files = { + "CHANGELOG.md", "LICENSE", "README.md", "pyproject.toml", @@ -71,6 +76,13 @@ def verify_distribution(path: Path) -> None: } missing = required_source_files - relative_names assert not missing, f"{path}: required source files are missing: {sorted(missing)}" + try: + validate_documents( + relative_files["CHANGELOG.md"].decode("utf-8"), + relative_files["pyproject.toml"].decode("utf-8"), + ) + except (UnicodeDecodeError, ValueError) as exc: + raise AssertionError(f"{path}: invalid changelog contract: {exc}") from exc metadata_files = [ content.decode("utf-8") @@ -104,8 +116,7 @@ def verify_distribution(path: Path) -> None: python_sources = "\n".join( content.decode("utf-8") for name, content in files.items() - if name.endswith(".py") - and "/openadapt_capture/" in f"/{name}" + if name.endswith(".py") and "/openadapt_capture/" in f"/{name}" ) for token in FORBIDDEN_SOURCE_TOKENS: assert token not in python_sources, ( diff --git a/tests/test_browser_release_boundary.py b/tests/test_browser_release_boundary.py new file mode 100644 index 0000000..15e3b29 --- /dev/null +++ b/tests/test_browser_release_boundary.py @@ -0,0 +1,112 @@ +"""Production artifact boundary for the repository-only browser prototype.""" + +from __future__ import annotations + +import socket +import time +import zipfile + +import pytest + +import openadapt_capture +from openadapt_capture.browser_events import BrowserClickEvent +from openadapt_capture.capture import CaptureSession +from openadapt_capture.cli import record as cli_record +from openadapt_capture.db import create_db +from openadapt_capture.db.crud import insert_browser_event, insert_recording +from openadapt_capture.recorder import Recorder +from scripts.verify_distribution import verify_distribution + + +def test_public_api_has_no_browser_bridge_or_replay_exports() -> None: + forbidden = { + "BrowserBridge", + "BrowserMode", + "BrowserEventRecord", + "run_browser_bridge", + } + + assert forbidden.isdisjoint(openadapt_capture.__all__) + for name in forbidden: + assert not hasattr(openadapt_capture, name) + + +def test_legacy_browser_opt_in_fails_before_socket_bind(monkeypatch, tmp_path) -> None: + bind_calls: list[object] = [] + + def reject_bind(self, address): + bind_calls.append(address) + raise AssertionError("legacy browser opt-in attempted to bind a listener") + + monkeypatch.setattr(socket.socket, "bind", reject_bind) + + with pytest.raises(ValueError, match="openadapt-flow Playwright"): + Recorder(str(tmp_path / "direct"), capture_browser_events=True) + with pytest.raises(SystemExit) as exc_info: + cli_record(str(tmp_path / "cli"), browser_events=True) + + assert exc_info.value.code == 2 + assert bind_calls == [] + + +def test_distribution_validator_rejects_repository_browser_bridge(tmp_path) -> None: + wheel = tmp_path / "openadapt_capture-1.2.2-py3-none-any.whl" + with zipfile.ZipFile(wheel, "w") as archive: + archive.writestr("openadapt_capture/browser_bridge.py", "EXECUTE_ACTION = 1\n") + archive.writestr("openadapt_capture-1.2.2.dist-info/licenses/LICENSE", "MIT\n") + archive.writestr("openadapt_capture-1.2.2.dist-info/METADATA", "Name: openadapt-capture\n") + + with pytest.raises(AssertionError, match="repository-only path"): + verify_distribution(wheel) + + +def test_passive_legacy_browser_event_still_loads(tmp_path) -> None: + engine, session_factory = create_db(str(tmp_path / "recording.db")) + session = session_factory() + timestamp = time.time() + recording = insert_recording( + session, + { + "timestamp": timestamp, + "monitor_width": 1920, + "monitor_height": 1080, + "double_click_interval_seconds": 0.5, + "double_click_distance_pixels": 5, + "platform": "test", + "task_description": "legacy browser data", + }, + ) + insert_browser_event( + session, + recording, + timestamp + 1, + { + "message": { + "type": "DOM_EVENT", + "tabId": 7, + "payload": { + "eventType": "click", + "url": "https://example.invalid/legacy", + "clientX": 20, + "clientY": 30, + "pageX": 20, + "pageY": 30, + "element": { + "role": "button", + "name": "Continue", + "bbox": {"x": 10, "y": 20, "width": 40, "height": 20}, + "xpath": "/html/body/button", + }, + }, + } + }, + ) + session.close() + engine.dispose() + + with CaptureSession.load(tmp_path) as capture: + events = capture.browser_events() + assert len(events) == 1 + assert isinstance(events[0], BrowserClickEvent) + assert events[0].element.name == "Continue" + assert events[0].tab_id == 7 diff --git a/tests/test_changelog_contract.py b/tests/test_changelog_contract.py new file mode 100644 index 0000000..9bd0276 --- /dev/null +++ b/tests/test_changelog_contract.py @@ -0,0 +1,127 @@ +from __future__ import annotations + +import io +import tarfile +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from scripts.check_changelog import ( + ChangelogContractError, + validate_documents, + validate_git_tags, +) +from scripts.verify_distribution import verify_distribution + +REPOSITORY = Path(__file__).resolve().parents[1] + + +def _documents() -> tuple[str, str]: + return ( + (REPOSITORY / "CHANGELOG.md").read_text(encoding="utf-8"), + (REPOSITORY / "pyproject.toml").read_text(encoding="utf-8"), + ) + + +def _add_tar_file(archive: tarfile.TarFile, name: str, content: str) -> None: + data = content.encode("utf-8") + member = tarfile.TarInfo(name) + member.size = len(data) + archive.addfile(member, io.BytesIO(data)) + + +def _project_with_next_patch(changelog: str, pyproject: str) -> tuple[str, str]: + current = validate_documents(changelog, pyproject)[0].version + major, minor, patch = (int(part) for part in current.split(".")) + next_version = f"{major}.{minor}.{patch + 1}" + updated = pyproject.replace( + f'version = "{current}"', + f'version = "{next_version}"', + 1, + ) + return updated, next_version + + +def test_changelog_matches_project_version_and_release_config() -> None: + changelog, pyproject = _documents() + releases = validate_documents(changelog, pyproject) + + assert { + "1.0.0", + "1.0.1", + "1.0.2", + "1.0.3", + "1.0.4", + "1.1.0", + "1.1.1", + "1.2.0", + "1.2.1", + "1.2.2", + } <= {release.version for release in releases} + assert releases[-1].version == "0.1.0" + + +def test_changelog_refuses_a_project_version_without_release_notes() -> None: + changelog, pyproject = _documents() + pyproject, _ = _project_with_next_patch(changelog, pyproject) + + with pytest.raises( + ChangelogContractError, + match="newest changelog release does not match project.version", + ): + validate_documents(changelog, pyproject) + + +def test_changelog_refuses_a_missing_semantic_release_insertion_flag() -> None: + changelog, pyproject = _documents() + changelog = changelog.replace("", "", 1) + + with pytest.raises(ChangelogContractError, match="exactly one"): + validate_documents(changelog, pyproject) + + +def test_changelog_refuses_a_missing_adjacent_tag_comparison() -> None: + changelog, pyproject = _documents() + changelog = changelog.replace("compare/v1.2.1...v1.2.2", "compare/v1.2.0...v1.2.2", 1) + + with pytest.raises(ChangelogContractError, match="exact adjacent-tag comparison"): + validate_documents(changelog, pyproject) + + +def test_git_tag_contract_refuses_an_incomplete_tag_inventory(monkeypatch) -> None: + changelog, pyproject = _documents() + releases = validate_documents(changelog, pyproject) + incomplete_tags = "\n".join(f"v{release.version}" for release in releases[1:]) + monkeypatch.setattr( + "scripts.check_changelog.subprocess.run", + lambda *args, **kwargs: SimpleNamespace(stdout=incomplete_tags), + ) + + with pytest.raises(ChangelogContractError, match="untagged stable release"): + validate_git_tags(releases, REPOSITORY) + + +def test_source_distribution_refuses_a_version_without_release_notes(tmp_path: Path) -> None: + changelog, pyproject = _documents() + pyproject, next_version = _project_with_next_patch(changelog, pyproject) + archive_path = tmp_path / f"openadapt_capture-{next_version}.tar.gz" + root = f"openadapt_capture-{next_version}" + files = { + "CHANGELOG.md": changelog, + "LICENSE": "MIT License\n", + "README.md": "# OpenAdapt Capture\n", + "pyproject.toml": pyproject, + "openadapt_capture/__init__.py": "", + f"openadapt_capture-{next_version}.dist-info/PKG-INFO": ( + "Metadata-Version: 2.4\n" + "Name: openadapt-capture\n" + f"Version: {next_version}\n" + ), + } + with tarfile.open(archive_path, "w:gz") as archive: + for name, content in files.items(): + _add_tar_file(archive, f"{root}/{name}", content) + + with pytest.raises(AssertionError, match="invalid changelog contract"): + verify_distribution(archive_path) diff --git a/tests/test_desktop_capture.py b/tests/test_desktop_capture.py new file mode 100644 index 0000000..0b455f8 --- /dev/null +++ b/tests/test_desktop_capture.py @@ -0,0 +1,149 @@ +"""Virtual-desktop coordinate and persistence contracts.""" + +from __future__ import annotations + +import json + +import pytest + +from openadapt_capture import utils +from openadapt_capture.capture import CaptureSession +from openadapt_capture.db import create_db, crud +from openadapt_capture.desktop_capture import DesktopCaptureError, DesktopCaptureScope +from openadapt_capture.recorder import create_recording, trigger_action_event + + +def _two_monitor_scope() -> DesktopCaptureScope: + return DesktopCaptureScope.from_monitors( + [ + {"left": -1920, "top": 0, "width": 4480, "height": 1440}, + {"left": -1920, "top": 0, "width": 1920, "height": 1080}, + {"left": 0, "top": 0, "width": 2560, "height": 1440}, + ] + ) + + +def test_multiple_monitor_scope_translates_negative_global_origin() -> None: + scope = _two_monitor_scope() + + assert scope.translate(-1920, 0) == (0, 0) + assert scope.translate(-1, 100) == (1919, 100) + assert scope.translate(0, 100) == (1920, 100) + assert scope.translate(2559, 1439) == (4479, 1439) + + +def test_live_scope_rejects_same_size_origin_and_layout_change() -> None: + original = [ + {"left": -1920, "top": 0, "width": 4480, "height": 1440}, + {"left": -1920, "top": 0, "width": 1920, "height": 1080}, + {"left": 0, "top": 0, "width": 2560, "height": 1440}, + ] + current = list(original) + scope = DesktopCaptureScope.from_monitors( + original, + topology_reader=lambda: current, + ) + assert scope.translate(-100, 50) == (1820, 50) + + # The combined viewport keeps the same size. Only the global origin and + # physical layout move. A video-frame dimension check cannot detect this. + current = [ + {"left": 0, "top": 0, "width": 4480, "height": 1440}, + {"left": 0, "top": 0, "width": 1920, "height": 1080}, + {"left": 1920, "top": 0, "width": 2560, "height": 1440}, + ] + + with pytest.raises(DesktopCaptureError, match="topology changed"): + scope.assert_current(force=True) + + +def test_multiple_monitor_snapshot_is_privacy_safe_geometry() -> None: + assert _two_monitor_scope().snapshot() == { + "coordinate_space": "virtual_desktop_pixels", + "origin": [-1920, 0], + "viewport": [4480, 1440], + "monitor_count": 2, + "monitors": [ + [-1920, 0, 1920, 1080], + [0, 0, 2560, 1440], + ], + } + + +def test_desktop_scope_rejects_missing_physical_monitor() -> None: + with pytest.raises(DesktopCaptureError, match="physical monitor"): + DesktopCaptureScope.from_monitors([{"left": 0, "top": 0, "width": 1920, "height": 1080}]) + + +@pytest.mark.parametrize("value", [True, 1.5, "1"]) +def test_desktop_scope_rejects_coerced_geometry(value) -> None: + with pytest.raises(DesktopCaptureError, match="must be an integer"): + DesktopCaptureScope.from_monitors( + [ + {"left": 0, "top": 0, "width": 1920, "height": 1080}, + {"left": value, "top": 0, "width": 1920, "height": 1080}, + ] + ) + + +def test_desktop_scope_rejects_inconsistent_combined_bounds() -> None: + with pytest.raises(DesktopCaptureError, match="do not span"): + DesktopCaptureScope.from_monitors( + [ + {"left": 0, "top": 0, "width": 3840, "height": 1080}, + {"left": 0, "top": 0, "width": 1920, "height": 1080}, + ] + ) + + +def test_action_event_uses_virtual_desktop_pixels() -> None: + import queue + + utils.set_start_time() + events = queue.Queue() + trigger_action_event( + events, + {"name": "click", "mouse_x": -100.0, "mouse_y": 50.0}, + _two_monitor_scope(), + ) + + event = events.get_nowait() + assert event.data["mouse_x"] == 1820.0 + assert event.data["mouse_y"] == 50.0 + + +def test_desktop_capture_metadata_round_trips(tmp_path) -> None: + capture_dir = tmp_path / "capture" + capture_dir.mkdir() + engine, session_factory = create_db(str(capture_dir / "recording.db")) + session = session_factory() + snapshot = _two_monitor_scope().snapshot() + crud.insert_recording( + session, + { + "timestamp": 1.0, + "monitor_width": 4480, + "monitor_height": 1440, + "double_click_interval_seconds": 0.5, + "double_click_distance_pixels": 5, + "platform": "test", + "task_description": "two monitors", + "config": json.loads(json.dumps({"capture_desktop": snapshot})), + }, + ) + session.close() + engine.dispose() + + with CaptureSession.load(capture_dir) as capture: + assert capture.desktop_capture == snapshot + assert capture.window_capture is None + + +def test_recording_rejects_ambiguous_coordinate_scopes(tmp_path) -> None: + with pytest.raises(ValueError, match="both window and desktop"): + create_recording( + "ambiguous", + str(tmp_path / "capture"), + window_capture_info={"coordinate_space": "window_pixels"}, + desktop_capture_info={"coordinate_space": "virtual_desktop_pixels"}, + ) diff --git a/tests/test_performance.py b/tests/test_performance.py index b4d04ff..bcf4ba0 100644 --- a/tests/test_performance.py +++ b/tests/test_performance.py @@ -10,9 +10,9 @@ NOTE: The recorder uses multiprocessing.Process for writer tasks. On macOS (Python "spawn" start method) writer processes historically failed to start because each child re-imported modules with display side effects; imports are -side-effect free since 0.5.4, but the spawn path on macOS/Linux is not yet -validated end to end. These tests target Windows (the primary recording -platform) and are exercised in CI on windows-latest. +side-effect free since 0.5.4. Windows runs these tests by default. Interactive +macOS/Linux production-qualification runners can opt in explicitly with +OPENADAPT_CAPTURE_PRODUCTION_QUALIFICATION=1. """ import os @@ -28,15 +28,22 @@ from openadapt_capture.capture import CaptureSession from openadapt_capture.recorder import Recorder -# Skip on non-Windows platforms where the live pipeline is not yet validated +# A non-Windows host must opt in explicitly. This prevents a developer or a +# hosted CI runner from accidentally starting native input injection when it +# does not have a visible desktop and the required OS permissions. The +# production qualification workflow uses controlled, interactive hosts. +_PRODUCTION_QUALIFICATION = ( + os.environ.get("OPENADAPT_CAPTURE_PRODUCTION_QUALIFICATION") == "1" +) +_ON_SUPPORTED_LIVE_PLATFORM = sys.platform == "win32" or ( + _PRODUCTION_QUALIFICATION and sys.platform in ("darwin", "linux") +) _SKIP_REASON = ( - "Live recorder integration tests target Windows (the primary recording " - "platform, exercised in CI on windows-latest). The multiprocessing " - "'spawn' writer path on macOS/Linux is not yet validated end to end; " - "on GitHub macOS runners synthetic input injection also needs Accessibility " - "permissions that cannot be granted." + "Live recorder integration tests run by default only on Windows. " + "Interactive macOS/Linux qualification requires " + "OPENADAPT_CAPTURE_PRODUCTION_QUALIFICATION=1 and the applicable screen, " + "input, and Accessibility permissions." ) -_ON_WINDOWS = sys.platform == "win32" # GitHub-hosted Windows runners execute jobs in a non-interactive session: # SendInput-injected events never reach native low-level hooks in this session, so @@ -129,7 +136,7 @@ def capture_dir(tmp_path): # --------------------------------------------------------------------------- @pytest.mark.slow -@pytest.mark.skipif(not _ON_WINDOWS, reason=_SKIP_REASON) +@pytest.mark.skipif(not _ON_SUPPORTED_LIVE_PLATFORM, reason=_SKIP_REASON) class TestRecorderIntegration: """Integration tests that run the full recording pipeline.""" diff --git a/tests/test_release_qualification_gates.py b/tests/test_release_qualification_gates.py new file mode 100644 index 0000000..2a6b190 --- /dev/null +++ b/tests/test_release_qualification_gates.py @@ -0,0 +1,188 @@ +"""Tests for the fail-closed production qualification and release gates.""" + +from __future__ import annotations + +import hashlib +from pathlib import Path + +import pytest + +from scripts.candidate_lifecycle import CandidateLifecycleError, verify_manifest +from scripts.check_display_topology import DisplayTopologyError, qualify_topology +from scripts.check_junit_no_skips import JUnitQualificationError, check_reports +from scripts.check_release_ci import ( + EXPECTED_QUALIFICATION_JOBS, + ReleaseEvidenceError, + check_once, + validate_qualification_jobs, +) + + +def _write_manifest(dist: Path, archives: list[Path]) -> Path: + manifest = dist / "SHA256SUMS" + manifest.write_text( + "".join( + f"{hashlib.sha256(path.read_bytes()).hexdigest()} {path.name}\n" + for path in archives + ), + encoding="utf-8", + ) + return manifest + + +def test_candidate_manifest_accounts_for_exact_wheel_and_sdist(tmp_path: Path) -> None: + wheel = tmp_path / "openadapt_capture-1.2.2-py3-none-any.whl" + sdist = tmp_path / "openadapt_capture-1.2.2.tar.gz" + wheel.write_bytes(b"wheel") + sdist.write_bytes(b"sdist") + manifest = _write_manifest(tmp_path, [wheel, sdist]) + + assert set(verify_manifest(tmp_path, manifest)) == {wheel.name, sdist.name} + + +def test_candidate_manifest_rejects_unaccounted_archive(tmp_path: Path) -> None: + wheel = tmp_path / "openadapt_capture-1.2.2-py3-none-any.whl" + sdist = tmp_path / "openadapt_capture-1.2.2.tar.gz" + wheel.write_bytes(b"wheel") + sdist.write_bytes(b"sdist") + manifest = _write_manifest(tmp_path, [wheel]) + + with pytest.raises(CandidateLifecycleError, match="manifest and candidate archives differ"): + verify_manifest(tmp_path, manifest) + + +def test_display_topology_requires_stable_multiple_monitors() -> None: + snapshot = { + "coordinate_space": "virtual_desktop_pixels", + "origin": [-1920, 0], + "viewport": [4480, 1440], + "monitor_count": 2, + "monitors": [[-1920, 0, 1920, 1080], [0, 0, 2560, 1440]], + } + + evidence = qualify_topology( + lambda: snapshot, + minimum_monitors=2, + samples=3, + interval_seconds=0, + ) + + assert evidence["stable"] is True + assert evidence["topology"] == snapshot + + +def test_display_topology_rejects_change_during_qualification() -> None: + snapshots = iter( + [ + { + "coordinate_space": "virtual_desktop_pixels", + "monitor_count": 2, + "monitors": [[0, 0, 10, 10], [10, 0, 10, 10]], + }, + { + "coordinate_space": "virtual_desktop_pixels", + "monitor_count": 2, + "monitors": [[0, 0, 10, 10], [10, 0, 20, 10]], + }, + ] + ) + + with pytest.raises(DisplayTopologyError, match="changed during qualification"): + qualify_topology( + lambda: next(snapshots), + minimum_monitors=2, + samples=2, + interval_seconds=0, + ) + + +def test_junit_gate_rejects_a_skipped_test(tmp_path: Path) -> None: + report = tmp_path / "qualification.xml" + report.write_text( + "" + "", + encoding="utf-8", + ) + + with pytest.raises(JUnitQualificationError, match="skipped 1 of 2"): + check_reports([report]) + + +def test_junit_gate_accepts_complete_tests(tmp_path: Path) -> None: + report = tmp_path / "qualification.xml" + report.write_text( + "" + "", + encoding="utf-8", + ) + + assert check_reports([report]) == { + "tests": 2, + "skipped": 0, + "failures": 0, + "errors": 0, + } + + +def _successful_jobs() -> list[dict[str, str]]: + return [ + {"name": name, "status": "completed", "conclusion": "success"} + for name in sorted(EXPECTED_QUALIFICATION_JOBS) + ] + + +def test_release_gate_rejects_missing_qualification_job() -> None: + jobs = _successful_jobs()[:-1] + + with pytest.raises(ReleaseEvidenceError, match="job set differs"): + validate_qualification_jobs(jobs) + + +def test_release_gate_rejects_skipped_qualification_job() -> None: + jobs = _successful_jobs() + jobs[0] = {**jobs[0], "conclusion": "skipped"} + + with pytest.raises(ReleaseEvidenceError, match="incomplete jobs"): + validate_qualification_jobs(jobs) + + +def test_release_gate_binds_both_workflows_and_jobs_to_exact_sha() -> None: + sha = "a" * 40 + + def get_json(url: str, _token: str): + if "/test.yml/runs?" in url: + return { + "workflow_runs": [ + { + "id": 10, + "head_sha": sha, + "event": "push", + "created_at": "2026-08-18T00:00:00Z", + "status": "completed", + "conclusion": "success", + } + ] + } + if "/production-qualification.yml/runs?" in url: + return { + "workflow_runs": [ + { + "id": 20, + "head_sha": sha, + "event": "workflow_dispatch", + "created_at": "2026-08-18T00:01:00Z", + "status": "completed", + "conclusion": "success", + } + ] + } + if "/actions/runs/20/jobs?" in url: + return {"jobs": _successful_jobs()} + raise AssertionError(f"unexpected URL {url}") + + assert check_once( + repository="OpenAdaptAI/openadapt-capture", + sha=sha, + token="test", + get_json=get_json, + ) == {"test.yml": 10, "production-qualification.yml": 20} diff --git a/tests/test_window_capture.py b/tests/test_window_capture.py index a278d41..0783851 100644 --- a/tests/test_window_capture.py +++ b/tests/test_window_capture.py @@ -14,15 +14,19 @@ """ import os +import queue import sys +import threading import time +from contextlib import contextmanager +from types import SimpleNamespace import pytest from PIL import Image from openadapt_capture.capture import CaptureSession from openadapt_capture.db import create_db, crud -from openadapt_capture.recorder import Recorder +from openadapt_capture.recorder import Recorder, read_screen_events from openadapt_capture.window_capture import ( TargetWindow, WindowCaptureError, @@ -192,8 +196,49 @@ def test_moved_window_flagged_changed(self, scope, fake): def test_resized_window_flagged_changed(self, scope, fake): scope.capture_frame() fake.bounds = (300.0, 150.0, 900.0, 700.0) - _, changed = scope.capture_frame() + image, changed = scope.capture_frame() assert changed is True + assert image.size == (1600, 1200) + state = scope.window_event_data()["state"] + assert state["viewport"] == [1600, 1200] + assert state["source_viewport"] == [1800, 1400] + + def test_resize_letterboxes_and_translates_into_fixed_viewport(self, scope, fake): + scope.capture_frame() + fake.bounds = (300.0, 150.0, 400.0, 600.0) + + image, changed = scope.capture_frame() + + assert changed is True + assert image.size == (1600, 1200) + state = scope.window_event_data()["state"] + assert state["source_viewport"] == [800, 1200] + assert state["content_rect"] == [400, 0, 800, 1200] + assert state["fit_scale"] == 1.0 + assert scope.translate(300.0, 150.0) == (400.0, 0.0) + assert scope.translate(500.0, 450.0) == (800.0, 600.0) + + def test_resize_uses_exact_axis_scales_after_integer_rounding(self, fake): + fake.bounds = (0.0, 0.0, 3.0, 3.0) + images = iter( + [ + Image.new("RGB", (5, 5)), + Image.new("RGB", (4, 3)), + ] + ) + scope = WindowCaptureScope( + WindowTarget(owner="Parallels"), + resolver=fake.resolver, + capturer=lambda _window: next(images), + ) + scope.capture_frame() + scope.capture_frame() + + state = scope.window_event_data()["state"] + assert state["content_rect"] == [0, 0, 5, 4] + assert state["scale_x"] == pytest.approx(5 / 3) + assert state["scale_y"] == pytest.approx(4 / 3) + assert scope.translate(1.5, 1.5) == pytest.approx((2.5, 2.0)) def test_translate_before_first_frame_raises(self, scope): with pytest.raises(WindowCaptureError, match="before the first"): @@ -208,11 +253,43 @@ def test_translate_uses_latest_bounds(self, scope, fake): scope.capture_frame() assert scope.translate(310.0, 170.0) == (420.0, 240.0) + def test_resolve_does_not_mix_new_bounds_with_previous_frame(self, scope, fake): + scope.capture_frame() + assert scope.translate(310.0, 170.0) == (20.0, 40.0) + + fake.bounds = (100.0, 50.0, 800.0, 600.0) + scope.resolve() + + # A resolver poll alone cannot commit geometry. Translation changes + # only after the corresponding frame has been captured. + assert scope.translate(310.0, 170.0) == (20.0, 40.0) + scope.capture_frame() + assert scope.translate(310.0, 170.0) == (420.0, 240.0) + + def test_window_identity_change_terminates_scope(self, scope, fake): + scope.capture_frame() + fake.window_id = 99 + + with pytest.raises(WindowCaptureError, match="changed window identity"): + scope.capture_frame() + def test_missing_window_raises_loudly(self, scope, fake): fake.missing = True with pytest.raises(WindowCaptureError, match="no window matching"): scope.capture_frame() + def test_screen_reader_propagates_capture_failure_without_retry(self, scope, fake): + fake.missing = True + + with pytest.raises(WindowCaptureError, match="no window matching"): + read_screen_events( + queue.Queue(), + threading.Event(), + SimpleNamespace(timestamp=time.time()), + threading.Event(), + window_scope=scope, + ) + def test_window_event_data_matches_window_event_columns(self, scope): scope.capture_frame() data = scope.window_event_data() @@ -234,6 +311,9 @@ def test_window_event_data_matches_window_event_columns(self, scope): assert state["scale"] == 2.0 assert state["bounds"] == [300.0, 150.0, 800.0, 600.0] assert state["viewport"] == [1600, 1200] + assert state["source_viewport"] == [1600, 1200] + assert state["content_rect"] == [0, 0, 1600, 1200] + assert state["fit_scale"] == 1.0 def test_window_event_data_before_frame_raises(self, scope): with pytest.raises(WindowCaptureError): @@ -247,6 +327,8 @@ def test_snapshot_shape(self, scope): assert snap["window_id"] == 42 assert snap["initial_bounds"] == [300.0, 150.0, 800.0, 600.0] assert snap["viewport"] == [1600, 1200] + assert snap["source_viewport"] == [1600, 1200] + assert snap["content_rect"] == [0, 0, 1600, 1200] def test_snapshot_before_frame_has_target_only(self, scope): snap = scope.snapshot() @@ -319,14 +401,12 @@ def test_mouse_action_translated(self, scope): utils.set_start_time() scope.capture_frame() q = queue.Queue() - trigger_action_event( - q, {"name": "click", "mouse_x": 310.0, "mouse_y": 170.0}, scope - ) + trigger_action_event(q, {"name": "click", "mouse_x": 310.0, "mouse_y": 170.0}, scope) (event,) = self._drain(q) assert event.data["mouse_x"] == 20.0 assert event.data["mouse_y"] == 40.0 - def test_mouse_action_before_first_frame_discarded(self, scope): + def test_mouse_action_before_first_frame_fails_session(self, scope): import queue from openadapt_capture import utils @@ -334,10 +414,13 @@ def test_mouse_action_before_first_frame_discarded(self, scope): utils.set_start_time() q = queue.Queue() - trigger_action_event( - q, {"name": "click", "mouse_x": 310.0, "mouse_y": 170.0}, scope - ) - assert self._drain(q) == [] # discarded loudly, not mis-recorded + with pytest.raises(WindowCaptureError, match="before the first"): + trigger_action_event( + q, + {"name": "click", "mouse_x": 310.0, "mouse_y": 170.0}, + scope, + ) + assert self._drain(q) == [] def test_key_action_unaffected(self, scope): import queue @@ -402,9 +485,7 @@ def test_window_capture_round_trips(self, tmp_path): "scale": 2.0, "viewport": [1600, 1200], } - capture_path = self._insert_recording( - str(tmp_path / "cap"), {"capture_window": scope_info} - ) + capture_path = self._insert_recording(str(tmp_path / "cap"), {"capture_window": scope_info}) with CaptureSession.load(capture_path) as capture: assert capture.window_capture == scope_info assert capture.window_capture["coordinate_space"] == "window_pixels" @@ -429,6 +510,7 @@ def test_fullscreen_recording_has_no_window_capture(self, tmp_path): # is no guarantee a resolvable/capturable application window exists. Run on # an interactive desktop (developer machine or the Parallels rig). _NO_INPUT_INJECTION = os.environ.get("OPENADAPT_CI_NO_INPUT_INJECTION") == "1" +_PRODUCTION_QUALIFICATION = os.environ.get("OPENADAPT_CAPTURE_PRODUCTION_QUALIFICATION") == "1" _SESSION_SKIP_REASON = ( "OPENADAPT_CI_NO_INPUT_INJECTION=1: non-interactive hosted-runner session " "has no guaranteed capturable application window (hosted CI limitation, " @@ -443,6 +525,247 @@ def test_fullscreen_recording_has_no_window_capture(self, tmp_path): _SMOKE_TITLE = os.environ.get("OPENADAPT_WINDOW_SMOKE_TITLE") or None +def _geometry_changed( + current: tuple[float, float, float, float], + original: tuple[float, float, float, float], +) -> bool: + """Return true only after both the position and the size change.""" + moved = abs(current[0] - original[0]) >= 1 or abs(current[1] - original[1]) >= 1 + resized = abs(current[2] - original[2]) >= 1 or abs(current[3] - original[3]) >= 1 + return moved and resized + + +def _capture_until_bounds( + scope: WindowCaptureScope, + predicate, + *, + timeout: float = 10.0, +): + """Capture until the live target has bounds accepted by ``predicate``.""" + deadline = time.monotonic() + timeout + last_bounds = None + saw_changed = False + while time.monotonic() < deadline: + image, changed = scope.capture_frame() + saw_changed = saw_changed or changed + data = scope.window_event_data() + last_bounds = tuple(data["state"]["bounds"]) + if predicate(last_bounds): + return image, saw_changed, data + time.sleep(0.1) + raise AssertionError( + f"window bounds did not reach the required state within {timeout}s; " + f"last bounds were {last_bounds!r}" + ) + + +@contextmanager +def _temporary_windows_geometry(window: TargetWindow): + """Move and resize a normal Win32 window, then restore its exact rectangle.""" + import ctypes + import ctypes.wintypes as wintypes + + user32 = ctypes.windll.user32 + hwnd = wintypes.HWND(window.window_id) + assert user32.IsWindow(hwnd), f"Win32 window {window.window_id} no longer exists" + assert not user32.IsIconic(hwnd), "qualification target must not be minimized" + assert not user32.IsZoomed(hwnd), "qualification target must not be maximized" + + original = wintypes.RECT() + assert user32.GetWindowRect(hwnd, ctypes.byref(original)), ( + f"GetWindowRect failed for Win32 window {window.window_id}" + ) + original_width = original.right - original.left + original_height = original.bottom - original.top + target_width = max(320, original_width - 137) + target_height = max(240, original_height - 83) + if target_width == original_width: + target_width += 137 + if target_height == original_height: + target_height += 83 + + mutated = False + try: + mutated = bool( + user32.MoveWindow( + hwnd, + original.left + 37, + original.top + 29, + target_width, + target_height, + True, + ) + ) + assert mutated, f"MoveWindow failed for Win32 window {window.window_id}" + yield + finally: + if mutated: + restored = user32.MoveWindow( + hwnd, + original.left, + original.top, + original_width, + original_height, + True, + ) + assert restored, f"could not restore Win32 window {window.window_id} geometry" + + +def _macos_ax_attribute(application_services, element, name: str): + """Read one AX attribute and return ``None`` when it is unavailable.""" + error, value = application_services.AXUIElementCopyAttributeValue( + element, + name, + None, + ) + if error != application_services.kAXErrorSuccess: + return None + return value + + +def _macos_ax_geometry(application_services, value, value_type): + """Read a CGPoint or CGSize from an AXValue.""" + success, geometry = application_services.AXValueGetValue( + value, + value_type, + None, + ) + assert success, "could not decode macOS accessibility geometry" + return geometry + + +@contextmanager +def _temporary_macos_geometry(window: TargetWindow): + """Move and resize one AX window, then restore its exact AX geometry.""" + import ApplicationServices + + app = ApplicationServices.AXUIElementCreateApplication(window.pid) + ax_windows = _macos_ax_attribute(ApplicationServices, app, "AXWindows") or [] + id_matches = [] + title_matches = [] + for candidate in ax_windows: + candidate_number = _macos_ax_attribute( + ApplicationServices, + candidate, + "AXWindowNumber", + ) + if candidate_number is not None and int(candidate_number) == window.window_id: + id_matches.append(candidate) + candidate_title = _macos_ax_attribute( + ApplicationServices, + candidate, + "AXTitle", + ) + if candidate_title is not None and str(candidate_title) == window.title: + title_matches.append(candidate) + + if id_matches: + ax_window = id_matches[0] + else: + assert len(title_matches) == 1, ( + "the macOS qualification target must expose a unique Accessibility " + f"window title; found {len(title_matches)} matches for {window.title!r}" + ) + ax_window = title_matches[0] + + fullscreen = _macos_ax_attribute( + ApplicationServices, + ax_window, + "AXFullScreen", + ) + movable = _macos_ax_attribute(ApplicationServices, ax_window, "AXMovable") + resizable = _macos_ax_attribute(ApplicationServices, ax_window, "AXResizable") + assert not fullscreen, "qualification target must not be full screen" + assert movable is not False, "qualification target must be movable" + assert resizable is not False, "qualification target must be resizable" + + original_position = _macos_ax_attribute( + ApplicationServices, + ax_window, + "AXPosition", + ) + original_size = _macos_ax_attribute(ApplicationServices, ax_window, "AXSize") + assert original_position is not None and original_size is not None, ( + "qualification target does not expose mutable Accessibility geometry" + ) + point = _macos_ax_geometry( + ApplicationServices, + original_position, + ApplicationServices.kAXValueCGPointType, + ) + size = _macos_ax_geometry( + ApplicationServices, + original_size, + ApplicationServices.kAXValueCGSizeType, + ) + target_width = max(320.0, float(size.width) - 137.0) + target_height = max(240.0, float(size.height) - 83.0) + if target_width == float(size.width): + target_width += 137.0 + if target_height == float(size.height): + target_height += 83.0 + + target_position_value = ApplicationServices.AXValueCreate( + ApplicationServices.kAXValueCGPointType, + (float(point.x) + 37.0, float(point.y) + 29.0), + ) + target_size_value = ApplicationServices.AXValueCreate( + ApplicationServices.kAXValueCGSizeType, + (target_width, target_height), + ) + + mutated = False + try: + size_error = ApplicationServices.AXUIElementSetAttributeValue( + ax_window, + "AXSize", + target_size_value, + ) + mutated = size_error == ApplicationServices.kAXErrorSuccess + assert mutated, f"could not resize macOS window (AX error {size_error})" + position_error = ApplicationServices.AXUIElementSetAttributeValue( + ax_window, + "AXPosition", + target_position_value, + ) + assert position_error == ApplicationServices.kAXErrorSuccess, ( + f"could not move macOS window (AX error {position_error})" + ) + yield + finally: + if mutated: + size_error = ApplicationServices.AXUIElementSetAttributeValue( + ax_window, + "AXSize", + original_size, + ) + position_error = ApplicationServices.AXUIElementSetAttributeValue( + ax_window, + "AXPosition", + original_position, + ) + assert size_error == ApplicationServices.kAXErrorSuccess, ( + f"could not restore macOS window size (AX error {size_error})" + ) + assert position_error == ApplicationServices.kAXErrorSuccess, ( + f"could not restore macOS window position (AX error {position_error})" + ) + + +@contextmanager +def _temporary_window_geometry(window: TargetWindow): + """Dispatch a reversible live geometry change to the current platform.""" + if sys.platform == "win32": + with _temporary_windows_geometry(window): + yield + return + if sys.platform == "darwin": + with _temporary_macos_geometry(window): + yield + return + raise AssertionError(f"no live window geometry controller for {sys.platform}") + + @pytest.mark.slow @pytest.mark.skipif(not _ON_SUPPORTED_PLATFORM, reason=_PLATFORM_SKIP_REASON) @pytest.mark.skipif(_NO_INPUT_INJECTION, reason=_SESSION_SKIP_REASON) @@ -450,12 +773,15 @@ class TestWindowCaptureLive: """Capture a real window end to end (resolve -> frame -> translate).""" def _scope(self) -> WindowCaptureScope: - scope = WindowCaptureScope( - WindowTarget(owner=_SMOKE_OWNER, title=_SMOKE_TITLE) - ) + scope = WindowCaptureScope(WindowTarget(owner=_SMOKE_OWNER, title=_SMOKE_TITLE)) try: scope.resolve() - except WindowCaptureError: + except WindowCaptureError as exc: + if _PRODUCTION_QUALIFICATION: + raise AssertionError( + f"production qualification requires an on-screen window " + f"matching owner {_SMOKE_OWNER!r} title {_SMOKE_TITLE!r}" + ) from exc pytest.skip( f"no on-screen window matching owner {_SMOKE_OWNER!r} " f"title {_SMOKE_TITLE!r} on this desktop; open one (or set " @@ -485,9 +811,79 @@ def test_live_window_frame_and_translation(self): data = scope.window_event_data() assert data["state"]["viewport"] == [image.width, image.height] - def test_live_missing_window_fails_loud(self): - scope = WindowCaptureScope( - WindowTarget(owner="no-such-app-obviously-not-running-xyz") + @pytest.mark.skipif( + not _PRODUCTION_QUALIFICATION, + reason=( + "live window move/resize changes are reserved for explicit " + "OPENADAPT_CAPTURE_PRODUCTION_QUALIFICATION=1 runs" + ), + ) + def test_live_move_resize_preserves_fixed_viewport_and_restores_window(self): + """Prove live move/resize normalization without changing final app state.""" + discovery_scope = self._scope() + target = discovery_scope.resolve() + assert target.title.strip(), ( + "production qualification requires a target with a stable window title" ) + + # Bind this test to the exact resolved application/title. This prevents + # an owner-only selector from switching to another large window after + # the target changes size. + scope = WindowCaptureScope(WindowTarget(owner=target.owner, title=target.title)) + initial_image, initial_changed = scope.capture_frame() + assert initial_changed is True + initial_data = scope.window_event_data() + initial_state = initial_data["state"] + initial_bounds = tuple(initial_state["bounds"]) + initial_viewport = initial_state["viewport"] + initial_source_viewport = initial_state["source_viewport"] + + with _temporary_window_geometry(target): + moved_image, moved_changed, moved_data = _capture_until_bounds( + scope, + lambda bounds: _geometry_changed(bounds, initial_bounds), + ) + + assert moved_changed is True + assert moved_data["window_id"] == str(target.window_id) + moved_state = moved_data["state"] + assert moved_image.size == tuple(initial_viewport) + assert moved_state["viewport"] == initial_viewport + assert moved_state["source_viewport"] != initial_source_viewport + + # A changed aspect ratio must be represented by a content rectangle + # inside the fixed output viewport, not by a dropped or stretched + # frame. + content_x, content_y, content_width, content_height = moved_state["content_rect"] + assert 0 <= content_x < initial_viewport[0] + assert 0 <= content_y < initial_viewport[1] + assert 0 < content_width <= initial_viewport[0] + assert 0 < content_height <= initial_viewport[1] + assert [content_x, content_y, content_width, content_height] != [ + 0, + 0, + *initial_viewport, + ] + + # Input at the live window center must map to the center of the + # non-letterboxed content, even after the move and resize. + x, y, width, height = moved_state["bounds"] + px, py = scope.translate(x + width / 2, y + height / 2) + tolerance = max(3.0, float(moved_state["scale"])) + assert px == pytest.approx(content_x + content_width / 2, abs=tolerance) + assert py == pytest.approx(content_y + content_height / 2, abs=tolerance) + + restored_image, restored_changed, restored_data = _capture_until_bounds( + scope, + lambda bounds: all( + abs(current - original) <= 4 for current, original in zip(bounds, initial_bounds) + ), + ) + assert restored_changed is True + assert restored_image.size == tuple(initial_viewport) + assert restored_data["window_id"] == str(target.window_id) + + def test_live_missing_window_fails_loud(self): + scope = WindowCaptureScope(WindowTarget(owner="no-such-app-obviously-not-running-xyz")) with pytest.raises(WindowCaptureError, match="no window matching"): scope.capture_frame()