Add report-only packaged release compatibility checks

.github/release_plan.md

@@ -90,6 +90,8 @@ NOTE: this PR should make all CI relating to `main` be passing again (such as th
     -   [ ] Click `run pipeline`.
 	-   [ ] for `branch/tag` select the release branch which is **`release/YYYY.minor`**.
 	-   NOTE: Please opt to release the python extension close to when VS Code is released to align when release notes go out. When we bump the VS Code engine number, our extension will not go out to stable until the VS Code stable release but this only occurs when we bump the engine number.
+-   [ ] After the official Build artifact exists, run the opt-in [packaged compatibility report](../build/releaseCompatibility/README.md) against that exact VSIX and a fixed supported Environments partner, with explicit on/off profiles. Record failed/infrastructure outcomes; this report does not replace the existing approval gate.
+-   [ ] Reconcile the tested VSIX SHA-256 with the final signed/published bytes without rebuilding when available. Until the shared post-sign integration is validated, record that acceptance as outstanding rather than claiming final-artifact certification.
 -   [ ] 🧍🧍 Get approval on the release on the [CD](https://dev.azure.com/monacotools/Monaco/_build?definitionId=299).
 -   [ ] Click "approve" in the publish step of [CD](https://dev.azure.com/monacotools/Monaco/_build?definitionId=299) to publish the release to the marketplace.  🎉
 -   [ ] Take the Github release out of draft.

.github/workflows/pr-check.yml

@@ -82,6 +82,115 @@ jobs:
           cargo_target: ${{ matrix.target }}
           vsix_target: ${{ matrix.vsix-target }}
 
+  candidate-compatibility:
+    name: Candidate Core packaged compatibility (${{ matrix.vsix-target }})
+    needs: [build-vsix]
+    if: ${{ !cancelled() }}
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 20
+    permissions:
+      contents: read
+      actions: read
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - os: windows-latest
+            vsix-target: win32-x64
+          - os: ubuntu-latest
+            vsix-target: linux-x64
+    steps:
+      - name: Checkout the same run's test orchestration
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+
+      - name: Initialize candidate infrastructure evidence
+        if: always()
+        shell: pwsh
+        run: |
+          $evidence = Join-Path $env:GITHUB_WORKSPACE 'candidate-core-evidence'
+          New-Item -ItemType Directory -Path $evidence -Force | Out-Null
+          @{
+            schemaVersion = 1
+            status = 'infrastructure-error'
+            scope = 'ci-built-candidate-core-compatibility'
+            message = 'Candidate download, preparation or execution has not completed.'
+            runId = $env:GITHUB_RUN_ID
+            sourceCommit = $env:GITHUB_SHA
+            artifactName = $env:PYTHON_CANDIDATE_ARTIFACT
+            certifiesOfficialCandidateOrSignedPublication = $false
+          } | ConvertTo-Json | Set-Content (Join-Path $evidence 'job-report.json')
+        env:
+          PYTHON_CANDIDATE_ARTIFACT: '${{ env.ARTIFACT_NAME_VSIX }}-${{ matrix.vsix-target }}'
+
+      - name: Select repository Node version
+        uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
+        with:
+          node-version-file: .nvmrc
+          cache: npm
+
+      - name: Select Python for real fixture venvs
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      - name: Download original candidate VSIX from this workflow run
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
+        with:
+          name: '${{ env.ARTIFACT_NAME_VSIX }}-${{ matrix.vsix-target }}'
+          path: candidate-core-artifact
+
+      - name: Restore only reviewed test orchestration dependencies
+        run: npm ci --ignore-scripts --no-audit --no-fund
+
+      - name: Exercise candidate Core and fixed released Environments on and off
+        shell: pwsh
+        run: |
+          $ErrorActionPreference = 'Stop'
+          $artifact = Join-Path $env:GITHUB_WORKSPACE 'candidate-core-artifact'
+          $files = @(Get-ChildItem -LiteralPath $artifact -Filter '*.vsix' -File -Recurse)
+          $candidate = Join-Path $artifact "ms-python-insiders-$($env:PYTHON_CANDIDATE_TARGET).vsix"
+          if ($files.Count -ne 1 -or -not (Test-Path -LiteralPath $candidate -PathType Leaf)) {
+            throw 'The exact same-run candidate artifact must contain its one expected original VSIX.'
+          }
+          $evidence = Join-Path $env:GITHUB_WORKSPACE 'candidate-core-evidence'
+          $output = Join-Path $evidence 'compatibility'
+          $script = Join-Path (Join-Path (Join-Path $env:GITHUB_WORKSPACE 'build') 'releaseCompatibility') 'selfTest.js'
+          $python = (Get-Command python).Source
+          if ($IsLinux) {
+            & xvfb-run -a node $script $output $python $candidate
+          } else {
+            & node $script $output $python $candidate
+          }
+          $result = $LASTEXITCODE
+          $report = Get-Content (Join-Path $output 'input-report.json') -Raw | ConvertFrom-Json
+          $report | ConvertTo-Json -Depth 20 | Set-Content (Join-Path $evidence 'job-report.json')
+          exit $result
+        env:
+          PYTHON_CANDIDATE_TARGET: ${{ matrix.vsix-target }}
+          PYTHON_CANDIDATE_ARTIFACT: '${{ env.ARTIFACT_NAME_VSIX }}-${{ matrix.vsix-target }}'
+          PYTHON_CANDIDATE_HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
+
+      - name: Retain candidate bytes provenance and all real-host outcomes
+        if: always()
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        with:
+          name: candidate-core-compatibility-${{ matrix.vsix-target }}
+          path: |
+            candidate-core-evidence/job-report.json
+            candidate-core-evidence/compatibility/input-report.json
+            candidate-core-evidence/compatibility/plan.json
+            candidate-core-evidence/compatibility/run/report.json
+            candidate-core-evidence/compatibility/run/environments-on/candidate/**
+            candidate-core-evidence/compatibility/run/environments-on/user-data/logs/**
+            candidate-core-evidence/compatibility/run/environments-on/workspace/probe-*.json
+            candidate-core-evidence/compatibility/run/explicit-off/candidate/**
+            candidate-core-evidence/compatibility/run/explicit-off/user-data/logs/**
+            candidate-core-evidence/compatibility/run/explicit-off/workspace/probe-*.json
+          if-no-files-found: error
+          retention-days: 7
+
   lint:
     name: Lint
     runs-on: ubuntu-latest

.github/workflows/release-compatibility.yml

@@ -0,0 +1,93 @@
+name: Packaged compatibility guards
+
+on:
+  pull_request:
+    paths:
+      - '.github/workflows/release-compatibility.yml'
+      - 'build/releaseCompatibility/**'
+      - 'build/azure-pipeline.release-compatibility.yml'
+      - 'package.json'
+      - 'package-lock.json'
+      - '.nvmrc'
+  push:
+    branches: [main, 'release/**']
+    paths:
+      - '.github/workflows/release-compatibility.yml'
+      - 'build/releaseCompatibility/**'
+      - 'build/azure-pipeline.release-compatibility.yml'
+      - 'package.json'
+      - 'package-lock.json'
+      - '.nvmrc'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  guards:
+    name: Guards and immutable harness (${{ matrix.os }})
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 15
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [windows-2022, ubuntu-22.04]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - name: Select repository Node version
+        uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
+        with:
+          node-version-file: .nvmrc
+          cache: npm
+      - name: Restore locked dependencies without install hooks
+        run: npm ci --ignore-scripts --no-audit --no-fund
+      - name: Run package, provenance, interpreter, mode, adapter and packer guards
+        run: npm run test:releaseCompatibility
+      - name: Lint compatibility implementation
+        run: npx --no-install eslint --max-warnings 0 build/releaseCompatibility
+      - name: Produce immutable reusable test harness
+        run: npm run package:releaseCompatibility -- "${{ github.workspace }}/harness.zip"
+      - name: Select Python for real fixture venvs
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+        with:
+          python-version: '3.10'
+      - name: Exercise the frozen released pair in real isolated on/off hosts
+        shell: pwsh
+        run: |
+          $output = Join-Path $env:GITHUB_WORKSPACE 'released-pair-evidence'
+          $script = Join-Path (Join-Path (Join-Path $env:GITHUB_WORKSPACE 'build') 'releaseCompatibility') 'selfTest.js'
+          $python = (Get-Command python).Source
+          if ($IsLinux) {
+            & xvfb-run -a node $script $output $python
+          } else {
+            & node $script $output $python
+          }
+          exit $LASTEXITCODE
+      - name: Retain verified harness bytes
+        if: always()
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        with:
+          name: python-release-compatibility-harness-${{ matrix.os }}
+          path: harness.zip
+          if-no-files-found: error
+          retention-days: 7
+      - name: Retain real-host pass, failure and infrastructure evidence
+        if: always()
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        with:
+          name: released-pair-evidence-${{ matrix.os }}
+          path: |
+            released-pair-evidence/input-report.json
+            released-pair-evidence/plan.json
+            released-pair-evidence/run/report.json
+            released-pair-evidence/run/environments-on/candidate/**
+            released-pair-evidence/run/environments-on/user-data/logs/**
+            released-pair-evidence/run/environments-on/workspace/probe-*.json
+            released-pair-evidence/run/explicit-off/candidate/**
+            released-pair-evidence/run/explicit-off/user-data/logs/**
+            released-pair-evidence/run/explicit-off/workspace/probe-*.json
+          if-no-files-found: warn
+          retention-days: 7

build/azure-pipeline.release-compatibility.yml

@@ -0,0 +1,178 @@
+# Opt-in report only. This pipeline never signs, publishes, or approves a release.
+trigger: none
+pr: none
+
+parameters:
+  - name: target
+    type: string
+    default: win32-x64
+    values: [win32-x64, linux-x64]
+  - name: coreDefinitionId
+    type: number
+    default: 0
+  - name: coreBuildId
+    type: number
+    default: 0
+  - name: coreArtifactName
+    type: string
+    default: ''
+  - name: coreVsixFile
+    type: string
+    default: ''
+  - name: coreVersion
+    type: string
+    default: ''
+  - name: coreSha256
+    type: string
+    default: ''
+  - name: inputsDefinitionId
+    type: number
+    default: 0
+  - name: inputsBuildId
+    type: number
+    default: 0
+  - name: inputsSha256
+    type: string
+    default: ''
+
+jobs:
+  - job: PackagedCompatibilityReport
+    displayName: Pre-sign packaged compatibility report
+    pool:
+      ${{ if eq(parameters.target, 'win32-x64') }}:
+        vmImage: windows-2022
+      ${{ else }}:
+        vmImage: ubuntu-22.04
+    steps:
+      - checkout: self
+        persistCredentials: false
+
+      - pwsh: |
+          $evidence = Join-Path $env:PIPELINE_WORKSPACE 'compatibility-evidence'
+          New-Item -ItemType Directory -Path $evidence -Force | Out-Null
+          @{
+            schemaVersion = 1
+            status = 'infrastructure-error'
+            validationScope = 'pre-sign-build-artifact'
+            requiresFinalSignedHashReconciliation = $true
+            message = 'Preparation or execution has not completed; inspect job output.'
+            sourceVersion = $env:BUILD_SOURCEVERSION
+            requestedTarget = $env:COMPATIBILITY_TARGET
+          } | ConvertTo-Json | Set-Content (Join-Path $evidence 'pipeline-report.json')
+          foreach ($id in @($env:CORE_DEFINITION, $env:CORE_BUILD, $env:INPUTS_DEFINITION, $env:INPUTS_BUILD)) {
+            if ($id -notmatch '^[1-9][0-9]*$') { throw 'Exact positive producer definition/build IDs are required.' }
+          }
+          if ([string]::IsNullOrWhiteSpace($env:CORE_ARTIFACT) -or [string]::IsNullOrWhiteSpace($env:CORE_FILE)) {
+            throw 'The exact final Build artifact and VSIX file are required; no artifact name is guessed.'
+          }
+          $nodeVersion = (Get-Content (Join-Path $env:BUILD_SOURCESDIRECTORY '.nvmrc') -Raw).Trim().TrimStart('v')
+          Write-Host "##vso[task.setvariable variable=nodeVersion;isReadOnly=true]$nodeVersion"
+        displayName: Initialize explicit infrastructure evidence
+        condition: always()
+        env:
+          COMPATIBILITY_TARGET: ${{ parameters.target }}
+          CORE_DEFINITION: ${{ parameters.coreDefinitionId }}
+          CORE_BUILD: ${{ parameters.coreBuildId }}
+          CORE_ARTIFACT: ${{ parameters.coreArtifactName }}
+          CORE_FILE: ${{ parameters.coreVsixFile }}
+          INPUTS_DEFINITION: ${{ parameters.inputsDefinitionId }}
+          INPUTS_BUILD: ${{ parameters.inputsBuildId }}
+
+      - task: UseNode@1
+        inputs:
+          version: $(nodeVersion)
+
+      - task: UsePythonVersion@0
+        inputs:
+          versionSpec: '3.10'
+          architecture: x64
+
+      - task: DownloadPipelineArtifact@2
+        inputs:
+          buildType: specific
+          project: $(System.TeamProject)
+          definition: ${{ parameters.coreDefinitionId }}
+          buildVersionToDownload: specific
+          pipelineId: ${{ parameters.coreBuildId }}
+          artifactName: ${{ parameters.coreArtifactName }}
+          targetPath: $(Pipeline.Workspace)/compatibility-core
+        displayName: Download one exact official final Build artifact
+
+      - task: DownloadPipelineArtifact@2
+        inputs:
+          buildType: specific
+          project: $(System.TeamProject)
+          definition: ${{ parameters.inputsDefinitionId }}
+          buildVersionToDownload: specific
+          pipelineId: ${{ parameters.inputsBuildId }}
+          artifactName: python-compatibility-inputs
+          targetPath: $(Pipeline.Workspace)/compatibility-inputs
+        displayName: Download one exact pinned partner input set
+
+      - script: npm ci --ignore-scripts --no-audit --no-fund
+        displayName: Restore reviewed harness dependencies, not product source
+
+      - pwsh: |
+          $ErrorActionPreference = 'Stop'
+          $root = Join-Path (Join-Path $env:BUILD_SOURCESDIRECTORY 'build') 'releaseCompatibility'
+          $evidence = Join-Path $env:PIPELINE_WORKSPACE 'compatibility-evidence'
+          $preparation = Join-Path $evidence 'preparation'
+          $output = Join-Path $evidence 'runner'
+          $coreDirectory = Join-Path $env:PIPELINE_WORKSPACE 'compatibility-core'
+          $core = [System.IO.Path]::GetFullPath((Join-Path $coreDirectory $env:CORE_FILE))
+          $allowed = [System.IO.Path]::GetFullPath($coreDirectory) + [System.IO.Path]::DirectorySeparatorChar
+          if (-not $core.StartsWith($allowed, [System.StringComparison]::Ordinal)) {
+            throw 'The candidate file must remain inside the downloaded artifact.'
+          }
+          $inputs = Join-Path (Join-Path $env:PIPELINE_WORKSPACE 'compatibility-inputs') 'inputs.zip'
+          $python = (Get-Command python).Source
+          function Save-Report($status, $exitCode, $phase) {
+            @{
+              schemaVersion = 1
+              status = $status
+              phase = $phase
+              validationScope = 'pre-sign-build-artifact'
+              requiresFinalSignedHashReconciliation = $true
+              sourceVersion = $env:BUILD_SOURCEVERSION
+              coreBuildId = $env:CORE_BUILD
+              inputsBuildId = $env:INPUTS_BUILD
+              runnerExitCode = $exitCode
+            } | ConvertTo-Json | Set-Content (Join-Path $evidence 'pipeline-report.json')
+          }
+          & node (Join-Path $root 'prepare.js') --inputs $inputs --inputs-sha256 $env:INPUTS_HASH `
+            --candidate $core --candidate-sha256 $env:CORE_HASH --candidate-version $env:CORE_VERSION `
+            --python $python --output $preparation
+          $preparationExit = $LASTEXITCODE
+          if ($preparationExit -ne 0) {
+            $adapter = Get-Content (Join-Path $preparation 'adapter-report.json') -Raw | ConvertFrom-Json
+            Save-Report $adapter.status $preparationExit 'preparation'
+            exit $preparationExit
+          }
+          if ($IsLinux) {
+            & xvfb-run -a node (Join-Path $root 'run.js') --plan (Join-Path $preparation 'plan.json') --output $output
+          } else {
+            & node (Join-Path $root 'run.js') --plan (Join-Path $preparation 'plan.json') --output $output
+          }
+          $result = $LASTEXITCODE
+          $report = Get-Content (Join-Path $output 'report.json') -Raw | ConvertFrom-Json
+          Save-Report $report.status $result 'execution'
+          if ($result -ne 0) {
+            Write-Host '##vso[task.logissue type=warning]Packaged compatibility failed or hit an infrastructure error. Report only; no publishing gate was changed.'
+          }
+          exit $result
+        displayName: Report real packaged on/off compatibility
+        continueOnError: true
+        env:
+          CORE_FILE: ${{ parameters.coreVsixFile }}
+          CORE_VERSION: ${{ parameters.coreVersion }}
+          CORE_HASH: ${{ parameters.coreSha256 }}
+          CORE_BUILD: ${{ parameters.coreBuildId }}
+          INPUTS_HASH: ${{ parameters.inputsSha256 }}
+          INPUTS_BUILD: ${{ parameters.inputsBuildId }}
+
+      - task: PublishPipelineArtifact@1
+        condition: always()
+        inputs:
+          targetPath: $(Pipeline.Workspace)/compatibility-evidence
+          artifact: packaged-compatibility-report-${{ parameters.target }}
+        displayName: Retain failures, infrastructure status, and raw evidence