Rust Driver EngSys Prototype

eng/apiview_reqs.txt

@@ -1,18 +1,19 @@
+aiohttp==3.13.3
 astroid==4.0.4
 charset-normalizer==3.4.1
 dill==0.3.9
 isodate==0.6.1
 isort==5.13.2
-lazy-object-proxy==1.10.0
+lazy-object-proxy==1.12.0
 mccabe==0.7.0
 pkginfo==1.12.1.2
 platformdirs==4.3.6
 pylint==4.0.4
-azure-pylint-guidelines-checker==0.5.7
+azure-pylint-guidelines-checker==0.5.9
 six==1.17.0
 tomli==2.2.1
 tomlkit==0.13.2
 typing_extensions==4.15.0
 wrapt==1.17.2
-apiview-stub-generator==0.3.28
-pip==24.0
+apiview-stub-generator==0.3.31
+pip==24.0

eng/common/TestResources/SubConfig-Helpers.ps1

@@ -82,6 +82,11 @@ function ShouldMarkValueAsSecret([string]$serviceName, [string]$key, [string]$va
         "SERVICE_MANAGEMENT_URL",
         "ENDPOINT_SUFFIX",
         "SERVICE_DIRECTORY",
+        "RUST_TEST_THREADS",
+        "RUST_BACKTRACE",
+        "COSMOS_RUSTFLAGS",
+        "DATABASE_NAME",
+        "ACCOUNT_HOST",
         # This is used in many places and is harder to extract from the base subscription config, so hardcode it for now.
         "STORAGE_ENDPOINT_SUFFIX",
         # Parameters

eng/common/TestResources/TestResources-Helpers.ps1

@@ -349,6 +349,36 @@ function SetDeploymentOutputs(
     return $deploymentEnvironmentVariables, $deploymentOutputs
 }
 
+<#
+   Writes Resource Manager errors which often have nested exceptions.
+   https://learn.microsoft.com/dotnet/api/microsoft.azure.commands.resourcemanager.cmdlets.sdkmodels.psresourcemanagererror
+#>
+function Write-PSResourceManagerError($resourceManagerError, [int]$level, [int]$maxLevel) {
+    if (!$resourceManagerError -or !$resourceManagerError.Message) {
+        return;
+    }
+
+    # Retrieve one or more messages then decode the strings for readability (remove quote escapes, fix link readability, etc.)
+    $parsedMessage = ($resourceManagerError.Message -join "$([System.Environment]::NewLine)$([System.Environment]::NewLine)")
+    $parsedMessage = [System.Net.WebUtility]::UrlDecode($parsedMessage)
+
+    $prefix = " " * ($level * 2) + "-"
+    Write-Host "$prefix $parsedMessage"
+
+    # Limit the level of nested exceptions to prevent overwhelming users with details and infinite recursive calls.
+    if ($level -ge $maxLevel) {
+        if ($resourceManagerError.Details.Count -gt 0) {
+            Write-Host "$prefix ... (additional nested errors not shown)"
+        }
+
+        return;
+    }
+
+    foreach ($detail in $resourceManagerError.Details) {
+        Write-PSResourceManagerError $detail ($level + 1) $maxLevel
+    }
+}
+
 function HandleTemplateDeploymentError($templateValidationResult) {
     Write-Warning "Deployment template validation failed"
 
@@ -357,14 +387,10 @@ function HandleTemplateDeploymentError($templateValidationResult) {
         return
     }
 
-    # Retrieve one or more messages then decode the strings for readability (remove quote escapes, fix link readability, etc.)
-    $parsedMessage = ($templateValidationResult.Details.Message -join "$([System.Environment]::NewLine)$([System.Environment]::NewLine)")
-    $parsedMessage = [System.Net.WebUtility]::UrlDecode($parsedMessage)
-
     Write-Warning "#####################################################"
     Write-Warning "######### TEMPLATE VALIDATION ERROR DETAILS #########"
     Write-Warning "#####################################################"
-    Write-Host $parsedMessage
+    Write-PSResourceManagerError $templateValidationResult 0 5
     Write-Warning "#####################################################"
 }
 

eng/common/TestResources/deploy-test-resources.yml

@@ -12,6 +12,7 @@ parameters:
   UseFederatedAuth: true
   PersistOidcToken: false
   SelfContainedPostScript: self-contained-test-resources-post.ps1
+  SkipEnvironmentSetup: false
 
 # SubscriptionConfiguration will be splatted into the parameters of the test
 # resources script. It should be JSON in the form:
@@ -42,7 +43,8 @@ parameters:
 steps:
   - template: /eng/common/pipelines/templates/steps/cache-ps-modules.yml
 
-  - template: /eng/common/TestResources/setup-environments.yml
+  - ${{ if eq(parameters.SkipEnvironmentSetup, false) }}:
+    - template: /eng/common/TestResources/setup-environments.yml
 
   - ${{ if eq(parameters.PersistOidcToken, true) }}:
     - task: AzureCLI@2

eng/common/instructions/azsdk-tools/typespec-to-sdk.instructions.md

@@ -52,7 +52,7 @@ Follow the steps in #file:.github/skills/azsdk-common-generate-sdk-locally/SKILL
         For data plane: `Python`, `.NET`, `JavaScript`, `Java`
     - Each SDK generation tool call should show a label to indicate the language being generated.
 2. Monitor pipeline status after 15 minutes and provide updates. If pipeline is in progress, inform user that it may take additional time and check the status later.
-3. Display generated SDK PR links when available. If pipeline fails, inform user with error details and suggest to check pipeline logs for more information. Use the `azsdk-common-pipeline-troubleshooting` skill to diagnose and resolve pipeline failures.
+3. Display generated SDK PR links when available. If pipeline fails, inform user with error details and suggest to check pipeline logs for more information. Use the `azsdk-common-pipeline-analysis` skill to diagnose and resolve pipeline failures.
 4. If SDK pull request is available for all languages, ask user to review generated SDK pull request and mark them as ready for review when they are ready to get them reviewed and merged. If APIView feedback is received, use the `azsdk-common-apiview-feedback-resolution` skill to analyze and resolve review comments.
 5. Inform the user that they can checkout generated SDK pull request locally and add more tests, samples or code customizations if needed using local SDK generation tools.
 6. If SDK pull request was created for test purposes, inform user to close the test SDK pull request.

eng/common/pipelines/live-eval.yml

@@ -0,0 +1,43 @@
+# Live-tier eval CI: nightly end-to-end run of the live workflow scenarios against the real
+# azsdk-cli MCP and real Azure DevOps (writes confined to a test area).
+
+# Nightly only — no CI/PR trigger.
+trigger: none
+pr: none
+
+schedules:
+  - cron: '0 9 * * *'        # 09:00 UTC daily
+    displayName: 'Nightly live eval'
+    branches:
+      include:
+        - main
+    always: true             # run even when main has not changed
+
+variables:
+  # Managed-pool image selection (LINUXPOOL/LINUXVMIMAGE). Repo-local.
+  - template: /eng/pipelines/templates/variables/image.yml
+  # Provides the secret azuresdk-copilot-github-pat, mapped into GITHUB_TOKEN in the invoke step.
+  - group: AzSDK_Eval_Variable_group
+
+extends:
+  template: /eng/common/pipelines/templates/stages/archetype-eval.yml
+  parameters:
+    # Shared mock/live builder; select the live tier (real Cli MCP).
+    mcpSetupTemplate: /eng/common/pipelines/templates/steps/eval-mcp-setup.yml
+    TestType: live
+    vallyRoot: evals
+    evalGlobs:
+      - 'workflows/live/*.eval.yaml'
+    # Run each shard under AzureCLI@2 so the real MCP's DevOps calls are authenticated.
+    UseAzSdkAuthentication: true
+    failOnFailedTests: true
+    # Live scenarios are end-to-end, so give each shard more headroom than the report-only tiers.
+    shardTimeoutInMinutes: 45
+    # Pass-rate gate for `vally eval`. This tier gates (failOnFailedTests: true), so the threshold
+    # is the real bar the nightly run must clear; raise it here as live coverage stabilizes.
+    threshold: 0.8
+    # This repo needs no repo-specific setup (the live MCP is built by the common BuildMcp job).
+    # A spec/language repo that must start its own bot / server / MCP copies the example hook and
+    # points these at it — see eng/common/pipelines/templates/steps/eval-hook-example.yml.
+    # preEvalTemplate: /eng/pipelines/eval/start-my-bot.yml
+    # postEvalTemplate: /eng/pipelines/eval/stop-my-bot.yml

eng/common/pipelines/skill-eval.yml

@@ -0,0 +1,38 @@
+# Skill-compliance eval CI: runs the per-skill Vally evals under .github/skills (one job per skill).
+
+trigger:
+  branches:
+    include:
+      - main
+  paths:
+    include:
+      - .github/skills/**
+      # Retrigger when the mock MCP's tool catalog changes (it can move results).
+      - tools/azsdk-cli/Azure.Sdk.Tools.Mock/**
+      - eng/common/pipelines/skill-eval.yml
+      - eng/common/pipelines/templates/jobs/**
+      - eng/common/pipelines/templates/steps/eval-invoke.yml
+      - eng/common/pipelines/templates/steps/eval-mcp-setup.yml
+      - eng/common/pipelines/templates/stages/archetype-eval.yml
+      - eng/common/scripts/eval/**
+
+pr: none
+
+variables:
+  # Managed-pool image selection (LINUXPOOL/LINUXVMIMAGE). Repo-local.
+  - template: /eng/pipelines/templates/variables/image.yml
+  # Provides the secret azuresdk-copilot-github-pat, mapped into GITHUB_TOKEN in the invoke step.
+  - group: AzSDK_Eval_Variable_group
+
+extends:
+  template: /eng/common/pipelines/templates/stages/archetype-eval.yml
+  parameters:
+    # Shared mock/live builder; select the mock tier.
+    mcpSetupTemplate: /eng/common/pipelines/templates/steps/eval-mcp-setup.yml
+    TestType: mock
+    vallyRoot: .github/skills
+    # Single-level glob excludes azure-typespec-author/evaluate/ (its own benchmark pipeline).
+    evalGlobs:
+      - '*/evals/*.eval.yaml'
+    # Per-shard job timeout (report-only tier).
+    shardTimeoutInMinutes: 20

eng/common/pipelines/templates/archetype-typespec-emitter.yml

@@ -303,7 +303,12 @@ extends:
         jobs:
         - job: Initialize
           steps:
+          # Regeneration does not read repository history, so fetch only the
+          # checked-out commit. git-branch-push.ps1's retry path diffs the branch
+          # tip against its parent, which stays inside the depth-1 boundary.
           - checkout: self
+            fetchDepth: 1
+            fetchTags: false
 
           - template: /eng/common/pipelines/templates/steps/login-to-github.yml
             parameters:
@@ -399,6 +404,8 @@ extends:
             emitterNpmrcPath: $(Agent.TempDirectory)/${{ parameters.EmitterPackagePath }}/.npmrc
           steps:
           - checkout: self
+            fetchDepth: 1
+            fetchTags: false
 
           - template: /eng/common/pipelines/templates/steps/login-to-github.yml
             parameters:
@@ -625,3 +632,5 @@ extends:
                   scriptLocation: "inlineScript"
                   inlineScript: npx tsp-spector upload-coverage --coverageFile $(Build.ArtifactStagingDirectory)/tsp-spector-coverage-azure.json --generatorName @azure-typespec/$(SpectorName) --storageAccountName typespec --containerName coverages --generatorVersion $(node -p -e "require('./package.json').version") --generatorMode azure
                   workingDirectory: $(Build.SourcesDirectory)/eng/packages/$(SpectorName)
+                env:
+                  npm_config_userconfig: $(emitterNpmrcPath)

eng/common/pipelines/templates/jobs/apireview-hub-job-base.yml

@@ -0,0 +1,75 @@
+# Base job wrapper for API Review Hub artifact creation. Language-specific job
+# templates provide setup steps and the request handler provides orchestration steps.
+parameters:
+  - name: jobName
+    type: string
+    default: CreateApiReviewArtifacts
+  - name: displayName
+    type: string
+    default: 'Create API review artifacts'
+  - name: poolName
+    type: string
+    default: 'azsdk-pool'
+  - name: imageOverride
+    type: string
+    default: 'ubuntu-24.04'
+  - name: variables
+    type: object
+    default: []
+  - name: sourceRepositoryFullName
+    type: string
+    default: ''
+  - name: sourceCheckoutDir
+    type: string
+    default: ''
+  - name: toolingDir
+    type: string
+    default: '$(Pipeline.Workspace)/apireview/tooling'
+  - name: setupSteps
+    type: stepList
+    default: []
+  - name: steps
+    type: stepList
+    default: []
+
+jobs:
+- job: ${{ parameters.jobName }}
+  displayName: ${{ parameters.displayName }}
+
+  pool:
+    name: ${{ parameters.poolName }}
+    demands: ImageOverride -equals ${{ parameters.imageOverride }}
+
+  variables:
+  - template: /eng/pipelines/templates/variables/globals.yml
+  - name: ApiReviewRepositoryFullName
+    value: ${{ parameters.sourceRepositoryFullName }}
+  - name: ApiReviewSourceDir
+    value: ${{ parameters.sourceCheckoutDir }}
+  - name: ApiReviewToolingDir
+    value: ${{ parameters.toolingDir }}
+  - ${{ each variable in parameters.variables }}:
+    - ${{ variable }}
+
+  steps:
+  - ${{ if ne(parameters.sourceCheckoutDir, '') }}:
+    - bash: |
+        set -Eeuo pipefail
+
+        source_repo="$SOURCE_CHECKOUT_DIR"
+        repository_full_name="$SOURCE_REPOSITORY_FULL_NAME"
+
+        rm -rf "$source_repo"
+        mkdir -p "$(dirname "$source_repo")"
+
+        echo "Cloning $repository_full_name into $source_repo"
+        git clone --filter=blob:none --no-checkout "https://github.com/$repository_full_name.git" "$source_repo"
+      env:
+        SOURCE_CHECKOUT_DIR: ${{ parameters.sourceCheckoutDir }}
+        SOURCE_REPOSITORY_FULL_NAME: ${{ parameters.sourceRepositoryFullName }}
+      displayName: 'Clone API review source repository'
+
+  - ${{ parameters.setupSteps }}
+
+  # These steps are assembled by the generic API Review Hub request pipeline.
+  - ${{ parameters.steps }}

eng/common/pipelines/templates/jobs/build-mcp.yml

@@ -0,0 +1,43 @@
+# BuildMcp job: builds the MCP server(s) via the swap-point mcpSetupTemplate and ships the
+# artifacts/mcp tree as the `mcp-servers` artifact every shard re-stages.
+
+parameters:
+  - name: mcpSetupTemplate
+    type: string
+  - name: TestType
+    # Which MCP tier to build (mock/live), forwarded to templates that build both tiers
+    # (e.g. eval-mcp-setup.yml). Use 'none' for param-free repo templates.
+    type: string
+    default: none
+  - name: toolsRepo
+    # Optional azure-sdk-tools repository-resource alias, forwarded to the MCP template when this
+    # pipeline runs OUTSIDE azure-sdk-tools (cross-repo build). Blank in the tools repo.
+    type: string
+    default: ''
+
+jobs:
+  - job: BuildMcp
+    displayName: 'Build + publish MCP server(s)'
+    pool:
+      name: $(LINUXPOOL)
+      image: $(LINUXVMIMAGE)
+      os: linux
+    # 1ES publishes via templateContext.outputs, not the `publish` shortcut.
+    templateContext:
+      outputs:
+        - output: pipelineArtifact
+          displayName: 'Publish MCP server artifact'
+          artifactName: mcp-servers
+          targetPath: $(Build.SourcesDirectory)/artifacts/mcp
+    steps:
+      - checkout: self
+        fetchDepth: 1
+
+      # Pass TestType only when not 'none', so param-free repo templates still resolve.
+      - ${{ if ne(parameters.TestType, 'none') }}:
+        - template: ${{ parameters.mcpSetupTemplate }}
+          parameters:
+            TestType: ${{ parameters.TestType }}
+            toolsRepo: ${{ parameters.toolsRepo }}
+      - ${{ else }}:
+        - template: ${{ parameters.mcpSetupTemplate }}