Sync eng/common directory with azure-sdk-tools for PR 17436

eng/common/pipelines/live-eval.yml

@@ -22,6 +22,7 @@ parameters:
 variables:
   # Managed-pool image selection (LINUXPOOL/LINUXVMIMAGE). Repo-local.
   - template: /eng/pipelines/templates/variables/image.yml
+  - template: /eng/pipelines/templates/variables/globals.yml
 
 extends:
   template: /eng/common/pipelines/templates/stages/archetype-eval.yml
@@ -37,6 +38,7 @@ extends:
       - 'workflows/live/*.eval.yaml'
     # Run each shard under AzureCLI@2 so the real MCP's DevOps calls are authenticated.
     UseAzSdkAuthentication: true
+    azureSubscription: ${{ variables.EvalAzureSubscription }}
     failOnFailedTests: true
     # Live scenarios are end-to-end, so give each shard more headroom than the report-only tiers.
     shardTimeoutInMinutes: 45

eng/common/pipelines/templates/jobs/eval-shard.yml

@@ -10,10 +10,22 @@ parameters:
     # Swap point: runs `vally eval` for the shard (the common template is auth-conditional).
     type: string
     default: /eng/common/pipelines/templates/steps/eval-invoke.yml
+  - name: vallyArguments
+    # --tag does not filter the supplied matrix; each planned eval must retain matching stimuli.
+    type: object
+    default: []
+  - name: failOnFailedTests
+    # Match Summary's gate; multi-run gating is not currently supported.
+    type: boolean
+    default: false
   - name: UseAzSdkAuthentication
     # Forwarded to the invoke template: true runs the shard under AzureCLI@2.
     type: boolean
     default: false
+  - name: azureSubscription
+    # Leave blank to preserve the invoke template's default connection.
+    type: string
+    default: ''
   - name: threshold
     # Pass-rate gate forwarded to the invoke template (`vally eval --threshold` + shard verdict).
     type: number
@@ -71,6 +83,14 @@ jobs:
         inputs:
           version: 24.x
 
+      - ${{ if eq(parameters.evalInvokeTemplate, '/eng/common/pipelines/templates/steps/eval-invoke.yml') }}:
+        - script: |
+            node --experimental-strip-types "$(Build.SourcesDirectory)/eng/common/scripts/eval/invoke-eval-shard.ts" --validate-only
+          displayName: 'Validate eval run configuration'
+          env:
+            VALLY_ARGUMENTS_JSON: ${{ replace(convertToJson(parameters.vallyArguments), '$', '\u0024') }}
+            EVAL_FAIL_ON_FAILED_TESTS: ${{ parameters.failOnFailedTests }}
+
       # 1ES Official image blocks registry.npmjs.org; restore via the azure-sdk npm mirror.
       - template: /eng/common/pipelines/templates/steps/create-authenticated-npmrc.yml
         parameters:
@@ -113,12 +133,17 @@ jobs:
 
       - template: /eng/common/pipelines/templates/steps/login-to-copilot.yml
 
-      # Runs the eval via the invoke template (auth-conditional in the common default).
       - template: ${{ parameters.evalInvokeTemplate }}
         parameters:
           vallyRoot: ${{ parameters.vallyRoot }}
           UseAzSdkAuthentication: ${{ parameters.UseAzSdkAuthentication }}
+          # Custom invoke templates receive this only when explicitly configured.
+          ${{ if ne(parameters.azureSubscription, '') }}:
+            azureSubscription: ${{ parameters.azureSubscription }}
           threshold: ${{ parameters.threshold }}
+          # Existing custom templates keep their contract unless opting into the argument list.
+          ${{ if ne(length(parameters.vallyArguments), 0) }}:
+            vallyArguments: ${{ parameters.vallyArguments }}
 
       # Optional repo-specific teardown after the eval.
       - ${{ if parameters.postEvalTemplate }}:

eng/common/pipelines/templates/jobs/eval-summarize.yml

@@ -3,6 +3,7 @@
 
 parameters:
   - name: failOnFailedTests
+    # Only single-run evaluations support this gate; multi-run observations must leave it off.
     type: boolean
     default: false
   - name: createDashboardBundle
@@ -40,7 +41,6 @@ jobs:
         inputs:
           version: 24.x
 
-      # Allowlist result data only, excluding debug, MCP, SDL and signing artifacts.
       - task: DownloadPipelineArtifact@2
         displayName: 'Download all shard results'
         inputs:

eng/common/pipelines/templates/stages/archetype-eval.yml

@@ -43,6 +43,14 @@ parameters:
     # When true, the invoke step runs under AzureCLI@2 (live MCP inherits the service connection).
     type: boolean
     default: false
+  - name: azureSubscription
+    # Optional override; blank preserves the invoke template's default connection.
+    type: string
+    default: ''
+  - name: vallyArguments
+    # --tag does not filter Prepare's matrix; align evalGlobs so every eval has matching stimuli.
+    type: object
+    default: []
   - name: threshold
     # Pass-rate gate forwarded to `vally eval --threshold` and the shard verdict. Repos may override.
     type: number
@@ -102,6 +110,9 @@ extends:
               vallyRoot: ${{ parameters.vallyRoot }}
               evalInvokeTemplate: ${{ parameters.evalInvokeTemplate }}
               UseAzSdkAuthentication: ${{ parameters.UseAzSdkAuthentication }}
+              azureSubscription: ${{ parameters.azureSubscription }}
+              vallyArguments: ${{ parameters.vallyArguments }}
+              failOnFailedTests: ${{ parameters.failOnFailedTests }}
               threshold: ${{ parameters.threshold }}
               preEvalTemplate: ${{ parameters.preEvalTemplate }}
               postEvalTemplate: ${{ parameters.postEvalTemplate }}

eng/common/pipelines/templates/steps/eval-invoke.yml

@@ -2,6 +2,7 @@
 # node script directly. UseAzSdkAuthentication=true wraps the same run in AzureCLI@2 so a live
 # MCP's Azure DevOps calls inherit the service-connection identity.
 # The shard's login-to-copilot template supplies the executor and prompt-judge token.
+# Callers may override the existing service connection through azureSubscription.
 
 parameters:
   - name: vallyRoot
@@ -16,9 +17,21 @@ parameters:
   - name: azureSubscription
     type: string
     default: opensource-api-connection
+  - name: vallyArguments
+    # Supplementary non-reserved --option value pairs and --switch flags, not shell text.
+    # --tag does not filter the matrix; callers must keep every planned eval non-empty.
+    # JSON-escape dollar signs below so agent macros cannot rewrite argument data.
+    type: object
+    default: []
 
 steps:
-  - ${{ if parameters.UseAzSdkAuthentication }}:
+  - ${{ if and(parameters.UseAzSdkAuthentication, eq(trim(parameters.azureSubscription), '')) }}:
+    - script: |
+        echo "##vso[task.logissue type=error]azureSubscription must be provided when UseAzSdkAuthentication is true."
+        exit 1
+      displayName: 'Validate authenticated eval service connection'
+
+  - ${{ elseif parameters.UseAzSdkAuthentication }}:
     - task: AzureCLI@2
       displayName: 'vally eval $(shardName) (authenticated)'
       inputs:
@@ -34,6 +47,7 @@ steps:
             --threshold ${{ parameters.threshold }}
       env:
         COPILOT_GITHUB_TOKEN: $(COPILOT_GITHUB_TOKEN)
+        VALLY_ARGUMENTS_JSON: ${{ replace(convertToJson(parameters.vallyArguments), '$', '\u0024') }}
 
   - ${{ else }}:
     - script: |
@@ -46,3 +60,4 @@ steps:
       workingDirectory: ${{ parameters.vallyRoot }}
       env:
         COPILOT_GITHUB_TOKEN: $(COPILOT_GITHUB_TOKEN)
+        VALLY_ARGUMENTS_JSON: ${{ replace(convertToJson(parameters.vallyArguments), '$', '\u0024') }}

eng/common/scripts/eval/README.md

@@ -13,6 +13,91 @@ templates.
   its own pnpm package/lock for **fflate only** and CLI/PowerShell tests. Shard evaluator
   restores do not install it; publication adds no Node Azure SDK dependencies.
 
+## Extending a shard job
+
+The shared `eng/common/pipelines/templates/jobs/eval-shard.yml` owns checkout,
+Node/Vally installation, MCP staging, git fixture initialization and Copilot login.
+Its defaults preserve the standard Prepare/Eval/Summary pipeline behavior.
+Repo-local adapters can customize it without copying that infrastructure:
+
+| Parameter | Contract |
+| --- | --- |
+| `preEvalTemplate` / `postEvalTemplate` | Existing parameter-free setup/teardown step templates. |
+| `evalInvokeTemplate` | Existing invocation template accepting `vallyRoot`, `UseAzSdkAuthentication` and `threshold`. |
+| `azureSubscription` | Optional service-connection override. Stage/job templates default to `''` and omit an empty override; the common invoke template retains its `opensource-api-connection` default. |
+| `vallyArguments` | Optional array of supplementary Vally options, values and switches. Strings are required except that trial/worker counts also accept numbers. Available through the stage, shard job and common invoke template. Defaults to `[]`. |
+
+**Not allowed in `vallyArguments`:**
+
+| Affected part | Reserved options | Reason |
+| --- | --- | --- |
+| Shard structure and execution context | `--eval-spec` / `-e`, `--no-eval-spec`, `--suite`, `--no-suite`, `--eval-plugin`, `--no-eval-plugin`, `--work-dir`, `--no-work-dir` | Prepare already assigns files to shards. These change eval selection/loading or the working directory; `--suite` also conflicts with the assigned `-e` arguments. Use `evalGlobs` and `vallyRoot` instead. |
+| Result collection and Summary | `--output-dir`, `--no-output-dir`, `--output`, `--no-output`, `--junit`, `--no-junit` | The pipeline owns result locations and enables JUnit. In Vally 0.19.0, `--output jsonl` sends results to stdout instead of `results.jsonl`. |
+| Validation and scoring | `--threshold`, `--no-threshold`, `--skip-grade`, `--no-skip-grade`, `--skip-validate`, `--no-skip-validate` | The runner does not support bypassing validation/grading, and scoring must match the shard gate. Set the template's `threshold` parameter instead. |
+| Exit-code gating | `--require-pass`, `--no-require-pass` | The shard decides pass/fail from `results.jsonl`, not Vally's exit code; this option would not control the pipeline gate. |
+
+**Allowed:** other Vally options using `--option value` or standalone switches such as
+`--verbose`. This includes `--tag`, `--runs`, `--workers`, `--timeout`,
+`--model` and supplementary plugin options. New options need no common-template changes;
+Vally validates their names, values and whether they require a value.
+Aliases, `--flag=value` and positional paths/globs are not supported.
+Values must be nonempty strings without null characters and must not start with `-`;
+`--runs` and `--workers` also accept positive safe integer numbers.
+Arguments are passed literally, without a shell or whitespace splitting.
+
+For example, replace `<tag-key>` and `<tag-value>` with a tag defined in your eval stimuli:
+
+```yaml
+vallyArguments:
+  - --tag
+  - '<tag-key>=<tag-value>'
+  - --runs
+  - 3
+  - --workers
+  - 2
+  - --verbose
+failOnFailedTests: false
+```
+
+Multiple runs require `failOnFailedTests: false`, including repetition configured
+outside `vallyArguments`. `--tag` uses keys and values defined in your eval stimuli,
+not predefined mode names. Tags filter stimuli, not the shard matrix: keep
+`evalGlobs` aligned so every selected eval has matching stimuli.
+Custom invoke templates must declare `vallyArguments` to receive a non-empty array.
+
+Use the existing pre/post-eval and MCP setup hooks for repository-specific work.
+Authenticated callers can pass `azureSubscription` explicitly to the stage or shard
+template. Nonempty overrides are forwarded to the common invoke step; otherwise
+the invoke template retains its existing `opensource-api-connection` default.
+An explicitly empty or whitespace-only value passed directly to the invoke template fails with
+a configuration error without creating an `AzureCLI@2` task. Unauthenticated runs
+do not need the parameter. Custom invoke templates receive it only when nonempty
+and must declare their own `azureSubscription` parameter to opt in.
+Configure an authorized connection in repo-owned YAML and explicitly pass it from
+the entrypoint to the stage template. Do not rely on an
+ambient variable being visible across nested templates, or on task/output variables
+that are unavailable during template expansion.
+The `eng/common/pipelines/live-eval.yml` entrypoint imports the repo-owned
+`eng/pipelines/templates/variables/globals.yml` and passes its `EvalAzureSubscription`
+value as `azureSubscription`. Nested stage, job and invoke templates use only the
+parameter. Scheduled runs use the same YAML configuration without a connection run
+parameter. Other repos must configure their own authorized connection in that
+repo-owned variable template.
+Do not replace authentication, result collection or retry/completeness checks.
+Supplementary reports must not rewrite JUnit thresholds or raw results.
+
+The shared job templates use the `Prepare` matrix and `Eval` timeline stage, with
+one build-wide Summary depending on both stages. Result/debug artifacts are named
+`eval-result-<shard>-<attempt>` / `eval-debug-<shard>-<attempt>`; the summary artifact
+is `eval-summary-<attempt>`. Summary downloads `eval-result-*` result data and selects
+only the latest verified attempt for each expected shard.
+Shard names must start with an ASCII letter, digit or underscore, contain only
+ASCII letters, digits, underscores or hyphens, and be at most 200 characters.
+
+Benchmarks select one mode per build through repo-owned pipeline parameters,
+aligned `evalGlobs` and `vallyArguments`. Run separate builds to compare modes;
+multiple independently published tracks within one build are not supported.
+
 ## Complete build results and direct Blob publishing
 
 Shards now retain the newest invocation's raw JSONL, JUnit and a completion marker
@@ -50,11 +135,11 @@ explicit bootstrap, without changing dependency checks or the frozen lock.
 
 | Control | Current use |
 | --- | --- |
-| `publishDashboardResults` | The only checkbox; defaults to `true` in the three tools entrypoints. Set `false` to disable both upload and Azure Storage egress |
+| `publishDashboardResults` | The only checkbox; defaults to `true` in the three standard tools entrypoints. Set `false` to disable both upload and Azure Storage egress |
 | `EvalStorageAccountName` | Pipeline-owner configuration. Upload is skipped when absent; set it to `evaltestsummary` for this dashboard |
 | `EvalDashboardRefreshUrl` | Pipeline-owner configuration: `https://azsdk-eval-bue6a7dwanatgpb3.westus3-01.azurewebsites.net/api/refresh` |
 | `EvalDashboardResource` | Pipeline-owner configuration: `api://258998df-81ec-460c-bdd7-56a9bdde1e48` (resource URI, not a `/.default` scope) |
-| `createDashboardBundle` | Shared-template artifact-only mode; enabled for all three tools entrypoints |
+| `createDashboardBundle` | Shared-template artifact-only mode; enabled for the three standard tools entrypoints |
 
 The pipeline owner must set **`EvalStorageAccountName=evaltestsummary`** and the two
 dashboard variables above in each onboarded DevOps pipeline, with **queue-time
@@ -155,6 +240,9 @@ then `node --experimental-strip-types --test` from this directory.
 These shared Node tests do not need Tools-owned CI files, PowerShell tests or
 the consuming repository's redirect settings. A consumer-copy regression runs
 them without Tools files or Git history, with absent and different redirect templates.
+Template contract checks use bounded text assertions, not a YAML parser, and need
+no evaluator dependencies. They do not evaluate Azure template expressions;
+server-side expansion and service-connection authorization require separate validation.
 
 In Tools, run `Invoke-Pester -Path eng/common-tests/eval/Publish-EvalResults.Tests.ps1 -CI`
 from the repository root for the **PowerShell 7.4+ / Pester 5 / Azure CLI** tests. That Tools-only

eng/common/scripts/eval/build-eval-summary.ts

@@ -413,7 +413,9 @@ async function main(argv) {
 
   if (process.env.TF_BUILD) {
     console.log(`##vso[task.uploadsummary]${path.resolve(options.outputPath)}`);
-    if (selection) console.log(`##vso[task.setvariable variable=EvalSummaryComplete]${selection.complete}`);
+  }
+  if (process.env.TF_BUILD && selection) {
+    console.log(`##vso[task.setvariable variable=EvalSummaryComplete]${selection.complete}`);
   }
 
   // Completed evaluation failures remain governed by failOnFailedTests. Missing result