From eed17ed927003d8b94a73a6687b08986e9950780 Mon Sep 17 00:00:00 2001 From: Andriy Svyryd Date: Tue, 15 Sep 2026 18:29:11 -0700 Subject: [PATCH 1/3] Add a Vally-based evaluation harness for repository-owned Copilot customizations. Discovers and validates instructions, skills, agents, prompts, and agentic workflows by convention. Runs paired treatment and unskilled-control evaluations with token grading and Vally comparison. Selectively evaluates changed components and publishes progressive PR results. Restricts secret-backed runs to trusted PR authors; later commits require manual reruns. Replace repository MCP configuration and duplicated skills with plugins. Update skill guidance and add matching repository-grounded evaluations. Add Dependabot coverage for Vally dependencies. --- .agents/skills/change-tracking/SKILL.md | 13 +- .agents/skills/make-custom-agent/SKILL.md | 3 +- .agents/skills/make-instructions/SKILL.md | 11 +- .agents/skills/make-skill/SKILL.md | 42 +- .../make-skill/references/testing-patterns.md | 15 +- .agents/skills/migrations/SKILL.md | 5 +- .agents/skills/run-apichief/SKILL.md | 2 +- .agents/skills/servicing-pr/SKILL.md | 3 +- .agents/skills/sqlite-adonet/SKILL.md | 1 + .agents/skills/testing/SKILL.md | 94 - .agents/skills/tooling/SKILL.md | 2 +- .agents/skills/update-pipeline/SKILL.md | 8 +- .claude/settings.json | 16 +- .github/copilot-instructions.md | 12 +- .github/copilot/settings.json | 22 +- .github/dependabot.yml | 12 + .github/workflows/agent-harness-coverage.yml | 52 + .../workflows/agent-harness-evaluation.yml | 786 +++++++ .github/workflows/pr-comment-command.yml | 53 + .vscode/mcp.json | 48 - eng/harness-evaluation/README.md | 76 + .../harness.experiment.yaml | 13 + .../copilot-instructions/eval.yaml | 87 + eng/harness-evaluation/package-lock.json | 2090 +++++++++++++++++ eng/harness-evaluation/package.json | 19 + .../skills/change-tracking/eval.yaml | 59 + .../skills/cosmos-provider/eval.yaml | 59 + .../skills/make-custom-agent/eval.yaml | 66 + .../make-github-actions-workflow/eval.yaml | 60 + .../skills/make-instructions/eval.yaml | 64 + .../skills/make-skill/eval.yaml | 62 + .../skills/migrations/eval.yaml | 57 + .../skills/model-building/eval.yaml | 56 + .../skills/query-pipeline/eval.yaml | 56 + .../skills/run-apichief/eval.yaml | 56 + .../skills/scaffolding/eval.yaml | 56 + .../skills/servicing-pr/eval.yaml | 58 + .../skills/sqlite-adonet/eval.yaml | 58 + .../skills/tooling/eval.yaml | 58 + .../skills/triage/eval.yaml | 83 + .../skills/update-pipeline/eval.yaml | 58 + eng/harness-evaluation/src/cli.mjs | 128 + eng/harness-evaluation/src/harness.mjs | 357 +++ eng/harness-evaluation/test/harness.test.mjs | 335 +++ 44 files changed, 5076 insertions(+), 195 deletions(-) delete mode 100644 .agents/skills/testing/SKILL.md create mode 100644 .github/workflows/agent-harness-coverage.yml create mode 100644 .github/workflows/agent-harness-evaluation.yml create mode 100644 .github/workflows/pr-comment-command.yml delete mode 100644 .vscode/mcp.json create mode 100644 eng/harness-evaluation/README.md create mode 100644 eng/harness-evaluation/harness.experiment.yaml create mode 100644 eng/harness-evaluation/instructions/copilot-instructions/eval.yaml create mode 100644 eng/harness-evaluation/package-lock.json create mode 100644 eng/harness-evaluation/package.json create mode 100644 eng/harness-evaluation/skills/change-tracking/eval.yaml create mode 100644 eng/harness-evaluation/skills/cosmos-provider/eval.yaml create mode 100644 eng/harness-evaluation/skills/make-custom-agent/eval.yaml create mode 100644 eng/harness-evaluation/skills/make-github-actions-workflow/eval.yaml create mode 100644 eng/harness-evaluation/skills/make-instructions/eval.yaml create mode 100644 eng/harness-evaluation/skills/make-skill/eval.yaml create mode 100644 eng/harness-evaluation/skills/migrations/eval.yaml create mode 100644 eng/harness-evaluation/skills/model-building/eval.yaml create mode 100644 eng/harness-evaluation/skills/query-pipeline/eval.yaml create mode 100644 eng/harness-evaluation/skills/run-apichief/eval.yaml create mode 100644 eng/harness-evaluation/skills/scaffolding/eval.yaml create mode 100644 eng/harness-evaluation/skills/servicing-pr/eval.yaml create mode 100644 eng/harness-evaluation/skills/sqlite-adonet/eval.yaml create mode 100644 eng/harness-evaluation/skills/tooling/eval.yaml create mode 100644 eng/harness-evaluation/skills/triage/eval.yaml create mode 100644 eng/harness-evaluation/skills/update-pipeline/eval.yaml create mode 100644 eng/harness-evaluation/src/cli.mjs create mode 100644 eng/harness-evaluation/src/harness.mjs create mode 100644 eng/harness-evaluation/test/harness.test.mjs diff --git a/.agents/skills/change-tracking/SKILL.md b/.agents/skills/change-tracking/SKILL.md index c7ef97dbe3a..5ce64693ddb 100644 --- a/.agents/skills/change-tracking/SKILL.md +++ b/.agents/skills/change-tracking/SKILL.md @@ -10,12 +10,19 @@ Manages entity states and detects changes for `SaveChanges()`. ## Core Components -- `StateManager` — central engine, identity maps, tracks all entities -- `InternalEntityEntry` — per-entity state, property flags, snapshots -- `SnapshotFactoryFactory` subclasses build snapshot factories for change detection +- `StateManager` — owns tracked entries, identity/reference maps, fixup, cascades, notifications, and changed counts +- `InternalEntryBase` and derived classes — own per-entry state, flags, values, snapshots, and ordered state transitions +- `ChangeDetector` — compares current values with baselines and reports changes through entry mutation APIs +- `SnapshotFactoryFactory` subclasses — create passive original- and relationship-value baselines +- `IdentityMap` — permits one active entry per key; shared identity pairs a replacement with the prior `Deleted` entry - `PropertyAccessorsFactory`, `ClrPropertyGetterFactory` and `ClrPropertySetterFactory` compile property accessors for efficient snapshotting and change detection - Ordinals in `indices` parameter specify element at each complex collection depth +## Change Detection + +- Snapshot and notification strategies both call `SetPropertyModified()` to keep property flags and entity state consistent. +- `SetEntityState()` validates values, updates flags and complex entries, changes state, then runs manager bookkeeping hooks. + ## Testing Unit tests: `test/EFCore.Tests/ChangeTracking/`. Functional tests: `test/EFCore.Specification.Tests/GraphUpdates/`. diff --git a/.agents/skills/make-custom-agent/SKILL.md b/.agents/skills/make-custom-agent/SKILL.md index e9c1f81a7e6..8c77ab0fa00 100644 --- a/.agents/skills/make-custom-agent/SKILL.md +++ b/.agents/skills/make-custom-agent/SKILL.md @@ -242,5 +242,4 @@ After creating or modifying an agent, verify: - [GitHub Copilot Extensions documentation](https://docs.github.com/en/copilot/building-copilot-extensions/about-building-copilot-extensions) - [GitHub Copilot Custom agents configuration](https://docs.github.com/en/copilot/reference/custom-agents-configuration) - [Agent Skills Specification](https://agentskills.io/specification) -- [make-skill](../make-skill/SKILL.md) -- [make-instructions](../make-instructions/SKILL.md) +- Related repository skills: `make-skill` and `make-instructions` diff --git a/.agents/skills/make-instructions/SKILL.md b/.agents/skills/make-instructions/SKILL.md index 551dc3eddd6..562c3b788c0 100644 --- a/.agents/skills/make-instructions/SKILL.md +++ b/.agents/skills/make-instructions/SKILL.md @@ -1,6 +1,6 @@ --- name: make-instructions -description: 'Create VS Code file-based instructions (.instructions.md files). Use when asked to create, scaffold, or add file-based instructions for Copilot. Generates .instructions.md with YAML frontmatter and background knowledge content.' +description: 'Create and evaluate VS Code file-based instructions (.instructions.md files). Use when asked to create, scaffold, or add file-based instructions for Copilot. Generates scoped instructions and a paired Vally harness eval.' --- # Create File-Based Instructions @@ -24,6 +24,8 @@ Build understanding of the area the instructions should cover. Identify: - [ ] Common pitfalls that Copilot should avoid - [ ] Non-obvious domain knowledge that isn't discoverable from code alone +Read the repository-wide instruction files that apply to the same paths and make an explicit exclusion list. Do not repeat those rules in the new file, even when they are relevant examples for the scoped area; include only guidance that becomes more specific or materially different at the narrower scope. + If the scope is unclear or overlaps with existing instructions, ask the user for clarification. ### Step 2: Choose the file location @@ -73,7 +75,11 @@ Recommended sections (adapt as needed): 5. **Key Files** — table of important files for orientation (optional) 6. **Common Pitfalls** — traps to avoid (optional) -### Step 5: Validate +### Step 5: Author and validate the harness evaluation + +Create `eng/harness-evaluation/instructions//eval.yaml`, where `` is the instruction path relative to `.github/instructions/` with its suffix removed and nested path separators replaced by `--`. + +Follow the authoring and validation rules in `eng/harness-evaluation/README.md`. Exercise guidance distinctive to the instruction and ensure the treatment meaningfully outperforms the omitted-instruction control. After creating the file, verify: @@ -84,6 +90,7 @@ After creating the file, verify: - [ ] Content is concise (aim for under 500 lines or 5000 tokens) — long instructions dilute effectiveness - [ ] No secrets, tokens, or internal URLs included - [ ] Instructions don't duplicate what's already in `.github/copilot-instructions.md` or under `.agents/skills/` +- [ ] The paired eval demonstrates behavior that the omitted-instruction control does not provide reliably ## Common Pitfalls diff --git a/.agents/skills/make-skill/SKILL.md b/.agents/skills/make-skill/SKILL.md index 9b8ec3dc1ba..850f2045b5d 100644 --- a/.agents/skills/make-skill/SKILL.md +++ b/.agents/skills/make-skill/SKILL.md @@ -1,6 +1,6 @@ --- name: make-skill -description: 'Create new Agent Skills for GitHub Copilot. Use when asked to create, scaffold, or add a skill. Generates SKILL.md with frontmatter, directory structure, and optional resources.' +description: 'Create and evaluate new Agent Skills for GitHub Copilot. Use when asked to create, scaffold, or add a skill. Generates SKILL.md, optional resources, and a paired Vally harness eval.' --- # Create Skill @@ -33,8 +33,7 @@ After investigating, verify: - [ ] Have search queries for deeper topics - [ ] Can determine if the skill should be user-invocable or background knowledge only -If there are any ambiguities, gaps in understanding, or multiple valid approaches, ask the user for clarification before proceeding to skill creation. -Also, evaluate whether the task might be better handled by a custom agent, agentic workflow, an existing skill or multiple narrower skills, and discuss this with the user if relevant. +If there are any ambiguities, gaps in understanding, or multiple valid approaches, ask the user for clarification before proceeding to skill creation. Also, evaluate whether the task might be better handled by a custom agent, agentic workflow, an existing skill or multiple narrower skills, and discuss this with the user if relevant. ### Step 2: Create the skill directory @@ -92,26 +91,22 @@ Include these recommended sections, following this file's structure: > ❌ **NEVER** count API failures as success. Return "Unknown" and exclude from positive counts. -### Step 7: Validate the skill - -Ensure the name: -- Does not start or end with a hyphen -- Does not contain consecutive hyphens -- Is between 1-64 characters -- YAML frontmatter name matches directory name exactly - -After creating a skill, verify: -- [ ] frontmatter fields are valid -- [ ] SKILL.md is under 500 lines and 5000 tokens, split into references if needed -- [ ] File references use relative paths -- [ ] Instructions are actionable and specific -- [ ] Instructions don't duplicate what's already in `.github/copilot-instructions.md` or under `.github/instructions/` -- [ ] Workflow has numbered steps with clear checkpoints -- [ ] Validation section exists with observable success criteria -- [ ] No secrets, tokens, or internal URLs included -- [ ] Common pitfalls are relevant and have solutions +### Step 7: Author and validate the harness evaluation + +Create `eng/harness-evaluation/skills//eval.yaml` and follow the authoring and validation rules in `eng/harness-evaluation/README.md`. The eval must require exact invocation of `` and meaningfully distinguish the skilled treatment from the unskilled control. + +Also verify: + +- [ ] The skill name does not start or end with a hyphen, contain consecutive hyphens, or exceed 64 characters +- [ ] YAML frontmatter name matches the directory name exactly and all frontmatter fields are valid +- [ ] SKILL.md is under 500 lines and 5000 tokens, splitting stable detail into references when needed +- [ ] File references are relative and instructions are actionable and specific +- [ ] Instructions do not duplicate `.github/copilot-instructions.md` or `.github/instructions/` +- [ ] The workflow has numbered steps and observable success criteria +- [ ] No secrets, tokens, or internal URLs are included - [ ] Optional directories are used appropriately -- [ ] Scripts handle edge cases gracefully and return structured outputs and helpful error messages when applicable +- [ ] Scripts handle edge cases, fail closed, and return structured, helpful errors +- [ ] The paired Vally comparison demonstrates distinctive value over the unskilled control ### Step 8: Test with Multi-Model Subagents @@ -141,5 +136,4 @@ Follow [references/testing-patterns.md](references/testing-patterns.md): ## References - [Agent Skills Specification](https://agentskills.io/specification) -- [Copilot Instructions](../../../.github/copilot-instructions.md) -- [Contributing Guidelines](../../../.github/CONTRIBUTING.md) +- Repository guidance: `.github/copilot-instructions.md` and `.github/CONTRIBUTING.md` diff --git a/.agents/skills/make-skill/references/testing-patterns.md b/.agents/skills/make-skill/references/testing-patterns.md index 44799155b23..f8acfa899fd 100644 --- a/.agents/skills/make-skill/references/testing-patterns.md +++ b/.agents/skills/make-skill/references/testing-patterns.md @@ -184,16 +184,16 @@ task agent_type="general-purpose" model="{different-model}" prompt="Review the s The two approaches complement each other: writer-critic for creation/iteration, multi-model for validation. -## Waza Eval Testing +## Vally Evaluation -For repeatable, quantitative skill testing, use the **waza-eval** skill. It provides: +For repeatable, quantitative skill testing, author the repository's paired Vally eval. It provides: -- **Structured eval suites** — define tasks with prompts, expected outputs, and graders -- **Progression testing** — compare tool efficiency across skill versions from git history -- **Session capture** — commit result transcripts as golden sessions for regression detection -- **CI integration** — gate PRs on eval pass rates +- **Structured eval suites** — define repository-grounded stimuli, expected outputs, and graders +- **Unskilled controls** — compare identical tasks with and without the target skill +- **Quality and efficiency comparison** — report judge preference and token, turn, tool-call, time, and error deltas +- **CI integration** — require the treatment threshold and fail statistically significant regressions -Use waza evals when you need to *measure* whether a skill change improved behavior. Use multi-model review (above) when you need *qualitative* structural feedback. +Use `node eng/harness-evaluation/src/cli.mjs eval --workers 1 --require-pass` to measure whether a skill improves behavior with its configured trial count. Use multi-model review (above) for qualitative structural feedback. ### Regression Heuristics @@ -220,6 +220,7 @@ Evals should include trigger tests (does the skill activate correctly?): Before shipping a skill change: - [ ] Description matches trigger tests (USE FOR phrases appear in should-trigger prompts) +- [ ] `eng/harness-evaluation/skills//eval.yaml` passes treatment/control comparison - [ ] Stop signals are explicit with numeric bounds - [ ] Domain examples present (not just tool schemas) - [ ] Token budget met (SKILL.md under 4K orchestrating / 15K knowledge) diff --git a/.agents/skills/migrations/SKILL.md b/.agents/skills/migrations/SKILL.md index 96e5223a200..c7ecd80c0e2 100644 --- a/.agents/skills/migrations/SKILL.md +++ b/.agents/skills/migrations/SKILL.md @@ -16,7 +16,10 @@ user-invocable: false - Model snapshots use `typeof(Dictionary)` (property bag format), not the actual CLR type. When examining the `ClrType` in a snapshot, don't assume it matches the real entity type. - `SnapshotModelProcessor.Process()` is used at design-time to fixup older model snapshots for backward compatibility. +- `MigrationsModelDiffer` uses provider-agnostic structural comparison between relational models to determine what migration operations are necessary. ## Testing -Migration operation tests: `test/EFCore.Relational.Tests/Migrations/`. Functional tests: `test/EFCore.{Provider}.FunctionalTests/Migrations/`. Model differ tests: `test/EFCore.Relational.Tests/Migrations/Internal/MigrationsModelDifferTest*.cs`. \ No newline at end of file +Migration operation tests: `test/EFCore.Relational.Tests/Migrations/`. Functional tests: `test/EFCore.{Provider}.FunctionalTests/Migrations/`. Model differ tests: `test/EFCore.Relational.Tests/Migrations/Internal/MigrationsModelDifferTest*.cs`. + +To simulate a snapshot model use `ModelBuilder` calls without conventions. diff --git a/.agents/skills/run-apichief/SKILL.md b/.agents/skills/run-apichief/SKILL.md index e78bbca1c44..b50d7084507 100644 --- a/.agents/skills/run-apichief/SKILL.md +++ b/.agents/skills/run-apichief/SKILL.md @@ -6,7 +6,7 @@ user-invocable: false # Run ApiChief -Use the [ApiChief tool](../../../eng/Tools/ApiChief/README.md) to inspect or refresh EF Core public API baselines for projects under `src/`. +Use the ApiChief tool documented at `eng/Tools/ApiChief/README.md` to inspect or refresh EF Core public API baselines for projects under `src/`. ApiChief can run against either a compiled assembly or a previously emitted baseline JSON file. Prefer the repo-local `.dotnet` SDK and the checked-in build scripts in this repo. diff --git a/.agents/skills/servicing-pr/SKILL.md b/.agents/skills/servicing-pr/SKILL.md index e9dabb573ed..1dbe0d53904 100644 --- a/.agents/skills/servicing-pr/SKILL.md +++ b/.agents/skills/servicing-pr/SKILL.md @@ -44,7 +44,7 @@ Brief risk assessment ranked from "extremely low" to "high". Note amount of code ## Quirk (AppContext Switch) -A quirk lets users opt out of the fix at runtime, reducing patch risk. Add for all cases where it makes sense. Skip when the fix is 100% obvious/risk-free, or when the quirk couldn't be used, like in tools or analyzers. +A quirk lets users opt out of the fix at runtime, reducing patch risk. Skip when the fix is 100% obvious/risk-free, or when the quirk couldn't be used, like in tools or analyzers. ### Adding a Quirk @@ -58,6 +58,7 @@ private static readonly bool UseOldBehavior37585 = - Change `37585` to the relevant issue number - Wrap changes with a condition on `!UseOldBehavior37585` so activating the switch bypasses the fix, prefer to minimize the number of times the switch is checked - If the PR closes multiple issues, pick the most appropriate one for the switch name +- Distinguish a recommendation from the staged implementation. Do not say "Quirk added" unless the source change actually includes the switch. ## Validation diff --git a/.agents/skills/sqlite-adonet/SKILL.md b/.agents/skills/sqlite-adonet/SKILL.md index 6f07513f044..d62d66cf967 100644 --- a/.agents/skills/sqlite-adonet/SKILL.md +++ b/.agents/skills/sqlite-adonet/SKILL.md @@ -12,3 +12,4 @@ Standalone ADO.NET provider in `src/Microsoft.Data.Sqlite.Core/`, independent of - Static constructor calls `SQLitePCL.Batteries_V2.Init()` reflectively - `CreateFunction()`/`CreateAggregate()` overloads generated from T4 templates (`.tt` files) +- `DbConnection.Close()` closes the connection but does not dispose commands associated with it; a reusable `DbCommand` may remain associated and execute after that same connection is reopened. Keep connection-driven statement cleanup separate from `DbCommand.Dispose()`, which is the terminal command-lifetime operation. diff --git a/.agents/skills/testing/SKILL.md b/.agents/skills/testing/SKILL.md deleted file mode 100644 index ea148cdd365..00000000000 --- a/.agents/skills/testing/SKILL.md +++ /dev/null @@ -1,94 +0,0 @@ ---- -name: testing -description: 'Implementation details for EF Core test infrastructure. Use when changing test fixtures, SQL baseline assertions, test helpers, the test class hierarchy, or when adding new tests.' -user-invocable: false ---- - -# Testing - -## Test Categories - -### Unit Tests (`test/EFCore.Tests/`, `test/EFCore.Relational.Tests/`, `test/EFCore.{Provider}.Tests/`) -Isolated logic tests. Build models via `*TestHelpers.Instance.CreateConventionBuilder()`, resolve services from `CreateContextServices()`. No database needed. - -### Specification Tests (provider-agnostic abstract bases) -Define WHAT to test (LINQ queries, expected results). Can't be run directly — provider tests override to verify HOW (generated SQL). - -- Core → `test/EFCore.Specification.Tests/` -- Relational → `test/EFCore.Relational.Specification.Tests/` - -### Functional Tests (`test/EFCore.{Provider}.FunctionalTests/`) -Concrete provider tests inheriting specification tests. Most include SQL baseline assertions. - -## Test Class Hierarchy (Query Example) - -``` -QueryTestBase # Core - └─ NorthwindWhereQueryTestBase # Specification - └─ NorthwindWhereQueryRelationalTestBase # Relational specification - └─ NorthwindWhereQuerySqlServerTest # Provider (asserts SQL) -``` - -Provider override pattern: -```csharp -public override async Task Where_simple(bool async) -{ - await base.Where_simple(async); // runs LINQ + asserts results - AssertSql("""..."""); // asserts provider-specific SQL -} -``` - -## TestHelpers Hierarchy - -``` -TestHelpers (abstract) # EFCore.Specification.Tests - ├─ InMemoryTestHelpers # non-relational - └─ RelationalTestHelpers (abstract) # EFCore.Relational.Specification.Tests - ├─ SqlServerTestHelpers - └─ SqliteTestHelpers -``` - -Key methods: `CreateConventionBuilder()`, `CreateContextServices(model)`, `CreateOptions()` - -## Fixtures - -### SharedStoreFixtureBase -Many tests share one database. Creates `TestStore` + pooled `DbContextFactory` in `InitializeAsync()`. Seeds data once. Use for read-heavy tests (e.g., Northwind query tests). - -### NonSharedModelTestBase -Each test gets a fresh model/store. Call `InitializeAsync(onModelCreating, seed, ...)` per test. Use for tests needing unique schemas. - -## SQL Baseline Assertions - -`TestSqlLoggerFactory` captures SQL. `AssertSql("""...""")` compares against expected. Set `EF_TEST_REWRITE_BASELINES=1` to auto-rewrite baselines via Roslyn. - -## Workflow: Adding New Tests - -1. **Specification test**: Add to `EFCore.Specification.Tests` (core) or `EFCore.Relational.Specification.Tests` (relational) -2. **Provider overrides**: Override in **every** provider functional test class (`EFCore.{Provider}.FunctionalTests`) that inherits the base with provider-appropriate assertions. -3. **Unit test**: Add to `EFCore.{Provider}.Tests` -4. Run with `EF_TEST_REWRITE_BASELINES=1` to capture initial baselines -5. Run tests with project rebuilding enabled (don't use `--no-build`) to ensure code changes are picked up -6. When testing cross-platform code (e.g., file paths, path separators), verify the fix works on both Windows and Linux/macOS - -## Common Pitfalls - -| Pitfall | Solution | -|---------|----------| -| Baseline mismatch (SQL or compiled model) | Re-run with `EF_TEST_REWRITE_BASELINES=1` | -| `Check_all_tests_overridden` fails | Override the new test in every inheriting provider class | -| SQL Server feature missing at lower compat level | Gate with `[SqlServerCondition(...)]`| - -## Running Tests Directly Against the Built Assembly - -When invoking the test DLL via `dotnet exec` (e.g. to isolate a single test method without rebuilding through `dotnet test`), always append `--filter-not-trait category=failing --ignore-exit-code 8`: - -```pwsh -dotnet exec ./Microsoft.EntityFrameworkCore.SqlServer.FunctionalTests.dll ` - --filter-method '*MigrationsSqlServerTest.Create_json_index_over_whole_complex_collection' ` - --filter-not-trait category=failing --ignore-exit-code 8 -``` - -These flags mirror what `test/Directory.Build.props` passes via `TestingPlatformCommandLineArguments`. - -Don't add `--no-build` unless the test assembly was built in the immediate previous step. diff --git a/.agents/skills/tooling/SKILL.md b/.agents/skills/tooling/SKILL.md index 480f8cc74bb..4b642c444ff 100644 --- a/.agents/skills/tooling/SKILL.md +++ b/.agents/skills/tooling/SKILL.md @@ -28,4 +28,4 @@ NuGet package `Microsoft.EntityFrameworkCore.Tasks` provides build/publish-time ## Validation - For tool changes, create a test project and manually run affected commands to verify behavior -- `dotnet ef migrations script` output matches expected DDL \ No newline at end of file +- `dotnet ef migrations script` output matches expected DDL diff --git a/.agents/skills/update-pipeline/SKILL.md b/.agents/skills/update-pipeline/SKILL.md index 400f9a95aea..a848839b57f 100644 --- a/.agents/skills/update-pipeline/SKILL.md +++ b/.agents/skills/update-pipeline/SKILL.md @@ -10,16 +10,12 @@ Converts tracked entity changes into database INSERT/UPDATE/DELETE commands duri ## Flow -`SaveChanges()` → `DetectChanges()` → `IDatabase.SaveChanges()` - → `UpdateAdapter` creates `IUpdateEntry` list - → `CommandBatchPreparer.BatchCommands()` +`SaveChanges()` → `DetectChanges()` → `IDatabase.SaveChanges()` → `UpdateAdapter` creates `IUpdateEntry` list → `CommandBatchPreparer.BatchCommands()` → `ModificationCommand` per table row, composed of `ColumnModification` per column → `SharedTableEntryMap` is used to track entries mapped to the same row → Topological sort via Multigraph (FK dependency ordering) → Groups into `ModificationCommandBatch` (respects max batch size) - → `UpdateSqlGenerator` generates SQL per batch - → `BatchExecutor` executes all batches in a transaction - → `StateManager.AcceptAllChanges()` + → `UpdateSqlGenerator` generates SQL per batch → `BatchExecutor` executes all batches in a transaction → `StateManager.AcceptAllChanges()` ## Concurrency diff --git a/.claude/settings.json b/.claude/settings.json index 35358294c16..89e87d2d918 100644 --- a/.claude/settings.json +++ b/.claude/settings.json @@ -5,9 +5,23 @@ "source": "github", "repo": "dotnet/arcade-skills" } + }, + "dotnet-skills": { + "source": { + "source": "github", + "repo": "dotnet/skills" + } } }, "enabledPlugins": { - "dotnet-dnceng@dotnet-arcade-skills": true + "dotnet-dnceng@dotnet-arcade-skills": true, + "dotnet-helix@dotnet-arcade-skills": true, + "dotnet-codeflow@dotnet-arcade-skills": true, + "dotnet@dotnet-skills": true, + "dotnet-test@dotnet-skills": true, + "dotnet-diag@dotnet-skills": true, + "dotnet-msbuild@dotnet-skills": true, + "dotnet-nuget@dotnet-skills": true, + "microsoft-docs@claude-plugins-official": true } } diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 035f331c7f2..e5e9045e2cd 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -2,9 +2,7 @@ This document provides guidance for working with code in the Entity Framework Core project. -If you are not sure, do not guess, just tell that you don't know or ask clarifying questions. -Don't just copy code that follows the same pattern in a different context. -Don't rely just on names to guess its function, evaluate the code based on the implementation and usage. +If you are not sure, do not guess, just tell that you don't know or ask clarifying questions. Don't just copy code that follows the same pattern in a different context. Don't rely just on names to guess its function, evaluate the code based on the implementation and usage. ## Code Style @@ -23,6 +21,14 @@ Don't rely just on names to guess its function, evaluate the code based on the i - **NEVER** hardcode package versions in `.csproj` files - Use `eng/Versions.props` and `Directory.Packages.props` for NuGet package version management +## Testing + +- Put provider-independent behavior in specification tests. Override those tests when a provider needs to add specific assertions such as `AssertSql` for providers that produce SQL. +- `NonSharedModelTestBase` supports both the tests that share a model as well as those that do not. +- Set `EF_TEST_REWRITE_BASELINES=1` to rewrite SQL and compiled-model baselines. +- Tests execute using MTP, call `dotnet exec --filter-method '' --filter-not-trait category=failing --ignore-exit-code 8` for focused runs. +- Do not add `--no-build` unless the test assembly was built in the immediately preceding step. + ## Implementation Guidelines - Write code that is secure by default. Avoid exposing potentially private or sensitive data diff --git a/.github/copilot/settings.json b/.github/copilot/settings.json index 35358294c16..2a22e9ff0b3 100644 --- a/.github/copilot/settings.json +++ b/.github/copilot/settings.json @@ -5,9 +5,29 @@ "source": "github", "repo": "dotnet/arcade-skills" } + }, + "dotnet-skills": { + "source": { + "source": "github", + "repo": "dotnet/skills" + } + }, + "microsoft-docs-marketplace": { + "source": { + "source": "github", + "repo": "microsoftdocs/mcp" + } } }, "enabledPlugins": { - "dotnet-dnceng@dotnet-arcade-skills": true + "dotnet-dnceng@dotnet-arcade-skills": true, + "dotnet-helix@dotnet-arcade-skills": true, + "dotnet-codeflow@dotnet-arcade-skills": true, + "dotnet@dotnet-skills": true, + "dotnet-test@dotnet-skills": true, + "dotnet-diag@dotnet-skills": true, + "dotnet-msbuild@dotnet-skills": true, + "dotnet-nuget@dotnet-skills": true, + "microsoft-docs@microsoft-docs-marketplace": true } } diff --git a/.github/dependabot.yml b/.github/dependabot.yml index b899b4dcda1..374f5c8788f 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -287,3 +287,15 @@ updates: interval: "weekly" labels: - "area-infrastructure" + + - package-ecosystem: "npm" + directory: "/eng/harness-evaluation" + open-pull-requests-limit: 5 + schedule: + interval: "weekly" + day: "monday" + labels: + - "area-infrastructure" + groups: + npm: + patterns: ["*"] diff --git a/.github/workflows/agent-harness-coverage.yml b/.github/workflows/agent-harness-coverage.yml new file mode 100644 index 00000000000..6d13e62ecee --- /dev/null +++ b/.github/workflows/agent-harness-coverage.yml @@ -0,0 +1,52 @@ +# Validates that every repository-owned instruction, skill, agent, agentic +# workflow, and prompt has a well-formed EF Core-grounded Vally evaluation. +# MCPs and plugins are excluded. + +name: Agent Harness Coverage + +on: + pull_request: + +permissions: + contents: read + +jobs: + coverage: + runs-on: ubuntu-24.04 + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Restore repository SDK + shell: bash + run: | + ./restore.sh + . ./activate.sh + echo "DOTNET_ROOT=$PWD/.dotnet" >> "$GITHUB_ENV" + echo "$PWD/.dotnet" >> "$GITHUB_PATH" + + - name: Set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 + with: + node-version: 22 + cache: npm + cache-dependency-path: eng/harness-evaluation/package-lock.json + + - name: Install evaluation tools + working-directory: eng/harness-evaluation + run: npm ci + + - name: Audit evaluation dependencies + working-directory: eng/harness-evaluation + run: npm audit --audit-level=moderate + + - name: Test evaluation harness + working-directory: eng/harness-evaluation + run: npm test + + - name: Validate customization coverage and lint skills and evaluations + working-directory: eng/harness-evaluation + run: npm run lint diff --git a/.github/workflows/agent-harness-evaluation.yml b/.github/workflows/agent-harness-evaluation.yml new file mode 100644 index 00000000000..dd6737f39b2 --- /dev/null +++ b/.github/workflows/agent-harness-evaluation.yml @@ -0,0 +1,786 @@ +# Discovers affected harness components for every PR. Automatic evaluation is +# limited to trusted authors; contributors can explicitly evaluate any PR. + +name: Agent Harness Evaluation + +on: + pull_request_target: + types: [opened, reopened, ready_for_review, synchronize] + workflow_dispatch: + inputs: + pull_request: + description: Pull request number to evaluate + required: true + type: number + commit_sha: + description: PR commit to evaluate; leave empty to evaluate the current head + required: false + type: string + component: + description: Affected component to evaluate; leave empty to evaluate all affected components + required: false + type: string + +permissions: + contents: read + +concurrency: + group: agent-harness-evaluation-${{ github.event.pull_request.number || inputs.pull_request }} + cancel-in-progress: true + +jobs: + discover: + runs-on: ubuntu-24.04 + permissions: + contents: read + issues: write + pull-requests: read + outputs: + base_sha: ${{ steps.pr.outputs.base_sha }} + head_sha: ${{ steps.pr.outputs.target_sha }} + pull_request: ${{ steps.pr.outputs.pull_request }} + can_use_secrets: ${{ steps.pr.outputs.can_use_secrets }} + component_filter: ${{ steps.pr.outputs.component_filter }} + author: ${{ steps.pr.outputs.author }} + matrix: ${{ steps.pr.outputs.matrix }} + has_entries: ${{ steps.pr.outputs.has_entries }} + components: ${{ steps.pr.outputs.components }} + steps: + - name: Resolve pull request and authorization + id: pr + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 + env: + MANUAL_COMPONENT: ${{ inputs.component }} + MANUAL_COMMIT_SHA: ${{ inputs.commit_sha }} + MANUAL_PULL_REQUEST: ${{ inputs.pull_request }} + with: + script: | + const manual = context.eventName === 'workflow_dispatch'; + if (manual && context.ref !== `refs/heads/${context.payload.repository.default_branch}`) { + throw new Error('Run manual harness evaluations from the repository default branch.'); + } + const pullNumber = manual + ? Number(process.env.MANUAL_PULL_REQUEST) + : context.payload.pull_request.number; + if (!Number.isInteger(pullNumber) || pullNumber <= 0) { + throw new Error(`Invalid pull request number: ${process.env.MANUAL_PULL_REQUEST}`); + } + const { data: pull } = await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: pullNumber + }); + const author = pull.user.login; + const componentFilter = manual ? process.env.MANUAL_COMPONENT.trim() : ''; + const requestedSha = manual ? process.env.MANUAL_COMMIT_SHA.trim() : ''; + if (requestedSha && !/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/i.test(requestedSha)) { + throw new Error(`Invalid commit SHA: ${requestedSha}`); + } + const targetSha = requestedSha || pull.head.sha; + if (requestedSha) { + const commits = await github.paginate(github.rest.pulls.listCommits, { + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: pullNumber, + per_page: 100 + }); + if (!commits.some(commit => commit.sha === targetSha)) { + throw new Error(`Commit ${targetSha} does not belong to PR #${pullNumber}.`); + } + } + + let permission = 'none'; + try { + const response = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: context.repo.owner, + repo: context.repo.repo, + username: author + }); + permission = response.data.permission; + } catch (error) { + if (error.status !== 404) throw error; + } + + // Helper function to check if a user is a bot + const isBot = (username) => { + const lower = username.toLowerCase(); + return lower === 'copilot' || + lower === 'dotnet-bot' || + lower.startsWith('app/') || + lower.includes('[bot]'); + }; + const fromBranch = pull.head.repo?.id === pull.base.repo.id; + const automatic = context.eventName === 'pull_request_target' + && context.payload.action !== 'synchronize'; + const canUseSecrets = isBot(author) + ? fromBranch && (manual || automatic) + : manual || (automatic && ['admin', 'maintain', 'write'].includes(permission)); + + const { data: targetCommit } = await github.rest.repos.getCommit({ + owner: context.repo.owner, + repo: context.repo.repo, + ref: targetSha + }); + const { data: tree } = await github.rest.git.getTree({ + owner: context.repo.owner, + repo: context.repo.repo, + tree_sha: targetCommit.commit.tree.sha, + recursive: 'true' + }); + if (tree.truncated) { + throw new Error(`The repository tree for ${targetSha} was truncated.`); + } + + const evaluationRoot = 'eng/harness-evaluation'; + const componentPatterns = [ + { directory: '.github/instructions', suffixes: ['.instructions.md'], kind: 'instruction', evalDirectory: 'instructions' }, + { directory: '.github/agents', suffixes: ['.agent.md', '.md'], kind: 'agent', evalDirectory: 'agents' }, + { directory: '.github/workflows', suffixes: ['.md'], kind: 'agentic-workflow', evalDirectory: 'workflows' }, + { directory: '.github/prompts', suffixes: ['.prompt.md'], kind: 'prompt', evalDirectory: 'prompts' } + ]; + const components = [{ + id: 'copilot-instructions', + kind: 'instruction', + source: '.github/copilot-instructions.md', + eval: `${evaluationRoot}/instructions/copilot-instructions/eval.yaml` + }]; + const paths = tree.tree + .filter(entry => entry.type === 'blob') + .map(entry => entry.path); + for (const path of paths) { + const skill = path.match(/^\.agents\/skills\/([^/]+)\/SKILL\.md$/); + if (skill) { + components.push({ + id: skill[1], + kind: 'skill', + source: path, + eval: `${evaluationRoot}/skills/${skill[1]}/eval.yaml` + }); + continue; + } + for (const componentPattern of componentPatterns) { + if (!path.startsWith(`${componentPattern.directory}/`)) continue; + const suffix = componentPattern.suffixes.find(candidate => path.endsWith(candidate)); + if (!suffix) continue; + const id = path.slice(componentPattern.directory.length + 1, -suffix.length).replaceAll('/', '--'); + components.push({ + id, + kind: componentPattern.kind, + source: path, + eval: `${evaluationRoot}/${componentPattern.evalDirectory}/${id}/eval.yaml` + }); + break; + } + } + + const pathSet = new Set(paths); + const componentIds = new Set(); + const componentEvals = new Set(); + for (const component of components) { + if (!/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(component.id) + || component.id.includes('..') + || componentIds.has(component.id)) { + throw new Error(`Invalid or duplicate component ID '${component.id}'.`); + } + componentIds.add(component.id); + componentEvals.add(component.eval); + if (!pathSet.has(component.source)) { + throw new Error(`${component.id}: source does not exist: ${component.source}`); + } + if (!pathSet.has(component.eval)) { + throw new Error(`${component.id}: eval does not exist: ${component.eval}`); + } + } + for (const path of paths) { + if (/^eng\/harness-evaluation\/(?:skills|instructions|agents|workflows|prompts)\/[^/]+\/eval\.yaml$/.test(path) + && !componentEvals.has(path)) { + throw new Error(`Evaluation has no matching harness component: ${path}`); + } + } + + let files; + if (targetSha === pull.head.sha) { + files = await github.paginate(github.rest.pulls.listFiles, { + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: pullNumber, + per_page: 100 + }); + } else { + const { data: comparison } = await github.rest.repos.compareCommitsWithBasehead({ + owner: context.repo.owner, + repo: context.repo.repo, + basehead: `${pull.base.sha}...${targetSha}` + }); + files = comparison.files ?? []; + if (files.length >= 300) { + throw new Error(`The changed-file list for ${targetSha} may be truncated.`); + } + } + const changedFiles = files.flatMap(file => + file.previous_filename ? [file.filename, file.previous_filename] : [file.filename]); + // package-lock.json is intentionally excluded because dependency-only changes should not affect eval results. + const infrastructurePrefixes = [ + `${evaluationRoot}/src/`, + `${evaluationRoot}/package.json`, + `${evaluationRoot}/harness.experiment.yaml` + ]; + const pathMatches = (path, candidate) => path === candidate || path.startsWith(`${candidate}/`); + let selected = changedFiles.some(path => + infrastructurePrefixes.some(prefix => path.startsWith(prefix))) + ? components.map(component => component.id) + : components + .filter(component => changedFiles.some(path => + pathMatches(path, component.eval.slice(0, component.eval.lastIndexOf('/'))) + || pathMatches(path, component.kind === 'skill' + ? component.source.slice(0, component.source.lastIndexOf('/')) + : component.source))) + .map(component => component.id); + selected = [...new Set(selected)].sort((left, right) => left.localeCompare(right)); + const byId = new Map(components.map(component => [component.id, component])); + if (componentFilter) { + if (!byId.has(componentFilter)) { + throw new Error(`Unknown component '${componentFilter}'.`); + } + if (!selected.includes(componentFilter)) { + throw new Error(`Component '${componentFilter}' is not affected by these changes.`); + } + selected = [componentFilter]; + } + const matrix = { + include: selected.map(id => ({ + id, + kind: byId.get(id).kind, + eval_path: byId.get(id).eval, + timeout_minutes: 360, + watchdog_minutes: 350 + })) + }; + + core.setOutput('base_sha', pull.base.sha); + core.setOutput('target_sha', targetSha); + core.setOutput('pull_request', String(pullNumber)); + core.setOutput('can_use_secrets', String(canUseSecrets)); + core.setOutput('component_filter', componentFilter); + core.setOutput('author', author); + core.setOutput('matrix', JSON.stringify(matrix)); + core.setOutput('has_entries', String(selected.length > 0)); + core.setOutput('components', JSON.stringify(selected)); + await core.summary + .addHeading('Agent Harness Evaluation') + .addRaw(`PR #${pullNumber} by ${author}: repository permission ${permission}; ` + + `evaluation ${canUseSecrets ? 'authorized' : 'requires explicit authorization'} for ${targetSha}.`) + .write(); + + - name: Update pull request metadata + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 + env: + CAN_USE_SECRETS: ${{ steps.pr.outputs.can_use_secrets }} + COMPONENT_FILTER: ${{ steps.pr.outputs.component_filter }} + HEAD_SHA: ${{ steps.pr.outputs.target_sha }} + PULL_REQUEST: ${{ steps.pr.outputs.pull_request }} + SELECTED_COMPONENTS: ${{ steps.pr.outputs.components }} + with: + script: | + const issueNumber = Number(process.env.PULL_REQUEST); + const selected = new Set(JSON.parse(process.env.SELECTED_COMPONENTS)); + const comments = await github.paginate(github.rest.issues.listComments, { + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issueNumber, + per_page: 100 + }); + if (!process.env.COMPONENT_FILTER) { + for (const comment of comments) { + const match = comment.body?.match(/^/); + if (comment.user?.login !== 'github-actions[bot]' || !match || selected.has(match[1])) continue; + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: comment.id, + body: `${match[0]}\n### Agent evaluation: \`${match[1]}\`\n\n` + + `Status: **superseded** by \`${process.env.HEAD_SHA}\`; this component is no longer affected.` + }); + } + } + + const rerunMarker = ''; + const rerunComment = comments.find(comment => + comment.user?.login === 'github-actions[bot]' && comment.body?.startsWith(rerunMarker)); + if (context.payload.action === 'synchronize' && selected.size > 0) { + const body = `${rerunMarker}\n### Agent harness evaluation is stale\n\n` + + `Evaluation does not run automatically for new commits. A contributor with write access can comment ` + + `\`/eval\` to run all affected components or \`/eval component-name\` to run one component.`; + if (rerunComment) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, repo: context.repo.repo, comment_id: rerunComment.id, body + }); + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, repo: context.repo.repo, issue_number: issueNumber, body + }); + } + } + + if (selected.size === 0) return; + + const label = 'area-harness'; + try { + await github.rest.issues.getLabel({ + owner: context.repo.owner, + repo: context.repo.repo, + name: label + }); + } catch (error) { + if (error.status === 404) { + throw new Error(`Required label '${label}' does not exist.`); + } + throw error; + } + await github.rest.issues.addLabels({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issueNumber, + labels: [label] + }); + + if (context.payload.action === 'synchronize') return; + + const promptMarker = ''; + const prompt = comments.find(comment => + comment.user?.login === 'github-actions[bot]' && comment.body?.startsWith(promptMarker)); + if (process.env.CAN_USE_SECRETS === 'true') { + if (prompt && !process.env.COMPONENT_FILTER) { + await github.rest.issues.deleteComment({ + owner: context.repo.owner, repo: context.repo.repo, comment_id: prompt.id + }); + } + if (rerunComment && !process.env.COMPONENT_FILTER) { + await github.rest.issues.deleteComment({ + owner: context.repo.owner, repo: context.repo.repo, comment_id: rerunComment.id + }); + } + return; + } + + const manualUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}` + + '/actions/workflows/agent-harness-evaluation.yml'; + const body = `${promptMarker}\n### Agent harness evaluation required\n\n` + + `This PR affects: ${[...selected].map(id => `\`${id}\``).join(', ')}. ` + + `Its author cannot use repository secrets in an automatic run. A contributor with write access can comment ` + + `\`/eval\` or \`/eval component-name\`, or ` + + `[run the workflow manually](${manualUrl}) for pull request **#${issueNumber}**.`; + if (prompt) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, repo: context.repo.repo, comment_id: prompt.id, body + }); + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, repo: context.repo.repo, issue_number: issueNumber, body + }); + } + + evaluate: + name: evaluate (${{ matrix.id }}) + needs: discover + if: needs.discover.outputs.has_entries == 'true' && needs.discover.outputs.can_use_secrets == 'true' + strategy: + fail-fast: false + max-parallel: 3 + matrix: ${{ fromJSON(needs.discover.outputs.matrix) }} + runs-on: ubuntu-24.04 + timeout-minutes: ${{ matrix.timeout_minutes }} + permissions: + contents: read + copilot-requests: write + issues: write + pull-requests: read + env: + COMPONENT: ${{ matrix.id }} + RESULT_ROOT: artifacts/TestResults/harness-evaluation/${{ matrix.id }} + steps: + - name: Check out pull request + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + ref: ${{ needs.discover.outputs.head_sha }} + fetch-depth: 0 + persist-credentials: false + + - name: Report evaluation started + continue-on-error: true + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 + env: + HEAD_SHA: ${{ needs.discover.outputs.head_sha }} + PULL_REQUEST: ${{ needs.discover.outputs.pull_request }} + COMPONENT: ${{ matrix.id }} + with: + script: | + const marker = ``; + const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + const manualUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}` + + '/actions/workflows/agent-harness-evaluation.yml'; + const body = `${marker}\n### Agent evaluation: \`${process.env.COMPONENT}\`\n\n` + + `Status: **running** for \`${process.env.HEAD_SHA}\`\n\n` + + `PR updates do not rerun evaluations automatically. [Run again manually](${manualUrl}) after later commits.\n\n` + + `[View workflow run](${runUrl})`; + const issueNumber = Number(process.env.PULL_REQUEST); + const comments = await github.paginate(github.rest.issues.listComments, { + owner: context.repo.owner, repo: context.repo.repo, issue_number: issueNumber, per_page: 100 + }); + const existing = comments.find(comment => + comment.user?.login === 'github-actions[bot]' && comment.body?.startsWith(marker)); + if (existing) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, repo: context.repo.repo, comment_id: existing.id, body + }); + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, repo: context.repo.repo, issue_number: issueNumber, body + }); + } + + - name: Restore repository SDK + shell: bash + run: | + ./restore.sh + . ./activate.sh + echo "DOTNET_ROOT=$PWD/.dotnet" >> "$GITHUB_ENV" + echo "$PWD/.dotnet" >> "$GITHUB_PATH" + + - name: Set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 + with: + node-version: 22 + cache: npm + cache-dependency-path: eng/harness-evaluation/package-lock.json + + - name: Install evaluation tools + working-directory: eng/harness-evaluation + run: npm ci + + - name: Run treatment, unskilled control, and comparison + id: run + shell: bash + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + timeout --signal=TERM --kill-after=30s "${{ matrix.watchdog_minutes }}m" \ + node eng/harness-evaluation/src/cli.mjs eval "$COMPONENT" \ + --workers 1 \ + --require-pass \ + --output "$RESULT_ROOT" + + - name: Upload component results + id: upload + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + with: + name: harness-eval-result-${{ matrix.id }} + path: ${{ env.RESULT_ROOT }}/ + include-hidden-files: true + if-no-files-found: warn + retention-days: 14 + + - name: Verify Vally reports + id: verify + if: steps.run.outcome == 'success' + shell: bash + run: | + node --input-type=module <<'EOF' + import fs from 'node:fs'; + import path from 'node:path'; + + const resultFiles = []; + const planFiles = []; + const visit = directory => { + for (const entry of fs.readdirSync(directory, { withFileTypes: true })) { + const fullPath = path.join(directory, entry.name); + if (entry.isDirectory()) visit(fullPath); + else if (entry.name === 'results.jsonl') resultFiles.push(fullPath); + else if (entry.name === 'plan-snapshot.json') planFiles.push(fullPath); + } + }; + visit(process.env.RESULT_ROOT); + const controlFile = resultFiles.find(file => file.split(path.sep).includes('control')); + const treatmentFile = resultFiles.find(file => file.split(path.sep).includes('treatment')); + if (!controlFile || !treatmentFile || resultFiles.length !== 2) { + throw new Error(`Expected control and treatment results.jsonl files, found ${resultFiles.length}.`); + } + if (planFiles.length !== 1) { + throw new Error(`Expected one plan-snapshot.json file, found ${planFiles.length}.`); + } + + const readRecords = file => fs.readFileSync(file, 'utf8').trim().split(/\r?\n/).map(JSON.parse); + const plan = JSON.parse(fs.readFileSync(planFiles[0], 'utf8')); + if (plan.type !== 'experiment-plan-snapshot' || !Array.isArray(plan.evals)) { + throw new Error('Vally did not produce a valid experiment plan snapshot.'); + } + const control = readRecords(controlFile); + const treatment = readRecords(treatmentFile); + const controlTrials = control.filter(record => record.type === 'trial-result'); + const treatmentTrials = treatment.filter(record => record.type === 'trial-result'); + const incomplete = trial => trial.status !== 'success' || !trial.gradeResult; + const trialKeys = trials => trials + .map(trial => `${trial.evalName}\0${trial.model ?? ''}\0${trial.stimulus}\0${trial.trialIndex ?? 0}`) + .sort(); + const plannedTrialKeys = variant => plan.evals + .filter(evalPlan => evalPlan.variant === variant && !evalPlan.failure) + .flatMap(evalPlan => { + if (!Number.isSafeInteger(evalPlan.runs) || evalPlan.runs <= 0 || + !Array.isArray(evalPlan.plannedStimulusNames)) { + throw new Error(`Invalid trial plan for ${variant}/${evalPlan.evalName}.`); + } + return evalPlan.plannedStimulusNames.flatMap(stimulus => + Array.from({ length: evalPlan.runs }, (_, trialIndex) => + `${evalPlan.evalName}\0${evalPlan.model ?? ''}\0${stimulus}\0${trialIndex}`)); + }) + .sort(); + const controlKeys = trialKeys(controlTrials); + const treatmentKeys = trialKeys(treatmentTrials); + if (controlTrials.length === 0 || + controlTrials.length !== treatmentTrials.length || + controlTrials.some(incomplete) || treatmentTrials.some(incomplete) || + JSON.stringify(controlKeys) !== JSON.stringify(plannedTrialKeys('control')) || + JSON.stringify(treatmentKeys) !== JSON.stringify(plannedTrialKeys('treatment')) || + JSON.stringify(controlKeys) !== JSON.stringify(treatmentKeys)) { + throw new Error('Vally did not produce complete control and passing treatment results.'); + } + + const comparisons = readRecords(path.join(process.env.RESULT_ROOT, 'comparison.jsonl')); + if (comparisons.length !== 1 || comparisons[0].type !== 'comparison' || + comparisons[0].summary.erroredCount !== 0 || + comparisons[0].summary.trialCount !== treatmentTrials.length || + !comparisons[0].summary.metricDeltas.some(metric => + metric.metric === 'totalTokens' && Number.isFinite(metric.mean))) { + throw new Error('Vally did not produce a complete comparison with token metrics.'); + } + EOF + + - name: Report component result + if: always() + continue-on-error: true + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 + env: + ARTIFACT_URL: ${{ steps.upload.outputs.artifact-url }} + BASE_SHA: ${{ needs.discover.outputs.base_sha }} + COMPONENT: ${{ matrix.id }} + EVAL_PATH: ${{ matrix.eval_path }} + EVAL_OUTCOME: ${{ steps.verify.outcome }} + HEAD_SHA: ${{ needs.discover.outputs.head_sha }} + PULL_REQUEST: ${{ needs.discover.outputs.pull_request }} + with: + script: | + const fs = require('fs'); + const path = require('path'); + const issueNumber = Number(process.env.PULL_REQUEST); + const marker = ``; + const passed = process.env.EVAL_OUTCOME === 'success'; + let details = '_No complete Vally comparison was produced._'; + let gradeBreakdown = ''; + let reportGenerationFailed = false; + + const readRecords = file => fs.readFileSync(file, 'utf8').split(/\r?\n/).filter(Boolean).map(JSON.parse); + const findFiles = (directory, name, matches = []) => { + for (const entry of fs.readdirSync(directory, { withFileTypes: true })) { + const fullPath = path.join(directory, entry.name); + if (entry.isDirectory()) findFiles(fullPath, name, matches); + else if (entry.name === name) matches.push(fullPath); + } + return matches; + }; + const escapeCell = value => String(value).replaceAll('|', '\\|').replaceAll(/\r?\n/g, ' '); + + try { + const comparisonPath = path.join(process.env.RESULT_ROOT, 'comparison.jsonl'); + if (fs.existsSync(comparisonPath)) { + const comparisons = readRecords(comparisonPath); + if (comparisons.length !== 1) { + throw new Error(`expected one comparison record, found ${comparisons.length}`); + } + const comparison = comparisons[0]; + const summary = comparison?.summary; + const tokens = summary?.metricDeltas?.find(metric => metric.metric === 'totalTokens'); + if (comparison?.type !== 'comparison' || + !Number.isFinite(summary?.meanScore) || + !Number.isFinite(summary?.wins) || + !Number.isFinite(summary?.ties) || + !Number.isFinite(summary?.losses) || + !Number.isFinite(tokens?.mean)) { + throw new Error('comparison summary or token metrics are incomplete'); + } + const tokenDelta = `${tokens.mean >= 0 ? '+' : ''}${Math.round(tokens.mean).toLocaleString()}`; + details = `Quality score (treatment relative to control): **${summary.meanScore.toFixed(2)}** ` + + `(${summary.wins} wins, ${summary.ties} ties, ${summary.losses} losses)\n\n` + + `Mean token delta: **${tokenDelta} tokens** per paired trial.`; + } + } catch (error) { + reportGenerationFailed = true; + core.warning(`Could not parse comparison report: ${error.message}`); + } + + try { + const treatmentFiles = findFiles(process.env.RESULT_ROOT, 'results.jsonl') + .filter(file => file.split(path.sep).includes('treatment')); + if (treatmentFiles.length !== 1) { + throw new Error(`expected one treatment results file, found ${treatmentFiles.length}`); + } + + const byStimulus = new Map(); + for (const trial of readRecords(treatmentFiles[0]).filter(record => record.type === 'trial-result')) { + const stimulus = trial.stimulus ?? trial.gradeResult?.stimulusName ?? 'unknown'; + const summary = byStimulus.get(stimulus) ?? { + trials: 0, + successful: 0, + scores: [], + graders: new Map(), + }; + summary.trials++; + if (trial.status === 'success') summary.successful++; + if (Number.isFinite(trial.gradeResult?.score)) summary.scores.push(trial.gradeResult.score); + for (const grade of trial.gradeResult?.details ?? []) { + const grader = grade.name ?? grade.graderType ?? 'unknown'; + const count = summary.graders.get(grader) ?? { passed: 0, total: 0 }; + count.total++; + if (grade.passed === true) count.passed++; + summary.graders.set(grader, count); + } + byStimulus.set(stimulus, summary); + } + + if (byStimulus.size > 0) { + const rows = [...byStimulus.entries()].map(([stimulus, summary]) => { + const meanScore = summary.scores.length > 0 + ? (summary.scores.reduce((sum, score) => sum + score, 0) / summary.scores.length).toFixed(2) + : 'N/A'; + const graders = [...summary.graders.entries()] + .map(([grader, count]) => `${escapeCell(grader)}: ${count.passed}/${count.total}`) + .join('
'); + return `| ${escapeCell(stimulus)} | ${summary.successful}/${summary.trials} | ${meanScore} | ${graders} |`; + }); + gradeBreakdown = '\n\n#### Treatment grades by stimulus\n\n' + + '| Stimulus | Successful trials | Mean score | Grader passes |\n' + + '| --- | ---: | ---: | --- |\n' + rows.join('\n'); + } + } catch (error) { + reportGenerationFailed = true; + core.warning(`Could not parse treatment grades: ${error.message}`); + } + + let baseline = ''; + try { + let existedAtBase = true; + try { + await github.rest.repos.getContent({ + owner: context.repo.owner, + repo: context.repo.repo, + path: process.env.EVAL_PATH, + ref: process.env.BASE_SHA + }); + } catch (error) { + if (error.status !== 404) throw error; + existedAtBase = false; + } + + if (existedAtBase) { + const query = `repo:${context.repo.owner}/${context.repo.repo} is:pr label:area-harness ` + + `in:comments "agent-harness-eval:${process.env.COMPONENT}"`; + const { data } = await github.rest.search.issuesAndPullRequests({ + q: query, sort: 'updated', order: 'desc', per_page: 100 + }); + let previous; + for (const pull of data.items) { + if (pull.number === issueNumber) continue; + const comments = await github.paginate(github.rest.issues.listComments, { + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: pull.number, + per_page: 100 + }); + for (const comment of comments) { + if (comment.user?.login !== 'github-actions[bot]' || + !comment.body?.startsWith(marker) || + !/Status: \*\*(passed|failed)\*\*/.test(comment.body)) continue; + if (!previous || Date.parse(comment.updated_at) > Date.parse(previous.comment.updated_at)) { + previous = { comment, pullNumber: pull.number }; + } + } + } + if (previous) { + baseline = `\n\n[Previous result from PR #${previous.pullNumber}]` + + `(${previous.comment.html_url}) provides the baseline.`; + } + } + } catch (error) { + reportGenerationFailed = true; + core.warning(`Could not locate a previous baseline: ${error.message}`); + } + + const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + const manualUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}` + + '/actions/workflows/agent-harness-evaluation.yml'; + const artifactLink = process.env.ARTIFACT_URL + ? `[Download Vally reports](${process.env.ARTIFACT_URL}) · ` + : ''; + const status = reportGenerationFailed ? 'error' : passed ? 'passed' : 'failed'; + const report = reportGenerationFailed + ? 'An error occurred while generating the evaluation report.' + : `${details}${gradeBreakdown}${baseline}`; + const body = `${marker}\n### Agent evaluation: \`${process.env.COMPONENT}\`\n\n` + + `Status: **${status}** for \`${process.env.HEAD_SHA}\`\n\n` + + `${report}\n\nPR updates do not rerun evaluations automatically. ` + + `[Run again manually](${manualUrl}) after later commits.\n\n` + + `${artifactLink}[View workflow run](${runUrl})`; + const comments = await github.paginate(github.rest.issues.listComments, { + owner: context.repo.owner, repo: context.repo.repo, issue_number: issueNumber, per_page: 100 + }); + const existing = comments.find(comment => + comment.user?.login === 'github-actions[bot]' && comment.body?.startsWith(marker)); + if (existing) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, repo: context.repo.repo, comment_id: existing.id, body + }); + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, repo: context.repo.repo, issue_number: issueNumber, body + }); + } + + evaluation: + needs: [discover, evaluate] + if: always() + runs-on: ubuntu-24.04 + steps: + - name: Fail when discovery did not complete + if: needs.discover.result != 'success' + run: | + echo "::error::Agent harness component discovery failed." + exit 1 + + - name: No affected components + if: needs.discover.result == 'success' && needs.discover.outputs.has_entries != 'true' + run: echo "No repository-owned harness evaluations are affected." + + - name: Report deferred evaluation + if: >- + needs.discover.outputs.has_entries == 'true' && + needs.discover.outputs.can_use_secrets != 'true' + run: echo "Evaluation requires a contributor-triggered manual run." + + - name: Fail when a component comparison failed + if: >- + needs.discover.outputs.has_entries == 'true' && + needs.discover.outputs.can_use_secrets == 'true' && + needs.evaluate.result != 'success' + run: | + echo "::error::At least one agent harness component comparison failed." + exit 1 + + - name: Report successful evaluation + if: >- + needs.discover.outputs.has_entries == 'true' && + needs.discover.outputs.can_use_secrets == 'true' && + needs.evaluate.result == 'success' + env: + COMPONENTS: ${{ needs.discover.outputs.components }} + run: | + echo "All selected components passed treatment and control comparison: $COMPONENTS." \ No newline at end of file diff --git a/.github/workflows/pr-comment-command.yml b/.github/workflows/pr-comment-command.yml new file mode 100644 index 00000000000..814fc37517f --- /dev/null +++ b/.github/workflows/pr-comment-command.yml @@ -0,0 +1,53 @@ +# Allows trusted contributors to trigger actions from a PR comment. + +name: PR Comment Command + +on: + issue_comment: + types: [created] + +permissions: + actions: write + contents: read + pull-requests: read + +jobs: + dispatch: + if: github.event.issue.pull_request + runs-on: ubuntu-latest + steps: + - name: Dispatch requested action + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 + with: + script: | + const match = context.payload.comment.body.trim().match( + /^\/eval(?:\s+([A-Za-z0-9][A-Za-z0-9._-]*))?$/); + if (!match) return; + + const commenter = context.payload.comment.user.login; + const { data: permission } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: context.repo.owner, + repo: context.repo.repo, + username: commenter + }); + if (!['admin', 'maintain', 'write'].includes(permission.permission)) { + throw new Error(`${commenter} does not have permission to trigger actions.`); + } + + const pullNumber = context.issue.number; + const { data: pull } = await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: pullNumber + }); + await github.rest.actions.createWorkflowDispatch({ + owner: context.repo.owner, + repo: context.repo.repo, + workflow_id: 'agent-harness-evaluation.yml', + ref: context.payload.repository.default_branch, + inputs: { + pull_request: String(pullNumber), + commit_sha: pull.head.sha, + component: match[1] ?? '' + } + }); \ No newline at end of file diff --git a/.vscode/mcp.json b/.vscode/mcp.json deleted file mode 100644 index 9d5b8f44c12..00000000000 --- a/.vscode/mcp.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "servers": { - "microsoft-docs": { - "type": "http", - "url": "https://learn.microsoft.com/api/mcp" - }, - "nuget": { - "type": "stdio", - "command": "dnx", - "args": [ - "NuGet.Mcp.Server", - "--prerelease", - "--yes" - ] - }, - "Community-Mcp-DotNet": { - "type": "stdio", - "command": "dnx", - "args": [ - "Community.Mcp.DotNet", - "--prerelease", - "--yes" - ], - "env": { - "DOTNET_SKIP_FIRST_TIME_EXPERIENCE": "1", - "DOTNET_NOLOGO": "1" - } - }, - "helix-azdo": { - "type": "stdio", - "command": "dnx", - "args": [ - "lewing.helix.mcp", - "--prerelease", - "--yes" - ] - }, - "msbuild-binlog": { - "type": "stdio", - "command": "dnx", - "args": [ - "baronfel.binlog.mcp", - "--prerelease", - "--yes" - ] - } - } -} diff --git a/eng/harness-evaluation/README.md b/eng/harness-evaluation/README.md new file mode 100644 index 00000000000..9d9455eea01 --- /dev/null +++ b/eng/harness-evaluation/README.md @@ -0,0 +1,76 @@ +# Agent harness evaluations + +This directory contains Vally evaluations for repository-owned Copilot instructions, skills, agents, agentic workflows, and prompts. + +## Layout + +- `instructions//eval.yaml` evaluates a repository instruction file. +- `skills//eval.yaml` evaluates `.agents/skills//SKILL.md`. +- `agents//eval.yaml` evaluates `.github/agents/.agent.md` or `.md`. +- `workflows//eval.yaml` evaluates a `.github/workflows/.md` agentic workflow source, not its compiled `.lock.yml`. +- `prompts//eval.yaml` evaluates `.github/prompts/.prompt.md`. + +Component IDs and eval paths are derived from these conventions. Nested instruction, agent, agentic-workflow, or prompt paths use `--` between path segments. + +MCP servers and plugins are explicitly outside behavioral-eval coverage. Prefer shared plugins over repository-owned MCP servers or skills when one owns the capability. This repository enables `dotnet`, `dotnet-test`, `dotnet-diag`, `dotnet-msbuild`, and `dotnet-nuget` from `dotnet/skills`, plus `dotnet-helix`, `dotnet-codeflow`, and `dotnet-dnceng` from `dotnet/arcade-skills`. `dotnet-msbuild` supplies binlog tooling; enabling Arcade's `dotnet-binlog` too would register a second MCP named `binlog`. The Microsoft Docs plugin supplies the Microsoft Learn MCP endpoint and its usage skills. `eng/common/AGENTS.md` is generated Arcade guidance and is also outside the inventory. + +## Authoring rules + +Every stimulus must: + +1. Set a positive `defaults.runs`. Use a parameterized default such as `${RUNS=5}` when local `--runs` overrides should be supported. +2. Declare the evaluated customization in the eval's root `agent_environment`: use `skills` for a skill and `files` for instructions, agents, workflows, or prompts. +3. Use stimulus-level `agent_environment.files` when repository inputs are needed. `src` is relative to the eval file; `dest` is relative to the isolated workspace. +4. Add a `token-budget` grader to every stimulus and define its weight in `scoring.weights`. +5. Use deterministic graders where possible and a narrow `prompt` rubric only for semantic quality. + +The runner passes declared input files from the evaluated commit to Vally, which validates and stages them in an isolated workspace. Inputs are optional. + +Skill evals load only their target skill and require a `skill-invocation` grader. The checked-in Vally experiment clears root `skills` and `files` for the unskilled control while the treatment inherits the eval's root environment. + +## Acceptance + +Pull request and manual evaluation use `vally experiment run` to execute the eval's configured number of unskilled-control and treatment trials, then use `vally compare` to judge paired trajectories. The treatment must meet the eval's committed `scoring.threshold`, comparison judging must complete, and `--fail-on-regression` rejects a statistically significant treatment regression. Reports include the treatment-relative quality verdict and mean token delta. + +The workflow discovers and labels affected PRs when they are opened, reopened, or marked ready for review. Evaluation starts automatically for authorized contributors and same-repository bot branches. A contributor with write access can comment `/eval` to rerun all affected components or `/eval ` to rerun one. Manual dispatch from the repository default branch accepts a PR number, PR commit SHA and optional component filter. + +## Local validation + +From the repository root in PowerShell: + +```powershell +# Restore the repository-managed .NET SDK and build dependencies. +.\restore.cmd + +# Use the repository-managed .NET SDK. +. .\activate.ps1 + +Set-Location eng\harness-evaluation + +# Install the Vally toolchain. +npm ci + +# Exercise component discovery, selection, experiment output, and timeout logic. +npm test + +# Check customization coverage and strict-lint all skills and source eval specifications. +npm run lint + +# Run the configured treatment and control trials, then compare quality and token use. +node src/cli.mjs eval --runs 5 --workers 1 --require-pass +``` + +Set `defaults.timeout` to five times the slowest observed trial, rounded up to the next five-minute boundary. + +Behavioral runs use the Copilot SDK and require a valid local Copilot login or `COPILOT_GITHUB_TOKEN`. Results are written below `artifacts/TestResults/harness-evaluation//` as one timestamped experiment containing separate `control` and `treatment` variants, plus `comparison.jsonl`. They may contain prompts, model output, and tool payloads; do not commit them. + +The runner defaults to one active trial per component. GitHub Actions parallelizes separate component jobs, so increasing Vally workers would multiply concurrent Copilot sessions and can cause session destruction or rate limiting. + +Before accepting a new or materially changed eval, inspect both arms and `comparison.jsonl`. A control that consistently matches or beats the treatment means the eval is not discriminating enough. + +## Adding components + +- **Skill:** add `.agents/skills//SKILL.md` and `eng/harness-evaluation/skills//eval.yaml`. +- **Instruction, agent, agentic workflow, or prompt:** add the customization and a same-ID eval under the corresponding `eng/harness-evaluation/` directory. + +The required coverage workflow rejects missing or orphaned repository-customization evals. MCP and plugin changes do not require eval coverage. diff --git a/eng/harness-evaluation/harness.experiment.yaml b/eng/harness-evaluation/harness.experiment.yaml new file mode 100644 index 00000000000..0e04735d0f2 --- /dev/null +++ b/eng/harness-evaluation/harness.experiment.yaml @@ -0,0 +1,13 @@ +name: efcore-agent-harness +evals: + - '{instructions,skills,agents,workflows,prompts}/**/eval.yaml' +vary: + - /environment/files + - /environment/skills +baseline: control +variants: + control: + environment: + files: [] + skills: [] + treatment: {} \ No newline at end of file diff --git a/eng/harness-evaluation/instructions/copilot-instructions/eval.yaml b/eng/harness-evaluation/instructions/copilot-instructions/eval.yaml new file mode 100644 index 00000000000..fb0a76ea960 --- /dev/null +++ b/eng/harness-evaluation/instructions/copilot-instructions/eval.yaml @@ -0,0 +1,87 @@ +name: copilot-instructions +description: Evaluates repository-wide Copilot guidance for EF Core internal APIs and provider test hierarchies. +type: capability +defaults: + runs: ${RUNS=5} + timeout: 10m + model: claude-sonnet-5 + judge_model: gpt-5.6-terra + executor: copilot-sdk +agent_environment: + files: + - src: ../../../../.github/copilot-instructions.md + dest: .github/copilot-instructions.md +stimuli: + - name: review-an-internal-public-api-change + prompt: |- + Inspect src/EFCore/Metadata/Internal/EntityType.cs. A proposed change adds a new public virtual member in this internal namespace; the member activates a CLR type through reflection and awaits a callback. Write api-review.md listing every repository-wide implementation, documentation, compatibility, and validation requirement that must be satisfied before accepting the change. Ground the review by naming at least two relevant existing members or annotations found in the staged file. Do not edit source. + constraints: + max_turns: 50 + max_tokens: 1000000 + max_duration: 10m + agent_environment: + files: + - src: ../../../../src/EFCore/Metadata/Internal/EntityType.cs + dest: src/EFCore/Metadata/Internal/EntityType.cs + rubric: + - The review requires the standard EF Core internal-API warning XML documentation on the public member. + - It addresses NativeAOT compatibility by avoiding reflection or requiring the appropriate annotations or exception. + - It requires ConfigureAwait(false) on the awaited callback and EFCore.ApiBaseline.Tests for the public API change. + - The advice names at least two relevant members or annotations from the staged EntityType implementation rather than giving generic library guidance. + graders: + - type: file-matches + config: + path: api-review.md + pattern: (?=[\s\S]*internal API)(?=[\s\S]*(NativeAOT|RequiresDynamicCode|DynamicallyAccessedMembers))(?=[\s\S]*ConfigureAwait\(false\))(?=[\s\S]*EFCore\.ApiBaseline\.Tests)[\s\S]+ + - type: prompt + config: + scoring: binary + threshold: 1.0 + evidence: + - trajectory + - repo + output_delivery: workspace + - type: token-budget + config: + max: 1000000 + - name: add-a-query-test-through-provider-hierarchies + prompt: |- + Inspect test/EFCore.Specification.Tests/Query/QueryTestBase.cs, test/EFCore.Specification.Tests/Query/NorthwindWhereQueryTestBase.cs, test/EFCore.SqlServer.FunctionalTests/Query/NorthwindWhereQuerySqlServerTest.cs, and test/EFCore.Sqlite.FunctionalTests/Query/NorthwindWhereQuerySqliteTest.cs. Write test-plan.md for adding a new Where_with_parameterized_limit case through this hierarchy, including result assertions, provider overrides, SQL baselines, and focused commands to validate it. Include `restore.cmd` and dot-sourced `activate.ps1` setup, then use `dotnet exec` on each built provider test DLL with `--filter-method`, `--filter-not-trait category=failing`, and `--ignore-exit-code 8`. Do not put `--no-build` on test commands. Do not edit source. + constraints: + max_turns: 40 + max_tokens: 750000 + max_duration: 5m + agent_environment: + files: + - src: ../../../../test/EFCore.Specification.Tests/Query/QueryTestBase.cs + dest: test/EFCore.Specification.Tests/Query/QueryTestBase.cs + - src: ../../../../test/EFCore.Specification.Tests/Query/NorthwindWhereQueryTestBase.cs + dest: test/EFCore.Specification.Tests/Query/NorthwindWhereQueryTestBase.cs + - src: ../../../../test/EFCore.SqlServer.FunctionalTests/Query/NorthwindWhereQuerySqlServerTest.cs + dest: test/EFCore.SqlServer.FunctionalTests/Query/NorthwindWhereQuerySqlServerTest.cs + - src: ../../../../test/EFCore.Sqlite.FunctionalTests/Query/NorthwindWhereQuerySqliteTest.cs + dest: test/EFCore.Sqlite.FunctionalTests/Query/NorthwindWhereQuerySqliteTest.cs + rubric: + - The plan puts provider-independent behavior in the specification test and identifies every relevant staged provider override. + - Provider overrides call the base test and use AssertSql only where SQL is produced. + - The commands follow this repository's restore, activation, and focused Microsoft Testing Platform conventions without stale --no-build usage. + graders: + - type: file-matches + config: + path: test-plan.md + pattern: (?=[\s\S]*Where_with_parameterized_limit)(?=[\s\S]*restore\.cmd)(?=[\s\S]*activate\.ps1)(?=[\s\S]*base)(?=[\s\S]*AssertSql)(?=[\s\S]*dotnet exec)(?=[\s\S]*--filter-method)(?=[\s\S]*--filter-not-trait category=failing)(?=[\s\S]*--ignore-exit-code 8)[\s\S]+ + - type: prompt + config: + scoring: binary + threshold: 1.0 + evidence: [ trajectory, repo ] + output_delivery: workspace + - type: token-budget + config: + max: 750000 +scoring: + weights: + file-matches: 0.36 + prompt: 0.54 + token-budget: 0.1 + threshold: 0.8 diff --git a/eng/harness-evaluation/package-lock.json b/eng/harness-evaluation/package-lock.json new file mode 100644 index 00000000000..986082430ad --- /dev/null +++ b/eng/harness-evaluation/package-lock.json @@ -0,0 +1,2090 @@ +{ + "name": "efcore-harness-evaluation", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "efcore-harness-evaluation", + "version": "0.1.0", + "devDependencies": { + "@microsoft/vally": "0.16.0", + "@microsoft/vally-cli": "0.16.0", + "yaml": "2.9.0" + }, + "engines": { + "node": ">=22.12.0" + } + }, + "node_modules/@azure-rest/core-client": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/@azure-rest/core-client/-/core-client-2.9.0.tgz", + "integrity": "sha512-6933vNLqh06RR7rumnrq3UZIeBtRLeBHDPMrieDPIljiCaMMwt0nRw++nd3TOxiohtasNtm29rC+b/9ZVKCOqg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2", + "@azure/core-auth": "^1.10.0", + "@azure/core-rest-pipeline": "^1.24.0", + "@azure/core-tracing": "^1.3.0", + "@typespec/ts-http-runtime": "^0.3.8", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/abort-controller": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-2.2.0.tgz", + "integrity": "sha512-fNAjWnA/nZ2jz31kxR/AqRaUT8ewHBw/WuBIosK0moMy1C9e5ValbDfFdIxJzVOOYaYkV/b2F1S4H/aHiqfVQg==", + "dev": true, + "license": "MIT", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/core-auth": { + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/@azure/core-auth/-/core-auth-1.11.0.tgz", + "integrity": "sha512-IUZydyTUkDnYdstOW9pFOOUQlBjAepK5teihDE3x6yxsPJs/hsAaaYpeGxdxrgtOiJbBKSjKW7MDk7AEhb4LRg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2", + "@azure/core-util": "^1.13.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/core-client": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@azure/core-client/-/core-client-1.11.1.tgz", + "integrity": "sha512-2QygG2F76ZpMP2eMztiJvAiFMu71M9rDeU7vO/QKg5Css7MgM4frUOslFjhVjRhbGaCNPtz/S8M6y46/fFKVuQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2", + "@azure/core-auth": "^1.10.0", + "@azure/core-rest-pipeline": "^1.22.0", + "@azure/core-tracing": "^1.3.0", + "@azure/core-util": "^1.13.0", + "@azure/logger": "^1.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/core-process": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@azure/core-process/-/core-process-1.0.0.tgz", + "integrity": "sha512-/shnJ+ooO8WPxDhPEeI/2oRQuubn16gZ6CvlbpWbEswZfzwI9tI/sMAHmF3x1LuQ9yZYXfLW3TjzGMLEC5blKg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/core-rest-pipeline": { + "version": "1.25.0", + "resolved": "https://registry.npmjs.org/@azure/core-rest-pipeline/-/core-rest-pipeline-1.25.0.tgz", + "integrity": "sha512-bMs8ekJLjX8wPV+9IPBges1SLPyuDtE9g5gLDWOpxzKcoOFQnpLGkbcT1tdw3FaAmDS1gnPmMmJ6y/T5B96kIA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2", + "@azure/core-auth": "^1.10.0", + "@azure/core-tracing": "^1.3.0", + "@azure/core-util": "^1.13.0", + "@azure/logger": "^1.3.0", + "@typespec/ts-http-runtime": "^0.3.4", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/core-tracing": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@azure/core-tracing/-/core-tracing-1.4.0.tgz", + "integrity": "sha512-eGwxD0AtncrxeBM4tG8R55Pc3rdX1hNW2WibJAgYpCVA6E93mvvVH+LcssoVjOBrSKWS55yEIHsk0X8ctHmfOQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/core-util": { + "version": "1.14.0", + "resolved": "https://registry.npmjs.org/@azure/core-util/-/core-util-1.14.0.tgz", + "integrity": "sha512-9n2pWK61veAuN0V20t9lOuoV4CFMdyAZ1ygZzvBGk/pBBJRib/PjL9PLXa/aI2CcPpyHfqVsxxqLCYl6uZlfDw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2", + "@typespec/ts-http-runtime": "^0.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/logger": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@azure/logger/-/logger-1.4.0.tgz", + "integrity": "sha512-rbAE25KUfjU/s3XHUdJgceoCP5dEOpMx85J04kF+QMdta73XkuG9JGHHinch+XIoKpBdqljin+KqURpJriSzLA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typespec/ts-http-runtime": "^0.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/monitor-opentelemetry-exporter": { + "version": "1.0.0-beta.45", + "resolved": "https://registry.npmjs.org/@azure/monitor-opentelemetry-exporter/-/monitor-opentelemetry-exporter-1.0.0-beta.45.tgz", + "integrity": "sha512-e23akpV6sm5YGlZEAD2TDidPZ5LdeNH99bGnx/OkrvWCOEYO8Js+HaNqjO6ctq252v1BtnTwJ53bXsc3/SGFBQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@azure-rest/core-client": "^2.5.2", + "@azure/core-auth": "^1.9.0", + "@azure/core-client": "^1.9.2", + "@azure/core-process": "^1.0.0", + "@azure/core-rest-pipeline": "^1.19.0", + "@azure/core-util": "^1.11.0", + "@opentelemetry/api": "^1.9.1", + "@opentelemetry/api-logs": "^0.221.0", + "@opentelemetry/core": "^2.10.0", + "@opentelemetry/resources": "^2.10.0", + "@opentelemetry/sdk-logs": "^0.221.0", + "@opentelemetry/sdk-metrics": "^2.10.0", + "@opentelemetry/sdk-trace-base": "^2.10.0", + "@opentelemetry/semantic-conventions": "^1.43.0", + "tslib": "^2.8.1" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@github/copilot": { + "version": "1.0.80", + "resolved": "https://registry.npmjs.org/@github/copilot/-/copilot-1.0.80.tgz", + "integrity": "sha512-6tf93ZF56KOiTTAjK/UhLZkl1W543IzaTQly288kockJZFswpRTnQEI00Yvacpb39DTvTYu3/ha9SeKpo/pgZQ==", + "dev": true, + "license": "SEE LICENSE IN LICENSE.md", + "dependencies": { + "detect-libc": "^2.1.2" + }, + "bin": { + "copilot": "npm-loader.js" + }, + "optionalDependencies": { + "@github/copilot-darwin-arm64": "1.0.80", + "@github/copilot-darwin-x64": "1.0.80", + "@github/copilot-linux-arm64": "1.0.80", + "@github/copilot-linux-x64": "1.0.80", + "@github/copilot-linuxmusl-arm64": "1.0.80", + "@github/copilot-linuxmusl-x64": "1.0.80", + "@github/copilot-win32-arm64": "1.0.80", + "@github/copilot-win32-x64": "1.0.80" + } + }, + "node_modules/@github/copilot-darwin-arm64": { + "version": "1.0.80", + "resolved": "https://registry.npmjs.org/@github/copilot-darwin-arm64/-/copilot-darwin-arm64-1.0.80.tgz", + "integrity": "sha512-fzn4PnSx3+O/a3ip72KVsjnzORsEygK+0i21bFAnFBYS+0Wi1Pk+o/CmNsJ7aRbf1enSJrcH8UDVkyc9pMGEBg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "SEE LICENSE IN LICENSE.md", + "optional": true, + "os": [ + "darwin" + ], + "bin": { + "copilot-darwin-arm64": "copilot" + } + }, + "node_modules/@github/copilot-darwin-x64": { + "version": "1.0.80", + "resolved": "https://registry.npmjs.org/@github/copilot-darwin-x64/-/copilot-darwin-x64-1.0.80.tgz", + "integrity": "sha512-PKsyGk5DccNzR3bYXcYTGB9N6sHzhzGqEwq/2t1qBwqPbrC98Zo2dOT2G40/QYpJ4XdrGmTmdmfPJQ9PJknlIQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "SEE LICENSE IN LICENSE.md", + "optional": true, + "os": [ + "darwin" + ], + "bin": { + "copilot-darwin-x64": "copilot" + } + }, + "node_modules/@github/copilot-linux-arm64": { + "version": "1.0.80", + "resolved": "https://registry.npmjs.org/@github/copilot-linux-arm64/-/copilot-linux-arm64-1.0.80.tgz", + "integrity": "sha512-8oXwN2luyHEjIoSk8AkATBjXDhRoQtuiUvC93GpfQKFHI+I1eoOVwIsAq5fKP8jNCF2rOrYFIcTjwmRt38kCcQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "SEE LICENSE IN LICENSE.md", + "optional": true, + "os": [ + "linux" + ], + "bin": { + "copilot-linux-arm64": "copilot" + } + }, + "node_modules/@github/copilot-linux-x64": { + "version": "1.0.80", + "resolved": "https://registry.npmjs.org/@github/copilot-linux-x64/-/copilot-linux-x64-1.0.80.tgz", + "integrity": "sha512-qv1ytVNwA3IDK7kcQow+fAikD67t42+AQ8X42bK/7oudNiv4frVZMO0yh1DYIebVRcmEhmPvbVPY/ptVUK3cbA==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "SEE LICENSE IN LICENSE.md", + "optional": true, + "os": [ + "linux" + ], + "bin": { + "copilot-linux-x64": "copilot" + } + }, + "node_modules/@github/copilot-linuxmusl-arm64": { + "version": "1.0.80", + "resolved": "https://registry.npmjs.org/@github/copilot-linuxmusl-arm64/-/copilot-linuxmusl-arm64-1.0.80.tgz", + "integrity": "sha512-Qjyi+OlVnPC4Lkuy7blDMMwMUQI/yELl7gDnqQlaN8TEbhZqZueuf3p0a+kEjXcNsw4XtNYQc0eMJqSIYy/Pjg==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "SEE LICENSE IN LICENSE.md", + "optional": true, + "os": [ + "linux" + ], + "bin": { + "copilot-linuxmusl-arm64": "copilot" + } + }, + "node_modules/@github/copilot-linuxmusl-x64": { + "version": "1.0.80", + "resolved": "https://registry.npmjs.org/@github/copilot-linuxmusl-x64/-/copilot-linuxmusl-x64-1.0.80.tgz", + "integrity": "sha512-rBg8pugf+5FhiZxi2zkOr+rlcOVF6Xg63j1FvryfwPT4DJ2w5Na7O3lpS4sgu8QmsP5H+dAqjlXYLYsvSoVQ0g==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "SEE LICENSE IN LICENSE.md", + "optional": true, + "os": [ + "linux" + ], + "bin": { + "copilot-linuxmusl-x64": "copilot" + } + }, + "node_modules/@github/copilot-sdk": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/@github/copilot-sdk/-/copilot-sdk-1.0.11.tgz", + "integrity": "sha512-ngrnfa9052fLTOMoY0iiQS2B6pFDYJpWNj3syCdjzdje0R5mWoij9b8exJZciLvX7BbJjKz2/lIdwo24av3e3A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@github/copilot": "^1.0.79", + "koffi": "^3.1.0", + "vscode-jsonrpc": "^8.2.1", + "zod": "^4.3.6" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@github/copilot-win32-arm64": { + "version": "1.0.80", + "resolved": "https://registry.npmjs.org/@github/copilot-win32-arm64/-/copilot-win32-arm64-1.0.80.tgz", + "integrity": "sha512-+f7Vkd3vt2DYOxRnS8dStvYu3DY638N/AuLuIjxZp1F9GgwCUZK69wspqIxg2L59PmRRQcH4AGTrRDR60ENIZA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "SEE LICENSE IN LICENSE.md", + "optional": true, + "os": [ + "win32" + ], + "bin": { + "copilot-win32-arm64": "copilot.exe" + } + }, + "node_modules/@github/copilot-win32-x64": { + "version": "1.0.80", + "resolved": "https://registry.npmjs.org/@github/copilot-win32-x64/-/copilot-win32-x64-1.0.80.tgz", + "integrity": "sha512-PO0kPqhRTWQfsqGaj4UN3cj8ttkcJYy4wmXiArtFm+03AIFu8xTvuhQDPn2xEOsUome7m7t2XomKoavcrCcRsw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "SEE LICENSE IN LICENSE.md", + "optional": true, + "os": [ + "win32" + ], + "bin": { + "copilot-win32-x64": "copilot.exe" + } + }, + "node_modules/@hono/node-server": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/@hono/node-server/-/node-server-2.1.1.tgz", + "integrity": "sha512-ELuehkj5VCBdgEw9zs+ivkKwyzzUCSQuE96YmiPvn1ECBoZCczbFXJLeEGMTYjphP6gydh4pHMqEYPVMYUVgQg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=20" + }, + "peerDependencies": { + "hono": "^4" + } + }, + "node_modules/@koromix/koffi-android-arm64": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/@koromix/koffi-android-arm64/-/koffi-android-arm64-3.3.0.tgz", + "integrity": "sha512-lHXE2mq+uG6CVeX1iepn2ATcAWMZPagr+J0ICYge7wTMDYc5pPRQMq0Rr/16775i6QlUqQqpk8c5Gu66+u3gGA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-android-x64": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/@koromix/koffi-android-x64/-/koffi-android-x64-3.3.0.tgz", + "integrity": "sha512-7EeVK/KS9XNbOsUfy0B3uuSjPQZenjJ35y9QSi6R6eK4Fv0oGvREQaBzIJRJwonxp7sJfGObnwnQTnKTVK2DWA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-darwin-arm64": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/@koromix/koffi-darwin-arm64/-/koffi-darwin-arm64-3.3.0.tgz", + "integrity": "sha512-2R9bihYmYFyRP98wRG2m5WQVKMONf7DoPJI8RkGCOUB2afOXJEVXFepiSKxBZGPml8i1USwAxMUwkzbJ5uMG1A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-darwin-x64": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/@koromix/koffi-darwin-x64/-/koffi-darwin-x64-3.3.0.tgz", + "integrity": "sha512-OecYCQfzhUx+XyAtatIHlk0jIz87wKNjtYvwlwgMwgaGQwO/TxFeReNo+s3167Rh1a9c8rUIOjq1jfzAPBxbKA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-freebsd-arm64": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/@koromix/koffi-freebsd-arm64/-/koffi-freebsd-arm64-3.3.0.tgz", + "integrity": "sha512-6ytHzV1DuyxTP5LtFmQ3imBk71j/F/Nb0Hcwhjcn35esnknm5rQbgRl6tNRldrkGnM4GIAJpsagooUBh1xAzYg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-freebsd-ia32": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/@koromix/koffi-freebsd-ia32/-/koffi-freebsd-ia32-3.3.0.tgz", + "integrity": "sha512-hINo5z1Iz2u3Rx3oHfzcmi5A03gYAoYf4nCJJYaY0bb5IuGdIC7noQTyyI9YxfyTJgoqw3wKFWWYtvsAmEo9Hw==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-freebsd-x64": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/@koromix/koffi-freebsd-x64/-/koffi-freebsd-x64-3.3.0.tgz", + "integrity": "sha512-7XQYAZh828nW6+y3IdyUNdo2VbVHL5ZC8Qb+yqHrlDjTjVXSyGYZJHAyJZ5c7eSTTKBhtIMTX238rIsdexxrog==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-linux-arm": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/@koromix/koffi-linux-arm/-/koffi-linux-arm-3.3.0.tgz", + "integrity": "sha512-NbahRKKKFkHoFN/ZpLY+w4F6eYSqV4KMN40PmjZ1xQjG/XjYb6PIYgE41c7/acAPxOYqujrcaI3B4BAgWYm5/A==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-linux-arm64": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/@koromix/koffi-linux-arm64/-/koffi-linux-arm64-3.3.0.tgz", + "integrity": "sha512-Eo/+wGeawGICzJkUlkzi/KerAJ6U1L/Hf7cnJKH7aGgCOY5RRnjUqXs/eL+UDpRcFHzGeUn+Fq+8q2VxYxvAfw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-linux-ia32": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/@koromix/koffi-linux-ia32/-/koffi-linux-ia32-3.3.0.tgz", + "integrity": "sha512-u9tKuj42RFWJU8ZgZqcBhkFCkIZE/SE6oR1ljpukttJUhOVuaf/lmsWxyOxiVu3U8T1gtD658FlLgrbMVIn6aQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-linux-loong64": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/@koromix/koffi-linux-loong64/-/koffi-linux-loong64-3.3.0.tgz", + "integrity": "sha512-2hKlNZ6OXkTUp6fcueZcP0sQ6yhpdfANg9/mWxDKzT1o8qIj4qJD7UcZLCcRIC8JXiP8JQwZBtlPuyXIqofDQg==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-linux-ppc64": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/@koromix/koffi-linux-ppc64/-/koffi-linux-ppc64-3.3.0.tgz", + "integrity": "sha512-bIVAoHgzyRN7xs529BK7rF3BbZUTJgIr0t/wvBvAgaIpreKsdZ/QWxhedKRbTN7ljlDHkPZFmu5XcCjb/vqUag==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-linux-riscv64": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/@koromix/koffi-linux-riscv64/-/koffi-linux-riscv64-3.3.0.tgz", + "integrity": "sha512-2niYAe+TBXbvf7yog/aiHiDUrKQn+RkMSH5x3ZYo65OQyII1CY3Jk2kP3Pe735gq0LzIYJsDuGMW+02ighSadQ==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-linux-x64": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/@koromix/koffi-linux-x64/-/koffi-linux-x64-3.3.0.tgz", + "integrity": "sha512-apU8ZMOrOh1eqMP/Pp0sDZ/WY6A4r6JWph26h63VM9nZ+5KCTYuPgojfoLPNvrfI/bn1PY3+bpXKRhZHI3gxbQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-openbsd-arm64": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/@koromix/koffi-openbsd-arm64/-/koffi-openbsd-arm64-3.3.0.tgz", + "integrity": "sha512-iTyHMCSSs+X579sWJv7XVFMp9gOfROI8IUEwp09YxJpGvmiSoipI0J7kytFuFq6dyuwO0Ymy4JdlmKnCM0qkxg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-openbsd-ia32": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/@koromix/koffi-openbsd-ia32/-/koffi-openbsd-ia32-3.3.0.tgz", + "integrity": "sha512-xCvMV+4VNHjTxgVPEsBzU0k8dlA7aoFO5QjRF9lVU/NzGhUedlYvFgaF6IWsGOoWEaNt9hgP4KK/wxMbm9v8EQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-openbsd-x64": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/@koromix/koffi-openbsd-x64/-/koffi-openbsd-x64-3.3.0.tgz", + "integrity": "sha512-xItIwx1elM8k1pTudEfuV/1U2gQ06+rFfxCVXrZkyWYYGTzmA5To00f8garJFR+E9gbjQJbn8QDFvWMI+i2sMA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-win32-arm64": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/@koromix/koffi-win32-arm64/-/koffi-win32-arm64-3.3.0.tgz", + "integrity": "sha512-fD41LgKBx5i1YlJDuds4tRHAaG5vV55tvagzJwKFDR/5BDeaFvH9QFMUMDoNl2LkmVyeug0dIElHr/oxejh3IA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-win32-ia32": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/@koromix/koffi-win32-ia32/-/koffi-win32-ia32-3.3.0.tgz", + "integrity": "sha512-BfDbDVEuQwPZOvWXawc3JT/7i9cN7QHYotgo014jjXb9pNv46Ykhb0RRglkcHg+SdytW/7pxYn18YwcQP6LLvQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-win32-x64": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/@koromix/koffi-win32-x64/-/koffi-win32-x64-3.3.0.tgz", + "integrity": "sha512-D+ORp73rTmvGVfq01+4uBmgVZqrPqU9N9xCZCfexaCDlVk0WQsaqid1dGsi6eScJuHt4Cp8s1i5/4CsecxFTKQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@microsoft/vally": { + "version": "0.16.0", + "resolved": "https://registry.npmjs.org/@microsoft/vally/-/vally-0.16.0.tgz", + "integrity": "sha512-SxpK/kfWaD+CZvPVRxmR8Y/I2MBwqBcamSvTXpU6sZCK352QdSOYiEPjvXFqx9vyt47GdhB3hOe3Nag7dM5/3w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@github/copilot": "1.0.80", + "@github/copilot-sdk": "1.0.11", + "@opentelemetry/api": "^1.9.1", + "js-tiktoken": "^1.0.21", + "mdast-util-from-markdown": "^2.0.3", + "picomatch": "^4.0.7", + "yaml": "^2.9.0", + "zod": "^4.4.3" + }, + "engines": { + "node": ">=22.12.0" + } + }, + "node_modules/@microsoft/vally-cli": { + "version": "0.16.0", + "resolved": "https://registry.npmjs.org/@microsoft/vally-cli/-/vally-cli-0.16.0.tgz", + "integrity": "sha512-mEX7xd4gNIOmM79GQJOWo1ewss6kc8Lg3AOAP95Cz+jWlQ+zK3MphRuA0skfr4x6Zddm7DfVy9lHOgqyNNBdQg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@azure/monitor-opentelemetry-exporter": "^1.0.0-beta.32", + "@microsoft/vally": "^0.16.0", + "@microsoft/vally-server": "^0.16.0", + "@opentelemetry/api": "^1.9.1", + "@opentelemetry/exporter-trace-otlp-http": "^0.221.0", + "@opentelemetry/resources": "^2.10.0", + "@opentelemetry/sdk-trace-base": "^2.10.0", + "@opentelemetry/sdk-trace-node": "^2.10.0", + "commander": "^15.0.0" + }, + "bin": { + "vally": "dist/index.js" + }, + "engines": { + "node": ">=22.12.0" + }, + "optionalDependencies": { + "@vscode/deviceid": "~0.1.5" + } + }, + "node_modules/@microsoft/vally-server": { + "version": "0.16.0", + "resolved": "https://registry.npmjs.org/@microsoft/vally-server/-/vally-server-0.16.0.tgz", + "integrity": "sha512-G11QONnAe+c+WzcRIjvz1xn9Ly2FctT3CZ3vK2Gcm74Vc4P+Yf9XK5jVzKM1ixB0rR1F2p4kv6ZfpQQjmTjMYw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@hono/node-server": "^2.1.1", + "@microsoft/vally": "^0.16.0", + "better-sqlite3": "^13.0.3", + "hono": "^4.13.4" + }, + "engines": { + "node": ">=22.12.0" + } + }, + "node_modules/@opentelemetry/api": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.1.tgz", + "integrity": "sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/@opentelemetry/api-logs": { + "version": "0.221.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/api-logs/-/api-logs-0.221.0.tgz", + "integrity": "sha512-OlanaW1vv7ufTqQ3/fPLI4arGt5ZoM+P8abOMki6uEYnpRazepSWDwDnnw+la7kE26SHVC18//SMccrDvLKOXQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api": "^1.3.0" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/@opentelemetry/context-async-hooks": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/context-async-hooks/-/context-async-hooks-2.11.0.tgz", + "integrity": "sha512-Tr79DyWI8itsBdg+jH+opjfrwLzX+erk1/ExkIwhWoAVjVrJIn2y5+cGjTC0Vy8fyNIA/y8wuJPZwr1T3xCZeQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/core": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.11.0.tgz", + "integrity": "sha512-7YP44XH0tV6+Mb54x2YGf84i7yi+31MBZlE8JwvozkxyTvXbSp10X7cI7YE49ChJ3shMJoBmCJF3+1QFBJctGA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/exporter-trace-otlp-http": { + "version": "0.221.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-trace-otlp-http/-/exporter-trace-otlp-http-0.221.0.tgz", + "integrity": "sha512-AySXiKoC+meiWm6zdVj5T2LnPDZuatveBby1cMOeQteIWsYXAUxs8Sru13G2pVSPrUXz6vF+og7QVBX6GdC/oQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/otlp-exporter-base": "0.221.0", + "@opentelemetry/otlp-transformer": "0.221.0", + "@opentelemetry/sdk-trace": "2.10.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/otlp-exporter-base": { + "version": "0.221.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/otlp-exporter-base/-/otlp-exporter-base-0.221.0.tgz", + "integrity": "sha512-UFPIq80OH3Ns/oPFHRj14d4DTOxUo+MUFU8hUiCq5jTqFhdeJnfVSANHT+xp92409cA+oxzvlZCe6NM1wvCuBA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.10.0", + "@opentelemetry/otlp-transformer": "0.221.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/otlp-exporter-base/node_modules/@opentelemetry/core": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.10.0.tgz", + "integrity": "sha512-/wNZ8twnEQQA4HoHu22+vcsdru6pWPWxW+7w+FlxT6Id7PE/WIbZmVKkte+PF72e0F2dnImFeHD2syyE1Mw6MQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/otlp-transformer": { + "version": "0.221.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/otlp-transformer/-/otlp-transformer-0.221.0.tgz", + "integrity": "sha512-lg6lkOU08Az23jVcn/0Els9HP+V8PnR4Km6p0KgpTggS0n/WuhnmY64rSh83Of9iR9nD+dpWr6adlcX8KzAwjg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api-logs": "0.221.0", + "@opentelemetry/core": "2.10.0", + "@opentelemetry/resources": "2.10.0", + "@opentelemetry/sdk-logs": "0.221.0", + "@opentelemetry/sdk-metrics": "2.10.0", + "@opentelemetry/sdk-trace": "2.10.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/otlp-transformer/node_modules/@opentelemetry/core": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.10.0.tgz", + "integrity": "sha512-/wNZ8twnEQQA4HoHu22+vcsdru6pWPWxW+7w+FlxT6Id7PE/WIbZmVKkte+PF72e0F2dnImFeHD2syyE1Mw6MQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/otlp-transformer/node_modules/@opentelemetry/resources": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-2.10.0.tgz", + "integrity": "sha512-q6MMm2zhggzsHVNbabYwut+a6nbuQQe3URUoxaojM/8K1IBfwwPzvxIjNi2/lI1TFe+fMHMW9MWhrtDLEXEnkA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.10.0", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/otlp-transformer/node_modules/@opentelemetry/sdk-metrics": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-metrics/-/sdk-metrics-2.10.0.tgz", + "integrity": "sha512-t6r1VSvXNtSDnPXU1FbZeetJb7yyovHmgu0wRSoftxtE0g2rSNhQZQUy69sRUCL+iioJpX8SN/S6wq6ZtvLySQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.10.0", + "@opentelemetry/resources": "2.10.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.9.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/resources": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-2.11.0.tgz", + "integrity": "sha512-Ie7+8q8MDF4FAEQCKVMTx3ReUvxiIAgIiiW3c9JdmP8+HMcDy20puT+AHjexnExgnbvBxjQ9fjkFDWrikJ2jQA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.11.0", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-logs": { + "version": "0.221.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-logs/-/sdk-logs-0.221.0.tgz", + "integrity": "sha512-FaDcazjyMp7TZZZAsqbo4IkovP0UegoCu0EBkiNt+qCqvUf7FPAsfcrZ3+ZEkKgXZ/jHafop+JoGPDk3A0SmLg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api-logs": "0.221.0", + "@opentelemetry/core": "2.10.0", + "@opentelemetry/resources": "2.10.0", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.4.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-logs/node_modules/@opentelemetry/core": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.10.0.tgz", + "integrity": "sha512-/wNZ8twnEQQA4HoHu22+vcsdru6pWPWxW+7w+FlxT6Id7PE/WIbZmVKkte+PF72e0F2dnImFeHD2syyE1Mw6MQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-logs/node_modules/@opentelemetry/resources": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-2.10.0.tgz", + "integrity": "sha512-q6MMm2zhggzsHVNbabYwut+a6nbuQQe3URUoxaojM/8K1IBfwwPzvxIjNi2/lI1TFe+fMHMW9MWhrtDLEXEnkA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.10.0", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-metrics": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-metrics/-/sdk-metrics-2.11.0.tgz", + "integrity": "sha512-7GXXcObyHyDUUSG+L+kJoquty01bzm7ivE7+SSgXXJcHuPzGviptxwARmI2c+bnnxjexGQbJnyNlN8HxBP/Y7A==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.11.0", + "@opentelemetry/resources": "2.11.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.9.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-trace": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace/-/sdk-trace-2.10.0.tgz", + "integrity": "sha512-MfQGq3GRmTh5fM/y+OjaO0vj6+luCB1XO2gfXCalKCfgKw0eHL++sm75DNweC6ohlp+aFvACqeE0fYayqdRaoQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.10.0", + "@opentelemetry/resources": "2.10.0", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-trace-base": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-2.11.0.tgz", + "integrity": "sha512-H19x/TX/LZdqiYOjM7fqtSxwlplC5pgelavqbQdHbhdq0q/AI/TGkM2dfGuuynTXmJPeF2HoZVoPDu+TGoW78A==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.11.0", + "@opentelemetry/resources": "2.11.0", + "@opentelemetry/sdk-trace": "2.11.0", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-trace-base/node_modules/@opentelemetry/sdk-trace": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace/-/sdk-trace-2.11.0.tgz", + "integrity": "sha512-fFnTqGm8/G73GQVnxYi7LXa1ZVYEUvgL6XI1LpvV0bPC7WQ/ZGgKxCSl8FnlZBKto9JHHEFTO6s6CUpvvtwFrA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.11.0", + "@opentelemetry/resources": "2.11.0", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-trace-node": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-node/-/sdk-trace-node-2.11.0.tgz", + "integrity": "sha512-CuvCMJmZxswhNLlM2LfuLOW3h3fZujA4hsG4B+Sz4dX2zvaXO8Ng74cnDHWD64gLszTlhiG3c0iNUjj4g+0/sA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/context-async-hooks": "2.11.0", + "@opentelemetry/core": "2.11.0", + "@opentelemetry/sdk-trace-base": "2.11.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-trace/node_modules/@opentelemetry/core": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.10.0.tgz", + "integrity": "sha512-/wNZ8twnEQQA4HoHu22+vcsdru6pWPWxW+7w+FlxT6Id7PE/WIbZmVKkte+PF72e0F2dnImFeHD2syyE1Mw6MQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-trace/node_modules/@opentelemetry/resources": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-2.10.0.tgz", + "integrity": "sha512-q6MMm2zhggzsHVNbabYwut+a6nbuQQe3URUoxaojM/8K1IBfwwPzvxIjNi2/lI1TFe+fMHMW9MWhrtDLEXEnkA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.10.0", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/semantic-conventions": { + "version": "1.43.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.43.0.tgz", + "integrity": "sha512-eSYWTm620tTk45EKSedaUL8MFYI8hW164hIXsgIHyxu3VobUB3fFCu5t0hQby6OoWRPsG1KkKUG2M5UadiLiVg==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=14" + } + }, + "node_modules/@types/debug": { + "version": "4.1.13", + "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.13.tgz", + "integrity": "sha512-KSVgmQmzMwPlmtljOomayoR89W4FynCAi3E8PPs7vmDVPe84hT+vGPKkJfThkmXs0x0jAaa9U8uW8bbfyS2fWw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/ms": "*" + } + }, + "node_modules/@types/mdast": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/@types/mdast/-/mdast-4.0.4.tgz", + "integrity": "sha512-kGaNbPh1k7AFzgpud/gMdvIm5xuECykRR+JnWKQno9TAXVa6WIVCGTPvYGekIDL4uwCZQSYbUxNBSb1aUo79oA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/unist": "*" + } + }, + "node_modules/@types/ms": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@types/ms/-/ms-2.1.0.tgz", + "integrity": "sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/unist": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/unist/-/unist-3.0.3.tgz", + "integrity": "sha512-ko/gIFJRv177XgZsZcBwnqJN5x/Gien8qNOn0D5bQU/zAzVf9Zt3BlcUiLqhV9y4ARk0GbT3tnUiPNgnTXzc/Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/@typespec/ts-http-runtime": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@typespec/ts-http-runtime/-/ts-http-runtime-0.3.9.tgz", + "integrity": "sha512-edSdeAqkdxBVzA1yL1LrLCml1YjyCVvPMtMqJpbF+6K609tHe8V6sQUzFQSGcYNhcuhOceZtjvN32+mpIth30A==", + "dev": true, + "license": "MIT", + "dependencies": { + "http-proxy-agent": "^7.0.0", + "https-proxy-agent": "^7.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@vscode/deviceid": { + "version": "0.1.5", + "resolved": "https://registry.npmjs.org/@vscode/deviceid/-/deviceid-0.1.5.tgz", + "integrity": "sha512-D0be67wWo7WyyBqHnRkL2bK7lp7CDH/EMN4kMV6INoKc7kxRL3nsTtngt9JZrOcZdnW59gquGRk+6KFIDyD3QA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "dependencies": { + "fs-extra": "^11.2.0", + "uuid": "^14.0.0" + } + }, + "node_modules/agent-base": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", + "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/better-sqlite3": { + "version": "13.0.3", + "resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-13.0.3.tgz", + "integrity": "sha512-RbOBxmLBG8uvFUc15X9+9SFemKcQ0WBuISBVkpuiaUB2qblC8UWlHEjdWVoZ8AdhSwmoEgsiXKfopX0CQxaACQ==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "dependencies": { + "node-addon-api": "^8.0.0" + }, + "engines": { + "node": ">=22" + } + }, + "node_modules/character-entities": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/character-entities/-/character-entities-2.0.2.tgz", + "integrity": "sha512-shx7oQ0Awen/BRIdkjkvz54PnEEI/EjwXDSIZp86/KKdbafHh1Df/RYGBhn4hbe2+uKC9FnT5UCEdyPz3ai9hQ==", + "dev": true, + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/commander": { + "version": "15.0.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-15.0.0.tgz", + "integrity": "sha512-z67u4ZhzCL/Tydu1lJARtEZYWbWaN7oYLHbsuzocr6y4N6WZAagG3RQ4FW61V1/0+jImpj293XfrcYnd1qxtPg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=22.12.0" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/decode-named-character-reference": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/decode-named-character-reference/-/decode-named-character-reference-1.3.0.tgz", + "integrity": "sha512-GtpQYB283KrPp6nRw50q3U9/VfOutZOe103qlN7BPP6Ad27xYnOIWv4lPzo8HCAL+mMZofJ9KEy30fq6MfaK6Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "character-entities": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/dequal": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz", + "integrity": "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=8" + } + }, + "node_modules/devlop": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/devlop/-/devlop-1.1.0.tgz", + "integrity": "sha512-RWmIqhcFf1lRYBvNmr7qTNuyCt/7/ns2jbpp1+PalgE/rDQcBT0fioSMUpJ93irlUhC5hrg4cYqe6U+0ImW0rA==", + "dev": true, + "license": "MIT", + "dependencies": { + "dequal": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/fs-extra": { + "version": "11.4.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.4.0.tgz", + "integrity": "sha512-EQsFzMUJkCKGr1ePqlYADkIUmHW1s3ZXr5Yqy6wbGrfUCphpl2maM/kyOIRA2HpP3AaFQTZXD4ldjek+nccddA==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + }, + "engines": { + "node": ">=14.14" + } + }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "dev": true, + "license": "ISC", + "optional": true + }, + "node_modules/hono": { + "version": "4.13.7", + "resolved": "https://registry.npmjs.org/hono/-/hono-4.13.7.tgz", + "integrity": "sha512-c8/gF9ac8Y78/agExVocyLevgR+JlpNB444Py0FSX8pJoPdYUfUzRcXtYEYGwt6l19qIlVZPN5Mfsw9jFShmQQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=16.9.0" + } + }, + "node_modules/http-proxy-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", + "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", + "dev": true, + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.0", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/https-proxy-agent": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", + "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", + "dev": true, + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/js-tiktoken": { + "version": "1.0.21", + "resolved": "https://registry.npmjs.org/js-tiktoken/-/js-tiktoken-1.0.21.tgz", + "integrity": "sha512-biOj/6M5qdgx5TKjDnFT1ymSpM5tbd3ylwDtrQvFQSu0Z7bBYko2dF+W/aUkXUPuk6IVpRxk/3Q2sHOzGlS36g==", + "dev": true, + "license": "MIT", + "dependencies": { + "base64-js": "^1.5.1" + } + }, + "node_modules/jsonfile": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.2.1.tgz", + "integrity": "sha512-zwOTdL3rFQ/lRdBnntKVOX6k5cKJwEc1HdilT71BWEu7J41gXIB2MRp+vxduPSwZJPWBxEzv4yH1wYLJGUHX4Q==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "universalify": "^2.0.0" + }, + "optionalDependencies": { + "graceful-fs": "^4.1.6" + } + }, + "node_modules/koffi": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/koffi/-/koffi-3.3.0.tgz", + "integrity": "sha512-UeyQtptuPeCP0BqKD+Q8ftTQhIUBHO7bXyK9DVIaW9hJSg0BrICm4K5N8Qtn7zKk20sb5vpnES9KBqVksM2kpQ==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "funding": { + "url": "https://liberapay.com/Koromix" + }, + "optionalDependencies": { + "@koromix/koffi-android-arm64": "3.3.0", + "@koromix/koffi-android-x64": "3.3.0", + "@koromix/koffi-darwin-arm64": "3.3.0", + "@koromix/koffi-darwin-x64": "3.3.0", + "@koromix/koffi-freebsd-arm64": "3.3.0", + "@koromix/koffi-freebsd-ia32": "3.3.0", + "@koromix/koffi-freebsd-x64": "3.3.0", + "@koromix/koffi-linux-arm": "3.3.0", + "@koromix/koffi-linux-arm64": "3.3.0", + "@koromix/koffi-linux-ia32": "3.3.0", + "@koromix/koffi-linux-loong64": "3.3.0", + "@koromix/koffi-linux-ppc64": "3.3.0", + "@koromix/koffi-linux-riscv64": "3.3.0", + "@koromix/koffi-linux-x64": "3.3.0", + "@koromix/koffi-openbsd-arm64": "3.3.0", + "@koromix/koffi-openbsd-ia32": "3.3.0", + "@koromix/koffi-openbsd-x64": "3.3.0", + "@koromix/koffi-win32-arm64": "3.3.0", + "@koromix/koffi-win32-ia32": "3.3.0", + "@koromix/koffi-win32-x64": "3.3.0" + } + }, + "node_modules/mdast-util-from-markdown": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/mdast-util-from-markdown/-/mdast-util-from-markdown-2.0.3.tgz", + "integrity": "sha512-W4mAWTvSlKvf8L6J+VN9yLSqQ9AOAAvHuoDAmPkz4dHf553m5gVj2ejadHJhoJmcmxEnOv6Pa8XJhpxE93kb8Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "@types/unist": "^3.0.0", + "decode-named-character-reference": "^1.0.0", + "devlop": "^1.0.0", + "mdast-util-to-string": "^4.0.0", + "micromark": "^4.0.0", + "micromark-util-decode-numeric-character-reference": "^2.0.0", + "micromark-util-decode-string": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0", + "unist-util-stringify-position": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-to-string": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/mdast-util-to-string/-/mdast-util-to-string-4.0.0.tgz", + "integrity": "sha512-0H44vDimn51F0YwvxSJSm0eCDOJTRlmN0R1yBh4HLj9wiV1Dn0QoXGbvFAWj2hSItVTlCmBF1hqKlIyUBVFLPg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/micromark/-/micromark-4.0.2.tgz", + "integrity": "sha512-zpe98Q6kvavpCr1NPVSCMebCKfD7CA2NqZ+rykeNhONIJBpc1tFKt9hucLGwha3jNTNI8lHpctWJWoimVF4PfA==", + "dev": true, + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "@types/debug": "^4.0.0", + "debug": "^4.0.0", + "decode-named-character-reference": "^1.0.0", + "devlop": "^1.0.0", + "micromark-core-commonmark": "^2.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-chunked": "^2.0.0", + "micromark-util-combine-extensions": "^2.0.0", + "micromark-util-decode-numeric-character-reference": "^2.0.0", + "micromark-util-encode": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0", + "micromark-util-resolve-all": "^2.0.0", + "micromark-util-sanitize-uri": "^2.0.0", + "micromark-util-subtokenize": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-core-commonmark": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/micromark-core-commonmark/-/micromark-core-commonmark-2.0.3.tgz", + "integrity": "sha512-RDBrHEMSxVFLg6xvnXmb1Ayr2WzLAWjeSATAoxwKYJV94TeNavgoIdA0a9ytzDSVzBy2YKFK+emCPOEibLeCrg==", + "dev": true, + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "decode-named-character-reference": "^1.0.0", + "devlop": "^1.0.0", + "micromark-factory-destination": "^2.0.0", + "micromark-factory-label": "^2.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-factory-title": "^2.0.0", + "micromark-factory-whitespace": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-chunked": "^2.0.0", + "micromark-util-classify-character": "^2.0.0", + "micromark-util-html-tag-name": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0", + "micromark-util-resolve-all": "^2.0.0", + "micromark-util-subtokenize": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-destination": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-destination/-/micromark-factory-destination-2.0.1.tgz", + "integrity": "sha512-Xe6rDdJlkmbFRExpTOmRj9N3MaWmbAgdpSrBQvCFqhezUn4AHqJHbaEnfbVYYiexVSs//tqOdY/DxhjdCiJnIA==", + "dev": true, + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-label": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-label/-/micromark-factory-label-2.0.1.tgz", + "integrity": "sha512-VFMekyQExqIW7xIChcXn4ok29YE3rnuyveW3wZQWWqF4Nv9Wk5rgJ99KzPvHjkmPXF93FXIbBp6YdW3t71/7Vg==", + "dev": true, + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-space": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-space/-/micromark-factory-space-2.0.1.tgz", + "integrity": "sha512-zRkxjtBxxLd2Sc0d+fbnEunsTj46SWXgXciZmHq0kDYGnck/ZSGj9/wULTV95uoeYiK5hRXP2mJ98Uo4cq/LQg==", + "dev": true, + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-title": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-title/-/micromark-factory-title-2.0.1.tgz", + "integrity": "sha512-5bZ+3CjhAd9eChYTHsjy6TGxpOFSKgKKJPJxr293jTbfry2KDoWkhBb6TcPVB4NmzaPhMs1Frm9AZH7OD4Cjzw==", + "dev": true, + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-whitespace": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-whitespace/-/micromark-factory-whitespace-2.0.1.tgz", + "integrity": "sha512-Ob0nuZ3PKt/n0hORHyvoD9uZhr+Za8sFoP+OnMcnWK5lngSzALgQYKMr9RJVOWLqQYuyn6ulqGWSXdwf6F80lQ==", + "dev": true, + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-character": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/micromark-util-character/-/micromark-util-character-2.1.1.tgz", + "integrity": "sha512-wv8tdUTJ3thSFFFJKtpYKOYiGP2+v96Hvk4Tu8KpCAsTMs6yi+nVmGh1syvSCsaxz45J6Jbw+9DD6g97+NV67Q==", + "dev": true, + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-chunked": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-chunked/-/micromark-util-chunked-2.0.1.tgz", + "integrity": "sha512-QUNFEOPELfmvv+4xiNg2sRYeS/P84pTW0TCgP5zc9FpXetHY0ab7SxKyAQCNCc1eK0459uoLI1y5oO5Vc1dbhA==", + "dev": true, + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-classify-character": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-classify-character/-/micromark-util-classify-character-2.0.1.tgz", + "integrity": "sha512-K0kHzM6afW/MbeWYWLjoHQv1sgg2Q9EccHEDzSkxiP/EaagNzCm7T/WMKZ3rjMbvIpvBiZgwR3dKMygtA4mG1Q==", + "dev": true, + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-combine-extensions": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-combine-extensions/-/micromark-util-combine-extensions-2.0.1.tgz", + "integrity": "sha512-OnAnH8Ujmy59JcyZw8JSbK9cGpdVY44NKgSM7E9Eh7DiLS2E9RNQf0dONaGDzEG9yjEl5hcqeIsj4hfRkLH/Bg==", + "dev": true, + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-chunked": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-decode-numeric-character-reference": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/micromark-util-decode-numeric-character-reference/-/micromark-util-decode-numeric-character-reference-2.0.2.tgz", + "integrity": "sha512-ccUbYk6CwVdkmCQMyr64dXz42EfHGkPQlBj5p7YVGzq8I7CtjXZJrubAYezf7Rp+bjPseiROqe7G6foFd+lEuw==", + "dev": true, + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-decode-string": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-decode-string/-/micromark-util-decode-string-2.0.1.tgz", + "integrity": "sha512-nDV/77Fj6eH1ynwscYTOsbK7rR//Uj0bZXBwJZRfaLEJ1iGBR6kIfNmlNqaqJf649EP0F3NWNdeJi03elllNUQ==", + "dev": true, + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "decode-named-character-reference": "^1.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-decode-numeric-character-reference": "^2.0.0", + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-encode": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-encode/-/micromark-util-encode-2.0.1.tgz", + "integrity": "sha512-c3cVx2y4KqUnwopcO9b/SCdo2O67LwJJ/UyqGfbigahfegL9myoEFoDYZgkT7f36T0bLrM9hZTAaAyH+PCAXjw==", + "dev": true, + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-util-html-tag-name": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-html-tag-name/-/micromark-util-html-tag-name-2.0.1.tgz", + "integrity": "sha512-2cNEiYDhCWKI+Gs9T0Tiysk136SnR13hhO8yW6BGNyhOC4qYFnwF1nKfD3HFAIXA5c45RrIG1ub11GiXeYd1xA==", + "dev": true, + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-util-normalize-identifier": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-normalize-identifier/-/micromark-util-normalize-identifier-2.0.1.tgz", + "integrity": "sha512-sxPqmo70LyARJs0w2UclACPUUEqltCkJ6PhKdMIDuJ3gSf/Q+/GIe3WKl0Ijb/GyH9lOpUkRAO2wp0GVkLvS9Q==", + "dev": true, + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-resolve-all": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-resolve-all/-/micromark-util-resolve-all-2.0.1.tgz", + "integrity": "sha512-VdQyxFWFT2/FGJgwQnJYbe1jjQoNTS4RjglmSjTUlpUMa95Htx9NHeYW4rGDJzbjvCsl9eLjMQwGeElsqmzcHg==", + "dev": true, + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-sanitize-uri": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-sanitize-uri/-/micromark-util-sanitize-uri-2.0.1.tgz", + "integrity": "sha512-9N9IomZ/YuGGZZmQec1MbgxtlgougxTodVwDzzEouPKo3qFWvymFHWcnDi2vzV1ff6kas9ucW+o3yzJK9YB1AQ==", + "dev": true, + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-encode": "^2.0.0", + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-subtokenize": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-util-subtokenize/-/micromark-util-subtokenize-2.1.0.tgz", + "integrity": "sha512-XQLu552iSctvnEcgXw6+Sx75GflAPNED1qx7eBJ+wydBb2KCbRZe+NwvIEEMM83uml1+2WSXpBAcp9IUCgCYWA==", + "dev": true, + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-util-chunked": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-symbol": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-symbol/-/micromark-util-symbol-2.0.1.tgz", + "integrity": "sha512-vs5t8Apaud9N28kgCrRUdEed4UJ+wWNvicHLPxCa9ENlYuAY31M0ETy5y1vA33YoNPDFTghEbnh6efaE8h4x0Q==", + "dev": true, + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-util-types": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/micromark-util-types/-/micromark-util-types-2.0.2.tgz", + "integrity": "sha512-Yw0ECSpJoViF1qTU4DC6NwtC4aWGt1EkzaQB8KPPyCRR8z9TWeV0HbEFGTO+ZY1wB22zmxnJqhPyTpOVCpeHTA==", + "dev": true, + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/node-addon-api": { + "version": "8.9.2", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.9.2.tgz", + "integrity": "sha512-VijLXbi3UACN69I0JVXJsX4tjACjNoQDgv2gTF6sx2wWEi8tkSg2eX8p5gSIFi8z2+DL3oHmY6OyKce38SDolg==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^18 || ^20 || >= 21" + } + }, + "node_modules/picomatch": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.7.tgz", + "integrity": "sha512-qcJu88Q2IWqJsDD529JKMdwGm/dvInW4HvQnRwiH9JtihJvzGOscDtHE3x1pBKeUOTysQ8kVmLnJ2kJu7yhcGA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "dev": true, + "license": "0BSD" + }, + "node_modules/unist-util-stringify-position": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/unist-util-stringify-position/-/unist-util-stringify-position-4.0.0.tgz", + "integrity": "sha512-0ASV06AAoKCDkS2+xw5RXJywruurpbC4JZSm7nr7MOt1ojAzvyyaO+UxZf18j8FCF6kmzCZKcAgN/yu2gm2XgQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/universalify": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.1.tgz", + "integrity": "sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==", + "dev": true, + "license": "MIT", + "optional": true, + "engines": { + "node": ">= 10.0.0" + } + }, + "node_modules/uuid": { + "version": "14.0.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-14.0.2.tgz", + "integrity": "sha512-xZe/16rV4aa+HGSOCiY2YeLT1OybRLrrkL/Rqaq7p7GMVXjFh+6wN4oMYgjFmnSnhY8t6Xpdl2l9qmnHYuMHwQ==", + "dev": true, + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "license": "MIT", + "optional": true, + "bin": { + "uuid": "dist-node/bin/uuid" + } + }, + "node_modules/vscode-jsonrpc": { + "version": "8.2.1", + "resolved": "https://registry.npmjs.org/vscode-jsonrpc/-/vscode-jsonrpc-8.2.1.tgz", + "integrity": "sha512-kdjOSJ2lLIn7r1rtrMbbNCHjyMPfRnowdKjBQ+mGq6NAW5QY2bEZC/khaC5OR8svbbjvLEaIXkOq45e2X9BIbQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/yaml": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.9.0.tgz", + "integrity": "sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA==", + "dev": true, + "license": "ISC", + "bin": { + "yaml": "bin.mjs" + }, + "engines": { + "node": ">= 14.6" + }, + "funding": { + "url": "https://github.com/sponsors/eemeli" + } + }, + "node_modules/zod": { + "version": "4.6.5", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.6.5.tgz", + "integrity": "sha512-v5l/aFXZQeai4awLbOpSoHecE9UiMrnfx75tEXLjNonXVARxQ5mOeipTjROUchszUNCqnE+hqAMujRsRHsut2Q==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + } + } +} diff --git a/eng/harness-evaluation/package.json b/eng/harness-evaluation/package.json new file mode 100644 index 00000000000..57bc62318b9 --- /dev/null +++ b/eng/harness-evaluation/package.json @@ -0,0 +1,19 @@ +{ + "name": "efcore-harness-evaluation", + "version": "0.1.0", + "private": true, + "type": "module", + "engines": { + "node": ">=22.12.0" + }, + "scripts": { + "test": "node --test test/*.test.mjs", + "lint": "node src/cli.mjs lint", + "eval": "node src/cli.mjs eval" + }, + "devDependencies": { + "@microsoft/vally": "0.16.0", + "@microsoft/vally-cli": "0.16.0", + "yaml": "2.9.0" + } +} diff --git a/eng/harness-evaluation/skills/change-tracking/eval.yaml b/eng/harness-evaluation/skills/change-tracking/eval.yaml new file mode 100644 index 00000000000..5b55702e7ca --- /dev/null +++ b/eng/harness-evaluation/skills/change-tracking/eval.yaml @@ -0,0 +1,59 @@ +name: change-tracking +description: Evaluates change-tracking guidance against EF Core snapshot and entry-state code. +type: capability +defaults: + runs: ${RUNS=5} + timeout: 30m + model: claude-sonnet-5 + judge_model: gpt-5.6-terra + executor: copilot-sdk +agent_environment: + skills: + - ../../../../.agents/skills/change-tracking +stimuli: + - name: diagnose-temporary-value-snapshot-transition + prompt: >- + Before analyzing files, use the available repository skill that applies to this task. Inspect src/EFCore/ChangeTracking/Internal/SnapshotFactoryFactory.cs, src/EFCore/ChangeTracking/Internal/InternalEntityEntry.cs, and test/EFCore.Tests/ChangeTracking/TemporaryValuesTest.cs. A proposed change causes a temporary property to lose its modified state when an Added entry becomes Modified. Write analysis.md identifying the controlling code path, the shared-identity edge case that must remain safe, and the focused regression test you would add. Do not edit source. + constraints: + max_turns: 80 + max_tokens: 2000000 + max_duration: 30m + agent_environment: + files: + - src: ../../../../src/EFCore/ChangeTracking/Internal/SnapshotFactoryFactory.cs + dest: src/EFCore/ChangeTracking/Internal/SnapshotFactoryFactory.cs + - src: ../../../../src/EFCore/ChangeTracking/Internal/InternalEntityEntry.cs + dest: src/EFCore/ChangeTracking/Internal/InternalEntityEntry.cs + - src: ../../../../test/EFCore.Tests/ChangeTracking/TemporaryValuesTest.cs + dest: test/EFCore.Tests/ChangeTracking/TemporaryValuesTest.cs + rubric: + - The analysis distinguishes snapshot comparison from state-transition handling in InternalEntityEntry. + - It explicitly considers SharedIdentityEntry when Added and Deleted entries can represent the same identity. + - The proposed test is located in TemporaryValuesTest.cs and verifies observable property/state behavior. + graders: + - type: skill-invocation + config: + required: + - change-tracking + - type: file-matches + config: + path: analysis.md + pattern: (InternalEntityEntry|SetEntityState)[\s\S]*(TemporaryValuesTest|temporary) + - type: prompt + config: + scoring: binary + threshold: 1 + evidence: + - trajectory + - repo + output_delivery: workspace + - type: token-budget + config: + max: 2000000 +scoring: + weights: + skill-invocation: 0.27 + file-matches: 0.27 + prompt: 0.36 + token-budget: 0.1 + threshold: 0.75 diff --git a/eng/harness-evaluation/skills/cosmos-provider/eval.yaml b/eng/harness-evaluation/skills/cosmos-provider/eval.yaml new file mode 100644 index 00000000000..94b8c87309a --- /dev/null +++ b/eng/harness-evaluation/skills/cosmos-provider/eval.yaml @@ -0,0 +1,59 @@ +name: cosmos-provider +description: Evaluates Cosmos provider guidance against EF Core ReadItem translation. +type: capability +defaults: + runs: ${RUNS=5} + timeout: 25m + model: claude-sonnet-5 + judge_model: gpt-5.6-terra + executor: copilot-sdk +agent_environment: + skills: + - ../../../../.agents/skills/cosmos-provider +stimuli: + - name: explain-readitem-discriminator-selection + prompt: >- + Before analyzing files, use the available repository skill that applies to this task. Inspect src/EFCore.Cosmos/Query/Internal/CosmosReadItemAndPartitionKeysExtractor.cs, test/EFCore.Cosmos.FunctionalTests/Query/ReadItemPartitionKeyQueryTestBase.cs, and test/EFCore.Cosmos.FunctionalTests/Query/ReadItemPartitionKeyQueryTest.cs. Write analysis.md explaining why ReadItem_with_single_explicit_discriminator_mapping emits ReadItem, while ReadItem_with_single_explicit_incorrect_discriminator_mapping and ReadItem_with_single_explicit_parameterized_discriminator_mapping emit general queries. Identify the exact extractor condition responsible, explain why retaining each predicate is required for correctness, and outline the assertions that preserve all three behaviors during a refactor. Do not edit source. Include the exact focused EFCore.Cosmos.FunctionalTests command and state whether these tests require the emulator or a Linux-emulator condition. + constraints: + max_turns: 80 + max_tokens: 2000000 + max_duration: 25m + agent_environment: + files: + - src: ../../../../src/EFCore.Cosmos/Query/Internal/CosmosReadItemAndPartitionKeysExtractor.cs + dest: src/EFCore.Cosmos/Query/Internal/CosmosReadItemAndPartitionKeysExtractor.cs + - src: ../../../../test/EFCore.Cosmos.FunctionalTests/Query/ReadItemPartitionKeyQueryTestBase.cs + dest: test/EFCore.Cosmos.FunctionalTests/Query/ReadItemPartitionKeyQueryTestBase.cs + - src: ../../../../test/EFCore.Cosmos.FunctionalTests/Query/ReadItemPartitionKeyQueryTest.cs + dest: test/EFCore.Cosmos.FunctionalTests/Query/ReadItemPartitionKeyQueryTest.cs + rubric: + - The analysis identifies the SqlConstantExpression discriminator condition in CosmosReadItemAndPartitionKeysExtractor. + - It correctly explains the existing ReadItem result for the matching constant and general-query results for incorrect and parameterized discriminator values. + - It preserves all three base/override assertions and includes the focused Cosmos functional-test command and emulator constraints. + graders: + - type: skill-invocation + config: + required: + - cosmos-provider + - type: file-matches + config: + path: analysis.md + pattern: (?=[\s\S]*CosmosReadItemAndPartitionKeysExtractor)(?=[\s\S]*ReadItem)(?=[\s\S]*(EFCore\.Cosmos\.FunctionalTests|dotnet test))(?=[\s\S]*emulator)[\s\S]+ + - type: prompt + config: + scoring: binary + threshold: 1 + evidence: + - trajectory + - repo + output_delivery: workspace + - type: token-budget + config: + max: 2000000 +scoring: + weights: + skill-invocation: 0.27 + file-matches: 0.27 + prompt: 0.36 + token-budget: 0.1 + threshold: 0.75 diff --git a/eng/harness-evaluation/skills/make-custom-agent/eval.yaml b/eng/harness-evaluation/skills/make-custom-agent/eval.yaml new file mode 100644 index 00000000000..ac8df9a5c48 --- /dev/null +++ b/eng/harness-evaluation/skills/make-custom-agent/eval.yaml @@ -0,0 +1,66 @@ +name: make-custom-agent +description: Evaluates custom-agent authoring for EF Core API review. +type: capability +defaults: + runs: ${RUNS=5} + timeout: 10m + model: claude-sonnet-5 + judge_model: gpt-5.6-terra + executor: copilot-sdk +agent_environment: + skills: + - ../../../../.agents/skills/make-custom-agent +stimuli: + - name: create-an-efcore-api-review-agent + prompt: >- + Before creating files, use the available repository skill that applies to this task. Inspect .github/workflows/api-review-baselines.yml, src/EFCore/EFCore.baseline.json, and .github/copilot-instructions.md. + Create a read-only declarative custom agent named efcore-api-review at .github/agents/efcore-api-review.agent.md that reviews EF Core ApiChief baseline deltas, explains compatibility impact, and points reviewers to evidence from the staged workflow and baseline. + Also create its paired harness evaluation at eng/harness-evaluation/agents/efcore-api-review/eval.yaml following eng/harness-evaluation/README.md. Do not create a skill or duplicate the root instructions. + constraints: + max_turns: 50 + max_tokens: 1000000 + max_duration: 10m + agent_environment: + files: + - src: ../../../../.github/workflows/api-review-baselines.yml + dest: .github/workflows/api-review-baselines.yml + - src: ../../../../src/EFCore/EFCore.baseline.json + dest: src/EFCore/EFCore.baseline.json + - src: ../../../../.github/copilot-instructions.md + dest: .github/copilot-instructions.md + rubric: + - The output is .github/agents/efcore-api-review.agent.md with valid frontmatter whose name is efcore-api-review. + - The output includes eng/harness-evaluation/agents/efcore-api-review/eval.yaml with a matching eval name and the required harness structure. + - Its responsibilities and allowed tools are specifically tied to EF Core ApiChief baseline review. + - It remains read-only, avoids secrets and internal URLs, and does not duplicate root repository instructions. + graders: + - type: skill-invocation + config: + required: + - make-custom-agent + - type: file-matches + config: + path: .github/agents/efcore-api-review.agent.md + pattern: '(?m)^---[\s\S]*^name:\s*["'']?efcore-api-review["'']?\s*$[\s\S]*^description:' + - type: file-matches + config: + path: eng/harness-evaluation/agents/efcore-api-review/eval.yaml + pattern: '(?m)^name:\s*["'']?efcore-api-review["'']?\s*$[\s\S]*runs:[\s\S]*agent_environment:[\s\S]*files:[\s\S]*token-budget[\s\S]*scoring:[\s\S]*threshold:' + - type: prompt + config: + scoring: binary + threshold: 1 + evidence: + - trajectory + - repo + output_delivery: workspace + - type: token-budget + config: + max: 1000000 +scoring: + weights: + skill-invocation: 0.25 + file-matches: 0.4 + prompt: 0.25 + token-budget: 0.1 + threshold: 0.75 diff --git a/eng/harness-evaluation/skills/make-github-actions-workflow/eval.yaml b/eng/harness-evaluation/skills/make-github-actions-workflow/eval.yaml new file mode 100644 index 00000000000..2c8574d5193 --- /dev/null +++ b/eng/harness-evaluation/skills/make-github-actions-workflow/eval.yaml @@ -0,0 +1,60 @@ +name: make-github-actions-workflow +description: Evaluates workflow authoring against EF Core's fork-safe PR automation. +type: capability +defaults: + runs: ${RUNS=5} + timeout: 5m + model: claude-sonnet-5 + judge_model: gpt-5.6-terra + executor: copilot-sdk +agent_environment: + skills: + - ../../../../.agents/skills/make-github-actions-workflow +stimuli: + - name: extend-the-release-target-validation-workflow + prompt: >- + Inspect .github/workflows/validate-pr-target-branch.yml and .github/workflows/label-and-milestone-issues.yml, and inspect .github/CONTRIBUTING.md for contributor context. Modify the staged validate-pr-target-branch.yml so an invalid external PR targeting a release branch also receives one idempotent explanatory comment. Preserve its existing policy, keep the pull_request_target path safe, preserve the existing immutable actions/github-script SHA reference, and do not execute or check out fork code. Use a stable hidden HTML marker and verify the matching comment's bot author before updating the existing bot comment across reruns. + constraints: &short_constraints + max_turns: 40 + max_tokens: 750000 + max_duration: 5m + agent_environment: + files: + - src: ../../../../.github/workflows/validate-pr-target-branch.yml + dest: .github/workflows/validate-pr-target-branch.yml + - src: ../../../../.github/workflows/label-and-milestone-issues.yml + dest: .github/workflows/label-and-milestone-issues.yml + - src: ../../../../.github/CONTRIBUTING.md + dest: .github/CONTRIBUTING.md + rubric: + - The edited workflow is valid YAML and preserves the existing release-target validation behavior. + - Permissions are explicit and minimal, and pull_request_target never executes or checks out untrusted PR content. + - Commenting preserves the staged workflow's immutable actions/github-script SHA reference and is idempotent across reruns without trusting a marker in a contributor-authored comment. + graders: + - type: skill-invocation + config: + required: + - make-github-actions-workflow + - type: file-matches + config: + path: .github/workflows/validate-pr-target-branch.yml + pattern: (?=[\s\S]*actions/github-script@)(?=[\s\S]*/); if (comment.user?.login !== 'github-actions[bot]' || !match || selected.has(match[1])) continue; @@ -397,14 +419,26 @@ jobs: pull-requests: read env: COMPONENT: ${{ matrix.id }} - RESULT_ROOT: artifacts/TestResults/harness-evaluation/${{ matrix.id }} + CANDIDATE_ROOT: candidate + RESULT_ROOT: candidate/artifacts/TestResults/harness-evaluation/${{ matrix.id }} + TRUSTED_ROOT: trusted steps: - - name: Check out pull request + - name: Check out trusted evaluation runner + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + ref: ${{ needs.discover.outputs.trusted_sha }} + path: ${{ env.TRUSTED_ROOT }} + fetch-depth: 0 + persist-credentials: false + + - name: Check out pull request candidate uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 with: ref: ${{ needs.discover.outputs.head_sha }} + path: ${{ env.CANDIDATE_ROOT }} fetch-depth: 0 persist-credentials: false + allow-unsafe-pr-checkout: true - name: Report evaluation started continue-on-error: true @@ -440,6 +474,7 @@ jobs: } - name: Restore repository SDK + working-directory: ${{ env.TRUSTED_ROOT }} shell: bash run: | ./restore.sh @@ -450,12 +485,12 @@ jobs: - name: Set up Node.js uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 with: - node-version: 22 + node-version: 22.12 cache: npm - cache-dependency-path: eng/harness-evaluation/package-lock.json + cache-dependency-path: ${{ env.TRUSTED_ROOT }}/eng/harness-evaluation/package-lock.json - name: Install evaluation tools - working-directory: eng/harness-evaluation + working-directory: ${{ env.TRUSTED_ROOT }}/eng/harness-evaluation run: npm ci - name: Run treatment, unskilled control, and comparison @@ -465,10 +500,11 @@ jobs: GITHUB_TOKEN: ${{ github.token }} run: | timeout --signal=TERM --kill-after=30s "${{ matrix.watchdog_minutes }}m" \ - node eng/harness-evaluation/src/cli.mjs eval "$COMPONENT" \ + node "$TRUSTED_ROOT/eng/harness-evaluation/src/cli.mjs" eval "$COMPONENT" \ + --repo-root "$GITHUB_WORKSPACE/$CANDIDATE_ROOT" \ --workers 1 \ --require-pass \ - --output "$RESULT_ROOT" + --output "$GITHUB_WORKSPACE/$RESULT_ROOT" - name: Upload component results id: upload @@ -501,8 +537,8 @@ jobs: } }; visit(process.env.RESULT_ROOT); - const controlFile = resultFiles.find(file => file.split(path.sep).includes('control')); - const treatmentFile = resultFiles.find(file => file.split(path.sep).includes('treatment')); + const controlFile = resultFiles.find(file => path.basename(path.dirname(file)) === 'control'); + const treatmentFile = resultFiles.find(file => path.basename(path.dirname(file)) === 'treatment'); if (!controlFile || !treatmentFile || resultFiles.length !== 2) { throw new Error(`Expected control and treatment results.jsonl files, found ${resultFiles.length}.`); } @@ -620,7 +656,7 @@ jobs: try { const treatmentFiles = findFiles(process.env.RESULT_ROOT, 'results.jsonl') - .filter(file => file.split(path.sep).includes('treatment')); + .filter(file => path.basename(path.dirname(file)) === 'treatment'); if (treatmentFiles.length !== 1) { throw new Error(`expected one treatment results file, found ${treatmentFiles.length}`); } @@ -682,8 +718,8 @@ jobs: } if (existedAtBase) { - const query = `repo:${context.repo.owner}/${context.repo.repo} is:pr label:area-harness ` + - `in:comments "agent-harness-eval:${process.env.COMPONENT}"`; + const query = `repo:${context.repo.owner}/${context.repo.repo} is:pr is:merged label:area-harness ` + + `in:comments "agent-harness-eval:${process.env.COMPONENT}"`; const { data } = await github.rest.search.issuesAndPullRequests({ q: query, sort: 'updated', order: 'desc', per_page: 100 }); diff --git a/eng/harness-evaluation/README.md b/eng/harness-evaluation/README.md index 9d9455eea01..0764d6d93cc 100644 --- a/eng/harness-evaluation/README.md +++ b/eng/harness-evaluation/README.md @@ -30,9 +30,11 @@ Skill evals load only their target skill and require a `skill-invocation` grader ## Acceptance -Pull request and manual evaluation use `vally experiment run` to execute the eval's configured number of unskilled-control and treatment trials, then use `vally compare` to judge paired trajectories. The treatment must meet the eval's committed `scoring.threshold`, comparison judging must complete, and `--fail-on-regression` rejects a statistically significant treatment regression. Reports include the treatment-relative quality verdict and mean token delta. +Pull request, post-merge harness, and manual evaluation use `vally experiment run` to execute the eval's configured number of unskilled-control and treatment trials, then use `vally compare` to judge paired trajectories. The treatment must meet the eval's committed `scoring.threshold`, comparison judging must complete, and `--fail-on-regression` rejects a statistically significant treatment regression. Reports include the treatment-relative quality verdict and mean token delta. -The workflow discovers and labels affected PRs when they are opened, reopened, or marked ready for review. Evaluation starts automatically for authorized contributors and same-repository bot branches. A contributor with write access can comment `/eval` to rerun all affected components or `/eval ` to rerun one. Manual dispatch from the repository default branch accepts a PR number, PR commit SHA and optional component filter. +GitHub Actions pins the evaluation runner and dependencies to the repository default branch, then evaluates the authorized pull request from a separate candidate checkout. The candidate's customizations, eval specifications, and declared input files are data consumed by the trusted runner; candidate harness scripts are not executed. + +The workflow discovers and labels affected PRs when they are opened, reopened, or marked ready for review. Component and eval changes are evaluated from authorized PRs, while harness infrastructure changes fan out to all components only after merging into the default branch. A contributor with write access can comment `/eval` to rerun all affected components or `/eval ` to rerun one. Manual dispatch from the repository default branch accepts a PR number, PR commit SHA and optional component filter. ## Local validation diff --git a/eng/harness-evaluation/skills/make-custom-agent/eval.yaml b/eng/harness-evaluation/skills/make-custom-agent/eval.yaml index ac8df9a5c48..e1344f0adfb 100644 --- a/eng/harness-evaluation/skills/make-custom-agent/eval.yaml +++ b/eng/harness-evaluation/skills/make-custom-agent/eval.yaml @@ -28,6 +28,8 @@ stimuli: dest: src/EFCore/EFCore.baseline.json - src: ../../../../.github/copilot-instructions.md dest: .github/copilot-instructions.md + - src: ../../../../eng/harness-evaluation/README.md + dest: eng/harness-evaluation/README.md rubric: - The output is .github/agents/efcore-api-review.agent.md with valid frontmatter whose name is efcore-api-review. - The output includes eng/harness-evaluation/agents/efcore-api-review/eval.yaml with a matching eval name and the required harness structure. diff --git a/eng/harness-evaluation/skills/make-instructions/eval.yaml b/eng/harness-evaluation/skills/make-instructions/eval.yaml index c557afd4be5..1b01558874a 100644 --- a/eng/harness-evaluation/skills/make-instructions/eval.yaml +++ b/eng/harness-evaluation/skills/make-instructions/eval.yaml @@ -13,7 +13,7 @@ agent_environment: stimuli: - name: create-query-pipeline-file-instructions prompt: >- - Before creating files, use the available repository skill that applies to this task. Inspect src/EFCore.Relational/Query/SqlExpressions/SelectExpression.cs, src/EFCore.Cosmos/Query/Internal/CosmosQueryableMethodTranslatingExpressionVisitor.cs, and .github/copilot-instructions.md. Create concise file-based instructions under .github/instructions/ for changes to these query areas. Create a same-ID Vally eval for the instructions file under eng/harness-evaluation/instructions/. Scope applyTo to the relevant source trees, capture non-obvious patterns visible in the staged files, and do not repeat repository-wide guidance or create a skill. + Before creating files, use the available repository skill that applies to this task. Inspect src/EFCore.Relational/Query/SqlExpressions/SelectExpression.cs, src/EFCore.Cosmos/Query/Internal/CosmosQueryableMethodTranslatingExpressionVisitor.cs, and .github/copilot-instructions.md. Create concise file-based instructions with the fixed ID `query-implementation` at `.github/instructions/query-implementation.instructions.md`, together with its Vally eval at `eng/harness-evaluation/instructions/query-implementation/eval.yaml`. Scope applyTo to the relevant source trees, capture non-obvious patterns visible in the staged files, and do not repeat repository-wide guidance or create a skill. constraints: max_turns: 50 max_tokens: 1000000 @@ -26,11 +26,13 @@ stimuli: dest: src/EFCore.Cosmos/Query/Internal/CosmosQueryableMethodTranslatingExpressionVisitor.cs - src: ../../../../.github/copilot-instructions.md dest: .github/copilot-instructions.md + - src: ../../../../eng/harness-evaluation/README.md + dest: eng/harness-evaluation/README.md rubric: - - The file is under .github/instructions, has valid frontmatter, and uses targeted query-tree applyTo patterns. + - The `.github/instructions/query-implementation.instructions.md` file has valid frontmatter and uses targeted query-tree applyTo patterns. - Guidance is concretely grounded in the staged relational and Cosmos query implementations. - It remains concise and avoids duplicating root instructions or turning the content into a procedural skill. - - Its matching eval declares the target instruction, bounded resources, token grading, deterministic checks where practical, a focused semantic rubric, and a committed scoring threshold. + - The matching `eng/harness-evaluation/instructions/query-implementation/eval.yaml` is named `query-implementation` and declares the exact target instruction, bounded resources, token grading, deterministic checks where practical, a focused semantic rubric, and a committed scoring threshold. graders: - type: skill-invocation config: @@ -38,12 +40,12 @@ stimuli: - make-instructions - type: file-matches config: - path: .github/instructions/*.instructions.md + path: .github/instructions/query-implementation.instructions.md pattern: '(?m)^---[\s\S]*^applyTo:[\s\S]*SelectExpression' - type: file-matches config: - path: eng/harness-evaluation/instructions/*/eval.yaml - pattern: '(?=[\s\S]*runs:)(?=[\s\S]*agent_environment:)(?=[\s\S]*files:)(?=[\s\S]*constraints:)(?=[\s\S]*token-budget)(?=[\s\S]*scoring:)(?=[\s\S]*threshold:)[\s\S]+' + path: eng/harness-evaluation/instructions/query-implementation/eval.yaml + pattern: '(?m)(?=[\s\S]*^name:\s*query-implementation\s*$)(?=[\s\S]*runs:)(?=[\s\S]*^agent_environment:\s*$\r?\n\s+files:\s*$\r?\n\s+-\s+src:\s+\.\./\.\./\.\./\.\./\.github/instructions/query-implementation\.instructions\.md\s*$\r?\n\s+dest:\s+\.github/instructions/query-implementation\.instructions\.md\s*$)(?=[\s\S]*constraints:)(?=[\s\S]*token-budget)(?=[\s\S]*scoring:)(?=[\s\S]*threshold:)[\s\S]+' - type: prompt config: scoring: binary diff --git a/eng/harness-evaluation/skills/make-skill/eval.yaml b/eng/harness-evaluation/skills/make-skill/eval.yaml index 763a0660c82..a8e4be9aed2 100644 --- a/eng/harness-evaluation/skills/make-skill/eval.yaml +++ b/eng/harness-evaluation/skills/make-skill/eval.yaml @@ -13,7 +13,7 @@ agent_environment: stimuli: - name: create-context-pool-diagnostics-skill prompt: >- - Before creating files, use the available repository skill that applies to this task. Inspect src/EFCore/Internal/DbContextPool.cs and test/EFCore.SqlServer.FunctionalTests/DbContextPoolingTest.cs. Create a background-only agent skill under .agents/skills/ for diagnosing EF Core context-pool reset and lifetime issues, together with its Vally eval under eng/harness-evaluation/skills/. Derive its workflow, pitfalls, and validation steps from the staged implementation and tests rather than generic pooling advice. + Before creating files, use the available repository skill that applies to this task. Inspect src/EFCore/Internal/DbContextPool.cs and test/EFCore.SqlServer.FunctionalTests/DbContextPoolingTest.cs. Create a background-only agent skill with the fixed ID `context-pooling` at `.agents/skills/context-pooling/SKILL.md`, together with its Vally eval at `eng/harness-evaluation/skills/context-pooling/eval.yaml`. Derive its workflow, pitfalls, and validation steps from the staged implementation and tests rather than generic pooling advice. constraints: max_turns: 60 max_tokens: 1500000 @@ -24,11 +24,15 @@ stimuli: dest: src/EFCore/Internal/DbContextPool.cs - src: ../../../../test/EFCore.SqlServer.FunctionalTests/DbContextPoolingTest.cs dest: test/EFCore.SqlServer.FunctionalTests/DbContextPoolingTest.cs + - src: ../../../../.github/copilot-instructions.md + dest: .github/copilot-instructions.md + - src: ../../../../eng/harness-evaluation/README.md + dest: eng/harness-evaluation/README.md rubric: - - The created SKILL.md has valid matching name/frontmatter and is explicitly non-user-invocable. + - The created `.agents/skills/context-pooling/SKILL.md` has `context-pooling` as its matching name/frontmatter and is explicitly non-user-invocable. - Its workflow cites concrete DbContextPool reset, lease, or lifetime behavior visible in the staged source and tests. - It contains observable validation steps, relevant pitfalls, and no duplicated generic repository instructions. - - The matching eval declares the target skill, bounded resources, token grading, an exact skill-invocation grader, deterministic output checks where possible, a focused semantic rubric, and a committed scoring threshold. + - The matching `eng/harness-evaluation/skills/context-pooling/eval.yaml` declares `context-pooling` as its name and target skill, bounded resources, token grading, an exact skill-invocation grader, deterministic output checks where possible, a focused semantic rubric, and a committed scoring threshold. graders: - type: skill-invocation config: @@ -36,12 +40,12 @@ stimuli: - make-skill - type: file-matches config: - path: .agents/skills/*/SKILL.md - pattern: '(?m)^---[\s\S]*^name:[\s\S]*DbContextPool' + path: .agents/skills/context-pooling/SKILL.md + pattern: '(?m)^---(?=[\s\S]*^name:\s*context-pooling\s*$)(?=[\s\S]*^user-invocable:\s*false\s*$)[\s\S]*DbContextPool' - type: file-matches config: - path: eng/harness-evaluation/skills/*/eval.yaml - pattern: '(?=[\s\S]*runs:)(?=[\s\S]*agent_environment:)(?=[\s\S]*skills:)(?=[\s\S]*skill-invocation)(?=[\s\S]*token-budget)(?=[\s\S]*scoring:)(?=[\s\S]*threshold:)[\s\S]+' + path: eng/harness-evaluation/skills/context-pooling/eval.yaml + pattern: '(?m)(?=[\s\S]*^name:\s*context-pooling\s*$)(?=[\s\S]*runs:)(?=[\s\S]*^agent_environment:\s*$\r?\n\s+skills:\s*$\r?\n\s+-\s+\.\./\.\./\.\./\.\./\.agents/skills/context-pooling\s*$)(?=[\s\S]*skill-invocation[\s\S]*required:\s*\[\s*context-pooling\s*\])(?=[\s\S]*token-budget)(?=[\s\S]*scoring:)(?=[\s\S]*threshold:)[\s\S]+' - type: prompt config: scoring: binary diff --git a/eng/harness-evaluation/skills/sqlite-adonet/eval.yaml b/eng/harness-evaluation/skills/sqlite-adonet/eval.yaml index d4a09149ef1..558615a7164 100644 --- a/eng/harness-evaluation/skills/sqlite-adonet/eval.yaml +++ b/eng/harness-evaluation/skills/sqlite-adonet/eval.yaml @@ -13,7 +13,7 @@ agent_environment: stimuli: - name: diagnose-reused-command-state prompt: |- - Before analyzing files, use the available repository skill that applies to this task. Inspect src/Microsoft.Data.Sqlite.Core/SqliteConnection.cs, src/Microsoft.Data.Sqlite.Core/SqliteCommand.cs, test/Microsoft.Data.Sqlite.Tests/SqliteConnectionTest.cs, and test/Microsoft.Data.Sqlite.Tests/SqliteCommandTest.cs. Write analysis.md for a bug where state from a reused command survives after its connection is closed and reopened. Identify the likely ADO.NET ownership boundary and the focused tests to add. Keep the fix independent of EF Core provider abstractions, preserve SQLitePCL initialization behavior, and identify the T4 owner before changing any generated API. Explicitly distinguish the `DbConnection.Close()` contract from terminal `DbCommand.Dispose()` and preserve command reuse after reopening the same connection. Do not edit source. + Before analyzing files, use the available repository skill that applies to this task. Inspect src/Microsoft.Data.Sqlite.Core/SqliteConnection.cs, src/Microsoft.Data.Sqlite.Core/SqliteCommand.cs, test/Microsoft.Data.Sqlite.Tests/SqliteConnectionTest.cs, and test/Microsoft.Data.Sqlite.Tests/SqliteCommandTest.cs. Write analysis.md assessing a report that state from a reused command survives after its connection is closed and reopened. Identify the ADO.NET ownership boundary and whether the reported lifecycle is possible. Keep the analysis independent of EF Core provider abstractions, preserve SQLitePCL initialization behavior, and identify the T4 owner before changing any generated API. Explicitly explain what `SqliteConnection.Close()` does to tracked commands and whether a disposed command can be reused after reopening the same connection. Do not edit source. constraints: &short_constraints max_turns: 40 max_tokens: 750000 @@ -31,7 +31,7 @@ stimuli: rubric: - The analysis is grounded in the staged SqliteConnection/SqliteCommand lifecycle and System.Data.Common contracts. - It keeps the fix independent of EF Core provider abstractions and preserves SQLitePCL initialization behavior. - - It proposes focused tests in the staged Microsoft.Data.Sqlite test classes and notes T4 ownership if generated APIs are touched. + - It proposes focused tests only if a reachable defect remains and notes T4 ownership if generated APIs are touched. graders: - type: skill-invocation config: diff --git a/eng/harness-evaluation/src/cli.mjs b/eng/harness-evaluation/src/cli.mjs index 12ea273f06d..8a627052f7d 100644 --- a/eng/harness-evaluation/src/cli.mjs +++ b/eng/harness-evaluation/src/cli.mjs @@ -22,7 +22,7 @@ function valueAfter(args, name, fallback) { function printUsage() { console.error(`Usage: node src/cli.mjs lint - node src/cli.mjs eval [--runs ] [--workers ] [--require-pass] [--output ]`); + node src/cli.mjs eval [--repo-root ] [--runs ] [--workers ] [--require-pass] [--output ]`); } async function lint() { @@ -52,18 +52,23 @@ async function evaluate(args) { } validateComponentId(componentId); - const component = await resolveComponent(componentId); + const repoRoot = resolve(valueAfter(args, '--repo-root', defaultRepoRoot)); + const component = await resolveComponent(componentId, repoRoot); const runsValue = valueAfter(args, '--runs'); const runs = runsValue === undefined ? undefined : Number(runsValue); if (runs !== undefined && (!Number.isSafeInteger(runs) || runs <= 0)) { throw new Error(`--runs must be a positive integer: ${runsValue}`); } - const workers = Number(valueAfter(args, '--workers', '1')); + const workersValue = valueAfter(args, '--workers', '1'); + const workers = Number(workersValue); + if (!Number.isSafeInteger(workers) || workers <= 0) { + throw new Error(`--workers must be a positive integer: ${workersValue}`); + } const requirePass = args.includes('--require-pass'); - const outputRoot = resolve(valueAfter(args, '--output', join(defaultRepoRoot, 'artifacts', 'TestResults', 'harness-evaluation', componentId))); - validateOutputRoot(outputRoot); - const evalPath = join(defaultRepoRoot, component.eval); - const experimentPath = join(defaultRepoRoot, 'eng', 'harness-evaluation', 'harness.experiment.yaml'); + const outputRoot = resolve(valueAfter(args, '--output', join(repoRoot, 'artifacts', 'TestResults', 'harness-evaluation', componentId))); + validateOutputRoot(outputRoot, repoRoot); + const evalPath = join(repoRoot, component.eval); + const experimentPath = join(repoRoot, 'eng', 'harness-evaluation', 'harness.experiment.yaml'); await rm(outputRoot, { recursive: true, force: true }); const experimentArguments = [ 'experiment', 'run', experimentPath, @@ -75,7 +80,7 @@ async function evaluate(args) { if (runs !== undefined) { experimentArguments.push('--param', `RUNS=${runs}`); } - const experimentResult = runVally(experimentArguments, { inherit: true }); + const experimentResult = runVally(experimentArguments, { cwd: repoRoot, inherit: true }); if (experimentResult.status !== 0) { process.exitCode = 1; return; @@ -83,8 +88,9 @@ async function evaluate(args) { const experimentDirectory = await findExperimentRunDirectory(outputRoot); const treatmentResults = join(experimentDirectory, 'treatment', 'results.jsonl'); + const experimentPlan = join(experimentDirectory, 'plan-snapshot.json'); const treatmentSpec = parse(await readFile(evalPath, 'utf8')); - const treatmentPass = !requirePass || await variantPassed(treatmentResults, evalPath); + const treatmentPass = !requirePass || await variantPassed(treatmentResults, evalPath, experimentPlan); if (!treatmentPass) { console.error(`Treatment '${componentId}' did not meet its committed scoring threshold.`); } @@ -98,7 +104,7 @@ async function evaluate(args) { if (treatmentSpec.defaults?.judge_model) { comparisonArguments.push('--judge-model', treatmentSpec.defaults.judge_model); } - const comparisonResult = runVally(comparisonArguments, { inherit: true }); + const comparisonResult = runVally(comparisonArguments, { cwd: repoRoot, inherit: true }); if (!treatmentPass || comparisonResult.status !== 0) { process.exitCode = 1; } diff --git a/eng/harness-evaluation/src/harness.mjs b/eng/harness-evaluation/src/harness.mjs index eba36e0793b..c3651a541bf 100644 --- a/eng/harness-evaluation/src/harness.mjs +++ b/eng/harness-evaluation/src/harness.mjs @@ -1,5 +1,5 @@ import { spawnSync } from 'node:child_process'; -import { existsSync, realpathSync } from 'node:fs'; +import { existsSync, lstatSync, realpathSync } from 'node:fs'; import { readFile, readdir, @@ -40,6 +40,21 @@ export function validateOutputRoot(outputRoot, repoRoot = defaultRepoRoot) { throw new Error(`Output path must be a descendant of '${artifactsRoot}': ${outputRoot}`); } + let currentPath = resolve(repoRoot); + for (const segment of relative(currentPath, resolvedOutput).split(sep)) { + currentPath = join(currentPath, segment); + try { + if (lstatSync(currentPath).isSymbolicLink()) { + throw new Error(`Output path must not contain symbolic links or junctions: ${currentPath}`); + } + } catch (error) { + if (error?.code === 'ENOENT') { + break; + } + throw error; + } + } + if (existsSync(artifactsRoot)) { const canonicalRepoRoot = realpathSync(repoRoot); const canonicalArtifactsRoot = realpathSync(artifactsRoot); @@ -154,9 +169,12 @@ export async function discoverComponents(repoRoot = defaultRepoRoot) { return discoverNonMcpComponents(repoRoot); } -export async function validateEval(evalPath) { +export async function validateEval(evalPath, componentId) { const spec = parse(await readFile(evalPath, 'utf8')); const errors = []; + if (componentId !== undefined && spec?.name !== componentId) { + errors.push(`eval name '${spec?.name}' must match component id '${componentId}'`); + } if (spec?.scoring?.weights?.['token-budget'] === undefined) { errors.push('scoring.weights.token-budget must be defined'); } @@ -268,7 +286,7 @@ export async function validateInventory(repoRoot = defaultRepoRoot) { errors.push(`${component.id}: eval does not exist: ${evalPath}`); } else { const absoluteEvalPath = join(repoRoot, evalPath); - errors.push(...(await validateEval(absoluteEvalPath)).map((error) => `${component.id}: ${error}`)); + errors.push(...(await validateEval(absoluteEvalPath, component.id)).map((error) => `${component.id}: ${error}`)); errors.push(...(await validateActivationGraders(component, absoluteEvalPath)).map((error) => `${component.id}: ${error}`)); errors.push(...(await validateComponentEnvironment(component, absoluteEvalPath, repoRoot)).map( (error) => `${component.id}: ${error}`, @@ -314,22 +332,49 @@ export async function findExperimentRunDirectory(directory) { return dirname(matches[0]); } -export async function variantPassed(resultsFile, evalFile) { +export async function variantPassed(resultsFile, evalFile, planFile, variant = 'treatment') { const content = await readFile(resultsFile, 'utf8'); const records = content.split(/\r?\n/).filter(Boolean).map(JSON.parse); + const trials = records.filter((record) => record.type === 'trial-result'); + const plan = JSON.parse(await readFile(planFile, 'utf8')); + if (plan.type !== 'experiment-plan-snapshot' || !Array.isArray(plan.evals)) { + return false; + } + + const variantPlans = plan.evals.filter((evalPlan) => evalPlan.variant === variant && !evalPlan.failure); + if (variantPlans.length === 0 || variantPlans.some((evalPlan) => + !Number.isSafeInteger(evalPlan.runs) || evalPlan.runs <= 0 + || !Array.isArray(evalPlan.plannedStimulusNames))) { + return false; + } + + const trialKeys = trials + .map((trial) => `${trial.evalName}\0${trial.model ?? ''}\0${trial.stimulus}\0${trial.trialIndex ?? 0}`) + .sort(); + const plannedTrialKeys = variantPlans + .flatMap((evalPlan) => evalPlan.plannedStimulusNames.flatMap((stimulus) => + Array.from( + { length: evalPlan.runs }, + (_, trialIndex) => `${evalPlan.evalName}\0${evalPlan.model ?? ''}\0${stimulus}\0${trialIndex}`, + ))) + .sort(); + if (trials.length === 0 || JSON.stringify(trialKeys) !== JSON.stringify(plannedTrialKeys)) { + return false; + } + const spec = parse(await readFile(evalFile, 'utf8')); const threshold = spec?.scoring?.threshold; const stimulusScores = []; for (const stimulus of spec.stimuli ?? []) { - const trials = records.filter((record) => record.type === 'trial-result' && record.stimulus === stimulus.name); - if (trials.length === 0 || trials.some((trial) => trial.status !== 'success' || !trial.gradeResult)) { + const stimulusTrials = trials.filter((trial) => trial.stimulus === stimulus.name); + if (stimulusTrials.some((trial) => trial.status !== 'success' || !trial.gradeResult)) { return false; } stimulusScores.push(computeStimulusScore( stimulus.name, - trials.map((trial) => ({ + stimulusTrials.map((trial) => ({ grade: trial.gradeResult, passed: resolveGradePass(trial.gradeResult, threshold), })), diff --git a/eng/harness-evaluation/test/harness.test.mjs b/eng/harness-evaluation/test/harness.test.mjs index 414d08ea4a4..c2f3b3bdb25 100644 --- a/eng/harness-evaluation/test/harness.test.mjs +++ b/eng/harness-evaluation/test/harness.test.mjs @@ -36,7 +36,7 @@ async function makeRepo() { await writeFile(join(root, 'eng/harness-evaluation/skills/example/eval.yaml'), evalYaml('example', ` - type: skill-invocation\n config:\n required: [example]`)); await writeFile( join(root, 'eng/harness-evaluation/instructions/copilot-instructions/eval.yaml'), - evalYaml('instructions', undefined, `agent_environment:\n files:\n - src: ../../../../.github/copilot-instructions.md\n dest: .github/copilot-instructions.md\n`), + evalYaml('copilot-instructions', undefined, `agent_environment:\n files:\n - src: ../../../../.github/copilot-instructions.md\n dest: .github/copilot-instructions.md\n`), ); return root; @@ -74,6 +74,26 @@ test('eval rejects unsafe component ids before deleting output', async () => { } }); +test('eval scopes component resolution and output validation to the selected repository', async () => { + const root = await makeRepo(); + try { + const cliPath = fileURLToPath(new URL('../src/cli.mjs', import.meta.url)); + const result = spawnSync(process.execPath, [ + cliPath, + 'eval', + 'example', + '--repo-root', root, + '--output', join(root, 'outside-artifacts'), + ], { encoding: 'utf8' }); + + assert.notEqual(result.status, 0); + assert.match(result.stderr, /Output path must be a descendant/); + assert.match(result.stderr, new RegExp(join(root, 'artifacts').replaceAll('\\', '\\\\'))); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + test('eval output is constrained to an artifacts descendant', async () => { const repoRoot = await mkdtemp(join(tmpdir(), 'efcore-agent-output-')); const outside = await mkdtemp(join(tmpdir(), 'efcore-agent-output-outside-')); @@ -98,7 +118,27 @@ test('eval output is constrained to an artifacts descendant', async () => { await symlink(outside, join(repoRoot, 'artifacts', 'escape'), 'junction'); assert.throws( () => validateOutputRoot(join(repoRoot, 'artifacts', 'escape', 'unrelated'), repoRoot), - /Output path must not escape repository artifacts/, + /Output path must not contain symbolic links or junctions/, + ); + + await rm(join(repoRoot, 'artifacts', 'escape')); + const missingTarget = join(outside, 'missing'); + await symlink( + missingTarget, + join(repoRoot, 'artifacts', 'dangling'), + process.platform === 'win32' ? 'junction' : 'dir', + ); + assert.throws( + () => validateOutputRoot(join(repoRoot, 'artifacts', 'dangling', 'unrelated'), repoRoot), + /Output path must not contain symbolic links or junctions/, + ); + + await rm(join(repoRoot, 'artifacts', 'dangling')); + await rm(join(repoRoot, 'artifacts'), { recursive: true }); + await symlink(missingTarget, join(repoRoot, 'artifacts'), process.platform === 'win32' ? 'junction' : 'dir'); + assert.throws( + () => validateOutputRoot(join(repoRoot, 'artifacts', 'harness-evaluation'), repoRoot), + /Output path must not contain symbolic links or junctions/, ); } finally { await rm(repoRoot, { recursive: true, force: true }); @@ -145,6 +185,18 @@ test('inventory fails closed for malformed eval YAML', async () => { } }); +test('inventory requires eval names to match component ids', async () => { + const root = await makeRepo(); + try { + const evalPath = join(root, 'eng/harness-evaluation/skills/example/eval.yaml'); + await writeFile(evalPath, evalYaml('other', ` - type: skill-invocation\n config:\n required: [example]`)); + const result = await validateInventory(root); + assert.match(result.errors.join('\n'), /example: eval name 'other' must match component id 'example'/); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + test('inventory requires exact target activation graders', async () => { const root = await makeRepo(); try { @@ -266,7 +318,7 @@ test('eval validation delegates missing input files to Vally', async () => { try { const evalPath = join(root, 'eng/harness-evaluation/skills/example/eval.yaml'); await writeFile(evalPath, `name: bad\ndefaults:\n runs: \${RUNS=2}\nstimuli:\n - name: missing-input\n prompt: Do a generic task.\n constraints:\n max_turns: 10\n max_tokens: 5000\n max_duration: 1m\n agent_environment:\n files:\n - src: src/EFCore/Missing.cs\n dest: src/EFCore/Missing.cs\n graders:\n - type: token-budget\n config:\n max: 5000\nscoring:\n weights:\n token-budget: 0.1\n threshold: 0.75\n`); - const errors = await validateEval(evalPath, root); + const errors = await validateEval(evalPath); assert.deepEqual(errors, []); } finally { await rm(root, { recursive: true, force: true }); @@ -313,22 +365,73 @@ test('treatment verdict uses the eval scoring threshold', async () => { try { const evalPath = join(root, 'eval.yaml'); const resultsPath = join(root, 'results.jsonl'); + const planPath = join(root, 'plan-snapshot.json'); await writeFile(evalPath, `name: example\nstimuli:\n - name: first\n prompt: Test.\n graders:\n - type: output-contains\n config: { substring: Test }\nscoring:\n weights:\n output-contains: 1\n threshold: 0.75\n`); + await writeFile(planPath, JSON.stringify({ + type: 'experiment-plan-snapshot', + evals: [{ + variant: 'treatment', + evalName: 'example', + model: 'mock', + runs: 1, + plannedStimulusNames: ['first'], + }], + })); await writeFile(resultsPath, `${JSON.stringify({ type: 'trial-result', + evalName: 'example', + model: 'mock', stimulus: 'first', status: 'success', gradeResult: { passed: true, score: 0.8 }, })}\n`); - assert.equal(await variantPassed(resultsPath, evalPath), true); + assert.equal(await variantPassed(resultsPath, evalPath, planPath), true); await writeFile(resultsPath, `${JSON.stringify({ type: 'trial-result', + evalName: 'example', + model: 'mock', stimulus: 'first', status: 'success', gradeResult: { passed: true, score: 0.7 }, })}\n`); - assert.equal(await variantPassed(resultsPath, evalPath), false); + assert.equal(await variantPassed(resultsPath, evalPath, planPath), false); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + +test('treatment verdict requires every planned trial index', async () => { + const root = await mkdtemp(join(tmpdir(), 'efcore-agent-results-')); + try { + const evalPath = join(root, 'eval.yaml'); + const resultsPath = join(root, 'results.jsonl'); + const planPath = join(root, 'plan-snapshot.json'); + await writeFile(evalPath, `name: example\nstimuli:\n - name: first\n prompt: Test.\n graders:\n - type: output-contains\n config: { substring: Test }\nscoring:\n weights:\n output-contains: 1\n threshold: 0.75\n`); + await writeFile(planPath, JSON.stringify({ + type: 'experiment-plan-snapshot', + evals: [{ + variant: 'treatment', + evalName: 'example', + model: 'mock', + runs: 2, + plannedStimulusNames: ['first'], + }], + })); + const passingTrial = { + type: 'trial-result', + evalName: 'example', + model: 'mock', + stimulus: 'first', + trialIndex: 0, + status: 'success', + gradeResult: { passed: true, score: 1 }, + }; + await writeFile(resultsPath, `${JSON.stringify(passingTrial)}\n`); + assert.equal(await variantPassed(resultsPath, evalPath, planPath), false); + + await writeFile(resultsPath, `${JSON.stringify(passingTrial)}\n${JSON.stringify(passingTrial)}\n`); + assert.equal(await variantPassed(resultsPath, evalPath, planPath), false); } finally { await rm(root, { recursive: true, force: true }); } From 7f52ab510a7f6d49249a9875bc9b4a3f3a537384 Mon Sep 17 00:00:00 2001 From: Andriy Svyryd Date: Fri, 18 Sep 2026 15:49:58 -0700 Subject: [PATCH 3/3] Modify condition for reporting results and update eval filter Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .github/workflows/agent-harness-evaluation.yml | 2 +- eng/harness-evaluation/src/cli.mjs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/agent-harness-evaluation.yml b/.github/workflows/agent-harness-evaluation.yml index 421a54e4e28..113c52ccde7 100644 --- a/.github/workflows/agent-harness-evaluation.yml +++ b/.github/workflows/agent-harness-evaluation.yml @@ -593,7 +593,7 @@ jobs: EOF - name: Report component result - if: always() + if: ${{ !cancelled() }} continue-on-error: true uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 env: diff --git a/eng/harness-evaluation/src/cli.mjs b/eng/harness-evaluation/src/cli.mjs index 8a627052f7d..92b64693528 100644 --- a/eng/harness-evaluation/src/cli.mjs +++ b/eng/harness-evaluation/src/cli.mjs @@ -72,7 +72,7 @@ async function evaluate(args) { await rm(outputRoot, { recursive: true, force: true }); const experimentArguments = [ 'experiment', 'run', experimentPath, - '--eval-filter', evalPath, + '--eval-filter', component.eval.slice('eng/harness-evaluation/'.length), '--output-dir', outputRoot, '--workers', String(workers), '--verbose',