From 79dba03b0b0d735891503311c36bbde4c943bf70 Mon Sep 17 00:00:00 2001
From: itelo <itelofilho@gmail.com>
Date: Fri, 21 Aug 2026 13:37:44 -0300
Subject: [PATCH] feat(harness): give pi adaptive thinking for a fair A/B
 (thinking parity)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

#43 disabled pi's thinking (reasoning:false) to dodge the 400 from pi-ai's
deprecated `thinking:{type:'enabled',budget_tokens}`. That unblocked pi but left
it running without extended thinking while the anthropic control uses adaptive —
an unfair A/B on that axis.

pi-ai supports the modern format via a model `compat` flag: set
`compat:{forceAdaptiveThinking:true}` so it sends `thinking:{type:'adaptive'}` +
output_config.effort (what Opus 4.8 / Sonnet 5 require), turn reasoning back on,
and pass thinkingLevel:'medium' to match the control's medium effort. Also set
supportsTemperature:false (Opus 4.7+ rejects a non-default temperature) — both
mirror how pi-ai's own built-in anthropic-messages models are configured.

Needs a real `SEAM_WIZARD_HARNESS=pi npm run eval` to confirm (adaptive isn't
exercised in CI).
---
 src/lib/steps/harness/pi.ts | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/src/lib/steps/harness/pi.ts b/src/lib/steps/harness/pi.ts
index 9c720c5..bfadaa4 100644
--- a/src/lib/steps/harness/pi.ts
+++ b/src/lib/steps/harness/pi.ts
@@ -59,14 +59,15 @@ export const piHarness: Harness = {
           name: modelId,
           api: 'anthropic-messages',
           baseUrl: inference.base_url,
-          // Must be false: pi-ai emits the deprecated `thinking:{type:'enabled',
-          // budget_tokens}` for reasoning models, which Opus 4.8 / Sonnet 5
-          // reject with a 400 (they only accept `thinking:{type:'adaptive'}`).
-          // With reasoning on, every request errored server-side → 0 tool calls,
-          // empty diff. Off means no extended thinking on the pi path, but the
-          // requests succeed. (The anthropic control uses the SDK's adaptive form;
-          // thinking parity needs a newer pi-ai.)
-          reasoning: false,
+          reasoning: true,
+          // forceAdaptiveThinking: send `thinking:{type:'adaptive'}` +
+          // output_config.effort (what Opus 4.8 / Sonnet 5 require) instead of
+          // pi-ai's default deprecated `thinking:{type:'enabled',budget_tokens}`,
+          // which those models reject with a 400 — matching the anthropic
+          // control's adaptive thinking. supportsTemperature:false because Opus
+          // 4.7+ rejects a non-default temperature. Both mirror how pi-ai's own
+          // built-in anthropic-messages models are configured.
+          compat: { forceAdaptiveThinking: true, supportsTemperature: false },
           input: ['text'],
           cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
           contextWindow: 1_000_000,
@@ -133,6 +134,8 @@ export const piHarness: Harness = {
     const { session } = await createAgentSession({
       model,
       modelRegistry: registry,
+      // Adaptive-thinking effort, matching the anthropic control's medium effort.
+      thinkingLevel: 'medium',
       cwd,
       sessionManager: SessionManager.inMemory(cwd),
       resourceLoader,