From 04c0f8eab5783fc3b481c47f8076246e2e1c3efd Mon Sep 17 00:00:00 2001 From: Colby McHenry Date: Wed, 5 Aug 2026 21:27:02 -0500 Subject: [PATCH] =?UTF-8?q?test(agent-eval):=20sum=20tokens=20per=20turn?= =?UTF-8?q?=20=E2=80=94=20result.usage=20stopped=20being=20cumulative?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit "Tokens processed" was read off result.usage. That was correct when the README figures were measured; in current Claude Code the field reports the LAST turn only. Nothing here changed — the host did, silently — and the harness kept reporting the smaller number. The error is one-sided, which makes it worse than noise: it under-counts whichever arm takes more turns, and that is always the WITHOUT arm. On the 2026-08-05 campaign it turned a real 62% token saving into 19% and invented a token REGRESSION on tokio (-41%) and alamofire (-25%) that does not exist. Those numbers were one push away from the README. Now summed per assistant request and deduped by message.id, the same rule the occupancy timeline already used — Claude Code emits one event per content block carrying identical usage, so summing per event double-counts (~1.7x measured). CLAUDE.md already warned about this field. The code did not follow; it does now. Co-Authored-By: Claude Opus 5 --- scripts/agent-eval/parse-run.mjs | 37 +++++++++++++++++++++++++------- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/scripts/agent-eval/parse-run.mjs b/scripts/agent-eval/parse-run.mjs index 58bff56..554f4af 100644 --- a/scripts/agent-eval/parse-run.mjs +++ b/scripts/agent-eval/parse-run.mjs @@ -297,14 +297,35 @@ export function parseSession(files) { // FIXED occupancy — its tool schema and MCP `initialize` instructions — which // it pays whether or not the agent ever calls it. const ctxBase = reqIdx.length ? timeline[reqIdx[0]].ctx : 0; - // Multi-turn: duration/cost/tokens are per-segment, so sum them. `result.usage` - // is cumulative WITHIN a segment (verified: its in+cache+out equals the sum of - // that segment's per-request prompts), so summing segments is correct and does - // NOT double-count. It is a "tokens processed" figure — every request re-counts - // the whole prefix — which is exactly why it can't answer the occupancy question. - const sumUsage = (k) => results.reduce((s, r) => s + (r.usage?.[k] || 0), 0); - const processed = sumUsage('input_tokens') + sumUsage('cache_read_input_tokens') - + sumUsage('cache_creation_input_tokens') + sumUsage('output_tokens'); + // "Tokens processed" = the SUM of every assistant request's whole prompt. Every + // request re-counts the prefix, which is what makes this a throughput figure and + // why it cannot answer the occupancy question (that is `residual`, above). + // + // It must be summed PER TURN, not read off `result.usage`. The comment that used + // to sit here asserted result.usage was "cumulative WITHIN a segment"; that was + // true when the README figures were measured and is NOT true in current Claude + // Code, where it reports the LAST turn only. Nothing in this repo changed — the + // host did, silently, and the harness kept reporting the smaller number. + // + // The damage is one-sided and therefore worse than noise: it under-counts + // whichever arm takes more turns, which is always the WITHOUT arm. Measured on + // the 2026-08-05 campaign it turned a real 62% token saving into 19%, and + // manufactured a token REGRESSION on tokio (-41%) and alamofire (-25%) that does + // not exist. CLAUDE.md already warned about this field; the code did not follow. + // + // Dedupe by message.id: Claude Code emits one assistant event per content block, + // each carrying the same `usage`, so summing per EVENT double-counts (~1.7x on a + // real run). Same rule the occupancy timeline above uses. + const seenUsageIds = new Set(); + let processed = 0; + for (const ev of events) { + if (ev.type !== 'assistant' || !ev.message?.id) continue; + if (seenUsageIds.has(ev.message.id)) continue; + seenUsageIds.add(ev.message.id); + const u = ev.message.usage || {}; + processed += (u.input_tokens || 0) + (u.cache_read_input_tokens || 0) + + (u.cache_creation_input_tokens || 0) + (u.output_tokens || 0); + } return { files, toolCalls, counts, initTools, result, results, raced, cliCalls, cliContaminated,