test(agent-eval): price codegraph's fixed context cost alongside its residual (CG-7)
The first request's prompt is system + tool schemas + the question, before any tool has answered, so differencing the arms' ctxBase prices what codegraph occupies whether or not the agent ever calls it. Measured on gin: +775 tokens, small because the tool is deferred -- only its name is in the initial listing.
This commit is contained in:
@@ -45,6 +45,7 @@ function parse(dir, label) {
|
|||||||
tokens: s.processed, cost: s.cost, raced: s.raced, turns: s.turns,
|
tokens: s.processed, cost: s.cost, raced: s.raced, turns: s.turns,
|
||||||
segments: files.length,
|
segments: files.length,
|
||||||
ctx: o.ctxFinal,
|
ctx: o.ctxFinal,
|
||||||
|
ctxBase: o.ctxBase,
|
||||||
occCg: o.residual.codegraph,
|
occCg: o.residual.codegraph,
|
||||||
occFile: o.residualFileAccess,
|
occFile: o.residualFileAccess,
|
||||||
// The arm's own retrieval residual: codegraph in the with-arm, Read/Grep/Bash
|
// The arm's own retrieval residual: codegraph in the with-arm, Read/Grep/Bash
|
||||||
@@ -107,10 +108,11 @@ console.log(`\n\nRESIDUAL CONTEXT OCCUPANCY — retrieval tokens still in the wi
|
|||||||
console.log(`(WITH = codegraph responses · WITHOUT = Read + Grep/Glob + Bash responses)`);
|
console.log(`(WITH = codegraph responses · WITHOUT = Read + Grep/Glob + Bash responses)`);
|
||||||
console.log(`${anyMulti ? 'multi-turn sessions' : 'SINGLE-TURN sessions — see the caveat below'}\n`);
|
console.log(`${anyMulti ? 'multi-turn sessions' : 'SINGLE-TURN sessions — see the caveat below'}\n`);
|
||||||
console.log('repo turns final ctx W→WO residual W→WO % of ctx W→WO % of window W→WO');
|
console.log('repo turns final ctx W→WO residual W→WO % of ctx W→WO % of window W→WO');
|
||||||
const occ = { resid: [], shareCtx: [] };
|
const occ = { resid: [], shareCtx: [], fixed: [] };
|
||||||
for (const { repo, W, WO } of rows) {
|
for (const { repo, W, WO } of rows) {
|
||||||
if (!W.length || !WO.length) { console.log(`${repo.padEnd(11)} (incomplete)`); continue; }
|
if (!W.length || !WO.length) { console.log(`${repo.padEnd(11)} (incomplete)`); continue; }
|
||||||
const m = (arr, k) => median(arr.map(x => x[k]));
|
const m = (arr, k) => median(arr.map(x => x[k]));
|
||||||
|
occ.fixed.push(m(W, 'ctxBase') - m(WO, 'ctxBase'));
|
||||||
const wR = m(W, 'occSelf'), woR = m(WO, 'occSelf');
|
const wR = m(W, 'occSelf'), woR = m(WO, 'occSelf');
|
||||||
const wCtx = m(W, 'ctx'), woCtx = m(WO, 'ctx');
|
const wCtx = m(W, 'ctx'), woCtx = m(WO, 'ctx');
|
||||||
const wSc = m(W, 'occShareCtx'), woSc = m(WO, 'occShareCtx');
|
const wSc = m(W, 'occShareCtx'), woSc = m(WO, 'occShareCtx');
|
||||||
@@ -125,6 +127,11 @@ for (const { repo, W, WO } of rows) {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
console.log(`\nAVERAGE: retrieval residual ${avg(occ.resid)}% lower with codegraph · share-of-context ${avg(occ.shareCtx)}% lower`);
|
console.log(`\nAVERAGE: retrieval residual ${avg(occ.resid)}% lower with codegraph · share-of-context ${avg(occ.shareCtx)}% lower`);
|
||||||
|
console.log(
|
||||||
|
`FIXED overhead: codegraph's tool schema + MCP instructions cost ${avg(occ.fixed) >= 0 ? '+' : ''}${avg(occ.fixed)} tok\n` +
|
||||||
|
` of context before any tool is called (median WITH ctxBase - median WITHOUT ctxBase, averaged\n` +
|
||||||
|
` over repos). It is paid whether or not the agent ever calls codegraph.`
|
||||||
|
);
|
||||||
if (!anyMulti) {
|
if (!anyMulti) {
|
||||||
console.log(
|
console.log(
|
||||||
`\nCAVEAT: every row above is a SINGLE-turn session, so the residual is measured at the\n` +
|
`\nCAVEAT: every row above is a SINGLE-turn session, so the residual is measured at the\n` +
|
||||||
|
|||||||
@@ -231,6 +231,11 @@ export function parseSession(files) {
|
|||||||
for (const q of queue) residual[q.family] += q.tokens;
|
for (const q of queue) residual[q.family] += q.tokens;
|
||||||
|
|
||||||
const ctxFinal = reqIdx.length ? timeline[reqIdx[reqIdx.length - 1]].ctx : 0;
|
const ctxFinal = reqIdx.length ? timeline[reqIdx[reqIdx.length - 1]].ctx : 0;
|
||||||
|
// The FIRST request's prompt is system + tool schemas + the question, before
|
||||||
|
// any tool has answered. Differencing the arms' ctxBase prices codegraph's
|
||||||
|
// FIXED occupancy — its tool schema and MCP `initialize` instructions — which
|
||||||
|
// it pays whether or not the agent ever calls it.
|
||||||
|
const ctxBase = reqIdx.length ? timeline[reqIdx[0]].ctx : 0;
|
||||||
// Multi-turn: duration/cost/tokens are per-segment, so sum them. `result.usage`
|
// Multi-turn: duration/cost/tokens are per-segment, so sum them. `result.usage`
|
||||||
// is cumulative WITHIN a segment (verified: its in+cache+out equals the sum of
|
// is cumulative WITHIN a segment (verified: its in+cache+out equals the sum of
|
||||||
// that segment's per-request prompts), so summing segments is correct and does
|
// that segment's per-request prompts), so summing segments is correct and does
|
||||||
@@ -252,7 +257,7 @@ export function parseSession(files) {
|
|||||||
cost: results.reduce((s, r) => s + (r.total_cost_usd || 0), 0),
|
cost: results.reduce((s, r) => s + (r.total_cost_usd || 0), 0),
|
||||||
processed,
|
processed,
|
||||||
occupancy: {
|
occupancy: {
|
||||||
ctxFinal, windowTokens: WINDOW_TOKENS,
|
ctxFinal, ctxBase, windowTokens: WINDOW_TOKENS,
|
||||||
charsPerToken, calibrated, compactions, evicted: Math.round(evicted),
|
charsPerToken, calibrated, compactions, evicted: Math.round(evicted),
|
||||||
residual: Object.fromEntries(FAMILIES.map((f) => [f, Math.round(residual[f])])),
|
residual: Object.fromEntries(FAMILIES.map((f) => [f, Math.round(residual[f])])),
|
||||||
contributed: Object.fromEntries(FAMILIES.map((f) => [f, Math.round(contributed[f])])),
|
contributed: Object.fromEntries(FAMILIES.map((f) => [f, Math.round(contributed[f])])),
|
||||||
@@ -287,6 +292,7 @@ export function formatOccupancy(s, indent = ' ') {
|
|||||||
row('other tools', o.residual.other, o.chars.other, o.results.other);
|
row('other tools', o.residual.other, o.chars.other, o.results.other);
|
||||||
const toolTotal = Object.values(o.residual).reduce((a, b) => a + b, 0);
|
const toolTotal = Object.values(o.residual).reduce((a, b) => a + b, 0);
|
||||||
row('base (prompt+prose)', Math.max(0, o.ctxFinal - toolTotal));
|
row('base (prompt+prose)', Math.max(0, o.ctxFinal - toolTotal));
|
||||||
|
out.push(`${indent} ${' of which fixed'.padEnd(18)}${(n(o.ctxBase) + ' tok').padStart(12)} system + tool schemas + question, before any tool answered`);
|
||||||
out.push(...rows);
|
out.push(...rows);
|
||||||
const dropped = o.contributed.codegraph + o.contributedFileAccess + o.contributed.other
|
const dropped = o.contributed.codegraph + o.contributedFileAccess + o.contributed.other
|
||||||
- (o.residual.codegraph + o.residualFileAccess + o.residual.other);
|
- (o.residual.codegraph + o.residualFileAccess + o.residual.other);
|
||||||
|
|||||||
Reference in New Issue
Block a user