merge main into CG-3: keep the envelope view alongside occupancy
CG-3 branched from main before CG-1 landed and rewrote parse-run.mjs wholesale into an exported parseSession(), which dropped CG-1's --envelope/--answer reporting entirely. That view is the instrument the CG-1/CG-22 allocation gate measures bar 2 with, and it is in that benchmark's documented reproduce steps, so it cannot be lost to the merge. Resolution takes CG-3's rewrite as the structure and ports the envelope feature into it: parseSession now collects codegraph_explore response text in call order, formatEnvelope renders the per-file share, and the CLI parses --envelope/--answer ahead of the positional filter so a glob is never mistaken for a log path. The glob sentinel stays written as a \u0000 escape, never a literal NUL byte -- a raw one makes git treat the whole script as binary, exactly as the comment there warns. Verified: --selftest 18/18, and a synthetic explore transcript reports the expected per-file shares and answer-set total. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Colby McHenry
Claude Opus 5
@@ -21,7 +21,19 @@
|
||||
# <indexed-repo> a repo with a .codegraph index (copied per arm)
|
||||
# "<task>" an implementation task, e.g. "Add X to Y and wire it through"
|
||||
# [baseline-ref] git ref for the BEFORE build (default: HEAD~1)
|
||||
# Env: AGENT_EVAL_OUT (default: /tmp/ab-new-vs-baseline)
|
||||
# Env:
|
||||
# AGENT_EVAL_OUT output dir (default: /tmp/ab-new-vs-baseline)
|
||||
# RUNS runs per arm (default 1). Run-to-run variance is large —
|
||||
# use >=2 and report the range, never a single run. Both arms
|
||||
# build/index ONCE and then run RUNS times, so raising this is
|
||||
# far cheaper than re-invoking the script.
|
||||
# MODEL / EFFORT default sonnet / high. Never raise without a reason: sonnet
|
||||
# is the deliberate floor model (see CLAUDE.md).
|
||||
#
|
||||
# Both arms run with CODEGRAPH_NO_PROMPT_HOOK=1: the machine's ambient
|
||||
# UserPromptSubmit front-load hook resolves to whichever build is currently in
|
||||
# dist/, so leaving it on injects context through a second, uncontrolled channel
|
||||
# and confounds the tool-call counts this script exists to compare.
|
||||
set -uo pipefail
|
||||
|
||||
TARGET="${1:?usage: ab-new-vs-baseline.sh <indexed-repo> \"<task>\" [baseline-ref]}"
|
||||
@@ -46,7 +58,9 @@ cleanup() {
|
||||
git -C "$ENGINE" checkout HEAD -- $CHANGED 2>/dev/null
|
||||
( cd "$ENGINE" && npm run build >/dev/null 2>&1 )
|
||||
}
|
||||
trap cleanup EXIT
|
||||
# INT/TERM too: killing the script mid-baseline-arm otherwise leaves the engine
|
||||
# checked out at the baseline ref, which silently poisons every later build.
|
||||
trap cleanup EXIT INT TERM
|
||||
|
||||
mkdir -p "$OUT"
|
||||
echo "###### engine=$ENGINE baseline=$BASE_REF"
|
||||
@@ -67,18 +81,27 @@ prewarm() { # target — spawn a persistent daemon (current $BIN) and wait for i
|
||||
&& echo " daemon warm: $1" || echo " WARN: daemon never bound for $1 (arm may run without codegraph)"
|
||||
}
|
||||
|
||||
run_arm() { # label, target-copy
|
||||
run_arm() { # label, target-copy — runs the task $RUNS times against one build
|
||||
local label="$1" tgt="$2" c="$OUT/mcp-$1.json"
|
||||
# Connect to the pre-warmed daemon; skip the startup re-exec for a fast attach.
|
||||
printf '{"mcpServers":{"codegraph":{"command":"env","args":["CODEGRAPH_WASM_RELAUNCHED=1","node","%s","serve","--mcp","--path","%s"]}}}' "$BIN" "$tgt" > "$c"
|
||||
prewarm "$tgt"
|
||||
# CODEGRAPH_EXPLORE_DEBUG points explore's per-file allocation diagnostic at a
|
||||
# sidecar (no-op on builds predating it; never perturbs the response).
|
||||
printf '{"mcpServers":{"codegraph":{"command":"env","args":["CODEGRAPH_WASM_RELAUNCHED=1","CODEGRAPH_EXPLORE_DEBUG=%s","node","%s","serve","--mcp","--path","%s"]}}}' \
|
||||
"$OUT/explore-$label.jsonl" "$BIN" "$tgt" > "$c"
|
||||
rm -f "$OUT/explore-$label.jsonl"
|
||||
echo "############## ARM [$label] ##############"
|
||||
( cd "$tgt" && claude -p "$TASK" \
|
||||
--output-format stream-json --verbose --permission-mode bypassPermissions \
|
||||
--model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" --max-budget-usd 4 --strict-mcp-config --mcp-config "$c" \
|
||||
</dev/null > "$OUT/run-$label.jsonl" 2>"$OUT/run-$label.err" )
|
||||
node "$PARSE" "$OUT/run-$label.jsonl" 2>&1 | grep -E "by type|Result" || echo " (parse failed — see $OUT/run-$label.jsonl)"
|
||||
pkill -9 -f "serve --mcp --path $tgt" 2>/dev/null
|
||||
for i in $(seq 1 "${RUNS:-1}"); do
|
||||
# Re-warm per run: the previous run's daemon is killed below, and a cold
|
||||
# attach is exactly the failure this pre-warm exists to prevent.
|
||||
prewarm "$tgt"
|
||||
( cd "$tgt" && CODEGRAPH_NO_PROMPT_HOOK=1 claude -p "$TASK" \
|
||||
--output-format stream-json --verbose --permission-mode bypassPermissions \
|
||||
--model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" --max-budget-usd 4 --strict-mcp-config --mcp-config "$c" \
|
||||
</dev/null > "$OUT/run-$label-$i.jsonl" 2>"$OUT/run-$label-$i.err" )
|
||||
echo "-- run $i --"
|
||||
node "$PARSE" "$OUT/run-$label-$i.jsonl" 2>&1 | grep -E "by type|Result" || echo " (parse failed — see $OUT/run-$label-$i.jsonl)"
|
||||
pkill -9 -f "serve --mcp --path $tgt" 2>/dev/null
|
||||
done
|
||||
echo
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,159 @@
|
||||
{
|
||||
"$comment": [
|
||||
"Regression fixtures for GitHub issue #1500 / epic CG-1 — relevance-proportional",
|
||||
"explore budget allocation. Run them with `node scripts/agent-eval/probe-allocation.mjs`",
|
||||
"against a built dist/.",
|
||||
"",
|
||||
"STATUS: BOTH FIXTURES PASS. CG-10 (relevance scoring) closed the RANKING half —",
|
||||
"nothing incidental reaches the envelope any more — and CG-12 (score-proportional",
|
||||
"allocation with a relative cliff) closed the BYTE SPLIT: each file's share is reserved",
|
||||
"before anything renders, and a file under 15% of the top weight gets no source at all,",
|
||||
"freeing both its bytes and its maxFiles slot. Each fixture records the pre-CG-10",
|
||||
"`baseline`, the interim `afterCG10`, and the current `afterCG12`.",
|
||||
"",
|
||||
"`groups` partitions the files explore rendered into `answer` (what the query is",
|
||||
"actually about) and `incidental` (what wins the envelope today on name collisions).",
|
||||
"Assertions are on the DELIVERED envelope unless suffixed `Allocated`; delivered is",
|
||||
"what the agent got, allocated is what the render loop chose before the hard ceiling.",
|
||||
"Shares are fractions of the whole response, meta-text included, so they never sum to 1."
|
||||
],
|
||||
"fixtures": [
|
||||
{
|
||||
"id": "payroll-go",
|
||||
"title": "#1500 — generated Go CRUD beside a hand-written payroll workflow",
|
||||
"kind": "fixture",
|
||||
"path": "__tests__/fixtures/payroll-go",
|
||||
"query": "how does payroll cycle create and calculate payslips?",
|
||||
"rationale": [
|
||||
"The reporter's repo shape: a Go service whose generated FKIT CRUD layer sits",
|
||||
"beside the hand-written use-case that does the real work. The query deliberately",
|
||||
"does NOT name runPayrollCycleAll / BuildPayslip / Upsert — an architecture",
|
||||
"question phrased the way a newcomer would phrase it. The generated layer",
|
||||
"name-collides on every query term (CreatePayslip, PayrollCycleCreateRequest,",
|
||||
"CalculatePayrollCycleTotals, a second BuildPayslip, a second Upsert), so a",
|
||||
"scorer that rewards incidental name matches surfaces the CRUD path.",
|
||||
"Half the generated files carry ORDINARY names and are only detectable by their",
|
||||
"`// Code generated ... DO NOT EDIT.` header (CG-5), which is what makes this",
|
||||
"the #1500 case rather than a .pb.go case."
|
||||
],
|
||||
"groups": {
|
||||
"answer": [
|
||||
"internal/usecase/**",
|
||||
"internal/store/**",
|
||||
"internal/transport/**",
|
||||
"internal/domain/**",
|
||||
"cmd/**"
|
||||
],
|
||||
"incidental": ["internal/gen/**"]
|
||||
},
|
||||
"assert": {
|
||||
"answerShareAtLeast": 0.55,
|
||||
"incidentalShareAtMost": 0.25,
|
||||
"topFileGroup": "answer",
|
||||
"mustDeliverBytes": [
|
||||
"internal/usecase/payroll/cycle.go",
|
||||
"internal/usecase/payroll/payslip_builder.go"
|
||||
],
|
||||
"$mustContainComment": "Needles are chosen to match the HAND-WRITTEN chain only — a bare `BuildPayslip`/`Upsert` also matches the generated collisions, which is the whole point of the fixture.",
|
||||
"mustContain": [
|
||||
"runPayrollCycleAll",
|
||||
"func (s *Service) BuildPayslip",
|
||||
"s.store.Upsert(ctx, slip)"
|
||||
]
|
||||
},
|
||||
"baseline": {
|
||||
"measuredOn": "2026-08-03",
|
||||
"note": "19 files → very-tiny tier (13,000 budget); 23,020 chars allocated against it, cut to 16,011 by the 19,500 hard ceiling.",
|
||||
"delivered": {
|
||||
"internal/gen/fkit/payroll/payslip.go": 0.307,
|
||||
"internal/gen/fkit/payroll/payroll_cycle.go": 0.266,
|
||||
"internal/domain/payroll/payslip.go": 0.256,
|
||||
"internal/usecase/payroll/cycle.go": 0.0
|
||||
},
|
||||
"verdict": "The workflow file is allocated the single largest slice (30.6%) and delivers ZERO — the hard ceiling drops its whole section. Generated CRUD takes 57.4% of what the agent actually receives; runPayrollCycleAll, BuildPayslip and the real Upsert never reach the response."
|
||||
},
|
||||
"afterCG10": {
|
||||
"measuredOn": "2026-08-04",
|
||||
"delivered": {
|
||||
"internal/usecase/payroll/cycle.go": 0.389,
|
||||
"internal/gen/fkit/payroll/payroll_cycle.go": 0.235,
|
||||
"internal/domain/payroll/payslip.go": 0.226,
|
||||
"internal/gen/fkit/payroll/payslip.go": 0.0
|
||||
},
|
||||
"verdict": "PASSES answerShareAtLeast (61.5%), incidentalShareAtMost (23.5%), topFileGroup, and the cycle.go + runPayrollCycleAll + real-Upsert needles. The generated files now rank #3/#4 instead of #1/#2 — kind weighting plus a 0.3x generated penalty on BOTH the score and the graph mass, which is the key the comparator sorts on. STILL FAILING for CG-12: payslip_builder.go ranks #6 against the tier's maxFiles of 4, so `func (s *Service) BuildPayslip` never renders."
|
||||
},
|
||||
"afterCG12": {
|
||||
"measuredOn": "2026-08-04",
|
||||
"note": "19,337 delivered, nothing truncated. Both generated files cliffed to pointers (weight 3.9 and 3.5 against a cliff of 5.4), which frees the two maxFiles slots the hand-written store and builder then take.",
|
||||
"delivered": {
|
||||
"internal/usecase/payroll/cycle.go": 0.306,
|
||||
"internal/domain/payroll/payslip.go": 0.212,
|
||||
"internal/usecase/payroll/payslip_builder.go": 0.151,
|
||||
"internal/store/payslipstore/store.go": 0.118,
|
||||
"internal/gen/fkit/payroll/payroll_cycle.go": 0.0,
|
||||
"internal/gen/fkit/payroll/payslip.go": 0.0
|
||||
},
|
||||
"verdict": "ALL GATES PASS. Answer group 78.7% (from 25.6% at baseline), generated layer 0.0% (from 57.4%). All four hand-written files deliver source, including payslip_builder.go — `func (s *Service) BuildPayslip`, the 'calculate' half of the question, finally reaches the agent. The generated files are still NAMED with their symbols and line numbers under 'Not shown above', so withholding their bytes costs ~100 chars each instead of ~4,500 and stays one follow-up explore away."
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "self-query",
|
||||
"title": "This repo — incidental `explore`/`BUDGET` matches in the agent-eval scripts",
|
||||
"kind": "self",
|
||||
"path": ".",
|
||||
"query": "how does explore allocate its output budget across files",
|
||||
"rationale": [
|
||||
"The same failure mode with no generated code in sight. `scripts/agent-eval/*.mjs`",
|
||||
"mention `explore` and `BUDGET` incidentally — they are eval harnesses, not the",
|
||||
"allocator — and they are small enough to ship WHOLE, while src/mcp/tools.ts (which",
|
||||
"carries getExploreOutputBudget and the render loop, and scores 4x higher on every",
|
||||
"signal) is large enough to be clipped at maxCharsPerFile. Allocation follows file",
|
||||
"size, not relevance.",
|
||||
"",
|
||||
"Unlike payroll-go this fixture reads THIS repo's live index, so its exact numbers",
|
||||
"move as the repo changes (indexed file count crossing 500 flips the budget tier).",
|
||||
"The assertions are therefore relative — answer-vs-incidental, not fixed percentages."
|
||||
],
|
||||
"groups": {
|
||||
"answer": ["src/mcp/**"],
|
||||
"incidental": ["scripts/**"]
|
||||
},
|
||||
"assert": {
|
||||
"answerShareAtLeast": 0.5,
|
||||
"incidentalShareAtMost": 0.25,
|
||||
"topFileGroup": "answer",
|
||||
"mustDeliverBytes": ["src/mcp/tools.ts"]
|
||||
},
|
||||
"baseline": {
|
||||
"measuredOn": "2026-08-03",
|
||||
"note": "493 files → small tier (18,000 budget); 27,518 chars allocated against it, cut to 19,749 by the 25,000 hard ceiling.",
|
||||
"delivered": {
|
||||
"scripts/agent-eval/offload-eval-hook.mjs": 0.25,
|
||||
"scripts/agent-eval/offload-eval-metrics.mjs": 0.236,
|
||||
"scripts/agent-eval/parse-session.mjs": 0.232,
|
||||
"src/mcp/tools.ts": 0.185,
|
||||
"scripts/agent-eval/offload-eval-cost.mjs": 0.0
|
||||
},
|
||||
"verdict": "The script corpus takes 71.8% of the delivered envelope (79.4% of what was allocated) against tools.ts's 18.5%, despite tools.ts scoring 46 vs 10, carrying 2.3x the graph mass and 3x the distinct term hits."
|
||||
},
|
||||
"afterCG10": {
|
||||
"measuredOn": "2026-08-04",
|
||||
"delivered": {
|
||||
"src/resolution/memory-budget.ts": 0.512,
|
||||
"src/mcp/tools.ts": 0.329
|
||||
},
|
||||
"verdict": "PASSES incidentalShareAtMost: every scripts/agent-eval/*.mjs file is gone (0.0%), which is CG-10's acceptance bar — their sole match was an unused file-scope `explore`/`BUDGET` constant, worth 0.08x weight, and the relative floor (8.2) then cut them. tools.ts ranks #1. STILL FAILING for CG-12: memory-budget.ts scores 18 to tools.ts's 41 yet takes 51.2% of the envelope to tools.ts's 32.9%, purely because it is small enough to ship whole while tools.ts is clipped at maxCharsPerFile. Allocation still follows file size, not relevance."
|
||||
},
|
||||
"afterCG12": {
|
||||
"measuredOn": "2026-08-04",
|
||||
"note": "18,134 delivered, nothing truncated. tools.ts scores 58 here (it grew by the allocator this task added), memory-budget.ts 18.",
|
||||
"delivered": {
|
||||
"src/mcp/tools.ts": 0.606,
|
||||
"src/resolution/memory-budget.ts": 0.172,
|
||||
"src/resolution/lru-cache.ts": 0.111
|
||||
},
|
||||
"verdict": "ALL GATES PASS. tools.ts takes 60.6% of the envelope, up from 18.5% at baseline and 32.9% after CG-10 — past the epic's >50% acceptance bar. The reversal is the whole point: memory-budget.ts no longer wins by being small enough to ship whole (it now clusters within its 3.1K reservation), and tools.ts is no longer clipped at maxCharsPerFile (11K reservation, ~3x the old flat cap). Exception to 'no previously-unclipped file becomes clipped': memory-budget.ts was unclipped-whole at 5,672 and is now clipped to its proportional share. That is the epic's own diagnosis of the bug, not a regression — it scored 18 against tools.ts's 58 and was taking the larger slice."
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -3,12 +3,19 @@
|
||||
// RESIDUAL CONTEXT OCCUPANCY — how many tokens of the context window each tool
|
||||
// family's responses still occupy when the run ends.
|
||||
//
|
||||
// Usage: parse-run.mjs <run.jsonl> [run.t2.jsonl ...]
|
||||
// Usage: parse-run.mjs <run.jsonl> [run.t2.jsonl ...] [--envelope] [--answer <glob>]...
|
||||
// Multiple files = one multi-turn session's segments, IN ORDER (run-all.sh
|
||||
// writes run-<label>.jsonl, run-<label>.t2.jsonl, … for a `Q1||Q2||Q3` set).
|
||||
// `--resume` does not replay prior messages, so the segments concatenate
|
||||
// cleanly and token accounting carries across the boundary.
|
||||
//
|
||||
// `--envelope` additionally reports how the codegraph_explore responses were
|
||||
// DIVIDED across files — the per-file share of the source envelope (#1500).
|
||||
// `--answer <glob>` (repeatable, implies --envelope) marks the files that
|
||||
// actually answer the question and reports their combined share: bar 2 of the
|
||||
// CG-1/CG-22 allocation gate. See formatEnvelope for why it parses the
|
||||
// rendered markdown rather than the CG-4 diagnostic sidecar.
|
||||
//
|
||||
// ---------------------------------------------------------------------------
|
||||
// Why occupancy, and how it's measured
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -102,6 +109,10 @@ export function parseSession(files) {
|
||||
let initTools = null, result = null, raced = false, cliCalls = 0, cliContaminated = 0;
|
||||
const results = []; // one `result` event per session segment (multi-turn)
|
||||
let compactions = 0;
|
||||
// Raw codegraph_explore response text, in call order. Feeds the envelope view
|
||||
// (see formatEnvelope) — kept here rather than re-parsed from the log later so
|
||||
// a multi-segment session's responses stay in one ordered list.
|
||||
const exploreTexts = [];
|
||||
|
||||
// A timeline of everything appended to the context, in order. `req` entries
|
||||
// are assistant requests (carrying that request's ctx); `add` entries are
|
||||
@@ -161,6 +172,7 @@ export function parseSession(files) {
|
||||
// by the binary being genuinely absent) and put nothing in context.
|
||||
if (cliById.has(b.tool_use_id) && !b.is_error) cliContaminated++;
|
||||
const name = nameById.get(b.tool_use_id) || '';
|
||||
if (/codegraph_explore/.test(name) && !b.is_error) exploreTexts.push(t);
|
||||
timeline.push({ kind: 'add', family: familyOf(name), chars: t.length, tool: name });
|
||||
} else {
|
||||
timeline.push({ kind: 'add', family: null, chars: textOf([b]).length });
|
||||
@@ -287,6 +299,7 @@ export function parseSession(files) {
|
||||
|
||||
return {
|
||||
files, toolCalls, counts, initTools, result, results, raced, cliCalls, cliContaminated,
|
||||
exploreTexts,
|
||||
ok: results.length > 0 && results.every((r) => r.subtype === 'success'),
|
||||
turns: reqIdx.length,
|
||||
tools: toolCalls.filter((t) => !t.startsWith('ToolSearch')).length,
|
||||
@@ -345,6 +358,68 @@ export function formatOccupancy(s, indent = ' ') {
|
||||
return out.join('\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* How the codegraph_explore responses the agent received were DIVIDED across
|
||||
* files — the per-file share of the source envelope (#1500 / epic CG-1).
|
||||
*
|
||||
* Parsed out of the RENDERED MARKDOWN, not the CG-4 diagnostic sidecar: the
|
||||
* sidecar only exists on a post-CG-4 build, so it cannot measure a baseline arm.
|
||||
* The markdown parse is the only instrument that measures both arms of a
|
||||
* new-vs-baseline A/B the same way.
|
||||
*
|
||||
* `answerGlobs` marks the files that actually answer the question; the summary
|
||||
* reports their combined share, which is bar 2 of the CG-1/CG-22 gate.
|
||||
*/
|
||||
export function formatEnvelope(exploreTexts, answerGlobs = [], indent = ' ') {
|
||||
// `tools/cache/**` -> /^tools\/cache\/.*$/ . Same semantics as probe-allocation.
|
||||
// The `**` sentinel is written as an escape, never a literal NUL byte — a raw
|
||||
// one makes git treat this whole script as binary and costs every future diff.
|
||||
const glob2re = (glob) => {
|
||||
const S = '\\u0000';
|
||||
const body = glob.replace(/[.+^${}()|[\]\\]/g, '\\$&')
|
||||
.replace(/\*\*/g, S).replace(/\*/g, '[^/]*').replaceAll(S, '.*');
|
||||
return new RegExp(`^${body}$`);
|
||||
};
|
||||
const answerRes = answerGlobs.map(glob2re);
|
||||
const isAnswer = (p) => answerRes.some((re) => re.test(p));
|
||||
|
||||
// Each rendered file section starts with **`path`** — its bytes run to the next
|
||||
// such header (or to the trailing guidance quote). Share is over the sum of the
|
||||
// sections, i.e. of the source envelope the allocator divides.
|
||||
const pooled = new Map();
|
||||
let envelope = 0;
|
||||
for (const text of exploreTexts) {
|
||||
const re = /^\*\*`([^`]+)`\*\*/gm;
|
||||
const marks = [];
|
||||
let m;
|
||||
while ((m = re.exec(text)) !== null) marks.push({ path: m[1], at: m.index });
|
||||
if (!marks.length) continue;
|
||||
const tail = text.indexOf('\n> ', marks[marks.length - 1].at);
|
||||
const end = tail === -1 ? text.length : tail;
|
||||
marks.forEach((mark, i) => {
|
||||
const chars = (i + 1 < marks.length ? marks[i + 1].at : end) - mark.at;
|
||||
pooled.set(mark.path, (pooled.get(mark.path) ?? 0) + chars);
|
||||
envelope += chars;
|
||||
});
|
||||
}
|
||||
const ranked = [...pooled.entries()]
|
||||
.map(([path, chars]) => ({ path, chars, share: envelope ? chars / envelope : 0, answer: isAnswer(path) }))
|
||||
.sort((a, b) => b.chars - a.chars);
|
||||
const answerChars = ranked.filter((r) => r.answer).reduce((s, r) => s + r.chars, 0);
|
||||
const pct = (f) => `${(f * 100).toFixed(1)}%`;
|
||||
|
||||
const out = [];
|
||||
out.push(`${indent}Explore envelope: ${envelope.toLocaleString('en-US')} chars over ${exploreTexts.length} response(s)`);
|
||||
if (answerGlobs.length) {
|
||||
out.push(`${indent} answer-set share: ${pct(envelope ? answerChars / envelope : 0)} | top file answers: ${ranked[0]?.answer ?? false}`);
|
||||
}
|
||||
for (const f of ranked.slice(0, 12)) {
|
||||
out.push(`${indent} ${f.answer ? '*' : ' '} ${pct(f.share).padStart(6)} ${String(f.chars).padStart(6)} ${f.path}`);
|
||||
}
|
||||
if (ranked.length > 12) out.push(`${indent} … ${ranked.length - 12} more files`);
|
||||
return out.join('\n');
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// `--selftest`: the occupancy math over synthetic transcripts with known
|
||||
// answers. It lives here rather than in a test file on purpose — a new
|
||||
@@ -462,8 +537,19 @@ function require0(m) { return process.getBuiltinModule(m); }
|
||||
const isMain = process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href;
|
||||
if (isMain && process.argv.includes('--selftest')) process.exit(selftest() ? 1 : 0);
|
||||
if (isMain) {
|
||||
const files = process.argv.slice(2).filter((a) => !a.startsWith('--'));
|
||||
if (!files.length) { console.error('usage: parse-run.mjs <run.jsonl> [run.t2.jsonl ...] | --selftest'); process.exit(1); }
|
||||
// `--answer <glob>` is repeatable and implies `--envelope`. Its VALUE is not a
|
||||
// run file, so consume it here rather than letting the positional filter below
|
||||
// mistake a glob for a log path.
|
||||
const argv = process.argv.slice(2);
|
||||
const files = [];
|
||||
const answerGlobs = [];
|
||||
let wantEnvelope = false;
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
if (argv[i] === '--envelope') wantEnvelope = true;
|
||||
else if (argv[i] === '--answer') { answerGlobs.push(argv[++i]); wantEnvelope = true; }
|
||||
else if (!argv[i].startsWith('--')) files.push(argv[i]);
|
||||
}
|
||||
if (!files.length) { console.error('usage: parse-run.mjs <run.jsonl> [run.t2.jsonl ...] [--envelope] [--answer <glob>]... | --selftest'); process.exit(1); }
|
||||
const s = parseSession(files);
|
||||
|
||||
console.log(`\n=== ${files.map((f) => f.split('/').pop()).join(' + ')} ===`);
|
||||
@@ -481,4 +567,8 @@ if (isMain) {
|
||||
}
|
||||
console.log('');
|
||||
console.log(formatOccupancy(s));
|
||||
if (wantEnvelope) {
|
||||
console.log('');
|
||||
console.log(formatEnvelope(s.exploreTexts, answerGlobs));
|
||||
}
|
||||
}
|
||||
|
||||
Executable
+296
@@ -0,0 +1,296 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Deterministic per-file budget-share probe for `codegraph_explore` (CG-6).
|
||||
*
|
||||
* `probe-explore.mjs` prints what explore returned. This prints how the response
|
||||
* was DIVIDED — which files won the byte envelope and in what proportion — and
|
||||
* checks that division against a declared expectation. It is the regression gate
|
||||
* for GitHub issue #1500 / epic CG-1: an architecture question that doesn't name
|
||||
* the exact use-case must concentrate the budget on the code that answers it, not
|
||||
* on a generated CRUD layer (or an eval script) that merely name-collides.
|
||||
*
|
||||
* The numbers come from the CG-4 diagnostic (`CODEGRAPH_EXPLORE_DEBUG`), read back
|
||||
* from a JSONL sidecar, so the probe measures the shipping allocator rather than
|
||||
* re-deriving shares from the markdown.
|
||||
*
|
||||
* Fixtures are declared in `allocation-fixtures.json`. A `kind: "fixture"` entry is
|
||||
* hermetic — the fixture tree is copied to a fresh temp dir and indexed per run, so
|
||||
* two runs on one build give identical numbers. A `kind: "self"` entry reads this
|
||||
* repo's live index and therefore moves as the repo changes; its assertions are
|
||||
* relative for that reason.
|
||||
*
|
||||
* Usage (needs a current `npm run build`):
|
||||
* node scripts/agent-eval/probe-allocation.mjs # every fixture
|
||||
* node scripts/agent-eval/probe-allocation.mjs payroll-go # one fixture
|
||||
* node scripts/agent-eval/probe-allocation.mjs --json # machine-readable
|
||||
* node scripts/agent-eval/probe-allocation.mjs --keep # keep the temp index
|
||||
*
|
||||
* Exit code: 0 if every assertion holds, 1 if any fails, 2 on a setup error.
|
||||
* BOTH FIXTURES ARE EXPECTED TO FAIL until CG-10/CG-12 land — that failure is the
|
||||
* documented bug. Use --expect-fail to invert the exit code while it is the state
|
||||
* of the world (0 = still broken, 1 = fixed, go flip the gate).
|
||||
*/
|
||||
import { cpSync, mkdtempSync, mkdirSync, readFileSync, rmSync, existsSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { dirname, join, resolve } from 'node:path';
|
||||
import { fileURLToPath, pathToFileURL } from 'node:url';
|
||||
|
||||
const HERE = dirname(fileURLToPath(import.meta.url));
|
||||
const REPO_ROOT = resolve(HERE, '../..');
|
||||
const SPEC_PATH = join(HERE, 'allocation-fixtures.json');
|
||||
|
||||
const argv = process.argv.slice(2);
|
||||
const flags = new Set(argv.filter((a) => a.startsWith('--')));
|
||||
const wanted = argv.filter((a) => !a.startsWith('--'));
|
||||
const asJson = flags.has('--json');
|
||||
const keepTemp = flags.has('--keep');
|
||||
const expectFail = flags.has('--expect-fail');
|
||||
|
||||
const say = (line = '') => { if (!asJson) console.log(line); };
|
||||
const pct = (f) => `${(f * 100).toFixed(1)}%`;
|
||||
const num = (n) => Math.round(n).toLocaleString('en-US');
|
||||
|
||||
/** Load the built dist — the probe measures the shipping allocator, not src. */
|
||||
async function loadDist() {
|
||||
const distIndex = join(REPO_ROOT, 'dist/index.js');
|
||||
if (!existsSync(distIndex)) {
|
||||
console.error('dist/ not built — run `npm run build` first.');
|
||||
process.exit(2);
|
||||
}
|
||||
const idx = await import(pathToFileURL(distIndex).href);
|
||||
const tools = await import(pathToFileURL(join(REPO_ROOT, 'dist/mcp/tools.js')).href);
|
||||
// esModuleInterop: dynamic import of CJS yields { default: module.exports, ...named }
|
||||
const CodeGraph = idx.default?.default ?? idx.default ?? idx.CodeGraph;
|
||||
const ToolHandler = tools.ToolHandler ?? tools.default?.ToolHandler;
|
||||
if (typeof CodeGraph?.openSync !== 'function' || typeof ToolHandler !== 'function') {
|
||||
console.error('could not resolve CodeGraph/ToolHandler from dist/');
|
||||
process.exit(2);
|
||||
}
|
||||
return { CodeGraph, ToolHandler };
|
||||
}
|
||||
|
||||
/** `internal/gen/**` → /^internal\/gen\/.*$/ . Supports `**`, `*` and literals. */
|
||||
function globToRegExp(glob) {
|
||||
// Park `**` on a sentinel no path can contain, so the `*` pass cannot eat it.
|
||||
// Written as an escape, not a literal byte — a raw NUL makes git treat this
|
||||
// whole script as binary, which costs every future diff of it.
|
||||
const DOUBLE_STAR = '\u0000';
|
||||
const escaped = glob.replace(/[.+^${}()|[\]\\]/g, '\\$&');
|
||||
const body = escaped
|
||||
.replace(/\*\*/g, DOUBLE_STAR)
|
||||
.replace(/\*/g, '[^/]*')
|
||||
.replaceAll(DOUBLE_STAR, '.*');
|
||||
return new RegExp(`^${body}$`);
|
||||
}
|
||||
|
||||
const groupOf = (path, groups) => {
|
||||
for (const [name, globs] of Object.entries(groups)) {
|
||||
if (globs.some((g) => globToRegExp(g).test(path))) return name;
|
||||
}
|
||||
return 'other';
|
||||
};
|
||||
|
||||
/**
|
||||
* Run one explore call with the diagnostic pointed at a sidecar, and return the
|
||||
* report plus the response text.
|
||||
*/
|
||||
async function runExplore({ CodeGraph, ToolHandler }, repoPath, query, sidecar) {
|
||||
const cg = CodeGraph.openSync(repoPath);
|
||||
const prior = process.env.CODEGRAPH_EXPLORE_DEBUG;
|
||||
process.env.CODEGRAPH_EXPLORE_DEBUG = sidecar;
|
||||
try {
|
||||
const res = await new ToolHandler(cg).execute('codegraph_explore', { query });
|
||||
const text = res.content?.[0]?.text ?? '';
|
||||
const lines = readFileSync(sidecar, 'utf-8').trim().split('\n').filter(Boolean);
|
||||
if (lines.length === 0) throw new Error('diagnostic produced no report');
|
||||
return { report: JSON.parse(lines[lines.length - 1]), text };
|
||||
} finally {
|
||||
if (prior === undefined) delete process.env.CODEGRAPH_EXPLORE_DEBUG;
|
||||
else process.env.CODEGRAPH_EXPLORE_DEBUG = prior;
|
||||
try { cg.close?.(); } catch {}
|
||||
}
|
||||
}
|
||||
|
||||
/** Copy a fixture tree to a fresh temp dir and index it — hermetic per run. */
|
||||
async function materializeFixture({ CodeGraph }, fixturePath) {
|
||||
const src = resolve(REPO_ROOT, fixturePath);
|
||||
if (!existsSync(src)) throw new Error(`fixture tree not found: ${src}`);
|
||||
const dir = mkdtempSync(join(tmpdir(), 'cg-alloc-'));
|
||||
cpSync(src, dir, { recursive: true });
|
||||
// A stray index inside the checked-in tree would be copied in and reused.
|
||||
rmSync(join(dir, '.codegraph'), { recursive: true, force: true });
|
||||
const cg = CodeGraph.initSync(dir);
|
||||
await cg.indexAll();
|
||||
cg.close?.();
|
||||
return dir;
|
||||
}
|
||||
|
||||
/** Evaluate one fixture's assertions against its report. Returns check rows. */
|
||||
function evaluate(fixture, report, text) {
|
||||
const { groups, assert: want } = fixture;
|
||||
const delivered = new Map();
|
||||
const allocated = new Map();
|
||||
for (const f of report.files) {
|
||||
const g = groupOf(f.path, groups);
|
||||
delivered.set(g, (delivered.get(g) ?? 0) + f.share);
|
||||
allocated.set(g, (allocated.get(g) ?? 0) + f.allocatedShare);
|
||||
}
|
||||
const share = (g) => delivered.get(g) ?? 0;
|
||||
const top = report.files
|
||||
.filter((f) => f.finalChars > 0)
|
||||
.sort((a, b) => b.finalChars - a.finalChars)[0];
|
||||
|
||||
const checks = [];
|
||||
const add = (name, pass, detail) => checks.push({ name, pass, detail });
|
||||
|
||||
if (want.answerShareAtLeast !== undefined) {
|
||||
add(
|
||||
`answer group takes >= ${pct(want.answerShareAtLeast)} of the envelope`,
|
||||
share('answer') >= want.answerShareAtLeast,
|
||||
`answer ${pct(share('answer'))} delivered (${pct(allocated.get('answer') ?? 0)} allocated)`,
|
||||
);
|
||||
}
|
||||
if (want.incidentalShareAtMost !== undefined) {
|
||||
add(
|
||||
`incidental group takes <= ${pct(want.incidentalShareAtMost)} of the envelope`,
|
||||
share('incidental') <= want.incidentalShareAtMost,
|
||||
`incidental ${pct(share('incidental'))} delivered (${pct(allocated.get('incidental') ?? 0)} allocated)`,
|
||||
);
|
||||
}
|
||||
if (want.topFileGroup) {
|
||||
const actual = top ? groupOf(top.path, groups) : '(nothing delivered)';
|
||||
add(
|
||||
`largest delivered file is in "${want.topFileGroup}"`,
|
||||
actual === want.topFileGroup,
|
||||
top ? `${top.path} (${pct(top.share)}, group "${actual}")` : 'no file delivered any source',
|
||||
);
|
||||
}
|
||||
for (const path of want.mustDeliverBytes ?? []) {
|
||||
const rec = report.files.find((f) => f.path === path);
|
||||
add(
|
||||
`${path} delivers source`,
|
||||
!!rec && rec.finalChars > 0,
|
||||
rec
|
||||
? `${num(rec.finalChars)} delivered of ${num(rec.emittedChars)} allocated` +
|
||||
(rec.finalChars === 0 && rec.emittedChars > 0 ? ' — hard ceiling dropped the whole section' : '') +
|
||||
(rec.emittedChars === 0 ? ` — never rendered (${rec.skipped ?? 'not reached'}, rank #${rec.rank})` : '')
|
||||
: 'not among the ranked candidates',
|
||||
);
|
||||
}
|
||||
for (const needle of want.mustContain ?? []) {
|
||||
add(`response contains "${needle}"`, text.includes(needle), text.includes(needle) ? 'present' : 'absent');
|
||||
}
|
||||
return { checks, delivered, allocated, top };
|
||||
}
|
||||
|
||||
function printReport(fixture, report, evaluated) {
|
||||
const { checks, delivered, allocated } = evaluated;
|
||||
const env = report.envelope;
|
||||
say('');
|
||||
say(`── ${fixture.id} — ${fixture.title}`);
|
||||
say(` query "${report.query}"`);
|
||||
say(` project ${report.projectRoot} · ${num(report.indexedFileCount)} files indexed`);
|
||||
say(
|
||||
` envelope ${num(env.chars)} delivered · ${num(env.allocatedChars)} allocated` +
|
||||
` of ${num(report.budget.maxOutputChars)} budget (hard ceiling ${num(report.budget.hardCeiling)})` +
|
||||
`${env.overBudget ? ' [over budget]' : ''}${env.truncated ? ' [TRUNCATED]' : ''}`,
|
||||
);
|
||||
say('');
|
||||
say(' group alloc% deliv%');
|
||||
for (const g of ['answer', 'incidental', 'other']) {
|
||||
if (!delivered.has(g) && !allocated.has(g)) continue;
|
||||
say(` ${g.padEnd(12)} ${pct(allocated.get(g) ?? 0).padStart(6)} ${pct(delivered.get(g) ?? 0).padStart(6)}`);
|
||||
}
|
||||
say('');
|
||||
say(' # alloc% deliv% bytes score graph hits gen render file');
|
||||
for (const f of report.files.filter((f) => f.emittedChars > 0 || f.finalChars > 0)) {
|
||||
say(
|
||||
' ' + String(f.rank).padStart(2) + ' ' +
|
||||
pct(f.allocatedShare).padStart(6) + ' ' +
|
||||
pct(f.share).padStart(6) + ' ' +
|
||||
num(f.emittedChars).padStart(7) + ' ' +
|
||||
String(f.score).padStart(5) + ' ' +
|
||||
f.graphScore.toFixed(5).padStart(7) + ' ' +
|
||||
String(f.termHits).padStart(4) + ' ' +
|
||||
(f.generated ? ' ✓ ' : ' ') + ' ' +
|
||||
((f.render ?? '-') + (f.clipped ? '*' : '')).padEnd(9) + ' ' +
|
||||
f.path,
|
||||
);
|
||||
}
|
||||
say('');
|
||||
for (const c of checks) say(` ${c.pass ? 'PASS' : 'FAIL'} ${c.name}\n ${c.detail}`);
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const spec = JSON.parse(readFileSync(SPEC_PATH, 'utf-8'));
|
||||
const fixtures = spec.fixtures.filter((f) => wanted.length === 0 || wanted.includes(f.id));
|
||||
if (fixtures.length === 0) {
|
||||
console.error(`no fixture matched ${JSON.stringify(wanted)}; known: ${spec.fixtures.map((f) => f.id).join(', ')}`);
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
const dist = await loadDist();
|
||||
const sidecarDir = mkdtempSync(join(tmpdir(), 'cg-alloc-diag-'));
|
||||
const results = [];
|
||||
const temps = [];
|
||||
|
||||
for (const fixture of fixtures) {
|
||||
let repoPath;
|
||||
if (fixture.kind === 'fixture') {
|
||||
repoPath = await materializeFixture(dist, fixture.path);
|
||||
temps.push(repoPath);
|
||||
} else {
|
||||
repoPath = resolve(REPO_ROOT, fixture.path);
|
||||
if (!existsSync(join(repoPath, '.codegraph'))) {
|
||||
console.error(`${fixture.id}: ${repoPath} has no .codegraph index — run \`codegraph init\` there first.`);
|
||||
process.exit(2);
|
||||
}
|
||||
}
|
||||
|
||||
const sidecar = join(sidecarDir, `${fixture.id}.jsonl`);
|
||||
mkdirSync(dirname(sidecar), { recursive: true });
|
||||
const { report, text } = await runExplore(dist, repoPath, fixture.query, sidecar);
|
||||
const evaluated = evaluate(fixture, report, text);
|
||||
printReport(fixture, report, evaluated);
|
||||
results.push({
|
||||
id: fixture.id,
|
||||
kind: fixture.kind,
|
||||
query: fixture.query,
|
||||
passed: evaluated.checks.every((c) => c.pass),
|
||||
checks: evaluated.checks,
|
||||
shares: {
|
||||
delivered: Object.fromEntries(evaluated.delivered),
|
||||
allocated: Object.fromEntries(evaluated.allocated),
|
||||
},
|
||||
envelope: report.envelope,
|
||||
files: report.files.filter((f) => f.emittedChars > 0 || f.finalChars > 0),
|
||||
});
|
||||
}
|
||||
|
||||
if (!keepTemp) {
|
||||
for (const dir of temps) rmSync(dir, { recursive: true, force: true });
|
||||
rmSync(sidecarDir, { recursive: true, force: true });
|
||||
} else {
|
||||
say('');
|
||||
say(` kept: ${[...temps, sidecarDir].join(' ')}`);
|
||||
}
|
||||
|
||||
const allPassed = results.every((r) => r.passed);
|
||||
if (asJson) {
|
||||
console.log(JSON.stringify({ passed: allPassed, fixtures: results }, null, 2));
|
||||
} else {
|
||||
say('');
|
||||
for (const r of results) say(`${r.passed ? 'PASS' : 'FAIL'} ${r.id}`);
|
||||
if (!allPassed) {
|
||||
say('');
|
||||
say('Failures here are the DOCUMENTED #1500 bug — the budget goes to files that merely');
|
||||
say('name-collide with the query. They become the pass gate once CG-10/CG-12 land.');
|
||||
}
|
||||
}
|
||||
process.exit(expectFail ? (allPassed ? 1 : 0) : (allPassed ? 0 : 1));
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error(err?.stack ?? String(err));
|
||||
process.exit(2);
|
||||
});
|
||||
Reference in New Issue
Block a user