feat(mcp): pare default tool surface to codegraph_explore alone + redux-thunk synthesizer
This commit is contained in:
Colby McHenry
@@ -0,0 +1,108 @@
|
||||
#!/usr/bin/env node
|
||||
// Effort A/B — does CODEGRAPH_OFFLOAD_EFFORT=high improve offload SYNTHESIS FIDELITY vs low?
|
||||
// Probe-based (no agent): for each repo × effort × rep, run codegraph_explore with the offload
|
||||
// ON on the canonical question, capture the synthesized answer + AI tokens/cost/latency, then
|
||||
// Sonnet-judge that answer's fidelity vs source-verified ground truth. Isolates the synthesis
|
||||
// from agent/adoption noise. Requires `codegraph login` (managed offload) + indexed repos.
|
||||
//
|
||||
// Env: REPS (default 3) · CG_ENGINE (engine repo) · AGENT_EVAL_OUT (repos under /repos) · CONC (judge concurrency)
|
||||
import { pathToFileURL, fileURLToPath } from 'node:url';
|
||||
import { resolve, dirname, join } from 'node:path';
|
||||
import { readFileSync, writeFileSync, existsSync, rmSync } from 'node:fs';
|
||||
import { execFile } from 'node:child_process';
|
||||
import { tmpdir } from 'node:os';
|
||||
|
||||
const HERE = dirname(fileURLToPath(import.meta.url));
|
||||
const ENGINE = process.env.CG_ENGINE || resolve(HERE, '..', '..');
|
||||
const OUT = process.env.AGENT_EVAL_OUT || '/tmp/cg-offload-eval';
|
||||
const REPOS = join(OUT, 'repos');
|
||||
const GT = JSON.parse(readFileSync(resolve(HERE, 'offload-eval-ground-truth.json'), 'utf8'));
|
||||
const REPS = Number(process.env.REPS || 3);
|
||||
const CONC = Number(process.env.CONC || 4);
|
||||
const EFFORTS = (process.env.EFFORTS_FILTER || 'low,high').split(',');
|
||||
const ONLY = process.env.REPOS_FILTER ? new Set(process.env.REPOS_FILTER.split(',')) : null;
|
||||
const TIER = { mtkruto: 'small', postybirb: 'medium', shapeshift: 'complex', trezor: 'large' };
|
||||
|
||||
const load = async (rel) => import(pathToFileURL(resolve(ENGINE, rel)).href);
|
||||
const idx = await load('dist/index.js');
|
||||
const toolsMod = await load('dist/mcp/tools.js');
|
||||
const CodeGraph = idx.default?.default ?? idx.default ?? idx.CodeGraph;
|
||||
const ToolHandler = toolsMod.ToolHandler ?? toolsMod.default?.ToolHandler;
|
||||
if (typeof CodeGraph?.openSync !== 'function' || typeof ToolHandler !== 'function') {
|
||||
console.error('could not load engine from', ENGINE); process.exit(2);
|
||||
}
|
||||
|
||||
const fidPrompt = (gt, ans) => `You are scoring the FIDELITY of a machine-synthesized code-exploration answer against verified ground truth. Do NOT use any tools.
|
||||
|
||||
QUESTION: ${gt.question}
|
||||
|
||||
VERIFIED GROUND TRUTH (the actual call path + files):
|
||||
${gt.truth}
|
||||
|
||||
SYNTHESIZED ANSWER (to score):
|
||||
${ans || '(empty)'}
|
||||
|
||||
Judge: (1) is the traced call path correct vs ground truth? (2) are the cited files/symbols correct (not fabricated)? (3) if it gave a "Coverage:" verdict, was it honest? A confident WRONG trace is the worst outcome — penalize it harder than an honest partial.
|
||||
Output ONLY minified JSON: {"verdict":"pass|partial|fail","score":<0-100>,"fabrication":<true|false>,"coverageHonest":<true|false>,"note":"<=20 words"}`;
|
||||
|
||||
const askJudge = (prompt) => new Promise((res) => {
|
||||
execFile('claude', ['-p', prompt, '--model', 'sonnet', '--effort', 'high', '--max-budget-usd', '0.5',
|
||||
'--strict-mcp-config', '--mcp-config', '{"mcpServers":{}}'],
|
||||
{ cwd: OUT, maxBuffer: 1 << 24, timeout: 120000 }, (err, stdout) => {
|
||||
const m = (stdout || '').match(/\{[\s\S]*\}/);
|
||||
if (!m) return res({ verdict: 'error', score: null, note: (err ? err.message : 'no json').slice(0, 60) });
|
||||
try { res(JSON.parse(m[0])); } catch { res({ verdict: 'error', score: null }); }
|
||||
});
|
||||
});
|
||||
|
||||
// ---- 1. Probe: collect synthesized answers at each effort -------------------
|
||||
const records = [];
|
||||
for (const repo of Object.keys(GT)) {
|
||||
if (ONLY && !ONLY.has(repo)) continue;
|
||||
const dir = join(REPOS, repo);
|
||||
if (!existsSync(join(dir, '.codegraph'))) { console.error('skip (not indexed):', repo); continue; }
|
||||
const cg = CodeGraph.openSync(dir);
|
||||
const h = new ToolHandler(cg);
|
||||
for (const effort of EFFORTS) {
|
||||
for (let rep = 1; rep <= REPS; rep++) {
|
||||
process.env.CODEGRAPH_OFFLOAD_EFFORT = effort;
|
||||
const usageLog = join(tmpdir(), `effort-${repo}-${effort}-${rep}.jsonl`);
|
||||
try { rmSync(usageLog); } catch { /* none */ }
|
||||
process.env.CODEGRAPH_OFFLOAD_USAGE_LOG = usageLog;
|
||||
let answer = '';
|
||||
try { answer = (await h.execute('codegraph_explore', { query: GT[repo].question }))?.content?.[0]?.text ?? ''; }
|
||||
catch (e) { console.error(` ${repo}/${effort}#${rep} explore failed: ${e?.message}`); }
|
||||
const fired = /Synthesized by CodeGraph/.test(answer);
|
||||
const ai = { tokens: 0, cost: 0, ms: 0 };
|
||||
if (existsSync(usageLog)) for (const e of readFileSync(usageLog, 'utf8').split('\n').filter(Boolean).map(JSON.parse)) {
|
||||
ai.tokens += e.totalTokens || 0; ai.cost += e.costUsd || 0; ai.ms += e.ms || 0;
|
||||
}
|
||||
records.push({ repo, tier: TIER[repo], effort, rep, fired, ai, answer });
|
||||
console.error(` ${repo}/${effort}#${rep}: fired=${fired} ${ai.tokens}tok $${ai.cost.toFixed(4)} ${ai.ms}ms`);
|
||||
}
|
||||
}
|
||||
try { cg.close?.(); } catch { /* none */ }
|
||||
}
|
||||
|
||||
// ---- 2. Judge fidelity (concurrency) ---------------------------------------
|
||||
console.error(`\njudging ${records.length} answers (concurrency ${CONC})...`);
|
||||
let done = 0;
|
||||
const q = [...records];
|
||||
async function worker() { while (q.length) { const r = q.shift(); r.fid = await askJudge(fidPrompt(GT[r.repo], r.answer)); console.error(` [${++done}/${records.length}] ${r.repo}/${r.effort}#${r.rep}: ${r.fid.verdict} ${r.fid.score ?? ''}`); } }
|
||||
await Promise.all(Array.from({ length: CONC }, worker));
|
||||
writeFileSync(join(OUT, 'effort-results.jsonl'), records.map((r) => JSON.stringify(r)).join('\n') + '\n');
|
||||
|
||||
// ---- 3. Aggregate: low vs high per repo ------------------------------------
|
||||
const med = (a) => { a = a.filter((x) => x != null).sort((x, y) => x - y); return a.length ? (a.length % 2 ? a[(a.length - 1) / 2] : (a[a.length / 2 - 1] + a[a.length / 2]) / 2) : null; };
|
||||
console.log(`\n${'='.repeat(80)}\nEFFORT A/B — offload synthesis fidelity (probe, n=${REPS}/cell)\n${'='.repeat(80)}`);
|
||||
console.log(`${'repo'.padEnd(11)} ${'tier'.padEnd(8)} ${'effort'.padEnd(6)} fired ${'fid(med)'.padStart(8)} ${'fab%'.padStart(5)} ${'AItok'.padStart(7)} ${'AIcost'.padStart(8)} ${'ms(med)'.padStart(8)}`);
|
||||
for (const repo of Object.keys(GT)) {
|
||||
for (const effort of EFFORTS) {
|
||||
const rs = records.filter((r) => r.repo === repo && r.effort === effort);
|
||||
if (!rs.length) continue;
|
||||
const fids = rs.map((r) => r.fid?.score).filter((x) => x != null);
|
||||
const fab = rs.filter((r) => r.fid?.fabrication === true).length;
|
||||
console.log(`${repo.padEnd(11)} ${TIER[repo].padEnd(8)} ${effort.padEnd(6)} ${rs.filter((r) => r.fired).length}/${rs.length} ${String(med(fids) ?? '—').padStart(8)} ${String(Math.round(100 * fab / rs.length) + '%').padStart(5)} ${String(Math.round(med(rs.map((r) => r.ai.tokens)) / 1000) + 'k').padStart(7)} ${('$' + (med(rs.map((r) => r.ai.cost)) ?? 0).toFixed(4)).padStart(8)} ${String(med(rs.map((r) => r.ai.ms)) ?? '—').padStart(8)}`);
|
||||
}
|
||||
}
|
||||
console.log('');
|
||||
@@ -43,7 +43,11 @@ for (const line of lines) {
|
||||
const text = Array.isArray(b.content)
|
||||
? b.content.map(c => (typeof c === 'string' ? c : c.text || '')).join('')
|
||||
: (typeof b.content === 'string' ? b.content : '');
|
||||
if (/Synthesized by CodeGraph/.test(text)) { offloadAnswers.push(text); exploreResults++; }
|
||||
// An offload answer is either the 'plain'/'report' synthesis (carries the
|
||||
// "Synthesized by CodeGraph" footer) or a 'refs' answer (carries the re-expanded
|
||||
// "### Referenced source — verbatim" appendix). A refs call that cited nothing
|
||||
// valid falls back to RAW source, which is correctly counted as a raw explore below.
|
||||
if (/Synthesized by CodeGraph|### Referenced source — verbatim/.test(text)) { offloadAnswers.push(text); exploreResults++; }
|
||||
else if (/Found \d+ symbols? across|## Exploration:/.test(text)) exploreResults++;
|
||||
}
|
||||
}
|
||||
|
||||
Executable
+50
@@ -0,0 +1,50 @@
|
||||
#!/usr/bin/env bash
|
||||
# ONE offload run on ONE indexed repo at a given offload STYLE (plain|refs), so we can
|
||||
# watch a single agent transcript at a time (the user's one-run-at-a-time methodology).
|
||||
# The OFFLOAD reasoning runs in the prewarmed DAEMON process, so the style env must be
|
||||
# set on BOTH the daemon and the client MCP config. Writes one metrics line to RESULTS
|
||||
# and leaves the raw stream-json at $RUNS/<repo>-<style>-<n>.jsonl for inspection.
|
||||
#
|
||||
# Usage: offload-eval-refs1.sh <indexed-repo> <style> <n> "<question>"
|
||||
set -uo pipefail
|
||||
HERE="$(cd "$(dirname "$0")" && pwd)"; ENGINE="$(cd "$HERE/../.." && pwd)"; BIN="$ENGINE/dist/bin/codegraph.js"
|
||||
OUT="${AGENT_EVAL_OUT:-/tmp/cg-offload-eval}"; RUNS="$OUT/runs"; EXTRACT="$HERE/offload-eval-metrics.mjs"
|
||||
TARGET="${1:?repo}"; STYLE="${2:?style}"; N="${3:?run-tag}"; Q="${4:?question}"
|
||||
RESULTS="${RESULTS:-$OUT/results-refs.jsonl}"; REPO=$(basename "$TARGET"); TARGET=$(cd "$TARGET" && pwd -P)
|
||||
mkdir -p "$RUNS"; command -v claude >/dev/null || { echo "no claude"; exit 1; }
|
||||
USAGE="$RUNS/$REPO-$STYLE-usage.jsonl"; : > "$USAGE"
|
||||
CFG="$RUNS/mcp-$REPO-$STYLE.json"
|
||||
# `raw` is a pseudo-style: codegraph attached but the offload DISABLED (the ceiling —
|
||||
# verbatim source, no reasoning model). Any other value is an offload style (plain|refs).
|
||||
if [ "$STYLE" = "raw" ]; then
|
||||
DAEMON_ENV="CODEGRAPH_OFFLOAD_DISABLE=1"
|
||||
printf '{"mcpServers":{"codegraph":{"command":"env","args":["CODEGRAPH_WASM_RELAUNCHED=1","CODEGRAPH_OFFLOAD_DISABLE=1","node","%s","serve","--mcp","--path","%s"]}}}' \
|
||||
"$BIN" "$TARGET" > "$CFG"
|
||||
USAGE="-"
|
||||
else
|
||||
DAEMON_ENV="CODEGRAPH_OFFLOAD_STYLE=$STYLE CODEGRAPH_OFFLOAD_USAGE_LOG=$USAGE"
|
||||
printf '{"mcpServers":{"codegraph":{"command":"env","args":["CODEGRAPH_WASM_RELAUNCHED=1","CODEGRAPH_OFFLOAD_STYLE=%s","CODEGRAPH_OFFLOAD_USAGE_LOG=%s","node","%s","serve","--mcp","--path","%s"]}}}' \
|
||||
"$STYLE" "$USAGE" "$BIN" "$TARGET" > "$CFG"
|
||||
fi
|
||||
|
||||
# Prewarm a persistent daemon carrying the SAME offload config (it does the reasoning).
|
||||
pkill -9 -f "serve --mcp --path $TARGET" 2>/dev/null; rm -f "$TARGET/.codegraph/daemon.sock" 2>/dev/null; sleep 0.6
|
||||
env $DAEMON_ENV CODEGRAPH_DAEMON_IDLE_TIMEOUT_MS=1800000 \
|
||||
node "$BIN" serve --mcp --path "$TARGET" </dev/null >/dev/null 2>&1 &
|
||||
node -e 'const fs=require("fs");let n=0;const t=setInterval(()=>{if(fs.existsSync(process.argv[1]+"/.codegraph/daemon.sock")){clearInterval(t);process.exit(0)}if(n++>150){clearInterval(t);process.exit(1)}},100)' "$TARGET" \
|
||||
&& echo "daemon warm ($STYLE)" || echo "WARN daemon never bound"
|
||||
|
||||
tag="$REPO-$STYLE-$N"
|
||||
echo "== run $tag =="
|
||||
# DISALLOW (optional): block tools that confound the offload-sufficiency signal —
|
||||
# chiefly "Agent" (sub-agent delegation: the spawned Explore subagent has low MCP
|
||||
# salience, ignores codegraph, and thrashes via Bash+Read, making the A/B noise).
|
||||
( cd "$TARGET" && claude -p "$Q" --output-format stream-json --verbose --permission-mode bypassPermissions \
|
||||
--model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" --max-budget-usd 4 \
|
||||
${DISALLOW:+--disallowedTools "$DISALLOW"} \
|
||||
--strict-mcp-config --mcp-config "$CFG" </dev/null > "$RUNS/$tag.jsonl" 2>"$RUNS/$tag.err" )
|
||||
node "$EXTRACT" --run "$RUNS/$tag.jsonl" --usage "$USAGE" --arm "offload-$STYLE" --rep "$N" \
|
||||
--repo "$REPO" --tier "complex" --q "$Q" >> "$RESULTS"
|
||||
node -e 'const o=JSON.parse(require("fs").readFileSync(process.argv[1],"utf8").trim().split("\n").pop());console.log(` [${o.arm} #${o.rep}] ${o.durationSec}s | main $${o.costUsdMain} ${o.tokBillable} tok | read=${o.read} grep=${o.grep} explore=${o.explore} offload=${o.offloadFired} | AI ${o.ai.calls}call/${o.ai.totalTokens}tok/$${o.ai.costUsd.toFixed(4)} | ok=${o.ok}`)' "$RESULTS"
|
||||
pkill -9 -f "serve --mcp --path $TARGET" 2>/dev/null; rm -f "$TARGET/.codegraph/daemon.sock" 2>/dev/null
|
||||
echo "raw transcript: $RUNS/$tag.jsonl"
|
||||
Reference in New Issue
Block a user