test(agent-eval): add output-style A/B harness, cost/token analyzer, and DISALLOW/REP_START controls
Three additions to tighten the eval loop: - offload-eval-styles.sh: new 4-arm eval (raw/refs/map/src) isolating the Worker's output shape's effect on main-session tokens, latency, and accuracy. Delegation blocked by default (DISALLOW=Agent) so variance from Haiku subagent spawning doesn't contaminate the measurement. - offload-eval-cost.mjs: cost/token analyzer that reads Claude Code's own per-model accounting (modelUsage.costUSD) rather than re-deriving from raw token counts, giving a correct main(Sonnet)/sub(Haiku) split with proper per-tier pricing. - offload-eval-3arm.sh: adds DISALLOW env to block sub-agent delegation across all arms, and REP_START to append reps to an existing run without clobbering earlier jsonls (e.g. REP_START=4 REPS=3 → reps 4,5,6). Also adds CODEGRAPH_OFFLOAD_STYLE forwarding to the managed gateway so the styles eval can drive output shape end-to-end; the field is stripped before the upstream model call and never sent to BYO endpoints.
This commit is contained in:
@@ -200,6 +200,9 @@ export async function synthesizeOffload({ query, context }: SynthArgs): Promise<
|
||||
const url = cfg.url.replace(/\/+$/, '') + '/chat/completions';
|
||||
const { system, footer } = promptFor(cfg.style);
|
||||
const ctx = cfg.strip ? stripAgentDirectives(context) : context;
|
||||
// Optional operator/eval flag forwarded verbatim to the managed Worker (see body below);
|
||||
// the Worker validates it and falls back to its default for anything it doesn't recognize.
|
||||
const workerStyle = (process.env.CODEGRAPH_OFFLOAD_STYLE || '').trim();
|
||||
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(() => controller.abort(), cfg.timeoutMs);
|
||||
@@ -217,6 +220,10 @@ export async function synthesizeOffload({ query, context }: SynthArgs): Promise<
|
||||
max_tokens: cfg.maxTokens,
|
||||
temperature: 0.2,
|
||||
reasoning_effort: cfg.effort,
|
||||
// Optional managed-tier flag, forwarded ONLY to the managed gateway (which strips it
|
||||
// before the upstream model call) and ONLY when an operator/eval sets it — so BYO
|
||||
// endpoints, which may reject unknown fields, never see it.
|
||||
...(cfg.managed && workerStyle ? { offload_style: workerStyle } : {}),
|
||||
messages: [
|
||||
{ role: 'system', content: system },
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user