fix(extraction): land upstream declaration initializer walks (#1511) (#1802)

Squash danusha2345's PR #1511 at d282f9e8 onto main 8c9c4761,
preserving its nine non-merge commits and main's existing Unreleased notes.
Calls in Kotlin, Java, TS/JS, Scala, Rust and Python declaration initializers
now retain the owner established by the upstream regression expectations.
Include the upstream CFML, dynamic-dispatch summary and viewer follow-ups.

Linux fail-to-pass validation (Node 22.19.0, rebuilt dist and native kernel):
- Before: TS load belonged to file:app.ts; Python/Kotlin/Scala/Rust calls
  vanished; Java lost the field-lambda, anonymous override and eager calls.
- After: all six languages PASS; 12 native/WASM LF/CRLF parity checks PASS.
- Focused initializer regressions: 10 passed with CODEGRAPH_KERNEL=0 and
  10 passed with the kernel enabled; Kotlin's grammar fallback is recorded.
- Related regression suites: 879 passed, 1 skipped across 15 test files.
- Evidence: /workspace/cg-1510-repro/before and /workspace/cg-1510-repro/after
  (combined test output: after/vitest.log).

Fixes #1510
Supersedes #1511

Co-authored-by: Colby McHenry <colbymchenry@users.noreply.github.com>
Co-authored-by: danusha2345 <ewidusoc498@gmail.com>
This commit is contained in:
Colby Mchenry
2026-09-08 17:33:45 -05:00
committed by GitHub
co-authored by Colby McHenry danusha2345
parent 8c9c4761b0
commit 9181dd1ef3
25 changed files with 884 additions and 108 deletions
+25 -6
View File
@@ -2204,6 +2204,12 @@ export class QueryBuilder {
* build script does its work on the way down the file. `instantiates` counts
* the same way — `new Server(...)` at module scope is the same act.
*
* A call made while initializing a module-level `variable` / `constant` —
* `const service = new Service()`, `app = FastAPI()` — is attributed to the
* declared name (#693), not to the file, so the file's own edges alone would
* miss most of what a real entry point runs. Those names are the file's
* top-level code too, so `tops` counts them alongside the file node.
*
* Ranking multiplies the two things an entry point does: it runs (calls), and
* it wires the project together (distinct other files its symbols reach). One
* alone is misleading — a registration table makes hundreds of module-level
@@ -2216,12 +2222,25 @@ export class QueryBuilder {
if (limit <= 0) return [];
return this.db
.prepare(
`WITH runs AS (
SELECT e.source AS id, COUNT(*) AS calls
FROM edges e
JOIN nodes n ON n.id = e.source
WHERE n.kind = 'file' AND e.kind IN ('calls', 'instantiates')
GROUP BY e.source
`WITH tops AS (
SELECT n.id AS file_id, n.id AS src
FROM nodes n
WHERE n.kind = 'file'
UNION ALL
SELECT c.source AS file_id, c.target AS src
FROM edges c
JOIN nodes f ON f.id = c.source
JOIN nodes v ON v.id = c.target
WHERE c.kind = 'contains'
AND f.kind = 'file'
AND v.kind IN ('variable', 'constant')
),
runs AS (
SELECT t.file_id AS id, COUNT(*) AS calls
FROM tops t
JOIN edges e ON e.source = t.src
WHERE e.kind IN ('calls', 'instantiates')
GROUP BY t.file_id
),
cand AS (
SELECT r.id AS id, n.file_path AS fp, r.calls AS calls
+15 -1
View File
@@ -356,6 +356,13 @@ export class CfmlExtractor {
.filter((e) => e.kind === 'contains' && e.source === innerFileNodeId)
.map((e) => e.target)
);
// Snippet-top-level non-callables: `var x = …` locals of the enclosing
// function that the fragment-as-module parse mints as declarations.
const localVarIds = new Set(
result.nodes
.filter((n) => topLevelIds.has(n.id) && (n.kind === 'variable' || n.kind === 'constant'))
.map((n) => n.id)
);
for (const node of result.nodes) {
if (node.kind === 'file') continue;
node.startLine += startLine;
@@ -385,7 +392,14 @@ export class CfmlExtractor {
// top-level script in a .cfm template, or any statement directly in
// the snippet body) attribute to the filtered-out snippet file node by
// default — redirect those (and any genuinely unset ones) to parentId.
if ((!ref.fromNodeId || ref.fromNodeId === innerFileNodeId) && parentId) ref.fromNodeId = parentId;
// Same for a snippet-top-level `var x = helper()`: the inner extractor
// parses the fragment as a whole module, so it mints a variable node and
// attributes the initializer's calls to it — but this fragment is a
// FUNCTION BODY, so `x` is a local and `helper` is the enclosing
// function's callee. Snippet-top-level FUNCTIONS keep their own calls.
if ((!ref.fromNodeId || ref.fromNodeId === innerFileNodeId || localVarIds.has(ref.fromNodeId)) && parentId) {
ref.fromNodeId = parentId;
}
this.unresolvedReferences.push(ref);
}
for (const error of result.errors) {
+140 -25
View File
@@ -42,6 +42,99 @@ function extractKotlinReturnType(node: SyntaxNode, source: string): string | und
return undefined;
}
/**
* A property's CODE children: the named child right after the `=` token, a
* `property_delegate` (`by lazy { … }`), and an accessor the grammar nested
* under the declaration (`val x: Int get() = compute()` written on ONE line;
* an accessor on its own line parses as a SIBLING of the property and is not
* reachable from here). What stays unwalked is the declaration itself
* modifiers, the `val`/`var` keyword, the name+type, and an extension
* receiver's type and type parameters. (Go's #693 fix walks the `value` field
* for the same reason; tree-sitter-kotlin exposes no fields at all, hence the
* `=` anchor.)
*/
function kotlinPropertyInitializers(node: SyntaxNode): SyntaxNode[] {
const out: SyntaxNode[] = [];
let afterEq = false;
for (let i = 0; i < node.childCount; i++) {
const c = node.child(i);
if (!c) continue;
if (!c.isNamed) {
if (c.type === '=') afterEq = true;
continue;
}
if (afterEq) {
out.push(c);
afterEq = false;
} else if (c.type === 'property_delegate' || c.type === 'getter' || c.type === 'setter') {
out.push(c);
}
}
return out;
}
/**
* A property's node kind, or null when the declaration mints no node at all:
* destructuring (`val (a, b) = …`), an unreadable name, or a local (one inside
* a function body / `init` block / lambda / accessor). Kind by enclosing scope:
* a singleton `object` / `companion object` and a top-level property holds
* *shared* values, so `val``constant` and `var``variable` (the Scala-object
* rule; a `const val` is just a val). A `class`/`interface`/`enum` instance
* `val`/`var` is per-instance state `field` (never a value-ref target, like a
* Java instance `final`).
*/
function kotlinPropertyKind(
node: SyntaxNode,
source: string
): 'field' | 'constant' | 'variable' | null {
const varDecl = node.namedChildren.find((c) => c.type === 'variable_declaration');
const nameNode = varDecl?.namedChildren.find((c) => c.type === 'simple_identifier');
if (!nameNode || !getNodeText(nameNode, source)) return null;
let scope: 'local' | 'const' | 'instance' = 'const';
for (let p = node.parent; p; p = p.parent) {
const pt = p.type;
if (
pt === 'function_body' || pt === 'function_declaration' ||
pt === 'lambda_literal' || pt === 'anonymous_initializer' ||
pt === 'control_structure_body' || pt === 'getter' || pt === 'setter'
) { scope = 'local'; break; }
if (pt === 'companion_object' || pt === 'object_declaration') { scope = 'const'; break; }
if (pt === 'class_declaration') { scope = 'instance'; break; }
}
if (scope === 'local') return null;
const binding = node.namedChildren.find((c) => c.type === 'binding_pattern_kind');
const isVal = binding != null && getNodeText(binding, source) === 'val';
return scope === 'instance' ? 'field' : isVal ? 'constant' : 'variable';
}
/**
* Accessors written on their OWN line parse as SIBLINGS of the property, not as
* children of it (same-line ones nest see kotlinPropertyInitializers). Walking
* back over any accessors between us and the declaration finds the property an
* accessor belongs to; null when this accessor stands alone (a grammar
* accident, or an accessor on a destructured/local declaration).
*/
function kotlinAccessorOwner(node: SyntaxNode): SyntaxNode | null {
for (let p = node.previousNamedSibling; p; p = p.previousNamedSibling) {
if (p.type === 'getter' || p.type === 'setter') continue;
return p.type === 'property_declaration' ? p : null;
}
return null;
}
/** The sibling accessors that follow a property declaration, in source order. */
function kotlinFollowingAccessors(node: SyntaxNode): SyntaxNode[] {
const out: SyntaxNode[] = [];
for (let n = node.nextNamedSibling; n; n = n.nextNamedSibling) {
if (n.type !== 'getter' && n.type !== 'setter') break;
out.push(n);
}
return out;
}
/** Check if a node matches the `fun interface` misparse pattern */
function isFunInterfaceNode(node: SyntaxNode): boolean {
let hasFun = false;
@@ -88,48 +181,70 @@ export const kotlinExtractor: LanguageExtractor = {
// Kotlin properties (`val` / `var` / `const val`). The name nests as
// property_declaration → variable_declaration → simple_identifier, which the
// generic variable/field path can't read — so nothing was extracted before.
// Kind by enclosing scope: a singleton `object` / `companion object` (and a
// top-level property) holds *shared* values — `val`→`constant`,
// `var`→`variable` (the Scala-object rule; a `const val` is a `val`). A
// `class`/`interface`/`enum` instance `val`/`var` is per-instance state →
// `field` (never a value-ref target, like a Java instance `final`). A
// property inside a function body / `init` block / lambda is a local and is
// skipped entirely.
// Kind comes from kotlinPropertyKind.
if (node.type === 'property_declaration') {
const varDecl = node.namedChildren.find((c) => c.type === 'variable_declaration');
const nameNode = varDecl?.namedChildren.find((c) => c.type === 'simple_identifier');
if (!nameNode) return false; // destructuring `val (a,b)` etc. — leave to default
// Destructuring (`val (a, b) = makePair()`): no symbol is minted for the
// destructured names either way — declining just routes the node to
// extractField/extractVariable, which both find nothing for Kotlin and
// end in the same fn-ref scan. But the RHS is CODE, and it was vanishing
// whole. Consume the node here and walk it at the ENCLOSING scope (there
// is no symbol of its own to attribute it to).
if (!nameNode) {
for (const init of kotlinPropertyInitializers(node)) ctx.visitFunctionBody(init, '');
return true;
}
const name = getNodeText(nameNode, ctx.source);
if (!name) return false;
// Walk to the nearest enclosing definition: a function body / init / lambda
// means it's a local; `object`/`companion object` is a constant scope; a
// `class_declaration` (covers class/interface/enum) is an instance scope.
let scope: 'local' | 'const' | 'instance' = 'const';
for (let p = node.parent; p; p = p.parent) {
const pt = p.type;
if (
pt === 'function_body' || pt === 'function_declaration' ||
pt === 'lambda_literal' || pt === 'anonymous_initializer' ||
pt === 'control_structure_body' || pt === 'getter' || pt === 'setter'
) { scope = 'local'; break; }
if (pt === 'companion_object' || pt === 'object_declaration') { scope = 'const'; break; }
if (pt === 'class_declaration') { scope = 'instance'; break; }
const kind = kotlinPropertyKind(node, ctx.source);
if (kind == null) {
// A local — no node is minted, but the initializer is still code. Walk
// it at the ENCLOSING scope: an `init { }` block's `val q = load()` is
// the CLASS calling load, and it used to disappear entirely (only the
// block's bare statements survived).
for (const init of kotlinPropertyInitializers(node)) ctx.visitFunctionBody(init, '');
return true;
}
if (scope === 'local') return true; // a local — don't extract
const binding = node.namedChildren.find((c) => c.type === 'binding_pattern_kind');
const isVal = binding != null && getNodeText(binding, ctx.source) === 'val';
const kind = scope === 'instance' ? 'field' : isVal ? 'constant' : 'variable';
const typeNode = node.childForFieldName('type');
const sig = typeNode
? `${isVal ? 'val' : 'var'} ${name}: ${getNodeText(typeNode, ctx.source)}`
: undefined;
ctx.createNode(kind, name, node, { signature: sig });
const created = ctx.createNode(kind, name, node, { signature: sig });
// Walk the initializer ATTRIBUTED to the declared symbol (#693, the Go
// fix, ported to Kotlin): the hook consumes this subtree, so without an
// explicit walk a lambda / SAM / object initializer
// (`private val cb = Runnable { target() }` — the idiomatic Android
// callback field) contributed NO call edge at all, and everything reached
// only through such a callback looked like it had no callers.
// The property also OWNS any accessor written on its own line, which the
// grammar makes a following SIBLING rather than a child; those bodies used
// to attribute to the enclosing class. Consumed here so the accessor
// branch below can skip them without any cross-node state.
const inits = created
? [...kotlinPropertyInitializers(node), ...kotlinFollowingAccessors(node)]
: [];
if (created && inits.length > 0) {
ctx.pushScope(created.id);
for (const init of inits) ctx.visitFunctionBody(init, created.id);
ctx.popScope();
}
return true;
}
// An own-line accessor already walked by its owning property above. The
// ownership test re-derives the property's kind rather than remembering it:
// a destructured or local declaration mints no node, so its accessors were
// NOT consumed and must keep falling through to the normal recursion.
if (node.type === 'getter' || node.type === 'setter') {
const owner = kotlinAccessorOwner(node);
return owner != null && kotlinPropertyKind(owner, ctx.source) != null;
}
// Handle Kotlin `fun interface` declarations.
// Tree-sitter-kotlin doesn't support `fun interface` syntax (Kotlin 1.4+).
// It produces two different misparse patterns:
+10
View File
@@ -166,6 +166,16 @@ export const scalaExtractor: LanguageExtractor = {
const created = ctx.createNode(kind, name, node, { signature: sig, visibility: extractVisibility(node) });
if (created && typeNode) emitScalaTypeRefs(typeNode, created.id, ctx, ctx.source);
// Walk the initializer ATTRIBUTED to the declared symbol (#693, the Go
// fix): the hook consumes this subtree and the dispatcher only scans it
// for function-as-value candidates, so `val cb = () => target()` — and
// even a plain `val x = compute()` — emitted no call edge at all.
const valueNode = node.childForFieldName('value');
if (created && valueNode) {
ctx.pushScope(created.id);
ctx.visitFunctionBody(valueNode, created.id);
ctx.popScope();
}
return true;
}
+66 -14
View File
@@ -2257,6 +2257,21 @@ export class TreeSitterExtractor {
// and the language-aware path in `extractTypeAnnotations` descends
// into that wrapper (#381).
this.extractTypeAnnotations(node, fieldNode.id);
// Walk the initializer ATTRIBUTED to the declared field (#693, the
// Go fix; same shape as the TS/JS class-field walk above). The
// dispatcher only scanned this subtree for function-as-value
// candidates, so a lambda / method reference / anonymous class in
// `private final Runnable r = () -> target();` contributed NO call
// edge at all and `target` looked callerless. Keyed on the `value`
// FIELD, which only Java's `variable_declarator` carries — C#,
// VB.NET and PHP spell their initializer differently and are
// deliberately untouched here.
const valueNode = getChildByField(decl, 'value');
if (valueNode) {
this.nodeStack.push(fieldNode.id);
this.visitFunctionBody(valueNode, fieldNode.id);
this.nodeStack.pop();
}
}
}
} else {
@@ -2818,19 +2833,24 @@ export class TreeSitterExtractor {
storeCollections.push(objectOfFns);
}
// Visit the initializer body for calls — EXCEPT object literals (their
// function-valued properties are extracted below) and the store-factory
// / createApi / store-collection call whose nested objects we extract
// method-by-method below (walking the whole call would re-visit those
// method arrows and mis-attribute their inner calls to the file scope).
if (valueNode &&
valueNode.type !== 'object' &&
valueNode.type !== 'object_expression' &&
!(extractObjectMethods && valueNode.type === 'call_expression') &&
!rtkEndpoints &&
!piniaSetup &&
storeCollections.length === 0) {
// Visit the initializer body for calls, ATTRIBUTED to the declared
// symbol (#693) — EXCEPT the shapes whose members are extracted
// one-by-one below (the store-factory / createApi / store-collection
// objects), where walking the whole initializer would re-visit each
// member arrow and double-count its calls.
//
// Two things were wrong here before. The walk ran with only the FILE
// on the stack, so `const cfg = load()` recorded the FILE as load's
// caller — the exact leak Go's #693 fixed. And an object literal was
// skipped outright, so `const obj = { handler: () => target() }`
// contributed nothing at all unless the const was exported (only then
// does extractObjectLiteralFunctions mint the members).
const membersExtractedSeparately =
extractObjectMethods || !!rtkEndpoints || !!piniaSetup || storeCollections.length > 0;
if (valueNode && !membersExtractedSeparately) {
if (varNode) this.nodeStack.push(varNode.id);
this.visitFunctionBody(valueNode, '');
if (varNode) this.nodeStack.pop();
}
if (extractObjectMethods && objectOfFns) {
@@ -2855,6 +2875,7 @@ export class TreeSitterExtractor {
// Ruby constant assignments (`MAX = 3`) have a `constant`-typed LHS, not
// `identifier`; without this they were never extracted as symbols at all.
let assigned: Node | null = null;
if (left && (left.type === 'identifier' || left.type === 'constant')) {
const name = getNodeText(left, this.source);
// Skip if name starts with lowercase and looks like a function call result
@@ -2862,11 +2883,23 @@ export class TreeSitterExtractor {
const initValue = right ? getNodeText(right, this.source).slice(0, 100) : undefined;
const initSignature = initValue ? `= ${initValue}${initValue.length >= 100 ? '...' : ''}` : undefined;
this.createNode(kind, name, node, {
assigned = this.createNode(kind, name, node, {
docstring,
signature: initSignature,
});
}
// Walk the initializer ATTRIBUTED to the assigned name (#693). A
// module-level `app = FastAPI()` / `ENGINE = create_engine(url)` /
// `handler = lambda: run()` dropped every call on the right-hand side, so
// whatever the module builds at import time linked to nothing. A tuple
// target (`a, b = f(), g()`) mints no symbol, so its RHS is walked at the
// enclosing scope rather than lost. Python only: Ruby shares this branch
// and gets its own turn.
if (this.language === 'python' && right) {
if (assigned) this.nodeStack.push(assigned.id);
this.visitFunctionBody(right, '');
if (assigned) this.nodeStack.pop();
}
} else if (this.language === 'go') {
// Go: var_declaration, short_var_declaration, const_declaration
// These can have multiple identifiers on the left
@@ -3019,6 +3052,8 @@ export class TreeSitterExtractor {
} else {
// Generic fallback for other languages
// Try to find identifier children
const nameField = getChildByField(node, 'name');
let declared: Node | null = null;
for (let i = 0; i < node.namedChildCount; i++) {
const child = node.namedChild(i);
if (child?.type === 'identifier' || child?.type === 'variable_declarator') {
@@ -3027,13 +3062,30 @@ export class TreeSitterExtractor {
: extractName(child, this.source, this.extractor);
if (name && name !== '<anonymous>') {
this.createNode(kind, name, child, {
const created = this.createNode(kind, name, child, {
docstring,
isExported,
});
if (created && nameField && child.startIndex === nameField.startIndex) {
declared = created;
}
}
}
}
// Walk the initializer ATTRIBUTED to the declared symbol (#693). Rust
// only for now: `const N: usize = compute()` and
// `static REGISTRY: Lazy<T> = Lazy::new(|| build())` dropped every call
// inside the initializer, so a handler table or a lazily-built singleton
// linked to nothing. The other languages sharing this fallback spell
// their initializer differently and get their own turn.
if (this.language === 'rust') {
const valueNode = getChildByField(node, 'value');
if (valueNode) {
if (declared) this.nodeStack.push(declared.id);
this.visitFunctionBody(valueNode, '');
if (declared) this.nodeStack.pop();
}
}
}
}
+9 -1
View File
@@ -277,8 +277,16 @@ export function resolveNamedTokens(
const segPool = new Set<string>();
for (const t of tokens) for (const s of t.toLowerCase().split(/::|\./)) if (s) segPool.add(s);
// RAW edges, not getCallers/getCallees: those return one row per NEIGHBOUR
// (the #1086 de-dup), so when a pair is joined by BOTH a static and a
// synthesized edge the static one wins and the synthesized one becomes
// invisible — which is exactly what happens once a thunk's `dispatch(x)`
// is walked statically. The question here is about the graph, not about
// callers, so ask the edges directly.
const hasHeuristicEdge = (id: string): boolean =>
[...cg.getCallers(id), ...cg.getCallees(id)].some(({ edge }) => edge.provenance === 'heuristic');
[...cg.getIncomingEdges(id), ...cg.getOutgoingEdges(id)].some(
(e) => e.provenance === 'heuristic'
);
for (const t of tokens) {
const hits = findAllSymbols(cg, t).nodes;
+9 -2
View File
@@ -2760,9 +2760,16 @@ export class ToolHandler {
const synthSeen = new Set<string>();
for (const n of [...named.values(), ...dynNamed.values()]) {
if (synthLines.length >= 6) break;
for (const { node: other, edge } of [...cg.getCallers(n.id), ...cg.getCallees(n.id)]) {
// RAW edges for the same reason as hasHeuristicEdge above — a static
// edge over the same pair hides the synthesized one from getCallers.
const incident = [...cg.getIncomingEdges(n.id), ...cg.getOutgoingEdges(n.id)];
for (const edge of incident) {
if (synthLines.length >= 6) break;
if (edge.provenance !== 'heuristic' || other.id === n.id) continue;
if (edge.provenance !== 'heuristic') continue;
const otherId = edge.source === n.id ? edge.target : edge.source;
if (otherId === n.id) continue;
const other = cg.getNode(otherId);
if (!other) continue;
if (skipInChain && skipInChain(edge)) continue;
const src = edge.source === n.id ? n : other;
const tgt = edge.source === n.id ? other : n;
+14 -1
View File
@@ -66,6 +66,7 @@ export function buildFile(cg: CodeGraph, projectRoot: string, requested: string)
const nodes = cg.getNodesInFile(storedPath);
const nodeIds = nodes.map((n) => n.id);
const inThisFile = new Set(nodeIds);
const nodeKindById = new Map(nodes.map((n) => [n.id, n.kind]));
const fileNode = nodes.find((n) => n.kind === 'file') ?? null;
// ---------------------------------------------------------------------------
@@ -106,8 +107,20 @@ export function buildFile(cg: CodeGraph, projectRoot: string, requested: string)
// the same signal `/api/entrypoints` ranks on. It is worth a line on this
// screen because the outline cannot show it: top-level code belongs to no
// symbol, so the only way to read it is to open the file node itself.
// A call made while initializing a module-level variable or constant is
// attributed to that name (#693), so those names are top-level code too and
// are counted with the file — the same set `getTopCallingFiles` ranks on.
const moduleLevelValueIds = fileNode
? cg
.getOutgoingEdgesFrom([fileNode.id], ['contains'])
.map((e) => e.target)
.filter((id) => {
const kind = nodeKindById.get(id);
return kind === 'variable' || kind === 'constant';
})
: [];
const topLevelEdges = fileNode
? cg.getOutgoingEdgesFrom([fileNode.id], ['calls', 'instantiates'])
? cg.getOutgoingEdgesFrom([fileNode.id, ...moduleLevelValueIds], ['calls', 'instantiates'])
: [];
return {