From a5b8cd8e25694565eb855091605a2dac5095bc22 Mon Sep 17 00:00:00 2001 From: Colby Mchenry Date: Fri, 3 Jul 2026 15:19:46 -0500 Subject: [PATCH] feat(extraction): Erlang macro-body call linkage (#635, #648) (#1168) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Calls hidden inside -define bodies were invisible: the extractor consumed pp_define without walking the replacement, and macro use sites produced no edges, so a call path routed through a macro (ejabberd's SQL upsert macros, logging wrappers) was completely dark. The macro's constant node now participates in the graph. The -define body's calls are attributed to the MACRO — true exactly once, instead of a per-use duplicate that would explode on logging macros — and each use site links in: ?MACRO(...) with arguments emits a `calls` ref (inlined code joins the call chain), a bare ?CONSTANT read emits `references` (answering "where is this macro used" without polluting call paths). Compiler-predefined macros (?MODULE, ?LINE, ?FUNCTION_NAME, ...) are excluded, macro-use arguments keep walking so a call nested in ?assertEqual(ok, do_thing()) still attributes to the enclosing function, and macro-to-macro chains connect. Validated: node counts unchanged on cowboy/ejabberd/emqx; edges +26/+7.3K/ +42K with honest hub shapes (?T i18n, ?SLOG logging, ?QOS_1 protocol constants); 40/40 sampled edges precise; +1.3s index cost on emqx's 2,273 files. The payoff chain on ejabberd: set_password_scram_t → ?SQL_UPSERT_T → ejabberd_sql:sql_query_t — database writes through SQL macros now trace end-to-end. Co-authored-by: Claude Fable 5 --- CHANGELOG.md | 2 +- __tests__/extraction.test.ts | 75 ++++++++++++++++++++++++++++++ src/extraction/languages/erlang.ts | 23 +++++++-- src/extraction/tree-sitter.ts | 30 ++++++++++++ 4 files changed, 124 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c1aee1..5145085 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### New Features - Erlang behaviour dispatch is now followed through the graph: a framework call through a variable module — cowboy's `Handler:init`/`Middleware:execute` folds, a plugin manager's `Mod:callback(...)` — links to the repo's implementations of the behaviour that declares that callback, so flow traces and impact cross the OTP callback boundary instead of stopping at it. The links are precision-gated: the callback arity must match, exactly one behaviour may own that callback shape (a collision stays unlinked rather than guessed), the implementer must actually export the callback, and the fan-out is bounded — a behaviour with hundreds of implementers stays a visibly dynamic boundary. Every bridged hop is labeled as dynamic dispatch with its wiring site, never shown as a plain static call. -- CodeGraph now indexes **Erlang** (`.erl`, `.hrl`) — functions, with clauses and arities of the same name grouped as one symbol spanning all of them, plus records with their fields, `-type`/`-opaque` aliases, `-define` macros, and `-spec` signatures attached to every function. Cross-module `mod:fn(...)` calls resolve to the target module's function, `fun name/arity` values are captured as references (so callback registrations like `lists:foreach(fun submit/1, ...)` link up), `-include`/`-include_lib` connect to the header files they pull in, `-behaviour` declarations link a callback module to its behaviour (and only ever to a module — a same-named macro or function elsewhere in the repo is never mistaken for one), and `-export` lists (plus `-compile(export_all)`) drive each function's public/private flag. OTP's indirection idioms are followed where the target is static: `spawn`/`apply`/`proc_lib`/`timer`/`rpc` calls that name their target as `(Module, Function, Args)` arguments produce call edges, and `gen_server:call`/`cast` connects to the target module's `handle_call`/`handle_cast` — its own when targeting `?MODULE` (including the `-define(SERVER, ?MODULE)` idiom), and the named module when a registered name follows OTP's name-the-server-after-its-module convention (`gen_server:call(other_mod, ...)`, directly or through a `-define(STORE, other_mod)` macro); a registered name that matches no module stays unlinked. Truly dynamic dispatch (`Mod:handle(...)`, message sends, var-module spawns) is deliberately left unlinked rather than guessed. `codegraph_explore` also understands Erlang-native symbol spelling in queries — `mod:fn/3` and `init/2` find the symbols they name. (#635, #648) +- CodeGraph now indexes **Erlang** (`.erl`, `.hrl`) — functions, with clauses and arities of the same name grouped as one symbol spanning all of them, plus records with their fields, `-type`/`-opaque` aliases, `-define` macros, and `-spec` signatures attached to every function. Cross-module `mod:fn(...)` calls resolve to the target module's function, `fun name/arity` values are captured as references (so callback registrations like `lists:foreach(fun submit/1, ...)` link up), `-include`/`-include_lib` connect to the header files they pull in, `-behaviour` declarations link a callback module to its behaviour (and only ever to a module — a same-named macro or function elsewhere in the repo is never mistaken for one), and `-export` lists (plus `-compile(export_all)`) drive each function's public/private flag. OTP's indirection idioms are followed where the target is static: `spawn`/`apply`/`proc_lib`/`timer`/`rpc` calls that name their target as `(Module, Function, Args)` arguments produce call edges, and `gen_server:call`/`cast` connects to the target module's `handle_call`/`handle_cast` — its own when targeting `?MODULE` (including the `-define(SERVER, ?MODULE)` idiom), and the named module when a registered name follows OTP's name-the-server-after-its-module convention (`gen_server:call(other_mod, ...)`, directly or through a `-define(STORE, other_mod)` macro); a registered name that matches no module stays unlinked. Macros participate in the graph too: a `-define` body's calls belong to the macro, each `?MACRO(...)` use site links into the call chain (and bare `?CONSTANT` reads are tracked as references), so a call path hidden behind a macro — `set_password → ?SQL_UPSERT_T → sql_query_t` — traces end-to-end and "where is this macro used" is answerable. Truly dynamic dispatch (`Mod:handle(...)`, message sends, var-module spawns) is deliberately left unlinked rather than guessed. `codegraph_explore` also understands Erlang-native symbol spelling in queries — `mod:fn/3` and `init/2` find the symbols they name. (#635, #648) - CodeGraph now indexes **Visual Basic .NET** (`.vb`) — classes, Modules, interfaces, structures, enums, properties, events, `MustOverride` abstract members, and `Declare` P/Invoke signatures, with `Inherits`/`Implements` hierarchy edges, call edges (resolved through VB's ambiguous call-vs-index parentheses), and `New`/`As New` instantiation links. Real-world VB styles parse cleanly: WinForms designer files, interpolated and multi-line strings, XML literals (embedded `<%= %>` expressions included), single-line and multi-line LINQ queries, multi-line lambdas, `Handles`/`WithEvents` event wiring, Custom Events, date literals, classic type-character identifiers (`i%`, `name$`), and non-English (Unicode) identifiers. (#648, #639, #170) - CodeGraph now indexes **COBOL** (`.cbl`, `.cob`, `.cpy`) — programs, sections and paragraphs with `PERFORM`/`GO TO` call edges, `CALL` cross-program calls, `COPY` copybook imports (standalone copybooks included), and DATA DIVISION records with 88-level condition names, in both fixed and free source format. Impact queries work on data items: every `MOVE`/`ADD`/`COMPUTE`/`SUBTRACT` write-site links back to the field it changes, so "what touches this copybook field" answers across programs. CICS flows connect too: `EXEC CICS LINK`/`XCTL` program targets, `EXEC SQL INCLUDE` copybooks, and pseudo-conversational `RETURN TRANSID(...)` hops resolve to the program owning the transaction id. (#590, #648) - CodeGraph now indexes **CFML** (`.cfc`, `.cfm`, `.cfs`) — both the classic tag-based style (``/``) and modern bare-script `component { ... }` syntax, including `extends`/`implements`, embedded `` blocks (at any nesting depth, including inside ``/``/``), call edges, and calls embedded in `#hash#` expressions inside `` SQL bodies. Files saved with a UTF-8 byte-order mark and tags with unquoted attribute values — both common in long-lived CFML codebases — are handled too. Thanks @ghedwards. (#1118) diff --git a/__tests__/extraction.test.ts b/__tests__/extraction.test.ts index 3941497..f95b66d 100644 --- a/__tests__/extraction.test.ts +++ b/__tests__/extraction.test.ts @@ -9134,6 +9134,81 @@ second(X) -> X. }); }); + describe('Macro linkage', () => { + it('should attribute macro-body calls to the macro and link function-like uses into the chain', () => { + const code = `-module(m). +-export([do_thing/1]). + +-define(LOG_AUDIT(Event), audit_logger:log(Event, ?MODULE)). + +do_thing(X) -> + ?LOG_AUDIT({thing, X}), + ok. +`; + const result = extractFromSource('src/m.erl', code); + const macro = result.nodes.find((n) => n.kind === 'constant' && n.name === 'LOG_AUDIT'); + const doThing = result.nodes.find((n) => n.kind === 'function' && n.name === 'do_thing'); + const refsFrom = (id?: string) => + result.unresolvedReferences.filter((r) => r.fromNodeId === id).map((r) => `${r.referenceKind}:${r.referenceName}`); + // The body's remote call belongs to the macro node — true exactly once. + expect(refsFrom(macro?.id)).toContain('calls:audit_logger::log'); + // The use site joins the call chain: do_thing -calls→ LOG_AUDIT. + expect(refsFrom(doThing?.id)).toContain('calls:LOG_AUDIT'); + }); + + it('should reference bare macro reads without polluting call chains', () => { + const code = `-module(m). +-export([wait/0]). + +-define(TIMEOUT, 5000). + +wait() -> + receive after ?TIMEOUT -> ok end. +`; + const result = extractFromSource('src/m.erl', code); + const refs = result.unresolvedReferences.map((r) => `${r.referenceKind}:${r.referenceName}`); + expect(refs).toContain('references:TIMEOUT'); + expect(refs).not.toContain('calls:TIMEOUT'); + }); + + it('should skip compiler-predefined macros and keep walking macro-use arguments', () => { + const code = `-module(m). +-export([check/0]). + +check() -> + ?assertEqual(ok, prepare()), + {?MODULE, ?LINE, ?FUNCTION_NAME}. + +prepare() -> ok. +`; + const result = extractFromSource('src/m.erl', code); + const refs = result.unresolvedReferences.map((r) => r.referenceName); + // The nested call inside the macro's arguments still attributes to check/0. + expect(refs).toContain('prepare'); + // ?assertEqual (an OTP header macro) is emitted and simply never resolves… + expect(refs).toContain('assertEqual'); + // …but predefined macros have no definition to link. + expect(refs).not.toContain('MODULE'); + expect(refs).not.toContain('LINE'); + expect(refs).not.toContain('FUNCTION_NAME'); + }); + + it('should chain macro-to-macro uses', () => { + const code = `-module(m). + +-define(TARGET, target_fn()). +-define(ALIAS, ?TARGET). +`; + const result = extractFromSource('src/m.erl', code); + const target = result.nodes.find((n) => n.kind === 'constant' && n.name === 'TARGET'); + const alias = result.nodes.find((n) => n.kind === 'constant' && n.name === 'ALIAS'); + const refsFrom = (id?: string) => + result.unresolvedReferences.filter((r) => r.fromNodeId === id).map((r) => `${r.referenceKind}:${r.referenceName}`); + expect(refsFrom(target?.id)).toContain('calls:target_fn'); + expect(refsFrom(alias?.id)).toContain('references:TARGET'); + }); + }); + describe('Behaviour extraction', () => { it('should emit an implements reference for -behaviour', () => { const code = `-module(m). diff --git a/src/extraction/languages/erlang.ts b/src/extraction/languages/erlang.ts index e35c467..7e122d5 100644 --- a/src/extraction/languages/erlang.ts +++ b/src/extraction/languages/erlang.ts @@ -175,12 +175,24 @@ function handleTypeAlias(node: SyntaxNode, ctx: ExtractorContext): boolean { function handlePpDefine(node: SyntaxNode, ctx: ExtractorContext): boolean { const lhs = getChildByField(node, 'lhs'); const nameNode = lhs ? getChildByField(lhs, 'name') : null; - if (nameNode) { - ctx.createNode('constant', getNodeText(nameNode, ctx.source), node, { - signature: collapseWs(getNodeText(node, ctx.source)).slice(0, 200), - }); + if (!nameNode) return true; + const macro = ctx.createNode('constant', getNodeText(nameNode, ctx.source), node, { + signature: collapseWs(getNodeText(node, ctx.source)).slice(0, 200), + }); + // The replacement's calls execute at expansion sites, but attributing them + // to the MACRO node keeps them true exactly once: `-define(LOG_AUDIT(E), + // audit_logger:log(E))` gives the LOG_AUDIT constant a `calls` edge to the + // logger, and each `?LOG_AUDIT(...)` use site links to the constant (see the + // macro_call_expr case in extractCall) — so the chain + // `caller → LOG_AUDIT → audit_logger:log` traverses without minting a + // per-use duplicate of the body's calls. + const replacement = getChildByField(node, 'replacement'); + if (macro && replacement) { + ctx.pushScope(macro.id); + ctx.visitFunctionBody(replacement, macro.id); + ctx.popScope(); } - return true; // the replacement's calls only exist at expansion sites + return true; } function handleBehaviour(node: SyntaxNode, ctx: ExtractorContext): boolean { @@ -218,6 +230,7 @@ export const erlangExtractor: LanguageExtractor = { 'record_update_expr', // X#rec{...} 'record_index_expr', // #rec.field 'record_field_expr', // X#rec.field + 'macro_call_expr', // ?MACRO / ?MACRO(...) — links use sites to the -define constant ], variableTypes: [], nameField: 'name', diff --git a/src/extraction/tree-sitter.ts b/src/extraction/tree-sitter.ts index 5cc0781..62ef311 100644 --- a/src/extraction/tree-sitter.ts +++ b/src/extraction/tree-sitter.ts @@ -67,6 +67,13 @@ const VUE_STORE_FILE_SIGNAL = /\bdefineStore\b|\bcreateStore\b|\bVuex\b|\bmutati * Used by the erlang branch of extractCall to lift a static MFA pair into a * call edge (the spawned/applied function is otherwise invisible to the graph). */ +/** Compiler-predefined Erlang macros — no `-define` exists to link a use to. */ +const ERLANG_PREDEFINED_MACROS = new Set([ + 'MODULE', 'MODULE_STRING', 'FILE', 'LINE', 'MACHINE', + 'FUNCTION_NAME', 'FUNCTION_ARITY', 'OTP_RELEASE', + 'FEATURE_AVAILABLE', 'FEATURE_ENABLED', +]); + const ERLANG_MFA_CALLS = new Set([ 'spawn', 'spawn_link', 'spawn_monitor', 'spawn_opt', 'apply', 'erlang:spawn', 'erlang:spawn_link', 'erlang:spawn_monitor', 'erlang:spawn_opt', 'erlang:apply', @@ -3752,6 +3759,29 @@ export class TreeSitterExtractor { }); return; } + if (node.type === 'macro_call_expr') { + // Macro use site → the `-define` constant node. Function-like uses + // (`?LOG_AUDIT(X)` — args present) are inlined code, so they join the + // call chain and connect through the macro node to the body's calls + // (attributed there by handlePpDefine); bare reads (`?TIMEOUT`) are + // `references`, answering "where is this macro used" without + // polluting call paths. Compiler-predefined macros carry no + // definition to link. The use site's ARGUMENTS are children and keep + // walking, so a call nested in `?assertEqual(ok, do_thing())` still + // attributes to the enclosing function. + const macroName = getChildByField(node, 'name'); + if (!macroName) return; + const name = getNodeText(macroName, this.source); + if (ERLANG_PREDEFINED_MACROS.has(name)) return; + this.unresolvedReferences.push({ + fromNodeId: callerId, + referenceName: name, + referenceKind: getChildByField(node, 'args') ? 'calls' : 'references', + line, + column, + }); + return; + } // record_expr / record_update_expr / record_index_expr / record_field_expr const recordName = getChildByField(node, 'name'); const recordAtom = recordName?.type === 'record_name' ? getChildByField(recordName, 'name') : null;