From 9ad5cd7ba2428d39b1c818247ac6c9c8a67d6a90 Mon Sep 17 00:00:00 2001 From: Colby McHenry Date: Thu, 16 Jul 2026 22:14:40 -0500 Subject: [PATCH] =?UTF-8?q?feat(kernel):=20R2=20=E2=80=94=20full=20TypeScr?= =?UTF-8?q?ipt/JavaScript=20extraction=20port,=20byte-parity=20with=20the?= =?UTF-8?q?=20wasm=20path?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the R1 seed .scm emitter with a bespoke Rust walker (codegraph-kernel/src/tsjs/) that mirrors TreeSitterExtractor's TS/JS paths function-for-function: declarations (incl. #808 field/property classification), qualified names, docstrings (#780 wrapper climbs), signatures, imports/re-exports + per-binding refs, calls with receiver-qualified callees (#1230 literal-receiver skip), instantiations, decorators, inheritance, type annotations (#381), type-alias members + tuple contracts (#359/#634), React component recognition (#841 forwardRef/memo/styled), object-of-functions / zustand-through-middleware / RTK Query endpoints + generated hooks / vuex + pinia store shapes, function-as-value capture with the flush gate (#756), and value-reference edges with the shadow prune (#895/#897). The generic query emitter is deleted — extraction parity needs logic .scm can't express; future languages get walkers too (migration plan §4a). Positions and JS string-slice semantics are emitted in UTF-16 code units natively, so kernel output is byte-identical to web-tree-sitter's — no column diff class exists. Parity evidence (macOS): scripts/kernel-parity.mjs (full-object multiset diff per file) — this repo 353/353 files, excalidraw 643/643 (10,650 nodes / 10,726 edges / 68,307 refs), plus torture fixtures checked into __tests__/fixtures/kernel-parity/ and enforced in npm test by kernel-tsjs-parity.test.ts. The strict compare caught one real decoder bug the loose harness missed: refs must NOT carry denormalized filePath/language at the extractFromSource seam (the store fills them). Perf: extraction 2.6× single-thread on excalidraw (487ms vs 1,255ms, identical outputs). Routing stays opt-in (CODEGRAPH_KERNEL_LANGS) until the R3 equivalence gate (large repo, DB dump-diff, retrieval invariants, agent A/B, Linux/Windows) passes. Co-Authored-By: Claude Fable 5 --- __tests__/fixtures/kernel-parity/torture.js | 75 + __tests__/fixtures/kernel-parity/torture.tsx | 214 +++ __tests__/kernel-scaffold.test.ts | 7 +- __tests__/kernel-tsjs-parity.test.ts | 119 ++ codegraph-kernel/Cargo.lock | 2 +- codegraph-kernel/Cargo.toml | 2 +- codegraph-kernel/queries/javascript.scm | 12 - codegraph-kernel/queries/typescript.scm | 21 - codegraph-kernel/src/buffers.rs | 9 + codegraph-kernel/src/emitter.rs | 300 ---- codegraph-kernel/src/langs.rs | 91 +- codegraph-kernel/src/lib.rs | 15 +- codegraph-kernel/src/tsjs/docstring.rs | 140 ++ codegraph-kernel/src/tsjs/extractors.rs | 1332 ++++++++++++++++++ codegraph-kernel/src/tsjs/fnref.rs | 133 ++ codegraph-kernel/src/tsjs/mod.rs | 876 ++++++++++++ codegraph-kernel/src/tsjs/util.rs | 190 +++ docs/design/rust-kernel-migration-plan.md | 44 +- scripts/kernel-parity.mjs | 214 +++ src/extraction/kernel/decode.ts | 5 +- 20 files changed, 3382 insertions(+), 419 deletions(-) create mode 100644 __tests__/fixtures/kernel-parity/torture.js create mode 100644 __tests__/fixtures/kernel-parity/torture.tsx create mode 100644 __tests__/kernel-tsjs-parity.test.ts delete mode 100644 codegraph-kernel/queries/javascript.scm delete mode 100644 codegraph-kernel/queries/typescript.scm delete mode 100644 codegraph-kernel/src/emitter.rs create mode 100644 codegraph-kernel/src/tsjs/docstring.rs create mode 100644 codegraph-kernel/src/tsjs/extractors.rs create mode 100644 codegraph-kernel/src/tsjs/fnref.rs create mode 100644 codegraph-kernel/src/tsjs/mod.rs create mode 100644 codegraph-kernel/src/tsjs/util.rs create mode 100644 scripts/kernel-parity.mjs diff --git a/__tests__/fixtures/kernel-parity/torture.js b/__tests__/fixtures/kernel-parity/torture.js new file mode 100644 index 0000000..50ddd02 --- /dev/null +++ b/__tests__/fixtures/kernel-parity/torture.js @@ -0,0 +1,75 @@ +/** + * JS-grammar torture fixture (javascript variant: no type machinery, JS class + * fields use `field_definition` with a `property` field). + */ +import { EventEmitter } from 'node:events'; +const { promisify } = require('node:util'); + +/** Legacy prototype-style helper. */ +function legacyHelper(a, b) { + return a + b; +} + +const arrow = (x) => legacyHelper(x, 1); + +class Widget extends EventEmitter { + static registry = new Map(); + #privateField = 1; + label = 'w'; + onTick = () => { + this.render(); + }; + wrapped = debounce(function () { + expensive(); + }, 50); + + constructor(opts) { + super(); + this.opts = opts; + register(this.onTick); + } + + render() { + paint(this.label); + } + + static create(opts) { + return new Widget(opts); + } +} + +// AMD-style wrapper — anonymous, but inner functions must still surface. +(function () { + function hiddenInner() { + return 7; + } + hiddenInner(); +})(); + +module.exports.makeWidget = function makeWidget(opts) { + return Widget.create(opts); +}; + +// Vuex module shape (store-file signals: mutations + actions + getters). +const mutations = { + SET_USER(state, user) { + state.user = user; + }, +}; +const actions = { + async loadUser({ commit }, id) { + const user = await fetchUser(id); + commit('SET_USER', user); + }, +}; +export default { + namespaced: true, + state: () => ({ user: null }), + mutations, + actions, + getters: { + userName(state) { + return state.user?.name; + }, + }, +}; diff --git a/__tests__/fixtures/kernel-parity/torture.tsx b/__tests__/fixtures/kernel-parity/torture.tsx new file mode 100644 index 0000000..de27c0a --- /dev/null +++ b/__tests__/fixtures/kernel-parity/torture.tsx @@ -0,0 +1,214 @@ +/** + * Torture fixture — exercises every TS/TSX extraction path the kernel ports. + */ +import React, { forwardRef, memo, useState as useStateAlias } from 'react'; +import * as NS from './namespace-module'; +import DefaultThing from './default-module'; +import './side-effect'; +import { helperFn, CONFIG_TABLE } from './helpers'; + +export { reExported, orig as aliased } from './barrel-source'; +export * from './star-source'; + +// A documented constant table (value-ref target). +const RETRY_LIMITS = { a: 1, b: 2 }; +export const API_BASE = 'https://example.test'; +let plainLet = 42; +var oldVar = 'x'; + +/** Class docs. + * Multi-line. + */ +@Injectable() +@scoped.Registry('name') +export abstract class BaseService extends EventTarget implements Disposable, Serializable { + static instances = 0; + private readonly cache: Map = new Map(); + public fonts: FontConfig; + count = 0; + onScroll = throttle((e: Event) => { + this.handleScroll(e); + }, 100); + handleClick = (ev: MouseEvent): void => { + emitTelemetry(ev); + new AbortController(); + }; + + @Get('/list') + async list(query: QueryOpts): Promise> { + const local: ItemMapper = makeMapper(); + const limit = RETRY_LIMITS; + return this.cache.get(query.key) ?? helperFn(query); + } + + private static compute(n: number): number { + return n * RETRY_LIMITS.a; + } + + get size(): number { + return this.count; + } + + protected dispose(): void { + listeners.forEach((l) => unregister(l)); + } +} + +class Plain { + constructor(private svc: BaseService) { + register(this.onEvent); + btn.on('click', this.handleClick); + } + onEvent() {} + handleClick() {} +} + +export interface Shape extends Named, Sized { + area: number; + resize(f: number): Shape; + onchange: (s: Shape) => void; +} + +export enum Direction { + Up, + Down = 2, + Left, +} + +export type Handle = { + stop: () => void; + id: string; + refresh(force: boolean): Promise; +}; + +export type ServiceList = [ + Service<'query_apply_record', Req, Resp>, + Service<'apply_confirm', Req, Resp>, +]; + +export type MaybeShape = Shape | null; + +export function topLevel(a: ShapeConfig): Shape { + const inner = () => reallyDeep(); + function namedInner(): void { + chained(a).value; + } + namedInner(); + return makeShape(a); +} + +async function* generatorFn(items: Item[]) { + yield* items.map((i) => transform(i)); +} + +export const arrowConst = async (x: number) => { + return x + plainLet; +}; + +const AnonClassHolder = class { + method() {} +}; + +// React components (#841). +export const Button = forwardRef((props: ButtonProps, ref) => { + const [state, setState] = useStateAlias(0); + useEffect(() => { + trackRender(); + }); + return setState(state + 1)} {...props} />; +}); + +export const MemoRow = memo(function Row(props: RowProps) { + return {props.children}; +}); + +export const Wrapped = React.memo(ImportedComponent); +export const Styled = styled.button` + color: red; +`; +const memoCache = memo(computeThing); // lowercase — stays a constant + +export function App() { + const nodes: GraphNode[] = []; + return ( +
+
+ ); +} + +// Object-of-functions (SvelteKit-style actions map). +export const actionsMap = { + create: async (input: CreateInput) => { + return persistNew(input); + }, + update(input: UpdateInput) { + return persistExisting(input); + }, +}; + +// Zustand-style store through middleware wrappers. +export const useStore = create( + persist( + (set, get) => ({ + fetchUser: async (id: string) => { + const user = await api.load(id); + set({ user }); + }, + reset: () => set({ user: null }), + }), + { name: 'store' } + ) +); + +// RTK Query. +export const widgetApi = createApi({ + reducerPath: 'widgets', + endpoints: (build) => ({ + getWidget: build.query({ + query: (id: string) => fetchWidget(id), + }), + updateWidget: build.mutation({ + queryFn: async (patch) => { + return applyPatch(patch); + }, + }), + prebuilt: build.query(makeEndpointConfig()), + }), +}); + +export const { useGetWidgetQuery, useUpdateWidgetMutation } = widgetApi; +const { notAHook } = widgetApi; + +// Value-ref shadowing: SHADOWED_LIMIT re-bound locally must be pruned. +const SHADOWED_LIMIT = 10; +export function readsShadowed() { + const SHADOWED_LIMIT = 20; + return SHADOWED_LIMIT; +} +export function readsTable() { + return RETRY_LIMITS.b + API_BASE.length; +} + +// Fn-ref registrations. +registerHandler(helperFn); +queueMicrotask(topLevel); +const routeTable = { home: topLevel, missing: notDefinedAnywhere }; +const handlerList = [helperFn, DefaultThing]; +target.cb = topLevel; +const aliasFn = topLevel; + +// Calls with interesting callees. +;(topLevel)(cfg); +NS.helper.deep(1); +"literal".includes('x'); +[1, 2].map(String); +obj?.optMethod?.(3); +import('./dynamic-module'); +new NS.Widget(makeArg()); +new Map(); +super_weird?.(); diff --git a/__tests__/kernel-scaffold.test.ts b/__tests__/kernel-scaffold.test.ts index 70bd224..0856012 100644 --- a/__tests__/kernel-scaffold.test.ts +++ b/__tests__/kernel-scaffold.test.ts @@ -124,8 +124,11 @@ describe.skipIf(!kernelBuilt)('kernel scaffold', () => { [file.id, 'helper'], ]); for (const r of calls) { - expect(r.filePath).toBe('src/utils.ts'); - expect(r.language).toBe('typescript'); + // No denormalized filePath/language at the extraction seam — the wasm + // extractors leave them unset (the store fills them, `?? filePath`), + // and the kernel matches that exactly (see decode.ts). + expect(r.filePath).toBeUndefined(); + expect(r.language).toBeUndefined(); expect(r.line).toBeGreaterThan(0); } }); diff --git a/__tests__/kernel-tsjs-parity.test.ts b/__tests__/kernel-tsjs-parity.test.ts new file mode 100644 index 0000000..f781fdc --- /dev/null +++ b/__tests__/kernel-tsjs-parity.test.ts @@ -0,0 +1,119 @@ +/** + * Kernel↔wasm TS/JS extraction parity (R2 of the kernel migration). + * + * Asserts the native walker (codegraph-kernel/src/tsjs/) produces the SAME + * ExtractionResult as the wasm TreeSitterExtractor — nodes, edges, and + * unresolved refs compared as canonicalized multisets — over: + * - the checked-in torture fixtures (every ported feature: components/HOCs, + * stores, RTK, vuex, fn-refs, value-ref shadowing, decorators, enums, + * type-alias members/tuple contracts, re-exports, JSX, field methods), and + * - this repo's own extraction sources (real-world TS). + * + * The full-repo sweep lives in scripts/kernel-parity.mjs (excalidraw et al., + * run for the §5 gate); this suite keeps the invariant alive in `npm test`. + * Skips when no kernel binary is staged; CODEGRAPH_KERNEL_EXPECT=1 turns that + * into a failure (wired in kernel-scaffold.test.ts). + */ + +import { describe, it, expect, beforeAll, beforeEach, afterEach } from 'vitest'; +import * as fs from 'fs'; +import * as path from 'path'; +import { extractFromSource } from '../src/extraction'; +import { initGrammars, loadGrammarsForLanguages } from '../src/extraction/grammars'; +import { tryKernelExtract, resetKernelForTests } from '../src/extraction/kernel'; +import type { ExtractionResult, Language } from '../src/types'; + +const KERNEL_PATH = path.join( + __dirname, + '..', + 'codegraph-kernel', + 'prebuilds', + `${process.platform}-${process.arch}`, + 'codegraph-kernel.node' +); +const kernelBuilt = fs.existsSync(KERNEL_PATH); + +const FIXTURE_DIR = path.join(__dirname, 'fixtures', 'kernel-parity'); +const REAL_SOURCES = [ + 'src/extraction/kernel/loader.ts', + 'src/extraction/kernel/decode.ts', + 'src/extraction/parse-pool.ts', + 'src/extraction/function-ref.ts', + 'src/mcp/tools.ts', +]; + +function canon(result: ExtractionResult): { nodes: string[]; edges: string[]; refs: string[] } { + return { + nodes: result.nodes + .map(({ updatedAt: _u, ...n }) => JSON.stringify(n, Object.keys(n).sort())) + .sort(), + edges: result.edges.map((e) => JSON.stringify(e, Object.keys(e).sort())).sort(), + refs: result.unresolvedReferences + .map((r) => JSON.stringify(r, Object.keys(r).sort())) + .sort(), + }; +} + +const ENV_KEYS = ['CODEGRAPH_KERNEL', 'CODEGRAPH_KERNEL_LANGS'] as const; +let savedEnv: Record; + +describe.skipIf(!kernelBuilt)('kernel TS/JS extraction parity', () => { + beforeAll(async () => { + await initGrammars(); + await loadGrammarsForLanguages(['typescript', 'tsx', 'javascript', 'jsx']); + }); + + beforeEach(() => { + savedEnv = Object.fromEntries(ENV_KEYS.map((k) => [k, process.env[k]])); + resetKernelForTests(); + }); + + afterEach(() => { + for (const k of ENV_KEYS) { + if (savedEnv[k] === undefined) delete process.env[k]; + else process.env[k] = savedEnv[k]; + } + resetKernelForTests(); + }); + + function assertParity(filePath: string, source: string, language: Language): void { + process.env.CODEGRAPH_KERNEL_LANGS = 'all'; + delete process.env.CODEGRAPH_KERNEL; + const viaKernel = tryKernelExtract(filePath, source, language); + expect(viaKernel, `kernel extraction failed for ${filePath}`).not.toBeNull(); + + process.env.CODEGRAPH_KERNEL = '0'; + const viaWasm = extractFromSource(filePath, source, language); + delete process.env.CODEGRAPH_KERNEL; + + const k = canon(viaKernel!); + const w = canon(viaWasm); + expect(k.nodes, `${filePath}: nodes`).toEqual(w.nodes); + expect(k.edges, `${filePath}: edges`).toEqual(w.edges); + expect(k.refs, `${filePath}: refs`).toEqual(w.refs); + // Meaningful comparison, not empty-vs-empty. + expect(viaWasm.nodes.length).toBeGreaterThan(3); + } + + it('torture fixture (tsx): components, stores, RTK, fn-refs, value-refs, decorators', () => { + const file = path.join(FIXTURE_DIR, 'torture.tsx'); + assertParity('fixtures/torture.tsx', fs.readFileSync(file, 'utf8'), 'tsx'); + }); + + it('torture fixture (js): field methods, wrappers, vuex module shape', () => { + const file = path.join(FIXTURE_DIR, 'torture.js'); + assertParity('fixtures/torture.js', fs.readFileSync(file, 'utf8'), 'javascript'); + }); + + it.each(REAL_SOURCES)('real source parity: %s', (rel) => { + const file = path.join(__dirname, '..', rel); + assertParity(rel, fs.readFileSync(file, 'utf8'), 'typescript'); + }); + + it('typescript fixture parsed as plain typescript variant', () => { + // Same content through the non-tsx grammar exercises the typescript + // (vs tsx) LangSpec pairing. + const file = path.join(__dirname, '..', 'src/extraction/kernel/index.ts'); + assertParity('src/extraction/kernel/index.ts', fs.readFileSync(file, 'utf8'), 'typescript'); + }); +}); diff --git a/codegraph-kernel/Cargo.lock b/codegraph-kernel/Cargo.lock index ee255a5..5f6243f 100644 --- a/codegraph-kernel/Cargo.lock +++ b/codegraph-kernel/Cargo.lock @@ -49,8 +49,8 @@ dependencies = [ "napi", "napi-build", "napi-derive", + "regex", "sha2", - "streaming-iterator", "tree-sitter", "tree-sitter-javascript", "tree-sitter-typescript", diff --git a/codegraph-kernel/Cargo.toml b/codegraph-kernel/Cargo.toml index 3a89ece..1257392 100644 --- a/codegraph-kernel/Cargo.toml +++ b/codegraph-kernel/Cargo.toml @@ -13,8 +13,8 @@ crate-type = ["cdylib"] napi = { version = "3", default-features = false, features = ["napi8"] } napi-derive = "3" tree-sitter = "0.25" -streaming-iterator = "0.1" sha2 = "0.10" +regex = "1" # Grammars — MUST stay revision-matched with the wasm grammars the fallback # path loads (tree-sitter-wasms npm package / src/extraction/wasm/). The diff --git a/codegraph-kernel/queries/javascript.scm b/codegraph-kernel/queries/javascript.scm deleted file mode 100644 index b887211..0000000 --- a/codegraph-kernel/queries/javascript.scm +++ /dev/null @@ -1,12 +0,0 @@ -; Seed query for the R1 scaffold (JavaScript / JSX grammar) — smoke-level -; coverage only; R2 replaces this with the full port. See typescript.scm for -; the capture convention. - -(class_declaration name: (identifier) @name) @def.class -(function_declaration name: (identifier) @name) @def.function -(generator_function_declaration name: (identifier) @name) @def.function -(method_definition name: (property_identifier) @name) @def.method - -(call_expression function: (identifier) @ref.calls) -(call_expression function: (member_expression property: (property_identifier) @ref.calls)) -(new_expression constructor: (identifier) @ref.instantiates) diff --git a/codegraph-kernel/queries/typescript.scm b/codegraph-kernel/queries/typescript.scm deleted file mode 100644 index 5692108..0000000 --- a/codegraph-kernel/queries/typescript.scm +++ /dev/null @@ -1,21 +0,0 @@ -; Seed query for the R1 scaffold — smoke-level coverage that proves the -; buffer contract and emitter mechanics end to end. NOT extraction parity: -; R2 replaces this with the full TypeScript/TSX port. -; -; Capture convention (see emitter.rs): -; @def. — the declaration node; pairs with @name in the pattern -; @name — the declaration's name node -; @ref. — a reference; the capture's own text is the name - -(class_declaration name: (type_identifier) @name) @def.class -(abstract_class_declaration name: (type_identifier) @name) @def.class -(interface_declaration name: (type_identifier) @name) @def.interface -(enum_declaration name: (identifier) @name) @def.enum -(type_alias_declaration name: (type_identifier) @name) @def.type_alias -(function_declaration name: (identifier) @name) @def.function -(generator_function_declaration name: (identifier) @name) @def.function -(method_definition name: (property_identifier) @name) @def.method - -(call_expression function: (identifier) @ref.calls) -(call_expression function: (member_expression property: (property_identifier) @ref.calls)) -(new_expression constructor: (identifier) @ref.instantiates) diff --git a/codegraph-kernel/src/buffers.rs b/codegraph-kernel/src/buffers.rs index f1b6594..8fe3479 100644 --- a/codegraph-kernel/src/buffers.rs +++ b/codegraph-kernel/src/buffers.rs @@ -316,6 +316,15 @@ impl Tables { } } +/// One file's encoded tables, ready to hand across the JS boundary. +pub struct EmitOut { + pub meta: Vec, + pub nodes: Vec, + pub edges: Vec, + pub refs: Vec, + pub arena: Vec, +} + pub fn build_meta(t: &Tables, arena_len: u32, errors_json: StrRef, duration_ms: f64) -> Vec { let mut m = Vec::with_capacity(META_SIZE); m.push(KERNEL_ABI_VERSION); diff --git a/codegraph-kernel/src/emitter.rs b/codegraph-kernel/src/emitter.rs deleted file mode 100644 index b390659..0000000 --- a/codegraph-kernel/src/emitter.rs +++ /dev/null @@ -1,300 +0,0 @@ -//! Generic query-driven emitter: parse the file, run the language's `.scm` -//! query, and emit flat rows. The whole tree walk happens native-side; the -//! only JS boundary crossing is the returned buffers. -//! -//! Mechanics mirrored from `TreeSitterExtractor` (src/extraction/tree-sitter.ts): -//! - node row 0 is the file node (`file:`, endLine = newline count + 1, -//! isExported present+false — byte-parity with the TS file node); -//! - definitions form a scope stack by byte-range nesting; qualifiedName is -//! the stack's names joined with `::` (file excluded); -//! - every definition gets a `contains` edge from its parent scope (the -//! file node when top-level); -//! - references attach to the innermost enclosing definition, falling back -//! to the file node — same as the TS extractor's nodeStack semantics; -//! - definitions with empty names are skipped (issue #42 semantics). - -use crate::buffers::{ - build_meta, edge_kind_index, node_kind_index, Arena, BoolFlags, EdgeRow, NodeRow, RefRow, - Tables, FLAG_IS_EXPORTED, FUNCTION_REF_CODE, NODE_KINDS, NONE, NONE_STR, -}; -use crate::ids; -use crate::langs::LangSpec; -use streaming_iterator::StreamingIterator; -use tree_sitter::{Node, Parser, QueryCursor}; - -pub struct EmitOut { - pub meta: Vec, - pub nodes: Vec, - pub edges: Vec, - pub refs: Vec, - pub arena: Vec, -} - -/// What a query capture name means. Resolved once per query. -#[derive(Clone, Copy)] -enum Role { - /// `@def.` — value is the NODE_KINDS index. - Def(u8), - /// `@name` — the paired definition's name node. - Name, - /// `@ref.` / `@ref.function_ref` — value is the wire code. - Ref(u8), - /// Helper captures (`@_anchor` etc.) — ignored. - Ignore, -} - -fn resolve_roles(capture_names: &[&str], lang: &str) -> Result, String> { - capture_names - .iter() - .map(|name| { - if let Some(kind) = name.strip_prefix("def.") { - let idx = node_kind_index(kind) - .ok_or_else(|| format!("{lang}: unknown NodeKind in capture @{name}"))?; - Ok(Role::Def(idx)) - } else if let Some(kind) = name.strip_prefix("ref.") { - if kind == "function_ref" { - return Ok(Role::Ref(FUNCTION_REF_CODE)); - } - let idx = edge_kind_index(kind) - .ok_or_else(|| format!("{lang}: unknown EdgeKind in capture @{name}"))?; - Ok(Role::Ref(idx)) - } else if *name == "name" { - Ok(Role::Name) - } else { - Ok(Role::Ignore) - } - }) - .collect() -} - -struct Def { - kind: u8, - name_start: usize, - name_end: usize, - start_byte: usize, - end_byte: usize, - start_line: u32, - end_line: u32, - start_column: u32, - end_column: u32, - /// Node-table row index, assigned during the scope sweep. - row: u32, -} - -struct RefCap { - kind: u8, - name_start: usize, - name_end: usize, - start_byte: usize, - line: u32, - column: u32, -} - -pub fn extract(file_path: &str, source: &str, spec: &LangSpec) -> Result { - let t0 = std::time::Instant::now(); - - let mut parser = Parser::new(); - parser - .set_language(spec.language()) - .map_err(|e| format!("set_language({}) failed: {e}", spec.name))?; - let tree = parser - .parse(source, None) - .ok_or_else(|| "parser returned null tree".to_string())?; - let root = tree.root_node(); - - let query = spec.query()?; - let roles = resolve_roles(&query.capture_names(), spec.name)?; - - // ---- Collect definition + reference captures from the query. ---- - let mut defs: Vec = Vec::new(); - let mut refs: Vec = Vec::new(); - // A node can match several patterns (e.g. nested alternations); first - // pattern wins, mirroring the TS walk's one-node-one-symbol behaviour. - let mut seen_defs = std::collections::HashSet::::new(); - - let mut cursor = QueryCursor::new(); - let mut matches = cursor.matches(query, root, source.as_bytes()); - while let Some(m) = matches.next() { - let mut def_node: Option<(Node, u8)> = None; - let mut name_node: Option = None; - for cap in m.captures { - match roles[cap.index as usize] { - Role::Def(kind) => def_node = Some((cap.node, kind)), - Role::Name => name_node = Some(cap.node), - Role::Ref(kind) => { - let p = cap.node.start_position(); - refs.push(RefCap { - kind, - name_start: cap.node.start_byte(), - name_end: cap.node.end_byte(), - start_byte: cap.node.start_byte(), - line: p.row as u32 + 1, - column: p.column as u32, - }); - } - Role::Ignore => {} - } - } - if let (Some((node, kind)), Some(name)) = (def_node, name_node) { - // Empty names are not meaningful symbols (issue #42). - if name.end_byte() > name.start_byte() && seen_defs.insert(node.id()) { - let sp = node.start_position(); - let ep = node.end_position(); - defs.push(Def { - kind, - name_start: name.start_byte(), - name_end: name.end_byte(), - start_byte: node.start_byte(), - end_byte: node.end_byte(), - start_line: sp.row as u32 + 1, - end_line: ep.row as u32 + 1, - start_column: sp.column as u32, - end_column: ep.column as u32, - row: 0, - }); - } - } - } - - // Deterministic pre-order regardless of query-match ordering. - defs.sort_by(|a, b| { - a.start_byte - .cmp(&b.start_byte) - .then(b.end_byte.cmp(&a.end_byte)) - }); - refs.sort_by_key(|r| r.start_byte); - - // ---- Emit rows: file node first, then the scope-stack sweep. ---- - let mut arena = Arena::default(); - let mut tables = Tables::default(); - - let line_count = source.bytes().filter(|b| *b == b'\n').count() as u32 + 1; - let base_name = file_path.rsplit(['/', '\\']).next().unwrap_or(file_path); - let mut file_flags = BoolFlags::default(); - file_flags.set(FLAG_IS_EXPORTED, false); - let file_id = arena.put(&ids::file_node_id(file_path)); - let file_name = arena.put(base_name); - let file_qn = arena.put(file_path); - tables.push_node(&NodeRow { - kind: node_kind_index("file").unwrap(), - visibility: 0, - flags: file_flags, - start_line: 1, - end_line: line_count, - start_column: 0, - end_column: 0, - name: file_name, - qualified_name: file_qn, - id: file_id, - docstring: NONE_STR, - signature: NONE_STR, - decorators: NONE_STR, - type_parameters: NONE_STR, - return_type: NONE_STR, - extra_json: NONE_STR, - }); - - // Merged sweep over definitions and references in byte order, maintaining - // the scope stack (indices into `defs`). - let mut stack: Vec = Vec::new(); - let mut ref_i = 0usize; - - fn pop_to(stack: &mut Vec, defs: &[Def], byte: usize) { - while let Some(&top) = stack.last() { - if defs[top].end_byte <= byte { - stack.pop(); - } else { - break; - } - } - } - - let emit_ref = |r: &RefCap, stack: &[usize], defs: &[Def], arena: &mut Arena, tables: &mut Tables| { - let from_idx = stack.last().map(|&i| defs[i].row).unwrap_or(0); - let name = arena.put(&source[r.name_start..r.name_end]); - tables.push_ref(&RefRow { - from_idx, - kind: r.kind, - line: r.line, - column: r.column, - reference_name: name, - candidates: NONE_STR, - from_id_str: NONE_STR, - }); - }; - - for i in 0..defs.len() { - let def_start = defs[i].start_byte; - while ref_i < refs.len() && refs[ref_i].start_byte < def_start { - pop_to(&mut stack, &defs, refs[ref_i].start_byte); - emit_ref(&refs[ref_i], &stack, &defs, &mut arena, &mut tables); - ref_i += 1; - } - pop_to(&mut stack, &defs, def_start); - - let name = &source[defs[i].name_start..defs[i].name_end]; - let kind_str = NODE_KINDS[defs[i].kind as usize]; - // qualifiedName = enclosing definition names + own name, `::`-joined - // (buildQualifiedName semantics; file node excluded). - let mut qn = String::new(); - for &s in stack.iter() { - qn.push_str(&source[defs[s].name_start..defs[s].name_end]); - qn.push_str("::"); - } - qn.push_str(name); - - let id = ids::node_id(file_path, kind_str, name, defs[i].start_line); - let id_ref = arena.put(&id); - let name_ref = arena.put(name); - let qn_ref = arena.put(&qn); - let row = tables.push_node(&NodeRow { - kind: defs[i].kind, - visibility: 0, - flags: BoolFlags::default(), - start_line: defs[i].start_line, - end_line: defs[i].end_line, - start_column: defs[i].start_column, - end_column: defs[i].end_column, - name: name_ref, - qualified_name: qn_ref, - id: id_ref, - docstring: NONE_STR, - signature: NONE_STR, - decorators: NONE_STR, - type_parameters: NONE_STR, - return_type: NONE_STR, - extra_json: NONE_STR, - }); - defs[i].row = row; - - let parent_row = stack.last().map(|&s| defs[s].row).unwrap_or(0); - tables.push_edge(&EdgeRow { - source_idx: parent_row, - target_idx: row, - kind: edge_kind_index("contains").unwrap(), - provenance: 0, - line: NONE, - column: NONE, - metadata_json: NONE_STR, - source_id_str: NONE_STR, - target_id_str: NONE_STR, - }); - - stack.push(i); - } - while ref_i < refs.len() { - pop_to(&mut stack, &defs, refs[ref_i].start_byte); - emit_ref(&refs[ref_i], &stack, &defs, &mut arena, &mut tables); - ref_i += 1; - } - - let duration_ms = t0.elapsed().as_secs_f64() * 1000.0; - let meta = build_meta(&tables, arena.len(), NONE_STR, duration_ms); - Ok(EmitOut { - meta, - nodes: tables.nodes, - edges: tables.edges, - refs: tables.refs, - arena: arena.into_vec(), - }) -} diff --git a/codegraph-kernel/src/langs.rs b/codegraph-kernel/src/langs.rs index 159447c..2152301 100644 --- a/codegraph-kernel/src/langs.rs +++ b/codegraph-kernel/src/langs.rs @@ -1,78 +1,27 @@ -//! Per-language specs: grammar + `.scm` query + (later) per-language config. +//! Grammar registry: codegraph `Language` string → native tree-sitter grammar. //! -//! Tier-1 languages are meant to be *mostly* a query file plus a small config -//! here; logic queries can't express stays TS-side as a per-language `post()` -//! hook over the returned buffers (see `src/extraction/kernel/route.ts`). +//! Mirrors the wasm side's `WASM_GRAMMAR_FILES` mapping (src/extraction/ +//! grammars.ts): `tsx` and `jsx` reuse another language's grammar exactly the +//! way the wasm map does. The kernel-grammar-parity test asserts each entry is +//! built from the SAME grammar revision as the vendored wasm — bump the crate +//! and the wasm together. //! -//! Language strings are codegraph `Language` values (src/types.ts), not -//! grammar names — `tsx` and `jsx` are separate entries that reuse another -//! entry's grammar exactly like `WASM_GRAMMAR_FILES` does on the wasm path. +//! (R1 shipped a generic `.scm`-query emitter here; R2 replaced it with the +//! bespoke per-language walker — see tsjs/ and the migration plan §3a — because +//! extraction parity needs logic queries can't express. New languages add a +//! grammar entry + a walker module.) -use std::sync::OnceLock; -use tree_sitter::{Language, Query}; +use tree_sitter::Language; -pub struct LangSpec { - /// codegraph Language string (src/types.ts). - pub name: &'static str, - get_language: fn() -> Language, - query_src: &'static str, - language: OnceLock, - query: OnceLock>, -} +/// Languages this kernel binary can extract (reported by contractInfo; +/// TS-side routing policy decides what actually routes). +pub const LANGUAGES: [&str; 4] = ["typescript", "tsx", "javascript", "jsx"]; -impl LangSpec { - const fn new(name: &'static str, get_language: fn() -> Language, query_src: &'static str) -> Self { - LangSpec { - name, - get_language, - query_src, - language: OnceLock::new(), - query: OnceLock::new(), - } - } - - pub fn language(&self) -> &Language { - self.language.get_or_init(self.get_language) - } - - pub fn query(&self) -> Result<&Query, String> { - self.query - .get_or_init(|| { - Query::new(self.language(), self.query_src) - .map_err(|e| format!("query compile failed for {}: {e}", self.name)) - }) - .as_ref() - .map_err(|e| e.clone()) +pub fn grammar_for(language: &str) -> Option { + match language { + "typescript" => Some(tree_sitter_typescript::LANGUAGE_TYPESCRIPT.into()), + "tsx" => Some(tree_sitter_typescript::LANGUAGE_TSX.into()), + "javascript" | "jsx" => Some(tree_sitter_javascript::LANGUAGE.into()), + _ => None, } } - -fn ts_language() -> Language { - tree_sitter_typescript::LANGUAGE_TYPESCRIPT.into() -} - -fn tsx_language() -> Language { - tree_sitter_typescript::LANGUAGE_TSX.into() -} - -fn js_language() -> Language { - tree_sitter_javascript::LANGUAGE.into() -} - -static TYPESCRIPT: LangSpec = LangSpec::new( - "typescript", - ts_language, - include_str!("../queries/typescript.scm"), -); -static TSX: LangSpec = LangSpec::new("tsx", tsx_language, include_str!("../queries/typescript.scm")); -static JAVASCRIPT: LangSpec = LangSpec::new( - "javascript", - js_language, - include_str!("../queries/javascript.scm"), -); -static JSX: LangSpec = LangSpec::new("jsx", js_language, include_str!("../queries/javascript.scm")); - -pub static ALL: [&LangSpec; 4] = [&TYPESCRIPT, &TSX, &JAVASCRIPT, &JSX]; - -pub fn spec_for(language: &str) -> Option<&'static LangSpec> { - ALL.iter().find(|s| s.name == language).copied() -} diff --git a/codegraph-kernel/src/lib.rs b/codegraph-kernel/src/lib.rs index 9d7e6c4..e1a4502 100644 --- a/codegraph-kernel/src/lib.rs +++ b/codegraph-kernel/src/lib.rs @@ -9,13 +9,17 @@ //! Calls are synchronous by design: the existing `ParseWorkerPool` workers //! already parallelize per-file, so each worker thread drives its own kernel //! call (do NOT rebuild the pool on the Rust side — see the migration plan §3). +//! +//! Per-language extraction lives in a dedicated walker module (tsjs/ for +//! typescript/tsx/javascript/jsx) that mirrors the TS extractor for behavioral +//! parity — verified by scripts/kernel-parity.mjs and the §5 gate. #![deny(clippy::all)] mod buffers; -mod emitter; mod ids; mod langs; +mod tsjs; use napi::bindgen_prelude::*; use napi_derive::napi; @@ -63,14 +67,13 @@ pub fn contract_info() -> ContractInfo { kernel_version: env!("CARGO_PKG_VERSION").to_string(), node_kinds: buffers::NODE_KINDS.iter().map(|s| s.to_string()).collect(), edge_kinds: buffers::EDGE_KINDS.iter().map(|s| s.to_string()).collect(), - languages: langs::ALL.iter().map(|s| s.name.to_string()).collect(), + languages: langs::LANGUAGES.iter().map(|s| s.to_string()).collect(), } } #[napi] pub fn grammar_info(language: String) -> Option { - let spec = langs::spec_for(&language)?; - let lang = spec.language(); + let lang = langs::grammar_for(&language)?; let node_kind_count = lang.node_kind_count(); let field_count = lang.field_count(); let node_kinds = (0..node_kind_count) @@ -91,9 +94,7 @@ pub fn grammar_info(language: String) -> Option { #[napi] pub fn extract_file(file_path: String, content: String, language: String) -> Result { - let spec = langs::spec_for(&language) - .ok_or_else(|| Error::from_reason(format!("kernel does not support language: {language}")))?; - let out = emitter::extract(&file_path, &content, spec).map_err(Error::from_reason)?; + let out = tsjs::extract(&file_path, &content, &language).map_err(Error::from_reason)?; Ok(ExtractBuffers { meta: out.meta.into(), nodes: out.nodes.into(), diff --git a/codegraph-kernel/src/tsjs/docstring.rs b/codegraph-kernel/src/tsjs/docstring.rs new file mode 100644 index 0000000..f4861a9 --- /dev/null +++ b/codegraph-kernel/src/tsjs/docstring.rs @@ -0,0 +1,140 @@ +//! getPrecedingDocstring / cleanCommentMarkers — faithful port of +//! src/extraction/tree-sitter-helpers.ts (#780 wrapper-climb semantics). + +use regex::Regex; +use std::sync::OnceLock; +use tree_sitter::Node; + +/// DOCSTRING_WRAPPER_TYPES (tree-sitter-helpers.ts). +fn is_wrapper(kind: &str) -> bool { + matches!( + kind, + "export_statement" + | "decorated_definition" + | "lexical_declaration" + | "variable_declaration" + | "variable_declarator" + | "ambient_declaration" + ) +} + +fn is_comment(kind: &str) -> bool { + matches!( + kind, + "comment" | "line_comment" | "block_comment" | "documentation_comment" + ) +} + +struct Cleaners { + block_open: Regex, + block_close: Regex, + lua_open: Regex, + lua_close: Regex, + paren_star_open: Regex, + paren_star_close: Regex, + brace_open: Regex, + brace_close: Regex, + slashes: Regex, + dashes: Regex, + hash: Regex, + percent: Regex, + star_cont: Regex, +} + +fn cleaners() -> &'static Cleaners { + static C: OnceLock = OnceLock::new(); + C.get_or_init(|| Cleaners { + block_open: Regex::new(r"^/\*+!?").unwrap(), + block_close: Regex::new(r"\*+/$").unwrap(), + lua_open: Regex::new(r"^--\[=*\[").unwrap(), + lua_close: Regex::new(r"\]=*\]$").unwrap(), + paren_star_open: Regex::new(r"^\(\*").unwrap(), + paren_star_close: Regex::new(r"\*\)$").unwrap(), + brace_open: Regex::new(r"^\{").unwrap(), + brace_close: Regex::new(r"\}$").unwrap(), + slashes: Regex::new(r"(?m)^//[/!]?\s?").unwrap(), + dashes: Regex::new(r"(?m)^--\s?").unwrap(), + hash: Regex::new(r"(?m)^#\s?").unwrap(), + percent: Regex::new(r"(?m)^%+\s?").unwrap(), + star_cont: Regex::new(r"(?m)^\s*\*\s?").unwrap(), + }) +} + +/// cleanCommentMarkers — strip comment syntax, keep the prose. +pub fn clean_comment_markers(comment: &str) -> String { + let c = cleaners(); + let mut s = comment.trim().to_string(); + if s.starts_with("/*") { + s = c.block_open.replace(&s, "").into_owned(); + s = c.block_close.replace(&s, "").into_owned(); + } else if s.starts_with("--[") { + s = c.lua_open.replace(&s, "").into_owned(); + s = c.lua_close.replace(&s, "").into_owned(); + } else if s.starts_with("(*") { + s = c.paren_star_open.replace(&s, "").into_owned(); + s = c.paren_star_close.replace(&s, "").into_owned(); + } else if s.starts_with('{') { + s = c.brace_open.replace(&s, "").into_owned(); + s = c.brace_close.replace(&s, "").into_owned(); + } + s = c.slashes.replace_all(&s, "").into_owned(); + s = c.dashes.replace_all(&s, "").into_owned(); + s = c.hash.replace_all(&s, "").into_owned(); + s = c.percent.replace_all(&s, "").into_owned(); + s = c.star_cont.replace_all(&s, "").into_owned(); + s.trim().to_string() +} + +/// getPrecedingDocstring — collect the comment run immediately preceding the +/// node (climbing out of declaration wrappers first), cleaned and joined. +/// Returns None when there is no preceding comment (a PRESENT-but-empty +/// docstring after cleaning still returns Some(""), matching the TS helper). +pub fn preceding_docstring(node: Node, src: &str) -> Option { + let mut anchor = node; + while let Some(parent) = anchor.parent() { + if is_wrapper(parent.kind()) { + anchor = parent; + } else { + break; + } + } + + let mut comments: Vec<&str> = Vec::new(); + let mut sibling = anchor.prev_named_sibling(); + while let Some(s) = sibling { + if is_comment(s.kind()) { + comments.push(&src[s.byte_range()]); + sibling = s.prev_named_sibling(); + } else { + break; + } + } + if comments.is_empty() { + return None; + } + comments.reverse(); // collected nearest-first; TS unshifts to keep source order + Some( + comments + .iter() + .map(|c| clean_comment_markers(c)) + .collect::>() + .join("\n") + .trim() + .to_string(), + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn strips_line_and_block_markers() { + assert_eq!(clean_comment_markers("// hello"), "hello"); + assert_eq!(clean_comment_markers("/// doc line"), "doc line"); + assert_eq!( + clean_comment_markers("/**\n * Adds things.\n * @param a first\n */"), + "Adds things.\n@param a first" + ); + } +} diff --git a/codegraph-kernel/src/tsjs/extractors.rs b/codegraph-kernel/src/tsjs/extractors.rs new file mode 100644 index 0000000..e9e3c13 --- /dev/null +++ b/codegraph-kernel/src/tsjs/extractors.rs @@ -0,0 +1,1332 @@ +//! The extract_* family — continuation of the Walker impl (see mod.rs for the +//! porting contract). Each function mirrors its namesake in +//! src/extraction/tree-sitter.ts; TS-file line references are as of the R2 +//! port. Bug-for-bug fidelity is deliberate — fix the TS side first. + +use super::util; +use super::{ + body_of, is_builtin_type, is_literal_receiver, is_react_hoc, is_variable_type, + is_vue_collection_name, Extra, Scope, Walker, +}; +use crate::buffers::edge_kind_index; +use tree_sitter::Node; + +impl<'t> Walker<'t> { + // --- extractFunction -------------------------------------------------------- + + pub(super) fn extract_function(&mut self, node: Node<'t>, name_override: Option) { + let mut name = name_override + .clone() + .unwrap_or_else(|| self.extract_name(node)); + + // Arrow/function-expression values: resolve the name from the parent + // variable_declarator (`export const useAuth = () => {}`). + if name_override.is_none() + && name == "" + && matches!(node.kind(), "arrow_function" | "function_expression") + { + if let Some(parent) = node.parent() { + if parent.kind() == "variable_declarator" { + if let Some(var_name) = parent.child_by_field_name("name") { + name = self.text(var_name).to_string(); + } + } + } + } + if name == "" { + // Still walk the body: module wrappers hold named inner functions + // and calls that would otherwise be lost (#528). + if let Some(body) = body_of(node) { + self.visit_function_body(body); + } + return; + } + + let extra = Extra { + docstring: super::docstring::preceding_docstring(node, self.src), + signature: self.signature_of(node), + visibility: self.visibility_of(node), + is_exported: Some(self.is_exported(node)), + is_async: Some(self.is_async(node)), + is_static: self.is_static(node), + ..Extra::default() + }; + let Some(row) = self.create_node("function", &name, node, extra) else { + return; + }; + + self.extract_type_annotations(node, row); + self.extract_decorators_for(node, row); + + self.stack.push(Scope { row, kind: "function", name }); + if let Some(body) = body_of(node) { + self.visit_function_body(body); + } + self.stack.pop(); + } + + // --- reactComponentHoc / extractReactComponentNode (#841) -------------------- + + /// Some(inner) when the initializer is a recognized component wrapper — + /// inner is the inline render function, or None for `styled.x`/`memo(Ref)`. + /// Outer None = not a component wrapper. + fn react_component_hoc(&self, value: Node<'t>) -> Option>> { + if value.kind() != "call_expression" { + return None; + } + let callee = value.child_by_field_name("function")?; + let callee_text = self.text(callee); + if util::styled_callee().is_match(callee_text) { + return Some(None); + } + if !is_react_hoc(callee_text) { + return None; + } + let mut inner: Option = None; + if let Some(args) = value.child_by_field_name("arguments") { + for i in 0..args.named_child_count() { + if let Some(a) = args.named_child(i) { + if matches!(a.kind(), "arrow_function" | "function_expression") { + inner = Some(a); + break; + } + } + } + } + Some(inner) + } + + fn extract_react_component_node( + &mut self, + name: &str, + declarator: Node<'t>, + inner_fn: Option>, + extra: Extra, + ) { + let Some(row) = self.create_node("component", name, declarator, extra) else { + return; + }; + let Some(inner) = inner_fn else { return }; + self.stack.push(Scope { row, kind: "component", name: name.to_string() }); + if let Some(body) = body_of(inner) { + self.visit_function_body(body); + } + self.stack.pop(); + } + + // --- extractClass ------------------------------------------------------------ + + pub(super) fn extract_class(&mut self, node: Node<'t>) { + let resolved_body = body_of(node); // skipBodilessClass unset for TS/JS + let name = self.extract_name(node); + let extra = Extra { + docstring: super::docstring::preceding_docstring(node, self.src), + visibility: self.visibility_of(node), + is_exported: Some(self.is_exported(node)), + ..Extra::default() + }; + let Some(row) = self.create_node("class", &name, node, extra) else { + return; + }; + + self.extract_inheritance(node, row); + self.extract_decorators_for(node, row); + + self.stack.push(Scope { row, kind: "class", name }); + let body = resolved_body.unwrap_or(node); + for i in 0..body.named_child_count() { + if let Some(c) = body.named_child(i) { + self.visit_node(c); + } + } + self.stack.pop(); + } + + // --- extractMethod ------------------------------------------------------------- + + pub(super) fn extract_method(&mut self, node: Node<'t>) { + if !self.inside_class_like() { + // Object-literal methods are ephemeral: walk the body only. + if let Some(parent) = node.parent() { + if matches!(parent.kind(), "object" | "object_expression") { + if let Some(body) = body_of(node) { + self.visit_function_body(body); + } + return; + } + } + self.extract_function(node, None); + return; + } + + let name = self.extract_name(node); + let extra = Extra { + docstring: super::docstring::preceding_docstring(node, self.src), + signature: self.signature_of(node), + visibility: self.visibility_of(node), + is_async: Some(self.is_async(node)), + is_static: self.is_static(node), + ..Extra::default() // methods carry no isExported (mirrors extractMethod) + }; + let Some(row) = self.create_node("method", &name, node, extra) else { + return; + }; + + self.extract_type_annotations(node, row); + self.extract_decorators_for(node, row); + + self.stack.push(Scope { row, kind: "method", name }); + if let Some(body) = body_of(node) { + self.visit_function_body(body); + } + self.stack.pop(); + } + + // --- extractInterface / extractEnum / members ----------------------------------- + + pub(super) fn extract_interface(&mut self, node: Node<'t>) { + let name = self.extract_name(node); + let extra = Extra { + docstring: super::docstring::preceding_docstring(node, self.src), + is_exported: Some(self.is_exported(node)), + ..Extra::default() + }; + let Some(row) = self.create_node("interface", &name, node, extra) else { + return; + }; + self.extract_inheritance(node, row); + self.stack.push(Scope { row, kind: "interface", name }); + let body = body_of(node).unwrap_or(node); + for i in 0..body.named_child_count() { + if let Some(c) = body.named_child(i) { + self.visit_node(c); + } + } + self.stack.pop(); + } + + pub(super) fn extract_enum(&mut self, node: Node<'t>) { + let Some(body) = body_of(node) else { return }; + let name = self.extract_name(node); + let extra = Extra { + docstring: super::docstring::preceding_docstring(node, self.src), + visibility: self.visibility_of(node), + is_exported: Some(self.is_exported(node)), + ..Extra::default() + }; + let Some(row) = self.create_node("enum", &name, node, extra) else { + return; + }; + self.extract_inheritance(node, row); + self.stack.push(Scope { row, kind: "enum", name }); + for i in 0..body.named_child_count() { + let Some(child) = body.named_child(i) else { continue }; + if matches!(child.kind(), "property_identifier" | "enum_assignment") { + self.extract_enum_members(child); + } else { + self.visit_node(child); + } + } + self.stack.pop(); + } + + fn extract_enum_members(&mut self, node: Node<'t>) { + if let Some(name_node) = node.child_by_field_name("name") { + let name = self.text(name_node).to_string(); + self.create_node("enum_member", &name, node, Extra::default()); + return; + } + let mut found = false; + for i in 0..node.named_child_count() { + if let Some(child) = node.named_child(i) { + if matches!(child.kind(), "simple_identifier" | "identifier" | "property_identifier") { + let name = self.text(child).to_string(); + self.create_node("enum_member", &name, child, Extra::default()); + found = true; + } + } + } + if !found && node.named_child_count() == 0 { + let name = self.text(node).to_string(); + self.create_node("enum_member", &name, node, Extra::default()); + } + } + + // --- extractProperty (#808 property-classified class fields) --------------------- + + pub(super) fn extract_property(&mut self, node: Node<'t>) -> Option<(u32, String)> { + let docstring = super::docstring::preceding_docstring(node, self.src); + let visibility = self.visibility_of(node); + let is_static = Some(self.is_static(node).unwrap_or(false)); // `?? false` — always present + + let name_node = node + .child_by_field_name("name") + .or_else(|| node.child_by_field_name("property")) + .or_else(|| { + (0..node.named_child_count()) + .filter_map(|i| node.named_child(i)) + .find(|c| c.kind() == "identifier") + })?; + let name = self.text(name_node).to_string(); + + // TS/JS field definitions carry an explicit `type` field; the generic + // scan is for other languages (#808). + let type_text = node.child_by_field_name("type").map(|t| { + let raw = self.text(t); + raw.strip_prefix(':').unwrap_or(raw).trim_start().to_string() + }); + let signature = match &type_text { + Some(t) => format!("{t} {name}"), + None => name.clone(), + }; + + let row = self.create_node( + "property", + &name, + node, + Extra { docstring, signature: Some(signature), visibility, is_static, ..Extra::default() }, + )?; + self.extract_decorators_for(node, row); + self.extract_type_annotations(node, row); + Some((row, name)) + } + + // --- extractVariable (TS/JS branch) ------------------------------------------------ + + pub(super) fn extract_variable(&mut self, node: Node<'t>) { + let is_const = self.is_const_decl(node); + let kind: &'static str = if is_const { "constant" } else { "variable" }; + let docstring = super::docstring::preceding_docstring(node, self.src); + let is_exported = self.is_exported(node); // `?? false` — always present + + for i in 0..node.named_child_count() { + let Some(child) = node.named_child(i) else { continue }; + if child.kind() != "variable_declarator" { + continue; + } + let Some(name_node) = child.child_by_field_name("name") else { continue }; + let value = child.child_by_field_name("value"); + + // Destructured patterns are skipped — except RTK Query generated + // hooks (`export const { useGetXQuery } = api`). + if matches!(name_node.kind(), "object_pattern" | "array_pattern") { + if name_node.kind() == "object_pattern" + && value.map(|v| v.kind() == "identifier").unwrap_or(false) + { + self.extract_rtk_hook_bindings(name_node, is_exported); + } + continue; + } + let name = self.text(name_node).to_string(); + + // Arrow/function values extract as functions, named by the declarator. + if let Some(v) = value { + if matches!(v.kind(), "arrow_function" | "function_expression") { + self.extract_function(v, None); + continue; + } + } + + let init_signature = value.map(|v| util::init_signature(self.text(v))); + + // React HOC-wrapped components (#841), PascalCase-gated. + if let Some(v) = value { + if util::pascal_case().is_match(&name) { + if let Some(inner) = self.react_component_hoc(v) { + self.extract_react_component_node( + &name, + child, + inner, + Extra { + docstring: docstring.clone(), + signature: init_signature.clone(), + is_exported: Some(is_exported), + ..Extra::default() + }, + ); + continue; + } + } + } + + let var_row = self.create_node( + kind, + &name, + child, + Extra { + docstring: docstring.clone(), + signature: init_signature.clone(), + is_exported: Some(is_exported), + ..Extra::default() + }, + ); + if let Some(row) = var_row { + self.extract_variable_type_annotation(child, row); + } + + // Exported const object-of-functions / store shapes. + let object_of_fns: Option = match value { + Some(v) if matches!(v.kind(), "object" | "object_expression") => Some(v), + Some(v) if v.kind() == "call_expression" => self.find_initializer_returned_object(v, 0), + _ => None, + }; + let has_inline_fns = object_of_fns + .map(|o| self.object_has_inline_functions(o)) + .unwrap_or(false); + let extract_object_methods = is_exported && object_of_fns.is_some() && has_inline_fns; + + let rtk_endpoints = match value { + Some(v) if v.kind() == "call_expression" => self.find_rtk_endpoints_object(v), + _ => None, + }; + let pinia_setup = match value { + Some(v) if v.kind() == "call_expression" => self.find_pinia_setup_fn(v), + _ => None, + }; + let mut store_collections: Vec = Vec::new(); + if let Some(v) = value { + if matches!(v.kind(), "call_expression" | "new_expression") { + store_collections.extend(self.find_vue_store_collection_objects(v)); + } + } + if let Some(obj) = object_of_fns { + if !extract_object_methods + && is_vue_collection_name(&name) + && self.looks_like_vue_store_file() + { + store_collections.push(obj); + } + } + + // Walk the initializer for calls — except the object/store shapes + // whose members are extracted method-by-method below. + if let Some(v) = value { + let vk = v.kind(); + if vk != "object" + && vk != "object_expression" + && !(extract_object_methods && vk == "call_expression") + && rtk_endpoints.is_none() + && pinia_setup.is_none() + && store_collections.is_empty() + { + self.visit_function_body(v); + } + } + + if extract_object_methods { + if let Some(obj) = object_of_fns { + self.extract_object_literal_functions(obj); + } + } + if let Some(rtk) = rtk_endpoints { + self.extract_rtk_endpoints(rtk); + } + if let Some(setup) = pinia_setup { + self.extract_pinia_setup_body(setup); + } + for coll in store_collections { + self.extract_object_literal_functions(coll); + } + } + } + + /// extractRtkHookBindings — `export const { useGetXQuery } = api`. + fn extract_rtk_hook_bindings(&mut self, pattern: Node<'t>, is_exported: bool) { + for i in 0..pattern.named_child_count() { + let Some(binding) = pattern.named_child(i) else { continue }; + if binding.kind() != "shorthand_property_identifier_pattern" { + continue; + } + let name = self.text(binding).to_string(); + if !util::rtk_hook_name().is_match(&name) { + continue; + } + self.create_node( + "function", + &name, + binding, + Extra { + is_exported: Some(is_exported), + signature: Some("= RTK Query generated hook".to_string()), + ..Extra::default() + }, + ); + } + } + + // --- object-literal / store helpers ------------------------------------------------- + + pub(super) fn extract_object_literal_functions(&mut self, obj: Node<'t>) { + for i in 0..obj.named_child_count() { + let Some(member) = obj.named_child(i) else { continue }; + if member.kind() == "pair" { + let key = member.child_by_field_name("key"); + let value = member.child_by_field_name("value"); + if let (Some(k), Some(v)) = (key, value) { + if matches!(v.kind(), "arrow_function" | "function_expression") { + let name = util::object_key_name(self.text(k)); + self.extract_function(v, Some(name)); + } + } + } else if member.kind() == "method_definition" { + if let Some(k) = member.child_by_field_name("name") { + let name = util::object_key_name(self.text(k)); + self.extract_function(member, Some(name)); + } + } + } + } + + fn find_initializer_returned_object(&self, call: Node<'t>, depth: u32) -> Option> { + if depth > 4 { + return None; + } + let args = call.child_by_field_name("arguments")?; + for i in 0..args.named_child_count() { + let Some(arg) = args.named_child(i) else { continue }; + if matches!(arg.kind(), "arrow_function" | "function_expression") { + if let Some(obj) = self.function_returned_object(arg) { + return Some(obj); + } + } else if arg.kind() == "call_expression" { + if let Some(obj) = self.find_initializer_returned_object(arg, depth + 1) { + return Some(obj); + } + } + } + None + } + + fn function_returned_object(&self, fn_node: Node<'t>) -> Option> { + fn as_object<'t>(n: Node<'t>) -> Option> { + match n.kind() { + "object" | "object_expression" => Some(n), + "parenthesized_expression" => { + for i in 0..n.named_child_count() { + if let Some(inner) = n.named_child(i).and_then(as_object) { + return Some(inner); + } + } + None + } + _ => None, + } + } + let body = fn_node.child_by_field_name("body")?; + if let Some(direct) = as_object(body) { + return Some(direct); + } + if body.kind() == "statement_block" { + for i in 0..body.named_child_count() { + let Some(stmt) = body.named_child(i) else { continue }; + if stmt.kind() != "return_statement" { + continue; + } + for j in 0..stmt.named_child_count() { + if let Some(obj) = stmt.named_child(j).and_then(as_object) { + return Some(obj); + } + } + } + } + None + } + + pub(super) fn object_has_inline_functions(&self, obj: Node) -> bool { + for i in 0..obj.named_child_count() { + let Some(member) = obj.named_child(i) else { continue }; + if member.kind() == "method_definition" { + return true; + } + if member.kind() == "pair" { + if let Some(v) = member.child_by_field_name("value") { + if matches!(v.kind(), "arrow_function" | "function_expression") { + return true; + } + } + } + } + false + } + + fn find_rtk_endpoints_object(&self, call: Node<'t>) -> Option> { + let callee = call.child_by_field_name("function")?; + let callee_name = match callee.kind() { + "identifier" => self.text(callee), + "member_expression" => { + let prop = callee.child_by_field_name("property").unwrap_or(callee); + self.text(prop) + } + _ => "", + }; + if callee_name != "createApi" && callee_name != "injectEndpoints" { + return None; + } + let args = call.child_by_field_name("arguments")?; + for i in 0..args.named_child_count() { + let Some(arg) = args.named_child(i) else { continue }; + if !matches!(arg.kind(), "object" | "object_expression") { + continue; + } + for j in 0..arg.named_child_count() { + let Some(member) = arg.named_child(j) else { continue }; + if member.kind() == "pair" { + let Some(key) = member.child_by_field_name("key") else { continue }; + if self.text(key) != "endpoints" { + continue; + } + if let Some(value) = member.child_by_field_name("value") { + if matches!(value.kind(), "arrow_function" | "function_expression") { + return self.function_returned_object(value); + } + } + } else if member.kind() == "method_definition" { + let Some(key) = member.child_by_field_name("name") else { continue }; + if self.text(key) != "endpoints" { + continue; + } + return self.function_returned_object(member); + } + } + } + None + } + + fn extract_rtk_endpoints(&mut self, obj: Node<'t>) { + for i in 0..obj.named_child_count() { + let Some(member) = obj.named_child(i) else { continue }; + if member.kind() != "pair" { + continue; + } + let key = member.child_by_field_name("key"); + let value = member.child_by_field_name("value"); + let (Some(key), Some(value)) = (key, value) else { continue }; + if value.kind() != "call_expression" { + continue; + } + let Some(callee) = value.child_by_field_name("function") else { continue }; + if callee.kind() != "member_expression" { + continue; + } + let method = self.text(callee.child_by_field_name("property").unwrap_or(callee)); + if method != "query" && method != "mutation" && method != "infiniteQuery" { + continue; + } + let key_name = util::object_key_name(self.text(key)); + if let Some(handler) = self.rtk_endpoint_handler(value) { + self.extract_function(handler, Some(key_name)); + } else { + // Config-only endpoint: bare node spanning the builder call. + let (sig, _) = util::slice_utf16(self.text(value), 80); + let row = self.create_node( + "function", + &key_name, + value, + Extra { signature: Some(sig), ..Extra::default() }, + ); + if let Some(row) = row { + self.stack.push(Scope { row, kind: "function", name: key_name }); + self.visit_function_body(value); + self.stack.pop(); + } + } + } + } + + fn rtk_endpoint_handler(&self, call: Node<'t>) -> Option> { + let args = call.child_by_field_name("arguments")?; + for i in 0..args.named_child_count() { + let Some(arg) = args.named_child(i) else { continue }; + if !matches!(arg.kind(), "object" | "object_expression") { + continue; + } + let mut query_fn: Option = None; + let mut query: Option = None; + let mut first_fn: Option = None; + for j in 0..arg.named_child_count() { + let Some(member) = arg.named_child(j) else { continue }; + let mut fn_node: Option = None; + let mut key_name = ""; + if member.kind() == "pair" { + if let Some(v) = member.child_by_field_name("value") { + if matches!(v.kind(), "arrow_function" | "function_expression") { + fn_node = Some(v); + if let Some(k) = member.child_by_field_name("key") { + key_name = self.text(k); + } + } + } + } else if member.kind() == "method_definition" { + fn_node = Some(member); + if let Some(k) = member.child_by_field_name("name") { + key_name = self.text(k); + } + } + let Some(f) = fn_node else { continue }; + if key_name == "queryFn" { + query_fn = Some(f); + } else if key_name == "query" { + query = Some(f); + } + if first_fn.is_none() { + first_fn = Some(f); + } + } + if let Some(f) = query_fn.or(query).or(first_fn) { + return Some(f); + } + } + None + } + + pub(super) fn looks_like_vue_store_file(&mut self) -> bool { + if let Some(v) = self.vue_store_file { + return v; + } + let mut seen: std::collections::HashSet<&str> = std::collections::HashSet::new(); + for m in util::vue_store_signal().find_iter(self.src) { + seen.insert(m.as_str()); + if seen.len() >= 2 { + break; + } + } + let v = seen.len() >= 2; + self.vue_store_file = Some(v); + v + } + + fn find_vue_store_collection_objects(&self, call: Node<'t>) -> Vec> { + let callee = call + .child_by_field_name("function") + .or_else(|| call.child_by_field_name("constructor")); + let Some(callee) = callee else { return vec![] }; + let callee_name = match callee.kind() { + "identifier" => self.text(callee), + "member_expression" => self.text(callee.child_by_field_name("property").unwrap_or(callee)), + _ => "", + }; + if !matches!(callee_name, "defineStore" | "createStore" | "Store") { + return vec![]; + } + let Some(args) = call.child_by_field_name("arguments") else { return vec![] }; + let mut objects = Vec::new(); + for i in 0..args.named_child_count() { + let Some(arg) = args.named_child(i) else { continue }; + if !matches!(arg.kind(), "object" | "object_expression") { + continue; + } + for j in 0..arg.named_child_count() { + let Some(member) = arg.named_child(j) else { continue }; + if member.kind() != "pair" { + continue; + } + let Some(key) = member.child_by_field_name("key") else { continue }; + if !is_vue_collection_name(self.text(key)) { + continue; + } + if let Some(value) = member.child_by_field_name("value") { + if matches!(value.kind(), "object" | "object_expression") { + objects.push(value); + } + } + } + } + objects + } + + pub(super) fn extract_store_collection_methods(&mut self, config: Node<'t>) { + for i in 0..config.named_child_count() { + let Some(member) = config.named_child(i) else { continue }; + if member.kind() != "pair" { + continue; + } + let Some(key) = member.child_by_field_name("key") else { continue }; + if !is_vue_collection_name(self.text(key)) { + continue; + } + if let Some(value) = member.child_by_field_name("value") { + if matches!(value.kind(), "object" | "object_expression") { + self.extract_object_literal_functions(value); + } + } + } + } + + fn find_pinia_setup_fn(&self, call: Node<'t>) -> Option> { + let callee = call.child_by_field_name("function")?; + if callee.kind() != "identifier" || self.text(callee) != "defineStore" { + return None; + } + let args = call.child_by_field_name("arguments")?; + for i in 0..args.named_child_count() { + let Some(arg) = args.named_child(i) else { continue }; + if !matches!(arg.kind(), "arrow_function" | "function_expression") { + continue; + } + if let Some(body) = arg.child_by_field_name("body") { + if body.kind() == "statement_block" { + return Some(arg); + } + } + } + None + } + + fn extract_pinia_setup_body(&mut self, setup: Node<'t>) { + let Some(body) = setup.child_by_field_name("body") else { return }; + if body.kind() != "statement_block" { + return; + } + for i in 0..body.named_child_count() { + let Some(stmt) = body.named_child(i) else { continue }; + if stmt.kind() == "function_declaration" { + self.extract_function(stmt, None); + } else if is_variable_type(stmt.kind()) { + for j in 0..stmt.named_child_count() { + let Some(decl) = stmt.named_child(j) else { continue }; + if decl.kind() != "variable_declarator" { + continue; + } + if let Some(v) = decl.child_by_field_name("value") { + if matches!(v.kind(), "arrow_function" | "function_expression") { + self.extract_function(v, None); + } + } + } + } + } + } + + // --- extractTypeAlias + members (#359, #634) ------------------------------------- + + /// Returns skipChildren (always false on the TS path — the alias value is + /// still traversed by the dispatcher). + pub(super) fn extract_type_alias(&mut self, node: Node<'t>) -> bool { + let name = self.extract_name(node); + if name == "" { + return false; + } + let extra = Extra { + docstring: super::docstring::preceding_docstring(node, self.src), + is_exported: Some(self.is_exported(node)), + ..Extra::default() + }; + let Some(row) = self.create_node("type_alias", &name, node, extra) else { + return false; + }; + if let Some(value) = node.child_by_field_name("value") { + self.extract_type_refs_from_subtree(value, row); + self.extract_ts_type_alias_members(value, row, &name); + self.extract_ts_tuple_contract_names(value, row, &name); + } + false + } + + fn extract_ts_type_alias_members(&mut self, value: Node<'t>, alias_row: u32, alias_name: &str) { + let mut object_types: Vec = Vec::new(); + if value.kind() == "object_type" { + object_types.push(value); + } else if value.kind() == "intersection_type" { + for i in 0..value.named_child_count() { + if let Some(op) = value.named_child(i) { + if op.kind() == "object_type" { + object_types.push(op); + } + } + } + } else { + return; + } + + self.stack.push(Scope { row: alias_row, kind: "type_alias", name: alias_name.to_string() }); + for obj_type in object_types { + for i in 0..obj_type.named_child_count() { + let Some(child) = obj_type.named_child(i) else { continue }; + if !matches!(child.kind(), "property_signature" | "method_signature") { + continue; + } + let Some(name_node) = child.child_by_field_name("name") else { continue }; + let member_name = self.text(name_node).to_string(); + if member_name.is_empty() { + continue; + } + let member_kind: &'static str = if child.kind() == "method_signature" + || self.is_ts_function_typed_property(child) + { + "method" + } else { + "property" + }; + let extra = Extra { + docstring: super::docstring::preceding_docstring(child, self.src), + signature: Some(self.text(child).to_string()), + qualified_name: Some(format!("{alias_name}::{member_name}")), + ..Extra::default() + }; + self.create_node(member_kind, &member_name, child, extra); + self.extract_type_annotations(child, alias_row); + } + } + self.stack.pop(); + } + + fn extract_ts_tuple_contract_names(&mut self, value: Node<'t>, alias_row: u32, alias_name: &str) { + let mut tuples: Vec = Vec::new(); + fn collect<'t>(n: Node<'t>, depth: u32, out: &mut Vec>) { + if depth > 6 { + return; + } + if n.kind() == "tuple_type" { + out.push(n); + } + for i in 0..n.named_child_count() { + if let Some(c) = n.named_child(i) { + collect(c, depth + 1, out); + } + } + } + collect(value, 0, &mut tuples); + if tuples.is_empty() { + return; + } + + self.stack.push(Scope { row: alias_row, kind: "type_alias", name: alias_name.to_string() }); + for tuple in tuples { + for i in 0..tuple.named_child_count() { + let Some(entry) = tuple.named_child(i) else { continue }; + if entry.kind() != "generic_type" { + continue; + } + let Some(type_args) = entry.child_by_field_name("type_arguments") else { continue }; + for j in 0..type_args.named_child_count() { + let Some(arg) = type_args.named_child(j) else { continue }; + if arg.kind() != "literal_type" { + continue; + } + let Some(str_node) = arg.named_child(0) else { continue }; + if str_node.kind() != "string" { + continue; + } + let name = util::object_key_name(self.text(str_node).trim()); + if !util::ident_dollar().is_match(&name) { + continue; + } + let collapsed = collapse_ws(self.text(entry)); + let (signature, _) = util::slice_utf16(collapsed.trim(), 120); + let extra = Extra { + signature: Some(signature), + qualified_name: Some(format!("{alias_name}::{name}")), + ..Extra::default() + }; + self.create_node("method", &name, entry, extra); + } + } + } + self.stack.pop(); + } + + fn is_ts_function_typed_property(&self, property_signature: Node) -> bool { + let Some(type_anno) = property_signature.child_by_field_name("type") else { + return false; + }; + for i in 0..type_anno.named_child_count() { + if let Some(inner) = type_anno.named_child(i) { + if inner.kind() == "function_type" { + return true; + } + } + } + false + } + + // --- extractImport + binding refs --------------------------------------------------- + + pub(super) fn extract_import(&mut self, node: Node<'t>) { + let import_text = self.text(node).trim().to_string(); + // typescriptExtractor.extractImport: the `source` field, quotes stripped + // globally. A missing/empty module means the hook declined — no node. + let Some(source_field) = node.child_by_field_name("source") else { return }; + let module_name: String = self + .text(source_field) + .chars() + .filter(|c| *c != '\'' && *c != '"') + .collect(); + if module_name.is_empty() { + return; + } + self.create_node( + "import", + &module_name, + node, + Extra { signature: Some(import_text), ..Extra::default() }, + ); + let parent = self.top_row(); + self.push_ref(parent, &module_name.clone(), edge_kind_index("imports").unwrap(), node); + self.emit_import_binding_refs(node, parent); + } + + fn emit_import_binding_refs(&mut self, node: Node<'t>, from_row: u32) { + let clause = (0..node.named_child_count()) + .filter_map(|i| node.named_child(i)) + .find(|c| c.kind() == "import_clause"); + let Some(clause) = clause else { return }; // side-effect import + + let imports_kind = edge_kind_index("imports").unwrap(); + let push = |w: &mut Self, name_node: Option| { + let Some(n) = name_node else { return }; + let name = w.text(n).to_string(); + if name.is_empty() { + return; + } + w.push_ref(from_row, &name, imports_kind, n); + }; + + for i in 0..clause.named_child_count() { + let Some(child) = clause.named_child(i) else { continue }; + match child.kind() { + "identifier" => push(self, Some(child)), + "named_imports" => { + for j in 0..child.named_child_count() { + let Some(spec) = child.named_child(j) else { continue }; + if spec.kind() != "import_specifier" { + continue; + } + let n = spec + .child_by_field_name("alias") + .or_else(|| spec.child_by_field_name("name")) + .or_else(|| spec.named_child(0)); + push(self, n); + } + } + "namespace_import" => { + let n = (0..child.named_child_count()) + .filter_map(|k| child.named_child(k)) + .find(|c| c.kind() == "identifier") + .or_else(|| child.named_child(0)); + push(self, n); + } + _ => {} + } + } + } + + pub(super) fn emit_re_export_refs(&mut self, node: Node<'t>) { + let from_row = self.top_row(); + let clause = (0..node.named_child_count()) + .filter_map(|i| node.named_child(i)) + .find(|c| c.kind() == "export_clause"); + let Some(clause) = clause else { return }; // `export * from './y'` + let imports_kind = edge_kind_index("imports").unwrap(); + for i in 0..clause.named_child_count() { + let Some(spec) = clause.named_child(i) else { continue }; + if spec.kind() != "export_specifier" { + continue; + } + let name_node = spec.child_by_field_name("name").or_else(|| spec.named_child(0)); + let Some(n) = name_node else { continue }; + let name = self.text(n).to_string(); + if name.is_empty() || name == "default" { + continue; + } + self.push_ref(from_row, &name, imports_kind, n); + } + } + + // --- extractCall (TS/JS generic tail) ------------------------------------------------- + + pub(super) fn extract_call(&mut self, node: Node<'t>) { + if self.stack.is_empty() { + return; + } + let func = node + .child_by_field_name("function") + .or_else(|| node.named_child(0)); + let mut callee_name = String::new(); + + if let Some(func) = func { + if func.kind() == "member_expression" { + let property = func + .child_by_field_name("property") + .or_else(|| func.child_by_field_name("field")) + .or_else(|| func.named_child(1)); + if let Some(property) = property { + let method_name = self.text(property); + let receiver = func + .child_by_field_name("object") + .or_else(|| func.child_by_field_name("operand")) + .or_else(|| func.child_by_field_name("argument")) + .or_else(|| func.named_child(0)); + // Literal receivers call builtins, never project symbols (#1230). + if let Some(r) = receiver { + if is_literal_receiver(r.kind()) { + return; + } + } + let recv_ident = receiver.filter(|r| { + matches!(r.kind(), "identifier" | "simple_identifier" | "field_identifier") + }); + if let Some(r) = recv_ident { + let receiver_name = self.text(r); + if !matches!(receiver_name, "self" | "this" | "cls" | "super") { + callee_name = format!("{receiver_name}.{method_name}"); + } else { + callee_name = method_name.to_string(); + } + } else { + // (the call-receiver re-encode branches are other + // languages'; TS/JS keeps the bare method name) + callee_name = method_name.to_string(); + } + } + } else { + callee_name = self.text(func).to_string(); + } + } + + // Parenthesized-callee normalization (`(fn)()` → fn). + if !callee_name.is_empty() { + if let Some(c) = util::paren_conversion().captures(&callee_name) { + callee_name = c[1].to_string(); + } + } + + if !callee_name.is_empty() { + self.push_call_ref(&callee_name.clone(), node); + } + } + + // --- extractInstantiation ----------------------------------------------------------- + + pub(super) fn extract_instantiation(&mut self, node: Node<'t>) { + if self.stack.is_empty() { + return; + } + let ctor = node + .child_by_field_name("constructor") + .or_else(|| node.child_by_field_name("type")) + .or_else(|| node.child_by_field_name("name")) + .or_else(|| node.named_child(0)); + let Some(ctor) = ctor else { return }; + + let mut class_name = self.text(ctor).to_string(); + // `new Map()` → Map. + if let Some(lt) = class_name.find('<') { + if lt > 0 { + class_name.truncate(lt); + } + } + // `new ns.Foo()` → Foo. + let last_dot = class_name + .rfind('.') + .map(|i| i as isize) + .unwrap_or(-1) + .max(class_name.rfind("::").map(|i| i as isize).unwrap_or(-1)); + if last_dot >= 0 { + class_name = class_name[(last_dot as usize + 1)..].to_string(); + // TS: .replace(/^[:.]/, '') — one leading colon-or-dot. + if class_name.starts_with(':') || class_name.starts_with('.') { + class_name.remove(0); + } + } + let class_name = class_name.trim().to_string(); + if !class_name.is_empty() { + let from = self.top_row(); + self.push_ref(from, &class_name, edge_kind_index("instantiates").unwrap(), node); + } + } + + // --- extractDecoratorsFor -------------------------------------------------------------- + + pub(super) fn extract_decorators_for(&mut self, decl: Node<'t>, decorated_row: u32) { + // 1. Direct children (method/property style). + for i in 0..decl.named_child_count() { + let Some(child) = decl.named_child(i) else { continue }; + self.consider_decorator(child, decorated_row); + if child.kind() == "modifiers" { + for j in 0..child.named_child_count() { + if let Some(m) = child.named_child(j) { + self.consider_decorator(m, decorated_row); + } + } + } + } + // 2. Preceding siblings (TypeScript class style), stopping at the + // first non-decorator so an earlier declaration's decorators never + // leak in. Matching by startIndex, not object identity. + let Some(parent) = decl.parent() else { return }; + let decl_start = decl.start_byte(); + let mut decl_idx: isize = -1; + for i in 0..parent.named_child_count() { + if let Some(sib) = parent.named_child(i) { + if sib.start_byte() == decl_start { + decl_idx = i as isize; + break; + } + } + } + if decl_idx > 0 { + let mut j = decl_idx - 1; + while j >= 0 { + let Some(sib) = parent.named_child(j as usize) else { + j -= 1; + continue; + }; + if !matches!(sib.kind(), "decorator" | "annotation" | "marker_annotation") { + break; + } + self.consider_decorator(sib, decorated_row); + j -= 1; + } + } + } + + fn consider_decorator(&mut self, n: Node<'t>, decorated_row: u32) { + if !matches!(n.kind(), "decorator" | "annotation" | "marker_annotation" | "attribute") { + return; + } + let mut target: Option = None; + for i in 0..n.named_child_count() { + let Some(child) = n.named_child(i) else { continue }; + if child.kind() == "call_expression" { + target = child.child_by_field_name("function").or_else(|| child.named_child(0)); + if target.is_some() { + break; + } + } + if matches!( + child.kind(), + "identifier" | "member_expression" | "scoped_identifier" | "navigation_expression" + | "user_type" | "type_identifier" + ) { + target = Some(child); + break; + } + } + let Some(target) = target else { return }; + let mut name = self.text(target).to_string(); + if let Some(lt) = name.find('<') { + if lt > 0 { + name.truncate(lt); + } + } + let last_dot = name + .rfind('.') + .map(|i| i as isize) + .unwrap_or(-1) + .max(name.rfind("::").map(|i| i as isize).unwrap_or(-1)); + if last_dot >= 0 { + name = name[(last_dot as usize + 1)..].to_string(); + if name.starts_with(':') || name.starts_with('.') { + name.remove(0); + } + } + let name = name.trim().to_string(); + if name.is_empty() { + return; + } + self.push_ref(decorated_row, &name, edge_kind_index("decorates").unwrap(), n); + } + + // --- extractInheritance (TS/JS clauses) --------------------------------------------------- + + pub(super) fn extract_inheritance(&mut self, node: Node<'t>, class_row: u32) { + let extends_kind = edge_kind_index("extends").unwrap(); + let implements_kind = edge_kind_index("implements").unwrap(); + for i in 0..node.named_child_count() { + let Some(child) = node.named_child(i) else { continue }; + match child.kind() { + // TS `extends_clause` (the other spellings are other grammars'). + "extends_clause" | "superclass" | "base_clause" | "extends_interfaces" => { + if let Some(target) = child.named_child(0) { + let name = self.text(target).to_string(); + self.push_ref(class_row, &name, extends_kind, target); + } + } + "implements_clause" | "class_interface_clause" | "super_interfaces" | "interfaces" => { + for j in 0..child.named_child_count() { + if let Some(iface) = child.named_child(j) { + let name = self.text(iface).to_string(); + self.push_ref(class_row, &name, implements_kind, iface); + } + } + } + // JS `class Foo extends Bar` — class_heritage holds a bare + // identifier without an extends_clause wrapper. + "identifier" | "type_identifier" if node.kind() == "class_heritage" => { + let name = self.text(child).to_string(); + self.push_ref(class_row, &name, extends_kind, child); + } + // TS class_heritage wraps extends/implements — recurse. + "field_declaration_list" | "class_heritage" => { + self.extract_inheritance(child, class_row); + } + _ => {} + } + } + } + + // --- type annotations (#381 — TS family only) ---------------------------------------------- + + pub(super) fn extract_type_annotations(&mut self, node: Node<'t>, from_row: u32) { + if !self.variant.is_ts() { + return; + } + if let Some(params) = node.child_by_field_name("parameters") { + self.extract_type_refs_from_subtree(params, from_row); + } + if let Some(ret) = node.child_by_field_name("return_type") { + self.extract_type_refs_from_subtree(ret, from_row); + } + let type_annotation = (0..node.named_child_count()) + .filter_map(|i| node.named_child(i)) + .find(|c| c.kind() == "type_annotation"); + if let Some(ta) = type_annotation { + self.extract_type_refs_from_subtree(ta, from_row); + } + } + + pub(super) fn extract_variable_type_annotation(&mut self, node: Node<'t>, from_row: u32) { + if !self.variant.is_ts() { + return; + } + let type_annotation = (0..node.named_child_count()) + .filter_map(|i| node.named_child(i)) + .find(|c| c.kind() == "type_annotation"); + if let Some(ta) = type_annotation { + self.extract_type_refs_from_subtree(ta, from_row); + } + } + + fn extract_type_refs_from_subtree(&mut self, node: Node<'t>, from_row: u32) { + if node.kind() == "type_identifier" { + let type_name = self.text(node).to_string(); + if !type_name.is_empty() && !is_builtin_type(&type_name) { + self.push_ref(from_row, &type_name, edge_kind_index("references").unwrap(), node); + } + return; + } + for i in 0..node.named_child_count() { + if let Some(c) = node.named_child(i) { + self.extract_type_refs_from_subtree(c, from_row); + } + } + } +} + +/// `.replace(/\s+/g, ' ')` for the tuple-contract signature. +fn collapse_ws(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + let mut in_ws = false; + for c in s.chars() { + if c.is_whitespace() { + if !in_ws { + out.push(' '); + in_ws = true; + } + } else { + out.push(c); + in_ws = false; + } + } + out +} diff --git a/codegraph-kernel/src/tsjs/fnref.rs b/codegraph-kernel/src/tsjs/fnref.rs new file mode 100644 index 0000000..8324710 --- /dev/null +++ b/codegraph-kernel/src/tsjs/fnref.rs @@ -0,0 +1,133 @@ +//! Function-as-value capture (#756) — the TS/JS slice of +//! src/extraction/function-ref.ts (TS_JS_SPEC): container dispatch, value +//! normalization, and the `this.member` special form. The flush-time gate +//! lives in the walker (it needs the file's nodes and import refs). + +use tree_sitter::Node; + +/// CaptureMode (function-ref.ts) — gate policy keys on it. +#[derive(Clone, Copy, PartialEq, Eq)] +pub enum Mode { + Args, + Rhs, + Value, + List, + VarInit, +} + +pub struct Candidate { + pub name: String, + pub line: u32, + pub column_byte: usize, // converted to UTF-16 at emit time + pub row: usize, +} + +/// NAME_STOPLIST (function-ref.ts). +fn stoplisted(name: &str) -> bool { + matches!( + name, + "this" | "self" | "super" | "null" | "nil" | "true" | "false" | "undefined" | "new" + | "NULL" | "nullptr" | "None" + ) +} + +/// TS_JS_SPEC.dispatch: container node type → capture mode. +pub fn dispatch(kind: &str) -> Option { + match kind { + "arguments" => Some(Mode::Args), + "assignment_expression" => Some(Mode::Rhs), + "variable_declarator" => Some(Mode::VarInit), + "pair" => Some(Mode::Value), + "array" => Some(Mode::List), + _ => None, + } +} + +/// captureFnRefCandidates for the TS/JS spec. Returns (candidate, mode) pairs. +pub fn capture(container: Node, mode: Mode, src: &str) -> Vec<(Candidate, Mode)> { + let mut value_nodes: Vec = Vec::new(); + + match mode { + Mode::Args | Mode::List => { + for i in 0..container.named_child_count() { + if let Some(c) = container.named_child(i) { + value_nodes.push(c); + } + } + } + Mode::Rhs => { + if let Some(rhs) = container.child_by_field_name("right") { + // Param-storage skip: `this.status = status` — LHS's trailing + // identifier equals the RHS text ⇒ a stored local/parameter. + let lhs_text = container + .child_by_field_name("left") + .map(|l| &src[l.byte_range()]) + .unwrap_or(""); + let lhs_last = super::util::lhs_last_name() + .captures(lhs_text) + .and_then(|c| c.get(1)) + .map(|m| m.as_str()); + let rhs_text = src[rhs.byte_range()].trim(); + if !(lhs_last.is_some() && lhs_last == Some(rhs_text)) { + value_nodes.push(rhs); + } + } + } + Mode::Value => { + if let Some(v) = container.child_by_field_name("value") { + value_nodes.push(v); + } + } + Mode::VarInit => { + // Destructuring extracts DATA, never a function alias. + let name_node = container.child_by_field_name("name"); + let is_pattern = name_node + .map(|n| matches!(n.kind(), "object_pattern" | "array_pattern")) + .unwrap_or(false); + if !is_pattern { + if let Some(v) = container.child_by_field_name("value") { + value_nodes.push(v); + } + } + } + } + + let mut out = Vec::new(); + for v in value_nodes { + for (name, node) in normalize(v, src) { + if name.is_empty() || stoplisted(&name) { + continue; + } + let p = node.start_position(); + out.push(( + Candidate { + name, + line: p.row as u32 + 1, + column_byte: node.start_byte(), + row: p.row, + }, + mode, + )); + } + } + out +} + +/// normalizeValue for the TS/JS spec: bare identifiers, plus the +/// `this.` member_expression special form (object EXACTLY `this`). +fn normalize<'t>(node: Node<'t>, src: &str) -> Vec<(String, Node<'t>)> { + match node.kind() { + "identifier" => vec![(src[node.byte_range()].to_string(), node)], + "member_expression" => { + let obj = node.child_by_field_name("object"); + let prop = node.child_by_field_name("property"); + if let (Some(o), Some(p)) = (obj, prop) { + if o.kind() == "this" && p.kind() == "property_identifier" { + return vec![(format!("this.{}", &src[p.byte_range()]), p)]; + } + } + vec![] + } + _ => vec![], + } +} diff --git a/codegraph-kernel/src/tsjs/mod.rs b/codegraph-kernel/src/tsjs/mod.rs new file mode 100644 index 0000000..c036e20 --- /dev/null +++ b/codegraph-kernel/src/tsjs/mod.rs @@ -0,0 +1,876 @@ +//! TypeScript / TSX / JavaScript / JSX extraction — a faithful Rust port of +//! `TreeSitterExtractor`'s TS/JS paths (src/extraction/tree-sitter.ts) plus +//! the typescript/javascript LanguageExtractor configs. +//! +//! Porting contract (R2 of the migration plan): behavior parity with the wasm +//! path, verified by scripts/kernel-parity.mjs over real repos — including +//! bug-for-bug fidelity where the TS code has quirks. Every function notes the +//! TS function it mirrors; if you change one side, change the other or the +//! parity gate fails. Positions are emitted in UTF-16 code units (what +//! web-tree-sitter reports), see util::col16. + +mod docstring; +mod extractors; +mod fnref; +pub(crate) mod util; + +use crate::buffers::{ + build_meta, edge_kind_index, node_kind_index, Arena, BoolFlags, EdgeRow, EmitOut, NodeRow, + RefRow, StrRef, Tables, FLAG_IS_ASYNC, FLAG_IS_EXPORTED, FLAG_IS_STATIC, FUNCTION_REF_CODE, + NONE, NONE_STR, +}; +use crate::ids; +use crate::langs; +use std::collections::{HashMap, HashSet}; +use tree_sitter::{Node, Parser}; + +#[derive(Clone, Copy, PartialEq, Eq)] +pub enum Variant { + Typescript, + Tsx, + Javascript, + Jsx, +} + +impl Variant { + pub fn from_language(language: &str) -> Option { + match language { + "typescript" => Some(Variant::Typescript), + "tsx" => Some(Variant::Tsx), + "javascript" => Some(Variant::Javascript), + "jsx" => Some(Variant::Jsx), + _ => None, + } + } + /// TS-family (typescript/tsx): type annotations, interfaces, enums, + /// aliases, visibility, isStatic. The JS family lacks all of those hooks. + fn is_ts(self) -> bool { + matches!(self, Variant::Typescript | Variant::Tsx) + } + /// VALUE_REF_LANGS includes typescript/tsx/javascript but NOT jsx. + fn value_refs(self) -> bool { + !matches!(self, Variant::Jsx) + } +} + +/// typescriptExtractor.methodTypes / javascriptExtractor.methodTypes. +fn is_method_type(v: Variant, kind: &str) -> bool { + kind == "method_definition" + || (v.is_ts() && kind == "public_field_definition") + || (!v.is_ts() && kind == "field_definition") +} + +fn is_function_type(kind: &str) -> bool { + matches!(kind, "function_declaration" | "arrow_function" | "function_expression") +} + +fn is_class_type(v: Variant, kind: &str) -> bool { + kind == "class_declaration" || (v.is_ts() && kind == "abstract_class_declaration") +} + +fn is_variable_type(kind: &str) -> bool { + matches!(kind, "lexical_declaration" | "variable_declaration") +} + +/// LITERAL_RECEIVER_TYPES (tree-sitter.ts) — full set; only a handful occur in +/// TS/JS grammars but membership is what the TS code tests. +fn is_literal_receiver(kind: &str) -> bool { + matches!( + kind, + "string" | "string_literal" | "interpreted_string_literal" | "raw_string_literal" + | "template_string" | "concatenated_string" | "formatted_string" | "f_string" + | "line_string_literal" | "string_content" | "heredoc_body" + | "number" | "number_literal" | "integer" | "integer_literal" | "float" + | "float_literal" | "int_literal" | "decimal_integer_literal" | "real_literal" + | "char_literal" | "character_literal" | "rune_literal" | "regex" | "regex_literal" + | "true" | "false" | "boolean_literal" | "bool_literal" | "none" | "null" | "nil" + | "null_literal" | "undefined" + | "list" | "list_literal" | "array" | "array_literal" | "array_creation_expression" + | "dictionary" | "dict_literal" | "object" | "tuple" | "set" + ) +} + +/// BUILTIN_TYPES (tree-sitter.ts) — names that never become type references. +fn is_builtin_type(name: &str) -> bool { + matches!( + name, + "string" | "number" | "boolean" | "void" | "null" | "undefined" | "never" | "any" + | "unknown" | "object" | "symbol" | "bigint" | "true" | "false" + | "str" | "bool" | "i8" | "i16" | "i32" | "i64" | "i128" | "isize" + | "u8" | "u16" | "u32" | "u64" | "u128" | "usize" | "f32" | "f64" | "char" + | "int" | "long" | "short" | "byte" | "float" | "double" + | "int8" | "int16" | "int32" | "int64" | "uint8" | "uint16" | "uint32" | "uint64" + | "float32" | "float64" | "complex64" | "complex128" | "rune" | "error" + | "Int" | "Long" | "Short" | "Byte" | "Float" | "Double" | "Boolean" | "Char" + | "Unit" | "String" | "Any" | "AnyRef" | "AnyVal" | "Nothing" | "Null" + ) +} + +/// REACT_COMPONENT_HOCS (tree-sitter.ts, #841). +fn is_react_hoc(callee: &str) -> bool { + matches!(callee, "forwardRef" | "memo" | "React.forwardRef" | "React.memo") +} + +fn is_vue_collection_name(name: &str) -> bool { + matches!(name, "actions" | "mutations" | "getters") +} + +/// One scope-stack entry (TS keeps node IDs; rows are our equivalent). +struct Scope { + row: u32, + kind: &'static str, + name: String, +} + +/// Extra node properties, per-extract-site (mirrors createNode's `extra`). +#[derive(Default)] +struct Extra { + docstring: Option, + signature: Option, + visibility: Option, + is_exported: Option, + is_async: Option, + is_static: Option, + qualified_name: Option, +} + +struct ValueScope<'t> { + row: u32, + node: Node<'t>, + name: String, +} + +pub struct Walker<'t> { + src: &'t str, + file_path: &'t str, + variant: Variant, + line_starts: Vec, + arena: Arena, + tables: Tables, + stack: Vec, + /// Function/method names defined in this file (fn-ref flush gate). + defined_fn_names: HashSet, + /// Simple names from `imports` refs (fn-ref flush gate). + imported_names: HashSet, + fn_ref_cands: Vec<(u32, fnref::Candidate)>, + // Value-reference bookkeeping (flushValueRefs). + fs_values: HashMap, + fs_value_counts: HashMap, + value_scopes: Vec>, + vue_store_file: Option, +} + +const MAX_VALUE_REF_NODES: usize = 20_000; + +pub fn extract(file_path: &str, source: &str, language: &str) -> Result { + let variant = Variant::from_language(language) + .ok_or_else(|| format!("tsjs walker does not handle language: {language}"))?; + let grammar = langs::grammar_for(language) + .ok_or_else(|| format!("no grammar for language: {language}"))?; + + let t0 = std::time::Instant::now(); + let mut parser = Parser::new(); + parser + .set_language(&grammar) + .map_err(|e| format!("set_language({language}) failed: {e}"))?; + let tree = parser + .parse(source, None) + .ok_or_else(|| "parser returned null tree".to_string())?; + + let mut w = Walker { + src: source, + file_path, + variant, + line_starts: util::line_starts(source), + arena: Arena::default(), + tables: Tables::default(), + stack: Vec::new(), + defined_fn_names: HashSet::new(), + imported_names: HashSet::new(), + fn_ref_cands: Vec::new(), + fs_values: HashMap::new(), + fs_value_counts: HashMap::new(), + value_scopes: Vec::new(), + vue_store_file: None, + }; + + // File node (TreeSitterExtractor.extract): id `file:`, endLine = + // newline count + 1, isExported explicitly false. + let line_count = source.bytes().filter(|b| *b == b'\n').count() as u32 + 1; + let base_name = file_path.rsplit(['/', '\\']).next().unwrap_or(file_path); + let mut flags = BoolFlags::default(); + flags.set(FLAG_IS_EXPORTED, false); + let file_id = w.arena.put(&ids::file_node_id(file_path)); + let name_ref = w.arena.put(base_name); + let qn_ref = w.arena.put(file_path); + w.tables.push_node(&NodeRow { + kind: node_kind_index("file").unwrap(), + visibility: 0, + flags, + start_line: 1, + end_line: line_count, + start_column: 0, + end_column: 0, + name: name_ref, + qualified_name: qn_ref, + id: file_id, + docstring: NONE_STR, + signature: NONE_STR, + decorators: NONE_STR, + type_parameters: NONE_STR, + return_type: NONE_STR, + extra_json: NONE_STR, + }); + w.stack.push(Scope { row: 0, kind: "file", name: base_name.to_string() }); + + w.visit_node(tree.root_node()); + + // End-of-file passes, in the TS extract() order. + w.flush_fn_ref_candidates(); + w.flush_value_refs(tree.root_node()); + w.stack.pop(); + + let duration_ms = t0.elapsed().as_secs_f64() * 1000.0; + let meta = build_meta(&w.tables, w.arena.len(), NONE_STR, duration_ms); + Ok(EmitOut { + meta, + nodes: w.tables.nodes, + edges: w.tables.edges, + refs: w.tables.refs, + arena: w.arena.into_vec(), + }) +} + +impl<'t> Walker<'t> { + // --- small helpers -------------------------------------------------------- + + fn text(&self, node: Node) -> &'t str { + &self.src[node.byte_range()] + } + + fn line_of(&self, node: Node) -> u32 { + node.start_position().row as u32 + 1 + } + + fn col_of(&self, node: Node) -> u32 { + util::col16(self.src, &self.line_starts, node.start_position().row, node.start_byte()) + } + + fn end_col_of(&self, node: Node) -> u32 { + util::col16(self.src, &self.line_starts, node.end_position().row, node.end_byte()) + } + + fn top_row(&self) -> u32 { + self.stack.last().map(|s| s.row).unwrap_or(0) + } + + /// isInsideClassLikeNode. + fn inside_class_like(&self) -> bool { + self.stack + .last() + .map(|s| matches!(s.kind, "class" | "struct" | "interface" | "trait" | "enum" | "module")) + .unwrap_or(false) + } + + fn push_ref(&mut self, from_row: u32, name: &str, kind_code: u8, node: Node) { + let name_ref = self.arena.put(name); + self.tables.push_ref(&RefRow { + from_idx: from_row, + kind: kind_code, + line: self.line_of(node), + column: self.col_of(node), + reference_name: name_ref, + candidates: NONE_STR, + from_id_str: NONE_STR, + }); + if kind_code == edge_kind_index("imports").unwrap() { + // Feed the fn-ref flush gate the same way flushFnRefCandidates + // derives importedNames from `imports` refs. + if util::simple_name().is_match(name) { + self.imported_names.insert(name.to_string()); + } else if let Some(c) = util::qualified_import().captures(name) { + self.imported_names.insert(c[1].to_string()); + } + } + } + + fn push_call_ref(&mut self, name: &str, node: Node) { + self.push_ref(self.top_row(), name, edge_kind_index("calls").unwrap(), node); + } + + // --- createNode ----------------------------------------------------------- + + /// createNode (tree-sitter.ts): id, qualified name from the scope stack, + /// contains edge from the parent scope, value-ref bookkeeping. + fn create_node(&mut self, kind: &'static str, name: &str, node: Node<'t>, extra: Extra) -> Option { + if name.is_empty() { + return None; + } + let start_line = self.line_of(node); + let id = ids::node_id(self.file_path, kind, name, start_line); + + // endLine body extension: resolveBody only (TS/JS: function-valued + // class fields whose body nests in the arrow / HOF-wrapped arrow). + let mut end_line = node.end_position().row as u32 + 1; + if (kind == "function" || kind == "method") && matches!(node.kind(), "public_field_definition" | "field_definition") + { + if let Some(body) = resolve_field_body(node) { + let be = body.end_position().row as u32 + 1; + if be > end_line { + end_line = be; + } + } + } + + let qualified = extra.qualified_name.unwrap_or_else(|| { + let mut parts: Vec<&str> = Vec::new(); + for s in &self.stack { + if s.kind != "file" { + parts.push(&s.name); + } + } + let mut qn = parts.join("::"); + if !qn.is_empty() { + qn.push_str("::"); + } + qn.push_str(name); + qn + }); + + let mut flags = BoolFlags::default(); + if let Some(v) = extra.is_exported { + flags.set(FLAG_IS_EXPORTED, v); + } + if let Some(v) = extra.is_async { + flags.set(FLAG_IS_ASYNC, v); + } + if let Some(v) = extra.is_static { + flags.set(FLAG_IS_STATIC, v); + } + + let name_ref = self.arena.put(name); + let qn_ref = self.arena.put(&qualified); + let id_ref = self.arena.put(&id); + let doc_ref = opt_str(&mut self.arena, extra.docstring.as_deref()); + let sig_ref = opt_str(&mut self.arena, extra.signature.as_deref()); + let row = self.tables.push_node(&NodeRow { + kind: node_kind_index(kind).unwrap(), + visibility: extra.visibility.unwrap_or(0), + flags, + start_line, + end_line, + start_column: self.col_of(node), + end_column: self.end_col_of(node), + name: name_ref, + qualified_name: qn_ref, + id: id_ref, + docstring: doc_ref, + signature: sig_ref, + decorators: NONE_STR, + type_parameters: NONE_STR, + return_type: NONE_STR, + extra_json: NONE_STR, + }); + + // Containment edge from the current scope. + let parent_row = self.top_row(); + self.tables.push_edge(&EdgeRow { + source_idx: parent_row, + target_idx: row, + kind: edge_kind_index("contains").unwrap(), + provenance: 0, + line: NONE, + column: NONE, + metadata_json: NONE_STR, + source_id_str: NONE_STR, + target_id_str: NONE_STR, + }); + + if kind == "function" || kind == "method" { + self.defined_fn_names.insert(name.to_string()); + } + self.capture_value_ref_scope(kind, name, row, node); + Some(row) + } + + // --- value references (captureValueRefScope / flushValueRefs) -------------- + + fn capture_value_ref_scope(&mut self, kind: &'static str, name: &str, row: u32, node: Node<'t>) { + if !self.variant.value_refs() { + return; + } + let target_kind_ok = kind == "constant" || kind == "variable"; + if target_kind_ok + && util::utf16_len(name) >= 3 + && util::has_upper_or_underscore().is_match(name) + { + let parent_ok = self + .stack + .last() + .map(|s| matches!(s.kind, "file" | "class" | "module" | "struct" | "enum")) + .unwrap_or(false); + if parent_ok { + self.fs_values.insert(name.to_string(), row); + *self.fs_value_counts.entry(name.to_string()).or_insert(0) += 1; + } + } + if matches!(kind, "function" | "method" | "constant" | "variable") { + self.value_scopes.push(ValueScope { row, node, name: name.to_string() }); + } + } + + fn flush_value_refs(&mut self, root: Node<'t>) { + let scopes = std::mem::take(&mut self.value_scopes); + let mut targets = std::mem::take(&mut self.fs_values); + let counts = std::mem::take(&mut self.fs_value_counts); + if !self.variant.value_refs() || std::env::var("CODEGRAPH_VALUE_REFS").as_deref() == Ok("0") { + return; + } + if targets.is_empty() || scopes.is_empty() || util::is_generated_file(self.file_path) { + return; + } + + // Shadow prune: count declarators of each target name across the whole + // tree; more declarators than file-scope nodes ⇒ an inner re-binding + // shadows the target. (TS/JS declarators are `variable_declarator`; + // the other kinds in the TS switch belong to other grammars.) + let mut decl_counts: HashMap<&str, u32> = HashMap::new(); + let mut dstack: Vec = vec![root]; + let mut dvisited = 0usize; + while let Some(n) = dstack.pop() { + if dvisited >= MAX_VALUE_REF_NODES { + break; + } + dvisited += 1; + if n.kind() == "variable_declarator" { + if let Some(first) = n.named_child(0) { + if first.kind() == "identifier" { + let nm = self.text(first); + if targets.contains_key(nm) { + *decl_counts.entry(nm).or_insert(0) += 1; + } + } + } + } + for i in 0..n.named_child_count() { + if let Some(c) = n.named_child(i) { + dstack.push(c); + } + } + } + let shadowed: Vec = decl_counts + .iter() + .filter(|(nm, c)| **c > counts.get(**nm).copied().unwrap_or(1)) + .map(|(nm, _)| nm.to_string()) + .collect(); + for nm in shadowed { + targets.remove(&nm); + } + if targets.is_empty() { + return; + } + + let refs_kind = edge_kind_index("references").unwrap(); + for scope in &scopes { + let mut seen: HashSet = HashSet::new(); + let mut stack: Vec = vec![scope.node]; + let mut visited = 0usize; + while let Some(n) = stack.pop() { + if visited >= MAX_VALUE_REF_NODES { + break; + } + visited += 1; + if matches!(n.kind(), "identifier" | "constant" | "name" | "simple_identifier") { + let ref_name = self.text(n); + if let Some(&target_row) = targets.get(ref_name) { + if target_row != scope.row && ref_name != scope.name && !seen.contains(&target_row) { + seen.insert(target_row); + let meta = self.arena.put(r#"{"valueRef":true}"#); + self.tables.push_edge(&EdgeRow { + source_idx: scope.row, + target_idx: target_row, + kind: refs_kind, + provenance: 0, + line: NONE, + column: NONE, + metadata_json: meta, + source_id_str: NONE_STR, + target_id_str: NONE_STR, + }); + } + } + } + for i in 0..n.named_child_count() { + if let Some(c) = n.named_child(i) { + stack.push(c); + } + } + } + } + } + + // --- function-as-value refs (#756) ----------------------------------------- + + fn maybe_capture_fn_refs(&mut self, node: Node<'t>) { + let Some(mode) = fnref::dispatch(node.kind()) else { return }; + if self.stack.is_empty() { + return; + } + let from = self.top_row(); + for (cand, _mode) in fnref::capture(node, mode, self.src) { + self.fn_ref_cands.push((from, cand)); + } + } + + /// scanFnRefSubtree: capture-only walk of subtrees the main walkers skip. + fn scan_fn_ref_subtree(&mut self, node: Node<'t>, depth: u32) { + if depth > 12 { + return; + } + let kind = node.kind(); + if depth > 0 + && (is_function_type(kind) || matches!(kind, "lambda_literal" | "lambda_expression")) + { + return; + } + self.maybe_capture_fn_refs(node); + for i in 0..node.named_child_count() { + if let Some(c) = node.named_child(i) { + self.scan_fn_ref_subtree(c, depth + 1); + } + } + } + + fn flush_fn_ref_candidates(&mut self) { + let cands = std::mem::take(&mut self.fn_ref_cands); + if cands.is_empty() || util::is_generated_file(self.file_path) { + return; + } + let mut seen: HashSet<(u32, String)> = HashSet::new(); + for (from, c) in cands { + // Gate: `this.` always flushes; everything else must match + // a same-file function/method or an imported name. (The `::` and + // ungated-mode policies belong to other languages' specs.) + if !c.name.starts_with("this.") + && !c.name.contains("::") + && !self.defined_fn_names.contains(&c.name) + && !self.imported_names.contains(&c.name) + { + continue; + } + if !seen.insert((from, c.name.clone())) { + continue; + } + let column = util::col16(self.src, &self.line_starts, c.row, c.column_byte); + let name_ref = self.arena.put(&c.name); + self.tables.push_ref(&RefRow { + from_idx: from, + kind: FUNCTION_REF_CODE, + line: c.line, + column, + reference_name: name_ref, + candidates: NONE_STR, + from_id_str: NONE_STR, + }); + } + } + + // --- the dispatcher (visitNode) -------------------------------------------- + + fn visit_node(&mut self, node: Node<'t>) { + let kind = node.kind(); + let mut skip_children = false; + + // Function-as-value capture — independent of the dispatch ladder. + self.maybe_capture_fn_refs(node); + + if is_function_type(kind) { + // (the isInsideClassLike + methodTypes overlap is Python/Ruby-only) + self.extract_function(node, None); + skip_children = true; + } else if is_class_type(self.variant, kind) { + self.extract_class(node); + skip_children = true; + } else if is_method_type(self.variant, kind) { + if classify_ts_class_member(node) == Member::Property { + let prop = self.extract_property(node); + if let (Some((row, name)), Some(value)) = (prop, node.child_by_field_name("value")) { + self.stack.push(Scope { row, kind: "property", name }); + self.visit_function_body(value); + self.stack.pop(); + } + self.scan_fn_ref_subtree(node, 0); + } else { + self.extract_method(node); + } + skip_children = true; + } else if self.variant.is_ts() && kind == "interface_declaration" { + self.extract_interface(node); + skip_children = true; + } else if self.variant.is_ts() && kind == "enum_declaration" { + self.extract_enum(node); + skip_children = true; + } else if self.variant.is_ts() && kind == "type_alias_declaration" { + skip_children = self.extract_type_alias(node); + } else if is_variable_type(kind) && !self.inside_class_like() { + self.extract_variable(node); + self.scan_fn_ref_subtree(node, 0); + skip_children = true; + } else if kind == "import_statement" { + self.extract_import(node); + } else if kind == "export_statement" && node.child_by_field_name("source").is_some() { + // Re-export: `export { X } from './y'`. + self.emit_re_export_refs(node); + } else if kind == "export_statement" && self.looks_like_vue_store_file() { + // Vuex MODULE default export (`export default { actions: {…} }`). + if let Some(exported) = node.child_by_field_name("value") { + if matches!(exported.kind(), "object" | "object_expression") { + self.extract_store_collection_methods(exported); + skip_children = true; + } + } + } else if kind == "call_expression" { + self.extract_call(node); + } else if kind == "new_expression" { + self.extract_instantiation(node); + } else if self.variant.is_ts() + && matches!(kind, "property_signature" | "method_signature") + && self.inside_class_like() + { + let parent = self.top_row(); + self.extract_type_annotations(node, parent); + } + + if !skip_children { + for i in 0..node.named_child_count() { + if let Some(c) = node.named_child(i) { + self.visit_node(c); + } + } + } + } + + // --- visitFunctionBody ------------------------------------------------------ + + fn visit_function_body(&mut self, body: Node<'t>) { + self.visit_for_calls_and_structure(body); + } + + fn visit_for_calls_and_structure(&mut self, node: Node<'t>) { + let kind = node.kind(); + self.maybe_capture_fn_refs(node); + + if kind == "call_expression" { + self.extract_call(node); + } else if kind == "new_expression" { + self.extract_instantiation(node); + } + + // Local variable type annotations (TS family only). + if self.variant.is_ts() && kind == "variable_declarator" { + let owner = self.top_row(); + self.extract_variable_type_annotation(node, owner); + } + + // Nested NAMED functions become their own nodes. + if is_function_type(kind) { + let name = self.extract_name(node); + if name != "" { + self.extract_function(node, None); + return; + } + } + + if is_class_type(self.variant, kind) { + self.extract_class(node); + return; + } + if self.variant.is_ts() && kind == "enum_declaration" { + self.extract_enum(node); + return; + } + if self.variant.is_ts() && kind == "interface_declaration" { + self.extract_interface(node); + return; + } + + for i in 0..node.named_child_count() { + if let Some(c) = node.named_child(i) { + self.visit_for_calls_and_structure(c); + } + } + } + + // --- name / signature / modifier helpers ------------------------------------ + + /// extractName / extractNameRaw for the TS/JS configs. + fn extract_name(&self, node: Node) -> String { + // javascriptExtractor.resolveName: field_definition names its key the + // `property` field. + if !self.variant.is_ts() && node.kind() == "field_definition" { + if let Some(prop) = node.child_by_field_name("property") { + return self.text(prop).to_string(); + } + } + if let Some(name_node) = node.child_by_field_name("name") { + return self.text(name_node).to_string(); + } + if matches!(node.kind(), "arrow_function" | "function_expression") { + return "".to_string(); + } + for i in 0..node.named_child_count() { + if let Some(c) = node.named_child(i) { + if matches!(c.kind(), "identifier" | "type_identifier" | "simple_identifier" | "constant") { + return self.text(c).to_string(); + } + } + } + "".to_string() + } + + /// typescriptExtractor.getSignature / javascriptExtractor.getSignature. + fn signature_of(&self, node: Node) -> Option { + let params = node.child_by_field_name("parameters")?; + let mut sig = self.text(params).to_string(); + if self.variant.is_ts() { + if let Some(ret) = node.child_by_field_name("return_type") { + let ret_text = self.text(ret); + let stripped = ret_text.strip_prefix(':').unwrap_or(ret_text).trim_start(); + sig.push_str(": "); + sig.push_str(stripped); + } + } + Some(sig) + } + + /// typescriptExtractor.getVisibility (TS only — JS has no hook). + fn visibility_of(&self, node: Node) -> Option { + if !self.variant.is_ts() { + return None; + } + for i in 0..node.child_count() { + let child = node.child(i)?; + if child.kind() == "accessibility_modifier" { + return match self.text(child) { + "public" => Some(1), + "private" => Some(2), + "protected" => Some(3), + _ => None, + }; + } + } + None + } + + /// isExported: walk the parent chain for an export_statement. + fn is_exported(&self, node: Node) -> bool { + let mut cur = node.parent(); + while let Some(p) = cur { + if p.kind() == "export_statement" { + return true; + } + cur = p.parent(); + } + false + } + + fn has_keyword_child(&self, node: Node, kw: &str) -> bool { + for i in 0..node.child_count() { + if let Some(c) = node.child(i) { + if c.kind() == kw { + return true; + } + } + } + false + } + + fn is_async(&self, node: Node) -> bool { + self.has_keyword_child(node, "async") + } + + /// TS has an isStatic hook; JS does not (None = field absent). + fn is_static(&self, node: Node) -> Option { + if self.variant.is_ts() { + Some(self.has_keyword_child(node, "static")) + } else { + None + } + } + + fn is_const_decl(&self, node: Node) -> bool { + node.kind() == "lexical_declaration" && self.has_keyword_child(node, "const") + } + + // (extract_* functions continue in impl blocks below) +} + +/// classifyTsClassMember (#808): a class field is a METHOD only when its value +/// is callable (arrow / function expression / HOF call wrapping one). +#[derive(PartialEq)] +enum Member { + Method, + Property, +} + +fn classify_ts_class_member(node: Node) -> Member { + if !matches!(node.kind(), "public_field_definition" | "field_definition") { + return Member::Method; + } + for i in 0..node.named_child_count() { + let Some(child) = node.named_child(i) else { continue }; + if matches!(child.kind(), "arrow_function" | "function_expression") { + return Member::Method; + } + if child.kind() == "call_expression" { + if let Some(args) = child.child_by_field_name("arguments") { + for j in 0..args.named_child_count() { + if let Some(arg) = args.named_child(j) { + if matches!(arg.kind(), "arrow_function" | "function_expression") { + return Member::Method; + } + } + } + } + } + } + Member::Property +} + +/// typescriptExtractor.resolveBody / javascriptExtractor.resolveBody: the body +/// of a function-valued class field, nested in the arrow / HOF-wrapped arrow. +fn resolve_field_body(node: Node) -> Option { + if !matches!(node.kind(), "public_field_definition" | "field_definition") { + return None; + } + for i in 0..node.named_child_count() { + let child = node.named_child(i)?; + if matches!(child.kind(), "arrow_function" | "function_expression") { + return child.child_by_field_name("body"); + } + if child.kind() == "call_expression" { + if let Some(args) = child.child_by_field_name("arguments") { + for j in 0..args.named_child_count() { + if let Some(arg) = args.named_child(j) { + if matches!(arg.kind(), "arrow_function" | "function_expression") { + return arg.child_by_field_name("body"); + } + } + } + } + } + } + None +} + +/// resolveBody ?? getChildByField(node, 'body') — the body-walk resolution. +fn body_of(node: Node) -> Option { + resolve_field_body(node).or_else(|| node.child_by_field_name("body")) +} + +fn opt_str(arena: &mut Arena, s: Option<&str>) -> StrRef { + match s { + Some(s) => arena.put(s), + None => NONE_STR, + } +} diff --git a/codegraph-kernel/src/tsjs/util.rs b/codegraph-kernel/src/tsjs/util.rs new file mode 100644 index 0000000..d322a19 --- /dev/null +++ b/codegraph-kernel/src/tsjs/util.rs @@ -0,0 +1,190 @@ +//! Shared utilities for the TS/JS walker: compiled regexes, UTF-16 position +//! conversion, generated-file detection, and small text helpers — each +//! mirroring a specific helper in src/extraction/tree-sitter.ts (noted inline). + +use regex::Regex; +use std::sync::OnceLock; + +macro_rules! re { + ($name:ident, $pat:expr) => { + pub fn $name() -> &'static Regex { + static RE: OnceLock = OnceLock::new(); + RE.get_or_init(|| Regex::new($pat).expect(concat!("regex ", stringify!($name)))) + } + }; +} + +// RTK_HOOK_NAME_RE (tree-sitter.ts) +re!(rtk_hook_name, r"^use[A-Z][A-Za-z0-9]*(?:Query|Mutation)$"); +// reactComponentHoc's styled test +re!(styled_callee, r"^styled\b"); +// PascalCase component gate (#841) +re!(pascal_case, r"^[A-Z]"); +// extractCall parenthesized-conversion normalization +re!(paren_conversion, r"^\(\s*\*?\s*([A-Za-z_][\w.]*)\s*\)$"); +// flushFnRefCandidates SIMPLE_NAME +re!(simple_name, r"^[A-Za-z_$][A-Za-z0-9_$]*$"); +// flushFnRefCandidates QUALIFIED_IMPORT +re!(qualified_import, r"^[A-Za-z_$][A-Za-z0-9_$.\\]*[.\\]([A-Za-z_$][A-Za-z0-9_$]*)$"); +// captureFnRefCandidates rhs param-storage skip — trailing identifier of LHS +re!(lhs_last_name, r"([A-Za-z_$][A-Za-z0-9_$]*)\s*$"); +// extractTsTupleContractNames identifier test +re!(ident_dollar, r"^[A-Za-z_$][A-Za-z0-9_$]*$"); +// looksLikeVueStoreFile signal (VUE_STORE_FILE_SIGNAL) +re!( + vue_store_signal, + r"\bdefineStore\b|\bcreateStore\b|\bVuex\b|\bmutations\b|\bactions\b|\bgetters\b|\bnamespaced\b" +); +// value-ref target-name distinctiveness: /[A-Z_]/ +re!(has_upper_or_underscore, r"[A-Z_]"); + +/// isGeneratedFile (src/extraction/generated-detection.ts) — full pattern list +/// ported so future language walkers share it. +pub fn is_generated_file(file_path: &str) -> bool { + static RES: OnceLock> = OnceLock::new(); + let patterns = RES.get_or_init(|| { + [ + r"\.pb\.go$", + r"\.pulsar\.go$", + r"_grpc\.pb\.go$", + r"_mock\.go$", + r"_mocks\.go$", + r"^mock_[^/]+\.go$", + r"\.generated\.[jt]sx?$", + r"\.gen\.[jt]sx?$", + r"\.pb\.[jt]s$", + r"_pb\.[jt]s$", + r"_grpc_pb\.[jt]s$", + r"\.min\.m?js$", + r"_pb2(_grpc)?\.py$", + r"_pb2\.pyi$", + r"\.pb\.(cc|h)$", + r"\.g\.cs$", + r"Grpc\.cs$", + r"OuterClass\.java$", + r"Grpc\.java$", + r"\.pb\.swift$", + r"\.g\.dart$", + r"\.freezed\.dart$", + r"\.pb\.dart$", + r"\.pbgrpc\.dart$", + r"\.chopper\.dart$", + r"\.generated\.rs$", + ] + .iter() + .map(|p| Regex::new(p).expect("generated pattern")) + .collect() + }); + patterns.iter().any(|p| p.is_match(file_path)) +} + +/// Byte offsets of each line start, for UTF-16 column conversion. +pub fn line_starts(src: &str) -> Vec { + let mut out = vec![0usize]; + for (i, b) in src.bytes().enumerate() { + if b == b'\n' { + out.push(i + 1); + } + } + out +} + +/// UTF-16 code units in `s` — what web-tree-sitter (and JS string ops) +/// count, so kernel-emitted columns are byte-identical to the wasm path's. +pub fn utf16_len(s: &str) -> usize { + s.chars().map(|c| c.len_utf16()).sum() +} + +/// Column (UTF-16 units) of `byte_pos` on line `row`, given `line_starts`. +pub fn col16(src: &str, starts: &[usize], row: usize, byte_pos: usize) -> u32 { + let ls = starts.get(row).copied().unwrap_or(0); + if byte_pos <= ls { + return 0; + } + utf16_len(&src[ls..byte_pos]) as u32 +} + +/// JS `String.prototype.slice(0, n)` in UTF-16 units, without splitting a +/// surrogate pair (when the cut would split one, we stop one code unit short — +/// a lone surrogate isn't representable in Rust and never round-trips through +/// SQLite anyway). Returns (sliced, was_truncated_at_or_beyond_n). +pub fn slice_utf16(s: &str, n: usize) -> (String, bool) { + let mut used = 0usize; + let mut out = String::new(); + for c in s.chars() { + let w = c.len_utf16(); + if used + w > n { + return (out, true); + } + used += w; + out.push(c); + if used == n { + // Exactly at the limit: truncated iff any source remains. + let truncated = out.len() < s.len(); + return (out, truncated); + } + } + (out, false) +} + +/// objectKeyName (tree-sitter.ts): strip ONE leading and ONE trailing quote +/// character (`'`, `"`, or backtick). +pub fn object_key_name(s: &str) -> String { + let mut out = s; + if let Some(first) = out.chars().next() { + if first == '\'' || first == '"' || first == '`' { + out = &out[first.len_utf8()..]; + } + } + if let Some(last) = out.chars().last() { + if last == '\'' || last == '"' || last == '`' { + out = &out[..out.len() - last.len_utf8()]; + } + } + out.to_string() +} + +/// The `= [...]` initializer signature used by +/// extractVariable (its `.length >= 100` check fires exactly when the slice +/// hit the cap). +pub fn init_signature(value_text: &str) -> String { + let (sliced, _) = slice_utf16(value_text, 100); + if utf16_len(&sliced) >= 100 { + format!("= {sliced}...") + } else { + format!("= {sliced}") + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn utf16_cols() { + let src = "aé😀b"; + // 'a'=1, 'é'=1, '😀'=2 utf16 units; bytes: a=1, é=2, 😀=4 + assert_eq!(utf16_len(src), 5); + let starts = line_starts(src); + assert_eq!(col16(src, &starts, 0, 1), 1); // after 'a' + assert_eq!(col16(src, &starts, 0, 3), 2); // after 'é' + assert_eq!(col16(src, &starts, 0, 7), 4); // after '😀' + } + + #[test] + fn init_sig_short_and_long() { + assert_eq!(init_signature("[1, 2]"), "= [1, 2]"); + let long = "x".repeat(150); + let sig = init_signature(&long); + assert!(sig.starts_with("= ")); + assert!(sig.ends_with("...")); + assert_eq!(utf16_len(&sig[2..sig.len() - 3]), 100); + } + + #[test] + fn generated_patterns() { + assert!(is_generated_file("src/api.generated.ts")); + assert!(is_generated_file("vendor/jquery.min.js")); + assert!(!is_generated_file("src/app.ts")); + } +} diff --git a/docs/design/rust-kernel-migration-plan.md b/docs/design/rust-kernel-migration-plan.md index 8a22c74..556ced1 100644 --- a/docs/design/rust-kernel-migration-plan.md +++ b/docs/design/rust-kernel-migration-plan.md @@ -14,7 +14,11 @@ Work top to bottom; each step has a section below with the detail. - [x] **R1. Scaffold the napi-rs crate** (`codegraph-kernel`): buffer contract, generic `.scm` emitter, build-pipeline integration, `CODEGRAPH_KERNEL=0` kill switch, wasm fallback, grammar-source-parity CI. (§3) — **done 2026-07-16, see §3a.** -- [ ] **R2. Port TypeScript/JavaScript extraction** (tsx/jsx included) as language one. (§4) +- [x] **R2. Port TypeScript/JavaScript extraction** (tsx/jsx included) as language one. (§4) + — **ported 2026-07-16, see §4a**: full-fidelity Rust walker, byte-parity on this repo + (353 files) + excalidraw (643 files) + torture fixtures; extraction 2.6× single-thread. + R3's gate (large repo, retrieval invariants, agent A/B, Linux/Windows) still gates + default-on. - [ ] **R3. Run TS/JS through the equivalence gate** — graph parity, retrieval invariants, agent A/B, perf + control repo. Ship behind the env flag, then default-on. (§5) - [ ] **R4. Port Java** → re-run the dubbo benchmark → the cbm-parity headline. (§4, §6) @@ -132,6 +136,42 @@ to wasm is the universal fallback. Zero-native-build-on-install stays true. - **Known R2 gate item:** native columns are UTF-8 byte offsets; web-tree-sitter's are UTF-16-derived — column NUMBERS on non-ASCII lines will differ in parity dumps (text, lines, IDs unaffected). Classify or normalize when it shows up. + **RESOLVED in R2:** the walker emits UTF-16 columns natively (util::col16), and JS + string-slicing semantics (signature truncation at 100/80/120 units) are reproduced in + UTF-16 units too — no column/slice diff class exists. + +### 4a. R2 — TS/JS port SHIPPED 2026-07-16 (and a §3 design revision) + +- **The generic `.scm` emitter is superseded.** Real TS/JS parity needs logic queries + can't express (extractCall's receiver-qualified callees, store/RTK/component + recognition, fn-ref capture+gating, value-ref shadow pruning, docstring wrapper + climbs) — so R2 replaced the R1 query emitter with a **bespoke per-language walker** + (`codegraph-kernel/src/tsjs/`, ~1,900 lines) that mirrors `TreeSitterExtractor`'s + TS/JS paths function-for-function, bug-for-bug. emitter.rs + queries/ are deleted + (git has them); expect T1 languages (java/python/go) to be walkers too. The + `post(result, source)` TS escape hatch remains available but TS/JS needed none. +- **Parity evidence (macOS):** `scripts/kernel-parity.mjs` (multiset diff of + canonicalized nodes/edges/refs per file, FULL objects) — this repo 353/353 files, + excalidraw 643/643 files (10,650 nodes / 10,726 edges / 68,307 refs), plus + checked-in torture fixtures (`__tests__/fixtures/kernel-parity/`) covering + components/HOCs/styled, zustand-through-middleware, RTK endpoints+hooks, vuex/pinia, + fn-refs (incl `this.x` + shadowing gates), value-refs (incl the shadow prune), + decorators, enums, type-alias members + tuple contracts, re-exports, JSX. Kept alive + in `npm test` by `__tests__/kernel-tsjs-parity.test.ts` (strict full-object compare). +- **One decoder bug found by the strict compare:** decode.ts pre-filled + `filePath`/`language` on refs; wasm extractors leave them unset (the store + denormalizes via `?? filePath`). Fixed — the seam contract is "exactly what + extractFromSource returns", not "what the store makes of it". +- **Perf (M3 Pro, excalidraw 643 files / 7MB):** extraction single-thread 487ms kernel + vs 1,255ms wasm (**2.6×**, identical outputs). End-to-end `init` on an 11-core host + moves only ~3.4s → ~3.2s — parse is a small, already-pool-parallelized slice there; + the win concentrates on constrained hardware (2-core CI class) and kernel-scale + parse (R6). Headroom if R4's dubbo target needs it: arena interning, memoized + UTF-16 line prefixes, and skipping wasm-grammar loads in workers for kernel-routed + languages (worker cold-start). +- **Not yet done (R3 gate):** large-repo parity (vscode-class), full-repo dump-diff + through the DB, retrieval invariants, agent A/B, Linux docker + Windows VM parity + runs, control-repo perf. Routing stays opt-in (`CODEGRAPH_KERNEL_LANGS`) until then. ## 4. Per-language tracker @@ -150,7 +190,7 @@ parity before porting the language. | Language(s) | Today | Tier | Grammar source | Migration notes / known traps | Status | |---|---|---|---|---|---| -| typescript, tsx, javascript, jsx | `languages/typescript.ts`, `javascript.ts` + shared branches | T1 | crates.io | First target. Value-reference edges (#895/#897) and component recognition (#841 forwardRef/memo/styled) must survive — they're extraction-side. Largest test surface; gate is strictest here. | ☐ | +| typescript, tsx, javascript, jsx | `languages/typescript.ts`, `javascript.ts` + shared branches | T1 | crates.io | First target. Value-reference edges (#895/#897) and component recognition (#841 forwardRef/memo/styled) must survive — they're extraction-side. Largest test surface; gate is strictest here. **PORTED (§4a) — value-refs, component recognition, fn-refs, stores all byte-parity; awaiting the R3 gate before default-on.** | ◐ | | java | `languages/java.ts` | T1 | crates.io | Second target; unlocks the dubbo-parity claim. Lombok member synthesis (#912) is a NODE synthesizer hook in extraction (`synthesizeMembers`) — port or keep as TS post-pass. | ☐ | | python | `languages/python.ts` | T1 | crates.io | Third. Decorator extraction feeds framework route detection — parity required. | ☐ | | go | `languages/go.ts` | T1 | crates.io | Third (tie). Value-reference edges ship here too (#897). | ☐ | diff --git a/scripts/kernel-parity.mjs b/scripts/kernel-parity.mjs new file mode 100644 index 0000000..6550334 --- /dev/null +++ b/scripts/kernel-parity.mjs @@ -0,0 +1,214 @@ +#!/usr/bin/env node +/** + * Kernel↔wasm extraction parity harness (R2/R3 of the kernel migration). + * + * Runs BOTH extraction paths over the given files/directories and diffs the + * per-file ExtractionResults as sets (nodes/edges/refs, canonicalized), so a + * behavioral gap in the native kernel shows up as a categorized diff instead + * of a graph-dump surprise later. This is the fast inner loop; the §5 gate's + * full-repo dump-diff still runs before any default-on. + * + * Usage: + * node scripts/kernel-parity.mjs ... [--lang typescript,tsx] + * [--max-samples N] [--list-files] + * + * Requires: npm run build (dist/) and a staged kernel (npm run build:kernel). + * Exit code: 0 = parity, 1 = diffs found, 2 = setup error. + */ + +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); +const dist = (p) => path.join(ROOT, 'dist', p); + +const args = process.argv.slice(2); +const paths = []; +let langFilter = null; +let maxSamples = 5; +let listFiles = false; +for (let i = 0; i < args.length; i++) { + if (args[i] === '--lang') langFilter = new Set(args[++i].split(',')); + else if (args[i] === '--max-samples') maxSamples = Number(args[++i]); + else if (args[i] === '--list-files') listFiles = true; + else paths.push(args[i]); +} +if (paths.length === 0) { + console.error('usage: kernel-parity.mjs ... [--lang ts,tsx] [--max-samples N]'); + process.exit(2); +} + +const KERNEL_LANGS = new Set(['typescript', 'tsx', 'javascript', 'jsx']); +const EXTS = new Map([ + ['.ts', 'typescript'], ['.mts', 'typescript'], ['.cts', 'typescript'], + ['.tsx', 'tsx'], ['.js', 'javascript'], ['.mjs', 'javascript'], + ['.cjs', 'javascript'], ['.jsx', 'jsx'], +]); + +/** Collect candidate files. */ +function collect(p, out) { + const st = fs.statSync(p); + if (st.isDirectory()) { + const base = path.basename(p); + if (base === 'node_modules' || base === '.git' || base === 'dist' || base === '.codegraph') return; + for (const e of fs.readdirSync(p)) collect(path.join(p, e), out); + } else if (EXTS.has(path.extname(p))) { + const lang = EXTS.get(path.extname(p)); + if (!langFilter || langFilter.has(lang)) out.push({ file: p, lang }); + } +} + +const files = []; +for (const p of paths) collect(path.resolve(p), files); +if (files.length === 0) { + console.error('no matching files'); + process.exit(2); +} + +// --- load the built engine --------------------------------------------------- +const { extractFromSource } = await import(dist('extraction/tree-sitter.js')); +const { initGrammars, loadGrammarsForLanguages } = await import(dist('extraction/grammars.js')); +const kernel = await import(dist('extraction/kernel/index.js')); + +await initGrammars(); +await loadGrammarsForLanguages([...KERNEL_LANGS]); + +if (!kernel.getKernel()) { + console.error('kernel .node not found — run: npm run build:kernel'); + process.exit(2); +} + +// --- canonicalization --------------------------------------------------------- +/** + * Node identity for cross-referencing edges/refs: the node id itself (both + * paths compute the same deterministic ids, and id embeds kind+name+line). + */ +function canonNode(n) { + const out = { + id: n.id, kind: n.kind, name: n.name, qualifiedName: n.qualifiedName, + filePath: n.filePath, language: n.language, + startLine: n.startLine, endLine: n.endLine, + startColumn: n.startColumn, endColumn: n.endColumn, + }; + for (const k of ['docstring', 'signature', 'visibility', 'isExported', 'isAsync', 'isStatic', 'isAbstract', 'returnType']) { + if (n[k] !== undefined) out[k] = n[k]; + } + if (n.decorators !== undefined) out.decorators = n.decorators; + if (n.typeParameters !== undefined) out.typeParameters = n.typeParameters; + return JSON.stringify(out); +} + +function canonEdge(e) { + const out = { source: e.source, target: e.target, kind: e.kind }; + if (e.line !== undefined) out.line = e.line; + if (e.column !== undefined) out.column = e.column; + if (e.provenance !== undefined) out.provenance = e.provenance; + if (e.metadata !== undefined) out.metadata = e.metadata; + return JSON.stringify(out); +} + +function canonRef(r) { + // FULL object — a field only one path sets is a parity bug (the vitest + // parity suite caught decode.ts pre-filling filePath/language this way). + const out = { + from: r.fromNodeId, name: r.referenceName, kind: r.referenceKind, + line: r.line, column: r.column, + }; + for (const k of ['filePath', 'language', 'candidates', 'rowId']) { + if (r[k] !== undefined) out[k] = r[k]; + } + return JSON.stringify(out); +} + +function diffSets(aList, bList) { + const a = new Map(); // canon -> count (multiset — duplicates matter) + const b = new Map(); + for (const x of aList) a.set(x, (a.get(x) ?? 0) + 1); + for (const x of bList) b.set(x, (b.get(x) ?? 0) + 1); + const onlyA = []; + const onlyB = []; + for (const [k, c] of a) { + const d = c - (b.get(k) ?? 0); + for (let i = 0; i < d; i++) onlyA.push(k); + } + for (const [k, c] of b) { + const d = c - (a.get(k) ?? 0); + for (let i = 0; i < d; i++) onlyB.push(k); + } + return { onlyA, onlyB }; +} + +// --- run ---------------------------------------------------------------------- +const buckets = new Map(); // category -> {count, samples[]} +function report(category, sample) { + let b = buckets.get(category); + if (!b) buckets.set(category, (b = { count: 0, samples: [] })); + b.count++; + if (b.samples.length < maxSamples) b.samples.push(sample); +} + +let filesWithDiffs = 0; +let filesOk = 0; +let kernelFailed = 0; +let totals = { nodes: 0, edges: 0, refs: 0 }; + +process.env.CODEGRAPH_KERNEL_LANGS = 'all'; + +for (const { file, lang } of files) { + const source = fs.readFileSync(file, 'utf8'); + const rel = path.relative(ROOT, file); + + delete process.env.CODEGRAPH_KERNEL; // kernel path on + const kres = kernel.tryKernelExtract(rel, source, lang); + if (!kres) { + kernelFailed++; + report('kernel-extract-failed', rel); + continue; + } + process.env.CODEGRAPH_KERNEL = '0'; // wasm path + const wres = extractFromSource(rel, source, lang); + delete process.env.CODEGRAPH_KERNEL; + + totals.nodes += wres.nodes.length; + totals.edges += wres.edges.length; + totals.refs += wres.unresolvedReferences.length; + + let fileHasDiff = false; + const tables = [ + ['node', wres.nodes.map(canonNode), kres.nodes.map(canonNode)], + ['edge', wres.edges.map(canonEdge), kres.edges.map(canonEdge)], + ['ref', wres.unresolvedReferences.map(canonRef), kres.unresolvedReferences.map(canonRef)], + ]; + for (const [table, wasm, kern] of tables) { + const { onlyA, onlyB } = diffSets(wasm, kern); + for (const x of onlyA) { + fileHasDiff = true; + const o = JSON.parse(x); + report(`${table}:missing-in-kernel:${o.kind ?? ''}`, `${rel}: ${x}`); + } + for (const x of onlyB) { + fileHasDiff = true; + const o = JSON.parse(x); + report(`${table}:extra-in-kernel:${o.kind ?? ''}`, `${rel}: ${x}`); + } + } + if (fileHasDiff) { + filesWithDiffs++; + if (listFiles) console.log(`DIFF ${rel}`); + } else { + filesOk++; + } +} + +console.log(`\n=== kernel parity: ${filesOk}/${files.length} files byte-parity` + + ` (${filesWithDiffs} with diffs, ${kernelFailed} kernel-failed)` + + ` | wasm totals: ${totals.nodes} nodes / ${totals.edges} edges / ${totals.refs} refs ===\n`); + +const sorted = [...buckets.entries()].sort((a, b) => b[1].count - a[1].count); +for (const [cat, { count, samples }] of sorted) { + console.log(`--- ${cat}: ${count}`); + for (const s of samples) console.log(` ${s.length > 400 ? s.slice(0, 400) + '…' : s}`); +} + +process.exit(filesWithDiffs > 0 || kernelFailed > 0 ? 1 : 0); diff --git a/src/extraction/kernel/decode.ts b/src/extraction/kernel/decode.ts index 2171aa5..59bad99 100644 --- a/src/extraction/kernel/decode.ts +++ b/src/extraction/kernel/decode.ts @@ -148,6 +148,9 @@ export function decodeExtractBuffers( const row = buffers.refs.subarray(i * REF_ROW_SIZE, (i + 1) * REF_ROW_SIZE); const fromIdx = row.readUInt32LE(REF.fromIdx); const kindByte = row.readUInt8(REF.kind); + // No filePath/language here: the wasm extractors emit refs WITHOUT the + // denormalized fields (the store fills them via `ref.filePath ?? filePath`), + // and the kernel must match the extractFromSource seam exactly. const ref: UnresolvedReference = { fromNodeId: fromIdx === NONE ? str(arena, row, REF.fromIdStr)! : idByRow[fromIdx]!, referenceName: str(arena, row, REF.referenceName)!, @@ -157,8 +160,6 @@ export function decodeExtractBuffers( : (EDGE_KINDS[kindByte] as ReferenceKind), line: row.readUInt32LE(REF.line), column: row.readUInt32LE(REF.column), - filePath, - language, }; const candidates = strList(arena, row, REF.candidates); if (candidates !== undefined) ref.candidates = candidates;