feat(kernel): R2 — full TypeScript/JavaScript extraction port, byte-parity with the wasm path
Replaces the R1 seed .scm emitter with a bespoke Rust walker (codegraph-kernel/src/tsjs/) that mirrors TreeSitterExtractor's TS/JS paths function-for-function: declarations (incl. #808 field/property classification), qualified names, docstrings (#780 wrapper climbs), signatures, imports/re-exports + per-binding refs, calls with receiver-qualified callees (#1230 literal-receiver skip), instantiations, decorators, inheritance, type annotations (#381), type-alias members + tuple contracts (#359/#634), React component recognition (#841 forwardRef/memo/styled), object-of-functions / zustand-through-middleware / RTK Query endpoints + generated hooks / vuex + pinia store shapes, function-as-value capture with the flush gate (#756), and value-reference edges with the shadow prune (#895/#897). The generic query emitter is deleted — extraction parity needs logic .scm can't express; future languages get walkers too (migration plan §4a). Positions and JS string-slice semantics are emitted in UTF-16 code units natively, so kernel output is byte-identical to web-tree-sitter's — no column diff class exists. Parity evidence (macOS): scripts/kernel-parity.mjs (full-object multiset diff per file) — this repo 353/353 files, excalidraw 643/643 (10,650 nodes / 10,726 edges / 68,307 refs), plus torture fixtures checked into __tests__/fixtures/kernel-parity/ and enforced in npm test by kernel-tsjs-parity.test.ts. The strict compare caught one real decoder bug the loose harness missed: refs must NOT carry denormalized filePath/language at the extractFromSource seam (the store fills them). Perf: extraction 2.6× single-thread on excalidraw (487ms vs 1,255ms, identical outputs). Routing stays opt-in (CODEGRAPH_KERNEL_LANGS) until the R3 equivalence gate (large repo, DB dump-diff, retrieval invariants, agent A/B, Linux/Windows) passes. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Colby McHenry
co-authored by
Claude Fable 5
Claude Fable 5
parent
c5eebe6beb
commit
9ad5cd7ba2
@@ -0,0 +1,75 @@
|
||||
/**
|
||||
* JS-grammar torture fixture (javascript variant: no type machinery, JS class
|
||||
* fields use `field_definition` with a `property` field).
|
||||
*/
|
||||
import { EventEmitter } from 'node:events';
|
||||
const { promisify } = require('node:util');
|
||||
|
||||
/** Legacy prototype-style helper. */
|
||||
function legacyHelper(a, b) {
|
||||
return a + b;
|
||||
}
|
||||
|
||||
const arrow = (x) => legacyHelper(x, 1);
|
||||
|
||||
class Widget extends EventEmitter {
|
||||
static registry = new Map();
|
||||
#privateField = 1;
|
||||
label = 'w';
|
||||
onTick = () => {
|
||||
this.render();
|
||||
};
|
||||
wrapped = debounce(function () {
|
||||
expensive();
|
||||
}, 50);
|
||||
|
||||
constructor(opts) {
|
||||
super();
|
||||
this.opts = opts;
|
||||
register(this.onTick);
|
||||
}
|
||||
|
||||
render() {
|
||||
paint(this.label);
|
||||
}
|
||||
|
||||
static create(opts) {
|
||||
return new Widget(opts);
|
||||
}
|
||||
}
|
||||
|
||||
// AMD-style wrapper — anonymous, but inner functions must still surface.
|
||||
(function () {
|
||||
function hiddenInner() {
|
||||
return 7;
|
||||
}
|
||||
hiddenInner();
|
||||
})();
|
||||
|
||||
module.exports.makeWidget = function makeWidget(opts) {
|
||||
return Widget.create(opts);
|
||||
};
|
||||
|
||||
// Vuex module shape (store-file signals: mutations + actions + getters).
|
||||
const mutations = {
|
||||
SET_USER(state, user) {
|
||||
state.user = user;
|
||||
},
|
||||
};
|
||||
const actions = {
|
||||
async loadUser({ commit }, id) {
|
||||
const user = await fetchUser(id);
|
||||
commit('SET_USER', user);
|
||||
},
|
||||
};
|
||||
export default {
|
||||
namespaced: true,
|
||||
state: () => ({ user: null }),
|
||||
mutations,
|
||||
actions,
|
||||
getters: {
|
||||
userName(state) {
|
||||
return state.user?.name;
|
||||
},
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,214 @@
|
||||
/**
|
||||
* Torture fixture — exercises every TS/TSX extraction path the kernel ports.
|
||||
*/
|
||||
import React, { forwardRef, memo, useState as useStateAlias } from 'react';
|
||||
import * as NS from './namespace-module';
|
||||
import DefaultThing from './default-module';
|
||||
import './side-effect';
|
||||
import { helperFn, CONFIG_TABLE } from './helpers';
|
||||
|
||||
export { reExported, orig as aliased } from './barrel-source';
|
||||
export * from './star-source';
|
||||
|
||||
// A documented constant table (value-ref target).
|
||||
const RETRY_LIMITS = { a: 1, b: 2 };
|
||||
export const API_BASE = 'https://example.test';
|
||||
let plainLet = 42;
|
||||
var oldVar = 'x';
|
||||
|
||||
/** Class docs.
|
||||
* Multi-line.
|
||||
*/
|
||||
@Injectable()
|
||||
@scoped.Registry<Config>('name')
|
||||
export abstract class BaseService extends EventTarget implements Disposable, Serializable {
|
||||
static instances = 0;
|
||||
private readonly cache: Map<string, Config> = new Map();
|
||||
public fonts: FontConfig;
|
||||
count = 0;
|
||||
onScroll = throttle((e: Event) => {
|
||||
this.handleScroll(e);
|
||||
}, 100);
|
||||
handleClick = (ev: MouseEvent): void => {
|
||||
emitTelemetry(ev);
|
||||
new AbortController();
|
||||
};
|
||||
|
||||
@Get('/list')
|
||||
async list(query: QueryOpts): Promise<Result<Item>> {
|
||||
const local: ItemMapper = makeMapper();
|
||||
const limit = RETRY_LIMITS;
|
||||
return this.cache.get(query.key) ?? helperFn(query);
|
||||
}
|
||||
|
||||
private static compute(n: number): number {
|
||||
return n * RETRY_LIMITS.a;
|
||||
}
|
||||
|
||||
get size(): number {
|
||||
return this.count;
|
||||
}
|
||||
|
||||
protected dispose(): void {
|
||||
listeners.forEach((l) => unregister(l));
|
||||
}
|
||||
}
|
||||
|
||||
class Plain {
|
||||
constructor(private svc: BaseService) {
|
||||
register(this.onEvent);
|
||||
btn.on('click', this.handleClick);
|
||||
}
|
||||
onEvent() {}
|
||||
handleClick() {}
|
||||
}
|
||||
|
||||
export interface Shape extends Named, Sized {
|
||||
area: number;
|
||||
resize(f: number): Shape;
|
||||
onchange: (s: Shape) => void;
|
||||
}
|
||||
|
||||
export enum Direction {
|
||||
Up,
|
||||
Down = 2,
|
||||
Left,
|
||||
}
|
||||
|
||||
export type Handle = {
|
||||
stop: () => void;
|
||||
id: string;
|
||||
refresh(force: boolean): Promise<void>;
|
||||
};
|
||||
|
||||
export type ServiceList = [
|
||||
Service<'query_apply_record', Req, Resp>,
|
||||
Service<'apply_confirm', Req, Resp>,
|
||||
];
|
||||
|
||||
export type MaybeShape = Shape | null;
|
||||
|
||||
export function topLevel(a: ShapeConfig): Shape {
|
||||
const inner = () => reallyDeep();
|
||||
function namedInner(): void {
|
||||
chained(a).value;
|
||||
}
|
||||
namedInner();
|
||||
return makeShape(a);
|
||||
}
|
||||
|
||||
async function* generatorFn(items: Item[]) {
|
||||
yield* items.map((i) => transform(i));
|
||||
}
|
||||
|
||||
export const arrowConst = async (x: number) => {
|
||||
return x + plainLet;
|
||||
};
|
||||
|
||||
const AnonClassHolder = class {
|
||||
method() {}
|
||||
};
|
||||
|
||||
// React components (#841).
|
||||
export const Button = forwardRef((props: ButtonProps, ref) => {
|
||||
const [state, setState] = useStateAlias(0);
|
||||
useEffect(() => {
|
||||
trackRender();
|
||||
});
|
||||
return <BaseButton ref={ref} onClick={() => setState(state + 1)} {...props} />;
|
||||
});
|
||||
|
||||
export const MemoRow = memo(function Row(props: RowProps) {
|
||||
return <tr className={props.cls}>{props.children}</tr>;
|
||||
});
|
||||
|
||||
export const Wrapped = React.memo(ImportedComponent);
|
||||
export const Styled = styled.button`
|
||||
color: red;
|
||||
`;
|
||||
const memoCache = memo(computeThing); // lowercase — stays a constant
|
||||
|
||||
export function App() {
|
||||
const nodes: GraphNode[] = [];
|
||||
return (
|
||||
<div>
|
||||
<Button label={CONFIG_TABLE.label} />
|
||||
<NS.Panel />
|
||||
{nodes.map((n) => (
|
||||
<MemoRow key={n.id} cls={n.cls} />
|
||||
))}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// Object-of-functions (SvelteKit-style actions map).
|
||||
export const actionsMap = {
|
||||
create: async (input: CreateInput) => {
|
||||
return persistNew(input);
|
||||
},
|
||||
update(input: UpdateInput) {
|
||||
return persistExisting(input);
|
||||
},
|
||||
};
|
||||
|
||||
// Zustand-style store through middleware wrappers.
|
||||
export const useStore = create(
|
||||
persist(
|
||||
(set, get) => ({
|
||||
fetchUser: async (id: string) => {
|
||||
const user = await api.load(id);
|
||||
set({ user });
|
||||
},
|
||||
reset: () => set({ user: null }),
|
||||
}),
|
||||
{ name: 'store' }
|
||||
)
|
||||
);
|
||||
|
||||
// RTK Query.
|
||||
export const widgetApi = createApi({
|
||||
reducerPath: 'widgets',
|
||||
endpoints: (build) => ({
|
||||
getWidget: build.query({
|
||||
query: (id: string) => fetchWidget(id),
|
||||
}),
|
||||
updateWidget: build.mutation({
|
||||
queryFn: async (patch) => {
|
||||
return applyPatch(patch);
|
||||
},
|
||||
}),
|
||||
prebuilt: build.query(makeEndpointConfig()),
|
||||
}),
|
||||
});
|
||||
|
||||
export const { useGetWidgetQuery, useUpdateWidgetMutation } = widgetApi;
|
||||
const { notAHook } = widgetApi;
|
||||
|
||||
// Value-ref shadowing: SHADOWED_LIMIT re-bound locally must be pruned.
|
||||
const SHADOWED_LIMIT = 10;
|
||||
export function readsShadowed() {
|
||||
const SHADOWED_LIMIT = 20;
|
||||
return SHADOWED_LIMIT;
|
||||
}
|
||||
export function readsTable() {
|
||||
return RETRY_LIMITS.b + API_BASE.length;
|
||||
}
|
||||
|
||||
// Fn-ref registrations.
|
||||
registerHandler(helperFn);
|
||||
queueMicrotask(topLevel);
|
||||
const routeTable = { home: topLevel, missing: notDefinedAnywhere };
|
||||
const handlerList = [helperFn, DefaultThing];
|
||||
target.cb = topLevel;
|
||||
const aliasFn = topLevel;
|
||||
|
||||
// Calls with interesting callees.
|
||||
;(topLevel)(cfg);
|
||||
NS.helper.deep(1);
|
||||
"literal".includes('x');
|
||||
[1, 2].map(String);
|
||||
obj?.optMethod?.(3);
|
||||
import('./dynamic-module');
|
||||
new NS.Widget(makeArg());
|
||||
new Map<string, number>();
|
||||
super_weird?.();
|
||||
@@ -124,8 +124,11 @@ describe.skipIf(!kernelBuilt)('kernel scaffold', () => {
|
||||
[file.id, 'helper'],
|
||||
]);
|
||||
for (const r of calls) {
|
||||
expect(r.filePath).toBe('src/utils.ts');
|
||||
expect(r.language).toBe('typescript');
|
||||
// No denormalized filePath/language at the extraction seam — the wasm
|
||||
// extractors leave them unset (the store fills them, `?? filePath`),
|
||||
// and the kernel matches that exactly (see decode.ts).
|
||||
expect(r.filePath).toBeUndefined();
|
||||
expect(r.language).toBeUndefined();
|
||||
expect(r.line).toBeGreaterThan(0);
|
||||
}
|
||||
});
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
/**
|
||||
* Kernel↔wasm TS/JS extraction parity (R2 of the kernel migration).
|
||||
*
|
||||
* Asserts the native walker (codegraph-kernel/src/tsjs/) produces the SAME
|
||||
* ExtractionResult as the wasm TreeSitterExtractor — nodes, edges, and
|
||||
* unresolved refs compared as canonicalized multisets — over:
|
||||
* - the checked-in torture fixtures (every ported feature: components/HOCs,
|
||||
* stores, RTK, vuex, fn-refs, value-ref shadowing, decorators, enums,
|
||||
* type-alias members/tuple contracts, re-exports, JSX, field methods), and
|
||||
* - this repo's own extraction sources (real-world TS).
|
||||
*
|
||||
* The full-repo sweep lives in scripts/kernel-parity.mjs (excalidraw et al.,
|
||||
* run for the §5 gate); this suite keeps the invariant alive in `npm test`.
|
||||
* Skips when no kernel binary is staged; CODEGRAPH_KERNEL_EXPECT=1 turns that
|
||||
* into a failure (wired in kernel-scaffold.test.ts).
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeAll, beforeEach, afterEach } from 'vitest';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import { extractFromSource } from '../src/extraction';
|
||||
import { initGrammars, loadGrammarsForLanguages } from '../src/extraction/grammars';
|
||||
import { tryKernelExtract, resetKernelForTests } from '../src/extraction/kernel';
|
||||
import type { ExtractionResult, Language } from '../src/types';
|
||||
|
||||
const KERNEL_PATH = path.join(
|
||||
__dirname,
|
||||
'..',
|
||||
'codegraph-kernel',
|
||||
'prebuilds',
|
||||
`${process.platform}-${process.arch}`,
|
||||
'codegraph-kernel.node'
|
||||
);
|
||||
const kernelBuilt = fs.existsSync(KERNEL_PATH);
|
||||
|
||||
const FIXTURE_DIR = path.join(__dirname, 'fixtures', 'kernel-parity');
|
||||
const REAL_SOURCES = [
|
||||
'src/extraction/kernel/loader.ts',
|
||||
'src/extraction/kernel/decode.ts',
|
||||
'src/extraction/parse-pool.ts',
|
||||
'src/extraction/function-ref.ts',
|
||||
'src/mcp/tools.ts',
|
||||
];
|
||||
|
||||
function canon(result: ExtractionResult): { nodes: string[]; edges: string[]; refs: string[] } {
|
||||
return {
|
||||
nodes: result.nodes
|
||||
.map(({ updatedAt: _u, ...n }) => JSON.stringify(n, Object.keys(n).sort()))
|
||||
.sort(),
|
||||
edges: result.edges.map((e) => JSON.stringify(e, Object.keys(e).sort())).sort(),
|
||||
refs: result.unresolvedReferences
|
||||
.map((r) => JSON.stringify(r, Object.keys(r).sort()))
|
||||
.sort(),
|
||||
};
|
||||
}
|
||||
|
||||
const ENV_KEYS = ['CODEGRAPH_KERNEL', 'CODEGRAPH_KERNEL_LANGS'] as const;
|
||||
let savedEnv: Record<string, string | undefined>;
|
||||
|
||||
describe.skipIf(!kernelBuilt)('kernel TS/JS extraction parity', () => {
|
||||
beforeAll(async () => {
|
||||
await initGrammars();
|
||||
await loadGrammarsForLanguages(['typescript', 'tsx', 'javascript', 'jsx']);
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
savedEnv = Object.fromEntries(ENV_KEYS.map((k) => [k, process.env[k]]));
|
||||
resetKernelForTests();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
for (const k of ENV_KEYS) {
|
||||
if (savedEnv[k] === undefined) delete process.env[k];
|
||||
else process.env[k] = savedEnv[k];
|
||||
}
|
||||
resetKernelForTests();
|
||||
});
|
||||
|
||||
function assertParity(filePath: string, source: string, language: Language): void {
|
||||
process.env.CODEGRAPH_KERNEL_LANGS = 'all';
|
||||
delete process.env.CODEGRAPH_KERNEL;
|
||||
const viaKernel = tryKernelExtract(filePath, source, language);
|
||||
expect(viaKernel, `kernel extraction failed for ${filePath}`).not.toBeNull();
|
||||
|
||||
process.env.CODEGRAPH_KERNEL = '0';
|
||||
const viaWasm = extractFromSource(filePath, source, language);
|
||||
delete process.env.CODEGRAPH_KERNEL;
|
||||
|
||||
const k = canon(viaKernel!);
|
||||
const w = canon(viaWasm);
|
||||
expect(k.nodes, `${filePath}: nodes`).toEqual(w.nodes);
|
||||
expect(k.edges, `${filePath}: edges`).toEqual(w.edges);
|
||||
expect(k.refs, `${filePath}: refs`).toEqual(w.refs);
|
||||
// Meaningful comparison, not empty-vs-empty.
|
||||
expect(viaWasm.nodes.length).toBeGreaterThan(3);
|
||||
}
|
||||
|
||||
it('torture fixture (tsx): components, stores, RTK, fn-refs, value-refs, decorators', () => {
|
||||
const file = path.join(FIXTURE_DIR, 'torture.tsx');
|
||||
assertParity('fixtures/torture.tsx', fs.readFileSync(file, 'utf8'), 'tsx');
|
||||
});
|
||||
|
||||
it('torture fixture (js): field methods, wrappers, vuex module shape', () => {
|
||||
const file = path.join(FIXTURE_DIR, 'torture.js');
|
||||
assertParity('fixtures/torture.js', fs.readFileSync(file, 'utf8'), 'javascript');
|
||||
});
|
||||
|
||||
it.each(REAL_SOURCES)('real source parity: %s', (rel) => {
|
||||
const file = path.join(__dirname, '..', rel);
|
||||
assertParity(rel, fs.readFileSync(file, 'utf8'), 'typescript');
|
||||
});
|
||||
|
||||
it('typescript fixture parsed as plain typescript variant', () => {
|
||||
// Same content through the non-tsx grammar exercises the typescript
|
||||
// (vs tsx) LangSpec pairing.
|
||||
const file = path.join(__dirname, '..', 'src/extraction/kernel/index.ts');
|
||||
assertParity('src/extraction/kernel/index.ts', fs.readFileSync(file, 'utf8'), 'typescript');
|
||||
});
|
||||
});
|
||||
Generated
+1
-1
@@ -49,8 +49,8 @@ dependencies = [
|
||||
"napi",
|
||||
"napi-build",
|
||||
"napi-derive",
|
||||
"regex",
|
||||
"sha2",
|
||||
"streaming-iterator",
|
||||
"tree-sitter",
|
||||
"tree-sitter-javascript",
|
||||
"tree-sitter-typescript",
|
||||
|
||||
@@ -13,8 +13,8 @@ crate-type = ["cdylib"]
|
||||
napi = { version = "3", default-features = false, features = ["napi8"] }
|
||||
napi-derive = "3"
|
||||
tree-sitter = "0.25"
|
||||
streaming-iterator = "0.1"
|
||||
sha2 = "0.10"
|
||||
regex = "1"
|
||||
|
||||
# Grammars — MUST stay revision-matched with the wasm grammars the fallback
|
||||
# path loads (tree-sitter-wasms npm package / src/extraction/wasm/). The
|
||||
|
||||
@@ -1,12 +0,0 @@
|
||||
; Seed query for the R1 scaffold (JavaScript / JSX grammar) — smoke-level
|
||||
; coverage only; R2 replaces this with the full port. See typescript.scm for
|
||||
; the capture convention.
|
||||
|
||||
(class_declaration name: (identifier) @name) @def.class
|
||||
(function_declaration name: (identifier) @name) @def.function
|
||||
(generator_function_declaration name: (identifier) @name) @def.function
|
||||
(method_definition name: (property_identifier) @name) @def.method
|
||||
|
||||
(call_expression function: (identifier) @ref.calls)
|
||||
(call_expression function: (member_expression property: (property_identifier) @ref.calls))
|
||||
(new_expression constructor: (identifier) @ref.instantiates)
|
||||
@@ -1,21 +0,0 @@
|
||||
; Seed query for the R1 scaffold — smoke-level coverage that proves the
|
||||
; buffer contract and emitter mechanics end to end. NOT extraction parity:
|
||||
; R2 replaces this with the full TypeScript/TSX port.
|
||||
;
|
||||
; Capture convention (see emitter.rs):
|
||||
; @def.<NodeKind> — the declaration node; pairs with @name in the pattern
|
||||
; @name — the declaration's name node
|
||||
; @ref.<EdgeKind> — a reference; the capture's own text is the name
|
||||
|
||||
(class_declaration name: (type_identifier) @name) @def.class
|
||||
(abstract_class_declaration name: (type_identifier) @name) @def.class
|
||||
(interface_declaration name: (type_identifier) @name) @def.interface
|
||||
(enum_declaration name: (identifier) @name) @def.enum
|
||||
(type_alias_declaration name: (type_identifier) @name) @def.type_alias
|
||||
(function_declaration name: (identifier) @name) @def.function
|
||||
(generator_function_declaration name: (identifier) @name) @def.function
|
||||
(method_definition name: (property_identifier) @name) @def.method
|
||||
|
||||
(call_expression function: (identifier) @ref.calls)
|
||||
(call_expression function: (member_expression property: (property_identifier) @ref.calls))
|
||||
(new_expression constructor: (identifier) @ref.instantiates)
|
||||
@@ -316,6 +316,15 @@ impl Tables {
|
||||
}
|
||||
}
|
||||
|
||||
/// One file's encoded tables, ready to hand across the JS boundary.
|
||||
pub struct EmitOut {
|
||||
pub meta: Vec<u8>,
|
||||
pub nodes: Vec<u8>,
|
||||
pub edges: Vec<u8>,
|
||||
pub refs: Vec<u8>,
|
||||
pub arena: Vec<u8>,
|
||||
}
|
||||
|
||||
pub fn build_meta(t: &Tables, arena_len: u32, errors_json: StrRef, duration_ms: f64) -> Vec<u8> {
|
||||
let mut m = Vec::with_capacity(META_SIZE);
|
||||
m.push(KERNEL_ABI_VERSION);
|
||||
|
||||
@@ -1,300 +0,0 @@
|
||||
//! Generic query-driven emitter: parse the file, run the language's `.scm`
|
||||
//! query, and emit flat rows. The whole tree walk happens native-side; the
|
||||
//! only JS boundary crossing is the returned buffers.
|
||||
//!
|
||||
//! Mechanics mirrored from `TreeSitterExtractor` (src/extraction/tree-sitter.ts):
|
||||
//! - node row 0 is the file node (`file:<path>`, endLine = newline count + 1,
|
||||
//! isExported present+false — byte-parity with the TS file node);
|
||||
//! - definitions form a scope stack by byte-range nesting; qualifiedName is
|
||||
//! the stack's names joined with `::` (file excluded);
|
||||
//! - every definition gets a `contains` edge from its parent scope (the
|
||||
//! file node when top-level);
|
||||
//! - references attach to the innermost enclosing definition, falling back
|
||||
//! to the file node — same as the TS extractor's nodeStack semantics;
|
||||
//! - definitions with empty names are skipped (issue #42 semantics).
|
||||
|
||||
use crate::buffers::{
|
||||
build_meta, edge_kind_index, node_kind_index, Arena, BoolFlags, EdgeRow, NodeRow, RefRow,
|
||||
Tables, FLAG_IS_EXPORTED, FUNCTION_REF_CODE, NODE_KINDS, NONE, NONE_STR,
|
||||
};
|
||||
use crate::ids;
|
||||
use crate::langs::LangSpec;
|
||||
use streaming_iterator::StreamingIterator;
|
||||
use tree_sitter::{Node, Parser, QueryCursor};
|
||||
|
||||
pub struct EmitOut {
|
||||
pub meta: Vec<u8>,
|
||||
pub nodes: Vec<u8>,
|
||||
pub edges: Vec<u8>,
|
||||
pub refs: Vec<u8>,
|
||||
pub arena: Vec<u8>,
|
||||
}
|
||||
|
||||
/// What a query capture name means. Resolved once per query.
|
||||
#[derive(Clone, Copy)]
|
||||
enum Role {
|
||||
/// `@def.<NodeKind>` — value is the NODE_KINDS index.
|
||||
Def(u8),
|
||||
/// `@name` — the paired definition's name node.
|
||||
Name,
|
||||
/// `@ref.<EdgeKind>` / `@ref.function_ref` — value is the wire code.
|
||||
Ref(u8),
|
||||
/// Helper captures (`@_anchor` etc.) — ignored.
|
||||
Ignore,
|
||||
}
|
||||
|
||||
fn resolve_roles(capture_names: &[&str], lang: &str) -> Result<Vec<Role>, String> {
|
||||
capture_names
|
||||
.iter()
|
||||
.map(|name| {
|
||||
if let Some(kind) = name.strip_prefix("def.") {
|
||||
let idx = node_kind_index(kind)
|
||||
.ok_or_else(|| format!("{lang}: unknown NodeKind in capture @{name}"))?;
|
||||
Ok(Role::Def(idx))
|
||||
} else if let Some(kind) = name.strip_prefix("ref.") {
|
||||
if kind == "function_ref" {
|
||||
return Ok(Role::Ref(FUNCTION_REF_CODE));
|
||||
}
|
||||
let idx = edge_kind_index(kind)
|
||||
.ok_or_else(|| format!("{lang}: unknown EdgeKind in capture @{name}"))?;
|
||||
Ok(Role::Ref(idx))
|
||||
} else if *name == "name" {
|
||||
Ok(Role::Name)
|
||||
} else {
|
||||
Ok(Role::Ignore)
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
struct Def {
|
||||
kind: u8,
|
||||
name_start: usize,
|
||||
name_end: usize,
|
||||
start_byte: usize,
|
||||
end_byte: usize,
|
||||
start_line: u32,
|
||||
end_line: u32,
|
||||
start_column: u32,
|
||||
end_column: u32,
|
||||
/// Node-table row index, assigned during the scope sweep.
|
||||
row: u32,
|
||||
}
|
||||
|
||||
struct RefCap {
|
||||
kind: u8,
|
||||
name_start: usize,
|
||||
name_end: usize,
|
||||
start_byte: usize,
|
||||
line: u32,
|
||||
column: u32,
|
||||
}
|
||||
|
||||
pub fn extract(file_path: &str, source: &str, spec: &LangSpec) -> Result<EmitOut, String> {
|
||||
let t0 = std::time::Instant::now();
|
||||
|
||||
let mut parser = Parser::new();
|
||||
parser
|
||||
.set_language(spec.language())
|
||||
.map_err(|e| format!("set_language({}) failed: {e}", spec.name))?;
|
||||
let tree = parser
|
||||
.parse(source, None)
|
||||
.ok_or_else(|| "parser returned null tree".to_string())?;
|
||||
let root = tree.root_node();
|
||||
|
||||
let query = spec.query()?;
|
||||
let roles = resolve_roles(&query.capture_names(), spec.name)?;
|
||||
|
||||
// ---- Collect definition + reference captures from the query. ----
|
||||
let mut defs: Vec<Def> = Vec::new();
|
||||
let mut refs: Vec<RefCap> = Vec::new();
|
||||
// A node can match several patterns (e.g. nested alternations); first
|
||||
// pattern wins, mirroring the TS walk's one-node-one-symbol behaviour.
|
||||
let mut seen_defs = std::collections::HashSet::<usize>::new();
|
||||
|
||||
let mut cursor = QueryCursor::new();
|
||||
let mut matches = cursor.matches(query, root, source.as_bytes());
|
||||
while let Some(m) = matches.next() {
|
||||
let mut def_node: Option<(Node, u8)> = None;
|
||||
let mut name_node: Option<Node> = None;
|
||||
for cap in m.captures {
|
||||
match roles[cap.index as usize] {
|
||||
Role::Def(kind) => def_node = Some((cap.node, kind)),
|
||||
Role::Name => name_node = Some(cap.node),
|
||||
Role::Ref(kind) => {
|
||||
let p = cap.node.start_position();
|
||||
refs.push(RefCap {
|
||||
kind,
|
||||
name_start: cap.node.start_byte(),
|
||||
name_end: cap.node.end_byte(),
|
||||
start_byte: cap.node.start_byte(),
|
||||
line: p.row as u32 + 1,
|
||||
column: p.column as u32,
|
||||
});
|
||||
}
|
||||
Role::Ignore => {}
|
||||
}
|
||||
}
|
||||
if let (Some((node, kind)), Some(name)) = (def_node, name_node) {
|
||||
// Empty names are not meaningful symbols (issue #42).
|
||||
if name.end_byte() > name.start_byte() && seen_defs.insert(node.id()) {
|
||||
let sp = node.start_position();
|
||||
let ep = node.end_position();
|
||||
defs.push(Def {
|
||||
kind,
|
||||
name_start: name.start_byte(),
|
||||
name_end: name.end_byte(),
|
||||
start_byte: node.start_byte(),
|
||||
end_byte: node.end_byte(),
|
||||
start_line: sp.row as u32 + 1,
|
||||
end_line: ep.row as u32 + 1,
|
||||
start_column: sp.column as u32,
|
||||
end_column: ep.column as u32,
|
||||
row: 0,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Deterministic pre-order regardless of query-match ordering.
|
||||
defs.sort_by(|a, b| {
|
||||
a.start_byte
|
||||
.cmp(&b.start_byte)
|
||||
.then(b.end_byte.cmp(&a.end_byte))
|
||||
});
|
||||
refs.sort_by_key(|r| r.start_byte);
|
||||
|
||||
// ---- Emit rows: file node first, then the scope-stack sweep. ----
|
||||
let mut arena = Arena::default();
|
||||
let mut tables = Tables::default();
|
||||
|
||||
let line_count = source.bytes().filter(|b| *b == b'\n').count() as u32 + 1;
|
||||
let base_name = file_path.rsplit(['/', '\\']).next().unwrap_or(file_path);
|
||||
let mut file_flags = BoolFlags::default();
|
||||
file_flags.set(FLAG_IS_EXPORTED, false);
|
||||
let file_id = arena.put(&ids::file_node_id(file_path));
|
||||
let file_name = arena.put(base_name);
|
||||
let file_qn = arena.put(file_path);
|
||||
tables.push_node(&NodeRow {
|
||||
kind: node_kind_index("file").unwrap(),
|
||||
visibility: 0,
|
||||
flags: file_flags,
|
||||
start_line: 1,
|
||||
end_line: line_count,
|
||||
start_column: 0,
|
||||
end_column: 0,
|
||||
name: file_name,
|
||||
qualified_name: file_qn,
|
||||
id: file_id,
|
||||
docstring: NONE_STR,
|
||||
signature: NONE_STR,
|
||||
decorators: NONE_STR,
|
||||
type_parameters: NONE_STR,
|
||||
return_type: NONE_STR,
|
||||
extra_json: NONE_STR,
|
||||
});
|
||||
|
||||
// Merged sweep over definitions and references in byte order, maintaining
|
||||
// the scope stack (indices into `defs`).
|
||||
let mut stack: Vec<usize> = Vec::new();
|
||||
let mut ref_i = 0usize;
|
||||
|
||||
fn pop_to(stack: &mut Vec<usize>, defs: &[Def], byte: usize) {
|
||||
while let Some(&top) = stack.last() {
|
||||
if defs[top].end_byte <= byte {
|
||||
stack.pop();
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let emit_ref = |r: &RefCap, stack: &[usize], defs: &[Def], arena: &mut Arena, tables: &mut Tables| {
|
||||
let from_idx = stack.last().map(|&i| defs[i].row).unwrap_or(0);
|
||||
let name = arena.put(&source[r.name_start..r.name_end]);
|
||||
tables.push_ref(&RefRow {
|
||||
from_idx,
|
||||
kind: r.kind,
|
||||
line: r.line,
|
||||
column: r.column,
|
||||
reference_name: name,
|
||||
candidates: NONE_STR,
|
||||
from_id_str: NONE_STR,
|
||||
});
|
||||
};
|
||||
|
||||
for i in 0..defs.len() {
|
||||
let def_start = defs[i].start_byte;
|
||||
while ref_i < refs.len() && refs[ref_i].start_byte < def_start {
|
||||
pop_to(&mut stack, &defs, refs[ref_i].start_byte);
|
||||
emit_ref(&refs[ref_i], &stack, &defs, &mut arena, &mut tables);
|
||||
ref_i += 1;
|
||||
}
|
||||
pop_to(&mut stack, &defs, def_start);
|
||||
|
||||
let name = &source[defs[i].name_start..defs[i].name_end];
|
||||
let kind_str = NODE_KINDS[defs[i].kind as usize];
|
||||
// qualifiedName = enclosing definition names + own name, `::`-joined
|
||||
// (buildQualifiedName semantics; file node excluded).
|
||||
let mut qn = String::new();
|
||||
for &s in stack.iter() {
|
||||
qn.push_str(&source[defs[s].name_start..defs[s].name_end]);
|
||||
qn.push_str("::");
|
||||
}
|
||||
qn.push_str(name);
|
||||
|
||||
let id = ids::node_id(file_path, kind_str, name, defs[i].start_line);
|
||||
let id_ref = arena.put(&id);
|
||||
let name_ref = arena.put(name);
|
||||
let qn_ref = arena.put(&qn);
|
||||
let row = tables.push_node(&NodeRow {
|
||||
kind: defs[i].kind,
|
||||
visibility: 0,
|
||||
flags: BoolFlags::default(),
|
||||
start_line: defs[i].start_line,
|
||||
end_line: defs[i].end_line,
|
||||
start_column: defs[i].start_column,
|
||||
end_column: defs[i].end_column,
|
||||
name: name_ref,
|
||||
qualified_name: qn_ref,
|
||||
id: id_ref,
|
||||
docstring: NONE_STR,
|
||||
signature: NONE_STR,
|
||||
decorators: NONE_STR,
|
||||
type_parameters: NONE_STR,
|
||||
return_type: NONE_STR,
|
||||
extra_json: NONE_STR,
|
||||
});
|
||||
defs[i].row = row;
|
||||
|
||||
let parent_row = stack.last().map(|&s| defs[s].row).unwrap_or(0);
|
||||
tables.push_edge(&EdgeRow {
|
||||
source_idx: parent_row,
|
||||
target_idx: row,
|
||||
kind: edge_kind_index("contains").unwrap(),
|
||||
provenance: 0,
|
||||
line: NONE,
|
||||
column: NONE,
|
||||
metadata_json: NONE_STR,
|
||||
source_id_str: NONE_STR,
|
||||
target_id_str: NONE_STR,
|
||||
});
|
||||
|
||||
stack.push(i);
|
||||
}
|
||||
while ref_i < refs.len() {
|
||||
pop_to(&mut stack, &defs, refs[ref_i].start_byte);
|
||||
emit_ref(&refs[ref_i], &stack, &defs, &mut arena, &mut tables);
|
||||
ref_i += 1;
|
||||
}
|
||||
|
||||
let duration_ms = t0.elapsed().as_secs_f64() * 1000.0;
|
||||
let meta = build_meta(&tables, arena.len(), NONE_STR, duration_ms);
|
||||
Ok(EmitOut {
|
||||
meta,
|
||||
nodes: tables.nodes,
|
||||
edges: tables.edges,
|
||||
refs: tables.refs,
|
||||
arena: arena.into_vec(),
|
||||
})
|
||||
}
|
||||
@@ -1,78 +1,27 @@
|
||||
//! Per-language specs: grammar + `.scm` query + (later) per-language config.
|
||||
//! Grammar registry: codegraph `Language` string → native tree-sitter grammar.
|
||||
//!
|
||||
//! Tier-1 languages are meant to be *mostly* a query file plus a small config
|
||||
//! here; logic queries can't express stays TS-side as a per-language `post()`
|
||||
//! hook over the returned buffers (see `src/extraction/kernel/route.ts`).
|
||||
//! Mirrors the wasm side's `WASM_GRAMMAR_FILES` mapping (src/extraction/
|
||||
//! grammars.ts): `tsx` and `jsx` reuse another language's grammar exactly the
|
||||
//! way the wasm map does. The kernel-grammar-parity test asserts each entry is
|
||||
//! built from the SAME grammar revision as the vendored wasm — bump the crate
|
||||
//! and the wasm together.
|
||||
//!
|
||||
//! Language strings are codegraph `Language` values (src/types.ts), not
|
||||
//! grammar names — `tsx` and `jsx` are separate entries that reuse another
|
||||
//! entry's grammar exactly like `WASM_GRAMMAR_FILES` does on the wasm path.
|
||||
//! (R1 shipped a generic `.scm`-query emitter here; R2 replaced it with the
|
||||
//! bespoke per-language walker — see tsjs/ and the migration plan §3a — because
|
||||
//! extraction parity needs logic queries can't express. New languages add a
|
||||
//! grammar entry + a walker module.)
|
||||
|
||||
use std::sync::OnceLock;
|
||||
use tree_sitter::{Language, Query};
|
||||
use tree_sitter::Language;
|
||||
|
||||
pub struct LangSpec {
|
||||
/// codegraph Language string (src/types.ts).
|
||||
pub name: &'static str,
|
||||
get_language: fn() -> Language,
|
||||
query_src: &'static str,
|
||||
language: OnceLock<Language>,
|
||||
query: OnceLock<Result<Query, String>>,
|
||||
}
|
||||
/// Languages this kernel binary can extract (reported by contractInfo;
|
||||
/// TS-side routing policy decides what actually routes).
|
||||
pub const LANGUAGES: [&str; 4] = ["typescript", "tsx", "javascript", "jsx"];
|
||||
|
||||
impl LangSpec {
|
||||
const fn new(name: &'static str, get_language: fn() -> Language, query_src: &'static str) -> Self {
|
||||
LangSpec {
|
||||
name,
|
||||
get_language,
|
||||
query_src,
|
||||
language: OnceLock::new(),
|
||||
query: OnceLock::new(),
|
||||
pub fn grammar_for(language: &str) -> Option<Language> {
|
||||
match language {
|
||||
"typescript" => Some(tree_sitter_typescript::LANGUAGE_TYPESCRIPT.into()),
|
||||
"tsx" => Some(tree_sitter_typescript::LANGUAGE_TSX.into()),
|
||||
"javascript" | "jsx" => Some(tree_sitter_javascript::LANGUAGE.into()),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn language(&self) -> &Language {
|
||||
self.language.get_or_init(self.get_language)
|
||||
}
|
||||
|
||||
pub fn query(&self) -> Result<&Query, String> {
|
||||
self.query
|
||||
.get_or_init(|| {
|
||||
Query::new(self.language(), self.query_src)
|
||||
.map_err(|e| format!("query compile failed for {}: {e}", self.name))
|
||||
})
|
||||
.as_ref()
|
||||
.map_err(|e| e.clone())
|
||||
}
|
||||
}
|
||||
|
||||
fn ts_language() -> Language {
|
||||
tree_sitter_typescript::LANGUAGE_TYPESCRIPT.into()
|
||||
}
|
||||
|
||||
fn tsx_language() -> Language {
|
||||
tree_sitter_typescript::LANGUAGE_TSX.into()
|
||||
}
|
||||
|
||||
fn js_language() -> Language {
|
||||
tree_sitter_javascript::LANGUAGE.into()
|
||||
}
|
||||
|
||||
static TYPESCRIPT: LangSpec = LangSpec::new(
|
||||
"typescript",
|
||||
ts_language,
|
||||
include_str!("../queries/typescript.scm"),
|
||||
);
|
||||
static TSX: LangSpec = LangSpec::new("tsx", tsx_language, include_str!("../queries/typescript.scm"));
|
||||
static JAVASCRIPT: LangSpec = LangSpec::new(
|
||||
"javascript",
|
||||
js_language,
|
||||
include_str!("../queries/javascript.scm"),
|
||||
);
|
||||
static JSX: LangSpec = LangSpec::new("jsx", js_language, include_str!("../queries/javascript.scm"));
|
||||
|
||||
pub static ALL: [&LangSpec; 4] = [&TYPESCRIPT, &TSX, &JAVASCRIPT, &JSX];
|
||||
|
||||
pub fn spec_for(language: &str) -> Option<&'static LangSpec> {
|
||||
ALL.iter().find(|s| s.name == language).copied()
|
||||
}
|
||||
|
||||
@@ -9,13 +9,17 @@
|
||||
//! Calls are synchronous by design: the existing `ParseWorkerPool` workers
|
||||
//! already parallelize per-file, so each worker thread drives its own kernel
|
||||
//! call (do NOT rebuild the pool on the Rust side — see the migration plan §3).
|
||||
//!
|
||||
//! Per-language extraction lives in a dedicated walker module (tsjs/ for
|
||||
//! typescript/tsx/javascript/jsx) that mirrors the TS extractor for behavioral
|
||||
//! parity — verified by scripts/kernel-parity.mjs and the §5 gate.
|
||||
|
||||
#![deny(clippy::all)]
|
||||
|
||||
mod buffers;
|
||||
mod emitter;
|
||||
mod ids;
|
||||
mod langs;
|
||||
mod tsjs;
|
||||
|
||||
use napi::bindgen_prelude::*;
|
||||
use napi_derive::napi;
|
||||
@@ -63,14 +67,13 @@ pub fn contract_info() -> ContractInfo {
|
||||
kernel_version: env!("CARGO_PKG_VERSION").to_string(),
|
||||
node_kinds: buffers::NODE_KINDS.iter().map(|s| s.to_string()).collect(),
|
||||
edge_kinds: buffers::EDGE_KINDS.iter().map(|s| s.to_string()).collect(),
|
||||
languages: langs::ALL.iter().map(|s| s.name.to_string()).collect(),
|
||||
languages: langs::LANGUAGES.iter().map(|s| s.to_string()).collect(),
|
||||
}
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub fn grammar_info(language: String) -> Option<GrammarInfo> {
|
||||
let spec = langs::spec_for(&language)?;
|
||||
let lang = spec.language();
|
||||
let lang = langs::grammar_for(&language)?;
|
||||
let node_kind_count = lang.node_kind_count();
|
||||
let field_count = lang.field_count();
|
||||
let node_kinds = (0..node_kind_count)
|
||||
@@ -91,9 +94,7 @@ pub fn grammar_info(language: String) -> Option<GrammarInfo> {
|
||||
|
||||
#[napi]
|
||||
pub fn extract_file(file_path: String, content: String, language: String) -> Result<ExtractBuffers> {
|
||||
let spec = langs::spec_for(&language)
|
||||
.ok_or_else(|| Error::from_reason(format!("kernel does not support language: {language}")))?;
|
||||
let out = emitter::extract(&file_path, &content, spec).map_err(Error::from_reason)?;
|
||||
let out = tsjs::extract(&file_path, &content, &language).map_err(Error::from_reason)?;
|
||||
Ok(ExtractBuffers {
|
||||
meta: out.meta.into(),
|
||||
nodes: out.nodes.into(),
|
||||
|
||||
@@ -0,0 +1,140 @@
|
||||
//! getPrecedingDocstring / cleanCommentMarkers — faithful port of
|
||||
//! src/extraction/tree-sitter-helpers.ts (#780 wrapper-climb semantics).
|
||||
|
||||
use regex::Regex;
|
||||
use std::sync::OnceLock;
|
||||
use tree_sitter::Node;
|
||||
|
||||
/// DOCSTRING_WRAPPER_TYPES (tree-sitter-helpers.ts).
|
||||
fn is_wrapper(kind: &str) -> bool {
|
||||
matches!(
|
||||
kind,
|
||||
"export_statement"
|
||||
| "decorated_definition"
|
||||
| "lexical_declaration"
|
||||
| "variable_declaration"
|
||||
| "variable_declarator"
|
||||
| "ambient_declaration"
|
||||
)
|
||||
}
|
||||
|
||||
fn is_comment(kind: &str) -> bool {
|
||||
matches!(
|
||||
kind,
|
||||
"comment" | "line_comment" | "block_comment" | "documentation_comment"
|
||||
)
|
||||
}
|
||||
|
||||
struct Cleaners {
|
||||
block_open: Regex,
|
||||
block_close: Regex,
|
||||
lua_open: Regex,
|
||||
lua_close: Regex,
|
||||
paren_star_open: Regex,
|
||||
paren_star_close: Regex,
|
||||
brace_open: Regex,
|
||||
brace_close: Regex,
|
||||
slashes: Regex,
|
||||
dashes: Regex,
|
||||
hash: Regex,
|
||||
percent: Regex,
|
||||
star_cont: Regex,
|
||||
}
|
||||
|
||||
fn cleaners() -> &'static Cleaners {
|
||||
static C: OnceLock<Cleaners> = OnceLock::new();
|
||||
C.get_or_init(|| Cleaners {
|
||||
block_open: Regex::new(r"^/\*+!?").unwrap(),
|
||||
block_close: Regex::new(r"\*+/$").unwrap(),
|
||||
lua_open: Regex::new(r"^--\[=*\[").unwrap(),
|
||||
lua_close: Regex::new(r"\]=*\]$").unwrap(),
|
||||
paren_star_open: Regex::new(r"^\(\*").unwrap(),
|
||||
paren_star_close: Regex::new(r"\*\)$").unwrap(),
|
||||
brace_open: Regex::new(r"^\{").unwrap(),
|
||||
brace_close: Regex::new(r"\}$").unwrap(),
|
||||
slashes: Regex::new(r"(?m)^//[/!]?\s?").unwrap(),
|
||||
dashes: Regex::new(r"(?m)^--\s?").unwrap(),
|
||||
hash: Regex::new(r"(?m)^#\s?").unwrap(),
|
||||
percent: Regex::new(r"(?m)^%+\s?").unwrap(),
|
||||
star_cont: Regex::new(r"(?m)^\s*\*\s?").unwrap(),
|
||||
})
|
||||
}
|
||||
|
||||
/// cleanCommentMarkers — strip comment syntax, keep the prose.
|
||||
pub fn clean_comment_markers(comment: &str) -> String {
|
||||
let c = cleaners();
|
||||
let mut s = comment.trim().to_string();
|
||||
if s.starts_with("/*") {
|
||||
s = c.block_open.replace(&s, "").into_owned();
|
||||
s = c.block_close.replace(&s, "").into_owned();
|
||||
} else if s.starts_with("--[") {
|
||||
s = c.lua_open.replace(&s, "").into_owned();
|
||||
s = c.lua_close.replace(&s, "").into_owned();
|
||||
} else if s.starts_with("(*") {
|
||||
s = c.paren_star_open.replace(&s, "").into_owned();
|
||||
s = c.paren_star_close.replace(&s, "").into_owned();
|
||||
} else if s.starts_with('{') {
|
||||
s = c.brace_open.replace(&s, "").into_owned();
|
||||
s = c.brace_close.replace(&s, "").into_owned();
|
||||
}
|
||||
s = c.slashes.replace_all(&s, "").into_owned();
|
||||
s = c.dashes.replace_all(&s, "").into_owned();
|
||||
s = c.hash.replace_all(&s, "").into_owned();
|
||||
s = c.percent.replace_all(&s, "").into_owned();
|
||||
s = c.star_cont.replace_all(&s, "").into_owned();
|
||||
s.trim().to_string()
|
||||
}
|
||||
|
||||
/// getPrecedingDocstring — collect the comment run immediately preceding the
|
||||
/// node (climbing out of declaration wrappers first), cleaned and joined.
|
||||
/// Returns None when there is no preceding comment (a PRESENT-but-empty
|
||||
/// docstring after cleaning still returns Some(""), matching the TS helper).
|
||||
pub fn preceding_docstring(node: Node, src: &str) -> Option<String> {
|
||||
let mut anchor = node;
|
||||
while let Some(parent) = anchor.parent() {
|
||||
if is_wrapper(parent.kind()) {
|
||||
anchor = parent;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
let mut comments: Vec<&str> = Vec::new();
|
||||
let mut sibling = anchor.prev_named_sibling();
|
||||
while let Some(s) = sibling {
|
||||
if is_comment(s.kind()) {
|
||||
comments.push(&src[s.byte_range()]);
|
||||
sibling = s.prev_named_sibling();
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if comments.is_empty() {
|
||||
return None;
|
||||
}
|
||||
comments.reverse(); // collected nearest-first; TS unshifts to keep source order
|
||||
Some(
|
||||
comments
|
||||
.iter()
|
||||
.map(|c| clean_comment_markers(c))
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n")
|
||||
.trim()
|
||||
.to_string(),
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn strips_line_and_block_markers() {
|
||||
assert_eq!(clean_comment_markers("// hello"), "hello");
|
||||
assert_eq!(clean_comment_markers("/// doc line"), "doc line");
|
||||
assert_eq!(
|
||||
clean_comment_markers("/**\n * Adds things.\n * @param a first\n */"),
|
||||
"Adds things.\n@param a first"
|
||||
);
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,133 @@
|
||||
//! Function-as-value capture (#756) — the TS/JS slice of
|
||||
//! src/extraction/function-ref.ts (TS_JS_SPEC): container dispatch, value
|
||||
//! normalization, and the `this.member` special form. The flush-time gate
|
||||
//! lives in the walker (it needs the file's nodes and import refs).
|
||||
|
||||
use tree_sitter::Node;
|
||||
|
||||
/// CaptureMode (function-ref.ts) — gate policy keys on it.
|
||||
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||
pub enum Mode {
|
||||
Args,
|
||||
Rhs,
|
||||
Value,
|
||||
List,
|
||||
VarInit,
|
||||
}
|
||||
|
||||
pub struct Candidate {
|
||||
pub name: String,
|
||||
pub line: u32,
|
||||
pub column_byte: usize, // converted to UTF-16 at emit time
|
||||
pub row: usize,
|
||||
}
|
||||
|
||||
/// NAME_STOPLIST (function-ref.ts).
|
||||
fn stoplisted(name: &str) -> bool {
|
||||
matches!(
|
||||
name,
|
||||
"this" | "self" | "super" | "null" | "nil" | "true" | "false" | "undefined" | "new"
|
||||
| "NULL" | "nullptr" | "None"
|
||||
)
|
||||
}
|
||||
|
||||
/// TS_JS_SPEC.dispatch: container node type → capture mode.
|
||||
pub fn dispatch(kind: &str) -> Option<Mode> {
|
||||
match kind {
|
||||
"arguments" => Some(Mode::Args),
|
||||
"assignment_expression" => Some(Mode::Rhs),
|
||||
"variable_declarator" => Some(Mode::VarInit),
|
||||
"pair" => Some(Mode::Value),
|
||||
"array" => Some(Mode::List),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// captureFnRefCandidates for the TS/JS spec. Returns (candidate, mode) pairs.
|
||||
pub fn capture(container: Node, mode: Mode, src: &str) -> Vec<(Candidate, Mode)> {
|
||||
let mut value_nodes: Vec<Node> = Vec::new();
|
||||
|
||||
match mode {
|
||||
Mode::Args | Mode::List => {
|
||||
for i in 0..container.named_child_count() {
|
||||
if let Some(c) = container.named_child(i) {
|
||||
value_nodes.push(c);
|
||||
}
|
||||
}
|
||||
}
|
||||
Mode::Rhs => {
|
||||
if let Some(rhs) = container.child_by_field_name("right") {
|
||||
// Param-storage skip: `this.status = status` — LHS's trailing
|
||||
// identifier equals the RHS text ⇒ a stored local/parameter.
|
||||
let lhs_text = container
|
||||
.child_by_field_name("left")
|
||||
.map(|l| &src[l.byte_range()])
|
||||
.unwrap_or("");
|
||||
let lhs_last = super::util::lhs_last_name()
|
||||
.captures(lhs_text)
|
||||
.and_then(|c| c.get(1))
|
||||
.map(|m| m.as_str());
|
||||
let rhs_text = src[rhs.byte_range()].trim();
|
||||
if !(lhs_last.is_some() && lhs_last == Some(rhs_text)) {
|
||||
value_nodes.push(rhs);
|
||||
}
|
||||
}
|
||||
}
|
||||
Mode::Value => {
|
||||
if let Some(v) = container.child_by_field_name("value") {
|
||||
value_nodes.push(v);
|
||||
}
|
||||
}
|
||||
Mode::VarInit => {
|
||||
// Destructuring extracts DATA, never a function alias.
|
||||
let name_node = container.child_by_field_name("name");
|
||||
let is_pattern = name_node
|
||||
.map(|n| matches!(n.kind(), "object_pattern" | "array_pattern"))
|
||||
.unwrap_or(false);
|
||||
if !is_pattern {
|
||||
if let Some(v) = container.child_by_field_name("value") {
|
||||
value_nodes.push(v);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut out = Vec::new();
|
||||
for v in value_nodes {
|
||||
for (name, node) in normalize(v, src) {
|
||||
if name.is_empty() || stoplisted(&name) {
|
||||
continue;
|
||||
}
|
||||
let p = node.start_position();
|
||||
out.push((
|
||||
Candidate {
|
||||
name,
|
||||
line: p.row as u32 + 1,
|
||||
column_byte: node.start_byte(),
|
||||
row: p.row,
|
||||
},
|
||||
mode,
|
||||
));
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// normalizeValue for the TS/JS spec: bare identifiers, plus the
|
||||
/// `this.<member>` member_expression special form (object EXACTLY `this`).
|
||||
fn normalize<'t>(node: Node<'t>, src: &str) -> Vec<(String, Node<'t>)> {
|
||||
match node.kind() {
|
||||
"identifier" => vec![(src[node.byte_range()].to_string(), node)],
|
||||
"member_expression" => {
|
||||
let obj = node.child_by_field_name("object");
|
||||
let prop = node.child_by_field_name("property");
|
||||
if let (Some(o), Some(p)) = (obj, prop) {
|
||||
if o.kind() == "this" && p.kind() == "property_identifier" {
|
||||
return vec![(format!("this.{}", &src[p.byte_range()]), p)];
|
||||
}
|
||||
}
|
||||
vec![]
|
||||
}
|
||||
_ => vec![],
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,876 @@
|
||||
//! TypeScript / TSX / JavaScript / JSX extraction — a faithful Rust port of
|
||||
//! `TreeSitterExtractor`'s TS/JS paths (src/extraction/tree-sitter.ts) plus
|
||||
//! the typescript/javascript LanguageExtractor configs.
|
||||
//!
|
||||
//! Porting contract (R2 of the migration plan): behavior parity with the wasm
|
||||
//! path, verified by scripts/kernel-parity.mjs over real repos — including
|
||||
//! bug-for-bug fidelity where the TS code has quirks. Every function notes the
|
||||
//! TS function it mirrors; if you change one side, change the other or the
|
||||
//! parity gate fails. Positions are emitted in UTF-16 code units (what
|
||||
//! web-tree-sitter reports), see util::col16.
|
||||
|
||||
mod docstring;
|
||||
mod extractors;
|
||||
mod fnref;
|
||||
pub(crate) mod util;
|
||||
|
||||
use crate::buffers::{
|
||||
build_meta, edge_kind_index, node_kind_index, Arena, BoolFlags, EdgeRow, EmitOut, NodeRow,
|
||||
RefRow, StrRef, Tables, FLAG_IS_ASYNC, FLAG_IS_EXPORTED, FLAG_IS_STATIC, FUNCTION_REF_CODE,
|
||||
NONE, NONE_STR,
|
||||
};
|
||||
use crate::ids;
|
||||
use crate::langs;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use tree_sitter::{Node, Parser};
|
||||
|
||||
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||
pub enum Variant {
|
||||
Typescript,
|
||||
Tsx,
|
||||
Javascript,
|
||||
Jsx,
|
||||
}
|
||||
|
||||
impl Variant {
|
||||
pub fn from_language(language: &str) -> Option<Variant> {
|
||||
match language {
|
||||
"typescript" => Some(Variant::Typescript),
|
||||
"tsx" => Some(Variant::Tsx),
|
||||
"javascript" => Some(Variant::Javascript),
|
||||
"jsx" => Some(Variant::Jsx),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
/// TS-family (typescript/tsx): type annotations, interfaces, enums,
|
||||
/// aliases, visibility, isStatic. The JS family lacks all of those hooks.
|
||||
fn is_ts(self) -> bool {
|
||||
matches!(self, Variant::Typescript | Variant::Tsx)
|
||||
}
|
||||
/// VALUE_REF_LANGS includes typescript/tsx/javascript but NOT jsx.
|
||||
fn value_refs(self) -> bool {
|
||||
!matches!(self, Variant::Jsx)
|
||||
}
|
||||
}
|
||||
|
||||
/// typescriptExtractor.methodTypes / javascriptExtractor.methodTypes.
|
||||
fn is_method_type(v: Variant, kind: &str) -> bool {
|
||||
kind == "method_definition"
|
||||
|| (v.is_ts() && kind == "public_field_definition")
|
||||
|| (!v.is_ts() && kind == "field_definition")
|
||||
}
|
||||
|
||||
fn is_function_type(kind: &str) -> bool {
|
||||
matches!(kind, "function_declaration" | "arrow_function" | "function_expression")
|
||||
}
|
||||
|
||||
fn is_class_type(v: Variant, kind: &str) -> bool {
|
||||
kind == "class_declaration" || (v.is_ts() && kind == "abstract_class_declaration")
|
||||
}
|
||||
|
||||
fn is_variable_type(kind: &str) -> bool {
|
||||
matches!(kind, "lexical_declaration" | "variable_declaration")
|
||||
}
|
||||
|
||||
/// LITERAL_RECEIVER_TYPES (tree-sitter.ts) — full set; only a handful occur in
|
||||
/// TS/JS grammars but membership is what the TS code tests.
|
||||
fn is_literal_receiver(kind: &str) -> bool {
|
||||
matches!(
|
||||
kind,
|
||||
"string" | "string_literal" | "interpreted_string_literal" | "raw_string_literal"
|
||||
| "template_string" | "concatenated_string" | "formatted_string" | "f_string"
|
||||
| "line_string_literal" | "string_content" | "heredoc_body"
|
||||
| "number" | "number_literal" | "integer" | "integer_literal" | "float"
|
||||
| "float_literal" | "int_literal" | "decimal_integer_literal" | "real_literal"
|
||||
| "char_literal" | "character_literal" | "rune_literal" | "regex" | "regex_literal"
|
||||
| "true" | "false" | "boolean_literal" | "bool_literal" | "none" | "null" | "nil"
|
||||
| "null_literal" | "undefined"
|
||||
| "list" | "list_literal" | "array" | "array_literal" | "array_creation_expression"
|
||||
| "dictionary" | "dict_literal" | "object" | "tuple" | "set"
|
||||
)
|
||||
}
|
||||
|
||||
/// BUILTIN_TYPES (tree-sitter.ts) — names that never become type references.
|
||||
fn is_builtin_type(name: &str) -> bool {
|
||||
matches!(
|
||||
name,
|
||||
"string" | "number" | "boolean" | "void" | "null" | "undefined" | "never" | "any"
|
||||
| "unknown" | "object" | "symbol" | "bigint" | "true" | "false"
|
||||
| "str" | "bool" | "i8" | "i16" | "i32" | "i64" | "i128" | "isize"
|
||||
| "u8" | "u16" | "u32" | "u64" | "u128" | "usize" | "f32" | "f64" | "char"
|
||||
| "int" | "long" | "short" | "byte" | "float" | "double"
|
||||
| "int8" | "int16" | "int32" | "int64" | "uint8" | "uint16" | "uint32" | "uint64"
|
||||
| "float32" | "float64" | "complex64" | "complex128" | "rune" | "error"
|
||||
| "Int" | "Long" | "Short" | "Byte" | "Float" | "Double" | "Boolean" | "Char"
|
||||
| "Unit" | "String" | "Any" | "AnyRef" | "AnyVal" | "Nothing" | "Null"
|
||||
)
|
||||
}
|
||||
|
||||
/// REACT_COMPONENT_HOCS (tree-sitter.ts, #841).
|
||||
fn is_react_hoc(callee: &str) -> bool {
|
||||
matches!(callee, "forwardRef" | "memo" | "React.forwardRef" | "React.memo")
|
||||
}
|
||||
|
||||
fn is_vue_collection_name(name: &str) -> bool {
|
||||
matches!(name, "actions" | "mutations" | "getters")
|
||||
}
|
||||
|
||||
/// One scope-stack entry (TS keeps node IDs; rows are our equivalent).
|
||||
struct Scope {
|
||||
row: u32,
|
||||
kind: &'static str,
|
||||
name: String,
|
||||
}
|
||||
|
||||
/// Extra node properties, per-extract-site (mirrors createNode's `extra`).
|
||||
#[derive(Default)]
|
||||
struct Extra {
|
||||
docstring: Option<String>,
|
||||
signature: Option<String>,
|
||||
visibility: Option<u8>,
|
||||
is_exported: Option<bool>,
|
||||
is_async: Option<bool>,
|
||||
is_static: Option<bool>,
|
||||
qualified_name: Option<String>,
|
||||
}
|
||||
|
||||
struct ValueScope<'t> {
|
||||
row: u32,
|
||||
node: Node<'t>,
|
||||
name: String,
|
||||
}
|
||||
|
||||
pub struct Walker<'t> {
|
||||
src: &'t str,
|
||||
file_path: &'t str,
|
||||
variant: Variant,
|
||||
line_starts: Vec<usize>,
|
||||
arena: Arena,
|
||||
tables: Tables,
|
||||
stack: Vec<Scope>,
|
||||
/// Function/method names defined in this file (fn-ref flush gate).
|
||||
defined_fn_names: HashSet<String>,
|
||||
/// Simple names from `imports` refs (fn-ref flush gate).
|
||||
imported_names: HashSet<String>,
|
||||
fn_ref_cands: Vec<(u32, fnref::Candidate)>,
|
||||
// Value-reference bookkeeping (flushValueRefs).
|
||||
fs_values: HashMap<String, u32>,
|
||||
fs_value_counts: HashMap<String, u32>,
|
||||
value_scopes: Vec<ValueScope<'t>>,
|
||||
vue_store_file: Option<bool>,
|
||||
}
|
||||
|
||||
const MAX_VALUE_REF_NODES: usize = 20_000;
|
||||
|
||||
pub fn extract(file_path: &str, source: &str, language: &str) -> Result<EmitOut, String> {
|
||||
let variant = Variant::from_language(language)
|
||||
.ok_or_else(|| format!("tsjs walker does not handle language: {language}"))?;
|
||||
let grammar = langs::grammar_for(language)
|
||||
.ok_or_else(|| format!("no grammar for language: {language}"))?;
|
||||
|
||||
let t0 = std::time::Instant::now();
|
||||
let mut parser = Parser::new();
|
||||
parser
|
||||
.set_language(&grammar)
|
||||
.map_err(|e| format!("set_language({language}) failed: {e}"))?;
|
||||
let tree = parser
|
||||
.parse(source, None)
|
||||
.ok_or_else(|| "parser returned null tree".to_string())?;
|
||||
|
||||
let mut w = Walker {
|
||||
src: source,
|
||||
file_path,
|
||||
variant,
|
||||
line_starts: util::line_starts(source),
|
||||
arena: Arena::default(),
|
||||
tables: Tables::default(),
|
||||
stack: Vec::new(),
|
||||
defined_fn_names: HashSet::new(),
|
||||
imported_names: HashSet::new(),
|
||||
fn_ref_cands: Vec::new(),
|
||||
fs_values: HashMap::new(),
|
||||
fs_value_counts: HashMap::new(),
|
||||
value_scopes: Vec::new(),
|
||||
vue_store_file: None,
|
||||
};
|
||||
|
||||
// File node (TreeSitterExtractor.extract): id `file:<path>`, endLine =
|
||||
// newline count + 1, isExported explicitly false.
|
||||
let line_count = source.bytes().filter(|b| *b == b'\n').count() as u32 + 1;
|
||||
let base_name = file_path.rsplit(['/', '\\']).next().unwrap_or(file_path);
|
||||
let mut flags = BoolFlags::default();
|
||||
flags.set(FLAG_IS_EXPORTED, false);
|
||||
let file_id = w.arena.put(&ids::file_node_id(file_path));
|
||||
let name_ref = w.arena.put(base_name);
|
||||
let qn_ref = w.arena.put(file_path);
|
||||
w.tables.push_node(&NodeRow {
|
||||
kind: node_kind_index("file").unwrap(),
|
||||
visibility: 0,
|
||||
flags,
|
||||
start_line: 1,
|
||||
end_line: line_count,
|
||||
start_column: 0,
|
||||
end_column: 0,
|
||||
name: name_ref,
|
||||
qualified_name: qn_ref,
|
||||
id: file_id,
|
||||
docstring: NONE_STR,
|
||||
signature: NONE_STR,
|
||||
decorators: NONE_STR,
|
||||
type_parameters: NONE_STR,
|
||||
return_type: NONE_STR,
|
||||
extra_json: NONE_STR,
|
||||
});
|
||||
w.stack.push(Scope { row: 0, kind: "file", name: base_name.to_string() });
|
||||
|
||||
w.visit_node(tree.root_node());
|
||||
|
||||
// End-of-file passes, in the TS extract() order.
|
||||
w.flush_fn_ref_candidates();
|
||||
w.flush_value_refs(tree.root_node());
|
||||
w.stack.pop();
|
||||
|
||||
let duration_ms = t0.elapsed().as_secs_f64() * 1000.0;
|
||||
let meta = build_meta(&w.tables, w.arena.len(), NONE_STR, duration_ms);
|
||||
Ok(EmitOut {
|
||||
meta,
|
||||
nodes: w.tables.nodes,
|
||||
edges: w.tables.edges,
|
||||
refs: w.tables.refs,
|
||||
arena: w.arena.into_vec(),
|
||||
})
|
||||
}
|
||||
|
||||
impl<'t> Walker<'t> {
|
||||
// --- small helpers --------------------------------------------------------
|
||||
|
||||
fn text(&self, node: Node) -> &'t str {
|
||||
&self.src[node.byte_range()]
|
||||
}
|
||||
|
||||
fn line_of(&self, node: Node) -> u32 {
|
||||
node.start_position().row as u32 + 1
|
||||
}
|
||||
|
||||
fn col_of(&self, node: Node) -> u32 {
|
||||
util::col16(self.src, &self.line_starts, node.start_position().row, node.start_byte())
|
||||
}
|
||||
|
||||
fn end_col_of(&self, node: Node) -> u32 {
|
||||
util::col16(self.src, &self.line_starts, node.end_position().row, node.end_byte())
|
||||
}
|
||||
|
||||
fn top_row(&self) -> u32 {
|
||||
self.stack.last().map(|s| s.row).unwrap_or(0)
|
||||
}
|
||||
|
||||
/// isInsideClassLikeNode.
|
||||
fn inside_class_like(&self) -> bool {
|
||||
self.stack
|
||||
.last()
|
||||
.map(|s| matches!(s.kind, "class" | "struct" | "interface" | "trait" | "enum" | "module"))
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
fn push_ref(&mut self, from_row: u32, name: &str, kind_code: u8, node: Node) {
|
||||
let name_ref = self.arena.put(name);
|
||||
self.tables.push_ref(&RefRow {
|
||||
from_idx: from_row,
|
||||
kind: kind_code,
|
||||
line: self.line_of(node),
|
||||
column: self.col_of(node),
|
||||
reference_name: name_ref,
|
||||
candidates: NONE_STR,
|
||||
from_id_str: NONE_STR,
|
||||
});
|
||||
if kind_code == edge_kind_index("imports").unwrap() {
|
||||
// Feed the fn-ref flush gate the same way flushFnRefCandidates
|
||||
// derives importedNames from `imports` refs.
|
||||
if util::simple_name().is_match(name) {
|
||||
self.imported_names.insert(name.to_string());
|
||||
} else if let Some(c) = util::qualified_import().captures(name) {
|
||||
self.imported_names.insert(c[1].to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn push_call_ref(&mut self, name: &str, node: Node) {
|
||||
self.push_ref(self.top_row(), name, edge_kind_index("calls").unwrap(), node);
|
||||
}
|
||||
|
||||
// --- createNode -----------------------------------------------------------
|
||||
|
||||
/// createNode (tree-sitter.ts): id, qualified name from the scope stack,
|
||||
/// contains edge from the parent scope, value-ref bookkeeping.
|
||||
fn create_node(&mut self, kind: &'static str, name: &str, node: Node<'t>, extra: Extra) -> Option<u32> {
|
||||
if name.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let start_line = self.line_of(node);
|
||||
let id = ids::node_id(self.file_path, kind, name, start_line);
|
||||
|
||||
// endLine body extension: resolveBody only (TS/JS: function-valued
|
||||
// class fields whose body nests in the arrow / HOF-wrapped arrow).
|
||||
let mut end_line = node.end_position().row as u32 + 1;
|
||||
if (kind == "function" || kind == "method") && matches!(node.kind(), "public_field_definition" | "field_definition")
|
||||
{
|
||||
if let Some(body) = resolve_field_body(node) {
|
||||
let be = body.end_position().row as u32 + 1;
|
||||
if be > end_line {
|
||||
end_line = be;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let qualified = extra.qualified_name.unwrap_or_else(|| {
|
||||
let mut parts: Vec<&str> = Vec::new();
|
||||
for s in &self.stack {
|
||||
if s.kind != "file" {
|
||||
parts.push(&s.name);
|
||||
}
|
||||
}
|
||||
let mut qn = parts.join("::");
|
||||
if !qn.is_empty() {
|
||||
qn.push_str("::");
|
||||
}
|
||||
qn.push_str(name);
|
||||
qn
|
||||
});
|
||||
|
||||
let mut flags = BoolFlags::default();
|
||||
if let Some(v) = extra.is_exported {
|
||||
flags.set(FLAG_IS_EXPORTED, v);
|
||||
}
|
||||
if let Some(v) = extra.is_async {
|
||||
flags.set(FLAG_IS_ASYNC, v);
|
||||
}
|
||||
if let Some(v) = extra.is_static {
|
||||
flags.set(FLAG_IS_STATIC, v);
|
||||
}
|
||||
|
||||
let name_ref = self.arena.put(name);
|
||||
let qn_ref = self.arena.put(&qualified);
|
||||
let id_ref = self.arena.put(&id);
|
||||
let doc_ref = opt_str(&mut self.arena, extra.docstring.as_deref());
|
||||
let sig_ref = opt_str(&mut self.arena, extra.signature.as_deref());
|
||||
let row = self.tables.push_node(&NodeRow {
|
||||
kind: node_kind_index(kind).unwrap(),
|
||||
visibility: extra.visibility.unwrap_or(0),
|
||||
flags,
|
||||
start_line,
|
||||
end_line,
|
||||
start_column: self.col_of(node),
|
||||
end_column: self.end_col_of(node),
|
||||
name: name_ref,
|
||||
qualified_name: qn_ref,
|
||||
id: id_ref,
|
||||
docstring: doc_ref,
|
||||
signature: sig_ref,
|
||||
decorators: NONE_STR,
|
||||
type_parameters: NONE_STR,
|
||||
return_type: NONE_STR,
|
||||
extra_json: NONE_STR,
|
||||
});
|
||||
|
||||
// Containment edge from the current scope.
|
||||
let parent_row = self.top_row();
|
||||
self.tables.push_edge(&EdgeRow {
|
||||
source_idx: parent_row,
|
||||
target_idx: row,
|
||||
kind: edge_kind_index("contains").unwrap(),
|
||||
provenance: 0,
|
||||
line: NONE,
|
||||
column: NONE,
|
||||
metadata_json: NONE_STR,
|
||||
source_id_str: NONE_STR,
|
||||
target_id_str: NONE_STR,
|
||||
});
|
||||
|
||||
if kind == "function" || kind == "method" {
|
||||
self.defined_fn_names.insert(name.to_string());
|
||||
}
|
||||
self.capture_value_ref_scope(kind, name, row, node);
|
||||
Some(row)
|
||||
}
|
||||
|
||||
// --- value references (captureValueRefScope / flushValueRefs) --------------
|
||||
|
||||
fn capture_value_ref_scope(&mut self, kind: &'static str, name: &str, row: u32, node: Node<'t>) {
|
||||
if !self.variant.value_refs() {
|
||||
return;
|
||||
}
|
||||
let target_kind_ok = kind == "constant" || kind == "variable";
|
||||
if target_kind_ok
|
||||
&& util::utf16_len(name) >= 3
|
||||
&& util::has_upper_or_underscore().is_match(name)
|
||||
{
|
||||
let parent_ok = self
|
||||
.stack
|
||||
.last()
|
||||
.map(|s| matches!(s.kind, "file" | "class" | "module" | "struct" | "enum"))
|
||||
.unwrap_or(false);
|
||||
if parent_ok {
|
||||
self.fs_values.insert(name.to_string(), row);
|
||||
*self.fs_value_counts.entry(name.to_string()).or_insert(0) += 1;
|
||||
}
|
||||
}
|
||||
if matches!(kind, "function" | "method" | "constant" | "variable") {
|
||||
self.value_scopes.push(ValueScope { row, node, name: name.to_string() });
|
||||
}
|
||||
}
|
||||
|
||||
fn flush_value_refs(&mut self, root: Node<'t>) {
|
||||
let scopes = std::mem::take(&mut self.value_scopes);
|
||||
let mut targets = std::mem::take(&mut self.fs_values);
|
||||
let counts = std::mem::take(&mut self.fs_value_counts);
|
||||
if !self.variant.value_refs() || std::env::var("CODEGRAPH_VALUE_REFS").as_deref() == Ok("0") {
|
||||
return;
|
||||
}
|
||||
if targets.is_empty() || scopes.is_empty() || util::is_generated_file(self.file_path) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Shadow prune: count declarators of each target name across the whole
|
||||
// tree; more declarators than file-scope nodes ⇒ an inner re-binding
|
||||
// shadows the target. (TS/JS declarators are `variable_declarator`;
|
||||
// the other kinds in the TS switch belong to other grammars.)
|
||||
let mut decl_counts: HashMap<&str, u32> = HashMap::new();
|
||||
let mut dstack: Vec<Node> = vec![root];
|
||||
let mut dvisited = 0usize;
|
||||
while let Some(n) = dstack.pop() {
|
||||
if dvisited >= MAX_VALUE_REF_NODES {
|
||||
break;
|
||||
}
|
||||
dvisited += 1;
|
||||
if n.kind() == "variable_declarator" {
|
||||
if let Some(first) = n.named_child(0) {
|
||||
if first.kind() == "identifier" {
|
||||
let nm = self.text(first);
|
||||
if targets.contains_key(nm) {
|
||||
*decl_counts.entry(nm).or_insert(0) += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for i in 0..n.named_child_count() {
|
||||
if let Some(c) = n.named_child(i) {
|
||||
dstack.push(c);
|
||||
}
|
||||
}
|
||||
}
|
||||
let shadowed: Vec<String> = decl_counts
|
||||
.iter()
|
||||
.filter(|(nm, c)| **c > counts.get(**nm).copied().unwrap_or(1))
|
||||
.map(|(nm, _)| nm.to_string())
|
||||
.collect();
|
||||
for nm in shadowed {
|
||||
targets.remove(&nm);
|
||||
}
|
||||
if targets.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
let refs_kind = edge_kind_index("references").unwrap();
|
||||
for scope in &scopes {
|
||||
let mut seen: HashSet<u32> = HashSet::new();
|
||||
let mut stack: Vec<Node> = vec![scope.node];
|
||||
let mut visited = 0usize;
|
||||
while let Some(n) = stack.pop() {
|
||||
if visited >= MAX_VALUE_REF_NODES {
|
||||
break;
|
||||
}
|
||||
visited += 1;
|
||||
if matches!(n.kind(), "identifier" | "constant" | "name" | "simple_identifier") {
|
||||
let ref_name = self.text(n);
|
||||
if let Some(&target_row) = targets.get(ref_name) {
|
||||
if target_row != scope.row && ref_name != scope.name && !seen.contains(&target_row) {
|
||||
seen.insert(target_row);
|
||||
let meta = self.arena.put(r#"{"valueRef":true}"#);
|
||||
self.tables.push_edge(&EdgeRow {
|
||||
source_idx: scope.row,
|
||||
target_idx: target_row,
|
||||
kind: refs_kind,
|
||||
provenance: 0,
|
||||
line: NONE,
|
||||
column: NONE,
|
||||
metadata_json: meta,
|
||||
source_id_str: NONE_STR,
|
||||
target_id_str: NONE_STR,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
for i in 0..n.named_child_count() {
|
||||
if let Some(c) = n.named_child(i) {
|
||||
stack.push(c);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- function-as-value refs (#756) -----------------------------------------
|
||||
|
||||
fn maybe_capture_fn_refs(&mut self, node: Node<'t>) {
|
||||
let Some(mode) = fnref::dispatch(node.kind()) else { return };
|
||||
if self.stack.is_empty() {
|
||||
return;
|
||||
}
|
||||
let from = self.top_row();
|
||||
for (cand, _mode) in fnref::capture(node, mode, self.src) {
|
||||
self.fn_ref_cands.push((from, cand));
|
||||
}
|
||||
}
|
||||
|
||||
/// scanFnRefSubtree: capture-only walk of subtrees the main walkers skip.
|
||||
fn scan_fn_ref_subtree(&mut self, node: Node<'t>, depth: u32) {
|
||||
if depth > 12 {
|
||||
return;
|
||||
}
|
||||
let kind = node.kind();
|
||||
if depth > 0
|
||||
&& (is_function_type(kind) || matches!(kind, "lambda_literal" | "lambda_expression"))
|
||||
{
|
||||
return;
|
||||
}
|
||||
self.maybe_capture_fn_refs(node);
|
||||
for i in 0..node.named_child_count() {
|
||||
if let Some(c) = node.named_child(i) {
|
||||
self.scan_fn_ref_subtree(c, depth + 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn flush_fn_ref_candidates(&mut self) {
|
||||
let cands = std::mem::take(&mut self.fn_ref_cands);
|
||||
if cands.is_empty() || util::is_generated_file(self.file_path) {
|
||||
return;
|
||||
}
|
||||
let mut seen: HashSet<(u32, String)> = HashSet::new();
|
||||
for (from, c) in cands {
|
||||
// Gate: `this.<member>` always flushes; everything else must match
|
||||
// a same-file function/method or an imported name. (The `::` and
|
||||
// ungated-mode policies belong to other languages' specs.)
|
||||
if !c.name.starts_with("this.")
|
||||
&& !c.name.contains("::")
|
||||
&& !self.defined_fn_names.contains(&c.name)
|
||||
&& !self.imported_names.contains(&c.name)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
if !seen.insert((from, c.name.clone())) {
|
||||
continue;
|
||||
}
|
||||
let column = util::col16(self.src, &self.line_starts, c.row, c.column_byte);
|
||||
let name_ref = self.arena.put(&c.name);
|
||||
self.tables.push_ref(&RefRow {
|
||||
from_idx: from,
|
||||
kind: FUNCTION_REF_CODE,
|
||||
line: c.line,
|
||||
column,
|
||||
reference_name: name_ref,
|
||||
candidates: NONE_STR,
|
||||
from_id_str: NONE_STR,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// --- the dispatcher (visitNode) --------------------------------------------
|
||||
|
||||
fn visit_node(&mut self, node: Node<'t>) {
|
||||
let kind = node.kind();
|
||||
let mut skip_children = false;
|
||||
|
||||
// Function-as-value capture — independent of the dispatch ladder.
|
||||
self.maybe_capture_fn_refs(node);
|
||||
|
||||
if is_function_type(kind) {
|
||||
// (the isInsideClassLike + methodTypes overlap is Python/Ruby-only)
|
||||
self.extract_function(node, None);
|
||||
skip_children = true;
|
||||
} else if is_class_type(self.variant, kind) {
|
||||
self.extract_class(node);
|
||||
skip_children = true;
|
||||
} else if is_method_type(self.variant, kind) {
|
||||
if classify_ts_class_member(node) == Member::Property {
|
||||
let prop = self.extract_property(node);
|
||||
if let (Some((row, name)), Some(value)) = (prop, node.child_by_field_name("value")) {
|
||||
self.stack.push(Scope { row, kind: "property", name });
|
||||
self.visit_function_body(value);
|
||||
self.stack.pop();
|
||||
}
|
||||
self.scan_fn_ref_subtree(node, 0);
|
||||
} else {
|
||||
self.extract_method(node);
|
||||
}
|
||||
skip_children = true;
|
||||
} else if self.variant.is_ts() && kind == "interface_declaration" {
|
||||
self.extract_interface(node);
|
||||
skip_children = true;
|
||||
} else if self.variant.is_ts() && kind == "enum_declaration" {
|
||||
self.extract_enum(node);
|
||||
skip_children = true;
|
||||
} else if self.variant.is_ts() && kind == "type_alias_declaration" {
|
||||
skip_children = self.extract_type_alias(node);
|
||||
} else if is_variable_type(kind) && !self.inside_class_like() {
|
||||
self.extract_variable(node);
|
||||
self.scan_fn_ref_subtree(node, 0);
|
||||
skip_children = true;
|
||||
} else if kind == "import_statement" {
|
||||
self.extract_import(node);
|
||||
} else if kind == "export_statement" && node.child_by_field_name("source").is_some() {
|
||||
// Re-export: `export { X } from './y'`.
|
||||
self.emit_re_export_refs(node);
|
||||
} else if kind == "export_statement" && self.looks_like_vue_store_file() {
|
||||
// Vuex MODULE default export (`export default { actions: {…} }`).
|
||||
if let Some(exported) = node.child_by_field_name("value") {
|
||||
if matches!(exported.kind(), "object" | "object_expression") {
|
||||
self.extract_store_collection_methods(exported);
|
||||
skip_children = true;
|
||||
}
|
||||
}
|
||||
} else if kind == "call_expression" {
|
||||
self.extract_call(node);
|
||||
} else if kind == "new_expression" {
|
||||
self.extract_instantiation(node);
|
||||
} else if self.variant.is_ts()
|
||||
&& matches!(kind, "property_signature" | "method_signature")
|
||||
&& self.inside_class_like()
|
||||
{
|
||||
let parent = self.top_row();
|
||||
self.extract_type_annotations(node, parent);
|
||||
}
|
||||
|
||||
if !skip_children {
|
||||
for i in 0..node.named_child_count() {
|
||||
if let Some(c) = node.named_child(i) {
|
||||
self.visit_node(c);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- visitFunctionBody ------------------------------------------------------
|
||||
|
||||
fn visit_function_body(&mut self, body: Node<'t>) {
|
||||
self.visit_for_calls_and_structure(body);
|
||||
}
|
||||
|
||||
fn visit_for_calls_and_structure(&mut self, node: Node<'t>) {
|
||||
let kind = node.kind();
|
||||
self.maybe_capture_fn_refs(node);
|
||||
|
||||
if kind == "call_expression" {
|
||||
self.extract_call(node);
|
||||
} else if kind == "new_expression" {
|
||||
self.extract_instantiation(node);
|
||||
}
|
||||
|
||||
// Local variable type annotations (TS family only).
|
||||
if self.variant.is_ts() && kind == "variable_declarator" {
|
||||
let owner = self.top_row();
|
||||
self.extract_variable_type_annotation(node, owner);
|
||||
}
|
||||
|
||||
// Nested NAMED functions become their own nodes.
|
||||
if is_function_type(kind) {
|
||||
let name = self.extract_name(node);
|
||||
if name != "<anonymous>" {
|
||||
self.extract_function(node, None);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if is_class_type(self.variant, kind) {
|
||||
self.extract_class(node);
|
||||
return;
|
||||
}
|
||||
if self.variant.is_ts() && kind == "enum_declaration" {
|
||||
self.extract_enum(node);
|
||||
return;
|
||||
}
|
||||
if self.variant.is_ts() && kind == "interface_declaration" {
|
||||
self.extract_interface(node);
|
||||
return;
|
||||
}
|
||||
|
||||
for i in 0..node.named_child_count() {
|
||||
if let Some(c) = node.named_child(i) {
|
||||
self.visit_for_calls_and_structure(c);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- name / signature / modifier helpers ------------------------------------
|
||||
|
||||
/// extractName / extractNameRaw for the TS/JS configs.
|
||||
fn extract_name(&self, node: Node) -> String {
|
||||
// javascriptExtractor.resolveName: field_definition names its key the
|
||||
// `property` field.
|
||||
if !self.variant.is_ts() && node.kind() == "field_definition" {
|
||||
if let Some(prop) = node.child_by_field_name("property") {
|
||||
return self.text(prop).to_string();
|
||||
}
|
||||
}
|
||||
if let Some(name_node) = node.child_by_field_name("name") {
|
||||
return self.text(name_node).to_string();
|
||||
}
|
||||
if matches!(node.kind(), "arrow_function" | "function_expression") {
|
||||
return "<anonymous>".to_string();
|
||||
}
|
||||
for i in 0..node.named_child_count() {
|
||||
if let Some(c) = node.named_child(i) {
|
||||
if matches!(c.kind(), "identifier" | "type_identifier" | "simple_identifier" | "constant") {
|
||||
return self.text(c).to_string();
|
||||
}
|
||||
}
|
||||
}
|
||||
"<anonymous>".to_string()
|
||||
}
|
||||
|
||||
/// typescriptExtractor.getSignature / javascriptExtractor.getSignature.
|
||||
fn signature_of(&self, node: Node) -> Option<String> {
|
||||
let params = node.child_by_field_name("parameters")?;
|
||||
let mut sig = self.text(params).to_string();
|
||||
if self.variant.is_ts() {
|
||||
if let Some(ret) = node.child_by_field_name("return_type") {
|
||||
let ret_text = self.text(ret);
|
||||
let stripped = ret_text.strip_prefix(':').unwrap_or(ret_text).trim_start();
|
||||
sig.push_str(": ");
|
||||
sig.push_str(stripped);
|
||||
}
|
||||
}
|
||||
Some(sig)
|
||||
}
|
||||
|
||||
/// typescriptExtractor.getVisibility (TS only — JS has no hook).
|
||||
fn visibility_of(&self, node: Node) -> Option<u8> {
|
||||
if !self.variant.is_ts() {
|
||||
return None;
|
||||
}
|
||||
for i in 0..node.child_count() {
|
||||
let child = node.child(i)?;
|
||||
if child.kind() == "accessibility_modifier" {
|
||||
return match self.text(child) {
|
||||
"public" => Some(1),
|
||||
"private" => Some(2),
|
||||
"protected" => Some(3),
|
||||
_ => None,
|
||||
};
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// isExported: walk the parent chain for an export_statement.
|
||||
fn is_exported(&self, node: Node) -> bool {
|
||||
let mut cur = node.parent();
|
||||
while let Some(p) = cur {
|
||||
if p.kind() == "export_statement" {
|
||||
return true;
|
||||
}
|
||||
cur = p.parent();
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
fn has_keyword_child(&self, node: Node, kw: &str) -> bool {
|
||||
for i in 0..node.child_count() {
|
||||
if let Some(c) = node.child(i) {
|
||||
if c.kind() == kw {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
fn is_async(&self, node: Node) -> bool {
|
||||
self.has_keyword_child(node, "async")
|
||||
}
|
||||
|
||||
/// TS has an isStatic hook; JS does not (None = field absent).
|
||||
fn is_static(&self, node: Node) -> Option<bool> {
|
||||
if self.variant.is_ts() {
|
||||
Some(self.has_keyword_child(node, "static"))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
fn is_const_decl(&self, node: Node) -> bool {
|
||||
node.kind() == "lexical_declaration" && self.has_keyword_child(node, "const")
|
||||
}
|
||||
|
||||
// (extract_* functions continue in impl blocks below)
|
||||
}
|
||||
|
||||
/// classifyTsClassMember (#808): a class field is a METHOD only when its value
|
||||
/// is callable (arrow / function expression / HOF call wrapping one).
|
||||
#[derive(PartialEq)]
|
||||
enum Member {
|
||||
Method,
|
||||
Property,
|
||||
}
|
||||
|
||||
fn classify_ts_class_member(node: Node) -> Member {
|
||||
if !matches!(node.kind(), "public_field_definition" | "field_definition") {
|
||||
return Member::Method;
|
||||
}
|
||||
for i in 0..node.named_child_count() {
|
||||
let Some(child) = node.named_child(i) else { continue };
|
||||
if matches!(child.kind(), "arrow_function" | "function_expression") {
|
||||
return Member::Method;
|
||||
}
|
||||
if child.kind() == "call_expression" {
|
||||
if let Some(args) = child.child_by_field_name("arguments") {
|
||||
for j in 0..args.named_child_count() {
|
||||
if let Some(arg) = args.named_child(j) {
|
||||
if matches!(arg.kind(), "arrow_function" | "function_expression") {
|
||||
return Member::Method;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Member::Property
|
||||
}
|
||||
|
||||
/// typescriptExtractor.resolveBody / javascriptExtractor.resolveBody: the body
|
||||
/// of a function-valued class field, nested in the arrow / HOF-wrapped arrow.
|
||||
fn resolve_field_body(node: Node) -> Option<Node> {
|
||||
if !matches!(node.kind(), "public_field_definition" | "field_definition") {
|
||||
return None;
|
||||
}
|
||||
for i in 0..node.named_child_count() {
|
||||
let child = node.named_child(i)?;
|
||||
if matches!(child.kind(), "arrow_function" | "function_expression") {
|
||||
return child.child_by_field_name("body");
|
||||
}
|
||||
if child.kind() == "call_expression" {
|
||||
if let Some(args) = child.child_by_field_name("arguments") {
|
||||
for j in 0..args.named_child_count() {
|
||||
if let Some(arg) = args.named_child(j) {
|
||||
if matches!(arg.kind(), "arrow_function" | "function_expression") {
|
||||
return arg.child_by_field_name("body");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// resolveBody ?? getChildByField(node, 'body') — the body-walk resolution.
|
||||
fn body_of(node: Node) -> Option<Node> {
|
||||
resolve_field_body(node).or_else(|| node.child_by_field_name("body"))
|
||||
}
|
||||
|
||||
fn opt_str(arena: &mut Arena, s: Option<&str>) -> StrRef {
|
||||
match s {
|
||||
Some(s) => arena.put(s),
|
||||
None => NONE_STR,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,190 @@
|
||||
//! Shared utilities for the TS/JS walker: compiled regexes, UTF-16 position
|
||||
//! conversion, generated-file detection, and small text helpers — each
|
||||
//! mirroring a specific helper in src/extraction/tree-sitter.ts (noted inline).
|
||||
|
||||
use regex::Regex;
|
||||
use std::sync::OnceLock;
|
||||
|
||||
macro_rules! re {
|
||||
($name:ident, $pat:expr) => {
|
||||
pub fn $name() -> &'static Regex {
|
||||
static RE: OnceLock<Regex> = OnceLock::new();
|
||||
RE.get_or_init(|| Regex::new($pat).expect(concat!("regex ", stringify!($name))))
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
// RTK_HOOK_NAME_RE (tree-sitter.ts)
|
||||
re!(rtk_hook_name, r"^use[A-Z][A-Za-z0-9]*(?:Query|Mutation)$");
|
||||
// reactComponentHoc's styled test
|
||||
re!(styled_callee, r"^styled\b");
|
||||
// PascalCase component gate (#841)
|
||||
re!(pascal_case, r"^[A-Z]");
|
||||
// extractCall parenthesized-conversion normalization
|
||||
re!(paren_conversion, r"^\(\s*\*?\s*([A-Za-z_][\w.]*)\s*\)$");
|
||||
// flushFnRefCandidates SIMPLE_NAME
|
||||
re!(simple_name, r"^[A-Za-z_$][A-Za-z0-9_$]*$");
|
||||
// flushFnRefCandidates QUALIFIED_IMPORT
|
||||
re!(qualified_import, r"^[A-Za-z_$][A-Za-z0-9_$.\\]*[.\\]([A-Za-z_$][A-Za-z0-9_$]*)$");
|
||||
// captureFnRefCandidates rhs param-storage skip — trailing identifier of LHS
|
||||
re!(lhs_last_name, r"([A-Za-z_$][A-Za-z0-9_$]*)\s*$");
|
||||
// extractTsTupleContractNames identifier test
|
||||
re!(ident_dollar, r"^[A-Za-z_$][A-Za-z0-9_$]*$");
|
||||
// looksLikeVueStoreFile signal (VUE_STORE_FILE_SIGNAL)
|
||||
re!(
|
||||
vue_store_signal,
|
||||
r"\bdefineStore\b|\bcreateStore\b|\bVuex\b|\bmutations\b|\bactions\b|\bgetters\b|\bnamespaced\b"
|
||||
);
|
||||
// value-ref target-name distinctiveness: /[A-Z_]/
|
||||
re!(has_upper_or_underscore, r"[A-Z_]");
|
||||
|
||||
/// isGeneratedFile (src/extraction/generated-detection.ts) — full pattern list
|
||||
/// ported so future language walkers share it.
|
||||
pub fn is_generated_file(file_path: &str) -> bool {
|
||||
static RES: OnceLock<Vec<Regex>> = OnceLock::new();
|
||||
let patterns = RES.get_or_init(|| {
|
||||
[
|
||||
r"\.pb\.go$",
|
||||
r"\.pulsar\.go$",
|
||||
r"_grpc\.pb\.go$",
|
||||
r"_mock\.go$",
|
||||
r"_mocks\.go$",
|
||||
r"^mock_[^/]+\.go$",
|
||||
r"\.generated\.[jt]sx?$",
|
||||
r"\.gen\.[jt]sx?$",
|
||||
r"\.pb\.[jt]s$",
|
||||
r"_pb\.[jt]s$",
|
||||
r"_grpc_pb\.[jt]s$",
|
||||
r"\.min\.m?js$",
|
||||
r"_pb2(_grpc)?\.py$",
|
||||
r"_pb2\.pyi$",
|
||||
r"\.pb\.(cc|h)$",
|
||||
r"\.g\.cs$",
|
||||
r"Grpc\.cs$",
|
||||
r"OuterClass\.java$",
|
||||
r"Grpc\.java$",
|
||||
r"\.pb\.swift$",
|
||||
r"\.g\.dart$",
|
||||
r"\.freezed\.dart$",
|
||||
r"\.pb\.dart$",
|
||||
r"\.pbgrpc\.dart$",
|
||||
r"\.chopper\.dart$",
|
||||
r"\.generated\.rs$",
|
||||
]
|
||||
.iter()
|
||||
.map(|p| Regex::new(p).expect("generated pattern"))
|
||||
.collect()
|
||||
});
|
||||
patterns.iter().any(|p| p.is_match(file_path))
|
||||
}
|
||||
|
||||
/// Byte offsets of each line start, for UTF-16 column conversion.
|
||||
pub fn line_starts(src: &str) -> Vec<usize> {
|
||||
let mut out = vec![0usize];
|
||||
for (i, b) in src.bytes().enumerate() {
|
||||
if b == b'\n' {
|
||||
out.push(i + 1);
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// UTF-16 code units in `s` — what web-tree-sitter (and JS string ops)
|
||||
/// count, so kernel-emitted columns are byte-identical to the wasm path's.
|
||||
pub fn utf16_len(s: &str) -> usize {
|
||||
s.chars().map(|c| c.len_utf16()).sum()
|
||||
}
|
||||
|
||||
/// Column (UTF-16 units) of `byte_pos` on line `row`, given `line_starts`.
|
||||
pub fn col16(src: &str, starts: &[usize], row: usize, byte_pos: usize) -> u32 {
|
||||
let ls = starts.get(row).copied().unwrap_or(0);
|
||||
if byte_pos <= ls {
|
||||
return 0;
|
||||
}
|
||||
utf16_len(&src[ls..byte_pos]) as u32
|
||||
}
|
||||
|
||||
/// JS `String.prototype.slice(0, n)` in UTF-16 units, without splitting a
|
||||
/// surrogate pair (when the cut would split one, we stop one code unit short —
|
||||
/// a lone surrogate isn't representable in Rust and never round-trips through
|
||||
/// SQLite anyway). Returns (sliced, was_truncated_at_or_beyond_n).
|
||||
pub fn slice_utf16(s: &str, n: usize) -> (String, bool) {
|
||||
let mut used = 0usize;
|
||||
let mut out = String::new();
|
||||
for c in s.chars() {
|
||||
let w = c.len_utf16();
|
||||
if used + w > n {
|
||||
return (out, true);
|
||||
}
|
||||
used += w;
|
||||
out.push(c);
|
||||
if used == n {
|
||||
// Exactly at the limit: truncated iff any source remains.
|
||||
let truncated = out.len() < s.len();
|
||||
return (out, truncated);
|
||||
}
|
||||
}
|
||||
(out, false)
|
||||
}
|
||||
|
||||
/// objectKeyName (tree-sitter.ts): strip ONE leading and ONE trailing quote
|
||||
/// character (`'`, `"`, or backtick).
|
||||
pub fn object_key_name(s: &str) -> String {
|
||||
let mut out = s;
|
||||
if let Some(first) = out.chars().next() {
|
||||
if first == '\'' || first == '"' || first == '`' {
|
||||
out = &out[first.len_utf8()..];
|
||||
}
|
||||
}
|
||||
if let Some(last) = out.chars().last() {
|
||||
if last == '\'' || last == '"' || last == '`' {
|
||||
out = &out[..out.len() - last.len_utf8()];
|
||||
}
|
||||
}
|
||||
out.to_string()
|
||||
}
|
||||
|
||||
/// The `= <first 100 UTF-16 units>[...]` initializer signature used by
|
||||
/// extractVariable (its `.length >= 100` check fires exactly when the slice
|
||||
/// hit the cap).
|
||||
pub fn init_signature(value_text: &str) -> String {
|
||||
let (sliced, _) = slice_utf16(value_text, 100);
|
||||
if utf16_len(&sliced) >= 100 {
|
||||
format!("= {sliced}...")
|
||||
} else {
|
||||
format!("= {sliced}")
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn utf16_cols() {
|
||||
let src = "aé😀b";
|
||||
// 'a'=1, 'é'=1, '😀'=2 utf16 units; bytes: a=1, é=2, 😀=4
|
||||
assert_eq!(utf16_len(src), 5);
|
||||
let starts = line_starts(src);
|
||||
assert_eq!(col16(src, &starts, 0, 1), 1); // after 'a'
|
||||
assert_eq!(col16(src, &starts, 0, 3), 2); // after 'é'
|
||||
assert_eq!(col16(src, &starts, 0, 7), 4); // after '😀'
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn init_sig_short_and_long() {
|
||||
assert_eq!(init_signature("[1, 2]"), "= [1, 2]");
|
||||
let long = "x".repeat(150);
|
||||
let sig = init_signature(&long);
|
||||
assert!(sig.starts_with("= "));
|
||||
assert!(sig.ends_with("..."));
|
||||
assert_eq!(utf16_len(&sig[2..sig.len() - 3]), 100);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn generated_patterns() {
|
||||
assert!(is_generated_file("src/api.generated.ts"));
|
||||
assert!(is_generated_file("vendor/jquery.min.js"));
|
||||
assert!(!is_generated_file("src/app.ts"));
|
||||
}
|
||||
}
|
||||
@@ -14,7 +14,11 @@ Work top to bottom; each step has a section below with the detail.
|
||||
- [x] **R1. Scaffold the napi-rs crate** (`codegraph-kernel`): buffer contract, generic
|
||||
`.scm` emitter, build-pipeline integration, `CODEGRAPH_KERNEL=0` kill switch,
|
||||
wasm fallback, grammar-source-parity CI. (§3) — **done 2026-07-16, see §3a.**
|
||||
- [ ] **R2. Port TypeScript/JavaScript extraction** (tsx/jsx included) as language one. (§4)
|
||||
- [x] **R2. Port TypeScript/JavaScript extraction** (tsx/jsx included) as language one. (§4)
|
||||
— **ported 2026-07-16, see §4a**: full-fidelity Rust walker, byte-parity on this repo
|
||||
(353 files) + excalidraw (643 files) + torture fixtures; extraction 2.6× single-thread.
|
||||
R3's gate (large repo, retrieval invariants, agent A/B, Linux/Windows) still gates
|
||||
default-on.
|
||||
- [ ] **R3. Run TS/JS through the equivalence gate** — graph parity, retrieval
|
||||
invariants, agent A/B, perf + control repo. Ship behind the env flag, then default-on. (§5)
|
||||
- [ ] **R4. Port Java** → re-run the dubbo benchmark → the cbm-parity headline. (§4, §6)
|
||||
@@ -132,6 +136,42 @@ to wasm is the universal fallback. Zero-native-build-on-install stays true.
|
||||
- **Known R2 gate item:** native columns are UTF-8 byte offsets; web-tree-sitter's are
|
||||
UTF-16-derived — column NUMBERS on non-ASCII lines will differ in parity dumps
|
||||
(text, lines, IDs unaffected). Classify or normalize when it shows up.
|
||||
**RESOLVED in R2:** the walker emits UTF-16 columns natively (util::col16), and JS
|
||||
string-slicing semantics (signature truncation at 100/80/120 units) are reproduced in
|
||||
UTF-16 units too — no column/slice diff class exists.
|
||||
|
||||
### 4a. R2 — TS/JS port SHIPPED 2026-07-16 (and a §3 design revision)
|
||||
|
||||
- **The generic `.scm` emitter is superseded.** Real TS/JS parity needs logic queries
|
||||
can't express (extractCall's receiver-qualified callees, store/RTK/component
|
||||
recognition, fn-ref capture+gating, value-ref shadow pruning, docstring wrapper
|
||||
climbs) — so R2 replaced the R1 query emitter with a **bespoke per-language walker**
|
||||
(`codegraph-kernel/src/tsjs/`, ~1,900 lines) that mirrors `TreeSitterExtractor`'s
|
||||
TS/JS paths function-for-function, bug-for-bug. emitter.rs + queries/ are deleted
|
||||
(git has them); expect T1 languages (java/python/go) to be walkers too. The
|
||||
`post(result, source)` TS escape hatch remains available but TS/JS needed none.
|
||||
- **Parity evidence (macOS):** `scripts/kernel-parity.mjs` (multiset diff of
|
||||
canonicalized nodes/edges/refs per file, FULL objects) — this repo 353/353 files,
|
||||
excalidraw 643/643 files (10,650 nodes / 10,726 edges / 68,307 refs), plus
|
||||
checked-in torture fixtures (`__tests__/fixtures/kernel-parity/`) covering
|
||||
components/HOCs/styled, zustand-through-middleware, RTK endpoints+hooks, vuex/pinia,
|
||||
fn-refs (incl `this.x` + shadowing gates), value-refs (incl the shadow prune),
|
||||
decorators, enums, type-alias members + tuple contracts, re-exports, JSX. Kept alive
|
||||
in `npm test` by `__tests__/kernel-tsjs-parity.test.ts` (strict full-object compare).
|
||||
- **One decoder bug found by the strict compare:** decode.ts pre-filled
|
||||
`filePath`/`language` on refs; wasm extractors leave them unset (the store
|
||||
denormalizes via `?? filePath`). Fixed — the seam contract is "exactly what
|
||||
extractFromSource returns", not "what the store makes of it".
|
||||
- **Perf (M3 Pro, excalidraw 643 files / 7MB):** extraction single-thread 487ms kernel
|
||||
vs 1,255ms wasm (**2.6×**, identical outputs). End-to-end `init` on an 11-core host
|
||||
moves only ~3.4s → ~3.2s — parse is a small, already-pool-parallelized slice there;
|
||||
the win concentrates on constrained hardware (2-core CI class) and kernel-scale
|
||||
parse (R6). Headroom if R4's dubbo target needs it: arena interning, memoized
|
||||
UTF-16 line prefixes, and skipping wasm-grammar loads in workers for kernel-routed
|
||||
languages (worker cold-start).
|
||||
- **Not yet done (R3 gate):** large-repo parity (vscode-class), full-repo dump-diff
|
||||
through the DB, retrieval invariants, agent A/B, Linux docker + Windows VM parity
|
||||
runs, control-repo perf. Routing stays opt-in (`CODEGRAPH_KERNEL_LANGS`) until then.
|
||||
|
||||
## 4. Per-language tracker
|
||||
|
||||
@@ -150,7 +190,7 @@ parity before porting the language.
|
||||
|
||||
| Language(s) | Today | Tier | Grammar source | Migration notes / known traps | Status |
|
||||
|---|---|---|---|---|---|
|
||||
| typescript, tsx, javascript, jsx | `languages/typescript.ts`, `javascript.ts` + shared branches | T1 | crates.io | First target. Value-reference edges (#895/#897) and component recognition (#841 forwardRef/memo/styled) must survive — they're extraction-side. Largest test surface; gate is strictest here. | ☐ |
|
||||
| typescript, tsx, javascript, jsx | `languages/typescript.ts`, `javascript.ts` + shared branches | T1 | crates.io | First target. Value-reference edges (#895/#897) and component recognition (#841 forwardRef/memo/styled) must survive — they're extraction-side. Largest test surface; gate is strictest here. **PORTED (§4a) — value-refs, component recognition, fn-refs, stores all byte-parity; awaiting the R3 gate before default-on.** | ◐ |
|
||||
| java | `languages/java.ts` | T1 | crates.io | Second target; unlocks the dubbo-parity claim. Lombok member synthesis (#912) is a NODE synthesizer hook in extraction (`synthesizeMembers`) — port or keep as TS post-pass. | ☐ |
|
||||
| python | `languages/python.ts` | T1 | crates.io | Third. Decorator extraction feeds framework route detection — parity required. | ☐ |
|
||||
| go | `languages/go.ts` | T1 | crates.io | Third (tie). Value-reference edges ship here too (#897). | ☐ |
|
||||
|
||||
@@ -0,0 +1,214 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Kernel↔wasm extraction parity harness (R2/R3 of the kernel migration).
|
||||
*
|
||||
* Runs BOTH extraction paths over the given files/directories and diffs the
|
||||
* per-file ExtractionResults as sets (nodes/edges/refs, canonicalized), so a
|
||||
* behavioral gap in the native kernel shows up as a categorized diff instead
|
||||
* of a graph-dump surprise later. This is the fast inner loop; the §5 gate's
|
||||
* full-repo dump-diff still runs before any default-on.
|
||||
*
|
||||
* Usage:
|
||||
* node scripts/kernel-parity.mjs <file-or-dir>... [--lang typescript,tsx]
|
||||
* [--max-samples N] [--list-files]
|
||||
*
|
||||
* Requires: npm run build (dist/) and a staged kernel (npm run build:kernel).
|
||||
* Exit code: 0 = parity, 1 = diffs found, 2 = setup error.
|
||||
*/
|
||||
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
|
||||
const dist = (p) => path.join(ROOT, 'dist', p);
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
const paths = [];
|
||||
let langFilter = null;
|
||||
let maxSamples = 5;
|
||||
let listFiles = false;
|
||||
for (let i = 0; i < args.length; i++) {
|
||||
if (args[i] === '--lang') langFilter = new Set(args[++i].split(','));
|
||||
else if (args[i] === '--max-samples') maxSamples = Number(args[++i]);
|
||||
else if (args[i] === '--list-files') listFiles = true;
|
||||
else paths.push(args[i]);
|
||||
}
|
||||
if (paths.length === 0) {
|
||||
console.error('usage: kernel-parity.mjs <file-or-dir>... [--lang ts,tsx] [--max-samples N]');
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
const KERNEL_LANGS = new Set(['typescript', 'tsx', 'javascript', 'jsx']);
|
||||
const EXTS = new Map([
|
||||
['.ts', 'typescript'], ['.mts', 'typescript'], ['.cts', 'typescript'],
|
||||
['.tsx', 'tsx'], ['.js', 'javascript'], ['.mjs', 'javascript'],
|
||||
['.cjs', 'javascript'], ['.jsx', 'jsx'],
|
||||
]);
|
||||
|
||||
/** Collect candidate files. */
|
||||
function collect(p, out) {
|
||||
const st = fs.statSync(p);
|
||||
if (st.isDirectory()) {
|
||||
const base = path.basename(p);
|
||||
if (base === 'node_modules' || base === '.git' || base === 'dist' || base === '.codegraph') return;
|
||||
for (const e of fs.readdirSync(p)) collect(path.join(p, e), out);
|
||||
} else if (EXTS.has(path.extname(p))) {
|
||||
const lang = EXTS.get(path.extname(p));
|
||||
if (!langFilter || langFilter.has(lang)) out.push({ file: p, lang });
|
||||
}
|
||||
}
|
||||
|
||||
const files = [];
|
||||
for (const p of paths) collect(path.resolve(p), files);
|
||||
if (files.length === 0) {
|
||||
console.error('no matching files');
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
// --- load the built engine ---------------------------------------------------
|
||||
const { extractFromSource } = await import(dist('extraction/tree-sitter.js'));
|
||||
const { initGrammars, loadGrammarsForLanguages } = await import(dist('extraction/grammars.js'));
|
||||
const kernel = await import(dist('extraction/kernel/index.js'));
|
||||
|
||||
await initGrammars();
|
||||
await loadGrammarsForLanguages([...KERNEL_LANGS]);
|
||||
|
||||
if (!kernel.getKernel()) {
|
||||
console.error('kernel .node not found — run: npm run build:kernel');
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
// --- canonicalization ---------------------------------------------------------
|
||||
/**
|
||||
* Node identity for cross-referencing edges/refs: the node id itself (both
|
||||
* paths compute the same deterministic ids, and id embeds kind+name+line).
|
||||
*/
|
||||
function canonNode(n) {
|
||||
const out = {
|
||||
id: n.id, kind: n.kind, name: n.name, qualifiedName: n.qualifiedName,
|
||||
filePath: n.filePath, language: n.language,
|
||||
startLine: n.startLine, endLine: n.endLine,
|
||||
startColumn: n.startColumn, endColumn: n.endColumn,
|
||||
};
|
||||
for (const k of ['docstring', 'signature', 'visibility', 'isExported', 'isAsync', 'isStatic', 'isAbstract', 'returnType']) {
|
||||
if (n[k] !== undefined) out[k] = n[k];
|
||||
}
|
||||
if (n.decorators !== undefined) out.decorators = n.decorators;
|
||||
if (n.typeParameters !== undefined) out.typeParameters = n.typeParameters;
|
||||
return JSON.stringify(out);
|
||||
}
|
||||
|
||||
function canonEdge(e) {
|
||||
const out = { source: e.source, target: e.target, kind: e.kind };
|
||||
if (e.line !== undefined) out.line = e.line;
|
||||
if (e.column !== undefined) out.column = e.column;
|
||||
if (e.provenance !== undefined) out.provenance = e.provenance;
|
||||
if (e.metadata !== undefined) out.metadata = e.metadata;
|
||||
return JSON.stringify(out);
|
||||
}
|
||||
|
||||
function canonRef(r) {
|
||||
// FULL object — a field only one path sets is a parity bug (the vitest
|
||||
// parity suite caught decode.ts pre-filling filePath/language this way).
|
||||
const out = {
|
||||
from: r.fromNodeId, name: r.referenceName, kind: r.referenceKind,
|
||||
line: r.line, column: r.column,
|
||||
};
|
||||
for (const k of ['filePath', 'language', 'candidates', 'rowId']) {
|
||||
if (r[k] !== undefined) out[k] = r[k];
|
||||
}
|
||||
return JSON.stringify(out);
|
||||
}
|
||||
|
||||
function diffSets(aList, bList) {
|
||||
const a = new Map(); // canon -> count (multiset — duplicates matter)
|
||||
const b = new Map();
|
||||
for (const x of aList) a.set(x, (a.get(x) ?? 0) + 1);
|
||||
for (const x of bList) b.set(x, (b.get(x) ?? 0) + 1);
|
||||
const onlyA = [];
|
||||
const onlyB = [];
|
||||
for (const [k, c] of a) {
|
||||
const d = c - (b.get(k) ?? 0);
|
||||
for (let i = 0; i < d; i++) onlyA.push(k);
|
||||
}
|
||||
for (const [k, c] of b) {
|
||||
const d = c - (a.get(k) ?? 0);
|
||||
for (let i = 0; i < d; i++) onlyB.push(k);
|
||||
}
|
||||
return { onlyA, onlyB };
|
||||
}
|
||||
|
||||
// --- run ----------------------------------------------------------------------
|
||||
const buckets = new Map(); // category -> {count, samples[]}
|
||||
function report(category, sample) {
|
||||
let b = buckets.get(category);
|
||||
if (!b) buckets.set(category, (b = { count: 0, samples: [] }));
|
||||
b.count++;
|
||||
if (b.samples.length < maxSamples) b.samples.push(sample);
|
||||
}
|
||||
|
||||
let filesWithDiffs = 0;
|
||||
let filesOk = 0;
|
||||
let kernelFailed = 0;
|
||||
let totals = { nodes: 0, edges: 0, refs: 0 };
|
||||
|
||||
process.env.CODEGRAPH_KERNEL_LANGS = 'all';
|
||||
|
||||
for (const { file, lang } of files) {
|
||||
const source = fs.readFileSync(file, 'utf8');
|
||||
const rel = path.relative(ROOT, file);
|
||||
|
||||
delete process.env.CODEGRAPH_KERNEL; // kernel path on
|
||||
const kres = kernel.tryKernelExtract(rel, source, lang);
|
||||
if (!kres) {
|
||||
kernelFailed++;
|
||||
report('kernel-extract-failed', rel);
|
||||
continue;
|
||||
}
|
||||
process.env.CODEGRAPH_KERNEL = '0'; // wasm path
|
||||
const wres = extractFromSource(rel, source, lang);
|
||||
delete process.env.CODEGRAPH_KERNEL;
|
||||
|
||||
totals.nodes += wres.nodes.length;
|
||||
totals.edges += wres.edges.length;
|
||||
totals.refs += wres.unresolvedReferences.length;
|
||||
|
||||
let fileHasDiff = false;
|
||||
const tables = [
|
||||
['node', wres.nodes.map(canonNode), kres.nodes.map(canonNode)],
|
||||
['edge', wres.edges.map(canonEdge), kres.edges.map(canonEdge)],
|
||||
['ref', wres.unresolvedReferences.map(canonRef), kres.unresolvedReferences.map(canonRef)],
|
||||
];
|
||||
for (const [table, wasm, kern] of tables) {
|
||||
const { onlyA, onlyB } = diffSets(wasm, kern);
|
||||
for (const x of onlyA) {
|
||||
fileHasDiff = true;
|
||||
const o = JSON.parse(x);
|
||||
report(`${table}:missing-in-kernel:${o.kind ?? ''}`, `${rel}: ${x}`);
|
||||
}
|
||||
for (const x of onlyB) {
|
||||
fileHasDiff = true;
|
||||
const o = JSON.parse(x);
|
||||
report(`${table}:extra-in-kernel:${o.kind ?? ''}`, `${rel}: ${x}`);
|
||||
}
|
||||
}
|
||||
if (fileHasDiff) {
|
||||
filesWithDiffs++;
|
||||
if (listFiles) console.log(`DIFF ${rel}`);
|
||||
} else {
|
||||
filesOk++;
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`\n=== kernel parity: ${filesOk}/${files.length} files byte-parity` +
|
||||
` (${filesWithDiffs} with diffs, ${kernelFailed} kernel-failed)` +
|
||||
` | wasm totals: ${totals.nodes} nodes / ${totals.edges} edges / ${totals.refs} refs ===\n`);
|
||||
|
||||
const sorted = [...buckets.entries()].sort((a, b) => b[1].count - a[1].count);
|
||||
for (const [cat, { count, samples }] of sorted) {
|
||||
console.log(`--- ${cat}: ${count}`);
|
||||
for (const s of samples) console.log(` ${s.length > 400 ? s.slice(0, 400) + '…' : s}`);
|
||||
}
|
||||
|
||||
process.exit(filesWithDiffs > 0 || kernelFailed > 0 ? 1 : 0);
|
||||
@@ -148,6 +148,9 @@ export function decodeExtractBuffers(
|
||||
const row = buffers.refs.subarray(i * REF_ROW_SIZE, (i + 1) * REF_ROW_SIZE);
|
||||
const fromIdx = row.readUInt32LE(REF.fromIdx);
|
||||
const kindByte = row.readUInt8(REF.kind);
|
||||
// No filePath/language here: the wasm extractors emit refs WITHOUT the
|
||||
// denormalized fields (the store fills them via `ref.filePath ?? filePath`),
|
||||
// and the kernel must match the extractFromSource seam exactly.
|
||||
const ref: UnresolvedReference = {
|
||||
fromNodeId: fromIdx === NONE ? str(arena, row, REF.fromIdStr)! : idByRow[fromIdx]!,
|
||||
referenceName: str(arena, row, REF.referenceName)!,
|
||||
@@ -157,8 +160,6 @@ export function decodeExtractBuffers(
|
||||
: (EDGE_KINDS[kindByte] as ReferenceKind),
|
||||
line: row.readUInt32LE(REF.line),
|
||||
column: row.readUInt32LE(REF.column),
|
||||
filePath,
|
||||
language,
|
||||
};
|
||||
const candidates = strList(arena, row, REF.candidates);
|
||||
if (candidates !== undefined) ref.candidates = candidates;
|
||||
|
||||
Reference in New Issue
Block a user