diff --git a/.dockerignore b/.dockerignore index 5e7e5a0..a98649d 100644 --- a/.dockerignore +++ b/.dockerignore @@ -5,3 +5,6 @@ dist .kommandr docs assets +codegraph-kernel/target +codegraph-kernel/prebuilds +release diff --git a/CHANGELOG.md b/CHANGELOG.md index 656fc7a..b447b83 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### New Features +- Indexing TypeScript, TSX, JavaScript, and JSX projects is faster: parsing and symbol extraction now run in a native engine when a prebuilt binary is available for your platform (release bundles include one), producing exactly the same graph — verified byte-for-byte against the previous engine on real repositories, from small libraries up to vscode-scale codebases. The speedup is largest on resource-constrained machines like CI runners. No setup needed: platforms without the native binary, and individual files with syntax errors, automatically use the previous engine, and `CODEGRAPH_KERNEL=0` turns the native path off entirely. - Reference resolution now runs in parallel on large projects. When a project has enough pending references to make it worthwhile (roughly 150k+, typical for big Java/Kotlin/Spring codebases), resolution fans out across worker threads while results are applied in the exact order the single-threaded path would have used — the graph comes out byte-for-byte identical, about twice as fast end-to-end on a 4,000-file Java project in our testing. Small projects keep the single-threaded path automatically (the fan-out costs more than it saves there). Set `CODEGRAPH_NO_PARALLEL_RESOLVE=1` to disable, or `CODEGRAPH_PARALLEL_RESOLVE_MIN=` to tune when it engages. - Indexing large projects got another sizeable speedup — about a quarter less wall-clock on the same 4,000-file Java project, with the graph still byte-for-byte identical. Two changes: the database no longer interleaves expensive checkpoint housekeeping into the middle of resolution on a fresh index (it's folded once at the end instead), and while one batch's results are being written out, the worker threads are already resolving the next batch instead of sitting idle. - The dynamic-dispatch analysis that runs at the end of indexing (callback, event, and framework wiring) now runs its passes in parallel on large projects, cutting that stage roughly in half there — and a pass that crashes now retries safely instead of failing the whole index, which also makes very large codebases that previously died in this stage more likely to index to completion. Graphs remain byte-for-byte identical. diff --git a/__tests__/kernel-scaffold.test.ts b/__tests__/kernel-scaffold.test.ts index 0856012..e055aad 100644 --- a/__tests__/kernel-scaffold.test.ts +++ b/__tests__/kernel-scaffold.test.ts @@ -72,9 +72,16 @@ describe.skipIf(!kernelBuilt)('kernel scaffold', () => { expect(info.languages).toContain('javascript'); }); - it('no language routes to the kernel by default (R1: wasm path unchanged)', () => { + it('TS/JS family routes to the kernel by default (R3 default-on); others stay wasm', () => { + for (const lang of ['typescript', 'tsx', 'javascript', 'jsx'] as const) { + expect(kernelRoutes(lang), lang).toBe(true); + } + expect(kernelRoutes('python')).toBe(false); + expect(tryKernelExtract('src/a.py', 'def f():\n pass\n', 'python')).toBeNull(); + // CODEGRAPH_KERNEL_LANGS REPLACES the default set when present. + process.env.CODEGRAPH_KERNEL_LANGS = 'tsx'; expect(kernelRoutes('typescript')).toBe(false); - expect(tryKernelExtract('src/a.ts', 'function f() {}', 'typescript')).toBeNull(); + expect(kernelRoutes('tsx')).toBe(true); }); describe('with typescript routed (CODEGRAPH_KERNEL_LANGS)', () => { @@ -170,12 +177,14 @@ describe.skipIf(!kernelBuilt)('kernel scaffold', () => { await loadGrammarsForLanguages(['typescript']); }); - it('unrouted language flows through the wasm extractor unchanged', () => { - // `const f = () => 1` yields a function node on the wasm path; the seed - // kernel query deliberately doesn't extract it — so its presence proves - // which path ran. + it('kill switch routes through the wasm extractor unchanged', () => { + process.env.CODEGRAPH_KERNEL = '0'; const result = extractFromSource('src/a.ts', 'export const f = () => 1;\n', 'typescript'); expect(result.nodes.some((n) => n.kind === 'function' && n.name === 'f')).toBe(true); + delete process.env.CODEGRAPH_KERNEL; + // Default-routed path produces the same node (R2 parity). + const viaKernel = extractFromSource('src/a.ts', 'export const f = () => 1;\n', 'typescript'); + expect(viaKernel.nodes.some((n) => n.kind === 'function' && n.name === 'f')).toBe(true); }); it('routed language takes the kernel and falls back per file on kernel absence', () => { diff --git a/__tests__/kernel-tsjs-parity.test.ts b/__tests__/kernel-tsjs-parity.test.ts index f781fdc..5732baa 100644 --- a/__tests__/kernel-tsjs-parity.test.ts +++ b/__tests__/kernel-tsjs-parity.test.ts @@ -110,6 +110,21 @@ describe.skipIf(!kernelBuilt)('kernel TS/JS extraction parity', () => { assertParity(rel, fs.readFileSync(file, 'utf8'), 'typescript'); }); + it('files with parse errors defer to the wasm extractor (recovery is encoding-dependent)', () => { + // tree-sitter error RECOVERY differs between UTF-8 (native) and UTF-16 + // (web-tree-sitter) parsing — same grammar, same core version — so the + // kernel defers any erroring file to keep routing graph-neutral. + const broken = 'export function f( {\n return }} 12 (\n'; + process.env.CODEGRAPH_KERNEL_LANGS = 'all'; + delete process.env.CODEGRAPH_KERNEL; + expect(tryKernelExtract('src/broken.ts', broken, 'typescript')).toBeNull(); + // The seam still serves the file — through the wasm path. + process.env.CODEGRAPH_KERNEL = '0'; + const viaWasm = extractFromSource('src/broken.ts', broken, 'typescript'); + delete process.env.CODEGRAPH_KERNEL; + expect(viaWasm.nodes.some((n) => n.kind === 'file')).toBe(true); + }); + it('typescript fixture parsed as plain typescript variant', () => { // Same content through the non-tsx grammar exercises the typescript // (vs tsx) LangSpec pairing. diff --git a/codegraph-kernel/src/tsjs/mod.rs b/codegraph-kernel/src/tsjs/mod.rs index c036e20..a499e9e 100644 --- a/codegraph-kernel/src/tsjs/mod.rs +++ b/codegraph-kernel/src/tsjs/mod.rs @@ -177,6 +177,19 @@ pub fn extract(file_path: &str, source: &str, language: &str) -> Result > a.dump + * node scripts/dump-graph.mjs > b.dump + * diff a.dump b.dump + * + * Volatile fields excluded: nodes.updated_at, files.modified_at/indexed_at/ + * content_hash+size (environment-dependent), edges.id / unresolved_refs.id + * (insertion rowids), and unresolved_refs.status (resolution bookkeeping — + * kept, actually: status is deterministic given the same input; excluded only + * if it proves flaky. We keep status.) + */ + +import { DatabaseSync } from 'node:sqlite'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; + +const arg = process.argv[2]; +if (!arg) { + console.error('usage: dump-graph.mjs '); + process.exit(2); +} +let dbPath = arg; +if (fs.statSync(arg).isDirectory()) { + dbPath = path.join(arg, '.codegraph', 'codegraph.db'); +} +const db = new DatabaseSync(dbPath, { readOnly: true }); + +function dump(title, sql) { + const rows = db.prepare(sql).all(); + const lines = rows.map((r) => JSON.stringify(r)).sort(); + process.stdout.write(`== ${title} (${lines.length})\n`); + for (const l of lines) process.stdout.write(l + '\n'); +} + +dump( + 'nodes', + `SELECT id, kind, name, qualified_name, file_path, language, start_line, end_line, + start_column, end_column, docstring, signature, visibility, is_exported, + is_async, is_static, is_abstract, decorators, type_parameters, return_type + FROM nodes` +); +dump( + 'edges', + `SELECT source, target, kind, metadata, line, col, provenance FROM edges` +); +dump( + 'refs', + `SELECT from_node_id, reference_name, reference_kind, line, col, candidates, + file_path, language, status, name_tail + FROM unresolved_refs` +); +dump('files', `SELECT path, language, node_count FROM files`); diff --git a/scripts/kernel-parity.mjs b/scripts/kernel-parity.mjs index 6550334..767178c 100644 --- a/scripts/kernel-parity.mjs +++ b/scripts/kernel-parity.mjs @@ -150,7 +150,7 @@ function report(category, sample) { let filesWithDiffs = 0; let filesOk = 0; -let kernelFailed = 0; +let deferred = 0; let totals = { nodes: 0, edges: 0, refs: 0 }; process.env.CODEGRAPH_KERNEL_LANGS = 'all'; @@ -162,8 +162,11 @@ for (const { file, lang } of files) { delete process.env.CODEGRAPH_KERNEL; // kernel path on const kres = kernel.tryKernelExtract(rel, source, lang); if (!kres) { - kernelFailed++; - report('kernel-extract-failed', rel); + // Expected: files with parse errors defer to wasm (parity by + // construction — both arms run the same extractor). Counted, and + // guarded below so a broken kernel can't silently defer everything. + deferred++; + report('kernel-deferred', rel); continue; } process.env.CODEGRAPH_KERNEL = '0'; // wasm path @@ -192,6 +195,19 @@ for (const { file, lang } of files) { const o = JSON.parse(x); report(`${table}:extra-in-kernel:${o.kind ?? ''}`, `${rel}: ${x}`); } + // ORDER matters too: identical multisets in a different emission order + // change DB rowids, and resolution iterates refs in rowid order — the + // full-index dump-diff would surface it as a downstream mystery. Catch it + // here instead. + if (onlyA.length === 0 && onlyB.length === 0) { + for (let i = 0; i < wasm.length; i++) { + if (wasm[i] !== kern[i]) { + fileHasDiff = true; + report(`${table}:order-mismatch`, `${rel}: index ${i}: wasm=${wasm[i]} kernel=${kern[i]}`); + break; + } + } + } } if (fileHasDiff) { filesWithDiffs++; @@ -202,7 +218,7 @@ for (const { file, lang } of files) { } console.log(`\n=== kernel parity: ${filesOk}/${files.length} files byte-parity` + - ` (${filesWithDiffs} with diffs, ${kernelFailed} kernel-failed)` + + ` (${filesWithDiffs} with diffs, ${deferred} deferred-to-wasm)` + ` | wasm totals: ${totals.nodes} nodes / ${totals.edges} edges / ${totals.refs} refs ===\n`); const sorted = [...buckets.entries()].sort((a, b) => b[1].count - a[1].count); @@ -211,4 +227,11 @@ for (const [cat, { count, samples }] of sorted) { for (const s of samples) console.log(` ${s.length > 400 ? s.slice(0, 400) + '…' : s}`); } -process.exit(filesWithDiffs > 0 || kernelFailed > 0 ? 1 : 0); +// Deferrals are per-file parse-error routing (expected, rare). A high rate +// means the kernel is broken and hiding behind the fallback — fail loudly. +const deferralRate = deferred / files.length; +if (deferralRate > 0.1) { + console.error(`deferral rate ${(deferralRate * 100).toFixed(1)}% exceeds 10% — kernel likely broken`); + process.exit(1); +} +process.exit(filesWithDiffs > 0 ? 1 : 0); diff --git a/src/extraction/kernel/index.ts b/src/extraction/kernel/index.ts index 3c27bd4..1cd235d 100644 --- a/src/extraction/kernel/index.ts +++ b/src/extraction/kernel/index.ts @@ -7,9 +7,11 @@ * everything else stays on the wasm path forever if need be. Rollback per * language = removing it from DEFAULT_ROUTED (or CODEGRAPH_KERNEL=0 for all). * - * R1 status: NO language is default-routed yet. Development/testing opt-in: - * CODEGRAPH_KERNEL_LANGS=typescript,tsx (or "all" for every kernel-capable - * language). R3 flips TS/JS into DEFAULT_ROUTED once the gate passes. + * Routing status: TypeScript/TSX/JavaScript/JSX are default-routed (R3 gate + * passed 2026-07-16 — full-index dumps byte-identical on express/excalidraw/ + * vscode, control repo unchanged; see the migration plan §4a). Override with + * CODEGRAPH_KERNEL_LANGS= (replaces the default set), or + * CODEGRAPH_KERNEL=0 (kill switch, everything → wasm). */ import type { ExtractionResult, Language } from '../../types'; @@ -22,8 +24,16 @@ export { decodeExtractBuffers } from './decode'; /** * Languages routed to the kernel by default (gate-passed only — see the * per-language tracker in docs/design/rust-kernel-migration-plan.md §4). + * Per-file safety valve regardless of routing: a file whose parse tree + * contains ERRORS defers to the wasm extractor (error recovery differs + * between UTF-8 and UTF-16 parsing — wasm's recovery is canonical). */ -const DEFAULT_ROUTED: ReadonlySet = new Set([]); +const DEFAULT_ROUTED: ReadonlySet = new Set([ + 'typescript', + 'tsx', + 'javascript', + 'jsx', +]); /** * Per-language TS post-pass over the decoded result — the escape hatch for @@ -82,12 +92,15 @@ export function tryKernelExtract( result.durationMs = Date.now() - t0; return result; } catch (err) { + const message = err instanceof Error ? err.message : String(err); + // `defer:` is the kernel's expected-routing signal (files with parse + // errors take the wasm path — its error RECOVERY is the canonical one; + // recovery differs between UTF-8 and UTF-16 parsing). Silent by design. + if (message.includes('defer:')) return null; if (!warned.has(language)) { warned.add(language); process.stderr.write( - `[codegraph-kernel] ${language} extraction failed (${ - err instanceof Error ? err.message : String(err) - }) — falling back to the wasm path\n` + `[codegraph-kernel] ${language} extraction failed (${message}) — falling back to the wasm path\n` ); } return null;