feat(kernel): R7b Lua+Luau walker — one lua module, vendored-grammar-C lua v0.4.1, tree-sitter-luau 1.2.0 pin, both default-routed (#1384)
R7b batch 4 #2 (docs/design/lua-luau-kernel-port-checklist.md is the authoritative quirk list). ONE walker for both dialects (ccpp precedent) — the differences are exactly four: luau's type_definition aliases, the `export `-slice isExported hook, the return-type signature suffix, and the grammar handle. Grammar prep is kernel-side only, no wasm change: lua is the SECOND vendored-grammar-C language (the vendored wasm is the v0.4.1 tag, a revision not on crates.io — tag artifacts compiled via build.rs, shas pinned); luau is a plain crate pin =1.2.0 whose tarball is sha-identical to the tag (the swift tag≠crate divergence does not recur). Grammar-parity rows replace the bump gate entirely. Preserved bug-for-bug (all probe-pinned): the require/visitNode-hook ASYMMETRIES (top-level requires — including inside top-level if/for/while — mint import nodes while the identical body-level statement emits `calls "require"`; top-level `local x = foo()` initializers are invisible while global `x = foo()` calls emit), the BFS string-win inside require args (`require(script:WaitForChild("Kid"))` → import Kid) and Roblox instance paths, receiver-QN methods (`M.sub.deep::chained`, `_G::installed`, stack-QN nested globals like `render::leakedGlobal`), the raw-text callee world (colon forms with `self` never stripped, bracket callees, newline-glued chains byte-verbatim, the `(handler)` paren-conversion), LUA_SPEC function-as-value capture with the `M.cb = cb` param-storage skip and first-occurrence dedupe, LuaDoc `---` keeping a leading `- ` plus `--!strict` joining docstring chains (block-comment docstrings keep interior CRLF bytes), variable nodes at the IDENTIFIER with positional value pairing, duplicate same-(kind,name,line) ids, and the lua↔luau isExported wire divergence (lua functions: flag absent; luau functions: present-false; methods: absent in both; variables: present-false in both; `export type`: true). Gates: parity sweeps first-run 0-diff on kong/lazy.nvim/lua-resty-core (lua) + lune/Fusion (luau) — 1,734 clean files byte-parity, deferrals 1/0/0/3/8 matching the survey's both-arm predictions exactly (kong's 1 = a deliberately invalid fixture; luau's = grammar-inherent generic type packs and default type params); full-init dumps byte-identical kernel-vs-wasm ×4 (kong 157,650 dump lines); kernel-lua-parity suite (both torture fixtures + in-memory CRLF variants + glue-chain, duplicate-id, and cross-dialect defer pins + kernel-arm wire-flag pins); full suite 2,647 green ×2 with CODEGRAPH_KERNEL_EXPECT=1. DEFAULT_ROUTED += lua, luau (18 langs). Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Colby Mchenry
co-authored by
Claude Fable 5
Claude Fable 5
parent
b2f9ab1800
commit
e32135171e
@@ -0,0 +1,99 @@
|
||||
#!/usr/bin/env lua
|
||||
-- File-header comment run line 1
|
||||
-- line 2 of the run
|
||||
local core = require("app.core")
|
||||
local Signal = require(script.Parent.Signal)
|
||||
local quoted = require 'app.quoted'
|
||||
local bracketed = require [[app.bracketed]]
|
||||
local viaChild = require(script:WaitForChild("Child"))
|
||||
local accessed = require("app.acc").field
|
||||
local dyn = require(dynName)
|
||||
local twoA, twoB = require("two.a"), require("two.b")
|
||||
local okG, guarded = pcall(require, "app.guarded")
|
||||
|
||||
--- LuaDoc summary for topFn.
|
||||
-- @param a number
|
||||
-- @return number
|
||||
function topFn(a)
|
||||
return a + 1
|
||||
end
|
||||
|
||||
--[[ Block comment doc
|
||||
spanning two lines ]]
|
||||
local function localFn(...)
|
||||
return select("#", ...)
|
||||
end
|
||||
|
||||
-- doc for anonAssigned (variable, initializer invisible)
|
||||
local anonAssigned = function(v)
|
||||
return hidden(v)
|
||||
end
|
||||
|
||||
local M = {}
|
||||
local UPPER_CONST = 42
|
||||
|
||||
function M.create(n)
|
||||
local inst = { n = n }
|
||||
return setmetatable(inst, { __index = M })
|
||||
end
|
||||
|
||||
function M:render(opts)
|
||||
local lazy = require("app.lazy")
|
||||
self.count = (self.count or 0) + 1
|
||||
self:helperMethod(opts)
|
||||
self.field.deep(1)
|
||||
M.create(2)
|
||||
M.registry[key](3)
|
||||
core.util.log("msg")
|
||||
core.run(M.create)
|
||||
local function inner()
|
||||
return topFn(9)
|
||||
end
|
||||
function M.attached(q)
|
||||
return q
|
||||
end
|
||||
function leakedGlobal()
|
||||
end
|
||||
return inner()
|
||||
end
|
||||
|
||||
function M.sub.deep:chained(x)
|
||||
return x
|
||||
end
|
||||
|
||||
function _G.installed()
|
||||
end
|
||||
|
||||
M.assigned = function(z)
|
||||
return topFn(z)
|
||||
end
|
||||
|
||||
M.handlers = { on_start = topFn, on_stop = localFn, skipped = missing }
|
||||
local tbl = { cb = topFn, [1] = localFn, nested = { deep_cb = topFn } }
|
||||
|
||||
globalAssign = topFn(10)
|
||||
M.cb = cb
|
||||
|
||||
topFn(11)
|
||||
M.create(12)
|
||||
M:render({})
|
||||
M.sub.deep:chained(13)
|
||||
core.util.log("direct")
|
||||
t2[k2](14)
|
||||
f2()(15)
|
||||
|
||||
function paren_conv()
|
||||
(handler)(16)
|
||||
end
|
||||
|
||||
local one = 1 local two = 2 print(one, two)
|
||||
|
||||
local s1 = [[long
|
||||
string]]
|
||||
local s2 = [==[nested ]] ok]==]
|
||||
local x <const> = 99
|
||||
local préfixe = "café"
|
||||
function after_unicode() end
|
||||
do goto done end
|
||||
::done::
|
||||
return M
|
||||
@@ -0,0 +1,61 @@
|
||||
--!strict
|
||||
-- header comment for torture.luau
|
||||
local core = require("app.core")
|
||||
local Signal = require(script.Parent.Signal)
|
||||
|
||||
-- doc for Point
|
||||
type Point = { x: number, y: number }
|
||||
--- doc for exported Handler
|
||||
export type Handler = (msg: string) -> ()
|
||||
export type Generic<T> = { value: T }
|
||||
type FromTypeof = typeof(require(script.Parent.Config))
|
||||
|
||||
--- doc for typedTop
|
||||
function typedTop(a: number, b: string?): boolean
|
||||
return a > 0 and b ~= nil
|
||||
end
|
||||
|
||||
function genericFn<T>(v: T): T
|
||||
return v
|
||||
end
|
||||
|
||||
function multiRet(): (number, string)
|
||||
return 1, "x"
|
||||
end
|
||||
|
||||
local function emptyOneLiner() end
|
||||
|
||||
local M = {}
|
||||
local CONST_N = 7
|
||||
|
||||
function M.make(n: number): Point
|
||||
return { x = n, y = n }
|
||||
end
|
||||
|
||||
function M:update(p: Point, cb: Handler)
|
||||
type BodyLocal = { z: number }
|
||||
local Config = require(script.Parent.Config)
|
||||
local msg = `point {p.x} of {Config.total()}`
|
||||
cb(msg)
|
||||
self:redraw()
|
||||
self.canvas.flush()
|
||||
M.make(3)
|
||||
local v = if p.x > 0 then bump() else drop()
|
||||
v += grow()
|
||||
local narrowed = p :: any
|
||||
for i = 1, 10 do
|
||||
if i % 2 == 0 then
|
||||
continue
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
M.registry = { on_make = M.make, plain = typedTop }
|
||||
|
||||
typedTop(1, "s")
|
||||
M.make(2)
|
||||
M:update({ x = 1, y = 2 }, print)
|
||||
|
||||
local préfixe = "café"
|
||||
function after_unicode() end
|
||||
return M
|
||||
@@ -36,7 +36,7 @@ const kernelBuilt = fs.existsSync(KERNEL_PATH);
|
||||
|
||||
// Every kernel-capable language. `jsx` shares the javascript grammar on BOTH
|
||||
// paths (langs.rs mirrors WASM_GRAMMAR_FILES), so the distinct grammars are:
|
||||
const GRAMMAR_LANGUAGES: Language[] = ['typescript', 'tsx', 'javascript', 'java', 'python', 'go', 'c', 'cpp', 'rust', 'csharp', 'ruby', 'php', 'swift', 'kotlin', 'r'];
|
||||
const GRAMMAR_LANGUAGES: Language[] = ['typescript', 'tsx', 'javascript', 'java', 'python', 'go', 'c', 'cpp', 'rust', 'csharp', 'ruby', 'php', 'swift', 'kotlin', 'r', 'lua', 'luau'];
|
||||
|
||||
describe.skipIf(!kernelBuilt)('kernel↔wasm grammar parity', () => {
|
||||
beforeAll(async () => {
|
||||
|
||||
@@ -0,0 +1,203 @@
|
||||
/**
|
||||
* Kernel↔wasm Lua + Luau extraction parity (R7b batch 4 of the kernel
|
||||
* migration).
|
||||
*
|
||||
* Asserts the native walker (codegraph-kernel/src/lua.rs — one module, two
|
||||
* dialects) produces the SAME ExtractionResult as the wasm
|
||||
* TreeSitterExtractor — nodes, edges, and unresolved refs compared as
|
||||
* canonicalized multisets — over the checked-in torture fixtures
|
||||
* (torture.lua: the require quintet incl. Roblox instance paths, the
|
||||
* BFS-string-win and .field/dynamic silences, receiver-QN methods
|
||||
* `M.sub.deep::chained` and `_G::installed`, the top-level
|
||||
* local-vs-global initializer-visibility inversion, body-level
|
||||
* `calls "require"`, table fn-ref registries with dedupe and the
|
||||
* `M.cb = cb` param-storage skip, the raw-text callee zoo with colon/
|
||||
* bracket/call-result callees and the `(handler)` conversion, LuaDoc
|
||||
* `- `-keeping docstrings, `<const>` attributes, one-line duplicate-id
|
||||
* declarations; torture.luau: `--!strict` docstring joining, `export type`
|
||||
* isExported, verbatim `Generic<T>` alias names, the typeof(require(...))
|
||||
* alias+import pair, typed signatures with return suffixes, interpolation/
|
||||
* if-expression/compound-assign call shapes) and their CRLF variants
|
||||
* (derived in-memory — #1329, pinning the block-comment `\r\n` docstring
|
||||
* byte), plus glue-chain and defer pins.
|
||||
*
|
||||
* The full-repo sweeps live in scripts/kernel-parity.mjs (kong/lazy.nvim/
|
||||
* lua-resty-core + lune/Fusion for the §5 gate); this suite keeps the
|
||||
* invariant alive in `npm test`. Skips when no kernel binary is staged;
|
||||
* CODEGRAPH_KERNEL_EXPECT=1 turns that into a failure.
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeAll, beforeEach, afterEach } from 'vitest';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import { extractFromSource } from '../src/extraction';
|
||||
import { initGrammars, loadGrammarsForLanguages } from '../src/extraction/grammars';
|
||||
import { tryKernelExtract, resetKernelForTests } from '../src/extraction/kernel';
|
||||
import type { ExtractionResult, Language } from '../src/types';
|
||||
|
||||
const KERNEL_PATH = path.join(
|
||||
__dirname,
|
||||
'..',
|
||||
'codegraph-kernel',
|
||||
'prebuilds',
|
||||
`${process.platform}-${process.arch}`,
|
||||
'codegraph-kernel.node'
|
||||
);
|
||||
const kernelBuilt = fs.existsSync(KERNEL_PATH);
|
||||
|
||||
const FIXTURE_DIR = path.join(__dirname, 'fixtures', 'kernel-parity');
|
||||
|
||||
function canon(result: ExtractionResult): { nodes: string[]; edges: string[]; refs: string[] } {
|
||||
return {
|
||||
nodes: result.nodes
|
||||
.map(({ updatedAt: _u, ...n }) => JSON.stringify(n, Object.keys(n).sort()))
|
||||
.sort(),
|
||||
edges: result.edges.map((e) => JSON.stringify(e, Object.keys(e).sort())).sort(),
|
||||
refs: result.unresolvedReferences
|
||||
.map((r) => JSON.stringify(r, Object.keys(r).sort()))
|
||||
.sort(),
|
||||
};
|
||||
}
|
||||
|
||||
const ENV_KEYS = ['CODEGRAPH_KERNEL', 'CODEGRAPH_KERNEL_LANGS'] as const;
|
||||
let savedEnv: Record<string, string | undefined>;
|
||||
|
||||
describe.skipIf(!kernelBuilt)('kernel Lua/Luau extraction parity', () => {
|
||||
beforeAll(async () => {
|
||||
await initGrammars();
|
||||
await loadGrammarsForLanguages(['lua', 'luau']);
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
savedEnv = Object.fromEntries(ENV_KEYS.map((k) => [k, process.env[k]]));
|
||||
resetKernelForTests();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
for (const k of ENV_KEYS) {
|
||||
if (savedEnv[k] === undefined) delete process.env[k];
|
||||
else process.env[k] = savedEnv[k];
|
||||
}
|
||||
resetKernelForTests();
|
||||
});
|
||||
|
||||
function assertParity(
|
||||
filePath: string,
|
||||
source: string,
|
||||
lang: Language,
|
||||
minNodes = 3
|
||||
): ExtractionResult {
|
||||
process.env.CODEGRAPH_KERNEL_LANGS = 'all';
|
||||
delete process.env.CODEGRAPH_KERNEL;
|
||||
const viaKernel = tryKernelExtract(filePath, source, lang);
|
||||
expect(viaKernel, `kernel extraction failed for ${filePath}`).not.toBeNull();
|
||||
|
||||
process.env.CODEGRAPH_KERNEL = '0';
|
||||
const viaWasm = extractFromSource(filePath, source, lang);
|
||||
delete process.env.CODEGRAPH_KERNEL;
|
||||
|
||||
const k = canon(viaKernel!);
|
||||
const w = canon(viaWasm);
|
||||
expect(k.nodes, `${filePath}: nodes`).toEqual(w.nodes);
|
||||
expect(k.edges, `${filePath}: edges`).toEqual(w.edges);
|
||||
expect(k.refs, `${filePath}: refs`).toEqual(w.refs);
|
||||
expect(viaWasm.nodes.length).toBeGreaterThanOrEqual(minNodes);
|
||||
return viaKernel!;
|
||||
}
|
||||
|
||||
it('torture.lua: requires, receiver QNs, visibility inversion, callee zoo, fn-refs', () => {
|
||||
const file = path.join(FIXTURE_DIR, 'torture.lua');
|
||||
const result = assertParity('fixtures/torture.lua', fs.readFileSync(file, 'utf8'), 'lua', 20);
|
||||
|
||||
// Kernel-arm pins so both arms drifting together can't silently lose the
|
||||
// dialect-defining quirks (checklist §Require hook / §extractCall):
|
||||
const refs = result.unresolvedReferences;
|
||||
// Top-level requires became imports; the body-level require is a CALL.
|
||||
expect(result.nodes.some((n) => n.kind === 'import' && n.name === 'app.core')).toBe(true);
|
||||
expect(refs.some((r) => r.referenceKind === 'calls' && r.referenceName === 'require')).toBe(
|
||||
true
|
||||
);
|
||||
// Roblox instance path → trailing segment; string-win beats the path.
|
||||
expect(result.nodes.some((n) => n.kind === 'import' && n.name === 'Signal')).toBe(true);
|
||||
expect(result.nodes.some((n) => n.kind === 'import' && n.name === 'Child')).toBe(true);
|
||||
// Colon callees keep the colon, self never stripped.
|
||||
expect(refs.some((r) => r.referenceName === 'self:helperMethod')).toBe(true);
|
||||
// Receiver-qualified method QNs are verbatim dotted receivers.
|
||||
expect(result.nodes.some((n) => n.kind === 'method' && n.qualifiedName === 'M.sub.deep::chained')).toBe(true);
|
||||
// lua functions carry NO isExported (undefined — not false).
|
||||
const fn = result.nodes.find((n) => n.kind === 'function' && n.name === 'topFn');
|
||||
expect(fn?.isExported).toBeUndefined();
|
||||
// variables DO carry isExported === false.
|
||||
const v = result.nodes.find((n) => n.kind === 'variable' && n.name === 'core');
|
||||
expect(v?.isExported).toBe(false);
|
||||
});
|
||||
|
||||
it('torture.lua CRLF parity (block-comment docstrings keep interior \\r\\n)', () => {
|
||||
const file = path.join(FIXTURE_DIR, 'torture.lua');
|
||||
const crlf = fs.readFileSync(file, 'utf8').replace(/(?<!\r)\n/g, '\r\n');
|
||||
assertParity('fixtures/torture.lua (crlf)', crlf, 'lua', 20);
|
||||
});
|
||||
|
||||
it('torture.luau: type aliases, export flags, typed signatures, typeof-require', () => {
|
||||
const file = path.join(FIXTURE_DIR, 'torture.luau');
|
||||
const result = assertParity(
|
||||
'fixtures/torture.luau',
|
||||
fs.readFileSync(file, 'utf8'),
|
||||
'luau',
|
||||
10
|
||||
);
|
||||
|
||||
// luau functions carry isExported === false (present bit); methods stay
|
||||
// undefined — the one lua↔luau node-payload flag divergence.
|
||||
const fn = result.nodes.find((n) => n.kind === 'function' && n.isExported === false);
|
||||
expect(fn).toBeTruthy();
|
||||
const method = result.nodes.find((n) => n.kind === 'method');
|
||||
if (method) expect(method.isExported).toBeUndefined();
|
||||
// export type → isExported true.
|
||||
expect(result.nodes.some((n) => n.kind === 'type_alias' && n.isExported === true)).toBe(true);
|
||||
});
|
||||
|
||||
it('torture.luau CRLF parity', () => {
|
||||
const file = path.join(FIXTURE_DIR, 'torture.luau');
|
||||
const crlf = fs.readFileSync(file, 'utf8').replace(/(?<!\r)\n/g, '\r\n');
|
||||
assertParity('fixtures/torture.luau (crlf)', crlf, 'luau', 10);
|
||||
});
|
||||
|
||||
it('newline-glue chains emit byte-verbatim multi-link refs', () => {
|
||||
// Lua's statement ambiguity: a call statement followed by a line starting
|
||||
// `(` parses as ONE glued chain — the middle links' "callees" are whole
|
||||
// inner function_call texts, embedded newline/tab included.
|
||||
const glued = 'local helper = require("app.helper")\nfunction M:go(obj)\n\tobj:foo():bar()\n\t(helper)(4)\nend\n';
|
||||
const result = assertParity('fixtures/glue.lua', glued, 'lua', 3);
|
||||
const names = result.unresolvedReferences
|
||||
.filter((r) => r.referenceKind === 'calls')
|
||||
.map((r) => r.referenceName);
|
||||
expect(names).toContain('obj:foo():bar()\n\t(helper)');
|
||||
expect(names).toContain('obj:foo():bar()');
|
||||
expect(names).toContain('obj:foo():bar');
|
||||
expect(names).toContain('obj:foo');
|
||||
});
|
||||
|
||||
it('one-line duplicate declarations emit duplicate-id rows verbatim', () => {
|
||||
const src = 'local x = 1; local x = 2\n';
|
||||
const result = assertParity('fixtures/dup.lua', src, 'lua', 2);
|
||||
const xs = result.nodes.filter((n) => n.kind === 'variable' && n.name === 'x');
|
||||
expect(xs).toHaveLength(2);
|
||||
expect(xs[0]!.id).toBe(xs[1]!.id);
|
||||
});
|
||||
|
||||
it('cross-dialect syntax defers to the wasm extractor', () => {
|
||||
// Luau syntax in a .lua file and a luau default type parameter both
|
||||
// ERROR (grammar-inherent, both-arm) — the kernel defers per-file.
|
||||
process.env.CODEGRAPH_KERNEL_LANGS = 'all';
|
||||
delete process.env.CODEGRAPH_KERNEL;
|
||||
expect(tryKernelExtract('src/compound.lua', 'x += 1\n', 'lua')).toBeNull();
|
||||
expect(
|
||||
tryKernelExtract('src/defaultparam.luau', 'type S<T = U> = {}\n', 'luau')
|
||||
).toBeNull();
|
||||
process.env.CODEGRAPH_KERNEL = '0';
|
||||
const viaWasm = extractFromSource('src/compound.lua', 'x += 1\n', 'lua');
|
||||
delete process.env.CODEGRAPH_KERNEL;
|
||||
expect(viaWasm.nodes.some((n) => n.kind === 'file')).toBe(true);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user