feat(kernel): R7b R walker — rlang module, tree-sitter-r 1.2.0 crate pin, r default-routed (#1383)
R7b batch 4 #1 (docs/design/r-kernel-port-checklist.md is the authoritative quirk list; survey + probe record therein). The lightest-shared-surface, heaviest-hook port: languages/r.ts works entirely through the visitNode hook (every type list empty except callTypes:['call']), so the walker is a file node + a faithful hook transcription + the generic extractCall + pre-order recursion — four shared machineries (value-refs, static-member reads, type annotations, fn-ref capture) are dead by language gates and stay dead. Grammar prep is the first true no-op of the arc: the crates.io tree-sitter-r 1.2.0 tarball ships parser.c AND scanner.c sha-identical to the r-lib v1.2.0 tag the vendored wasm was built from — crate pin only, no wasm change, no bump gate; kernel-grammar-parity gains the r row (ABI 14, same-revision). Preserved bug-for-bug (all probe-pinned): calls "return" on every return(x) (named node in v1.2.0), the import quintet's silent dynamic-arg consumption vs class/generic fall-through asymmetry, library(help = pkg) importing the named arg, class-idiom variable suppression by callee name, chained/right- assign/precedence-ghost gaps, env$fn body-leak-to-file, raw-text callees verbatim (pkg::fn, obj$meth, "strfn" quotes kept, (handler) conversion), duplicate same-(kind,name,line) ids, roxygen dropped entirely, UTF-16 columns/slices. Gates: parity sweeps first-run 0-diff on AnomalyDetection/dplyr/ggplot2/ shiny (838 files; deferrals exactly 0/0/0/1 — the 1 is the moustache- template pseudo-R file, both-arm) — kernel-parity.mjs gained lowercased- extension matching so .R files sweep (matches detectLanguage routing); full-init dumps byte-identical kernel-vs-wasm on dplyr/ggplot2/shiny; kernel-r-parity suite (torture fixture + in-memory CRLF + BOM variants + defer pin + kernel-arm quirk pins); full suite 2,638 green ×2 with CODEGRAPH_KERNEL_EXPECT=1. DEFAULT_ROUTED += r (16 langs). Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Colby Mchenry
co-authored by
Claude Fable 5
Claude Fable 5
parent
45a53eb5b5
commit
b2f9ab1800
@@ -0,0 +1,204 @@
|
||||
# Kernel↔wasm R parity torture fixture — every hook branch, extractCall shape,
|
||||
# and known-gap behavior from docs/design/r-kernel-port-checklist.md. Parses
|
||||
# CLEAN (no ERROR/MISSING) — deferral shapes live in the test file, and the
|
||||
# CRLF variant is derived in-memory by kernel-r-parity.test.ts.
|
||||
|
||||
#' Roxygen title for top_fn (dropped — R nodes never carry docstrings)
|
||||
#' @param a first
|
||||
top_fn <- function(a, b = 2, ...) {
|
||||
a + b
|
||||
}
|
||||
|
||||
# plain comment run above eq_fn (also dropped)
|
||||
# second line
|
||||
eq_fn = function(x) x * 2
|
||||
|
||||
lam <- \(x) x + 1
|
||||
|
||||
gfun <<- function() 0
|
||||
|
||||
(function(x) x * 3) -> trpl
|
||||
|
||||
function(y) y - 1 -> ghost
|
||||
|
||||
MAX_RETRIES <- 3L
|
||||
A.CONST = 2.5
|
||||
lower_var <- "hello"
|
||||
dotted.var <- 1
|
||||
x2 <<- 4
|
||||
9 -> right_var
|
||||
10 ->> RIGHT.CONST
|
||||
chain_a <- chain_b <- 5
|
||||
|
||||
nester <- function(x) {
|
||||
inner <- function(y) {
|
||||
innermost <- function(z) z + 1
|
||||
innermost(y)
|
||||
}
|
||||
CAPS_LOCAL <- 99
|
||||
z <- inner(x)
|
||||
log_it(z)
|
||||
z
|
||||
}
|
||||
|
||||
if (TRUE) f_in_if <- function() 1
|
||||
{
|
||||
hidden_var <- 42
|
||||
braced_fn <- function() 2
|
||||
}
|
||||
|
||||
top_call(nested_call(1))
|
||||
x %>% p_one() %>% p_two()
|
||||
z |> p_three()
|
||||
res <- data %>% p_four()
|
||||
|
||||
# --- imports -----------------------------------------------------------------
|
||||
library(dplyr)
|
||||
require(stats)
|
||||
requireNamespace("jsonlite")
|
||||
loadNamespace("tools")
|
||||
source("helpers.R")
|
||||
source(file.path("R", "dyn.R"))
|
||||
library()
|
||||
suppressPackageStartupMessages(library(quietpkg))
|
||||
base::library(magrittr)
|
||||
library(help = docpkg)
|
||||
requireNamespace(quietly = TRUE, package = "namedpkg")
|
||||
library("")
|
||||
pkg::fn_q(1)
|
||||
pkg:::fn_h(2)
|
||||
use_it <- function() {
|
||||
library(inside_fn)
|
||||
fn_q(3)
|
||||
}
|
||||
|
||||
# --- classes -----------------------------------------------------------------
|
||||
setClass("Patient", representation(name = "character"), contains = "Person")
|
||||
|
||||
setGeneric("describe", function(obj) standardGeneric("describe"))
|
||||
|
||||
setMethod("describe", "Patient", function(obj) {
|
||||
fmt(obj)
|
||||
})
|
||||
|
||||
Account <- setRefClass("Account",
|
||||
fields = list(balance = "numeric"),
|
||||
contains = "BaseAccount",
|
||||
methods = list(
|
||||
deposit = function(x) {
|
||||
balance <<- balance + x
|
||||
audit(x)
|
||||
},
|
||||
withdraw = function(x) balance <<- balance - x
|
||||
))
|
||||
|
||||
Stack <- R6Class("Stack",
|
||||
inherit = AbstractCollection,
|
||||
public = list(
|
||||
items = NULL,
|
||||
push = function(x) {
|
||||
self$items <- c(self$items, x)
|
||||
invisible(self)
|
||||
}
|
||||
),
|
||||
private = list(
|
||||
validate_it = function() TRUE
|
||||
),
|
||||
active = list(
|
||||
size = function() length(private$items)
|
||||
))
|
||||
|
||||
GeomX <- ggproto("GeomX", Geom,
|
||||
extra_param = "no",
|
||||
draw_panel = function(data, panel) {
|
||||
render_geom(data)
|
||||
}
|
||||
)
|
||||
|
||||
methods::setClass("QualClass", contains = "QBase")
|
||||
R6::R6Class("QualR6", public = list(qm = function() do_q()))
|
||||
Gen <- R6Class(GenName, public = list(gm = function() 1))
|
||||
BadGG <- ggproto(NULL, Geom, draw_key = function(x) render_key(x))
|
||||
NoInherit <- R6Class("NoInherit", inherit = pkg::Parent)
|
||||
factory <- function() {
|
||||
Local <- setRefClass("LocalCls", methods = list(lm = function() lcall()))
|
||||
Local
|
||||
}
|
||||
Empty <- R6Class("Empty", public = list())
|
||||
Pos <- setRefClass("PosCls", methods = list(function() 1))
|
||||
s3.method <- print.myclass <- NULL
|
||||
print.data.frame2 <- function(x, ...) {
|
||||
format_it(x)
|
||||
}
|
||||
env$attached <- function(x) side_call(x)
|
||||
"strname" <- function() 1
|
||||
setMethod("show", signature("Cls"), function(object) cat_it(object))
|
||||
setGeneric("area")
|
||||
setValidity("Cls", function(object) TRUE)
|
||||
Late <- R6Class("Late",
|
||||
public = list(pm = function() p_call()),
|
||||
inherit = LateBase)
|
||||
|
||||
# --- call zoo & lhs shapes ---------------------------------------------------
|
||||
"strassign" <- 6
|
||||
x[1] <- 7
|
||||
attr(x, "who") <- 8
|
||||
names(x) <- c("a")
|
||||
obj$field <- 9
|
||||
obj@slot <- 10
|
||||
assign("via_assign", 11)
|
||||
delayedAssign("lazy_one", compute_it())
|
||||
makeActiveBinding("active_one", function() 1, environment())
|
||||
obj$meth(3)
|
||||
lst$a$b(4)
|
||||
o@s$m(5)
|
||||
Negate(`%in%`)(6)
|
||||
"strfn"(7)
|
||||
(handler)(8)
|
||||
lst[[1]](9)
|
||||
`weird name` <- 12
|
||||
`%+%` <- function(a, b) paste(a, b)
|
||||
result <- if (cond) f_yes() else f_no()
|
||||
for (i in seq_len(10)) body_call(i)
|
||||
while (keep_going()) step_once()
|
||||
repeat break
|
||||
local({
|
||||
local_hidden <- 13
|
||||
local_fn <- function() 14
|
||||
})
|
||||
try(risky_call())
|
||||
Recall(1)
|
||||
UseMethod("generic_dispatch")
|
||||
do.call("dyn_target", list(1))
|
||||
do.call(real_target, list(2))
|
||||
match.fun("fun_by_name")(3)
|
||||
stopifnot(is_ok(x))
|
||||
on.exit(cleanup_fn())
|
||||
invisible(NULL)
|
||||
|
||||
# --- return-as-named-node, duplicates, right-assign in a body ----------------
|
||||
f <- function(x) {
|
||||
if (x > 0) return(g(x))
|
||||
h(x)
|
||||
}
|
||||
ret_val <- return
|
||||
setMethod("area", "Sq", area_impl)
|
||||
require(pkg2, quietly = TRUE)
|
||||
dupline <- function() 1; dupline <- function() 2
|
||||
dup_var <- 1; dup_var <- 2
|
||||
runner <- function() {
|
||||
fetch() -> got
|
||||
got
|
||||
}
|
||||
dt[, b := compute_b(a)]
|
||||
|
||||
# --- parse-clean battery shapes (must NOT defer) -----------------------------
|
||||
rs <- r"(no \escape here)"
|
||||
rs2 <- r"#(one "quoted" bit)#"
|
||||
plot(1, )
|
||||
sliced <- x[1, ]
|
||||
piped <- x |> f_ph(y = _)
|
||||
|
||||
# --- UTF-16 columns ----------------------------------------------------------
|
||||
msg <- "héllo 🎉"
|
||||
emoji_caller <- function() after_emoji("🎉🎉", target_fn())
|
||||
@@ -36,7 +36,7 @@ const kernelBuilt = fs.existsSync(KERNEL_PATH);
|
||||
|
||||
// Every kernel-capable language. `jsx` shares the javascript grammar on BOTH
|
||||
// paths (langs.rs mirrors WASM_GRAMMAR_FILES), so the distinct grammars are:
|
||||
const GRAMMAR_LANGUAGES: Language[] = ['typescript', 'tsx', 'javascript', 'java', 'python', 'go', 'c', 'cpp', 'rust', 'csharp', 'ruby', 'php', 'swift', 'kotlin'];
|
||||
const GRAMMAR_LANGUAGES: Language[] = ['typescript', 'tsx', 'javascript', 'java', 'python', 'go', 'c', 'cpp', 'rust', 'csharp', 'ruby', 'php', 'swift', 'kotlin', 'r'];
|
||||
|
||||
describe.skipIf(!kernelBuilt)('kernel↔wasm grammar parity', () => {
|
||||
beforeAll(async () => {
|
||||
|
||||
@@ -0,0 +1,148 @@
|
||||
/**
|
||||
* Kernel↔wasm R extraction parity (R7b batch 4 of the kernel migration).
|
||||
*
|
||||
* Asserts the native walker (codegraph-kernel/src/rlang.rs) produces the SAME
|
||||
* ExtractionResult as the wasm TreeSitterExtractor — nodes, edges, and
|
||||
* unresolved refs compared as canonicalized multisets — over the checked-in
|
||||
* torture fixture (torture.R: every visitNode-hook branch — function/variable/
|
||||
* constant assignments in all five operators, the class quartet
|
||||
* setClass/setRefClass/R6Class/ggproto with list()+direct methods and
|
||||
* extends refs, setGeneric/setMethod, the import quintet with its five
|
||||
* silent-consumption shapes, class-idiom variable suppression, chained/
|
||||
* right-assign/precedence-ghost gaps — plus the raw-text callee zoo
|
||||
* (`pkg::fn`, `obj$meth`, `"strfn"` quotes kept, `(handler)` conversion,
|
||||
* `calls "return"`), duplicate same-(kind,name,line) ids, parse-clean raw
|
||||
* strings/underscore-pipe/trailing commas, and UTF-16 emoji columns) and its
|
||||
* CRLF variant (derived in-memory — #1329).
|
||||
*
|
||||
* The full-repo sweep lives in scripts/kernel-parity.mjs (dplyr/ggplot2/shiny
|
||||
* for the §5 gate); this suite keeps the invariant alive in `npm test`.
|
||||
* Skips when no kernel binary is staged; CODEGRAPH_KERNEL_EXPECT=1 turns that
|
||||
* into a failure (kernel-scaffold.test.ts).
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeAll, beforeEach, afterEach } from 'vitest';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import { extractFromSource } from '../src/extraction';
|
||||
import { initGrammars, loadGrammarsForLanguages } from '../src/extraction/grammars';
|
||||
import { tryKernelExtract, resetKernelForTests } from '../src/extraction/kernel';
|
||||
import type { ExtractionResult } from '../src/types';
|
||||
|
||||
const KERNEL_PATH = path.join(
|
||||
__dirname,
|
||||
'..',
|
||||
'codegraph-kernel',
|
||||
'prebuilds',
|
||||
`${process.platform}-${process.arch}`,
|
||||
'codegraph-kernel.node'
|
||||
);
|
||||
const kernelBuilt = fs.existsSync(KERNEL_PATH);
|
||||
|
||||
const FIXTURE_DIR = path.join(__dirname, 'fixtures', 'kernel-parity');
|
||||
|
||||
function canon(result: ExtractionResult): { nodes: string[]; edges: string[]; refs: string[] } {
|
||||
return {
|
||||
nodes: result.nodes
|
||||
.map(({ updatedAt: _u, ...n }) => JSON.stringify(n, Object.keys(n).sort()))
|
||||
.sort(),
|
||||
edges: result.edges.map((e) => JSON.stringify(e, Object.keys(e).sort())).sort(),
|
||||
refs: result.unresolvedReferences
|
||||
.map((r) => JSON.stringify(r, Object.keys(r).sort()))
|
||||
.sort(),
|
||||
};
|
||||
}
|
||||
|
||||
const ENV_KEYS = ['CODEGRAPH_KERNEL', 'CODEGRAPH_KERNEL_LANGS'] as const;
|
||||
let savedEnv: Record<string, string | undefined>;
|
||||
|
||||
describe.skipIf(!kernelBuilt)('kernel R extraction parity', () => {
|
||||
beforeAll(async () => {
|
||||
await initGrammars();
|
||||
await loadGrammarsForLanguages(['r']);
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
savedEnv = Object.fromEntries(ENV_KEYS.map((k) => [k, process.env[k]]));
|
||||
resetKernelForTests();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
for (const k of ENV_KEYS) {
|
||||
if (savedEnv[k] === undefined) delete process.env[k];
|
||||
else process.env[k] = savedEnv[k];
|
||||
}
|
||||
resetKernelForTests();
|
||||
});
|
||||
|
||||
function assertParity(filePath: string, source: string, minNodes = 3): ExtractionResult {
|
||||
process.env.CODEGRAPH_KERNEL_LANGS = 'all';
|
||||
delete process.env.CODEGRAPH_KERNEL;
|
||||
const viaKernel = tryKernelExtract(filePath, source, 'r');
|
||||
expect(viaKernel, `kernel extraction failed for ${filePath}`).not.toBeNull();
|
||||
|
||||
process.env.CODEGRAPH_KERNEL = '0';
|
||||
const viaWasm = extractFromSource(filePath, source, 'r');
|
||||
delete process.env.CODEGRAPH_KERNEL;
|
||||
|
||||
const k = canon(viaKernel!);
|
||||
const w = canon(viaWasm);
|
||||
expect(k.nodes, `${filePath}: nodes`).toEqual(w.nodes);
|
||||
expect(k.edges, `${filePath}: edges`).toEqual(w.edges);
|
||||
expect(k.refs, `${filePath}: refs`).toEqual(w.refs);
|
||||
expect(viaWasm.nodes.length).toBeGreaterThanOrEqual(minNodes);
|
||||
return viaKernel!;
|
||||
}
|
||||
|
||||
it('torture fixture: hook branches, class quartet, import quintet, call zoo', () => {
|
||||
const file = path.join(FIXTURE_DIR, 'torture.R');
|
||||
const result = assertParity('fixtures/torture.R', fs.readFileSync(file, 'utf8'), 40);
|
||||
|
||||
// Pin the R-distinctive quirks on the KERNEL arm so both arms drifting
|
||||
// together can't silently lose them (checklist §The visitNode hook):
|
||||
// return/next/break are named nodes in v1.2.0 — `return(g(x))` emits a
|
||||
// literal `calls "return"` ref alongside `calls g`.
|
||||
const refNames = result.unresolvedReferences.map((r) => r.referenceName);
|
||||
expect(refNames).toContain('return');
|
||||
// Dynamic-arg imports are consumed SILENTLY — the `file.path` call inside
|
||||
// `source(file.path("R", "dyn.R"))` vanishes (subtree never visited).
|
||||
expect(refNames).not.toContain('file.path');
|
||||
// The named-first-argument bug: `library(help = docpkg)` imports docpkg.
|
||||
expect(result.nodes.some((n) => n.kind === 'import' && n.name === 'docpkg')).toBe(true);
|
||||
// Class-idiom suppression: Account has a class node but NO variable twin.
|
||||
expect(result.nodes.some((n) => n.kind === 'class' && n.name === 'Account')).toBe(true);
|
||||
expect(result.nodes.some((n) => n.kind === 'variable' && n.name === 'Account')).toBe(false);
|
||||
// No R node ever carries a docstring (roxygen is dropped).
|
||||
expect(result.nodes.every((n) => n.docstring === undefined)).toBe(true);
|
||||
});
|
||||
|
||||
// CRLF variant — the shape every Windows autocrlf checkout has. Derived in
|
||||
// memory so no platform or editor can silently normalize it away. The only
|
||||
// LF↔CRLF extraction difference for R is `\r\n` bytes inside multi-line
|
||||
// import signatures — both arms must agree byte-for-byte.
|
||||
it('torture fixture CRLF parity', () => {
|
||||
const file = path.join(FIXTURE_DIR, 'torture.R');
|
||||
const crlf = fs.readFileSync(file, 'utf8').replace(/(?<!\r)\n/g, '\r\n');
|
||||
assertParity('fixtures/torture.R (crlf)', crlf, 40);
|
||||
});
|
||||
|
||||
// BOM variant — the err-battery pinned BOM sources as parse-clean; derive it
|
||||
// in-memory for the same reason as CRLF.
|
||||
it('torture fixture BOM parity', () => {
|
||||
const file = path.join(FIXTURE_DIR, 'torture.R');
|
||||
const bom = '' + fs.readFileSync(file, 'utf8');
|
||||
assertParity('fixtures/torture.R (bom)', bom, 40);
|
||||
});
|
||||
|
||||
it('files with parse errors defer to the wasm extractor (recovery is encoding-dependent)', () => {
|
||||
// `x <-` with no rhs is a MISSING-node incomplete (genuinely broken).
|
||||
const broken = 'ok_fn <- function() 1\nx <-\n';
|
||||
process.env.CODEGRAPH_KERNEL_LANGS = 'all';
|
||||
delete process.env.CODEGRAPH_KERNEL;
|
||||
expect(tryKernelExtract('src/broken.R', broken, 'r')).toBeNull();
|
||||
process.env.CODEGRAPH_KERNEL = '0';
|
||||
const viaWasm = extractFromSource('src/broken.R', broken, 'r');
|
||||
delete process.env.CODEGRAPH_KERNEL;
|
||||
expect(viaWasm.nodes.some((n) => n.kind === 'file')).toBe(true);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user