feat(kernel): R4 — Java port with Lombok synthesis, gate passed, default-on
Java joins the native kernel (codegraph-kernel/src/java.rs), mirroring the wasm extractor's Java paths bug-for-bug: package namespaces, imports, javadoc, annotations→decorates, type_list inheritance, static-final constants, enum constants, anonymous classes (including the TS side's 0-based-line quirk on the extends ref), method_invocation calls with the this.field unwrap and the Foo.getInstance().bar() chain encoding (#645/#608), static-member value reads, method-reference fn-refs (#756), value-reference edges, and the full Lombok member synthesizer (#912: Getter/Setter/Data/Value/Builder/ToString/ EqualsAndHashCode/Slf4j-family with taken-member dedup). The shared docstring/textutil modules moved to crate level. Grammar: tree-sitter-java 0.23.5, with the wasm grammar vendored from the same tag (parser.c sha-matched) replacing tree-sitter-wasms' 2023-era build. Gate (plan §4c): extraction sweeps 100% — gson 262/262, retrofit 341/341, dubbo 4,048/4,048 — plus a Java torture fixture in npm test; full-init dump-diffs byte-identical on gson (49,766 rows), retrofit (62,735), and dubbo (441,266 rows); all R2/R3 repos re-verified; Linux container runs all 23 kernel tests green under CODEGRAPH_KERNEL_EXPECT=1. The gate caught a real cross-language bug: fn-ref dedupe and value-ref self-target checks must compare node ID STRINGS, not node-table rows — ids collide for same-(kind, name, line) nodes, which minified one-line bundles hit routinely (retrofit's website JS exposed it; latent in the TS/JS walker since R2, never released). Fixed in both walkers. Benchmark honesty: dubbo fresh-init on an 11-core Mac is ~flat (parse-loop wall 5,020→4,394ms; total ~11.3s both arms) because that wall is main-thread-bound (reads + store), not worker-CPU-bound — the §6 expectation assumed otherwise. Where worker CPU binds the kernel delivers: dubbo on a 2-CPU/6GB container drops 27.8-28.6s → 22.3-22.8s (~1.25×). The identified lever for the many-core headline is decoding kernel buffers directly into store rows (skipping per-node JS object materialization); the buffer contract already carries everything. DEFAULT_ROUTED now includes java. Full suite: 2,467 tests pass. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
c8cca9a601
commit
03d54e47a1
Generated
+11
@@ -52,6 +52,7 @@ dependencies = [
|
||||
"regex",
|
||||
"sha2",
|
||||
"tree-sitter",
|
||||
"tree-sitter-java",
|
||||
"tree-sitter-javascript",
|
||||
"tree-sitter-typescript",
|
||||
]
|
||||
@@ -479,6 +480,16 @@ dependencies = [
|
||||
"tree-sitter-language",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tree-sitter-java"
|
||||
version = "0.23.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0aa6cbcdc8c679b214e616fd3300da67da0e492e066df01bcf5a5921a71e90d6"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"tree-sitter-language",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tree-sitter-javascript"
|
||||
version = "0.25.0"
|
||||
|
||||
@@ -22,6 +22,7 @@ regex = "1"
|
||||
# bump these together with the wasm side or that gate fails.
|
||||
tree-sitter-typescript = "0.23"
|
||||
tree-sitter-javascript = "0.25"
|
||||
tree-sitter-java = "0.23"
|
||||
|
||||
[build-dependencies]
|
||||
napi-build = "2"
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -15,13 +15,14 @@ use tree_sitter::Language;
|
||||
|
||||
/// Languages this kernel binary can extract (reported by contractInfo;
|
||||
/// TS-side routing policy decides what actually routes).
|
||||
pub const LANGUAGES: [&str; 4] = ["typescript", "tsx", "javascript", "jsx"];
|
||||
pub const LANGUAGES: [&str; 5] = ["typescript", "tsx", "javascript", "jsx", "java"];
|
||||
|
||||
pub fn grammar_for(language: &str) -> Option<Language> {
|
||||
match language {
|
||||
"typescript" => Some(tree_sitter_typescript::LANGUAGE_TYPESCRIPT.into()),
|
||||
"tsx" => Some(tree_sitter_typescript::LANGUAGE_TSX.into()),
|
||||
"javascript" | "jsx" => Some(tree_sitter_javascript::LANGUAGE.into()),
|
||||
"java" => Some(tree_sitter_java::LANGUAGE.into()),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,8 +17,11 @@
|
||||
#![deny(clippy::all)]
|
||||
|
||||
mod buffers;
|
||||
mod docstring;
|
||||
mod ids;
|
||||
mod java;
|
||||
mod langs;
|
||||
mod textutil;
|
||||
mod tsjs;
|
||||
|
||||
use napi::bindgen_prelude::*;
|
||||
@@ -94,7 +97,10 @@ pub fn grammar_info(language: String) -> Option<GrammarInfo> {
|
||||
|
||||
#[napi]
|
||||
pub fn extract_file(file_path: String, content: String, language: String) -> Result<ExtractBuffers> {
|
||||
let out = tsjs::extract(&file_path, &content, &language).map_err(Error::from_reason)?;
|
||||
let out = match language.as_str() {
|
||||
"java" => java::extract(&file_path, &content).map_err(Error::from_reason)?,
|
||||
_ => tsjs::extract(&file_path, &content, &language).map_err(Error::from_reason)?,
|
||||
};
|
||||
Ok(ExtractBuffers {
|
||||
meta: out.meta.into(),
|
||||
nodes: out.nodes.into(),
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
//! src/extraction/tree-sitter.ts; TS-file line references are as of the R2
|
||||
//! port. Bug-for-bug fidelity is deliberate — fix the TS side first.
|
||||
|
||||
use super::util;
|
||||
use crate::textutil as util;
|
||||
use super::{
|
||||
body_of, is_builtin_type, is_literal_receiver, is_react_hoc, is_variable_type,
|
||||
is_vue_collection_name, Extra, Scope, Walker,
|
||||
@@ -43,7 +43,7 @@ impl<'t> Walker<'t> {
|
||||
}
|
||||
|
||||
let extra = Extra {
|
||||
docstring: super::docstring::preceding_docstring(node, self.src),
|
||||
docstring: crate::docstring::preceding_docstring(node, self.src),
|
||||
signature: self.signature_of(node),
|
||||
visibility: self.visibility_of(node),
|
||||
is_exported: Some(self.is_exported(node)),
|
||||
@@ -120,7 +120,7 @@ impl<'t> Walker<'t> {
|
||||
let resolved_body = body_of(node); // skipBodilessClass unset for TS/JS
|
||||
let name = self.extract_name(node);
|
||||
let extra = Extra {
|
||||
docstring: super::docstring::preceding_docstring(node, self.src),
|
||||
docstring: crate::docstring::preceding_docstring(node, self.src),
|
||||
visibility: self.visibility_of(node),
|
||||
is_exported: Some(self.is_exported(node)),
|
||||
..Extra::default()
|
||||
@@ -161,7 +161,7 @@ impl<'t> Walker<'t> {
|
||||
|
||||
let name = self.extract_name(node);
|
||||
let extra = Extra {
|
||||
docstring: super::docstring::preceding_docstring(node, self.src),
|
||||
docstring: crate::docstring::preceding_docstring(node, self.src),
|
||||
signature: self.signature_of(node),
|
||||
visibility: self.visibility_of(node),
|
||||
is_async: Some(self.is_async(node)),
|
||||
@@ -187,7 +187,7 @@ impl<'t> Walker<'t> {
|
||||
pub(super) fn extract_interface(&mut self, node: Node<'t>) {
|
||||
let name = self.extract_name(node);
|
||||
let extra = Extra {
|
||||
docstring: super::docstring::preceding_docstring(node, self.src),
|
||||
docstring: crate::docstring::preceding_docstring(node, self.src),
|
||||
is_exported: Some(self.is_exported(node)),
|
||||
..Extra::default()
|
||||
};
|
||||
@@ -209,7 +209,7 @@ impl<'t> Walker<'t> {
|
||||
let Some(body) = body_of(node) else { return };
|
||||
let name = self.extract_name(node);
|
||||
let extra = Extra {
|
||||
docstring: super::docstring::preceding_docstring(node, self.src),
|
||||
docstring: crate::docstring::preceding_docstring(node, self.src),
|
||||
visibility: self.visibility_of(node),
|
||||
is_exported: Some(self.is_exported(node)),
|
||||
..Extra::default()
|
||||
@@ -255,7 +255,7 @@ impl<'t> Walker<'t> {
|
||||
// --- extractProperty (#808 property-classified class fields) ---------------------
|
||||
|
||||
pub(super) fn extract_property(&mut self, node: Node<'t>) -> Option<(u32, String)> {
|
||||
let docstring = super::docstring::preceding_docstring(node, self.src);
|
||||
let docstring = crate::docstring::preceding_docstring(node, self.src);
|
||||
let visibility = self.visibility_of(node);
|
||||
let is_static = Some(self.is_static(node).unwrap_or(false)); // `?? false` — always present
|
||||
|
||||
@@ -296,7 +296,7 @@ impl<'t> Walker<'t> {
|
||||
pub(super) fn extract_variable(&mut self, node: Node<'t>) {
|
||||
let is_const = self.is_const_decl(node);
|
||||
let kind: &'static str = if is_const { "constant" } else { "variable" };
|
||||
let docstring = super::docstring::preceding_docstring(node, self.src);
|
||||
let docstring = crate::docstring::preceding_docstring(node, self.src);
|
||||
let is_exported = self.is_exported(node); // `?? false` — always present
|
||||
|
||||
for i in 0..node.named_child_count() {
|
||||
@@ -807,7 +807,7 @@ impl<'t> Walker<'t> {
|
||||
return false;
|
||||
}
|
||||
let extra = Extra {
|
||||
docstring: super::docstring::preceding_docstring(node, self.src),
|
||||
docstring: crate::docstring::preceding_docstring(node, self.src),
|
||||
is_exported: Some(self.is_exported(node)),
|
||||
..Extra::default()
|
||||
};
|
||||
@@ -858,7 +858,7 @@ impl<'t> Walker<'t> {
|
||||
"property"
|
||||
};
|
||||
let extra = Extra {
|
||||
docstring: super::docstring::preceding_docstring(child, self.src),
|
||||
docstring: crate::docstring::preceding_docstring(child, self.src),
|
||||
signature: Some(self.text(child).to_string()),
|
||||
qualified_name: Some(format!("{alias_name}::{member_name}")),
|
||||
..Extra::default()
|
||||
|
||||
@@ -9,10 +9,9 @@
|
||||
//! parity gate fails. Positions are emitted in UTF-16 code units (what
|
||||
//! web-tree-sitter reports), see util::col16.
|
||||
|
||||
mod docstring;
|
||||
mod extractors;
|
||||
mod fnref;
|
||||
pub(crate) mod util;
|
||||
use crate::textutil as util;
|
||||
|
||||
use crate::buffers::{
|
||||
build_meta, edge_kind_index, node_kind_index, Arena, BoolFlags, EdgeRow, EmitOut, NodeRow,
|
||||
@@ -148,6 +147,11 @@ pub struct Walker<'t> {
|
||||
arena: Arena,
|
||||
tables: Tables,
|
||||
stack: Vec<Scope>,
|
||||
/// Node id string per row. Rows are unique but IDS COLLIDE for same
|
||||
/// (kind, name, line) nodes — routine in minified one-line files — and the
|
||||
/// TS extractor's fn-ref dedupe and value-ref self-checks key on the ID,
|
||||
/// so parity requires comparing ids, not rows.
|
||||
node_ids: Vec<String>,
|
||||
/// Function/method names defined in this file (fn-ref flush gate).
|
||||
defined_fn_names: HashSet<String>,
|
||||
/// Simple names from `imports` refs (fn-ref flush gate).
|
||||
@@ -198,6 +202,7 @@ pub fn extract(file_path: &str, source: &str, language: &str) -> Result<EmitOut,
|
||||
arena: Arena::default(),
|
||||
tables: Tables::default(),
|
||||
stack: Vec::new(),
|
||||
node_ids: Vec::new(),
|
||||
defined_fn_names: HashSet::new(),
|
||||
imported_names: HashSet::new(),
|
||||
fn_ref_cands: Vec::new(),
|
||||
@@ -234,6 +239,7 @@ pub fn extract(file_path: &str, source: &str, language: &str) -> Result<EmitOut,
|
||||
return_type: NONE_STR,
|
||||
extra_json: NONE_STR,
|
||||
});
|
||||
w.node_ids.push(ids::file_node_id(file_path));
|
||||
w.stack.push(Scope { row: 0, kind: "file", name: base_name.to_string() });
|
||||
|
||||
w.visit_node(tree.root_node());
|
||||
@@ -399,6 +405,7 @@ impl<'t> Walker<'t> {
|
||||
target_id_str: NONE_STR,
|
||||
});
|
||||
|
||||
self.node_ids.push(id);
|
||||
if kind == "function" || kind == "method" {
|
||||
self.defined_fn_names.insert(name.to_string());
|
||||
}
|
||||
@@ -485,7 +492,9 @@ impl<'t> Walker<'t> {
|
||||
|
||||
let refs_kind = edge_kind_index("references").unwrap();
|
||||
for scope in &scopes {
|
||||
let mut seen: HashSet<u32> = HashSet::new();
|
||||
// Self-skip and per-scope dedupe compare node ID STRINGS (which
|
||||
// collide for same-(kind, name, line) nodes), matching the TS side.
|
||||
let mut seen: HashSet<&str> = HashSet::new();
|
||||
let mut stack: Vec<Node> = vec![scope.node];
|
||||
let mut visited = 0usize;
|
||||
while let Some(n) = stack.pop() {
|
||||
@@ -496,8 +505,12 @@ impl<'t> Walker<'t> {
|
||||
if matches!(n.kind(), "identifier" | "constant" | "name" | "simple_identifier") {
|
||||
let ref_name = self.text(n);
|
||||
if let Some(&target_row) = targets.get(ref_name) {
|
||||
if target_row != scope.row && ref_name != scope.name && !seen.contains(&target_row) {
|
||||
seen.insert(target_row);
|
||||
let target_id = self.node_ids[target_row as usize].as_str();
|
||||
if target_id != self.node_ids[scope.row as usize]
|
||||
&& ref_name != scope.name
|
||||
&& !seen.contains(&target_id)
|
||||
{
|
||||
seen.insert(target_id);
|
||||
let meta = self.arena.put(r#"{"valueRef":true}"#);
|
||||
self.tables.push_edge(&EdgeRow {
|
||||
source_idx: scope.row,
|
||||
@@ -559,7 +572,7 @@ impl<'t> Walker<'t> {
|
||||
if cands.is_empty() || util::is_generated_file(self.file_path) {
|
||||
return;
|
||||
}
|
||||
let mut seen: HashSet<(u32, String)> = HashSet::new();
|
||||
let mut seen: HashSet<(String, String)> = HashSet::new();
|
||||
for (from, c) in cands {
|
||||
// Gate: `this.<member>` always flushes; everything else must match
|
||||
// a same-file function/method or an imported name. (The `::` and
|
||||
@@ -571,7 +584,10 @@ impl<'t> Walker<'t> {
|
||||
{
|
||||
continue;
|
||||
}
|
||||
if !seen.insert((from, c.name.clone())) {
|
||||
// Dedupe on the node ID STRING, not the row — ids collide for
|
||||
// same-(kind, name, line) nodes (minified one-liners) and the TS
|
||||
// side keys its dedupe on `${fromNodeId}|${name}`.
|
||||
if !seen.insert((self.node_ids[from as usize].clone(), c.name.clone())) {
|
||||
continue;
|
||||
}
|
||||
let column = util::col16(self.src, &self.line_starts, c.row, c.column_byte);
|
||||
|
||||
Reference in New Issue
Block a user