feat(extraction): add ArkTS language support with ArkUI dispatch bridges (#396, #512, #890 via #648) (#1186)

Adds ArkTS (.ets, HarmonyOS/OpenHarmony) as a first-class language:
full TypeScript-grade extraction via the harmony-contrib tree-sitter
grammar (MIT, vendored byte-identical from the tree-sitter-arkts 0.2.0
npm tarball), plus the ArkUI constructs that make HarmonyOS apps
traceable:

- @Component/@ComponentV2 structs with decorators from both grammar
  positions; members extract as class members with qualified names.
- build() component trees: child instantiation edges via
  arkui_component_expression, no synthesizer needed.
- Attribute chains emitted dot-prefixed and resolved ONLY against
  @Extend/@Styles/@AnimatableExtend/@Builder helpers (unique-or-drop) —
  bare-name fallthrough produced 36,840 wrong edges (17% of calls) on
  the OpenHarmony samples monorepo. All four grammar chain shapes
  handled, including the detached-chain forms.
- .onClick(this.handler) method-reference bindings.
- ohpm workspace modules: bare imports follow oh-package.json5 file:
  deps (ambiguous names dropped), honoring each module's main entry —
  which also lets .ts consumers resolve .ets modules.
- ArkUI dynamic-dispatch bridges, all provenance:'heuristic' with
  wiring-site metadata: assignment-gated state->build() re-render
  (V1 @State family + V2 @Local/@Provider/@Consumer),
  @ohos.events.emitter emit->subscriber pairing on static event keys
  (numeric ids same-file, named constants same-module, fan-out capped),
  and router.pushUrl literal urls -> the target page's @Entry struct.
- $r/$rawfile resource intrinsics treated as built-ins; arkts joins the
  web language family, value-reference edges, re-export chase, and the
  other TS-applicable gates.

Also ships a language-agnostic index-completeness guard: indexAll
stamps index_state (indexing -> complete/partial/failed), reconciles
discovered vs accounted files (a loaded run silently dropped 37 files),
and codegraph status surfaces truncated/partial indexes in human and
--json output.

Validated on HarmoneyOpenEye (82 files), CoolMallArkTS (528, modular
ohpm + ArkUI V2), and openharmony/applications_app_samples (11,693
files, 202,890 nodes stable across re-index, attribute false-positive
audit 36,840 -> 588 residual all-plausible). Supersedes PRs #656 and
#988 with credit — both informed this implementation.

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Colby Mchenry
2026-07-06 09:07:15 -05:00
committed by GitHub
co-authored by Claude Fable 5
parent f8cdbe3c67
commit 99152212a9
21 changed files with 1695 additions and 21 deletions
+13 -1
View File
@@ -47,6 +47,7 @@ const WASM_GRAMMAR_FILES: Record<GrammarLanguage, string> = {
erlang: 'tree-sitter-erlang.wasm',
solidity: 'tree-sitter-solidity.wasm',
terraform: 'tree-sitter-terraform.wasm',
arkts: 'tree-sitter-arkts.wasm',
};
/**
@@ -58,6 +59,10 @@ export const EXTENSION_MAP: Record<string, Language> = {
// ESM/CJS TypeScript module extensions — parsed as TS (no JSX). (#366)
'.mts': 'typescript',
'.cts': 'typescript',
// ArkTS (HarmonyOS / OpenHarmony) — a TypeScript superset with declarative
// UI (`@Component struct` + `build()`). Own grammar (a tree-sitter-typescript
// -style fork); plain `.ts` in an ArkTS project stays TypeScript. (#648)
'.ets': 'arkts',
'.js': 'javascript',
'.mjs': 'javascript',
'.cjs': 'javascript',
@@ -292,7 +297,13 @@ export async function loadGrammarsForLanguages(languages: Language[]): Promise<v
// ship HCL/Terraform at all, so we vendor the prebuilt
// tree-sitter-terraform.wasm from @tree-sitter-grammars/tree-sitter-hcl
// 1.2.0 (Apache-2.0) — byte-identical to the npm package's artifact.
const wasmPath = (lang === 'pascal' || lang === 'scala' || lang === 'lua' || lang === 'luau' || lang === 'csharp' || lang === 'r' || lang === 'cfml' || lang === 'cfscript' || lang === 'cfquery' || lang === 'cobol' || lang === 'vbnet' || lang === 'erlang' || lang === 'terraform')
// ArkTS: tree-sitter-wasms doesn't ship it either; we vendor the prebuilt
// tree-sitter-arkts.wasm from the tree-sitter-arkts 0.2.0 npm package
// (harmony-contrib/tree-sitter-arkts, MIT) — byte-identical to the npm
// tarball's artifact. It extends the tree-sitter-javascript grammar the
// same way tree-sitter-typescript does, adding `struct_declaration` and
// the `arkui_component_expression` build() DSL.
const wasmPath = (lang === 'pascal' || lang === 'scala' || lang === 'lua' || lang === 'luau' || lang === 'csharp' || lang === 'r' || lang === 'cfml' || lang === 'cfscript' || lang === 'cfquery' || lang === 'cobol' || lang === 'vbnet' || lang === 'erlang' || lang === 'terraform' || lang === 'arkts')
? path.join(__dirname, 'wasm', wasmFile)
: require.resolve(`tree-sitter-wasms/out/${wasmFile}`);
const language = await WasmLanguage.load(wasmPath);
@@ -518,6 +529,7 @@ export function getLanguageDisplayName(language: Language): string {
vbnet: 'Visual Basic .NET',
erlang: 'Erlang',
terraform: 'Terraform',
arkts: 'ArkTS',
unknown: 'Unknown',
};
return names[language] || language;
+10
View File
@@ -83,6 +83,14 @@ export interface IndexResult {
filesIndexed: number;
filesSkipped: number;
filesErrored: number;
/**
* How many indexable files the scan discovered — the ground truth the
* indexed/skipped/errored tallies must add up to. A shortfall means files
* were silently dropped mid-pipeline (e.g. a killed worker under load) and
* the index is PARTIAL; callers surface that rather than trusting the
* counts. Only set by full-index runs (indexAll), not indexFiles/sync.
*/
filesDiscovered?: number;
nodesCreated: number;
edgesCreated: number;
errors: ExtractionError[];
@@ -1512,6 +1520,7 @@ export class ExtractionOrchestrator {
filesIndexed,
filesSkipped,
filesErrored,
filesDiscovered: total,
nodesCreated: totalNodes,
edgesCreated: totalEdges,
errors: [{ message: 'Aborted', severity: 'error' }, ...errors],
@@ -1645,6 +1654,7 @@ export class ExtractionOrchestrator {
filesIndexed,
filesSkipped,
filesErrored,
filesDiscovered: total,
nodesCreated: totalNodes,
edgesCreated: totalEdges,
errors,
+128
View File
@@ -0,0 +1,128 @@
import type { LanguageExtractor } from '../tree-sitter-types';
import { typescriptExtractor } from './typescript';
import type { Node as SyntaxNode } from 'web-tree-sitter';
/**
* ArkTS (HarmonyOS / OpenHarmony, `.ets`) — a TypeScript superset whose
* headline feature is declarative UI: an `@Component struct` with a `build()`
* method describing the view tree, `@State`/`@Prop`/`@Link` reactive
* properties, and global `@Builder`/`@Extend`/`@Styles` functions.
*
* The vendored grammar (harmony-contrib/tree-sitter-arkts) extends
* tree-sitter-javascript exactly the way tree-sitter-typescript does, so every
* TS node type — and therefore the whole typescriptExtractor — applies
* verbatim. ArkTS-specific shapes it adds:
*
* - `struct_declaration` / `struct_body` — the `@Component struct`. Same
* `name:`/`body:` fields as class_declaration; members are ordinary
* `method_definition` / `public_field_definition` nodes, so struct members
* extract through the standard class-member paths.
* - `arkui_component_expression` — a build()-DSL component instantiation
* (`Column() { … }`). Carries a `function:` field (the component), an
* optional `children:` block, and — unlike TS — the CHAINED ATTRIBUTES as
* repeated `property:`/`arguments:` field pairs on the SAME node
* (`Text(x).fontSize(16).opacity(0.6)` is ONE node, not nested calls).
* Handled by the arkts branch in extractCall (tree-sitter.ts).
* - Decorators on functions (`@Builder function F() {}`) — invalid in TS,
* first-class here (a `decorator:` field on function_declaration), so the
* core's existing extractDecoratorsFor path captures them.
*/
/** Reactive/state decorators that make a member worth flagging (searchable). */
const DECORATED_MEMBER_TYPES = new Set([
'struct_declaration',
'public_field_definition',
'method_definition',
'function_declaration',
]);
/**
* Collect decorator names for a declaration from BOTH positions the grammar
* produces: direct `decorator` children (`@Entry @Component struct X`,
* `@State count` on a field) and preceding `decorator` siblings (`@Builder`
* before a method_definition inside struct_body; `@Component` on the
* export_statement wrapping `export struct X`). The backwards sibling walk
* stops at the first non-decorator so an earlier declaration's decorators
* never leak in (mirrors extractDecoratorsFor's sibling pass).
*/
function collectDecoratorNames(node: SyntaxNode): string[] | undefined {
const names: string[] = [];
const nameOf = (dec: SyntaxNode): string | undefined => {
for (let i = 0; i < dec.namedChildCount; i++) {
const child = dec.namedChild(i);
if (!child) continue;
if (child.type === 'identifier') return child.text;
if (child.type === 'call_expression') {
// `@StorageLink('theme')` / `@Extend(Text)` — the decorator name is
// the callee.
const fn = child.childForFieldName('function');
if (fn?.type === 'identifier') return fn.text;
}
}
return undefined;
};
for (let i = 0; i < node.namedChildCount; i++) {
const child = node.namedChild(i);
if (child?.type === 'decorator') {
const n = nameOf(child);
if (n) names.push(n);
}
}
const parent = node.parent;
if (parent) {
// Find this node among the parent's named children by start offset
// (wrapper identity is not stable across navigation), then walk backwards.
const start = node.startIndex;
let idx = -1;
for (let i = 0; i < parent.namedChildCount; i++) {
const sib = parent.namedChild(i);
if (sib && sib.startIndex === start) {
idx = i;
break;
}
}
for (let i = idx - 1; i >= 0; i--) {
const sib = parent.namedChild(i);
if (!sib || sib.type !== 'decorator') break;
const n = nameOf(sib);
if (n) names.unshift(n);
}
}
return names.length > 0 ? names : undefined;
}
export const arktsExtractor: LanguageExtractor = {
...typescriptExtractor,
// `@Component struct X { … }` — extractStruct handles it (kind `struct`,
// members extracted like class members, `this.m()` resolution and the
// class/struct containment gates in the name-matcher all apply as-is). The
// component-ness is preserved on the node's decorators (`Component`,
// `Entry`, `CustomDialog`, `Reusable`), captured by extractModifiers below.
structTypes: ['struct_declaration'],
// build()-DSL component instantiations are call sites: `TodoRow({...})`
// inside a parent's build() is the parent→child component edge, resolved by
// the ordinary call pipeline against the child's struct node. The arkts
// branch in extractCall also lifts each chained `.attr(...)` (emitted
// dot-prefixed so it can ONLY resolve to `@Extend`/`@Styles`/`@Builder`
// attribute helpers — see matchReference) and `.onXxx(this.handler)`
// method-reference bindings. `leading_dot_expression` is the detached-chain
// shape the grammar produces when a nested component's chain starts on the
// line after its closing `}` inside arkui_children.
callTypes: ['call_expression', 'arkui_component_expression', 'leading_dot_expression'],
// Surface ArkTS decorators on the node's `decorators` list (searchable, and
// the hook a future ArkUI state→build synthesizer keys off). Core paths
// already emit `decorates` REFERENCES for classes/methods/properties/
// functions; this hook is what puts the names on struct nodes too —
// extractStruct has no extractDecoratorsFor call, and node.decorators is
// only populated via extractModifiers (see createNode).
extractModifiers: (node) => {
if (!DECORATED_MEMBER_TYPES.has(node.type)) return undefined;
return collectDecoratorNames(node);
},
};
+2
View File
@@ -34,6 +34,7 @@ import { vbnetExtractor } from './vbnet';
import { erlangExtractor } from './erlang';
import { solidityExtractor } from './solidity';
import { terraformExtractor } from './terraform';
import { arktsExtractor } from './arkts';
export const EXTRACTORS: Partial<Record<Language, LanguageExtractor>> = {
typescript: typescriptExtractor,
@@ -65,4 +66,5 @@ export const EXTRACTORS: Partial<Record<Language, LanguageExtractor>> = {
erlang: erlangExtractor,
solidity: solidityExtractor,
terraform: terraformExtractor,
arkts: arktsExtractor,
};
+179 -6
View File
@@ -373,7 +373,7 @@ export class TreeSitterExtractor {
// Value-reference edges (default ON; set CODEGRAPH_VALUE_REFS=0 to disable; see flushValueRefs).
// Same-file reads of file-scope const/var symbols → `references` edges so impact analysis catches
// value consumers ("change this constant/table, affect its readers").
private static readonly VALUE_REF_LANGS = new Set<string>(['typescript', 'javascript', 'tsx', 'go', 'python', 'rust', 'ruby', 'c', 'java', 'csharp', 'php', 'scala', 'kotlin', 'swift', 'dart', 'pascal']);
private static readonly VALUE_REF_LANGS = new Set<string>(['typescript', 'javascript', 'tsx', 'arkts', 'go', 'python', 'rust', 'ruby', 'c', 'java', 'csharp', 'php', 'scala', 'kotlin', 'swift', 'dart', 'pascal']);
private static readonly MAX_VALUE_REF_NODES = 20_000;
private readonly valueRefsEnabled = process.env.CODEGRAPH_VALUE_REFS !== '0';
private fileScopeValues = new Map<string, string>();
@@ -1183,7 +1183,8 @@ export class TreeSitterExtractor {
else if (
nodeType === 'export_statement' &&
(this.language === 'typescript' || this.language === 'tsx' ||
this.language === 'javascript' || this.language === 'jsx') &&
this.language === 'javascript' || this.language === 'jsx' ||
this.language === 'arkts') &&
getChildByField(node, 'source')
) {
const parentId = this.nodeStack[this.nodeStack.length - 1];
@@ -2487,7 +2488,8 @@ export class TreeSitterExtractor {
// Extract variable declarators based on language
if (this.language === 'typescript' || this.language === 'javascript' ||
this.language === 'tsx' || this.language === 'jsx' || this.language === 'cfscript') {
this.language === 'tsx' || this.language === 'jsx' || this.language === 'cfscript' ||
this.language === 'arkts') {
// Handle lexical_declaration and variable_declaration
// These contain one or more variable_declarator children
for (let i = 0; i < node.namedChildCount; i++) {
@@ -2916,7 +2918,7 @@ export class TreeSitterExtractor {
// property/method nodes under the type alias so `recorder.stop()`
// can attach the call edge to `RecorderHandle.stop` instead of
// an unrelated class method picked by path-proximity (#359).
if (this.language === 'typescript' || this.language === 'tsx') {
if (this.language === 'typescript' || this.language === 'tsx' || this.language === 'arkts') {
this.extractTsTypeAliasMembers(value, typeAliasNode);
// `type List = [ Service<'name', Req, Resp>, … ]` — surface each
// entry's string-literal name as a searchable member (issue #634).
@@ -3132,7 +3134,8 @@ export class TreeSitterExtractor {
// called/typed symbols still record a cross-file dependency (TS/JS only).
if (
this.language === 'typescript' || this.language === 'tsx' ||
this.language === 'javascript' || this.language === 'jsx'
this.language === 'javascript' || this.language === 'jsx' ||
this.language === 'arkts'
) {
const parentId = this.nodeStack[this.nodeStack.length - 1];
if (parentId) this.emitImportBindingRefs(node, parentId);
@@ -3894,6 +3897,176 @@ export class TreeSitterExtractor {
return;
}
// ArkTS build()-DSL handling. Three shapes carry UI-attribute chains, and
// all of their attribute names are emitted with a LEADING DOT
// (`.titleStyle`, `.width`) — an impossible identifier that routes them to
// a dedicated matcher strategy resolving ONLY to decorator-marked
// attribute helpers (`@Extend`/`@Styles`/`@AnimatableExtend`/`@Builder`
// functions). Bare names would go through global name matching, where
// framework attributes (`.width`, `.fontSize`, appearing on nearly every
// UI line) hit arbitrary same-named symbols — measured on the OpenHarmony
// samples monorepo, that produced 36k wrong edges (17% of all calls),
// including single properties with 3,400+ false callers.
//
// 1. `Column({space:8}) { … }.height('100%')` — ONE
// arkui_component_expression: `function:` = the component, chained
// attributes as repeated `property:`/`arguments:` field pairs.
// The component ref (`Column`, `TodoRow`) stays a PLAIN name — it
// resolves to the child `@Component struct`, giving the parent→child
// component-tree edge the way JSX children do for React.
// 2. `Image(x).width(10).onClick(this.f)` — ordinary nested
// call_expressions whose `function:` is a member_expression chained
// on a CALL RESULT (never a named receiver, so `svc.save()` /
// `this.vm.load()` are untouched and fall through to the generic
// paths below).
// 3. A nested component whose chain starts on the line AFTER its
// closing `}` inside arkui_children — the grammar detaches the chain
// into sibling `leading_dot_expression(identifier)` +
// `parenthesized_expression(args)` statement pairs; reassemble from
// the siblings.
//
// `.onXxx(this.handler)` METHOD-REFERENCE bindings (no call parens, so
// nothing else records them) additionally emit a call ref to the bare
// handler name — same-class resolution links the tap→handler hop.
// Arrow-function handlers need nothing: their bodies' calls already
// attribute to the enclosing build(). Children/argument subtrees are
// still walked by the caller, so nested components extract normally.
if (this.language === 'arkts') {
const emitAttr = (nameNode: SyntaxNode): void => {
const attrName = getNodeText(nameNode, this.source);
if (!attrName) return;
this.unresolvedReferences.push({
fromNodeId: callerId,
referenceName: '.' + attrName,
referenceKind: 'calls',
line: nameNode.startPosition.row + 1,
column: nameNode.startPosition.column,
});
};
// Emit `handler` for each bare `this.handler` among an on-attribute's
// arguments.
const emitThisHandlers = (args: SyntaxNode | null): void => {
if (!args) return;
for (let j = 0; j < args.namedChildCount; j++) {
const arg = args.namedChild(j);
if (arg?.type !== 'member_expression') continue;
const obj = getChildByField(arg, 'object');
const prop = getChildByField(arg, 'property');
if (obj?.type === 'this' && prop) {
this.unresolvedReferences.push({
fromNodeId: callerId,
referenceName: getNodeText(prop, this.source),
referenceKind: 'calls',
line: arg.startPosition.row + 1,
column: arg.startPosition.column,
});
}
}
};
// Shape 1: arkui_component_expression with property/arguments pairs.
if (node.type === 'arkui_component_expression') {
const componentField = getChildByField(node, 'function');
if (componentField && componentField.type === 'identifier') {
this.unresolvedReferences.push({
fromNodeId: callerId,
referenceName: getNodeText(componentField, this.source),
referenceKind: 'calls',
line: node.startPosition.row + 1,
column: node.startPosition.column,
});
}
for (let i = 0; i < node.childCount; i++) {
const child = node.child(i);
if (!child || child.type !== 'property_identifier') continue;
emitAttr(child);
if (/^on[A-Z]/.test(getNodeText(child, this.source))) {
// The attribute's arguments node is the next `arguments`-typed
// child before the following attribute name.
let args: SyntaxNode | null = null;
for (let k = i + 1; k < node.childCount; k++) {
const next = node.child(k);
if (!next) continue;
if (next.type === 'property_identifier') break;
if (next.type === 'arguments') {
args = next;
break;
}
}
emitThisHandlers(args);
}
}
return;
}
// Shape 2: fluent chain on a call result —
// call_expression(function: member_expression(object: <call>)), or the
// grammar's DSL-specific arkui_dsl_decorator_member_expression (same
// object/property fields; produced e.g. by `Column() { … }.alignItems(x)`
// in some chain positions — it ONLY occurs in attribute chains).
if (node.type === 'call_expression') {
const fn = getChildByField(node, 'function');
if (fn?.type === 'member_expression' || fn?.type === 'arkui_dsl_decorator_member_expression') {
const obj = getChildByField(fn, 'object');
const prop = getChildByField(fn, 'property');
if (
prop &&
(fn.type === 'arkui_dsl_decorator_member_expression' ||
obj?.type === 'call_expression' ||
obj?.type === 'arkui_component_expression')
) {
emitAttr(prop);
if (/^on[A-Z]/.test(getNodeText(prop, this.source))) {
emitThisHandlers(getChildByField(node, 'arguments'));
}
return;
}
}
// The INNERMOST call of a proper-form detached chain
// (`.alignItems(x).layoutWeight(1)…` under a leading_dot_expression)
// has a BARE IDENTIFIER function — the leading dot was consumed by
// the wrapper, so it masquerades as a plain `alignItems(...)` call.
// Walk up the member/call alternation; topping out at
// leading_dot_expression means the dot belongs to this chain.
if (fn?.type === 'identifier') {
let p: SyntaxNode | null = node.parent;
while (p && (p.type === 'member_expression' || p.type === 'call_expression')) {
p = p.parent;
}
if (p?.type === 'leading_dot_expression') {
emitAttr(fn);
if (/^on[A-Z]/.test(getNodeText(fn, this.source))) {
emitThisHandlers(getChildByField(node, 'arguments'));
}
return;
}
}
// Not a chained attribute — fall through to the generic call paths.
}
// Shape 3: detached chain segment — leading_dot_expression whose only
// named child is a bare identifier; its arguments sit in the NEXT
// sibling statement as a parenthesized_expression.
if (node.type === 'leading_dot_expression') {
const only = node.namedChildCount === 1 ? node.namedChild(0) : null;
if (only && only.type === 'identifier') {
emitAttr(only);
if (/^on[A-Z]/.test(getNodeText(only, this.source))) {
const stmt = node.parent; // expression_statement
const nextStmt = stmt?.nextNamedSibling;
const paren = nextStmt?.namedChild(0);
if (paren?.type === 'parenthesized_expression') {
emitThisHandlers(paren);
}
}
}
// The proper form (child is a call_expression chain, as inside
// `@Extend` bodies) needs nothing here — the walker descends into it
// and the inner call_expressions take the paths above.
return;
}
}
// Get the function/method being called
let calleeName = '';
@@ -5442,7 +5615,7 @@ export class TreeSitterExtractor {
* Languages that support type annotations (TypeScript, etc.)
*/
private readonly TYPE_ANNOTATION_LANGUAGES = new Set([
'typescript', 'tsx', 'dart', 'kotlin', 'swift', 'rust', 'go', 'java', 'csharp', 'scala', 'php',
'typescript', 'tsx', 'arkts', 'dart', 'kotlin', 'swift', 'rust', 'go', 'java', 'csharp', 'scala', 'php',
]);
/**
Binary file not shown.