feat(extraction): add Terraform/OpenTofu language support with module-boundary bridging (#83, #310, #648 — carries #706) (#1173)

* feat(extraction): add Terraform and OpenTofu language support

Index .tf, .tfvars, and .tofu files via the tree-sitter-terraform dialect
of HCL (vendored from @tree-sitter-grammars/tree-sitter-hcl, Apache-2.0).

Symbols extracted:
- resource / data  → class  (qualified "type.name" / "data.type.name")
- module           → module (qualified "module.name")
- variable         → variable (qualified "var.name")
- output           → variable (qualified "output.name")
- provider         → namespace
- locals           → constant per attribute (qualified "local.key")

References resolved cross-file:
- var.X, local.X, module.M[.out], data.T.N[.attr], <type>.<name>[.attr]
- built-ins skipped: each.*, count.*, self.*, path.*, terraform.workspace

The Terraform framework resolver disambiguates same-named candidates
across modules by preferring the one in the same directory as the
reference site, then by closest common-ancestor path, falling back to
the generic name matcher only when neither applies.

Validated on two Terraform monorepos (277 and 470 .tf files): indexing
runs in 1.3s and 2.4s respectively, query latency stays under 200ms,
and cross-module references resolve to the correct module 100% of the
time on inspected samples.

18 new extraction tests; full suite 1146/1148 green (2 pre-existing
flaky skips, 0 regressions).

* feat(terraform): bridge the module boundary and enforce directory scoping

Builds on #706. The module declaration was a dead end: module.M.out
resolved to the declaration and stopped, module inputs never reached the
child module's variables, and impact could not cross the boundary — on
real multi-module repos that breaks the core blast-radius question
("what breaks upstream if I change this module's variable/output").

- module blocks now wire across the boundary through :-scoped refs only
  the Terraform resolver understands: module.M:var.<input> → the child's
  variable node, module.M:output.<o> → the child's output node (emitted
  alongside the module.M declaration ref), and module.M:file → the local
  source directory's entry file (imports). Registry/git sources emit no
  file ref and resolve nothing — an out-of-repo module stays a visible
  boundary instead of a guess.
- .tfvars top-level assignments reference the variable they set, walking
  up to the nearest ancestor directory (envs/prod.tfvars → root vars).
- Resolution now enforces Terraform's real scoping: same-directory only
  (no cross-module fallback by common path prefix, no single-candidate
  anywhere-in-tree binding), and terraform refs never fall through to
  the generic name matcher — var.X can never legally bind outside its
  module directory, so the fallback could only add wrong edges.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* docs(terraform): README language table + changelog entry + agent-eval corpus

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

---------

Co-authored-by: Javier Rodríguez Fernández <jfernandez@freepik.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Colby Mchenry
2026-07-03 18:37:07 -05:00
committed by GitHub
co-authored by Claude Fable 5 Javier Rodríguez Fernández
parent e1a8d888e5
commit 6c24f4bddf
13 changed files with 1139 additions and 4 deletions
+11 -2
View File
@@ -46,6 +46,7 @@ const WASM_GRAMMAR_FILES: Record<GrammarLanguage, string> = {
vbnet: 'tree-sitter-vbnet.wasm',
erlang: 'tree-sitter-erlang.wasm',
solidity: 'tree-sitter-solidity.wasm',
terraform: 'tree-sitter-terraform.wasm',
};
/**
@@ -157,6 +158,10 @@ export const EXTENSION_MAP: Record<string, Language> = {
// shape as the `.yml` variants — the YAML/properties extractor emits one node
// per leaf key, and the Spring resolver links `@Value("${k}")` references.
'.properties': 'properties',
// Terraform / OpenTofu / HCL config — tree-sitter-terraform dialect of HCL.
'.tf': 'terraform',
'.tfvars': 'terraform',
'.tofu': 'terraform',
};
/**
@@ -283,8 +288,11 @@ export async function loadGrammarsForLanguages(languages: Language[]): Promise<v
// build (ABI 13) has no primary-constructor support and parses
// `class Foo(...)` as an ERROR that swallows the whole class (#237); we
// vendor the upstream ABI-15 tree-sitter-c-sharp 0.23.5 wasm, which parses
// primary constructors natively.
const wasmPath = (lang === 'pascal' || lang === 'scala' || lang === 'lua' || lang === 'luau' || lang === 'csharp' || lang === 'r' || lang === 'cfml' || lang === 'cfscript' || lang === 'cfquery' || lang === 'cobol' || lang === 'vbnet' || lang === 'erlang')
// primary constructors natively. Terraform: tree-sitter-wasms does not
// ship HCL/Terraform at all, so we vendor the prebuilt
// tree-sitter-terraform.wasm from @tree-sitter-grammars/tree-sitter-hcl
// 1.2.0 (Apache-2.0) — byte-identical to the npm package's artifact.
const wasmPath = (lang === 'pascal' || lang === 'scala' || lang === 'lua' || lang === 'luau' || lang === 'csharp' || lang === 'r' || lang === 'cfml' || lang === 'cfscript' || lang === 'cfquery' || lang === 'cobol' || lang === 'vbnet' || lang === 'erlang' || lang === 'terraform')
? path.join(__dirname, 'wasm', wasmFile)
: require.resolve(`tree-sitter-wasms/out/${wasmFile}`);
const language = await WasmLanguage.load(wasmPath);
@@ -509,6 +517,7 @@ export function getLanguageDisplayName(language: Language): string {
cobol: 'COBOL',
vbnet: 'Visual Basic .NET',
erlang: 'Erlang',
terraform: 'Terraform',
unknown: 'Unknown',
};
return names[language] || language;
+2
View File
@@ -33,6 +33,7 @@ import { cobolExtractor } from './cobol';
import { vbnetExtractor } from './vbnet';
import { erlangExtractor } from './erlang';
import { solidityExtractor } from './solidity';
import { terraformExtractor } from './terraform';
export const EXTRACTORS: Partial<Record<Language, LanguageExtractor>> = {
typescript: typescriptExtractor,
@@ -63,4 +64,5 @@ export const EXTRACTORS: Partial<Record<Language, LanguageExtractor>> = {
vbnet: vbnetExtractor,
erlang: erlangExtractor,
solidity: solidityExtractor,
terraform: terraformExtractor,
};
+475
View File
@@ -0,0 +1,475 @@
import type { Node as SyntaxNode } from 'web-tree-sitter';
import { getNodeText } from '../tree-sitter-helpers';
import type { LanguageExtractor } from '../tree-sitter-types';
// Grammar: tree-sitter-terraform (vendored at src/extraction/wasm/tree-sitter-terraform.wasm,
// built from @tree-sitter-grammars/tree-sitter-hcl, Apache-2.0). The HCL grammar
// is intentionally generic: ALL Terraform top-level constructs share the same
// AST node type `block`, distinguished only by the first `identifier` child
// (the block "type": resource, variable, data, module, output, locals, …).
// Labels for resources/data/modules/variables come from `string_lit` children
// AFTER that first identifier.
//
// resource "aws_s3_bucket" "my_bucket" { ... }
// └─ block
// ├─ identifier ("resource")
// ├─ string_lit ("aws_s3_bucket") ← type label
// ├─ string_lit ("my_bucket") ← name label
// ├─ block_start
// ├─ body
// │ ├─ attribute (identifier "bucket" "=" expression)
// │ └─ block ("tags" { ... }) ← nested block (skipped)
// └─ block_end
//
// References live inside `expression` subtrees: a leading `identifier` followed
// by zero or more `get_attr` (`.foo`) nodes. We synthesise qualified-name refs
// matching the node names emitted above (e.g. `var.region` → unresolved ref
// `var.region`, which the matcher resolves to the `variable "region"` node).
/** Built-in references that should NOT be resolved to project nodes. */
const BUILTIN_HEADS = new Set([
'each', // for_each iterator: each.key / each.value
'count', // count meta-argument: count.index
'self', // provisioner connection self.*
'path', // path.module / path.root / path.cwd
'terraform', // terraform.workspace
]);
/** Bare strings that we never want to treat as references. */
const BUILTIN_KEYWORDS = new Set(['null', 'true', 'false']);
/** Read a string_lit value (skipping the quotes / template start/end tokens). */
function stringLitValue(node: SyntaxNode, source: string): string {
const literal = node.namedChildren.find((c) => c?.type === 'template_literal');
if (literal) return getNodeText(literal, source);
// Empty string ("") parses as quoted_template_start + quoted_template_end
// with no template_literal — return empty.
return '';
}
/** Block "type" and its label values. Returns null if the block is malformed. */
function readBlockHeader(block: SyntaxNode, source: string): { type: string; labels: string[] } | null {
const named = block.namedChildren.filter((c): c is SyntaxNode => c !== null);
const first = named[0];
if (!first || first.type !== 'identifier') return null;
const type = getNodeText(first, source);
const labels: string[] = [];
for (let i = 1; i < named.length; i++) {
const child = named[i];
if (!child) continue;
if (child.type === 'string_lit') {
labels.push(stringLitValue(child, source));
} else if (child.type === 'identifier') {
// HCL allows unquoted identifier labels (rare in Terraform but legal).
labels.push(getNodeText(child, source));
} else {
break;
}
}
return { type, labels };
}
/** Find the `body` child of a block (it's after the labels and block_start). */
function getBlockBody(block: SyntaxNode): SyntaxNode | null {
return block.namedChildren.find((c) => c?.type === 'body') ?? null;
}
/**
* Walk an `expression` subtree and emit a reference for every dotted name
* whose head is a Terraform reference root (var / local / module / data / a
* resource type name). Skips built-ins.
*
* Patterns we recognise:
* var.X → ref "var.X" (variable "X")
* local.X → ref "local.X" (locals.X)
* module.M.O → ref "module.M" (module "M")
* data.T.N.A → ref "data.T.N" (data "T" "N")
* T.N[.A] → ref "T.N" (resource "T" "N", e.g. aws_x.y)
*/
function collectReferences(
expr: SyntaxNode,
source: string,
onRef: (qualifiedName: string, line: number, column: number) => void
): void {
// BFS for variable_expr and inspect each. variable_expr's only child is an
// identifier (the head); its siblings via get_attr / index chains live on
// the parent _expr_term, so walk the parent chain to collect them.
const queue: SyntaxNode[] = [expr];
while (queue.length) {
const n = queue.shift()!;
if (n.type === 'variable_expr') {
emitRefFromVariableExpr(n, source, onRef);
// Don't recurse into the chain we just read — but DO continue scanning
// siblings (e.g. function call arguments).
}
for (const c of n.namedChildren) {
if (c) queue.push(c);
}
}
}
function emitRefFromVariableExpr(
varExpr: SyntaxNode,
source: string,
onRef: (qualifiedName: string, line: number, column: number) => void
): void {
const id = varExpr.namedChildren.find((c) => c?.type === 'identifier');
if (!id) return;
const head = getNodeText(id, source);
if (BUILTIN_HEADS.has(head) || BUILTIN_KEYWORDS.has(head)) return;
// Walk get_attr siblings on the parent. The AST shape is roughly:
// expression > _expr_term (hidden) → variable_expr + get_attr + get_attr + ...
// tree-sitter exposes _expr_term children flattened on `expression`.
const attrs: string[] = [];
let cursor: SyntaxNode | null = varExpr.nextNamedSibling;
while (cursor) {
if (cursor.type === 'get_attr') {
const attrId = cursor.namedChildren.find((c) => c?.type === 'identifier');
if (!attrId) break;
attrs.push(getNodeText(attrId, source));
cursor = cursor.nextNamedSibling;
} else if (cursor.type === 'index' || cursor.type === 'new_index' || cursor.type === 'legacy_index' || cursor.type === 'splat' || cursor.type === 'attr_splat' || cursor.type === 'full_splat') {
// foo[0], foo[*], foo.* — keep walking but don't add a segment.
cursor = cursor.nextNamedSibling;
} else {
break;
}
}
const line = varExpr.startPosition.row + 1;
const col = varExpr.startPosition.column;
for (const qname of qualifyReference(head, attrs)) onRef(qname, line, col);
}
function qualifyReference(head: string, attrs: string[]): string[] {
switch (head) {
case 'var':
// var.X — variable "X"
return attrs[0] ? [`var.${attrs[0]}`] : [];
case 'local':
// local.K — locals attribute K
return attrs[0] ? [`local.${attrs[0]}`] : [];
case 'module':
// module.M[.OUTPUT] — module "M". A two-segment chain (`module.M.out`)
// additionally emits a scoped `module.M:output.out` ref that the
// Terraform resolver bridges to the `output "out"` node inside the
// module's source directory — the edge that carries impact across the
// module boundary instead of dead-ending at the declaration. Only the
// Terraform framework resolver understands the `:`-scoped spelling; if
// the module's source is a registry/git address the ref simply stays
// unresolved and the boundary remains visible.
if (!attrs[0]) return [];
return attrs[1]
? [`module.${attrs[0]}`, `module.${attrs[0]}:output.${attrs[1]}`]
: [`module.${attrs[0]}`];
case 'data':
// data.TYPE.NAME[.ATTR] — data "TYPE" "NAME"
return attrs[0] && attrs[1] ? [`data.${attrs[0]}.${attrs[1]}`] : [];
default:
// <type>.<name>[.<attr>...] — managed resource (e.g. aws_s3_bucket.my)
// Skip plain identifiers with no dotted chain — those are function calls,
// local-only variables, or template params.
if (!attrs[0]) return [];
return [`${head}.${attrs[0]}`];
}
}
export const terraformExtractor: LanguageExtractor = {
// The HCL grammar exposes everything as `block` / `attribute`; the default
// dispatcher does not know how to read Terraform's first-identifier-as-type
// convention, so we drive extraction entirely from visitNode below.
functionTypes: [],
classTypes: [],
methodTypes: [],
interfaceTypes: [],
structTypes: [],
enumTypes: [],
typeAliasTypes: [],
importTypes: [],
callTypes: [],
variableTypes: [],
nameField: '',
bodyField: '',
paramsField: '',
visitNode: (node, ctx) => {
if (node.type !== 'block') {
// .tfvars files carry no blocks — just top-level `name = value`
// assignments, each of which SETS the root module variable of that
// name. Reference the variable from the file node so "what sets
// var.region" is answerable from the graph.
if (
node.type === 'attribute' &&
ctx.filePath.endsWith('.tfvars') &&
node.parent?.type === 'body' &&
node.parent.parent?.type === 'config_file'
) {
const idNode = node.namedChildren.find((c) => c?.type === 'identifier');
const fileNodeId = ctx.nodeStack[0];
if (idNode && fileNodeId) {
ctx.addUnresolvedReference({
fromNodeId: fileNodeId,
referenceName: `var.${getNodeText(idNode, ctx.source)}`,
referenceKind: 'references',
line: node.startPosition.row + 1,
column: node.startPosition.column,
});
}
return true;
}
// Let the default walker descend into bodies/expressions; we only claim
// top-level blocks.
return false;
}
const header = readBlockHeader(node, ctx.source);
if (!header) return false;
const { type, labels } = header;
const body = getBlockBody(node);
// --- locals: every attribute becomes its own constant ---
if (type === 'locals' && labels.length === 0) {
emitLocals(body, ctx);
return true; // we handled everything inside this block
}
// --- terraform { ... } settings block — no symbols, no refs to project ---
if (type === 'terraform' && labels.length === 0) {
return true;
}
// --- resource / data / module / variable / output / provider ---
const decl = describeBlock(type, labels);
if (!decl) {
// Unknown top-level block (e.g. nested block hoisted as top-level via
// walker). Let the default walker continue.
return false;
}
const created = ctx.createNode(decl.kind, decl.name, node, {
qualifiedName: decl.qualifiedName,
signature: decl.signature,
isExported: decl.kind === 'variable',
});
if (!created) return true;
// Collect references inside this block's body (attribute expressions).
if (body) {
ctx.pushScope(created.id);
try {
emitReferencesInBody(body, ctx, created.id);
if (type === 'module' && labels[0]) {
emitModuleWiring(labels[0], node, body, ctx, created.id);
}
} finally {
ctx.popScope();
}
}
return true;
},
};
/**
* Module meta-arguments — attributes of a `module` block that configure the
* call itself rather than set one of the child module's input variables.
*/
const MODULE_META_ARGS = new Set(['source', 'version', 'count', 'for_each', 'providers', 'depends_on']);
/**
* Bridge a `module "M" { ... }` block across the module boundary with
* `:`-scoped references that only the Terraform framework resolver
* understands (a plain qualified name would let the generic matcher bind
* them to a same-named symbol in an unrelated module — a wrong edge is
* worse than none):
*
* - `module.M:file` (imports) → the module source directory's
* entry file, when `source` is a local `./`/`../` path. Registry and
* git sources emit nothing — an out-of-repo module stays a visible
* boundary instead of a guessed edge.
* - `module.M:var.<in>` (references) → the child module's
* `variable "<in>"` node, one per input attribute. This is what lets
* "what depends on modules/vpc's var.cidr" reach the callers.
*/
function emitModuleWiring(
moduleName: string,
block: SyntaxNode,
body: SyntaxNode,
ctx: Parameters<NonNullable<LanguageExtractor['visitNode']>>[1],
fromNodeId: string
): void {
for (const attr of body.namedChildren) {
if (!attr || attr.type !== 'attribute') continue;
const idNode = attr.namedChildren.find((c) => c?.type === 'identifier');
if (!idNode) continue;
const attrName = getNodeText(idNode, ctx.source);
if (attrName === 'source') {
const expr = attr.namedChildren.find((c) => c?.type === 'expression');
const lit = expr ? findStringLit(expr) : null;
const source = lit ? stringLitValue(lit, ctx.source) : '';
if (source.startsWith('./') || source.startsWith('../')) {
ctx.addUnresolvedReference({
fromNodeId,
referenceName: `module.${moduleName}:file`,
referenceKind: 'imports',
line: block.startPosition.row + 1,
column: block.startPosition.column,
});
}
continue;
}
if (MODULE_META_ARGS.has(attrName)) continue;
ctx.addUnresolvedReference({
fromNodeId,
referenceName: `module.${moduleName}:var.${attrName}`,
referenceKind: 'references',
line: attr.startPosition.row + 1,
column: attr.startPosition.column,
});
}
}
/** First string_lit anywhere under an expression (source = "./modules/x"). */
function findStringLit(expr: SyntaxNode): SyntaxNode | null {
const queue: SyntaxNode[] = [expr];
while (queue.length) {
const n = queue.shift()!;
if (n.type === 'string_lit') return n;
for (const c of n.namedChildren) {
if (c) queue.push(c);
}
}
return null;
}
interface BlockDecl {
kind: 'class' | 'module' | 'variable' | 'namespace';
name: string;
qualifiedName: string;
signature: string;
}
function describeBlock(type: string, labels: string[]): BlockDecl | null {
const [first, second] = labels;
switch (type) {
case 'resource': {
if (!first || !second) return null;
return {
kind: 'class',
name: `${first}.${second}`,
qualifiedName: `${first}.${second}`,
signature: `resource "${first}" "${second}"`,
};
}
case 'data': {
if (!first || !second) return null;
return {
kind: 'class',
name: `${first}.${second}`,
qualifiedName: `data.${first}.${second}`,
signature: `data "${first}" "${second}"`,
};
}
case 'module': {
if (!first) return null;
return {
kind: 'module',
name: first,
qualifiedName: `module.${first}`,
signature: `module "${first}"`,
};
}
case 'variable': {
if (!first) return null;
return {
kind: 'variable',
name: first,
qualifiedName: `var.${first}`,
signature: `variable "${first}"`,
};
}
case 'output': {
if (!first) return null;
return {
kind: 'variable',
name: first,
qualifiedName: `output.${first}`,
signature: `output "${first}"`,
};
}
case 'provider': {
if (!first) return null;
return {
kind: 'namespace',
name: first,
qualifiedName: `provider.${first}`,
signature: `provider "${first}"`,
};
}
default:
return null;
}
}
function emitLocals(
body: SyntaxNode | null,
ctx: Parameters<NonNullable<LanguageExtractor['visitNode']>>[1]
): void {
if (!body) return;
for (const attr of body.namedChildren) {
if (!attr || attr.type !== 'attribute') continue;
const idNode = attr.namedChildren.find((c) => c?.type === 'identifier');
if (!idNode) continue;
const name = getNodeText(idNode, ctx.source);
const created = ctx.createNode('constant', name, attr, {
qualifiedName: `local.${name}`,
signature: `local.${name}`,
});
if (!created) continue;
const expr = attr.namedChildren.find((c) => c?.type === 'expression');
if (expr) {
ctx.pushScope(created.id);
try {
collectReferences(expr, ctx.source, (qname, line, column) => {
ctx.addUnresolvedReference({
fromNodeId: created.id,
referenceName: qname,
referenceKind: 'references',
line,
column,
});
});
} finally {
ctx.popScope();
}
}
}
}
function emitReferencesInBody(
body: SyntaxNode,
ctx: Parameters<NonNullable<LanguageExtractor['visitNode']>>[1],
fromNodeId: string
): void {
const queue: SyntaxNode[] = [body];
while (queue.length) {
const n = queue.shift()!;
if (n.type === 'expression') {
collectReferences(n, ctx.source, (qname, line, column) => {
ctx.addUnresolvedReference({
fromNodeId,
referenceName: qname,
referenceKind: 'references',
line,
column,
});
});
// Don't descend into expression — collectReferences already does.
continue;
}
for (const c of n.namedChildren) {
if (c) queue.push(c);
}
}
}
Binary file not shown.