feat(kernel): R1 scaffold — napi-rs extraction kernel, buffer contract, routing + fallback, grammar-parity CI
Phase 0 of the Rust extraction-kernel migration (docs/design/ rust-kernel-migration-plan.md, now checked in with §3a recording the shipped state): - codegraph-kernel/ napi-rs crate: extractFile(path, content, language) → five flat buffers (meta/nodes/edges/refs/arena), one JS boundary crossing per file. Node ids computed Rust-side, byte-identical to generateNodeId (pinned by test vector). Reserved per-node metrics slot for the Arc 3.2 code-metrics work. - Generic .scm-driven emitter (@def.<kind>/@name/@ref.<kind> captures, byte-range scope stack → ::-joined qualified names, contains edges, refs attributed to the innermost enclosing symbol). Seed TS/JS queries are smoke-level; R2 replaces them with the full port. - Routing seam in extractFromSource with per-file wasm fallback. DEFAULT_ROUTED is empty — no behavior change until a language passes its equivalence gate (R3). Dev opt-in: CODEGRAPH_KERNEL_LANGS. Kill switch: CODEGRAPH_KERNEL=0. Loader verifies ABI + kind tables before routing; EDGE_KINDS became a runtime array because kind order is now wire contract. - Grammar-source parity: vendored TS/TSX/JS wasm grammars built from the exact crate revisions (tree-sitter-typescript v0.23.2, tree-sitter-javascript v0.25.0, checked-in parser.c, ts-cli 0.25.10) — the tree-sitter-wasms builds were 2023-era, which the new kernel-grammar-parity test caught on day one. Production TS/JS parsing gets 2.5 years of grammar fixes; full suite green (2456 tests). - Build/release wiring: scripts/build-kernel.sh + npm run build:kernel; release.yml kernel prebuild matrix (continue-on-error — the kernel is optional everywhere, bundles fall back to the wasm path); bundles stage lib/kernel/codegraph-kernel.node; release job runs the kernel suites with CODEGRAPH_KERNEL_EXPECT=1. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Colby McHenry
co-authored by
Claude Fable 5
Claude Fable 5
parent
4efc6c70e2
commit
c5eebe6beb
@@ -68,6 +68,26 @@ echo "[bundle] installing production dependencies"
|
||||
( cd "$STAGE/lib" && npm ci --omit=dev --ignore-scripts >/dev/null 2>&1 )
|
||||
rm -f "$STAGE/lib/package-lock.json"
|
||||
|
||||
# 3b. Native extraction kernel (optional). Included when a prebuilt .node for
|
||||
# the target exists — release/kernel/<target>/codegraph-kernel.node (the
|
||||
# release workflow's prebuild artifacts) or the locally staged
|
||||
# codegraph-kernel/prebuilds/<target>/ (scripts/build-kernel.sh). Absent →
|
||||
# the bundle simply runs the wasm extraction path; the kernel is a
|
||||
# per-language speedup, never a requirement (see
|
||||
# docs/design/rust-kernel-migration-plan.md).
|
||||
KERNEL_NODE=""
|
||||
for candidate in "$ROOT/release/kernel/${TARGET}/codegraph-kernel.node" \
|
||||
"$ROOT/codegraph-kernel/prebuilds/${TARGET}/codegraph-kernel.node"; do
|
||||
if [ -f "$candidate" ]; then KERNEL_NODE="$candidate"; break; fi
|
||||
done
|
||||
if [ -n "$KERNEL_NODE" ]; then
|
||||
mkdir -p "$STAGE/lib/kernel"
|
||||
cp "$KERNEL_NODE" "$STAGE/lib/kernel/codegraph-kernel.node"
|
||||
echo "[bundle] native kernel included ($KERNEL_NODE)"
|
||||
else
|
||||
echo "[bundle] no native kernel for ${TARGET} — bundle uses the wasm extraction path"
|
||||
fi
|
||||
|
||||
# 4. Vendored Node + launcher (the launcher uses the bundled Node by relative
|
||||
# path, so no system Node is ever needed).
|
||||
#
|
||||
|
||||
Executable
+82
@@ -0,0 +1,82 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Build the native extraction kernel (codegraph-kernel) and stage the .node
|
||||
# where the TS loader (src/extraction/kernel/loader.ts) finds it for
|
||||
# from-source runs and tests:
|
||||
#
|
||||
# codegraph-kernel/prebuilds/<platform>-<arch>/codegraph-kernel.node
|
||||
#
|
||||
# The kernel is OPTIONAL everywhere: when the .node is absent the extraction
|
||||
# path falls back to the wasm pipeline. This script needs a Rust toolchain
|
||||
# (rustup.rs); nothing else in the repo does.
|
||||
#
|
||||
# Usage:
|
||||
# scripts/build-kernel.sh # host platform
|
||||
# scripts/build-kernel.sh --target <rust-triple> [--platform <plat-arch>]
|
||||
#
|
||||
# The cross-compile form is what the release workflow uses (e.g.
|
||||
# --target x86_64-apple-darwin --platform darwin-x64 on a macos-arm runner).
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
CRATE="$ROOT/codegraph-kernel"
|
||||
|
||||
TARGET=""
|
||||
PLATFORM=""
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--target) TARGET="$2"; shift 2 ;;
|
||||
--platform) PLATFORM="$2"; shift 2 ;;
|
||||
*) echo "unknown arg: $1" >&2; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Map a rust triple (or the host) to the bundle-target naming used across the
|
||||
# release pipeline (darwin-arm64, linux-x64, win32-arm64, ...).
|
||||
if [ -z "$PLATFORM" ]; then
|
||||
if [ -n "$TARGET" ]; then
|
||||
case "$TARGET" in
|
||||
aarch64-apple-darwin) PLATFORM="darwin-arm64" ;;
|
||||
x86_64-apple-darwin) PLATFORM="darwin-x64" ;;
|
||||
x86_64-unknown-linux-gnu) PLATFORM="linux-x64" ;;
|
||||
aarch64-unknown-linux-gnu) PLATFORM="linux-arm64" ;;
|
||||
x86_64-pc-windows-msvc) PLATFORM="win32-x64" ;;
|
||||
aarch64-pc-windows-msvc) PLATFORM="win32-arm64" ;;
|
||||
*) echo "cannot map rust target '$TARGET' to a platform name; pass --platform" >&2; exit 1 ;;
|
||||
esac
|
||||
else
|
||||
case "$(uname -s)-$(uname -m)" in
|
||||
Darwin-arm64) PLATFORM="darwin-arm64" ;;
|
||||
Darwin-x86_64) PLATFORM="darwin-x64" ;;
|
||||
Linux-x86_64) PLATFORM="linux-x64" ;;
|
||||
Linux-aarch64) PLATFORM="linux-arm64" ;;
|
||||
MINGW*-x86_64|MSYS*-x86_64) PLATFORM="win32-x64" ;;
|
||||
MINGW*-aarch64|MSYS*-aarch64) PLATFORM="win32-arm64" ;;
|
||||
*) echo "unrecognized host $(uname -s)-$(uname -m); pass --platform" >&2; exit 1 ;;
|
||||
esac
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "[kernel] building codegraph-kernel for ${PLATFORM}${TARGET:+ (target $TARGET)}"
|
||||
cd "$CRATE"
|
||||
if [ -n "$TARGET" ]; then
|
||||
rustup target add "$TARGET" >/dev/null 2>&1 || true
|
||||
cargo build --release --target "$TARGET"
|
||||
OUTDIR="$CRATE/target/$TARGET/release"
|
||||
else
|
||||
cargo build --release
|
||||
OUTDIR="$CRATE/target/release"
|
||||
fi
|
||||
|
||||
# cdylib name differs per OS; the staged name is always codegraph-kernel.node.
|
||||
case "$PLATFORM" in
|
||||
darwin-*) LIB="$OUTDIR/libcodegraph_kernel.dylib" ;;
|
||||
linux-*) LIB="$OUTDIR/libcodegraph_kernel.so" ;;
|
||||
win32-*) LIB="$OUTDIR/codegraph_kernel.dll" ;;
|
||||
esac
|
||||
[ -f "$LIB" ] || { echo "[kernel] error: built library not found at $LIB" >&2; exit 1; }
|
||||
|
||||
DEST="$CRATE/prebuilds/$PLATFORM"
|
||||
mkdir -p "$DEST"
|
||||
cp "$LIB" "$DEST/codegraph-kernel.node"
|
||||
echo "[kernel] staged $DEST/codegraph-kernel.node ($(du -h "$DEST/codegraph-kernel.node" | cut -f1))"
|
||||
Reference in New Issue
Block a user