The viewer ran a second highlighter over source the engine had already parsed
with a real grammar: Shiki, plus 56 pruned TextMate grammars shipped in
dist/textmate/. The classification now comes off that tree instead, so a file is
read by exactly the grammar that decided what its symbols are.
The swap is complete rather than flagged: @shikijs/core, @shikijs/engine-javascript
and @shikijs/langs are off the dependency list, scripts/prune-grammars.mjs and
`npm run build:textmate` are deleted, and check-ui-build.mjs asserts the
tree-sitter grammars in dist/extraction/wasm instead of dist/textmate.
The wire contract is unchanged — `[classId, text]` pairs with the class names
alongside — so the viewer's decoder and code blocks did not have to be rewritten.
Two classes are added to the six: `type` (a named type reference, painted at
plain ink) and `def` (the name a definition declares, weight 600), the latter
taken from the extractors' own definition tables so it cannot drift from what
indexing calls a definition.
Three differences are not cosmetic:
* Interpolations (`${…}`, `#{…}`, `$"{…}"`, f-strings) are classified as code,
not as string. The call-site overlay refuses to claim a token classed string,
so calls written inside interpolated strings now link.
* Built-in type words are emitted whole and classed `type` in every language.
The grammars disagree about whether `string` is a type_identifier or an
anonymous token inside a predefined_type, and TextMate scoped them
inconsistently too.
* 3 000 lines of TypeScript cost 24-41 ms instead of ~700 ms.
Given up deliberately: Liquid, Razor, YAML, Twig, XML and .properties render
plain. .svelte/.vue/.astro are classified through their <script> blocks, the same
delegation the SFC extractors do. Pulling html/css/vue out of tree-sitter-wasms
would cover them, but those ABI-13 builds are the known cause of shared-WASM-heap
corruption for every other language in the same process.
Measured parity, per-language before/after screenshots and the reproduction
recipe: docs/design/cg57-highlighting-parity.md.
158 lines
6.1 KiB
JavaScript
158 lines
6.1 KiB
JavaScript
#!/usr/bin/env node
|
|
/**
|
|
* Assert that the browser viewer actually built.
|
|
*
|
|
* `codegraph ui` serves dist/viewer/ as static files. If that tree is missing
|
|
* or half-written, the CLI still starts and the browser gets a 404 — a failure
|
|
* that would otherwise surface after the release is published. So the build
|
|
* fails here instead: index.html must exist, be non-trivial, and every local
|
|
* asset it references must be on disk next to it.
|
|
*
|
|
* It also re-asserts that the compiled engine is still there. The viewer build
|
|
* empties its own output directory, and `dist/ui/` — the obvious name — is
|
|
* where tsc puts the TERMINAL ui, so a mis-pointed outDir silently deletes
|
|
* modules the CLI requires at startup.
|
|
*
|
|
* The tree-sitter grammars in dist/extraction/wasm/ are checked the same way
|
|
* and for the same reason. They are copied by `npm run copy-assets`, they are
|
|
* what both indexing and the viewer's syntax classification parse with, and
|
|
* their absence is survivable at runtime — source is served unhighlighted —
|
|
* which is exactly why it has to fail here: nothing downstream would complain.
|
|
*
|
|
* Usage: node scripts/check-ui-build.mjs [--root <dir>]
|
|
* --root directory holding dist/ (default: the repo root). The release
|
|
* bundler points this at its staging dir to verify the copy.
|
|
*/
|
|
import { existsSync, readFileSync, readdirSync, statSync } from 'node:fs';
|
|
import { dirname, join, resolve, sep } from 'node:path';
|
|
import { fileURLToPath } from 'node:url';
|
|
|
|
const argv = process.argv.slice(2);
|
|
const rootFlag = argv.indexOf('--root');
|
|
const staged = rootFlag >= 0 && Boolean(argv[rootFlag + 1]);
|
|
const root = staged
|
|
? resolve(argv[rootFlag + 1])
|
|
: resolve(dirname(fileURLToPath(import.meta.url)), '..');
|
|
|
|
const viewerDir = join(root, 'dist', 'viewer');
|
|
const indexHtml = join(viewerDir, 'index.html');
|
|
|
|
function fail(message, hint) {
|
|
console.error(`[check-ui-build] ${message}`);
|
|
if (hint) console.error(`[check-ui-build] ${hint}`);
|
|
process.exit(1);
|
|
}
|
|
|
|
if (!existsSync(indexHtml)) {
|
|
fail(
|
|
`missing ${indexHtml}`,
|
|
staged
|
|
? 'this bundle predates the UI or was assembled from a stale archive — rebuild it with scripts/build-bundle.sh'
|
|
: 'the UI workspace did not build — run `npm run build:ui` (or `npm ci` if ui/ has no node_modules)'
|
|
);
|
|
}
|
|
|
|
const html = readFileSync(indexHtml, 'utf8');
|
|
if (html.length < 200 || !/<div id="app">/.test(html)) {
|
|
fail(`${indexHtml} does not look like the built viewer (${html.length} bytes)`);
|
|
}
|
|
|
|
// Every local src=/href= in the document must resolve inside dist/ui. This is
|
|
// what catches a partial write: index.html naming a hashed bundle that the
|
|
// build never emitted.
|
|
const referenced = [...html.matchAll(/\s(?:src|href)="([^"]+)"/g)].map((m) => m[1]);
|
|
const local = referenced.filter(
|
|
(url) => !/^(?:[a-z][a-z0-9+.-]*:|\/\/)/i.test(url) && !url.startsWith('#')
|
|
);
|
|
|
|
const missing = [];
|
|
let assets = 0;
|
|
for (const url of local) {
|
|
const rel = url.replace(/^\.\//, '').replace(/[?#].*$/, '');
|
|
if (!rel) continue;
|
|
const onDisk = join(viewerDir, ...rel.split('/'));
|
|
if (!existsSync(onDisk) || !statSync(onDisk).isFile()) missing.push(rel);
|
|
else assets += 1;
|
|
}
|
|
|
|
if (missing.length > 0) {
|
|
fail(
|
|
`index.html references ${missing.length} file(s) that are not in dist/viewer: ${missing.join(', ')}`,
|
|
'the UI build was interrupted or dist/viewer was copied incompletely'
|
|
);
|
|
}
|
|
|
|
if (assets === 0) {
|
|
fail('index.html references no bundled assets — the UI build produced no JS/CSS');
|
|
}
|
|
|
|
// The viewer build must never have eaten the tsc output next door.
|
|
for (const compiled of [join('bin', 'codegraph.js'), 'index.js', join('ui', 'shimmer-progress.js')]) {
|
|
if (!existsSync(join(root, 'dist', compiled))) {
|
|
fail(
|
|
`dist/${compiled.split(sep).join('/')} is missing — the compiled engine is incomplete`,
|
|
"if this appeared with a UI change, check ui/vite.config.ts: build.outDir must stay dist/viewer, and emptyOutDir must never point at a directory tsc writes (dist/ui is the TERMINAL ui)"
|
|
);
|
|
}
|
|
}
|
|
|
|
// The vendored tree-sitter grammars (`npm run copy-assets`). The viewer reads
|
|
// every file with the same grammar the engine indexed it with, so a missing
|
|
// wasm is both an extraction gap and a silently unhighlighted screen.
|
|
const wasmDir = join(root, 'dist', 'extraction', 'wasm');
|
|
|
|
/**
|
|
* The grammars the syntax classification is gated on — the eight languages
|
|
* CG-57 measured parity against, plus the two the TS family needs. Every one is
|
|
* vendored (see VENDORED_WASM_LANGS), so all of them must be in this directory
|
|
* rather than resolved out of node_modules.
|
|
*/
|
|
const GATE_GRAMMARS = [
|
|
'tree-sitter-typescript.wasm',
|
|
'tree-sitter-tsx.wasm',
|
|
'tree-sitter-javascript.wasm',
|
|
'tree-sitter-go.wasm',
|
|
'tree-sitter-python.wasm',
|
|
'tree-sitter-rust.wasm',
|
|
'tree-sitter-swift.wasm',
|
|
'tree-sitter-c_sharp.wasm',
|
|
'tree-sitter-ruby.wasm',
|
|
'tree-sitter-php.wasm',
|
|
];
|
|
|
|
if (!existsSync(wasmDir)) {
|
|
fail(
|
|
`missing ${wasmDir}`,
|
|
staged
|
|
? 'dist/extraction/wasm was not copied into the bundle — re-run scripts/build-bundle.sh'
|
|
: 'run `npm run copy-assets` (it copies src/extraction/wasm/*.wasm into dist/)'
|
|
);
|
|
}
|
|
|
|
// Against the source tree, the source directory IS the list — nothing to drift.
|
|
// Inside a staged bundle there is no src/, so the gate list carries it.
|
|
const expectedGrammars = new Set(GATE_GRAMMARS);
|
|
const srcWasmDir = join(root, 'src', 'extraction', 'wasm');
|
|
if (!staged && existsSync(srcWasmDir)) {
|
|
for (const name of readdirSync(srcWasmDir)) {
|
|
if (name.endsWith('.wasm')) expectedGrammars.add(name);
|
|
}
|
|
}
|
|
|
|
const missingGrammars = [...expectedGrammars].filter(
|
|
(name) => !existsSync(join(wasmDir, name))
|
|
);
|
|
if (missingGrammars.length > 0) {
|
|
fail(
|
|
`dist/extraction/wasm is missing ${missingGrammars.length} grammar(s): ${missingGrammars.join(', ')}`,
|
|
'the copy-assets step was interrupted or dist/extraction/wasm was copied incompletely'
|
|
);
|
|
}
|
|
|
|
const grammarCount = readdirSync(wasmDir).filter((n) => n.endsWith('.wasm')).length;
|
|
|
|
console.log(
|
|
`[check-ui-build] dist/viewer ok (index.html + ${assets} referenced asset(s)); ` +
|
|
`dist/extraction/wasm ok (${grammarCount} grammars); dist/ engine intact`
|
|
);
|