feat(ui): highlight source server-side with a near-monochrome Shiki theme (CG-43)
The viewer's code block stops lexing with a hand-rolled dialect table and reads real TextMate grammars instead, run once in `/api/source`. Three things make that safe to depend on: * Highlighting never fails a request. A missing grammar, an oversized slice, an ESM import that did not resolve — every one of them answers `engine: 'plain'` with a reason and the source still goes out. * Identifiers survive whatever token boundaries a grammar chose. Every code token is split into identifier runs before it goes on the wire, so the graph's call-site overlay claims a token the highlighter produced rather than re-cutting the line. `assignRefs` now matches on a token's text rather than on the class a grammar gave it, so a language that scopes type names as `storage.type` still links. * The theme classifies rather than colours: its foregrounds are sentinels the server maps back to class names, and the viewer paints them from CSS custom properties — one token stream serves light and dark with no refetch, and the ramp lives only in app.css. Comments move from --ink-3 to a new --code-comment. --ink-3 measures 3.46:1 on paper and 3.00:1 on the hot-line tint, both under AA for 12.5px text; --code-comment is the smallest step along the same ramp that clears 4.5:1 on every background a code line can have, and stays quieter than the strings and numbers above it. Shipping: @shikijs/core and @shikijs/engine-javascript are runtime dependencies (no wasm, no native module); @shikijs/langs stays a devDependency and `npm run build:textmate` writes only the closure the engine's 40-odd languages reach — 56 grammars, 2.6 MB, against 11 MB for all 722. check-ui-build.mjs asserts the tree after every build and inside every release archive.
This commit is contained in:
Colby McHenry
@@ -13,6 +13,11 @@
|
||||
* where tsc puts the TERMINAL ui, so a mis-pointed outDir silently deletes
|
||||
* modules the CLI requires at startup.
|
||||
*
|
||||
* The pruned TextMate grammars in dist/textmate/ are checked the same way and
|
||||
* for the same reason: without them every file the viewer shows falls back to
|
||||
* unhighlighted text, which looks like a styling bug rather than a missing
|
||||
* build step.
|
||||
*
|
||||
* Usage: node scripts/check-ui-build.mjs [--root <dir>]
|
||||
* --root directory holding dist/ (default: the repo root). The release
|
||||
* bundler points this at its staging dir to verify the copy.
|
||||
@@ -90,4 +95,36 @@ for (const compiled of [join('bin', 'codegraph.js'), 'index.js', join('ui', 'shi
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`[check-ui-build] dist/viewer ok (index.html + ${assets} referenced asset(s)); dist/ engine intact`);
|
||||
// The pruned syntax grammars (scripts/prune-grammars.mjs). Their absence is
|
||||
// survivable at runtime — source is served unhighlighted — which is exactly why
|
||||
// it has to fail here: nothing downstream would ever complain.
|
||||
const textmateDir = join(root, 'dist', 'textmate');
|
||||
const manifestPath = join(textmateDir, 'manifest.json');
|
||||
if (!existsSync(manifestPath)) {
|
||||
fail(
|
||||
`missing ${manifestPath}`,
|
||||
staged
|
||||
? 'this bundle was assembled before the syntax grammars were added, or dist/textmate was not copied'
|
||||
: 'run `npm run build:textmate` (it needs @shikijs/langs from devDependencies)'
|
||||
);
|
||||
}
|
||||
|
||||
const manifest = JSON.parse(readFileSync(manifestPath, 'utf8'));
|
||||
const languages = Object.keys(manifest.languages ?? {});
|
||||
if (languages.length === 0) fail('dist/textmate/manifest.json lists no languages');
|
||||
|
||||
const grammarFiles = new Set(Object.values(manifest.languages).flat());
|
||||
const missingGrammars = [...grammarFiles].filter(
|
||||
(name) => !existsSync(join(textmateDir, `${name}.json`))
|
||||
);
|
||||
if (missingGrammars.length > 0) {
|
||||
fail(
|
||||
`dist/textmate is missing ${missingGrammars.length} grammar file(s): ${missingGrammars.join(', ')}`,
|
||||
'the prune step was interrupted or dist/textmate was copied incompletely'
|
||||
);
|
||||
}
|
||||
|
||||
console.log(
|
||||
`[check-ui-build] dist/viewer ok (index.html + ${assets} referenced asset(s)); ` +
|
||||
`dist/textmate ok (${languages.length} languages, ${grammarFiles.size} grammars); dist/ engine intact`
|
||||
);
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Write the TextMate grammars the viewer needs into `dist/textmate/`.
|
||||
*
|
||||
* Shiki carries about 700 grammars, 11 MB of JSON. The engine indexes about 40
|
||||
* languages. Shipping the other 660 to every user of a code-intelligence CLI is
|
||||
* not a trade worth making, so `@shikijs/langs` stays a devDependency and this
|
||||
* step copies out exactly the closure the viewer can reach: every grammar named
|
||||
* in `src/ui-server/highlight/languages.ts`, plus every grammar those embed
|
||||
* (`vue` needs html, css, typescript, json and four Vue-specific ones before it
|
||||
* will highlight a single-file component).
|
||||
*
|
||||
* Run from `npm run build`, after `tsc`, because the language table is read
|
||||
* from the compiled `dist/ui-server/highlight/languages.js` rather than being
|
||||
* duplicated here — one source of truth for what ships.
|
||||
*
|
||||
* Output:
|
||||
* dist/textmate/manifest.json grammar id -> files to load, deps first
|
||||
* dist/textmate/<name>.json one TextMate grammar, verbatim
|
||||
*/
|
||||
|
||||
import { createRequire } from 'node:module';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
|
||||
const OUT = path.join(ROOT, 'dist', 'textmate');
|
||||
const require = createRequire(import.meta.url);
|
||||
|
||||
function fail(message) {
|
||||
process.stderr.write(`[prune-grammars] ${message}\n`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const languagesModule = path.join(ROOT, 'dist', 'ui-server', 'highlight', 'languages.js');
|
||||
if (!fs.existsSync(languagesModule)) {
|
||||
fail(`${path.relative(ROOT, languagesModule)} is missing — run tsc before this script.`);
|
||||
}
|
||||
const { REQUIRED_GRAMMARS } = require(languagesModule);
|
||||
if (!Array.isArray(REQUIRED_GRAMMARS) || REQUIRED_GRAMMARS.length === 0) {
|
||||
fail('REQUIRED_GRAMMARS is empty — the language table did not compile as expected.');
|
||||
}
|
||||
|
||||
const shikiVersion = JSON.parse(
|
||||
fs.readFileSync(path.join(ROOT, 'node_modules', '@shikijs', 'langs', 'package.json'), 'utf-8')
|
||||
).version;
|
||||
|
||||
/**
|
||||
* Load one Shiki language module and return its registrations.
|
||||
*
|
||||
* The default export is already the flattened chain — embedded grammars first,
|
||||
* the language itself last — which is exactly the order Shiki's registry needs
|
||||
* to resolve `embeddedLangs`. Keeping that order is the whole reason the
|
||||
* manifest stores a list rather than a single filename.
|
||||
*/
|
||||
async function loadChain(id) {
|
||||
const mod = await import(`@shikijs/langs/${id}`);
|
||||
const chain = mod.default;
|
||||
if (!Array.isArray(chain) || chain.length === 0) {
|
||||
fail(`@shikijs/langs/${id} did not export a grammar array.`);
|
||||
}
|
||||
return chain;
|
||||
}
|
||||
|
||||
fs.rmSync(OUT, { recursive: true, force: true });
|
||||
fs.mkdirSync(OUT, { recursive: true });
|
||||
|
||||
const manifest = { shikiVersion, languages: {} };
|
||||
const written = new Map();
|
||||
let bytes = 0;
|
||||
|
||||
for (const id of REQUIRED_GRAMMARS) {
|
||||
let chain;
|
||||
try {
|
||||
chain = await loadChain(id);
|
||||
} catch (err) {
|
||||
fail(`could not load the ${id} grammar: ${err?.message ?? err}`);
|
||||
}
|
||||
|
||||
const files = [];
|
||||
for (const grammar of chain) {
|
||||
// A chain can name the same dependency more than once (Vue reaches
|
||||
// JavaScript four different ways). Registering it twice is wasted work and
|
||||
// a confusing manifest; the FIRST occurrence is the one that keeps the
|
||||
// dependencies-before-dependents ordering intact.
|
||||
// `name` is the grammar's own id and is unique across the bundle, so two
|
||||
// languages that embed html write (and share) exactly one html.json.
|
||||
const file = grammar.name;
|
||||
if (typeof file !== 'string' || !/^[\w.+-]+$/.test(file)) {
|
||||
fail(`the ${id} chain contains a grammar with an unusable name: ${JSON.stringify(file)}`);
|
||||
}
|
||||
if (files.includes(file)) continue;
|
||||
if (!written.has(file)) {
|
||||
const json = JSON.stringify(grammar);
|
||||
fs.writeFileSync(path.join(OUT, `${file}.json`), json);
|
||||
written.set(file, json.length);
|
||||
bytes += json.length;
|
||||
}
|
||||
files.push(file);
|
||||
}
|
||||
manifest.languages[id] = files;
|
||||
}
|
||||
|
||||
fs.writeFileSync(path.join(OUT, 'manifest.json'), JSON.stringify(manifest, null, 2));
|
||||
|
||||
process.stdout.write(
|
||||
`[prune-grammars] ${REQUIRED_GRAMMARS.length} languages -> ${written.size} grammars, ` +
|
||||
`${(bytes / 1024 / 1024).toFixed(1)} MB in dist/textmate (shiki ${shikiVersion})\n`
|
||||
);
|
||||
Reference in New Issue
Block a user