feat(ui): highlight source server-side with a near-monochrome Shiki theme (CG-43)
The viewer's code block stops lexing with a hand-rolled dialect table and reads real TextMate grammars instead, run once in `/api/source`. Three things make that safe to depend on: * Highlighting never fails a request. A missing grammar, an oversized slice, an ESM import that did not resolve — every one of them answers `engine: 'plain'` with a reason and the source still goes out. * Identifiers survive whatever token boundaries a grammar chose. Every code token is split into identifier runs before it goes on the wire, so the graph's call-site overlay claims a token the highlighter produced rather than re-cutting the line. `assignRefs` now matches on a token's text rather than on the class a grammar gave it, so a language that scopes type names as `storage.type` still links. * The theme classifies rather than colours: its foregrounds are sentinels the server maps back to class names, and the viewer paints them from CSS custom properties — one token stream serves light and dark with no refetch, and the ramp lives only in app.css. Comments move from --ink-3 to a new --code-comment. --ink-3 measures 3.46:1 on paper and 3.00:1 on the hot-line tint, both under AA for 12.5px text; --code-comment is the smallest step along the same ramp that clears 4.5:1 on every background a code line can have, and stays quieter than the strings and numbers above it. Shipping: @shikijs/core and @shikijs/engine-javascript are runtime dependencies (no wasm, no native module); @shikijs/langs stays a devDependency and `npm run build:textmate` writes only the closure the engine's 40-odd languages reach — 56 grammars, 2.6 MB, against 11 MB for all 722. check-ui-build.mjs asserts the tree after every build and inside every release archive.
This commit is contained in:
Colby McHenry
@@ -87,7 +87,9 @@ const API_INDEX = {
|
||||
export function createGraphApi(options: GraphApiOptions): GraphApi {
|
||||
const session = new GraphSession(options.projectRoot);
|
||||
|
||||
const handler: UiApiHandler = (_req, res, ctx) => {
|
||||
// Async because `/api/source` highlights: everything else answers straight
|
||||
// out of SQLite and resolves on the same tick.
|
||||
const handler: UiApiHandler = async (_req, res, ctx) => {
|
||||
const route = normalize(ctx.pathname);
|
||||
try {
|
||||
switch (route) {
|
||||
@@ -104,7 +106,7 @@ export function createGraphApi(options: GraphApiOptions): GraphApi {
|
||||
case '/api/nodes':
|
||||
return ok(res, buildNodeRefs(session.acquire(), ctx.query), ctx.method);
|
||||
case '/api/source':
|
||||
return ok(res, buildSource(session.acquire(), ctx.projectRoot, ctx.query), ctx.method);
|
||||
return ok(res, await buildSource(session.acquire(), ctx.projectRoot, ctx.query), ctx.method);
|
||||
default:
|
||||
return dispatchPathRoutes(route, res, ctx, session);
|
||||
}
|
||||
|
||||
@@ -30,6 +30,7 @@ import * as path from 'path';
|
||||
import type { FileRecord } from '../../types';
|
||||
import type { CodeGraph } from '../../index';
|
||||
import { resolveProjectFile } from '../security';
|
||||
import { highlightLines, type HighlightResult } from '../highlight';
|
||||
import { ApiError, badRequest, intParam, notFound, textParam } from './respond';
|
||||
|
||||
/**
|
||||
@@ -186,13 +187,23 @@ export interface SourceResult {
|
||||
lines?: string[];
|
||||
truncated?: boolean;
|
||||
reason?: string;
|
||||
/**
|
||||
* The same lines, classified for the code block — one entry per line, each a
|
||||
* list of `[classId, text]` pairs indexed into `highlight.classes`.
|
||||
*
|
||||
* It rides with the slice rather than living behind its own endpoint because
|
||||
* the two are only ever wanted together, and because a second round-trip
|
||||
* would let the viewer paint unhighlighted source and then reflow it. Absent
|
||||
* whenever `lines` is — a drifted file is not served at all.
|
||||
*/
|
||||
highlight?: HighlightResult;
|
||||
}
|
||||
|
||||
export function buildSource(
|
||||
export async function buildSource(
|
||||
cg: CodeGraph,
|
||||
projectRoot: string,
|
||||
query: URLSearchParams
|
||||
): SourceResult {
|
||||
): Promise<SourceResult> {
|
||||
const requested = textParam(query, 'file');
|
||||
// Refusal first, index lookup second — see `resolveRequestedFile`.
|
||||
const { record, storedPath, absolute } = resolveRequestedFile(cg, projectRoot, requested);
|
||||
@@ -265,13 +276,21 @@ export function buildSource(
|
||||
const start = from;
|
||||
const requestedEnd = to === 0 ? all.length : Math.min(to, all.length);
|
||||
const end = Math.min(requestedEnd, start + MAX_SOURCE_LINES - 1);
|
||||
const slice = all.slice(start - 1, end);
|
||||
|
||||
return {
|
||||
...base,
|
||||
totalLines: all.length,
|
||||
from: start,
|
||||
to: end,
|
||||
lines: all.slice(start - 1, end),
|
||||
lines: slice,
|
||||
truncated: end < requestedEnd,
|
||||
// Keyed on the content hash, so the cache is invalidated by the file
|
||||
// changing rather than by a clock, and two viewers looking at the same
|
||||
// symbol share one tokenisation.
|
||||
highlight: await highlightLines(slice, {
|
||||
language: record.language,
|
||||
cacheKey: `${record.contentHash}:${start}:${end}`,
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
/**
|
||||
* Finding and reading the pruned TextMate grammars on disk.
|
||||
*
|
||||
* Shiki ships 700-odd grammars; the engine indexes 40-odd languages. The build
|
||||
* writes only the closure those 40 need — including the grammars they embed, so
|
||||
* a `.vue` file still gets its `<script lang="ts">` — into `dist/textmate/`,
|
||||
* and `@shikijs/langs` stays a devDependency that never reaches a user's disk.
|
||||
* See `scripts/prune-grammars.mjs`.
|
||||
*
|
||||
* They are located the way `db/index.ts` finds `schema.sql` and `assets.ts`
|
||||
* finds the viewer: relative to `__dirname`, never to `process.cwd()`, which is
|
||||
* whatever directory the user happened to be standing in.
|
||||
*
|
||||
* `dist/textmate`, not `dist/highlight` — `src/ui-server/highlight/` is this
|
||||
* module and tsc already owns `dist/ui-server/highlight/`. The same collision
|
||||
* that put the viewer in `dist/viewer` rather than `dist/ui`.
|
||||
*/
|
||||
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
|
||||
/** Overrides where the grammars are read from. For tests and for packagers. */
|
||||
export const TEXTMATE_PATH_ENV = 'CODEGRAPH_TEXTMATE_PATH';
|
||||
|
||||
/** What `scripts/prune-grammars.mjs` writes beside the grammar files. */
|
||||
export interface GrammarManifest {
|
||||
/** Shiki version the grammars were pruned from — surfaced when one fails. */
|
||||
shikiVersion: string;
|
||||
/** Grammar id → the files to load, dependencies first, the grammar itself last. */
|
||||
languages: Record<string, string[]>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Candidate locations, most-specific first.
|
||||
*
|
||||
* 1. The `CODEGRAPH_TEXTMATE_PATH` override.
|
||||
* 2. `<__dirname>/../../textmate` — the shipped layout
|
||||
* (`dist/ui-server/highlight/` → `dist/textmate/`).
|
||||
* 3. `<__dirname>/../../../dist/textmate` — running the TypeScript straight out
|
||||
* of `src/` (vitest, tsx), where `__dirname` is `src/ui-server/highlight/`.
|
||||
*/
|
||||
export function grammarDirCandidates(): string[] {
|
||||
const override = process.env[TEXTMATE_PATH_ENV]?.trim();
|
||||
const candidates = [
|
||||
path.join(__dirname, '..', '..', 'textmate'),
|
||||
path.join(__dirname, '..', '..', '..', 'dist', 'textmate'),
|
||||
];
|
||||
return override ? [path.resolve(override), ...candidates] : candidates;
|
||||
}
|
||||
|
||||
/**
|
||||
* The grammar directory and its manifest, or null when the build did not run.
|
||||
*
|
||||
* Null is a normal outcome, not an error: a source checkout that has only had
|
||||
* `tsc` run against it has no `dist/textmate`, and the right answer there is
|
||||
* plain text, not a 500 on a request for source.
|
||||
*/
|
||||
export function loadManifest(): { dir: string; manifest: GrammarManifest } | null {
|
||||
for (const dir of grammarDirCandidates()) {
|
||||
try {
|
||||
const raw = fs.readFileSync(path.join(dir, 'manifest.json'), 'utf-8');
|
||||
const manifest = JSON.parse(raw) as GrammarManifest;
|
||||
if (manifest && typeof manifest === 'object' && manifest.languages) return { dir, manifest };
|
||||
} catch {
|
||||
// Not here — try the next candidate.
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the grammar registrations one language needs, dependencies first.
|
||||
*
|
||||
* Shiki resolves a grammar's `embeddedLangs` against what is already in its
|
||||
* registry, so the order the manifest records matters: `vue` must arrive after
|
||||
* the `html`, `css` and `typescript` it embeds, or the embedded blocks come
|
||||
* back unhighlighted.
|
||||
*/
|
||||
export function readGrammarChain(dir: string, manifest: GrammarManifest, id: string): unknown[] {
|
||||
const files = manifest.languages[id];
|
||||
if (!files) return [];
|
||||
return files.map((file) => {
|
||||
const full = path.join(dir, `${file}.json`);
|
||||
return JSON.parse(fs.readFileSync(full, 'utf-8')) as unknown;
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,407 @@
|
||||
/**
|
||||
* Server-side syntax classification for the viewer's code block (CG-43).
|
||||
*
|
||||
* The viewer used to lex on the client with a hand-rolled dialect table. This
|
||||
* replaces it with real TextMate grammars, run once here, so a Go file reads as
|
||||
* Go rather than as "something with braces". Three properties keep that from
|
||||
* becoming a liability:
|
||||
*
|
||||
* * **It never fails a request.** A missing grammar, a corrupt grammar file, an
|
||||
* ESM import that did not resolve, a slice too big to be worth tokenising —
|
||||
* every one of them answers `engine: 'plain'` with a reason and the source
|
||||
* still goes out. Highlighting is the part that degrades; nothing else does.
|
||||
* * **Identifiers survive whatever token boundaries the grammar chose.** Every
|
||||
* code token is split into identifier runs before it goes on the wire, which
|
||||
* is what lets the viewer wrap a call site as a link by *claiming a token*
|
||||
* rather than re-tokenising the line on top of the highlighter's answer.
|
||||
* * **The classification is a class name, not a colour.** See `theme.ts` — the
|
||||
* viewer paints from CSS custom properties, so one token stream serves light
|
||||
* and dark and the design tokens live in exactly one place.
|
||||
*
|
||||
* ## Cost, measured
|
||||
*
|
||||
* Shiki's JavaScript regex engine (no oniguruma wasm, no native module) runs at
|
||||
* roughly 17 us/line on Go, 34 us/line on Python and 230 us/line on TypeScript,
|
||||
* whose TextMate grammar is by a wide margin the most expensive one here. A
|
||||
* 3 000-line TypeScript file is therefore ~700 ms cold, which is why
|
||||
* {@link SLICE_CACHE_LIMIT} exists: a slice is keyed by the file's content hash
|
||||
* and its line range, so every re-render — a theme flip, a resize, stepping
|
||||
* back to a symbol — is a map lookup. Phase 1 only ever asks for one symbol's
|
||||
* range (tens of lines); the whole-file view is CG-52 and tree-sitter tokens
|
||||
* from the engine's own parse replace this module entirely in CG-57.
|
||||
*/
|
||||
|
||||
import type {
|
||||
ShikiCoreModule,
|
||||
ShikiHighlighter,
|
||||
ShikiJavaScriptEngineModule,
|
||||
ShikiThemedToken,
|
||||
} from './shiki-types';
|
||||
import { CLASS_ID, MONO_THEME, TOKEN_CLASSES, classOf, type TokenClassName } from './theme';
|
||||
import { grammarFor } from './languages';
|
||||
import { loadManifest, readGrammarChain, type GrammarManifest } from './grammars';
|
||||
|
||||
export { TOKEN_CLASSES } from './theme';
|
||||
export { LANGUAGE_GRAMMAR, REQUIRED_GRAMMARS, grammarFor } from './languages';
|
||||
export { TEXTMATE_PATH_ENV } from './grammars';
|
||||
|
||||
/** One token on the wire: its class id, then its text. */
|
||||
export type WireToken = [number, string];
|
||||
|
||||
export interface HighlightResult {
|
||||
/** `shiki` when a grammar produced the classes; `plain` when nothing did. */
|
||||
engine: 'shiki' | 'plain';
|
||||
/** The TextMate grammar used, or null. */
|
||||
grammar: string | null;
|
||||
/** Class names, indexed by the first element of every {@link WireToken}. */
|
||||
classes: readonly string[];
|
||||
/** One entry per source line, in order. */
|
||||
lines: WireToken[][];
|
||||
/** Why the answer is plain, when it is. Absent on the happy path. */
|
||||
reason?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Lines above this are not tokenised.
|
||||
*
|
||||
* Matches `MAX_SOURCE_LINES`, so anything the source endpoint will serve, this
|
||||
* will try to highlight.
|
||||
*/
|
||||
export const MAX_HIGHLIGHT_LINES = 4000;
|
||||
|
||||
/**
|
||||
* Characters above this are not tokenised.
|
||||
*
|
||||
* The line cap alone does not bound the work: one minified bundle line can be
|
||||
* two megabytes, and a TextMate scanner walks it character by character. This
|
||||
* is the guard that keeps a single request from wedging a single-threaded
|
||||
* loopback server, and it is generous — 600 kB is far more source than any
|
||||
* screen renders.
|
||||
*/
|
||||
export const MAX_HIGHLIGHT_CHARS = 600_000;
|
||||
|
||||
/** Highlighted slices kept in memory. Most are one symbol's body. */
|
||||
export const SLICE_CACHE_LIMIT = 96;
|
||||
|
||||
/**
|
||||
* Total cached lines, which is the bound that actually matters.
|
||||
*
|
||||
* The entry count alone does not bound memory: 96 slices of a symbol body is a
|
||||
* megabyte, 96 whole 4 000-line files is two orders of magnitude more, and this
|
||||
* process is a reader someone leaves open all day. Twenty thousand lines is
|
||||
* roughly a working set of every symbol a session visits, or a handful of whole
|
||||
* files, and the eviction is the same recency order.
|
||||
*/
|
||||
export const SLICE_CACHE_LINES = 20_000;
|
||||
|
||||
/* ----------------------------------------------------------- the runtime -- */
|
||||
|
||||
/**
|
||||
* tsc compiles `import()` to `require()` under `module: commonjs`, which fails
|
||||
* for an ESM-only package. Same escape hatch `src/bin/codegraph.ts` uses.
|
||||
*/
|
||||
const importESM = new Function('specifier', 'return import(specifier)') as (
|
||||
specifier: string
|
||||
) => Promise<unknown>;
|
||||
|
||||
/**
|
||||
* Import an ESM-only package from this CommonJS build.
|
||||
*
|
||||
* The `new Function` route is the one that runs in production. It does NOT run
|
||||
* under Vitest, whose module runner evaluates this file without a dynamic-import
|
||||
* callback ("A dynamic import callback was not specified") — there, the
|
||||
* transformed `import()` below is the working one, and in the shipped CommonJS
|
||||
* build it is the one that cannot work. Each covers exactly the other's gap;
|
||||
* neither alone is enough, which is why both are here.
|
||||
*/
|
||||
async function loadEsm<T>(specifier: string): Promise<T> {
|
||||
try {
|
||||
return (await importESM(specifier)) as T;
|
||||
} catch (err) {
|
||||
if (!(err instanceof Error) || !/dynamic import callback/i.test(err.message)) throw err;
|
||||
return (await import(/* @vite-ignore */ specifier)) as T;
|
||||
}
|
||||
}
|
||||
|
||||
interface Runtime {
|
||||
highlighter: ShikiHighlighter;
|
||||
dir: string;
|
||||
manifest: GrammarManifest;
|
||||
}
|
||||
|
||||
let runtimePromise: Promise<Runtime | null> | null = null;
|
||||
/** Why the runtime is unavailable, for the `reason` on a plain answer. */
|
||||
let runtimeFailure: string | null = null;
|
||||
|
||||
async function getRuntime(): Promise<Runtime | null> {
|
||||
if (!runtimePromise) runtimePromise = createRuntime();
|
||||
return runtimePromise;
|
||||
}
|
||||
|
||||
async function createRuntime(): Promise<Runtime | null> {
|
||||
const found = loadManifest();
|
||||
if (!found) {
|
||||
runtimeFailure =
|
||||
'No syntax grammars are installed with this build, so source is shown unhighlighted.';
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
const core = await loadEsm<ShikiCoreModule>('@shikijs/core');
|
||||
const engineModule = await loadEsm<ShikiJavaScriptEngineModule>('@shikijs/engine-javascript');
|
||||
const highlighter = core.createHighlighterCoreSync({
|
||||
themes: [MONO_THEME],
|
||||
langs: [],
|
||||
// The JavaScript regex engine, deliberately: no oniguruma wasm and no
|
||||
// native module, so the viewer adds nothing to the install that has to
|
||||
// be compiled or fetched per platform. `forgiving` skips the handful of
|
||||
// Oniguruma-only patterns it cannot translate rather than refusing the
|
||||
// whole grammar over them.
|
||||
engine: engineModule.createJavaScriptRegexEngine({ forgiving: true, cache: new Map() }),
|
||||
});
|
||||
return { highlighter, dir: found.dir, manifest: found.manifest };
|
||||
} catch (err) {
|
||||
runtimeFailure = `Syntax highlighting is unavailable (${
|
||||
err instanceof Error ? err.message : String(err)
|
||||
}).`;
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Grammar ids already handed to the highlighter, and the ones that failed. */
|
||||
const loadedGrammars = new Set<string>();
|
||||
const brokenGrammars = new Map<string, string>();
|
||||
|
||||
function ensureGrammar(runtime: Runtime, id: string): string | null {
|
||||
if (loadedGrammars.has(id)) return null;
|
||||
const broken = brokenGrammars.get(id);
|
||||
if (broken !== undefined) return broken;
|
||||
try {
|
||||
const chain = readGrammarChain(runtime.dir, runtime.manifest, id);
|
||||
if (chain.length === 0) {
|
||||
const reason = `No ${id} grammar shipped with this build, so it is shown unhighlighted.`;
|
||||
brokenGrammars.set(id, reason);
|
||||
return reason;
|
||||
}
|
||||
runtime.highlighter.loadLanguageSync(chain);
|
||||
loadedGrammars.add(id);
|
||||
return null;
|
||||
} catch (err) {
|
||||
const reason = `The ${id} grammar could not be loaded (${
|
||||
err instanceof Error ? err.message : String(err)
|
||||
}).`;
|
||||
brokenGrammars.set(id, reason);
|
||||
return reason;
|
||||
}
|
||||
}
|
||||
|
||||
/* -------------------------------------------------------------- the cache -- */
|
||||
|
||||
const sliceCache = new Map<string, HighlightResult>();
|
||||
let cachedLines = 0;
|
||||
|
||||
function cacheGet(key: string): HighlightResult | undefined {
|
||||
const hit = sliceCache.get(key);
|
||||
// Re-insert so the map's insertion order is a recency order and the first
|
||||
// key is always the coldest.
|
||||
if (hit) {
|
||||
sliceCache.delete(key);
|
||||
sliceCache.set(key, hit);
|
||||
}
|
||||
return hit;
|
||||
}
|
||||
|
||||
function cachePut(key: string, value: HighlightResult): void {
|
||||
sliceCache.set(key, value);
|
||||
cachedLines += value.lines.length;
|
||||
while (
|
||||
sliceCache.size > SLICE_CACHE_LIMIT ||
|
||||
(cachedLines > SLICE_CACHE_LINES && sliceCache.size > 1)
|
||||
) {
|
||||
const oldest = sliceCache.keys().next();
|
||||
if (oldest.done) break;
|
||||
cachedLines -= sliceCache.get(oldest.value)?.lines.length ?? 0;
|
||||
sliceCache.delete(oldest.value);
|
||||
}
|
||||
}
|
||||
|
||||
/** Drop everything cached. Tests use it; nothing in the server needs to. */
|
||||
export function clearHighlightCache(): void {
|
||||
sliceCache.clear();
|
||||
cachedLines = 0;
|
||||
}
|
||||
|
||||
/** What the slice cache is holding — for tests, and for anyone diagnosing it. */
|
||||
export function highlightCacheStats(): { entries: number; lines: number } {
|
||||
return { entries: sliceCache.size, lines: cachedLines };
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------- the entry -- */
|
||||
|
||||
export interface HighlightOptions {
|
||||
/** The engine's language for the file, e.g. `typescript`. */
|
||||
language?: string | null;
|
||||
/**
|
||||
* A key that changes whenever the text does — the file's content hash plus
|
||||
* the requested range. Omit it and the slice is tokenised every time.
|
||||
*/
|
||||
cacheKey?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Classify `lines` for the viewer's code block.
|
||||
*
|
||||
* Never throws and never rejects: every failure path returns a plain result
|
||||
* carrying the reason, because the caller is serving source and the source is
|
||||
* the part that matters.
|
||||
*/
|
||||
export async function highlightLines(
|
||||
lines: readonly string[],
|
||||
options: HighlightOptions = {}
|
||||
): Promise<HighlightResult> {
|
||||
const grammar = grammarFor(options.language);
|
||||
const key = options.cacheKey ? `${grammar ?? '-'} ${options.cacheKey}` : null;
|
||||
if (key) {
|
||||
const hit = cacheGet(key);
|
||||
if (hit) return hit;
|
||||
}
|
||||
|
||||
const result = await highlightUncached(lines, grammar);
|
||||
if (key) cachePut(key, result);
|
||||
return result;
|
||||
}
|
||||
|
||||
async function highlightUncached(
|
||||
lines: readonly string[],
|
||||
grammar: string | null
|
||||
): Promise<HighlightResult> {
|
||||
if (!grammar) {
|
||||
return plain(lines, null, 'No syntax grammar covers this file type.');
|
||||
}
|
||||
if (lines.length > MAX_HIGHLIGHT_LINES) {
|
||||
return plain(lines, grammar, `Too many lines to highlight (over ${MAX_HIGHLIGHT_LINES}).`);
|
||||
}
|
||||
let chars = 0;
|
||||
for (const line of lines) chars += line.length + 1;
|
||||
if (chars > MAX_HIGHLIGHT_CHARS) {
|
||||
return plain(lines, grammar, 'Too much text on too few lines to highlight (minified?).');
|
||||
}
|
||||
|
||||
const runtime = await getRuntime();
|
||||
if (!runtime) return plain(lines, grammar, runtimeFailure ?? undefined);
|
||||
|
||||
const failure = ensureGrammar(runtime, grammar);
|
||||
if (failure) return plain(lines, grammar, failure);
|
||||
|
||||
let tokenized: ShikiThemedToken[][];
|
||||
try {
|
||||
tokenized = runtime.highlighter.codeToTokensBase(lines.join('\n'), {
|
||||
lang: grammar,
|
||||
theme: MONO_THEME.name,
|
||||
});
|
||||
} catch (err) {
|
||||
// A grammar that throws once will throw again on the next request for the
|
||||
// same file type, so it is retired rather than retried.
|
||||
const reason = `The ${grammar} grammar failed on this file (${
|
||||
err instanceof Error ? err.message : String(err)
|
||||
}).`;
|
||||
brokenGrammars.set(grammar, reason);
|
||||
loadedGrammars.delete(grammar);
|
||||
return plain(lines, grammar, reason);
|
||||
}
|
||||
|
||||
// A trailing empty line, or a grammar that answered short, must not shift the
|
||||
// viewer's line numbering — the rows are indexed positionally.
|
||||
const out: WireToken[][] = lines.map((line, i) => {
|
||||
const row = tokenized[i];
|
||||
return row ? atomize(row) : atomizePlain(line);
|
||||
});
|
||||
|
||||
return { engine: 'shiki', grammar, classes: TOKEN_CLASSES, lines: out };
|
||||
}
|
||||
|
||||
function plain(lines: readonly string[], grammar: string | null, reason?: string): HighlightResult {
|
||||
return {
|
||||
engine: 'plain',
|
||||
grammar,
|
||||
classes: TOKEN_CLASSES,
|
||||
lines: lines.map(atomizePlain),
|
||||
...(reason ? { reason } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
/* ---------------------------------------------------------- atomisation -- */
|
||||
|
||||
/**
|
||||
* An identifier, in the loosest sense every indexed language agrees on.
|
||||
*
|
||||
* The high range is there because `\w` is ASCII-only in JavaScript and a symbol
|
||||
* name can be Chinese, Japanese or Cyrillic; a call site in those repositories
|
||||
* has to be linkable too.
|
||||
*/
|
||||
const IDENT = /[A-Za-z_$À-][\w$À-]*/g;
|
||||
|
||||
/**
|
||||
* Split a grammar's tokens into identifier runs, merging everything else.
|
||||
*
|
||||
* This is the step that makes the graph's call-site links independent of how a
|
||||
* grammar chose to chunk a line. TextMate is free to emit `this.mutex.withLock`
|
||||
* as one token, three, or five, and the viewer has to be able to wrap exactly
|
||||
* `withLock`; giving it identifier-sized atoms up front means the overlay only
|
||||
* ever *claims* a token, never re-cuts one.
|
||||
*
|
||||
* Comments and strings are left whole on purpose: no edge points inside one,
|
||||
* and a doc comment split into forty atoms is forty times the wire bytes for
|
||||
* nothing.
|
||||
*/
|
||||
function atomize(tokens: readonly ShikiThemedToken[]): WireToken[] {
|
||||
const out: WireToken[] = [];
|
||||
for (const token of tokens) {
|
||||
const cls = classOf(token.color);
|
||||
if (cls === 'comment' || cls === 'string') {
|
||||
push(out, cls, token.content);
|
||||
continue;
|
||||
}
|
||||
splitIdentifiers(out, token.content, cls);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function atomizePlain(line: string): WireToken[] {
|
||||
const out: WireToken[] = [];
|
||||
splitIdentifiers(out, line, 'other');
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Emit `text` as alternating non-identifier and identifier runs.
|
||||
*
|
||||
* An identifier inside a token the grammar called a keyword keeps the keyword
|
||||
* class — `func` should still carry its weight — while the overlay's matcher
|
||||
* looks at a token's *text*, not its class, so a language whose grammar scopes
|
||||
* type names as `storage.type` still links.
|
||||
*/
|
||||
function splitIdentifiers(out: WireToken[], text: string, cls: TokenClassName): void {
|
||||
if (text === '') return;
|
||||
IDENT.lastIndex = 0;
|
||||
let at = 0;
|
||||
let match: RegExpExecArray | null;
|
||||
while ((match = IDENT.exec(text)) !== null) {
|
||||
if (match.index > at) push(out, cls === 'ident' ? 'other' : cls, text.slice(at, match.index));
|
||||
push(out, cls === 'other' ? 'ident' : cls, match[0]);
|
||||
at = match.index + match[0].length;
|
||||
}
|
||||
if (at < text.length) push(out, cls === 'ident' ? 'other' : cls, text.slice(at));
|
||||
}
|
||||
|
||||
/** Append, merging into the previous token when it carries the same class. */
|
||||
function push(out: WireToken[], cls: TokenClassName, text: string): void {
|
||||
if (text === '') return;
|
||||
const id = CLASS_ID[cls];
|
||||
const last = out[out.length - 1];
|
||||
// Identifiers are never merged: each one has to stay claimable on its own.
|
||||
if (last && last[0] === id && id !== CLASS_ID.ident) {
|
||||
last[1] += text;
|
||||
return;
|
||||
}
|
||||
out.push([id, text]);
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
/**
|
||||
* Engine `Language` → TextMate grammar, and the closure of grammars that has
|
||||
* to ship for those to load.
|
||||
*
|
||||
* The engine indexes 40-odd languages; Shiki carries 700-odd grammars. Shipping
|
||||
* all of them would put 11 MB of JSON in the bundle to serve 40, so the build
|
||||
* prunes them (`scripts/prune-grammars.mjs`) to exactly the closure this table
|
||||
* names — which is why the table lives in its own module: the build script
|
||||
* reads the compiled `dist/ui-server/highlight/languages.js` rather than keeping
|
||||
* a second copy of the mapping that could drift from the runtime's.
|
||||
*
|
||||
* A language with no entry (or with `null`) is not an error. It renders as
|
||||
* plain text with its identifiers still split out, so the graph's call-site
|
||||
* links land exactly as they do everywhere else — highlighting is the part that
|
||||
* degrades, never the linking.
|
||||
*/
|
||||
|
||||
import type { Language } from '../../types';
|
||||
|
||||
/**
|
||||
* The grammar each indexed language is read with.
|
||||
*
|
||||
* Three of these are deliberate approximations, marked below: Shiki has no
|
||||
* ColdFusion grammar, and the three CFML dialects the engine distinguishes are
|
||||
* each a close relative of something it does have. An approximate keyword set
|
||||
* is a better answer than no colouring at all, and nothing downstream depends
|
||||
* on the grammar being exact — the links come from the graph.
|
||||
*/
|
||||
export const LANGUAGE_GRAMMAR: Record<Language, string | null> = {
|
||||
typescript: 'typescript',
|
||||
javascript: 'javascript',
|
||||
tsx: 'tsx',
|
||||
jsx: 'jsx',
|
||||
// ArkTS is TypeScript plus HarmonyOS decorators — the TS grammar reads it.
|
||||
arkts: 'typescript',
|
||||
python: 'python',
|
||||
go: 'go',
|
||||
rust: 'rust',
|
||||
java: 'java',
|
||||
c: 'c',
|
||||
cpp: 'cpp',
|
||||
csharp: 'csharp',
|
||||
razor: 'razor',
|
||||
php: 'php',
|
||||
ruby: 'ruby',
|
||||
swift: 'swift',
|
||||
kotlin: 'kotlin',
|
||||
dart: 'dart',
|
||||
svelte: 'svelte',
|
||||
vue: 'vue',
|
||||
astro: 'astro',
|
||||
liquid: 'liquid',
|
||||
pascal: 'pascal',
|
||||
scala: 'scala',
|
||||
lua: 'lua',
|
||||
luau: 'luau',
|
||||
objc: 'objective-c',
|
||||
r: 'r',
|
||||
solidity: 'solidity',
|
||||
nix: 'nix',
|
||||
yaml: 'yaml',
|
||||
twig: 'twig',
|
||||
xml: 'xml',
|
||||
properties: 'properties',
|
||||
// Approximations — no CFML grammar exists. Tag soup reads as HTML, cfscript
|
||||
// is a JavaScript-shaped dialect, and a <cfquery> body is SQL.
|
||||
cfml: 'html',
|
||||
cfscript: 'javascript',
|
||||
cfquery: 'sql',
|
||||
cobol: 'cobol',
|
||||
vbnet: 'vb',
|
||||
erlang: 'erlang',
|
||||
terraform: 'terraform',
|
||||
// Not a language, the absence of one: a file no extractor claimed.
|
||||
unknown: null,
|
||||
};
|
||||
|
||||
/** Every grammar the build must prune to, de-duplicated, in a stable order. */
|
||||
export const REQUIRED_GRAMMARS: readonly string[] = [
|
||||
...new Set(Object.values(LANGUAGE_GRAMMAR).filter((id): id is string => id !== null)),
|
||||
].sort();
|
||||
|
||||
/**
|
||||
* The grammar for an indexed language, or null when it has none.
|
||||
*
|
||||
* Accepts the raw string off a `FileRecord` rather than a `Language`, because
|
||||
* an index written by an older engine can hold a language this build has since
|
||||
* renamed, and a viewer must not throw over that.
|
||||
*/
|
||||
export function grammarFor(language: string | undefined | null): string | null {
|
||||
if (!language) return null;
|
||||
return LANGUAGE_GRAMMAR[language as Language] ?? null;
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
/**
|
||||
* The slice of Shiki's surface this server uses, declared locally.
|
||||
*
|
||||
* `@shikijs/core` is ESM-only and the engine compiles to CommonJS, so it is
|
||||
* loaded through the same `new Function('return import(...)')` escape hatch the
|
||||
* CLI uses for `@clack/prompts` — which means tsc never sees the import and
|
||||
* cannot type it. Rather than fight `.d.mts` resolution under
|
||||
* `module: commonjs`, the four shapes actually touched are written out here.
|
||||
* They are checked against the real package by the highlighter's tests: a
|
||||
* signature change shows up as a failing highlight, not as a silent `any`.
|
||||
*/
|
||||
|
||||
export interface ShikiThemedToken {
|
||||
content: string;
|
||||
color?: string;
|
||||
fontStyle?: number;
|
||||
}
|
||||
|
||||
export interface ShikiHighlighter {
|
||||
loadLanguageSync(lang: unknown): void;
|
||||
getLoadedLanguages(): string[];
|
||||
codeToTokensBase(code: string, options: { lang: string; theme: string }): ShikiThemedToken[][];
|
||||
dispose?(): void;
|
||||
}
|
||||
|
||||
export interface ShikiCoreModule {
|
||||
createHighlighterCoreSync(options: {
|
||||
themes: unknown[];
|
||||
langs: unknown[];
|
||||
engine: unknown;
|
||||
}): ShikiHighlighter;
|
||||
}
|
||||
|
||||
export interface ShikiJavaScriptEngineModule {
|
||||
createJavaScriptRegexEngine(options?: {
|
||||
forgiving?: boolean;
|
||||
target?: 'auto' | 'ES2025' | 'ES2024' | 'ES2018';
|
||||
cache?: Map<string, RegExp | Error> | null;
|
||||
}): unknown;
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
/**
|
||||
* The near-monochrome code theme (design spec §2.2).
|
||||
*
|
||||
* The colouring is deliberately almost absent: comments and strings recede,
|
||||
* keywords carry weight rather than hue, and the ONLY colour in the body is a
|
||||
* resolved call site. A six-colour syntax theme buries exactly the thing the
|
||||
* screen exists to show.
|
||||
*
|
||||
* ## Why the theme's colours are sentinels, not colours
|
||||
*
|
||||
* A TextMate theme classifies by mapping scopes to colours, so that is how the
|
||||
* classification is *expressed* — but the values here are placeholders that
|
||||
* mean "comment", "string", "keyword", "number", nothing. The server turns each
|
||||
* one back into a class name; the viewer paints it from a CSS custom property.
|
||||
*
|
||||
* That indirection is load-bearing, not decoration:
|
||||
*
|
||||
* * **One token stream serves both modes.** The viewer flips light/dark from
|
||||
* `prefers-color-scheme` with no reload and no refetch. Baking `#6a675d` into
|
||||
* the payload would make dark mode a second request for the same source, and
|
||||
* would put the design tokens in two places at once.
|
||||
* * **Contrast is fixed where the tokens live.** `ui/src/app.css` owns the
|
||||
* ramp; a colour change there cannot leave the server's copy behind.
|
||||
*
|
||||
* The sentinels are arbitrary but must be distinct and must never be a colour a
|
||||
* grammar could plausibly emit through some other path, hence the `#00000n`
|
||||
* block: TextMate themes only ever return values *this* theme defines.
|
||||
*/
|
||||
|
||||
/** The classes a token can carry — the viewer's `TokenClass`, server side. */
|
||||
export const TOKEN_CLASSES = ['other', 'ident', 'comment', 'string', 'keyword', 'number'] as const;
|
||||
|
||||
export type TokenClassName = (typeof TOKEN_CLASSES)[number];
|
||||
|
||||
/** Class name → its index in {@link TOKEN_CLASSES}, which is what the wire carries. */
|
||||
export const CLASS_ID: Record<TokenClassName, number> = {
|
||||
other: 0,
|
||||
ident: 1,
|
||||
comment: 2,
|
||||
string: 3,
|
||||
keyword: 4,
|
||||
number: 5,
|
||||
};
|
||||
|
||||
const FG_DEFAULT = '#000001';
|
||||
const FG_COMMENT = '#000002';
|
||||
const FG_STRING = '#000003';
|
||||
const FG_KEYWORD = '#000004';
|
||||
const FG_NUMBER = '#000005';
|
||||
|
||||
/** Sentinel foreground → the class it stands for. */
|
||||
export const SENTINEL_CLASS: Record<string, TokenClassName> = {
|
||||
[FG_DEFAULT]: 'other',
|
||||
[FG_COMMENT]: 'comment',
|
||||
[FG_STRING]: 'string',
|
||||
[FG_KEYWORD]: 'keyword',
|
||||
[FG_NUMBER]: 'number',
|
||||
};
|
||||
|
||||
/**
|
||||
* The theme itself.
|
||||
*
|
||||
* Scope selection follows the spec exactly: `comment` recedes furthest,
|
||||
* `string`/`constant.numeric` sit one step in, `keyword`/`storage` stay ink and
|
||||
* gain weight, everything else is ink. Nothing sets a background — a token that
|
||||
* painted its own would fight the hovered-line and hot-line tints the rails use
|
||||
* to point at it.
|
||||
*/
|
||||
export const MONO_THEME = {
|
||||
name: 'codegraph-mono',
|
||||
type: 'light' as const,
|
||||
fg: FG_DEFAULT,
|
||||
// TextMate wants a background; the viewer never reads it (the code block
|
||||
// paints `--paper`), and it must not equal a foreground sentinel.
|
||||
bg: '#ffffff',
|
||||
settings: [
|
||||
{ settings: { foreground: FG_DEFAULT } },
|
||||
{ scope: ['comment', 'punctuation.definition.comment'], settings: { foreground: FG_COMMENT } },
|
||||
{
|
||||
scope: [
|
||||
'string',
|
||||
'string.template',
|
||||
'punctuation.definition.string',
|
||||
'constant.character.escape',
|
||||
],
|
||||
settings: { foreground: FG_STRING },
|
||||
},
|
||||
{
|
||||
scope: ['constant.numeric', 'constant.language', 'keyword.other.unit'],
|
||||
settings: { foreground: FG_NUMBER },
|
||||
},
|
||||
{
|
||||
scope: ['keyword', 'keyword.control', 'storage', 'storage.type', 'storage.modifier'],
|
||||
settings: { foreground: FG_KEYWORD },
|
||||
},
|
||||
// `keyword.operator` is a keyword scope by name only: it covers `=`, `+`,
|
||||
// `=>` and `?.`. Weighting punctuation buys nothing and costs the calm the
|
||||
// rest of the block is built on, so it drops back to plain ink — while the
|
||||
// operators that are actually WORDS (`new`, `typeof`, `instanceof`, `in`)
|
||||
// keep their weight through the more specific rule below. Shiki resolves
|
||||
// the longest matching scope, so the order here is the order of rescue,
|
||||
// not of priority.
|
||||
{ scope: ['keyword.operator'], settings: { foreground: FG_DEFAULT } },
|
||||
{
|
||||
scope: ['keyword.operator.expression', 'keyword.operator.word', 'keyword.operator.new'],
|
||||
settings: { foreground: FG_KEYWORD },
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
/** The class a Shiki token's resolved colour stands for. */
|
||||
export function classOf(color: string | undefined): TokenClassName {
|
||||
if (!color) return 'other';
|
||||
return SENTINEL_CLASS[color.toLowerCase()] ?? 'other';
|
||||
}
|
||||
Reference in New Issue
Block a user