From 2ad836d93501cefcb99a8607361d591311b95bf6 Mon Sep 17 00:00:00 2001 From: Colby McHenry Date: Thu, 27 Aug 2026 01:23:10 -0500 Subject: [PATCH] feat(ui): highlight source server-side with a near-monochrome Shiki theme (CG-43) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The viewer's code block stops lexing with a hand-rolled dialect table and reads real TextMate grammars instead, run once in `/api/source`. Three things make that safe to depend on: * Highlighting never fails a request. A missing grammar, an oversized slice, an ESM import that did not resolve — every one of them answers `engine: 'plain'` with a reason and the source still goes out. * Identifiers survive whatever token boundaries a grammar chose. Every code token is split into identifier runs before it goes on the wire, so the graph's call-site overlay claims a token the highlighter produced rather than re-cutting the line. `assignRefs` now matches on a token's text rather than on the class a grammar gave it, so a language that scopes type names as `storage.type` still links. * The theme classifies rather than colours: its foregrounds are sentinels the server maps back to class names, and the viewer paints them from CSS custom properties — one token stream serves light and dark with no refetch, and the ramp lives only in app.css. Comments move from --ink-3 to a new --code-comment. --ink-3 measures 3.46:1 on paper and 3.00:1 on the hot-line tint, both under AA for 12.5px text; --code-comment is the smallest step along the same ramp that clears 4.5:1 on every background a code line can have, and stays quieter than the strings and numbers above it. Shipping: @shikijs/core and @shikijs/engine-javascript are runtime dependencies (no wasm, no native module); @shikijs/langs stays a devDependency and `npm run build:textmate` writes only the closure the engine's 40-odd languages reach — 56 grammars, 2.6 MB, against 11 MB for all 722. check-ui-build.mjs asserts the tree after every build and inside every release archive. --- CLAUDE.md | 7 +- __tests__/ui-highlight.test.ts | 349 +++++++++++++ __tests__/ui-server-api.test.ts | 26 + __tests__/ui-symbol-model.test.ts | 180 ++++--- docs/design/codegraph-ui-design-spec.md | 24 +- package-lock.json | 518 ++++++++++++++++++++ package.json | 8 +- scripts/check-ui-build.mjs | 39 +- scripts/prune-grammars.mjs | 110 +++++ src/ui-server/api/index.ts | 6 +- src/ui-server/api/source.ts | 25 +- src/ui-server/highlight/grammars.ts | 86 ++++ src/ui-server/highlight/index.ts | 407 +++++++++++++++ src/ui-server/highlight/languages.ts | 93 ++++ src/ui-server/highlight/shiki-types.ts | 40 ++ src/ui-server/highlight/theme.ts | 115 +++++ ui/src/app.css | 14 + ui/src/components/symbol/SourceBlock.svelte | 50 +- ui/src/lib/api.ts | 9 + ui/src/lib/highlight.ts | 391 ++++----------- ui/src/lib/symbol-model.ts | 10 +- ui/src/views/SymbolView.svelte | 15 +- 22 files changed, 2117 insertions(+), 405 deletions(-) create mode 100644 __tests__/ui-highlight.test.ts create mode 100644 scripts/prune-grammars.mjs create mode 100644 src/ui-server/highlight/grammars.ts create mode 100644 src/ui-server/highlight/index.ts create mode 100644 src/ui-server/highlight/languages.ts create mode 100644 src/ui-server/highlight/shiki-types.ts create mode 100644 src/ui-server/highlight/theme.ts diff --git a/CLAUDE.md b/CLAUDE.md index 0b24349..143e6d3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -11,7 +11,7 @@ Distributed as `@colbymchenry/codegraph` on npm; same binary serves as installer ## Build, Test, Run ```bash -npm run build # tsc + copy schema.sql and *.wasm into dist/; chmods dist/bin/codegraph.js +npm run build # tsc + copy schema.sql and *.wasm + prune TextMate grammars + build the viewer into dist/; chmods dist/bin/codegraph.js npm run dev # tsc --watch npm run clean # rm -rf dist @@ -29,6 +29,11 @@ npx vitest run __tests__/extraction.test.ts -t "TypeScript" `copy-assets` (called from `build`) copies `src/db/schema.sql` and all `src/extraction/wasm/*.wasm` files into `dist/`. **Any new SQL or grammar wasm must be copied or it won't ship.** +Two other build steps write into `dist/` and are subject to the same rule: +`build:textmate` (`scripts/prune-grammars.mjs`) writes the viewer's syntax grammars to `dist/textmate/`, and +`build:ui` builds the browser viewer into `dist/viewer/` (never `dist/ui/` — that's the terminal ui). +`scripts/check-ui-build.mjs` asserts both trees after every build and inside every release archive. + Node engines: `>=20.0.0 <25.0.0`. There is a hard exit on Node 25.x and below 20 (see `src/bin/node-version-check.ts`). ## Architecture diff --git a/__tests__/ui-highlight.test.ts b/__tests__/ui-highlight.test.ts new file mode 100644 index 0000000..816b861 --- /dev/null +++ b/__tests__/ui-highlight.test.ts @@ -0,0 +1,349 @@ +/** + * The viewer's server-side syntax classification (CG-43). + * + * Two things are worth pinning here and they are not the colours. The first is + * that a call-site link lands on the callee's own name — the accent underline + * is the only colour in the code block, and putting it on the receiver or on a + * word inside a comment is worse than not drawing it. The second is that + * highlighting never becomes a way for a source request to fail: a missing + * grammar, an oversized slice, a language nobody wrote a grammar for all have + * to answer with the source and an honest `engine: 'plain'`. + * + * The end-to-end shape is deliberate: the server's tokens are fed straight + * through the viewer's own `decodeLine` and `assignRefs`, because the seam + * between "how a grammar chose to cut a line" and "which token the overlay + * claims" is exactly where this breaks. + */ + +import { describe, it, expect, beforeAll, vi } from 'vitest'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { + clearHighlightCache, + grammarFor, + highlightCacheStats, + highlightLines, + LANGUAGE_GRAMMAR, + MAX_HIGHLIGHT_CHARS, + REQUIRED_GRAMMARS, + SLICE_CACHE_LINES, + TOKEN_CLASSES, + type HighlightResult, +} from '../src/ui-server/highlight'; +import { loadManifest } from '../src/ui-server/highlight/grammars'; +import { LANGUAGES } from '../src/types'; +import { decodeLine, type Token } from '../ui/src/lib/highlight'; +import { assignRefs, type LineRef } from '../ui/src/lib/symbol-model'; + +/** + * The pruned grammars live in `dist/textmate`, written by `npm run build`. A + * source tree that has only ever been type-checked has none, and the right + * behaviour there is plain text — which is itself asserted below, so the + * grammar-dependent cases skip rather than fail. + */ +const HAS_GRAMMARS = loadManifest() !== null; +const withGrammars = HAS_GRAMMARS ? it : it.skip; + +function tokensOf(result: HighlightResult, line: number): Token[] { + return decodeLine(result.lines[line] ?? [], result.classes); +} + +/** What the code block would render for one line: `class:text` per token. */ +function shape(result: HighlightResult, line: number): string[] { + return tokensOf(result, line).map((t) => `${t.cls}:${t.text}`); +} + +function lineRef(over: Partial): LineRef { + return { + ident: 'x', + col: null, + targetId: 'method:x', + uncertain: false, + outside: false, + title: '', + ...over, + }; +} + +/** Which token an overlay ref claims — the whole point of the atomisation. */ +function claimedText(result: HighlightResult, line: number, ref: LineRef): string | undefined { + const tokens = tokensOf(result, line); + const claimed = assignRefs(tokens, [ref]); + const [index] = [...claimed.keys()]; + return index === undefined ? undefined : tokens[index]?.text; +} + +describe('the language table', () => { + it('has an entry for every language the engine indexes', () => { + for (const language of LANGUAGES) { + expect(LANGUAGE_GRAMMAR).toHaveProperty(language); + } + }); + + it('answers null rather than throwing for a language this build never heard of', () => { + expect(grammarFor('some-future-language')).toBeNull(); + expect(grammarFor(undefined)).toBeNull(); + expect(grammarFor('')).toBeNull(); + }); + + withGrammars('ships a grammar for every id the table names', () => { + const found = loadManifest(); + expect(found).not.toBeNull(); + for (const id of REQUIRED_GRAMMARS) { + expect(Object.keys((found as NonNullable).manifest.languages)).toContain(id); + } + }); + + withGrammars('loads a grammar chain dependencies-first, so embedded blocks highlight', () => { + const found = loadManifest(); + const vue = (found as NonNullable).manifest.languages['vue'] ?? []; + // The single-file component's own grammar is last; everything it embeds + // has to be registered before Shiki resolves `embeddedLangs`. + expect(vue[vue.length - 1]).toBe('vue'); + expect(vue).toContain('typescript'); + expect(vue.indexOf('typescript')).toBeLessThan(vue.length - 1); + }); +}); + +describe('classification', () => { + beforeAll(() => clearHighlightCache()); + + withGrammars('reads TypeScript with the four classes the theme paints', async () => { + const result = await highlightLines(['const answer = 42; // note'], { + language: 'typescript', + }); + expect(result.engine).toBe('shiki'); + expect(result.grammar).toBe('typescript'); + expect(result.classes).toEqual([...TOKEN_CLASSES]); + const rendered = shape(result, 0); + expect(rendered).toContain('keyword:const'); + expect(rendered).toContain('ident:answer'); + expect(rendered).toContain('number:42'); + expect(rendered).toContain('comment:// note'); + }); + + withGrammars('reads a # comment as a comment in Python and as code in TypeScript', async () => { + const python = await highlightLines(['x = 1 # note'], { language: 'python' }); + expect(shape(python, 0).at(-1)).toBe('comment:# note'); + + const ts = await highlightLines(['x = 1 # note'], { language: 'typescript' }); + expect(shape(ts, 0).at(-1)).not.toBe('comment:# note'); + }); + + withGrammars('carries a block comment across lines within one slice', async () => { + const result = await highlightLines(['/* open', 'still comment', 'done */ const x = 1;'], { + language: 'typescript', + }); + expect(shape(result, 1)).toEqual(['comment:still comment']); + expect(shape(result, 2)[0]).toBe('comment:done */'); + expect(shape(result, 2)).toContain('keyword:const'); + }); + + withGrammars('reads Go, which has its own idea of what a keyword is', async () => { + const result = await highlightLines(['func Greet(name string) string {'], { language: 'go' }); + expect(shape(result, 0)).toContain('keyword:func'); + expect(shape(result, 0)).toContain('ident:Greet'); + }); + + withGrammars('reads ArkTS with the TypeScript grammar', async () => { + const result = await highlightLines(['@Entry struct Index { build() {} }'], { + language: 'arkts', + }); + expect(result.engine).toBe('shiki'); + expect(result.grammar).toBe('typescript'); + }); + + withGrammars('emits one entry per source line, always', async () => { + const lines = ['a();', '', 'b();', '']; + const result = await highlightLines(lines, { language: 'typescript' }); + // The code block indexes rows positionally: one short answer and every + // line below it renders the wrong source. + expect(result.lines).toHaveLength(lines.length); + expect(result.lines[1]).toEqual([]); + }); +}); + +describe('the plain fallback', () => { + beforeAll(() => clearHighlightCache()); + + it('answers plain, with a reason, for a language no grammar covers', async () => { + const result = await highlightLines(['whatever this is'], { language: 'unknown' }); + expect(result.engine).toBe('plain'); + expect(result.grammar).toBeNull(); + expect(result.reason).toBeTruthy(); + expect(result.lines).toHaveLength(1); + }); + + it('still splits identifiers when it cannot highlight, so the links land', async () => { + const result = await highlightLines([' return this.mutex.withLock();'], { + language: 'unknown', + }); + expect(claimedText(result, 0, lineRef({ ident: 'withLock', col: 9 }))).toBe('withLock'); + }); + + it('refuses to tokenise a minified line rather than wedging on it', async () => { + const enormous = 'a'.repeat(MAX_HIGHLIGHT_CHARS + 1); + const result = await highlightLines([enormous], { language: 'javascript' }); + expect(result.engine).toBe('plain'); + expect(result.reason).toMatch(/minified/); + // The source still comes back whole — that is the part that matters. + expect(result.lines[0]?.map(([, text]) => text).join('')).toHaveLength(enormous.length); + }); + + it('answers plain when a shipped grammar file is missing or unreadable', async () => { + // The install is half there: a manifest that names a grammar whose file + // never made it. The viewer must still get its source. + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-textmate-')); + fs.writeFileSync( + path.join(dir, 'manifest.json'), + JSON.stringify({ shikiVersion: 'test', languages: { typescript: ['typescript'] } }) + ); + + const previous = process.env.CODEGRAPH_TEXTMATE_PATH; + process.env.CODEGRAPH_TEXTMATE_PATH = dir; + // A fresh module registry: the highlighter and its grammar bookkeeping are + // created once per process, and this case is about that first attempt. + vi.resetModules(); + try { + const mod = await import('../src/ui-server/highlight'); + const result: HighlightResult = await mod.highlightLines(['const x = 1;'], { + language: 'typescript', + }); + expect(result.engine).toBe('plain'); + expect(result.reason).toBeTruthy(); + expect(result.lines[0]?.map(([, text]) => text).join('')).toBe('const x = 1;'); + } finally { + if (previous === undefined) delete process.env.CODEGRAPH_TEXTMATE_PATH; + else process.env.CODEGRAPH_TEXTMATE_PATH = previous; + fs.rmSync(dir, { recursive: true, force: true }); + vi.resetModules(); + } + }); +}); + +describe('graph links land on the right token', () => { + beforeAll(() => clearHighlightCache()); + + withGrammars('marks the callee, not the receiver the recorded column points at', async () => { + // The recorded column is the start of the calling EXPRESSION — `this` — + // and the underline has to end up on `withLock`. + const line = ' return this.indexMutex.withLock(async () => {'; + const result = await highlightLines([line], { language: 'typescript' }); + expect(claimedText(result, 0, lineRef({ ident: 'withLock', col: line.indexOf('this') }))).toBe( + 'withLock' + ); + }); + + withGrammars('lands on a real call site in the engine’s own src/index.ts', async () => { + const file = path.join(__dirname, '..', 'src', 'index.ts'); + const source = fs.readFileSync(file, 'utf-8').split('\n'); + // A line the engine actually contains, found rather than hard-coded, so a + // refactor of index.ts retires this test instead of silently passing. + const index = source.findIndex((l) => /^\s*(?:return |const \w+ = )?this\.\w+\.\w+\(/.test(l)); + expect(index).toBeGreaterThanOrEqual(0); + const line = source[index] as string; + const match = /this\.(\w+)\.(\w+)\(/.exec(line) as RegExpExecArray; + const callee = match[2] as string; + + const result = await highlightLines([line], { language: 'typescript' }); + expect(claimedText(result, 0, lineRef({ ident: callee, col: line.indexOf('this') }))).toBe( + callee + ); + }); + + withGrammars('lands on a Go method call', async () => { + const line = '\tresult := s.repo.FindByID(ctx, id)'; + const result = await highlightLines([line], { language: 'go' }); + expect(claimedText(result, 0, lineRef({ ident: 'FindByID', col: line.indexOf('s.repo') }))).toBe( + 'FindByID' + ); + }); + + withGrammars('lands on a Python method call, not on the receiver of the same name', async () => { + const line = ' return self.store.join(self.store.path)'; + const result = await highlightLines([line], { language: 'python' }); + expect(claimedText(result, 0, lineRef({ ident: 'join', col: line.indexOf('self') }))).toBe( + 'join' + ); + }); + + withGrammars('leaves a word inside a comment or a string alone', async () => { + const result = await highlightLines( + [' // call render here', ' const s = "render";'], + { language: 'typescript' } + ); + expect(claimedText(result, 0, lineRef({ ident: 'render' }))).toBeUndefined(); + expect(claimedText(result, 1, lineRef({ ident: 'render' }))).toBeUndefined(); + }); + + withGrammars('keeps every identifier separately claimable', async () => { + const result = await highlightLines(['render(); render();'], { language: 'typescript' }); + const tokens = tokensOf(result, 0); + const claimed = assignRefs(tokens, [ + lineRef({ ident: 'render', targetId: 'a' }), + lineRef({ ident: 'render', targetId: 'b' }), + ]); + expect(claimed.size).toBe(2); + }); + + withGrammars('reproduces the line exactly — the code block renders these tokens', async () => { + const line = ' const s = `a ${b.c()} d`; // 1 + 2'; + const result = await highlightLines([line], { language: 'typescript' }); + expect( + tokensOf(result, 0) + .map((t) => t.text) + .join('') + ).toBe(line); + }); +}); + +describe('cost', () => { + withGrammars('answers a cached slice without re-tokenising it', async () => { + clearHighlightCache(); + const lines = fs + .readFileSync(path.join(__dirname, '..', 'src', 'ui-server', 'api', 'source.ts'), 'utf-8') + .split('\n'); + + const cold = Date.now(); + await highlightLines(lines, { language: 'typescript', cacheKey: 'a:1:9999' }); + const coldMs = Date.now() - cold; + + const warm = Date.now(); + const second = await highlightLines(lines, { language: 'typescript', cacheKey: 'a:1:9999' }); + const warmMs = Date.now() - warm; + + expect(second.engine).toBe('shiki'); + // The cache is what makes a re-render free: every resize, theme flip and + // step back through the trail re-asks for the same slice. + expect(warmMs).toBeLessThan(Math.max(20, coldMs / 4)); + }); + + it('bounds the cache by total lines, not just by entry count', async () => { + clearHighlightCache(); + const big = new Array(Math.ceil(SLICE_CACHE_LINES / 2) + 10).fill('x'); + // The entry count alone would let a reader left open on a big repo grow + // without limit: three of these is well inside SLICE_CACHE_LIMIT and well + // over the line budget. + for (const key of ['one', 'two', 'three']) { + await highlightLines(big, { language: 'unknown', cacheKey: key }); + } + const stats = highlightCacheStats(); + expect(stats.entries).toBeLessThan(3); + expect(stats.lines).toBeLessThanOrEqual(SLICE_CACHE_LINES); + }); + + withGrammars('keys the cache on the content, so an edited file re-highlights', async () => { + clearHighlightCache(); + const first = await highlightLines(['const a = 1;'], { + language: 'typescript', + cacheKey: 'hash-one:1:1', + }); + const second = await highlightLines(['const bbb = 2;'], { + language: 'typescript', + cacheKey: 'hash-two:1:1', + }); + expect(first.lines[0]?.map(([, t]) => t).join('')).toBe('const a = 1;'); + expect(second.lines[0]?.map(([, t]) => t).join('')).toBe('const bbb = 2;'); + }); +}); diff --git a/__tests__/ui-server-api.test.ts b/__tests__/ui-server-api.test.ts index 3877265..156d7c3 100644 --- a/__tests__/ui-server-api.test.ts +++ b/__tests__/ui-server-api.test.ts @@ -658,6 +658,29 @@ describe('GET /api/source', () => { expect(body.lines).toHaveLength(node.node.lines); }); + it('carries the classified source beside the lines, one entry per line', async () => { + const body = await getJson('/api/source?file=src/cache.ts&from=1&to=3'); + // Highlighting rides with the slice rather than behind its own endpoint: + // the two are only ever wanted together, and a second round-trip would let + // the code block paint unhighlighted source and then reflow it. + expect(body.highlight).toBeTruthy(); + expect(body.highlight.classes).toEqual([ + 'other', + 'ident', + 'comment', + 'string', + 'keyword', + 'number', + ]); + expect(body.highlight.lines).toHaveLength(body.lines.length); + // Every line's tokens reproduce that line exactly — the code block renders + // these, not the raw string. + for (let i = 0; i < body.lines.length; i++) { + const rebuilt = body.highlight.lines[i].map(([, text]: [number, string]) => text).join(''); + expect(rebuilt).toBe(body.lines[i]); + } + }); + it('refuses to slice a file that changed on disk after the last sync', async () => { const target = path.join(projectRoot, 'src', 'handler.ts'); const original = fs.readFileSync(target); @@ -668,6 +691,9 @@ describe('GET /api/source', () => { expect(body.drift).toBe(true); // The whole point: no slice, rather than a slice of the wrong lines. expect(body.lines).toBeUndefined(); + // And nothing to render it with either — a highlight with no source is + // just a second way to draw the wrong lines. + expect(body.highlight).toBeUndefined(); expect(body.reason).toContain('changed on disk after the last index sync'); // And every screen that renders indexed line ranges is told. diff --git a/__tests__/ui-symbol-model.test.ts b/__tests__/ui-symbol-model.test.ts index 10f8060..d96e7c8 100644 --- a/__tests__/ui-symbol-model.test.ts +++ b/__tests__/ui-symbol-model.test.ts @@ -26,7 +26,7 @@ import { HEAD_LINES, type LineRef, } from '../ui/src/lib/symbol-model'; -import { newLexState, tokenize } from '../ui/src/lib/highlight'; +import { decodeLine, plainLine, tokensByLine } from '../ui/src/lib/highlight'; import type { WireRelation, WireSymbolPayload } from '../ui/src/lib/api'; /* ------------------------------------------------------------- fixtures -- */ @@ -204,7 +204,7 @@ describe('buildCodeBlock', () => { /* ----------------------------------------------------------------- refs -- */ describe('assignRefs', () => { - const toks = (line: string) => tokenize(line, newLexState(), 'typescript'); + const toks = (line: string) => plainLine(line); const ref = (over: Partial): LineRef => ({ ident: 'withLock', col: null, @@ -249,10 +249,36 @@ describe('assignRefs', () => { expect(assignRefs(toks('return 1;'), [ref({ ident: 'nowhere' })]).size).toBe(0); }); - it('never marks a keyword, a string or a comment as a call site', () => { - const tokens = toks('// call render here'); - expect(assignRefs(tokens, [ref({ ident: 'render' })]).size).toBe(0); - expect(assignRefs(toks('const s = "render";'), [ref({ ident: 'render' })]).size).toBe(0); + it('never marks a word inside a comment or a string as a call site', () => { + // The classification comes from the server's grammar; what this pins is + // that the overlay respects it. Anything else — a keyword, a type name a + // grammar happened to scope as `storage.type` — stays claimable, because a + // grammar's opinion about a scope name must not decide what navigates. + const comment = [ + { cls: 'comment', text: '// call render here', col: 0 }, + ]; + expect(assignRefs(comment, [ref({ ident: 'render' })]).size).toBe(0); + + const string = [ + { cls: 'keyword', text: 'const', col: 0 }, + { cls: 'other', text: ' s = ', col: 5 }, + { cls: 'string', text: '"render"', col: 10 }, + { cls: 'other', text: ';', col: 18 }, + ]; + expect(assignRefs(string, [ref({ ident: 'render' })]).size).toBe(0); + }); + + it('still claims an identifier a grammar classified as something else', () => { + // Go scopes `string` as storage.type; Java does the same to a declared + // type name. A link that disappeared over that would be a highlighting + // change silently breaking navigation. + const tokens = [ + { cls: 'keyword', text: 'Duration', col: 0 }, + { cls: 'other', text: '.Since(t)', col: 8 }, + ]; + const claimed = assignRefs(tokens, [ref({ ident: 'Duration', col: 0 })]); + expect(claimed.size).toBe(1); + expect(tokens[[...claimed.keys()][0] as number]?.text).toBe('Duration'); }); }); @@ -456,90 +482,96 @@ describe('showsBody', () => { /* ---------------------------------------------------------------- lexer -- */ -describe('tokenize', () => { - const kinds = (line: string, state = newLexState(), language = 'typescript') => - tokenize(line, state, language).map((t) => `${t.cls}:${t.text}`); +describe('client-side token decoding', () => { + // The classification itself is the server's job (`src/ui-server/highlight/`, + // real TextMate grammars); what is worth pinning here is the decoding — the + // columns the call-site overlay matches against, and the plain fallback that + // has to keep links working when no grammar covers a file. + const CLASSES = ['other', 'ident', 'comment', 'string', 'keyword', 'number']; - it('separates the four things the near-monochrome theme colours', () => { - expect(kinds('const x = 1; // note')).toEqual([ + it('resolves class ids through the payload table', () => { + const tokens = decodeLine( + [ + [4, 'const'], + [0, ' '], + [1, 'x'], + [0, ' = '], + [5, '1'], + [0, '; '], + [2, '// note'], + ], + CLASSES + ); + expect(tokens.map((t) => `${t.cls}:${t.text}`)).toEqual([ 'keyword:const', - 'space: ', + 'other: ', 'ident:x', - 'space: ', - 'punct:=', - 'space: ', + 'other: = ', 'number:1', - 'punct:;', - 'space: ', + 'other:; ', 'comment:// note', ]); }); - it('carries a block comment across lines so the next line is not read as code', () => { - const state = newLexState(); - expect(kinds('/* open', state)).toEqual(['comment:/* open']); - expect(state.block).toBe(true); - expect(kinds('still comment', state)).toEqual(['comment:still comment']); - expect(kinds('done */ const x = 1;', state)).toEqual([ - 'comment:done */', - 'space: ', - 'keyword:const', - 'space: ', - 'ident:x', - 'space: ', - 'punct:=', - 'space: ', - 'number:1', - 'punct:;', - ]); - expect(state.block).toBe(false); + it('derives each column from the running text, which is how a ref finds its identifier', () => { + const tokens = decodeLine( + [ + [0, ' '], + [4, 'return'], + [0, ' '], + [1, 'render'], + [0, '();'], + ], + CLASSES + ); + expect(tokens.find((t) => t.text === 'render')?.col).toBe(' return '.length); + expect(tokens.at(-1)?.col).toBe(' return render'.length); }); - it('carries a template literal across lines, and closes it on the right backtick', () => { - const state = newLexState(); - expect(kinds('const s = `open', state)).toContain('string:`open'); - expect(state.stringEnd).toBe('`'); - expect(kinds('closed` + x', state)).toEqual([ - 'string:closed`', - 'space: ', - 'punct:+', - 'space: ', - 'ident:x', + it('treats an unknown class id as unstyled rather than throwing', () => { + expect(decodeLine([[99, 'x']], CLASSES)[0]?.cls).toBe('other'); + }); + + it('splits identifiers even with no grammar, so the links still land', () => { + expect(plainLine(' return this.mutex.withLock();').map((t) => `${t.cls}:${t.text}`)).toEqual([ + 'other: ', + 'ident:return', + 'other: ', + 'ident:this', + 'other:.', + 'ident:mutex', + 'other:.', + 'ident:withLock', + 'other:();', ]); }); - it('does not eat the rest of a window on an apostrophe in prose', () => { - // An unterminated single-line quote is punctuation in English far more - // often than a real string, so it stops at the line. - const state = newLexState(); - kinds("// it's fine", state); - expect(state.stringEnd).toBeNull(); - const after = kinds('const x = 1;', state); - expect(after[0]).toBe('keyword:const'); + it('splits non-ASCII identifiers, because a symbol name can be one', () => { + expect(plainLine('取得データ()').map((t) => t.cls)).toEqual(['ident', 'other']); }); - it('reads a # comment as a comment in Python and as code in TypeScript', () => { - expect(kinds('# note', newLexState(), 'python')).toEqual(['comment:# note']); - expect(kinds('x = 1 # note', newLexState(), 'python').at(-1)).toBe('comment:# note'); - expect(kinds('# note', newLexState(), 'typescript')[0]).not.toBe('comment:# note'); + it('keys a slice by real file line, not by offset into the slice', () => { + const byLine = tokensByLine(['a();', 'b();'], 120, { + engine: 'shiki', + grammar: 'typescript', + classes: CLASSES, + lines: [ + [ + [1, 'a'], + [0, '();'], + ], + [ + [1, 'b'], + [0, '();'], + ], + ], + }); + expect([...byLine.keys()]).toEqual([120, 121]); + expect(byLine.get(121)?.[0]?.text).toBe('b'); }); - it('closes a Python triple-quoted string on the triple, not on the first quote', () => { - const state = newLexState(); - expect(kinds('"""docstring', state, 'python')).toEqual(['string:"""docstring']); - expect(state.stringEnd).toBe('"""'); - expect(kinds('more"""', state, 'python')).toEqual(['string:more"""']); - }); - - it("reports each token's column, which is how a ref finds its identifier", () => { - const tokens = tokenize(' return render();', newLexState(), 'typescript'); - const render = tokens.find((t) => t.text === 'render'); - expect(render?.col).toBe(' return '.length); - }); - - it('falls back to a C-family reading for a language it has no table for', () => { - // Silence beats a wrong claim, but a plain `//` comment is not a claim - // worth getting wrong in a language we have not enumerated. - expect(kinds('// note', newLexState(), 'some-new-language')).toEqual(['comment:// note']); + it('falls back per line when the payload carries no highlight block at all', () => { + const byLine = tokensByLine(['render();'], 5, undefined); + expect(byLine.get(5)?.map((t) => t.cls)).toEqual(['ident', 'other']); }); }); diff --git a/docs/design/codegraph-ui-design-spec.md b/docs/design/codegraph-ui-design-spec.md index ae7010d..9dfada1 100644 --- a/docs/design/codegraph-ui-design-spec.md +++ b/docs/design/codegraph-ui-design-spec.md @@ -61,10 +61,18 @@ media/`[data-theme]` block. `body { background: var(--paper); color: var(--ink) section labels (`Called by`, `Calls`, `Blast radius`) `600 13px` sans; rail rows `12.5px` mono name + `11px` sans meta; chips `11px` mono; line numbers `11px` mono in `--ink-4`; badges `11.5px`; map node label `13px` mono, count `11px`; flow card name `600 13px` mono, window `12px/19px` mono; trail `12px` mono. Headings sentence case, `text-wrap: balance`. -- Code token classes: comment `--ink-3`; string `--ink-2`; keyword weight 500 (same ink); number `--ink-2`; definition +- Code token classes: comment `--code-comment`; string `--ink-2`; keyword weight 500 (same ink); number `--ink-2`; definition name on its own line weight 600; **call-site link** = `--accent`, underline `--accent-line`, offset 3px, hover/hot fill `--accent-soft`; uncertain link = `--ink-2`, dotted underline `--ink-4`; link to a symbol outside the index = `--ink-2`, underline `--rule-soft`, not clickable. + - *As built (CG-43) — comments are `--code-comment`, not `--ink-3`.* `--ink-3` measures 3.46:1 on `--paper` and 3.00:1 on + the hot-line tint `--accent-soft`, both under the 4.5:1 that 12.5px body text needs. `--code-comment` is the smallest + step along the same warm-grey ramp that clears 4.5:1 on every background a code line can have (`#6a675d` light — + paper 5.23, paper-2 4.92, accent-soft 4.53; `#8e8b81` dark — 5.36 / 5.10 / 4.51) while staying quieter than the + `--ink-2` strings and numbers use, so the recession order above is unchanged. Everything else in this list passes as + specified: ink 16.9/16.2, ink-2 7.03/8.89, accent 9.25/6.91 (8.02/5.80 on `--accent-soft`). + - *Line numbers remain `--ink-4` (1.99:1 light, 2.69:1 dark) — a known contrast gap, left as specified rather than + changed inside a rendering task. Worth a design call before phase 2.* ### 2.3 Kind glyphs @@ -188,6 +196,20 @@ point. fallback if crossing quality demands it (never ELK). Symbol view = DOM + one SVG overlay (`ResizeObserver` re-layout). - Shiki (JavaScript regex engine, lazy grammars, custom near-monochrome theme as in §2.2) server-side in `/api/source`; tree-sitter-derived tokens replace it in phase 3. + - *As built (CG-43).* `@shikijs/core` + `@shikijs/engine-javascript` are runtime dependencies (~5 MB installed, no wasm, no native + module); `@shikijs/langs` is a **devDependency** and `npm run build:textmate` (`scripts/prune-grammars.mjs`) writes only the + closure the engine's 40-odd languages reach — 56 grammars, 2.6 MB — into **`dist/textmate/`**, checked by `scripts/check-ui-build.mjs`. + Shipping all 722 grammars would have been 11 MB. + - The theme classifies rather than colours: its foregrounds are sentinels the server maps to class names (`comment`, `string`, + `keyword`, `number`, `ident`, `other`), and the viewer paints them from the CSS custom properties above — so **one token stream + serves light and dark** with no refetch when `prefers-color-scheme` flips, and the ramp lives only in `ui/src/app.css`. + - Every code token is split into identifier runs before it goes on the wire, so the graph's call-site overlay claims a token the + highlighter produced rather than re-cutting a line — which is what keeps a link landing on the callee's own name whatever + boundaries a grammar chose, and keeps links working in the plain-text fallback. + - Measured on this machine (Shiki 4.4.3, JS regex engine, 3 000 lines cold): Python 35–47 ms, Go 43–57 ms, **TypeScript ~700 ms** — + the TS TextMate grammar is 5–7× the cost of any other and the oniguruma-wasm engine would run it in ~120 ms. Slices are therefore + cached by content hash + range, so a re-render (resize, theme flip, stepping back through the trail) is a map lookup (< 10 ms); + a symbol-sized slice (~280 lines of TS) is ~50 ms cold. Phase 1 only ever requests one symbol's range. - No native modules; no runtime dependency for the UI itself; the CLI serves **`dist/viewer/`** over `node:http`, loopback only. (Not `dist/ui/` — `src/ui/` is the engine's *terminal* ui and tsc already compiles it there; see `ui/README.md`.) diff --git a/package-lock.json b/package-lock.json index 523b46d..f568884 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,6 +13,8 @@ ], "dependencies": { "@clack/prompts": "^1.3.0", + "@shikijs/core": "^4.4.3", + "@shikijs/engine-javascript": "^4.4.3", "commander": "^14.0.2", "fast-string-width": "^3.0.2", "fast-wrap-ansi": "^0.2.0", @@ -27,6 +29,7 @@ "codegraph": "dist/bin/codegraph.js" }, "devDependencies": { + "@shikijs/langs": "^4.4.3", "@types/better-sqlite3": "^7.6.0", "@types/node": "^20.19.30", "@types/picomatch": "^4.0.2", @@ -927,6 +930,82 @@ "win32" ] }, + "node_modules/@shikijs/core": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/@shikijs/core/-/core-4.4.3.tgz", + "integrity": "sha512-QCR4q2ZO/ILJEuwiBMel4wdcTDb1JGwfjKTxPDF6x8ixOaluPrVqIn06C99AcRPhmYlBR56d/Fb+GN58GzExpg==", + "license": "MIT", + "dependencies": { + "@shikijs/primitive": "4.4.3", + "@shikijs/types": "4.4.3", + "@shikijs/vscode-textmate": "^10.0.2", + "@types/hast": "^3.0.5", + "hast-util-to-html": "^9.0.5" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@shikijs/engine-javascript": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/@shikijs/engine-javascript/-/engine-javascript-4.4.3.tgz", + "integrity": "sha512-FbOjFJp9VLdo1Wevs10BBtVxiTWwNLqZh5Gkhjgda/ioL15YOgeSl9n+6XMa3qRlPQzfhFNe641SrynFHYG0nQ==", + "license": "MIT", + "dependencies": { + "@shikijs/types": "4.4.3", + "@shikijs/vscode-textmate": "^10.0.2", + "oniguruma-to-es": "^4.3.6" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@shikijs/langs": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/@shikijs/langs/-/langs-4.4.3.tgz", + "integrity": "sha512-ePic0yfAJGOF83D5wBHK/00EjK65oahBYxFk5epgq33WRv7X9UuxLEV8PtR0szC0z8dl7INIpIodB99JRFlR+A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@shikijs/types": "4.4.3" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@shikijs/primitive": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/@shikijs/primitive/-/primitive-4.4.3.tgz", + "integrity": "sha512-m0wBeLDQDeIxRdUmrCPdQqfuUamDwRL5isCfYbguKD6NiaKpVbsv+3J81DyIKgNW5h4WAIIr8T4EkgQrBBxvaQ==", + "license": "MIT", + "dependencies": { + "@shikijs/types": "4.4.3", + "@shikijs/vscode-textmate": "^10.0.2", + "@types/hast": "^3.0.5" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@shikijs/types": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/@shikijs/types/-/types-4.4.3.tgz", + "integrity": "sha512-UEJxmRR++MAGR6hugn0vgVS2W/6lWAts84FFSrnlH9sP0LNol7E5+NQ792pH8liWUhyMyjhTgSUH3k7iD7tc5g==", + "license": "MIT", + "dependencies": { + "@shikijs/vscode-textmate": "^10.0.2", + "@types/hast": "^3.0.5" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@shikijs/vscode-textmate": { + "version": "10.0.2", + "resolved": "https://registry.npmjs.org/@shikijs/vscode-textmate/-/vscode-textmate-10.0.2.tgz", + "integrity": "sha512-83yeghZ2xxin3Nj8z1NMd/NCuca+gsYXswywDy5bHvwlWL8tpTQmzGeUuHd9FC3E/SBEMvzJRwWEOz5gGes9Qg==", + "license": "MIT" + }, "node_modules/@sveltejs/acorn-typescript": { "version": "1.0.13", "resolved": "https://registry.npmjs.org/@sveltejs/acorn-typescript/-/acorn-typescript-1.0.13.tgz", @@ -964,6 +1043,24 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/hast": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/@types/hast/-/hast-3.0.5.tgz", + "integrity": "sha512-rp/ezSWaD1m44dPKICGhiskI13nVr7qTloFwDa/IYkhhf5nzwP+zIQcIJh3WIFSBOy/H1PzB40jPjMDksN4F+g==", + "license": "MIT", + "dependencies": { + "@types/unist": "*" + } + }, + "node_modules/@types/mdast": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/@types/mdast/-/mdast-4.0.4.tgz", + "integrity": "sha512-kGaNbPh1k7AFzgpud/gMdvIm5xuECykRR+JnWKQno9TAXVa6WIVCGTPvYGekIDL4uwCZQSYbUxNBSb1aUo79oA==", + "license": "MIT", + "dependencies": { + "@types/unist": "*" + } + }, "node_modules/@types/node": { "version": "20.19.33", "resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.33.tgz", @@ -989,6 +1086,18 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/unist": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/unist/-/unist-3.0.3.tgz", + "integrity": "sha512-ko/gIFJRv177XgZsZcBwnqJN5x/Gien8qNOn0D5bQU/zAzVf9Zt3BlcUiLqhV9y4ARk0GbT3tnUiPNgnTXzc/Q==", + "license": "MIT" + }, + "node_modules/@ungap/structured-clone": { + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.3.4.tgz", + "integrity": "sha512-JL+CF0GeLHyPWI0rXu7UnxgiuOm9UQWzadi0OYOJNhNO2q6EZElpwlgXkNkfU1PzANDHq3YcwKVZprdvS+BrbQ==", + "license": "ISC" + }, "node_modules/@vitest/expect": { "version": "2.1.9", "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-2.1.9.tgz", @@ -1156,6 +1265,16 @@ "node": ">=8" } }, + "node_modules/ccount": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/ccount/-/ccount-2.0.1.tgz", + "integrity": "sha512-eyrF0jiFpY+3drT6383f1qhkbGsLSifNAjA61IUjZjmLCWjItY6LB9ft9YhoDgwfmclB2zhu51Lc7+95b8NRAg==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/chai": { "version": "5.3.3", "resolved": "https://registry.npmjs.org/chai/-/chai-5.3.3.tgz", @@ -1173,6 +1292,26 @@ "node": ">=18" } }, + "node_modules/character-entities-html4": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/character-entities-html4/-/character-entities-html4-2.1.0.tgz", + "integrity": "sha512-1v7fgQRj6hnSwFpq1Eu0ynr/CDEw0rXo2B61qXrLNdHZmPKgb7fqS1a2JwF0rISo9q77jDI8VMEHoApn8qDoZA==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/character-entities-legacy": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/character-entities-legacy/-/character-entities-legacy-3.0.0.tgz", + "integrity": "sha512-RpPp0asT/6ufRm//AJVwpViZbGM/MkjQFxJccQRHmISF/22NBtsHqAWmL+/pmkPWoIUJdWyeVleTl1wydHATVQ==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/check-error": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/check-error/-/check-error-2.1.3.tgz", @@ -1213,6 +1352,16 @@ "resolved": "ui", "link": true }, + "node_modules/comma-separated-tokens": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/comma-separated-tokens/-/comma-separated-tokens-2.0.3.tgz", + "integrity": "sha512-Fu4hJdvzeylCfQPp9SGWidpzrMs7tTrlu6Vb8XGaRGck8QSNZJJp538Wrb60Lax4fPwR64ViY468OIUTbRlGZg==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/commander": { "version": "14.0.3", "resolved": "https://registry.npmjs.org/commander/-/commander-14.0.3.tgz", @@ -1260,6 +1409,15 @@ "node": ">=0.10.0" } }, + "node_modules/dequal": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz", + "integrity": "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/devalue": { "version": "5.9.1", "resolved": "https://registry.npmjs.org/devalue/-/devalue-5.9.1.tgz", @@ -1267,6 +1425,19 @@ "dev": true, "license": "MIT" }, + "node_modules/devlop": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/devlop/-/devlop-1.1.0.tgz", + "integrity": "sha512-RWmIqhcFf1lRYBvNmr7qTNuyCt/7/ns2jbpp1+PalgE/rDQcBT0fioSMUpJ93irlUhC5hrg4cYqe6U+0ImW0rA==", + "license": "MIT", + "dependencies": { + "dequal": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/es-module-lexer": { "version": "1.7.0", "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-1.7.0.tgz", @@ -1415,6 +1586,52 @@ "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, + "node_modules/hast-util-to-html": { + "version": "9.0.5", + "resolved": "https://registry.npmjs.org/hast-util-to-html/-/hast-util-to-html-9.0.5.tgz", + "integrity": "sha512-OguPdidb+fbHQSU4Q4ZiLKnzWo8Wwsf5bZfbvu7//a9oTYoqD/fWpe96NuHkoS9h0ccGOTe0C4NGXdtS0iObOw==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "@types/unist": "^3.0.0", + "ccount": "^2.0.0", + "comma-separated-tokens": "^2.0.0", + "hast-util-whitespace": "^3.0.0", + "html-void-elements": "^3.0.0", + "mdast-util-to-hast": "^13.0.0", + "property-information": "^7.0.0", + "space-separated-tokens": "^2.0.0", + "stringify-entities": "^4.0.0", + "zwitch": "^2.0.4" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-whitespace": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/hast-util-whitespace/-/hast-util-whitespace-3.0.0.tgz", + "integrity": "sha512-88JUN06ipLwsnv+dVn+OIYOvAuvBMy/Qoi6O7mQHxdPXpjy+Cd6xRkWwux7DKO+4sYILtLBRIKgsdpS2gQc7qw==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/html-void-elements": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/html-void-elements/-/html-void-elements-3.0.0.tgz", + "integrity": "sha512-bEqo66MRXsUGxWHV5IP0PUiAWwoEjba4VCzg0LjFJBpchPaTfyfCKTG6bc5F8ucKec3q5y6qOdGyYTSBEvhCrg==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/ignore": { "version": "7.0.5", "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.5.tgz", @@ -1464,6 +1681,116 @@ "@jridgewell/sourcemap-codec": "^1.5.5" } }, + "node_modules/mdast-util-to-hast": { + "version": "13.2.1", + "resolved": "https://registry.npmjs.org/mdast-util-to-hast/-/mdast-util-to-hast-13.2.1.tgz", + "integrity": "sha512-cctsq2wp5vTsLIcaymblUriiTcZd0CwWtCbLvrOzYCDZoWyMNV8sZ7krj09FSnsiJi3WVsHLM4k6Dq/yaPyCXA==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "@types/mdast": "^4.0.0", + "@ungap/structured-clone": "^1.0.0", + "devlop": "^1.0.0", + "micromark-util-sanitize-uri": "^2.0.0", + "trim-lines": "^3.0.0", + "unist-util-position": "^5.0.0", + "unist-util-visit": "^5.0.0", + "vfile": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-util-character": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/micromark-util-character/-/micromark-util-character-2.1.1.tgz", + "integrity": "sha512-wv8tdUTJ3thSFFFJKtpYKOYiGP2+v96Hvk4Tu8KpCAsTMs6yi+nVmGh1syvSCsaxz45J6Jbw+9DD6g97+NV67Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-encode": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-encode/-/micromark-util-encode-2.0.1.tgz", + "integrity": "sha512-c3cVx2y4KqUnwopcO9b/SCdo2O67LwJJ/UyqGfbigahfegL9myoEFoDYZgkT7f36T0bLrM9hZTAaAyH+PCAXjw==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-util-sanitize-uri": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-sanitize-uri/-/micromark-util-sanitize-uri-2.0.1.tgz", + "integrity": "sha512-9N9IomZ/YuGGZZmQec1MbgxtlgougxTodVwDzzEouPKo3qFWvymFHWcnDi2vzV1ff6kas9ucW+o3yzJK9YB1AQ==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-encode": "^2.0.0", + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-symbol": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-symbol/-/micromark-util-symbol-2.0.1.tgz", + "integrity": "sha512-vs5t8Apaud9N28kgCrRUdEed4UJ+wWNvicHLPxCa9ENlYuAY31M0ETy5y1vA33YoNPDFTghEbnh6efaE8h4x0Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-util-types": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/micromark-util-types/-/micromark-util-types-2.0.2.tgz", + "integrity": "sha512-Yw0ECSpJoViF1qTU4DC6NwtC4aWGt1EkzaQB8KPPyCRR8z9TWeV0HbEFGTO+ZY1wB22zmxnJqhPyTpOVCpeHTA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, "node_modules/mri": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/mri/-/mri-1.2.0.tgz", @@ -1514,6 +1841,23 @@ "node": ">=12.20.0" } }, + "node_modules/oniguruma-parser": { + "version": "0.12.2", + "resolved": "https://registry.npmjs.org/oniguruma-parser/-/oniguruma-parser-0.12.2.tgz", + "integrity": "sha512-6HVa5oIrgMC6aA6WF6XyyqbhRPJrKR02L20+2+zpDtO5QAzGHAUGw5TKQvwi5vctNnRHkJYmjAhRVQF2EKdTQw==", + "license": "MIT" + }, + "node_modules/oniguruma-to-es": { + "version": "4.3.6", + "resolved": "https://registry.npmjs.org/oniguruma-to-es/-/oniguruma-to-es-4.3.6.tgz", + "integrity": "sha512-csuQ9x3Yr0cEIs/Zgx/OEt9iBw9vqIunAPQkx19R/fiMq2oGVTgcMqO/V3Ybqefr1TBvosI6jU539ksaBULJyA==", + "license": "MIT", + "dependencies": { + "oniguruma-parser": "^0.12.2", + "regex": "^6.1.0", + "regex-recursion": "^6.0.2" + } + }, "node_modules/pathe": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/pathe/-/pathe-1.1.2.tgz", @@ -1580,6 +1924,16 @@ "node": "^10 || ^12 || >=14" } }, + "node_modules/property-information": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/property-information/-/property-information-7.2.0.tgz", + "integrity": "sha512-IAtzIB6sUiWaJYrX9smp3V46pBGbBeLFRGdh25kg1334VcBlD8HzhPeNIWQH9zhGmo2itIe25EHt9dQP7G5hmg==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/readdirp": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-4.1.2.tgz", @@ -1594,6 +1948,30 @@ "url": "https://paulmillr.com/funding/" } }, + "node_modules/regex": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/regex/-/regex-6.1.0.tgz", + "integrity": "sha512-6VwtthbV4o/7+OaAF9I5L5V3llLEsoPyq9P1JVXkedTP33c7MfCG0/5NOPcSJn0TzXcG9YUrR0gQSWioew3LDg==", + "license": "MIT", + "dependencies": { + "regex-utilities": "^2.3.0" + } + }, + "node_modules/regex-recursion": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/regex-recursion/-/regex-recursion-6.0.2.tgz", + "integrity": "sha512-0YCaSCq2VRIebiaUviZNs0cBz1kg5kVS2UKUfNIx8YVs1cN3AV7NTctO5FOKBA+UT2BPJIWZauYHPqJODG50cg==", + "license": "MIT", + "dependencies": { + "regex-utilities": "^2.3.0" + } + }, + "node_modules/regex-utilities": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/regex-utilities/-/regex-utilities-2.3.0.tgz", + "integrity": "sha512-8VhliFJAWRaUiVvREIiW2NXXTmHs4vMNnSzuJVhscgmGav3g9VDxLrQndI3dZZVVdp0ZO/5v0xmX516/7M9cng==", + "license": "MIT" + }, "node_modules/rollup": { "version": "4.57.1", "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.57.1.tgz", @@ -1675,6 +2053,16 @@ "node": ">=0.10.0" } }, + "node_modules/space-separated-tokens": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/space-separated-tokens/-/space-separated-tokens-2.0.2.tgz", + "integrity": "sha512-PEGlAwrG8yXGXRjW32fGbg66JAlOAwbObuqVoJpv/mRgoWDQfgH1wDPvtzWyUSNAXBGSk8h755YDbbcEy3SH2Q==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/stackback": { "version": "0.0.2", "resolved": "https://registry.npmjs.org/stackback/-/stackback-0.0.2.tgz", @@ -1689,6 +2077,20 @@ "dev": true, "license": "MIT" }, + "node_modules/stringify-entities": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/stringify-entities/-/stringify-entities-4.0.4.tgz", + "integrity": "sha512-IwfBptatlO+QCJUo19AqvrPNqlVMpW9YEL2LIVY+Rpv2qsjCGxaDLNRgeGsQWJhfItebuJhsGSLjaBbNSQ+ieg==", + "license": "MIT", + "dependencies": { + "character-entities-html4": "^2.0.0", + "character-entities-legacy": "^3.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/svelte": { "version": "5.56.10", "resolved": "https://registry.npmjs.org/svelte/-/svelte-5.56.10.tgz", @@ -1813,6 +2215,16 @@ "tree-sitter-wasms": "^0.1.11" } }, + "node_modules/trim-lines": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/trim-lines/-/trim-lines-3.0.1.tgz", + "integrity": "sha512-kRj8B+YHZCc9kQYdWfJB2/oUl9rA99qbowYYBtr4ui4mZyAQ2JpvVBd/6U2YloATfqBhBTSMhTpgBHtU0Mf3Rg==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/typescript": { "version": "5.9.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", @@ -1835,6 +2247,102 @@ "dev": true, "license": "MIT" }, + "node_modules/unist-util-is": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/unist-util-is/-/unist-util-is-6.0.1.tgz", + "integrity": "sha512-LsiILbtBETkDz8I9p1dQ0uyRUWuaQzd/cuEeS1hoRSyW5E5XGmTzlwY1OrNzzakGowI9Dr/I8HVaw4hTtnxy8g==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-position": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/unist-util-position/-/unist-util-position-5.0.0.tgz", + "integrity": "sha512-fucsC7HjXvkB5R3kTCO7kUjRdrS0BJt3M/FPxmHMBOm8JQi2BsHAHFsy27E0EolP8rp0NzXsJ+jNPyDWvOJZPA==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-stringify-position": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/unist-util-stringify-position/-/unist-util-stringify-position-4.0.0.tgz", + "integrity": "sha512-0ASV06AAoKCDkS2+xw5RXJywruurpbC4JZSm7nr7MOt1ojAzvyyaO+UxZf18j8FCF6kmzCZKcAgN/yu2gm2XgQ==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-visit": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/unist-util-visit/-/unist-util-visit-5.1.0.tgz", + "integrity": "sha512-m+vIdyeCOpdr/QeQCu2EzxX/ohgS8KbnPDgFni4dQsfSCtpz8UqDyY5GjRru8PDKuYn7Fq19j1CQ+nJSsGKOzg==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "unist-util-is": "^6.0.0", + "unist-util-visit-parents": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-visit-parents": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/unist-util-visit-parents/-/unist-util-visit-parents-6.0.2.tgz", + "integrity": "sha512-goh1s1TBrqSqukSc8wrjwWhL0hiJxgA8m4kFxGlQ+8FYQ3C/m11FcTs4YYem7V664AhHVvgoQLk890Ssdsr2IQ==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "unist-util-is": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/vfile": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/vfile/-/vfile-6.0.3.tgz", + "integrity": "sha512-KzIbH/9tXat2u30jf+smMwFCsno4wHVdNmzFyL+T/L3UGqqk6JKfVqOFOZEpZSHADH1k40ab6NUIXZq422ov3Q==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "vfile-message": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/vfile-message": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/vfile-message/-/vfile-message-4.0.3.tgz", + "integrity": "sha512-QTHzsGd1EhbZs4AsQ20JX1rC3cOlt/IWJruk893DfLRr57lcnOeMaWG4K0JrRta4mIJZKth2Au3mM3u03/JWKw==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "unist-util-stringify-position": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/vite": { "version": "5.4.21", "resolved": "https://registry.npmjs.org/vite/-/vite-5.4.21.tgz", @@ -2043,6 +2551,16 @@ "dev": true, "license": "MIT" }, + "node_modules/zwitch": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/zwitch/-/zwitch-2.0.4.tgz", + "integrity": "sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "ui": { "name": "codegraph-ui", "version": "0.0.0", diff --git a/package.json b/package.json index 5bb2d81..8e16340 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,7 @@ "ui" ], "scripts": { - "build": "tsc && npm run copy-assets && npm run build:ui && node -e \"require('fs').chmodSync('dist/bin/codegraph.js', 0o755)\"", + "build": "tsc && npm run copy-assets && npm run build:textmate && npm run build:ui && node -e \"require('fs').chmodSync('dist/bin/codegraph.js', 0o755)\"", "build:ui": "npm run build --workspace ui && node scripts/check-ui-build.mjs", "preuninstall": "node dist/bin/uninstall.js", "copy-assets": "node -e \"const fs=require('fs');fs.mkdirSync('dist/db',{recursive:true});fs.copyFileSync('src/db/schema.sql','dist/db/schema.sql');fs.mkdirSync('dist/extraction/wasm',{recursive:true});fs.readdirSync('src/extraction/wasm').filter(f=>f.endsWith('.wasm')).forEach(f=>fs.copyFileSync('src/extraction/wasm/'+f,'dist/extraction/wasm/'+f))\"", @@ -31,7 +31,8 @@ "test:watch": "vitest", "test:eval": "vitest run __tests__/evaluation/", "eval": "npm run build && npx tsx __tests__/evaluation/runner.ts", - "clean": "node -e \"const fs=require('fs');fs.rmSync('dist',{recursive:true,force:true})\"" + "clean": "node -e \"const fs=require('fs');fs.rmSync('dist',{recursive:true,force:true})\"", + "build:textmate": "node scripts/prune-grammars.mjs" }, "keywords": [ "code-intelligence", @@ -42,6 +43,8 @@ "license": "MIT", "dependencies": { "@clack/prompts": "^1.3.0", + "@shikijs/core": "^4.4.3", + "@shikijs/engine-javascript": "^4.4.3", "commander": "^14.0.2", "fast-string-width": "^3.0.2", "fast-wrap-ansi": "^0.2.0", @@ -53,6 +56,7 @@ "web-tree-sitter": "^0.25.3" }, "devDependencies": { + "@shikijs/langs": "^4.4.3", "@types/better-sqlite3": "^7.6.0", "@types/node": "^20.19.30", "@types/picomatch": "^4.0.2", diff --git a/scripts/check-ui-build.mjs b/scripts/check-ui-build.mjs index 80db622..eebcbdd 100644 --- a/scripts/check-ui-build.mjs +++ b/scripts/check-ui-build.mjs @@ -13,6 +13,11 @@ * where tsc puts the TERMINAL ui, so a mis-pointed outDir silently deletes * modules the CLI requires at startup. * + * The pruned TextMate grammars in dist/textmate/ are checked the same way and + * for the same reason: without them every file the viewer shows falls back to + * unhighlighted text, which looks like a styling bug rather than a missing + * build step. + * * Usage: node scripts/check-ui-build.mjs [--root ] * --root directory holding dist/ (default: the repo root). The release * bundler points this at its staging dir to verify the copy. @@ -90,4 +95,36 @@ for (const compiled of [join('bin', 'codegraph.js'), 'index.js', join('ui', 'shi } } -console.log(`[check-ui-build] dist/viewer ok (index.html + ${assets} referenced asset(s)); dist/ engine intact`); +// The pruned syntax grammars (scripts/prune-grammars.mjs). Their absence is +// survivable at runtime — source is served unhighlighted — which is exactly why +// it has to fail here: nothing downstream would ever complain. +const textmateDir = join(root, 'dist', 'textmate'); +const manifestPath = join(textmateDir, 'manifest.json'); +if (!existsSync(manifestPath)) { + fail( + `missing ${manifestPath}`, + staged + ? 'this bundle was assembled before the syntax grammars were added, or dist/textmate was not copied' + : 'run `npm run build:textmate` (it needs @shikijs/langs from devDependencies)' + ); +} + +const manifest = JSON.parse(readFileSync(manifestPath, 'utf8')); +const languages = Object.keys(manifest.languages ?? {}); +if (languages.length === 0) fail('dist/textmate/manifest.json lists no languages'); + +const grammarFiles = new Set(Object.values(manifest.languages).flat()); +const missingGrammars = [...grammarFiles].filter( + (name) => !existsSync(join(textmateDir, `${name}.json`)) +); +if (missingGrammars.length > 0) { + fail( + `dist/textmate is missing ${missingGrammars.length} grammar file(s): ${missingGrammars.join(', ')}`, + 'the prune step was interrupted or dist/textmate was copied incompletely' + ); +} + +console.log( + `[check-ui-build] dist/viewer ok (index.html + ${assets} referenced asset(s)); ` + + `dist/textmate ok (${languages.length} languages, ${grammarFiles.size} grammars); dist/ engine intact` +); diff --git a/scripts/prune-grammars.mjs b/scripts/prune-grammars.mjs new file mode 100644 index 0000000..c22f931 --- /dev/null +++ b/scripts/prune-grammars.mjs @@ -0,0 +1,110 @@ +#!/usr/bin/env node +/** + * Write the TextMate grammars the viewer needs into `dist/textmate/`. + * + * Shiki carries about 700 grammars, 11 MB of JSON. The engine indexes about 40 + * languages. Shipping the other 660 to every user of a code-intelligence CLI is + * not a trade worth making, so `@shikijs/langs` stays a devDependency and this + * step copies out exactly the closure the viewer can reach: every grammar named + * in `src/ui-server/highlight/languages.ts`, plus every grammar those embed + * (`vue` needs html, css, typescript, json and four Vue-specific ones before it + * will highlight a single-file component). + * + * Run from `npm run build`, after `tsc`, because the language table is read + * from the compiled `dist/ui-server/highlight/languages.js` rather than being + * duplicated here — one source of truth for what ships. + * + * Output: + * dist/textmate/manifest.json grammar id -> files to load, deps first + * dist/textmate/.json one TextMate grammar, verbatim + */ + +import { createRequire } from 'node:module'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); +const OUT = path.join(ROOT, 'dist', 'textmate'); +const require = createRequire(import.meta.url); + +function fail(message) { + process.stderr.write(`[prune-grammars] ${message}\n`); + process.exit(1); +} + +const languagesModule = path.join(ROOT, 'dist', 'ui-server', 'highlight', 'languages.js'); +if (!fs.existsSync(languagesModule)) { + fail(`${path.relative(ROOT, languagesModule)} is missing — run tsc before this script.`); +} +const { REQUIRED_GRAMMARS } = require(languagesModule); +if (!Array.isArray(REQUIRED_GRAMMARS) || REQUIRED_GRAMMARS.length === 0) { + fail('REQUIRED_GRAMMARS is empty — the language table did not compile as expected.'); +} + +const shikiVersion = JSON.parse( + fs.readFileSync(path.join(ROOT, 'node_modules', '@shikijs', 'langs', 'package.json'), 'utf-8') +).version; + +/** + * Load one Shiki language module and return its registrations. + * + * The default export is already the flattened chain — embedded grammars first, + * the language itself last — which is exactly the order Shiki's registry needs + * to resolve `embeddedLangs`. Keeping that order is the whole reason the + * manifest stores a list rather than a single filename. + */ +async function loadChain(id) { + const mod = await import(`@shikijs/langs/${id}`); + const chain = mod.default; + if (!Array.isArray(chain) || chain.length === 0) { + fail(`@shikijs/langs/${id} did not export a grammar array.`); + } + return chain; +} + +fs.rmSync(OUT, { recursive: true, force: true }); +fs.mkdirSync(OUT, { recursive: true }); + +const manifest = { shikiVersion, languages: {} }; +const written = new Map(); +let bytes = 0; + +for (const id of REQUIRED_GRAMMARS) { + let chain; + try { + chain = await loadChain(id); + } catch (err) { + fail(`could not load the ${id} grammar: ${err?.message ?? err}`); + } + + const files = []; + for (const grammar of chain) { + // A chain can name the same dependency more than once (Vue reaches + // JavaScript four different ways). Registering it twice is wasted work and + // a confusing manifest; the FIRST occurrence is the one that keeps the + // dependencies-before-dependents ordering intact. + // `name` is the grammar's own id and is unique across the bundle, so two + // languages that embed html write (and share) exactly one html.json. + const file = grammar.name; + if (typeof file !== 'string' || !/^[\w.+-]+$/.test(file)) { + fail(`the ${id} chain contains a grammar with an unusable name: ${JSON.stringify(file)}`); + } + if (files.includes(file)) continue; + if (!written.has(file)) { + const json = JSON.stringify(grammar); + fs.writeFileSync(path.join(OUT, `${file}.json`), json); + written.set(file, json.length); + bytes += json.length; + } + files.push(file); + } + manifest.languages[id] = files; +} + +fs.writeFileSync(path.join(OUT, 'manifest.json'), JSON.stringify(manifest, null, 2)); + +process.stdout.write( + `[prune-grammars] ${REQUIRED_GRAMMARS.length} languages -> ${written.size} grammars, ` + + `${(bytes / 1024 / 1024).toFixed(1)} MB in dist/textmate (shiki ${shikiVersion})\n` +); diff --git a/src/ui-server/api/index.ts b/src/ui-server/api/index.ts index cef8be1..58e73f5 100644 --- a/src/ui-server/api/index.ts +++ b/src/ui-server/api/index.ts @@ -87,7 +87,9 @@ const API_INDEX = { export function createGraphApi(options: GraphApiOptions): GraphApi { const session = new GraphSession(options.projectRoot); - const handler: UiApiHandler = (_req, res, ctx) => { + // Async because `/api/source` highlights: everything else answers straight + // out of SQLite and resolves on the same tick. + const handler: UiApiHandler = async (_req, res, ctx) => { const route = normalize(ctx.pathname); try { switch (route) { @@ -104,7 +106,7 @@ export function createGraphApi(options: GraphApiOptions): GraphApi { case '/api/nodes': return ok(res, buildNodeRefs(session.acquire(), ctx.query), ctx.method); case '/api/source': - return ok(res, buildSource(session.acquire(), ctx.projectRoot, ctx.query), ctx.method); + return ok(res, await buildSource(session.acquire(), ctx.projectRoot, ctx.query), ctx.method); default: return dispatchPathRoutes(route, res, ctx, session); } diff --git a/src/ui-server/api/source.ts b/src/ui-server/api/source.ts index 650b1a2..072702a 100644 --- a/src/ui-server/api/source.ts +++ b/src/ui-server/api/source.ts @@ -30,6 +30,7 @@ import * as path from 'path'; import type { FileRecord } from '../../types'; import type { CodeGraph } from '../../index'; import { resolveProjectFile } from '../security'; +import { highlightLines, type HighlightResult } from '../highlight'; import { ApiError, badRequest, intParam, notFound, textParam } from './respond'; /** @@ -186,13 +187,23 @@ export interface SourceResult { lines?: string[]; truncated?: boolean; reason?: string; + /** + * The same lines, classified for the code block — one entry per line, each a + * list of `[classId, text]` pairs indexed into `highlight.classes`. + * + * It rides with the slice rather than living behind its own endpoint because + * the two are only ever wanted together, and because a second round-trip + * would let the viewer paint unhighlighted source and then reflow it. Absent + * whenever `lines` is — a drifted file is not served at all. + */ + highlight?: HighlightResult; } -export function buildSource( +export async function buildSource( cg: CodeGraph, projectRoot: string, query: URLSearchParams -): SourceResult { +): Promise { const requested = textParam(query, 'file'); // Refusal first, index lookup second — see `resolveRequestedFile`. const { record, storedPath, absolute } = resolveRequestedFile(cg, projectRoot, requested); @@ -265,13 +276,21 @@ export function buildSource( const start = from; const requestedEnd = to === 0 ? all.length : Math.min(to, all.length); const end = Math.min(requestedEnd, start + MAX_SOURCE_LINES - 1); + const slice = all.slice(start - 1, end); return { ...base, totalLines: all.length, from: start, to: end, - lines: all.slice(start - 1, end), + lines: slice, truncated: end < requestedEnd, + // Keyed on the content hash, so the cache is invalidated by the file + // changing rather than by a clock, and two viewers looking at the same + // symbol share one tokenisation. + highlight: await highlightLines(slice, { + language: record.language, + cacheKey: `${record.contentHash}:${start}:${end}`, + }), }; } diff --git a/src/ui-server/highlight/grammars.ts b/src/ui-server/highlight/grammars.ts new file mode 100644 index 0000000..a8d2d08 --- /dev/null +++ b/src/ui-server/highlight/grammars.ts @@ -0,0 +1,86 @@ +/** + * Finding and reading the pruned TextMate grammars on disk. + * + * Shiki ships 700-odd grammars; the engine indexes 40-odd languages. The build + * writes only the closure those 40 need — including the grammars they embed, so + * a `.vue` file still gets its `