diff --git a/CHANGELOG.md b/CHANGELOG.md index 3f7a892..e558807 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -139,6 +139,8 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). #### MCP / indexing +- C++ functions following anonymous namespaces containing raw-string templates are now indexed correctly, even when template text resembles an unfinished macro call. (#1505) + - Indexing now warns when parser errors leave a file with no symbols, including C++ raw strings with 16-character delimiters, so missing code is no longer silent. (#1522) - `codegraph index ` now refuses uninitialized paths and names the nearest initialized parent instead of silently rebuilding it; thanks @danusha2345. (#1524, #1689) diff --git a/__tests__/cpp-raw-string-preparse-1505.test.ts b/__tests__/cpp-raw-string-preparse-1505.test.ts new file mode 100644 index 0000000..982e802 --- /dev/null +++ b/__tests__/cpp-raw-string-preparse-1505.test.ts @@ -0,0 +1,177 @@ +import { beforeAll, describe, expect, it } from 'vitest'; +import { extractFromSource } from '../src/extraction'; +import { getParser, initGrammars, loadGrammarsForLanguages } from '../src/extraction/grammars'; +import { + blankCppAnnotationMacroCalls, + blankCppInlineAnnotationMacros, + blankCStatementMacroCalls, + blankCTypeKeywordArgs, + blankCFileScopePrefixedDeclMacros, + blankCParameterizedAnnotationMacros, + blankCDesignatedMacroArgs, + cExtractor, + cppExtractor, +} from '../src/extraction/languages/c-cpp'; + +// The original parses cleanly; #1505 is preParse corrupting its short delimiter, +// distinct from the vendored grammar's 16-character delimiter error (#1522). +function scaffoldSource(delimiter = 'GEN'): string { + return `#include + +namespace { +const char* kTpl = R"${delimiter}( +DECLARE_THING( +struct Ignored { int v; }; +int nested_fn() { return 1; } +)${delimiter}"; +} + +int create_scaffold(int x) { + return x; +} + +int helper_after(int y) { + return y + 1; +} +`; +} + +const annotationBlankers = [ + { name: 'line-leading annotations', blank: blankCppAnnotationMacroCalls }, + { name: 'inline annotations', blank: blankCppInlineAnnotationMacros }, +]; + +describe('C/C++ raw strings survive preParse (#1505)', () => { + beforeAll(async () => { + await initGrammars(); + await loadGrammarsForLanguages(['cpp']); + }); + + it('indexes both functions after the anonymous namespace without introducing a parse error', () => { + const source = scaffoldSource(); + const rewritten = cppExtractor.preParse!(source, 'scaffold.cpp'); + for (const text of [source, rewritten]) { + const tree = getParser('cpp')!.parse(text)!; + try { + expect(tree.rootNode.hasError).toBe(false); + } finally { + tree.delete(); + } + } + const result = extractFromSource('scaffold.cpp', source); + expect(result.nodes.filter((node) => node.kind === 'function').map((node) => node.name)) + .toEqual(['create_scaffold', 'helper_after']); + expect(result.nodes.some((node) => node.name === 'Ignored')).toBe(false); + expect(result.errors).toEqual([]); + expect(rewritten).toBe(source); + }); + + it('keeps the raw-string terminator at its original offset', () => { + const source = scaffoldSource('TAG'); + const closer = source.indexOf(')TAG"'); + const blanked = blankCppAnnotationMacroCalls(source); + expect(blanked.slice(closer, closer + 5)).toBe(')TAG"'); + expect(blanked).toBe(source); + }); + + describe.each(annotationBlankers)('$name', ({ blank }) => { + it.each(['R', 'LR', 'u8R', 'uR', 'UR'])('leaves %s raw-string contents untouched', (prefix) => { + const source = `const auto* text = ${prefix}"TAG( +DECLARE_THING( +"quoted ) text" and 'characters' and a backslash \\ +)OTHER" +value UPARAM(ref) UE_DEPRECATED( +)TAG"; +`; + expect(blank(source)).toBe(source); + }); + + it.each(['', 'FIFTEEN_CHARS__', 'SIXTEEN_CHARS___'])('protects delimiter %j with CRLF', (delimiter) => { + const source = `const char* text = R"${delimiter}(\r\nUE_DEPRECATED(\r\n)${delimiter}";\r\n`; + expect(blank(source)).toBe(source); + }); + + it('leaves an unterminated raw string untouched', () => { + const source = 'const char* text = R"TAG(\nUE_DEPRECATED(1)\nint example;'; + expect(blank(source)).toBe(source); + }); + }); + + it.each([ + { name: 'line-leading', blank: blankCppAnnotationMacroCalls, head: '', macro: 'ANNOTATE', tail: '\nint helper_after() { return 1; }\n' }, + { name: 'inline', blank: blankCppInlineAnnotationMacros, head: 'using Alias ', macro: 'UE_DEPRECATED', tail: ' = int;\n' }, + { name: 'C parameterized', blank: blankCParameterizedAnnotationMacros, head: 'static void ', macro: '__section', tail: ' helper_after(void) {}\n' }, + { name: 'C iterator', blank: blankCStatementMacroCalls, head: 'void iterate() {\n ', macro: 'for_each_item', tail: ' {\n visit();\n }\n}\n' }, + ])('balances a genuine $name macro containing a raw-string argument', ({ blank, head, macro, tail }) => { + const annotation = `${macro}(R"TAG(" ) unbalanced ( " \\ +UE_DEPRECATED( +)TAG")`; + expect(blank(head + annotation + tail)) + .toBe(head + annotation.replace(/[^\r\n]/g, ' ') + tail); + }); + + it('balances a C declaration macro with a raw-string argument', () => { + const macro = 'static DECLARE_THING(R"TAG(" ) unbalanced ( ")TAG");'; + const tail = '\nint helper_after(void) {}\n'; + expect(blankCFileScopePrefixedDeclMacros(macro + tail)) + .toBe(' '.repeat(macro.length) + tail); + }); + + it('preserves a raw argument while blanking a later C type-keyword argument', () => { + const literal = 'R"TAG(" ), struct Fake, ( ")TAG"'; + const source = `take(${literal}, struct RealType);`; + expect(blankCTypeKeywordArgs(source)).toBe(`take(${literal}, RealType);`); + }); + + it('only counts designators outside raw arguments when blanking a C macro call', () => { + const literal = 'R"TAG(" ) .fake = 1 ( ")TAG"'; + const head = 'void reset(void) {\n RESET_THING('; + const tail = ');\n}\n'; + expect(blankCDesignatedMacroArgs(head + literal + tail)).toBe(head + literal + tail); + const args = literal + ', .field = 1'; + expect(blankCDesignatedMacroArgs(head + args + tail)) + .toBe(head + ' '.repeat(args.length) + tail); + }); + + it.each([ + { name: 'C iterator macros', blank: blankCStatementMacroCalls }, + { name: 'C type arguments', blank: blankCTypeKeywordArgs }, + { name: 'C declaration macros', blank: blankCFileScopePrefixedDeclMacros }, + { name: 'C parameterized annotations', blank: blankCParameterizedAnnotationMacros }, + { name: 'C designated initializer arguments', blank: blankCDesignatedMacroArgs }, + { name: 'C preParse', blank: (source: string) => cExtractor.preParse!(source) }, + { name: 'C++ preParse', blank: (source: string) => cppExtractor.preParse!(source, 'template.cpp') }, + ])('$name leaves macro-like raw-string contents untouched', ({ blank }) => { + const source = `const auto* text = u8R"TAG( + for_each_item(item, list) { + visit(item); + } +static DECLARE_THING(value); +use(struct Example); + RESET_THING(.field = 1); +class EXAMPLE_API Example { +FORCEINLINE int example() {} +}; +FMT_BEGIN_NAMESPACE +int example; +__section( +)TAG"; +`; + expect(blank(source)).toBe(source); + }); + + it('ignores raw-string openers in comments and ordinary literals, then resumes blanking after a real raw string', () => { + const before = [ + '// R"COMMENT(', + '/* LR"COMMENT( */', + 'const char* quoted = "escaped R\\"STRING(";', + "const auto digit = 1'000;", + 'const char quote = \'"\';', + scaffoldSource(), + ].join('\n'); + const annotation = 'UPROPERTY(EditAnywhere)'; + const tail = '\nint actual_field;\n'; + expect(blankCppAnnotationMacroCalls(before + annotation + tail)) + .toBe(before + ' '.repeat(annotation.length) + tail); + }); +}); diff --git a/src/extraction/languages/c-cpp.ts b/src/extraction/languages/c-cpp.ts index 7dff163..afce3a0 100644 --- a/src/extraction/languages/c-cpp.ts +++ b/src/extraction/languages/c-cpp.ts @@ -480,6 +480,54 @@ export function blankMetalAttributes(source: string): string { return source.replace(METAL_ATTRIBUTE_RE, (m) => ' '.repeat(m.length)); } +/** + * Hide C++ raw literals from the offset-preserving preParse scans (#1505). + * Neither macro-shaped text in a raw body nor its `)delim"` closer is code. + * Mask the whole literal with non-whitespace, non-paren tokens, keeping line + * endings, so even line-based scanners treat a multiline raw argument as opaque. + * Restore untouched bytes afterward; a real enclosing annotation can still be + * removed in full. The pipeline masks once, and individual paren blankers also + * use this helper so they are safe when called directly. + */ +function maskCppRawStrings(source: string): { source: string; restore: (blanked: string) => string } { + const unchanged = { source, restore: (blanked: string): string => blanked }; + if (source.indexOf('R"') === -1) return unchanged; + // Skip comments and ordinary literals before looking for a raw opener. The + // char-literal boundary leaves numeric digit separators (1'000) alone. + const re = /\/\/[^\r\n]*|\/\*[\s\S]*?(?:\*\/|$)|\b(?:u8|[LuU])?R"([^ \t\v\f\r\n()\\]{0,16})\(|"(?:\\[\s\S]|[^"\\])*(?:"|$)|(? = []; + const parts: string[] = []; + let last = 0; + let m: RegExpExecArray | null; + while ((m = re.exec(source)) !== null) { + if (m[1] === undefined) continue; + const closer = `)${m[1]}"`; + const close = source.indexOf(closer, re.lastIndex); + // An unterminated raw literal owns the rest of the file too. + const end = close < 0 ? source.length : close + closer.length; + spans.push({ start: m.index, end }); + parts.push(source.slice(last, m.index), source.slice(m.index, end).replace(/[^\r\n]/g, '\0')); + last = re.lastIndex = end; + } + if (spans.length === 0) return unchanged; + parts.push(source.slice(last)); + const masked = parts.join(''); + return { + source: masked, + restore(blanked): string { + // A length-changing rewrite cannot be restored at the original offsets. + if (blanked === masked || blanked.length !== source.length) return source; + const chars = blanked.split(''); + for (const { start, end } of spans) { + for (let i = start; i < end; i++) { + if (chars[i] === '\0') chars[i] = source[i] as string; + } + } + return chars.join(''); + }, + }; +} + /** * Blank annotation-style macro invocations that decorate a declaration but carry * NO terminating semicolon — the pervasive Unreal-Engine reflection markup @@ -519,6 +567,8 @@ export function blankMetalAttributes(source: string): string { */ export function blankCppAnnotationMacroCalls(source: string): string { if (!/^[ \t]*[A-Z][A-Z0-9_]{2,}\s*\(/m.test(source)) return source; + const rawStrings = maskCppRawStrings(source); + source = rawStrings.source; const chars = source.split(''); const re = /^([ \t]*)([A-Z][A-Z0-9_]{2,})(\s*)\(/gm; let m: RegExpExecArray | null; @@ -556,7 +606,7 @@ export function blankCppAnnotationMacroCalls(source: string): string { } re.lastIndex = end; } - return chars.join(''); + return rawStrings.restore(chars.join('')); } /** @@ -670,6 +720,8 @@ export function blankCppApiPrefixMacros(source: string): string { const CPP_INLINE_ANNOTATION_RE = /\b(?:UMETA|UPARAM|UE_DEPRECATED\w*)\s*\(/g; export function blankCppInlineAnnotationMacros(source: string): string { if (!/\b(?:UMETA|UPARAM|UE_DEPRECATED)/.test(source)) return source; + const rawStrings = maskCppRawStrings(source); + source = rawStrings.source; const chars = source.split(''); const re = new RegExp(CPP_INLINE_ANNOTATION_RE.source, 'g'); let m: RegExpExecArray | null; @@ -700,7 +752,7 @@ export function blankCppInlineAnnotationMacros(source: string): string { } re.lastIndex = end; } - return chars.join(''); + return rawStrings.restore(chars.join('')); } /** @@ -824,6 +876,8 @@ function restoreDirectiveLines(original: string, blanked: string): string { * or by content, for CUDA living in `.h`/`.hpp` headers). Offset-preserving; * directive lines are restored at the end (see restoreDirectiveLines). */ function preParseCppSource(source: string, filePath?: string): string { + const rawStrings = maskCppRawStrings(source); + source = rawStrings.source; // blankCLeadingAttrMacros runs AFTER the api-prefix blank so a stacked // `FMT_NORETURN FMT_API void f(…)` reduces to the `MACRO Ret name(` shape // it matches (the _API token is already spaces by then). @@ -842,7 +896,7 @@ function preParseCppSource(source: string, filePath?: string): string { } else if (lower.endsWith('.cu') || lower.endsWith('.cuh') || looksLikeCudaSource(source)) { blanked = blankCudaConstructs(blanked); } - return restoreDirectiveLines(source, blanked); + return rawStrings.restore(restoreDirectiveLines(source, blanked)); } /** @@ -970,6 +1024,8 @@ const C_STMT_MACRO_KEYWORDS = new Set([ 'if', 'while', 'for', 'switch', 'return', 'do', 'else', 'sizeof', ]); export function blankCStatementMacroCalls(source: string): string { + const rawStrings = maskCppRawStrings(source); + source = rawStrings.source; const lines = source.split('\n'); let changed = false; const content = (l: string): string => l.replace(/\r$/, '').trim(); @@ -1073,7 +1129,7 @@ export function blankCStatementMacroCalls(source: string): string { } changed = true; } - return changed ? lines.join('\n') : source; + return rawStrings.restore(changed ? lines.join('\n') : source); } /** @@ -1244,8 +1300,8 @@ const C_PARAM_ANNOTATION_RE = new RegExp( ); export function blankCParameterizedAnnotationMacros(source: string): string { if (source.indexOf('__') === -1) return source; - C_PARAM_ANNOTATION_RE.lastIndex = 0; - if (!C_PARAM_ANNOTATION_RE.test(source)) return source; + const rawStrings = maskCppRawStrings(source); + source = rawStrings.source; C_PARAM_ANNOTATION_RE.lastIndex = 0; let result = ''; let last = 0; @@ -1265,7 +1321,7 @@ export function blankCParameterizedAnnotationMacros(source: string): string { result += source.slice(last, start) + source.slice(start, end).replace(/[^\n\r]/g, ' '); last = end; } - return result + source.slice(last); + return rawStrings.restore(result + source.slice(last)); } /** @@ -1308,6 +1364,8 @@ const C_TYPE_ARG_OPENER_RE = /^(struct|union|enum)([ \t\r\n]+)([A-Za-z_]\w*)([ \ const C_TYPE_ARG_SCAN_CAP = 600; export function blankCTypeKeywordArgs(source: string): string { if (!/\b(?:struct|union|enum)[ \t\r\n]/.test(source)) return source; + const rawStrings = maskCppRawStrings(source); + source = rawStrings.source; let chars: string[] | null = null; C_TYPE_ARG_HEAD_RE.lastIndex = 0; let m: RegExpExecArray | null; @@ -1375,7 +1433,7 @@ export function blankCTypeKeywordArgs(source: string): string { atArgStart = false; } } - return chars ? chars.join('') : source; + return rawStrings.restore(chars ? chars.join('') : source); } /** @@ -1403,6 +1461,8 @@ export function blankCTypeKeywordArgs(source: string): string { const C_PREFIXED_DECL_MACRO_RE = /^[ \t]*(?:static|extern)[ \t]+[A-Z][A-Z0-9_]{2,}[ \t]*\(/; export function blankCFileScopePrefixedDeclMacros(source: string): string { if (!/^[ \t]*(?:static|extern)[ \t]+[A-Z]/m.test(source)) return source; + const rawStrings = maskCppRawStrings(source); + source = rawStrings.source; const lines = source.split('\n'); let changed = false; for (let i = 0; i < lines.length; i++) { @@ -1437,7 +1497,7 @@ export function blankCFileScopePrefixedDeclMacros(source: string): string { lines[i] = line.replace(/[^\n\r]/g, ' '); changed = true; } - return changed ? lines.join('\n') : source; + return rawStrings.restore(changed ? lines.join('\n') : source); } /** @@ -1568,6 +1628,8 @@ export function blankCNamedVariadicDefineDots(source: string): string { */ export function blankCDesignatedMacroArgs(source: string): string { if (source.indexOf('=') === -1) return source; + const rawStrings = maskCppRawStrings(source); + source = rawStrings.source; const out = source.split(''); const re = /^[ \t]*([A-Z_][A-Z0-9_]*)\s*\(/gm; let m: RegExpExecArray | null; @@ -1589,10 +1651,12 @@ export function blankCDesignatedMacroArgs(source: string): string { for (let k = open + 1; k < close; k++) if (out[k] !== '\n') out[k] = ' '; re.lastIndex = close; } - return out.join(''); + return rawStrings.restore(out.join('')); } function preParseCSource(source: string): string { + const rawStrings = maskCppRawStrings(source); + source = rawStrings.source; const inner = blankCDesignatedMacroArgs(blankCKernelAnnotations(blankCCplusplusGuardBodies(source))); let blanked = blankCLeadingAttrMacros( blankLoneMacroLines( @@ -1618,7 +1682,7 @@ function preParseCSource(source: string): string { if (looksLikeCudaSource(blanked)) blanked = blankCudaConstructs(blanked); // The named-variadic `#define` pass runs AFTER the directive restore — it // deliberately edits directive lines (see its doc comment). - return blankCNamedVariadicDefineDots(restoreDirectiveLines(source, blanked)); + return rawStrings.restore(blankCNamedVariadicDefineDots(restoreDirectiveLines(source, blanked))); } export const cppExtractor: LanguageExtractor = {