From a961491cea5a0e67d29cf9a998030396b38be65c Mon Sep 17 00:00:00 2001 From: Colby Mchenry Date: Tue, 8 Sep 2026 14:37:46 -0500 Subject: [PATCH] fix(extraction): keep git-aware scans working before Git 2.36 (#1793) Land upstream PR #1604 by @maxmilian from commit 27c149524d968485164fb43bdec994fdb9323c75 on current main cece0720. Fixes #1549. Use the upstream lsFilesStaged helper at both collection and embedded-repo discovery call sites. Retry without --recurse-submodules when staged recursive listing fails, retaining the mode bits needed for gitlinks. Keep the upstream regression test byte-for-byte and preserve main's existing extraction/watcher changes, including #1728. Resolve the changelog conflict with a concise Unreleased entry, and keep the existing collectGitFiles documentation attached to its function. Verified on Linux x86_64 with Node 22.19.0 and Git 2.47.3: - The unchanged upstream test fails against main: expected [ 'a.ts' ] to include 'dir_b/b.ts'; it passes with the fix. - The same persistent fixture under a PATH shim rejecting -s with --recurse-submodules (exit 128) changes scanDirectory from [a.ts] to [a.ts, dir_b/b.ts], and gitlink watcher roots from [] to [lib/]. - Real-Git controls return both files and the watcher root in both arms. - 69 related scope/config tests and 23 extraction scanning tests pass. - npx tsc --noEmit passes. Co-authored-by: Colby McHenry Co-authored-by: Max Hsu Co-authored-by: newshowardz777 --- CHANGELOG.md | 2 + __tests__/extraction-old-git.test.ts | 107 +++++++++++++++++++++++++++ src/extraction/index.ts | 30 +++++++- 3 files changed, 135 insertions(+), 4 deletions(-) create mode 100644 __tests__/extraction-old-git.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 851f8e9..4e76318 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -151,6 +151,8 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - `codegraph install` now honors `CLAUDE_CONFIG_DIR` and `CODEX_HOME` for global Claude Code and Codex setup so CodeGraph loads in your chosen profile (thanks @seanchann; #1627). +- Files opted in with `includeIgnored` now stay indexed on Git older than 2.36, and embedded repositories remain visible to the watcher (thanks @maxmilian and @newshowardz777; #1549). + #### Screens, links and navigation - **Where the app goes after login is a fork, not two always-es.** A navigation whose destination comes back from a helper — `router.replace(await resolvePostLoginRoute())` over `return (await hasSeenWelcome(…)) ? '/home/' : '/welcome/'` — drew both screens with no condition, reading as if the welcome screen always shows. The two arms share a line, and only a column can tell them apart; each synthesized edge now carries its literal's own position, so the guard reader says which arm it is: `WHEN await hasSeenWelcome(…)` → home, and its negation → welcome. And the scan starts at the helper's body, so a literal-union return type — `Promise<'/welcome/' | '/home/'>`, whose routes are string literals too, written first — no longer stands in for the navigation itself. Re-index after upgrading to pick the positions up. diff --git a/__tests__/extraction-old-git.test.ts b/__tests__/extraction-old-git.test.ts new file mode 100644 index 0000000..38f6239 --- /dev/null +++ b/__tests__/extraction-old-git.test.ts @@ -0,0 +1,107 @@ +/** + * Regression: git older than 2.36 rejects `ls-files -s --recurse-submodules` (#1549). + * + * Kept in its own file rather than appended to extraction.test.ts: that suite + * loads every tree-sitter grammar in `beforeAll`, and running a git-scan case + * after it pushed the worker past its memory ceiling. + */ + +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import * as fs from 'fs'; +import * as path from 'path'; +import * as os from 'os'; +import { execFileSync } from 'child_process'; +import { scanDirectory } from '../src/extraction'; + +function createTempDir(): string { + return fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-test-')); +} + +// git < 2.36 rejects `ls-files -s --recurse-submodules` outright: the guard in +// builtin/ls-files.c listed `show_stage` among the modes that die, and it was +// only dropped in 2.36. The die is unconditional — it does not check whether the +// repo has submodules — so on Ubuntu 22.04 (git 2.34.1), Debian 11 (2.30.2) and +// older, every call threw, `getGitVisibleFiles` swallowed it, and the whole +// git-visible path went with it: `includeIgnored`, gitlink recursion and the +// `codegraph.json` `include` allowlist all silently stopped applying (#1549). +// +// A PATH shim reproduces that on any git version, which is what makes this +// testable in CI at all. +describe('Old git without `ls-files -s --recurse-submodules` support (#1549)', () => { + let tempDir: string; + let originalPath: string | undefined; + + const runGit = (cwd: string, ...args: string[]) => + execFileSync('git', args, { cwd, stdio: 'pipe' }); + + const makeRepo = (dir: string, base: string) => { + fs.mkdirSync(dir, { recursive: true }); + runGit(dir, 'init', '-q'); + runGit(dir, 'config', 'user.email', 'test@test.com'); + runGit(dir, 'config', 'user.name', 'Test'); + fs.writeFileSync(path.join(dir, `${base}.ts`), `export const ${base} = 1;`); + runGit(dir, 'add', '-A'); + runGit(dir, 'commit', '-q', '-m', `${base} init`); + }; + + /** A `git` that dies exactly like < 2.36 when it sees -s with --recurse-submodules. */ + const installOldGitShim = () => { + const shimDir = path.join(tempDir, '.shim'); + fs.mkdirSync(shimDir, { recursive: true }); + const realGit = execFileSync('which', ['git']).toString().trim(); + const shim = path.join(shimDir, 'git'); + fs.writeFileSync( + shim, + [ + '#!/bin/sh', + 'for a in "$@"; do', + ' [ "$a" = "--recurse-submodules" ] && rs=1', + ' [ "$a" = "-s" ] && st=1', + 'done', + 'if [ -n "$rs" ] && [ -n "$st" ]; then', + ' echo "fatal: ls-files --recurse-submodules unsupported mode" >&2', + ' exit 128', + 'fi', + `exec ${JSON.stringify(realGit)} "$@"`, + ].join('\n'), + ); + fs.chmodSync(shim, 0o755); + originalPath = process.env.PATH; + process.env.PATH = `${shimDir}:${originalPath ?? ''}`; + }; + + beforeEach(() => { + tempDir = createTempDir(); + }); + + afterEach(() => { + if (originalPath !== undefined) process.env.PATH = originalPath; + originalPath = undefined; + }); + + it('still honours includeIgnored when `ls-files --recurse-submodules` is unsupported', () => { + const root = path.join(tempDir, 'root'); + makeRepo(root, 'a'); + // An embedded repo that .gitignore excludes but codegraph.json opts back in. + makeRepo(path.join(root, 'dir_b'), 'b'); + fs.writeFileSync(path.join(root, '.gitignore'), 'dir_b/\n'); + fs.writeFileSync( + path.join(root, 'codegraph.json'), + JSON.stringify({ includeIgnored: ['dir_b/'] }), + ); + runGit(root, 'add', '-A'); + runGit(root, 'commit', '-q', '-m', 'ignore dir_b'); + + // Baseline: the real git resolves both files. + const withRealGit = scanDirectory(root); + expect(withRealGit).toContain('a.ts'); + expect(withRealGit).toContain(path.join('dir_b', 'b.ts')); + + installOldGitShim(); + + // The opted-in file must survive the unsupported-mode failure, not vanish. + const withOldGit = scanDirectory(root); + expect(withOldGit).toContain('a.ts'); + expect(withOldGit).toContain(path.join('dir_b', 'b.ts')); + }); +}); diff --git a/src/extraction/index.ts b/src/extraction/index.ts index fc9f904..4608a5f 100644 --- a/src/extraction/index.ts +++ b/src/extraction/index.ts @@ -925,9 +925,7 @@ export function discoverEmbeddedRepoRoots(rootDir: string): string[] { // same way collectGitFiles does, keeping watcher scope == indexer scope. // (#1031, #1033) try { - const staged = execFileSync( - 'git', - ['ls-files', '-z', '-s', '--recurse-submodules'], + const staged = lsFilesStaged( { cwd: repoAbs, encoding: 'utf-8', timeout: 30000, maxBuffer: 50 * 1024 * 1024, stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true } ); const repoIgnore = buildDefaultIgnore(repoAbs); @@ -1040,6 +1038,30 @@ function findIgnoredEmbeddedRepos(repoDir: string, includeIgnored: Ignore | null return repos; } +/** + * `git ls-files -z -s`, expanding submodules where git allows it. + * + * `--recurse-submodules` could not be combined with `-s` before git 2.36: + * `builtin/ls-files.c` listed `show_stage` among the modes that die, and the + * check is unconditional — it does not look at whether the repo actually has + * submodules, so every call fails on older git. Ubuntu 22.04 LTS (2.34.1) and + * Debian 11 (2.30.2) are both below that line. + * + * Letting the throw escape cost far more than submodule expansion: it unwound + * the whole git-visible pass, so `includeIgnored`, gitlink recursion and the + * `codegraph.json` include allowlist silently stopped applying and files went + * missing from the index with no error (#1549). Retry without the flag instead + * — `-s` is the part that matters here, since gitlink detection reads the mode + * bits, and embedded repos are reached through the gitlink recursion anyway. + */ +function lsFilesStaged(gitOpts: Parameters[2]): string { + try { + return execFileSync('git', ['ls-files', '-z', '-s', '--recurse-submodules'], gitOpts) as unknown as string; + } catch { + return execFileSync('git', ['ls-files', '-z', '-s'], gitOpts) as unknown as string; + } +} + /** * Collect git-visible files (tracked + untracked, .gitignore-respected) from the * git repository rooted at `repoDir`, adding each to `files` with `prefix` @@ -1085,7 +1107,7 @@ function collectGitFiles(repoDir: string, prefix: string, files: Set, em // on disk → those files are silently dropped from the index. (#541) With -s the // path follows a TAB after the ` ` prefix. const gitlinkRels: string[] = []; - const tracked = execFileSync('git', ['ls-files', '-z', '-s', '--recurse-submodules'], gitOpts); + const tracked = lsFilesStaged(gitOpts); for (const entry of tracked.split('\0')) { if (!entry) continue; const tab = entry.indexOf('\t');