FileWatcher.flush() bounded only two failure modes — lock contention (backoff + degrade past MAX_LOCK_RETRIES) and watch-resource exhaustion (degrade at setup). Its generic catch branch — any *other* sync error — reset the only circuit breaker (lockRetryCount = 0) and fell through to scheduleSync() at the normal debounce cadence, forever, with no backoff and no degrade(). The trigger is realistic, not synthetic: CodeGraph.sync() runs the whole extract -> resolve -> maintenance pipeline inside try/finally(release) with no catch, so a deterministic failure (a tree-sitter extractor that crashes on one file, SQLITE_FULL, an OOM in batched resolution) propagates straight into that unbounded branch — wedging a long-running daemon/MCP session into ~1,800 failing syncs + log lines/hour while the auto-update guarantee is silently dead. Mirror the lock circuit breaker for the generic branch: a separate consecutive-failure counter (syncFailureRetryCount) reset only by a clean sync, exponential backoff via the shared finally, and degrade() past MAX_SYNC_FAILURE_RETRIES with an actionable reason naming the underlying error. degrade() -> onDegraded/isDegraded() is what surfaces the dead guarantee (the staleness banner already consumes it) — a lighter flat-retry would keep it hidden, which is the core of the #876/#1127 complaint. Reset-on-success means a transient hiccup never degrades. The lock path is behaviorally unchanged: in any pure-lock scenario syncFailureRetryCount stays 0, so Math.max(lockRetryCount, syncFailureRetryCount) and the degrade threshold behave exactly as before. Renamed MAX_LOCK_RETRY_DELAY_MS -> MAX_RETRY_BACKOFF_MS (shared cap). Adds two regression tests mirroring the lock-contention ones: a persistent non-lock failure degrades past the budget; a transient one recovers without degrading. Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
3460accda8
commit
7c7514f43f
@@ -324,6 +324,71 @@ describe('FileWatcher', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('persistent sync-failure degradation (#1127)', () => {
|
||||
it('disables auto-sync after a persistent non-lock sync failure, with bounded retries', async () => {
|
||||
// A deterministic pipeline failure (broken extractor on a file, DB
|
||||
// corruption, SQLITE_FULL, OOM) recurs every cycle. Unbounded it retried
|
||||
// forever at the debounce cadence; it must now back off and degrade.
|
||||
const syncFn = vi.fn().mockRejectedValue(new Error('extractor crashed on src/bad.ts'));
|
||||
const onSyncComplete = vi.fn();
|
||||
const onSyncError = vi.fn();
|
||||
const onDegraded = vi.fn();
|
||||
const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});
|
||||
const watcher = newWatcher(syncFn, {
|
||||
debounceMs: 25,
|
||||
onSyncComplete,
|
||||
onSyncError,
|
||||
onDegraded,
|
||||
});
|
||||
watcher.start();
|
||||
await watcher.waitUntilReady();
|
||||
__emitWatchEventForTests(testDir, 'src/persistent-fail.ts');
|
||||
|
||||
// 5 backoff retries (25·1,2,4,8,16 ms), then degrade on the 6th attempt.
|
||||
await waitFor(() => !watcher.isActive(), 8000, 20);
|
||||
|
||||
expect(syncFn.mock.calls.length).toBeGreaterThanOrEqual(6); // MAX_SYNC_FAILURE_RETRIES + 1
|
||||
expect(watcher.isDegraded()).toBe(true);
|
||||
expect(onDegraded).toHaveBeenCalledTimes(1);
|
||||
expect(onDegraded).toHaveBeenCalledWith(expect.stringContaining('auto-sync disabled'));
|
||||
// The degrade reason carries the underlying error so the user can act.
|
||||
expect(onDegraded).toHaveBeenCalledWith(expect.stringContaining('extractor crashed'));
|
||||
// Unlike a held lock, a generic failure IS surfaced per-attempt.
|
||||
expect(onSyncError.mock.calls.length).toBeGreaterThanOrEqual(6);
|
||||
expect(onSyncComplete).not.toHaveBeenCalled();
|
||||
// Degrade stops the watcher, which clears pending state.
|
||||
expect(watcher.getPendingFiles()).toEqual([]);
|
||||
const disableWarnings = warnSpy.mock.calls.filter(
|
||||
(c) => typeof c[0] === 'string' && c[0].includes('File watcher disabled')
|
||||
);
|
||||
expect(disableWarnings).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('does NOT degrade on a transient sync failure — backoff resets after a clean sync', async () => {
|
||||
const syncFn = vi
|
||||
.fn()
|
||||
.mockRejectedValueOnce(new Error('transient blip'))
|
||||
.mockRejectedValueOnce(new Error('transient blip'))
|
||||
.mockRejectedValueOnce(new Error('transient blip'))
|
||||
.mockResolvedValue({ filesChanged: 1, durationMs: 5 });
|
||||
const onDegraded = vi.fn();
|
||||
const onSyncComplete = vi.fn();
|
||||
const watcher = newWatcher(syncFn, { debounceMs: 25, onDegraded, onSyncComplete });
|
||||
watcher.start();
|
||||
await watcher.waitUntilReady();
|
||||
__emitWatchEventForTests(testDir, 'src/transient-fail.ts');
|
||||
|
||||
await waitFor(() => onSyncComplete.mock.calls.length > 0, 4000, 20);
|
||||
|
||||
expect(onDegraded).not.toHaveBeenCalled();
|
||||
expect(watcher.isDegraded()).toBe(false);
|
||||
expect(watcher.isActive()).toBe(true);
|
||||
expect(watcher.getPendingFiles().some((p) => p.path === 'src/transient-fail.ts')).toBe(false);
|
||||
|
||||
watcher.stop();
|
||||
});
|
||||
});
|
||||
|
||||
describe('debounced sync', () => {
|
||||
it('should trigger sync after file change', async () => {
|
||||
const syncFn = vi.fn().mockResolvedValue({ filesChanged: 1, durationMs: 10 });
|
||||
|
||||
Reference in New Issue
Block a user