feat: wire up framework route extraction (#89)

* docs: add framework extract wiring plan

* feat(resolution): replace extractNodes with extract() returning nodes and references

* feat(resolution): add getApplicableFrameworks helper for per-language dispatch

* feat(django): emit route nodes and route->view references in extract()

* feat(flask,fastapi): emit route nodes and route->handler references

* feat(express): emit route nodes and route->handler references

* feat(laravel): emit route nodes and route->handler references

* feat(rails): emit route nodes and route->handler references

* feat(spring): emit route nodes and route->handler references

* feat(go): emit route nodes and route->handler references

* feat(rust): emit route nodes and route->handler references

* feat(aspnet): emit route nodes and route->handler references

* feat(swift,vapor): emit route nodes and route->handler references

* chore(react,svelte): migrate resolvers to extract() interface

* feat(extraction): run framework extractors after tree-sitter parse

* docs: document framework route extraction

* feat(strip-comments): add per-language comment stripper for framework extractors

Replaces comment characters and string-literal contents with spaces (not
removal) so source offsets stay valid for downstream regex match index ->
line number conversion. Handles Python triple-quoted docstrings, Ruby
=begin/=end, Rust nested block comments, and the standard //, #, /* */
forms across the supported languages.

This is consumed by framework extract() methods in a follow-up commit so
that commented-out / docstring routing examples don't surface as phantom
route nodes in the graph.

* feat(frameworks): strip comments before regex extraction (prevents phantom routes)

Pipes the per-language stripCommentsForRegex helper into every framework
extract() that scans raw source: django/flask/fastapi (python.ts),
express, laravel, rails, spring, go, rust, aspnet, vapor, plus
swiftui/uikit struct extraction in swift.ts.

Without this, examples like:

    # path('/admin/', AdminPanel.as_view())
    """ path('/users/', UserListView.as_view()) """
    urlpatterns = [path('/real/', RealView.as_view())]

produced 3 phantom route nodes. Now only the real one is extracted.

Each framework gets a regression test in __tests__/frameworks.test.ts
asserting that line-, block-, docstring- and (where relevant)
heredoc-style commented-out routes do not surface as nodes.

---------

Co-authored-by: Colby McHenry <me@colbymchenry.com>
Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
timomeara
2026-05-07 22:03:33 -05:00
committed by GitHub
co-authored by Colby McHenry Claude Opus 4.7
parent 5ab81746e8
commit 74327814ee
23 changed files with 3080 additions and 719 deletions
+80 -4
View File
@@ -22,6 +22,8 @@ import { detectLanguage, isLanguageSupported, initGrammars, loadGrammarsForLangu
import { logDebug, logWarn } from '../errors';
import { validatePathWithinRoot, normalizePath } from '../utils';
import picomatch from 'picomatch';
import { detectFrameworks } from '../resolution/frameworks';
import type { ResolutionContext } from '../resolution/types';
/**
* Number of files to read in parallel during indexing.
@@ -399,6 +401,13 @@ export class ExtractionOrchestrator {
private rootDir: string;
private config: CodeGraphConfig;
private queries: QueryBuilder;
/**
* Names of frameworks detected for this project, populated by indexAll().
* Passed to extractFromSource so framework-specific extractors (route nodes,
* middleware, etc.) run after the tree-sitter pass. Cleared if detection
* hasn't run yet so single-file re-index paths can detect on the spot.
*/
private detectedFrameworkNames: string[] | null = null;
constructor(rootDir: string, config: CodeGraphConfig, queries: QueryBuilder) {
this.rootDir = rootDir;
@@ -406,6 +415,57 @@ export class ExtractionOrchestrator {
this.queries = queries;
}
/**
* Build a filesystem-backed ResolutionContext sufficient for framework
* detection. Graph-query methods (getNodesByName etc.) return empty because
* the DB hasn't been populated yet, but detect() only uses readFile,
* fileExists, and getAllFiles, so that's fine.
*/
private buildDetectionContext(files: string[]): ResolutionContext {
const rootDir = this.rootDir;
return {
getNodesInFile: () => [],
getNodesByName: () => [],
getNodesByQualifiedName: () => [],
getNodesByKind: () => [],
getNodesByLowerName: () => [],
getImportMappings: () => [],
getAllFiles: () => files,
getProjectRoot: () => rootDir,
fileExists: (relativePath: string) => {
const full = validatePathWithinRoot(rootDir, relativePath);
if (!full) return false;
try {
return fs.existsSync(full);
} catch {
return false;
}
},
readFile: (relativePath: string) => {
const full = validatePathWithinRoot(rootDir, relativePath);
if (!full) return null;
try {
return fs.readFileSync(full, 'utf-8');
} catch {
return null;
}
},
};
}
/**
* Detect frameworks on demand using the current scanned files (or a fresh
* scan if none are provided). Cached on the orchestrator so repeat calls
* inside a single run don't re-scan.
*/
private ensureDetectedFrameworks(files?: string[]): string[] {
if (this.detectedFrameworkNames !== null) return this.detectedFrameworkNames;
const fileList = files ?? scanDirectory(this.rootDir, this.config);
const context = this.buildDetectionContext(fileList);
this.detectedFrameworkNames = detectFrameworks(context).map((r) => r.name);
return this.detectedFrameworkNames;
}
/**
* Index all files in the project
*/
@@ -443,6 +503,14 @@ export class ExtractionOrchestrator {
});
});
// Detect frameworks once per indexAll run using the scanned file list.
// Names are passed to each parse call so framework-specific extractors
// (route nodes, middleware, etc.) run after the tree-sitter pass.
// Framework detection is reset each run so adding e.g. requirements.txt
// between runs is picked up without restarting the process.
this.detectedFrameworkNames = null;
const frameworkNames = this.ensureDetectedFrameworks(files);
if (signal?.aborted) {
return {
success: false,
@@ -584,7 +652,12 @@ export class ExtractionOrchestrator {
async function requestParse(filePath: string, content: string): Promise<ExtractionResult> {
if (!WorkerClass) {
// In-process fallback
return extractFromSource(filePath, content, detectLanguage(filePath, content));
return extractFromSource(
filePath,
content,
detectLanguage(filePath, content),
frameworkNames
);
}
// Recycle the worker before the next parse if we've hit the threshold.
@@ -614,7 +687,7 @@ export class ExtractionOrchestrator {
}, timeoutMs);
pendingParses.set(id, { resolve, reject, timer });
worker.postMessage({ type: 'parse', id, filePath, content });
worker.postMessage({ type: 'parse', id, filePath, content, frameworkNames });
});
}
@@ -1024,8 +1097,11 @@ export class ExtractionOrchestrator {
};
}
// Extract from source
const result = extractFromSource(relativePath, content, language);
// Extract from source. Use cached framework names if indexAll has run,
// otherwise detect on the spot so single-file re-index paths still emit
// route nodes / middleware / etc.
const frameworkNames = this.ensureDetectedFrameworks();
const result = extractFromSource(relativePath, content, language, frameworkNames);
// Store in database
if (result.nodes.length > 0 || result.errors.length === 0) {