63e1b5a23a
Vendored patched govindbanura/tree-sitter-vbnet grammar (MIT, ~20-fix patch + new external scanner for XML literals and multi-line LINQ continuation; provenance + rebuild instructions in docs/grammars/tree-sitter-vbnet.md), vbnet extractor with VB-specific call/index disambiguation, Inherits/ Implements heritage, As New instantiation, events, Declare P/Invoke, and MustOverride abstract members. Parse health on five real repos: PolicyPlus 100%, CompactGUI 100%, staxrip 95.2%, SCrawler 87.2%, PCL 87.5% (upstream grammar: 3-18%). Retrieval A/B (sonnet): 26-43% faster with 0-5 file reads vs 7-20 without. Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
473 lines
15 KiB
JSON
473 lines
15 KiB
JSON
{
|
|
"_comment": "Test corpus for /agent-eval. Add entries freely. size: Small (<~150 files), Medium (~150-1500), Large (>~1500). 'question' is a representative architectural question that exercises cross-file understanding.",
|
|
"TypeScript": [
|
|
{
|
|
"name": "ky",
|
|
"repo": "https://github.com/sindresorhus/ky",
|
|
"size": "Small",
|
|
"files": "~25",
|
|
"question": "How does ky implement request retries and timeouts?"
|
|
},
|
|
{
|
|
"name": "excalidraw",
|
|
"repo": "https://github.com/excalidraw/excalidraw",
|
|
"size": "Medium",
|
|
"files": "~600",
|
|
"question": "How does Excalidraw render and update canvas elements?"
|
|
},
|
|
{
|
|
"name": "vscode",
|
|
"repo": "https://github.com/microsoft/vscode",
|
|
"size": "Large",
|
|
"files": "~10000",
|
|
"question": "How does the extension host communicate with the main process?"
|
|
}
|
|
],
|
|
"JavaScript": [
|
|
{
|
|
"name": "express",
|
|
"repo": "https://github.com/expressjs/express",
|
|
"size": "Small",
|
|
"files": "~50",
|
|
"question": "How does Express route a request through its middleware stack?"
|
|
}
|
|
],
|
|
"Go": [
|
|
{
|
|
"name": "cobra",
|
|
"repo": "https://github.com/spf13/cobra",
|
|
"size": "Small",
|
|
"files": "~50",
|
|
"question": "How does cobra parse commands and flags?"
|
|
},
|
|
{
|
|
"name": "gin",
|
|
"repo": "https://github.com/gin-gonic/gin",
|
|
"size": "Medium",
|
|
"files": "~150",
|
|
"question": "How does gin route requests through its middleware chain?"
|
|
},
|
|
{
|
|
"name": "terraform",
|
|
"repo": "https://github.com/hashicorp/terraform",
|
|
"size": "Large",
|
|
"files": "~4000",
|
|
"question": "How does Terraform build and walk the resource dependency graph?"
|
|
},
|
|
{
|
|
"name": "cosmos-sdk",
|
|
"repo": "https://github.com/cosmos/cosmos-sdk",
|
|
"size": "Large",
|
|
"files": "~5000",
|
|
"question": "How does a bank module MsgSend message reach the account balance update? Trace the cross-module call path from the bank keeper's Send handler through to the account/balance store update."
|
|
}
|
|
],
|
|
"Python": [
|
|
{
|
|
"name": "click",
|
|
"repo": "https://github.com/pallets/click",
|
|
"size": "Small",
|
|
"files": "~60",
|
|
"question": "How does click parse command-line arguments into commands?"
|
|
},
|
|
{
|
|
"name": "flask",
|
|
"repo": "https://github.com/pallets/flask",
|
|
"size": "Medium",
|
|
"files": "~90",
|
|
"question": "How does Flask dispatch a request to a view function?"
|
|
},
|
|
{
|
|
"name": "django",
|
|
"repo": "https://github.com/django/django",
|
|
"size": "Large",
|
|
"files": "~2700",
|
|
"question": "How does Django's ORM build and execute a query from a QuerySet?"
|
|
}
|
|
],
|
|
"Rust": [
|
|
{
|
|
"name": "clap",
|
|
"repo": "https://github.com/clap-rs/clap",
|
|
"size": "Medium",
|
|
"files": "~200",
|
|
"question": "How does clap parse arguments against a derived command definition?"
|
|
},
|
|
{
|
|
"name": "tokio",
|
|
"repo": "https://github.com/tokio-rs/tokio",
|
|
"size": "Large",
|
|
"files": "~700",
|
|
"question": "How does tokio schedule and run async tasks on its runtime?"
|
|
},
|
|
{
|
|
"name": "deno",
|
|
"repo": "https://github.com/denoland/deno",
|
|
"size": "Large",
|
|
"files": "~1500",
|
|
"question": "How does Deno load and execute a TypeScript module?"
|
|
}
|
|
],
|
|
"Java": [
|
|
{
|
|
"name": "gson",
|
|
"repo": "https://github.com/google/gson",
|
|
"size": "Medium",
|
|
"files": "~200",
|
|
"question": "How does Gson serialize an object to JSON?"
|
|
},
|
|
{
|
|
"name": "okhttp",
|
|
"repo": "https://github.com/square/okhttp",
|
|
"size": "Medium",
|
|
"files": "~640",
|
|
"question": "How does OkHttp process a request through its interceptor chain?"
|
|
},
|
|
{
|
|
"name": "guava",
|
|
"repo": "https://github.com/google/guava",
|
|
"size": "Large",
|
|
"files": "~3000",
|
|
"question": "How does Guava's CacheBuilder build and configure a cache?"
|
|
}
|
|
],
|
|
"Kotlin": [
|
|
{
|
|
"name": "koin",
|
|
"repo": "https://github.com/InsertKoinIO/koin",
|
|
"size": "Medium",
|
|
"files": "~300",
|
|
"question": "How does Koin resolve and inject dependencies?"
|
|
},
|
|
{
|
|
"name": "leakcanary",
|
|
"repo": "https://github.com/square/leakcanary",
|
|
"size": "Medium",
|
|
"files": "~250",
|
|
"question": "How does LeakCanary detect and analyze a memory leak?"
|
|
}
|
|
],
|
|
"Swift": [
|
|
{
|
|
"name": "alamofire",
|
|
"repo": "https://github.com/Alamofire/Alamofire",
|
|
"size": "Small",
|
|
"files": "~100",
|
|
"question": "How does Alamofire build, send, and validate a request?"
|
|
}
|
|
],
|
|
"C#": [
|
|
{
|
|
"name": "serilog",
|
|
"repo": "https://github.com/serilog/serilog",
|
|
"size": "Medium",
|
|
"files": "~250",
|
|
"question": "How does Serilog route a log event to its sinks?"
|
|
},
|
|
{
|
|
"name": "jellyfin",
|
|
"repo": "https://github.com/jellyfin/jellyfin",
|
|
"size": "Large",
|
|
"files": "~2500",
|
|
"question": "How does Jellyfin scan and identify items in a media library?"
|
|
}
|
|
],
|
|
"Ruby": [
|
|
{
|
|
"name": "sinatra",
|
|
"repo": "https://github.com/sinatra/sinatra",
|
|
"size": "Small",
|
|
"files": "~60",
|
|
"question": "How does Sinatra match a request to a route handler?"
|
|
},
|
|
{
|
|
"name": "discourse",
|
|
"repo": "https://github.com/discourse/discourse",
|
|
"size": "Large",
|
|
"files": "~3000",
|
|
"question": "How does Discourse create and render a new post?"
|
|
}
|
|
],
|
|
"PHP": [
|
|
{
|
|
"name": "slim",
|
|
"repo": "https://github.com/slimphp/Slim",
|
|
"size": "Small",
|
|
"files": "~80",
|
|
"question": "How does Slim handle a request through its middleware?"
|
|
},
|
|
{
|
|
"name": "laravel",
|
|
"repo": "https://github.com/laravel/framework",
|
|
"size": "Large",
|
|
"files": "~3000",
|
|
"question": "How does Laravel resolve and dispatch a route to a controller?"
|
|
}
|
|
],
|
|
"C": [
|
|
{
|
|
"name": "redis",
|
|
"repo": "https://github.com/redis/redis",
|
|
"size": "Large",
|
|
"files": "~600",
|
|
"question": "How does Redis parse and dispatch a client command?"
|
|
}
|
|
],
|
|
"C++": [
|
|
{
|
|
"name": "json",
|
|
"repo": "https://github.com/nlohmann/json",
|
|
"size": "Small",
|
|
"files": "~100",
|
|
"question": "How does nlohmann::json parse a JSON string into a value?"
|
|
},
|
|
{
|
|
"name": "grpc",
|
|
"repo": "https://github.com/grpc/grpc",
|
|
"size": "Large",
|
|
"files": "~3000",
|
|
"question": "How does gRPC dispatch an incoming RPC to its handler?"
|
|
}
|
|
],
|
|
"Dart": [
|
|
{
|
|
"name": "flutter",
|
|
"repo": "https://github.com/flutter/flutter",
|
|
"size": "Large",
|
|
"files": "~6000",
|
|
"question": "How does Flutter build and lay out a widget tree?"
|
|
}
|
|
],
|
|
"Svelte": [
|
|
{
|
|
"name": "shadcn-svelte",
|
|
"repo": "https://github.com/huntabyte/shadcn-svelte",
|
|
"size": "Medium",
|
|
"files": "~600",
|
|
"question": "How do shadcn-svelte components compose and apply their styling?"
|
|
}
|
|
],
|
|
"Lua": [
|
|
{
|
|
"name": "lualine.nvim",
|
|
"repo": "https://github.com/nvim-lualine/lualine.nvim",
|
|
"size": "Small",
|
|
"files": "~120",
|
|
"question": "How does lualine assemble and render its statusline sections and components?"
|
|
},
|
|
{
|
|
"name": "telescope.nvim",
|
|
"repo": "https://github.com/nvim-telescope/telescope.nvim",
|
|
"size": "Medium",
|
|
"files": "~80",
|
|
"question": "How does Telescope wire a picker to its finder, sorter, and previewer?"
|
|
},
|
|
{
|
|
"name": "kong",
|
|
"repo": "https://github.com/Kong/kong",
|
|
"size": "Large",
|
|
"files": "~1330",
|
|
"question": "How does Kong execute plugins across a request's lifecycle phases?"
|
|
}
|
|
],
|
|
"Luau": [
|
|
{
|
|
"name": "Knit",
|
|
"repo": "https://github.com/Sleitnick/Knit",
|
|
"size": "Small",
|
|
"files": "~10",
|
|
"question": "How does Knit register services and expose them to clients?"
|
|
},
|
|
{
|
|
"name": "vide",
|
|
"repo": "https://github.com/centau/vide",
|
|
"size": "Small",
|
|
"files": "~40",
|
|
"question": "How does vide track reactive sources and re-run effects when state changes?"
|
|
},
|
|
{
|
|
"name": "Fusion",
|
|
"repo": "https://github.com/dphfox/Fusion",
|
|
"size": "Medium",
|
|
"files": "~115",
|
|
"question": "How does Fusion build and update its reactive UI graph from state objects?"
|
|
}
|
|
],
|
|
"Objective-C": [
|
|
{
|
|
"name": "Masonry",
|
|
"repo": "https://github.com/SnapKit/Masonry",
|
|
"size": "Small",
|
|
"files": "~50",
|
|
"question": "How does Masonry build and activate Auto Layout constraints from its block DSL?"
|
|
},
|
|
{
|
|
"name": "FMDB",
|
|
"repo": "https://github.com/ccgus/fmdb",
|
|
"size": "Medium",
|
|
"files": "~80",
|
|
"question": "How does FMDB execute a prepared SQL statement and bind parameters?"
|
|
},
|
|
{
|
|
"name": "SDWebImage",
|
|
"repo": "https://github.com/SDWebImage/SDWebImage",
|
|
"size": "Large",
|
|
"files": "~400",
|
|
"question": "How does SDWebImage download, cache, and decode an image for a UIImageView?"
|
|
}
|
|
],
|
|
"Mixed iOS (Swift+ObjC)": [
|
|
{
|
|
"name": "Charts",
|
|
"repo": "https://github.com/danielgindi/Charts",
|
|
"size": "Small",
|
|
"files": "~270",
|
|
"question": "How does the ChartsDemo ObjC demo controller drive the Swift Charts library to animate and notify a data update?"
|
|
},
|
|
{
|
|
"name": "realm-swift",
|
|
"repo": "https://github.com/realm/realm-swift",
|
|
"size": "Medium",
|
|
"files": "~370",
|
|
"question": "How does a Swift `Realm.write { realm.add(obj) }` reach the Objective-C persistence layer?"
|
|
},
|
|
{
|
|
"name": "wikipedia-ios",
|
|
"repo": "https://github.com/wikimedia/wikipedia-ios",
|
|
"size": "Large",
|
|
"files": "~1700",
|
|
"question": "How does tapping a search result reach the article-fetch network call across the Swift / ObjC boundary?"
|
|
}
|
|
],
|
|
"React Native (legacy bridge + TurboModule)": [
|
|
{
|
|
"name": "@react-native-async-storage",
|
|
"repo": "https://github.com/react-native-async-storage/async-storage",
|
|
"size": "Small",
|
|
"files": "~60",
|
|
"question": "How does `setItem` in JS reach the native `legacy_multiSet` implementation?"
|
|
},
|
|
{
|
|
"name": "react-native-svg",
|
|
"repo": "https://github.com/software-mansion/react-native-svg",
|
|
"size": "Medium",
|
|
"files": "~700",
|
|
"question": "How does a JS `Svg.getTotalLength(...)` reach the iOS / Android native implementation via TurboModule?"
|
|
},
|
|
{
|
|
"name": "react-native-firebase",
|
|
"repo": "https://github.com/invertase/react-native-firebase",
|
|
"size": "Large",
|
|
"files": "~1100",
|
|
"question": "How does a native iOS push notification reach the JS `messaging().onMessage(...)` listener?"
|
|
}
|
|
],
|
|
"Expo Modules": [
|
|
{
|
|
"name": "expo-haptics",
|
|
"repo": "https://github.com/expo/expo/tree/main/packages/expo-haptics",
|
|
"size": "Small",
|
|
"files": "~15",
|
|
"question": "How does `Haptics.notificationAsync(...)` in JS reach `UINotificationFeedbackGenerator` in the Swift Module?"
|
|
},
|
|
{
|
|
"name": "expo-camera",
|
|
"repo": "https://github.com/expo/expo/tree/main/packages/expo-camera",
|
|
"size": "Medium",
|
|
"files": "~70",
|
|
"question": "How does a JS `CameraView.takePictureAsync(options)` reach the native AVCaptureSession / CameraDevice call?"
|
|
}
|
|
],
|
|
"React Native Fabric (view components)": [
|
|
{
|
|
"name": "react-native-segmented-control",
|
|
"repo": "https://github.com/react-native-segmented-control/segmented-control",
|
|
"size": "Small",
|
|
"files": "~25",
|
|
"question": "How does JSX `<SegmentedControl onChange={cb}/>` reach the native onChange handler on iOS/Android?"
|
|
},
|
|
{
|
|
"name": "react-native-screens",
|
|
"repo": "https://github.com/software-mansion/react-native-screens",
|
|
"size": "Medium",
|
|
"files": "~1200",
|
|
"question": "How does JSX `<ScreenStack>` reach the native RNSScreenStackView component?"
|
|
},
|
|
{
|
|
"name": "react-native-skia",
|
|
"repo": "https://github.com/Shopify/react-native-skia",
|
|
"size": "Large",
|
|
"files": "~1000",
|
|
"question": "How does a `<SkiaPictureView/>` JSX usage reach the iOS / Android native renderer?"
|
|
}
|
|
],
|
|
"R": [
|
|
{
|
|
"name": "AnomalyDetection",
|
|
"repo": "https://github.com/twitter/AnomalyDetection",
|
|
"size": "Small",
|
|
"files": "~24",
|
|
"question": "How does AnomalyDetectionTs go from the exported entry function to the underlying S-H-ESD statistical test? Name the functions on the path in order."
|
|
},
|
|
{
|
|
"name": "dplyr",
|
|
"repo": "https://github.com/tidyverse/dplyr",
|
|
"size": "Medium",
|
|
"files": "~450",
|
|
"question": "When mutate() is called on a grouped data frame, which functions handle the grouping and expression evaluation, in order, from mutate() down?"
|
|
},
|
|
{
|
|
"name": "ggplot2",
|
|
"repo": "https://github.com/tidyverse/ggplot2",
|
|
"size": "Large",
|
|
"files": "~1150",
|
|
"question": "When a ggplot object is printed, how does the plot actually get built and drawn — trace the path from print/plot to where geoms render. Name the key functions in order."
|
|
}
|
|
],
|
|
"COBOL": [
|
|
{
|
|
"name": "cics-genapp",
|
|
"repo": "https://github.com/cicsdev/cics-genapp",
|
|
"size": "Small",
|
|
"files": "~60",
|
|
"question": "When a new insurance policy is added through the LGTESTP1 CICS test harness, which programs handle the request on the way to the DB2 insert? Trace the path and name the programs in order."
|
|
},
|
|
{
|
|
"name": "carddemo",
|
|
"repo": "https://github.com/aws-samples/aws-mainframe-modernization-carddemo",
|
|
"size": "Medium",
|
|
"files": "~75",
|
|
"question": "How does the online bill-payment flow work — from the COBIL00C screen program to the transaction record being written? Name the paragraphs performed in order and the copybooks that define the records involved."
|
|
},
|
|
{
|
|
"name": "CobolCraft",
|
|
"repo": "https://github.com/meyfa/CobolCraft",
|
|
"size": "Medium",
|
|
"files": "~270",
|
|
"question": "How does an incoming player chat message travel from packet handling to being broadcast to the other connected players? Name the programs on the path in order."
|
|
}
|
|
],
|
|
"VB.NET": [
|
|
{
|
|
"name": "policyplus",
|
|
"repo": "https://github.com/Fleex255/PolicyPlus",
|
|
"size": "Small",
|
|
"files": "~94",
|
|
"question": "When the user toggles a policy to Enabled in the policy-setting editor and clicks OK, how does the new state end up written into the loaded policy source (POL file or registry)? Trace the path from the EditSetting dialog to the concrete write."
|
|
},
|
|
{
|
|
"name": "scrawler",
|
|
"repo": "https://github.com/AAndyProgram/SCrawler",
|
|
"size": "Medium",
|
|
"files": "~320",
|
|
"question": "When a user download is started for a Reddit user, how does the request flow from the user-level download entry point through the shared downloader base into the Reddit site plugin, and where do downloaded media items get appended to the user's content list?"
|
|
},
|
|
{
|
|
"name": "staxrip",
|
|
"repo": "https://github.com/staxrip/staxrip",
|
|
"size": "Medium",
|
|
"files": "~145",
|
|
"question": "When a job finishes video encoding, how does staxrip decide which muxer runs and how does the muxer command line get built and executed? Trace from job processing to the mkvmerge invocation."
|
|
}
|
|
]
|
|
} |