fix(prompt-hook): make the structural-question gate multilingual (#1126) (#1134)

* fix(prompt-hook): fire the structural gate for Latin-script, Cyrillic, and JA/KO prompts (#1126) The prompt-hook's keyword gate only knew English and simplified-Chinese keywords, so a structural question in French (or Spanish, German, Italian, Portuguese, Russian, Japanese, Korean, traditional Chinese) silently no-op'd unless it happened to contain an identifier-shaped code token — the #994 symptom, resurfaced for every other language. Root causes fixed: - JS \b is ASCII-only: a keyword whose first/last char is accented or non-Latin (où, qué, Cyrillic, kana) can never match \bkeyword\b — the same mechanism behind #994. Keyword matching now uses Unicode lookaround boundaries ((?<![\p{L}\p{N}_]) … (?![\p{L}\p{N}_])). - Bare-stem English entries never matched their own derived forms (\barchitect\b can't match "architecture", \bdepend\b can't match "dependencies"). Stems are now matched as word prefixes (leading boundary only), which also lets one shared stem cover the Romance/ Germanic spellings that coincide. - The "CJK" set was simplified-Chinese-only: Japanese (呼び出し, 仕組み, 実装 — and 追跡 ≠ 追踪), Korean, and traditional-Chinese terms are now in the unsegmented substring set. Code-token extraction and the graph-verification path are unchanged; non-structural prose stays a zero-cost no-op in every language. Fixes #1126 Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * fix(prompt-hook): extend the gate to tier-2 languages (VI/TR/ID/PL/UA/NL/CS/RO/HU/EL/Nordics/FI/HI/AR/FA/HE/TH) The first pass covered the 10 largest languages; this closes the rest of the major-developer-population set (~29 total). Notable per-language mechanics the curation had to respect: - Agglutinative languages (Turkish, Finnish, Hungarian) need stems, not exact words — suffixes attach to everything (akışı, riippuu, működik). - Indonesian me-/di-/ber- prefixes block leading-boundary stems, so affixed forms are listed explicitly (memanggil, dipanggil, berfungsi). - Arabic/Farsi/Hebrew are spaced but proclitics attach to the word (وكيف = and-how), so they join the substring class with Thai. - Ukrainian і/и spellings diverge from Russian (архітектур ≠ архитектур). - Excluded terms that collide with English or code words: NL "pad", SV "var", CS "tok", Catalan "com" (matches every .com domain) — with regression tests pinning the exclusions. Vietnamese was the sharpest gap: spaced Latin with heavy diacritics — exactly the ASCII-\b failure class #1126 reports. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --------- Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
2026-07-02 14:34:17 -05:00
parent 04e23917d0
commit 317e7f4d3d
4 changed files with 292 additions and 22 deletions
@@ -130,7 +130,7 @@ describe('findIndexedSubprojectRoots', () => {
 });

 describe('hasStructuralKeyword — keyword signal fires the hook directly (#994)', () => {
-  it('English keywords match, with `\\b` so "flow" ≠ "flower"', () => {
+  it('English keywords match with word boundaries so "flow" ≠ "flower"', () => {
    expect(hasStructuralKeyword('how does article publish work')).toBe(true);
    expect(hasStructuralKeyword('where is the token validated')).toBe(true);
    expect(hasStructuralKeyword('trace the request flow')).toBe(true);
@@ -152,6 +152,100 @@ describe('hasStructuralKeyword — keyword signal fires the hook directly (#994)
  });
 });

+describe('hasStructuralKeyword — Latin-script languages, Cyrillic, JA/KO (#1126)', () => {
+  it('French structural prompts fire — including the prompts from the report', () => {
+    expect(hasStructuralKeyword('comment marche la state machine des commandes ?')).toBe(true);
+    expect(hasStructuralKeyword("explique l'architecture du module de stock")).toBe(true);
+    expect(hasStructuralKeyword('qui appelle cette fonction de parsing ?')).toBe(true);
+    expect(hasStructuralKeyword('de quoi dépend le module de paiement ?')).toBe(true);
+  });
+
+  it('accented keyword edges match — ASCII `\\b` could never bound "où"', () => {
+    expect(hasStructuralKeyword('où est validé le token ?')).toBe(true);
+    expect(hasStructuralKeyword("d'où vient cette valeur ?")).toBe(true);
+  });
+
+  it('Spanish / Portuguese / German / Italian fire', () => {
+    expect(hasStructuralKeyword('¿cómo funciona la máquina de estados de pedidos?')).toBe(true);
+    expect(hasStructuralKeyword('¿qué rompe este cambio?')).toBe(true);
+    expect(hasStructuralKeyword('como funciona a máquina de estados dos pedidos?')).toBe(true);
+    expect(hasStructuralKeyword('qual é a arquitetura do módulo de estoque?')).toBe(true);
+    expect(hasStructuralKeyword('wie funktioniert die Zustandsmaschine für Bestellungen?')).toBe(true);
+    expect(hasStructuralKeyword('wovon hängt das Zahlungsmodul ab?')).toBe(true);
+    expect(hasStructuralKeyword('come funziona la macchina a stati degli ordini?')).toBe(true);
+    expect(hasStructuralKeyword('spiegami la struttura del modulo ordini')).toBe(true);
+  });
+
+  it('Russian / Japanese / Korean / traditional Chinese fire', () => {
+    expect(hasStructuralKeyword('как работает конечный автомат заказов?')).toBe(true);
+    expect(hasStructuralKeyword('от чего зависит модуль оплаты?')).toBe(true);
+    expect(hasStructuralKeyword('注文のステートマシンの仕組みを説明して')).toBe(true);
+    expect(hasStructuralKeyword('この関数の呼び出しの流れは?')).toBe(true);
+    expect(hasStructuralKeyword('주문 상태 머신은 어떻게 작동하나요?')).toBe(true);
+    expect(hasStructuralKeyword('訂單狀態機的架構是怎麼實現的?')).toBe(true);
+  });
+
+  it('English derived forms fire — "architecture"/"dependencies" failed the old exact-word list', () => {
+    expect(hasStructuralKeyword('explain the architecture of the stock module')).toBe(true);
+    expect(hasStructuralKeyword('what are the dependencies of the parser?')).toBe(true);
+  });
+
+  it('second-tier languages fire — VI/TR/ID/PL/UA/NL/CS/RO/HU/EL/SV/NO/FI/HI', () => {
+    expect(hasStructuralKeyword('state machine của đơn hàng hoạt động thế nào?')).toBe(true);   // Vietnamese
+    expect(hasStructuralKeyword('sipariş durum makinesi nasıl çalışıyor?')).toBe(true);          // Turkish
+    expect(hasStructuralKeyword('bu fonksiyonun bağımlılıkları neler?')).toBe(true);             // Turkish (stem)
+    expect(hasStructuralKeyword('bagaimana cara kerja mesin status pesanan?')).toBe(true);       // Indonesian
+    expect(hasStructuralKeyword('jak działa maszyna stanów zamówień?')).toBe(true);              // Polish
+    expect(hasStructuralKeyword('co wywołuje tę funkcję?')).toBe(true);                          // Polish (stem)
+    expect(hasStructuralKeyword('як працює кінцевий автомат замовлень?')).toBe(true);            // Ukrainian
+    expect(hasStructuralKeyword('від чого залежить модуль оплати?')).toBe(true);                 // Ukrainian (stem)
+    expect(hasStructuralKeyword('hoe werkt de state machine van bestellingen?')).toBe(true);     // Dutch
+    expect(hasStructuralKeyword('jak funguje stavový automat objednávek?')).toBe(true);          // Czech
+    expect(hasStructuralKeyword('cum funcționează mașina de stări a comenzilor?')).toBe(true);   // Romanian
+    expect(hasStructuralKeyword('hogyan működik a rendelések állapotgépe?')).toBe(true);         // Hungarian
+    expect(hasStructuralKeyword('πώς λειτουργεί η μηχανή καταστάσεων παραγγελιών;')).toBe(true); // Greek
+    expect(hasStructuralKeyword('hur fungerar orderns tillståndsmaskin?')).toBe(true);           // Swedish
+    expect(hasStructuralKeyword('hvordan fungerer ordrenes tilstandsmaskin?')).toBe(true);       // Norwegian/Danish
+    expect(hasStructuralKeyword('miten tilausten tilakone toimii?')).toBe(true);                 // Finnish
+    expect(hasStructuralKeyword('ऑर्डर स्टेट मशीन कैसे काम करती है?')).toBe(true);                 // Hindi
+  });
+
+  it('RTL scripts and Thai fire — proclitics/unsegmented text uses substring matching', () => {
+    expect(hasStructuralKeyword('كيف تعمل آلة حالات الطلبات؟')).toBe(true);        // Arabic
+    expect(hasStructuralKeyword('وكيف يعتمد هذا على قاعدة البيانات؟')).toBe(true); // Arabic, proclitic و attached
+    expect(hasStructuralKeyword('ماشین وضعیت سفارش‌ها چگونه کار می‌کند؟')).toBe(true); // Farsi
+    expect(hasStructuralKeyword('איך עובדת מכונת המצבים של ההזמנות?')).toBe(true);  // Hebrew
+    expect(hasStructuralKeyword('สถาปัตยกรรมของระบบทำงานอย่างไร')).toBe(true);       // Thai
+  });
+
+  it('terms that collide with English or code words are deliberately excluded', () => {
+    expect(hasStructuralKeyword('pad the buffer with zeros')).toBe(false);     // NL pad=path skipped
+    expect(hasStructuralKeyword('declare a var for the count')).toBe(false);   // SV var=where skipped
+    expect(hasStructuralKeyword('refresh the token')).toBe(false);             // CS tok=flow skipped
+    expect(hasStructuralKeyword('run the llama model locally')).toBe(false);   // ES bare llama skipped
+    expect(hasStructuralKeyword('come back to this later')).toBe(false);       // IT bare come skipped
+  });
+
+  it('stems match only at word start — no mid-word false positives', () => {
+    expect(hasStructuralKeyword('restructure this paragraph')).toBe(false); // "structur" mid-word
+    expect(hasStructuralKeyword('an independent module')).toBe(false);      // "depend" mid-word
+    expect(hasStructuralKeyword('water the flower')).toBe(false);           // unchanged guarantee
+  });
+
+  it('non-structural prose stays a no-op in every covered language', () => {
+    expect(hasStructuralKeyword('corrige cette faute de frappe')).toBe(false);   // FR "fix this typo"
+    expect(hasStructuralKeyword('arregla este error tipográfico')).toBe(false);  // ES
+    expect(hasStructuralKeyword('behebe diesen Tippfehler')).toBe(false);        // DE
+    expect(hasStructuralKeyword('исправь эту опечатку')).toBe(false);            // RU
+    expect(hasStructuralKeyword('このタイプミスを直して')).toBe(false);            // JA
+    expect(hasStructuralKeyword('이 오타를 수정해줘')).toBe(false);                // KO
+    expect(hasStructuralKeyword('sửa lỗi chính tả này')).toBe(false);            // VI
+    expect(hasStructuralKeyword('bu yazım hatasını düzelt')).toBe(false);        // TR
+    expect(hasStructuralKeyword('popraw tę literówkę')).toBe(false);             // PL
+    expect(hasStructuralKeyword('صحح هذا الخطأ الإملائي')).toBe(false);          // AR
+  });
+});
+
 describe('extractCodeTokens — candidate symbols the hook verifies against the graph', () => {
  it('pulls camelCase / PascalCase / snake_case / call / member tokens', () => {
    expect(extractCodeTokens('prepareArticlePublish 的调用链')).toContain('prepareArticlePublish');