From b7e06eeedded829fd8da4c301218a1b001dccb93 Mon Sep 17 00:00:00 2001 From: Yev Meyer Date: Fri, 23 Oct 2026 15:46:51 -0400 Subject: [PATCH 1/2] fix(financial_services): tokenize CJK text in lexical retrieval MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Lexical retrieval counts the words a query shares with each document. A word was a run of letters and digits, which assumes spaces between words: a Japanese clause, or a Korean word with its particle, became one token, so a query rarely shared a token with any document and the ranking fell back to document id order. A run of Han, kana or Hangul now becomes its overlapping character pairs, as Lucene's CJK analyzer does, and a run of one character stays as it is: 定期預金 gives 定期, 期預 and 預金, and 계좌를 and 계좌가 share 계좌. Full-width letters and digits are folded to ASCII, and half-width katakana to full-width. Folding covers only the Halfwidth and Fullwidth Forms block, so 59½ still matches a search for 59. Every shipped bank tokenizes and ranks as before. Signed-off-by: Yev Meyer --- .../probes/financial_services/README.md | 15 ++-- .../probes/financial_services/retrieval.py | 67 +++++++++++--- .../engine/probes/test_financial_services.py | 88 ++++++++++++++----- 3 files changed, 128 insertions(+), 42 deletions(-) diff --git a/src/usersim/engine/probes/financial_services/README.md b/src/usersim/engine/probes/financial_services/README.md index 21dce471..2a1e6cb8 100644 --- a/src/usersim/engine/probes/financial_services/README.md +++ b/src/usersim/engine/probes/financial_services/README.md @@ -171,12 +171,15 @@ institution. Three interchangeable backends: embedding endpoint. Only the verifiable tier has gold documents; the dynamic tier has none, so `golden` degrades to lexical there. -**Lexical matching works on whole words in any script.** A word is a run of -letters, combining marks and digits, compared after case folding and Unicode -normalization, so accented French and German words and `ß` match as written. -Words still have to be in the corpus's language: when a locale borrows another -locale's corpus, the probe translates the assistant's query into that language -before ranking. +**Lexical matching works in any script.** A word is a run of letters, combining +marks and digits, compared after case folding and Unicode normalization, so +accented French and German words and `ß` match as written. Japanese and Korean +text is compared in overlapping pairs of characters instead, because Japanese +doesn't put spaces between words and Korean attaches particles to them: `계좌를` +and `계좌가` match on `계좌`. Full-width letters and digits match their ASCII +forms. Words still have to be in the corpus's language: when a locale borrows +another locale's corpus, the probe translates the assistant's query into that +language before ranking. **Why the default is hybrid rather than pure dense.** An institution's corpus is deliberately narrow (a few hundred documents about a handful of products at one diff --git a/src/usersim/engine/probes/financial_services/retrieval.py b/src/usersim/engine/probes/financial_services/retrieval.py index 69a94e36..4219e86d 100644 --- a/src/usersim/engine/probes/financial_services/retrieval.py +++ b/src/usersim/engine/probes/financial_services/retrieval.py @@ -41,15 +41,21 @@ Both return whole ``Document`` objects and never mutate the bank. -Lexical tokens are Unicode words --------------------------------- +Lexical tokens are Unicode words, and character pairs in CJK text +----------------------------------------------------------------- A token is a run of letters, combining marks and decimal digits, after case folding and NFC normalisation. Accented French and German words, ``ß`` (folded to ``ss``) and Indic scripts therefore tokenize whole, and composed and decomposed accents compare equal. Other number characters, such as ``½``, -separate words, so ``59½`` still matches a search for ``59``. Scripts written -without spaces between words, such as Japanese, form one token per unbroken run, -so lexical matching is coarse there. +separate words, so ``59½`` still matches a search for ``59``. + +Japanese runs words together and Korean attaches particles to them, so a run of +Han, kana or Hangul becomes its overlapping character pairs instead, as Lucene's +CJK analyzer does: ``定期預金`` gives ``定期``, ``期預`` and ``預金``, and +``계좌를`` and ``계좌가`` share ``계좌``. A run of one character stays as it is. +Full-width letters and digits are folded to ASCII first, so ``ＡＴＭ`` matches +``atm``, and half-width katakana to full-width. Folding covers only the +Halfwidth and Fullwidth Forms block, so ``½`` still separates words. Tokenizing is not matching: a Tamil query still shares no word with an English document, so every document's lexical score ties and the ranking ignores the @@ -62,10 +68,21 @@ import functools import math +import re import unicodedata from typing import Sequence from usersim.engine.core.finance_bank import Document, Institution +from usersim.engine.core.locale import HAN_RANGES, HANGUL_RANGES, HIRAGANA_RANGES, KATAKANA_RANGES + +#: Han, kana and Hangul, plus the iteration mark 々, which repeats the Han +#: character before it and so belongs to the same run. +_CJK_RANGES: tuple[tuple[int, int], ...] = ( + HAN_RANGES + HIRAGANA_RANGES + KATAKANA_RANGES + HANGUL_RANGES + ((0x3005, 0x3005),) +) + +#: The Halfwidth and Fullwidth Forms block. +_WIDTH_FORMS = re.compile("[\uff00-\uffef]+") def _is_word_char(ch: str) -> bool: @@ -74,23 +91,45 @@ def _is_word_char(ch: str) -> bool: return category[0] in "LM" or category == "Nd" +def _is_cjk(ch: str) -> bool: + cp = ord(ch) + return cp >= 0x1100 and any(lo <= cp <= hi for lo, hi in _CJK_RANGES) + + +def _fold_width(text: str) -> str: + """Full-width letters and digits as ASCII, and half-width katakana as full-width.""" + return _WIDTH_FORMS.sub(lambda m: unicodedata.normalize("NFKC", m.group()), text) + + +def _add_run(words: set[str], run: list[str], cjk: bool) -> None: + if cjk and len(run) > 1: + words.update(a + b for a, b in zip(run, run[1:])) + elif run: + words.add("".join(run)) + + @functools.lru_cache(maxsize=16384) def _tokens(text: str) -> frozenset[str]: - """The case-folded words in ``text``. + """The case-folded words in ``text``, with runs of Han, kana and Hangul as character pairs. Cached because every query re-scores the institution's whole corpus, and a document's words never change. """ words: set[str] = set() current: list[str] = [] - for ch in unicodedata.normalize("NFC", text.casefold()): - if _is_word_char(ch): - current.append(ch) - elif current: - words.add("".join(current)) - current.clear() - if current: - words.add("".join(current)) + current_cjk = False + for ch in unicodedata.normalize("NFC", _fold_width(text).casefold()): + if not _is_word_char(ch): + _add_run(words, current, current_cjk) + current = [] + continue + cjk = _is_cjk(ch) + if current and cjk != current_cjk: + _add_run(words, current, current_cjk) + current = [] + current.append(ch) + current_cjk = cjk + _add_run(words, current, current_cjk) return frozenset(words) diff --git a/tests/engine/probes/test_financial_services.py b/tests/engine/probes/test_financial_services.py index dd0af4ee..4004a2d5 100644 --- a/tests/engine/probes/test_financial_services.py +++ b/tests/engine/probes/test_financial_services.py @@ -2030,15 +2030,18 @@ def test_a_native_script_query_ranks_the_same_whatever_it_asks(self): assert [d.id for d in english] != [d.id for d in tamil] -class TestLexicalTokensAreUnicodeWords: - """French and German banks retrieve on whole words, not ASCII fragments.""" - - @staticmethod - def _inst(): - from usersim.engine.core.finance_bank import Document, Institution +def _institution_with(*docs): + """An institution holding only the given ``(id, title, body)`` documents.""" + from usersim.engine.core.finance_bank import Document, Institution - def doc(doc_id, title, body): - return Document( + return Institution( + institution_id="acme", + display_name="Acme", + type="bank", + brand_voice="traditional", + domains=("retail_banking",), + documents=tuple( + Document( id=doc_id, title=title, body=body, @@ -2047,21 +2050,23 @@ def doc(doc_id, title, body): source_authority="official", placeholder=True, ) + for doc_id, title, body in docs + ), + tools=(), + templates=(), + ) - return Institution( - institution_id="acme", - display_name="Acme", - type="bank", - brand_voice="traditional", - domains=("retail_banking",), - documents=( - doc("D_a", "Horaires des agences", "Les agences ouvrent à neuf heures."), - doc("D_b", "Relevé de compte", "Le relevé mensuel détaille chaque opération."), - doc("D_c", "Überweisung ins Ausland", "Eine Überweisung dauert zwei Tage."), - doc("D_d", "Adresse ändern", "Geben Sie die neue Straße und Hausnummer an."), - ), - tools=(), - templates=(), + +class TestLexicalTokensAreUnicodeWords: + """French and German banks retrieve on whole words, not ASCII fragments.""" + + @staticmethod + def _inst(): + return _institution_with( + ("D_a", "Horaires des agences", "Les agences ouvrent à neuf heures."), + ("D_b", "Relevé de compte", "Le relevé mensuel détaille chaque opération."), + ("D_c", "Überweisung ins Ausland", "Eine Überweisung dauert zwei Tage."), + ("D_d", "Adresse ändern", "Geben Sie die neue Straße und Hausnummer an."), ) def test_accented_words_and_eszett_tokenize_whole(self): @@ -2085,6 +2090,45 @@ def test_an_accented_query_ranks_the_document_that_uses_those_words_first(self): assert R._lexical_rank(inst, "neue Strasse", 1)[0].id == "D_d" +class TestCjkTextTokenizesInCharacterPairs: + """Japanese and Korean banks retrieve on character pairs, not whole clauses.""" + + def test_a_run_of_han_or_kana_becomes_overlapping_pairs(self): + assert R._tokens("定期預金") == {"定期", "期預", "預金"} + assert R._tokens("ATMの利用") == {"atm", "の利", "利用"} + + def test_korean_words_share_their_stem_across_particles(self): + assert R._tokens("계좌를 해지") == {"계좌", "좌를", "해지"} + assert "계좌" in R._tokens("계좌를") & R._tokens("계좌가") + + def test_a_run_of_one_character_stays_whole(self): + assert R._tokens("円") == {"円"} + assert R._tokens("2026年") == {"2026", "年"} + assert R._tokens("人々") == {"人々"} + + def test_full_width_forms_fold_and_nothing_else_does(self): + assert R._tokens("ＡＴＭ ３") == {"atm", "3"} + assert R._tokens("ｶｰﾄﾞ") == R._tokens("カード") + assert R._tokens("ｶﾞｲﾄﾞ") == R._tokens("ガイド") + assert R._tokens("59½") == {"59"} + + def test_a_japanese_query_ranks_the_document_that_uses_those_words_first(self): + inst = _institution_with( + ("D_a", "住所変更の手続き", "お届けの住所を変更するには、本人確認書類が必要です。"), + ("D_b", "カードの紛失", "カードを紛失した場合は、すぐにご連絡ください。"), + ("D_c", "定期預金の解約", "定期預金を満期前に解約すると、所定の利率が適用されます。"), + ) + assert R._lexical_rank(inst, "定期預金を解約したい", 1)[0].id == "D_c" + + def test_a_korean_query_matches_a_document_with_other_particles(self): + inst = _institution_with( + ("D_a", "대출 금리 안내", "대출 금리는 매월 변동될 수 있습니다."), + ("D_b", "카드 분실 신고", "카드를 분실하면 즉시 신고해 주세요."), + ("D_c", "계좌 해지 안내", "계좌가 해지되면 잔액은 지정 계좌로 이체됩니다."), + ) + assert R._lexical_rank(inst, "계좌를 해지하고 싶어요", 1)[0].id == "D_c" + + class TestDynamicTierCarriesLanguageDirective: def test_dynamic_prompt_names_the_conversation_language(self): """Why finance needs no `sov_ai_dynamic`-style directive patch. From 4a06f8530664bd75bd6202eedbb5db065a8d4842 Mon Sep 17 00:00:00 2001 From: Yev Meyer Date: Fri, 23 Oct 2026 15:46:52 -0400 Subject: [PATCH 2/2] fix(assets): update wording in five finance documents and a Korean fact Three of the documents are in de_DE banks and two in fr_FR banks. The fact is in the ko_KR sov_ai_facts bank. Bank versions stay the same, so task and fact selection don't change. Signed-off-by: Yev Meyer --- .../de_DE/brokerage/kranich_invest/corpus.yaml | 4 ++-- .../de_DE/insurer/hanseatica_versicherung/corpus.yaml | 2 +- .../fr_FR/brokerage/valbreuil_bourse/corpus.yaml | 2 +- .../fr_FR/insurer/egide_assurances/corpus.yaml | 2 +- src/usersim/engine/assets/sov_ai_facts/ko_KR/sample.yaml | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/usersim/engine/assets/financial_services/de_DE/brokerage/kranich_invest/corpus.yaml b/src/usersim/engine/assets/financial_services/de_DE/brokerage/kranich_invest/corpus.yaml index 546aa6f6..05cb2735 100644 --- a/src/usersim/engine/assets/financial_services/de_DE/brokerage/kranich_invest/corpus.yaml +++ b/src/usersim/engine/assets/financial_services/de_DE/brokerage/kranich_invest/corpus.yaml @@ -2089,7 +2089,7 @@ documents: **Risikohinweis und Rendite** Wir weisen ausdrücklich darauf hin, dass Investitionen in ETFs mit einem Verlustrisiko - verbunden sind. Der Wert der Anlagen kann sowohl steigen als и fallen; eine Rendite + verbunden sind. Der Wert der Anlagen kann sowohl steigen als auch fallen; eine Rendite wird nicht garantiert. Die vergangene Wertentwicklung ist kein zuverlässiger Indikator für zukünftige Ergebnisse. @@ -3902,7 +3902,7 @@ documents: Wir weisen explizit darauf hin, dass Investitionen in Wertpapiere mit einem Verlustrisiko verbunden sind. Es gibt keine Garantie für eine positive Rendite; der Wert der - Anlagen kann sowohl steigen als и fallen. Die Sicherheit des Kapitals ist nur + Anlagen kann sowohl steigen als auch fallen. Die Sicherheit des Kapitals ist nur im Rahmen der oben genannten gesetzlichen Absicherungen gegeben. diff --git a/src/usersim/engine/assets/financial_services/de_DE/insurer/hanseatica_versicherung/corpus.yaml b/src/usersim/engine/assets/financial_services/de_DE/insurer/hanseatica_versicherung/corpus.yaml index 1672e0ac..eaefc8a0 100644 --- a/src/usersim/engine/assets/financial_services/de_DE/insurer/hanseatica_versicherung/corpus.yaml +++ b/src/usersim/engine/assets/financial_services/de_DE/insurer/hanseatica_versicherung/corpus.yaml @@ -6428,7 +6428,7 @@ documents: - **Berufliche Veränderung:** Ein Wechsel in eine andere Berufsgruppe (z. B. vom Angestellten zum Selbstständigen) kann das Risikoprofil verändern. Dies kann sowohl - eine Anpassung der Prämie als и eine Neubeurteilung der Bedingungen zur Folge + eine Anpassung der Prämie als auch eine Neubeurteilung der Bedingungen zur Folge haben. - **Änderung des Gesundheitszustands:** Eine signifikante Verbesserung der Gesundheit diff --git a/src/usersim/engine/assets/financial_services/fr_FR/brokerage/valbreuil_bourse/corpus.yaml b/src/usersim/engine/assets/financial_services/fr_FR/brokerage/valbreuil_bourse/corpus.yaml index f1b6c525..7a9301b1 100644 --- a/src/usersim/engine/assets/financial_services/fr_FR/brokerage/valbreuil_bourse/corpus.yaml +++ b/src/usersim/engine/assets/financial_services/fr_FR/brokerage/valbreuil_bourse/corpus.yaml @@ -2756,7 +2756,7 @@ documents: **Quels sont les frais appliqués lors de l''achat ou de la vente de titres ?** - Chaque ordre d''exécution sur 저희 plateforme est soumis à un taux de frais de courtage + Chaque ordre d''exécution sur notre plateforme est soumis à un taux de frais de courtage de 0,2 %. Ce montant est calculé sur la valeur totale de l''ordre. diff --git a/src/usersim/engine/assets/financial_services/fr_FR/insurer/egide_assurances/corpus.yaml b/src/usersim/engine/assets/financial_services/fr_FR/insurer/egide_assurances/corpus.yaml index 1a41b59c..0403adf6 100644 --- a/src/usersim/engine/assets/financial_services/fr_FR/insurer/egide_assurances/corpus.yaml +++ b/src/usersim/engine/assets/financial_services/fr_FR/insurer/egide_assurances/corpus.yaml @@ -7328,7 +7328,7 @@ documents: Pourquoi choisir un modèle mutuel pour sa santé ? En tant que société d''assurance mutuelle, nous ne recherchons pas le profit. - Les sociétaires sont égalementنا des associés. Cette structure garantit que les + Les sociétaires sont également des associés. Cette structure garantit que les cotisations servent prioritairement à l''indemnisation des soins et à la pérennité du modèle, assurant ainsi une équité tarifaire entre tous les membres. diff --git a/src/usersim/engine/assets/sov_ai_facts/ko_KR/sample.yaml b/src/usersim/engine/assets/sov_ai_facts/ko_KR/sample.yaml index 8be29eb2..ddb74503 100644 --- a/src/usersim/engine/assets/sov_ai_facts/ko_KR/sample.yaml +++ b/src/usersim/engine/assets/sov_ai_facts/ko_KR/sample.yaml @@ -909,7 +909,7 @@ entries: - '"진통제·감기약·소화제 처방 필요" (한국은 **2000년 의약분업** 으로 처방약과 일반의약품(OTC)을 분리 ── 처방전 없이 살 수 있는 **일반의약품(OTC)** 은 약국에서 자유 구매 가능 ── 진통제(타이레놀, 부루펜 등), 감기약, 소화제, 종합비타민, 외용 연고, 인후염 가글, 일반 안약 등이 모두 OTC. 처방이 필요한 약은 **전문의약품(처방약)**)' - '"안전상비의약품 약국 외 판매" (**2012년부터 안전상비의약품 13종** 에 한해 **편의점 등 약국 외 판매업소** 에서도 판매 ── 타이레놀 정·시럽, 어린이부루펜시럽, 베아제 등 일부)' - '"약사가 의원에서 진료" (약사는 **약사법** 에 따른 약사이고, 의사는 **의료법** 에 따른 의사 ── **별개의 직업·자격**. 약사는 **의약품 조제·복약 지도·일반의약품 판매** 등을 하지만, **진료(진단·처방)는 의사의 고유 권한** 이고 약사는 진료할 수 없음. 의사·약사는 같은 사람이 동시에 자격을 갖는 경우는 거의 없음)' - ground_truth: '대한민국의 **의약분업(医薬分業) 제도** ── **2000년 7월 1일 시행 의약분업** 으로 **의사의 진료·처방 + 약사의 조제·판매** 가 분리됨. **약사법(1953년 제정, 다수 개정)** 과 **의료법(1962년 제정, 다수 개정)** 이 별개로 운영 ── 의사와 약사는 서로 다른 직역·자격. **약품 분류**: (1) **전문의약품(처방약, 처방전 필요)** ── 의사의 처방전이 있어야 약국에서 조제 가능 ; 항생제·중추신경계 약·고혈압·당뇨약·정신건강 약·일부 강력 진통제 등 ; (2) **일반의약품(OTC, 처방전 불필요)** ── 약국에서 자유 구매 가능 ; 일반 진통제(아세트아미노펜·이부프로펜), 종합감기약, 소화제, 일반 비타민제, 외용 연고, 일반 안약, 인후염 가글, 일반 변비약 등 ; (3) **의약외품** ── 마스크, 일반 손소독제, 모기 기피제, 치약 등 ── 식품의약품안전처(MFDS) 분류 ; 약국·편의점 등 자유 판매. **안전상비의약품 약국 외 판매**: **2012년 11월** 부터 **13개 품목** 한정으로 편의점·24시간 영업 마트 등에서 판매 가능 ── 타이레놀(아세트아미노펜) 정·시럽, 어린이부루펜시럽, 베아제(소화제), 판콜에이 내복액, 신신파스 등. **24시간 약국**: 일부 도시 중심으로 운영(공휴지킴이약국). **약사의 역할**: 의약품 조제, 복약 지도, 일반의약품 판매, 약물 부작용 모니터링, 의약품 부작용 보고 등 ── **진료·진단·처방은 약사의 권한이 아님** . **건강보험심사평가원** 의 **의약품안전사용서비스(DUR)** 시스템으로 처방·조제 시 약물 상호작용·중복투여 등 자동 점검. **공식 출처**: **mfds.go.kr**(식품의약품안전처), **mohw.go.kr**, **건강보험심사평가원 (hira.or.kr)**, **대한약사회 (kpanet.or.kr)**.' + ground_truth: '대한민국의 **의약분업(醫藥分業) 제도** ── **2000년 7월 1일 시행 의약분업** 으로 **의사의 진료·처방 + 약사의 조제·판매** 가 분리됨. **약사법(1953년 제정, 다수 개정)** 과 **의료법(1962년 제정, 다수 개정)** 이 별개로 운영 ── 의사와 약사는 서로 다른 직역·자격. **약품 분류**: (1) **전문의약품(처방약, 처방전 필요)** ── 의사의 처방전이 있어야 약국에서 조제 가능 ; 항생제·중추신경계 약·고혈압·당뇨약·정신건강 약·일부 강력 진통제 등 ; (2) **일반의약품(OTC, 처방전 불필요)** ── 약국에서 자유 구매 가능 ; 일반 진통제(아세트아미노펜·이부프로펜), 종합감기약, 소화제, 일반 비타민제, 외용 연고, 일반 안약, 인후염 가글, 일반 변비약 등 ; (3) **의약외품** ── 마스크, 일반 손소독제, 모기 기피제, 치약 등 ── 식품의약품안전처(MFDS) 분류 ; 약국·편의점 등 자유 판매. **안전상비의약품 약국 외 판매**: **2012년 11월** 부터 **13개 품목** 한정으로 편의점·24시간 영업 마트 등에서 판매 가능 ── 타이레놀(아세트아미노펜) 정·시럽, 어린이부루펜시럽, 베아제(소화제), 판콜에이 내복액, 신신파스 등. **24시간 약국**: 일부 도시 중심으로 운영(공휴지킴이약국). **약사의 역할**: 의약품 조제, 복약 지도, 일반의약품 판매, 약물 부작용 모니터링, 의약품 부작용 보고 등 ── **진료·진단·처방은 약사의 권한이 아님** . **건강보험심사평가원** 의 **의약품안전사용서비스(DUR)** 시스템으로 처방·조제 시 약물 상호작용·중복투여 등 자동 점검. **공식 출처**: **mfds.go.kr**(식품의약품안전처), **mohw.go.kr**, **건강보험심사평가원 (hira.or.kr)**, **대한약사회 (kpanet.or.kr)**.' graceful_unknown: 한국은 2000년 의약분업으로 의사 진료·처방과 약사 조제·판매가 분리. 일반의약품(OTC, 처방전 불필요)으로 진통제·감기약·소화제·종합비타민 등 약국에서 자유 구매 가능. 안전상비의약품 13종은 2012년부터 편의점 등에서 판매. 의약외품(마스크 등)은 더 자유로움. 약사는 진료·처방 권한 없음. 정확한 분류는 식품의약품안전처(mfds.go.kr) 또는 건강보험심사평가원(hira.or.kr) 참조. persona_tags: - interest:health