diff --git a/src/usersim/asset_gen/financial_services/spec.py b/src/usersim/asset_gen/financial_services/spec.py index d1425ef0..c99260c6 100644 --- a/src/usersim/asset_gen/financial_services/spec.py +++ b/src/usersim/asset_gen/financial_services/spec.py @@ -514,10 +514,35 @@ def _templates_keep_their_slots(self) -> "DocTitleVocabulary": return self +def _whole_word_pattern(forms: list[str]) -> re.Pattern[str]: + """A case-insensitive regex whose group 1 is any of ``forms`` as a whole word + or phrase, longest first. + + An edge of a form in Japanese or Korean script may meet any character. Any + other edge must not run into a letter or digit of a script that spaces its + words. The forms that need that check before them share one, which keeps the + regex as fast as a plain word-boundary pattern. + """ + from usersim.engine.core.locale import HAN_RANGES, HANGUL_RANGES, HIRAGANA_RANGES, KATAKANA_RANGES + + ranges = HAN_RANGES + HIRAGANA_RANGES + KATAKANA_RANGES + HANGUL_RANGES + unspaced = "".join(f"{chr(lo)}-{chr(hi)}" for lo, hi in ranges) + spaced = rf"[^\W{unspaced}]" + checked: list[str] = [] + open_start: list[str] = [] + for form in sorted(forms, key=len, reverse=True): + alternative = re.escape(form) + ("" if re.search(f"[{unspaced}]$", form) else rf"(?!{spaced})") + (open_start if re.match(f"[{unspaced}]", form) else checked).append(alternative) + branches = [rf"(? dict[str, str]: return {form.strip(): replacement.strip() for form, replacement in v.items()} def _pattern(self) -> re.Pattern[str] | None: - if not self.respell: - return None - alternatives = "|".join(sorted((re.escape(form) for form in self.respell), key=len, reverse=True)) - return re.compile(rf"(? list[str]: """The forms in ``respell`` that ``text`` uses, lowercased, in order.""" diff --git a/src/usersim/asset_gen/financial_services/validate.py b/src/usersim/asset_gen/financial_services/validate.py index e523ba71..9dc75f4f 100644 --- a/src/usersim/asset_gen/financial_services/validate.py +++ b/src/usersim/asset_gen/financial_services/validate.py @@ -189,6 +189,32 @@ def _scale_word_forms(iv: int) -> set: return out +def _myriad_forms(iv: int) -> set: + """Japanese and Korean renderings of a value (``10000000`` -> ``'1,000万'``, ``'1천만'``). + + Both languages count large amounts in units of 10^4 (万, 만) and 10^8 (億, 억), + so ten million yen is ``1,000万円`` and 150 million won ``1억 5,000만 원``: the + exact value is present without its digit string. A value that isn't a multiple + of 10^4 has none. + """ + out: set = set() + if iv < 10_000 or iv % 10_000: + return out + oku, man = divmod(iv // 10_000, 10_000) + for man_unit, oku_unit, thousand in (("万", "億", "千"), ("만", "억", "천")): + lows = {f"{man:,}{man_unit}", f"{man}{man_unit}"} + if man % 1000 == 0: + lows.add(f"{man // 1000}{thousand}{man_unit}") + highs = {f"{oku:,}{oku_unit}", f"{oku}{oku_unit}"} + if not oku: + out |= lows + elif not man: + out |= highs + else: + out |= {f"{high}{space}{low}" for high in highs for low in lows for space in ("", " ")} + return out + + #: Digit-group separators of the number formats that write a decimal comma, keyed #: by language: French writes ``1 234,56`` (the space is often a no-break or narrow #: no-break one), German and Brazilian Portuguese ``1.234,56``. @@ -235,11 +261,13 @@ def _value_candidates(val: Any, number_format: str | None = None) -> list[str]: return list(cands) if isinstance(val, int) or (isinstance(val, float) and float(val).is_integer()): iv = int(val) - # Plain digits, Western grouping, and Indian grouping / scale words, so a - # locale-natural rendering counts as faithful (a US corpus will never - # contain "lakh", so this cannot mask a wrong number). + # Plain digits, Western grouping, Indian grouping and scale words, and + # Japanese and Korean units of 10^4, so a locale-natural rendering counts + # as faithful (a US corpus will never contain "lakh" or "万", so none of + # these can mask a wrong number). cands.update({str(iv), f"{iv:,}", _indian_grouped(iv)}) cands.update(_scale_word_forms(iv)) + cands.update(_myriad_forms(iv)) if isinstance(val, float): cands.update({f"{val}", f"{val:.2f}", f"{val:,.2f}"}) return [c for c in cands if c] diff --git a/src/usersim/engine/core/finance_bank.py b/src/usersim/engine/core/finance_bank.py index 470e371f..b18b3de6 100644 --- a/src/usersim/engine/core/finance_bank.py +++ b/src/usersim/engine/core/finance_bank.py @@ -140,6 +140,10 @@ def to_openai_tool(self) -> dict[str, Any]: "de": (".", ",", True), } +#: Languages whose currency has no minor unit and is written straight after the +#: number: ``1,234円`` in Japanese and ``1,234원`` in Korean. +_WHOLE_UNIT_AFTER: frozenset[str] = frozenset({"ja", "ko"}) + #: Date patterns by locale, then by language. Anything else writes day/month/year, #: the most common convention. _DATE_FORMATS: dict[str, str] = { @@ -156,9 +160,10 @@ def format_amount(amount: Any, *, currency_symbol: str, number_format: str, drop """``amount`` in a region's currency and number format. ``$1,234.50`` in the US, ``₹1,23,456.00`` in India, ``1 234,50 €`` in France - and ``1.234,50 €`` in Germany. ``drop_zero_cents`` writes a whole amount - without decimals (``22 950 €``). A value that isn't a number comes back as - given, after the symbol. + and ``1.234,50 €`` in Germany. Japan and Korea write a whole amount without + decimals, since yen and won have no minor unit (``1,234円``, ``1,234원``); + elsewhere ``drop_zero_cents`` does that (``22 950 €``). A value that isn't a + number comes back as given, after the symbol. """ try: value = Decimal(str(amount)) @@ -168,6 +173,9 @@ def format_amount(amount: Any, *, currency_symbol: str, number_format: str, drop return f"{currency_symbol}{amount}" sign = "-" if value < 0 else "" whole, cents = f"{abs(value):.2f}".split(".") + if number_format.split("_")[0].lower() in _WHOLE_UNIT_AFTER: + number = f"{int(whole):,}" if cents == "00" else f"{int(whole):,}.{cents}" + return f"{sign}{number}{currency_symbol}" if number_format.endswith("_IN"): grouped, decimal_mark, symbol_after = _indian_grouped(whole), ".", False else: diff --git a/tests/engine/probes/test_financial_services.py b/tests/engine/probes/test_financial_services.py index dd0af4ee..0a1cc290 100644 --- a/tests/engine/probes/test_financial_services.py +++ b/tests/engine/probes/test_financial_services.py @@ -2286,6 +2286,18 @@ def test_percentages_and_whole_amounts_follow_the_region(self): assert format_amount(7.9, currency_symbol="€", number_format="de_DE", drop_zero_cents=True) == "7,90 €" assert format_amount(7.9, currency_symbol="£", number_format="en_GB") == "£7.90" + def test_yen_and_won_are_whole_amounts_with_the_unit_after(self): + from usersim.engine.core.finance_bank import format_percent + + us = load_finance_bank_for_locale("en_US") + japan = _regional(us, "ja_JP", currency_symbol="円", number_format="ja_JP") + korea = _regional(us, "ko_KR", currency_symbol="원", number_format="ko_KR") + assert japan.format_money("1234567") == "1,234,567円" + assert korea.format_money("-50000") == "-50,000원" + assert japan.format_money("1234.5") == "1,234.50円" + assert format_percent(0.25, number_format="ja_JP") == "0.25%" + assert format_percent(3.5, number_format="ko_KR") == "3.5%" + def test_dates_follow_the_region(self): day = datetime.date(1980, 3, 27) us = load_finance_bank_for_locale("en_US") diff --git a/tests/test_asset_gen_region_spec.py b/tests/test_asset_gen_region_spec.py index d389330a..1be449b9 100644 --- a/tests/test_asset_gen_region_spec.py +++ b/tests/test_asset_gen_region_spec.py @@ -964,6 +964,14 @@ def test_respelling_matches_whole_words_and_keeps_capitalisation(): ) +def test_respelling_matches_inside_japanese_and_korean_words(): + """A form in Japanese or Korean script needs no space beside it; a Latin edge still does.""" + conventions = _with_conventions("", {"ATMカード": "キャッシュカード", "직불카드": "체크카드"}).language_conventions + text = "新しいATMカードが届きました。직불카드를 해지하려면 XATMカード" + assert conventions.find(text) == ["atmカード", "직불카드"] + assert conventions.apply(text) == "新しいキャッシュカードが届きました。체크카드를 해지하려면 XATMカード" + + @pytest.mark.parametrize("prompt_name", ["BRIEF_PROMPT", "DOCSET_PROMPT"]) def test_language_guidance_adds_exactly_one_line_to_each_prompt(prompt_name): """Without conventions both prompts render as they do for a spec that has none; diff --git a/tests/test_asset_gen_validate.py b/tests/test_asset_gen_validate.py index 816b8605..01cdbd4b 100644 --- a/tests/test_asset_gen_validate.py +++ b/tests/test_asset_gen_validate.py @@ -217,6 +217,19 @@ def test_numeric_faithfulness_accepts_lakh_and_crore_phrasing(): assert _value_present(10_000_000, "cover up to Rs 1 crore") +def test_numeric_faithfulness_accepts_japanese_and_korean_units_of_ten_thousand(): + """``1,000万円`` and ``1억 원`` state the value without its digit string.""" + from usersim.asset_gen.financial_services.validate import _value_present + + assert _value_present(10_000_000, "預金保険で1,000万円まで保護されます", "ja_JP") + assert _value_present(18_000_000, "非課税保有限度額は1,800万円です", "ja_JP") + assert _value_present(2_400_000, "年間240万円まで投資できます", "ja_JP") + assert _value_present(150_000_000, "保障額は1億5,000万円です", "ja_JP") + assert _value_present(100_000_000, "예금자 보호 한도는 1억 원입니다", "ko_KR") + assert _value_present(50_000_000, "최대 5천만 원까지 보호됩니다", "ko_KR") + assert _value_present(150_000_000, "대출 한도는 1억 5,000만 원입니다", "ko_KR") + + def test_numeric_faithfulness_still_rejects_a_wrong_number(): """Tolerating locale formats must not tolerate a DIFFERENT value.""" from usersim.asset_gen.financial_services.validate import _value_present @@ -224,6 +237,9 @@ def test_numeric_faithfulness_still_rejects_a_wrong_number(): assert not _value_present(250_000, "minimum balance of Rs 2,60,000 applies") assert not _value_present(250_000, "a ceiling of Rs 3 lakh per year") assert not _value_present(250_000, "no figure at all here") + assert not _value_present(10_000_000, "上限は2,000万円です", "ja_JP") + assert not _value_present(150_000_000, "保障額は1億6,000万円です", "ja_JP") + assert not _value_present(100_000_000, "한도는 2억 원입니다", "ko_KR") def test_numeric_faithfulness_accepts_french_and_german_number_formats():