From 2e7f816ecd9108339ce535531e173edd4e7e3cdf Mon Sep 17 00:00:00 2001 From: DazedAnon Date: Sat, 25 Jul 2026 14:01:04 -0500 Subject: [PATCH] fix(rpgmaker): enforce glossary spellings for speaker names - Prefer curated character names over cached or generated translations - Preserve name spellings inside compound labels and parsed speakers - Reload vocab before runs and refine default event-code handling - Add regression coverage for glossary precedence and matching --- modules/rpgmakermvmz.py | 149 ++++++++++++++++++++++++++++--- tests/test_mvmz_speaker_vocab.py | 131 +++++++++++++++++++++++++++ util/translation.py | 37 +++++++- 3 files changed, 303 insertions(+), 14 deletions(-) create mode 100644 tests/test_mvmz_speaker_vocab.py diff --git a/modules/rpgmakermvmz.py b/modules/rpgmakermvmz.py index 3d03f59..f80b9b1 100644 --- a/modules/rpgmakermvmz.py +++ b/modules/rpgmakermvmz.py @@ -13,7 +13,7 @@ from colorama import Fore from dotenv import load_dotenv from retry import retry from tqdm import tqdm -from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost, getPricingConfig, calculateCost, get_var_translation, set_var_translations_batch, convert_corner_brackets +from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost, getPricingConfig, calculateCost, get_var_translation, set_var_translations_batch, convert_corner_brackets, parseVocabWithCategories from util.speakers import SPEAKER_BRACKET_INNER, strip_speaker_prefix from util.skills import ctx, load_system_prompt from util.paths import VOCAB_PATH @@ -46,6 +46,10 @@ SPEAKER_PARSE_MODE = False _speakerCache = {} _speakerCacheLock = threading.Lock() SPEAKER_COLLECTED = [] # Original speaker names collected during parse mode (untranslated) +_speakerVocabLock = threading.Lock() +_speakerVocabSource = None +_speakerVocabExact = {} +_speakerVocabCharacterPairs = [] # Actor variable substitution (\n[X] -> name before AI, name -> \n[X] after) _ACTOR_MAP_CACHE: dict | None = None @@ -144,13 +148,13 @@ TLSYSTEMSWITCHES = False JOIN408 = False # Dialogue / Scroll / Choices (Main Codes) -CODE101 = True -CODE401 = True -CODE405 = True -CODE102 = True +CODE101 = False +CODE401 = False +CODE405 = False +CODE102 = False # Optional -CODE408 = True +CODE408 = False # Variables CODE122 = False @@ -165,7 +169,7 @@ CODE356 = False CODE320 = False CODE324 = False CODE325 = False -CODE111 = False +CODE111 = True CODE108 = False # ─── Plugin Manager ────────────────────────────────────────────────────────── @@ -240,6 +244,16 @@ def _pat355655_captured_text(match): return match.group(match.lastindex) +def _reload_vocab(): + """Reload vocab.txt so edits made after module import take effect.""" + global VOCAB + try: + VOCAB = VOCAB_PATH.read_text(encoding="utf-8") + except FileNotFoundError: + VOCAB = "" + TRANSLATION_CONFIG.vocab = VOCAB + + def handleMVMZ(filename, estimate): global ESTIMATE, TOKENS, FILENAME, MISMATCH, PROMPT ESTIMATE = estimate @@ -254,6 +268,7 @@ def handleMVMZ(filename, estimate): # Reload system.md + per-game overlays (game.md / quirks / custom) via DAZED_GAME_ROOT. PROMPT = load_system_prompt() TRANSLATION_CONFIG.prompt = PROMPT + _reload_vocab() # Translate start = time.time() @@ -4791,6 +4806,75 @@ def _normalize_speaker_nameplate(translated: str) -> str: return out +def _is_character_vocab_category(category) -> bool: + """True for glossary sections that define character/speaker names.""" + name = str(category or "").lstrip("#").strip().casefold() + primary = re.split(r"\s*[·・|/]\s*", name, maxsplit=1)[0] + return primary in {"game characters", "speakers"} + + +def _vocab_category_priority(category) -> int: + """Prefer curated character entries over generated speaker entries.""" + name = str(category or "").lstrip("#").strip().casefold() + primary = re.split(r"\s*[·・|/]\s*", name, maxsplit=1)[0] + if primary == "game characters": + return 3 + if primary == "speakers": + return 2 + return 1 + + +def _speaker_vocab_indexes(): + """Return cached exact and character-only mappings from the current vocab.""" + global _speakerVocabSource, _speakerVocabExact, _speakerVocabCharacterPairs + vocab_text = VOCAB or "" + with _speakerVocabLock: + if _speakerVocabSource == vocab_text: + return _speakerVocabExact, _speakerVocabCharacterPairs + + exact = {} + exact_priorities = {} + characters = {} + for term, _line, category in parseVocabWithCategories(vocab_text): + if not isinstance(term, tuple) or len(term) != 2: + continue + source, translated = (str(part).strip() for part in term) + if not source or not translated or re.search(LANGREGEX, translated): + continue + priority = _vocab_category_priority(category) + if priority > exact_priorities.get(source, 0): + exact[source] = translated + exact_priorities[source] = priority + if _is_character_vocab_category(category): + previous = characters.get(source) + if previous is None or priority > previous[0]: + characters[source] = (priority, translated) + + _speakerVocabSource = vocab_text + _speakerVocabExact = exact + _speakerVocabCharacterPairs = sorted( + ((source, value[1]) for source, value in characters.items()), + key=lambda pair: len(pair[0]), + reverse=True, + ) + return _speakerVocabExact, _speakerVocabCharacterPairs + + +def _vocab_speaker_lookup(speaker: str): + """Resolve an exact speaker/label from vocab without asking the model.""" + exact, _characters = _speaker_vocab_indexes() + return exact.get(speaker) + + +def _substitute_vocab_character_names(text: str) -> str: + """Enforce known character spellings inside compound labels.""" + _exact, characters = _speaker_vocab_indexes() + for source, translated in characters: + if source in text: + text = text.replace(source, translated) + return text + + # Save some money and enter the character before translation def getSpeaker(speaker: str): """Return (and possibly collect) speaker name. @@ -4816,18 +4900,38 @@ def getSpeaker(speaker: str): SPEAKER_COLLECTED.append(speaker) return [speaker, [0, 0]] + # vocab.txt is authoritative. Check it before either cache so an old model + # romanization can never override a newly curated spelling. + vocab_hit = _vocab_speaker_lookup(speaker) + if vocab_hit is not None: + with _speakerCacheLock: + _speakerCache[speaker] = vocab_hit + for item in NAMESLIST: + if item[0] == speaker: + item[1] = vocab_hit + break + else: + NAMESLIST.append([speaker, vocab_hit]) + return [vocab_hit, [0, 0]] + # Normal mode translation path with _speakerCacheLock: cached = _speakerCache.get(speaker) if cached is not None: return [cached, [0, 0]] + # A few RPG Maker fields routed through getSpeaker are compound labels + # (for example, "ユウイベント5"). Replace curated character names before + # translation so the model cannot choose a different romanization. This + # also avoids reusing a stale cache entry keyed to the all-Japanese label. + translation_source = _substitute_vocab_character_names(speaker) + try: THREAD_CTX.in_speaker = True except Exception: pass response = translateAI( - speaker, + translation_source, ctx("names.speaker"), False, ) @@ -4843,7 +4947,7 @@ def getSpeaker(speaker: str): except Exception: pass response = translateAI( - speaker, + translation_source, ctx("names.speaker"), False, ) @@ -5052,12 +5156,31 @@ def finalizeSpeakerParse(): if not SPEAKER_PARSE_MODE: return try: - # Step 1: batch translate any collected speakers not already translated + # Step 1: seed curated vocab hits, then batch translate only unresolved + # speakers. This avoids generating a contradictory # Speakers spelling + # for a name already defined under # Game Characters. + _reload_vocab() to_translate = [] + vocab_resolved = [] + for s in SPEAKER_COLLECTED: + if not s: + continue + vocab_hit = _vocab_speaker_lookup(s) + if vocab_hit is not None: + vocab_resolved.append((s, vocab_hit)) + else: + to_translate.append(s) + with _speakerCacheLock: - for s in SPEAKER_COLLECTED: - if s not in _speakerCache and s != "": - to_translate.append(s) + for source, translated in vocab_resolved: + _speakerCache[source] = translated + for item in NAMESLIST: + if item[0] == source: + item[1] = translated + break + else: + NAMESLIST.append([source, translated]) + to_translate = [s for s in to_translate if s not in _speakerCache] if to_translate: try: THREAD_CTX.in_speaker = True diff --git a/tests/test_mvmz_speaker_vocab.py b/tests/test_mvmz_speaker_vocab.py new file mode 100644 index 0000000..19511ab --- /dev/null +++ b/tests/test_mvmz_speaker_vocab.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python3 +"""Regression tests for deterministic RPG Maker speaker glossary handling.""" + +import unittest +from pathlib import Path +from tempfile import TemporaryDirectory +from unittest.mock import patch + +import modules.rpgmakermvmz as mvmz +from util.translation import buildMatchedVocabText, parseVocabWithCategories + + +VOCAB = ( + "# Game Characters\n" + "ユウ (Yuu) - Male protagonist.\n" + "ハルカ (Haruka) - Female heroine.\n\n" + "# Terms\n" + "教室 (Classroom)\n" +) + + +class MVMZSpeakerVocabTests(unittest.TestCase): + def setUp(self): + self.original_vocab = mvmz.VOCAB + self.original_config_vocab = mvmz.TRANSLATION_CONFIG.vocab + self.original_names = mvmz.NAMESLIST + self.original_collected = mvmz.SPEAKER_COLLECTED + self.original_parse_mode = mvmz.SPEAKER_PARSE_MODE + self.original_preflight = mvmz.PREFLIGHT_COUNT_MODE + mvmz.VOCAB = VOCAB + mvmz.NAMESLIST = [] + mvmz.SPEAKER_COLLECTED = [] + mvmz.SPEAKER_PARSE_MODE = False + mvmz.PREFLIGHT_COUNT_MODE = False + with mvmz._speakerCacheLock: + mvmz._speakerCache.clear() + + def tearDown(self): + mvmz.VOCAB = self.original_vocab + mvmz.TRANSLATION_CONFIG.vocab = self.original_config_vocab + mvmz.NAMESLIST = self.original_names + mvmz.SPEAKER_COLLECTED = self.original_collected + mvmz.SPEAKER_PARSE_MODE = self.original_parse_mode + mvmz.PREFLIGHT_COUNT_MODE = self.original_preflight + with mvmz._speakerCacheLock: + mvmz._speakerCache.clear() + + def test_exact_vocab_name_bypasses_model_and_stale_cache(self): + with mvmz._speakerCacheLock: + mvmz._speakerCache["ユウ"] = "Yu" + + with patch.object(mvmz, "translateAI") as translate: + result = mvmz.getSpeaker("ユウ") + + self.assertEqual(result, ["Yuu", [0, 0]]) + translate.assert_not_called() + + def test_character_name_is_substituted_inside_compound_label(self): + def translate(text, _context, _batch=False): + self.assertEqual(text, "Yuuイベント5") + return ["Yuu Event 5", [3, 2]] + + with patch.object(mvmz, "translateAI", side_effect=translate): + result = mvmz.getSpeaker("ユウイベント5") + + self.assertEqual(result, ["Yuu Event 5", [3, 2]]) + + def test_game_characters_override_generated_speaker_spelling(self): + mvmz.VOCAB = ( + "# Game Characters\nユウ (Yuu) - Male protagonist.\n\n" + "# Speakers\nユウ (Yu)\n" + ) + + with patch.object(mvmz, "translateAI") as translate: + result = mvmz.getSpeaker("ユウ") + + self.assertEqual(result, ["Yuu", [0, 0]]) + translate.assert_not_called() + + def test_reload_vocab_updates_translation_config(self): + with TemporaryDirectory() as tmp: + vocab_path = Path(tmp) / "vocab.txt" + vocab_path.write_text("# Game Characters\nユウ (You)\n", encoding="utf-8") + with patch.object(mvmz, "VOCAB_PATH", vocab_path): + mvmz._reload_vocab() + + self.assertIn("ユウ (You)", mvmz.VOCAB) + self.assertEqual(mvmz.TRANSLATION_CONFIG.vocab, mvmz.VOCAB) + + def test_parse_speakers_reuses_curated_name(self): + mvmz.SPEAKER_PARSE_MODE = True + mvmz.SPEAKER_COLLECTED = ["ユウ"] + with TemporaryDirectory() as tmp: + vocab_path = Path(tmp) / "vocab.txt" + vocab_path.write_text(VOCAB, encoding="utf-8") + with ( + patch.object(mvmz, "VOCAB_PATH", vocab_path), + patch.object(mvmz, "translateAI") as translate, + ): + mvmz.finalizeSpeakerParse() + written = vocab_path.read_text(encoding="utf-8") + + translate.assert_not_called() + self.assertIn("# Speakers\nユウ (Yuu)", written) + + +class CharacterCompoundMatchingTests(unittest.TestCase): + def test_character_entry_matches_inside_katakana_compound(self): + pairs = parseVocabWithCategories(VOCAB) + matched = buildMatchedVocabText(pairs, "ユウイベント5") + + self.assertIn("ユウ (Yuu)", matched) + + def test_non_character_short_term_keeps_script_boundaries(self): + pairs = parseVocabWithCategories("# Terms\nキス (Kiss)\n") + matched = buildMatchedVocabText(pairs, "テキスト") + + self.assertEqual(matched, "") + + def test_curated_character_spelling_suppresses_generated_conflict(self): + pairs = parseVocabWithCategories( + "# Game Characters\nユウ (Yuu)\n\n# Speakers\nユウ (Yu)\n" + ) + matched = buildMatchedVocabText(pairs, "ユウイベント5") + + self.assertIn("ユウ (Yuu)", matched) + self.assertNotIn("ユウ (Yu)", matched) + + +if __name__ == "__main__": + unittest.main() diff --git a/util/translation.py b/util/translation.py index b2a8870..052f98d 100644 --- a/util/translation.py +++ b/util/translation.py @@ -1917,6 +1917,24 @@ def buildMatchedVocabText(vocabPairs, subbedText, history=None): """Build formatted vocabulary text for terms found in the current batch.""" matchedCategories = {} + # Generated # Speakers entries can overlap hand-curated character entries. + # Keep only the highest-authority spelling for each character source. + character_authority = {} + for candidate_term, candidate_line, candidate_category in vocabPairs: + if not isinstance(candidate_term, tuple) or len(candidate_term) != 2: + continue + candidate_name = str(candidate_category or "").lstrip("#").strip().casefold() + candidate_primary = re.split( + r"\s*[·・|/]\s*", candidate_name, maxsplit=1 + )[0] + priority = {"game characters": 2, "speakers": 1}.get(candidate_primary, 0) + if not priority: + continue + source = candidate_term[0] + previous = character_authority.get(source) + if previous is None or priority > previous[0]: + character_authority[source] = (priority, candidate_line) + # Only match against the current request text. History is deliberately not # searched so stale terms are not resent in unrelated batches. textToSearch = _text_for_vocab_search(subbedText) @@ -1928,7 +1946,24 @@ def buildMatchedVocabText(vocabPairs, subbedText, history=None): if isinstance(term, tuple): # Check both Japanese and English terms japanese_term, english_term = term - if _vocab_term_in_text(japanese_term, textToSearch) or _vocab_term_in_text(english_term, textToSearch): + category_name = str(category or "").lstrip("#").strip().casefold() + category_primary = re.split( + r"\s*[·・|/]\s*", category_name, maxsplit=1 + )[0] + authoritative = character_authority.get(japanese_term) + if authoritative is not None and authoritative[1] != line: + continue + japanese_match = _vocab_term_in_text(japanese_term, textToSearch) + # Character names often appear inside compound event/map labels, + # e.g. ユウイベント. For character sections only, a substring is + # intentional and should still attach the authoritative spelling. + if ( + not japanese_match + and category_primary in {"game characters", "speakers"} + and japanese_term in textToSearch + ): + japanese_match = True + if japanese_match or _vocab_term_in_text(english_term, textToSearch): term_found = True else: # Single term check