fix(rpgmaker): enforce glossary spellings for speaker names
- Prefer curated character names over cached or generated translations - Preserve name spellings inside compound labels and parsed speakers - Reload vocab before runs and refine default event-code handling - Add regression coverage for glossary precedence and matching
This commit is contained in:
parent
38c4f5350d
commit
2e7f816ecd
3 changed files with 303 additions and 14 deletions
|
|
@ -13,7 +13,7 @@ from colorama import Fore
|
|||
from dotenv import load_dotenv
|
||||
from retry import retry
|
||||
from tqdm import tqdm
|
||||
from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost, getPricingConfig, calculateCost, get_var_translation, set_var_translations_batch, convert_corner_brackets
|
||||
from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost, getPricingConfig, calculateCost, get_var_translation, set_var_translations_batch, convert_corner_brackets, parseVocabWithCategories
|
||||
from util.speakers import SPEAKER_BRACKET_INNER, strip_speaker_prefix
|
||||
from util.skills import ctx, load_system_prompt
|
||||
from util.paths import VOCAB_PATH
|
||||
|
|
@ -46,6 +46,10 @@ SPEAKER_PARSE_MODE = False
|
|||
_speakerCache = {}
|
||||
_speakerCacheLock = threading.Lock()
|
||||
SPEAKER_COLLECTED = [] # Original speaker names collected during parse mode (untranslated)
|
||||
_speakerVocabLock = threading.Lock()
|
||||
_speakerVocabSource = None
|
||||
_speakerVocabExact = {}
|
||||
_speakerVocabCharacterPairs = []
|
||||
|
||||
# Actor variable substitution (\n[X] -> name before AI, name -> \n[X] after)
|
||||
_ACTOR_MAP_CACHE: dict | None = None
|
||||
|
|
@ -144,13 +148,13 @@ TLSYSTEMSWITCHES = False
|
|||
JOIN408 = False
|
||||
|
||||
# Dialogue / Scroll / Choices (Main Codes)
|
||||
CODE101 = True
|
||||
CODE401 = True
|
||||
CODE405 = True
|
||||
CODE102 = True
|
||||
CODE101 = False
|
||||
CODE401 = False
|
||||
CODE405 = False
|
||||
CODE102 = False
|
||||
|
||||
# Optional
|
||||
CODE408 = True
|
||||
CODE408 = False
|
||||
|
||||
# Variables
|
||||
CODE122 = False
|
||||
|
|
@ -165,7 +169,7 @@ CODE356 = False
|
|||
CODE320 = False
|
||||
CODE324 = False
|
||||
CODE325 = False
|
||||
CODE111 = False
|
||||
CODE111 = True
|
||||
CODE108 = False
|
||||
|
||||
# ─── Plugin Manager ──────────────────────────────────────────────────────────
|
||||
|
|
@ -240,6 +244,16 @@ def _pat355655_captured_text(match):
|
|||
return match.group(match.lastindex)
|
||||
|
||||
|
||||
def _reload_vocab():
|
||||
"""Reload vocab.txt so edits made after module import take effect."""
|
||||
global VOCAB
|
||||
try:
|
||||
VOCAB = VOCAB_PATH.read_text(encoding="utf-8")
|
||||
except FileNotFoundError:
|
||||
VOCAB = ""
|
||||
TRANSLATION_CONFIG.vocab = VOCAB
|
||||
|
||||
|
||||
def handleMVMZ(filename, estimate):
|
||||
global ESTIMATE, TOKENS, FILENAME, MISMATCH, PROMPT
|
||||
ESTIMATE = estimate
|
||||
|
|
@ -254,6 +268,7 @@ def handleMVMZ(filename, estimate):
|
|||
# Reload system.md + per-game overlays (game.md / quirks / custom) via DAZED_GAME_ROOT.
|
||||
PROMPT = load_system_prompt()
|
||||
TRANSLATION_CONFIG.prompt = PROMPT
|
||||
_reload_vocab()
|
||||
|
||||
# Translate
|
||||
start = time.time()
|
||||
|
|
@ -4791,6 +4806,75 @@ def _normalize_speaker_nameplate(translated: str) -> str:
|
|||
return out
|
||||
|
||||
|
||||
def _is_character_vocab_category(category) -> bool:
|
||||
"""True for glossary sections that define character/speaker names."""
|
||||
name = str(category or "").lstrip("#").strip().casefold()
|
||||
primary = re.split(r"\s*[·・|/]\s*", name, maxsplit=1)[0]
|
||||
return primary in {"game characters", "speakers"}
|
||||
|
||||
|
||||
def _vocab_category_priority(category) -> int:
|
||||
"""Prefer curated character entries over generated speaker entries."""
|
||||
name = str(category or "").lstrip("#").strip().casefold()
|
||||
primary = re.split(r"\s*[·・|/]\s*", name, maxsplit=1)[0]
|
||||
if primary == "game characters":
|
||||
return 3
|
||||
if primary == "speakers":
|
||||
return 2
|
||||
return 1
|
||||
|
||||
|
||||
def _speaker_vocab_indexes():
|
||||
"""Return cached exact and character-only mappings from the current vocab."""
|
||||
global _speakerVocabSource, _speakerVocabExact, _speakerVocabCharacterPairs
|
||||
vocab_text = VOCAB or ""
|
||||
with _speakerVocabLock:
|
||||
if _speakerVocabSource == vocab_text:
|
||||
return _speakerVocabExact, _speakerVocabCharacterPairs
|
||||
|
||||
exact = {}
|
||||
exact_priorities = {}
|
||||
characters = {}
|
||||
for term, _line, category in parseVocabWithCategories(vocab_text):
|
||||
if not isinstance(term, tuple) or len(term) != 2:
|
||||
continue
|
||||
source, translated = (str(part).strip() for part in term)
|
||||
if not source or not translated or re.search(LANGREGEX, translated):
|
||||
continue
|
||||
priority = _vocab_category_priority(category)
|
||||
if priority > exact_priorities.get(source, 0):
|
||||
exact[source] = translated
|
||||
exact_priorities[source] = priority
|
||||
if _is_character_vocab_category(category):
|
||||
previous = characters.get(source)
|
||||
if previous is None or priority > previous[0]:
|
||||
characters[source] = (priority, translated)
|
||||
|
||||
_speakerVocabSource = vocab_text
|
||||
_speakerVocabExact = exact
|
||||
_speakerVocabCharacterPairs = sorted(
|
||||
((source, value[1]) for source, value in characters.items()),
|
||||
key=lambda pair: len(pair[0]),
|
||||
reverse=True,
|
||||
)
|
||||
return _speakerVocabExact, _speakerVocabCharacterPairs
|
||||
|
||||
|
||||
def _vocab_speaker_lookup(speaker: str):
|
||||
"""Resolve an exact speaker/label from vocab without asking the model."""
|
||||
exact, _characters = _speaker_vocab_indexes()
|
||||
return exact.get(speaker)
|
||||
|
||||
|
||||
def _substitute_vocab_character_names(text: str) -> str:
|
||||
"""Enforce known character spellings inside compound labels."""
|
||||
_exact, characters = _speaker_vocab_indexes()
|
||||
for source, translated in characters:
|
||||
if source in text:
|
||||
text = text.replace(source, translated)
|
||||
return text
|
||||
|
||||
|
||||
# Save some money and enter the character before translation
|
||||
def getSpeaker(speaker: str):
|
||||
"""Return (and possibly collect) speaker name.
|
||||
|
|
@ -4816,18 +4900,38 @@ def getSpeaker(speaker: str):
|
|||
SPEAKER_COLLECTED.append(speaker)
|
||||
return [speaker, [0, 0]]
|
||||
|
||||
# vocab.txt is authoritative. Check it before either cache so an old model
|
||||
# romanization can never override a newly curated spelling.
|
||||
vocab_hit = _vocab_speaker_lookup(speaker)
|
||||
if vocab_hit is not None:
|
||||
with _speakerCacheLock:
|
||||
_speakerCache[speaker] = vocab_hit
|
||||
for item in NAMESLIST:
|
||||
if item[0] == speaker:
|
||||
item[1] = vocab_hit
|
||||
break
|
||||
else:
|
||||
NAMESLIST.append([speaker, vocab_hit])
|
||||
return [vocab_hit, [0, 0]]
|
||||
|
||||
# Normal mode translation path
|
||||
with _speakerCacheLock:
|
||||
cached = _speakerCache.get(speaker)
|
||||
if cached is not None:
|
||||
return [cached, [0, 0]]
|
||||
|
||||
# A few RPG Maker fields routed through getSpeaker are compound labels
|
||||
# (for example, "ユウイベント5"). Replace curated character names before
|
||||
# translation so the model cannot choose a different romanization. This
|
||||
# also avoids reusing a stale cache entry keyed to the all-Japanese label.
|
||||
translation_source = _substitute_vocab_character_names(speaker)
|
||||
|
||||
try:
|
||||
THREAD_CTX.in_speaker = True
|
||||
except Exception:
|
||||
pass
|
||||
response = translateAI(
|
||||
speaker,
|
||||
translation_source,
|
||||
ctx("names.speaker"),
|
||||
False,
|
||||
)
|
||||
|
|
@ -4843,7 +4947,7 @@ def getSpeaker(speaker: str):
|
|||
except Exception:
|
||||
pass
|
||||
response = translateAI(
|
||||
speaker,
|
||||
translation_source,
|
||||
ctx("names.speaker"),
|
||||
False,
|
||||
)
|
||||
|
|
@ -5052,12 +5156,31 @@ def finalizeSpeakerParse():
|
|||
if not SPEAKER_PARSE_MODE:
|
||||
return
|
||||
try:
|
||||
# Step 1: batch translate any collected speakers not already translated
|
||||
# Step 1: seed curated vocab hits, then batch translate only unresolved
|
||||
# speakers. This avoids generating a contradictory # Speakers spelling
|
||||
# for a name already defined under # Game Characters.
|
||||
_reload_vocab()
|
||||
to_translate = []
|
||||
vocab_resolved = []
|
||||
for s in SPEAKER_COLLECTED:
|
||||
if not s:
|
||||
continue
|
||||
vocab_hit = _vocab_speaker_lookup(s)
|
||||
if vocab_hit is not None:
|
||||
vocab_resolved.append((s, vocab_hit))
|
||||
else:
|
||||
to_translate.append(s)
|
||||
|
||||
with _speakerCacheLock:
|
||||
for s in SPEAKER_COLLECTED:
|
||||
if s not in _speakerCache and s != "":
|
||||
to_translate.append(s)
|
||||
for source, translated in vocab_resolved:
|
||||
_speakerCache[source] = translated
|
||||
for item in NAMESLIST:
|
||||
if item[0] == source:
|
||||
item[1] = translated
|
||||
break
|
||||
else:
|
||||
NAMESLIST.append([source, translated])
|
||||
to_translate = [s for s in to_translate if s not in _speakerCache]
|
||||
if to_translate:
|
||||
try:
|
||||
THREAD_CTX.in_speaker = True
|
||||
|
|
|
|||
131
tests/test_mvmz_speaker_vocab.py
Normal file
131
tests/test_mvmz_speaker_vocab.py
Normal file
|
|
@ -0,0 +1,131 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Regression tests for deterministic RPG Maker speaker glossary handling."""
|
||||
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
from unittest.mock import patch
|
||||
|
||||
import modules.rpgmakermvmz as mvmz
|
||||
from util.translation import buildMatchedVocabText, parseVocabWithCategories
|
||||
|
||||
|
||||
VOCAB = (
|
||||
"# Game Characters\n"
|
||||
"ユウ (Yuu) - Male protagonist.\n"
|
||||
"ハルカ (Haruka) - Female heroine.\n\n"
|
||||
"# Terms\n"
|
||||
"教室 (Classroom)\n"
|
||||
)
|
||||
|
||||
|
||||
class MVMZSpeakerVocabTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.original_vocab = mvmz.VOCAB
|
||||
self.original_config_vocab = mvmz.TRANSLATION_CONFIG.vocab
|
||||
self.original_names = mvmz.NAMESLIST
|
||||
self.original_collected = mvmz.SPEAKER_COLLECTED
|
||||
self.original_parse_mode = mvmz.SPEAKER_PARSE_MODE
|
||||
self.original_preflight = mvmz.PREFLIGHT_COUNT_MODE
|
||||
mvmz.VOCAB = VOCAB
|
||||
mvmz.NAMESLIST = []
|
||||
mvmz.SPEAKER_COLLECTED = []
|
||||
mvmz.SPEAKER_PARSE_MODE = False
|
||||
mvmz.PREFLIGHT_COUNT_MODE = False
|
||||
with mvmz._speakerCacheLock:
|
||||
mvmz._speakerCache.clear()
|
||||
|
||||
def tearDown(self):
|
||||
mvmz.VOCAB = self.original_vocab
|
||||
mvmz.TRANSLATION_CONFIG.vocab = self.original_config_vocab
|
||||
mvmz.NAMESLIST = self.original_names
|
||||
mvmz.SPEAKER_COLLECTED = self.original_collected
|
||||
mvmz.SPEAKER_PARSE_MODE = self.original_parse_mode
|
||||
mvmz.PREFLIGHT_COUNT_MODE = self.original_preflight
|
||||
with mvmz._speakerCacheLock:
|
||||
mvmz._speakerCache.clear()
|
||||
|
||||
def test_exact_vocab_name_bypasses_model_and_stale_cache(self):
|
||||
with mvmz._speakerCacheLock:
|
||||
mvmz._speakerCache["ユウ"] = "Yu"
|
||||
|
||||
with patch.object(mvmz, "translateAI") as translate:
|
||||
result = mvmz.getSpeaker("ユウ")
|
||||
|
||||
self.assertEqual(result, ["Yuu", [0, 0]])
|
||||
translate.assert_not_called()
|
||||
|
||||
def test_character_name_is_substituted_inside_compound_label(self):
|
||||
def translate(text, _context, _batch=False):
|
||||
self.assertEqual(text, "Yuuイベント5")
|
||||
return ["Yuu Event 5", [3, 2]]
|
||||
|
||||
with patch.object(mvmz, "translateAI", side_effect=translate):
|
||||
result = mvmz.getSpeaker("ユウイベント5")
|
||||
|
||||
self.assertEqual(result, ["Yuu Event 5", [3, 2]])
|
||||
|
||||
def test_game_characters_override_generated_speaker_spelling(self):
|
||||
mvmz.VOCAB = (
|
||||
"# Game Characters\nユウ (Yuu) - Male protagonist.\n\n"
|
||||
"# Speakers\nユウ (Yu)\n"
|
||||
)
|
||||
|
||||
with patch.object(mvmz, "translateAI") as translate:
|
||||
result = mvmz.getSpeaker("ユウ")
|
||||
|
||||
self.assertEqual(result, ["Yuu", [0, 0]])
|
||||
translate.assert_not_called()
|
||||
|
||||
def test_reload_vocab_updates_translation_config(self):
|
||||
with TemporaryDirectory() as tmp:
|
||||
vocab_path = Path(tmp) / "vocab.txt"
|
||||
vocab_path.write_text("# Game Characters\nユウ (You)\n", encoding="utf-8")
|
||||
with patch.object(mvmz, "VOCAB_PATH", vocab_path):
|
||||
mvmz._reload_vocab()
|
||||
|
||||
self.assertIn("ユウ (You)", mvmz.VOCAB)
|
||||
self.assertEqual(mvmz.TRANSLATION_CONFIG.vocab, mvmz.VOCAB)
|
||||
|
||||
def test_parse_speakers_reuses_curated_name(self):
|
||||
mvmz.SPEAKER_PARSE_MODE = True
|
||||
mvmz.SPEAKER_COLLECTED = ["ユウ"]
|
||||
with TemporaryDirectory() as tmp:
|
||||
vocab_path = Path(tmp) / "vocab.txt"
|
||||
vocab_path.write_text(VOCAB, encoding="utf-8")
|
||||
with (
|
||||
patch.object(mvmz, "VOCAB_PATH", vocab_path),
|
||||
patch.object(mvmz, "translateAI") as translate,
|
||||
):
|
||||
mvmz.finalizeSpeakerParse()
|
||||
written = vocab_path.read_text(encoding="utf-8")
|
||||
|
||||
translate.assert_not_called()
|
||||
self.assertIn("# Speakers\nユウ (Yuu)", written)
|
||||
|
||||
|
||||
class CharacterCompoundMatchingTests(unittest.TestCase):
|
||||
def test_character_entry_matches_inside_katakana_compound(self):
|
||||
pairs = parseVocabWithCategories(VOCAB)
|
||||
matched = buildMatchedVocabText(pairs, "ユウイベント5")
|
||||
|
||||
self.assertIn("ユウ (Yuu)", matched)
|
||||
|
||||
def test_non_character_short_term_keeps_script_boundaries(self):
|
||||
pairs = parseVocabWithCategories("# Terms\nキス (Kiss)\n")
|
||||
matched = buildMatchedVocabText(pairs, "テキスト")
|
||||
|
||||
self.assertEqual(matched, "")
|
||||
|
||||
def test_curated_character_spelling_suppresses_generated_conflict(self):
|
||||
pairs = parseVocabWithCategories(
|
||||
"# Game Characters\nユウ (Yuu)\n\n# Speakers\nユウ (Yu)\n"
|
||||
)
|
||||
matched = buildMatchedVocabText(pairs, "ユウイベント5")
|
||||
|
||||
self.assertIn("ユウ (Yuu)", matched)
|
||||
self.assertNotIn("ユウ (Yu)", matched)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
@ -1917,6 +1917,24 @@ def buildMatchedVocabText(vocabPairs, subbedText, history=None):
|
|||
"""Build formatted vocabulary text for terms found in the current batch."""
|
||||
matchedCategories = {}
|
||||
|
||||
# Generated # Speakers entries can overlap hand-curated character entries.
|
||||
# Keep only the highest-authority spelling for each character source.
|
||||
character_authority = {}
|
||||
for candidate_term, candidate_line, candidate_category in vocabPairs:
|
||||
if not isinstance(candidate_term, tuple) or len(candidate_term) != 2:
|
||||
continue
|
||||
candidate_name = str(candidate_category or "").lstrip("#").strip().casefold()
|
||||
candidate_primary = re.split(
|
||||
r"\s*[·・|/]\s*", candidate_name, maxsplit=1
|
||||
)[0]
|
||||
priority = {"game characters": 2, "speakers": 1}.get(candidate_primary, 0)
|
||||
if not priority:
|
||||
continue
|
||||
source = candidate_term[0]
|
||||
previous = character_authority.get(source)
|
||||
if previous is None or priority > previous[0]:
|
||||
character_authority[source] = (priority, candidate_line)
|
||||
|
||||
# Only match against the current request text. History is deliberately not
|
||||
# searched so stale terms are not resent in unrelated batches.
|
||||
textToSearch = _text_for_vocab_search(subbedText)
|
||||
|
|
@ -1928,7 +1946,24 @@ def buildMatchedVocabText(vocabPairs, subbedText, history=None):
|
|||
if isinstance(term, tuple):
|
||||
# Check both Japanese and English terms
|
||||
japanese_term, english_term = term
|
||||
if _vocab_term_in_text(japanese_term, textToSearch) or _vocab_term_in_text(english_term, textToSearch):
|
||||
category_name = str(category or "").lstrip("#").strip().casefold()
|
||||
category_primary = re.split(
|
||||
r"\s*[·・|/]\s*", category_name, maxsplit=1
|
||||
)[0]
|
||||
authoritative = character_authority.get(japanese_term)
|
||||
if authoritative is not None and authoritative[1] != line:
|
||||
continue
|
||||
japanese_match = _vocab_term_in_text(japanese_term, textToSearch)
|
||||
# Character names often appear inside compound event/map labels,
|
||||
# e.g. ユウイベント. For character sections only, a substring is
|
||||
# intentional and should still attach the authoritative spelling.
|
||||
if (
|
||||
not japanese_match
|
||||
and category_primary in {"game characters", "speakers"}
|
||||
and japanese_term in textToSearch
|
||||
):
|
||||
japanese_match = True
|
||||
if japanese_match or _vocab_term_in_text(english_term, textToSearch):
|
||||
term_found = True
|
||||
else:
|
||||
# Single term check
|
||||
|
|
|
|||
Loading…
Reference in a new issue