fix(rpgmaker): preserve 408 comments across batch failures
This commit is contained in:
parent
25ed3406d6
commit
ca7cb81929
3 changed files with 112 additions and 14 deletions
|
|
@ -13,7 +13,7 @@ from colorama import Fore
|
|||
from dotenv import load_dotenv
|
||||
from retry import retry
|
||||
from tqdm import tqdm
|
||||
from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost, getPricingConfig, calculateCost, get_var_translation, set_var_translations_batch, convert_corner_brackets, parseVocabWithCategories
|
||||
from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost, getPricingConfig, calculateCost, get_var_translation, set_var_translations_batch, convert_corner_brackets, parseVocabWithCategories, last_translation_had_mismatch
|
||||
from util.speakers import SPEAKER_BRACKET_INNER, strip_speaker_prefix
|
||||
from util.skills import ctx, load_system_prompt
|
||||
from util.paths import VOCAB_PATH
|
||||
|
|
@ -142,13 +142,13 @@ TLSYSTEMSWITCHES = False
|
|||
JOIN408 = False
|
||||
|
||||
# Dialogue / Scroll / Choices (Main Codes)
|
||||
CODE101 = True
|
||||
CODE401 = True
|
||||
CODE405 = True
|
||||
CODE102 = True
|
||||
CODE101 = False
|
||||
CODE401 = False
|
||||
CODE405 = False
|
||||
CODE102 = False
|
||||
|
||||
# Optional
|
||||
CODE408 = True
|
||||
CODE408 = False
|
||||
|
||||
# Variables
|
||||
CODE122 = False
|
||||
|
|
@ -163,7 +163,7 @@ CODE356 = False
|
|||
CODE320 = False
|
||||
CODE324 = False
|
||||
CODE325 = False
|
||||
CODE111 = False
|
||||
CODE111 = True
|
||||
CODE108 = False
|
||||
|
||||
# ─── Plugin Manager ──────────────────────────────────────────────────────────
|
||||
|
|
@ -3677,14 +3677,12 @@ def searchCodes(page, pbar, jobList, filename):
|
|||
|
||||
## Event Code: 408 (Script)
|
||||
if "code" in codeList[i] and (codeList[i]["code"] == 408) and CODE408 is True:
|
||||
# Only translate if preceded by a 108 with "選択肢ヘルプ" or another 408
|
||||
# A 108 starts an RPG Maker comment and each following 408
|
||||
# continues it. Accept the first continuation regardless of the
|
||||
# 108 text so the entire comment block is handled consistently.
|
||||
if i > 0:
|
||||
prevCode = codeList[i - 1].get("code", None)
|
||||
if prevCode == 408:
|
||||
pass # Consecutive 408s are allowed
|
||||
elif prevCode == 108 and len(codeList[i - 1].get("parameters", [])) > 0 and codeList[i - 1]["parameters"][0] == "選択肢ヘルプ":
|
||||
pass # 108 with 選択肢ヘルプ is allowed
|
||||
else:
|
||||
if prevCode not in [108, 408]:
|
||||
i += 1
|
||||
continue
|
||||
|
||||
|
|
@ -4339,10 +4337,15 @@ def searchCodes(page, pbar, jobList, filename):
|
|||
list408TL = response[0]
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
if len(list408TL) != len(list408):
|
||||
failed408Batch = bool(getattr(THREAD_CTX, "last_translation_had_mismatch", False))
|
||||
if failed408Batch or len(list408TL) != len(list408):
|
||||
with LOCK:
|
||||
if filename not in MISMATCH:
|
||||
MISMATCH.append(filename)
|
||||
# A failed internal chunk or short result would make this pass
|
||||
# partially apply or misalign later entries. Keep the entire
|
||||
# 408 batch untouched so it can be retried safely.
|
||||
list408TL = []
|
||||
|
||||
# 324
|
||||
if len(list324) > 0:
|
||||
|
|
@ -5204,6 +5207,7 @@ def translateAI(text, history, history_ctx=None):
|
|||
lock=LOCK,
|
||||
mismatchList=MISMATCH
|
||||
)
|
||||
THREAD_CTX.last_translation_had_mismatch = last_translation_had_mismatch()
|
||||
|
||||
# ── Restore \n[X] codes in translated output ───────────────────────────
|
||||
def _restore(s: str, reverse_map: dict[str, str]) -> str:
|
||||
|
|
|
|||
|
|
@ -118,12 +118,15 @@ def _run_search_codes(
|
|||
preserve_original=True,
|
||||
speaker_fn=None,
|
||||
ignore_tl_text=False,
|
||||
translate_fn=None,
|
||||
):
|
||||
"""Full Pass 1 -> mock translate -> Pass 2 cycle."""
|
||||
captured = []
|
||||
|
||||
def translate(text, history, batch=False):
|
||||
captured.append(copy.deepcopy(text))
|
||||
if translate_fn is not None:
|
||||
return translate_fn(text, history, batch)
|
||||
return _mock_translate(text, history, batch)
|
||||
|
||||
def speaker(name):
|
||||
|
|
@ -139,6 +142,12 @@ def _run_search_codes(
|
|||
orig_102 = mvmz.CODE102
|
||||
orig_preserve = mvmz.PRESERVEORIGINAL
|
||||
orig_ignore = mvmz.IGNORETLTEXT
|
||||
missing_marker = object()
|
||||
orig_mismatch_marker = getattr(
|
||||
mvmz.THREAD_CTX,
|
||||
"last_translation_had_mismatch",
|
||||
missing_marker,
|
||||
)
|
||||
mvmz.translateAI = translate
|
||||
mvmz.getSpeaker = speaker
|
||||
mvmz.CODE122 = True
|
||||
|
|
@ -150,6 +159,7 @@ def _run_search_codes(
|
|||
mvmz.PRESERVEORIGINAL = preserve_original
|
||||
mvmz.IGNORETLTEXT = ignore_tl_text
|
||||
try:
|
||||
mvmz.THREAD_CTX.last_translation_had_mismatch = False
|
||||
page_copy = copy.deepcopy(page)
|
||||
mvmz.searchCodes(page_copy, None, [], "TestMap.json")
|
||||
return page_copy, captured
|
||||
|
|
@ -164,6 +174,13 @@ def _run_search_codes(
|
|||
mvmz.CODE102 = orig_102
|
||||
mvmz.PRESERVEORIGINAL = orig_preserve
|
||||
mvmz.IGNORETLTEXT = orig_ignore
|
||||
if orig_mismatch_marker is missing_marker:
|
||||
try:
|
||||
del mvmz.THREAD_CTX.last_translation_had_mismatch
|
||||
except AttributeError:
|
||||
pass
|
||||
else:
|
||||
mvmz.THREAD_CTX.last_translation_had_mismatch = orig_mismatch_marker
|
||||
|
||||
|
||||
def _find_commands(page, code):
|
||||
|
|
@ -425,6 +442,76 @@ class TestMVMZSourceOriginal(unittest.TestCase):
|
|||
continue
|
||||
self.assertTrue(_has_japanese(item), f"408 re-run sent non-Japanese: {item!r}")
|
||||
|
||||
def test_first_408_after_empty_108_is_translated_and_preserved(self):
|
||||
page = {
|
||||
"list": [
|
||||
{"code": 108, "indent": 0, "parameters": [""]},
|
||||
{"code": 408, "indent": 0, "parameters": ["テレポート。"]},
|
||||
{"code": 408, "indent": 0, "parameters": ["条件スイッチ。"]},
|
||||
]
|
||||
}
|
||||
|
||||
page, _ = _run_search_codes(page)
|
||||
comments = _find_commands(page, 408)
|
||||
|
||||
self.assertEqual(comments[0].get("_original"), "テレポート。")
|
||||
self.assertEqual(comments[1].get("_original"), "条件スイッチ。")
|
||||
self.assertEqual(comments[0]["parameters"][0], "EN_TRANSLATED")
|
||||
self.assertEqual(comments[1]["parameters"][0], "EN_TRANSLATED")
|
||||
|
||||
def test_failed_408_fallback_does_not_write_originals(self):
|
||||
page = {
|
||||
"list": [
|
||||
{"code": 108, "indent": 0, "parameters": [""]},
|
||||
{"code": 408, "indent": 0, "parameters": ["第一行"]},
|
||||
{"code": 408, "indent": 0, "parameters": ["第二行"]},
|
||||
]
|
||||
}
|
||||
|
||||
def failed_translation(text, _history, _batch=False):
|
||||
if isinstance(text, list) and text == ["第一行", "第二行"]:
|
||||
if "TestMap.json" not in mvmz.MISMATCH:
|
||||
mvmz.MISMATCH.append("TestMap.json")
|
||||
mvmz.THREAD_CTX.last_translation_had_mismatch = True
|
||||
# Simulate one successful internal chunk and one chunk that
|
||||
# exhausted retries and fell back to its source text.
|
||||
return [["First line", text[1]], [0, 0]]
|
||||
return _mock_translate(text, _history, _batch)
|
||||
|
||||
original_mismatches = mvmz.MISMATCH[:]
|
||||
try:
|
||||
page, _ = _run_search_codes(page, translate_fn=failed_translation)
|
||||
finally:
|
||||
mvmz.MISMATCH[:] = original_mismatches
|
||||
|
||||
comments = _find_commands(page, 408)
|
||||
self.assertEqual([cmd["parameters"][0] for cmd in comments], ["第一行", "第二行"])
|
||||
self.assertTrue(all("_original" not in cmd for cmd in comments))
|
||||
|
||||
def test_short_408_batch_is_not_partially_applied(self):
|
||||
page = {
|
||||
"list": [
|
||||
{"code": 108, "indent": 0, "parameters": [""]},
|
||||
{"code": 408, "indent": 0, "parameters": ["第一行"]},
|
||||
{"code": 408, "indent": 0, "parameters": ["第二行"]},
|
||||
]
|
||||
}
|
||||
|
||||
def short_translation(text, _history, _batch=False):
|
||||
if isinstance(text, list) and text == ["第一行", "第二行"]:
|
||||
return [["First line"], [0, 0]]
|
||||
return _mock_translate(text, _history, _batch)
|
||||
|
||||
original_mismatches = mvmz.MISMATCH[:]
|
||||
try:
|
||||
page, _ = _run_search_codes(page, translate_fn=short_translation)
|
||||
finally:
|
||||
mvmz.MISMATCH[:] = original_mismatches
|
||||
|
||||
comments = _find_commands(page, 408)
|
||||
self.assertEqual([cmd["parameters"][0] for cmd in comments], ["第一行", "第二行"])
|
||||
self.assertTrue(all("_original" not in cmd for cmd in comments))
|
||||
|
||||
|
||||
class TestFixtureMapOriginal(unittest.TestCase):
|
||||
"""Full fixture map covering every _original preservation code path."""
|
||||
|
|
|
|||
|
|
@ -2637,6 +2637,11 @@ def countTokens(system, user, history):
|
|||
return [inputTotalTokens, outputTotalTokens]
|
||||
|
||||
|
||||
def last_translation_had_mismatch() -> bool:
|
||||
"""Return whether the latest translateAI call on this thread fell back."""
|
||||
return bool(getattr(_thread_local, "last_translation_had_mismatch", False))
|
||||
|
||||
|
||||
@retry(exceptions=Exception, tries=5, delay=5)
|
||||
def translateAI(text, history, config, filename=None, pbar=None, lock=None, mismatchList=None):
|
||||
"""
|
||||
|
|
@ -2644,6 +2649,7 @@ def translateAI(text, history, config, filename=None, pbar=None, lock=None, mism
|
|||
|
||||
Returns [translatedText, [inputTokens, outputTokens]].
|
||||
"""
|
||||
_thread_local.last_translation_had_mismatch = False
|
||||
if not text:
|
||||
return [text, [0, 0]]
|
||||
|
||||
|
|
@ -3237,6 +3243,7 @@ def translateAI(text, history, config, filename=None, pbar=None, lock=None, mism
|
|||
pbar.update(len(tItem) if isinstance(tItem, list) else 1)
|
||||
|
||||
else: # Failure case after all retries
|
||||
_thread_local.last_translation_had_mismatch = True
|
||||
if pbar: pbar.write(f"Translation failed after {max_retries + 1} attempts. Check mismatch log.")
|
||||
|
||||
# Emit a machine-readable marker on stdout so the GUI worker
|
||||
|
|
|
|||
Loading…
Reference in a new issue