From d38a8a2725f9f7c70e356831e9ed050bd70e297d Mon Sep 17 00:00:00 2001 From: DazedAnon Date: Mon, 27 Jul 2026 21:39:10 -0500 Subject: [PATCH] fix(rpgmaker): prevent control-code and choice-prefix corruption --- modules/rpgmakermvmz.py | 31 ++++-- tests/test_mvmz_source_original.py | 29 +++++ tests/test_translation_validation.py | 63 +++++++++++ util/translation.py | 156 +++++++++++++++++++++------ 4 files changed, 242 insertions(+), 37 deletions(-) create mode 100644 tests/test_translation_validation.py diff --git a/modules/rpgmakermvmz.py b/modules/rpgmakermvmz.py index eb852c8..9359500 100644 --- a/modules/rpgmakermvmz.py +++ b/modules/rpgmakermvmz.py @@ -611,6 +611,28 @@ def _apply_choice_original(cmd, index: int, raw_source: str) -> None: orig_list[index] = raw_source +def _split_choice_condition_prefix(text: str) -> tuple[str, str]: + """Split a leading lowercase ``if(...)``/``en(...)`` plugin condition safely. + + Conditions can contain nested calls such as ``if($gameSwitches.value(1))``. + Return the input untouched when the prefix is absent or unbalanced. + """ + if not isinstance(text, str) or not text.startswith(("if(", "en(")): + return "", text + + depth = 0 + for index, char in enumerate(text[2:], start=2): + if char == "(": + depth += 1 + elif char == ")": + depth -= 1 + if depth == 0: + return text[: index + 1], text[index + 1 :] + if depth < 0: + break + return "", text + + def _122_inner_source(cmd) -> str | None: """Inner quoted value for code 122: _original or extract from parameters[4].""" orig = _scalar_original(cmd) @@ -4052,13 +4074,8 @@ def searchCodes(page, pbar, jobList, filename): if not _text_needs_translation(currentChoice): continue - # If and En Statements - ifVar = "" - ifList = re.findall(r"([ei][nf]\(.+?\)\)?\)?)", jaString) - if len(ifList) != 0: - for var in ifList: - jaString = jaString.replace(var, "") - ifVar += var + # Preserve plugin condition prefixes outside the model. + ifVar, jaString = _split_choice_condition_prefix(jaString) # Store the formatting and cleaned string varList.append(ifVar) diff --git a/tests/test_mvmz_source_original.py b/tests/test_mvmz_source_original.py index c06b521..1ab6640 100644 --- a/tests/test_mvmz_source_original.py +++ b/tests/test_mvmz_source_original.py @@ -192,6 +192,35 @@ def _has_japanese(s: str) -> bool: class TestMVMZSourceOriginal(unittest.TestCase): + def test_choice_condition_prefix_parser_handles_nested_calls(self): + prefix, label = mvmz._split_choice_condition_prefix( + "if($gameSwitches.value(1) && v[31]>=4)迷宮四階" + ) + + self.assertEqual(prefix, "if($gameSwitches.value(1) && v[31]>=4)") + self.assertEqual(label, "迷宮四階") + + def test_choice_condition_prefix_parser_leaves_malformed_input_untouched(self): + source = "if(v[31]>=4迷宮四階" + self.assertEqual(mvmz._split_choice_condition_prefix(source), ("", source)) + + def test_choice_translation_restores_condition_prefix_byte_for_byte(self): + source = "if(v[31]>=4)迷宮四階：図書館" + page = { + "list": [ + {"code": 102, "indent": 0, "parameters": [[source], -1, 0, 2, 0]}, + ] + } + + def translate(text, _history, _batch=False): + self.assertEqual(text, ["迷宮四階：図書館"]) + return [["labyrinth floor 4: library"], [0, 0]] + + translated, _ = _run_search_codes(page, translate_fn=translate) + + choice = _find_commands(translated, 102)[0]["parameters"][0][0] + self.assertEqual(choice, "if(v[31]>=4)Labyrinth floor 4: library") + def test_first_pass_writes_original(self): page, _ = _run_search_codes(_load_map_excerpt()) diff --git a/tests/test_translation_validation.py b/tests/test_translation_validation.py new file mode 100644 index 0000000..f42b492 --- /dev/null +++ b/tests/test_translation_validation.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +"""Regression tests for shared translation safety validation.""" + +from __future__ import annotations + +import unittest + +from util import translation as tr + + +class ControlCodeProtectionTests(unittest.TestCase): + def test_general_rpgmaker_controls_round_trip_exactly(self): + source = r"\SHADOW[3]試練\SHADOW[0] \I[14]\>文字\}\C[0]" + + protected, replacements = tr.protect_script_codes(source) + + self.assertNotIn(r"\SHADOW[3]", protected) + self.assertNotIn(r"\I[14]", protected) + self.assertEqual(tr.restore_script_codes(protected, replacements), source) + + def test_control_validation_rejects_changed_parameter_and_added_escape(self): + source = r"\SHADOW[3]Trial \I[14]" + translated = r"\SHADOW[08]Trial \I[14]\Coming" + + valid, reasons = tr.validate_control_codes(source, translated) + + self.assertFalse(valid) + self.assertIn(r"\SHADOW[3]", reasons[0]) + self.assertIn(r"\SHADOW[08]", reasons[0]) + self.assertIn(r"\Coming", reasons[0]) + + def test_control_validation_preserves_duplicate_counts(self): + valid, _ = tr.validate_control_codes(r"\I[14]a\I[14]", r"\I[14]a") + self.assertFalse(valid) + + def test_control_validation_rejects_reordered_scopes(self): + valid, reasons = tr.validate_control_codes( + r"\C[2]Name\C[0] \I[14]", r"\I[14] \C[2]Name\C[0]" + ) + self.assertFalse(valid) + self.assertIn("order changed", reasons[0]) + + +class TranslationContentValidationTests(unittest.TestCase): + def test_rejects_source_language_residue(self): + valid, indices, reasons = tr.validate_translation_content( + ["そのとおり"], ["Exactly(そのとおり)!!"], r"[一-龠ぁ-ゔァ-ヴー]+" + ) + self.assertFalse(valid) + self.assertEqual(indices, [0]) + self.assertIn("Source-language text remains", reasons[0]) + + def test_rejects_leaked_line_marker(self): + valid, indices, reasons = tr.validate_translation_content( + ["役人"], ["}Line1:"], r"[一-龠ぁ-ゔァ-ヴー]+" + ) + self.assertFalse(valid) + self.assertEqual(indices, [0]) + self.assertIn("Structured response marker", reasons[0]) + + +if __name__ == "__main__": + unittest.main() diff --git a/util/translation.py b/util/translation.py index 9912680..60129ba 100644 --- a/util/translation.py +++ b/util/translation.py @@ -16,6 +16,7 @@ import urllib.request from openai import APIError, APIConnectionError, RateLimitError, APIStatusError import hashlib import threading +from collections import Counter from contextlib import contextmanager from dotenv import load_dotenv from pathlib import Path @@ -177,8 +178,49 @@ PROTECTED_PATTERNS = [ r'\\BGS\[[^\]]+\]', # \BGS[background_sound_name] r'_pum\[[^\]]+\]', # _pum[name] r'\\VS\[[^\]]+\]', # \VS[name] + # General RPG Maker/plugin controls. Preserve the full spelling, parameter, + # slash count, and order rather than asking the model to reproduce them. + r'[\\]+(?:[A-Za-z_][A-Za-z0-9_]*(?:\[(?:[^\[\]]|\[[^\]]*\])*\])?|[{}.!|^><#$@,\-])', ] +_CONTROL_CODE_RE = re.compile(PROTECTED_PATTERNS[-1]) + + +def extract_control_codes(text): + """Return runtime control tokens in source order.""" + if not isinstance(text, str) or not text: + return [] + return _CONTROL_CODE_RE.findall(text) + + +def validate_control_codes(original_items, translated_items): + """Require each translated line to preserve the exact source control-token sequence.""" + originals = original_items if isinstance(original_items, list) else [original_items] + translations = translated_items if isinstance(translated_items, list) else [translated_items] + if len(originals) != len(translations): + return False, [f"line count differs ({len(originals)} source, {len(translations)} translated)"] + + errors = [] + for index, (original, translated) in enumerate(zip(originals, translations), start=1): + source_sequence = extract_control_codes(str(original)) + target_sequence = extract_control_codes(str(translated)) + if source_sequence != target_sequence: + source_codes = Counter(source_sequence) + target_codes = Counter(target_sequence) + missing = list((source_codes - target_codes).elements()) + extra = list((target_codes - source_codes).elements()) + details = [] + if missing: + details.append(f"missing {missing}") + if extra: + details.append(f"extra/altered {extra}") + if not missing and not extra: + details.append( + f"order changed from {source_sequence} to {target_sequence}" + ) + errors.append(f"Line{index}: " + "; ".join(details)) + return not errors, errors + def protect_script_codes(text): """ Replace script codes (like \\SE[タイプライター]) with unique placeholders before translation. @@ -369,6 +411,21 @@ def validate_translation_content(original_items, translated_items, langRegex): reasons.append(f"Line{i+1}: Excessive character repetition (possible model glitch) in translation") continue + # Check 6: Source-language residue must not survive in player text. + # Japanese inside protected runtime-code parameters is absent here + # and is restored only after validation. + if re.search(langRegex, trans_str): + invalid_indices.append(i) + reasons.append(f"Line{i+1}: Source-language text remains in translation") + continue + + # Check 7: Reject leaked structured-response scaffolding such as + # `}Line1:` that can otherwise become a speaker label. + if re.search(r"(?:^|[}\]])\s*Line\d+\s*:", trans_str, re.IGNORECASE): + invalid_indices.append(i) + reasons.append(f"Line{i+1}: Structured response marker leaked into translation") + continue + is_valid = len(invalid_indices) == 0 return is_valid, invalid_indices, reasons @@ -2849,6 +2906,18 @@ def translateAI(text, history, config, filename=None, pbar=None, lock=None, mism cached_result = peek_cached_translation(subbedT, config.language) else: cached_result = get_cached_translation(subbedT, config.language) + if cached_result is not None: + cached_values = cached_result if isinstance(cached_result, list) else [cached_result] + source_values = clean_tItem if isinstance(clean_tItem, list) else [clean_tItem] + controls_ok, _control_errors = validate_control_codes(source_values, cached_values) + content_ok, _invalid_indices, _content_reasons = validate_translation_content( + source_values, cached_values, config.langRegex + ) + if len(source_values) != len(cached_values) or not controls_ok or not content_ok: + # Ignore stale/corrupt cache entries. A successful live result + # below overwrites the same key. + cached_result = None + if cached_result is not None: # Estimate mode: keep original tList[index]; cached length may differ. if not config.estimateMode: @@ -3116,29 +3185,43 @@ def translateAI(text, history, config, filename=None, pbar=None, lock=None, mism if extra: pbar.write(f"Extra placeholders: {', '.join(extra)}") else: - # Check 3: Validate that translations are not empty or nearly empty - content_valid, invalid_indices, content_reasons = validate_translation_content( - clean_tItem, extracted, config.langRegex + restored_for_validation = [ + restore_script_codes(line, all_replacements.get(j, {})) + for j, line in enumerate(extracted) + ] + control_valid, control_reasons = validate_control_codes( + clean_tItem, restored_for_validation ) - - if not content_valid: + if not control_valid: is_valid = False if pbar: - pbar.write(f"Invalid translation content detected:") - for reason in content_reasons[:5]: # Show first 5 issues + pbar.write("Control-code mismatch detected:") + for reason in control_reasons[:5]: pbar.write(f" - {reason}") - if len(content_reasons) > 5: - pbar.write(f" ... and {len(content_reasons) - 5} more issues") else: - # Set translations (line count matches, placeholders valid, and content is good) - # Strip "Placeholder Text" from individual lines (AI placeholder for untranslatable input) - # Also apply the 「→" / 」→" replacements here per-line (safe now that JSON is parsed) - def _clean_extracted_line(line): - if not isinstance(line, str): - return line - line = line.replace("Placeholder Text", "").strip() - return convert_corner_brackets(line, config.convertQuotes) - final_translations = [_clean_extracted_line(line) for line in extracted] + # Check 3: Validate that translations are not empty or nearly empty + content_valid, invalid_indices, content_reasons = validate_translation_content( + clean_tItem, extracted, config.langRegex + ) + + if not content_valid: + is_valid = False + if pbar: + pbar.write(f"Invalid translation content detected:") + for reason in content_reasons[:5]: # Show first 5 issues + pbar.write(f" - {reason}") + if len(content_reasons) > 5: + pbar.write(f" ... and {len(content_reasons) - 5} more issues") + else: + # Set translations (line count matches, placeholders valid, and content is good) + # Strip "Placeholder Text" from individual lines (AI placeholder for untranslatable input) + # Also apply the 「→" / 」→" replacements here per-line (safe now that JSON is parsed) + def _clean_extracted_line(line): + if not isinstance(line, str): + return line + line = line.replace("Placeholder Text", "").strip() + return convert_corner_brackets(line, config.convertQuotes) + final_translations = [_clean_extracted_line(line) for line in extracted] else: # Single string: extract from JSON schema response extracted = extractTranslation(cleaned_text, False, pbar) @@ -3158,24 +3241,37 @@ def translateAI(text, history, config, filename=None, pbar=None, lock=None, mism if extra: pbar.write(f"Extra placeholders: {', '.join(extra)}") else: - # Validate content for single string - final_cleaned = extracted.replace("Placeholder Text", "") - content_valid, _, content_reasons = validate_translation_content( - tItem, final_cleaned, config.langRegex + restored_for_validation = restore_script_codes( + extracted, all_replacements[0] ) - - if not content_valid: + control_valid, control_reasons = validate_control_codes( + tItem, restored_for_validation + ) + if not control_valid: is_valid = False if pbar: - pbar.write(f"Invalid translation content:") - for reason in content_reasons: + pbar.write("Control-code mismatch detected:") + for reason in control_reasons: pbar.write(f" - {reason}") else: - # Accept output - all validations passed - final_cleaned = convert_corner_brackets( - final_cleaned, config.convertQuotes + # Validate content for single string + final_cleaned = extracted.replace("Placeholder Text", "") + content_valid, _, content_reasons = validate_translation_content( + tItem, final_cleaned, config.langRegex ) - final_translations = final_cleaned + + if not content_valid: + is_valid = False + if pbar: + pbar.write(f"Invalid translation content:") + for reason in content_reasons: + pbar.write(f" - {reason}") + else: + # Accept output - all validations passed + final_cleaned = convert_corner_brackets( + final_cleaned, config.convertQuotes + ) + final_translations = final_cleaned else: is_valid = False if pbar: pbar.write(f"AI Refused: {tItem}\n")