fix(rpgmaker): prevent control-code and choice-prefix corruption
This commit is contained in:
parent
2ddb47dce1
commit
d38a8a2725
4 changed files with 242 additions and 37 deletions
|
|
@ -611,6 +611,28 @@ def _apply_choice_original(cmd, index: int, raw_source: str) -> None:
|
|||
orig_list[index] = raw_source
|
||||
|
||||
|
||||
def _split_choice_condition_prefix(text: str) -> tuple[str, str]:
|
||||
"""Split a leading lowercase ``if(...)``/``en(...)`` plugin condition safely.
|
||||
|
||||
Conditions can contain nested calls such as ``if($gameSwitches.value(1))``.
|
||||
Return the input untouched when the prefix is absent or unbalanced.
|
||||
"""
|
||||
if not isinstance(text, str) or not text.startswith(("if(", "en(")):
|
||||
return "", text
|
||||
|
||||
depth = 0
|
||||
for index, char in enumerate(text[2:], start=2):
|
||||
if char == "(":
|
||||
depth += 1
|
||||
elif char == ")":
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
return text[: index + 1], text[index + 1 :]
|
||||
if depth < 0:
|
||||
break
|
||||
return "", text
|
||||
|
||||
|
||||
def _122_inner_source(cmd) -> str | None:
|
||||
"""Inner quoted value for code 122: _original or extract from parameters[4]."""
|
||||
orig = _scalar_original(cmd)
|
||||
|
|
@ -4052,13 +4074,8 @@ def searchCodes(page, pbar, jobList, filename):
|
|||
if not _text_needs_translation(currentChoice):
|
||||
continue
|
||||
|
||||
# If and En Statements
|
||||
ifVar = ""
|
||||
ifList = re.findall(r"([ei][nf]\(.+?\)\)?\)?)", jaString)
|
||||
if len(ifList) != 0:
|
||||
for var in ifList:
|
||||
jaString = jaString.replace(var, "")
|
||||
ifVar += var
|
||||
# Preserve plugin condition prefixes outside the model.
|
||||
ifVar, jaString = _split_choice_condition_prefix(jaString)
|
||||
|
||||
# Store the formatting and cleaned string
|
||||
varList.append(ifVar)
|
||||
|
|
|
|||
|
|
@ -192,6 +192,35 @@ def _has_japanese(s: str) -> bool:
|
|||
|
||||
|
||||
class TestMVMZSourceOriginal(unittest.TestCase):
|
||||
def test_choice_condition_prefix_parser_handles_nested_calls(self):
|
||||
prefix, label = mvmz._split_choice_condition_prefix(
|
||||
"if($gameSwitches.value(1) && v[31]>=4)迷宮四階"
|
||||
)
|
||||
|
||||
self.assertEqual(prefix, "if($gameSwitches.value(1) && v[31]>=4)")
|
||||
self.assertEqual(label, "迷宮四階")
|
||||
|
||||
def test_choice_condition_prefix_parser_leaves_malformed_input_untouched(self):
|
||||
source = "if(v[31]>=4迷宮四階"
|
||||
self.assertEqual(mvmz._split_choice_condition_prefix(source), ("", source))
|
||||
|
||||
def test_choice_translation_restores_condition_prefix_byte_for_byte(self):
|
||||
source = "if(v[31]>=4)迷宮四階：図書館"
|
||||
page = {
|
||||
"list": [
|
||||
{"code": 102, "indent": 0, "parameters": [[source], -1, 0, 2, 0]},
|
||||
]
|
||||
}
|
||||
|
||||
def translate(text, _history, _batch=False):
|
||||
self.assertEqual(text, ["迷宮四階：図書館"])
|
||||
return [["labyrinth floor 4: library"], [0, 0]]
|
||||
|
||||
translated, _ = _run_search_codes(page, translate_fn=translate)
|
||||
|
||||
choice = _find_commands(translated, 102)[0]["parameters"][0][0]
|
||||
self.assertEqual(choice, "if(v[31]>=4)Labyrinth floor 4: library")
|
||||
|
||||
def test_first_pass_writes_original(self):
|
||||
page, _ = _run_search_codes(_load_map_excerpt())
|
||||
|
||||
|
|
|
|||
63
tests/test_translation_validation.py
Normal file
63
tests/test_translation_validation.py
Normal file
|
|
@ -0,0 +1,63 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Regression tests for shared translation safety validation."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
|
||||
from util import translation as tr
|
||||
|
||||
|
||||
class ControlCodeProtectionTests(unittest.TestCase):
|
||||
def test_general_rpgmaker_controls_round_trip_exactly(self):
|
||||
source = r"\SHADOW[3]試練\SHADOW[0] \I[14]\>文字\}\C[0]"
|
||||
|
||||
protected, replacements = tr.protect_script_codes(source)
|
||||
|
||||
self.assertNotIn(r"\SHADOW[3]", protected)
|
||||
self.assertNotIn(r"\I[14]", protected)
|
||||
self.assertEqual(tr.restore_script_codes(protected, replacements), source)
|
||||
|
||||
def test_control_validation_rejects_changed_parameter_and_added_escape(self):
|
||||
source = r"\SHADOW[3]Trial \I[14]"
|
||||
translated = r"\SHADOW[08]Trial \I[14]\Coming"
|
||||
|
||||
valid, reasons = tr.validate_control_codes(source, translated)
|
||||
|
||||
self.assertFalse(valid)
|
||||
self.assertIn(r"\SHADOW[3]", reasons[0])
|
||||
self.assertIn(r"\SHADOW[08]", reasons[0])
|
||||
self.assertIn(r"\Coming", reasons[0])
|
||||
|
||||
def test_control_validation_preserves_duplicate_counts(self):
|
||||
valid, _ = tr.validate_control_codes(r"\I[14]a\I[14]", r"\I[14]a")
|
||||
self.assertFalse(valid)
|
||||
|
||||
def test_control_validation_rejects_reordered_scopes(self):
|
||||
valid, reasons = tr.validate_control_codes(
|
||||
r"\C[2]Name\C[0] \I[14]", r"\I[14] \C[2]Name\C[0]"
|
||||
)
|
||||
self.assertFalse(valid)
|
||||
self.assertIn("order changed", reasons[0])
|
||||
|
||||
|
||||
class TranslationContentValidationTests(unittest.TestCase):
|
||||
def test_rejects_source_language_residue(self):
|
||||
valid, indices, reasons = tr.validate_translation_content(
|
||||
["そのとおり"], ["Exactly(そのとおり)!!"], r"[一-龠ぁ-ゔァ-ヴー]+"
|
||||
)
|
||||
self.assertFalse(valid)
|
||||
self.assertEqual(indices, [0])
|
||||
self.assertIn("Source-language text remains", reasons[0])
|
||||
|
||||
def test_rejects_leaked_line_marker(self):
|
||||
valid, indices, reasons = tr.validate_translation_content(
|
||||
["役人"], ["}Line1:"], r"[一-龠ぁ-ゔァ-ヴー]+"
|
||||
)
|
||||
self.assertFalse(valid)
|
||||
self.assertEqual(indices, [0])
|
||||
self.assertIn("Structured response marker", reasons[0])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
@ -16,6 +16,7 @@ import urllib.request
|
|||
from openai import APIError, APIConnectionError, RateLimitError, APIStatusError
|
||||
import hashlib
|
||||
import threading
|
||||
from collections import Counter
|
||||
from contextlib import contextmanager
|
||||
from dotenv import load_dotenv
|
||||
from pathlib import Path
|
||||
|
|
@ -177,8 +178,49 @@ PROTECTED_PATTERNS = [
|
|||
r'\\BGS\[[^\]]+\]', # \BGS[background_sound_name]
|
||||
r'_pum\[[^\]]+\]', # _pum[name]
|
||||
r'\\VS\[[^\]]+\]', # \VS[name]
|
||||
# General RPG Maker/plugin controls. Preserve the full spelling, parameter,
|
||||
# slash count, and order rather than asking the model to reproduce them.
|
||||
r'[\\]+(?:[A-Za-z_][A-Za-z0-9_]*(?:\[(?:[^\[\]]|\[[^\]]*\])*\])?|[{}.!|^><#$@,\-])',
|
||||
]
|
||||
|
||||
_CONTROL_CODE_RE = re.compile(PROTECTED_PATTERNS[-1])
|
||||
|
||||
|
||||
def extract_control_codes(text):
|
||||
"""Return runtime control tokens in source order."""
|
||||
if not isinstance(text, str) or not text:
|
||||
return []
|
||||
return _CONTROL_CODE_RE.findall(text)
|
||||
|
||||
|
||||
def validate_control_codes(original_items, translated_items):
|
||||
"""Require each translated line to preserve the exact source control-token sequence."""
|
||||
originals = original_items if isinstance(original_items, list) else [original_items]
|
||||
translations = translated_items if isinstance(translated_items, list) else [translated_items]
|
||||
if len(originals) != len(translations):
|
||||
return False, [f"line count differs ({len(originals)} source, {len(translations)} translated)"]
|
||||
|
||||
errors = []
|
||||
for index, (original, translated) in enumerate(zip(originals, translations), start=1):
|
||||
source_sequence = extract_control_codes(str(original))
|
||||
target_sequence = extract_control_codes(str(translated))
|
||||
if source_sequence != target_sequence:
|
||||
source_codes = Counter(source_sequence)
|
||||
target_codes = Counter(target_sequence)
|
||||
missing = list((source_codes - target_codes).elements())
|
||||
extra = list((target_codes - source_codes).elements())
|
||||
details = []
|
||||
if missing:
|
||||
details.append(f"missing {missing}")
|
||||
if extra:
|
||||
details.append(f"extra/altered {extra}")
|
||||
if not missing and not extra:
|
||||
details.append(
|
||||
f"order changed from {source_sequence} to {target_sequence}"
|
||||
)
|
||||
errors.append(f"Line{index}: " + "; ".join(details))
|
||||
return not errors, errors
|
||||
|
||||
def protect_script_codes(text):
|
||||
"""
|
||||
Replace script codes (like \\SE[タイプライター]) with unique placeholders before translation.
|
||||
|
|
@ -369,6 +411,21 @@ def validate_translation_content(original_items, translated_items, langRegex):
|
|||
reasons.append(f"Line{i+1}: Excessive character repetition (possible model glitch) in translation")
|
||||
continue
|
||||
|
||||
# Check 6: Source-language residue must not survive in player text.
|
||||
# Japanese inside protected runtime-code parameters is absent here
|
||||
# and is restored only after validation.
|
||||
if re.search(langRegex, trans_str):
|
||||
invalid_indices.append(i)
|
||||
reasons.append(f"Line{i+1}: Source-language text remains in translation")
|
||||
continue
|
||||
|
||||
# Check 7: Reject leaked structured-response scaffolding such as
|
||||
# `}Line1:` that can otherwise become a speaker label.
|
||||
if re.search(r"(?:^|[}\]])\s*Line\d+\s*:", trans_str, re.IGNORECASE):
|
||||
invalid_indices.append(i)
|
||||
reasons.append(f"Line{i+1}: Structured response marker leaked into translation")
|
||||
continue
|
||||
|
||||
is_valid = len(invalid_indices) == 0
|
||||
return is_valid, invalid_indices, reasons
|
||||
|
||||
|
|
@ -2849,6 +2906,18 @@ def translateAI(text, history, config, filename=None, pbar=None, lock=None, mism
|
|||
cached_result = peek_cached_translation(subbedT, config.language)
|
||||
else:
|
||||
cached_result = get_cached_translation(subbedT, config.language)
|
||||
if cached_result is not None:
|
||||
cached_values = cached_result if isinstance(cached_result, list) else [cached_result]
|
||||
source_values = clean_tItem if isinstance(clean_tItem, list) else [clean_tItem]
|
||||
controls_ok, _control_errors = validate_control_codes(source_values, cached_values)
|
||||
content_ok, _invalid_indices, _content_reasons = validate_translation_content(
|
||||
source_values, cached_values, config.langRegex
|
||||
)
|
||||
if len(source_values) != len(cached_values) or not controls_ok or not content_ok:
|
||||
# Ignore stale/corrupt cache entries. A successful live result
|
||||
# below overwrites the same key.
|
||||
cached_result = None
|
||||
|
||||
if cached_result is not None:
|
||||
# Estimate mode: keep original tList[index]; cached length may differ.
|
||||
if not config.estimateMode:
|
||||
|
|
@ -3116,29 +3185,43 @@ def translateAI(text, history, config, filename=None, pbar=None, lock=None, mism
|
|||
if extra:
|
||||
pbar.write(f"Extra placeholders: {', '.join(extra)}")
|
||||
else:
|
||||
# Check 3: Validate that translations are not empty or nearly empty
|
||||
content_valid, invalid_indices, content_reasons = validate_translation_content(
|
||||
clean_tItem, extracted, config.langRegex
|
||||
restored_for_validation = [
|
||||
restore_script_codes(line, all_replacements.get(j, {}))
|
||||
for j, line in enumerate(extracted)
|
||||
]
|
||||
control_valid, control_reasons = validate_control_codes(
|
||||
clean_tItem, restored_for_validation
|
||||
)
|
||||
|
||||
if not content_valid:
|
||||
if not control_valid:
|
||||
is_valid = False
|
||||
if pbar:
|
||||
pbar.write(f"Invalid translation content detected:")
|
||||
for reason in content_reasons[:5]: # Show first 5 issues
|
||||
pbar.write("Control-code mismatch detected:")
|
||||
for reason in control_reasons[:5]:
|
||||
pbar.write(f" - {reason}")
|
||||
if len(content_reasons) > 5:
|
||||
pbar.write(f" ... and {len(content_reasons) - 5} more issues")
|
||||
else:
|
||||
# Set translations (line count matches, placeholders valid, and content is good)
|
||||
# Strip "Placeholder Text" from individual lines (AI placeholder for untranslatable input)
|
||||
# Also apply the 「→" / 」→" replacements here per-line (safe now that JSON is parsed)
|
||||
def _clean_extracted_line(line):
|
||||
if not isinstance(line, str):
|
||||
return line
|
||||
line = line.replace("Placeholder Text", "").strip()
|
||||
return convert_corner_brackets(line, config.convertQuotes)
|
||||
final_translations = [_clean_extracted_line(line) for line in extracted]
|
||||
# Check 3: Validate that translations are not empty or nearly empty
|
||||
content_valid, invalid_indices, content_reasons = validate_translation_content(
|
||||
clean_tItem, extracted, config.langRegex
|
||||
)
|
||||
|
||||
if not content_valid:
|
||||
is_valid = False
|
||||
if pbar:
|
||||
pbar.write(f"Invalid translation content detected:")
|
||||
for reason in content_reasons[:5]: # Show first 5 issues
|
||||
pbar.write(f" - {reason}")
|
||||
if len(content_reasons) > 5:
|
||||
pbar.write(f" ... and {len(content_reasons) - 5} more issues")
|
||||
else:
|
||||
# Set translations (line count matches, placeholders valid, and content is good)
|
||||
# Strip "Placeholder Text" from individual lines (AI placeholder for untranslatable input)
|
||||
# Also apply the 「→" / 」→" replacements here per-line (safe now that JSON is parsed)
|
||||
def _clean_extracted_line(line):
|
||||
if not isinstance(line, str):
|
||||
return line
|
||||
line = line.replace("Placeholder Text", "").strip()
|
||||
return convert_corner_brackets(line, config.convertQuotes)
|
||||
final_translations = [_clean_extracted_line(line) for line in extracted]
|
||||
else:
|
||||
# Single string: extract from JSON schema response
|
||||
extracted = extractTranslation(cleaned_text, False, pbar)
|
||||
|
|
@ -3158,24 +3241,37 @@ def translateAI(text, history, config, filename=None, pbar=None, lock=None, mism
|
|||
if extra:
|
||||
pbar.write(f"Extra placeholders: {', '.join(extra)}")
|
||||
else:
|
||||
# Validate content for single string
|
||||
final_cleaned = extracted.replace("Placeholder Text", "")
|
||||
content_valid, _, content_reasons = validate_translation_content(
|
||||
tItem, final_cleaned, config.langRegex
|
||||
restored_for_validation = restore_script_codes(
|
||||
extracted, all_replacements[0]
|
||||
)
|
||||
|
||||
if not content_valid:
|
||||
control_valid, control_reasons = validate_control_codes(
|
||||
tItem, restored_for_validation
|
||||
)
|
||||
if not control_valid:
|
||||
is_valid = False
|
||||
if pbar:
|
||||
pbar.write(f"Invalid translation content:")
|
||||
for reason in content_reasons:
|
||||
pbar.write("Control-code mismatch detected:")
|
||||
for reason in control_reasons:
|
||||
pbar.write(f" - {reason}")
|
||||
else:
|
||||
# Accept output - all validations passed
|
||||
final_cleaned = convert_corner_brackets(
|
||||
final_cleaned, config.convertQuotes
|
||||
# Validate content for single string
|
||||
final_cleaned = extracted.replace("Placeholder Text", "")
|
||||
content_valid, _, content_reasons = validate_translation_content(
|
||||
tItem, final_cleaned, config.langRegex
|
||||
)
|
||||
final_translations = final_cleaned
|
||||
|
||||
if not content_valid:
|
||||
is_valid = False
|
||||
if pbar:
|
||||
pbar.write(f"Invalid translation content:")
|
||||
for reason in content_reasons:
|
||||
pbar.write(f" - {reason}")
|
||||
else:
|
||||
# Accept output - all validations passed
|
||||
final_cleaned = convert_corner_brackets(
|
||||
final_cleaned, config.convertQuotes
|
||||
)
|
||||
final_translations = final_cleaned
|
||||
else:
|
||||
is_valid = False
|
||||
if pbar: pbar.write(f"AI Refused: {tItem}\n")
|
||||
|
|
|
|||
Loading…
Reference in a new issue