From de93d8d8e1c02c7482a91a447430b0047a5b9ad4 Mon Sep 17 00:00:00 2001 From: DazedAnon Date: Sun, 14 Jun 2026 10:41:40 -0500 Subject: [PATCH] Preserve Original for 401, 405, 408, 101, 102 directly in text. --- modules/rpgmakermvmz.py | 404 +++++++++++++++++++++-------- tests/__init__.py | 0 tests/fixtures/README.md | 51 ++++ tests/run_tests.sh | 30 +++ tests/test_mvmz_source_original.py | 363 ++++++++++++++++++++++++++ 5 files changed, 741 insertions(+), 107 deletions(-) create mode 100644 tests/__init__.py create mode 100644 tests/fixtures/README.md create mode 100644 tests/run_tests.sh create mode 100644 tests/test_mvmz_source_original.py diff --git a/modules/rpgmakermvmz.py b/modules/rpgmakermvmz.py index 472b65c..be84dfa 100644 --- a/modules/rpgmakermvmz.py +++ b/modules/rpgmakermvmz.py @@ -435,6 +435,150 @@ def saveProgress(data, filename): traceback.print_exc() +def _scalar_original(cmd) -> str | None: + """Return scalar _original on an event command, or None if absent/empty.""" + orig = cmd.get("_original") + if orig is not None and not isinstance(orig, list) and str(orig).strip(): + return str(orig) + return None + + +def _param_source(cmd, index: int) -> str: + """Prefer scalar _original; else parameters[index] (401/405 dialogue lines).""" + orig = _scalar_original(cmd) + if orig is not None: + return orig + params = cmd.get("parameters") or [] + if index < len(params) and params[index] is not None: + return str(params[index]) + return "" + + +def _group_source(codeList, start: int, end: int) -> str: + """Join source text for a merged 401/405 group (indices start..end inclusive).""" + if start < len(codeList): + orig = _scalar_original(codeList[start]) + if orig is not None: + return orig + parts = [] + for idx in range(start, end + 1): + if idx >= len(codeList): + break + cmd = codeList[idx] + if not cmd or cmd.get("code") not in (401, 405, -1): + continue + params = cmd.get("parameters") or [] + if not params: + continue + src = _param_source(cmd, 0) + if src.strip(): + parts.append(src) + return "\n".join(parts) + + +def _group_raw_source(codeList, group_start: int, source_parts: list[str]) -> str: + """Batch source for merged 401/405; anchor _original wins on re-run.""" + if group_start < len(codeList): + orig = _scalar_original(codeList[group_start]) + if orig is not None: + return orig + return "\n".join(source_parts) + + +def _apply_original(cmd, raw_source: str) -> None: + """Set scalar _original only when not already present (re-run safe).""" + if not raw_source or not str(raw_source).strip(): + return + if _scalar_original(cmd) is not None: + return + cmd["_original"] = raw_source + + +def _choice_source(cmd, index: int) -> str: + """Prefer _original[index] for code 102 choices; else parameters[0][index].""" + orig_list = cmd.get("_original") + if isinstance(orig_list, list) and index < len(orig_list): + slot = orig_list[index] + if slot is not None and str(slot).strip(): + return str(slot) + params = cmd.get("parameters") or [[]] + choices = params[0] if params else [] + if isinstance(choices, list) and index < len(choices) and choices[index] is not None: + return str(choices[index]) + return "" + + +def _apply_choice_original(cmd, index: int, raw_source: str) -> None: + """Set _original[index] for code 102 only when that slot is empty.""" + if not raw_source or not str(raw_source).strip(): + return + params = cmd.get("parameters") or [[]] + choices = params[0] if params else [] + n = len(choices) if isinstance(choices, list) else 0 + orig_list = cmd.get("_original") + if not isinstance(orig_list, list): + orig_list = [None] * n + cmd["_original"] = orig_list + while len(orig_list) < n: + orig_list.append(None) + if index < len(orig_list): + existing = orig_list[index] + if existing is not None and str(existing).strip(): + return + orig_list[index] = raw_source + + +def _122_inner_source(cmd) -> str | None: + """Inner quoted value for code 122: _original or extract from parameters[4].""" + orig = _scalar_original(cmd) + if orig is not None: + return orig + params = cmd.get("parameters") or [] + if len(params) <= 4: + return None + jaString = params[4] + if not isinstance(jaString, str): + return None + if len(re.findall(r"([\'\"\`])", jaString)) >= 2: + matchedText = re.search(r"[\'\"\`](.*)[\'\"\`]", jaString) + if matchedText and matchedText.group(1).strip(): + return matchedText.group(1) + return None + + +def _101_name_source(cmd, is_var: bool) -> str: + """Speaker name field for code 101: _original or parameters[4]/[0].""" + orig = _scalar_original(cmd) + if orig is not None: + return orig + params = cmd.get("parameters") or [] + if is_var and len(params) > 0 and params[0] is not None: + return str(params[0]) + if not is_var and len(params) > 4 and params[4] is not None: + return str(params[4]) + return "" + + +_COLOR_SPEAKER_RE = re.compile( + r"^[\\]+[cC]\[\d+\]【?(.+?)】?[\\]+[cC]\[\d+\](?:[\\]+[A-Za-z]+(?:\[[^\]]*\])?)*[\\]*$" +) + + +def _replace_speaker_in_param(param_str: str, source_name: str, translated_name: str) -> str: + """Replace a speaker name inside a 401/101 parameter while keeping colour/bracket wrappers.""" + if not param_str or not translated_name: + return param_str + m = _COLOR_SPEAKER_RE.match(param_str) + if m: + return param_str.replace(m.group(1), translated_name, 1) + bracket_disp = re.findall(r"【(.+?)】", param_str) + if bracket_disp: + return param_str.replace(bracket_disp[0], translated_name, 1) + if source_name and source_name in param_str: + return param_str.replace(source_name, translated_name, 1) + return param_str + + def checkSave(data, filename, tokens): """Save progress only if the given tokens reflect an actual translation. tokens should be a [input_tokens, output_tokens] pair returned by a search/translate call. @@ -1799,6 +1943,7 @@ def searchCodes(page, pbar, jobList, filename): # Declare Varss currentGroup = [] + sourceGroup = [] nametag = "" ## Event Code: 401 Show Text @@ -1806,13 +1951,15 @@ def searchCodes(page, pbar, jobList, filename): # Save Code and starting index (j) code = codeList[i]["code"] j = i + groupStart = j endtag = "" instantLineFlag = False # Grab String if len(codeList[i]["parameters"]) > 0: jaString = codeList[i]["parameters"][0] - oldjaString = jaString + oldjaString = _param_source(codeList[i], 0) + speakerWork = oldjaString else: codeList[i]["code"] = -1 i += 1 @@ -1846,7 +1993,7 @@ def searchCodes(page, pbar, jobList, filename): nametag += ffMatch.group(0) # m and z Codes - match = re.search(r"(.*?)[\\]+m\[\d+?\][\\]+z\[\d+?\]", jaString) + match = re.search(r"(.*?)[\\]+m\[\d+?\][\\]+z\[\d+?\]", speakerWork) if match: speakerList.append(match.group(1)) if "\\c" in speakerList[0]: @@ -1858,7 +2005,7 @@ def searchCodes(page, pbar, jobList, filename): # Brackets (support multiple names like 【A】【B】) if len(speakerList) == 0: # Check for bracket at start with dialogue following (【name】dialogue...) - inlineBracketMatch = re.match(r"^\s*【([^】]+)】(.+)", jaString, re.DOTALL) + inlineBracketMatch = re.match(r"^\s*【([^】]+)】(.+)", speakerWork, re.DOTALL) if inlineBracketMatch: # Inline bracket with dialogue on same line @@ -1866,14 +2013,14 @@ def searchCodes(page, pbar, jobList, filename): else: # Only consider bracketed names when the line starts with '【' and # ends with either '】' or trailing variable/control codes like \n[2], \FF[\w[3]], etc. - startsWithBracket = re.match(r"^\s*【", jaString) is not None + startsWithBracket = re.match(r"^\s*【", speakerWork) is not None endsWithBracket = re.search( r"(】\s*|(?:[\\]+[A-Za-z]+(?:\[(?:[^\[\]]|\[[^\]]*\])*\])+\s*)$)", - jaString, + speakerWork, ) is not None if startsWithBracket and endsWithBracket: - candidates = re.findall(r"【(.*?)】", jaString) + candidates = re.findall(r"【(.*?)】", speakerWork) if candidates: candidates = [c.strip() for c in candidates] if candidates: @@ -1883,19 +2030,19 @@ def searchCodes(page, pbar, jobList, filename): if len(speakerList) == 0: speakerList = re.findall( r"^[\\]+[cC]\[\d+\]【?(.+?)】?[\\]+[cC]\[\d+\](?:[\\]+[A-Za-z]+(?:\[[^\]]*\])?)*[\\]*$", - jaString, + speakerWork, ) # Colons if len(speakerList) == 0: speakerList = re.findall( r"(.+)：$", - jaString, + speakerWork, ) # [Speaker] standalone line format (written back by inline re-export) if len(speakerList) == 0: - inlineFmtMatch = re.match(r"^\[([^\[\]\n]+)\]\s*$", jaString) + inlineFmtMatch = re.match(r"^\[([^\[\]\n]+)\]\s*$", speakerWork) if inlineFmtMatch: speakerList = [inlineFmtMatch.group(1).strip()] @@ -1903,7 +2050,7 @@ def searchCodes(page, pbar, jobList, filename): if len(speakerList) == 0 and INLINE401SPEAKERS: inlineSpeakerMatch = re.match( r'^(?:\[([^\]]{1,30})\]\s*|([^\s「」。、！？…\\\n“”"(:\[\]]{1,20})(?:[::：]?\s*)(?=[「“"(]))(.*)', - jaString, re.DOTALL + speakerWork, re.DOTALL ) if inlineSpeakerMatch: speakerList = [(inlineSpeakerMatch.group(1) or inlineSpeakerMatch.group(2)).strip()] @@ -1914,7 +2061,7 @@ def searchCodes(page, pbar, jobList, filename): if len(speakerList) == 0 and FIRSTLINESPEAKERS is True: # Test Speaker if ( - len(jaString) < 40 + len(speakerWork) < 40 and "code" in codeList[i + 1] and codeList[i + 1]["code"] in [401, 405, -1] and len(codeList[i + 1]["parameters"]) > 0 @@ -1941,12 +2088,12 @@ def searchCodes(page, pbar, jobList, filename): "*", "[", ]: - speakerList = re.findall(r".+", jaString) + speakerList = re.findall(r".+", speakerWork) # Replace Speaker if len(speakerList) != 0: # Check if speaker+dialogue are on same line - sameLineMatch = re.match(r"^\s*【([^】]+)】(.+)", jaString, re.DOTALL) + sameLineMatch = re.match(r"^\s*【([^】]+)】(.+)", speakerWork, re.DOTALL) if inlineSpeakerMatch and len(speakerList) == 1: # Strip speaker prefix, keep everything after as dialogue response = getSpeaker(speakerList[0]) @@ -1995,8 +2142,13 @@ def searchCodes(page, pbar, jobList, filename): # Set Data if not setData and len(speakerList) > 1: codeList[i]["parameters"][0] = nametag + jaStringUpdated + _apply_original(codeList[i], oldjaString) elif not setData and len(speakerList) == 1: - codeList[i]["parameters"][0] = nametag + jaString.replace(speakerList[0], speaker) + paramStr = codeList[i]["parameters"][0] + codeList[i]["parameters"][0] = nametag + _replace_speaker_in_param( + paramStr, speakerList[0], speaker + ) + _apply_original(codeList[i], oldjaString) nametag = "" # Iterate to next string @@ -2006,14 +2158,12 @@ def searchCodes(page, pbar, jobList, filename): i += 1 j = i jaString = codeList[i]["parameters"][0] + groupStart = i - # Validate Japanese Text - if not re.search(LANGREGEX, jaString) and IGNORETLTEXT: - i += 1 - continue - - # Using this to keep track of 401's in a row. + # Using this to keep track of 401's in a row (display text for Pass 2 formatting). currentGroup.append(jaString) + anchor_has_orig = _scalar_original(codeList[groupStart]) is not None + sourceGroup.append(_param_source(codeList[i], 0)) # Join Up 401's into single string if len(codeList) > i + 1: @@ -2027,6 +2177,8 @@ def searchCodes(page, pbar, jobList, filename): jaString = codeList[i]["parameters"][0] if jaString.strip(): currentGroup.append(jaString) + if not anchor_has_orig: + sourceGroup.append(_param_source(codeList[i], 0)) # Make sure not the end of the list. if len(codeList) <= i + 1: @@ -2034,8 +2186,16 @@ def searchCodes(page, pbar, jobList, filename): # Format String if len(currentGroup) > 0: - finalJAString = "\n".join(currentGroup) - oldjaString = finalJAString + rawSource = _group_raw_source(codeList, groupStart, sourceGroup) + if not rawSource.strip(): + i += 1 + continue + if not re.search(LANGREGEX, rawSource) and IGNORETLTEXT: + i += 1 + continue + + finalJAString = rawSource + oldjaString = rawSource # Set Back if not setData: @@ -2137,6 +2297,7 @@ def searchCodes(page, pbar, jobList, filename): match = [] nametag = "" currentGroup = [] + sourceGroup = [] syncIndex = i + 1 # Keep textHistory list at length maxHistory @@ -2148,6 +2309,7 @@ def searchCodes(page, pbar, jobList, filename): else: # Grab Translated String if len(list401) > 0: + rawSource = _group_raw_source(codeList, groupStart, sourceGroup) translatedText = list401[0] # Remove speaker prefix if present @@ -2235,10 +2397,13 @@ def searchCodes(page, pbar, jobList, filename): codeList[j]["code"] = code syncIndex = i + 1 + _apply_original(codeList[j], rawSource) + # Reset speaker = "" match = [] currentGroup = [] + sourceGroup = [] list401.pop(0) ## Event Code: 122 [Set Variables] @@ -2277,48 +2442,49 @@ def searchCodes(page, pbar, jobList, filename): # continue # Set String - matchedText = None - if len(re.findall(r"([\'\"\`])", jaString)) >= 2: - matchedText = re.search(r"[\'\"\`](.*)[\'\"\`]", jaString) - if matchedText and matchedText.group(1).strip(): - # Skip if IGNORETLTEXT is enabled and no Japanese text - if IGNORETLTEXT and not re.search(LANGREGEX, matchedText.group(1)): - i += 1 - continue + innerSource = _122_inner_source(codeList[i]) + if innerSource is not None and innerSource.strip(): + # Skip if IGNORETLTEXT is enabled and no Japanese text + if IGNORETLTEXT and not re.search(LANGREGEX, innerSource): + i += 1 + continue - # Remove Textwrap - finalJAString = matchedText.group(1).replace("\\n", " ") + # Remove Textwrap + finalJAString = innerSource.replace("\\n", " ") - # Pass 1 - if setData: - if finalJAString != "": - list122.append(finalJAString) + # Pass 1 + if setData: + if finalJAString != "": + list122.append(finalJAString) - # Pass 2 - else: - if len(list122) > 0: - # Grab and Replace - translatedText = list122[0] - translatedText = jaString.replace(jaString, translatedText) + # Pass 2 + else: + if len(list122) > 0: + rawInner = innerSource + hadSemicolon = ';' in jaString + # Grab and Replace + translatedText = list122[0] + translatedText = jaString.replace(jaString, translatedText) - # Remove characters that may break scripts - charList = ['"', "\\n"] - for char in charList: - translatedText = translatedText.replace(char, "") + # Remove characters that may break scripts + charList = ['"', "\\n"] + for char in charList: + translatedText = translatedText.replace(char, "") - # Force 4 Escapes - translatedText = re.sub(r'(? 0: - if isVar == False: - codeList[i]["parameters"][4] = codeList[i]["parameters"][4].replace(jaString, speaker) - i += 1 - continue - else: - codeList[i]["parameters"][0] = codeList[i]["parameters"][0].replace(jaString, speaker) - isVar = False - i += 1 - continue + paramIdx = 0 if isVar else 4 + paramStr = codeList[i]["parameters"][paramIdx] + codeList[i]["parameters"][paramIdx] = _replace_speaker_in_param( + paramStr, sourceName, speaker + ) + _apply_original(codeList[i], rawName) + isVar = False + i += 1 + continue else: speaker = "" @@ -3155,52 +3322,72 @@ def searchCodes(page, pbar, jobList, filename): i += 1 continue - jaString = codeList[i]["parameters"][0] - match = re.search(r"(.+)", jaString) - if match: - # Skip if IGNORETLTEXT is enabled and no Japanese text - if IGNORETLTEXT and not re.search(LANGREGEX, jaString): + if not codeList[i].get("parameters"): + i += 1 + continue + + groupStart408 = i + j = i + source408Parts = [] + rawSource = _param_source(codeList[i], 0) + ojaString = rawSource + anchor408HasOrig = _scalar_original(codeList[groupStart408]) is not None + source408Parts.append(rawSource) + + if not rawSource.strip(): + i += 1 + continue + + # Skip if IGNORETLTEXT is enabled and no Japanese text + if IGNORETLTEXT and not re.search(LANGREGEX, rawSource): + i += 1 + continue + + # Join Up 408's into single string + if len(codeList) > i + 1 and JOIN408 is True: + while codeList[i + 1]["code"] in [408] and len(codeList[i]["parameters"]) > 0 and len(codeList[i + 1]["parameters"]) > 0 and not re.match(r"^(\s*[\\]+[aAbBdDeEfFgGhHjJlLmMoOpPqQrRsStTuUwWxXyYzZ]+\[[\w\d\[\]\\]+\])", codeList[i+1]["parameters"][0]): + if not setData: + codeList[i]["parameters"] = [] + codeList[i]["code"] = -1 i += 1 - continue + j = i - # Remove Textwrap - jaString = codeList[i]["parameters"][0] - ojaString = jaString - jaString = jaString.replace("\n", " ") + lineSource = _param_source(codeList[i], 0) + if lineSource.strip() and not anchor408HasOrig: + source408Parts.append(lineSource) - # Join Up 408's into single string - if len(codeList) > i + 1 and JOIN408 is True: - while codeList[i + 1]["code"] in [408] and len(codeList[i]["parameters"]) > 0 and len(codeList[i + 1]["parameters"]) > 0 and not re.match(r"^(\s*[\\]+[aAbBdDeEfFgGhHjJlLmMoOpPqQrRsStTuUwWxXyYzZ]+\[[\w\d\[\]\\]+\])", codeList[i+1]["parameters"][0]): - if not setData: - codeList[i]["parameters"] = [] - codeList[i]["code"] = -1 - i += 1 - j = i + if len(codeList) <= i + 1: + break - jaString = codeList[i]["parameters"][0] - if jaString.strip(): - currentGroup.append(jaString) + rawSource = _group_raw_source(codeList, groupStart408, source408Parts) + ojaString = rawSource + jaString = rawSource.replace("\n", " ") - # Make sure not the end of the list. - if len(codeList) <= i + 1: - break + # Pass 1 + if setData: + list408.append(jaString) - # Pass 1 - if setData: - # Remove Textwrap - jaString = jaString.replace("\n", " ") - list408.append(jaString) - - # Pass 2 - else: + # Pass 2 + else: + if len(list408) > 0: translatedText = list408[0] list408.pop(0) - # Textwrap - # translatedText = dazedwrap.wrapText(translatedText, width=WIDTH) + merged408 = len(source408Parts) > 1 + if merged408: + codeList[i]["parameters"] = [translatedText] + else: + param0 = codeList[i]["parameters"][0] + if ojaString in param0: + codeList[i]["parameters"][0] = param0.replace(ojaString, translatedText) + else: + flatSource = ojaString.replace("\n", " ") + if flatSource in param0: + codeList[i]["parameters"][0] = param0.replace(flatSource, translatedText) + else: + codeList[i]["parameters"][0] = translatedText - # Set Data - codeList[i]["parameters"][0] = codeList[i]["parameters"][0].replace(ojaString, translatedText) + _apply_original(codeList[i], rawSource) ## Event Code: 108 (Script) if "code" in codeList[i] and (codeList[i]["code"] == 108) and CODE108 is True: @@ -3484,18 +3671,19 @@ def searchCodes(page, pbar, jobList, filename): choiceList = [] varList = [] choiceIndexMap = [] # Track which original indices we're processing + choiceSourceList = [] # Process each string in the parameters list for choice in range(len(codeList[i]["parameters"][0])): - jaString = codeList[i]["parameters"][0][choice] - jaString = jaString.replace(" 。", ".") + rawSource = _choice_source(codeList[i], choice) + jaString = rawSource.replace(" 。", ".") # Avoid Empty Strings if not jaString.strip(): continue # Skip if IGNORETLTEXT is enabled and no Japanese text - if IGNORETLTEXT and not re.search(LANGREGEX, jaString): + if IGNORETLTEXT and not re.search(LANGREGEX, rawSource): continue # If and En Statements @@ -3510,6 +3698,7 @@ def searchCodes(page, pbar, jobList, filename): varList.append(ifVar) choiceList.append(jaString) choiceIndexMap.append(choice) + choiceSourceList.append(rawSource) # Translate the list if len(choiceList) > 0: @@ -3539,6 +3728,7 @@ def searchCodes(page, pbar, jobList, filename): # Set the translation back to the original position codeList[i]["parameters"][0][originalIndex] = translatedText + _apply_choice_original(codeList[i], originalIndex, choiceSourceList[idx]) else: if filename not in MISMATCH: MISMATCH.append(filename) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/fixtures/README.md b/tests/fixtures/README.md new file mode 100644 index 0000000..ff4a557 --- /dev/null +++ b/tests/fixtures/README.md @@ -0,0 +1,51 @@ +# Map `_original` fixture + +## Files + +- **`Map_original_fixture.json`** — Minimal RPG Maker MV map (17×13) with one event exercising every `_original` code path: 401, 405, 102, 101, and 122. +- **`Map_original_fixture_manifest.json`** — Expected `_original` values after translation (used by tests). + +## Event layout (event id 1, page 0) + +| Order | Code | Purpose | +|------:|------|---------| +| 1–2 | 401 | Color speaker line `\C[2]テスト子\C[0]` + following dialogue | +| 3–4 | 401 | Merged multi-line dialogue (two 401s → one batch) | +| 5–6 | 101, 405 | Empty face setup + scrolling text line | +| 7 | 102 | Three choices (middle choice has `if(...)` prefix) | +| 8–9 | 108, 408 | Choice-help marker (`選択肢ヘルプ`) + help comment line | +| 10–11 | 101, 401 | Name box with `\C[2]アリス\C[0]` + dialogue | +| 10–11 | 122 | Variable string `` `変数の中身` `` and `` `セミコロン`; `` | + +## Run the test + +Using the project venv (recommended): + +```bash +./tests/run_tests.sh +``` + +Or explicitly: + +```bash +./tests/run_tests.sh tests.test_mvmz_source_original -v +./tests/run_tests.sh tests.test_mvmz_source_original.TestFixtureMapOriginal -v +``` + +Manual equivalent (must run from project root, with venv activated): + +```bash +cd /path/to/DazedMTLTool +source .venv/bin/activate # or: source venv/bin/activate +python -m unittest tests.test_mvmz_source_original -v +``` + +Do **not** use `pytest` unless you install it yourself — this project uses the stdlib `unittest` runner. + +If you see `ModuleNotFoundError: No module named 'colorama'` (or similar), activate the venv or use `./tests/run_tests.sh` instead of system `python3`. + +Tests mock `translateAI` / `getSpeaker` — no API key required. Enable `CODE122` for the duration of the run. + +## Manual check + +Copy the fixture into `files/` and run the tool with batch/consume or live translate, then diff against the manifest’s `expected_original` fields in `translated/Map_original_fixture.json`. diff --git a/tests/run_tests.sh b/tests/run_tests.sh new file mode 100644 index 0000000..d141c79 --- /dev/null +++ b/tests/run_tests.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# Run DazedMTLTool unit tests using the project venv (cwd = project root). + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +cd "$ROOT" + +PYTHON="" +if [[ -x "$ROOT/.venv/bin/python" ]]; then + PYTHON="$ROOT/.venv/bin/python" +elif [[ -x "$ROOT/venv/bin/python" ]]; then + PYTHON="$ROOT/venv/bin/python" +else + echo "ERROR: No virtual environment found (.venv or venv)." >&2 + echo "Create one first, e.g.: python3 -m venv .venv && .venv/bin/pip install -r requirements.txt" >&2 + exit 1 +fi + +if ! "$PYTHON" -c "import colorama, dotenv, tqdm" >/dev/null 2>&1; then + echo "Installing test dependencies from requirements.txt..." + "$PYTHON" -m pip install -r requirements.txt +fi + +if [[ "$#" -eq 0 ]]; then + set -- tests.test_mvmz_source_original -v +fi + +exec "$PYTHON" -m unittest "$@" diff --git a/tests/test_mvmz_source_original.py b/tests/test_mvmz_source_original.py new file mode 100644 index 0000000..5cd7309 --- /dev/null +++ b/tests/test_mvmz_source_original.py @@ -0,0 +1,363 @@ +#!/usr/bin/env python3 +"""Integration tests for _original source preservation in rpgmakermvmz searchCodes.""" + +from __future__ import annotations + +import copy +import json +import os +import re +import sys +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +os.chdir(ROOT) +sys.path.insert(0, str(ROOT)) + +import modules.rpgmakermvmz as mvmz # noqa: E402 + +LANGREGEX = mvmz.LANGREGEX + + +def _mock_translate(text, history, batch=False): + def tr(s): + if not isinstance(s, str): + return s + m = re.match(r"^(\[[^\]]+\]:\s?)", s) + if m: + return m.group(1) + "EN_TRANSLATED" + return "EN_TRANSLATED" + + if isinstance(text, list): + return [[tr(t) for t in text], [0, 0]] + return [tr(text), [0, 0]] + + +def _mock_speaker(name): + return [f"Speaker_{name}", [0, 0]] + + +FIXTURE_MAP = ROOT / "tests" / "fixtures" / "Map_original_fixture.json" +FIXTURE_MANIFEST = ROOT / "tests" / "fixtures" / "Map_original_fixture_manifest.json" +CASE_MARKER_RE = re.compile(r"# CASE:(\S+)") + + +def _load_fixture_page(): + data = json.loads(FIXTURE_MAP.read_text(encoding="utf-8-sig")) + event = next(e for e in data["events"] if e and e.get("id") == 1) + return {"list": copy.deepcopy(event["pages"][0]["list"])} + + +def _case_commands(page_list): + """Map manifest case id -> command immediately following its 108 marker.""" + cases = {} + pending = None + for cmd in page_list: + if not cmd: + continue + if cmd.get("code") == 108: + m = CASE_MARKER_RE.search(str(cmd.get("parameters", [""])[0])) + pending = m.group(1) if m else None + continue + if pending: + cases[pending] = cmd + pending = None + return cases + + +def _load_fixture_manifest(): + return json.loads(FIXTURE_MANIFEST.read_text(encoding="utf-8")) + + +def _load_map_excerpt(): + """Small real snippets from Map002 event 17 + Map001 102 + synthetic 101/122.""" + map2 = json.loads((ROOT / "files" / "Map002.json").read_text(encoding="utf-8-sig")) + ev17 = next(e for e in map2["events"] if e and e.get("id") == 17) + real = copy.deepcopy(ev17["pages"][0]["list"][7:14]) # 101 + 401 dialogue block + + map1 = json.loads((ROOT / "files" / "Map001.json").read_text(encoding="utf-8-sig")) + choice_cmd = None + for ev in map1["events"]: + if not ev: + continue + for pg in ev.get("pages") or []: + if not pg: + continue + for cmd in pg.get("list") or []: + if cmd and cmd.get("code") == 102: + choice_cmd = copy.deepcopy(cmd) + break + if choice_cmd: + break + if choice_cmd: + break + assert choice_cmd is not None, "Map001 should contain a 102 choice command" + + synthetic = [ + {"code": 101, "indent": 0, "parameters": ["", 0, 0, 2, "\\C[2]アリス\\C[0]"]}, + {"code": 401, "indent": 0, "parameters": ["こんにちは、世界。"]}, + { + "code": 122, + "indent": 0, + "parameters": [101, 101, 0, 0, "`変数テスト`"], + }, + ] + + return {"list": real + [choice_cmd] + synthetic} + + +def _resolve_case_command(page_list, entry, marked_cases=None): + """Find manifest case command by CASE marker or by code + expected _original.""" + marked_cases = marked_cases if marked_cases is not None else _case_commands(page_list) + cid = entry["id"] + cmd = marked_cases.get(cid) + if cmd is not None and cmd.get("code") == entry.get("code"): + return cmd + exp = entry.get("expected_original") + code = entry.get("code") + if code is not None and exp is not None: + for candidate in page_list: + if candidate and candidate.get("code") == code and candidate.get("_original") == exp: + return candidate + return cmd + + +def _run_search_codes(page): + """Full Pass 1 -> mock translate -> Pass 2 cycle.""" + captured = [] + + def translate(text, history, batch=False): + captured.append(copy.deepcopy(text)) + return _mock_translate(text, history, batch) + + def speaker(name): + return _mock_speaker(name) + + orig_t = mvmz.translateAI + orig_s = mvmz.getSpeaker + orig_122 = mvmz.CODE122 + orig_408 = mvmz.CODE408 + mvmz.translateAI = translate + mvmz.getSpeaker = speaker + mvmz.CODE122 = True + mvmz.CODE408 = True + try: + page_copy = copy.deepcopy(page) + mvmz.searchCodes(page_copy, None, [], "TestMap.json") + return page_copy, captured + finally: + mvmz.translateAI = orig_t + mvmz.getSpeaker = orig_s + mvmz.CODE122 = orig_122 + mvmz.CODE408 = orig_408 + + +def _find_commands(page, code): + return [cmd for cmd in page["list"] if cmd and cmd.get("code") == code] + + +def _has_japanese(s: str) -> bool: + return bool(re.search(LANGREGEX, s or "")) + + +class TestMVMZSourceOriginal(unittest.TestCase): + def test_first_pass_writes_original(self): + page, _ = _run_search_codes(_load_map_excerpt()) + + cmds401 = _find_commands(page, 401) + with_orig = [c for c in cmds401 if c.get("_original")] + self.assertGreater(len(with_orig), 0, "401 dialogue should have _original") + for c in with_orig: + self.assertTrue(_has_japanese(c["_original"])) + self.assertNotEqual(c["parameters"][0], c["_original"]) + + c102 = _find_commands(page, 102)[0] + self.assertIsInstance(c102.get("_original"), list) + self.assertEqual(len(c102["_original"]), len(c102["parameters"][0])) + for i, orig in enumerate(c102["_original"]): + if orig: + self.assertTrue(_has_japanese(orig), f"choice {i} _original should be Japanese") + self.assertNotEqual(c102["parameters"][0][i], orig) + + c101 = next(c for c in _find_commands(page, 101) if len(c.get("parameters", [])) > 4) + self.assertIn("_original", c101) + self.assertIn("アリス", c101["_original"]) + self.assertIn("\\C[2]", c101["_original"]) + self.assertIn("Speaker_", c101["parameters"][4]) + + c122 = _find_commands(page, 122)[0] + self.assertEqual(c122["_original"], "変数テスト") + self.assertIn("EN_TRANSLATED", c122["parameters"][4]) + + def test_rerun_uses_original_not_display_text(self): + page1, captured1 = _run_search_codes(_load_map_excerpt()) + originals_snapshot = json.dumps( + {i: cmd.get("_original") for i, cmd in enumerate(page1["list"]) if cmd}, + ensure_ascii=False, + ) + + page2, captured2 = _run_search_codes(page1) + originals_after = json.dumps( + {i: cmd.get("_original") for i, cmd in enumerate(page2["list"]) if cmd}, + ensure_ascii=False, + ) + self.assertEqual(originals_snapshot, originals_after, "_original must not change on re-run") + + # Every batch sent to translateAI on re-run should still contain Japanese + for payload in captured2: + items = payload if isinstance(payload, list) else [payload] + for item in items: + if not isinstance(item, str): + continue + if item == "EN_TRANSLATED": + continue + if _has_japanese(item): + continue + if re.match(r"^\[.+?\]:\s*EN_TRANSLATED$", item): + continue + self.fail(f"Re-run sent non-Japanese to translateAI: {item!r}") + + def test_map002_micro_page_real_data(self): + """Translate a tiny slice of Map002 event 17 only (7 commands).""" + map2 = json.loads((ROOT / "files" / "Map002.json").read_text(encoding="utf-8-sig")) + ev17 = next(e for e in map2["events"] if e and e.get("id") == 17) + micro = {"list": copy.deepcopy(ev17["pages"][0]["list"][8:11])} # 3x401 only + + page, _ = _run_search_codes(micro) + c401 = _find_commands(page, 401) + self.assertGreaterEqual(len(c401), 1) + with_orig = [c for c in c401 if c.get("_original")] + self.assertGreaterEqual(len(with_orig), 1) + for c in with_orig: + self.assertTrue(_has_japanese(c["_original"])) + + def test_speaker_color_line_full_original(self): + """Standalone \\C[n]Name\\C[n] speaker 401 lines keep the full string in _original.""" + page = { + "list": [ + {"code": 401, "indent": 0, "parameters": ["\\C[2]エルーシャ\\C[0]"]}, + {"code": 401, "indent": 0, "parameters": ["「テストセリフ」"]}, + ] + } + page, _ = _run_search_codes(page) + speaker_cmd = page["list"][0] + self.assertEqual(speaker_cmd.get("_original"), "\\C[2]エルーシャ\\C[0]") + self.assertIn("\\C[2]", speaker_cmd["parameters"][0]) + self.assertIn("Speaker_エルーシャ", speaker_cmd["parameters"][0]) + + # Re-run: _original unchanged, getSpeaker still receives Japanese name + speakers_seen = [] + + def speaker(name): + speakers_seen.append(name) + return _mock_speaker(name) + + orig_t, orig_s = mvmz.translateAI, mvmz.getSpeaker + mvmz.getSpeaker = speaker + mvmz.translateAI = lambda text, history, batch=False: _mock_translate(text, history, batch) + try: + mvmz.searchCodes(page, None, [], "TestMap.json") + finally: + mvmz.getSpeaker = orig_s + mvmz.translateAI = orig_t + self.assertEqual(speaker_cmd["_original"], "\\C[2]エルーシャ\\C[0]") + self.assertIn("エルーシャ", speakers_seen) + + def test_405_split_rerun_uses_anchor_original_only(self): + """English 405 siblings after a split must not pollute re-run source.""" + page = { + "list": [ + { + "code": 405, + "indent": 0, + "parameters": ["EN_LINE_1"], + "_original": "第一行\n第二行", + }, + {"code": 405, "indent": 0, "parameters": ["EN_LINE_2"]}, + ] + } + _, captured = _run_search_codes(page) + self.assertGreater(len(captured), 0) + payloads = captured if isinstance(captured[0], str) else captured + for payload in payloads: + if not isinstance(payload, str): + continue + self.assertIn("第一行", payload) + self.assertNotIn("EN_LINE_1", payload) + self.assertNotIn("EN_LINE_2", payload) + + + def test_408_choice_help_original(self): + page = { + "list": [ + {"code": 108, "indent": 0, "parameters": ["選択肢ヘルプ"]}, + {"code": 408, "indent": 0, "parameters": ["これは選択肢のヘルプです。"]}, + ] + } + page, captured = _run_search_codes(page) + cmd = _find_commands(page, 408)[0] + self.assertEqual(cmd.get("_original"), "これは選択肢のヘルプです。") + self.assertNotEqual(cmd["parameters"][0], cmd["_original"]) + self.assertGreater(len(captured), 0) + + page2, captured2 = _run_search_codes(page) + self.assertEqual(cmd["_original"], _find_commands(page2, 408)[0]["_original"]) + for payload in captured2: + items = payload if isinstance(payload, list) else [payload] + for item in items: + if not isinstance(item, str) or item == "EN_TRANSLATED": + continue + self.assertTrue(_has_japanese(item), f"408 re-run sent non-Japanese: {item!r}") + + +class TestFixtureMapOriginal(unittest.TestCase): + """Full fixture map covering every _original preservation code path.""" + + def test_fixture_all_cases_original(self): + page, _ = _run_search_codes(_load_fixture_page()) + marked = _case_commands(page["list"]) + manifest = _load_fixture_manifest() + + for entry in manifest["cases"]: + cid = entry["id"] + cmd = _resolve_case_command(page["list"], entry, marked) + expected = entry["expected_original"] + with self.subTest(case=cid): + self.assertIsNotNone(cmd, f"could not resolve fixture case {cid}") + self.assertEqual(cmd.get("_original"), expected, entry.get("summary", cid)) + if isinstance(expected, str): + self.assertTrue(_has_japanese(expected)) + if cmd.get("parameters"): + display = cmd["parameters"][0] + if isinstance(display, str): + self.assertNotEqual(display, expected) + + def test_fixture_rerun_preserves_original(self): + page1, _ = _run_search_codes(_load_fixture_page()) + page2, captured2 = _run_search_codes(page1) + manifest = _load_fixture_manifest() + + for entry in manifest["cases"]: + cid = entry["id"] + cmd1 = _resolve_case_command(page1["list"], entry) + cmd2 = _resolve_case_command(page2["list"], entry) + with self.subTest(case=cid): + self.assertIsNotNone(cmd1) + self.assertIsNotNone(cmd2) + self.assertEqual(cmd1.get("_original"), cmd2.get("_original")) + + for payload in captured2: + items = payload if isinstance(payload, list) else [payload] + for item in items: + if not isinstance(item, str) or item == "EN_TRANSLATED": + continue + if _has_japanese(item): + continue + if re.match(r"^\[.+?\]:\s*EN_TRANSLATED$", item): + continue + self.fail(f"Fixture re-run sent non-Japanese to translateAI: {item!r}") + + +if __name__ == "__main__": + unittest.main(verbosity=2)