This commit is contained in:
DazedAnon 2026-07-08 11:56:27 -05:00
parent b9ab810c26
commit 7a1109d8be
9 changed files with 760 additions and 100 deletions

View file

@ -87,7 +87,7 @@ from gui.workflow_tab import (
_make_text_btn, _make_text_btn,
_make_section_label, _make_section_label,
) )
from util import wolf_names from util.wolfdawn import names as wolf_names
from util.paths import PROJECT_ROOT from util.paths import PROJECT_ROOT
from util.project_scanner import ( from util.project_scanner import (
detect_wolf_layout, detect_wolf_layout,
@ -1696,8 +1696,10 @@ class WolfWorkflowTab(QWidget):
layout.addWidget(self._desc( layout.addWidget(self._desc(
"Sends the extracted files in files/ to the Translation tab using the Wolf RPG " "Sends the extracted files in files/ to the Translation tab using the Wolf RPG "
"(WolfDawn) module. Only the 'text' fields are filled in; 'source' is preserved so " "(WolfDawn) module. Only the 'text' fields are filled in; 'source' is preserved so "
"injection can verify each line. Run Step 3 (names) first so vocab.txt is seeded, " "injection can verify each line. Lines whose text is already translated (no Japanese) "
"then Phase 1 (database text) and Phase 2 (maps / events) in order." "are skipped, so re-running a phase only sends unfinished lines. Run Step 3 (names) "
"first so vocab.txt is seeded, then Phase 1 (database text) and Phase 2 "
"(maps / events) in order."
)) ))
self._add_phase_buttons(layout) self._add_phase_buttons(layout)
@ -2166,7 +2168,9 @@ class WolfWorkflowTab(QWidget):
"byte-exact. Translated safe/refs name values from names.json are applied across Data/ " "byte-exact. Translated safe/refs name values from names.json are applied across Data/ "
"(only the entries you translated change). Lines whose inline codes changed are skipped and " "(only the entries you translated change). Lines whose inline codes changed are skipped and "
"reported (unless you allow drift). Full inject and any inject that includes " "reported (unless you allow drift). Full inject and any inject that includes "
f"{NAMES_JSON} reset live Data/ from {WORK_DIR_NAME}/originals/ first." f"{NAMES_JSON} reset live Data/ from {WORK_DIR_NAME}/originals/ first. "
f"Before each inject, inline \\\\codes are auto-repaired from source; on a full inject "
f"every translated/ JSON is copied into {WORK_DIR_NAME}/ for the game-repo diff."
)) ))
self.en_punct_cb = QCheckBox("Convert Japanese punctuation to ASCII (--en-punct)") self.en_punct_cb = QCheckBox("Convert Japanese punctuation to ASCII (--en-punct)")
@ -2346,14 +2350,73 @@ class WolfWorkflowTab(QWidget):
now_btn.clicked.connect(lambda: self._run_relayout(manual=True)) now_btn.clicked.connect(lambda: self._run_relayout(manual=True))
layout.addWidget(now_btn) layout.addWidget(now_btn)
def _tool_root(self) -> Path:
"""DazedMTLTool project root (``files/``, ``translated/``, etc.)."""
return Path(__file__).resolve().parent.parent
def _translated_path(self, json_name: str) -> Path | None:
"""Absolute path to ``translated/<json_name>`` when it exists."""
p = self._tool_root() / "translated" / json_name
return p if p.is_file() else None
def _translated_or_source(self, json_name: str) -> Path | None: def _translated_or_source(self, json_name: str) -> Path | None:
"""Prefer translated/<name>; fall back to files/<name>.""" """Prefer translated/<name>; fall back to files/<name>."""
for base in ("translated", "files"): root = self._tool_root()
p = Path(base) / json_name for sub in ("translated", "files"):
p = root / sub / json_name
if p.is_file(): if p.is_file():
return p return p
return None return None
def _repair_inject_json(self, src: Path, log) -> Path:
"""Auto-repair WOLF inline codes in a translated JSON before inject."""
from util.wolfdawn import codes as wolf_codes
data = json.loads(src.read_text(encoding="utf-8-sig"))
data, repairs = wolf_codes.repair_document(data)
if repairs:
log(
f" Auto-repaired {len(repairs)} line(s) with control-code drift in {src.name}"
)
src.write_text(
json.dumps(data, ensure_ascii=False, indent=4) + "\n",
encoding="utf-8",
)
return src
def _sync_translated_json_to_wolf_json(
self, json_name: str, log, game_json_dir: Path
) -> bool:
"""Copy one translated/ JSON into the game's wolf_json/ folder."""
src = self._translated_path(json_name)
if src is None:
return False
dest = game_json_dir / json_name
try:
if src.resolve() == dest.resolve():
return True
except Exception:
pass
try:
dest.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(src, dest)
return True
except Exception as exc:
log(f" ⚠ could not update {json_name} in {WORK_DIR_NAME}/: {exc}")
return False
def _log_inject_guard_lines(self, mismatches: list[str], untranslated: int | None, log):
"""Surface WolfDawn guard failures instead of burying them in the scrollback."""
if untranslated:
log(f"{untranslated} line(s) left untranslated by WolfDawn safety guards")
if not mismatches:
return
log(f"{len(mismatches)} control-code mismatch(es) (first 10):")
for line in mismatches[:10]:
log(f" {line}")
if len(mismatches) > 10:
log(f" … and {len(mismatches) - 10} more (see log above)")
def _on_step_changed(self, idx: int): def _on_step_changed(self, idx: int):
previous = self._current_step_index previous = self._current_step_index
self._current_step_index = idx self._current_step_index = idx
@ -2382,7 +2445,9 @@ class WolfWorkflowTab(QWidget):
json_name = entry.get("json") json_name = entry.get("json")
if not json_name: if not json_name:
continue continue
if not (Path("translated") / json_name).is_file(): if entry.get("kind") == "names":
continue
if not self._translated_path(json_name):
continue continue
item = QListWidgetItem(json_name) item = QListWidgetItem(json_name)
item.setData(Qt.UserRole, json_name) item.setData(Qt.UserRole, json_name)
@ -2465,21 +2530,6 @@ class WolfWorkflowTab(QWidget):
quick = only_json is not None quick = only_json is not None
game_json_dir = self._work_dir() game_json_dir = self._work_dir()
def _sync_json(json_name: str, src: Path, log):
"""Copy the translated JSON into the game's git-tracked wolf_json/ so the
diff shows both the updated JSON source and the injected binary."""
dest = game_json_dir / json_name
try:
if src.resolve() == dest.resolve():
return
except Exception:
pass
try:
dest.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(src, dest)
except Exception as e:
log(f" ⚠ could not update {json_name} in {WORK_DIR_NAME}/: {e}")
def task(log, progress=None): def task(log, progress=None):
from util import wolfdawn from util import wolfdawn
@ -2492,6 +2542,7 @@ class WolfWorkflowTab(QWidget):
data_dir_path = Path(data_dir) data_dir_path = Path(data_dir)
applied = 0 applied = 0
failed = 0 failed = 0
guard_failures: list[str] = []
names_entry = next((e for e in entries if e["kind"] == "names"), None) names_entry = next((e for e in entries if e["kind"] == "names"), None)
names_src = ( names_src = (
@ -2520,7 +2571,7 @@ class WolfWorkflowTab(QWidget):
e["json"] e["json"]
for e in entries for e in entries
if e.get("kind") != "names" if e.get("kind") != "names"
and (Path("translated") / e["json"]).is_file() and self._translated_path(e["json"]) is not None
} }
strings_targets |= {j for j in only_json if j != NAMES_JSON} strings_targets |= {j for j in only_json if j != NAMES_JSON}
else: else:
@ -2532,10 +2583,13 @@ class WolfWorkflowTab(QWidget):
continue continue
if entry["json"] not in strings_targets: if entry["json"] not in strings_targets:
continue continue
src = self._translated_or_source(entry["json"]) inject_src = self._translated_path(entry["json"])
if src is None: if inject_src is None:
inject_src = self._translated_or_source(entry["json"])
if inject_src is None:
log(f" ⚠ no JSON for {entry['json']} — skipped") log(f" ⚠ no JSON for {entry['json']} — skipped")
continue continue
inject_src = self._repair_inject_json(inject_src, log)
out = entry["base"] # live game binary (or txt-dir) we write into out = entry["base"] # live game binary (or txt-dir) we write into
orig = self._orig_base_for(entry, data_dir_path) orig = self._orig_base_for(entry, data_dir_path)
base = str(orig) if orig.exists() else out base = str(orig) if orig.exists() else out
@ -2546,24 +2600,32 @@ class WolfWorkflowTab(QWidget):
) )
log(f"Injecting {entry['json']}{Path(out).name}") log(f"Injecting {entry['json']}{Path(out).name}")
res = wolfdawn.strings_inject( res = wolfdawn.strings_inject(
str(src), base, out, str(inject_src), base, out,
allow_code_drift=allow_drift, en_punct=en_punct, log_fn=log, allow_code_drift=allow_drift, en_punct=en_punct, log_fn=log,
) )
a, d = wolfdawn.parse_strings_inject_counts(res.stdout) a, d = wolfdawn.parse_strings_inject_counts(res.stdout, res.stderr)
untranslated = wolfdawn.parse_strings_inject_untranslated(
res.stdout, res.stderr
)
mismatches = wolfdawn.parse_strings_inject_mismatches(
res.stdout, res.stderr
)
strings_ok = wolfdawn.inject_had_applied(a) or ( strings_ok = wolfdawn.inject_had_applied(a) or (
res.ok and not (a == 0 and (d or 0) > 0) res.ok and not (a == 0 and (d or 0) > 0)
) )
if strings_ok: if strings_ok:
applied += 1 applied += 1
_sync_json(entry["json"], src, log) if mismatches or (untranslated or 0) > 0:
guard_failures.append(entry["json"])
self._log_inject_guard_lines(mismatches, untranslated, log)
if not res.ok and wolfdawn.inject_had_applied(a): if not res.ok and wolfdawn.inject_had_applied(a):
log( log(
f"{entry['json']}: wolf exited {res.returncode} " f"{entry['json']}: wolf exited {res.returncode} "
f"after applying {a} change(s) — see warnings above" f"after applying {a} change(s) — see warnings above"
) )
elif res.ok: elif res.ok:
# Exit 0 but nothing applied and lines drifted = stale/injected base.
failed += 1 failed += 1
guard_failures.append(entry["json"])
log( log(
f"{entry['json']}: 0 applied, {d} skipped as drift — the base " f"{entry['json']}: 0 applied, {d} skipped as drift — the base "
"looks already translated. Restore the pristine original (re-run " "looks already translated. Restore the pristine original (re-run "
@ -2571,6 +2633,8 @@ class WolfWorkflowTab(QWidget):
) )
else: else:
failed += 1 failed += 1
guard_failures.append(entry["json"])
self._log_inject_guard_lines(mismatches, untranslated, log)
log(f" ⚠ inject exit {res.returncode} for {entry['json']}") log(f" ⚠ inject exit {res.returncode} for {entry['json']}")
# Name values: apply across the whole data dir when translated/names.json # Name values: apply across the whole data dir when translated/names.json
@ -2589,10 +2653,7 @@ class WolfWorkflowTab(QWidget):
allow_code_drift=allow_drift, en_punct=en_punct, log_fn=log, allow_code_drift=allow_drift, en_punct=en_punct, log_fn=log,
) )
out = (res.stdout or "") + (res.stderr or "") out = (res.stdout or "") + (res.stderr or "")
a, d = wolfdawn.parse_names_inject_counts(out) a, d = wolfdawn.parse_names_inject_counts(res.stdout, res.stderr)
# Keep wolf_json/names.json aligned with translated/ even when the
# binary pass is a no-op (stale base) or exits 2 after partial guards.
_sync_json(names_entry["json"], names_src, log)
if wolfdawn.inject_had_applied(a): if wolfdawn.inject_had_applied(a):
applied += 1 applied += 1
if not res.ok: if not res.ok:
@ -2620,9 +2681,30 @@ class WolfWorkflowTab(QWidget):
failed += 1 failed += 1
log(f" ⚠ names-inject exit {res.returncode}") log(f" ⚠ names-inject exit {res.returncode}")
# Refresh wolf_json/ from translated/ for git (full inject: all docs;
# quick inject: only the files we touched).
if only_json is None:
sync_targets = [
e["json"] for e in entries if e.get("kind") != "names"
]
else:
sync_targets = sorted(strings_targets)
if will_names and names_entry:
sync_targets.append(names_entry["json"])
synced = 0
for json_name in sync_targets:
if self._sync_translated_json_to_wolf_json(
json_name, log, game_json_dir
):
synced += 1
if synced:
log(f"Synced {synced} translated JSON file(s) into {WORK_DIR_NAME}/")
msg = f"Injected {applied} document(s)" msg = f"Injected {applied} document(s)"
if guard_failures:
msg += f"; {len(guard_failures)} had guard warnings (see log)"
if failed: if failed:
msg += f", {failed} had guard/errors (see log)" msg += f"; {failed} failed"
if quick: if quick:
msg += ". Review the git diff in the game project and test in-game." msg += ". Review the git diff in the game project and test in-game."
else: else:

View file

@ -15,7 +15,10 @@ Supported document ``kind``s (all share the ``{source, text}`` leaf pattern):
Only entries whose ``source`` contains target-language (Japanese by default) Only entries whose ``source`` contains target-language (Japanese by default)
text are sent to the model; everything else keeps ``text == source`` so inject text are sent to the model; everything else keeps ``text == source`` so inject
is a no-op for it. is a no-op for it. When ``IGNORETLTEXT`` is on (default), entries whose ``text``
is already translated are skipped so Phase 2 can resume a partial run without
retranslating completed lines. Skip looks past Japanese nameplates on line 1
and Japanese embedded only in WOLF control codes (e.g. ``\\r[...]`` ruby).
names.json safety: WolfDawn tags every value name with a static ``safety`` badge names.json safety: WolfDawn tags every value name with a static ``safety`` badge
(``safe``, ``refs``, or ``verify``). For ``kind == "names"`` documents, only (``safe``, ``refs``, or ``verify``). For ``kind == "names"`` documents, only
@ -65,7 +68,8 @@ from util.translation import (
) )
from util import speakers as wolf_speakers from util import speakers as wolf_speakers
from util import vocab as wolf_vocab from util import vocab as wolf_vocab
from util import wolf_names from util.wolfdawn import codes as wolf_codes
from util.wolfdawn import names as wolf_names
# Globals (mirror the other engine modules; populated from .env at import time) # Globals (mirror the other engine modules; populated from .env at import time)
MODEL = os.getenv("model") MODEL = os.getenv("model")
@ -84,13 +88,24 @@ FILENAME = None
# Regex - default matches Japanese (kanji, kana, full-width forms). # Regex - default matches Japanese (kanji, kana, full-width forms).
LANGREGEX = r"[一-龠ぁ-ゔァ-ヴーa---\uFF61-\uFF9F]+" LANGREGEX = r"[一-龠ぁ-ゔァ-ヴーa---\uFF61-\uFF9F]+"
# Skip lines whose ``text`` is already translated. WolfDawn keeps Japanese in
# ``source`` forever, so skip-translated must look at ``text``, not ``source``.
# Set False for a forced full retranslate.
IGNORETLTEXT = True
# Control codes that can embed Japanese inside an otherwise-translated line
# (ruby ``\r[kanji,kana]``, colour / font indexes, ``\cself[n]``, etc.).
_WOLF_CODE_RE = re.compile(
r"\\(?:r\[[^\]]*\]|c(?:self)?\[[^\]]*\]|[A-Za-z]+\[[^\]]*\]|[A-Za-z])"
)
# Speaker handling: for first-line-speaker formats, reshape the line into the # Speaker handling: for first-line-speaker formats, reshape the line into the
# shared "[Speaker]: line" transport before translating and restore WOLF's # shared "[Speaker]: line" transport before translating and restore WOLF's
# native "Speaker\nline" layout on write-back. Which formats are reshaped is # native "Speaker\nline" layout on write-back. Which formats are reshaped is
# configurable from the workflow (data/wolf_speakers.json). # configurable from the workflow (data/wolf_speakers.json).
SPEAKER_CONFIG = wolf_speakers.load_config() SPEAKER_CONFIG = wolf_speakers.load_config()
# names.json: translate per-entry safe/refs badges only (see util.wolf_names). # names.json: translate per-entry safe/refs badges only (see util.wolfdawn.names).
# Text wrapping: optional advanced rewrap via dazedwrap (wolfWrap/wolfWidth). Defaults # Text wrapping: optional advanced rewrap via dazedwrap (wolfWrap/wolfWidth). Defaults
# off - the workflow uses WolfDawn relayout after inject instead. # off - the workflow uses WolfDawn relayout after inject instead.
@ -134,6 +149,11 @@ TRANSLATION_CONFIG = TranslationConfig(
) )
def _batch_phase() -> str:
"""Current Anthropic batch phase from the GUI subprocess env (``collect`` / ``consume``)."""
return (os.getenv("BATCH_PHASE") or "").strip().lower()
def handleWolfDawn(filename, estimate): def handleWolfDawn(filename, estimate):
"""Entry point used by the CLI/GUI dispatchers. Returns a summary string or 'Fail'.""" """Entry point used by the CLI/GUI dispatchers. Returns a summary string or 'Fail'."""
global ESTIMATE, TOKENS, FILENAME, SPEAKER_CONFIG, VOCAB global ESTIMATE, TOKENS, FILENAME, SPEAKER_CONFIG, VOCAB
@ -151,7 +171,11 @@ def handleWolfDawn(filename, estimate):
start = time.time() start = time.time()
translatedData = openFiles(filename) translatedData = openFiles(filename)
if not estimate: # Batch collect only queues API requests; text is still Japanese. Writing
# translated/ here would overwrite prior work with rewrapped source (and with
# wolfWrap on, the git diff looks like "translations" that never changed).
# Real English is written on the consume pass (or a live non-batch run).
if not estimate and _batch_phase() != "collect":
try: try:
with open("translated/" + filename, "w", encoding="utf-8", newline="\n") as outFile: with open("translated/" + filename, "w", encoding="utf-8", newline="\n") as outFile:
json.dump(translatedData[0], outFile, ensure_ascii=False, indent=4) json.dump(translatedData[0], outFile, ensure_ascii=False, indent=4)
@ -252,6 +276,44 @@ def _wrap_plain(text, is_firstline):
return prefix + _wrap_body(rest) return prefix + _wrap_body(rest)
def _text_check_body(text: str, speaker_src: str = "") -> str:
"""Body text used for skip-translated checks (nameplates / codes ignored).
Already-translated WOLF dialogue often keeps a Japanese speaker name on
line 1 (``司祭\\nSorry to keep you...``) and may embed Japanese only inside
control codes like ``\\r[,]``. Those lines are finished translations.
"""
if not isinstance(text, str):
return ""
_prefix, rest = wolf_speakers.split_window_prefix(text)
if "\n" in rest:
first, body = rest.split("\n", 1)
# Known first-line speaker formats, or a short nameplate-like first line.
if (
speaker_src in wolf_speakers.FIRSTLINE_SRCS
or speaker_src in ("ui", "narration")
or (0 < len(first.strip()) <= 20 and re.search(LANGREGEX, first))
):
rest = body
return _WOLF_CODE_RE.sub("", rest)
def _text_still_needs_translation(entry) -> bool:
"""True when ``text`` still needs the model (empty, Japanese, or identical)."""
txt = entry.get("text")
if not isinstance(txt, str) or not txt.strip():
return True
src = entry.get("source")
# Fresh / unfinished extract: text still mirrors Japanese source.
if isinstance(src, str) and txt == src:
return True
# Fully English (no Japanese chars at all).
if not re.search(LANGREGEX, txt):
return False
# Japanese only in the nameplate / control codes while the body is done.
return bool(re.search(LANGREGEX, _text_check_body(txt, entry.get("speaker_src", ""))))
def parseDocument(data, filename): def parseDocument(data, filename):
"""Translate every translatable leaf entry and return [data, tokens, error].""" """Translate every translatable leaf entry and return [data, tokens, error]."""
global PBAR global PBAR
@ -264,71 +326,92 @@ def parseDocument(data, filename):
src = e.get("source") src = e.get("source")
if not (isinstance(src, str) and re.search(LANGREGEX, src)): if not (isinstance(src, str) and re.search(LANGREGEX, src)):
return False return False
# Already translated: look at ``text`` (source stays Japanese forever).
# Fresh extracts keep ``text == source``, so they still queue.
if IGNORETLTEXT and not _text_still_needs_translation(e):
return False
# names.json: only translate WolfDawn safe/refs entries (per-name badge). # names.json: only translate WolfDawn safe/refs entries (per-name badge).
if is_names and not wolf_names.is_name_translatable(e): if is_names and not wolf_names.is_name_translatable(e):
return False return False
return True return True
# Only translate entries that actually contain target-language text (and, for # Only translate entries that still need work; names.json also requires a
# names.json, carry a translatable safety badge); the rest keep text == source so # safe/refs badge. Untouched leaves keep ``text == source`` so WolfDawn
# WolfDawn treats them as untouched on inject. # treats them as no-ops on inject.
translatable = [e for e in entries if _translatable(e)] translatable = [e for e in entries if _translatable(e)]
with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=len(translatable), leave=LEAVE) as pbar: with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=len(translatable), leave=LEAVE) as pbar:
pbar.desc = filename pbar.desc = filename
PBAR = pbar PBAR = pbar
if not translatable: if translatable:
return [data, totalTokens, None] # Reshape first-line-speaker lines into the shared "[Speaker]: line"
# transport format. plans[i] carries what is needed to restore each
# Reshape first-line-speaker lines into the shared "[Speaker]: line" # entry after translation.
# transport format. plans[i] carries what is needed to restore each sources = []
# entry after translation. plans = [] # (entry, prefix, has_speaker, is_firstline, code_map)
sources = [] for entry in translatable:
plans = [] # (entry, prefix, has_speaker, is_firstline) src = entry["source"]
for entry in translatable: protected_src, code_map = wolf_codes.protect_wolf_codes(src)
src = entry["source"] is_firstline = entry.get("speaker_src", "") in wolf_speakers.FIRSTLINE_SRCS
is_firstline = entry.get("speaker_src", "") in wolf_speakers.FIRSTLINE_SRCS split = wolf_speakers.split_source(
split = wolf_speakers.split_source(src, entry.get("speaker_src", ""), SPEAKER_CONFIG) protected_src, entry.get("speaker_src", ""), SPEAKER_CONFIG
if split is not None: )
prefix, speaker, body = split if split is not None:
sources.append(wolf_speakers.to_prefixed(speaker, body)) prefix, speaker, body = split
plans.append((entry, prefix, True, is_firstline)) sources.append(wolf_speakers.to_prefixed(speaker, body))
else: plans.append((entry, prefix, True, is_firstline, code_map))
sources.append(src)
plans.append((entry, "", False, is_firstline))
try:
response = translateAI(sources, [])
except Exception as e:
return [data, totalTokens, e]
translated, tokens = response[0], response[1]
totalTokens[0] += tokens[0]
totalTokens[1] += tokens[1]
# Write translations back (skip in estimate mode: translated == sources).
if not ESTIMATE and isinstance(translated, list) and len(translated) == len(plans):
for (entry, prefix, has_speaker, is_firstline), text in zip(plans, translated):
if not isinstance(text, str):
continue
if is_names:
entry["text"] = _wrap_names_entry(text, entry)
elif has_speaker:
speaker_en, body_en = wolf_speakers.parse_prefixed(text)
if speaker_en is not None:
# Wrap only the body; the name stays on its own line.
entry["text"] = wolf_speakers.restore_source(
prefix, speaker_en, _wrap_body(body_en)
)
else:
# Model dropped the [Speaker]: prefix; keep its output as-is.
entry["text"] = prefix + _wrap_body(text)
else: else:
entry["text"] = _wrap_plain(text, is_firstline) sources.append(protected_src)
plans.append((entry, "", False, is_firstline, code_map))
# Phase 0 feeds the glossary: harvest the just-translated safe name values try:
# into vocab.txt (grouped by note) so the DB-text and dialogue phases keep response = translateAI(sources, [])
# item/skill/term names consistent. Mirrors the RPGMaker DB-first strategy. except Exception as e:
return [data, totalTokens, e]
translated, tokens = response[0], response[1]
totalTokens[0] += tokens[0]
totalTokens[1] += tokens[1]
# Write translations back. Skip estimate mode, and skip the batch
# collect pass (response is still the Japanese source list — wrapping
# it into ``text`` only reflows JP and is what poisoned translated/).
collecting = _batch_phase() == "collect"
if (
not ESTIMATE
and not collecting
and isinstance(translated, list)
and len(translated) == len(plans)
):
for (entry, prefix, has_speaker, is_firstline, code_map), text, src in zip(
plans, translated, sources
):
if not isinstance(text, str):
continue
# Model / collect echoed the sent payload — leave the entry alone.
if text == src:
continue
text = wolf_codes.restore_wolf_code_placeholders(text, code_map)
if is_names:
entry["text"] = _wrap_names_entry(text, entry)
elif has_speaker:
speaker_en, body_en = wolf_speakers.parse_prefixed(text)
if speaker_en is not None:
# Wrap only the body; the name stays on its own line.
entry["text"] = wolf_speakers.restore_source(
prefix, speaker_en, _wrap_body(body_en)
)
else:
# Model dropped the [Speaker]: prefix; keep its output as-is.
entry["text"] = prefix + _wrap_body(text)
else:
entry["text"] = _wrap_plain(text, is_firstline)
wolf_codes.repair_entry(entry)
# Phase 0 feeds the glossary: harvest safe name values into vocab.txt
# (grouped by note) so the DB-text and dialogue phases keep item/skill/term
# names consistent. Runs even when every name was already translated (skip),
# so a resume still seeds vocab. Mirrors the RPGMaker DB-first strategy.
if is_names and not ESTIMATE: if is_names and not ESTIMATE:
_harvest_names_to_vocab(data) _harvest_names_to_vocab(data)

64
tests/test_wolf_codes.py Normal file
View file

@ -0,0 +1,64 @@
#!/usr/bin/env python3
"""Tests for WOLF inline control-code repair."""
from __future__ import annotations
import os
import sys
import unittest
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
os.chdir(ROOT)
sys.path.insert(0, str(ROOT))
from util.wolfdawn import codes as wolf_codes # noqa: E402
class WolfCodesRepairTests(unittest.TestCase):
def test_fixes_spurious_space_before_caret(self):
source = "占い師\nはぁ!\\^"
text = "Fortune-teller\nHa!\\ ^"
fixed = wolf_codes.rebuild_text_preserving_source_codes(source, text)
self.assertEqual(fixed, "Fortune-teller\nHa!\\^")
def test_rebuild_preserves_multiple_codes(self):
source = "A\\c[1]B\\f[2]C"
text = "A\\c[ 1]B\\f[ 2]C"
fixed = wolf_codes.rebuild_text_preserving_source_codes(source, text)
self.assertEqual(fixed, "A\\c[1]B\\f[2]C")
def test_protect_and_restore_roundtrip(self):
src = "Line with \\^ and \\cself[8]"
protected, mapping = wolf_codes.protect_wolf_codes(src)
self.assertNotIn("\\^", protected)
restored = wolf_codes.restore_wolf_code_placeholders(protected, mapping)
self.assertEqual(restored, src)
def test_repair_document_updates_leaf(self):
doc = {
"kind": "map",
"scenes": [
{
"event": 374,
"lines": [
{
"cmd": 233,
"str": 0,
"source": "占い師\nはぁ!\\^",
"text": "Fortune-teller\nHa!\\ ^",
}
],
}
],
}
_doc, notes = wolf_codes.repair_document(doc)
self.assertEqual(len(notes), 1)
self.assertEqual(
doc["scenes"][0]["lines"][0]["text"],
"Fortune-teller\nHa!\\^",
)
if __name__ == "__main__":
unittest.main()

View file

@ -33,6 +33,20 @@ class WolfInjectCountsTests(unittest.TestCase):
self.assertEqual(applied, 91) self.assertEqual(applied, 91)
self.assertEqual(drifted, 3) self.assertEqual(drifted, 3)
def test_parse_strings_inject_counts_with_untranslated_on_stderr(self):
stderr = (
"event 374 cmd 233 str 0: control-code mismatch - source has [\"\\\\^\"], "
"translation has [\"\\\\ \"]\n"
"applied 24547 translation(s) (37 untranslated, 0 drifted); wrote CommonEvent.dat\n"
)
applied, drifted = wolfdawn.parse_strings_inject_counts("", stderr)
self.assertEqual(applied, 24547)
self.assertEqual(drifted, 0)
self.assertEqual(wolfdawn.parse_strings_inject_untranslated("", stderr), 37)
mismatches = wolfdawn.parse_strings_inject_mismatches("", stderr)
self.assertEqual(len(mismatches), 1)
self.assertIn("control-code mismatch", mismatches[0])
def test_inject_had_applied_rejects_zero(self): def test_inject_had_applied_rejects_zero(self):
self.assertFalse(wolfdawn.inject_had_applied(0)) self.assertFalse(wolfdawn.inject_had_applied(0))
self.assertFalse(wolfdawn.inject_had_applied(None)) self.assertFalse(wolfdawn.inject_had_applied(None))

View file

@ -1,5 +1,5 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
"""Unit tests for WolfDawn names.json helpers in util/wolf_names.py.""" """Unit tests for WolfDawn names.json helpers in util/wolfdawn/names.py."""
from __future__ import annotations from __future__ import annotations
@ -14,7 +14,7 @@ ROOT = Path(__file__).resolve().parents[1]
os.chdir(ROOT) os.chdir(ROOT)
sys.path.insert(0, str(ROOT)) sys.path.insert(0, str(ROOT))
from util import wolf_names as wn # noqa: E402 from util.wolfdawn import names as wn # noqa: E402
class TestNameTranslatable(unittest.TestCase): class TestNameTranslatable(unittest.TestCase):

View file

@ -41,7 +41,7 @@ class _WolfTranslateHarness:
self.vocab_writes = [] self.vocab_writes = []
self.vocab_remove_writes = [] self.vocab_remove_writes = []
def run(self, data, filename="doc.json", estimate=False): def run(self, data, filename="doc.json", estimate=False, ignore_tl_text=True):
def translate(text, history, history_ctx=None): def translate(text, history, history_ctx=None):
self.captured.append(copy.deepcopy(text)) self.captured.append(copy.deepcopy(text))
return _mock_translate(text, history, history_ctx) return _mock_translate(text, history, history_ctx)
@ -52,11 +52,13 @@ class _WolfTranslateHarness:
orig_t = wd.translateAI orig_t = wd.translateAI
orig_estimate = wd.ESTIMATE orig_estimate = wd.ESTIMATE
orig_wrap = wd.WRAP orig_wrap = wd.WRAP
orig_ignore = wd.IGNORETLTEXT
orig_update = wd.wolf_vocab.update_vocab_section orig_update = wd.wolf_vocab.update_vocab_section
orig_labels = wd.wolf_names.derive_db_labels orig_labels = wd.wolf_names.derive_db_labels
wd.translateAI = translate wd.translateAI = translate
wd.ESTIMATE = estimate wd.ESTIMATE = estimate
wd.WRAP = False # keep write-back byte-faithful; wrapping tested separately wd.WRAP = False # keep write-back byte-faithful; wrapping tested separately
wd.IGNORETLTEXT = ignore_tl_text
# Never touch the real glossary / DB files during tests. # Never touch the real glossary / DB files during tests.
wd.wolf_vocab.update_vocab_section = capture_vocab wd.wolf_vocab.update_vocab_section = capture_vocab
wd.wolf_names.derive_db_labels = lambda _p: {} wd.wolf_names.derive_db_labels = lambda _p: {}
@ -68,6 +70,7 @@ class _WolfTranslateHarness:
wd.translateAI = orig_t wd.translateAI = orig_t
wd.ESTIMATE = orig_estimate wd.ESTIMATE = orig_estimate
wd.WRAP = orig_wrap wd.WRAP = orig_wrap
wd.IGNORETLTEXT = orig_ignore
wd.wolf_vocab.update_vocab_section = orig_update wd.wolf_vocab.update_vocab_section = orig_update
wd.wolf_names.derive_db_labels = orig_labels wd.wolf_names.derive_db_labels = orig_labels
@ -244,6 +247,213 @@ class TestTranslationWriteback(unittest.TestCase):
# In estimate mode text stays equal to source. # In estimate mode text stays equal to source.
self.assertEqual(data["scenes"][0]["lines"][0]["text"], "こんにちは") self.assertEqual(data["scenes"][0]["lines"][0]["text"], "こんにちは")
def test_collect_pass_does_not_mutate_or_write(self):
"""Batch collect must not reflow JP into text or overwrite translated/."""
orig_phase = os.environ.get("BATCH_PHASE")
os.environ["BATCH_PHASE"] = "collect"
orig_wrap = wd.WRAP
orig_width = wd.WRAPWIDTH
wd.WRAP = True
wd.WRAPWIDTH = 20
try:
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
(root / "files").mkdir()
(root / "translated").mkdir()
doc = copy.deepcopy(MAP_DOC)
# A long JP line that wrap would reflow if write-back ran.
doc["scenes"][0]["lines"][0]["source"] = "" * 40
doc["scenes"][0]["lines"][0]["text"] = "" * 40
src_path = root / "files" / "Map001.mps.json"
src_path.write_text(json.dumps(doc, ensure_ascii=False), encoding="utf-8")
out_path = root / "translated" / "Map001.mps.json"
out_path.write_text('{"kind":"map","marker":"keep-me"}', encoding="utf-8")
old_cwd = os.getcwd()
os.chdir(root)
try:
# Echo sources (what collect's queue path effectively returns).
def echo(text, history, history_ctx=None):
return [text if isinstance(text, list) else text, [0, 0]]
orig_t = wd.translateAI
wd.translateAI = echo
try:
result = wd.handleWolfDawn("Map001.mps.json", estimate=False)
finally:
wd.translateAI = orig_t
finally:
os.chdir(old_cwd)
self.assertNotEqual(result, "Fail")
# Prior translated/ content must be left alone.
self.assertEqual(
out_path.read_text(encoding="utf-8"),
'{"kind":"map","marker":"keep-me"}',
)
finally:
wd.WRAP = orig_wrap
wd.WRAPWIDTH = orig_width
if orig_phase is None:
os.environ.pop("BATCH_PHASE", None)
else:
os.environ["BATCH_PHASE"] = orig_phase
def test_echoed_source_does_not_get_wrapped_into_text(self):
"""If the model/collect echoes JP source, leave text alone even with wrap on."""
orig_wrap = wd.WRAP
orig_width = wd.WRAPWIDTH
wd.WRAP = True
wd.WRAPWIDTH = 10
try:
doc = {
"kind": "map",
"scenes": [
{
"event": 1,
"name": "ev",
"lines": [
{
"cmd": 0,
"str": 0,
"source": "あいうえおかきくけこさしすせそ",
"text": "あいうえおかきくけこさしすせそ",
},
],
}
],
}
def echo(text, history, history_ctx=None):
return [text if isinstance(text, list) else text, [1, 1]]
orig_t = wd.translateAI
wd.translateAI = echo
try:
data, _tok, err = wd.parseDocument(copy.deepcopy(doc), "echo.mps.json")
finally:
wd.translateAI = orig_t
self.assertIsNone(err)
# Must still be the unbroken source, not a wrapped JP reflow.
self.assertEqual(
data["scenes"][0]["lines"][0]["text"],
"あいうえおかきくけこさしすせそ",
)
finally:
wd.WRAP = orig_wrap
wd.WRAPWIDTH = orig_width
def test_skips_already_translated_text(self):
doc = {
"kind": "map",
"scenes": [
{
"event": 1,
"name": "ev",
"lines": [
{
"cmd": 0,
"str": 0,
"source": "こんにちは",
"text": "Hello",
},
{
"cmd": 1,
"str": 0,
"source": "さようなら",
"text": "さようなら",
},
{
"cmd": 2,
"str": 0,
"source": "まだ日本語",
"text": "Partial 日本語 left",
},
],
}
],
}
(data, _t, err), captured = _WolfTranslateHarness().run(doc, "partial.mps.json")
self.assertIsNone(err)
lines = data["scenes"][0]["lines"]
self.assertEqual(lines[0]["text"], "Hello")
self.assertEqual(lines[1]["text"], "EN_さようなら")
# Still-Japanese body is not skipped; the model still gets ``source``.
self.assertEqual(lines[2]["text"], "EN_まだ日本語")
self.assertEqual(captured, [["さようなら", "まだ日本語"]])
def test_skips_english_body_with_japanese_nameplate(self):
doc = {
"kind": "map",
"scenes": [
{
"event": 1,
"name": "ev",
"lines": [
{
"cmd": 0,
"str": 0,
"speaker": "司祭",
"speaker_src": "literal_line1_lowconf",
"source": "司祭\n皆さんお待たせしました……。",
"text": "司祭\nSorry to keep you all waiting......",
},
{
"cmd": 1,
"str": 0,
"speaker": "UI",
"speaker_src": "ui",
"source": "まだだ",
"text": "まだだ",
},
],
}
],
}
(data, _t, err), captured = _WolfTranslateHarness().run(doc, "nameplate.mps.json")
self.assertIsNone(err)
lines = data["scenes"][0]["lines"]
self.assertEqual(lines[0]["text"], "司祭\nSorry to keep you all waiting......")
self.assertEqual(lines[1]["text"], "EN_まだだ")
self.assertEqual(captured, [["まだだ"]])
def test_ignore_tl_text_false_retranslates(self):
doc = {
"kind": "map",
"scenes": [
{
"event": 1,
"name": "ev",
"lines": [
{"cmd": 0, "str": 0, "source": "こんにちは", "text": "Hello"},
],
}
],
}
(data, _t, err), captured = _WolfTranslateHarness().run(
doc, "force.mps.json", ignore_tl_text=False
)
self.assertIsNone(err)
self.assertEqual(data["scenes"][0]["lines"][0]["text"], "EN_こんにちは")
self.assertEqual(captured, [["こんにちは"]])
def test_names_already_translated_still_harvest(self):
doc = {
"kind": "names",
"names": [
{"source": "", "text": "Sword", "note": "武器", "safety": "safe"},
{"source": "", "text": "Spear", "note": "武器", "safety": "refs"},
],
}
harness = _WolfTranslateHarness()
(_data, _t, err), captured = harness.run(doc, "names.json")
self.assertIsNone(err)
self.assertEqual(captured, [])
self.assertEqual(
harness.vocab_writes,
[("Weapon · 武器", [("", "Sword"), ("", "Spear")])],
)
import re # noqa: E402 import re # noqa: E402

View file

@ -42,6 +42,8 @@ __all__ = [
"strings_inject", "strings_inject",
"names_inject", "names_inject",
"parse_strings_inject_counts", "parse_strings_inject_counts",
"parse_strings_inject_untranslated",
"parse_strings_inject_mismatches",
"parse_names_inject_counts", "parse_names_inject_counts",
"inject_had_applied", "inject_had_applied",
"pack", "pack",
@ -55,6 +57,17 @@ __all__ = [
_INJECT_COUNTS_RE = re.compile( _INJECT_COUNTS_RE = re.compile(
r"applied\s+(\d+)\s+translation.*?(\d+)\s+drifted", re.IGNORECASE | re.DOTALL r"applied\s+(\d+)\s+translation.*?(\d+)\s+drifted", re.IGNORECASE | re.DOTALL
) )
_INJECT_UNTRANSLATED_RE = re.compile(
r"applied\s+\d+\s+translation.*?\((\d+)\s+untranslated", re.IGNORECASE | re.DOTALL
)
_INJECT_MISMATCH_RE = re.compile(
r"^(?:event\s+\d+\s+)?cmd\s+(\d+)\s+str\s+(\d+):\s+control-code mismatch",
re.IGNORECASE | re.MULTILINE,
)
_INJECT_MISMATCH_EVENT_RE = re.compile(
r"^event\s+(\d+)\s+cmd\s+(\d+)\s+str\s+(\d+):\s+control-code mismatch",
re.IGNORECASE | re.MULTILINE,
)
_NAMES_INJECT_COUNTS_RE = re.compile( _NAMES_INJECT_COUNTS_RE = re.compile(
r"applied\s+(\d+)\s+name change.*?(\d+)\s+drifted", re.IGNORECASE | re.DOTALL r"applied\s+(\d+)\s+name change.*?(\d+)\s+drifted", re.IGNORECASE | re.DOTALL
) )
@ -69,21 +82,46 @@ _UNPACK_SUMMARY_RE = re.compile(
) )
def parse_strings_inject_counts(stdout: str) -> tuple[int | None, int | None]: def _inject_output_text(stdout: str, stderr: str) -> str:
"""WolfDawn prints per-line guards to stderr and the summary to either stream."""
return "\n".join(part for part in (stdout or "", stderr or "") if part)
def parse_strings_inject_counts(stdout: str, stderr: str = "") -> tuple[int | None, int | None]:
"""Return (applied, drifted) from wolf strings-inject output, or (None, None).""" """Return (applied, drifted) from wolf strings-inject output, or (None, None)."""
if not stdout: text = _inject_output_text(stdout, stderr)
if not text:
return None, None return None, None
m = _INJECT_COUNTS_RE.search(stdout) m = _INJECT_COUNTS_RE.search(text)
if not m: if not m:
return None, None return None, None
return int(m.group(1)), int(m.group(2)) return int(m.group(1)), int(m.group(2))
def parse_names_inject_counts(stdout: str) -> tuple[int | None, int | None]: def parse_strings_inject_untranslated(stdout: str, stderr: str = "") -> int | None:
"""Return the untranslated line count from strings-inject summary, if present."""
text = _inject_output_text(stdout, stderr)
m = _INJECT_UNTRANSLATED_RE.search(text)
return int(m.group(1)) if m else None
def parse_strings_inject_mismatches(stdout: str, stderr: str = "") -> list[str]:
"""Return human-readable control-code mismatch locations from inject output."""
text = _inject_output_text(stdout, stderr)
out: list[str] = []
for line in text.splitlines():
line = line.strip()
if "control-code mismatch" in line:
out.append(line)
return out
def parse_names_inject_counts(stdout: str, stderr: str = "") -> tuple[int | None, int | None]:
"""Return (applied, drifted) from wolf names-inject output, or (None, None).""" """Return (applied, drifted) from wolf names-inject output, or (None, None)."""
if not stdout: text = _inject_output_text(stdout, stderr)
if not text:
return None, None return None, None
m = _NAMES_INJECT_COUNTS_RE.search(stdout) m = _NAMES_INJECT_COUNTS_RE.search(text)
if not m: if not m:
return None, None return None, None
return int(m.group(1)), int(m.group(2)) return int(m.group(1)), int(m.group(2))

169
util/wolfdawn/codes.py Normal file
View file

@ -0,0 +1,169 @@
"""WOLF RPG Editor inline control-code helpers for WolfDawn JSON.
WolfDawn ``strings-inject`` requires each ``text`` line to carry the same
backslash control codes as its ``source`` (only the human-readable words may
change). Models often break codes (e.g. ``\\^`` becomes ``\\ ^``). These
helpers protect codes during translation and repair them before inject.
"""
from __future__ import annotations
import re
from typing import Any
from util import speakers as wolf_speakers
# Inline codes WolfDawn compares during inject (see strings-inject guard output).
# Order matters: longer bracketed forms before single-letter fallbacks.
WOLF_INLINE_CODE_RE = re.compile(
r"\\(?:"
r"r\[[^\]]*\]|" # ruby \r[kanji,kana]
r"c(?:self)?\[[^\]]*\]|" # \c[...] / \cself[...]
r"[A-Za-z]+\[[^\]]*\]|" # \font[0], \cdb[21:78:0], \wE[1], ...
r"\^|" # \^ (wait / beat — often mangled to "\ ^")
r"[ @<>\-]|" # other single-char escapes Wolf uses
r"[A-Za-z]" # \n, \c, \f, ...
r")"
)
# Placeholder transport (same convention as util.translation.protect_script_codes).
_PLACEHOLDER_PREFIX = "__WOLF_CODE_"
def _tokenize(rest: str) -> list[tuple[str, str]]:
"""Split *rest* into ``('lit', ...)`` and ``('code', ...)`` tokens."""
if not rest:
return []
out: list[tuple[str, str]] = []
last = 0
for m in WOLF_INLINE_CODE_RE.finditer(rest):
if m.start() > last:
out.append(("lit", rest[last : m.start()]))
out.append(("code", m.group(0)))
last = m.end()
if last < len(rest):
out.append(("lit", rest[last:]))
return out
def _fix_spurious_spaces_in_codes(source: str, text: str) -> str:
"""Undo common model errors like ``\\ ^`` when *source* has ``\\^``."""
for m in WOLF_INLINE_CODE_RE.finditer(source):
code = m.group(0)
# Single-char escapes: \^ \c etc. Models often insert a space after \\.
if len(code) == 2 and code[1] not in (" ", "\t"):
spaced = f"{code[0]} {code[1]}"
if spaced in text:
text = text.replace(spaced, code, 1)
return text
def rebuild_text_preserving_source_codes(source: str, text: str) -> str:
"""Return *text* with *source*'s inline code tokens and translated literals.
Keeps any leading ``@<option>\\n`` window prefix from the translated line when
present; otherwise preserves the source prefix byte-for-byte.
"""
if not isinstance(source, str) or not isinstance(text, str):
return text
if source == text:
return text
src_prefix, src_rest = wolf_speakers.split_window_prefix(source)
txt_prefix, txt_rest = wolf_speakers.split_window_prefix(text)
out_prefix = txt_prefix if txt_prefix else src_prefix
txt_rest = _fix_spurious_spaces_in_codes(src_rest, txt_rest)
src_parts = _tokenize(src_rest)
txt_parts = _tokenize(txt_rest)
src_lit = [p[1] for p in src_parts if p[0] == "lit"]
txt_lit = [p[1] for p in txt_parts if p[0] == "lit"]
if not src_parts:
return out_prefix + txt_rest
# Rebuild using source code skeleton + translated literal slots.
rebuilt: list[str] = []
lit_i = 0
for kind, val in src_parts:
if kind == "code":
rebuilt.append(val)
else:
if lit_i < len(txt_lit):
rebuilt.append(txt_lit[lit_i])
lit_i += 1
else:
rebuilt.append(val)
return out_prefix + "".join(rebuilt)
def protect_wolf_codes(text: str) -> tuple[str, dict[str, str]]:
"""Replace inline WOLF codes with placeholders before sending text to the model."""
if not text or not isinstance(text, str):
return text, {}
replacements: dict[str, str] = {}
counter = 0
def _sub(m: re.Match) -> str:
nonlocal counter
key = f"{_PLACEHOLDER_PREFIX}{counter}__"
replacements[key] = m.group(0)
counter += 1
return key
protected = WOLF_INLINE_CODE_RE.sub(_sub, text)
return protected, replacements
def restore_wolf_code_placeholders(text: str, replacements: dict[str, str]) -> str:
if not text or not replacements:
return text
out = text
for key, original in replacements.items():
out = out.replace(key, original)
return out
def repair_entry(entry: dict[str, Any]) -> tuple[bool, str]:
"""Repair one ``{source, text}`` leaf. Returns (changed, reason)."""
src, txt = entry.get("source"), entry.get("text")
if not isinstance(src, str) or not isinstance(txt, str) or not src or src == txt:
return False, ""
fixed = rebuild_text_preserving_source_codes(src, txt)
if fixed == txt:
return False, ""
entry["text"] = fixed
return True, "codes"
def _walk_entries(data: dict) -> list[dict]:
kind = data.get("kind")
entries: list[dict] = []
if kind in ("map", "common"):
for scene in data.get("scenes") or []:
entries.extend(scene.get("lines") or [])
elif kind == "db":
for group in data.get("groups") or []:
entries.extend(group.get("lines") or [])
elif kind in ("gamedat", "txt"):
entries.extend(data.get("lines") or [])
elif kind == "txt-dir":
for file_doc in data.get("files") or []:
entries.extend(file_doc.get("lines") or [])
return entries
def repair_document(data: dict) -> tuple[dict, list[str]]:
"""Repair every ``{source, text}`` leaf in a WolfDawn document in place."""
notes: list[str] = []
for entry in _walk_entries(data):
changed, _reason = repair_entry(entry)
if changed:
row = entry.get("rowName") or entry.get("speaker") or ""
notes.append(
f"event {entry.get('event', entry.get('row', '?'))} "
f"cmd {entry.get('cmd', entry.get('field', '?'))} "
f"str {entry.get('str', '')} {row}".strip()
)
return data, notes