diff --git a/gui/wolf_workflow_tab.py b/gui/wolf_workflow_tab.py index d6cec27..6a63023 100644 --- a/gui/wolf_workflow_tab.py +++ b/gui/wolf_workflow_tab.py @@ -12,7 +12,8 @@ Mirrors the RPGMaker WorkflowTab, driven by the vendored WolfDawn ``wolf`` CLI are translated per-name; refs and verify names are skipped. Harvests short name terms into vocab.txt. Step 4 Database - discover content layout; translate foundation DB sheets - (items, skills, descriptions) before narrative custom sheets + (items, skills, descriptions) before narrative custom sheets; + harvests short label fields (map names, titles) into vocab.txt Step 5 Maps/Events - .mps maps, CommonEvent, Game.dat, Evtext; speaker handling Step 6 Precheck - dry-run selected JSON for safety-guard skips; edit those lines before writing binaries @@ -1784,7 +1785,8 @@ class WolfWorkflowTab(QWidget): note = self._desc( "You don't need to add item / skill / enemy value names here: Phase 0 translates " - "WolfDawn safe entries from names.json and harvests them into vocab.txt " + "WolfDawn safe entries from names.json and harvests them into vocab.txt. " + "Foundation DB also merges short labels (map names, titles) after Step 4. " "Focus this glossary on characters and worldbuilding terms." ) layout.addWidget(note) @@ -1883,7 +1885,8 @@ class WolfWorkflowTab(QWidget): layout.addWidget(self._subheading("Translate database")) layout.addWidget(self._desc( "Run after Step 3 (names) so vocab.txt stays consistent. Only checked sheets " - "are sent to the model. Re-running skips lines already translated." + "are sent to the model. Re-running skips lines already translated. Short " + "foundation labels (map names, titles) are merged into vocab.txt." )) self._add_tl_mode_selector(layout) diff --git a/modules/wolfdawn.py b/modules/wolfdawn.py index 97f4914..89b0642 100644 --- a/modules/wolfdawn.py +++ b/modules/wolfdawn.py @@ -24,9 +24,10 @@ names.json safety: WolfDawn tags every value name with a static ``safety`` badge (``safe``, ``refs``, or ``verify``). For ``kind == "names"`` documents, only entries whose badge is ``safe`` are translated; ``refs`` and ``verify`` names and legacy entries without a badge keep ``text == source``. After translating -names.json, translatable entries are harvested into ``vocab.txt`` (grouped by -``note``, with bilingual headers) so later phases keep item / skill / term names -consistent. +names.json, short name-like entries are harvested into ``vocab.txt`` (grouped by +``note``, with bilingual headers). After foundation DB sheets are translated, +short label fields (map names, titles, etc.) are merged into the same glossary +so later phases keep place / term names consistent without overwriting names.json. Database sheet filter (Step 4): optional ``wolfDbIncludeGroups`` or ``wolfDbIncludeTiers`` in ``.env`` limit which ``kind: db`` lines are collected. @@ -353,6 +354,10 @@ def parseDocument(data, filename): # so a resume still seeds vocab. Mirrors the RPGMaker DB-first strategy. if is_names and not ESTIMATE: _harvest_names_to_vocab(data) + # Foundation DB label fields (map names, titles) merge into vocab.txt without + # replacing entries already seeded from names.json. + elif data.get("kind") == "db" and not ESTIMATE: + _harvest_db_to_vocab(data) return [data, totalTokens, None] @@ -389,6 +394,25 @@ def _harvest_names_to_vocab(data): traceback.print_exc() +def _harvest_db_to_vocab(data): + """Merge short foundation DB labels into vocab.txt, grouped by sheet. + + Uses merge mode so names.json-seeded entries for the same section stay put. + Descriptions and narrative dialogue are filtered out. + """ + try: + include_tiers, include_groups = wolf_db.load_db_filter_config() + by_sheet = wolf_db.collect_db_vocab_pairs( + data, + include_tiers=include_tiers, + include_groups=include_groups, + ) + for type_name, pairs in by_sheet.items(): + wolf_vocab.update_vocab_section(type_name, pairs, merge=True) + except Exception: + traceback.print_exc() + + def getResultString(translatedData, translationTime, filename): """Format the per-file / total cost + status line for the console log.""" cost = calculateCost(translatedData[1][0], translatedData[1][1], MODEL) diff --git a/tests/test_vocab.py b/tests/test_vocab.py index 3efd77e..8e629bf 100644 --- a/tests/test_vocab.py +++ b/tests/test_vocab.py @@ -63,6 +63,20 @@ class TestUpdateVocabSection(unittest.TestCase): self.assertIn("ヒール (Cure)", text) self.assertNotIn("ヒール (Heal)", text) + def test_merge_keeps_existing_and_adds_new(self): + vocab.update_vocab_section("Map Setting · マップ設定", [("礼拝堂", "Chapel")]) + vocab.update_vocab_section( + "Map Setting · マップ設定", + [("礼拝堂", "Chapel Hall"), ("大通り", "Main Street")], + merge=True, + ) + + text = self.vocab_path.read_text(encoding="utf-8") + self.assertIn("礼拝堂 (Chapel)", text) + self.assertNotIn("礼拝堂 (Chapel Hall)", text) + self.assertIn("大通り (Main Street)", text) + self.assertEqual(text.count("# Map Setting · マップ設定"), 1) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_wolf_db_classify.py b/tests/test_wolf_db_classify.py index 07f2059..8e6554f 100644 --- a/tests/test_wolf_db_classify.py +++ b/tests/test_wolf_db_classify.py @@ -238,5 +238,93 @@ class TestDbProfile(unittest.TestCase): self.assertFalse(checks[groups[1].key]) +class TestDbVocabHarvest(unittest.TestCase): + def test_map_name_is_candidate(self): + line = {"fieldName": "マップ名", "source": "礼拝堂", "text": "Chapel"} + self.assertTrue( + wdb.is_db_vocab_harvest_candidate( + line, + type_name="Map Setting · マップ設定", + tier=wdb.TIER_UNKNOWN, + ) + ) + + def test_description_is_not_candidate(self): + line = { + "fieldName": "Description · 説明", + "source": "味方一人のHPを回復", + "text": "Restores one ally's HP", + } + self.assertFalse( + wdb.is_db_vocab_harvest_candidate( + line, + type_name="Skill · 技能", + tier=wdb.TIER_FOUNDATION, + ) + ) + + def test_usage_message_with_name_placeholder_is_not_candidate(self): + line = { + "fieldName": "Usage Message [Battle] (Name~ · 使用時文章[戦闘](人名~", + "source": "の攻撃！", + "text": "'s attack!", + } + self.assertFalse( + wdb.is_db_vocab_harvest_candidate( + line, + type_name="Skill · 技能", + tier=wdb.TIER_FOUNDATION, + ) + ) + + def test_narrative_sheet_is_not_candidate(self): + line = {"fieldName": "マップ名", "source": "礼拝堂", "text": "Chapel"} + self.assertFalse( + wdb.is_db_vocab_harvest_candidate( + line, + type_name="■イベント(セルリア)", + tier=wdb.TIER_NARRATIVE, + ) + ) + + def test_collect_pairs_groups_by_sheet(self): + doc = { + "file": "SysDatabase.project", + "kind": "db", + "groups": [ + { + "typeName": "Map Setting · マップ設定", + "lines": [ + { + "fieldName": "マップ名", + "source": "礼拝堂", + "text": "Chapel", + }, + { + "fieldName": "Description · 説明", + "source": "静かな礼拝堂", + "text": "A quiet chapel", + }, + ], + }, + { + "typeName": "■イベント(セルリア)", + "lines": [ + { + "fieldName": "現在の行動", + "source": "礼拝堂で黙とう中", + "text": "Praying in the chapel", + }, + ], + }, + ], + } + pairs = wdb.collect_db_vocab_pairs(doc) + self.assertEqual( + pairs, + {"Map Setting · マップ設定": [("礼拝堂", "Chapel")]}, + ) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_wolfdawn.py b/tests/test_wolfdawn.py index 6cd8a07..cf89bda 100644 --- a/tests/test_wolfdawn.py +++ b/tests/test_wolfdawn.py @@ -46,22 +46,22 @@ class _WolfTranslateHarness: self.captured.append(copy.deepcopy(text)) return _mock_translate(text, history, history_ctx) - def capture_vocab(category, pairs): - self.vocab_writes.append((category, list(pairs))) + def capture_vocab(category, pairs, merge=False): + self.vocab_writes.append((category, list(pairs), merge)) orig_t = wd.translateAI orig_estimate = wd.ESTIMATE - orig_wrap = wd.WRAP orig_ignore = wd.IGNORETLTEXT orig_update = wd.wolf_vocab.update_vocab_section orig_labels = wd.wolf_names.derive_db_labels + orig_db_filter = wd.wolf_db.load_db_filter_config wd.translateAI = translate wd.ESTIMATE = estimate - wd.WRAP = False # keep write-back byte-faithful; wrapping tested separately wd.IGNORETLTEXT = ignore_tl_text # Never touch the real glossary / DB files during tests. wd.wolf_vocab.update_vocab_section = capture_vocab wd.wolf_names.derive_db_labels = lambda _p: {} + wd.wolf_db.load_db_filter_config = lambda: (frozenset(), frozenset()) try: data_copy = copy.deepcopy(data) result = wd.parseDocument(data_copy, filename) @@ -69,10 +69,10 @@ class _WolfTranslateHarness: finally: wd.translateAI = orig_t wd.ESTIMATE = orig_estimate - wd.WRAP = orig_wrap wd.IGNORETLTEXT = orig_ignore wd.wolf_vocab.update_vocab_section = orig_update wd.wolf_names.derive_db_labels = orig_labels + wd.wolf_db.load_db_filter_config = orig_db_filter MAP_DOC = { @@ -198,7 +198,7 @@ class TestTranslationWriteback(unittest.TestCase): self.assertEqual( harness.vocab_writes, [ - ("Weapon · 武器", [("剣", "EN_剣")]), + ("Weapon · 武器", [("剣", "EN_剣")], False), ], ) @@ -233,7 +233,7 @@ class TestTranslationWriteback(unittest.TestCase): finally: wd.wolf_vocab.remove_vocab_section = orig_remove self.assertIsNone(err) - self.assertEqual(harness.vocab_writes, [("Weapon · 武器", [("ダガー", "EN_ダガー")])]) + self.assertEqual(harness.vocab_writes, [("Weapon · 武器", [("ダガー", "EN_ダガー")], False)]) self.assertEqual(harness.vocab_remove_writes, ["├■プロフィール"]) def test_txtdir_translates(self): @@ -451,9 +451,94 @@ class TestTranslationWriteback(unittest.TestCase): self.assertEqual(captured, []) self.assertEqual( harness.vocab_writes, - [("Weapon · 武器", [("剣", "Sword")])], + [("Weapon · 武器", [("剣", "Sword")], False)], ) + def test_db_foundation_labels_harvest_to_vocab(self): + doc = { + "file": "SysDatabase.project", + "kind": "db", + "groups": [ + { + "type": 0, + "typeName": "Map Setting · マップ設定", + "lines": [ + { + "row": 0, + "field": 0, + "fieldName": "マップ名", + "source": "礼拝堂", + "text": "礼拝堂", + }, + { + "row": 1, + "field": 0, + "fieldName": "マップ名", + "source": "大通り", + "text": "大通り", + }, + { + "row": 0, + "field": 1, + "fieldName": "Description · 説明", + "source": "静かな礼拝堂", + "text": "静かな礼拝堂", + }, + ], + }, + { + "type": 1, + "typeName": "■イベント(セルリア)", + "lines": [ + { + "row": 0, + "field": 0, + "fieldName": "現在の行動", + "source": "礼拝堂で黙とう中", + "text": "礼拝堂で黙とう中", + }, + ], + }, + ], + } + harness = _WolfTranslateHarness() + (_data, _t, err), _c = harness.run(doc, "SysDatabase.project.json") + self.assertIsNone(err) + self.assertEqual( + harness.vocab_writes, + [ + ( + "Map Setting · マップ設定", + [("礼拝堂", "EN_礼拝堂"), ("大通り", "EN_大通り")], + True, + ), + ], + ) + + def test_db_harvest_skipped_in_estimate(self): + doc = { + "file": "SysDatabase.project", + "kind": "db", + "groups": [ + { + "type": 0, + "typeName": "Map Setting · マップ設定", + "lines": [ + { + "row": 0, + "field": 0, + "fieldName": "マップ名", + "source": "礼拝堂", + "text": "礼拝堂", + }, + ], + }, + ], + } + harness = _WolfTranslateHarness() + harness.run(doc, "SysDatabase.project.json", estimate=True) + self.assertEqual(harness.vocab_writes, []) + import re # noqa: E402 diff --git a/util/vocab.py b/util/vocab.py index 0d87bd2..1d2af53 100644 --- a/util/vocab.py +++ b/util/vocab.py @@ -62,7 +62,26 @@ def _norm(s: str) -> str: return re.sub(r"\s+", " ", str(s)).strip().casefold() -def update_vocab_section(category: str, pairs) -> None: +_SECTION_PAIR_RE = re.compile( + r"^(.+?)\s+\((.+)\)\s*$", +) + + +def _parse_section_pairs(section_body: str) -> dict[str, str]: + """Parse ``src (dst)`` lines from a vocab section body (no header).""" + pairs: dict[str, str] = {} + for raw in section_body.splitlines(): + line = raw.strip() + if not line or line.startswith("#"): + continue + m = _SECTION_PAIR_RE.match(line) + if not m: + continue + pairs[m.group(1)] = m.group(2) + return pairs + + +def update_vocab_section(category: str, pairs, *, merge: bool = False) -> None: """Insert or replace a ``# {category}`` section in the game-specific vocab. Mirrors the RPGMaker auto-glossary behaviour (translated DB names feed @@ -74,6 +93,8 @@ def update_vocab_section(category: str, pairs) -> None: - ``pairs``: iterable of ``(source, translated)``. Deduped by source (last wins); no-ops (empty translation or unchanged after normalisation) are dropped. When nothing survives filtering the file is left untouched. + - ``merge``: when True, keep existing entries for this category and only add + sources that are not already present (names.json stays authoritative). """ dedup: dict[str, str] = {} for src, dst in pairs: @@ -97,16 +118,26 @@ def update_vocab_section(category: str, pairs) -> None: game_part = existing base_part = "" - block_lines = [f"{src} ({dst})" for src, dst in dedup.items()] - new_block = f"# {category}\n" + "\n".join(block_lines) + "\n\n" - # Match this category's section up to the next '#' header or end of the # game portion. Handles '#Cat', '# Cat', '## Cat', etc. pattern = re.compile( - rf"^[\t ]*#+\s*{re.escape(category)}\s*$\r?\n.*?(?=^[\t ]*#|\Z)", + rf"^([\t ]*#+\s*{re.escape(category)}\s*$\r?\n)(.*?)(?=^[\t ]*#|\Z)", re.MULTILINE | re.DOTALL, ) - if pattern.search(game_part): + match = pattern.search(game_part) + if merge and match: + merged = _parse_section_pairs(match.group(2)) + for src, dst in dedup.items(): + if src not in merged: + merged[src] = dst + dedup = merged + if not dedup: + return + + block_lines = [f"{src} ({dst})" for src, dst in dedup.items()] + new_block = f"# {category}\n" + "\n".join(block_lines) + "\n\n" + + if match: new_game = pattern.sub(lambda _m: new_block, game_part, count=1) else: new_game = game_part.rstrip("\n") diff --git a/util/wolfdawn/db_classify.py b/util/wolfdawn/db_classify.py index 0b75e66..4153524 100644 --- a/util/wolfdawn/db_classify.py +++ b/util/wolfdawn/db_classify.py @@ -64,6 +64,28 @@ DIALOGUE_FIELDS_RE = re.compile( r"セリフ|コメント|相手|行為|台詞|会話|メッセージ_", ) +# Foundation DB lines that are short display labels (map names, term titles), +# not descriptions / battle messages / dialogue. Used when seeding vocab.txt. +_DB_VOCAB_FIELD_RE = re.compile( + r"マップ名|" + r"属性名|" + r"用語名|" + r"コマンド名|" + r"(?:^|·\s*)タイトル(?:$|[（(\s])|" + r"(?:^|·\s*)Name(?:\s*·|$)|" + r"(?:^|·\s*)Title(?:\s*·|$)|" + r"(?:^|·\s*)名称(?:$|[（(\s·])|" + r"(?:^|·\s*)称号(?!.*コメント)|" + r"(?:^|·\s*)好きなもの(?!.*コメント)", + re.IGNORECASE, +) +_DB_VOCAB_SKIP_FIELD_RE = re.compile( + r"説明|Description|文章|メッセージ|Message|セリフ|コメント|Comment|" + r"行為|相手|台詞|会話", + re.IGNORECASE, +) +_DB_VOCAB_HARVEST_MAX_LEN = 72 + ARCHETYPE_CLASSIC = "classic_rpg" ARCHETYPE_DB_HEAVY = "simulation_db_heavy" ARCHETYPE_HYBRID = "hybrid" @@ -458,6 +480,79 @@ def load_db_filter_config() -> tuple[frozenset[str], frozenset[str]]: ) +def is_db_vocab_harvest_candidate( + line: dict[str, Any], + *, + type_name: str = "", + tier: str | None = None, +) -> bool: + """True when a foundation DB line should seed ``vocab.txt``. + + Only short label-like fields on foundation/system/unknown sheets qualify. + Descriptions, battle messages, and narrative dialogue stay out. + """ + resolved = tier + if resolved is None: + resolved = classify_group_tier(type_name, [line]) + if resolved not in FOUNDATION_TIERS: + return False + field_name = str(line.get("fieldName") or "") + if _DB_VOCAB_SKIP_FIELD_RE.search(field_name): + return False + if not _DB_VOCAB_FIELD_RE.search(field_name): + return False + if DIALOGUE_FIELDS_RE.search(field_name): + return False + src = str(line.get("source") or "") + if not src.strip(): + return False + if "\n" in src or "\r" in src: + return False + if len(src) > _DB_VOCAB_HARVEST_MAX_LEN: + return False + if len(re.findall(r"[。！？]", src)) >= 2: + return False + return True + + +def collect_db_vocab_pairs( + doc: dict[str, Any], + *, + include_tiers: frozenset[str] | None = None, + include_groups: frozenset[str] | None = None, +) -> dict[str, list[tuple[str, str]]]: + """Return ``{typeName: [(source, text), ...]}`` for harvestable DB terms. + + Respects the same sheet filter used for translation when *include_tiers* / + *include_groups* are provided. Only foundation-tier sheets contribute. + """ + if doc.get("kind") != "db": + return {} + json_file = json_file_from_doc(doc) + tiers = include_tiers if include_tiers is not None else frozenset() + groups = include_groups if include_groups is not None else frozenset() + by_sheet: dict[str, list[tuple[str, str]]] = {} + for group in doc.get("groups") or []: + type_name = str(group.get("typeName") or "") + lines = group.get("lines") or [] + if not group_matches_filter(json_file, type_name, tiers, groups): + continue + tier = classify_group_tier(type_name, lines) + if tier not in FOUNDATION_TIERS: + continue + pairs: list[tuple[str, str]] = [] + for line in lines: + if not is_db_vocab_harvest_candidate(line, type_name=type_name, tier=tier): + continue + src, dst = line.get("source"), line.get("text") + if not isinstance(src, str) or not isinstance(dst, str): + continue + pairs.append((src, dst)) + if pairs: + by_sheet[type_name] = pairs + return by_sheet + + def group_matches_filter( json_file: str, type_name: str,