From 3dea0b89725948a80a7f7d4187e2efc4ad56b28b Mon Sep 17 00:00:00 2001 From: DazedAnon Date: Mon, 22 Jun 2026 05:25:52 -0500 Subject: [PATCH] Fix Speaker bug and add tests --- modules/csv.py | 4 ++-- modules/json.py | 15 ++++++------ modules/kirikiri.py | 7 +++--- modules/regex.py | 5 ++-- modules/renpy.py | 5 ++-- modules/rpgmakermvmz.py | 27 ++++++++++------------ modules/srpg.py | 21 ++++------------- modules/text.py | 18 +++++---------- modules/tyrano.py | 5 ++-- modules/wolf.py | 9 +++----- modules/wolf2.py | 5 ++-- modules/yuris.py | 8 +++---- tests/test_speaker_prefix.py | 44 ++++++++++++++++++++++++++++++++++++ util/speaker_prefix.py | 43 +++++++++++++++++++++++++++++++++++ 14 files changed, 139 insertions(+), 77 deletions(-) create mode 100644 tests/test_speaker_prefix.py create mode 100644 util/speaker_prefix.py diff --git a/modules/csv.py b/modules/csv.py index 81e36aa..ade91cf 100644 --- a/modules/csv.py +++ b/modules/csv.py @@ -14,6 +14,7 @@ from dotenv import load_dotenv from retry import retry from tqdm import tqdm from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost +from util.speaker_prefix import strip_speaker_prefix import tempfile # Globals @@ -330,8 +331,7 @@ def translateCSV(data, pbar, writeFile, writer, filename, translatedList): translatedList.pop(0) # Remove speaker prefix from translation if present - if speaker: - translatedText = re.sub(r"^\[?(.+?)\]?\s?[|:]\s?", "", translatedText) + translatedText = strip_speaker_prefix(translatedText) # Add wordwrap translatedText = dazedwrap.wrapText(translatedText, WIDTH) diff --git a/modules/json.py b/modules/json.py index e306dc3..1186a5f 100644 --- a/modules/json.py +++ b/modules/json.py @@ -13,6 +13,7 @@ from dotenv import load_dotenv from retry import retry from tqdm import tqdm from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost, getPricingConfig, calculateCost +from util.speaker_prefix import strip_speaker_prefix import tempfile # Globals @@ -912,10 +913,9 @@ def translateDialogueSrcTl(data, filename): # Strip speaker prefix if the AI echoed it back # Handles: [Speaker]: text | Speaker: text | Speaker(text) / CJK(text) - match = re.search(r'^\[.+?\]\s?[|:]\s?', translatedText) - if match: - translatedText = translatedText[match.end():] - else: + before_strip = translatedText + translatedText = strip_speaker_prefix(translatedText) + if translatedText == before_strip: # Fallback: strip any leading Japanese/CJK name followed by ( or : cjk_m = re.match(r'^[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+\s*[\(（:]\s*', translatedText) if cjk_m: @@ -1149,10 +1149,9 @@ def translateJSON(data, filename, translatedList): stringList = None # Remove speaker prefix — handles [Name]: / Name: / CJK(text) - match = re.search(r'(^\[.+?\]\s?[|:]\s?)', translatedText) - if match: - translatedText = translatedText.replace(match.group(1), "") - else: + before_strip = translatedText + translatedText = strip_speaker_prefix(translatedText) + if translatedText == before_strip: cjk_m = re.match(r'^[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+\s*[\(（:]\s*', translatedText) if cjk_m: translatedText = translatedText[cjk_m.end():] diff --git a/modules/kirikiri.py b/modules/kirikiri.py index 0c70174..66a2500 100644 --- a/modules/kirikiri.py +++ b/modules/kirikiri.py @@ -13,6 +13,7 @@ from dotenv import load_dotenv from retry import retry from tqdm import tqdm from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost, getPricingConfig, calculateCost +from util.speaker_prefix import strip_speaker_prefix import tempfile # Globals @@ -324,7 +325,7 @@ def translateKiriKiri(data, pbar, filename, jobList): translatedText = stringList[0] stringList.pop(0) # Remove Speaker label if present - translatedText = re.sub(r"\[.*?\]:\s", "", translatedText) + translatedText = strip_speaker_prefix(translatedText) # Wrap and convert newlines to [r] translatedText = dazedwrap.wrapText(translatedText, width=WIDTH) translatedText = translatedText.replace("\n", "[r]") @@ -362,7 +363,7 @@ def translateKiriKiri(data, pbar, filename, jobList): if len(stringList) > 0: translatedText = stringList[0] stringList.pop(0) - translatedText = re.sub(r"\[.*?\]:\s", "", translatedText) + translatedText = strip_speaker_prefix(translatedText) translatedText = dazedwrap.wrapText(translatedText, width=WIDTH) translatedText = translatedText.replace("\n", "[r]") indent_match = re.match(r"^([ \t\u3000]+)", data[i]) @@ -402,7 +403,7 @@ def translateKiriKiri(data, pbar, filename, jobList): stringList.pop(0) # Remove Speaker - translatedText = re.sub(r"\[.*?\]:\s", "", translatedText) + translatedText = strip_speaker_prefix(translatedText) # Textwrap translatedText = dazedwrap.wrapText(translatedText, width=WIDTH) diff --git a/modules/regex.py b/modules/regex.py index af881e5..0ab8c3f 100644 --- a/modules/regex.py +++ b/modules/regex.py @@ -13,6 +13,7 @@ from dotenv import load_dotenv from retry import retry from tqdm import tqdm from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost +from util.speaker_prefix import strip_speaker_prefix import tempfile # Globals @@ -291,7 +292,7 @@ def translateRegex(data, filename, translatedList): stringList = None # Remove speaker prefix from translation - translatedText = re.sub(r"^\[?(.+?)\]?\s?[|:]\s?", "", translatedText) + translatedText = strip_speaker_prefix(translatedText) # Textwrap translatedText = dazedwrap.wrapText(translatedText, width=WIDTH) @@ -383,7 +384,7 @@ def translateRegex(data, filename, translatedList): stringList = None # Remove speaker - translatedText = re.sub(r"^\[?(.+?)\]?\s?[|:]\s?", "", translatedText) + translatedText = strip_speaker_prefix(translatedText) # Escape Quotes translatedText = re.sub(r'(?\s*\[)(?P(?:\\n\[\d+\]|[^\]\n])+)(?P\]\s*[|:]\s*)", + tag_match = re.match( + rf"^(?P\s*\[)(?P{SPEAKER_BRACKET_INNER})(?P\]\s*[|:：]\s*)", s, re.IGNORECASE, ) - if speaker_prefix: - speaker = _VAR_ACTOR_RE.sub(_display_actor_name, speaker_prefix.group("speaker")) - body = _VAR_ACTOR_RE.sub(_repl, s[speaker_prefix.end():]) - return f"{speaker_prefix.group('open')}{speaker}{speaker_prefix.group('close')}{body}" + if tag_match: + speaker = _VAR_ACTOR_RE.sub(_display_actor_name, tag_match.group("speaker")) + body = _VAR_ACTOR_RE.sub(_repl, s[tag_match.end():]) + return f"{tag_match.group('open')}{speaker}{tag_match.group('close')}{body}" return _VAR_ACTOR_RE.sub(_repl, s) diff --git a/modules/srpg.py b/modules/srpg.py index f89e39f..483a9a0 100644 --- a/modules/srpg.py +++ b/modules/srpg.py @@ -14,6 +14,7 @@ from dotenv import load_dotenv from retry import retry from tqdm import tqdm from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost +from util.speaker_prefix import strip_speaker_prefix # Globals MODEL = os.getenv("model") @@ -501,10 +502,7 @@ def translateBookmark(data, filename, translatedDataList=None, pbar=None): else: if dataList: translated = dataList[0] - if speaker: - match = re.search(r'(^\[.+?\]\s?[|:]\s?)', translated) - if match: - translated = translated.replace(match.group(1), "") + translated = strip_speaker_prefix(translated) translated = dazedwrap.wrapText(translated, width=WIDTH) command["data"][i] = translated dataList.pop(0) @@ -1389,10 +1387,7 @@ def translateRecollection(data, filename, translatedDataList=None, pbar=None): translated = dataList[0] # Remove speaker - if speaker: - match = re.search(r'(^\[.+?\]\s?[|:]\s?)', translated) - if match: - translated = translated.replace(match.group(1), "") + translated = strip_speaker_prefix(translated) # Textwrap translated = dazedwrap.wrapText(translated, width=WIDTH) @@ -1843,10 +1838,7 @@ def translateMap(data, filename, translatedDataList=None, pbar=None): else: if dataList: translated = dataList[0] - if speaker: - match = re.search(r'(^\[.+?\]\s?[|:]\s?)', translated) - if match: - translated = translated.replace(match.group(1), "") + translated = strip_speaker_prefix(translated) translated = dazedwrap.wrapText(translated, width=WIDTH) command["data"][i] = translated dataList.pop(0) @@ -1919,10 +1911,7 @@ def translateMap(data, filename, translatedDataList=None, pbar=None): else: if dataList: translated = dataList[0] - if speaker: - match = re.search(r'(^\[.+?\]\s?[|:]\s?)', translated) - if match: - translated = translated.replace(match.group(1), "") + translated = strip_speaker_prefix(translated) translated = dazedwrap.wrapText(translated, width=WIDTH) command["data"][i] = translated dataList.pop(0) diff --git a/modules/text.py b/modules/text.py index e4cd2ff..bf964d1 100644 --- a/modules/text.py +++ b/modules/text.py @@ -13,6 +13,7 @@ from dotenv import load_dotenv from retry import retry from tqdm import tqdm from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost, getPricingConfig, calculateCost +from util.speaker_prefix import SPEAKER_TAG_RE, extract_dialogue_after_speaker, strip_speaker_prefix import tempfile # Globals @@ -208,27 +209,20 @@ def translateTxt(data, filename, translatedList): global LOCK, ESTIMATE, FILENAME, PBAR, MISMATCH i = 0 - # Regex - lineTextRegex = r"(?:^\[.+?\]:)?(.+)" - speakerTextRegex = r"^\[(.+?)\]" - while i < len(data): # Speaker - match = re.search(speakerTextRegex, data[i]) + match = SPEAKER_TAG_RE.match(data[i]) if match: # Get Speaker speakerData = getSpeaker(match.group(1)) speaker = speakerData[0] tokens[0] += speakerData[1][0] tokens[1] += speakerData[1][1] - data[i] = data[i].replace(match.group(1), speaker) + data[i] = data[i].replace(match.group(1), speaker, 1) # Dialogue - match = re.search(lineTextRegex, data[i]) - jaString = None - if match: - # Set String - jaString = match.group(1) + jaString = extract_dialogue_after_speaker(data[i]) + if jaString is not None: # Pass 1 if not translatedList: @@ -254,7 +248,7 @@ def translateTxt(data, filename, translatedList): stringList = None # Remove speaker - translatedText = re.sub(r"(^\[.+?\]\s?[|:]\s?)", "", translatedText) + translatedText = strip_speaker_prefix(translatedText) # # Textwrap # translatedText = dazedwrap.wrapText(translatedText, width=WIDTH) diff --git a/modules/tyrano.py b/modules/tyrano.py index afda46e..adac9c8 100644 --- a/modules/tyrano.py +++ b/modules/tyrano.py @@ -13,6 +13,7 @@ from dotenv import load_dotenv from retry import retry from tqdm import tqdm from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost +from util.speaker_prefix import strip_speaker_prefix import tempfile # OpenAI initialization centralized in util/translation.py @@ -320,9 +321,7 @@ def translateTyrano(data, filename, translatedList): stringList = None # Remove speaker - if speaker != "": - matchSpeakerList = re.findall(r"^\[?(.+?)\]?\s?[|:]\s?", translatedText) - translatedText = re.sub(r"^\[?(.+?)\]?\s?[|:]\s?", "", translatedText) + translatedText = strip_speaker_prefix(translatedText) # Avoid Crashes translatedText = translatedText.replace("[", "(") diff --git a/modules/wolf.py b/modules/wolf.py index 20a1868..5010498 100644 --- a/modules/wolf.py +++ b/modules/wolf.py @@ -14,6 +14,7 @@ from dotenv import load_dotenv from retry import retry from tqdm import tqdm from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost, getPricingConfig, calculateCost +from util.speaker_prefix import strip_speaker_prefix # OpenAI initialization centralized in util/translation.py @@ -391,9 +392,7 @@ def searchCodes(events, pbar, jobList, filename): translatedText = stringList[0] # Remove speaker - matchSpeakerList = re.findall(r"^(\[.+?\]\s?[|:]\s?)\s?", translatedText) - if len(matchSpeakerList) > 0: - translatedText = translatedText.replace(matchSpeakerList[0], "") + translatedText = strip_speaker_prefix(translatedText) # Textwrap if FIXTEXTWRAP is True: @@ -667,9 +666,7 @@ def searchCodes(events, pbar, jobList, filename): translatedText = list300[0] # Remove speaker - matchSpeakerList = re.findall(r"^(\[.+?\]\s?[|:]\s?)\s?", translatedText) - if len(matchSpeakerList) > 0: - translatedText = translatedText.replace(matchSpeakerList[0], "") + translatedText = strip_speaker_prefix(translatedText) # Textwrap if FIXTEXTWRAP is True: diff --git a/modules/wolf2.py b/modules/wolf2.py index 194f8d2..64700ed 100644 --- a/modules/wolf2.py +++ b/modules/wolf2.py @@ -14,6 +14,7 @@ from dotenv import load_dotenv from retry import retry from tqdm import tqdm from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost +from util.speaker_prefix import strip_speaker_prefix import tempfile # OpenAI initialization centralized in util/translation.py @@ -323,9 +324,7 @@ def translateWOLF(data, translatedList, pbar, filename): translatedList = None # Remove speaker - matchSpeakerList = re.findall(r"^(\[.+?\]\s?[|:]\s?)\s?", translatedText) - if len(matchSpeakerList) > 0: - translatedText = translatedText.replace(matchSpeakerList[0], "") + translatedText = strip_speaker_prefix(translatedText) # Textwrap translatedText = dazedwrap.wrapText(translatedText, width=WIDTH) diff --git a/modules/yuris.py b/modules/yuris.py index 739b205..343f5be 100644 --- a/modules/yuris.py +++ b/modules/yuris.py @@ -11,6 +11,7 @@ from colorama import Fore from tqdm import tqdm import util.dazedwrap as dazedwrap +from util.speaker_prefix import strip_speaker_prefix from util.translation import ( TranslationConfig, calculateCost, @@ -302,10 +303,9 @@ def translateYuris(data, filename): def stripSpeakerPrefix(translated_text): """Same behavior as modules/json.translateJSON: strip [Name]: prefix from TL lines.""" - match = re.search(r"(^\[.+?\]\s?[|:]\s?)", translated_text) - if match: - translated_text = translated_text.replace(match.group(1), "") - else: + before_strip = translated_text + translated_text = strip_speaker_prefix(translated_text) + if translated_text == before_strip: cjk_m = re.match( r"^[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+\s*[\(（:]\s*", translated_text, diff --git a/tests/test_speaker_prefix.py b/tests/test_speaker_prefix.py new file mode 100644 index 0000000..2124ad6 --- /dev/null +++ b/tests/test_speaker_prefix.py @@ -0,0 +1,44 @@ +import unittest + +from util.speaker_prefix import ( + SPEAKER_TAG_RE, + extract_dialogue_after_speaker, + strip_speaker_prefix, +) + + +class SpeakerPrefixTests(unittest.TestCase): + def test_strip_plain_speaker(self): + self.assertEqual(strip_speaker_prefix("[Kurone]: Hello"), "Hello") + + def test_strip_color_coded_speaker(self): + line = r"[\C[10]Hp Drink\C[0]]: Received item!" + self.assertEqual(strip_speaker_prefix(line), "Received item!") + + def test_strip_fullwidth_colon(self): + line = r"[\C[10]HPドリンク\C[0]]：をもらった！" + self.assertEqual(strip_speaker_prefix(line), "をもらった！") + + def test_tag_captures_color_coded_speaker(self): + line = r"[\C[10]Hp Drink\C[0]]: 【\c[10]HPドリンク\c[0]】をもらった！" + m = SPEAKER_TAG_RE.match(line) + self.assertIsNotNone(m) + self.assertEqual(m.group(1), r"\C[10]Hp Drink\C[0]") + + def test_extract_dialogue_skips_color_speaker(self): + line = r"[\C[10]Hp Drink\C[0]]: 【\c[10]HPドリンク\c[0]】をもらった！" + self.assertEqual( + extract_dialogue_after_speaker(line), + "【\\c[10]HPドリンク\\c[0]】をもらった！", + ) + + def test_actor_variable_in_speaker(self): + line = r"[\n[1]]: Actor line" + m = SPEAKER_TAG_RE.match(line) + self.assertIsNotNone(m) + self.assertEqual(m.group(1), r"\n[1]") + self.assertEqual(extract_dialogue_after_speaker(line), "Actor line") + + +if __name__ == "__main__": + unittest.main() diff --git a/util/speaker_prefix.py b/util/speaker_prefix.py new file mode 100644 index 0000000..fc4aa02 --- /dev/null +++ b/util/speaker_prefix.py @@ -0,0 +1,43 @@ +"""Shared [Speaker]: prefix parsing for dialogue lines. + +Handles RPG Maker control codes inside speaker brackets, e.g. +``[\\C[10]Hp Drink\\C[0]]: dialogue`` where inner ``[10]`` must not end the match early. +""" + +from __future__ import annotations + +import re + +# Inner text of [Speaker] — allows \C[n], \c[n], \i[n], \n[actorId], etc. +# The control-code branch (\X[...]) keeps the inner [..] from ending the bracket early. +SPEAKER_BRACKET_INNER = r"(?:\\[A-Za-z]+\[[^\]]*\]|[^\]\n])+" + +# A bare [Speaker]: / [Speaker]| / [Speaker]： prefix. +_PREFIX_PATTERN = rf"^\[{SPEAKER_BRACKET_INNER}\]\s*[|:：]\s*" +SPEAKER_PREFIX_RE = re.compile(_PREFIX_PATTERN, re.IGNORECASE) + +SPEAKER_TAG_RE = re.compile( + rf"^\[({SPEAKER_BRACKET_INNER})\]", + re.IGNORECASE, +) + +# Dialogue body after an optional [Speaker]: prefix (text.py lineTextRegex, etc.) +SPEAKER_PREFIX_OPTIONAL_RE = re.compile( + rf"(?:{_PREFIX_PATTERN})?(.+)", + re.IGNORECASE, +) + + +def strip_speaker_prefix(text: str) -> str: + """Remove a leading ``[Speaker]:`` / ``[Speaker]|`` / ``[Speaker]：`` prefix.""" + if not text: + return text + return SPEAKER_PREFIX_RE.sub("", text, count=1) + + +def extract_dialogue_after_speaker(text: str) -> str | None: + """Return dialogue body after an optional ``[Speaker]:`` prefix, or ``None``.""" + if not text: + return None + m = SPEAKER_PREFIX_OPTIONAL_RE.match(text) + return m.group(1) if m else None