From 4dd9d2bbd4115cb18b1714c5ddb5c39d88959d3f Mon Sep 17 00:00:00 2001 From: DazedAnon Date: Mon, 15 Jun 2026 17:34:32 -0500 Subject: [PATCH] Surely cleaning up all this mess wont break anything. surely --- .gitignore | 64 +- .pre-commit-config.yaml | 2 +- DazedMTLTool.desktop | 10 + README.md | 23 +- START.bat | 25 +- START.sh | 23 +- prompt.txt => data/prompt.txt | 0 vocab_base.txt => data/vocab_base.txt | 0 gui/main.py | 29 +- gui/rpgmaker_tab.py | 3 +- gui/srpg_tab.py | 3 +- gui/wolf_tab.py | 3 +- gui/workflow_tab.py | 12 +- modules/aquedi4.py | 430 +- modules/csv.py | 918 ++-- modules/images.py | 880 ++-- modules/json.py | 2720 +++++------ modules/kirikiri.py | 1064 ++--- modules/lune.py | 746 +-- modules/main.py | 764 +-- modules/nscript.py | 854 ++-- modules/regex.py | 1056 ++--- modules/renpy.py | 704 +-- modules/rpgmakermvmz.py | 10 +- modules/rpgmakerplugin.py | 2272 ++++----- modules/srpg.py | 4524 +++++++++--------- modules/text.py | 726 +-- modules/tyrano.py | 916 ++-- modules/unity.py | 708 +-- modules/wolf.py | 5400 +++++++++++----------- modules/wolf2.py | 842 ++-- modules/yuris.py | 6 +- scripts/launch.sh | 13 + start_gui.py => scripts/start_gui.py | 146 +- set_defaults.py => util/defaults.py | 6 +- util/linux_desktop.py | 72 + util/paths.py | 45 + util/translation.py | 6119 +++++++++++++------------ 38 files changed, 16162 insertions(+), 15976 deletions(-) create mode 100644 DazedMTLTool.desktop rename prompt.txt => data/prompt.txt (100%) rename vocab_base.txt => data/vocab_base.txt (100%) create mode 100644 scripts/launch.sh rename start_gui.py => scripts/start_gui.py (76%) rename set_defaults.py => util/defaults.py (90%) create mode 100644 util/linux_desktop.py create mode 100644 util/paths.py diff --git a/.gitignore b/.gitignore index c1b01d1..ba96724 100644 --- a/.gitignore +++ b/.gitignore @@ -1,32 +1,32 @@ - -.env -*.tmp -*.json -*.js -*.yaml -*.yml -*.txt -*.log -logs/ -*.rpy -*.csv -*.ks -*.cid -*.png -*.script -__pycache__ - -!assets/*.png -!.pre-commit-config.yaml -!prompt.txt -!vocab_base.txt -!requirements.txt -!util/tl_inspector/TLInspector.js -!util/forge/Forge_MZ.js -!util/forge/Forge_MV.js -!util/forge/upstream/Forge_MZ.js -!util/forge/upstream/Forge_MV.js -util/forge/.forge_version.json -util/ace/*.exe -util/ace/.tools_version.json -!util/ace/offline/*.exe + +.env +*.tmp +*.json +*.js +*.yaml +*.yml +*.txt +*.log +logs/ +*.rpy +*.csv +*.ks +*.cid +*.png +*.script +__pycache__ + +!assets/*.png +!.pre-commit-config.yaml +!data/prompt.txt +!data/vocab_base.txt +!requirements.txt +!util/tl_inspector/TLInspector.js +!util/forge/Forge_MZ.js +!util/forge/Forge_MV.js +!util/forge/upstream/Forge_MZ.js +!util/forge/upstream/Forge_MV.js +util/forge/.forge_version.json +util/ace/*.exe +util/ace/.tools_version.json +!util/ace/offline/*.exe diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6b31d65..e1497a5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,6 +3,6 @@ 'hooks': - 'id': 'set-defaults' 'name': 'Set Defaults in rpgmakermvmz.py' - 'entry': 'python set_defaults.py' + 'entry': 'python -m util.defaults' 'language': 'python' 'files': '^modules/rpgmakermvmz.py$' diff --git a/DazedMTLTool.desktop b/DazedMTLTool.desktop new file mode 100644 index 0000000..14da745 --- /dev/null +++ b/DazedMTLTool.desktop @@ -0,0 +1,10 @@ +[Desktop Entry] +Type=Application +Name=DazedMTLTool +GenericName=Translation Tool +Comment=AI translation tool for visual novels and RPG games +Exec=bash -c 'exec "$(cd "$(dirname "%k")" && pwd)/scripts/launch.sh"' +Icon=%k/assets/icon.png +Terminal=false +Categories=Development;Utility; +StartupWMClass=DazedMTLTool diff --git a/README.md b/README.md index 9c9cb5e..ae2c356 100644 --- a/README.md +++ b/README.md @@ -124,12 +124,11 @@ This means Python wasn't added to your PATH. You have two options: ### 3. Launch the GUI -**Double-click `START.bat`**. It will: -- Create a virtual environment automatically. -- Install all dependencies. -- Launch the GUI. +**Windows:** Double-click `START.bat`. It will create a virtual environment, install dependencies, and open the GUI. -That's it! From now on, just double-click `START.bat` to open the tool. +**Linux/macOS:** Run `./START.sh`, or double-click `DazedMTLTool.desktop` (choose **Allow Launching** when your file manager asks). From then on, either method works. + +That's it! Use the same launcher each time you want to open the tool. --- @@ -160,10 +159,10 @@ Specialized tabs with extra options for those specific engines. ## Vocab & Prompt -### vocab.txt +### data/vocab.txt This file gives the AI context about your game — character names, genders, recurring terms, etc. The better your vocab file, the more consistent the translation. -Open `vocab.txt` (or copy `vocab.txt.example` to `vocab.txt` if it doesn't exist) and add entries like: +On first run, `data/vocab.txt` is created automatically from `data/vocab_base.txt` if it does not exist yet. Add entries like: ```plaintext # Game Characters @@ -176,14 +175,14 @@ Format: Japanese name, English name in parentheses, then gender. > **Note:** A very large vocab file can increase API costs and potentially reduce quality. Focus on the most important characters and terms. -### prompt.txt -This is the system prompt sent to the AI. A default `prompt.txt` is included and works well for most games. You generally don't need to edit it unless you want to customize the translation style. +### data/prompt.txt +This is the system prompt sent to the AI. A default `data/prompt.txt` is included and works well for most games. You generally don't need to edit it unless you want to customize the translation style. --- ## Tips -- **Check `log/translations.txt`** after a run to see what was translated. You can copy useful terms from it into `vocab.txt` for consistency in future runs. +- **Check `log/translations.txt`** after a run to see what was translated. You can copy useful terms from it into `data/vocab.txt` for consistency in future runs. - **Start small** — Translate a few files first to make sure the output looks good before doing the whole game. - **Wordwrap** — If text overflows or looks awkward in-game, adjust the `width` setting in `.env` or the Config tab. `60` is a good default for most RPG Maker games. - **Version control** — Using [Git](https://git-scm.com/) with the game folder is highly recommended. It lets you track every change the translation makes, compare with original files, and roll back if needed. @@ -300,8 +299,8 @@ Here's the recommended step-by-step process for translating an RPG Maker MV/MZ g | Step | Action | |------|--------| -| **1** | **Parse speakers → vocab.txt** — Use the Parse Speakers feature to pull character names from the game files into `vocab.txt`. | -| **2** | **Identify speaker genders** — Figure out which characters are male/female and update `vocab.txt` accordingly. This helps the AI use correct pronouns. | +| **1** | **Parse speakers → vocab** — Use the Parse Speakers feature to pull character names from the game files into `data/vocab.txt`. | +| **2** | **Identify speaker genders** — Figure out which characters are male/female and update `data/vocab.txt` accordingly. This helps the AI use correct pronouns. | | **3** | **Translate Actors.json, MapInfos.json** — These are small files with character and map names. Good to do first. | | **4** | **Translate Items, System, Weapons, etc.** — All the data files that aren't maps or events. Place them in `files/`, translate, then copy results back. | | **5** | **Find speaker names** — Enable CODE 101 (Speakers), check for bracketed names, or use the "First Line = Speaker" option to capture speaker names properly. | diff --git a/START.bat b/START.bat index e2e3f3f..b910a60 100644 --- a/START.bat +++ b/START.bat @@ -184,28 +184,29 @@ echo Launching DazedMTLTool GUI... echo ========================================== echo. -:: Ensure vocab.txt exists (create from example if available) -if not exist "vocab.txt" ( - if exist "vocab.txt.example" ( - echo vocab.txt not found - creating from vocab.txt.example... - copy /Y "vocab.txt.example" "vocab.txt" >nul 2>&1 +:: Ensure data/vocab.txt exists (create from vocab_base.txt if available) +if not exist "data\\vocab.txt" ( + if not exist "data" mkdir data + if exist "data\\vocab_base.txt" ( + echo data\vocab.txt not found - creating from data\vocab_base.txt... + copy /Y "data\vocab_base.txt" "data\vocab.txt" >nul 2>&1 if errorlevel 1 ( - echo ERROR: Failed to copy vocab.txt.example to vocab.txt. + echo ERROR: Failed to copy data\vocab_base.txt to data\vocab.txt. ) else ( - echo Created vocab.txt from vocab.txt.example + echo Created data\vocab.txt from data\vocab_base.txt ) ) else ( - echo vocab.txt and vocab.txt.example not found - creating empty vocab.txt to avoid import errors... - type NUL > "vocab.txt" + echo data\vocab.txt not found - creating empty file to avoid import errors... + type NUL > "data\vocab.txt" if errorlevel 1 ( - echo ERROR: Failed to create empty vocab.txt. + echo ERROR: Failed to create empty data\vocab.txt. ) else ( - echo Created empty vocab.txt + echo Created empty data\vocab.txt ) ) ) -python start_gui.py +python scripts/start_gui.py :: Check if GUI launched successfully if errorlevel 1 ( diff --git a/START.sh b/START.sh index 6bfa62e..f087cf1 100755 --- a/START.sh +++ b/START.sh @@ -127,23 +127,24 @@ activate_venv() { } ensure_vocab_file() { - if [[ -f "vocab.txt" ]]; then + mkdir -p data + if [[ -f "data/vocab.txt" ]]; then return 0 fi - if [[ -f "vocab.txt.example" ]]; then - echo "vocab.txt not found - creating from vocab.txt.example..." - if cp "vocab.txt.example" "vocab.txt"; then - echo "Created vocab.txt from vocab.txt.example" + if [[ -f "data/vocab_base.txt" ]]; then + echo "data/vocab.txt not found - creating from data/vocab_base.txt..." + if cp "data/vocab_base.txt" "data/vocab.txt"; then + echo "Created data/vocab.txt from data/vocab_base.txt" else - echo "ERROR: Failed to copy vocab.txt.example to vocab.txt." + echo "ERROR: Failed to copy data/vocab_base.txt to data/vocab.txt." fi else - echo "vocab.txt and vocab.txt.example not found - creating empty vocab.txt to avoid import errors..." - if : > "vocab.txt"; then - echo "Created empty vocab.txt" + echo "data/vocab.txt not found - creating empty file to avoid import errors..." + if : > "data/vocab.txt"; then + echo "Created empty data/vocab.txt" else - echo "ERROR: Failed to create empty vocab.txt." + echo "ERROR: Failed to create empty data/vocab.txt." fi fi } @@ -227,7 +228,7 @@ echo ensure_vocab_file -if ! python start_gui.py; then +if ! exec python "$SCRIPT_DIR/scripts/start_gui.py"; then echo echo "ERROR: Failed to launch GUI." echo "Check the error messages above." diff --git a/prompt.txt b/data/prompt.txt similarity index 100% rename from prompt.txt rename to data/prompt.txt diff --git a/vocab_base.txt b/data/vocab_base.txt similarity index 100% rename from vocab_base.txt rename to data/vocab_base.txt diff --git a/gui/main.py b/gui/main.py index fd35eb5..7d6c3b6 100644 --- a/gui/main.py +++ b/gui/main.py @@ -19,20 +19,16 @@ from PyQt5.QtWidgets import ( QTextEdit, QSplitter, QGroupBox, QStatusBar, QStackedWidget, QToolButton, QDialog ) -from PyQt5.QtCore import Qt, QThread, pyqtSignal, QTimer, QSettings -from PyQt5.QtGui import QIcon, QFont, QPixmap, QScreen +from PyQt5.QtCore import Qt, QThread, pyqtSignal, QTimer, QSettings, QCoreApplication +from PyQt5.QtGui import QIcon, QFont, QPixmap, QScreen, QGuiApplication -def _project_root() -> Path: - """Directory containing start_gui.py / requirements.txt (parent of gui/).""" - return Path(__file__).resolve().parent.parent +from util.paths import PROJECT_ROOT, ICON_PATH, LAST_UPDATE_SHA_PATH def load_application_icon() -> QIcon: """Prefer ICO on Windows; PNG is fine for all platforms. Empty if missing.""" - root = _project_root() - for rel in ("assets/icon.ico", "assets/icon.png"): - path = root / rel + for path in (PROJECT_ROOT / "assets/icon.ico", ICON_PATH): if path.is_file(): icon = QIcon(str(path)) if not icon.isNull(): @@ -126,14 +122,13 @@ class UpdateThread(QThread): REPO_USER = "DazedAnon" REPO_NAME = "DazedMTLTool" REPO_BRANCH = "main" - SHA_FILE = "last_update_sha.txt" + SHA_FILE = str(LAST_UPDATE_SHA_PATH) # Paths (relative, top-level) that should never be touched during update - PROTECTED = {".env", "venv", "log", "files", "translated", - "vocab.txt", "last_update_sha.txt"} + PROTECTED = {".env", "venv", "log", "files", "translated", "data"} # GitLab zip archives do not preserve Unix execute bits; restore after apply. - EXECUTABLE_SUFFIXES = {".sh"} + EXECUTABLE_SUFFIXES = {".sh", ".desktop"} progress = pyqtSignal(str) # status message finished = pyqtSignal(bool, str) # (success, message) @@ -902,6 +897,14 @@ def main(): # Set high DPI scale factor policy if hasattr(QApplication, 'setHighDpiScaleFactorRoundingPolicy'): QApplication.setHighDpiScaleFactorRoundingPolicy(Qt.HighDpiScaleFactorRoundingPolicy.PassThrough) + + if sys.platform.startswith("linux"): + from util.linux_desktop import ensure_linux_desktop_entry + ensure_linux_desktop_entry() + QGuiApplication.setDesktopFileName("DazedMTLTool") + + QCoreApplication.setOrganizationName("DazedTranslations") + QCoreApplication.setApplicationName("DazedMTLTool") app = QApplication(sys.argv) @@ -934,9 +937,7 @@ def main(): return 1 # Set application properties - app.setApplicationName("DazedMTLTool") app.setApplicationVersion("1.0") - app.setOrganizationName("DazedTranslations") # Apply dark theme with cleaner, more compact styling app.setStyleSheet(""" diff --git a/gui/rpgmaker_tab.py b/gui/rpgmaker_tab.py index ef7c86b..60c8418 100644 --- a/gui/rpgmaker_tab.py +++ b/gui/rpgmaker_tab.py @@ -19,8 +19,7 @@ except ImportError: ConfigIntegration = None try: # Prefer importing the project's canonical defaults if available - import set_defaults - CANONICAL_DEFAULTS = getattr(set_defaults, 'DEFAULTS', None) + from util.defaults import DEFAULTS as CANONICAL_DEFAULTS except Exception: CANONICAL_DEFAULTS = None diff --git a/gui/srpg_tab.py b/gui/srpg_tab.py index 43f0262..476f497 100644 --- a/gui/srpg_tab.py +++ b/gui/srpg_tab.py @@ -12,8 +12,7 @@ import re from gui.platform_glyph import platform_glyph try: - import set_defaults - CANONICAL_DEFAULTS = getattr(set_defaults, 'DEFAULTS', None) + from util.defaults import DEFAULTS as CANONICAL_DEFAULTS except Exception: CANONICAL_DEFAULTS = None diff --git a/gui/wolf_tab.py b/gui/wolf_tab.py index cc507e1..c37b9e5 100644 --- a/gui/wolf_tab.py +++ b/gui/wolf_tab.py @@ -11,8 +11,7 @@ from PyQt5.QtCore import pyqtSignal import re from gui.platform_glyph import platform_glyph try: - import set_defaults - CANONICAL_DEFAULTS = getattr(set_defaults, 'DEFAULTS', None) + from util.defaults import DEFAULTS as CANONICAL_DEFAULTS except Exception: CANONICAL_DEFAULTS = None diff --git a/gui/workflow_tab.py b/gui/workflow_tab.py index d130995..6148b2f 100644 --- a/gui/workflow_tab.py +++ b/gui/workflow_tab.py @@ -22,6 +22,8 @@ import sys import threading from pathlib import Path +from util.paths import VOCAB_BASE_PATH, VOCAB_PATH + import jsbeautifier from PyQt5.QtCore import Qt, QEvent, QSettings, QThread, QTimer, pyqtSignal @@ -3576,7 +3578,7 @@ class WorkflowTab(QWidget): def _read_vocab_speakers(self) -> list[tuple[str, str]]: """Parse the '# Speakers' section from vocab.txt and return (orig, tl) pairs.""" - vocab_path = Path("vocab.txt") + vocab_path = VOCAB_PATH if not vocab_path.exists(): return [] try: @@ -3606,7 +3608,7 @@ class WorkflowTab(QWidget): return results def _reload_vocab(self): - vocab_path = Path("vocab.txt") + vocab_path = VOCAB_PATH try: if vocab_path.exists(): text = vocab_path.read_text(encoding="utf-8") @@ -3623,10 +3625,10 @@ class WorkflowTab(QWidget): def _save_vocab(self): try: game_text = self.vocab_editor.toPlainText().rstrip("\n") - base_path = Path("vocab_base.txt") + base_path = VOCAB_BASE_PATH base_text = base_path.read_text(encoding="utf-8") if base_path.exists() else "" combined = game_text + "\n\n" + self._BASE_SEPARATOR + base_text - Path("vocab.txt").write_text(combined, encoding="utf-8") + VOCAB_PATH.write_text(combined, encoding="utf-8") self._log("✅ vocab.txt saved (base terms from vocab_base.txt appended).") except Exception as exc: self._log(f"❌ Could not save vocab.txt: {exc}") @@ -3795,7 +3797,7 @@ class WorkflowTab(QWidget): self._log("⚠ No game folder set. Complete Step 0 first.") return - src = Path("vocab.txt") + src = VOCAB_PATH if not src.exists(): self._log("⚠ vocab.txt not found — save it in Step 3 first.") return diff --git a/modules/aquedi4.py b/modules/aquedi4.py index 7f1a43e..655835e 100644 --- a/modules/aquedi4.py +++ b/modules/aquedi4.py @@ -1,215 +1,217 @@ -# modules/aquedi4.py - -# Libraries -import json -import os -import re -import util.dazedwrap as dazedwrap -import threading -import time -import traceback -import tiktoken -from pathlib import Path -from colorama import Fore -from dotenv import load_dotenv -from retry import retry -from tqdm import tqdm -from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost -# from modules.json import translateSimpleKeyValueJSON -import tempfile - -# Boilerplate Globals -MODEL = os.getenv("model") -TIMEOUT = int(os.getenv("timeout")) -LANGUAGE = os.getenv("language").capitalize() -PROMPT = Path("prompt.txt").read_text(encoding="utf-8") -VOCAB = Path("vocab.txt").read_text(encoding="utf-8") -LOCK = threading.Lock() -ESTIMATE = "" -TOKENS = [0, 0] -MISMATCH = [] -FILENAME = None -PBAR = None -BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}" - -# Flag to control line break replacement -REPLACE_LINEBREAKS_WITH_PIPES = True - -LANGREGEX = r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+" -TRANSLATION_CONFIG = TranslationConfig( - model=MODEL, - language=LANGUAGE, - prompt=PROMPT, - vocab=VOCAB, - langRegex=LANGREGEX, - batchSize=int(os.getenv("batchsize", 10)), - maxHistory=10, - estimateMode=False -) -# End Boilerplate - -def handleAquedi4(filename, estimate): - """Main handler for Aquedi4 JSON files.""" - global ESTIMATE, TOKENS, FILENAME - ESTIMATE = estimate - FILENAME = filename - - try: - start = time.time() - translatedData = openFiles(filename) - - if not estimate: - # Write final result after translation is complete - with open("translated/" + filename, "w", encoding="utf-8", newline="\n") as outFile: - json.dump(translatedData[0], outFile, ensure_ascii=False, indent=4) - - end = time.time() - tqdm.write(getResultString(translatedData, end - start, filename)) - with LOCK: - TOKENS[0] += translatedData[1][0] - TOKENS[1] += translatedData[1][1] - - except Exception as e: - traceback.print_exc() - return "Fail" - - return getResultString(["", TOKENS, None], end - start, "TOTAL") - - -def openFiles(filename): - """Opens and parses the JSON file.""" - with open("files/" + filename, "r", encoding="UTF-8-sig") as f: - data = json.load(f) - translatedData = parseAquedi4JSON(data, filename) - return translatedData - - -def parseAquedi4JSON(data, filename): - """Parses and translates the simple key-value JSON.""" - totalTokens = [0, 0] - totalLines = 0 - totalLines = len(data) - global LOCK, PBAR - - stringList = [] - keyList = [] - if isinstance(data, dict) and all(isinstance(v, str) for v in data.values()): - for key, value in data.items(): - if re.search(LANGREGEX, key): - stringList.append(key) - keyList.append(key) - - with tqdm(total=len(stringList), bar_format=BAR_FORMAT, position=0, leave=False) as pbar: - pbar.desc = filename - PBAR = pbar - - # from modules.json import translateSimpleKeyValueJSON - - try: - if stringList: - result = translateSimpleKeyValueJSON(data, filename, stringList, keyList) - totalTokens[0] += result[0] - totalTokens[1] += result[1] - else: - pbar.write(f"Warning: {filename} is not in the expected simple key-value format. Skipping.") - except Exception as e: - traceback.print_exc() - return [data, totalTokens, e] - - return [data, totalTokens, None] - - -def translateSimpleKeyValueJSON(data, filename, stringList, keyList): - """ - Translate simple key-value JSON format where keys contain Japanese text - and values are placeholder strings like "TODO". - - Format example: - { - "\\r\\nスタビライザー": "TODO", - "\\r\\nプラグインなし": "TODO" - } - """ - global LOCK, ESTIMATE, FILENAME, PBAR, MISMATCH - tokens = [0, 0] - - preparedStringList = [] - if REPLACE_LINEBREAKS_WITH_PIPES: - for s in stringList: - preparedStringList.append(s.replace("\r\n", "|").replace("\n", "|")) - else: - preparedStringList = stringList - - # Translate all keys if any were found - if preparedStringList: - PBAR.total = len(preparedStringList) - PBAR.refresh() - - response = translateAI(preparedStringList, "Reply with the English Translation") - tokens[0] += response[1][0] - tokens[1] += response[1][1] - translatedList = response[0] - - # Check for mismatch - if len(preparedStringList) != len(translatedList): - with LOCK: - if FILENAME not in MISMATCH: - MISMATCH.append(FILENAME) - - # Pass 2: Update the values with translations - for i, original_key in enumerate(keyList): - if i < len(translatedList): - translatedText = translatedList[i] - - if REPLACE_LINEBREAKS_WITH_PIPES: - translatedText = translatedText.replace("|", "\r\n") - # Set the translated text as the value - data[original_key] = translatedText - - # Save progress after each translation - save_progress_json(data, filename) - - return tokens - -def save_progress_json(data, filename): - """Save current JSON translation progress.""" - try: - if ESTIMATE: return - os.makedirs("translated", exist_ok=True) - tmp_path = os.path.join("translated", f"{filename}.tmp") - final_path = os.path.join("translated", filename) - with open(tmp_path, "w", encoding="utf-8", newline="\n") as outFile: - json.dump(data, outFile, ensure_ascii=False, indent=4) - os.replace(tmp_path, final_path) - except Exception: - traceback.print_exc() - - -def getResultString(translatedData, translationTime, filename): - cost = calculateCost(translatedData[1][0], translatedData[1][1], MODEL) - totalTokenstring = ( - Fore.YELLOW + f"[Input: {translatedData[1][0]}]" - f"[Output: {translatedData[1][1]}]" - f"[Cost: ${cost:,.4f}]" - ) - timeString = Fore.BLUE + f"[{round(translationTime, 1)}s]" - - if translatedData[2] is None: - return f"{filename}: {totalTokenstring}{timeString}" + Fore.GREEN + " \u2713 " + Fore.RESET - else: - errorString = str(translatedData[2]) + Fore.RED - return f"{filename}: {totalTokenstring}{timeString}" + Fore.RED + " \u2717 " + errorString + Fore.RESET - -def translateAI(text, history, history_ctx=None): - """Legacy wrapper for the shared translation utility.""" - global PBAR, MISMATCH, FILENAME - TRANSLATION_CONFIG.estimateMode = bool(ESTIMATE) - return sharedtranslateAI( - text=text, - history=history, - config=TRANSLATION_CONFIG, - filename=FILENAME, - pbar=PBAR, - lock=LOCK, - mismatchList=MISMATCH +# modules/aquedi4.py + +# Libraries +import json +import os +import re +import util.dazedwrap as dazedwrap +import threading +import time +import traceback +import tiktoken +from pathlib import Path +from colorama import Fore +from dotenv import load_dotenv +from retry import retry +from tqdm import tqdm +from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost +# from modules.json import translateSimpleKeyValueJSON +import tempfile + +# Boilerplate Globals +MODEL = os.getenv("model") +TIMEOUT = int(os.getenv("timeout")) +LANGUAGE = os.getenv("language").capitalize() +from util.paths import PROMPT_PATH, VOCAB_PATH + +PROMPT = PROMPT_PATH.read_text(encoding="utf-8") +VOCAB = VOCAB_PATH.read_text(encoding="utf-8") +LOCK = threading.Lock() +ESTIMATE = "" +TOKENS = [0, 0] +MISMATCH = [] +FILENAME = None +PBAR = None +BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}" + +# Flag to control line break replacement +REPLACE_LINEBREAKS_WITH_PIPES = True + +LANGREGEX = r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+" +TRANSLATION_CONFIG = TranslationConfig( + model=MODEL, + language=LANGUAGE, + prompt=PROMPT, + vocab=VOCAB, + langRegex=LANGREGEX, + batchSize=int(os.getenv("batchsize", 10)), + maxHistory=10, + estimateMode=False +) +# End Boilerplate + +def handleAquedi4(filename, estimate): + """Main handler for Aquedi4 JSON files.""" + global ESTIMATE, TOKENS, FILENAME + ESTIMATE = estimate + FILENAME = filename + + try: + start = time.time() + translatedData = openFiles(filename) + + if not estimate: + # Write final result after translation is complete + with open("translated/" + filename, "w", encoding="utf-8", newline="\n") as outFile: + json.dump(translatedData[0], outFile, ensure_ascii=False, indent=4) + + end = time.time() + tqdm.write(getResultString(translatedData, end - start, filename)) + with LOCK: + TOKENS[0] += translatedData[1][0] + TOKENS[1] += translatedData[1][1] + + except Exception as e: + traceback.print_exc() + return "Fail" + + return getResultString(["", TOKENS, None], end - start, "TOTAL") + + +def openFiles(filename): + """Opens and parses the JSON file.""" + with open("files/" + filename, "r", encoding="UTF-8-sig") as f: + data = json.load(f) + translatedData = parseAquedi4JSON(data, filename) + return translatedData + + +def parseAquedi4JSON(data, filename): + """Parses and translates the simple key-value JSON.""" + totalTokens = [0, 0] + totalLines = 0 + totalLines = len(data) + global LOCK, PBAR + + stringList = [] + keyList = [] + if isinstance(data, dict) and all(isinstance(v, str) for v in data.values()): + for key, value in data.items(): + if re.search(LANGREGEX, key): + stringList.append(key) + keyList.append(key) + + with tqdm(total=len(stringList), bar_format=BAR_FORMAT, position=0, leave=False) as pbar: + pbar.desc = filename + PBAR = pbar + + # from modules.json import translateSimpleKeyValueJSON + + try: + if stringList: + result = translateSimpleKeyValueJSON(data, filename, stringList, keyList) + totalTokens[0] += result[0] + totalTokens[1] += result[1] + else: + pbar.write(f"Warning: {filename} is not in the expected simple key-value format. Skipping.") + except Exception as e: + traceback.print_exc() + return [data, totalTokens, e] + + return [data, totalTokens, None] + + +def translateSimpleKeyValueJSON(data, filename, stringList, keyList): + """ + Translate simple key-value JSON format where keys contain Japanese text + and values are placeholder strings like "TODO". + + Format example: + { + "\\r\\nスタビライザー": "TODO", + "\\r\\nプラグインなし": "TODO" + } + """ + global LOCK, ESTIMATE, FILENAME, PBAR, MISMATCH + tokens = [0, 0] + + preparedStringList = [] + if REPLACE_LINEBREAKS_WITH_PIPES: + for s in stringList: + preparedStringList.append(s.replace("\r\n", "|").replace("\n", "|")) + else: + preparedStringList = stringList + + # Translate all keys if any were found + if preparedStringList: + PBAR.total = len(preparedStringList) + PBAR.refresh() + + response = translateAI(preparedStringList, "Reply with the English Translation") + tokens[0] += response[1][0] + tokens[1] += response[1][1] + translatedList = response[0] + + # Check for mismatch + if len(preparedStringList) != len(translatedList): + with LOCK: + if FILENAME not in MISMATCH: + MISMATCH.append(FILENAME) + + # Pass 2: Update the values with translations + for i, original_key in enumerate(keyList): + if i < len(translatedList): + translatedText = translatedList[i] + + if REPLACE_LINEBREAKS_WITH_PIPES: + translatedText = translatedText.replace("|", "\r\n") + # Set the translated text as the value + data[original_key] = translatedText + + # Save progress after each translation + save_progress_json(data, filename) + + return tokens + +def save_progress_json(data, filename): + """Save current JSON translation progress.""" + try: + if ESTIMATE: return + os.makedirs("translated", exist_ok=True) + tmp_path = os.path.join("translated", f"{filename}.tmp") + final_path = os.path.join("translated", filename) + with open(tmp_path, "w", encoding="utf-8", newline="\n") as outFile: + json.dump(data, outFile, ensure_ascii=False, indent=4) + os.replace(tmp_path, final_path) + except Exception: + traceback.print_exc() + + +def getResultString(translatedData, translationTime, filename): + cost = calculateCost(translatedData[1][0], translatedData[1][1], MODEL) + totalTokenstring = ( + Fore.YELLOW + f"[Input: {translatedData[1][0]}]" + f"[Output: {translatedData[1][1]}]" + f"[Cost: ${cost:,.4f}]" + ) + timeString = Fore.BLUE + f"[{round(translationTime, 1)}s]" + + if translatedData[2] is None: + return f"{filename}: {totalTokenstring}{timeString}" + Fore.GREEN + " \u2713 " + Fore.RESET + else: + errorString = str(translatedData[2]) + Fore.RED + return f"{filename}: {totalTokenstring}{timeString}" + Fore.RED + " \u2717 " + errorString + Fore.RESET + +def translateAI(text, history, history_ctx=None): + """Legacy wrapper for the shared translation utility.""" + global PBAR, MISMATCH, FILENAME + TRANSLATION_CONFIG.estimateMode = bool(ESTIMATE) + return sharedtranslateAI( + text=text, + history=history, + config=TRANSLATION_CONFIG, + filename=FILENAME, + pbar=PBAR, + lock=LOCK, + mismatchList=MISMATCH ) \ No newline at end of file diff --git a/modules/csv.py b/modules/csv.py index 88535fd..81e36aa 100644 --- a/modules/csv.py +++ b/modules/csv.py @@ -1,458 +1,460 @@ -# Libraries -import json -import os -import re -import util.dazedwrap as dazedwrap -import threading -import time -import traceback -import tiktoken -import csv -from pathlib import Path -from colorama import Fore -from dotenv import load_dotenv -from retry import retry -from tqdm import tqdm -from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost -import tempfile - -# Globals -MODEL = os.getenv("model") -TIMEOUT = int(os.getenv("timeout")) -LANGUAGE = os.getenv("language").capitalize() -PROMPT = Path("prompt.txt").read_text(encoding="utf-8") -VOCAB = Path("vocab.txt").read_text(encoding="utf-8") -LOCK = threading.Lock() -WIDTH = int(os.getenv("width")) -LISTWIDTH = int(os.getenv("listWidth")) -NOTEWIDTH = int(os.getenv("noteWidth")) -MAXHISTORY = 10 -ESTIMATE = "" -TOKENS = [0, 0] -NAMESLIST = [] -NAMES = False # Output a list of all the character names found -BRFLAG = False # If the game uses
instead -FIXTEXTWRAP = True # Overwrites textwrap -IGNORETLTEXT = True # Ignores all translated text. -MISMATCH = [] # Lists files that thdata a mismatch error (Length of GPT list response is wrong) -FILENAME = None -BRACKETNAMES = False - -# CSV Configuration Settings (configurable via GUI) -CSV_DELIMITER = " " # CSV delimiter character (comma, semicolon, tab) -SOURCE_COLUMN = 2 # Which column has the source text to translate (0-indexed) -TARGET_COLUMN = 3 # Which column to write translations to -SPEAKER_COLUMN = 1 # Which column has speaker names (-1 = none) -SKIP_HEADER_ROW = False # Skip the first row (header) -USE_TARGET_IF_NOT_EMPTY = False # Use target column text if not empty (T++ style) -WRITE_TO_NEXT_COLUMN = False # Write to column after target instead of overwriting -PARSE_NAME_TAGS = False # Parse :name[] tags in text -PARSE_M_MARKERS = False # Parse \M markers in text -REMOVE_FURIGANA = True # Remove furigana annotations ＜＝＞ -SKIP_COMMENT_ROWS = False # Skip rows starting with 'comment' - -# Regex - Need to change this if you want to translate from/to other languages. Default is Japanese Regex -LANGREGEX = r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+" - -# Get pricing configuration based on the model -PRICING_CONFIG = getPricingConfig(MODEL) -INPUTAPICOST = PRICING_CONFIG["inputAPICost"] -OUTPUTAPICOST = PRICING_CONFIG["outputAPICost"] -BATCHSIZE = PRICING_CONFIG["batchSize"] -FREQUENCY_PENALTY = PRICING_CONFIG["frequencyPenalty"] - -# tqdm Globals -BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}" -POSITION = 0 -LEAVE = False -PBAR = None -ENCODING = "utf8" - -# Initialize Translation Config -TRANSLATION_CONFIG = TranslationConfig( - model=MODEL, - language=LANGUAGE, - prompt=PROMPT, - vocab=VOCAB, - langRegex=LANGREGEX, - batchSize=BATCHSIZE, - maxHistory=MAXHISTORY, - estimateMode=False # Will be set dynamically based on ESTIMATE -) -LEAVE = False - -def handleCSV(filename, estimate): - global ESTIMATE, TOKENS, FILENAME - ESTIMATE = estimate - FILENAME = filename - - if not ESTIMATE: - with open("translated/" + filename, "w+t", newline="", encoding=ENCODING, errors="xmlcharrefreplace") as writeFile: - # Translate - start = time.time() - translatedData = openFiles(filename, writeFile) - - # Print Result - end = time.time() - tqdm.write(getResultString(translatedData, end - start, filename)) - with LOCK: - TOKENS[0] += translatedData[1][0] - TOKENS[1] += translatedData[1][1] - else: - # Translate - start = time.time() - translatedData = openFilesEstimate(filename) - - # Print Result - end = time.time() - tqdm.write(getResultString(translatedData, end - start, filename)) - with LOCK: - TOKENS[0] += translatedData[1][0] - TOKENS[1] += translatedData[1][1] - - # Print Total - totalString = getResultString(["", TOKENS, None], end - start, "TOTAL") - - # Print any errors on maps - if len(MISMATCH) > 0: - return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET - else: - return totalString - - -def openFiles(filename, writeFile): - with open("files/" + filename, "r", encoding=ENCODING) as readFile, writeFile: - translatedData = parseCSV(readFile, writeFile, filename) - - return translatedData - - -def openFilesEstimate(filename): - with open("files/" + filename, "r", encoding="utf8") as readFile: - translatedData = parseCSV(readFile, "", filename) - - return translatedData - - -def getResultString(translatedData, translationTime, filename): - # File Print String - cost = calculateCost(translatedData[1][0], translatedData[1][1], MODEL) - totalTokenstring = ( - Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]" - "[Output: " - + str(translatedData[1][1]) - + "]" "[Cost: ${:,.4f}".format(cost) - + "]" - ) - timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]" - - if translatedData[2] is None: - # Success - return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET - else: - # Fail - try: - raise translatedData[2] - except Exception as e: - traceback.print_exc() - errorString = str(e) + Fore.RED - return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET - - -def parseCSV(readFile, writeFile, filename): - totalTokens = [0, 0] - totalLines = 0 - global LOCK - - # Get total for progress bar - totalLines = len(readFile.readlines()) - readFile.seek(0) - - reader = csv.reader(readFile, delimiter=CSV_DELIMITER) - if not ESTIMATE: - writer = csv.writer( - writeFile, - delimiter=CSV_DELIMITER, - ) - else: - writer = "" - - with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar: - pbar.desc = filename - pbar.total = totalLines - - # Grab All Rows - data = [] - for row in reader: - data.append(row) - - try: - response = translateCSV(data, pbar, writeFile, writer, filename, None) - totalTokens[0] = response[0] - totalTokens[1] = response[1] - except Exception: - traceback.print_exc() - return [data, totalTokens, None] - - -def flush_progress_csv(writeFile, writer, rows): - """Flush current CSV progress to the already-open output file (Windows-safe).""" - try: - if ESTIMATE or writeFile is None: - return - with LOCK: - writeFile.seek(0) - # Recreate writer at current position to avoid state issues - tmp_writer = csv.writer(writeFile, delimiter=CSV_DELIMITER) - tmp_writer.writerows(rows) - writeFile.truncate() - writeFile.flush() - os.fsync(writeFile.fileno()) - except Exception: - traceback.print_exc() - - -def translateCSV(data, pbar, writeFile, writer, filename, translatedList): - """ - Unified CSV translation function using configurable settings. - - Uses global settings: - - SOURCE_COLUMN: column index for source text - - TARGET_COLUMN: column index to write translations - - SPEAKER_COLUMN: column index for speaker names (-1 = none) - - SKIP_HEADER_ROW: whether to skip first row - - USE_TARGET_IF_NOT_EMPTY: use existing target text if present (T++ style) - - WRITE_TO_NEXT_COLUMN: write to column after target - - PARSE_NAME_TAGS: parse :name[] tags - - PARSE_M_MARKERS: parse \\M markers - - REMOVE_FURIGANA: remove furigana - - SKIP_COMMENT_ROWS: skip rows with 'comment' in first column - """ - global LOCK, ESTIMATE, PBAR - PBAR = pbar - translatedText = "" - totalTokens = [0, 0] - i = 0 - stringList = [] - - try: - # Translate - while i < len(data): - # Skip header row if configured - if SKIP_HEADER_ROW and i == 0: - i += 1 - continue - - # Skip comment rows if configured - if SKIP_COMMENT_ROWS and len(data[i]) > 0 and 'comment' in str(data[i][0]).lower(): - i += 1 - continue - - # Check if row has enough columns - if len(data[i]) <= SOURCE_COLUMN: - i += 1 - continue - - # Get source text - jaString = "" - speaker = "" - - # If USE_TARGET_IF_NOT_EMPTY is enabled (T++ style), check target column first - if USE_TARGET_IF_NOT_EMPTY and len(data[i]) > TARGET_COLUMN and data[i][TARGET_COLUMN]: - jaString = data[i][TARGET_COLUMN] - else: - jaString = data[i][SOURCE_COLUMN] if data[i][SOURCE_COLUMN] else "" - - # Skip empty strings - if not jaString: - i += 1 - continue - - # Handle speaker column if configured - if SPEAKER_COLUMN >= 0 and len(data[i]) > SPEAKER_COLUMN and data[i][SPEAKER_COLUMN]: - speakerResponse = getSpeaker(data[i][SPEAKER_COLUMN]) - totalTokens[0] += speakerResponse[1][0] - totalTokens[1] += speakerResponse[1][1] - speaker = speakerResponse[0] - data[i][SPEAKER_COLUMN] = speaker - - # Parse :name[] tags if configured - if PARSE_NAME_TAGS and ':name' in jaString: - match = re.search(r":name\[([^\]]+?)\]\n([\w\W]*)", jaString) - if match: - # Translate speaker name - response = getSpeaker(match.group(1)) - speaker = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - data[i][SOURCE_COLUMN] = data[i][SOURCE_COLUMN].replace(match.group(1), speaker) - - # Extract text portion - jaString = match.group(2) - # Remove voice markers - voMatch = re.search(r"\\[vfF]+\[.+]", jaString) - if voMatch: - jaString = jaString.replace(voMatch.group(0), "") - - # Parse \M markers if configured - if PARSE_M_MARKERS and '\\M' in jaString: - match = re.search(r"\\M.+\n([\w\W]*)", jaString) - if match: - jaString = match.group(1) - voMatch = re.search(r"\\[vfF]+\[.+]", jaString) - if voMatch: - jaString = jaString.replace(voMatch.group(0), "") - - # Remove furigana if configured - if REMOVE_FURIGANA: - jaString = re.sub(r"＜(.*)＝.*＞", r"\1", jaString) - - # Store original for replacement - ojaString = jaString - - # Remove textwrap - jaString = jaString.replace("\\n", " ") - jaString = jaString.replace("\n", " ") - - # Pass 1: Collect strings - if not translatedList: - if speaker: - stringList.append(f"[{speaker}]: {jaString}") - else: - stringList.append(jaString) - - # Pass 2: Apply translations - else: - # Grab and pop translation - translatedText = translatedList[0] - translatedList.pop(0) - - # Remove speaker prefix from translation if present - if speaker: - translatedText = re.sub(r"^\[?(.+?)\]?\s?[|:]\s?", "", translatedText) - - # Add wordwrap - translatedText = dazedwrap.wrapText(translatedText, WIDTH) - translatedText = translatedText.replace("\n", "\\n") - - # Determine target column - actual_target = TARGET_COLUMN + 1 if WRITE_TO_NEXT_COLUMN else TARGET_COLUMN - - # Ensure row has enough columns - while len(data[i]) <= actual_target: - data[i].append("") - - # Set data - if PARSE_NAME_TAGS or PARSE_M_MARKERS: - # Replace original text portion - data[i][actual_target] = data[i][actual_target].replace(ojaString, translatedText) if data[i][actual_target] else translatedText - else: - data[i][actual_target] = translatedText - - flush_progress_csv(writeFile, writer, data) - - # Iterate - i += 1 - - # EOF - Process collected strings - if len(stringList) > 0: - # Set Progress - pbar.total = len(stringList) - pbar.refresh() - - # Translate - response = translateAI(stringList, "") - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - translatedList = response[0] - - # Set Strings - if len(stringList) == len(translatedList): - translateCSV(data, pbar, writeFile, writer, filename, translatedList) - # Mismatch - else: - with LOCK: - if filename not in MISMATCH: - MISMATCH.append(filename) - - # Write all Data - with LOCK: - if not ESTIMATE: - for row in data: - writer.writerow(row) - flush_progress_csv(writeFile, writer, data) - - except Exception: - traceback.print_exc() - - # Write all Data - with LOCK: - if not ESTIMATE: - for row in data: - writer.writerow(row) - flush_progress_csv(writeFile, writer, data) - return totalTokens - - return totalTokens - -# Save some money and enter the character before translation -def getSpeaker(speaker): - match speaker: - case "ファイン": - return ["Fine", [0, 0]] - case "": - return ["", [0, 0]] - case _: - # Find Speaker - for i in range(len(NAMESLIST)): - if speaker == NAMESLIST[i][0]: - return [NAMESLIST[i][1], [0, 0]] - - if speaker == "？？？": - return ["???", [0, 0]] - - # Translate and Store Speaker - response = translateAI( - f"{speaker}", - "Reply with the " + LANGUAGE + " translation of the NPC name.", - False, - ) - response[0] = response[0].title() - response[0] = response[0].replace("'S", "'s") - response[0] = response[0].replace("Speaker: ", "") - - # Retry if name doesn't translate for some reason - if re.search(r"([a-zA-Z？?])", response[0]) == None: - response = translateAI( - f"{speaker}", - "Reply with the " + LANGUAGE + " translation of the NPC name.", - False, - ) - response[0] = response[0].title() - response[0] = response[0].replace("'S", "'s") - - speakerList = [speaker, response[0]] - NAMESLIST.append(speakerList) - return response - return [speaker, [0, 0]] - -def translateAI(text, history, history_ctx=None): - """ - Legacy wrapper function for the new shared translation utility. - This maintains compatibility with existing code while using the new shared implementation. - """ - global PBAR, MISMATCH, FILENAME - - # Update config estimate mode based on global ESTIMATE - TRANSLATION_CONFIG.estimateMode = bool(ESTIMATE) - - # Call the new shared translation function - return sharedtranslateAI( - text=text, - history=history, - config=TRANSLATION_CONFIG, - filename=FILENAME, - pbar=PBAR, - lock=LOCK, - mismatchList=MISMATCH - ) +# Libraries +import json +import os +import re +import util.dazedwrap as dazedwrap +import threading +import time +import traceback +import tiktoken +import csv +from pathlib import Path +from colorama import Fore +from dotenv import load_dotenv +from retry import retry +from tqdm import tqdm +from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost +import tempfile + +# Globals +MODEL = os.getenv("model") +TIMEOUT = int(os.getenv("timeout")) +LANGUAGE = os.getenv("language").capitalize() +from util.paths import PROMPT_PATH, VOCAB_PATH + +PROMPT = PROMPT_PATH.read_text(encoding="utf-8") +VOCAB = VOCAB_PATH.read_text(encoding="utf-8") +LOCK = threading.Lock() +WIDTH = int(os.getenv("width")) +LISTWIDTH = int(os.getenv("listWidth")) +NOTEWIDTH = int(os.getenv("noteWidth")) +MAXHISTORY = 10 +ESTIMATE = "" +TOKENS = [0, 0] +NAMESLIST = [] +NAMES = False # Output a list of all the character names found +BRFLAG = False # If the game uses
instead +FIXTEXTWRAP = True # Overwrites textwrap +IGNORETLTEXT = True # Ignores all translated text. +MISMATCH = [] # Lists files that thdata a mismatch error (Length of GPT list response is wrong) +FILENAME = None +BRACKETNAMES = False + +# CSV Configuration Settings (configurable via GUI) +CSV_DELIMITER = " " # CSV delimiter character (comma, semicolon, tab) +SOURCE_COLUMN = 2 # Which column has the source text to translate (0-indexed) +TARGET_COLUMN = 3 # Which column to write translations to +SPEAKER_COLUMN = 1 # Which column has speaker names (-1 = none) +SKIP_HEADER_ROW = False # Skip the first row (header) +USE_TARGET_IF_NOT_EMPTY = False # Use target column text if not empty (T++ style) +WRITE_TO_NEXT_COLUMN = False # Write to column after target instead of overwriting +PARSE_NAME_TAGS = False # Parse :name[] tags in text +PARSE_M_MARKERS = False # Parse \M markers in text +REMOVE_FURIGANA = True # Remove furigana annotations ＜＝＞ +SKIP_COMMENT_ROWS = False # Skip rows starting with 'comment' + +# Regex - Need to change this if you want to translate from/to other languages. Default is Japanese Regex +LANGREGEX = r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+" + +# Get pricing configuration based on the model +PRICING_CONFIG = getPricingConfig(MODEL) +INPUTAPICOST = PRICING_CONFIG["inputAPICost"] +OUTPUTAPICOST = PRICING_CONFIG["outputAPICost"] +BATCHSIZE = PRICING_CONFIG["batchSize"] +FREQUENCY_PENALTY = PRICING_CONFIG["frequencyPenalty"] + +# tqdm Globals +BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}" +POSITION = 0 +LEAVE = False +PBAR = None +ENCODING = "utf8" + +# Initialize Translation Config +TRANSLATION_CONFIG = TranslationConfig( + model=MODEL, + language=LANGUAGE, + prompt=PROMPT, + vocab=VOCAB, + langRegex=LANGREGEX, + batchSize=BATCHSIZE, + maxHistory=MAXHISTORY, + estimateMode=False # Will be set dynamically based on ESTIMATE +) +LEAVE = False + +def handleCSV(filename, estimate): + global ESTIMATE, TOKENS, FILENAME + ESTIMATE = estimate + FILENAME = filename + + if not ESTIMATE: + with open("translated/" + filename, "w+t", newline="", encoding=ENCODING, errors="xmlcharrefreplace") as writeFile: + # Translate + start = time.time() + translatedData = openFiles(filename, writeFile) + + # Print Result + end = time.time() + tqdm.write(getResultString(translatedData, end - start, filename)) + with LOCK: + TOKENS[0] += translatedData[1][0] + TOKENS[1] += translatedData[1][1] + else: + # Translate + start = time.time() + translatedData = openFilesEstimate(filename) + + # Print Result + end = time.time() + tqdm.write(getResultString(translatedData, end - start, filename)) + with LOCK: + TOKENS[0] += translatedData[1][0] + TOKENS[1] += translatedData[1][1] + + # Print Total + totalString = getResultString(["", TOKENS, None], end - start, "TOTAL") + + # Print any errors on maps + if len(MISMATCH) > 0: + return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET + else: + return totalString + + +def openFiles(filename, writeFile): + with open("files/" + filename, "r", encoding=ENCODING) as readFile, writeFile: + translatedData = parseCSV(readFile, writeFile, filename) + + return translatedData + + +def openFilesEstimate(filename): + with open("files/" + filename, "r", encoding="utf8") as readFile: + translatedData = parseCSV(readFile, "", filename) + + return translatedData + + +def getResultString(translatedData, translationTime, filename): + # File Print String + cost = calculateCost(translatedData[1][0], translatedData[1][1], MODEL) + totalTokenstring = ( + Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]" + "[Output: " + + str(translatedData[1][1]) + + "]" "[Cost: ${:,.4f}".format(cost) + + "]" + ) + timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]" + + if translatedData[2] is None: + # Success + return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET + else: + # Fail + try: + raise translatedData[2] + except Exception as e: + traceback.print_exc() + errorString = str(e) + Fore.RED + return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET + + +def parseCSV(readFile, writeFile, filename): + totalTokens = [0, 0] + totalLines = 0 + global LOCK + + # Get total for progress bar + totalLines = len(readFile.readlines()) + readFile.seek(0) + + reader = csv.reader(readFile, delimiter=CSV_DELIMITER) + if not ESTIMATE: + writer = csv.writer( + writeFile, + delimiter=CSV_DELIMITER, + ) + else: + writer = "" + + with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar: + pbar.desc = filename + pbar.total = totalLines + + # Grab All Rows + data = [] + for row in reader: + data.append(row) + + try: + response = translateCSV(data, pbar, writeFile, writer, filename, None) + totalTokens[0] = response[0] + totalTokens[1] = response[1] + except Exception: + traceback.print_exc() + return [data, totalTokens, None] + + +def flush_progress_csv(writeFile, writer, rows): + """Flush current CSV progress to the already-open output file (Windows-safe).""" + try: + if ESTIMATE or writeFile is None: + return + with LOCK: + writeFile.seek(0) + # Recreate writer at current position to avoid state issues + tmp_writer = csv.writer(writeFile, delimiter=CSV_DELIMITER) + tmp_writer.writerows(rows) + writeFile.truncate() + writeFile.flush() + os.fsync(writeFile.fileno()) + except Exception: + traceback.print_exc() + + +def translateCSV(data, pbar, writeFile, writer, filename, translatedList): + """ + Unified CSV translation function using configurable settings. + + Uses global settings: + - SOURCE_COLUMN: column index for source text + - TARGET_COLUMN: column index to write translations + - SPEAKER_COLUMN: column index for speaker names (-1 = none) + - SKIP_HEADER_ROW: whether to skip first row + - USE_TARGET_IF_NOT_EMPTY: use existing target text if present (T++ style) + - WRITE_TO_NEXT_COLUMN: write to column after target + - PARSE_NAME_TAGS: parse :name[] tags + - PARSE_M_MARKERS: parse \\M markers + - REMOVE_FURIGANA: remove furigana + - SKIP_COMMENT_ROWS: skip rows with 'comment' in first column + """ + global LOCK, ESTIMATE, PBAR + PBAR = pbar + translatedText = "" + totalTokens = [0, 0] + i = 0 + stringList = [] + + try: + # Translate + while i < len(data): + # Skip header row if configured + if SKIP_HEADER_ROW and i == 0: + i += 1 + continue + + # Skip comment rows if configured + if SKIP_COMMENT_ROWS and len(data[i]) > 0 and 'comment' in str(data[i][0]).lower(): + i += 1 + continue + + # Check if row has enough columns + if len(data[i]) <= SOURCE_COLUMN: + i += 1 + continue + + # Get source text + jaString = "" + speaker = "" + + # If USE_TARGET_IF_NOT_EMPTY is enabled (T++ style), check target column first + if USE_TARGET_IF_NOT_EMPTY and len(data[i]) > TARGET_COLUMN and data[i][TARGET_COLUMN]: + jaString = data[i][TARGET_COLUMN] + else: + jaString = data[i][SOURCE_COLUMN] if data[i][SOURCE_COLUMN] else "" + + # Skip empty strings + if not jaString: + i += 1 + continue + + # Handle speaker column if configured + if SPEAKER_COLUMN >= 0 and len(data[i]) > SPEAKER_COLUMN and data[i][SPEAKER_COLUMN]: + speakerResponse = getSpeaker(data[i][SPEAKER_COLUMN]) + totalTokens[0] += speakerResponse[1][0] + totalTokens[1] += speakerResponse[1][1] + speaker = speakerResponse[0] + data[i][SPEAKER_COLUMN] = speaker + + # Parse :name[] tags if configured + if PARSE_NAME_TAGS and ':name' in jaString: + match = re.search(r":name\[([^\]]+?)\]\n([\w\W]*)", jaString) + if match: + # Translate speaker name + response = getSpeaker(match.group(1)) + speaker = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + data[i][SOURCE_COLUMN] = data[i][SOURCE_COLUMN].replace(match.group(1), speaker) + + # Extract text portion + jaString = match.group(2) + # Remove voice markers + voMatch = re.search(r"\\[vfF]+\[.+]", jaString) + if voMatch: + jaString = jaString.replace(voMatch.group(0), "") + + # Parse \M markers if configured + if PARSE_M_MARKERS and '\\M' in jaString: + match = re.search(r"\\M.+\n([\w\W]*)", jaString) + if match: + jaString = match.group(1) + voMatch = re.search(r"\\[vfF]+\[.+]", jaString) + if voMatch: + jaString = jaString.replace(voMatch.group(0), "") + + # Remove furigana if configured + if REMOVE_FURIGANA: + jaString = re.sub(r"＜(.*)＝.*＞", r"\1", jaString) + + # Store original for replacement + ojaString = jaString + + # Remove textwrap + jaString = jaString.replace("\\n", " ") + jaString = jaString.replace("\n", " ") + + # Pass 1: Collect strings + if not translatedList: + if speaker: + stringList.append(f"[{speaker}]: {jaString}") + else: + stringList.append(jaString) + + # Pass 2: Apply translations + else: + # Grab and pop translation + translatedText = translatedList[0] + translatedList.pop(0) + + # Remove speaker prefix from translation if present + if speaker: + translatedText = re.sub(r"^\[?(.+?)\]?\s?[|:]\s?", "", translatedText) + + # Add wordwrap + translatedText = dazedwrap.wrapText(translatedText, WIDTH) + translatedText = translatedText.replace("\n", "\\n") + + # Determine target column + actual_target = TARGET_COLUMN + 1 if WRITE_TO_NEXT_COLUMN else TARGET_COLUMN + + # Ensure row has enough columns + while len(data[i]) <= actual_target: + data[i].append("") + + # Set data + if PARSE_NAME_TAGS or PARSE_M_MARKERS: + # Replace original text portion + data[i][actual_target] = data[i][actual_target].replace(ojaString, translatedText) if data[i][actual_target] else translatedText + else: + data[i][actual_target] = translatedText + + flush_progress_csv(writeFile, writer, data) + + # Iterate + i += 1 + + # EOF - Process collected strings + if len(stringList) > 0: + # Set Progress + pbar.total = len(stringList) + pbar.refresh() + + # Translate + response = translateAI(stringList, "") + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + translatedList = response[0] + + # Set Strings + if len(stringList) == len(translatedList): + translateCSV(data, pbar, writeFile, writer, filename, translatedList) + # Mismatch + else: + with LOCK: + if filename not in MISMATCH: + MISMATCH.append(filename) + + # Write all Data + with LOCK: + if not ESTIMATE: + for row in data: + writer.writerow(row) + flush_progress_csv(writeFile, writer, data) + + except Exception: + traceback.print_exc() + + # Write all Data + with LOCK: + if not ESTIMATE: + for row in data: + writer.writerow(row) + flush_progress_csv(writeFile, writer, data) + return totalTokens + + return totalTokens + +# Save some money and enter the character before translation +def getSpeaker(speaker): + match speaker: + case "ファイン": + return ["Fine", [0, 0]] + case "": + return ["", [0, 0]] + case _: + # Find Speaker + for i in range(len(NAMESLIST)): + if speaker == NAMESLIST[i][0]: + return [NAMESLIST[i][1], [0, 0]] + + if speaker == "？？？": + return ["???", [0, 0]] + + # Translate and Store Speaker + response = translateAI( + f"{speaker}", + "Reply with the " + LANGUAGE + " translation of the NPC name.", + False, + ) + response[0] = response[0].title() + response[0] = response[0].replace("'S", "'s") + response[0] = response[0].replace("Speaker: ", "") + + # Retry if name doesn't translate for some reason + if re.search(r"([a-zA-Z？?])", response[0]) == None: + response = translateAI( + f"{speaker}", + "Reply with the " + LANGUAGE + " translation of the NPC name.", + False, + ) + response[0] = response[0].title() + response[0] = response[0].replace("'S", "'s") + + speakerList = [speaker, response[0]] + NAMESLIST.append(speakerList) + return response + return [speaker, [0, 0]] + +def translateAI(text, history, history_ctx=None): + """ + Legacy wrapper function for the new shared translation utility. + This maintains compatibility with existing code while using the new shared implementation. + """ + global PBAR, MISMATCH, FILENAME + + # Update config estimate mode based on global ESTIMATE + TRANSLATION_CONFIG.estimateMode = bool(ESTIMATE) + + # Call the new shared translation function + return sharedtranslateAI( + text=text, + history=history, + config=TRANSLATION_CONFIG, + filename=FILENAME, + pbar=PBAR, + lock=LOCK, + mismatchList=MISMATCH + ) diff --git a/modules/images.py b/modules/images.py index e6fd04b..65af059 100644 --- a/modules/images.py +++ b/modules/images.py @@ -1,439 +1,441 @@ -# Libraries -from PIL import Image, ImageDraw, ImageFont -import json -import os -import re -import threading -import time -import traceback -import tiktoken -from pathlib import Path -from colorama import Fore -from dotenv import load_dotenv -from retry import retry -from tqdm import tqdm -from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost - -# Globals -MODEL = os.getenv("model") -TIMEOUT = int(os.getenv("timeout")) -LANGUAGE = os.getenv("language").capitalize() -PROMPT = Path("prompt.txt").read_text(encoding="utf-8") -VOCAB = Path("vocab.txt").read_text(encoding="utf-8") -LOCK = threading.Lock() -PBAR = None -WIDTH = int(os.getenv("width")) -LISTWIDTH = int(os.getenv("listWidth")) -NOTEWIDTH = int(os.getenv("noteWidth")) -MAXHISTORY = 10 -ESTIMATE = "" -TOKENS = [0, 0] -NAMESLIST = [] -MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong) -FILENAME = None - -# Regex - Need to change this if you want to translate from/to other languages. Default is Japanese Regex -LANGREGEX = r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+" - -# Get pricing configuration based on the model -PRICING_CONFIG = getPricingConfig(MODEL) -INPUTAPICOST = PRICING_CONFIG["inputAPICost"] -OUTPUTAPICOST = PRICING_CONFIG["outputAPICost"] -BATCHSIZE = PRICING_CONFIG["batchSize"] -FREQUENCY_PENALTY = PRICING_CONFIG["frequencyPenalty"] - -# Initialize Translation Config -TRANSLATION_CONFIG = TranslationConfig( - model=MODEL, - language=LANGUAGE, - prompt=PROMPT, - vocab=VOCAB, - langRegex=LANGREGEX, - batchSize=BATCHSIZE, - maxHistory=MAXHISTORY, - estimateMode=False # Will be set dynamically based on ESTIMATE -) -LEAVE = False - -# tqdm Globals -BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}" -POSITION = 0 -LEAVE = False - - -def handleImages(folderName, estimate): - global ESTIMATE, TOKENS, FILENAME - ESTIMATE = estimate - FILENAME = folderName - start = time.time() - - # Translate Strings - translatedData = openFiles(f"files/{folderName}") - - # Custom Names - # customList = [[], []] - # customList = processImagesDir("Custom", customList) - - # Write TL To Images - try: - translatedList, originalList, dimensionsList = translatedData[0] - for i in range(len(translatedList)): - try: - # Create image from string - image = stringToImageOutline(translatedList[i], dimensionsList[i][0], dimensionsList[i][1]) - # Save image using the corresponding original filename - image.save(rf"translated/{folderName}/{originalList[i]}.png", quality=100) - except Exception as e: - # Log error if image saving fails - PBAR.write(f"Error processing {translatedList[i]}: {str(e)}") - except IndexError: - PBAR.write("Translated data is incomplete. Please check your input.") - - # Print File - end = time.time() - tqdm.write(getResultString(translatedData, end - start, folderName)) - with LOCK: - TOKENS[0] += translatedData[1][0] - TOKENS[1] += translatedData[1][1] - - # Print Total - totalString = getResultString(["", TOKENS, None], end - start, "TOTAL") - - # Print any errors on maps - if len(MISMATCH) > 0: - return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET - else: - return totalString - - -def openFiles(folderName): - global PBAR - - if os.path.isdir(folderName): - imageList = [[], [], []] - imageList = processImagesDir(folderName, imageList) - - # Start Translation - with tqdm( - bar_format=BAR_FORMAT, - position=POSITION, - leave=LEAVE, - desc=folderName, - total=len(imageList[0]), - ) as PBAR: - translatedData = translateImages(imageList) - translatedData = [ - [translatedData[0], imageList[2], imageList[1]], - translatedData[1], - translatedData[2], - ] - - return translatedData - else: - print("The provided directory path does not exist.") - - -def getResultString(translatedData, translationTime, filename): - # File Print String - cost = calculateCost(translatedData[1][0], translatedData[1][1], MODEL) - totalTokenstring = ( - Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]" - "[Output: " - + str(translatedData[1][1]) - + "]" "[Cost: ${:,.4f}".format(cost) - + "]" - ) - timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]" - - if translatedData[2] is None: - # Success - return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET - else: - # Fail - try: - raise translatedData[2] - except Exception as e: - traceback.print_exc() - errorString = str(e) + Fore.RED - return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET - - -def getFontSize(text, image_width, image_height, font_path): - # Start with a high font size and keep reducing it until the text fits within the image bounds - font_size = min(image_width, image_height) - - while font_size > 0: - font = ImageFont.truetype(font_path, font_size) - text_bbox = ImageDraw.Draw(Image.new("RGB", (1, 1))).textbbox((0, 0), text, font=font) - text_width = text_bbox[2] - text_bbox[0] - text_height = text_bbox[3] - text_bbox[1] - - if text_width <= image_width and text_height <= image_height: - return font_size - font_size -= 1 - - return font_size - - -def stringToImage(text, width, height, font_path="fonts/TsunagiGothic.ttf", scale_factor=4): - # Increase the resolution - scaled_width = int(width * scale_factor) - scaled_height = int(height * scale_factor) - - # Find the appropriate font size for the scaled up image - font_size = getFontSize(text, scaled_width, scaled_height, font_path) - if font_size == 0: - raise ValueError("Text is too long to fit in the supplied dimensions.") - - # Create a new image with the scaled width and height and a transparent background - image = Image.new("RGBA", (scaled_width, scaled_height), (255, 255, 255, 0)) - - # Create a drawing context - draw = ImageDraw.Draw(image) - - # Load the appropriate font - font = ImageFont.truetype(font_path, font_size) - - # Calculate the size of the text to center it - text_bbox = draw.textbbox((0, 0), text, font=font) - text_width = text_bbox[2] - text_bbox[0] - text_height = text_bbox[3] - text_bbox[1] + 20 - x = 0 - - x = (scaled_width - text_width) // 2 - y = (scaled_height - text_height) // 2 - - # Draw the text on the image - draw.text((x, y), text, font=font, fill=(255, 255, 255, 255)) - - # Resize back to the original dimensions to get a clearer text rendering - image = image.resize( - (width, height), - Image.LANCZOS, - ) - - return image - - -from PIL import Image, ImageDraw, ImageFont - - -def stringToImageOutline(text, width, height, font_path="fonts/TsunagiGothic.ttf", scale_factor=4): - # Outline - outline_color = (255, 255, 255, 255) - text_color = (0, 0, 0, 255) - outline_thickness = 4 - - # Increase the resolution - scaled_width = int(width * scale_factor) - scaled_height = int(height * scale_factor) - - # Find the appropriate font size for the scaled up image - font_size = getFontSize(text, scaled_width, scaled_height, font_path) - if font_size == 0: - raise ValueError("Text is too long to fit in the supplied dimensions.") - - # Create a new image with the scaled width and height and a transparent background - image = Image.new("RGBA", (scaled_width, scaled_height), (255, 255, 255, 0)) - - # Create a drawing context - draw = ImageDraw.Draw(image) - - # Load the appropriate font - font = ImageFont.truetype(font_path, font_size) - - # Calculate the size of the text to center it - text_bbox = draw.textbbox((0, 0), text, font=font) - text_width = text_bbox[2] - text_bbox[0] - text_height = text_bbox[3] - text_bbox[1] + 20 - x = (scaled_width - text_width) // 2 - y = (scaled_height - text_height) // 2 - - # Draw the text outline by applying the text multiple times with small offsets - for dx in range(-outline_thickness, outline_thickness + 1): - for dy in range(-outline_thickness, outline_thickness + 1): - if dx != 0 or dy != 0: - draw.text((x + dx, y + dy), text, font=font, fill=outline_color) - - # Draw the main text - draw.text((x, y), text, font=font, fill=text_color) - - # Resize back to the original dimensions to get a clearer text rendering - image = image.resize((width, height), Image.LANCZOS) - - return image - - -def stringToImageBox(text, width, height, font_path="fonts/TsunagiGothic.ttf", scale_factor=4): - # Increase the resolution - scaled_width = int(width * scale_factor) - scaled_height = int(height * scale_factor) - - # Padding around the text - padding = 10 - - # Calculate the dimensions available for text placement - available_width = scaled_width - 2 * padding - available_height = scaled_height - 2 * padding - - # Determine the best font size to fit within the available dimensions - font_size = getFontSize(text, available_width, available_height, font_path) - if font_size <= 0: - raise ValueError("Text is too long to fit in the supplied dimensions.") - - # Create a new image with increased resolution - image = Image.new("RGBA", (scaled_width, scaled_height), (255, 255, 255, 0)) - draw = ImageDraw.Draw(image) - - # Load the calculated font - font = ImageFont.truetype(font_path, font_size) - - # Calculate the size and bounding box of the text - text_bbox = draw.textbbox((0, 0), text, font=font) - text_width = text_bbox[2] - text_bbox[0] - text_height = text_bbox[3] - text_bbox[1] + 20 - - # Determine centered position for the text while considering padding - # Additional adjustment ensures text appears centrally aligned - x = (scaled_width - text_width) // 2 - y = (scaled_height - text_height) // 2 - - # Draw a black box with a white outline that fits the image dimensions precisely - draw.rectangle([0, 0, scaled_width - 1, scaled_height - 1], outline=(255, 255, 255, 255), width=1) - - # Fill the inside box with black color - draw.rectangle([1, 1, scaled_width - 2, scaled_height - 2], fill=(0, 0, 0, 255)) - - # Render the text within the image - draw.text((x, y), text, font=font, fill=(255, 255, 255, 255)) - - # Shrink the image back to original dimensions with high-quality interpolation - image = image.resize( - (width, height), - Image.LANCZOS, - ) - - return image - - -def getImageDimensions(file_path): - try: - with Image.open(file_path) as img: - width, height = img.size - return width, height - except Exception as e: - print(f"Error reading {file_path}: {e}") - return None, None - - -def processImagesDir(directory_path, imageList): - for file_name in os.listdir(directory_path): - # .png and Japanese - if ".png" in file_name: - file_path = os.path.join(directory_path, file_name) - if os.path.isfile(file_path): - # Check if the file is an image - try: - width, height = getImageDimensions(file_path) - if width is not None and height is not None: - placeholders = { - ".png": "", - } - for target, replacement in placeholders.items(): - file_name = file_name.replace(target, replacement) - match = re.search(r"[\[【].+?[\]】](.*)", file_name) - if match: - text = match.group(1) - else: - text = file_name - imageList[0].append(text) - imageList[1].append([width, height]) - imageList[2].append(file_name) - except Exception as e: - print(f"Error processing {file_name}: {e}") - - if ".txt" in file_name: - try: - with open(f"{directory_path}/{file_name}", "r", encoding="utf8") as file: - for line in file: - line = line.strip() - line = line.replace(":", "：") - line = line.replace("/", "／") - line = line.replace("?", "？") - imageList[0].append(line) # Using strip() to remove any extra newlines or spaces - imageList[1].append([104, 15]) - except FileNotFoundError: - print(f"The file at {file_path} was not found.") - except IOError: - print(f"An error occurred while reading the file at {file_path}.") - return imageList - - -def translateImages(imageList): - totalTokens = [0, 0] - - # Translate GPT - response = translateAI(imageList[0], "Keep the Translation as brief as possible") - translatedList = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - - return [translatedList, totalTokens, None] - -# Save some money and enter the character before translation -def getSpeaker(speaker): - match speaker: - case "ファイン": - return ["Fine", [0, 0]] - case "": - return ["", [0, 0]] - case _: - # Find Speaker - for i in range(len(NAMESLIST)): - if speaker == NAMESLIST[i][0]: - return [NAMESLIST[i][1], [0, 0]] - - # Translate and Store Speaker - response = translateAI( - f"{speaker}", - "Reply with the " + LANGUAGE + " translation of the NPC name.", - False, - ) - response[0] = response[0].title() - response[0] = response[0].replace("'S", "'s") - response[0] = response[0].replace("Speaker: ", "") - - # Retry if name doesn't translate for some reason - if re.search(r"([a-zA-Z？?])", response[0]) == None: - response = translateAI( - f"{speaker}", - "Reply with the " + LANGUAGE + " translation of the NPC name.", - False, - ) - response[0] = response[0].title() - response[0] = response[0].replace("'S", "'s") - - speakerList = [speaker, response[0]] - NAMESLIST.append(speakerList) - return response - return [speaker, [0, 0]] -def translateAI(text, history, history_ctx=None): - """ - Legacy wrapper function for the new shared translation utility. - This maintains compatibility with existing code while using the new shared implementation. - """ - global PBAR, MISMATCH, FILENAME - - # Update config estimate mode based on global ESTIMATE - TRANSLATION_CONFIG.estimateMode = bool(ESTIMATE) - - # Call the new shared translation function - return sharedtranslateAI( - text=text, - history=history, - config=TRANSLATION_CONFIG, - filename=FILENAME, - pbar=PBAR, - lock=LOCK, - mismatchList=MISMATCH - ) +# Libraries +from PIL import Image, ImageDraw, ImageFont +import json +import os +import re +import threading +import time +import traceback +import tiktoken +from pathlib import Path +from colorama import Fore +from dotenv import load_dotenv +from retry import retry +from tqdm import tqdm +from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost + +# Globals +MODEL = os.getenv("model") +TIMEOUT = int(os.getenv("timeout")) +LANGUAGE = os.getenv("language").capitalize() +from util.paths import PROMPT_PATH, VOCAB_PATH + +PROMPT = PROMPT_PATH.read_text(encoding="utf-8") +VOCAB = VOCAB_PATH.read_text(encoding="utf-8") +LOCK = threading.Lock() +PBAR = None +WIDTH = int(os.getenv("width")) +LISTWIDTH = int(os.getenv("listWidth")) +NOTEWIDTH = int(os.getenv("noteWidth")) +MAXHISTORY = 10 +ESTIMATE = "" +TOKENS = [0, 0] +NAMESLIST = [] +MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong) +FILENAME = None + +# Regex - Need to change this if you want to translate from/to other languages. Default is Japanese Regex +LANGREGEX = r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+" + +# Get pricing configuration based on the model +PRICING_CONFIG = getPricingConfig(MODEL) +INPUTAPICOST = PRICING_CONFIG["inputAPICost"] +OUTPUTAPICOST = PRICING_CONFIG["outputAPICost"] +BATCHSIZE = PRICING_CONFIG["batchSize"] +FREQUENCY_PENALTY = PRICING_CONFIG["frequencyPenalty"] + +# Initialize Translation Config +TRANSLATION_CONFIG = TranslationConfig( + model=MODEL, + language=LANGUAGE, + prompt=PROMPT, + vocab=VOCAB, + langRegex=LANGREGEX, + batchSize=BATCHSIZE, + maxHistory=MAXHISTORY, + estimateMode=False # Will be set dynamically based on ESTIMATE +) +LEAVE = False + +# tqdm Globals +BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}" +POSITION = 0 +LEAVE = False + + +def handleImages(folderName, estimate): + global ESTIMATE, TOKENS, FILENAME + ESTIMATE = estimate + FILENAME = folderName + start = time.time() + + # Translate Strings + translatedData = openFiles(f"files/{folderName}") + + # Custom Names + # customList = [[], []] + # customList = processImagesDir("Custom", customList) + + # Write TL To Images + try: + translatedList, originalList, dimensionsList = translatedData[0] + for i in range(len(translatedList)): + try: + # Create image from string + image = stringToImageOutline(translatedList[i], dimensionsList[i][0], dimensionsList[i][1]) + # Save image using the corresponding original filename + image.save(rf"translated/{folderName}/{originalList[i]}.png", quality=100) + except Exception as e: + # Log error if image saving fails + PBAR.write(f"Error processing {translatedList[i]}: {str(e)}") + except IndexError: + PBAR.write("Translated data is incomplete. Please check your input.") + + # Print File + end = time.time() + tqdm.write(getResultString(translatedData, end - start, folderName)) + with LOCK: + TOKENS[0] += translatedData[1][0] + TOKENS[1] += translatedData[1][1] + + # Print Total + totalString = getResultString(["", TOKENS, None], end - start, "TOTAL") + + # Print any errors on maps + if len(MISMATCH) > 0: + return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET + else: + return totalString + + +def openFiles(folderName): + global PBAR + + if os.path.isdir(folderName): + imageList = [[], [], []] + imageList = processImagesDir(folderName, imageList) + + # Start Translation + with tqdm( + bar_format=BAR_FORMAT, + position=POSITION, + leave=LEAVE, + desc=folderName, + total=len(imageList[0]), + ) as PBAR: + translatedData = translateImages(imageList) + translatedData = [ + [translatedData[0], imageList[2], imageList[1]], + translatedData[1], + translatedData[2], + ] + + return translatedData + else: + print("The provided directory path does not exist.") + + +def getResultString(translatedData, translationTime, filename): + # File Print String + cost = calculateCost(translatedData[1][0], translatedData[1][1], MODEL) + totalTokenstring = ( + Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]" + "[Output: " + + str(translatedData[1][1]) + + "]" "[Cost: ${:,.4f}".format(cost) + + "]" + ) + timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]" + + if translatedData[2] is None: + # Success + return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET + else: + # Fail + try: + raise translatedData[2] + except Exception as e: + traceback.print_exc() + errorString = str(e) + Fore.RED + return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET + + +def getFontSize(text, image_width, image_height, font_path): + # Start with a high font size and keep reducing it until the text fits within the image bounds + font_size = min(image_width, image_height) + + while font_size > 0: + font = ImageFont.truetype(font_path, font_size) + text_bbox = ImageDraw.Draw(Image.new("RGB", (1, 1))).textbbox((0, 0), text, font=font) + text_width = text_bbox[2] - text_bbox[0] + text_height = text_bbox[3] - text_bbox[1] + + if text_width <= image_width and text_height <= image_height: + return font_size + font_size -= 1 + + return font_size + + +def stringToImage(text, width, height, font_path="fonts/TsunagiGothic.ttf", scale_factor=4): + # Increase the resolution + scaled_width = int(width * scale_factor) + scaled_height = int(height * scale_factor) + + # Find the appropriate font size for the scaled up image + font_size = getFontSize(text, scaled_width, scaled_height, font_path) + if font_size == 0: + raise ValueError("Text is too long to fit in the supplied dimensions.") + + # Create a new image with the scaled width and height and a transparent background + image = Image.new("RGBA", (scaled_width, scaled_height), (255, 255, 255, 0)) + + # Create a drawing context + draw = ImageDraw.Draw(image) + + # Load the appropriate font + font = ImageFont.truetype(font_path, font_size) + + # Calculate the size of the text to center it + text_bbox = draw.textbbox((0, 0), text, font=font) + text_width = text_bbox[2] - text_bbox[0] + text_height = text_bbox[3] - text_bbox[1] + 20 + x = 0 + + x = (scaled_width - text_width) // 2 + y = (scaled_height - text_height) // 2 + + # Draw the text on the image + draw.text((x, y), text, font=font, fill=(255, 255, 255, 255)) + + # Resize back to the original dimensions to get a clearer text rendering + image = image.resize( + (width, height), + Image.LANCZOS, + ) + + return image + + +from PIL import Image, ImageDraw, ImageFont + + +def stringToImageOutline(text, width, height, font_path="fonts/TsunagiGothic.ttf", scale_factor=4): + # Outline + outline_color = (255, 255, 255, 255) + text_color = (0, 0, 0, 255) + outline_thickness = 4 + + # Increase the resolution + scaled_width = int(width * scale_factor) + scaled_height = int(height * scale_factor) + + # Find the appropriate font size for the scaled up image + font_size = getFontSize(text, scaled_width, scaled_height, font_path) + if font_size == 0: + raise ValueError("Text is too long to fit in the supplied dimensions.") + + # Create a new image with the scaled width and height and a transparent background + image = Image.new("RGBA", (scaled_width, scaled_height), (255, 255, 255, 0)) + + # Create a drawing context + draw = ImageDraw.Draw(image) + + # Load the appropriate font + font = ImageFont.truetype(font_path, font_size) + + # Calculate the size of the text to center it + text_bbox = draw.textbbox((0, 0), text, font=font) + text_width = text_bbox[2] - text_bbox[0] + text_height = text_bbox[3] - text_bbox[1] + 20 + x = (scaled_width - text_width) // 2 + y = (scaled_height - text_height) // 2 + + # Draw the text outline by applying the text multiple times with small offsets + for dx in range(-outline_thickness, outline_thickness + 1): + for dy in range(-outline_thickness, outline_thickness + 1): + if dx != 0 or dy != 0: + draw.text((x + dx, y + dy), text, font=font, fill=outline_color) + + # Draw the main text + draw.text((x, y), text, font=font, fill=text_color) + + # Resize back to the original dimensions to get a clearer text rendering + image = image.resize((width, height), Image.LANCZOS) + + return image + + +def stringToImageBox(text, width, height, font_path="fonts/TsunagiGothic.ttf", scale_factor=4): + # Increase the resolution + scaled_width = int(width * scale_factor) + scaled_height = int(height * scale_factor) + + # Padding around the text + padding = 10 + + # Calculate the dimensions available for text placement + available_width = scaled_width - 2 * padding + available_height = scaled_height - 2 * padding + + # Determine the best font size to fit within the available dimensions + font_size = getFontSize(text, available_width, available_height, font_path) + if font_size <= 0: + raise ValueError("Text is too long to fit in the supplied dimensions.") + + # Create a new image with increased resolution + image = Image.new("RGBA", (scaled_width, scaled_height), (255, 255, 255, 0)) + draw = ImageDraw.Draw(image) + + # Load the calculated font + font = ImageFont.truetype(font_path, font_size) + + # Calculate the size and bounding box of the text + text_bbox = draw.textbbox((0, 0), text, font=font) + text_width = text_bbox[2] - text_bbox[0] + text_height = text_bbox[3] - text_bbox[1] + 20 + + # Determine centered position for the text while considering padding + # Additional adjustment ensures text appears centrally aligned + x = (scaled_width - text_width) // 2 + y = (scaled_height - text_height) // 2 + + # Draw a black box with a white outline that fits the image dimensions precisely + draw.rectangle([0, 0, scaled_width - 1, scaled_height - 1], outline=(255, 255, 255, 255), width=1) + + # Fill the inside box with black color + draw.rectangle([1, 1, scaled_width - 2, scaled_height - 2], fill=(0, 0, 0, 255)) + + # Render the text within the image + draw.text((x, y), text, font=font, fill=(255, 255, 255, 255)) + + # Shrink the image back to original dimensions with high-quality interpolation + image = image.resize( + (width, height), + Image.LANCZOS, + ) + + return image + + +def getImageDimensions(file_path): + try: + with Image.open(file_path) as img: + width, height = img.size + return width, height + except Exception as e: + print(f"Error reading {file_path}: {e}") + return None, None + + +def processImagesDir(directory_path, imageList): + for file_name in os.listdir(directory_path): + # .png and Japanese + if ".png" in file_name: + file_path = os.path.join(directory_path, file_name) + if os.path.isfile(file_path): + # Check if the file is an image + try: + width, height = getImageDimensions(file_path) + if width is not None and height is not None: + placeholders = { + ".png": "", + } + for target, replacement in placeholders.items(): + file_name = file_name.replace(target, replacement) + match = re.search(r"[\[【].+?[\]】](.*)", file_name) + if match: + text = match.group(1) + else: + text = file_name + imageList[0].append(text) + imageList[1].append([width, height]) + imageList[2].append(file_name) + except Exception as e: + print(f"Error processing {file_name}: {e}") + + if ".txt" in file_name: + try: + with open(f"{directory_path}/{file_name}", "r", encoding="utf8") as file: + for line in file: + line = line.strip() + line = line.replace(":", "：") + line = line.replace("/", "／") + line = line.replace("?", "？") + imageList[0].append(line) # Using strip() to remove any extra newlines or spaces + imageList[1].append([104, 15]) + except FileNotFoundError: + print(f"The file at {file_path} was not found.") + except IOError: + print(f"An error occurred while reading the file at {file_path}.") + return imageList + + +def translateImages(imageList): + totalTokens = [0, 0] + + # Translate GPT + response = translateAI(imageList[0], "Keep the Translation as brief as possible") + translatedList = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + + return [translatedList, totalTokens, None] + +# Save some money and enter the character before translation +def getSpeaker(speaker): + match speaker: + case "ファイン": + return ["Fine", [0, 0]] + case "": + return ["", [0, 0]] + case _: + # Find Speaker + for i in range(len(NAMESLIST)): + if speaker == NAMESLIST[i][0]: + return [NAMESLIST[i][1], [0, 0]] + + # Translate and Store Speaker + response = translateAI( + f"{speaker}", + "Reply with the " + LANGUAGE + " translation of the NPC name.", + False, + ) + response[0] = response[0].title() + response[0] = response[0].replace("'S", "'s") + response[0] = response[0].replace("Speaker: ", "") + + # Retry if name doesn't translate for some reason + if re.search(r"([a-zA-Z？?])", response[0]) == None: + response = translateAI( + f"{speaker}", + "Reply with the " + LANGUAGE + " translation of the NPC name.", + False, + ) + response[0] = response[0].title() + response[0] = response[0].replace("'S", "'s") + + speakerList = [speaker, response[0]] + NAMESLIST.append(speakerList) + return response + return [speaker, [0, 0]] +def translateAI(text, history, history_ctx=None): + """ + Legacy wrapper function for the new shared translation utility. + This maintains compatibility with existing code while using the new shared implementation. + """ + global PBAR, MISMATCH, FILENAME + + # Update config estimate mode based on global ESTIMATE + TRANSLATION_CONFIG.estimateMode = bool(ESTIMATE) + + # Call the new shared translation function + return sharedtranslateAI( + text=text, + history=history, + config=TRANSLATION_CONFIG, + filename=FILENAME, + pbar=PBAR, + lock=LOCK, + mismatchList=MISMATCH + ) diff --git a/modules/json.py b/modules/json.py index 8be6378..e306dc3 100644 --- a/modules/json.py +++ b/modules/json.py @@ -1,1359 +1,1361 @@ -# Libraries -import json -import os -import re -import util.dazedwrap as dazedwrap -import threading -import time -import traceback -import tiktoken -from pathlib import Path -from colorama import Fore -from dotenv import load_dotenv -from retry import retry -from tqdm import tqdm -from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost, getPricingConfig, calculateCost -import tempfile - -# Globals -MODEL = os.getenv("model") -TIMEOUT = int(os.getenv("timeout")) -LANGUAGE = os.getenv("language").capitalize() -PROMPT = Path("prompt.txt").read_text(encoding="utf-8") -VOCAB = Path("vocab.txt").read_text(encoding="utf-8") -LOCK = threading.Lock() -WIDTH = int(os.getenv("width")) -LISTWIDTH = int(os.getenv("listWidth")) -NOTEWIDTH = 70 -MAXHISTORY = 10 -ESTIMATE = "" -TOKENS = [0, 0] -NAMESLIST = [] -FILENAME = None -NAMES = False # Output a list of all the character names found -BRFLAG = False # If the game uses
instead -FIXTEXTWRAP = True # Overwrites textwrap -IGNORETLTEXT = False # Ignores all translated text. -MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong) - -# tqdm Globals -BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}" -POSITION = 0 -LEAVE = False -PBAR = None - -# Regex - Need to change this if you want to translate from/to other languages. Default is Japanese Regex -LANGREGEX = r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+" - -# Get pricing configuration based on the model -PRICING_CONFIG = getPricingConfig(MODEL) -INPUTAPICOST = PRICING_CONFIG["inputAPICost"] -OUTPUTAPICOST = PRICING_CONFIG["outputAPICost"] -BATCHSIZE = PRICING_CONFIG["batchSize"] -FREQUENCY_PENALTY = PRICING_CONFIG["frequencyPenalty"] - -# Initialize Translation Config -TRANSLATION_CONFIG = TranslationConfig( - model=MODEL, - language=LANGUAGE, - prompt=PROMPT, - vocab=VOCAB, - langRegex=LANGREGEX, - batchSize=BATCHSIZE, - maxHistory=MAXHISTORY, - estimateMode=False # Will be set dynamically based on ESTIMATE -) -LEAVE = False - -def handleJSON(filename, estimate): - global ESTIMATE, totalTokens, FILENAME - ESTIMATE = estimate - FILENAME = filename - - if estimate: - start = time.time() - translatedData = openFiles(filename) - - # Print Result - end = time.time() - tqdm.write(getResultString(translatedData, end - start, filename)) - with LOCK: - TOKENS[0] += translatedData[1][0] - TOKENS[1] += translatedData[1][1] - - return getResultString(["", TOKENS, None], end - start, "TOTAL") - - else: - try: - start = time.time() - translatedData = openFiles(filename) - - # Write final result after translation is complete - with open("translated/" + filename, "w", encoding="utf-8", newline="\n") as outFile: - json.dump(translatedData[0], outFile, ensure_ascii=False, indent=4) - - # Print Result - end = time.time() - tqdm.write(getResultString(translatedData, end - start, filename)) - with LOCK: - TOKENS[0] += translatedData[1][0] - TOKENS[1] += translatedData[1][1] - except Exception: - return "Fail" - - return getResultString(["", TOKENS, None], end - start, "TOTAL") - - -def openFiles(filename): - with open("files/" + filename, "r", encoding="UTF-8-sig") as f: - data = json.load(f) - - # Map Files - if ".json" in filename: - translatedData = parseJSON(data, filename) - - else: - raise NameError(filename + " Not Supported") - - return translatedData - - -def getResultString(translatedData, translationTime, filename): - # File Print String - cost = calculateCost(translatedData[1][0], translatedData[1][1], MODEL) - totalTokenstring = ( - Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]" - "[Output: " - + str(translatedData[1][1]) - + "]" "[Cost: ${:,.4f}".format(cost) - + "]" - ) - timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]" - - if translatedData[2] == None: - # Success - return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET - - else: - # Fail - try: - raise translatedData[2] - except Exception as e: - traceback.print_exc() - errorString = str(e) + Fore.RED - return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET - - -def parseJSON(data, filename): - totalTokens = [0, 0] - totalLines = 0 - totalLines = len(data) - global LOCK, PBAR - - with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar: - pbar.desc = filename - PBAR = pbar - try: - # Check if data is a simple key-value dict (not a list) - if isinstance(data, dict) and not isinstance(data, list): - # Check if it's a simple key-value format (all values are strings, not nested objects) - if all(isinstance(v, str) for v in data.values()): - result = translateSimpleKeyValueJSON(data, filename) - # PSData format (Nupu_PSData.json - MailDatas, MemoryData, ShopDatas, hStData) - elif any(k in data for k in ["MailDatas", "MemoryData", "ShopDatas", "hStData"]): - result = translatePSData(data, filename) - # RdData format (RdData.json - dgnDatas with dungeon/dialogue data) - elif "dgnDatas" in data: - result = translateRdData(data, filename) - # GameSetting format (Nupu_GameSetting.json - fzCardDatas, fzDeckDatas, rentalDecks) - elif any(k in data for k in ["fzCardDatas", "fzDeckDatas", "rentalDecks"]): - result = translateGameSetting(data, filename) - # Src/Tl dialogue format (numeric-keyed entries with src and tl fields) - elif data and all(isinstance(v, dict) and "src" in v for v in data.values()): - result = translateDialogueSrcTl(data, filename) - else: - result = translateJSON(data, filename, []) - else: - result = translateJSON(data, filename, []) - totalTokens[0] += result[0] - totalTokens[1] += result[1] - except Exception as e: - traceback.print_exc() - return [data, totalTokens, e] - return [data, totalTokens, None] - - -def save_progress_json(data, filename): - """Save current JSON translation progress.""" - try: - if ESTIMATE: - return - os.makedirs("translated", exist_ok=True) - tmp_path = os.path.join("translated", f"{filename}.tmp") - final_path = os.path.join("translated", filename) - with open(tmp_path, "w", encoding="utf-8", newline="\n") as outFile: - json.dump(data, outFile, ensure_ascii=False, indent=4) - os.replace(tmp_path, final_path) - except Exception: - traceback.print_exc() - - -def translateSimpleKeyValueJSON(data, filename): - """ - Translate simple key-value JSON format where keys contain Japanese text - and values are placeholder strings like "TODO". - - Format example: - { - "\\r\\nスタビライザー": "TODO", - "\\r\\nプラグインなし": "TODO" - } - """ - global LOCK, ESTIMATE, FILENAME, PBAR, MISMATCH - tokens = [0, 0] - stringList = [] - keyList = [] - - # Pass 1: Collect all keys that need translation - for key, value in data.items(): - # Check if the key contains text matching the language regex - if re.search(LANGREGEX, key): - # Strip whitespace and newline characters for translation - # cleanKey = key.strip().replace("\r\n", "").replace("\n", "").replace("\r", "") - # if cleanKey: - stringList.append(key) - keyList.append(key) # Store original key for reference - - # Translate all keys if any were found - if stringList: - PBAR.total = len(stringList) - PBAR.refresh() - - response = translateAI(stringList, "Reply with the English Translation") - tokens[0] += response[1][0] - tokens[1] += response[1][1] - translatedList = response[0] - - # Check for mismatch - if len(stringList) != len(translatedList): - with LOCK: - if FILENAME not in MISMATCH: - MISMATCH.append(FILENAME) - - # Pass 2: Update the values with translations - for i, key in enumerate(keyList): - if i < len(translatedList): - translatedText = translatedList[i] - # Set the translated text as the value - data[key] = translatedText - - # Save progress after each translation - save_progress_json(data, filename) - - return tokens - - -def translatePSData(data, filename): - """Translate PSData JSON format (e.g. Nupu_PSData.json). - - Handles four sections: - - MailDatas: In-game mail/message system - - MemoryData: Scene/CG gallery memories - - ShopDatas: Shop configurations and dialogue - - hStData: Status rank descriptions - - Only translates player-visible text fields. - """ - global LOCK, ESTIMATE, FILENAME, PBAR, MISMATCH - tokens = [0, 0] - - def batchTranslate(stringList, context): - """Translate a list of strings and return translations. Returns [] on mismatch.""" - nonlocal tokens - if not stringList: - return [] - response = translateAI(stringList, context) - tokens[0] += response[1][0] - tokens[1] += response[1][1] - if len(stringList) != len(response[0]): - with LOCK: - if FILENAME not in MISMATCH: - MISMATCH.append(FILENAME) - return [] - return response[0] - - # ================================================================ - # PASS 1: Collect all translatable strings from every section - # ================================================================ - - # -- MailDatas -- - mailNameS, mailNameI = [], [] - mailTitleS, mailTitleI = [], [] - mailListNameS, mailListNameI = [], [] - mailTextS, mailTextI = [], [] - - if "MailDatas" in data: - for i, m in enumerate(data["MailDatas"]): - if m.get("Name"): - mailNameS.append(m["Name"]) - mailNameI.append(i) - if m.get("Title"): - mailTitleS.append(m["Title"]) - mailTitleI.append(i) - if m.get("MailListName"): - mailListNameS.append(m["MailListName"]) - mailListNameI.append(i) - if m.get("MailText"): - mailTextS.append(m["MailText"].replace("\r\n", " ").replace("\n", " ").strip()) - mailTextI.append(i) - - # -- MemoryData -- - memTitleS, memTitleI = [], [] - memSetuS, memSetuI = [], [] - memHintS, memHintI = [], [] - memPtnS, memPtnI = [], [] # indices are (memIdx, ptnIdx) - memDtlSetuS, memDtlSetuI = [], [] # indices are (memIdx, dtlIdx) - - if "MemoryData" in data: - for i, mem in enumerate(data["MemoryData"]): - if mem.get("_Title"): - memTitleS.append(mem["_Title"]) - memTitleI.append(i) - if mem.get("_Setu"): - memSetuS.append(mem["_Setu"].replace("\r\n", " ").replace("\n", " ").strip()) - memSetuI.append(i) - if mem.get("_Hint"): - memHintS.append(mem["_Hint"]) - memHintI.append(i) - if "_Ptn" in mem and isinstance(mem["_Ptn"], list): - for j, p in enumerate(mem["_Ptn"]): - if p: - memPtnS.append(p) - memPtnI.append((i, j)) - if "_DtlCheck" in mem and isinstance(mem["_DtlCheck"], list): - for j, dtl in enumerate(mem["_DtlCheck"]): - if isinstance(dtl, dict) and dtl.get("_Setu"): - memDtlSetuS.append(dtl["_Setu"].replace("\r\n", " ").replace("\n", " ").strip()) - memDtlSetuI.append((i, j)) - - # -- ShopDatas -- - shopNameS, shopNameI = [], [] - shopTextFields = ["buySelf", "emptySelf", "firstSelf", "husokuSelf", - "randSelf1", "randSelf2", "randSelf3"] - shopTextS = {f: [] for f in shopTextFields} - shopTextI = {f: [] for f in shopTextFields} - - if "ShopDatas" in data: - for i, shop in enumerate(data["ShopDatas"]): - if shop.get("name"): - shopNameS.append(shop["name"]) - shopNameI.append(i) - for field in shopTextFields: - if field in shop and isinstance(shop[field], dict): - text = shop[field].get("text", "") - if text: - shopTextS[field].append(text.replace("\r\n", " ").replace("\n", " ").strip()) - shopTextI[field].append(i) - - # -- hStData -- - rankArrayNames = ["KutiRankDatas", "MuneRankDatas", "SiriRankDatas", "TituRankDatas"] - rankSetuS, rankSetuI = [], [] # indices are (arrayName, idx) - rankText1S, rankText1I = [], [] - rankText2S, rankText2I = [], [] - - if "hStData" in data: - hst = data["hStData"] - for arrName in rankArrayNames: - if arrName in hst and isinstance(hst[arrName], list): - for i, rank in enumerate(hst[arrName]): - if rank.get("rankSetu"): - rankSetuS.append(rank["rankSetu"].replace("\r\n", " ").replace("\n", " ").strip()) - rankSetuI.append((arrName, i)) - if rank.get("rankText1"): - rankText1S.append(rank["rankText1"]) - rankText1I.append((arrName, i)) - if rank.get("rankText2"): - rankText2S.append(rank["rankText2"]) - rankText2I.append((arrName, i)) - - # Set progress bar total - totalItems = ( - len(mailNameS) + len(mailTitleS) + len(mailListNameS) + len(mailTextS) - + len(memTitleS) + len(memSetuS) + len(memHintS) + len(memPtnS) + len(memDtlSetuS) - + len(shopNameS) + sum(len(v) for v in shopTextS.values()) - + len(rankSetuS) + len(rankText1S) + len(rankText2S) - ) - PBAR.total = totalItems - PBAR.refresh() - - # ================================================================ - # PASS 2: Translate each batch and apply results back to data - # ================================================================ - - # -- MailDatas -- - if "MailDatas" in data: - mails = data["MailDatas"] - - for tl, idx in zip(batchTranslate(mailNameS, "Character Name"), mailNameI): - mails[idx]["Name"] = tl - if mailNameS: - save_progress_json(data, filename) - - for tl, idx in zip(batchTranslate(mailTitleS, "Mail Subject"), mailTitleI): - mails[idx]["Title"] = tl - if mailTitleS: - save_progress_json(data, filename) - - for tl, idx in zip(batchTranslate(mailListNameS, "Mail List Entry"), mailListNameI): - mails[idx]["MailListName"] = tl - if mailListNameS: - save_progress_json(data, filename) - - for tl, idx in zip(batchTranslate(mailTextS, "Mail Body Text"), mailTextI): - mails[idx]["MailText"] = dazedwrap.wrapText(tl, WIDTH) - if mailTextS: - save_progress_json(data, filename) - - # -- MemoryData -- - if "MemoryData" in data: - memories = data["MemoryData"] - - for tl, idx in zip(batchTranslate(memTitleS, "Scene Title"), memTitleI): - memories[idx]["_Title"] = tl - if memTitleS: - save_progress_json(data, filename) - - for tl, idx in zip(batchTranslate(memSetuS, "Scene Description"), memSetuI): - memories[idx]["_Setu"] = tl - if memSetuS: - save_progress_json(data, filename) - - for tl, idx in zip(batchTranslate(memHintS, "Hint Text"), memHintI): - memories[idx]["_Hint"] = tl - if memHintS: - save_progress_json(data, filename) - - for tl, (mi, pi) in zip(batchTranslate(memPtnS, "Scene Description"), memPtnI): - memories[mi]["_Ptn"][pi] = tl - if memPtnS: - save_progress_json(data, filename) - - for tl, (mi, di) in zip(batchTranslate(memDtlSetuS, "Character Commentary"), memDtlSetuI): - memories[mi]["_DtlCheck"][di]["_Setu"] = dazedwrap.wrapText(tl, LISTWIDTH) - if memDtlSetuS: - save_progress_json(data, filename) - - # -- ShopDatas -- - if "ShopDatas" in data: - shops = data["ShopDatas"] - - for tl, idx in zip(batchTranslate(shopNameS, "Shop Name"), shopNameI): - shops[idx]["name"] = tl - if shopNameS: - save_progress_json(data, filename) - - for field in shopTextFields: - for tl, idx in zip(batchTranslate(shopTextS[field], "Shop Dialogue"), shopTextI[field]): - shops[idx][field]["text"] = dazedwrap.wrapText(tl, WIDTH) - if shopTextS[field]: - save_progress_json(data, filename) - - # -- hStData -- - if "hStData" in data: - hst = data["hStData"] - - for tl, (an, idx) in zip(batchTranslate(rankSetuS, "Rank Description"), rankSetuI): - hst[an][idx]["rankSetu"] = dazedwrap.wrapText(tl, WIDTH) - if rankSetuS: - save_progress_json(data, filename) - - for tl, (an, idx) in zip(batchTranslate(rankText1S, "Rank Label"), rankText1I): - hst[an][idx]["rankText1"] = tl - if rankText1S: - save_progress_json(data, filename) - - for tl, (an, idx) in zip(batchTranslate(rankText2S, "Rank Subtitle"), rankText2I): - hst[an][idx]["rankText2"] = tl - if rankText2S: - save_progress_json(data, filename) - - return tokens - - -def translateRdData(data, filename): - """Translate RdData JSON format (e.g. RdData.json). - - Handles dungeon data with nested dialogue, choices, and UI text. - Only translates player-visible text fields. - """ - global LOCK, ESTIMATE, FILENAME, PBAR, MISMATCH - tokens = [0, 0] - - def batchTranslate(stringList, context): - """Translate a list of strings and return translations. Returns [] on mismatch.""" - nonlocal tokens - if not stringList: - return [] - response = translateAI(stringList, context) - tokens[0] += response[1][0] - tokens[1] += response[1][1] - if len(stringList) != len(response[0]): - with LOCK: - if FILENAME not in MISMATCH: - MISMATCH.append(FILENAME) - return [] - return response[0] - - if "dgnDatas" not in data: - return tokens - - dgns = data["dgnDatas"] - - # ================================================================ - # PASS 1: Collect all translatable strings - # ================================================================ - - # -- Dungeon-level short fields -- - dgnNameS, dgnNameI = [], [] - clearTxtS, clearTxtI = [], [] - hosyuTxtS, hosyuTxtI = [], [] - firstHosyuTxtS, firstHosyuTxtI = [], [] - memoTxtS, memoTxtI = [], [] - appMemoTxtS, appMemoTxtI = [], [] - areaMemoS, areaMemoI = [], [] - - # -- Dungeon-level long text fields -- - naiyoTxtS, naiyoTxtI = [], [] - naiyoTxtExS, naiyoTxtExI = [], [] - johoTxtS, johoTxtI = [], [] - johoTxtExS, johoTxtExI = [], [] - - # -- Floor names -- - kaisoNameS, kaisoNameI = [], [] # (dgnIdx, kaisoIdx) - - # -- Dialogue (talkDatas / talkDatasEx) -- - talkNameS, talkNameI = [], [] # (dgnIdx, talkKey, talkIdx) - speakerNameS, speakerNameI = [], [] # (dgnIdx, talkKey, talkIdx, speechIdx) - dialogueS, dialogueI = [], [] # (dgnIdx, talkKey, talkIdx, speechIdx) - - # -- Choice events (sijiDatas) -- - sijiNameS, sijiNameI = [], [] # (dgnIdx, sijiIdx) - sijiChoiceS, sijiChoiceI = [], [] # (dgnIdx, sijiIdx) - - for di, dgn in enumerate(dgns): - if dgn is None: - continue - - # Dungeon short fields - if dgn.get("name"): - dgnNameS.append(dgn["name"]) - dgnNameI.append(di) - if dgn.get("clearTxt"): - clearTxtS.append(dgn["clearTxt"]) - clearTxtI.append(di) - if dgn.get("hosyuTxt"): - hosyuTxtS.append(dgn["hosyuTxt"]) - hosyuTxtI.append(di) - if dgn.get("firstHosyuTxt"): - firstHosyuTxtS.append(dgn["firstHosyuTxt"]) - firstHosyuTxtI.append(di) - if dgn.get("memoTxt"): - memoTxtS.append(dgn["memoTxt"]) - memoTxtI.append(di) - if dgn.get("appMemoTxt"): - appMemoTxtS.append(dgn["appMemoTxt"]) - appMemoTxtI.append(di) - if dgn.get("areaMemo"): - areaMemoS.append(dgn["areaMemo"]) - areaMemoI.append(di) - - # Dungeon long text fields (strip line breaks for translation) - if dgn.get("naiyoTxt"): - naiyoTxtS.append(dgn["naiyoTxt"].replace("\r\n", " ").replace("\n", " ").strip()) - naiyoTxtI.append(di) - if dgn.get("naiyoTxtEx"): - naiyoTxtExS.append(dgn["naiyoTxtEx"].replace("\r\n", " ").replace("\n", " ").strip()) - naiyoTxtExI.append(di) - if dgn.get("johoTxt"): - johoTxtS.append(dgn["johoTxt"].replace("\r\n", " ").replace("\n", " ").strip()) - johoTxtI.append(di) - if dgn.get("johoTxtEx"): - johoTxtExS.append(dgn["johoTxtEx"].replace("\r\n", " ").replace("\n", " ").strip()) - johoTxtExI.append(di) - - # Floor names - if "kaisoDatas" in dgn and isinstance(dgn["kaisoDatas"], list): - for ki, kaiso in enumerate(dgn["kaisoDatas"]): - if isinstance(kaiso, dict) and kaiso.get("kaisoName"): - kaisoNameS.append(kaiso["kaisoName"]) - kaisoNameI.append((di, ki)) - - # Dialogue from talkDatas and talkDatasEx - for talkKey in ["talkDatas", "talkDatasEx"]: - if talkKey in dgn and isinstance(dgn[talkKey], list): - for ti, talk in enumerate(dgn[talkKey]): - if not isinstance(talk, dict): - continue - if talk.get("name"): - talkNameS.append(talk["name"]) - talkNameI.append((di, talkKey, ti)) - if "speachDatas" in talk and isinstance(talk["speachDatas"], list): - for si, speech in enumerate(talk["speachDatas"]): - if not isinstance(speech, dict): - continue - if speech.get("name"): - speakerNameS.append(speech["name"]) - speakerNameI.append((di, talkKey, ti, si)) - if speech.get("text"): - dialogueS.append(speech["text"].replace("\r\n", " ").replace("\n", " ").strip()) - dialogueI.append((di, talkKey, ti, si)) - - # Choice events (sijiDatas) - if "sijiDatas" in dgn and isinstance(dgn["sijiDatas"], list): - for si, siji in enumerate(dgn["sijiDatas"]): - if not isinstance(siji, dict): - continue - if siji.get("name"): - sijiNameS.append(siji["name"]) - sijiNameI.append((di, si)) - if siji.get("choiceSijiTxt"): - sijiChoiceS.append(siji["choiceSijiTxt"].replace("\r\n", " ").replace("\n", " ").strip()) - sijiChoiceI.append((di, si)) - - # Set progress bar total - totalItems = ( - len(dgnNameS) + len(clearTxtS) + len(hosyuTxtS) + len(firstHosyuTxtS) - + len(memoTxtS) + len(appMemoTxtS) + len(areaMemoS) - + len(naiyoTxtS) + len(naiyoTxtExS) + len(johoTxtS) + len(johoTxtExS) - + len(kaisoNameS) - + len(talkNameS) + len(speakerNameS) + len(dialogueS) - + len(sijiNameS) + len(sijiChoiceS) - ) - PBAR.total = totalItems - PBAR.refresh() - - # ================================================================ - # PASS 2: Translate each batch and apply results back to data - # ================================================================ - - # -- Dungeon short fields -- - for tl, idx in zip(batchTranslate(dgnNameS, "Dungeon Name"), dgnNameI): - dgns[idx]["name"] = tl - if dgnNameS: - save_progress_json(data, filename) - - for tl, idx in zip(batchTranslate(clearTxtS, "Clear Condition"), clearTxtI): - dgns[idx]["clearTxt"] = tl - if clearTxtS: - save_progress_json(data, filename) - - for tl, idx in zip(batchTranslate(hosyuTxtS, "Reward Description"), hosyuTxtI): - dgns[idx]["hosyuTxt"] = tl - if hosyuTxtS: - save_progress_json(data, filename) - - for tl, idx in zip(batchTranslate(firstHosyuTxtS, "First Clear Reward"), firstHosyuTxtI): - dgns[idx]["firstHosyuTxt"] = tl - if firstHosyuTxtS: - save_progress_json(data, filename) - - for tl, idx in zip(batchTranslate(memoTxtS, "Memo Label"), memoTxtI): - dgns[idx]["memoTxt"] = tl - if memoTxtS: - save_progress_json(data, filename) - - for tl, idx in zip(batchTranslate(appMemoTxtS, "App Memo"), appMemoTxtI): - dgns[idx]["appMemoTxt"] = tl - if appMemoTxtS: - save_progress_json(data, filename) - - for tl, idx in zip(batchTranslate(areaMemoS, "Area Label"), areaMemoI): - dgns[idx]["areaMemo"] = tl - if areaMemoS: - save_progress_json(data, filename) - - # -- Dungeon long text fields (with word wrap) -- - for tl, idx in zip(batchTranslate(naiyoTxtS, "Mission Description"), naiyoTxtI): - dgns[idx]["naiyoTxt"] = dazedwrap.wrapText(tl, WIDTH) - if naiyoTxtS: - save_progress_json(data, filename) - - for tl, idx in zip(batchTranslate(naiyoTxtExS, "Extended Mission Description"), naiyoTxtExI): - dgns[idx]["naiyoTxtEx"] = dazedwrap.wrapText(tl, WIDTH) - if naiyoTxtExS: - save_progress_json(data, filename) - - for tl, idx in zip(batchTranslate(johoTxtS, "Info Text"), johoTxtI): - dgns[idx]["johoTxt"] = dazedwrap.wrapText(tl, WIDTH) - if johoTxtS: - save_progress_json(data, filename) - - for tl, idx in zip(batchTranslate(johoTxtExS, "Extended Info Text"), johoTxtExI): - dgns[idx]["johoTxtEx"] = dazedwrap.wrapText(tl, WIDTH) - if johoTxtExS: - save_progress_json(data, filename) - - # -- Floor names -- - for tl, (di, ki) in zip(batchTranslate(kaisoNameS, "Floor Name"), kaisoNameI): - dgns[di]["kaisoDatas"][ki]["kaisoName"] = tl - if kaisoNameS: - save_progress_json(data, filename) - - # -- Dialogue event names -- - for tl, (di, tk, ti) in zip(batchTranslate(talkNameS, "Event Label"), talkNameI): - dgns[di][tk][ti]["name"] = tl - if talkNameS: - save_progress_json(data, filename) - - # -- Speaker names -- - for tl, (di, tk, ti, si) in zip(batchTranslate(speakerNameS, "Character Name"), speakerNameI): - dgns[di][tk][ti]["speachDatas"][si]["name"] = tl - if speakerNameS: - save_progress_json(data, filename) - - # -- Dialogue text (with word wrap) -- - for tl, (di, tk, ti, si) in zip(batchTranslate(dialogueS, "Dialogue"), dialogueI): - dgns[di][tk][ti]["speachDatas"][si]["text"] = dazedwrap.wrapText(tl, WIDTH) - if dialogueS: - save_progress_json(data, filename) - - # -- Choice event names -- - for tl, (di, si) in zip(batchTranslate(sijiNameS, "Event Name"), sijiNameI): - dgns[di]["sijiDatas"][si]["name"] = tl - if sijiNameS: - save_progress_json(data, filename) - - # -- Choice text (with word wrap) -- - for tl, (di, si) in zip(batchTranslate(sijiChoiceS, "Choice Event Text"), sijiChoiceI): - dgns[di]["sijiDatas"][si]["choiceSijiTxt"] = dazedwrap.wrapText(tl, WIDTH) - if sijiChoiceS: - save_progress_json(data, filename) - - return tokens - - -def translateGameSetting(data, filename): - """Translate GameSetting JSON format (e.g. Nupu_GameSetting.json). - - Handles card data, deck definitions, and rental decks. - Only translates player-visible text fields. - """ - global LOCK, ESTIMATE, FILENAME, PBAR, MISMATCH - tokens = [0, 0] - - def batchTranslate(stringList, context): - """Translate a list of strings and return translations. Returns [] on mismatch.""" - nonlocal tokens - if not stringList: - return [] - response = translateAI(stringList, context) - tokens[0] += response[1][0] - tokens[1] += response[1][1] - if len(stringList) != len(response[0]): - with LOCK: - if FILENAME not in MISMATCH: - MISMATCH.append(FILENAME) - return [] - return response[0] - - # ================================================================ - # PASS 1: Collect all translatable strings - # ================================================================ - - # -- fzCardDatas: card descriptions -- - desc1S, desc1I = [], [] - desc2S, desc2I = [], [] - - if "fzCardDatas" in data: - for i, card in enumerate(data["fzCardDatas"]): - if card.get("desc1"): - desc1S.append(card["desc1"].replace("\r\n", " ").replace("\n", " ").strip()) - desc1I.append(i) - if card.get("desc2"): - desc2S.append(card["desc2"].replace("\r\n", " ").replace("\n", " ").strip()) - desc2I.append(i) - - # -- fzDeckDatas: deck name and description -- - deckNameS, deckNameI = [], [] - deckDescS, deckDescI = [], [] - - if "fzDeckDatas" in data: - for i, deck in enumerate(data["fzDeckDatas"]): - if deck.get("name"): - deckNameS.append(deck["name"]) - deckNameI.append(i) - if deck.get("desc"): - deckDescS.append(deck["desc"].replace("\r\n", " ").replace("\n", " ").strip()) - deckDescI.append(i) - - # -- rentalDecks: memo label -- - rentalMemoS, rentalMemoI = [], [] - - if "rentalDecks" in data: - for i, rental in enumerate(data["rentalDecks"]): - if rental.get("memo"): - rentalMemoS.append(rental["memo"]) - rentalMemoI.append(i) - - # Set progress bar total - totalItems = ( - len(desc1S) + len(desc2S) - + len(deckNameS) + len(deckDescS) - + len(rentalMemoS) - ) - PBAR.total = totalItems - PBAR.refresh() - - # ================================================================ - # PASS 2: Translate each batch and apply results back to data - # ================================================================ - - # -- fzCardDatas -- - if "fzCardDatas" in data: - cards = data["fzCardDatas"] - - for tl, idx in zip(batchTranslate(desc1S, "Card Description"), desc1I): - cards[idx]["desc1"] = dazedwrap.wrapText(tl, WIDTH) - if desc1S: - save_progress_json(data, filename) - - for tl, idx in zip(batchTranslate(desc2S, "Card Description"), desc2I): - cards[idx]["desc2"] = dazedwrap.wrapText(tl, WIDTH) - if desc2S: - save_progress_json(data, filename) - - # -- fzDeckDatas -- - if "fzDeckDatas" in data: - decks = data["fzDeckDatas"] - - for tl, idx in zip(batchTranslate(deckNameS, "Deck Name"), deckNameI): - decks[idx]["name"] = tl - if deckNameS: - save_progress_json(data, filename) - - for tl, idx in zip(batchTranslate(deckDescS, "Deck Description"), deckDescI): - decks[idx]["desc"] = dazedwrap.wrapText(tl, WIDTH) - if deckDescS: - save_progress_json(data, filename) - - # -- rentalDecks -- - if "rentalDecks" in data: - rentals = data["rentalDecks"] - - for tl, idx in zip(batchTranslate(rentalMemoS, "Deck Label"), rentalMemoI): - rentals[idx]["memo"] = tl - if rentalMemoS: - save_progress_json(data, filename) - - return tokens - - -def translateDialogueSrcTl(data, filename): - """Translate src/tl dialogue JSON format. - - Format: - { - "0": {"type": "dialogue", "src": "Japanese text", "tl": ""}, - "4": {"type": "dialogue", "speaker": "琴音", "src": "「...」", "tl": ""} - } - - Translates 'src' into 'tl'. Also translates Japanese 'speaker' fields. - Skips entries where 'tl' is already populated. - """ - global LOCK, ESTIMATE, FILENAME, PBAR, MISMATCH - tokens = [0, 0] - srcList = [] - keyList = [] # list of (key, entry_type) - - # Collect all entries with translatable content - for key, entry in data.items(): - if not isinstance(entry, dict): - continue - src = entry.get("src", "") - if not src: - continue - tl = entry.get("tl", "") - entry_type = entry.get("type", "") - speaker = entry.get("speaker", "") - - if tl: - # Already has a translation — send tl so the module can decide to skip it - srcList.append(tl) - else: - if not re.search(LANGREGEX, src): - continue - srcClean = src.replace("\r\n", " ").replace("\n", " ").strip() - if speaker: - srcList.append(f"[{speaker}]: {srcClean}") - else: - srcList.append(srcClean) - keyList.append((key, entry_type)) - - if srcList: - PBAR.total = len(srcList) - PBAR.refresh() - - response = translateAI(srcList, "Reply with the English Translation") - tokens[0] += response[1][0] - tokens[1] += response[1][1] - translatedList = response[0] - - if len(srcList) != len(translatedList): - with LOCK: - if FILENAME not in MISMATCH: - MISMATCH.append(FILENAME) - - for i, (key, entry_type) in enumerate(keyList): - if i >= len(translatedList): - break - translatedText = translatedList[i] - - # Strip speaker prefix if the AI echoed it back - # Handles: [Speaker]: text | Speaker: text | Speaker(text) / CJK(text) - match = re.search(r'^\[.+?\]\s?[|:]\s?', translatedText) - if match: - translatedText = translatedText[match.end():] - else: - # Fallback: strip any leading Japanese/CJK name followed by ( or : - cjk_m = re.match(r'^[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+\s*[\(（:]\s*', translatedText) - if cjk_m: - translatedText = translatedText[cjk_m.end():] - translatedText = re.sub(r'\s*[)）]\s*$', '', translatedText) - - data[key]["tl"] = translatedText - save_progress_json(data, filename) - - # Translate any Japanese speaker names found in the data - seen_speakers = {} - speaker_updated = False - for key, entry in data.items(): - if not isinstance(entry, dict): - continue - speaker = entry.get("speaker", "") - if not speaker or not re.search(LANGREGEX, speaker): - continue - if speaker not in seen_speakers: - response = getSpeaker(speaker) - tokens[0] += response[1][0] - tokens[1] += response[1][1] - seen_speakers[speaker] = response[0] - data[key]["speaker"] = seen_speakers[speaker] - speaker_updated = True - if speaker_updated: - save_progress_json(data, filename) - - return tokens - - -def translateJSON(data, filename, translatedList): - global LOCK, ESTIMATE, FILENAME, PBAR, MISMATCH - if translatedList: - stringList = translatedList[0] - eventList = translatedList[1] - else: - stringList = [] - eventList = [[], [], [], [], [], [], []] # [title, process, text, key, target, job, place] - tokens = [0, 0] - speaker = "" - i = 0 - stringListTL = [] - eventListTL = [[], [], [], [], [], [], []] - - while i < len(data): - speakerKey = "character_nameText" - messageKey = "m_text" - - # Event List Format - Key - if "key" in data[i] and data[i]["key"]: - jaString = data[i]["key"] - - # Pass 1 - if not translatedList: - eventList[3].append(jaString.strip()) - - # Pass 2 - else: - if eventList[3]: - translatedText = eventList[3][0] - eventList[3].pop(0) - - # Set Data - data[i]["key"] = translatedText - save_progress_json(data, filename) - - # Event List Format - Title - if "title" in data[i] and data[i]["title"]: - jaString = data[i]["title"] - - # Pass 1 - if not translatedList: - eventList[0].append(jaString.strip()) - - # Pass 2 - else: - if eventList[0]: - translatedText = eventList[0][0] - eventList[0].pop(0) - - # Set Data - data[i]["title"] = translatedText - save_progress_json(data, filename) - - # Event List Format - Target - if "target" in data[i] and data[i]["target"]: - jaString = data[i]["target"] - - # Pass 1 - if not translatedList: - eventList[4].append(jaString.strip()) - - # Pass 2 - else: - if eventList[4]: - translatedText = eventList[4][0] - eventList[4].pop(0) - - # Set Data - data[i]["target"] = translatedText - save_progress_json(data, filename) - - # Event List Format - Job - if "job" in data[i] and data[i]["job"]: - jaString = data[i]["job"] - - # Pass 1 - if not translatedList: - eventList[5].append(jaString.strip()) - - # Pass 2 - else: - if eventList[5]: - translatedText = eventList[5][0] - eventList[5].pop(0) - - # Set Data - data[i]["job"] = translatedText - save_progress_json(data, filename) - - # Event List Format - Place - if "place" in data[i] and data[i]["place"]: - jaString = data[i]["place"] - - # Pass 1 - if not translatedList: - eventList[6].append(jaString.strip()) - - # Pass 2 - else: - if eventList[6]: - translatedText = eventList[6][0] - eventList[6].pop(0) - - # Set Data - data[i]["place"] = translatedText - save_progress_json(data, filename) - - # Event List Format - Process - if "process" in data[i] and data[i]["process"]: - jaString = data[i]["process"] - - # Pass 1 - if not translatedList: - eventList[1].append(jaString.strip()) - - # Pass 2 - else: - if eventList[1]: - translatedText = eventList[1][0] - eventList[1].pop(0) - - # Set Data - data[i]["process"] = translatedText - save_progress_json(data, filename) - - # Event List Format - Text - if "text" in data[i] and data[i]["text"]: - jaString = data[i]["text"] - # Pass 1 - if not translatedList: - # Replace \n with space for translation - jaStringClean = jaString.replace("\n", " ") - eventList[2].append(jaStringClean.strip()) - - # Pass 2 - else: - if eventList[2]: - translatedText = eventList[2][0] - eventList[2].pop(0) - - # Apply text wrapping and restore linebreaks - translatedText = dazedwrap.wrapText(translatedText, 70) - - # Set Data - data[i]["text"] = translatedText - save_progress_json(data, filename) - - # Speaker - if speakerKey in data[i] and data[i][speakerKey]: - # Grab and TL - speaker = data[i][speakerKey] - response = getSpeaker(speaker) - speaker = response[0] - tokens[0] += response[1][0] - tokens[1] += response[1][1] - - # Set Speaker - data[i][speakerKey] = speaker - - # Dialogue - if messageKey in data[i]\ - and data[i][messageKey].strip()\ - and data[i][messageKey] != "a"\ - and data[i][messageKey].replace("\u3000", "").strip() != "": - jaString = data[i][messageKey] - - # Save Original String - originalString = jaString - - # If there isn't any Japanese in the text just skip - if not re.search(LANGREGEX, jaString): - i += 1 - continue - - # Pass 1 - if not translatedList: - # Strip Spaces - jaString = jaString.strip() - - # Remove Textwrap - # jaString = jaString.replace('\n', ' ') - - if jaString: - if speaker: - stringList.append(f"[{speaker}]: {jaString}") - else: - stringList.append(jaString) - - # Pass 2 - else: - # Get Text - if stringList: - # Grab and Pop - translatedText = stringList[0] - stringList.pop(0) - - # Set to None if empty list - if len(stringList) <= 0: - stringList = None - - # Remove speaker prefix — handles [Name]: / Name: / CJK(text) - match = re.search(r'(^\[.+?\]\s?[|:]\s?)', translatedText) - if match: - translatedText = translatedText.replace(match.group(1), "") - else: - cjk_m = re.match(r'^[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+\s*[\(（:]\s*', translatedText) - if cjk_m: - translatedText = translatedText[cjk_m.end():] - translatedText = re.sub(r'\s*[)）]\s*$', '', translatedText) - - # Escape Quotes - translatedText = re.sub(r'(?", ")") - translatedText = translatedText.replace("『", "") - translatedText = translatedText.replace("』", "") - - # Remove GPT ' Quotes - if translatedText: - if translatedText[0] == "'": - translatedText = translatedText[1:] - if translatedText[-1] == "'": - translatedText = translatedText[:-1] - else: - print("Warning: Empty Translation for", originalString) - - # Textwrap - # translatedText = dazedwrap.wrapText(translatedText, width=WIDTH) - - # Set Data - if "『" in data[i][messageKey] and "』" not in translatedText: - data[i][messageKey] = data[i][messageKey].replace(originalString, f"『{translatedText}』") - else: - data[i][messageKey] = data[i][messageKey].replace(originalString, f"{translatedText}") - - # Save progress after each message replacement - save_progress_json(data, filename) - # Next Value - i += 1 - - # EOF - Only do translation if this is Pass 1 (collecting strings) - if not translatedList: - # Event List - if any(eventList): - PBAR.total = sum(len(event) for event in eventList) - PBAR.refresh() - - # Event Title - if eventList[0]: - response = translateAI(eventList[0], "Event Title") - tokens[0] += response[1][0] - tokens[1] += response[1][1] - eventListTL[0] = response[0] - - if len(eventList[0]) != len(eventListTL[0]): - with LOCK: - if FILENAME not in MISMATCH: - MISMATCH.append(FILENAME) - - # Event Process - if eventList[1]: - response = translateAI(eventList[1], "Event Process") - tokens[0] += response[1][0] - tokens[1] += response[1][1] - eventListTL[1] = response[0] - - if len(eventList[1]) != len(eventListTL[1]): - with LOCK: - if FILENAME not in MISMATCH: - MISMATCH.append(FILENAME) - - # Event Text - if eventList[2]: - response = translateAI(eventList[2], "Event Description") - tokens[0] += response[1][0] - tokens[1] += response[1][1] - eventListTL[2] = response[0] - - if len(eventList[2]) != len(eventListTL[2]): - with LOCK: - if FILENAME not in MISMATCH: - MISMATCH.append(FILENAME) - - # Event Key - if eventList[3]: - response = translateAI(eventList[3], "Event Key") - tokens[0] += response[1][0] - tokens[1] += response[1][1] - eventListTL[3] = response[0] - - if len(eventList[3]) != len(eventListTL[3]): - with LOCK: - if FILENAME not in MISMATCH: - MISMATCH.append(FILENAME) - - # Event Target - if eventList[4]: - response = translateAI(eventList[4], "Character Name") - tokens[0] += response[1][0] - tokens[1] += response[1][1] - eventListTL[4] = response[0] - - if len(eventList[4]) != len(eventListTL[4]): - with LOCK: - if FILENAME not in MISMATCH: - MISMATCH.append(FILENAME) - - # Event Job - if eventList[5]: - response = translateAI(eventList[5], "Job/Occupation") - tokens[0] += response[1][0] - tokens[1] += response[1][1] - eventListTL[5] = response[0] - - if len(eventList[5]) != len(eventListTL[5]): - with LOCK: - if FILENAME not in MISMATCH: - MISMATCH.append(FILENAME) - - # Event Place - if eventList[6]: - response = translateAI(eventList[6], "Location Name") - tokens[0] += response[1][0] - tokens[1] += response[1][1] - eventListTL[6] = response[0] - - if len(eventList[6]) != len(eventListTL[6]): - with LOCK: - if FILENAME not in MISMATCH: - MISMATCH.append(FILENAME) - - # String List - if stringList: - PBAR.total = len(stringList) - PBAR.refresh() - response = translateAI(stringList, "Reply with the English Translation") - tokens[0] += response[1][0] - tokens[1] += response[1][1] - stringListTL = response[0] - - if len(stringList) != len(stringListTL): - # Mismatch - with LOCK: - if FILENAME not in MISMATCH: - MISMATCH.append(FILENAME) - - # Pass 2: Set Strings (recursive call) - translateJSON(data, filename, [stringListTL, eventListTL]) - - return tokens - -# Save some money and enter the character before translation -def getSpeaker(speaker): - match speaker: - case "ファイン": - return ["Fine", [0, 0]] - case "": - return ["", [0, 0]] - case _: - # Find Speaker - for i in range(len(NAMESLIST)): - if speaker == NAMESLIST[i][0]: - return [NAMESLIST[i][1], [0, 0]] - - # Translate and Store Speaker - response = translateAI( - f"{speaker}", - "Reply with the " + LANGUAGE + " translation of the NPC name.", - False, - ) - response[0] = response[0].title() - response[0] = response[0].replace("'S", "'s") - response[0] = response[0].replace("Speaker: ", "") - - # Retry if name doesn't translate for some reason - if re.search(r"([a-zA-Z？?])", response[0]) == None: - response = translateAI( - f"{speaker}", - "Reply with the " + LANGUAGE + " translation of the NPC name.", - False, - ) - response[0] = response[0].title() - response[0] = response[0].replace("'S", "'s") - - speakerList = [speaker, response[0]] - NAMESLIST.append(speakerList) - return response - return [speaker, [0, 0]] - -def translateAI(text, history, history_ctx=None): - """ - Legacy wrapper function for the new shared translation utility. - This maintains compatibility with existing code while using the new shared implementation. - """ - global PBAR, MISMATCH, FILENAME - - # Update config estimate mode based on global ESTIMATE - TRANSLATION_CONFIG.estimateMode = bool(ESTIMATE) - - # Call the new shared translation function - return sharedtranslateAI( - text=text, - history=history, - config=TRANSLATION_CONFIG, - filename=FILENAME, - pbar=PBAR, - lock=LOCK, - mismatchList=MISMATCH - ) +# Libraries +import json +import os +import re +import util.dazedwrap as dazedwrap +import threading +import time +import traceback +import tiktoken +from pathlib import Path +from colorama import Fore +from dotenv import load_dotenv +from retry import retry +from tqdm import tqdm +from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost, getPricingConfig, calculateCost +import tempfile + +# Globals +MODEL = os.getenv("model") +TIMEOUT = int(os.getenv("timeout")) +LANGUAGE = os.getenv("language").capitalize() +from util.paths import PROMPT_PATH, VOCAB_PATH + +PROMPT = PROMPT_PATH.read_text(encoding="utf-8") +VOCAB = VOCAB_PATH.read_text(encoding="utf-8") +LOCK = threading.Lock() +WIDTH = int(os.getenv("width")) +LISTWIDTH = int(os.getenv("listWidth")) +NOTEWIDTH = 70 +MAXHISTORY = 10 +ESTIMATE = "" +TOKENS = [0, 0] +NAMESLIST = [] +FILENAME = None +NAMES = False # Output a list of all the character names found +BRFLAG = False # If the game uses
instead +FIXTEXTWRAP = True # Overwrites textwrap +IGNORETLTEXT = False # Ignores all translated text. +MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong) + +# tqdm Globals +BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}" +POSITION = 0 +LEAVE = False +PBAR = None + +# Regex - Need to change this if you want to translate from/to other languages. Default is Japanese Regex +LANGREGEX = r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+" + +# Get pricing configuration based on the model +PRICING_CONFIG = getPricingConfig(MODEL) +INPUTAPICOST = PRICING_CONFIG["inputAPICost"] +OUTPUTAPICOST = PRICING_CONFIG["outputAPICost"] +BATCHSIZE = PRICING_CONFIG["batchSize"] +FREQUENCY_PENALTY = PRICING_CONFIG["frequencyPenalty"] + +# Initialize Translation Config +TRANSLATION_CONFIG = TranslationConfig( + model=MODEL, + language=LANGUAGE, + prompt=PROMPT, + vocab=VOCAB, + langRegex=LANGREGEX, + batchSize=BATCHSIZE, + maxHistory=MAXHISTORY, + estimateMode=False # Will be set dynamically based on ESTIMATE +) +LEAVE = False + +def handleJSON(filename, estimate): + global ESTIMATE, totalTokens, FILENAME + ESTIMATE = estimate + FILENAME = filename + + if estimate: + start = time.time() + translatedData = openFiles(filename) + + # Print Result + end = time.time() + tqdm.write(getResultString(translatedData, end - start, filename)) + with LOCK: + TOKENS[0] += translatedData[1][0] + TOKENS[1] += translatedData[1][1] + + return getResultString(["", TOKENS, None], end - start, "TOTAL") + + else: + try: + start = time.time() + translatedData = openFiles(filename) + + # Write final result after translation is complete + with open("translated/" + filename, "w", encoding="utf-8", newline="\n") as outFile: + json.dump(translatedData[0], outFile, ensure_ascii=False, indent=4) + + # Print Result + end = time.time() + tqdm.write(getResultString(translatedData, end - start, filename)) + with LOCK: + TOKENS[0] += translatedData[1][0] + TOKENS[1] += translatedData[1][1] + except Exception: + return "Fail" + + return getResultString(["", TOKENS, None], end - start, "TOTAL") + + +def openFiles(filename): + with open("files/" + filename, "r", encoding="UTF-8-sig") as f: + data = json.load(f) + + # Map Files + if ".json" in filename: + translatedData = parseJSON(data, filename) + + else: + raise NameError(filename + " Not Supported") + + return translatedData + + +def getResultString(translatedData, translationTime, filename): + # File Print String + cost = calculateCost(translatedData[1][0], translatedData[1][1], MODEL) + totalTokenstring = ( + Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]" + "[Output: " + + str(translatedData[1][1]) + + "]" "[Cost: ${:,.4f}".format(cost) + + "]" + ) + timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]" + + if translatedData[2] == None: + # Success + return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET + + else: + # Fail + try: + raise translatedData[2] + except Exception as e: + traceback.print_exc() + errorString = str(e) + Fore.RED + return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET + + +def parseJSON(data, filename): + totalTokens = [0, 0] + totalLines = 0 + totalLines = len(data) + global LOCK, PBAR + + with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar: + pbar.desc = filename + PBAR = pbar + try: + # Check if data is a simple key-value dict (not a list) + if isinstance(data, dict) and not isinstance(data, list): + # Check if it's a simple key-value format (all values are strings, not nested objects) + if all(isinstance(v, str) for v in data.values()): + result = translateSimpleKeyValueJSON(data, filename) + # PSData format (Nupu_PSData.json - MailDatas, MemoryData, ShopDatas, hStData) + elif any(k in data for k in ["MailDatas", "MemoryData", "ShopDatas", "hStData"]): + result = translatePSData(data, filename) + # RdData format (RdData.json - dgnDatas with dungeon/dialogue data) + elif "dgnDatas" in data: + result = translateRdData(data, filename) + # GameSetting format (Nupu_GameSetting.json - fzCardDatas, fzDeckDatas, rentalDecks) + elif any(k in data for k in ["fzCardDatas", "fzDeckDatas", "rentalDecks"]): + result = translateGameSetting(data, filename) + # Src/Tl dialogue format (numeric-keyed entries with src and tl fields) + elif data and all(isinstance(v, dict) and "src" in v for v in data.values()): + result = translateDialogueSrcTl(data, filename) + else: + result = translateJSON(data, filename, []) + else: + result = translateJSON(data, filename, []) + totalTokens[0] += result[0] + totalTokens[1] += result[1] + except Exception as e: + traceback.print_exc() + return [data, totalTokens, e] + return [data, totalTokens, None] + + +def save_progress_json(data, filename): + """Save current JSON translation progress.""" + try: + if ESTIMATE: + return + os.makedirs("translated", exist_ok=True) + tmp_path = os.path.join("translated", f"{filename}.tmp") + final_path = os.path.join("translated", filename) + with open(tmp_path, "w", encoding="utf-8", newline="\n") as outFile: + json.dump(data, outFile, ensure_ascii=False, indent=4) + os.replace(tmp_path, final_path) + except Exception: + traceback.print_exc() + + +def translateSimpleKeyValueJSON(data, filename): + """ + Translate simple key-value JSON format where keys contain Japanese text + and values are placeholder strings like "TODO". + + Format example: + { + "\\r\\nスタビライザー": "TODO", + "\\r\\nプラグインなし": "TODO" + } + """ + global LOCK, ESTIMATE, FILENAME, PBAR, MISMATCH + tokens = [0, 0] + stringList = [] + keyList = [] + + # Pass 1: Collect all keys that need translation + for key, value in data.items(): + # Check if the key contains text matching the language regex + if re.search(LANGREGEX, key): + # Strip whitespace and newline characters for translation + # cleanKey = key.strip().replace("\r\n", "").replace("\n", "").replace("\r", "") + # if cleanKey: + stringList.append(key) + keyList.append(key) # Store original key for reference + + # Translate all keys if any were found + if stringList: + PBAR.total = len(stringList) + PBAR.refresh() + + response = translateAI(stringList, "Reply with the English Translation") + tokens[0] += response[1][0] + tokens[1] += response[1][1] + translatedList = response[0] + + # Check for mismatch + if len(stringList) != len(translatedList): + with LOCK: + if FILENAME not in MISMATCH: + MISMATCH.append(FILENAME) + + # Pass 2: Update the values with translations + for i, key in enumerate(keyList): + if i < len(translatedList): + translatedText = translatedList[i] + # Set the translated text as the value + data[key] = translatedText + + # Save progress after each translation + save_progress_json(data, filename) + + return tokens + + +def translatePSData(data, filename): + """Translate PSData JSON format (e.g. Nupu_PSData.json). + + Handles four sections: + - MailDatas: In-game mail/message system + - MemoryData: Scene/CG gallery memories + - ShopDatas: Shop configurations and dialogue + - hStData: Status rank descriptions + + Only translates player-visible text fields. + """ + global LOCK, ESTIMATE, FILENAME, PBAR, MISMATCH + tokens = [0, 0] + + def batchTranslate(stringList, context): + """Translate a list of strings and return translations. Returns [] on mismatch.""" + nonlocal tokens + if not stringList: + return [] + response = translateAI(stringList, context) + tokens[0] += response[1][0] + tokens[1] += response[1][1] + if len(stringList) != len(response[0]): + with LOCK: + if FILENAME not in MISMATCH: + MISMATCH.append(FILENAME) + return [] + return response[0] + + # ================================================================ + # PASS 1: Collect all translatable strings from every section + # ================================================================ + + # -- MailDatas -- + mailNameS, mailNameI = [], [] + mailTitleS, mailTitleI = [], [] + mailListNameS, mailListNameI = [], [] + mailTextS, mailTextI = [], [] + + if "MailDatas" in data: + for i, m in enumerate(data["MailDatas"]): + if m.get("Name"): + mailNameS.append(m["Name"]) + mailNameI.append(i) + if m.get("Title"): + mailTitleS.append(m["Title"]) + mailTitleI.append(i) + if m.get("MailListName"): + mailListNameS.append(m["MailListName"]) + mailListNameI.append(i) + if m.get("MailText"): + mailTextS.append(m["MailText"].replace("\r\n", " ").replace("\n", " ").strip()) + mailTextI.append(i) + + # -- MemoryData -- + memTitleS, memTitleI = [], [] + memSetuS, memSetuI = [], [] + memHintS, memHintI = [], [] + memPtnS, memPtnI = [], [] # indices are (memIdx, ptnIdx) + memDtlSetuS, memDtlSetuI = [], [] # indices are (memIdx, dtlIdx) + + if "MemoryData" in data: + for i, mem in enumerate(data["MemoryData"]): + if mem.get("_Title"): + memTitleS.append(mem["_Title"]) + memTitleI.append(i) + if mem.get("_Setu"): + memSetuS.append(mem["_Setu"].replace("\r\n", " ").replace("\n", " ").strip()) + memSetuI.append(i) + if mem.get("_Hint"): + memHintS.append(mem["_Hint"]) + memHintI.append(i) + if "_Ptn" in mem and isinstance(mem["_Ptn"], list): + for j, p in enumerate(mem["_Ptn"]): + if p: + memPtnS.append(p) + memPtnI.append((i, j)) + if "_DtlCheck" in mem and isinstance(mem["_DtlCheck"], list): + for j, dtl in enumerate(mem["_DtlCheck"]): + if isinstance(dtl, dict) and dtl.get("_Setu"): + memDtlSetuS.append(dtl["_Setu"].replace("\r\n", " ").replace("\n", " ").strip()) + memDtlSetuI.append((i, j)) + + # -- ShopDatas -- + shopNameS, shopNameI = [], [] + shopTextFields = ["buySelf", "emptySelf", "firstSelf", "husokuSelf", + "randSelf1", "randSelf2", "randSelf3"] + shopTextS = {f: [] for f in shopTextFields} + shopTextI = {f: [] for f in shopTextFields} + + if "ShopDatas" in data: + for i, shop in enumerate(data["ShopDatas"]): + if shop.get("name"): + shopNameS.append(shop["name"]) + shopNameI.append(i) + for field in shopTextFields: + if field in shop and isinstance(shop[field], dict): + text = shop[field].get("text", "") + if text: + shopTextS[field].append(text.replace("\r\n", " ").replace("\n", " ").strip()) + shopTextI[field].append(i) + + # -- hStData -- + rankArrayNames = ["KutiRankDatas", "MuneRankDatas", "SiriRankDatas", "TituRankDatas"] + rankSetuS, rankSetuI = [], [] # indices are (arrayName, idx) + rankText1S, rankText1I = [], [] + rankText2S, rankText2I = [], [] + + if "hStData" in data: + hst = data["hStData"] + for arrName in rankArrayNames: + if arrName in hst and isinstance(hst[arrName], list): + for i, rank in enumerate(hst[arrName]): + if rank.get("rankSetu"): + rankSetuS.append(rank["rankSetu"].replace("\r\n", " ").replace("\n", " ").strip()) + rankSetuI.append((arrName, i)) + if rank.get("rankText1"): + rankText1S.append(rank["rankText1"]) + rankText1I.append((arrName, i)) + if rank.get("rankText2"): + rankText2S.append(rank["rankText2"]) + rankText2I.append((arrName, i)) + + # Set progress bar total + totalItems = ( + len(mailNameS) + len(mailTitleS) + len(mailListNameS) + len(mailTextS) + + len(memTitleS) + len(memSetuS) + len(memHintS) + len(memPtnS) + len(memDtlSetuS) + + len(shopNameS) + sum(len(v) for v in shopTextS.values()) + + len(rankSetuS) + len(rankText1S) + len(rankText2S) + ) + PBAR.total = totalItems + PBAR.refresh() + + # ================================================================ + # PASS 2: Translate each batch and apply results back to data + # ================================================================ + + # -- MailDatas -- + if "MailDatas" in data: + mails = data["MailDatas"] + + for tl, idx in zip(batchTranslate(mailNameS, "Character Name"), mailNameI): + mails[idx]["Name"] = tl + if mailNameS: + save_progress_json(data, filename) + + for tl, idx in zip(batchTranslate(mailTitleS, "Mail Subject"), mailTitleI): + mails[idx]["Title"] = tl + if mailTitleS: + save_progress_json(data, filename) + + for tl, idx in zip(batchTranslate(mailListNameS, "Mail List Entry"), mailListNameI): + mails[idx]["MailListName"] = tl + if mailListNameS: + save_progress_json(data, filename) + + for tl, idx in zip(batchTranslate(mailTextS, "Mail Body Text"), mailTextI): + mails[idx]["MailText"] = dazedwrap.wrapText(tl, WIDTH) + if mailTextS: + save_progress_json(data, filename) + + # -- MemoryData -- + if "MemoryData" in data: + memories = data["MemoryData"] + + for tl, idx in zip(batchTranslate(memTitleS, "Scene Title"), memTitleI): + memories[idx]["_Title"] = tl + if memTitleS: + save_progress_json(data, filename) + + for tl, idx in zip(batchTranslate(memSetuS, "Scene Description"), memSetuI): + memories[idx]["_Setu"] = tl + if memSetuS: + save_progress_json(data, filename) + + for tl, idx in zip(batchTranslate(memHintS, "Hint Text"), memHintI): + memories[idx]["_Hint"] = tl + if memHintS: + save_progress_json(data, filename) + + for tl, (mi, pi) in zip(batchTranslate(memPtnS, "Scene Description"), memPtnI): + memories[mi]["_Ptn"][pi] = tl + if memPtnS: + save_progress_json(data, filename) + + for tl, (mi, di) in zip(batchTranslate(memDtlSetuS, "Character Commentary"), memDtlSetuI): + memories[mi]["_DtlCheck"][di]["_Setu"] = dazedwrap.wrapText(tl, LISTWIDTH) + if memDtlSetuS: + save_progress_json(data, filename) + + # -- ShopDatas -- + if "ShopDatas" in data: + shops = data["ShopDatas"] + + for tl, idx in zip(batchTranslate(shopNameS, "Shop Name"), shopNameI): + shops[idx]["name"] = tl + if shopNameS: + save_progress_json(data, filename) + + for field in shopTextFields: + for tl, idx in zip(batchTranslate(shopTextS[field], "Shop Dialogue"), shopTextI[field]): + shops[idx][field]["text"] = dazedwrap.wrapText(tl, WIDTH) + if shopTextS[field]: + save_progress_json(data, filename) + + # -- hStData -- + if "hStData" in data: + hst = data["hStData"] + + for tl, (an, idx) in zip(batchTranslate(rankSetuS, "Rank Description"), rankSetuI): + hst[an][idx]["rankSetu"] = dazedwrap.wrapText(tl, WIDTH) + if rankSetuS: + save_progress_json(data, filename) + + for tl, (an, idx) in zip(batchTranslate(rankText1S, "Rank Label"), rankText1I): + hst[an][idx]["rankText1"] = tl + if rankText1S: + save_progress_json(data, filename) + + for tl, (an, idx) in zip(batchTranslate(rankText2S, "Rank Subtitle"), rankText2I): + hst[an][idx]["rankText2"] = tl + if rankText2S: + save_progress_json(data, filename) + + return tokens + + +def translateRdData(data, filename): + """Translate RdData JSON format (e.g. RdData.json). + + Handles dungeon data with nested dialogue, choices, and UI text. + Only translates player-visible text fields. + """ + global LOCK, ESTIMATE, FILENAME, PBAR, MISMATCH + tokens = [0, 0] + + def batchTranslate(stringList, context): + """Translate a list of strings and return translations. Returns [] on mismatch.""" + nonlocal tokens + if not stringList: + return [] + response = translateAI(stringList, context) + tokens[0] += response[1][0] + tokens[1] += response[1][1] + if len(stringList) != len(response[0]): + with LOCK: + if FILENAME not in MISMATCH: + MISMATCH.append(FILENAME) + return [] + return response[0] + + if "dgnDatas" not in data: + return tokens + + dgns = data["dgnDatas"] + + # ================================================================ + # PASS 1: Collect all translatable strings + # ================================================================ + + # -- Dungeon-level short fields -- + dgnNameS, dgnNameI = [], [] + clearTxtS, clearTxtI = [], [] + hosyuTxtS, hosyuTxtI = [], [] + firstHosyuTxtS, firstHosyuTxtI = [], [] + memoTxtS, memoTxtI = [], [] + appMemoTxtS, appMemoTxtI = [], [] + areaMemoS, areaMemoI = [], [] + + # -- Dungeon-level long text fields -- + naiyoTxtS, naiyoTxtI = [], [] + naiyoTxtExS, naiyoTxtExI = [], [] + johoTxtS, johoTxtI = [], [] + johoTxtExS, johoTxtExI = [], [] + + # -- Floor names -- + kaisoNameS, kaisoNameI = [], [] # (dgnIdx, kaisoIdx) + + # -- Dialogue (talkDatas / talkDatasEx) -- + talkNameS, talkNameI = [], [] # (dgnIdx, talkKey, talkIdx) + speakerNameS, speakerNameI = [], [] # (dgnIdx, talkKey, talkIdx, speechIdx) + dialogueS, dialogueI = [], [] # (dgnIdx, talkKey, talkIdx, speechIdx) + + # -- Choice events (sijiDatas) -- + sijiNameS, sijiNameI = [], [] # (dgnIdx, sijiIdx) + sijiChoiceS, sijiChoiceI = [], [] # (dgnIdx, sijiIdx) + + for di, dgn in enumerate(dgns): + if dgn is None: + continue + + # Dungeon short fields + if dgn.get("name"): + dgnNameS.append(dgn["name"]) + dgnNameI.append(di) + if dgn.get("clearTxt"): + clearTxtS.append(dgn["clearTxt"]) + clearTxtI.append(di) + if dgn.get("hosyuTxt"): + hosyuTxtS.append(dgn["hosyuTxt"]) + hosyuTxtI.append(di) + if dgn.get("firstHosyuTxt"): + firstHosyuTxtS.append(dgn["firstHosyuTxt"]) + firstHosyuTxtI.append(di) + if dgn.get("memoTxt"): + memoTxtS.append(dgn["memoTxt"]) + memoTxtI.append(di) + if dgn.get("appMemoTxt"): + appMemoTxtS.append(dgn["appMemoTxt"]) + appMemoTxtI.append(di) + if dgn.get("areaMemo"): + areaMemoS.append(dgn["areaMemo"]) + areaMemoI.append(di) + + # Dungeon long text fields (strip line breaks for translation) + if dgn.get("naiyoTxt"): + naiyoTxtS.append(dgn["naiyoTxt"].replace("\r\n", " ").replace("\n", " ").strip()) + naiyoTxtI.append(di) + if dgn.get("naiyoTxtEx"): + naiyoTxtExS.append(dgn["naiyoTxtEx"].replace("\r\n", " ").replace("\n", " ").strip()) + naiyoTxtExI.append(di) + if dgn.get("johoTxt"): + johoTxtS.append(dgn["johoTxt"].replace("\r\n", " ").replace("\n", " ").strip()) + johoTxtI.append(di) + if dgn.get("johoTxtEx"): + johoTxtExS.append(dgn["johoTxtEx"].replace("\r\n", " ").replace("\n", " ").strip()) + johoTxtExI.append(di) + + # Floor names + if "kaisoDatas" in dgn and isinstance(dgn["kaisoDatas"], list): + for ki, kaiso in enumerate(dgn["kaisoDatas"]): + if isinstance(kaiso, dict) and kaiso.get("kaisoName"): + kaisoNameS.append(kaiso["kaisoName"]) + kaisoNameI.append((di, ki)) + + # Dialogue from talkDatas and talkDatasEx + for talkKey in ["talkDatas", "talkDatasEx"]: + if talkKey in dgn and isinstance(dgn[talkKey], list): + for ti, talk in enumerate(dgn[talkKey]): + if not isinstance(talk, dict): + continue + if talk.get("name"): + talkNameS.append(talk["name"]) + talkNameI.append((di, talkKey, ti)) + if "speachDatas" in talk and isinstance(talk["speachDatas"], list): + for si, speech in enumerate(talk["speachDatas"]): + if not isinstance(speech, dict): + continue + if speech.get("name"): + speakerNameS.append(speech["name"]) + speakerNameI.append((di, talkKey, ti, si)) + if speech.get("text"): + dialogueS.append(speech["text"].replace("\r\n", " ").replace("\n", " ").strip()) + dialogueI.append((di, talkKey, ti, si)) + + # Choice events (sijiDatas) + if "sijiDatas" in dgn and isinstance(dgn["sijiDatas"], list): + for si, siji in enumerate(dgn["sijiDatas"]): + if not isinstance(siji, dict): + continue + if siji.get("name"): + sijiNameS.append(siji["name"]) + sijiNameI.append((di, si)) + if siji.get("choiceSijiTxt"): + sijiChoiceS.append(siji["choiceSijiTxt"].replace("\r\n", " ").replace("\n", " ").strip()) + sijiChoiceI.append((di, si)) + + # Set progress bar total + totalItems = ( + len(dgnNameS) + len(clearTxtS) + len(hosyuTxtS) + len(firstHosyuTxtS) + + len(memoTxtS) + len(appMemoTxtS) + len(areaMemoS) + + len(naiyoTxtS) + len(naiyoTxtExS) + len(johoTxtS) + len(johoTxtExS) + + len(kaisoNameS) + + len(talkNameS) + len(speakerNameS) + len(dialogueS) + + len(sijiNameS) + len(sijiChoiceS) + ) + PBAR.total = totalItems + PBAR.refresh() + + # ================================================================ + # PASS 2: Translate each batch and apply results back to data + # ================================================================ + + # -- Dungeon short fields -- + for tl, idx in zip(batchTranslate(dgnNameS, "Dungeon Name"), dgnNameI): + dgns[idx]["name"] = tl + if dgnNameS: + save_progress_json(data, filename) + + for tl, idx in zip(batchTranslate(clearTxtS, "Clear Condition"), clearTxtI): + dgns[idx]["clearTxt"] = tl + if clearTxtS: + save_progress_json(data, filename) + + for tl, idx in zip(batchTranslate(hosyuTxtS, "Reward Description"), hosyuTxtI): + dgns[idx]["hosyuTxt"] = tl + if hosyuTxtS: + save_progress_json(data, filename) + + for tl, idx in zip(batchTranslate(firstHosyuTxtS, "First Clear Reward"), firstHosyuTxtI): + dgns[idx]["firstHosyuTxt"] = tl + if firstHosyuTxtS: + save_progress_json(data, filename) + + for tl, idx in zip(batchTranslate(memoTxtS, "Memo Label"), memoTxtI): + dgns[idx]["memoTxt"] = tl + if memoTxtS: + save_progress_json(data, filename) + + for tl, idx in zip(batchTranslate(appMemoTxtS, "App Memo"), appMemoTxtI): + dgns[idx]["appMemoTxt"] = tl + if appMemoTxtS: + save_progress_json(data, filename) + + for tl, idx in zip(batchTranslate(areaMemoS, "Area Label"), areaMemoI): + dgns[idx]["areaMemo"] = tl + if areaMemoS: + save_progress_json(data, filename) + + # -- Dungeon long text fields (with word wrap) -- + for tl, idx in zip(batchTranslate(naiyoTxtS, "Mission Description"), naiyoTxtI): + dgns[idx]["naiyoTxt"] = dazedwrap.wrapText(tl, WIDTH) + if naiyoTxtS: + save_progress_json(data, filename) + + for tl, idx in zip(batchTranslate(naiyoTxtExS, "Extended Mission Description"), naiyoTxtExI): + dgns[idx]["naiyoTxtEx"] = dazedwrap.wrapText(tl, WIDTH) + if naiyoTxtExS: + save_progress_json(data, filename) + + for tl, idx in zip(batchTranslate(johoTxtS, "Info Text"), johoTxtI): + dgns[idx]["johoTxt"] = dazedwrap.wrapText(tl, WIDTH) + if johoTxtS: + save_progress_json(data, filename) + + for tl, idx in zip(batchTranslate(johoTxtExS, "Extended Info Text"), johoTxtExI): + dgns[idx]["johoTxtEx"] = dazedwrap.wrapText(tl, WIDTH) + if johoTxtExS: + save_progress_json(data, filename) + + # -- Floor names -- + for tl, (di, ki) in zip(batchTranslate(kaisoNameS, "Floor Name"), kaisoNameI): + dgns[di]["kaisoDatas"][ki]["kaisoName"] = tl + if kaisoNameS: + save_progress_json(data, filename) + + # -- Dialogue event names -- + for tl, (di, tk, ti) in zip(batchTranslate(talkNameS, "Event Label"), talkNameI): + dgns[di][tk][ti]["name"] = tl + if talkNameS: + save_progress_json(data, filename) + + # -- Speaker names -- + for tl, (di, tk, ti, si) in zip(batchTranslate(speakerNameS, "Character Name"), speakerNameI): + dgns[di][tk][ti]["speachDatas"][si]["name"] = tl + if speakerNameS: + save_progress_json(data, filename) + + # -- Dialogue text (with word wrap) -- + for tl, (di, tk, ti, si) in zip(batchTranslate(dialogueS, "Dialogue"), dialogueI): + dgns[di][tk][ti]["speachDatas"][si]["text"] = dazedwrap.wrapText(tl, WIDTH) + if dialogueS: + save_progress_json(data, filename) + + # -- Choice event names -- + for tl, (di, si) in zip(batchTranslate(sijiNameS, "Event Name"), sijiNameI): + dgns[di]["sijiDatas"][si]["name"] = tl + if sijiNameS: + save_progress_json(data, filename) + + # -- Choice text (with word wrap) -- + for tl, (di, si) in zip(batchTranslate(sijiChoiceS, "Choice Event Text"), sijiChoiceI): + dgns[di]["sijiDatas"][si]["choiceSijiTxt"] = dazedwrap.wrapText(tl, WIDTH) + if sijiChoiceS: + save_progress_json(data, filename) + + return tokens + + +def translateGameSetting(data, filename): + """Translate GameSetting JSON format (e.g. Nupu_GameSetting.json). + + Handles card data, deck definitions, and rental decks. + Only translates player-visible text fields. + """ + global LOCK, ESTIMATE, FILENAME, PBAR, MISMATCH + tokens = [0, 0] + + def batchTranslate(stringList, context): + """Translate a list of strings and return translations. Returns [] on mismatch.""" + nonlocal tokens + if not stringList: + return [] + response = translateAI(stringList, context) + tokens[0] += response[1][0] + tokens[1] += response[1][1] + if len(stringList) != len(response[0]): + with LOCK: + if FILENAME not in MISMATCH: + MISMATCH.append(FILENAME) + return [] + return response[0] + + # ================================================================ + # PASS 1: Collect all translatable strings + # ================================================================ + + # -- fzCardDatas: card descriptions -- + desc1S, desc1I = [], [] + desc2S, desc2I = [], [] + + if "fzCardDatas" in data: + for i, card in enumerate(data["fzCardDatas"]): + if card.get("desc1"): + desc1S.append(card["desc1"].replace("\r\n", " ").replace("\n", " ").strip()) + desc1I.append(i) + if card.get("desc2"): + desc2S.append(card["desc2"].replace("\r\n", " ").replace("\n", " ").strip()) + desc2I.append(i) + + # -- fzDeckDatas: deck name and description -- + deckNameS, deckNameI = [], [] + deckDescS, deckDescI = [], [] + + if "fzDeckDatas" in data: + for i, deck in enumerate(data["fzDeckDatas"]): + if deck.get("name"): + deckNameS.append(deck["name"]) + deckNameI.append(i) + if deck.get("desc"): + deckDescS.append(deck["desc"].replace("\r\n", " ").replace("\n", " ").strip()) + deckDescI.append(i) + + # -- rentalDecks: memo label -- + rentalMemoS, rentalMemoI = [], [] + + if "rentalDecks" in data: + for i, rental in enumerate(data["rentalDecks"]): + if rental.get("memo"): + rentalMemoS.append(rental["memo"]) + rentalMemoI.append(i) + + # Set progress bar total + totalItems = ( + len(desc1S) + len(desc2S) + + len(deckNameS) + len(deckDescS) + + len(rentalMemoS) + ) + PBAR.total = totalItems + PBAR.refresh() + + # ================================================================ + # PASS 2: Translate each batch and apply results back to data + # ================================================================ + + # -- fzCardDatas -- + if "fzCardDatas" in data: + cards = data["fzCardDatas"] + + for tl, idx in zip(batchTranslate(desc1S, "Card Description"), desc1I): + cards[idx]["desc1"] = dazedwrap.wrapText(tl, WIDTH) + if desc1S: + save_progress_json(data, filename) + + for tl, idx in zip(batchTranslate(desc2S, "Card Description"), desc2I): + cards[idx]["desc2"] = dazedwrap.wrapText(tl, WIDTH) + if desc2S: + save_progress_json(data, filename) + + # -- fzDeckDatas -- + if "fzDeckDatas" in data: + decks = data["fzDeckDatas"] + + for tl, idx in zip(batchTranslate(deckNameS, "Deck Name"), deckNameI): + decks[idx]["name"] = tl + if deckNameS: + save_progress_json(data, filename) + + for tl, idx in zip(batchTranslate(deckDescS, "Deck Description"), deckDescI): + decks[idx]["desc"] = dazedwrap.wrapText(tl, WIDTH) + if deckDescS: + save_progress_json(data, filename) + + # -- rentalDecks -- + if "rentalDecks" in data: + rentals = data["rentalDecks"] + + for tl, idx in zip(batchTranslate(rentalMemoS, "Deck Label"), rentalMemoI): + rentals[idx]["memo"] = tl + if rentalMemoS: + save_progress_json(data, filename) + + return tokens + + +def translateDialogueSrcTl(data, filename): + """Translate src/tl dialogue JSON format. + + Format: + { + "0": {"type": "dialogue", "src": "Japanese text", "tl": ""}, + "4": {"type": "dialogue", "speaker": "琴音", "src": "「...」", "tl": ""} + } + + Translates 'src' into 'tl'. Also translates Japanese 'speaker' fields. + Skips entries where 'tl' is already populated. + """ + global LOCK, ESTIMATE, FILENAME, PBAR, MISMATCH + tokens = [0, 0] + srcList = [] + keyList = [] # list of (key, entry_type) + + # Collect all entries with translatable content + for key, entry in data.items(): + if not isinstance(entry, dict): + continue + src = entry.get("src", "") + if not src: + continue + tl = entry.get("tl", "") + entry_type = entry.get("type", "") + speaker = entry.get("speaker", "") + + if tl: + # Already has a translation — send tl so the module can decide to skip it + srcList.append(tl) + else: + if not re.search(LANGREGEX, src): + continue + srcClean = src.replace("\r\n", " ").replace("\n", " ").strip() + if speaker: + srcList.append(f"[{speaker}]: {srcClean}") + else: + srcList.append(srcClean) + keyList.append((key, entry_type)) + + if srcList: + PBAR.total = len(srcList) + PBAR.refresh() + + response = translateAI(srcList, "Reply with the English Translation") + tokens[0] += response[1][0] + tokens[1] += response[1][1] + translatedList = response[0] + + if len(srcList) != len(translatedList): + with LOCK: + if FILENAME not in MISMATCH: + MISMATCH.append(FILENAME) + + for i, (key, entry_type) in enumerate(keyList): + if i >= len(translatedList): + break + translatedText = translatedList[i] + + # Strip speaker prefix if the AI echoed it back + # Handles: [Speaker]: text | Speaker: text | Speaker(text) / CJK(text) + match = re.search(r'^\[.+?\]\s?[|:]\s?', translatedText) + if match: + translatedText = translatedText[match.end():] + else: + # Fallback: strip any leading Japanese/CJK name followed by ( or : + cjk_m = re.match(r'^[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+\s*[\(（:]\s*', translatedText) + if cjk_m: + translatedText = translatedText[cjk_m.end():] + translatedText = re.sub(r'\s*[)）]\s*$', '', translatedText) + + data[key]["tl"] = translatedText + save_progress_json(data, filename) + + # Translate any Japanese speaker names found in the data + seen_speakers = {} + speaker_updated = False + for key, entry in data.items(): + if not isinstance(entry, dict): + continue + speaker = entry.get("speaker", "") + if not speaker or not re.search(LANGREGEX, speaker): + continue + if speaker not in seen_speakers: + response = getSpeaker(speaker) + tokens[0] += response[1][0] + tokens[1] += response[1][1] + seen_speakers[speaker] = response[0] + data[key]["speaker"] = seen_speakers[speaker] + speaker_updated = True + if speaker_updated: + save_progress_json(data, filename) + + return tokens + + +def translateJSON(data, filename, translatedList): + global LOCK, ESTIMATE, FILENAME, PBAR, MISMATCH + if translatedList: + stringList = translatedList[0] + eventList = translatedList[1] + else: + stringList = [] + eventList = [[], [], [], [], [], [], []] # [title, process, text, key, target, job, place] + tokens = [0, 0] + speaker = "" + i = 0 + stringListTL = [] + eventListTL = [[], [], [], [], [], [], []] + + while i < len(data): + speakerKey = "character_nameText" + messageKey = "m_text" + + # Event List Format - Key + if "key" in data[i] and data[i]["key"]: + jaString = data[i]["key"] + + # Pass 1 + if not translatedList: + eventList[3].append(jaString.strip()) + + # Pass 2 + else: + if eventList[3]: + translatedText = eventList[3][0] + eventList[3].pop(0) + + # Set Data + data[i]["key"] = translatedText + save_progress_json(data, filename) + + # Event List Format - Title + if "title" in data[i] and data[i]["title"]: + jaString = data[i]["title"] + + # Pass 1 + if not translatedList: + eventList[0].append(jaString.strip()) + + # Pass 2 + else: + if eventList[0]: + translatedText = eventList[0][0] + eventList[0].pop(0) + + # Set Data + data[i]["title"] = translatedText + save_progress_json(data, filename) + + # Event List Format - Target + if "target" in data[i] and data[i]["target"]: + jaString = data[i]["target"] + + # Pass 1 + if not translatedList: + eventList[4].append(jaString.strip()) + + # Pass 2 + else: + if eventList[4]: + translatedText = eventList[4][0] + eventList[4].pop(0) + + # Set Data + data[i]["target"] = translatedText + save_progress_json(data, filename) + + # Event List Format - Job + if "job" in data[i] and data[i]["job"]: + jaString = data[i]["job"] + + # Pass 1 + if not translatedList: + eventList[5].append(jaString.strip()) + + # Pass 2 + else: + if eventList[5]: + translatedText = eventList[5][0] + eventList[5].pop(0) + + # Set Data + data[i]["job"] = translatedText + save_progress_json(data, filename) + + # Event List Format - Place + if "place" in data[i] and data[i]["place"]: + jaString = data[i]["place"] + + # Pass 1 + if not translatedList: + eventList[6].append(jaString.strip()) + + # Pass 2 + else: + if eventList[6]: + translatedText = eventList[6][0] + eventList[6].pop(0) + + # Set Data + data[i]["place"] = translatedText + save_progress_json(data, filename) + + # Event List Format - Process + if "process" in data[i] and data[i]["process"]: + jaString = data[i]["process"] + + # Pass 1 + if not translatedList: + eventList[1].append(jaString.strip()) + + # Pass 2 + else: + if eventList[1]: + translatedText = eventList[1][0] + eventList[1].pop(0) + + # Set Data + data[i]["process"] = translatedText + save_progress_json(data, filename) + + # Event List Format - Text + if "text" in data[i] and data[i]["text"]: + jaString = data[i]["text"] + # Pass 1 + if not translatedList: + # Replace \n with space for translation + jaStringClean = jaString.replace("\n", " ") + eventList[2].append(jaStringClean.strip()) + + # Pass 2 + else: + if eventList[2]: + translatedText = eventList[2][0] + eventList[2].pop(0) + + # Apply text wrapping and restore linebreaks + translatedText = dazedwrap.wrapText(translatedText, 70) + + # Set Data + data[i]["text"] = translatedText + save_progress_json(data, filename) + + # Speaker + if speakerKey in data[i] and data[i][speakerKey]: + # Grab and TL + speaker = data[i][speakerKey] + response = getSpeaker(speaker) + speaker = response[0] + tokens[0] += response[1][0] + tokens[1] += response[1][1] + + # Set Speaker + data[i][speakerKey] = speaker + + # Dialogue + if messageKey in data[i]\ + and data[i][messageKey].strip()\ + and data[i][messageKey] != "a"\ + and data[i][messageKey].replace("\u3000", "").strip() != "": + jaString = data[i][messageKey] + + # Save Original String + originalString = jaString + + # If there isn't any Japanese in the text just skip + if not re.search(LANGREGEX, jaString): + i += 1 + continue + + # Pass 1 + if not translatedList: + # Strip Spaces + jaString = jaString.strip() + + # Remove Textwrap + # jaString = jaString.replace('\n', ' ') + + if jaString: + if speaker: + stringList.append(f"[{speaker}]: {jaString}") + else: + stringList.append(jaString) + + # Pass 2 + else: + # Get Text + if stringList: + # Grab and Pop + translatedText = stringList[0] + stringList.pop(0) + + # Set to None if empty list + if len(stringList) <= 0: + stringList = None + + # Remove speaker prefix — handles [Name]: / Name: / CJK(text) + match = re.search(r'(^\[.+?\]\s?[|:]\s?)', translatedText) + if match: + translatedText = translatedText.replace(match.group(1), "") + else: + cjk_m = re.match(r'^[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+\s*[\(（:]\s*', translatedText) + if cjk_m: + translatedText = translatedText[cjk_m.end():] + translatedText = re.sub(r'\s*[)）]\s*$', '', translatedText) + + # Escape Quotes + translatedText = re.sub(r'(?", ")") + translatedText = translatedText.replace("『", "") + translatedText = translatedText.replace("』", "") + + # Remove GPT ' Quotes + if translatedText: + if translatedText[0] == "'": + translatedText = translatedText[1:] + if translatedText[-1] == "'": + translatedText = translatedText[:-1] + else: + print("Warning: Empty Translation for", originalString) + + # Textwrap + # translatedText = dazedwrap.wrapText(translatedText, width=WIDTH) + + # Set Data + if "『" in data[i][messageKey] and "』" not in translatedText: + data[i][messageKey] = data[i][messageKey].replace(originalString, f"『{translatedText}』") + else: + data[i][messageKey] = data[i][messageKey].replace(originalString, f"{translatedText}") + + # Save progress after each message replacement + save_progress_json(data, filename) + # Next Value + i += 1 + + # EOF - Only do translation if this is Pass 1 (collecting strings) + if not translatedList: + # Event List + if any(eventList): + PBAR.total = sum(len(event) for event in eventList) + PBAR.refresh() + + # Event Title + if eventList[0]: + response = translateAI(eventList[0], "Event Title") + tokens[0] += response[1][0] + tokens[1] += response[1][1] + eventListTL[0] = response[0] + + if len(eventList[0]) != len(eventListTL[0]): + with LOCK: + if FILENAME not in MISMATCH: + MISMATCH.append(FILENAME) + + # Event Process + if eventList[1]: + response = translateAI(eventList[1], "Event Process") + tokens[0] += response[1][0] + tokens[1] += response[1][1] + eventListTL[1] = response[0] + + if len(eventList[1]) != len(eventListTL[1]): + with LOCK: + if FILENAME not in MISMATCH: + MISMATCH.append(FILENAME) + + # Event Text + if eventList[2]: + response = translateAI(eventList[2], "Event Description") + tokens[0] += response[1][0] + tokens[1] += response[1][1] + eventListTL[2] = response[0] + + if len(eventList[2]) != len(eventListTL[2]): + with LOCK: + if FILENAME not in MISMATCH: + MISMATCH.append(FILENAME) + + # Event Key + if eventList[3]: + response = translateAI(eventList[3], "Event Key") + tokens[0] += response[1][0] + tokens[1] += response[1][1] + eventListTL[3] = response[0] + + if len(eventList[3]) != len(eventListTL[3]): + with LOCK: + if FILENAME not in MISMATCH: + MISMATCH.append(FILENAME) + + # Event Target + if eventList[4]: + response = translateAI(eventList[4], "Character Name") + tokens[0] += response[1][0] + tokens[1] += response[1][1] + eventListTL[4] = response[0] + + if len(eventList[4]) != len(eventListTL[4]): + with LOCK: + if FILENAME not in MISMATCH: + MISMATCH.append(FILENAME) + + # Event Job + if eventList[5]: + response = translateAI(eventList[5], "Job/Occupation") + tokens[0] += response[1][0] + tokens[1] += response[1][1] + eventListTL[5] = response[0] + + if len(eventList[5]) != len(eventListTL[5]): + with LOCK: + if FILENAME not in MISMATCH: + MISMATCH.append(FILENAME) + + # Event Place + if eventList[6]: + response = translateAI(eventList[6], "Location Name") + tokens[0] += response[1][0] + tokens[1] += response[1][1] + eventListTL[6] = response[0] + + if len(eventList[6]) != len(eventListTL[6]): + with LOCK: + if FILENAME not in MISMATCH: + MISMATCH.append(FILENAME) + + # String List + if stringList: + PBAR.total = len(stringList) + PBAR.refresh() + response = translateAI(stringList, "Reply with the English Translation") + tokens[0] += response[1][0] + tokens[1] += response[1][1] + stringListTL = response[0] + + if len(stringList) != len(stringListTL): + # Mismatch + with LOCK: + if FILENAME not in MISMATCH: + MISMATCH.append(FILENAME) + + # Pass 2: Set Strings (recursive call) + translateJSON(data, filename, [stringListTL, eventListTL]) + + return tokens + +# Save some money and enter the character before translation +def getSpeaker(speaker): + match speaker: + case "ファイン": + return ["Fine", [0, 0]] + case "": + return ["", [0, 0]] + case _: + # Find Speaker + for i in range(len(NAMESLIST)): + if speaker == NAMESLIST[i][0]: + return [NAMESLIST[i][1], [0, 0]] + + # Translate and Store Speaker + response = translateAI( + f"{speaker}", + "Reply with the " + LANGUAGE + " translation of the NPC name.", + False, + ) + response[0] = response[0].title() + response[0] = response[0].replace("'S", "'s") + response[0] = response[0].replace("Speaker: ", "") + + # Retry if name doesn't translate for some reason + if re.search(r"([a-zA-Z？?])", response[0]) == None: + response = translateAI( + f"{speaker}", + "Reply with the " + LANGUAGE + " translation of the NPC name.", + False, + ) + response[0] = response[0].title() + response[0] = response[0].replace("'S", "'s") + + speakerList = [speaker, response[0]] + NAMESLIST.append(speakerList) + return response + return [speaker, [0, 0]] + +def translateAI(text, history, history_ctx=None): + """ + Legacy wrapper function for the new shared translation utility. + This maintains compatibility with existing code while using the new shared implementation. + """ + global PBAR, MISMATCH, FILENAME + + # Update config estimate mode based on global ESTIMATE + TRANSLATION_CONFIG.estimateMode = bool(ESTIMATE) + + # Call the new shared translation function + return sharedtranslateAI( + text=text, + history=history, + config=TRANSLATION_CONFIG, + filename=FILENAME, + pbar=PBAR, + lock=LOCK, + mismatchList=MISMATCH + ) diff --git a/modules/kirikiri.py b/modules/kirikiri.py index 0e7d236..0c70174 100644 --- a/modules/kirikiri.py +++ b/modules/kirikiri.py @@ -1,531 +1,533 @@ -# Libraries -import json -import os -import re -import util.dazedwrap as dazedwrap -import threading -import time -import traceback -import tiktoken -from pathlib import Path -from colorama import Fore -from dotenv import load_dotenv -from retry import retry -from tqdm import tqdm -from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost, getPricingConfig, calculateCost -import tempfile - -# Globals -MODEL = os.getenv("model") -TIMEOUT = int(os.getenv("timeout")) -LANGUAGE = os.getenv("language").capitalize() -PROMPT = Path("prompt.txt").read_text(encoding="utf-8") -VOCAB = Path("vocab.txt").read_text(encoding="utf-8") -LOCK = threading.Lock() -WIDTH = int(os.getenv("width")) -LISTWIDTH = int(os.getenv("listWidth")) -NOTEWIDTH = 70 -MAXHISTORY = 10 -ESTIMATE = "" -TOKENS = [0, 0] -NAMESLIST = [] -NAMES = False # Output a list of all the character names found -BRFLAG = False # If the game uses
instead -FIXTEXTWRAP = True # Overwrites textwrap -IGNORETLTEXT = False # Ignores all translated text. -MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong) -PBAR = None -FILENAME = None - -# tqdm Globals -BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}" -POSITION = 0 -LEAVE = False - -# Flags -SPEAKERS = True -CHOICES = True -DIALOGUE = True - -# Regex - Need to change this if you want to translate from/to other languages. Default is Japanese Regex -LANGREGEX = r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+" - -# Get pricing configuration based on the model -PRICING_CONFIG = getPricingConfig(MODEL) -INPUTAPICOST = PRICING_CONFIG["inputAPICost"] -OUTPUTAPICOST = PRICING_CONFIG["outputAPICost"] -BATCHSIZE = PRICING_CONFIG["batchSize"] -FREQUENCY_PENALTY = PRICING_CONFIG["frequencyPenalty"] - -# Initialize Translation Config -TRANSLATION_CONFIG = TranslationConfig( - model=MODEL, - language=LANGUAGE, - prompt=PROMPT, - vocab=VOCAB, - langRegex=LANGREGEX, - batchSize=BATCHSIZE, - maxHistory=MAXHISTORY, - estimateMode=False # Will be set dynamically based on ESTIMATE -) -LEAVE = False - -def handleKirikiri(filename, estimate): - global ESTIMATE, FILENAME - ESTIMATE = estimate - FILENAME = filename - - if ESTIMATE: - start = time.time() - translatedData = openFiles(filename) - - # Print Result - end = time.time() - tqdm.write(getResultString(translatedData, end - start, filename)) - with LOCK: - TOKENS[0] += translatedData[1][0] - TOKENS[1] += translatedData[1][1] - - # Print Total - totalString = getResultString(["", TOKENS, None], end - start, "TOTAL") - - # Print any errors on maps - if len(MISMATCH) > 0: - return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET - else: - return totalString - - else: - try: - start = time.time() - translatedData = openFiles(filename) - end = time.time() - - # Final write safeguard: if for some reason the progress file was - # never written (e.g. no translatable lines triggered saves), write it now. - try: - if translatedData[0]: - os.makedirs("translated", exist_ok=True) - final_path = os.path.join("translated", filename) - # Write directly (small risk window acceptable on final flush) - with open(final_path, "w", encoding="cp932", errors="ignore", newline="\n") as f: - f.writelines(translatedData[0]) - except Exception: - traceback.print_exc() - - tqdm.write(getResultString(translatedData, end - start, filename)) - with LOCK: - TOKENS[0] += translatedData[1][0] - TOKENS[1] += translatedData[1][1] - except Exception: - traceback.print_exc() - # Don't blindly remove the file; it may contain partial progress. - return "Fail" - - return getResultString(["", TOKENS, None], end - start, "TOTAL") - - -def getResultString(translatedData, translationTime, filename): - # File Print String - cost = calculateCost(translatedData[1][0], translatedData[1][1], MODEL) - totalTokenstring = ( - Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]" - "[Output: " - + str(translatedData[1][1]) - + "]" "[Cost: ${:,.4f}".format(cost) - + "]" - ) - timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]" - - if translatedData[2] == None: - # Success - return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET - - else: - # Fail - try: - raise translatedData[2] - except Exception as e: - traceback.print_exc() - errorString = str(e) + Fore.RED - return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET - - -def openFiles(filename): - with open("files/" + filename, "r", encoding="cp932") as readFile: - translatedData = parseKiriKiri(readFile, filename) - return translatedData - - -def parseKiriKiri(readFile, filename): - global PBAR - totalTokens = [0, 0] - - # Read File into data - data = readFile.readlines() - - # Create Progress Bar - with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as PBAR: - PBAR.desc = filename - - try: - result = translateKiriKiri(data, PBAR, filename, []) - totalTokens[0] += result[0] - totalTokens[1] += result[1] - except Exception as e: - traceback.print_exc() - return [data, totalTokens, e] - return [data, totalTokens, None] - - -def save_progress_lines(lines, filename, encoding="cp932"): - """Atomically (with retries) save current line-based translation progress. - - Rationale: - Windows raises PermissionError if the destination file is open elsewhere. - We avoid holding an open handle outside this function and add a small - exponential backoff retry loop to handle transient locks (e.g. AV scanners). - """ - if ESTIMATE: - return - - max_attempts = 5 - backoff = 0.05 # seconds - tmp_fd = None - tmp_path = None - - for attempt in range(1, max_attempts + 1): - try: - os.makedirs("translated", exist_ok=True) - - # Create temp file every attempt (prior one cleaned in finally). - tmp_fd, tmp_path = tempfile.mkstemp(prefix=f"{filename}.", suffix=".tmp", dir="translated") - with os.fdopen(tmp_fd, "w", encoding=encoding, newline="\n", errors="ignore") as tmp_file: - tmp_file.writelines(lines) - tmp_file.flush() - os.fsync(tmp_file.fileno()) - - dest_path = os.path.join("translated", filename) - try: - os.replace(tmp_path, dest_path) - except PermissionError as e: - # Retry on Windows-specific sharing violation - if attempt < max_attempts: - time.sleep(backoff) - backoff *= 2 - continue - else: - raise e - # Success, break loop - break - except Exception: - if attempt == max_attempts: - traceback.print_exc() - finally: - # Ensure temp file removed if it still exists - if tmp_path and os.path.exists(tmp_path): - try: - os.remove(tmp_path) - except OSError: - pass - tmp_fd = None - tmp_path = None - - -def translateKiriKiri(data, pbar, filename, jobList): - # Check Job Data - if len(jobList) > 0: - stringList = jobList[0] - choiceList = jobList[1] - setData = True - else: - stringList = [] - choiceList = [] - setData = False - tokens = [0, 0] - speaker = "" - global LOCK, ESTIMATE - i = 0 - - # Regex - speakerRegex = r"【(.*)】\[CR\]" - dialogueRegex = r"^\[text\](.*).*\[KeyWait\]|\[\w+\](.*)\[\/\w+\].*\[KeyWait\]" - furiganaRegex = r'(\[eruby\sstr="(.*?)"\stext.*?\])' - choicesRegex = r"^\s*\[button\d\sclickse=sys_decide.*text='(.*?)'.*" - taggedDialogueRegex = r"^\[(?P[^\s\]/]+)(?:\s[^\]]*)?\](?P.*?)\[/\1\]" - - while i < len(data): - speaker = "" - # Speaker - match = re.search(speakerRegex, data[i]) - if match and SPEAKERS: - speakerJA = match.group(1) - response = getSpeaker(speakerJA) - speaker = response[0] - tokens[0] += response[1][0] - tokens[1] += response[1][1] - data[i] = data[i].replace(speakerJA, speaker) - save_progress_lines(data, filename) - i += 1 - - # Choices - match = re.search(choicesRegex, data[i]) - if match and CHOICES: - jaString = match.group(1) - - # Pass 1 - if not setData: - choiceList.append(jaString) - - # Pass 2 - else: - # Grab and Pop and Set - translatedText = choiceList[0] - choiceList.pop(0) - - # Replace Quotes - data[i] = data[i].replace("'", '"') - translatedText = translatedText.replace('"', "'") - data[i] = data[i].replace(jaString, translatedText) - save_progress_lines(data, filename) - - # Tagged dialogue lines e.g., [思考 storage="..."]text[/思考] - tagged = re.match(taggedDialogueRegex, data[i]) - if tagged and DIALOGUE: - tag_name = tagged.group('tag') - jaString = tagged.group('text') - - # Pass 1: enqueue with speaker from closing tag - if not setData: - # Remove inline wrapping - jaString_clean = jaString.replace("[r]", " ") - # Remove furigana - matchList = re.findall(furiganaRegex, jaString_clean) - if matchList: - for fm in matchList: - jaString_clean = jaString_clean.replace(fm[0], fm[1]) - - # Resolve speaker via getSpeaker - resolved = getSpeaker(tag_name) - tag_speaker = resolved[0] - tokens[0] += resolved[1][0] - tokens[1] += resolved[1][1] - - if tag_speaker: - stringList.append(f"[{tag_speaker}]: {jaString_clean.strip()}") - else: - stringList.append(jaString_clean.strip()) - - # Pass 2: apply translated text back between tags - else: - if len(stringList) > 0: - translatedText = stringList[0] - stringList.pop(0) - # Remove Speaker label if present - translatedText = re.sub(r"\[.*?\]:\s", "", translatedText) - # Wrap and convert newlines to [r] - translatedText = dazedwrap.wrapText(translatedText, width=WIDTH) - translatedText = translatedText.replace("\n", "[r]") - # Replace quotes as per convention - data[i] = data[i].replace("'", '"') - translatedText = translatedText.replace('"', "'") - # Replace only inner content - data[i] = data[i].replace(jaString, translatedText) - save_progress_lines(data, filename) - - # Simple narrative line handling: translate each whitespace-led, non-tag, non-command line independently. - # This avoids reflowing or merging blocks, preventing misplaced text. - if not re.match(r"^\[", data[i]) and not data[i].lstrip().startswith("@"): - if re.match(r"^[ \t\u3000]+", data[i]): - # Skip standalone markers like [▼] - if data[i].strip() == "[▼]": - pass - else: - # Pass 1 - if not setData: - line_content = data[i].rstrip("\n") - # Remove inline wrapping markers and glyph markers - line_content = line_content.replace("[r]", " ") - line_content = re.sub(r"\[▼\]", "", line_content) - # Remove furigana blocks - matchList = re.findall(furiganaRegex, line_content) - if matchList: - for fm in matchList: - line_content = line_content.replace(fm[0], fm[1]) - cleaned = line_content.strip() - if cleaned: - stringList.append(cleaned) - # Pass 2 - else: - if len(stringList) > 0: - translatedText = stringList[0] - stringList.pop(0) - translatedText = re.sub(r"\[.*?\]:\s", "", translatedText) - translatedText = dazedwrap.wrapText(translatedText, width=WIDTH) - translatedText = translatedText.replace("\n", "[r]") - indent_match = re.match(r"^([ \t\u3000]+)", data[i]) - indent = indent_match.group(1) if indent_match else "" - data[i] = f"{indent}{translatedText}\n" - save_progress_lines(data, filename) - - # Dialogue - match = re.search(dialogueRegex, data[i]) - if match and DIALOGUE: - jaString = match.group(1) - if not jaString: - jaString = match.group(2) - - # Pass 1 - if not setData: - # Remove any textwrap - jaString = jaString.replace("[r]", " ") - - # Remove Furigana - matchList = re.findall(furiganaRegex, jaString) - if matchList: - for match in matchList: - jaString = jaString.replace(match[0], match[1]) - - # Add String - if speaker: - stringList.append(f"[{speaker}]: {jaString.strip()}") - else: - stringList.append(f"{jaString.strip()}") - - # Pass 2 - else: - if len(stringList) > 0: - # Grab and Pop - translatedText = stringList[0] - stringList.pop(0) - - # Remove Speaker - translatedText = re.sub(r"\[.*?\]:\s", "", translatedText) - - # Textwrap - translatedText = dazedwrap.wrapText(translatedText, width=WIDTH) - translatedText = translatedText.replace("\n", "[r]") - - # Replace Quotes - data[i] = data[i].replace("'", '"') - translatedText = translatedText.replace('"', "'") - data[i] = data[i].replace(jaString, translatedText) - save_progress_lines(data, filename) - - # Next Line - i += 1 - - # EOF - stringListTL = [] - choiceListTL = [] - - # Dialogue - if len(stringList) > 0: - # Set Progress - pbar.total = len(stringList) - pbar.refresh() - - # Translate - response = translateAI( - stringList, - "", - True, - ) - tokens[0] += response[1][0] - tokens[1] += response[1][1] - stringListTL = response[0] - - # Validate - if len(stringList) != len(stringListTL): - with LOCK: - if filename not in MISMATCH: - MISMATCH.append(filename) - stringListTL = stringList - - # Choices - if len(choiceList) > 0: - # Set Progress - pbar.total = len(choiceList) - pbar.refresh() - - # Translate - response = translateAI( - choiceList, - "", - True, - ) - tokens[0] += response[1][0] - tokens[1] += response[1][1] - choiceListTL = response[0] - - # Validate - if len(choiceList) != len(choiceListTL): - with LOCK: - if filename not in MISMATCH: - MISMATCH.append(filename) - choiceListTL = choiceList - - # Proceed to Pass 2 - if not setData: - translateKiriKiri(data, pbar, filename, [stringListTL, choiceListTL]) - - return tokens - -# Save some money and enter the character before translation -def getSpeaker(speaker): - match speaker: - case "ファイン": - return ["Fine", [0, 0]] - case "": - return ["", [0, 0]] - case _: - # Find Speaker - for i in range(len(NAMESLIST)): - if speaker == NAMESLIST[i][0]: - return [NAMESLIST[i][1], [0, 0]] - - # Translate and Store Speaker - response = translateAI( - f"{speaker}", - "Reply with the " + LANGUAGE + " translation of the NPC name.", - False, - ) - response[0] = response[0].title() - response[0] = response[0].replace("'S", "'s") - response[0] = response[0].replace("Speaker: ", "") - - # Retry if name doesn't translate for some reason - if re.search(r"([a-zA-Z？?])", response[0]) == None: - response = translateAI( - f"{speaker}", - "Reply with the " + LANGUAGE + " translation of the NPC name.", - False, - ) - response[0] = response[0].title() - response[0] = response[0].replace("'S", "'s") - - speakerList = [speaker, response[0]] - NAMESLIST.append(speakerList) - return response - return [speaker, [0, 0]] - -def translateAI(text, history, history_ctx=None): - """ - Legacy wrapper function for the new shared translation utility. - This maintains compatibility with existing code while using the new shared implementation. - """ - global PBAR, MISMATCH, FILENAME - - # Update config estimate mode based on global ESTIMATE - TRANSLATION_CONFIG.estimateMode = bool(ESTIMATE) - - # Call the new shared translation function - return sharedtranslateAI( - text=text, - history=history, - config=TRANSLATION_CONFIG, - filename=FILENAME, - pbar=PBAR, - lock=LOCK, - mismatchList=MISMATCH - ) +# Libraries +import json +import os +import re +import util.dazedwrap as dazedwrap +import threading +import time +import traceback +import tiktoken +from pathlib import Path +from colorama import Fore +from dotenv import load_dotenv +from retry import retry +from tqdm import tqdm +from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost, getPricingConfig, calculateCost +import tempfile + +# Globals +MODEL = os.getenv("model") +TIMEOUT = int(os.getenv("timeout")) +LANGUAGE = os.getenv("language").capitalize() +from util.paths import PROMPT_PATH, VOCAB_PATH + +PROMPT = PROMPT_PATH.read_text(encoding="utf-8") +VOCAB = VOCAB_PATH.read_text(encoding="utf-8") +LOCK = threading.Lock() +WIDTH = int(os.getenv("width")) +LISTWIDTH = int(os.getenv("listWidth")) +NOTEWIDTH = 70 +MAXHISTORY = 10 +ESTIMATE = "" +TOKENS = [0, 0] +NAMESLIST = [] +NAMES = False # Output a list of all the character names found +BRFLAG = False # If the game uses
instead +FIXTEXTWRAP = True # Overwrites textwrap +IGNORETLTEXT = False # Ignores all translated text. +MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong) +PBAR = None +FILENAME = None + +# tqdm Globals +BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}" +POSITION = 0 +LEAVE = False + +# Flags +SPEAKERS = True +CHOICES = True +DIALOGUE = True + +# Regex - Need to change this if you want to translate from/to other languages. Default is Japanese Regex +LANGREGEX = r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+" + +# Get pricing configuration based on the model +PRICING_CONFIG = getPricingConfig(MODEL) +INPUTAPICOST = PRICING_CONFIG["inputAPICost"] +OUTPUTAPICOST = PRICING_CONFIG["outputAPICost"] +BATCHSIZE = PRICING_CONFIG["batchSize"] +FREQUENCY_PENALTY = PRICING_CONFIG["frequencyPenalty"] + +# Initialize Translation Config +TRANSLATION_CONFIG = TranslationConfig( + model=MODEL, + language=LANGUAGE, + prompt=PROMPT, + vocab=VOCAB, + langRegex=LANGREGEX, + batchSize=BATCHSIZE, + maxHistory=MAXHISTORY, + estimateMode=False # Will be set dynamically based on ESTIMATE +) +LEAVE = False + +def handleKirikiri(filename, estimate): + global ESTIMATE, FILENAME + ESTIMATE = estimate + FILENAME = filename + + if ESTIMATE: + start = time.time() + translatedData = openFiles(filename) + + # Print Result + end = time.time() + tqdm.write(getResultString(translatedData, end - start, filename)) + with LOCK: + TOKENS[0] += translatedData[1][0] + TOKENS[1] += translatedData[1][1] + + # Print Total + totalString = getResultString(["", TOKENS, None], end - start, "TOTAL") + + # Print any errors on maps + if len(MISMATCH) > 0: + return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET + else: + return totalString + + else: + try: + start = time.time() + translatedData = openFiles(filename) + end = time.time() + + # Final write safeguard: if for some reason the progress file was + # never written (e.g. no translatable lines triggered saves), write it now. + try: + if translatedData[0]: + os.makedirs("translated", exist_ok=True) + final_path = os.path.join("translated", filename) + # Write directly (small risk window acceptable on final flush) + with open(final_path, "w", encoding="cp932", errors="ignore", newline="\n") as f: + f.writelines(translatedData[0]) + except Exception: + traceback.print_exc() + + tqdm.write(getResultString(translatedData, end - start, filename)) + with LOCK: + TOKENS[0] += translatedData[1][0] + TOKENS[1] += translatedData[1][1] + except Exception: + traceback.print_exc() + # Don't blindly remove the file; it may contain partial progress. + return "Fail" + + return getResultString(["", TOKENS, None], end - start, "TOTAL") + + +def getResultString(translatedData, translationTime, filename): + # File Print String + cost = calculateCost(translatedData[1][0], translatedData[1][1], MODEL) + totalTokenstring = ( + Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]" + "[Output: " + + str(translatedData[1][1]) + + "]" "[Cost: ${:,.4f}".format(cost) + + "]" + ) + timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]" + + if translatedData[2] == None: + # Success + return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET + + else: + # Fail + try: + raise translatedData[2] + except Exception as e: + traceback.print_exc() + errorString = str(e) + Fore.RED + return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET + + +def openFiles(filename): + with open("files/" + filename, "r", encoding="cp932") as readFile: + translatedData = parseKiriKiri(readFile, filename) + return translatedData + + +def parseKiriKiri(readFile, filename): + global PBAR + totalTokens = [0, 0] + + # Read File into data + data = readFile.readlines() + + # Create Progress Bar + with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as PBAR: + PBAR.desc = filename + + try: + result = translateKiriKiri(data, PBAR, filename, []) + totalTokens[0] += result[0] + totalTokens[1] += result[1] + except Exception as e: + traceback.print_exc() + return [data, totalTokens, e] + return [data, totalTokens, None] + + +def save_progress_lines(lines, filename, encoding="cp932"): + """Atomically (with retries) save current line-based translation progress. + + Rationale: + Windows raises PermissionError if the destination file is open elsewhere. + We avoid holding an open handle outside this function and add a small + exponential backoff retry loop to handle transient locks (e.g. AV scanners). + """ + if ESTIMATE: + return + + max_attempts = 5 + backoff = 0.05 # seconds + tmp_fd = None + tmp_path = None + + for attempt in range(1, max_attempts + 1): + try: + os.makedirs("translated", exist_ok=True) + + # Create temp file every attempt (prior one cleaned in finally). + tmp_fd, tmp_path = tempfile.mkstemp(prefix=f"{filename}.", suffix=".tmp", dir="translated") + with os.fdopen(tmp_fd, "w", encoding=encoding, newline="\n", errors="ignore") as tmp_file: + tmp_file.writelines(lines) + tmp_file.flush() + os.fsync(tmp_file.fileno()) + + dest_path = os.path.join("translated", filename) + try: + os.replace(tmp_path, dest_path) + except PermissionError as e: + # Retry on Windows-specific sharing violation + if attempt < max_attempts: + time.sleep(backoff) + backoff *= 2 + continue + else: + raise e + # Success, break loop + break + except Exception: + if attempt == max_attempts: + traceback.print_exc() + finally: + # Ensure temp file removed if it still exists + if tmp_path and os.path.exists(tmp_path): + try: + os.remove(tmp_path) + except OSError: + pass + tmp_fd = None + tmp_path = None + + +def translateKiriKiri(data, pbar, filename, jobList): + # Check Job Data + if len(jobList) > 0: + stringList = jobList[0] + choiceList = jobList[1] + setData = True + else: + stringList = [] + choiceList = [] + setData = False + tokens = [0, 0] + speaker = "" + global LOCK, ESTIMATE + i = 0 + + # Regex + speakerRegex = r"【(.*)】\[CR\]" + dialogueRegex = r"^\[text\](.*).*\[KeyWait\]|\[\w+\](.*)\[\/\w+\].*\[KeyWait\]" + furiganaRegex = r'(\[eruby\sstr="(.*?)"\stext.*?\])' + choicesRegex = r"^\s*\[button\d\sclickse=sys_decide.*text='(.*?)'.*" + taggedDialogueRegex = r"^\[(?P[^\s\]/]+)(?:\s[^\]]*)?\](?P.*?)\[/\1\]" + + while i < len(data): + speaker = "" + # Speaker + match = re.search(speakerRegex, data[i]) + if match and SPEAKERS: + speakerJA = match.group(1) + response = getSpeaker(speakerJA) + speaker = response[0] + tokens[0] += response[1][0] + tokens[1] += response[1][1] + data[i] = data[i].replace(speakerJA, speaker) + save_progress_lines(data, filename) + i += 1 + + # Choices + match = re.search(choicesRegex, data[i]) + if match and CHOICES: + jaString = match.group(1) + + # Pass 1 + if not setData: + choiceList.append(jaString) + + # Pass 2 + else: + # Grab and Pop and Set + translatedText = choiceList[0] + choiceList.pop(0) + + # Replace Quotes + data[i] = data[i].replace("'", '"') + translatedText = translatedText.replace('"', "'") + data[i] = data[i].replace(jaString, translatedText) + save_progress_lines(data, filename) + + # Tagged dialogue lines e.g., [思考 storage="..."]text[/思考] + tagged = re.match(taggedDialogueRegex, data[i]) + if tagged and DIALOGUE: + tag_name = tagged.group('tag') + jaString = tagged.group('text') + + # Pass 1: enqueue with speaker from closing tag + if not setData: + # Remove inline wrapping + jaString_clean = jaString.replace("[r]", " ") + # Remove furigana + matchList = re.findall(furiganaRegex, jaString_clean) + if matchList: + for fm in matchList: + jaString_clean = jaString_clean.replace(fm[0], fm[1]) + + # Resolve speaker via getSpeaker + resolved = getSpeaker(tag_name) + tag_speaker = resolved[0] + tokens[0] += resolved[1][0] + tokens[1] += resolved[1][1] + + if tag_speaker: + stringList.append(f"[{tag_speaker}]: {jaString_clean.strip()}") + else: + stringList.append(jaString_clean.strip()) + + # Pass 2: apply translated text back between tags + else: + if len(stringList) > 0: + translatedText = stringList[0] + stringList.pop(0) + # Remove Speaker label if present + translatedText = re.sub(r"\[.*?\]:\s", "", translatedText) + # Wrap and convert newlines to [r] + translatedText = dazedwrap.wrapText(translatedText, width=WIDTH) + translatedText = translatedText.replace("\n", "[r]") + # Replace quotes as per convention + data[i] = data[i].replace("'", '"') + translatedText = translatedText.replace('"', "'") + # Replace only inner content + data[i] = data[i].replace(jaString, translatedText) + save_progress_lines(data, filename) + + # Simple narrative line handling: translate each whitespace-led, non-tag, non-command line independently. + # This avoids reflowing or merging blocks, preventing misplaced text. + if not re.match(r"^\[", data[i]) and not data[i].lstrip().startswith("@"): + if re.match(r"^[ \t\u3000]+", data[i]): + # Skip standalone markers like [▼] + if data[i].strip() == "[▼]": + pass + else: + # Pass 1 + if not setData: + line_content = data[i].rstrip("\n") + # Remove inline wrapping markers and glyph markers + line_content = line_content.replace("[r]", " ") + line_content = re.sub(r"\[▼\]", "", line_content) + # Remove furigana blocks + matchList = re.findall(furiganaRegex, line_content) + if matchList: + for fm in matchList: + line_content = line_content.replace(fm[0], fm[1]) + cleaned = line_content.strip() + if cleaned: + stringList.append(cleaned) + # Pass 2 + else: + if len(stringList) > 0: + translatedText = stringList[0] + stringList.pop(0) + translatedText = re.sub(r"\[.*?\]:\s", "", translatedText) + translatedText = dazedwrap.wrapText(translatedText, width=WIDTH) + translatedText = translatedText.replace("\n", "[r]") + indent_match = re.match(r"^([ \t\u3000]+)", data[i]) + indent = indent_match.group(1) if indent_match else "" + data[i] = f"{indent}{translatedText}\n" + save_progress_lines(data, filename) + + # Dialogue + match = re.search(dialogueRegex, data[i]) + if match and DIALOGUE: + jaString = match.group(1) + if not jaString: + jaString = match.group(2) + + # Pass 1 + if not setData: + # Remove any textwrap + jaString = jaString.replace("[r]", " ") + + # Remove Furigana + matchList = re.findall(furiganaRegex, jaString) + if matchList: + for match in matchList: + jaString = jaString.replace(match[0], match[1]) + + # Add String + if speaker: + stringList.append(f"[{speaker}]: {jaString.strip()}") + else: + stringList.append(f"{jaString.strip()}") + + # Pass 2 + else: + if len(stringList) > 0: + # Grab and Pop + translatedText = stringList[0] + stringList.pop(0) + + # Remove Speaker + translatedText = re.sub(r"\[.*?\]:\s", "", translatedText) + + # Textwrap + translatedText = dazedwrap.wrapText(translatedText, width=WIDTH) + translatedText = translatedText.replace("\n", "[r]") + + # Replace Quotes + data[i] = data[i].replace("'", '"') + translatedText = translatedText.replace('"', "'") + data[i] = data[i].replace(jaString, translatedText) + save_progress_lines(data, filename) + + # Next Line + i += 1 + + # EOF + stringListTL = [] + choiceListTL = [] + + # Dialogue + if len(stringList) > 0: + # Set Progress + pbar.total = len(stringList) + pbar.refresh() + + # Translate + response = translateAI( + stringList, + "", + True, + ) + tokens[0] += response[1][0] + tokens[1] += response[1][1] + stringListTL = response[0] + + # Validate + if len(stringList) != len(stringListTL): + with LOCK: + if filename not in MISMATCH: + MISMATCH.append(filename) + stringListTL = stringList + + # Choices + if len(choiceList) > 0: + # Set Progress + pbar.total = len(choiceList) + pbar.refresh() + + # Translate + response = translateAI( + choiceList, + "", + True, + ) + tokens[0] += response[1][0] + tokens[1] += response[1][1] + choiceListTL = response[0] + + # Validate + if len(choiceList) != len(choiceListTL): + with LOCK: + if filename not in MISMATCH: + MISMATCH.append(filename) + choiceListTL = choiceList + + # Proceed to Pass 2 + if not setData: + translateKiriKiri(data, pbar, filename, [stringListTL, choiceListTL]) + + return tokens + +# Save some money and enter the character before translation +def getSpeaker(speaker): + match speaker: + case "ファイン": + return ["Fine", [0, 0]] + case "": + return ["", [0, 0]] + case _: + # Find Speaker + for i in range(len(NAMESLIST)): + if speaker == NAMESLIST[i][0]: + return [NAMESLIST[i][1], [0, 0]] + + # Translate and Store Speaker + response = translateAI( + f"{speaker}", + "Reply with the " + LANGUAGE + " translation of the NPC name.", + False, + ) + response[0] = response[0].title() + response[0] = response[0].replace("'S", "'s") + response[0] = response[0].replace("Speaker: ", "") + + # Retry if name doesn't translate for some reason + if re.search(r"([a-zA-Z？?])", response[0]) == None: + response = translateAI( + f"{speaker}", + "Reply with the " + LANGUAGE + " translation of the NPC name.", + False, + ) + response[0] = response[0].title() + response[0] = response[0].replace("'S", "'s") + + speakerList = [speaker, response[0]] + NAMESLIST.append(speakerList) + return response + return [speaker, [0, 0]] + +def translateAI(text, history, history_ctx=None): + """ + Legacy wrapper function for the new shared translation utility. + This maintains compatibility with existing code while using the new shared implementation. + """ + global PBAR, MISMATCH, FILENAME + + # Update config estimate mode based on global ESTIMATE + TRANSLATION_CONFIG.estimateMode = bool(ESTIMATE) + + # Call the new shared translation function + return sharedtranslateAI( + text=text, + history=history, + config=TRANSLATION_CONFIG, + filename=FILENAME, + pbar=PBAR, + lock=LOCK, + mismatchList=MISMATCH + ) diff --git a/modules/lune.py b/modules/lune.py index 377b173..3e036c3 100644 --- a/modules/lune.py +++ b/modules/lune.py @@ -1,372 +1,374 @@ -# Libraries -import json -import os -import re -import util.dazedwrap as dazedwrap -import threading -import time -import traceback -import tiktoken -from pathlib import Path -from colorama import Fore -from dotenv import load_dotenv -from retry import retry -from tqdm import tqdm -from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost -import tempfile - -# Globals -MODEL = os.getenv("model") -TIMEOUT = int(os.getenv("timeout")) -LANGUAGE = os.getenv("language").capitalize() -PROMPT = Path("prompt.txt").read_text(encoding="utf-8") -VOCAB = Path("vocab.txt").read_text(encoding="utf-8") -LOCK = threading.Lock() -WIDTH = int(os.getenv("width")) -LISTWIDTH = int(os.getenv("listWidth")) -NOTEWIDTH = 70 -MAXHISTORY = 10 -ESTIMATE = "" -TOKENS = [0, 0] -NAMESLIST = [] -NAMES = False # Output a list of all the character names found -BRFLAG = False # If the game uses
instead -FIXTEXTWRAP = True # Overwrites textwrap -IGNORETLTEXT = False # Ignores all translated text. -MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong) -PBAR = None -FILENAME = None - -# tqdm Globals -BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}" -POSITION = 0 -LEAVE = False - -# Regex - Need to change this if you want to translate from/to other languages. Default is Japanese Regex -LANGREGEX = r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+" - -# Get pricing configuration based on the model -PRICING_CONFIG = getPricingConfig(MODEL) -INPUTAPICOST = PRICING_CONFIG["inputAPICost"] -OUTPUTAPICOST = PRICING_CONFIG["outputAPICost"] -BATCHSIZE = PRICING_CONFIG["batchSize"] -FREQUENCY_PENALTY = PRICING_CONFIG["frequencyPenalty"] - -# Initialize Translation Config -TRANSLATION_CONFIG = TranslationConfig( - model=MODEL, - language=LANGUAGE, - prompt=PROMPT, - vocab=VOCAB, - langRegex=LANGREGEX, - batchSize=BATCHSIZE, - maxHistory=MAXHISTORY, - estimateMode=False # Will be set dynamically based on ESTIMATE -) -LEAVE = False - -def handleLune(filename, estimate): - global FILENAME, ESTIMATE, totalTokens - ESTIMATE = estimate - FILENAME = filename - - if estimate: - start = time.time() - translatedData = openFiles(filename) - - # Print Result - end = time.time() - tqdm.write(getResultString(translatedData, end - start, filename)) - with LOCK: - TOKENS[0] += translatedData[1][0] - TOKENS[1] += translatedData[1][1] - - return getResultString(["", TOKENS, None], end - start, "TOTAL") - - else: - try: - with open("translated/" + filename, "w", encoding="utf-8", newline="\n") as outFile: - start = time.time() - translatedData = openFiles(filename) - - # Print Result - end = time.time() - json.dump(translatedData[0], outFile, ensure_ascii=False, indent=4) - tqdm.write(getResultString(translatedData, end - start, filename)) - with LOCK: - TOKENS[0] += translatedData[1][0] - TOKENS[1] += translatedData[1][1] - except Exception: - return "Fail" - - return getResultString(["", TOKENS, None], end - start, "TOTAL") - - -def openFiles(filename): - with open("files/" + filename, "r", encoding="UTF-8-sig") as f: - data = json.load(f) - - # Map Files - if ".json" in filename: - translatedData = parseJSON(data, filename) - - else: - raise NameError(filename + " Not Supported") - - return translatedData - - -def getResultString(translatedData, translationTime, filename): - # File Print String - cost = calculateCost(translatedData[1][0], translatedData[1][1], MODEL) - totalTokenstring = ( - Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]" - "[Output: " - + str(translatedData[1][1]) - + "]" "[Cost: ${:,.4f}".format(cost) - + "]" - ) - timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]" - - if translatedData[2] == None: - # Success - return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET - - else: - # Fail - try: - raise translatedData[2] - except Exception as e: - traceback.print_exc() - errorString = str(e) + Fore.RED - return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET - - -def parseJSON(data, filename): - totalTokens = [0, 0] - totalLines = 0 - totalLines = len(data) - global LOCK - - with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar: - pbar.desc = filename - pbar.total = totalLines - try: - result = translateJSON(data, pbar) - totalTokens[0] += result[0] - totalTokens[1] += result[1] - except Exception as e: - return [data, totalTokens, e] - return [data, totalTokens, None] - - -def save_progress_json(data, filename): - """Atomically save current JSON translation progress.""" - try: - if ESTIMATE: - return - os.makedirs("translated", exist_ok=True) - tmp_fd, tmp_path = tempfile.mkstemp(prefix=f"{filename}.", suffix=".tmp", dir="translated") - try: - with os.fdopen(tmp_fd, "w", encoding="utf-8", newline="\n") as tmp_file: - json.dump(data, tmp_file, ensure_ascii=False, indent=4) - os.replace(tmp_path, os.path.join("translated", filename)) - finally: - if os.path.exists(tmp_path): - try: - os.remove(tmp_path) - except OSError: - pass - except Exception: - traceback.print_exc() - - -def translateJSON(data, pbar): - global PBAR - PBAR = pbar - textHistory = [] - batch = [] - maxHistory = MAXHISTORY - tokens = [0, 0] - speaker = "None" - insertBool = False - i = 0 - batchStartIndex = 0 - - while i < len(data): - item = data[i] - # Speaker - if "name" in item: - if item["name"] not in [None, "-"]: - response = getSpeaker(item["name"]) - speaker = response[0] - tokens[0] += response[1][0] - tokens[1] += response[1][1] - item["name"] = speaker - save_progress_json(data, FILENAME or "output.json") - else: - speaker = "None" - - # Text - if "message" in item: - for text in [ - "text", - "text2", - "help1", - "help2", - "help3", - "like", - "message", - "me", - ]: - if text in item: - if item[text] != None: - jaString = item[text] - - # Remove any textwrap - if FIXTEXTWRAP == True: - finalJAString = jaString.replace("\n", " ") - - # [Passthrough 1] Pulling From File - if insertBool is False: - # Append to List and Clear Values - batch.append(finalJAString) - speaker = "" - - # Translate Batch if Full - if len(batch) == BATCHSIZE: - # Translate - response = translateAI(batch, textHistory) - tokens[0] += response[1][0] - tokens[1] += response[1][1] - translatedBatch = response[0] - textHistory = translatedBatch[-10:] - - # Set Values - if len(batch) == len(translatedBatch): - i = batchStartIndex - insertBool = True - - # Mismatch - else: - pbar.write(f"Mismatch: {batchStartIndex} - {i}") - MISMATCH.append(batch) - batchStartIndex = i - batch.clear() - - if insertBool is False: - i += 1 - - currentGroup = [] - - # [Passthrough 2] Setting Data - else: - # Get Text - translatedText = translatedBatch[0] - - # Remove added speaker - translatedText = re.sub(r"^.+?:\s", "", translatedText) - - # Textwrap - translatedText = dazedwrap.wrapText(translatedText, width=WIDTH) - - # Set Text - item[text] = translatedText - save_progress_json(data, FILENAME or "output.json") - translatedBatch.pop(0) - speaker = "" - currentGroup = [] - i += 1 - - # If Batch is empty. Move on. - if len(translatedBatch) == 0: - insertBool = False - batchStartIndex = i - batch.clear() - else: - i += 1 - - # Translate Batch if not empty and EOF - if len(batch) != 0 and i >= len(data): - # Translate - response = translateAI(batch, textHistory) - tokens[0] += response[1][0] - tokens[1] += response[1][1] - translatedBatch = response[0] - textHistory = translatedBatch[-10:] - - # Set Values - if len(batch) == len(translatedBatch): - i = batchStartIndex - insertBool = True - - # Mismatch - else: - pbar.write(f"Mismatch: {batchStartIndex} - {i}") - MISMATCH.append(batch) - batchStartIndex = i - batch.clear() - - currentGroup = [] - # After applying a batch, persist progress - save_progress_json(data, FILENAME or "output.json") - return tokens - -# Save some money and enter the character before translation -def getSpeaker(speaker): - match speaker: - case "ファイン": - return ["Fine", [0, 0]] - case "": - return ["", [0, 0]] - case _: - # Find Speaker - for i in range(len(NAMESLIST)): - if speaker == NAMESLIST[i][0]: - return [NAMESLIST[i][1], [0, 0]] - - # Translate and Store Speaker - response = translateAI( - f"{speaker}", - "Reply with the " + LANGUAGE + " translation of the NPC name.", - False, - ) - response[0] = response[0].title() - response[0] = response[0].replace("'S", "'s") - response[0] = response[0].replace("Speaker: ", "") - - # Retry if name doesn't translate for some reason - if re.search(r"([a-zA-Z？?])", response[0]) == None: - response = translateAI( - f"{speaker}", - "Reply with the " + LANGUAGE + " translation of the NPC name.", - False, - ) - response[0] = response[0].title() - response[0] = response[0].replace("'S", "'s") - - speakerList = [speaker, response[0]] - NAMESLIST.append(speakerList) - return response - return [speaker, [0, 0]] - -def translateAI(text, history, history_ctx=None): - """ - Legacy wrapper function for the new shared translation utility. - This maintains compatibility with existing code while using the new shared implementation. - """ - global PBAR, MISMATCH, FILENAME - - # Update config estimate mode based on global ESTIMATE - TRANSLATION_CONFIG.estimateMode = bool(ESTIMATE) - - # Call the new shared translation function - return sharedtranslateAI( - text=text, - history=history, - config=TRANSLATION_CONFIG, - filename=FILENAME, - pbar=PBAR, - lock=LOCK, - mismatchList=MISMATCH - ) +# Libraries +import json +import os +import re +import util.dazedwrap as dazedwrap +import threading +import time +import traceback +import tiktoken +from pathlib import Path +from colorama import Fore +from dotenv import load_dotenv +from retry import retry +from tqdm import tqdm +from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost +import tempfile + +# Globals +MODEL = os.getenv("model") +TIMEOUT = int(os.getenv("timeout")) +LANGUAGE = os.getenv("language").capitalize() +from util.paths import PROMPT_PATH, VOCAB_PATH + +PROMPT = PROMPT_PATH.read_text(encoding="utf-8") +VOCAB = VOCAB_PATH.read_text(encoding="utf-8") +LOCK = threading.Lock() +WIDTH = int(os.getenv("width")) +LISTWIDTH = int(os.getenv("listWidth")) +NOTEWIDTH = 70 +MAXHISTORY = 10 +ESTIMATE = "" +TOKENS = [0, 0] +NAMESLIST = [] +NAMES = False # Output a list of all the character names found +BRFLAG = False # If the game uses
instead +FIXTEXTWRAP = True # Overwrites textwrap +IGNORETLTEXT = False # Ignores all translated text. +MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong) +PBAR = None +FILENAME = None + +# tqdm Globals +BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}" +POSITION = 0 +LEAVE = False + +# Regex - Need to change this if you want to translate from/to other languages. Default is Japanese Regex +LANGREGEX = r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+" + +# Get pricing configuration based on the model +PRICING_CONFIG = getPricingConfig(MODEL) +INPUTAPICOST = PRICING_CONFIG["inputAPICost"] +OUTPUTAPICOST = PRICING_CONFIG["outputAPICost"] +BATCHSIZE = PRICING_CONFIG["batchSize"] +FREQUENCY_PENALTY = PRICING_CONFIG["frequencyPenalty"] + +# Initialize Translation Config +TRANSLATION_CONFIG = TranslationConfig( + model=MODEL, + language=LANGUAGE, + prompt=PROMPT, + vocab=VOCAB, + langRegex=LANGREGEX, + batchSize=BATCHSIZE, + maxHistory=MAXHISTORY, + estimateMode=False # Will be set dynamically based on ESTIMATE +) +LEAVE = False + +def handleLune(filename, estimate): + global FILENAME, ESTIMATE, totalTokens + ESTIMATE = estimate + FILENAME = filename + + if estimate: + start = time.time() + translatedData = openFiles(filename) + + # Print Result + end = time.time() + tqdm.write(getResultString(translatedData, end - start, filename)) + with LOCK: + TOKENS[0] += translatedData[1][0] + TOKENS[1] += translatedData[1][1] + + return getResultString(["", TOKENS, None], end - start, "TOTAL") + + else: + try: + with open("translated/" + filename, "w", encoding="utf-8", newline="\n") as outFile: + start = time.time() + translatedData = openFiles(filename) + + # Print Result + end = time.time() + json.dump(translatedData[0], outFile, ensure_ascii=False, indent=4) + tqdm.write(getResultString(translatedData, end - start, filename)) + with LOCK: + TOKENS[0] += translatedData[1][0] + TOKENS[1] += translatedData[1][1] + except Exception: + return "Fail" + + return getResultString(["", TOKENS, None], end - start, "TOTAL") + + +def openFiles(filename): + with open("files/" + filename, "r", encoding="UTF-8-sig") as f: + data = json.load(f) + + # Map Files + if ".json" in filename: + translatedData = parseJSON(data, filename) + + else: + raise NameError(filename + " Not Supported") + + return translatedData + + +def getResultString(translatedData, translationTime, filename): + # File Print String + cost = calculateCost(translatedData[1][0], translatedData[1][1], MODEL) + totalTokenstring = ( + Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]" + "[Output: " + + str(translatedData[1][1]) + + "]" "[Cost: ${:,.4f}".format(cost) + + "]" + ) + timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]" + + if translatedData[2] == None: + # Success + return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET + + else: + # Fail + try: + raise translatedData[2] + except Exception as e: + traceback.print_exc() + errorString = str(e) + Fore.RED + return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET + + +def parseJSON(data, filename): + totalTokens = [0, 0] + totalLines = 0 + totalLines = len(data) + global LOCK + + with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar: + pbar.desc = filename + pbar.total = totalLines + try: + result = translateJSON(data, pbar) + totalTokens[0] += result[0] + totalTokens[1] += result[1] + except Exception as e: + return [data, totalTokens, e] + return [data, totalTokens, None] + + +def save_progress_json(data, filename): + """Atomically save current JSON translation progress.""" + try: + if ESTIMATE: + return + os.makedirs("translated", exist_ok=True) + tmp_fd, tmp_path = tempfile.mkstemp(prefix=f"{filename}.", suffix=".tmp", dir="translated") + try: + with os.fdopen(tmp_fd, "w", encoding="utf-8", newline="\n") as tmp_file: + json.dump(data, tmp_file, ensure_ascii=False, indent=4) + os.replace(tmp_path, os.path.join("translated", filename)) + finally: + if os.path.exists(tmp_path): + try: + os.remove(tmp_path) + except OSError: + pass + except Exception: + traceback.print_exc() + + +def translateJSON(data, pbar): + global PBAR + PBAR = pbar + textHistory = [] + batch = [] + maxHistory = MAXHISTORY + tokens = [0, 0] + speaker = "None" + insertBool = False + i = 0 + batchStartIndex = 0 + + while i < len(data): + item = data[i] + # Speaker + if "name" in item: + if item["name"] not in [None, "-"]: + response = getSpeaker(item["name"]) + speaker = response[0] + tokens[0] += response[1][0] + tokens[1] += response[1][1] + item["name"] = speaker + save_progress_json(data, FILENAME or "output.json") + else: + speaker = "None" + + # Text + if "message" in item: + for text in [ + "text", + "text2", + "help1", + "help2", + "help3", + "like", + "message", + "me", + ]: + if text in item: + if item[text] != None: + jaString = item[text] + + # Remove any textwrap + if FIXTEXTWRAP == True: + finalJAString = jaString.replace("\n", " ") + + # [Passthrough 1] Pulling From File + if insertBool is False: + # Append to List and Clear Values + batch.append(finalJAString) + speaker = "" + + # Translate Batch if Full + if len(batch) == BATCHSIZE: + # Translate + response = translateAI(batch, textHistory) + tokens[0] += response[1][0] + tokens[1] += response[1][1] + translatedBatch = response[0] + textHistory = translatedBatch[-10:] + + # Set Values + if len(batch) == len(translatedBatch): + i = batchStartIndex + insertBool = True + + # Mismatch + else: + pbar.write(f"Mismatch: {batchStartIndex} - {i}") + MISMATCH.append(batch) + batchStartIndex = i + batch.clear() + + if insertBool is False: + i += 1 + + currentGroup = [] + + # [Passthrough 2] Setting Data + else: + # Get Text + translatedText = translatedBatch[0] + + # Remove added speaker + translatedText = re.sub(r"^.+?:\s", "", translatedText) + + # Textwrap + translatedText = dazedwrap.wrapText(translatedText, width=WIDTH) + + # Set Text + item[text] = translatedText + save_progress_json(data, FILENAME or "output.json") + translatedBatch.pop(0) + speaker = "" + currentGroup = [] + i += 1 + + # If Batch is empty. Move on. + if len(translatedBatch) == 0: + insertBool = False + batchStartIndex = i + batch.clear() + else: + i += 1 + + # Translate Batch if not empty and EOF + if len(batch) != 0 and i >= len(data): + # Translate + response = translateAI(batch, textHistory) + tokens[0] += response[1][0] + tokens[1] += response[1][1] + translatedBatch = response[0] + textHistory = translatedBatch[-10:] + + # Set Values + if len(batch) == len(translatedBatch): + i = batchStartIndex + insertBool = True + + # Mismatch + else: + pbar.write(f"Mismatch: {batchStartIndex} - {i}") + MISMATCH.append(batch) + batchStartIndex = i + batch.clear() + + currentGroup = [] + # After applying a batch, persist progress + save_progress_json(data, FILENAME or "output.json") + return tokens + +# Save some money and enter the character before translation +def getSpeaker(speaker): + match speaker: + case "ファイン": + return ["Fine", [0, 0]] + case "": + return ["", [0, 0]] + case _: + # Find Speaker + for i in range(len(NAMESLIST)): + if speaker == NAMESLIST[i][0]: + return [NAMESLIST[i][1], [0, 0]] + + # Translate and Store Speaker + response = translateAI( + f"{speaker}", + "Reply with the " + LANGUAGE + " translation of the NPC name.", + False, + ) + response[0] = response[0].title() + response[0] = response[0].replace("'S", "'s") + response[0] = response[0].replace("Speaker: ", "") + + # Retry if name doesn't translate for some reason + if re.search(r"([a-zA-Z？?])", response[0]) == None: + response = translateAI( + f"{speaker}", + "Reply with the " + LANGUAGE + " translation of the NPC name.", + False, + ) + response[0] = response[0].title() + response[0] = response[0].replace("'S", "'s") + + speakerList = [speaker, response[0]] + NAMESLIST.append(speakerList) + return response + return [speaker, [0, 0]] + +def translateAI(text, history, history_ctx=None): + """ + Legacy wrapper function for the new shared translation utility. + This maintains compatibility with existing code while using the new shared implementation. + """ + global PBAR, MISMATCH, FILENAME + + # Update config estimate mode based on global ESTIMATE + TRANSLATION_CONFIG.estimateMode = bool(ESTIMATE) + + # Call the new shared translation function + return sharedtranslateAI( + text=text, + history=history, + config=TRANSLATION_CONFIG, + filename=FILENAME, + pbar=PBAR, + lock=LOCK, + mismatchList=MISMATCH + ) diff --git a/modules/main.py b/modules/main.py index 2fd2752..ca93613 100644 --- a/modules/main.py +++ b/modules/main.py @@ -1,381 +1,385 @@ -import sys -import os -import traceback -import datetime -from pathlib import Path -from concurrent.futures import ThreadPoolExecutor, as_completed -from colorama import Fore -from tqdm import tqdm -from dotenv import load_dotenv - -# This needs to be before the module imports as some of them currently try to read and use some of these values -# upon import, in which case if they are unset the script will crash before we can output these messages. -load_dotenv() -_missing_envs = [ - env for env in [ - "api", - "key", - "model", - "language", - "timeout", - "fileThreads", - "threads", - "width", - "listWidth", - ] - if os.getenv(env) is None or str(os.getenv(env))[:1] == "<" -] -if _missing_envs: - names = ", ".join(_missing_envs) - tqdm.write( - Fore.RED - + f"Missing required environment variable(s): {names}. " - + "Set them in a .env file (see .env.example)." - ) - -from modules.rpgmakermvmz import handleMVMZ, setSpeakerParseMode as setSpeakerParseMVMZ, finalizeSpeakerParse as finalizeSpeakerParseMVMZ -from modules.csv import handleCSV -from modules.tyrano import handleTyrano -from modules.kirikiri import handleKirikiri -from modules.json import handleJSON -from modules.lune import handleLune -from modules.yuris import handleYuris -from modules.nscript import handleOnscripter -from modules.wolf import handleWOLF -from modules.wolf2 import handleWOLF2 -from modules.regex import handleRegex -from modules.text import handleText -from modules.renpy import handleRenpy -from modules.unity import handleUnity -from modules.images import handleImages -from modules.rpgmakerplugin import handlePlugin -from modules.srpg import handleSRPG -from modules.aquedi4 import handleAquedi4 - -# For GPT4 rate limit will be hit if you have more than 1 thread. -# 1 Thread for each file. Controls how many files are worked on at once. -THREADS = int(os.getenv("fileThreads")) - -# [Display name, file extension, handle function] -MODULES = [ - ["RPGMaker MV/MZ", ["json"], handleMVMZ], - ["RPGMaker Plugins", ["js", "rb"], handlePlugin], - ["CSV (From Translator++)", ["csv"], handleCSV], - ["Tyrano", ["ks"], handleTyrano], - ["Kirikiri", ["ks", "tjs", "ssd", "asd"], handleKirikiri], - ["JSON", ["json"], handleJSON], - ["Lune", ["json"], handleLune], - ["Yuris", ["json"], handleYuris], - ["NScript", ["txt"], handleOnscripter], - ["Wolf", ["json"], handleWOLF], - ["Wolf", ["txt"], handleWOLF2], - ["Regex", ["txt", "json", "script", "csv"], handleRegex], - ["Text", ["txt", "srt"], handleText], - ["Renpy", ["rpy"], handleRenpy], - ["Unity", ["txt"], handleUnity], - ["SRPG Studio", ["json"], handleSRPG], - ["Images", [""], handleImages], - ["Aquedi4 Prepared JSON", [".json"], handleAquedi4], -] - -# Info Message -tqdm.write( - Fore.CYAN - + "-Dazed MTL Tool -" - + Fore.RESET, - end="\n\n", -) - - -def main(): - from util.translation import clear_cache - - estimate = "" - batch_mode = False - speaker_parse = False # Deferred until after engine select - while estimate == "": - estimate = input("Select Mode:\n\n 1. Translate\n 2. Estimate\n 3. Batch Translate (Anthropic Batches API, 50% off)\n") - match estimate: - case "1": - estimate = False - case "2": - estimate = True - case "3": - estimate = False - batch_mode = True - case _: - estimate = "" - - resume_state = None - if batch_mode: - from util.translation import isClaudeNative, batchRunState - if not isClaudeNative(os.getenv("model", "")): - tqdm.write( - Fore.RED - + "Batch Translate requires a Claude model with the 'api' env var unset or pointing at anthropic.com." - + Fore.RESET - ) - return - # An interrupted batch run can be resumed instead of re-collecting - # (a second submission would be billed again). - resume_state = batchRunState() - if resume_state: - confirm = "" - while confirm not in ("y", "n"): - confirm = input(f"A previous batch run was interrupted ({resume_state}). Resume it? (y/n)\n").strip().lower() - if confirm == "n": - resume_state = None - - # Clear the translation cache at the start of the run. Kept when resuming a - # batch so names translated during collect stay consistent with the queued - # payloads in the consume pass. - if not resume_state: - clear_cache() - - version = "" - while True: - tqdm.write("Select game engine:\n") - for position, module in enumerate(MODULES): - tqdm.write(f"{str(position + 1).rjust(2)}. {module[0]} (.{module[1]})") - version = input() - try: - version = int(version) - 1 - except: - continue - if version in range(len(MODULES)): - break - - totalCost = ( - Fore.RED - + "Translation module didn't return the total cost. Make sure the \ -files to translate are in the /files folder and that you picked the right game engine." - ) - - # If translating RPGMaker MV/MZ, prompt for speaker parse mode - speaker_parse = False - if version == 0 and not estimate and not batch_mode: - sub = "" - while sub == "": - sub = input("RPGMaker MV/MZ options:\n\n 1. Standard Translate\n 2. Parse Speakers (collect speaker names only)\n") - match sub: - case "1": - speaker_parse = False - case "2": - speaker_parse = True - case _: - sub = "" - if speaker_parse: - setSpeakerParseMVMZ(True) - - # Open File (Threads) - recursively walk 'files' and preserve directory structure - # Prepare per-run log file so CLI runs also write to a run-specific history file - try: - hist_dir = Path("log") / "history" - hist_dir.mkdir(parents=True, exist_ok=True) - - # Clean up old log files, keeping only the 10 most recent - try: - log_files = sorted(hist_dir.glob("translationHistory_*.txt"), key=lambda p: p.stat().st_mtime, reverse=True) - # Keep only the 10 most recent, delete the rest - for old_log in log_files[10:]: - try: - old_log.unlink() - except Exception: - pass - except Exception: - pass - - fname = datetime.datetime.now().strftime("translationHistory_%Y%m%d_%H%M%S.txt") - run_log_path = hist_dir / fname - # Don't create the file yet - it will be created when first log is written - # Store the path in environment variable - try: - os.environ['TRANSLATION_RUN_LOG'] = str(run_log_path) - except Exception: - pass - - # Try to create a hard link from legacy path to this run file for compatibility - # This will be created when the run_log_path file is first written to - legacy = Path("log") / "translationHistory.txt" - try: - if legacy.exists(): - try: - legacy.unlink() - except Exception: - pass - except Exception: - pass - except Exception: - pass - - def runFiles(estimate): - runCost = totalCost - # Use single worker for estimate mode to prevent race conditions - max_workers = 1 if estimate else THREADS - with ThreadPoolExecutor(max_workers=max_workers) as executor: - futures = [] - files_root = "files" - - # Special-case: Images engine expects a folder, not a file; schedule per directory containing assets - if MODULES[version][0] == "Images": - for root, dirs, filenames in os.walk(files_root): - # Skip hidden/system directories - dirs[:] = [d for d in dirs if d not in {".git", "__pycache__"}] - - # Skip the root 'files' itself to avoid processing everything twice - # We'll still allow scheduling for root if it contains assets - - # Only schedule directories that contain potential assets - has_assets = any(fn.lower().endswith((".png", ".txt")) for fn in filenames) - if not has_assets: - continue - - # Compute relative directory path and ensure translated mirror exists - rel_dir = os.path.relpath(root, files_root).replace(os.sep, "/") - if rel_dir == ".": - # Represent root as empty string so handler creates files under translated/ directly - rel_dir = "" - try: - target_dir = os.path.join("translated", rel_dir.replace("/", os.sep)) if rel_dir else "translated" - os.makedirs(target_dir, exist_ok=True) - except Exception: - pass - - futures.append( - executor.submit(MODULES[version][2], rel_dir, estimate) - ) - else: - # Gather all candidate files recursively - for root, dirs, filenames in os.walk(files_root): - # Skip hidden/system directories if any - dirs[:] = [d for d in dirs if d not in {".git", "__pycache__"}] - - for fname in filenames: - if fname == ".gitkeep": - continue - - abs_path = os.path.join(root, fname) - # Build relative path from 'files' root using POSIX-style separators so handlers can do 'files/' + rel - rel_path = os.path.relpath(abs_path, files_root) - rel_path_posix = rel_path.replace(os.sep, "/") - - # Check extension match for the selected module version - for m in MODULES[version][1]: - if rel_path_posix.endswith(m): - # Ensure the corresponding directory exists under 'translated' - rel_dir = os.path.dirname(rel_path_posix) - if rel_dir: - try: - os.makedirs(os.path.join("translated", rel_dir.replace("/", os.sep)), exist_ok=True) - except Exception: - # Best-effort; handler may attempt write and fail if permissions are insufficient - pass - - futures.append( - executor.submit(MODULES[version][2], rel_path_posix, estimate) - ) - break # Avoid double-adding if multiple ext entries match - - for future in as_completed(futures): - try: - runCost = future.result() - except Exception as e: - tracebackLineNo = str(traceback.extract_tb(sys.exc_info()[2])[-1].lineno) - tqdm.write(Fore.RED + str(e) + "|" + tracebackLineNo + Fore.RESET) - return runCost - - if batch_mode: - from util.translation import ( - set_batch_phase, - clearBatchFiles, - pendingBatchRequests, - estimateBatchCost, - runTranslationBatches, - ) - poll = int(os.getenv("batchPollInterval", "60") or 60) - run_consume = True - - if resume_state is None: - clearBatchFiles() - - # Pass 1 — queue dialogue for the batch; speaker/variable strings still hit live API. - tqdm.write(Fore.CYAN + "[BATCH] Pass 1/2: collecting requests..." + Fore.RESET) - tqdm.write( - Fore.YELLOW - + "[BATCH] Note: speaker names and similar short strings translate at live " - "API rates during collect (dialogue is batched after you confirm)." - + Fore.RESET - ) - set_batch_phase("collect") - try: - totalCost = runFiles(False) - finally: - set_batch_phase(None) - - if pendingBatchRequests() == 0: - tqdm.write("[BATCH] No requests queued — nothing needed the API.") - run_consume = False - else: - estimateBatchCost() - confirm = "" - while confirm not in ("y", "n"): - confirm = input("Submit batch? (y/n)\n").strip().lower() - if confirm == "n": - tqdm.write("[BATCH] Not submitted. The queue is kept in log/batch_requests.json.") - return - runTranslationBatches(poll) - elif resume_state == "submitted": - tqdm.write(Fore.CYAN + "[BATCH] Resuming the submitted batch..." + Fore.RESET) - runTranslationBatches(poll) - else: # "fetched" — results already downloaded, just write the files - tqdm.write(Fore.CYAN + "[BATCH] Resuming from fetched results..." + Fore.RESET) - - if run_consume: - # Pass 2 — write the translated files from the fetched results. - # Anything the batch missed falls back to the live API. - tqdm.write(Fore.CYAN + "[BATCH] Pass 2/2: writing translated files..." + Fore.RESET) - set_batch_phase("consume") - try: - totalCost = runFiles(False) - finally: - set_batch_phase(None) - else: - totalCost = runFiles(estimate) - - # Finalize speaker parse mode by writing collected speakers to vocab - if speaker_parse: - finalizeSpeakerParseMVMZ() - - # Delete Tmp Files - if os.path.isfile("csv.tmp"): - os.remove("csv.tmp") - - # Sweep any leftover temp files in translated/ - try: - translated_dir = os.path.join("translated") - if os.path.isdir(translated_dir): - for fname in os.listdir(translated_dir): - if fname.endswith(".tmp"): - fpath = os.path.join(translated_dir, fname) - try: - os.remove(fpath) - except Exception: - # Best-effort cleanup; ignore files locked by other processes - pass - except Exception: - pass - - # Finish - if totalCost != "Fail": - # if estimate is False: - # This is to encourage people to grab what's in /translated instead - # deleteFolderFiles("files") - - tqdm.write(str(totalCost)) - - -def deleteFolderFiles(folderPath): - for filename in os.listdir(folderPath): - file_path = os.path.join(folderPath, filename) - if file_path.endswith((".json", ".ks")): +import sys +import os +import traceback +import datetime +from pathlib import Path +from concurrent.futures import ThreadPoolExecutor, as_completed +from colorama import Fore +from tqdm import tqdm +from dotenv import load_dotenv + +# This needs to be before the module imports as some of them currently try to read and use some of these values +# upon import, in which case if they are unset the script will crash before we can output these messages. +load_dotenv() +from util.paths import migrate_root_data_files, ensure_vocab_file + +migrate_root_data_files() +ensure_vocab_file() +_missing_envs = [ + env for env in [ + "api", + "key", + "model", + "language", + "timeout", + "fileThreads", + "threads", + "width", + "listWidth", + ] + if os.getenv(env) is None or str(os.getenv(env))[:1] == "<" +] +if _missing_envs: + names = ", ".join(_missing_envs) + tqdm.write( + Fore.RED + + f"Missing required environment variable(s): {names}. " + + "Set them in a .env file (see .env.example)." + ) + +from modules.rpgmakermvmz import handleMVMZ, setSpeakerParseMode as setSpeakerParseMVMZ, finalizeSpeakerParse as finalizeSpeakerParseMVMZ +from modules.csv import handleCSV +from modules.tyrano import handleTyrano +from modules.kirikiri import handleKirikiri +from modules.json import handleJSON +from modules.lune import handleLune +from modules.yuris import handleYuris +from modules.nscript import handleOnscripter +from modules.wolf import handleWOLF +from modules.wolf2 import handleWOLF2 +from modules.regex import handleRegex +from modules.text import handleText +from modules.renpy import handleRenpy +from modules.unity import handleUnity +from modules.images import handleImages +from modules.rpgmakerplugin import handlePlugin +from modules.srpg import handleSRPG +from modules.aquedi4 import handleAquedi4 + +# For GPT4 rate limit will be hit if you have more than 1 thread. +# 1 Thread for each file. Controls how many files are worked on at once. +THREADS = int(os.getenv("fileThreads")) + +# [Display name, file extension, handle function] +MODULES = [ + ["RPGMaker MV/MZ", ["json"], handleMVMZ], + ["RPGMaker Plugins", ["js", "rb"], handlePlugin], + ["CSV (From Translator++)", ["csv"], handleCSV], + ["Tyrano", ["ks"], handleTyrano], + ["Kirikiri", ["ks", "tjs", "ssd", "asd"], handleKirikiri], + ["JSON", ["json"], handleJSON], + ["Lune", ["json"], handleLune], + ["Yuris", ["json"], handleYuris], + ["NScript", ["txt"], handleOnscripter], + ["Wolf", ["json"], handleWOLF], + ["Wolf", ["txt"], handleWOLF2], + ["Regex", ["txt", "json", "script", "csv"], handleRegex], + ["Text", ["txt", "srt"], handleText], + ["Renpy", ["rpy"], handleRenpy], + ["Unity", ["txt"], handleUnity], + ["SRPG Studio", ["json"], handleSRPG], + ["Images", [""], handleImages], + ["Aquedi4 Prepared JSON", [".json"], handleAquedi4], +] + +# Info Message +tqdm.write( + Fore.CYAN + + "-Dazed MTL Tool -" + + Fore.RESET, + end="\n\n", +) + + +def main(): + from util.translation import clear_cache + + estimate = "" + batch_mode = False + speaker_parse = False # Deferred until after engine select + while estimate == "": + estimate = input("Select Mode:\n\n 1. Translate\n 2. Estimate\n 3. Batch Translate (Anthropic Batches API, 50% off)\n") + match estimate: + case "1": + estimate = False + case "2": + estimate = True + case "3": + estimate = False + batch_mode = True + case _: + estimate = "" + + resume_state = None + if batch_mode: + from util.translation import isClaudeNative, batchRunState + if not isClaudeNative(os.getenv("model", "")): + tqdm.write( + Fore.RED + + "Batch Translate requires a Claude model with the 'api' env var unset or pointing at anthropic.com." + + Fore.RESET + ) + return + # An interrupted batch run can be resumed instead of re-collecting + # (a second submission would be billed again). + resume_state = batchRunState() + if resume_state: + confirm = "" + while confirm not in ("y", "n"): + confirm = input(f"A previous batch run was interrupted ({resume_state}). Resume it? (y/n)\n").strip().lower() + if confirm == "n": + resume_state = None + + # Clear the translation cache at the start of the run. Kept when resuming a + # batch so names translated during collect stay consistent with the queued + # payloads in the consume pass. + if not resume_state: + clear_cache() + + version = "" + while True: + tqdm.write("Select game engine:\n") + for position, module in enumerate(MODULES): + tqdm.write(f"{str(position + 1).rjust(2)}. {module[0]} (.{module[1]})") + version = input() + try: + version = int(version) - 1 + except: + continue + if version in range(len(MODULES)): + break + + totalCost = ( + Fore.RED + + "Translation module didn't return the total cost. Make sure the \ +files to translate are in the /files folder and that you picked the right game engine." + ) + + # If translating RPGMaker MV/MZ, prompt for speaker parse mode + speaker_parse = False + if version == 0 and not estimate and not batch_mode: + sub = "" + while sub == "": + sub = input("RPGMaker MV/MZ options:\n\n 1. Standard Translate\n 2. Parse Speakers (collect speaker names only)\n") + match sub: + case "1": + speaker_parse = False + case "2": + speaker_parse = True + case _: + sub = "" + if speaker_parse: + setSpeakerParseMVMZ(True) + + # Open File (Threads) - recursively walk 'files' and preserve directory structure + # Prepare per-run log file so CLI runs also write to a run-specific history file + try: + hist_dir = Path("log") / "history" + hist_dir.mkdir(parents=True, exist_ok=True) + + # Clean up old log files, keeping only the 10 most recent + try: + log_files = sorted(hist_dir.glob("translationHistory_*.txt"), key=lambda p: p.stat().st_mtime, reverse=True) + # Keep only the 10 most recent, delete the rest + for old_log in log_files[10:]: + try: + old_log.unlink() + except Exception: + pass + except Exception: + pass + + fname = datetime.datetime.now().strftime("translationHistory_%Y%m%d_%H%M%S.txt") + run_log_path = hist_dir / fname + # Don't create the file yet - it will be created when first log is written + # Store the path in environment variable + try: + os.environ['TRANSLATION_RUN_LOG'] = str(run_log_path) + except Exception: + pass + + # Try to create a hard link from legacy path to this run file for compatibility + # This will be created when the run_log_path file is first written to + legacy = Path("log") / "translationHistory.txt" + try: + if legacy.exists(): + try: + legacy.unlink() + except Exception: + pass + except Exception: + pass + except Exception: + pass + + def runFiles(estimate): + runCost = totalCost + # Use single worker for estimate mode to prevent race conditions + max_workers = 1 if estimate else THREADS + with ThreadPoolExecutor(max_workers=max_workers) as executor: + futures = [] + files_root = "files" + + # Special-case: Images engine expects a folder, not a file; schedule per directory containing assets + if MODULES[version][0] == "Images": + for root, dirs, filenames in os.walk(files_root): + # Skip hidden/system directories + dirs[:] = [d for d in dirs if d not in {".git", "__pycache__"}] + + # Skip the root 'files' itself to avoid processing everything twice + # We'll still allow scheduling for root if it contains assets + + # Only schedule directories that contain potential assets + has_assets = any(fn.lower().endswith((".png", ".txt")) for fn in filenames) + if not has_assets: + continue + + # Compute relative directory path and ensure translated mirror exists + rel_dir = os.path.relpath(root, files_root).replace(os.sep, "/") + if rel_dir == ".": + # Represent root as empty string so handler creates files under translated/ directly + rel_dir = "" + try: + target_dir = os.path.join("translated", rel_dir.replace("/", os.sep)) if rel_dir else "translated" + os.makedirs(target_dir, exist_ok=True) + except Exception: + pass + + futures.append( + executor.submit(MODULES[version][2], rel_dir, estimate) + ) + else: + # Gather all candidate files recursively + for root, dirs, filenames in os.walk(files_root): + # Skip hidden/system directories if any + dirs[:] = [d for d in dirs if d not in {".git", "__pycache__"}] + + for fname in filenames: + if fname == ".gitkeep": + continue + + abs_path = os.path.join(root, fname) + # Build relative path from 'files' root using POSIX-style separators so handlers can do 'files/' + rel + rel_path = os.path.relpath(abs_path, files_root) + rel_path_posix = rel_path.replace(os.sep, "/") + + # Check extension match for the selected module version + for m in MODULES[version][1]: + if rel_path_posix.endswith(m): + # Ensure the corresponding directory exists under 'translated' + rel_dir = os.path.dirname(rel_path_posix) + if rel_dir: + try: + os.makedirs(os.path.join("translated", rel_dir.replace("/", os.sep)), exist_ok=True) + except Exception: + # Best-effort; handler may attempt write and fail if permissions are insufficient + pass + + futures.append( + executor.submit(MODULES[version][2], rel_path_posix, estimate) + ) + break # Avoid double-adding if multiple ext entries match + + for future in as_completed(futures): + try: + runCost = future.result() + except Exception as e: + tracebackLineNo = str(traceback.extract_tb(sys.exc_info()[2])[-1].lineno) + tqdm.write(Fore.RED + str(e) + "|" + tracebackLineNo + Fore.RESET) + return runCost + + if batch_mode: + from util.translation import ( + set_batch_phase, + clearBatchFiles, + pendingBatchRequests, + estimateBatchCost, + runTranslationBatches, + ) + poll = int(os.getenv("batchPollInterval", "60") or 60) + run_consume = True + + if resume_state is None: + clearBatchFiles() + + # Pass 1 — queue dialogue for the batch; speaker/variable strings still hit live API. + tqdm.write(Fore.CYAN + "[BATCH] Pass 1/2: collecting requests..." + Fore.RESET) + tqdm.write( + Fore.YELLOW + + "[BATCH] Note: speaker names and similar short strings translate at live " + "API rates during collect (dialogue is batched after you confirm)." + + Fore.RESET + ) + set_batch_phase("collect") + try: + totalCost = runFiles(False) + finally: + set_batch_phase(None) + + if pendingBatchRequests() == 0: + tqdm.write("[BATCH] No requests queued — nothing needed the API.") + run_consume = False + else: + estimateBatchCost() + confirm = "" + while confirm not in ("y", "n"): + confirm = input("Submit batch? (y/n)\n").strip().lower() + if confirm == "n": + tqdm.write("[BATCH] Not submitted. The queue is kept in log/batch_requests.json.") + return + runTranslationBatches(poll) + elif resume_state == "submitted": + tqdm.write(Fore.CYAN + "[BATCH] Resuming the submitted batch..." + Fore.RESET) + runTranslationBatches(poll) + else: # "fetched" — results already downloaded, just write the files + tqdm.write(Fore.CYAN + "[BATCH] Resuming from fetched results..." + Fore.RESET) + + if run_consume: + # Pass 2 — write the translated files from the fetched results. + # Anything the batch missed falls back to the live API. + tqdm.write(Fore.CYAN + "[BATCH] Pass 2/2: writing translated files..." + Fore.RESET) + set_batch_phase("consume") + try: + totalCost = runFiles(False) + finally: + set_batch_phase(None) + else: + totalCost = runFiles(estimate) + + # Finalize speaker parse mode by writing collected speakers to vocab + if speaker_parse: + finalizeSpeakerParseMVMZ() + + # Delete Tmp Files + if os.path.isfile("csv.tmp"): + os.remove("csv.tmp") + + # Sweep any leftover temp files in translated/ + try: + translated_dir = os.path.join("translated") + if os.path.isdir(translated_dir): + for fname in os.listdir(translated_dir): + if fname.endswith(".tmp"): + fpath = os.path.join(translated_dir, fname) + try: + os.remove(fpath) + except Exception: + # Best-effort cleanup; ignore files locked by other processes + pass + except Exception: + pass + + # Finish + if totalCost != "Fail": + # if estimate is False: + # This is to encourage people to grab what's in /translated instead + # deleteFolderFiles("files") + + tqdm.write(str(totalCost)) + + +def deleteFolderFiles(folderPath): + for filename in os.listdir(folderPath): + file_path = os.path.join(folderPath, filename) + if file_path.endswith((".json", ".ks")): os.remove(file_path) \ No newline at end of file diff --git a/modules/nscript.py b/modules/nscript.py index f6eecaf..60d5816 100644 --- a/modules/nscript.py +++ b/modules/nscript.py @@ -1,426 +1,428 @@ -# Libraries -import json -import os -import re -import util.dazedwrap as dazedwrap -import threading -import time -import traceback -import tiktoken -from pathlib import Path -from colorama import Fore -from dotenv import load_dotenv -from retry import retry -from tqdm import tqdm -from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost -import tempfile - -# Globals -MODEL = os.getenv("model") -TIMEOUT = int(os.getenv("timeout")) -LANGUAGE = os.getenv("language").capitalize() -PROMPT = Path("prompt.txt").read_text(encoding="utf-8") -VOCAB = Path("vocab.txt").read_text(encoding="utf-8") -LOCK = threading.Lock() -WIDTH = int(os.getenv("width")) -LISTWIDTH = int(os.getenv("listWidth")) -NOTEWIDTH = 70 -MAXHISTORY = 10 -ESTIMATE = "" -TOKENS = [0, 0] -NAMESLIST = [] -NAMES = False # Output a list of all the character names found -BRFLAG = False # If the game uses
instead -FIXTEXTWRAP = True # Overwrites textwrap -IGNORETLTEXT = False # Ignores all translated text. -MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong) -CONVERTTOWIDE = False # Default (False) - -# tqdm Globals -BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}" -POSITION = 0 -LEAVE = False -PBAR = None -FILENAME = None - -# Full Width -ascii_to_wide = dict((i, chr(i + 0xFEE0)) for i in range(0x21, 0x7F)) -ascii_to_wide.update({0x20: "\u3000", 0x2D: "\u2212"}) # space and minus -wide_to_ascii = dict((i, chr(i - 0xFEE0)) for i in range(0xFF01, 0xFF5F)) -wide_to_ascii.update({0x3000: " ", 0x2212: "-"}) # space and minus - -# Regex - Need to change this if you want to translate from/to other languages. Default is Japanese Regex -LANGREGEX = r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+" - -# Get pricing configuration based on the model -PRICING_CONFIG = getPricingConfig(MODEL) -INPUTAPICOST = PRICING_CONFIG["inputAPICost"] -OUTPUTAPICOST = PRICING_CONFIG["outputAPICost"] -BATCHSIZE = PRICING_CONFIG["batchSize"] -FREQUENCY_PENALTY = PRICING_CONFIG["frequencyPenalty"] - -# Initialize Translation Config -TRANSLATION_CONFIG = TranslationConfig( - model=MODEL, - language=LANGUAGE, - prompt=PROMPT, - vocab=VOCAB, - langRegex=LANGREGEX, - batchSize=BATCHSIZE, - maxHistory=MAXHISTORY, - estimateMode=False # Will be set dynamically based on ESTIMATE -) -LEAVE = False - -def handleOnscripter(filename, estimate): - global ESTIMATE, FILENAME - ESTIMATE = estimate - FILENAME = filename - - if ESTIMATE: - start = time.time() - translatedData = openFiles(filename) - - # Print Result - end = time.time() - tqdm.write(getResultString(translatedData, end - start, filename)) - with LOCK: - TOKENS[0] += translatedData[1][0] - TOKENS[1] += translatedData[1][1] - - # Print Total - totalString = getResultString(["", TOKENS, None], end - start, "TOTAL") - - # Print any errors on maps - if len(MISMATCH) > 0: - return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET - else: - return totalString - - else: - try: - with open("translated/" + filename, "w", encoding="cp932", errors="ignore") as outFile: - start = time.time() - translatedData = openFiles(filename) - - # Print Result - end = time.time() - outFile.writelines(translatedData[0]) - tqdm.write(getResultString(translatedData, end - start, filename)) - with LOCK: - TOKENS[0] += translatedData[1][0] - TOKENS[1] += translatedData[1][1] - except Exception: - traceback.print_exc() - return "Fail" - - return getResultString(["", TOKENS, None], end - start, "TOTAL") - - -def getResultString(translatedData, translationTime, filename): - # File Print String - cost = calculateCost(translatedData[1][0], translatedData[1][1], MODEL) - totalTokenstring = ( - Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]" - "[Output: " - + str(translatedData[1][1]) - + "]" "[Cost: ${:,.4f}".format(cost) - + "]" - ) - timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]" - - if translatedData[2] == None: - # Success - return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET - - else: - # Fail - try: - raise translatedData[2] - except Exception as e: - traceback.print_exc() - errorString = str(e) + Fore.RED - return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET - - -def openFiles(filename): - with open("files/" + filename, "r", encoding="cp932") as readFile: - translatedData = parseOnscripter(readFile, filename) - - # Delete lines marked for deletion - finalData = [] - for line in translatedData[0]: - if line != "\\d\n": - finalData.append(line) - translatedData[0] = finalData - - return translatedData - - -def parseOnscripter(readFile, filename): - totalTokens = [0, 0] - - # Read File into data - data = readFile.readlines() - - # Create Progress Bar - with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar: - pbar.desc = filename - - try: - result = translateOnscripter(data, pbar, filename, []) - totalTokens[0] += result[0] - totalTokens[1] += result[1] - except Exception as e: - traceback.print_exc() - return [data, totalTokens, e] - return [data, totalTokens, None] - - -def save_progress_lines(lines, filename, encoding="cp932"): - """Atomically save current line-based translation progress.""" - try: - if ESTIMATE: - return - os.makedirs("translated", exist_ok=True) - tmp_fd, tmp_path = tempfile.mkstemp(prefix=f"{filename}.", suffix=".tmp", dir="translated") - try: - with os.fdopen(tmp_fd, "w", encoding=encoding, newline="\n", errors="ignore") as tmp_file: - tmp_file.writelines(lines) - os.replace(tmp_path, os.path.join("translated", filename)) - finally: - if os.path.exists(tmp_path): - try: - os.remove(tmp_path) - except OSError: - pass - except Exception: - traceback.print_exc() - - -def translateOnscripter(data, pbar, filename, translatedList): - stringList = [] - currentGroup = [] - tokens = [0, 0] - speaker = "" - voice = False - global LOCK, ESTIMATE, PBAR - PBAR = pbar - i = 0 - - # Dialogue - while i < len(data): - # Lines - regex = r"^([\u3000「（【[][^\n]+)" - match = re.search(regex, data[i]) - if match != None and match.group(1) != "": - originalString = match.group(1) - # Pass 1 - if translatedList == []: - # Grab Consecutive Strings - jaString = match.group(1) - while len(data) > i + 1 and re.match(regex, data[i + 1]): - data[i] = "" - i += 1 - jaString = f"{jaString} {data[i]}" - - # Convert from Wide - jaString = jaString.translate(wide_to_ascii) - - # Remove any textwrap and \u3000 and \ - jaString = jaString.replace("\n", "") - jaString = jaString.replace("\u3000", "") - jaString = jaString.replace("\\", "") - jaString = jaString.replace("　＞", ")") - jaString = jaString.replace("＜　", "(") - - # Remove Furigana - furiMatch = re.findall(r"({(.+?)\/(.+?)})", jaString) - if furiMatch: - for match in furiMatch: - jaString = jaString.replace(match[0], match[2]) - - # Add String - stringList.append(jaString.strip()) - - # Pass 2 - else: - # Get Text - if translatedList: - # Grab and Pop - translatedText = translatedList[0] - translatedList.pop(0) - - # Set to None if empty list - if len(translatedList) <= 0: - translatedList = None - - # Textwrap & Other Text - translatedText = dazedwrap.wrapText(translatedText, width=WIDTH) - translatedText = translatedText.replace("\n", "\n\u3000") - - # Split the string into lines - lines = translatedText.split("\n") - - # Add a backslash after every 3rd line - j = 0 - while j < len(lines): - if j == 4: - lines[j - 1] = f"{lines[j-1]}\\" - lines[j] = f"\n{lines[j]}" - j += 1 - - # Join the lines back into a single string - translatedText = "\n".join(lines) - - # Remove Double Spaces - translatedText = translatedText.replace(" ", " ") - - # Convert to Wide - if CONVERTTOWIDE: - translatedText = translatedText.translate(ascii_to_wide) - - # Fix Formatting - translatedText = fixText(translatedText) - - # Set Data - data[i] = data[i].replace(originalString, f"{translatedText}") - save_progress_lines(data, filename) - i += 1 - - # Choices - elif "csel" in data[i] and translatedList != []: - choiceList = [] - jaString = data[i] - - choiceList = re.findall(r"\"(.*?)\"", jaString) - if len(choiceList) > 0: - # Translate - response = translateAI(choiceList, "This will be a dialogue option") - translatedTextList = response[0] - tokens[0] += response[1][0] - tokens[1] += response[1][1] - - # Set Data - for j in range(len(translatedTextList)): - # Convert to Wide - if CONVERTTOWIDE: - translatedText = translatedTextList[j].translate(ascii_to_wide) - - # Set - data[i] = data[i].replace(choiceList[j], translatedText) - save_progress_lines(data, filename) - i += 1 - - # Nothing relevant. Skip Line. - else: - i += 1 - - # EOF - if len(stringList) > 0: - # Set Progress - pbar.total = len(stringList) - pbar.refresh() - - # Translate - response = translateAI(stringList, "") - tokens[0] += response[1][0] - tokens[1] += response[1][1] - translatedList = response[0] - - # Set Strings - if len(stringList) == len(translatedList): - translateOnscripter(data, pbar, filename, translatedList) - - # Mismatch - else: - with LOCK: - if filename not in MISMATCH: - MISMATCH.append(filename) - return tokens - - -def fixText(translatedText): - # Add Break - translatedText = translatedText.replace('"', "'") - translatedText = f"\u3000{translatedText}\\" - - # Unconvert Codes - matchList = re.findall(r"([＄].+?)[^\w]", translatedText) - if matchList: - for match in matchList: - translatedText = translatedText.replace(match, match.translate(wide_to_ascii)) - - # Unconvert Color Codes - matchList = re.findall(r"([＃][\w\d]{6})", translatedText) - if matchList: - for match in matchList: - translatedText = translatedText.replace(match, match.translate(wide_to_ascii)) - - # Unconvert Variables - matchList = re.findall(r"([％]\w.+?)[^\w＿]", translatedText) - if matchList: - for match in matchList: - translatedText = translatedText.replace(match, match.translate(wide_to_ascii)) - - # Unconvert Backslashes - matchList = re.findall(r"＼", translatedText) - if matchList: - for match in matchList: - translatedText = translatedText.replace(match, match.translate(wide_to_ascii)) - - return translatedText - -# Save some money and enter the character before translation -def getSpeaker(speaker): - match speaker: - case "ファイン": - return ["Fine", [0, 0]] - case "": - return ["", [0, 0]] - case _: - # Find Speaker - for i in range(len(NAMESLIST)): - if speaker == NAMESLIST[i][0]: - return [NAMESLIST[i][1], [0, 0]] - - # Translate and Store Speaker - response = translateAI( - f"{speaker}", - "Reply with the " + LANGUAGE + " translation of the NPC name.", - False, - ) - response[0] = response[0].title() - response[0] = response[0].replace("'S", "'s") - response[0] = response[0].replace("Speaker: ", "") - - # Retry if name doesn't translate for some reason - if re.search(r"([a-zA-Z？?])", response[0]) == None: - response = translateAI( - f"{speaker}", - "Reply with the " + LANGUAGE + " translation of the NPC name.", - False, - ) - response[0] = response[0].title() - response[0] = response[0].replace("'S", "'s") - - speakerList = [speaker, response[0]] - NAMESLIST.append(speakerList) - return response - return [speaker, [0, 0]] - - -def translateAI(text, history, history_ctx=None): - """ - Translate text using the shared translation utility. - This function maintains compatibility with existing code while using the new shared implementation. - """ - return sharedtranslateAI( - text=text, - history=history, - config=TRANSLATION_CONFIG, - filename=FILENAME, - pbar=PBAR, - lock=LOCK, - mismatchList=MISMATCH - ) +# Libraries +import json +import os +import re +import util.dazedwrap as dazedwrap +import threading +import time +import traceback +import tiktoken +from pathlib import Path +from colorama import Fore +from dotenv import load_dotenv +from retry import retry +from tqdm import tqdm +from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost +import tempfile + +# Globals +MODEL = os.getenv("model") +TIMEOUT = int(os.getenv("timeout")) +LANGUAGE = os.getenv("language").capitalize() +from util.paths import PROMPT_PATH, VOCAB_PATH + +PROMPT = PROMPT_PATH.read_text(encoding="utf-8") +VOCAB = VOCAB_PATH.read_text(encoding="utf-8") +LOCK = threading.Lock() +WIDTH = int(os.getenv("width")) +LISTWIDTH = int(os.getenv("listWidth")) +NOTEWIDTH = 70 +MAXHISTORY = 10 +ESTIMATE = "" +TOKENS = [0, 0] +NAMESLIST = [] +NAMES = False # Output a list of all the character names found +BRFLAG = False # If the game uses
instead +FIXTEXTWRAP = True # Overwrites textwrap +IGNORETLTEXT = False # Ignores all translated text. +MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong) +CONVERTTOWIDE = False # Default (False) + +# tqdm Globals +BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}" +POSITION = 0 +LEAVE = False +PBAR = None +FILENAME = None + +# Full Width +ascii_to_wide = dict((i, chr(i + 0xFEE0)) for i in range(0x21, 0x7F)) +ascii_to_wide.update({0x20: "\u3000", 0x2D: "\u2212"}) # space and minus +wide_to_ascii = dict((i, chr(i - 0xFEE0)) for i in range(0xFF01, 0xFF5F)) +wide_to_ascii.update({0x3000: " ", 0x2212: "-"}) # space and minus + +# Regex - Need to change this if you want to translate from/to other languages. Default is Japanese Regex +LANGREGEX = r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+" + +# Get pricing configuration based on the model +PRICING_CONFIG = getPricingConfig(MODEL) +INPUTAPICOST = PRICING_CONFIG["inputAPICost"] +OUTPUTAPICOST = PRICING_CONFIG["outputAPICost"] +BATCHSIZE = PRICING_CONFIG["batchSize"] +FREQUENCY_PENALTY = PRICING_CONFIG["frequencyPenalty"] + +# Initialize Translation Config +TRANSLATION_CONFIG = TranslationConfig( + model=MODEL, + language=LANGUAGE, + prompt=PROMPT, + vocab=VOCAB, + langRegex=LANGREGEX, + batchSize=BATCHSIZE, + maxHistory=MAXHISTORY, + estimateMode=False # Will be set dynamically based on ESTIMATE +) +LEAVE = False + +def handleOnscripter(filename, estimate): + global ESTIMATE, FILENAME + ESTIMATE = estimate + FILENAME = filename + + if ESTIMATE: + start = time.time() + translatedData = openFiles(filename) + + # Print Result + end = time.time() + tqdm.write(getResultString(translatedData, end - start, filename)) + with LOCK: + TOKENS[0] += translatedData[1][0] + TOKENS[1] += translatedData[1][1] + + # Print Total + totalString = getResultString(["", TOKENS, None], end - start, "TOTAL") + + # Print any errors on maps + if len(MISMATCH) > 0: + return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET + else: + return totalString + + else: + try: + with open("translated/" + filename, "w", encoding="cp932", errors="ignore") as outFile: + start = time.time() + translatedData = openFiles(filename) + + # Print Result + end = time.time() + outFile.writelines(translatedData[0]) + tqdm.write(getResultString(translatedData, end - start, filename)) + with LOCK: + TOKENS[0] += translatedData[1][0] + TOKENS[1] += translatedData[1][1] + except Exception: + traceback.print_exc() + return "Fail" + + return getResultString(["", TOKENS, None], end - start, "TOTAL") + + +def getResultString(translatedData, translationTime, filename): + # File Print String + cost = calculateCost(translatedData[1][0], translatedData[1][1], MODEL) + totalTokenstring = ( + Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]" + "[Output: " + + str(translatedData[1][1]) + + "]" "[Cost: ${:,.4f}".format(cost) + + "]" + ) + timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]" + + if translatedData[2] == None: + # Success + return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET + + else: + # Fail + try: + raise translatedData[2] + except Exception as e: + traceback.print_exc() + errorString = str(e) + Fore.RED + return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET + + +def openFiles(filename): + with open("files/" + filename, "r", encoding="cp932") as readFile: + translatedData = parseOnscripter(readFile, filename) + + # Delete lines marked for deletion + finalData = [] + for line in translatedData[0]: + if line != "\\d\n": + finalData.append(line) + translatedData[0] = finalData + + return translatedData + + +def parseOnscripter(readFile, filename): + totalTokens = [0, 0] + + # Read File into data + data = readFile.readlines() + + # Create Progress Bar + with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar: + pbar.desc = filename + + try: + result = translateOnscripter(data, pbar, filename, []) + totalTokens[0] += result[0] + totalTokens[1] += result[1] + except Exception as e: + traceback.print_exc() + return [data, totalTokens, e] + return [data, totalTokens, None] + + +def save_progress_lines(lines, filename, encoding="cp932"): + """Atomically save current line-based translation progress.""" + try: + if ESTIMATE: + return + os.makedirs("translated", exist_ok=True) + tmp_fd, tmp_path = tempfile.mkstemp(prefix=f"{filename}.", suffix=".tmp", dir="translated") + try: + with os.fdopen(tmp_fd, "w", encoding=encoding, newline="\n", errors="ignore") as tmp_file: + tmp_file.writelines(lines) + os.replace(tmp_path, os.path.join("translated", filename)) + finally: + if os.path.exists(tmp_path): + try: + os.remove(tmp_path) + except OSError: + pass + except Exception: + traceback.print_exc() + + +def translateOnscripter(data, pbar, filename, translatedList): + stringList = [] + currentGroup = [] + tokens = [0, 0] + speaker = "" + voice = False + global LOCK, ESTIMATE, PBAR + PBAR = pbar + i = 0 + + # Dialogue + while i < len(data): + # Lines + regex = r"^([\u3000「（【[][^\n]+)" + match = re.search(regex, data[i]) + if match != None and match.group(1) != "": + originalString = match.group(1) + # Pass 1 + if translatedList == []: + # Grab Consecutive Strings + jaString = match.group(1) + while len(data) > i + 1 and re.match(regex, data[i + 1]): + data[i] = "" + i += 1 + jaString = f"{jaString} {data[i]}" + + # Convert from Wide + jaString = jaString.translate(wide_to_ascii) + + # Remove any textwrap and \u3000 and \ + jaString = jaString.replace("\n", "") + jaString = jaString.replace("\u3000", "") + jaString = jaString.replace("\\", "") + jaString = jaString.replace("　＞", ")") + jaString = jaString.replace("＜　", "(") + + # Remove Furigana + furiMatch = re.findall(r"({(.+?)\/(.+?)})", jaString) + if furiMatch: + for match in furiMatch: + jaString = jaString.replace(match[0], match[2]) + + # Add String + stringList.append(jaString.strip()) + + # Pass 2 + else: + # Get Text + if translatedList: + # Grab and Pop + translatedText = translatedList[0] + translatedList.pop(0) + + # Set to None if empty list + if len(translatedList) <= 0: + translatedList = None + + # Textwrap & Other Text + translatedText = dazedwrap.wrapText(translatedText, width=WIDTH) + translatedText = translatedText.replace("\n", "\n\u3000") + + # Split the string into lines + lines = translatedText.split("\n") + + # Add a backslash after every 3rd line + j = 0 + while j < len(lines): + if j == 4: + lines[j - 1] = f"{lines[j-1]}\\" + lines[j] = f"\n{lines[j]}" + j += 1 + + # Join the lines back into a single string + translatedText = "\n".join(lines) + + # Remove Double Spaces + translatedText = translatedText.replace(" ", " ") + + # Convert to Wide + if CONVERTTOWIDE: + translatedText = translatedText.translate(ascii_to_wide) + + # Fix Formatting + translatedText = fixText(translatedText) + + # Set Data + data[i] = data[i].replace(originalString, f"{translatedText}") + save_progress_lines(data, filename) + i += 1 + + # Choices + elif "csel" in data[i] and translatedList != []: + choiceList = [] + jaString = data[i] + + choiceList = re.findall(r"\"(.*?)\"", jaString) + if len(choiceList) > 0: + # Translate + response = translateAI(choiceList, "This will be a dialogue option") + translatedTextList = response[0] + tokens[0] += response[1][0] + tokens[1] += response[1][1] + + # Set Data + for j in range(len(translatedTextList)): + # Convert to Wide + if CONVERTTOWIDE: + translatedText = translatedTextList[j].translate(ascii_to_wide) + + # Set + data[i] = data[i].replace(choiceList[j], translatedText) + save_progress_lines(data, filename) + i += 1 + + # Nothing relevant. Skip Line. + else: + i += 1 + + # EOF + if len(stringList) > 0: + # Set Progress + pbar.total = len(stringList) + pbar.refresh() + + # Translate + response = translateAI(stringList, "") + tokens[0] += response[1][0] + tokens[1] += response[1][1] + translatedList = response[0] + + # Set Strings + if len(stringList) == len(translatedList): + translateOnscripter(data, pbar, filename, translatedList) + + # Mismatch + else: + with LOCK: + if filename not in MISMATCH: + MISMATCH.append(filename) + return tokens + + +def fixText(translatedText): + # Add Break + translatedText = translatedText.replace('"', "'") + translatedText = f"\u3000{translatedText}\\" + + # Unconvert Codes + matchList = re.findall(r"([＄].+?)[^\w]", translatedText) + if matchList: + for match in matchList: + translatedText = translatedText.replace(match, match.translate(wide_to_ascii)) + + # Unconvert Color Codes + matchList = re.findall(r"([＃][\w\d]{6})", translatedText) + if matchList: + for match in matchList: + translatedText = translatedText.replace(match, match.translate(wide_to_ascii)) + + # Unconvert Variables + matchList = re.findall(r"([％]\w.+?)[^\w＿]", translatedText) + if matchList: + for match in matchList: + translatedText = translatedText.replace(match, match.translate(wide_to_ascii)) + + # Unconvert Backslashes + matchList = re.findall(r"＼", translatedText) + if matchList: + for match in matchList: + translatedText = translatedText.replace(match, match.translate(wide_to_ascii)) + + return translatedText + +# Save some money and enter the character before translation +def getSpeaker(speaker): + match speaker: + case "ファイン": + return ["Fine", [0, 0]] + case "": + return ["", [0, 0]] + case _: + # Find Speaker + for i in range(len(NAMESLIST)): + if speaker == NAMESLIST[i][0]: + return [NAMESLIST[i][1], [0, 0]] + + # Translate and Store Speaker + response = translateAI( + f"{speaker}", + "Reply with the " + LANGUAGE + " translation of the NPC name.", + False, + ) + response[0] = response[0].title() + response[0] = response[0].replace("'S", "'s") + response[0] = response[0].replace("Speaker: ", "") + + # Retry if name doesn't translate for some reason + if re.search(r"([a-zA-Z？?])", response[0]) == None: + response = translateAI( + f"{speaker}", + "Reply with the " + LANGUAGE + " translation of the NPC name.", + False, + ) + response[0] = response[0].title() + response[0] = response[0].replace("'S", "'s") + + speakerList = [speaker, response[0]] + NAMESLIST.append(speakerList) + return response + return [speaker, [0, 0]] + + +def translateAI(text, history, history_ctx=None): + """ + Translate text using the shared translation utility. + This function maintains compatibility with existing code while using the new shared implementation. + """ + return sharedtranslateAI( + text=text, + history=history, + config=TRANSLATION_CONFIG, + filename=FILENAME, + pbar=PBAR, + lock=LOCK, + mismatchList=MISMATCH + ) diff --git a/modules/regex.py b/modules/regex.py index b4e69ae..af881e5 100644 --- a/modules/regex.py +++ b/modules/regex.py @@ -1,528 +1,530 @@ -# Libraries -import json -import os -import re -import util.dazedwrap as dazedwrap -import threading -import time -import traceback -import tiktoken -from pathlib import Path -from colorama import Fore -from dotenv import load_dotenv -from retry import retry -from tqdm import tqdm -from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost -import tempfile - -# Globals -MODEL = os.getenv("model") -TIMEOUT = int(os.getenv("timeout")) -LANGUAGE = os.getenv("language").capitalize() -PROMPT = Path("prompt.txt").read_text(encoding="utf-8") -VOCAB = Path("vocab.txt").read_text(encoding="utf-8") -LOCK = threading.Lock() -WIDTH = int(os.getenv("width")) -LISTWIDTH = int(os.getenv("listWidth")) -NOTEWIDTH = 70 -MAXHISTORY = 10 -ESTIMATE = "" -TOKENS = [0, 0] -NAMESLIST = [] -NAMES = False # Output a list of all the character names found -BRFLAG = False # If the game uses
instead -FIXTEXTWRAP = True # Overwrites textwrap -IGNORETLTEXT = False # Ignores all translated text. -MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong) -FILENAME = None -TIMETOTAL = 0 # Total Time Taken for all translations -TIMETOTAL = 0 # Total Time Taken for all translations - -# tqdm Globals -BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}" -POSITION = 0 -LEAVE = False -PBAR = None - -# Regex - Need to change this if you want to translate from/to other languages. Default is Japanese Regex -LANGREGEX = r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+" - -# Get pricing configuration based on the model -PRICING_CONFIG = getPricingConfig(MODEL) -INPUTAPICOST = PRICING_CONFIG["inputAPICost"] -OUTPUTAPICOST = PRICING_CONFIG["outputAPICost"] -BATCHSIZE = PRICING_CONFIG["batchSize"] -FREQUENCY_PENALTY = PRICING_CONFIG["frequencyPenalty"] - -# Initialize Translation Config -TRANSLATION_CONFIG = TranslationConfig( - model=MODEL, - language=LANGUAGE, - prompt=PROMPT, - vocab=VOCAB, - langRegex=LANGREGEX, - batchSize=BATCHSIZE, - maxHistory=MAXHISTORY, - estimateMode=False # Will be set dynamically based on ESTIMATE -) -LEAVE = False - -def handleRegex(filename, estimate): - global ESTIMATE, TOKENS, FILENAME - ESTIMATE = estimate - FILENAME = filename - - # Translate - start = time.time() - translatedData = openFiles(filename) - - # Translate - if not estimate: - try: - with open("translated/" + filename, "w", encoding="utf-8-sig") as outFile: - outFile.writelines(translatedData[0]) - except Exception: - traceback.print_exc() - return "Fail" - - # Print File - end = time.time() - tqdm.write(getResultString(translatedData, end - start, filename)) - with LOCK: - TOKENS[0] += translatedData[1][0] - TOKENS[1] += translatedData[1][1] - - # Print Total - totalString = getResultString(["", TOKENS, None], end - start, "TOTAL") - - # Print any errors on maps - if len(MISMATCH) > 0: - return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET - else: - return totalString - - -def getResultString(translatedData, translationTime, filename): - global TIMETOTAL - # File Print String - cost = calculateCost(translatedData[1][0], translatedData[1][1], MODEL) - totalTokenstring = ( - Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]" - "[Output: " - + str(translatedData[1][1]) - + "]" "[Cost: ${:,.4f}".format(cost) - + "]" - ) - if filename != "TOTAL": - timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]" - TIMETOTAL += round(translationTime, 1) - else: - timeString = Fore.BLUE + "[" + str(round(TIMETOTAL, 1)) + "s]" - - if translatedData[2] == None: - # Success - return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET - - else: - # Fail - try: - raise translatedData[2] - except Exception as e: - traceback.print_exc() - errorString = str(e) + Fore.RED - return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET - - -def openFiles(filename): - with open("files/" + filename, "r", encoding="utf-8-sig") as readFile: - translatedData = parseRegex(readFile, filename) - - # Delete lines marked for deletion - finalData = [] - for line in translatedData[0]: - if line != "\\d\n": - finalData.append(line) - translatedData[0] = finalData - - return translatedData - - -def parseRegex(readFile, filename): - global PBAR - totalTokens = [0, 0] - - # Read File into data - data = readFile.readlines() - - # Create Progress Bar - with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar: - pbar.desc = filename - PBAR = pbar - - try: - result = translateRegex(data, filename, []) - totalTokens[0] += result[0] - totalTokens[1] += result[1] - except Exception as e: - traceback.print_exc() - return [data, totalTokens, e] - return [data, totalTokens, None] - - -def save_progress_lines(lines, filename, encoding="utf-8-sig"): - """Atomically save current line-based translation progress.""" - try: - if ESTIMATE: - return - os.makedirs("translated", exist_ok=True) - tmp_fd, tmp_path = tempfile.mkstemp(prefix=f"{filename}.", suffix=".tmp", dir="translated") - try: - with os.fdopen(tmp_fd, "w", encoding=encoding, newline="\n", errors="ignore") as tmp_file: - tmp_file.writelines(lines) - os.replace(tmp_path, os.path.join("translated", filename)) - finally: - if os.path.exists(tmp_path): - try: - os.remove(tmp_path) - except OSError: - pass - except Exception: - traceback.print_exc() - - -def translateRegex(data, filename, translatedList): - if translatedList: - stringList = translatedList[0] - choiceList = translatedList[1] - else: - stringList = [] - choiceList = [] - tokens = [0, 0] - speaker = "" - global LOCK, ESTIMATE, FILENAME, PBAR, MISMATCH - i = 0 - - # Define regex patterns outside loop - dialogueRegex = r'\"text\":\s?\"(.+)\",' - lineRegexSpeaker = r'"name":\s?"(.+)",' - choiceRegex = r"\$menu_item.+?,(.*?)," - titleRegex = r"title\s'(.*)'$" - setgamedatatitleRegex = r'\\setgamedatatitle\("(.+?)"\)' - selRegex = r'\\sel\((.+)\)' - - while i < len(data): - - # Setgamedatatitle - match = re.search(setgamedatatitleRegex, data[i]) - if match: - # Pass 1 - Translate immediately (not batched) - if not translatedList: - response = translateAI( - match.group(1).replace('\\n', '\n'), - f"Reply with the {LANGUAGE} translation of the chapter title", - True, - ) - tokens[0] += response[1][0] - tokens[1] += response[1][1] - title = response[0] - - # Convert newlines back to escape sequences and escape quotes - title = title.replace('\n', '\\n').replace('"', '\\"') - data[i] = data[i].replace(match.group(1), title) - save_progress_lines(data, filename) - - # Title - match = re.search(titleRegex, data[i]) - if match: - # Pass 1 - Translate immediately (not batched) - if not translatedList: - response = translateAI( - match.group(1), - f"Reply with the {LANGUAGE} translation of the chapter title", - True, - ) - tokens[0] += response[1][0] - tokens[1] += response[1][1] - title = response[0] - - # Set - title = re.sub(r"(?", ")") - - # Textwrap - translatedText = dazedwrap.wrapText(translatedText, width=WIDTH) - - # Set Data - while "\\endtext" not in data[i].strip(): - del data[i] - data.insert(i, f"{translatedText}\n") - save_progress_lines(data, filename) - - # Choices - match = re.search(choiceRegex, data[i]) - if match: - # Pass 1 - if not translatedList: - choiceList.append(match.group(1)) - - # Pass 2 - else: - # Grab and Pop - translatedText = choiceList[0] - choiceList.pop(0) - - # Replace Spaces - translatedText = translatedText.replace("\u3000", " ") - - # Escape Quotes - translatedText = re.sub(r'(? 0: - PBAR.total = totalItems - PBAR.refresh() - - # String List - if stringList: - response = translateAI(stringList, "Reply with the English Translation") - tokens[0] += response[1][0] - tokens[1] += response[1][1] - stringListTL = response[0] - - if len(stringList) != len(stringListTL): - # Mismatch - with LOCK: - if FILENAME not in MISMATCH: - MISMATCH.append(FILENAME) - - # Choice List - if choiceList: - response = translateAI(choiceList, "Reply with the English TL of the Dialogue Choice") - tokens[0] += response[1][0] - tokens[1] += response[1][1] - choiceListTL = response[0] - - if len(choiceList) != len(choiceListTL): - # Mismatch - with LOCK: - if FILENAME not in MISMATCH: - MISMATCH.append(FILENAME) - - # Set Strings - translateRegex(data, filename, [stringListTL, choiceListTL]) - return tokens - -# Save some money and enter the character before translation -def getSpeaker(speaker): - match speaker: - case "ファイン": - return ["Fine", [0, 0]] - case "": - return ["", [0, 0]] - case _: - # Find Speaker - for i in range(len(NAMESLIST)): - if speaker == NAMESLIST[i][0]: - return [NAMESLIST[i][1], [0, 0]] - - # Translate and Store Speaker - response = translateAI( - f"{speaker}", - "Reply with the " + LANGUAGE + " translation of the NPC name.", - False, - ) - response[0] = response[0].title() - response[0] = response[0].replace("'S", "'s") - response[0] = response[0].replace("Speaker: ", "") - - # Retry if name doesn't translate for some reason - if re.search(r"([a-zA-Z？?])", response[0]) == None: - response = translateAI( - f"{speaker}", - "Reply with the " + LANGUAGE + " translation of the NPC name.", - False, - ) - response[0] = response[0].title() - response[0] = response[0].replace("'S", "'s") - - speakerList = [speaker, response[0]] - NAMESLIST.append(speakerList) - return response - return [speaker, [0, 0]] - -def translateAI(text, history, history_ctx=None): - """ - Legacy wrapper function for the new shared translation utility. - This maintains compatibility with existing code while using the new shared implementation. - """ - global PBAR, MISMATCH, FILENAME - - # Update config estimate mode based on global ESTIMATE - TRANSLATION_CONFIG.estimateMode = bool(ESTIMATE) - - # Call the new shared translation function - return sharedtranslateAI( - text=text, - history=history, - config=TRANSLATION_CONFIG, - filename=FILENAME, - pbar=PBAR, - lock=LOCK, - mismatchList=MISMATCH +# Libraries +import json +import os +import re +import util.dazedwrap as dazedwrap +import threading +import time +import traceback +import tiktoken +from pathlib import Path +from colorama import Fore +from dotenv import load_dotenv +from retry import retry +from tqdm import tqdm +from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost +import tempfile + +# Globals +MODEL = os.getenv("model") +TIMEOUT = int(os.getenv("timeout")) +LANGUAGE = os.getenv("language").capitalize() +from util.paths import PROMPT_PATH, VOCAB_PATH + +PROMPT = PROMPT_PATH.read_text(encoding="utf-8") +VOCAB = VOCAB_PATH.read_text(encoding="utf-8") +LOCK = threading.Lock() +WIDTH = int(os.getenv("width")) +LISTWIDTH = int(os.getenv("listWidth")) +NOTEWIDTH = 70 +MAXHISTORY = 10 +ESTIMATE = "" +TOKENS = [0, 0] +NAMESLIST = [] +NAMES = False # Output a list of all the character names found +BRFLAG = False # If the game uses
instead +FIXTEXTWRAP = True # Overwrites textwrap +IGNORETLTEXT = False # Ignores all translated text. +MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong) +FILENAME = None +TIMETOTAL = 0 # Total Time Taken for all translations +TIMETOTAL = 0 # Total Time Taken for all translations + +# tqdm Globals +BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}" +POSITION = 0 +LEAVE = False +PBAR = None + +# Regex - Need to change this if you want to translate from/to other languages. Default is Japanese Regex +LANGREGEX = r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+" + +# Get pricing configuration based on the model +PRICING_CONFIG = getPricingConfig(MODEL) +INPUTAPICOST = PRICING_CONFIG["inputAPICost"] +OUTPUTAPICOST = PRICING_CONFIG["outputAPICost"] +BATCHSIZE = PRICING_CONFIG["batchSize"] +FREQUENCY_PENALTY = PRICING_CONFIG["frequencyPenalty"] + +# Initialize Translation Config +TRANSLATION_CONFIG = TranslationConfig( + model=MODEL, + language=LANGUAGE, + prompt=PROMPT, + vocab=VOCAB, + langRegex=LANGREGEX, + batchSize=BATCHSIZE, + maxHistory=MAXHISTORY, + estimateMode=False # Will be set dynamically based on ESTIMATE +) +LEAVE = False + +def handleRegex(filename, estimate): + global ESTIMATE, TOKENS, FILENAME + ESTIMATE = estimate + FILENAME = filename + + # Translate + start = time.time() + translatedData = openFiles(filename) + + # Translate + if not estimate: + try: + with open("translated/" + filename, "w", encoding="utf-8-sig") as outFile: + outFile.writelines(translatedData[0]) + except Exception: + traceback.print_exc() + return "Fail" + + # Print File + end = time.time() + tqdm.write(getResultString(translatedData, end - start, filename)) + with LOCK: + TOKENS[0] += translatedData[1][0] + TOKENS[1] += translatedData[1][1] + + # Print Total + totalString = getResultString(["", TOKENS, None], end - start, "TOTAL") + + # Print any errors on maps + if len(MISMATCH) > 0: + return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET + else: + return totalString + + +def getResultString(translatedData, translationTime, filename): + global TIMETOTAL + # File Print String + cost = calculateCost(translatedData[1][0], translatedData[1][1], MODEL) + totalTokenstring = ( + Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]" + "[Output: " + + str(translatedData[1][1]) + + "]" "[Cost: ${:,.4f}".format(cost) + + "]" + ) + if filename != "TOTAL": + timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]" + TIMETOTAL += round(translationTime, 1) + else: + timeString = Fore.BLUE + "[" + str(round(TIMETOTAL, 1)) + "s]" + + if translatedData[2] == None: + # Success + return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET + + else: + # Fail + try: + raise translatedData[2] + except Exception as e: + traceback.print_exc() + errorString = str(e) + Fore.RED + return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET + + +def openFiles(filename): + with open("files/" + filename, "r", encoding="utf-8-sig") as readFile: + translatedData = parseRegex(readFile, filename) + + # Delete lines marked for deletion + finalData = [] + for line in translatedData[0]: + if line != "\\d\n": + finalData.append(line) + translatedData[0] = finalData + + return translatedData + + +def parseRegex(readFile, filename): + global PBAR + totalTokens = [0, 0] + + # Read File into data + data = readFile.readlines() + + # Create Progress Bar + with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar: + pbar.desc = filename + PBAR = pbar + + try: + result = translateRegex(data, filename, []) + totalTokens[0] += result[0] + totalTokens[1] += result[1] + except Exception as e: + traceback.print_exc() + return [data, totalTokens, e] + return [data, totalTokens, None] + + +def save_progress_lines(lines, filename, encoding="utf-8-sig"): + """Atomically save current line-based translation progress.""" + try: + if ESTIMATE: + return + os.makedirs("translated", exist_ok=True) + tmp_fd, tmp_path = tempfile.mkstemp(prefix=f"{filename}.", suffix=".tmp", dir="translated") + try: + with os.fdopen(tmp_fd, "w", encoding=encoding, newline="\n", errors="ignore") as tmp_file: + tmp_file.writelines(lines) + os.replace(tmp_path, os.path.join("translated", filename)) + finally: + if os.path.exists(tmp_path): + try: + os.remove(tmp_path) + except OSError: + pass + except Exception: + traceback.print_exc() + + +def translateRegex(data, filename, translatedList): + if translatedList: + stringList = translatedList[0] + choiceList = translatedList[1] + else: + stringList = [] + choiceList = [] + tokens = [0, 0] + speaker = "" + global LOCK, ESTIMATE, FILENAME, PBAR, MISMATCH + i = 0 + + # Define regex patterns outside loop + dialogueRegex = r'\"text\":\s?\"(.+)\",' + lineRegexSpeaker = r'"name":\s?"(.+)",' + choiceRegex = r"\$menu_item.+?,(.*?)," + titleRegex = r"title\s'(.*)'$" + setgamedatatitleRegex = r'\\setgamedatatitle\("(.+?)"\)' + selRegex = r'\\sel\((.+)\)' + + while i < len(data): + + # Setgamedatatitle + match = re.search(setgamedatatitleRegex, data[i]) + if match: + # Pass 1 - Translate immediately (not batched) + if not translatedList: + response = translateAI( + match.group(1).replace('\\n', '\n'), + f"Reply with the {LANGUAGE} translation of the chapter title", + True, + ) + tokens[0] += response[1][0] + tokens[1] += response[1][1] + title = response[0] + + # Convert newlines back to escape sequences and escape quotes + title = title.replace('\n', '\\n').replace('"', '\\"') + data[i] = data[i].replace(match.group(1), title) + save_progress_lines(data, filename) + + # Title + match = re.search(titleRegex, data[i]) + if match: + # Pass 1 - Translate immediately (not batched) + if not translatedList: + response = translateAI( + match.group(1), + f"Reply with the {LANGUAGE} translation of the chapter title", + True, + ) + tokens[0] += response[1][0] + tokens[1] += response[1][1] + title = response[0] + + # Set + title = re.sub(r"(?", ")") + + # Textwrap + translatedText = dazedwrap.wrapText(translatedText, width=WIDTH) + + # Set Data + while "\\endtext" not in data[i].strip(): + del data[i] + data.insert(i, f"{translatedText}\n") + save_progress_lines(data, filename) + + # Choices + match = re.search(choiceRegex, data[i]) + if match: + # Pass 1 + if not translatedList: + choiceList.append(match.group(1)) + + # Pass 2 + else: + # Grab and Pop + translatedText = choiceList[0] + choiceList.pop(0) + + # Replace Spaces + translatedText = translatedText.replace("\u3000", " ") + + # Escape Quotes + translatedText = re.sub(r'(? 0: + PBAR.total = totalItems + PBAR.refresh() + + # String List + if stringList: + response = translateAI(stringList, "Reply with the English Translation") + tokens[0] += response[1][0] + tokens[1] += response[1][1] + stringListTL = response[0] + + if len(stringList) != len(stringListTL): + # Mismatch + with LOCK: + if FILENAME not in MISMATCH: + MISMATCH.append(FILENAME) + + # Choice List + if choiceList: + response = translateAI(choiceList, "Reply with the English TL of the Dialogue Choice") + tokens[0] += response[1][0] + tokens[1] += response[1][1] + choiceListTL = response[0] + + if len(choiceList) != len(choiceListTL): + # Mismatch + with LOCK: + if FILENAME not in MISMATCH: + MISMATCH.append(FILENAME) + + # Set Strings + translateRegex(data, filename, [stringListTL, choiceListTL]) + return tokens + +# Save some money and enter the character before translation +def getSpeaker(speaker): + match speaker: + case "ファイン": + return ["Fine", [0, 0]] + case "": + return ["", [0, 0]] + case _: + # Find Speaker + for i in range(len(NAMESLIST)): + if speaker == NAMESLIST[i][0]: + return [NAMESLIST[i][1], [0, 0]] + + # Translate and Store Speaker + response = translateAI( + f"{speaker}", + "Reply with the " + LANGUAGE + " translation of the NPC name.", + False, + ) + response[0] = response[0].title() + response[0] = response[0].replace("'S", "'s") + response[0] = response[0].replace("Speaker: ", "") + + # Retry if name doesn't translate for some reason + if re.search(r"([a-zA-Z？?])", response[0]) == None: + response = translateAI( + f"{speaker}", + "Reply with the " + LANGUAGE + " translation of the NPC name.", + False, + ) + response[0] = response[0].title() + response[0] = response[0].replace("'S", "'s") + + speakerList = [speaker, response[0]] + NAMESLIST.append(speakerList) + return response + return [speaker, [0, 0]] + +def translateAI(text, history, history_ctx=None): + """ + Legacy wrapper function for the new shared translation utility. + This maintains compatibility with existing code while using the new shared implementation. + """ + global PBAR, MISMATCH, FILENAME + + # Update config estimate mode based on global ESTIMATE + TRANSLATION_CONFIG.estimateMode = bool(ESTIMATE) + + # Call the new shared translation function + return sharedtranslateAI( + text=text, + history=history, + config=TRANSLATION_CONFIG, + filename=FILENAME, + pbar=PBAR, + lock=LOCK, + mismatchList=MISMATCH ) \ No newline at end of file diff --git a/modules/renpy.py b/modules/renpy.py index 289efb0..414b099 100644 --- a/modules/renpy.py +++ b/modules/renpy.py @@ -1,351 +1,353 @@ -# Libraries -import json -import os -import re -import util.dazedwrap as dazedwrap -import threading -import time -import traceback -import tiktoken -from pathlib import Path -from colorama import Fore -from dotenv import load_dotenv -from retry import retry -from tqdm import tqdm -from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost, getPricingConfig, calculateCost -import tempfile - -# Globals -MODEL = os.getenv("model") -TIMEOUT = int(os.getenv("timeout")) -LANGUAGE = os.getenv("language").capitalize() -PROMPT = Path("prompt.txt").read_text(encoding="utf-8") -VOCAB = Path("vocab.txt").read_text(encoding="utf-8") -LOCK = threading.Lock() -WIDTH = int(os.getenv("width")) -LISTWIDTH = int(os.getenv("listWidth")) -NOTEWIDTH = 70 -MAXHISTORY = 10 -ESTIMATE = "" -TOKENS = [0, 0] -NAMESLIST = [] -NAMES = False # Output a list of all the character names found -BRFLAG = False # If the game uses
instead -FIXTEXTWRAP = True # Overwrites textwrap -IGNORETLTEXT = False # Ignores all translated text. -MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong) -FILENAME = None - -# tqdm Globals -BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}" -POSITION = 0 -LEAVE = False -PBAR = None - -# Regex - Need to change this if you want to translate from/to other languages. Default is Japanese Regex -LANGREGEX = r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+" - -# Get pricing configuration based on the model -PRICING_CONFIG = getPricingConfig(MODEL) -INPUTAPICOST = PRICING_CONFIG["inputAPICost"] -OUTPUTAPICOST = PRICING_CONFIG["outputAPICost"] -BATCHSIZE = PRICING_CONFIG["batchSize"] -FREQUENCY_PENALTY = PRICING_CONFIG["frequencyPenalty"] - -# Initialize Translation Config -TRANSLATION_CONFIG = TranslationConfig( - model=MODEL, - language=LANGUAGE, - prompt=PROMPT, - vocab=VOCAB, - langRegex=LANGREGEX, - batchSize=BATCHSIZE, - maxHistory=MAXHISTORY, - estimateMode=False # Will be set dynamically based on ESTIMATE -) -LEAVE = False - -def handleRenpy(filename, estimate): - global ESTIMATE - global FILENAME - FILENAME = filename - ESTIMATE = estimate - - if ESTIMATE: - start = time.time() - translatedData = openFiles(filename) - - # Print Result - end = time.time() - tqdm.write(getResultString(translatedData, end - start, filename)) - with LOCK: - TOKENS[0] += translatedData[1][0] - TOKENS[1] += translatedData[1][1] - - # Print Total - totalString = getResultString(["", TOKENS, None], end - start, "TOTAL") - - # Print any errors on maps - if len(MISMATCH) > 0: - return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET - else: - return totalString - - else: - try: - with open("translated/" + filename, "w", encoding="utf8", errors="ignore") as outFile: - start = time.time() - translatedData = openFiles(filename) - - # Print Result - end = time.time() - outFile.writelines(translatedData[0]) - tqdm.write(getResultString(translatedData, end - start, filename)) - with LOCK: - TOKENS[0] += translatedData[1][0] - TOKENS[1] += translatedData[1][1] - except Exception: - traceback.print_exc() - return "Fail" - - return getResultString(["", TOKENS, None], end - start, "TOTAL") - - -def getResultString(translatedData, translationTime, filename): - # File Print String - cost = calculateCost(translatedData[1][0], translatedData[1][1], MODEL) - totalTokenstring = ( - Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]" - "[Output: " - + str(translatedData[1][1]) - + "]" "[Cost: ${:,.4f}".format(cost) - + "]" - ) - timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]" - - if translatedData[2] == None: - # Success - return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET - - else: - # Fail - try: - raise translatedData[2] - except Exception as e: - traceback.print_exc() - errorString = str(e) + Fore.RED - return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET - - -def openFiles(filename): - with open("files/" + filename, "r", encoding="utf8") as readFile: - translatedData = parseRenpy(readFile, filename) - - # Delete lines marked for deletion - finalData = [] - for line in translatedData[0]: - if line != "\\d\n": - finalData.append(line) - translatedData[0] = finalData - - return translatedData - - -def parseRenpy(readFile, filename): - global PBAR - totalTokens = [0, 0] - - # Read File into data - data = readFile.readlines() - - # Create Progress Bar - with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar: - pbar.desc = filename - PBAR = pbar - - try: - result = translateRenpy(data, filename, []) - totalTokens[0] += result[0] - totalTokens[1] += result[1] - except Exception as e: - traceback.print_exc() - return [data, totalTokens, e] - return [data, totalTokens, None] - - -def save_progress_lines(lines, filename, encoding="utf-8"): - """Atomically save current line-based translation progress.""" - try: - if ESTIMATE: - return - os.makedirs("translated", exist_ok=True) - tmp_fd, tmp_path = tempfile.mkstemp(prefix=f"{filename}.", suffix=".tmp", dir="translated") - try: - with os.fdopen(tmp_fd, "w", encoding=encoding, newline="\n", errors="ignore") as tmp_file: - tmp_file.writelines(lines) - os.replace(tmp_path, os.path.join("translated", filename)) - finally: - if os.path.exists(tmp_path): - try: - os.remove(tmp_path) - except OSError: - pass - except Exception: - traceback.print_exc() - - -def translateRenpy(data, filename, translatedList): - stringList = [] - currentGroup = [] - tokens = [0, 0] - speaker = "" - voice = False - global LOCK, ESTIMATE, FILENAME, PBAR - i = 0 - - while i < len(data): - voice = False - speaker = "" - lineRegexNoSpeaker = r'^\s\s\s\s"(.*)"' - lineRegexSpeaker = r'^\s\s\s\s(.+?)\s"(.*)"' - - # Grab Line - match = re.search(lineRegexSpeaker, data[i]) - if match: - response = getSpeaker(match.group(1)) - jaString = match.group(2) - speaker = response[0] - tokens[0] += response[1][0] - tokens[1] += response[1][1] - else: - match = re.search(lineRegexNoSpeaker, data[i]) - if match: - jaString = match.group(1) - - # Valid Line - if match and "voice" not in data[i]: - originalString = jaString - # Pass 1 - if translatedList == []: - # Remove any textwrap - jaString = jaString.replace("\\n", " ") - - # Add String - if speaker: - stringList.append(f"[{speaker}]: {jaString.strip()}") - else: - stringList.append(jaString.strip()) - - # Pass 2 - else: - # Get Text - if translatedList: - # Grab and Pop - translatedText = translatedList[0] - translatedList.pop(0) - - # Set to None if empty list - if len(translatedList) <= 0: - translatedList = None - - # Remove speaker - if speaker != "": - matchSpeakerList = re.findall(r"^\[?(.+?)\]?\s?[|:]\s?", translatedText) - translatedText = re.sub(r"^\[?(.+?)\]?\s?[|:]\s?", "", translatedText) - - # Escape Quotes - translatedText = re.sub(r'[\\]*(")', '\\"', translatedText) - translatedText = re.sub(r"[\\]*(')", "\\'", translatedText) - - # Textwrap - translatedText = dazedwrap.wrapText(translatedText, width=WIDTH) - translatedText = translatedText.replace("\n", "\\n") - - # Set Data - data[i] = data[i].replace(originalString, translatedText) - save_progress_lines(data, filename) - i += 1 - else: - i += 1 - - # EOF - if len(stringList) > 0: - # Set Progress - PBAR.total = len(stringList) - PBAR.refresh() - - # Translate - response = translateAI(stringList, "Reply with the English TL of the NPC Name") - tokens[0] += response[1][0] - tokens[1] += response[1][1] - translatedList = response[0] - - # Set Strings - if len(stringList) == len(translatedList): - translateRenpy(data, filename, translatedList) - - # Mismatch - else: - with LOCK: - if FILENAME not in MISMATCH: - MISMATCH.append(FILENAME) - return tokens - -# Save some money and enter the character before translation -def getSpeaker(speaker): - match speaker: - case "ファイン": - return ["Fine", [0, 0]] - case "": - return ["", [0, 0]] - case _: - # Find Speaker - for i in range(len(NAMESLIST)): - if speaker == NAMESLIST[i][0]: - return [NAMESLIST[i][1], [0, 0]] - - # Translate and Store Speaker - response = translateAI( - f"{speaker}", - "Reply with the " + LANGUAGE + " translation of the NPC name.", - False, - ) - response[0] = response[0].title() - response[0] = response[0].replace("'S", "'s") - response[0] = response[0].replace("Speaker: ", "") - - # Retry if name doesn't translate for some reason - if re.search(r"([a-zA-Z？?])", response[0]) == None: - response = translateAI( - f"{speaker}", - "Reply with the " + LANGUAGE + " translation of the NPC name.", - False, - ) - response[0] = response[0].title() - response[0] = response[0].replace("'S", "'s") - - speakerList = [speaker, response[0]] - NAMESLIST.append(speakerList) - return response - return [speaker, [0, 0]] - -def translateAI(text, history, history_ctx=None): - """ - Legacy wrapper function for the new shared translation utility. - This maintains compatibility with existing code while using the new shared implementation. - """ - global PBAR, MISMATCH, FILENAME - - # Update config estimate mode based on global ESTIMATE - TRANSLATION_CONFIG.estimateMode = bool(ESTIMATE) - - # Call the new shared translation function - return sharedtranslateAI( - text=text, - history=history, - config=TRANSLATION_CONFIG, - filename=FILENAME, - pbar=PBAR, - lock=LOCK, - mismatchList=MISMATCH - ) +# Libraries +import json +import os +import re +import util.dazedwrap as dazedwrap +import threading +import time +import traceback +import tiktoken +from pathlib import Path +from colorama import Fore +from dotenv import load_dotenv +from retry import retry +from tqdm import tqdm +from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost, getPricingConfig, calculateCost +import tempfile + +# Globals +MODEL = os.getenv("model") +TIMEOUT = int(os.getenv("timeout")) +LANGUAGE = os.getenv("language").capitalize() +from util.paths import PROMPT_PATH, VOCAB_PATH + +PROMPT = PROMPT_PATH.read_text(encoding="utf-8") +VOCAB = VOCAB_PATH.read_text(encoding="utf-8") +LOCK = threading.Lock() +WIDTH = int(os.getenv("width")) +LISTWIDTH = int(os.getenv("listWidth")) +NOTEWIDTH = 70 +MAXHISTORY = 10 +ESTIMATE = "" +TOKENS = [0, 0] +NAMESLIST = [] +NAMES = False # Output a list of all the character names found +BRFLAG = False # If the game uses
instead +FIXTEXTWRAP = True # Overwrites textwrap +IGNORETLTEXT = False # Ignores all translated text. +MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong) +FILENAME = None + +# tqdm Globals +BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}" +POSITION = 0 +LEAVE = False +PBAR = None + +# Regex - Need to change this if you want to translate from/to other languages. Default is Japanese Regex +LANGREGEX = r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+" + +# Get pricing configuration based on the model +PRICING_CONFIG = getPricingConfig(MODEL) +INPUTAPICOST = PRICING_CONFIG["inputAPICost"] +OUTPUTAPICOST = PRICING_CONFIG["outputAPICost"] +BATCHSIZE = PRICING_CONFIG["batchSize"] +FREQUENCY_PENALTY = PRICING_CONFIG["frequencyPenalty"] + +# Initialize Translation Config +TRANSLATION_CONFIG = TranslationConfig( + model=MODEL, + language=LANGUAGE, + prompt=PROMPT, + vocab=VOCAB, + langRegex=LANGREGEX, + batchSize=BATCHSIZE, + maxHistory=MAXHISTORY, + estimateMode=False # Will be set dynamically based on ESTIMATE +) +LEAVE = False + +def handleRenpy(filename, estimate): + global ESTIMATE + global FILENAME + FILENAME = filename + ESTIMATE = estimate + + if ESTIMATE: + start = time.time() + translatedData = openFiles(filename) + + # Print Result + end = time.time() + tqdm.write(getResultString(translatedData, end - start, filename)) + with LOCK: + TOKENS[0] += translatedData[1][0] + TOKENS[1] += translatedData[1][1] + + # Print Total + totalString = getResultString(["", TOKENS, None], end - start, "TOTAL") + + # Print any errors on maps + if len(MISMATCH) > 0: + return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET + else: + return totalString + + else: + try: + with open("translated/" + filename, "w", encoding="utf8", errors="ignore") as outFile: + start = time.time() + translatedData = openFiles(filename) + + # Print Result + end = time.time() + outFile.writelines(translatedData[0]) + tqdm.write(getResultString(translatedData, end - start, filename)) + with LOCK: + TOKENS[0] += translatedData[1][0] + TOKENS[1] += translatedData[1][1] + except Exception: + traceback.print_exc() + return "Fail" + + return getResultString(["", TOKENS, None], end - start, "TOTAL") + + +def getResultString(translatedData, translationTime, filename): + # File Print String + cost = calculateCost(translatedData[1][0], translatedData[1][1], MODEL) + totalTokenstring = ( + Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]" + "[Output: " + + str(translatedData[1][1]) + + "]" "[Cost: ${:,.4f}".format(cost) + + "]" + ) + timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]" + + if translatedData[2] == None: + # Success + return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET + + else: + # Fail + try: + raise translatedData[2] + except Exception as e: + traceback.print_exc() + errorString = str(e) + Fore.RED + return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET + + +def openFiles(filename): + with open("files/" + filename, "r", encoding="utf8") as readFile: + translatedData = parseRenpy(readFile, filename) + + # Delete lines marked for deletion + finalData = [] + for line in translatedData[0]: + if line != "\\d\n": + finalData.append(line) + translatedData[0] = finalData + + return translatedData + + +def parseRenpy(readFile, filename): + global PBAR + totalTokens = [0, 0] + + # Read File into data + data = readFile.readlines() + + # Create Progress Bar + with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar: + pbar.desc = filename + PBAR = pbar + + try: + result = translateRenpy(data, filename, []) + totalTokens[0] += result[0] + totalTokens[1] += result[1] + except Exception as e: + traceback.print_exc() + return [data, totalTokens, e] + return [data, totalTokens, None] + + +def save_progress_lines(lines, filename, encoding="utf-8"): + """Atomically save current line-based translation progress.""" + try: + if ESTIMATE: + return + os.makedirs("translated", exist_ok=True) + tmp_fd, tmp_path = tempfile.mkstemp(prefix=f"{filename}.", suffix=".tmp", dir="translated") + try: + with os.fdopen(tmp_fd, "w", encoding=encoding, newline="\n", errors="ignore") as tmp_file: + tmp_file.writelines(lines) + os.replace(tmp_path, os.path.join("translated", filename)) + finally: + if os.path.exists(tmp_path): + try: + os.remove(tmp_path) + except OSError: + pass + except Exception: + traceback.print_exc() + + +def translateRenpy(data, filename, translatedList): + stringList = [] + currentGroup = [] + tokens = [0, 0] + speaker = "" + voice = False + global LOCK, ESTIMATE, FILENAME, PBAR + i = 0 + + while i < len(data): + voice = False + speaker = "" + lineRegexNoSpeaker = r'^\s\s\s\s"(.*)"' + lineRegexSpeaker = r'^\s\s\s\s(.+?)\s"(.*)"' + + # Grab Line + match = re.search(lineRegexSpeaker, data[i]) + if match: + response = getSpeaker(match.group(1)) + jaString = match.group(2) + speaker = response[0] + tokens[0] += response[1][0] + tokens[1] += response[1][1] + else: + match = re.search(lineRegexNoSpeaker, data[i]) + if match: + jaString = match.group(1) + + # Valid Line + if match and "voice" not in data[i]: + originalString = jaString + # Pass 1 + if translatedList == []: + # Remove any textwrap + jaString = jaString.replace("\\n", " ") + + # Add String + if speaker: + stringList.append(f"[{speaker}]: {jaString.strip()}") + else: + stringList.append(jaString.strip()) + + # Pass 2 + else: + # Get Text + if translatedList: + # Grab and Pop + translatedText = translatedList[0] + translatedList.pop(0) + + # Set to None if empty list + if len(translatedList) <= 0: + translatedList = None + + # Remove speaker + if speaker != "": + matchSpeakerList = re.findall(r"^\[?(.+?)\]?\s?[|:]\s?", translatedText) + translatedText = re.sub(r"^\[?(.+?)\]?\s?[|:]\s?", "", translatedText) + + # Escape Quotes + translatedText = re.sub(r'[\\]*(")', '\\"', translatedText) + translatedText = re.sub(r"[\\]*(')", "\\'", translatedText) + + # Textwrap + translatedText = dazedwrap.wrapText(translatedText, width=WIDTH) + translatedText = translatedText.replace("\n", "\\n") + + # Set Data + data[i] = data[i].replace(originalString, translatedText) + save_progress_lines(data, filename) + i += 1 + else: + i += 1 + + # EOF + if len(stringList) > 0: + # Set Progress + PBAR.total = len(stringList) + PBAR.refresh() + + # Translate + response = translateAI(stringList, "Reply with the English TL of the NPC Name") + tokens[0] += response[1][0] + tokens[1] += response[1][1] + translatedList = response[0] + + # Set Strings + if len(stringList) == len(translatedList): + translateRenpy(data, filename, translatedList) + + # Mismatch + else: + with LOCK: + if FILENAME not in MISMATCH: + MISMATCH.append(FILENAME) + return tokens + +# Save some money and enter the character before translation +def getSpeaker(speaker): + match speaker: + case "ファイン": + return ["Fine", [0, 0]] + case "": + return ["", [0, 0]] + case _: + # Find Speaker + for i in range(len(NAMESLIST)): + if speaker == NAMESLIST[i][0]: + return [NAMESLIST[i][1], [0, 0]] + + # Translate and Store Speaker + response = translateAI( + f"{speaker}", + "Reply with the " + LANGUAGE + " translation of the NPC name.", + False, + ) + response[0] = response[0].title() + response[0] = response[0].replace("'S", "'s") + response[0] = response[0].replace("Speaker: ", "") + + # Retry if name doesn't translate for some reason + if re.search(r"([a-zA-Z？?])", response[0]) == None: + response = translateAI( + f"{speaker}", + "Reply with the " + LANGUAGE + " translation of the NPC name.", + False, + ) + response[0] = response[0].title() + response[0] = response[0].replace("'S", "'s") + + speakerList = [speaker, response[0]] + NAMESLIST.append(speakerList) + return response + return [speaker, [0, 0]] + +def translateAI(text, history, history_ctx=None): + """ + Legacy wrapper function for the new shared translation utility. + This maintains compatibility with existing code while using the new shared implementation. + """ + global PBAR, MISMATCH, FILENAME + + # Update config estimate mode based on global ESTIMATE + TRANSLATION_CONFIG.estimateMode = bool(ESTIMATE) + + # Call the new shared translation function + return sharedtranslateAI( + text=text, + history=history, + config=TRANSLATION_CONFIG, + filename=FILENAME, + pbar=PBAR, + lock=LOCK, + mismatchList=MISMATCH + ) diff --git a/modules/rpgmakermvmz.py b/modules/rpgmakermvmz.py index b6a5a5e..e4350ad 100644 --- a/modules/rpgmakermvmz.py +++ b/modules/rpgmakermvmz.py @@ -19,8 +19,10 @@ from util.translation import TranslationConfig, translateAI as sharedtranslateAI MODEL = os.getenv("model") TIMEOUT = int(os.getenv("timeout")) LANGUAGE = os.getenv("language").capitalize() -PROMPT = Path("prompt.txt").read_text(encoding="utf-8") -VOCAB = Path("vocab.txt").read_text(encoding="utf-8") +from util.paths import PROMPT_PATH, VOCAB_PATH + +PROMPT = PROMPT_PATH.read_text(encoding="utf-8") +VOCAB = VOCAB_PATH.read_text(encoding="utf-8") LOCK = threading.Lock() THREAD_CTX = threading.local() WIDTH = int(os.getenv("width")) @@ -803,7 +805,7 @@ def update_vocab_section(category: str, pairs: list[tuple[str, str]]): The existing section is replaced entirely; other sections are preserved. """ try: - vocab_path = Path("vocab.txt") + vocab_path = VOCAB_PATH # Helper: normalized comparison to detect no-op translations def _norm(s: str) -> str: @@ -5071,7 +5073,7 @@ def finalizeSpeakerParse(): _speakerCache[orig] = norm NAMESLIST.append([orig, norm]) - vocab_path = Path("vocab.txt") + vocab_path = VOCAB_PATH if not vocab_path.exists(): return content = vocab_path.read_text(encoding="utf-8") diff --git a/modules/rpgmakerplugin.py b/modules/rpgmakerplugin.py index 161a719..343993a 100644 --- a/modules/rpgmakerplugin.py +++ b/modules/rpgmakerplugin.py @@ -1,1135 +1,1137 @@ -# Libraries -import json -import os -import re -import util.dazedwrap as dazedwrap -import threading -import time -import traceback -import tiktoken -from pathlib import Path -from colorama import Fore -from dotenv import load_dotenv -from retry import retry -from tqdm import tqdm -from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost -import tempfile - -# Globals -MODEL = os.getenv("model") -TIMEOUT = int(os.getenv("timeout")) -LANGUAGE = os.getenv("language").capitalize() -PROMPT = Path("prompt.txt").read_text(encoding="utf-8") -VOCAB = Path("vocab.txt").read_text(encoding="utf-8") -LOCK = threading.Lock() -WIDTH = int(os.getenv("width")) -LISTWIDTH = int(os.getenv("listWidth")) -NOTEWIDTH = 70 -MAXHISTORY = 10 -ESTIMATE = "" -TOKENS = [0, 0] -NAMESLIST = [] -NAMES = False # Output a list of all the character names found -BRFLAG = False # If the game uses
instead -FIXTEXTWRAP = True # Overwrites textwrap -IGNORETLTEXT = False # Ignores all translated text. -MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong) -FILENAME = None - -# tqdm Globals -BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}" -POSITION = 0 -LEAVE = False -PBAR = None - -# Regex - Need to change this if you want to translate from/to other languages. Default is Japanese Regex -LANGREGEX = r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+" - -# Get pricing configuration based on the model -PRICING_CONFIG = getPricingConfig(MODEL) -INPUTAPICOST = PRICING_CONFIG["inputAPICost"] -OUTPUTAPICOST = PRICING_CONFIG["outputAPICost"] -BATCHSIZE = PRICING_CONFIG["batchSize"] -FREQUENCY_PENALTY = PRICING_CONFIG["frequencyPenalty"] - -# Initialize Translation Config -TRANSLATION_CONFIG = TranslationConfig( - model=MODEL, - language=LANGUAGE, - prompt=PROMPT, - vocab=VOCAB, - langRegex=LANGREGEX, - batchSize=BATCHSIZE, - maxHistory=MAXHISTORY, - estimateMode=False # Will be set dynamically based on ESTIMATE -) -LEAVE = False - -def handlePlugin(filename, estimate): - global ESTIMATE, PBAR, FILENAME - ESTIMATE = estimate - FILENAME = filename - - if ESTIMATE: - start = time.time() - translatedData = openFiles(filename) - - # Print Result - end = time.time() - tqdm.write(getResultString(translatedData, end - start, filename)) - with LOCK: - TOKENS[0] += translatedData[1][0] - TOKENS[1] += translatedData[1][1] - - # Print Total - totalString = getResultString(["", TOKENS, None], end - start, "TOTAL") - - # Print any errors on maps - if len(MISMATCH) > 0: - return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET - else: - return totalString - - else: - try: - # Perform translation first; incremental progress writes happen during translation. - # We no longer keep the destination file open simultaneously to avoid Windows locking issues - # during atomic os.replace operations inside save_progress_lines. - start = time.time() - translatedData = openFiles(filename) - - # Ensure final state is flushed (in case no incremental writes occurred for some reason). - save_progress_lines(translatedData[0], filename) - - # Print Result - end = time.time() - tqdm.write(getResultString(translatedData, end - start, filename)) - with LOCK: - TOKENS[0] += translatedData[1][0] - TOKENS[1] += translatedData[1][1] - except Exception: - traceback.print_exc() - return "Fail" - - return getResultString(["", TOKENS, None], end - start, "TOTAL") - - -def getResultString(translatedData, translationTime, filename): - # File Print String - cost = calculateCost(translatedData[1][0], translatedData[1][1], MODEL) - totalTokenstring = ( - Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]" - "[Output: " - + str(translatedData[1][1]) - + "]" "[Cost: ${:,.4f}".format(cost) - + "]" - ) - timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]" - - if translatedData[2] == None: - # Success - return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET - - else: - # Fail - try: - raise translatedData[2] - except Exception as e: - traceback.print_exc() - errorString = str(e) + Fore.RED - return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET - - -def openFiles(filename): - with open("files/" + filename, "r", encoding="utf_8") as readFile: - translatedData = parsePlugin(readFile, filename) - - # Delete lines marked for deletion - finalData = [] - for line in translatedData[0]: - if line != "\\d\n": - finalData.append(line) - translatedData[0] = finalData - - return translatedData - - -def parsePlugin(readFile, filename): - global PBAR - totalTokens = [0, 0] - - # Read File into data - data = readFile.readlines() - - # Create Progress Bar - with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar: - pbar.desc = filename - PBAR = pbar - - try: - result = translatePlugin(data, pbar, filename, []) - totalTokens[0] += result[0] - totalTokens[1] += result[1] - except Exception as e: - traceback.print_exc() - return [data, totalTokens, e] - return [data, totalTokens, None] - - -def save_progress_lines(lines, filename, encoding="utf_8"): - """Atomically save current line-based translation progress.""" - try: - if ESTIMATE: - return - global LOCK - os.makedirs("translated", exist_ok=True) - # Use a single lock to prevent concurrent replace attempts on Windows (which causes PermissionError) - with LOCK: - tmp_fd, tmp_path = tempfile.mkstemp(prefix=f"{filename}.", suffix=".tmp", dir="translated") - try: - with os.fdopen(tmp_fd, "w", encoding=encoding, newline="\n", errors="ignore") as tmp_file: - tmp_file.writelines(lines) - - dest_path = os.path.join("translated", filename) - # Retry a few times in case another process/thread still has the file open momentarily. - for attempt in range(5): - try: - os.replace(tmp_path, dest_path) - break - except PermissionError: - if attempt == 4: - raise - time.sleep(0.1 * (attempt + 1)) - finally: - if os.path.exists(tmp_path): - try: - os.remove(tmp_path) - except OSError: - pass - except Exception: - traceback.print_exc() - - -def saveCheckLines(lines, filename, tokens=None, encoding="utf_8"): - """Save progress only when tokens indicate work or when explicitly called after a mutation.""" - try: - if tokens is not None: - if not (isinstance(tokens, (list, tuple)) and len(tokens) >= 2 and (tokens[0] or tokens[1])): - return - save_progress_lines(lines, filename, encoding=encoding) - except Exception: - traceback.print_exc() - - -def translatePlugin(data, pbar, filename, translatedList): - if len(translatedList) > 0: - questList = translatedList[0] - custom = translatedList[1] - sceneMenuText = translatedList[2] if len(translatedList) > 2 else [] - sceneMenuCommonHelpText = translatedList[3] if len(translatedList) > 3 else [] - sceneMenuHelpText = translatedList[4] if len(translatedList) > 4 else [] - sceneMenuDrawText = translatedList[5] if len(translatedList) > 5 else [] - sceneMenuStatLabel = translatedList[6] if len(translatedList) > 6 else [] - saveParamName = translatedList[7] if len(translatedList) > 7 else [] - setData = True - else: - questList = [[], [], [], [], [], []] - custom = [] - sceneMenuText = [] - sceneMenuCommonHelpText = [] - sceneMenuHelpText = [] - sceneMenuDrawText = [] - sceneMenuStatLabel = [] - saveParamName = [] - setData = False - currentGroup = [] - tokens = [0, 0] - speaker = "" - voice = False - global LOCK, ESTIMATE - i = 0 - in_disp_list = False - brace_count = 0 - _pending_drawtext = {} # Deferred drawTextEx brace label replacements (longest-first) - _pending_statlabel = {} # Deferred drawTextEx stat label replacements (longest-first) - - # Category - with open("log/translations.txt", "a+", encoding="utf-8") as tlFile: - tlFile.write(f"\nCustom:\n") - tlFile.close() - - while i < len(data): - voice = False - speaker = "" - - # Track if we're inside CBR_travel_data block - if 'data_map_name_list' in data[i]: - in_disp_list = True - brace_count = 0 - - if in_disp_list: - # Count braces to know when we exit the object - brace_count += data[i].count('{') - data[i].count('}') - if brace_count < 0: - in_disp_list = False - - # Nested array strings (e.g., this.disp_list = { key: { subkey: ["string1", "string2"] } }) - # Matches strings inside arrays within object literals - if in_disp_list: - regex_nested = r'"([^"]*[一-龠ぁ-ゔァ-ヴー]+[^"]*)"' - matchList = re.findall(regex_nested, data[i]) - if len(matchList) > 0: - for match in matchList: - # Save Original String - jaString = match - originalString = jaString - - # Replace \n for translation - jaStringClean = jaString.replace("\\n", " ") - - if jaStringClean.strip(): - # Pass 1 - if setData == False: - custom.append(jaStringClean.strip()) - - # Pass 2 - else: - if custom: - # Grab and Pop - translatedText = custom[0] - custom.pop(0) - - # Set to None if empty list - if len(custom) <= 0: - custom = [] - - # Restore original newlines but keep translated text - # Count original newlines - newline_count = jaString.count("\\n") - if newline_count > 0: - # Textwrap the translation - translatedText = dazedwrap.wrapText(translatedText, 50) - translatedText = translatedText.replace("\n", "\\n") - - # Escape quotes in translation - translatedText = translatedText.replace('"', '\\"') - - # Set Data - data[i] = data[i].replace(f'"{originalString}"', f'"{translatedText}"') - saveCheckLines(data, filename) - - # Custom - # Useful Regex's - # r'"Text[\\]+":[\\]+"(.+?)[\\]+",' - # r'"HelpText[\\]+":[\\]+"(.+?)[\\]+",' - # r"this.drawTextEx\(\\'(.+?)\'," - # r'txtSubject.+?"(.+)"' - regex = r'txtSubject.+?"(.+)"' - matchList = re.findall(regex, data[i]) - if len(matchList) > 0: - for match in matchList: - # Save Original String - jaString = match - originalString = jaString - newline = None - colorCode = None - - # Make sure it contains Japanese - if not re.search(LANGREGEX, jaString): - continue - - # Make sure didn't grab \\ - if re.search(r"^[\\]+$", jaString): - continue - - # Replace \n and \c - if re.search(r"\\+n", jaString): - newline = re.search(r"\\+n", jaString).group(0) - jaString = re.sub(r"\\+n", r"\\n", jaString) - if re.search(r"\\+C", jaString): - colorCode = re.search(r"\\+C", jaString).group(0) - jaString = re.sub(r"\\+C", r"\\C", jaString) - - # Remove any textwrap - jaString = jaString.replace("\\n", " ") - - if jaString.replace("\u3000", "") and jaString: - # Pass 1 - if setData == False and jaString.strip(): - custom.append(jaString.strip()) - - # Pass 2 - else: - if custom: - # Grab and Pop - translatedText = custom[0] - custom.pop(0) - - # Set to None if empty list - if len(translatedList) <= 0: - translatedList = None - - # Replace Single Quotes - translatedText = re.sub(r"([^\\'])'", r"\1՚", translatedText) - translatedText = re.sub(r"([^\\'])\"", r"\1՚", translatedText) - - # Replace \n and \c - if newline: - # Textwrap - # translatedText = dazedwrap.wrapText(translatedText, WIDTH) - translatedText = re.sub(r"\n", re.escape(newline), translatedText) - if colorCode: - translatedText = re.sub(r"\n", re.escape(colorCode), translatedText) - - # Set Data - with open("log/translations.txt", "a+", encoding="utf-8") as tlFile: - tlFile.write(f"{originalString} ({translatedText})\n") - data[i] = data[i].replace(originalString, translatedText) - saveCheckLines(data, filename) - - # Quest Name - regex = r'[\\]+"QuestName[\\]+":[\\]+"(.*?)[\\]+"' - matchList = re.findall(regex, data[i]) - if len(matchList) > 0: - for match in matchList: - if match: - # Save Original String - originalString = match - - # Remove any textwrap - match = match.replace(newline, " ") - - # Pass 1 - if setData == False: - # Add String - if match != "\\\\\\\\": - questList[0].append(match.strip()) - - # Pass 2 - else: - if questList[0]: - # Grab and Pop - translatedText = questList[0][0] - questList[0].pop(0) - - # Set to None if empty list - if len(translatedList) <= 0: - translatedList = None - - # Replace Single Quotes - translatedText = translatedText.replace('"', "'") - translatedText = re.sub(r"([^\\'])'", r"\1\\'", translatedText) - - # Set Data - data[i] = data[i].replace(originalString, translatedText) - saveCheckLines(data, filename) - - # Quest Client - regex = r'QuestClientName[\\]+":[\\]+"(.*?)[\\]+"' - matchList = re.findall(regex, data[i]) - if len(matchList) > 0: - for match in matchList: - if match: - # Save Original String - originalString = match - - # Pass 1 - if setData == False: - # Add String - if match != "\\\\\\\\": - questList[1].append(match.strip()) - - # Pass 2 - else: - if questList[1]: - # Grab and Pop - translatedText = questList[1][0] - questList[1].pop(0) - - # Set to None if empty list - if len(translatedList) <= 0: - translatedList = None - - # Replace Single Quotes - translatedText = translatedText.replace('"', "'") - translatedText = re.sub(r"([^\\'])'", r"\1\\'", translatedText) - - # Set Data - data[i] = data[i].replace(originalString, translatedText) - saveCheckLines(data, filename) - - # Quest Location - regex = r'QuestLocation[\\]+":[\\]+"(.*?)[\\]+"' - matchList = re.findall(regex, data[i]) - if len(matchList) > 0: - for match in matchList: - if match: - # Save Original String - originalString = match - - # Pass 1 - if setData == False: - # Add String - if match != "\\\\\\\\": - questList[2].append(match.strip()) - - # Pass 2 - else: - if questList[2]: - # Grab and Pop - translatedText = questList[2][0] - questList[2].pop(0) - - # Set to None if empty list - if len(translatedList) <= 0: - translatedList = None - - # Replace Single Quotes - translatedText = translatedText.replace('"', "'") - translatedText = re.sub(r"([^\\'])'", r"\1\\'", translatedText) - - # Set Data - data[i] = data[i].replace(originalString, translatedText) - saveCheckLines(data, filename) - - # Quest Target - regex = r'PlaceInformation[\\]+":[\\]+"(.*?)[\\]+"' - matchList = re.findall(regex, data[i]) - if len(matchList) > 0: - for match in matchList: - if match: - # Save Original String - originalString = match - - # Pass 1 - if setData == False: - # Add String - if match != "\\\\\\\\": - questList[3].append(match.strip()) - - # Pass 2 - else: - if questList[3]: - # Grab and Pop - translatedText = questList[3][0] - questList[3].pop(0) - - # Set to None if empty list - if len(translatedList) <= 0: - translatedList = None - - # Replace Single Quotes - translatedText = translatedText.replace('"', "'") - translatedText = re.sub(r"([^\\'])'", r"\1\\'", translatedText) - - # Set Data - data[i] = data[i].replace(originalString, translatedText) - saveCheckLines(data, filename) - - # Quest Summary - regex = r'[\\]+"QuestContent[\\]+":[\\]+"[\\]+"(.*?)[\\]+"[\\]+"' - matchList = re.findall(regex, data[i]) - if len(matchList) > 0: - for match in matchList: - if match: - # Save Original String - originalString = match - - # Remove any textwrap - match = match.replace(r"\\\\\\\\n", " ") - match = match.replace(r"\\\\\\\\\\\\\\\\c", "\\c") - - # Pass 1 - if setData == False: - # Add String - if match != "\\\\\\\\": - questList[4].append(match.strip()) - - # Pass 2 - else: - if questList[4]: - # Grab and Pop - translatedText = questList[4][0] - questList[4].pop(0) - - # Set to None if empty list - if len(translatedList) <= 0: - translatedList = None - - # Textwrap - translatedText = dazedwrap.wrapText(translatedText, width=WIDTH) - translatedText = translatedText.replace("\n", r"\\\\\\\\n") - match = match.replace("\\c", r"\\\\\\\\\\\\\\\\c") - - # Replace Single Quotes - translatedText = translatedText.replace('"', "'") - translatedText = re.sub(r"([^\\'])'", r"\1\\'", translatedText) - - # Set Data - data[i] = data[i].replace(originalString, translatedText) - saveCheckLines(data, filename) - - # Quest Goal 1 - regex = r'ObjectiveContent[\\]+":[\\]+"[\\]+"(.*?)[\\]+"' - matchList = re.findall(regex, data[i]) - if len(matchList) > 0: - for match in matchList: - if match: - # Save Original String - originalString = match - - # Remove any textwrap - match = match.replace(r"\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\n", " ") - - # Pass 1 - if setData == False: - # Add String - if match != "\\\\\\\\": - questList[5].append(match.strip()) - - # Pass 2 - else: - if questList[5]: - # Grab and Pop - translatedText = questList[5][0] - questList[5].pop(0) - - # Set to None if empty list - if len(translatedList) <= 0: - translatedList = None - - # Textwrap - translatedText = dazedwrap.wrapText(translatedText, width=WIDTH) - translatedText = translatedText.replace("\n", r"\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\n") - - # Replace Single Quotes - translatedText = translatedText.replace('"', "'") - translatedText = re.sub(r"([^\\'])'", r"\1\\'", translatedText) - - # Set Data - data[i] = data[i].replace(originalString, translatedText) - saveCheckLines(data, filename) - - # SceneCustomMenu - Text (Command Labels) - regex = r'[\\]+"Text[\\]+":[\\]+"(.+?)[\\]+"' - matchList = re.findall(regex, data[i]) - if len(matchList) > 0: - for match in matchList: - if match: - jaString = match - originalString = jaString - - # Make sure it contains Japanese - if not re.search(LANGREGEX, jaString): - continue - - # Make sure didn't grab only backslashes - if re.search(r"^[\\]+$", jaString): - continue - - if jaString.replace("\u3000", "").strip(): - # Pass 1 - if setData == False: - sceneMenuText.append(jaString.strip()) - - # Pass 2 - else: - if sceneMenuText: - # Grab and Pop - translatedText = sceneMenuText[0] - sceneMenuText.pop(0) - - # Replace quotes - translatedText = translatedText.replace('"', "'") - translatedText = re.sub(r"([^\\'])'" , r"\1\\'", translatedText) - - # Set Data - data[i] = data[i].replace(originalString, translatedText) - saveCheckLines(data, filename) - - # SceneCustomMenu - CommonHelpText - regex = r'[\\]+"CommonHelpText[\\]+":[\\]+"(.+?)[\\]+"' - matchList = re.findall(regex, data[i]) - if len(matchList) > 0: - for match in matchList: - if match: - jaString = match - originalString = jaString - - # Make sure it contains Japanese - if not re.search(LANGREGEX, jaString): - continue - - # Make sure didn't grab only backslashes - if re.search(r"^[\\]+$", jaString): - continue - - if jaString.replace("\u3000", "").strip(): - # Pass 1 - if setData == False: - sceneMenuCommonHelpText.append(jaString.strip()) - - # Pass 2 - else: - if sceneMenuCommonHelpText: - # Grab and Pop - translatedText = sceneMenuCommonHelpText[0] - sceneMenuCommonHelpText.pop(0) - - # Replace quotes - translatedText = translatedText.replace('"', "'") - translatedText = re.sub(r"([^\\'])'" , r"\1\\'", translatedText) - - # Set Data - data[i] = data[i].replace(originalString, translatedText) - saveCheckLines(data, filename) - - # SceneCustomMenu - HelpText (Command Help) - regex = r'[\\]+"HelpText[\\]+":[\\]+"(.+?)[\\]+"' - matchList = re.findall(regex, data[i]) - if len(matchList) > 0: - for match in matchList: - if match: - jaString = match - originalString = jaString - - # Make sure it contains Japanese - if not re.search(LANGREGEX, jaString): - continue - - # Make sure didn't grab only backslashes - if re.search(r"^[\\]+$", jaString): - continue - - if jaString.replace("\u3000", "").strip(): - # Pass 1 - if setData == False: - sceneMenuHelpText.append(jaString.strip()) - - # Pass 2 - else: - if sceneMenuHelpText: - # Grab and Pop - translatedText = sceneMenuHelpText[0] - sceneMenuHelpText.pop(0) - - # Replace quotes - translatedText = translatedText.replace('"', "'") - translatedText = re.sub(r"([^\\'])'" , r"\1\\'", translatedText) - - # Set Data - data[i] = data[i].replace(originalString, translatedText) - saveCheckLines(data, filename) - - # NUUN_SaveScreen - ParamName (save screen labels like 現在地, プレイ時間, 所持金) - regex = r'[\\]+"ParamName[\\]+":[\\]+"(.+?)[\\]+"' - matchList = re.findall(regex, data[i]) - if len(matchList) > 0: - for match in matchList: - if match: - jaString = match - originalString = jaString - - # Make sure it contains Japanese - if not re.search(LANGREGEX, jaString): - continue - - # Make sure didn't grab only backslashes - if re.search(r"^[\\]+$", jaString): - continue - - if jaString.replace("\u3000", "").strip(): - # Pass 1 - if setData == False: - saveParamName.append(jaString.strip()) - - # Pass 2 - else: - if saveParamName: - # Grab and Pop - translatedText = saveParamName[0] - saveParamName.pop(0) - - # Replace quotes - translatedText = translatedText.replace('"', "'") - translatedText = re.sub(r"([^\\'])'" , r"\1\\'", translatedText) - - # Set Data - data[i] = data[i].replace(originalString, translatedText) - saveCheckLines(data, filename) - - # DrawTextEx - Brace-delimited labels (e.g., \}【口づけ】\{, \}回数：\{) - # Only process lines containing drawTextEx calls - if 'drawTextEx' in data[i]: - regex = r'[\\]+\}(.+?)[\\]+\{' - matchList = re.findall(regex, data[i]) - if len(matchList) > 0: - for match in matchList: - if match: - jaString = match - originalString = jaString - - # Skip game variable references (handled separately) - if '$gameVariables' in jaString: - continue - - # Make sure it contains Japanese - if not re.search(LANGREGEX, jaString): - continue - - if jaString.replace("\u3000", "").strip(): - # Pass 1 (deduplicated) - if setData == False: - if jaString.strip() not in sceneMenuDrawText: - sceneMenuDrawText.append(jaString.strip()) - - # Pass 2 (collect mapping, skip duplicates) - else: - if sceneMenuDrawText and originalString not in _pending_drawtext: - # Grab and Pop - translatedText = sceneMenuDrawText[0] - sceneMenuDrawText.pop(0) - - # Replace quotes - translatedText = translatedText.replace('"', "'") - - # Queue for deferred replacement - _pending_drawtext[originalString] = translatedText - - # Apply deferred replacements longest-first to avoid substring collisions - # (e.g., replacing 回数： before 使用回数： would corrupt the longer string) - if setData and _pending_drawtext: - for orig in sorted(_pending_drawtext, key=len, reverse=True): - data[i] = data[i].replace(orig, _pending_drawtext[orig]) - _pending_drawtext.clear() - saveCheckLines(data, filename) - - # DrawTextEx - Stat labels (e.g., C[16]攻撃力\\...C[0]) - if 'drawTextEx' in data[i]: - regex = r'C\[\d+\]([^C\\,`\[\]]+?)[\\]+C\[0\]' - matchList = re.findall(regex, data[i]) - if len(matchList) > 0: - for match in matchList: - if match: - jaString = match - originalString = jaString - - # Make sure it contains Japanese - if not re.search(LANGREGEX, jaString): - continue - - if jaString.replace("\u3000", "").strip(): - # Pass 1 (deduplicated) - if setData == False: - if jaString.strip() not in sceneMenuStatLabel: - sceneMenuStatLabel.append(jaString.strip()) - - # Pass 2 (collect mapping, skip duplicates) - else: - if sceneMenuStatLabel and originalString not in _pending_statlabel: - # Grab and Pop - translatedText = sceneMenuStatLabel[0] - sceneMenuStatLabel.pop(0) - - # Queue for deferred replacement - _pending_statlabel[originalString] = translatedText - - # Apply deferred replacements longest-first to avoid substring collisions - if setData and _pending_statlabel: - for orig in sorted(_pending_statlabel, key=len, reverse=True): - data[i] = data[i].replace(orig, _pending_statlabel[orig]) - _pending_statlabel.clear() - saveCheckLines(data, filename) - - # DrawTextEx - Game variable counter word (hardcoded: 回 → time(s)) - if setData and 'drawTextEx' in data[i] and '$gameVariables' in data[i]: - data[i] = re.sub( - r'(\$\{\$gameVariables\.value\(\d+\)\}\s*)回', - r'\1x', - data[i] - ) - saveCheckLines(data, filename) - - # Next Line - i += 1 - - # EOF - translate = False - questListTL = [[], [], [], [], [], []] - customTL = [] - - # Compute combined total for progress bar across all categories - combinedTotal = ( - sum(len(quest) for quest in questList) - + len(custom) - + len(sceneMenuText) - + len(sceneMenuCommonHelpText) - + len(sceneMenuHelpText) - + len(sceneMenuDrawText) - + len(sceneMenuStatLabel) - + len(saveParamName) - ) - if combinedTotal > 0: - pbar.total = combinedTotal - pbar.refresh() - - # Quest - if len(questList) > 0: - # Quest Name - response = translateAI(questList[0], "Quest Name") - tokens[0] += response[1][0] - tokens[1] += response[1][1] - questName = response[0] - - # Quest Client - response = translateAI(questList[1], "Quest Client") - tokens[0] += response[1][0] - tokens[1] += response[1][1] - questClient = response[0] - - # Quest Location - response = translateAI(questList[2], "Quest Location") - tokens[0] += response[1][0] - tokens[1] += response[1][1] - questLocation = response[0] - - # Quest Target - response = translateAI(questList[3], "Quest Location") - tokens[0] += response[1][0] - tokens[1] += response[1][1] - questTarget = response[0] - - # Quest Summary - response = translateAI(questList[4], "Quest Summary") - tokens[0] += response[1][0] - tokens[1] += response[1][1] - questSummary = response[0] - - # Quest Goal 1 - response = translateAI(questList[5], "Quest Goal") - tokens[0] += response[1][0] - tokens[1] += response[1][1] - questGoal1 = response[0] - - # Check Mismatch - if ( - len(questName) == len(questList[0]) - or len(questClient) == len(questList[1]) - or len(questLocation) == len(questList[2]) - or len(questTarget) == len(questList[3]) - or len(questSummary) == len(questList[4]) - or len(questGoal1) == len(questList[5]) - ): - # Set Strings - questListTL = [questName, questClient, questLocation, questTarget, questSummary, questGoal1] - translate = True - - # Mismatch - else: - with LOCK: - if filename not in MISMATCH: - MISMATCH.append(filename) - - # Custom - if custom: - # TL - response = translateAI(custom, "Relic Name") - tokens[0] += response[1][0] - tokens[1] += response[1][1] - customResponse = response[0] - - # Check Mismatch - if len(custom) == len(customResponse): - customTL = customResponse - translate = True - - # Mismatch - else: - with LOCK: - if filename not in MISMATCH: - MISMATCH.append(filename) - - # SceneCustomMenu Text - sceneMenuTextTL = [] - sceneMenuCommonHelpTextTL = [] - sceneMenuHelpTextTL = [] - sceneMenuDrawTextTL = [] - sceneMenuStatLabelTL = [] - saveParamNameTL = [] - - if sceneMenuText: - # TL - response = translateAI(sceneMenuText, "Menu Item") - tokens[0] += response[1][0] - tokens[1] += response[1][1] - sceneMenuTextResponse = response[0] - - # Check Mismatch - if len(sceneMenuText) == len(sceneMenuTextResponse): - sceneMenuTextTL = sceneMenuTextResponse - translate = True - - # Mismatch - else: - with LOCK: - if filename not in MISMATCH: - MISMATCH.append(filename) - - # SceneCustomMenu CommonHelpText - if sceneMenuCommonHelpText: - # TL - response = translateAI(sceneMenuCommonHelpText, "Menu Help Text") - tokens[0] += response[1][0] - tokens[1] += response[1][1] - sceneMenuCommonHelpTextResponse = response[0] - - # Check Mismatch - if len(sceneMenuCommonHelpText) == len(sceneMenuCommonHelpTextResponse): - sceneMenuCommonHelpTextTL = sceneMenuCommonHelpTextResponse - translate = True - - # Mismatch - else: - with LOCK: - if filename not in MISMATCH: - MISMATCH.append(filename) - - # SceneCustomMenu HelpText - if sceneMenuHelpText: - # TL - response = translateAI(sceneMenuHelpText, "Menu Help Text") - tokens[0] += response[1][0] - tokens[1] += response[1][1] - sceneMenuHelpTextResponse = response[0] - - # Check Mismatch - if len(sceneMenuHelpText) == len(sceneMenuHelpTextResponse): - sceneMenuHelpTextTL = sceneMenuHelpTextResponse - translate = True - - # Mismatch - else: - with LOCK: - if filename not in MISMATCH: - MISMATCH.append(filename) - - # DrawTextEx - Brace-delimited labels (tracker section headers and field labels) - if sceneMenuDrawText: - # TL - response = translateAI(sceneMenuDrawText, "Status Tracker Label") - tokens[0] += response[1][0] - tokens[1] += response[1][1] - sceneMenuDrawTextResponse = response[0] - - # Check Mismatch - if len(sceneMenuDrawText) == len(sceneMenuDrawTextResponse): - sceneMenuDrawTextTL = sceneMenuDrawTextResponse - translate = True - - # Mismatch - else: - with LOCK: - if filename not in MISMATCH: - MISMATCH.append(filename) - - # DrawTextEx - Stat labels (e.g., 攻撃力, 魔力, etc.) - if sceneMenuStatLabel: - # TL - response = translateAI(sceneMenuStatLabel, "Character Stat Name") - tokens[0] += response[1][0] - tokens[1] += response[1][1] - sceneMenuStatLabelResponse = response[0] - - # Check Mismatch - if len(sceneMenuStatLabel) == len(sceneMenuStatLabelResponse): - sceneMenuStatLabelTL = sceneMenuStatLabelResponse - translate = True - - # Mismatch - else: - with LOCK: - if filename not in MISMATCH: - MISMATCH.append(filename) - - # NUUN_SaveScreen ParamName - if saveParamName: - # TL - response = translateAI(saveParamName, "Save Screen Label") - tokens[0] += response[1][0] - tokens[1] += response[1][1] - saveParamNameResponse = response[0] - - # Check Mismatch - if len(saveParamName) == len(saveParamNameResponse): - saveParamNameTL = saveParamNameResponse - translate = True - - # Mismatch - else: - with LOCK: - if filename not in MISMATCH: - MISMATCH.append(filename) - - # Pass 2 - if translate and not setData: - translatePlugin(data, pbar, filename, [questListTL, customTL, sceneMenuTextTL, sceneMenuCommonHelpTextTL, sceneMenuHelpTextTL, sceneMenuDrawTextTL, sceneMenuStatLabelTL, saveParamNameTL]) - return tokens - -# Save some money and enter the character before translation -def getSpeaker(speaker): - match speaker: - case "ファイン": - return ["Fine", [0, 0]] - case "": - return ["", [0, 0]] - case _: - # Find Speaker - for i in range(len(NAMESLIST)): - if speaker == NAMESLIST[i][0]: - return [NAMESLIST[i][1], [0, 0]] - - # Translate and Store Speaker - response = translateAI( - f"{speaker}", - "Reply with the " + LANGUAGE + " translation of the NPC name.", - False, - ) - response[0] = response[0].title() - response[0] = response[0].replace("'S", "'s") - response[0] = response[0].replace("Speaker: ", "") - - # Retry if name doesn't translate for some reason - if re.search(r"([a-zA-Z？?])", response[0]) == None: - response = translateAI( - f"{speaker}", - "Reply with the " + LANGUAGE + " translation of the NPC name.", - False, - ) - response[0] = response[0].title() - response[0] = response[0].replace("'S", "'s") - - speakerList = [speaker, response[0]] - NAMESLIST.append(speakerList) - return response - return [speaker, [0, 0]] - -def translateAI(text, history, history_ctx=None): - """ - Legacy wrapper function for the new shared translation utility. - This maintains compatibility with existing code while using the new shared implementation. - """ - global PBAR, MISMATCH, FILENAME - - # Update config estimate mode based on global ESTIMATE - TRANSLATION_CONFIG.estimateMode = bool(ESTIMATE) - - # Call the new shared translation function - return sharedtranslateAI( - text=text, - history=history, - config=TRANSLATION_CONFIG, - filename=FILENAME, - pbar=PBAR, - lock=LOCK, - mismatchList=MISMATCH - ) +# Libraries +import json +import os +import re +import util.dazedwrap as dazedwrap +import threading +import time +import traceback +import tiktoken +from pathlib import Path +from colorama import Fore +from dotenv import load_dotenv +from retry import retry +from tqdm import tqdm +from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost +import tempfile + +# Globals +MODEL = os.getenv("model") +TIMEOUT = int(os.getenv("timeout")) +LANGUAGE = os.getenv("language").capitalize() +from util.paths import PROMPT_PATH, VOCAB_PATH + +PROMPT = PROMPT_PATH.read_text(encoding="utf-8") +VOCAB = VOCAB_PATH.read_text(encoding="utf-8") +LOCK = threading.Lock() +WIDTH = int(os.getenv("width")) +LISTWIDTH = int(os.getenv("listWidth")) +NOTEWIDTH = 70 +MAXHISTORY = 10 +ESTIMATE = "" +TOKENS = [0, 0] +NAMESLIST = [] +NAMES = False # Output a list of all the character names found +BRFLAG = False # If the game uses
instead +FIXTEXTWRAP = True # Overwrites textwrap +IGNORETLTEXT = False # Ignores all translated text. +MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong) +FILENAME = None + +# tqdm Globals +BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}" +POSITION = 0 +LEAVE = False +PBAR = None + +# Regex - Need to change this if you want to translate from/to other languages. Default is Japanese Regex +LANGREGEX = r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+" + +# Get pricing configuration based on the model +PRICING_CONFIG = getPricingConfig(MODEL) +INPUTAPICOST = PRICING_CONFIG["inputAPICost"] +OUTPUTAPICOST = PRICING_CONFIG["outputAPICost"] +BATCHSIZE = PRICING_CONFIG["batchSize"] +FREQUENCY_PENALTY = PRICING_CONFIG["frequencyPenalty"] + +# Initialize Translation Config +TRANSLATION_CONFIG = TranslationConfig( + model=MODEL, + language=LANGUAGE, + prompt=PROMPT, + vocab=VOCAB, + langRegex=LANGREGEX, + batchSize=BATCHSIZE, + maxHistory=MAXHISTORY, + estimateMode=False # Will be set dynamically based on ESTIMATE +) +LEAVE = False + +def handlePlugin(filename, estimate): + global ESTIMATE, PBAR, FILENAME + ESTIMATE = estimate + FILENAME = filename + + if ESTIMATE: + start = time.time() + translatedData = openFiles(filename) + + # Print Result + end = time.time() + tqdm.write(getResultString(translatedData, end - start, filename)) + with LOCK: + TOKENS[0] += translatedData[1][0] + TOKENS[1] += translatedData[1][1] + + # Print Total + totalString = getResultString(["", TOKENS, None], end - start, "TOTAL") + + # Print any errors on maps + if len(MISMATCH) > 0: + return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET + else: + return totalString + + else: + try: + # Perform translation first; incremental progress writes happen during translation. + # We no longer keep the destination file open simultaneously to avoid Windows locking issues + # during atomic os.replace operations inside save_progress_lines. + start = time.time() + translatedData = openFiles(filename) + + # Ensure final state is flushed (in case no incremental writes occurred for some reason). + save_progress_lines(translatedData[0], filename) + + # Print Result + end = time.time() + tqdm.write(getResultString(translatedData, end - start, filename)) + with LOCK: + TOKENS[0] += translatedData[1][0] + TOKENS[1] += translatedData[1][1] + except Exception: + traceback.print_exc() + return "Fail" + + return getResultString(["", TOKENS, None], end - start, "TOTAL") + + +def getResultString(translatedData, translationTime, filename): + # File Print String + cost = calculateCost(translatedData[1][0], translatedData[1][1], MODEL) + totalTokenstring = ( + Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]" + "[Output: " + + str(translatedData[1][1]) + + "]" "[Cost: ${:,.4f}".format(cost) + + "]" + ) + timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]" + + if translatedData[2] == None: + # Success + return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET + + else: + # Fail + try: + raise translatedData[2] + except Exception as e: + traceback.print_exc() + errorString = str(e) + Fore.RED + return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET + + +def openFiles(filename): + with open("files/" + filename, "r", encoding="utf_8") as readFile: + translatedData = parsePlugin(readFile, filename) + + # Delete lines marked for deletion + finalData = [] + for line in translatedData[0]: + if line != "\\d\n": + finalData.append(line) + translatedData[0] = finalData + + return translatedData + + +def parsePlugin(readFile, filename): + global PBAR + totalTokens = [0, 0] + + # Read File into data + data = readFile.readlines() + + # Create Progress Bar + with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar: + pbar.desc = filename + PBAR = pbar + + try: + result = translatePlugin(data, pbar, filename, []) + totalTokens[0] += result[0] + totalTokens[1] += result[1] + except Exception as e: + traceback.print_exc() + return [data, totalTokens, e] + return [data, totalTokens, None] + + +def save_progress_lines(lines, filename, encoding="utf_8"): + """Atomically save current line-based translation progress.""" + try: + if ESTIMATE: + return + global LOCK + os.makedirs("translated", exist_ok=True) + # Use a single lock to prevent concurrent replace attempts on Windows (which causes PermissionError) + with LOCK: + tmp_fd, tmp_path = tempfile.mkstemp(prefix=f"{filename}.", suffix=".tmp", dir="translated") + try: + with os.fdopen(tmp_fd, "w", encoding=encoding, newline="\n", errors="ignore") as tmp_file: + tmp_file.writelines(lines) + + dest_path = os.path.join("translated", filename) + # Retry a few times in case another process/thread still has the file open momentarily. + for attempt in range(5): + try: + os.replace(tmp_path, dest_path) + break + except PermissionError: + if attempt == 4: + raise + time.sleep(0.1 * (attempt + 1)) + finally: + if os.path.exists(tmp_path): + try: + os.remove(tmp_path) + except OSError: + pass + except Exception: + traceback.print_exc() + + +def saveCheckLines(lines, filename, tokens=None, encoding="utf_8"): + """Save progress only when tokens indicate work or when explicitly called after a mutation.""" + try: + if tokens is not None: + if not (isinstance(tokens, (list, tuple)) and len(tokens) >= 2 and (tokens[0] or tokens[1])): + return + save_progress_lines(lines, filename, encoding=encoding) + except Exception: + traceback.print_exc() + + +def translatePlugin(data, pbar, filename, translatedList): + if len(translatedList) > 0: + questList = translatedList[0] + custom = translatedList[1] + sceneMenuText = translatedList[2] if len(translatedList) > 2 else [] + sceneMenuCommonHelpText = translatedList[3] if len(translatedList) > 3 else [] + sceneMenuHelpText = translatedList[4] if len(translatedList) > 4 else [] + sceneMenuDrawText = translatedList[5] if len(translatedList) > 5 else [] + sceneMenuStatLabel = translatedList[6] if len(translatedList) > 6 else [] + saveParamName = translatedList[7] if len(translatedList) > 7 else [] + setData = True + else: + questList = [[], [], [], [], [], []] + custom = [] + sceneMenuText = [] + sceneMenuCommonHelpText = [] + sceneMenuHelpText = [] + sceneMenuDrawText = [] + sceneMenuStatLabel = [] + saveParamName = [] + setData = False + currentGroup = [] + tokens = [0, 0] + speaker = "" + voice = False + global LOCK, ESTIMATE + i = 0 + in_disp_list = False + brace_count = 0 + _pending_drawtext = {} # Deferred drawTextEx brace label replacements (longest-first) + _pending_statlabel = {} # Deferred drawTextEx stat label replacements (longest-first) + + # Category + with open("log/translations.txt", "a+", encoding="utf-8") as tlFile: + tlFile.write(f"\nCustom:\n") + tlFile.close() + + while i < len(data): + voice = False + speaker = "" + + # Track if we're inside CBR_travel_data block + if 'data_map_name_list' in data[i]: + in_disp_list = True + brace_count = 0 + + if in_disp_list: + # Count braces to know when we exit the object + brace_count += data[i].count('{') - data[i].count('}') + if brace_count < 0: + in_disp_list = False + + # Nested array strings (e.g., this.disp_list = { key: { subkey: ["string1", "string2"] } }) + # Matches strings inside arrays within object literals + if in_disp_list: + regex_nested = r'"([^"]*[一-龠ぁ-ゔァ-ヴー]+[^"]*)"' + matchList = re.findall(regex_nested, data[i]) + if len(matchList) > 0: + for match in matchList: + # Save Original String + jaString = match + originalString = jaString + + # Replace \n for translation + jaStringClean = jaString.replace("\\n", " ") + + if jaStringClean.strip(): + # Pass 1 + if setData == False: + custom.append(jaStringClean.strip()) + + # Pass 2 + else: + if custom: + # Grab and Pop + translatedText = custom[0] + custom.pop(0) + + # Set to None if empty list + if len(custom) <= 0: + custom = [] + + # Restore original newlines but keep translated text + # Count original newlines + newline_count = jaString.count("\\n") + if newline_count > 0: + # Textwrap the translation + translatedText = dazedwrap.wrapText(translatedText, 50) + translatedText = translatedText.replace("\n", "\\n") + + # Escape quotes in translation + translatedText = translatedText.replace('"', '\\"') + + # Set Data + data[i] = data[i].replace(f'"{originalString}"', f'"{translatedText}"') + saveCheckLines(data, filename) + + # Custom + # Useful Regex's + # r'"Text[\\]+":[\\]+"(.+?)[\\]+",' + # r'"HelpText[\\]+":[\\]+"(.+?)[\\]+",' + # r"this.drawTextEx\(\\'(.+?)\'," + # r'txtSubject.+?"(.+)"' + regex = r'txtSubject.+?"(.+)"' + matchList = re.findall(regex, data[i]) + if len(matchList) > 0: + for match in matchList: + # Save Original String + jaString = match + originalString = jaString + newline = None + colorCode = None + + # Make sure it contains Japanese + if not re.search(LANGREGEX, jaString): + continue + + # Make sure didn't grab \\ + if re.search(r"^[\\]+$", jaString): + continue + + # Replace \n and \c + if re.search(r"\\+n", jaString): + newline = re.search(r"\\+n", jaString).group(0) + jaString = re.sub(r"\\+n", r"\\n", jaString) + if re.search(r"\\+C", jaString): + colorCode = re.search(r"\\+C", jaString).group(0) + jaString = re.sub(r"\\+C", r"\\C", jaString) + + # Remove any textwrap + jaString = jaString.replace("\\n", " ") + + if jaString.replace("\u3000", "") and jaString: + # Pass 1 + if setData == False and jaString.strip(): + custom.append(jaString.strip()) + + # Pass 2 + else: + if custom: + # Grab and Pop + translatedText = custom[0] + custom.pop(0) + + # Set to None if empty list + if len(translatedList) <= 0: + translatedList = None + + # Replace Single Quotes + translatedText = re.sub(r"([^\\'])'", r"\1՚", translatedText) + translatedText = re.sub(r"([^\\'])\"", r"\1՚", translatedText) + + # Replace \n and \c + if newline: + # Textwrap + # translatedText = dazedwrap.wrapText(translatedText, WIDTH) + translatedText = re.sub(r"\n", re.escape(newline), translatedText) + if colorCode: + translatedText = re.sub(r"\n", re.escape(colorCode), translatedText) + + # Set Data + with open("log/translations.txt", "a+", encoding="utf-8") as tlFile: + tlFile.write(f"{originalString} ({translatedText})\n") + data[i] = data[i].replace(originalString, translatedText) + saveCheckLines(data, filename) + + # Quest Name + regex = r'[\\]+"QuestName[\\]+":[\\]+"(.*?)[\\]+"' + matchList = re.findall(regex, data[i]) + if len(matchList) > 0: + for match in matchList: + if match: + # Save Original String + originalString = match + + # Remove any textwrap + match = match.replace(newline, " ") + + # Pass 1 + if setData == False: + # Add String + if match != "\\\\\\\\": + questList[0].append(match.strip()) + + # Pass 2 + else: + if questList[0]: + # Grab and Pop + translatedText = questList[0][0] + questList[0].pop(0) + + # Set to None if empty list + if len(translatedList) <= 0: + translatedList = None + + # Replace Single Quotes + translatedText = translatedText.replace('"', "'") + translatedText = re.sub(r"([^\\'])'", r"\1\\'", translatedText) + + # Set Data + data[i] = data[i].replace(originalString, translatedText) + saveCheckLines(data, filename) + + # Quest Client + regex = r'QuestClientName[\\]+":[\\]+"(.*?)[\\]+"' + matchList = re.findall(regex, data[i]) + if len(matchList) > 0: + for match in matchList: + if match: + # Save Original String + originalString = match + + # Pass 1 + if setData == False: + # Add String + if match != "\\\\\\\\": + questList[1].append(match.strip()) + + # Pass 2 + else: + if questList[1]: + # Grab and Pop + translatedText = questList[1][0] + questList[1].pop(0) + + # Set to None if empty list + if len(translatedList) <= 0: + translatedList = None + + # Replace Single Quotes + translatedText = translatedText.replace('"', "'") + translatedText = re.sub(r"([^\\'])'", r"\1\\'", translatedText) + + # Set Data + data[i] = data[i].replace(originalString, translatedText) + saveCheckLines(data, filename) + + # Quest Location + regex = r'QuestLocation[\\]+":[\\]+"(.*?)[\\]+"' + matchList = re.findall(regex, data[i]) + if len(matchList) > 0: + for match in matchList: + if match: + # Save Original String + originalString = match + + # Pass 1 + if setData == False: + # Add String + if match != "\\\\\\\\": + questList[2].append(match.strip()) + + # Pass 2 + else: + if questList[2]: + # Grab and Pop + translatedText = questList[2][0] + questList[2].pop(0) + + # Set to None if empty list + if len(translatedList) <= 0: + translatedList = None + + # Replace Single Quotes + translatedText = translatedText.replace('"', "'") + translatedText = re.sub(r"([^\\'])'", r"\1\\'", translatedText) + + # Set Data + data[i] = data[i].replace(originalString, translatedText) + saveCheckLines(data, filename) + + # Quest Target + regex = r'PlaceInformation[\\]+":[\\]+"(.*?)[\\]+"' + matchList = re.findall(regex, data[i]) + if len(matchList) > 0: + for match in matchList: + if match: + # Save Original String + originalString = match + + # Pass 1 + if setData == False: + # Add String + if match != "\\\\\\\\": + questList[3].append(match.strip()) + + # Pass 2 + else: + if questList[3]: + # Grab and Pop + translatedText = questList[3][0] + questList[3].pop(0) + + # Set to None if empty list + if len(translatedList) <= 0: + translatedList = None + + # Replace Single Quotes + translatedText = translatedText.replace('"', "'") + translatedText = re.sub(r"([^\\'])'", r"\1\\'", translatedText) + + # Set Data + data[i] = data[i].replace(originalString, translatedText) + saveCheckLines(data, filename) + + # Quest Summary + regex = r'[\\]+"QuestContent[\\]+":[\\]+"[\\]+"(.*?)[\\]+"[\\]+"' + matchList = re.findall(regex, data[i]) + if len(matchList) > 0: + for match in matchList: + if match: + # Save Original String + originalString = match + + # Remove any textwrap + match = match.replace(r"\\\\\\\\n", " ") + match = match.replace(r"\\\\\\\\\\\\\\\\c", "\\c") + + # Pass 1 + if setData == False: + # Add String + if match != "\\\\\\\\": + questList[4].append(match.strip()) + + # Pass 2 + else: + if questList[4]: + # Grab and Pop + translatedText = questList[4][0] + questList[4].pop(0) + + # Set to None if empty list + if len(translatedList) <= 0: + translatedList = None + + # Textwrap + translatedText = dazedwrap.wrapText(translatedText, width=WIDTH) + translatedText = translatedText.replace("\n", r"\\\\\\\\n") + match = match.replace("\\c", r"\\\\\\\\\\\\\\\\c") + + # Replace Single Quotes + translatedText = translatedText.replace('"', "'") + translatedText = re.sub(r"([^\\'])'", r"\1\\'", translatedText) + + # Set Data + data[i] = data[i].replace(originalString, translatedText) + saveCheckLines(data, filename) + + # Quest Goal 1 + regex = r'ObjectiveContent[\\]+":[\\]+"[\\]+"(.*?)[\\]+"' + matchList = re.findall(regex, data[i]) + if len(matchList) > 0: + for match in matchList: + if match: + # Save Original String + originalString = match + + # Remove any textwrap + match = match.replace(r"\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\n", " ") + + # Pass 1 + if setData == False: + # Add String + if match != "\\\\\\\\": + questList[5].append(match.strip()) + + # Pass 2 + else: + if questList[5]: + # Grab and Pop + translatedText = questList[5][0] + questList[5].pop(0) + + # Set to None if empty list + if len(translatedList) <= 0: + translatedList = None + + # Textwrap + translatedText = dazedwrap.wrapText(translatedText, width=WIDTH) + translatedText = translatedText.replace("\n", r"\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\n") + + # Replace Single Quotes + translatedText = translatedText.replace('"', "'") + translatedText = re.sub(r"([^\\'])'", r"\1\\'", translatedText) + + # Set Data + data[i] = data[i].replace(originalString, translatedText) + saveCheckLines(data, filename) + + # SceneCustomMenu - Text (Command Labels) + regex = r'[\\]+"Text[\\]+":[\\]+"(.+?)[\\]+"' + matchList = re.findall(regex, data[i]) + if len(matchList) > 0: + for match in matchList: + if match: + jaString = match + originalString = jaString + + # Make sure it contains Japanese + if not re.search(LANGREGEX, jaString): + continue + + # Make sure didn't grab only backslashes + if re.search(r"^[\\]+$", jaString): + continue + + if jaString.replace("\u3000", "").strip(): + # Pass 1 + if setData == False: + sceneMenuText.append(jaString.strip()) + + # Pass 2 + else: + if sceneMenuText: + # Grab and Pop + translatedText = sceneMenuText[0] + sceneMenuText.pop(0) + + # Replace quotes + translatedText = translatedText.replace('"', "'") + translatedText = re.sub(r"([^\\'])'" , r"\1\\'", translatedText) + + # Set Data + data[i] = data[i].replace(originalString, translatedText) + saveCheckLines(data, filename) + + # SceneCustomMenu - CommonHelpText + regex = r'[\\]+"CommonHelpText[\\]+":[\\]+"(.+?)[\\]+"' + matchList = re.findall(regex, data[i]) + if len(matchList) > 0: + for match in matchList: + if match: + jaString = match + originalString = jaString + + # Make sure it contains Japanese + if not re.search(LANGREGEX, jaString): + continue + + # Make sure didn't grab only backslashes + if re.search(r"^[\\]+$", jaString): + continue + + if jaString.replace("\u3000", "").strip(): + # Pass 1 + if setData == False: + sceneMenuCommonHelpText.append(jaString.strip()) + + # Pass 2 + else: + if sceneMenuCommonHelpText: + # Grab and Pop + translatedText = sceneMenuCommonHelpText[0] + sceneMenuCommonHelpText.pop(0) + + # Replace quotes + translatedText = translatedText.replace('"', "'") + translatedText = re.sub(r"([^\\'])'" , r"\1\\'", translatedText) + + # Set Data + data[i] = data[i].replace(originalString, translatedText) + saveCheckLines(data, filename) + + # SceneCustomMenu - HelpText (Command Help) + regex = r'[\\]+"HelpText[\\]+":[\\]+"(.+?)[\\]+"' + matchList = re.findall(regex, data[i]) + if len(matchList) > 0: + for match in matchList: + if match: + jaString = match + originalString = jaString + + # Make sure it contains Japanese + if not re.search(LANGREGEX, jaString): + continue + + # Make sure didn't grab only backslashes + if re.search(r"^[\\]+$", jaString): + continue + + if jaString.replace("\u3000", "").strip(): + # Pass 1 + if setData == False: + sceneMenuHelpText.append(jaString.strip()) + + # Pass 2 + else: + if sceneMenuHelpText: + # Grab and Pop + translatedText = sceneMenuHelpText[0] + sceneMenuHelpText.pop(0) + + # Replace quotes + translatedText = translatedText.replace('"', "'") + translatedText = re.sub(r"([^\\'])'" , r"\1\\'", translatedText) + + # Set Data + data[i] = data[i].replace(originalString, translatedText) + saveCheckLines(data, filename) + + # NUUN_SaveScreen - ParamName (save screen labels like 現在地, プレイ時間, 所持金) + regex = r'[\\]+"ParamName[\\]+":[\\]+"(.+?)[\\]+"' + matchList = re.findall(regex, data[i]) + if len(matchList) > 0: + for match in matchList: + if match: + jaString = match + originalString = jaString + + # Make sure it contains Japanese + if not re.search(LANGREGEX, jaString): + continue + + # Make sure didn't grab only backslashes + if re.search(r"^[\\]+$", jaString): + continue + + if jaString.replace("\u3000", "").strip(): + # Pass 1 + if setData == False: + saveParamName.append(jaString.strip()) + + # Pass 2 + else: + if saveParamName: + # Grab and Pop + translatedText = saveParamName[0] + saveParamName.pop(0) + + # Replace quotes + translatedText = translatedText.replace('"', "'") + translatedText = re.sub(r"([^\\'])'" , r"\1\\'", translatedText) + + # Set Data + data[i] = data[i].replace(originalString, translatedText) + saveCheckLines(data, filename) + + # DrawTextEx - Brace-delimited labels (e.g., \}【口づけ】\{, \}回数：\{) + # Only process lines containing drawTextEx calls + if 'drawTextEx' in data[i]: + regex = r'[\\]+\}(.+?)[\\]+\{' + matchList = re.findall(regex, data[i]) + if len(matchList) > 0: + for match in matchList: + if match: + jaString = match + originalString = jaString + + # Skip game variable references (handled separately) + if '$gameVariables' in jaString: + continue + + # Make sure it contains Japanese + if not re.search(LANGREGEX, jaString): + continue + + if jaString.replace("\u3000", "").strip(): + # Pass 1 (deduplicated) + if setData == False: + if jaString.strip() not in sceneMenuDrawText: + sceneMenuDrawText.append(jaString.strip()) + + # Pass 2 (collect mapping, skip duplicates) + else: + if sceneMenuDrawText and originalString not in _pending_drawtext: + # Grab and Pop + translatedText = sceneMenuDrawText[0] + sceneMenuDrawText.pop(0) + + # Replace quotes + translatedText = translatedText.replace('"', "'") + + # Queue for deferred replacement + _pending_drawtext[originalString] = translatedText + + # Apply deferred replacements longest-first to avoid substring collisions + # (e.g., replacing 回数： before 使用回数： would corrupt the longer string) + if setData and _pending_drawtext: + for orig in sorted(_pending_drawtext, key=len, reverse=True): + data[i] = data[i].replace(orig, _pending_drawtext[orig]) + _pending_drawtext.clear() + saveCheckLines(data, filename) + + # DrawTextEx - Stat labels (e.g., C[16]攻撃力\\...C[0]) + if 'drawTextEx' in data[i]: + regex = r'C\[\d+\]([^C\\,`\[\]]+?)[\\]+C\[0\]' + matchList = re.findall(regex, data[i]) + if len(matchList) > 0: + for match in matchList: + if match: + jaString = match + originalString = jaString + + # Make sure it contains Japanese + if not re.search(LANGREGEX, jaString): + continue + + if jaString.replace("\u3000", "").strip(): + # Pass 1 (deduplicated) + if setData == False: + if jaString.strip() not in sceneMenuStatLabel: + sceneMenuStatLabel.append(jaString.strip()) + + # Pass 2 (collect mapping, skip duplicates) + else: + if sceneMenuStatLabel and originalString not in _pending_statlabel: + # Grab and Pop + translatedText = sceneMenuStatLabel[0] + sceneMenuStatLabel.pop(0) + + # Queue for deferred replacement + _pending_statlabel[originalString] = translatedText + + # Apply deferred replacements longest-first to avoid substring collisions + if setData and _pending_statlabel: + for orig in sorted(_pending_statlabel, key=len, reverse=True): + data[i] = data[i].replace(orig, _pending_statlabel[orig]) + _pending_statlabel.clear() + saveCheckLines(data, filename) + + # DrawTextEx - Game variable counter word (hardcoded: 回 → time(s)) + if setData and 'drawTextEx' in data[i] and '$gameVariables' in data[i]: + data[i] = re.sub( + r'(\$\{\$gameVariables\.value\(\d+\)\}\s*)回', + r'\1x', + data[i] + ) + saveCheckLines(data, filename) + + # Next Line + i += 1 + + # EOF + translate = False + questListTL = [[], [], [], [], [], []] + customTL = [] + + # Compute combined total for progress bar across all categories + combinedTotal = ( + sum(len(quest) for quest in questList) + + len(custom) + + len(sceneMenuText) + + len(sceneMenuCommonHelpText) + + len(sceneMenuHelpText) + + len(sceneMenuDrawText) + + len(sceneMenuStatLabel) + + len(saveParamName) + ) + if combinedTotal > 0: + pbar.total = combinedTotal + pbar.refresh() + + # Quest + if len(questList) > 0: + # Quest Name + response = translateAI(questList[0], "Quest Name") + tokens[0] += response[1][0] + tokens[1] += response[1][1] + questName = response[0] + + # Quest Client + response = translateAI(questList[1], "Quest Client") + tokens[0] += response[1][0] + tokens[1] += response[1][1] + questClient = response[0] + + # Quest Location + response = translateAI(questList[2], "Quest Location") + tokens[0] += response[1][0] + tokens[1] += response[1][1] + questLocation = response[0] + + # Quest Target + response = translateAI(questList[3], "Quest Location") + tokens[0] += response[1][0] + tokens[1] += response[1][1] + questTarget = response[0] + + # Quest Summary + response = translateAI(questList[4], "Quest Summary") + tokens[0] += response[1][0] + tokens[1] += response[1][1] + questSummary = response[0] + + # Quest Goal 1 + response = translateAI(questList[5], "Quest Goal") + tokens[0] += response[1][0] + tokens[1] += response[1][1] + questGoal1 = response[0] + + # Check Mismatch + if ( + len(questName) == len(questList[0]) + or len(questClient) == len(questList[1]) + or len(questLocation) == len(questList[2]) + or len(questTarget) == len(questList[3]) + or len(questSummary) == len(questList[4]) + or len(questGoal1) == len(questList[5]) + ): + # Set Strings + questListTL = [questName, questClient, questLocation, questTarget, questSummary, questGoal1] + translate = True + + # Mismatch + else: + with LOCK: + if filename not in MISMATCH: + MISMATCH.append(filename) + + # Custom + if custom: + # TL + response = translateAI(custom, "Relic Name") + tokens[0] += response[1][0] + tokens[1] += response[1][1] + customResponse = response[0] + + # Check Mismatch + if len(custom) == len(customResponse): + customTL = customResponse + translate = True + + # Mismatch + else: + with LOCK: + if filename not in MISMATCH: + MISMATCH.append(filename) + + # SceneCustomMenu Text + sceneMenuTextTL = [] + sceneMenuCommonHelpTextTL = [] + sceneMenuHelpTextTL = [] + sceneMenuDrawTextTL = [] + sceneMenuStatLabelTL = [] + saveParamNameTL = [] + + if sceneMenuText: + # TL + response = translateAI(sceneMenuText, "Menu Item") + tokens[0] += response[1][0] + tokens[1] += response[1][1] + sceneMenuTextResponse = response[0] + + # Check Mismatch + if len(sceneMenuText) == len(sceneMenuTextResponse): + sceneMenuTextTL = sceneMenuTextResponse + translate = True + + # Mismatch + else: + with LOCK: + if filename not in MISMATCH: + MISMATCH.append(filename) + + # SceneCustomMenu CommonHelpText + if sceneMenuCommonHelpText: + # TL + response = translateAI(sceneMenuCommonHelpText, "Menu Help Text") + tokens[0] += response[1][0] + tokens[1] += response[1][1] + sceneMenuCommonHelpTextResponse = response[0] + + # Check Mismatch + if len(sceneMenuCommonHelpText) == len(sceneMenuCommonHelpTextResponse): + sceneMenuCommonHelpTextTL = sceneMenuCommonHelpTextResponse + translate = True + + # Mismatch + else: + with LOCK: + if filename not in MISMATCH: + MISMATCH.append(filename) + + # SceneCustomMenu HelpText + if sceneMenuHelpText: + # TL + response = translateAI(sceneMenuHelpText, "Menu Help Text") + tokens[0] += response[1][0] + tokens[1] += response[1][1] + sceneMenuHelpTextResponse = response[0] + + # Check Mismatch + if len(sceneMenuHelpText) == len(sceneMenuHelpTextResponse): + sceneMenuHelpTextTL = sceneMenuHelpTextResponse + translate = True + + # Mismatch + else: + with LOCK: + if filename not in MISMATCH: + MISMATCH.append(filename) + + # DrawTextEx - Brace-delimited labels (tracker section headers and field labels) + if sceneMenuDrawText: + # TL + response = translateAI(sceneMenuDrawText, "Status Tracker Label") + tokens[0] += response[1][0] + tokens[1] += response[1][1] + sceneMenuDrawTextResponse = response[0] + + # Check Mismatch + if len(sceneMenuDrawText) == len(sceneMenuDrawTextResponse): + sceneMenuDrawTextTL = sceneMenuDrawTextResponse + translate = True + + # Mismatch + else: + with LOCK: + if filename not in MISMATCH: + MISMATCH.append(filename) + + # DrawTextEx - Stat labels (e.g., 攻撃力, 魔力, etc.) + if sceneMenuStatLabel: + # TL + response = translateAI(sceneMenuStatLabel, "Character Stat Name") + tokens[0] += response[1][0] + tokens[1] += response[1][1] + sceneMenuStatLabelResponse = response[0] + + # Check Mismatch + if len(sceneMenuStatLabel) == len(sceneMenuStatLabelResponse): + sceneMenuStatLabelTL = sceneMenuStatLabelResponse + translate = True + + # Mismatch + else: + with LOCK: + if filename not in MISMATCH: + MISMATCH.append(filename) + + # NUUN_SaveScreen ParamName + if saveParamName: + # TL + response = translateAI(saveParamName, "Save Screen Label") + tokens[0] += response[1][0] + tokens[1] += response[1][1] + saveParamNameResponse = response[0] + + # Check Mismatch + if len(saveParamName) == len(saveParamNameResponse): + saveParamNameTL = saveParamNameResponse + translate = True + + # Mismatch + else: + with LOCK: + if filename not in MISMATCH: + MISMATCH.append(filename) + + # Pass 2 + if translate and not setData: + translatePlugin(data, pbar, filename, [questListTL, customTL, sceneMenuTextTL, sceneMenuCommonHelpTextTL, sceneMenuHelpTextTL, sceneMenuDrawTextTL, sceneMenuStatLabelTL, saveParamNameTL]) + return tokens + +# Save some money and enter the character before translation +def getSpeaker(speaker): + match speaker: + case "ファイン": + return ["Fine", [0, 0]] + case "": + return ["", [0, 0]] + case _: + # Find Speaker + for i in range(len(NAMESLIST)): + if speaker == NAMESLIST[i][0]: + return [NAMESLIST[i][1], [0, 0]] + + # Translate and Store Speaker + response = translateAI( + f"{speaker}", + "Reply with the " + LANGUAGE + " translation of the NPC name.", + False, + ) + response[0] = response[0].title() + response[0] = response[0].replace("'S", "'s") + response[0] = response[0].replace("Speaker: ", "") + + # Retry if name doesn't translate for some reason + if re.search(r"([a-zA-Z？?])", response[0]) == None: + response = translateAI( + f"{speaker}", + "Reply with the " + LANGUAGE + " translation of the NPC name.", + False, + ) + response[0] = response[0].title() + response[0] = response[0].replace("'S", "'s") + + speakerList = [speaker, response[0]] + NAMESLIST.append(speakerList) + return response + return [speaker, [0, 0]] + +def translateAI(text, history, history_ctx=None): + """ + Legacy wrapper function for the new shared translation utility. + This maintains compatibility with existing code while using the new shared implementation. + """ + global PBAR, MISMATCH, FILENAME + + # Update config estimate mode based on global ESTIMATE + TRANSLATION_CONFIG.estimateMode = bool(ESTIMATE) + + # Call the new shared translation function + return sharedtranslateAI( + text=text, + history=history, + config=TRANSLATION_CONFIG, + filename=FILENAME, + pbar=PBAR, + lock=LOCK, + mismatchList=MISMATCH + ) diff --git a/modules/srpg.py b/modules/srpg.py index 9568574..f89e39f 100644 --- a/modules/srpg.py +++ b/modules/srpg.py @@ -1,2261 +1,2263 @@ -# Libraries -import json -import os -import re -import shutil -import util.dazedwrap as dazedwrap -import threading -import time -import traceback -from dotenv import load_dotenv -from pathlib import Path -from colorama import Fore -from dotenv import load_dotenv -from retry import retry -from tqdm import tqdm -from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost - -# Globals -MODEL = os.getenv("model") -TIMEOUT = int(os.getenv("timeout")) -LANGUAGE = os.getenv("language").capitalize() -PROMPT = Path("prompt.txt").read_text(encoding="utf-8") -VOCAB = Path("vocab.txt").read_text(encoding="utf-8") -LOCK = threading.Lock() -VOCAB_LOCK = threading.Lock() # Dedicated lock for vocab.txt updates -WIDTH = int(os.getenv("width")) -LISTWIDTH = int(os.getenv("listWidth")) -NOTEWIDTH = int(os.getenv("noteWidth")) -MAXHISTORY = 10 -ESTIMATE = "" -TOKENS = [0, 0] -MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong) -NAMESLIST = [] # List of speaker names and their translations -PBAR = None -FILENAME = None -TIMETOTAL = 0 # Total Time Taken for all translations - -# Regex - Need to change this if you want to translate from/to other languages. Default is Japanese Regex -LANGREGEX = r"[\u3000-\u303F\u3040-\u309F\u30A0-\u30FF\u31F0-\u31FF\u3400-\u4DBF\u4E00-\u9FFF\uF900-\uFAFF\uFF61-\uFF9F]+" - -# Get pricing configuration based on the model -PRICING_CONFIG = getPricingConfig(MODEL) -INPUTAPICOST = PRICING_CONFIG["inputAPICost"] -OUTPUTAPICOST = PRICING_CONFIG["outputAPICost"] -BATCHSIZE = PRICING_CONFIG["batchSize"] -FREQUENCY_PENALTY = PRICING_CONFIG["frequencyPenalty"] - -# tqdm Globals -BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}" -POSITION = 0 -LEAVE = False - -# Initialize Translation Config -TRANSLATION_CONFIG = TranslationConfig( - model=MODEL, - language=LANGUAGE, - prompt=PROMPT, - vocab=VOCAB, - langRegex=LANGREGEX, - batchSize=BATCHSIZE, - maxHistory=MAXHISTORY, - estimateMode=False # Will be set dynamically based on ESTIMATE -) - -# Config (Default) -FIXTEXTWRAP = True # Rewrap text to WIDTH -IGNORETLTEXT = False # Skip Translated Text - -# List of file patterns that use parseGeneric -# Add more patterns here as needed -GENERIC_FILES = [ - "quests", - "shops", - "shoplayout", - "bonuses", - "base", - "battleprep", - "manage", - "mapcommands", - "title", - "classes", - "classesgroups", - "classtypes", - "races", - "skills", - "weapons", - "states", - "difficulties", - "fonts", - "fusionsettings", - "transformations", - "characters", - "glossary", - "npc", - "screens", - "strings", - "originalterrains", - "runtimeterrains", - "archers", - "fighters", - "mages", - "items", -] - -# List of file patterns that use parseMap -# Be specific to avoid catching non-map files like CommandLayout/mapcommands.json -# SRPG Studio map files in this project follow the pattern Maps/map_XXX.json -MAP_FILES = [ - "map_", # e.g., Maps/map_000.json -] - - -def update_vocab_section(category: str, pairs: list[tuple[str, str]]): - """Update or insert a section in vocab.txt for the given category with provided pairs. - Only writes when there's an actual translation (dst is non-empty and differs from src after normalization). - - category: e.g., "Items", "Weapons", "Speakers", etc. Section header will be "# {category}". - - pairs: list of (source, translated) strings. Duplicates by source are deduped (last wins). - The existing section is replaced entirely; other sections are preserved. - """ - try: - vocab_path = Path("vocab.txt") - - # Helper: normalized comparison to detect no-op translations - def _norm(s: str) -> str: - if s is None: - return "" - # Collapse whitespace and case-fold; leave punctuation to avoid over-matching - return re.sub(r"\s+", " ", str(s)).strip().casefold() - - # Filter and deduplicate by source term (last mapping wins) - dedup: dict[str, str] = {} - for src, dst in pairs: - if not src: - continue - # Skip when no destination or no actual change - if dst is None or _norm(dst) == "" or _norm(dst) == _norm(src): - continue - dedup[src] = dst - - # If nothing to add after filtering, skip touching the file - if not dedup: - return - - # Guard the read-modify-write with a dedicated lock to avoid races - with VOCAB_LOCK: - existing = vocab_path.read_text(encoding="utf-8") if vocab_path.exists() else "" - - lines = [f"{src} ({dst})" for src, dst in dedup.items()] - # Always terminate a section with a blank line to separate from next header - new_block = f"# {category}\n" + "\n".join(lines) - if not new_block.endswith("\n\n"): - if not new_block.endswith("\n"): - new_block += "\n" - new_block += "\n" - - # Regex to find the specific section starting at the header for this category - # and ending right before the next header (any number of '#') or EOF. - # - Handles headers like '#Category', '# Category', '## Category', etc. - # - Uses non-greedy matching for the body to avoid spanning multiple sections. - pattern = re.compile( - rf"^[\t ]*#+\s*{re.escape(category)}\s*$\r?\n.*?(?=^[\t ]*#|\Z)", - re.MULTILINE | re.DOTALL, - ) - if pattern.search(existing): - # Replace only the first matching section for this category. - updated = pattern.sub(lambda m: new_block, existing, count=1) - else: - updated = existing - if updated and not updated.endswith("\n\n"): - # Ensure a blank line before appending new section if file not empty - if not updated.endswith("\n"): - updated += "\n" - updated += "\n" - updated += new_block - - # Avoid writing if nothing changed - if updated == existing: - return - # Atomic write: write to unique temp and replace with retries on Windows - tmp_path = vocab_path.with_suffix(vocab_path.suffix + f".{os.getpid()}.{threading.get_ident()}.tmp") - tmp_path.write_text(updated, encoding="utf-8") - - attempts = 6 - delay = 0.1 - last_err = None - for attempt in range(attempts): - try: - os.replace(tmp_path, vocab_path) - last_err = None - break - except PermissionError as e: - last_err = e - # Try relaxing permissions then retry - try: - if vocab_path.exists(): - os.chmod(vocab_path, 0o666) - except Exception: - pass - time.sleep(delay) - delay = min(1.0, delay * 2) - except Exception as e: - last_err = e - break - if last_err is not None: - try: - shutil.move(str(tmp_path), str(vocab_path)) - except Exception: - try: - if tmp_path.exists(): - tmp_path.unlink(missing_ok=True) - except Exception: - pass - raise last_err - except Exception: - traceback.print_exc() - - -def handleSRPG(filename, estimate): - """ - Main handler function for SRPG Studio files. - - Args: - filename: Name of the file to translate - estimate: Boolean indicating if this is an estimate run - - Returns: - String with translation results or error message - """ - global ESTIMATE, TOKENS, FILENAME, TIMETOTAL - ESTIMATE = estimate - FILENAME = filename - - # Translate - start = time.time() - translatedData = openFiles(filename) - - # Write output file if not in estimate mode - if not estimate: - try: - with open("translated/" + filename, "w", encoding="utf-8", newline="\n") as outFile: - json.dump(translatedData[0], outFile, ensure_ascii=False, indent=4) - except Exception: - traceback.print_exc() - return "Fail" - - # Print File - end = time.time() - tqdm.write(getResultString(translatedData, end - start, filename)) - with LOCK: - TOKENS[0] += translatedData[1][0] - TOKENS[1] += translatedData[1][1] - - # Print Total - totalString = getResultString(["", TOKENS, None], end - start, "TOTAL") - - # Print any errors - if len(MISMATCH) > 0: - return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET - else: - return totalString - - -def openFiles(filename): - """ - Opens and routes SRPG Studio files to appropriate parsing functions. - - Args: - filename: Name of the file to open and parse - - Returns: - Tuple of (translated data, token counts, error) - """ - with open("files/" + filename, "r", encoding="utf-8-sig") as f: - data = json.load(f) - - # Check if filename matches recollection pattern - if "recollection" in filename.lower(): - translatedData = parseRecollection(data, filename) - # mapcommonevents is a list of events like recollection - elif "mapcommonevents" in filename.lower(): - translatedData = parseRecollection(data, filename) - # Bookmark events use the same structure as recollection (list of events with pages/commands) - elif "bookmarkevents" in filename.lower(): - translatedData = parseRecollection(data, filename) - # Event files (autoevents, placeevents, talkevents, communicationevents, openingevents, endingevents) - # all have name, desc, and pages->commands structure like recollection - elif any(event_type in filename.lower() for event_type in ["autoevents", "placeevents", "talkevents", "communicationevents", "openingevents", "endingevents"]): - translatedData = parseRecollection(data, filename) - # Check if filename matches bookmark pattern (top-level entries with name/desc + events) - elif "bookmark" in filename.lower(): - translatedData = parseBookmark(data, filename) - # Players have the same shape as bookmark entries - elif "players" in filename.lower(): - translatedData = parseBookmark(data, filename) - # Titles.json is a small dict of title strings - elif os.path.basename(filename).lower() == "titles.json": - translatedData = parseTitles(data, filename) - # Check if filename matches map pattern - elif any(pattern in filename.lower() for pattern in MAP_FILES): - translatedData = parseMap(data, filename) - # Check if filename matches any pattern in GENERIC_FILES - elif any(pattern in filename.lower() for pattern in GENERIC_FILES): - translatedData = parseGeneric(data, filename) - - # TODO: Add other SRPG Studio file types here - else: - raise NameError(filename + " Not Supported") - - return translatedData - - -def parseBookmark(data, filename): - """ - Parser for SRPG Studio bookmark.json files. - Structure: List of entries, each with id, name, desc, and events containing pages -> commands - where commands have data (array of strings) and optional speaker. - - Args: - data: Parsed JSON data (list of bookmark entries) - filename: Name of the file being parsed - - Returns: - Tuple of (translated data, token counts, error) - """ - global PBAR - totalTokens = [0, 0] - - pbar = None - try: - # Count work units: names, descs, dialogue lines, and speakers - total_units = 0 - - for entry in data: - if not entry: - continue - - # name and desc - for field in ["name", "desc"]: - if field in entry and entry[field]: - total_units += 1 - - # events -> pages -> commands - if "events" in entry and isinstance(entry["events"], list): - for event in entry["events"]: - if not event: - continue - - # Count event-level name and desc fields - for field in ["name", "desc"]: - if field in event and event[field]: - total_units += 1 - - if "pages" not in event or not isinstance(event["pages"], list): - continue - - for page in event["pages"]: - if not page or "commands" not in page or not isinstance(page["commands"], list): - continue - for command in page["commands"]: - if not command: - continue - if "data" in command and isinstance(command["data"], list): - for text in command["data"]: - if text: - total_units += 1 - if "speaker" in command and command["speaker"]: - total_units += 1 - - # Setup progress bar - with LOCK: - pbar = tqdm( - desc=filename, - total=total_units, - bar_format=BAR_FORMAT, - position=POSITION, - leave=LEAVE, - ) - PBAR = pbar - - # Translate using two-pass approach - result = translateBookmark(data, filename, pbar=pbar) - totalTokens[0] += result[0] - totalTokens[1] += result[1] - - return (data, totalTokens, None) - - except Exception as e: - traceback.print_exc() - return (data, totalTokens, e) - finally: - try: - if pbar is not None: - pbar.close() - except Exception: - pass - PBAR = None - - -def translateBookmark(data, filename, translatedDataList=None, pbar=None): - """ - Translates bookmark.json data structure. - Two-pass approach via recursion: - - Pass 1: Collect strings (names, descs, dialogue data with speaker prefix, speakers) - - Pass 2: Apply translations back into data - - Returns: - [input tokens, output tokens] - """ - totalTokens = [0, 0] - - # Initialize or extract lists - if translatedDataList is None: - nameList = [] - descList = [] - dataList = [] - speakerList = [] - else: - nameList = translatedDataList[0] - descList = translatedDataList[1] - dataList = translatedDataList[2] - speakerList = translatedDataList[3] - - for entry in data: - if not entry: - continue - - # name - if "name" in entry and entry["name"]: - if translatedDataList is None: - nameList.append(entry["name"]) - else: - if nameList: - entry["name"] = nameList[0] - nameList.pop(0) - - # desc - if "desc" in entry and entry["desc"]: - if translatedDataList is None: - # Remove newlines for translation - descList.append(entry["desc"].replace("\n", " ")) - else: - if descList: - translatedText = descList[0] - # Apply text wrapping - translatedText = dazedwrap.wrapText(translatedText, width=WIDTH) - entry["desc"] = translatedText - descList.pop(0) - - # events -> pages -> commands - if "events" in entry and isinstance(entry["events"], list): - for event in entry["events"]: - if not event: - continue - - # Handle event-level name field - if "name" in event and event["name"]: - if translatedDataList is None: - nameList.append(event["name"]) - else: - if nameList: - event["name"] = nameList[0] - nameList.pop(0) - - # Handle event-level desc field - if "desc" in event and event["desc"]: - if translatedDataList is None: - # Remove newlines for translation - descList.append(event["desc"].replace("\n", " ")) - else: - if descList: - translatedText = descList[0] - # Apply text wrapping - translatedText = dazedwrap.wrapText(translatedText, width=WIDTH) - event["desc"] = translatedText - descList.pop(0) - - if "pages" not in event or not isinstance(event["pages"], list): - continue - - for page in event["pages"]: - if not page or "commands" not in page or not isinstance(page["commands"], list): - continue - for command in page["commands"]: - if not command: - continue - - speaker = command.get("speaker", "") - - # data array - if "data" in command and isinstance(command["data"], list): - for i, text in enumerate(command["data"]): - if text: - if translatedDataList is None: - text = text.replace("\n", " ") - if speaker: - dataList.append(f"[{speaker}]: {text}") - else: - dataList.append(text) - else: - if dataList: - translated = dataList[0] - if speaker: - match = re.search(r'(^\[.+?\]\s?[|:]\s?)', translated) - if match: - translated = translated.replace(match.group(1), "") - translated = dazedwrap.wrapText(translated, width=WIDTH) - command["data"][i] = translated - dataList.pop(0) - - # speaker field - if "speaker" in command and command["speaker"]: - if translatedDataList is None: - speakerList.append(command["speaker"]) - else: - if speakerList: - command["speaker"] = speakerList[0] - speakerList.pop(0) - - # If this was Pass 1, perform translations and recurse - if translatedDataList is None: - originalNameCount = len(nameList) - originalDescCount = len(descList) - originalDataCount = len(dataList) - originalSpeakerCount = len(speakerList) - - if nameList: - response = translateAI( - nameList, - "Reply with only the " + LANGUAGE + " translation of the bookmark name.", - True, - filename, - pbar, - ) - nameList = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - - if descList: - response = translateAI( - descList, - "Reply with only the " + LANGUAGE + " translation of the bookmark description.", - True, - filename, - pbar, - ) - descList = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - - if dataList: - response = translateAI( - dataList, - "Reply with only the " + LANGUAGE + " translation of the dialogue text.", - True, - filename, - pbar, - ) - dataList = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - - if speakerList: - response = translateAI( - speakerList, - "Reply with only the " + LANGUAGE + " translation of the speaker name.", - True, - filename, - pbar, - ) - speakerList = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - - # Mismatch checks - if len(nameList) != originalNameCount: - with LOCK: - if filename not in MISMATCH: - MISMATCH.append(filename) - if len(descList) != originalDescCount: - with LOCK: - if filename not in MISMATCH: - MISMATCH.append(filename) - if len(dataList) != originalDataCount: - with LOCK: - if filename not in MISMATCH: - MISMATCH.append(filename) - if len(speakerList) != originalSpeakerCount: - with LOCK: - if filename not in MISMATCH: - MISMATCH.append(filename) - - # PASS 2 - translateBookmark(data, filename, [nameList, descList, dataList, speakerList], pbar) - - return totalTokens - - -def parseGeneric(data, filename): - """ - Generic parser for SRPG Studio files with id, name, desc structure. - Handles files like quests.json. - Uses a two-pass approach: first pass collects all strings to translate, - then batch translates them, then second pass applies translations. - - Args: - data: Parsed JSON data (list of objects with id, name, desc) - filename: Name of the file being parsed - - Returns: - Tuple of (data, token counts, error) - """ - global PBAR - - totalTokens = [0, 0] - - pbar = None - try: - # Count work units (all translatable fields that need translation) - total_units = 0 - translatable_fields = ["name", "desc", "commandName", "command"] - - for entry in data: - if entry: - for field in translatable_fields: - if field in entry and entry[field]: - # If command is a list (e.g., fusionsettings), count per element - if field == "command" and isinstance(entry[field], list): - total_units += sum(1 for x in entry[field] if x) - # Otherwise simple increment - elif not isinstance(entry[field], list): - total_units += 1 - - # Handle pages array separately - if "pages" in entry and entry["pages"] and isinstance(entry["pages"], list): - for page in entry["pages"]: - if page: - total_units += 1 - - # Handle msg arrays (e.g., shoplayout) - if "msg" in entry and isinstance(entry["msg"], list): - total_units += sum(1 for m in entry["msg"] if m) - - # Handle rewardData arrays (e.g., quests) - if "rewardData" in entry and isinstance(entry["rewardData"], list): - total_units += sum(1 for r in entry["rewardData"] if r) - - # Handle customParameters name field (e.g., {name:'シャルロット強制売春'}) - if "customParameters" in entry and entry["customParameters"]: - match = re.search(r"name:\s*['\"]([^'\"]+)['\"]", entry["customParameters"]) - if match: - total_units += 1 - - # Handle terrains array (nested structure) - if "terrains" in entry and entry["terrains"] and isinstance(entry["terrains"], list): - for terrain in entry["terrains"]: - if terrain: - for field in ["name", "desc"]: - if field in terrain and terrain[field]: - total_units += 1 - - # Setup progress bar (use a per-file instance to avoid cross-thread clashes) - with LOCK: - pbar = tqdm( - desc=filename, - total=total_units, - bar_format=BAR_FORMAT, - position=POSITION, - leave=LEAVE, - ) - PBAR = pbar - - # Translate the data using two-pass approach - result = translateGeneric(data, filename, pbar=pbar) - totalTokens[0] += result[0] - totalTokens[1] += result[1] - - return (data, totalTokens, None) - - except Exception as e: - traceback.print_exc() - return (data, totalTokens, e) - finally: - # Ensure progress bar is closed - try: - if pbar is not None: - pbar.close() - except Exception: - pass - PBAR = None - - -def translateGeneric(data, filename, translatedDataList=None, pbar=None): - """ - Translates generic SRPG Studio data with id, name, desc, commandName, command, pages structure. - Uses two-pass approach via recursion: - - Pass 1 (translatedDataList=None): Collect strings and batch translate - - Pass 2 (translatedDataList set): Apply translations back to the data - - Args: - data: List of objects with id, name, desc, commandName, command, pages keys - filename: Name of the file being translated - translatedDataList: List containing [nameList, descList, commandNameList, commandList, pagesList] - - Pass 1: Empty lists to collect originals - - Pass 2: Filled lists with translations - - Returns: - Tuple of [input tokens, output tokens] - """ - global PBAR - - totalTokens = [0, 0] - - # Initialize or extract lists - if translatedDataList is None: - # PASS 1: Create empty lists to collect strings - nameList = [] - descList = [] - commandNameList = [] - commandList = [] - pagesList = [] - msgList = [] - commandArrayList = [] - rewardDataList = [] - customParametersNameList = [] - else: - # PASS 2: Use provided translated lists - nameList = translatedDataList[0] - descList = translatedDataList[1] - commandNameList = translatedDataList[2] - commandList = translatedDataList[3] - pagesList = translatedDataList[4] - msgList = translatedDataList[5] - commandArrayList = translatedDataList[6] - rewardDataList = translatedDataList[7] - customParametersNameList = translatedDataList[8] - - # Single loop - behavior depends on which pass we're in - for entry in data: - if not entry: - continue - - # Handle name field - if "name" in entry and entry["name"]: - # PASS 1: Collect original - if translatedDataList is None: - nameList.append(entry["name"]) - # PASS 2: Apply translation - else: - if nameList: - entry["name"] = nameList[0] - nameList.pop(0) - - # Handle desc field - if "desc" in entry and entry["desc"]: - # PASS 1: Collect original - if translatedDataList is None: - # Nuke Wordwrap - descList.append(entry["desc"].replace("\n", " ")) - # PASS 2: Apply translation - else: - if descList: - translatedText = descList[0] - # Wordwrap - translatedText = dazedwrap.wrapText(translatedText, width=WIDTH) - - # Set Data - entry["desc"] = translatedText - descList.pop(0) - - # Handle commandName field - if "commandName" in entry and entry["commandName"]: - # PASS 1: Collect original - if translatedDataList is None: - commandNameList.append(entry["commandName"]) - # PASS 2: Apply translation - else: - if commandNameList: - entry["commandName"] = commandNameList[0] - commandNameList.pop(0) - - # Handle command field (string or list) - if "command" in entry and entry["command"] is not None: - # If list of strings - if isinstance(entry["command"], list): - for i, val in enumerate(entry["command"]): - if val: - if translatedDataList is None: - # Nuke Wrap - commandArrayList.append(val.replace("\n", " ")) - else: - if commandArrayList: - translatedText = dazedwrap.wrapText(commandArrayList[0], width=WIDTH) - entry["command"][i] = translatedText - commandArrayList.pop(0) - # If simple string - elif isinstance(entry["command"], str): - if translatedDataList is None: - commandList.append(entry["command"]) - else: - if commandList: - entry["command"] = commandList[0] - commandList.pop(0) - - # Handle pages field (array of strings) - if "pages" in entry and entry["pages"] and isinstance(entry["pages"], list): - for i, page in enumerate(entry["pages"]): - if page: - # PASS 1: Collect original - if translatedDataList is None: - # Nuke Wordwrap - page = page.replace("\n", " ") - pagesList.append(page) - # PASS 2: Apply translation - else: - if pagesList: - translatedText = pagesList[0] - - # Wordwrap - translatedText = dazedwrap.wrapText(translatedText, width=WIDTH) - - # Set Data - entry["pages"][i] = translatedText - pagesList.pop(0) - - # Handle msg field (array of strings) - if "msg" in entry and isinstance(entry["msg"], list): - for i, val in enumerate(entry["msg"]): - if val is not None: - if translatedDataList is None: - msgList.append(str(val).replace("\n", " ")) - else: - if msgList: - translatedText = dazedwrap.wrapText(msgList[0], width=WIDTH) - entry["msg"][i] = translatedText - msgList.pop(0) - - # Handle rewardData field (array of strings) - if "rewardData" in entry and isinstance(entry["rewardData"], list): - for i, val in enumerate(entry["rewardData"]): - if val: - # PASS 1: Collect original - if translatedDataList is None: - rewardDataList.append(str(val).replace("\n", " ")) - # PASS 2: Apply translation - else: - if rewardDataList: - translatedText = rewardDataList[0] - # Set Data (no wordwrap for reward data) - entry["rewardData"][i] = translatedText - rewardDataList.pop(0) - - # Handle customParameters name field (e.g., {name:'シャルロット強制売春'}) - if "customParameters" in entry and entry["customParameters"]: - match = re.search(r"name:\s*['\"]([^'\"]+)['\"]", entry["customParameters"]) - if match: - # PASS 1: Collect original - if translatedDataList is None: - customParametersNameList.append(match.group(1)) - # PASS 2: Apply translation - else: - if customParametersNameList: - translatedName = customParametersNameList[0] - # Replace the name value in customParameters - entry["customParameters"] = re.sub( - r"(name:\s*['\"])([^'\"]+)(['\"])", - r"\1" + translatedName.replace("\\", "\\\\") + r"\3", - entry["customParameters"] - ) - customParametersNameList.pop(0) - - # Handle terrains field (nested array with name and desc) - if "terrains" in entry and entry["terrains"] and isinstance(entry["terrains"], list): - for terrain in entry["terrains"]: - if not terrain: - continue - - # Handle terrain name - if "name" in terrain and terrain["name"]: - # PASS 1: Collect original - if translatedDataList is None: - nameList.append(terrain["name"]) - # PASS 2: Apply translation - else: - if nameList: - terrain["name"] = nameList[0] - nameList.pop(0) - - # Handle terrain desc - if "desc" in terrain and terrain["desc"]: - # PASS 1: Collect original - if translatedDataList is None: - descList.append(terrain["desc"]).replace("\n", " ") - # PASS 2: Apply translation - else: - if descList: - translatedText = descList[0] - # Wordwrap - translatedText = dazedwrap.wrapText(translatedText, width=WIDTH) - - # Set Data - terrain["desc"] = translatedText - descList.pop(0) - - # If this was Pass 1, do the translation and recurse for Pass 2 - if translatedDataList is None: - # Store original counts for mismatch checking - originalNameCount = len(nameList) - originalDescCount = len(descList) - originalCommandNameCount = len(commandNameList) - originalCommandCount = len(commandList) - originalPagesCount = len(pagesList) - originalMsgCount = len(msgList) - originalCommandArrayCount = len(commandArrayList) - originalRewardDataCount = len(rewardDataList) - originalCustomParametersNameCount = len(customParametersNameList) - - # Keep a copy of original names for vocab update (for characters/items/skills/classes/weapons) - vocab_name_files = ["characters", "items", "skills", "classes", "weapons"] - originalNameList = nameList.copy() if any(tag in filename.lower() for tag in vocab_name_files) else [] - - # Batch translate names - if nameList: - response = translateAI( - nameList, - "Reply with only the " + LANGUAGE + " translation of the quest name.", - True, - filename, - pbar - ) - nameList = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - - # Update vocab.txt for name-bearing files - if originalNameList and nameList: - try: - file_lower = filename.lower() - section = None - if "characters" in file_lower: - section = "Speakers" - elif "items" in file_lower: - section = "Items" - elif "skills" in file_lower: - section = "Skills" - elif "classes" in file_lower: - section = "Classes" - elif "weapons" in file_lower: - section = "Weapons" - - if section: - vocab_pairs = list(zip(originalNameList, nameList)) - update_vocab_section(section, vocab_pairs) - except Exception: - traceback.print_exc() - - # Batch translate descriptions - if descList: - response = translateAI( - descList, - "Reply with only the " + LANGUAGE + " translation of the quest description.", - True, - filename, - pbar - ) - descList = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - - # Batch translate command names - if commandNameList: - response = translateAI( - commandNameList, - "Reply with only the " + LANGUAGE + " translation of the command name.", - True, - filename, - pbar - ) - commandNameList = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - - # Batch translate commands - if commandList: - response = translateAI( - commandList, - "Reply with only the " + LANGUAGE + " translation of the command.", - True, - filename, - pbar - ) - commandList = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - - # Batch translate pages - if pagesList: - response = translateAI( - pagesList, - "Reply with only the " + LANGUAGE + " translation of the page content.", - True, - filename, - pbar - ) - pagesList = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - - # Batch translate msg arrays - if msgList: - response = translateAI( - msgList, - "Reply with only the " + LANGUAGE + " translation of the message text.", - True, - filename, - pbar - ) - msgList = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - - # Batch translate command arrays - if commandArrayList: - response = translateAI( - commandArrayList, - "Reply with only the " + LANGUAGE + " translation of the command text.", - True, - filename, - pbar - ) - commandArrayList = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - - # Batch translate rewardData arrays - if rewardDataList: - response = translateAI( - rewardDataList, - "Reply with only the " + LANGUAGE + " translation of the reward data text.", - True, - filename, - pbar - ) - rewardDataList = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - - # Batch translate customParameters name fields - if customParametersNameList: - response = translateAI( - customParametersNameList, - "Reply with only the " + LANGUAGE + " translation of the custom parameter name.", - True, - filename, - pbar - ) - customParametersNameList = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - - # Check for mismatch errors - if len(nameList) != originalNameCount: - with LOCK: - if filename not in MISMATCH: - MISMATCH.append(filename) - - if len(descList) != originalDescCount: - with LOCK: - if filename not in MISMATCH: - MISMATCH.append(filename) - - if len(commandNameList) != originalCommandNameCount: - with LOCK: - if filename not in MISMATCH: - MISMATCH.append(filename) - - if len(commandList) != originalCommandCount: - with LOCK: - if filename not in MISMATCH: - MISMATCH.append(filename) - - if len(pagesList) != originalPagesCount: - with LOCK: - if filename not in MISMATCH: - MISMATCH.append(filename) - if len(msgList) != originalMsgCount: - with LOCK: - if filename not in MISMATCH: - MISMATCH.append(filename) - if len(commandArrayList) != originalCommandArrayCount: - with LOCK: - if filename not in MISMATCH: - MISMATCH.append(filename) - if len(rewardDataList) != originalRewardDataCount: - with LOCK: - if filename not in MISMATCH: - MISMATCH.append(filename) - if len(customParametersNameList) != originalCustomParametersNameCount: - with LOCK: - if filename not in MISMATCH: - MISMATCH.append(filename) - - # PASS 2: Recursively call to apply translations - translateGeneric( - data, - filename, - [nameList, descList, commandNameList, commandList, pagesList, msgList, commandArrayList, rewardDataList, customParametersNameList], - pbar, - ) - - return totalTokens - - -def parseTitles(data, filename): - """Parser for titles.json (small dict of title strings).""" - global PBAR - totalTokens = [0, 0] - pbar = None - try: - # Collect fields to translate - keys = [k for k in ["windowTitle", "gameTitle", "saveFileTitle"] if k in data and data[k]] - values = [data[k] for k in keys] - - with LOCK: - pbar = tqdm( - desc=filename, - total=len(values), - bar_format=BAR_FORMAT, - position=POSITION, - leave=LEAVE, - ) - PBAR = pbar - - if values: - response = translateAI( - values, - "Reply with only the " + LANGUAGE + " translation of the title text.", - True, - filename, - pbar, - ) - translations = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - - # Apply - for i, k in enumerate(keys): - data[k] = translations[i] - - return (data, totalTokens, None) - except Exception as e: - traceback.print_exc() - return (data, totalTokens, e) - finally: - try: - if pbar is not None: - pbar.close() - except Exception: - pass - PBAR = None - - -def parseRecollection(data, filename): - """ - Parser for SRPG Studio recollection.json files. - Structure: List of entries, each with pages containing commands with data arrays and speakers. - - Args: - data: Parsed JSON data (list of objects with id, pages structure) - filename: Name of the file being parsed - - Returns: - Tuple of (data, token counts, error) - """ - global PBAR - - totalTokens = [0, 0] - - pbar = None - try: - # Count work units (data entries and speakers that need translation) - total_units = 0 - - # Iterate through structure and count - for entry in data: - if not entry: - continue - - # Count name and desc fields at entry level - for field in ["name", "desc"]: - if field in entry and entry[field]: - total_units += 1 - - # Count customParameters hints - if "customParameters" in entry and entry["customParameters"]: - match = re.search(r'hint:"((?:[^"\\]|\\.)*)"', entry["customParameters"]) - if match: - total_units += 1 - - if "pages" not in entry or not isinstance(entry["pages"], list): - continue - - for page in entry["pages"]: - if not page or "commands" not in page or not isinstance(page["commands"], list): - continue - - for command in page["commands"]: - if not command: - continue - - # Count data array items (count any non-empty text; do not gate on LANGREGEX) - if "data" in command and isinstance(command["data"], list): - for text in command["data"]: - if text: - total_units += 1 - - # Count speaker field (count any non-empty speaker; do not gate on LANGREGEX) - if "speaker" in command and command["speaker"]: - total_units += 1 - - # Setup progress bar (per-file instance) - with LOCK: - pbar = tqdm( - desc=filename, - total=total_units, - bar_format=BAR_FORMAT, - position=POSITION, - leave=LEAVE, - ) - PBAR = pbar - - # Translate the data using two-pass approach - result = translateRecollection(data, filename, pbar=pbar) - totalTokens[0] += result[0] - totalTokens[1] += result[1] - - return (data, totalTokens, None) - - except Exception as e: - traceback.print_exc() - return (data, totalTokens, e) - finally: - try: - if pbar is not None: - pbar.close() - except Exception: - pass - PBAR = None - - -def translateRecollection(data, filename, translatedDataList=None, pbar=None): - """ - Translates recollection.json data structure. - Uses two-pass approach via recursion: - - Pass 1 (translatedDataList=None): Collect strings and batch translate - - Pass 2 (translatedDataList set): Apply translations back to the data - - Args: - data: List of objects with name, desc, customParameters, and pages->commands->data structure - filename: Name of the file being translated - translatedDataList: List containing [nameList, descList, dataList, speakerList, customParamsList] - - Pass 1: Empty lists to collect originals - - Pass 2: Filled lists with translations - - Returns: - Tuple of [input tokens, output tokens] - """ - global PBAR - - totalTokens = [0, 0] - - # Initialize or extract lists - if translatedDataList is None: - # PASS 1: Create empty lists to collect strings - nameList = [] - descList = [] - dataList = [] - speakerList = [] - customParamsList = [] - originalSpeakerList = [] # For vocab update - else: - # PASS 2: Use provided translated lists - nameList = translatedDataList[0] - descList = translatedDataList[1] - dataList = translatedDataList[2] - speakerList = translatedDataList[3] - customParamsList = translatedDataList[4] - - # Single loop - behavior depends on which pass we're in - for entry in data: - if not entry: - continue - - # Handle name field - if "name" in entry and entry["name"]: - # PASS 1: Collect original - if translatedDataList is None: - nameList.append(entry["name"]) - # PASS 2: Apply translation - else: - if nameList: - entry["name"] = nameList[0] - nameList.pop(0) - - # Handle desc field - if "desc" in entry and entry["desc"]: - # PASS 1: Collect original - if translatedDataList is None: - # Remove newlines for translation - descList.append(entry["desc"].replace("\n", " ")) - # PASS 2: Apply translation - else: - if descList: - translatedText = descList[0] - # Apply text wrapping - translatedText = dazedwrap.wrapText(translatedText, width=WIDTH) - entry["desc"] = translatedText - descList.pop(0) - - # Handle customParameters at entry level - if "customParameters" in entry and entry["customParameters"]: - # PASS 1: Collect hint text - if translatedDataList is None: - # Extract hint value using regex (captures content between hint:" and ") - match = re.search(r'hint:"((?:[^"\\]|\\.)*)"', entry["customParameters"]) - if match: - hintText = match.group(1) - hintText = hintText.replace("\\n", " ") - customParamsList.append(hintText) - else: - # No hint found, add empty string as placeholder - customParamsList.append("") - # PASS 2: Apply translation - else: - if customParamsList: - translatedHint = customParamsList[0] - customParamsList.pop(0) - - if translatedHint: # Only replace if we translated something - # Replace double quotes with single quotes since \" is used as delimiter - translatedHint = translatedHint.replace('"', "'") - # Wrap text using dazedwrap with WIDTH and replace newlines with \\n - translatedHint = dazedwrap.wrapText(translatedHint, width=WIDTH) - translatedHint = translatedHint.replace("\n", "\\\\n") - # Replace the hint value in customParameters - entry["customParameters"] = re.sub( - r'(hint:")((?:[^"\\]|\\.)*)"', - r'\1' + translatedHint + '"', - entry["customParameters"] - ) - - # Check if pages exists before processing - if "pages" not in entry or not isinstance(entry["pages"], list): - continue - - for page in entry["pages"]: - if not page or "commands" not in page: - continue - - if not isinstance(page["commands"], list): - continue - - for command in page["commands"]: - if not command: - continue - - # Get the speaker for this command (if available) - speaker = command.get("speaker", "") - - # Handle data array - if "data" in command and isinstance(command["data"], list): - for i, text in enumerate(command["data"]): - if text: - # PASS 1: Collect original with speaker prefix - if translatedDataList is None: - # Remove Wrap - text = text.replace("\n", " ") - - # Attach speaker to data for translation - if speaker: - dataList.append(f"[{speaker}]: {text}") - else: - dataList.append(text) - - # PASS 2: Apply translation and strip speaker prefix - else: - if dataList: - translated = dataList[0] - - # Remove speaker - if speaker: - match = re.search(r'(^\[.+?\]\s?[|:]\s?)', translated) - if match: - translated = translated.replace(match.group(1), "") - - # Textwrap - translated = dazedwrap.wrapText(translated, width=WIDTH) - - # Set Data - command["data"][i] = translated - dataList.pop(0) - - # Handle speaker field - if "speaker" in command and command["speaker"]: - # PASS 1: Collect original - if translatedDataList is None: - originalSpeakerList.append(command["speaker"]) - speakerList.append(command["speaker"]) - # PASS 2: Apply translation - else: - if speakerList: - command["speaker"] = speakerList[0] - speakerList.pop(0) - - # If this was Pass 1, do the translation and recurse for Pass 2 - if translatedDataList is None: - # Store original counts for mismatch checking - originalNameCount = len(nameList) - originalDescCount = len(descList) - originalDataCount = len(dataList) - originalSpeakerCount = len(speakerList) - originalCustomParamsCount = len(customParamsList) - - # Batch translate names - if nameList: - response = translateAI( - nameList, - "Reply with only the " + LANGUAGE + " translation of the name.", - True, - filename, - pbar - ) - nameList = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - - # Batch translate descriptions - if descList: - response = translateAI( - descList, - "Reply with only the " + LANGUAGE + " translation of the description.", - True, - filename, - pbar - ) - descList = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - - # Batch translate data text - if dataList: - response = translateAI( - dataList, - "Reply with only the " + LANGUAGE + " translation of the dialogue text.", - True, - filename, - pbar - ) - dataList = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - - # Batch translate speakers - if speakerList: - response = translateAI( - speakerList, - "Reply with only the " + LANGUAGE + " translation of the speaker name.", - True, - filename, - pbar - ) - speakerList = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - - # Batch translate customParameters hints (only non-empty ones) - if customParamsList: - # Filter out empty strings for translation - hintsToTranslate = [h for h in customParamsList if h] - if hintsToTranslate: - response = translateAI( - hintsToTranslate, - "Reply with only the " + LANGUAGE + " translation of the hint text.", - True, - filename, - pbar - ) - translatedHints = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - - # Reconstruct customParamsList with translations in place - translatedIndex = 0 - for i, hint in enumerate(customParamsList): - if hint: # If it was non-empty, use the translation - customParamsList[i] = translatedHints[translatedIndex] - translatedIndex += 1 - - # Check for mismatch errors - if len(nameList) != originalNameCount: - with LOCK: - if filename not in MISMATCH: - MISMATCH.append(filename) - - if len(descList) != originalDescCount: - with LOCK: - if filename not in MISMATCH: - MISMATCH.append(filename) - - if len(dataList) != originalDataCount: - with LOCK: - if filename not in MISMATCH: - MISMATCH.append(filename) - - if len(speakerList) != originalSpeakerCount: - with LOCK: - if filename not in MISMATCH: - MISMATCH.append(filename) - - if len(customParamsList) != originalCustomParamsCount: - with LOCK: - if filename not in MISMATCH: - MISMATCH.append(filename) - - # PASS 2: Recursively call to apply translations - translateRecollection(data, filename, [nameList, descList, dataList, speakerList, customParamsList], pbar) - - return totalTokens - - -def parseMap(data, filename): - """ - Parser for SRPG Studio map.json files. - Structure: Single object with: - - id, name, desc, mapName - - victoryConds (array of strings) - - defeatConds (array of strings) - - EnemyUnits (array of objects with id, name, desc, events structure similar to recollection) - - Args: - data: Parsed JSON data (single map object) - filename: Name of the file being parsed - - Returns: - Tuple of (data, token counts, error) - """ - global PBAR - - totalTokens = [0, 0] - - pbar = None - try: - # Count work units (all translatable fields) - total_units = 0 - - # Count top-level fields: desc, mapName (name is an identifier and should not be translated) - for field in ["desc", "mapName"]: - if field in data and data[field]: - total_units += 1 - - # Count victoryConds array items - if "victoryConds" in data and isinstance(data["victoryConds"], list): - for cond in data["victoryConds"]: - if cond: - total_units += 1 - - # Count defeatConds array items - if "defeatConds" in data and isinstance(data["defeatConds"], list): - for cond in data["defeatConds"]: - if cond: - total_units += 1 - - # Count units with events (EnemyUnits, EvEnemyUnits) - for unitsKey in ["EnemyUnits", "EvEnemyUnits"]: - if unitsKey in data and isinstance(data[unitsKey], list): - for unit in data[unitsKey]: - if not unit: - continue - - # Count unit name and desc - for field in ["name", "desc"]: - if field in unit and unit[field]: - total_units += 1 - - # Count events->pages->commands->data and speaker - if "events" in unit and isinstance(unit["events"], list): - for event in unit["events"]: - if not event or "pages" not in event: - continue - - if not isinstance(event["pages"], list): - continue - - for page in event["pages"]: - if not page or "commands" not in page: - continue - - if not isinstance(page["commands"], list): - continue - - for command in page["commands"]: - if not command: - continue - - # Count data array items - if "data" in command and isinstance(command["data"], list): - for text in command["data"]: - if text: - total_units += 1 - - # Count speaker field - if "speaker" in command and command["speaker"]: - total_units += 1 - - # Count events (placeEvents, autoEvents, openingEvents, communicationEvents) - for eventsKey in ["placeEvents", "autoEvents", "openingEvents", "communicationEvents"]: - if eventsKey in data and isinstance(data[eventsKey], list): - for event in data[eventsKey]: - if not event: - continue - - # Count event-level name and desc fields - for field in ["name", "desc"]: - if field in event and event[field]: - total_units += 1 - - if "pages" not in event or not isinstance(event["pages"], list): - continue - - for page in event["pages"]: - if not page or "commands" not in page: - continue - - if not isinstance(page["commands"], list): - continue - - for command in page["commands"]: - if not command: - continue - - # Count data array items - if "data" in command and isinstance(command["data"], list): - for text in command["data"]: - if text: - total_units += 1 - - # Count speaker field - if "speaker" in command and command["speaker"]: - total_units += 1 - - # Setup progress bar (per-file instance) - with LOCK: - pbar = tqdm( - desc=filename, - total=total_units, - bar_format=BAR_FORMAT, - position=POSITION, - leave=LEAVE, - ) - PBAR = pbar - - # Translate the data using two-pass approach - result = translateMap(data, filename, pbar=pbar) - totalTokens[0] += result[0] - totalTokens[1] += result[1] - - return (data, totalTokens, None) - - except Exception as e: - traceback.print_exc() - return (data, totalTokens, e) - finally: - try: - if pbar is not None: - pbar.close() - except Exception: - pass - PBAR = None - - -def translateMap(data, filename, translatedDataList=None, pbar=None): - """ - Translates map.json data structure. - Uses two-pass approach via recursion: - - Pass 1 (translatedDataList=None): Collect strings and batch translate - - Pass 2 (translatedDataList set): Apply translations back to the data - - Args: - data: Single map object with name, desc, mapName, victoryConds, defeatConds, and EnemyUnits - filename: Name of the file being translated - translatedDataList: List containing translation lists - - Pass 1: Empty lists to collect originals - - Pass 2: Filled lists with translations - - Returns: - Tuple of [input tokens, output tokens] - """ - global PBAR - - totalTokens = [0, 0] - - # Initialize or extract lists - if translatedDataList is None: - # PASS 1: Create empty lists to collect strings - descList = [] - mapNameList = [] - victoryCondsList = [] - defeatCondsList = [] - unitNameList = [] - unitDescList = [] - eventNameList = [] - eventDescList = [] - dataList = [] - speakerList = [] - originalSpeakerList = [] # For vocab update - else: - # PASS 2: Use provided translated lists - descList = translatedDataList[0] - mapNameList = translatedDataList[1] - victoryCondsList = translatedDataList[2] - defeatCondsList = translatedDataList[3] - unitNameList = translatedDataList[4] - unitDescList = translatedDataList[5] - eventNameList = translatedDataList[6] - eventDescList = translatedDataList[7] - dataList = translatedDataList[8] - speakerList = translatedDataList[9] - - # Note: name field is not translated as it's an identifier (e.g., "ch3_瘴気の森") - - # Handle desc field - if "desc" in data and data["desc"]: - # PASS 1: Collect original - if translatedDataList is None: - descList.append(data["desc"].replace("\n", " ")) - # PASS 2: Apply translation - else: - if descList: - translatedText = descList[0] - # Wordwrap - translatedText = dazedwrap.wrapText(translatedText, width=WIDTH) - data["desc"] = translatedText - descList.pop(0) - - # Handle mapName field - if "mapName" in data and data["mapName"]: - # PASS 1: Collect original - if translatedDataList is None: - mapNameList.append(data["mapName"]) - # PASS 2: Apply translation - else: - if mapNameList: - data["mapName"] = mapNameList[0] - mapNameList.pop(0) - - # Handle victoryConds array - if "victoryConds" in data and isinstance(data["victoryConds"], list): - for i, cond in enumerate(data["victoryConds"]): - if cond: - # PASS 1: Collect original - if translatedDataList is None: - victoryCondsList.append(cond) - # PASS 2: Apply translation - else: - if victoryCondsList: - data["victoryConds"][i] = victoryCondsList[0] - victoryCondsList.pop(0) - - # Handle defeatConds array - if "defeatConds" in data and isinstance(data["defeatConds"], list): - for i, cond in enumerate(data["defeatConds"]): - if cond: - # PASS 1: Collect original - if translatedDataList is None: - defeatCondsList.append(cond) - # PASS 2: Apply translation - else: - if defeatCondsList: - data["defeatConds"][i] = defeatCondsList[0] - defeatCondsList.pop(0) - - # Process all unit arrays (EnemyUnits, EvEnemyUnits) - for unitsKey in ["EnemyUnits", "EvEnemyUnits"]: - if unitsKey in data and isinstance(data[unitsKey], list): - for unit in data[unitsKey]: - if not unit: - continue - - # Handle unit name - if "name" in unit and unit["name"]: - if translatedDataList is None: - unitNameList.append(unit["name"]) - else: - if unitNameList: - unit["name"] = unitNameList[0] - unitNameList.pop(0) - - # Handle unit desc - if "desc" in unit and unit["desc"]: - if translatedDataList is None: - unitDescList.append(unit["desc"].replace("\n", " ")) - else: - if unitDescList: - translatedText = unitDescList[0] - # Wordwrap - translatedText = dazedwrap.wrapText(translatedText, width=WIDTH) - unit["desc"] = translatedText - unitDescList.pop(0) - - # Process events within units - if "events" in unit and isinstance(unit["events"], list): - for event in unit["events"]: - if not event or "pages" not in event: - continue - - if not isinstance(event["pages"], list): - continue - - for page in event["pages"]: - if not page or "commands" not in page: - continue - - if not isinstance(page["commands"], list): - continue - - for command in page["commands"]: - if not command: - continue - - speaker = command.get("speaker", "") - - # Handle data array - if "data" in command and isinstance(command["data"], list): - for i, text in enumerate(command["data"]): - if text: - if translatedDataList is None: - text = text.replace("\n", " ") - if speaker: - dataList.append(f"[{speaker}]: {text}") - else: - dataList.append(text) - else: - if dataList: - translated = dataList[0] - if speaker: - match = re.search(r'(^\[.+?\]\s?[|:]\s?)', translated) - if match: - translated = translated.replace(match.group(1), "") - translated = dazedwrap.wrapText(translated, width=WIDTH) - command["data"][i] = translated - dataList.pop(0) - - # Handle speaker field - if "speaker" in command and command["speaker"]: - if translatedDataList is None: - originalSpeakerList.append(command["speaker"]) - speakerList.append(command["speaker"]) - else: - if speakerList: - command["speaker"] = speakerList[0] - speakerList.pop(0) - - # Process all event arrays (placeEvents, autoEvents, openingEvents, communicationEvents) - for eventsKey in ["placeEvents", "autoEvents", "openingEvents", "communicationEvents"]: - if eventsKey in data and isinstance(data[eventsKey], list): - for event in data[eventsKey]: - if not event: - continue - - # Handle event-level name field - if "name" in event and event["name"]: - if translatedDataList is None: - eventNameList.append(event["name"]) - else: - if eventNameList: - event["name"] = eventNameList[0] - eventNameList.pop(0) - - # Handle event-level desc field - if "desc" in event and event["desc"]: - if translatedDataList is None: - # Remove newlines for translation - eventDescList.append(event["desc"].replace("\n", " ")) - else: - if eventDescList: - translatedText = eventDescList[0] - # Apply text wrapping - translatedText = dazedwrap.wrapText(translatedText, width=WIDTH) - event["desc"] = translatedText - eventDescList.pop(0) - - if "pages" not in event or not isinstance(event["pages"], list): - continue - - for page in event["pages"]: - if not page or "commands" not in page: - continue - - if not isinstance(page["commands"], list): - continue - - for command in page["commands"]: - if not command: - continue - - speaker = command.get("speaker", "") - - # Handle data array - if "data" in command and isinstance(command["data"], list): - for i, text in enumerate(command["data"]): - if text: - if translatedDataList is None: - text = text.replace("\n", " ") - if speaker: - dataList.append(f"[{speaker}]: {text}") - else: - dataList.append(text) - else: - if dataList: - translated = dataList[0] - if speaker: - match = re.search(r'(^\[.+?\]\s?[|:]\s?)', translated) - if match: - translated = translated.replace(match.group(1), "") - translated = dazedwrap.wrapText(translated, width=WIDTH) - command["data"][i] = translated - dataList.pop(0) - - # Handle speaker field - if "speaker" in command and command["speaker"]: - if translatedDataList is None: - originalSpeakerList.append(command["speaker"]) - speakerList.append(command["speaker"]) - else: - if speakerList: - command["speaker"] = speakerList[0] - speakerList.pop(0) - - # If this was Pass 1, do the translation and recurse for Pass 2 - if translatedDataList is None: - # Store original counts for mismatch checking - originalDescCount = len(descList) - originalMapNameCount = len(mapNameList) - originalVictoryCondsCount = len(victoryCondsList) - originalDefeatCondsCount = len(defeatCondsList) - originalUnitNameCount = len(unitNameList) - originalUnitDescCount = len(unitDescList) - originalEventNameCount = len(eventNameList) - originalEventDescCount = len(eventDescList) - originalDataCount = len(dataList) - originalSpeakerCount = len(speakerList) - - # Batch translate map descriptions - if descList: - response = translateAI( - descList, - "Reply with only the " + LANGUAGE + " translation of the map description.", - True, - filename, - pbar - ) - descList = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - - # Batch translate map display names - if mapNameList: - response = translateAI( - mapNameList, - "Reply with only the " + LANGUAGE + " translation of the map display name.", - True, - filename, - pbar - ) - mapNameList = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - - # Batch translate victory conditions - if victoryCondsList: - response = translateAI( - victoryCondsList, - "Reply with only the " + LANGUAGE + " translation of the victory condition.", - True, - filename, - pbar - ) - victoryCondsList = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - - # Batch translate defeat conditions - if defeatCondsList: - response = translateAI( - defeatCondsList, - "Reply with only the " + LANGUAGE + " translation of the defeat condition.", - True, - filename, - pbar - ) - defeatCondsList = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - - # Batch translate unit names - if unitNameList: - response = translateAI( - unitNameList, - "Reply with only the " + LANGUAGE + " translation of the enemy unit name.", - True, - filename, - pbar - ) - unitNameList = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - - # Batch translate unit descriptions - if unitDescList: - response = translateAI( - unitDescList, - "Reply with only the " + LANGUAGE + " translation of the enemy unit description.", - True, - filename, - pbar - ) - unitDescList = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - - # Batch translate event names - if eventNameList: - response = translateAI( - eventNameList, - "Reply with only the " + LANGUAGE + " translation of the event name.", - True, - filename, - pbar - ) - eventNameList = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - - # Batch translate event descriptions - if eventDescList: - response = translateAI( - eventDescList, - "Reply with only the " + LANGUAGE + " translation of the event description.", - True, - filename, - pbar - ) - eventDescList = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - - # Batch translate dialogue data - if dataList: - response = translateAI( - dataList, - "Reply with only the " + LANGUAGE + " translation of the dialogue text.", - True, - filename, - pbar - ) - dataList = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - - # Batch translate speakers - if speakerList: - response = translateAI( - speakerList, - "Reply with only the " + LANGUAGE + " translation of the speaker name.", - True, - filename, - pbar - ) - speakerList = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - - # Check for mismatch errors - if len(descList) != originalDescCount: - with LOCK: - if filename not in MISMATCH: - MISMATCH.append(filename) - - if len(mapNameList) != originalMapNameCount: - with LOCK: - if filename not in MISMATCH: - MISMATCH.append(filename) - - if len(victoryCondsList) != originalVictoryCondsCount: - with LOCK: - if filename not in MISMATCH: - MISMATCH.append(filename) - - if len(defeatCondsList) != originalDefeatCondsCount: - with LOCK: - if filename not in MISMATCH: - MISMATCH.append(filename) - - if len(unitNameList) != originalUnitNameCount: - with LOCK: - if filename not in MISMATCH: - MISMATCH.append(filename) - - if len(unitDescList) != originalUnitDescCount: - with LOCK: - if filename not in MISMATCH: - MISMATCH.append(filename) - - if len(eventNameList) != originalEventNameCount: - with LOCK: - if filename not in MISMATCH: - MISMATCH.append(filename) - - if len(eventDescList) != originalEventDescCount: - with LOCK: - if filename not in MISMATCH: - MISMATCH.append(filename) - - if len(dataList) != originalDataCount: - with LOCK: - if filename not in MISMATCH: - MISMATCH.append(filename) - - if len(speakerList) != originalSpeakerCount: - with LOCK: - if filename not in MISMATCH: - MISMATCH.append(filename) - - # PASS 2: Recursively call to apply translations - translateMap(data, filename, [descList, mapNameList, victoryCondsList, defeatCondsList, unitNameList, unitDescList, eventNameList, eventDescList, dataList, speakerList], pbar) - - return totalTokens - - -def getResultString(translatedData, translationTime, filename): - """ - Formats the translation result string with token counts, cost, and time. - - Args: - translatedData: Tuple of (data, tokens, error) - translationTime: Time taken for translation - filename: Name of the file - - Returns: - Formatted result string - """ - global TIMETOTAL - - # Calculate cost - cost = calculateCost(translatedData[1][0], translatedData[1][1], MODEL) - totalTokenstring = ( - Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]" - "[Output: " - + str(translatedData[1][1]) - + "]" "[Cost: ${:,.4f}".format(cost) - + "]" - ) - - # Format time string - if filename != "TOTAL": - timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]" - TIMETOTAL += round(translationTime, 1) - else: - timeString = Fore.BLUE + "[" + str(round(TIMETOTAL, 1)) + "s]" - - # Return success or failure string - if translatedData[2] is None: - # Success - return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET - else: - # Fail - try: - raise translatedData[2] - except Exception as e: - traceback.print_exc() - return ( - filename - + ": " - + totalTokenstring - + timeString - + Fore.RED - + " \u2717 " - + Fore.RESET - ) - - -def getSpeaker(speaker): - """ - Translates speaker/character names with caching to avoid redundant translations. - - Args: - speaker: The original speaker name to translate - - Returns: - List containing [translated name, [input tokens, output tokens]] - """ - if speaker == "": - return ["", [0, 0]] - - # Check if speaker has already been translated - for i in range(len(NAMESLIST)): - if speaker == NAMESLIST[i][0]: - return [NAMESLIST[i][1], [0, 0]] - - # Translate and Store Speaker - response = translateAI( - speaker, - "Reply with the " + LANGUAGE + " translation of the NPC name.", - False, - ) - response[0] = response[0].title() - response[0] = response[0].replace("'S", "'s") - response[0] = response[0].replace("Speaker: ", "") - - # Retry if name doesn't translate for some reason - if re.search(r"([a-zA-Z？?])", response[0]) is None: - response = translateAI( - speaker, - "Reply with the " + LANGUAGE + " translation of the NPC name.", - False, - ) - response[0] = response[0].title() - response[0] = response[0].replace("'S", "'s") - - speakerList = [speaker, response[0]] - NAMESLIST.append(speakerList) - return response - - -def translateAI(text, history, history_ctx=None, filename=None, pbar=None): - """ - Legacy wrapper function for the new shared translation utility. - This maintains compatibility with existing code while using the new shared implementation. - - Args: - text: Text to translate (can be string or list) - history: History/context for the translation - - Returns: - List containing [translated text, [input tokens, output tokens]] - """ - global PBAR, MISMATCH, FILENAME - - # Update config estimate mode based on global ESTIMATE - TRANSLATION_CONFIG.estimateMode = bool(ESTIMATE) - - # Call the new shared translation function - return sharedtranslateAI( - text=text, - history=history, - config=TRANSLATION_CONFIG, - filename=filename if filename is not None else FILENAME, - pbar=pbar if pbar is not None else PBAR, - lock=LOCK, - mismatchList=MISMATCH - ) - +# Libraries +import json +import os +import re +import shutil +import util.dazedwrap as dazedwrap +import threading +import time +import traceback +from dotenv import load_dotenv +from pathlib import Path +from colorama import Fore +from dotenv import load_dotenv +from retry import retry +from tqdm import tqdm +from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost + +# Globals +MODEL = os.getenv("model") +TIMEOUT = int(os.getenv("timeout")) +LANGUAGE = os.getenv("language").capitalize() +from util.paths import PROMPT_PATH, VOCAB_PATH + +PROMPT = PROMPT_PATH.read_text(encoding="utf-8") +VOCAB = VOCAB_PATH.read_text(encoding="utf-8") +LOCK = threading.Lock() +VOCAB_LOCK = threading.Lock() # Dedicated lock for vocab.txt updates +WIDTH = int(os.getenv("width")) +LISTWIDTH = int(os.getenv("listWidth")) +NOTEWIDTH = int(os.getenv("noteWidth")) +MAXHISTORY = 10 +ESTIMATE = "" +TOKENS = [0, 0] +MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong) +NAMESLIST = [] # List of speaker names and their translations +PBAR = None +FILENAME = None +TIMETOTAL = 0 # Total Time Taken for all translations + +# Regex - Need to change this if you want to translate from/to other languages. Default is Japanese Regex +LANGREGEX = r"[\u3000-\u303F\u3040-\u309F\u30A0-\u30FF\u31F0-\u31FF\u3400-\u4DBF\u4E00-\u9FFF\uF900-\uFAFF\uFF61-\uFF9F]+" + +# Get pricing configuration based on the model +PRICING_CONFIG = getPricingConfig(MODEL) +INPUTAPICOST = PRICING_CONFIG["inputAPICost"] +OUTPUTAPICOST = PRICING_CONFIG["outputAPICost"] +BATCHSIZE = PRICING_CONFIG["batchSize"] +FREQUENCY_PENALTY = PRICING_CONFIG["frequencyPenalty"] + +# tqdm Globals +BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}" +POSITION = 0 +LEAVE = False + +# Initialize Translation Config +TRANSLATION_CONFIG = TranslationConfig( + model=MODEL, + language=LANGUAGE, + prompt=PROMPT, + vocab=VOCAB, + langRegex=LANGREGEX, + batchSize=BATCHSIZE, + maxHistory=MAXHISTORY, + estimateMode=False # Will be set dynamically based on ESTIMATE +) + +# Config (Default) +FIXTEXTWRAP = True # Rewrap text to WIDTH +IGNORETLTEXT = False # Skip Translated Text + +# List of file patterns that use parseGeneric +# Add more patterns here as needed +GENERIC_FILES = [ + "quests", + "shops", + "shoplayout", + "bonuses", + "base", + "battleprep", + "manage", + "mapcommands", + "title", + "classes", + "classesgroups", + "classtypes", + "races", + "skills", + "weapons", + "states", + "difficulties", + "fonts", + "fusionsettings", + "transformations", + "characters", + "glossary", + "npc", + "screens", + "strings", + "originalterrains", + "runtimeterrains", + "archers", + "fighters", + "mages", + "items", +] + +# List of file patterns that use parseMap +# Be specific to avoid catching non-map files like CommandLayout/mapcommands.json +# SRPG Studio map files in this project follow the pattern Maps/map_XXX.json +MAP_FILES = [ + "map_", # e.g., Maps/map_000.json +] + + +def update_vocab_section(category: str, pairs: list[tuple[str, str]]): + """Update or insert a section in vocab.txt for the given category with provided pairs. + Only writes when there's an actual translation (dst is non-empty and differs from src after normalization). + - category: e.g., "Items", "Weapons", "Speakers", etc. Section header will be "# {category}". + - pairs: list of (source, translated) strings. Duplicates by source are deduped (last wins). + The existing section is replaced entirely; other sections are preserved. + """ + try: + vocab_path = VOCAB_PATH + + # Helper: normalized comparison to detect no-op translations + def _norm(s: str) -> str: + if s is None: + return "" + # Collapse whitespace and case-fold; leave punctuation to avoid over-matching + return re.sub(r"\s+", " ", str(s)).strip().casefold() + + # Filter and deduplicate by source term (last mapping wins) + dedup: dict[str, str] = {} + for src, dst in pairs: + if not src: + continue + # Skip when no destination or no actual change + if dst is None or _norm(dst) == "" or _norm(dst) == _norm(src): + continue + dedup[src] = dst + + # If nothing to add after filtering, skip touching the file + if not dedup: + return + + # Guard the read-modify-write with a dedicated lock to avoid races + with VOCAB_LOCK: + existing = vocab_path.read_text(encoding="utf-8") if vocab_path.exists() else "" + + lines = [f"{src} ({dst})" for src, dst in dedup.items()] + # Always terminate a section with a blank line to separate from next header + new_block = f"# {category}\n" + "\n".join(lines) + if not new_block.endswith("\n\n"): + if not new_block.endswith("\n"): + new_block += "\n" + new_block += "\n" + + # Regex to find the specific section starting at the header for this category + # and ending right before the next header (any number of '#') or EOF. + # - Handles headers like '#Category', '# Category', '## Category', etc. + # - Uses non-greedy matching for the body to avoid spanning multiple sections. + pattern = re.compile( + rf"^[\t ]*#+\s*{re.escape(category)}\s*$\r?\n.*?(?=^[\t ]*#|\Z)", + re.MULTILINE | re.DOTALL, + ) + if pattern.search(existing): + # Replace only the first matching section for this category. + updated = pattern.sub(lambda m: new_block, existing, count=1) + else: + updated = existing + if updated and not updated.endswith("\n\n"): + # Ensure a blank line before appending new section if file not empty + if not updated.endswith("\n"): + updated += "\n" + updated += "\n" + updated += new_block + + # Avoid writing if nothing changed + if updated == existing: + return + # Atomic write: write to unique temp and replace with retries on Windows + tmp_path = vocab_path.with_suffix(vocab_path.suffix + f".{os.getpid()}.{threading.get_ident()}.tmp") + tmp_path.write_text(updated, encoding="utf-8") + + attempts = 6 + delay = 0.1 + last_err = None + for attempt in range(attempts): + try: + os.replace(tmp_path, vocab_path) + last_err = None + break + except PermissionError as e: + last_err = e + # Try relaxing permissions then retry + try: + if vocab_path.exists(): + os.chmod(vocab_path, 0o666) + except Exception: + pass + time.sleep(delay) + delay = min(1.0, delay * 2) + except Exception as e: + last_err = e + break + if last_err is not None: + try: + shutil.move(str(tmp_path), str(vocab_path)) + except Exception: + try: + if tmp_path.exists(): + tmp_path.unlink(missing_ok=True) + except Exception: + pass + raise last_err + except Exception: + traceback.print_exc() + + +def handleSRPG(filename, estimate): + """ + Main handler function for SRPG Studio files. + + Args: + filename: Name of the file to translate + estimate: Boolean indicating if this is an estimate run + + Returns: + String with translation results or error message + """ + global ESTIMATE, TOKENS, FILENAME, TIMETOTAL + ESTIMATE = estimate + FILENAME = filename + + # Translate + start = time.time() + translatedData = openFiles(filename) + + # Write output file if not in estimate mode + if not estimate: + try: + with open("translated/" + filename, "w", encoding="utf-8", newline="\n") as outFile: + json.dump(translatedData[0], outFile, ensure_ascii=False, indent=4) + except Exception: + traceback.print_exc() + return "Fail" + + # Print File + end = time.time() + tqdm.write(getResultString(translatedData, end - start, filename)) + with LOCK: + TOKENS[0] += translatedData[1][0] + TOKENS[1] += translatedData[1][1] + + # Print Total + totalString = getResultString(["", TOKENS, None], end - start, "TOTAL") + + # Print any errors + if len(MISMATCH) > 0: + return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET + else: + return totalString + + +def openFiles(filename): + """ + Opens and routes SRPG Studio files to appropriate parsing functions. + + Args: + filename: Name of the file to open and parse + + Returns: + Tuple of (translated data, token counts, error) + """ + with open("files/" + filename, "r", encoding="utf-8-sig") as f: + data = json.load(f) + + # Check if filename matches recollection pattern + if "recollection" in filename.lower(): + translatedData = parseRecollection(data, filename) + # mapcommonevents is a list of events like recollection + elif "mapcommonevents" in filename.lower(): + translatedData = parseRecollection(data, filename) + # Bookmark events use the same structure as recollection (list of events with pages/commands) + elif "bookmarkevents" in filename.lower(): + translatedData = parseRecollection(data, filename) + # Event files (autoevents, placeevents, talkevents, communicationevents, openingevents, endingevents) + # all have name, desc, and pages->commands structure like recollection + elif any(event_type in filename.lower() for event_type in ["autoevents", "placeevents", "talkevents", "communicationevents", "openingevents", "endingevents"]): + translatedData = parseRecollection(data, filename) + # Check if filename matches bookmark pattern (top-level entries with name/desc + events) + elif "bookmark" in filename.lower(): + translatedData = parseBookmark(data, filename) + # Players have the same shape as bookmark entries + elif "players" in filename.lower(): + translatedData = parseBookmark(data, filename) + # Titles.json is a small dict of title strings + elif os.path.basename(filename).lower() == "titles.json": + translatedData = parseTitles(data, filename) + # Check if filename matches map pattern + elif any(pattern in filename.lower() for pattern in MAP_FILES): + translatedData = parseMap(data, filename) + # Check if filename matches any pattern in GENERIC_FILES + elif any(pattern in filename.lower() for pattern in GENERIC_FILES): + translatedData = parseGeneric(data, filename) + + # TODO: Add other SRPG Studio file types here + else: + raise NameError(filename + " Not Supported") + + return translatedData + + +def parseBookmark(data, filename): + """ + Parser for SRPG Studio bookmark.json files. + Structure: List of entries, each with id, name, desc, and events containing pages -> commands + where commands have data (array of strings) and optional speaker. + + Args: + data: Parsed JSON data (list of bookmark entries) + filename: Name of the file being parsed + + Returns: + Tuple of (translated data, token counts, error) + """ + global PBAR + totalTokens = [0, 0] + + pbar = None + try: + # Count work units: names, descs, dialogue lines, and speakers + total_units = 0 + + for entry in data: + if not entry: + continue + + # name and desc + for field in ["name", "desc"]: + if field in entry and entry[field]: + total_units += 1 + + # events -> pages -> commands + if "events" in entry and isinstance(entry["events"], list): + for event in entry["events"]: + if not event: + continue + + # Count event-level name and desc fields + for field in ["name", "desc"]: + if field in event and event[field]: + total_units += 1 + + if "pages" not in event or not isinstance(event["pages"], list): + continue + + for page in event["pages"]: + if not page or "commands" not in page or not isinstance(page["commands"], list): + continue + for command in page["commands"]: + if not command: + continue + if "data" in command and isinstance(command["data"], list): + for text in command["data"]: + if text: + total_units += 1 + if "speaker" in command and command["speaker"]: + total_units += 1 + + # Setup progress bar + with LOCK: + pbar = tqdm( + desc=filename, + total=total_units, + bar_format=BAR_FORMAT, + position=POSITION, + leave=LEAVE, + ) + PBAR = pbar + + # Translate using two-pass approach + result = translateBookmark(data, filename, pbar=pbar) + totalTokens[0] += result[0] + totalTokens[1] += result[1] + + return (data, totalTokens, None) + + except Exception as e: + traceback.print_exc() + return (data, totalTokens, e) + finally: + try: + if pbar is not None: + pbar.close() + except Exception: + pass + PBAR = None + + +def translateBookmark(data, filename, translatedDataList=None, pbar=None): + """ + Translates bookmark.json data structure. + Two-pass approach via recursion: + - Pass 1: Collect strings (names, descs, dialogue data with speaker prefix, speakers) + - Pass 2: Apply translations back into data + + Returns: + [input tokens, output tokens] + """ + totalTokens = [0, 0] + + # Initialize or extract lists + if translatedDataList is None: + nameList = [] + descList = [] + dataList = [] + speakerList = [] + else: + nameList = translatedDataList[0] + descList = translatedDataList[1] + dataList = translatedDataList[2] + speakerList = translatedDataList[3] + + for entry in data: + if not entry: + continue + + # name + if "name" in entry and entry["name"]: + if translatedDataList is None: + nameList.append(entry["name"]) + else: + if nameList: + entry["name"] = nameList[0] + nameList.pop(0) + + # desc + if "desc" in entry and entry["desc"]: + if translatedDataList is None: + # Remove newlines for translation + descList.append(entry["desc"].replace("\n", " ")) + else: + if descList: + translatedText = descList[0] + # Apply text wrapping + translatedText = dazedwrap.wrapText(translatedText, width=WIDTH) + entry["desc"] = translatedText + descList.pop(0) + + # events -> pages -> commands + if "events" in entry and isinstance(entry["events"], list): + for event in entry["events"]: + if not event: + continue + + # Handle event-level name field + if "name" in event and event["name"]: + if translatedDataList is None: + nameList.append(event["name"]) + else: + if nameList: + event["name"] = nameList[0] + nameList.pop(0) + + # Handle event-level desc field + if "desc" in event and event["desc"]: + if translatedDataList is None: + # Remove newlines for translation + descList.append(event["desc"].replace("\n", " ")) + else: + if descList: + translatedText = descList[0] + # Apply text wrapping + translatedText = dazedwrap.wrapText(translatedText, width=WIDTH) + event["desc"] = translatedText + descList.pop(0) + + if "pages" not in event or not isinstance(event["pages"], list): + continue + + for page in event["pages"]: + if not page or "commands" not in page or not isinstance(page["commands"], list): + continue + for command in page["commands"]: + if not command: + continue + + speaker = command.get("speaker", "") + + # data array + if "data" in command and isinstance(command["data"], list): + for i, text in enumerate(command["data"]): + if text: + if translatedDataList is None: + text = text.replace("\n", " ") + if speaker: + dataList.append(f"[{speaker}]: {text}") + else: + dataList.append(text) + else: + if dataList: + translated = dataList[0] + if speaker: + match = re.search(r'(^\[.+?\]\s?[|:]\s?)', translated) + if match: + translated = translated.replace(match.group(1), "") + translated = dazedwrap.wrapText(translated, width=WIDTH) + command["data"][i] = translated + dataList.pop(0) + + # speaker field + if "speaker" in command and command["speaker"]: + if translatedDataList is None: + speakerList.append(command["speaker"]) + else: + if speakerList: + command["speaker"] = speakerList[0] + speakerList.pop(0) + + # If this was Pass 1, perform translations and recurse + if translatedDataList is None: + originalNameCount = len(nameList) + originalDescCount = len(descList) + originalDataCount = len(dataList) + originalSpeakerCount = len(speakerList) + + if nameList: + response = translateAI( + nameList, + "Reply with only the " + LANGUAGE + " translation of the bookmark name.", + True, + filename, + pbar, + ) + nameList = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + + if descList: + response = translateAI( + descList, + "Reply with only the " + LANGUAGE + " translation of the bookmark description.", + True, + filename, + pbar, + ) + descList = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + + if dataList: + response = translateAI( + dataList, + "Reply with only the " + LANGUAGE + " translation of the dialogue text.", + True, + filename, + pbar, + ) + dataList = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + + if speakerList: + response = translateAI( + speakerList, + "Reply with only the " + LANGUAGE + " translation of the speaker name.", + True, + filename, + pbar, + ) + speakerList = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + + # Mismatch checks + if len(nameList) != originalNameCount: + with LOCK: + if filename not in MISMATCH: + MISMATCH.append(filename) + if len(descList) != originalDescCount: + with LOCK: + if filename not in MISMATCH: + MISMATCH.append(filename) + if len(dataList) != originalDataCount: + with LOCK: + if filename not in MISMATCH: + MISMATCH.append(filename) + if len(speakerList) != originalSpeakerCount: + with LOCK: + if filename not in MISMATCH: + MISMATCH.append(filename) + + # PASS 2 + translateBookmark(data, filename, [nameList, descList, dataList, speakerList], pbar) + + return totalTokens + + +def parseGeneric(data, filename): + """ + Generic parser for SRPG Studio files with id, name, desc structure. + Handles files like quests.json. + Uses a two-pass approach: first pass collects all strings to translate, + then batch translates them, then second pass applies translations. + + Args: + data: Parsed JSON data (list of objects with id, name, desc) + filename: Name of the file being parsed + + Returns: + Tuple of (data, token counts, error) + """ + global PBAR + + totalTokens = [0, 0] + + pbar = None + try: + # Count work units (all translatable fields that need translation) + total_units = 0 + translatable_fields = ["name", "desc", "commandName", "command"] + + for entry in data: + if entry: + for field in translatable_fields: + if field in entry and entry[field]: + # If command is a list (e.g., fusionsettings), count per element + if field == "command" and isinstance(entry[field], list): + total_units += sum(1 for x in entry[field] if x) + # Otherwise simple increment + elif not isinstance(entry[field], list): + total_units += 1 + + # Handle pages array separately + if "pages" in entry and entry["pages"] and isinstance(entry["pages"], list): + for page in entry["pages"]: + if page: + total_units += 1 + + # Handle msg arrays (e.g., shoplayout) + if "msg" in entry and isinstance(entry["msg"], list): + total_units += sum(1 for m in entry["msg"] if m) + + # Handle rewardData arrays (e.g., quests) + if "rewardData" in entry and isinstance(entry["rewardData"], list): + total_units += sum(1 for r in entry["rewardData"] if r) + + # Handle customParameters name field (e.g., {name:'シャルロット強制売春'}) + if "customParameters" in entry and entry["customParameters"]: + match = re.search(r"name:\s*['\"]([^'\"]+)['\"]", entry["customParameters"]) + if match: + total_units += 1 + + # Handle terrains array (nested structure) + if "terrains" in entry and entry["terrains"] and isinstance(entry["terrains"], list): + for terrain in entry["terrains"]: + if terrain: + for field in ["name", "desc"]: + if field in terrain and terrain[field]: + total_units += 1 + + # Setup progress bar (use a per-file instance to avoid cross-thread clashes) + with LOCK: + pbar = tqdm( + desc=filename, + total=total_units, + bar_format=BAR_FORMAT, + position=POSITION, + leave=LEAVE, + ) + PBAR = pbar + + # Translate the data using two-pass approach + result = translateGeneric(data, filename, pbar=pbar) + totalTokens[0] += result[0] + totalTokens[1] += result[1] + + return (data, totalTokens, None) + + except Exception as e: + traceback.print_exc() + return (data, totalTokens, e) + finally: + # Ensure progress bar is closed + try: + if pbar is not None: + pbar.close() + except Exception: + pass + PBAR = None + + +def translateGeneric(data, filename, translatedDataList=None, pbar=None): + """ + Translates generic SRPG Studio data with id, name, desc, commandName, command, pages structure. + Uses two-pass approach via recursion: + - Pass 1 (translatedDataList=None): Collect strings and batch translate + - Pass 2 (translatedDataList set): Apply translations back to the data + + Args: + data: List of objects with id, name, desc, commandName, command, pages keys + filename: Name of the file being translated + translatedDataList: List containing [nameList, descList, commandNameList, commandList, pagesList] + - Pass 1: Empty lists to collect originals + - Pass 2: Filled lists with translations + + Returns: + Tuple of [input tokens, output tokens] + """ + global PBAR + + totalTokens = [0, 0] + + # Initialize or extract lists + if translatedDataList is None: + # PASS 1: Create empty lists to collect strings + nameList = [] + descList = [] + commandNameList = [] + commandList = [] + pagesList = [] + msgList = [] + commandArrayList = [] + rewardDataList = [] + customParametersNameList = [] + else: + # PASS 2: Use provided translated lists + nameList = translatedDataList[0] + descList = translatedDataList[1] + commandNameList = translatedDataList[2] + commandList = translatedDataList[3] + pagesList = translatedDataList[4] + msgList = translatedDataList[5] + commandArrayList = translatedDataList[6] + rewardDataList = translatedDataList[7] + customParametersNameList = translatedDataList[8] + + # Single loop - behavior depends on which pass we're in + for entry in data: + if not entry: + continue + + # Handle name field + if "name" in entry and entry["name"]: + # PASS 1: Collect original + if translatedDataList is None: + nameList.append(entry["name"]) + # PASS 2: Apply translation + else: + if nameList: + entry["name"] = nameList[0] + nameList.pop(0) + + # Handle desc field + if "desc" in entry and entry["desc"]: + # PASS 1: Collect original + if translatedDataList is None: + # Nuke Wordwrap + descList.append(entry["desc"].replace("\n", " ")) + # PASS 2: Apply translation + else: + if descList: + translatedText = descList[0] + # Wordwrap + translatedText = dazedwrap.wrapText(translatedText, width=WIDTH) + + # Set Data + entry["desc"] = translatedText + descList.pop(0) + + # Handle commandName field + if "commandName" in entry and entry["commandName"]: + # PASS 1: Collect original + if translatedDataList is None: + commandNameList.append(entry["commandName"]) + # PASS 2: Apply translation + else: + if commandNameList: + entry["commandName"] = commandNameList[0] + commandNameList.pop(0) + + # Handle command field (string or list) + if "command" in entry and entry["command"] is not None: + # If list of strings + if isinstance(entry["command"], list): + for i, val in enumerate(entry["command"]): + if val: + if translatedDataList is None: + # Nuke Wrap + commandArrayList.append(val.replace("\n", " ")) + else: + if commandArrayList: + translatedText = dazedwrap.wrapText(commandArrayList[0], width=WIDTH) + entry["command"][i] = translatedText + commandArrayList.pop(0) + # If simple string + elif isinstance(entry["command"], str): + if translatedDataList is None: + commandList.append(entry["command"]) + else: + if commandList: + entry["command"] = commandList[0] + commandList.pop(0) + + # Handle pages field (array of strings) + if "pages" in entry and entry["pages"] and isinstance(entry["pages"], list): + for i, page in enumerate(entry["pages"]): + if page: + # PASS 1: Collect original + if translatedDataList is None: + # Nuke Wordwrap + page = page.replace("\n", " ") + pagesList.append(page) + # PASS 2: Apply translation + else: + if pagesList: + translatedText = pagesList[0] + + # Wordwrap + translatedText = dazedwrap.wrapText(translatedText, width=WIDTH) + + # Set Data + entry["pages"][i] = translatedText + pagesList.pop(0) + + # Handle msg field (array of strings) + if "msg" in entry and isinstance(entry["msg"], list): + for i, val in enumerate(entry["msg"]): + if val is not None: + if translatedDataList is None: + msgList.append(str(val).replace("\n", " ")) + else: + if msgList: + translatedText = dazedwrap.wrapText(msgList[0], width=WIDTH) + entry["msg"][i] = translatedText + msgList.pop(0) + + # Handle rewardData field (array of strings) + if "rewardData" in entry and isinstance(entry["rewardData"], list): + for i, val in enumerate(entry["rewardData"]): + if val: + # PASS 1: Collect original + if translatedDataList is None: + rewardDataList.append(str(val).replace("\n", " ")) + # PASS 2: Apply translation + else: + if rewardDataList: + translatedText = rewardDataList[0] + # Set Data (no wordwrap for reward data) + entry["rewardData"][i] = translatedText + rewardDataList.pop(0) + + # Handle customParameters name field (e.g., {name:'シャルロット強制売春'}) + if "customParameters" in entry and entry["customParameters"]: + match = re.search(r"name:\s*['\"]([^'\"]+)['\"]", entry["customParameters"]) + if match: + # PASS 1: Collect original + if translatedDataList is None: + customParametersNameList.append(match.group(1)) + # PASS 2: Apply translation + else: + if customParametersNameList: + translatedName = customParametersNameList[0] + # Replace the name value in customParameters + entry["customParameters"] = re.sub( + r"(name:\s*['\"])([^'\"]+)(['\"])", + r"\1" + translatedName.replace("\\", "\\\\") + r"\3", + entry["customParameters"] + ) + customParametersNameList.pop(0) + + # Handle terrains field (nested array with name and desc) + if "terrains" in entry and entry["terrains"] and isinstance(entry["terrains"], list): + for terrain in entry["terrains"]: + if not terrain: + continue + + # Handle terrain name + if "name" in terrain and terrain["name"]: + # PASS 1: Collect original + if translatedDataList is None: + nameList.append(terrain["name"]) + # PASS 2: Apply translation + else: + if nameList: + terrain["name"] = nameList[0] + nameList.pop(0) + + # Handle terrain desc + if "desc" in terrain and terrain["desc"]: + # PASS 1: Collect original + if translatedDataList is None: + descList.append(terrain["desc"]).replace("\n", " ") + # PASS 2: Apply translation + else: + if descList: + translatedText = descList[0] + # Wordwrap + translatedText = dazedwrap.wrapText(translatedText, width=WIDTH) + + # Set Data + terrain["desc"] = translatedText + descList.pop(0) + + # If this was Pass 1, do the translation and recurse for Pass 2 + if translatedDataList is None: + # Store original counts for mismatch checking + originalNameCount = len(nameList) + originalDescCount = len(descList) + originalCommandNameCount = len(commandNameList) + originalCommandCount = len(commandList) + originalPagesCount = len(pagesList) + originalMsgCount = len(msgList) + originalCommandArrayCount = len(commandArrayList) + originalRewardDataCount = len(rewardDataList) + originalCustomParametersNameCount = len(customParametersNameList) + + # Keep a copy of original names for vocab update (for characters/items/skills/classes/weapons) + vocab_name_files = ["characters", "items", "skills", "classes", "weapons"] + originalNameList = nameList.copy() if any(tag in filename.lower() for tag in vocab_name_files) else [] + + # Batch translate names + if nameList: + response = translateAI( + nameList, + "Reply with only the " + LANGUAGE + " translation of the quest name.", + True, + filename, + pbar + ) + nameList = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + + # Update vocab.txt for name-bearing files + if originalNameList and nameList: + try: + file_lower = filename.lower() + section = None + if "characters" in file_lower: + section = "Speakers" + elif "items" in file_lower: + section = "Items" + elif "skills" in file_lower: + section = "Skills" + elif "classes" in file_lower: + section = "Classes" + elif "weapons" in file_lower: + section = "Weapons" + + if section: + vocab_pairs = list(zip(originalNameList, nameList)) + update_vocab_section(section, vocab_pairs) + except Exception: + traceback.print_exc() + + # Batch translate descriptions + if descList: + response = translateAI( + descList, + "Reply with only the " + LANGUAGE + " translation of the quest description.", + True, + filename, + pbar + ) + descList = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + + # Batch translate command names + if commandNameList: + response = translateAI( + commandNameList, + "Reply with only the " + LANGUAGE + " translation of the command name.", + True, + filename, + pbar + ) + commandNameList = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + + # Batch translate commands + if commandList: + response = translateAI( + commandList, + "Reply with only the " + LANGUAGE + " translation of the command.", + True, + filename, + pbar + ) + commandList = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + + # Batch translate pages + if pagesList: + response = translateAI( + pagesList, + "Reply with only the " + LANGUAGE + " translation of the page content.", + True, + filename, + pbar + ) + pagesList = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + + # Batch translate msg arrays + if msgList: + response = translateAI( + msgList, + "Reply with only the " + LANGUAGE + " translation of the message text.", + True, + filename, + pbar + ) + msgList = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + + # Batch translate command arrays + if commandArrayList: + response = translateAI( + commandArrayList, + "Reply with only the " + LANGUAGE + " translation of the command text.", + True, + filename, + pbar + ) + commandArrayList = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + + # Batch translate rewardData arrays + if rewardDataList: + response = translateAI( + rewardDataList, + "Reply with only the " + LANGUAGE + " translation of the reward data text.", + True, + filename, + pbar + ) + rewardDataList = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + + # Batch translate customParameters name fields + if customParametersNameList: + response = translateAI( + customParametersNameList, + "Reply with only the " + LANGUAGE + " translation of the custom parameter name.", + True, + filename, + pbar + ) + customParametersNameList = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + + # Check for mismatch errors + if len(nameList) != originalNameCount: + with LOCK: + if filename not in MISMATCH: + MISMATCH.append(filename) + + if len(descList) != originalDescCount: + with LOCK: + if filename not in MISMATCH: + MISMATCH.append(filename) + + if len(commandNameList) != originalCommandNameCount: + with LOCK: + if filename not in MISMATCH: + MISMATCH.append(filename) + + if len(commandList) != originalCommandCount: + with LOCK: + if filename not in MISMATCH: + MISMATCH.append(filename) + + if len(pagesList) != originalPagesCount: + with LOCK: + if filename not in MISMATCH: + MISMATCH.append(filename) + if len(msgList) != originalMsgCount: + with LOCK: + if filename not in MISMATCH: + MISMATCH.append(filename) + if len(commandArrayList) != originalCommandArrayCount: + with LOCK: + if filename not in MISMATCH: + MISMATCH.append(filename) + if len(rewardDataList) != originalRewardDataCount: + with LOCK: + if filename not in MISMATCH: + MISMATCH.append(filename) + if len(customParametersNameList) != originalCustomParametersNameCount: + with LOCK: + if filename not in MISMATCH: + MISMATCH.append(filename) + + # PASS 2: Recursively call to apply translations + translateGeneric( + data, + filename, + [nameList, descList, commandNameList, commandList, pagesList, msgList, commandArrayList, rewardDataList, customParametersNameList], + pbar, + ) + + return totalTokens + + +def parseTitles(data, filename): + """Parser for titles.json (small dict of title strings).""" + global PBAR + totalTokens = [0, 0] + pbar = None + try: + # Collect fields to translate + keys = [k for k in ["windowTitle", "gameTitle", "saveFileTitle"] if k in data and data[k]] + values = [data[k] for k in keys] + + with LOCK: + pbar = tqdm( + desc=filename, + total=len(values), + bar_format=BAR_FORMAT, + position=POSITION, + leave=LEAVE, + ) + PBAR = pbar + + if values: + response = translateAI( + values, + "Reply with only the " + LANGUAGE + " translation of the title text.", + True, + filename, + pbar, + ) + translations = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + + # Apply + for i, k in enumerate(keys): + data[k] = translations[i] + + return (data, totalTokens, None) + except Exception as e: + traceback.print_exc() + return (data, totalTokens, e) + finally: + try: + if pbar is not None: + pbar.close() + except Exception: + pass + PBAR = None + + +def parseRecollection(data, filename): + """ + Parser for SRPG Studio recollection.json files. + Structure: List of entries, each with pages containing commands with data arrays and speakers. + + Args: + data: Parsed JSON data (list of objects with id, pages structure) + filename: Name of the file being parsed + + Returns: + Tuple of (data, token counts, error) + """ + global PBAR + + totalTokens = [0, 0] + + pbar = None + try: + # Count work units (data entries and speakers that need translation) + total_units = 0 + + # Iterate through structure and count + for entry in data: + if not entry: + continue + + # Count name and desc fields at entry level + for field in ["name", "desc"]: + if field in entry and entry[field]: + total_units += 1 + + # Count customParameters hints + if "customParameters" in entry and entry["customParameters"]: + match = re.search(r'hint:"((?:[^"\\]|\\.)*)"', entry["customParameters"]) + if match: + total_units += 1 + + if "pages" not in entry or not isinstance(entry["pages"], list): + continue + + for page in entry["pages"]: + if not page or "commands" not in page or not isinstance(page["commands"], list): + continue + + for command in page["commands"]: + if not command: + continue + + # Count data array items (count any non-empty text; do not gate on LANGREGEX) + if "data" in command and isinstance(command["data"], list): + for text in command["data"]: + if text: + total_units += 1 + + # Count speaker field (count any non-empty speaker; do not gate on LANGREGEX) + if "speaker" in command and command["speaker"]: + total_units += 1 + + # Setup progress bar (per-file instance) + with LOCK: + pbar = tqdm( + desc=filename, + total=total_units, + bar_format=BAR_FORMAT, + position=POSITION, + leave=LEAVE, + ) + PBAR = pbar + + # Translate the data using two-pass approach + result = translateRecollection(data, filename, pbar=pbar) + totalTokens[0] += result[0] + totalTokens[1] += result[1] + + return (data, totalTokens, None) + + except Exception as e: + traceback.print_exc() + return (data, totalTokens, e) + finally: + try: + if pbar is not None: + pbar.close() + except Exception: + pass + PBAR = None + + +def translateRecollection(data, filename, translatedDataList=None, pbar=None): + """ + Translates recollection.json data structure. + Uses two-pass approach via recursion: + - Pass 1 (translatedDataList=None): Collect strings and batch translate + - Pass 2 (translatedDataList set): Apply translations back to the data + + Args: + data: List of objects with name, desc, customParameters, and pages->commands->data structure + filename: Name of the file being translated + translatedDataList: List containing [nameList, descList, dataList, speakerList, customParamsList] + - Pass 1: Empty lists to collect originals + - Pass 2: Filled lists with translations + + Returns: + Tuple of [input tokens, output tokens] + """ + global PBAR + + totalTokens = [0, 0] + + # Initialize or extract lists + if translatedDataList is None: + # PASS 1: Create empty lists to collect strings + nameList = [] + descList = [] + dataList = [] + speakerList = [] + customParamsList = [] + originalSpeakerList = [] # For vocab update + else: + # PASS 2: Use provided translated lists + nameList = translatedDataList[0] + descList = translatedDataList[1] + dataList = translatedDataList[2] + speakerList = translatedDataList[3] + customParamsList = translatedDataList[4] + + # Single loop - behavior depends on which pass we're in + for entry in data: + if not entry: + continue + + # Handle name field + if "name" in entry and entry["name"]: + # PASS 1: Collect original + if translatedDataList is None: + nameList.append(entry["name"]) + # PASS 2: Apply translation + else: + if nameList: + entry["name"] = nameList[0] + nameList.pop(0) + + # Handle desc field + if "desc" in entry and entry["desc"]: + # PASS 1: Collect original + if translatedDataList is None: + # Remove newlines for translation + descList.append(entry["desc"].replace("\n", " ")) + # PASS 2: Apply translation + else: + if descList: + translatedText = descList[0] + # Apply text wrapping + translatedText = dazedwrap.wrapText(translatedText, width=WIDTH) + entry["desc"] = translatedText + descList.pop(0) + + # Handle customParameters at entry level + if "customParameters" in entry and entry["customParameters"]: + # PASS 1: Collect hint text + if translatedDataList is None: + # Extract hint value using regex (captures content between hint:" and ") + match = re.search(r'hint:"((?:[^"\\]|\\.)*)"', entry["customParameters"]) + if match: + hintText = match.group(1) + hintText = hintText.replace("\\n", " ") + customParamsList.append(hintText) + else: + # No hint found, add empty string as placeholder + customParamsList.append("") + # PASS 2: Apply translation + else: + if customParamsList: + translatedHint = customParamsList[0] + customParamsList.pop(0) + + if translatedHint: # Only replace if we translated something + # Replace double quotes with single quotes since \" is used as delimiter + translatedHint = translatedHint.replace('"', "'") + # Wrap text using dazedwrap with WIDTH and replace newlines with \\n + translatedHint = dazedwrap.wrapText(translatedHint, width=WIDTH) + translatedHint = translatedHint.replace("\n", "\\\\n") + # Replace the hint value in customParameters + entry["customParameters"] = re.sub( + r'(hint:")((?:[^"\\]|\\.)*)"', + r'\1' + translatedHint + '"', + entry["customParameters"] + ) + + # Check if pages exists before processing + if "pages" not in entry or not isinstance(entry["pages"], list): + continue + + for page in entry["pages"]: + if not page or "commands" not in page: + continue + + if not isinstance(page["commands"], list): + continue + + for command in page["commands"]: + if not command: + continue + + # Get the speaker for this command (if available) + speaker = command.get("speaker", "") + + # Handle data array + if "data" in command and isinstance(command["data"], list): + for i, text in enumerate(command["data"]): + if text: + # PASS 1: Collect original with speaker prefix + if translatedDataList is None: + # Remove Wrap + text = text.replace("\n", " ") + + # Attach speaker to data for translation + if speaker: + dataList.append(f"[{speaker}]: {text}") + else: + dataList.append(text) + + # PASS 2: Apply translation and strip speaker prefix + else: + if dataList: + translated = dataList[0] + + # Remove speaker + if speaker: + match = re.search(r'(^\[.+?\]\s?[|:]\s?)', translated) + if match: + translated = translated.replace(match.group(1), "") + + # Textwrap + translated = dazedwrap.wrapText(translated, width=WIDTH) + + # Set Data + command["data"][i] = translated + dataList.pop(0) + + # Handle speaker field + if "speaker" in command and command["speaker"]: + # PASS 1: Collect original + if translatedDataList is None: + originalSpeakerList.append(command["speaker"]) + speakerList.append(command["speaker"]) + # PASS 2: Apply translation + else: + if speakerList: + command["speaker"] = speakerList[0] + speakerList.pop(0) + + # If this was Pass 1, do the translation and recurse for Pass 2 + if translatedDataList is None: + # Store original counts for mismatch checking + originalNameCount = len(nameList) + originalDescCount = len(descList) + originalDataCount = len(dataList) + originalSpeakerCount = len(speakerList) + originalCustomParamsCount = len(customParamsList) + + # Batch translate names + if nameList: + response = translateAI( + nameList, + "Reply with only the " + LANGUAGE + " translation of the name.", + True, + filename, + pbar + ) + nameList = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + + # Batch translate descriptions + if descList: + response = translateAI( + descList, + "Reply with only the " + LANGUAGE + " translation of the description.", + True, + filename, + pbar + ) + descList = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + + # Batch translate data text + if dataList: + response = translateAI( + dataList, + "Reply with only the " + LANGUAGE + " translation of the dialogue text.", + True, + filename, + pbar + ) + dataList = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + + # Batch translate speakers + if speakerList: + response = translateAI( + speakerList, + "Reply with only the " + LANGUAGE + " translation of the speaker name.", + True, + filename, + pbar + ) + speakerList = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + + # Batch translate customParameters hints (only non-empty ones) + if customParamsList: + # Filter out empty strings for translation + hintsToTranslate = [h for h in customParamsList if h] + if hintsToTranslate: + response = translateAI( + hintsToTranslate, + "Reply with only the " + LANGUAGE + " translation of the hint text.", + True, + filename, + pbar + ) + translatedHints = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + + # Reconstruct customParamsList with translations in place + translatedIndex = 0 + for i, hint in enumerate(customParamsList): + if hint: # If it was non-empty, use the translation + customParamsList[i] = translatedHints[translatedIndex] + translatedIndex += 1 + + # Check for mismatch errors + if len(nameList) != originalNameCount: + with LOCK: + if filename not in MISMATCH: + MISMATCH.append(filename) + + if len(descList) != originalDescCount: + with LOCK: + if filename not in MISMATCH: + MISMATCH.append(filename) + + if len(dataList) != originalDataCount: + with LOCK: + if filename not in MISMATCH: + MISMATCH.append(filename) + + if len(speakerList) != originalSpeakerCount: + with LOCK: + if filename not in MISMATCH: + MISMATCH.append(filename) + + if len(customParamsList) != originalCustomParamsCount: + with LOCK: + if filename not in MISMATCH: + MISMATCH.append(filename) + + # PASS 2: Recursively call to apply translations + translateRecollection(data, filename, [nameList, descList, dataList, speakerList, customParamsList], pbar) + + return totalTokens + + +def parseMap(data, filename): + """ + Parser for SRPG Studio map.json files. + Structure: Single object with: + - id, name, desc, mapName + - victoryConds (array of strings) + - defeatConds (array of strings) + - EnemyUnits (array of objects with id, name, desc, events structure similar to recollection) + + Args: + data: Parsed JSON data (single map object) + filename: Name of the file being parsed + + Returns: + Tuple of (data, token counts, error) + """ + global PBAR + + totalTokens = [0, 0] + + pbar = None + try: + # Count work units (all translatable fields) + total_units = 0 + + # Count top-level fields: desc, mapName (name is an identifier and should not be translated) + for field in ["desc", "mapName"]: + if field in data and data[field]: + total_units += 1 + + # Count victoryConds array items + if "victoryConds" in data and isinstance(data["victoryConds"], list): + for cond in data["victoryConds"]: + if cond: + total_units += 1 + + # Count defeatConds array items + if "defeatConds" in data and isinstance(data["defeatConds"], list): + for cond in data["defeatConds"]: + if cond: + total_units += 1 + + # Count units with events (EnemyUnits, EvEnemyUnits) + for unitsKey in ["EnemyUnits", "EvEnemyUnits"]: + if unitsKey in data and isinstance(data[unitsKey], list): + for unit in data[unitsKey]: + if not unit: + continue + + # Count unit name and desc + for field in ["name", "desc"]: + if field in unit and unit[field]: + total_units += 1 + + # Count events->pages->commands->data and speaker + if "events" in unit and isinstance(unit["events"], list): + for event in unit["events"]: + if not event or "pages" not in event: + continue + + if not isinstance(event["pages"], list): + continue + + for page in event["pages"]: + if not page or "commands" not in page: + continue + + if not isinstance(page["commands"], list): + continue + + for command in page["commands"]: + if not command: + continue + + # Count data array items + if "data" in command and isinstance(command["data"], list): + for text in command["data"]: + if text: + total_units += 1 + + # Count speaker field + if "speaker" in command and command["speaker"]: + total_units += 1 + + # Count events (placeEvents, autoEvents, openingEvents, communicationEvents) + for eventsKey in ["placeEvents", "autoEvents", "openingEvents", "communicationEvents"]: + if eventsKey in data and isinstance(data[eventsKey], list): + for event in data[eventsKey]: + if not event: + continue + + # Count event-level name and desc fields + for field in ["name", "desc"]: + if field in event and event[field]: + total_units += 1 + + if "pages" not in event or not isinstance(event["pages"], list): + continue + + for page in event["pages"]: + if not page or "commands" not in page: + continue + + if not isinstance(page["commands"], list): + continue + + for command in page["commands"]: + if not command: + continue + + # Count data array items + if "data" in command and isinstance(command["data"], list): + for text in command["data"]: + if text: + total_units += 1 + + # Count speaker field + if "speaker" in command and command["speaker"]: + total_units += 1 + + # Setup progress bar (per-file instance) + with LOCK: + pbar = tqdm( + desc=filename, + total=total_units, + bar_format=BAR_FORMAT, + position=POSITION, + leave=LEAVE, + ) + PBAR = pbar + + # Translate the data using two-pass approach + result = translateMap(data, filename, pbar=pbar) + totalTokens[0] += result[0] + totalTokens[1] += result[1] + + return (data, totalTokens, None) + + except Exception as e: + traceback.print_exc() + return (data, totalTokens, e) + finally: + try: + if pbar is not None: + pbar.close() + except Exception: + pass + PBAR = None + + +def translateMap(data, filename, translatedDataList=None, pbar=None): + """ + Translates map.json data structure. + Uses two-pass approach via recursion: + - Pass 1 (translatedDataList=None): Collect strings and batch translate + - Pass 2 (translatedDataList set): Apply translations back to the data + + Args: + data: Single map object with name, desc, mapName, victoryConds, defeatConds, and EnemyUnits + filename: Name of the file being translated + translatedDataList: List containing translation lists + - Pass 1: Empty lists to collect originals + - Pass 2: Filled lists with translations + + Returns: + Tuple of [input tokens, output tokens] + """ + global PBAR + + totalTokens = [0, 0] + + # Initialize or extract lists + if translatedDataList is None: + # PASS 1: Create empty lists to collect strings + descList = [] + mapNameList = [] + victoryCondsList = [] + defeatCondsList = [] + unitNameList = [] + unitDescList = [] + eventNameList = [] + eventDescList = [] + dataList = [] + speakerList = [] + originalSpeakerList = [] # For vocab update + else: + # PASS 2: Use provided translated lists + descList = translatedDataList[0] + mapNameList = translatedDataList[1] + victoryCondsList = translatedDataList[2] + defeatCondsList = translatedDataList[3] + unitNameList = translatedDataList[4] + unitDescList = translatedDataList[5] + eventNameList = translatedDataList[6] + eventDescList = translatedDataList[7] + dataList = translatedDataList[8] + speakerList = translatedDataList[9] + + # Note: name field is not translated as it's an identifier (e.g., "ch3_瘴気の森") + + # Handle desc field + if "desc" in data and data["desc"]: + # PASS 1: Collect original + if translatedDataList is None: + descList.append(data["desc"].replace("\n", " ")) + # PASS 2: Apply translation + else: + if descList: + translatedText = descList[0] + # Wordwrap + translatedText = dazedwrap.wrapText(translatedText, width=WIDTH) + data["desc"] = translatedText + descList.pop(0) + + # Handle mapName field + if "mapName" in data and data["mapName"]: + # PASS 1: Collect original + if translatedDataList is None: + mapNameList.append(data["mapName"]) + # PASS 2: Apply translation + else: + if mapNameList: + data["mapName"] = mapNameList[0] + mapNameList.pop(0) + + # Handle victoryConds array + if "victoryConds" in data and isinstance(data["victoryConds"], list): + for i, cond in enumerate(data["victoryConds"]): + if cond: + # PASS 1: Collect original + if translatedDataList is None: + victoryCondsList.append(cond) + # PASS 2: Apply translation + else: + if victoryCondsList: + data["victoryConds"][i] = victoryCondsList[0] + victoryCondsList.pop(0) + + # Handle defeatConds array + if "defeatConds" in data and isinstance(data["defeatConds"], list): + for i, cond in enumerate(data["defeatConds"]): + if cond: + # PASS 1: Collect original + if translatedDataList is None: + defeatCondsList.append(cond) + # PASS 2: Apply translation + else: + if defeatCondsList: + data["defeatConds"][i] = defeatCondsList[0] + defeatCondsList.pop(0) + + # Process all unit arrays (EnemyUnits, EvEnemyUnits) + for unitsKey in ["EnemyUnits", "EvEnemyUnits"]: + if unitsKey in data and isinstance(data[unitsKey], list): + for unit in data[unitsKey]: + if not unit: + continue + + # Handle unit name + if "name" in unit and unit["name"]: + if translatedDataList is None: + unitNameList.append(unit["name"]) + else: + if unitNameList: + unit["name"] = unitNameList[0] + unitNameList.pop(0) + + # Handle unit desc + if "desc" in unit and unit["desc"]: + if translatedDataList is None: + unitDescList.append(unit["desc"].replace("\n", " ")) + else: + if unitDescList: + translatedText = unitDescList[0] + # Wordwrap + translatedText = dazedwrap.wrapText(translatedText, width=WIDTH) + unit["desc"] = translatedText + unitDescList.pop(0) + + # Process events within units + if "events" in unit and isinstance(unit["events"], list): + for event in unit["events"]: + if not event or "pages" not in event: + continue + + if not isinstance(event["pages"], list): + continue + + for page in event["pages"]: + if not page or "commands" not in page: + continue + + if not isinstance(page["commands"], list): + continue + + for command in page["commands"]: + if not command: + continue + + speaker = command.get("speaker", "") + + # Handle data array + if "data" in command and isinstance(command["data"], list): + for i, text in enumerate(command["data"]): + if text: + if translatedDataList is None: + text = text.replace("\n", " ") + if speaker: + dataList.append(f"[{speaker}]: {text}") + else: + dataList.append(text) + else: + if dataList: + translated = dataList[0] + if speaker: + match = re.search(r'(^\[.+?\]\s?[|:]\s?)', translated) + if match: + translated = translated.replace(match.group(1), "") + translated = dazedwrap.wrapText(translated, width=WIDTH) + command["data"][i] = translated + dataList.pop(0) + + # Handle speaker field + if "speaker" in command and command["speaker"]: + if translatedDataList is None: + originalSpeakerList.append(command["speaker"]) + speakerList.append(command["speaker"]) + else: + if speakerList: + command["speaker"] = speakerList[0] + speakerList.pop(0) + + # Process all event arrays (placeEvents, autoEvents, openingEvents, communicationEvents) + for eventsKey in ["placeEvents", "autoEvents", "openingEvents", "communicationEvents"]: + if eventsKey in data and isinstance(data[eventsKey], list): + for event in data[eventsKey]: + if not event: + continue + + # Handle event-level name field + if "name" in event and event["name"]: + if translatedDataList is None: + eventNameList.append(event["name"]) + else: + if eventNameList: + event["name"] = eventNameList[0] + eventNameList.pop(0) + + # Handle event-level desc field + if "desc" in event and event["desc"]: + if translatedDataList is None: + # Remove newlines for translation + eventDescList.append(event["desc"].replace("\n", " ")) + else: + if eventDescList: + translatedText = eventDescList[0] + # Apply text wrapping + translatedText = dazedwrap.wrapText(translatedText, width=WIDTH) + event["desc"] = translatedText + eventDescList.pop(0) + + if "pages" not in event or not isinstance(event["pages"], list): + continue + + for page in event["pages"]: + if not page or "commands" not in page: + continue + + if not isinstance(page["commands"], list): + continue + + for command in page["commands"]: + if not command: + continue + + speaker = command.get("speaker", "") + + # Handle data array + if "data" in command and isinstance(command["data"], list): + for i, text in enumerate(command["data"]): + if text: + if translatedDataList is None: + text = text.replace("\n", " ") + if speaker: + dataList.append(f"[{speaker}]: {text}") + else: + dataList.append(text) + else: + if dataList: + translated = dataList[0] + if speaker: + match = re.search(r'(^\[.+?\]\s?[|:]\s?)', translated) + if match: + translated = translated.replace(match.group(1), "") + translated = dazedwrap.wrapText(translated, width=WIDTH) + command["data"][i] = translated + dataList.pop(0) + + # Handle speaker field + if "speaker" in command and command["speaker"]: + if translatedDataList is None: + originalSpeakerList.append(command["speaker"]) + speakerList.append(command["speaker"]) + else: + if speakerList: + command["speaker"] = speakerList[0] + speakerList.pop(0) + + # If this was Pass 1, do the translation and recurse for Pass 2 + if translatedDataList is None: + # Store original counts for mismatch checking + originalDescCount = len(descList) + originalMapNameCount = len(mapNameList) + originalVictoryCondsCount = len(victoryCondsList) + originalDefeatCondsCount = len(defeatCondsList) + originalUnitNameCount = len(unitNameList) + originalUnitDescCount = len(unitDescList) + originalEventNameCount = len(eventNameList) + originalEventDescCount = len(eventDescList) + originalDataCount = len(dataList) + originalSpeakerCount = len(speakerList) + + # Batch translate map descriptions + if descList: + response = translateAI( + descList, + "Reply with only the " + LANGUAGE + " translation of the map description.", + True, + filename, + pbar + ) + descList = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + + # Batch translate map display names + if mapNameList: + response = translateAI( + mapNameList, + "Reply with only the " + LANGUAGE + " translation of the map display name.", + True, + filename, + pbar + ) + mapNameList = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + + # Batch translate victory conditions + if victoryCondsList: + response = translateAI( + victoryCondsList, + "Reply with only the " + LANGUAGE + " translation of the victory condition.", + True, + filename, + pbar + ) + victoryCondsList = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + + # Batch translate defeat conditions + if defeatCondsList: + response = translateAI( + defeatCondsList, + "Reply with only the " + LANGUAGE + " translation of the defeat condition.", + True, + filename, + pbar + ) + defeatCondsList = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + + # Batch translate unit names + if unitNameList: + response = translateAI( + unitNameList, + "Reply with only the " + LANGUAGE + " translation of the enemy unit name.", + True, + filename, + pbar + ) + unitNameList = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + + # Batch translate unit descriptions + if unitDescList: + response = translateAI( + unitDescList, + "Reply with only the " + LANGUAGE + " translation of the enemy unit description.", + True, + filename, + pbar + ) + unitDescList = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + + # Batch translate event names + if eventNameList: + response = translateAI( + eventNameList, + "Reply with only the " + LANGUAGE + " translation of the event name.", + True, + filename, + pbar + ) + eventNameList = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + + # Batch translate event descriptions + if eventDescList: + response = translateAI( + eventDescList, + "Reply with only the " + LANGUAGE + " translation of the event description.", + True, + filename, + pbar + ) + eventDescList = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + + # Batch translate dialogue data + if dataList: + response = translateAI( + dataList, + "Reply with only the " + LANGUAGE + " translation of the dialogue text.", + True, + filename, + pbar + ) + dataList = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + + # Batch translate speakers + if speakerList: + response = translateAI( + speakerList, + "Reply with only the " + LANGUAGE + " translation of the speaker name.", + True, + filename, + pbar + ) + speakerList = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + + # Check for mismatch errors + if len(descList) != originalDescCount: + with LOCK: + if filename not in MISMATCH: + MISMATCH.append(filename) + + if len(mapNameList) != originalMapNameCount: + with LOCK: + if filename not in MISMATCH: + MISMATCH.append(filename) + + if len(victoryCondsList) != originalVictoryCondsCount: + with LOCK: + if filename not in MISMATCH: + MISMATCH.append(filename) + + if len(defeatCondsList) != originalDefeatCondsCount: + with LOCK: + if filename not in MISMATCH: + MISMATCH.append(filename) + + if len(unitNameList) != originalUnitNameCount: + with LOCK: + if filename not in MISMATCH: + MISMATCH.append(filename) + + if len(unitDescList) != originalUnitDescCount: + with LOCK: + if filename not in MISMATCH: + MISMATCH.append(filename) + + if len(eventNameList) != originalEventNameCount: + with LOCK: + if filename not in MISMATCH: + MISMATCH.append(filename) + + if len(eventDescList) != originalEventDescCount: + with LOCK: + if filename not in MISMATCH: + MISMATCH.append(filename) + + if len(dataList) != originalDataCount: + with LOCK: + if filename not in MISMATCH: + MISMATCH.append(filename) + + if len(speakerList) != originalSpeakerCount: + with LOCK: + if filename not in MISMATCH: + MISMATCH.append(filename) + + # PASS 2: Recursively call to apply translations + translateMap(data, filename, [descList, mapNameList, victoryCondsList, defeatCondsList, unitNameList, unitDescList, eventNameList, eventDescList, dataList, speakerList], pbar) + + return totalTokens + + +def getResultString(translatedData, translationTime, filename): + """ + Formats the translation result string with token counts, cost, and time. + + Args: + translatedData: Tuple of (data, tokens, error) + translationTime: Time taken for translation + filename: Name of the file + + Returns: + Formatted result string + """ + global TIMETOTAL + + # Calculate cost + cost = calculateCost(translatedData[1][0], translatedData[1][1], MODEL) + totalTokenstring = ( + Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]" + "[Output: " + + str(translatedData[1][1]) + + "]" "[Cost: ${:,.4f}".format(cost) + + "]" + ) + + # Format time string + if filename != "TOTAL": + timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]" + TIMETOTAL += round(translationTime, 1) + else: + timeString = Fore.BLUE + "[" + str(round(TIMETOTAL, 1)) + "s]" + + # Return success or failure string + if translatedData[2] is None: + # Success + return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET + else: + # Fail + try: + raise translatedData[2] + except Exception as e: + traceback.print_exc() + return ( + filename + + ": " + + totalTokenstring + + timeString + + Fore.RED + + " \u2717 " + + Fore.RESET + ) + + +def getSpeaker(speaker): + """ + Translates speaker/character names with caching to avoid redundant translations. + + Args: + speaker: The original speaker name to translate + + Returns: + List containing [translated name, [input tokens, output tokens]] + """ + if speaker == "": + return ["", [0, 0]] + + # Check if speaker has already been translated + for i in range(len(NAMESLIST)): + if speaker == NAMESLIST[i][0]: + return [NAMESLIST[i][1], [0, 0]] + + # Translate and Store Speaker + response = translateAI( + speaker, + "Reply with the " + LANGUAGE + " translation of the NPC name.", + False, + ) + response[0] = response[0].title() + response[0] = response[0].replace("'S", "'s") + response[0] = response[0].replace("Speaker: ", "") + + # Retry if name doesn't translate for some reason + if re.search(r"([a-zA-Z？?])", response[0]) is None: + response = translateAI( + speaker, + "Reply with the " + LANGUAGE + " translation of the NPC name.", + False, + ) + response[0] = response[0].title() + response[0] = response[0].replace("'S", "'s") + + speakerList = [speaker, response[0]] + NAMESLIST.append(speakerList) + return response + + +def translateAI(text, history, history_ctx=None, filename=None, pbar=None): + """ + Legacy wrapper function for the new shared translation utility. + This maintains compatibility with existing code while using the new shared implementation. + + Args: + text: Text to translate (can be string or list) + history: History/context for the translation + + Returns: + List containing [translated text, [input tokens, output tokens]] + """ + global PBAR, MISMATCH, FILENAME + + # Update config estimate mode based on global ESTIMATE + TRANSLATION_CONFIG.estimateMode = bool(ESTIMATE) + + # Call the new shared translation function + return sharedtranslateAI( + text=text, + history=history, + config=TRANSLATION_CONFIG, + filename=filename if filename is not None else FILENAME, + pbar=pbar if pbar is not None else PBAR, + lock=LOCK, + mismatchList=MISMATCH + ) + diff --git a/modules/text.py b/modules/text.py index d257d64..e4cd2ff 100644 --- a/modules/text.py +++ b/modules/text.py @@ -1,363 +1,365 @@ -# Libraries -import json -import os -import re -import util.dazedwrap as dazedwrap -import threading -import time -import traceback -import tiktoken -from pathlib import Path -from colorama import Fore -from dotenv import load_dotenv -from retry import retry -from tqdm import tqdm -from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost, getPricingConfig, calculateCost -import tempfile - -# Globals -MODEL = os.getenv("model") -TIMEOUT = int(os.getenv("timeout")) -LANGUAGE = os.getenv("language").capitalize() -PROMPT = Path("prompt.txt").read_text(encoding="utf-8") -VOCAB = Path("vocab.txt").read_text(encoding="utf-8") -LOCK = threading.Lock() -WIDTH = int(os.getenv("width")) -LISTWIDTH = int(os.getenv("listWidth")) -NOTEWIDTH = 70 -MAXHISTORY = 10 -ESTIMATE = "" -TOKENS = [0, 0] -NAMESLIST = [] -NAMES = False # Output a list of all the character names found -BRFLAG = False # If the game uses
instead -FIXTEXTWRAP = True # Overwrites textwrap -IGNORETLTEXT = False # Ignores all translated text. -MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong) -FILENAME = None - -# tqdm Globals -BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}" -POSITION = 0 -LEAVE = False -PBAR = None - -# Regex - Need to change this if you want to translate from/to other languages. Default is Japanese Regex -LANGREGEX = r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+" - -# Get pricing configuration based on the model -PRICING_CONFIG = getPricingConfig(MODEL) -INPUTAPICOST = PRICING_CONFIG["inputAPICost"] -OUTPUTAPICOST = PRICING_CONFIG["outputAPICost"] -BATCHSIZE = PRICING_CONFIG["batchSize"] -FREQUENCY_PENALTY = PRICING_CONFIG["frequencyPenalty"] - -# Initialize Translation Config -TRANSLATION_CONFIG = TranslationConfig( - model=MODEL, - language=LANGUAGE, - prompt=PROMPT, - vocab=VOCAB, - langRegex=LANGREGEX, - batchSize=BATCHSIZE, - maxHistory=MAXHISTORY, - estimateMode=False # Will be set dynamically based on ESTIMATE -) -LEAVE = False - -def handleText(filename, estimate): - global ESTIMATE, TOKENS, FILENAME - ESTIMATE = estimate - FILENAME = filename - - # Translate - start = time.time() - translatedData = openFiles(filename) - - # Translate - if not estimate: - try: - with open("translated/" + filename, "w", encoding="utf-8", newline="\n") as outFile: - outFile.writelines(translatedData[0]) - except Exception: - traceback.print_exc() - return "Fail" - - # Print File - end = time.time() - tqdm.write(getResultString(translatedData, end - start, filename)) - with LOCK: - TOKENS[0] += translatedData[1][0] - TOKENS[1] += translatedData[1][1] - - # Print Total - totalString = getResultString(["", TOKENS, None], end - start, "TOTAL") - - # Print any errors on maps - if len(MISMATCH) > 0: - return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET - else: - return totalString - - -def getResultString(translatedData, translationTime, filename): - # File Print String - cost = calculateCost(translatedData[1][0], translatedData[1][1], MODEL) - totalTokenstring = ( - Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]" - "[Output: " - + str(translatedData[1][1]) - + "]" "[Cost: ${:,.4f}".format(cost) - + "]" - ) - timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]" - - if translatedData[2] == None: - # Success - return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET - - else: - # Fail - try: - raise translatedData[2] - except Exception as e: - traceback.print_exc() - errorString = str(e) + Fore.RED - return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET - - -def openFiles(filename): - with open("files/" + filename, "r", encoding="utf8") as readFile: - translatedData = parseText(readFile, filename) - - # Delete lines marked for deletion - finalData = [] - for line in translatedData[0]: - if line != "\\d\n": - finalData.append(line) - translatedData[0] = finalData - - return translatedData - - -def parseText(readFile, filename): - global PBAR - totalTokens = [0, 0] - - # Read File into data - data = readFile.readlines() - - # Create Progress Bar - with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar: - pbar.desc = filename - PBAR = pbar - - try: - result = translateTxt(data, filename, []) - totalTokens[0] += result[0] - totalTokens[1] += result[1] - except Exception as e: - traceback.print_exc() - return [data, totalTokens, e] - return [data, totalTokens, None] - - -def save_progress_lines(lines, filename, encoding="utf-8"): - """Atomically save current line-based translation progress.""" - try: - if ESTIMATE: - return - os.makedirs("translated", exist_ok=True) - tmp_fd, tmp_path = tempfile.mkstemp(prefix=f"{filename}.", suffix=".tmp", dir="translated") - try: - with os.fdopen(tmp_fd, "w", encoding=encoding, newline="\n", errors="ignore") as tmp_file: - tmp_file.writelines(lines) - os.replace(tmp_path, os.path.join("translated", filename)) - finally: - if os.path.exists(tmp_path): - try: - os.remove(tmp_path) - except OSError: - pass - except Exception: - traceback.print_exc() - - -def saveCheckLines(lines, filename, tokens=None, encoding="utf-8"): - """Save progress only when tokens indicate work or when explicitly called after a mutation.""" - try: - if tokens is not None: - if not (isinstance(tokens, (list, tuple)) and len(tokens) >= 2 and (tokens[0] or tokens[1])): - return - save_progress_lines(lines, filename, encoding=encoding) - except Exception: - traceback.print_exc() - - -def translateTxt(data, filename, translatedList): - if translatedList: - stringList = translatedList[0] - choiceList = translatedList[1] - else: - stringList = [] - choiceList = [] - tokens = [0, 0] - speaker = "" - global LOCK, ESTIMATE, FILENAME, PBAR, MISMATCH - i = 0 - - # Regex - lineTextRegex = r"(?:^\[.+?\]:)?(.+)" - speakerTextRegex = r"^\[(.+?)\]" - - while i < len(data): - # Speaker - match = re.search(speakerTextRegex, data[i]) - if match: - # Get Speaker - speakerData = getSpeaker(match.group(1)) - speaker = speakerData[0] - tokens[0] += speakerData[1][0] - tokens[1] += speakerData[1][1] - data[i] = data[i].replace(match.group(1), speaker) - - # Dialogue - match = re.search(lineTextRegex, data[i]) - jaString = None - if match: - # Set String - jaString = match.group(1) - - # Pass 1 - if not translatedList: - # Strip Spaces - jaString = jaString.strip() - - if jaString: - if speaker: - stringList.append(f"[{speaker}]: {jaString}") - else: - stringList.append(jaString) - - # Pass 2 - else: - # Get Text - if stringList: - # Grab and Pop - translatedText = stringList[0] - stringList.pop(0) - - # Set to None if empty list - if len(stringList) <= 0: - stringList = None - - # Remove speaker - translatedText = re.sub(r"(^\[.+?\]\s?[|:]\s?)", "", translatedText) - - # # Textwrap - # translatedText = dazedwrap.wrapText(translatedText, width=WIDTH) - # translatedText = translatedText.replace("\n", "\\n") - - # Set Data - data[i] = f"{translatedText}\n" - saveCheckLines(data, filename) - - i += 1 - else: - i += 1 - - # EOF - if not translatedList: - stringListTL = [] - choiceListTL = [] - - # String List - if stringList: - PBAR.total = len(stringList) - PBAR.refresh() - response = translateAI(stringList, "Reply with the English Translation") - tokens[0] += response[1][0] - tokens[1] += response[1][1] - stringListTL = response[0] - - if len(stringList) != len(stringListTL): - # Mismatch - with LOCK: - if FILENAME not in MISMATCH: - MISMATCH.append(FILENAME) - - # Choice List - if choiceList: - response = translateAI(choiceList, "Reply with the English TL of the Dialogue Choice") - tokens[0] += response[1][0] - tokens[1] += response[1][1] - choiceListTL = response[0] - - if len(choiceList) != len(choiceListTL): - # Mismatch - with LOCK: - if FILENAME not in MISMATCH: - MISMATCH.append(FILENAME) - - # Set Strings - translateTxt(data, filename, [stringListTL, choiceListTL]) - return tokens - -# Save some money and enter the character before translation -def getSpeaker(speaker): - match speaker: - case "ファイン": - return ["Fine", [0, 0]] - case "": - return ["", [0, 0]] - case _: - # Find Speaker - for i in range(len(NAMESLIST)): - if speaker == NAMESLIST[i][0]: - return [NAMESLIST[i][1], [0, 0]] - - # Translate and Store Speaker - response = translateAI( - f"{speaker}", - "Reply with the " + LANGUAGE + " translation of the NPC name.", - False, - ) - response[0] = response[0].title() - response[0] = response[0].replace("'S", "'s") - response[0] = response[0].replace("Speaker: ", "") - - # Retry if name doesn't translate for some reason - if re.search(r"([a-zA-Z？?])", response[0]) == None: - response = translateAI( - f"{speaker}", - "Reply with the " + LANGUAGE + " translation of the NPC name.", - False, - ) - response[0] = response[0].title() - response[0] = response[0].replace("'S", "'s") - - speakerList = [speaker, response[0]] - NAMESLIST.append(speakerList) - return response - return [speaker, [0, 0]] - -def translateAI(text, history, history_ctx=None): - """ - Legacy wrapper function for the new shared translation utility. - This maintains compatibility with existing code while using the new shared implementation. - """ - global PBAR, MISMATCH, FILENAME - - # Update config estimate mode based on global ESTIMATE - TRANSLATION_CONFIG.estimateMode = bool(ESTIMATE) - - # Call the new shared translation function - return sharedtranslateAI( - text=text, - history=history, - config=TRANSLATION_CONFIG, - filename=FILENAME, - pbar=PBAR, - lock=LOCK, - mismatchList=MISMATCH +# Libraries +import json +import os +import re +import util.dazedwrap as dazedwrap +import threading +import time +import traceback +import tiktoken +from pathlib import Path +from colorama import Fore +from dotenv import load_dotenv +from retry import retry +from tqdm import tqdm +from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost, getPricingConfig, calculateCost +import tempfile + +# Globals +MODEL = os.getenv("model") +TIMEOUT = int(os.getenv("timeout")) +LANGUAGE = os.getenv("language").capitalize() +from util.paths import PROMPT_PATH, VOCAB_PATH + +PROMPT = PROMPT_PATH.read_text(encoding="utf-8") +VOCAB = VOCAB_PATH.read_text(encoding="utf-8") +LOCK = threading.Lock() +WIDTH = int(os.getenv("width")) +LISTWIDTH = int(os.getenv("listWidth")) +NOTEWIDTH = 70 +MAXHISTORY = 10 +ESTIMATE = "" +TOKENS = [0, 0] +NAMESLIST = [] +NAMES = False # Output a list of all the character names found +BRFLAG = False # If the game uses
instead +FIXTEXTWRAP = True # Overwrites textwrap +IGNORETLTEXT = False # Ignores all translated text. +MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong) +FILENAME = None + +# tqdm Globals +BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}" +POSITION = 0 +LEAVE = False +PBAR = None + +# Regex - Need to change this if you want to translate from/to other languages. Default is Japanese Regex +LANGREGEX = r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+" + +# Get pricing configuration based on the model +PRICING_CONFIG = getPricingConfig(MODEL) +INPUTAPICOST = PRICING_CONFIG["inputAPICost"] +OUTPUTAPICOST = PRICING_CONFIG["outputAPICost"] +BATCHSIZE = PRICING_CONFIG["batchSize"] +FREQUENCY_PENALTY = PRICING_CONFIG["frequencyPenalty"] + +# Initialize Translation Config +TRANSLATION_CONFIG = TranslationConfig( + model=MODEL, + language=LANGUAGE, + prompt=PROMPT, + vocab=VOCAB, + langRegex=LANGREGEX, + batchSize=BATCHSIZE, + maxHistory=MAXHISTORY, + estimateMode=False # Will be set dynamically based on ESTIMATE +) +LEAVE = False + +def handleText(filename, estimate): + global ESTIMATE, TOKENS, FILENAME + ESTIMATE = estimate + FILENAME = filename + + # Translate + start = time.time() + translatedData = openFiles(filename) + + # Translate + if not estimate: + try: + with open("translated/" + filename, "w", encoding="utf-8", newline="\n") as outFile: + outFile.writelines(translatedData[0]) + except Exception: + traceback.print_exc() + return "Fail" + + # Print File + end = time.time() + tqdm.write(getResultString(translatedData, end - start, filename)) + with LOCK: + TOKENS[0] += translatedData[1][0] + TOKENS[1] += translatedData[1][1] + + # Print Total + totalString = getResultString(["", TOKENS, None], end - start, "TOTAL") + + # Print any errors on maps + if len(MISMATCH) > 0: + return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET + else: + return totalString + + +def getResultString(translatedData, translationTime, filename): + # File Print String + cost = calculateCost(translatedData[1][0], translatedData[1][1], MODEL) + totalTokenstring = ( + Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]" + "[Output: " + + str(translatedData[1][1]) + + "]" "[Cost: ${:,.4f}".format(cost) + + "]" + ) + timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]" + + if translatedData[2] == None: + # Success + return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET + + else: + # Fail + try: + raise translatedData[2] + except Exception as e: + traceback.print_exc() + errorString = str(e) + Fore.RED + return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET + + +def openFiles(filename): + with open("files/" + filename, "r", encoding="utf8") as readFile: + translatedData = parseText(readFile, filename) + + # Delete lines marked for deletion + finalData = [] + for line in translatedData[0]: + if line != "\\d\n": + finalData.append(line) + translatedData[0] = finalData + + return translatedData + + +def parseText(readFile, filename): + global PBAR + totalTokens = [0, 0] + + # Read File into data + data = readFile.readlines() + + # Create Progress Bar + with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar: + pbar.desc = filename + PBAR = pbar + + try: + result = translateTxt(data, filename, []) + totalTokens[0] += result[0] + totalTokens[1] += result[1] + except Exception as e: + traceback.print_exc() + return [data, totalTokens, e] + return [data, totalTokens, None] + + +def save_progress_lines(lines, filename, encoding="utf-8"): + """Atomically save current line-based translation progress.""" + try: + if ESTIMATE: + return + os.makedirs("translated", exist_ok=True) + tmp_fd, tmp_path = tempfile.mkstemp(prefix=f"{filename}.", suffix=".tmp", dir="translated") + try: + with os.fdopen(tmp_fd, "w", encoding=encoding, newline="\n", errors="ignore") as tmp_file: + tmp_file.writelines(lines) + os.replace(tmp_path, os.path.join("translated", filename)) + finally: + if os.path.exists(tmp_path): + try: + os.remove(tmp_path) + except OSError: + pass + except Exception: + traceback.print_exc() + + +def saveCheckLines(lines, filename, tokens=None, encoding="utf-8"): + """Save progress only when tokens indicate work or when explicitly called after a mutation.""" + try: + if tokens is not None: + if not (isinstance(tokens, (list, tuple)) and len(tokens) >= 2 and (tokens[0] or tokens[1])): + return + save_progress_lines(lines, filename, encoding=encoding) + except Exception: + traceback.print_exc() + + +def translateTxt(data, filename, translatedList): + if translatedList: + stringList = translatedList[0] + choiceList = translatedList[1] + else: + stringList = [] + choiceList = [] + tokens = [0, 0] + speaker = "" + global LOCK, ESTIMATE, FILENAME, PBAR, MISMATCH + i = 0 + + # Regex + lineTextRegex = r"(?:^\[.+?\]:)?(.+)" + speakerTextRegex = r"^\[(.+?)\]" + + while i < len(data): + # Speaker + match = re.search(speakerTextRegex, data[i]) + if match: + # Get Speaker + speakerData = getSpeaker(match.group(1)) + speaker = speakerData[0] + tokens[0] += speakerData[1][0] + tokens[1] += speakerData[1][1] + data[i] = data[i].replace(match.group(1), speaker) + + # Dialogue + match = re.search(lineTextRegex, data[i]) + jaString = None + if match: + # Set String + jaString = match.group(1) + + # Pass 1 + if not translatedList: + # Strip Spaces + jaString = jaString.strip() + + if jaString: + if speaker: + stringList.append(f"[{speaker}]: {jaString}") + else: + stringList.append(jaString) + + # Pass 2 + else: + # Get Text + if stringList: + # Grab and Pop + translatedText = stringList[0] + stringList.pop(0) + + # Set to None if empty list + if len(stringList) <= 0: + stringList = None + + # Remove speaker + translatedText = re.sub(r"(^\[.+?\]\s?[|:]\s?)", "", translatedText) + + # # Textwrap + # translatedText = dazedwrap.wrapText(translatedText, width=WIDTH) + # translatedText = translatedText.replace("\n", "\\n") + + # Set Data + data[i] = f"{translatedText}\n" + saveCheckLines(data, filename) + + i += 1 + else: + i += 1 + + # EOF + if not translatedList: + stringListTL = [] + choiceListTL = [] + + # String List + if stringList: + PBAR.total = len(stringList) + PBAR.refresh() + response = translateAI(stringList, "Reply with the English Translation") + tokens[0] += response[1][0] + tokens[1] += response[1][1] + stringListTL = response[0] + + if len(stringList) != len(stringListTL): + # Mismatch + with LOCK: + if FILENAME not in MISMATCH: + MISMATCH.append(FILENAME) + + # Choice List + if choiceList: + response = translateAI(choiceList, "Reply with the English TL of the Dialogue Choice") + tokens[0] += response[1][0] + tokens[1] += response[1][1] + choiceListTL = response[0] + + if len(choiceList) != len(choiceListTL): + # Mismatch + with LOCK: + if FILENAME not in MISMATCH: + MISMATCH.append(FILENAME) + + # Set Strings + translateTxt(data, filename, [stringListTL, choiceListTL]) + return tokens + +# Save some money and enter the character before translation +def getSpeaker(speaker): + match speaker: + case "ファイン": + return ["Fine", [0, 0]] + case "": + return ["", [0, 0]] + case _: + # Find Speaker + for i in range(len(NAMESLIST)): + if speaker == NAMESLIST[i][0]: + return [NAMESLIST[i][1], [0, 0]] + + # Translate and Store Speaker + response = translateAI( + f"{speaker}", + "Reply with the " + LANGUAGE + " translation of the NPC name.", + False, + ) + response[0] = response[0].title() + response[0] = response[0].replace("'S", "'s") + response[0] = response[0].replace("Speaker: ", "") + + # Retry if name doesn't translate for some reason + if re.search(r"([a-zA-Z？?])", response[0]) == None: + response = translateAI( + f"{speaker}", + "Reply with the " + LANGUAGE + " translation of the NPC name.", + False, + ) + response[0] = response[0].title() + response[0] = response[0].replace("'S", "'s") + + speakerList = [speaker, response[0]] + NAMESLIST.append(speakerList) + return response + return [speaker, [0, 0]] + +def translateAI(text, history, history_ctx=None): + """ + Legacy wrapper function for the new shared translation utility. + This maintains compatibility with existing code while using the new shared implementation. + """ + global PBAR, MISMATCH, FILENAME + + # Update config estimate mode based on global ESTIMATE + TRANSLATION_CONFIG.estimateMode = bool(ESTIMATE) + + # Call the new shared translation function + return sharedtranslateAI( + text=text, + history=history, + config=TRANSLATION_CONFIG, + filename=FILENAME, + pbar=PBAR, + lock=LOCK, + mismatchList=MISMATCH ) \ No newline at end of file diff --git a/modules/tyrano.py b/modules/tyrano.py index 733e312..afda46e 100644 --- a/modules/tyrano.py +++ b/modules/tyrano.py @@ -1,457 +1,459 @@ -# Libraries -import json -import os -import re -import util.dazedwrap as dazedwrap -import threading -import time -import traceback -import tiktoken -from pathlib import Path -from colorama import Fore -from dotenv import load_dotenv -from retry import retry -from tqdm import tqdm -from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost -import tempfile - -# OpenAI initialization centralized in util/translation.py - -# Globals -MODEL = os.getenv("model") -TIMEOUT = int(os.getenv("timeout")) -LANGUAGE = os.getenv("language").capitalize() -PROMPT = Path("prompt.txt").read_text(encoding="utf-8") -VOCAB = Path("vocab.txt").read_text(encoding="utf-8") -LOCK = threading.Lock() -WIDTH = int(os.getenv("width")) -LISTWIDTH = int(os.getenv("listWidth")) -NOTEWIDTH = 70 -MAXHISTORY = 10 -ESTIMATE = "" -TOKENS = [0, 0] -NAMESLIST = [] -NAMES = False # Output a list of all the character names found -BRFLAG = False # If the game uses
instead -FIXTEXTWRAP = True # Overwrites textwrap -IGNORETLTEXT = False # Ignores all translated text. -MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong) -FILENAME = None - -# tqdm Globals -BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}" -POSITION = 0 -LEAVE = False -PBAR = None - -# Regex - Need to change this if you want to translate from/to other languages. Default is Japanese Regex -LANGREGEX = r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+" - -# Get pricing configuration based on the model -PRICING_CONFIG = getPricingConfig(MODEL) -INPUTAPICOST = PRICING_CONFIG["inputAPICost"] -OUTPUTAPICOST = PRICING_CONFIG["outputAPICost"] -BATCHSIZE = PRICING_CONFIG["batchSize"] -FREQUENCY_PENALTY = PRICING_CONFIG["frequencyPenalty"] - -# Initialize Translation Config -TRANSLATION_CONFIG = TranslationConfig( - model=MODEL, - language=LANGUAGE, - prompt=PROMPT, - vocab=VOCAB, - langRegex=LANGREGEX, - batchSize=BATCHSIZE, - maxHistory=MAXHISTORY, - estimateMode=False # Will be set dynamically based on ESTIMATE -) -LEAVE = False - -def handleTyrano(filename, estimate): - global ESTIMATE, TOKENS, FILENAME - ESTIMATE = estimate - FILENAME = filename - - # Translate - start = time.time() - translatedData = openFiles(filename) - - # Translate - if not estimate: - try: - with open("translated/" + filename, "w", encoding="utf-8", newline="\n") as outFile: - outFile.writelines(translatedData[0]) - except Exception: - traceback.print_exc() - return "Fail" - - # Print File - end = time.time() - tqdm.write(getResultString(translatedData, end - start, filename)) - with LOCK: - TOKENS[0] += translatedData[1][0] - TOKENS[1] += translatedData[1][1] - - # Print Total - totalString = getResultString(["", TOKENS, None], end - start, "TOTAL") - - # Print any errors on maps - if len(MISMATCH) > 0: - return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET - else: - return totalString - - -def getResultString(translatedData, translationTime, filename): - # File Print String - cost = calculateCost(translatedData[1][0], translatedData[1][1], MODEL) - totalTokenstring = ( - Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]" - "[Output: " - + str(translatedData[1][1]) - + "]" "[Cost: ${:,.4f}".format(cost) - + "]" - ) - timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]" - - if translatedData[2] == None: - # Success - return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET - - else: - # Fail - try: - raise translatedData[2] - except Exception as e: - traceback.print_exc() - errorString = str(e) + Fore.RED - return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET - - -def openFiles(filename): - with open("files/" + filename, "r", encoding="utf8") as readFile: - translatedData = parseTyrano(readFile, filename) - - # Delete lines marked for deletion - finalData = [] - for line in translatedData[0]: - if line != "\\d\n": - finalData.append(line) - translatedData[0] = finalData - - return translatedData - - -def parseTyrano(readFile, filename): - global PBAR - totalTokens = [0, 0] - - # Read File into data - data = readFile.readlines() - - # Create Progress Bar - with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar: - pbar.desc = filename - PBAR = pbar - - try: - result = translateTyrano(data, filename, []) - totalTokens[0] += result[0] - totalTokens[1] += result[1] - except Exception as e: - traceback.print_exc() - return [data, totalTokens, e] - return [data, totalTokens, None] - - -def save_progress_lines(lines, filename, encoding="utf-8"): - """Atomically save current line-based translation progress.""" - try: - if ESTIMATE: - return - os.makedirs("translated", exist_ok=True) - tmp_fd, tmp_path = tempfile.mkstemp(prefix=f"{filename}.", suffix=".tmp", dir="translated") - try: - with os.fdopen(tmp_fd, "w", encoding=encoding, newline="\n", errors="ignore") as tmp_file: - tmp_file.writelines(lines) - os.replace(tmp_path, os.path.join("translated", filename)) - finally: - if os.path.exists(tmp_path): - try: - os.remove(tmp_path) - except OSError: - pass - except Exception: - traceback.print_exc() - - -def saveCheckLines(lines, filename, tokens=None, encoding="utf-8"): - """Save progress only when tokens indicate work or when tokens is None but we know a mutation occurred.""" - try: - # If explicit tokens provided, gate on tokens; otherwise assume we were called on actual mutation - if tokens is not None: - if not (isinstance(tokens, (list, tuple)) and len(tokens) >= 2 and (tokens[0] or tokens[1])): - return - save_progress_lines(lines, filename, encoding=encoding) - except Exception: - traceback.print_exc() - - -def translateTyrano(data, filename, translatedList): - if translatedList: - stringList = translatedList[0] - choiceList = translatedList[1] - else: - stringList = [] - choiceList = [] - tokens = [0, 0] - speaker = "" - stringListTL = [] - choiceListTL = [] - global LOCK, ESTIMATE, FILENAME, PBAR, MISMATCH - i = 0 - - while i < len(data): - voice = False - lineRegexNoSpeaker = r"^([^\[#;*@\n]+)\[p\]|^([^\[#;*@\n]+)\[l\]\[[rp]\]|^([^\[#;*@\n]+)\[[rpl]\]|^([^\[#;*@_\n]+)\n$" - lineRegexSpeaker = r"^#(.*)" - furiganaRegex = r"(\[ruby\stext=(.*?)\])" - choiceRegex = r'\[glink.+?text="(.*?)"' - - # Speaker - match = re.search(lineRegexSpeaker, data[i]) - if match: - if match.group(1): - response = getSpeaker(match.group(1)) - speaker = response[0] - tokens[0] += response[1][0] - tokens[1] += response[1][1] - data[i] = data[i].replace(match.group(1), speaker) - else: - speaker = None - - # Furigana - match = re.search(r"^\[ruby\stext", data[i]) - furiganaList = [] - if match: - # Check next line and combine - while match: - furiganaList.append(data[i].replace("\n", "")) - del data[i] - match = re.search(r"^\[ruby\stext", data[i]) - jaString = "".join(furiganaList) - - # Ruby Text - furiganaList = re.findall(furiganaRegex, jaString) - for furigana in furiganaList: - jaString = jaString.replace(furigana[0], furigana[1]) - - data.insert(i, f"{jaString}[r]") - - # Dialogue - match = re.search(lineRegexNoSpeaker, data[i]) - jaString = None - if match: - # Find which group matched - for group_num in range(1, 5): - try: - if match.group(group_num): - jaString = match.group(group_num) - break - except IndexError: - break - - # Skip if no valid string found - if not jaString: - i += 1 - continue - - # Combine w/ next line if necessary - repeatRegex = r"(.+?)\[[rpl]\]" - if i + 1 < len(data): - match = re.search(repeatRegex, data[i + 1]) - while match and "[p]" not in data[i]: - jaString = jaString + match.group(1) - jaString = jaString.replace("_　", "") - if "[p]" in data[i + 1]: - data[i] = f"{jaString}[p]\n" - else: - data[i] = jaString - del data[i + 1] - if i + 1 < len(data): - match = re.search(repeatRegex, data[i + 1]) - else: - break - - originalString = jaString - - # Pass 1 - if not translatedList: - # Remove any textwrap and commands - jaString = jaString.replace("[r]", " ") - jaString = jaString.replace("[l]", "") - - # Ruby Text - furiganaList = re.findall(furiganaRegex, jaString) - for furigana in furiganaList: - jaString = jaString.replace(furigana[0], furigana[1]) - - # Strip Spaces - jaString = jaString.strip() - - if jaString: - if speaker: - stringList.append(f"[{speaker}]: {jaString}") - else: - stringList.append(jaString) - - # Pass 2 - else: - # Get Text - if stringList: - # Grab and Pop - translatedText = stringList[0] - stringList.pop(0) - - # Set to None if empty list - if len(stringList) <= 0: - stringList = None - - # Remove speaker - if speaker != "": - matchSpeakerList = re.findall(r"^\[?(.+?)\]?\s?[|:]\s?", translatedText) - translatedText = re.sub(r"^\[?(.+?)\]?\s?[|:]\s?", "", translatedText) - - # Avoid Crashes - translatedText = translatedText.replace("[", "(") - translatedText = translatedText.replace("]", ")") - - # Textwrap - translatedText = dazedwrap.wrapText(translatedText, width=WIDTH) - translatedText = translatedText.replace("\n", "[r]") - - # Set Data - data[i] = data[i].replace(originalString, translatedText) - # Save progress after each line change (we mutated, so tokens gate optional) - saveCheckLines(data, filename) - - # Choices - match = re.search(choiceRegex, data[i]) - if match: - # Pass 1 - if not translatedList: - choiceList.append(match.group(1)) - match = re.search(choiceRegex, data[i + 1]) - - # Pass 2 - else: - # Grab and Pop - translatedText = choiceList[0] - choiceList.pop(0) - - # Replace Spaces - translatedText = translatedText.replace(" ", "\u3000") - - # Set - data[i] = data[i].replace(match.group(1), translatedText) - saveCheckLines(data, filename) - - i += 1 - else: - i += 1 - - # EOF - if not translatedList: - # String List - if stringList: - PBAR.total = len(stringList) - PBAR.refresh() - response = translateAI(stringList, "Reply with the English Translation") - tokens[0] += response[1][0] - tokens[1] += response[1][1] - stringListTL = response[0] - - if len(stringList) != len(stringListTL): - # Mismatch - with LOCK: - if FILENAME not in MISMATCH: - MISMATCH.append(FILENAME) - - # Choice List - if choiceList: - response = translateAI(choiceList, "Reply with the English TL of the Dialogue Choice") - tokens[0] += response[1][0] - tokens[1] += response[1][1] - choiceListTL = response[0] - - if len(choiceList) != len(choiceListTL): - # Mismatch - with LOCK: - if FILENAME not in MISMATCH: - MISMATCH.append(FILENAME) - - # Recursive call for Pass 2 with translated strings - result = translateTyrano(data, filename, [stringListTL, choiceListTL]) - tokens[0] += result[0] - tokens[1] += result[1] - - return tokens - -# Save some money and enter the character before translation -def getSpeaker(speaker): - match speaker: - case "ファイン": - return ["Fine", [0, 0]] - case "": - return ["", [0, 0]] - case _: - # Find Speaker - for i in range(len(NAMESLIST)): - if speaker == NAMESLIST[i][0]: - return [NAMESLIST[i][1], [0, 0]] - - # Translate and Store Speaker - response = translateAI( - f"{speaker}", - "Reply with the " + LANGUAGE + " translation of the NPC name.", - False, - ) - response[0] = response[0].title() - response[0] = response[0].replace("'S", "'s") - response[0] = response[0].replace("Speaker: ", "") - - # Retry if name doesn't translate for some reason - if re.search(r"([a-zA-Z？?])", response[0]) == None: - response = translateAI( - f"{speaker}", - "Reply with the " + LANGUAGE + " translation of the NPC name.", - False, - ) - response[0] = response[0].title() - response[0] = response[0].replace("'S", "'s") - - speakerList = [speaker, response[0]] - NAMESLIST.append(speakerList) - return response - return [speaker, [0, 0]] - -def translateAI(text, history, history_ctx=None): - """ - Legacy wrapper function for the new shared translation utility. - This maintains compatibility with existing code while using the new shared implementation. - """ - global PBAR, MISMATCH, FILENAME - - # Update config estimate mode based on global ESTIMATE - TRANSLATION_CONFIG.estimateMode = bool(ESTIMATE) - - # Call the new shared translation function - return sharedtranslateAI( - text=text, - history=history, - config=TRANSLATION_CONFIG, - filename=FILENAME, - pbar=PBAR, - lock=LOCK, - mismatchList=MISMATCH - ) +# Libraries +import json +import os +import re +import util.dazedwrap as dazedwrap +import threading +import time +import traceback +import tiktoken +from pathlib import Path +from colorama import Fore +from dotenv import load_dotenv +from retry import retry +from tqdm import tqdm +from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost +import tempfile + +# OpenAI initialization centralized in util/translation.py + +# Globals +MODEL = os.getenv("model") +TIMEOUT = int(os.getenv("timeout")) +LANGUAGE = os.getenv("language").capitalize() +from util.paths import PROMPT_PATH, VOCAB_PATH + +PROMPT = PROMPT_PATH.read_text(encoding="utf-8") +VOCAB = VOCAB_PATH.read_text(encoding="utf-8") +LOCK = threading.Lock() +WIDTH = int(os.getenv("width")) +LISTWIDTH = int(os.getenv("listWidth")) +NOTEWIDTH = 70 +MAXHISTORY = 10 +ESTIMATE = "" +TOKENS = [0, 0] +NAMESLIST = [] +NAMES = False # Output a list of all the character names found +BRFLAG = False # If the game uses
instead +FIXTEXTWRAP = True # Overwrites textwrap +IGNORETLTEXT = False # Ignores all translated text. +MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong) +FILENAME = None + +# tqdm Globals +BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}" +POSITION = 0 +LEAVE = False +PBAR = None + +# Regex - Need to change this if you want to translate from/to other languages. Default is Japanese Regex +LANGREGEX = r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+" + +# Get pricing configuration based on the model +PRICING_CONFIG = getPricingConfig(MODEL) +INPUTAPICOST = PRICING_CONFIG["inputAPICost"] +OUTPUTAPICOST = PRICING_CONFIG["outputAPICost"] +BATCHSIZE = PRICING_CONFIG["batchSize"] +FREQUENCY_PENALTY = PRICING_CONFIG["frequencyPenalty"] + +# Initialize Translation Config +TRANSLATION_CONFIG = TranslationConfig( + model=MODEL, + language=LANGUAGE, + prompt=PROMPT, + vocab=VOCAB, + langRegex=LANGREGEX, + batchSize=BATCHSIZE, + maxHistory=MAXHISTORY, + estimateMode=False # Will be set dynamically based on ESTIMATE +) +LEAVE = False + +def handleTyrano(filename, estimate): + global ESTIMATE, TOKENS, FILENAME + ESTIMATE = estimate + FILENAME = filename + + # Translate + start = time.time() + translatedData = openFiles(filename) + + # Translate + if not estimate: + try: + with open("translated/" + filename, "w", encoding="utf-8", newline="\n") as outFile: + outFile.writelines(translatedData[0]) + except Exception: + traceback.print_exc() + return "Fail" + + # Print File + end = time.time() + tqdm.write(getResultString(translatedData, end - start, filename)) + with LOCK: + TOKENS[0] += translatedData[1][0] + TOKENS[1] += translatedData[1][1] + + # Print Total + totalString = getResultString(["", TOKENS, None], end - start, "TOTAL") + + # Print any errors on maps + if len(MISMATCH) > 0: + return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET + else: + return totalString + + +def getResultString(translatedData, translationTime, filename): + # File Print String + cost = calculateCost(translatedData[1][0], translatedData[1][1], MODEL) + totalTokenstring = ( + Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]" + "[Output: " + + str(translatedData[1][1]) + + "]" "[Cost: ${:,.4f}".format(cost) + + "]" + ) + timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]" + + if translatedData[2] == None: + # Success + return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET + + else: + # Fail + try: + raise translatedData[2] + except Exception as e: + traceback.print_exc() + errorString = str(e) + Fore.RED + return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET + + +def openFiles(filename): + with open("files/" + filename, "r", encoding="utf8") as readFile: + translatedData = parseTyrano(readFile, filename) + + # Delete lines marked for deletion + finalData = [] + for line in translatedData[0]: + if line != "\\d\n": + finalData.append(line) + translatedData[0] = finalData + + return translatedData + + +def parseTyrano(readFile, filename): + global PBAR + totalTokens = [0, 0] + + # Read File into data + data = readFile.readlines() + + # Create Progress Bar + with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar: + pbar.desc = filename + PBAR = pbar + + try: + result = translateTyrano(data, filename, []) + totalTokens[0] += result[0] + totalTokens[1] += result[1] + except Exception as e: + traceback.print_exc() + return [data, totalTokens, e] + return [data, totalTokens, None] + + +def save_progress_lines(lines, filename, encoding="utf-8"): + """Atomically save current line-based translation progress.""" + try: + if ESTIMATE: + return + os.makedirs("translated", exist_ok=True) + tmp_fd, tmp_path = tempfile.mkstemp(prefix=f"{filename}.", suffix=".tmp", dir="translated") + try: + with os.fdopen(tmp_fd, "w", encoding=encoding, newline="\n", errors="ignore") as tmp_file: + tmp_file.writelines(lines) + os.replace(tmp_path, os.path.join("translated", filename)) + finally: + if os.path.exists(tmp_path): + try: + os.remove(tmp_path) + except OSError: + pass + except Exception: + traceback.print_exc() + + +def saveCheckLines(lines, filename, tokens=None, encoding="utf-8"): + """Save progress only when tokens indicate work or when tokens is None but we know a mutation occurred.""" + try: + # If explicit tokens provided, gate on tokens; otherwise assume we were called on actual mutation + if tokens is not None: + if not (isinstance(tokens, (list, tuple)) and len(tokens) >= 2 and (tokens[0] or tokens[1])): + return + save_progress_lines(lines, filename, encoding=encoding) + except Exception: + traceback.print_exc() + + +def translateTyrano(data, filename, translatedList): + if translatedList: + stringList = translatedList[0] + choiceList = translatedList[1] + else: + stringList = [] + choiceList = [] + tokens = [0, 0] + speaker = "" + stringListTL = [] + choiceListTL = [] + global LOCK, ESTIMATE, FILENAME, PBAR, MISMATCH + i = 0 + + while i < len(data): + voice = False + lineRegexNoSpeaker = r"^([^\[#;*@\n]+)\[p\]|^([^\[#;*@\n]+)\[l\]\[[rp]\]|^([^\[#;*@\n]+)\[[rpl]\]|^([^\[#;*@_\n]+)\n$" + lineRegexSpeaker = r"^#(.*)" + furiganaRegex = r"(\[ruby\stext=(.*?)\])" + choiceRegex = r'\[glink.+?text="(.*?)"' + + # Speaker + match = re.search(lineRegexSpeaker, data[i]) + if match: + if match.group(1): + response = getSpeaker(match.group(1)) + speaker = response[0] + tokens[0] += response[1][0] + tokens[1] += response[1][1] + data[i] = data[i].replace(match.group(1), speaker) + else: + speaker = None + + # Furigana + match = re.search(r"^\[ruby\stext", data[i]) + furiganaList = [] + if match: + # Check next line and combine + while match: + furiganaList.append(data[i].replace("\n", "")) + del data[i] + match = re.search(r"^\[ruby\stext", data[i]) + jaString = "".join(furiganaList) + + # Ruby Text + furiganaList = re.findall(furiganaRegex, jaString) + for furigana in furiganaList: + jaString = jaString.replace(furigana[0], furigana[1]) + + data.insert(i, f"{jaString}[r]") + + # Dialogue + match = re.search(lineRegexNoSpeaker, data[i]) + jaString = None + if match: + # Find which group matched + for group_num in range(1, 5): + try: + if match.group(group_num): + jaString = match.group(group_num) + break + except IndexError: + break + + # Skip if no valid string found + if not jaString: + i += 1 + continue + + # Combine w/ next line if necessary + repeatRegex = r"(.+?)\[[rpl]\]" + if i + 1 < len(data): + match = re.search(repeatRegex, data[i + 1]) + while match and "[p]" not in data[i]: + jaString = jaString + match.group(1) + jaString = jaString.replace("_　", "") + if "[p]" in data[i + 1]: + data[i] = f"{jaString}[p]\n" + else: + data[i] = jaString + del data[i + 1] + if i + 1 < len(data): + match = re.search(repeatRegex, data[i + 1]) + else: + break + + originalString = jaString + + # Pass 1 + if not translatedList: + # Remove any textwrap and commands + jaString = jaString.replace("[r]", " ") + jaString = jaString.replace("[l]", "") + + # Ruby Text + furiganaList = re.findall(furiganaRegex, jaString) + for furigana in furiganaList: + jaString = jaString.replace(furigana[0], furigana[1]) + + # Strip Spaces + jaString = jaString.strip() + + if jaString: + if speaker: + stringList.append(f"[{speaker}]: {jaString}") + else: + stringList.append(jaString) + + # Pass 2 + else: + # Get Text + if stringList: + # Grab and Pop + translatedText = stringList[0] + stringList.pop(0) + + # Set to None if empty list + if len(stringList) <= 0: + stringList = None + + # Remove speaker + if speaker != "": + matchSpeakerList = re.findall(r"^\[?(.+?)\]?\s?[|:]\s?", translatedText) + translatedText = re.sub(r"^\[?(.+?)\]?\s?[|:]\s?", "", translatedText) + + # Avoid Crashes + translatedText = translatedText.replace("[", "(") + translatedText = translatedText.replace("]", ")") + + # Textwrap + translatedText = dazedwrap.wrapText(translatedText, width=WIDTH) + translatedText = translatedText.replace("\n", "[r]") + + # Set Data + data[i] = data[i].replace(originalString, translatedText) + # Save progress after each line change (we mutated, so tokens gate optional) + saveCheckLines(data, filename) + + # Choices + match = re.search(choiceRegex, data[i]) + if match: + # Pass 1 + if not translatedList: + choiceList.append(match.group(1)) + match = re.search(choiceRegex, data[i + 1]) + + # Pass 2 + else: + # Grab and Pop + translatedText = choiceList[0] + choiceList.pop(0) + + # Replace Spaces + translatedText = translatedText.replace(" ", "\u3000") + + # Set + data[i] = data[i].replace(match.group(1), translatedText) + saveCheckLines(data, filename) + + i += 1 + else: + i += 1 + + # EOF + if not translatedList: + # String List + if stringList: + PBAR.total = len(stringList) + PBAR.refresh() + response = translateAI(stringList, "Reply with the English Translation") + tokens[0] += response[1][0] + tokens[1] += response[1][1] + stringListTL = response[0] + + if len(stringList) != len(stringListTL): + # Mismatch + with LOCK: + if FILENAME not in MISMATCH: + MISMATCH.append(FILENAME) + + # Choice List + if choiceList: + response = translateAI(choiceList, "Reply with the English TL of the Dialogue Choice") + tokens[0] += response[1][0] + tokens[1] += response[1][1] + choiceListTL = response[0] + + if len(choiceList) != len(choiceListTL): + # Mismatch + with LOCK: + if FILENAME not in MISMATCH: + MISMATCH.append(FILENAME) + + # Recursive call for Pass 2 with translated strings + result = translateTyrano(data, filename, [stringListTL, choiceListTL]) + tokens[0] += result[0] + tokens[1] += result[1] + + return tokens + +# Save some money and enter the character before translation +def getSpeaker(speaker): + match speaker: + case "ファイン": + return ["Fine", [0, 0]] + case "": + return ["", [0, 0]] + case _: + # Find Speaker + for i in range(len(NAMESLIST)): + if speaker == NAMESLIST[i][0]: + return [NAMESLIST[i][1], [0, 0]] + + # Translate and Store Speaker + response = translateAI( + f"{speaker}", + "Reply with the " + LANGUAGE + " translation of the NPC name.", + False, + ) + response[0] = response[0].title() + response[0] = response[0].replace("'S", "'s") + response[0] = response[0].replace("Speaker: ", "") + + # Retry if name doesn't translate for some reason + if re.search(r"([a-zA-Z？?])", response[0]) == None: + response = translateAI( + f"{speaker}", + "Reply with the " + LANGUAGE + " translation of the NPC name.", + False, + ) + response[0] = response[0].title() + response[0] = response[0].replace("'S", "'s") + + speakerList = [speaker, response[0]] + NAMESLIST.append(speakerList) + return response + return [speaker, [0, 0]] + +def translateAI(text, history, history_ctx=None): + """ + Legacy wrapper function for the new shared translation utility. + This maintains compatibility with existing code while using the new shared implementation. + """ + global PBAR, MISMATCH, FILENAME + + # Update config estimate mode based on global ESTIMATE + TRANSLATION_CONFIG.estimateMode = bool(ESTIMATE) + + # Call the new shared translation function + return sharedtranslateAI( + text=text, + history=history, + config=TRANSLATION_CONFIG, + filename=FILENAME, + pbar=PBAR, + lock=LOCK, + mismatchList=MISMATCH + ) diff --git a/modules/unity.py b/modules/unity.py index d1c2f11..ec92c3e 100644 --- a/modules/unity.py +++ b/modules/unity.py @@ -1,353 +1,355 @@ -# Libraries -import json -import os -import re -import util.dazedwrap as dazedwrap -import threading -import time -import traceback -import tiktoken -from pathlib import Path -from colorama import Fore -from dotenv import load_dotenv -from retry import retry -from tqdm import tqdm -from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost, getPricingConfig, calculateCost -import tempfile - -# OpenAI initialization centralized in util/translation.py - -# Globals -MODEL = os.getenv("model") -TIMEOUT = int(os.getenv("timeout")) -LANGUAGE = os.getenv("language").capitalize() -PROMPT = Path("prompt.txt").read_text(encoding="utf-8") -VOCAB = Path("vocab.txt").read_text(encoding="utf-8") -LOCK = threading.Lock() -WIDTH = int(os.getenv("width")) -LISTWIDTH = int(os.getenv("listWidth")) -NOTEWIDTH = 70 -MAXHISTORY = 10 -ESTIMATE = "" -TOKENS = [0, 0] -NAMESLIST = [] -NAMES = False # Output a list of all the character names found -BRFLAG = False # If the game uses
instead -FIXTEXTWRAP = True # Overwrites textwrap -IGNORETLTEXT = False # Ignores all translated text. -MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong) - -# tqdm Globals -BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}" -POSITION = 0 -LEAVE = False -PBAR = None -FILENAME = None - -# Full Width -ascii_to_wide = dict((i, chr(i + 0xFEE0)) for i in range(0x21, 0x7F)) -ascii_to_wide.update({0x20: "\u3000", 0x2D: "\u2212"}) # space and minus -wide_to_ascii = dict((i, chr(i - 0xFEE0)) for i in range(0xFF01, 0xFF5F)) -wide_to_ascii.update({0x3000: " ", 0x2212: "-"}) # space and minus - -# Regex - Need to change this if you want to translate from/to other languages. Default is Japanese Regex -LANGREGEX = r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+" - -# Get pricing configuration based on the model -PRICING_CONFIG = getPricingConfig(MODEL) -INPUTAPICOST = PRICING_CONFIG["inputAPICost"] -OUTPUTAPICOST = PRICING_CONFIG["outputAPICost"] -BATCHSIZE = PRICING_CONFIG["batchSize"] -FREQUENCY_PENALTY = PRICING_CONFIG["frequencyPenalty"] - -# Initialize Translation Config -TRANSLATION_CONFIG = TranslationConfig( - model=MODEL, - language=LANGUAGE, - prompt=PROMPT, - vocab=VOCAB, - langRegex=LANGREGEX, - batchSize=BATCHSIZE, - maxHistory=MAXHISTORY, - estimateMode=False # Will be set dynamically based on ESTIMATE -) -LEAVE = False - -def handleUnity(filename, estimate): - global ESTIMATE, FILENAME - ESTIMATE = estimate - FILENAME = filename - - if ESTIMATE: - start = time.time() - translatedData = openFiles(filename) - - # Print Result - end = time.time() - tqdm.write(getResultString(translatedData, end - start, filename)) - with LOCK: - TOKENS[0] += translatedData[1][0] - TOKENS[1] += translatedData[1][1] - - # Print Total - totalString = getResultString(["", TOKENS, None], end - start, "TOTAL") - - # Print any errors on maps - if len(MISMATCH) > 0: - return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET - else: - return totalString - - else: - try: - with open("translated/" + filename, "w", encoding="utf8", errors="ignore") as outFile: - start = time.time() - translatedData = openFiles(filename) - - # Print Result - end = time.time() - outFile.writelines(translatedData[0]) - tqdm.write(getResultString(translatedData, end - start, filename)) - with LOCK: - TOKENS[0] += translatedData[1][0] - TOKENS[1] += translatedData[1][1] - except Exception: - traceback.print_exc() - return "Fail" - - return getResultString(["", TOKENS, None], end - start, "TOTAL") - - -def getResultString(translatedData, translationTime, filename): - # File Print String - cost = calculateCost(translatedData[1][0], translatedData[1][1], MODEL) - totalTokenstring = ( - Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]" - "[Output: " - + str(translatedData[1][1]) - + "]" "[Cost: ${:,.4f}".format(cost) - + "]" - ) - timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]" - - if translatedData[2] == None: - # Success - return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET - - else: - # Fail - try: - raise translatedData[2] - except Exception as e: - traceback.print_exc() - errorString = str(e) + Fore.RED - return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET - - -def openFiles(filename): - with open("files/" + filename, "r", encoding="utf8") as readFile: - translatedData = parseUnity(readFile, filename) - - # Delete lines marked for deletion - finalData = [] - for line in translatedData[0]: - if line != "\\d\n": - finalData.append(line) - translatedData[0] = finalData - - return translatedData - - -def parseUnity(readFile, filename): - totalTokens = [0, 0] - - # Read File into data - data = readFile.readlines() - - # Create Progress Bar - with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar: - pbar.desc = filename - - try: - result = translateUnity(data, pbar, filename, []) - totalTokens[0] += result[0] - totalTokens[1] += result[1] - except Exception as e: - traceback.print_exc() - return [data, totalTokens, e] - return [data, totalTokens, None] - - -def save_progress_lines(lines, filename, encoding="utf-8"): - """Atomically save current line-based translation progress.""" - try: - if ESTIMATE: - return - os.makedirs("translated", exist_ok=True) - tmp_fd, tmp_path = tempfile.mkstemp(prefix=f"{filename}.", suffix=".tmp", dir="translated") - try: - with os.fdopen(tmp_fd, "w", encoding=encoding, newline="\n", errors="ignore") as tmp_file: - tmp_file.writelines(lines) - os.replace(tmp_path, os.path.join("translated", filename)) - finally: - if os.path.exists(tmp_path): - try: - os.remove(tmp_path) - except OSError: - pass - except Exception: - traceback.print_exc() - - -def saveCheckLines(lines, filename, tokens=None, encoding="utf-8"): - """Save progress only when tokens indicate work or when explicitly called after a mutation.""" - try: - if tokens is not None: - if not (isinstance(tokens, (list, tuple)) and len(tokens) >= 2 and (tokens[0] or tokens[1])): - return - save_progress_lines(lines, filename, encoding=encoding) - except Exception: - traceback.print_exc() - - -def translateUnity(data, pbar, filename, translatedList): - stringList = [] - tokens = [0, 0] - global LOCK, ESTIMATE, PBAR - PBAR = pbar - i = 0 - - # Dialogue - while i < len(data): - # Lines - regex = r"(.*?)=(.*)" - match = re.search(regex, data[i]) - if match: - leftString = match.group(1) - rightString = match.group(2) - - # Validate Japanese Text - if not re.search(LANGREGEX, leftString) and IGNORETLTEXT: - i += 1 - continue - - # Pass 1 - if translatedList == []: - jaString = leftString - - # Remove textwrap - # jaString = jaString.replace("\\n", " ") - - # Add String - stringList.append(jaString.strip()) - - # Pass 2 - else: - # Get Text - if translatedList: - # Grab and Pop - translatedText = translatedList[0] - translatedList.pop(0) - - # Set to None if empty list - if len(translatedList) <= 0: - translatedList = None - - # Textwrap - # translatedText = dazedwrap.wrapText(translatedText, width=WIDTH) - # translatedText = translatedText.replace("\n", "\\n") - - # Remove Double Spaces and = - translatedText = translatedText.replace(" ", " ") - translatedText = translatedText.replace("=", "->") - - # Set Data - data[i] = f"{leftString}={translatedText}\n" - saveCheckLines(data, filename) - i += 1 - - # Nothing relevant. Skip Line. - else: - i += 1 - - # EOF - if len(stringList) > 0: - # Set Progress - pbar.total = len(stringList) - pbar.refresh() - - # Translate - response = translateAI(stringList, "") - tokens[0] += response[1][0] - tokens[1] += response[1][1] - translatedList = response[0] - - # Set Strings - if len(stringList) == len(translatedList): - translateUnity(data, pbar, filename, translatedList) - - # Mismatch - else: - with LOCK: - if filename not in MISMATCH: - MISMATCH.append(filename) - return tokens - -# Save some money and enter the character before translation -def getSpeaker(speaker): - match speaker: - case "ファイン": - return ["Fine", [0, 0]] - case "": - return ["", [0, 0]] - case _: - # Find Speaker - for i in range(len(NAMESLIST)): - if speaker == NAMESLIST[i][0]: - return [NAMESLIST[i][1], [0, 0]] - - # Translate and Store Speaker - response = translateAI( - f"{speaker}", - "Reply with the " + LANGUAGE + " translation of the NPC name.", - False, - ) - response[0] = response[0].title() - response[0] = response[0].replace("'S", "'s") - response[0] = response[0].replace("Speaker: ", "") - - # Retry if name doesn't translate for some reason - if re.search(r"([a-zA-Z？?])", response[0]) == None: - response = translateAI( - f"{speaker}", - "Reply with the " + LANGUAGE + " translation of the NPC name.", - False, - ) - response[0] = response[0].title() - response[0] = response[0].replace("'S", "'s") - - speakerList = [speaker, response[0]] - NAMESLIST.append(speakerList) - return response - return [speaker, [0, 0]] - -def translateAI(text, history, history_ctx=None): - """ - Legacy wrapper function for the new shared translation utility. - This maintains compatibility with existing code while using the new shared implementation. - """ - global PBAR, MISMATCH, FILENAME - - # Update config estimate mode based on global ESTIMATE - TRANSLATION_CONFIG.estimateMode = bool(ESTIMATE) - - # Call the new shared translation function - return sharedtranslateAI( - text=text, - history=history, - config=TRANSLATION_CONFIG, - filename=FILENAME, - pbar=PBAR, - lock=LOCK, - mismatchList=MISMATCH - ) +# Libraries +import json +import os +import re +import util.dazedwrap as dazedwrap +import threading +import time +import traceback +import tiktoken +from pathlib import Path +from colorama import Fore +from dotenv import load_dotenv +from retry import retry +from tqdm import tqdm +from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost, getPricingConfig, calculateCost +import tempfile + +# OpenAI initialization centralized in util/translation.py + +# Globals +MODEL = os.getenv("model") +TIMEOUT = int(os.getenv("timeout")) +LANGUAGE = os.getenv("language").capitalize() +from util.paths import PROMPT_PATH, VOCAB_PATH + +PROMPT = PROMPT_PATH.read_text(encoding="utf-8") +VOCAB = VOCAB_PATH.read_text(encoding="utf-8") +LOCK = threading.Lock() +WIDTH = int(os.getenv("width")) +LISTWIDTH = int(os.getenv("listWidth")) +NOTEWIDTH = 70 +MAXHISTORY = 10 +ESTIMATE = "" +TOKENS = [0, 0] +NAMESLIST = [] +NAMES = False # Output a list of all the character names found +BRFLAG = False # If the game uses
instead +FIXTEXTWRAP = True # Overwrites textwrap +IGNORETLTEXT = False # Ignores all translated text. +MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong) + +# tqdm Globals +BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}" +POSITION = 0 +LEAVE = False +PBAR = None +FILENAME = None + +# Full Width +ascii_to_wide = dict((i, chr(i + 0xFEE0)) for i in range(0x21, 0x7F)) +ascii_to_wide.update({0x20: "\u3000", 0x2D: "\u2212"}) # space and minus +wide_to_ascii = dict((i, chr(i - 0xFEE0)) for i in range(0xFF01, 0xFF5F)) +wide_to_ascii.update({0x3000: " ", 0x2212: "-"}) # space and minus + +# Regex - Need to change this if you want to translate from/to other languages. Default is Japanese Regex +LANGREGEX = r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+" + +# Get pricing configuration based on the model +PRICING_CONFIG = getPricingConfig(MODEL) +INPUTAPICOST = PRICING_CONFIG["inputAPICost"] +OUTPUTAPICOST = PRICING_CONFIG["outputAPICost"] +BATCHSIZE = PRICING_CONFIG["batchSize"] +FREQUENCY_PENALTY = PRICING_CONFIG["frequencyPenalty"] + +# Initialize Translation Config +TRANSLATION_CONFIG = TranslationConfig( + model=MODEL, + language=LANGUAGE, + prompt=PROMPT, + vocab=VOCAB, + langRegex=LANGREGEX, + batchSize=BATCHSIZE, + maxHistory=MAXHISTORY, + estimateMode=False # Will be set dynamically based on ESTIMATE +) +LEAVE = False + +def handleUnity(filename, estimate): + global ESTIMATE, FILENAME + ESTIMATE = estimate + FILENAME = filename + + if ESTIMATE: + start = time.time() + translatedData = openFiles(filename) + + # Print Result + end = time.time() + tqdm.write(getResultString(translatedData, end - start, filename)) + with LOCK: + TOKENS[0] += translatedData[1][0] + TOKENS[1] += translatedData[1][1] + + # Print Total + totalString = getResultString(["", TOKENS, None], end - start, "TOTAL") + + # Print any errors on maps + if len(MISMATCH) > 0: + return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET + else: + return totalString + + else: + try: + with open("translated/" + filename, "w", encoding="utf8", errors="ignore") as outFile: + start = time.time() + translatedData = openFiles(filename) + + # Print Result + end = time.time() + outFile.writelines(translatedData[0]) + tqdm.write(getResultString(translatedData, end - start, filename)) + with LOCK: + TOKENS[0] += translatedData[1][0] + TOKENS[1] += translatedData[1][1] + except Exception: + traceback.print_exc() + return "Fail" + + return getResultString(["", TOKENS, None], end - start, "TOTAL") + + +def getResultString(translatedData, translationTime, filename): + # File Print String + cost = calculateCost(translatedData[1][0], translatedData[1][1], MODEL) + totalTokenstring = ( + Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]" + "[Output: " + + str(translatedData[1][1]) + + "]" "[Cost: ${:,.4f}".format(cost) + + "]" + ) + timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]" + + if translatedData[2] == None: + # Success + return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET + + else: + # Fail + try: + raise translatedData[2] + except Exception as e: + traceback.print_exc() + errorString = str(e) + Fore.RED + return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET + + +def openFiles(filename): + with open("files/" + filename, "r", encoding="utf8") as readFile: + translatedData = parseUnity(readFile, filename) + + # Delete lines marked for deletion + finalData = [] + for line in translatedData[0]: + if line != "\\d\n": + finalData.append(line) + translatedData[0] = finalData + + return translatedData + + +def parseUnity(readFile, filename): + totalTokens = [0, 0] + + # Read File into data + data = readFile.readlines() + + # Create Progress Bar + with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar: + pbar.desc = filename + + try: + result = translateUnity(data, pbar, filename, []) + totalTokens[0] += result[0] + totalTokens[1] += result[1] + except Exception as e: + traceback.print_exc() + return [data, totalTokens, e] + return [data, totalTokens, None] + + +def save_progress_lines(lines, filename, encoding="utf-8"): + """Atomically save current line-based translation progress.""" + try: + if ESTIMATE: + return + os.makedirs("translated", exist_ok=True) + tmp_fd, tmp_path = tempfile.mkstemp(prefix=f"{filename}.", suffix=".tmp", dir="translated") + try: + with os.fdopen(tmp_fd, "w", encoding=encoding, newline="\n", errors="ignore") as tmp_file: + tmp_file.writelines(lines) + os.replace(tmp_path, os.path.join("translated", filename)) + finally: + if os.path.exists(tmp_path): + try: + os.remove(tmp_path) + except OSError: + pass + except Exception: + traceback.print_exc() + + +def saveCheckLines(lines, filename, tokens=None, encoding="utf-8"): + """Save progress only when tokens indicate work or when explicitly called after a mutation.""" + try: + if tokens is not None: + if not (isinstance(tokens, (list, tuple)) and len(tokens) >= 2 and (tokens[0] or tokens[1])): + return + save_progress_lines(lines, filename, encoding=encoding) + except Exception: + traceback.print_exc() + + +def translateUnity(data, pbar, filename, translatedList): + stringList = [] + tokens = [0, 0] + global LOCK, ESTIMATE, PBAR + PBAR = pbar + i = 0 + + # Dialogue + while i < len(data): + # Lines + regex = r"(.*?)=(.*)" + match = re.search(regex, data[i]) + if match: + leftString = match.group(1) + rightString = match.group(2) + + # Validate Japanese Text + if not re.search(LANGREGEX, leftString) and IGNORETLTEXT: + i += 1 + continue + + # Pass 1 + if translatedList == []: + jaString = leftString + + # Remove textwrap + # jaString = jaString.replace("\\n", " ") + + # Add String + stringList.append(jaString.strip()) + + # Pass 2 + else: + # Get Text + if translatedList: + # Grab and Pop + translatedText = translatedList[0] + translatedList.pop(0) + + # Set to None if empty list + if len(translatedList) <= 0: + translatedList = None + + # Textwrap + # translatedText = dazedwrap.wrapText(translatedText, width=WIDTH) + # translatedText = translatedText.replace("\n", "\\n") + + # Remove Double Spaces and = + translatedText = translatedText.replace(" ", " ") + translatedText = translatedText.replace("=", "->") + + # Set Data + data[i] = f"{leftString}={translatedText}\n" + saveCheckLines(data, filename) + i += 1 + + # Nothing relevant. Skip Line. + else: + i += 1 + + # EOF + if len(stringList) > 0: + # Set Progress + pbar.total = len(stringList) + pbar.refresh() + + # Translate + response = translateAI(stringList, "") + tokens[0] += response[1][0] + tokens[1] += response[1][1] + translatedList = response[0] + + # Set Strings + if len(stringList) == len(translatedList): + translateUnity(data, pbar, filename, translatedList) + + # Mismatch + else: + with LOCK: + if filename not in MISMATCH: + MISMATCH.append(filename) + return tokens + +# Save some money and enter the character before translation +def getSpeaker(speaker): + match speaker: + case "ファイン": + return ["Fine", [0, 0]] + case "": + return ["", [0, 0]] + case _: + # Find Speaker + for i in range(len(NAMESLIST)): + if speaker == NAMESLIST[i][0]: + return [NAMESLIST[i][1], [0, 0]] + + # Translate and Store Speaker + response = translateAI( + f"{speaker}", + "Reply with the " + LANGUAGE + " translation of the NPC name.", + False, + ) + response[0] = response[0].title() + response[0] = response[0].replace("'S", "'s") + response[0] = response[0].replace("Speaker: ", "") + + # Retry if name doesn't translate for some reason + if re.search(r"([a-zA-Z？?])", response[0]) == None: + response = translateAI( + f"{speaker}", + "Reply with the " + LANGUAGE + " translation of the NPC name.", + False, + ) + response[0] = response[0].title() + response[0] = response[0].replace("'S", "'s") + + speakerList = [speaker, response[0]] + NAMESLIST.append(speakerList) + return response + return [speaker, [0, 0]] + +def translateAI(text, history, history_ctx=None): + """ + Legacy wrapper function for the new shared translation utility. + This maintains compatibility with existing code while using the new shared implementation. + """ + global PBAR, MISMATCH, FILENAME + + # Update config estimate mode based on global ESTIMATE + TRANSLATION_CONFIG.estimateMode = bool(ESTIMATE) + + # Call the new shared translation function + return sharedtranslateAI( + text=text, + history=history, + config=TRANSLATION_CONFIG, + filename=FILENAME, + pbar=PBAR, + lock=LOCK, + mismatchList=MISMATCH + ) diff --git a/modules/wolf.py b/modules/wolf.py index 6b61cdc..20a1868 100644 --- a/modules/wolf.py +++ b/modules/wolf.py @@ -1,2699 +1,2701 @@ -# Libraries -import json -import os -import re -import util.dazedwrap as dazedwrap -import threading -import time -import traceback -import tiktoken -# Removed concurrent.futures usage for simplicity; running synchronously -from pathlib import Path -from colorama import Fore -from dotenv import load_dotenv -from retry import retry -from tqdm import tqdm -from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost, getPricingConfig, calculateCost - -# OpenAI initialization centralized in util/translation.py - -# Globals -MODEL = os.getenv("model") -TIMEOUT = int(os.getenv("timeout")) -LANGUAGE = os.getenv("language").capitalize() -PROMPT = Path("prompt.txt").read_text(encoding="utf-8") -VOCAB = Path("vocab.txt").read_text(encoding="utf-8") -LOCK = threading.Lock() -WIDTH = int(os.getenv("width")) -LISTWIDTH = int(os.getenv("listWidth")) -NOTEWIDTH = int(os.getenv("noteWidth")) -MAXHISTORY = 10 -ESTIMATE = "" -TOKENS = [0, 0] -NAMESLIST = [] # Keep list for consistency -TERMSLIST = [] # Keep list for consistency -NAMES = False # Output a list of all the character names found -BRFLAG = False # If the game uses
instead -FIXTEXTWRAP = True # Overwrites textwrap -IGNORETLTEXT = False # Ignores all translated text. -MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong) -FILENAME = None -BRACKETNAMES = False - -# Regex - Need to change this if you want to translate from/to other languages. Default is Japanese Regex -LANGREGEX = r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+" - -# Get pricing configuration based on the model -PRICING_CONFIG = getPricingConfig(MODEL) -INPUTAPICOST = PRICING_CONFIG["inputAPICost"] -OUTPUTAPICOST = PRICING_CONFIG["outputAPICost"] -BATCHSIZE = PRICING_CONFIG["batchSize"] -FREQUENCY_PENALTY = PRICING_CONFIG["frequencyPenalty"] - -# tqdm Globals -BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}" -POSITION = 0 -LEAVE = False -PBAR = None -FILENAME = None - -# Dialogue / Choices -CODE101 = False -CODE102 = False - -# Picture -CODE150 = False - -# Set String (Fragile but necessary) -CODE122 = False - -# Other -CODE210 = False -CODE300 = False -CODE250 = False - -# Database -SCENARIOFLAG = False -OPTIONSFLAG = True -NPCFLAG = False -DBNAMEFLAG = False -DBVALUEFLAG = False -ITEMFLAG = False -STATEFLAG = False -ENEMYFLAG = False -ARMORFLAG = False -WEAPONFLAG = False -SKILLFLAG = False - -# Initialize Translation Config -TRANSLATION_CONFIG = TranslationConfig( - model=MODEL, - language=LANGUAGE, - prompt=PROMPT, - vocab=VOCAB, - langRegex=LANGREGEX, - batchSize=BATCHSIZE, - maxHistory=MAXHISTORY, - estimateMode=False # Will be set dynamically based on ESTIMATE -) -LEAVE = False - -def handleWOLF(filename, estimate): - global ESTIMATE, TOKENS, FILENAME - ESTIMATE = estimate - FILENAME = filename - - # Translate - start = time.time() - translatedData = openFiles(filename) - - # Translate - if not estimate: - try: - with open("translated/" + filename, "w", encoding="utf-8", newline="\n") as outFile: - json.dump(translatedData[0], outFile, ensure_ascii=False, indent=4) - except Exception: - traceback.print_exc() - return "Fail" - - # Print File - end = time.time() - tqdm.write(getResultString(translatedData, end - start, filename)) - with LOCK: - TOKENS[0] += translatedData[1][0] - TOKENS[1] += translatedData[1][1] - - # Print Total - totalString = getResultString(["", TOKENS, None], end - start, "TOTAL") - - # Print any errors on maps - if len(MISMATCH) > 0: - return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET - else: - return totalString - - -def openFiles(filename): - with open("files/" + filename, "r", encoding="utf-8-sig") as f: - data = json.load(f) - - # Map Files - if "'events':" in str(data): - if len(data["events"]) > 0: - translatedData = parseMap(data, filename) - else: - return [data, [0, 0], None] - - # Map Files - elif "'types':" in str(data): - translatedData = parseDB(data, filename) - - # Other Files - elif "'commands':" in str(data): - translatedData = parseOther(data, filename) - - else: - raise NameError(filename + " Not Supported") - - return translatedData - - -def getResultString(translatedData, translationTime, filename): - # File Print String - cost = calculateCost(translatedData[1][0], translatedData[1][1], MODEL) - totalTokenstring = ( - Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]" - "[Output: " - + str(translatedData[1][1]) - + "]" "[Cost: ${:,.4f}".format(cost) - + "]" - ) - timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]" - - if translatedData[2] is None: - # Success - return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET - else: - # Fail - try: - raise translatedData[2] - except Exception as e: - traceback.print_exc() - errorString = str(e) + Fore.RED - return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET - - -def save_progress_json(data, filename): - """Atomically write current JSON data to translated/filename; skip in estimate mode.""" - try: - if ESTIMATE: - return - os.makedirs("translated", exist_ok=True) - tmp_path = os.path.join("translated", f"{filename}.tmp") - final_path = os.path.join("translated", filename) - with open(tmp_path, "w", encoding="utf-8", newline="\n") as outFile: - json.dump(data, outFile, ensure_ascii=False, indent=4) - os.replace(tmp_path, final_path) - except Exception: - traceback.print_exc() - - -def maybe_save_progress_json(data, filename, tokens): - """Save JSON progress only when tokens indicate actual translation work.""" - try: - if not tokens: - return - if isinstance(tokens, (list, tuple)) and len(tokens) >= 2 and (tokens[0] or tokens[1]): - save_progress_json(data, filename) - except Exception: - traceback.print_exc() - - -def parseOther(data, filename): - totalTokens = [0, 0] - totalLines = 0 - events = data["commands"] - global LOCK - - # Thread for each page in file - with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar: - pbar.desc = filename - pbar.total = totalLines - translationData = searchCodes(events, pbar, [], filename) - try: - totalTokens[0] += translationData[0] - totalTokens[1] += translationData[1] - except Exception as e: - return [data, totalTokens, e] - finally: - maybe_save_progress_json(data, filename, translationData) - return [data, totalTokens, None] - - -def parseDB(data, filename): - totalTokens = [0, 0] - totalLines = 0 - events = data["types"] - global LOCK - - # Thread for each page in file - with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar: - pbar.desc = filename - pbar.total = totalLines - translationData = searchDB(events, pbar, [], filename) - try: - totalTokens[0] += translationData[0] - totalTokens[1] += translationData[1] - except Exception as e: - return [data, totalTokens, e] - finally: - maybe_save_progress_json(data, filename, translationData) - return [data, totalTokens, None] - - -def parseMap(data, filename): - totalTokens = [0, 0] - totalLines = 0 - events = data["events"] - global LOCK - - # Get total for progress bar - for event in events: - if event is not None: - for page in event["pages"]: - totalLines += len(page["list"]) - - # Process pages synchronously and persist after each - with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar: - pbar.desc = filename - pbar.total = totalLines - for event in events: - if event is not None: - for page in event["pages"]: - if page is not None: - try: - tt = searchCodes(page["list"], pbar, None, filename) - totalTokens[0] += tt[0] - totalTokens[1] += tt[1] - except Exception as e: - return [data, totalTokens, e] - finally: - maybe_save_progress_json(data, filename, tt) - return [data, totalTokens, None] - - -def searchCodes(events, pbar, jobList, filename): - # Lists - if jobList: - stringList = jobList[0] - list210 = jobList[1] - list300 = jobList[2] - list150 = jobList[3] - list250 = jobList[4] - list122 = jobList[5] - setData = True - else: - stringList = [] - list210 = [] - list300 = [] - list150 = [] - list250 = [] - list122 = [] - setData = False - - # Other - codeList = events - textHistory = [] - totalTokens = [0, 0] - translatedText = "" - speaker = "" - nametag = "" - initialJAString = "" - global LOCK, NAMESLIST, MISMATCH, PBAR, FILENAME - FILENAME = filename - PBAR = pbar - - # Calculate Total Length - code_flags = {102: CODE102, 122: CODE122, 300: CODE300, 250: CODE250} - totalList = 0 - for code_item in codeList: - if code_flags.get(code_item["code"], False): - totalList += 1 - pbar.total = totalList - pbar.refresh() - - # Begin Parsing File - try: - # Iterate through events - i = 0 - while i < len(codeList): - ### Event Code: 101 Message - if codeList[i]["code"] == 101 and CODE101 == True: - speakerRegex = r"@\d+\r?\n(.*?)：?\r?\n" # Default: r"@\d+\r?\n(.*)：?\r?\n"" - textRegex = r"@\d+\r?\n.*?：?\r?\n(.*)" # Default: r"@\d+\r?\n.*：?\r?\n(.*)" - # Alternative patterns for messages without @\d+ prefix - speakerRegexAlt = r"^(.*?)：\r?\n" # Matches "Name：\n" - textRegexAlt = r"^.*?：\r?\n(.*)" # Matches text after "Name：\n" - - # Grab String - jaString = codeList[i]["stringArgs"][0] - speaker = "" - - # Grab Speaker - if "\n" in jaString: - match = re.search(speakerRegex, jaString) - if not match: - # Try alternative pattern without @\d+ prefix - match = re.search(speakerRegexAlt, jaString) - if match: - # TL Speaker - response = getSpeaker(match.group(1)) - speaker = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - - # Set Name - codeList[i]["stringArgs"][0] = codeList[i]["stringArgs"][0].replace(match.group(1), speaker) - - # Grab Only Text - match = re.search(textRegex, jaString, flags=re.DOTALL) - if not match: - # Try alternative pattern without @\d+ prefix - match = re.search(textRegexAlt, jaString, flags=re.DOTALL) - if not match and re.search(LANGREGEX, jaString): - # No speaker pattern found, treat entire string as text to translate - jaString = jaString - initialJAString = jaString - elif match: - jaString = match.group(1) - initialJAString = jaString - else: - jaString = None # Skip non-translatable text - - if jaString is not None: - - # Remove Textwrap - jaString = jaString.replace("\r", "") - jaString = jaString.replace("\n", " ") - - # 1st Pass (Save Text to List) - if not setData: - if speaker == "": - stringList.append(jaString) - else: - stringList.append(f"[{speaker}]: {jaString}") - - # 2nd Pass (Set Text) - else: - # Grab Translated String - translatedText = stringList[0] - - # Remove speaker - matchSpeakerList = re.findall(r"^(\[.+?\]\s?[|:]\s?)\s?", translatedText) - if len(matchSpeakerList) > 0: - translatedText = translatedText.replace(matchSpeakerList[0], "") - - # Textwrap - if FIXTEXTWRAP is True: - translatedText = dazedwrap.wrapText(translatedText, width=WIDTH) - - # Set Data - codeList[i]["stringArgs"][0] = codeList[i]["stringArgs"][0].replace(initialJAString, translatedText) - - # Reset Data and Pop Item - stringList.pop(0) - - ### Event Code: 102 Choices - if codeList[i]["code"] == 102 and CODE102 == True: - # Grab Choice List - choiceList = [] - jaChoiceList = codeList[i]["stringArgs"] - - # Filter Empty - for j in range(len(jaChoiceList)): - if jaChoiceList[j]: - choiceList.append(jaChoiceList[j]) - - # Translate - if 'jaString' in locals(): - choiceString = f"Previous Line: {jaString}\n\nReply with the {LANGUAGE} translation of the dialogue choice" - else: - choiceString = f"Reply with the {LANGUAGE} translation of the dialogue choice" - response = translateAI( - choiceList, - choiceString, - True, - ) - translatedChoiceList = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - - # Validate and Set Data - if len(translatedChoiceList) == len(choiceList): - for j in range(len(jaChoiceList)): - if jaChoiceList[j]: - codeList[i]["stringArgs"][j] = translatedChoiceList[0] - translatedChoiceList.pop(0) - - ### Event Code: 210 Common Event - if codeList[i]["code"] == 210 and CODE210 == True: - # Speaker Event - if "stringArgs" in codeList[i] and codeList[i]["intArgs"][0] == None and len(codeList[i]["stringArgs"]) == 2: - response = getSpeaker(codeList[i]["stringArgs"][1]) - totalTokens[1] += response[1][0] - totalTokens[1] += response[1][1] - - # Set Data - codeList[i]["stringArgs"][1] = response[0] - - # Logs - elif "stringArgs" in codeList[i] and codeList[i]["intArgs"][0] == 500725 and len(codeList[i]["stringArgs"]) == 2: - # Grab String - jaString = codeList[i]["stringArgs"][1] - initialJAString = jaString - - # Remove Textwrap - jaString = jaString.replace("\r", "") - jaString = jaString.replace("\n", " ") - - # 1st Pass (Save Text to List) - if not setData: - list210.append(jaString) - - # 2nd Pass (Set Text) - else: - # Grab Translated String - translatedText = list210[0] - - # Textwrap - if FIXTEXTWRAP is True: - translatedText = dazedwrap.wrapText(translatedText, width=WIDTH) - - # Set Data - codeList[i]["stringArgs"][1] = translatedText - - # Pop Item - list210.pop(0) - - ### Event Code: 122 SetString - if codeList[i]["code"] == 122 and CODE122 == True: - if "stringArgs" in codeList[i] and len(codeList[i]["stringArgs"]) > 0: - # Grab String - jaString = re.search(r"^\n?(.*)\n?$", codeList[i]["stringArgs"][0]) - if jaString: - jaString = jaString.group(1) - else: - jaString = codeList[i]["stringArgs"][0] - - originalString = jaString - - # Remove Textwrap - jaString = jaString.replace("\r", "") - jaString = jaString.replace("\n", " ") - - # Check if this is a translatable string - if ( - not re.search(r"\.[\w]+$", jaString) - and jaString != "" - and "_" not in jaString - and '",' not in jaString - and "/" not in jaString - and re.search(LANGREGEX, jaString) - ): - # 1st Pass (Save Text to List) - if not setData: - list122.append(jaString) - - # 2nd Pass (Set Text) - else: - if len(list122) > 0: - # Grab Translated String - translatedText = list122[0] - - # Textwrap - if FIXTEXTWRAP is True: - translatedText = dazedwrap.wrapText(translatedText, width=WIDTH) - - # Set Data - codeList[i]["stringArgs"][0] = codeList[i]["stringArgs"][0].replace(originalString, translatedText) - - # Pop Item - list122.pop(0) - - ### Event Code: 150 Picture String - if codeList[i]["code"] == 150 and CODE150 == True: - if "stringArgs" in codeList[i] and len(codeList[i]["stringArgs"]) > 0: - font150 = "\\f[8]" - # Grab String - jaString = codeList[i]["stringArgs"][0] - - # Japanses Text Only - if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", jaString): - i += 1 - continue - - # Remove Textwrap - # jaString = jaString.replace("\n", " ") - # jaString = jaString.replace("\r", "") - - # Translate Other Strings [Specific Files Only] - if ( - not re.search(r"\.[\w]+$", jaString) - and jaString != "" - and "_" not in jaString - and '",' not in jaString - # and ">" not in jaString - # and "<" not in jaString - ): - # Pass 1 (Save Text to List) - if not setData: - list150.append(jaString) - - # Pass 2 (Set Text) - else: - translatedText = list150[0] - - # Textwrap - # translatedText = dazedwrap.wrapText(translatedText, WIDTH) - - # Set String with font formatting - codeList[i]["stringArgs"][0] = re.sub(r'\\f\[(\d+)\]', lambda x: f'\\f[{int(x.group(1))-2}]', translatedText) - - # Pop processed item - list150.pop(0) - - ### Event Code: 300 Common Events - if codeList[i]["code"] == 300 and CODE300 == True and "stringArgs" in codeList[i] and len(codeList[i]["stringArgs"]) > 1: - # Choices - if codeList[i]["stringArgs"][0] == "[共]汎用ウィンドウ生成" or codeList[i]["stringArgs"][0] == "選択肢/確認": - # Grab String - choiceList = codeList[i]["stringArgs"][1].split(",") - - # Translate Question - question = codeList[i]['stringArgs'][2] - response = translateAI(question, "Reply with the {LANGUAGE} translation") - translatedText = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - - # Translate Question - jaString = translatedText - codeList[i]['stringArgs'][2] = translatedText - - # Translate Choices - if 'jaString' in locals() and jaString: - response = translateAI(choiceList, jaString) - else: - response = translateAI(choiceList, f"Reply with the {LANGUAGE} translation of the dialogue choice") - choiceListTL = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - - # Replace Commas - for j in range(len(choiceListTL)): - choiceListTL[j] = choiceListTL[j].replace(", ", "、") - - # Convert to String and Set - translatedText = ",".join(choiceListTL) - codeList[i]["stringArgs"][1] = translatedText - - # Dialogue - elif codeList[i]["stringArgs"][0] == "BTLメッセージ" or codeList[i]["stringArgs"][0] == "X[移]メニュー時文章表示": - jaString = codeList[i]["stringArgs"][1] - - # Pass 1 - if not setData: - # Remove Textwrap - jaString = jaString.replace("\n", " ") - jaString = jaString.replace("\r", "") - - # Append - list300.append(jaString) - - # Pass 2 - else: - # Add Textwrap and Font - translatedText = dazedwrap.wrapText(list300[0], WIDTH) - list300.pop(0) - - # Write to File - codeList[i]["stringArgs"][1] = translatedText - - # Dialogue - elif codeList[i]["stringArgs"][0] == "援護文章": - speakerRegex = r"@\d+\r?\n(.*)：\r?\n" # Default: r"@\d+\r?\n(.*)：\r?\n" - textRegex = r"@?\d*\r?\n?\u3000*([\w\W]+)\r?\n?" # Default: r"@?\d*\r?\n?\u3000*([\w\W]+)\r?\n?" - - # Grab String - jaString = codeList[i]["stringArgs"][1] - speaker = "" - - # Grab Speaker - if "：\n" in jaString or "：\r\n" in jaString: - match = re.search(speakerRegex, jaString) - if match: - # TL Speaker - response = getSpeaker(match.group(1)) - speaker = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - - # Set nametag and remove from string - codeList[i]["stringArgs"][1] = codeList[i]["stringArgs"][1].replace(match.group(1), speaker) - jaString = jaString.replace(match.group(0), "") - - # Grab Only Text - match = re.search(textRegex, jaString) - if match: - jaString = match.group(1) - initialJAString = jaString - - # Remove Textwrap - jaString = jaString.replace("\r", "") - jaString = jaString.replace("\n", " ") - - # 1st Pass (Save Text to List) - if not setData: - if speaker == "": - list300.append(jaString) - else: - list300.append(f"[{speaker}]: {jaString}") - - # 2nd Pass (Set Text) - else: - # Grab Translated String - translatedText = list300[0] - - # Remove speaker - matchSpeakerList = re.findall(r"^(\[.+?\]\s?[|:]\s?)\s?", translatedText) - if len(matchSpeakerList) > 0: - translatedText = translatedText.replace(matchSpeakerList[0], "") - - # Textwrap - if FIXTEXTWRAP is True: - translatedText = dazedwrap.wrapText(translatedText, width=WIDTH) - - # Set Data - codeList[i]["stringArgs"][1] = codeList[i]["stringArgs"][1].replace(initialJAString, translatedText) - - # Reset Data and Pop Item - list300.pop(0) - - ### Event Code: 250 DB Read/Writes - if codeList[i]["code"] == 250 and CODE250 == True: - # Validate size - stringArg = 0 - if len(codeList[i]["stringArgs"]) == 4: - if codeList[i]["stringArgs"][1] == "万能ｳｨﾝﾄﾞｳ一時DB"\ - and codeList[i]["stringArgs"][stringArg] != "": - # Font Size - fontSize = 0 - - # Grab String - jaString = codeList[i]["stringArgs"][stringArg] - - # Remove Textwrap - # jaString = jaString.replace("\r", "") - # jaString = jaString.replace("\n", " ") - - # Pass 1 (Save Text to List) - if not setData: - list250.append(jaString) - - # Pass 2 (Set Text) - else: - translatedText = list250[0] - list250.pop(0) - - # Textwrap - # translatedText = dazedwrap.wrapText(translatedText, WIDTH) - - # Add/Replace font formatting - # Remove existing font command if present - translatedText = re.sub(r'\\f\[\d+\]', '', translatedText) - if fontSize: - # Add new font command if fontSize is not 0 - if fontSize != "0" and fontSize != 0: - translatedText = f"\\f[{fontSize}]{translatedText}" - - # Set Data - codeList[i]["stringArgs"][stringArg] = translatedText - - ### Iterate - i += 1 - - # EOF - stringListTL = [] - list210TL = [] - list150TL = [] - list250TL = [] - list300TL = [] - list122TL = [] - setData = False - - # String List - if len(stringList) > 0: - pbar.total = len(stringList) - pbar.refresh() - response = translateAI(stringList, textHistory) - stringListTL = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - if len(stringListTL) != len(stringList): - with LOCK: - if filename not in MISMATCH: - MISMATCH.append(filename) - else: - setData = True - - # 210 List - if len(list210) > 0: - pbar.total = len(list210) - pbar.refresh() - response = translateAI(list210, textHistory) - list210TL = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - if len(list210TL) != len(list210): - with LOCK: - if filename not in MISMATCH: - MISMATCH.append(filename) - else: - setData = True - - # 250 List - if len(list250) > 0: - pbar.total = len(list250) - pbar.refresh() - response = translateAI(list250, textHistory) - list250TL = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - if len(list250TL) != len(list250): - with LOCK: - if filename not in MISMATCH: - MISMATCH.append(filename) - else: - setData = True - - # 300 List - if len(list300) > 0: - pbar.total = len(list300) - pbar.refresh() - response = translateAI(list300, textHistory) - list300TL = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - if len(list300TL) != len(list300): - with LOCK: - if filename not in MISMATCH: - MISMATCH.append(filename) - else: - setData = True - - # 150 List - if len(list150) > 0: - # Progress Bar - pbar.total = len(list150) - pbar.refresh() - - # Translate - response = translateAI( - list150, - textHistory, - True, - ) - list150TL = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - - # Check Mismatch - if len(list150TL) != len(list150): - with LOCK: - if filename not in MISMATCH: - MISMATCH.append(filename) - else: - list150 = list150TL - setData = True - - # 122 List - if len(list122) > 0: - pbar.total = len(list122) - pbar.refresh() - response = translateAI(list122, textHistory) - list122TL = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - if len(list122TL) != len(list122): - with LOCK: - if filename not in MISMATCH: - MISMATCH.append(filename) - else: - setData = True - - # Update jobList before recursive call - if setData == True: - stringList = [] - jobList = [stringListTL, list210TL, list300TL, list150TL, list250TL, list122TL] # Add list122TL - searchCodes(events, pbar, jobList, filename) - - else: - # Set Data - events = codeList - - except IndexError as e: - traceback.print_exc() - raise Exception(str(e) + "Failed to translate: " + initialJAString) from None - except Exception as e: - traceback.print_exc() - raise Exception(str(e) + "Failed to translate: " + initialJAString) from None - - return totalTokens - - -def formatDramon(jaString): - imageRegex = r"(\r?\n?_[a-zA-Z_\d/.]+\r?\n?)|(@-?\d?\r\n)|(@-?\d?)([^\r]\d?-?\d?[^\r]+?)(\r\n|$)|(_PDC)|(>\r\n)|(^[#])|(\r\n@$)|(/)|(_SS_)|(\r\n[@/]\s?-?\d?\r\n)" - - jaString = jaString.replace("\u3000", " ") - jaString = jaString.replace("#", "") - jaString = re.sub(r"([^\r])\n", r"\1\r\n", jaString) - - # Grab and Split - jaStringList = re.split(imageRegex, jaString) - - # Clean List - cleanedList = [x for x in jaStringList if x is not None and x != "" and x != "\r\n" and x != "_SS_"] - - # Iterate Through List - j = 0 - translatedText = "" - while j < len(cleanedList): - if ( - ("@" in cleanedList[j] or "/" in cleanedList[j]) - and j < len(cleanedList) - 1 - and re.search(r"([@/]-?\d?\r\n)", cleanedList[j]) is None - and ".ogg" not in cleanedList[j] - ): - # Setup @ - if j > 0 and "@" not in cleanedList[j - 1] and "/" not in cleanedList[j - 1] and "_" not in cleanedList[j - 1]: - cleanedList[j - 1] = cleanedList[j - 1] + cleanedList[j + 1] - else: - cleanedList.insert(j, cleanedList[j + 1]) - j += 1 - cleanedList[j] = f"\r\n{cleanedList[j]}\r\n" - cleanedList.pop(j + 1) - j += 1 - - return cleanedList - -def handleScenarioScript(jaString, scriptString=""): - # Extract Speaker - # Updated regex to match speaker names with spaces, hyphens, and other characters - # [^\n：]+ matches any character except newline and ： (the speaker name) - scriptRegex = r"\n?(//.*?\n|//.*|/b.*?\n|[^\n：]+：\d*\r?\n)" - match = re.search(scriptRegex, jaString) - if match: - if "//" in match.group(1): - scriptStringLocal = match.group(1) - jaString = jaString.replace(scriptStringLocal, "") - scriptString += scriptStringLocal - match = re.search(scriptRegex, jaString) - if not match: - return None - else: - if "//" in jaString: - return handleScenarioScript(jaString, scriptString) - return [jaString, match.group(1), scriptString] - else: - return None - -# Database -def searchDB(events, pbar, jobList, filename): - # Set Lists - if len(jobList) > 0: - scenarioList = jobList[0] - npcList = jobList[1] - itemList = jobList[2] - stateList = jobList[3] - armorList = jobList[4] - enemyList = jobList[5] - weaponsList = jobList[6] - skillList = jobList[7] - optionsList = jobList[8] - dbNameList = jobList[9] - dbValueList = jobList[10] - setData = True - else: - scenarioList = [[], [], []] - npcList = [[], [], [], []] - itemList = [[], [], [], []] - armorList = [[], []] - enemyList = [[], []] - weaponsList = [[], [], [], []] - skillList = [[], [], [], [], []] - stateList = [[], [], [], [], [], [], [], []] - optionsList = [[], [], []] - dbNameList = [[]] - dbValueList = [[]] - setData = False - - # Vars/Globals - totalTokens = [0, 0] - initialJAString = "" - tableList = events - font = "" - global LOCK - global NAMESLIST - global MISMATCH - global PBAR - PBAR = pbar - - # Begin Parsing File - try: - for table in tableList: - # Grab NPCs - if table["name"] == "アイドル" and NPCFLAG == True: - with open("log/translations.txt", "a", encoding="utf-8") as file: - if setData: - file.write(f"\n#Actors\n") - for npc in table["data"]: - dataList = npc["data"] - - # Parse - for j in range(len(dataList)): - # Name - if dataList[j].get("name") == "出身": - # Pass 1 (Grab Data) - if setData == False: - if dataList[j].get("value") != "": - npcList[0].append(dataList[j].get("value")) - - # Pass 2 (Set Data) - else: - if dataList[j].get("value") != "": - # Write Name - file.write(f"{dataList[j].get('value')} ({npcList[0][0]})\n") - - # Set Data - dataList[j].update({"value": npcList[0][0]}) - npcList[0].pop(0) - - # Description - if dataList[j].get("name") == "説明": - # Pass 1 (Grab Data) - if setData == False: - if dataList[j].get("value") != "": - # Remove Textwrap - jaString = dataList[j].get("value") - jaString = jaString.replace("\n", " ") - jaString = jaString.replace("\r", "") - jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString) - - # Append Data - npcList[1].append(jaString) - # Pass 2 (Set Data) - else: - if dataList[j].get("value") != "": - # Textwrap - translatedText = npcList[1][0] - translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH) - translatedText = font + translatedText - - # Set Data - dataList[j].update({"value": translatedText}) - npcList[1].pop(0) - - # Grab Scenario - if "デートキャラ" in table["name"] and SCENARIOFLAG == True: - for scenario in table["data"]: - dataList = scenario["data"] - - # Parse - for j in range(len(dataList)): - # Name - if dataList[j].get("name") == "NULL": - if dataList[j].get("value"): - jaStringList = re.split(r'\r\n\r\n|\n\n', dataList[j].get("value")) - for jaString in jaStringList: - speakerNum = None - ogString = jaString - # Extract Speaker - speakerList = handleScenarioScript(jaString) - - # Handle simple names without speaker pattern - if not speakerList: - # Pass 1 (Grab Data) - Simple name - if setData == False: - jaString = jaString.replace("\n", " ") - jaString = jaString.replace("\r", "") - if jaString.strip(): - scenarioList[0].append(jaString) - # Pass 2 (Set Data) - Simple name - else: - if ogString.strip(): - translatedText = scenarioList[0][0] - scenarioList[0].pop(0) - dataList[j].update({"value": dataList[j].get("value").replace(ogString, translatedText)}) - continue - - if speakerList[2]: - jaString = speakerList[0] - speakerNumMatch = re.search(r"：(\d+)", speakerList[1]) - if speakerNumMatch: - speakerNum = speakerNumMatch.group(1) - speaker = re.sub(r"：\d*", "", speakerList[1]) - speaker = re.sub(r"/b", "NPC", speaker) - speaker = speaker.replace("\r\n", "") - - # Add Speaker - jaString = jaString.replace(speakerList[1], f"[{speaker}]: ") - - # Pass 1 (Grab Data) - if setData == False: - jaString = jaString.replace("\n", " ") - jaString = jaString.replace("\r", "") - scenarioList[0].append(jaString) - - # Pass 2 (Set Data) - else: - translatedText = scenarioList[0][0] - scenarioList[0].pop(0) - - # Remove Speaker before wrapping so WIDTH applies to dialogue only - speakerMatch = re.search(r"\[(.+?)\]\s?[|:]\s?", translatedText) - if speakerMatch: - speaker = speakerMatch.group(1).strip() - match = re.search(r'(^\[.+?\]\s?[|:]\s?)', translatedText) - if match: - translatedText = translatedText.replace(match.group(1), "") - - # Wrap text after removing speaker - translatedText = dazedwrap.wrapText(translatedText, WIDTH) - - # Redo Old Speaker Format - speaker = speaker.replace("Speaker", "\b") - if speakerNum: - translatedText = f"{speaker}：{speakerNum}\r\n{translatedText}" - elif speaker == "NPC": - translatedText = f"/b\r\n{translatedText}" - else: - translatedText = f"{speaker}：\r\n{translatedText}" - - # Add Script String - if speakerList[2]: - translatedText = f"{speakerList[2]}{translatedText}" - - # Set Data - dataList[j].update({"value": dataList[j].get("value").replace(ogString, translatedText)}) - # Description - if dataList[j].get("name") == "不正解会話": - if dataList[j].get("value"): - jaStringList = re.split(r'\r\n\r\n|\n\n', dataList[j].get("value")) - for jaString in jaStringList: - speakerNum = None - ogString = jaString - # Extract Speaker - speakerList = handleScenarioScript(jaString) - - # Handle simple text without speaker pattern - if not speakerList: - # Pass 1 (Grab Data) - Simple text - if setData == False: - jaString = jaString.replace("\n", " ") - jaString = jaString.replace("\r", "") - if jaString.strip(): - scenarioList[1].append(jaString) - # Pass 2 (Set Data) - Simple text - else: - if ogString.strip(): - translatedText = scenarioList[1][0] - scenarioList[1].pop(0) - translatedText = dazedwrap.wrapText(translatedText, WIDTH) - dataList[j].update({"value": dataList[j].get("value").replace(ogString, translatedText)}) - continue - - if speakerList[2]: - jaString = speakerList[0] - speakerNumMatch = re.search(r"：(\d+)", speakerList[1]) - if speakerNumMatch: - speakerNum = speakerNumMatch.group(1) - speaker = re.sub(r"：\d*", "", speakerList[1]) - speaker = re.sub(r"/b", "NPC", speaker) - speaker = speaker.replace("\r\n", "") - - # Add Speaker - jaString = jaString.replace(speakerList[1], f"[{speaker}]: ") - - # Pass 1 (Grab Data) - if setData == False: - jaString = jaString.replace("\n", " ") - jaString = jaString.replace("\r", "") - scenarioList[1].append(jaString) - - # Pass 2 (Set Data) - else: - translatedText = scenarioList[1][0] - scenarioList[1].pop(0) - - # Remove Speaker before wrapping so WIDTH applies to dialogue only - speakerMatch = re.search(r"\[(.+?)\]\s?[|:]\s?", translatedText) - if speakerMatch: - speaker = speakerMatch.group(1).strip() - match = re.search(r'(^\[.+?\]\s?[|:]\s?)', translatedText) - if match: - translatedText = translatedText.replace(match.group(1), "") - - # Wrap text after removing speaker - translatedText = dazedwrap.wrapText(translatedText, WIDTH) - - # Redo Old Speaker Format - speaker = speaker.replace("Speaker", "\b") - if speakerNum: - translatedText = f"{speaker}：{speakerNum}\r\n{translatedText}" - elif speaker == "NPC": - translatedText = f"/b\r\n{translatedText}" - else: - translatedText = f"{speaker}：\r\n{translatedText}" - - # Add Script String - if speakerList[2]: - translatedText = f"{speakerList[2]}{translatedText}" - - # Set Data - dataList[j].update({"value": dataList[j].get("value").replace(ogString, translatedText)}) - - # Description - if dataList[j].get("name") == "正解会話": - if dataList[j].get("value"): - jaStringList = re.split(r'\r\n\r\n|\n\n', dataList[j].get("value")) - for jaString in jaStringList: - speakerNum = None - ogString = jaString - # Extract Speaker - speakerList = handleScenarioScript(jaString) - - # Handle simple text without speaker pattern - if not speakerList: - # Pass 1 (Grab Data) - Simple text - if setData == False: - jaString = jaString.replace("\n", " ") - jaString = jaString.replace("\r", "") - if jaString.strip(): - scenarioList[2].append(jaString) - # Pass 2 (Set Data) - Simple text - else: - if ogString.strip(): - translatedText = scenarioList[2][0] - scenarioList[2].pop(0) - translatedText = dazedwrap.wrapText(translatedText, WIDTH) - dataList[j].update({"value": dataList[j].get("value").replace(ogString, translatedText)}) - continue - if speakerList[2]: - jaString = speakerList[0] - speakerNumMatch = re.search(r"：(\d+)", speakerList[1]) - if speakerNumMatch: - speakerNum = speakerNumMatch.group(1) - speaker = re.sub(r"：\d*", "", speakerList[1]) - speaker = re.sub(r"/b", "NPC", speaker) - speaker = speaker.replace("\r\n", "") - - # Add Speaker - jaString = jaString.replace(speakerList[1], f"[{speaker}]: ") - - # Pass 1 (Grab Data) - if setData == False: - jaString = jaString.replace("\n", " ") - jaString = jaString.replace("\r", "") - scenarioList[2].append(jaString) - - # Pass 2 (Set Data) - else: - translatedText = scenarioList[2][0] - scenarioList[2].pop(0) - - # Remove Speaker before wrapping so WIDTH applies to dialogue only - speakerMatch = re.search(r"\[(.+?)\]\s?[|:]\s?", translatedText) - if speakerMatch: - speaker = speakerMatch.group(1).strip() - match = re.search(r'(^\[.+?\]\s?[|:]\s?)', translatedText) - if match: - translatedText = translatedText.replace(match.group(1), "") - - # Wrap text after removing speaker - translatedText = dazedwrap.wrapText(translatedText, WIDTH) - - # Redo Old Speaker Format - speaker = speaker.replace("Speaker", "\b") - if speakerNum: - translatedText = f"{speaker}：{speakerNum}\r\n{translatedText}" - elif speaker == "NPC": - translatedText = f"/b\r\n{translatedText}" - else: - translatedText = f"{speaker}：\r\n{translatedText}" - - # Add Script String - if speakerList[2]: - translatedText = f"{speakerList[2]}{translatedText}" - - # Set Data - dataList[j].update({"value": dataList[j].get("value").replace(ogString, translatedText)}) - - # Grab Options - if table["name"] == "クイズ" and OPTIONSFLAG == True: - for option in table["data"]: - dataList = option["data"] - - # Parse - for j in range(len(dataList)): - # Name - if dataList[j].get("name") == "問": - if dataList[j].get("value"): - jaString = dataList[j].get("value") - # Pass 1 (Grab Data) - if setData == False: - optionsList[0].append(jaString) - - # Pass 2 (Set Data) - else: - translatedText = optionsList[0][0] - optionsList[0].pop(0) - dataList[j].update({"value": dataList[j].get("value").replace(jaString, translatedText)}) - - # Description 1 - if dataList[j].get("name") == "3": - if dataList[j].get("value"): - jaString = dataList[j].get("value") - # Pass 1 (Grab Data) - if setData == False: - jaString = jaString.replace("\n", " ") - jaString = jaString.replace("\r", "") - optionsList[1].append(jaString) - - # Pass 2 (Set Data) - else: - translatedText = optionsList[1][0] - optionsList[1].pop(0) - translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH) - dataList[j].update({"value": dataList[j].get("value").replace(jaString, translatedText)}) - - # Description 2 - if dataList[j].get("name") == "4": - if dataList[j].get("value"): - jaString = dataList[j].get("value") - # Pass 1 (Grab Data) - if setData == False: - jaString = jaString.replace("\n", " ") - jaString = jaString.replace("\r", "") - optionsList[2].append(jaString) - - # Pass 2 (Set Data) - else: - translatedText = optionsList[2][0] - optionsList[2].pop(0) - translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH) - dataList[j].update({"value": dataList[j].get("value").replace(jaString, translatedText)}) - - # Grab DB Names - if table["name"] == "マップ設定" and DBNAMEFLAG == True: - font = None - dbName = table["data"] - for j in range(len(dbName)): - # Pass 1 (Grab Data) - if setData == False: - if dbName[j].get("name") != "": - dbNameList[0].append(dbName[j].get("name")) - - # Pass 2 (Set Data) - else: - if dbName[j].get("name") != "": - dbName[j].update({"name": dbNameList[0][0]}) - dbNameList[0].pop(0) - - # Grab DB Values - if table["name"] == "近況文章" and DBVALUEFLAG == True: - font = 16 - subject = "" - for entry in table["data"]: - dbValue = entry["data"] - for j in range(len(dbValue)): - if dbValue[j].get("value"): - jaString = dbValue[j].get("value") - - # Speaker - match = re.search(r"(^[\\]+f\[\d+\].+?\r\n).+", jaString) - if match: - subject = match.group(1) - jaString = jaString.replace(subject, "") - - # Nuke Textwrap & Font - jaString = jaString.replace("\r", "") - jaString = jaString.replace("\n", " ") - jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString) - - # Pass 1 (Grab Data) - if setData == False: - # Append - dbValueList[0].append(jaString) - - # Pass 2 (Set Data) - else: - if dbValue[j].get("value"): - translatedText = dbValueList[0][0] - - # Textwrap - translatedText = dazedwrap.wrapText(translatedText, 30) - translatedText = translatedText.replace("\n", f"\r\n\\f[{font}]") - - # Subject - if subject: - translatedText = f"{subject}{translatedText}" - - # Set - dbValue[j].update({"value": translatedText}) - dbValueList[0].pop(0) - - # Grab Items - if table["name"] == "道具" and ITEMFLAG == True: - # Write Category - if setData: - with open("log/translations.txt", "a", encoding="utf-8") as file: - file.write(f"\n#Items\n") - - # Begin Translation - for item in table["data"]: - dataList = item["data"] - - # Parse - font = 18 - for j in range(len(dataList)): - # Name - if dataList[j].get("name") == "名前": - jaString = dataList[j].get("value") - if jaString != "": - # Pass 1 (Grab Data) - if setData == False: - if jaString != "": - itemList[0].append(jaString) - - # Pass 2 (Set Data) - else: - # Write to TL File - with open("log/translations.txt", "a", encoding="utf-8") as file: - file.write(f"{jaString} ({itemList[0][0]})\n") - - dataList[j].update({"value": itemList[0][0]}) - itemList[0].pop(0) - - # Description - if dataList[j].get("name") == "説明": - # Pass 1 (Grab Data) - if setData == False: - if dataList[j].get("value") != "": - # Remove Textwrap & Font - jaString = dataList[j].get("value") - jaString = jaString.replace("\n", " ") - jaString = jaString.replace("\r", "") - jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString) - - # Append Data - itemList[1].append(jaString) - # Pass 2 (Set Data) - else: - if dataList[j].get("value") != "": - # Textwrap - translatedText = itemList[1][0] - translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH) - - # Font - if font: - translatedText = f"\\f[{font}]{translatedText}" - - # Set Data - dataList[j].update({"value": translatedText}) - itemList[1].pop(0) - - # Log - if dataList[j].get("name") == "使用時文章[戦](人名~": - # Pass 1 (Grab Data) - if setData == False: - if dataList[j].get("value") != "": - # Remove Textwrap - jaString = dataList[j].get("value") - jaString = jaString.replace("\n", " ") - jaString = jaString.replace("\r", "") - jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString) - - # Skill Action - if jaString[0] in [ - "は", - "を", - "の", - "に", - "が", - ]: - jaString = f"Taro{jaString}" - - # Append Data - itemList[2].append(jaString) - # Pass 2 (Set Data) - else: - if dataList[j].get("value") != "": - # Textwrap - translatedText = itemList[2][0] - translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH) - # translatedText = font + translatedText - - # Remove Taro - translatedText = re.sub(r"\bTaro\b", "", translatedText) - - # Set Data - dataList[j].update({"value": translatedText}) - itemList[2].pop(0) - - # Description - if dataList[j].get("name") == "使用後文章[移動]": - # Pass 1 (Grab Data) - if setData == False: - if dataList[j].get("value") != "": - # Remove Textwrap - jaString = dataList[j].get("value") - jaString = jaString.replace("\n", " ") - jaString = jaString.replace("\r", "") - jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString) - - # Append Data - itemList[3].append(jaString) - # Pass 2 (Set Data) - else: - if dataList[j].get("value") != "": - # Textwrap - translatedText = itemList[3][0] - translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH) - - # Font - if font: - translatedText = f"\\f[{font}]{translatedText}" - - # Set Data - dataList[j].update({"value": translatedText}) - itemList[3].pop(0) - - # Grab Armors - if table["name"] == "防具" and ARMORFLAG == True: - font = "24" - for armor in table["data"]: - dataList = armor["data"] - - # Parse - for j in range(len(dataList)): - # Name - if dataList[j].get("name") == "名前": - # Pass 1 (Grab Data) - if setData == False: - if dataList[j].get("value") != "": - armorList[0].append(dataList[j].get("value")) - - # Pass 2 (Set Data) - else: - if dataList[j].get("value") != "": - dataList[j].update({"value": armorList[0][0]}) - armorList[0].pop(0) - - # Description - if dataList[j].get("name") == "説明": - # Pass 1 (Grab Data) - if setData == False: - if dataList[j].get("value") != "": - # Remove Textwrap - jaString = dataList[j].get("value") - jaString = jaString.replace("\n", " ") - jaString = jaString.replace("\r", "") - jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString) - - # Append Data - armorList[1].append(jaString) - # Pass 2 (Set Data) - else: - if dataList[j].get("value") != "": - # Textwrap - translatedText = armorList[1][0] - translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH) - - # Font - if font: - translatedText = f"\\f[{font}]{translatedText}" - - # Set Data - dataList[j].update({"value": translatedText}) - armorList[1].pop(0) - - # Grab Enemies - if table["name"] == "敵" and ENEMYFLAG == True: - for enemy in table["data"]: - dataList = enemy["data"] - - # Parse - for j in range(len(dataList)): - # Name - if dataList[j].get("name") == "敵キャラ名": - # Pass 1 (Grab Data) - if setData == False: - if dataList[j].get("value") != "": - enemyList[0].append(dataList[j].get("value")) - - # Pass 2 (Set Data) - else: - if dataList[j].get("value") != "": - dataList[j].update({"value": enemyList[0][0]}) - enemyList[0].pop(0) - - # Description - if dataList[j].get("name") == "NULL": - # Pass 1 (Grab Data) - if setData == False: - if dataList[j].get("value") != "": - # Remove Textwrap - jaString = dataList[j].get("value") - jaString = jaString.replace("\n", " ") - jaString = jaString.replace("\r", "") - jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString) - - # Append Data - enemyList[1].append(jaString) - # Pass 2 (Set Data) - else: - if dataList[j].get("value") != "": - # Textwrap - translatedText = enemyList[1][0] - translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH) - - # Font - if font: - translatedText = f"\\f[{font}]{translatedText}" - - # Set Data - dataList[j].update({"value": translatedText}) - enemyList[1].pop(0) - - # Grab Weapons - if table["name"] == "武器" and WEAPONFLAG == True: - font = "24" - for weapon in table["data"]: - dataList = weapon["data"] - - # Parse - for j in range(len(dataList)): - # Name - if dataList[j].get("name") == "名前": - # Pass 1 (Grab Data) - if setData == False: - if dataList[j].get("value") != "": - weaponsList[0].append(dataList[j].get("value")) - - # Pass 2 (Set Data) - else: - if dataList[j].get("value") != "": - dataList[j].update({"value": weaponsList[0][0]}) - weaponsList[0].pop(0) - - # Description - if dataList[j].get("name") == "説明": - # Pass 1 (Grab Data) - if setData == False: - if dataList[j].get("value") != "": - # Remove Textwrap - jaString = dataList[j].get("value") - jaString = jaString.replace("\n", " ") - jaString = jaString.replace("\r", "") - jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString) - - # Append Data - weaponsList[1].append(jaString) - # Pass 2 (Set Data) - else: - if dataList[j].get("value") != "": - # Textwrap - translatedText = weaponsList[1][0] - translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH) - - # Font - if font: - translatedText = f"\\f[{font}]{translatedText}" - - # Set Data - dataList[j].update({"value": translatedText}) - weaponsList[1].pop(0) - - # Grab Skills - if table["name"] == "技能" and SKILLFLAG == True: - font = "24" - for skill in table["data"]: - dataList = skill["data"] - - # Parse - for j in range(len(dataList)): - # Name - if dataList[j].get("name") == "名前": - # Pass 1 (Grab Data) - if setData == False: - if dataList[j].get("value") != "": - skillList[0].append(dataList[j].get("value")) - - # Pass 2 (Set Data) - else: - if dataList[j].get("value") != "": - dataList[j].update({"value": skillList[0][0]}) - skillList[0].pop(0) - - # Description - if dataList[j].get("name") == "説明": - # Pass 1 (Grab Data) - if setData == False: - if dataList[j].get("value") != "": - # Remove Textwrap - jaString = dataList[j].get("value") - jaString = jaString.replace("\n", " ") - jaString = jaString.replace("\r", "") - jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString) - - # Append Data - skillList[1].append(jaString) - - # Pass 2 (Set Data) - else: - if dataList[j].get("value") != "": - # Textwrap - translatedText = skillList[1][0] - translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH) - - # Font - if font: - translatedText = f"\\f[{font}]{translatedText}" - - # Set Data - dataList[j].update({"value": translatedText}) - skillList[1].pop(0) - - # Log - if dataList[j].get("name") == "発動時文章": - # Pass 1 (Grab Data) - if setData == False: - if dataList[j].get("value") != "": - # Remove Textwrap - jaString = dataList[j].get("value") - jaString = jaString.replace("\n", " ") - jaString = jaString.replace("\r", "") - jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString) - - # Skill Action - if jaString[0] in [ - "は", - "を", - "の", - "に", - "が", - ]: - jaString = f"Taro{jaString}" - - # Append Data - skillList[2].append(jaString) - # Pass 2 (Set Data) - else: - if dataList[j].get("value") != "": - # Textwrap - translatedText = skillList[2][0] - translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH) - translatedText = font + translatedText - - # Remove Taro - translatedText = re.sub(r"\bTaro\b", "", translatedText) - - # Set Data - dataList[j].update({"value": translatedText}) - skillList[2].pop(0) - - # Log - if dataList[j].get("name") == "使用時文章[戦闘](人名~": - # Pass 1 (Grab Data) - if setData == False: - if dataList[j].get("value") != "": - # Remove Textwrap - jaString = dataList[j].get("value") - jaString = jaString.replace("\n", " ") - jaString = jaString.replace("\r", "") - jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString) - - # Skill Action - if jaString[0] in [ - "は", - "を", - "の", - "に", - "が", - ]: - jaString = f"Taro{jaString}" - - # Append Data - skillList[3].append(jaString) - # Pass 2 (Set Data) - else: - if dataList[j].get("value") != "": - # Textwrap - translatedText = skillList[3][0] - translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH) - - # Remove Taro - translatedText = re.sub(r"\bTaro\b", "", translatedText) - - # Font - translatedText = font + translatedText - - # Set Data - dataList[j].update({"value": translatedText}) - skillList[3].pop(0) - - # Log - if dataList[j].get("name") == "失敗時文章[(対象)～]": - # Pass 1 (Grab Data) - if setData == False: - if dataList[j].get("value") != "": - # Remove Textwrap - jaString = dataList[j].get("value") - jaString = jaString.replace("\n", " ") - jaString = jaString.replace("\r", "") - jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString) - - # Skill Action - if jaString[0] in [ - "は", - "を", - "の", - "に", - "が", - ]: - jaString = f"Taro{jaString}" - - # Append Data - skillList[4].append(jaString) - # Pass 2 (Set Data) - else: - if dataList[j].get("value") != "": - # Textwrap - translatedText = skillList[4][0] - translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH) - translatedText = font + translatedText - - # Remove Taro - translatedText = re.sub(r"\bTaro\b", "", translatedText) - - # Set Data - dataList[j].update({"value": translatedText}) - skillList[4].pop(0) - - # Grab States - if table["name"] == "ステート" and STATEFLAG == True: - for state in table["data"]: - dataList = state["data"] - - # Parse - for j in range(len(dataList)): - # Name - if dataList[j].get("name") == "名前": - # Pass 1 (Grab Data) - if setData == False: - if dataList[j].get("value") != "": - stateList[0].append(dataList[j].get("value")) - - # Pass 2 (Set Data) - else: - if dataList[j].get("value") != "": - dataList[j].update({"value": stateList[0][0]}) - stateList[0].pop(0) - - # Description - if dataList[j].get("name") == "表示名": - # Pass 1 (Grab Data) - if setData == False: - if dataList[j].get("value") != "": - # Remove Textwrap - jaString = dataList[j].get("value") - jaString = jaString.replace("\n", " ") - jaString = jaString.replace("\r", "") - jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString) - - # Append Data - stateList[1].append(jaString) - # Pass 2 (Set Data) - else: - if dataList[j].get("value") != "": - # Textwrap - translatedText = stateList[1][0] - translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH) - translatedText = font + translatedText - - # Set Data - dataList[j].update({"value": translatedText}) - stateList[1].pop(0) - - # Log - if dataList[j].get("name") == "付与時文章": - # Pass 1 (Grab Data) - if setData == False: - if dataList[j].get("value") != "": - # Remove Textwrap - jaString = dataList[j].get("value") - jaString = jaString.replace("\n", " ") - jaString = jaString.replace("\r", "") - jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString) - - # State Action - if jaString[0] in [ - "は", - "を", - "の", - "に", - "が", - ]: - jaString = f"Taro{jaString}" - - # Append Data - stateList[2].append(jaString) - # Pass 2 (Set Data) - else: - if dataList[j].get("value") != "": - # Textwrap - translatedText = stateList[2][0] - translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH) - translatedText = font + translatedText - - # Remove Taro - translatedText = re.sub(r"\bTaro\b", "", translatedText) - - # Set Data - dataList[j].update({"value": translatedText}) - stateList[2].pop(0) - - # Log - if dataList[j].get("name") == "解除時文章": - # Pass 1 (Grab Data) - if setData == False: - if dataList[j].get("value") != "": - # Remove Textwrap - jaString = dataList[j].get("value") - jaString = jaString.replace("\n", " ") - jaString = jaString.replace("\r", "") - jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString) - - # State Action - if jaString[0] in [ - "は", - "を", - "の", - "に", - "が", - ]: - jaString = f"Taro{jaString}" - - # Append Data - stateList[3].append(jaString) - # Pass 2 (Set Data) - else: - if dataList[j].get("value") != "": - # Textwrap - translatedText = stateList[3][0] - translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH) - translatedText = font + translatedText - - # Remove Taro - translatedText = re.sub(r"\bTaro\b", "", translatedText) - - # Set Data - dataList[j].update({"value": translatedText}) - stateList[3].pop(0) - - # Log - if dataList[j].get("name") == "回復時の文章[(人名)～]": - # Pass 1 (Grab Data) - if setData == False: - if dataList[j].get("value") != "": - # Remove Textwrap - jaString = dataList[j].get("value") - jaString = jaString.replace("\n", " ") - jaString = jaString.replace("\r", "") - jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString) - - # State Action - if jaString[0] in [ - "は", - "を", - "の", - "に", - "が", - ]: - jaString = f"Taro{jaString}" - - # Append Data - stateList[4].append(jaString) - # Pass 2 (Set Data) - else: - if dataList[j].get("value") != "": - # Textwrap - translatedText = stateList[4][0] - translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH) - translatedText = font + translatedText - - # Remove Taro - translatedText = re.sub(r"\bTaro\b", "", translatedText) - - # Set Data - dataList[j].update({"value": translatedText}) - stateList[4].pop(0) - # Log - if dataList[j].get("name") == "┣ ｶｳﾝﾀｰ発動文[対象～": - # Pass 1 (Grab Data) - if setData == False: - if dataList[j].get("value") != "": - # Remove Textwrap - jaString = dataList[j].get("value") - jaString = jaString.replace("\n", " ") - jaString = jaString.replace("\r", "") - jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString) - - # State Action - if jaString[0] in [ - "は", - "を", - "の", - "に", - "が", - ]: - jaString = f"Taro{jaString}" - - # Append Data - stateList[5].append(jaString) - # Pass 2 (Set Data) - else: - if dataList[j].get("value") != "": - # Textwrap - translatedText = stateList[5][0] - translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH) - translatedText = font + translatedText - - # Remove Taro - translatedText = re.sub(r"\bTaro\b", "", translatedText) - - # Set Data - dataList[j].update({"value": translatedText}) - stateList[5].pop(0) - - # Log - if dataList[j].get("name") == "尻もち　行動不能　持続3ターン": - # Pass 1 (Grab Data) - if setData == False: - if dataList[j].get("value") != "": - # Remove Textwrap - jaString = dataList[j].get("value") - jaString = jaString.replace("\n", " ") - jaString = jaString.replace("\r", "") - jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString) - - # State Action - if jaString[0] in [ - "は", - "を", - "の", - "に", - "が", - ]: - jaString = f"Taro{jaString}" - - # Append Data - stateList[6].append(jaString) - # Pass 2 (Set Data) - else: - if dataList[j].get("value") != "": - # Textwrap - translatedText = stateList[6][0] - translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH) - translatedText = font + translatedText - - # Remove Taro - translatedText = re.sub(r"\bTaro\b", "", translatedText) - - # Set Data - dataList[j].update({"value": translatedText}) - stateList[6].pop(0) - - # Log - if dataList[j].get("name") == "状態異常の説明": - # Pass 1 (Grab Data) - if setData == False: - if dataList[j].get("value") != "": - # Remove Textwrap - jaString = dataList[j].get("value") - jaString = jaString.replace("\n", " ") - jaString = jaString.replace("\r", "") - jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString) - - # State Action - if jaString[0] in [ - "は", - "を", - "の", - "に", - "が", - ]: - jaString = f"Taro{jaString}" - - # Append Data - stateList[7].append(jaString) - # Pass 2 (Set Data) - else: - if dataList[j].get("value") != "": - # Textwrap - translatedText = stateList[7][0] - translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH) - translatedText = font + translatedText - - # Remove Taro - translatedText = re.sub(r"\bTaro\b", "", translatedText) - - # Set Data - dataList[j].update({"value": translatedText}) - stateList[7].pop(0) - - # Translation - scenarioListTL = [[], [], []] - optionsListTL = [[], [], []] - npcListTL = [[], [], [], []] - itemListTL = [[], [], [], []] - stateListTL = [[], [], [], [], [], [], [], []] - armorListTL = [[], []] - enemyListTL = [[], []] - weaponsListTL = [[], [], []] - skillListTL = [[], [], [], [], []] - dbNameListTL = [[]] - dbValueListTL = [[]] - - translate = False - - # NPCs - if len(npcList[0]) > 0: - # Progress Bar - total = 0 - for itemArray in npcList: - total += len(itemArray) - pbar.total = total - pbar.refresh() - - # Name - response = translateAI( - npcList[0], - "Reply with only the " + LANGUAGE + " translation of the RPG enemy name", - True, - ) - nameListTL = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - # Desc 1 - response = translateAI(npcList[1], "Reply with only the " + LANGUAGE + " translation") - descListTL1 = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - # Desc 2 - response = translateAI(npcList[2], "Reply with only the " + LANGUAGE + " translation") - descListTL2 = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - # Desc 3 - response = translateAI(npcList[3], "Reply with only the " + LANGUAGE + " translation") - descListTL3 = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - - # Check Mismatch - if ( - len(nameListTL) != len(npcList[0]) - or len(descListTL1) != len(npcList[1]) - or len(descListTL2) != len(npcList[2]) - or len(descListTL3) != len(npcList[3]) - ): - with LOCK: - if filename not in MISMATCH: - MISMATCH.append(filename) - else: - npcListTL = [nameListTL, descListTL1, descListTL2, descListTL3] - translate = True - - # SCENARIO - if scenarioList[0] or scenarioList[1]: - # Progress Bar - total = 0 - for scenarioArray in scenarioList: - total += len(scenarioArray) - pbar.total = total - pbar.refresh() - - # Name - response = translateAI( - scenarioList[0], - "Reply with only the " + LANGUAGE + " translation", - True, - ) - nameListTL = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - # Desc 1 - response = translateAI( - scenarioList[1], - "reply with only the gender neutral " + LANGUAGE + " translation of the NPC name", - True, - ) - descListTL1 = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - # Desc 2 - response = translateAI( - scenarioList[2], - "reply with only the gender neutral " + LANGUAGE + " translation of the NPC name", - True, - ) - descListTL2 = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - - # Check Mismatch - if len(nameListTL) != len(scenarioList[0]) or len(descListTL1) != len(scenarioList[1]) or len(descListTL2) != len(scenarioList[2]): - with LOCK: - if filename not in MISMATCH: - MISMATCH.append(filename) - else: - scenarioListTL = [nameListTL, descListTL1, descListTL2] - translate = True - - # OPTIONS - if optionsList[0] or optionsList[1]: - # Progress Bar - total = 0 - for optionsArray in optionsList: - total += len(optionsArray) - pbar.total = total - pbar.refresh() - - # Name - response = translateAI( - optionsList[0], - "Reply with only the " + LANGUAGE + " translation", - True, - ) - nameListTL = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - # Desc 1 - response = translateAI( - optionsList[1], - "reply with only the gender neutral " + LANGUAGE + " translation of the NPC name", - True, - ) - descListTL1 = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - # Desc 2 - response = translateAI( - optionsList[2], - "reply with only the gender neutral " + LANGUAGE + " translation of the NPC name", - True, - ) - descListTL2 = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - - # Check Mismatch - if len(nameListTL) != len(optionsList[0]) or len(descListTL1) != len(optionsList[1]) or len(descListTL2) != len(optionsList[2]): - with LOCK: - if filename not in MISMATCH: - MISMATCH.append(filename) - else: - optionsListTL = [nameListTL, descListTL1, descListTL2] - translate = True - - # ITEMS - if len(itemList[0]) > 0 or len(itemList[1]) > 0: - # Progress Bar - total = 0 - for itemArray in itemList: - total += len(itemArray) - pbar.total = total - pbar.refresh() - - # Name - response = translateAI(itemList[0], "Reply with only the " + LANGUAGE + " translation") - nameListTL = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - # Desc 1 - response = translateAI(itemList[1], "Reply with only the " + LANGUAGE + " translation") - descListTL1 = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - # Desc 2 - response = translateAI(itemList[2], "Reply with only the " + LANGUAGE + " translation") - descListTL2 = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - # Desc 3 - response = translateAI(itemList[3], "Reply with only the " + LANGUAGE + " translation") - descListTL3 = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - - # Check Mismatch - if ( - len(nameListTL) != len(itemList[0]) - or len(descListTL1) != len(itemList[1]) - or len(descListTL2) != len(itemList[2]) - or len(descListTL3) != len(itemList[3]) - ): - with LOCK: - if filename not in MISMATCH: - MISMATCH.append(filename) - else: - itemListTL = [nameListTL, descListTL1, descListTL2, descListTL3] - translate = True - - # Armor - if len(armorList[0]) > 0: - # Progress Bar - total = 0 - for armorArray in armorList: - total += len(armorArray) - pbar.total = total - pbar.refresh() - - # Name - response = translateAI( - armorList[0], - "Reply with only the " + LANGUAGE + " translation of the NPC name", - True, - ) - nameListTL = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - # Desc 1 - response = translateAI(armorList[1], "Reply with only the " + LANGUAGE + " translation") - descListTL1 = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - - # Check Mismatch - if len(nameListTL) != len(armorList[0]) or len(descListTL1) != len(armorList[1]): - with LOCK: - if filename not in MISMATCH: - MISMATCH.append(filename) - else: - armorListTL = [nameListTL, descListTL1] - translate = True - - # Enemies - if len(enemyList[0]) > 0: - # Progress Bar - total = 0 - for enemyArray in enemyList: - total += len(enemyArray) - pbar.total = total - pbar.refresh() - - # Name - response = translateAI( - enemyList[0], - "Reply with only the " + LANGUAGE + " translation of the enemy NPC name", - True, - ) - nameListTL = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - # Desc 1 - response = translateAI(enemyList[1], "Reply with only the " + LANGUAGE + " translation") - descListTL1 = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - - # Check Mismatch - if len(nameListTL) != len(enemyList[0]) or len(descListTL1) != len(enemyList[1]): - with LOCK: - if filename not in MISMATCH: - MISMATCH.append(filename) - else: - enemyListTL = [nameListTL, descListTL1] - translate = True - - # Weapons - if len(weaponsList[0]) > 0: - # Progress Bar - total = 0 - for weaponsArray in weaponsList: - total += len(weaponsArray) - pbar.total = total - pbar.refresh() - - # Name - response = translateAI( - weaponsList[0], - "Reply with only the " + LANGUAGE + " translation of the RPG weapon name", - True, - ) - nameListTL = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - # Desc 1 - response = translateAI(weaponsList[1], "") - descListTL1 = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - # Desc 2 - response = translateAI(weaponsList[2], "") - descListTL2 = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - - # Check Mismatch - if len(nameListTL) != len(weaponsList[0]) or len(descListTL1) != len(weaponsList[1]) or len(descListTL2) != len(weaponsList[2]): - with LOCK: - if filename not in MISMATCH: - MISMATCH.append(filename) - else: - weaponsListTL = [nameListTL, descListTL1, descListTL2] - translate = True - - # Skills - if len(skillList[0]) > 0: - # Progress Bar - total = 0 - for skillArray in skillList: - total += len(skillArray) - pbar.total = total - pbar.refresh() - - # Name - response = translateAI( - skillList[0], - "Reply with only the " + LANGUAGE + " translation of the RPG skill name", - True, - ) - nameListTL = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - - # Desc - response = translateAI( - skillList[1], - "Reply with only the " + LANGUAGE + " translation of the RPG skill description", - True, - ) - descListTL1 = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - - # Log 1 - response = translateAI( - skillList[2], - "reply with only the gender neutral " - + LANGUAGE - + " translation of the action log. Always start the sentence with Taro. For example, Translate 'Taroを倒した！' as 'Taro was defeated!'", - True, - ) - descListTL2 = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - - # Log 2 - response = translateAI( - skillList[3], - "reply with only the gender neutral " - + LANGUAGE - + " translation of the action log. Always start the sentence with Taro. For example, Translate 'Taroを倒した！' as 'Taro was defeated!'", - True, - ) - descListTL3 = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - - # Log 3 - response = translateAI( - skillList[4], - "reply with only the gender neutral " - + LANGUAGE - + " translation of the action log. Always start the sentence with Taro. For example, Translate 'Taroを倒した！' as 'Taro was defeated!'", - True, - ) - descListTL4 = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - - # Check Mismatch - if ( - len(nameListTL) != len(skillList[0]) - or len(descListTL1) != len(skillList[1]) - or len(descListTL2) != len(skillList[2]) - or len(descListTL3) != len(skillList[3]) - or len(descListTL4) != len(skillList[4]) - ): - with LOCK: - if filename not in MISMATCH: - MISMATCH.append(filename) - else: - skillListTL = [nameListTL, descListTL1, descListTL2, descListTL3, descListTL4] - translate = True - - # State - for list in stateList: - if len(list) > 0: - # Progress Bar - total = 0 - for stateArray in stateList: - total += len(stateArray) - pbar.total = total - pbar.refresh() - - # Name - response = translateAI( - stateList[0], - f"Reply with the {LANGUAGE} translation of the status effect.", - True, - ) - nameListTL = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - - # Desc 1 - response = translateAI(stateList[1], "") - descListTL1 = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - - # Log 1 - response = translateAI( - stateList[2], - f"Reply with the {LANGUAGE} translation of the status effect.", - True, - ) - descListTL2 = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - - # Log 2 - response = translateAI( - stateList[3], - "reply with only the gender neutral " - + LANGUAGE - + " translation of the action log. Always start the sentence with Taro. For example, Translate 'Taroを倒した！' as 'Taro was defeated!'", - True, - ) - descListTL3 = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - - # Log 3 - response = translateAI( - stateList[4], - "reply with only the gender neutral " - + LANGUAGE - + " translation of the action log. Always start the sentence with Taro. For example, Translate 'Taroを倒した！' as 'Taro was defeated!'", - True, - ) - descListTL4 = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - - # Log 4 - response = translateAI( - stateList[5], - "reply with only the gender neutral " - + LANGUAGE - + " translation of the action log. Always start the sentence with Taro. For example, Translate 'Taroを倒した！' as 'Taro was defeated!'", - True, - ) - descListTL5 = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - - # Log 1 - response = translateAI( - stateList[6], - "reply with only the gender neutral " - + LANGUAGE - + " translation of the action log. Always start the sentence with Taro. For example, Translate 'Taroを倒した！' as 'Taro was defeated!'", - True, - ) - descListTL6 = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - - # Log 1 - response = translateAI( - stateList[7], - "reply with only the gender neutral " - + LANGUAGE - + " translation of the action log. Always start the sentence with Taro. For example, Translate 'Taroを倒した！' as 'Taro was defeated!'", - True, - ) - descListTL7 = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - - # Check Mismatch - if ( - len(nameListTL) != len(stateList[0]) - or len(descListTL1) != len(stateList[1]) - or len(descListTL2) != len(stateList[2]) - or len(descListTL3) != len(stateList[3]) - or len(descListTL4) != len(stateList[4]) - or len(descListTL5) != len(stateList[5]) - or len(descListTL6) != len(stateList[6]) - ): - with LOCK: - if filename not in MISMATCH: - MISMATCH.append(filename) - else: - stateListTL = [ - nameListTL, - descListTL1, - descListTL2, - descListTL3, - descListTL4, - descListTL5, - descListTL6, - descListTL7, - ] - translate = True - - # DB Names - if len(dbNameList[0]) > 0: - # Progress Bar - total = 0 - for dbNameArray in dbNameList: - total += len(dbNameArray) - pbar.total = total - pbar.refresh() - - # Name - response = translateAI(dbNameList[0], "Reply with only the " + LANGUAGE + " translation") - nameListTL = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - - # Check Mismatch - if len(nameListTL) != len(dbNameList[0]): - with LOCK: - if filename not in MISMATCH: - MISMATCH.append(filename) - else: - dbNameListTL = [nameListTL] - translate = True - - # DB Values - if len(dbValueList[0]) > 0: - # Progress Bar - total = 0 - for dbValueArray in dbValueList: - total += len(dbValueArray) - pbar.total = total - pbar.refresh() - - # Name - response = translateAI(dbValueList[0], "Reply with only the " + LANGUAGE + " translation") - valueListTL = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - - # Check Mismatch - if len(valueListTL) != len(dbValueList[0]): - with LOCK: - if filename not in MISMATCH: - MISMATCH.append(filename) - else: - dbValueListTL = [valueListTL] - translate = True - - # Start Pass 2 - if translate == True: - jobList.append(scenarioListTL) - jobList.append(npcListTL) - jobList.append(itemListTL) - jobList.append(stateListTL) - jobList.append(armorListTL) - jobList.append(enemyListTL) - jobList.append(weaponsListTL) - jobList.append(skillListTL) - jobList.append(optionsListTL) - jobList.append(dbNameListTL) - jobList.append(dbValueListTL) - searchDB(events, pbar, jobList, filename) - - except IndexError as e: - traceback.print_exc() - raise Exception(str(e) + "Failed to translate: " + initialJAString) from None - except Exception as e: - traceback.print_exc() - raise Exception(str(e) + "Failed to translate: " + initialJAString) from None - - return totalTokens - -# Save some money and enter the character before translation -def getSpeaker(speaker): - match speaker: - case "ファイン": - return ["Fine", [0, 0]] - case "": - return ["", [0, 0]] - case _: - # Find Speaker - for i in range(len(NAMESLIST)): - if speaker == NAMESLIST[i][0]: - return [NAMESLIST[i][1], [0, 0]] - - # Translate and Store Speaker - response = translateAI( - f"{speaker}", - "Reply with the " + LANGUAGE + " translation of the NPC name.", - False, - ) - response[0] = response[0].title() - response[0] = response[0].replace("'S", "'s") - response[0] = response[0].replace("Speaker: ", "") - - # Retry if name doesn't translate for some reason - if re.search(r"([a-zA-Z？?])", response[0]) == None: - response = translateAI( - f"{speaker}", - "Reply with the " + LANGUAGE + " translation of the NPC name.", - False, - ) - response[0] = response[0].title() - response[0] = response[0].replace("'S", "'s") - - speakerList = [speaker, response[0]] - NAMESLIST.append(speakerList) - return response - return [speaker, [0, 0]] - -def translateAI(text, history, history_ctx=None): - """ - Legacy wrapper function for the new shared translation utility. - This maintains compatibility with existing code while using the new shared implementation. - """ - global PBAR, MISMATCH, FILENAME - - # Update config estimate mode based on global ESTIMATE - TRANSLATION_CONFIG.estimateMode = bool(ESTIMATE) - - # Call the new shared translation function - return sharedtranslateAI( - text=text, - history=history, - config=TRANSLATION_CONFIG, - filename=FILENAME, - pbar=PBAR, - lock=LOCK, - mismatchList=MISMATCH - ) +# Libraries +import json +import os +import re +import util.dazedwrap as dazedwrap +import threading +import time +import traceback +import tiktoken +# Removed concurrent.futures usage for simplicity; running synchronously +from pathlib import Path +from colorama import Fore +from dotenv import load_dotenv +from retry import retry +from tqdm import tqdm +from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost, getPricingConfig, calculateCost + +# OpenAI initialization centralized in util/translation.py + +# Globals +MODEL = os.getenv("model") +TIMEOUT = int(os.getenv("timeout")) +LANGUAGE = os.getenv("language").capitalize() +from util.paths import PROMPT_PATH, VOCAB_PATH + +PROMPT = PROMPT_PATH.read_text(encoding="utf-8") +VOCAB = VOCAB_PATH.read_text(encoding="utf-8") +LOCK = threading.Lock() +WIDTH = int(os.getenv("width")) +LISTWIDTH = int(os.getenv("listWidth")) +NOTEWIDTH = int(os.getenv("noteWidth")) +MAXHISTORY = 10 +ESTIMATE = "" +TOKENS = [0, 0] +NAMESLIST = [] # Keep list for consistency +TERMSLIST = [] # Keep list for consistency +NAMES = False # Output a list of all the character names found +BRFLAG = False # If the game uses
instead +FIXTEXTWRAP = True # Overwrites textwrap +IGNORETLTEXT = False # Ignores all translated text. +MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong) +FILENAME = None +BRACKETNAMES = False + +# Regex - Need to change this if you want to translate from/to other languages. Default is Japanese Regex +LANGREGEX = r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+" + +# Get pricing configuration based on the model +PRICING_CONFIG = getPricingConfig(MODEL) +INPUTAPICOST = PRICING_CONFIG["inputAPICost"] +OUTPUTAPICOST = PRICING_CONFIG["outputAPICost"] +BATCHSIZE = PRICING_CONFIG["batchSize"] +FREQUENCY_PENALTY = PRICING_CONFIG["frequencyPenalty"] + +# tqdm Globals +BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}" +POSITION = 0 +LEAVE = False +PBAR = None +FILENAME = None + +# Dialogue / Choices +CODE101 = False +CODE102 = False + +# Picture +CODE150 = False + +# Set String (Fragile but necessary) +CODE122 = False + +# Other +CODE210 = False +CODE300 = False +CODE250 = False + +# Database +SCENARIOFLAG = False +OPTIONSFLAG = True +NPCFLAG = False +DBNAMEFLAG = False +DBVALUEFLAG = False +ITEMFLAG = False +STATEFLAG = False +ENEMYFLAG = False +ARMORFLAG = False +WEAPONFLAG = False +SKILLFLAG = False + +# Initialize Translation Config +TRANSLATION_CONFIG = TranslationConfig( + model=MODEL, + language=LANGUAGE, + prompt=PROMPT, + vocab=VOCAB, + langRegex=LANGREGEX, + batchSize=BATCHSIZE, + maxHistory=MAXHISTORY, + estimateMode=False # Will be set dynamically based on ESTIMATE +) +LEAVE = False + +def handleWOLF(filename, estimate): + global ESTIMATE, TOKENS, FILENAME + ESTIMATE = estimate + FILENAME = filename + + # Translate + start = time.time() + translatedData = openFiles(filename) + + # Translate + if not estimate: + try: + with open("translated/" + filename, "w", encoding="utf-8", newline="\n") as outFile: + json.dump(translatedData[0], outFile, ensure_ascii=False, indent=4) + except Exception: + traceback.print_exc() + return "Fail" + + # Print File + end = time.time() + tqdm.write(getResultString(translatedData, end - start, filename)) + with LOCK: + TOKENS[0] += translatedData[1][0] + TOKENS[1] += translatedData[1][1] + + # Print Total + totalString = getResultString(["", TOKENS, None], end - start, "TOTAL") + + # Print any errors on maps + if len(MISMATCH) > 0: + return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET + else: + return totalString + + +def openFiles(filename): + with open("files/" + filename, "r", encoding="utf-8-sig") as f: + data = json.load(f) + + # Map Files + if "'events':" in str(data): + if len(data["events"]) > 0: + translatedData = parseMap(data, filename) + else: + return [data, [0, 0], None] + + # Map Files + elif "'types':" in str(data): + translatedData = parseDB(data, filename) + + # Other Files + elif "'commands':" in str(data): + translatedData = parseOther(data, filename) + + else: + raise NameError(filename + " Not Supported") + + return translatedData + + +def getResultString(translatedData, translationTime, filename): + # File Print String + cost = calculateCost(translatedData[1][0], translatedData[1][1], MODEL) + totalTokenstring = ( + Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]" + "[Output: " + + str(translatedData[1][1]) + + "]" "[Cost: ${:,.4f}".format(cost) + + "]" + ) + timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]" + + if translatedData[2] is None: + # Success + return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET + else: + # Fail + try: + raise translatedData[2] + except Exception as e: + traceback.print_exc() + errorString = str(e) + Fore.RED + return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET + + +def save_progress_json(data, filename): + """Atomically write current JSON data to translated/filename; skip in estimate mode.""" + try: + if ESTIMATE: + return + os.makedirs("translated", exist_ok=True) + tmp_path = os.path.join("translated", f"{filename}.tmp") + final_path = os.path.join("translated", filename) + with open(tmp_path, "w", encoding="utf-8", newline="\n") as outFile: + json.dump(data, outFile, ensure_ascii=False, indent=4) + os.replace(tmp_path, final_path) + except Exception: + traceback.print_exc() + + +def maybe_save_progress_json(data, filename, tokens): + """Save JSON progress only when tokens indicate actual translation work.""" + try: + if not tokens: + return + if isinstance(tokens, (list, tuple)) and len(tokens) >= 2 and (tokens[0] or tokens[1]): + save_progress_json(data, filename) + except Exception: + traceback.print_exc() + + +def parseOther(data, filename): + totalTokens = [0, 0] + totalLines = 0 + events = data["commands"] + global LOCK + + # Thread for each page in file + with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar: + pbar.desc = filename + pbar.total = totalLines + translationData = searchCodes(events, pbar, [], filename) + try: + totalTokens[0] += translationData[0] + totalTokens[1] += translationData[1] + except Exception as e: + return [data, totalTokens, e] + finally: + maybe_save_progress_json(data, filename, translationData) + return [data, totalTokens, None] + + +def parseDB(data, filename): + totalTokens = [0, 0] + totalLines = 0 + events = data["types"] + global LOCK + + # Thread for each page in file + with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar: + pbar.desc = filename + pbar.total = totalLines + translationData = searchDB(events, pbar, [], filename) + try: + totalTokens[0] += translationData[0] + totalTokens[1] += translationData[1] + except Exception as e: + return [data, totalTokens, e] + finally: + maybe_save_progress_json(data, filename, translationData) + return [data, totalTokens, None] + + +def parseMap(data, filename): + totalTokens = [0, 0] + totalLines = 0 + events = data["events"] + global LOCK + + # Get total for progress bar + for event in events: + if event is not None: + for page in event["pages"]: + totalLines += len(page["list"]) + + # Process pages synchronously and persist after each + with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar: + pbar.desc = filename + pbar.total = totalLines + for event in events: + if event is not None: + for page in event["pages"]: + if page is not None: + try: + tt = searchCodes(page["list"], pbar, None, filename) + totalTokens[0] += tt[0] + totalTokens[1] += tt[1] + except Exception as e: + return [data, totalTokens, e] + finally: + maybe_save_progress_json(data, filename, tt) + return [data, totalTokens, None] + + +def searchCodes(events, pbar, jobList, filename): + # Lists + if jobList: + stringList = jobList[0] + list210 = jobList[1] + list300 = jobList[2] + list150 = jobList[3] + list250 = jobList[4] + list122 = jobList[5] + setData = True + else: + stringList = [] + list210 = [] + list300 = [] + list150 = [] + list250 = [] + list122 = [] + setData = False + + # Other + codeList = events + textHistory = [] + totalTokens = [0, 0] + translatedText = "" + speaker = "" + nametag = "" + initialJAString = "" + global LOCK, NAMESLIST, MISMATCH, PBAR, FILENAME + FILENAME = filename + PBAR = pbar + + # Calculate Total Length + code_flags = {102: CODE102, 122: CODE122, 300: CODE300, 250: CODE250} + totalList = 0 + for code_item in codeList: + if code_flags.get(code_item["code"], False): + totalList += 1 + pbar.total = totalList + pbar.refresh() + + # Begin Parsing File + try: + # Iterate through events + i = 0 + while i < len(codeList): + ### Event Code: 101 Message + if codeList[i]["code"] == 101 and CODE101 == True: + speakerRegex = r"@\d+\r?\n(.*?)：?\r?\n" # Default: r"@\d+\r?\n(.*)：?\r?\n"" + textRegex = r"@\d+\r?\n.*?：?\r?\n(.*)" # Default: r"@\d+\r?\n.*：?\r?\n(.*)" + # Alternative patterns for messages without @\d+ prefix + speakerRegexAlt = r"^(.*?)：\r?\n" # Matches "Name：\n" + textRegexAlt = r"^.*?：\r?\n(.*)" # Matches text after "Name：\n" + + # Grab String + jaString = codeList[i]["stringArgs"][0] + speaker = "" + + # Grab Speaker + if "\n" in jaString: + match = re.search(speakerRegex, jaString) + if not match: + # Try alternative pattern without @\d+ prefix + match = re.search(speakerRegexAlt, jaString) + if match: + # TL Speaker + response = getSpeaker(match.group(1)) + speaker = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + + # Set Name + codeList[i]["stringArgs"][0] = codeList[i]["stringArgs"][0].replace(match.group(1), speaker) + + # Grab Only Text + match = re.search(textRegex, jaString, flags=re.DOTALL) + if not match: + # Try alternative pattern without @\d+ prefix + match = re.search(textRegexAlt, jaString, flags=re.DOTALL) + if not match and re.search(LANGREGEX, jaString): + # No speaker pattern found, treat entire string as text to translate + jaString = jaString + initialJAString = jaString + elif match: + jaString = match.group(1) + initialJAString = jaString + else: + jaString = None # Skip non-translatable text + + if jaString is not None: + + # Remove Textwrap + jaString = jaString.replace("\r", "") + jaString = jaString.replace("\n", " ") + + # 1st Pass (Save Text to List) + if not setData: + if speaker == "": + stringList.append(jaString) + else: + stringList.append(f"[{speaker}]: {jaString}") + + # 2nd Pass (Set Text) + else: + # Grab Translated String + translatedText = stringList[0] + + # Remove speaker + matchSpeakerList = re.findall(r"^(\[.+?\]\s?[|:]\s?)\s?", translatedText) + if len(matchSpeakerList) > 0: + translatedText = translatedText.replace(matchSpeakerList[0], "") + + # Textwrap + if FIXTEXTWRAP is True: + translatedText = dazedwrap.wrapText(translatedText, width=WIDTH) + + # Set Data + codeList[i]["stringArgs"][0] = codeList[i]["stringArgs"][0].replace(initialJAString, translatedText) + + # Reset Data and Pop Item + stringList.pop(0) + + ### Event Code: 102 Choices + if codeList[i]["code"] == 102 and CODE102 == True: + # Grab Choice List + choiceList = [] + jaChoiceList = codeList[i]["stringArgs"] + + # Filter Empty + for j in range(len(jaChoiceList)): + if jaChoiceList[j]: + choiceList.append(jaChoiceList[j]) + + # Translate + if 'jaString' in locals(): + choiceString = f"Previous Line: {jaString}\n\nReply with the {LANGUAGE} translation of the dialogue choice" + else: + choiceString = f"Reply with the {LANGUAGE} translation of the dialogue choice" + response = translateAI( + choiceList, + choiceString, + True, + ) + translatedChoiceList = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + + # Validate and Set Data + if len(translatedChoiceList) == len(choiceList): + for j in range(len(jaChoiceList)): + if jaChoiceList[j]: + codeList[i]["stringArgs"][j] = translatedChoiceList[0] + translatedChoiceList.pop(0) + + ### Event Code: 210 Common Event + if codeList[i]["code"] == 210 and CODE210 == True: + # Speaker Event + if "stringArgs" in codeList[i] and codeList[i]["intArgs"][0] == None and len(codeList[i]["stringArgs"]) == 2: + response = getSpeaker(codeList[i]["stringArgs"][1]) + totalTokens[1] += response[1][0] + totalTokens[1] += response[1][1] + + # Set Data + codeList[i]["stringArgs"][1] = response[0] + + # Logs + elif "stringArgs" in codeList[i] and codeList[i]["intArgs"][0] == 500725 and len(codeList[i]["stringArgs"]) == 2: + # Grab String + jaString = codeList[i]["stringArgs"][1] + initialJAString = jaString + + # Remove Textwrap + jaString = jaString.replace("\r", "") + jaString = jaString.replace("\n", " ") + + # 1st Pass (Save Text to List) + if not setData: + list210.append(jaString) + + # 2nd Pass (Set Text) + else: + # Grab Translated String + translatedText = list210[0] + + # Textwrap + if FIXTEXTWRAP is True: + translatedText = dazedwrap.wrapText(translatedText, width=WIDTH) + + # Set Data + codeList[i]["stringArgs"][1] = translatedText + + # Pop Item + list210.pop(0) + + ### Event Code: 122 SetString + if codeList[i]["code"] == 122 and CODE122 == True: + if "stringArgs" in codeList[i] and len(codeList[i]["stringArgs"]) > 0: + # Grab String + jaString = re.search(r"^\n?(.*)\n?$", codeList[i]["stringArgs"][0]) + if jaString: + jaString = jaString.group(1) + else: + jaString = codeList[i]["stringArgs"][0] + + originalString = jaString + + # Remove Textwrap + jaString = jaString.replace("\r", "") + jaString = jaString.replace("\n", " ") + + # Check if this is a translatable string + if ( + not re.search(r"\.[\w]+$", jaString) + and jaString != "" + and "_" not in jaString + and '",' not in jaString + and "/" not in jaString + and re.search(LANGREGEX, jaString) + ): + # 1st Pass (Save Text to List) + if not setData: + list122.append(jaString) + + # 2nd Pass (Set Text) + else: + if len(list122) > 0: + # Grab Translated String + translatedText = list122[0] + + # Textwrap + if FIXTEXTWRAP is True: + translatedText = dazedwrap.wrapText(translatedText, width=WIDTH) + + # Set Data + codeList[i]["stringArgs"][0] = codeList[i]["stringArgs"][0].replace(originalString, translatedText) + + # Pop Item + list122.pop(0) + + ### Event Code: 150 Picture String + if codeList[i]["code"] == 150 and CODE150 == True: + if "stringArgs" in codeList[i] and len(codeList[i]["stringArgs"]) > 0: + font150 = "\\f[8]" + # Grab String + jaString = codeList[i]["stringArgs"][0] + + # Japanses Text Only + if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", jaString): + i += 1 + continue + + # Remove Textwrap + # jaString = jaString.replace("\n", " ") + # jaString = jaString.replace("\r", "") + + # Translate Other Strings [Specific Files Only] + if ( + not re.search(r"\.[\w]+$", jaString) + and jaString != "" + and "_" not in jaString + and '",' not in jaString + # and ">" not in jaString + # and "<" not in jaString + ): + # Pass 1 (Save Text to List) + if not setData: + list150.append(jaString) + + # Pass 2 (Set Text) + else: + translatedText = list150[0] + + # Textwrap + # translatedText = dazedwrap.wrapText(translatedText, WIDTH) + + # Set String with font formatting + codeList[i]["stringArgs"][0] = re.sub(r'\\f\[(\d+)\]', lambda x: f'\\f[{int(x.group(1))-2}]', translatedText) + + # Pop processed item + list150.pop(0) + + ### Event Code: 300 Common Events + if codeList[i]["code"] == 300 and CODE300 == True and "stringArgs" in codeList[i] and len(codeList[i]["stringArgs"]) > 1: + # Choices + if codeList[i]["stringArgs"][0] == "[共]汎用ウィンドウ生成" or codeList[i]["stringArgs"][0] == "選択肢/確認": + # Grab String + choiceList = codeList[i]["stringArgs"][1].split(",") + + # Translate Question + question = codeList[i]['stringArgs'][2] + response = translateAI(question, "Reply with the {LANGUAGE} translation") + translatedText = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + + # Translate Question + jaString = translatedText + codeList[i]['stringArgs'][2] = translatedText + + # Translate Choices + if 'jaString' in locals() and jaString: + response = translateAI(choiceList, jaString) + else: + response = translateAI(choiceList, f"Reply with the {LANGUAGE} translation of the dialogue choice") + choiceListTL = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + + # Replace Commas + for j in range(len(choiceListTL)): + choiceListTL[j] = choiceListTL[j].replace(", ", "、") + + # Convert to String and Set + translatedText = ",".join(choiceListTL) + codeList[i]["stringArgs"][1] = translatedText + + # Dialogue + elif codeList[i]["stringArgs"][0] == "BTLメッセージ" or codeList[i]["stringArgs"][0] == "X[移]メニュー時文章表示": + jaString = codeList[i]["stringArgs"][1] + + # Pass 1 + if not setData: + # Remove Textwrap + jaString = jaString.replace("\n", " ") + jaString = jaString.replace("\r", "") + + # Append + list300.append(jaString) + + # Pass 2 + else: + # Add Textwrap and Font + translatedText = dazedwrap.wrapText(list300[0], WIDTH) + list300.pop(0) + + # Write to File + codeList[i]["stringArgs"][1] = translatedText + + # Dialogue + elif codeList[i]["stringArgs"][0] == "援護文章": + speakerRegex = r"@\d+\r?\n(.*)：\r?\n" # Default: r"@\d+\r?\n(.*)：\r?\n" + textRegex = r"@?\d*\r?\n?\u3000*([\w\W]+)\r?\n?" # Default: r"@?\d*\r?\n?\u3000*([\w\W]+)\r?\n?" + + # Grab String + jaString = codeList[i]["stringArgs"][1] + speaker = "" + + # Grab Speaker + if "：\n" in jaString or "：\r\n" in jaString: + match = re.search(speakerRegex, jaString) + if match: + # TL Speaker + response = getSpeaker(match.group(1)) + speaker = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + + # Set nametag and remove from string + codeList[i]["stringArgs"][1] = codeList[i]["stringArgs"][1].replace(match.group(1), speaker) + jaString = jaString.replace(match.group(0), "") + + # Grab Only Text + match = re.search(textRegex, jaString) + if match: + jaString = match.group(1) + initialJAString = jaString + + # Remove Textwrap + jaString = jaString.replace("\r", "") + jaString = jaString.replace("\n", " ") + + # 1st Pass (Save Text to List) + if not setData: + if speaker == "": + list300.append(jaString) + else: + list300.append(f"[{speaker}]: {jaString}") + + # 2nd Pass (Set Text) + else: + # Grab Translated String + translatedText = list300[0] + + # Remove speaker + matchSpeakerList = re.findall(r"^(\[.+?\]\s?[|:]\s?)\s?", translatedText) + if len(matchSpeakerList) > 0: + translatedText = translatedText.replace(matchSpeakerList[0], "") + + # Textwrap + if FIXTEXTWRAP is True: + translatedText = dazedwrap.wrapText(translatedText, width=WIDTH) + + # Set Data + codeList[i]["stringArgs"][1] = codeList[i]["stringArgs"][1].replace(initialJAString, translatedText) + + # Reset Data and Pop Item + list300.pop(0) + + ### Event Code: 250 DB Read/Writes + if codeList[i]["code"] == 250 and CODE250 == True: + # Validate size + stringArg = 0 + if len(codeList[i]["stringArgs"]) == 4: + if codeList[i]["stringArgs"][1] == "万能ｳｨﾝﾄﾞｳ一時DB"\ + and codeList[i]["stringArgs"][stringArg] != "": + # Font Size + fontSize = 0 + + # Grab String + jaString = codeList[i]["stringArgs"][stringArg] + + # Remove Textwrap + # jaString = jaString.replace("\r", "") + # jaString = jaString.replace("\n", " ") + + # Pass 1 (Save Text to List) + if not setData: + list250.append(jaString) + + # Pass 2 (Set Text) + else: + translatedText = list250[0] + list250.pop(0) + + # Textwrap + # translatedText = dazedwrap.wrapText(translatedText, WIDTH) + + # Add/Replace font formatting + # Remove existing font command if present + translatedText = re.sub(r'\\f\[\d+\]', '', translatedText) + if fontSize: + # Add new font command if fontSize is not 0 + if fontSize != "0" and fontSize != 0: + translatedText = f"\\f[{fontSize}]{translatedText}" + + # Set Data + codeList[i]["stringArgs"][stringArg] = translatedText + + ### Iterate + i += 1 + + # EOF + stringListTL = [] + list210TL = [] + list150TL = [] + list250TL = [] + list300TL = [] + list122TL = [] + setData = False + + # String List + if len(stringList) > 0: + pbar.total = len(stringList) + pbar.refresh() + response = translateAI(stringList, textHistory) + stringListTL = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + if len(stringListTL) != len(stringList): + with LOCK: + if filename not in MISMATCH: + MISMATCH.append(filename) + else: + setData = True + + # 210 List + if len(list210) > 0: + pbar.total = len(list210) + pbar.refresh() + response = translateAI(list210, textHistory) + list210TL = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + if len(list210TL) != len(list210): + with LOCK: + if filename not in MISMATCH: + MISMATCH.append(filename) + else: + setData = True + + # 250 List + if len(list250) > 0: + pbar.total = len(list250) + pbar.refresh() + response = translateAI(list250, textHistory) + list250TL = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + if len(list250TL) != len(list250): + with LOCK: + if filename not in MISMATCH: + MISMATCH.append(filename) + else: + setData = True + + # 300 List + if len(list300) > 0: + pbar.total = len(list300) + pbar.refresh() + response = translateAI(list300, textHistory) + list300TL = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + if len(list300TL) != len(list300): + with LOCK: + if filename not in MISMATCH: + MISMATCH.append(filename) + else: + setData = True + + # 150 List + if len(list150) > 0: + # Progress Bar + pbar.total = len(list150) + pbar.refresh() + + # Translate + response = translateAI( + list150, + textHistory, + True, + ) + list150TL = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + + # Check Mismatch + if len(list150TL) != len(list150): + with LOCK: + if filename not in MISMATCH: + MISMATCH.append(filename) + else: + list150 = list150TL + setData = True + + # 122 List + if len(list122) > 0: + pbar.total = len(list122) + pbar.refresh() + response = translateAI(list122, textHistory) + list122TL = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + if len(list122TL) != len(list122): + with LOCK: + if filename not in MISMATCH: + MISMATCH.append(filename) + else: + setData = True + + # Update jobList before recursive call + if setData == True: + stringList = [] + jobList = [stringListTL, list210TL, list300TL, list150TL, list250TL, list122TL] # Add list122TL + searchCodes(events, pbar, jobList, filename) + + else: + # Set Data + events = codeList + + except IndexError as e: + traceback.print_exc() + raise Exception(str(e) + "Failed to translate: " + initialJAString) from None + except Exception as e: + traceback.print_exc() + raise Exception(str(e) + "Failed to translate: " + initialJAString) from None + + return totalTokens + + +def formatDramon(jaString): + imageRegex = r"(\r?\n?_[a-zA-Z_\d/.]+\r?\n?)|(@-?\d?\r\n)|(@-?\d?)([^\r]\d?-?\d?[^\r]+?)(\r\n|$)|(_PDC)|(>\r\n)|(^[#])|(\r\n@$)|(/)|(_SS_)|(\r\n[@/]\s?-?\d?\r\n)" + + jaString = jaString.replace("\u3000", " ") + jaString = jaString.replace("#", "") + jaString = re.sub(r"([^\r])\n", r"\1\r\n", jaString) + + # Grab and Split + jaStringList = re.split(imageRegex, jaString) + + # Clean List + cleanedList = [x for x in jaStringList if x is not None and x != "" and x != "\r\n" and x != "_SS_"] + + # Iterate Through List + j = 0 + translatedText = "" + while j < len(cleanedList): + if ( + ("@" in cleanedList[j] or "/" in cleanedList[j]) + and j < len(cleanedList) - 1 + and re.search(r"([@/]-?\d?\r\n)", cleanedList[j]) is None + and ".ogg" not in cleanedList[j] + ): + # Setup @ + if j > 0 and "@" not in cleanedList[j - 1] and "/" not in cleanedList[j - 1] and "_" not in cleanedList[j - 1]: + cleanedList[j - 1] = cleanedList[j - 1] + cleanedList[j + 1] + else: + cleanedList.insert(j, cleanedList[j + 1]) + j += 1 + cleanedList[j] = f"\r\n{cleanedList[j]}\r\n" + cleanedList.pop(j + 1) + j += 1 + + return cleanedList + +def handleScenarioScript(jaString, scriptString=""): + # Extract Speaker + # Updated regex to match speaker names with spaces, hyphens, and other characters + # [^\n：]+ matches any character except newline and ： (the speaker name) + scriptRegex = r"\n?(//.*?\n|//.*|/b.*?\n|[^\n：]+：\d*\r?\n)" + match = re.search(scriptRegex, jaString) + if match: + if "//" in match.group(1): + scriptStringLocal = match.group(1) + jaString = jaString.replace(scriptStringLocal, "") + scriptString += scriptStringLocal + match = re.search(scriptRegex, jaString) + if not match: + return None + else: + if "//" in jaString: + return handleScenarioScript(jaString, scriptString) + return [jaString, match.group(1), scriptString] + else: + return None + +# Database +def searchDB(events, pbar, jobList, filename): + # Set Lists + if len(jobList) > 0: + scenarioList = jobList[0] + npcList = jobList[1] + itemList = jobList[2] + stateList = jobList[3] + armorList = jobList[4] + enemyList = jobList[5] + weaponsList = jobList[6] + skillList = jobList[7] + optionsList = jobList[8] + dbNameList = jobList[9] + dbValueList = jobList[10] + setData = True + else: + scenarioList = [[], [], []] + npcList = [[], [], [], []] + itemList = [[], [], [], []] + armorList = [[], []] + enemyList = [[], []] + weaponsList = [[], [], [], []] + skillList = [[], [], [], [], []] + stateList = [[], [], [], [], [], [], [], []] + optionsList = [[], [], []] + dbNameList = [[]] + dbValueList = [[]] + setData = False + + # Vars/Globals + totalTokens = [0, 0] + initialJAString = "" + tableList = events + font = "" + global LOCK + global NAMESLIST + global MISMATCH + global PBAR + PBAR = pbar + + # Begin Parsing File + try: + for table in tableList: + # Grab NPCs + if table["name"] == "アイドル" and NPCFLAG == True: + with open("log/translations.txt", "a", encoding="utf-8") as file: + if setData: + file.write(f"\n#Actors\n") + for npc in table["data"]: + dataList = npc["data"] + + # Parse + for j in range(len(dataList)): + # Name + if dataList[j].get("name") == "出身": + # Pass 1 (Grab Data) + if setData == False: + if dataList[j].get("value") != "": + npcList[0].append(dataList[j].get("value")) + + # Pass 2 (Set Data) + else: + if dataList[j].get("value") != "": + # Write Name + file.write(f"{dataList[j].get('value')} ({npcList[0][0]})\n") + + # Set Data + dataList[j].update({"value": npcList[0][0]}) + npcList[0].pop(0) + + # Description + if dataList[j].get("name") == "説明": + # Pass 1 (Grab Data) + if setData == False: + if dataList[j].get("value") != "": + # Remove Textwrap + jaString = dataList[j].get("value") + jaString = jaString.replace("\n", " ") + jaString = jaString.replace("\r", "") + jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString) + + # Append Data + npcList[1].append(jaString) + # Pass 2 (Set Data) + else: + if dataList[j].get("value") != "": + # Textwrap + translatedText = npcList[1][0] + translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH) + translatedText = font + translatedText + + # Set Data + dataList[j].update({"value": translatedText}) + npcList[1].pop(0) + + # Grab Scenario + if "デートキャラ" in table["name"] and SCENARIOFLAG == True: + for scenario in table["data"]: + dataList = scenario["data"] + + # Parse + for j in range(len(dataList)): + # Name + if dataList[j].get("name") == "NULL": + if dataList[j].get("value"): + jaStringList = re.split(r'\r\n\r\n|\n\n', dataList[j].get("value")) + for jaString in jaStringList: + speakerNum = None + ogString = jaString + # Extract Speaker + speakerList = handleScenarioScript(jaString) + + # Handle simple names without speaker pattern + if not speakerList: + # Pass 1 (Grab Data) - Simple name + if setData == False: + jaString = jaString.replace("\n", " ") + jaString = jaString.replace("\r", "") + if jaString.strip(): + scenarioList[0].append(jaString) + # Pass 2 (Set Data) - Simple name + else: + if ogString.strip(): + translatedText = scenarioList[0][0] + scenarioList[0].pop(0) + dataList[j].update({"value": dataList[j].get("value").replace(ogString, translatedText)}) + continue + + if speakerList[2]: + jaString = speakerList[0] + speakerNumMatch = re.search(r"：(\d+)", speakerList[1]) + if speakerNumMatch: + speakerNum = speakerNumMatch.group(1) + speaker = re.sub(r"：\d*", "", speakerList[1]) + speaker = re.sub(r"/b", "NPC", speaker) + speaker = speaker.replace("\r\n", "") + + # Add Speaker + jaString = jaString.replace(speakerList[1], f"[{speaker}]: ") + + # Pass 1 (Grab Data) + if setData == False: + jaString = jaString.replace("\n", " ") + jaString = jaString.replace("\r", "") + scenarioList[0].append(jaString) + + # Pass 2 (Set Data) + else: + translatedText = scenarioList[0][0] + scenarioList[0].pop(0) + + # Remove Speaker before wrapping so WIDTH applies to dialogue only + speakerMatch = re.search(r"\[(.+?)\]\s?[|:]\s?", translatedText) + if speakerMatch: + speaker = speakerMatch.group(1).strip() + match = re.search(r'(^\[.+?\]\s?[|:]\s?)', translatedText) + if match: + translatedText = translatedText.replace(match.group(1), "") + + # Wrap text after removing speaker + translatedText = dazedwrap.wrapText(translatedText, WIDTH) + + # Redo Old Speaker Format + speaker = speaker.replace("Speaker", "\b") + if speakerNum: + translatedText = f"{speaker}：{speakerNum}\r\n{translatedText}" + elif speaker == "NPC": + translatedText = f"/b\r\n{translatedText}" + else: + translatedText = f"{speaker}：\r\n{translatedText}" + + # Add Script String + if speakerList[2]: + translatedText = f"{speakerList[2]}{translatedText}" + + # Set Data + dataList[j].update({"value": dataList[j].get("value").replace(ogString, translatedText)}) + # Description + if dataList[j].get("name") == "不正解会話": + if dataList[j].get("value"): + jaStringList = re.split(r'\r\n\r\n|\n\n', dataList[j].get("value")) + for jaString in jaStringList: + speakerNum = None + ogString = jaString + # Extract Speaker + speakerList = handleScenarioScript(jaString) + + # Handle simple text without speaker pattern + if not speakerList: + # Pass 1 (Grab Data) - Simple text + if setData == False: + jaString = jaString.replace("\n", " ") + jaString = jaString.replace("\r", "") + if jaString.strip(): + scenarioList[1].append(jaString) + # Pass 2 (Set Data) - Simple text + else: + if ogString.strip(): + translatedText = scenarioList[1][0] + scenarioList[1].pop(0) + translatedText = dazedwrap.wrapText(translatedText, WIDTH) + dataList[j].update({"value": dataList[j].get("value").replace(ogString, translatedText)}) + continue + + if speakerList[2]: + jaString = speakerList[0] + speakerNumMatch = re.search(r"：(\d+)", speakerList[1]) + if speakerNumMatch: + speakerNum = speakerNumMatch.group(1) + speaker = re.sub(r"：\d*", "", speakerList[1]) + speaker = re.sub(r"/b", "NPC", speaker) + speaker = speaker.replace("\r\n", "") + + # Add Speaker + jaString = jaString.replace(speakerList[1], f"[{speaker}]: ") + + # Pass 1 (Grab Data) + if setData == False: + jaString = jaString.replace("\n", " ") + jaString = jaString.replace("\r", "") + scenarioList[1].append(jaString) + + # Pass 2 (Set Data) + else: + translatedText = scenarioList[1][0] + scenarioList[1].pop(0) + + # Remove Speaker before wrapping so WIDTH applies to dialogue only + speakerMatch = re.search(r"\[(.+?)\]\s?[|:]\s?", translatedText) + if speakerMatch: + speaker = speakerMatch.group(1).strip() + match = re.search(r'(^\[.+?\]\s?[|:]\s?)', translatedText) + if match: + translatedText = translatedText.replace(match.group(1), "") + + # Wrap text after removing speaker + translatedText = dazedwrap.wrapText(translatedText, WIDTH) + + # Redo Old Speaker Format + speaker = speaker.replace("Speaker", "\b") + if speakerNum: + translatedText = f"{speaker}：{speakerNum}\r\n{translatedText}" + elif speaker == "NPC": + translatedText = f"/b\r\n{translatedText}" + else: + translatedText = f"{speaker}：\r\n{translatedText}" + + # Add Script String + if speakerList[2]: + translatedText = f"{speakerList[2]}{translatedText}" + + # Set Data + dataList[j].update({"value": dataList[j].get("value").replace(ogString, translatedText)}) + + # Description + if dataList[j].get("name") == "正解会話": + if dataList[j].get("value"): + jaStringList = re.split(r'\r\n\r\n|\n\n', dataList[j].get("value")) + for jaString in jaStringList: + speakerNum = None + ogString = jaString + # Extract Speaker + speakerList = handleScenarioScript(jaString) + + # Handle simple text without speaker pattern + if not speakerList: + # Pass 1 (Grab Data) - Simple text + if setData == False: + jaString = jaString.replace("\n", " ") + jaString = jaString.replace("\r", "") + if jaString.strip(): + scenarioList[2].append(jaString) + # Pass 2 (Set Data) - Simple text + else: + if ogString.strip(): + translatedText = scenarioList[2][0] + scenarioList[2].pop(0) + translatedText = dazedwrap.wrapText(translatedText, WIDTH) + dataList[j].update({"value": dataList[j].get("value").replace(ogString, translatedText)}) + continue + if speakerList[2]: + jaString = speakerList[0] + speakerNumMatch = re.search(r"：(\d+)", speakerList[1]) + if speakerNumMatch: + speakerNum = speakerNumMatch.group(1) + speaker = re.sub(r"：\d*", "", speakerList[1]) + speaker = re.sub(r"/b", "NPC", speaker) + speaker = speaker.replace("\r\n", "") + + # Add Speaker + jaString = jaString.replace(speakerList[1], f"[{speaker}]: ") + + # Pass 1 (Grab Data) + if setData == False: + jaString = jaString.replace("\n", " ") + jaString = jaString.replace("\r", "") + scenarioList[2].append(jaString) + + # Pass 2 (Set Data) + else: + translatedText = scenarioList[2][0] + scenarioList[2].pop(0) + + # Remove Speaker before wrapping so WIDTH applies to dialogue only + speakerMatch = re.search(r"\[(.+?)\]\s?[|:]\s?", translatedText) + if speakerMatch: + speaker = speakerMatch.group(1).strip() + match = re.search(r'(^\[.+?\]\s?[|:]\s?)', translatedText) + if match: + translatedText = translatedText.replace(match.group(1), "") + + # Wrap text after removing speaker + translatedText = dazedwrap.wrapText(translatedText, WIDTH) + + # Redo Old Speaker Format + speaker = speaker.replace("Speaker", "\b") + if speakerNum: + translatedText = f"{speaker}：{speakerNum}\r\n{translatedText}" + elif speaker == "NPC": + translatedText = f"/b\r\n{translatedText}" + else: + translatedText = f"{speaker}：\r\n{translatedText}" + + # Add Script String + if speakerList[2]: + translatedText = f"{speakerList[2]}{translatedText}" + + # Set Data + dataList[j].update({"value": dataList[j].get("value").replace(ogString, translatedText)}) + + # Grab Options + if table["name"] == "クイズ" and OPTIONSFLAG == True: + for option in table["data"]: + dataList = option["data"] + + # Parse + for j in range(len(dataList)): + # Name + if dataList[j].get("name") == "問": + if dataList[j].get("value"): + jaString = dataList[j].get("value") + # Pass 1 (Grab Data) + if setData == False: + optionsList[0].append(jaString) + + # Pass 2 (Set Data) + else: + translatedText = optionsList[0][0] + optionsList[0].pop(0) + dataList[j].update({"value": dataList[j].get("value").replace(jaString, translatedText)}) + + # Description 1 + if dataList[j].get("name") == "3": + if dataList[j].get("value"): + jaString = dataList[j].get("value") + # Pass 1 (Grab Data) + if setData == False: + jaString = jaString.replace("\n", " ") + jaString = jaString.replace("\r", "") + optionsList[1].append(jaString) + + # Pass 2 (Set Data) + else: + translatedText = optionsList[1][0] + optionsList[1].pop(0) + translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH) + dataList[j].update({"value": dataList[j].get("value").replace(jaString, translatedText)}) + + # Description 2 + if dataList[j].get("name") == "4": + if dataList[j].get("value"): + jaString = dataList[j].get("value") + # Pass 1 (Grab Data) + if setData == False: + jaString = jaString.replace("\n", " ") + jaString = jaString.replace("\r", "") + optionsList[2].append(jaString) + + # Pass 2 (Set Data) + else: + translatedText = optionsList[2][0] + optionsList[2].pop(0) + translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH) + dataList[j].update({"value": dataList[j].get("value").replace(jaString, translatedText)}) + + # Grab DB Names + if table["name"] == "マップ設定" and DBNAMEFLAG == True: + font = None + dbName = table["data"] + for j in range(len(dbName)): + # Pass 1 (Grab Data) + if setData == False: + if dbName[j].get("name") != "": + dbNameList[0].append(dbName[j].get("name")) + + # Pass 2 (Set Data) + else: + if dbName[j].get("name") != "": + dbName[j].update({"name": dbNameList[0][0]}) + dbNameList[0].pop(0) + + # Grab DB Values + if table["name"] == "近況文章" and DBVALUEFLAG == True: + font = 16 + subject = "" + for entry in table["data"]: + dbValue = entry["data"] + for j in range(len(dbValue)): + if dbValue[j].get("value"): + jaString = dbValue[j].get("value") + + # Speaker + match = re.search(r"(^[\\]+f\[\d+\].+?\r\n).+", jaString) + if match: + subject = match.group(1) + jaString = jaString.replace(subject, "") + + # Nuke Textwrap & Font + jaString = jaString.replace("\r", "") + jaString = jaString.replace("\n", " ") + jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString) + + # Pass 1 (Grab Data) + if setData == False: + # Append + dbValueList[0].append(jaString) + + # Pass 2 (Set Data) + else: + if dbValue[j].get("value"): + translatedText = dbValueList[0][0] + + # Textwrap + translatedText = dazedwrap.wrapText(translatedText, 30) + translatedText = translatedText.replace("\n", f"\r\n\\f[{font}]") + + # Subject + if subject: + translatedText = f"{subject}{translatedText}" + + # Set + dbValue[j].update({"value": translatedText}) + dbValueList[0].pop(0) + + # Grab Items + if table["name"] == "道具" and ITEMFLAG == True: + # Write Category + if setData: + with open("log/translations.txt", "a", encoding="utf-8") as file: + file.write(f"\n#Items\n") + + # Begin Translation + for item in table["data"]: + dataList = item["data"] + + # Parse + font = 18 + for j in range(len(dataList)): + # Name + if dataList[j].get("name") == "名前": + jaString = dataList[j].get("value") + if jaString != "": + # Pass 1 (Grab Data) + if setData == False: + if jaString != "": + itemList[0].append(jaString) + + # Pass 2 (Set Data) + else: + # Write to TL File + with open("log/translations.txt", "a", encoding="utf-8") as file: + file.write(f"{jaString} ({itemList[0][0]})\n") + + dataList[j].update({"value": itemList[0][0]}) + itemList[0].pop(0) + + # Description + if dataList[j].get("name") == "説明": + # Pass 1 (Grab Data) + if setData == False: + if dataList[j].get("value") != "": + # Remove Textwrap & Font + jaString = dataList[j].get("value") + jaString = jaString.replace("\n", " ") + jaString = jaString.replace("\r", "") + jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString) + + # Append Data + itemList[1].append(jaString) + # Pass 2 (Set Data) + else: + if dataList[j].get("value") != "": + # Textwrap + translatedText = itemList[1][0] + translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH) + + # Font + if font: + translatedText = f"\\f[{font}]{translatedText}" + + # Set Data + dataList[j].update({"value": translatedText}) + itemList[1].pop(0) + + # Log + if dataList[j].get("name") == "使用時文章[戦](人名~": + # Pass 1 (Grab Data) + if setData == False: + if dataList[j].get("value") != "": + # Remove Textwrap + jaString = dataList[j].get("value") + jaString = jaString.replace("\n", " ") + jaString = jaString.replace("\r", "") + jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString) + + # Skill Action + if jaString[0] in [ + "は", + "を", + "の", + "に", + "が", + ]: + jaString = f"Taro{jaString}" + + # Append Data + itemList[2].append(jaString) + # Pass 2 (Set Data) + else: + if dataList[j].get("value") != "": + # Textwrap + translatedText = itemList[2][0] + translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH) + # translatedText = font + translatedText + + # Remove Taro + translatedText = re.sub(r"\bTaro\b", "", translatedText) + + # Set Data + dataList[j].update({"value": translatedText}) + itemList[2].pop(0) + + # Description + if dataList[j].get("name") == "使用後文章[移動]": + # Pass 1 (Grab Data) + if setData == False: + if dataList[j].get("value") != "": + # Remove Textwrap + jaString = dataList[j].get("value") + jaString = jaString.replace("\n", " ") + jaString = jaString.replace("\r", "") + jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString) + + # Append Data + itemList[3].append(jaString) + # Pass 2 (Set Data) + else: + if dataList[j].get("value") != "": + # Textwrap + translatedText = itemList[3][0] + translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH) + + # Font + if font: + translatedText = f"\\f[{font}]{translatedText}" + + # Set Data + dataList[j].update({"value": translatedText}) + itemList[3].pop(0) + + # Grab Armors + if table["name"] == "防具" and ARMORFLAG == True: + font = "24" + for armor in table["data"]: + dataList = armor["data"] + + # Parse + for j in range(len(dataList)): + # Name + if dataList[j].get("name") == "名前": + # Pass 1 (Grab Data) + if setData == False: + if dataList[j].get("value") != "": + armorList[0].append(dataList[j].get("value")) + + # Pass 2 (Set Data) + else: + if dataList[j].get("value") != "": + dataList[j].update({"value": armorList[0][0]}) + armorList[0].pop(0) + + # Description + if dataList[j].get("name") == "説明": + # Pass 1 (Grab Data) + if setData == False: + if dataList[j].get("value") != "": + # Remove Textwrap + jaString = dataList[j].get("value") + jaString = jaString.replace("\n", " ") + jaString = jaString.replace("\r", "") + jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString) + + # Append Data + armorList[1].append(jaString) + # Pass 2 (Set Data) + else: + if dataList[j].get("value") != "": + # Textwrap + translatedText = armorList[1][0] + translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH) + + # Font + if font: + translatedText = f"\\f[{font}]{translatedText}" + + # Set Data + dataList[j].update({"value": translatedText}) + armorList[1].pop(0) + + # Grab Enemies + if table["name"] == "敵" and ENEMYFLAG == True: + for enemy in table["data"]: + dataList = enemy["data"] + + # Parse + for j in range(len(dataList)): + # Name + if dataList[j].get("name") == "敵キャラ名": + # Pass 1 (Grab Data) + if setData == False: + if dataList[j].get("value") != "": + enemyList[0].append(dataList[j].get("value")) + + # Pass 2 (Set Data) + else: + if dataList[j].get("value") != "": + dataList[j].update({"value": enemyList[0][0]}) + enemyList[0].pop(0) + + # Description + if dataList[j].get("name") == "NULL": + # Pass 1 (Grab Data) + if setData == False: + if dataList[j].get("value") != "": + # Remove Textwrap + jaString = dataList[j].get("value") + jaString = jaString.replace("\n", " ") + jaString = jaString.replace("\r", "") + jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString) + + # Append Data + enemyList[1].append(jaString) + # Pass 2 (Set Data) + else: + if dataList[j].get("value") != "": + # Textwrap + translatedText = enemyList[1][0] + translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH) + + # Font + if font: + translatedText = f"\\f[{font}]{translatedText}" + + # Set Data + dataList[j].update({"value": translatedText}) + enemyList[1].pop(0) + + # Grab Weapons + if table["name"] == "武器" and WEAPONFLAG == True: + font = "24" + for weapon in table["data"]: + dataList = weapon["data"] + + # Parse + for j in range(len(dataList)): + # Name + if dataList[j].get("name") == "名前": + # Pass 1 (Grab Data) + if setData == False: + if dataList[j].get("value") != "": + weaponsList[0].append(dataList[j].get("value")) + + # Pass 2 (Set Data) + else: + if dataList[j].get("value") != "": + dataList[j].update({"value": weaponsList[0][0]}) + weaponsList[0].pop(0) + + # Description + if dataList[j].get("name") == "説明": + # Pass 1 (Grab Data) + if setData == False: + if dataList[j].get("value") != "": + # Remove Textwrap + jaString = dataList[j].get("value") + jaString = jaString.replace("\n", " ") + jaString = jaString.replace("\r", "") + jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString) + + # Append Data + weaponsList[1].append(jaString) + # Pass 2 (Set Data) + else: + if dataList[j].get("value") != "": + # Textwrap + translatedText = weaponsList[1][0] + translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH) + + # Font + if font: + translatedText = f"\\f[{font}]{translatedText}" + + # Set Data + dataList[j].update({"value": translatedText}) + weaponsList[1].pop(0) + + # Grab Skills + if table["name"] == "技能" and SKILLFLAG == True: + font = "24" + for skill in table["data"]: + dataList = skill["data"] + + # Parse + for j in range(len(dataList)): + # Name + if dataList[j].get("name") == "名前": + # Pass 1 (Grab Data) + if setData == False: + if dataList[j].get("value") != "": + skillList[0].append(dataList[j].get("value")) + + # Pass 2 (Set Data) + else: + if dataList[j].get("value") != "": + dataList[j].update({"value": skillList[0][0]}) + skillList[0].pop(0) + + # Description + if dataList[j].get("name") == "説明": + # Pass 1 (Grab Data) + if setData == False: + if dataList[j].get("value") != "": + # Remove Textwrap + jaString = dataList[j].get("value") + jaString = jaString.replace("\n", " ") + jaString = jaString.replace("\r", "") + jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString) + + # Append Data + skillList[1].append(jaString) + + # Pass 2 (Set Data) + else: + if dataList[j].get("value") != "": + # Textwrap + translatedText = skillList[1][0] + translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH) + + # Font + if font: + translatedText = f"\\f[{font}]{translatedText}" + + # Set Data + dataList[j].update({"value": translatedText}) + skillList[1].pop(0) + + # Log + if dataList[j].get("name") == "発動時文章": + # Pass 1 (Grab Data) + if setData == False: + if dataList[j].get("value") != "": + # Remove Textwrap + jaString = dataList[j].get("value") + jaString = jaString.replace("\n", " ") + jaString = jaString.replace("\r", "") + jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString) + + # Skill Action + if jaString[0] in [ + "は", + "を", + "の", + "に", + "が", + ]: + jaString = f"Taro{jaString}" + + # Append Data + skillList[2].append(jaString) + # Pass 2 (Set Data) + else: + if dataList[j].get("value") != "": + # Textwrap + translatedText = skillList[2][0] + translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH) + translatedText = font + translatedText + + # Remove Taro + translatedText = re.sub(r"\bTaro\b", "", translatedText) + + # Set Data + dataList[j].update({"value": translatedText}) + skillList[2].pop(0) + + # Log + if dataList[j].get("name") == "使用時文章[戦闘](人名~": + # Pass 1 (Grab Data) + if setData == False: + if dataList[j].get("value") != "": + # Remove Textwrap + jaString = dataList[j].get("value") + jaString = jaString.replace("\n", " ") + jaString = jaString.replace("\r", "") + jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString) + + # Skill Action + if jaString[0] in [ + "は", + "を", + "の", + "に", + "が", + ]: + jaString = f"Taro{jaString}" + + # Append Data + skillList[3].append(jaString) + # Pass 2 (Set Data) + else: + if dataList[j].get("value") != "": + # Textwrap + translatedText = skillList[3][0] + translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH) + + # Remove Taro + translatedText = re.sub(r"\bTaro\b", "", translatedText) + + # Font + translatedText = font + translatedText + + # Set Data + dataList[j].update({"value": translatedText}) + skillList[3].pop(0) + + # Log + if dataList[j].get("name") == "失敗時文章[(対象)～]": + # Pass 1 (Grab Data) + if setData == False: + if dataList[j].get("value") != "": + # Remove Textwrap + jaString = dataList[j].get("value") + jaString = jaString.replace("\n", " ") + jaString = jaString.replace("\r", "") + jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString) + + # Skill Action + if jaString[0] in [ + "は", + "を", + "の", + "に", + "が", + ]: + jaString = f"Taro{jaString}" + + # Append Data + skillList[4].append(jaString) + # Pass 2 (Set Data) + else: + if dataList[j].get("value") != "": + # Textwrap + translatedText = skillList[4][0] + translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH) + translatedText = font + translatedText + + # Remove Taro + translatedText = re.sub(r"\bTaro\b", "", translatedText) + + # Set Data + dataList[j].update({"value": translatedText}) + skillList[4].pop(0) + + # Grab States + if table["name"] == "ステート" and STATEFLAG == True: + for state in table["data"]: + dataList = state["data"] + + # Parse + for j in range(len(dataList)): + # Name + if dataList[j].get("name") == "名前": + # Pass 1 (Grab Data) + if setData == False: + if dataList[j].get("value") != "": + stateList[0].append(dataList[j].get("value")) + + # Pass 2 (Set Data) + else: + if dataList[j].get("value") != "": + dataList[j].update({"value": stateList[0][0]}) + stateList[0].pop(0) + + # Description + if dataList[j].get("name") == "表示名": + # Pass 1 (Grab Data) + if setData == False: + if dataList[j].get("value") != "": + # Remove Textwrap + jaString = dataList[j].get("value") + jaString = jaString.replace("\n", " ") + jaString = jaString.replace("\r", "") + jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString) + + # Append Data + stateList[1].append(jaString) + # Pass 2 (Set Data) + else: + if dataList[j].get("value") != "": + # Textwrap + translatedText = stateList[1][0] + translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH) + translatedText = font + translatedText + + # Set Data + dataList[j].update({"value": translatedText}) + stateList[1].pop(0) + + # Log + if dataList[j].get("name") == "付与時文章": + # Pass 1 (Grab Data) + if setData == False: + if dataList[j].get("value") != "": + # Remove Textwrap + jaString = dataList[j].get("value") + jaString = jaString.replace("\n", " ") + jaString = jaString.replace("\r", "") + jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString) + + # State Action + if jaString[0] in [ + "は", + "を", + "の", + "に", + "が", + ]: + jaString = f"Taro{jaString}" + + # Append Data + stateList[2].append(jaString) + # Pass 2 (Set Data) + else: + if dataList[j].get("value") != "": + # Textwrap + translatedText = stateList[2][0] + translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH) + translatedText = font + translatedText + + # Remove Taro + translatedText = re.sub(r"\bTaro\b", "", translatedText) + + # Set Data + dataList[j].update({"value": translatedText}) + stateList[2].pop(0) + + # Log + if dataList[j].get("name") == "解除時文章": + # Pass 1 (Grab Data) + if setData == False: + if dataList[j].get("value") != "": + # Remove Textwrap + jaString = dataList[j].get("value") + jaString = jaString.replace("\n", " ") + jaString = jaString.replace("\r", "") + jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString) + + # State Action + if jaString[0] in [ + "は", + "を", + "の", + "に", + "が", + ]: + jaString = f"Taro{jaString}" + + # Append Data + stateList[3].append(jaString) + # Pass 2 (Set Data) + else: + if dataList[j].get("value") != "": + # Textwrap + translatedText = stateList[3][0] + translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH) + translatedText = font + translatedText + + # Remove Taro + translatedText = re.sub(r"\bTaro\b", "", translatedText) + + # Set Data + dataList[j].update({"value": translatedText}) + stateList[3].pop(0) + + # Log + if dataList[j].get("name") == "回復時の文章[(人名)～]": + # Pass 1 (Grab Data) + if setData == False: + if dataList[j].get("value") != "": + # Remove Textwrap + jaString = dataList[j].get("value") + jaString = jaString.replace("\n", " ") + jaString = jaString.replace("\r", "") + jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString) + + # State Action + if jaString[0] in [ + "は", + "を", + "の", + "に", + "が", + ]: + jaString = f"Taro{jaString}" + + # Append Data + stateList[4].append(jaString) + # Pass 2 (Set Data) + else: + if dataList[j].get("value") != "": + # Textwrap + translatedText = stateList[4][0] + translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH) + translatedText = font + translatedText + + # Remove Taro + translatedText = re.sub(r"\bTaro\b", "", translatedText) + + # Set Data + dataList[j].update({"value": translatedText}) + stateList[4].pop(0) + # Log + if dataList[j].get("name") == "┣ ｶｳﾝﾀｰ発動文[対象～": + # Pass 1 (Grab Data) + if setData == False: + if dataList[j].get("value") != "": + # Remove Textwrap + jaString = dataList[j].get("value") + jaString = jaString.replace("\n", " ") + jaString = jaString.replace("\r", "") + jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString) + + # State Action + if jaString[0] in [ + "は", + "を", + "の", + "に", + "が", + ]: + jaString = f"Taro{jaString}" + + # Append Data + stateList[5].append(jaString) + # Pass 2 (Set Data) + else: + if dataList[j].get("value") != "": + # Textwrap + translatedText = stateList[5][0] + translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH) + translatedText = font + translatedText + + # Remove Taro + translatedText = re.sub(r"\bTaro\b", "", translatedText) + + # Set Data + dataList[j].update({"value": translatedText}) + stateList[5].pop(0) + + # Log + if dataList[j].get("name") == "尻もち　行動不能　持続3ターン": + # Pass 1 (Grab Data) + if setData == False: + if dataList[j].get("value") != "": + # Remove Textwrap + jaString = dataList[j].get("value") + jaString = jaString.replace("\n", " ") + jaString = jaString.replace("\r", "") + jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString) + + # State Action + if jaString[0] in [ + "は", + "を", + "の", + "に", + "が", + ]: + jaString = f"Taro{jaString}" + + # Append Data + stateList[6].append(jaString) + # Pass 2 (Set Data) + else: + if dataList[j].get("value") != "": + # Textwrap + translatedText = stateList[6][0] + translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH) + translatedText = font + translatedText + + # Remove Taro + translatedText = re.sub(r"\bTaro\b", "", translatedText) + + # Set Data + dataList[j].update({"value": translatedText}) + stateList[6].pop(0) + + # Log + if dataList[j].get("name") == "状態異常の説明": + # Pass 1 (Grab Data) + if setData == False: + if dataList[j].get("value") != "": + # Remove Textwrap + jaString = dataList[j].get("value") + jaString = jaString.replace("\n", " ") + jaString = jaString.replace("\r", "") + jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString) + + # State Action + if jaString[0] in [ + "は", + "を", + "の", + "に", + "が", + ]: + jaString = f"Taro{jaString}" + + # Append Data + stateList[7].append(jaString) + # Pass 2 (Set Data) + else: + if dataList[j].get("value") != "": + # Textwrap + translatedText = stateList[7][0] + translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH) + translatedText = font + translatedText + + # Remove Taro + translatedText = re.sub(r"\bTaro\b", "", translatedText) + + # Set Data + dataList[j].update({"value": translatedText}) + stateList[7].pop(0) + + # Translation + scenarioListTL = [[], [], []] + optionsListTL = [[], [], []] + npcListTL = [[], [], [], []] + itemListTL = [[], [], [], []] + stateListTL = [[], [], [], [], [], [], [], []] + armorListTL = [[], []] + enemyListTL = [[], []] + weaponsListTL = [[], [], []] + skillListTL = [[], [], [], [], []] + dbNameListTL = [[]] + dbValueListTL = [[]] + + translate = False + + # NPCs + if len(npcList[0]) > 0: + # Progress Bar + total = 0 + for itemArray in npcList: + total += len(itemArray) + pbar.total = total + pbar.refresh() + + # Name + response = translateAI( + npcList[0], + "Reply with only the " + LANGUAGE + " translation of the RPG enemy name", + True, + ) + nameListTL = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + # Desc 1 + response = translateAI(npcList[1], "Reply with only the " + LANGUAGE + " translation") + descListTL1 = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + # Desc 2 + response = translateAI(npcList[2], "Reply with only the " + LANGUAGE + " translation") + descListTL2 = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + # Desc 3 + response = translateAI(npcList[3], "Reply with only the " + LANGUAGE + " translation") + descListTL3 = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + + # Check Mismatch + if ( + len(nameListTL) != len(npcList[0]) + or len(descListTL1) != len(npcList[1]) + or len(descListTL2) != len(npcList[2]) + or len(descListTL3) != len(npcList[3]) + ): + with LOCK: + if filename not in MISMATCH: + MISMATCH.append(filename) + else: + npcListTL = [nameListTL, descListTL1, descListTL2, descListTL3] + translate = True + + # SCENARIO + if scenarioList[0] or scenarioList[1]: + # Progress Bar + total = 0 + for scenarioArray in scenarioList: + total += len(scenarioArray) + pbar.total = total + pbar.refresh() + + # Name + response = translateAI( + scenarioList[0], + "Reply with only the " + LANGUAGE + " translation", + True, + ) + nameListTL = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + # Desc 1 + response = translateAI( + scenarioList[1], + "reply with only the gender neutral " + LANGUAGE + " translation of the NPC name", + True, + ) + descListTL1 = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + # Desc 2 + response = translateAI( + scenarioList[2], + "reply with only the gender neutral " + LANGUAGE + " translation of the NPC name", + True, + ) + descListTL2 = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + + # Check Mismatch + if len(nameListTL) != len(scenarioList[0]) or len(descListTL1) != len(scenarioList[1]) or len(descListTL2) != len(scenarioList[2]): + with LOCK: + if filename not in MISMATCH: + MISMATCH.append(filename) + else: + scenarioListTL = [nameListTL, descListTL1, descListTL2] + translate = True + + # OPTIONS + if optionsList[0] or optionsList[1]: + # Progress Bar + total = 0 + for optionsArray in optionsList: + total += len(optionsArray) + pbar.total = total + pbar.refresh() + + # Name + response = translateAI( + optionsList[0], + "Reply with only the " + LANGUAGE + " translation", + True, + ) + nameListTL = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + # Desc 1 + response = translateAI( + optionsList[1], + "reply with only the gender neutral " + LANGUAGE + " translation of the NPC name", + True, + ) + descListTL1 = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + # Desc 2 + response = translateAI( + optionsList[2], + "reply with only the gender neutral " + LANGUAGE + " translation of the NPC name", + True, + ) + descListTL2 = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + + # Check Mismatch + if len(nameListTL) != len(optionsList[0]) or len(descListTL1) != len(optionsList[1]) or len(descListTL2) != len(optionsList[2]): + with LOCK: + if filename not in MISMATCH: + MISMATCH.append(filename) + else: + optionsListTL = [nameListTL, descListTL1, descListTL2] + translate = True + + # ITEMS + if len(itemList[0]) > 0 or len(itemList[1]) > 0: + # Progress Bar + total = 0 + for itemArray in itemList: + total += len(itemArray) + pbar.total = total + pbar.refresh() + + # Name + response = translateAI(itemList[0], "Reply with only the " + LANGUAGE + " translation") + nameListTL = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + # Desc 1 + response = translateAI(itemList[1], "Reply with only the " + LANGUAGE + " translation") + descListTL1 = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + # Desc 2 + response = translateAI(itemList[2], "Reply with only the " + LANGUAGE + " translation") + descListTL2 = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + # Desc 3 + response = translateAI(itemList[3], "Reply with only the " + LANGUAGE + " translation") + descListTL3 = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + + # Check Mismatch + if ( + len(nameListTL) != len(itemList[0]) + or len(descListTL1) != len(itemList[1]) + or len(descListTL2) != len(itemList[2]) + or len(descListTL3) != len(itemList[3]) + ): + with LOCK: + if filename not in MISMATCH: + MISMATCH.append(filename) + else: + itemListTL = [nameListTL, descListTL1, descListTL2, descListTL3] + translate = True + + # Armor + if len(armorList[0]) > 0: + # Progress Bar + total = 0 + for armorArray in armorList: + total += len(armorArray) + pbar.total = total + pbar.refresh() + + # Name + response = translateAI( + armorList[0], + "Reply with only the " + LANGUAGE + " translation of the NPC name", + True, + ) + nameListTL = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + # Desc 1 + response = translateAI(armorList[1], "Reply with only the " + LANGUAGE + " translation") + descListTL1 = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + + # Check Mismatch + if len(nameListTL) != len(armorList[0]) or len(descListTL1) != len(armorList[1]): + with LOCK: + if filename not in MISMATCH: + MISMATCH.append(filename) + else: + armorListTL = [nameListTL, descListTL1] + translate = True + + # Enemies + if len(enemyList[0]) > 0: + # Progress Bar + total = 0 + for enemyArray in enemyList: + total += len(enemyArray) + pbar.total = total + pbar.refresh() + + # Name + response = translateAI( + enemyList[0], + "Reply with only the " + LANGUAGE + " translation of the enemy NPC name", + True, + ) + nameListTL = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + # Desc 1 + response = translateAI(enemyList[1], "Reply with only the " + LANGUAGE + " translation") + descListTL1 = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + + # Check Mismatch + if len(nameListTL) != len(enemyList[0]) or len(descListTL1) != len(enemyList[1]): + with LOCK: + if filename not in MISMATCH: + MISMATCH.append(filename) + else: + enemyListTL = [nameListTL, descListTL1] + translate = True + + # Weapons + if len(weaponsList[0]) > 0: + # Progress Bar + total = 0 + for weaponsArray in weaponsList: + total += len(weaponsArray) + pbar.total = total + pbar.refresh() + + # Name + response = translateAI( + weaponsList[0], + "Reply with only the " + LANGUAGE + " translation of the RPG weapon name", + True, + ) + nameListTL = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + # Desc 1 + response = translateAI(weaponsList[1], "") + descListTL1 = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + # Desc 2 + response = translateAI(weaponsList[2], "") + descListTL2 = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + + # Check Mismatch + if len(nameListTL) != len(weaponsList[0]) or len(descListTL1) != len(weaponsList[1]) or len(descListTL2) != len(weaponsList[2]): + with LOCK: + if filename not in MISMATCH: + MISMATCH.append(filename) + else: + weaponsListTL = [nameListTL, descListTL1, descListTL2] + translate = True + + # Skills + if len(skillList[0]) > 0: + # Progress Bar + total = 0 + for skillArray in skillList: + total += len(skillArray) + pbar.total = total + pbar.refresh() + + # Name + response = translateAI( + skillList[0], + "Reply with only the " + LANGUAGE + " translation of the RPG skill name", + True, + ) + nameListTL = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + + # Desc + response = translateAI( + skillList[1], + "Reply with only the " + LANGUAGE + " translation of the RPG skill description", + True, + ) + descListTL1 = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + + # Log 1 + response = translateAI( + skillList[2], + "reply with only the gender neutral " + + LANGUAGE + + " translation of the action log. Always start the sentence with Taro. For example, Translate 'Taroを倒した！' as 'Taro was defeated!'", + True, + ) + descListTL2 = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + + # Log 2 + response = translateAI( + skillList[3], + "reply with only the gender neutral " + + LANGUAGE + + " translation of the action log. Always start the sentence with Taro. For example, Translate 'Taroを倒した！' as 'Taro was defeated!'", + True, + ) + descListTL3 = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + + # Log 3 + response = translateAI( + skillList[4], + "reply with only the gender neutral " + + LANGUAGE + + " translation of the action log. Always start the sentence with Taro. For example, Translate 'Taroを倒した！' as 'Taro was defeated!'", + True, + ) + descListTL4 = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + + # Check Mismatch + if ( + len(nameListTL) != len(skillList[0]) + or len(descListTL1) != len(skillList[1]) + or len(descListTL2) != len(skillList[2]) + or len(descListTL3) != len(skillList[3]) + or len(descListTL4) != len(skillList[4]) + ): + with LOCK: + if filename not in MISMATCH: + MISMATCH.append(filename) + else: + skillListTL = [nameListTL, descListTL1, descListTL2, descListTL3, descListTL4] + translate = True + + # State + for list in stateList: + if len(list) > 0: + # Progress Bar + total = 0 + for stateArray in stateList: + total += len(stateArray) + pbar.total = total + pbar.refresh() + + # Name + response = translateAI( + stateList[0], + f"Reply with the {LANGUAGE} translation of the status effect.", + True, + ) + nameListTL = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + + # Desc 1 + response = translateAI(stateList[1], "") + descListTL1 = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + + # Log 1 + response = translateAI( + stateList[2], + f"Reply with the {LANGUAGE} translation of the status effect.", + True, + ) + descListTL2 = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + + # Log 2 + response = translateAI( + stateList[3], + "reply with only the gender neutral " + + LANGUAGE + + " translation of the action log. Always start the sentence with Taro. For example, Translate 'Taroを倒した！' as 'Taro was defeated!'", + True, + ) + descListTL3 = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + + # Log 3 + response = translateAI( + stateList[4], + "reply with only the gender neutral " + + LANGUAGE + + " translation of the action log. Always start the sentence with Taro. For example, Translate 'Taroを倒した！' as 'Taro was defeated!'", + True, + ) + descListTL4 = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + + # Log 4 + response = translateAI( + stateList[5], + "reply with only the gender neutral " + + LANGUAGE + + " translation of the action log. Always start the sentence with Taro. For example, Translate 'Taroを倒した！' as 'Taro was defeated!'", + True, + ) + descListTL5 = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + + # Log 1 + response = translateAI( + stateList[6], + "reply with only the gender neutral " + + LANGUAGE + + " translation of the action log. Always start the sentence with Taro. For example, Translate 'Taroを倒した！' as 'Taro was defeated!'", + True, + ) + descListTL6 = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + + # Log 1 + response = translateAI( + stateList[7], + "reply with only the gender neutral " + + LANGUAGE + + " translation of the action log. Always start the sentence with Taro. For example, Translate 'Taroを倒した！' as 'Taro was defeated!'", + True, + ) + descListTL7 = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + + # Check Mismatch + if ( + len(nameListTL) != len(stateList[0]) + or len(descListTL1) != len(stateList[1]) + or len(descListTL2) != len(stateList[2]) + or len(descListTL3) != len(stateList[3]) + or len(descListTL4) != len(stateList[4]) + or len(descListTL5) != len(stateList[5]) + or len(descListTL6) != len(stateList[6]) + ): + with LOCK: + if filename not in MISMATCH: + MISMATCH.append(filename) + else: + stateListTL = [ + nameListTL, + descListTL1, + descListTL2, + descListTL3, + descListTL4, + descListTL5, + descListTL6, + descListTL7, + ] + translate = True + + # DB Names + if len(dbNameList[0]) > 0: + # Progress Bar + total = 0 + for dbNameArray in dbNameList: + total += len(dbNameArray) + pbar.total = total + pbar.refresh() + + # Name + response = translateAI(dbNameList[0], "Reply with only the " + LANGUAGE + " translation") + nameListTL = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + + # Check Mismatch + if len(nameListTL) != len(dbNameList[0]): + with LOCK: + if filename not in MISMATCH: + MISMATCH.append(filename) + else: + dbNameListTL = [nameListTL] + translate = True + + # DB Values + if len(dbValueList[0]) > 0: + # Progress Bar + total = 0 + for dbValueArray in dbValueList: + total += len(dbValueArray) + pbar.total = total + pbar.refresh() + + # Name + response = translateAI(dbValueList[0], "Reply with only the " + LANGUAGE + " translation") + valueListTL = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + + # Check Mismatch + if len(valueListTL) != len(dbValueList[0]): + with LOCK: + if filename not in MISMATCH: + MISMATCH.append(filename) + else: + dbValueListTL = [valueListTL] + translate = True + + # Start Pass 2 + if translate == True: + jobList.append(scenarioListTL) + jobList.append(npcListTL) + jobList.append(itemListTL) + jobList.append(stateListTL) + jobList.append(armorListTL) + jobList.append(enemyListTL) + jobList.append(weaponsListTL) + jobList.append(skillListTL) + jobList.append(optionsListTL) + jobList.append(dbNameListTL) + jobList.append(dbValueListTL) + searchDB(events, pbar, jobList, filename) + + except IndexError as e: + traceback.print_exc() + raise Exception(str(e) + "Failed to translate: " + initialJAString) from None + except Exception as e: + traceback.print_exc() + raise Exception(str(e) + "Failed to translate: " + initialJAString) from None + + return totalTokens + +# Save some money and enter the character before translation +def getSpeaker(speaker): + match speaker: + case "ファイン": + return ["Fine", [0, 0]] + case "": + return ["", [0, 0]] + case _: + # Find Speaker + for i in range(len(NAMESLIST)): + if speaker == NAMESLIST[i][0]: + return [NAMESLIST[i][1], [0, 0]] + + # Translate and Store Speaker + response = translateAI( + f"{speaker}", + "Reply with the " + LANGUAGE + " translation of the NPC name.", + False, + ) + response[0] = response[0].title() + response[0] = response[0].replace("'S", "'s") + response[0] = response[0].replace("Speaker: ", "") + + # Retry if name doesn't translate for some reason + if re.search(r"([a-zA-Z？?])", response[0]) == None: + response = translateAI( + f"{speaker}", + "Reply with the " + LANGUAGE + " translation of the NPC name.", + False, + ) + response[0] = response[0].title() + response[0] = response[0].replace("'S", "'s") + + speakerList = [speaker, response[0]] + NAMESLIST.append(speakerList) + return response + return [speaker, [0, 0]] + +def translateAI(text, history, history_ctx=None): + """ + Legacy wrapper function for the new shared translation utility. + This maintains compatibility with existing code while using the new shared implementation. + """ + global PBAR, MISMATCH, FILENAME + + # Update config estimate mode based on global ESTIMATE + TRANSLATION_CONFIG.estimateMode = bool(ESTIMATE) + + # Call the new shared translation function + return sharedtranslateAI( + text=text, + history=history, + config=TRANSLATION_CONFIG, + filename=FILENAME, + pbar=PBAR, + lock=LOCK, + mismatchList=MISMATCH + ) diff --git a/modules/wolf2.py b/modules/wolf2.py index 3b6ec64..194f8d2 100644 --- a/modules/wolf2.py +++ b/modules/wolf2.py @@ -1,420 +1,422 @@ -# Libraries -import json -import io -import os -import re -import util.dazedwrap as dazedwrap -import threading -import time -import traceback -import tiktoken -from pathlib import Path -from colorama import Fore -from dotenv import load_dotenv -from retry import retry -from tqdm import tqdm -from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost -import tempfile - -# OpenAI initialization centralized in util/translation.py - -# Globals -MODEL = os.getenv("model") -TIMEOUT = int(os.getenv("timeout")) -LANGUAGE = os.getenv("language").capitalize() -PROMPT = Path("prompt.txt").read_text(encoding="utf-8") -VOCAB = Path("vocab.txt").read_text(encoding="utf-8") -LOCK = threading.Lock() -WIDTH = int(os.getenv("width")) -LISTWIDTH = int(os.getenv("listWidth")) -NOTEWIDTH = 70 -MAXHISTORY = 10 -ESTIMATE = "" -TOKENS = [0, 0] -NAMESLIST = [] -NAMES = False # Output a list of all the character names found -BRFLAG = False # If the game uses
instead -FIXTEXTWRAP = True # Overwrites textwrap -IGNORETLTEXT = False # Ignores all translated text. -MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong) -FILENAME = "" # Current file being processed, used by translateAI wrapper - -# tqdm Globals -BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}" -POSITION = 0 -LEAVE = False - -# Regex - Need to change this if you want to translate from/to other languages. Default is Japanese Regex -LANGREGEX = r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+" - -# Get pricing configuration based on the model -PRICING_CONFIG = getPricingConfig(MODEL) -INPUTAPICOST = PRICING_CONFIG["inputAPICost"] -OUTPUTAPICOST = PRICING_CONFIG["outputAPICost"] -BATCHSIZE = PRICING_CONFIG["batchSize"] -FREQUENCY_PENALTY = PRICING_CONFIG["frequencyPenalty"] - -# Initialize Translation Config -TRANSLATION_CONFIG = TranslationConfig( - model=MODEL, - language=LANGUAGE, - prompt=PROMPT, - vocab=VOCAB, - langRegex=LANGREGEX, - batchSize=BATCHSIZE, - maxHistory=MAXHISTORY, - estimateMode=False # Will be set dynamically based on ESTIMATE -) -LEAVE = False - -def handleWOLF2(filename, estimate): - global ESTIMATE, FILENAME - ESTIMATE = estimate - FILENAME = filename - - if ESTIMATE: - start = time.time() - translatedData = openFiles(filename) - - # Print Result - end = time.time() - tqdm.write(getResultString(translatedData, end - start, filename)) - with LOCK: - TOKENS[0] += translatedData[1][0] - TOKENS[1] += translatedData[1][1] - - # Print Total - totalString = getResultString(["", TOKENS, None], end - start, "TOTAL") - - # Print any errors on maps - if len(MISMATCH) > 0: - return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET - else: - return totalString - - else: - try: - with open("translated/" + filename, "w", encoding="shift_jis", errors="ignore") as outFile: - start = time.time() - translatedData = openFiles(filename) - - # Print Result - end = time.time() - outFile.writelines(translatedData[0]) - tqdm.write(getResultString(translatedData, end - start, filename)) - with LOCK: - TOKENS[0] += translatedData[1][0] - TOKENS[1] += translatedData[1][1] - except Exception: - traceback.print_exc() - return "Fail" - - return getResultString(["", TOKENS, None], end - start, "TOTAL") - - -def getResultString(translatedData, translationTime, filename): - # File Print String - cost = calculateCost(translatedData[1][0], translatedData[1][1], MODEL) - totalTokenstring = ( - Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]" - "[Output: " - + str(translatedData[1][1]) - + "]" "[Cost: ${:,.4f}".format(cost) - + "]" - ) - timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]" - - if translatedData[2] == None: - # Success - return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET - - else: - # Fail - try: - raise translatedData[2] - except Exception as e: - traceback.print_exc() - errorString = str(e) + Fore.RED - return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET - - -def openFiles(filename): - # Use a robust reader to handle cp932/Shift_JIS variants and occasional bad bytes - def read_file_lines_with_fallback(path: str): - encodings = ["cp932", "shift_jis", "utf-8-sig", "utf-8"] - for enc in encodings: - try: - with open(path, "r", encoding=enc) as f: - return f.readlines() - except UnicodeDecodeError: - continue - # Last resort: ignore undecodable bytes under cp932 - with open(path, "rb") as f: - raw = f.read() - try: - text = raw.decode("cp932", errors="ignore") - except Exception: - text = raw.decode("latin-1", errors="ignore") - return text.splitlines(keepends=True) - - path = os.path.join("files", filename) - lines = read_file_lines_with_fallback(path) - - # Keep parseWOLF API by wrapping lines in a file-like object - with io.StringIO("".join(lines)) as readFile: - translatedData = parseWOLF(readFile, filename) - - # Delete lines marked for deletion - finalData = [] - for line in translatedData[0]: - if line != "\\d\n": - finalData.append(line) - translatedData[0] = finalData - - return translatedData - - -def parseWOLF(readFile, filename): - totalTokens = [0, 0] - - # Read File into data - data = readFile.readlines() - - # Create Progress Bar - with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar: - pbar.desc = filename - - try: - result = translateWOLF(data, [], pbar, filename) - totalTokens[0] += result[0] - totalTokens[1] += result[1] - except Exception as e: - traceback.print_exc() - return [data, totalTokens, e] - return [data, totalTokens, None] - - -def save_progress_lines(lines, filename, encoding="shift_jis"): - """Atomically save current line-based translation progress.""" - try: - if ESTIMATE: - return - os.makedirs("translated", exist_ok=True) - tmp_fd, tmp_path = tempfile.mkstemp(prefix=f"{filename}.", suffix=".tmp", dir="translated") - try: - with os.fdopen(tmp_fd, "w", encoding=encoding, newline="\n", errors="ignore") as tmp_file: - tmp_file.writelines(lines) - os.replace(tmp_path, os.path.join("translated", filename)) - finally: - if os.path.exists(tmp_path): - try: - os.remove(tmp_path) - except OSError: - pass - except Exception: - traceback.print_exc() - - -def saveCheckLines(lines, filename, tokens=None, encoding="shift_jis"): - try: - if tokens is not None: - if not (isinstance(tokens, (list, tuple)) and len(tokens) >= 2 and (tokens[0] or tokens[1])): - return - save_progress_lines(lines, filename, encoding=encoding) - except Exception: - traceback.print_exc() - - -def translateWOLF(data, translatedList, pbar, filename): - stringList = [] - currentGroup = [] - tokens = [0, 0] - speaker = "" - global LOCK, ESTIMATE, PBAR - PBAR = pbar - i = 0 - - while i < len(data): - # Speaker - matchList = re.findall(r"^([^/]*)：", data[i]) - if len(matchList) != 0: - response = getSpeaker(matchList[0]) - speaker = response[0] - tokens[0] += response[1][0] - tokens[1] += response[1][1] - data[i] = data[i].replace(matchList[0], f"{speaker}") - saveCheckLines(data, filename) - i += 1 - else: - speaker = "" - - # Options - if "//選択肢" in data[i]: - i += 1 - choiceList = [] - initialIndex = i - while "//" in data[i] and "の場合" not in data[i]: - choiceList.append(re.search(r"\/\/(.*)", data[i]).group(1)) - i += 1 - - # Translate - response = translateAI(choiceList, "This will be a dialogue option") - tokens[0] += response[1][0] - tokens[1] += response[1][1] - choiceListTL = response[0] - - # Set Data - if len(choiceList) == len(choiceListTL): - # Set Data - i = initialIndex - while "//" in data[i] and "の場合" not in data[i]: - choiceListTL[0] = choiceListTL[0].replace(", ", "、") - data[i] = f"//{choiceListTL[0]}\n" - choiceListTL.pop(0) - i += 1 - saveCheckLines(data, filename) - - # Mismatch - else: - with LOCK: - if filename not in MISMATCH: - MISMATCH.append(filename) - - # Lines - if r"/" not in data[i] and "@" not in data[i] and data[i] != "\n": - # Pass 1 - if translatedList == []: - # Grab Consecutive Strings - currentGroup.append(data[i]) - i += 1 - while i < len(data) and r"/" not in data[i] and "@" not in data[i] and data[i] != "\n": - currentGroup.append(data[i]) - i += 1 - - # Join up 401 groups for better translation. - if len(currentGroup) > 0: - jaString = "".join(currentGroup) - currentGroup = [] - - # Remove any textwrap - jaString = jaString.replace("\n", " ") - - # Add Speaker (If there is one) - if speaker != "": - jaString = f"[{speaker}]: {jaString}" - - # Add String - stringList.append(jaString) - i += 1 - - # Pass 2 - else: - # Insert Strings - while i < len(data) and r"/" not in data[i] and "@" not in data[i] and data[i] != "\n": - data.pop(i) - - # Get Text - translatedText = translatedList[0] - translatedList.pop(0) - - if len(translatedList) <= 0: - translatedList = None - - # Remove speaker - matchSpeakerList = re.findall(r"^(\[.+?\]\s?[|:]\s?)\s?", translatedText) - if len(matchSpeakerList) > 0: - translatedText = translatedText.replace(matchSpeakerList[0], "") - - # Textwrap - translatedText = dazedwrap.wrapText(translatedText, width=WIDTH) - - # Set Data - data.insert(i, f"{translatedText}\n") - saveCheckLines(data, filename) - i += 1 - - # Nothing relevant. Skip Line. - else: - i += 1 - - # EOF - if len(stringList) > 0: - # Set Progress - pbar.total = len(stringList) - pbar.refresh() - - # Translate - response = translateAI(stringList, "") - tokens[0] += response[1][0] - tokens[1] += response[1][1] - translatedList = response[0] - - # Set Strings - if len(stringList) == len(translatedList): - translateWOLF(data, translatedList, pbar, filename) - - # Mismatch - else: - with LOCK: - if filename not in MISMATCH: - MISMATCH.append(filename) - return tokens - -# Save some money and enter the character before translation -def getSpeaker(speaker): - match speaker: - case "ファイン": - return ["Fine", [0, 0]] - case "": - return ["", [0, 0]] - case _: - # Find Speaker - for i in range(len(NAMESLIST)): - if speaker == NAMESLIST[i][0]: - return [NAMESLIST[i][1], [0, 0]] - - # Translate and Store Speaker - response = translateAI( - f"{speaker}", - "Reply with the " + LANGUAGE + " translation of the NPC name.", - False, - ) - response[0] = response[0].title() - response[0] = response[0].replace("'S", "'s") - response[0] = response[0].replace("Speaker: ", "") - - # Retry if name doesn't translate for some reason - if re.search(r"([a-zA-Z？?])", response[0]) == None: - response = translateAI( - f"{speaker}", - "Reply with the " + LANGUAGE + " translation of the NPC name.", - False, - ) - response[0] = response[0].title() - response[0] = response[0].replace("'S", "'s") - - speakerList = [speaker, response[0]] - NAMESLIST.append(speakerList) - return response - return [speaker, [0, 0]] - -def translateAI(text, history, history_ctx=None): - """ - Legacy wrapper function for the new shared translation utility. - This maintains compatibility with existing code while using the new shared implementation. - """ - global PBAR, MISMATCH, FILENAME - - # Update config estimate mode based on global ESTIMATE - TRANSLATION_CONFIG.estimateMode = bool(ESTIMATE) - - # Call the new shared translation function - return sharedtranslateAI( - text=text, - history=history, - config=TRANSLATION_CONFIG, - filename=FILENAME, - pbar=PBAR, - lock=LOCK, - mismatchList=MISMATCH - ) +# Libraries +import json +import io +import os +import re +import util.dazedwrap as dazedwrap +import threading +import time +import traceback +import tiktoken +from pathlib import Path +from colorama import Fore +from dotenv import load_dotenv +from retry import retry +from tqdm import tqdm +from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost +import tempfile + +# OpenAI initialization centralized in util/translation.py + +# Globals +MODEL = os.getenv("model") +TIMEOUT = int(os.getenv("timeout")) +LANGUAGE = os.getenv("language").capitalize() +from util.paths import PROMPT_PATH, VOCAB_PATH + +PROMPT = PROMPT_PATH.read_text(encoding="utf-8") +VOCAB = VOCAB_PATH.read_text(encoding="utf-8") +LOCK = threading.Lock() +WIDTH = int(os.getenv("width")) +LISTWIDTH = int(os.getenv("listWidth")) +NOTEWIDTH = 70 +MAXHISTORY = 10 +ESTIMATE = "" +TOKENS = [0, 0] +NAMESLIST = [] +NAMES = False # Output a list of all the character names found +BRFLAG = False # If the game uses
instead +FIXTEXTWRAP = True # Overwrites textwrap +IGNORETLTEXT = False # Ignores all translated text. +MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong) +FILENAME = "" # Current file being processed, used by translateAI wrapper + +# tqdm Globals +BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}" +POSITION = 0 +LEAVE = False + +# Regex - Need to change this if you want to translate from/to other languages. Default is Japanese Regex +LANGREGEX = r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+" + +# Get pricing configuration based on the model +PRICING_CONFIG = getPricingConfig(MODEL) +INPUTAPICOST = PRICING_CONFIG["inputAPICost"] +OUTPUTAPICOST = PRICING_CONFIG["outputAPICost"] +BATCHSIZE = PRICING_CONFIG["batchSize"] +FREQUENCY_PENALTY = PRICING_CONFIG["frequencyPenalty"] + +# Initialize Translation Config +TRANSLATION_CONFIG = TranslationConfig( + model=MODEL, + language=LANGUAGE, + prompt=PROMPT, + vocab=VOCAB, + langRegex=LANGREGEX, + batchSize=BATCHSIZE, + maxHistory=MAXHISTORY, + estimateMode=False # Will be set dynamically based on ESTIMATE +) +LEAVE = False + +def handleWOLF2(filename, estimate): + global ESTIMATE, FILENAME + ESTIMATE = estimate + FILENAME = filename + + if ESTIMATE: + start = time.time() + translatedData = openFiles(filename) + + # Print Result + end = time.time() + tqdm.write(getResultString(translatedData, end - start, filename)) + with LOCK: + TOKENS[0] += translatedData[1][0] + TOKENS[1] += translatedData[1][1] + + # Print Total + totalString = getResultString(["", TOKENS, None], end - start, "TOTAL") + + # Print any errors on maps + if len(MISMATCH) > 0: + return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET + else: + return totalString + + else: + try: + with open("translated/" + filename, "w", encoding="shift_jis", errors="ignore") as outFile: + start = time.time() + translatedData = openFiles(filename) + + # Print Result + end = time.time() + outFile.writelines(translatedData[0]) + tqdm.write(getResultString(translatedData, end - start, filename)) + with LOCK: + TOKENS[0] += translatedData[1][0] + TOKENS[1] += translatedData[1][1] + except Exception: + traceback.print_exc() + return "Fail" + + return getResultString(["", TOKENS, None], end - start, "TOTAL") + + +def getResultString(translatedData, translationTime, filename): + # File Print String + cost = calculateCost(translatedData[1][0], translatedData[1][1], MODEL) + totalTokenstring = ( + Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]" + "[Output: " + + str(translatedData[1][1]) + + "]" "[Cost: ${:,.4f}".format(cost) + + "]" + ) + timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]" + + if translatedData[2] == None: + # Success + return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET + + else: + # Fail + try: + raise translatedData[2] + except Exception as e: + traceback.print_exc() + errorString = str(e) + Fore.RED + return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET + + +def openFiles(filename): + # Use a robust reader to handle cp932/Shift_JIS variants and occasional bad bytes + def read_file_lines_with_fallback(path: str): + encodings = ["cp932", "shift_jis", "utf-8-sig", "utf-8"] + for enc in encodings: + try: + with open(path, "r", encoding=enc) as f: + return f.readlines() + except UnicodeDecodeError: + continue + # Last resort: ignore undecodable bytes under cp932 + with open(path, "rb") as f: + raw = f.read() + try: + text = raw.decode("cp932", errors="ignore") + except Exception: + text = raw.decode("latin-1", errors="ignore") + return text.splitlines(keepends=True) + + path = os.path.join("files", filename) + lines = read_file_lines_with_fallback(path) + + # Keep parseWOLF API by wrapping lines in a file-like object + with io.StringIO("".join(lines)) as readFile: + translatedData = parseWOLF(readFile, filename) + + # Delete lines marked for deletion + finalData = [] + for line in translatedData[0]: + if line != "\\d\n": + finalData.append(line) + translatedData[0] = finalData + + return translatedData + + +def parseWOLF(readFile, filename): + totalTokens = [0, 0] + + # Read File into data + data = readFile.readlines() + + # Create Progress Bar + with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar: + pbar.desc = filename + + try: + result = translateWOLF(data, [], pbar, filename) + totalTokens[0] += result[0] + totalTokens[1] += result[1] + except Exception as e: + traceback.print_exc() + return [data, totalTokens, e] + return [data, totalTokens, None] + + +def save_progress_lines(lines, filename, encoding="shift_jis"): + """Atomically save current line-based translation progress.""" + try: + if ESTIMATE: + return + os.makedirs("translated", exist_ok=True) + tmp_fd, tmp_path = tempfile.mkstemp(prefix=f"{filename}.", suffix=".tmp", dir="translated") + try: + with os.fdopen(tmp_fd, "w", encoding=encoding, newline="\n", errors="ignore") as tmp_file: + tmp_file.writelines(lines) + os.replace(tmp_path, os.path.join("translated", filename)) + finally: + if os.path.exists(tmp_path): + try: + os.remove(tmp_path) + except OSError: + pass + except Exception: + traceback.print_exc() + + +def saveCheckLines(lines, filename, tokens=None, encoding="shift_jis"): + try: + if tokens is not None: + if not (isinstance(tokens, (list, tuple)) and len(tokens) >= 2 and (tokens[0] or tokens[1])): + return + save_progress_lines(lines, filename, encoding=encoding) + except Exception: + traceback.print_exc() + + +def translateWOLF(data, translatedList, pbar, filename): + stringList = [] + currentGroup = [] + tokens = [0, 0] + speaker = "" + global LOCK, ESTIMATE, PBAR + PBAR = pbar + i = 0 + + while i < len(data): + # Speaker + matchList = re.findall(r"^([^/]*)：", data[i]) + if len(matchList) != 0: + response = getSpeaker(matchList[0]) + speaker = response[0] + tokens[0] += response[1][0] + tokens[1] += response[1][1] + data[i] = data[i].replace(matchList[0], f"{speaker}") + saveCheckLines(data, filename) + i += 1 + else: + speaker = "" + + # Options + if "//選択肢" in data[i]: + i += 1 + choiceList = [] + initialIndex = i + while "//" in data[i] and "の場合" not in data[i]: + choiceList.append(re.search(r"\/\/(.*)", data[i]).group(1)) + i += 1 + + # Translate + response = translateAI(choiceList, "This will be a dialogue option") + tokens[0] += response[1][0] + tokens[1] += response[1][1] + choiceListTL = response[0] + + # Set Data + if len(choiceList) == len(choiceListTL): + # Set Data + i = initialIndex + while "//" in data[i] and "の場合" not in data[i]: + choiceListTL[0] = choiceListTL[0].replace(", ", "、") + data[i] = f"//{choiceListTL[0]}\n" + choiceListTL.pop(0) + i += 1 + saveCheckLines(data, filename) + + # Mismatch + else: + with LOCK: + if filename not in MISMATCH: + MISMATCH.append(filename) + + # Lines + if r"/" not in data[i] and "@" not in data[i] and data[i] != "\n": + # Pass 1 + if translatedList == []: + # Grab Consecutive Strings + currentGroup.append(data[i]) + i += 1 + while i < len(data) and r"/" not in data[i] and "@" not in data[i] and data[i] != "\n": + currentGroup.append(data[i]) + i += 1 + + # Join up 401 groups for better translation. + if len(currentGroup) > 0: + jaString = "".join(currentGroup) + currentGroup = [] + + # Remove any textwrap + jaString = jaString.replace("\n", " ") + + # Add Speaker (If there is one) + if speaker != "": + jaString = f"[{speaker}]: {jaString}" + + # Add String + stringList.append(jaString) + i += 1 + + # Pass 2 + else: + # Insert Strings + while i < len(data) and r"/" not in data[i] and "@" not in data[i] and data[i] != "\n": + data.pop(i) + + # Get Text + translatedText = translatedList[0] + translatedList.pop(0) + + if len(translatedList) <= 0: + translatedList = None + + # Remove speaker + matchSpeakerList = re.findall(r"^(\[.+?\]\s?[|:]\s?)\s?", translatedText) + if len(matchSpeakerList) > 0: + translatedText = translatedText.replace(matchSpeakerList[0], "") + + # Textwrap + translatedText = dazedwrap.wrapText(translatedText, width=WIDTH) + + # Set Data + data.insert(i, f"{translatedText}\n") + saveCheckLines(data, filename) + i += 1 + + # Nothing relevant. Skip Line. + else: + i += 1 + + # EOF + if len(stringList) > 0: + # Set Progress + pbar.total = len(stringList) + pbar.refresh() + + # Translate + response = translateAI(stringList, "") + tokens[0] += response[1][0] + tokens[1] += response[1][1] + translatedList = response[0] + + # Set Strings + if len(stringList) == len(translatedList): + translateWOLF(data, translatedList, pbar, filename) + + # Mismatch + else: + with LOCK: + if filename not in MISMATCH: + MISMATCH.append(filename) + return tokens + +# Save some money and enter the character before translation +def getSpeaker(speaker): + match speaker: + case "ファイン": + return ["Fine", [0, 0]] + case "": + return ["", [0, 0]] + case _: + # Find Speaker + for i in range(len(NAMESLIST)): + if speaker == NAMESLIST[i][0]: + return [NAMESLIST[i][1], [0, 0]] + + # Translate and Store Speaker + response = translateAI( + f"{speaker}", + "Reply with the " + LANGUAGE + " translation of the NPC name.", + False, + ) + response[0] = response[0].title() + response[0] = response[0].replace("'S", "'s") + response[0] = response[0].replace("Speaker: ", "") + + # Retry if name doesn't translate for some reason + if re.search(r"([a-zA-Z？?])", response[0]) == None: + response = translateAI( + f"{speaker}", + "Reply with the " + LANGUAGE + " translation of the NPC name.", + False, + ) + response[0] = response[0].title() + response[0] = response[0].replace("'S", "'s") + + speakerList = [speaker, response[0]] + NAMESLIST.append(speakerList) + return response + return [speaker, [0, 0]] + +def translateAI(text, history, history_ctx=None): + """ + Legacy wrapper function for the new shared translation utility. + This maintains compatibility with existing code while using the new shared implementation. + """ + global PBAR, MISMATCH, FILENAME + + # Update config estimate mode based on global ESTIMATE + TRANSLATION_CONFIG.estimateMode = bool(ESTIMATE) + + # Call the new shared translation function + return sharedtranslateAI( + text=text, + history=history, + config=TRANSLATION_CONFIG, + filename=FILENAME, + pbar=PBAR, + lock=LOCK, + mismatchList=MISMATCH + ) diff --git a/modules/yuris.py b/modules/yuris.py index 6995515..739b205 100644 --- a/modules/yuris.py +++ b/modules/yuris.py @@ -21,8 +21,10 @@ from util.translation import ( MODEL = os.getenv("model") LANGUAGE = os.getenv("language").capitalize() -PROMPT = Path("prompt.txt").read_text(encoding="utf-8") -VOCAB = Path("vocab.txt").read_text(encoding="utf-8") +from util.paths import PROMPT_PATH, VOCAB_PATH + +PROMPT = PROMPT_PATH.read_text(encoding="utf-8") +VOCAB = VOCAB_PATH.read_text(encoding="utf-8") LOCK = threading.Lock() WIDTH = int(os.getenv("width")) MAXHISTORY = 10 diff --git a/scripts/launch.sh b/scripts/launch.sh new file mode 100644 index 0000000..ec96110 --- /dev/null +++ b/scripts/launch.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# Launcher for DazedMTLTool.desktop (finds venv, then starts the GUI). +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT" || exit 1 + +PYTHON="python3" +if [[ -x "$ROOT/.venv/bin/python" ]]; then + PYTHON="$ROOT/.venv/bin/python" +elif [[ -x "$ROOT/venv/bin/python" ]]; then + PYTHON="$ROOT/venv/bin/python" +fi + +exec "$PYTHON" "$ROOT/scripts/start_gui.py" diff --git a/start_gui.py b/scripts/start_gui.py similarity index 76% rename from start_gui.py rename to scripts/start_gui.py index 276c76b..ef1ee19 100644 --- a/start_gui.py +++ b/scripts/start_gui.py @@ -1,72 +1,74 @@ -#!/usr/bin/env python3 -""" -Launch script for DazedMTLTool GUI -""" - -import sys -import os -from pathlib import Path - -# Add the project root to Python path -project_root = Path(__file__).parent -sys.path.insert(0, str(project_root)) - -def check_dependencies(): - """Check if required dependencies are installed.""" - missing_deps = [] - - try: - import PyQt5 - except ImportError: - missing_deps.append("PyQt5") - - try: - from dotenv import load_dotenv - except ImportError: - missing_deps.append("python-dotenv") - - if missing_deps: - print("Missing dependencies:") - for dep in missing_deps: - print(f" - {dep}") - print("\nPlease install them using:") - print(" pip install -r requirements_gui.txt") - return False - - return True - -def main(): - """Main entry point.""" - print("DazedMTLTool GUI Launcher") - print("=" * 40) - - # Check dependencies - if not check_dependencies(): - sys.exit(1) - - try: - from util.ace.update_tools import seed_ace_tools - seed_ace_tools() - except Exception as exc: - print(f"Warning: Ace tool setup failed ({exc}). Ace features may be unavailable.") - - try: - from util.forge.update_tools import seed_forge_plugins - seed_forge_plugins() - except Exception as exc: - print(f"Warning: Forge plugin setup failed ({exc}). Playtest Forge may be unavailable.") - - # Import and run GUI - try: - from gui.main import main as gui_main - gui_main() - except ImportError as e: - print(f"Error importing GUI modules: {e}") - print("Make sure all GUI files are in the 'gui' directory") - sys.exit(1) - except Exception as e: - print(f"Error starting GUI: {e}") - sys.exit(1) - -if __name__ == "__main__": - main() +#!/usr/bin/env python3 +"""Launch script for DazedMTLTool GUI.""" + +import sys +from pathlib import Path + +PROJECT_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(PROJECT_ROOT)) + + +def check_dependencies(): + """Check if required dependencies are installed.""" + missing_deps = [] + + try: + import PyQt5 # noqa: F401 + except ImportError: + missing_deps.append("PyQt5") + + try: + from dotenv import load_dotenv # noqa: F401 + except ImportError: + missing_deps.append("python-dotenv") + + if missing_deps: + print("Missing dependencies:") + for dep in missing_deps: + print(f" - {dep}") + print("\nPlease install them using:") + print(" pip install -r requirements.txt") + return False + + return True + + +def main(): + """Main entry point.""" + print("DazedMTLTool GUI Launcher") + print("=" * 40) + + from util.paths import ensure_vocab_file, migrate_root_data_files + + migrate_root_data_files() + ensure_vocab_file() + + if not check_dependencies(): + sys.exit(1) + + try: + from util.ace.update_tools import seed_ace_tools + seed_ace_tools() + except Exception as exc: + print(f"Warning: Ace tool setup failed ({exc}). Ace features may be unavailable.") + + try: + from util.forge.update_tools import seed_forge_plugins + seed_forge_plugins() + except Exception as exc: + print(f"Warning: Forge plugin setup failed ({exc}). Playtest Forge may be unavailable.") + + try: + from gui.main import main as gui_main + gui_main() + except ImportError as e: + print(f"Error importing GUI modules: {e}") + print("Make sure all GUI files are in the 'gui' directory") + sys.exit(1) + except Exception as e: + print(f"Error starting GUI: {e}") + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/set_defaults.py b/util/defaults.py similarity index 90% rename from set_defaults.py rename to util/defaults.py index 7f6c24f..2e92992 100644 --- a/set_defaults.py +++ b/util/defaults.py @@ -1,7 +1,9 @@ import re from copy import deepcopy -# Canonical defaults used by the GUI and the `set_defaults` script. +from util.paths import PROJECT_ROOT + +# Canonical defaults used by the GUI and the pre-commit hook. # Stored as booleans for easier consumption by the GUI. DEFAULTS = { 'FIRSTLINESPEAKERS': False, @@ -56,4 +58,4 @@ def set_defaults(file_path): if __name__ == "__main__": - set_defaults('modules/rpgmakermvmz.py') + set_defaults(PROJECT_ROOT / "modules/rpgmakermvmz.py") diff --git a/util/linux_desktop.py b/util/linux_desktop.py new file mode 100644 index 0000000..541c439 --- /dev/null +++ b/util/linux_desktop.py @@ -0,0 +1,72 @@ +"""Install/update the Linux .desktop entry for taskbar and window icons.""" + +from __future__ import annotations + +import os +import subprocess +import sys +from pathlib import Path + +from util.paths import ICON_PATH, PROJECT_ROOT + +DESKTOP_ID = "DazedMTLTool" +LAUNCH_SCRIPT = PROJECT_ROOT / "scripts" / "launch.sh" + + +def installed_desktop_path() -> Path: + data_home = os.environ.get("XDG_DATA_HOME", "") + if data_home: + base = Path(data_home) + else: + base = Path.home() / ".local" / "share" + return base / "applications" / f"{DESKTOP_ID}.desktop" + + +def _desktop_content(root: Path, icon: Path, launch: Path) -> str: + return ( + "[Desktop Entry]\n" + "Type=Application\n" + f"Name={DESKTOP_ID}\n" + "GenericName=Translation Tool\n" + "Comment=AI translation tool for visual novels and RPG games\n" + f"Exec={launch}\n" + f"Icon={icon}\n" + f"Path={root}\n" + "Terminal=false\n" + "Categories=Development;Utility;\n" + f"StartupWMClass={DESKTOP_ID}\n" + ) + + +def ensure_linux_desktop_entry() -> None: + """Write ~/.local/share/applications/DazedMTLTool.desktop with absolute paths.""" + if not sys.platform.startswith("linux"): + return + if not LAUNCH_SCRIPT.is_file() or not ICON_PATH.is_file(): + return + + desktop_path = installed_desktop_path() + content = _desktop_content(PROJECT_ROOT, ICON_PATH, LAUNCH_SCRIPT) + + try: + if desktop_path.is_file() and desktop_path.read_text(encoding="utf-8") == content: + return + desktop_path.parent.mkdir(parents=True, exist_ok=True) + desktop_path.write_text(content, encoding="utf-8") + desktop_path.chmod(0o755) + except OSError as exc: + print(f"Warning: could not install Linux desktop entry ({exc})") + return + + _refresh_desktop_database(desktop_path.parent) + + +def _refresh_desktop_database(applications_dir: Path) -> None: + for cmd in ( + ["update-desktop-database", str(applications_dir)], + ["kbuildsycoca5", "--noincremental"], + ): + try: + subprocess.run(cmd, check=False, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + except OSError: + pass diff --git a/util/paths.py b/util/paths.py new file mode 100644 index 0000000..a8f1648 --- /dev/null +++ b/util/paths.py @@ -0,0 +1,45 @@ +"""Canonical project paths (repo root, data files, config).""" + +from __future__ import annotations + +from pathlib import Path + +PROJECT_ROOT = Path(__file__).resolve().parent.parent +DATA_DIR = PROJECT_ROOT / "data" +VOCAB_PATH = DATA_DIR / "vocab.txt" +VOCAB_BASE_PATH = DATA_DIR / "vocab_base.txt" +PROMPT_PATH = DATA_DIR / "prompt.txt" +LAST_UPDATE_SHA_PATH = DATA_DIR / "last_update_sha.txt" +ENV_PATH = PROJECT_ROOT / ".env" +ICON_PATH = PROJECT_ROOT / "assets" / "icon.png" + +_ROOT_DATA_FILES = ( + "vocab.txt", + "vocab_base.txt", + "prompt.txt", + "last_update_sha.txt", +) + + +def migrate_root_data_files() -> None: + """Move legacy root-level data files into data/ on first run.""" + DATA_DIR.mkdir(parents=True, exist_ok=True) + for name in _ROOT_DATA_FILES: + src = PROJECT_ROOT / name + dst = DATA_DIR / name + if src.is_file() and not dst.exists(): + src.rename(dst) + + +migrate_root_data_files() + + +def ensure_vocab_file() -> None: + """Create data/vocab.txt from vocab_base.txt when missing.""" + migrate_root_data_files() + if VOCAB_PATH.is_file(): + return + if VOCAB_BASE_PATH.is_file(): + VOCAB_PATH.write_text(VOCAB_BASE_PATH.read_text(encoding="utf-8"), encoding="utf-8") + else: + VOCAB_PATH.write_text("", encoding="utf-8") diff --git a/util/translation.py b/util/translation.py index 39d335a..39685f4 100644 --- a/util/translation.py +++ b/util/translation.py @@ -1,3060 +1,3061 @@ -""" -Shared translation utilities for DazedMTLTool. -Centralized translation function used across all modules. -""" - -import os -import re -import json -import time -import random -import unicodedata -import tiktoken -import openai -import anthropic -import urllib.request -from openai import APIError, APIConnectionError, RateLimitError, APIStatusError -import hashlib -import threading -from contextlib import contextmanager -from dotenv import load_dotenv -from pathlib import Path -from retry import retry - -# Set to True to enable debug logging (token counts, cache costs, etc.) -DEBUG = True -_debug_request_log_lock = threading.Lock() - -# Set to True to disable Claude prompt caching for baseline cost comparison. -DISABLE_CACHE = False - -# Thread-local per-file token breakdown; read by calculateCost() for Claude. -_thread_local = threading.local() - -# Cross-thread running total of accurate cache-discounted cost (protected by lock). -_global_accurate_cost = 0.0 -_global_accurate_cost_lock = threading.Lock() - - -def _usage_to_debug_dict(usage): - """Extract token counts from provider usage objects for request debugging.""" - if not usage: - return {} - - usage_dict = {} - for field in ( - "prompt_tokens", - "completion_tokens", - "input_tokens", - "output_tokens", - "cache_read_input_tokens", - "cache_creation_input_tokens", - ): - value = getattr(usage, field, None) - if value is not None: - usage_dict[field] = value - - extra = getattr(usage, "model_extra", None) - if isinstance(extra, dict): - for field in ("cache_read_input_tokens", "cache_creation_input_tokens"): - value = extra.get(field) - if value is not None and field not in usage_dict: - usage_dict[field] = value - - return usage_dict - - -def _write_request_debug_log(provider, request_payload, usage): - """Write the exact SDK payload text and returned token usage.""" - if not DEBUG: - return - - try: - log_dir = Path("log") - log_dir.mkdir(parents=True, exist_ok=True) - usage_dict = _usage_to_debug_dict(usage) - payload_text = json.dumps(request_payload, indent=2, ensure_ascii=False, default=str) - usage_text = json.dumps(usage_dict, indent=2, ensure_ascii=False, default=str) - - with _debug_request_log_lock: - with open(log_dir / "request_debug.log", "a", encoding="utf-8") as debug_file: - debug_file.write("\n=== API Request ===\n") - debug_file.write(f"Timestamp: {time.strftime('%Y-%m-%d %H:%M:%S')}\n") - debug_file.write(f"Provider: {provider}\n") - debug_file.write("Usage:\n") - debug_file.write(f"{usage_text}\n") - debug_file.write("Payload:\n") - debug_file.write(f"{payload_text}\n") - debug_file.flush() - except Exception: - pass - -def _normalize_openai_base_url(url: str) -> str: - """Ensure OpenAI SDK global base_url has a trailing slash.""" - _url = (url or "").strip() - if _url and not _url.endswith("/"): - _url += "/" - return _url - - -def isClaudeModel(model): - """True when the model name looks like an Anthropic Claude model.""" - return bool(model) and any(x in model.lower() for x in ("claude", "sonnet", "haiku", "opus")) - - -def isClaudeNative(model): - """True when this model routes to the native Anthropic SDK. - - Mirrors the routing check in translateText: the model must look like Claude - AND the configured API URL must be unset or point at anthropic.com. Any - other custom URL (e.g. DeepSeek, OpenAI proxy) uses the OpenAI-compatible - path even for Claude-named models. - """ - live_api = os.getenv("api", "").strip() - return isClaudeModel(model) and (not live_api or "anthropic" in live_api.lower()) - - -def isMistralAPI(): - """True when requests go to the Mistral platform API (la Plateforme). - - Detection is URL-based: the adaptive rate limiter keys off x-ratelimit-* - headers that only api.mistral.ai sends. Mistral models served through other - OpenAI-compatible providers (Nvidia, OpenRouter, ...) use the generic path. - """ - live_api = os.getenv("api", "").strip().lower() - if "mistral.ai" in live_api: - return True - return not live_api and os.getenv("API_PROVIDER", "").strip().lower() == "mistral" - - -# Models that REJECT sampling params (temperature/top_p/top_k) with a 400 — -# Claude Opus 4.7 and up retired them in favour of adaptive thinking. Matches -# opus-4-7, opus-4-8, opus-4-10+ and fable, but NOT opus-4-6 or Sonnet/Haiku. -_NO_SAMPLING_RE = re.compile(r"opus-4-(?:[7-9]\b|[1-9]\d)|fable", re.I) - -# Tracks which distinct batch sizes have already been cache-written during this estimate run. -# Each unique numLines value maps to a distinct output_config schema → one write per size. -# Persisted to disk so sequential GUI subprocesses share state. -_estimate_written_sizes: set = set() -_ESTIMATE_SIZES_FILE = Path("log/estimate_written_sizes.json") - -def _load_estimate_written_sizes(): - """Load persisted written-sizes set from disk (for GUI subprocess sharing).""" - global _estimate_written_sizes - try: - if _ESTIMATE_SIZES_FILE.exists(): - with open(_ESTIMATE_SIZES_FILE, "r", encoding="utf-8") as f: - _estimate_written_sizes = set(json.load(f)) - except Exception: - _estimate_written_sizes = set() - -def _save_estimate_written_sizes(): - """Persist written-sizes set to disk.""" - try: - _ESTIMATE_SIZES_FILE.parent.mkdir(parents=True, exist_ok=True) - with open(_ESTIMATE_SIZES_FILE, "w", encoding="utf-8") as f: - json.dump(list(_estimate_written_sizes), f) - except Exception: - pass - -def clear_estimate_written_sizes(): - """Reset the written-sizes file at the start of a new estimate run.""" - global _estimate_written_sizes - _estimate_written_sizes = set() - try: - if _ESTIMATE_SIZES_FILE.exists(): - _ESTIMATE_SIZES_FILE.unlink() - except Exception: - pass - - -# ===== Placeholder Protection System ===== -# Patterns to protect from translation (sound effects, control codes, etc.) -PROTECTED_PATTERNS = [ - r'\\SE\[[^\]]+\]', # \SE[sound_effect_name] - r'\\ME\[[^\]]+\]', # \ME[music_effect_name] - r'\\BGM\[[^\]]+\]', # \BGM[background_music_name] - r'\\BGS\[[^\]]+\]', # \BGS[background_sound_name] - r'_pum\[[^\]]+\]', # _pum[name] - r'\\VS\[[^\]]+\]', # \VS[name] -] - -def protect_script_codes(text): - """ - Replace script codes (like \\SE[タイプライター]) with unique placeholders before translation. - Returns: (protected_text, replacements_dict) - """ - if not text or not isinstance(text, str): - return text, {} - - # Normalize curly/smart quotes to ASCII equivalents BEFORE building the JSON - # payload. When these characters appear inside a JSON string value the AI - # tends to treat them as regular ASCII double-quotes, which makes the value - # appear empty (e.g. `"スキルを"リセットする` → AI sees empty + stray text). - # This mirrors the identical normalization already applied to the AI's OUTPUT - # inside extractTranslation's translation_table. - quote_norm_table = str.maketrans({ - '\u201C': "'", # " left double quotation mark - '\u201D': "'", # " right double quotation mark - '\uFF02': "'", # ＂ fullwidth quotation mark - '\u2018': "'", # ' left single quotation mark - '\u2019': "'", # ' right single quotation mark - '\u201B': "'", # ‛ single high-reversed-9 quotation mark - '\u02BC': "'", # ʼ modifier letter apostrophe - '\uFF07': "'", # ＇ fullwidth apostrophe - }) - text = text.translate(quote_norm_table) - - # Convert half-width katakana (U+FF61–U+FF9F) to full-width katakana so the - # AI recognises them as Japanese text and translates them correctly. - # NFKC is applied only to matched half-width kana spans to avoid altering - # intentional fullwidth Latin/digit characters elsewhere in the string. - text = re.sub(r'[\uFF61-\uFF9F]+', lambda m: unicodedata.normalize('NFKC', m.group(0)), text) - - replacements = {} - protected_text = text - counter = 0 - - # Combine all patterns - combined_pattern = '|'.join(f'({pattern})' for pattern in PROTECTED_PATTERNS) - - def replace_match(match): - nonlocal counter - original = match.group(0) - # Create a unique placeholder that won't be translated - placeholder = f"__PROTECTED_{counter}__" - replacements[placeholder] = original - counter += 1 - return placeholder - - if combined_pattern: - protected_text = re.sub(combined_pattern, replace_match, protected_text) - - return protected_text, replacements - - -def restore_script_codes(text, replacements): - """ - Restore protected script codes from placeholders after translation. - """ - if not text or not replacements: - return text - - if isinstance(text, str): - result = text - for placeholder, original in replacements.items(): - result = result.replace(placeholder, original) - return result - elif isinstance(text, list): - return [restore_script_codes(item, replacements) for item in text] - else: - return text - - -def validate_placeholders(original_text, translated_text, replacements): - """ - Validate that all placeholders from the original text appear in the translation. - Returns: (is_valid, missing_placeholders, extra_placeholders) - """ - if not replacements: - return True, [], [] - - # Get all placeholders - all_placeholders = set(replacements.keys()) - - # Count placeholders in original - original_counts = {} - for placeholder in all_placeholders: - if isinstance(original_text, str): - original_counts[placeholder] = original_text.count(placeholder) - elif isinstance(original_text, list): - original_counts[placeholder] = sum(str(item).count(placeholder) for item in original_text) - - # Count placeholders in translation - translated_counts = {} - for placeholder in all_placeholders: - if isinstance(translated_text, str): - translated_counts[placeholder] = translated_text.count(placeholder) - elif isinstance(translated_text, list): - translated_counts[placeholder] = sum(str(item).count(placeholder) for item in translated_text) - - # Find mismatches - missing = [] - extra = [] - for placeholder in all_placeholders: - orig_count = original_counts.get(placeholder, 0) - trans_count = translated_counts.get(placeholder, 0) - - if trans_count < orig_count: - missing.append(f"{placeholder} (expected {orig_count}, found {trans_count})") - elif trans_count > orig_count: - extra.append(f"{placeholder} (expected {orig_count}, found {trans_count})") - - is_valid = len(missing) == 0 and len(extra) == 0 - return is_valid, missing, extra - - -def validate_translation_content(original_items, translated_items, langRegex): - """ - Validate that translated items are not empty or nearly empty. - Returns: (is_valid, invalid_indices, reasons) - - Rules: - 1. If original has content, translation must not be empty or just whitespace - 2. If original has Japanese text, translation must not be a single punctuation mark - 3. Translation should have meaningful content (more than 1-2 characters for substantial originals) - """ - if not isinstance(original_items, list): - original_items = [original_items] - translated_items = [translated_items] - - invalid_indices = [] - reasons = [] - - for i, (orig, trans) in enumerate(zip(original_items, translated_items)): - orig_str = str(orig).strip() - trans_str = str(trans).strip() - - # Skip if original is empty or placeholder - if not orig_str or orig_str == "Placeholder Text": - continue - - # Check if original has content that needs translation - has_source_text = bool(re.search(langRegex, orig_str)) - - if has_source_text: - # Original has Japanese text - translation must be substantial - - # Check 1: Translation is empty or just whitespace - if not trans_str: - invalid_indices.append(i) - reasons.append(f"Line{i+1}: Empty translation for '{orig_str[:50]}...'") - continue - - # Check 2: Translation is just a single punctuation mark or very short - # Allow control codes like \\C[27]\\V[45] but not just ":" or "" - # Use <= 1 so real 2-char words like "No", "Go", "Hi" are not rejected - if len(trans_str) <= 1 and not re.search(r'\\[A-Z]\[', trans_str): - # Exception: if original is also very short (like "回" -> "x"), that's ok - if len(orig_str) > 3: - invalid_indices.append(i) - reasons.append(f"Line{i+1}: Translation too short ('{trans_str}') for '{orig_str[:50]}...'") - continue - - # Check 3: For longer originals (>10 chars), translation should be more than just 1-2 chars - # unless it's a special case like numbers or codes - if len(orig_str) > 10 and len(trans_str) <= 2: - # Allow if it contains control codes or is just a replacement word - if not re.search(r'\\[A-Z]\[', trans_str) and not trans_str.isalnum(): - invalid_indices.append(i) - reasons.append(f"Line{i+1}: Translation suspiciously short ('{trans_str}') for '{orig_str[:50]}...'") - continue - - # Check 4: Runaway translation - translation is excessively long relative to original - # Catches cases where the model repeats words endlessly (e.g. "it hurts it hurts it hurts...") - ratio_limit = max(len(orig_str) * 8, 120) - if len(orig_str) > 10 and len(trans_str) > ratio_limit: - invalid_indices.append(i) - reasons.append(f"Line{i+1}: Runaway translation (output {len(trans_str)} chars vs input {len(orig_str)} chars) for '{orig_str[:50]}...'") - continue - # Absolute cap: garbage outputs that are not caught by ratio alone - if len(trans_str) > 4000 and len(trans_str) > len(orig_str) * 3: - invalid_indices.append(i) - reasons.append(f"Line{i+1}: Runaway translation (output {len(trans_str)} chars exceeds cap) for '{orig_str[:50]}...'") - continue - - # Check 5: Same character repeated many times (common API glitch / broken JSON tail) - if re.search(r"(.)\1{44,}", trans_str): - invalid_indices.append(i) - reasons.append(f"Line{i+1}: Excessive character repetition (possible model glitch) in translation") - continue - - is_valid = len(invalid_indices) == 0 - return is_valid, invalid_indices, reasons - -# Load .env, strip accidental whitespace, set base URL / org / API key. -# Gemini/Mistral use their endpoints only when no custom API URL is set. -load_dotenv() -api_provider = os.getenv("API_PROVIDER", "openai").lower() -env_api = os.getenv("api", "").strip() -if api_provider == "gemini" and not env_api: - # Use Google Generative Language compatibility endpoint only as fallback. - openai.base_url = "https://generativelanguage.googleapis.com/v1beta/openai/" - openai.organization = None -elif api_provider == "mistral" and not env_api: - openai.base_url = "https://api.mistral.ai/v1/" - openai.organization = None -else: - if env_api: - openai.base_url = _normalize_openai_base_url(env_api) - # Support both 'organization' (gui/.env.example) and legacy 'org' names - org = os.getenv("organization") or os.getenv("org") - if org: - openai.organization = org.strip() - -# Always set API key from 'key' env var (trim whitespace) -openai.api_key = os.getenv("key", "").strip() - -# Translation cache management -CACHE_FILE = Path("log/translation_cache.json") -CACHE_LOCK_FILE = Path("log/translation_cache.lock") -CACHE_LOCK = threading.RLock() -CACHE_PENDING_MARKER = "__translation_pending__" -CACHE_PENDING_TTL = 600 -CACHE_WAIT_INTERVAL = 0.25 -_cache = None - -@contextmanager -def _translation_cache_file_lock(): - """Cross-process lock for translation_cache.json.""" - CACHE_LOCK_FILE.parent.mkdir(parents=True, exist_ok=True) - with open(CACHE_LOCK_FILE, "a+b") as lock_file: - if os.name == "nt": - import msvcrt - lock_file.seek(0) - msvcrt.locking(lock_file.fileno(), msvcrt.LK_LOCK, 1) - try: - yield - finally: - lock_file.seek(0) - msvcrt.locking(lock_file.fileno(), msvcrt.LK_UNLCK, 1) - else: - import fcntl - fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX) - try: - yield - finally: - fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN) - -def _read_cache_from_disk(): - """Read the disk cache; return an empty dict if it is unavailable.""" - try: - if CACHE_FILE.exists(): - with open(CACHE_FILE, "r", encoding="utf-8") as f: - data = json.load(f) - return data if isinstance(data, dict) else {} - except Exception: - pass - return {} - -def _write_cache_to_disk(cache): - """Atomically write the cache using a process/thread-unique temp file.""" - CACHE_FILE.parent.mkdir(parents=True, exist_ok=True) - tmp_file = CACHE_FILE.with_name( - f"{CACHE_FILE.name}.{os.getpid()}.{threading.get_ident()}.tmp" - ) - with open(tmp_file, "w", encoding="utf-8") as f: - json.dump(cache, f, ensure_ascii=False, indent=2) - tmp_file.replace(CACHE_FILE) - -def _is_pending_cache_entry(value): - return isinstance(value, dict) and value.get(CACHE_PENDING_MARKER) is True - -def _is_stale_pending_cache_entry(value): - if not _is_pending_cache_entry(value): - return False - try: - return time.time() - float(value.get("time", 0)) > CACHE_PENDING_TTL - except Exception: - return True - -def _is_own_pending_cache_entry(value): - return ( - _is_pending_cache_entry(value) - and value.get("pid") == os.getpid() - and value.get("thread") == threading.get_ident() - ) - -def _pending_cache_entry(): - return { - CACHE_PENDING_MARKER: True, - "pid": os.getpid(), - "thread": threading.get_ident(), - "time": time.time(), - } - -def _merge_translation_caches(base, overlay): - """Merge cache dictionaries while never replacing a translation with pending.""" - merged = dict(base or {}) - for key, value in (overlay or {}).items(): - existing = merged.get(key) - if _is_pending_cache_entry(value) and existing is not None: - if not _is_pending_cache_entry(existing): - continue - if not _is_stale_pending_cache_entry(existing): - continue - merged[key] = value - return merged - -def clear_cache(): - """Clear the translation cache (called at start of each run)""" - global _cache - with CACHE_LOCK: - _cache = {} - with _translation_cache_file_lock(): - try: - if CACHE_FILE.exists(): - CACHE_FILE.unlink() - except Exception: - pass - -def load_cache(): - """Load the translation cache from disk.""" - global _cache - with CACHE_LOCK: - with _translation_cache_file_lock(): - disk_cache = _read_cache_from_disk() - if _cache: - disk_cache = _merge_translation_caches(disk_cache, _cache) - _cache = disk_cache - return _cache - -def save_cache(): - """Save the translation cache to disk, preserving entries from other workers.""" - global _cache - if _cache is None: - return - - with CACHE_LOCK: - try: - with _translation_cache_file_lock(): - disk_cache = _read_cache_from_disk() - disk_cache = _merge_translation_caches(disk_cache, _cache) - _cache = disk_cache - _write_cache_to_disk(_cache) - except Exception: - pass - -def get_cache_key(payload, language): - """Generate a cache key for a payload (can be single string or JSON batch)""" - # Use hash to keep keys short but unique - payload_str = str(payload) if payload is not None else "" - combined = f"{payload_str}|{language}" - return hashlib.md5(combined.encode("utf-8")).hexdigest() - -def get_cached_translation(payload, language): - """Get cached translation if it exists""" - global _cache - key = get_cache_key(payload, language) - while True: - with CACHE_LOCK: - with _translation_cache_file_lock(): - cache = _read_cache_from_disk() - if _cache: - cache = _merge_translation_caches(cache, _cache) - - entry = cache.get(key) - if ( - entry is None - or _is_stale_pending_cache_entry(entry) - or _is_own_pending_cache_entry(entry) - ): - cache[key] = _pending_cache_entry() - _cache = cache - _write_cache_to_disk(cache) - return None - - _cache = cache - if not _is_pending_cache_entry(entry): - return entry - - time.sleep(CACHE_WAIT_INTERVAL) - -def cache_translation(payload, translation, language): - """Cache a translation payload and its response""" - global _cache - key = get_cache_key(payload, language) - - with CACHE_LOCK: - with _translation_cache_file_lock(): - cache = _read_cache_from_disk() - if _cache: - cache = _merge_translation_caches(cache, _cache) - cache[key] = translation - _cache = cache - _write_cache_to_disk(cache) - - -# Variable translation map (code 122 <-> code 111 consistency) -VAR_MAP_FILE = Path("log/var_translation_map.json") -VAR_MAP_LOCK = threading.Lock() -_var_map = None - -def clear_var_map(): - """Clear the variable translation map (called at start of each run)""" - global _var_map - with VAR_MAP_LOCK: - _var_map = {} - try: - if VAR_MAP_FILE.exists(): - VAR_MAP_FILE.unlink() - except Exception: - pass - -def _load_var_map(): - """Load the variable translation map from disk (always re-reads to pick up - entries written by other subprocesses).""" - global _var_map - _var_map = {} - try: - if VAR_MAP_FILE.exists(): - with open(VAR_MAP_FILE, "r", encoding="utf-8") as f: - _var_map = json.load(f) - except Exception: - _var_map = {} - return _var_map - -def _save_var_map(): - """Save the variable translation map to disk. - Re-reads the file first and merges so entries from other subprocesses - are never lost.""" - global _var_map - if _var_map is None: - return - try: - VAR_MAP_FILE.parent.mkdir(parents=True, exist_ok=True) - # Re-read the on-disk version and merge our entries on top - disk_map = {} - try: - if VAR_MAP_FILE.exists(): - with open(VAR_MAP_FILE, "r", encoding="utf-8") as f: - disk_map = json.load(f) - except Exception: - disk_map = {} - disk_map.update(_var_map) - _var_map = disk_map - tmp_file = VAR_MAP_FILE.with_suffix(".tmp") - with open(tmp_file, "w", encoding="utf-8") as f: - json.dump(_var_map, f, ensure_ascii=False, indent=2) - tmp_file.replace(VAR_MAP_FILE) - except Exception: - pass - -def get_var_translation(original): - """Look up a cached variable translation. Returns the translation or None.""" - with VAR_MAP_LOCK: - m = _load_var_map() - return m.get(original) - -def set_var_translation(original, translated): - """Store a variable translation and persist to disk. - Skips if the translation is identical to the original (untranslated). - """ - if original == translated: - return - with VAR_MAP_LOCK: - m = _load_var_map() - m[original] = translated - _save_var_map() - -def set_var_translations_batch(pairs): - """Store multiple variable translations at once and persist to disk. - pairs: list of (original, translated) tuples - Skips pairs where the translation is identical to the original (untranslated). - """ - with VAR_MAP_LOCK: - m = _load_var_map() - for original, translated in pairs: - if original != translated: - m[original] = translated - _save_var_map() - - -# ===== Anthropic Message Batches (50% off all token usage) ===== -# Batch integration by Len — two-pass collect/consume flow; see README Credits. -# Batch translation is a two-pass flow driven by the batch phase (kept in the -# BATCH_PHASE env var so GUI subprocesses inherit it): -# collect: translateAI builds each cache-missed request (byte-identical to a -# live request, including the cached system block) and queues it -# instead of calling the API. Text is left untranslated. -# consume: translateAI feeds the fetched batch responses through the normal -# validation/restore path; anything missing or invalid falls back -# to the live API automatically. -# Between the passes, submit/poll/fetch the queue with runTranslationBatches(). -BATCH_QUEUE_FILE = Path("log/batch_requests.json") -BATCH_STATE_FILE = Path("log/batch_state.json") -BATCH_RESULTS_FILE = Path("log/batch_results.json") -BATCH_LOCK_FILE = Path("log/batch_files.lock") -BATCH_LOCK = threading.RLock() -# API limits are 100,000 requests / 256 MB per batch; stay safely under both. -BATCH_MAX_REQUESTS = 100_000 -BATCH_MAX_BYTES = 200 * 1024 * 1024 - -_batch_phase = None -_batch_results = None # in-memory copy of BATCH_RESULTS_FILE (read-only during consume) -_batch_queue_pending = {} # process-local queue entries not yet flushed to disk - - -def set_batch_phase(phase): - """Set the batch phase ('collect', 'consume' or None) for this process and - any subprocesses it spawns (the GUI runs one per file).""" - global _batch_phase, _batch_results - _batch_phase = phase if phase in ("collect", "consume") else None - if _batch_phase: - os.environ["BATCH_PHASE"] = _batch_phase - else: - os.environ.pop("BATCH_PHASE", None) - _batch_results = None # phase change invalidates the in-memory results copy - - -def get_batch_phase(): - """Current batch phase, or None when batch translation is off.""" - if _batch_phase: - return _batch_phase - phase = os.getenv("BATCH_PHASE", "").strip().lower() - return phase if phase in ("collect", "consume") else None - - -@contextmanager -def _batch_file_lock(): - """Cross-process lock for the batch queue/state/results files.""" - BATCH_LOCK_FILE.parent.mkdir(parents=True, exist_ok=True) - with open(BATCH_LOCK_FILE, "a+b") as lock_file: - if os.name == "nt": - import msvcrt - lock_file.seek(0) - msvcrt.locking(lock_file.fileno(), msvcrt.LK_LOCK, 1) - try: - yield - finally: - lock_file.seek(0) - msvcrt.locking(lock_file.fileno(), msvcrt.LK_UNLCK, 1) - else: - import fcntl - fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX) - try: - yield - finally: - fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN) - - -def _read_batch_file(path): - """Read a batch JSON file; return an empty dict if it is unavailable.""" - try: - if path.exists(): - with open(path, "r", encoding="utf-8") as f: - data = json.load(f) - return data if isinstance(data, dict) else {} - except Exception: - pass - return {} - - -def _write_batch_file(path, data): - """Atomically write a batch JSON file (no indent — queues can be large).""" - path.parent.mkdir(parents=True, exist_ok=True) - tmp_file = path.with_name(f"{path.name}.{os.getpid()}.{threading.get_ident()}.tmp") - with open(tmp_file, "w", encoding="utf-8") as f: - json.dump(data, f, ensure_ascii=False) - tmp_file.replace(path) - - -def peek_cached_translation(payload, language): - """Cache lookup that never blocks or writes a pending marker. - - The collect pass uses this instead of get_cached_translation so abandoned - pending markers can't stall the consume pass for CACHE_PENDING_TTL.""" - key = get_cache_key(payload, language) - with CACHE_LOCK: - with _translation_cache_file_lock(): - cache = _read_cache_from_disk() - if _cache: - cache = _merge_translation_caches(cache, _cache) - entry = cache.get(key) - if entry is None or _is_pending_cache_entry(entry): - return None - return entry - - -def queue_batch_request(payload, language, params): - """Queue one Batches API request during the collect pass. - - Deduped by the same key the translation cache uses, so identical payloads - across files are only paid for once. Entries are buffered in memory and - merged to disk by flush_batch_queue() at the end of each translateAI call. - """ - key = get_cache_key(payload, language) - with BATCH_LOCK: - _batch_queue_pending[key] = {"payload": payload, "language": language, "params": params} - return key - - -def flush_batch_queue(): - """Merge this process's pending queue entries into the on-disk queue.""" - global _batch_queue_pending - with BATCH_LOCK: - if not _batch_queue_pending: - return - pending, _batch_queue_pending = _batch_queue_pending, {} - try: - with _batch_file_lock(): - queue = _read_batch_file(BATCH_QUEUE_FILE) - for key, entry in pending.items(): - queue.setdefault(key, entry) - _write_batch_file(BATCH_QUEUE_FILE, queue) - except Exception: - _batch_queue_pending.update(pending) # keep entries for the next flush - - -def take_batch_result(payload, language): - """Return the fetched batch response dict for a payload, or None. - - The results file is loaded once per process — it is written before the - consume pass starts and never changes mid-consume.""" - global _batch_results - if _batch_results is None: - with BATCH_LOCK: - if _batch_results is None: - with _batch_file_lock(): - _batch_results = _read_batch_file(BATCH_RESULTS_FILE) - return _batch_results.get(get_cache_key(payload, language)) - - -def pendingBatchRequests(): - """Number of queued batch requests (call after the collect pass).""" - flush_batch_queue() - with _batch_file_lock(): - return len(_read_batch_file(BATCH_QUEUE_FILE)) - - -def batchRunState(): - """'submitted' when a batch is still in flight, 'fetched' when results are - waiting to be consumed, else None. Lets an interrupted batch run resume - instead of re-collecting and paying for a second submission.""" - with _batch_file_lock(): - if _read_batch_file(BATCH_STATE_FILE).get("batches"): - return "submitted" - if _read_batch_file(BATCH_RESULTS_FILE): - return "fetched" - return None - - -def clearBatchFiles(): - """Remove queue/state/results left over from any previous batch run.""" - global _batch_results, _batch_queue_pending - with BATCH_LOCK: - _batch_results = None - _batch_queue_pending = {} - with _batch_file_lock(): - for path in (BATCH_QUEUE_FILE, BATCH_STATE_FILE, BATCH_RESULTS_FILE): - try: - if path.exists(): - path.unlink() - except Exception: - pass - - -def _get_anthropic_client(): - key = os.getenv("key", "").strip() - if not key: - raise Exception("Batch translation requires the 'key' env var (see .env).") - return anthropic.Anthropic(api_key=key) - - -def estimateBatchCost(model=None): - """Print a cost estimate for the queued batch requests and return it. - - Cached-prefix accounting mirrors what Anthropic bills: each distinct cached - prefix is written once (2x input rate at the 1h TTL) and read by every other - request that shares it (0.10x); everything is then halved by the batch - discount. Cache hits inside a batch are best-effort, so the no-cache batch - figure is the worst-case bound. - """ - flush_batch_queue() - with _batch_file_lock(): - queue = _read_batch_file(BATCH_QUEUE_FILE) - if not queue: - print("[BATCH] No batch requests queued.", flush=True) - return None - - enc = tiktoken.encoding_for_model("gpt-4") - prefix_count = {} # cached prefix text -> how many requests reuse it - prefix_tokens = {} # cached prefix text -> token count - dynamic_tokens = 0 - output_tokens = 0 - models = set() - for entry in queue.values(): - params = entry.get("params", {}) - if params.get("model"): - models.add(params["model"]) - blocks = params.get("system") or [] - cut = 0 # split system blocks at the cache breakpoint (0 = nothing cached) - for i, b in enumerate(blocks): - if "cache_control" in b: - cut = i + 1 - break - prefix = "".join(b.get("text", "") for b in blocks[:cut]) - dyn = "".join(b.get("text", "") for b in blocks[cut:]) - if prefix: - prefix_count[prefix] = prefix_count.get(prefix, 0) + 1 - if prefix not in prefix_tokens: - prefix_tokens[prefix] = len(enc.encode(prefix)) - msg_text = "\n".join(str(m.get("content", "")) for m in params.get("messages", [])) - dynamic_tokens += len(enc.encode(dyn)) + len(enc.encode(msg_text)) + 8 - # Output heuristic mirrors countTokens(): payload tokens x 2.5 covers - # the echoed JSON scaffold plus EN expansion. - output_tokens += round(len(enc.encode(str(entry.get("payload", "")))) * 2.5) - - est_model = model or next(iter(models), None) or os.getenv("model", "") - # Use Anthropic's count_tokens for the exact cached-prefix size when possible. - if prefix_tokens: - try: - client = _get_anthropic_client() - for prefix in list(prefix_tokens.keys()): - resp = client.messages.count_tokens( - model=est_model, - system=[{"type": "text", "text": prefix}], - messages=[{"role": "user", "content": "x"}], - ) - prefix_tokens[prefix] = resp.input_tokens - except Exception: - pass # tiktoken estimate already in place - - cache_write_tok = sum(prefix_tokens.values()) - cache_read_tok = sum(prefix_tokens[p] * (prefix_count[p] - 1) for p in prefix_count) - raw_input_tok = sum(prefix_tokens[p] * prefix_count[p] for p in prefix_count) + dynamic_tokens - - pricing = getPricingConfig(est_model) - in_rate = pricing["inputAPICost"] / 1_000_000 - out_rate = pricing["outputAPICost"] / 1_000_000 - - # Batch = 50% off every token, including cache writes (2x, 1h TTL) and reads (0.10x). - batch_cached = (cache_write_tok * in_rate * 2.00 - + cache_read_tok * in_rate * 0.10 - + dynamic_tokens * in_rate - + output_tokens * out_rate) * 0.50 - batch_nocache = (raw_input_tok * in_rate + output_tokens * out_rate) * 0.50 - live_cost = raw_input_tok * in_rate + output_tokens * out_rate - - n_reread = sum(prefix_count.values()) - len(prefix_count) - print(f"[BATCH] {len(queue)} requests queued for {est_model}", flush=True) - print(f"[BATCH] cached prefix: {cache_write_tok:,} tokens (written once, re-read by {n_reread:,} requests)", flush=True) - print(f"[BATCH] dynamic input: {dynamic_tokens:,} tokens | estimated output: {output_tokens:,} tokens", flush=True) - print(f"[BATCH] estimated cost: ${batch_cached:.2f} (batch + prompt cache)", flush=True) - print(f"[BATCH] ${batch_nocache:.2f} (batch, worst case no cache hits)", flush=True) - print(f"[BATCH] ${live_cost:.2f} (live API, no batch discount)", flush=True) - return { - "requests": len(queue), - "model": est_model, - "cache_write_tokens": cache_write_tok, - "cache_read_tokens": cache_read_tok, - "dynamic_tokens": dynamic_tokens, - "output_tokens": output_tokens, - "batch_cached_cost": batch_cached, - "batch_nocache_cost": batch_nocache, - "live_cost": live_cost, - } - - -def submitTranslationBatches(): - """Submit the queued requests to the Anthropic Message Batches API. - - Splits at the API limits and saves the custom_id -> cache-key mapping so - fetchTranslationBatches can route results back. Returns the batch ids.""" - flush_batch_queue() - with _batch_file_lock(): - queue = _read_batch_file(BATCH_QUEUE_FILE) - if not queue: - print("[BATCH] No batch requests queued.", flush=True) - return [] - - client = _get_anthropic_client() - batches = [] - requests, id_map, size = [], {}, 0 - - def _submit(): - nonlocal requests, id_map, size - if not requests: - return - batch = client.messages.batches.create(requests=requests) - batches.append({"id": batch.id, "custom_ids": id_map}) - print(f"[BATCH] submitted {batch.id} ({len(requests)} requests)", flush=True) - requests, id_map, size = [], {}, 0 - - for i, (key, entry) in enumerate(queue.items()): - custom_id = f"req-{i:06d}" - params = entry["params"] - requests.append({"custom_id": custom_id, "params": params}) - id_map[custom_id] = key - size += len(json.dumps(params, ensure_ascii=False).encode("utf-8")) - if len(requests) >= BATCH_MAX_REQUESTS or size >= BATCH_MAX_BYTES: - _submit() - _submit() - - with BATCH_LOCK: - with _batch_file_lock(): - _write_batch_file(BATCH_STATE_FILE, {"batches": batches}) - return [b["id"] for b in batches] - - -def checkTranslationBatches(): - """Print the processing status of submitted batches. True when all ended.""" - with _batch_file_lock(): - state = _read_batch_file(BATCH_STATE_FILE) - if not state.get("batches"): - print("[BATCH] No submitted batches — submit the queue first.", flush=True) - return False - client = _get_anthropic_client() - all_ended = True - for info in state["batches"]: - b = client.messages.batches.retrieve(info["id"]) - counts = getattr(b, "request_counts", None) - suffix = f" counts: {counts}" if counts else "" - print(f"[BATCH] {time.strftime('%H:%M:%S')} {b.id}: {b.processing_status}{suffix}", flush=True) - if b.processing_status != "ended": - all_ended = False - return all_ended - - -def fetchTranslationBatches(): - """Download finished batch results into the local results store. - - Successes are stored keyed by the payload cache key for the consume pass; - errored/expired requests are reported and simply fall back to the live API - during consume. Returns (succeeded, errored) counts.""" - global _batch_results - with _batch_file_lock(): - state = _read_batch_file(BATCH_STATE_FILE) - if not state.get("batches"): - print("[BATCH] No submitted batches — nothing to fetch.", flush=True) - return 0, 0 - client = _get_anthropic_client() - results, errored = {}, [] - for info in state["batches"]: - id_map = info.get("custom_ids", {}) - for r in client.messages.batches.results(info["id"]): - key = id_map.get(r.custom_id) - if key is None: - continue - res = r.result - if res.type != "succeeded": - detail = res.type - err = getattr(res, "error", None) - if err is not None: - inner = getattr(err, "error", err) - detail = f"{res.type} | {getattr(inner, 'type', '')}: {str(getattr(inner, 'message', '') or err)[:200]}" - errored.append((r.custom_id, detail)) - continue - msg = res.message - text = "".join(getattr(b, "text", "") or "" for b in msg.content) - u = msg.usage - cr = getattr(u, "cache_read_input_tokens", 0) or 0 - cw = getattr(u, "cache_creation_input_tokens", 0) or 0 - inp = getattr(u, "input_tokens", 0) or 0 - out = getattr(u, "output_tokens", 0) or 0 - results[key] = { - "text": text, - # prompt_tokens matches _AnthropicCompat: total incl. cache fields. - "prompt_tokens": inp + cr + cw, - "completion_tokens": out, - "cache_read_input_tokens": cr, - "cache_creation_input_tokens": cw, - } - with BATCH_LOCK: - with _batch_file_lock(): - merged = _read_batch_file(BATCH_RESULTS_FILE) - merged.update(results) - _write_batch_file(BATCH_RESULTS_FILE, merged) - # Queue and state are consumed; only the results store remains. - for path in (BATCH_QUEUE_FILE, BATCH_STATE_FILE): - try: - if path.exists(): - path.unlink() - except Exception: - pass - _batch_results = None - print(f"[BATCH] fetched {len(results)} results ({len(errored)} errored).", flush=True) - for cid, why in errored[:20]: - print(f"[BATCH] ! {cid}: {why}", flush=True) - if len(errored) > 20: - print(f"[BATCH] ... ({len(errored) - 20} more)", flush=True) - return len(results), len(errored) - - -def runTranslationBatches(poll=60): - """Submit the queue (unless already submitted), poll to completion, fetch.""" - with _batch_file_lock(): - state = _read_batch_file(BATCH_STATE_FILE) - if not state.get("batches"): - if not submitTranslationBatches(): - return 0, 0 - print(f"[BATCH] polling every {poll}s (Ctrl-C is safe — resume later with fetchTranslationBatches)...", flush=True) - while not checkTranslationBatches(): - time.sleep(poll) - return fetchTranslationBatches() - - -# ===== Mistral (la Plateforme) adaptive rate limiting ===== -# Mistral enforces independent request and token limits, both PER-MODEL. -# Verified live against api.mistral.ai (the response headers are authoritative): -# * x-ratelimit-limit-req-minute — requests/minute. PER-MODEL and varies a -# lot: mistral-medium=25, mistral-small=50, codestral=125, ministral-3b=750. -# Mistral's dashboard shows this /60 as "RPS" (25/min = 0.42 "RPS"), so the -# effective rate spans well below AND above 1 req/sec. There is NO -# per-second header; we divide req-minute by 60 to get the pacing interval. -# * x-ratelimit-limit-tokens-minute — input+output throughput, a true minute -# window, also per-model (e.g. 50k–1.3M). -# * Tokens per month — overall cap (not paced here; surfaces as a 429). -# The limiter paces requests >= 1/RPS apart (so a burst of file threads can't -# overrun the per-minute request budget) AND charges them against a -# minute-windowed token budget. It starts from a conservative seed, then the -# live headers on the first response pin req_per_sec / tok_per_min to the exact -# per-model values. One limiter is shared across all file threads (this tool -# runs one model per session). Override the seeds with -# mistralReqPerSec / mistralTokPerMin / mistralTokenHeadroom. -_mistral_limiter = None -_mistral_limiter_lock = threading.Lock() - - -def _estimate_mistral_tokens(text): - # JP ~1 token/char with Mistral tokenizers; EN ~1 per 3.5 chars. Be pessimistic. - return int(len(str(text)) * 1.1) + 8 - - -def _header_int(headers, *names): - """First present header among names parsed as int, else None.""" - for n in names: - v = headers.get(n) - if v is not None and str(v).strip() != "": - try: - return int(float(v)) - except (TypeError, ValueError): - pass - return None - - -def _header_float(headers, *names): - """First present header among names parsed as float, else None. - - Used for the RPS limit header, which per-model is often FRACTIONAL - (e.g. 0.08, 0.42, 0.83) — parsing it as int would floor those to 0.""" - for n in names: - v = headers.get(n) - if v is not None and str(v).strip() != "": - try: - return float(v) - except (TypeError, ValueError): - pass - return None - - -class AdaptiveLimiter: - """Paces requests off live ratelimit headers. - - Two dimensions, enforced together by acquire(): - * requests: spaced >= min_interval (1/RPS) apart — Mistral's request cap - is per-second, so this prevents bursts from tripping it. - * tokens: a minute-windowed budget (input+output), with a headroom margin. - """ - - def __init__(self, req_per_sec, tok_per_min, headroom): - self.lock = threading.Lock() - self.req_per_sec = max(0.001, float(req_per_sec)) - self.tok_per_min = tok_per_min - self.headroom = headroom # margin left under the TPM cap - self.min_interval = 1.0 / self.req_per_sec - self.next_request_at = time.monotonic() - self.tokens_remaining = tok_per_min - self.window_reset = time.monotonic() + 60 - - def acquire(self, est_tokens): - """Block until the request clears both the RPS pace and the TPM budget.""" - while True: - with self.lock: - now = time.monotonic() - if now >= self.window_reset: - self.tokens_remaining = self.tok_per_min - self.window_reset = now + 60 - # token gate - if self.tokens_remaining - est_tokens <= self.headroom: - sleep_for = max(0.05, self.window_reset - now) - # request-pace gate - elif now < self.next_request_at: - sleep_for = self.next_request_at - now - else: - # both gates clear — reserve this slot - self.next_request_at = max(now, self.next_request_at) + self.min_interval - self.tokens_remaining -= est_tokens - return - time.sleep(min(sleep_for, 5)) - - def update(self, headers): - """Sync budgets from the live ratelimit headers (authoritative). - - Accepts both the per-second request headers and (for forward/back compat) - the older per-minute request header names; the token headers are minute.""" - if not headers: - return - with self.lock: - # The authoritative request limit is x-ratelimit-limit-req-MINUTE - # (verified live: mistral-medium=25, ministral-3b=750, codestral=125; - # Mistral's dashboard "RPS" is just this number / 60). There is no - # per-second header. Convert to RPS for the pacing interval. A - # per-second header is still accepted first in case Mistral adds one. - limit_rps = _header_float(headers, "x-ratelimit-limit-req-second", - "x-ratelimit-limit-requests-second") - if limit_rps is None: - limit_rpm = _header_float(headers, "x-ratelimit-limit-req-minute", - "x-ratelimit-limit-requests-minute") - if limit_rpm is not None: - limit_rps = limit_rpm / 60.0 - if limit_rps and limit_rps > 0: - self.req_per_sec = limit_rps - self.min_interval = 1.0 / self.req_per_sec - limit_tok = _header_int(headers, "x-ratelimit-limit-tokens-minute", - "x-ratelimit-limit-tokens") - if limit_tok and limit_tok > 0: - self.tok_per_min = limit_tok - rem_tok = _header_int(headers, "x-ratelimit-remaining-tokens-minute", - "x-ratelimit-remaining-tokens") - if rem_tok is not None: - self.tokens_remaining = rem_tok - # Remaining requests this minute -> also cap the pace so a fresh - # limiter that joins mid-window doesn't burst the leftover budget. - rem_req = _header_int(headers, "x-ratelimit-remaining-req-minute", - "x-ratelimit-remaining-requests-minute") - if rem_req is not None and rem_req <= 0: - # budget already spent this minute — hold off ~ a minute - self.next_request_at = max(self.next_request_at, time.monotonic() + 60.0) - - -def _get_mistral_limiter(): - global _mistral_limiter - with _mistral_limiter_lock: - if _mistral_limiter is None: - # mistralReqPerMin kept as a deprecated alias: a per-minute number is - # converted to RPS so old .env files keep pacing sanely. - rps_env = os.getenv("mistralReqPerSec") - if rps_env: - req_per_sec = float(rps_env) - elif os.getenv("mistralReqPerMin"): - req_per_sec = max(0.05, float(os.getenv("mistralReqPerMin")) / 60.0) - else: - # Per-model RPS varies and is often well under 1 (free-tier - # mistral-medium ~0.83, magistral ~0.08). Start conservative so - # the FIRST request doesn't over-burst; the live header from that - # first response then corrects req_per_sec to the model's exact - # value (which is why few large batches > many tiny ones). - req_per_sec = 0.5 - _mistral_limiter = AdaptiveLimiter( - req_per_sec, - int(os.getenv("mistralTokPerMin", "50000") or 50000), - int(os.getenv("mistralTokenHeadroom", "4000") or 4000), - ) - return _mistral_limiter - - -def callMistral(params, retries=6): - """Call the Mistral chat completions endpoint with adaptive pacing. - - Acquires from the shared limiter before each attempt, syncs budgets from - the live x-ratelimit headers after each response, honours Retry-After on - 429, backs off with jitter on 5xx/network errors, and downgrades - json_schema -> json_object for models without structured-output support. - """ - limiter = _get_mistral_limiter() - est = sum(_estimate_mistral_tokens(m.get("content", "")) for m in params.get("messages", [])) - est += params.get("max_tokens", 0) // 2 - last_error = None - for attempt in range(retries + 1): - limiter.acquire(est) - try: - raw = openai.chat.completions.with_raw_response.create(**params) - response = raw.parse() - limiter.update(raw.headers) - return response - except RateLimitError as e: - last_error = e - resp = getattr(e, "response", None) - limiter.update(getattr(resp, "headers", None)) - retry_after = None - try: - retry_after = float(resp.headers.get("retry-after")) - except (AttributeError, TypeError, ValueError): - pass - time.sleep(retry_after if retry_after is not None else min(60, 2 ** attempt + random.random() * 2)) - except APIStatusError as e: - last_error = e - # Mistral rejects unsupported params with 400/422 — downgrade the - # structured-output format once and retry immediately. - if e.status_code in (400, 422) and (params.get("response_format") or {}).get("type") == "json_schema": - params = dict(params) - params["response_format"] = {"type": "json_object"} - continue - if e.status_code >= 500 and attempt < retries: - time.sleep(min(45, 2 ** attempt + random.random())) - continue - raise Exception(f"Mistral API error ({e.status_code}): {e}") - except APIConnectionError as e: - last_error = e - if attempt < retries: - time.sleep(min(45, 2 ** attempt + random.random())) - continue - raise Exception(f"Mistral connection error: {e}") - raise Exception(f"Mistral API failed after {retries + 1} attempts: {last_error}") - - -class TranslationConfig: - """Configuration class to hold all translation settings""" - - def __init__(self, - model=None, - language=None, - prompt=None, - vocab=None, - langRegex=None, - batchSize=None, - maxHistory=10, - estimateMode=False, - logFilePath="log/translationHistory.txt", - mismatchLogPath="log/mismatchHistory.txt"): - - # Load from environment if not provided - self.model = model or os.getenv("model") - self.language = (language or os.getenv("language", "english")).capitalize() - - # Load prompt and vocab files if not provided - if prompt is None: - try: - self.prompt = Path("prompt.txt").read_text(encoding="utf-8") - except FileNotFoundError: - self.prompt = "" - else: - self.prompt = prompt - - if vocab is None: - try: - self.vocab = Path("vocab.txt").read_text(encoding="utf-8") - except FileNotFoundError: - self.vocab = "" - else: - self.vocab = vocab - - # Set language regex (default is Japanese) - self.langRegex = langRegex or r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+" - - # Set batch size — derive from pricing config unless explicitly supplied - if batchSize is None: - self.batchSize = getPricingConfig(self.model)["batchSize"] - else: - self.batchSize = batchSize - - self.maxHistory = maxHistory - self.estimateMode = estimateMode - self.logFilePath = logFilePath - self.mismatchLogPath = mismatchLogPath - - -_LITELLM_PRICING_URL = ( - "https://raw.githubusercontent.com/BerriAI/litellm/main" - "/model_prices_and_context_window.json" -) -_PRICING_CACHE_FILE = Path("log/litellm_pricing.json") -_PRICING_CACHE_TTL = 86_400 # 24 hours -_pricing_db: dict | None = None -_pricing_db_fetched_at: float = 0.0 -_pricing_db_lock = threading.Lock() -_pricing_fetch_warned: bool = False # print fetch-failure warning at most once per session - - -def _load_litellm_pricing() -> dict | None: - """Return the LiteLLM pricing DB, using a 24-hour disk cache.""" - global _pricing_db, _pricing_db_fetched_at, _pricing_fetch_warned - - with _pricing_db_lock: - now = time.time() - - # In-memory cache still fresh - if _pricing_db is not None and (now - _pricing_db_fetched_at) < _PRICING_CACHE_TTL: - return _pricing_db - - # Try disk cache - if _PRICING_CACHE_FILE.exists(): - try: - disk = json.loads(_PRICING_CACHE_FILE.read_text(encoding="utf-8")) - if (now - disk.get("fetched_at", 0)) < _PRICING_CACHE_TTL: - _pricing_db = disk["prices"] - _pricing_db_fetched_at = disk["fetched_at"] - return _pricing_db - except Exception: - pass - - # Fetch from GitHub - try: - with urllib.request.urlopen(_LITELLM_PRICING_URL, timeout=5) as resp: - data = json.loads(resp.read().decode("utf-8")) - _pricing_db = data - _pricing_db_fetched_at = now - _pricing_fetch_warned = False # reset if a later fetch succeeds - try: - _PRICING_CACHE_FILE.parent.mkdir(parents=True, exist_ok=True) - _PRICING_CACHE_FILE.write_text( - json.dumps({"fetched_at": now, "prices": data}), - encoding="utf-8", - ) - except Exception: - pass - return _pricing_db - except Exception as fetch_err: - # No internet / GitHub unreachable — warn once, then fall back - if not _pricing_fetch_warned: - _pricing_fetch_warned = True - print( - f"[PRICING] Warning: Could not fetch live model pricing " - f"({fetch_err}). Cost estimates may be inaccurate — " - f"using built-in fallback prices.", - flush=True, - ) - # Use stale disk cache if available - if _pricing_db is not None: - return _pricing_db - try: - disk = json.loads(_PRICING_CACHE_FILE.read_text(encoding="utf-8")) - _pricing_db = disk["prices"] - return _pricing_db - except Exception: - return None - - -def _lookup_model_price(model: str): - """Look up (input_per_1M, output_per_1M) from the LiteLLM pricing DB. - - Returns a (float, float) tuple or None if not found. - Matching priority: - 1. Exact key match - 2. Exact match on the model portion after a provider prefix (e.g. "deepseek/deepseek-chat") - 3. The user's model name is a prefix of a DB key (handles dated suffixes like -20241022) - 4. A DB key model-part is a prefix of the user's model name - """ - db = _load_litellm_pricing() - if not db: - return None - - model_lower = model.lower() - - def _extract(entry): - inp = entry.get("input_cost_per_token") - out = entry.get("output_cost_per_token") - if inp is not None and out is not None: - return round(inp * 1_000_000, 6), round(out * 1_000_000, 6) - return None - - # Pass 1: exact key - if model_lower in db: - result = _extract(db[model_lower]) - if result: - return result - - # Build a lookup of (stripped_key → original_key) for passes 2-4 - stripped: list[tuple[str, str]] = [] - for key in db: - stripped.append((key.split("/")[-1].lower(), key)) - - # Pass 2: exact match on stripped key - for skey, orig in stripped: - if skey == model_lower: - result = _extract(db[orig]) - if result: - return result - - # Pass 3: model name is a prefix of the DB key (e.g. "claude-3-5-sonnet" matches - # "claude-3-5-sonnet-20241022") - candidates = [(skey, orig) for skey, orig in stripped if skey.startswith(model_lower)] - if candidates: - # Prefer the shortest (most generic) key - skey, orig = min(candidates, key=lambda x: len(x[0])) - result = _extract(db[orig]) - if result: - return result - - # Pass 4: DB key is a prefix of the model name (e.g. "gemini-2.0-flash" matches - # "gemini-2.0-flash-exp") - candidates = [(skey, orig) for skey, orig in stripped if model_lower.startswith(skey)] - if candidates: - skey, orig = max(candidates, key=lambda x: len(x[0])) # longest = most specific - result = _extract(db[orig]) - if result: - return result - - return None - - -def getPricingConfig(model): - """ - Get pricing configuration for a given model. - - Args: - model: The model name string - - Returns: - dict: Dictionary containing inputAPICost, outputAPICost, batchSize, and frequencyPenalty - """ - # Try to resolve pricing from the LiteLLM community pricing DB first. - # This keeps costs accurate as providers update their prices without requiring - # a code change. Falls back to the hardcoded table below on failure. - live_price = _lookup_model_price(model) - if live_price: - inp, out = live_price - # Preserve model-specific batch / penalty overrides from the hardcoded table - # by still running through the if-chain but replacing the cost fields. - _live_override = {"inputAPICost": inp, "outputAPICost": out} - else: - _live_override = None - - # Hardcoded fallback table — used for batchSize / frequencyPenalty tuning and - # as a cost fallback when the LiteLLM DB is unavailable. - # Batch Size: GPT-3.5 struggles past 15 lines; GPT-4 struggles past 50. - # If you get a MISMATCH LENGTH error, lower the batch size. - if "gpt-3.5" in model: - cfg = {"inputAPICost": 3.00, "outputAPICost": 5.00, "batchSize": 10, "frequencyPenalty": 0.2} - elif "gpt-4.1-mini" in model: - cfg = {"inputAPICost": 0.40, "outputAPICost": 1.60, "batchSize": 30, "frequencyPenalty": 0.05} - elif "gpt-4.1" in model: - cfg = {"inputAPICost": 2.00, "outputAPICost": 8.00, "batchSize": 30, "frequencyPenalty": 0.05} - elif "gpt-5" in model: - cfg = {"inputAPICost": 1.25, "outputAPICost": 10.00, "batchSize": 30, "frequencyPenalty": 0.05} - elif "deepseek" in model: - cfg = {"inputAPICost": 0.27, "outputAPICost": 1.10, "batchSize": 30, "frequencyPenalty": 0.05} - # Mistral — system prompt is resent per request (no prompt caching), so - # throughput/cost favor larger batches. Live LiteLLM pricing overrides these. - elif "mistral-large" in model or "pixtral-large" in model: - cfg = {"inputAPICost": 2.00, "outputAPICost": 6.00, "batchSize": 40, "frequencyPenalty": 0.0} - elif "magistral-medium" in model: - cfg = {"inputAPICost": 2.00, "outputAPICost": 5.00, "batchSize": 40, "frequencyPenalty": 0.0} - elif "mistral-medium-3.5" in model or "mistral-medium-3-5" in model or "mistral-medium-26" in model: - # Medium 3.5 (v26.04) — also matches dated 26xx ids - cfg = {"inputAPICost": 1.50, "outputAPICost": 7.50, "batchSize": 40, "frequencyPenalty": 0.0} - elif "mistral-medium" in model: - # Medium 3 / 3.1 (what -latest still points at) - cfg = {"inputAPICost": 0.40, "outputAPICost": 2.00, "batchSize": 40, "frequencyPenalty": 0.0} - elif "magistral-small" in model: - cfg = {"inputAPICost": 0.50, "outputAPICost": 1.50, "batchSize": 40, "frequencyPenalty": 0.0} - elif "mistral-small" in model: - cfg = {"inputAPICost": 0.10, "outputAPICost": 0.30, "batchSize": 40, "frequencyPenalty": 0.0} - elif "ministral" in model or "open-mistral" in model or "mistral-nemo" in model: - cfg = {"inputAPICost": 0.10, "outputAPICost": 0.10, "batchSize": 40, "frequencyPenalty": 0.0} - elif "codestral" in model: - cfg = {"inputAPICost": 0.30, "outputAPICost": 0.90, "batchSize": 40, "frequencyPenalty": 0.0} - elif "mistral" in model or "pixtral" in model: - cfg = {"inputAPICost": 2.00, "outputAPICost": 6.00, "batchSize": 40, "frequencyPenalty": 0.0} - elif "claude-opus-4-5" in model or "claude-opus-4-6" in model: - cfg = {"inputAPICost": 5.00, "outputAPICost": 25.00, "batchSize": 30, "frequencyPenalty": 0.05} - elif "claude-opus" in model or model == "claude-3-opus": - # Opus 4, 4.1, 3 — $15/$75 - cfg = {"inputAPICost": 15.00, "outputAPICost": 75.00, "batchSize": 30, "frequencyPenalty": 0.05} - elif "claude-haiku-4-5" in model or "claude-haiku-4-6" in model: - cfg = {"inputAPICost": 1.00, "outputAPICost": 5.00, "batchSize": 30, "frequencyPenalty": 0.05} - elif "claude-haiku-3-5" in model: - cfg = {"inputAPICost": 0.80, "outputAPICost": 4.00, "batchSize": 30, "frequencyPenalty": 0.05} - elif "claude-3-haiku" in model: - cfg = {"inputAPICost": 0.25, "outputAPICost": 1.25, "batchSize": 30, "frequencyPenalty": 0.05} - elif "haiku" in model: - # Unknown haiku version — use current flagship pricing as best guess - cfg = {"inputAPICost": 1.00, "outputAPICost": 5.00, "batchSize": 30, "frequencyPenalty": 0.05} - elif "sonnet" in model or "claude" in model: - cfg = {"inputAPICost": 3.00, "outputAPICost": 15.00, "batchSize": 30, "frequencyPenalty": 0.05} - elif "gemini-2.0-flash-lite" in model: - cfg = {"inputAPICost": 0.075, "outputAPICost": 0.30, "batchSize": 30, "frequencyPenalty": 0.0} - elif "gemini-2.0-flash" in model: - cfg = {"inputAPICost": 0.10, "outputAPICost": 0.40, "batchSize": 30, "frequencyPenalty": 0.0} - elif "gemini-2.5-flash-lite" in model: - cfg = {"inputAPICost": 0.10, "outputAPICost": 0.40, "batchSize": 30, "frequencyPenalty": 0.0} - elif "gemini-2.5-flash" in model: - cfg = {"inputAPICost": 0.30, "outputAPICost": 2.50, "batchSize": 30, "frequencyPenalty": 0.0} - elif "gemini-2.5-pro" in model: - cfg = {"inputAPICost": 1.25, "outputAPICost": 10.00, "batchSize": 30, "frequencyPenalty": 0.0} - else: - cfg = { - "inputAPICost": float(os.getenv("input_cost", 3.00)), - "outputAPICost": float(os.getenv("output_cost", 6.00)), - "batchSize": int(os.getenv("batchsize", 10)), - "frequencyPenalty": float(os.getenv("frequency_penalty", 0.2)), - } - - # Apply live pricing from LiteLLM if available — keeps costs up-to-date - # without requiring code changes when providers reprice their models. - if _live_override: - cfg.update(_live_override) - - return cfg - - -def batchList(inputList, batchSize): - """Split a list into batches of specified size""" - if not isinstance(batchSize, int) or batchSize <= 0: - raise ValueError("batchSize must be a positive integer") - - return [inputList[i : i + batchSize] for i in range(0, len(inputList), batchSize)] - - -def parseVocabWithCategories(vocabText): - """Parse vocabulary text and extract terms with their categories.""" - pairs = [] - seen = set() - currentCategory = None - - for line in vocabText.splitlines(): - line = line.strip() - if not line or line.startswith('```') or line.startswith('Here are some vocabulary'): - continue - - # Check if this is a category header - if line.startswith('#'): - currentCategory = line - continue - - # Parse vocabulary term - extract both Japanese and English parts. - # Rich entries may continue after the first parenthesized translation, - # e.g. "サンク (Sank) - Male; protagonist..."; only "Sank" is the match key. - paren_match = re.match(r'^(.+?)\s*\(([^()]*)\)', line) - dash_match = re.match(r'^(.+?)\s+[–-]\s+(.+)$', line) - if paren_match: - japanese_term = paren_match.group(1).strip() - english_term = paren_match.group(2).strip() - - # Create a tuple with both terms for matching - term_pair = (japanese_term, english_term) - if term_pair not in seen: - pairs.append((term_pair, line, currentCategory)) - seen.add(term_pair) - elif dash_match: - japanese_term = dash_match.group(1).strip() - english_term = dash_match.group(2).strip() - - # Create a tuple with both terms for matching - term_pair = (japanese_term, english_term) - if term_pair not in seen: - pairs.append((term_pair, line, currentCategory)) - seen.add(term_pair) - elif line and not line.startswith('#'): - # Fallback for lines without parentheses - treat as single term - term = line.strip() - if term and term not in seen: - pairs.append((term, line, currentCategory)) - seen.add(term) - - return pairs - - -def _japanese_term_in_text(term, text): - """ - Check if a Japanese term appears in text as a standalone word, not as a - substring of a longer run of the same script (katakana/hiragana/kanji). - E.g. 'キス' will NOT match inside 'テキスト' because both neighbours are katakana. - Falls back to plain substring check for non-Japanese or mixed terms. - """ - if term not in text: - return False - KATAKANA = r'ァ-ヴーｦ-ﾟ' - HIRAGANA = r'ぁ-ゔ' - KANJI = r'一-龠' - if re.search(rf'[{KATAKANA}]', term) and not re.search(rf'[{HIRAGANA}{KANJI}]', term): - pattern = rf'(?= 500: - raise Exception(f"API server error ({e.status_code}) - retrying... Error: {e}") - elif e.status_code == 400 and formatType == "json" and "json_schema" in str(responseFormat): - # Only fall back to json_object if the error is NOT "Input should be 'json_schema'" - # (that message means json_schema IS required and json_object would also be rejected) - if "input should be 'json_schema'" in str(e).lower() or "input should be \"json_schema\"" in str(e).lower(): - raise Exception(f"API status error ({e.status_code}): {e}") - # Provider doesn't support json_schema (e.g. Claude) — fall back to json_object - responseFormat = {"type": "json_object"} - params["response_format"] = responseFormat - try: - response = openai.chat.completions.create(**params) - except APIStatusError as fallback_error: - if fallback_error.status_code == 400 and "input should be 'json_schema'" in str(fallback_error).lower(): - raise Exception(f"API requires json_schema response format but rejected the schema. Original error: {e}") - raise Exception(f"API call failed: {e}. Fallback also failed: {fallback_error}") - except Exception as fallback_error: - raise Exception(f"API call failed: {e}. Fallback also failed: {fallback_error}") - elif e.status_code == 400 and "input should be 'json_schema'" in str(e).lower(): - # response_format.type was rejected (e.g. sent "text" or "json_object" to a model - # that only accepts json_schema). Remove response_format and retry with no constraint. - params.pop("response_format", None) - try: - response = openai.chat.completions.create(**params) - except Exception as fallback_error: - raise Exception(f"API call failed: {e}. Fallback also failed: {fallback_error}") - else: - raise Exception(f"API status error ({e.status_code}): {e}") - except (APIConnectionError, RateLimitError) as e: - # These should always be retried - raise Exception(f"API connection/rate limit error - retrying... Error: {e}") - except Exception as e: - # Check if it's a 404 error or other HTTP error that should be retried - error_str = str(e).lower() - if "404" in error_str or "not found" in error_str: - raise Exception(f"API returned 404 Not Found - check your API configuration. Original error: {e}") - - # If structured output fails, fallback to json_object (unless the error - # explicitly states json_schema is required — falling back would just fail again) - if formatType == "json" and "json_schema" in str(responseFormat) and \ - "input should be 'json_schema'" not in error_str: - responseFormat = {"type": "json_object"} - params["response_format"] = responseFormat - try: - response = openai.chat.completions.create(**params) - except Exception as fallback_error: - # If fallback also fails, raise the original error for retry - raise Exception(f"API call failed: {e}. Fallback also failed: {fallback_error}") - else: - raise e - - # Validate response before returning - if not response or not hasattr(response, 'choices') or not response.choices: - raise Exception("API returned invalid or empty response - retrying...") - - _write_request_debug_log(api_provider, params, getattr(response, "usage", None)) - return response - - -def cleanTranslatedText(translatedText, language): - """Clean and format translated text""" - placeholders = { - f"{language} Translation: ": "", - "Translation: ": "", - "っ": "", - "〜": "~", - "ッ": "", - "。": ".", - # Note: 「 and 」 are NOT replaced here — replacing them with ASCII " would - # corrupt raw JSON strings before extraction. They are handled per-line - # in _clean_extracted_line() after JSON parsing. - "—": "―", - "】": "]", - "【": "[", - "é": "e", - "’": "'", - "this guy": "this bastard", - "This guy": "This bastard", - "```json": "", - "```": "", - } - - for target, replacement in placeholders.items(): - translatedText = translatedText.replace(target, replacement) - - # Remove Repeating Characters - pattern = re.compile(r"(.)\s*\1(?:\s*\1){" + str(20 - 1) + r",}") - translatedText = pattern.sub(lambda match: match.group(0).replace(" ", "")[:20], translatedText) - - # Elongate Long Dashes (Since GPT Ignores them...) - translatedText = elongateCharacters(translatedText) - return translatedText - - -def elongateCharacters(text): - """Replace ー sequences with elongated characters""" - # Define a pattern to match one character followed by two or more ー characters. - # The lookbehind is restricted to non-ー Japanese/CJK characters so that: - # - standalone ー separators (e.g. "ーーーーーーーーーー") are left untouched - # - ー sequences preceded by a JSON quote or other non-Japanese char are not corrupted - pattern = r"(?<=([\u3040-\u309F\u30A0-\u30FB\u30FD-\u30FF\u4E00-\u9FEF\uFF61-\uFF9F]))ー{2,}" - - # Define a replacement function that elongates the captured character - def repl(match): - char = match.group(1) # The character before the ー sequence - count = len(match.group(0)) - 1 # Number of ー characters - return char * count # Replace ー sequence with the character repeated - - # Use re.sub() to replace the pattern in the text - return re.sub(pattern, repl, text) - - -def extractTranslation(translatedTextList, isList, pbar=None): - """Extract translation from JSON response. - - This function is resilient to a few common model mistakes: - - Wraps output in code fences or outer quotes - - Uses smart quotes instead of straight quotes - - Inserts an extra leading quote in values (e.g. :""Word" -> :"Word") - - Trailing commas before } or ] - - If strict JSON parsing fails, falls back to a regex-based extractor that - captures LineN values in numeric order. - """ - s = str(translatedTextList or "").strip() - - # Fast exit - if not s: - return None - - # Remove code fences if present - if s.startswith("```"): - s = re.sub(r"^```(?:json)?\s*", "", s, flags=re.IGNORECASE) - s = re.sub(r"\s*```$", "", s) - - # Trim wrapping quotes around the whole JSON blob (common in logs) - if len(s) >= 2 and s[0] == s[-1] and s[0] in {'"', "'"}: - # Only strip if it still looks like JSON inside - if s[1:2] == "{" and s[-2:-1] == "}": - s = s[1:-1] - - # Normalize a broad set of Unicode “smart” quotes to ASCII equivalents. - translation_table = { - 0x201C: "'", # “ left double quotation mark - 0x201D: "'", # ” right double quotation mark - 0xFF02: "'", # ＂ fullwidth quotation mark - - 0x2018: "'", # ‘ left single quotation mark - 0x2019: "'", # ’ right single quotation mark - 0x201B: "'", # ‛ single high-reversed-9 quotation mark - 0x02BC: "'", # ʼ modifier letter apostrophe - 0xFF07: "'", # ＇ fullwidth apostrophe - } - s = s.translate(translation_table) - - # Remove trailing commas before object/array closures - s = re.sub(r",(\s*[}\]])", r"\1", s) - - # Repair common doubled leading quote in values: :""Word" -> :"Word" - # Ensure we don't alter legitimate empty strings (:"") - s = re.sub(r":\s*\"\"(?=[^\",}\]\s])", r':"', s) - - # Attempt strict parse first - try: - lineDict = json.loads(s) - - # Handle array-based schema: {"translations": ["...", ...]} - if isinstance(lineDict, dict) and "translations" in lineDict and isinstance(lineDict["translations"], list): - stringList = [str(v) for v in lineDict["translations"]] - return stringList if isList else (stringList[0] if stringList else None) - - # Build list in numeric order if keys are LineN - numeric_keys = [] - for k in lineDict.keys(): - m = re.fullmatch(r"Line(\d+)", str(k)) - if m: - numeric_keys.append(int(m.group(1))) - - if numeric_keys: - stringList = [lineDict.get(f"Line{n}", "") for n in sorted(numeric_keys)] - else: - # Fallback to values order if no LineN keys found - stringList = list(lineDict.values()) - - return stringList if isList else (stringList[0] if stringList else None) - - except Exception as e: - # Fallback: regex-based extraction tolerant to one or two opening quotes - # Captures escaped quotes within values too - try: - pairs = re.findall(r'"Line(\d+)"\s*:\s*"{1,2}((?:\\.|[^"\\])*)"', s) - if not pairs: - raise ValueError("No LineN pairs found") - - # Sort numerically and unescape JSON string content - items = [] - for n_str, v in sorted(((int(n), v) for n, v in pairs), key=lambda x: x[0]): - try: - # Decode JSON escapes reliably by round-tripping as a JSON string - decoded = json.loads(f'"{v}"') - except Exception: - decoded = v - items.append(decoded) - - return items if isList else (items[0] if items else None) - except Exception as e2: - if pbar: - pbar.write(f"extractTranslation Error: {e2} after JSON error {e} on String {translatedTextList}") - return None - - -def calculateCost(inputTokens, outputTokens, model): - """ - Calculate the cost of translation based on token usage and model pricing. - - For Claude models the cost is derived from the actual cache token breakdown - recorded by translateAI, so cache discounts are reflected accurately: - - Cache reads: 10 % of the base input rate - - Cache writes: 125 % of the base input rate - - Regular input: 100 % of the base input rate - - Call pattern (no module changes required): - Per-file call: file_cost_ready flag is True → read thread-local per-file - accumulators (which span all translateAI calls for the file), - compute cost, reset accumulators, clear flag, return cost. - TOTAL call: file_cost_ready is False (already cleared) → return the - cross-thread _global_accurate_cost running sum. - - Falls back to naive token × rate calculation for non-Claude models. - """ - _is_claude = model and any(x in model.lower() for x in ("claude", "sonnet", "haiku", "opus")) - if _is_claude: - if getattr(_thread_local, 'file_cost_ready', False): - # Per-file call: compute from accumulators (may be 0 for disk-cached files), - # reset everything, return the file cost. - cr = getattr(_thread_local, 'file_cache_read', 0) - cw = getattr(_thread_local, 'file_cache_write', 0) - reg = getattr(_thread_local, 'file_regular', 0) - out = getattr(_thread_local, 'file_output', 0) - bcr = getattr(_thread_local, 'file_batch_read', 0) - bcw = getattr(_thread_local, 'file_batch_write', 0) - breg = getattr(_thread_local, 'file_batch_regular', 0) - bout = getattr(_thread_local, 'file_batch_output', 0) - pricing = getPricingConfig(model) - br = pricing["inputAPICost"] / 1_000_000 - orr = pricing["outputAPICost"] / 1_000_000 - cost = (cr * br * 0.10 + cw * br * 2.00 + reg * br + out * orr - # Batch API tokens: same rates, then the 50% batch discount. - + (bcr * br * 0.10 + bcw * br * 2.00 + breg * br + bout * orr) * 0.50) - _thread_local.file_cache_read = 0 - _thread_local.file_cache_write = 0 - _thread_local.file_regular = 0 - _thread_local.file_output = 0 - _thread_local.file_batch_read = 0 - _thread_local.file_batch_write = 0 - _thread_local.file_batch_regular = 0 - _thread_local.file_batch_output = 0 - _thread_local.file_cost_ready = False - return cost - # TOTAL call (flag already cleared): return the cross-thread running total. - # If _global_accurate_cost is 0 it means no real API calls were made - # (e.g. estimate mode), so fall through to the naive calculation below. - with _global_accurate_cost_lock: - accurate = _global_accurate_cost - if accurate > 0: - return accurate - - # Non-Claude, estimate mode, or no accurate data: naive calculation. - # For Claude models, use the accumulated static_system token count (the portion - # that is always cache-written at the 1hr TTL rate = 2x input rate). - # Remaining tokens are billed at the regular input rate. - pricing = getPricingConfig(model) - _is_claude_naive = model and any(x in model.lower() for x in ("claude", "sonnet", "haiku", "opus")) - if _is_claude_naive: - static_tok = getattr(_thread_local, 'estimate_static_tokens', 0) - regular_tok = getattr(_thread_local, 'estimate_regular_tokens', 0) - batch_count = max(1, getattr(_thread_local, 'estimate_batch_count', 1)) - _thread_local.estimate_static_tokens = 0 - _thread_local.estimate_regular_tokens = 0 - _thread_local.estimate_batch_count = 0 - # If cache is disabled, every batch is a write (2x) — no reads ever. - # Otherwise: each distinct batch size (= distinct output_config schema) gets exactly - # one cache write on first use; all subsequent batches of that size are reads (0.10x). - # Load from disk first so GUI subprocesses (one per file) share warm-cache state. - global _estimate_written_sizes - if DISABLE_CACHE: - write_batches = batch_count - read_batches = 0 - else: - _load_estimate_written_sizes() - seen_sizes = getattr(_thread_local, 'estimate_seen_sizes', set()) - new_sizes = seen_sizes - _estimate_written_sizes - write_batches = len(new_sizes) # one write per newly-seen size - read_batches = batch_count - write_batches - _estimate_written_sizes.update(new_sizes) - _save_estimate_written_sizes() - _thread_local.estimate_seen_sizes = set() - write_cost = (write_batches * static_tok / 1_000_000) * pricing["inputAPICost"] * 2.0 - read_cost = (read_batches * static_tok / 1_000_000) * pricing["inputAPICost"] * 0.10 - regular_cost = (regular_tok / 1_000_000) * pricing["inputAPICost"] - inputCost = write_cost + read_cost + regular_cost - else: - inputCost = (inputTokens / 1_000_000) * pricing["inputAPICost"] - outputCost = (outputTokens / 1_000_000) * pricing["outputAPICost"] - return inputCost + outputCost - - -def countTokens(system, user, history): - """Count tokens for cost estimation""" - inputTotalTokens = 0 - outputTotalTokens = 0 - enc = tiktoken.encoding_for_model("gpt-4") - - # Input - if isinstance(history, list): - for line in history: - inputTotalTokens += len(enc.encode(line)) - else: - inputTotalTokens += len(enc.encode(history)) - inputTotalTokens += len(enc.encode(system)) - inputTotalTokens += len(enc.encode(user)) - - # Output - outputTotalTokens += round(len(enc.encode(user)) * 2.5) - - return [inputTotalTokens, outputTotalTokens] - - -@retry(exceptions=Exception, tries=5, delay=5) -def translateAI(text, history, config, filename=None, pbar=None, lock=None, mismatchList=None): - """ - Main translation entry point used by all modules. - - Returns [translatedText, [inputTokens, outputTokens]]. - """ - if not text: - return [text, [0, 0]] - - # Use TRANSLATION_RUN_LOG env var as log path if set. - run_log = os.getenv("TRANSLATION_RUN_LOG") - if run_log: - # Make sure parent dir exists - try: - Path(run_log).parent.mkdir(parents=True, exist_ok=True) - except Exception: - pass - config.logFilePath = run_log - - # Ensure log directory exists for the configured path - try: - Path(config.logFilePath).parent.mkdir(parents=True, exist_ok=True) - except Exception: - pass - - # Token tracking: [input, output]. - totalTokens = [0, 0] - - # Init per-file accumulators on first call on this thread (never reset here — - # they span all translateAI calls for a file; reset by calculateCost). - if not hasattr(_thread_local, 'file_cache_read'): - _thread_local.file_cache_read = 0 - _thread_local.file_cache_write = 0 - _thread_local.file_regular = 0 - _thread_local.file_output = 0 - # Batch API usage is billed at 50% so it is accumulated separately. - if not hasattr(_thread_local, 'file_batch_read'): - _thread_local.file_batch_read = 0 - _thread_local.file_batch_write = 0 - _thread_local.file_batch_regular = 0 - _thread_local.file_batch_output = 0 - # Snapshot accumulators so end-of-call delta only counts tokens from this call. - _prev_cr = _thread_local.file_cache_read - _prev_cw = _thread_local.file_cache_write - _prev_reg = _thread_local.file_regular - _prev_out = _thread_local.file_output - _prev_bcr = _thread_local.file_batch_read - _prev_bcw = _thread_local.file_batch_write - _prev_breg = _thread_local.file_batch_regular - _prev_bout = _thread_local.file_batch_output - _thread_local.file_cost_ready = False # will be set True at end of translateAI - - # Anthropic batch phase ('collect'/'consume'); None when off, in estimate - # mode, or when the model doesn't route to the native Anthropic SDK. - batch_phase = get_batch_phase() - if batch_phase and (config.estimateMode or not isClaudeNative(config.model)): - batch_phase = None - - if isinstance(text, list): - formatType = "json" - tList = batchList(text, config.batchSize) - else: - formatType = "json" - tList = [text] - - for index, tItem in enumerate(tList): - # Check if text contains target language - if not re.search(config.langRegex, str(tItem)): - if pbar is not None: - pbar.update(len(tItem) if isinstance(tItem, list) else 1) - if isinstance(tItem, list): - for j in range(len(tItem)): - tItem[j] = cleanTranslatedText(tItem[j], config.language) - tList[index] = tItem - else: - tList[index] = cleanTranslatedText(tItem, config.language) - history = tItem[-config.maxHistory:] if isinstance(tItem, list) else tItem - continue - - # Ellipsis-only bypass: strings whose translatable content is purely '…' characters - # (e.g. "「………」") should never be sent to the AI — just convert brackets and pass through. - def _is_ellipsis_only(s): - inner = str(s).strip().lstrip('「『').rstrip('」』').strip() - return bool(inner) and all(c in '\u2026\u30FC' for c in inner) - - def _convert_ellipsis(s): - return str(s).replace('「', '"').replace('」', '"').replace('『', '"').replace('』', '"') - - if isinstance(tItem, list): - if all(_is_ellipsis_only(s) for s in tItem): - tList[index] = [_convert_ellipsis(s) for s in tItem] - if pbar is not None: - pbar.update(len(tItem)) - continue - else: - if _is_ellipsis_only(tItem): - tList[index] = _convert_ellipsis(tItem) - if pbar is not None: - pbar.update(1) - continue - - # Protect script codes before translation - protected_items = [] - all_replacements = {} - - if isinstance(tItem, list): - for j in range(len(tItem)): - if not tItem[j] or not str(tItem[j]).strip(): - protected_items.append("Placeholder Text") - all_replacements[j] = {} - else: - collapsed = re.sub(r'(.)\1{9,}', lambda m: m.group(1) * 10, tItem[j]) - protected_text, replacements = protect_script_codes(collapsed) - protected_items.append(protected_text) - all_replacements[j] = replacements - else: - if not tItem or not str(tItem).strip(): - protected_items = "Placeholder Text" - all_replacements[0] = {} - else: - collapsed = re.sub(r'(.)\1{9,}', lambda m: m.group(1) * 10, tItem) - protected_items, all_replacements[0] = protect_script_codes(collapsed) - - # Filter out corrupted/mojibake text (U+FFFD) from the batch before API call - corrupted_map = {} # original_index -> original_text - if isinstance(tItem, list): - for j in range(len(tItem)): - if tItem[j] and "\ufffd" in str(tItem[j]): - corrupted_map[j] = tItem[j] - elif tItem and "\ufffd" in str(tItem): - # Single corrupted string - skip translation entirely - tList[index] = tItem - if pbar is not None: - pbar.update(1) - history = tItem - continue - - # Filter out items that have content but no Japanese — they need no translation - # and the AI tends to empty them (e.g. "「………」" -> ""). Apply the same - # cleanup that would happen post-translation and restore them afterwards. - no_japanese_map = {} # original_index -> already-cleaned text - if isinstance(tItem, list): - for j in range(len(tItem)): - if j in corrupted_map: - continue - item_str = str(tItem[j]).strip() if tItem[j] else "" - if item_str and item_str != "Placeholder Text" and not re.search(config.langRegex, item_str): - cleaned = cleanTranslatedText(tItem[j], config.language) - cleaned = cleaned.replace("「", '"').replace("」", '"').strip() - no_japanese_map[j] = cleaned - - # Combine skip sets and rebuild protected_items / all_replacements - skip_indices = set(corrupted_map.keys()) | set(no_japanese_map.keys()) - if isinstance(tItem, list) and skip_indices: - clean_indices = [j for j in range(len(tItem)) if j not in skip_indices] - - if not clean_indices: - # Every item is either corrupted or untranslatable — reassemble and move on - result = [] - for j in range(len(tItem)): - if j in corrupted_map: - result.append(corrupted_map[j]) - elif j in no_japanese_map: - result.append(no_japanese_map[j]) - else: - result.append(tItem[j]) - tList[index] = result - if pbar is not None: - pbar.update(len(tItem)) - history = result[-config.maxHistory:] - continue - - # Rebuild protected_items and all_replacements for translatable items only - protected_items = [protected_items[j] for j in clean_indices] - new_replacements = {} - for new_idx, old_idx in enumerate(clean_indices): - new_replacements[new_idx] = all_replacements.get(old_idx, {}) - all_replacements = new_replacements - - # Build filtered tItem for validation (excludes skipped items) - if isinstance(tItem, list) and skip_indices: - clean_tItem = [tItem[j] for j in range(len(tItem)) if j not in skip_indices] - else: - clean_tItem = tItem - - # Format for translation - if isinstance(tItem, list): - payload = {f"Line{i+1}": string for i, string in enumerate(protected_items)} - payload = json.dumps(payload, indent=4, ensure_ascii=False) - subbedT = payload - else: - subbedT = json.dumps({"Line1": protected_items}, indent=4, ensure_ascii=False) - - # Batch collect queues list payloads only. Single strings (speaker and - # variable names) translate live — modules memoize them and embed the - # results into later payloads, so they must resolve identically in both - # passes or the consume pass couldn't match the queued payload keys. - # This is the names-first phase; names are a tiny share of the volume. - queue_for_batch = batch_phase == "collect" and isinstance(tItem, list) - - # Check cache for this exact payload (the collect pass uses a - # non-blocking peek so no pending markers are left behind for the - # consume pass to wait on) - if queue_for_batch: - cached_result = peek_cached_translation(subbedT, config.language) - else: - cached_result = get_cached_translation(subbedT, config.language) - if cached_result is not None: - # In estimate mode, never replace tList[index] from cache — the cached value - # may have been stored for a batch with a different number of skip_indices, - # so its length can differ from the current tItem. Keeping tList[index] as - # the original tItem ensures the returned list always has the correct length. - if not config.estimateMode: - if isinstance(tItem, list): - tList[index] = cached_result - history = cached_result[-config.maxHistory:] - else: - tList[index] = cached_result - history = cached_result - else: - if isinstance(cached_result, list) and cached_result: - history = cached_result[-config.maxHistory:] - elif cached_result: - history = cached_result - - if lock and pbar is not None: - with lock: - pbar.update(len(tItem) if isinstance(tItem, list) else 1) - - continue - - # Create context — static_system is the stable prompt.txt content; - # vocab_text is the per-batch matched vocabulary (dynamic). - static_system, vocab_text, user = createContext(config, subbedT, formatType, history) - - # Batch collect pass: queue the request (built exactly like a live one) - # instead of calling the API. The text stays untranslated; the consume - # pass fills it in from the fetched batch results. History carries the - # preceding source lines so the model still sees scene context. - if queue_for_batch: - numLines = len(clean_tItem) if isinstance(tItem, list) else 1 - params = buildClaudeRequest(static_system, user, history, formatType, - config.model, numLines, vocab_text=vocab_text) - queue_batch_request(subbedT, config.language, params) - if lock and pbar is not None: - with lock: - pbar.update(len(tItem)) - history = tItem[-config.maxHistory:] - continue - - # Calculate estimate if in estimate mode - if config.estimateMode: - estimate = countTokens(static_system + vocab_text, user, history) - totalTokens[0] += estimate[0] - totalTokens[1] += estimate[1] - - # Track exact cache write size (static_system, constant across batches) - # and accumulate non-cached (vocab + user + history) tokens per batch. - _est_api = os.getenv("api", "").strip() - _is_claude_est = ( - config.model - and any(x in config.model.lower() for x in ("claude", "sonnet", "haiku", "opus")) - and (not _est_api or "anthropic" in _est_api.lower()) - ) - if _is_claude_est: - # Use Anthropic's count_tokens API once to get the exact cached token count. - # Only called on the first batch; result reused for all subsequent batches. - if not getattr(_thread_local, 'estimate_static_tokens', 0): - try: - _ant_count_client = anthropic.Anthropic(api_key=openai.api_key) - backtick = chr(96) * 3 - _sys_block = [{"type": "text", "text": backtick + "\n" + static_system + "\n" + backtick, "cache_control": {"type": "ephemeral", "ttl": "1h"}}] - _count_resp = _ant_count_client.beta.messages.count_tokens( - betas=["token-counting-2024-11-01"], - model=config.model, - system=_sys_block, - messages=[{"role": "user", "content": "x"}] - ) - _thread_local.estimate_static_tokens = _count_resp.input_tokens - except Exception: - # Fallback to tiktoken if count_tokens fails - enc = tiktoken.encoding_for_model("gpt-4") - _thread_local.estimate_static_tokens = len(enc.encode(static_system)) - regular_tok = max(0, estimate[0] - getattr(_thread_local, 'estimate_static_tokens', 0)) - _thread_local.estimate_regular_tokens = getattr(_thread_local, 'estimate_regular_tokens', 0) + regular_tok - _thread_local.estimate_batch_count = getattr(_thread_local, 'estimate_batch_count', 0) + 1 - # Track unique batch sizes seen this file (each maps to a distinct schema) - _size = len(clean_tItem) if isinstance(clean_tItem, list) else 1 - _seen = getattr(_thread_local, 'estimate_seen_sizes', set()) - _seen.add(_size) - _thread_local.estimate_seen_sizes = _seen - - # Cache the payload with original text as placeholder for future estimates - if isinstance(tItem, list): - cache_translation(subbedT, tItem, config.language) - else: - cache_translation(subbedT, [tItem], config.language) - - continue - - # --- Translation and Validation Retry Block --- - max_retries = 2 # 1 initial attempt + 2 retries - final_translations = None - last_raw_translation = "" - numLines = len(clean_tItem) if isinstance(tItem, list) else 1 - - for attempt in range(max_retries + 1): - is_valid = True - - # On retries, prepend the correction note to the USER message so the - # cached static_system block is never modified (avoids cache busting). - current_user = user - if attempt > 0: - retry_note = ( - f"IMPORTANT: Your previous attempt was incorrect or incomplete. Please ensure:\n" - f"1. The entire output is translated to {config.language} with no untranslated characters\n" - f"2. The JSON structure is correct with NO EMPTY or near-empty translations\n" - f" - Every line with Japanese text MUST be fully translated\n" - f" - Do NOT leave translations empty (\"\") or as single punctuation marks (\":\")\n" - f"3. ALL placeholders (like __PROTECTED_0__, __PROTECTED_1__, etc.) are preserved EXACTLY as they appear in the input\n" - f" - Do not modify, translate, or remove any __PROTECTED_N__ placeholders\n" - f" - Keep them in the exact same position in your translation\n" - f"4. Do NOT repeat the same letter or symbol many times in a row (e.g. uuuuuuuu... or broken tails)\n" - f" - Keep moans/effects natural; never output long runs of one character\n\n" - ) - current_user = retry_note + user - if pbar: - pbar.write(f"Retrying translation... (Attempt {attempt + 1}/{max_retries + 1})") - - # Translate — the consume pass tries the fetched batch result first; - # a missing or invalid result falls through to the live API. - from_batch = False - if batch_phase == "consume" and attempt == 0: - batch_result = take_batch_result(subbedT, config.language) - if batch_result is not None: - response = _AnthropicCompat( - batch_result.get("text", ""), - batch_result.get("prompt_tokens", 0) or 0, - batch_result.get("completion_tokens", 0) or 0, - batch_result.get("cache_read_input_tokens", 0) or 0, - batch_result.get("cache_creation_input_tokens", 0) or 0, - ) - from_batch = True - _write_request_debug_log("anthropic-batch", {"payload": subbedT}, response.usage) - if not from_batch: - try: - response = translateText(static_system, current_user, history, 0.05, formatType, config.model, numLines, vocab_text=vocab_text) - except Exception as api_err: - err_msg = f"[API_ERROR] {api_err}" - # Print to stdout so the GUI captures it immediately - print(err_msg, flush=True) - if pbar: - pbar.write(err_msg) - # Also write to the translation log file for persistence - try: - Path(config.logFilePath).parent.mkdir(parents=True, exist_ok=True) - with open(config.logFilePath, "a", encoding="utf-8") as _lf: - _lf.write(f"{err_msg}\n") - _lf.flush() - except Exception: - pass - raise # Let retry decorator handle it - translatedText = response.choices[0].message.content - last_raw_translation = translatedText - - # Update token count for this attempt - totalTokens[0] += response.usage.prompt_tokens - totalTokens[1] += response.usage.completion_tokens - - # --- Cache cost tracking (Claude only) --- - _is_claude_model = config.model and any(x in config.model.lower() for x in ("claude", "sonnet", "haiku", "opus")) - if _is_claude_model: - usage = response.usage - - # Read cache fields from _AnthropicCompat._Usage; fall back to model_extra. - def _get_usage_field(field): - v = getattr(usage, field, None) - if v is None: - v = (getattr(usage, "model_extra", None) or {}).get(field) - return int(v) if v else 0 - - batch_cache_read = _get_usage_field("cache_read_input_tokens") - batch_cache_write = _get_usage_field("cache_creation_input_tokens") - batch_prompt_total = getattr(usage, "prompt_tokens", 0) or 0 - batch_regular = max(0, batch_prompt_total - batch_cache_read - batch_cache_write) - batch_output = getattr(usage, "completion_tokens", 0) or 0 - - # Accumulate into per-file thread-local counters. Batch API - # usage is billed at 50% so it goes into its own counters. - if from_batch: - _thread_local.file_batch_read += batch_cache_read - _thread_local.file_batch_write += batch_cache_write - _thread_local.file_batch_regular += batch_regular - _thread_local.file_batch_output += batch_output - else: - _thread_local.file_cache_read += batch_cache_read - _thread_local.file_cache_write += batch_cache_write - _thread_local.file_regular += batch_regular - _thread_local.file_output += batch_output - - # --- Debug Token Logging --- - if DEBUG: - try: - _dbg_dir = Path("log") - _dbg_dir.mkdir(parents=True, exist_ok=True) - with open(_dbg_dir / "debug.log", "a", encoding="utf-8") as _dbf: - _dbf.write(f"\n--- Batch ({len(clean_tItem) if isinstance(tItem, list) else 1} lines) ---\n") - _dbf.write(f"Prompt: {response.usage.prompt_tokens} tokens | Output: {response.usage.completion_tokens} tokens\n") - if hasattr(response.usage, "cache_read_input_tokens"): - cr = getattr(response.usage, "cache_read_input_tokens", 0) or 0 - cw = getattr(response.usage, "cache_creation_input_tokens", 0) or 0 - cache_status = "HIT" if cr > 0 else ("WRITE" if cw > 0 else "MISS") - _dbf.write(f"Cache: {cache_status} (read={cr}, write={cw})\n") - _dbf.flush() - except Exception: - pass - - # Clean the translation first for consistency - cleaned_text = cleanTranslatedText(translatedText, config.language) - - # Process and validate translation result - if cleaned_text: - if isinstance(tItem, list): - extracted = extractTranslation(cleaned_text, True, pbar) - - # Check 1: Mismatch in length -> still a hard failure - if extracted is None or len(clean_tItem) != len(extracted): - is_valid = False - if pbar: - pbar.write(f"Length mismatch: expected {len(clean_tItem)}, got {len(extracted) if extracted else 0}") - else: - # Check 2: Validate placeholders are preserved - # Flatten all_replacements for batch validation - all_protected_text = protected_items # The list we sent - placeholder_valid, missing, extra = validate_placeholders(all_protected_text, extracted, - {k: v for replacements in all_replacements.values() for k, v in replacements.items()}) - - if not placeholder_valid: - is_valid = False - if pbar: - if missing: - pbar.write(f"Missing placeholders: {', '.join(missing)}") - if extra: - pbar.write(f"Extra placeholders: {', '.join(extra)}") - else: - # Check 3: Validate that translations are not empty or nearly empty - content_valid, invalid_indices, content_reasons = validate_translation_content( - clean_tItem, extracted, config.langRegex - ) - - if not content_valid: - is_valid = False - if pbar: - pbar.write(f"Invalid translation content detected:") - for reason in content_reasons[:5]: # Show first 5 issues - pbar.write(f" - {reason}") - if len(content_reasons) > 5: - pbar.write(f" ... and {len(content_reasons) - 5} more issues") - else: - # Set translations (line count matches, placeholders valid, and content is good) - # Strip "Placeholder Text" from individual lines (AI placeholder for untranslatable input) - # Also apply the 「→" / 」→" replacements here per-line (safe now that JSON is parsed) - def _clean_extracted_line(line): - if not isinstance(line, str): - return line - line = line.replace("Placeholder Text", "").strip() - line = line.replace("「", '"').replace("」", '"') - line = line.replace("『", '"').replace("』", '"') - return line - final_translations = [_clean_extracted_line(line) for line in extracted] - else: - # Single string: extract from JSON schema response - extracted = extractTranslation(cleaned_text, False, pbar) - if extracted is None: - is_valid = False - if pbar: - pbar.write(f"Failed to extract translation from response: {cleaned_text[:100]}") - else: - # Validate placeholders against extracted value - placeholder_valid, missing, extra = validate_placeholders(protected_items, extracted, all_replacements[0]) - - if not placeholder_valid: - is_valid = False - if pbar: - if missing: - pbar.write(f"Missing placeholders: {', '.join(missing)}") - if extra: - pbar.write(f"Extra placeholders: {', '.join(extra)}") - else: - # Validate content for single string - final_cleaned = extracted.replace("Placeholder Text", "") - content_valid, _, content_reasons = validate_translation_content( - tItem, final_cleaned, config.langRegex - ) - - if not content_valid: - is_valid = False - if pbar: - pbar.write(f"Invalid translation content:") - for reason in content_reasons: - pbar.write(f" - {reason}") - else: - # Accept output - all validations passed - final_translations = final_cleaned - else: - is_valid = False - if pbar: pbar.write(f"AI Refused: {tItem}\n") - - # If translation is valid, break the retry loop - if is_valid: - break - - # --- End of Retry Block --- - - # After the loop, handle the final result - if final_translations is not None: # Success case - # Restore protected script codes - if isinstance(tItem, list): - for j in range(len(final_translations)): - if j in all_replacements: - final_translations[j] = restore_script_codes(final_translations[j], all_replacements[j]) - - # Re-insert corrupted / no-japanese originals at their original positions - if corrupted_map or no_japanese_map: - expanded = [] - clean_idx = 0 - for j in range(len(tItem)): - if j in corrupted_map: - expanded.append(corrupted_map[j]) - elif j in no_japanese_map: - expanded.append(no_japanese_map[j]) - else: - expanded.append(final_translations[clean_idx]) - clean_idx += 1 - final_translations = expanded - else: - final_translations = restore_script_codes(final_translations, all_replacements[0]) - - formatted_output = last_raw_translation - try: - parsed_json = json.loads(last_raw_translation) - # Normalize array-based output to LineN format for log readability - if isinstance(parsed_json, dict) and "translations" in parsed_json and isinstance(parsed_json["translations"], list): - parsed_json = {f"Line{i+1}": v for i, v in enumerate(parsed_json["translations"])} - formatted_output = json.dumps(parsed_json, indent=4, ensure_ascii=False) - except (json.JSONDecodeError, ValueError): - pass - - # Only open and write to log file when we have something to log - try: - with open(config.logFilePath, "a", encoding="utf-8") as logFile: - logFile.write(f"Input:\n{subbedT}\n") - logFile.write(f"Output:\n{formatted_output}\n") - logFile.flush() # Ensure data is written to disk immediately - except Exception: - pass # Don't fail if logging fails - - # Cache the entire payload and its translation - if not config.estimateMode: - cache_translation(subbedT, final_translations, config.language) - - if isinstance(tItem, list): - tList[index] = final_translations - history = final_translations[-config.maxHistory:] - else: - tList[index] = final_translations - history = final_translations - - if lock and pbar is not None: - with lock: - pbar.update(len(tItem) if isinstance(tItem, list) else 1) - - else: # Failure case after all retries - if pbar: pbar.write(f"Translation failed after {max_retries + 1} attempts. Check mismatch log.") - - # Emit a machine-readable marker on stdout so the GUI worker - # thread can detect the mismatch reliably (stdout is captured - # synchronously, unlike file-tail polling which can be racy). - try: - print(f"MISMATCH_EVENT:{filename}", flush=True) - except Exception: - pass - - formatted_mismatch_output = last_raw_translation - try: - parsed_json = json.loads(last_raw_translation) - if isinstance(parsed_json, dict) and "translations" in parsed_json and isinstance(parsed_json["translations"], list): - parsed_json = {f"Line{i+1}": v for i, v in enumerate(parsed_json["translations"])} - formatted_mismatch_output = json.dumps(parsed_json, indent=4, ensure_ascii=False) - except (json.JSONDecodeError, ValueError): - pass - with open(config.mismatchLogPath, "a+", encoding="utf-8") as mismatchFile: - mismatchFile.write(f"Failed after retries: {filename}\n") - mismatchFile.write(f"Input:\n{subbedT}\n") - mismatchFile.write(f"Final Output:\n{formatted_mismatch_output}\n") - mismatchFile.flush() # Ensure data is written to disk immediately - - # Also write to the main translation log so the GUI log viewer can display it - try: - with open(config.logFilePath, "a", encoding="utf-8") as logFile: - logFile.write(f"[MISMATCH] Failed after retries: {filename}\n") - logFile.write(f"[MISMATCH] Input:\n") - for mline in subbedT.splitlines(): - logFile.write(f"[MISMATCH] {mline}\n") - logFile.write(f"[MISMATCH] Final Output:\n") - for mline in formatted_mismatch_output.splitlines(): - logFile.write(f"[MISMATCH] {mline}\n") - logFile.flush() - except Exception: - pass # Don't fail if logging fails - - if filename and mismatchList is not None and filename not in mismatchList: - mismatchList.append(filename) - - tList[index] = tItem - history = text[-config.maxHistory:] if isinstance(text, list) else text - - # Combine if multilist - if tList and isinstance(tList[0], list): - tList = [t for sublist in tList for t in sublist] - - # Save cache after processing (for both estimate and translation modes) - save_cache() - - # Batch collect pass: merge this call's queued requests into the disk queue. - if batch_phase == "collect": - flush_batch_queue() - - # For Claude: accumulate only this call's delta into the cross-thread total. - # file_* accumulators hold full per-file totals; calculateCost() reads them. - _is_claude_final = config.model and any(x in config.model.lower() for x in ("claude", "sonnet", "haiku", "opus")) - if _is_claude_final and not config.estimateMode: - _pricing = getPricingConfig(config.model) - _br = _pricing["inputAPICost"] / 1_000_000 - _or = _pricing["outputAPICost"] / 1_000_000 - # Delta = tokens added in this call only (not earlier calls for same file). - _delta_cr = getattr(_thread_local, 'file_cache_read', 0) - _prev_cr - _delta_cw = getattr(_thread_local, 'file_cache_write', 0) - _prev_cw - _delta_reg = getattr(_thread_local, 'file_regular', 0) - _prev_reg - _delta_out = getattr(_thread_local, 'file_output', 0) - _prev_out - _delta_bcr = getattr(_thread_local, 'file_batch_read', 0) - _prev_bcr - _delta_bcw = getattr(_thread_local, 'file_batch_write', 0) - _prev_bcw - _delta_breg = getattr(_thread_local, 'file_batch_regular', 0) - _prev_breg - _delta_bout = getattr(_thread_local, 'file_batch_output', 0) - _prev_bout - _call_cost = ( - _delta_cr * _br * 0.10 + - _delta_cw * _br * 2.00 + - _delta_reg * _br + - _delta_out * _or + - # Batch API tokens: same rates, then the 50% batch discount. - (_delta_bcr * _br * 0.10 + - _delta_bcw * _br * 2.00 + - _delta_breg * _br + - _delta_bout * _or) * 0.50 - ) - global _global_accurate_cost - with _global_accurate_cost_lock: - _global_accurate_cost += _call_cost - _thread_local.file_cost_ready = True # signals calculateCost to use file accumulators - - # Return result - if isinstance(text, list): - return [tList, totalTokens] - else: +""" +Shared translation utilities for DazedMTLTool. +Centralized translation function used across all modules. +""" + +import os +import re +import json +import time +import random +import unicodedata +import tiktoken +import openai +import anthropic +import urllib.request +from openai import APIError, APIConnectionError, RateLimitError, APIStatusError +import hashlib +import threading +from contextlib import contextmanager +from dotenv import load_dotenv +from pathlib import Path +from retry import retry + +# Set to True to enable debug logging (token counts, cache costs, etc.) +DEBUG = True +_debug_request_log_lock = threading.Lock() + +# Set to True to disable Claude prompt caching for baseline cost comparison. +DISABLE_CACHE = False + +# Thread-local per-file token breakdown; read by calculateCost() for Claude. +_thread_local = threading.local() + +# Cross-thread running total of accurate cache-discounted cost (protected by lock). +_global_accurate_cost = 0.0 +_global_accurate_cost_lock = threading.Lock() + + +def _usage_to_debug_dict(usage): + """Extract token counts from provider usage objects for request debugging.""" + if not usage: + return {} + + usage_dict = {} + for field in ( + "prompt_tokens", + "completion_tokens", + "input_tokens", + "output_tokens", + "cache_read_input_tokens", + "cache_creation_input_tokens", + ): + value = getattr(usage, field, None) + if value is not None: + usage_dict[field] = value + + extra = getattr(usage, "model_extra", None) + if isinstance(extra, dict): + for field in ("cache_read_input_tokens", "cache_creation_input_tokens"): + value = extra.get(field) + if value is not None and field not in usage_dict: + usage_dict[field] = value + + return usage_dict + + +def _write_request_debug_log(provider, request_payload, usage): + """Write the exact SDK payload text and returned token usage.""" + if not DEBUG: + return + + try: + log_dir = Path("log") + log_dir.mkdir(parents=True, exist_ok=True) + usage_dict = _usage_to_debug_dict(usage) + payload_text = json.dumps(request_payload, indent=2, ensure_ascii=False, default=str) + usage_text = json.dumps(usage_dict, indent=2, ensure_ascii=False, default=str) + + with _debug_request_log_lock: + with open(log_dir / "request_debug.log", "a", encoding="utf-8") as debug_file: + debug_file.write("\n=== API Request ===\n") + debug_file.write(f"Timestamp: {time.strftime('%Y-%m-%d %H:%M:%S')}\n") + debug_file.write(f"Provider: {provider}\n") + debug_file.write("Usage:\n") + debug_file.write(f"{usage_text}\n") + debug_file.write("Payload:\n") + debug_file.write(f"{payload_text}\n") + debug_file.flush() + except Exception: + pass + +def _normalize_openai_base_url(url: str) -> str: + """Ensure OpenAI SDK global base_url has a trailing slash.""" + _url = (url or "").strip() + if _url and not _url.endswith("/"): + _url += "/" + return _url + + +def isClaudeModel(model): + """True when the model name looks like an Anthropic Claude model.""" + return bool(model) and any(x in model.lower() for x in ("claude", "sonnet", "haiku", "opus")) + + +def isClaudeNative(model): + """True when this model routes to the native Anthropic SDK. + + Mirrors the routing check in translateText: the model must look like Claude + AND the configured API URL must be unset or point at anthropic.com. Any + other custom URL (e.g. DeepSeek, OpenAI proxy) uses the OpenAI-compatible + path even for Claude-named models. + """ + live_api = os.getenv("api", "").strip() + return isClaudeModel(model) and (not live_api or "anthropic" in live_api.lower()) + + +def isMistralAPI(): + """True when requests go to the Mistral platform API (la Plateforme). + + Detection is URL-based: the adaptive rate limiter keys off x-ratelimit-* + headers that only api.mistral.ai sends. Mistral models served through other + OpenAI-compatible providers (Nvidia, OpenRouter, ...) use the generic path. + """ + live_api = os.getenv("api", "").strip().lower() + if "mistral.ai" in live_api: + return True + return not live_api and os.getenv("API_PROVIDER", "").strip().lower() == "mistral" + + +# Models that REJECT sampling params (temperature/top_p/top_k) with a 400 — +# Claude Opus 4.7 and up retired them in favour of adaptive thinking. Matches +# opus-4-7, opus-4-8, opus-4-10+ and fable, but NOT opus-4-6 or Sonnet/Haiku. +_NO_SAMPLING_RE = re.compile(r"opus-4-(?:[7-9]\b|[1-9]\d)|fable", re.I) + +# Tracks which distinct batch sizes have already been cache-written during this estimate run. +# Each unique numLines value maps to a distinct output_config schema → one write per size. +# Persisted to disk so sequential GUI subprocesses share state. +_estimate_written_sizes: set = set() +_ESTIMATE_SIZES_FILE = Path("log/estimate_written_sizes.json") + +def _load_estimate_written_sizes(): + """Load persisted written-sizes set from disk (for GUI subprocess sharing).""" + global _estimate_written_sizes + try: + if _ESTIMATE_SIZES_FILE.exists(): + with open(_ESTIMATE_SIZES_FILE, "r", encoding="utf-8") as f: + _estimate_written_sizes = set(json.load(f)) + except Exception: + _estimate_written_sizes = set() + +def _save_estimate_written_sizes(): + """Persist written-sizes set to disk.""" + try: + _ESTIMATE_SIZES_FILE.parent.mkdir(parents=True, exist_ok=True) + with open(_ESTIMATE_SIZES_FILE, "w", encoding="utf-8") as f: + json.dump(list(_estimate_written_sizes), f) + except Exception: + pass + +def clear_estimate_written_sizes(): + """Reset the written-sizes file at the start of a new estimate run.""" + global _estimate_written_sizes + _estimate_written_sizes = set() + try: + if _ESTIMATE_SIZES_FILE.exists(): + _ESTIMATE_SIZES_FILE.unlink() + except Exception: + pass + + +# ===== Placeholder Protection System ===== +# Patterns to protect from translation (sound effects, control codes, etc.) +PROTECTED_PATTERNS = [ + r'\\SE\[[^\]]+\]', # \SE[sound_effect_name] + r'\\ME\[[^\]]+\]', # \ME[music_effect_name] + r'\\BGM\[[^\]]+\]', # \BGM[background_music_name] + r'\\BGS\[[^\]]+\]', # \BGS[background_sound_name] + r'_pum\[[^\]]+\]', # _pum[name] + r'\\VS\[[^\]]+\]', # \VS[name] +] + +def protect_script_codes(text): + """ + Replace script codes (like \\SE[タイプライター]) with unique placeholders before translation. + Returns: (protected_text, replacements_dict) + """ + if not text or not isinstance(text, str): + return text, {} + + # Normalize curly/smart quotes to ASCII equivalents BEFORE building the JSON + # payload. When these characters appear inside a JSON string value the AI + # tends to treat them as regular ASCII double-quotes, which makes the value + # appear empty (e.g. `"スキルを"リセットする` → AI sees empty + stray text). + # This mirrors the identical normalization already applied to the AI's OUTPUT + # inside extractTranslation's translation_table. + quote_norm_table = str.maketrans({ + '\u201C': "'", # " left double quotation mark + '\u201D': "'", # " right double quotation mark + '\uFF02': "'", # ＂ fullwidth quotation mark + '\u2018': "'", # ' left single quotation mark + '\u2019': "'", # ' right single quotation mark + '\u201B': "'", # ‛ single high-reversed-9 quotation mark + '\u02BC': "'", # ʼ modifier letter apostrophe + '\uFF07': "'", # ＇ fullwidth apostrophe + }) + text = text.translate(quote_norm_table) + + # Convert half-width katakana (U+FF61–U+FF9F) to full-width katakana so the + # AI recognises them as Japanese text and translates them correctly. + # NFKC is applied only to matched half-width kana spans to avoid altering + # intentional fullwidth Latin/digit characters elsewhere in the string. + text = re.sub(r'[\uFF61-\uFF9F]+', lambda m: unicodedata.normalize('NFKC', m.group(0)), text) + + replacements = {} + protected_text = text + counter = 0 + + # Combine all patterns + combined_pattern = '|'.join(f'({pattern})' for pattern in PROTECTED_PATTERNS) + + def replace_match(match): + nonlocal counter + original = match.group(0) + # Create a unique placeholder that won't be translated + placeholder = f"__PROTECTED_{counter}__" + replacements[placeholder] = original + counter += 1 + return placeholder + + if combined_pattern: + protected_text = re.sub(combined_pattern, replace_match, protected_text) + + return protected_text, replacements + + +def restore_script_codes(text, replacements): + """ + Restore protected script codes from placeholders after translation. + """ + if not text or not replacements: + return text + + if isinstance(text, str): + result = text + for placeholder, original in replacements.items(): + result = result.replace(placeholder, original) + return result + elif isinstance(text, list): + return [restore_script_codes(item, replacements) for item in text] + else: + return text + + +def validate_placeholders(original_text, translated_text, replacements): + """ + Validate that all placeholders from the original text appear in the translation. + Returns: (is_valid, missing_placeholders, extra_placeholders) + """ + if not replacements: + return True, [], [] + + # Get all placeholders + all_placeholders = set(replacements.keys()) + + # Count placeholders in original + original_counts = {} + for placeholder in all_placeholders: + if isinstance(original_text, str): + original_counts[placeholder] = original_text.count(placeholder) + elif isinstance(original_text, list): + original_counts[placeholder] = sum(str(item).count(placeholder) for item in original_text) + + # Count placeholders in translation + translated_counts = {} + for placeholder in all_placeholders: + if isinstance(translated_text, str): + translated_counts[placeholder] = translated_text.count(placeholder) + elif isinstance(translated_text, list): + translated_counts[placeholder] = sum(str(item).count(placeholder) for item in translated_text) + + # Find mismatches + missing = [] + extra = [] + for placeholder in all_placeholders: + orig_count = original_counts.get(placeholder, 0) + trans_count = translated_counts.get(placeholder, 0) + + if trans_count < orig_count: + missing.append(f"{placeholder} (expected {orig_count}, found {trans_count})") + elif trans_count > orig_count: + extra.append(f"{placeholder} (expected {orig_count}, found {trans_count})") + + is_valid = len(missing) == 0 and len(extra) == 0 + return is_valid, missing, extra + + +def validate_translation_content(original_items, translated_items, langRegex): + """ + Validate that translated items are not empty or nearly empty. + Returns: (is_valid, invalid_indices, reasons) + + Rules: + 1. If original has content, translation must not be empty or just whitespace + 2. If original has Japanese text, translation must not be a single punctuation mark + 3. Translation should have meaningful content (more than 1-2 characters for substantial originals) + """ + if not isinstance(original_items, list): + original_items = [original_items] + translated_items = [translated_items] + + invalid_indices = [] + reasons = [] + + for i, (orig, trans) in enumerate(zip(original_items, translated_items)): + orig_str = str(orig).strip() + trans_str = str(trans).strip() + + # Skip if original is empty or placeholder + if not orig_str or orig_str == "Placeholder Text": + continue + + # Check if original has content that needs translation + has_source_text = bool(re.search(langRegex, orig_str)) + + if has_source_text: + # Original has Japanese text - translation must be substantial + + # Check 1: Translation is empty or just whitespace + if not trans_str: + invalid_indices.append(i) + reasons.append(f"Line{i+1}: Empty translation for '{orig_str[:50]}...'") + continue + + # Check 2: Translation is just a single punctuation mark or very short + # Allow control codes like \\C[27]\\V[45] but not just ":" or "" + # Use <= 1 so real 2-char words like "No", "Go", "Hi" are not rejected + if len(trans_str) <= 1 and not re.search(r'\\[A-Z]\[', trans_str): + # Exception: if original is also very short (like "回" -> "x"), that's ok + if len(orig_str) > 3: + invalid_indices.append(i) + reasons.append(f"Line{i+1}: Translation too short ('{trans_str}') for '{orig_str[:50]}...'") + continue + + # Check 3: For longer originals (>10 chars), translation should be more than just 1-2 chars + # unless it's a special case like numbers or codes + if len(orig_str) > 10 and len(trans_str) <= 2: + # Allow if it contains control codes or is just a replacement word + if not re.search(r'\\[A-Z]\[', trans_str) and not trans_str.isalnum(): + invalid_indices.append(i) + reasons.append(f"Line{i+1}: Translation suspiciously short ('{trans_str}') for '{orig_str[:50]}...'") + continue + + # Check 4: Runaway translation - translation is excessively long relative to original + # Catches cases where the model repeats words endlessly (e.g. "it hurts it hurts it hurts...") + ratio_limit = max(len(orig_str) * 8, 120) + if len(orig_str) > 10 and len(trans_str) > ratio_limit: + invalid_indices.append(i) + reasons.append(f"Line{i+1}: Runaway translation (output {len(trans_str)} chars vs input {len(orig_str)} chars) for '{orig_str[:50]}...'") + continue + # Absolute cap: garbage outputs that are not caught by ratio alone + if len(trans_str) > 4000 and len(trans_str) > len(orig_str) * 3: + invalid_indices.append(i) + reasons.append(f"Line{i+1}: Runaway translation (output {len(trans_str)} chars exceeds cap) for '{orig_str[:50]}...'") + continue + + # Check 5: Same character repeated many times (common API glitch / broken JSON tail) + if re.search(r"(.)\1{44,}", trans_str): + invalid_indices.append(i) + reasons.append(f"Line{i+1}: Excessive character repetition (possible model glitch) in translation") + continue + + is_valid = len(invalid_indices) == 0 + return is_valid, invalid_indices, reasons + +# Load .env, strip accidental whitespace, set base URL / org / API key. +# Gemini/Mistral use their endpoints only when no custom API URL is set. +load_dotenv() +api_provider = os.getenv("API_PROVIDER", "openai").lower() +env_api = os.getenv("api", "").strip() +if api_provider == "gemini" and not env_api: + # Use Google Generative Language compatibility endpoint only as fallback. + openai.base_url = "https://generativelanguage.googleapis.com/v1beta/openai/" + openai.organization = None +elif api_provider == "mistral" and not env_api: + openai.base_url = "https://api.mistral.ai/v1/" + openai.organization = None +else: + if env_api: + openai.base_url = _normalize_openai_base_url(env_api) + # Support both 'organization' (gui/.env.example) and legacy 'org' names + org = os.getenv("organization") or os.getenv("org") + if org: + openai.organization = org.strip() + +# Always set API key from 'key' env var (trim whitespace) +openai.api_key = os.getenv("key", "").strip() + +# Translation cache management +CACHE_FILE = Path("log/translation_cache.json") +CACHE_LOCK_FILE = Path("log/translation_cache.lock") +CACHE_LOCK = threading.RLock() +CACHE_PENDING_MARKER = "__translation_pending__" +CACHE_PENDING_TTL = 600 +CACHE_WAIT_INTERVAL = 0.25 +_cache = None + +@contextmanager +def _translation_cache_file_lock(): + """Cross-process lock for translation_cache.json.""" + CACHE_LOCK_FILE.parent.mkdir(parents=True, exist_ok=True) + with open(CACHE_LOCK_FILE, "a+b") as lock_file: + if os.name == "nt": + import msvcrt + lock_file.seek(0) + msvcrt.locking(lock_file.fileno(), msvcrt.LK_LOCK, 1) + try: + yield + finally: + lock_file.seek(0) + msvcrt.locking(lock_file.fileno(), msvcrt.LK_UNLCK, 1) + else: + import fcntl + fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX) + try: + yield + finally: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN) + +def _read_cache_from_disk(): + """Read the disk cache; return an empty dict if it is unavailable.""" + try: + if CACHE_FILE.exists(): + with open(CACHE_FILE, "r", encoding="utf-8") as f: + data = json.load(f) + return data if isinstance(data, dict) else {} + except Exception: + pass + return {} + +def _write_cache_to_disk(cache): + """Atomically write the cache using a process/thread-unique temp file.""" + CACHE_FILE.parent.mkdir(parents=True, exist_ok=True) + tmp_file = CACHE_FILE.with_name( + f"{CACHE_FILE.name}.{os.getpid()}.{threading.get_ident()}.tmp" + ) + with open(tmp_file, "w", encoding="utf-8") as f: + json.dump(cache, f, ensure_ascii=False, indent=2) + tmp_file.replace(CACHE_FILE) + +def _is_pending_cache_entry(value): + return isinstance(value, dict) and value.get(CACHE_PENDING_MARKER) is True + +def _is_stale_pending_cache_entry(value): + if not _is_pending_cache_entry(value): + return False + try: + return time.time() - float(value.get("time", 0)) > CACHE_PENDING_TTL + except Exception: + return True + +def _is_own_pending_cache_entry(value): + return ( + _is_pending_cache_entry(value) + and value.get("pid") == os.getpid() + and value.get("thread") == threading.get_ident() + ) + +def _pending_cache_entry(): + return { + CACHE_PENDING_MARKER: True, + "pid": os.getpid(), + "thread": threading.get_ident(), + "time": time.time(), + } + +def _merge_translation_caches(base, overlay): + """Merge cache dictionaries while never replacing a translation with pending.""" + merged = dict(base or {}) + for key, value in (overlay or {}).items(): + existing = merged.get(key) + if _is_pending_cache_entry(value) and existing is not None: + if not _is_pending_cache_entry(existing): + continue + if not _is_stale_pending_cache_entry(existing): + continue + merged[key] = value + return merged + +def clear_cache(): + """Clear the translation cache (called at start of each run)""" + global _cache + with CACHE_LOCK: + _cache = {} + with _translation_cache_file_lock(): + try: + if CACHE_FILE.exists(): + CACHE_FILE.unlink() + except Exception: + pass + +def load_cache(): + """Load the translation cache from disk.""" + global _cache + with CACHE_LOCK: + with _translation_cache_file_lock(): + disk_cache = _read_cache_from_disk() + if _cache: + disk_cache = _merge_translation_caches(disk_cache, _cache) + _cache = disk_cache + return _cache + +def save_cache(): + """Save the translation cache to disk, preserving entries from other workers.""" + global _cache + if _cache is None: + return + + with CACHE_LOCK: + try: + with _translation_cache_file_lock(): + disk_cache = _read_cache_from_disk() + disk_cache = _merge_translation_caches(disk_cache, _cache) + _cache = disk_cache + _write_cache_to_disk(_cache) + except Exception: + pass + +def get_cache_key(payload, language): + """Generate a cache key for a payload (can be single string or JSON batch)""" + # Use hash to keep keys short but unique + payload_str = str(payload) if payload is not None else "" + combined = f"{payload_str}|{language}" + return hashlib.md5(combined.encode("utf-8")).hexdigest() + +def get_cached_translation(payload, language): + """Get cached translation if it exists""" + global _cache + key = get_cache_key(payload, language) + while True: + with CACHE_LOCK: + with _translation_cache_file_lock(): + cache = _read_cache_from_disk() + if _cache: + cache = _merge_translation_caches(cache, _cache) + + entry = cache.get(key) + if ( + entry is None + or _is_stale_pending_cache_entry(entry) + or _is_own_pending_cache_entry(entry) + ): + cache[key] = _pending_cache_entry() + _cache = cache + _write_cache_to_disk(cache) + return None + + _cache = cache + if not _is_pending_cache_entry(entry): + return entry + + time.sleep(CACHE_WAIT_INTERVAL) + +def cache_translation(payload, translation, language): + """Cache a translation payload and its response""" + global _cache + key = get_cache_key(payload, language) + + with CACHE_LOCK: + with _translation_cache_file_lock(): + cache = _read_cache_from_disk() + if _cache: + cache = _merge_translation_caches(cache, _cache) + cache[key] = translation + _cache = cache + _write_cache_to_disk(cache) + + +# Variable translation map (code 122 <-> code 111 consistency) +VAR_MAP_FILE = Path("log/var_translation_map.json") +VAR_MAP_LOCK = threading.Lock() +_var_map = None + +def clear_var_map(): + """Clear the variable translation map (called at start of each run)""" + global _var_map + with VAR_MAP_LOCK: + _var_map = {} + try: + if VAR_MAP_FILE.exists(): + VAR_MAP_FILE.unlink() + except Exception: + pass + +def _load_var_map(): + """Load the variable translation map from disk (always re-reads to pick up + entries written by other subprocesses).""" + global _var_map + _var_map = {} + try: + if VAR_MAP_FILE.exists(): + with open(VAR_MAP_FILE, "r", encoding="utf-8") as f: + _var_map = json.load(f) + except Exception: + _var_map = {} + return _var_map + +def _save_var_map(): + """Save the variable translation map to disk. + Re-reads the file first and merges so entries from other subprocesses + are never lost.""" + global _var_map + if _var_map is None: + return + try: + VAR_MAP_FILE.parent.mkdir(parents=True, exist_ok=True) + # Re-read the on-disk version and merge our entries on top + disk_map = {} + try: + if VAR_MAP_FILE.exists(): + with open(VAR_MAP_FILE, "r", encoding="utf-8") as f: + disk_map = json.load(f) + except Exception: + disk_map = {} + disk_map.update(_var_map) + _var_map = disk_map + tmp_file = VAR_MAP_FILE.with_suffix(".tmp") + with open(tmp_file, "w", encoding="utf-8") as f: + json.dump(_var_map, f, ensure_ascii=False, indent=2) + tmp_file.replace(VAR_MAP_FILE) + except Exception: + pass + +def get_var_translation(original): + """Look up a cached variable translation. Returns the translation or None.""" + with VAR_MAP_LOCK: + m = _load_var_map() + return m.get(original) + +def set_var_translation(original, translated): + """Store a variable translation and persist to disk. + Skips if the translation is identical to the original (untranslated). + """ + if original == translated: + return + with VAR_MAP_LOCK: + m = _load_var_map() + m[original] = translated + _save_var_map() + +def set_var_translations_batch(pairs): + """Store multiple variable translations at once and persist to disk. + pairs: list of (original, translated) tuples + Skips pairs where the translation is identical to the original (untranslated). + """ + with VAR_MAP_LOCK: + m = _load_var_map() + for original, translated in pairs: + if original != translated: + m[original] = translated + _save_var_map() + + +# ===== Anthropic Message Batches (50% off all token usage) ===== +# Batch integration by Len — two-pass collect/consume flow; see README Credits. +# Batch translation is a two-pass flow driven by the batch phase (kept in the +# BATCH_PHASE env var so GUI subprocesses inherit it): +# collect: translateAI builds each cache-missed request (byte-identical to a +# live request, including the cached system block) and queues it +# instead of calling the API. Text is left untranslated. +# consume: translateAI feeds the fetched batch responses through the normal +# validation/restore path; anything missing or invalid falls back +# to the live API automatically. +# Between the passes, submit/poll/fetch the queue with runTranslationBatches(). +BATCH_QUEUE_FILE = Path("log/batch_requests.json") +BATCH_STATE_FILE = Path("log/batch_state.json") +BATCH_RESULTS_FILE = Path("log/batch_results.json") +BATCH_LOCK_FILE = Path("log/batch_files.lock") +BATCH_LOCK = threading.RLock() +# API limits are 100,000 requests / 256 MB per batch; stay safely under both. +BATCH_MAX_REQUESTS = 100_000 +BATCH_MAX_BYTES = 200 * 1024 * 1024 + +_batch_phase = None +_batch_results = None # in-memory copy of BATCH_RESULTS_FILE (read-only during consume) +_batch_queue_pending = {} # process-local queue entries not yet flushed to disk + + +def set_batch_phase(phase): + """Set the batch phase ('collect', 'consume' or None) for this process and + any subprocesses it spawns (the GUI runs one per file).""" + global _batch_phase, _batch_results + _batch_phase = phase if phase in ("collect", "consume") else None + if _batch_phase: + os.environ["BATCH_PHASE"] = _batch_phase + else: + os.environ.pop("BATCH_PHASE", None) + _batch_results = None # phase change invalidates the in-memory results copy + + +def get_batch_phase(): + """Current batch phase, or None when batch translation is off.""" + if _batch_phase: + return _batch_phase + phase = os.getenv("BATCH_PHASE", "").strip().lower() + return phase if phase in ("collect", "consume") else None + + +@contextmanager +def _batch_file_lock(): + """Cross-process lock for the batch queue/state/results files.""" + BATCH_LOCK_FILE.parent.mkdir(parents=True, exist_ok=True) + with open(BATCH_LOCK_FILE, "a+b") as lock_file: + if os.name == "nt": + import msvcrt + lock_file.seek(0) + msvcrt.locking(lock_file.fileno(), msvcrt.LK_LOCK, 1) + try: + yield + finally: + lock_file.seek(0) + msvcrt.locking(lock_file.fileno(), msvcrt.LK_UNLCK, 1) + else: + import fcntl + fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX) + try: + yield + finally: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN) + + +def _read_batch_file(path): + """Read a batch JSON file; return an empty dict if it is unavailable.""" + try: + if path.exists(): + with open(path, "r", encoding="utf-8") as f: + data = json.load(f) + return data if isinstance(data, dict) else {} + except Exception: + pass + return {} + + +def _write_batch_file(path, data): + """Atomically write a batch JSON file (no indent — queues can be large).""" + path.parent.mkdir(parents=True, exist_ok=True) + tmp_file = path.with_name(f"{path.name}.{os.getpid()}.{threading.get_ident()}.tmp") + with open(tmp_file, "w", encoding="utf-8") as f: + json.dump(data, f, ensure_ascii=False) + tmp_file.replace(path) + + +def peek_cached_translation(payload, language): + """Cache lookup that never blocks or writes a pending marker. + + The collect pass uses this instead of get_cached_translation so abandoned + pending markers can't stall the consume pass for CACHE_PENDING_TTL.""" + key = get_cache_key(payload, language) + with CACHE_LOCK: + with _translation_cache_file_lock(): + cache = _read_cache_from_disk() + if _cache: + cache = _merge_translation_caches(cache, _cache) + entry = cache.get(key) + if entry is None or _is_pending_cache_entry(entry): + return None + return entry + + +def queue_batch_request(payload, language, params): + """Queue one Batches API request during the collect pass. + + Deduped by the same key the translation cache uses, so identical payloads + across files are only paid for once. Entries are buffered in memory and + merged to disk by flush_batch_queue() at the end of each translateAI call. + """ + key = get_cache_key(payload, language) + with BATCH_LOCK: + _batch_queue_pending[key] = {"payload": payload, "language": language, "params": params} + return key + + +def flush_batch_queue(): + """Merge this process's pending queue entries into the on-disk queue.""" + global _batch_queue_pending + with BATCH_LOCK: + if not _batch_queue_pending: + return + pending, _batch_queue_pending = _batch_queue_pending, {} + try: + with _batch_file_lock(): + queue = _read_batch_file(BATCH_QUEUE_FILE) + for key, entry in pending.items(): + queue.setdefault(key, entry) + _write_batch_file(BATCH_QUEUE_FILE, queue) + except Exception: + _batch_queue_pending.update(pending) # keep entries for the next flush + + +def take_batch_result(payload, language): + """Return the fetched batch response dict for a payload, or None. + + The results file is loaded once per process — it is written before the + consume pass starts and never changes mid-consume.""" + global _batch_results + if _batch_results is None: + with BATCH_LOCK: + if _batch_results is None: + with _batch_file_lock(): + _batch_results = _read_batch_file(BATCH_RESULTS_FILE) + return _batch_results.get(get_cache_key(payload, language)) + + +def pendingBatchRequests(): + """Number of queued batch requests (call after the collect pass).""" + flush_batch_queue() + with _batch_file_lock(): + return len(_read_batch_file(BATCH_QUEUE_FILE)) + + +def batchRunState(): + """'submitted' when a batch is still in flight, 'fetched' when results are + waiting to be consumed, else None. Lets an interrupted batch run resume + instead of re-collecting and paying for a second submission.""" + with _batch_file_lock(): + if _read_batch_file(BATCH_STATE_FILE).get("batches"): + return "submitted" + if _read_batch_file(BATCH_RESULTS_FILE): + return "fetched" + return None + + +def clearBatchFiles(): + """Remove queue/state/results left over from any previous batch run.""" + global _batch_results, _batch_queue_pending + with BATCH_LOCK: + _batch_results = None + _batch_queue_pending = {} + with _batch_file_lock(): + for path in (BATCH_QUEUE_FILE, BATCH_STATE_FILE, BATCH_RESULTS_FILE): + try: + if path.exists(): + path.unlink() + except Exception: + pass + + +def _get_anthropic_client(): + key = os.getenv("key", "").strip() + if not key: + raise Exception("Batch translation requires the 'key' env var (see .env).") + return anthropic.Anthropic(api_key=key) + + +def estimateBatchCost(model=None): + """Print a cost estimate for the queued batch requests and return it. + + Cached-prefix accounting mirrors what Anthropic bills: each distinct cached + prefix is written once (2x input rate at the 1h TTL) and read by every other + request that shares it (0.10x); everything is then halved by the batch + discount. Cache hits inside a batch are best-effort, so the no-cache batch + figure is the worst-case bound. + """ + flush_batch_queue() + with _batch_file_lock(): + queue = _read_batch_file(BATCH_QUEUE_FILE) + if not queue: + print("[BATCH] No batch requests queued.", flush=True) + return None + + enc = tiktoken.encoding_for_model("gpt-4") + prefix_count = {} # cached prefix text -> how many requests reuse it + prefix_tokens = {} # cached prefix text -> token count + dynamic_tokens = 0 + output_tokens = 0 + models = set() + for entry in queue.values(): + params = entry.get("params", {}) + if params.get("model"): + models.add(params["model"]) + blocks = params.get("system") or [] + cut = 0 # split system blocks at the cache breakpoint (0 = nothing cached) + for i, b in enumerate(blocks): + if "cache_control" in b: + cut = i + 1 + break + prefix = "".join(b.get("text", "") for b in blocks[:cut]) + dyn = "".join(b.get("text", "") for b in blocks[cut:]) + if prefix: + prefix_count[prefix] = prefix_count.get(prefix, 0) + 1 + if prefix not in prefix_tokens: + prefix_tokens[prefix] = len(enc.encode(prefix)) + msg_text = "\n".join(str(m.get("content", "")) for m in params.get("messages", [])) + dynamic_tokens += len(enc.encode(dyn)) + len(enc.encode(msg_text)) + 8 + # Output heuristic mirrors countTokens(): payload tokens x 2.5 covers + # the echoed JSON scaffold plus EN expansion. + output_tokens += round(len(enc.encode(str(entry.get("payload", "")))) * 2.5) + + est_model = model or next(iter(models), None) or os.getenv("model", "") + # Use Anthropic's count_tokens for the exact cached-prefix size when possible. + if prefix_tokens: + try: + client = _get_anthropic_client() + for prefix in list(prefix_tokens.keys()): + resp = client.messages.count_tokens( + model=est_model, + system=[{"type": "text", "text": prefix}], + messages=[{"role": "user", "content": "x"}], + ) + prefix_tokens[prefix] = resp.input_tokens + except Exception: + pass # tiktoken estimate already in place + + cache_write_tok = sum(prefix_tokens.values()) + cache_read_tok = sum(prefix_tokens[p] * (prefix_count[p] - 1) for p in prefix_count) + raw_input_tok = sum(prefix_tokens[p] * prefix_count[p] for p in prefix_count) + dynamic_tokens + + pricing = getPricingConfig(est_model) + in_rate = pricing["inputAPICost"] / 1_000_000 + out_rate = pricing["outputAPICost"] / 1_000_000 + + # Batch = 50% off every token, including cache writes (2x, 1h TTL) and reads (0.10x). + batch_cached = (cache_write_tok * in_rate * 2.00 + + cache_read_tok * in_rate * 0.10 + + dynamic_tokens * in_rate + + output_tokens * out_rate) * 0.50 + batch_nocache = (raw_input_tok * in_rate + output_tokens * out_rate) * 0.50 + live_cost = raw_input_tok * in_rate + output_tokens * out_rate + + n_reread = sum(prefix_count.values()) - len(prefix_count) + print(f"[BATCH] {len(queue)} requests queued for {est_model}", flush=True) + print(f"[BATCH] cached prefix: {cache_write_tok:,} tokens (written once, re-read by {n_reread:,} requests)", flush=True) + print(f"[BATCH] dynamic input: {dynamic_tokens:,} tokens | estimated output: {output_tokens:,} tokens", flush=True) + print(f"[BATCH] estimated cost: ${batch_cached:.2f} (batch + prompt cache)", flush=True) + print(f"[BATCH] ${batch_nocache:.2f} (batch, worst case no cache hits)", flush=True) + print(f"[BATCH] ${live_cost:.2f} (live API, no batch discount)", flush=True) + return { + "requests": len(queue), + "model": est_model, + "cache_write_tokens": cache_write_tok, + "cache_read_tokens": cache_read_tok, + "dynamic_tokens": dynamic_tokens, + "output_tokens": output_tokens, + "batch_cached_cost": batch_cached, + "batch_nocache_cost": batch_nocache, + "live_cost": live_cost, + } + + +def submitTranslationBatches(): + """Submit the queued requests to the Anthropic Message Batches API. + + Splits at the API limits and saves the custom_id -> cache-key mapping so + fetchTranslationBatches can route results back. Returns the batch ids.""" + flush_batch_queue() + with _batch_file_lock(): + queue = _read_batch_file(BATCH_QUEUE_FILE) + if not queue: + print("[BATCH] No batch requests queued.", flush=True) + return [] + + client = _get_anthropic_client() + batches = [] + requests, id_map, size = [], {}, 0 + + def _submit(): + nonlocal requests, id_map, size + if not requests: + return + batch = client.messages.batches.create(requests=requests) + batches.append({"id": batch.id, "custom_ids": id_map}) + print(f"[BATCH] submitted {batch.id} ({len(requests)} requests)", flush=True) + requests, id_map, size = [], {}, 0 + + for i, (key, entry) in enumerate(queue.items()): + custom_id = f"req-{i:06d}" + params = entry["params"] + requests.append({"custom_id": custom_id, "params": params}) + id_map[custom_id] = key + size += len(json.dumps(params, ensure_ascii=False).encode("utf-8")) + if len(requests) >= BATCH_MAX_REQUESTS or size >= BATCH_MAX_BYTES: + _submit() + _submit() + + with BATCH_LOCK: + with _batch_file_lock(): + _write_batch_file(BATCH_STATE_FILE, {"batches": batches}) + return [b["id"] for b in batches] + + +def checkTranslationBatches(): + """Print the processing status of submitted batches. True when all ended.""" + with _batch_file_lock(): + state = _read_batch_file(BATCH_STATE_FILE) + if not state.get("batches"): + print("[BATCH] No submitted batches — submit the queue first.", flush=True) + return False + client = _get_anthropic_client() + all_ended = True + for info in state["batches"]: + b = client.messages.batches.retrieve(info["id"]) + counts = getattr(b, "request_counts", None) + suffix = f" counts: {counts}" if counts else "" + print(f"[BATCH] {time.strftime('%H:%M:%S')} {b.id}: {b.processing_status}{suffix}", flush=True) + if b.processing_status != "ended": + all_ended = False + return all_ended + + +def fetchTranslationBatches(): + """Download finished batch results into the local results store. + + Successes are stored keyed by the payload cache key for the consume pass; + errored/expired requests are reported and simply fall back to the live API + during consume. Returns (succeeded, errored) counts.""" + global _batch_results + with _batch_file_lock(): + state = _read_batch_file(BATCH_STATE_FILE) + if not state.get("batches"): + print("[BATCH] No submitted batches — nothing to fetch.", flush=True) + return 0, 0 + client = _get_anthropic_client() + results, errored = {}, [] + for info in state["batches"]: + id_map = info.get("custom_ids", {}) + for r in client.messages.batches.results(info["id"]): + key = id_map.get(r.custom_id) + if key is None: + continue + res = r.result + if res.type != "succeeded": + detail = res.type + err = getattr(res, "error", None) + if err is not None: + inner = getattr(err, "error", err) + detail = f"{res.type} | {getattr(inner, 'type', '')}: {str(getattr(inner, 'message', '') or err)[:200]}" + errored.append((r.custom_id, detail)) + continue + msg = res.message + text = "".join(getattr(b, "text", "") or "" for b in msg.content) + u = msg.usage + cr = getattr(u, "cache_read_input_tokens", 0) or 0 + cw = getattr(u, "cache_creation_input_tokens", 0) or 0 + inp = getattr(u, "input_tokens", 0) or 0 + out = getattr(u, "output_tokens", 0) or 0 + results[key] = { + "text": text, + # prompt_tokens matches _AnthropicCompat: total incl. cache fields. + "prompt_tokens": inp + cr + cw, + "completion_tokens": out, + "cache_read_input_tokens": cr, + "cache_creation_input_tokens": cw, + } + with BATCH_LOCK: + with _batch_file_lock(): + merged = _read_batch_file(BATCH_RESULTS_FILE) + merged.update(results) + _write_batch_file(BATCH_RESULTS_FILE, merged) + # Queue and state are consumed; only the results store remains. + for path in (BATCH_QUEUE_FILE, BATCH_STATE_FILE): + try: + if path.exists(): + path.unlink() + except Exception: + pass + _batch_results = None + print(f"[BATCH] fetched {len(results)} results ({len(errored)} errored).", flush=True) + for cid, why in errored[:20]: + print(f"[BATCH] ! {cid}: {why}", flush=True) + if len(errored) > 20: + print(f"[BATCH] ... ({len(errored) - 20} more)", flush=True) + return len(results), len(errored) + + +def runTranslationBatches(poll=60): + """Submit the queue (unless already submitted), poll to completion, fetch.""" + with _batch_file_lock(): + state = _read_batch_file(BATCH_STATE_FILE) + if not state.get("batches"): + if not submitTranslationBatches(): + return 0, 0 + print(f"[BATCH] polling every {poll}s (Ctrl-C is safe — resume later with fetchTranslationBatches)...", flush=True) + while not checkTranslationBatches(): + time.sleep(poll) + return fetchTranslationBatches() + + +# ===== Mistral (la Plateforme) adaptive rate limiting ===== +# Mistral enforces independent request and token limits, both PER-MODEL. +# Verified live against api.mistral.ai (the response headers are authoritative): +# * x-ratelimit-limit-req-minute — requests/minute. PER-MODEL and varies a +# lot: mistral-medium=25, mistral-small=50, codestral=125, ministral-3b=750. +# Mistral's dashboard shows this /60 as "RPS" (25/min = 0.42 "RPS"), so the +# effective rate spans well below AND above 1 req/sec. There is NO +# per-second header; we divide req-minute by 60 to get the pacing interval. +# * x-ratelimit-limit-tokens-minute — input+output throughput, a true minute +# window, also per-model (e.g. 50k–1.3M). +# * Tokens per month — overall cap (not paced here; surfaces as a 429). +# The limiter paces requests >= 1/RPS apart (so a burst of file threads can't +# overrun the per-minute request budget) AND charges them against a +# minute-windowed token budget. It starts from a conservative seed, then the +# live headers on the first response pin req_per_sec / tok_per_min to the exact +# per-model values. One limiter is shared across all file threads (this tool +# runs one model per session). Override the seeds with +# mistralReqPerSec / mistralTokPerMin / mistralTokenHeadroom. +_mistral_limiter = None +_mistral_limiter_lock = threading.Lock() + + +def _estimate_mistral_tokens(text): + # JP ~1 token/char with Mistral tokenizers; EN ~1 per 3.5 chars. Be pessimistic. + return int(len(str(text)) * 1.1) + 8 + + +def _header_int(headers, *names): + """First present header among names parsed as int, else None.""" + for n in names: + v = headers.get(n) + if v is not None and str(v).strip() != "": + try: + return int(float(v)) + except (TypeError, ValueError): + pass + return None + + +def _header_float(headers, *names): + """First present header among names parsed as float, else None. + + Used for the RPS limit header, which per-model is often FRACTIONAL + (e.g. 0.08, 0.42, 0.83) — parsing it as int would floor those to 0.""" + for n in names: + v = headers.get(n) + if v is not None and str(v).strip() != "": + try: + return float(v) + except (TypeError, ValueError): + pass + return None + + +class AdaptiveLimiter: + """Paces requests off live ratelimit headers. + + Two dimensions, enforced together by acquire(): + * requests: spaced >= min_interval (1/RPS) apart — Mistral's request cap + is per-second, so this prevents bursts from tripping it. + * tokens: a minute-windowed budget (input+output), with a headroom margin. + """ + + def __init__(self, req_per_sec, tok_per_min, headroom): + self.lock = threading.Lock() + self.req_per_sec = max(0.001, float(req_per_sec)) + self.tok_per_min = tok_per_min + self.headroom = headroom # margin left under the TPM cap + self.min_interval = 1.0 / self.req_per_sec + self.next_request_at = time.monotonic() + self.tokens_remaining = tok_per_min + self.window_reset = time.monotonic() + 60 + + def acquire(self, est_tokens): + """Block until the request clears both the RPS pace and the TPM budget.""" + while True: + with self.lock: + now = time.monotonic() + if now >= self.window_reset: + self.tokens_remaining = self.tok_per_min + self.window_reset = now + 60 + # token gate + if self.tokens_remaining - est_tokens <= self.headroom: + sleep_for = max(0.05, self.window_reset - now) + # request-pace gate + elif now < self.next_request_at: + sleep_for = self.next_request_at - now + else: + # both gates clear — reserve this slot + self.next_request_at = max(now, self.next_request_at) + self.min_interval + self.tokens_remaining -= est_tokens + return + time.sleep(min(sleep_for, 5)) + + def update(self, headers): + """Sync budgets from the live ratelimit headers (authoritative). + + Accepts both the per-second request headers and (for forward/back compat) + the older per-minute request header names; the token headers are minute.""" + if not headers: + return + with self.lock: + # The authoritative request limit is x-ratelimit-limit-req-MINUTE + # (verified live: mistral-medium=25, ministral-3b=750, codestral=125; + # Mistral's dashboard "RPS" is just this number / 60). There is no + # per-second header. Convert to RPS for the pacing interval. A + # per-second header is still accepted first in case Mistral adds one. + limit_rps = _header_float(headers, "x-ratelimit-limit-req-second", + "x-ratelimit-limit-requests-second") + if limit_rps is None: + limit_rpm = _header_float(headers, "x-ratelimit-limit-req-minute", + "x-ratelimit-limit-requests-minute") + if limit_rpm is not None: + limit_rps = limit_rpm / 60.0 + if limit_rps and limit_rps > 0: + self.req_per_sec = limit_rps + self.min_interval = 1.0 / self.req_per_sec + limit_tok = _header_int(headers, "x-ratelimit-limit-tokens-minute", + "x-ratelimit-limit-tokens") + if limit_tok and limit_tok > 0: + self.tok_per_min = limit_tok + rem_tok = _header_int(headers, "x-ratelimit-remaining-tokens-minute", + "x-ratelimit-remaining-tokens") + if rem_tok is not None: + self.tokens_remaining = rem_tok + # Remaining requests this minute -> also cap the pace so a fresh + # limiter that joins mid-window doesn't burst the leftover budget. + rem_req = _header_int(headers, "x-ratelimit-remaining-req-minute", + "x-ratelimit-remaining-requests-minute") + if rem_req is not None and rem_req <= 0: + # budget already spent this minute — hold off ~ a minute + self.next_request_at = max(self.next_request_at, time.monotonic() + 60.0) + + +def _get_mistral_limiter(): + global _mistral_limiter + with _mistral_limiter_lock: + if _mistral_limiter is None: + # mistralReqPerMin kept as a deprecated alias: a per-minute number is + # converted to RPS so old .env files keep pacing sanely. + rps_env = os.getenv("mistralReqPerSec") + if rps_env: + req_per_sec = float(rps_env) + elif os.getenv("mistralReqPerMin"): + req_per_sec = max(0.05, float(os.getenv("mistralReqPerMin")) / 60.0) + else: + # Per-model RPS varies and is often well under 1 (free-tier + # mistral-medium ~0.83, magistral ~0.08). Start conservative so + # the FIRST request doesn't over-burst; the live header from that + # first response then corrects req_per_sec to the model's exact + # value (which is why few large batches > many tiny ones). + req_per_sec = 0.5 + _mistral_limiter = AdaptiveLimiter( + req_per_sec, + int(os.getenv("mistralTokPerMin", "50000") or 50000), + int(os.getenv("mistralTokenHeadroom", "4000") or 4000), + ) + return _mistral_limiter + + +def callMistral(params, retries=6): + """Call the Mistral chat completions endpoint with adaptive pacing. + + Acquires from the shared limiter before each attempt, syncs budgets from + the live x-ratelimit headers after each response, honours Retry-After on + 429, backs off with jitter on 5xx/network errors, and downgrades + json_schema -> json_object for models without structured-output support. + """ + limiter = _get_mistral_limiter() + est = sum(_estimate_mistral_tokens(m.get("content", "")) for m in params.get("messages", [])) + est += params.get("max_tokens", 0) // 2 + last_error = None + for attempt in range(retries + 1): + limiter.acquire(est) + try: + raw = openai.chat.completions.with_raw_response.create(**params) + response = raw.parse() + limiter.update(raw.headers) + return response + except RateLimitError as e: + last_error = e + resp = getattr(e, "response", None) + limiter.update(getattr(resp, "headers", None)) + retry_after = None + try: + retry_after = float(resp.headers.get("retry-after")) + except (AttributeError, TypeError, ValueError): + pass + time.sleep(retry_after if retry_after is not None else min(60, 2 ** attempt + random.random() * 2)) + except APIStatusError as e: + last_error = e + # Mistral rejects unsupported params with 400/422 — downgrade the + # structured-output format once and retry immediately. + if e.status_code in (400, 422) and (params.get("response_format") or {}).get("type") == "json_schema": + params = dict(params) + params["response_format"] = {"type": "json_object"} + continue + if e.status_code >= 500 and attempt < retries: + time.sleep(min(45, 2 ** attempt + random.random())) + continue + raise Exception(f"Mistral API error ({e.status_code}): {e}") + except APIConnectionError as e: + last_error = e + if attempt < retries: + time.sleep(min(45, 2 ** attempt + random.random())) + continue + raise Exception(f"Mistral connection error: {e}") + raise Exception(f"Mistral API failed after {retries + 1} attempts: {last_error}") + + +class TranslationConfig: + """Configuration class to hold all translation settings""" + + def __init__(self, + model=None, + language=None, + prompt=None, + vocab=None, + langRegex=None, + batchSize=None, + maxHistory=10, + estimateMode=False, + logFilePath="log/translationHistory.txt", + mismatchLogPath="log/mismatchHistory.txt"): + + # Load from environment if not provided + self.model = model or os.getenv("model") + self.language = (language or os.getenv("language", "english")).capitalize() + + # Load prompt and vocab files if not provided + if prompt is None: + try: + from util.paths import PROMPT_PATH, VOCAB_PATH + self.prompt = PROMPT_PATH.read_text(encoding="utf-8") + except FileNotFoundError: + self.prompt = "" + else: + self.prompt = prompt + + if vocab is None: + try: + self.vocab = VOCAB_PATH.read_text(encoding="utf-8") + except FileNotFoundError: + self.vocab = "" + else: + self.vocab = vocab + + # Set language regex (default is Japanese) + self.langRegex = langRegex or r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+" + + # Set batch size — derive from pricing config unless explicitly supplied + if batchSize is None: + self.batchSize = getPricingConfig(self.model)["batchSize"] + else: + self.batchSize = batchSize + + self.maxHistory = maxHistory + self.estimateMode = estimateMode + self.logFilePath = logFilePath + self.mismatchLogPath = mismatchLogPath + + +_LITELLM_PRICING_URL = ( + "https://raw.githubusercontent.com/BerriAI/litellm/main" + "/model_prices_and_context_window.json" +) +_PRICING_CACHE_FILE = Path("log/litellm_pricing.json") +_PRICING_CACHE_TTL = 86_400 # 24 hours +_pricing_db: dict | None = None +_pricing_db_fetched_at: float = 0.0 +_pricing_db_lock = threading.Lock() +_pricing_fetch_warned: bool = False # print fetch-failure warning at most once per session + + +def _load_litellm_pricing() -> dict | None: + """Return the LiteLLM pricing DB, using a 24-hour disk cache.""" + global _pricing_db, _pricing_db_fetched_at, _pricing_fetch_warned + + with _pricing_db_lock: + now = time.time() + + # In-memory cache still fresh + if _pricing_db is not None and (now - _pricing_db_fetched_at) < _PRICING_CACHE_TTL: + return _pricing_db + + # Try disk cache + if _PRICING_CACHE_FILE.exists(): + try: + disk = json.loads(_PRICING_CACHE_FILE.read_text(encoding="utf-8")) + if (now - disk.get("fetched_at", 0)) < _PRICING_CACHE_TTL: + _pricing_db = disk["prices"] + _pricing_db_fetched_at = disk["fetched_at"] + return _pricing_db + except Exception: + pass + + # Fetch from GitHub + try: + with urllib.request.urlopen(_LITELLM_PRICING_URL, timeout=5) as resp: + data = json.loads(resp.read().decode("utf-8")) + _pricing_db = data + _pricing_db_fetched_at = now + _pricing_fetch_warned = False # reset if a later fetch succeeds + try: + _PRICING_CACHE_FILE.parent.mkdir(parents=True, exist_ok=True) + _PRICING_CACHE_FILE.write_text( + json.dumps({"fetched_at": now, "prices": data}), + encoding="utf-8", + ) + except Exception: + pass + return _pricing_db + except Exception as fetch_err: + # No internet / GitHub unreachable — warn once, then fall back + if not _pricing_fetch_warned: + _pricing_fetch_warned = True + print( + f"[PRICING] Warning: Could not fetch live model pricing " + f"({fetch_err}). Cost estimates may be inaccurate — " + f"using built-in fallback prices.", + flush=True, + ) + # Use stale disk cache if available + if _pricing_db is not None: + return _pricing_db + try: + disk = json.loads(_PRICING_CACHE_FILE.read_text(encoding="utf-8")) + _pricing_db = disk["prices"] + return _pricing_db + except Exception: + return None + + +def _lookup_model_price(model: str): + """Look up (input_per_1M, output_per_1M) from the LiteLLM pricing DB. + + Returns a (float, float) tuple or None if not found. + Matching priority: + 1. Exact key match + 2. Exact match on the model portion after a provider prefix (e.g. "deepseek/deepseek-chat") + 3. The user's model name is a prefix of a DB key (handles dated suffixes like -20241022) + 4. A DB key model-part is a prefix of the user's model name + """ + db = _load_litellm_pricing() + if not db: + return None + + model_lower = model.lower() + + def _extract(entry): + inp = entry.get("input_cost_per_token") + out = entry.get("output_cost_per_token") + if inp is not None and out is not None: + return round(inp * 1_000_000, 6), round(out * 1_000_000, 6) + return None + + # Pass 1: exact key + if model_lower in db: + result = _extract(db[model_lower]) + if result: + return result + + # Build a lookup of (stripped_key → original_key) for passes 2-4 + stripped: list[tuple[str, str]] = [] + for key in db: + stripped.append((key.split("/")[-1].lower(), key)) + + # Pass 2: exact match on stripped key + for skey, orig in stripped: + if skey == model_lower: + result = _extract(db[orig]) + if result: + return result + + # Pass 3: model name is a prefix of the DB key (e.g. "claude-3-5-sonnet" matches + # "claude-3-5-sonnet-20241022") + candidates = [(skey, orig) for skey, orig in stripped if skey.startswith(model_lower)] + if candidates: + # Prefer the shortest (most generic) key + skey, orig = min(candidates, key=lambda x: len(x[0])) + result = _extract(db[orig]) + if result: + return result + + # Pass 4: DB key is a prefix of the model name (e.g. "gemini-2.0-flash" matches + # "gemini-2.0-flash-exp") + candidates = [(skey, orig) for skey, orig in stripped if model_lower.startswith(skey)] + if candidates: + skey, orig = max(candidates, key=lambda x: len(x[0])) # longest = most specific + result = _extract(db[orig]) + if result: + return result + + return None + + +def getPricingConfig(model): + """ + Get pricing configuration for a given model. + + Args: + model: The model name string + + Returns: + dict: Dictionary containing inputAPICost, outputAPICost, batchSize, and frequencyPenalty + """ + # Try to resolve pricing from the LiteLLM community pricing DB first. + # This keeps costs accurate as providers update their prices without requiring + # a code change. Falls back to the hardcoded table below on failure. + live_price = _lookup_model_price(model) + if live_price: + inp, out = live_price + # Preserve model-specific batch / penalty overrides from the hardcoded table + # by still running through the if-chain but replacing the cost fields. + _live_override = {"inputAPICost": inp, "outputAPICost": out} + else: + _live_override = None + + # Hardcoded fallback table — used for batchSize / frequencyPenalty tuning and + # as a cost fallback when the LiteLLM DB is unavailable. + # Batch Size: GPT-3.5 struggles past 15 lines; GPT-4 struggles past 50. + # If you get a MISMATCH LENGTH error, lower the batch size. + if "gpt-3.5" in model: + cfg = {"inputAPICost": 3.00, "outputAPICost": 5.00, "batchSize": 10, "frequencyPenalty": 0.2} + elif "gpt-4.1-mini" in model: + cfg = {"inputAPICost": 0.40, "outputAPICost": 1.60, "batchSize": 30, "frequencyPenalty": 0.05} + elif "gpt-4.1" in model: + cfg = {"inputAPICost": 2.00, "outputAPICost": 8.00, "batchSize": 30, "frequencyPenalty": 0.05} + elif "gpt-5" in model: + cfg = {"inputAPICost": 1.25, "outputAPICost": 10.00, "batchSize": 30, "frequencyPenalty": 0.05} + elif "deepseek" in model: + cfg = {"inputAPICost": 0.27, "outputAPICost": 1.10, "batchSize": 30, "frequencyPenalty": 0.05} + # Mistral — system prompt is resent per request (no prompt caching), so + # throughput/cost favor larger batches. Live LiteLLM pricing overrides these. + elif "mistral-large" in model or "pixtral-large" in model: + cfg = {"inputAPICost": 2.00, "outputAPICost": 6.00, "batchSize": 40, "frequencyPenalty": 0.0} + elif "magistral-medium" in model: + cfg = {"inputAPICost": 2.00, "outputAPICost": 5.00, "batchSize": 40, "frequencyPenalty": 0.0} + elif "mistral-medium-3.5" in model or "mistral-medium-3-5" in model or "mistral-medium-26" in model: + # Medium 3.5 (v26.04) — also matches dated 26xx ids + cfg = {"inputAPICost": 1.50, "outputAPICost": 7.50, "batchSize": 40, "frequencyPenalty": 0.0} + elif "mistral-medium" in model: + # Medium 3 / 3.1 (what -latest still points at) + cfg = {"inputAPICost": 0.40, "outputAPICost": 2.00, "batchSize": 40, "frequencyPenalty": 0.0} + elif "magistral-small" in model: + cfg = {"inputAPICost": 0.50, "outputAPICost": 1.50, "batchSize": 40, "frequencyPenalty": 0.0} + elif "mistral-small" in model: + cfg = {"inputAPICost": 0.10, "outputAPICost": 0.30, "batchSize": 40, "frequencyPenalty": 0.0} + elif "ministral" in model or "open-mistral" in model or "mistral-nemo" in model: + cfg = {"inputAPICost": 0.10, "outputAPICost": 0.10, "batchSize": 40, "frequencyPenalty": 0.0} + elif "codestral" in model: + cfg = {"inputAPICost": 0.30, "outputAPICost": 0.90, "batchSize": 40, "frequencyPenalty": 0.0} + elif "mistral" in model or "pixtral" in model: + cfg = {"inputAPICost": 2.00, "outputAPICost": 6.00, "batchSize": 40, "frequencyPenalty": 0.0} + elif "claude-opus-4-5" in model or "claude-opus-4-6" in model: + cfg = {"inputAPICost": 5.00, "outputAPICost": 25.00, "batchSize": 30, "frequencyPenalty": 0.05} + elif "claude-opus" in model or model == "claude-3-opus": + # Opus 4, 4.1, 3 — $15/$75 + cfg = {"inputAPICost": 15.00, "outputAPICost": 75.00, "batchSize": 30, "frequencyPenalty": 0.05} + elif "claude-haiku-4-5" in model or "claude-haiku-4-6" in model: + cfg = {"inputAPICost": 1.00, "outputAPICost": 5.00, "batchSize": 30, "frequencyPenalty": 0.05} + elif "claude-haiku-3-5" in model: + cfg = {"inputAPICost": 0.80, "outputAPICost": 4.00, "batchSize": 30, "frequencyPenalty": 0.05} + elif "claude-3-haiku" in model: + cfg = {"inputAPICost": 0.25, "outputAPICost": 1.25, "batchSize": 30, "frequencyPenalty": 0.05} + elif "haiku" in model: + # Unknown haiku version — use current flagship pricing as best guess + cfg = {"inputAPICost": 1.00, "outputAPICost": 5.00, "batchSize": 30, "frequencyPenalty": 0.05} + elif "sonnet" in model or "claude" in model: + cfg = {"inputAPICost": 3.00, "outputAPICost": 15.00, "batchSize": 30, "frequencyPenalty": 0.05} + elif "gemini-2.0-flash-lite" in model: + cfg = {"inputAPICost": 0.075, "outputAPICost": 0.30, "batchSize": 30, "frequencyPenalty": 0.0} + elif "gemini-2.0-flash" in model: + cfg = {"inputAPICost": 0.10, "outputAPICost": 0.40, "batchSize": 30, "frequencyPenalty": 0.0} + elif "gemini-2.5-flash-lite" in model: + cfg = {"inputAPICost": 0.10, "outputAPICost": 0.40, "batchSize": 30, "frequencyPenalty": 0.0} + elif "gemini-2.5-flash" in model: + cfg = {"inputAPICost": 0.30, "outputAPICost": 2.50, "batchSize": 30, "frequencyPenalty": 0.0} + elif "gemini-2.5-pro" in model: + cfg = {"inputAPICost": 1.25, "outputAPICost": 10.00, "batchSize": 30, "frequencyPenalty": 0.0} + else: + cfg = { + "inputAPICost": float(os.getenv("input_cost", 3.00)), + "outputAPICost": float(os.getenv("output_cost", 6.00)), + "batchSize": int(os.getenv("batchsize", 10)), + "frequencyPenalty": float(os.getenv("frequency_penalty", 0.2)), + } + + # Apply live pricing from LiteLLM if available — keeps costs up-to-date + # without requiring code changes when providers reprice their models. + if _live_override: + cfg.update(_live_override) + + return cfg + + +def batchList(inputList, batchSize): + """Split a list into batches of specified size""" + if not isinstance(batchSize, int) or batchSize <= 0: + raise ValueError("batchSize must be a positive integer") + + return [inputList[i : i + batchSize] for i in range(0, len(inputList), batchSize)] + + +def parseVocabWithCategories(vocabText): + """Parse vocabulary text and extract terms with their categories.""" + pairs = [] + seen = set() + currentCategory = None + + for line in vocabText.splitlines(): + line = line.strip() + if not line or line.startswith('```') or line.startswith('Here are some vocabulary'): + continue + + # Check if this is a category header + if line.startswith('#'): + currentCategory = line + continue + + # Parse vocabulary term - extract both Japanese and English parts. + # Rich entries may continue after the first parenthesized translation, + # e.g. "サンク (Sank) - Male; protagonist..."; only "Sank" is the match key. + paren_match = re.match(r'^(.+?)\s*\(([^()]*)\)', line) + dash_match = re.match(r'^(.+?)\s+[–-]\s+(.+)$', line) + if paren_match: + japanese_term = paren_match.group(1).strip() + english_term = paren_match.group(2).strip() + + # Create a tuple with both terms for matching + term_pair = (japanese_term, english_term) + if term_pair not in seen: + pairs.append((term_pair, line, currentCategory)) + seen.add(term_pair) + elif dash_match: + japanese_term = dash_match.group(1).strip() + english_term = dash_match.group(2).strip() + + # Create a tuple with both terms for matching + term_pair = (japanese_term, english_term) + if term_pair not in seen: + pairs.append((term_pair, line, currentCategory)) + seen.add(term_pair) + elif line and not line.startswith('#'): + # Fallback for lines without parentheses - treat as single term + term = line.strip() + if term and term not in seen: + pairs.append((term, line, currentCategory)) + seen.add(term) + + return pairs + + +def _japanese_term_in_text(term, text): + """ + Check if a Japanese term appears in text as a standalone word, not as a + substring of a longer run of the same script (katakana/hiragana/kanji). + E.g. 'キス' will NOT match inside 'テキスト' because both neighbours are katakana. + Falls back to plain substring check for non-Japanese or mixed terms. + """ + if term not in text: + return False + KATAKANA = r'ァ-ヴーｦ-ﾟ' + HIRAGANA = r'ぁ-ゔ' + KANJI = r'一-龠' + if re.search(rf'[{KATAKANA}]', term) and not re.search(rf'[{HIRAGANA}{KANJI}]', term): + pattern = rf'(?= 500: + raise Exception(f"API server error ({e.status_code}) - retrying... Error: {e}") + elif e.status_code == 400 and formatType == "json" and "json_schema" in str(responseFormat): + # Only fall back to json_object if the error is NOT "Input should be 'json_schema'" + # (that message means json_schema IS required and json_object would also be rejected) + if "input should be 'json_schema'" in str(e).lower() or "input should be \"json_schema\"" in str(e).lower(): + raise Exception(f"API status error ({e.status_code}): {e}") + # Provider doesn't support json_schema (e.g. Claude) — fall back to json_object + responseFormat = {"type": "json_object"} + params["response_format"] = responseFormat + try: + response = openai.chat.completions.create(**params) + except APIStatusError as fallback_error: + if fallback_error.status_code == 400 and "input should be 'json_schema'" in str(fallback_error).lower(): + raise Exception(f"API requires json_schema response format but rejected the schema. Original error: {e}") + raise Exception(f"API call failed: {e}. Fallback also failed: {fallback_error}") + except Exception as fallback_error: + raise Exception(f"API call failed: {e}. Fallback also failed: {fallback_error}") + elif e.status_code == 400 and "input should be 'json_schema'" in str(e).lower(): + # response_format.type was rejected (e.g. sent "text" or "json_object" to a model + # that only accepts json_schema). Remove response_format and retry with no constraint. + params.pop("response_format", None) + try: + response = openai.chat.completions.create(**params) + except Exception as fallback_error: + raise Exception(f"API call failed: {e}. Fallback also failed: {fallback_error}") + else: + raise Exception(f"API status error ({e.status_code}): {e}") + except (APIConnectionError, RateLimitError) as e: + # These should always be retried + raise Exception(f"API connection/rate limit error - retrying... Error: {e}") + except Exception as e: + # Check if it's a 404 error or other HTTP error that should be retried + error_str = str(e).lower() + if "404" in error_str or "not found" in error_str: + raise Exception(f"API returned 404 Not Found - check your API configuration. Original error: {e}") + + # If structured output fails, fallback to json_object (unless the error + # explicitly states json_schema is required — falling back would just fail again) + if formatType == "json" and "json_schema" in str(responseFormat) and \ + "input should be 'json_schema'" not in error_str: + responseFormat = {"type": "json_object"} + params["response_format"] = responseFormat + try: + response = openai.chat.completions.create(**params) + except Exception as fallback_error: + # If fallback also fails, raise the original error for retry + raise Exception(f"API call failed: {e}. Fallback also failed: {fallback_error}") + else: + raise e + + # Validate response before returning + if not response or not hasattr(response, 'choices') or not response.choices: + raise Exception("API returned invalid or empty response - retrying...") + + _write_request_debug_log(api_provider, params, getattr(response, "usage", None)) + return response + + +def cleanTranslatedText(translatedText, language): + """Clean and format translated text""" + placeholders = { + f"{language} Translation: ": "", + "Translation: ": "", + "っ": "", + "〜": "~", + "ッ": "", + "。": ".", + # Note: 「 and 」 are NOT replaced here — replacing them with ASCII " would + # corrupt raw JSON strings before extraction. They are handled per-line + # in _clean_extracted_line() after JSON parsing. + "—": "―", + "】": "]", + "【": "[", + "é": "e", + "’": "'", + "this guy": "this bastard", + "This guy": "This bastard", + "```json": "", + "```": "", + } + + for target, replacement in placeholders.items(): + translatedText = translatedText.replace(target, replacement) + + # Remove Repeating Characters + pattern = re.compile(r"(.)\s*\1(?:\s*\1){" + str(20 - 1) + r",}") + translatedText = pattern.sub(lambda match: match.group(0).replace(" ", "")[:20], translatedText) + + # Elongate Long Dashes (Since GPT Ignores them...) + translatedText = elongateCharacters(translatedText) + return translatedText + + +def elongateCharacters(text): + """Replace ー sequences with elongated characters""" + # Define a pattern to match one character followed by two or more ー characters. + # The lookbehind is restricted to non-ー Japanese/CJK characters so that: + # - standalone ー separators (e.g. "ーーーーーーーーーー") are left untouched + # - ー sequences preceded by a JSON quote or other non-Japanese char are not corrupted + pattern = r"(?<=([\u3040-\u309F\u30A0-\u30FB\u30FD-\u30FF\u4E00-\u9FEF\uFF61-\uFF9F]))ー{2,}" + + # Define a replacement function that elongates the captured character + def repl(match): + char = match.group(1) # The character before the ー sequence + count = len(match.group(0)) - 1 # Number of ー characters + return char * count # Replace ー sequence with the character repeated + + # Use re.sub() to replace the pattern in the text + return re.sub(pattern, repl, text) + + +def extractTranslation(translatedTextList, isList, pbar=None): + """Extract translation from JSON response. + + This function is resilient to a few common model mistakes: + - Wraps output in code fences or outer quotes + - Uses smart quotes instead of straight quotes + - Inserts an extra leading quote in values (e.g. :""Word" -> :"Word") + - Trailing commas before } or ] + + If strict JSON parsing fails, falls back to a regex-based extractor that + captures LineN values in numeric order. + """ + s = str(translatedTextList or "").strip() + + # Fast exit + if not s: + return None + + # Remove code fences if present + if s.startswith("```"): + s = re.sub(r"^```(?:json)?\s*", "", s, flags=re.IGNORECASE) + s = re.sub(r"\s*```$", "", s) + + # Trim wrapping quotes around the whole JSON blob (common in logs) + if len(s) >= 2 and s[0] == s[-1] and s[0] in {'"', "'"}: + # Only strip if it still looks like JSON inside + if s[1:2] == "{" and s[-2:-1] == "}": + s = s[1:-1] + + # Normalize a broad set of Unicode “smart” quotes to ASCII equivalents. + translation_table = { + 0x201C: "'", # “ left double quotation mark + 0x201D: "'", # ” right double quotation mark + 0xFF02: "'", # ＂ fullwidth quotation mark + + 0x2018: "'", # ‘ left single quotation mark + 0x2019: "'", # ’ right single quotation mark + 0x201B: "'", # ‛ single high-reversed-9 quotation mark + 0x02BC: "'", # ʼ modifier letter apostrophe + 0xFF07: "'", # ＇ fullwidth apostrophe + } + s = s.translate(translation_table) + + # Remove trailing commas before object/array closures + s = re.sub(r",(\s*[}\]])", r"\1", s) + + # Repair common doubled leading quote in values: :""Word" -> :"Word" + # Ensure we don't alter legitimate empty strings (:"") + s = re.sub(r":\s*\"\"(?=[^\",}\]\s])", r':"', s) + + # Attempt strict parse first + try: + lineDict = json.loads(s) + + # Handle array-based schema: {"translations": ["...", ...]} + if isinstance(lineDict, dict) and "translations" in lineDict and isinstance(lineDict["translations"], list): + stringList = [str(v) for v in lineDict["translations"]] + return stringList if isList else (stringList[0] if stringList else None) + + # Build list in numeric order if keys are LineN + numeric_keys = [] + for k in lineDict.keys(): + m = re.fullmatch(r"Line(\d+)", str(k)) + if m: + numeric_keys.append(int(m.group(1))) + + if numeric_keys: + stringList = [lineDict.get(f"Line{n}", "") for n in sorted(numeric_keys)] + else: + # Fallback to values order if no LineN keys found + stringList = list(lineDict.values()) + + return stringList if isList else (stringList[0] if stringList else None) + + except Exception as e: + # Fallback: regex-based extraction tolerant to one or two opening quotes + # Captures escaped quotes within values too + try: + pairs = re.findall(r'"Line(\d+)"\s*:\s*"{1,2}((?:\\.|[^"\\])*)"', s) + if not pairs: + raise ValueError("No LineN pairs found") + + # Sort numerically and unescape JSON string content + items = [] + for n_str, v in sorted(((int(n), v) for n, v in pairs), key=lambda x: x[0]): + try: + # Decode JSON escapes reliably by round-tripping as a JSON string + decoded = json.loads(f'"{v}"') + except Exception: + decoded = v + items.append(decoded) + + return items if isList else (items[0] if items else None) + except Exception as e2: + if pbar: + pbar.write(f"extractTranslation Error: {e2} after JSON error {e} on String {translatedTextList}") + return None + + +def calculateCost(inputTokens, outputTokens, model): + """ + Calculate the cost of translation based on token usage and model pricing. + + For Claude models the cost is derived from the actual cache token breakdown + recorded by translateAI, so cache discounts are reflected accurately: + - Cache reads: 10 % of the base input rate + - Cache writes: 125 % of the base input rate + - Regular input: 100 % of the base input rate + + Call pattern (no module changes required): + Per-file call: file_cost_ready flag is True → read thread-local per-file + accumulators (which span all translateAI calls for the file), + compute cost, reset accumulators, clear flag, return cost. + TOTAL call: file_cost_ready is False (already cleared) → return the + cross-thread _global_accurate_cost running sum. + + Falls back to naive token × rate calculation for non-Claude models. + """ + _is_claude = model and any(x in model.lower() for x in ("claude", "sonnet", "haiku", "opus")) + if _is_claude: + if getattr(_thread_local, 'file_cost_ready', False): + # Per-file call: compute from accumulators (may be 0 for disk-cached files), + # reset everything, return the file cost. + cr = getattr(_thread_local, 'file_cache_read', 0) + cw = getattr(_thread_local, 'file_cache_write', 0) + reg = getattr(_thread_local, 'file_regular', 0) + out = getattr(_thread_local, 'file_output', 0) + bcr = getattr(_thread_local, 'file_batch_read', 0) + bcw = getattr(_thread_local, 'file_batch_write', 0) + breg = getattr(_thread_local, 'file_batch_regular', 0) + bout = getattr(_thread_local, 'file_batch_output', 0) + pricing = getPricingConfig(model) + br = pricing["inputAPICost"] / 1_000_000 + orr = pricing["outputAPICost"] / 1_000_000 + cost = (cr * br * 0.10 + cw * br * 2.00 + reg * br + out * orr + # Batch API tokens: same rates, then the 50% batch discount. + + (bcr * br * 0.10 + bcw * br * 2.00 + breg * br + bout * orr) * 0.50) + _thread_local.file_cache_read = 0 + _thread_local.file_cache_write = 0 + _thread_local.file_regular = 0 + _thread_local.file_output = 0 + _thread_local.file_batch_read = 0 + _thread_local.file_batch_write = 0 + _thread_local.file_batch_regular = 0 + _thread_local.file_batch_output = 0 + _thread_local.file_cost_ready = False + return cost + # TOTAL call (flag already cleared): return the cross-thread running total. + # If _global_accurate_cost is 0 it means no real API calls were made + # (e.g. estimate mode), so fall through to the naive calculation below. + with _global_accurate_cost_lock: + accurate = _global_accurate_cost + if accurate > 0: + return accurate + + # Non-Claude, estimate mode, or no accurate data: naive calculation. + # For Claude models, use the accumulated static_system token count (the portion + # that is always cache-written at the 1hr TTL rate = 2x input rate). + # Remaining tokens are billed at the regular input rate. + pricing = getPricingConfig(model) + _is_claude_naive = model and any(x in model.lower() for x in ("claude", "sonnet", "haiku", "opus")) + if _is_claude_naive: + static_tok = getattr(_thread_local, 'estimate_static_tokens', 0) + regular_tok = getattr(_thread_local, 'estimate_regular_tokens', 0) + batch_count = max(1, getattr(_thread_local, 'estimate_batch_count', 1)) + _thread_local.estimate_static_tokens = 0 + _thread_local.estimate_regular_tokens = 0 + _thread_local.estimate_batch_count = 0 + # If cache is disabled, every batch is a write (2x) — no reads ever. + # Otherwise: each distinct batch size (= distinct output_config schema) gets exactly + # one cache write on first use; all subsequent batches of that size are reads (0.10x). + # Load from disk first so GUI subprocesses (one per file) share warm-cache state. + global _estimate_written_sizes + if DISABLE_CACHE: + write_batches = batch_count + read_batches = 0 + else: + _load_estimate_written_sizes() + seen_sizes = getattr(_thread_local, 'estimate_seen_sizes', set()) + new_sizes = seen_sizes - _estimate_written_sizes + write_batches = len(new_sizes) # one write per newly-seen size + read_batches = batch_count - write_batches + _estimate_written_sizes.update(new_sizes) + _save_estimate_written_sizes() + _thread_local.estimate_seen_sizes = set() + write_cost = (write_batches * static_tok / 1_000_000) * pricing["inputAPICost"] * 2.0 + read_cost = (read_batches * static_tok / 1_000_000) * pricing["inputAPICost"] * 0.10 + regular_cost = (regular_tok / 1_000_000) * pricing["inputAPICost"] + inputCost = write_cost + read_cost + regular_cost + else: + inputCost = (inputTokens / 1_000_000) * pricing["inputAPICost"] + outputCost = (outputTokens / 1_000_000) * pricing["outputAPICost"] + return inputCost + outputCost + + +def countTokens(system, user, history): + """Count tokens for cost estimation""" + inputTotalTokens = 0 + outputTotalTokens = 0 + enc = tiktoken.encoding_for_model("gpt-4") + + # Input + if isinstance(history, list): + for line in history: + inputTotalTokens += len(enc.encode(line)) + else: + inputTotalTokens += len(enc.encode(history)) + inputTotalTokens += len(enc.encode(system)) + inputTotalTokens += len(enc.encode(user)) + + # Output + outputTotalTokens += round(len(enc.encode(user)) * 2.5) + + return [inputTotalTokens, outputTotalTokens] + + +@retry(exceptions=Exception, tries=5, delay=5) +def translateAI(text, history, config, filename=None, pbar=None, lock=None, mismatchList=None): + """ + Main translation entry point used by all modules. + + Returns [translatedText, [inputTokens, outputTokens]]. + """ + if not text: + return [text, [0, 0]] + + # Use TRANSLATION_RUN_LOG env var as log path if set. + run_log = os.getenv("TRANSLATION_RUN_LOG") + if run_log: + # Make sure parent dir exists + try: + Path(run_log).parent.mkdir(parents=True, exist_ok=True) + except Exception: + pass + config.logFilePath = run_log + + # Ensure log directory exists for the configured path + try: + Path(config.logFilePath).parent.mkdir(parents=True, exist_ok=True) + except Exception: + pass + + # Token tracking: [input, output]. + totalTokens = [0, 0] + + # Init per-file accumulators on first call on this thread (never reset here — + # they span all translateAI calls for a file; reset by calculateCost). + if not hasattr(_thread_local, 'file_cache_read'): + _thread_local.file_cache_read = 0 + _thread_local.file_cache_write = 0 + _thread_local.file_regular = 0 + _thread_local.file_output = 0 + # Batch API usage is billed at 50% so it is accumulated separately. + if not hasattr(_thread_local, 'file_batch_read'): + _thread_local.file_batch_read = 0 + _thread_local.file_batch_write = 0 + _thread_local.file_batch_regular = 0 + _thread_local.file_batch_output = 0 + # Snapshot accumulators so end-of-call delta only counts tokens from this call. + _prev_cr = _thread_local.file_cache_read + _prev_cw = _thread_local.file_cache_write + _prev_reg = _thread_local.file_regular + _prev_out = _thread_local.file_output + _prev_bcr = _thread_local.file_batch_read + _prev_bcw = _thread_local.file_batch_write + _prev_breg = _thread_local.file_batch_regular + _prev_bout = _thread_local.file_batch_output + _thread_local.file_cost_ready = False # will be set True at end of translateAI + + # Anthropic batch phase ('collect'/'consume'); None when off, in estimate + # mode, or when the model doesn't route to the native Anthropic SDK. + batch_phase = get_batch_phase() + if batch_phase and (config.estimateMode or not isClaudeNative(config.model)): + batch_phase = None + + if isinstance(text, list): + formatType = "json" + tList = batchList(text, config.batchSize) + else: + formatType = "json" + tList = [text] + + for index, tItem in enumerate(tList): + # Check if text contains target language + if not re.search(config.langRegex, str(tItem)): + if pbar is not None: + pbar.update(len(tItem) if isinstance(tItem, list) else 1) + if isinstance(tItem, list): + for j in range(len(tItem)): + tItem[j] = cleanTranslatedText(tItem[j], config.language) + tList[index] = tItem + else: + tList[index] = cleanTranslatedText(tItem, config.language) + history = tItem[-config.maxHistory:] if isinstance(tItem, list) else tItem + continue + + # Ellipsis-only bypass: strings whose translatable content is purely '…' characters + # (e.g. "「………」") should never be sent to the AI — just convert brackets and pass through. + def _is_ellipsis_only(s): + inner = str(s).strip().lstrip('「『').rstrip('」』').strip() + return bool(inner) and all(c in '\u2026\u30FC' for c in inner) + + def _convert_ellipsis(s): + return str(s).replace('「', '"').replace('」', '"').replace('『', '"').replace('』', '"') + + if isinstance(tItem, list): + if all(_is_ellipsis_only(s) for s in tItem): + tList[index] = [_convert_ellipsis(s) for s in tItem] + if pbar is not None: + pbar.update(len(tItem)) + continue + else: + if _is_ellipsis_only(tItem): + tList[index] = _convert_ellipsis(tItem) + if pbar is not None: + pbar.update(1) + continue + + # Protect script codes before translation + protected_items = [] + all_replacements = {} + + if isinstance(tItem, list): + for j in range(len(tItem)): + if not tItem[j] or not str(tItem[j]).strip(): + protected_items.append("Placeholder Text") + all_replacements[j] = {} + else: + collapsed = re.sub(r'(.)\1{9,}', lambda m: m.group(1) * 10, tItem[j]) + protected_text, replacements = protect_script_codes(collapsed) + protected_items.append(protected_text) + all_replacements[j] = replacements + else: + if not tItem or not str(tItem).strip(): + protected_items = "Placeholder Text" + all_replacements[0] = {} + else: + collapsed = re.sub(r'(.)\1{9,}', lambda m: m.group(1) * 10, tItem) + protected_items, all_replacements[0] = protect_script_codes(collapsed) + + # Filter out corrupted/mojibake text (U+FFFD) from the batch before API call + corrupted_map = {} # original_index -> original_text + if isinstance(tItem, list): + for j in range(len(tItem)): + if tItem[j] and "\ufffd" in str(tItem[j]): + corrupted_map[j] = tItem[j] + elif tItem and "\ufffd" in str(tItem): + # Single corrupted string - skip translation entirely + tList[index] = tItem + if pbar is not None: + pbar.update(1) + history = tItem + continue + + # Filter out items that have content but no Japanese — they need no translation + # and the AI tends to empty them (e.g. "「………」" -> ""). Apply the same + # cleanup that would happen post-translation and restore them afterwards. + no_japanese_map = {} # original_index -> already-cleaned text + if isinstance(tItem, list): + for j in range(len(tItem)): + if j in corrupted_map: + continue + item_str = str(tItem[j]).strip() if tItem[j] else "" + if item_str and item_str != "Placeholder Text" and not re.search(config.langRegex, item_str): + cleaned = cleanTranslatedText(tItem[j], config.language) + cleaned = cleaned.replace("「", '"').replace("」", '"').strip() + no_japanese_map[j] = cleaned + + # Combine skip sets and rebuild protected_items / all_replacements + skip_indices = set(corrupted_map.keys()) | set(no_japanese_map.keys()) + if isinstance(tItem, list) and skip_indices: + clean_indices = [j for j in range(len(tItem)) if j not in skip_indices] + + if not clean_indices: + # Every item is either corrupted or untranslatable — reassemble and move on + result = [] + for j in range(len(tItem)): + if j in corrupted_map: + result.append(corrupted_map[j]) + elif j in no_japanese_map: + result.append(no_japanese_map[j]) + else: + result.append(tItem[j]) + tList[index] = result + if pbar is not None: + pbar.update(len(tItem)) + history = result[-config.maxHistory:] + continue + + # Rebuild protected_items and all_replacements for translatable items only + protected_items = [protected_items[j] for j in clean_indices] + new_replacements = {} + for new_idx, old_idx in enumerate(clean_indices): + new_replacements[new_idx] = all_replacements.get(old_idx, {}) + all_replacements = new_replacements + + # Build filtered tItem for validation (excludes skipped items) + if isinstance(tItem, list) and skip_indices: + clean_tItem = [tItem[j] for j in range(len(tItem)) if j not in skip_indices] + else: + clean_tItem = tItem + + # Format for translation + if isinstance(tItem, list): + payload = {f"Line{i+1}": string for i, string in enumerate(protected_items)} + payload = json.dumps(payload, indent=4, ensure_ascii=False) + subbedT = payload + else: + subbedT = json.dumps({"Line1": protected_items}, indent=4, ensure_ascii=False) + + # Batch collect queues list payloads only. Single strings (speaker and + # variable names) translate live — modules memoize them and embed the + # results into later payloads, so they must resolve identically in both + # passes or the consume pass couldn't match the queued payload keys. + # This is the names-first phase; names are a tiny share of the volume. + queue_for_batch = batch_phase == "collect" and isinstance(tItem, list) + + # Check cache for this exact payload (the collect pass uses a + # non-blocking peek so no pending markers are left behind for the + # consume pass to wait on) + if queue_for_batch: + cached_result = peek_cached_translation(subbedT, config.language) + else: + cached_result = get_cached_translation(subbedT, config.language) + if cached_result is not None: + # In estimate mode, never replace tList[index] from cache — the cached value + # may have been stored for a batch with a different number of skip_indices, + # so its length can differ from the current tItem. Keeping tList[index] as + # the original tItem ensures the returned list always has the correct length. + if not config.estimateMode: + if isinstance(tItem, list): + tList[index] = cached_result + history = cached_result[-config.maxHistory:] + else: + tList[index] = cached_result + history = cached_result + else: + if isinstance(cached_result, list) and cached_result: + history = cached_result[-config.maxHistory:] + elif cached_result: + history = cached_result + + if lock and pbar is not None: + with lock: + pbar.update(len(tItem) if isinstance(tItem, list) else 1) + + continue + + # Create context — static_system is the stable prompt.txt content; + # vocab_text is the per-batch matched vocabulary (dynamic). + static_system, vocab_text, user = createContext(config, subbedT, formatType, history) + + # Batch collect pass: queue the request (built exactly like a live one) + # instead of calling the API. The text stays untranslated; the consume + # pass fills it in from the fetched batch results. History carries the + # preceding source lines so the model still sees scene context. + if queue_for_batch: + numLines = len(clean_tItem) if isinstance(tItem, list) else 1 + params = buildClaudeRequest(static_system, user, history, formatType, + config.model, numLines, vocab_text=vocab_text) + queue_batch_request(subbedT, config.language, params) + if lock and pbar is not None: + with lock: + pbar.update(len(tItem)) + history = tItem[-config.maxHistory:] + continue + + # Calculate estimate if in estimate mode + if config.estimateMode: + estimate = countTokens(static_system + vocab_text, user, history) + totalTokens[0] += estimate[0] + totalTokens[1] += estimate[1] + + # Track exact cache write size (static_system, constant across batches) + # and accumulate non-cached (vocab + user + history) tokens per batch. + _est_api = os.getenv("api", "").strip() + _is_claude_est = ( + config.model + and any(x in config.model.lower() for x in ("claude", "sonnet", "haiku", "opus")) + and (not _est_api or "anthropic" in _est_api.lower()) + ) + if _is_claude_est: + # Use Anthropic's count_tokens API once to get the exact cached token count. + # Only called on the first batch; result reused for all subsequent batches. + if not getattr(_thread_local, 'estimate_static_tokens', 0): + try: + _ant_count_client = anthropic.Anthropic(api_key=openai.api_key) + backtick = chr(96) * 3 + _sys_block = [{"type": "text", "text": backtick + "\n" + static_system + "\n" + backtick, "cache_control": {"type": "ephemeral", "ttl": "1h"}}] + _count_resp = _ant_count_client.beta.messages.count_tokens( + betas=["token-counting-2024-11-01"], + model=config.model, + system=_sys_block, + messages=[{"role": "user", "content": "x"}] + ) + _thread_local.estimate_static_tokens = _count_resp.input_tokens + except Exception: + # Fallback to tiktoken if count_tokens fails + enc = tiktoken.encoding_for_model("gpt-4") + _thread_local.estimate_static_tokens = len(enc.encode(static_system)) + regular_tok = max(0, estimate[0] - getattr(_thread_local, 'estimate_static_tokens', 0)) + _thread_local.estimate_regular_tokens = getattr(_thread_local, 'estimate_regular_tokens', 0) + regular_tok + _thread_local.estimate_batch_count = getattr(_thread_local, 'estimate_batch_count', 0) + 1 + # Track unique batch sizes seen this file (each maps to a distinct schema) + _size = len(clean_tItem) if isinstance(clean_tItem, list) else 1 + _seen = getattr(_thread_local, 'estimate_seen_sizes', set()) + _seen.add(_size) + _thread_local.estimate_seen_sizes = _seen + + # Cache the payload with original text as placeholder for future estimates + if isinstance(tItem, list): + cache_translation(subbedT, tItem, config.language) + else: + cache_translation(subbedT, [tItem], config.language) + + continue + + # --- Translation and Validation Retry Block --- + max_retries = 2 # 1 initial attempt + 2 retries + final_translations = None + last_raw_translation = "" + numLines = len(clean_tItem) if isinstance(tItem, list) else 1 + + for attempt in range(max_retries + 1): + is_valid = True + + # On retries, prepend the correction note to the USER message so the + # cached static_system block is never modified (avoids cache busting). + current_user = user + if attempt > 0: + retry_note = ( + f"IMPORTANT: Your previous attempt was incorrect or incomplete. Please ensure:\n" + f"1. The entire output is translated to {config.language} with no untranslated characters\n" + f"2. The JSON structure is correct with NO EMPTY or near-empty translations\n" + f" - Every line with Japanese text MUST be fully translated\n" + f" - Do NOT leave translations empty (\"\") or as single punctuation marks (\":\")\n" + f"3. ALL placeholders (like __PROTECTED_0__, __PROTECTED_1__, etc.) are preserved EXACTLY as they appear in the input\n" + f" - Do not modify, translate, or remove any __PROTECTED_N__ placeholders\n" + f" - Keep them in the exact same position in your translation\n" + f"4. Do NOT repeat the same letter or symbol many times in a row (e.g. uuuuuuuu... or broken tails)\n" + f" - Keep moans/effects natural; never output long runs of one character\n\n" + ) + current_user = retry_note + user + if pbar: + pbar.write(f"Retrying translation... (Attempt {attempt + 1}/{max_retries + 1})") + + # Translate — the consume pass tries the fetched batch result first; + # a missing or invalid result falls through to the live API. + from_batch = False + if batch_phase == "consume" and attempt == 0: + batch_result = take_batch_result(subbedT, config.language) + if batch_result is not None: + response = _AnthropicCompat( + batch_result.get("text", ""), + batch_result.get("prompt_tokens", 0) or 0, + batch_result.get("completion_tokens", 0) or 0, + batch_result.get("cache_read_input_tokens", 0) or 0, + batch_result.get("cache_creation_input_tokens", 0) or 0, + ) + from_batch = True + _write_request_debug_log("anthropic-batch", {"payload": subbedT}, response.usage) + if not from_batch: + try: + response = translateText(static_system, current_user, history, 0.05, formatType, config.model, numLines, vocab_text=vocab_text) + except Exception as api_err: + err_msg = f"[API_ERROR] {api_err}" + # Print to stdout so the GUI captures it immediately + print(err_msg, flush=True) + if pbar: + pbar.write(err_msg) + # Also write to the translation log file for persistence + try: + Path(config.logFilePath).parent.mkdir(parents=True, exist_ok=True) + with open(config.logFilePath, "a", encoding="utf-8") as _lf: + _lf.write(f"{err_msg}\n") + _lf.flush() + except Exception: + pass + raise # Let retry decorator handle it + translatedText = response.choices[0].message.content + last_raw_translation = translatedText + + # Update token count for this attempt + totalTokens[0] += response.usage.prompt_tokens + totalTokens[1] += response.usage.completion_tokens + + # --- Cache cost tracking (Claude only) --- + _is_claude_model = config.model and any(x in config.model.lower() for x in ("claude", "sonnet", "haiku", "opus")) + if _is_claude_model: + usage = response.usage + + # Read cache fields from _AnthropicCompat._Usage; fall back to model_extra. + def _get_usage_field(field): + v = getattr(usage, field, None) + if v is None: + v = (getattr(usage, "model_extra", None) or {}).get(field) + return int(v) if v else 0 + + batch_cache_read = _get_usage_field("cache_read_input_tokens") + batch_cache_write = _get_usage_field("cache_creation_input_tokens") + batch_prompt_total = getattr(usage, "prompt_tokens", 0) or 0 + batch_regular = max(0, batch_prompt_total - batch_cache_read - batch_cache_write) + batch_output = getattr(usage, "completion_tokens", 0) or 0 + + # Accumulate into per-file thread-local counters. Batch API + # usage is billed at 50% so it goes into its own counters. + if from_batch: + _thread_local.file_batch_read += batch_cache_read + _thread_local.file_batch_write += batch_cache_write + _thread_local.file_batch_regular += batch_regular + _thread_local.file_batch_output += batch_output + else: + _thread_local.file_cache_read += batch_cache_read + _thread_local.file_cache_write += batch_cache_write + _thread_local.file_regular += batch_regular + _thread_local.file_output += batch_output + + # --- Debug Token Logging --- + if DEBUG: + try: + _dbg_dir = Path("log") + _dbg_dir.mkdir(parents=True, exist_ok=True) + with open(_dbg_dir / "debug.log", "a", encoding="utf-8") as _dbf: + _dbf.write(f"\n--- Batch ({len(clean_tItem) if isinstance(tItem, list) else 1} lines) ---\n") + _dbf.write(f"Prompt: {response.usage.prompt_tokens} tokens | Output: {response.usage.completion_tokens} tokens\n") + if hasattr(response.usage, "cache_read_input_tokens"): + cr = getattr(response.usage, "cache_read_input_tokens", 0) or 0 + cw = getattr(response.usage, "cache_creation_input_tokens", 0) or 0 + cache_status = "HIT" if cr > 0 else ("WRITE" if cw > 0 else "MISS") + _dbf.write(f"Cache: {cache_status} (read={cr}, write={cw})\n") + _dbf.flush() + except Exception: + pass + + # Clean the translation first for consistency + cleaned_text = cleanTranslatedText(translatedText, config.language) + + # Process and validate translation result + if cleaned_text: + if isinstance(tItem, list): + extracted = extractTranslation(cleaned_text, True, pbar) + + # Check 1: Mismatch in length -> still a hard failure + if extracted is None or len(clean_tItem) != len(extracted): + is_valid = False + if pbar: + pbar.write(f"Length mismatch: expected {len(clean_tItem)}, got {len(extracted) if extracted else 0}") + else: + # Check 2: Validate placeholders are preserved + # Flatten all_replacements for batch validation + all_protected_text = protected_items # The list we sent + placeholder_valid, missing, extra = validate_placeholders(all_protected_text, extracted, + {k: v for replacements in all_replacements.values() for k, v in replacements.items()}) + + if not placeholder_valid: + is_valid = False + if pbar: + if missing: + pbar.write(f"Missing placeholders: {', '.join(missing)}") + if extra: + pbar.write(f"Extra placeholders: {', '.join(extra)}") + else: + # Check 3: Validate that translations are not empty or nearly empty + content_valid, invalid_indices, content_reasons = validate_translation_content( + clean_tItem, extracted, config.langRegex + ) + + if not content_valid: + is_valid = False + if pbar: + pbar.write(f"Invalid translation content detected:") + for reason in content_reasons[:5]: # Show first 5 issues + pbar.write(f" - {reason}") + if len(content_reasons) > 5: + pbar.write(f" ... and {len(content_reasons) - 5} more issues") + else: + # Set translations (line count matches, placeholders valid, and content is good) + # Strip "Placeholder Text" from individual lines (AI placeholder for untranslatable input) + # Also apply the 「→" / 」→" replacements here per-line (safe now that JSON is parsed) + def _clean_extracted_line(line): + if not isinstance(line, str): + return line + line = line.replace("Placeholder Text", "").strip() + line = line.replace("「", '"').replace("」", '"') + line = line.replace("『", '"').replace("』", '"') + return line + final_translations = [_clean_extracted_line(line) for line in extracted] + else: + # Single string: extract from JSON schema response + extracted = extractTranslation(cleaned_text, False, pbar) + if extracted is None: + is_valid = False + if pbar: + pbar.write(f"Failed to extract translation from response: {cleaned_text[:100]}") + else: + # Validate placeholders against extracted value + placeholder_valid, missing, extra = validate_placeholders(protected_items, extracted, all_replacements[0]) + + if not placeholder_valid: + is_valid = False + if pbar: + if missing: + pbar.write(f"Missing placeholders: {', '.join(missing)}") + if extra: + pbar.write(f"Extra placeholders: {', '.join(extra)}") + else: + # Validate content for single string + final_cleaned = extracted.replace("Placeholder Text", "") + content_valid, _, content_reasons = validate_translation_content( + tItem, final_cleaned, config.langRegex + ) + + if not content_valid: + is_valid = False + if pbar: + pbar.write(f"Invalid translation content:") + for reason in content_reasons: + pbar.write(f" - {reason}") + else: + # Accept output - all validations passed + final_translations = final_cleaned + else: + is_valid = False + if pbar: pbar.write(f"AI Refused: {tItem}\n") + + # If translation is valid, break the retry loop + if is_valid: + break + + # --- End of Retry Block --- + + # After the loop, handle the final result + if final_translations is not None: # Success case + # Restore protected script codes + if isinstance(tItem, list): + for j in range(len(final_translations)): + if j in all_replacements: + final_translations[j] = restore_script_codes(final_translations[j], all_replacements[j]) + + # Re-insert corrupted / no-japanese originals at their original positions + if corrupted_map or no_japanese_map: + expanded = [] + clean_idx = 0 + for j in range(len(tItem)): + if j in corrupted_map: + expanded.append(corrupted_map[j]) + elif j in no_japanese_map: + expanded.append(no_japanese_map[j]) + else: + expanded.append(final_translations[clean_idx]) + clean_idx += 1 + final_translations = expanded + else: + final_translations = restore_script_codes(final_translations, all_replacements[0]) + + formatted_output = last_raw_translation + try: + parsed_json = json.loads(last_raw_translation) + # Normalize array-based output to LineN format for log readability + if isinstance(parsed_json, dict) and "translations" in parsed_json and isinstance(parsed_json["translations"], list): + parsed_json = {f"Line{i+1}": v for i, v in enumerate(parsed_json["translations"])} + formatted_output = json.dumps(parsed_json, indent=4, ensure_ascii=False) + except (json.JSONDecodeError, ValueError): + pass + + # Only open and write to log file when we have something to log + try: + with open(config.logFilePath, "a", encoding="utf-8") as logFile: + logFile.write(f"Input:\n{subbedT}\n") + logFile.write(f"Output:\n{formatted_output}\n") + logFile.flush() # Ensure data is written to disk immediately + except Exception: + pass # Don't fail if logging fails + + # Cache the entire payload and its translation + if not config.estimateMode: + cache_translation(subbedT, final_translations, config.language) + + if isinstance(tItem, list): + tList[index] = final_translations + history = final_translations[-config.maxHistory:] + else: + tList[index] = final_translations + history = final_translations + + if lock and pbar is not None: + with lock: + pbar.update(len(tItem) if isinstance(tItem, list) else 1) + + else: # Failure case after all retries + if pbar: pbar.write(f"Translation failed after {max_retries + 1} attempts. Check mismatch log.") + + # Emit a machine-readable marker on stdout so the GUI worker + # thread can detect the mismatch reliably (stdout is captured + # synchronously, unlike file-tail polling which can be racy). + try: + print(f"MISMATCH_EVENT:{filename}", flush=True) + except Exception: + pass + + formatted_mismatch_output = last_raw_translation + try: + parsed_json = json.loads(last_raw_translation) + if isinstance(parsed_json, dict) and "translations" in parsed_json and isinstance(parsed_json["translations"], list): + parsed_json = {f"Line{i+1}": v for i, v in enumerate(parsed_json["translations"])} + formatted_mismatch_output = json.dumps(parsed_json, indent=4, ensure_ascii=False) + except (json.JSONDecodeError, ValueError): + pass + with open(config.mismatchLogPath, "a+", encoding="utf-8") as mismatchFile: + mismatchFile.write(f"Failed after retries: {filename}\n") + mismatchFile.write(f"Input:\n{subbedT}\n") + mismatchFile.write(f"Final Output:\n{formatted_mismatch_output}\n") + mismatchFile.flush() # Ensure data is written to disk immediately + + # Also write to the main translation log so the GUI log viewer can display it + try: + with open(config.logFilePath, "a", encoding="utf-8") as logFile: + logFile.write(f"[MISMATCH] Failed after retries: {filename}\n") + logFile.write(f"[MISMATCH] Input:\n") + for mline in subbedT.splitlines(): + logFile.write(f"[MISMATCH] {mline}\n") + logFile.write(f"[MISMATCH] Final Output:\n") + for mline in formatted_mismatch_output.splitlines(): + logFile.write(f"[MISMATCH] {mline}\n") + logFile.flush() + except Exception: + pass # Don't fail if logging fails + + if filename and mismatchList is not None and filename not in mismatchList: + mismatchList.append(filename) + + tList[index] = tItem + history = text[-config.maxHistory:] if isinstance(text, list) else text + + # Combine if multilist + if tList and isinstance(tList[0], list): + tList = [t for sublist in tList for t in sublist] + + # Save cache after processing (for both estimate and translation modes) + save_cache() + + # Batch collect pass: merge this call's queued requests into the disk queue. + if batch_phase == "collect": + flush_batch_queue() + + # For Claude: accumulate only this call's delta into the cross-thread total. + # file_* accumulators hold full per-file totals; calculateCost() reads them. + _is_claude_final = config.model and any(x in config.model.lower() for x in ("claude", "sonnet", "haiku", "opus")) + if _is_claude_final and not config.estimateMode: + _pricing = getPricingConfig(config.model) + _br = _pricing["inputAPICost"] / 1_000_000 + _or = _pricing["outputAPICost"] / 1_000_000 + # Delta = tokens added in this call only (not earlier calls for same file). + _delta_cr = getattr(_thread_local, 'file_cache_read', 0) - _prev_cr + _delta_cw = getattr(_thread_local, 'file_cache_write', 0) - _prev_cw + _delta_reg = getattr(_thread_local, 'file_regular', 0) - _prev_reg + _delta_out = getattr(_thread_local, 'file_output', 0) - _prev_out + _delta_bcr = getattr(_thread_local, 'file_batch_read', 0) - _prev_bcr + _delta_bcw = getattr(_thread_local, 'file_batch_write', 0) - _prev_bcw + _delta_breg = getattr(_thread_local, 'file_batch_regular', 0) - _prev_breg + _delta_bout = getattr(_thread_local, 'file_batch_output', 0) - _prev_bout + _call_cost = ( + _delta_cr * _br * 0.10 + + _delta_cw * _br * 2.00 + + _delta_reg * _br + + _delta_out * _or + + # Batch API tokens: same rates, then the 50% batch discount. + (_delta_bcr * _br * 0.10 + + _delta_bcw * _br * 2.00 + + _delta_breg * _br + + _delta_bout * _or) * 0.50 + ) + global _global_accurate_cost + with _global_accurate_cost_lock: + _global_accurate_cost += _call_cost + _thread_local.file_cost_ready = True # signals calculateCost to use file accumulators + + # Return result + if isinstance(text, list): + return [tList, totalTokens] + else: return [tList[0], totalTokens] \ No newline at end of file