diff --git a/.gitignore b/.gitignore
index c1b01d1..ba96724 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,32 +1,32 @@
-
-.env
-*.tmp
-*.json
-*.js
-*.yaml
-*.yml
-*.txt
-*.log
-logs/
-*.rpy
-*.csv
-*.ks
-*.cid
-*.png
-*.script
-__pycache__
-
-!assets/*.png
-!.pre-commit-config.yaml
-!prompt.txt
-!vocab_base.txt
-!requirements.txt
-!util/tl_inspector/TLInspector.js
-!util/forge/Forge_MZ.js
-!util/forge/Forge_MV.js
-!util/forge/upstream/Forge_MZ.js
-!util/forge/upstream/Forge_MV.js
-util/forge/.forge_version.json
-util/ace/*.exe
-util/ace/.tools_version.json
-!util/ace/offline/*.exe
+
+.env
+*.tmp
+*.json
+*.js
+*.yaml
+*.yml
+*.txt
+*.log
+logs/
+*.rpy
+*.csv
+*.ks
+*.cid
+*.png
+*.script
+__pycache__
+
+!assets/*.png
+!.pre-commit-config.yaml
+!data/prompt.txt
+!data/vocab_base.txt
+!requirements.txt
+!util/tl_inspector/TLInspector.js
+!util/forge/Forge_MZ.js
+!util/forge/Forge_MV.js
+!util/forge/upstream/Forge_MZ.js
+!util/forge/upstream/Forge_MV.js
+util/forge/.forge_version.json
+util/ace/*.exe
+util/ace/.tools_version.json
+!util/ace/offline/*.exe
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 6b31d65..e1497a5 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -3,6 +3,6 @@
'hooks':
- 'id': 'set-defaults'
'name': 'Set Defaults in rpgmakermvmz.py'
- 'entry': 'python set_defaults.py'
+ 'entry': 'python -m util.defaults'
'language': 'python'
'files': '^modules/rpgmakermvmz.py$'
diff --git a/DazedMTLTool.desktop b/DazedMTLTool.desktop
new file mode 100644
index 0000000..14da745
--- /dev/null
+++ b/DazedMTLTool.desktop
@@ -0,0 +1,10 @@
+[Desktop Entry]
+Type=Application
+Name=DazedMTLTool
+GenericName=Translation Tool
+Comment=AI translation tool for visual novels and RPG games
+Exec=bash -c 'exec "$(cd "$(dirname "%k")" && pwd)/scripts/launch.sh"'
+Icon=%k/assets/icon.png
+Terminal=false
+Categories=Development;Utility;
+StartupWMClass=DazedMTLTool
diff --git a/README.md b/README.md
index 9c9cb5e..ae2c356 100644
--- a/README.md
+++ b/README.md
@@ -124,12 +124,11 @@ This means Python wasn't added to your PATH. You have two options:
### 3. Launch the GUI
-**Double-click `START.bat`**. It will:
-- Create a virtual environment automatically.
-- Install all dependencies.
-- Launch the GUI.
+**Windows:** Double-click `START.bat`. It will create a virtual environment, install dependencies, and open the GUI.
-That's it! From now on, just double-click `START.bat` to open the tool.
+**Linux/macOS:** Run `./START.sh`, or double-click `DazedMTLTool.desktop` (choose **Allow Launching** when your file manager asks). From then on, either method works.
+
+That's it! Use the same launcher each time you want to open the tool.
---
@@ -160,10 +159,10 @@ Specialized tabs with extra options for those specific engines.
## Vocab & Prompt
-### vocab.txt
+### data/vocab.txt
This file gives the AI context about your game — character names, genders, recurring terms, etc. The better your vocab file, the more consistent the translation.
-Open `vocab.txt` (or copy `vocab.txt.example` to `vocab.txt` if it doesn't exist) and add entries like:
+On first run, `data/vocab.txt` is created automatically from `data/vocab_base.txt` if it does not exist yet. Add entries like:
```plaintext
# Game Characters
@@ -176,14 +175,14 @@ Format: Japanese name, English name in parentheses, then gender.
> **Note:** A very large vocab file can increase API costs and potentially reduce quality. Focus on the most important characters and terms.
-### prompt.txt
-This is the system prompt sent to the AI. A default `prompt.txt` is included and works well for most games. You generally don't need to edit it unless you want to customize the translation style.
+### data/prompt.txt
+This is the system prompt sent to the AI. A default `data/prompt.txt` is included and works well for most games. You generally don't need to edit it unless you want to customize the translation style.
---
## Tips
-- **Check `log/translations.txt`** after a run to see what was translated. You can copy useful terms from it into `vocab.txt` for consistency in future runs.
+- **Check `log/translations.txt`** after a run to see what was translated. You can copy useful terms from it into `data/vocab.txt` for consistency in future runs.
- **Start small** — Translate a few files first to make sure the output looks good before doing the whole game.
- **Wordwrap** — If text overflows or looks awkward in-game, adjust the `width` setting in `.env` or the Config tab. `60` is a good default for most RPG Maker games.
- **Version control** — Using [Git](https://git-scm.com/) with the game folder is highly recommended. It lets you track every change the translation makes, compare with original files, and roll back if needed.
@@ -300,8 +299,8 @@ Here's the recommended step-by-step process for translating an RPG Maker MV/MZ g
| Step | Action |
|------|--------|
-| **1** | **Parse speakers → vocab.txt** — Use the Parse Speakers feature to pull character names from the game files into `vocab.txt`. |
-| **2** | **Identify speaker genders** — Figure out which characters are male/female and update `vocab.txt` accordingly. This helps the AI use correct pronouns. |
+| **1** | **Parse speakers → vocab** — Use the Parse Speakers feature to pull character names from the game files into `data/vocab.txt`. |
+| **2** | **Identify speaker genders** — Figure out which characters are male/female and update `data/vocab.txt` accordingly. This helps the AI use correct pronouns. |
| **3** | **Translate Actors.json, MapInfos.json** — These are small files with character and map names. Good to do first. |
| **4** | **Translate Items, System, Weapons, etc.** — All the data files that aren't maps or events. Place them in `files/`, translate, then copy results back. |
| **5** | **Find speaker names** — Enable CODE 101 (Speakers), check for bracketed names, or use the "First Line = Speaker" option to capture speaker names properly. |
diff --git a/START.bat b/START.bat
index e2e3f3f..b910a60 100644
--- a/START.bat
+++ b/START.bat
@@ -184,28 +184,29 @@ echo Launching DazedMTLTool GUI...
echo ==========================================
echo.
-:: Ensure vocab.txt exists (create from example if available)
-if not exist "vocab.txt" (
- if exist "vocab.txt.example" (
- echo vocab.txt not found - creating from vocab.txt.example...
- copy /Y "vocab.txt.example" "vocab.txt" >nul 2>&1
+:: Ensure data/vocab.txt exists (create from vocab_base.txt if available)
+if not exist "data\\vocab.txt" (
+ if not exist "data" mkdir data
+ if exist "data\\vocab_base.txt" (
+ echo data\vocab.txt not found - creating from data\vocab_base.txt...
+ copy /Y "data\vocab_base.txt" "data\vocab.txt" >nul 2>&1
if errorlevel 1 (
- echo ERROR: Failed to copy vocab.txt.example to vocab.txt.
+ echo ERROR: Failed to copy data\vocab_base.txt to data\vocab.txt.
) else (
- echo Created vocab.txt from vocab.txt.example
+ echo Created data\vocab.txt from data\vocab_base.txt
)
) else (
- echo vocab.txt and vocab.txt.example not found - creating empty vocab.txt to avoid import errors...
- type NUL > "vocab.txt"
+ echo data\vocab.txt not found - creating empty file to avoid import errors...
+ type NUL > "data\vocab.txt"
if errorlevel 1 (
- echo ERROR: Failed to create empty vocab.txt.
+ echo ERROR: Failed to create empty data\vocab.txt.
) else (
- echo Created empty vocab.txt
+ echo Created empty data\vocab.txt
)
)
)
-python start_gui.py
+python scripts/start_gui.py
:: Check if GUI launched successfully
if errorlevel 1 (
diff --git a/START.sh b/START.sh
index 6bfa62e..f087cf1 100755
--- a/START.sh
+++ b/START.sh
@@ -127,23 +127,24 @@ activate_venv() {
}
ensure_vocab_file() {
- if [[ -f "vocab.txt" ]]; then
+ mkdir -p data
+ if [[ -f "data/vocab.txt" ]]; then
return 0
fi
- if [[ -f "vocab.txt.example" ]]; then
- echo "vocab.txt not found - creating from vocab.txt.example..."
- if cp "vocab.txt.example" "vocab.txt"; then
- echo "Created vocab.txt from vocab.txt.example"
+ if [[ -f "data/vocab_base.txt" ]]; then
+ echo "data/vocab.txt not found - creating from data/vocab_base.txt..."
+ if cp "data/vocab_base.txt" "data/vocab.txt"; then
+ echo "Created data/vocab.txt from data/vocab_base.txt"
else
- echo "ERROR: Failed to copy vocab.txt.example to vocab.txt."
+ echo "ERROR: Failed to copy data/vocab_base.txt to data/vocab.txt."
fi
else
- echo "vocab.txt and vocab.txt.example not found - creating empty vocab.txt to avoid import errors..."
- if : > "vocab.txt"; then
- echo "Created empty vocab.txt"
+ echo "data/vocab.txt not found - creating empty file to avoid import errors..."
+ if : > "data/vocab.txt"; then
+ echo "Created empty data/vocab.txt"
else
- echo "ERROR: Failed to create empty vocab.txt."
+ echo "ERROR: Failed to create empty data/vocab.txt."
fi
fi
}
@@ -227,7 +228,7 @@ echo
ensure_vocab_file
-if ! python start_gui.py; then
+if ! exec python "$SCRIPT_DIR/scripts/start_gui.py"; then
echo
echo "ERROR: Failed to launch GUI."
echo "Check the error messages above."
diff --git a/prompt.txt b/data/prompt.txt
similarity index 100%
rename from prompt.txt
rename to data/prompt.txt
diff --git a/vocab_base.txt b/data/vocab_base.txt
similarity index 100%
rename from vocab_base.txt
rename to data/vocab_base.txt
diff --git a/gui/main.py b/gui/main.py
index fd35eb5..7d6c3b6 100644
--- a/gui/main.py
+++ b/gui/main.py
@@ -19,20 +19,16 @@ from PyQt5.QtWidgets import (
QTextEdit, QSplitter, QGroupBox, QStatusBar, QStackedWidget, QToolButton,
QDialog
)
-from PyQt5.QtCore import Qt, QThread, pyqtSignal, QTimer, QSettings
-from PyQt5.QtGui import QIcon, QFont, QPixmap, QScreen
+from PyQt5.QtCore import Qt, QThread, pyqtSignal, QTimer, QSettings, QCoreApplication
+from PyQt5.QtGui import QIcon, QFont, QPixmap, QScreen, QGuiApplication
-def _project_root() -> Path:
- """Directory containing start_gui.py / requirements.txt (parent of gui/)."""
- return Path(__file__).resolve().parent.parent
+from util.paths import PROJECT_ROOT, ICON_PATH, LAST_UPDATE_SHA_PATH
def load_application_icon() -> QIcon:
"""Prefer ICO on Windows; PNG is fine for all platforms. Empty if missing."""
- root = _project_root()
- for rel in ("assets/icon.ico", "assets/icon.png"):
- path = root / rel
+ for path in (PROJECT_ROOT / "assets/icon.ico", ICON_PATH):
if path.is_file():
icon = QIcon(str(path))
if not icon.isNull():
@@ -126,14 +122,13 @@ class UpdateThread(QThread):
REPO_USER = "DazedAnon"
REPO_NAME = "DazedMTLTool"
REPO_BRANCH = "main"
- SHA_FILE = "last_update_sha.txt"
+ SHA_FILE = str(LAST_UPDATE_SHA_PATH)
# Paths (relative, top-level) that should never be touched during update
- PROTECTED = {".env", "venv", "log", "files", "translated",
- "vocab.txt", "last_update_sha.txt"}
+ PROTECTED = {".env", "venv", "log", "files", "translated", "data"}
# GitLab zip archives do not preserve Unix execute bits; restore after apply.
- EXECUTABLE_SUFFIXES = {".sh"}
+ EXECUTABLE_SUFFIXES = {".sh", ".desktop"}
progress = pyqtSignal(str) # status message
finished = pyqtSignal(bool, str) # (success, message)
@@ -902,6 +897,14 @@ def main():
# Set high DPI scale factor policy
if hasattr(QApplication, 'setHighDpiScaleFactorRoundingPolicy'):
QApplication.setHighDpiScaleFactorRoundingPolicy(Qt.HighDpiScaleFactorRoundingPolicy.PassThrough)
+
+ if sys.platform.startswith("linux"):
+ from util.linux_desktop import ensure_linux_desktop_entry
+ ensure_linux_desktop_entry()
+ QGuiApplication.setDesktopFileName("DazedMTLTool")
+
+ QCoreApplication.setOrganizationName("DazedTranslations")
+ QCoreApplication.setApplicationName("DazedMTLTool")
app = QApplication(sys.argv)
@@ -934,9 +937,7 @@ def main():
return 1
# Set application properties
- app.setApplicationName("DazedMTLTool")
app.setApplicationVersion("1.0")
- app.setOrganizationName("DazedTranslations")
# Apply dark theme with cleaner, more compact styling
app.setStyleSheet("""
diff --git a/gui/rpgmaker_tab.py b/gui/rpgmaker_tab.py
index ef7c86b..60c8418 100644
--- a/gui/rpgmaker_tab.py
+++ b/gui/rpgmaker_tab.py
@@ -19,8 +19,7 @@ except ImportError:
ConfigIntegration = None
try:
# Prefer importing the project's canonical defaults if available
- import set_defaults
- CANONICAL_DEFAULTS = getattr(set_defaults, 'DEFAULTS', None)
+ from util.defaults import DEFAULTS as CANONICAL_DEFAULTS
except Exception:
CANONICAL_DEFAULTS = None
diff --git a/gui/srpg_tab.py b/gui/srpg_tab.py
index 43f0262..476f497 100644
--- a/gui/srpg_tab.py
+++ b/gui/srpg_tab.py
@@ -12,8 +12,7 @@ import re
from gui.platform_glyph import platform_glyph
try:
- import set_defaults
- CANONICAL_DEFAULTS = getattr(set_defaults, 'DEFAULTS', None)
+ from util.defaults import DEFAULTS as CANONICAL_DEFAULTS
except Exception:
CANONICAL_DEFAULTS = None
diff --git a/gui/wolf_tab.py b/gui/wolf_tab.py
index cc507e1..c37b9e5 100644
--- a/gui/wolf_tab.py
+++ b/gui/wolf_tab.py
@@ -11,8 +11,7 @@ from PyQt5.QtCore import pyqtSignal
import re
from gui.platform_glyph import platform_glyph
try:
- import set_defaults
- CANONICAL_DEFAULTS = getattr(set_defaults, 'DEFAULTS', None)
+ from util.defaults import DEFAULTS as CANONICAL_DEFAULTS
except Exception:
CANONICAL_DEFAULTS = None
diff --git a/gui/workflow_tab.py b/gui/workflow_tab.py
index d130995..6148b2f 100644
--- a/gui/workflow_tab.py
+++ b/gui/workflow_tab.py
@@ -22,6 +22,8 @@ import sys
import threading
from pathlib import Path
+from util.paths import VOCAB_BASE_PATH, VOCAB_PATH
+
import jsbeautifier
from PyQt5.QtCore import Qt, QEvent, QSettings, QThread, QTimer, pyqtSignal
@@ -3576,7 +3578,7 @@ class WorkflowTab(QWidget):
def _read_vocab_speakers(self) -> list[tuple[str, str]]:
"""Parse the '# Speakers' section from vocab.txt and return (orig, tl) pairs."""
- vocab_path = Path("vocab.txt")
+ vocab_path = VOCAB_PATH
if not vocab_path.exists():
return []
try:
@@ -3606,7 +3608,7 @@ class WorkflowTab(QWidget):
return results
def _reload_vocab(self):
- vocab_path = Path("vocab.txt")
+ vocab_path = VOCAB_PATH
try:
if vocab_path.exists():
text = vocab_path.read_text(encoding="utf-8")
@@ -3623,10 +3625,10 @@ class WorkflowTab(QWidget):
def _save_vocab(self):
try:
game_text = self.vocab_editor.toPlainText().rstrip("\n")
- base_path = Path("vocab_base.txt")
+ base_path = VOCAB_BASE_PATH
base_text = base_path.read_text(encoding="utf-8") if base_path.exists() else ""
combined = game_text + "\n\n" + self._BASE_SEPARATOR + base_text
- Path("vocab.txt").write_text(combined, encoding="utf-8")
+ VOCAB_PATH.write_text(combined, encoding="utf-8")
self._log("✅ vocab.txt saved (base terms from vocab_base.txt appended).")
except Exception as exc:
self._log(f"❌ Could not save vocab.txt: {exc}")
@@ -3795,7 +3797,7 @@ class WorkflowTab(QWidget):
self._log("⚠ No game folder set. Complete Step 0 first.")
return
- src = Path("vocab.txt")
+ src = VOCAB_PATH
if not src.exists():
self._log("⚠ vocab.txt not found — save it in Step 3 first.")
return
diff --git a/modules/aquedi4.py b/modules/aquedi4.py
index 7f1a43e..655835e 100644
--- a/modules/aquedi4.py
+++ b/modules/aquedi4.py
@@ -1,215 +1,217 @@
-# modules/aquedi4.py
-
-# Libraries
-import json
-import os
-import re
-import util.dazedwrap as dazedwrap
-import threading
-import time
-import traceback
-import tiktoken
-from pathlib import Path
-from colorama import Fore
-from dotenv import load_dotenv
-from retry import retry
-from tqdm import tqdm
-from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost
-# from modules.json import translateSimpleKeyValueJSON
-import tempfile
-
-# Boilerplate Globals
-MODEL = os.getenv("model")
-TIMEOUT = int(os.getenv("timeout"))
-LANGUAGE = os.getenv("language").capitalize()
-PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
-VOCAB = Path("vocab.txt").read_text(encoding="utf-8")
-LOCK = threading.Lock()
-ESTIMATE = ""
-TOKENS = [0, 0]
-MISMATCH = []
-FILENAME = None
-PBAR = None
-BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
-
-# Flag to control line break replacement
-REPLACE_LINEBREAKS_WITH_PIPES = True
-
-LANGREGEX = r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+"
-TRANSLATION_CONFIG = TranslationConfig(
- model=MODEL,
- language=LANGUAGE,
- prompt=PROMPT,
- vocab=VOCAB,
- langRegex=LANGREGEX,
- batchSize=int(os.getenv("batchsize", 10)),
- maxHistory=10,
- estimateMode=False
-)
-# End Boilerplate
-
-def handleAquedi4(filename, estimate):
- """Main handler for Aquedi4 JSON files."""
- global ESTIMATE, TOKENS, FILENAME
- ESTIMATE = estimate
- FILENAME = filename
-
- try:
- start = time.time()
- translatedData = openFiles(filename)
-
- if not estimate:
- # Write final result after translation is complete
- with open("translated/" + filename, "w", encoding="utf-8", newline="\n") as outFile:
- json.dump(translatedData[0], outFile, ensure_ascii=False, indent=4)
-
- end = time.time()
- tqdm.write(getResultString(translatedData, end - start, filename))
- with LOCK:
- TOKENS[0] += translatedData[1][0]
- TOKENS[1] += translatedData[1][1]
-
- except Exception as e:
- traceback.print_exc()
- return "Fail"
-
- return getResultString(["", TOKENS, None], end - start, "TOTAL")
-
-
-def openFiles(filename):
- """Opens and parses the JSON file."""
- with open("files/" + filename, "r", encoding="UTF-8-sig") as f:
- data = json.load(f)
- translatedData = parseAquedi4JSON(data, filename)
- return translatedData
-
-
-def parseAquedi4JSON(data, filename):
- """Parses and translates the simple key-value JSON."""
- totalTokens = [0, 0]
- totalLines = 0
- totalLines = len(data)
- global LOCK, PBAR
-
- stringList = []
- keyList = []
- if isinstance(data, dict) and all(isinstance(v, str) for v in data.values()):
- for key, value in data.items():
- if re.search(LANGREGEX, key):
- stringList.append(key)
- keyList.append(key)
-
- with tqdm(total=len(stringList), bar_format=BAR_FORMAT, position=0, leave=False) as pbar:
- pbar.desc = filename
- PBAR = pbar
-
- # from modules.json import translateSimpleKeyValueJSON
-
- try:
- if stringList:
- result = translateSimpleKeyValueJSON(data, filename, stringList, keyList)
- totalTokens[0] += result[0]
- totalTokens[1] += result[1]
- else:
- pbar.write(f"Warning: {filename} is not in the expected simple key-value format. Skipping.")
- except Exception as e:
- traceback.print_exc()
- return [data, totalTokens, e]
-
- return [data, totalTokens, None]
-
-
-def translateSimpleKeyValueJSON(data, filename, stringList, keyList):
- """
- Translate simple key-value JSON format where keys contain Japanese text
- and values are placeholder strings like "TODO".
-
- Format example:
- {
- "\\r\\nスタビライザー": "TODO",
- "\\r\\nプラグインなし": "TODO"
- }
- """
- global LOCK, ESTIMATE, FILENAME, PBAR, MISMATCH
- tokens = [0, 0]
-
- preparedStringList = []
- if REPLACE_LINEBREAKS_WITH_PIPES:
- for s in stringList:
- preparedStringList.append(s.replace("\r\n", "|").replace("\n", "|"))
- else:
- preparedStringList = stringList
-
- # Translate all keys if any were found
- if preparedStringList:
- PBAR.total = len(preparedStringList)
- PBAR.refresh()
-
- response = translateAI(preparedStringList, "Reply with the English Translation")
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- translatedList = response[0]
-
- # Check for mismatch
- if len(preparedStringList) != len(translatedList):
- with LOCK:
- if FILENAME not in MISMATCH:
- MISMATCH.append(FILENAME)
-
- # Pass 2: Update the values with translations
- for i, original_key in enumerate(keyList):
- if i < len(translatedList):
- translatedText = translatedList[i]
-
- if REPLACE_LINEBREAKS_WITH_PIPES:
- translatedText = translatedText.replace("|", "\r\n")
- # Set the translated text as the value
- data[original_key] = translatedText
-
- # Save progress after each translation
- save_progress_json(data, filename)
-
- return tokens
-
-def save_progress_json(data, filename):
- """Save current JSON translation progress."""
- try:
- if ESTIMATE: return
- os.makedirs("translated", exist_ok=True)
- tmp_path = os.path.join("translated", f"{filename}.tmp")
- final_path = os.path.join("translated", filename)
- with open(tmp_path, "w", encoding="utf-8", newline="\n") as outFile:
- json.dump(data, outFile, ensure_ascii=False, indent=4)
- os.replace(tmp_path, final_path)
- except Exception:
- traceback.print_exc()
-
-
-def getResultString(translatedData, translationTime, filename):
- cost = calculateCost(translatedData[1][0], translatedData[1][1], MODEL)
- totalTokenstring = (
- Fore.YELLOW + f"[Input: {translatedData[1][0]}]"
- f"[Output: {translatedData[1][1]}]"
- f"[Cost: ${cost:,.4f}]"
- )
- timeString = Fore.BLUE + f"[{round(translationTime, 1)}s]"
-
- if translatedData[2] is None:
- return f"{filename}: {totalTokenstring}{timeString}" + Fore.GREEN + " \u2713 " + Fore.RESET
- else:
- errorString = str(translatedData[2]) + Fore.RED
- return f"{filename}: {totalTokenstring}{timeString}" + Fore.RED + " \u2717 " + errorString + Fore.RESET
-
-def translateAI(text, history, history_ctx=None):
- """Legacy wrapper for the shared translation utility."""
- global PBAR, MISMATCH, FILENAME
- TRANSLATION_CONFIG.estimateMode = bool(ESTIMATE)
- return sharedtranslateAI(
- text=text,
- history=history,
- config=TRANSLATION_CONFIG,
- filename=FILENAME,
- pbar=PBAR,
- lock=LOCK,
- mismatchList=MISMATCH
+# modules/aquedi4.py
+
+# Libraries
+import json
+import os
+import re
+import util.dazedwrap as dazedwrap
+import threading
+import time
+import traceback
+import tiktoken
+from pathlib import Path
+from colorama import Fore
+from dotenv import load_dotenv
+from retry import retry
+from tqdm import tqdm
+from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost
+# from modules.json import translateSimpleKeyValueJSON
+import tempfile
+
+# Boilerplate Globals
+MODEL = os.getenv("model")
+TIMEOUT = int(os.getenv("timeout"))
+LANGUAGE = os.getenv("language").capitalize()
+from util.paths import PROMPT_PATH, VOCAB_PATH
+
+PROMPT = PROMPT_PATH.read_text(encoding="utf-8")
+VOCAB = VOCAB_PATH.read_text(encoding="utf-8")
+LOCK = threading.Lock()
+ESTIMATE = ""
+TOKENS = [0, 0]
+MISMATCH = []
+FILENAME = None
+PBAR = None
+BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
+
+# Flag to control line break replacement
+REPLACE_LINEBREAKS_WITH_PIPES = True
+
+LANGREGEX = r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+"
+TRANSLATION_CONFIG = TranslationConfig(
+ model=MODEL,
+ language=LANGUAGE,
+ prompt=PROMPT,
+ vocab=VOCAB,
+ langRegex=LANGREGEX,
+ batchSize=int(os.getenv("batchsize", 10)),
+ maxHistory=10,
+ estimateMode=False
+)
+# End Boilerplate
+
+def handleAquedi4(filename, estimate):
+ """Main handler for Aquedi4 JSON files."""
+ global ESTIMATE, TOKENS, FILENAME
+ ESTIMATE = estimate
+ FILENAME = filename
+
+ try:
+ start = time.time()
+ translatedData = openFiles(filename)
+
+ if not estimate:
+ # Write final result after translation is complete
+ with open("translated/" + filename, "w", encoding="utf-8", newline="\n") as outFile:
+ json.dump(translatedData[0], outFile, ensure_ascii=False, indent=4)
+
+ end = time.time()
+ tqdm.write(getResultString(translatedData, end - start, filename))
+ with LOCK:
+ TOKENS[0] += translatedData[1][0]
+ TOKENS[1] += translatedData[1][1]
+
+ except Exception as e:
+ traceback.print_exc()
+ return "Fail"
+
+ return getResultString(["", TOKENS, None], end - start, "TOTAL")
+
+
+def openFiles(filename):
+ """Opens and parses the JSON file."""
+ with open("files/" + filename, "r", encoding="UTF-8-sig") as f:
+ data = json.load(f)
+ translatedData = parseAquedi4JSON(data, filename)
+ return translatedData
+
+
+def parseAquedi4JSON(data, filename):
+ """Parses and translates the simple key-value JSON."""
+ totalTokens = [0, 0]
+ totalLines = 0
+ totalLines = len(data)
+ global LOCK, PBAR
+
+ stringList = []
+ keyList = []
+ if isinstance(data, dict) and all(isinstance(v, str) for v in data.values()):
+ for key, value in data.items():
+ if re.search(LANGREGEX, key):
+ stringList.append(key)
+ keyList.append(key)
+
+ with tqdm(total=len(stringList), bar_format=BAR_FORMAT, position=0, leave=False) as pbar:
+ pbar.desc = filename
+ PBAR = pbar
+
+ # from modules.json import translateSimpleKeyValueJSON
+
+ try:
+ if stringList:
+ result = translateSimpleKeyValueJSON(data, filename, stringList, keyList)
+ totalTokens[0] += result[0]
+ totalTokens[1] += result[1]
+ else:
+ pbar.write(f"Warning: {filename} is not in the expected simple key-value format. Skipping.")
+ except Exception as e:
+ traceback.print_exc()
+ return [data, totalTokens, e]
+
+ return [data, totalTokens, None]
+
+
+def translateSimpleKeyValueJSON(data, filename, stringList, keyList):
+ """
+ Translate simple key-value JSON format where keys contain Japanese text
+ and values are placeholder strings like "TODO".
+
+ Format example:
+ {
+ "\\r\\nスタビライザー": "TODO",
+ "\\r\\nプラグインなし": "TODO"
+ }
+ """
+ global LOCK, ESTIMATE, FILENAME, PBAR, MISMATCH
+ tokens = [0, 0]
+
+ preparedStringList = []
+ if REPLACE_LINEBREAKS_WITH_PIPES:
+ for s in stringList:
+ preparedStringList.append(s.replace("\r\n", "|").replace("\n", "|"))
+ else:
+ preparedStringList = stringList
+
+ # Translate all keys if any were found
+ if preparedStringList:
+ PBAR.total = len(preparedStringList)
+ PBAR.refresh()
+
+ response = translateAI(preparedStringList, "Reply with the English Translation")
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ translatedList = response[0]
+
+ # Check for mismatch
+ if len(preparedStringList) != len(translatedList):
+ with LOCK:
+ if FILENAME not in MISMATCH:
+ MISMATCH.append(FILENAME)
+
+ # Pass 2: Update the values with translations
+ for i, original_key in enumerate(keyList):
+ if i < len(translatedList):
+ translatedText = translatedList[i]
+
+ if REPLACE_LINEBREAKS_WITH_PIPES:
+ translatedText = translatedText.replace("|", "\r\n")
+ # Set the translated text as the value
+ data[original_key] = translatedText
+
+ # Save progress after each translation
+ save_progress_json(data, filename)
+
+ return tokens
+
+def save_progress_json(data, filename):
+ """Save current JSON translation progress."""
+ try:
+ if ESTIMATE: return
+ os.makedirs("translated", exist_ok=True)
+ tmp_path = os.path.join("translated", f"{filename}.tmp")
+ final_path = os.path.join("translated", filename)
+ with open(tmp_path, "w", encoding="utf-8", newline="\n") as outFile:
+ json.dump(data, outFile, ensure_ascii=False, indent=4)
+ os.replace(tmp_path, final_path)
+ except Exception:
+ traceback.print_exc()
+
+
+def getResultString(translatedData, translationTime, filename):
+ cost = calculateCost(translatedData[1][0], translatedData[1][1], MODEL)
+ totalTokenstring = (
+ Fore.YELLOW + f"[Input: {translatedData[1][0]}]"
+ f"[Output: {translatedData[1][1]}]"
+ f"[Cost: ${cost:,.4f}]"
+ )
+ timeString = Fore.BLUE + f"[{round(translationTime, 1)}s]"
+
+ if translatedData[2] is None:
+ return f"{filename}: {totalTokenstring}{timeString}" + Fore.GREEN + " \u2713 " + Fore.RESET
+ else:
+ errorString = str(translatedData[2]) + Fore.RED
+ return f"{filename}: {totalTokenstring}{timeString}" + Fore.RED + " \u2717 " + errorString + Fore.RESET
+
+def translateAI(text, history, history_ctx=None):
+ """Legacy wrapper for the shared translation utility."""
+ global PBAR, MISMATCH, FILENAME
+ TRANSLATION_CONFIG.estimateMode = bool(ESTIMATE)
+ return sharedtranslateAI(
+ text=text,
+ history=history,
+ config=TRANSLATION_CONFIG,
+ filename=FILENAME,
+ pbar=PBAR,
+ lock=LOCK,
+ mismatchList=MISMATCH
)
\ No newline at end of file
diff --git a/modules/csv.py b/modules/csv.py
index 88535fd..81e36aa 100644
--- a/modules/csv.py
+++ b/modules/csv.py
@@ -1,458 +1,460 @@
-# Libraries
-import json
-import os
-import re
-import util.dazedwrap as dazedwrap
-import threading
-import time
-import traceback
-import tiktoken
-import csv
-from pathlib import Path
-from colorama import Fore
-from dotenv import load_dotenv
-from retry import retry
-from tqdm import tqdm
-from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost
-import tempfile
-
-# Globals
-MODEL = os.getenv("model")
-TIMEOUT = int(os.getenv("timeout"))
-LANGUAGE = os.getenv("language").capitalize()
-PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
-VOCAB = Path("vocab.txt").read_text(encoding="utf-8")
-LOCK = threading.Lock()
-WIDTH = int(os.getenv("width"))
-LISTWIDTH = int(os.getenv("listWidth"))
-NOTEWIDTH = int(os.getenv("noteWidth"))
-MAXHISTORY = 10
-ESTIMATE = ""
-TOKENS = [0, 0]
-NAMESLIST = []
-NAMES = False # Output a list of all the character names found
-BRFLAG = False # If the game uses
instead
-FIXTEXTWRAP = True # Overwrites textwrap
-IGNORETLTEXT = True # Ignores all translated text.
-MISMATCH = [] # Lists files that thdata a mismatch error (Length of GPT list response is wrong)
-FILENAME = None
-BRACKETNAMES = False
-
-# CSV Configuration Settings (configurable via GUI)
-CSV_DELIMITER = " " # CSV delimiter character (comma, semicolon, tab)
-SOURCE_COLUMN = 2 # Which column has the source text to translate (0-indexed)
-TARGET_COLUMN = 3 # Which column to write translations to
-SPEAKER_COLUMN = 1 # Which column has speaker names (-1 = none)
-SKIP_HEADER_ROW = False # Skip the first row (header)
-USE_TARGET_IF_NOT_EMPTY = False # Use target column text if not empty (T++ style)
-WRITE_TO_NEXT_COLUMN = False # Write to column after target instead of overwriting
-PARSE_NAME_TAGS = False # Parse :name[] tags in text
-PARSE_M_MARKERS = False # Parse \M markers in text
-REMOVE_FURIGANA = True # Remove furigana annotations ＜＝＞
-SKIP_COMMENT_ROWS = False # Skip rows starting with 'comment'
-
-# Regex - Need to change this if you want to translate from/to other languages. Default is Japanese Regex
-LANGREGEX = r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+"
-
-# Get pricing configuration based on the model
-PRICING_CONFIG = getPricingConfig(MODEL)
-INPUTAPICOST = PRICING_CONFIG["inputAPICost"]
-OUTPUTAPICOST = PRICING_CONFIG["outputAPICost"]
-BATCHSIZE = PRICING_CONFIG["batchSize"]
-FREQUENCY_PENALTY = PRICING_CONFIG["frequencyPenalty"]
-
-# tqdm Globals
-BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
-POSITION = 0
-LEAVE = False
-PBAR = None
-ENCODING = "utf8"
-
-# Initialize Translation Config
-TRANSLATION_CONFIG = TranslationConfig(
- model=MODEL,
- language=LANGUAGE,
- prompt=PROMPT,
- vocab=VOCAB,
- langRegex=LANGREGEX,
- batchSize=BATCHSIZE,
- maxHistory=MAXHISTORY,
- estimateMode=False # Will be set dynamically based on ESTIMATE
-)
-LEAVE = False
-
-def handleCSV(filename, estimate):
- global ESTIMATE, TOKENS, FILENAME
- ESTIMATE = estimate
- FILENAME = filename
-
- if not ESTIMATE:
- with open("translated/" + filename, "w+t", newline="", encoding=ENCODING, errors="xmlcharrefreplace") as writeFile:
- # Translate
- start = time.time()
- translatedData = openFiles(filename, writeFile)
-
- # Print Result
- end = time.time()
- tqdm.write(getResultString(translatedData, end - start, filename))
- with LOCK:
- TOKENS[0] += translatedData[1][0]
- TOKENS[1] += translatedData[1][1]
- else:
- # Translate
- start = time.time()
- translatedData = openFilesEstimate(filename)
-
- # Print Result
- end = time.time()
- tqdm.write(getResultString(translatedData, end - start, filename))
- with LOCK:
- TOKENS[0] += translatedData[1][0]
- TOKENS[1] += translatedData[1][1]
-
- # Print Total
- totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
-
- # Print any errors on maps
- if len(MISMATCH) > 0:
- return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
- else:
- return totalString
-
-
-def openFiles(filename, writeFile):
- with open("files/" + filename, "r", encoding=ENCODING) as readFile, writeFile:
- translatedData = parseCSV(readFile, writeFile, filename)
-
- return translatedData
-
-
-def openFilesEstimate(filename):
- with open("files/" + filename, "r", encoding="utf8") as readFile:
- translatedData = parseCSV(readFile, "", filename)
-
- return translatedData
-
-
-def getResultString(translatedData, translationTime, filename):
- # File Print String
- cost = calculateCost(translatedData[1][0], translatedData[1][1], MODEL)
- totalTokenstring = (
- Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
- "[Output: "
- + str(translatedData[1][1])
- + "]" "[Cost: ${:,.4f}".format(cost)
- + "]"
- )
- timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
-
- if translatedData[2] is None:
- # Success
- return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET
- else:
- # Fail
- try:
- raise translatedData[2]
- except Exception as e:
- traceback.print_exc()
- errorString = str(e) + Fore.RED
- return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET
-
-
-def parseCSV(readFile, writeFile, filename):
- totalTokens = [0, 0]
- totalLines = 0
- global LOCK
-
- # Get total for progress bar
- totalLines = len(readFile.readlines())
- readFile.seek(0)
-
- reader = csv.reader(readFile, delimiter=CSV_DELIMITER)
- if not ESTIMATE:
- writer = csv.writer(
- writeFile,
- delimiter=CSV_DELIMITER,
- )
- else:
- writer = ""
-
- with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar:
- pbar.desc = filename
- pbar.total = totalLines
-
- # Grab All Rows
- data = []
- for row in reader:
- data.append(row)
-
- try:
- response = translateCSV(data, pbar, writeFile, writer, filename, None)
- totalTokens[0] = response[0]
- totalTokens[1] = response[1]
- except Exception:
- traceback.print_exc()
- return [data, totalTokens, None]
-
-
-def flush_progress_csv(writeFile, writer, rows):
- """Flush current CSV progress to the already-open output file (Windows-safe)."""
- try:
- if ESTIMATE or writeFile is None:
- return
- with LOCK:
- writeFile.seek(0)
- # Recreate writer at current position to avoid state issues
- tmp_writer = csv.writer(writeFile, delimiter=CSV_DELIMITER)
- tmp_writer.writerows(rows)
- writeFile.truncate()
- writeFile.flush()
- os.fsync(writeFile.fileno())
- except Exception:
- traceback.print_exc()
-
-
-def translateCSV(data, pbar, writeFile, writer, filename, translatedList):
- """
- Unified CSV translation function using configurable settings.
-
- Uses global settings:
- - SOURCE_COLUMN: column index for source text
- - TARGET_COLUMN: column index to write translations
- - SPEAKER_COLUMN: column index for speaker names (-1 = none)
- - SKIP_HEADER_ROW: whether to skip first row
- - USE_TARGET_IF_NOT_EMPTY: use existing target text if present (T++ style)
- - WRITE_TO_NEXT_COLUMN: write to column after target
- - PARSE_NAME_TAGS: parse :name[] tags
- - PARSE_M_MARKERS: parse \\M markers
- - REMOVE_FURIGANA: remove furigana
- - SKIP_COMMENT_ROWS: skip rows with 'comment' in first column
- """
- global LOCK, ESTIMATE, PBAR
- PBAR = pbar
- translatedText = ""
- totalTokens = [0, 0]
- i = 0
- stringList = []
-
- try:
- # Translate
- while i < len(data):
- # Skip header row if configured
- if SKIP_HEADER_ROW and i == 0:
- i += 1
- continue
-
- # Skip comment rows if configured
- if SKIP_COMMENT_ROWS and len(data[i]) > 0 and 'comment' in str(data[i][0]).lower():
- i += 1
- continue
-
- # Check if row has enough columns
- if len(data[i]) <= SOURCE_COLUMN:
- i += 1
- continue
-
- # Get source text
- jaString = ""
- speaker = ""
-
- # If USE_TARGET_IF_NOT_EMPTY is enabled (T++ style), check target column first
- if USE_TARGET_IF_NOT_EMPTY and len(data[i]) > TARGET_COLUMN and data[i][TARGET_COLUMN]:
- jaString = data[i][TARGET_COLUMN]
- else:
- jaString = data[i][SOURCE_COLUMN] if data[i][SOURCE_COLUMN] else ""
-
- # Skip empty strings
- if not jaString:
- i += 1
- continue
-
- # Handle speaker column if configured
- if SPEAKER_COLUMN >= 0 and len(data[i]) > SPEAKER_COLUMN and data[i][SPEAKER_COLUMN]:
- speakerResponse = getSpeaker(data[i][SPEAKER_COLUMN])
- totalTokens[0] += speakerResponse[1][0]
- totalTokens[1] += speakerResponse[1][1]
- speaker = speakerResponse[0]
- data[i][SPEAKER_COLUMN] = speaker
-
- # Parse :name[] tags if configured
- if PARSE_NAME_TAGS and ':name' in jaString:
- match = re.search(r":name\[([^\]]+?)\]\n([\w\W]*)", jaString)
- if match:
- # Translate speaker name
- response = getSpeaker(match.group(1))
- speaker = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
- data[i][SOURCE_COLUMN] = data[i][SOURCE_COLUMN].replace(match.group(1), speaker)
-
- # Extract text portion
- jaString = match.group(2)
- # Remove voice markers
- voMatch = re.search(r"\\[vfF]+\[.+]", jaString)
- if voMatch:
- jaString = jaString.replace(voMatch.group(0), "")
-
- # Parse \M markers if configured
- if PARSE_M_MARKERS and '\\M' in jaString:
- match = re.search(r"\\M.+\n([\w\W]*)", jaString)
- if match:
- jaString = match.group(1)
- voMatch = re.search(r"\\[vfF]+\[.+]", jaString)
- if voMatch:
- jaString = jaString.replace(voMatch.group(0), "")
-
- # Remove furigana if configured
- if REMOVE_FURIGANA:
- jaString = re.sub(r"＜(.*)＝.*＞", r"\1", jaString)
-
- # Store original for replacement
- ojaString = jaString
-
- # Remove textwrap
- jaString = jaString.replace("\\n", " ")
- jaString = jaString.replace("\n", " ")
-
- # Pass 1: Collect strings
- if not translatedList:
- if speaker:
- stringList.append(f"[{speaker}]: {jaString}")
- else:
- stringList.append(jaString)
-
- # Pass 2: Apply translations
- else:
- # Grab and pop translation
- translatedText = translatedList[0]
- translatedList.pop(0)
-
- # Remove speaker prefix from translation if present
- if speaker:
- translatedText = re.sub(r"^\[?(.+?)\]?\s?[|:]\s?", "", translatedText)
-
- # Add wordwrap
- translatedText = dazedwrap.wrapText(translatedText, WIDTH)
- translatedText = translatedText.replace("\n", "\\n")
-
- # Determine target column
- actual_target = TARGET_COLUMN + 1 if WRITE_TO_NEXT_COLUMN else TARGET_COLUMN
-
- # Ensure row has enough columns
- while len(data[i]) <= actual_target:
- data[i].append("")
-
- # Set data
- if PARSE_NAME_TAGS or PARSE_M_MARKERS:
- # Replace original text portion
- data[i][actual_target] = data[i][actual_target].replace(ojaString, translatedText) if data[i][actual_target] else translatedText
- else:
- data[i][actual_target] = translatedText
-
- flush_progress_csv(writeFile, writer, data)
-
- # Iterate
- i += 1
-
- # EOF - Process collected strings
- if len(stringList) > 0:
- # Set Progress
- pbar.total = len(stringList)
- pbar.refresh()
-
- # Translate
- response = translateAI(stringList, "")
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
- translatedList = response[0]
-
- # Set Strings
- if len(stringList) == len(translatedList):
- translateCSV(data, pbar, writeFile, writer, filename, translatedList)
- # Mismatch
- else:
- with LOCK:
- if filename not in MISMATCH:
- MISMATCH.append(filename)
-
- # Write all Data
- with LOCK:
- if not ESTIMATE:
- for row in data:
- writer.writerow(row)
- flush_progress_csv(writeFile, writer, data)
-
- except Exception:
- traceback.print_exc()
-
- # Write all Data
- with LOCK:
- if not ESTIMATE:
- for row in data:
- writer.writerow(row)
- flush_progress_csv(writeFile, writer, data)
- return totalTokens
-
- return totalTokens
-
-# Save some money and enter the character before translation
-def getSpeaker(speaker):
- match speaker:
- case "ファイン":
- return ["Fine", [0, 0]]
- case "":
- return ["", [0, 0]]
- case _:
- # Find Speaker
- for i in range(len(NAMESLIST)):
- if speaker == NAMESLIST[i][0]:
- return [NAMESLIST[i][1], [0, 0]]
-
- if speaker == "？？？":
- return ["???", [0, 0]]
-
- # Translate and Store Speaker
- response = translateAI(
- f"{speaker}",
- "Reply with the " + LANGUAGE + " translation of the NPC name.",
- False,
- )
- response[0] = response[0].title()
- response[0] = response[0].replace("'S", "'s")
- response[0] = response[0].replace("Speaker: ", "")
-
- # Retry if name doesn't translate for some reason
- if re.search(r"([a-zA-Z？?])", response[0]) == None:
- response = translateAI(
- f"{speaker}",
- "Reply with the " + LANGUAGE + " translation of the NPC name.",
- False,
- )
- response[0] = response[0].title()
- response[0] = response[0].replace("'S", "'s")
-
- speakerList = [speaker, response[0]]
- NAMESLIST.append(speakerList)
- return response
- return [speaker, [0, 0]]
-
-def translateAI(text, history, history_ctx=None):
- """
- Legacy wrapper function for the new shared translation utility.
- This maintains compatibility with existing code while using the new shared implementation.
- """
- global PBAR, MISMATCH, FILENAME
-
- # Update config estimate mode based on global ESTIMATE
- TRANSLATION_CONFIG.estimateMode = bool(ESTIMATE)
-
- # Call the new shared translation function
- return sharedtranslateAI(
- text=text,
- history=history,
- config=TRANSLATION_CONFIG,
- filename=FILENAME,
- pbar=PBAR,
- lock=LOCK,
- mismatchList=MISMATCH
- )
+# Libraries
+import json
+import os
+import re
+import util.dazedwrap as dazedwrap
+import threading
+import time
+import traceback
+import tiktoken
+import csv
+from pathlib import Path
+from colorama import Fore
+from dotenv import load_dotenv
+from retry import retry
+from tqdm import tqdm
+from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost
+import tempfile
+
+# Globals
+MODEL = os.getenv("model")
+TIMEOUT = int(os.getenv("timeout"))
+LANGUAGE = os.getenv("language").capitalize()
+from util.paths import PROMPT_PATH, VOCAB_PATH
+
+PROMPT = PROMPT_PATH.read_text(encoding="utf-8")
+VOCAB = VOCAB_PATH.read_text(encoding="utf-8")
+LOCK = threading.Lock()
+WIDTH = int(os.getenv("width"))
+LISTWIDTH = int(os.getenv("listWidth"))
+NOTEWIDTH = int(os.getenv("noteWidth"))
+MAXHISTORY = 10
+ESTIMATE = ""
+TOKENS = [0, 0]
+NAMESLIST = []
+NAMES = False # Output a list of all the character names found
+BRFLAG = False # If the game uses
instead
+FIXTEXTWRAP = True # Overwrites textwrap
+IGNORETLTEXT = True # Ignores all translated text.
+MISMATCH = [] # Lists files that thdata a mismatch error (Length of GPT list response is wrong)
+FILENAME = None
+BRACKETNAMES = False
+
+# CSV Configuration Settings (configurable via GUI)
+CSV_DELIMITER = " " # CSV delimiter character (comma, semicolon, tab)
+SOURCE_COLUMN = 2 # Which column has the source text to translate (0-indexed)
+TARGET_COLUMN = 3 # Which column to write translations to
+SPEAKER_COLUMN = 1 # Which column has speaker names (-1 = none)
+SKIP_HEADER_ROW = False # Skip the first row (header)
+USE_TARGET_IF_NOT_EMPTY = False # Use target column text if not empty (T++ style)
+WRITE_TO_NEXT_COLUMN = False # Write to column after target instead of overwriting
+PARSE_NAME_TAGS = False # Parse :name[] tags in text
+PARSE_M_MARKERS = False # Parse \M markers in text
+REMOVE_FURIGANA = True # Remove furigana annotations ＜＝＞
+SKIP_COMMENT_ROWS = False # Skip rows starting with 'comment'
+
+# Regex - Need to change this if you want to translate from/to other languages. Default is Japanese Regex
+LANGREGEX = r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+"
+
+# Get pricing configuration based on the model
+PRICING_CONFIG = getPricingConfig(MODEL)
+INPUTAPICOST = PRICING_CONFIG["inputAPICost"]
+OUTPUTAPICOST = PRICING_CONFIG["outputAPICost"]
+BATCHSIZE = PRICING_CONFIG["batchSize"]
+FREQUENCY_PENALTY = PRICING_CONFIG["frequencyPenalty"]
+
+# tqdm Globals
+BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
+POSITION = 0
+LEAVE = False
+PBAR = None
+ENCODING = "utf8"
+
+# Initialize Translation Config
+TRANSLATION_CONFIG = TranslationConfig(
+ model=MODEL,
+ language=LANGUAGE,
+ prompt=PROMPT,
+ vocab=VOCAB,
+ langRegex=LANGREGEX,
+ batchSize=BATCHSIZE,
+ maxHistory=MAXHISTORY,
+ estimateMode=False # Will be set dynamically based on ESTIMATE
+)
+LEAVE = False
+
+def handleCSV(filename, estimate):
+ global ESTIMATE, TOKENS, FILENAME
+ ESTIMATE = estimate
+ FILENAME = filename
+
+ if not ESTIMATE:
+ with open("translated/" + filename, "w+t", newline="", encoding=ENCODING, errors="xmlcharrefreplace") as writeFile:
+ # Translate
+ start = time.time()
+ translatedData = openFiles(filename, writeFile)
+
+ # Print Result
+ end = time.time()
+ tqdm.write(getResultString(translatedData, end - start, filename))
+ with LOCK:
+ TOKENS[0] += translatedData[1][0]
+ TOKENS[1] += translatedData[1][1]
+ else:
+ # Translate
+ start = time.time()
+ translatedData = openFilesEstimate(filename)
+
+ # Print Result
+ end = time.time()
+ tqdm.write(getResultString(translatedData, end - start, filename))
+ with LOCK:
+ TOKENS[0] += translatedData[1][0]
+ TOKENS[1] += translatedData[1][1]
+
+ # Print Total
+ totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
+
+ # Print any errors on maps
+ if len(MISMATCH) > 0:
+ return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
+ else:
+ return totalString
+
+
+def openFiles(filename, writeFile):
+ with open("files/" + filename, "r", encoding=ENCODING) as readFile, writeFile:
+ translatedData = parseCSV(readFile, writeFile, filename)
+
+ return translatedData
+
+
+def openFilesEstimate(filename):
+ with open("files/" + filename, "r", encoding="utf8") as readFile:
+ translatedData = parseCSV(readFile, "", filename)
+
+ return translatedData
+
+
+def getResultString(translatedData, translationTime, filename):
+ # File Print String
+ cost = calculateCost(translatedData[1][0], translatedData[1][1], MODEL)
+ totalTokenstring = (
+ Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
+ "[Output: "
+ + str(translatedData[1][1])
+ + "]" "[Cost: ${:,.4f}".format(cost)
+ + "]"
+ )
+ timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
+
+ if translatedData[2] is None:
+ # Success
+ return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET
+ else:
+ # Fail
+ try:
+ raise translatedData[2]
+ except Exception as e:
+ traceback.print_exc()
+ errorString = str(e) + Fore.RED
+ return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET
+
+
+def parseCSV(readFile, writeFile, filename):
+ totalTokens = [0, 0]
+ totalLines = 0
+ global LOCK
+
+ # Get total for progress bar
+ totalLines = len(readFile.readlines())
+ readFile.seek(0)
+
+ reader = csv.reader(readFile, delimiter=CSV_DELIMITER)
+ if not ESTIMATE:
+ writer = csv.writer(
+ writeFile,
+ delimiter=CSV_DELIMITER,
+ )
+ else:
+ writer = ""
+
+ with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar:
+ pbar.desc = filename
+ pbar.total = totalLines
+
+ # Grab All Rows
+ data = []
+ for row in reader:
+ data.append(row)
+
+ try:
+ response = translateCSV(data, pbar, writeFile, writer, filename, None)
+ totalTokens[0] = response[0]
+ totalTokens[1] = response[1]
+ except Exception:
+ traceback.print_exc()
+ return [data, totalTokens, None]
+
+
+def flush_progress_csv(writeFile, writer, rows):
+ """Flush current CSV progress to the already-open output file (Windows-safe)."""
+ try:
+ if ESTIMATE or writeFile is None:
+ return
+ with LOCK:
+ writeFile.seek(0)
+ # Recreate writer at current position to avoid state issues
+ tmp_writer = csv.writer(writeFile, delimiter=CSV_DELIMITER)
+ tmp_writer.writerows(rows)
+ writeFile.truncate()
+ writeFile.flush()
+ os.fsync(writeFile.fileno())
+ except Exception:
+ traceback.print_exc()
+
+
+def translateCSV(data, pbar, writeFile, writer, filename, translatedList):
+ """
+ Unified CSV translation function using configurable settings.
+
+ Uses global settings:
+ - SOURCE_COLUMN: column index for source text
+ - TARGET_COLUMN: column index to write translations
+ - SPEAKER_COLUMN: column index for speaker names (-1 = none)
+ - SKIP_HEADER_ROW: whether to skip first row
+ - USE_TARGET_IF_NOT_EMPTY: use existing target text if present (T++ style)
+ - WRITE_TO_NEXT_COLUMN: write to column after target
+ - PARSE_NAME_TAGS: parse :name[] tags
+ - PARSE_M_MARKERS: parse \\M markers
+ - REMOVE_FURIGANA: remove furigana
+ - SKIP_COMMENT_ROWS: skip rows with 'comment' in first column
+ """
+ global LOCK, ESTIMATE, PBAR
+ PBAR = pbar
+ translatedText = ""
+ totalTokens = [0, 0]
+ i = 0
+ stringList = []
+
+ try:
+ # Translate
+ while i < len(data):
+ # Skip header row if configured
+ if SKIP_HEADER_ROW and i == 0:
+ i += 1
+ continue
+
+ # Skip comment rows if configured
+ if SKIP_COMMENT_ROWS and len(data[i]) > 0 and 'comment' in str(data[i][0]).lower():
+ i += 1
+ continue
+
+ # Check if row has enough columns
+ if len(data[i]) <= SOURCE_COLUMN:
+ i += 1
+ continue
+
+ # Get source text
+ jaString = ""
+ speaker = ""
+
+ # If USE_TARGET_IF_NOT_EMPTY is enabled (T++ style), check target column first
+ if USE_TARGET_IF_NOT_EMPTY and len(data[i]) > TARGET_COLUMN and data[i][TARGET_COLUMN]:
+ jaString = data[i][TARGET_COLUMN]
+ else:
+ jaString = data[i][SOURCE_COLUMN] if data[i][SOURCE_COLUMN] else ""
+
+ # Skip empty strings
+ if not jaString:
+ i += 1
+ continue
+
+ # Handle speaker column if configured
+ if SPEAKER_COLUMN >= 0 and len(data[i]) > SPEAKER_COLUMN and data[i][SPEAKER_COLUMN]:
+ speakerResponse = getSpeaker(data[i][SPEAKER_COLUMN])
+ totalTokens[0] += speakerResponse[1][0]
+ totalTokens[1] += speakerResponse[1][1]
+ speaker = speakerResponse[0]
+ data[i][SPEAKER_COLUMN] = speaker
+
+ # Parse :name[] tags if configured
+ if PARSE_NAME_TAGS and ':name' in jaString:
+ match = re.search(r":name\[([^\]]+?)\]\n([\w\W]*)", jaString)
+ if match:
+ # Translate speaker name
+ response = getSpeaker(match.group(1))
+ speaker = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+ data[i][SOURCE_COLUMN] = data[i][SOURCE_COLUMN].replace(match.group(1), speaker)
+
+ # Extract text portion
+ jaString = match.group(2)
+ # Remove voice markers
+ voMatch = re.search(r"\\[vfF]+\[.+]", jaString)
+ if voMatch:
+ jaString = jaString.replace(voMatch.group(0), "")
+
+ # Parse \M markers if configured
+ if PARSE_M_MARKERS and '\\M' in jaString:
+ match = re.search(r"\\M.+\n([\w\W]*)", jaString)
+ if match:
+ jaString = match.group(1)
+ voMatch = re.search(r"\\[vfF]+\[.+]", jaString)
+ if voMatch:
+ jaString = jaString.replace(voMatch.group(0), "")
+
+ # Remove furigana if configured
+ if REMOVE_FURIGANA:
+ jaString = re.sub(r"＜(.*)＝.*＞", r"\1", jaString)
+
+ # Store original for replacement
+ ojaString = jaString
+
+ # Remove textwrap
+ jaString = jaString.replace("\\n", " ")
+ jaString = jaString.replace("\n", " ")
+
+ # Pass 1: Collect strings
+ if not translatedList:
+ if speaker:
+ stringList.append(f"[{speaker}]: {jaString}")
+ else:
+ stringList.append(jaString)
+
+ # Pass 2: Apply translations
+ else:
+ # Grab and pop translation
+ translatedText = translatedList[0]
+ translatedList.pop(0)
+
+ # Remove speaker prefix from translation if present
+ if speaker:
+ translatedText = re.sub(r"^\[?(.+?)\]?\s?[|:]\s?", "", translatedText)
+
+ # Add wordwrap
+ translatedText = dazedwrap.wrapText(translatedText, WIDTH)
+ translatedText = translatedText.replace("\n", "\\n")
+
+ # Determine target column
+ actual_target = TARGET_COLUMN + 1 if WRITE_TO_NEXT_COLUMN else TARGET_COLUMN
+
+ # Ensure row has enough columns
+ while len(data[i]) <= actual_target:
+ data[i].append("")
+
+ # Set data
+ if PARSE_NAME_TAGS or PARSE_M_MARKERS:
+ # Replace original text portion
+ data[i][actual_target] = data[i][actual_target].replace(ojaString, translatedText) if data[i][actual_target] else translatedText
+ else:
+ data[i][actual_target] = translatedText
+
+ flush_progress_csv(writeFile, writer, data)
+
+ # Iterate
+ i += 1
+
+ # EOF - Process collected strings
+ if len(stringList) > 0:
+ # Set Progress
+ pbar.total = len(stringList)
+ pbar.refresh()
+
+ # Translate
+ response = translateAI(stringList, "")
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+ translatedList = response[0]
+
+ # Set Strings
+ if len(stringList) == len(translatedList):
+ translateCSV(data, pbar, writeFile, writer, filename, translatedList)
+ # Mismatch
+ else:
+ with LOCK:
+ if filename not in MISMATCH:
+ MISMATCH.append(filename)
+
+ # Write all Data
+ with LOCK:
+ if not ESTIMATE:
+ for row in data:
+ writer.writerow(row)
+ flush_progress_csv(writeFile, writer, data)
+
+ except Exception:
+ traceback.print_exc()
+
+ # Write all Data
+ with LOCK:
+ if not ESTIMATE:
+ for row in data:
+ writer.writerow(row)
+ flush_progress_csv(writeFile, writer, data)
+ return totalTokens
+
+ return totalTokens
+
+# Save some money and enter the character before translation
+def getSpeaker(speaker):
+ match speaker:
+ case "ファイン":
+ return ["Fine", [0, 0]]
+ case "":
+ return ["", [0, 0]]
+ case _:
+ # Find Speaker
+ for i in range(len(NAMESLIST)):
+ if speaker == NAMESLIST[i][0]:
+ return [NAMESLIST[i][1], [0, 0]]
+
+ if speaker == "？？？":
+ return ["???", [0, 0]]
+
+ # Translate and Store Speaker
+ response = translateAI(
+ f"{speaker}",
+ "Reply with the " + LANGUAGE + " translation of the NPC name.",
+ False,
+ )
+ response[0] = response[0].title()
+ response[0] = response[0].replace("'S", "'s")
+ response[0] = response[0].replace("Speaker: ", "")
+
+ # Retry if name doesn't translate for some reason
+ if re.search(r"([a-zA-Z？?])", response[0]) == None:
+ response = translateAI(
+ f"{speaker}",
+ "Reply with the " + LANGUAGE + " translation of the NPC name.",
+ False,
+ )
+ response[0] = response[0].title()
+ response[0] = response[0].replace("'S", "'s")
+
+ speakerList = [speaker, response[0]]
+ NAMESLIST.append(speakerList)
+ return response
+ return [speaker, [0, 0]]
+
+def translateAI(text, history, history_ctx=None):
+ """
+ Legacy wrapper function for the new shared translation utility.
+ This maintains compatibility with existing code while using the new shared implementation.
+ """
+ global PBAR, MISMATCH, FILENAME
+
+ # Update config estimate mode based on global ESTIMATE
+ TRANSLATION_CONFIG.estimateMode = bool(ESTIMATE)
+
+ # Call the new shared translation function
+ return sharedtranslateAI(
+ text=text,
+ history=history,
+ config=TRANSLATION_CONFIG,
+ filename=FILENAME,
+ pbar=PBAR,
+ lock=LOCK,
+ mismatchList=MISMATCH
+ )
diff --git a/modules/images.py b/modules/images.py
index e6fd04b..65af059 100644
--- a/modules/images.py
+++ b/modules/images.py
@@ -1,439 +1,441 @@
-# Libraries
-from PIL import Image, ImageDraw, ImageFont
-import json
-import os
-import re
-import threading
-import time
-import traceback
-import tiktoken
-from pathlib import Path
-from colorama import Fore
-from dotenv import load_dotenv
-from retry import retry
-from tqdm import tqdm
-from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost
-
-# Globals
-MODEL = os.getenv("model")
-TIMEOUT = int(os.getenv("timeout"))
-LANGUAGE = os.getenv("language").capitalize()
-PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
-VOCAB = Path("vocab.txt").read_text(encoding="utf-8")
-LOCK = threading.Lock()
-PBAR = None
-WIDTH = int(os.getenv("width"))
-LISTWIDTH = int(os.getenv("listWidth"))
-NOTEWIDTH = int(os.getenv("noteWidth"))
-MAXHISTORY = 10
-ESTIMATE = ""
-TOKENS = [0, 0]
-NAMESLIST = []
-MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
-FILENAME = None
-
-# Regex - Need to change this if you want to translate from/to other languages. Default is Japanese Regex
-LANGREGEX = r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+"
-
-# Get pricing configuration based on the model
-PRICING_CONFIG = getPricingConfig(MODEL)
-INPUTAPICOST = PRICING_CONFIG["inputAPICost"]
-OUTPUTAPICOST = PRICING_CONFIG["outputAPICost"]
-BATCHSIZE = PRICING_CONFIG["batchSize"]
-FREQUENCY_PENALTY = PRICING_CONFIG["frequencyPenalty"]
-
-# Initialize Translation Config
-TRANSLATION_CONFIG = TranslationConfig(
- model=MODEL,
- language=LANGUAGE,
- prompt=PROMPT,
- vocab=VOCAB,
- langRegex=LANGREGEX,
- batchSize=BATCHSIZE,
- maxHistory=MAXHISTORY,
- estimateMode=False # Will be set dynamically based on ESTIMATE
-)
-LEAVE = False
-
-# tqdm Globals
-BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
-POSITION = 0
-LEAVE = False
-
-
-def handleImages(folderName, estimate):
- global ESTIMATE, TOKENS, FILENAME
- ESTIMATE = estimate
- FILENAME = folderName
- start = time.time()
-
- # Translate Strings
- translatedData = openFiles(f"files/{folderName}")
-
- # Custom Names
- # customList = [[], []]
- # customList = processImagesDir("Custom", customList)
-
- # Write TL To Images
- try:
- translatedList, originalList, dimensionsList = translatedData[0]
- for i in range(len(translatedList)):
- try:
- # Create image from string
- image = stringToImageOutline(translatedList[i], dimensionsList[i][0], dimensionsList[i][1])
- # Save image using the corresponding original filename
- image.save(rf"translated/{folderName}/{originalList[i]}.png", quality=100)
- except Exception as e:
- # Log error if image saving fails
- PBAR.write(f"Error processing {translatedList[i]}: {str(e)}")
- except IndexError:
- PBAR.write("Translated data is incomplete. Please check your input.")
-
- # Print File
- end = time.time()
- tqdm.write(getResultString(translatedData, end - start, folderName))
- with LOCK:
- TOKENS[0] += translatedData[1][0]
- TOKENS[1] += translatedData[1][1]
-
- # Print Total
- totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
-
- # Print any errors on maps
- if len(MISMATCH) > 0:
- return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
- else:
- return totalString
-
-
-def openFiles(folderName):
- global PBAR
-
- if os.path.isdir(folderName):
- imageList = [[], [], []]
- imageList = processImagesDir(folderName, imageList)
-
- # Start Translation
- with tqdm(
- bar_format=BAR_FORMAT,
- position=POSITION,
- leave=LEAVE,
- desc=folderName,
- total=len(imageList[0]),
- ) as PBAR:
- translatedData = translateImages(imageList)
- translatedData = [
- [translatedData[0], imageList[2], imageList[1]],
- translatedData[1],
- translatedData[2],
- ]
-
- return translatedData
- else:
- print("The provided directory path does not exist.")
-
-
-def getResultString(translatedData, translationTime, filename):
- # File Print String
- cost = calculateCost(translatedData[1][0], translatedData[1][1], MODEL)
- totalTokenstring = (
- Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
- "[Output: "
- + str(translatedData[1][1])
- + "]" "[Cost: ${:,.4f}".format(cost)
- + "]"
- )
- timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
-
- if translatedData[2] is None:
- # Success
- return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET
- else:
- # Fail
- try:
- raise translatedData[2]
- except Exception as e:
- traceback.print_exc()
- errorString = str(e) + Fore.RED
- return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET
-
-
-def getFontSize(text, image_width, image_height, font_path):
- # Start with a high font size and keep reducing it until the text fits within the image bounds
- font_size = min(image_width, image_height)
-
- while font_size > 0:
- font = ImageFont.truetype(font_path, font_size)
- text_bbox = ImageDraw.Draw(Image.new("RGB", (1, 1))).textbbox((0, 0), text, font=font)
- text_width = text_bbox[2] - text_bbox[0]
- text_height = text_bbox[3] - text_bbox[1]
-
- if text_width <= image_width and text_height <= image_height:
- return font_size
- font_size -= 1
-
- return font_size
-
-
-def stringToImage(text, width, height, font_path="fonts/TsunagiGothic.ttf", scale_factor=4):
- # Increase the resolution
- scaled_width = int(width * scale_factor)
- scaled_height = int(height * scale_factor)
-
- # Find the appropriate font size for the scaled up image
- font_size = getFontSize(text, scaled_width, scaled_height, font_path)
- if font_size == 0:
- raise ValueError("Text is too long to fit in the supplied dimensions.")
-
- # Create a new image with the scaled width and height and a transparent background
- image = Image.new("RGBA", (scaled_width, scaled_height), (255, 255, 255, 0))
-
- # Create a drawing context
- draw = ImageDraw.Draw(image)
-
- # Load the appropriate font
- font = ImageFont.truetype(font_path, font_size)
-
- # Calculate the size of the text to center it
- text_bbox = draw.textbbox((0, 0), text, font=font)
- text_width = text_bbox[2] - text_bbox[0]
- text_height = text_bbox[3] - text_bbox[1] + 20
- x = 0
-
- x = (scaled_width - text_width) // 2
- y = (scaled_height - text_height) // 2
-
- # Draw the text on the image
- draw.text((x, y), text, font=font, fill=(255, 255, 255, 255))
-
- # Resize back to the original dimensions to get a clearer text rendering
- image = image.resize(
- (width, height),
- Image.LANCZOS,
- )
-
- return image
-
-
-from PIL import Image, ImageDraw, ImageFont
-
-
-def stringToImageOutline(text, width, height, font_path="fonts/TsunagiGothic.ttf", scale_factor=4):
- # Outline
- outline_color = (255, 255, 255, 255)
- text_color = (0, 0, 0, 255)
- outline_thickness = 4
-
- # Increase the resolution
- scaled_width = int(width * scale_factor)
- scaled_height = int(height * scale_factor)
-
- # Find the appropriate font size for the scaled up image
- font_size = getFontSize(text, scaled_width, scaled_height, font_path)
- if font_size == 0:
- raise ValueError("Text is too long to fit in the supplied dimensions.")
-
- # Create a new image with the scaled width and height and a transparent background
- image = Image.new("RGBA", (scaled_width, scaled_height), (255, 255, 255, 0))
-
- # Create a drawing context
- draw = ImageDraw.Draw(image)
-
- # Load the appropriate font
- font = ImageFont.truetype(font_path, font_size)
-
- # Calculate the size of the text to center it
- text_bbox = draw.textbbox((0, 0), text, font=font)
- text_width = text_bbox[2] - text_bbox[0]
- text_height = text_bbox[3] - text_bbox[1] + 20
- x = (scaled_width - text_width) // 2
- y = (scaled_height - text_height) // 2
-
- # Draw the text outline by applying the text multiple times with small offsets
- for dx in range(-outline_thickness, outline_thickness + 1):
- for dy in range(-outline_thickness, outline_thickness + 1):
- if dx != 0 or dy != 0:
- draw.text((x + dx, y + dy), text, font=font, fill=outline_color)
-
- # Draw the main text
- draw.text((x, y), text, font=font, fill=text_color)
-
- # Resize back to the original dimensions to get a clearer text rendering
- image = image.resize((width, height), Image.LANCZOS)
-
- return image
-
-
-def stringToImageBox(text, width, height, font_path="fonts/TsunagiGothic.ttf", scale_factor=4):
- # Increase the resolution
- scaled_width = int(width * scale_factor)
- scaled_height = int(height * scale_factor)
-
- # Padding around the text
- padding = 10
-
- # Calculate the dimensions available for text placement
- available_width = scaled_width - 2 * padding
- available_height = scaled_height - 2 * padding
-
- # Determine the best font size to fit within the available dimensions
- font_size = getFontSize(text, available_width, available_height, font_path)
- if font_size <= 0:
- raise ValueError("Text is too long to fit in the supplied dimensions.")
-
- # Create a new image with increased resolution
- image = Image.new("RGBA", (scaled_width, scaled_height), (255, 255, 255, 0))
- draw = ImageDraw.Draw(image)
-
- # Load the calculated font
- font = ImageFont.truetype(font_path, font_size)
-
- # Calculate the size and bounding box of the text
- text_bbox = draw.textbbox((0, 0), text, font=font)
- text_width = text_bbox[2] - text_bbox[0]
- text_height = text_bbox[3] - text_bbox[1] + 20
-
- # Determine centered position for the text while considering padding
- # Additional adjustment ensures text appears centrally aligned
- x = (scaled_width - text_width) // 2
- y = (scaled_height - text_height) // 2
-
- # Draw a black box with a white outline that fits the image dimensions precisely
- draw.rectangle([0, 0, scaled_width - 1, scaled_height - 1], outline=(255, 255, 255, 255), width=1)
-
- # Fill the inside box with black color
- draw.rectangle([1, 1, scaled_width - 2, scaled_height - 2], fill=(0, 0, 0, 255))
-
- # Render the text within the image
- draw.text((x, y), text, font=font, fill=(255, 255, 255, 255))
-
- # Shrink the image back to original dimensions with high-quality interpolation
- image = image.resize(
- (width, height),
- Image.LANCZOS,
- )
-
- return image
-
-
-def getImageDimensions(file_path):
- try:
- with Image.open(file_path) as img:
- width, height = img.size
- return width, height
- except Exception as e:
- print(f"Error reading {file_path}: {e}")
- return None, None
-
-
-def processImagesDir(directory_path, imageList):
- for file_name in os.listdir(directory_path):
- # .png and Japanese
- if ".png" in file_name:
- file_path = os.path.join(directory_path, file_name)
- if os.path.isfile(file_path):
- # Check if the file is an image
- try:
- width, height = getImageDimensions(file_path)
- if width is not None and height is not None:
- placeholders = {
- ".png": "",
- }
- for target, replacement in placeholders.items():
- file_name = file_name.replace(target, replacement)
- match = re.search(r"[\[【].+?[\]】](.*)", file_name)
- if match:
- text = match.group(1)
- else:
- text = file_name
- imageList[0].append(text)
- imageList[1].append([width, height])
- imageList[2].append(file_name)
- except Exception as e:
- print(f"Error processing {file_name}: {e}")
-
- if ".txt" in file_name:
- try:
- with open(f"{directory_path}/{file_name}", "r", encoding="utf8") as file:
- for line in file:
- line = line.strip()
- line = line.replace(":", "：")
- line = line.replace("/", "／")
- line = line.replace("?", "？")
- imageList[0].append(line) # Using strip() to remove any extra newlines or spaces
- imageList[1].append([104, 15])
- except FileNotFoundError:
- print(f"The file at {file_path} was not found.")
- except IOError:
- print(f"An error occurred while reading the file at {file_path}.")
- return imageList
-
-
-def translateImages(imageList):
- totalTokens = [0, 0]
-
- # Translate GPT
- response = translateAI(imageList[0], "Keep the Translation as brief as possible")
- translatedList = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- return [translatedList, totalTokens, None]
-
-# Save some money and enter the character before translation
-def getSpeaker(speaker):
- match speaker:
- case "ファイン":
- return ["Fine", [0, 0]]
- case "":
- return ["", [0, 0]]
- case _:
- # Find Speaker
- for i in range(len(NAMESLIST)):
- if speaker == NAMESLIST[i][0]:
- return [NAMESLIST[i][1], [0, 0]]
-
- # Translate and Store Speaker
- response = translateAI(
- f"{speaker}",
- "Reply with the " + LANGUAGE + " translation of the NPC name.",
- False,
- )
- response[0] = response[0].title()
- response[0] = response[0].replace("'S", "'s")
- response[0] = response[0].replace("Speaker: ", "")
-
- # Retry if name doesn't translate for some reason
- if re.search(r"([a-zA-Z？?])", response[0]) == None:
- response = translateAI(
- f"{speaker}",
- "Reply with the " + LANGUAGE + " translation of the NPC name.",
- False,
- )
- response[0] = response[0].title()
- response[0] = response[0].replace("'S", "'s")
-
- speakerList = [speaker, response[0]]
- NAMESLIST.append(speakerList)
- return response
- return [speaker, [0, 0]]
-def translateAI(text, history, history_ctx=None):
- """
- Legacy wrapper function for the new shared translation utility.
- This maintains compatibility with existing code while using the new shared implementation.
- """
- global PBAR, MISMATCH, FILENAME
-
- # Update config estimate mode based on global ESTIMATE
- TRANSLATION_CONFIG.estimateMode = bool(ESTIMATE)
-
- # Call the new shared translation function
- return sharedtranslateAI(
- text=text,
- history=history,
- config=TRANSLATION_CONFIG,
- filename=FILENAME,
- pbar=PBAR,
- lock=LOCK,
- mismatchList=MISMATCH
- )
+# Libraries
+from PIL import Image, ImageDraw, ImageFont
+import json
+import os
+import re
+import threading
+import time
+import traceback
+import tiktoken
+from pathlib import Path
+from colorama import Fore
+from dotenv import load_dotenv
+from retry import retry
+from tqdm import tqdm
+from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost
+
+# Globals
+MODEL = os.getenv("model")
+TIMEOUT = int(os.getenv("timeout"))
+LANGUAGE = os.getenv("language").capitalize()
+from util.paths import PROMPT_PATH, VOCAB_PATH
+
+PROMPT = PROMPT_PATH.read_text(encoding="utf-8")
+VOCAB = VOCAB_PATH.read_text(encoding="utf-8")
+LOCK = threading.Lock()
+PBAR = None
+WIDTH = int(os.getenv("width"))
+LISTWIDTH = int(os.getenv("listWidth"))
+NOTEWIDTH = int(os.getenv("noteWidth"))
+MAXHISTORY = 10
+ESTIMATE = ""
+TOKENS = [0, 0]
+NAMESLIST = []
+MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
+FILENAME = None
+
+# Regex - Need to change this if you want to translate from/to other languages. Default is Japanese Regex
+LANGREGEX = r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+"
+
+# Get pricing configuration based on the model
+PRICING_CONFIG = getPricingConfig(MODEL)
+INPUTAPICOST = PRICING_CONFIG["inputAPICost"]
+OUTPUTAPICOST = PRICING_CONFIG["outputAPICost"]
+BATCHSIZE = PRICING_CONFIG["batchSize"]
+FREQUENCY_PENALTY = PRICING_CONFIG["frequencyPenalty"]
+
+# Initialize Translation Config
+TRANSLATION_CONFIG = TranslationConfig(
+ model=MODEL,
+ language=LANGUAGE,
+ prompt=PROMPT,
+ vocab=VOCAB,
+ langRegex=LANGREGEX,
+ batchSize=BATCHSIZE,
+ maxHistory=MAXHISTORY,
+ estimateMode=False # Will be set dynamically based on ESTIMATE
+)
+LEAVE = False
+
+# tqdm Globals
+BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
+POSITION = 0
+LEAVE = False
+
+
+def handleImages(folderName, estimate):
+ global ESTIMATE, TOKENS, FILENAME
+ ESTIMATE = estimate
+ FILENAME = folderName
+ start = time.time()
+
+ # Translate Strings
+ translatedData = openFiles(f"files/{folderName}")
+
+ # Custom Names
+ # customList = [[], []]
+ # customList = processImagesDir("Custom", customList)
+
+ # Write TL To Images
+ try:
+ translatedList, originalList, dimensionsList = translatedData[0]
+ for i in range(len(translatedList)):
+ try:
+ # Create image from string
+ image = stringToImageOutline(translatedList[i], dimensionsList[i][0], dimensionsList[i][1])
+ # Save image using the corresponding original filename
+ image.save(rf"translated/{folderName}/{originalList[i]}.png", quality=100)
+ except Exception as e:
+ # Log error if image saving fails
+ PBAR.write(f"Error processing {translatedList[i]}: {str(e)}")
+ except IndexError:
+ PBAR.write("Translated data is incomplete. Please check your input.")
+
+ # Print File
+ end = time.time()
+ tqdm.write(getResultString(translatedData, end - start, folderName))
+ with LOCK:
+ TOKENS[0] += translatedData[1][0]
+ TOKENS[1] += translatedData[1][1]
+
+ # Print Total
+ totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
+
+ # Print any errors on maps
+ if len(MISMATCH) > 0:
+ return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
+ else:
+ return totalString
+
+
+def openFiles(folderName):
+ global PBAR
+
+ if os.path.isdir(folderName):
+ imageList = [[], [], []]
+ imageList = processImagesDir(folderName, imageList)
+
+ # Start Translation
+ with tqdm(
+ bar_format=BAR_FORMAT,
+ position=POSITION,
+ leave=LEAVE,
+ desc=folderName,
+ total=len(imageList[0]),
+ ) as PBAR:
+ translatedData = translateImages(imageList)
+ translatedData = [
+ [translatedData[0], imageList[2], imageList[1]],
+ translatedData[1],
+ translatedData[2],
+ ]
+
+ return translatedData
+ else:
+ print("The provided directory path does not exist.")
+
+
+def getResultString(translatedData, translationTime, filename):
+ # File Print String
+ cost = calculateCost(translatedData[1][0], translatedData[1][1], MODEL)
+ totalTokenstring = (
+ Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
+ "[Output: "
+ + str(translatedData[1][1])
+ + "]" "[Cost: ${:,.4f}".format(cost)
+ + "]"
+ )
+ timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
+
+ if translatedData[2] is None:
+ # Success
+ return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET
+ else:
+ # Fail
+ try:
+ raise translatedData[2]
+ except Exception as e:
+ traceback.print_exc()
+ errorString = str(e) + Fore.RED
+ return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET
+
+
+def getFontSize(text, image_width, image_height, font_path):
+ # Start with a high font size and keep reducing it until the text fits within the image bounds
+ font_size = min(image_width, image_height)
+
+ while font_size > 0:
+ font = ImageFont.truetype(font_path, font_size)
+ text_bbox = ImageDraw.Draw(Image.new("RGB", (1, 1))).textbbox((0, 0), text, font=font)
+ text_width = text_bbox[2] - text_bbox[0]
+ text_height = text_bbox[3] - text_bbox[1]
+
+ if text_width <= image_width and text_height <= image_height:
+ return font_size
+ font_size -= 1
+
+ return font_size
+
+
+def stringToImage(text, width, height, font_path="fonts/TsunagiGothic.ttf", scale_factor=4):
+ # Increase the resolution
+ scaled_width = int(width * scale_factor)
+ scaled_height = int(height * scale_factor)
+
+ # Find the appropriate font size for the scaled up image
+ font_size = getFontSize(text, scaled_width, scaled_height, font_path)
+ if font_size == 0:
+ raise ValueError("Text is too long to fit in the supplied dimensions.")
+
+ # Create a new image with the scaled width and height and a transparent background
+ image = Image.new("RGBA", (scaled_width, scaled_height), (255, 255, 255, 0))
+
+ # Create a drawing context
+ draw = ImageDraw.Draw(image)
+
+ # Load the appropriate font
+ font = ImageFont.truetype(font_path, font_size)
+
+ # Calculate the size of the text to center it
+ text_bbox = draw.textbbox((0, 0), text, font=font)
+ text_width = text_bbox[2] - text_bbox[0]
+ text_height = text_bbox[3] - text_bbox[1] + 20
+ x = 0
+
+ x = (scaled_width - text_width) // 2
+ y = (scaled_height - text_height) // 2
+
+ # Draw the text on the image
+ draw.text((x, y), text, font=font, fill=(255, 255, 255, 255))
+
+ # Resize back to the original dimensions to get a clearer text rendering
+ image = image.resize(
+ (width, height),
+ Image.LANCZOS,
+ )
+
+ return image
+
+
+from PIL import Image, ImageDraw, ImageFont
+
+
+def stringToImageOutline(text, width, height, font_path="fonts/TsunagiGothic.ttf", scale_factor=4):
+ # Outline
+ outline_color = (255, 255, 255, 255)
+ text_color = (0, 0, 0, 255)
+ outline_thickness = 4
+
+ # Increase the resolution
+ scaled_width = int(width * scale_factor)
+ scaled_height = int(height * scale_factor)
+
+ # Find the appropriate font size for the scaled up image
+ font_size = getFontSize(text, scaled_width, scaled_height, font_path)
+ if font_size == 0:
+ raise ValueError("Text is too long to fit in the supplied dimensions.")
+
+ # Create a new image with the scaled width and height and a transparent background
+ image = Image.new("RGBA", (scaled_width, scaled_height), (255, 255, 255, 0))
+
+ # Create a drawing context
+ draw = ImageDraw.Draw(image)
+
+ # Load the appropriate font
+ font = ImageFont.truetype(font_path, font_size)
+
+ # Calculate the size of the text to center it
+ text_bbox = draw.textbbox((0, 0), text, font=font)
+ text_width = text_bbox[2] - text_bbox[0]
+ text_height = text_bbox[3] - text_bbox[1] + 20
+ x = (scaled_width - text_width) // 2
+ y = (scaled_height - text_height) // 2
+
+ # Draw the text outline by applying the text multiple times with small offsets
+ for dx in range(-outline_thickness, outline_thickness + 1):
+ for dy in range(-outline_thickness, outline_thickness + 1):
+ if dx != 0 or dy != 0:
+ draw.text((x + dx, y + dy), text, font=font, fill=outline_color)
+
+ # Draw the main text
+ draw.text((x, y), text, font=font, fill=text_color)
+
+ # Resize back to the original dimensions to get a clearer text rendering
+ image = image.resize((width, height), Image.LANCZOS)
+
+ return image
+
+
+def stringToImageBox(text, width, height, font_path="fonts/TsunagiGothic.ttf", scale_factor=4):
+ # Increase the resolution
+ scaled_width = int(width * scale_factor)
+ scaled_height = int(height * scale_factor)
+
+ # Padding around the text
+ padding = 10
+
+ # Calculate the dimensions available for text placement
+ available_width = scaled_width - 2 * padding
+ available_height = scaled_height - 2 * padding
+
+ # Determine the best font size to fit within the available dimensions
+ font_size = getFontSize(text, available_width, available_height, font_path)
+ if font_size <= 0:
+ raise ValueError("Text is too long to fit in the supplied dimensions.")
+
+ # Create a new image with increased resolution
+ image = Image.new("RGBA", (scaled_width, scaled_height), (255, 255, 255, 0))
+ draw = ImageDraw.Draw(image)
+
+ # Load the calculated font
+ font = ImageFont.truetype(font_path, font_size)
+
+ # Calculate the size and bounding box of the text
+ text_bbox = draw.textbbox((0, 0), text, font=font)
+ text_width = text_bbox[2] - text_bbox[0]
+ text_height = text_bbox[3] - text_bbox[1] + 20
+
+ # Determine centered position for the text while considering padding
+ # Additional adjustment ensures text appears centrally aligned
+ x = (scaled_width - text_width) // 2
+ y = (scaled_height - text_height) // 2
+
+ # Draw a black box with a white outline that fits the image dimensions precisely
+ draw.rectangle([0, 0, scaled_width - 1, scaled_height - 1], outline=(255, 255, 255, 255), width=1)
+
+ # Fill the inside box with black color
+ draw.rectangle([1, 1, scaled_width - 2, scaled_height - 2], fill=(0, 0, 0, 255))
+
+ # Render the text within the image
+ draw.text((x, y), text, font=font, fill=(255, 255, 255, 255))
+
+ # Shrink the image back to original dimensions with high-quality interpolation
+ image = image.resize(
+ (width, height),
+ Image.LANCZOS,
+ )
+
+ return image
+
+
+def getImageDimensions(file_path):
+ try:
+ with Image.open(file_path) as img:
+ width, height = img.size
+ return width, height
+ except Exception as e:
+ print(f"Error reading {file_path}: {e}")
+ return None, None
+
+
+def processImagesDir(directory_path, imageList):
+ for file_name in os.listdir(directory_path):
+ # .png and Japanese
+ if ".png" in file_name:
+ file_path = os.path.join(directory_path, file_name)
+ if os.path.isfile(file_path):
+ # Check if the file is an image
+ try:
+ width, height = getImageDimensions(file_path)
+ if width is not None and height is not None:
+ placeholders = {
+ ".png": "",
+ }
+ for target, replacement in placeholders.items():
+ file_name = file_name.replace(target, replacement)
+ match = re.search(r"[\[【].+?[\]】](.*)", file_name)
+ if match:
+ text = match.group(1)
+ else:
+ text = file_name
+ imageList[0].append(text)
+ imageList[1].append([width, height])
+ imageList[2].append(file_name)
+ except Exception as e:
+ print(f"Error processing {file_name}: {e}")
+
+ if ".txt" in file_name:
+ try:
+ with open(f"{directory_path}/{file_name}", "r", encoding="utf8") as file:
+ for line in file:
+ line = line.strip()
+ line = line.replace(":", "：")
+ line = line.replace("/", "／")
+ line = line.replace("?", "？")
+ imageList[0].append(line) # Using strip() to remove any extra newlines or spaces
+ imageList[1].append([104, 15])
+ except FileNotFoundError:
+ print(f"The file at {file_path} was not found.")
+ except IOError:
+ print(f"An error occurred while reading the file at {file_path}.")
+ return imageList
+
+
+def translateImages(imageList):
+ totalTokens = [0, 0]
+
+ # Translate GPT
+ response = translateAI(imageList[0], "Keep the Translation as brief as possible")
+ translatedList = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ return [translatedList, totalTokens, None]
+
+# Save some money and enter the character before translation
+def getSpeaker(speaker):
+ match speaker:
+ case "ファイン":
+ return ["Fine", [0, 0]]
+ case "":
+ return ["", [0, 0]]
+ case _:
+ # Find Speaker
+ for i in range(len(NAMESLIST)):
+ if speaker == NAMESLIST[i][0]:
+ return [NAMESLIST[i][1], [0, 0]]
+
+ # Translate and Store Speaker
+ response = translateAI(
+ f"{speaker}",
+ "Reply with the " + LANGUAGE + " translation of the NPC name.",
+ False,
+ )
+ response[0] = response[0].title()
+ response[0] = response[0].replace("'S", "'s")
+ response[0] = response[0].replace("Speaker: ", "")
+
+ # Retry if name doesn't translate for some reason
+ if re.search(r"([a-zA-Z？?])", response[0]) == None:
+ response = translateAI(
+ f"{speaker}",
+ "Reply with the " + LANGUAGE + " translation of the NPC name.",
+ False,
+ )
+ response[0] = response[0].title()
+ response[0] = response[0].replace("'S", "'s")
+
+ speakerList = [speaker, response[0]]
+ NAMESLIST.append(speakerList)
+ return response
+ return [speaker, [0, 0]]
+def translateAI(text, history, history_ctx=None):
+ """
+ Legacy wrapper function for the new shared translation utility.
+ This maintains compatibility with existing code while using the new shared implementation.
+ """
+ global PBAR, MISMATCH, FILENAME
+
+ # Update config estimate mode based on global ESTIMATE
+ TRANSLATION_CONFIG.estimateMode = bool(ESTIMATE)
+
+ # Call the new shared translation function
+ return sharedtranslateAI(
+ text=text,
+ history=history,
+ config=TRANSLATION_CONFIG,
+ filename=FILENAME,
+ pbar=PBAR,
+ lock=LOCK,
+ mismatchList=MISMATCH
+ )
diff --git a/modules/json.py b/modules/json.py
index 8be6378..e306dc3 100644
--- a/modules/json.py
+++ b/modules/json.py
@@ -1,1359 +1,1361 @@
-# Libraries
-import json
-import os
-import re
-import util.dazedwrap as dazedwrap
-import threading
-import time
-import traceback
-import tiktoken
-from pathlib import Path
-from colorama import Fore
-from dotenv import load_dotenv
-from retry import retry
-from tqdm import tqdm
-from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost, getPricingConfig, calculateCost
-import tempfile
-
-# Globals
-MODEL = os.getenv("model")
-TIMEOUT = int(os.getenv("timeout"))
-LANGUAGE = os.getenv("language").capitalize()
-PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
-VOCAB = Path("vocab.txt").read_text(encoding="utf-8")
-LOCK = threading.Lock()
-WIDTH = int(os.getenv("width"))
-LISTWIDTH = int(os.getenv("listWidth"))
-NOTEWIDTH = 70
-MAXHISTORY = 10
-ESTIMATE = ""
-TOKENS = [0, 0]
-NAMESLIST = []
-FILENAME = None
-NAMES = False # Output a list of all the character names found
-BRFLAG = False # If the game uses
instead
-FIXTEXTWRAP = True # Overwrites textwrap
-IGNORETLTEXT = False # Ignores all translated text.
-MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
-
-# tqdm Globals
-BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
-POSITION = 0
-LEAVE = False
-PBAR = None
-
-# Regex - Need to change this if you want to translate from/to other languages. Default is Japanese Regex
-LANGREGEX = r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+"
-
-# Get pricing configuration based on the model
-PRICING_CONFIG = getPricingConfig(MODEL)
-INPUTAPICOST = PRICING_CONFIG["inputAPICost"]
-OUTPUTAPICOST = PRICING_CONFIG["outputAPICost"]
-BATCHSIZE = PRICING_CONFIG["batchSize"]
-FREQUENCY_PENALTY = PRICING_CONFIG["frequencyPenalty"]
-
-# Initialize Translation Config
-TRANSLATION_CONFIG = TranslationConfig(
- model=MODEL,
- language=LANGUAGE,
- prompt=PROMPT,
- vocab=VOCAB,
- langRegex=LANGREGEX,
- batchSize=BATCHSIZE,
- maxHistory=MAXHISTORY,
- estimateMode=False # Will be set dynamically based on ESTIMATE
-)
-LEAVE = False
-
-def handleJSON(filename, estimate):
- global ESTIMATE, totalTokens, FILENAME
- ESTIMATE = estimate
- FILENAME = filename
-
- if estimate:
- start = time.time()
- translatedData = openFiles(filename)
-
- # Print Result
- end = time.time()
- tqdm.write(getResultString(translatedData, end - start, filename))
- with LOCK:
- TOKENS[0] += translatedData[1][0]
- TOKENS[1] += translatedData[1][1]
-
- return getResultString(["", TOKENS, None], end - start, "TOTAL")
-
- else:
- try:
- start = time.time()
- translatedData = openFiles(filename)
-
- # Write final result after translation is complete
- with open("translated/" + filename, "w", encoding="utf-8", newline="\n") as outFile:
- json.dump(translatedData[0], outFile, ensure_ascii=False, indent=4)
-
- # Print Result
- end = time.time()
- tqdm.write(getResultString(translatedData, end - start, filename))
- with LOCK:
- TOKENS[0] += translatedData[1][0]
- TOKENS[1] += translatedData[1][1]
- except Exception:
- return "Fail"
-
- return getResultString(["", TOKENS, None], end - start, "TOTAL")
-
-
-def openFiles(filename):
- with open("files/" + filename, "r", encoding="UTF-8-sig") as f:
- data = json.load(f)
-
- # Map Files
- if ".json" in filename:
- translatedData = parseJSON(data, filename)
-
- else:
- raise NameError(filename + " Not Supported")
-
- return translatedData
-
-
-def getResultString(translatedData, translationTime, filename):
- # File Print String
- cost = calculateCost(translatedData[1][0], translatedData[1][1], MODEL)
- totalTokenstring = (
- Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
- "[Output: "
- + str(translatedData[1][1])
- + "]" "[Cost: ${:,.4f}".format(cost)
- + "]"
- )
- timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
-
- if translatedData[2] == None:
- # Success
- return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET
-
- else:
- # Fail
- try:
- raise translatedData[2]
- except Exception as e:
- traceback.print_exc()
- errorString = str(e) + Fore.RED
- return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET
-
-
-def parseJSON(data, filename):
- totalTokens = [0, 0]
- totalLines = 0
- totalLines = len(data)
- global LOCK, PBAR
-
- with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar:
- pbar.desc = filename
- PBAR = pbar
- try:
- # Check if data is a simple key-value dict (not a list)
- if isinstance(data, dict) and not isinstance(data, list):
- # Check if it's a simple key-value format (all values are strings, not nested objects)
- if all(isinstance(v, str) for v in data.values()):
- result = translateSimpleKeyValueJSON(data, filename)
- # PSData format (Nupu_PSData.json - MailDatas, MemoryData, ShopDatas, hStData)
- elif any(k in data for k in ["MailDatas", "MemoryData", "ShopDatas", "hStData"]):
- result = translatePSData(data, filename)
- # RdData format (RdData.json - dgnDatas with dungeon/dialogue data)
- elif "dgnDatas" in data:
- result = translateRdData(data, filename)
- # GameSetting format (Nupu_GameSetting.json - fzCardDatas, fzDeckDatas, rentalDecks)
- elif any(k in data for k in ["fzCardDatas", "fzDeckDatas", "rentalDecks"]):
- result = translateGameSetting(data, filename)
- # Src/Tl dialogue format (numeric-keyed entries with src and tl fields)
- elif data and all(isinstance(v, dict) and "src" in v for v in data.values()):
- result = translateDialogueSrcTl(data, filename)
- else:
- result = translateJSON(data, filename, [])
- else:
- result = translateJSON(data, filename, [])
- totalTokens[0] += result[0]
- totalTokens[1] += result[1]
- except Exception as e:
- traceback.print_exc()
- return [data, totalTokens, e]
- return [data, totalTokens, None]
-
-
-def save_progress_json(data, filename):
- """Save current JSON translation progress."""
- try:
- if ESTIMATE:
- return
- os.makedirs("translated", exist_ok=True)
- tmp_path = os.path.join("translated", f"{filename}.tmp")
- final_path = os.path.join("translated", filename)
- with open(tmp_path, "w", encoding="utf-8", newline="\n") as outFile:
- json.dump(data, outFile, ensure_ascii=False, indent=4)
- os.replace(tmp_path, final_path)
- except Exception:
- traceback.print_exc()
-
-
-def translateSimpleKeyValueJSON(data, filename):
- """
- Translate simple key-value JSON format where keys contain Japanese text
- and values are placeholder strings like "TODO".
-
- Format example:
- {
- "\\r\\nスタビライザー": "TODO",
- "\\r\\nプラグインなし": "TODO"
- }
- """
- global LOCK, ESTIMATE, FILENAME, PBAR, MISMATCH
- tokens = [0, 0]
- stringList = []
- keyList = []
-
- # Pass 1: Collect all keys that need translation
- for key, value in data.items():
- # Check if the key contains text matching the language regex
- if re.search(LANGREGEX, key):
- # Strip whitespace and newline characters for translation
- # cleanKey = key.strip().replace("\r\n", "").replace("\n", "").replace("\r", "")
- # if cleanKey:
- stringList.append(key)
- keyList.append(key) # Store original key for reference
-
- # Translate all keys if any were found
- if stringList:
- PBAR.total = len(stringList)
- PBAR.refresh()
-
- response = translateAI(stringList, "Reply with the English Translation")
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- translatedList = response[0]
-
- # Check for mismatch
- if len(stringList) != len(translatedList):
- with LOCK:
- if FILENAME not in MISMATCH:
- MISMATCH.append(FILENAME)
-
- # Pass 2: Update the values with translations
- for i, key in enumerate(keyList):
- if i < len(translatedList):
- translatedText = translatedList[i]
- # Set the translated text as the value
- data[key] = translatedText
-
- # Save progress after each translation
- save_progress_json(data, filename)
-
- return tokens
-
-
-def translatePSData(data, filename):
- """Translate PSData JSON format (e.g. Nupu_PSData.json).
-
- Handles four sections:
- - MailDatas: In-game mail/message system
- - MemoryData: Scene/CG gallery memories
- - ShopDatas: Shop configurations and dialogue
- - hStData: Status rank descriptions
-
- Only translates player-visible text fields.
- """
- global LOCK, ESTIMATE, FILENAME, PBAR, MISMATCH
- tokens = [0, 0]
-
- def batchTranslate(stringList, context):
- """Translate a list of strings and return translations. Returns [] on mismatch."""
- nonlocal tokens
- if not stringList:
- return []
- response = translateAI(stringList, context)
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- if len(stringList) != len(response[0]):
- with LOCK:
- if FILENAME not in MISMATCH:
- MISMATCH.append(FILENAME)
- return []
- return response[0]
-
- # ================================================================
- # PASS 1: Collect all translatable strings from every section
- # ================================================================
-
- # -- MailDatas --
- mailNameS, mailNameI = [], []
- mailTitleS, mailTitleI = [], []
- mailListNameS, mailListNameI = [], []
- mailTextS, mailTextI = [], []
-
- if "MailDatas" in data:
- for i, m in enumerate(data["MailDatas"]):
- if m.get("Name"):
- mailNameS.append(m["Name"])
- mailNameI.append(i)
- if m.get("Title"):
- mailTitleS.append(m["Title"])
- mailTitleI.append(i)
- if m.get("MailListName"):
- mailListNameS.append(m["MailListName"])
- mailListNameI.append(i)
- if m.get("MailText"):
- mailTextS.append(m["MailText"].replace("\r\n", " ").replace("\n", " ").strip())
- mailTextI.append(i)
-
- # -- MemoryData --
- memTitleS, memTitleI = [], []
- memSetuS, memSetuI = [], []
- memHintS, memHintI = [], []
- memPtnS, memPtnI = [], [] # indices are (memIdx, ptnIdx)
- memDtlSetuS, memDtlSetuI = [], [] # indices are (memIdx, dtlIdx)
-
- if "MemoryData" in data:
- for i, mem in enumerate(data["MemoryData"]):
- if mem.get("_Title"):
- memTitleS.append(mem["_Title"])
- memTitleI.append(i)
- if mem.get("_Setu"):
- memSetuS.append(mem["_Setu"].replace("\r\n", " ").replace("\n", " ").strip())
- memSetuI.append(i)
- if mem.get("_Hint"):
- memHintS.append(mem["_Hint"])
- memHintI.append(i)
- if "_Ptn" in mem and isinstance(mem["_Ptn"], list):
- for j, p in enumerate(mem["_Ptn"]):
- if p:
- memPtnS.append(p)
- memPtnI.append((i, j))
- if "_DtlCheck" in mem and isinstance(mem["_DtlCheck"], list):
- for j, dtl in enumerate(mem["_DtlCheck"]):
- if isinstance(dtl, dict) and dtl.get("_Setu"):
- memDtlSetuS.append(dtl["_Setu"].replace("\r\n", " ").replace("\n", " ").strip())
- memDtlSetuI.append((i, j))
-
- # -- ShopDatas --
- shopNameS, shopNameI = [], []
- shopTextFields = ["buySelf", "emptySelf", "firstSelf", "husokuSelf",
- "randSelf1", "randSelf2", "randSelf3"]
- shopTextS = {f: [] for f in shopTextFields}
- shopTextI = {f: [] for f in shopTextFields}
-
- if "ShopDatas" in data:
- for i, shop in enumerate(data["ShopDatas"]):
- if shop.get("name"):
- shopNameS.append(shop["name"])
- shopNameI.append(i)
- for field in shopTextFields:
- if field in shop and isinstance(shop[field], dict):
- text = shop[field].get("text", "")
- if text:
- shopTextS[field].append(text.replace("\r\n", " ").replace("\n", " ").strip())
- shopTextI[field].append(i)
-
- # -- hStData --
- rankArrayNames = ["KutiRankDatas", "MuneRankDatas", "SiriRankDatas", "TituRankDatas"]
- rankSetuS, rankSetuI = [], [] # indices are (arrayName, idx)
- rankText1S, rankText1I = [], []
- rankText2S, rankText2I = [], []
-
- if "hStData" in data:
- hst = data["hStData"]
- for arrName in rankArrayNames:
- if arrName in hst and isinstance(hst[arrName], list):
- for i, rank in enumerate(hst[arrName]):
- if rank.get("rankSetu"):
- rankSetuS.append(rank["rankSetu"].replace("\r\n", " ").replace("\n", " ").strip())
- rankSetuI.append((arrName, i))
- if rank.get("rankText1"):
- rankText1S.append(rank["rankText1"])
- rankText1I.append((arrName, i))
- if rank.get("rankText2"):
- rankText2S.append(rank["rankText2"])
- rankText2I.append((arrName, i))
-
- # Set progress bar total
- totalItems = (
- len(mailNameS) + len(mailTitleS) + len(mailListNameS) + len(mailTextS)
- + len(memTitleS) + len(memSetuS) + len(memHintS) + len(memPtnS) + len(memDtlSetuS)
- + len(shopNameS) + sum(len(v) for v in shopTextS.values())
- + len(rankSetuS) + len(rankText1S) + len(rankText2S)
- )
- PBAR.total = totalItems
- PBAR.refresh()
-
- # ================================================================
- # PASS 2: Translate each batch and apply results back to data
- # ================================================================
-
- # -- MailDatas --
- if "MailDatas" in data:
- mails = data["MailDatas"]
-
- for tl, idx in zip(batchTranslate(mailNameS, "Character Name"), mailNameI):
- mails[idx]["Name"] = tl
- if mailNameS:
- save_progress_json(data, filename)
-
- for tl, idx in zip(batchTranslate(mailTitleS, "Mail Subject"), mailTitleI):
- mails[idx]["Title"] = tl
- if mailTitleS:
- save_progress_json(data, filename)
-
- for tl, idx in zip(batchTranslate(mailListNameS, "Mail List Entry"), mailListNameI):
- mails[idx]["MailListName"] = tl
- if mailListNameS:
- save_progress_json(data, filename)
-
- for tl, idx in zip(batchTranslate(mailTextS, "Mail Body Text"), mailTextI):
- mails[idx]["MailText"] = dazedwrap.wrapText(tl, WIDTH)
- if mailTextS:
- save_progress_json(data, filename)
-
- # -- MemoryData --
- if "MemoryData" in data:
- memories = data["MemoryData"]
-
- for tl, idx in zip(batchTranslate(memTitleS, "Scene Title"), memTitleI):
- memories[idx]["_Title"] = tl
- if memTitleS:
- save_progress_json(data, filename)
-
- for tl, idx in zip(batchTranslate(memSetuS, "Scene Description"), memSetuI):
- memories[idx]["_Setu"] = tl
- if memSetuS:
- save_progress_json(data, filename)
-
- for tl, idx in zip(batchTranslate(memHintS, "Hint Text"), memHintI):
- memories[idx]["_Hint"] = tl
- if memHintS:
- save_progress_json(data, filename)
-
- for tl, (mi, pi) in zip(batchTranslate(memPtnS, "Scene Description"), memPtnI):
- memories[mi]["_Ptn"][pi] = tl
- if memPtnS:
- save_progress_json(data, filename)
-
- for tl, (mi, di) in zip(batchTranslate(memDtlSetuS, "Character Commentary"), memDtlSetuI):
- memories[mi]["_DtlCheck"][di]["_Setu"] = dazedwrap.wrapText(tl, LISTWIDTH)
- if memDtlSetuS:
- save_progress_json(data, filename)
-
- # -- ShopDatas --
- if "ShopDatas" in data:
- shops = data["ShopDatas"]
-
- for tl, idx in zip(batchTranslate(shopNameS, "Shop Name"), shopNameI):
- shops[idx]["name"] = tl
- if shopNameS:
- save_progress_json(data, filename)
-
- for field in shopTextFields:
- for tl, idx in zip(batchTranslate(shopTextS[field], "Shop Dialogue"), shopTextI[field]):
- shops[idx][field]["text"] = dazedwrap.wrapText(tl, WIDTH)
- if shopTextS[field]:
- save_progress_json(data, filename)
-
- # -- hStData --
- if "hStData" in data:
- hst = data["hStData"]
-
- for tl, (an, idx) in zip(batchTranslate(rankSetuS, "Rank Description"), rankSetuI):
- hst[an][idx]["rankSetu"] = dazedwrap.wrapText(tl, WIDTH)
- if rankSetuS:
- save_progress_json(data, filename)
-
- for tl, (an, idx) in zip(batchTranslate(rankText1S, "Rank Label"), rankText1I):
- hst[an][idx]["rankText1"] = tl
- if rankText1S:
- save_progress_json(data, filename)
-
- for tl, (an, idx) in zip(batchTranslate(rankText2S, "Rank Subtitle"), rankText2I):
- hst[an][idx]["rankText2"] = tl
- if rankText2S:
- save_progress_json(data, filename)
-
- return tokens
-
-
-def translateRdData(data, filename):
- """Translate RdData JSON format (e.g. RdData.json).
-
- Handles dungeon data with nested dialogue, choices, and UI text.
- Only translates player-visible text fields.
- """
- global LOCK, ESTIMATE, FILENAME, PBAR, MISMATCH
- tokens = [0, 0]
-
- def batchTranslate(stringList, context):
- """Translate a list of strings and return translations. Returns [] on mismatch."""
- nonlocal tokens
- if not stringList:
- return []
- response = translateAI(stringList, context)
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- if len(stringList) != len(response[0]):
- with LOCK:
- if FILENAME not in MISMATCH:
- MISMATCH.append(FILENAME)
- return []
- return response[0]
-
- if "dgnDatas" not in data:
- return tokens
-
- dgns = data["dgnDatas"]
-
- # ================================================================
- # PASS 1: Collect all translatable strings
- # ================================================================
-
- # -- Dungeon-level short fields --
- dgnNameS, dgnNameI = [], []
- clearTxtS, clearTxtI = [], []
- hosyuTxtS, hosyuTxtI = [], []
- firstHosyuTxtS, firstHosyuTxtI = [], []
- memoTxtS, memoTxtI = [], []
- appMemoTxtS, appMemoTxtI = [], []
- areaMemoS, areaMemoI = [], []
-
- # -- Dungeon-level long text fields --
- naiyoTxtS, naiyoTxtI = [], []
- naiyoTxtExS, naiyoTxtExI = [], []
- johoTxtS, johoTxtI = [], []
- johoTxtExS, johoTxtExI = [], []
-
- # -- Floor names --
- kaisoNameS, kaisoNameI = [], [] # (dgnIdx, kaisoIdx)
-
- # -- Dialogue (talkDatas / talkDatasEx) --
- talkNameS, talkNameI = [], [] # (dgnIdx, talkKey, talkIdx)
- speakerNameS, speakerNameI = [], [] # (dgnIdx, talkKey, talkIdx, speechIdx)
- dialogueS, dialogueI = [], [] # (dgnIdx, talkKey, talkIdx, speechIdx)
-
- # -- Choice events (sijiDatas) --
- sijiNameS, sijiNameI = [], [] # (dgnIdx, sijiIdx)
- sijiChoiceS, sijiChoiceI = [], [] # (dgnIdx, sijiIdx)
-
- for di, dgn in enumerate(dgns):
- if dgn is None:
- continue
-
- # Dungeon short fields
- if dgn.get("name"):
- dgnNameS.append(dgn["name"])
- dgnNameI.append(di)
- if dgn.get("clearTxt"):
- clearTxtS.append(dgn["clearTxt"])
- clearTxtI.append(di)
- if dgn.get("hosyuTxt"):
- hosyuTxtS.append(dgn["hosyuTxt"])
- hosyuTxtI.append(di)
- if dgn.get("firstHosyuTxt"):
- firstHosyuTxtS.append(dgn["firstHosyuTxt"])
- firstHosyuTxtI.append(di)
- if dgn.get("memoTxt"):
- memoTxtS.append(dgn["memoTxt"])
- memoTxtI.append(di)
- if dgn.get("appMemoTxt"):
- appMemoTxtS.append(dgn["appMemoTxt"])
- appMemoTxtI.append(di)
- if dgn.get("areaMemo"):
- areaMemoS.append(dgn["areaMemo"])
- areaMemoI.append(di)
-
- # Dungeon long text fields (strip line breaks for translation)
- if dgn.get("naiyoTxt"):
- naiyoTxtS.append(dgn["naiyoTxt"].replace("\r\n", " ").replace("\n", " ").strip())
- naiyoTxtI.append(di)
- if dgn.get("naiyoTxtEx"):
- naiyoTxtExS.append(dgn["naiyoTxtEx"].replace("\r\n", " ").replace("\n", " ").strip())
- naiyoTxtExI.append(di)
- if dgn.get("johoTxt"):
- johoTxtS.append(dgn["johoTxt"].replace("\r\n", " ").replace("\n", " ").strip())
- johoTxtI.append(di)
- if dgn.get("johoTxtEx"):
- johoTxtExS.append(dgn["johoTxtEx"].replace("\r\n", " ").replace("\n", " ").strip())
- johoTxtExI.append(di)
-
- # Floor names
- if "kaisoDatas" in dgn and isinstance(dgn["kaisoDatas"], list):
- for ki, kaiso in enumerate(dgn["kaisoDatas"]):
- if isinstance(kaiso, dict) and kaiso.get("kaisoName"):
- kaisoNameS.append(kaiso["kaisoName"])
- kaisoNameI.append((di, ki))
-
- # Dialogue from talkDatas and talkDatasEx
- for talkKey in ["talkDatas", "talkDatasEx"]:
- if talkKey in dgn and isinstance(dgn[talkKey], list):
- for ti, talk in enumerate(dgn[talkKey]):
- if not isinstance(talk, dict):
- continue
- if talk.get("name"):
- talkNameS.append(talk["name"])
- talkNameI.append((di, talkKey, ti))
- if "speachDatas" in talk and isinstance(talk["speachDatas"], list):
- for si, speech in enumerate(talk["speachDatas"]):
- if not isinstance(speech, dict):
- continue
- if speech.get("name"):
- speakerNameS.append(speech["name"])
- speakerNameI.append((di, talkKey, ti, si))
- if speech.get("text"):
- dialogueS.append(speech["text"].replace("\r\n", " ").replace("\n", " ").strip())
- dialogueI.append((di, talkKey, ti, si))
-
- # Choice events (sijiDatas)
- if "sijiDatas" in dgn and isinstance(dgn["sijiDatas"], list):
- for si, siji in enumerate(dgn["sijiDatas"]):
- if not isinstance(siji, dict):
- continue
- if siji.get("name"):
- sijiNameS.append(siji["name"])
- sijiNameI.append((di, si))
- if siji.get("choiceSijiTxt"):
- sijiChoiceS.append(siji["choiceSijiTxt"].replace("\r\n", " ").replace("\n", " ").strip())
- sijiChoiceI.append((di, si))
-
- # Set progress bar total
- totalItems = (
- len(dgnNameS) + len(clearTxtS) + len(hosyuTxtS) + len(firstHosyuTxtS)
- + len(memoTxtS) + len(appMemoTxtS) + len(areaMemoS)
- + len(naiyoTxtS) + len(naiyoTxtExS) + len(johoTxtS) + len(johoTxtExS)
- + len(kaisoNameS)
- + len(talkNameS) + len(speakerNameS) + len(dialogueS)
- + len(sijiNameS) + len(sijiChoiceS)
- )
- PBAR.total = totalItems
- PBAR.refresh()
-
- # ================================================================
- # PASS 2: Translate each batch and apply results back to data
- # ================================================================
-
- # -- Dungeon short fields --
- for tl, idx in zip(batchTranslate(dgnNameS, "Dungeon Name"), dgnNameI):
- dgns[idx]["name"] = tl
- if dgnNameS:
- save_progress_json(data, filename)
-
- for tl, idx in zip(batchTranslate(clearTxtS, "Clear Condition"), clearTxtI):
- dgns[idx]["clearTxt"] = tl
- if clearTxtS:
- save_progress_json(data, filename)
-
- for tl, idx in zip(batchTranslate(hosyuTxtS, "Reward Description"), hosyuTxtI):
- dgns[idx]["hosyuTxt"] = tl
- if hosyuTxtS:
- save_progress_json(data, filename)
-
- for tl, idx in zip(batchTranslate(firstHosyuTxtS, "First Clear Reward"), firstHosyuTxtI):
- dgns[idx]["firstHosyuTxt"] = tl
- if firstHosyuTxtS:
- save_progress_json(data, filename)
-
- for tl, idx in zip(batchTranslate(memoTxtS, "Memo Label"), memoTxtI):
- dgns[idx]["memoTxt"] = tl
- if memoTxtS:
- save_progress_json(data, filename)
-
- for tl, idx in zip(batchTranslate(appMemoTxtS, "App Memo"), appMemoTxtI):
- dgns[idx]["appMemoTxt"] = tl
- if appMemoTxtS:
- save_progress_json(data, filename)
-
- for tl, idx in zip(batchTranslate(areaMemoS, "Area Label"), areaMemoI):
- dgns[idx]["areaMemo"] = tl
- if areaMemoS:
- save_progress_json(data, filename)
-
- # -- Dungeon long text fields (with word wrap) --
- for tl, idx in zip(batchTranslate(naiyoTxtS, "Mission Description"), naiyoTxtI):
- dgns[idx]["naiyoTxt"] = dazedwrap.wrapText(tl, WIDTH)
- if naiyoTxtS:
- save_progress_json(data, filename)
-
- for tl, idx in zip(batchTranslate(naiyoTxtExS, "Extended Mission Description"), naiyoTxtExI):
- dgns[idx]["naiyoTxtEx"] = dazedwrap.wrapText(tl, WIDTH)
- if naiyoTxtExS:
- save_progress_json(data, filename)
-
- for tl, idx in zip(batchTranslate(johoTxtS, "Info Text"), johoTxtI):
- dgns[idx]["johoTxt"] = dazedwrap.wrapText(tl, WIDTH)
- if johoTxtS:
- save_progress_json(data, filename)
-
- for tl, idx in zip(batchTranslate(johoTxtExS, "Extended Info Text"), johoTxtExI):
- dgns[idx]["johoTxtEx"] = dazedwrap.wrapText(tl, WIDTH)
- if johoTxtExS:
- save_progress_json(data, filename)
-
- # -- Floor names --
- for tl, (di, ki) in zip(batchTranslate(kaisoNameS, "Floor Name"), kaisoNameI):
- dgns[di]["kaisoDatas"][ki]["kaisoName"] = tl
- if kaisoNameS:
- save_progress_json(data, filename)
-
- # -- Dialogue event names --
- for tl, (di, tk, ti) in zip(batchTranslate(talkNameS, "Event Label"), talkNameI):
- dgns[di][tk][ti]["name"] = tl
- if talkNameS:
- save_progress_json(data, filename)
-
- # -- Speaker names --
- for tl, (di, tk, ti, si) in zip(batchTranslate(speakerNameS, "Character Name"), speakerNameI):
- dgns[di][tk][ti]["speachDatas"][si]["name"] = tl
- if speakerNameS:
- save_progress_json(data, filename)
-
- # -- Dialogue text (with word wrap) --
- for tl, (di, tk, ti, si) in zip(batchTranslate(dialogueS, "Dialogue"), dialogueI):
- dgns[di][tk][ti]["speachDatas"][si]["text"] = dazedwrap.wrapText(tl, WIDTH)
- if dialogueS:
- save_progress_json(data, filename)
-
- # -- Choice event names --
- for tl, (di, si) in zip(batchTranslate(sijiNameS, "Event Name"), sijiNameI):
- dgns[di]["sijiDatas"][si]["name"] = tl
- if sijiNameS:
- save_progress_json(data, filename)
-
- # -- Choice text (with word wrap) --
- for tl, (di, si) in zip(batchTranslate(sijiChoiceS, "Choice Event Text"), sijiChoiceI):
- dgns[di]["sijiDatas"][si]["choiceSijiTxt"] = dazedwrap.wrapText(tl, WIDTH)
- if sijiChoiceS:
- save_progress_json(data, filename)
-
- return tokens
-
-
-def translateGameSetting(data, filename):
- """Translate GameSetting JSON format (e.g. Nupu_GameSetting.json).
-
- Handles card data, deck definitions, and rental decks.
- Only translates player-visible text fields.
- """
- global LOCK, ESTIMATE, FILENAME, PBAR, MISMATCH
- tokens = [0, 0]
-
- def batchTranslate(stringList, context):
- """Translate a list of strings and return translations. Returns [] on mismatch."""
- nonlocal tokens
- if not stringList:
- return []
- response = translateAI(stringList, context)
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- if len(stringList) != len(response[0]):
- with LOCK:
- if FILENAME not in MISMATCH:
- MISMATCH.append(FILENAME)
- return []
- return response[0]
-
- # ================================================================
- # PASS 1: Collect all translatable strings
- # ================================================================
-
- # -- fzCardDatas: card descriptions --
- desc1S, desc1I = [], []
- desc2S, desc2I = [], []
-
- if "fzCardDatas" in data:
- for i, card in enumerate(data["fzCardDatas"]):
- if card.get("desc1"):
- desc1S.append(card["desc1"].replace("\r\n", " ").replace("\n", " ").strip())
- desc1I.append(i)
- if card.get("desc2"):
- desc2S.append(card["desc2"].replace("\r\n", " ").replace("\n", " ").strip())
- desc2I.append(i)
-
- # -- fzDeckDatas: deck name and description --
- deckNameS, deckNameI = [], []
- deckDescS, deckDescI = [], []
-
- if "fzDeckDatas" in data:
- for i, deck in enumerate(data["fzDeckDatas"]):
- if deck.get("name"):
- deckNameS.append(deck["name"])
- deckNameI.append(i)
- if deck.get("desc"):
- deckDescS.append(deck["desc"].replace("\r\n", " ").replace("\n", " ").strip())
- deckDescI.append(i)
-
- # -- rentalDecks: memo label --
- rentalMemoS, rentalMemoI = [], []
-
- if "rentalDecks" in data:
- for i, rental in enumerate(data["rentalDecks"]):
- if rental.get("memo"):
- rentalMemoS.append(rental["memo"])
- rentalMemoI.append(i)
-
- # Set progress bar total
- totalItems = (
- len(desc1S) + len(desc2S)
- + len(deckNameS) + len(deckDescS)
- + len(rentalMemoS)
- )
- PBAR.total = totalItems
- PBAR.refresh()
-
- # ================================================================
- # PASS 2: Translate each batch and apply results back to data
- # ================================================================
-
- # -- fzCardDatas --
- if "fzCardDatas" in data:
- cards = data["fzCardDatas"]
-
- for tl, idx in zip(batchTranslate(desc1S, "Card Description"), desc1I):
- cards[idx]["desc1"] = dazedwrap.wrapText(tl, WIDTH)
- if desc1S:
- save_progress_json(data, filename)
-
- for tl, idx in zip(batchTranslate(desc2S, "Card Description"), desc2I):
- cards[idx]["desc2"] = dazedwrap.wrapText(tl, WIDTH)
- if desc2S:
- save_progress_json(data, filename)
-
- # -- fzDeckDatas --
- if "fzDeckDatas" in data:
- decks = data["fzDeckDatas"]
-
- for tl, idx in zip(batchTranslate(deckNameS, "Deck Name"), deckNameI):
- decks[idx]["name"] = tl
- if deckNameS:
- save_progress_json(data, filename)
-
- for tl, idx in zip(batchTranslate(deckDescS, "Deck Description"), deckDescI):
- decks[idx]["desc"] = dazedwrap.wrapText(tl, WIDTH)
- if deckDescS:
- save_progress_json(data, filename)
-
- # -- rentalDecks --
- if "rentalDecks" in data:
- rentals = data["rentalDecks"]
-
- for tl, idx in zip(batchTranslate(rentalMemoS, "Deck Label"), rentalMemoI):
- rentals[idx]["memo"] = tl
- if rentalMemoS:
- save_progress_json(data, filename)
-
- return tokens
-
-
-def translateDialogueSrcTl(data, filename):
- """Translate src/tl dialogue JSON format.
-
- Format:
- {
- "0": {"type": "dialogue", "src": "Japanese text", "tl": ""},
- "4": {"type": "dialogue", "speaker": "琴音", "src": "「...」", "tl": ""}
- }
-
- Translates 'src' into 'tl'. Also translates Japanese 'speaker' fields.
- Skips entries where 'tl' is already populated.
- """
- global LOCK, ESTIMATE, FILENAME, PBAR, MISMATCH
- tokens = [0, 0]
- srcList = []
- keyList = [] # list of (key, entry_type)
-
- # Collect all entries with translatable content
- for key, entry in data.items():
- if not isinstance(entry, dict):
- continue
- src = entry.get("src", "")
- if not src:
- continue
- tl = entry.get("tl", "")
- entry_type = entry.get("type", "")
- speaker = entry.get("speaker", "")
-
- if tl:
- # Already has a translation — send tl so the module can decide to skip it
- srcList.append(tl)
- else:
- if not re.search(LANGREGEX, src):
- continue
- srcClean = src.replace("\r\n", " ").replace("\n", " ").strip()
- if speaker:
- srcList.append(f"[{speaker}]: {srcClean}")
- else:
- srcList.append(srcClean)
- keyList.append((key, entry_type))
-
- if srcList:
- PBAR.total = len(srcList)
- PBAR.refresh()
-
- response = translateAI(srcList, "Reply with the English Translation")
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- translatedList = response[0]
-
- if len(srcList) != len(translatedList):
- with LOCK:
- if FILENAME not in MISMATCH:
- MISMATCH.append(FILENAME)
-
- for i, (key, entry_type) in enumerate(keyList):
- if i >= len(translatedList):
- break
- translatedText = translatedList[i]
-
- # Strip speaker prefix if the AI echoed it back
- # Handles: [Speaker]: text | Speaker: text | Speaker(text) / CJK(text)
- match = re.search(r'^\[.+?\]\s?[|:]\s?', translatedText)
- if match:
- translatedText = translatedText[match.end():]
- else:
- # Fallback: strip any leading Japanese/CJK name followed by ( or :
- cjk_m = re.match(r'^[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+\s*[\(（:]\s*', translatedText)
- if cjk_m:
- translatedText = translatedText[cjk_m.end():]
- translatedText = re.sub(r'\s*[)）]\s*$', '', translatedText)
-
- data[key]["tl"] = translatedText
- save_progress_json(data, filename)
-
- # Translate any Japanese speaker names found in the data
- seen_speakers = {}
- speaker_updated = False
- for key, entry in data.items():
- if not isinstance(entry, dict):
- continue
- speaker = entry.get("speaker", "")
- if not speaker or not re.search(LANGREGEX, speaker):
- continue
- if speaker not in seen_speakers:
- response = getSpeaker(speaker)
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- seen_speakers[speaker] = response[0]
- data[key]["speaker"] = seen_speakers[speaker]
- speaker_updated = True
- if speaker_updated:
- save_progress_json(data, filename)
-
- return tokens
-
-
-def translateJSON(data, filename, translatedList):
- global LOCK, ESTIMATE, FILENAME, PBAR, MISMATCH
- if translatedList:
- stringList = translatedList[0]
- eventList = translatedList[1]
- else:
- stringList = []
- eventList = [[], [], [], [], [], [], []] # [title, process, text, key, target, job, place]
- tokens = [0, 0]
- speaker = ""
- i = 0
- stringListTL = []
- eventListTL = [[], [], [], [], [], [], []]
-
- while i < len(data):
- speakerKey = "character_nameText"
- messageKey = "m_text"
-
- # Event List Format - Key
- if "key" in data[i] and data[i]["key"]:
- jaString = data[i]["key"]
-
- # Pass 1
- if not translatedList:
- eventList[3].append(jaString.strip())
-
- # Pass 2
- else:
- if eventList[3]:
- translatedText = eventList[3][0]
- eventList[3].pop(0)
-
- # Set Data
- data[i]["key"] = translatedText
- save_progress_json(data, filename)
-
- # Event List Format - Title
- if "title" in data[i] and data[i]["title"]:
- jaString = data[i]["title"]
-
- # Pass 1
- if not translatedList:
- eventList[0].append(jaString.strip())
-
- # Pass 2
- else:
- if eventList[0]:
- translatedText = eventList[0][0]
- eventList[0].pop(0)
-
- # Set Data
- data[i]["title"] = translatedText
- save_progress_json(data, filename)
-
- # Event List Format - Target
- if "target" in data[i] and data[i]["target"]:
- jaString = data[i]["target"]
-
- # Pass 1
- if not translatedList:
- eventList[4].append(jaString.strip())
-
- # Pass 2
- else:
- if eventList[4]:
- translatedText = eventList[4][0]
- eventList[4].pop(0)
-
- # Set Data
- data[i]["target"] = translatedText
- save_progress_json(data, filename)
-
- # Event List Format - Job
- if "job" in data[i] and data[i]["job"]:
- jaString = data[i]["job"]
-
- # Pass 1
- if not translatedList:
- eventList[5].append(jaString.strip())
-
- # Pass 2
- else:
- if eventList[5]:
- translatedText = eventList[5][0]
- eventList[5].pop(0)
-
- # Set Data
- data[i]["job"] = translatedText
- save_progress_json(data, filename)
-
- # Event List Format - Place
- if "place" in data[i] and data[i]["place"]:
- jaString = data[i]["place"]
-
- # Pass 1
- if not translatedList:
- eventList[6].append(jaString.strip())
-
- # Pass 2
- else:
- if eventList[6]:
- translatedText = eventList[6][0]
- eventList[6].pop(0)
-
- # Set Data
- data[i]["place"] = translatedText
- save_progress_json(data, filename)
-
- # Event List Format - Process
- if "process" in data[i] and data[i]["process"]:
- jaString = data[i]["process"]
-
- # Pass 1
- if not translatedList:
- eventList[1].append(jaString.strip())
-
- # Pass 2
- else:
- if eventList[1]:
- translatedText = eventList[1][0]
- eventList[1].pop(0)
-
- # Set Data
- data[i]["process"] = translatedText
- save_progress_json(data, filename)
-
- # Event List Format - Text
- if "text" in data[i] and data[i]["text"]:
- jaString = data[i]["text"]
- # Pass 1
- if not translatedList:
- # Replace \n with space for translation
- jaStringClean = jaString.replace("\n", " ")
- eventList[2].append(jaStringClean.strip())
-
- # Pass 2
- else:
- if eventList[2]:
- translatedText = eventList[2][0]
- eventList[2].pop(0)
-
- # Apply text wrapping and restore linebreaks
- translatedText = dazedwrap.wrapText(translatedText, 70)
-
- # Set Data
- data[i]["text"] = translatedText
- save_progress_json(data, filename)
-
- # Speaker
- if speakerKey in data[i] and data[i][speakerKey]:
- # Grab and TL
- speaker = data[i][speakerKey]
- response = getSpeaker(speaker)
- speaker = response[0]
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
-
- # Set Speaker
- data[i][speakerKey] = speaker
-
- # Dialogue
- if messageKey in data[i]\
- and data[i][messageKey].strip()\
- and data[i][messageKey] != "a"\
- and data[i][messageKey].replace("\u3000", "").strip() != "":
- jaString = data[i][messageKey]
-
- # Save Original String
- originalString = jaString
-
- # If there isn't any Japanese in the text just skip
- if not re.search(LANGREGEX, jaString):
- i += 1
- continue
-
- # Pass 1
- if not translatedList:
- # Strip Spaces
- jaString = jaString.strip()
-
- # Remove Textwrap
- # jaString = jaString.replace('\n', ' ')
-
- if jaString:
- if speaker:
- stringList.append(f"[{speaker}]: {jaString}")
- else:
- stringList.append(jaString)
-
- # Pass 2
- else:
- # Get Text
- if stringList:
- # Grab and Pop
- translatedText = stringList[0]
- stringList.pop(0)
-
- # Set to None if empty list
- if len(stringList) <= 0:
- stringList = None
-
- # Remove speaker prefix — handles [Name]: / Name: / CJK(text)
- match = re.search(r'(^\[.+?\]\s?[|:]\s?)', translatedText)
- if match:
- translatedText = translatedText.replace(match.group(1), "")
- else:
- cjk_m = re.match(r'^[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+\s*[\(（:]\s*', translatedText)
- if cjk_m:
- translatedText = translatedText[cjk_m.end():]
- translatedText = re.sub(r'\s*[)）]\s*$', '', translatedText)
-
- # Escape Quotes
- translatedText = re.sub(r'(?", ")")
- translatedText = translatedText.replace("『", "")
- translatedText = translatedText.replace("』", "")
-
- # Remove GPT ' Quotes
- if translatedText:
- if translatedText[0] == "'":
- translatedText = translatedText[1:]
- if translatedText[-1] == "'":
- translatedText = translatedText[:-1]
- else:
- print("Warning: Empty Translation for", originalString)
-
- # Textwrap
- # translatedText = dazedwrap.wrapText(translatedText, width=WIDTH)
-
- # Set Data
- if "『" in data[i][messageKey] and "』" not in translatedText:
- data[i][messageKey] = data[i][messageKey].replace(originalString, f"『{translatedText}』")
- else:
- data[i][messageKey] = data[i][messageKey].replace(originalString, f"{translatedText}")
-
- # Save progress after each message replacement
- save_progress_json(data, filename)
- # Next Value
- i += 1
-
- # EOF - Only do translation if this is Pass 1 (collecting strings)
- if not translatedList:
- # Event List
- if any(eventList):
- PBAR.total = sum(len(event) for event in eventList)
- PBAR.refresh()
-
- # Event Title
- if eventList[0]:
- response = translateAI(eventList[0], "Event Title")
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- eventListTL[0] = response[0]
-
- if len(eventList[0]) != len(eventListTL[0]):
- with LOCK:
- if FILENAME not in MISMATCH:
- MISMATCH.append(FILENAME)
-
- # Event Process
- if eventList[1]:
- response = translateAI(eventList[1], "Event Process")
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- eventListTL[1] = response[0]
-
- if len(eventList[1]) != len(eventListTL[1]):
- with LOCK:
- if FILENAME not in MISMATCH:
- MISMATCH.append(FILENAME)
-
- # Event Text
- if eventList[2]:
- response = translateAI(eventList[2], "Event Description")
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- eventListTL[2] = response[0]
-
- if len(eventList[2]) != len(eventListTL[2]):
- with LOCK:
- if FILENAME not in MISMATCH:
- MISMATCH.append(FILENAME)
-
- # Event Key
- if eventList[3]:
- response = translateAI(eventList[3], "Event Key")
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- eventListTL[3] = response[0]
-
- if len(eventList[3]) != len(eventListTL[3]):
- with LOCK:
- if FILENAME not in MISMATCH:
- MISMATCH.append(FILENAME)
-
- # Event Target
- if eventList[4]:
- response = translateAI(eventList[4], "Character Name")
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- eventListTL[4] = response[0]
-
- if len(eventList[4]) != len(eventListTL[4]):
- with LOCK:
- if FILENAME not in MISMATCH:
- MISMATCH.append(FILENAME)
-
- # Event Job
- if eventList[5]:
- response = translateAI(eventList[5], "Job/Occupation")
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- eventListTL[5] = response[0]
-
- if len(eventList[5]) != len(eventListTL[5]):
- with LOCK:
- if FILENAME not in MISMATCH:
- MISMATCH.append(FILENAME)
-
- # Event Place
- if eventList[6]:
- response = translateAI(eventList[6], "Location Name")
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- eventListTL[6] = response[0]
-
- if len(eventList[6]) != len(eventListTL[6]):
- with LOCK:
- if FILENAME not in MISMATCH:
- MISMATCH.append(FILENAME)
-
- # String List
- if stringList:
- PBAR.total = len(stringList)
- PBAR.refresh()
- response = translateAI(stringList, "Reply with the English Translation")
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- stringListTL = response[0]
-
- if len(stringList) != len(stringListTL):
- # Mismatch
- with LOCK:
- if FILENAME not in MISMATCH:
- MISMATCH.append(FILENAME)
-
- # Pass 2: Set Strings (recursive call)
- translateJSON(data, filename, [stringListTL, eventListTL])
-
- return tokens
-
-# Save some money and enter the character before translation
-def getSpeaker(speaker):
- match speaker:
- case "ファイン":
- return ["Fine", [0, 0]]
- case "":
- return ["", [0, 0]]
- case _:
- # Find Speaker
- for i in range(len(NAMESLIST)):
- if speaker == NAMESLIST[i][0]:
- return [NAMESLIST[i][1], [0, 0]]
-
- # Translate and Store Speaker
- response = translateAI(
- f"{speaker}",
- "Reply with the " + LANGUAGE + " translation of the NPC name.",
- False,
- )
- response[0] = response[0].title()
- response[0] = response[0].replace("'S", "'s")
- response[0] = response[0].replace("Speaker: ", "")
-
- # Retry if name doesn't translate for some reason
- if re.search(r"([a-zA-Z？?])", response[0]) == None:
- response = translateAI(
- f"{speaker}",
- "Reply with the " + LANGUAGE + " translation of the NPC name.",
- False,
- )
- response[0] = response[0].title()
- response[0] = response[0].replace("'S", "'s")
-
- speakerList = [speaker, response[0]]
- NAMESLIST.append(speakerList)
- return response
- return [speaker, [0, 0]]
-
-def translateAI(text, history, history_ctx=None):
- """
- Legacy wrapper function for the new shared translation utility.
- This maintains compatibility with existing code while using the new shared implementation.
- """
- global PBAR, MISMATCH, FILENAME
-
- # Update config estimate mode based on global ESTIMATE
- TRANSLATION_CONFIG.estimateMode = bool(ESTIMATE)
-
- # Call the new shared translation function
- return sharedtranslateAI(
- text=text,
- history=history,
- config=TRANSLATION_CONFIG,
- filename=FILENAME,
- pbar=PBAR,
- lock=LOCK,
- mismatchList=MISMATCH
- )
+# Libraries
+import json
+import os
+import re
+import util.dazedwrap as dazedwrap
+import threading
+import time
+import traceback
+import tiktoken
+from pathlib import Path
+from colorama import Fore
+from dotenv import load_dotenv
+from retry import retry
+from tqdm import tqdm
+from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost, getPricingConfig, calculateCost
+import tempfile
+
+# Globals
+MODEL = os.getenv("model")
+TIMEOUT = int(os.getenv("timeout"))
+LANGUAGE = os.getenv("language").capitalize()
+from util.paths import PROMPT_PATH, VOCAB_PATH
+
+PROMPT = PROMPT_PATH.read_text(encoding="utf-8")
+VOCAB = VOCAB_PATH.read_text(encoding="utf-8")
+LOCK = threading.Lock()
+WIDTH = int(os.getenv("width"))
+LISTWIDTH = int(os.getenv("listWidth"))
+NOTEWIDTH = 70
+MAXHISTORY = 10
+ESTIMATE = ""
+TOKENS = [0, 0]
+NAMESLIST = []
+FILENAME = None
+NAMES = False # Output a list of all the character names found
+BRFLAG = False # If the game uses
instead
+FIXTEXTWRAP = True # Overwrites textwrap
+IGNORETLTEXT = False # Ignores all translated text.
+MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
+
+# tqdm Globals
+BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
+POSITION = 0
+LEAVE = False
+PBAR = None
+
+# Regex - Need to change this if you want to translate from/to other languages. Default is Japanese Regex
+LANGREGEX = r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+"
+
+# Get pricing configuration based on the model
+PRICING_CONFIG = getPricingConfig(MODEL)
+INPUTAPICOST = PRICING_CONFIG["inputAPICost"]
+OUTPUTAPICOST = PRICING_CONFIG["outputAPICost"]
+BATCHSIZE = PRICING_CONFIG["batchSize"]
+FREQUENCY_PENALTY = PRICING_CONFIG["frequencyPenalty"]
+
+# Initialize Translation Config
+TRANSLATION_CONFIG = TranslationConfig(
+ model=MODEL,
+ language=LANGUAGE,
+ prompt=PROMPT,
+ vocab=VOCAB,
+ langRegex=LANGREGEX,
+ batchSize=BATCHSIZE,
+ maxHistory=MAXHISTORY,
+ estimateMode=False # Will be set dynamically based on ESTIMATE
+)
+LEAVE = False
+
+def handleJSON(filename, estimate):
+ global ESTIMATE, totalTokens, FILENAME
+ ESTIMATE = estimate
+ FILENAME = filename
+
+ if estimate:
+ start = time.time()
+ translatedData = openFiles(filename)
+
+ # Print Result
+ end = time.time()
+ tqdm.write(getResultString(translatedData, end - start, filename))
+ with LOCK:
+ TOKENS[0] += translatedData[1][0]
+ TOKENS[1] += translatedData[1][1]
+
+ return getResultString(["", TOKENS, None], end - start, "TOTAL")
+
+ else:
+ try:
+ start = time.time()
+ translatedData = openFiles(filename)
+
+ # Write final result after translation is complete
+ with open("translated/" + filename, "w", encoding="utf-8", newline="\n") as outFile:
+ json.dump(translatedData[0], outFile, ensure_ascii=False, indent=4)
+
+ # Print Result
+ end = time.time()
+ tqdm.write(getResultString(translatedData, end - start, filename))
+ with LOCK:
+ TOKENS[0] += translatedData[1][0]
+ TOKENS[1] += translatedData[1][1]
+ except Exception:
+ return "Fail"
+
+ return getResultString(["", TOKENS, None], end - start, "TOTAL")
+
+
+def openFiles(filename):
+ with open("files/" + filename, "r", encoding="UTF-8-sig") as f:
+ data = json.load(f)
+
+ # Map Files
+ if ".json" in filename:
+ translatedData = parseJSON(data, filename)
+
+ else:
+ raise NameError(filename + " Not Supported")
+
+ return translatedData
+
+
+def getResultString(translatedData, translationTime, filename):
+ # File Print String
+ cost = calculateCost(translatedData[1][0], translatedData[1][1], MODEL)
+ totalTokenstring = (
+ Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
+ "[Output: "
+ + str(translatedData[1][1])
+ + "]" "[Cost: ${:,.4f}".format(cost)
+ + "]"
+ )
+ timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
+
+ if translatedData[2] == None:
+ # Success
+ return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET
+
+ else:
+ # Fail
+ try:
+ raise translatedData[2]
+ except Exception as e:
+ traceback.print_exc()
+ errorString = str(e) + Fore.RED
+ return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET
+
+
+def parseJSON(data, filename):
+ totalTokens = [0, 0]
+ totalLines = 0
+ totalLines = len(data)
+ global LOCK, PBAR
+
+ with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar:
+ pbar.desc = filename
+ PBAR = pbar
+ try:
+ # Check if data is a simple key-value dict (not a list)
+ if isinstance(data, dict) and not isinstance(data, list):
+ # Check if it's a simple key-value format (all values are strings, not nested objects)
+ if all(isinstance(v, str) for v in data.values()):
+ result = translateSimpleKeyValueJSON(data, filename)
+ # PSData format (Nupu_PSData.json - MailDatas, MemoryData, ShopDatas, hStData)
+ elif any(k in data for k in ["MailDatas", "MemoryData", "ShopDatas", "hStData"]):
+ result = translatePSData(data, filename)
+ # RdData format (RdData.json - dgnDatas with dungeon/dialogue data)
+ elif "dgnDatas" in data:
+ result = translateRdData(data, filename)
+ # GameSetting format (Nupu_GameSetting.json - fzCardDatas, fzDeckDatas, rentalDecks)
+ elif any(k in data for k in ["fzCardDatas", "fzDeckDatas", "rentalDecks"]):
+ result = translateGameSetting(data, filename)
+ # Src/Tl dialogue format (numeric-keyed entries with src and tl fields)
+ elif data and all(isinstance(v, dict) and "src" in v for v in data.values()):
+ result = translateDialogueSrcTl(data, filename)
+ else:
+ result = translateJSON(data, filename, [])
+ else:
+ result = translateJSON(data, filename, [])
+ totalTokens[0] += result[0]
+ totalTokens[1] += result[1]
+ except Exception as e:
+ traceback.print_exc()
+ return [data, totalTokens, e]
+ return [data, totalTokens, None]
+
+
+def save_progress_json(data, filename):
+ """Save current JSON translation progress."""
+ try:
+ if ESTIMATE:
+ return
+ os.makedirs("translated", exist_ok=True)
+ tmp_path = os.path.join("translated", f"{filename}.tmp")
+ final_path = os.path.join("translated", filename)
+ with open(tmp_path, "w", encoding="utf-8", newline="\n") as outFile:
+ json.dump(data, outFile, ensure_ascii=False, indent=4)
+ os.replace(tmp_path, final_path)
+ except Exception:
+ traceback.print_exc()
+
+
+def translateSimpleKeyValueJSON(data, filename):
+ """
+ Translate simple key-value JSON format where keys contain Japanese text
+ and values are placeholder strings like "TODO".
+
+ Format example:
+ {
+ "\\r\\nスタビライザー": "TODO",
+ "\\r\\nプラグインなし": "TODO"
+ }
+ """
+ global LOCK, ESTIMATE, FILENAME, PBAR, MISMATCH
+ tokens = [0, 0]
+ stringList = []
+ keyList = []
+
+ # Pass 1: Collect all keys that need translation
+ for key, value in data.items():
+ # Check if the key contains text matching the language regex
+ if re.search(LANGREGEX, key):
+ # Strip whitespace and newline characters for translation
+ # cleanKey = key.strip().replace("\r\n", "").replace("\n", "").replace("\r", "")
+ # if cleanKey:
+ stringList.append(key)
+ keyList.append(key) # Store original key for reference
+
+ # Translate all keys if any were found
+ if stringList:
+ PBAR.total = len(stringList)
+ PBAR.refresh()
+
+ response = translateAI(stringList, "Reply with the English Translation")
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ translatedList = response[0]
+
+ # Check for mismatch
+ if len(stringList) != len(translatedList):
+ with LOCK:
+ if FILENAME not in MISMATCH:
+ MISMATCH.append(FILENAME)
+
+ # Pass 2: Update the values with translations
+ for i, key in enumerate(keyList):
+ if i < len(translatedList):
+ translatedText = translatedList[i]
+ # Set the translated text as the value
+ data[key] = translatedText
+
+ # Save progress after each translation
+ save_progress_json(data, filename)
+
+ return tokens
+
+
+def translatePSData(data, filename):
+ """Translate PSData JSON format (e.g. Nupu_PSData.json).
+
+ Handles four sections:
+ - MailDatas: In-game mail/message system
+ - MemoryData: Scene/CG gallery memories
+ - ShopDatas: Shop configurations and dialogue
+ - hStData: Status rank descriptions
+
+ Only translates player-visible text fields.
+ """
+ global LOCK, ESTIMATE, FILENAME, PBAR, MISMATCH
+ tokens = [0, 0]
+
+ def batchTranslate(stringList, context):
+ """Translate a list of strings and return translations. Returns [] on mismatch."""
+ nonlocal tokens
+ if not stringList:
+ return []
+ response = translateAI(stringList, context)
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ if len(stringList) != len(response[0]):
+ with LOCK:
+ if FILENAME not in MISMATCH:
+ MISMATCH.append(FILENAME)
+ return []
+ return response[0]
+
+ # ================================================================
+ # PASS 1: Collect all translatable strings from every section
+ # ================================================================
+
+ # -- MailDatas --
+ mailNameS, mailNameI = [], []
+ mailTitleS, mailTitleI = [], []
+ mailListNameS, mailListNameI = [], []
+ mailTextS, mailTextI = [], []
+
+ if "MailDatas" in data:
+ for i, m in enumerate(data["MailDatas"]):
+ if m.get("Name"):
+ mailNameS.append(m["Name"])
+ mailNameI.append(i)
+ if m.get("Title"):
+ mailTitleS.append(m["Title"])
+ mailTitleI.append(i)
+ if m.get("MailListName"):
+ mailListNameS.append(m["MailListName"])
+ mailListNameI.append(i)
+ if m.get("MailText"):
+ mailTextS.append(m["MailText"].replace("\r\n", " ").replace("\n", " ").strip())
+ mailTextI.append(i)
+
+ # -- MemoryData --
+ memTitleS, memTitleI = [], []
+ memSetuS, memSetuI = [], []
+ memHintS, memHintI = [], []
+ memPtnS, memPtnI = [], [] # indices are (memIdx, ptnIdx)
+ memDtlSetuS, memDtlSetuI = [], [] # indices are (memIdx, dtlIdx)
+
+ if "MemoryData" in data:
+ for i, mem in enumerate(data["MemoryData"]):
+ if mem.get("_Title"):
+ memTitleS.append(mem["_Title"])
+ memTitleI.append(i)
+ if mem.get("_Setu"):
+ memSetuS.append(mem["_Setu"].replace("\r\n", " ").replace("\n", " ").strip())
+ memSetuI.append(i)
+ if mem.get("_Hint"):
+ memHintS.append(mem["_Hint"])
+ memHintI.append(i)
+ if "_Ptn" in mem and isinstance(mem["_Ptn"], list):
+ for j, p in enumerate(mem["_Ptn"]):
+ if p:
+ memPtnS.append(p)
+ memPtnI.append((i, j))
+ if "_DtlCheck" in mem and isinstance(mem["_DtlCheck"], list):
+ for j, dtl in enumerate(mem["_DtlCheck"]):
+ if isinstance(dtl, dict) and dtl.get("_Setu"):
+ memDtlSetuS.append(dtl["_Setu"].replace("\r\n", " ").replace("\n", " ").strip())
+ memDtlSetuI.append((i, j))
+
+ # -- ShopDatas --
+ shopNameS, shopNameI = [], []
+ shopTextFields = ["buySelf", "emptySelf", "firstSelf", "husokuSelf",
+ "randSelf1", "randSelf2", "randSelf3"]
+ shopTextS = {f: [] for f in shopTextFields}
+ shopTextI = {f: [] for f in shopTextFields}
+
+ if "ShopDatas" in data:
+ for i, shop in enumerate(data["ShopDatas"]):
+ if shop.get("name"):
+ shopNameS.append(shop["name"])
+ shopNameI.append(i)
+ for field in shopTextFields:
+ if field in shop and isinstance(shop[field], dict):
+ text = shop[field].get("text", "")
+ if text:
+ shopTextS[field].append(text.replace("\r\n", " ").replace("\n", " ").strip())
+ shopTextI[field].append(i)
+
+ # -- hStData --
+ rankArrayNames = ["KutiRankDatas", "MuneRankDatas", "SiriRankDatas", "TituRankDatas"]
+ rankSetuS, rankSetuI = [], [] # indices are (arrayName, idx)
+ rankText1S, rankText1I = [], []
+ rankText2S, rankText2I = [], []
+
+ if "hStData" in data:
+ hst = data["hStData"]
+ for arrName in rankArrayNames:
+ if arrName in hst and isinstance(hst[arrName], list):
+ for i, rank in enumerate(hst[arrName]):
+ if rank.get("rankSetu"):
+ rankSetuS.append(rank["rankSetu"].replace("\r\n", " ").replace("\n", " ").strip())
+ rankSetuI.append((arrName, i))
+ if rank.get("rankText1"):
+ rankText1S.append(rank["rankText1"])
+ rankText1I.append((arrName, i))
+ if rank.get("rankText2"):
+ rankText2S.append(rank["rankText2"])
+ rankText2I.append((arrName, i))
+
+ # Set progress bar total
+ totalItems = (
+ len(mailNameS) + len(mailTitleS) + len(mailListNameS) + len(mailTextS)
+ + len(memTitleS) + len(memSetuS) + len(memHintS) + len(memPtnS) + len(memDtlSetuS)
+ + len(shopNameS) + sum(len(v) for v in shopTextS.values())
+ + len(rankSetuS) + len(rankText1S) + len(rankText2S)
+ )
+ PBAR.total = totalItems
+ PBAR.refresh()
+
+ # ================================================================
+ # PASS 2: Translate each batch and apply results back to data
+ # ================================================================
+
+ # -- MailDatas --
+ if "MailDatas" in data:
+ mails = data["MailDatas"]
+
+ for tl, idx in zip(batchTranslate(mailNameS, "Character Name"), mailNameI):
+ mails[idx]["Name"] = tl
+ if mailNameS:
+ save_progress_json(data, filename)
+
+ for tl, idx in zip(batchTranslate(mailTitleS, "Mail Subject"), mailTitleI):
+ mails[idx]["Title"] = tl
+ if mailTitleS:
+ save_progress_json(data, filename)
+
+ for tl, idx in zip(batchTranslate(mailListNameS, "Mail List Entry"), mailListNameI):
+ mails[idx]["MailListName"] = tl
+ if mailListNameS:
+ save_progress_json(data, filename)
+
+ for tl, idx in zip(batchTranslate(mailTextS, "Mail Body Text"), mailTextI):
+ mails[idx]["MailText"] = dazedwrap.wrapText(tl, WIDTH)
+ if mailTextS:
+ save_progress_json(data, filename)
+
+ # -- MemoryData --
+ if "MemoryData" in data:
+ memories = data["MemoryData"]
+
+ for tl, idx in zip(batchTranslate(memTitleS, "Scene Title"), memTitleI):
+ memories[idx]["_Title"] = tl
+ if memTitleS:
+ save_progress_json(data, filename)
+
+ for tl, idx in zip(batchTranslate(memSetuS, "Scene Description"), memSetuI):
+ memories[idx]["_Setu"] = tl
+ if memSetuS:
+ save_progress_json(data, filename)
+
+ for tl, idx in zip(batchTranslate(memHintS, "Hint Text"), memHintI):
+ memories[idx]["_Hint"] = tl
+ if memHintS:
+ save_progress_json(data, filename)
+
+ for tl, (mi, pi) in zip(batchTranslate(memPtnS, "Scene Description"), memPtnI):
+ memories[mi]["_Ptn"][pi] = tl
+ if memPtnS:
+ save_progress_json(data, filename)
+
+ for tl, (mi, di) in zip(batchTranslate(memDtlSetuS, "Character Commentary"), memDtlSetuI):
+ memories[mi]["_DtlCheck"][di]["_Setu"] = dazedwrap.wrapText(tl, LISTWIDTH)
+ if memDtlSetuS:
+ save_progress_json(data, filename)
+
+ # -- ShopDatas --
+ if "ShopDatas" in data:
+ shops = data["ShopDatas"]
+
+ for tl, idx in zip(batchTranslate(shopNameS, "Shop Name"), shopNameI):
+ shops[idx]["name"] = tl
+ if shopNameS:
+ save_progress_json(data, filename)
+
+ for field in shopTextFields:
+ for tl, idx in zip(batchTranslate(shopTextS[field], "Shop Dialogue"), shopTextI[field]):
+ shops[idx][field]["text"] = dazedwrap.wrapText(tl, WIDTH)
+ if shopTextS[field]:
+ save_progress_json(data, filename)
+
+ # -- hStData --
+ if "hStData" in data:
+ hst = data["hStData"]
+
+ for tl, (an, idx) in zip(batchTranslate(rankSetuS, "Rank Description"), rankSetuI):
+ hst[an][idx]["rankSetu"] = dazedwrap.wrapText(tl, WIDTH)
+ if rankSetuS:
+ save_progress_json(data, filename)
+
+ for tl, (an, idx) in zip(batchTranslate(rankText1S, "Rank Label"), rankText1I):
+ hst[an][idx]["rankText1"] = tl
+ if rankText1S:
+ save_progress_json(data, filename)
+
+ for tl, (an, idx) in zip(batchTranslate(rankText2S, "Rank Subtitle"), rankText2I):
+ hst[an][idx]["rankText2"] = tl
+ if rankText2S:
+ save_progress_json(data, filename)
+
+ return tokens
+
+
+def translateRdData(data, filename):
+ """Translate RdData JSON format (e.g. RdData.json).
+
+ Handles dungeon data with nested dialogue, choices, and UI text.
+ Only translates player-visible text fields.
+ """
+ global LOCK, ESTIMATE, FILENAME, PBAR, MISMATCH
+ tokens = [0, 0]
+
+ def batchTranslate(stringList, context):
+ """Translate a list of strings and return translations. Returns [] on mismatch."""
+ nonlocal tokens
+ if not stringList:
+ return []
+ response = translateAI(stringList, context)
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ if len(stringList) != len(response[0]):
+ with LOCK:
+ if FILENAME not in MISMATCH:
+ MISMATCH.append(FILENAME)
+ return []
+ return response[0]
+
+ if "dgnDatas" not in data:
+ return tokens
+
+ dgns = data["dgnDatas"]
+
+ # ================================================================
+ # PASS 1: Collect all translatable strings
+ # ================================================================
+
+ # -- Dungeon-level short fields --
+ dgnNameS, dgnNameI = [], []
+ clearTxtS, clearTxtI = [], []
+ hosyuTxtS, hosyuTxtI = [], []
+ firstHosyuTxtS, firstHosyuTxtI = [], []
+ memoTxtS, memoTxtI = [], []
+ appMemoTxtS, appMemoTxtI = [], []
+ areaMemoS, areaMemoI = [], []
+
+ # -- Dungeon-level long text fields --
+ naiyoTxtS, naiyoTxtI = [], []
+ naiyoTxtExS, naiyoTxtExI = [], []
+ johoTxtS, johoTxtI = [], []
+ johoTxtExS, johoTxtExI = [], []
+
+ # -- Floor names --
+ kaisoNameS, kaisoNameI = [], [] # (dgnIdx, kaisoIdx)
+
+ # -- Dialogue (talkDatas / talkDatasEx) --
+ talkNameS, talkNameI = [], [] # (dgnIdx, talkKey, talkIdx)
+ speakerNameS, speakerNameI = [], [] # (dgnIdx, talkKey, talkIdx, speechIdx)
+ dialogueS, dialogueI = [], [] # (dgnIdx, talkKey, talkIdx, speechIdx)
+
+ # -- Choice events (sijiDatas) --
+ sijiNameS, sijiNameI = [], [] # (dgnIdx, sijiIdx)
+ sijiChoiceS, sijiChoiceI = [], [] # (dgnIdx, sijiIdx)
+
+ for di, dgn in enumerate(dgns):
+ if dgn is None:
+ continue
+
+ # Dungeon short fields
+ if dgn.get("name"):
+ dgnNameS.append(dgn["name"])
+ dgnNameI.append(di)
+ if dgn.get("clearTxt"):
+ clearTxtS.append(dgn["clearTxt"])
+ clearTxtI.append(di)
+ if dgn.get("hosyuTxt"):
+ hosyuTxtS.append(dgn["hosyuTxt"])
+ hosyuTxtI.append(di)
+ if dgn.get("firstHosyuTxt"):
+ firstHosyuTxtS.append(dgn["firstHosyuTxt"])
+ firstHosyuTxtI.append(di)
+ if dgn.get("memoTxt"):
+ memoTxtS.append(dgn["memoTxt"])
+ memoTxtI.append(di)
+ if dgn.get("appMemoTxt"):
+ appMemoTxtS.append(dgn["appMemoTxt"])
+ appMemoTxtI.append(di)
+ if dgn.get("areaMemo"):
+ areaMemoS.append(dgn["areaMemo"])
+ areaMemoI.append(di)
+
+ # Dungeon long text fields (strip line breaks for translation)
+ if dgn.get("naiyoTxt"):
+ naiyoTxtS.append(dgn["naiyoTxt"].replace("\r\n", " ").replace("\n", " ").strip())
+ naiyoTxtI.append(di)
+ if dgn.get("naiyoTxtEx"):
+ naiyoTxtExS.append(dgn["naiyoTxtEx"].replace("\r\n", " ").replace("\n", " ").strip())
+ naiyoTxtExI.append(di)
+ if dgn.get("johoTxt"):
+ johoTxtS.append(dgn["johoTxt"].replace("\r\n", " ").replace("\n", " ").strip())
+ johoTxtI.append(di)
+ if dgn.get("johoTxtEx"):
+ johoTxtExS.append(dgn["johoTxtEx"].replace("\r\n", " ").replace("\n", " ").strip())
+ johoTxtExI.append(di)
+
+ # Floor names
+ if "kaisoDatas" in dgn and isinstance(dgn["kaisoDatas"], list):
+ for ki, kaiso in enumerate(dgn["kaisoDatas"]):
+ if isinstance(kaiso, dict) and kaiso.get("kaisoName"):
+ kaisoNameS.append(kaiso["kaisoName"])
+ kaisoNameI.append((di, ki))
+
+ # Dialogue from talkDatas and talkDatasEx
+ for talkKey in ["talkDatas", "talkDatasEx"]:
+ if talkKey in dgn and isinstance(dgn[talkKey], list):
+ for ti, talk in enumerate(dgn[talkKey]):
+ if not isinstance(talk, dict):
+ continue
+ if talk.get("name"):
+ talkNameS.append(talk["name"])
+ talkNameI.append((di, talkKey, ti))
+ if "speachDatas" in talk and isinstance(talk["speachDatas"], list):
+ for si, speech in enumerate(talk["speachDatas"]):
+ if not isinstance(speech, dict):
+ continue
+ if speech.get("name"):
+ speakerNameS.append(speech["name"])
+ speakerNameI.append((di, talkKey, ti, si))
+ if speech.get("text"):
+ dialogueS.append(speech["text"].replace("\r\n", " ").replace("\n", " ").strip())
+ dialogueI.append((di, talkKey, ti, si))
+
+ # Choice events (sijiDatas)
+ if "sijiDatas" in dgn and isinstance(dgn["sijiDatas"], list):
+ for si, siji in enumerate(dgn["sijiDatas"]):
+ if not isinstance(siji, dict):
+ continue
+ if siji.get("name"):
+ sijiNameS.append(siji["name"])
+ sijiNameI.append((di, si))
+ if siji.get("choiceSijiTxt"):
+ sijiChoiceS.append(siji["choiceSijiTxt"].replace("\r\n", " ").replace("\n", " ").strip())
+ sijiChoiceI.append((di, si))
+
+ # Set progress bar total
+ totalItems = (
+ len(dgnNameS) + len(clearTxtS) + len(hosyuTxtS) + len(firstHosyuTxtS)
+ + len(memoTxtS) + len(appMemoTxtS) + len(areaMemoS)
+ + len(naiyoTxtS) + len(naiyoTxtExS) + len(johoTxtS) + len(johoTxtExS)
+ + len(kaisoNameS)
+ + len(talkNameS) + len(speakerNameS) + len(dialogueS)
+ + len(sijiNameS) + len(sijiChoiceS)
+ )
+ PBAR.total = totalItems
+ PBAR.refresh()
+
+ # ================================================================
+ # PASS 2: Translate each batch and apply results back to data
+ # ================================================================
+
+ # -- Dungeon short fields --
+ for tl, idx in zip(batchTranslate(dgnNameS, "Dungeon Name"), dgnNameI):
+ dgns[idx]["name"] = tl
+ if dgnNameS:
+ save_progress_json(data, filename)
+
+ for tl, idx in zip(batchTranslate(clearTxtS, "Clear Condition"), clearTxtI):
+ dgns[idx]["clearTxt"] = tl
+ if clearTxtS:
+ save_progress_json(data, filename)
+
+ for tl, idx in zip(batchTranslate(hosyuTxtS, "Reward Description"), hosyuTxtI):
+ dgns[idx]["hosyuTxt"] = tl
+ if hosyuTxtS:
+ save_progress_json(data, filename)
+
+ for tl, idx in zip(batchTranslate(firstHosyuTxtS, "First Clear Reward"), firstHosyuTxtI):
+ dgns[idx]["firstHosyuTxt"] = tl
+ if firstHosyuTxtS:
+ save_progress_json(data, filename)
+
+ for tl, idx in zip(batchTranslate(memoTxtS, "Memo Label"), memoTxtI):
+ dgns[idx]["memoTxt"] = tl
+ if memoTxtS:
+ save_progress_json(data, filename)
+
+ for tl, idx in zip(batchTranslate(appMemoTxtS, "App Memo"), appMemoTxtI):
+ dgns[idx]["appMemoTxt"] = tl
+ if appMemoTxtS:
+ save_progress_json(data, filename)
+
+ for tl, idx in zip(batchTranslate(areaMemoS, "Area Label"), areaMemoI):
+ dgns[idx]["areaMemo"] = tl
+ if areaMemoS:
+ save_progress_json(data, filename)
+
+ # -- Dungeon long text fields (with word wrap) --
+ for tl, idx in zip(batchTranslate(naiyoTxtS, "Mission Description"), naiyoTxtI):
+ dgns[idx]["naiyoTxt"] = dazedwrap.wrapText(tl, WIDTH)
+ if naiyoTxtS:
+ save_progress_json(data, filename)
+
+ for tl, idx in zip(batchTranslate(naiyoTxtExS, "Extended Mission Description"), naiyoTxtExI):
+ dgns[idx]["naiyoTxtEx"] = dazedwrap.wrapText(tl, WIDTH)
+ if naiyoTxtExS:
+ save_progress_json(data, filename)
+
+ for tl, idx in zip(batchTranslate(johoTxtS, "Info Text"), johoTxtI):
+ dgns[idx]["johoTxt"] = dazedwrap.wrapText(tl, WIDTH)
+ if johoTxtS:
+ save_progress_json(data, filename)
+
+ for tl, idx in zip(batchTranslate(johoTxtExS, "Extended Info Text"), johoTxtExI):
+ dgns[idx]["johoTxtEx"] = dazedwrap.wrapText(tl, WIDTH)
+ if johoTxtExS:
+ save_progress_json(data, filename)
+
+ # -- Floor names --
+ for tl, (di, ki) in zip(batchTranslate(kaisoNameS, "Floor Name"), kaisoNameI):
+ dgns[di]["kaisoDatas"][ki]["kaisoName"] = tl
+ if kaisoNameS:
+ save_progress_json(data, filename)
+
+ # -- Dialogue event names --
+ for tl, (di, tk, ti) in zip(batchTranslate(talkNameS, "Event Label"), talkNameI):
+ dgns[di][tk][ti]["name"] = tl
+ if talkNameS:
+ save_progress_json(data, filename)
+
+ # -- Speaker names --
+ for tl, (di, tk, ti, si) in zip(batchTranslate(speakerNameS, "Character Name"), speakerNameI):
+ dgns[di][tk][ti]["speachDatas"][si]["name"] = tl
+ if speakerNameS:
+ save_progress_json(data, filename)
+
+ # -- Dialogue text (with word wrap) --
+ for tl, (di, tk, ti, si) in zip(batchTranslate(dialogueS, "Dialogue"), dialogueI):
+ dgns[di][tk][ti]["speachDatas"][si]["text"] = dazedwrap.wrapText(tl, WIDTH)
+ if dialogueS:
+ save_progress_json(data, filename)
+
+ # -- Choice event names --
+ for tl, (di, si) in zip(batchTranslate(sijiNameS, "Event Name"), sijiNameI):
+ dgns[di]["sijiDatas"][si]["name"] = tl
+ if sijiNameS:
+ save_progress_json(data, filename)
+
+ # -- Choice text (with word wrap) --
+ for tl, (di, si) in zip(batchTranslate(sijiChoiceS, "Choice Event Text"), sijiChoiceI):
+ dgns[di]["sijiDatas"][si]["choiceSijiTxt"] = dazedwrap.wrapText(tl, WIDTH)
+ if sijiChoiceS:
+ save_progress_json(data, filename)
+
+ return tokens
+
+
+def translateGameSetting(data, filename):
+ """Translate GameSetting JSON format (e.g. Nupu_GameSetting.json).
+
+ Handles card data, deck definitions, and rental decks.
+ Only translates player-visible text fields.
+ """
+ global LOCK, ESTIMATE, FILENAME, PBAR, MISMATCH
+ tokens = [0, 0]
+
+ def batchTranslate(stringList, context):
+ """Translate a list of strings and return translations. Returns [] on mismatch."""
+ nonlocal tokens
+ if not stringList:
+ return []
+ response = translateAI(stringList, context)
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ if len(stringList) != len(response[0]):
+ with LOCK:
+ if FILENAME not in MISMATCH:
+ MISMATCH.append(FILENAME)
+ return []
+ return response[0]
+
+ # ================================================================
+ # PASS 1: Collect all translatable strings
+ # ================================================================
+
+ # -- fzCardDatas: card descriptions --
+ desc1S, desc1I = [], []
+ desc2S, desc2I = [], []
+
+ if "fzCardDatas" in data:
+ for i, card in enumerate(data["fzCardDatas"]):
+ if card.get("desc1"):
+ desc1S.append(card["desc1"].replace("\r\n", " ").replace("\n", " ").strip())
+ desc1I.append(i)
+ if card.get("desc2"):
+ desc2S.append(card["desc2"].replace("\r\n", " ").replace("\n", " ").strip())
+ desc2I.append(i)
+
+ # -- fzDeckDatas: deck name and description --
+ deckNameS, deckNameI = [], []
+ deckDescS, deckDescI = [], []
+
+ if "fzDeckDatas" in data:
+ for i, deck in enumerate(data["fzDeckDatas"]):
+ if deck.get("name"):
+ deckNameS.append(deck["name"])
+ deckNameI.append(i)
+ if deck.get("desc"):
+ deckDescS.append(deck["desc"].replace("\r\n", " ").replace("\n", " ").strip())
+ deckDescI.append(i)
+
+ # -- rentalDecks: memo label --
+ rentalMemoS, rentalMemoI = [], []
+
+ if "rentalDecks" in data:
+ for i, rental in enumerate(data["rentalDecks"]):
+ if rental.get("memo"):
+ rentalMemoS.append(rental["memo"])
+ rentalMemoI.append(i)
+
+ # Set progress bar total
+ totalItems = (
+ len(desc1S) + len(desc2S)
+ + len(deckNameS) + len(deckDescS)
+ + len(rentalMemoS)
+ )
+ PBAR.total = totalItems
+ PBAR.refresh()
+
+ # ================================================================
+ # PASS 2: Translate each batch and apply results back to data
+ # ================================================================
+
+ # -- fzCardDatas --
+ if "fzCardDatas" in data:
+ cards = data["fzCardDatas"]
+
+ for tl, idx in zip(batchTranslate(desc1S, "Card Description"), desc1I):
+ cards[idx]["desc1"] = dazedwrap.wrapText(tl, WIDTH)
+ if desc1S:
+ save_progress_json(data, filename)
+
+ for tl, idx in zip(batchTranslate(desc2S, "Card Description"), desc2I):
+ cards[idx]["desc2"] = dazedwrap.wrapText(tl, WIDTH)
+ if desc2S:
+ save_progress_json(data, filename)
+
+ # -- fzDeckDatas --
+ if "fzDeckDatas" in data:
+ decks = data["fzDeckDatas"]
+
+ for tl, idx in zip(batchTranslate(deckNameS, "Deck Name"), deckNameI):
+ decks[idx]["name"] = tl
+ if deckNameS:
+ save_progress_json(data, filename)
+
+ for tl, idx in zip(batchTranslate(deckDescS, "Deck Description"), deckDescI):
+ decks[idx]["desc"] = dazedwrap.wrapText(tl, WIDTH)
+ if deckDescS:
+ save_progress_json(data, filename)
+
+ # -- rentalDecks --
+ if "rentalDecks" in data:
+ rentals = data["rentalDecks"]
+
+ for tl, idx in zip(batchTranslate(rentalMemoS, "Deck Label"), rentalMemoI):
+ rentals[idx]["memo"] = tl
+ if rentalMemoS:
+ save_progress_json(data, filename)
+
+ return tokens
+
+
+def translateDialogueSrcTl(data, filename):
+ """Translate src/tl dialogue JSON format.
+
+ Format:
+ {
+ "0": {"type": "dialogue", "src": "Japanese text", "tl": ""},
+ "4": {"type": "dialogue", "speaker": "琴音", "src": "「...」", "tl": ""}
+ }
+
+ Translates 'src' into 'tl'. Also translates Japanese 'speaker' fields.
+ Skips entries where 'tl' is already populated.
+ """
+ global LOCK, ESTIMATE, FILENAME, PBAR, MISMATCH
+ tokens = [0, 0]
+ srcList = []
+ keyList = [] # list of (key, entry_type)
+
+ # Collect all entries with translatable content
+ for key, entry in data.items():
+ if not isinstance(entry, dict):
+ continue
+ src = entry.get("src", "")
+ if not src:
+ continue
+ tl = entry.get("tl", "")
+ entry_type = entry.get("type", "")
+ speaker = entry.get("speaker", "")
+
+ if tl:
+ # Already has a translation — send tl so the module can decide to skip it
+ srcList.append(tl)
+ else:
+ if not re.search(LANGREGEX, src):
+ continue
+ srcClean = src.replace("\r\n", " ").replace("\n", " ").strip()
+ if speaker:
+ srcList.append(f"[{speaker}]: {srcClean}")
+ else:
+ srcList.append(srcClean)
+ keyList.append((key, entry_type))
+
+ if srcList:
+ PBAR.total = len(srcList)
+ PBAR.refresh()
+
+ response = translateAI(srcList, "Reply with the English Translation")
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ translatedList = response[0]
+
+ if len(srcList) != len(translatedList):
+ with LOCK:
+ if FILENAME not in MISMATCH:
+ MISMATCH.append(FILENAME)
+
+ for i, (key, entry_type) in enumerate(keyList):
+ if i >= len(translatedList):
+ break
+ translatedText = translatedList[i]
+
+ # Strip speaker prefix if the AI echoed it back
+ # Handles: [Speaker]: text | Speaker: text | Speaker(text) / CJK(text)
+ match = re.search(r'^\[.+?\]\s?[|:]\s?', translatedText)
+ if match:
+ translatedText = translatedText[match.end():]
+ else:
+ # Fallback: strip any leading Japanese/CJK name followed by ( or :
+ cjk_m = re.match(r'^[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+\s*[\(（:]\s*', translatedText)
+ if cjk_m:
+ translatedText = translatedText[cjk_m.end():]
+ translatedText = re.sub(r'\s*[)）]\s*$', '', translatedText)
+
+ data[key]["tl"] = translatedText
+ save_progress_json(data, filename)
+
+ # Translate any Japanese speaker names found in the data
+ seen_speakers = {}
+ speaker_updated = False
+ for key, entry in data.items():
+ if not isinstance(entry, dict):
+ continue
+ speaker = entry.get("speaker", "")
+ if not speaker or not re.search(LANGREGEX, speaker):
+ continue
+ if speaker not in seen_speakers:
+ response = getSpeaker(speaker)
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ seen_speakers[speaker] = response[0]
+ data[key]["speaker"] = seen_speakers[speaker]
+ speaker_updated = True
+ if speaker_updated:
+ save_progress_json(data, filename)
+
+ return tokens
+
+
+def translateJSON(data, filename, translatedList):
+ global LOCK, ESTIMATE, FILENAME, PBAR, MISMATCH
+ if translatedList:
+ stringList = translatedList[0]
+ eventList = translatedList[1]
+ else:
+ stringList = []
+ eventList = [[], [], [], [], [], [], []] # [title, process, text, key, target, job, place]
+ tokens = [0, 0]
+ speaker = ""
+ i = 0
+ stringListTL = []
+ eventListTL = [[], [], [], [], [], [], []]
+
+ while i < len(data):
+ speakerKey = "character_nameText"
+ messageKey = "m_text"
+
+ # Event List Format - Key
+ if "key" in data[i] and data[i]["key"]:
+ jaString = data[i]["key"]
+
+ # Pass 1
+ if not translatedList:
+ eventList[3].append(jaString.strip())
+
+ # Pass 2
+ else:
+ if eventList[3]:
+ translatedText = eventList[3][0]
+ eventList[3].pop(0)
+
+ # Set Data
+ data[i]["key"] = translatedText
+ save_progress_json(data, filename)
+
+ # Event List Format - Title
+ if "title" in data[i] and data[i]["title"]:
+ jaString = data[i]["title"]
+
+ # Pass 1
+ if not translatedList:
+ eventList[0].append(jaString.strip())
+
+ # Pass 2
+ else:
+ if eventList[0]:
+ translatedText = eventList[0][0]
+ eventList[0].pop(0)
+
+ # Set Data
+ data[i]["title"] = translatedText
+ save_progress_json(data, filename)
+
+ # Event List Format - Target
+ if "target" in data[i] and data[i]["target"]:
+ jaString = data[i]["target"]
+
+ # Pass 1
+ if not translatedList:
+ eventList[4].append(jaString.strip())
+
+ # Pass 2
+ else:
+ if eventList[4]:
+ translatedText = eventList[4][0]
+ eventList[4].pop(0)
+
+ # Set Data
+ data[i]["target"] = translatedText
+ save_progress_json(data, filename)
+
+ # Event List Format - Job
+ if "job" in data[i] and data[i]["job"]:
+ jaString = data[i]["job"]
+
+ # Pass 1
+ if not translatedList:
+ eventList[5].append(jaString.strip())
+
+ # Pass 2
+ else:
+ if eventList[5]:
+ translatedText = eventList[5][0]
+ eventList[5].pop(0)
+
+ # Set Data
+ data[i]["job"] = translatedText
+ save_progress_json(data, filename)
+
+ # Event List Format - Place
+ if "place" in data[i] and data[i]["place"]:
+ jaString = data[i]["place"]
+
+ # Pass 1
+ if not translatedList:
+ eventList[6].append(jaString.strip())
+
+ # Pass 2
+ else:
+ if eventList[6]:
+ translatedText = eventList[6][0]
+ eventList[6].pop(0)
+
+ # Set Data
+ data[i]["place"] = translatedText
+ save_progress_json(data, filename)
+
+ # Event List Format - Process
+ if "process" in data[i] and data[i]["process"]:
+ jaString = data[i]["process"]
+
+ # Pass 1
+ if not translatedList:
+ eventList[1].append(jaString.strip())
+
+ # Pass 2
+ else:
+ if eventList[1]:
+ translatedText = eventList[1][0]
+ eventList[1].pop(0)
+
+ # Set Data
+ data[i]["process"] = translatedText
+ save_progress_json(data, filename)
+
+ # Event List Format - Text
+ if "text" in data[i] and data[i]["text"]:
+ jaString = data[i]["text"]
+ # Pass 1
+ if not translatedList:
+ # Replace \n with space for translation
+ jaStringClean = jaString.replace("\n", " ")
+ eventList[2].append(jaStringClean.strip())
+
+ # Pass 2
+ else:
+ if eventList[2]:
+ translatedText = eventList[2][0]
+ eventList[2].pop(0)
+
+ # Apply text wrapping and restore linebreaks
+ translatedText = dazedwrap.wrapText(translatedText, 70)
+
+ # Set Data
+ data[i]["text"] = translatedText
+ save_progress_json(data, filename)
+
+ # Speaker
+ if speakerKey in data[i] and data[i][speakerKey]:
+ # Grab and TL
+ speaker = data[i][speakerKey]
+ response = getSpeaker(speaker)
+ speaker = response[0]
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+
+ # Set Speaker
+ data[i][speakerKey] = speaker
+
+ # Dialogue
+ if messageKey in data[i]\
+ and data[i][messageKey].strip()\
+ and data[i][messageKey] != "a"\
+ and data[i][messageKey].replace("\u3000", "").strip() != "":
+ jaString = data[i][messageKey]
+
+ # Save Original String
+ originalString = jaString
+
+ # If there isn't any Japanese in the text just skip
+ if not re.search(LANGREGEX, jaString):
+ i += 1
+ continue
+
+ # Pass 1
+ if not translatedList:
+ # Strip Spaces
+ jaString = jaString.strip()
+
+ # Remove Textwrap
+ # jaString = jaString.replace('\n', ' ')
+
+ if jaString:
+ if speaker:
+ stringList.append(f"[{speaker}]: {jaString}")
+ else:
+ stringList.append(jaString)
+
+ # Pass 2
+ else:
+ # Get Text
+ if stringList:
+ # Grab and Pop
+ translatedText = stringList[0]
+ stringList.pop(0)
+
+ # Set to None if empty list
+ if len(stringList) <= 0:
+ stringList = None
+
+ # Remove speaker prefix — handles [Name]: / Name: / CJK(text)
+ match = re.search(r'(^\[.+?\]\s?[|:]\s?)', translatedText)
+ if match:
+ translatedText = translatedText.replace(match.group(1), "")
+ else:
+ cjk_m = re.match(r'^[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+\s*[\(（:]\s*', translatedText)
+ if cjk_m:
+ translatedText = translatedText[cjk_m.end():]
+ translatedText = re.sub(r'\s*[)）]\s*$', '', translatedText)
+
+ # Escape Quotes
+ translatedText = re.sub(r'(?", ")")
+ translatedText = translatedText.replace("『", "")
+ translatedText = translatedText.replace("』", "")
+
+ # Remove GPT ' Quotes
+ if translatedText:
+ if translatedText[0] == "'":
+ translatedText = translatedText[1:]
+ if translatedText[-1] == "'":
+ translatedText = translatedText[:-1]
+ else:
+ print("Warning: Empty Translation for", originalString)
+
+ # Textwrap
+ # translatedText = dazedwrap.wrapText(translatedText, width=WIDTH)
+
+ # Set Data
+ if "『" in data[i][messageKey] and "』" not in translatedText:
+ data[i][messageKey] = data[i][messageKey].replace(originalString, f"『{translatedText}』")
+ else:
+ data[i][messageKey] = data[i][messageKey].replace(originalString, f"{translatedText}")
+
+ # Save progress after each message replacement
+ save_progress_json(data, filename)
+ # Next Value
+ i += 1
+
+ # EOF - Only do translation if this is Pass 1 (collecting strings)
+ if not translatedList:
+ # Event List
+ if any(eventList):
+ PBAR.total = sum(len(event) for event in eventList)
+ PBAR.refresh()
+
+ # Event Title
+ if eventList[0]:
+ response = translateAI(eventList[0], "Event Title")
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ eventListTL[0] = response[0]
+
+ if len(eventList[0]) != len(eventListTL[0]):
+ with LOCK:
+ if FILENAME not in MISMATCH:
+ MISMATCH.append(FILENAME)
+
+ # Event Process
+ if eventList[1]:
+ response = translateAI(eventList[1], "Event Process")
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ eventListTL[1] = response[0]
+
+ if len(eventList[1]) != len(eventListTL[1]):
+ with LOCK:
+ if FILENAME not in MISMATCH:
+ MISMATCH.append(FILENAME)
+
+ # Event Text
+ if eventList[2]:
+ response = translateAI(eventList[2], "Event Description")
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ eventListTL[2] = response[0]
+
+ if len(eventList[2]) != len(eventListTL[2]):
+ with LOCK:
+ if FILENAME not in MISMATCH:
+ MISMATCH.append(FILENAME)
+
+ # Event Key
+ if eventList[3]:
+ response = translateAI(eventList[3], "Event Key")
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ eventListTL[3] = response[0]
+
+ if len(eventList[3]) != len(eventListTL[3]):
+ with LOCK:
+ if FILENAME not in MISMATCH:
+ MISMATCH.append(FILENAME)
+
+ # Event Target
+ if eventList[4]:
+ response = translateAI(eventList[4], "Character Name")
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ eventListTL[4] = response[0]
+
+ if len(eventList[4]) != len(eventListTL[4]):
+ with LOCK:
+ if FILENAME not in MISMATCH:
+ MISMATCH.append(FILENAME)
+
+ # Event Job
+ if eventList[5]:
+ response = translateAI(eventList[5], "Job/Occupation")
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ eventListTL[5] = response[0]
+
+ if len(eventList[5]) != len(eventListTL[5]):
+ with LOCK:
+ if FILENAME not in MISMATCH:
+ MISMATCH.append(FILENAME)
+
+ # Event Place
+ if eventList[6]:
+ response = translateAI(eventList[6], "Location Name")
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ eventListTL[6] = response[0]
+
+ if len(eventList[6]) != len(eventListTL[6]):
+ with LOCK:
+ if FILENAME not in MISMATCH:
+ MISMATCH.append(FILENAME)
+
+ # String List
+ if stringList:
+ PBAR.total = len(stringList)
+ PBAR.refresh()
+ response = translateAI(stringList, "Reply with the English Translation")
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ stringListTL = response[0]
+
+ if len(stringList) != len(stringListTL):
+ # Mismatch
+ with LOCK:
+ if FILENAME not in MISMATCH:
+ MISMATCH.append(FILENAME)
+
+ # Pass 2: Set Strings (recursive call)
+ translateJSON(data, filename, [stringListTL, eventListTL])
+
+ return tokens
+
+# Save some money and enter the character before translation
+def getSpeaker(speaker):
+ match speaker:
+ case "ファイン":
+ return ["Fine", [0, 0]]
+ case "":
+ return ["", [0, 0]]
+ case _:
+ # Find Speaker
+ for i in range(len(NAMESLIST)):
+ if speaker == NAMESLIST[i][0]:
+ return [NAMESLIST[i][1], [0, 0]]
+
+ # Translate and Store Speaker
+ response = translateAI(
+ f"{speaker}",
+ "Reply with the " + LANGUAGE + " translation of the NPC name.",
+ False,
+ )
+ response[0] = response[0].title()
+ response[0] = response[0].replace("'S", "'s")
+ response[0] = response[0].replace("Speaker: ", "")
+
+ # Retry if name doesn't translate for some reason
+ if re.search(r"([a-zA-Z？?])", response[0]) == None:
+ response = translateAI(
+ f"{speaker}",
+ "Reply with the " + LANGUAGE + " translation of the NPC name.",
+ False,
+ )
+ response[0] = response[0].title()
+ response[0] = response[0].replace("'S", "'s")
+
+ speakerList = [speaker, response[0]]
+ NAMESLIST.append(speakerList)
+ return response
+ return [speaker, [0, 0]]
+
+def translateAI(text, history, history_ctx=None):
+ """
+ Legacy wrapper function for the new shared translation utility.
+ This maintains compatibility with existing code while using the new shared implementation.
+ """
+ global PBAR, MISMATCH, FILENAME
+
+ # Update config estimate mode based on global ESTIMATE
+ TRANSLATION_CONFIG.estimateMode = bool(ESTIMATE)
+
+ # Call the new shared translation function
+ return sharedtranslateAI(
+ text=text,
+ history=history,
+ config=TRANSLATION_CONFIG,
+ filename=FILENAME,
+ pbar=PBAR,
+ lock=LOCK,
+ mismatchList=MISMATCH
+ )
diff --git a/modules/kirikiri.py b/modules/kirikiri.py
index 0e7d236..0c70174 100644
--- a/modules/kirikiri.py
+++ b/modules/kirikiri.py
@@ -1,531 +1,533 @@
-# Libraries
-import json
-import os
-import re
-import util.dazedwrap as dazedwrap
-import threading
-import time
-import traceback
-import tiktoken
-from pathlib import Path
-from colorama import Fore
-from dotenv import load_dotenv
-from retry import retry
-from tqdm import tqdm
-from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost, getPricingConfig, calculateCost
-import tempfile
-
-# Globals
-MODEL = os.getenv("model")
-TIMEOUT = int(os.getenv("timeout"))
-LANGUAGE = os.getenv("language").capitalize()
-PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
-VOCAB = Path("vocab.txt").read_text(encoding="utf-8")
-LOCK = threading.Lock()
-WIDTH = int(os.getenv("width"))
-LISTWIDTH = int(os.getenv("listWidth"))
-NOTEWIDTH = 70
-MAXHISTORY = 10
-ESTIMATE = ""
-TOKENS = [0, 0]
-NAMESLIST = []
-NAMES = False # Output a list of all the character names found
-BRFLAG = False # If the game uses
instead
-FIXTEXTWRAP = True # Overwrites textwrap
-IGNORETLTEXT = False # Ignores all translated text.
-MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
-PBAR = None
-FILENAME = None
-
-# tqdm Globals
-BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
-POSITION = 0
-LEAVE = False
-
-# Flags
-SPEAKERS = True
-CHOICES = True
-DIALOGUE = True
-
-# Regex - Need to change this if you want to translate from/to other languages. Default is Japanese Regex
-LANGREGEX = r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+"
-
-# Get pricing configuration based on the model
-PRICING_CONFIG = getPricingConfig(MODEL)
-INPUTAPICOST = PRICING_CONFIG["inputAPICost"]
-OUTPUTAPICOST = PRICING_CONFIG["outputAPICost"]
-BATCHSIZE = PRICING_CONFIG["batchSize"]
-FREQUENCY_PENALTY = PRICING_CONFIG["frequencyPenalty"]
-
-# Initialize Translation Config
-TRANSLATION_CONFIG = TranslationConfig(
- model=MODEL,
- language=LANGUAGE,
- prompt=PROMPT,
- vocab=VOCAB,
- langRegex=LANGREGEX,
- batchSize=BATCHSIZE,
- maxHistory=MAXHISTORY,
- estimateMode=False # Will be set dynamically based on ESTIMATE
-)
-LEAVE = False
-
-def handleKirikiri(filename, estimate):
- global ESTIMATE, FILENAME
- ESTIMATE = estimate
- FILENAME = filename
-
- if ESTIMATE:
- start = time.time()
- translatedData = openFiles(filename)
-
- # Print Result
- end = time.time()
- tqdm.write(getResultString(translatedData, end - start, filename))
- with LOCK:
- TOKENS[0] += translatedData[1][0]
- TOKENS[1] += translatedData[1][1]
-
- # Print Total
- totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
-
- # Print any errors on maps
- if len(MISMATCH) > 0:
- return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
- else:
- return totalString
-
- else:
- try:
- start = time.time()
- translatedData = openFiles(filename)
- end = time.time()
-
- # Final write safeguard: if for some reason the progress file was
- # never written (e.g. no translatable lines triggered saves), write it now.
- try:
- if translatedData[0]:
- os.makedirs("translated", exist_ok=True)
- final_path = os.path.join("translated", filename)
- # Write directly (small risk window acceptable on final flush)
- with open(final_path, "w", encoding="cp932", errors="ignore", newline="\n") as f:
- f.writelines(translatedData[0])
- except Exception:
- traceback.print_exc()
-
- tqdm.write(getResultString(translatedData, end - start, filename))
- with LOCK:
- TOKENS[0] += translatedData[1][0]
- TOKENS[1] += translatedData[1][1]
- except Exception:
- traceback.print_exc()
- # Don't blindly remove the file; it may contain partial progress.
- return "Fail"
-
- return getResultString(["", TOKENS, None], end - start, "TOTAL")
-
-
-def getResultString(translatedData, translationTime, filename):
- # File Print String
- cost = calculateCost(translatedData[1][0], translatedData[1][1], MODEL)
- totalTokenstring = (
- Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
- "[Output: "
- + str(translatedData[1][1])
- + "]" "[Cost: ${:,.4f}".format(cost)
- + "]"
- )
- timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
-
- if translatedData[2] == None:
- # Success
- return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET
-
- else:
- # Fail
- try:
- raise translatedData[2]
- except Exception as e:
- traceback.print_exc()
- errorString = str(e) + Fore.RED
- return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET
-
-
-def openFiles(filename):
- with open("files/" + filename, "r", encoding="cp932") as readFile:
- translatedData = parseKiriKiri(readFile, filename)
- return translatedData
-
-
-def parseKiriKiri(readFile, filename):
- global PBAR
- totalTokens = [0, 0]
-
- # Read File into data
- data = readFile.readlines()
-
- # Create Progress Bar
- with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as PBAR:
- PBAR.desc = filename
-
- try:
- result = translateKiriKiri(data, PBAR, filename, [])
- totalTokens[0] += result[0]
- totalTokens[1] += result[1]
- except Exception as e:
- traceback.print_exc()
- return [data, totalTokens, e]
- return [data, totalTokens, None]
-
-
-def save_progress_lines(lines, filename, encoding="cp932"):
- """Atomically (with retries) save current line-based translation progress.
-
- Rationale:
- Windows raises PermissionError if the destination file is open elsewhere.
- We avoid holding an open handle outside this function and add a small
- exponential backoff retry loop to handle transient locks (e.g. AV scanners).
- """
- if ESTIMATE:
- return
-
- max_attempts = 5
- backoff = 0.05 # seconds
- tmp_fd = None
- tmp_path = None
-
- for attempt in range(1, max_attempts + 1):
- try:
- os.makedirs("translated", exist_ok=True)
-
- # Create temp file every attempt (prior one cleaned in finally).
- tmp_fd, tmp_path = tempfile.mkstemp(prefix=f"{filename}.", suffix=".tmp", dir="translated")
- with os.fdopen(tmp_fd, "w", encoding=encoding, newline="\n", errors="ignore") as tmp_file:
- tmp_file.writelines(lines)
- tmp_file.flush()
- os.fsync(tmp_file.fileno())
-
- dest_path = os.path.join("translated", filename)
- try:
- os.replace(tmp_path, dest_path)
- except PermissionError as e:
- # Retry on Windows-specific sharing violation
- if attempt < max_attempts:
- time.sleep(backoff)
- backoff *= 2
- continue
- else:
- raise e
- # Success, break loop
- break
- except Exception:
- if attempt == max_attempts:
- traceback.print_exc()
- finally:
- # Ensure temp file removed if it still exists
- if tmp_path and os.path.exists(tmp_path):
- try:
- os.remove(tmp_path)
- except OSError:
- pass
- tmp_fd = None
- tmp_path = None
-
-
-def translateKiriKiri(data, pbar, filename, jobList):
- # Check Job Data
- if len(jobList) > 0:
- stringList = jobList[0]
- choiceList = jobList[1]
- setData = True
- else:
- stringList = []
- choiceList = []
- setData = False
- tokens = [0, 0]
- speaker = ""
- global LOCK, ESTIMATE
- i = 0
-
- # Regex
- speakerRegex = r"【(.*)】\[CR\]"
- dialogueRegex = r"^\[text\](.*).*\[KeyWait\]|\[\w+\](.*)\[\/\w+\].*\[KeyWait\]"
- furiganaRegex = r'(\[eruby\sstr="(.*?)"\stext.*?\])'
- choicesRegex = r"^\s*\[button\d\sclickse=sys_decide.*text='(.*?)'.*"
- taggedDialogueRegex = r"^\[(?P[^\s\]/]+)(?:\s[^\]]*)?\](?P.*?)\[/\1\]"
-
- while i < len(data):
- speaker = ""
- # Speaker
- match = re.search(speakerRegex, data[i])
- if match and SPEAKERS:
- speakerJA = match.group(1)
- response = getSpeaker(speakerJA)
- speaker = response[0]
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- data[i] = data[i].replace(speakerJA, speaker)
- save_progress_lines(data, filename)
- i += 1
-
- # Choices
- match = re.search(choicesRegex, data[i])
- if match and CHOICES:
- jaString = match.group(1)
-
- # Pass 1
- if not setData:
- choiceList.append(jaString)
-
- # Pass 2
- else:
- # Grab and Pop and Set
- translatedText = choiceList[0]
- choiceList.pop(0)
-
- # Replace Quotes
- data[i] = data[i].replace("'", '"')
- translatedText = translatedText.replace('"', "'")
- data[i] = data[i].replace(jaString, translatedText)
- save_progress_lines(data, filename)
-
- # Tagged dialogue lines e.g., [思考 storage="..."]text[/思考]
- tagged = re.match(taggedDialogueRegex, data[i])
- if tagged and DIALOGUE:
- tag_name = tagged.group('tag')
- jaString = tagged.group('text')
-
- # Pass 1: enqueue with speaker from closing tag
- if not setData:
- # Remove inline wrapping
- jaString_clean = jaString.replace("[r]", " ")
- # Remove furigana
- matchList = re.findall(furiganaRegex, jaString_clean)
- if matchList:
- for fm in matchList:
- jaString_clean = jaString_clean.replace(fm[0], fm[1])
-
- # Resolve speaker via getSpeaker
- resolved = getSpeaker(tag_name)
- tag_speaker = resolved[0]
- tokens[0] += resolved[1][0]
- tokens[1] += resolved[1][1]
-
- if tag_speaker:
- stringList.append(f"[{tag_speaker}]: {jaString_clean.strip()}")
- else:
- stringList.append(jaString_clean.strip())
-
- # Pass 2: apply translated text back between tags
- else:
- if len(stringList) > 0:
- translatedText = stringList[0]
- stringList.pop(0)
- # Remove Speaker label if present
- translatedText = re.sub(r"\[.*?\]:\s", "", translatedText)
- # Wrap and convert newlines to [r]
- translatedText = dazedwrap.wrapText(translatedText, width=WIDTH)
- translatedText = translatedText.replace("\n", "[r]")
- # Replace quotes as per convention
- data[i] = data[i].replace("'", '"')
- translatedText = translatedText.replace('"', "'")
- # Replace only inner content
- data[i] = data[i].replace(jaString, translatedText)
- save_progress_lines(data, filename)
-
- # Simple narrative line handling: translate each whitespace-led, non-tag, non-command line independently.
- # This avoids reflowing or merging blocks, preventing misplaced text.
- if not re.match(r"^\[", data[i]) and not data[i].lstrip().startswith("@"):
- if re.match(r"^[ \t\u3000]+", data[i]):
- # Skip standalone markers like [▼]
- if data[i].strip() == "[▼]":
- pass
- else:
- # Pass 1
- if not setData:
- line_content = data[i].rstrip("\n")
- # Remove inline wrapping markers and glyph markers
- line_content = line_content.replace("[r]", " ")
- line_content = re.sub(r"\[▼\]", "", line_content)
- # Remove furigana blocks
- matchList = re.findall(furiganaRegex, line_content)
- if matchList:
- for fm in matchList:
- line_content = line_content.replace(fm[0], fm[1])
- cleaned = line_content.strip()
- if cleaned:
- stringList.append(cleaned)
- # Pass 2
- else:
- if len(stringList) > 0:
- translatedText = stringList[0]
- stringList.pop(0)
- translatedText = re.sub(r"\[.*?\]:\s", "", translatedText)
- translatedText = dazedwrap.wrapText(translatedText, width=WIDTH)
- translatedText = translatedText.replace("\n", "[r]")
- indent_match = re.match(r"^([ \t\u3000]+)", data[i])
- indent = indent_match.group(1) if indent_match else ""
- data[i] = f"{indent}{translatedText}\n"
- save_progress_lines(data, filename)
-
- # Dialogue
- match = re.search(dialogueRegex, data[i])
- if match and DIALOGUE:
- jaString = match.group(1)
- if not jaString:
- jaString = match.group(2)
-
- # Pass 1
- if not setData:
- # Remove any textwrap
- jaString = jaString.replace("[r]", " ")
-
- # Remove Furigana
- matchList = re.findall(furiganaRegex, jaString)
- if matchList:
- for match in matchList:
- jaString = jaString.replace(match[0], match[1])
-
- # Add String
- if speaker:
- stringList.append(f"[{speaker}]: {jaString.strip()}")
- else:
- stringList.append(f"{jaString.strip()}")
-
- # Pass 2
- else:
- if len(stringList) > 0:
- # Grab and Pop
- translatedText = stringList[0]
- stringList.pop(0)
-
- # Remove Speaker
- translatedText = re.sub(r"\[.*?\]:\s", "", translatedText)
-
- # Textwrap
- translatedText = dazedwrap.wrapText(translatedText, width=WIDTH)
- translatedText = translatedText.replace("\n", "[r]")
-
- # Replace Quotes
- data[i] = data[i].replace("'", '"')
- translatedText = translatedText.replace('"', "'")
- data[i] = data[i].replace(jaString, translatedText)
- save_progress_lines(data, filename)
-
- # Next Line
- i += 1
-
- # EOF
- stringListTL = []
- choiceListTL = []
-
- # Dialogue
- if len(stringList) > 0:
- # Set Progress
- pbar.total = len(stringList)
- pbar.refresh()
-
- # Translate
- response = translateAI(
- stringList,
- "",
- True,
- )
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- stringListTL = response[0]
-
- # Validate
- if len(stringList) != len(stringListTL):
- with LOCK:
- if filename not in MISMATCH:
- MISMATCH.append(filename)
- stringListTL = stringList
-
- # Choices
- if len(choiceList) > 0:
- # Set Progress
- pbar.total = len(choiceList)
- pbar.refresh()
-
- # Translate
- response = translateAI(
- choiceList,
- "",
- True,
- )
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- choiceListTL = response[0]
-
- # Validate
- if len(choiceList) != len(choiceListTL):
- with LOCK:
- if filename not in MISMATCH:
- MISMATCH.append(filename)
- choiceListTL = choiceList
-
- # Proceed to Pass 2
- if not setData:
- translateKiriKiri(data, pbar, filename, [stringListTL, choiceListTL])
-
- return tokens
-
-# Save some money and enter the character before translation
-def getSpeaker(speaker):
- match speaker:
- case "ファイン":
- return ["Fine", [0, 0]]
- case "":
- return ["", [0, 0]]
- case _:
- # Find Speaker
- for i in range(len(NAMESLIST)):
- if speaker == NAMESLIST[i][0]:
- return [NAMESLIST[i][1], [0, 0]]
-
- # Translate and Store Speaker
- response = translateAI(
- f"{speaker}",
- "Reply with the " + LANGUAGE + " translation of the NPC name.",
- False,
- )
- response[0] = response[0].title()
- response[0] = response[0].replace("'S", "'s")
- response[0] = response[0].replace("Speaker: ", "")
-
- # Retry if name doesn't translate for some reason
- if re.search(r"([a-zA-Z？?])", response[0]) == None:
- response = translateAI(
- f"{speaker}",
- "Reply with the " + LANGUAGE + " translation of the NPC name.",
- False,
- )
- response[0] = response[0].title()
- response[0] = response[0].replace("'S", "'s")
-
- speakerList = [speaker, response[0]]
- NAMESLIST.append(speakerList)
- return response
- return [speaker, [0, 0]]
-
-def translateAI(text, history, history_ctx=None):
- """
- Legacy wrapper function for the new shared translation utility.
- This maintains compatibility with existing code while using the new shared implementation.
- """
- global PBAR, MISMATCH, FILENAME
-
- # Update config estimate mode based on global ESTIMATE
- TRANSLATION_CONFIG.estimateMode = bool(ESTIMATE)
-
- # Call the new shared translation function
- return sharedtranslateAI(
- text=text,
- history=history,
- config=TRANSLATION_CONFIG,
- filename=FILENAME,
- pbar=PBAR,
- lock=LOCK,
- mismatchList=MISMATCH
- )
+# Libraries
+import json
+import os
+import re
+import util.dazedwrap as dazedwrap
+import threading
+import time
+import traceback
+import tiktoken
+from pathlib import Path
+from colorama import Fore
+from dotenv import load_dotenv
+from retry import retry
+from tqdm import tqdm
+from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost, getPricingConfig, calculateCost
+import tempfile
+
+# Globals
+MODEL = os.getenv("model")
+TIMEOUT = int(os.getenv("timeout"))
+LANGUAGE = os.getenv("language").capitalize()
+from util.paths import PROMPT_PATH, VOCAB_PATH
+
+PROMPT = PROMPT_PATH.read_text(encoding="utf-8")
+VOCAB = VOCAB_PATH.read_text(encoding="utf-8")
+LOCK = threading.Lock()
+WIDTH = int(os.getenv("width"))
+LISTWIDTH = int(os.getenv("listWidth"))
+NOTEWIDTH = 70
+MAXHISTORY = 10
+ESTIMATE = ""
+TOKENS = [0, 0]
+NAMESLIST = []
+NAMES = False # Output a list of all the character names found
+BRFLAG = False # If the game uses
instead
+FIXTEXTWRAP = True # Overwrites textwrap
+IGNORETLTEXT = False # Ignores all translated text.
+MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
+PBAR = None
+FILENAME = None
+
+# tqdm Globals
+BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
+POSITION = 0
+LEAVE = False
+
+# Flags
+SPEAKERS = True
+CHOICES = True
+DIALOGUE = True
+
+# Regex - Need to change this if you want to translate from/to other languages. Default is Japanese Regex
+LANGREGEX = r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+"
+
+# Get pricing configuration based on the model
+PRICING_CONFIG = getPricingConfig(MODEL)
+INPUTAPICOST = PRICING_CONFIG["inputAPICost"]
+OUTPUTAPICOST = PRICING_CONFIG["outputAPICost"]
+BATCHSIZE = PRICING_CONFIG["batchSize"]
+FREQUENCY_PENALTY = PRICING_CONFIG["frequencyPenalty"]
+
+# Initialize Translation Config
+TRANSLATION_CONFIG = TranslationConfig(
+ model=MODEL,
+ language=LANGUAGE,
+ prompt=PROMPT,
+ vocab=VOCAB,
+ langRegex=LANGREGEX,
+ batchSize=BATCHSIZE,
+ maxHistory=MAXHISTORY,
+ estimateMode=False # Will be set dynamically based on ESTIMATE
+)
+LEAVE = False
+
+def handleKirikiri(filename, estimate):
+ global ESTIMATE, FILENAME
+ ESTIMATE = estimate
+ FILENAME = filename
+
+ if ESTIMATE:
+ start = time.time()
+ translatedData = openFiles(filename)
+
+ # Print Result
+ end = time.time()
+ tqdm.write(getResultString(translatedData, end - start, filename))
+ with LOCK:
+ TOKENS[0] += translatedData[1][0]
+ TOKENS[1] += translatedData[1][1]
+
+ # Print Total
+ totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
+
+ # Print any errors on maps
+ if len(MISMATCH) > 0:
+ return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
+ else:
+ return totalString
+
+ else:
+ try:
+ start = time.time()
+ translatedData = openFiles(filename)
+ end = time.time()
+
+ # Final write safeguard: if for some reason the progress file was
+ # never written (e.g. no translatable lines triggered saves), write it now.
+ try:
+ if translatedData[0]:
+ os.makedirs("translated", exist_ok=True)
+ final_path = os.path.join("translated", filename)
+ # Write directly (small risk window acceptable on final flush)
+ with open(final_path, "w", encoding="cp932", errors="ignore", newline="\n") as f:
+ f.writelines(translatedData[0])
+ except Exception:
+ traceback.print_exc()
+
+ tqdm.write(getResultString(translatedData, end - start, filename))
+ with LOCK:
+ TOKENS[0] += translatedData[1][0]
+ TOKENS[1] += translatedData[1][1]
+ except Exception:
+ traceback.print_exc()
+ # Don't blindly remove the file; it may contain partial progress.
+ return "Fail"
+
+ return getResultString(["", TOKENS, None], end - start, "TOTAL")
+
+
+def getResultString(translatedData, translationTime, filename):
+ # File Print String
+ cost = calculateCost(translatedData[1][0], translatedData[1][1], MODEL)
+ totalTokenstring = (
+ Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
+ "[Output: "
+ + str(translatedData[1][1])
+ + "]" "[Cost: ${:,.4f}".format(cost)
+ + "]"
+ )
+ timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
+
+ if translatedData[2] == None:
+ # Success
+ return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET
+
+ else:
+ # Fail
+ try:
+ raise translatedData[2]
+ except Exception as e:
+ traceback.print_exc()
+ errorString = str(e) + Fore.RED
+ return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET
+
+
+def openFiles(filename):
+ with open("files/" + filename, "r", encoding="cp932") as readFile:
+ translatedData = parseKiriKiri(readFile, filename)
+ return translatedData
+
+
+def parseKiriKiri(readFile, filename):
+ global PBAR
+ totalTokens = [0, 0]
+
+ # Read File into data
+ data = readFile.readlines()
+
+ # Create Progress Bar
+ with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as PBAR:
+ PBAR.desc = filename
+
+ try:
+ result = translateKiriKiri(data, PBAR, filename, [])
+ totalTokens[0] += result[0]
+ totalTokens[1] += result[1]
+ except Exception as e:
+ traceback.print_exc()
+ return [data, totalTokens, e]
+ return [data, totalTokens, None]
+
+
+def save_progress_lines(lines, filename, encoding="cp932"):
+ """Atomically (with retries) save current line-based translation progress.
+
+ Rationale:
+ Windows raises PermissionError if the destination file is open elsewhere.
+ We avoid holding an open handle outside this function and add a small
+ exponential backoff retry loop to handle transient locks (e.g. AV scanners).
+ """
+ if ESTIMATE:
+ return
+
+ max_attempts = 5
+ backoff = 0.05 # seconds
+ tmp_fd = None
+ tmp_path = None
+
+ for attempt in range(1, max_attempts + 1):
+ try:
+ os.makedirs("translated", exist_ok=True)
+
+ # Create temp file every attempt (prior one cleaned in finally).
+ tmp_fd, tmp_path = tempfile.mkstemp(prefix=f"{filename}.", suffix=".tmp", dir="translated")
+ with os.fdopen(tmp_fd, "w", encoding=encoding, newline="\n", errors="ignore") as tmp_file:
+ tmp_file.writelines(lines)
+ tmp_file.flush()
+ os.fsync(tmp_file.fileno())
+
+ dest_path = os.path.join("translated", filename)
+ try:
+ os.replace(tmp_path, dest_path)
+ except PermissionError as e:
+ # Retry on Windows-specific sharing violation
+ if attempt < max_attempts:
+ time.sleep(backoff)
+ backoff *= 2
+ continue
+ else:
+ raise e
+ # Success, break loop
+ break
+ except Exception:
+ if attempt == max_attempts:
+ traceback.print_exc()
+ finally:
+ # Ensure temp file removed if it still exists
+ if tmp_path and os.path.exists(tmp_path):
+ try:
+ os.remove(tmp_path)
+ except OSError:
+ pass
+ tmp_fd = None
+ tmp_path = None
+
+
+def translateKiriKiri(data, pbar, filename, jobList):
+ # Check Job Data
+ if len(jobList) > 0:
+ stringList = jobList[0]
+ choiceList = jobList[1]
+ setData = True
+ else:
+ stringList = []
+ choiceList = []
+ setData = False
+ tokens = [0, 0]
+ speaker = ""
+ global LOCK, ESTIMATE
+ i = 0
+
+ # Regex
+ speakerRegex = r"【(.*)】\[CR\]"
+ dialogueRegex = r"^\[text\](.*).*\[KeyWait\]|\[\w+\](.*)\[\/\w+\].*\[KeyWait\]"
+ furiganaRegex = r'(\[eruby\sstr="(.*?)"\stext.*?\])'
+ choicesRegex = r"^\s*\[button\d\sclickse=sys_decide.*text='(.*?)'.*"
+ taggedDialogueRegex = r"^\[(?P[^\s\]/]+)(?:\s[^\]]*)?\](?P.*?)\[/\1\]"
+
+ while i < len(data):
+ speaker = ""
+ # Speaker
+ match = re.search(speakerRegex, data[i])
+ if match and SPEAKERS:
+ speakerJA = match.group(1)
+ response = getSpeaker(speakerJA)
+ speaker = response[0]
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ data[i] = data[i].replace(speakerJA, speaker)
+ save_progress_lines(data, filename)
+ i += 1
+
+ # Choices
+ match = re.search(choicesRegex, data[i])
+ if match and CHOICES:
+ jaString = match.group(1)
+
+ # Pass 1
+ if not setData:
+ choiceList.append(jaString)
+
+ # Pass 2
+ else:
+ # Grab and Pop and Set
+ translatedText = choiceList[0]
+ choiceList.pop(0)
+
+ # Replace Quotes
+ data[i] = data[i].replace("'", '"')
+ translatedText = translatedText.replace('"', "'")
+ data[i] = data[i].replace(jaString, translatedText)
+ save_progress_lines(data, filename)
+
+ # Tagged dialogue lines e.g., [思考 storage="..."]text[/思考]
+ tagged = re.match(taggedDialogueRegex, data[i])
+ if tagged and DIALOGUE:
+ tag_name = tagged.group('tag')
+ jaString = tagged.group('text')
+
+ # Pass 1: enqueue with speaker from closing tag
+ if not setData:
+ # Remove inline wrapping
+ jaString_clean = jaString.replace("[r]", " ")
+ # Remove furigana
+ matchList = re.findall(furiganaRegex, jaString_clean)
+ if matchList:
+ for fm in matchList:
+ jaString_clean = jaString_clean.replace(fm[0], fm[1])
+
+ # Resolve speaker via getSpeaker
+ resolved = getSpeaker(tag_name)
+ tag_speaker = resolved[0]
+ tokens[0] += resolved[1][0]
+ tokens[1] += resolved[1][1]
+
+ if tag_speaker:
+ stringList.append(f"[{tag_speaker}]: {jaString_clean.strip()}")
+ else:
+ stringList.append(jaString_clean.strip())
+
+ # Pass 2: apply translated text back between tags
+ else:
+ if len(stringList) > 0:
+ translatedText = stringList[0]
+ stringList.pop(0)
+ # Remove Speaker label if present
+ translatedText = re.sub(r"\[.*?\]:\s", "", translatedText)
+ # Wrap and convert newlines to [r]
+ translatedText = dazedwrap.wrapText(translatedText, width=WIDTH)
+ translatedText = translatedText.replace("\n", "[r]")
+ # Replace quotes as per convention
+ data[i] = data[i].replace("'", '"')
+ translatedText = translatedText.replace('"', "'")
+ # Replace only inner content
+ data[i] = data[i].replace(jaString, translatedText)
+ save_progress_lines(data, filename)
+
+ # Simple narrative line handling: translate each whitespace-led, non-tag, non-command line independently.
+ # This avoids reflowing or merging blocks, preventing misplaced text.
+ if not re.match(r"^\[", data[i]) and not data[i].lstrip().startswith("@"):
+ if re.match(r"^[ \t\u3000]+", data[i]):
+ # Skip standalone markers like [▼]
+ if data[i].strip() == "[▼]":
+ pass
+ else:
+ # Pass 1
+ if not setData:
+ line_content = data[i].rstrip("\n")
+ # Remove inline wrapping markers and glyph markers
+ line_content = line_content.replace("[r]", " ")
+ line_content = re.sub(r"\[▼\]", "", line_content)
+ # Remove furigana blocks
+ matchList = re.findall(furiganaRegex, line_content)
+ if matchList:
+ for fm in matchList:
+ line_content = line_content.replace(fm[0], fm[1])
+ cleaned = line_content.strip()
+ if cleaned:
+ stringList.append(cleaned)
+ # Pass 2
+ else:
+ if len(stringList) > 0:
+ translatedText = stringList[0]
+ stringList.pop(0)
+ translatedText = re.sub(r"\[.*?\]:\s", "", translatedText)
+ translatedText = dazedwrap.wrapText(translatedText, width=WIDTH)
+ translatedText = translatedText.replace("\n", "[r]")
+ indent_match = re.match(r"^([ \t\u3000]+)", data[i])
+ indent = indent_match.group(1) if indent_match else ""
+ data[i] = f"{indent}{translatedText}\n"
+ save_progress_lines(data, filename)
+
+ # Dialogue
+ match = re.search(dialogueRegex, data[i])
+ if match and DIALOGUE:
+ jaString = match.group(1)
+ if not jaString:
+ jaString = match.group(2)
+
+ # Pass 1
+ if not setData:
+ # Remove any textwrap
+ jaString = jaString.replace("[r]", " ")
+
+ # Remove Furigana
+ matchList = re.findall(furiganaRegex, jaString)
+ if matchList:
+ for match in matchList:
+ jaString = jaString.replace(match[0], match[1])
+
+ # Add String
+ if speaker:
+ stringList.append(f"[{speaker}]: {jaString.strip()}")
+ else:
+ stringList.append(f"{jaString.strip()}")
+
+ # Pass 2
+ else:
+ if len(stringList) > 0:
+ # Grab and Pop
+ translatedText = stringList[0]
+ stringList.pop(0)
+
+ # Remove Speaker
+ translatedText = re.sub(r"\[.*?\]:\s", "", translatedText)
+
+ # Textwrap
+ translatedText = dazedwrap.wrapText(translatedText, width=WIDTH)
+ translatedText = translatedText.replace("\n", "[r]")
+
+ # Replace Quotes
+ data[i] = data[i].replace("'", '"')
+ translatedText = translatedText.replace('"', "'")
+ data[i] = data[i].replace(jaString, translatedText)
+ save_progress_lines(data, filename)
+
+ # Next Line
+ i += 1
+
+ # EOF
+ stringListTL = []
+ choiceListTL = []
+
+ # Dialogue
+ if len(stringList) > 0:
+ # Set Progress
+ pbar.total = len(stringList)
+ pbar.refresh()
+
+ # Translate
+ response = translateAI(
+ stringList,
+ "",
+ True,
+ )
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ stringListTL = response[0]
+
+ # Validate
+ if len(stringList) != len(stringListTL):
+ with LOCK:
+ if filename not in MISMATCH:
+ MISMATCH.append(filename)
+ stringListTL = stringList
+
+ # Choices
+ if len(choiceList) > 0:
+ # Set Progress
+ pbar.total = len(choiceList)
+ pbar.refresh()
+
+ # Translate
+ response = translateAI(
+ choiceList,
+ "",
+ True,
+ )
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ choiceListTL = response[0]
+
+ # Validate
+ if len(choiceList) != len(choiceListTL):
+ with LOCK:
+ if filename not in MISMATCH:
+ MISMATCH.append(filename)
+ choiceListTL = choiceList
+
+ # Proceed to Pass 2
+ if not setData:
+ translateKiriKiri(data, pbar, filename, [stringListTL, choiceListTL])
+
+ return tokens
+
+# Save some money and enter the character before translation
+def getSpeaker(speaker):
+ match speaker:
+ case "ファイン":
+ return ["Fine", [0, 0]]
+ case "":
+ return ["", [0, 0]]
+ case _:
+ # Find Speaker
+ for i in range(len(NAMESLIST)):
+ if speaker == NAMESLIST[i][0]:
+ return [NAMESLIST[i][1], [0, 0]]
+
+ # Translate and Store Speaker
+ response = translateAI(
+ f"{speaker}",
+ "Reply with the " + LANGUAGE + " translation of the NPC name.",
+ False,
+ )
+ response[0] = response[0].title()
+ response[0] = response[0].replace("'S", "'s")
+ response[0] = response[0].replace("Speaker: ", "")
+
+ # Retry if name doesn't translate for some reason
+ if re.search(r"([a-zA-Z？?])", response[0]) == None:
+ response = translateAI(
+ f"{speaker}",
+ "Reply with the " + LANGUAGE + " translation of the NPC name.",
+ False,
+ )
+ response[0] = response[0].title()
+ response[0] = response[0].replace("'S", "'s")
+
+ speakerList = [speaker, response[0]]
+ NAMESLIST.append(speakerList)
+ return response
+ return [speaker, [0, 0]]
+
+def translateAI(text, history, history_ctx=None):
+ """
+ Legacy wrapper function for the new shared translation utility.
+ This maintains compatibility with existing code while using the new shared implementation.
+ """
+ global PBAR, MISMATCH, FILENAME
+
+ # Update config estimate mode based on global ESTIMATE
+ TRANSLATION_CONFIG.estimateMode = bool(ESTIMATE)
+
+ # Call the new shared translation function
+ return sharedtranslateAI(
+ text=text,
+ history=history,
+ config=TRANSLATION_CONFIG,
+ filename=FILENAME,
+ pbar=PBAR,
+ lock=LOCK,
+ mismatchList=MISMATCH
+ )
diff --git a/modules/lune.py b/modules/lune.py
index 377b173..3e036c3 100644
--- a/modules/lune.py
+++ b/modules/lune.py
@@ -1,372 +1,374 @@
-# Libraries
-import json
-import os
-import re
-import util.dazedwrap as dazedwrap
-import threading
-import time
-import traceback
-import tiktoken
-from pathlib import Path
-from colorama import Fore
-from dotenv import load_dotenv
-from retry import retry
-from tqdm import tqdm
-from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost
-import tempfile
-
-# Globals
-MODEL = os.getenv("model")
-TIMEOUT = int(os.getenv("timeout"))
-LANGUAGE = os.getenv("language").capitalize()
-PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
-VOCAB = Path("vocab.txt").read_text(encoding="utf-8")
-LOCK = threading.Lock()
-WIDTH = int(os.getenv("width"))
-LISTWIDTH = int(os.getenv("listWidth"))
-NOTEWIDTH = 70
-MAXHISTORY = 10
-ESTIMATE = ""
-TOKENS = [0, 0]
-NAMESLIST = []
-NAMES = False # Output a list of all the character names found
-BRFLAG = False # If the game uses
instead
-FIXTEXTWRAP = True # Overwrites textwrap
-IGNORETLTEXT = False # Ignores all translated text.
-MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
-PBAR = None
-FILENAME = None
-
-# tqdm Globals
-BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
-POSITION = 0
-LEAVE = False
-
-# Regex - Need to change this if you want to translate from/to other languages. Default is Japanese Regex
-LANGREGEX = r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+"
-
-# Get pricing configuration based on the model
-PRICING_CONFIG = getPricingConfig(MODEL)
-INPUTAPICOST = PRICING_CONFIG["inputAPICost"]
-OUTPUTAPICOST = PRICING_CONFIG["outputAPICost"]
-BATCHSIZE = PRICING_CONFIG["batchSize"]
-FREQUENCY_PENALTY = PRICING_CONFIG["frequencyPenalty"]
-
-# Initialize Translation Config
-TRANSLATION_CONFIG = TranslationConfig(
- model=MODEL,
- language=LANGUAGE,
- prompt=PROMPT,
- vocab=VOCAB,
- langRegex=LANGREGEX,
- batchSize=BATCHSIZE,
- maxHistory=MAXHISTORY,
- estimateMode=False # Will be set dynamically based on ESTIMATE
-)
-LEAVE = False
-
-def handleLune(filename, estimate):
- global FILENAME, ESTIMATE, totalTokens
- ESTIMATE = estimate
- FILENAME = filename
-
- if estimate:
- start = time.time()
- translatedData = openFiles(filename)
-
- # Print Result
- end = time.time()
- tqdm.write(getResultString(translatedData, end - start, filename))
- with LOCK:
- TOKENS[0] += translatedData[1][0]
- TOKENS[1] += translatedData[1][1]
-
- return getResultString(["", TOKENS, None], end - start, "TOTAL")
-
- else:
- try:
- with open("translated/" + filename, "w", encoding="utf-8", newline="\n") as outFile:
- start = time.time()
- translatedData = openFiles(filename)
-
- # Print Result
- end = time.time()
- json.dump(translatedData[0], outFile, ensure_ascii=False, indent=4)
- tqdm.write(getResultString(translatedData, end - start, filename))
- with LOCK:
- TOKENS[0] += translatedData[1][0]
- TOKENS[1] += translatedData[1][1]
- except Exception:
- return "Fail"
-
- return getResultString(["", TOKENS, None], end - start, "TOTAL")
-
-
-def openFiles(filename):
- with open("files/" + filename, "r", encoding="UTF-8-sig") as f:
- data = json.load(f)
-
- # Map Files
- if ".json" in filename:
- translatedData = parseJSON(data, filename)
-
- else:
- raise NameError(filename + " Not Supported")
-
- return translatedData
-
-
-def getResultString(translatedData, translationTime, filename):
- # File Print String
- cost = calculateCost(translatedData[1][0], translatedData[1][1], MODEL)
- totalTokenstring = (
- Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
- "[Output: "
- + str(translatedData[1][1])
- + "]" "[Cost: ${:,.4f}".format(cost)
- + "]"
- )
- timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
-
- if translatedData[2] == None:
- # Success
- return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET
-
- else:
- # Fail
- try:
- raise translatedData[2]
- except Exception as e:
- traceback.print_exc()
- errorString = str(e) + Fore.RED
- return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET
-
-
-def parseJSON(data, filename):
- totalTokens = [0, 0]
- totalLines = 0
- totalLines = len(data)
- global LOCK
-
- with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar:
- pbar.desc = filename
- pbar.total = totalLines
- try:
- result = translateJSON(data, pbar)
- totalTokens[0] += result[0]
- totalTokens[1] += result[1]
- except Exception as e:
- return [data, totalTokens, e]
- return [data, totalTokens, None]
-
-
-def save_progress_json(data, filename):
- """Atomically save current JSON translation progress."""
- try:
- if ESTIMATE:
- return
- os.makedirs("translated", exist_ok=True)
- tmp_fd, tmp_path = tempfile.mkstemp(prefix=f"{filename}.", suffix=".tmp", dir="translated")
- try:
- with os.fdopen(tmp_fd, "w", encoding="utf-8", newline="\n") as tmp_file:
- json.dump(data, tmp_file, ensure_ascii=False, indent=4)
- os.replace(tmp_path, os.path.join("translated", filename))
- finally:
- if os.path.exists(tmp_path):
- try:
- os.remove(tmp_path)
- except OSError:
- pass
- except Exception:
- traceback.print_exc()
-
-
-def translateJSON(data, pbar):
- global PBAR
- PBAR = pbar
- textHistory = []
- batch = []
- maxHistory = MAXHISTORY
- tokens = [0, 0]
- speaker = "None"
- insertBool = False
- i = 0
- batchStartIndex = 0
-
- while i < len(data):
- item = data[i]
- # Speaker
- if "name" in item:
- if item["name"] not in [None, "-"]:
- response = getSpeaker(item["name"])
- speaker = response[0]
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- item["name"] = speaker
- save_progress_json(data, FILENAME or "output.json")
- else:
- speaker = "None"
-
- # Text
- if "message" in item:
- for text in [
- "text",
- "text2",
- "help1",
- "help2",
- "help3",
- "like",
- "message",
- "me",
- ]:
- if text in item:
- if item[text] != None:
- jaString = item[text]
-
- # Remove any textwrap
- if FIXTEXTWRAP == True:
- finalJAString = jaString.replace("\n", " ")
-
- # [Passthrough 1] Pulling From File
- if insertBool is False:
- # Append to List and Clear Values
- batch.append(finalJAString)
- speaker = ""
-
- # Translate Batch if Full
- if len(batch) == BATCHSIZE:
- # Translate
- response = translateAI(batch, textHistory)
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- translatedBatch = response[0]
- textHistory = translatedBatch[-10:]
-
- # Set Values
- if len(batch) == len(translatedBatch):
- i = batchStartIndex
- insertBool = True
-
- # Mismatch
- else:
- pbar.write(f"Mismatch: {batchStartIndex} - {i}")
- MISMATCH.append(batch)
- batchStartIndex = i
- batch.clear()
-
- if insertBool is False:
- i += 1
-
- currentGroup = []
-
- # [Passthrough 2] Setting Data
- else:
- # Get Text
- translatedText = translatedBatch[0]
-
- # Remove added speaker
- translatedText = re.sub(r"^.+?:\s", "", translatedText)
-
- # Textwrap
- translatedText = dazedwrap.wrapText(translatedText, width=WIDTH)
-
- # Set Text
- item[text] = translatedText
- save_progress_json(data, FILENAME or "output.json")
- translatedBatch.pop(0)
- speaker = ""
- currentGroup = []
- i += 1
-
- # If Batch is empty. Move on.
- if len(translatedBatch) == 0:
- insertBool = False
- batchStartIndex = i
- batch.clear()
- else:
- i += 1
-
- # Translate Batch if not empty and EOF
- if len(batch) != 0 and i >= len(data):
- # Translate
- response = translateAI(batch, textHistory)
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- translatedBatch = response[0]
- textHistory = translatedBatch[-10:]
-
- # Set Values
- if len(batch) == len(translatedBatch):
- i = batchStartIndex
- insertBool = True
-
- # Mismatch
- else:
- pbar.write(f"Mismatch: {batchStartIndex} - {i}")
- MISMATCH.append(batch)
- batchStartIndex = i
- batch.clear()
-
- currentGroup = []
- # After applying a batch, persist progress
- save_progress_json(data, FILENAME or "output.json")
- return tokens
-
-# Save some money and enter the character before translation
-def getSpeaker(speaker):
- match speaker:
- case "ファイン":
- return ["Fine", [0, 0]]
- case "":
- return ["", [0, 0]]
- case _:
- # Find Speaker
- for i in range(len(NAMESLIST)):
- if speaker == NAMESLIST[i][0]:
- return [NAMESLIST[i][1], [0, 0]]
-
- # Translate and Store Speaker
- response = translateAI(
- f"{speaker}",
- "Reply with the " + LANGUAGE + " translation of the NPC name.",
- False,
- )
- response[0] = response[0].title()
- response[0] = response[0].replace("'S", "'s")
- response[0] = response[0].replace("Speaker: ", "")
-
- # Retry if name doesn't translate for some reason
- if re.search(r"([a-zA-Z？?])", response[0]) == None:
- response = translateAI(
- f"{speaker}",
- "Reply with the " + LANGUAGE + " translation of the NPC name.",
- False,
- )
- response[0] = response[0].title()
- response[0] = response[0].replace("'S", "'s")
-
- speakerList = [speaker, response[0]]
- NAMESLIST.append(speakerList)
- return response
- return [speaker, [0, 0]]
-
-def translateAI(text, history, history_ctx=None):
- """
- Legacy wrapper function for the new shared translation utility.
- This maintains compatibility with existing code while using the new shared implementation.
- """
- global PBAR, MISMATCH, FILENAME
-
- # Update config estimate mode based on global ESTIMATE
- TRANSLATION_CONFIG.estimateMode = bool(ESTIMATE)
-
- # Call the new shared translation function
- return sharedtranslateAI(
- text=text,
- history=history,
- config=TRANSLATION_CONFIG,
- filename=FILENAME,
- pbar=PBAR,
- lock=LOCK,
- mismatchList=MISMATCH
- )
+# Libraries
+import json
+import os
+import re
+import util.dazedwrap as dazedwrap
+import threading
+import time
+import traceback
+import tiktoken
+from pathlib import Path
+from colorama import Fore
+from dotenv import load_dotenv
+from retry import retry
+from tqdm import tqdm
+from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost
+import tempfile
+
+# Globals
+MODEL = os.getenv("model")
+TIMEOUT = int(os.getenv("timeout"))
+LANGUAGE = os.getenv("language").capitalize()
+from util.paths import PROMPT_PATH, VOCAB_PATH
+
+PROMPT = PROMPT_PATH.read_text(encoding="utf-8")
+VOCAB = VOCAB_PATH.read_text(encoding="utf-8")
+LOCK = threading.Lock()
+WIDTH = int(os.getenv("width"))
+LISTWIDTH = int(os.getenv("listWidth"))
+NOTEWIDTH = 70
+MAXHISTORY = 10
+ESTIMATE = ""
+TOKENS = [0, 0]
+NAMESLIST = []
+NAMES = False # Output a list of all the character names found
+BRFLAG = False # If the game uses
instead
+FIXTEXTWRAP = True # Overwrites textwrap
+IGNORETLTEXT = False # Ignores all translated text.
+MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
+PBAR = None
+FILENAME = None
+
+# tqdm Globals
+BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
+POSITION = 0
+LEAVE = False
+
+# Regex - Need to change this if you want to translate from/to other languages. Default is Japanese Regex
+LANGREGEX = r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+"
+
+# Get pricing configuration based on the model
+PRICING_CONFIG = getPricingConfig(MODEL)
+INPUTAPICOST = PRICING_CONFIG["inputAPICost"]
+OUTPUTAPICOST = PRICING_CONFIG["outputAPICost"]
+BATCHSIZE = PRICING_CONFIG["batchSize"]
+FREQUENCY_PENALTY = PRICING_CONFIG["frequencyPenalty"]
+
+# Initialize Translation Config
+TRANSLATION_CONFIG = TranslationConfig(
+ model=MODEL,
+ language=LANGUAGE,
+ prompt=PROMPT,
+ vocab=VOCAB,
+ langRegex=LANGREGEX,
+ batchSize=BATCHSIZE,
+ maxHistory=MAXHISTORY,
+ estimateMode=False # Will be set dynamically based on ESTIMATE
+)
+LEAVE = False
+
+def handleLune(filename, estimate):
+ global FILENAME, ESTIMATE, totalTokens
+ ESTIMATE = estimate
+ FILENAME = filename
+
+ if estimate:
+ start = time.time()
+ translatedData = openFiles(filename)
+
+ # Print Result
+ end = time.time()
+ tqdm.write(getResultString(translatedData, end - start, filename))
+ with LOCK:
+ TOKENS[0] += translatedData[1][0]
+ TOKENS[1] += translatedData[1][1]
+
+ return getResultString(["", TOKENS, None], end - start, "TOTAL")
+
+ else:
+ try:
+ with open("translated/" + filename, "w", encoding="utf-8", newline="\n") as outFile:
+ start = time.time()
+ translatedData = openFiles(filename)
+
+ # Print Result
+ end = time.time()
+ json.dump(translatedData[0], outFile, ensure_ascii=False, indent=4)
+ tqdm.write(getResultString(translatedData, end - start, filename))
+ with LOCK:
+ TOKENS[0] += translatedData[1][0]
+ TOKENS[1] += translatedData[1][1]
+ except Exception:
+ return "Fail"
+
+ return getResultString(["", TOKENS, None], end - start, "TOTAL")
+
+
+def openFiles(filename):
+ with open("files/" + filename, "r", encoding="UTF-8-sig") as f:
+ data = json.load(f)
+
+ # Map Files
+ if ".json" in filename:
+ translatedData = parseJSON(data, filename)
+
+ else:
+ raise NameError(filename + " Not Supported")
+
+ return translatedData
+
+
+def getResultString(translatedData, translationTime, filename):
+ # File Print String
+ cost = calculateCost(translatedData[1][0], translatedData[1][1], MODEL)
+ totalTokenstring = (
+ Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
+ "[Output: "
+ + str(translatedData[1][1])
+ + "]" "[Cost: ${:,.4f}".format(cost)
+ + "]"
+ )
+ timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
+
+ if translatedData[2] == None:
+ # Success
+ return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET
+
+ else:
+ # Fail
+ try:
+ raise translatedData[2]
+ except Exception as e:
+ traceback.print_exc()
+ errorString = str(e) + Fore.RED
+ return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET
+
+
+def parseJSON(data, filename):
+ totalTokens = [0, 0]
+ totalLines = 0
+ totalLines = len(data)
+ global LOCK
+
+ with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar:
+ pbar.desc = filename
+ pbar.total = totalLines
+ try:
+ result = translateJSON(data, pbar)
+ totalTokens[0] += result[0]
+ totalTokens[1] += result[1]
+ except Exception as e:
+ return [data, totalTokens, e]
+ return [data, totalTokens, None]
+
+
+def save_progress_json(data, filename):
+ """Atomically save current JSON translation progress."""
+ try:
+ if ESTIMATE:
+ return
+ os.makedirs("translated", exist_ok=True)
+ tmp_fd, tmp_path = tempfile.mkstemp(prefix=f"{filename}.", suffix=".tmp", dir="translated")
+ try:
+ with os.fdopen(tmp_fd, "w", encoding="utf-8", newline="\n") as tmp_file:
+ json.dump(data, tmp_file, ensure_ascii=False, indent=4)
+ os.replace(tmp_path, os.path.join("translated", filename))
+ finally:
+ if os.path.exists(tmp_path):
+ try:
+ os.remove(tmp_path)
+ except OSError:
+ pass
+ except Exception:
+ traceback.print_exc()
+
+
+def translateJSON(data, pbar):
+ global PBAR
+ PBAR = pbar
+ textHistory = []
+ batch = []
+ maxHistory = MAXHISTORY
+ tokens = [0, 0]
+ speaker = "None"
+ insertBool = False
+ i = 0
+ batchStartIndex = 0
+
+ while i < len(data):
+ item = data[i]
+ # Speaker
+ if "name" in item:
+ if item["name"] not in [None, "-"]:
+ response = getSpeaker(item["name"])
+ speaker = response[0]
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ item["name"] = speaker
+ save_progress_json(data, FILENAME or "output.json")
+ else:
+ speaker = "None"
+
+ # Text
+ if "message" in item:
+ for text in [
+ "text",
+ "text2",
+ "help1",
+ "help2",
+ "help3",
+ "like",
+ "message",
+ "me",
+ ]:
+ if text in item:
+ if item[text] != None:
+ jaString = item[text]
+
+ # Remove any textwrap
+ if FIXTEXTWRAP == True:
+ finalJAString = jaString.replace("\n", " ")
+
+ # [Passthrough 1] Pulling From File
+ if insertBool is False:
+ # Append to List and Clear Values
+ batch.append(finalJAString)
+ speaker = ""
+
+ # Translate Batch if Full
+ if len(batch) == BATCHSIZE:
+ # Translate
+ response = translateAI(batch, textHistory)
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ translatedBatch = response[0]
+ textHistory = translatedBatch[-10:]
+
+ # Set Values
+ if len(batch) == len(translatedBatch):
+ i = batchStartIndex
+ insertBool = True
+
+ # Mismatch
+ else:
+ pbar.write(f"Mismatch: {batchStartIndex} - {i}")
+ MISMATCH.append(batch)
+ batchStartIndex = i
+ batch.clear()
+
+ if insertBool is False:
+ i += 1
+
+ currentGroup = []
+
+ # [Passthrough 2] Setting Data
+ else:
+ # Get Text
+ translatedText = translatedBatch[0]
+
+ # Remove added speaker
+ translatedText = re.sub(r"^.+?:\s", "", translatedText)
+
+ # Textwrap
+ translatedText = dazedwrap.wrapText(translatedText, width=WIDTH)
+
+ # Set Text
+ item[text] = translatedText
+ save_progress_json(data, FILENAME or "output.json")
+ translatedBatch.pop(0)
+ speaker = ""
+ currentGroup = []
+ i += 1
+
+ # If Batch is empty. Move on.
+ if len(translatedBatch) == 0:
+ insertBool = False
+ batchStartIndex = i
+ batch.clear()
+ else:
+ i += 1
+
+ # Translate Batch if not empty and EOF
+ if len(batch) != 0 and i >= len(data):
+ # Translate
+ response = translateAI(batch, textHistory)
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ translatedBatch = response[0]
+ textHistory = translatedBatch[-10:]
+
+ # Set Values
+ if len(batch) == len(translatedBatch):
+ i = batchStartIndex
+ insertBool = True
+
+ # Mismatch
+ else:
+ pbar.write(f"Mismatch: {batchStartIndex} - {i}")
+ MISMATCH.append(batch)
+ batchStartIndex = i
+ batch.clear()
+
+ currentGroup = []
+ # After applying a batch, persist progress
+ save_progress_json(data, FILENAME or "output.json")
+ return tokens
+
+# Save some money and enter the character before translation
+def getSpeaker(speaker):
+ match speaker:
+ case "ファイン":
+ return ["Fine", [0, 0]]
+ case "":
+ return ["", [0, 0]]
+ case _:
+ # Find Speaker
+ for i in range(len(NAMESLIST)):
+ if speaker == NAMESLIST[i][0]:
+ return [NAMESLIST[i][1], [0, 0]]
+
+ # Translate and Store Speaker
+ response = translateAI(
+ f"{speaker}",
+ "Reply with the " + LANGUAGE + " translation of the NPC name.",
+ False,
+ )
+ response[0] = response[0].title()
+ response[0] = response[0].replace("'S", "'s")
+ response[0] = response[0].replace("Speaker: ", "")
+
+ # Retry if name doesn't translate for some reason
+ if re.search(r"([a-zA-Z？?])", response[0]) == None:
+ response = translateAI(
+ f"{speaker}",
+ "Reply with the " + LANGUAGE + " translation of the NPC name.",
+ False,
+ )
+ response[0] = response[0].title()
+ response[0] = response[0].replace("'S", "'s")
+
+ speakerList = [speaker, response[0]]
+ NAMESLIST.append(speakerList)
+ return response
+ return [speaker, [0, 0]]
+
+def translateAI(text, history, history_ctx=None):
+ """
+ Legacy wrapper function for the new shared translation utility.
+ This maintains compatibility with existing code while using the new shared implementation.
+ """
+ global PBAR, MISMATCH, FILENAME
+
+ # Update config estimate mode based on global ESTIMATE
+ TRANSLATION_CONFIG.estimateMode = bool(ESTIMATE)
+
+ # Call the new shared translation function
+ return sharedtranslateAI(
+ text=text,
+ history=history,
+ config=TRANSLATION_CONFIG,
+ filename=FILENAME,
+ pbar=PBAR,
+ lock=LOCK,
+ mismatchList=MISMATCH
+ )
diff --git a/modules/main.py b/modules/main.py
index 2fd2752..ca93613 100644
--- a/modules/main.py
+++ b/modules/main.py
@@ -1,381 +1,385 @@
-import sys
-import os
-import traceback
-import datetime
-from pathlib import Path
-from concurrent.futures import ThreadPoolExecutor, as_completed
-from colorama import Fore
-from tqdm import tqdm
-from dotenv import load_dotenv
-
-# This needs to be before the module imports as some of them currently try to read and use some of these values
-# upon import, in which case if they are unset the script will crash before we can output these messages.
-load_dotenv()
-_missing_envs = [
- env for env in [
- "api",
- "key",
- "model",
- "language",
- "timeout",
- "fileThreads",
- "threads",
- "width",
- "listWidth",
- ]
- if os.getenv(env) is None or str(os.getenv(env))[:1] == "<"
-]
-if _missing_envs:
- names = ", ".join(_missing_envs)
- tqdm.write(
- Fore.RED
- + f"Missing required environment variable(s): {names}. "
- + "Set them in a .env file (see .env.example)."
- )
-
-from modules.rpgmakermvmz import handleMVMZ, setSpeakerParseMode as setSpeakerParseMVMZ, finalizeSpeakerParse as finalizeSpeakerParseMVMZ
-from modules.csv import handleCSV
-from modules.tyrano import handleTyrano
-from modules.kirikiri import handleKirikiri
-from modules.json import handleJSON
-from modules.lune import handleLune
-from modules.yuris import handleYuris
-from modules.nscript import handleOnscripter
-from modules.wolf import handleWOLF
-from modules.wolf2 import handleWOLF2
-from modules.regex import handleRegex
-from modules.text import handleText
-from modules.renpy import handleRenpy
-from modules.unity import handleUnity
-from modules.images import handleImages
-from modules.rpgmakerplugin import handlePlugin
-from modules.srpg import handleSRPG
-from modules.aquedi4 import handleAquedi4
-
-# For GPT4 rate limit will be hit if you have more than 1 thread.
-# 1 Thread for each file. Controls how many files are worked on at once.
-THREADS = int(os.getenv("fileThreads"))
-
-# [Display name, file extension, handle function]
-MODULES = [
- ["RPGMaker MV/MZ", ["json"], handleMVMZ],
- ["RPGMaker Plugins", ["js", "rb"], handlePlugin],
- ["CSV (From Translator++)", ["csv"], handleCSV],
- ["Tyrano", ["ks"], handleTyrano],
- ["Kirikiri", ["ks", "tjs", "ssd", "asd"], handleKirikiri],
- ["JSON", ["json"], handleJSON],
- ["Lune", ["json"], handleLune],
- ["Yuris", ["json"], handleYuris],
- ["NScript", ["txt"], handleOnscripter],
- ["Wolf", ["json"], handleWOLF],
- ["Wolf", ["txt"], handleWOLF2],
- ["Regex", ["txt", "json", "script", "csv"], handleRegex],
- ["Text", ["txt", "srt"], handleText],
- ["Renpy", ["rpy"], handleRenpy],
- ["Unity", ["txt"], handleUnity],
- ["SRPG Studio", ["json"], handleSRPG],
- ["Images", [""], handleImages],
- ["Aquedi4 Prepared JSON", [".json"], handleAquedi4],
-]
-
-# Info Message
-tqdm.write(
- Fore.CYAN
- + "-Dazed MTL Tool -"
- + Fore.RESET,
- end="\n\n",
-)
-
-
-def main():
- from util.translation import clear_cache
-
- estimate = ""
- batch_mode = False
- speaker_parse = False # Deferred until after engine select
- while estimate == "":
- estimate = input("Select Mode:\n\n 1. Translate\n 2. Estimate\n 3. Batch Translate (Anthropic Batches API, 50% off)\n")
- match estimate:
- case "1":
- estimate = False
- case "2":
- estimate = True
- case "3":
- estimate = False
- batch_mode = True
- case _:
- estimate = ""
-
- resume_state = None
- if batch_mode:
- from util.translation import isClaudeNative, batchRunState
- if not isClaudeNative(os.getenv("model", "")):
- tqdm.write(
- Fore.RED
- + "Batch Translate requires a Claude model with the 'api' env var unset or pointing at anthropic.com."
- + Fore.RESET
- )
- return
- # An interrupted batch run can be resumed instead of re-collecting
- # (a second submission would be billed again).
- resume_state = batchRunState()
- if resume_state:
- confirm = ""
- while confirm not in ("y", "n"):
- confirm = input(f"A previous batch run was interrupted ({resume_state}). Resume it? (y/n)\n").strip().lower()
- if confirm == "n":
- resume_state = None
-
- # Clear the translation cache at the start of the run. Kept when resuming a
- # batch so names translated during collect stay consistent with the queued
- # payloads in the consume pass.
- if not resume_state:
- clear_cache()
-
- version = ""
- while True:
- tqdm.write("Select game engine:\n")
- for position, module in enumerate(MODULES):
- tqdm.write(f"{str(position + 1).rjust(2)}. {module[0]} (.{module[1]})")
- version = input()
- try:
- version = int(version) - 1
- except:
- continue
- if version in range(len(MODULES)):
- break
-
- totalCost = (
- Fore.RED
- + "Translation module didn't return the total cost. Make sure the \
-files to translate are in the /files folder and that you picked the right game engine."
- )
-
- # If translating RPGMaker MV/MZ, prompt for speaker parse mode
- speaker_parse = False
- if version == 0 and not estimate and not batch_mode:
- sub = ""
- while sub == "":
- sub = input("RPGMaker MV/MZ options:\n\n 1. Standard Translate\n 2. Parse Speakers (collect speaker names only)\n")
- match sub:
- case "1":
- speaker_parse = False
- case "2":
- speaker_parse = True
- case _:
- sub = ""
- if speaker_parse:
- setSpeakerParseMVMZ(True)
-
- # Open File (Threads) - recursively walk 'files' and preserve directory structure
- # Prepare per-run log file so CLI runs also write to a run-specific history file
- try:
- hist_dir = Path("log") / "history"
- hist_dir.mkdir(parents=True, exist_ok=True)
-
- # Clean up old log files, keeping only the 10 most recent
- try:
- log_files = sorted(hist_dir.glob("translationHistory_*.txt"), key=lambda p: p.stat().st_mtime, reverse=True)
- # Keep only the 10 most recent, delete the rest
- for old_log in log_files[10:]:
- try:
- old_log.unlink()
- except Exception:
- pass
- except Exception:
- pass
-
- fname = datetime.datetime.now().strftime("translationHistory_%Y%m%d_%H%M%S.txt")
- run_log_path = hist_dir / fname
- # Don't create the file yet - it will be created when first log is written
- # Store the path in environment variable
- try:
- os.environ['TRANSLATION_RUN_LOG'] = str(run_log_path)
- except Exception:
- pass
-
- # Try to create a hard link from legacy path to this run file for compatibility
- # This will be created when the run_log_path file is first written to
- legacy = Path("log") / "translationHistory.txt"
- try:
- if legacy.exists():
- try:
- legacy.unlink()
- except Exception:
- pass
- except Exception:
- pass
- except Exception:
- pass
-
- def runFiles(estimate):
- runCost = totalCost
- # Use single worker for estimate mode to prevent race conditions
- max_workers = 1 if estimate else THREADS
- with ThreadPoolExecutor(max_workers=max_workers) as executor:
- futures = []
- files_root = "files"
-
- # Special-case: Images engine expects a folder, not a file; schedule per directory containing assets
- if MODULES[version][0] == "Images":
- for root, dirs, filenames in os.walk(files_root):
- # Skip hidden/system directories
- dirs[:] = [d for d in dirs if d not in {".git", "__pycache__"}]
-
- # Skip the root 'files' itself to avoid processing everything twice
- # We'll still allow scheduling for root if it contains assets
-
- # Only schedule directories that contain potential assets
- has_assets = any(fn.lower().endswith((".png", ".txt")) for fn in filenames)
- if not has_assets:
- continue
-
- # Compute relative directory path and ensure translated mirror exists
- rel_dir = os.path.relpath(root, files_root).replace(os.sep, "/")
- if rel_dir == ".":
- # Represent root as empty string so handler creates files under translated/ directly
- rel_dir = ""
- try:
- target_dir = os.path.join("translated", rel_dir.replace("/", os.sep)) if rel_dir else "translated"
- os.makedirs(target_dir, exist_ok=True)
- except Exception:
- pass
-
- futures.append(
- executor.submit(MODULES[version][2], rel_dir, estimate)
- )
- else:
- # Gather all candidate files recursively
- for root, dirs, filenames in os.walk(files_root):
- # Skip hidden/system directories if any
- dirs[:] = [d for d in dirs if d not in {".git", "__pycache__"}]
-
- for fname in filenames:
- if fname == ".gitkeep":
- continue
-
- abs_path = os.path.join(root, fname)
- # Build relative path from 'files' root using POSIX-style separators so handlers can do 'files/' + rel
- rel_path = os.path.relpath(abs_path, files_root)
- rel_path_posix = rel_path.replace(os.sep, "/")
-
- # Check extension match for the selected module version
- for m in MODULES[version][1]:
- if rel_path_posix.endswith(m):
- # Ensure the corresponding directory exists under 'translated'
- rel_dir = os.path.dirname(rel_path_posix)
- if rel_dir:
- try:
- os.makedirs(os.path.join("translated", rel_dir.replace("/", os.sep)), exist_ok=True)
- except Exception:
- # Best-effort; handler may attempt write and fail if permissions are insufficient
- pass
-
- futures.append(
- executor.submit(MODULES[version][2], rel_path_posix, estimate)
- )
- break # Avoid double-adding if multiple ext entries match
-
- for future in as_completed(futures):
- try:
- runCost = future.result()
- except Exception as e:
- tracebackLineNo = str(traceback.extract_tb(sys.exc_info()[2])[-1].lineno)
- tqdm.write(Fore.RED + str(e) + "|" + tracebackLineNo + Fore.RESET)
- return runCost
-
- if batch_mode:
- from util.translation import (
- set_batch_phase,
- clearBatchFiles,
- pendingBatchRequests,
- estimateBatchCost,
- runTranslationBatches,
- )
- poll = int(os.getenv("batchPollInterval", "60") or 60)
- run_consume = True
-
- if resume_state is None:
- clearBatchFiles()
-
- # Pass 1 — queue dialogue for the batch; speaker/variable strings still hit live API.
- tqdm.write(Fore.CYAN + "[BATCH] Pass 1/2: collecting requests..." + Fore.RESET)
- tqdm.write(
- Fore.YELLOW
- + "[BATCH] Note: speaker names and similar short strings translate at live "
- "API rates during collect (dialogue is batched after you confirm)."
- + Fore.RESET
- )
- set_batch_phase("collect")
- try:
- totalCost = runFiles(False)
- finally:
- set_batch_phase(None)
-
- if pendingBatchRequests() == 0:
- tqdm.write("[BATCH] No requests queued — nothing needed the API.")
- run_consume = False
- else:
- estimateBatchCost()
- confirm = ""
- while confirm not in ("y", "n"):
- confirm = input("Submit batch? (y/n)\n").strip().lower()
- if confirm == "n":
- tqdm.write("[BATCH] Not submitted. The queue is kept in log/batch_requests.json.")
- return
- runTranslationBatches(poll)
- elif resume_state == "submitted":
- tqdm.write(Fore.CYAN + "[BATCH] Resuming the submitted batch..." + Fore.RESET)
- runTranslationBatches(poll)
- else: # "fetched" — results already downloaded, just write the files
- tqdm.write(Fore.CYAN + "[BATCH] Resuming from fetched results..." + Fore.RESET)
-
- if run_consume:
- # Pass 2 — write the translated files from the fetched results.
- # Anything the batch missed falls back to the live API.
- tqdm.write(Fore.CYAN + "[BATCH] Pass 2/2: writing translated files..." + Fore.RESET)
- set_batch_phase("consume")
- try:
- totalCost = runFiles(False)
- finally:
- set_batch_phase(None)
- else:
- totalCost = runFiles(estimate)
-
- # Finalize speaker parse mode by writing collected speakers to vocab
- if speaker_parse:
- finalizeSpeakerParseMVMZ()
-
- # Delete Tmp Files
- if os.path.isfile("csv.tmp"):
- os.remove("csv.tmp")
-
- # Sweep any leftover temp files in translated/
- try:
- translated_dir = os.path.join("translated")
- if os.path.isdir(translated_dir):
- for fname in os.listdir(translated_dir):
- if fname.endswith(".tmp"):
- fpath = os.path.join(translated_dir, fname)
- try:
- os.remove(fpath)
- except Exception:
- # Best-effort cleanup; ignore files locked by other processes
- pass
- except Exception:
- pass
-
- # Finish
- if totalCost != "Fail":
- # if estimate is False:
- # This is to encourage people to grab what's in /translated instead
- # deleteFolderFiles("files")
-
- tqdm.write(str(totalCost))
-
-
-def deleteFolderFiles(folderPath):
- for filename in os.listdir(folderPath):
- file_path = os.path.join(folderPath, filename)
- if file_path.endswith((".json", ".ks")):
+import sys
+import os
+import traceback
+import datetime
+from pathlib import Path
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from colorama import Fore
+from tqdm import tqdm
+from dotenv import load_dotenv
+
+# This needs to be before the module imports as some of them currently try to read and use some of these values
+# upon import, in which case if they are unset the script will crash before we can output these messages.
+load_dotenv()
+from util.paths import migrate_root_data_files, ensure_vocab_file
+
+migrate_root_data_files()
+ensure_vocab_file()
+_missing_envs = [
+ env for env in [
+ "api",
+ "key",
+ "model",
+ "language",
+ "timeout",
+ "fileThreads",
+ "threads",
+ "width",
+ "listWidth",
+ ]
+ if os.getenv(env) is None or str(os.getenv(env))[:1] == "<"
+]
+if _missing_envs:
+ names = ", ".join(_missing_envs)
+ tqdm.write(
+ Fore.RED
+ + f"Missing required environment variable(s): {names}. "
+ + "Set them in a .env file (see .env.example)."
+ )
+
+from modules.rpgmakermvmz import handleMVMZ, setSpeakerParseMode as setSpeakerParseMVMZ, finalizeSpeakerParse as finalizeSpeakerParseMVMZ
+from modules.csv import handleCSV
+from modules.tyrano import handleTyrano
+from modules.kirikiri import handleKirikiri
+from modules.json import handleJSON
+from modules.lune import handleLune
+from modules.yuris import handleYuris
+from modules.nscript import handleOnscripter
+from modules.wolf import handleWOLF
+from modules.wolf2 import handleWOLF2
+from modules.regex import handleRegex
+from modules.text import handleText
+from modules.renpy import handleRenpy
+from modules.unity import handleUnity
+from modules.images import handleImages
+from modules.rpgmakerplugin import handlePlugin
+from modules.srpg import handleSRPG
+from modules.aquedi4 import handleAquedi4
+
+# For GPT4 rate limit will be hit if you have more than 1 thread.
+# 1 Thread for each file. Controls how many files are worked on at once.
+THREADS = int(os.getenv("fileThreads"))
+
+# [Display name, file extension, handle function]
+MODULES = [
+ ["RPGMaker MV/MZ", ["json"], handleMVMZ],
+ ["RPGMaker Plugins", ["js", "rb"], handlePlugin],
+ ["CSV (From Translator++)", ["csv"], handleCSV],
+ ["Tyrano", ["ks"], handleTyrano],
+ ["Kirikiri", ["ks", "tjs", "ssd", "asd"], handleKirikiri],
+ ["JSON", ["json"], handleJSON],
+ ["Lune", ["json"], handleLune],
+ ["Yuris", ["json"], handleYuris],
+ ["NScript", ["txt"], handleOnscripter],
+ ["Wolf", ["json"], handleWOLF],
+ ["Wolf", ["txt"], handleWOLF2],
+ ["Regex", ["txt", "json", "script", "csv"], handleRegex],
+ ["Text", ["txt", "srt"], handleText],
+ ["Renpy", ["rpy"], handleRenpy],
+ ["Unity", ["txt"], handleUnity],
+ ["SRPG Studio", ["json"], handleSRPG],
+ ["Images", [""], handleImages],
+ ["Aquedi4 Prepared JSON", [".json"], handleAquedi4],
+]
+
+# Info Message
+tqdm.write(
+ Fore.CYAN
+ + "-Dazed MTL Tool -"
+ + Fore.RESET,
+ end="\n\n",
+)
+
+
+def main():
+ from util.translation import clear_cache
+
+ estimate = ""
+ batch_mode = False
+ speaker_parse = False # Deferred until after engine select
+ while estimate == "":
+ estimate = input("Select Mode:\n\n 1. Translate\n 2. Estimate\n 3. Batch Translate (Anthropic Batches API, 50% off)\n")
+ match estimate:
+ case "1":
+ estimate = False
+ case "2":
+ estimate = True
+ case "3":
+ estimate = False
+ batch_mode = True
+ case _:
+ estimate = ""
+
+ resume_state = None
+ if batch_mode:
+ from util.translation import isClaudeNative, batchRunState
+ if not isClaudeNative(os.getenv("model", "")):
+ tqdm.write(
+ Fore.RED
+ + "Batch Translate requires a Claude model with the 'api' env var unset or pointing at anthropic.com."
+ + Fore.RESET
+ )
+ return
+ # An interrupted batch run can be resumed instead of re-collecting
+ # (a second submission would be billed again).
+ resume_state = batchRunState()
+ if resume_state:
+ confirm = ""
+ while confirm not in ("y", "n"):
+ confirm = input(f"A previous batch run was interrupted ({resume_state}). Resume it? (y/n)\n").strip().lower()
+ if confirm == "n":
+ resume_state = None
+
+ # Clear the translation cache at the start of the run. Kept when resuming a
+ # batch so names translated during collect stay consistent with the queued
+ # payloads in the consume pass.
+ if not resume_state:
+ clear_cache()
+
+ version = ""
+ while True:
+ tqdm.write("Select game engine:\n")
+ for position, module in enumerate(MODULES):
+ tqdm.write(f"{str(position + 1).rjust(2)}. {module[0]} (.{module[1]})")
+ version = input()
+ try:
+ version = int(version) - 1
+ except:
+ continue
+ if version in range(len(MODULES)):
+ break
+
+ totalCost = (
+ Fore.RED
+ + "Translation module didn't return the total cost. Make sure the \
+files to translate are in the /files folder and that you picked the right game engine."
+ )
+
+ # If translating RPGMaker MV/MZ, prompt for speaker parse mode
+ speaker_parse = False
+ if version == 0 and not estimate and not batch_mode:
+ sub = ""
+ while sub == "":
+ sub = input("RPGMaker MV/MZ options:\n\n 1. Standard Translate\n 2. Parse Speakers (collect speaker names only)\n")
+ match sub:
+ case "1":
+ speaker_parse = False
+ case "2":
+ speaker_parse = True
+ case _:
+ sub = ""
+ if speaker_parse:
+ setSpeakerParseMVMZ(True)
+
+ # Open File (Threads) - recursively walk 'files' and preserve directory structure
+ # Prepare per-run log file so CLI runs also write to a run-specific history file
+ try:
+ hist_dir = Path("log") / "history"
+ hist_dir.mkdir(parents=True, exist_ok=True)
+
+ # Clean up old log files, keeping only the 10 most recent
+ try:
+ log_files = sorted(hist_dir.glob("translationHistory_*.txt"), key=lambda p: p.stat().st_mtime, reverse=True)
+ # Keep only the 10 most recent, delete the rest
+ for old_log in log_files[10:]:
+ try:
+ old_log.unlink()
+ except Exception:
+ pass
+ except Exception:
+ pass
+
+ fname = datetime.datetime.now().strftime("translationHistory_%Y%m%d_%H%M%S.txt")
+ run_log_path = hist_dir / fname
+ # Don't create the file yet - it will be created when first log is written
+ # Store the path in environment variable
+ try:
+ os.environ['TRANSLATION_RUN_LOG'] = str(run_log_path)
+ except Exception:
+ pass
+
+ # Try to create a hard link from legacy path to this run file for compatibility
+ # This will be created when the run_log_path file is first written to
+ legacy = Path("log") / "translationHistory.txt"
+ try:
+ if legacy.exists():
+ try:
+ legacy.unlink()
+ except Exception:
+ pass
+ except Exception:
+ pass
+ except Exception:
+ pass
+
+ def runFiles(estimate):
+ runCost = totalCost
+ # Use single worker for estimate mode to prevent race conditions
+ max_workers = 1 if estimate else THREADS
+ with ThreadPoolExecutor(max_workers=max_workers) as executor:
+ futures = []
+ files_root = "files"
+
+ # Special-case: Images engine expects a folder, not a file; schedule per directory containing assets
+ if MODULES[version][0] == "Images":
+ for root, dirs, filenames in os.walk(files_root):
+ # Skip hidden/system directories
+ dirs[:] = [d for d in dirs if d not in {".git", "__pycache__"}]
+
+ # Skip the root 'files' itself to avoid processing everything twice
+ # We'll still allow scheduling for root if it contains assets
+
+ # Only schedule directories that contain potential assets
+ has_assets = any(fn.lower().endswith((".png", ".txt")) for fn in filenames)
+ if not has_assets:
+ continue
+
+ # Compute relative directory path and ensure translated mirror exists
+ rel_dir = os.path.relpath(root, files_root).replace(os.sep, "/")
+ if rel_dir == ".":
+ # Represent root as empty string so handler creates files under translated/ directly
+ rel_dir = ""
+ try:
+ target_dir = os.path.join("translated", rel_dir.replace("/", os.sep)) if rel_dir else "translated"
+ os.makedirs(target_dir, exist_ok=True)
+ except Exception:
+ pass
+
+ futures.append(
+ executor.submit(MODULES[version][2], rel_dir, estimate)
+ )
+ else:
+ # Gather all candidate files recursively
+ for root, dirs, filenames in os.walk(files_root):
+ # Skip hidden/system directories if any
+ dirs[:] = [d for d in dirs if d not in {".git", "__pycache__"}]
+
+ for fname in filenames:
+ if fname == ".gitkeep":
+ continue
+
+ abs_path = os.path.join(root, fname)
+ # Build relative path from 'files' root using POSIX-style separators so handlers can do 'files/' + rel
+ rel_path = os.path.relpath(abs_path, files_root)
+ rel_path_posix = rel_path.replace(os.sep, "/")
+
+ # Check extension match for the selected module version
+ for m in MODULES[version][1]:
+ if rel_path_posix.endswith(m):
+ # Ensure the corresponding directory exists under 'translated'
+ rel_dir = os.path.dirname(rel_path_posix)
+ if rel_dir:
+ try:
+ os.makedirs(os.path.join("translated", rel_dir.replace("/", os.sep)), exist_ok=True)
+ except Exception:
+ # Best-effort; handler may attempt write and fail if permissions are insufficient
+ pass
+
+ futures.append(
+ executor.submit(MODULES[version][2], rel_path_posix, estimate)
+ )
+ break # Avoid double-adding if multiple ext entries match
+
+ for future in as_completed(futures):
+ try:
+ runCost = future.result()
+ except Exception as e:
+ tracebackLineNo = str(traceback.extract_tb(sys.exc_info()[2])[-1].lineno)
+ tqdm.write(Fore.RED + str(e) + "|" + tracebackLineNo + Fore.RESET)
+ return runCost
+
+ if batch_mode:
+ from util.translation import (
+ set_batch_phase,
+ clearBatchFiles,
+ pendingBatchRequests,
+ estimateBatchCost,
+ runTranslationBatches,
+ )
+ poll = int(os.getenv("batchPollInterval", "60") or 60)
+ run_consume = True
+
+ if resume_state is None:
+ clearBatchFiles()
+
+ # Pass 1 — queue dialogue for the batch; speaker/variable strings still hit live API.
+ tqdm.write(Fore.CYAN + "[BATCH] Pass 1/2: collecting requests..." + Fore.RESET)
+ tqdm.write(
+ Fore.YELLOW
+ + "[BATCH] Note: speaker names and similar short strings translate at live "
+ "API rates during collect (dialogue is batched after you confirm)."
+ + Fore.RESET
+ )
+ set_batch_phase("collect")
+ try:
+ totalCost = runFiles(False)
+ finally:
+ set_batch_phase(None)
+
+ if pendingBatchRequests() == 0:
+ tqdm.write("[BATCH] No requests queued — nothing needed the API.")
+ run_consume = False
+ else:
+ estimateBatchCost()
+ confirm = ""
+ while confirm not in ("y", "n"):
+ confirm = input("Submit batch? (y/n)\n").strip().lower()
+ if confirm == "n":
+ tqdm.write("[BATCH] Not submitted. The queue is kept in log/batch_requests.json.")
+ return
+ runTranslationBatches(poll)
+ elif resume_state == "submitted":
+ tqdm.write(Fore.CYAN + "[BATCH] Resuming the submitted batch..." + Fore.RESET)
+ runTranslationBatches(poll)
+ else: # "fetched" — results already downloaded, just write the files
+ tqdm.write(Fore.CYAN + "[BATCH] Resuming from fetched results..." + Fore.RESET)
+
+ if run_consume:
+ # Pass 2 — write the translated files from the fetched results.
+ # Anything the batch missed falls back to the live API.
+ tqdm.write(Fore.CYAN + "[BATCH] Pass 2/2: writing translated files..." + Fore.RESET)
+ set_batch_phase("consume")
+ try:
+ totalCost = runFiles(False)
+ finally:
+ set_batch_phase(None)
+ else:
+ totalCost = runFiles(estimate)
+
+ # Finalize speaker parse mode by writing collected speakers to vocab
+ if speaker_parse:
+ finalizeSpeakerParseMVMZ()
+
+ # Delete Tmp Files
+ if os.path.isfile("csv.tmp"):
+ os.remove("csv.tmp")
+
+ # Sweep any leftover temp files in translated/
+ try:
+ translated_dir = os.path.join("translated")
+ if os.path.isdir(translated_dir):
+ for fname in os.listdir(translated_dir):
+ if fname.endswith(".tmp"):
+ fpath = os.path.join(translated_dir, fname)
+ try:
+ os.remove(fpath)
+ except Exception:
+ # Best-effort cleanup; ignore files locked by other processes
+ pass
+ except Exception:
+ pass
+
+ # Finish
+ if totalCost != "Fail":
+ # if estimate is False:
+ # This is to encourage people to grab what's in /translated instead
+ # deleteFolderFiles("files")
+
+ tqdm.write(str(totalCost))
+
+
+def deleteFolderFiles(folderPath):
+ for filename in os.listdir(folderPath):
+ file_path = os.path.join(folderPath, filename)
+ if file_path.endswith((".json", ".ks")):
os.remove(file_path)
\ No newline at end of file
diff --git a/modules/nscript.py b/modules/nscript.py
index f6eecaf..60d5816 100644
--- a/modules/nscript.py
+++ b/modules/nscript.py
@@ -1,426 +1,428 @@
-# Libraries
-import json
-import os
-import re
-import util.dazedwrap as dazedwrap
-import threading
-import time
-import traceback
-import tiktoken
-from pathlib import Path
-from colorama import Fore
-from dotenv import load_dotenv
-from retry import retry
-from tqdm import tqdm
-from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost
-import tempfile
-
-# Globals
-MODEL = os.getenv("model")
-TIMEOUT = int(os.getenv("timeout"))
-LANGUAGE = os.getenv("language").capitalize()
-PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
-VOCAB = Path("vocab.txt").read_text(encoding="utf-8")
-LOCK = threading.Lock()
-WIDTH = int(os.getenv("width"))
-LISTWIDTH = int(os.getenv("listWidth"))
-NOTEWIDTH = 70
-MAXHISTORY = 10
-ESTIMATE = ""
-TOKENS = [0, 0]
-NAMESLIST = []
-NAMES = False # Output a list of all the character names found
-BRFLAG = False # If the game uses
instead
-FIXTEXTWRAP = True # Overwrites textwrap
-IGNORETLTEXT = False # Ignores all translated text.
-MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
-CONVERTTOWIDE = False # Default (False)
-
-# tqdm Globals
-BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
-POSITION = 0
-LEAVE = False
-PBAR = None
-FILENAME = None
-
-# Full Width
-ascii_to_wide = dict((i, chr(i + 0xFEE0)) for i in range(0x21, 0x7F))
-ascii_to_wide.update({0x20: "\u3000", 0x2D: "\u2212"}) # space and minus
-wide_to_ascii = dict((i, chr(i - 0xFEE0)) for i in range(0xFF01, 0xFF5F))
-wide_to_ascii.update({0x3000: " ", 0x2212: "-"}) # space and minus
-
-# Regex - Need to change this if you want to translate from/to other languages. Default is Japanese Regex
-LANGREGEX = r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+"
-
-# Get pricing configuration based on the model
-PRICING_CONFIG = getPricingConfig(MODEL)
-INPUTAPICOST = PRICING_CONFIG["inputAPICost"]
-OUTPUTAPICOST = PRICING_CONFIG["outputAPICost"]
-BATCHSIZE = PRICING_CONFIG["batchSize"]
-FREQUENCY_PENALTY = PRICING_CONFIG["frequencyPenalty"]
-
-# Initialize Translation Config
-TRANSLATION_CONFIG = TranslationConfig(
- model=MODEL,
- language=LANGUAGE,
- prompt=PROMPT,
- vocab=VOCAB,
- langRegex=LANGREGEX,
- batchSize=BATCHSIZE,
- maxHistory=MAXHISTORY,
- estimateMode=False # Will be set dynamically based on ESTIMATE
-)
-LEAVE = False
-
-def handleOnscripter(filename, estimate):
- global ESTIMATE, FILENAME
- ESTIMATE = estimate
- FILENAME = filename
-
- if ESTIMATE:
- start = time.time()
- translatedData = openFiles(filename)
-
- # Print Result
- end = time.time()
- tqdm.write(getResultString(translatedData, end - start, filename))
- with LOCK:
- TOKENS[0] += translatedData[1][0]
- TOKENS[1] += translatedData[1][1]
-
- # Print Total
- totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
-
- # Print any errors on maps
- if len(MISMATCH) > 0:
- return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
- else:
- return totalString
-
- else:
- try:
- with open("translated/" + filename, "w", encoding="cp932", errors="ignore") as outFile:
- start = time.time()
- translatedData = openFiles(filename)
-
- # Print Result
- end = time.time()
- outFile.writelines(translatedData[0])
- tqdm.write(getResultString(translatedData, end - start, filename))
- with LOCK:
- TOKENS[0] += translatedData[1][0]
- TOKENS[1] += translatedData[1][1]
- except Exception:
- traceback.print_exc()
- return "Fail"
-
- return getResultString(["", TOKENS, None], end - start, "TOTAL")
-
-
-def getResultString(translatedData, translationTime, filename):
- # File Print String
- cost = calculateCost(translatedData[1][0], translatedData[1][1], MODEL)
- totalTokenstring = (
- Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
- "[Output: "
- + str(translatedData[1][1])
- + "]" "[Cost: ${:,.4f}".format(cost)
- + "]"
- )
- timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
-
- if translatedData[2] == None:
- # Success
- return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET
-
- else:
- # Fail
- try:
- raise translatedData[2]
- except Exception as e:
- traceback.print_exc()
- errorString = str(e) + Fore.RED
- return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET
-
-
-def openFiles(filename):
- with open("files/" + filename, "r", encoding="cp932") as readFile:
- translatedData = parseOnscripter(readFile, filename)
-
- # Delete lines marked for deletion
- finalData = []
- for line in translatedData[0]:
- if line != "\\d\n":
- finalData.append(line)
- translatedData[0] = finalData
-
- return translatedData
-
-
-def parseOnscripter(readFile, filename):
- totalTokens = [0, 0]
-
- # Read File into data
- data = readFile.readlines()
-
- # Create Progress Bar
- with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
- pbar.desc = filename
-
- try:
- result = translateOnscripter(data, pbar, filename, [])
- totalTokens[0] += result[0]
- totalTokens[1] += result[1]
- except Exception as e:
- traceback.print_exc()
- return [data, totalTokens, e]
- return [data, totalTokens, None]
-
-
-def save_progress_lines(lines, filename, encoding="cp932"):
- """Atomically save current line-based translation progress."""
- try:
- if ESTIMATE:
- return
- os.makedirs("translated", exist_ok=True)
- tmp_fd, tmp_path = tempfile.mkstemp(prefix=f"{filename}.", suffix=".tmp", dir="translated")
- try:
- with os.fdopen(tmp_fd, "w", encoding=encoding, newline="\n", errors="ignore") as tmp_file:
- tmp_file.writelines(lines)
- os.replace(tmp_path, os.path.join("translated", filename))
- finally:
- if os.path.exists(tmp_path):
- try:
- os.remove(tmp_path)
- except OSError:
- pass
- except Exception:
- traceback.print_exc()
-
-
-def translateOnscripter(data, pbar, filename, translatedList):
- stringList = []
- currentGroup = []
- tokens = [0, 0]
- speaker = ""
- voice = False
- global LOCK, ESTIMATE, PBAR
- PBAR = pbar
- i = 0
-
- # Dialogue
- while i < len(data):
- # Lines
- regex = r"^([\u3000「（【[][^\n]+)"
- match = re.search(regex, data[i])
- if match != None and match.group(1) != "":
- originalString = match.group(1)
- # Pass 1
- if translatedList == []:
- # Grab Consecutive Strings
- jaString = match.group(1)
- while len(data) > i + 1 and re.match(regex, data[i + 1]):
- data[i] = ""
- i += 1
- jaString = f"{jaString} {data[i]}"
-
- # Convert from Wide
- jaString = jaString.translate(wide_to_ascii)
-
- # Remove any textwrap and \u3000 and \
- jaString = jaString.replace("\n", "")
- jaString = jaString.replace("\u3000", "")
- jaString = jaString.replace("\\", "")
- jaString = jaString.replace(" ＞", ")")
- jaString = jaString.replace("＜ ", "(")
-
- # Remove Furigana
- furiMatch = re.findall(r"({(.+?)\/(.+?)})", jaString)
- if furiMatch:
- for match in furiMatch:
- jaString = jaString.replace(match[0], match[2])
-
- # Add String
- stringList.append(jaString.strip())
-
- # Pass 2
- else:
- # Get Text
- if translatedList:
- # Grab and Pop
- translatedText = translatedList[0]
- translatedList.pop(0)
-
- # Set to None if empty list
- if len(translatedList) <= 0:
- translatedList = None
-
- # Textwrap & Other Text
- translatedText = dazedwrap.wrapText(translatedText, width=WIDTH)
- translatedText = translatedText.replace("\n", "\n\u3000")
-
- # Split the string into lines
- lines = translatedText.split("\n")
-
- # Add a backslash after every 3rd line
- j = 0
- while j < len(lines):
- if j == 4:
- lines[j - 1] = f"{lines[j-1]}\\"
- lines[j] = f"\n{lines[j]}"
- j += 1
-
- # Join the lines back into a single string
- translatedText = "\n".join(lines)
-
- # Remove Double Spaces
- translatedText = translatedText.replace(" ", " ")
-
- # Convert to Wide
- if CONVERTTOWIDE:
- translatedText = translatedText.translate(ascii_to_wide)
-
- # Fix Formatting
- translatedText = fixText(translatedText)
-
- # Set Data
- data[i] = data[i].replace(originalString, f"{translatedText}")
- save_progress_lines(data, filename)
- i += 1
-
- # Choices
- elif "csel" in data[i] and translatedList != []:
- choiceList = []
- jaString = data[i]
-
- choiceList = re.findall(r"\"(.*?)\"", jaString)
- if len(choiceList) > 0:
- # Translate
- response = translateAI(choiceList, "This will be a dialogue option")
- translatedTextList = response[0]
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
-
- # Set Data
- for j in range(len(translatedTextList)):
- # Convert to Wide
- if CONVERTTOWIDE:
- translatedText = translatedTextList[j].translate(ascii_to_wide)
-
- # Set
- data[i] = data[i].replace(choiceList[j], translatedText)
- save_progress_lines(data, filename)
- i += 1
-
- # Nothing relevant. Skip Line.
- else:
- i += 1
-
- # EOF
- if len(stringList) > 0:
- # Set Progress
- pbar.total = len(stringList)
- pbar.refresh()
-
- # Translate
- response = translateAI(stringList, "")
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- translatedList = response[0]
-
- # Set Strings
- if len(stringList) == len(translatedList):
- translateOnscripter(data, pbar, filename, translatedList)
-
- # Mismatch
- else:
- with LOCK:
- if filename not in MISMATCH:
- MISMATCH.append(filename)
- return tokens
-
-
-def fixText(translatedText):
- # Add Break
- translatedText = translatedText.replace('"', "'")
- translatedText = f"\u3000{translatedText}\\"
-
- # Unconvert Codes
- matchList = re.findall(r"([＄].+?)[^\w]", translatedText)
- if matchList:
- for match in matchList:
- translatedText = translatedText.replace(match, match.translate(wide_to_ascii))
-
- # Unconvert Color Codes
- matchList = re.findall(r"([＃][\w\d]{6})", translatedText)
- if matchList:
- for match in matchList:
- translatedText = translatedText.replace(match, match.translate(wide_to_ascii))
-
- # Unconvert Variables
- matchList = re.findall(r"([％]\w.+?)[^\w＿]", translatedText)
- if matchList:
- for match in matchList:
- translatedText = translatedText.replace(match, match.translate(wide_to_ascii))
-
- # Unconvert Backslashes
- matchList = re.findall(r"＼", translatedText)
- if matchList:
- for match in matchList:
- translatedText = translatedText.replace(match, match.translate(wide_to_ascii))
-
- return translatedText
-
-# Save some money and enter the character before translation
-def getSpeaker(speaker):
- match speaker:
- case "ファイン":
- return ["Fine", [0, 0]]
- case "":
- return ["", [0, 0]]
- case _:
- # Find Speaker
- for i in range(len(NAMESLIST)):
- if speaker == NAMESLIST[i][0]:
- return [NAMESLIST[i][1], [0, 0]]
-
- # Translate and Store Speaker
- response = translateAI(
- f"{speaker}",
- "Reply with the " + LANGUAGE + " translation of the NPC name.",
- False,
- )
- response[0] = response[0].title()
- response[0] = response[0].replace("'S", "'s")
- response[0] = response[0].replace("Speaker: ", "")
-
- # Retry if name doesn't translate for some reason
- if re.search(r"([a-zA-Z？?])", response[0]) == None:
- response = translateAI(
- f"{speaker}",
- "Reply with the " + LANGUAGE + " translation of the NPC name.",
- False,
- )
- response[0] = response[0].title()
- response[0] = response[0].replace("'S", "'s")
-
- speakerList = [speaker, response[0]]
- NAMESLIST.append(speakerList)
- return response
- return [speaker, [0, 0]]
-
-
-def translateAI(text, history, history_ctx=None):
- """
- Translate text using the shared translation utility.
- This function maintains compatibility with existing code while using the new shared implementation.
- """
- return sharedtranslateAI(
- text=text,
- history=history,
- config=TRANSLATION_CONFIG,
- filename=FILENAME,
- pbar=PBAR,
- lock=LOCK,
- mismatchList=MISMATCH
- )
+# Libraries
+import json
+import os
+import re
+import util.dazedwrap as dazedwrap
+import threading
+import time
+import traceback
+import tiktoken
+from pathlib import Path
+from colorama import Fore
+from dotenv import load_dotenv
+from retry import retry
+from tqdm import tqdm
+from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost
+import tempfile
+
+# Globals
+MODEL = os.getenv("model")
+TIMEOUT = int(os.getenv("timeout"))
+LANGUAGE = os.getenv("language").capitalize()
+from util.paths import PROMPT_PATH, VOCAB_PATH
+
+PROMPT = PROMPT_PATH.read_text(encoding="utf-8")
+VOCAB = VOCAB_PATH.read_text(encoding="utf-8")
+LOCK = threading.Lock()
+WIDTH = int(os.getenv("width"))
+LISTWIDTH = int(os.getenv("listWidth"))
+NOTEWIDTH = 70
+MAXHISTORY = 10
+ESTIMATE = ""
+TOKENS = [0, 0]
+NAMESLIST = []
+NAMES = False # Output a list of all the character names found
+BRFLAG = False # If the game uses
instead
+FIXTEXTWRAP = True # Overwrites textwrap
+IGNORETLTEXT = False # Ignores all translated text.
+MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
+CONVERTTOWIDE = False # Default (False)
+
+# tqdm Globals
+BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
+POSITION = 0
+LEAVE = False
+PBAR = None
+FILENAME = None
+
+# Full Width
+ascii_to_wide = dict((i, chr(i + 0xFEE0)) for i in range(0x21, 0x7F))
+ascii_to_wide.update({0x20: "\u3000", 0x2D: "\u2212"}) # space and minus
+wide_to_ascii = dict((i, chr(i - 0xFEE0)) for i in range(0xFF01, 0xFF5F))
+wide_to_ascii.update({0x3000: " ", 0x2212: "-"}) # space and minus
+
+# Regex - Need to change this if you want to translate from/to other languages. Default is Japanese Regex
+LANGREGEX = r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+"
+
+# Get pricing configuration based on the model
+PRICING_CONFIG = getPricingConfig(MODEL)
+INPUTAPICOST = PRICING_CONFIG["inputAPICost"]
+OUTPUTAPICOST = PRICING_CONFIG["outputAPICost"]
+BATCHSIZE = PRICING_CONFIG["batchSize"]
+FREQUENCY_PENALTY = PRICING_CONFIG["frequencyPenalty"]
+
+# Initialize Translation Config
+TRANSLATION_CONFIG = TranslationConfig(
+ model=MODEL,
+ language=LANGUAGE,
+ prompt=PROMPT,
+ vocab=VOCAB,
+ langRegex=LANGREGEX,
+ batchSize=BATCHSIZE,
+ maxHistory=MAXHISTORY,
+ estimateMode=False # Will be set dynamically based on ESTIMATE
+)
+LEAVE = False
+
+def handleOnscripter(filename, estimate):
+ global ESTIMATE, FILENAME
+ ESTIMATE = estimate
+ FILENAME = filename
+
+ if ESTIMATE:
+ start = time.time()
+ translatedData = openFiles(filename)
+
+ # Print Result
+ end = time.time()
+ tqdm.write(getResultString(translatedData, end - start, filename))
+ with LOCK:
+ TOKENS[0] += translatedData[1][0]
+ TOKENS[1] += translatedData[1][1]
+
+ # Print Total
+ totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
+
+ # Print any errors on maps
+ if len(MISMATCH) > 0:
+ return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
+ else:
+ return totalString
+
+ else:
+ try:
+ with open("translated/" + filename, "w", encoding="cp932", errors="ignore") as outFile:
+ start = time.time()
+ translatedData = openFiles(filename)
+
+ # Print Result
+ end = time.time()
+ outFile.writelines(translatedData[0])
+ tqdm.write(getResultString(translatedData, end - start, filename))
+ with LOCK:
+ TOKENS[0] += translatedData[1][0]
+ TOKENS[1] += translatedData[1][1]
+ except Exception:
+ traceback.print_exc()
+ return "Fail"
+
+ return getResultString(["", TOKENS, None], end - start, "TOTAL")
+
+
+def getResultString(translatedData, translationTime, filename):
+ # File Print String
+ cost = calculateCost(translatedData[1][0], translatedData[1][1], MODEL)
+ totalTokenstring = (
+ Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
+ "[Output: "
+ + str(translatedData[1][1])
+ + "]" "[Cost: ${:,.4f}".format(cost)
+ + "]"
+ )
+ timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
+
+ if translatedData[2] == None:
+ # Success
+ return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET
+
+ else:
+ # Fail
+ try:
+ raise translatedData[2]
+ except Exception as e:
+ traceback.print_exc()
+ errorString = str(e) + Fore.RED
+ return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET
+
+
+def openFiles(filename):
+ with open("files/" + filename, "r", encoding="cp932") as readFile:
+ translatedData = parseOnscripter(readFile, filename)
+
+ # Delete lines marked for deletion
+ finalData = []
+ for line in translatedData[0]:
+ if line != "\\d\n":
+ finalData.append(line)
+ translatedData[0] = finalData
+
+ return translatedData
+
+
+def parseOnscripter(readFile, filename):
+ totalTokens = [0, 0]
+
+ # Read File into data
+ data = readFile.readlines()
+
+ # Create Progress Bar
+ with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
+ pbar.desc = filename
+
+ try:
+ result = translateOnscripter(data, pbar, filename, [])
+ totalTokens[0] += result[0]
+ totalTokens[1] += result[1]
+ except Exception as e:
+ traceback.print_exc()
+ return [data, totalTokens, e]
+ return [data, totalTokens, None]
+
+
+def save_progress_lines(lines, filename, encoding="cp932"):
+ """Atomically save current line-based translation progress."""
+ try:
+ if ESTIMATE:
+ return
+ os.makedirs("translated", exist_ok=True)
+ tmp_fd, tmp_path = tempfile.mkstemp(prefix=f"{filename}.", suffix=".tmp", dir="translated")
+ try:
+ with os.fdopen(tmp_fd, "w", encoding=encoding, newline="\n", errors="ignore") as tmp_file:
+ tmp_file.writelines(lines)
+ os.replace(tmp_path, os.path.join("translated", filename))
+ finally:
+ if os.path.exists(tmp_path):
+ try:
+ os.remove(tmp_path)
+ except OSError:
+ pass
+ except Exception:
+ traceback.print_exc()
+
+
+def translateOnscripter(data, pbar, filename, translatedList):
+ stringList = []
+ currentGroup = []
+ tokens = [0, 0]
+ speaker = ""
+ voice = False
+ global LOCK, ESTIMATE, PBAR
+ PBAR = pbar
+ i = 0
+
+ # Dialogue
+ while i < len(data):
+ # Lines
+ regex = r"^([\u3000「（【[][^\n]+)"
+ match = re.search(regex, data[i])
+ if match != None and match.group(1) != "":
+ originalString = match.group(1)
+ # Pass 1
+ if translatedList == []:
+ # Grab Consecutive Strings
+ jaString = match.group(1)
+ while len(data) > i + 1 and re.match(regex, data[i + 1]):
+ data[i] = ""
+ i += 1
+ jaString = f"{jaString} {data[i]}"
+
+ # Convert from Wide
+ jaString = jaString.translate(wide_to_ascii)
+
+ # Remove any textwrap and \u3000 and \
+ jaString = jaString.replace("\n", "")
+ jaString = jaString.replace("\u3000", "")
+ jaString = jaString.replace("\\", "")
+ jaString = jaString.replace(" ＞", ")")
+ jaString = jaString.replace("＜ ", "(")
+
+ # Remove Furigana
+ furiMatch = re.findall(r"({(.+?)\/(.+?)})", jaString)
+ if furiMatch:
+ for match in furiMatch:
+ jaString = jaString.replace(match[0], match[2])
+
+ # Add String
+ stringList.append(jaString.strip())
+
+ # Pass 2
+ else:
+ # Get Text
+ if translatedList:
+ # Grab and Pop
+ translatedText = translatedList[0]
+ translatedList.pop(0)
+
+ # Set to None if empty list
+ if len(translatedList) <= 0:
+ translatedList = None
+
+ # Textwrap & Other Text
+ translatedText = dazedwrap.wrapText(translatedText, width=WIDTH)
+ translatedText = translatedText.replace("\n", "\n\u3000")
+
+ # Split the string into lines
+ lines = translatedText.split("\n")
+
+ # Add a backslash after every 3rd line
+ j = 0
+ while j < len(lines):
+ if j == 4:
+ lines[j - 1] = f"{lines[j-1]}\\"
+ lines[j] = f"\n{lines[j]}"
+ j += 1
+
+ # Join the lines back into a single string
+ translatedText = "\n".join(lines)
+
+ # Remove Double Spaces
+ translatedText = translatedText.replace(" ", " ")
+
+ # Convert to Wide
+ if CONVERTTOWIDE:
+ translatedText = translatedText.translate(ascii_to_wide)
+
+ # Fix Formatting
+ translatedText = fixText(translatedText)
+
+ # Set Data
+ data[i] = data[i].replace(originalString, f"{translatedText}")
+ save_progress_lines(data, filename)
+ i += 1
+
+ # Choices
+ elif "csel" in data[i] and translatedList != []:
+ choiceList = []
+ jaString = data[i]
+
+ choiceList = re.findall(r"\"(.*?)\"", jaString)
+ if len(choiceList) > 0:
+ # Translate
+ response = translateAI(choiceList, "This will be a dialogue option")
+ translatedTextList = response[0]
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+
+ # Set Data
+ for j in range(len(translatedTextList)):
+ # Convert to Wide
+ if CONVERTTOWIDE:
+ translatedText = translatedTextList[j].translate(ascii_to_wide)
+
+ # Set
+ data[i] = data[i].replace(choiceList[j], translatedText)
+ save_progress_lines(data, filename)
+ i += 1
+
+ # Nothing relevant. Skip Line.
+ else:
+ i += 1
+
+ # EOF
+ if len(stringList) > 0:
+ # Set Progress
+ pbar.total = len(stringList)
+ pbar.refresh()
+
+ # Translate
+ response = translateAI(stringList, "")
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ translatedList = response[0]
+
+ # Set Strings
+ if len(stringList) == len(translatedList):
+ translateOnscripter(data, pbar, filename, translatedList)
+
+ # Mismatch
+ else:
+ with LOCK:
+ if filename not in MISMATCH:
+ MISMATCH.append(filename)
+ return tokens
+
+
+def fixText(translatedText):
+ # Add Break
+ translatedText = translatedText.replace('"', "'")
+ translatedText = f"\u3000{translatedText}\\"
+
+ # Unconvert Codes
+ matchList = re.findall(r"([＄].+?)[^\w]", translatedText)
+ if matchList:
+ for match in matchList:
+ translatedText = translatedText.replace(match, match.translate(wide_to_ascii))
+
+ # Unconvert Color Codes
+ matchList = re.findall(r"([＃][\w\d]{6})", translatedText)
+ if matchList:
+ for match in matchList:
+ translatedText = translatedText.replace(match, match.translate(wide_to_ascii))
+
+ # Unconvert Variables
+ matchList = re.findall(r"([％]\w.+?)[^\w＿]", translatedText)
+ if matchList:
+ for match in matchList:
+ translatedText = translatedText.replace(match, match.translate(wide_to_ascii))
+
+ # Unconvert Backslashes
+ matchList = re.findall(r"＼", translatedText)
+ if matchList:
+ for match in matchList:
+ translatedText = translatedText.replace(match, match.translate(wide_to_ascii))
+
+ return translatedText
+
+# Save some money and enter the character before translation
+def getSpeaker(speaker):
+ match speaker:
+ case "ファイン":
+ return ["Fine", [0, 0]]
+ case "":
+ return ["", [0, 0]]
+ case _:
+ # Find Speaker
+ for i in range(len(NAMESLIST)):
+ if speaker == NAMESLIST[i][0]:
+ return [NAMESLIST[i][1], [0, 0]]
+
+ # Translate and Store Speaker
+ response = translateAI(
+ f"{speaker}",
+ "Reply with the " + LANGUAGE + " translation of the NPC name.",
+ False,
+ )
+ response[0] = response[0].title()
+ response[0] = response[0].replace("'S", "'s")
+ response[0] = response[0].replace("Speaker: ", "")
+
+ # Retry if name doesn't translate for some reason
+ if re.search(r"([a-zA-Z？?])", response[0]) == None:
+ response = translateAI(
+ f"{speaker}",
+ "Reply with the " + LANGUAGE + " translation of the NPC name.",
+ False,
+ )
+ response[0] = response[0].title()
+ response[0] = response[0].replace("'S", "'s")
+
+ speakerList = [speaker, response[0]]
+ NAMESLIST.append(speakerList)
+ return response
+ return [speaker, [0, 0]]
+
+
+def translateAI(text, history, history_ctx=None):
+ """
+ Translate text using the shared translation utility.
+ This function maintains compatibility with existing code while using the new shared implementation.
+ """
+ return sharedtranslateAI(
+ text=text,
+ history=history,
+ config=TRANSLATION_CONFIG,
+ filename=FILENAME,
+ pbar=PBAR,
+ lock=LOCK,
+ mismatchList=MISMATCH
+ )
diff --git a/modules/regex.py b/modules/regex.py
index b4e69ae..af881e5 100644
--- a/modules/regex.py
+++ b/modules/regex.py
@@ -1,528 +1,530 @@
-# Libraries
-import json
-import os
-import re
-import util.dazedwrap as dazedwrap
-import threading
-import time
-import traceback
-import tiktoken
-from pathlib import Path
-from colorama import Fore
-from dotenv import load_dotenv
-from retry import retry
-from tqdm import tqdm
-from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost
-import tempfile
-
-# Globals
-MODEL = os.getenv("model")
-TIMEOUT = int(os.getenv("timeout"))
-LANGUAGE = os.getenv("language").capitalize()
-PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
-VOCAB = Path("vocab.txt").read_text(encoding="utf-8")
-LOCK = threading.Lock()
-WIDTH = int(os.getenv("width"))
-LISTWIDTH = int(os.getenv("listWidth"))
-NOTEWIDTH = 70
-MAXHISTORY = 10
-ESTIMATE = ""
-TOKENS = [0, 0]
-NAMESLIST = []
-NAMES = False # Output a list of all the character names found
-BRFLAG = False # If the game uses
instead
-FIXTEXTWRAP = True # Overwrites textwrap
-IGNORETLTEXT = False # Ignores all translated text.
-MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
-FILENAME = None
-TIMETOTAL = 0 # Total Time Taken for all translations
-TIMETOTAL = 0 # Total Time Taken for all translations
-
-# tqdm Globals
-BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
-POSITION = 0
-LEAVE = False
-PBAR = None
-
-# Regex - Need to change this if you want to translate from/to other languages. Default is Japanese Regex
-LANGREGEX = r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+"
-
-# Get pricing configuration based on the model
-PRICING_CONFIG = getPricingConfig(MODEL)
-INPUTAPICOST = PRICING_CONFIG["inputAPICost"]
-OUTPUTAPICOST = PRICING_CONFIG["outputAPICost"]
-BATCHSIZE = PRICING_CONFIG["batchSize"]
-FREQUENCY_PENALTY = PRICING_CONFIG["frequencyPenalty"]
-
-# Initialize Translation Config
-TRANSLATION_CONFIG = TranslationConfig(
- model=MODEL,
- language=LANGUAGE,
- prompt=PROMPT,
- vocab=VOCAB,
- langRegex=LANGREGEX,
- batchSize=BATCHSIZE,
- maxHistory=MAXHISTORY,
- estimateMode=False # Will be set dynamically based on ESTIMATE
-)
-LEAVE = False
-
-def handleRegex(filename, estimate):
- global ESTIMATE, TOKENS, FILENAME
- ESTIMATE = estimate
- FILENAME = filename
-
- # Translate
- start = time.time()
- translatedData = openFiles(filename)
-
- # Translate
- if not estimate:
- try:
- with open("translated/" + filename, "w", encoding="utf-8-sig") as outFile:
- outFile.writelines(translatedData[0])
- except Exception:
- traceback.print_exc()
- return "Fail"
-
- # Print File
- end = time.time()
- tqdm.write(getResultString(translatedData, end - start, filename))
- with LOCK:
- TOKENS[0] += translatedData[1][0]
- TOKENS[1] += translatedData[1][1]
-
- # Print Total
- totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
-
- # Print any errors on maps
- if len(MISMATCH) > 0:
- return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
- else:
- return totalString
-
-
-def getResultString(translatedData, translationTime, filename):
- global TIMETOTAL
- # File Print String
- cost = calculateCost(translatedData[1][0], translatedData[1][1], MODEL)
- totalTokenstring = (
- Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
- "[Output: "
- + str(translatedData[1][1])
- + "]" "[Cost: ${:,.4f}".format(cost)
- + "]"
- )
- if filename != "TOTAL":
- timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
- TIMETOTAL += round(translationTime, 1)
- else:
- timeString = Fore.BLUE + "[" + str(round(TIMETOTAL, 1)) + "s]"
-
- if translatedData[2] == None:
- # Success
- return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET
-
- else:
- # Fail
- try:
- raise translatedData[2]
- except Exception as e:
- traceback.print_exc()
- errorString = str(e) + Fore.RED
- return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET
-
-
-def openFiles(filename):
- with open("files/" + filename, "r", encoding="utf-8-sig") as readFile:
- translatedData = parseRegex(readFile, filename)
-
- # Delete lines marked for deletion
- finalData = []
- for line in translatedData[0]:
- if line != "\\d\n":
- finalData.append(line)
- translatedData[0] = finalData
-
- return translatedData
-
-
-def parseRegex(readFile, filename):
- global PBAR
- totalTokens = [0, 0]
-
- # Read File into data
- data = readFile.readlines()
-
- # Create Progress Bar
- with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
- pbar.desc = filename
- PBAR = pbar
-
- try:
- result = translateRegex(data, filename, [])
- totalTokens[0] += result[0]
- totalTokens[1] += result[1]
- except Exception as e:
- traceback.print_exc()
- return [data, totalTokens, e]
- return [data, totalTokens, None]
-
-
-def save_progress_lines(lines, filename, encoding="utf-8-sig"):
- """Atomically save current line-based translation progress."""
- try:
- if ESTIMATE:
- return
- os.makedirs("translated", exist_ok=True)
- tmp_fd, tmp_path = tempfile.mkstemp(prefix=f"{filename}.", suffix=".tmp", dir="translated")
- try:
- with os.fdopen(tmp_fd, "w", encoding=encoding, newline="\n", errors="ignore") as tmp_file:
- tmp_file.writelines(lines)
- os.replace(tmp_path, os.path.join("translated", filename))
- finally:
- if os.path.exists(tmp_path):
- try:
- os.remove(tmp_path)
- except OSError:
- pass
- except Exception:
- traceback.print_exc()
-
-
-def translateRegex(data, filename, translatedList):
- if translatedList:
- stringList = translatedList[0]
- choiceList = translatedList[1]
- else:
- stringList = []
- choiceList = []
- tokens = [0, 0]
- speaker = ""
- global LOCK, ESTIMATE, FILENAME, PBAR, MISMATCH
- i = 0
-
- # Define regex patterns outside loop
- dialogueRegex = r'\"text\":\s?\"(.+)\",'
- lineRegexSpeaker = r'"name":\s?"(.+)",'
- choiceRegex = r"\$menu_item.+?,(.*?),"
- titleRegex = r"title\s'(.*)'$"
- setgamedatatitleRegex = r'\\setgamedatatitle\("(.+?)"\)'
- selRegex = r'\\sel\((.+)\)'
-
- while i < len(data):
-
- # Setgamedatatitle
- match = re.search(setgamedatatitleRegex, data[i])
- if match:
- # Pass 1 - Translate immediately (not batched)
- if not translatedList:
- response = translateAI(
- match.group(1).replace('\\n', '\n'),
- f"Reply with the {LANGUAGE} translation of the chapter title",
- True,
- )
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- title = response[0]
-
- # Convert newlines back to escape sequences and escape quotes
- title = title.replace('\n', '\\n').replace('"', '\\"')
- data[i] = data[i].replace(match.group(1), title)
- save_progress_lines(data, filename)
-
- # Title
- match = re.search(titleRegex, data[i])
- if match:
- # Pass 1 - Translate immediately (not batched)
- if not translatedList:
- response = translateAI(
- match.group(1),
- f"Reply with the {LANGUAGE} translation of the chapter title",
- True,
- )
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- title = response[0]
-
- # Set
- title = re.sub(r"(?", ")")
-
- # Textwrap
- translatedText = dazedwrap.wrapText(translatedText, width=WIDTH)
-
- # Set Data
- while "\\endtext" not in data[i].strip():
- del data[i]
- data.insert(i, f"{translatedText}\n")
- save_progress_lines(data, filename)
-
- # Choices
- match = re.search(choiceRegex, data[i])
- if match:
- # Pass 1
- if not translatedList:
- choiceList.append(match.group(1))
-
- # Pass 2
- else:
- # Grab and Pop
- translatedText = choiceList[0]
- choiceList.pop(0)
-
- # Replace Spaces
- translatedText = translatedText.replace("\u3000", " ")
-
- # Escape Quotes
- translatedText = re.sub(r'(? 0:
- PBAR.total = totalItems
- PBAR.refresh()
-
- # String List
- if stringList:
- response = translateAI(stringList, "Reply with the English Translation")
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- stringListTL = response[0]
-
- if len(stringList) != len(stringListTL):
- # Mismatch
- with LOCK:
- if FILENAME not in MISMATCH:
- MISMATCH.append(FILENAME)
-
- # Choice List
- if choiceList:
- response = translateAI(choiceList, "Reply with the English TL of the Dialogue Choice")
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- choiceListTL = response[0]
-
- if len(choiceList) != len(choiceListTL):
- # Mismatch
- with LOCK:
- if FILENAME not in MISMATCH:
- MISMATCH.append(FILENAME)
-
- # Set Strings
- translateRegex(data, filename, [stringListTL, choiceListTL])
- return tokens
-
-# Save some money and enter the character before translation
-def getSpeaker(speaker):
- match speaker:
- case "ファイン":
- return ["Fine", [0, 0]]
- case "":
- return ["", [0, 0]]
- case _:
- # Find Speaker
- for i in range(len(NAMESLIST)):
- if speaker == NAMESLIST[i][0]:
- return [NAMESLIST[i][1], [0, 0]]
-
- # Translate and Store Speaker
- response = translateAI(
- f"{speaker}",
- "Reply with the " + LANGUAGE + " translation of the NPC name.",
- False,
- )
- response[0] = response[0].title()
- response[0] = response[0].replace("'S", "'s")
- response[0] = response[0].replace("Speaker: ", "")
-
- # Retry if name doesn't translate for some reason
- if re.search(r"([a-zA-Z？?])", response[0]) == None:
- response = translateAI(
- f"{speaker}",
- "Reply with the " + LANGUAGE + " translation of the NPC name.",
- False,
- )
- response[0] = response[0].title()
- response[0] = response[0].replace("'S", "'s")
-
- speakerList = [speaker, response[0]]
- NAMESLIST.append(speakerList)
- return response
- return [speaker, [0, 0]]
-
-def translateAI(text, history, history_ctx=None):
- """
- Legacy wrapper function for the new shared translation utility.
- This maintains compatibility with existing code while using the new shared implementation.
- """
- global PBAR, MISMATCH, FILENAME
-
- # Update config estimate mode based on global ESTIMATE
- TRANSLATION_CONFIG.estimateMode = bool(ESTIMATE)
-
- # Call the new shared translation function
- return sharedtranslateAI(
- text=text,
- history=history,
- config=TRANSLATION_CONFIG,
- filename=FILENAME,
- pbar=PBAR,
- lock=LOCK,
- mismatchList=MISMATCH
+# Libraries
+import json
+import os
+import re
+import util.dazedwrap as dazedwrap
+import threading
+import time
+import traceback
+import tiktoken
+from pathlib import Path
+from colorama import Fore
+from dotenv import load_dotenv
+from retry import retry
+from tqdm import tqdm
+from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost
+import tempfile
+
+# Globals
+MODEL = os.getenv("model")
+TIMEOUT = int(os.getenv("timeout"))
+LANGUAGE = os.getenv("language").capitalize()
+from util.paths import PROMPT_PATH, VOCAB_PATH
+
+PROMPT = PROMPT_PATH.read_text(encoding="utf-8")
+VOCAB = VOCAB_PATH.read_text(encoding="utf-8")
+LOCK = threading.Lock()
+WIDTH = int(os.getenv("width"))
+LISTWIDTH = int(os.getenv("listWidth"))
+NOTEWIDTH = 70
+MAXHISTORY = 10
+ESTIMATE = ""
+TOKENS = [0, 0]
+NAMESLIST = []
+NAMES = False # Output a list of all the character names found
+BRFLAG = False # If the game uses
instead
+FIXTEXTWRAP = True # Overwrites textwrap
+IGNORETLTEXT = False # Ignores all translated text.
+MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
+FILENAME = None
+TIMETOTAL = 0 # Total Time Taken for all translations
+TIMETOTAL = 0 # Total Time Taken for all translations
+
+# tqdm Globals
+BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
+POSITION = 0
+LEAVE = False
+PBAR = None
+
+# Regex - Need to change this if you want to translate from/to other languages. Default is Japanese Regex
+LANGREGEX = r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+"
+
+# Get pricing configuration based on the model
+PRICING_CONFIG = getPricingConfig(MODEL)
+INPUTAPICOST = PRICING_CONFIG["inputAPICost"]
+OUTPUTAPICOST = PRICING_CONFIG["outputAPICost"]
+BATCHSIZE = PRICING_CONFIG["batchSize"]
+FREQUENCY_PENALTY = PRICING_CONFIG["frequencyPenalty"]
+
+# Initialize Translation Config
+TRANSLATION_CONFIG = TranslationConfig(
+ model=MODEL,
+ language=LANGUAGE,
+ prompt=PROMPT,
+ vocab=VOCAB,
+ langRegex=LANGREGEX,
+ batchSize=BATCHSIZE,
+ maxHistory=MAXHISTORY,
+ estimateMode=False # Will be set dynamically based on ESTIMATE
+)
+LEAVE = False
+
+def handleRegex(filename, estimate):
+ global ESTIMATE, TOKENS, FILENAME
+ ESTIMATE = estimate
+ FILENAME = filename
+
+ # Translate
+ start = time.time()
+ translatedData = openFiles(filename)
+
+ # Translate
+ if not estimate:
+ try:
+ with open("translated/" + filename, "w", encoding="utf-8-sig") as outFile:
+ outFile.writelines(translatedData[0])
+ except Exception:
+ traceback.print_exc()
+ return "Fail"
+
+ # Print File
+ end = time.time()
+ tqdm.write(getResultString(translatedData, end - start, filename))
+ with LOCK:
+ TOKENS[0] += translatedData[1][0]
+ TOKENS[1] += translatedData[1][1]
+
+ # Print Total
+ totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
+
+ # Print any errors on maps
+ if len(MISMATCH) > 0:
+ return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
+ else:
+ return totalString
+
+
+def getResultString(translatedData, translationTime, filename):
+ global TIMETOTAL
+ # File Print String
+ cost = calculateCost(translatedData[1][0], translatedData[1][1], MODEL)
+ totalTokenstring = (
+ Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
+ "[Output: "
+ + str(translatedData[1][1])
+ + "]" "[Cost: ${:,.4f}".format(cost)
+ + "]"
+ )
+ if filename != "TOTAL":
+ timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
+ TIMETOTAL += round(translationTime, 1)
+ else:
+ timeString = Fore.BLUE + "[" + str(round(TIMETOTAL, 1)) + "s]"
+
+ if translatedData[2] == None:
+ # Success
+ return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET
+
+ else:
+ # Fail
+ try:
+ raise translatedData[2]
+ except Exception as e:
+ traceback.print_exc()
+ errorString = str(e) + Fore.RED
+ return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET
+
+
+def openFiles(filename):
+ with open("files/" + filename, "r", encoding="utf-8-sig") as readFile:
+ translatedData = parseRegex(readFile, filename)
+
+ # Delete lines marked for deletion
+ finalData = []
+ for line in translatedData[0]:
+ if line != "\\d\n":
+ finalData.append(line)
+ translatedData[0] = finalData
+
+ return translatedData
+
+
+def parseRegex(readFile, filename):
+ global PBAR
+ totalTokens = [0, 0]
+
+ # Read File into data
+ data = readFile.readlines()
+
+ # Create Progress Bar
+ with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
+ pbar.desc = filename
+ PBAR = pbar
+
+ try:
+ result = translateRegex(data, filename, [])
+ totalTokens[0] += result[0]
+ totalTokens[1] += result[1]
+ except Exception as e:
+ traceback.print_exc()
+ return [data, totalTokens, e]
+ return [data, totalTokens, None]
+
+
+def save_progress_lines(lines, filename, encoding="utf-8-sig"):
+ """Atomically save current line-based translation progress."""
+ try:
+ if ESTIMATE:
+ return
+ os.makedirs("translated", exist_ok=True)
+ tmp_fd, tmp_path = tempfile.mkstemp(prefix=f"{filename}.", suffix=".tmp", dir="translated")
+ try:
+ with os.fdopen(tmp_fd, "w", encoding=encoding, newline="\n", errors="ignore") as tmp_file:
+ tmp_file.writelines(lines)
+ os.replace(tmp_path, os.path.join("translated", filename))
+ finally:
+ if os.path.exists(tmp_path):
+ try:
+ os.remove(tmp_path)
+ except OSError:
+ pass
+ except Exception:
+ traceback.print_exc()
+
+
+def translateRegex(data, filename, translatedList):
+ if translatedList:
+ stringList = translatedList[0]
+ choiceList = translatedList[1]
+ else:
+ stringList = []
+ choiceList = []
+ tokens = [0, 0]
+ speaker = ""
+ global LOCK, ESTIMATE, FILENAME, PBAR, MISMATCH
+ i = 0
+
+ # Define regex patterns outside loop
+ dialogueRegex = r'\"text\":\s?\"(.+)\",'
+ lineRegexSpeaker = r'"name":\s?"(.+)",'
+ choiceRegex = r"\$menu_item.+?,(.*?),"
+ titleRegex = r"title\s'(.*)'$"
+ setgamedatatitleRegex = r'\\setgamedatatitle\("(.+?)"\)'
+ selRegex = r'\\sel\((.+)\)'
+
+ while i < len(data):
+
+ # Setgamedatatitle
+ match = re.search(setgamedatatitleRegex, data[i])
+ if match:
+ # Pass 1 - Translate immediately (not batched)
+ if not translatedList:
+ response = translateAI(
+ match.group(1).replace('\\n', '\n'),
+ f"Reply with the {LANGUAGE} translation of the chapter title",
+ True,
+ )
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ title = response[0]
+
+ # Convert newlines back to escape sequences and escape quotes
+ title = title.replace('\n', '\\n').replace('"', '\\"')
+ data[i] = data[i].replace(match.group(1), title)
+ save_progress_lines(data, filename)
+
+ # Title
+ match = re.search(titleRegex, data[i])
+ if match:
+ # Pass 1 - Translate immediately (not batched)
+ if not translatedList:
+ response = translateAI(
+ match.group(1),
+ f"Reply with the {LANGUAGE} translation of the chapter title",
+ True,
+ )
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ title = response[0]
+
+ # Set
+ title = re.sub(r"(?", ")")
+
+ # Textwrap
+ translatedText = dazedwrap.wrapText(translatedText, width=WIDTH)
+
+ # Set Data
+ while "\\endtext" not in data[i].strip():
+ del data[i]
+ data.insert(i, f"{translatedText}\n")
+ save_progress_lines(data, filename)
+
+ # Choices
+ match = re.search(choiceRegex, data[i])
+ if match:
+ # Pass 1
+ if not translatedList:
+ choiceList.append(match.group(1))
+
+ # Pass 2
+ else:
+ # Grab and Pop
+ translatedText = choiceList[0]
+ choiceList.pop(0)
+
+ # Replace Spaces
+ translatedText = translatedText.replace("\u3000", " ")
+
+ # Escape Quotes
+ translatedText = re.sub(r'(? 0:
+ PBAR.total = totalItems
+ PBAR.refresh()
+
+ # String List
+ if stringList:
+ response = translateAI(stringList, "Reply with the English Translation")
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ stringListTL = response[0]
+
+ if len(stringList) != len(stringListTL):
+ # Mismatch
+ with LOCK:
+ if FILENAME not in MISMATCH:
+ MISMATCH.append(FILENAME)
+
+ # Choice List
+ if choiceList:
+ response = translateAI(choiceList, "Reply with the English TL of the Dialogue Choice")
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ choiceListTL = response[0]
+
+ if len(choiceList) != len(choiceListTL):
+ # Mismatch
+ with LOCK:
+ if FILENAME not in MISMATCH:
+ MISMATCH.append(FILENAME)
+
+ # Set Strings
+ translateRegex(data, filename, [stringListTL, choiceListTL])
+ return tokens
+
+# Save some money and enter the character before translation
+def getSpeaker(speaker):
+ match speaker:
+ case "ファイン":
+ return ["Fine", [0, 0]]
+ case "":
+ return ["", [0, 0]]
+ case _:
+ # Find Speaker
+ for i in range(len(NAMESLIST)):
+ if speaker == NAMESLIST[i][0]:
+ return [NAMESLIST[i][1], [0, 0]]
+
+ # Translate and Store Speaker
+ response = translateAI(
+ f"{speaker}",
+ "Reply with the " + LANGUAGE + " translation of the NPC name.",
+ False,
+ )
+ response[0] = response[0].title()
+ response[0] = response[0].replace("'S", "'s")
+ response[0] = response[0].replace("Speaker: ", "")
+
+ # Retry if name doesn't translate for some reason
+ if re.search(r"([a-zA-Z？?])", response[0]) == None:
+ response = translateAI(
+ f"{speaker}",
+ "Reply with the " + LANGUAGE + " translation of the NPC name.",
+ False,
+ )
+ response[0] = response[0].title()
+ response[0] = response[0].replace("'S", "'s")
+
+ speakerList = [speaker, response[0]]
+ NAMESLIST.append(speakerList)
+ return response
+ return [speaker, [0, 0]]
+
+def translateAI(text, history, history_ctx=None):
+ """
+ Legacy wrapper function for the new shared translation utility.
+ This maintains compatibility with existing code while using the new shared implementation.
+ """
+ global PBAR, MISMATCH, FILENAME
+
+ # Update config estimate mode based on global ESTIMATE
+ TRANSLATION_CONFIG.estimateMode = bool(ESTIMATE)
+
+ # Call the new shared translation function
+ return sharedtranslateAI(
+ text=text,
+ history=history,
+ config=TRANSLATION_CONFIG,
+ filename=FILENAME,
+ pbar=PBAR,
+ lock=LOCK,
+ mismatchList=MISMATCH
)
\ No newline at end of file
diff --git a/modules/renpy.py b/modules/renpy.py
index 289efb0..414b099 100644
--- a/modules/renpy.py
+++ b/modules/renpy.py
@@ -1,351 +1,353 @@
-# Libraries
-import json
-import os
-import re
-import util.dazedwrap as dazedwrap
-import threading
-import time
-import traceback
-import tiktoken
-from pathlib import Path
-from colorama import Fore
-from dotenv import load_dotenv
-from retry import retry
-from tqdm import tqdm
-from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost, getPricingConfig, calculateCost
-import tempfile
-
-# Globals
-MODEL = os.getenv("model")
-TIMEOUT = int(os.getenv("timeout"))
-LANGUAGE = os.getenv("language").capitalize()
-PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
-VOCAB = Path("vocab.txt").read_text(encoding="utf-8")
-LOCK = threading.Lock()
-WIDTH = int(os.getenv("width"))
-LISTWIDTH = int(os.getenv("listWidth"))
-NOTEWIDTH = 70
-MAXHISTORY = 10
-ESTIMATE = ""
-TOKENS = [0, 0]
-NAMESLIST = []
-NAMES = False # Output a list of all the character names found
-BRFLAG = False # If the game uses
instead
-FIXTEXTWRAP = True # Overwrites textwrap
-IGNORETLTEXT = False # Ignores all translated text.
-MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
-FILENAME = None
-
-# tqdm Globals
-BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
-POSITION = 0
-LEAVE = False
-PBAR = None
-
-# Regex - Need to change this if you want to translate from/to other languages. Default is Japanese Regex
-LANGREGEX = r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+"
-
-# Get pricing configuration based on the model
-PRICING_CONFIG = getPricingConfig(MODEL)
-INPUTAPICOST = PRICING_CONFIG["inputAPICost"]
-OUTPUTAPICOST = PRICING_CONFIG["outputAPICost"]
-BATCHSIZE = PRICING_CONFIG["batchSize"]
-FREQUENCY_PENALTY = PRICING_CONFIG["frequencyPenalty"]
-
-# Initialize Translation Config
-TRANSLATION_CONFIG = TranslationConfig(
- model=MODEL,
- language=LANGUAGE,
- prompt=PROMPT,
- vocab=VOCAB,
- langRegex=LANGREGEX,
- batchSize=BATCHSIZE,
- maxHistory=MAXHISTORY,
- estimateMode=False # Will be set dynamically based on ESTIMATE
-)
-LEAVE = False
-
-def handleRenpy(filename, estimate):
- global ESTIMATE
- global FILENAME
- FILENAME = filename
- ESTIMATE = estimate
-
- if ESTIMATE:
- start = time.time()
- translatedData = openFiles(filename)
-
- # Print Result
- end = time.time()
- tqdm.write(getResultString(translatedData, end - start, filename))
- with LOCK:
- TOKENS[0] += translatedData[1][0]
- TOKENS[1] += translatedData[1][1]
-
- # Print Total
- totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
-
- # Print any errors on maps
- if len(MISMATCH) > 0:
- return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
- else:
- return totalString
-
- else:
- try:
- with open("translated/" + filename, "w", encoding="utf8", errors="ignore") as outFile:
- start = time.time()
- translatedData = openFiles(filename)
-
- # Print Result
- end = time.time()
- outFile.writelines(translatedData[0])
- tqdm.write(getResultString(translatedData, end - start, filename))
- with LOCK:
- TOKENS[0] += translatedData[1][0]
- TOKENS[1] += translatedData[1][1]
- except Exception:
- traceback.print_exc()
- return "Fail"
-
- return getResultString(["", TOKENS, None], end - start, "TOTAL")
-
-
-def getResultString(translatedData, translationTime, filename):
- # File Print String
- cost = calculateCost(translatedData[1][0], translatedData[1][1], MODEL)
- totalTokenstring = (
- Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
- "[Output: "
- + str(translatedData[1][1])
- + "]" "[Cost: ${:,.4f}".format(cost)
- + "]"
- )
- timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
-
- if translatedData[2] == None:
- # Success
- return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET
-
- else:
- # Fail
- try:
- raise translatedData[2]
- except Exception as e:
- traceback.print_exc()
- errorString = str(e) + Fore.RED
- return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET
-
-
-def openFiles(filename):
- with open("files/" + filename, "r", encoding="utf8") as readFile:
- translatedData = parseRenpy(readFile, filename)
-
- # Delete lines marked for deletion
- finalData = []
- for line in translatedData[0]:
- if line != "\\d\n":
- finalData.append(line)
- translatedData[0] = finalData
-
- return translatedData
-
-
-def parseRenpy(readFile, filename):
- global PBAR
- totalTokens = [0, 0]
-
- # Read File into data
- data = readFile.readlines()
-
- # Create Progress Bar
- with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
- pbar.desc = filename
- PBAR = pbar
-
- try:
- result = translateRenpy(data, filename, [])
- totalTokens[0] += result[0]
- totalTokens[1] += result[1]
- except Exception as e:
- traceback.print_exc()
- return [data, totalTokens, e]
- return [data, totalTokens, None]
-
-
-def save_progress_lines(lines, filename, encoding="utf-8"):
- """Atomically save current line-based translation progress."""
- try:
- if ESTIMATE:
- return
- os.makedirs("translated", exist_ok=True)
- tmp_fd, tmp_path = tempfile.mkstemp(prefix=f"{filename}.", suffix=".tmp", dir="translated")
- try:
- with os.fdopen(tmp_fd, "w", encoding=encoding, newline="\n", errors="ignore") as tmp_file:
- tmp_file.writelines(lines)
- os.replace(tmp_path, os.path.join("translated", filename))
- finally:
- if os.path.exists(tmp_path):
- try:
- os.remove(tmp_path)
- except OSError:
- pass
- except Exception:
- traceback.print_exc()
-
-
-def translateRenpy(data, filename, translatedList):
- stringList = []
- currentGroup = []
- tokens = [0, 0]
- speaker = ""
- voice = False
- global LOCK, ESTIMATE, FILENAME, PBAR
- i = 0
-
- while i < len(data):
- voice = False
- speaker = ""
- lineRegexNoSpeaker = r'^\s\s\s\s"(.*)"'
- lineRegexSpeaker = r'^\s\s\s\s(.+?)\s"(.*)"'
-
- # Grab Line
- match = re.search(lineRegexSpeaker, data[i])
- if match:
- response = getSpeaker(match.group(1))
- jaString = match.group(2)
- speaker = response[0]
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- else:
- match = re.search(lineRegexNoSpeaker, data[i])
- if match:
- jaString = match.group(1)
-
- # Valid Line
- if match and "voice" not in data[i]:
- originalString = jaString
- # Pass 1
- if translatedList == []:
- # Remove any textwrap
- jaString = jaString.replace("\\n", " ")
-
- # Add String
- if speaker:
- stringList.append(f"[{speaker}]: {jaString.strip()}")
- else:
- stringList.append(jaString.strip())
-
- # Pass 2
- else:
- # Get Text
- if translatedList:
- # Grab and Pop
- translatedText = translatedList[0]
- translatedList.pop(0)
-
- # Set to None if empty list
- if len(translatedList) <= 0:
- translatedList = None
-
- # Remove speaker
- if speaker != "":
- matchSpeakerList = re.findall(r"^\[?(.+?)\]?\s?[|:]\s?", translatedText)
- translatedText = re.sub(r"^\[?(.+?)\]?\s?[|:]\s?", "", translatedText)
-
- # Escape Quotes
- translatedText = re.sub(r'[\\]*(")', '\\"', translatedText)
- translatedText = re.sub(r"[\\]*(')", "\\'", translatedText)
-
- # Textwrap
- translatedText = dazedwrap.wrapText(translatedText, width=WIDTH)
- translatedText = translatedText.replace("\n", "\\n")
-
- # Set Data
- data[i] = data[i].replace(originalString, translatedText)
- save_progress_lines(data, filename)
- i += 1
- else:
- i += 1
-
- # EOF
- if len(stringList) > 0:
- # Set Progress
- PBAR.total = len(stringList)
- PBAR.refresh()
-
- # Translate
- response = translateAI(stringList, "Reply with the English TL of the NPC Name")
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- translatedList = response[0]
-
- # Set Strings
- if len(stringList) == len(translatedList):
- translateRenpy(data, filename, translatedList)
-
- # Mismatch
- else:
- with LOCK:
- if FILENAME not in MISMATCH:
- MISMATCH.append(FILENAME)
- return tokens
-
-# Save some money and enter the character before translation
-def getSpeaker(speaker):
- match speaker:
- case "ファイン":
- return ["Fine", [0, 0]]
- case "":
- return ["", [0, 0]]
- case _:
- # Find Speaker
- for i in range(len(NAMESLIST)):
- if speaker == NAMESLIST[i][0]:
- return [NAMESLIST[i][1], [0, 0]]
-
- # Translate and Store Speaker
- response = translateAI(
- f"{speaker}",
- "Reply with the " + LANGUAGE + " translation of the NPC name.",
- False,
- )
- response[0] = response[0].title()
- response[0] = response[0].replace("'S", "'s")
- response[0] = response[0].replace("Speaker: ", "")
-
- # Retry if name doesn't translate for some reason
- if re.search(r"([a-zA-Z？?])", response[0]) == None:
- response = translateAI(
- f"{speaker}",
- "Reply with the " + LANGUAGE + " translation of the NPC name.",
- False,
- )
- response[0] = response[0].title()
- response[0] = response[0].replace("'S", "'s")
-
- speakerList = [speaker, response[0]]
- NAMESLIST.append(speakerList)
- return response
- return [speaker, [0, 0]]
-
-def translateAI(text, history, history_ctx=None):
- """
- Legacy wrapper function for the new shared translation utility.
- This maintains compatibility with existing code while using the new shared implementation.
- """
- global PBAR, MISMATCH, FILENAME
-
- # Update config estimate mode based on global ESTIMATE
- TRANSLATION_CONFIG.estimateMode = bool(ESTIMATE)
-
- # Call the new shared translation function
- return sharedtranslateAI(
- text=text,
- history=history,
- config=TRANSLATION_CONFIG,
- filename=FILENAME,
- pbar=PBAR,
- lock=LOCK,
- mismatchList=MISMATCH
- )
+# Libraries
+import json
+import os
+import re
+import util.dazedwrap as dazedwrap
+import threading
+import time
+import traceback
+import tiktoken
+from pathlib import Path
+from colorama import Fore
+from dotenv import load_dotenv
+from retry import retry
+from tqdm import tqdm
+from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost, getPricingConfig, calculateCost
+import tempfile
+
+# Globals
+MODEL = os.getenv("model")
+TIMEOUT = int(os.getenv("timeout"))
+LANGUAGE = os.getenv("language").capitalize()
+from util.paths import PROMPT_PATH, VOCAB_PATH
+
+PROMPT = PROMPT_PATH.read_text(encoding="utf-8")
+VOCAB = VOCAB_PATH.read_text(encoding="utf-8")
+LOCK = threading.Lock()
+WIDTH = int(os.getenv("width"))
+LISTWIDTH = int(os.getenv("listWidth"))
+NOTEWIDTH = 70
+MAXHISTORY = 10
+ESTIMATE = ""
+TOKENS = [0, 0]
+NAMESLIST = []
+NAMES = False # Output a list of all the character names found
+BRFLAG = False # If the game uses
instead
+FIXTEXTWRAP = True # Overwrites textwrap
+IGNORETLTEXT = False # Ignores all translated text.
+MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
+FILENAME = None
+
+# tqdm Globals
+BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
+POSITION = 0
+LEAVE = False
+PBAR = None
+
+# Regex - Need to change this if you want to translate from/to other languages. Default is Japanese Regex
+LANGREGEX = r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+"
+
+# Get pricing configuration based on the model
+PRICING_CONFIG = getPricingConfig(MODEL)
+INPUTAPICOST = PRICING_CONFIG["inputAPICost"]
+OUTPUTAPICOST = PRICING_CONFIG["outputAPICost"]
+BATCHSIZE = PRICING_CONFIG["batchSize"]
+FREQUENCY_PENALTY = PRICING_CONFIG["frequencyPenalty"]
+
+# Initialize Translation Config
+TRANSLATION_CONFIG = TranslationConfig(
+ model=MODEL,
+ language=LANGUAGE,
+ prompt=PROMPT,
+ vocab=VOCAB,
+ langRegex=LANGREGEX,
+ batchSize=BATCHSIZE,
+ maxHistory=MAXHISTORY,
+ estimateMode=False # Will be set dynamically based on ESTIMATE
+)
+LEAVE = False
+
+def handleRenpy(filename, estimate):
+ global ESTIMATE
+ global FILENAME
+ FILENAME = filename
+ ESTIMATE = estimate
+
+ if ESTIMATE:
+ start = time.time()
+ translatedData = openFiles(filename)
+
+ # Print Result
+ end = time.time()
+ tqdm.write(getResultString(translatedData, end - start, filename))
+ with LOCK:
+ TOKENS[0] += translatedData[1][0]
+ TOKENS[1] += translatedData[1][1]
+
+ # Print Total
+ totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
+
+ # Print any errors on maps
+ if len(MISMATCH) > 0:
+ return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
+ else:
+ return totalString
+
+ else:
+ try:
+ with open("translated/" + filename, "w", encoding="utf8", errors="ignore") as outFile:
+ start = time.time()
+ translatedData = openFiles(filename)
+
+ # Print Result
+ end = time.time()
+ outFile.writelines(translatedData[0])
+ tqdm.write(getResultString(translatedData, end - start, filename))
+ with LOCK:
+ TOKENS[0] += translatedData[1][0]
+ TOKENS[1] += translatedData[1][1]
+ except Exception:
+ traceback.print_exc()
+ return "Fail"
+
+ return getResultString(["", TOKENS, None], end - start, "TOTAL")
+
+
+def getResultString(translatedData, translationTime, filename):
+ # File Print String
+ cost = calculateCost(translatedData[1][0], translatedData[1][1], MODEL)
+ totalTokenstring = (
+ Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
+ "[Output: "
+ + str(translatedData[1][1])
+ + "]" "[Cost: ${:,.4f}".format(cost)
+ + "]"
+ )
+ timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
+
+ if translatedData[2] == None:
+ # Success
+ return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET
+
+ else:
+ # Fail
+ try:
+ raise translatedData[2]
+ except Exception as e:
+ traceback.print_exc()
+ errorString = str(e) + Fore.RED
+ return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET
+
+
+def openFiles(filename):
+ with open("files/" + filename, "r", encoding="utf8") as readFile:
+ translatedData = parseRenpy(readFile, filename)
+
+ # Delete lines marked for deletion
+ finalData = []
+ for line in translatedData[0]:
+ if line != "\\d\n":
+ finalData.append(line)
+ translatedData[0] = finalData
+
+ return translatedData
+
+
+def parseRenpy(readFile, filename):
+ global PBAR
+ totalTokens = [0, 0]
+
+ # Read File into data
+ data = readFile.readlines()
+
+ # Create Progress Bar
+ with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
+ pbar.desc = filename
+ PBAR = pbar
+
+ try:
+ result = translateRenpy(data, filename, [])
+ totalTokens[0] += result[0]
+ totalTokens[1] += result[1]
+ except Exception as e:
+ traceback.print_exc()
+ return [data, totalTokens, e]
+ return [data, totalTokens, None]
+
+
+def save_progress_lines(lines, filename, encoding="utf-8"):
+ """Atomically save current line-based translation progress."""
+ try:
+ if ESTIMATE:
+ return
+ os.makedirs("translated", exist_ok=True)
+ tmp_fd, tmp_path = tempfile.mkstemp(prefix=f"{filename}.", suffix=".tmp", dir="translated")
+ try:
+ with os.fdopen(tmp_fd, "w", encoding=encoding, newline="\n", errors="ignore") as tmp_file:
+ tmp_file.writelines(lines)
+ os.replace(tmp_path, os.path.join("translated", filename))
+ finally:
+ if os.path.exists(tmp_path):
+ try:
+ os.remove(tmp_path)
+ except OSError:
+ pass
+ except Exception:
+ traceback.print_exc()
+
+
+def translateRenpy(data, filename, translatedList):
+ stringList = []
+ currentGroup = []
+ tokens = [0, 0]
+ speaker = ""
+ voice = False
+ global LOCK, ESTIMATE, FILENAME, PBAR
+ i = 0
+
+ while i < len(data):
+ voice = False
+ speaker = ""
+ lineRegexNoSpeaker = r'^\s\s\s\s"(.*)"'
+ lineRegexSpeaker = r'^\s\s\s\s(.+?)\s"(.*)"'
+
+ # Grab Line
+ match = re.search(lineRegexSpeaker, data[i])
+ if match:
+ response = getSpeaker(match.group(1))
+ jaString = match.group(2)
+ speaker = response[0]
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ else:
+ match = re.search(lineRegexNoSpeaker, data[i])
+ if match:
+ jaString = match.group(1)
+
+ # Valid Line
+ if match and "voice" not in data[i]:
+ originalString = jaString
+ # Pass 1
+ if translatedList == []:
+ # Remove any textwrap
+ jaString = jaString.replace("\\n", " ")
+
+ # Add String
+ if speaker:
+ stringList.append(f"[{speaker}]: {jaString.strip()}")
+ else:
+ stringList.append(jaString.strip())
+
+ # Pass 2
+ else:
+ # Get Text
+ if translatedList:
+ # Grab and Pop
+ translatedText = translatedList[0]
+ translatedList.pop(0)
+
+ # Set to None if empty list
+ if len(translatedList) <= 0:
+ translatedList = None
+
+ # Remove speaker
+ if speaker != "":
+ matchSpeakerList = re.findall(r"^\[?(.+?)\]?\s?[|:]\s?", translatedText)
+ translatedText = re.sub(r"^\[?(.+?)\]?\s?[|:]\s?", "", translatedText)
+
+ # Escape Quotes
+ translatedText = re.sub(r'[\\]*(")', '\\"', translatedText)
+ translatedText = re.sub(r"[\\]*(')", "\\'", translatedText)
+
+ # Textwrap
+ translatedText = dazedwrap.wrapText(translatedText, width=WIDTH)
+ translatedText = translatedText.replace("\n", "\\n")
+
+ # Set Data
+ data[i] = data[i].replace(originalString, translatedText)
+ save_progress_lines(data, filename)
+ i += 1
+ else:
+ i += 1
+
+ # EOF
+ if len(stringList) > 0:
+ # Set Progress
+ PBAR.total = len(stringList)
+ PBAR.refresh()
+
+ # Translate
+ response = translateAI(stringList, "Reply with the English TL of the NPC Name")
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ translatedList = response[0]
+
+ # Set Strings
+ if len(stringList) == len(translatedList):
+ translateRenpy(data, filename, translatedList)
+
+ # Mismatch
+ else:
+ with LOCK:
+ if FILENAME not in MISMATCH:
+ MISMATCH.append(FILENAME)
+ return tokens
+
+# Save some money and enter the character before translation
+def getSpeaker(speaker):
+ match speaker:
+ case "ファイン":
+ return ["Fine", [0, 0]]
+ case "":
+ return ["", [0, 0]]
+ case _:
+ # Find Speaker
+ for i in range(len(NAMESLIST)):
+ if speaker == NAMESLIST[i][0]:
+ return [NAMESLIST[i][1], [0, 0]]
+
+ # Translate and Store Speaker
+ response = translateAI(
+ f"{speaker}",
+ "Reply with the " + LANGUAGE + " translation of the NPC name.",
+ False,
+ )
+ response[0] = response[0].title()
+ response[0] = response[0].replace("'S", "'s")
+ response[0] = response[0].replace("Speaker: ", "")
+
+ # Retry if name doesn't translate for some reason
+ if re.search(r"([a-zA-Z？?])", response[0]) == None:
+ response = translateAI(
+ f"{speaker}",
+ "Reply with the " + LANGUAGE + " translation of the NPC name.",
+ False,
+ )
+ response[0] = response[0].title()
+ response[0] = response[0].replace("'S", "'s")
+
+ speakerList = [speaker, response[0]]
+ NAMESLIST.append(speakerList)
+ return response
+ return [speaker, [0, 0]]
+
+def translateAI(text, history, history_ctx=None):
+ """
+ Legacy wrapper function for the new shared translation utility.
+ This maintains compatibility with existing code while using the new shared implementation.
+ """
+ global PBAR, MISMATCH, FILENAME
+
+ # Update config estimate mode based on global ESTIMATE
+ TRANSLATION_CONFIG.estimateMode = bool(ESTIMATE)
+
+ # Call the new shared translation function
+ return sharedtranslateAI(
+ text=text,
+ history=history,
+ config=TRANSLATION_CONFIG,
+ filename=FILENAME,
+ pbar=PBAR,
+ lock=LOCK,
+ mismatchList=MISMATCH
+ )
diff --git a/modules/rpgmakermvmz.py b/modules/rpgmakermvmz.py
index b6a5a5e..e4350ad 100644
--- a/modules/rpgmakermvmz.py
+++ b/modules/rpgmakermvmz.py
@@ -19,8 +19,10 @@ from util.translation import TranslationConfig, translateAI as sharedtranslateAI
MODEL = os.getenv("model")
TIMEOUT = int(os.getenv("timeout"))
LANGUAGE = os.getenv("language").capitalize()
-PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
-VOCAB = Path("vocab.txt").read_text(encoding="utf-8")
+from util.paths import PROMPT_PATH, VOCAB_PATH
+
+PROMPT = PROMPT_PATH.read_text(encoding="utf-8")
+VOCAB = VOCAB_PATH.read_text(encoding="utf-8")
LOCK = threading.Lock()
THREAD_CTX = threading.local()
WIDTH = int(os.getenv("width"))
@@ -803,7 +805,7 @@ def update_vocab_section(category: str, pairs: list[tuple[str, str]]):
The existing section is replaced entirely; other sections are preserved.
"""
try:
- vocab_path = Path("vocab.txt")
+ vocab_path = VOCAB_PATH
# Helper: normalized comparison to detect no-op translations
def _norm(s: str) -> str:
@@ -5071,7 +5073,7 @@ def finalizeSpeakerParse():
_speakerCache[orig] = norm
NAMESLIST.append([orig, norm])
- vocab_path = Path("vocab.txt")
+ vocab_path = VOCAB_PATH
if not vocab_path.exists():
return
content = vocab_path.read_text(encoding="utf-8")
diff --git a/modules/rpgmakerplugin.py b/modules/rpgmakerplugin.py
index 161a719..343993a 100644
--- a/modules/rpgmakerplugin.py
+++ b/modules/rpgmakerplugin.py
@@ -1,1135 +1,1137 @@
-# Libraries
-import json
-import os
-import re
-import util.dazedwrap as dazedwrap
-import threading
-import time
-import traceback
-import tiktoken
-from pathlib import Path
-from colorama import Fore
-from dotenv import load_dotenv
-from retry import retry
-from tqdm import tqdm
-from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost
-import tempfile
-
-# Globals
-MODEL = os.getenv("model")
-TIMEOUT = int(os.getenv("timeout"))
-LANGUAGE = os.getenv("language").capitalize()
-PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
-VOCAB = Path("vocab.txt").read_text(encoding="utf-8")
-LOCK = threading.Lock()
-WIDTH = int(os.getenv("width"))
-LISTWIDTH = int(os.getenv("listWidth"))
-NOTEWIDTH = 70
-MAXHISTORY = 10
-ESTIMATE = ""
-TOKENS = [0, 0]
-NAMESLIST = []
-NAMES = False # Output a list of all the character names found
-BRFLAG = False # If the game uses
instead
-FIXTEXTWRAP = True # Overwrites textwrap
-IGNORETLTEXT = False # Ignores all translated text.
-MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
-FILENAME = None
-
-# tqdm Globals
-BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
-POSITION = 0
-LEAVE = False
-PBAR = None
-
-# Regex - Need to change this if you want to translate from/to other languages. Default is Japanese Regex
-LANGREGEX = r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+"
-
-# Get pricing configuration based on the model
-PRICING_CONFIG = getPricingConfig(MODEL)
-INPUTAPICOST = PRICING_CONFIG["inputAPICost"]
-OUTPUTAPICOST = PRICING_CONFIG["outputAPICost"]
-BATCHSIZE = PRICING_CONFIG["batchSize"]
-FREQUENCY_PENALTY = PRICING_CONFIG["frequencyPenalty"]
-
-# Initialize Translation Config
-TRANSLATION_CONFIG = TranslationConfig(
- model=MODEL,
- language=LANGUAGE,
- prompt=PROMPT,
- vocab=VOCAB,
- langRegex=LANGREGEX,
- batchSize=BATCHSIZE,
- maxHistory=MAXHISTORY,
- estimateMode=False # Will be set dynamically based on ESTIMATE
-)
-LEAVE = False
-
-def handlePlugin(filename, estimate):
- global ESTIMATE, PBAR, FILENAME
- ESTIMATE = estimate
- FILENAME = filename
-
- if ESTIMATE:
- start = time.time()
- translatedData = openFiles(filename)
-
- # Print Result
- end = time.time()
- tqdm.write(getResultString(translatedData, end - start, filename))
- with LOCK:
- TOKENS[0] += translatedData[1][0]
- TOKENS[1] += translatedData[1][1]
-
- # Print Total
- totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
-
- # Print any errors on maps
- if len(MISMATCH) > 0:
- return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
- else:
- return totalString
-
- else:
- try:
- # Perform translation first; incremental progress writes happen during translation.
- # We no longer keep the destination file open simultaneously to avoid Windows locking issues
- # during atomic os.replace operations inside save_progress_lines.
- start = time.time()
- translatedData = openFiles(filename)
-
- # Ensure final state is flushed (in case no incremental writes occurred for some reason).
- save_progress_lines(translatedData[0], filename)
-
- # Print Result
- end = time.time()
- tqdm.write(getResultString(translatedData, end - start, filename))
- with LOCK:
- TOKENS[0] += translatedData[1][0]
- TOKENS[1] += translatedData[1][1]
- except Exception:
- traceback.print_exc()
- return "Fail"
-
- return getResultString(["", TOKENS, None], end - start, "TOTAL")
-
-
-def getResultString(translatedData, translationTime, filename):
- # File Print String
- cost = calculateCost(translatedData[1][0], translatedData[1][1], MODEL)
- totalTokenstring = (
- Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
- "[Output: "
- + str(translatedData[1][1])
- + "]" "[Cost: ${:,.4f}".format(cost)
- + "]"
- )
- timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
-
- if translatedData[2] == None:
- # Success
- return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET
-
- else:
- # Fail
- try:
- raise translatedData[2]
- except Exception as e:
- traceback.print_exc()
- errorString = str(e) + Fore.RED
- return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET
-
-
-def openFiles(filename):
- with open("files/" + filename, "r", encoding="utf_8") as readFile:
- translatedData = parsePlugin(readFile, filename)
-
- # Delete lines marked for deletion
- finalData = []
- for line in translatedData[0]:
- if line != "\\d\n":
- finalData.append(line)
- translatedData[0] = finalData
-
- return translatedData
-
-
-def parsePlugin(readFile, filename):
- global PBAR
- totalTokens = [0, 0]
-
- # Read File into data
- data = readFile.readlines()
-
- # Create Progress Bar
- with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
- pbar.desc = filename
- PBAR = pbar
-
- try:
- result = translatePlugin(data, pbar, filename, [])
- totalTokens[0] += result[0]
- totalTokens[1] += result[1]
- except Exception as e:
- traceback.print_exc()
- return [data, totalTokens, e]
- return [data, totalTokens, None]
-
-
-def save_progress_lines(lines, filename, encoding="utf_8"):
- """Atomically save current line-based translation progress."""
- try:
- if ESTIMATE:
- return
- global LOCK
- os.makedirs("translated", exist_ok=True)
- # Use a single lock to prevent concurrent replace attempts on Windows (which causes PermissionError)
- with LOCK:
- tmp_fd, tmp_path = tempfile.mkstemp(prefix=f"{filename}.", suffix=".tmp", dir="translated")
- try:
- with os.fdopen(tmp_fd, "w", encoding=encoding, newline="\n", errors="ignore") as tmp_file:
- tmp_file.writelines(lines)
-
- dest_path = os.path.join("translated", filename)
- # Retry a few times in case another process/thread still has the file open momentarily.
- for attempt in range(5):
- try:
- os.replace(tmp_path, dest_path)
- break
- except PermissionError:
- if attempt == 4:
- raise
- time.sleep(0.1 * (attempt + 1))
- finally:
- if os.path.exists(tmp_path):
- try:
- os.remove(tmp_path)
- except OSError:
- pass
- except Exception:
- traceback.print_exc()
-
-
-def saveCheckLines(lines, filename, tokens=None, encoding="utf_8"):
- """Save progress only when tokens indicate work or when explicitly called after a mutation."""
- try:
- if tokens is not None:
- if not (isinstance(tokens, (list, tuple)) and len(tokens) >= 2 and (tokens[0] or tokens[1])):
- return
- save_progress_lines(lines, filename, encoding=encoding)
- except Exception:
- traceback.print_exc()
-
-
-def translatePlugin(data, pbar, filename, translatedList):
- if len(translatedList) > 0:
- questList = translatedList[0]
- custom = translatedList[1]
- sceneMenuText = translatedList[2] if len(translatedList) > 2 else []
- sceneMenuCommonHelpText = translatedList[3] if len(translatedList) > 3 else []
- sceneMenuHelpText = translatedList[4] if len(translatedList) > 4 else []
- sceneMenuDrawText = translatedList[5] if len(translatedList) > 5 else []
- sceneMenuStatLabel = translatedList[6] if len(translatedList) > 6 else []
- saveParamName = translatedList[7] if len(translatedList) > 7 else []
- setData = True
- else:
- questList = [[], [], [], [], [], []]
- custom = []
- sceneMenuText = []
- sceneMenuCommonHelpText = []
- sceneMenuHelpText = []
- sceneMenuDrawText = []
- sceneMenuStatLabel = []
- saveParamName = []
- setData = False
- currentGroup = []
- tokens = [0, 0]
- speaker = ""
- voice = False
- global LOCK, ESTIMATE
- i = 0
- in_disp_list = False
- brace_count = 0
- _pending_drawtext = {} # Deferred drawTextEx brace label replacements (longest-first)
- _pending_statlabel = {} # Deferred drawTextEx stat label replacements (longest-first)
-
- # Category
- with open("log/translations.txt", "a+", encoding="utf-8") as tlFile:
- tlFile.write(f"\nCustom:\n")
- tlFile.close()
-
- while i < len(data):
- voice = False
- speaker = ""
-
- # Track if we're inside CBR_travel_data block
- if 'data_map_name_list' in data[i]:
- in_disp_list = True
- brace_count = 0
-
- if in_disp_list:
- # Count braces to know when we exit the object
- brace_count += data[i].count('{') - data[i].count('}')
- if brace_count < 0:
- in_disp_list = False
-
- # Nested array strings (e.g., this.disp_list = { key: { subkey: ["string1", "string2"] } })
- # Matches strings inside arrays within object literals
- if in_disp_list:
- regex_nested = r'"([^"]*[一-龠ぁ-ゔァ-ヴー]+[^"]*)"'
- matchList = re.findall(regex_nested, data[i])
- if len(matchList) > 0:
- for match in matchList:
- # Save Original String
- jaString = match
- originalString = jaString
-
- # Replace \n for translation
- jaStringClean = jaString.replace("\\n", " ")
-
- if jaStringClean.strip():
- # Pass 1
- if setData == False:
- custom.append(jaStringClean.strip())
-
- # Pass 2
- else:
- if custom:
- # Grab and Pop
- translatedText = custom[0]
- custom.pop(0)
-
- # Set to None if empty list
- if len(custom) <= 0:
- custom = []
-
- # Restore original newlines but keep translated text
- # Count original newlines
- newline_count = jaString.count("\\n")
- if newline_count > 0:
- # Textwrap the translation
- translatedText = dazedwrap.wrapText(translatedText, 50)
- translatedText = translatedText.replace("\n", "\\n")
-
- # Escape quotes in translation
- translatedText = translatedText.replace('"', '\\"')
-
- # Set Data
- data[i] = data[i].replace(f'"{originalString}"', f'"{translatedText}"')
- saveCheckLines(data, filename)
-
- # Custom
- # Useful Regex's
- # r'"Text[\\]+":[\\]+"(.+?)[\\]+",'
- # r'"HelpText[\\]+":[\\]+"(.+?)[\\]+",'
- # r"this.drawTextEx\(\\'(.+?)\',"
- # r'txtSubject.+?"(.+)"'
- regex = r'txtSubject.+?"(.+)"'
- matchList = re.findall(regex, data[i])
- if len(matchList) > 0:
- for match in matchList:
- # Save Original String
- jaString = match
- originalString = jaString
- newline = None
- colorCode = None
-
- # Make sure it contains Japanese
- if not re.search(LANGREGEX, jaString):
- continue
-
- # Make sure didn't grab \\
- if re.search(r"^[\\]+$", jaString):
- continue
-
- # Replace \n and \c
- if re.search(r"\\+n", jaString):
- newline = re.search(r"\\+n", jaString).group(0)
- jaString = re.sub(r"\\+n", r"\\n", jaString)
- if re.search(r"\\+C", jaString):
- colorCode = re.search(r"\\+C", jaString).group(0)
- jaString = re.sub(r"\\+C", r"\\C", jaString)
-
- # Remove any textwrap
- jaString = jaString.replace("\\n", " ")
-
- if jaString.replace("\u3000", "") and jaString:
- # Pass 1
- if setData == False and jaString.strip():
- custom.append(jaString.strip())
-
- # Pass 2
- else:
- if custom:
- # Grab and Pop
- translatedText = custom[0]
- custom.pop(0)
-
- # Set to None if empty list
- if len(translatedList) <= 0:
- translatedList = None
-
- # Replace Single Quotes
- translatedText = re.sub(r"([^\\'])'", r"\1՚", translatedText)
- translatedText = re.sub(r"([^\\'])\"", r"\1՚", translatedText)
-
- # Replace \n and \c
- if newline:
- # Textwrap
- # translatedText = dazedwrap.wrapText(translatedText, WIDTH)
- translatedText = re.sub(r"\n", re.escape(newline), translatedText)
- if colorCode:
- translatedText = re.sub(r"\n", re.escape(colorCode), translatedText)
-
- # Set Data
- with open("log/translations.txt", "a+", encoding="utf-8") as tlFile:
- tlFile.write(f"{originalString} ({translatedText})\n")
- data[i] = data[i].replace(originalString, translatedText)
- saveCheckLines(data, filename)
-
- # Quest Name
- regex = r'[\\]+"QuestName[\\]+":[\\]+"(.*?)[\\]+"'
- matchList = re.findall(regex, data[i])
- if len(matchList) > 0:
- for match in matchList:
- if match:
- # Save Original String
- originalString = match
-
- # Remove any textwrap
- match = match.replace(newline, " ")
-
- # Pass 1
- if setData == False:
- # Add String
- if match != "\\\\\\\\":
- questList[0].append(match.strip())
-
- # Pass 2
- else:
- if questList[0]:
- # Grab and Pop
- translatedText = questList[0][0]
- questList[0].pop(0)
-
- # Set to None if empty list
- if len(translatedList) <= 0:
- translatedList = None
-
- # Replace Single Quotes
- translatedText = translatedText.replace('"', "'")
- translatedText = re.sub(r"([^\\'])'", r"\1\\'", translatedText)
-
- # Set Data
- data[i] = data[i].replace(originalString, translatedText)
- saveCheckLines(data, filename)
-
- # Quest Client
- regex = r'QuestClientName[\\]+":[\\]+"(.*?)[\\]+"'
- matchList = re.findall(regex, data[i])
- if len(matchList) > 0:
- for match in matchList:
- if match:
- # Save Original String
- originalString = match
-
- # Pass 1
- if setData == False:
- # Add String
- if match != "\\\\\\\\":
- questList[1].append(match.strip())
-
- # Pass 2
- else:
- if questList[1]:
- # Grab and Pop
- translatedText = questList[1][0]
- questList[1].pop(0)
-
- # Set to None if empty list
- if len(translatedList) <= 0:
- translatedList = None
-
- # Replace Single Quotes
- translatedText = translatedText.replace('"', "'")
- translatedText = re.sub(r"([^\\'])'", r"\1\\'", translatedText)
-
- # Set Data
- data[i] = data[i].replace(originalString, translatedText)
- saveCheckLines(data, filename)
-
- # Quest Location
- regex = r'QuestLocation[\\]+":[\\]+"(.*?)[\\]+"'
- matchList = re.findall(regex, data[i])
- if len(matchList) > 0:
- for match in matchList:
- if match:
- # Save Original String
- originalString = match
-
- # Pass 1
- if setData == False:
- # Add String
- if match != "\\\\\\\\":
- questList[2].append(match.strip())
-
- # Pass 2
- else:
- if questList[2]:
- # Grab and Pop
- translatedText = questList[2][0]
- questList[2].pop(0)
-
- # Set to None if empty list
- if len(translatedList) <= 0:
- translatedList = None
-
- # Replace Single Quotes
- translatedText = translatedText.replace('"', "'")
- translatedText = re.sub(r"([^\\'])'", r"\1\\'", translatedText)
-
- # Set Data
- data[i] = data[i].replace(originalString, translatedText)
- saveCheckLines(data, filename)
-
- # Quest Target
- regex = r'PlaceInformation[\\]+":[\\]+"(.*?)[\\]+"'
- matchList = re.findall(regex, data[i])
- if len(matchList) > 0:
- for match in matchList:
- if match:
- # Save Original String
- originalString = match
-
- # Pass 1
- if setData == False:
- # Add String
- if match != "\\\\\\\\":
- questList[3].append(match.strip())
-
- # Pass 2
- else:
- if questList[3]:
- # Grab and Pop
- translatedText = questList[3][0]
- questList[3].pop(0)
-
- # Set to None if empty list
- if len(translatedList) <= 0:
- translatedList = None
-
- # Replace Single Quotes
- translatedText = translatedText.replace('"', "'")
- translatedText = re.sub(r"([^\\'])'", r"\1\\'", translatedText)
-
- # Set Data
- data[i] = data[i].replace(originalString, translatedText)
- saveCheckLines(data, filename)
-
- # Quest Summary
- regex = r'[\\]+"QuestContent[\\]+":[\\]+"[\\]+"(.*?)[\\]+"[\\]+"'
- matchList = re.findall(regex, data[i])
- if len(matchList) > 0:
- for match in matchList:
- if match:
- # Save Original String
- originalString = match
-
- # Remove any textwrap
- match = match.replace(r"\\\\\\\\n", " ")
- match = match.replace(r"\\\\\\\\\\\\\\\\c", "\\c")
-
- # Pass 1
- if setData == False:
- # Add String
- if match != "\\\\\\\\":
- questList[4].append(match.strip())
-
- # Pass 2
- else:
- if questList[4]:
- # Grab and Pop
- translatedText = questList[4][0]
- questList[4].pop(0)
-
- # Set to None if empty list
- if len(translatedList) <= 0:
- translatedList = None
-
- # Textwrap
- translatedText = dazedwrap.wrapText(translatedText, width=WIDTH)
- translatedText = translatedText.replace("\n", r"\\\\\\\\n")
- match = match.replace("\\c", r"\\\\\\\\\\\\\\\\c")
-
- # Replace Single Quotes
- translatedText = translatedText.replace('"', "'")
- translatedText = re.sub(r"([^\\'])'", r"\1\\'", translatedText)
-
- # Set Data
- data[i] = data[i].replace(originalString, translatedText)
- saveCheckLines(data, filename)
-
- # Quest Goal 1
- regex = r'ObjectiveContent[\\]+":[\\]+"[\\]+"(.*?)[\\]+"'
- matchList = re.findall(regex, data[i])
- if len(matchList) > 0:
- for match in matchList:
- if match:
- # Save Original String
- originalString = match
-
- # Remove any textwrap
- match = match.replace(r"\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\n", " ")
-
- # Pass 1
- if setData == False:
- # Add String
- if match != "\\\\\\\\":
- questList[5].append(match.strip())
-
- # Pass 2
- else:
- if questList[5]:
- # Grab and Pop
- translatedText = questList[5][0]
- questList[5].pop(0)
-
- # Set to None if empty list
- if len(translatedList) <= 0:
- translatedList = None
-
- # Textwrap
- translatedText = dazedwrap.wrapText(translatedText, width=WIDTH)
- translatedText = translatedText.replace("\n", r"\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\n")
-
- # Replace Single Quotes
- translatedText = translatedText.replace('"', "'")
- translatedText = re.sub(r"([^\\'])'", r"\1\\'", translatedText)
-
- # Set Data
- data[i] = data[i].replace(originalString, translatedText)
- saveCheckLines(data, filename)
-
- # SceneCustomMenu - Text (Command Labels)
- regex = r'[\\]+"Text[\\]+":[\\]+"(.+?)[\\]+"'
- matchList = re.findall(regex, data[i])
- if len(matchList) > 0:
- for match in matchList:
- if match:
- jaString = match
- originalString = jaString
-
- # Make sure it contains Japanese
- if not re.search(LANGREGEX, jaString):
- continue
-
- # Make sure didn't grab only backslashes
- if re.search(r"^[\\]+$", jaString):
- continue
-
- if jaString.replace("\u3000", "").strip():
- # Pass 1
- if setData == False:
- sceneMenuText.append(jaString.strip())
-
- # Pass 2
- else:
- if sceneMenuText:
- # Grab and Pop
- translatedText = sceneMenuText[0]
- sceneMenuText.pop(0)
-
- # Replace quotes
- translatedText = translatedText.replace('"', "'")
- translatedText = re.sub(r"([^\\'])'" , r"\1\\'", translatedText)
-
- # Set Data
- data[i] = data[i].replace(originalString, translatedText)
- saveCheckLines(data, filename)
-
- # SceneCustomMenu - CommonHelpText
- regex = r'[\\]+"CommonHelpText[\\]+":[\\]+"(.+?)[\\]+"'
- matchList = re.findall(regex, data[i])
- if len(matchList) > 0:
- for match in matchList:
- if match:
- jaString = match
- originalString = jaString
-
- # Make sure it contains Japanese
- if not re.search(LANGREGEX, jaString):
- continue
-
- # Make sure didn't grab only backslashes
- if re.search(r"^[\\]+$", jaString):
- continue
-
- if jaString.replace("\u3000", "").strip():
- # Pass 1
- if setData == False:
- sceneMenuCommonHelpText.append(jaString.strip())
-
- # Pass 2
- else:
- if sceneMenuCommonHelpText:
- # Grab and Pop
- translatedText = sceneMenuCommonHelpText[0]
- sceneMenuCommonHelpText.pop(0)
-
- # Replace quotes
- translatedText = translatedText.replace('"', "'")
- translatedText = re.sub(r"([^\\'])'" , r"\1\\'", translatedText)
-
- # Set Data
- data[i] = data[i].replace(originalString, translatedText)
- saveCheckLines(data, filename)
-
- # SceneCustomMenu - HelpText (Command Help)
- regex = r'[\\]+"HelpText[\\]+":[\\]+"(.+?)[\\]+"'
- matchList = re.findall(regex, data[i])
- if len(matchList) > 0:
- for match in matchList:
- if match:
- jaString = match
- originalString = jaString
-
- # Make sure it contains Japanese
- if not re.search(LANGREGEX, jaString):
- continue
-
- # Make sure didn't grab only backslashes
- if re.search(r"^[\\]+$", jaString):
- continue
-
- if jaString.replace("\u3000", "").strip():
- # Pass 1
- if setData == False:
- sceneMenuHelpText.append(jaString.strip())
-
- # Pass 2
- else:
- if sceneMenuHelpText:
- # Grab and Pop
- translatedText = sceneMenuHelpText[0]
- sceneMenuHelpText.pop(0)
-
- # Replace quotes
- translatedText = translatedText.replace('"', "'")
- translatedText = re.sub(r"([^\\'])'" , r"\1\\'", translatedText)
-
- # Set Data
- data[i] = data[i].replace(originalString, translatedText)
- saveCheckLines(data, filename)
-
- # NUUN_SaveScreen - ParamName (save screen labels like 現在地, プレイ時間, 所持金)
- regex = r'[\\]+"ParamName[\\]+":[\\]+"(.+?)[\\]+"'
- matchList = re.findall(regex, data[i])
- if len(matchList) > 0:
- for match in matchList:
- if match:
- jaString = match
- originalString = jaString
-
- # Make sure it contains Japanese
- if not re.search(LANGREGEX, jaString):
- continue
-
- # Make sure didn't grab only backslashes
- if re.search(r"^[\\]+$", jaString):
- continue
-
- if jaString.replace("\u3000", "").strip():
- # Pass 1
- if setData == False:
- saveParamName.append(jaString.strip())
-
- # Pass 2
- else:
- if saveParamName:
- # Grab and Pop
- translatedText = saveParamName[0]
- saveParamName.pop(0)
-
- # Replace quotes
- translatedText = translatedText.replace('"', "'")
- translatedText = re.sub(r"([^\\'])'" , r"\1\\'", translatedText)
-
- # Set Data
- data[i] = data[i].replace(originalString, translatedText)
- saveCheckLines(data, filename)
-
- # DrawTextEx - Brace-delimited labels (e.g., \}【口づけ】\{, \}回数：\{)
- # Only process lines containing drawTextEx calls
- if 'drawTextEx' in data[i]:
- regex = r'[\\]+\}(.+?)[\\]+\{'
- matchList = re.findall(regex, data[i])
- if len(matchList) > 0:
- for match in matchList:
- if match:
- jaString = match
- originalString = jaString
-
- # Skip game variable references (handled separately)
- if '$gameVariables' in jaString:
- continue
-
- # Make sure it contains Japanese
- if not re.search(LANGREGEX, jaString):
- continue
-
- if jaString.replace("\u3000", "").strip():
- # Pass 1 (deduplicated)
- if setData == False:
- if jaString.strip() not in sceneMenuDrawText:
- sceneMenuDrawText.append(jaString.strip())
-
- # Pass 2 (collect mapping, skip duplicates)
- else:
- if sceneMenuDrawText and originalString not in _pending_drawtext:
- # Grab and Pop
- translatedText = sceneMenuDrawText[0]
- sceneMenuDrawText.pop(0)
-
- # Replace quotes
- translatedText = translatedText.replace('"', "'")
-
- # Queue for deferred replacement
- _pending_drawtext[originalString] = translatedText
-
- # Apply deferred replacements longest-first to avoid substring collisions
- # (e.g., replacing 回数： before 使用回数： would corrupt the longer string)
- if setData and _pending_drawtext:
- for orig in sorted(_pending_drawtext, key=len, reverse=True):
- data[i] = data[i].replace(orig, _pending_drawtext[orig])
- _pending_drawtext.clear()
- saveCheckLines(data, filename)
-
- # DrawTextEx - Stat labels (e.g., C[16]攻撃力\\...C[0])
- if 'drawTextEx' in data[i]:
- regex = r'C\[\d+\]([^C\\,`\[\]]+?)[\\]+C\[0\]'
- matchList = re.findall(regex, data[i])
- if len(matchList) > 0:
- for match in matchList:
- if match:
- jaString = match
- originalString = jaString
-
- # Make sure it contains Japanese
- if not re.search(LANGREGEX, jaString):
- continue
-
- if jaString.replace("\u3000", "").strip():
- # Pass 1 (deduplicated)
- if setData == False:
- if jaString.strip() not in sceneMenuStatLabel:
- sceneMenuStatLabel.append(jaString.strip())
-
- # Pass 2 (collect mapping, skip duplicates)
- else:
- if sceneMenuStatLabel and originalString not in _pending_statlabel:
- # Grab and Pop
- translatedText = sceneMenuStatLabel[0]
- sceneMenuStatLabel.pop(0)
-
- # Queue for deferred replacement
- _pending_statlabel[originalString] = translatedText
-
- # Apply deferred replacements longest-first to avoid substring collisions
- if setData and _pending_statlabel:
- for orig in sorted(_pending_statlabel, key=len, reverse=True):
- data[i] = data[i].replace(orig, _pending_statlabel[orig])
- _pending_statlabel.clear()
- saveCheckLines(data, filename)
-
- # DrawTextEx - Game variable counter word (hardcoded: 回 → time(s))
- if setData and 'drawTextEx' in data[i] and '$gameVariables' in data[i]:
- data[i] = re.sub(
- r'(\$\{\$gameVariables\.value\(\d+\)\}\s*)回',
- r'\1x',
- data[i]
- )
- saveCheckLines(data, filename)
-
- # Next Line
- i += 1
-
- # EOF
- translate = False
- questListTL = [[], [], [], [], [], []]
- customTL = []
-
- # Compute combined total for progress bar across all categories
- combinedTotal = (
- sum(len(quest) for quest in questList)
- + len(custom)
- + len(sceneMenuText)
- + len(sceneMenuCommonHelpText)
- + len(sceneMenuHelpText)
- + len(sceneMenuDrawText)
- + len(sceneMenuStatLabel)
- + len(saveParamName)
- )
- if combinedTotal > 0:
- pbar.total = combinedTotal
- pbar.refresh()
-
- # Quest
- if len(questList) > 0:
- # Quest Name
- response = translateAI(questList[0], "Quest Name")
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- questName = response[0]
-
- # Quest Client
- response = translateAI(questList[1], "Quest Client")
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- questClient = response[0]
-
- # Quest Location
- response = translateAI(questList[2], "Quest Location")
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- questLocation = response[0]
-
- # Quest Target
- response = translateAI(questList[3], "Quest Location")
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- questTarget = response[0]
-
- # Quest Summary
- response = translateAI(questList[4], "Quest Summary")
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- questSummary = response[0]
-
- # Quest Goal 1
- response = translateAI(questList[5], "Quest Goal")
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- questGoal1 = response[0]
-
- # Check Mismatch
- if (
- len(questName) == len(questList[0])
- or len(questClient) == len(questList[1])
- or len(questLocation) == len(questList[2])
- or len(questTarget) == len(questList[3])
- or len(questSummary) == len(questList[4])
- or len(questGoal1) == len(questList[5])
- ):
- # Set Strings
- questListTL = [questName, questClient, questLocation, questTarget, questSummary, questGoal1]
- translate = True
-
- # Mismatch
- else:
- with LOCK:
- if filename not in MISMATCH:
- MISMATCH.append(filename)
-
- # Custom
- if custom:
- # TL
- response = translateAI(custom, "Relic Name")
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- customResponse = response[0]
-
- # Check Mismatch
- if len(custom) == len(customResponse):
- customTL = customResponse
- translate = True
-
- # Mismatch
- else:
- with LOCK:
- if filename not in MISMATCH:
- MISMATCH.append(filename)
-
- # SceneCustomMenu Text
- sceneMenuTextTL = []
- sceneMenuCommonHelpTextTL = []
- sceneMenuHelpTextTL = []
- sceneMenuDrawTextTL = []
- sceneMenuStatLabelTL = []
- saveParamNameTL = []
-
- if sceneMenuText:
- # TL
- response = translateAI(sceneMenuText, "Menu Item")
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- sceneMenuTextResponse = response[0]
-
- # Check Mismatch
- if len(sceneMenuText) == len(sceneMenuTextResponse):
- sceneMenuTextTL = sceneMenuTextResponse
- translate = True
-
- # Mismatch
- else:
- with LOCK:
- if filename not in MISMATCH:
- MISMATCH.append(filename)
-
- # SceneCustomMenu CommonHelpText
- if sceneMenuCommonHelpText:
- # TL
- response = translateAI(sceneMenuCommonHelpText, "Menu Help Text")
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- sceneMenuCommonHelpTextResponse = response[0]
-
- # Check Mismatch
- if len(sceneMenuCommonHelpText) == len(sceneMenuCommonHelpTextResponse):
- sceneMenuCommonHelpTextTL = sceneMenuCommonHelpTextResponse
- translate = True
-
- # Mismatch
- else:
- with LOCK:
- if filename not in MISMATCH:
- MISMATCH.append(filename)
-
- # SceneCustomMenu HelpText
- if sceneMenuHelpText:
- # TL
- response = translateAI(sceneMenuHelpText, "Menu Help Text")
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- sceneMenuHelpTextResponse = response[0]
-
- # Check Mismatch
- if len(sceneMenuHelpText) == len(sceneMenuHelpTextResponse):
- sceneMenuHelpTextTL = sceneMenuHelpTextResponse
- translate = True
-
- # Mismatch
- else:
- with LOCK:
- if filename not in MISMATCH:
- MISMATCH.append(filename)
-
- # DrawTextEx - Brace-delimited labels (tracker section headers and field labels)
- if sceneMenuDrawText:
- # TL
- response = translateAI(sceneMenuDrawText, "Status Tracker Label")
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- sceneMenuDrawTextResponse = response[0]
-
- # Check Mismatch
- if len(sceneMenuDrawText) == len(sceneMenuDrawTextResponse):
- sceneMenuDrawTextTL = sceneMenuDrawTextResponse
- translate = True
-
- # Mismatch
- else:
- with LOCK:
- if filename not in MISMATCH:
- MISMATCH.append(filename)
-
- # DrawTextEx - Stat labels (e.g., 攻撃力, 魔力, etc.)
- if sceneMenuStatLabel:
- # TL
- response = translateAI(sceneMenuStatLabel, "Character Stat Name")
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- sceneMenuStatLabelResponse = response[0]
-
- # Check Mismatch
- if len(sceneMenuStatLabel) == len(sceneMenuStatLabelResponse):
- sceneMenuStatLabelTL = sceneMenuStatLabelResponse
- translate = True
-
- # Mismatch
- else:
- with LOCK:
- if filename not in MISMATCH:
- MISMATCH.append(filename)
-
- # NUUN_SaveScreen ParamName
- if saveParamName:
- # TL
- response = translateAI(saveParamName, "Save Screen Label")
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- saveParamNameResponse = response[0]
-
- # Check Mismatch
- if len(saveParamName) == len(saveParamNameResponse):
- saveParamNameTL = saveParamNameResponse
- translate = True
-
- # Mismatch
- else:
- with LOCK:
- if filename not in MISMATCH:
- MISMATCH.append(filename)
-
- # Pass 2
- if translate and not setData:
- translatePlugin(data, pbar, filename, [questListTL, customTL, sceneMenuTextTL, sceneMenuCommonHelpTextTL, sceneMenuHelpTextTL, sceneMenuDrawTextTL, sceneMenuStatLabelTL, saveParamNameTL])
- return tokens
-
-# Save some money and enter the character before translation
-def getSpeaker(speaker):
- match speaker:
- case "ファイン":
- return ["Fine", [0, 0]]
- case "":
- return ["", [0, 0]]
- case _:
- # Find Speaker
- for i in range(len(NAMESLIST)):
- if speaker == NAMESLIST[i][0]:
- return [NAMESLIST[i][1], [0, 0]]
-
- # Translate and Store Speaker
- response = translateAI(
- f"{speaker}",
- "Reply with the " + LANGUAGE + " translation of the NPC name.",
- False,
- )
- response[0] = response[0].title()
- response[0] = response[0].replace("'S", "'s")
- response[0] = response[0].replace("Speaker: ", "")
-
- # Retry if name doesn't translate for some reason
- if re.search(r"([a-zA-Z？?])", response[0]) == None:
- response = translateAI(
- f"{speaker}",
- "Reply with the " + LANGUAGE + " translation of the NPC name.",
- False,
- )
- response[0] = response[0].title()
- response[0] = response[0].replace("'S", "'s")
-
- speakerList = [speaker, response[0]]
- NAMESLIST.append(speakerList)
- return response
- return [speaker, [0, 0]]
-
-def translateAI(text, history, history_ctx=None):
- """
- Legacy wrapper function for the new shared translation utility.
- This maintains compatibility with existing code while using the new shared implementation.
- """
- global PBAR, MISMATCH, FILENAME
-
- # Update config estimate mode based on global ESTIMATE
- TRANSLATION_CONFIG.estimateMode = bool(ESTIMATE)
-
- # Call the new shared translation function
- return sharedtranslateAI(
- text=text,
- history=history,
- config=TRANSLATION_CONFIG,
- filename=FILENAME,
- pbar=PBAR,
- lock=LOCK,
- mismatchList=MISMATCH
- )
+# Libraries
+import json
+import os
+import re
+import util.dazedwrap as dazedwrap
+import threading
+import time
+import traceback
+import tiktoken
+from pathlib import Path
+from colorama import Fore
+from dotenv import load_dotenv
+from retry import retry
+from tqdm import tqdm
+from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost
+import tempfile
+
+# Globals
+MODEL = os.getenv("model")
+TIMEOUT = int(os.getenv("timeout"))
+LANGUAGE = os.getenv("language").capitalize()
+from util.paths import PROMPT_PATH, VOCAB_PATH
+
+PROMPT = PROMPT_PATH.read_text(encoding="utf-8")
+VOCAB = VOCAB_PATH.read_text(encoding="utf-8")
+LOCK = threading.Lock()
+WIDTH = int(os.getenv("width"))
+LISTWIDTH = int(os.getenv("listWidth"))
+NOTEWIDTH = 70
+MAXHISTORY = 10
+ESTIMATE = ""
+TOKENS = [0, 0]
+NAMESLIST = []
+NAMES = False # Output a list of all the character names found
+BRFLAG = False # If the game uses
instead
+FIXTEXTWRAP = True # Overwrites textwrap
+IGNORETLTEXT = False # Ignores all translated text.
+MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
+FILENAME = None
+
+# tqdm Globals
+BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
+POSITION = 0
+LEAVE = False
+PBAR = None
+
+# Regex - Need to change this if you want to translate from/to other languages. Default is Japanese Regex
+LANGREGEX = r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+"
+
+# Get pricing configuration based on the model
+PRICING_CONFIG = getPricingConfig(MODEL)
+INPUTAPICOST = PRICING_CONFIG["inputAPICost"]
+OUTPUTAPICOST = PRICING_CONFIG["outputAPICost"]
+BATCHSIZE = PRICING_CONFIG["batchSize"]
+FREQUENCY_PENALTY = PRICING_CONFIG["frequencyPenalty"]
+
+# Initialize Translation Config
+TRANSLATION_CONFIG = TranslationConfig(
+ model=MODEL,
+ language=LANGUAGE,
+ prompt=PROMPT,
+ vocab=VOCAB,
+ langRegex=LANGREGEX,
+ batchSize=BATCHSIZE,
+ maxHistory=MAXHISTORY,
+ estimateMode=False # Will be set dynamically based on ESTIMATE
+)
+LEAVE = False
+
+def handlePlugin(filename, estimate):
+ global ESTIMATE, PBAR, FILENAME
+ ESTIMATE = estimate
+ FILENAME = filename
+
+ if ESTIMATE:
+ start = time.time()
+ translatedData = openFiles(filename)
+
+ # Print Result
+ end = time.time()
+ tqdm.write(getResultString(translatedData, end - start, filename))
+ with LOCK:
+ TOKENS[0] += translatedData[1][0]
+ TOKENS[1] += translatedData[1][1]
+
+ # Print Total
+ totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
+
+ # Print any errors on maps
+ if len(MISMATCH) > 0:
+ return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
+ else:
+ return totalString
+
+ else:
+ try:
+ # Perform translation first; incremental progress writes happen during translation.
+ # We no longer keep the destination file open simultaneously to avoid Windows locking issues
+ # during atomic os.replace operations inside save_progress_lines.
+ start = time.time()
+ translatedData = openFiles(filename)
+
+ # Ensure final state is flushed (in case no incremental writes occurred for some reason).
+ save_progress_lines(translatedData[0], filename)
+
+ # Print Result
+ end = time.time()
+ tqdm.write(getResultString(translatedData, end - start, filename))
+ with LOCK:
+ TOKENS[0] += translatedData[1][0]
+ TOKENS[1] += translatedData[1][1]
+ except Exception:
+ traceback.print_exc()
+ return "Fail"
+
+ return getResultString(["", TOKENS, None], end - start, "TOTAL")
+
+
+def getResultString(translatedData, translationTime, filename):
+ # File Print String
+ cost = calculateCost(translatedData[1][0], translatedData[1][1], MODEL)
+ totalTokenstring = (
+ Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
+ "[Output: "
+ + str(translatedData[1][1])
+ + "]" "[Cost: ${:,.4f}".format(cost)
+ + "]"
+ )
+ timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
+
+ if translatedData[2] == None:
+ # Success
+ return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET
+
+ else:
+ # Fail
+ try:
+ raise translatedData[2]
+ except Exception as e:
+ traceback.print_exc()
+ errorString = str(e) + Fore.RED
+ return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET
+
+
+def openFiles(filename):
+ with open("files/" + filename, "r", encoding="utf_8") as readFile:
+ translatedData = parsePlugin(readFile, filename)
+
+ # Delete lines marked for deletion
+ finalData = []
+ for line in translatedData[0]:
+ if line != "\\d\n":
+ finalData.append(line)
+ translatedData[0] = finalData
+
+ return translatedData
+
+
+def parsePlugin(readFile, filename):
+ global PBAR
+ totalTokens = [0, 0]
+
+ # Read File into data
+ data = readFile.readlines()
+
+ # Create Progress Bar
+ with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
+ pbar.desc = filename
+ PBAR = pbar
+
+ try:
+ result = translatePlugin(data, pbar, filename, [])
+ totalTokens[0] += result[0]
+ totalTokens[1] += result[1]
+ except Exception as e:
+ traceback.print_exc()
+ return [data, totalTokens, e]
+ return [data, totalTokens, None]
+
+
+def save_progress_lines(lines, filename, encoding="utf_8"):
+ """Atomically save current line-based translation progress."""
+ try:
+ if ESTIMATE:
+ return
+ global LOCK
+ os.makedirs("translated", exist_ok=True)
+ # Use a single lock to prevent concurrent replace attempts on Windows (which causes PermissionError)
+ with LOCK:
+ tmp_fd, tmp_path = tempfile.mkstemp(prefix=f"{filename}.", suffix=".tmp", dir="translated")
+ try:
+ with os.fdopen(tmp_fd, "w", encoding=encoding, newline="\n", errors="ignore") as tmp_file:
+ tmp_file.writelines(lines)
+
+ dest_path = os.path.join("translated", filename)
+ # Retry a few times in case another process/thread still has the file open momentarily.
+ for attempt in range(5):
+ try:
+ os.replace(tmp_path, dest_path)
+ break
+ except PermissionError:
+ if attempt == 4:
+ raise
+ time.sleep(0.1 * (attempt + 1))
+ finally:
+ if os.path.exists(tmp_path):
+ try:
+ os.remove(tmp_path)
+ except OSError:
+ pass
+ except Exception:
+ traceback.print_exc()
+
+
+def saveCheckLines(lines, filename, tokens=None, encoding="utf_8"):
+ """Save progress only when tokens indicate work or when explicitly called after a mutation."""
+ try:
+ if tokens is not None:
+ if not (isinstance(tokens, (list, tuple)) and len(tokens) >= 2 and (tokens[0] or tokens[1])):
+ return
+ save_progress_lines(lines, filename, encoding=encoding)
+ except Exception:
+ traceback.print_exc()
+
+
+def translatePlugin(data, pbar, filename, translatedList):
+ if len(translatedList) > 0:
+ questList = translatedList[0]
+ custom = translatedList[1]
+ sceneMenuText = translatedList[2] if len(translatedList) > 2 else []
+ sceneMenuCommonHelpText = translatedList[3] if len(translatedList) > 3 else []
+ sceneMenuHelpText = translatedList[4] if len(translatedList) > 4 else []
+ sceneMenuDrawText = translatedList[5] if len(translatedList) > 5 else []
+ sceneMenuStatLabel = translatedList[6] if len(translatedList) > 6 else []
+ saveParamName = translatedList[7] if len(translatedList) > 7 else []
+ setData = True
+ else:
+ questList = [[], [], [], [], [], []]
+ custom = []
+ sceneMenuText = []
+ sceneMenuCommonHelpText = []
+ sceneMenuHelpText = []
+ sceneMenuDrawText = []
+ sceneMenuStatLabel = []
+ saveParamName = []
+ setData = False
+ currentGroup = []
+ tokens = [0, 0]
+ speaker = ""
+ voice = False
+ global LOCK, ESTIMATE
+ i = 0
+ in_disp_list = False
+ brace_count = 0
+ _pending_drawtext = {} # Deferred drawTextEx brace label replacements (longest-first)
+ _pending_statlabel = {} # Deferred drawTextEx stat label replacements (longest-first)
+
+ # Category
+ with open("log/translations.txt", "a+", encoding="utf-8") as tlFile:
+ tlFile.write(f"\nCustom:\n")
+ tlFile.close()
+
+ while i < len(data):
+ voice = False
+ speaker = ""
+
+ # Track if we're inside CBR_travel_data block
+ if 'data_map_name_list' in data[i]:
+ in_disp_list = True
+ brace_count = 0
+
+ if in_disp_list:
+ # Count braces to know when we exit the object
+ brace_count += data[i].count('{') - data[i].count('}')
+ if brace_count < 0:
+ in_disp_list = False
+
+ # Nested array strings (e.g., this.disp_list = { key: { subkey: ["string1", "string2"] } })
+ # Matches strings inside arrays within object literals
+ if in_disp_list:
+ regex_nested = r'"([^"]*[一-龠ぁ-ゔァ-ヴー]+[^"]*)"'
+ matchList = re.findall(regex_nested, data[i])
+ if len(matchList) > 0:
+ for match in matchList:
+ # Save Original String
+ jaString = match
+ originalString = jaString
+
+ # Replace \n for translation
+ jaStringClean = jaString.replace("\\n", " ")
+
+ if jaStringClean.strip():
+ # Pass 1
+ if setData == False:
+ custom.append(jaStringClean.strip())
+
+ # Pass 2
+ else:
+ if custom:
+ # Grab and Pop
+ translatedText = custom[0]
+ custom.pop(0)
+
+ # Set to None if empty list
+ if len(custom) <= 0:
+ custom = []
+
+ # Restore original newlines but keep translated text
+ # Count original newlines
+ newline_count = jaString.count("\\n")
+ if newline_count > 0:
+ # Textwrap the translation
+ translatedText = dazedwrap.wrapText(translatedText, 50)
+ translatedText = translatedText.replace("\n", "\\n")
+
+ # Escape quotes in translation
+ translatedText = translatedText.replace('"', '\\"')
+
+ # Set Data
+ data[i] = data[i].replace(f'"{originalString}"', f'"{translatedText}"')
+ saveCheckLines(data, filename)
+
+ # Custom
+ # Useful Regex's
+ # r'"Text[\\]+":[\\]+"(.+?)[\\]+",'
+ # r'"HelpText[\\]+":[\\]+"(.+?)[\\]+",'
+ # r"this.drawTextEx\(\\'(.+?)\',"
+ # r'txtSubject.+?"(.+)"'
+ regex = r'txtSubject.+?"(.+)"'
+ matchList = re.findall(regex, data[i])
+ if len(matchList) > 0:
+ for match in matchList:
+ # Save Original String
+ jaString = match
+ originalString = jaString
+ newline = None
+ colorCode = None
+
+ # Make sure it contains Japanese
+ if not re.search(LANGREGEX, jaString):
+ continue
+
+ # Make sure didn't grab \\
+ if re.search(r"^[\\]+$", jaString):
+ continue
+
+ # Replace \n and \c
+ if re.search(r"\\+n", jaString):
+ newline = re.search(r"\\+n", jaString).group(0)
+ jaString = re.sub(r"\\+n", r"\\n", jaString)
+ if re.search(r"\\+C", jaString):
+ colorCode = re.search(r"\\+C", jaString).group(0)
+ jaString = re.sub(r"\\+C", r"\\C", jaString)
+
+ # Remove any textwrap
+ jaString = jaString.replace("\\n", " ")
+
+ if jaString.replace("\u3000", "") and jaString:
+ # Pass 1
+ if setData == False and jaString.strip():
+ custom.append(jaString.strip())
+
+ # Pass 2
+ else:
+ if custom:
+ # Grab and Pop
+ translatedText = custom[0]
+ custom.pop(0)
+
+ # Set to None if empty list
+ if len(translatedList) <= 0:
+ translatedList = None
+
+ # Replace Single Quotes
+ translatedText = re.sub(r"([^\\'])'", r"\1՚", translatedText)
+ translatedText = re.sub(r"([^\\'])\"", r"\1՚", translatedText)
+
+ # Replace \n and \c
+ if newline:
+ # Textwrap
+ # translatedText = dazedwrap.wrapText(translatedText, WIDTH)
+ translatedText = re.sub(r"\n", re.escape(newline), translatedText)
+ if colorCode:
+ translatedText = re.sub(r"\n", re.escape(colorCode), translatedText)
+
+ # Set Data
+ with open("log/translations.txt", "a+", encoding="utf-8") as tlFile:
+ tlFile.write(f"{originalString} ({translatedText})\n")
+ data[i] = data[i].replace(originalString, translatedText)
+ saveCheckLines(data, filename)
+
+ # Quest Name
+ regex = r'[\\]+"QuestName[\\]+":[\\]+"(.*?)[\\]+"'
+ matchList = re.findall(regex, data[i])
+ if len(matchList) > 0:
+ for match in matchList:
+ if match:
+ # Save Original String
+ originalString = match
+
+ # Remove any textwrap
+ match = match.replace(newline, " ")
+
+ # Pass 1
+ if setData == False:
+ # Add String
+ if match != "\\\\\\\\":
+ questList[0].append(match.strip())
+
+ # Pass 2
+ else:
+ if questList[0]:
+ # Grab and Pop
+ translatedText = questList[0][0]
+ questList[0].pop(0)
+
+ # Set to None if empty list
+ if len(translatedList) <= 0:
+ translatedList = None
+
+ # Replace Single Quotes
+ translatedText = translatedText.replace('"', "'")
+ translatedText = re.sub(r"([^\\'])'", r"\1\\'", translatedText)
+
+ # Set Data
+ data[i] = data[i].replace(originalString, translatedText)
+ saveCheckLines(data, filename)
+
+ # Quest Client
+ regex = r'QuestClientName[\\]+":[\\]+"(.*?)[\\]+"'
+ matchList = re.findall(regex, data[i])
+ if len(matchList) > 0:
+ for match in matchList:
+ if match:
+ # Save Original String
+ originalString = match
+
+ # Pass 1
+ if setData == False:
+ # Add String
+ if match != "\\\\\\\\":
+ questList[1].append(match.strip())
+
+ # Pass 2
+ else:
+ if questList[1]:
+ # Grab and Pop
+ translatedText = questList[1][0]
+ questList[1].pop(0)
+
+ # Set to None if empty list
+ if len(translatedList) <= 0:
+ translatedList = None
+
+ # Replace Single Quotes
+ translatedText = translatedText.replace('"', "'")
+ translatedText = re.sub(r"([^\\'])'", r"\1\\'", translatedText)
+
+ # Set Data
+ data[i] = data[i].replace(originalString, translatedText)
+ saveCheckLines(data, filename)
+
+ # Quest Location
+ regex = r'QuestLocation[\\]+":[\\]+"(.*?)[\\]+"'
+ matchList = re.findall(regex, data[i])
+ if len(matchList) > 0:
+ for match in matchList:
+ if match:
+ # Save Original String
+ originalString = match
+
+ # Pass 1
+ if setData == False:
+ # Add String
+ if match != "\\\\\\\\":
+ questList[2].append(match.strip())
+
+ # Pass 2
+ else:
+ if questList[2]:
+ # Grab and Pop
+ translatedText = questList[2][0]
+ questList[2].pop(0)
+
+ # Set to None if empty list
+ if len(translatedList) <= 0:
+ translatedList = None
+
+ # Replace Single Quotes
+ translatedText = translatedText.replace('"', "'")
+ translatedText = re.sub(r"([^\\'])'", r"\1\\'", translatedText)
+
+ # Set Data
+ data[i] = data[i].replace(originalString, translatedText)
+ saveCheckLines(data, filename)
+
+ # Quest Target
+ regex = r'PlaceInformation[\\]+":[\\]+"(.*?)[\\]+"'
+ matchList = re.findall(regex, data[i])
+ if len(matchList) > 0:
+ for match in matchList:
+ if match:
+ # Save Original String
+ originalString = match
+
+ # Pass 1
+ if setData == False:
+ # Add String
+ if match != "\\\\\\\\":
+ questList[3].append(match.strip())
+
+ # Pass 2
+ else:
+ if questList[3]:
+ # Grab and Pop
+ translatedText = questList[3][0]
+ questList[3].pop(0)
+
+ # Set to None if empty list
+ if len(translatedList) <= 0:
+ translatedList = None
+
+ # Replace Single Quotes
+ translatedText = translatedText.replace('"', "'")
+ translatedText = re.sub(r"([^\\'])'", r"\1\\'", translatedText)
+
+ # Set Data
+ data[i] = data[i].replace(originalString, translatedText)
+ saveCheckLines(data, filename)
+
+ # Quest Summary
+ regex = r'[\\]+"QuestContent[\\]+":[\\]+"[\\]+"(.*?)[\\]+"[\\]+"'
+ matchList = re.findall(regex, data[i])
+ if len(matchList) > 0:
+ for match in matchList:
+ if match:
+ # Save Original String
+ originalString = match
+
+ # Remove any textwrap
+ match = match.replace(r"\\\\\\\\n", " ")
+ match = match.replace(r"\\\\\\\\\\\\\\\\c", "\\c")
+
+ # Pass 1
+ if setData == False:
+ # Add String
+ if match != "\\\\\\\\":
+ questList[4].append(match.strip())
+
+ # Pass 2
+ else:
+ if questList[4]:
+ # Grab and Pop
+ translatedText = questList[4][0]
+ questList[4].pop(0)
+
+ # Set to None if empty list
+ if len(translatedList) <= 0:
+ translatedList = None
+
+ # Textwrap
+ translatedText = dazedwrap.wrapText(translatedText, width=WIDTH)
+ translatedText = translatedText.replace("\n", r"\\\\\\\\n")
+ match = match.replace("\\c", r"\\\\\\\\\\\\\\\\c")
+
+ # Replace Single Quotes
+ translatedText = translatedText.replace('"', "'")
+ translatedText = re.sub(r"([^\\'])'", r"\1\\'", translatedText)
+
+ # Set Data
+ data[i] = data[i].replace(originalString, translatedText)
+ saveCheckLines(data, filename)
+
+ # Quest Goal 1
+ regex = r'ObjectiveContent[\\]+":[\\]+"[\\]+"(.*?)[\\]+"'
+ matchList = re.findall(regex, data[i])
+ if len(matchList) > 0:
+ for match in matchList:
+ if match:
+ # Save Original String
+ originalString = match
+
+ # Remove any textwrap
+ match = match.replace(r"\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\n", " ")
+
+ # Pass 1
+ if setData == False:
+ # Add String
+ if match != "\\\\\\\\":
+ questList[5].append(match.strip())
+
+ # Pass 2
+ else:
+ if questList[5]:
+ # Grab and Pop
+ translatedText = questList[5][0]
+ questList[5].pop(0)
+
+ # Set to None if empty list
+ if len(translatedList) <= 0:
+ translatedList = None
+
+ # Textwrap
+ translatedText = dazedwrap.wrapText(translatedText, width=WIDTH)
+ translatedText = translatedText.replace("\n", r"\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\n")
+
+ # Replace Single Quotes
+ translatedText = translatedText.replace('"', "'")
+ translatedText = re.sub(r"([^\\'])'", r"\1\\'", translatedText)
+
+ # Set Data
+ data[i] = data[i].replace(originalString, translatedText)
+ saveCheckLines(data, filename)
+
+ # SceneCustomMenu - Text (Command Labels)
+ regex = r'[\\]+"Text[\\]+":[\\]+"(.+?)[\\]+"'
+ matchList = re.findall(regex, data[i])
+ if len(matchList) > 0:
+ for match in matchList:
+ if match:
+ jaString = match
+ originalString = jaString
+
+ # Make sure it contains Japanese
+ if not re.search(LANGREGEX, jaString):
+ continue
+
+ # Make sure didn't grab only backslashes
+ if re.search(r"^[\\]+$", jaString):
+ continue
+
+ if jaString.replace("\u3000", "").strip():
+ # Pass 1
+ if setData == False:
+ sceneMenuText.append(jaString.strip())
+
+ # Pass 2
+ else:
+ if sceneMenuText:
+ # Grab and Pop
+ translatedText = sceneMenuText[0]
+ sceneMenuText.pop(0)
+
+ # Replace quotes
+ translatedText = translatedText.replace('"', "'")
+ translatedText = re.sub(r"([^\\'])'" , r"\1\\'", translatedText)
+
+ # Set Data
+ data[i] = data[i].replace(originalString, translatedText)
+ saveCheckLines(data, filename)
+
+ # SceneCustomMenu - CommonHelpText
+ regex = r'[\\]+"CommonHelpText[\\]+":[\\]+"(.+?)[\\]+"'
+ matchList = re.findall(regex, data[i])
+ if len(matchList) > 0:
+ for match in matchList:
+ if match:
+ jaString = match
+ originalString = jaString
+
+ # Make sure it contains Japanese
+ if not re.search(LANGREGEX, jaString):
+ continue
+
+ # Make sure didn't grab only backslashes
+ if re.search(r"^[\\]+$", jaString):
+ continue
+
+ if jaString.replace("\u3000", "").strip():
+ # Pass 1
+ if setData == False:
+ sceneMenuCommonHelpText.append(jaString.strip())
+
+ # Pass 2
+ else:
+ if sceneMenuCommonHelpText:
+ # Grab and Pop
+ translatedText = sceneMenuCommonHelpText[0]
+ sceneMenuCommonHelpText.pop(0)
+
+ # Replace quotes
+ translatedText = translatedText.replace('"', "'")
+ translatedText = re.sub(r"([^\\'])'" , r"\1\\'", translatedText)
+
+ # Set Data
+ data[i] = data[i].replace(originalString, translatedText)
+ saveCheckLines(data, filename)
+
+ # SceneCustomMenu - HelpText (Command Help)
+ regex = r'[\\]+"HelpText[\\]+":[\\]+"(.+?)[\\]+"'
+ matchList = re.findall(regex, data[i])
+ if len(matchList) > 0:
+ for match in matchList:
+ if match:
+ jaString = match
+ originalString = jaString
+
+ # Make sure it contains Japanese
+ if not re.search(LANGREGEX, jaString):
+ continue
+
+ # Make sure didn't grab only backslashes
+ if re.search(r"^[\\]+$", jaString):
+ continue
+
+ if jaString.replace("\u3000", "").strip():
+ # Pass 1
+ if setData == False:
+ sceneMenuHelpText.append(jaString.strip())
+
+ # Pass 2
+ else:
+ if sceneMenuHelpText:
+ # Grab and Pop
+ translatedText = sceneMenuHelpText[0]
+ sceneMenuHelpText.pop(0)
+
+ # Replace quotes
+ translatedText = translatedText.replace('"', "'")
+ translatedText = re.sub(r"([^\\'])'" , r"\1\\'", translatedText)
+
+ # Set Data
+ data[i] = data[i].replace(originalString, translatedText)
+ saveCheckLines(data, filename)
+
+ # NUUN_SaveScreen - ParamName (save screen labels like 現在地, プレイ時間, 所持金)
+ regex = r'[\\]+"ParamName[\\]+":[\\]+"(.+?)[\\]+"'
+ matchList = re.findall(regex, data[i])
+ if len(matchList) > 0:
+ for match in matchList:
+ if match:
+ jaString = match
+ originalString = jaString
+
+ # Make sure it contains Japanese
+ if not re.search(LANGREGEX, jaString):
+ continue
+
+ # Make sure didn't grab only backslashes
+ if re.search(r"^[\\]+$", jaString):
+ continue
+
+ if jaString.replace("\u3000", "").strip():
+ # Pass 1
+ if setData == False:
+ saveParamName.append(jaString.strip())
+
+ # Pass 2
+ else:
+ if saveParamName:
+ # Grab and Pop
+ translatedText = saveParamName[0]
+ saveParamName.pop(0)
+
+ # Replace quotes
+ translatedText = translatedText.replace('"', "'")
+ translatedText = re.sub(r"([^\\'])'" , r"\1\\'", translatedText)
+
+ # Set Data
+ data[i] = data[i].replace(originalString, translatedText)
+ saveCheckLines(data, filename)
+
+ # DrawTextEx - Brace-delimited labels (e.g., \}【口づけ】\{, \}回数：\{)
+ # Only process lines containing drawTextEx calls
+ if 'drawTextEx' in data[i]:
+ regex = r'[\\]+\}(.+?)[\\]+\{'
+ matchList = re.findall(regex, data[i])
+ if len(matchList) > 0:
+ for match in matchList:
+ if match:
+ jaString = match
+ originalString = jaString
+
+ # Skip game variable references (handled separately)
+ if '$gameVariables' in jaString:
+ continue
+
+ # Make sure it contains Japanese
+ if not re.search(LANGREGEX, jaString):
+ continue
+
+ if jaString.replace("\u3000", "").strip():
+ # Pass 1 (deduplicated)
+ if setData == False:
+ if jaString.strip() not in sceneMenuDrawText:
+ sceneMenuDrawText.append(jaString.strip())
+
+ # Pass 2 (collect mapping, skip duplicates)
+ else:
+ if sceneMenuDrawText and originalString not in _pending_drawtext:
+ # Grab and Pop
+ translatedText = sceneMenuDrawText[0]
+ sceneMenuDrawText.pop(0)
+
+ # Replace quotes
+ translatedText = translatedText.replace('"', "'")
+
+ # Queue for deferred replacement
+ _pending_drawtext[originalString] = translatedText
+
+ # Apply deferred replacements longest-first to avoid substring collisions
+ # (e.g., replacing 回数： before 使用回数： would corrupt the longer string)
+ if setData and _pending_drawtext:
+ for orig in sorted(_pending_drawtext, key=len, reverse=True):
+ data[i] = data[i].replace(orig, _pending_drawtext[orig])
+ _pending_drawtext.clear()
+ saveCheckLines(data, filename)
+
+ # DrawTextEx - Stat labels (e.g., C[16]攻撃力\\...C[0])
+ if 'drawTextEx' in data[i]:
+ regex = r'C\[\d+\]([^C\\,`\[\]]+?)[\\]+C\[0\]'
+ matchList = re.findall(regex, data[i])
+ if len(matchList) > 0:
+ for match in matchList:
+ if match:
+ jaString = match
+ originalString = jaString
+
+ # Make sure it contains Japanese
+ if not re.search(LANGREGEX, jaString):
+ continue
+
+ if jaString.replace("\u3000", "").strip():
+ # Pass 1 (deduplicated)
+ if setData == False:
+ if jaString.strip() not in sceneMenuStatLabel:
+ sceneMenuStatLabel.append(jaString.strip())
+
+ # Pass 2 (collect mapping, skip duplicates)
+ else:
+ if sceneMenuStatLabel and originalString not in _pending_statlabel:
+ # Grab and Pop
+ translatedText = sceneMenuStatLabel[0]
+ sceneMenuStatLabel.pop(0)
+
+ # Queue for deferred replacement
+ _pending_statlabel[originalString] = translatedText
+
+ # Apply deferred replacements longest-first to avoid substring collisions
+ if setData and _pending_statlabel:
+ for orig in sorted(_pending_statlabel, key=len, reverse=True):
+ data[i] = data[i].replace(orig, _pending_statlabel[orig])
+ _pending_statlabel.clear()
+ saveCheckLines(data, filename)
+
+ # DrawTextEx - Game variable counter word (hardcoded: 回 → time(s))
+ if setData and 'drawTextEx' in data[i] and '$gameVariables' in data[i]:
+ data[i] = re.sub(
+ r'(\$\{\$gameVariables\.value\(\d+\)\}\s*)回',
+ r'\1x',
+ data[i]
+ )
+ saveCheckLines(data, filename)
+
+ # Next Line
+ i += 1
+
+ # EOF
+ translate = False
+ questListTL = [[], [], [], [], [], []]
+ customTL = []
+
+ # Compute combined total for progress bar across all categories
+ combinedTotal = (
+ sum(len(quest) for quest in questList)
+ + len(custom)
+ + len(sceneMenuText)
+ + len(sceneMenuCommonHelpText)
+ + len(sceneMenuHelpText)
+ + len(sceneMenuDrawText)
+ + len(sceneMenuStatLabel)
+ + len(saveParamName)
+ )
+ if combinedTotal > 0:
+ pbar.total = combinedTotal
+ pbar.refresh()
+
+ # Quest
+ if len(questList) > 0:
+ # Quest Name
+ response = translateAI(questList[0], "Quest Name")
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ questName = response[0]
+
+ # Quest Client
+ response = translateAI(questList[1], "Quest Client")
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ questClient = response[0]
+
+ # Quest Location
+ response = translateAI(questList[2], "Quest Location")
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ questLocation = response[0]
+
+ # Quest Target
+ response = translateAI(questList[3], "Quest Location")
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ questTarget = response[0]
+
+ # Quest Summary
+ response = translateAI(questList[4], "Quest Summary")
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ questSummary = response[0]
+
+ # Quest Goal 1
+ response = translateAI(questList[5], "Quest Goal")
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ questGoal1 = response[0]
+
+ # Check Mismatch
+ if (
+ len(questName) == len(questList[0])
+ or len(questClient) == len(questList[1])
+ or len(questLocation) == len(questList[2])
+ or len(questTarget) == len(questList[3])
+ or len(questSummary) == len(questList[4])
+ or len(questGoal1) == len(questList[5])
+ ):
+ # Set Strings
+ questListTL = [questName, questClient, questLocation, questTarget, questSummary, questGoal1]
+ translate = True
+
+ # Mismatch
+ else:
+ with LOCK:
+ if filename not in MISMATCH:
+ MISMATCH.append(filename)
+
+ # Custom
+ if custom:
+ # TL
+ response = translateAI(custom, "Relic Name")
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ customResponse = response[0]
+
+ # Check Mismatch
+ if len(custom) == len(customResponse):
+ customTL = customResponse
+ translate = True
+
+ # Mismatch
+ else:
+ with LOCK:
+ if filename not in MISMATCH:
+ MISMATCH.append(filename)
+
+ # SceneCustomMenu Text
+ sceneMenuTextTL = []
+ sceneMenuCommonHelpTextTL = []
+ sceneMenuHelpTextTL = []
+ sceneMenuDrawTextTL = []
+ sceneMenuStatLabelTL = []
+ saveParamNameTL = []
+
+ if sceneMenuText:
+ # TL
+ response = translateAI(sceneMenuText, "Menu Item")
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ sceneMenuTextResponse = response[0]
+
+ # Check Mismatch
+ if len(sceneMenuText) == len(sceneMenuTextResponse):
+ sceneMenuTextTL = sceneMenuTextResponse
+ translate = True
+
+ # Mismatch
+ else:
+ with LOCK:
+ if filename not in MISMATCH:
+ MISMATCH.append(filename)
+
+ # SceneCustomMenu CommonHelpText
+ if sceneMenuCommonHelpText:
+ # TL
+ response = translateAI(sceneMenuCommonHelpText, "Menu Help Text")
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ sceneMenuCommonHelpTextResponse = response[0]
+
+ # Check Mismatch
+ if len(sceneMenuCommonHelpText) == len(sceneMenuCommonHelpTextResponse):
+ sceneMenuCommonHelpTextTL = sceneMenuCommonHelpTextResponse
+ translate = True
+
+ # Mismatch
+ else:
+ with LOCK:
+ if filename not in MISMATCH:
+ MISMATCH.append(filename)
+
+ # SceneCustomMenu HelpText
+ if sceneMenuHelpText:
+ # TL
+ response = translateAI(sceneMenuHelpText, "Menu Help Text")
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ sceneMenuHelpTextResponse = response[0]
+
+ # Check Mismatch
+ if len(sceneMenuHelpText) == len(sceneMenuHelpTextResponse):
+ sceneMenuHelpTextTL = sceneMenuHelpTextResponse
+ translate = True
+
+ # Mismatch
+ else:
+ with LOCK:
+ if filename not in MISMATCH:
+ MISMATCH.append(filename)
+
+ # DrawTextEx - Brace-delimited labels (tracker section headers and field labels)
+ if sceneMenuDrawText:
+ # TL
+ response = translateAI(sceneMenuDrawText, "Status Tracker Label")
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ sceneMenuDrawTextResponse = response[0]
+
+ # Check Mismatch
+ if len(sceneMenuDrawText) == len(sceneMenuDrawTextResponse):
+ sceneMenuDrawTextTL = sceneMenuDrawTextResponse
+ translate = True
+
+ # Mismatch
+ else:
+ with LOCK:
+ if filename not in MISMATCH:
+ MISMATCH.append(filename)
+
+ # DrawTextEx - Stat labels (e.g., 攻撃力, 魔力, etc.)
+ if sceneMenuStatLabel:
+ # TL
+ response = translateAI(sceneMenuStatLabel, "Character Stat Name")
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ sceneMenuStatLabelResponse = response[0]
+
+ # Check Mismatch
+ if len(sceneMenuStatLabel) == len(sceneMenuStatLabelResponse):
+ sceneMenuStatLabelTL = sceneMenuStatLabelResponse
+ translate = True
+
+ # Mismatch
+ else:
+ with LOCK:
+ if filename not in MISMATCH:
+ MISMATCH.append(filename)
+
+ # NUUN_SaveScreen ParamName
+ if saveParamName:
+ # TL
+ response = translateAI(saveParamName, "Save Screen Label")
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ saveParamNameResponse = response[0]
+
+ # Check Mismatch
+ if len(saveParamName) == len(saveParamNameResponse):
+ saveParamNameTL = saveParamNameResponse
+ translate = True
+
+ # Mismatch
+ else:
+ with LOCK:
+ if filename not in MISMATCH:
+ MISMATCH.append(filename)
+
+ # Pass 2
+ if translate and not setData:
+ translatePlugin(data, pbar, filename, [questListTL, customTL, sceneMenuTextTL, sceneMenuCommonHelpTextTL, sceneMenuHelpTextTL, sceneMenuDrawTextTL, sceneMenuStatLabelTL, saveParamNameTL])
+ return tokens
+
+# Save some money and enter the character before translation
+def getSpeaker(speaker):
+ match speaker:
+ case "ファイン":
+ return ["Fine", [0, 0]]
+ case "":
+ return ["", [0, 0]]
+ case _:
+ # Find Speaker
+ for i in range(len(NAMESLIST)):
+ if speaker == NAMESLIST[i][0]:
+ return [NAMESLIST[i][1], [0, 0]]
+
+ # Translate and Store Speaker
+ response = translateAI(
+ f"{speaker}",
+ "Reply with the " + LANGUAGE + " translation of the NPC name.",
+ False,
+ )
+ response[0] = response[0].title()
+ response[0] = response[0].replace("'S", "'s")
+ response[0] = response[0].replace("Speaker: ", "")
+
+ # Retry if name doesn't translate for some reason
+ if re.search(r"([a-zA-Z？?])", response[0]) == None:
+ response = translateAI(
+ f"{speaker}",
+ "Reply with the " + LANGUAGE + " translation of the NPC name.",
+ False,
+ )
+ response[0] = response[0].title()
+ response[0] = response[0].replace("'S", "'s")
+
+ speakerList = [speaker, response[0]]
+ NAMESLIST.append(speakerList)
+ return response
+ return [speaker, [0, 0]]
+
+def translateAI(text, history, history_ctx=None):
+ """
+ Legacy wrapper function for the new shared translation utility.
+ This maintains compatibility with existing code while using the new shared implementation.
+ """
+ global PBAR, MISMATCH, FILENAME
+
+ # Update config estimate mode based on global ESTIMATE
+ TRANSLATION_CONFIG.estimateMode = bool(ESTIMATE)
+
+ # Call the new shared translation function
+ return sharedtranslateAI(
+ text=text,
+ history=history,
+ config=TRANSLATION_CONFIG,
+ filename=FILENAME,
+ pbar=PBAR,
+ lock=LOCK,
+ mismatchList=MISMATCH
+ )
diff --git a/modules/srpg.py b/modules/srpg.py
index 9568574..f89e39f 100644
--- a/modules/srpg.py
+++ b/modules/srpg.py
@@ -1,2261 +1,2263 @@
-# Libraries
-import json
-import os
-import re
-import shutil
-import util.dazedwrap as dazedwrap
-import threading
-import time
-import traceback
-from dotenv import load_dotenv
-from pathlib import Path
-from colorama import Fore
-from dotenv import load_dotenv
-from retry import retry
-from tqdm import tqdm
-from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost
-
-# Globals
-MODEL = os.getenv("model")
-TIMEOUT = int(os.getenv("timeout"))
-LANGUAGE = os.getenv("language").capitalize()
-PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
-VOCAB = Path("vocab.txt").read_text(encoding="utf-8")
-LOCK = threading.Lock()
-VOCAB_LOCK = threading.Lock() # Dedicated lock for vocab.txt updates
-WIDTH = int(os.getenv("width"))
-LISTWIDTH = int(os.getenv("listWidth"))
-NOTEWIDTH = int(os.getenv("noteWidth"))
-MAXHISTORY = 10
-ESTIMATE = ""
-TOKENS = [0, 0]
-MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
-NAMESLIST = [] # List of speaker names and their translations
-PBAR = None
-FILENAME = None
-TIMETOTAL = 0 # Total Time Taken for all translations
-
-# Regex - Need to change this if you want to translate from/to other languages. Default is Japanese Regex
-LANGREGEX = r"[\u3000-\u303F\u3040-\u309F\u30A0-\u30FF\u31F0-\u31FF\u3400-\u4DBF\u4E00-\u9FFF\uF900-\uFAFF\uFF61-\uFF9F]+"
-
-# Get pricing configuration based on the model
-PRICING_CONFIG = getPricingConfig(MODEL)
-INPUTAPICOST = PRICING_CONFIG["inputAPICost"]
-OUTPUTAPICOST = PRICING_CONFIG["outputAPICost"]
-BATCHSIZE = PRICING_CONFIG["batchSize"]
-FREQUENCY_PENALTY = PRICING_CONFIG["frequencyPenalty"]
-
-# tqdm Globals
-BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
-POSITION = 0
-LEAVE = False
-
-# Initialize Translation Config
-TRANSLATION_CONFIG = TranslationConfig(
- model=MODEL,
- language=LANGUAGE,
- prompt=PROMPT,
- vocab=VOCAB,
- langRegex=LANGREGEX,
- batchSize=BATCHSIZE,
- maxHistory=MAXHISTORY,
- estimateMode=False # Will be set dynamically based on ESTIMATE
-)
-
-# Config (Default)
-FIXTEXTWRAP = True # Rewrap text to WIDTH
-IGNORETLTEXT = False # Skip Translated Text
-
-# List of file patterns that use parseGeneric
-# Add more patterns here as needed
-GENERIC_FILES = [
- "quests",
- "shops",
- "shoplayout",
- "bonuses",
- "base",
- "battleprep",
- "manage",
- "mapcommands",
- "title",
- "classes",
- "classesgroups",
- "classtypes",
- "races",
- "skills",
- "weapons",
- "states",
- "difficulties",
- "fonts",
- "fusionsettings",
- "transformations",
- "characters",
- "glossary",
- "npc",
- "screens",
- "strings",
- "originalterrains",
- "runtimeterrains",
- "archers",
- "fighters",
- "mages",
- "items",
-]
-
-# List of file patterns that use parseMap
-# Be specific to avoid catching non-map files like CommandLayout/mapcommands.json
-# SRPG Studio map files in this project follow the pattern Maps/map_XXX.json
-MAP_FILES = [
- "map_", # e.g., Maps/map_000.json
-]
-
-
-def update_vocab_section(category: str, pairs: list[tuple[str, str]]):
- """Update or insert a section in vocab.txt for the given category with provided pairs.
- Only writes when there's an actual translation (dst is non-empty and differs from src after normalization).
- - category: e.g., "Items", "Weapons", "Speakers", etc. Section header will be "# {category}".
- - pairs: list of (source, translated) strings. Duplicates by source are deduped (last wins).
- The existing section is replaced entirely; other sections are preserved.
- """
- try:
- vocab_path = Path("vocab.txt")
-
- # Helper: normalized comparison to detect no-op translations
- def _norm(s: str) -> str:
- if s is None:
- return ""
- # Collapse whitespace and case-fold; leave punctuation to avoid over-matching
- return re.sub(r"\s+", " ", str(s)).strip().casefold()
-
- # Filter and deduplicate by source term (last mapping wins)
- dedup: dict[str, str] = {}
- for src, dst in pairs:
- if not src:
- continue
- # Skip when no destination or no actual change
- if dst is None or _norm(dst) == "" or _norm(dst) == _norm(src):
- continue
- dedup[src] = dst
-
- # If nothing to add after filtering, skip touching the file
- if not dedup:
- return
-
- # Guard the read-modify-write with a dedicated lock to avoid races
- with VOCAB_LOCK:
- existing = vocab_path.read_text(encoding="utf-8") if vocab_path.exists() else ""
-
- lines = [f"{src} ({dst})" for src, dst in dedup.items()]
- # Always terminate a section with a blank line to separate from next header
- new_block = f"# {category}\n" + "\n".join(lines)
- if not new_block.endswith("\n\n"):
- if not new_block.endswith("\n"):
- new_block += "\n"
- new_block += "\n"
-
- # Regex to find the specific section starting at the header for this category
- # and ending right before the next header (any number of '#') or EOF.
- # - Handles headers like '#Category', '# Category', '## Category', etc.
- # - Uses non-greedy matching for the body to avoid spanning multiple sections.
- pattern = re.compile(
- rf"^[\t ]*#+\s*{re.escape(category)}\s*$\r?\n.*?(?=^[\t ]*#|\Z)",
- re.MULTILINE | re.DOTALL,
- )
- if pattern.search(existing):
- # Replace only the first matching section for this category.
- updated = pattern.sub(lambda m: new_block, existing, count=1)
- else:
- updated = existing
- if updated and not updated.endswith("\n\n"):
- # Ensure a blank line before appending new section if file not empty
- if not updated.endswith("\n"):
- updated += "\n"
- updated += "\n"
- updated += new_block
-
- # Avoid writing if nothing changed
- if updated == existing:
- return
- # Atomic write: write to unique temp and replace with retries on Windows
- tmp_path = vocab_path.with_suffix(vocab_path.suffix + f".{os.getpid()}.{threading.get_ident()}.tmp")
- tmp_path.write_text(updated, encoding="utf-8")
-
- attempts = 6
- delay = 0.1
- last_err = None
- for attempt in range(attempts):
- try:
- os.replace(tmp_path, vocab_path)
- last_err = None
- break
- except PermissionError as e:
- last_err = e
- # Try relaxing permissions then retry
- try:
- if vocab_path.exists():
- os.chmod(vocab_path, 0o666)
- except Exception:
- pass
- time.sleep(delay)
- delay = min(1.0, delay * 2)
- except Exception as e:
- last_err = e
- break
- if last_err is not None:
- try:
- shutil.move(str(tmp_path), str(vocab_path))
- except Exception:
- try:
- if tmp_path.exists():
- tmp_path.unlink(missing_ok=True)
- except Exception:
- pass
- raise last_err
- except Exception:
- traceback.print_exc()
-
-
-def handleSRPG(filename, estimate):
- """
- Main handler function for SRPG Studio files.
-
- Args:
- filename: Name of the file to translate
- estimate: Boolean indicating if this is an estimate run
-
- Returns:
- String with translation results or error message
- """
- global ESTIMATE, TOKENS, FILENAME, TIMETOTAL
- ESTIMATE = estimate
- FILENAME = filename
-
- # Translate
- start = time.time()
- translatedData = openFiles(filename)
-
- # Write output file if not in estimate mode
- if not estimate:
- try:
- with open("translated/" + filename, "w", encoding="utf-8", newline="\n") as outFile:
- json.dump(translatedData[0], outFile, ensure_ascii=False, indent=4)
- except Exception:
- traceback.print_exc()
- return "Fail"
-
- # Print File
- end = time.time()
- tqdm.write(getResultString(translatedData, end - start, filename))
- with LOCK:
- TOKENS[0] += translatedData[1][0]
- TOKENS[1] += translatedData[1][1]
-
- # Print Total
- totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
-
- # Print any errors
- if len(MISMATCH) > 0:
- return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
- else:
- return totalString
-
-
-def openFiles(filename):
- """
- Opens and routes SRPG Studio files to appropriate parsing functions.
-
- Args:
- filename: Name of the file to open and parse
-
- Returns:
- Tuple of (translated data, token counts, error)
- """
- with open("files/" + filename, "r", encoding="utf-8-sig") as f:
- data = json.load(f)
-
- # Check if filename matches recollection pattern
- if "recollection" in filename.lower():
- translatedData = parseRecollection(data, filename)
- # mapcommonevents is a list of events like recollection
- elif "mapcommonevents" in filename.lower():
- translatedData = parseRecollection(data, filename)
- # Bookmark events use the same structure as recollection (list of events with pages/commands)
- elif "bookmarkevents" in filename.lower():
- translatedData = parseRecollection(data, filename)
- # Event files (autoevents, placeevents, talkevents, communicationevents, openingevents, endingevents)
- # all have name, desc, and pages->commands structure like recollection
- elif any(event_type in filename.lower() for event_type in ["autoevents", "placeevents", "talkevents", "communicationevents", "openingevents", "endingevents"]):
- translatedData = parseRecollection(data, filename)
- # Check if filename matches bookmark pattern (top-level entries with name/desc + events)
- elif "bookmark" in filename.lower():
- translatedData = parseBookmark(data, filename)
- # Players have the same shape as bookmark entries
- elif "players" in filename.lower():
- translatedData = parseBookmark(data, filename)
- # Titles.json is a small dict of title strings
- elif os.path.basename(filename).lower() == "titles.json":
- translatedData = parseTitles(data, filename)
- # Check if filename matches map pattern
- elif any(pattern in filename.lower() for pattern in MAP_FILES):
- translatedData = parseMap(data, filename)
- # Check if filename matches any pattern in GENERIC_FILES
- elif any(pattern in filename.lower() for pattern in GENERIC_FILES):
- translatedData = parseGeneric(data, filename)
-
- # TODO: Add other SRPG Studio file types here
- else:
- raise NameError(filename + " Not Supported")
-
- return translatedData
-
-
-def parseBookmark(data, filename):
- """
- Parser for SRPG Studio bookmark.json files.
- Structure: List of entries, each with id, name, desc, and events containing pages -> commands
- where commands have data (array of strings) and optional speaker.
-
- Args:
- data: Parsed JSON data (list of bookmark entries)
- filename: Name of the file being parsed
-
- Returns:
- Tuple of (translated data, token counts, error)
- """
- global PBAR
- totalTokens = [0, 0]
-
- pbar = None
- try:
- # Count work units: names, descs, dialogue lines, and speakers
- total_units = 0
-
- for entry in data:
- if not entry:
- continue
-
- # name and desc
- for field in ["name", "desc"]:
- if field in entry and entry[field]:
- total_units += 1
-
- # events -> pages -> commands
- if "events" in entry and isinstance(entry["events"], list):
- for event in entry["events"]:
- if not event:
- continue
-
- # Count event-level name and desc fields
- for field in ["name", "desc"]:
- if field in event and event[field]:
- total_units += 1
-
- if "pages" not in event or not isinstance(event["pages"], list):
- continue
-
- for page in event["pages"]:
- if not page or "commands" not in page or not isinstance(page["commands"], list):
- continue
- for command in page["commands"]:
- if not command:
- continue
- if "data" in command and isinstance(command["data"], list):
- for text in command["data"]:
- if text:
- total_units += 1
- if "speaker" in command and command["speaker"]:
- total_units += 1
-
- # Setup progress bar
- with LOCK:
- pbar = tqdm(
- desc=filename,
- total=total_units,
- bar_format=BAR_FORMAT,
- position=POSITION,
- leave=LEAVE,
- )
- PBAR = pbar
-
- # Translate using two-pass approach
- result = translateBookmark(data, filename, pbar=pbar)
- totalTokens[0] += result[0]
- totalTokens[1] += result[1]
-
- return (data, totalTokens, None)
-
- except Exception as e:
- traceback.print_exc()
- return (data, totalTokens, e)
- finally:
- try:
- if pbar is not None:
- pbar.close()
- except Exception:
- pass
- PBAR = None
-
-
-def translateBookmark(data, filename, translatedDataList=None, pbar=None):
- """
- Translates bookmark.json data structure.
- Two-pass approach via recursion:
- - Pass 1: Collect strings (names, descs, dialogue data with speaker prefix, speakers)
- - Pass 2: Apply translations back into data
-
- Returns:
- [input tokens, output tokens]
- """
- totalTokens = [0, 0]
-
- # Initialize or extract lists
- if translatedDataList is None:
- nameList = []
- descList = []
- dataList = []
- speakerList = []
- else:
- nameList = translatedDataList[0]
- descList = translatedDataList[1]
- dataList = translatedDataList[2]
- speakerList = translatedDataList[3]
-
- for entry in data:
- if not entry:
- continue
-
- # name
- if "name" in entry and entry["name"]:
- if translatedDataList is None:
- nameList.append(entry["name"])
- else:
- if nameList:
- entry["name"] = nameList[0]
- nameList.pop(0)
-
- # desc
- if "desc" in entry and entry["desc"]:
- if translatedDataList is None:
- # Remove newlines for translation
- descList.append(entry["desc"].replace("\n", " "))
- else:
- if descList:
- translatedText = descList[0]
- # Apply text wrapping
- translatedText = dazedwrap.wrapText(translatedText, width=WIDTH)
- entry["desc"] = translatedText
- descList.pop(0)
-
- # events -> pages -> commands
- if "events" in entry and isinstance(entry["events"], list):
- for event in entry["events"]:
- if not event:
- continue
-
- # Handle event-level name field
- if "name" in event and event["name"]:
- if translatedDataList is None:
- nameList.append(event["name"])
- else:
- if nameList:
- event["name"] = nameList[0]
- nameList.pop(0)
-
- # Handle event-level desc field
- if "desc" in event and event["desc"]:
- if translatedDataList is None:
- # Remove newlines for translation
- descList.append(event["desc"].replace("\n", " "))
- else:
- if descList:
- translatedText = descList[0]
- # Apply text wrapping
- translatedText = dazedwrap.wrapText(translatedText, width=WIDTH)
- event["desc"] = translatedText
- descList.pop(0)
-
- if "pages" not in event or not isinstance(event["pages"], list):
- continue
-
- for page in event["pages"]:
- if not page or "commands" not in page or not isinstance(page["commands"], list):
- continue
- for command in page["commands"]:
- if not command:
- continue
-
- speaker = command.get("speaker", "")
-
- # data array
- if "data" in command and isinstance(command["data"], list):
- for i, text in enumerate(command["data"]):
- if text:
- if translatedDataList is None:
- text = text.replace("\n", " ")
- if speaker:
- dataList.append(f"[{speaker}]: {text}")
- else:
- dataList.append(text)
- else:
- if dataList:
- translated = dataList[0]
- if speaker:
- match = re.search(r'(^\[.+?\]\s?[|:]\s?)', translated)
- if match:
- translated = translated.replace(match.group(1), "")
- translated = dazedwrap.wrapText(translated, width=WIDTH)
- command["data"][i] = translated
- dataList.pop(0)
-
- # speaker field
- if "speaker" in command and command["speaker"]:
- if translatedDataList is None:
- speakerList.append(command["speaker"])
- else:
- if speakerList:
- command["speaker"] = speakerList[0]
- speakerList.pop(0)
-
- # If this was Pass 1, perform translations and recurse
- if translatedDataList is None:
- originalNameCount = len(nameList)
- originalDescCount = len(descList)
- originalDataCount = len(dataList)
- originalSpeakerCount = len(speakerList)
-
- if nameList:
- response = translateAI(
- nameList,
- "Reply with only the " + LANGUAGE + " translation of the bookmark name.",
- True,
- filename,
- pbar,
- )
- nameList = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- if descList:
- response = translateAI(
- descList,
- "Reply with only the " + LANGUAGE + " translation of the bookmark description.",
- True,
- filename,
- pbar,
- )
- descList = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- if dataList:
- response = translateAI(
- dataList,
- "Reply with only the " + LANGUAGE + " translation of the dialogue text.",
- True,
- filename,
- pbar,
- )
- dataList = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- if speakerList:
- response = translateAI(
- speakerList,
- "Reply with only the " + LANGUAGE + " translation of the speaker name.",
- True,
- filename,
- pbar,
- )
- speakerList = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Mismatch checks
- if len(nameList) != originalNameCount:
- with LOCK:
- if filename not in MISMATCH:
- MISMATCH.append(filename)
- if len(descList) != originalDescCount:
- with LOCK:
- if filename not in MISMATCH:
- MISMATCH.append(filename)
- if len(dataList) != originalDataCount:
- with LOCK:
- if filename not in MISMATCH:
- MISMATCH.append(filename)
- if len(speakerList) != originalSpeakerCount:
- with LOCK:
- if filename not in MISMATCH:
- MISMATCH.append(filename)
-
- # PASS 2
- translateBookmark(data, filename, [nameList, descList, dataList, speakerList], pbar)
-
- return totalTokens
-
-
-def parseGeneric(data, filename):
- """
- Generic parser for SRPG Studio files with id, name, desc structure.
- Handles files like quests.json.
- Uses a two-pass approach: first pass collects all strings to translate,
- then batch translates them, then second pass applies translations.
-
- Args:
- data: Parsed JSON data (list of objects with id, name, desc)
- filename: Name of the file being parsed
-
- Returns:
- Tuple of (data, token counts, error)
- """
- global PBAR
-
- totalTokens = [0, 0]
-
- pbar = None
- try:
- # Count work units (all translatable fields that need translation)
- total_units = 0
- translatable_fields = ["name", "desc", "commandName", "command"]
-
- for entry in data:
- if entry:
- for field in translatable_fields:
- if field in entry and entry[field]:
- # If command is a list (e.g., fusionsettings), count per element
- if field == "command" and isinstance(entry[field], list):
- total_units += sum(1 for x in entry[field] if x)
- # Otherwise simple increment
- elif not isinstance(entry[field], list):
- total_units += 1
-
- # Handle pages array separately
- if "pages" in entry and entry["pages"] and isinstance(entry["pages"], list):
- for page in entry["pages"]:
- if page:
- total_units += 1
-
- # Handle msg arrays (e.g., shoplayout)
- if "msg" in entry and isinstance(entry["msg"], list):
- total_units += sum(1 for m in entry["msg"] if m)
-
- # Handle rewardData arrays (e.g., quests)
- if "rewardData" in entry and isinstance(entry["rewardData"], list):
- total_units += sum(1 for r in entry["rewardData"] if r)
-
- # Handle customParameters name field (e.g., {name:'シャルロット強制売春'})
- if "customParameters" in entry and entry["customParameters"]:
- match = re.search(r"name:\s*['\"]([^'\"]+)['\"]", entry["customParameters"])
- if match:
- total_units += 1
-
- # Handle terrains array (nested structure)
- if "terrains" in entry and entry["terrains"] and isinstance(entry["terrains"], list):
- for terrain in entry["terrains"]:
- if terrain:
- for field in ["name", "desc"]:
- if field in terrain and terrain[field]:
- total_units += 1
-
- # Setup progress bar (use a per-file instance to avoid cross-thread clashes)
- with LOCK:
- pbar = tqdm(
- desc=filename,
- total=total_units,
- bar_format=BAR_FORMAT,
- position=POSITION,
- leave=LEAVE,
- )
- PBAR = pbar
-
- # Translate the data using two-pass approach
- result = translateGeneric(data, filename, pbar=pbar)
- totalTokens[0] += result[0]
- totalTokens[1] += result[1]
-
- return (data, totalTokens, None)
-
- except Exception as e:
- traceback.print_exc()
- return (data, totalTokens, e)
- finally:
- # Ensure progress bar is closed
- try:
- if pbar is not None:
- pbar.close()
- except Exception:
- pass
- PBAR = None
-
-
-def translateGeneric(data, filename, translatedDataList=None, pbar=None):
- """
- Translates generic SRPG Studio data with id, name, desc, commandName, command, pages structure.
- Uses two-pass approach via recursion:
- - Pass 1 (translatedDataList=None): Collect strings and batch translate
- - Pass 2 (translatedDataList set): Apply translations back to the data
-
- Args:
- data: List of objects with id, name, desc, commandName, command, pages keys
- filename: Name of the file being translated
- translatedDataList: List containing [nameList, descList, commandNameList, commandList, pagesList]
- - Pass 1: Empty lists to collect originals
- - Pass 2: Filled lists with translations
-
- Returns:
- Tuple of [input tokens, output tokens]
- """
- global PBAR
-
- totalTokens = [0, 0]
-
- # Initialize or extract lists
- if translatedDataList is None:
- # PASS 1: Create empty lists to collect strings
- nameList = []
- descList = []
- commandNameList = []
- commandList = []
- pagesList = []
- msgList = []
- commandArrayList = []
- rewardDataList = []
- customParametersNameList = []
- else:
- # PASS 2: Use provided translated lists
- nameList = translatedDataList[0]
- descList = translatedDataList[1]
- commandNameList = translatedDataList[2]
- commandList = translatedDataList[3]
- pagesList = translatedDataList[4]
- msgList = translatedDataList[5]
- commandArrayList = translatedDataList[6]
- rewardDataList = translatedDataList[7]
- customParametersNameList = translatedDataList[8]
-
- # Single loop - behavior depends on which pass we're in
- for entry in data:
- if not entry:
- continue
-
- # Handle name field
- if "name" in entry and entry["name"]:
- # PASS 1: Collect original
- if translatedDataList is None:
- nameList.append(entry["name"])
- # PASS 2: Apply translation
- else:
- if nameList:
- entry["name"] = nameList[0]
- nameList.pop(0)
-
- # Handle desc field
- if "desc" in entry and entry["desc"]:
- # PASS 1: Collect original
- if translatedDataList is None:
- # Nuke Wordwrap
- descList.append(entry["desc"].replace("\n", " "))
- # PASS 2: Apply translation
- else:
- if descList:
- translatedText = descList[0]
- # Wordwrap
- translatedText = dazedwrap.wrapText(translatedText, width=WIDTH)
-
- # Set Data
- entry["desc"] = translatedText
- descList.pop(0)
-
- # Handle commandName field
- if "commandName" in entry and entry["commandName"]:
- # PASS 1: Collect original
- if translatedDataList is None:
- commandNameList.append(entry["commandName"])
- # PASS 2: Apply translation
- else:
- if commandNameList:
- entry["commandName"] = commandNameList[0]
- commandNameList.pop(0)
-
- # Handle command field (string or list)
- if "command" in entry and entry["command"] is not None:
- # If list of strings
- if isinstance(entry["command"], list):
- for i, val in enumerate(entry["command"]):
- if val:
- if translatedDataList is None:
- # Nuke Wrap
- commandArrayList.append(val.replace("\n", " "))
- else:
- if commandArrayList:
- translatedText = dazedwrap.wrapText(commandArrayList[0], width=WIDTH)
- entry["command"][i] = translatedText
- commandArrayList.pop(0)
- # If simple string
- elif isinstance(entry["command"], str):
- if translatedDataList is None:
- commandList.append(entry["command"])
- else:
- if commandList:
- entry["command"] = commandList[0]
- commandList.pop(0)
-
- # Handle pages field (array of strings)
- if "pages" in entry and entry["pages"] and isinstance(entry["pages"], list):
- for i, page in enumerate(entry["pages"]):
- if page:
- # PASS 1: Collect original
- if translatedDataList is None:
- # Nuke Wordwrap
- page = page.replace("\n", " ")
- pagesList.append(page)
- # PASS 2: Apply translation
- else:
- if pagesList:
- translatedText = pagesList[0]
-
- # Wordwrap
- translatedText = dazedwrap.wrapText(translatedText, width=WIDTH)
-
- # Set Data
- entry["pages"][i] = translatedText
- pagesList.pop(0)
-
- # Handle msg field (array of strings)
- if "msg" in entry and isinstance(entry["msg"], list):
- for i, val in enumerate(entry["msg"]):
- if val is not None:
- if translatedDataList is None:
- msgList.append(str(val).replace("\n", " "))
- else:
- if msgList:
- translatedText = dazedwrap.wrapText(msgList[0], width=WIDTH)
- entry["msg"][i] = translatedText
- msgList.pop(0)
-
- # Handle rewardData field (array of strings)
- if "rewardData" in entry and isinstance(entry["rewardData"], list):
- for i, val in enumerate(entry["rewardData"]):
- if val:
- # PASS 1: Collect original
- if translatedDataList is None:
- rewardDataList.append(str(val).replace("\n", " "))
- # PASS 2: Apply translation
- else:
- if rewardDataList:
- translatedText = rewardDataList[0]
- # Set Data (no wordwrap for reward data)
- entry["rewardData"][i] = translatedText
- rewardDataList.pop(0)
-
- # Handle customParameters name field (e.g., {name:'シャルロット強制売春'})
- if "customParameters" in entry and entry["customParameters"]:
- match = re.search(r"name:\s*['\"]([^'\"]+)['\"]", entry["customParameters"])
- if match:
- # PASS 1: Collect original
- if translatedDataList is None:
- customParametersNameList.append(match.group(1))
- # PASS 2: Apply translation
- else:
- if customParametersNameList:
- translatedName = customParametersNameList[0]
- # Replace the name value in customParameters
- entry["customParameters"] = re.sub(
- r"(name:\s*['\"])([^'\"]+)(['\"])",
- r"\1" + translatedName.replace("\\", "\\\\") + r"\3",
- entry["customParameters"]
- )
- customParametersNameList.pop(0)
-
- # Handle terrains field (nested array with name and desc)
- if "terrains" in entry and entry["terrains"] and isinstance(entry["terrains"], list):
- for terrain in entry["terrains"]:
- if not terrain:
- continue
-
- # Handle terrain name
- if "name" in terrain and terrain["name"]:
- # PASS 1: Collect original
- if translatedDataList is None:
- nameList.append(terrain["name"])
- # PASS 2: Apply translation
- else:
- if nameList:
- terrain["name"] = nameList[0]
- nameList.pop(0)
-
- # Handle terrain desc
- if "desc" in terrain and terrain["desc"]:
- # PASS 1: Collect original
- if translatedDataList is None:
- descList.append(terrain["desc"]).replace("\n", " ")
- # PASS 2: Apply translation
- else:
- if descList:
- translatedText = descList[0]
- # Wordwrap
- translatedText = dazedwrap.wrapText(translatedText, width=WIDTH)
-
- # Set Data
- terrain["desc"] = translatedText
- descList.pop(0)
-
- # If this was Pass 1, do the translation and recurse for Pass 2
- if translatedDataList is None:
- # Store original counts for mismatch checking
- originalNameCount = len(nameList)
- originalDescCount = len(descList)
- originalCommandNameCount = len(commandNameList)
- originalCommandCount = len(commandList)
- originalPagesCount = len(pagesList)
- originalMsgCount = len(msgList)
- originalCommandArrayCount = len(commandArrayList)
- originalRewardDataCount = len(rewardDataList)
- originalCustomParametersNameCount = len(customParametersNameList)
-
- # Keep a copy of original names for vocab update (for characters/items/skills/classes/weapons)
- vocab_name_files = ["characters", "items", "skills", "classes", "weapons"]
- originalNameList = nameList.copy() if any(tag in filename.lower() for tag in vocab_name_files) else []
-
- # Batch translate names
- if nameList:
- response = translateAI(
- nameList,
- "Reply with only the " + LANGUAGE + " translation of the quest name.",
- True,
- filename,
- pbar
- )
- nameList = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Update vocab.txt for name-bearing files
- if originalNameList and nameList:
- try:
- file_lower = filename.lower()
- section = None
- if "characters" in file_lower:
- section = "Speakers"
- elif "items" in file_lower:
- section = "Items"
- elif "skills" in file_lower:
- section = "Skills"
- elif "classes" in file_lower:
- section = "Classes"
- elif "weapons" in file_lower:
- section = "Weapons"
-
- if section:
- vocab_pairs = list(zip(originalNameList, nameList))
- update_vocab_section(section, vocab_pairs)
- except Exception:
- traceback.print_exc()
-
- # Batch translate descriptions
- if descList:
- response = translateAI(
- descList,
- "Reply with only the " + LANGUAGE + " translation of the quest description.",
- True,
- filename,
- pbar
- )
- descList = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Batch translate command names
- if commandNameList:
- response = translateAI(
- commandNameList,
- "Reply with only the " + LANGUAGE + " translation of the command name.",
- True,
- filename,
- pbar
- )
- commandNameList = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Batch translate commands
- if commandList:
- response = translateAI(
- commandList,
- "Reply with only the " + LANGUAGE + " translation of the command.",
- True,
- filename,
- pbar
- )
- commandList = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Batch translate pages
- if pagesList:
- response = translateAI(
- pagesList,
- "Reply with only the " + LANGUAGE + " translation of the page content.",
- True,
- filename,
- pbar
- )
- pagesList = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Batch translate msg arrays
- if msgList:
- response = translateAI(
- msgList,
- "Reply with only the " + LANGUAGE + " translation of the message text.",
- True,
- filename,
- pbar
- )
- msgList = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Batch translate command arrays
- if commandArrayList:
- response = translateAI(
- commandArrayList,
- "Reply with only the " + LANGUAGE + " translation of the command text.",
- True,
- filename,
- pbar
- )
- commandArrayList = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Batch translate rewardData arrays
- if rewardDataList:
- response = translateAI(
- rewardDataList,
- "Reply with only the " + LANGUAGE + " translation of the reward data text.",
- True,
- filename,
- pbar
- )
- rewardDataList = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Batch translate customParameters name fields
- if customParametersNameList:
- response = translateAI(
- customParametersNameList,
- "Reply with only the " + LANGUAGE + " translation of the custom parameter name.",
- True,
- filename,
- pbar
- )
- customParametersNameList = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Check for mismatch errors
- if len(nameList) != originalNameCount:
- with LOCK:
- if filename not in MISMATCH:
- MISMATCH.append(filename)
-
- if len(descList) != originalDescCount:
- with LOCK:
- if filename not in MISMATCH:
- MISMATCH.append(filename)
-
- if len(commandNameList) != originalCommandNameCount:
- with LOCK:
- if filename not in MISMATCH:
- MISMATCH.append(filename)
-
- if len(commandList) != originalCommandCount:
- with LOCK:
- if filename not in MISMATCH:
- MISMATCH.append(filename)
-
- if len(pagesList) != originalPagesCount:
- with LOCK:
- if filename not in MISMATCH:
- MISMATCH.append(filename)
- if len(msgList) != originalMsgCount:
- with LOCK:
- if filename not in MISMATCH:
- MISMATCH.append(filename)
- if len(commandArrayList) != originalCommandArrayCount:
- with LOCK:
- if filename not in MISMATCH:
- MISMATCH.append(filename)
- if len(rewardDataList) != originalRewardDataCount:
- with LOCK:
- if filename not in MISMATCH:
- MISMATCH.append(filename)
- if len(customParametersNameList) != originalCustomParametersNameCount:
- with LOCK:
- if filename not in MISMATCH:
- MISMATCH.append(filename)
-
- # PASS 2: Recursively call to apply translations
- translateGeneric(
- data,
- filename,
- [nameList, descList, commandNameList, commandList, pagesList, msgList, commandArrayList, rewardDataList, customParametersNameList],
- pbar,
- )
-
- return totalTokens
-
-
-def parseTitles(data, filename):
- """Parser for titles.json (small dict of title strings)."""
- global PBAR
- totalTokens = [0, 0]
- pbar = None
- try:
- # Collect fields to translate
- keys = [k for k in ["windowTitle", "gameTitle", "saveFileTitle"] if k in data and data[k]]
- values = [data[k] for k in keys]
-
- with LOCK:
- pbar = tqdm(
- desc=filename,
- total=len(values),
- bar_format=BAR_FORMAT,
- position=POSITION,
- leave=LEAVE,
- )
- PBAR = pbar
-
- if values:
- response = translateAI(
- values,
- "Reply with only the " + LANGUAGE + " translation of the title text.",
- True,
- filename,
- pbar,
- )
- translations = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Apply
- for i, k in enumerate(keys):
- data[k] = translations[i]
-
- return (data, totalTokens, None)
- except Exception as e:
- traceback.print_exc()
- return (data, totalTokens, e)
- finally:
- try:
- if pbar is not None:
- pbar.close()
- except Exception:
- pass
- PBAR = None
-
-
-def parseRecollection(data, filename):
- """
- Parser for SRPG Studio recollection.json files.
- Structure: List of entries, each with pages containing commands with data arrays and speakers.
-
- Args:
- data: Parsed JSON data (list of objects with id, pages structure)
- filename: Name of the file being parsed
-
- Returns:
- Tuple of (data, token counts, error)
- """
- global PBAR
-
- totalTokens = [0, 0]
-
- pbar = None
- try:
- # Count work units (data entries and speakers that need translation)
- total_units = 0
-
- # Iterate through structure and count
- for entry in data:
- if not entry:
- continue
-
- # Count name and desc fields at entry level
- for field in ["name", "desc"]:
- if field in entry and entry[field]:
- total_units += 1
-
- # Count customParameters hints
- if "customParameters" in entry and entry["customParameters"]:
- match = re.search(r'hint:"((?:[^"\\]|\\.)*)"', entry["customParameters"])
- if match:
- total_units += 1
-
- if "pages" not in entry or not isinstance(entry["pages"], list):
- continue
-
- for page in entry["pages"]:
- if not page or "commands" not in page or not isinstance(page["commands"], list):
- continue
-
- for command in page["commands"]:
- if not command:
- continue
-
- # Count data array items (count any non-empty text; do not gate on LANGREGEX)
- if "data" in command and isinstance(command["data"], list):
- for text in command["data"]:
- if text:
- total_units += 1
-
- # Count speaker field (count any non-empty speaker; do not gate on LANGREGEX)
- if "speaker" in command and command["speaker"]:
- total_units += 1
-
- # Setup progress bar (per-file instance)
- with LOCK:
- pbar = tqdm(
- desc=filename,
- total=total_units,
- bar_format=BAR_FORMAT,
- position=POSITION,
- leave=LEAVE,
- )
- PBAR = pbar
-
- # Translate the data using two-pass approach
- result = translateRecollection(data, filename, pbar=pbar)
- totalTokens[0] += result[0]
- totalTokens[1] += result[1]
-
- return (data, totalTokens, None)
-
- except Exception as e:
- traceback.print_exc()
- return (data, totalTokens, e)
- finally:
- try:
- if pbar is not None:
- pbar.close()
- except Exception:
- pass
- PBAR = None
-
-
-def translateRecollection(data, filename, translatedDataList=None, pbar=None):
- """
- Translates recollection.json data structure.
- Uses two-pass approach via recursion:
- - Pass 1 (translatedDataList=None): Collect strings and batch translate
- - Pass 2 (translatedDataList set): Apply translations back to the data
-
- Args:
- data: List of objects with name, desc, customParameters, and pages->commands->data structure
- filename: Name of the file being translated
- translatedDataList: List containing [nameList, descList, dataList, speakerList, customParamsList]
- - Pass 1: Empty lists to collect originals
- - Pass 2: Filled lists with translations
-
- Returns:
- Tuple of [input tokens, output tokens]
- """
- global PBAR
-
- totalTokens = [0, 0]
-
- # Initialize or extract lists
- if translatedDataList is None:
- # PASS 1: Create empty lists to collect strings
- nameList = []
- descList = []
- dataList = []
- speakerList = []
- customParamsList = []
- originalSpeakerList = [] # For vocab update
- else:
- # PASS 2: Use provided translated lists
- nameList = translatedDataList[0]
- descList = translatedDataList[1]
- dataList = translatedDataList[2]
- speakerList = translatedDataList[3]
- customParamsList = translatedDataList[4]
-
- # Single loop - behavior depends on which pass we're in
- for entry in data:
- if not entry:
- continue
-
- # Handle name field
- if "name" in entry and entry["name"]:
- # PASS 1: Collect original
- if translatedDataList is None:
- nameList.append(entry["name"])
- # PASS 2: Apply translation
- else:
- if nameList:
- entry["name"] = nameList[0]
- nameList.pop(0)
-
- # Handle desc field
- if "desc" in entry and entry["desc"]:
- # PASS 1: Collect original
- if translatedDataList is None:
- # Remove newlines for translation
- descList.append(entry["desc"].replace("\n", " "))
- # PASS 2: Apply translation
- else:
- if descList:
- translatedText = descList[0]
- # Apply text wrapping
- translatedText = dazedwrap.wrapText(translatedText, width=WIDTH)
- entry["desc"] = translatedText
- descList.pop(0)
-
- # Handle customParameters at entry level
- if "customParameters" in entry and entry["customParameters"]:
- # PASS 1: Collect hint text
- if translatedDataList is None:
- # Extract hint value using regex (captures content between hint:" and ")
- match = re.search(r'hint:"((?:[^"\\]|\\.)*)"', entry["customParameters"])
- if match:
- hintText = match.group(1)
- hintText = hintText.replace("\\n", " ")
- customParamsList.append(hintText)
- else:
- # No hint found, add empty string as placeholder
- customParamsList.append("")
- # PASS 2: Apply translation
- else:
- if customParamsList:
- translatedHint = customParamsList[0]
- customParamsList.pop(0)
-
- if translatedHint: # Only replace if we translated something
- # Replace double quotes with single quotes since \" is used as delimiter
- translatedHint = translatedHint.replace('"', "'")
- # Wrap text using dazedwrap with WIDTH and replace newlines with \\n
- translatedHint = dazedwrap.wrapText(translatedHint, width=WIDTH)
- translatedHint = translatedHint.replace("\n", "\\\\n")
- # Replace the hint value in customParameters
- entry["customParameters"] = re.sub(
- r'(hint:")((?:[^"\\]|\\.)*)"',
- r'\1' + translatedHint + '"',
- entry["customParameters"]
- )
-
- # Check if pages exists before processing
- if "pages" not in entry or not isinstance(entry["pages"], list):
- continue
-
- for page in entry["pages"]:
- if not page or "commands" not in page:
- continue
-
- if not isinstance(page["commands"], list):
- continue
-
- for command in page["commands"]:
- if not command:
- continue
-
- # Get the speaker for this command (if available)
- speaker = command.get("speaker", "")
-
- # Handle data array
- if "data" in command and isinstance(command["data"], list):
- for i, text in enumerate(command["data"]):
- if text:
- # PASS 1: Collect original with speaker prefix
- if translatedDataList is None:
- # Remove Wrap
- text = text.replace("\n", " ")
-
- # Attach speaker to data for translation
- if speaker:
- dataList.append(f"[{speaker}]: {text}")
- else:
- dataList.append(text)
-
- # PASS 2: Apply translation and strip speaker prefix
- else:
- if dataList:
- translated = dataList[0]
-
- # Remove speaker
- if speaker:
- match = re.search(r'(^\[.+?\]\s?[|:]\s?)', translated)
- if match:
- translated = translated.replace(match.group(1), "")
-
- # Textwrap
- translated = dazedwrap.wrapText(translated, width=WIDTH)
-
- # Set Data
- command["data"][i] = translated
- dataList.pop(0)
-
- # Handle speaker field
- if "speaker" in command and command["speaker"]:
- # PASS 1: Collect original
- if translatedDataList is None:
- originalSpeakerList.append(command["speaker"])
- speakerList.append(command["speaker"])
- # PASS 2: Apply translation
- else:
- if speakerList:
- command["speaker"] = speakerList[0]
- speakerList.pop(0)
-
- # If this was Pass 1, do the translation and recurse for Pass 2
- if translatedDataList is None:
- # Store original counts for mismatch checking
- originalNameCount = len(nameList)
- originalDescCount = len(descList)
- originalDataCount = len(dataList)
- originalSpeakerCount = len(speakerList)
- originalCustomParamsCount = len(customParamsList)
-
- # Batch translate names
- if nameList:
- response = translateAI(
- nameList,
- "Reply with only the " + LANGUAGE + " translation of the name.",
- True,
- filename,
- pbar
- )
- nameList = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Batch translate descriptions
- if descList:
- response = translateAI(
- descList,
- "Reply with only the " + LANGUAGE + " translation of the description.",
- True,
- filename,
- pbar
- )
- descList = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Batch translate data text
- if dataList:
- response = translateAI(
- dataList,
- "Reply with only the " + LANGUAGE + " translation of the dialogue text.",
- True,
- filename,
- pbar
- )
- dataList = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Batch translate speakers
- if speakerList:
- response = translateAI(
- speakerList,
- "Reply with only the " + LANGUAGE + " translation of the speaker name.",
- True,
- filename,
- pbar
- )
- speakerList = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Batch translate customParameters hints (only non-empty ones)
- if customParamsList:
- # Filter out empty strings for translation
- hintsToTranslate = [h for h in customParamsList if h]
- if hintsToTranslate:
- response = translateAI(
- hintsToTranslate,
- "Reply with only the " + LANGUAGE + " translation of the hint text.",
- True,
- filename,
- pbar
- )
- translatedHints = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Reconstruct customParamsList with translations in place
- translatedIndex = 0
- for i, hint in enumerate(customParamsList):
- if hint: # If it was non-empty, use the translation
- customParamsList[i] = translatedHints[translatedIndex]
- translatedIndex += 1
-
- # Check for mismatch errors
- if len(nameList) != originalNameCount:
- with LOCK:
- if filename not in MISMATCH:
- MISMATCH.append(filename)
-
- if len(descList) != originalDescCount:
- with LOCK:
- if filename not in MISMATCH:
- MISMATCH.append(filename)
-
- if len(dataList) != originalDataCount:
- with LOCK:
- if filename not in MISMATCH:
- MISMATCH.append(filename)
-
- if len(speakerList) != originalSpeakerCount:
- with LOCK:
- if filename not in MISMATCH:
- MISMATCH.append(filename)
-
- if len(customParamsList) != originalCustomParamsCount:
- with LOCK:
- if filename not in MISMATCH:
- MISMATCH.append(filename)
-
- # PASS 2: Recursively call to apply translations
- translateRecollection(data, filename, [nameList, descList, dataList, speakerList, customParamsList], pbar)
-
- return totalTokens
-
-
-def parseMap(data, filename):
- """
- Parser for SRPG Studio map.json files.
- Structure: Single object with:
- - id, name, desc, mapName
- - victoryConds (array of strings)
- - defeatConds (array of strings)
- - EnemyUnits (array of objects with id, name, desc, events structure similar to recollection)
-
- Args:
- data: Parsed JSON data (single map object)
- filename: Name of the file being parsed
-
- Returns:
- Tuple of (data, token counts, error)
- """
- global PBAR
-
- totalTokens = [0, 0]
-
- pbar = None
- try:
- # Count work units (all translatable fields)
- total_units = 0
-
- # Count top-level fields: desc, mapName (name is an identifier and should not be translated)
- for field in ["desc", "mapName"]:
- if field in data and data[field]:
- total_units += 1
-
- # Count victoryConds array items
- if "victoryConds" in data and isinstance(data["victoryConds"], list):
- for cond in data["victoryConds"]:
- if cond:
- total_units += 1
-
- # Count defeatConds array items
- if "defeatConds" in data and isinstance(data["defeatConds"], list):
- for cond in data["defeatConds"]:
- if cond:
- total_units += 1
-
- # Count units with events (EnemyUnits, EvEnemyUnits)
- for unitsKey in ["EnemyUnits", "EvEnemyUnits"]:
- if unitsKey in data and isinstance(data[unitsKey], list):
- for unit in data[unitsKey]:
- if not unit:
- continue
-
- # Count unit name and desc
- for field in ["name", "desc"]:
- if field in unit and unit[field]:
- total_units += 1
-
- # Count events->pages->commands->data and speaker
- if "events" in unit and isinstance(unit["events"], list):
- for event in unit["events"]:
- if not event or "pages" not in event:
- continue
-
- if not isinstance(event["pages"], list):
- continue
-
- for page in event["pages"]:
- if not page or "commands" not in page:
- continue
-
- if not isinstance(page["commands"], list):
- continue
-
- for command in page["commands"]:
- if not command:
- continue
-
- # Count data array items
- if "data" in command and isinstance(command["data"], list):
- for text in command["data"]:
- if text:
- total_units += 1
-
- # Count speaker field
- if "speaker" in command and command["speaker"]:
- total_units += 1
-
- # Count events (placeEvents, autoEvents, openingEvents, communicationEvents)
- for eventsKey in ["placeEvents", "autoEvents", "openingEvents", "communicationEvents"]:
- if eventsKey in data and isinstance(data[eventsKey], list):
- for event in data[eventsKey]:
- if not event:
- continue
-
- # Count event-level name and desc fields
- for field in ["name", "desc"]:
- if field in event and event[field]:
- total_units += 1
-
- if "pages" not in event or not isinstance(event["pages"], list):
- continue
-
- for page in event["pages"]:
- if not page or "commands" not in page:
- continue
-
- if not isinstance(page["commands"], list):
- continue
-
- for command in page["commands"]:
- if not command:
- continue
-
- # Count data array items
- if "data" in command and isinstance(command["data"], list):
- for text in command["data"]:
- if text:
- total_units += 1
-
- # Count speaker field
- if "speaker" in command and command["speaker"]:
- total_units += 1
-
- # Setup progress bar (per-file instance)
- with LOCK:
- pbar = tqdm(
- desc=filename,
- total=total_units,
- bar_format=BAR_FORMAT,
- position=POSITION,
- leave=LEAVE,
- )
- PBAR = pbar
-
- # Translate the data using two-pass approach
- result = translateMap(data, filename, pbar=pbar)
- totalTokens[0] += result[0]
- totalTokens[1] += result[1]
-
- return (data, totalTokens, None)
-
- except Exception as e:
- traceback.print_exc()
- return (data, totalTokens, e)
- finally:
- try:
- if pbar is not None:
- pbar.close()
- except Exception:
- pass
- PBAR = None
-
-
-def translateMap(data, filename, translatedDataList=None, pbar=None):
- """
- Translates map.json data structure.
- Uses two-pass approach via recursion:
- - Pass 1 (translatedDataList=None): Collect strings and batch translate
- - Pass 2 (translatedDataList set): Apply translations back to the data
-
- Args:
- data: Single map object with name, desc, mapName, victoryConds, defeatConds, and EnemyUnits
- filename: Name of the file being translated
- translatedDataList: List containing translation lists
- - Pass 1: Empty lists to collect originals
- - Pass 2: Filled lists with translations
-
- Returns:
- Tuple of [input tokens, output tokens]
- """
- global PBAR
-
- totalTokens = [0, 0]
-
- # Initialize or extract lists
- if translatedDataList is None:
- # PASS 1: Create empty lists to collect strings
- descList = []
- mapNameList = []
- victoryCondsList = []
- defeatCondsList = []
- unitNameList = []
- unitDescList = []
- eventNameList = []
- eventDescList = []
- dataList = []
- speakerList = []
- originalSpeakerList = [] # For vocab update
- else:
- # PASS 2: Use provided translated lists
- descList = translatedDataList[0]
- mapNameList = translatedDataList[1]
- victoryCondsList = translatedDataList[2]
- defeatCondsList = translatedDataList[3]
- unitNameList = translatedDataList[4]
- unitDescList = translatedDataList[5]
- eventNameList = translatedDataList[6]
- eventDescList = translatedDataList[7]
- dataList = translatedDataList[8]
- speakerList = translatedDataList[9]
-
- # Note: name field is not translated as it's an identifier (e.g., "ch3_瘴気の森")
-
- # Handle desc field
- if "desc" in data and data["desc"]:
- # PASS 1: Collect original
- if translatedDataList is None:
- descList.append(data["desc"].replace("\n", " "))
- # PASS 2: Apply translation
- else:
- if descList:
- translatedText = descList[0]
- # Wordwrap
- translatedText = dazedwrap.wrapText(translatedText, width=WIDTH)
- data["desc"] = translatedText
- descList.pop(0)
-
- # Handle mapName field
- if "mapName" in data and data["mapName"]:
- # PASS 1: Collect original
- if translatedDataList is None:
- mapNameList.append(data["mapName"])
- # PASS 2: Apply translation
- else:
- if mapNameList:
- data["mapName"] = mapNameList[0]
- mapNameList.pop(0)
-
- # Handle victoryConds array
- if "victoryConds" in data and isinstance(data["victoryConds"], list):
- for i, cond in enumerate(data["victoryConds"]):
- if cond:
- # PASS 1: Collect original
- if translatedDataList is None:
- victoryCondsList.append(cond)
- # PASS 2: Apply translation
- else:
- if victoryCondsList:
- data["victoryConds"][i] = victoryCondsList[0]
- victoryCondsList.pop(0)
-
- # Handle defeatConds array
- if "defeatConds" in data and isinstance(data["defeatConds"], list):
- for i, cond in enumerate(data["defeatConds"]):
- if cond:
- # PASS 1: Collect original
- if translatedDataList is None:
- defeatCondsList.append(cond)
- # PASS 2: Apply translation
- else:
- if defeatCondsList:
- data["defeatConds"][i] = defeatCondsList[0]
- defeatCondsList.pop(0)
-
- # Process all unit arrays (EnemyUnits, EvEnemyUnits)
- for unitsKey in ["EnemyUnits", "EvEnemyUnits"]:
- if unitsKey in data and isinstance(data[unitsKey], list):
- for unit in data[unitsKey]:
- if not unit:
- continue
-
- # Handle unit name
- if "name" in unit and unit["name"]:
- if translatedDataList is None:
- unitNameList.append(unit["name"])
- else:
- if unitNameList:
- unit["name"] = unitNameList[0]
- unitNameList.pop(0)
-
- # Handle unit desc
- if "desc" in unit and unit["desc"]:
- if translatedDataList is None:
- unitDescList.append(unit["desc"].replace("\n", " "))
- else:
- if unitDescList:
- translatedText = unitDescList[0]
- # Wordwrap
- translatedText = dazedwrap.wrapText(translatedText, width=WIDTH)
- unit["desc"] = translatedText
- unitDescList.pop(0)
-
- # Process events within units
- if "events" in unit and isinstance(unit["events"], list):
- for event in unit["events"]:
- if not event or "pages" not in event:
- continue
-
- if not isinstance(event["pages"], list):
- continue
-
- for page in event["pages"]:
- if not page or "commands" not in page:
- continue
-
- if not isinstance(page["commands"], list):
- continue
-
- for command in page["commands"]:
- if not command:
- continue
-
- speaker = command.get("speaker", "")
-
- # Handle data array
- if "data" in command and isinstance(command["data"], list):
- for i, text in enumerate(command["data"]):
- if text:
- if translatedDataList is None:
- text = text.replace("\n", " ")
- if speaker:
- dataList.append(f"[{speaker}]: {text}")
- else:
- dataList.append(text)
- else:
- if dataList:
- translated = dataList[0]
- if speaker:
- match = re.search(r'(^\[.+?\]\s?[|:]\s?)', translated)
- if match:
- translated = translated.replace(match.group(1), "")
- translated = dazedwrap.wrapText(translated, width=WIDTH)
- command["data"][i] = translated
- dataList.pop(0)
-
- # Handle speaker field
- if "speaker" in command and command["speaker"]:
- if translatedDataList is None:
- originalSpeakerList.append(command["speaker"])
- speakerList.append(command["speaker"])
- else:
- if speakerList:
- command["speaker"] = speakerList[0]
- speakerList.pop(0)
-
- # Process all event arrays (placeEvents, autoEvents, openingEvents, communicationEvents)
- for eventsKey in ["placeEvents", "autoEvents", "openingEvents", "communicationEvents"]:
- if eventsKey in data and isinstance(data[eventsKey], list):
- for event in data[eventsKey]:
- if not event:
- continue
-
- # Handle event-level name field
- if "name" in event and event["name"]:
- if translatedDataList is None:
- eventNameList.append(event["name"])
- else:
- if eventNameList:
- event["name"] = eventNameList[0]
- eventNameList.pop(0)
-
- # Handle event-level desc field
- if "desc" in event and event["desc"]:
- if translatedDataList is None:
- # Remove newlines for translation
- eventDescList.append(event["desc"].replace("\n", " "))
- else:
- if eventDescList:
- translatedText = eventDescList[0]
- # Apply text wrapping
- translatedText = dazedwrap.wrapText(translatedText, width=WIDTH)
- event["desc"] = translatedText
- eventDescList.pop(0)
-
- if "pages" not in event or not isinstance(event["pages"], list):
- continue
-
- for page in event["pages"]:
- if not page or "commands" not in page:
- continue
-
- if not isinstance(page["commands"], list):
- continue
-
- for command in page["commands"]:
- if not command:
- continue
-
- speaker = command.get("speaker", "")
-
- # Handle data array
- if "data" in command and isinstance(command["data"], list):
- for i, text in enumerate(command["data"]):
- if text:
- if translatedDataList is None:
- text = text.replace("\n", " ")
- if speaker:
- dataList.append(f"[{speaker}]: {text}")
- else:
- dataList.append(text)
- else:
- if dataList:
- translated = dataList[0]
- if speaker:
- match = re.search(r'(^\[.+?\]\s?[|:]\s?)', translated)
- if match:
- translated = translated.replace(match.group(1), "")
- translated = dazedwrap.wrapText(translated, width=WIDTH)
- command["data"][i] = translated
- dataList.pop(0)
-
- # Handle speaker field
- if "speaker" in command and command["speaker"]:
- if translatedDataList is None:
- originalSpeakerList.append(command["speaker"])
- speakerList.append(command["speaker"])
- else:
- if speakerList:
- command["speaker"] = speakerList[0]
- speakerList.pop(0)
-
- # If this was Pass 1, do the translation and recurse for Pass 2
- if translatedDataList is None:
- # Store original counts for mismatch checking
- originalDescCount = len(descList)
- originalMapNameCount = len(mapNameList)
- originalVictoryCondsCount = len(victoryCondsList)
- originalDefeatCondsCount = len(defeatCondsList)
- originalUnitNameCount = len(unitNameList)
- originalUnitDescCount = len(unitDescList)
- originalEventNameCount = len(eventNameList)
- originalEventDescCount = len(eventDescList)
- originalDataCount = len(dataList)
- originalSpeakerCount = len(speakerList)
-
- # Batch translate map descriptions
- if descList:
- response = translateAI(
- descList,
- "Reply with only the " + LANGUAGE + " translation of the map description.",
- True,
- filename,
- pbar
- )
- descList = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Batch translate map display names
- if mapNameList:
- response = translateAI(
- mapNameList,
- "Reply with only the " + LANGUAGE + " translation of the map display name.",
- True,
- filename,
- pbar
- )
- mapNameList = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Batch translate victory conditions
- if victoryCondsList:
- response = translateAI(
- victoryCondsList,
- "Reply with only the " + LANGUAGE + " translation of the victory condition.",
- True,
- filename,
- pbar
- )
- victoryCondsList = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Batch translate defeat conditions
- if defeatCondsList:
- response = translateAI(
- defeatCondsList,
- "Reply with only the " + LANGUAGE + " translation of the defeat condition.",
- True,
- filename,
- pbar
- )
- defeatCondsList = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Batch translate unit names
- if unitNameList:
- response = translateAI(
- unitNameList,
- "Reply with only the " + LANGUAGE + " translation of the enemy unit name.",
- True,
- filename,
- pbar
- )
- unitNameList = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Batch translate unit descriptions
- if unitDescList:
- response = translateAI(
- unitDescList,
- "Reply with only the " + LANGUAGE + " translation of the enemy unit description.",
- True,
- filename,
- pbar
- )
- unitDescList = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Batch translate event names
- if eventNameList:
- response = translateAI(
- eventNameList,
- "Reply with only the " + LANGUAGE + " translation of the event name.",
- True,
- filename,
- pbar
- )
- eventNameList = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Batch translate event descriptions
- if eventDescList:
- response = translateAI(
- eventDescList,
- "Reply with only the " + LANGUAGE + " translation of the event description.",
- True,
- filename,
- pbar
- )
- eventDescList = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Batch translate dialogue data
- if dataList:
- response = translateAI(
- dataList,
- "Reply with only the " + LANGUAGE + " translation of the dialogue text.",
- True,
- filename,
- pbar
- )
- dataList = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Batch translate speakers
- if speakerList:
- response = translateAI(
- speakerList,
- "Reply with only the " + LANGUAGE + " translation of the speaker name.",
- True,
- filename,
- pbar
- )
- speakerList = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Check for mismatch errors
- if len(descList) != originalDescCount:
- with LOCK:
- if filename not in MISMATCH:
- MISMATCH.append(filename)
-
- if len(mapNameList) != originalMapNameCount:
- with LOCK:
- if filename not in MISMATCH:
- MISMATCH.append(filename)
-
- if len(victoryCondsList) != originalVictoryCondsCount:
- with LOCK:
- if filename not in MISMATCH:
- MISMATCH.append(filename)
-
- if len(defeatCondsList) != originalDefeatCondsCount:
- with LOCK:
- if filename not in MISMATCH:
- MISMATCH.append(filename)
-
- if len(unitNameList) != originalUnitNameCount:
- with LOCK:
- if filename not in MISMATCH:
- MISMATCH.append(filename)
-
- if len(unitDescList) != originalUnitDescCount:
- with LOCK:
- if filename not in MISMATCH:
- MISMATCH.append(filename)
-
- if len(eventNameList) != originalEventNameCount:
- with LOCK:
- if filename not in MISMATCH:
- MISMATCH.append(filename)
-
- if len(eventDescList) != originalEventDescCount:
- with LOCK:
- if filename not in MISMATCH:
- MISMATCH.append(filename)
-
- if len(dataList) != originalDataCount:
- with LOCK:
- if filename not in MISMATCH:
- MISMATCH.append(filename)
-
- if len(speakerList) != originalSpeakerCount:
- with LOCK:
- if filename not in MISMATCH:
- MISMATCH.append(filename)
-
- # PASS 2: Recursively call to apply translations
- translateMap(data, filename, [descList, mapNameList, victoryCondsList, defeatCondsList, unitNameList, unitDescList, eventNameList, eventDescList, dataList, speakerList], pbar)
-
- return totalTokens
-
-
-def getResultString(translatedData, translationTime, filename):
- """
- Formats the translation result string with token counts, cost, and time.
-
- Args:
- translatedData: Tuple of (data, tokens, error)
- translationTime: Time taken for translation
- filename: Name of the file
-
- Returns:
- Formatted result string
- """
- global TIMETOTAL
-
- # Calculate cost
- cost = calculateCost(translatedData[1][0], translatedData[1][1], MODEL)
- totalTokenstring = (
- Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
- "[Output: "
- + str(translatedData[1][1])
- + "]" "[Cost: ${:,.4f}".format(cost)
- + "]"
- )
-
- # Format time string
- if filename != "TOTAL":
- timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
- TIMETOTAL += round(translationTime, 1)
- else:
- timeString = Fore.BLUE + "[" + str(round(TIMETOTAL, 1)) + "s]"
-
- # Return success or failure string
- if translatedData[2] is None:
- # Success
- return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET
- else:
- # Fail
- try:
- raise translatedData[2]
- except Exception as e:
- traceback.print_exc()
- return (
- filename
- + ": "
- + totalTokenstring
- + timeString
- + Fore.RED
- + " \u2717 "
- + Fore.RESET
- )
-
-
-def getSpeaker(speaker):
- """
- Translates speaker/character names with caching to avoid redundant translations.
-
- Args:
- speaker: The original speaker name to translate
-
- Returns:
- List containing [translated name, [input tokens, output tokens]]
- """
- if speaker == "":
- return ["", [0, 0]]
-
- # Check if speaker has already been translated
- for i in range(len(NAMESLIST)):
- if speaker == NAMESLIST[i][0]:
- return [NAMESLIST[i][1], [0, 0]]
-
- # Translate and Store Speaker
- response = translateAI(
- speaker,
- "Reply with the " + LANGUAGE + " translation of the NPC name.",
- False,
- )
- response[0] = response[0].title()
- response[0] = response[0].replace("'S", "'s")
- response[0] = response[0].replace("Speaker: ", "")
-
- # Retry if name doesn't translate for some reason
- if re.search(r"([a-zA-Z？?])", response[0]) is None:
- response = translateAI(
- speaker,
- "Reply with the " + LANGUAGE + " translation of the NPC name.",
- False,
- )
- response[0] = response[0].title()
- response[0] = response[0].replace("'S", "'s")
-
- speakerList = [speaker, response[0]]
- NAMESLIST.append(speakerList)
- return response
-
-
-def translateAI(text, history, history_ctx=None, filename=None, pbar=None):
- """
- Legacy wrapper function for the new shared translation utility.
- This maintains compatibility with existing code while using the new shared implementation.
-
- Args:
- text: Text to translate (can be string or list)
- history: History/context for the translation
-
- Returns:
- List containing [translated text, [input tokens, output tokens]]
- """
- global PBAR, MISMATCH, FILENAME
-
- # Update config estimate mode based on global ESTIMATE
- TRANSLATION_CONFIG.estimateMode = bool(ESTIMATE)
-
- # Call the new shared translation function
- return sharedtranslateAI(
- text=text,
- history=history,
- config=TRANSLATION_CONFIG,
- filename=filename if filename is not None else FILENAME,
- pbar=pbar if pbar is not None else PBAR,
- lock=LOCK,
- mismatchList=MISMATCH
- )
-
+# Libraries
+import json
+import os
+import re
+import shutil
+import util.dazedwrap as dazedwrap
+import threading
+import time
+import traceback
+from dotenv import load_dotenv
+from pathlib import Path
+from colorama import Fore
+from dotenv import load_dotenv
+from retry import retry
+from tqdm import tqdm
+from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost
+
+# Globals
+MODEL = os.getenv("model")
+TIMEOUT = int(os.getenv("timeout"))
+LANGUAGE = os.getenv("language").capitalize()
+from util.paths import PROMPT_PATH, VOCAB_PATH
+
+PROMPT = PROMPT_PATH.read_text(encoding="utf-8")
+VOCAB = VOCAB_PATH.read_text(encoding="utf-8")
+LOCK = threading.Lock()
+VOCAB_LOCK = threading.Lock() # Dedicated lock for vocab.txt updates
+WIDTH = int(os.getenv("width"))
+LISTWIDTH = int(os.getenv("listWidth"))
+NOTEWIDTH = int(os.getenv("noteWidth"))
+MAXHISTORY = 10
+ESTIMATE = ""
+TOKENS = [0, 0]
+MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
+NAMESLIST = [] # List of speaker names and their translations
+PBAR = None
+FILENAME = None
+TIMETOTAL = 0 # Total Time Taken for all translations
+
+# Regex - Need to change this if you want to translate from/to other languages. Default is Japanese Regex
+LANGREGEX = r"[\u3000-\u303F\u3040-\u309F\u30A0-\u30FF\u31F0-\u31FF\u3400-\u4DBF\u4E00-\u9FFF\uF900-\uFAFF\uFF61-\uFF9F]+"
+
+# Get pricing configuration based on the model
+PRICING_CONFIG = getPricingConfig(MODEL)
+INPUTAPICOST = PRICING_CONFIG["inputAPICost"]
+OUTPUTAPICOST = PRICING_CONFIG["outputAPICost"]
+BATCHSIZE = PRICING_CONFIG["batchSize"]
+FREQUENCY_PENALTY = PRICING_CONFIG["frequencyPenalty"]
+
+# tqdm Globals
+BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
+POSITION = 0
+LEAVE = False
+
+# Initialize Translation Config
+TRANSLATION_CONFIG = TranslationConfig(
+ model=MODEL,
+ language=LANGUAGE,
+ prompt=PROMPT,
+ vocab=VOCAB,
+ langRegex=LANGREGEX,
+ batchSize=BATCHSIZE,
+ maxHistory=MAXHISTORY,
+ estimateMode=False # Will be set dynamically based on ESTIMATE
+)
+
+# Config (Default)
+FIXTEXTWRAP = True # Rewrap text to WIDTH
+IGNORETLTEXT = False # Skip Translated Text
+
+# List of file patterns that use parseGeneric
+# Add more patterns here as needed
+GENERIC_FILES = [
+ "quests",
+ "shops",
+ "shoplayout",
+ "bonuses",
+ "base",
+ "battleprep",
+ "manage",
+ "mapcommands",
+ "title",
+ "classes",
+ "classesgroups",
+ "classtypes",
+ "races",
+ "skills",
+ "weapons",
+ "states",
+ "difficulties",
+ "fonts",
+ "fusionsettings",
+ "transformations",
+ "characters",
+ "glossary",
+ "npc",
+ "screens",
+ "strings",
+ "originalterrains",
+ "runtimeterrains",
+ "archers",
+ "fighters",
+ "mages",
+ "items",
+]
+
+# List of file patterns that use parseMap
+# Be specific to avoid catching non-map files like CommandLayout/mapcommands.json
+# SRPG Studio map files in this project follow the pattern Maps/map_XXX.json
+MAP_FILES = [
+ "map_", # e.g., Maps/map_000.json
+]
+
+
+def update_vocab_section(category: str, pairs: list[tuple[str, str]]):
+ """Update or insert a section in vocab.txt for the given category with provided pairs.
+ Only writes when there's an actual translation (dst is non-empty and differs from src after normalization).
+ - category: e.g., "Items", "Weapons", "Speakers", etc. Section header will be "# {category}".
+ - pairs: list of (source, translated) strings. Duplicates by source are deduped (last wins).
+ The existing section is replaced entirely; other sections are preserved.
+ """
+ try:
+ vocab_path = VOCAB_PATH
+
+ # Helper: normalized comparison to detect no-op translations
+ def _norm(s: str) -> str:
+ if s is None:
+ return ""
+ # Collapse whitespace and case-fold; leave punctuation to avoid over-matching
+ return re.sub(r"\s+", " ", str(s)).strip().casefold()
+
+ # Filter and deduplicate by source term (last mapping wins)
+ dedup: dict[str, str] = {}
+ for src, dst in pairs:
+ if not src:
+ continue
+ # Skip when no destination or no actual change
+ if dst is None or _norm(dst) == "" or _norm(dst) == _norm(src):
+ continue
+ dedup[src] = dst
+
+ # If nothing to add after filtering, skip touching the file
+ if not dedup:
+ return
+
+ # Guard the read-modify-write with a dedicated lock to avoid races
+ with VOCAB_LOCK:
+ existing = vocab_path.read_text(encoding="utf-8") if vocab_path.exists() else ""
+
+ lines = [f"{src} ({dst})" for src, dst in dedup.items()]
+ # Always terminate a section with a blank line to separate from next header
+ new_block = f"# {category}\n" + "\n".join(lines)
+ if not new_block.endswith("\n\n"):
+ if not new_block.endswith("\n"):
+ new_block += "\n"
+ new_block += "\n"
+
+ # Regex to find the specific section starting at the header for this category
+ # and ending right before the next header (any number of '#') or EOF.
+ # - Handles headers like '#Category', '# Category', '## Category', etc.
+ # - Uses non-greedy matching for the body to avoid spanning multiple sections.
+ pattern = re.compile(
+ rf"^[\t ]*#+\s*{re.escape(category)}\s*$\r?\n.*?(?=^[\t ]*#|\Z)",
+ re.MULTILINE | re.DOTALL,
+ )
+ if pattern.search(existing):
+ # Replace only the first matching section for this category.
+ updated = pattern.sub(lambda m: new_block, existing, count=1)
+ else:
+ updated = existing
+ if updated and not updated.endswith("\n\n"):
+ # Ensure a blank line before appending new section if file not empty
+ if not updated.endswith("\n"):
+ updated += "\n"
+ updated += "\n"
+ updated += new_block
+
+ # Avoid writing if nothing changed
+ if updated == existing:
+ return
+ # Atomic write: write to unique temp and replace with retries on Windows
+ tmp_path = vocab_path.with_suffix(vocab_path.suffix + f".{os.getpid()}.{threading.get_ident()}.tmp")
+ tmp_path.write_text(updated, encoding="utf-8")
+
+ attempts = 6
+ delay = 0.1
+ last_err = None
+ for attempt in range(attempts):
+ try:
+ os.replace(tmp_path, vocab_path)
+ last_err = None
+ break
+ except PermissionError as e:
+ last_err = e
+ # Try relaxing permissions then retry
+ try:
+ if vocab_path.exists():
+ os.chmod(vocab_path, 0o666)
+ except Exception:
+ pass
+ time.sleep(delay)
+ delay = min(1.0, delay * 2)
+ except Exception as e:
+ last_err = e
+ break
+ if last_err is not None:
+ try:
+ shutil.move(str(tmp_path), str(vocab_path))
+ except Exception:
+ try:
+ if tmp_path.exists():
+ tmp_path.unlink(missing_ok=True)
+ except Exception:
+ pass
+ raise last_err
+ except Exception:
+ traceback.print_exc()
+
+
+def handleSRPG(filename, estimate):
+ """
+ Main handler function for SRPG Studio files.
+
+ Args:
+ filename: Name of the file to translate
+ estimate: Boolean indicating if this is an estimate run
+
+ Returns:
+ String with translation results or error message
+ """
+ global ESTIMATE, TOKENS, FILENAME, TIMETOTAL
+ ESTIMATE = estimate
+ FILENAME = filename
+
+ # Translate
+ start = time.time()
+ translatedData = openFiles(filename)
+
+ # Write output file if not in estimate mode
+ if not estimate:
+ try:
+ with open("translated/" + filename, "w", encoding="utf-8", newline="\n") as outFile:
+ json.dump(translatedData[0], outFile, ensure_ascii=False, indent=4)
+ except Exception:
+ traceback.print_exc()
+ return "Fail"
+
+ # Print File
+ end = time.time()
+ tqdm.write(getResultString(translatedData, end - start, filename))
+ with LOCK:
+ TOKENS[0] += translatedData[1][0]
+ TOKENS[1] += translatedData[1][1]
+
+ # Print Total
+ totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
+
+ # Print any errors
+ if len(MISMATCH) > 0:
+ return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
+ else:
+ return totalString
+
+
+def openFiles(filename):
+ """
+ Opens and routes SRPG Studio files to appropriate parsing functions.
+
+ Args:
+ filename: Name of the file to open and parse
+
+ Returns:
+ Tuple of (translated data, token counts, error)
+ """
+ with open("files/" + filename, "r", encoding="utf-8-sig") as f:
+ data = json.load(f)
+
+ # Check if filename matches recollection pattern
+ if "recollection" in filename.lower():
+ translatedData = parseRecollection(data, filename)
+ # mapcommonevents is a list of events like recollection
+ elif "mapcommonevents" in filename.lower():
+ translatedData = parseRecollection(data, filename)
+ # Bookmark events use the same structure as recollection (list of events with pages/commands)
+ elif "bookmarkevents" in filename.lower():
+ translatedData = parseRecollection(data, filename)
+ # Event files (autoevents, placeevents, talkevents, communicationevents, openingevents, endingevents)
+ # all have name, desc, and pages->commands structure like recollection
+ elif any(event_type in filename.lower() for event_type in ["autoevents", "placeevents", "talkevents", "communicationevents", "openingevents", "endingevents"]):
+ translatedData = parseRecollection(data, filename)
+ # Check if filename matches bookmark pattern (top-level entries with name/desc + events)
+ elif "bookmark" in filename.lower():
+ translatedData = parseBookmark(data, filename)
+ # Players have the same shape as bookmark entries
+ elif "players" in filename.lower():
+ translatedData = parseBookmark(data, filename)
+ # Titles.json is a small dict of title strings
+ elif os.path.basename(filename).lower() == "titles.json":
+ translatedData = parseTitles(data, filename)
+ # Check if filename matches map pattern
+ elif any(pattern in filename.lower() for pattern in MAP_FILES):
+ translatedData = parseMap(data, filename)
+ # Check if filename matches any pattern in GENERIC_FILES
+ elif any(pattern in filename.lower() for pattern in GENERIC_FILES):
+ translatedData = parseGeneric(data, filename)
+
+ # TODO: Add other SRPG Studio file types here
+ else:
+ raise NameError(filename + " Not Supported")
+
+ return translatedData
+
+
+def parseBookmark(data, filename):
+ """
+ Parser for SRPG Studio bookmark.json files.
+ Structure: List of entries, each with id, name, desc, and events containing pages -> commands
+ where commands have data (array of strings) and optional speaker.
+
+ Args:
+ data: Parsed JSON data (list of bookmark entries)
+ filename: Name of the file being parsed
+
+ Returns:
+ Tuple of (translated data, token counts, error)
+ """
+ global PBAR
+ totalTokens = [0, 0]
+
+ pbar = None
+ try:
+ # Count work units: names, descs, dialogue lines, and speakers
+ total_units = 0
+
+ for entry in data:
+ if not entry:
+ continue
+
+ # name and desc
+ for field in ["name", "desc"]:
+ if field in entry and entry[field]:
+ total_units += 1
+
+ # events -> pages -> commands
+ if "events" in entry and isinstance(entry["events"], list):
+ for event in entry["events"]:
+ if not event:
+ continue
+
+ # Count event-level name and desc fields
+ for field in ["name", "desc"]:
+ if field in event and event[field]:
+ total_units += 1
+
+ if "pages" not in event or not isinstance(event["pages"], list):
+ continue
+
+ for page in event["pages"]:
+ if not page or "commands" not in page or not isinstance(page["commands"], list):
+ continue
+ for command in page["commands"]:
+ if not command:
+ continue
+ if "data" in command and isinstance(command["data"], list):
+ for text in command["data"]:
+ if text:
+ total_units += 1
+ if "speaker" in command and command["speaker"]:
+ total_units += 1
+
+ # Setup progress bar
+ with LOCK:
+ pbar = tqdm(
+ desc=filename,
+ total=total_units,
+ bar_format=BAR_FORMAT,
+ position=POSITION,
+ leave=LEAVE,
+ )
+ PBAR = pbar
+
+ # Translate using two-pass approach
+ result = translateBookmark(data, filename, pbar=pbar)
+ totalTokens[0] += result[0]
+ totalTokens[1] += result[1]
+
+ return (data, totalTokens, None)
+
+ except Exception as e:
+ traceback.print_exc()
+ return (data, totalTokens, e)
+ finally:
+ try:
+ if pbar is not None:
+ pbar.close()
+ except Exception:
+ pass
+ PBAR = None
+
+
+def translateBookmark(data, filename, translatedDataList=None, pbar=None):
+ """
+ Translates bookmark.json data structure.
+ Two-pass approach via recursion:
+ - Pass 1: Collect strings (names, descs, dialogue data with speaker prefix, speakers)
+ - Pass 2: Apply translations back into data
+
+ Returns:
+ [input tokens, output tokens]
+ """
+ totalTokens = [0, 0]
+
+ # Initialize or extract lists
+ if translatedDataList is None:
+ nameList = []
+ descList = []
+ dataList = []
+ speakerList = []
+ else:
+ nameList = translatedDataList[0]
+ descList = translatedDataList[1]
+ dataList = translatedDataList[2]
+ speakerList = translatedDataList[3]
+
+ for entry in data:
+ if not entry:
+ continue
+
+ # name
+ if "name" in entry and entry["name"]:
+ if translatedDataList is None:
+ nameList.append(entry["name"])
+ else:
+ if nameList:
+ entry["name"] = nameList[0]
+ nameList.pop(0)
+
+ # desc
+ if "desc" in entry and entry["desc"]:
+ if translatedDataList is None:
+ # Remove newlines for translation
+ descList.append(entry["desc"].replace("\n", " "))
+ else:
+ if descList:
+ translatedText = descList[0]
+ # Apply text wrapping
+ translatedText = dazedwrap.wrapText(translatedText, width=WIDTH)
+ entry["desc"] = translatedText
+ descList.pop(0)
+
+ # events -> pages -> commands
+ if "events" in entry and isinstance(entry["events"], list):
+ for event in entry["events"]:
+ if not event:
+ continue
+
+ # Handle event-level name field
+ if "name" in event and event["name"]:
+ if translatedDataList is None:
+ nameList.append(event["name"])
+ else:
+ if nameList:
+ event["name"] = nameList[0]
+ nameList.pop(0)
+
+ # Handle event-level desc field
+ if "desc" in event and event["desc"]:
+ if translatedDataList is None:
+ # Remove newlines for translation
+ descList.append(event["desc"].replace("\n", " "))
+ else:
+ if descList:
+ translatedText = descList[0]
+ # Apply text wrapping
+ translatedText = dazedwrap.wrapText(translatedText, width=WIDTH)
+ event["desc"] = translatedText
+ descList.pop(0)
+
+ if "pages" not in event or not isinstance(event["pages"], list):
+ continue
+
+ for page in event["pages"]:
+ if not page or "commands" not in page or not isinstance(page["commands"], list):
+ continue
+ for command in page["commands"]:
+ if not command:
+ continue
+
+ speaker = command.get("speaker", "")
+
+ # data array
+ if "data" in command and isinstance(command["data"], list):
+ for i, text in enumerate(command["data"]):
+ if text:
+ if translatedDataList is None:
+ text = text.replace("\n", " ")
+ if speaker:
+ dataList.append(f"[{speaker}]: {text}")
+ else:
+ dataList.append(text)
+ else:
+ if dataList:
+ translated = dataList[0]
+ if speaker:
+ match = re.search(r'(^\[.+?\]\s?[|:]\s?)', translated)
+ if match:
+ translated = translated.replace(match.group(1), "")
+ translated = dazedwrap.wrapText(translated, width=WIDTH)
+ command["data"][i] = translated
+ dataList.pop(0)
+
+ # speaker field
+ if "speaker" in command and command["speaker"]:
+ if translatedDataList is None:
+ speakerList.append(command["speaker"])
+ else:
+ if speakerList:
+ command["speaker"] = speakerList[0]
+ speakerList.pop(0)
+
+ # If this was Pass 1, perform translations and recurse
+ if translatedDataList is None:
+ originalNameCount = len(nameList)
+ originalDescCount = len(descList)
+ originalDataCount = len(dataList)
+ originalSpeakerCount = len(speakerList)
+
+ if nameList:
+ response = translateAI(
+ nameList,
+ "Reply with only the " + LANGUAGE + " translation of the bookmark name.",
+ True,
+ filename,
+ pbar,
+ )
+ nameList = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ if descList:
+ response = translateAI(
+ descList,
+ "Reply with only the " + LANGUAGE + " translation of the bookmark description.",
+ True,
+ filename,
+ pbar,
+ )
+ descList = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ if dataList:
+ response = translateAI(
+ dataList,
+ "Reply with only the " + LANGUAGE + " translation of the dialogue text.",
+ True,
+ filename,
+ pbar,
+ )
+ dataList = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ if speakerList:
+ response = translateAI(
+ speakerList,
+ "Reply with only the " + LANGUAGE + " translation of the speaker name.",
+ True,
+ filename,
+ pbar,
+ )
+ speakerList = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Mismatch checks
+ if len(nameList) != originalNameCount:
+ with LOCK:
+ if filename not in MISMATCH:
+ MISMATCH.append(filename)
+ if len(descList) != originalDescCount:
+ with LOCK:
+ if filename not in MISMATCH:
+ MISMATCH.append(filename)
+ if len(dataList) != originalDataCount:
+ with LOCK:
+ if filename not in MISMATCH:
+ MISMATCH.append(filename)
+ if len(speakerList) != originalSpeakerCount:
+ with LOCK:
+ if filename not in MISMATCH:
+ MISMATCH.append(filename)
+
+ # PASS 2
+ translateBookmark(data, filename, [nameList, descList, dataList, speakerList], pbar)
+
+ return totalTokens
+
+
+def parseGeneric(data, filename):
+ """
+ Generic parser for SRPG Studio files with id, name, desc structure.
+ Handles files like quests.json.
+ Uses a two-pass approach: first pass collects all strings to translate,
+ then batch translates them, then second pass applies translations.
+
+ Args:
+ data: Parsed JSON data (list of objects with id, name, desc)
+ filename: Name of the file being parsed
+
+ Returns:
+ Tuple of (data, token counts, error)
+ """
+ global PBAR
+
+ totalTokens = [0, 0]
+
+ pbar = None
+ try:
+ # Count work units (all translatable fields that need translation)
+ total_units = 0
+ translatable_fields = ["name", "desc", "commandName", "command"]
+
+ for entry in data:
+ if entry:
+ for field in translatable_fields:
+ if field in entry and entry[field]:
+ # If command is a list (e.g., fusionsettings), count per element
+ if field == "command" and isinstance(entry[field], list):
+ total_units += sum(1 for x in entry[field] if x)
+ # Otherwise simple increment
+ elif not isinstance(entry[field], list):
+ total_units += 1
+
+ # Handle pages array separately
+ if "pages" in entry and entry["pages"] and isinstance(entry["pages"], list):
+ for page in entry["pages"]:
+ if page:
+ total_units += 1
+
+ # Handle msg arrays (e.g., shoplayout)
+ if "msg" in entry and isinstance(entry["msg"], list):
+ total_units += sum(1 for m in entry["msg"] if m)
+
+ # Handle rewardData arrays (e.g., quests)
+ if "rewardData" in entry and isinstance(entry["rewardData"], list):
+ total_units += sum(1 for r in entry["rewardData"] if r)
+
+ # Handle customParameters name field (e.g., {name:'シャルロット強制売春'})
+ if "customParameters" in entry and entry["customParameters"]:
+ match = re.search(r"name:\s*['\"]([^'\"]+)['\"]", entry["customParameters"])
+ if match:
+ total_units += 1
+
+ # Handle terrains array (nested structure)
+ if "terrains" in entry and entry["terrains"] and isinstance(entry["terrains"], list):
+ for terrain in entry["terrains"]:
+ if terrain:
+ for field in ["name", "desc"]:
+ if field in terrain and terrain[field]:
+ total_units += 1
+
+ # Setup progress bar (use a per-file instance to avoid cross-thread clashes)
+ with LOCK:
+ pbar = tqdm(
+ desc=filename,
+ total=total_units,
+ bar_format=BAR_FORMAT,
+ position=POSITION,
+ leave=LEAVE,
+ )
+ PBAR = pbar
+
+ # Translate the data using two-pass approach
+ result = translateGeneric(data, filename, pbar=pbar)
+ totalTokens[0] += result[0]
+ totalTokens[1] += result[1]
+
+ return (data, totalTokens, None)
+
+ except Exception as e:
+ traceback.print_exc()
+ return (data, totalTokens, e)
+ finally:
+ # Ensure progress bar is closed
+ try:
+ if pbar is not None:
+ pbar.close()
+ except Exception:
+ pass
+ PBAR = None
+
+
+def translateGeneric(data, filename, translatedDataList=None, pbar=None):
+ """
+ Translates generic SRPG Studio data with id, name, desc, commandName, command, pages structure.
+ Uses two-pass approach via recursion:
+ - Pass 1 (translatedDataList=None): Collect strings and batch translate
+ - Pass 2 (translatedDataList set): Apply translations back to the data
+
+ Args:
+ data: List of objects with id, name, desc, commandName, command, pages keys
+ filename: Name of the file being translated
+ translatedDataList: List containing [nameList, descList, commandNameList, commandList, pagesList]
+ - Pass 1: Empty lists to collect originals
+ - Pass 2: Filled lists with translations
+
+ Returns:
+ Tuple of [input tokens, output tokens]
+ """
+ global PBAR
+
+ totalTokens = [0, 0]
+
+ # Initialize or extract lists
+ if translatedDataList is None:
+ # PASS 1: Create empty lists to collect strings
+ nameList = []
+ descList = []
+ commandNameList = []
+ commandList = []
+ pagesList = []
+ msgList = []
+ commandArrayList = []
+ rewardDataList = []
+ customParametersNameList = []
+ else:
+ # PASS 2: Use provided translated lists
+ nameList = translatedDataList[0]
+ descList = translatedDataList[1]
+ commandNameList = translatedDataList[2]
+ commandList = translatedDataList[3]
+ pagesList = translatedDataList[4]
+ msgList = translatedDataList[5]
+ commandArrayList = translatedDataList[6]
+ rewardDataList = translatedDataList[7]
+ customParametersNameList = translatedDataList[8]
+
+ # Single loop - behavior depends on which pass we're in
+ for entry in data:
+ if not entry:
+ continue
+
+ # Handle name field
+ if "name" in entry and entry["name"]:
+ # PASS 1: Collect original
+ if translatedDataList is None:
+ nameList.append(entry["name"])
+ # PASS 2: Apply translation
+ else:
+ if nameList:
+ entry["name"] = nameList[0]
+ nameList.pop(0)
+
+ # Handle desc field
+ if "desc" in entry and entry["desc"]:
+ # PASS 1: Collect original
+ if translatedDataList is None:
+ # Nuke Wordwrap
+ descList.append(entry["desc"].replace("\n", " "))
+ # PASS 2: Apply translation
+ else:
+ if descList:
+ translatedText = descList[0]
+ # Wordwrap
+ translatedText = dazedwrap.wrapText(translatedText, width=WIDTH)
+
+ # Set Data
+ entry["desc"] = translatedText
+ descList.pop(0)
+
+ # Handle commandName field
+ if "commandName" in entry and entry["commandName"]:
+ # PASS 1: Collect original
+ if translatedDataList is None:
+ commandNameList.append(entry["commandName"])
+ # PASS 2: Apply translation
+ else:
+ if commandNameList:
+ entry["commandName"] = commandNameList[0]
+ commandNameList.pop(0)
+
+ # Handle command field (string or list)
+ if "command" in entry and entry["command"] is not None:
+ # If list of strings
+ if isinstance(entry["command"], list):
+ for i, val in enumerate(entry["command"]):
+ if val:
+ if translatedDataList is None:
+ # Nuke Wrap
+ commandArrayList.append(val.replace("\n", " "))
+ else:
+ if commandArrayList:
+ translatedText = dazedwrap.wrapText(commandArrayList[0], width=WIDTH)
+ entry["command"][i] = translatedText
+ commandArrayList.pop(0)
+ # If simple string
+ elif isinstance(entry["command"], str):
+ if translatedDataList is None:
+ commandList.append(entry["command"])
+ else:
+ if commandList:
+ entry["command"] = commandList[0]
+ commandList.pop(0)
+
+ # Handle pages field (array of strings)
+ if "pages" in entry and entry["pages"] and isinstance(entry["pages"], list):
+ for i, page in enumerate(entry["pages"]):
+ if page:
+ # PASS 1: Collect original
+ if translatedDataList is None:
+ # Nuke Wordwrap
+ page = page.replace("\n", " ")
+ pagesList.append(page)
+ # PASS 2: Apply translation
+ else:
+ if pagesList:
+ translatedText = pagesList[0]
+
+ # Wordwrap
+ translatedText = dazedwrap.wrapText(translatedText, width=WIDTH)
+
+ # Set Data
+ entry["pages"][i] = translatedText
+ pagesList.pop(0)
+
+ # Handle msg field (array of strings)
+ if "msg" in entry and isinstance(entry["msg"], list):
+ for i, val in enumerate(entry["msg"]):
+ if val is not None:
+ if translatedDataList is None:
+ msgList.append(str(val).replace("\n", " "))
+ else:
+ if msgList:
+ translatedText = dazedwrap.wrapText(msgList[0], width=WIDTH)
+ entry["msg"][i] = translatedText
+ msgList.pop(0)
+
+ # Handle rewardData field (array of strings)
+ if "rewardData" in entry and isinstance(entry["rewardData"], list):
+ for i, val in enumerate(entry["rewardData"]):
+ if val:
+ # PASS 1: Collect original
+ if translatedDataList is None:
+ rewardDataList.append(str(val).replace("\n", " "))
+ # PASS 2: Apply translation
+ else:
+ if rewardDataList:
+ translatedText = rewardDataList[0]
+ # Set Data (no wordwrap for reward data)
+ entry["rewardData"][i] = translatedText
+ rewardDataList.pop(0)
+
+ # Handle customParameters name field (e.g., {name:'シャルロット強制売春'})
+ if "customParameters" in entry and entry["customParameters"]:
+ match = re.search(r"name:\s*['\"]([^'\"]+)['\"]", entry["customParameters"])
+ if match:
+ # PASS 1: Collect original
+ if translatedDataList is None:
+ customParametersNameList.append(match.group(1))
+ # PASS 2: Apply translation
+ else:
+ if customParametersNameList:
+ translatedName = customParametersNameList[0]
+ # Replace the name value in customParameters
+ entry["customParameters"] = re.sub(
+ r"(name:\s*['\"])([^'\"]+)(['\"])",
+ r"\1" + translatedName.replace("\\", "\\\\") + r"\3",
+ entry["customParameters"]
+ )
+ customParametersNameList.pop(0)
+
+ # Handle terrains field (nested array with name and desc)
+ if "terrains" in entry and entry["terrains"] and isinstance(entry["terrains"], list):
+ for terrain in entry["terrains"]:
+ if not terrain:
+ continue
+
+ # Handle terrain name
+ if "name" in terrain and terrain["name"]:
+ # PASS 1: Collect original
+ if translatedDataList is None:
+ nameList.append(terrain["name"])
+ # PASS 2: Apply translation
+ else:
+ if nameList:
+ terrain["name"] = nameList[0]
+ nameList.pop(0)
+
+ # Handle terrain desc
+ if "desc" in terrain and terrain["desc"]:
+ # PASS 1: Collect original
+ if translatedDataList is None:
+ descList.append(terrain["desc"]).replace("\n", " ")
+ # PASS 2: Apply translation
+ else:
+ if descList:
+ translatedText = descList[0]
+ # Wordwrap
+ translatedText = dazedwrap.wrapText(translatedText, width=WIDTH)
+
+ # Set Data
+ terrain["desc"] = translatedText
+ descList.pop(0)
+
+ # If this was Pass 1, do the translation and recurse for Pass 2
+ if translatedDataList is None:
+ # Store original counts for mismatch checking
+ originalNameCount = len(nameList)
+ originalDescCount = len(descList)
+ originalCommandNameCount = len(commandNameList)
+ originalCommandCount = len(commandList)
+ originalPagesCount = len(pagesList)
+ originalMsgCount = len(msgList)
+ originalCommandArrayCount = len(commandArrayList)
+ originalRewardDataCount = len(rewardDataList)
+ originalCustomParametersNameCount = len(customParametersNameList)
+
+ # Keep a copy of original names for vocab update (for characters/items/skills/classes/weapons)
+ vocab_name_files = ["characters", "items", "skills", "classes", "weapons"]
+ originalNameList = nameList.copy() if any(tag in filename.lower() for tag in vocab_name_files) else []
+
+ # Batch translate names
+ if nameList:
+ response = translateAI(
+ nameList,
+ "Reply with only the " + LANGUAGE + " translation of the quest name.",
+ True,
+ filename,
+ pbar
+ )
+ nameList = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Update vocab.txt for name-bearing files
+ if originalNameList and nameList:
+ try:
+ file_lower = filename.lower()
+ section = None
+ if "characters" in file_lower:
+ section = "Speakers"
+ elif "items" in file_lower:
+ section = "Items"
+ elif "skills" in file_lower:
+ section = "Skills"
+ elif "classes" in file_lower:
+ section = "Classes"
+ elif "weapons" in file_lower:
+ section = "Weapons"
+
+ if section:
+ vocab_pairs = list(zip(originalNameList, nameList))
+ update_vocab_section(section, vocab_pairs)
+ except Exception:
+ traceback.print_exc()
+
+ # Batch translate descriptions
+ if descList:
+ response = translateAI(
+ descList,
+ "Reply with only the " + LANGUAGE + " translation of the quest description.",
+ True,
+ filename,
+ pbar
+ )
+ descList = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Batch translate command names
+ if commandNameList:
+ response = translateAI(
+ commandNameList,
+ "Reply with only the " + LANGUAGE + " translation of the command name.",
+ True,
+ filename,
+ pbar
+ )
+ commandNameList = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Batch translate commands
+ if commandList:
+ response = translateAI(
+ commandList,
+ "Reply with only the " + LANGUAGE + " translation of the command.",
+ True,
+ filename,
+ pbar
+ )
+ commandList = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Batch translate pages
+ if pagesList:
+ response = translateAI(
+ pagesList,
+ "Reply with only the " + LANGUAGE + " translation of the page content.",
+ True,
+ filename,
+ pbar
+ )
+ pagesList = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Batch translate msg arrays
+ if msgList:
+ response = translateAI(
+ msgList,
+ "Reply with only the " + LANGUAGE + " translation of the message text.",
+ True,
+ filename,
+ pbar
+ )
+ msgList = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Batch translate command arrays
+ if commandArrayList:
+ response = translateAI(
+ commandArrayList,
+ "Reply with only the " + LANGUAGE + " translation of the command text.",
+ True,
+ filename,
+ pbar
+ )
+ commandArrayList = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Batch translate rewardData arrays
+ if rewardDataList:
+ response = translateAI(
+ rewardDataList,
+ "Reply with only the " + LANGUAGE + " translation of the reward data text.",
+ True,
+ filename,
+ pbar
+ )
+ rewardDataList = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Batch translate customParameters name fields
+ if customParametersNameList:
+ response = translateAI(
+ customParametersNameList,
+ "Reply with only the " + LANGUAGE + " translation of the custom parameter name.",
+ True,
+ filename,
+ pbar
+ )
+ customParametersNameList = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Check for mismatch errors
+ if len(nameList) != originalNameCount:
+ with LOCK:
+ if filename not in MISMATCH:
+ MISMATCH.append(filename)
+
+ if len(descList) != originalDescCount:
+ with LOCK:
+ if filename not in MISMATCH:
+ MISMATCH.append(filename)
+
+ if len(commandNameList) != originalCommandNameCount:
+ with LOCK:
+ if filename not in MISMATCH:
+ MISMATCH.append(filename)
+
+ if len(commandList) != originalCommandCount:
+ with LOCK:
+ if filename not in MISMATCH:
+ MISMATCH.append(filename)
+
+ if len(pagesList) != originalPagesCount:
+ with LOCK:
+ if filename not in MISMATCH:
+ MISMATCH.append(filename)
+ if len(msgList) != originalMsgCount:
+ with LOCK:
+ if filename not in MISMATCH:
+ MISMATCH.append(filename)
+ if len(commandArrayList) != originalCommandArrayCount:
+ with LOCK:
+ if filename not in MISMATCH:
+ MISMATCH.append(filename)
+ if len(rewardDataList) != originalRewardDataCount:
+ with LOCK:
+ if filename not in MISMATCH:
+ MISMATCH.append(filename)
+ if len(customParametersNameList) != originalCustomParametersNameCount:
+ with LOCK:
+ if filename not in MISMATCH:
+ MISMATCH.append(filename)
+
+ # PASS 2: Recursively call to apply translations
+ translateGeneric(
+ data,
+ filename,
+ [nameList, descList, commandNameList, commandList, pagesList, msgList, commandArrayList, rewardDataList, customParametersNameList],
+ pbar,
+ )
+
+ return totalTokens
+
+
+def parseTitles(data, filename):
+ """Parser for titles.json (small dict of title strings)."""
+ global PBAR
+ totalTokens = [0, 0]
+ pbar = None
+ try:
+ # Collect fields to translate
+ keys = [k for k in ["windowTitle", "gameTitle", "saveFileTitle"] if k in data and data[k]]
+ values = [data[k] for k in keys]
+
+ with LOCK:
+ pbar = tqdm(
+ desc=filename,
+ total=len(values),
+ bar_format=BAR_FORMAT,
+ position=POSITION,
+ leave=LEAVE,
+ )
+ PBAR = pbar
+
+ if values:
+ response = translateAI(
+ values,
+ "Reply with only the " + LANGUAGE + " translation of the title text.",
+ True,
+ filename,
+ pbar,
+ )
+ translations = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Apply
+ for i, k in enumerate(keys):
+ data[k] = translations[i]
+
+ return (data, totalTokens, None)
+ except Exception as e:
+ traceback.print_exc()
+ return (data, totalTokens, e)
+ finally:
+ try:
+ if pbar is not None:
+ pbar.close()
+ except Exception:
+ pass
+ PBAR = None
+
+
+def parseRecollection(data, filename):
+ """
+ Parser for SRPG Studio recollection.json files.
+ Structure: List of entries, each with pages containing commands with data arrays and speakers.
+
+ Args:
+ data: Parsed JSON data (list of objects with id, pages structure)
+ filename: Name of the file being parsed
+
+ Returns:
+ Tuple of (data, token counts, error)
+ """
+ global PBAR
+
+ totalTokens = [0, 0]
+
+ pbar = None
+ try:
+ # Count work units (data entries and speakers that need translation)
+ total_units = 0
+
+ # Iterate through structure and count
+ for entry in data:
+ if not entry:
+ continue
+
+ # Count name and desc fields at entry level
+ for field in ["name", "desc"]:
+ if field in entry and entry[field]:
+ total_units += 1
+
+ # Count customParameters hints
+ if "customParameters" in entry and entry["customParameters"]:
+ match = re.search(r'hint:"((?:[^"\\]|\\.)*)"', entry["customParameters"])
+ if match:
+ total_units += 1
+
+ if "pages" not in entry or not isinstance(entry["pages"], list):
+ continue
+
+ for page in entry["pages"]:
+ if not page or "commands" not in page or not isinstance(page["commands"], list):
+ continue
+
+ for command in page["commands"]:
+ if not command:
+ continue
+
+ # Count data array items (count any non-empty text; do not gate on LANGREGEX)
+ if "data" in command and isinstance(command["data"], list):
+ for text in command["data"]:
+ if text:
+ total_units += 1
+
+ # Count speaker field (count any non-empty speaker; do not gate on LANGREGEX)
+ if "speaker" in command and command["speaker"]:
+ total_units += 1
+
+ # Setup progress bar (per-file instance)
+ with LOCK:
+ pbar = tqdm(
+ desc=filename,
+ total=total_units,
+ bar_format=BAR_FORMAT,
+ position=POSITION,
+ leave=LEAVE,
+ )
+ PBAR = pbar
+
+ # Translate the data using two-pass approach
+ result = translateRecollection(data, filename, pbar=pbar)
+ totalTokens[0] += result[0]
+ totalTokens[1] += result[1]
+
+ return (data, totalTokens, None)
+
+ except Exception as e:
+ traceback.print_exc()
+ return (data, totalTokens, e)
+ finally:
+ try:
+ if pbar is not None:
+ pbar.close()
+ except Exception:
+ pass
+ PBAR = None
+
+
+def translateRecollection(data, filename, translatedDataList=None, pbar=None):
+ """
+ Translates recollection.json data structure.
+ Uses two-pass approach via recursion:
+ - Pass 1 (translatedDataList=None): Collect strings and batch translate
+ - Pass 2 (translatedDataList set): Apply translations back to the data
+
+ Args:
+ data: List of objects with name, desc, customParameters, and pages->commands->data structure
+ filename: Name of the file being translated
+ translatedDataList: List containing [nameList, descList, dataList, speakerList, customParamsList]
+ - Pass 1: Empty lists to collect originals
+ - Pass 2: Filled lists with translations
+
+ Returns:
+ Tuple of [input tokens, output tokens]
+ """
+ global PBAR
+
+ totalTokens = [0, 0]
+
+ # Initialize or extract lists
+ if translatedDataList is None:
+ # PASS 1: Create empty lists to collect strings
+ nameList = []
+ descList = []
+ dataList = []
+ speakerList = []
+ customParamsList = []
+ originalSpeakerList = [] # For vocab update
+ else:
+ # PASS 2: Use provided translated lists
+ nameList = translatedDataList[0]
+ descList = translatedDataList[1]
+ dataList = translatedDataList[2]
+ speakerList = translatedDataList[3]
+ customParamsList = translatedDataList[4]
+
+ # Single loop - behavior depends on which pass we're in
+ for entry in data:
+ if not entry:
+ continue
+
+ # Handle name field
+ if "name" in entry and entry["name"]:
+ # PASS 1: Collect original
+ if translatedDataList is None:
+ nameList.append(entry["name"])
+ # PASS 2: Apply translation
+ else:
+ if nameList:
+ entry["name"] = nameList[0]
+ nameList.pop(0)
+
+ # Handle desc field
+ if "desc" in entry and entry["desc"]:
+ # PASS 1: Collect original
+ if translatedDataList is None:
+ # Remove newlines for translation
+ descList.append(entry["desc"].replace("\n", " "))
+ # PASS 2: Apply translation
+ else:
+ if descList:
+ translatedText = descList[0]
+ # Apply text wrapping
+ translatedText = dazedwrap.wrapText(translatedText, width=WIDTH)
+ entry["desc"] = translatedText
+ descList.pop(0)
+
+ # Handle customParameters at entry level
+ if "customParameters" in entry and entry["customParameters"]:
+ # PASS 1: Collect hint text
+ if translatedDataList is None:
+ # Extract hint value using regex (captures content between hint:" and ")
+ match = re.search(r'hint:"((?:[^"\\]|\\.)*)"', entry["customParameters"])
+ if match:
+ hintText = match.group(1)
+ hintText = hintText.replace("\\n", " ")
+ customParamsList.append(hintText)
+ else:
+ # No hint found, add empty string as placeholder
+ customParamsList.append("")
+ # PASS 2: Apply translation
+ else:
+ if customParamsList:
+ translatedHint = customParamsList[0]
+ customParamsList.pop(0)
+
+ if translatedHint: # Only replace if we translated something
+ # Replace double quotes with single quotes since \" is used as delimiter
+ translatedHint = translatedHint.replace('"', "'")
+ # Wrap text using dazedwrap with WIDTH and replace newlines with \\n
+ translatedHint = dazedwrap.wrapText(translatedHint, width=WIDTH)
+ translatedHint = translatedHint.replace("\n", "\\\\n")
+ # Replace the hint value in customParameters
+ entry["customParameters"] = re.sub(
+ r'(hint:")((?:[^"\\]|\\.)*)"',
+ r'\1' + translatedHint + '"',
+ entry["customParameters"]
+ )
+
+ # Check if pages exists before processing
+ if "pages" not in entry or not isinstance(entry["pages"], list):
+ continue
+
+ for page in entry["pages"]:
+ if not page or "commands" not in page:
+ continue
+
+ if not isinstance(page["commands"], list):
+ continue
+
+ for command in page["commands"]:
+ if not command:
+ continue
+
+ # Get the speaker for this command (if available)
+ speaker = command.get("speaker", "")
+
+ # Handle data array
+ if "data" in command and isinstance(command["data"], list):
+ for i, text in enumerate(command["data"]):
+ if text:
+ # PASS 1: Collect original with speaker prefix
+ if translatedDataList is None:
+ # Remove Wrap
+ text = text.replace("\n", " ")
+
+ # Attach speaker to data for translation
+ if speaker:
+ dataList.append(f"[{speaker}]: {text}")
+ else:
+ dataList.append(text)
+
+ # PASS 2: Apply translation and strip speaker prefix
+ else:
+ if dataList:
+ translated = dataList[0]
+
+ # Remove speaker
+ if speaker:
+ match = re.search(r'(^\[.+?\]\s?[|:]\s?)', translated)
+ if match:
+ translated = translated.replace(match.group(1), "")
+
+ # Textwrap
+ translated = dazedwrap.wrapText(translated, width=WIDTH)
+
+ # Set Data
+ command["data"][i] = translated
+ dataList.pop(0)
+
+ # Handle speaker field
+ if "speaker" in command and command["speaker"]:
+ # PASS 1: Collect original
+ if translatedDataList is None:
+ originalSpeakerList.append(command["speaker"])
+ speakerList.append(command["speaker"])
+ # PASS 2: Apply translation
+ else:
+ if speakerList:
+ command["speaker"] = speakerList[0]
+ speakerList.pop(0)
+
+ # If this was Pass 1, do the translation and recurse for Pass 2
+ if translatedDataList is None:
+ # Store original counts for mismatch checking
+ originalNameCount = len(nameList)
+ originalDescCount = len(descList)
+ originalDataCount = len(dataList)
+ originalSpeakerCount = len(speakerList)
+ originalCustomParamsCount = len(customParamsList)
+
+ # Batch translate names
+ if nameList:
+ response = translateAI(
+ nameList,
+ "Reply with only the " + LANGUAGE + " translation of the name.",
+ True,
+ filename,
+ pbar
+ )
+ nameList = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Batch translate descriptions
+ if descList:
+ response = translateAI(
+ descList,
+ "Reply with only the " + LANGUAGE + " translation of the description.",
+ True,
+ filename,
+ pbar
+ )
+ descList = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Batch translate data text
+ if dataList:
+ response = translateAI(
+ dataList,
+ "Reply with only the " + LANGUAGE + " translation of the dialogue text.",
+ True,
+ filename,
+ pbar
+ )
+ dataList = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Batch translate speakers
+ if speakerList:
+ response = translateAI(
+ speakerList,
+ "Reply with only the " + LANGUAGE + " translation of the speaker name.",
+ True,
+ filename,
+ pbar
+ )
+ speakerList = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Batch translate customParameters hints (only non-empty ones)
+ if customParamsList:
+ # Filter out empty strings for translation
+ hintsToTranslate = [h for h in customParamsList if h]
+ if hintsToTranslate:
+ response = translateAI(
+ hintsToTranslate,
+ "Reply with only the " + LANGUAGE + " translation of the hint text.",
+ True,
+ filename,
+ pbar
+ )
+ translatedHints = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Reconstruct customParamsList with translations in place
+ translatedIndex = 0
+ for i, hint in enumerate(customParamsList):
+ if hint: # If it was non-empty, use the translation
+ customParamsList[i] = translatedHints[translatedIndex]
+ translatedIndex += 1
+
+ # Check for mismatch errors
+ if len(nameList) != originalNameCount:
+ with LOCK:
+ if filename not in MISMATCH:
+ MISMATCH.append(filename)
+
+ if len(descList) != originalDescCount:
+ with LOCK:
+ if filename not in MISMATCH:
+ MISMATCH.append(filename)
+
+ if len(dataList) != originalDataCount:
+ with LOCK:
+ if filename not in MISMATCH:
+ MISMATCH.append(filename)
+
+ if len(speakerList) != originalSpeakerCount:
+ with LOCK:
+ if filename not in MISMATCH:
+ MISMATCH.append(filename)
+
+ if len(customParamsList) != originalCustomParamsCount:
+ with LOCK:
+ if filename not in MISMATCH:
+ MISMATCH.append(filename)
+
+ # PASS 2: Recursively call to apply translations
+ translateRecollection(data, filename, [nameList, descList, dataList, speakerList, customParamsList], pbar)
+
+ return totalTokens
+
+
+def parseMap(data, filename):
+ """
+ Parser for SRPG Studio map.json files.
+ Structure: Single object with:
+ - id, name, desc, mapName
+ - victoryConds (array of strings)
+ - defeatConds (array of strings)
+ - EnemyUnits (array of objects with id, name, desc, events structure similar to recollection)
+
+ Args:
+ data: Parsed JSON data (single map object)
+ filename: Name of the file being parsed
+
+ Returns:
+ Tuple of (data, token counts, error)
+ """
+ global PBAR
+
+ totalTokens = [0, 0]
+
+ pbar = None
+ try:
+ # Count work units (all translatable fields)
+ total_units = 0
+
+ # Count top-level fields: desc, mapName (name is an identifier and should not be translated)
+ for field in ["desc", "mapName"]:
+ if field in data and data[field]:
+ total_units += 1
+
+ # Count victoryConds array items
+ if "victoryConds" in data and isinstance(data["victoryConds"], list):
+ for cond in data["victoryConds"]:
+ if cond:
+ total_units += 1
+
+ # Count defeatConds array items
+ if "defeatConds" in data and isinstance(data["defeatConds"], list):
+ for cond in data["defeatConds"]:
+ if cond:
+ total_units += 1
+
+ # Count units with events (EnemyUnits, EvEnemyUnits)
+ for unitsKey in ["EnemyUnits", "EvEnemyUnits"]:
+ if unitsKey in data and isinstance(data[unitsKey], list):
+ for unit in data[unitsKey]:
+ if not unit:
+ continue
+
+ # Count unit name and desc
+ for field in ["name", "desc"]:
+ if field in unit and unit[field]:
+ total_units += 1
+
+ # Count events->pages->commands->data and speaker
+ if "events" in unit and isinstance(unit["events"], list):
+ for event in unit["events"]:
+ if not event or "pages" not in event:
+ continue
+
+ if not isinstance(event["pages"], list):
+ continue
+
+ for page in event["pages"]:
+ if not page or "commands" not in page:
+ continue
+
+ if not isinstance(page["commands"], list):
+ continue
+
+ for command in page["commands"]:
+ if not command:
+ continue
+
+ # Count data array items
+ if "data" in command and isinstance(command["data"], list):
+ for text in command["data"]:
+ if text:
+ total_units += 1
+
+ # Count speaker field
+ if "speaker" in command and command["speaker"]:
+ total_units += 1
+
+ # Count events (placeEvents, autoEvents, openingEvents, communicationEvents)
+ for eventsKey in ["placeEvents", "autoEvents", "openingEvents", "communicationEvents"]:
+ if eventsKey in data and isinstance(data[eventsKey], list):
+ for event in data[eventsKey]:
+ if not event:
+ continue
+
+ # Count event-level name and desc fields
+ for field in ["name", "desc"]:
+ if field in event and event[field]:
+ total_units += 1
+
+ if "pages" not in event or not isinstance(event["pages"], list):
+ continue
+
+ for page in event["pages"]:
+ if not page or "commands" not in page:
+ continue
+
+ if not isinstance(page["commands"], list):
+ continue
+
+ for command in page["commands"]:
+ if not command:
+ continue
+
+ # Count data array items
+ if "data" in command and isinstance(command["data"], list):
+ for text in command["data"]:
+ if text:
+ total_units += 1
+
+ # Count speaker field
+ if "speaker" in command and command["speaker"]:
+ total_units += 1
+
+ # Setup progress bar (per-file instance)
+ with LOCK:
+ pbar = tqdm(
+ desc=filename,
+ total=total_units,
+ bar_format=BAR_FORMAT,
+ position=POSITION,
+ leave=LEAVE,
+ )
+ PBAR = pbar
+
+ # Translate the data using two-pass approach
+ result = translateMap(data, filename, pbar=pbar)
+ totalTokens[0] += result[0]
+ totalTokens[1] += result[1]
+
+ return (data, totalTokens, None)
+
+ except Exception as e:
+ traceback.print_exc()
+ return (data, totalTokens, e)
+ finally:
+ try:
+ if pbar is not None:
+ pbar.close()
+ except Exception:
+ pass
+ PBAR = None
+
+
+def translateMap(data, filename, translatedDataList=None, pbar=None):
+ """
+ Translates map.json data structure.
+ Uses two-pass approach via recursion:
+ - Pass 1 (translatedDataList=None): Collect strings and batch translate
+ - Pass 2 (translatedDataList set): Apply translations back to the data
+
+ Args:
+ data: Single map object with name, desc, mapName, victoryConds, defeatConds, and EnemyUnits
+ filename: Name of the file being translated
+ translatedDataList: List containing translation lists
+ - Pass 1: Empty lists to collect originals
+ - Pass 2: Filled lists with translations
+
+ Returns:
+ Tuple of [input tokens, output tokens]
+ """
+ global PBAR
+
+ totalTokens = [0, 0]
+
+ # Initialize or extract lists
+ if translatedDataList is None:
+ # PASS 1: Create empty lists to collect strings
+ descList = []
+ mapNameList = []
+ victoryCondsList = []
+ defeatCondsList = []
+ unitNameList = []
+ unitDescList = []
+ eventNameList = []
+ eventDescList = []
+ dataList = []
+ speakerList = []
+ originalSpeakerList = [] # For vocab update
+ else:
+ # PASS 2: Use provided translated lists
+ descList = translatedDataList[0]
+ mapNameList = translatedDataList[1]
+ victoryCondsList = translatedDataList[2]
+ defeatCondsList = translatedDataList[3]
+ unitNameList = translatedDataList[4]
+ unitDescList = translatedDataList[5]
+ eventNameList = translatedDataList[6]
+ eventDescList = translatedDataList[7]
+ dataList = translatedDataList[8]
+ speakerList = translatedDataList[9]
+
+ # Note: name field is not translated as it's an identifier (e.g., "ch3_瘴気の森")
+
+ # Handle desc field
+ if "desc" in data and data["desc"]:
+ # PASS 1: Collect original
+ if translatedDataList is None:
+ descList.append(data["desc"].replace("\n", " "))
+ # PASS 2: Apply translation
+ else:
+ if descList:
+ translatedText = descList[0]
+ # Wordwrap
+ translatedText = dazedwrap.wrapText(translatedText, width=WIDTH)
+ data["desc"] = translatedText
+ descList.pop(0)
+
+ # Handle mapName field
+ if "mapName" in data and data["mapName"]:
+ # PASS 1: Collect original
+ if translatedDataList is None:
+ mapNameList.append(data["mapName"])
+ # PASS 2: Apply translation
+ else:
+ if mapNameList:
+ data["mapName"] = mapNameList[0]
+ mapNameList.pop(0)
+
+ # Handle victoryConds array
+ if "victoryConds" in data and isinstance(data["victoryConds"], list):
+ for i, cond in enumerate(data["victoryConds"]):
+ if cond:
+ # PASS 1: Collect original
+ if translatedDataList is None:
+ victoryCondsList.append(cond)
+ # PASS 2: Apply translation
+ else:
+ if victoryCondsList:
+ data["victoryConds"][i] = victoryCondsList[0]
+ victoryCondsList.pop(0)
+
+ # Handle defeatConds array
+ if "defeatConds" in data and isinstance(data["defeatConds"], list):
+ for i, cond in enumerate(data["defeatConds"]):
+ if cond:
+ # PASS 1: Collect original
+ if translatedDataList is None:
+ defeatCondsList.append(cond)
+ # PASS 2: Apply translation
+ else:
+ if defeatCondsList:
+ data["defeatConds"][i] = defeatCondsList[0]
+ defeatCondsList.pop(0)
+
+ # Process all unit arrays (EnemyUnits, EvEnemyUnits)
+ for unitsKey in ["EnemyUnits", "EvEnemyUnits"]:
+ if unitsKey in data and isinstance(data[unitsKey], list):
+ for unit in data[unitsKey]:
+ if not unit:
+ continue
+
+ # Handle unit name
+ if "name" in unit and unit["name"]:
+ if translatedDataList is None:
+ unitNameList.append(unit["name"])
+ else:
+ if unitNameList:
+ unit["name"] = unitNameList[0]
+ unitNameList.pop(0)
+
+ # Handle unit desc
+ if "desc" in unit and unit["desc"]:
+ if translatedDataList is None:
+ unitDescList.append(unit["desc"].replace("\n", " "))
+ else:
+ if unitDescList:
+ translatedText = unitDescList[0]
+ # Wordwrap
+ translatedText = dazedwrap.wrapText(translatedText, width=WIDTH)
+ unit["desc"] = translatedText
+ unitDescList.pop(0)
+
+ # Process events within units
+ if "events" in unit and isinstance(unit["events"], list):
+ for event in unit["events"]:
+ if not event or "pages" not in event:
+ continue
+
+ if not isinstance(event["pages"], list):
+ continue
+
+ for page in event["pages"]:
+ if not page or "commands" not in page:
+ continue
+
+ if not isinstance(page["commands"], list):
+ continue
+
+ for command in page["commands"]:
+ if not command:
+ continue
+
+ speaker = command.get("speaker", "")
+
+ # Handle data array
+ if "data" in command and isinstance(command["data"], list):
+ for i, text in enumerate(command["data"]):
+ if text:
+ if translatedDataList is None:
+ text = text.replace("\n", " ")
+ if speaker:
+ dataList.append(f"[{speaker}]: {text}")
+ else:
+ dataList.append(text)
+ else:
+ if dataList:
+ translated = dataList[0]
+ if speaker:
+ match = re.search(r'(^\[.+?\]\s?[|:]\s?)', translated)
+ if match:
+ translated = translated.replace(match.group(1), "")
+ translated = dazedwrap.wrapText(translated, width=WIDTH)
+ command["data"][i] = translated
+ dataList.pop(0)
+
+ # Handle speaker field
+ if "speaker" in command and command["speaker"]:
+ if translatedDataList is None:
+ originalSpeakerList.append(command["speaker"])
+ speakerList.append(command["speaker"])
+ else:
+ if speakerList:
+ command["speaker"] = speakerList[0]
+ speakerList.pop(0)
+
+ # Process all event arrays (placeEvents, autoEvents, openingEvents, communicationEvents)
+ for eventsKey in ["placeEvents", "autoEvents", "openingEvents", "communicationEvents"]:
+ if eventsKey in data and isinstance(data[eventsKey], list):
+ for event in data[eventsKey]:
+ if not event:
+ continue
+
+ # Handle event-level name field
+ if "name" in event and event["name"]:
+ if translatedDataList is None:
+ eventNameList.append(event["name"])
+ else:
+ if eventNameList:
+ event["name"] = eventNameList[0]
+ eventNameList.pop(0)
+
+ # Handle event-level desc field
+ if "desc" in event and event["desc"]:
+ if translatedDataList is None:
+ # Remove newlines for translation
+ eventDescList.append(event["desc"].replace("\n", " "))
+ else:
+ if eventDescList:
+ translatedText = eventDescList[0]
+ # Apply text wrapping
+ translatedText = dazedwrap.wrapText(translatedText, width=WIDTH)
+ event["desc"] = translatedText
+ eventDescList.pop(0)
+
+ if "pages" not in event or not isinstance(event["pages"], list):
+ continue
+
+ for page in event["pages"]:
+ if not page or "commands" not in page:
+ continue
+
+ if not isinstance(page["commands"], list):
+ continue
+
+ for command in page["commands"]:
+ if not command:
+ continue
+
+ speaker = command.get("speaker", "")
+
+ # Handle data array
+ if "data" in command and isinstance(command["data"], list):
+ for i, text in enumerate(command["data"]):
+ if text:
+ if translatedDataList is None:
+ text = text.replace("\n", " ")
+ if speaker:
+ dataList.append(f"[{speaker}]: {text}")
+ else:
+ dataList.append(text)
+ else:
+ if dataList:
+ translated = dataList[0]
+ if speaker:
+ match = re.search(r'(^\[.+?\]\s?[|:]\s?)', translated)
+ if match:
+ translated = translated.replace(match.group(1), "")
+ translated = dazedwrap.wrapText(translated, width=WIDTH)
+ command["data"][i] = translated
+ dataList.pop(0)
+
+ # Handle speaker field
+ if "speaker" in command and command["speaker"]:
+ if translatedDataList is None:
+ originalSpeakerList.append(command["speaker"])
+ speakerList.append(command["speaker"])
+ else:
+ if speakerList:
+ command["speaker"] = speakerList[0]
+ speakerList.pop(0)
+
+ # If this was Pass 1, do the translation and recurse for Pass 2
+ if translatedDataList is None:
+ # Store original counts for mismatch checking
+ originalDescCount = len(descList)
+ originalMapNameCount = len(mapNameList)
+ originalVictoryCondsCount = len(victoryCondsList)
+ originalDefeatCondsCount = len(defeatCondsList)
+ originalUnitNameCount = len(unitNameList)
+ originalUnitDescCount = len(unitDescList)
+ originalEventNameCount = len(eventNameList)
+ originalEventDescCount = len(eventDescList)
+ originalDataCount = len(dataList)
+ originalSpeakerCount = len(speakerList)
+
+ # Batch translate map descriptions
+ if descList:
+ response = translateAI(
+ descList,
+ "Reply with only the " + LANGUAGE + " translation of the map description.",
+ True,
+ filename,
+ pbar
+ )
+ descList = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Batch translate map display names
+ if mapNameList:
+ response = translateAI(
+ mapNameList,
+ "Reply with only the " + LANGUAGE + " translation of the map display name.",
+ True,
+ filename,
+ pbar
+ )
+ mapNameList = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Batch translate victory conditions
+ if victoryCondsList:
+ response = translateAI(
+ victoryCondsList,
+ "Reply with only the " + LANGUAGE + " translation of the victory condition.",
+ True,
+ filename,
+ pbar
+ )
+ victoryCondsList = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Batch translate defeat conditions
+ if defeatCondsList:
+ response = translateAI(
+ defeatCondsList,
+ "Reply with only the " + LANGUAGE + " translation of the defeat condition.",
+ True,
+ filename,
+ pbar
+ )
+ defeatCondsList = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Batch translate unit names
+ if unitNameList:
+ response = translateAI(
+ unitNameList,
+ "Reply with only the " + LANGUAGE + " translation of the enemy unit name.",
+ True,
+ filename,
+ pbar
+ )
+ unitNameList = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Batch translate unit descriptions
+ if unitDescList:
+ response = translateAI(
+ unitDescList,
+ "Reply with only the " + LANGUAGE + " translation of the enemy unit description.",
+ True,
+ filename,
+ pbar
+ )
+ unitDescList = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Batch translate event names
+ if eventNameList:
+ response = translateAI(
+ eventNameList,
+ "Reply with only the " + LANGUAGE + " translation of the event name.",
+ True,
+ filename,
+ pbar
+ )
+ eventNameList = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Batch translate event descriptions
+ if eventDescList:
+ response = translateAI(
+ eventDescList,
+ "Reply with only the " + LANGUAGE + " translation of the event description.",
+ True,
+ filename,
+ pbar
+ )
+ eventDescList = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Batch translate dialogue data
+ if dataList:
+ response = translateAI(
+ dataList,
+ "Reply with only the " + LANGUAGE + " translation of the dialogue text.",
+ True,
+ filename,
+ pbar
+ )
+ dataList = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Batch translate speakers
+ if speakerList:
+ response = translateAI(
+ speakerList,
+ "Reply with only the " + LANGUAGE + " translation of the speaker name.",
+ True,
+ filename,
+ pbar
+ )
+ speakerList = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Check for mismatch errors
+ if len(descList) != originalDescCount:
+ with LOCK:
+ if filename not in MISMATCH:
+ MISMATCH.append(filename)
+
+ if len(mapNameList) != originalMapNameCount:
+ with LOCK:
+ if filename not in MISMATCH:
+ MISMATCH.append(filename)
+
+ if len(victoryCondsList) != originalVictoryCondsCount:
+ with LOCK:
+ if filename not in MISMATCH:
+ MISMATCH.append(filename)
+
+ if len(defeatCondsList) != originalDefeatCondsCount:
+ with LOCK:
+ if filename not in MISMATCH:
+ MISMATCH.append(filename)
+
+ if len(unitNameList) != originalUnitNameCount:
+ with LOCK:
+ if filename not in MISMATCH:
+ MISMATCH.append(filename)
+
+ if len(unitDescList) != originalUnitDescCount:
+ with LOCK:
+ if filename not in MISMATCH:
+ MISMATCH.append(filename)
+
+ if len(eventNameList) != originalEventNameCount:
+ with LOCK:
+ if filename not in MISMATCH:
+ MISMATCH.append(filename)
+
+ if len(eventDescList) != originalEventDescCount:
+ with LOCK:
+ if filename not in MISMATCH:
+ MISMATCH.append(filename)
+
+ if len(dataList) != originalDataCount:
+ with LOCK:
+ if filename not in MISMATCH:
+ MISMATCH.append(filename)
+
+ if len(speakerList) != originalSpeakerCount:
+ with LOCK:
+ if filename not in MISMATCH:
+ MISMATCH.append(filename)
+
+ # PASS 2: Recursively call to apply translations
+ translateMap(data, filename, [descList, mapNameList, victoryCondsList, defeatCondsList, unitNameList, unitDescList, eventNameList, eventDescList, dataList, speakerList], pbar)
+
+ return totalTokens
+
+
+def getResultString(translatedData, translationTime, filename):
+ """
+ Formats the translation result string with token counts, cost, and time.
+
+ Args:
+ translatedData: Tuple of (data, tokens, error)
+ translationTime: Time taken for translation
+ filename: Name of the file
+
+ Returns:
+ Formatted result string
+ """
+ global TIMETOTAL
+
+ # Calculate cost
+ cost = calculateCost(translatedData[1][0], translatedData[1][1], MODEL)
+ totalTokenstring = (
+ Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
+ "[Output: "
+ + str(translatedData[1][1])
+ + "]" "[Cost: ${:,.4f}".format(cost)
+ + "]"
+ )
+
+ # Format time string
+ if filename != "TOTAL":
+ timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
+ TIMETOTAL += round(translationTime, 1)
+ else:
+ timeString = Fore.BLUE + "[" + str(round(TIMETOTAL, 1)) + "s]"
+
+ # Return success or failure string
+ if translatedData[2] is None:
+ # Success
+ return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET
+ else:
+ # Fail
+ try:
+ raise translatedData[2]
+ except Exception as e:
+ traceback.print_exc()
+ return (
+ filename
+ + ": "
+ + totalTokenstring
+ + timeString
+ + Fore.RED
+ + " \u2717 "
+ + Fore.RESET
+ )
+
+
+def getSpeaker(speaker):
+ """
+ Translates speaker/character names with caching to avoid redundant translations.
+
+ Args:
+ speaker: The original speaker name to translate
+
+ Returns:
+ List containing [translated name, [input tokens, output tokens]]
+ """
+ if speaker == "":
+ return ["", [0, 0]]
+
+ # Check if speaker has already been translated
+ for i in range(len(NAMESLIST)):
+ if speaker == NAMESLIST[i][0]:
+ return [NAMESLIST[i][1], [0, 0]]
+
+ # Translate and Store Speaker
+ response = translateAI(
+ speaker,
+ "Reply with the " + LANGUAGE + " translation of the NPC name.",
+ False,
+ )
+ response[0] = response[0].title()
+ response[0] = response[0].replace("'S", "'s")
+ response[0] = response[0].replace("Speaker: ", "")
+
+ # Retry if name doesn't translate for some reason
+ if re.search(r"([a-zA-Z？?])", response[0]) is None:
+ response = translateAI(
+ speaker,
+ "Reply with the " + LANGUAGE + " translation of the NPC name.",
+ False,
+ )
+ response[0] = response[0].title()
+ response[0] = response[0].replace("'S", "'s")
+
+ speakerList = [speaker, response[0]]
+ NAMESLIST.append(speakerList)
+ return response
+
+
+def translateAI(text, history, history_ctx=None, filename=None, pbar=None):
+ """
+ Legacy wrapper function for the new shared translation utility.
+ This maintains compatibility with existing code while using the new shared implementation.
+
+ Args:
+ text: Text to translate (can be string or list)
+ history: History/context for the translation
+
+ Returns:
+ List containing [translated text, [input tokens, output tokens]]
+ """
+ global PBAR, MISMATCH, FILENAME
+
+ # Update config estimate mode based on global ESTIMATE
+ TRANSLATION_CONFIG.estimateMode = bool(ESTIMATE)
+
+ # Call the new shared translation function
+ return sharedtranslateAI(
+ text=text,
+ history=history,
+ config=TRANSLATION_CONFIG,
+ filename=filename if filename is not None else FILENAME,
+ pbar=pbar if pbar is not None else PBAR,
+ lock=LOCK,
+ mismatchList=MISMATCH
+ )
+
diff --git a/modules/text.py b/modules/text.py
index d257d64..e4cd2ff 100644
--- a/modules/text.py
+++ b/modules/text.py
@@ -1,363 +1,365 @@
-# Libraries
-import json
-import os
-import re
-import util.dazedwrap as dazedwrap
-import threading
-import time
-import traceback
-import tiktoken
-from pathlib import Path
-from colorama import Fore
-from dotenv import load_dotenv
-from retry import retry
-from tqdm import tqdm
-from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost, getPricingConfig, calculateCost
-import tempfile
-
-# Globals
-MODEL = os.getenv("model")
-TIMEOUT = int(os.getenv("timeout"))
-LANGUAGE = os.getenv("language").capitalize()
-PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
-VOCAB = Path("vocab.txt").read_text(encoding="utf-8")
-LOCK = threading.Lock()
-WIDTH = int(os.getenv("width"))
-LISTWIDTH = int(os.getenv("listWidth"))
-NOTEWIDTH = 70
-MAXHISTORY = 10
-ESTIMATE = ""
-TOKENS = [0, 0]
-NAMESLIST = []
-NAMES = False # Output a list of all the character names found
-BRFLAG = False # If the game uses
instead
-FIXTEXTWRAP = True # Overwrites textwrap
-IGNORETLTEXT = False # Ignores all translated text.
-MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
-FILENAME = None
-
-# tqdm Globals
-BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
-POSITION = 0
-LEAVE = False
-PBAR = None
-
-# Regex - Need to change this if you want to translate from/to other languages. Default is Japanese Regex
-LANGREGEX = r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+"
-
-# Get pricing configuration based on the model
-PRICING_CONFIG = getPricingConfig(MODEL)
-INPUTAPICOST = PRICING_CONFIG["inputAPICost"]
-OUTPUTAPICOST = PRICING_CONFIG["outputAPICost"]
-BATCHSIZE = PRICING_CONFIG["batchSize"]
-FREQUENCY_PENALTY = PRICING_CONFIG["frequencyPenalty"]
-
-# Initialize Translation Config
-TRANSLATION_CONFIG = TranslationConfig(
- model=MODEL,
- language=LANGUAGE,
- prompt=PROMPT,
- vocab=VOCAB,
- langRegex=LANGREGEX,
- batchSize=BATCHSIZE,
- maxHistory=MAXHISTORY,
- estimateMode=False # Will be set dynamically based on ESTIMATE
-)
-LEAVE = False
-
-def handleText(filename, estimate):
- global ESTIMATE, TOKENS, FILENAME
- ESTIMATE = estimate
- FILENAME = filename
-
- # Translate
- start = time.time()
- translatedData = openFiles(filename)
-
- # Translate
- if not estimate:
- try:
- with open("translated/" + filename, "w", encoding="utf-8", newline="\n") as outFile:
- outFile.writelines(translatedData[0])
- except Exception:
- traceback.print_exc()
- return "Fail"
-
- # Print File
- end = time.time()
- tqdm.write(getResultString(translatedData, end - start, filename))
- with LOCK:
- TOKENS[0] += translatedData[1][0]
- TOKENS[1] += translatedData[1][1]
-
- # Print Total
- totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
-
- # Print any errors on maps
- if len(MISMATCH) > 0:
- return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
- else:
- return totalString
-
-
-def getResultString(translatedData, translationTime, filename):
- # File Print String
- cost = calculateCost(translatedData[1][0], translatedData[1][1], MODEL)
- totalTokenstring = (
- Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
- "[Output: "
- + str(translatedData[1][1])
- + "]" "[Cost: ${:,.4f}".format(cost)
- + "]"
- )
- timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
-
- if translatedData[2] == None:
- # Success
- return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET
-
- else:
- # Fail
- try:
- raise translatedData[2]
- except Exception as e:
- traceback.print_exc()
- errorString = str(e) + Fore.RED
- return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET
-
-
-def openFiles(filename):
- with open("files/" + filename, "r", encoding="utf8") as readFile:
- translatedData = parseText(readFile, filename)
-
- # Delete lines marked for deletion
- finalData = []
- for line in translatedData[0]:
- if line != "\\d\n":
- finalData.append(line)
- translatedData[0] = finalData
-
- return translatedData
-
-
-def parseText(readFile, filename):
- global PBAR
- totalTokens = [0, 0]
-
- # Read File into data
- data = readFile.readlines()
-
- # Create Progress Bar
- with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
- pbar.desc = filename
- PBAR = pbar
-
- try:
- result = translateTxt(data, filename, [])
- totalTokens[0] += result[0]
- totalTokens[1] += result[1]
- except Exception as e:
- traceback.print_exc()
- return [data, totalTokens, e]
- return [data, totalTokens, None]
-
-
-def save_progress_lines(lines, filename, encoding="utf-8"):
- """Atomically save current line-based translation progress."""
- try:
- if ESTIMATE:
- return
- os.makedirs("translated", exist_ok=True)
- tmp_fd, tmp_path = tempfile.mkstemp(prefix=f"{filename}.", suffix=".tmp", dir="translated")
- try:
- with os.fdopen(tmp_fd, "w", encoding=encoding, newline="\n", errors="ignore") as tmp_file:
- tmp_file.writelines(lines)
- os.replace(tmp_path, os.path.join("translated", filename))
- finally:
- if os.path.exists(tmp_path):
- try:
- os.remove(tmp_path)
- except OSError:
- pass
- except Exception:
- traceback.print_exc()
-
-
-def saveCheckLines(lines, filename, tokens=None, encoding="utf-8"):
- """Save progress only when tokens indicate work or when explicitly called after a mutation."""
- try:
- if tokens is not None:
- if not (isinstance(tokens, (list, tuple)) and len(tokens) >= 2 and (tokens[0] or tokens[1])):
- return
- save_progress_lines(lines, filename, encoding=encoding)
- except Exception:
- traceback.print_exc()
-
-
-def translateTxt(data, filename, translatedList):
- if translatedList:
- stringList = translatedList[0]
- choiceList = translatedList[1]
- else:
- stringList = []
- choiceList = []
- tokens = [0, 0]
- speaker = ""
- global LOCK, ESTIMATE, FILENAME, PBAR, MISMATCH
- i = 0
-
- # Regex
- lineTextRegex = r"(?:^\[.+?\]:)?(.+)"
- speakerTextRegex = r"^\[(.+?)\]"
-
- while i < len(data):
- # Speaker
- match = re.search(speakerTextRegex, data[i])
- if match:
- # Get Speaker
- speakerData = getSpeaker(match.group(1))
- speaker = speakerData[0]
- tokens[0] += speakerData[1][0]
- tokens[1] += speakerData[1][1]
- data[i] = data[i].replace(match.group(1), speaker)
-
- # Dialogue
- match = re.search(lineTextRegex, data[i])
- jaString = None
- if match:
- # Set String
- jaString = match.group(1)
-
- # Pass 1
- if not translatedList:
- # Strip Spaces
- jaString = jaString.strip()
-
- if jaString:
- if speaker:
- stringList.append(f"[{speaker}]: {jaString}")
- else:
- stringList.append(jaString)
-
- # Pass 2
- else:
- # Get Text
- if stringList:
- # Grab and Pop
- translatedText = stringList[0]
- stringList.pop(0)
-
- # Set to None if empty list
- if len(stringList) <= 0:
- stringList = None
-
- # Remove speaker
- translatedText = re.sub(r"(^\[.+?\]\s?[|:]\s?)", "", translatedText)
-
- # # Textwrap
- # translatedText = dazedwrap.wrapText(translatedText, width=WIDTH)
- # translatedText = translatedText.replace("\n", "\\n")
-
- # Set Data
- data[i] = f"{translatedText}\n"
- saveCheckLines(data, filename)
-
- i += 1
- else:
- i += 1
-
- # EOF
- if not translatedList:
- stringListTL = []
- choiceListTL = []
-
- # String List
- if stringList:
- PBAR.total = len(stringList)
- PBAR.refresh()
- response = translateAI(stringList, "Reply with the English Translation")
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- stringListTL = response[0]
-
- if len(stringList) != len(stringListTL):
- # Mismatch
- with LOCK:
- if FILENAME not in MISMATCH:
- MISMATCH.append(FILENAME)
-
- # Choice List
- if choiceList:
- response = translateAI(choiceList, "Reply with the English TL of the Dialogue Choice")
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- choiceListTL = response[0]
-
- if len(choiceList) != len(choiceListTL):
- # Mismatch
- with LOCK:
- if FILENAME not in MISMATCH:
- MISMATCH.append(FILENAME)
-
- # Set Strings
- translateTxt(data, filename, [stringListTL, choiceListTL])
- return tokens
-
-# Save some money and enter the character before translation
-def getSpeaker(speaker):
- match speaker:
- case "ファイン":
- return ["Fine", [0, 0]]
- case "":
- return ["", [0, 0]]
- case _:
- # Find Speaker
- for i in range(len(NAMESLIST)):
- if speaker == NAMESLIST[i][0]:
- return [NAMESLIST[i][1], [0, 0]]
-
- # Translate and Store Speaker
- response = translateAI(
- f"{speaker}",
- "Reply with the " + LANGUAGE + " translation of the NPC name.",
- False,
- )
- response[0] = response[0].title()
- response[0] = response[0].replace("'S", "'s")
- response[0] = response[0].replace("Speaker: ", "")
-
- # Retry if name doesn't translate for some reason
- if re.search(r"([a-zA-Z？?])", response[0]) == None:
- response = translateAI(
- f"{speaker}",
- "Reply with the " + LANGUAGE + " translation of the NPC name.",
- False,
- )
- response[0] = response[0].title()
- response[0] = response[0].replace("'S", "'s")
-
- speakerList = [speaker, response[0]]
- NAMESLIST.append(speakerList)
- return response
- return [speaker, [0, 0]]
-
-def translateAI(text, history, history_ctx=None):
- """
- Legacy wrapper function for the new shared translation utility.
- This maintains compatibility with existing code while using the new shared implementation.
- """
- global PBAR, MISMATCH, FILENAME
-
- # Update config estimate mode based on global ESTIMATE
- TRANSLATION_CONFIG.estimateMode = bool(ESTIMATE)
-
- # Call the new shared translation function
- return sharedtranslateAI(
- text=text,
- history=history,
- config=TRANSLATION_CONFIG,
- filename=FILENAME,
- pbar=PBAR,
- lock=LOCK,
- mismatchList=MISMATCH
+# Libraries
+import json
+import os
+import re
+import util.dazedwrap as dazedwrap
+import threading
+import time
+import traceback
+import tiktoken
+from pathlib import Path
+from colorama import Fore
+from dotenv import load_dotenv
+from retry import retry
+from tqdm import tqdm
+from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost, getPricingConfig, calculateCost
+import tempfile
+
+# Globals
+MODEL = os.getenv("model")
+TIMEOUT = int(os.getenv("timeout"))
+LANGUAGE = os.getenv("language").capitalize()
+from util.paths import PROMPT_PATH, VOCAB_PATH
+
+PROMPT = PROMPT_PATH.read_text(encoding="utf-8")
+VOCAB = VOCAB_PATH.read_text(encoding="utf-8")
+LOCK = threading.Lock()
+WIDTH = int(os.getenv("width"))
+LISTWIDTH = int(os.getenv("listWidth"))
+NOTEWIDTH = 70
+MAXHISTORY = 10
+ESTIMATE = ""
+TOKENS = [0, 0]
+NAMESLIST = []
+NAMES = False # Output a list of all the character names found
+BRFLAG = False # If the game uses
instead
+FIXTEXTWRAP = True # Overwrites textwrap
+IGNORETLTEXT = False # Ignores all translated text.
+MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
+FILENAME = None
+
+# tqdm Globals
+BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
+POSITION = 0
+LEAVE = False
+PBAR = None
+
+# Regex - Need to change this if you want to translate from/to other languages. Default is Japanese Regex
+LANGREGEX = r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+"
+
+# Get pricing configuration based on the model
+PRICING_CONFIG = getPricingConfig(MODEL)
+INPUTAPICOST = PRICING_CONFIG["inputAPICost"]
+OUTPUTAPICOST = PRICING_CONFIG["outputAPICost"]
+BATCHSIZE = PRICING_CONFIG["batchSize"]
+FREQUENCY_PENALTY = PRICING_CONFIG["frequencyPenalty"]
+
+# Initialize Translation Config
+TRANSLATION_CONFIG = TranslationConfig(
+ model=MODEL,
+ language=LANGUAGE,
+ prompt=PROMPT,
+ vocab=VOCAB,
+ langRegex=LANGREGEX,
+ batchSize=BATCHSIZE,
+ maxHistory=MAXHISTORY,
+ estimateMode=False # Will be set dynamically based on ESTIMATE
+)
+LEAVE = False
+
+def handleText(filename, estimate):
+ global ESTIMATE, TOKENS, FILENAME
+ ESTIMATE = estimate
+ FILENAME = filename
+
+ # Translate
+ start = time.time()
+ translatedData = openFiles(filename)
+
+ # Translate
+ if not estimate:
+ try:
+ with open("translated/" + filename, "w", encoding="utf-8", newline="\n") as outFile:
+ outFile.writelines(translatedData[0])
+ except Exception:
+ traceback.print_exc()
+ return "Fail"
+
+ # Print File
+ end = time.time()
+ tqdm.write(getResultString(translatedData, end - start, filename))
+ with LOCK:
+ TOKENS[0] += translatedData[1][0]
+ TOKENS[1] += translatedData[1][1]
+
+ # Print Total
+ totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
+
+ # Print any errors on maps
+ if len(MISMATCH) > 0:
+ return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
+ else:
+ return totalString
+
+
+def getResultString(translatedData, translationTime, filename):
+ # File Print String
+ cost = calculateCost(translatedData[1][0], translatedData[1][1], MODEL)
+ totalTokenstring = (
+ Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
+ "[Output: "
+ + str(translatedData[1][1])
+ + "]" "[Cost: ${:,.4f}".format(cost)
+ + "]"
+ )
+ timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
+
+ if translatedData[2] == None:
+ # Success
+ return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET
+
+ else:
+ # Fail
+ try:
+ raise translatedData[2]
+ except Exception as e:
+ traceback.print_exc()
+ errorString = str(e) + Fore.RED
+ return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET
+
+
+def openFiles(filename):
+ with open("files/" + filename, "r", encoding="utf8") as readFile:
+ translatedData = parseText(readFile, filename)
+
+ # Delete lines marked for deletion
+ finalData = []
+ for line in translatedData[0]:
+ if line != "\\d\n":
+ finalData.append(line)
+ translatedData[0] = finalData
+
+ return translatedData
+
+
+def parseText(readFile, filename):
+ global PBAR
+ totalTokens = [0, 0]
+
+ # Read File into data
+ data = readFile.readlines()
+
+ # Create Progress Bar
+ with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
+ pbar.desc = filename
+ PBAR = pbar
+
+ try:
+ result = translateTxt(data, filename, [])
+ totalTokens[0] += result[0]
+ totalTokens[1] += result[1]
+ except Exception as e:
+ traceback.print_exc()
+ return [data, totalTokens, e]
+ return [data, totalTokens, None]
+
+
+def save_progress_lines(lines, filename, encoding="utf-8"):
+ """Atomically save current line-based translation progress."""
+ try:
+ if ESTIMATE:
+ return
+ os.makedirs("translated", exist_ok=True)
+ tmp_fd, tmp_path = tempfile.mkstemp(prefix=f"{filename}.", suffix=".tmp", dir="translated")
+ try:
+ with os.fdopen(tmp_fd, "w", encoding=encoding, newline="\n", errors="ignore") as tmp_file:
+ tmp_file.writelines(lines)
+ os.replace(tmp_path, os.path.join("translated", filename))
+ finally:
+ if os.path.exists(tmp_path):
+ try:
+ os.remove(tmp_path)
+ except OSError:
+ pass
+ except Exception:
+ traceback.print_exc()
+
+
+def saveCheckLines(lines, filename, tokens=None, encoding="utf-8"):
+ """Save progress only when tokens indicate work or when explicitly called after a mutation."""
+ try:
+ if tokens is not None:
+ if not (isinstance(tokens, (list, tuple)) and len(tokens) >= 2 and (tokens[0] or tokens[1])):
+ return
+ save_progress_lines(lines, filename, encoding=encoding)
+ except Exception:
+ traceback.print_exc()
+
+
+def translateTxt(data, filename, translatedList):
+ if translatedList:
+ stringList = translatedList[0]
+ choiceList = translatedList[1]
+ else:
+ stringList = []
+ choiceList = []
+ tokens = [0, 0]
+ speaker = ""
+ global LOCK, ESTIMATE, FILENAME, PBAR, MISMATCH
+ i = 0
+
+ # Regex
+ lineTextRegex = r"(?:^\[.+?\]:)?(.+)"
+ speakerTextRegex = r"^\[(.+?)\]"
+
+ while i < len(data):
+ # Speaker
+ match = re.search(speakerTextRegex, data[i])
+ if match:
+ # Get Speaker
+ speakerData = getSpeaker(match.group(1))
+ speaker = speakerData[0]
+ tokens[0] += speakerData[1][0]
+ tokens[1] += speakerData[1][1]
+ data[i] = data[i].replace(match.group(1), speaker)
+
+ # Dialogue
+ match = re.search(lineTextRegex, data[i])
+ jaString = None
+ if match:
+ # Set String
+ jaString = match.group(1)
+
+ # Pass 1
+ if not translatedList:
+ # Strip Spaces
+ jaString = jaString.strip()
+
+ if jaString:
+ if speaker:
+ stringList.append(f"[{speaker}]: {jaString}")
+ else:
+ stringList.append(jaString)
+
+ # Pass 2
+ else:
+ # Get Text
+ if stringList:
+ # Grab and Pop
+ translatedText = stringList[0]
+ stringList.pop(0)
+
+ # Set to None if empty list
+ if len(stringList) <= 0:
+ stringList = None
+
+ # Remove speaker
+ translatedText = re.sub(r"(^\[.+?\]\s?[|:]\s?)", "", translatedText)
+
+ # # Textwrap
+ # translatedText = dazedwrap.wrapText(translatedText, width=WIDTH)
+ # translatedText = translatedText.replace("\n", "\\n")
+
+ # Set Data
+ data[i] = f"{translatedText}\n"
+ saveCheckLines(data, filename)
+
+ i += 1
+ else:
+ i += 1
+
+ # EOF
+ if not translatedList:
+ stringListTL = []
+ choiceListTL = []
+
+ # String List
+ if stringList:
+ PBAR.total = len(stringList)
+ PBAR.refresh()
+ response = translateAI(stringList, "Reply with the English Translation")
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ stringListTL = response[0]
+
+ if len(stringList) != len(stringListTL):
+ # Mismatch
+ with LOCK:
+ if FILENAME not in MISMATCH:
+ MISMATCH.append(FILENAME)
+
+ # Choice List
+ if choiceList:
+ response = translateAI(choiceList, "Reply with the English TL of the Dialogue Choice")
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ choiceListTL = response[0]
+
+ if len(choiceList) != len(choiceListTL):
+ # Mismatch
+ with LOCK:
+ if FILENAME not in MISMATCH:
+ MISMATCH.append(FILENAME)
+
+ # Set Strings
+ translateTxt(data, filename, [stringListTL, choiceListTL])
+ return tokens
+
+# Save some money and enter the character before translation
+def getSpeaker(speaker):
+ match speaker:
+ case "ファイン":
+ return ["Fine", [0, 0]]
+ case "":
+ return ["", [0, 0]]
+ case _:
+ # Find Speaker
+ for i in range(len(NAMESLIST)):
+ if speaker == NAMESLIST[i][0]:
+ return [NAMESLIST[i][1], [0, 0]]
+
+ # Translate and Store Speaker
+ response = translateAI(
+ f"{speaker}",
+ "Reply with the " + LANGUAGE + " translation of the NPC name.",
+ False,
+ )
+ response[0] = response[0].title()
+ response[0] = response[0].replace("'S", "'s")
+ response[0] = response[0].replace("Speaker: ", "")
+
+ # Retry if name doesn't translate for some reason
+ if re.search(r"([a-zA-Z？?])", response[0]) == None:
+ response = translateAI(
+ f"{speaker}",
+ "Reply with the " + LANGUAGE + " translation of the NPC name.",
+ False,
+ )
+ response[0] = response[0].title()
+ response[0] = response[0].replace("'S", "'s")
+
+ speakerList = [speaker, response[0]]
+ NAMESLIST.append(speakerList)
+ return response
+ return [speaker, [0, 0]]
+
+def translateAI(text, history, history_ctx=None):
+ """
+ Legacy wrapper function for the new shared translation utility.
+ This maintains compatibility with existing code while using the new shared implementation.
+ """
+ global PBAR, MISMATCH, FILENAME
+
+ # Update config estimate mode based on global ESTIMATE
+ TRANSLATION_CONFIG.estimateMode = bool(ESTIMATE)
+
+ # Call the new shared translation function
+ return sharedtranslateAI(
+ text=text,
+ history=history,
+ config=TRANSLATION_CONFIG,
+ filename=FILENAME,
+ pbar=PBAR,
+ lock=LOCK,
+ mismatchList=MISMATCH
)
\ No newline at end of file
diff --git a/modules/tyrano.py b/modules/tyrano.py
index 733e312..afda46e 100644
--- a/modules/tyrano.py
+++ b/modules/tyrano.py
@@ -1,457 +1,459 @@
-# Libraries
-import json
-import os
-import re
-import util.dazedwrap as dazedwrap
-import threading
-import time
-import traceback
-import tiktoken
-from pathlib import Path
-from colorama import Fore
-from dotenv import load_dotenv
-from retry import retry
-from tqdm import tqdm
-from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost
-import tempfile
-
-# OpenAI initialization centralized in util/translation.py
-
-# Globals
-MODEL = os.getenv("model")
-TIMEOUT = int(os.getenv("timeout"))
-LANGUAGE = os.getenv("language").capitalize()
-PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
-VOCAB = Path("vocab.txt").read_text(encoding="utf-8")
-LOCK = threading.Lock()
-WIDTH = int(os.getenv("width"))
-LISTWIDTH = int(os.getenv("listWidth"))
-NOTEWIDTH = 70
-MAXHISTORY = 10
-ESTIMATE = ""
-TOKENS = [0, 0]
-NAMESLIST = []
-NAMES = False # Output a list of all the character names found
-BRFLAG = False # If the game uses
instead
-FIXTEXTWRAP = True # Overwrites textwrap
-IGNORETLTEXT = False # Ignores all translated text.
-MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
-FILENAME = None
-
-# tqdm Globals
-BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
-POSITION = 0
-LEAVE = False
-PBAR = None
-
-# Regex - Need to change this if you want to translate from/to other languages. Default is Japanese Regex
-LANGREGEX = r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+"
-
-# Get pricing configuration based on the model
-PRICING_CONFIG = getPricingConfig(MODEL)
-INPUTAPICOST = PRICING_CONFIG["inputAPICost"]
-OUTPUTAPICOST = PRICING_CONFIG["outputAPICost"]
-BATCHSIZE = PRICING_CONFIG["batchSize"]
-FREQUENCY_PENALTY = PRICING_CONFIG["frequencyPenalty"]
-
-# Initialize Translation Config
-TRANSLATION_CONFIG = TranslationConfig(
- model=MODEL,
- language=LANGUAGE,
- prompt=PROMPT,
- vocab=VOCAB,
- langRegex=LANGREGEX,
- batchSize=BATCHSIZE,
- maxHistory=MAXHISTORY,
- estimateMode=False # Will be set dynamically based on ESTIMATE
-)
-LEAVE = False
-
-def handleTyrano(filename, estimate):
- global ESTIMATE, TOKENS, FILENAME
- ESTIMATE = estimate
- FILENAME = filename
-
- # Translate
- start = time.time()
- translatedData = openFiles(filename)
-
- # Translate
- if not estimate:
- try:
- with open("translated/" + filename, "w", encoding="utf-8", newline="\n") as outFile:
- outFile.writelines(translatedData[0])
- except Exception:
- traceback.print_exc()
- return "Fail"
-
- # Print File
- end = time.time()
- tqdm.write(getResultString(translatedData, end - start, filename))
- with LOCK:
- TOKENS[0] += translatedData[1][0]
- TOKENS[1] += translatedData[1][1]
-
- # Print Total
- totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
-
- # Print any errors on maps
- if len(MISMATCH) > 0:
- return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
- else:
- return totalString
-
-
-def getResultString(translatedData, translationTime, filename):
- # File Print String
- cost = calculateCost(translatedData[1][0], translatedData[1][1], MODEL)
- totalTokenstring = (
- Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
- "[Output: "
- + str(translatedData[1][1])
- + "]" "[Cost: ${:,.4f}".format(cost)
- + "]"
- )
- timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
-
- if translatedData[2] == None:
- # Success
- return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET
-
- else:
- # Fail
- try:
- raise translatedData[2]
- except Exception as e:
- traceback.print_exc()
- errorString = str(e) + Fore.RED
- return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET
-
-
-def openFiles(filename):
- with open("files/" + filename, "r", encoding="utf8") as readFile:
- translatedData = parseTyrano(readFile, filename)
-
- # Delete lines marked for deletion
- finalData = []
- for line in translatedData[0]:
- if line != "\\d\n":
- finalData.append(line)
- translatedData[0] = finalData
-
- return translatedData
-
-
-def parseTyrano(readFile, filename):
- global PBAR
- totalTokens = [0, 0]
-
- # Read File into data
- data = readFile.readlines()
-
- # Create Progress Bar
- with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
- pbar.desc = filename
- PBAR = pbar
-
- try:
- result = translateTyrano(data, filename, [])
- totalTokens[0] += result[0]
- totalTokens[1] += result[1]
- except Exception as e:
- traceback.print_exc()
- return [data, totalTokens, e]
- return [data, totalTokens, None]
-
-
-def save_progress_lines(lines, filename, encoding="utf-8"):
- """Atomically save current line-based translation progress."""
- try:
- if ESTIMATE:
- return
- os.makedirs("translated", exist_ok=True)
- tmp_fd, tmp_path = tempfile.mkstemp(prefix=f"{filename}.", suffix=".tmp", dir="translated")
- try:
- with os.fdopen(tmp_fd, "w", encoding=encoding, newline="\n", errors="ignore") as tmp_file:
- tmp_file.writelines(lines)
- os.replace(tmp_path, os.path.join("translated", filename))
- finally:
- if os.path.exists(tmp_path):
- try:
- os.remove(tmp_path)
- except OSError:
- pass
- except Exception:
- traceback.print_exc()
-
-
-def saveCheckLines(lines, filename, tokens=None, encoding="utf-8"):
- """Save progress only when tokens indicate work or when tokens is None but we know a mutation occurred."""
- try:
- # If explicit tokens provided, gate on tokens; otherwise assume we were called on actual mutation
- if tokens is not None:
- if not (isinstance(tokens, (list, tuple)) and len(tokens) >= 2 and (tokens[0] or tokens[1])):
- return
- save_progress_lines(lines, filename, encoding=encoding)
- except Exception:
- traceback.print_exc()
-
-
-def translateTyrano(data, filename, translatedList):
- if translatedList:
- stringList = translatedList[0]
- choiceList = translatedList[1]
- else:
- stringList = []
- choiceList = []
- tokens = [0, 0]
- speaker = ""
- stringListTL = []
- choiceListTL = []
- global LOCK, ESTIMATE, FILENAME, PBAR, MISMATCH
- i = 0
-
- while i < len(data):
- voice = False
- lineRegexNoSpeaker = r"^([^\[#;*@\n]+)\[p\]|^([^\[#;*@\n]+)\[l\]\[[rp]\]|^([^\[#;*@\n]+)\[[rpl]\]|^([^\[#;*@_\n]+)\n$"
- lineRegexSpeaker = r"^#(.*)"
- furiganaRegex = r"(\[ruby\stext=(.*?)\])"
- choiceRegex = r'\[glink.+?text="(.*?)"'
-
- # Speaker
- match = re.search(lineRegexSpeaker, data[i])
- if match:
- if match.group(1):
- response = getSpeaker(match.group(1))
- speaker = response[0]
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- data[i] = data[i].replace(match.group(1), speaker)
- else:
- speaker = None
-
- # Furigana
- match = re.search(r"^\[ruby\stext", data[i])
- furiganaList = []
- if match:
- # Check next line and combine
- while match:
- furiganaList.append(data[i].replace("\n", ""))
- del data[i]
- match = re.search(r"^\[ruby\stext", data[i])
- jaString = "".join(furiganaList)
-
- # Ruby Text
- furiganaList = re.findall(furiganaRegex, jaString)
- for furigana in furiganaList:
- jaString = jaString.replace(furigana[0], furigana[1])
-
- data.insert(i, f"{jaString}[r]")
-
- # Dialogue
- match = re.search(lineRegexNoSpeaker, data[i])
- jaString = None
- if match:
- # Find which group matched
- for group_num in range(1, 5):
- try:
- if match.group(group_num):
- jaString = match.group(group_num)
- break
- except IndexError:
- break
-
- # Skip if no valid string found
- if not jaString:
- i += 1
- continue
-
- # Combine w/ next line if necessary
- repeatRegex = r"(.+?)\[[rpl]\]"
- if i + 1 < len(data):
- match = re.search(repeatRegex, data[i + 1])
- while match and "[p]" not in data[i]:
- jaString = jaString + match.group(1)
- jaString = jaString.replace("_ ", "")
- if "[p]" in data[i + 1]:
- data[i] = f"{jaString}[p]\n"
- else:
- data[i] = jaString
- del data[i + 1]
- if i + 1 < len(data):
- match = re.search(repeatRegex, data[i + 1])
- else:
- break
-
- originalString = jaString
-
- # Pass 1
- if not translatedList:
- # Remove any textwrap and commands
- jaString = jaString.replace("[r]", " ")
- jaString = jaString.replace("[l]", "")
-
- # Ruby Text
- furiganaList = re.findall(furiganaRegex, jaString)
- for furigana in furiganaList:
- jaString = jaString.replace(furigana[0], furigana[1])
-
- # Strip Spaces
- jaString = jaString.strip()
-
- if jaString:
- if speaker:
- stringList.append(f"[{speaker}]: {jaString}")
- else:
- stringList.append(jaString)
-
- # Pass 2
- else:
- # Get Text
- if stringList:
- # Grab and Pop
- translatedText = stringList[0]
- stringList.pop(0)
-
- # Set to None if empty list
- if len(stringList) <= 0:
- stringList = None
-
- # Remove speaker
- if speaker != "":
- matchSpeakerList = re.findall(r"^\[?(.+?)\]?\s?[|:]\s?", translatedText)
- translatedText = re.sub(r"^\[?(.+?)\]?\s?[|:]\s?", "", translatedText)
-
- # Avoid Crashes
- translatedText = translatedText.replace("[", "(")
- translatedText = translatedText.replace("]", ")")
-
- # Textwrap
- translatedText = dazedwrap.wrapText(translatedText, width=WIDTH)
- translatedText = translatedText.replace("\n", "[r]")
-
- # Set Data
- data[i] = data[i].replace(originalString, translatedText)
- # Save progress after each line change (we mutated, so tokens gate optional)
- saveCheckLines(data, filename)
-
- # Choices
- match = re.search(choiceRegex, data[i])
- if match:
- # Pass 1
- if not translatedList:
- choiceList.append(match.group(1))
- match = re.search(choiceRegex, data[i + 1])
-
- # Pass 2
- else:
- # Grab and Pop
- translatedText = choiceList[0]
- choiceList.pop(0)
-
- # Replace Spaces
- translatedText = translatedText.replace(" ", "\u3000")
-
- # Set
- data[i] = data[i].replace(match.group(1), translatedText)
- saveCheckLines(data, filename)
-
- i += 1
- else:
- i += 1
-
- # EOF
- if not translatedList:
- # String List
- if stringList:
- PBAR.total = len(stringList)
- PBAR.refresh()
- response = translateAI(stringList, "Reply with the English Translation")
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- stringListTL = response[0]
-
- if len(stringList) != len(stringListTL):
- # Mismatch
- with LOCK:
- if FILENAME not in MISMATCH:
- MISMATCH.append(FILENAME)
-
- # Choice List
- if choiceList:
- response = translateAI(choiceList, "Reply with the English TL of the Dialogue Choice")
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- choiceListTL = response[0]
-
- if len(choiceList) != len(choiceListTL):
- # Mismatch
- with LOCK:
- if FILENAME not in MISMATCH:
- MISMATCH.append(FILENAME)
-
- # Recursive call for Pass 2 with translated strings
- result = translateTyrano(data, filename, [stringListTL, choiceListTL])
- tokens[0] += result[0]
- tokens[1] += result[1]
-
- return tokens
-
-# Save some money and enter the character before translation
-def getSpeaker(speaker):
- match speaker:
- case "ファイン":
- return ["Fine", [0, 0]]
- case "":
- return ["", [0, 0]]
- case _:
- # Find Speaker
- for i in range(len(NAMESLIST)):
- if speaker == NAMESLIST[i][0]:
- return [NAMESLIST[i][1], [0, 0]]
-
- # Translate and Store Speaker
- response = translateAI(
- f"{speaker}",
- "Reply with the " + LANGUAGE + " translation of the NPC name.",
- False,
- )
- response[0] = response[0].title()
- response[0] = response[0].replace("'S", "'s")
- response[0] = response[0].replace("Speaker: ", "")
-
- # Retry if name doesn't translate for some reason
- if re.search(r"([a-zA-Z？?])", response[0]) == None:
- response = translateAI(
- f"{speaker}",
- "Reply with the " + LANGUAGE + " translation of the NPC name.",
- False,
- )
- response[0] = response[0].title()
- response[0] = response[0].replace("'S", "'s")
-
- speakerList = [speaker, response[0]]
- NAMESLIST.append(speakerList)
- return response
- return [speaker, [0, 0]]
-
-def translateAI(text, history, history_ctx=None):
- """
- Legacy wrapper function for the new shared translation utility.
- This maintains compatibility with existing code while using the new shared implementation.
- """
- global PBAR, MISMATCH, FILENAME
-
- # Update config estimate mode based on global ESTIMATE
- TRANSLATION_CONFIG.estimateMode = bool(ESTIMATE)
-
- # Call the new shared translation function
- return sharedtranslateAI(
- text=text,
- history=history,
- config=TRANSLATION_CONFIG,
- filename=FILENAME,
- pbar=PBAR,
- lock=LOCK,
- mismatchList=MISMATCH
- )
+# Libraries
+import json
+import os
+import re
+import util.dazedwrap as dazedwrap
+import threading
+import time
+import traceback
+import tiktoken
+from pathlib import Path
+from colorama import Fore
+from dotenv import load_dotenv
+from retry import retry
+from tqdm import tqdm
+from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost
+import tempfile
+
+# OpenAI initialization centralized in util/translation.py
+
+# Globals
+MODEL = os.getenv("model")
+TIMEOUT = int(os.getenv("timeout"))
+LANGUAGE = os.getenv("language").capitalize()
+from util.paths import PROMPT_PATH, VOCAB_PATH
+
+PROMPT = PROMPT_PATH.read_text(encoding="utf-8")
+VOCAB = VOCAB_PATH.read_text(encoding="utf-8")
+LOCK = threading.Lock()
+WIDTH = int(os.getenv("width"))
+LISTWIDTH = int(os.getenv("listWidth"))
+NOTEWIDTH = 70
+MAXHISTORY = 10
+ESTIMATE = ""
+TOKENS = [0, 0]
+NAMESLIST = []
+NAMES = False # Output a list of all the character names found
+BRFLAG = False # If the game uses
instead
+FIXTEXTWRAP = True # Overwrites textwrap
+IGNORETLTEXT = False # Ignores all translated text.
+MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
+FILENAME = None
+
+# tqdm Globals
+BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
+POSITION = 0
+LEAVE = False
+PBAR = None
+
+# Regex - Need to change this if you want to translate from/to other languages. Default is Japanese Regex
+LANGREGEX = r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+"
+
+# Get pricing configuration based on the model
+PRICING_CONFIG = getPricingConfig(MODEL)
+INPUTAPICOST = PRICING_CONFIG["inputAPICost"]
+OUTPUTAPICOST = PRICING_CONFIG["outputAPICost"]
+BATCHSIZE = PRICING_CONFIG["batchSize"]
+FREQUENCY_PENALTY = PRICING_CONFIG["frequencyPenalty"]
+
+# Initialize Translation Config
+TRANSLATION_CONFIG = TranslationConfig(
+ model=MODEL,
+ language=LANGUAGE,
+ prompt=PROMPT,
+ vocab=VOCAB,
+ langRegex=LANGREGEX,
+ batchSize=BATCHSIZE,
+ maxHistory=MAXHISTORY,
+ estimateMode=False # Will be set dynamically based on ESTIMATE
+)
+LEAVE = False
+
+def handleTyrano(filename, estimate):
+ global ESTIMATE, TOKENS, FILENAME
+ ESTIMATE = estimate
+ FILENAME = filename
+
+ # Translate
+ start = time.time()
+ translatedData = openFiles(filename)
+
+ # Translate
+ if not estimate:
+ try:
+ with open("translated/" + filename, "w", encoding="utf-8", newline="\n") as outFile:
+ outFile.writelines(translatedData[0])
+ except Exception:
+ traceback.print_exc()
+ return "Fail"
+
+ # Print File
+ end = time.time()
+ tqdm.write(getResultString(translatedData, end - start, filename))
+ with LOCK:
+ TOKENS[0] += translatedData[1][0]
+ TOKENS[1] += translatedData[1][1]
+
+ # Print Total
+ totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
+
+ # Print any errors on maps
+ if len(MISMATCH) > 0:
+ return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
+ else:
+ return totalString
+
+
+def getResultString(translatedData, translationTime, filename):
+ # File Print String
+ cost = calculateCost(translatedData[1][0], translatedData[1][1], MODEL)
+ totalTokenstring = (
+ Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
+ "[Output: "
+ + str(translatedData[1][1])
+ + "]" "[Cost: ${:,.4f}".format(cost)
+ + "]"
+ )
+ timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
+
+ if translatedData[2] == None:
+ # Success
+ return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET
+
+ else:
+ # Fail
+ try:
+ raise translatedData[2]
+ except Exception as e:
+ traceback.print_exc()
+ errorString = str(e) + Fore.RED
+ return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET
+
+
+def openFiles(filename):
+ with open("files/" + filename, "r", encoding="utf8") as readFile:
+ translatedData = parseTyrano(readFile, filename)
+
+ # Delete lines marked for deletion
+ finalData = []
+ for line in translatedData[0]:
+ if line != "\\d\n":
+ finalData.append(line)
+ translatedData[0] = finalData
+
+ return translatedData
+
+
+def parseTyrano(readFile, filename):
+ global PBAR
+ totalTokens = [0, 0]
+
+ # Read File into data
+ data = readFile.readlines()
+
+ # Create Progress Bar
+ with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
+ pbar.desc = filename
+ PBAR = pbar
+
+ try:
+ result = translateTyrano(data, filename, [])
+ totalTokens[0] += result[0]
+ totalTokens[1] += result[1]
+ except Exception as e:
+ traceback.print_exc()
+ return [data, totalTokens, e]
+ return [data, totalTokens, None]
+
+
+def save_progress_lines(lines, filename, encoding="utf-8"):
+ """Atomically save current line-based translation progress."""
+ try:
+ if ESTIMATE:
+ return
+ os.makedirs("translated", exist_ok=True)
+ tmp_fd, tmp_path = tempfile.mkstemp(prefix=f"{filename}.", suffix=".tmp", dir="translated")
+ try:
+ with os.fdopen(tmp_fd, "w", encoding=encoding, newline="\n", errors="ignore") as tmp_file:
+ tmp_file.writelines(lines)
+ os.replace(tmp_path, os.path.join("translated", filename))
+ finally:
+ if os.path.exists(tmp_path):
+ try:
+ os.remove(tmp_path)
+ except OSError:
+ pass
+ except Exception:
+ traceback.print_exc()
+
+
+def saveCheckLines(lines, filename, tokens=None, encoding="utf-8"):
+ """Save progress only when tokens indicate work or when tokens is None but we know a mutation occurred."""
+ try:
+ # If explicit tokens provided, gate on tokens; otherwise assume we were called on actual mutation
+ if tokens is not None:
+ if not (isinstance(tokens, (list, tuple)) and len(tokens) >= 2 and (tokens[0] or tokens[1])):
+ return
+ save_progress_lines(lines, filename, encoding=encoding)
+ except Exception:
+ traceback.print_exc()
+
+
+def translateTyrano(data, filename, translatedList):
+ if translatedList:
+ stringList = translatedList[0]
+ choiceList = translatedList[1]
+ else:
+ stringList = []
+ choiceList = []
+ tokens = [0, 0]
+ speaker = ""
+ stringListTL = []
+ choiceListTL = []
+ global LOCK, ESTIMATE, FILENAME, PBAR, MISMATCH
+ i = 0
+
+ while i < len(data):
+ voice = False
+ lineRegexNoSpeaker = r"^([^\[#;*@\n]+)\[p\]|^([^\[#;*@\n]+)\[l\]\[[rp]\]|^([^\[#;*@\n]+)\[[rpl]\]|^([^\[#;*@_\n]+)\n$"
+ lineRegexSpeaker = r"^#(.*)"
+ furiganaRegex = r"(\[ruby\stext=(.*?)\])"
+ choiceRegex = r'\[glink.+?text="(.*?)"'
+
+ # Speaker
+ match = re.search(lineRegexSpeaker, data[i])
+ if match:
+ if match.group(1):
+ response = getSpeaker(match.group(1))
+ speaker = response[0]
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ data[i] = data[i].replace(match.group(1), speaker)
+ else:
+ speaker = None
+
+ # Furigana
+ match = re.search(r"^\[ruby\stext", data[i])
+ furiganaList = []
+ if match:
+ # Check next line and combine
+ while match:
+ furiganaList.append(data[i].replace("\n", ""))
+ del data[i]
+ match = re.search(r"^\[ruby\stext", data[i])
+ jaString = "".join(furiganaList)
+
+ # Ruby Text
+ furiganaList = re.findall(furiganaRegex, jaString)
+ for furigana in furiganaList:
+ jaString = jaString.replace(furigana[0], furigana[1])
+
+ data.insert(i, f"{jaString}[r]")
+
+ # Dialogue
+ match = re.search(lineRegexNoSpeaker, data[i])
+ jaString = None
+ if match:
+ # Find which group matched
+ for group_num in range(1, 5):
+ try:
+ if match.group(group_num):
+ jaString = match.group(group_num)
+ break
+ except IndexError:
+ break
+
+ # Skip if no valid string found
+ if not jaString:
+ i += 1
+ continue
+
+ # Combine w/ next line if necessary
+ repeatRegex = r"(.+?)\[[rpl]\]"
+ if i + 1 < len(data):
+ match = re.search(repeatRegex, data[i + 1])
+ while match and "[p]" not in data[i]:
+ jaString = jaString + match.group(1)
+ jaString = jaString.replace("_ ", "")
+ if "[p]" in data[i + 1]:
+ data[i] = f"{jaString}[p]\n"
+ else:
+ data[i] = jaString
+ del data[i + 1]
+ if i + 1 < len(data):
+ match = re.search(repeatRegex, data[i + 1])
+ else:
+ break
+
+ originalString = jaString
+
+ # Pass 1
+ if not translatedList:
+ # Remove any textwrap and commands
+ jaString = jaString.replace("[r]", " ")
+ jaString = jaString.replace("[l]", "")
+
+ # Ruby Text
+ furiganaList = re.findall(furiganaRegex, jaString)
+ for furigana in furiganaList:
+ jaString = jaString.replace(furigana[0], furigana[1])
+
+ # Strip Spaces
+ jaString = jaString.strip()
+
+ if jaString:
+ if speaker:
+ stringList.append(f"[{speaker}]: {jaString}")
+ else:
+ stringList.append(jaString)
+
+ # Pass 2
+ else:
+ # Get Text
+ if stringList:
+ # Grab and Pop
+ translatedText = stringList[0]
+ stringList.pop(0)
+
+ # Set to None if empty list
+ if len(stringList) <= 0:
+ stringList = None
+
+ # Remove speaker
+ if speaker != "":
+ matchSpeakerList = re.findall(r"^\[?(.+?)\]?\s?[|:]\s?", translatedText)
+ translatedText = re.sub(r"^\[?(.+?)\]?\s?[|:]\s?", "", translatedText)
+
+ # Avoid Crashes
+ translatedText = translatedText.replace("[", "(")
+ translatedText = translatedText.replace("]", ")")
+
+ # Textwrap
+ translatedText = dazedwrap.wrapText(translatedText, width=WIDTH)
+ translatedText = translatedText.replace("\n", "[r]")
+
+ # Set Data
+ data[i] = data[i].replace(originalString, translatedText)
+ # Save progress after each line change (we mutated, so tokens gate optional)
+ saveCheckLines(data, filename)
+
+ # Choices
+ match = re.search(choiceRegex, data[i])
+ if match:
+ # Pass 1
+ if not translatedList:
+ choiceList.append(match.group(1))
+ match = re.search(choiceRegex, data[i + 1])
+
+ # Pass 2
+ else:
+ # Grab and Pop
+ translatedText = choiceList[0]
+ choiceList.pop(0)
+
+ # Replace Spaces
+ translatedText = translatedText.replace(" ", "\u3000")
+
+ # Set
+ data[i] = data[i].replace(match.group(1), translatedText)
+ saveCheckLines(data, filename)
+
+ i += 1
+ else:
+ i += 1
+
+ # EOF
+ if not translatedList:
+ # String List
+ if stringList:
+ PBAR.total = len(stringList)
+ PBAR.refresh()
+ response = translateAI(stringList, "Reply with the English Translation")
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ stringListTL = response[0]
+
+ if len(stringList) != len(stringListTL):
+ # Mismatch
+ with LOCK:
+ if FILENAME not in MISMATCH:
+ MISMATCH.append(FILENAME)
+
+ # Choice List
+ if choiceList:
+ response = translateAI(choiceList, "Reply with the English TL of the Dialogue Choice")
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ choiceListTL = response[0]
+
+ if len(choiceList) != len(choiceListTL):
+ # Mismatch
+ with LOCK:
+ if FILENAME not in MISMATCH:
+ MISMATCH.append(FILENAME)
+
+ # Recursive call for Pass 2 with translated strings
+ result = translateTyrano(data, filename, [stringListTL, choiceListTL])
+ tokens[0] += result[0]
+ tokens[1] += result[1]
+
+ return tokens
+
+# Save some money and enter the character before translation
+def getSpeaker(speaker):
+ match speaker:
+ case "ファイン":
+ return ["Fine", [0, 0]]
+ case "":
+ return ["", [0, 0]]
+ case _:
+ # Find Speaker
+ for i in range(len(NAMESLIST)):
+ if speaker == NAMESLIST[i][0]:
+ return [NAMESLIST[i][1], [0, 0]]
+
+ # Translate and Store Speaker
+ response = translateAI(
+ f"{speaker}",
+ "Reply with the " + LANGUAGE + " translation of the NPC name.",
+ False,
+ )
+ response[0] = response[0].title()
+ response[0] = response[0].replace("'S", "'s")
+ response[0] = response[0].replace("Speaker: ", "")
+
+ # Retry if name doesn't translate for some reason
+ if re.search(r"([a-zA-Z？?])", response[0]) == None:
+ response = translateAI(
+ f"{speaker}",
+ "Reply with the " + LANGUAGE + " translation of the NPC name.",
+ False,
+ )
+ response[0] = response[0].title()
+ response[0] = response[0].replace("'S", "'s")
+
+ speakerList = [speaker, response[0]]
+ NAMESLIST.append(speakerList)
+ return response
+ return [speaker, [0, 0]]
+
+def translateAI(text, history, history_ctx=None):
+ """
+ Legacy wrapper function for the new shared translation utility.
+ This maintains compatibility with existing code while using the new shared implementation.
+ """
+ global PBAR, MISMATCH, FILENAME
+
+ # Update config estimate mode based on global ESTIMATE
+ TRANSLATION_CONFIG.estimateMode = bool(ESTIMATE)
+
+ # Call the new shared translation function
+ return sharedtranslateAI(
+ text=text,
+ history=history,
+ config=TRANSLATION_CONFIG,
+ filename=FILENAME,
+ pbar=PBAR,
+ lock=LOCK,
+ mismatchList=MISMATCH
+ )
diff --git a/modules/unity.py b/modules/unity.py
index d1c2f11..ec92c3e 100644
--- a/modules/unity.py
+++ b/modules/unity.py
@@ -1,353 +1,355 @@
-# Libraries
-import json
-import os
-import re
-import util.dazedwrap as dazedwrap
-import threading
-import time
-import traceback
-import tiktoken
-from pathlib import Path
-from colorama import Fore
-from dotenv import load_dotenv
-from retry import retry
-from tqdm import tqdm
-from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost, getPricingConfig, calculateCost
-import tempfile
-
-# OpenAI initialization centralized in util/translation.py
-
-# Globals
-MODEL = os.getenv("model")
-TIMEOUT = int(os.getenv("timeout"))
-LANGUAGE = os.getenv("language").capitalize()
-PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
-VOCAB = Path("vocab.txt").read_text(encoding="utf-8")
-LOCK = threading.Lock()
-WIDTH = int(os.getenv("width"))
-LISTWIDTH = int(os.getenv("listWidth"))
-NOTEWIDTH = 70
-MAXHISTORY = 10
-ESTIMATE = ""
-TOKENS = [0, 0]
-NAMESLIST = []
-NAMES = False # Output a list of all the character names found
-BRFLAG = False # If the game uses
instead
-FIXTEXTWRAP = True # Overwrites textwrap
-IGNORETLTEXT = False # Ignores all translated text.
-MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
-
-# tqdm Globals
-BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
-POSITION = 0
-LEAVE = False
-PBAR = None
-FILENAME = None
-
-# Full Width
-ascii_to_wide = dict((i, chr(i + 0xFEE0)) for i in range(0x21, 0x7F))
-ascii_to_wide.update({0x20: "\u3000", 0x2D: "\u2212"}) # space and minus
-wide_to_ascii = dict((i, chr(i - 0xFEE0)) for i in range(0xFF01, 0xFF5F))
-wide_to_ascii.update({0x3000: " ", 0x2212: "-"}) # space and minus
-
-# Regex - Need to change this if you want to translate from/to other languages. Default is Japanese Regex
-LANGREGEX = r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+"
-
-# Get pricing configuration based on the model
-PRICING_CONFIG = getPricingConfig(MODEL)
-INPUTAPICOST = PRICING_CONFIG["inputAPICost"]
-OUTPUTAPICOST = PRICING_CONFIG["outputAPICost"]
-BATCHSIZE = PRICING_CONFIG["batchSize"]
-FREQUENCY_PENALTY = PRICING_CONFIG["frequencyPenalty"]
-
-# Initialize Translation Config
-TRANSLATION_CONFIG = TranslationConfig(
- model=MODEL,
- language=LANGUAGE,
- prompt=PROMPT,
- vocab=VOCAB,
- langRegex=LANGREGEX,
- batchSize=BATCHSIZE,
- maxHistory=MAXHISTORY,
- estimateMode=False # Will be set dynamically based on ESTIMATE
-)
-LEAVE = False
-
-def handleUnity(filename, estimate):
- global ESTIMATE, FILENAME
- ESTIMATE = estimate
- FILENAME = filename
-
- if ESTIMATE:
- start = time.time()
- translatedData = openFiles(filename)
-
- # Print Result
- end = time.time()
- tqdm.write(getResultString(translatedData, end - start, filename))
- with LOCK:
- TOKENS[0] += translatedData[1][0]
- TOKENS[1] += translatedData[1][1]
-
- # Print Total
- totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
-
- # Print any errors on maps
- if len(MISMATCH) > 0:
- return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
- else:
- return totalString
-
- else:
- try:
- with open("translated/" + filename, "w", encoding="utf8", errors="ignore") as outFile:
- start = time.time()
- translatedData = openFiles(filename)
-
- # Print Result
- end = time.time()
- outFile.writelines(translatedData[0])
- tqdm.write(getResultString(translatedData, end - start, filename))
- with LOCK:
- TOKENS[0] += translatedData[1][0]
- TOKENS[1] += translatedData[1][1]
- except Exception:
- traceback.print_exc()
- return "Fail"
-
- return getResultString(["", TOKENS, None], end - start, "TOTAL")
-
-
-def getResultString(translatedData, translationTime, filename):
- # File Print String
- cost = calculateCost(translatedData[1][0], translatedData[1][1], MODEL)
- totalTokenstring = (
- Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
- "[Output: "
- + str(translatedData[1][1])
- + "]" "[Cost: ${:,.4f}".format(cost)
- + "]"
- )
- timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
-
- if translatedData[2] == None:
- # Success
- return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET
-
- else:
- # Fail
- try:
- raise translatedData[2]
- except Exception as e:
- traceback.print_exc()
- errorString = str(e) + Fore.RED
- return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET
-
-
-def openFiles(filename):
- with open("files/" + filename, "r", encoding="utf8") as readFile:
- translatedData = parseUnity(readFile, filename)
-
- # Delete lines marked for deletion
- finalData = []
- for line in translatedData[0]:
- if line != "\\d\n":
- finalData.append(line)
- translatedData[0] = finalData
-
- return translatedData
-
-
-def parseUnity(readFile, filename):
- totalTokens = [0, 0]
-
- # Read File into data
- data = readFile.readlines()
-
- # Create Progress Bar
- with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
- pbar.desc = filename
-
- try:
- result = translateUnity(data, pbar, filename, [])
- totalTokens[0] += result[0]
- totalTokens[1] += result[1]
- except Exception as e:
- traceback.print_exc()
- return [data, totalTokens, e]
- return [data, totalTokens, None]
-
-
-def save_progress_lines(lines, filename, encoding="utf-8"):
- """Atomically save current line-based translation progress."""
- try:
- if ESTIMATE:
- return
- os.makedirs("translated", exist_ok=True)
- tmp_fd, tmp_path = tempfile.mkstemp(prefix=f"{filename}.", suffix=".tmp", dir="translated")
- try:
- with os.fdopen(tmp_fd, "w", encoding=encoding, newline="\n", errors="ignore") as tmp_file:
- tmp_file.writelines(lines)
- os.replace(tmp_path, os.path.join("translated", filename))
- finally:
- if os.path.exists(tmp_path):
- try:
- os.remove(tmp_path)
- except OSError:
- pass
- except Exception:
- traceback.print_exc()
-
-
-def saveCheckLines(lines, filename, tokens=None, encoding="utf-8"):
- """Save progress only when tokens indicate work or when explicitly called after a mutation."""
- try:
- if tokens is not None:
- if not (isinstance(tokens, (list, tuple)) and len(tokens) >= 2 and (tokens[0] or tokens[1])):
- return
- save_progress_lines(lines, filename, encoding=encoding)
- except Exception:
- traceback.print_exc()
-
-
-def translateUnity(data, pbar, filename, translatedList):
- stringList = []
- tokens = [0, 0]
- global LOCK, ESTIMATE, PBAR
- PBAR = pbar
- i = 0
-
- # Dialogue
- while i < len(data):
- # Lines
- regex = r"(.*?)=(.*)"
- match = re.search(regex, data[i])
- if match:
- leftString = match.group(1)
- rightString = match.group(2)
-
- # Validate Japanese Text
- if not re.search(LANGREGEX, leftString) and IGNORETLTEXT:
- i += 1
- continue
-
- # Pass 1
- if translatedList == []:
- jaString = leftString
-
- # Remove textwrap
- # jaString = jaString.replace("\\n", " ")
-
- # Add String
- stringList.append(jaString.strip())
-
- # Pass 2
- else:
- # Get Text
- if translatedList:
- # Grab and Pop
- translatedText = translatedList[0]
- translatedList.pop(0)
-
- # Set to None if empty list
- if len(translatedList) <= 0:
- translatedList = None
-
- # Textwrap
- # translatedText = dazedwrap.wrapText(translatedText, width=WIDTH)
- # translatedText = translatedText.replace("\n", "\\n")
-
- # Remove Double Spaces and =
- translatedText = translatedText.replace(" ", " ")
- translatedText = translatedText.replace("=", "->")
-
- # Set Data
- data[i] = f"{leftString}={translatedText}\n"
- saveCheckLines(data, filename)
- i += 1
-
- # Nothing relevant. Skip Line.
- else:
- i += 1
-
- # EOF
- if len(stringList) > 0:
- # Set Progress
- pbar.total = len(stringList)
- pbar.refresh()
-
- # Translate
- response = translateAI(stringList, "")
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- translatedList = response[0]
-
- # Set Strings
- if len(stringList) == len(translatedList):
- translateUnity(data, pbar, filename, translatedList)
-
- # Mismatch
- else:
- with LOCK:
- if filename not in MISMATCH:
- MISMATCH.append(filename)
- return tokens
-
-# Save some money and enter the character before translation
-def getSpeaker(speaker):
- match speaker:
- case "ファイン":
- return ["Fine", [0, 0]]
- case "":
- return ["", [0, 0]]
- case _:
- # Find Speaker
- for i in range(len(NAMESLIST)):
- if speaker == NAMESLIST[i][0]:
- return [NAMESLIST[i][1], [0, 0]]
-
- # Translate and Store Speaker
- response = translateAI(
- f"{speaker}",
- "Reply with the " + LANGUAGE + " translation of the NPC name.",
- False,
- )
- response[0] = response[0].title()
- response[0] = response[0].replace("'S", "'s")
- response[0] = response[0].replace("Speaker: ", "")
-
- # Retry if name doesn't translate for some reason
- if re.search(r"([a-zA-Z？?])", response[0]) == None:
- response = translateAI(
- f"{speaker}",
- "Reply with the " + LANGUAGE + " translation of the NPC name.",
- False,
- )
- response[0] = response[0].title()
- response[0] = response[0].replace("'S", "'s")
-
- speakerList = [speaker, response[0]]
- NAMESLIST.append(speakerList)
- return response
- return [speaker, [0, 0]]
-
-def translateAI(text, history, history_ctx=None):
- """
- Legacy wrapper function for the new shared translation utility.
- This maintains compatibility with existing code while using the new shared implementation.
- """
- global PBAR, MISMATCH, FILENAME
-
- # Update config estimate mode based on global ESTIMATE
- TRANSLATION_CONFIG.estimateMode = bool(ESTIMATE)
-
- # Call the new shared translation function
- return sharedtranslateAI(
- text=text,
- history=history,
- config=TRANSLATION_CONFIG,
- filename=FILENAME,
- pbar=PBAR,
- lock=LOCK,
- mismatchList=MISMATCH
- )
+# Libraries
+import json
+import os
+import re
+import util.dazedwrap as dazedwrap
+import threading
+import time
+import traceback
+import tiktoken
+from pathlib import Path
+from colorama import Fore
+from dotenv import load_dotenv
+from retry import retry
+from tqdm import tqdm
+from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost, getPricingConfig, calculateCost
+import tempfile
+
+# OpenAI initialization centralized in util/translation.py
+
+# Globals
+MODEL = os.getenv("model")
+TIMEOUT = int(os.getenv("timeout"))
+LANGUAGE = os.getenv("language").capitalize()
+from util.paths import PROMPT_PATH, VOCAB_PATH
+
+PROMPT = PROMPT_PATH.read_text(encoding="utf-8")
+VOCAB = VOCAB_PATH.read_text(encoding="utf-8")
+LOCK = threading.Lock()
+WIDTH = int(os.getenv("width"))
+LISTWIDTH = int(os.getenv("listWidth"))
+NOTEWIDTH = 70
+MAXHISTORY = 10
+ESTIMATE = ""
+TOKENS = [0, 0]
+NAMESLIST = []
+NAMES = False # Output a list of all the character names found
+BRFLAG = False # If the game uses
instead
+FIXTEXTWRAP = True # Overwrites textwrap
+IGNORETLTEXT = False # Ignores all translated text.
+MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
+
+# tqdm Globals
+BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
+POSITION = 0
+LEAVE = False
+PBAR = None
+FILENAME = None
+
+# Full Width
+ascii_to_wide = dict((i, chr(i + 0xFEE0)) for i in range(0x21, 0x7F))
+ascii_to_wide.update({0x20: "\u3000", 0x2D: "\u2212"}) # space and minus
+wide_to_ascii = dict((i, chr(i - 0xFEE0)) for i in range(0xFF01, 0xFF5F))
+wide_to_ascii.update({0x3000: " ", 0x2212: "-"}) # space and minus
+
+# Regex - Need to change this if you want to translate from/to other languages. Default is Japanese Regex
+LANGREGEX = r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+"
+
+# Get pricing configuration based on the model
+PRICING_CONFIG = getPricingConfig(MODEL)
+INPUTAPICOST = PRICING_CONFIG["inputAPICost"]
+OUTPUTAPICOST = PRICING_CONFIG["outputAPICost"]
+BATCHSIZE = PRICING_CONFIG["batchSize"]
+FREQUENCY_PENALTY = PRICING_CONFIG["frequencyPenalty"]
+
+# Initialize Translation Config
+TRANSLATION_CONFIG = TranslationConfig(
+ model=MODEL,
+ language=LANGUAGE,
+ prompt=PROMPT,
+ vocab=VOCAB,
+ langRegex=LANGREGEX,
+ batchSize=BATCHSIZE,
+ maxHistory=MAXHISTORY,
+ estimateMode=False # Will be set dynamically based on ESTIMATE
+)
+LEAVE = False
+
+def handleUnity(filename, estimate):
+ global ESTIMATE, FILENAME
+ ESTIMATE = estimate
+ FILENAME = filename
+
+ if ESTIMATE:
+ start = time.time()
+ translatedData = openFiles(filename)
+
+ # Print Result
+ end = time.time()
+ tqdm.write(getResultString(translatedData, end - start, filename))
+ with LOCK:
+ TOKENS[0] += translatedData[1][0]
+ TOKENS[1] += translatedData[1][1]
+
+ # Print Total
+ totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
+
+ # Print any errors on maps
+ if len(MISMATCH) > 0:
+ return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
+ else:
+ return totalString
+
+ else:
+ try:
+ with open("translated/" + filename, "w", encoding="utf8", errors="ignore") as outFile:
+ start = time.time()
+ translatedData = openFiles(filename)
+
+ # Print Result
+ end = time.time()
+ outFile.writelines(translatedData[0])
+ tqdm.write(getResultString(translatedData, end - start, filename))
+ with LOCK:
+ TOKENS[0] += translatedData[1][0]
+ TOKENS[1] += translatedData[1][1]
+ except Exception:
+ traceback.print_exc()
+ return "Fail"
+
+ return getResultString(["", TOKENS, None], end - start, "TOTAL")
+
+
+def getResultString(translatedData, translationTime, filename):
+ # File Print String
+ cost = calculateCost(translatedData[1][0], translatedData[1][1], MODEL)
+ totalTokenstring = (
+ Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
+ "[Output: "
+ + str(translatedData[1][1])
+ + "]" "[Cost: ${:,.4f}".format(cost)
+ + "]"
+ )
+ timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
+
+ if translatedData[2] == None:
+ # Success
+ return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET
+
+ else:
+ # Fail
+ try:
+ raise translatedData[2]
+ except Exception as e:
+ traceback.print_exc()
+ errorString = str(e) + Fore.RED
+ return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET
+
+
+def openFiles(filename):
+ with open("files/" + filename, "r", encoding="utf8") as readFile:
+ translatedData = parseUnity(readFile, filename)
+
+ # Delete lines marked for deletion
+ finalData = []
+ for line in translatedData[0]:
+ if line != "\\d\n":
+ finalData.append(line)
+ translatedData[0] = finalData
+
+ return translatedData
+
+
+def parseUnity(readFile, filename):
+ totalTokens = [0, 0]
+
+ # Read File into data
+ data = readFile.readlines()
+
+ # Create Progress Bar
+ with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
+ pbar.desc = filename
+
+ try:
+ result = translateUnity(data, pbar, filename, [])
+ totalTokens[0] += result[0]
+ totalTokens[1] += result[1]
+ except Exception as e:
+ traceback.print_exc()
+ return [data, totalTokens, e]
+ return [data, totalTokens, None]
+
+
+def save_progress_lines(lines, filename, encoding="utf-8"):
+ """Atomically save current line-based translation progress."""
+ try:
+ if ESTIMATE:
+ return
+ os.makedirs("translated", exist_ok=True)
+ tmp_fd, tmp_path = tempfile.mkstemp(prefix=f"{filename}.", suffix=".tmp", dir="translated")
+ try:
+ with os.fdopen(tmp_fd, "w", encoding=encoding, newline="\n", errors="ignore") as tmp_file:
+ tmp_file.writelines(lines)
+ os.replace(tmp_path, os.path.join("translated", filename))
+ finally:
+ if os.path.exists(tmp_path):
+ try:
+ os.remove(tmp_path)
+ except OSError:
+ pass
+ except Exception:
+ traceback.print_exc()
+
+
+def saveCheckLines(lines, filename, tokens=None, encoding="utf-8"):
+ """Save progress only when tokens indicate work or when explicitly called after a mutation."""
+ try:
+ if tokens is not None:
+ if not (isinstance(tokens, (list, tuple)) and len(tokens) >= 2 and (tokens[0] or tokens[1])):
+ return
+ save_progress_lines(lines, filename, encoding=encoding)
+ except Exception:
+ traceback.print_exc()
+
+
+def translateUnity(data, pbar, filename, translatedList):
+ stringList = []
+ tokens = [0, 0]
+ global LOCK, ESTIMATE, PBAR
+ PBAR = pbar
+ i = 0
+
+ # Dialogue
+ while i < len(data):
+ # Lines
+ regex = r"(.*?)=(.*)"
+ match = re.search(regex, data[i])
+ if match:
+ leftString = match.group(1)
+ rightString = match.group(2)
+
+ # Validate Japanese Text
+ if not re.search(LANGREGEX, leftString) and IGNORETLTEXT:
+ i += 1
+ continue
+
+ # Pass 1
+ if translatedList == []:
+ jaString = leftString
+
+ # Remove textwrap
+ # jaString = jaString.replace("\\n", " ")
+
+ # Add String
+ stringList.append(jaString.strip())
+
+ # Pass 2
+ else:
+ # Get Text
+ if translatedList:
+ # Grab and Pop
+ translatedText = translatedList[0]
+ translatedList.pop(0)
+
+ # Set to None if empty list
+ if len(translatedList) <= 0:
+ translatedList = None
+
+ # Textwrap
+ # translatedText = dazedwrap.wrapText(translatedText, width=WIDTH)
+ # translatedText = translatedText.replace("\n", "\\n")
+
+ # Remove Double Spaces and =
+ translatedText = translatedText.replace(" ", " ")
+ translatedText = translatedText.replace("=", "->")
+
+ # Set Data
+ data[i] = f"{leftString}={translatedText}\n"
+ saveCheckLines(data, filename)
+ i += 1
+
+ # Nothing relevant. Skip Line.
+ else:
+ i += 1
+
+ # EOF
+ if len(stringList) > 0:
+ # Set Progress
+ pbar.total = len(stringList)
+ pbar.refresh()
+
+ # Translate
+ response = translateAI(stringList, "")
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ translatedList = response[0]
+
+ # Set Strings
+ if len(stringList) == len(translatedList):
+ translateUnity(data, pbar, filename, translatedList)
+
+ # Mismatch
+ else:
+ with LOCK:
+ if filename not in MISMATCH:
+ MISMATCH.append(filename)
+ return tokens
+
+# Save some money and enter the character before translation
+def getSpeaker(speaker):
+ match speaker:
+ case "ファイン":
+ return ["Fine", [0, 0]]
+ case "":
+ return ["", [0, 0]]
+ case _:
+ # Find Speaker
+ for i in range(len(NAMESLIST)):
+ if speaker == NAMESLIST[i][0]:
+ return [NAMESLIST[i][1], [0, 0]]
+
+ # Translate and Store Speaker
+ response = translateAI(
+ f"{speaker}",
+ "Reply with the " + LANGUAGE + " translation of the NPC name.",
+ False,
+ )
+ response[0] = response[0].title()
+ response[0] = response[0].replace("'S", "'s")
+ response[0] = response[0].replace("Speaker: ", "")
+
+ # Retry if name doesn't translate for some reason
+ if re.search(r"([a-zA-Z？?])", response[0]) == None:
+ response = translateAI(
+ f"{speaker}",
+ "Reply with the " + LANGUAGE + " translation of the NPC name.",
+ False,
+ )
+ response[0] = response[0].title()
+ response[0] = response[0].replace("'S", "'s")
+
+ speakerList = [speaker, response[0]]
+ NAMESLIST.append(speakerList)
+ return response
+ return [speaker, [0, 0]]
+
+def translateAI(text, history, history_ctx=None):
+ """
+ Legacy wrapper function for the new shared translation utility.
+ This maintains compatibility with existing code while using the new shared implementation.
+ """
+ global PBAR, MISMATCH, FILENAME
+
+ # Update config estimate mode based on global ESTIMATE
+ TRANSLATION_CONFIG.estimateMode = bool(ESTIMATE)
+
+ # Call the new shared translation function
+ return sharedtranslateAI(
+ text=text,
+ history=history,
+ config=TRANSLATION_CONFIG,
+ filename=FILENAME,
+ pbar=PBAR,
+ lock=LOCK,
+ mismatchList=MISMATCH
+ )
diff --git a/modules/wolf.py b/modules/wolf.py
index 6b61cdc..20a1868 100644
--- a/modules/wolf.py
+++ b/modules/wolf.py
@@ -1,2699 +1,2701 @@
-# Libraries
-import json
-import os
-import re
-import util.dazedwrap as dazedwrap
-import threading
-import time
-import traceback
-import tiktoken
-# Removed concurrent.futures usage for simplicity; running synchronously
-from pathlib import Path
-from colorama import Fore
-from dotenv import load_dotenv
-from retry import retry
-from tqdm import tqdm
-from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost, getPricingConfig, calculateCost
-
-# OpenAI initialization centralized in util/translation.py
-
-# Globals
-MODEL = os.getenv("model")
-TIMEOUT = int(os.getenv("timeout"))
-LANGUAGE = os.getenv("language").capitalize()
-PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
-VOCAB = Path("vocab.txt").read_text(encoding="utf-8")
-LOCK = threading.Lock()
-WIDTH = int(os.getenv("width"))
-LISTWIDTH = int(os.getenv("listWidth"))
-NOTEWIDTH = int(os.getenv("noteWidth"))
-MAXHISTORY = 10
-ESTIMATE = ""
-TOKENS = [0, 0]
-NAMESLIST = [] # Keep list for consistency
-TERMSLIST = [] # Keep list for consistency
-NAMES = False # Output a list of all the character names found
-BRFLAG = False # If the game uses
instead
-FIXTEXTWRAP = True # Overwrites textwrap
-IGNORETLTEXT = False # Ignores all translated text.
-MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
-FILENAME = None
-BRACKETNAMES = False
-
-# Regex - Need to change this if you want to translate from/to other languages. Default is Japanese Regex
-LANGREGEX = r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+"
-
-# Get pricing configuration based on the model
-PRICING_CONFIG = getPricingConfig(MODEL)
-INPUTAPICOST = PRICING_CONFIG["inputAPICost"]
-OUTPUTAPICOST = PRICING_CONFIG["outputAPICost"]
-BATCHSIZE = PRICING_CONFIG["batchSize"]
-FREQUENCY_PENALTY = PRICING_CONFIG["frequencyPenalty"]
-
-# tqdm Globals
-BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
-POSITION = 0
-LEAVE = False
-PBAR = None
-FILENAME = None
-
-# Dialogue / Choices
-CODE101 = False
-CODE102 = False
-
-# Picture
-CODE150 = False
-
-# Set String (Fragile but necessary)
-CODE122 = False
-
-# Other
-CODE210 = False
-CODE300 = False
-CODE250 = False
-
-# Database
-SCENARIOFLAG = False
-OPTIONSFLAG = True
-NPCFLAG = False
-DBNAMEFLAG = False
-DBVALUEFLAG = False
-ITEMFLAG = False
-STATEFLAG = False
-ENEMYFLAG = False
-ARMORFLAG = False
-WEAPONFLAG = False
-SKILLFLAG = False
-
-# Initialize Translation Config
-TRANSLATION_CONFIG = TranslationConfig(
- model=MODEL,
- language=LANGUAGE,
- prompt=PROMPT,
- vocab=VOCAB,
- langRegex=LANGREGEX,
- batchSize=BATCHSIZE,
- maxHistory=MAXHISTORY,
- estimateMode=False # Will be set dynamically based on ESTIMATE
-)
-LEAVE = False
-
-def handleWOLF(filename, estimate):
- global ESTIMATE, TOKENS, FILENAME
- ESTIMATE = estimate
- FILENAME = filename
-
- # Translate
- start = time.time()
- translatedData = openFiles(filename)
-
- # Translate
- if not estimate:
- try:
- with open("translated/" + filename, "w", encoding="utf-8", newline="\n") as outFile:
- json.dump(translatedData[0], outFile, ensure_ascii=False, indent=4)
- except Exception:
- traceback.print_exc()
- return "Fail"
-
- # Print File
- end = time.time()
- tqdm.write(getResultString(translatedData, end - start, filename))
- with LOCK:
- TOKENS[0] += translatedData[1][0]
- TOKENS[1] += translatedData[1][1]
-
- # Print Total
- totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
-
- # Print any errors on maps
- if len(MISMATCH) > 0:
- return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
- else:
- return totalString
-
-
-def openFiles(filename):
- with open("files/" + filename, "r", encoding="utf-8-sig") as f:
- data = json.load(f)
-
- # Map Files
- if "'events':" in str(data):
- if len(data["events"]) > 0:
- translatedData = parseMap(data, filename)
- else:
- return [data, [0, 0], None]
-
- # Map Files
- elif "'types':" in str(data):
- translatedData = parseDB(data, filename)
-
- # Other Files
- elif "'commands':" in str(data):
- translatedData = parseOther(data, filename)
-
- else:
- raise NameError(filename + " Not Supported")
-
- return translatedData
-
-
-def getResultString(translatedData, translationTime, filename):
- # File Print String
- cost = calculateCost(translatedData[1][0], translatedData[1][1], MODEL)
- totalTokenstring = (
- Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
- "[Output: "
- + str(translatedData[1][1])
- + "]" "[Cost: ${:,.4f}".format(cost)
- + "]"
- )
- timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
-
- if translatedData[2] is None:
- # Success
- return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET
- else:
- # Fail
- try:
- raise translatedData[2]
- except Exception as e:
- traceback.print_exc()
- errorString = str(e) + Fore.RED
- return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET
-
-
-def save_progress_json(data, filename):
- """Atomically write current JSON data to translated/filename; skip in estimate mode."""
- try:
- if ESTIMATE:
- return
- os.makedirs("translated", exist_ok=True)
- tmp_path = os.path.join("translated", f"{filename}.tmp")
- final_path = os.path.join("translated", filename)
- with open(tmp_path, "w", encoding="utf-8", newline="\n") as outFile:
- json.dump(data, outFile, ensure_ascii=False, indent=4)
- os.replace(tmp_path, final_path)
- except Exception:
- traceback.print_exc()
-
-
-def maybe_save_progress_json(data, filename, tokens):
- """Save JSON progress only when tokens indicate actual translation work."""
- try:
- if not tokens:
- return
- if isinstance(tokens, (list, tuple)) and len(tokens) >= 2 and (tokens[0] or tokens[1]):
- save_progress_json(data, filename)
- except Exception:
- traceback.print_exc()
-
-
-def parseOther(data, filename):
- totalTokens = [0, 0]
- totalLines = 0
- events = data["commands"]
- global LOCK
-
- # Thread for each page in file
- with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
- pbar.desc = filename
- pbar.total = totalLines
- translationData = searchCodes(events, pbar, [], filename)
- try:
- totalTokens[0] += translationData[0]
- totalTokens[1] += translationData[1]
- except Exception as e:
- return [data, totalTokens, e]
- finally:
- maybe_save_progress_json(data, filename, translationData)
- return [data, totalTokens, None]
-
-
-def parseDB(data, filename):
- totalTokens = [0, 0]
- totalLines = 0
- events = data["types"]
- global LOCK
-
- # Thread for each page in file
- with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
- pbar.desc = filename
- pbar.total = totalLines
- translationData = searchDB(events, pbar, [], filename)
- try:
- totalTokens[0] += translationData[0]
- totalTokens[1] += translationData[1]
- except Exception as e:
- return [data, totalTokens, e]
- finally:
- maybe_save_progress_json(data, filename, translationData)
- return [data, totalTokens, None]
-
-
-def parseMap(data, filename):
- totalTokens = [0, 0]
- totalLines = 0
- events = data["events"]
- global LOCK
-
- # Get total for progress bar
- for event in events:
- if event is not None:
- for page in event["pages"]:
- totalLines += len(page["list"])
-
- # Process pages synchronously and persist after each
- with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar:
- pbar.desc = filename
- pbar.total = totalLines
- for event in events:
- if event is not None:
- for page in event["pages"]:
- if page is not None:
- try:
- tt = searchCodes(page["list"], pbar, None, filename)
- totalTokens[0] += tt[0]
- totalTokens[1] += tt[1]
- except Exception as e:
- return [data, totalTokens, e]
- finally:
- maybe_save_progress_json(data, filename, tt)
- return [data, totalTokens, None]
-
-
-def searchCodes(events, pbar, jobList, filename):
- # Lists
- if jobList:
- stringList = jobList[0]
- list210 = jobList[1]
- list300 = jobList[2]
- list150 = jobList[3]
- list250 = jobList[4]
- list122 = jobList[5]
- setData = True
- else:
- stringList = []
- list210 = []
- list300 = []
- list150 = []
- list250 = []
- list122 = []
- setData = False
-
- # Other
- codeList = events
- textHistory = []
- totalTokens = [0, 0]
- translatedText = ""
- speaker = ""
- nametag = ""
- initialJAString = ""
- global LOCK, NAMESLIST, MISMATCH, PBAR, FILENAME
- FILENAME = filename
- PBAR = pbar
-
- # Calculate Total Length
- code_flags = {102: CODE102, 122: CODE122, 300: CODE300, 250: CODE250}
- totalList = 0
- for code_item in codeList:
- if code_flags.get(code_item["code"], False):
- totalList += 1
- pbar.total = totalList
- pbar.refresh()
-
- # Begin Parsing File
- try:
- # Iterate through events
- i = 0
- while i < len(codeList):
- ### Event Code: 101 Message
- if codeList[i]["code"] == 101 and CODE101 == True:
- speakerRegex = r"@\d+\r?\n(.*?)：?\r?\n" # Default: r"@\d+\r?\n(.*)：?\r?\n""
- textRegex = r"@\d+\r?\n.*?：?\r?\n(.*)" # Default: r"@\d+\r?\n.*：?\r?\n(.*)"
- # Alternative patterns for messages without @\d+ prefix
- speakerRegexAlt = r"^(.*?)：\r?\n" # Matches "Name：\n"
- textRegexAlt = r"^.*?：\r?\n(.*)" # Matches text after "Name：\n"
-
- # Grab String
- jaString = codeList[i]["stringArgs"][0]
- speaker = ""
-
- # Grab Speaker
- if "\n" in jaString:
- match = re.search(speakerRegex, jaString)
- if not match:
- # Try alternative pattern without @\d+ prefix
- match = re.search(speakerRegexAlt, jaString)
- if match:
- # TL Speaker
- response = getSpeaker(match.group(1))
- speaker = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Set Name
- codeList[i]["stringArgs"][0] = codeList[i]["stringArgs"][0].replace(match.group(1), speaker)
-
- # Grab Only Text
- match = re.search(textRegex, jaString, flags=re.DOTALL)
- if not match:
- # Try alternative pattern without @\d+ prefix
- match = re.search(textRegexAlt, jaString, flags=re.DOTALL)
- if not match and re.search(LANGREGEX, jaString):
- # No speaker pattern found, treat entire string as text to translate
- jaString = jaString
- initialJAString = jaString
- elif match:
- jaString = match.group(1)
- initialJAString = jaString
- else:
- jaString = None # Skip non-translatable text
-
- if jaString is not None:
-
- # Remove Textwrap
- jaString = jaString.replace("\r", "")
- jaString = jaString.replace("\n", " ")
-
- # 1st Pass (Save Text to List)
- if not setData:
- if speaker == "":
- stringList.append(jaString)
- else:
- stringList.append(f"[{speaker}]: {jaString}")
-
- # 2nd Pass (Set Text)
- else:
- # Grab Translated String
- translatedText = stringList[0]
-
- # Remove speaker
- matchSpeakerList = re.findall(r"^(\[.+?\]\s?[|:]\s?)\s?", translatedText)
- if len(matchSpeakerList) > 0:
- translatedText = translatedText.replace(matchSpeakerList[0], "")
-
- # Textwrap
- if FIXTEXTWRAP is True:
- translatedText = dazedwrap.wrapText(translatedText, width=WIDTH)
-
- # Set Data
- codeList[i]["stringArgs"][0] = codeList[i]["stringArgs"][0].replace(initialJAString, translatedText)
-
- # Reset Data and Pop Item
- stringList.pop(0)
-
- ### Event Code: 102 Choices
- if codeList[i]["code"] == 102 and CODE102 == True:
- # Grab Choice List
- choiceList = []
- jaChoiceList = codeList[i]["stringArgs"]
-
- # Filter Empty
- for j in range(len(jaChoiceList)):
- if jaChoiceList[j]:
- choiceList.append(jaChoiceList[j])
-
- # Translate
- if 'jaString' in locals():
- choiceString = f"Previous Line: {jaString}\n\nReply with the {LANGUAGE} translation of the dialogue choice"
- else:
- choiceString = f"Reply with the {LANGUAGE} translation of the dialogue choice"
- response = translateAI(
- choiceList,
- choiceString,
- True,
- )
- translatedChoiceList = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Validate and Set Data
- if len(translatedChoiceList) == len(choiceList):
- for j in range(len(jaChoiceList)):
- if jaChoiceList[j]:
- codeList[i]["stringArgs"][j] = translatedChoiceList[0]
- translatedChoiceList.pop(0)
-
- ### Event Code: 210 Common Event
- if codeList[i]["code"] == 210 and CODE210 == True:
- # Speaker Event
- if "stringArgs" in codeList[i] and codeList[i]["intArgs"][0] == None and len(codeList[i]["stringArgs"]) == 2:
- response = getSpeaker(codeList[i]["stringArgs"][1])
- totalTokens[1] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Set Data
- codeList[i]["stringArgs"][1] = response[0]
-
- # Logs
- elif "stringArgs" in codeList[i] and codeList[i]["intArgs"][0] == 500725 and len(codeList[i]["stringArgs"]) == 2:
- # Grab String
- jaString = codeList[i]["stringArgs"][1]
- initialJAString = jaString
-
- # Remove Textwrap
- jaString = jaString.replace("\r", "")
- jaString = jaString.replace("\n", " ")
-
- # 1st Pass (Save Text to List)
- if not setData:
- list210.append(jaString)
-
- # 2nd Pass (Set Text)
- else:
- # Grab Translated String
- translatedText = list210[0]
-
- # Textwrap
- if FIXTEXTWRAP is True:
- translatedText = dazedwrap.wrapText(translatedText, width=WIDTH)
-
- # Set Data
- codeList[i]["stringArgs"][1] = translatedText
-
- # Pop Item
- list210.pop(0)
-
- ### Event Code: 122 SetString
- if codeList[i]["code"] == 122 and CODE122 == True:
- if "stringArgs" in codeList[i] and len(codeList[i]["stringArgs"]) > 0:
- # Grab String
- jaString = re.search(r"^\n?(.*)\n?$", codeList[i]["stringArgs"][0])
- if jaString:
- jaString = jaString.group(1)
- else:
- jaString = codeList[i]["stringArgs"][0]
-
- originalString = jaString
-
- # Remove Textwrap
- jaString = jaString.replace("\r", "")
- jaString = jaString.replace("\n", " ")
-
- # Check if this is a translatable string
- if (
- not re.search(r"\.[\w]+$", jaString)
- and jaString != ""
- and "_" not in jaString
- and '",' not in jaString
- and "/" not in jaString
- and re.search(LANGREGEX, jaString)
- ):
- # 1st Pass (Save Text to List)
- if not setData:
- list122.append(jaString)
-
- # 2nd Pass (Set Text)
- else:
- if len(list122) > 0:
- # Grab Translated String
- translatedText = list122[0]
-
- # Textwrap
- if FIXTEXTWRAP is True:
- translatedText = dazedwrap.wrapText(translatedText, width=WIDTH)
-
- # Set Data
- codeList[i]["stringArgs"][0] = codeList[i]["stringArgs"][0].replace(originalString, translatedText)
-
- # Pop Item
- list122.pop(0)
-
- ### Event Code: 150 Picture String
- if codeList[i]["code"] == 150 and CODE150 == True:
- if "stringArgs" in codeList[i] and len(codeList[i]["stringArgs"]) > 0:
- font150 = "\\f[8]"
- # Grab String
- jaString = codeList[i]["stringArgs"][0]
-
- # Japanses Text Only
- if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", jaString):
- i += 1
- continue
-
- # Remove Textwrap
- # jaString = jaString.replace("\n", " ")
- # jaString = jaString.replace("\r", "")
-
- # Translate Other Strings [Specific Files Only]
- if (
- not re.search(r"\.[\w]+$", jaString)
- and jaString != ""
- and "_" not in jaString
- and '",' not in jaString
- # and ">" not in jaString
- # and "<" not in jaString
- ):
- # Pass 1 (Save Text to List)
- if not setData:
- list150.append(jaString)
-
- # Pass 2 (Set Text)
- else:
- translatedText = list150[0]
-
- # Textwrap
- # translatedText = dazedwrap.wrapText(translatedText, WIDTH)
-
- # Set String with font formatting
- codeList[i]["stringArgs"][0] = re.sub(r'\\f\[(\d+)\]', lambda x: f'\\f[{int(x.group(1))-2}]', translatedText)
-
- # Pop processed item
- list150.pop(0)
-
- ### Event Code: 300 Common Events
- if codeList[i]["code"] == 300 and CODE300 == True and "stringArgs" in codeList[i] and len(codeList[i]["stringArgs"]) > 1:
- # Choices
- if codeList[i]["stringArgs"][0] == "[共]汎用ウィンドウ生成" or codeList[i]["stringArgs"][0] == "選択肢/確認":
- # Grab String
- choiceList = codeList[i]["stringArgs"][1].split(",")
-
- # Translate Question
- question = codeList[i]['stringArgs'][2]
- response = translateAI(question, "Reply with the {LANGUAGE} translation")
- translatedText = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Translate Question
- jaString = translatedText
- codeList[i]['stringArgs'][2] = translatedText
-
- # Translate Choices
- if 'jaString' in locals() and jaString:
- response = translateAI(choiceList, jaString)
- else:
- response = translateAI(choiceList, f"Reply with the {LANGUAGE} translation of the dialogue choice")
- choiceListTL = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Replace Commas
- for j in range(len(choiceListTL)):
- choiceListTL[j] = choiceListTL[j].replace(", ", "、")
-
- # Convert to String and Set
- translatedText = ",".join(choiceListTL)
- codeList[i]["stringArgs"][1] = translatedText
-
- # Dialogue
- elif codeList[i]["stringArgs"][0] == "BTLメッセージ" or codeList[i]["stringArgs"][0] == "X[移]メニュー時文章表示":
- jaString = codeList[i]["stringArgs"][1]
-
- # Pass 1
- if not setData:
- # Remove Textwrap
- jaString = jaString.replace("\n", " ")
- jaString = jaString.replace("\r", "")
-
- # Append
- list300.append(jaString)
-
- # Pass 2
- else:
- # Add Textwrap and Font
- translatedText = dazedwrap.wrapText(list300[0], WIDTH)
- list300.pop(0)
-
- # Write to File
- codeList[i]["stringArgs"][1] = translatedText
-
- # Dialogue
- elif codeList[i]["stringArgs"][0] == "援護文章":
- speakerRegex = r"@\d+\r?\n(.*)：\r?\n" # Default: r"@\d+\r?\n(.*)：\r?\n"
- textRegex = r"@?\d*\r?\n?\u3000*([\w\W]+)\r?\n?" # Default: r"@?\d*\r?\n?\u3000*([\w\W]+)\r?\n?"
-
- # Grab String
- jaString = codeList[i]["stringArgs"][1]
- speaker = ""
-
- # Grab Speaker
- if "：\n" in jaString or "：\r\n" in jaString:
- match = re.search(speakerRegex, jaString)
- if match:
- # TL Speaker
- response = getSpeaker(match.group(1))
- speaker = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Set nametag and remove from string
- codeList[i]["stringArgs"][1] = codeList[i]["stringArgs"][1].replace(match.group(1), speaker)
- jaString = jaString.replace(match.group(0), "")
-
- # Grab Only Text
- match = re.search(textRegex, jaString)
- if match:
- jaString = match.group(1)
- initialJAString = jaString
-
- # Remove Textwrap
- jaString = jaString.replace("\r", "")
- jaString = jaString.replace("\n", " ")
-
- # 1st Pass (Save Text to List)
- if not setData:
- if speaker == "":
- list300.append(jaString)
- else:
- list300.append(f"[{speaker}]: {jaString}")
-
- # 2nd Pass (Set Text)
- else:
- # Grab Translated String
- translatedText = list300[0]
-
- # Remove speaker
- matchSpeakerList = re.findall(r"^(\[.+?\]\s?[|:]\s?)\s?", translatedText)
- if len(matchSpeakerList) > 0:
- translatedText = translatedText.replace(matchSpeakerList[0], "")
-
- # Textwrap
- if FIXTEXTWRAP is True:
- translatedText = dazedwrap.wrapText(translatedText, width=WIDTH)
-
- # Set Data
- codeList[i]["stringArgs"][1] = codeList[i]["stringArgs"][1].replace(initialJAString, translatedText)
-
- # Reset Data and Pop Item
- list300.pop(0)
-
- ### Event Code: 250 DB Read/Writes
- if codeList[i]["code"] == 250 and CODE250 == True:
- # Validate size
- stringArg = 0
- if len(codeList[i]["stringArgs"]) == 4:
- if codeList[i]["stringArgs"][1] == "万能ｳｨﾝﾄﾞｳ一時DB"\
- and codeList[i]["stringArgs"][stringArg] != "":
- # Font Size
- fontSize = 0
-
- # Grab String
- jaString = codeList[i]["stringArgs"][stringArg]
-
- # Remove Textwrap
- # jaString = jaString.replace("\r", "")
- # jaString = jaString.replace("\n", " ")
-
- # Pass 1 (Save Text to List)
- if not setData:
- list250.append(jaString)
-
- # Pass 2 (Set Text)
- else:
- translatedText = list250[0]
- list250.pop(0)
-
- # Textwrap
- # translatedText = dazedwrap.wrapText(translatedText, WIDTH)
-
- # Add/Replace font formatting
- # Remove existing font command if present
- translatedText = re.sub(r'\\f\[\d+\]', '', translatedText)
- if fontSize:
- # Add new font command if fontSize is not 0
- if fontSize != "0" and fontSize != 0:
- translatedText = f"\\f[{fontSize}]{translatedText}"
-
- # Set Data
- codeList[i]["stringArgs"][stringArg] = translatedText
-
- ### Iterate
- i += 1
-
- # EOF
- stringListTL = []
- list210TL = []
- list150TL = []
- list250TL = []
- list300TL = []
- list122TL = []
- setData = False
-
- # String List
- if len(stringList) > 0:
- pbar.total = len(stringList)
- pbar.refresh()
- response = translateAI(stringList, textHistory)
- stringListTL = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
- if len(stringListTL) != len(stringList):
- with LOCK:
- if filename not in MISMATCH:
- MISMATCH.append(filename)
- else:
- setData = True
-
- # 210 List
- if len(list210) > 0:
- pbar.total = len(list210)
- pbar.refresh()
- response = translateAI(list210, textHistory)
- list210TL = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
- if len(list210TL) != len(list210):
- with LOCK:
- if filename not in MISMATCH:
- MISMATCH.append(filename)
- else:
- setData = True
-
- # 250 List
- if len(list250) > 0:
- pbar.total = len(list250)
- pbar.refresh()
- response = translateAI(list250, textHistory)
- list250TL = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
- if len(list250TL) != len(list250):
- with LOCK:
- if filename not in MISMATCH:
- MISMATCH.append(filename)
- else:
- setData = True
-
- # 300 List
- if len(list300) > 0:
- pbar.total = len(list300)
- pbar.refresh()
- response = translateAI(list300, textHistory)
- list300TL = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
- if len(list300TL) != len(list300):
- with LOCK:
- if filename not in MISMATCH:
- MISMATCH.append(filename)
- else:
- setData = True
-
- # 150 List
- if len(list150) > 0:
- # Progress Bar
- pbar.total = len(list150)
- pbar.refresh()
-
- # Translate
- response = translateAI(
- list150,
- textHistory,
- True,
- )
- list150TL = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Check Mismatch
- if len(list150TL) != len(list150):
- with LOCK:
- if filename not in MISMATCH:
- MISMATCH.append(filename)
- else:
- list150 = list150TL
- setData = True
-
- # 122 List
- if len(list122) > 0:
- pbar.total = len(list122)
- pbar.refresh()
- response = translateAI(list122, textHistory)
- list122TL = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
- if len(list122TL) != len(list122):
- with LOCK:
- if filename not in MISMATCH:
- MISMATCH.append(filename)
- else:
- setData = True
-
- # Update jobList before recursive call
- if setData == True:
- stringList = []
- jobList = [stringListTL, list210TL, list300TL, list150TL, list250TL, list122TL] # Add list122TL
- searchCodes(events, pbar, jobList, filename)
-
- else:
- # Set Data
- events = codeList
-
- except IndexError as e:
- traceback.print_exc()
- raise Exception(str(e) + "Failed to translate: " + initialJAString) from None
- except Exception as e:
- traceback.print_exc()
- raise Exception(str(e) + "Failed to translate: " + initialJAString) from None
-
- return totalTokens
-
-
-def formatDramon(jaString):
- imageRegex = r"(\r?\n?_[a-zA-Z_\d/.]+\r?\n?)|(@-?\d?\r\n)|(@-?\d?)([^\r]\d?-?\d?[^\r]+?)(\r\n|$)|(_PDC)|(>\r\n)|(^[#])|(\r\n@$)|(/)|(_SS_)|(\r\n[@/]\s?-?\d?\r\n)"
-
- jaString = jaString.replace("\u3000", " ")
- jaString = jaString.replace("#", "")
- jaString = re.sub(r"([^\r])\n", r"\1\r\n", jaString)
-
- # Grab and Split
- jaStringList = re.split(imageRegex, jaString)
-
- # Clean List
- cleanedList = [x for x in jaStringList if x is not None and x != "" and x != "\r\n" and x != "_SS_"]
-
- # Iterate Through List
- j = 0
- translatedText = ""
- while j < len(cleanedList):
- if (
- ("@" in cleanedList[j] or "/" in cleanedList[j])
- and j < len(cleanedList) - 1
- and re.search(r"([@/]-?\d?\r\n)", cleanedList[j]) is None
- and ".ogg" not in cleanedList[j]
- ):
- # Setup @
- if j > 0 and "@" not in cleanedList[j - 1] and "/" not in cleanedList[j - 1] and "_" not in cleanedList[j - 1]:
- cleanedList[j - 1] = cleanedList[j - 1] + cleanedList[j + 1]
- else:
- cleanedList.insert(j, cleanedList[j + 1])
- j += 1
- cleanedList[j] = f"\r\n{cleanedList[j]}\r\n"
- cleanedList.pop(j + 1)
- j += 1
-
- return cleanedList
-
-def handleScenarioScript(jaString, scriptString=""):
- # Extract Speaker
- # Updated regex to match speaker names with spaces, hyphens, and other characters
- # [^\n：]+ matches any character except newline and ： (the speaker name)
- scriptRegex = r"\n?(//.*?\n|//.*|/b.*?\n|[^\n：]+：\d*\r?\n)"
- match = re.search(scriptRegex, jaString)
- if match:
- if "//" in match.group(1):
- scriptStringLocal = match.group(1)
- jaString = jaString.replace(scriptStringLocal, "")
- scriptString += scriptStringLocal
- match = re.search(scriptRegex, jaString)
- if not match:
- return None
- else:
- if "//" in jaString:
- return handleScenarioScript(jaString, scriptString)
- return [jaString, match.group(1), scriptString]
- else:
- return None
-
-# Database
-def searchDB(events, pbar, jobList, filename):
- # Set Lists
- if len(jobList) > 0:
- scenarioList = jobList[0]
- npcList = jobList[1]
- itemList = jobList[2]
- stateList = jobList[3]
- armorList = jobList[4]
- enemyList = jobList[5]
- weaponsList = jobList[6]
- skillList = jobList[7]
- optionsList = jobList[8]
- dbNameList = jobList[9]
- dbValueList = jobList[10]
- setData = True
- else:
- scenarioList = [[], [], []]
- npcList = [[], [], [], []]
- itemList = [[], [], [], []]
- armorList = [[], []]
- enemyList = [[], []]
- weaponsList = [[], [], [], []]
- skillList = [[], [], [], [], []]
- stateList = [[], [], [], [], [], [], [], []]
- optionsList = [[], [], []]
- dbNameList = [[]]
- dbValueList = [[]]
- setData = False
-
- # Vars/Globals
- totalTokens = [0, 0]
- initialJAString = ""
- tableList = events
- font = ""
- global LOCK
- global NAMESLIST
- global MISMATCH
- global PBAR
- PBAR = pbar
-
- # Begin Parsing File
- try:
- for table in tableList:
- # Grab NPCs
- if table["name"] == "アイドル" and NPCFLAG == True:
- with open("log/translations.txt", "a", encoding="utf-8") as file:
- if setData:
- file.write(f"\n#Actors\n")
- for npc in table["data"]:
- dataList = npc["data"]
-
- # Parse
- for j in range(len(dataList)):
- # Name
- if dataList[j].get("name") == "出身":
- # Pass 1 (Grab Data)
- if setData == False:
- if dataList[j].get("value") != "":
- npcList[0].append(dataList[j].get("value"))
-
- # Pass 2 (Set Data)
- else:
- if dataList[j].get("value") != "":
- # Write Name
- file.write(f"{dataList[j].get('value')} ({npcList[0][0]})\n")
-
- # Set Data
- dataList[j].update({"value": npcList[0][0]})
- npcList[0].pop(0)
-
- # Description
- if dataList[j].get("name") == "説明":
- # Pass 1 (Grab Data)
- if setData == False:
- if dataList[j].get("value") != "":
- # Remove Textwrap
- jaString = dataList[j].get("value")
- jaString = jaString.replace("\n", " ")
- jaString = jaString.replace("\r", "")
- jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString)
-
- # Append Data
- npcList[1].append(jaString)
- # Pass 2 (Set Data)
- else:
- if dataList[j].get("value") != "":
- # Textwrap
- translatedText = npcList[1][0]
- translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH)
- translatedText = font + translatedText
-
- # Set Data
- dataList[j].update({"value": translatedText})
- npcList[1].pop(0)
-
- # Grab Scenario
- if "デートキャラ" in table["name"] and SCENARIOFLAG == True:
- for scenario in table["data"]:
- dataList = scenario["data"]
-
- # Parse
- for j in range(len(dataList)):
- # Name
- if dataList[j].get("name") == "NULL":
- if dataList[j].get("value"):
- jaStringList = re.split(r'\r\n\r\n|\n\n', dataList[j].get("value"))
- for jaString in jaStringList:
- speakerNum = None
- ogString = jaString
- # Extract Speaker
- speakerList = handleScenarioScript(jaString)
-
- # Handle simple names without speaker pattern
- if not speakerList:
- # Pass 1 (Grab Data) - Simple name
- if setData == False:
- jaString = jaString.replace("\n", " ")
- jaString = jaString.replace("\r", "")
- if jaString.strip():
- scenarioList[0].append(jaString)
- # Pass 2 (Set Data) - Simple name
- else:
- if ogString.strip():
- translatedText = scenarioList[0][0]
- scenarioList[0].pop(0)
- dataList[j].update({"value": dataList[j].get("value").replace(ogString, translatedText)})
- continue
-
- if speakerList[2]:
- jaString = speakerList[0]
- speakerNumMatch = re.search(r"：(\d+)", speakerList[1])
- if speakerNumMatch:
- speakerNum = speakerNumMatch.group(1)
- speaker = re.sub(r"：\d*", "", speakerList[1])
- speaker = re.sub(r"/b", "NPC", speaker)
- speaker = speaker.replace("\r\n", "")
-
- # Add Speaker
- jaString = jaString.replace(speakerList[1], f"[{speaker}]: ")
-
- # Pass 1 (Grab Data)
- if setData == False:
- jaString = jaString.replace("\n", " ")
- jaString = jaString.replace("\r", "")
- scenarioList[0].append(jaString)
-
- # Pass 2 (Set Data)
- else:
- translatedText = scenarioList[0][0]
- scenarioList[0].pop(0)
-
- # Remove Speaker before wrapping so WIDTH applies to dialogue only
- speakerMatch = re.search(r"\[(.+?)\]\s?[|:]\s?", translatedText)
- if speakerMatch:
- speaker = speakerMatch.group(1).strip()
- match = re.search(r'(^\[.+?\]\s?[|:]\s?)', translatedText)
- if match:
- translatedText = translatedText.replace(match.group(1), "")
-
- # Wrap text after removing speaker
- translatedText = dazedwrap.wrapText(translatedText, WIDTH)
-
- # Redo Old Speaker Format
- speaker = speaker.replace("Speaker", "\b")
- if speakerNum:
- translatedText = f"{speaker}：{speakerNum}\r\n{translatedText}"
- elif speaker == "NPC":
- translatedText = f"/b\r\n{translatedText}"
- else:
- translatedText = f"{speaker}：\r\n{translatedText}"
-
- # Add Script String
- if speakerList[2]:
- translatedText = f"{speakerList[2]}{translatedText}"
-
- # Set Data
- dataList[j].update({"value": dataList[j].get("value").replace(ogString, translatedText)})
- # Description
- if dataList[j].get("name") == "不正解会話":
- if dataList[j].get("value"):
- jaStringList = re.split(r'\r\n\r\n|\n\n', dataList[j].get("value"))
- for jaString in jaStringList:
- speakerNum = None
- ogString = jaString
- # Extract Speaker
- speakerList = handleScenarioScript(jaString)
-
- # Handle simple text without speaker pattern
- if not speakerList:
- # Pass 1 (Grab Data) - Simple text
- if setData == False:
- jaString = jaString.replace("\n", " ")
- jaString = jaString.replace("\r", "")
- if jaString.strip():
- scenarioList[1].append(jaString)
- # Pass 2 (Set Data) - Simple text
- else:
- if ogString.strip():
- translatedText = scenarioList[1][0]
- scenarioList[1].pop(0)
- translatedText = dazedwrap.wrapText(translatedText, WIDTH)
- dataList[j].update({"value": dataList[j].get("value").replace(ogString, translatedText)})
- continue
-
- if speakerList[2]:
- jaString = speakerList[0]
- speakerNumMatch = re.search(r"：(\d+)", speakerList[1])
- if speakerNumMatch:
- speakerNum = speakerNumMatch.group(1)
- speaker = re.sub(r"：\d*", "", speakerList[1])
- speaker = re.sub(r"/b", "NPC", speaker)
- speaker = speaker.replace("\r\n", "")
-
- # Add Speaker
- jaString = jaString.replace(speakerList[1], f"[{speaker}]: ")
-
- # Pass 1 (Grab Data)
- if setData == False:
- jaString = jaString.replace("\n", " ")
- jaString = jaString.replace("\r", "")
- scenarioList[1].append(jaString)
-
- # Pass 2 (Set Data)
- else:
- translatedText = scenarioList[1][0]
- scenarioList[1].pop(0)
-
- # Remove Speaker before wrapping so WIDTH applies to dialogue only
- speakerMatch = re.search(r"\[(.+?)\]\s?[|:]\s?", translatedText)
- if speakerMatch:
- speaker = speakerMatch.group(1).strip()
- match = re.search(r'(^\[.+?\]\s?[|:]\s?)', translatedText)
- if match:
- translatedText = translatedText.replace(match.group(1), "")
-
- # Wrap text after removing speaker
- translatedText = dazedwrap.wrapText(translatedText, WIDTH)
-
- # Redo Old Speaker Format
- speaker = speaker.replace("Speaker", "\b")
- if speakerNum:
- translatedText = f"{speaker}：{speakerNum}\r\n{translatedText}"
- elif speaker == "NPC":
- translatedText = f"/b\r\n{translatedText}"
- else:
- translatedText = f"{speaker}：\r\n{translatedText}"
-
- # Add Script String
- if speakerList[2]:
- translatedText = f"{speakerList[2]}{translatedText}"
-
- # Set Data
- dataList[j].update({"value": dataList[j].get("value").replace(ogString, translatedText)})
-
- # Description
- if dataList[j].get("name") == "正解会話":
- if dataList[j].get("value"):
- jaStringList = re.split(r'\r\n\r\n|\n\n', dataList[j].get("value"))
- for jaString in jaStringList:
- speakerNum = None
- ogString = jaString
- # Extract Speaker
- speakerList = handleScenarioScript(jaString)
-
- # Handle simple text without speaker pattern
- if not speakerList:
- # Pass 1 (Grab Data) - Simple text
- if setData == False:
- jaString = jaString.replace("\n", " ")
- jaString = jaString.replace("\r", "")
- if jaString.strip():
- scenarioList[2].append(jaString)
- # Pass 2 (Set Data) - Simple text
- else:
- if ogString.strip():
- translatedText = scenarioList[2][0]
- scenarioList[2].pop(0)
- translatedText = dazedwrap.wrapText(translatedText, WIDTH)
- dataList[j].update({"value": dataList[j].get("value").replace(ogString, translatedText)})
- continue
- if speakerList[2]:
- jaString = speakerList[0]
- speakerNumMatch = re.search(r"：(\d+)", speakerList[1])
- if speakerNumMatch:
- speakerNum = speakerNumMatch.group(1)
- speaker = re.sub(r"：\d*", "", speakerList[1])
- speaker = re.sub(r"/b", "NPC", speaker)
- speaker = speaker.replace("\r\n", "")
-
- # Add Speaker
- jaString = jaString.replace(speakerList[1], f"[{speaker}]: ")
-
- # Pass 1 (Grab Data)
- if setData == False:
- jaString = jaString.replace("\n", " ")
- jaString = jaString.replace("\r", "")
- scenarioList[2].append(jaString)
-
- # Pass 2 (Set Data)
- else:
- translatedText = scenarioList[2][0]
- scenarioList[2].pop(0)
-
- # Remove Speaker before wrapping so WIDTH applies to dialogue only
- speakerMatch = re.search(r"\[(.+?)\]\s?[|:]\s?", translatedText)
- if speakerMatch:
- speaker = speakerMatch.group(1).strip()
- match = re.search(r'(^\[.+?\]\s?[|:]\s?)', translatedText)
- if match:
- translatedText = translatedText.replace(match.group(1), "")
-
- # Wrap text after removing speaker
- translatedText = dazedwrap.wrapText(translatedText, WIDTH)
-
- # Redo Old Speaker Format
- speaker = speaker.replace("Speaker", "\b")
- if speakerNum:
- translatedText = f"{speaker}：{speakerNum}\r\n{translatedText}"
- elif speaker == "NPC":
- translatedText = f"/b\r\n{translatedText}"
- else:
- translatedText = f"{speaker}：\r\n{translatedText}"
-
- # Add Script String
- if speakerList[2]:
- translatedText = f"{speakerList[2]}{translatedText}"
-
- # Set Data
- dataList[j].update({"value": dataList[j].get("value").replace(ogString, translatedText)})
-
- # Grab Options
- if table["name"] == "クイズ" and OPTIONSFLAG == True:
- for option in table["data"]:
- dataList = option["data"]
-
- # Parse
- for j in range(len(dataList)):
- # Name
- if dataList[j].get("name") == "問":
- if dataList[j].get("value"):
- jaString = dataList[j].get("value")
- # Pass 1 (Grab Data)
- if setData == False:
- optionsList[0].append(jaString)
-
- # Pass 2 (Set Data)
- else:
- translatedText = optionsList[0][0]
- optionsList[0].pop(0)
- dataList[j].update({"value": dataList[j].get("value").replace(jaString, translatedText)})
-
- # Description 1
- if dataList[j].get("name") == "3":
- if dataList[j].get("value"):
- jaString = dataList[j].get("value")
- # Pass 1 (Grab Data)
- if setData == False:
- jaString = jaString.replace("\n", " ")
- jaString = jaString.replace("\r", "")
- optionsList[1].append(jaString)
-
- # Pass 2 (Set Data)
- else:
- translatedText = optionsList[1][0]
- optionsList[1].pop(0)
- translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH)
- dataList[j].update({"value": dataList[j].get("value").replace(jaString, translatedText)})
-
- # Description 2
- if dataList[j].get("name") == "4":
- if dataList[j].get("value"):
- jaString = dataList[j].get("value")
- # Pass 1 (Grab Data)
- if setData == False:
- jaString = jaString.replace("\n", " ")
- jaString = jaString.replace("\r", "")
- optionsList[2].append(jaString)
-
- # Pass 2 (Set Data)
- else:
- translatedText = optionsList[2][0]
- optionsList[2].pop(0)
- translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH)
- dataList[j].update({"value": dataList[j].get("value").replace(jaString, translatedText)})
-
- # Grab DB Names
- if table["name"] == "マップ設定" and DBNAMEFLAG == True:
- font = None
- dbName = table["data"]
- for j in range(len(dbName)):
- # Pass 1 (Grab Data)
- if setData == False:
- if dbName[j].get("name") != "":
- dbNameList[0].append(dbName[j].get("name"))
-
- # Pass 2 (Set Data)
- else:
- if dbName[j].get("name") != "":
- dbName[j].update({"name": dbNameList[0][0]})
- dbNameList[0].pop(0)
-
- # Grab DB Values
- if table["name"] == "近況文章" and DBVALUEFLAG == True:
- font = 16
- subject = ""
- for entry in table["data"]:
- dbValue = entry["data"]
- for j in range(len(dbValue)):
- if dbValue[j].get("value"):
- jaString = dbValue[j].get("value")
-
- # Speaker
- match = re.search(r"(^[\\]+f\[\d+\].+?\r\n).+", jaString)
- if match:
- subject = match.group(1)
- jaString = jaString.replace(subject, "")
-
- # Nuke Textwrap & Font
- jaString = jaString.replace("\r", "")
- jaString = jaString.replace("\n", " ")
- jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString)
-
- # Pass 1 (Grab Data)
- if setData == False:
- # Append
- dbValueList[0].append(jaString)
-
- # Pass 2 (Set Data)
- else:
- if dbValue[j].get("value"):
- translatedText = dbValueList[0][0]
-
- # Textwrap
- translatedText = dazedwrap.wrapText(translatedText, 30)
- translatedText = translatedText.replace("\n", f"\r\n\\f[{font}]")
-
- # Subject
- if subject:
- translatedText = f"{subject}{translatedText}"
-
- # Set
- dbValue[j].update({"value": translatedText})
- dbValueList[0].pop(0)
-
- # Grab Items
- if table["name"] == "道具" and ITEMFLAG == True:
- # Write Category
- if setData:
- with open("log/translations.txt", "a", encoding="utf-8") as file:
- file.write(f"\n#Items\n")
-
- # Begin Translation
- for item in table["data"]:
- dataList = item["data"]
-
- # Parse
- font = 18
- for j in range(len(dataList)):
- # Name
- if dataList[j].get("name") == "名前":
- jaString = dataList[j].get("value")
- if jaString != "":
- # Pass 1 (Grab Data)
- if setData == False:
- if jaString != "":
- itemList[0].append(jaString)
-
- # Pass 2 (Set Data)
- else:
- # Write to TL File
- with open("log/translations.txt", "a", encoding="utf-8") as file:
- file.write(f"{jaString} ({itemList[0][0]})\n")
-
- dataList[j].update({"value": itemList[0][0]})
- itemList[0].pop(0)
-
- # Description
- if dataList[j].get("name") == "説明":
- # Pass 1 (Grab Data)
- if setData == False:
- if dataList[j].get("value") != "":
- # Remove Textwrap & Font
- jaString = dataList[j].get("value")
- jaString = jaString.replace("\n", " ")
- jaString = jaString.replace("\r", "")
- jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString)
-
- # Append Data
- itemList[1].append(jaString)
- # Pass 2 (Set Data)
- else:
- if dataList[j].get("value") != "":
- # Textwrap
- translatedText = itemList[1][0]
- translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH)
-
- # Font
- if font:
- translatedText = f"\\f[{font}]{translatedText}"
-
- # Set Data
- dataList[j].update({"value": translatedText})
- itemList[1].pop(0)
-
- # Log
- if dataList[j].get("name") == "使用時文章[戦](人名~":
- # Pass 1 (Grab Data)
- if setData == False:
- if dataList[j].get("value") != "":
- # Remove Textwrap
- jaString = dataList[j].get("value")
- jaString = jaString.replace("\n", " ")
- jaString = jaString.replace("\r", "")
- jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString)
-
- # Skill Action
- if jaString[0] in [
- "は",
- "を",
- "の",
- "に",
- "が",
- ]:
- jaString = f"Taro{jaString}"
-
- # Append Data
- itemList[2].append(jaString)
- # Pass 2 (Set Data)
- else:
- if dataList[j].get("value") != "":
- # Textwrap
- translatedText = itemList[2][0]
- translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH)
- # translatedText = font + translatedText
-
- # Remove Taro
- translatedText = re.sub(r"\bTaro\b", "", translatedText)
-
- # Set Data
- dataList[j].update({"value": translatedText})
- itemList[2].pop(0)
-
- # Description
- if dataList[j].get("name") == "使用後文章[移動]":
- # Pass 1 (Grab Data)
- if setData == False:
- if dataList[j].get("value") != "":
- # Remove Textwrap
- jaString = dataList[j].get("value")
- jaString = jaString.replace("\n", " ")
- jaString = jaString.replace("\r", "")
- jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString)
-
- # Append Data
- itemList[3].append(jaString)
- # Pass 2 (Set Data)
- else:
- if dataList[j].get("value") != "":
- # Textwrap
- translatedText = itemList[3][0]
- translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH)
-
- # Font
- if font:
- translatedText = f"\\f[{font}]{translatedText}"
-
- # Set Data
- dataList[j].update({"value": translatedText})
- itemList[3].pop(0)
-
- # Grab Armors
- if table["name"] == "防具" and ARMORFLAG == True:
- font = "24"
- for armor in table["data"]:
- dataList = armor["data"]
-
- # Parse
- for j in range(len(dataList)):
- # Name
- if dataList[j].get("name") == "名前":
- # Pass 1 (Grab Data)
- if setData == False:
- if dataList[j].get("value") != "":
- armorList[0].append(dataList[j].get("value"))
-
- # Pass 2 (Set Data)
- else:
- if dataList[j].get("value") != "":
- dataList[j].update({"value": armorList[0][0]})
- armorList[0].pop(0)
-
- # Description
- if dataList[j].get("name") == "説明":
- # Pass 1 (Grab Data)
- if setData == False:
- if dataList[j].get("value") != "":
- # Remove Textwrap
- jaString = dataList[j].get("value")
- jaString = jaString.replace("\n", " ")
- jaString = jaString.replace("\r", "")
- jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString)
-
- # Append Data
- armorList[1].append(jaString)
- # Pass 2 (Set Data)
- else:
- if dataList[j].get("value") != "":
- # Textwrap
- translatedText = armorList[1][0]
- translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH)
-
- # Font
- if font:
- translatedText = f"\\f[{font}]{translatedText}"
-
- # Set Data
- dataList[j].update({"value": translatedText})
- armorList[1].pop(0)
-
- # Grab Enemies
- if table["name"] == "敵" and ENEMYFLAG == True:
- for enemy in table["data"]:
- dataList = enemy["data"]
-
- # Parse
- for j in range(len(dataList)):
- # Name
- if dataList[j].get("name") == "敵キャラ名":
- # Pass 1 (Grab Data)
- if setData == False:
- if dataList[j].get("value") != "":
- enemyList[0].append(dataList[j].get("value"))
-
- # Pass 2 (Set Data)
- else:
- if dataList[j].get("value") != "":
- dataList[j].update({"value": enemyList[0][0]})
- enemyList[0].pop(0)
-
- # Description
- if dataList[j].get("name") == "NULL":
- # Pass 1 (Grab Data)
- if setData == False:
- if dataList[j].get("value") != "":
- # Remove Textwrap
- jaString = dataList[j].get("value")
- jaString = jaString.replace("\n", " ")
- jaString = jaString.replace("\r", "")
- jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString)
-
- # Append Data
- enemyList[1].append(jaString)
- # Pass 2 (Set Data)
- else:
- if dataList[j].get("value") != "":
- # Textwrap
- translatedText = enemyList[1][0]
- translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH)
-
- # Font
- if font:
- translatedText = f"\\f[{font}]{translatedText}"
-
- # Set Data
- dataList[j].update({"value": translatedText})
- enemyList[1].pop(0)
-
- # Grab Weapons
- if table["name"] == "武器" and WEAPONFLAG == True:
- font = "24"
- for weapon in table["data"]:
- dataList = weapon["data"]
-
- # Parse
- for j in range(len(dataList)):
- # Name
- if dataList[j].get("name") == "名前":
- # Pass 1 (Grab Data)
- if setData == False:
- if dataList[j].get("value") != "":
- weaponsList[0].append(dataList[j].get("value"))
-
- # Pass 2 (Set Data)
- else:
- if dataList[j].get("value") != "":
- dataList[j].update({"value": weaponsList[0][0]})
- weaponsList[0].pop(0)
-
- # Description
- if dataList[j].get("name") == "説明":
- # Pass 1 (Grab Data)
- if setData == False:
- if dataList[j].get("value") != "":
- # Remove Textwrap
- jaString = dataList[j].get("value")
- jaString = jaString.replace("\n", " ")
- jaString = jaString.replace("\r", "")
- jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString)
-
- # Append Data
- weaponsList[1].append(jaString)
- # Pass 2 (Set Data)
- else:
- if dataList[j].get("value") != "":
- # Textwrap
- translatedText = weaponsList[1][0]
- translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH)
-
- # Font
- if font:
- translatedText = f"\\f[{font}]{translatedText}"
-
- # Set Data
- dataList[j].update({"value": translatedText})
- weaponsList[1].pop(0)
-
- # Grab Skills
- if table["name"] == "技能" and SKILLFLAG == True:
- font = "24"
- for skill in table["data"]:
- dataList = skill["data"]
-
- # Parse
- for j in range(len(dataList)):
- # Name
- if dataList[j].get("name") == "名前":
- # Pass 1 (Grab Data)
- if setData == False:
- if dataList[j].get("value") != "":
- skillList[0].append(dataList[j].get("value"))
-
- # Pass 2 (Set Data)
- else:
- if dataList[j].get("value") != "":
- dataList[j].update({"value": skillList[0][0]})
- skillList[0].pop(0)
-
- # Description
- if dataList[j].get("name") == "説明":
- # Pass 1 (Grab Data)
- if setData == False:
- if dataList[j].get("value") != "":
- # Remove Textwrap
- jaString = dataList[j].get("value")
- jaString = jaString.replace("\n", " ")
- jaString = jaString.replace("\r", "")
- jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString)
-
- # Append Data
- skillList[1].append(jaString)
-
- # Pass 2 (Set Data)
- else:
- if dataList[j].get("value") != "":
- # Textwrap
- translatedText = skillList[1][0]
- translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH)
-
- # Font
- if font:
- translatedText = f"\\f[{font}]{translatedText}"
-
- # Set Data
- dataList[j].update({"value": translatedText})
- skillList[1].pop(0)
-
- # Log
- if dataList[j].get("name") == "発動時文章":
- # Pass 1 (Grab Data)
- if setData == False:
- if dataList[j].get("value") != "":
- # Remove Textwrap
- jaString = dataList[j].get("value")
- jaString = jaString.replace("\n", " ")
- jaString = jaString.replace("\r", "")
- jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString)
-
- # Skill Action
- if jaString[0] in [
- "は",
- "を",
- "の",
- "に",
- "が",
- ]:
- jaString = f"Taro{jaString}"
-
- # Append Data
- skillList[2].append(jaString)
- # Pass 2 (Set Data)
- else:
- if dataList[j].get("value") != "":
- # Textwrap
- translatedText = skillList[2][0]
- translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH)
- translatedText = font + translatedText
-
- # Remove Taro
- translatedText = re.sub(r"\bTaro\b", "", translatedText)
-
- # Set Data
- dataList[j].update({"value": translatedText})
- skillList[2].pop(0)
-
- # Log
- if dataList[j].get("name") == "使用時文章[戦闘](人名~":
- # Pass 1 (Grab Data)
- if setData == False:
- if dataList[j].get("value") != "":
- # Remove Textwrap
- jaString = dataList[j].get("value")
- jaString = jaString.replace("\n", " ")
- jaString = jaString.replace("\r", "")
- jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString)
-
- # Skill Action
- if jaString[0] in [
- "は",
- "を",
- "の",
- "に",
- "が",
- ]:
- jaString = f"Taro{jaString}"
-
- # Append Data
- skillList[3].append(jaString)
- # Pass 2 (Set Data)
- else:
- if dataList[j].get("value") != "":
- # Textwrap
- translatedText = skillList[3][0]
- translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH)
-
- # Remove Taro
- translatedText = re.sub(r"\bTaro\b", "", translatedText)
-
- # Font
- translatedText = font + translatedText
-
- # Set Data
- dataList[j].update({"value": translatedText})
- skillList[3].pop(0)
-
- # Log
- if dataList[j].get("name") == "失敗時文章[(対象)～]":
- # Pass 1 (Grab Data)
- if setData == False:
- if dataList[j].get("value") != "":
- # Remove Textwrap
- jaString = dataList[j].get("value")
- jaString = jaString.replace("\n", " ")
- jaString = jaString.replace("\r", "")
- jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString)
-
- # Skill Action
- if jaString[0] in [
- "は",
- "を",
- "の",
- "に",
- "が",
- ]:
- jaString = f"Taro{jaString}"
-
- # Append Data
- skillList[4].append(jaString)
- # Pass 2 (Set Data)
- else:
- if dataList[j].get("value") != "":
- # Textwrap
- translatedText = skillList[4][0]
- translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH)
- translatedText = font + translatedText
-
- # Remove Taro
- translatedText = re.sub(r"\bTaro\b", "", translatedText)
-
- # Set Data
- dataList[j].update({"value": translatedText})
- skillList[4].pop(0)
-
- # Grab States
- if table["name"] == "ステート" and STATEFLAG == True:
- for state in table["data"]:
- dataList = state["data"]
-
- # Parse
- for j in range(len(dataList)):
- # Name
- if dataList[j].get("name") == "名前":
- # Pass 1 (Grab Data)
- if setData == False:
- if dataList[j].get("value") != "":
- stateList[0].append(dataList[j].get("value"))
-
- # Pass 2 (Set Data)
- else:
- if dataList[j].get("value") != "":
- dataList[j].update({"value": stateList[0][0]})
- stateList[0].pop(0)
-
- # Description
- if dataList[j].get("name") == "表示名":
- # Pass 1 (Grab Data)
- if setData == False:
- if dataList[j].get("value") != "":
- # Remove Textwrap
- jaString = dataList[j].get("value")
- jaString = jaString.replace("\n", " ")
- jaString = jaString.replace("\r", "")
- jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString)
-
- # Append Data
- stateList[1].append(jaString)
- # Pass 2 (Set Data)
- else:
- if dataList[j].get("value") != "":
- # Textwrap
- translatedText = stateList[1][0]
- translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH)
- translatedText = font + translatedText
-
- # Set Data
- dataList[j].update({"value": translatedText})
- stateList[1].pop(0)
-
- # Log
- if dataList[j].get("name") == "付与時文章":
- # Pass 1 (Grab Data)
- if setData == False:
- if dataList[j].get("value") != "":
- # Remove Textwrap
- jaString = dataList[j].get("value")
- jaString = jaString.replace("\n", " ")
- jaString = jaString.replace("\r", "")
- jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString)
-
- # State Action
- if jaString[0] in [
- "は",
- "を",
- "の",
- "に",
- "が",
- ]:
- jaString = f"Taro{jaString}"
-
- # Append Data
- stateList[2].append(jaString)
- # Pass 2 (Set Data)
- else:
- if dataList[j].get("value") != "":
- # Textwrap
- translatedText = stateList[2][0]
- translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH)
- translatedText = font + translatedText
-
- # Remove Taro
- translatedText = re.sub(r"\bTaro\b", "", translatedText)
-
- # Set Data
- dataList[j].update({"value": translatedText})
- stateList[2].pop(0)
-
- # Log
- if dataList[j].get("name") == "解除時文章":
- # Pass 1 (Grab Data)
- if setData == False:
- if dataList[j].get("value") != "":
- # Remove Textwrap
- jaString = dataList[j].get("value")
- jaString = jaString.replace("\n", " ")
- jaString = jaString.replace("\r", "")
- jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString)
-
- # State Action
- if jaString[0] in [
- "は",
- "を",
- "の",
- "に",
- "が",
- ]:
- jaString = f"Taro{jaString}"
-
- # Append Data
- stateList[3].append(jaString)
- # Pass 2 (Set Data)
- else:
- if dataList[j].get("value") != "":
- # Textwrap
- translatedText = stateList[3][0]
- translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH)
- translatedText = font + translatedText
-
- # Remove Taro
- translatedText = re.sub(r"\bTaro\b", "", translatedText)
-
- # Set Data
- dataList[j].update({"value": translatedText})
- stateList[3].pop(0)
-
- # Log
- if dataList[j].get("name") == "回復時の文章[(人名)～]":
- # Pass 1 (Grab Data)
- if setData == False:
- if dataList[j].get("value") != "":
- # Remove Textwrap
- jaString = dataList[j].get("value")
- jaString = jaString.replace("\n", " ")
- jaString = jaString.replace("\r", "")
- jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString)
-
- # State Action
- if jaString[0] in [
- "は",
- "を",
- "の",
- "に",
- "が",
- ]:
- jaString = f"Taro{jaString}"
-
- # Append Data
- stateList[4].append(jaString)
- # Pass 2 (Set Data)
- else:
- if dataList[j].get("value") != "":
- # Textwrap
- translatedText = stateList[4][0]
- translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH)
- translatedText = font + translatedText
-
- # Remove Taro
- translatedText = re.sub(r"\bTaro\b", "", translatedText)
-
- # Set Data
- dataList[j].update({"value": translatedText})
- stateList[4].pop(0)
- # Log
- if dataList[j].get("name") == "┣ ｶｳﾝﾀｰ発動文[対象～":
- # Pass 1 (Grab Data)
- if setData == False:
- if dataList[j].get("value") != "":
- # Remove Textwrap
- jaString = dataList[j].get("value")
- jaString = jaString.replace("\n", " ")
- jaString = jaString.replace("\r", "")
- jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString)
-
- # State Action
- if jaString[0] in [
- "は",
- "を",
- "の",
- "に",
- "が",
- ]:
- jaString = f"Taro{jaString}"
-
- # Append Data
- stateList[5].append(jaString)
- # Pass 2 (Set Data)
- else:
- if dataList[j].get("value") != "":
- # Textwrap
- translatedText = stateList[5][0]
- translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH)
- translatedText = font + translatedText
-
- # Remove Taro
- translatedText = re.sub(r"\bTaro\b", "", translatedText)
-
- # Set Data
- dataList[j].update({"value": translatedText})
- stateList[5].pop(0)
-
- # Log
- if dataList[j].get("name") == "尻もち 行動不能 持続3ターン":
- # Pass 1 (Grab Data)
- if setData == False:
- if dataList[j].get("value") != "":
- # Remove Textwrap
- jaString = dataList[j].get("value")
- jaString = jaString.replace("\n", " ")
- jaString = jaString.replace("\r", "")
- jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString)
-
- # State Action
- if jaString[0] in [
- "は",
- "を",
- "の",
- "に",
- "が",
- ]:
- jaString = f"Taro{jaString}"
-
- # Append Data
- stateList[6].append(jaString)
- # Pass 2 (Set Data)
- else:
- if dataList[j].get("value") != "":
- # Textwrap
- translatedText = stateList[6][0]
- translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH)
- translatedText = font + translatedText
-
- # Remove Taro
- translatedText = re.sub(r"\bTaro\b", "", translatedText)
-
- # Set Data
- dataList[j].update({"value": translatedText})
- stateList[6].pop(0)
-
- # Log
- if dataList[j].get("name") == "状態異常の説明":
- # Pass 1 (Grab Data)
- if setData == False:
- if dataList[j].get("value") != "":
- # Remove Textwrap
- jaString = dataList[j].get("value")
- jaString = jaString.replace("\n", " ")
- jaString = jaString.replace("\r", "")
- jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString)
-
- # State Action
- if jaString[0] in [
- "は",
- "を",
- "の",
- "に",
- "が",
- ]:
- jaString = f"Taro{jaString}"
-
- # Append Data
- stateList[7].append(jaString)
- # Pass 2 (Set Data)
- else:
- if dataList[j].get("value") != "":
- # Textwrap
- translatedText = stateList[7][0]
- translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH)
- translatedText = font + translatedText
-
- # Remove Taro
- translatedText = re.sub(r"\bTaro\b", "", translatedText)
-
- # Set Data
- dataList[j].update({"value": translatedText})
- stateList[7].pop(0)
-
- # Translation
- scenarioListTL = [[], [], []]
- optionsListTL = [[], [], []]
- npcListTL = [[], [], [], []]
- itemListTL = [[], [], [], []]
- stateListTL = [[], [], [], [], [], [], [], []]
- armorListTL = [[], []]
- enemyListTL = [[], []]
- weaponsListTL = [[], [], []]
- skillListTL = [[], [], [], [], []]
- dbNameListTL = [[]]
- dbValueListTL = [[]]
-
- translate = False
-
- # NPCs
- if len(npcList[0]) > 0:
- # Progress Bar
- total = 0
- for itemArray in npcList:
- total += len(itemArray)
- pbar.total = total
- pbar.refresh()
-
- # Name
- response = translateAI(
- npcList[0],
- "Reply with only the " + LANGUAGE + " translation of the RPG enemy name",
- True,
- )
- nameListTL = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
- # Desc 1
- response = translateAI(npcList[1], "Reply with only the " + LANGUAGE + " translation")
- descListTL1 = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
- # Desc 2
- response = translateAI(npcList[2], "Reply with only the " + LANGUAGE + " translation")
- descListTL2 = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
- # Desc 3
- response = translateAI(npcList[3], "Reply with only the " + LANGUAGE + " translation")
- descListTL3 = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Check Mismatch
- if (
- len(nameListTL) != len(npcList[0])
- or len(descListTL1) != len(npcList[1])
- or len(descListTL2) != len(npcList[2])
- or len(descListTL3) != len(npcList[3])
- ):
- with LOCK:
- if filename not in MISMATCH:
- MISMATCH.append(filename)
- else:
- npcListTL = [nameListTL, descListTL1, descListTL2, descListTL3]
- translate = True
-
- # SCENARIO
- if scenarioList[0] or scenarioList[1]:
- # Progress Bar
- total = 0
- for scenarioArray in scenarioList:
- total += len(scenarioArray)
- pbar.total = total
- pbar.refresh()
-
- # Name
- response = translateAI(
- scenarioList[0],
- "Reply with only the " + LANGUAGE + " translation",
- True,
- )
- nameListTL = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
- # Desc 1
- response = translateAI(
- scenarioList[1],
- "reply with only the gender neutral " + LANGUAGE + " translation of the NPC name",
- True,
- )
- descListTL1 = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
- # Desc 2
- response = translateAI(
- scenarioList[2],
- "reply with only the gender neutral " + LANGUAGE + " translation of the NPC name",
- True,
- )
- descListTL2 = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Check Mismatch
- if len(nameListTL) != len(scenarioList[0]) or len(descListTL1) != len(scenarioList[1]) or len(descListTL2) != len(scenarioList[2]):
- with LOCK:
- if filename not in MISMATCH:
- MISMATCH.append(filename)
- else:
- scenarioListTL = [nameListTL, descListTL1, descListTL2]
- translate = True
-
- # OPTIONS
- if optionsList[0] or optionsList[1]:
- # Progress Bar
- total = 0
- for optionsArray in optionsList:
- total += len(optionsArray)
- pbar.total = total
- pbar.refresh()
-
- # Name
- response = translateAI(
- optionsList[0],
- "Reply with only the " + LANGUAGE + " translation",
- True,
- )
- nameListTL = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
- # Desc 1
- response = translateAI(
- optionsList[1],
- "reply with only the gender neutral " + LANGUAGE + " translation of the NPC name",
- True,
- )
- descListTL1 = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
- # Desc 2
- response = translateAI(
- optionsList[2],
- "reply with only the gender neutral " + LANGUAGE + " translation of the NPC name",
- True,
- )
- descListTL2 = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Check Mismatch
- if len(nameListTL) != len(optionsList[0]) or len(descListTL1) != len(optionsList[1]) or len(descListTL2) != len(optionsList[2]):
- with LOCK:
- if filename not in MISMATCH:
- MISMATCH.append(filename)
- else:
- optionsListTL = [nameListTL, descListTL1, descListTL2]
- translate = True
-
- # ITEMS
- if len(itemList[0]) > 0 or len(itemList[1]) > 0:
- # Progress Bar
- total = 0
- for itemArray in itemList:
- total += len(itemArray)
- pbar.total = total
- pbar.refresh()
-
- # Name
- response = translateAI(itemList[0], "Reply with only the " + LANGUAGE + " translation")
- nameListTL = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
- # Desc 1
- response = translateAI(itemList[1], "Reply with only the " + LANGUAGE + " translation")
- descListTL1 = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
- # Desc 2
- response = translateAI(itemList[2], "Reply with only the " + LANGUAGE + " translation")
- descListTL2 = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
- # Desc 3
- response = translateAI(itemList[3], "Reply with only the " + LANGUAGE + " translation")
- descListTL3 = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Check Mismatch
- if (
- len(nameListTL) != len(itemList[0])
- or len(descListTL1) != len(itemList[1])
- or len(descListTL2) != len(itemList[2])
- or len(descListTL3) != len(itemList[3])
- ):
- with LOCK:
- if filename not in MISMATCH:
- MISMATCH.append(filename)
- else:
- itemListTL = [nameListTL, descListTL1, descListTL2, descListTL3]
- translate = True
-
- # Armor
- if len(armorList[0]) > 0:
- # Progress Bar
- total = 0
- for armorArray in armorList:
- total += len(armorArray)
- pbar.total = total
- pbar.refresh()
-
- # Name
- response = translateAI(
- armorList[0],
- "Reply with only the " + LANGUAGE + " translation of the NPC name",
- True,
- )
- nameListTL = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
- # Desc 1
- response = translateAI(armorList[1], "Reply with only the " + LANGUAGE + " translation")
- descListTL1 = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Check Mismatch
- if len(nameListTL) != len(armorList[0]) or len(descListTL1) != len(armorList[1]):
- with LOCK:
- if filename not in MISMATCH:
- MISMATCH.append(filename)
- else:
- armorListTL = [nameListTL, descListTL1]
- translate = True
-
- # Enemies
- if len(enemyList[0]) > 0:
- # Progress Bar
- total = 0
- for enemyArray in enemyList:
- total += len(enemyArray)
- pbar.total = total
- pbar.refresh()
-
- # Name
- response = translateAI(
- enemyList[0],
- "Reply with only the " + LANGUAGE + " translation of the enemy NPC name",
- True,
- )
- nameListTL = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
- # Desc 1
- response = translateAI(enemyList[1], "Reply with only the " + LANGUAGE + " translation")
- descListTL1 = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Check Mismatch
- if len(nameListTL) != len(enemyList[0]) or len(descListTL1) != len(enemyList[1]):
- with LOCK:
- if filename not in MISMATCH:
- MISMATCH.append(filename)
- else:
- enemyListTL = [nameListTL, descListTL1]
- translate = True
-
- # Weapons
- if len(weaponsList[0]) > 0:
- # Progress Bar
- total = 0
- for weaponsArray in weaponsList:
- total += len(weaponsArray)
- pbar.total = total
- pbar.refresh()
-
- # Name
- response = translateAI(
- weaponsList[0],
- "Reply with only the " + LANGUAGE + " translation of the RPG weapon name",
- True,
- )
- nameListTL = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
- # Desc 1
- response = translateAI(weaponsList[1], "")
- descListTL1 = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
- # Desc 2
- response = translateAI(weaponsList[2], "")
- descListTL2 = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Check Mismatch
- if len(nameListTL) != len(weaponsList[0]) or len(descListTL1) != len(weaponsList[1]) or len(descListTL2) != len(weaponsList[2]):
- with LOCK:
- if filename not in MISMATCH:
- MISMATCH.append(filename)
- else:
- weaponsListTL = [nameListTL, descListTL1, descListTL2]
- translate = True
-
- # Skills
- if len(skillList[0]) > 0:
- # Progress Bar
- total = 0
- for skillArray in skillList:
- total += len(skillArray)
- pbar.total = total
- pbar.refresh()
-
- # Name
- response = translateAI(
- skillList[0],
- "Reply with only the " + LANGUAGE + " translation of the RPG skill name",
- True,
- )
- nameListTL = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Desc
- response = translateAI(
- skillList[1],
- "Reply with only the " + LANGUAGE + " translation of the RPG skill description",
- True,
- )
- descListTL1 = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Log 1
- response = translateAI(
- skillList[2],
- "reply with only the gender neutral "
- + LANGUAGE
- + " translation of the action log. Always start the sentence with Taro. For example, Translate 'Taroを倒した！' as 'Taro was defeated!'",
- True,
- )
- descListTL2 = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Log 2
- response = translateAI(
- skillList[3],
- "reply with only the gender neutral "
- + LANGUAGE
- + " translation of the action log. Always start the sentence with Taro. For example, Translate 'Taroを倒した！' as 'Taro was defeated!'",
- True,
- )
- descListTL3 = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Log 3
- response = translateAI(
- skillList[4],
- "reply with only the gender neutral "
- + LANGUAGE
- + " translation of the action log. Always start the sentence with Taro. For example, Translate 'Taroを倒した！' as 'Taro was defeated!'",
- True,
- )
- descListTL4 = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Check Mismatch
- if (
- len(nameListTL) != len(skillList[0])
- or len(descListTL1) != len(skillList[1])
- or len(descListTL2) != len(skillList[2])
- or len(descListTL3) != len(skillList[3])
- or len(descListTL4) != len(skillList[4])
- ):
- with LOCK:
- if filename not in MISMATCH:
- MISMATCH.append(filename)
- else:
- skillListTL = [nameListTL, descListTL1, descListTL2, descListTL3, descListTL4]
- translate = True
-
- # State
- for list in stateList:
- if len(list) > 0:
- # Progress Bar
- total = 0
- for stateArray in stateList:
- total += len(stateArray)
- pbar.total = total
- pbar.refresh()
-
- # Name
- response = translateAI(
- stateList[0],
- f"Reply with the {LANGUAGE} translation of the status effect.",
- True,
- )
- nameListTL = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Desc 1
- response = translateAI(stateList[1], "")
- descListTL1 = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Log 1
- response = translateAI(
- stateList[2],
- f"Reply with the {LANGUAGE} translation of the status effect.",
- True,
- )
- descListTL2 = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Log 2
- response = translateAI(
- stateList[3],
- "reply with only the gender neutral "
- + LANGUAGE
- + " translation of the action log. Always start the sentence with Taro. For example, Translate 'Taroを倒した！' as 'Taro was defeated!'",
- True,
- )
- descListTL3 = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Log 3
- response = translateAI(
- stateList[4],
- "reply with only the gender neutral "
- + LANGUAGE
- + " translation of the action log. Always start the sentence with Taro. For example, Translate 'Taroを倒した！' as 'Taro was defeated!'",
- True,
- )
- descListTL4 = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Log 4
- response = translateAI(
- stateList[5],
- "reply with only the gender neutral "
- + LANGUAGE
- + " translation of the action log. Always start the sentence with Taro. For example, Translate 'Taroを倒した！' as 'Taro was defeated!'",
- True,
- )
- descListTL5 = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Log 1
- response = translateAI(
- stateList[6],
- "reply with only the gender neutral "
- + LANGUAGE
- + " translation of the action log. Always start the sentence with Taro. For example, Translate 'Taroを倒した！' as 'Taro was defeated!'",
- True,
- )
- descListTL6 = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Log 1
- response = translateAI(
- stateList[7],
- "reply with only the gender neutral "
- + LANGUAGE
- + " translation of the action log. Always start the sentence with Taro. For example, Translate 'Taroを倒した！' as 'Taro was defeated!'",
- True,
- )
- descListTL7 = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Check Mismatch
- if (
- len(nameListTL) != len(stateList[0])
- or len(descListTL1) != len(stateList[1])
- or len(descListTL2) != len(stateList[2])
- or len(descListTL3) != len(stateList[3])
- or len(descListTL4) != len(stateList[4])
- or len(descListTL5) != len(stateList[5])
- or len(descListTL6) != len(stateList[6])
- ):
- with LOCK:
- if filename not in MISMATCH:
- MISMATCH.append(filename)
- else:
- stateListTL = [
- nameListTL,
- descListTL1,
- descListTL2,
- descListTL3,
- descListTL4,
- descListTL5,
- descListTL6,
- descListTL7,
- ]
- translate = True
-
- # DB Names
- if len(dbNameList[0]) > 0:
- # Progress Bar
- total = 0
- for dbNameArray in dbNameList:
- total += len(dbNameArray)
- pbar.total = total
- pbar.refresh()
-
- # Name
- response = translateAI(dbNameList[0], "Reply with only the " + LANGUAGE + " translation")
- nameListTL = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Check Mismatch
- if len(nameListTL) != len(dbNameList[0]):
- with LOCK:
- if filename not in MISMATCH:
- MISMATCH.append(filename)
- else:
- dbNameListTL = [nameListTL]
- translate = True
-
- # DB Values
- if len(dbValueList[0]) > 0:
- # Progress Bar
- total = 0
- for dbValueArray in dbValueList:
- total += len(dbValueArray)
- pbar.total = total
- pbar.refresh()
-
- # Name
- response = translateAI(dbValueList[0], "Reply with only the " + LANGUAGE + " translation")
- valueListTL = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Check Mismatch
- if len(valueListTL) != len(dbValueList[0]):
- with LOCK:
- if filename not in MISMATCH:
- MISMATCH.append(filename)
- else:
- dbValueListTL = [valueListTL]
- translate = True
-
- # Start Pass 2
- if translate == True:
- jobList.append(scenarioListTL)
- jobList.append(npcListTL)
- jobList.append(itemListTL)
- jobList.append(stateListTL)
- jobList.append(armorListTL)
- jobList.append(enemyListTL)
- jobList.append(weaponsListTL)
- jobList.append(skillListTL)
- jobList.append(optionsListTL)
- jobList.append(dbNameListTL)
- jobList.append(dbValueListTL)
- searchDB(events, pbar, jobList, filename)
-
- except IndexError as e:
- traceback.print_exc()
- raise Exception(str(e) + "Failed to translate: " + initialJAString) from None
- except Exception as e:
- traceback.print_exc()
- raise Exception(str(e) + "Failed to translate: " + initialJAString) from None
-
- return totalTokens
-
-# Save some money and enter the character before translation
-def getSpeaker(speaker):
- match speaker:
- case "ファイン":
- return ["Fine", [0, 0]]
- case "":
- return ["", [0, 0]]
- case _:
- # Find Speaker
- for i in range(len(NAMESLIST)):
- if speaker == NAMESLIST[i][0]:
- return [NAMESLIST[i][1], [0, 0]]
-
- # Translate and Store Speaker
- response = translateAI(
- f"{speaker}",
- "Reply with the " + LANGUAGE + " translation of the NPC name.",
- False,
- )
- response[0] = response[0].title()
- response[0] = response[0].replace("'S", "'s")
- response[0] = response[0].replace("Speaker: ", "")
-
- # Retry if name doesn't translate for some reason
- if re.search(r"([a-zA-Z？?])", response[0]) == None:
- response = translateAI(
- f"{speaker}",
- "Reply with the " + LANGUAGE + " translation of the NPC name.",
- False,
- )
- response[0] = response[0].title()
- response[0] = response[0].replace("'S", "'s")
-
- speakerList = [speaker, response[0]]
- NAMESLIST.append(speakerList)
- return response
- return [speaker, [0, 0]]
-
-def translateAI(text, history, history_ctx=None):
- """
- Legacy wrapper function for the new shared translation utility.
- This maintains compatibility with existing code while using the new shared implementation.
- """
- global PBAR, MISMATCH, FILENAME
-
- # Update config estimate mode based on global ESTIMATE
- TRANSLATION_CONFIG.estimateMode = bool(ESTIMATE)
-
- # Call the new shared translation function
- return sharedtranslateAI(
- text=text,
- history=history,
- config=TRANSLATION_CONFIG,
- filename=FILENAME,
- pbar=PBAR,
- lock=LOCK,
- mismatchList=MISMATCH
- )
+# Libraries
+import json
+import os
+import re
+import util.dazedwrap as dazedwrap
+import threading
+import time
+import traceback
+import tiktoken
+# Removed concurrent.futures usage for simplicity; running synchronously
+from pathlib import Path
+from colorama import Fore
+from dotenv import load_dotenv
+from retry import retry
+from tqdm import tqdm
+from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost, getPricingConfig, calculateCost
+
+# OpenAI initialization centralized in util/translation.py
+
+# Globals
+MODEL = os.getenv("model")
+TIMEOUT = int(os.getenv("timeout"))
+LANGUAGE = os.getenv("language").capitalize()
+from util.paths import PROMPT_PATH, VOCAB_PATH
+
+PROMPT = PROMPT_PATH.read_text(encoding="utf-8")
+VOCAB = VOCAB_PATH.read_text(encoding="utf-8")
+LOCK = threading.Lock()
+WIDTH = int(os.getenv("width"))
+LISTWIDTH = int(os.getenv("listWidth"))
+NOTEWIDTH = int(os.getenv("noteWidth"))
+MAXHISTORY = 10
+ESTIMATE = ""
+TOKENS = [0, 0]
+NAMESLIST = [] # Keep list for consistency
+TERMSLIST = [] # Keep list for consistency
+NAMES = False # Output a list of all the character names found
+BRFLAG = False # If the game uses
instead
+FIXTEXTWRAP = True # Overwrites textwrap
+IGNORETLTEXT = False # Ignores all translated text.
+MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
+FILENAME = None
+BRACKETNAMES = False
+
+# Regex - Need to change this if you want to translate from/to other languages. Default is Japanese Regex
+LANGREGEX = r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+"
+
+# Get pricing configuration based on the model
+PRICING_CONFIG = getPricingConfig(MODEL)
+INPUTAPICOST = PRICING_CONFIG["inputAPICost"]
+OUTPUTAPICOST = PRICING_CONFIG["outputAPICost"]
+BATCHSIZE = PRICING_CONFIG["batchSize"]
+FREQUENCY_PENALTY = PRICING_CONFIG["frequencyPenalty"]
+
+# tqdm Globals
+BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
+POSITION = 0
+LEAVE = False
+PBAR = None
+FILENAME = None
+
+# Dialogue / Choices
+CODE101 = False
+CODE102 = False
+
+# Picture
+CODE150 = False
+
+# Set String (Fragile but necessary)
+CODE122 = False
+
+# Other
+CODE210 = False
+CODE300 = False
+CODE250 = False
+
+# Database
+SCENARIOFLAG = False
+OPTIONSFLAG = True
+NPCFLAG = False
+DBNAMEFLAG = False
+DBVALUEFLAG = False
+ITEMFLAG = False
+STATEFLAG = False
+ENEMYFLAG = False
+ARMORFLAG = False
+WEAPONFLAG = False
+SKILLFLAG = False
+
+# Initialize Translation Config
+TRANSLATION_CONFIG = TranslationConfig(
+ model=MODEL,
+ language=LANGUAGE,
+ prompt=PROMPT,
+ vocab=VOCAB,
+ langRegex=LANGREGEX,
+ batchSize=BATCHSIZE,
+ maxHistory=MAXHISTORY,
+ estimateMode=False # Will be set dynamically based on ESTIMATE
+)
+LEAVE = False
+
+def handleWOLF(filename, estimate):
+ global ESTIMATE, TOKENS, FILENAME
+ ESTIMATE = estimate
+ FILENAME = filename
+
+ # Translate
+ start = time.time()
+ translatedData = openFiles(filename)
+
+ # Translate
+ if not estimate:
+ try:
+ with open("translated/" + filename, "w", encoding="utf-8", newline="\n") as outFile:
+ json.dump(translatedData[0], outFile, ensure_ascii=False, indent=4)
+ except Exception:
+ traceback.print_exc()
+ return "Fail"
+
+ # Print File
+ end = time.time()
+ tqdm.write(getResultString(translatedData, end - start, filename))
+ with LOCK:
+ TOKENS[0] += translatedData[1][0]
+ TOKENS[1] += translatedData[1][1]
+
+ # Print Total
+ totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
+
+ # Print any errors on maps
+ if len(MISMATCH) > 0:
+ return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
+ else:
+ return totalString
+
+
+def openFiles(filename):
+ with open("files/" + filename, "r", encoding="utf-8-sig") as f:
+ data = json.load(f)
+
+ # Map Files
+ if "'events':" in str(data):
+ if len(data["events"]) > 0:
+ translatedData = parseMap(data, filename)
+ else:
+ return [data, [0, 0], None]
+
+ # Map Files
+ elif "'types':" in str(data):
+ translatedData = parseDB(data, filename)
+
+ # Other Files
+ elif "'commands':" in str(data):
+ translatedData = parseOther(data, filename)
+
+ else:
+ raise NameError(filename + " Not Supported")
+
+ return translatedData
+
+
+def getResultString(translatedData, translationTime, filename):
+ # File Print String
+ cost = calculateCost(translatedData[1][0], translatedData[1][1], MODEL)
+ totalTokenstring = (
+ Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
+ "[Output: "
+ + str(translatedData[1][1])
+ + "]" "[Cost: ${:,.4f}".format(cost)
+ + "]"
+ )
+ timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
+
+ if translatedData[2] is None:
+ # Success
+ return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET
+ else:
+ # Fail
+ try:
+ raise translatedData[2]
+ except Exception as e:
+ traceback.print_exc()
+ errorString = str(e) + Fore.RED
+ return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET
+
+
+def save_progress_json(data, filename):
+ """Atomically write current JSON data to translated/filename; skip in estimate mode."""
+ try:
+ if ESTIMATE:
+ return
+ os.makedirs("translated", exist_ok=True)
+ tmp_path = os.path.join("translated", f"{filename}.tmp")
+ final_path = os.path.join("translated", filename)
+ with open(tmp_path, "w", encoding="utf-8", newline="\n") as outFile:
+ json.dump(data, outFile, ensure_ascii=False, indent=4)
+ os.replace(tmp_path, final_path)
+ except Exception:
+ traceback.print_exc()
+
+
+def maybe_save_progress_json(data, filename, tokens):
+ """Save JSON progress only when tokens indicate actual translation work."""
+ try:
+ if not tokens:
+ return
+ if isinstance(tokens, (list, tuple)) and len(tokens) >= 2 and (tokens[0] or tokens[1]):
+ save_progress_json(data, filename)
+ except Exception:
+ traceback.print_exc()
+
+
+def parseOther(data, filename):
+ totalTokens = [0, 0]
+ totalLines = 0
+ events = data["commands"]
+ global LOCK
+
+ # Thread for each page in file
+ with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
+ pbar.desc = filename
+ pbar.total = totalLines
+ translationData = searchCodes(events, pbar, [], filename)
+ try:
+ totalTokens[0] += translationData[0]
+ totalTokens[1] += translationData[1]
+ except Exception as e:
+ return [data, totalTokens, e]
+ finally:
+ maybe_save_progress_json(data, filename, translationData)
+ return [data, totalTokens, None]
+
+
+def parseDB(data, filename):
+ totalTokens = [0, 0]
+ totalLines = 0
+ events = data["types"]
+ global LOCK
+
+ # Thread for each page in file
+ with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
+ pbar.desc = filename
+ pbar.total = totalLines
+ translationData = searchDB(events, pbar, [], filename)
+ try:
+ totalTokens[0] += translationData[0]
+ totalTokens[1] += translationData[1]
+ except Exception as e:
+ return [data, totalTokens, e]
+ finally:
+ maybe_save_progress_json(data, filename, translationData)
+ return [data, totalTokens, None]
+
+
+def parseMap(data, filename):
+ totalTokens = [0, 0]
+ totalLines = 0
+ events = data["events"]
+ global LOCK
+
+ # Get total for progress bar
+ for event in events:
+ if event is not None:
+ for page in event["pages"]:
+ totalLines += len(page["list"])
+
+ # Process pages synchronously and persist after each
+ with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar:
+ pbar.desc = filename
+ pbar.total = totalLines
+ for event in events:
+ if event is not None:
+ for page in event["pages"]:
+ if page is not None:
+ try:
+ tt = searchCodes(page["list"], pbar, None, filename)
+ totalTokens[0] += tt[0]
+ totalTokens[1] += tt[1]
+ except Exception as e:
+ return [data, totalTokens, e]
+ finally:
+ maybe_save_progress_json(data, filename, tt)
+ return [data, totalTokens, None]
+
+
+def searchCodes(events, pbar, jobList, filename):
+ # Lists
+ if jobList:
+ stringList = jobList[0]
+ list210 = jobList[1]
+ list300 = jobList[2]
+ list150 = jobList[3]
+ list250 = jobList[4]
+ list122 = jobList[5]
+ setData = True
+ else:
+ stringList = []
+ list210 = []
+ list300 = []
+ list150 = []
+ list250 = []
+ list122 = []
+ setData = False
+
+ # Other
+ codeList = events
+ textHistory = []
+ totalTokens = [0, 0]
+ translatedText = ""
+ speaker = ""
+ nametag = ""
+ initialJAString = ""
+ global LOCK, NAMESLIST, MISMATCH, PBAR, FILENAME
+ FILENAME = filename
+ PBAR = pbar
+
+ # Calculate Total Length
+ code_flags = {102: CODE102, 122: CODE122, 300: CODE300, 250: CODE250}
+ totalList = 0
+ for code_item in codeList:
+ if code_flags.get(code_item["code"], False):
+ totalList += 1
+ pbar.total = totalList
+ pbar.refresh()
+
+ # Begin Parsing File
+ try:
+ # Iterate through events
+ i = 0
+ while i < len(codeList):
+ ### Event Code: 101 Message
+ if codeList[i]["code"] == 101 and CODE101 == True:
+ speakerRegex = r"@\d+\r?\n(.*?)：?\r?\n" # Default: r"@\d+\r?\n(.*)：?\r?\n""
+ textRegex = r"@\d+\r?\n.*?：?\r?\n(.*)" # Default: r"@\d+\r?\n.*：?\r?\n(.*)"
+ # Alternative patterns for messages without @\d+ prefix
+ speakerRegexAlt = r"^(.*?)：\r?\n" # Matches "Name：\n"
+ textRegexAlt = r"^.*?：\r?\n(.*)" # Matches text after "Name：\n"
+
+ # Grab String
+ jaString = codeList[i]["stringArgs"][0]
+ speaker = ""
+
+ # Grab Speaker
+ if "\n" in jaString:
+ match = re.search(speakerRegex, jaString)
+ if not match:
+ # Try alternative pattern without @\d+ prefix
+ match = re.search(speakerRegexAlt, jaString)
+ if match:
+ # TL Speaker
+ response = getSpeaker(match.group(1))
+ speaker = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Set Name
+ codeList[i]["stringArgs"][0] = codeList[i]["stringArgs"][0].replace(match.group(1), speaker)
+
+ # Grab Only Text
+ match = re.search(textRegex, jaString, flags=re.DOTALL)
+ if not match:
+ # Try alternative pattern without @\d+ prefix
+ match = re.search(textRegexAlt, jaString, flags=re.DOTALL)
+ if not match and re.search(LANGREGEX, jaString):
+ # No speaker pattern found, treat entire string as text to translate
+ jaString = jaString
+ initialJAString = jaString
+ elif match:
+ jaString = match.group(1)
+ initialJAString = jaString
+ else:
+ jaString = None # Skip non-translatable text
+
+ if jaString is not None:
+
+ # Remove Textwrap
+ jaString = jaString.replace("\r", "")
+ jaString = jaString.replace("\n", " ")
+
+ # 1st Pass (Save Text to List)
+ if not setData:
+ if speaker == "":
+ stringList.append(jaString)
+ else:
+ stringList.append(f"[{speaker}]: {jaString}")
+
+ # 2nd Pass (Set Text)
+ else:
+ # Grab Translated String
+ translatedText = stringList[0]
+
+ # Remove speaker
+ matchSpeakerList = re.findall(r"^(\[.+?\]\s?[|:]\s?)\s?", translatedText)
+ if len(matchSpeakerList) > 0:
+ translatedText = translatedText.replace(matchSpeakerList[0], "")
+
+ # Textwrap
+ if FIXTEXTWRAP is True:
+ translatedText = dazedwrap.wrapText(translatedText, width=WIDTH)
+
+ # Set Data
+ codeList[i]["stringArgs"][0] = codeList[i]["stringArgs"][0].replace(initialJAString, translatedText)
+
+ # Reset Data and Pop Item
+ stringList.pop(0)
+
+ ### Event Code: 102 Choices
+ if codeList[i]["code"] == 102 and CODE102 == True:
+ # Grab Choice List
+ choiceList = []
+ jaChoiceList = codeList[i]["stringArgs"]
+
+ # Filter Empty
+ for j in range(len(jaChoiceList)):
+ if jaChoiceList[j]:
+ choiceList.append(jaChoiceList[j])
+
+ # Translate
+ if 'jaString' in locals():
+ choiceString = f"Previous Line: {jaString}\n\nReply with the {LANGUAGE} translation of the dialogue choice"
+ else:
+ choiceString = f"Reply with the {LANGUAGE} translation of the dialogue choice"
+ response = translateAI(
+ choiceList,
+ choiceString,
+ True,
+ )
+ translatedChoiceList = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Validate and Set Data
+ if len(translatedChoiceList) == len(choiceList):
+ for j in range(len(jaChoiceList)):
+ if jaChoiceList[j]:
+ codeList[i]["stringArgs"][j] = translatedChoiceList[0]
+ translatedChoiceList.pop(0)
+
+ ### Event Code: 210 Common Event
+ if codeList[i]["code"] == 210 and CODE210 == True:
+ # Speaker Event
+ if "stringArgs" in codeList[i] and codeList[i]["intArgs"][0] == None and len(codeList[i]["stringArgs"]) == 2:
+ response = getSpeaker(codeList[i]["stringArgs"][1])
+ totalTokens[1] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Set Data
+ codeList[i]["stringArgs"][1] = response[0]
+
+ # Logs
+ elif "stringArgs" in codeList[i] and codeList[i]["intArgs"][0] == 500725 and len(codeList[i]["stringArgs"]) == 2:
+ # Grab String
+ jaString = codeList[i]["stringArgs"][1]
+ initialJAString = jaString
+
+ # Remove Textwrap
+ jaString = jaString.replace("\r", "")
+ jaString = jaString.replace("\n", " ")
+
+ # 1st Pass (Save Text to List)
+ if not setData:
+ list210.append(jaString)
+
+ # 2nd Pass (Set Text)
+ else:
+ # Grab Translated String
+ translatedText = list210[0]
+
+ # Textwrap
+ if FIXTEXTWRAP is True:
+ translatedText = dazedwrap.wrapText(translatedText, width=WIDTH)
+
+ # Set Data
+ codeList[i]["stringArgs"][1] = translatedText
+
+ # Pop Item
+ list210.pop(0)
+
+ ### Event Code: 122 SetString
+ if codeList[i]["code"] == 122 and CODE122 == True:
+ if "stringArgs" in codeList[i] and len(codeList[i]["stringArgs"]) > 0:
+ # Grab String
+ jaString = re.search(r"^\n?(.*)\n?$", codeList[i]["stringArgs"][0])
+ if jaString:
+ jaString = jaString.group(1)
+ else:
+ jaString = codeList[i]["stringArgs"][0]
+
+ originalString = jaString
+
+ # Remove Textwrap
+ jaString = jaString.replace("\r", "")
+ jaString = jaString.replace("\n", " ")
+
+ # Check if this is a translatable string
+ if (
+ not re.search(r"\.[\w]+$", jaString)
+ and jaString != ""
+ and "_" not in jaString
+ and '",' not in jaString
+ and "/" not in jaString
+ and re.search(LANGREGEX, jaString)
+ ):
+ # 1st Pass (Save Text to List)
+ if not setData:
+ list122.append(jaString)
+
+ # 2nd Pass (Set Text)
+ else:
+ if len(list122) > 0:
+ # Grab Translated String
+ translatedText = list122[0]
+
+ # Textwrap
+ if FIXTEXTWRAP is True:
+ translatedText = dazedwrap.wrapText(translatedText, width=WIDTH)
+
+ # Set Data
+ codeList[i]["stringArgs"][0] = codeList[i]["stringArgs"][0].replace(originalString, translatedText)
+
+ # Pop Item
+ list122.pop(0)
+
+ ### Event Code: 150 Picture String
+ if codeList[i]["code"] == 150 and CODE150 == True:
+ if "stringArgs" in codeList[i] and len(codeList[i]["stringArgs"]) > 0:
+ font150 = "\\f[8]"
+ # Grab String
+ jaString = codeList[i]["stringArgs"][0]
+
+ # Japanses Text Only
+ if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", jaString):
+ i += 1
+ continue
+
+ # Remove Textwrap
+ # jaString = jaString.replace("\n", " ")
+ # jaString = jaString.replace("\r", "")
+
+ # Translate Other Strings [Specific Files Only]
+ if (
+ not re.search(r"\.[\w]+$", jaString)
+ and jaString != ""
+ and "_" not in jaString
+ and '",' not in jaString
+ # and ">" not in jaString
+ # and "<" not in jaString
+ ):
+ # Pass 1 (Save Text to List)
+ if not setData:
+ list150.append(jaString)
+
+ # Pass 2 (Set Text)
+ else:
+ translatedText = list150[0]
+
+ # Textwrap
+ # translatedText = dazedwrap.wrapText(translatedText, WIDTH)
+
+ # Set String with font formatting
+ codeList[i]["stringArgs"][0] = re.sub(r'\\f\[(\d+)\]', lambda x: f'\\f[{int(x.group(1))-2}]', translatedText)
+
+ # Pop processed item
+ list150.pop(0)
+
+ ### Event Code: 300 Common Events
+ if codeList[i]["code"] == 300 and CODE300 == True and "stringArgs" in codeList[i] and len(codeList[i]["stringArgs"]) > 1:
+ # Choices
+ if codeList[i]["stringArgs"][0] == "[共]汎用ウィンドウ生成" or codeList[i]["stringArgs"][0] == "選択肢/確認":
+ # Grab String
+ choiceList = codeList[i]["stringArgs"][1].split(",")
+
+ # Translate Question
+ question = codeList[i]['stringArgs'][2]
+ response = translateAI(question, "Reply with the {LANGUAGE} translation")
+ translatedText = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Translate Question
+ jaString = translatedText
+ codeList[i]['stringArgs'][2] = translatedText
+
+ # Translate Choices
+ if 'jaString' in locals() and jaString:
+ response = translateAI(choiceList, jaString)
+ else:
+ response = translateAI(choiceList, f"Reply with the {LANGUAGE} translation of the dialogue choice")
+ choiceListTL = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Replace Commas
+ for j in range(len(choiceListTL)):
+ choiceListTL[j] = choiceListTL[j].replace(", ", "、")
+
+ # Convert to String and Set
+ translatedText = ",".join(choiceListTL)
+ codeList[i]["stringArgs"][1] = translatedText
+
+ # Dialogue
+ elif codeList[i]["stringArgs"][0] == "BTLメッセージ" or codeList[i]["stringArgs"][0] == "X[移]メニュー時文章表示":
+ jaString = codeList[i]["stringArgs"][1]
+
+ # Pass 1
+ if not setData:
+ # Remove Textwrap
+ jaString = jaString.replace("\n", " ")
+ jaString = jaString.replace("\r", "")
+
+ # Append
+ list300.append(jaString)
+
+ # Pass 2
+ else:
+ # Add Textwrap and Font
+ translatedText = dazedwrap.wrapText(list300[0], WIDTH)
+ list300.pop(0)
+
+ # Write to File
+ codeList[i]["stringArgs"][1] = translatedText
+
+ # Dialogue
+ elif codeList[i]["stringArgs"][0] == "援護文章":
+ speakerRegex = r"@\d+\r?\n(.*)：\r?\n" # Default: r"@\d+\r?\n(.*)：\r?\n"
+ textRegex = r"@?\d*\r?\n?\u3000*([\w\W]+)\r?\n?" # Default: r"@?\d*\r?\n?\u3000*([\w\W]+)\r?\n?"
+
+ # Grab String
+ jaString = codeList[i]["stringArgs"][1]
+ speaker = ""
+
+ # Grab Speaker
+ if "：\n" in jaString or "：\r\n" in jaString:
+ match = re.search(speakerRegex, jaString)
+ if match:
+ # TL Speaker
+ response = getSpeaker(match.group(1))
+ speaker = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Set nametag and remove from string
+ codeList[i]["stringArgs"][1] = codeList[i]["stringArgs"][1].replace(match.group(1), speaker)
+ jaString = jaString.replace(match.group(0), "")
+
+ # Grab Only Text
+ match = re.search(textRegex, jaString)
+ if match:
+ jaString = match.group(1)
+ initialJAString = jaString
+
+ # Remove Textwrap
+ jaString = jaString.replace("\r", "")
+ jaString = jaString.replace("\n", " ")
+
+ # 1st Pass (Save Text to List)
+ if not setData:
+ if speaker == "":
+ list300.append(jaString)
+ else:
+ list300.append(f"[{speaker}]: {jaString}")
+
+ # 2nd Pass (Set Text)
+ else:
+ # Grab Translated String
+ translatedText = list300[0]
+
+ # Remove speaker
+ matchSpeakerList = re.findall(r"^(\[.+?\]\s?[|:]\s?)\s?", translatedText)
+ if len(matchSpeakerList) > 0:
+ translatedText = translatedText.replace(matchSpeakerList[0], "")
+
+ # Textwrap
+ if FIXTEXTWRAP is True:
+ translatedText = dazedwrap.wrapText(translatedText, width=WIDTH)
+
+ # Set Data
+ codeList[i]["stringArgs"][1] = codeList[i]["stringArgs"][1].replace(initialJAString, translatedText)
+
+ # Reset Data and Pop Item
+ list300.pop(0)
+
+ ### Event Code: 250 DB Read/Writes
+ if codeList[i]["code"] == 250 and CODE250 == True:
+ # Validate size
+ stringArg = 0
+ if len(codeList[i]["stringArgs"]) == 4:
+ if codeList[i]["stringArgs"][1] == "万能ｳｨﾝﾄﾞｳ一時DB"\
+ and codeList[i]["stringArgs"][stringArg] != "":
+ # Font Size
+ fontSize = 0
+
+ # Grab String
+ jaString = codeList[i]["stringArgs"][stringArg]
+
+ # Remove Textwrap
+ # jaString = jaString.replace("\r", "")
+ # jaString = jaString.replace("\n", " ")
+
+ # Pass 1 (Save Text to List)
+ if not setData:
+ list250.append(jaString)
+
+ # Pass 2 (Set Text)
+ else:
+ translatedText = list250[0]
+ list250.pop(0)
+
+ # Textwrap
+ # translatedText = dazedwrap.wrapText(translatedText, WIDTH)
+
+ # Add/Replace font formatting
+ # Remove existing font command if present
+ translatedText = re.sub(r'\\f\[\d+\]', '', translatedText)
+ if fontSize:
+ # Add new font command if fontSize is not 0
+ if fontSize != "0" and fontSize != 0:
+ translatedText = f"\\f[{fontSize}]{translatedText}"
+
+ # Set Data
+ codeList[i]["stringArgs"][stringArg] = translatedText
+
+ ### Iterate
+ i += 1
+
+ # EOF
+ stringListTL = []
+ list210TL = []
+ list150TL = []
+ list250TL = []
+ list300TL = []
+ list122TL = []
+ setData = False
+
+ # String List
+ if len(stringList) > 0:
+ pbar.total = len(stringList)
+ pbar.refresh()
+ response = translateAI(stringList, textHistory)
+ stringListTL = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+ if len(stringListTL) != len(stringList):
+ with LOCK:
+ if filename not in MISMATCH:
+ MISMATCH.append(filename)
+ else:
+ setData = True
+
+ # 210 List
+ if len(list210) > 0:
+ pbar.total = len(list210)
+ pbar.refresh()
+ response = translateAI(list210, textHistory)
+ list210TL = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+ if len(list210TL) != len(list210):
+ with LOCK:
+ if filename not in MISMATCH:
+ MISMATCH.append(filename)
+ else:
+ setData = True
+
+ # 250 List
+ if len(list250) > 0:
+ pbar.total = len(list250)
+ pbar.refresh()
+ response = translateAI(list250, textHistory)
+ list250TL = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+ if len(list250TL) != len(list250):
+ with LOCK:
+ if filename not in MISMATCH:
+ MISMATCH.append(filename)
+ else:
+ setData = True
+
+ # 300 List
+ if len(list300) > 0:
+ pbar.total = len(list300)
+ pbar.refresh()
+ response = translateAI(list300, textHistory)
+ list300TL = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+ if len(list300TL) != len(list300):
+ with LOCK:
+ if filename not in MISMATCH:
+ MISMATCH.append(filename)
+ else:
+ setData = True
+
+ # 150 List
+ if len(list150) > 0:
+ # Progress Bar
+ pbar.total = len(list150)
+ pbar.refresh()
+
+ # Translate
+ response = translateAI(
+ list150,
+ textHistory,
+ True,
+ )
+ list150TL = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Check Mismatch
+ if len(list150TL) != len(list150):
+ with LOCK:
+ if filename not in MISMATCH:
+ MISMATCH.append(filename)
+ else:
+ list150 = list150TL
+ setData = True
+
+ # 122 List
+ if len(list122) > 0:
+ pbar.total = len(list122)
+ pbar.refresh()
+ response = translateAI(list122, textHistory)
+ list122TL = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+ if len(list122TL) != len(list122):
+ with LOCK:
+ if filename not in MISMATCH:
+ MISMATCH.append(filename)
+ else:
+ setData = True
+
+ # Update jobList before recursive call
+ if setData == True:
+ stringList = []
+ jobList = [stringListTL, list210TL, list300TL, list150TL, list250TL, list122TL] # Add list122TL
+ searchCodes(events, pbar, jobList, filename)
+
+ else:
+ # Set Data
+ events = codeList
+
+ except IndexError as e:
+ traceback.print_exc()
+ raise Exception(str(e) + "Failed to translate: " + initialJAString) from None
+ except Exception as e:
+ traceback.print_exc()
+ raise Exception(str(e) + "Failed to translate: " + initialJAString) from None
+
+ return totalTokens
+
+
+def formatDramon(jaString):
+ imageRegex = r"(\r?\n?_[a-zA-Z_\d/.]+\r?\n?)|(@-?\d?\r\n)|(@-?\d?)([^\r]\d?-?\d?[^\r]+?)(\r\n|$)|(_PDC)|(>\r\n)|(^[#])|(\r\n@$)|(/)|(_SS_)|(\r\n[@/]\s?-?\d?\r\n)"
+
+ jaString = jaString.replace("\u3000", " ")
+ jaString = jaString.replace("#", "")
+ jaString = re.sub(r"([^\r])\n", r"\1\r\n", jaString)
+
+ # Grab and Split
+ jaStringList = re.split(imageRegex, jaString)
+
+ # Clean List
+ cleanedList = [x for x in jaStringList if x is not None and x != "" and x != "\r\n" and x != "_SS_"]
+
+ # Iterate Through List
+ j = 0
+ translatedText = ""
+ while j < len(cleanedList):
+ if (
+ ("@" in cleanedList[j] or "/" in cleanedList[j])
+ and j < len(cleanedList) - 1
+ and re.search(r"([@/]-?\d?\r\n)", cleanedList[j]) is None
+ and ".ogg" not in cleanedList[j]
+ ):
+ # Setup @
+ if j > 0 and "@" not in cleanedList[j - 1] and "/" not in cleanedList[j - 1] and "_" not in cleanedList[j - 1]:
+ cleanedList[j - 1] = cleanedList[j - 1] + cleanedList[j + 1]
+ else:
+ cleanedList.insert(j, cleanedList[j + 1])
+ j += 1
+ cleanedList[j] = f"\r\n{cleanedList[j]}\r\n"
+ cleanedList.pop(j + 1)
+ j += 1
+
+ return cleanedList
+
+def handleScenarioScript(jaString, scriptString=""):
+ # Extract Speaker
+ # Updated regex to match speaker names with spaces, hyphens, and other characters
+ # [^\n：]+ matches any character except newline and ： (the speaker name)
+ scriptRegex = r"\n?(//.*?\n|//.*|/b.*?\n|[^\n：]+：\d*\r?\n)"
+ match = re.search(scriptRegex, jaString)
+ if match:
+ if "//" in match.group(1):
+ scriptStringLocal = match.group(1)
+ jaString = jaString.replace(scriptStringLocal, "")
+ scriptString += scriptStringLocal
+ match = re.search(scriptRegex, jaString)
+ if not match:
+ return None
+ else:
+ if "//" in jaString:
+ return handleScenarioScript(jaString, scriptString)
+ return [jaString, match.group(1), scriptString]
+ else:
+ return None
+
+# Database
+def searchDB(events, pbar, jobList, filename):
+ # Set Lists
+ if len(jobList) > 0:
+ scenarioList = jobList[0]
+ npcList = jobList[1]
+ itemList = jobList[2]
+ stateList = jobList[3]
+ armorList = jobList[4]
+ enemyList = jobList[5]
+ weaponsList = jobList[6]
+ skillList = jobList[7]
+ optionsList = jobList[8]
+ dbNameList = jobList[9]
+ dbValueList = jobList[10]
+ setData = True
+ else:
+ scenarioList = [[], [], []]
+ npcList = [[], [], [], []]
+ itemList = [[], [], [], []]
+ armorList = [[], []]
+ enemyList = [[], []]
+ weaponsList = [[], [], [], []]
+ skillList = [[], [], [], [], []]
+ stateList = [[], [], [], [], [], [], [], []]
+ optionsList = [[], [], []]
+ dbNameList = [[]]
+ dbValueList = [[]]
+ setData = False
+
+ # Vars/Globals
+ totalTokens = [0, 0]
+ initialJAString = ""
+ tableList = events
+ font = ""
+ global LOCK
+ global NAMESLIST
+ global MISMATCH
+ global PBAR
+ PBAR = pbar
+
+ # Begin Parsing File
+ try:
+ for table in tableList:
+ # Grab NPCs
+ if table["name"] == "アイドル" and NPCFLAG == True:
+ with open("log/translations.txt", "a", encoding="utf-8") as file:
+ if setData:
+ file.write(f"\n#Actors\n")
+ for npc in table["data"]:
+ dataList = npc["data"]
+
+ # Parse
+ for j in range(len(dataList)):
+ # Name
+ if dataList[j].get("name") == "出身":
+ # Pass 1 (Grab Data)
+ if setData == False:
+ if dataList[j].get("value") != "":
+ npcList[0].append(dataList[j].get("value"))
+
+ # Pass 2 (Set Data)
+ else:
+ if dataList[j].get("value") != "":
+ # Write Name
+ file.write(f"{dataList[j].get('value')} ({npcList[0][0]})\n")
+
+ # Set Data
+ dataList[j].update({"value": npcList[0][0]})
+ npcList[0].pop(0)
+
+ # Description
+ if dataList[j].get("name") == "説明":
+ # Pass 1 (Grab Data)
+ if setData == False:
+ if dataList[j].get("value") != "":
+ # Remove Textwrap
+ jaString = dataList[j].get("value")
+ jaString = jaString.replace("\n", " ")
+ jaString = jaString.replace("\r", "")
+ jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString)
+
+ # Append Data
+ npcList[1].append(jaString)
+ # Pass 2 (Set Data)
+ else:
+ if dataList[j].get("value") != "":
+ # Textwrap
+ translatedText = npcList[1][0]
+ translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH)
+ translatedText = font + translatedText
+
+ # Set Data
+ dataList[j].update({"value": translatedText})
+ npcList[1].pop(0)
+
+ # Grab Scenario
+ if "デートキャラ" in table["name"] and SCENARIOFLAG == True:
+ for scenario in table["data"]:
+ dataList = scenario["data"]
+
+ # Parse
+ for j in range(len(dataList)):
+ # Name
+ if dataList[j].get("name") == "NULL":
+ if dataList[j].get("value"):
+ jaStringList = re.split(r'\r\n\r\n|\n\n', dataList[j].get("value"))
+ for jaString in jaStringList:
+ speakerNum = None
+ ogString = jaString
+ # Extract Speaker
+ speakerList = handleScenarioScript(jaString)
+
+ # Handle simple names without speaker pattern
+ if not speakerList:
+ # Pass 1 (Grab Data) - Simple name
+ if setData == False:
+ jaString = jaString.replace("\n", " ")
+ jaString = jaString.replace("\r", "")
+ if jaString.strip():
+ scenarioList[0].append(jaString)
+ # Pass 2 (Set Data) - Simple name
+ else:
+ if ogString.strip():
+ translatedText = scenarioList[0][0]
+ scenarioList[0].pop(0)
+ dataList[j].update({"value": dataList[j].get("value").replace(ogString, translatedText)})
+ continue
+
+ if speakerList[2]:
+ jaString = speakerList[0]
+ speakerNumMatch = re.search(r"：(\d+)", speakerList[1])
+ if speakerNumMatch:
+ speakerNum = speakerNumMatch.group(1)
+ speaker = re.sub(r"：\d*", "", speakerList[1])
+ speaker = re.sub(r"/b", "NPC", speaker)
+ speaker = speaker.replace("\r\n", "")
+
+ # Add Speaker
+ jaString = jaString.replace(speakerList[1], f"[{speaker}]: ")
+
+ # Pass 1 (Grab Data)
+ if setData == False:
+ jaString = jaString.replace("\n", " ")
+ jaString = jaString.replace("\r", "")
+ scenarioList[0].append(jaString)
+
+ # Pass 2 (Set Data)
+ else:
+ translatedText = scenarioList[0][0]
+ scenarioList[0].pop(0)
+
+ # Remove Speaker before wrapping so WIDTH applies to dialogue only
+ speakerMatch = re.search(r"\[(.+?)\]\s?[|:]\s?", translatedText)
+ if speakerMatch:
+ speaker = speakerMatch.group(1).strip()
+ match = re.search(r'(^\[.+?\]\s?[|:]\s?)', translatedText)
+ if match:
+ translatedText = translatedText.replace(match.group(1), "")
+
+ # Wrap text after removing speaker
+ translatedText = dazedwrap.wrapText(translatedText, WIDTH)
+
+ # Redo Old Speaker Format
+ speaker = speaker.replace("Speaker", "\b")
+ if speakerNum:
+ translatedText = f"{speaker}：{speakerNum}\r\n{translatedText}"
+ elif speaker == "NPC":
+ translatedText = f"/b\r\n{translatedText}"
+ else:
+ translatedText = f"{speaker}：\r\n{translatedText}"
+
+ # Add Script String
+ if speakerList[2]:
+ translatedText = f"{speakerList[2]}{translatedText}"
+
+ # Set Data
+ dataList[j].update({"value": dataList[j].get("value").replace(ogString, translatedText)})
+ # Description
+ if dataList[j].get("name") == "不正解会話":
+ if dataList[j].get("value"):
+ jaStringList = re.split(r'\r\n\r\n|\n\n', dataList[j].get("value"))
+ for jaString in jaStringList:
+ speakerNum = None
+ ogString = jaString
+ # Extract Speaker
+ speakerList = handleScenarioScript(jaString)
+
+ # Handle simple text without speaker pattern
+ if not speakerList:
+ # Pass 1 (Grab Data) - Simple text
+ if setData == False:
+ jaString = jaString.replace("\n", " ")
+ jaString = jaString.replace("\r", "")
+ if jaString.strip():
+ scenarioList[1].append(jaString)
+ # Pass 2 (Set Data) - Simple text
+ else:
+ if ogString.strip():
+ translatedText = scenarioList[1][0]
+ scenarioList[1].pop(0)
+ translatedText = dazedwrap.wrapText(translatedText, WIDTH)
+ dataList[j].update({"value": dataList[j].get("value").replace(ogString, translatedText)})
+ continue
+
+ if speakerList[2]:
+ jaString = speakerList[0]
+ speakerNumMatch = re.search(r"：(\d+)", speakerList[1])
+ if speakerNumMatch:
+ speakerNum = speakerNumMatch.group(1)
+ speaker = re.sub(r"：\d*", "", speakerList[1])
+ speaker = re.sub(r"/b", "NPC", speaker)
+ speaker = speaker.replace("\r\n", "")
+
+ # Add Speaker
+ jaString = jaString.replace(speakerList[1], f"[{speaker}]: ")
+
+ # Pass 1 (Grab Data)
+ if setData == False:
+ jaString = jaString.replace("\n", " ")
+ jaString = jaString.replace("\r", "")
+ scenarioList[1].append(jaString)
+
+ # Pass 2 (Set Data)
+ else:
+ translatedText = scenarioList[1][0]
+ scenarioList[1].pop(0)
+
+ # Remove Speaker before wrapping so WIDTH applies to dialogue only
+ speakerMatch = re.search(r"\[(.+?)\]\s?[|:]\s?", translatedText)
+ if speakerMatch:
+ speaker = speakerMatch.group(1).strip()
+ match = re.search(r'(^\[.+?\]\s?[|:]\s?)', translatedText)
+ if match:
+ translatedText = translatedText.replace(match.group(1), "")
+
+ # Wrap text after removing speaker
+ translatedText = dazedwrap.wrapText(translatedText, WIDTH)
+
+ # Redo Old Speaker Format
+ speaker = speaker.replace("Speaker", "\b")
+ if speakerNum:
+ translatedText = f"{speaker}：{speakerNum}\r\n{translatedText}"
+ elif speaker == "NPC":
+ translatedText = f"/b\r\n{translatedText}"
+ else:
+ translatedText = f"{speaker}：\r\n{translatedText}"
+
+ # Add Script String
+ if speakerList[2]:
+ translatedText = f"{speakerList[2]}{translatedText}"
+
+ # Set Data
+ dataList[j].update({"value": dataList[j].get("value").replace(ogString, translatedText)})
+
+ # Description
+ if dataList[j].get("name") == "正解会話":
+ if dataList[j].get("value"):
+ jaStringList = re.split(r'\r\n\r\n|\n\n', dataList[j].get("value"))
+ for jaString in jaStringList:
+ speakerNum = None
+ ogString = jaString
+ # Extract Speaker
+ speakerList = handleScenarioScript(jaString)
+
+ # Handle simple text without speaker pattern
+ if not speakerList:
+ # Pass 1 (Grab Data) - Simple text
+ if setData == False:
+ jaString = jaString.replace("\n", " ")
+ jaString = jaString.replace("\r", "")
+ if jaString.strip():
+ scenarioList[2].append(jaString)
+ # Pass 2 (Set Data) - Simple text
+ else:
+ if ogString.strip():
+ translatedText = scenarioList[2][0]
+ scenarioList[2].pop(0)
+ translatedText = dazedwrap.wrapText(translatedText, WIDTH)
+ dataList[j].update({"value": dataList[j].get("value").replace(ogString, translatedText)})
+ continue
+ if speakerList[2]:
+ jaString = speakerList[0]
+ speakerNumMatch = re.search(r"：(\d+)", speakerList[1])
+ if speakerNumMatch:
+ speakerNum = speakerNumMatch.group(1)
+ speaker = re.sub(r"：\d*", "", speakerList[1])
+ speaker = re.sub(r"/b", "NPC", speaker)
+ speaker = speaker.replace("\r\n", "")
+
+ # Add Speaker
+ jaString = jaString.replace(speakerList[1], f"[{speaker}]: ")
+
+ # Pass 1 (Grab Data)
+ if setData == False:
+ jaString = jaString.replace("\n", " ")
+ jaString = jaString.replace("\r", "")
+ scenarioList[2].append(jaString)
+
+ # Pass 2 (Set Data)
+ else:
+ translatedText = scenarioList[2][0]
+ scenarioList[2].pop(0)
+
+ # Remove Speaker before wrapping so WIDTH applies to dialogue only
+ speakerMatch = re.search(r"\[(.+?)\]\s?[|:]\s?", translatedText)
+ if speakerMatch:
+ speaker = speakerMatch.group(1).strip()
+ match = re.search(r'(^\[.+?\]\s?[|:]\s?)', translatedText)
+ if match:
+ translatedText = translatedText.replace(match.group(1), "")
+
+ # Wrap text after removing speaker
+ translatedText = dazedwrap.wrapText(translatedText, WIDTH)
+
+ # Redo Old Speaker Format
+ speaker = speaker.replace("Speaker", "\b")
+ if speakerNum:
+ translatedText = f"{speaker}：{speakerNum}\r\n{translatedText}"
+ elif speaker == "NPC":
+ translatedText = f"/b\r\n{translatedText}"
+ else:
+ translatedText = f"{speaker}：\r\n{translatedText}"
+
+ # Add Script String
+ if speakerList[2]:
+ translatedText = f"{speakerList[2]}{translatedText}"
+
+ # Set Data
+ dataList[j].update({"value": dataList[j].get("value").replace(ogString, translatedText)})
+
+ # Grab Options
+ if table["name"] == "クイズ" and OPTIONSFLAG == True:
+ for option in table["data"]:
+ dataList = option["data"]
+
+ # Parse
+ for j in range(len(dataList)):
+ # Name
+ if dataList[j].get("name") == "問":
+ if dataList[j].get("value"):
+ jaString = dataList[j].get("value")
+ # Pass 1 (Grab Data)
+ if setData == False:
+ optionsList[0].append(jaString)
+
+ # Pass 2 (Set Data)
+ else:
+ translatedText = optionsList[0][0]
+ optionsList[0].pop(0)
+ dataList[j].update({"value": dataList[j].get("value").replace(jaString, translatedText)})
+
+ # Description 1
+ if dataList[j].get("name") == "3":
+ if dataList[j].get("value"):
+ jaString = dataList[j].get("value")
+ # Pass 1 (Grab Data)
+ if setData == False:
+ jaString = jaString.replace("\n", " ")
+ jaString = jaString.replace("\r", "")
+ optionsList[1].append(jaString)
+
+ # Pass 2 (Set Data)
+ else:
+ translatedText = optionsList[1][0]
+ optionsList[1].pop(0)
+ translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH)
+ dataList[j].update({"value": dataList[j].get("value").replace(jaString, translatedText)})
+
+ # Description 2
+ if dataList[j].get("name") == "4":
+ if dataList[j].get("value"):
+ jaString = dataList[j].get("value")
+ # Pass 1 (Grab Data)
+ if setData == False:
+ jaString = jaString.replace("\n", " ")
+ jaString = jaString.replace("\r", "")
+ optionsList[2].append(jaString)
+
+ # Pass 2 (Set Data)
+ else:
+ translatedText = optionsList[2][0]
+ optionsList[2].pop(0)
+ translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH)
+ dataList[j].update({"value": dataList[j].get("value").replace(jaString, translatedText)})
+
+ # Grab DB Names
+ if table["name"] == "マップ設定" and DBNAMEFLAG == True:
+ font = None
+ dbName = table["data"]
+ for j in range(len(dbName)):
+ # Pass 1 (Grab Data)
+ if setData == False:
+ if dbName[j].get("name") != "":
+ dbNameList[0].append(dbName[j].get("name"))
+
+ # Pass 2 (Set Data)
+ else:
+ if dbName[j].get("name") != "":
+ dbName[j].update({"name": dbNameList[0][0]})
+ dbNameList[0].pop(0)
+
+ # Grab DB Values
+ if table["name"] == "近況文章" and DBVALUEFLAG == True:
+ font = 16
+ subject = ""
+ for entry in table["data"]:
+ dbValue = entry["data"]
+ for j in range(len(dbValue)):
+ if dbValue[j].get("value"):
+ jaString = dbValue[j].get("value")
+
+ # Speaker
+ match = re.search(r"(^[\\]+f\[\d+\].+?\r\n).+", jaString)
+ if match:
+ subject = match.group(1)
+ jaString = jaString.replace(subject, "")
+
+ # Nuke Textwrap & Font
+ jaString = jaString.replace("\r", "")
+ jaString = jaString.replace("\n", " ")
+ jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString)
+
+ # Pass 1 (Grab Data)
+ if setData == False:
+ # Append
+ dbValueList[0].append(jaString)
+
+ # Pass 2 (Set Data)
+ else:
+ if dbValue[j].get("value"):
+ translatedText = dbValueList[0][0]
+
+ # Textwrap
+ translatedText = dazedwrap.wrapText(translatedText, 30)
+ translatedText = translatedText.replace("\n", f"\r\n\\f[{font}]")
+
+ # Subject
+ if subject:
+ translatedText = f"{subject}{translatedText}"
+
+ # Set
+ dbValue[j].update({"value": translatedText})
+ dbValueList[0].pop(0)
+
+ # Grab Items
+ if table["name"] == "道具" and ITEMFLAG == True:
+ # Write Category
+ if setData:
+ with open("log/translations.txt", "a", encoding="utf-8") as file:
+ file.write(f"\n#Items\n")
+
+ # Begin Translation
+ for item in table["data"]:
+ dataList = item["data"]
+
+ # Parse
+ font = 18
+ for j in range(len(dataList)):
+ # Name
+ if dataList[j].get("name") == "名前":
+ jaString = dataList[j].get("value")
+ if jaString != "":
+ # Pass 1 (Grab Data)
+ if setData == False:
+ if jaString != "":
+ itemList[0].append(jaString)
+
+ # Pass 2 (Set Data)
+ else:
+ # Write to TL File
+ with open("log/translations.txt", "a", encoding="utf-8") as file:
+ file.write(f"{jaString} ({itemList[0][0]})\n")
+
+ dataList[j].update({"value": itemList[0][0]})
+ itemList[0].pop(0)
+
+ # Description
+ if dataList[j].get("name") == "説明":
+ # Pass 1 (Grab Data)
+ if setData == False:
+ if dataList[j].get("value") != "":
+ # Remove Textwrap & Font
+ jaString = dataList[j].get("value")
+ jaString = jaString.replace("\n", " ")
+ jaString = jaString.replace("\r", "")
+ jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString)
+
+ # Append Data
+ itemList[1].append(jaString)
+ # Pass 2 (Set Data)
+ else:
+ if dataList[j].get("value") != "":
+ # Textwrap
+ translatedText = itemList[1][0]
+ translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH)
+
+ # Font
+ if font:
+ translatedText = f"\\f[{font}]{translatedText}"
+
+ # Set Data
+ dataList[j].update({"value": translatedText})
+ itemList[1].pop(0)
+
+ # Log
+ if dataList[j].get("name") == "使用時文章[戦](人名~":
+ # Pass 1 (Grab Data)
+ if setData == False:
+ if dataList[j].get("value") != "":
+ # Remove Textwrap
+ jaString = dataList[j].get("value")
+ jaString = jaString.replace("\n", " ")
+ jaString = jaString.replace("\r", "")
+ jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString)
+
+ # Skill Action
+ if jaString[0] in [
+ "は",
+ "を",
+ "の",
+ "に",
+ "が",
+ ]:
+ jaString = f"Taro{jaString}"
+
+ # Append Data
+ itemList[2].append(jaString)
+ # Pass 2 (Set Data)
+ else:
+ if dataList[j].get("value") != "":
+ # Textwrap
+ translatedText = itemList[2][0]
+ translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH)
+ # translatedText = font + translatedText
+
+ # Remove Taro
+ translatedText = re.sub(r"\bTaro\b", "", translatedText)
+
+ # Set Data
+ dataList[j].update({"value": translatedText})
+ itemList[2].pop(0)
+
+ # Description
+ if dataList[j].get("name") == "使用後文章[移動]":
+ # Pass 1 (Grab Data)
+ if setData == False:
+ if dataList[j].get("value") != "":
+ # Remove Textwrap
+ jaString = dataList[j].get("value")
+ jaString = jaString.replace("\n", " ")
+ jaString = jaString.replace("\r", "")
+ jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString)
+
+ # Append Data
+ itemList[3].append(jaString)
+ # Pass 2 (Set Data)
+ else:
+ if dataList[j].get("value") != "":
+ # Textwrap
+ translatedText = itemList[3][0]
+ translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH)
+
+ # Font
+ if font:
+ translatedText = f"\\f[{font}]{translatedText}"
+
+ # Set Data
+ dataList[j].update({"value": translatedText})
+ itemList[3].pop(0)
+
+ # Grab Armors
+ if table["name"] == "防具" and ARMORFLAG == True:
+ font = "24"
+ for armor in table["data"]:
+ dataList = armor["data"]
+
+ # Parse
+ for j in range(len(dataList)):
+ # Name
+ if dataList[j].get("name") == "名前":
+ # Pass 1 (Grab Data)
+ if setData == False:
+ if dataList[j].get("value") != "":
+ armorList[0].append(dataList[j].get("value"))
+
+ # Pass 2 (Set Data)
+ else:
+ if dataList[j].get("value") != "":
+ dataList[j].update({"value": armorList[0][0]})
+ armorList[0].pop(0)
+
+ # Description
+ if dataList[j].get("name") == "説明":
+ # Pass 1 (Grab Data)
+ if setData == False:
+ if dataList[j].get("value") != "":
+ # Remove Textwrap
+ jaString = dataList[j].get("value")
+ jaString = jaString.replace("\n", " ")
+ jaString = jaString.replace("\r", "")
+ jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString)
+
+ # Append Data
+ armorList[1].append(jaString)
+ # Pass 2 (Set Data)
+ else:
+ if dataList[j].get("value") != "":
+ # Textwrap
+ translatedText = armorList[1][0]
+ translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH)
+
+ # Font
+ if font:
+ translatedText = f"\\f[{font}]{translatedText}"
+
+ # Set Data
+ dataList[j].update({"value": translatedText})
+ armorList[1].pop(0)
+
+ # Grab Enemies
+ if table["name"] == "敵" and ENEMYFLAG == True:
+ for enemy in table["data"]:
+ dataList = enemy["data"]
+
+ # Parse
+ for j in range(len(dataList)):
+ # Name
+ if dataList[j].get("name") == "敵キャラ名":
+ # Pass 1 (Grab Data)
+ if setData == False:
+ if dataList[j].get("value") != "":
+ enemyList[0].append(dataList[j].get("value"))
+
+ # Pass 2 (Set Data)
+ else:
+ if dataList[j].get("value") != "":
+ dataList[j].update({"value": enemyList[0][0]})
+ enemyList[0].pop(0)
+
+ # Description
+ if dataList[j].get("name") == "NULL":
+ # Pass 1 (Grab Data)
+ if setData == False:
+ if dataList[j].get("value") != "":
+ # Remove Textwrap
+ jaString = dataList[j].get("value")
+ jaString = jaString.replace("\n", " ")
+ jaString = jaString.replace("\r", "")
+ jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString)
+
+ # Append Data
+ enemyList[1].append(jaString)
+ # Pass 2 (Set Data)
+ else:
+ if dataList[j].get("value") != "":
+ # Textwrap
+ translatedText = enemyList[1][0]
+ translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH)
+
+ # Font
+ if font:
+ translatedText = f"\\f[{font}]{translatedText}"
+
+ # Set Data
+ dataList[j].update({"value": translatedText})
+ enemyList[1].pop(0)
+
+ # Grab Weapons
+ if table["name"] == "武器" and WEAPONFLAG == True:
+ font = "24"
+ for weapon in table["data"]:
+ dataList = weapon["data"]
+
+ # Parse
+ for j in range(len(dataList)):
+ # Name
+ if dataList[j].get("name") == "名前":
+ # Pass 1 (Grab Data)
+ if setData == False:
+ if dataList[j].get("value") != "":
+ weaponsList[0].append(dataList[j].get("value"))
+
+ # Pass 2 (Set Data)
+ else:
+ if dataList[j].get("value") != "":
+ dataList[j].update({"value": weaponsList[0][0]})
+ weaponsList[0].pop(0)
+
+ # Description
+ if dataList[j].get("name") == "説明":
+ # Pass 1 (Grab Data)
+ if setData == False:
+ if dataList[j].get("value") != "":
+ # Remove Textwrap
+ jaString = dataList[j].get("value")
+ jaString = jaString.replace("\n", " ")
+ jaString = jaString.replace("\r", "")
+ jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString)
+
+ # Append Data
+ weaponsList[1].append(jaString)
+ # Pass 2 (Set Data)
+ else:
+ if dataList[j].get("value") != "":
+ # Textwrap
+ translatedText = weaponsList[1][0]
+ translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH)
+
+ # Font
+ if font:
+ translatedText = f"\\f[{font}]{translatedText}"
+
+ # Set Data
+ dataList[j].update({"value": translatedText})
+ weaponsList[1].pop(0)
+
+ # Grab Skills
+ if table["name"] == "技能" and SKILLFLAG == True:
+ font = "24"
+ for skill in table["data"]:
+ dataList = skill["data"]
+
+ # Parse
+ for j in range(len(dataList)):
+ # Name
+ if dataList[j].get("name") == "名前":
+ # Pass 1 (Grab Data)
+ if setData == False:
+ if dataList[j].get("value") != "":
+ skillList[0].append(dataList[j].get("value"))
+
+ # Pass 2 (Set Data)
+ else:
+ if dataList[j].get("value") != "":
+ dataList[j].update({"value": skillList[0][0]})
+ skillList[0].pop(0)
+
+ # Description
+ if dataList[j].get("name") == "説明":
+ # Pass 1 (Grab Data)
+ if setData == False:
+ if dataList[j].get("value") != "":
+ # Remove Textwrap
+ jaString = dataList[j].get("value")
+ jaString = jaString.replace("\n", " ")
+ jaString = jaString.replace("\r", "")
+ jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString)
+
+ # Append Data
+ skillList[1].append(jaString)
+
+ # Pass 2 (Set Data)
+ else:
+ if dataList[j].get("value") != "":
+ # Textwrap
+ translatedText = skillList[1][0]
+ translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH)
+
+ # Font
+ if font:
+ translatedText = f"\\f[{font}]{translatedText}"
+
+ # Set Data
+ dataList[j].update({"value": translatedText})
+ skillList[1].pop(0)
+
+ # Log
+ if dataList[j].get("name") == "発動時文章":
+ # Pass 1 (Grab Data)
+ if setData == False:
+ if dataList[j].get("value") != "":
+ # Remove Textwrap
+ jaString = dataList[j].get("value")
+ jaString = jaString.replace("\n", " ")
+ jaString = jaString.replace("\r", "")
+ jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString)
+
+ # Skill Action
+ if jaString[0] in [
+ "は",
+ "を",
+ "の",
+ "に",
+ "が",
+ ]:
+ jaString = f"Taro{jaString}"
+
+ # Append Data
+ skillList[2].append(jaString)
+ # Pass 2 (Set Data)
+ else:
+ if dataList[j].get("value") != "":
+ # Textwrap
+ translatedText = skillList[2][0]
+ translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH)
+ translatedText = font + translatedText
+
+ # Remove Taro
+ translatedText = re.sub(r"\bTaro\b", "", translatedText)
+
+ # Set Data
+ dataList[j].update({"value": translatedText})
+ skillList[2].pop(0)
+
+ # Log
+ if dataList[j].get("name") == "使用時文章[戦闘](人名~":
+ # Pass 1 (Grab Data)
+ if setData == False:
+ if dataList[j].get("value") != "":
+ # Remove Textwrap
+ jaString = dataList[j].get("value")
+ jaString = jaString.replace("\n", " ")
+ jaString = jaString.replace("\r", "")
+ jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString)
+
+ # Skill Action
+ if jaString[0] in [
+ "は",
+ "を",
+ "の",
+ "に",
+ "が",
+ ]:
+ jaString = f"Taro{jaString}"
+
+ # Append Data
+ skillList[3].append(jaString)
+ # Pass 2 (Set Data)
+ else:
+ if dataList[j].get("value") != "":
+ # Textwrap
+ translatedText = skillList[3][0]
+ translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH)
+
+ # Remove Taro
+ translatedText = re.sub(r"\bTaro\b", "", translatedText)
+
+ # Font
+ translatedText = font + translatedText
+
+ # Set Data
+ dataList[j].update({"value": translatedText})
+ skillList[3].pop(0)
+
+ # Log
+ if dataList[j].get("name") == "失敗時文章[(対象)～]":
+ # Pass 1 (Grab Data)
+ if setData == False:
+ if dataList[j].get("value") != "":
+ # Remove Textwrap
+ jaString = dataList[j].get("value")
+ jaString = jaString.replace("\n", " ")
+ jaString = jaString.replace("\r", "")
+ jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString)
+
+ # Skill Action
+ if jaString[0] in [
+ "は",
+ "を",
+ "の",
+ "に",
+ "が",
+ ]:
+ jaString = f"Taro{jaString}"
+
+ # Append Data
+ skillList[4].append(jaString)
+ # Pass 2 (Set Data)
+ else:
+ if dataList[j].get("value") != "":
+ # Textwrap
+ translatedText = skillList[4][0]
+ translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH)
+ translatedText = font + translatedText
+
+ # Remove Taro
+ translatedText = re.sub(r"\bTaro\b", "", translatedText)
+
+ # Set Data
+ dataList[j].update({"value": translatedText})
+ skillList[4].pop(0)
+
+ # Grab States
+ if table["name"] == "ステート" and STATEFLAG == True:
+ for state in table["data"]:
+ dataList = state["data"]
+
+ # Parse
+ for j in range(len(dataList)):
+ # Name
+ if dataList[j].get("name") == "名前":
+ # Pass 1 (Grab Data)
+ if setData == False:
+ if dataList[j].get("value") != "":
+ stateList[0].append(dataList[j].get("value"))
+
+ # Pass 2 (Set Data)
+ else:
+ if dataList[j].get("value") != "":
+ dataList[j].update({"value": stateList[0][0]})
+ stateList[0].pop(0)
+
+ # Description
+ if dataList[j].get("name") == "表示名":
+ # Pass 1 (Grab Data)
+ if setData == False:
+ if dataList[j].get("value") != "":
+ # Remove Textwrap
+ jaString = dataList[j].get("value")
+ jaString = jaString.replace("\n", " ")
+ jaString = jaString.replace("\r", "")
+ jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString)
+
+ # Append Data
+ stateList[1].append(jaString)
+ # Pass 2 (Set Data)
+ else:
+ if dataList[j].get("value") != "":
+ # Textwrap
+ translatedText = stateList[1][0]
+ translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH)
+ translatedText = font + translatedText
+
+ # Set Data
+ dataList[j].update({"value": translatedText})
+ stateList[1].pop(0)
+
+ # Log
+ if dataList[j].get("name") == "付与時文章":
+ # Pass 1 (Grab Data)
+ if setData == False:
+ if dataList[j].get("value") != "":
+ # Remove Textwrap
+ jaString = dataList[j].get("value")
+ jaString = jaString.replace("\n", " ")
+ jaString = jaString.replace("\r", "")
+ jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString)
+
+ # State Action
+ if jaString[0] in [
+ "は",
+ "を",
+ "の",
+ "に",
+ "が",
+ ]:
+ jaString = f"Taro{jaString}"
+
+ # Append Data
+ stateList[2].append(jaString)
+ # Pass 2 (Set Data)
+ else:
+ if dataList[j].get("value") != "":
+ # Textwrap
+ translatedText = stateList[2][0]
+ translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH)
+ translatedText = font + translatedText
+
+ # Remove Taro
+ translatedText = re.sub(r"\bTaro\b", "", translatedText)
+
+ # Set Data
+ dataList[j].update({"value": translatedText})
+ stateList[2].pop(0)
+
+ # Log
+ if dataList[j].get("name") == "解除時文章":
+ # Pass 1 (Grab Data)
+ if setData == False:
+ if dataList[j].get("value") != "":
+ # Remove Textwrap
+ jaString = dataList[j].get("value")
+ jaString = jaString.replace("\n", " ")
+ jaString = jaString.replace("\r", "")
+ jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString)
+
+ # State Action
+ if jaString[0] in [
+ "は",
+ "を",
+ "の",
+ "に",
+ "が",
+ ]:
+ jaString = f"Taro{jaString}"
+
+ # Append Data
+ stateList[3].append(jaString)
+ # Pass 2 (Set Data)
+ else:
+ if dataList[j].get("value") != "":
+ # Textwrap
+ translatedText = stateList[3][0]
+ translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH)
+ translatedText = font + translatedText
+
+ # Remove Taro
+ translatedText = re.sub(r"\bTaro\b", "", translatedText)
+
+ # Set Data
+ dataList[j].update({"value": translatedText})
+ stateList[3].pop(0)
+
+ # Log
+ if dataList[j].get("name") == "回復時の文章[(人名)～]":
+ # Pass 1 (Grab Data)
+ if setData == False:
+ if dataList[j].get("value") != "":
+ # Remove Textwrap
+ jaString = dataList[j].get("value")
+ jaString = jaString.replace("\n", " ")
+ jaString = jaString.replace("\r", "")
+ jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString)
+
+ # State Action
+ if jaString[0] in [
+ "は",
+ "を",
+ "の",
+ "に",
+ "が",
+ ]:
+ jaString = f"Taro{jaString}"
+
+ # Append Data
+ stateList[4].append(jaString)
+ # Pass 2 (Set Data)
+ else:
+ if dataList[j].get("value") != "":
+ # Textwrap
+ translatedText = stateList[4][0]
+ translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH)
+ translatedText = font + translatedText
+
+ # Remove Taro
+ translatedText = re.sub(r"\bTaro\b", "", translatedText)
+
+ # Set Data
+ dataList[j].update({"value": translatedText})
+ stateList[4].pop(0)
+ # Log
+ if dataList[j].get("name") == "┣ ｶｳﾝﾀｰ発動文[対象～":
+ # Pass 1 (Grab Data)
+ if setData == False:
+ if dataList[j].get("value") != "":
+ # Remove Textwrap
+ jaString = dataList[j].get("value")
+ jaString = jaString.replace("\n", " ")
+ jaString = jaString.replace("\r", "")
+ jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString)
+
+ # State Action
+ if jaString[0] in [
+ "は",
+ "を",
+ "の",
+ "に",
+ "が",
+ ]:
+ jaString = f"Taro{jaString}"
+
+ # Append Data
+ stateList[5].append(jaString)
+ # Pass 2 (Set Data)
+ else:
+ if dataList[j].get("value") != "":
+ # Textwrap
+ translatedText = stateList[5][0]
+ translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH)
+ translatedText = font + translatedText
+
+ # Remove Taro
+ translatedText = re.sub(r"\bTaro\b", "", translatedText)
+
+ # Set Data
+ dataList[j].update({"value": translatedText})
+ stateList[5].pop(0)
+
+ # Log
+ if dataList[j].get("name") == "尻もち 行動不能 持続3ターン":
+ # Pass 1 (Grab Data)
+ if setData == False:
+ if dataList[j].get("value") != "":
+ # Remove Textwrap
+ jaString = dataList[j].get("value")
+ jaString = jaString.replace("\n", " ")
+ jaString = jaString.replace("\r", "")
+ jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString)
+
+ # State Action
+ if jaString[0] in [
+ "は",
+ "を",
+ "の",
+ "に",
+ "が",
+ ]:
+ jaString = f"Taro{jaString}"
+
+ # Append Data
+ stateList[6].append(jaString)
+ # Pass 2 (Set Data)
+ else:
+ if dataList[j].get("value") != "":
+ # Textwrap
+ translatedText = stateList[6][0]
+ translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH)
+ translatedText = font + translatedText
+
+ # Remove Taro
+ translatedText = re.sub(r"\bTaro\b", "", translatedText)
+
+ # Set Data
+ dataList[j].update({"value": translatedText})
+ stateList[6].pop(0)
+
+ # Log
+ if dataList[j].get("name") == "状態異常の説明":
+ # Pass 1 (Grab Data)
+ if setData == False:
+ if dataList[j].get("value") != "":
+ # Remove Textwrap
+ jaString = dataList[j].get("value")
+ jaString = jaString.replace("\n", " ")
+ jaString = jaString.replace("\r", "")
+ jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString)
+
+ # State Action
+ if jaString[0] in [
+ "は",
+ "を",
+ "の",
+ "に",
+ "が",
+ ]:
+ jaString = f"Taro{jaString}"
+
+ # Append Data
+ stateList[7].append(jaString)
+ # Pass 2 (Set Data)
+ else:
+ if dataList[j].get("value") != "":
+ # Textwrap
+ translatedText = stateList[7][0]
+ translatedText = dazedwrap.wrapText(translatedText, LISTWIDTH)
+ translatedText = font + translatedText
+
+ # Remove Taro
+ translatedText = re.sub(r"\bTaro\b", "", translatedText)
+
+ # Set Data
+ dataList[j].update({"value": translatedText})
+ stateList[7].pop(0)
+
+ # Translation
+ scenarioListTL = [[], [], []]
+ optionsListTL = [[], [], []]
+ npcListTL = [[], [], [], []]
+ itemListTL = [[], [], [], []]
+ stateListTL = [[], [], [], [], [], [], [], []]
+ armorListTL = [[], []]
+ enemyListTL = [[], []]
+ weaponsListTL = [[], [], []]
+ skillListTL = [[], [], [], [], []]
+ dbNameListTL = [[]]
+ dbValueListTL = [[]]
+
+ translate = False
+
+ # NPCs
+ if len(npcList[0]) > 0:
+ # Progress Bar
+ total = 0
+ for itemArray in npcList:
+ total += len(itemArray)
+ pbar.total = total
+ pbar.refresh()
+
+ # Name
+ response = translateAI(
+ npcList[0],
+ "Reply with only the " + LANGUAGE + " translation of the RPG enemy name",
+ True,
+ )
+ nameListTL = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+ # Desc 1
+ response = translateAI(npcList[1], "Reply with only the " + LANGUAGE + " translation")
+ descListTL1 = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+ # Desc 2
+ response = translateAI(npcList[2], "Reply with only the " + LANGUAGE + " translation")
+ descListTL2 = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+ # Desc 3
+ response = translateAI(npcList[3], "Reply with only the " + LANGUAGE + " translation")
+ descListTL3 = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Check Mismatch
+ if (
+ len(nameListTL) != len(npcList[0])
+ or len(descListTL1) != len(npcList[1])
+ or len(descListTL2) != len(npcList[2])
+ or len(descListTL3) != len(npcList[3])
+ ):
+ with LOCK:
+ if filename not in MISMATCH:
+ MISMATCH.append(filename)
+ else:
+ npcListTL = [nameListTL, descListTL1, descListTL2, descListTL3]
+ translate = True
+
+ # SCENARIO
+ if scenarioList[0] or scenarioList[1]:
+ # Progress Bar
+ total = 0
+ for scenarioArray in scenarioList:
+ total += len(scenarioArray)
+ pbar.total = total
+ pbar.refresh()
+
+ # Name
+ response = translateAI(
+ scenarioList[0],
+ "Reply with only the " + LANGUAGE + " translation",
+ True,
+ )
+ nameListTL = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+ # Desc 1
+ response = translateAI(
+ scenarioList[1],
+ "reply with only the gender neutral " + LANGUAGE + " translation of the NPC name",
+ True,
+ )
+ descListTL1 = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+ # Desc 2
+ response = translateAI(
+ scenarioList[2],
+ "reply with only the gender neutral " + LANGUAGE + " translation of the NPC name",
+ True,
+ )
+ descListTL2 = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Check Mismatch
+ if len(nameListTL) != len(scenarioList[0]) or len(descListTL1) != len(scenarioList[1]) or len(descListTL2) != len(scenarioList[2]):
+ with LOCK:
+ if filename not in MISMATCH:
+ MISMATCH.append(filename)
+ else:
+ scenarioListTL = [nameListTL, descListTL1, descListTL2]
+ translate = True
+
+ # OPTIONS
+ if optionsList[0] or optionsList[1]:
+ # Progress Bar
+ total = 0
+ for optionsArray in optionsList:
+ total += len(optionsArray)
+ pbar.total = total
+ pbar.refresh()
+
+ # Name
+ response = translateAI(
+ optionsList[0],
+ "Reply with only the " + LANGUAGE + " translation",
+ True,
+ )
+ nameListTL = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+ # Desc 1
+ response = translateAI(
+ optionsList[1],
+ "reply with only the gender neutral " + LANGUAGE + " translation of the NPC name",
+ True,
+ )
+ descListTL1 = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+ # Desc 2
+ response = translateAI(
+ optionsList[2],
+ "reply with only the gender neutral " + LANGUAGE + " translation of the NPC name",
+ True,
+ )
+ descListTL2 = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Check Mismatch
+ if len(nameListTL) != len(optionsList[0]) or len(descListTL1) != len(optionsList[1]) or len(descListTL2) != len(optionsList[2]):
+ with LOCK:
+ if filename not in MISMATCH:
+ MISMATCH.append(filename)
+ else:
+ optionsListTL = [nameListTL, descListTL1, descListTL2]
+ translate = True
+
+ # ITEMS
+ if len(itemList[0]) > 0 or len(itemList[1]) > 0:
+ # Progress Bar
+ total = 0
+ for itemArray in itemList:
+ total += len(itemArray)
+ pbar.total = total
+ pbar.refresh()
+
+ # Name
+ response = translateAI(itemList[0], "Reply with only the " + LANGUAGE + " translation")
+ nameListTL = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+ # Desc 1
+ response = translateAI(itemList[1], "Reply with only the " + LANGUAGE + " translation")
+ descListTL1 = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+ # Desc 2
+ response = translateAI(itemList[2], "Reply with only the " + LANGUAGE + " translation")
+ descListTL2 = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+ # Desc 3
+ response = translateAI(itemList[3], "Reply with only the " + LANGUAGE + " translation")
+ descListTL3 = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Check Mismatch
+ if (
+ len(nameListTL) != len(itemList[0])
+ or len(descListTL1) != len(itemList[1])
+ or len(descListTL2) != len(itemList[2])
+ or len(descListTL3) != len(itemList[3])
+ ):
+ with LOCK:
+ if filename not in MISMATCH:
+ MISMATCH.append(filename)
+ else:
+ itemListTL = [nameListTL, descListTL1, descListTL2, descListTL3]
+ translate = True
+
+ # Armor
+ if len(armorList[0]) > 0:
+ # Progress Bar
+ total = 0
+ for armorArray in armorList:
+ total += len(armorArray)
+ pbar.total = total
+ pbar.refresh()
+
+ # Name
+ response = translateAI(
+ armorList[0],
+ "Reply with only the " + LANGUAGE + " translation of the NPC name",
+ True,
+ )
+ nameListTL = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+ # Desc 1
+ response = translateAI(armorList[1], "Reply with only the " + LANGUAGE + " translation")
+ descListTL1 = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Check Mismatch
+ if len(nameListTL) != len(armorList[0]) or len(descListTL1) != len(armorList[1]):
+ with LOCK:
+ if filename not in MISMATCH:
+ MISMATCH.append(filename)
+ else:
+ armorListTL = [nameListTL, descListTL1]
+ translate = True
+
+ # Enemies
+ if len(enemyList[0]) > 0:
+ # Progress Bar
+ total = 0
+ for enemyArray in enemyList:
+ total += len(enemyArray)
+ pbar.total = total
+ pbar.refresh()
+
+ # Name
+ response = translateAI(
+ enemyList[0],
+ "Reply with only the " + LANGUAGE + " translation of the enemy NPC name",
+ True,
+ )
+ nameListTL = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+ # Desc 1
+ response = translateAI(enemyList[1], "Reply with only the " + LANGUAGE + " translation")
+ descListTL1 = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Check Mismatch
+ if len(nameListTL) != len(enemyList[0]) or len(descListTL1) != len(enemyList[1]):
+ with LOCK:
+ if filename not in MISMATCH:
+ MISMATCH.append(filename)
+ else:
+ enemyListTL = [nameListTL, descListTL1]
+ translate = True
+
+ # Weapons
+ if len(weaponsList[0]) > 0:
+ # Progress Bar
+ total = 0
+ for weaponsArray in weaponsList:
+ total += len(weaponsArray)
+ pbar.total = total
+ pbar.refresh()
+
+ # Name
+ response = translateAI(
+ weaponsList[0],
+ "Reply with only the " + LANGUAGE + " translation of the RPG weapon name",
+ True,
+ )
+ nameListTL = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+ # Desc 1
+ response = translateAI(weaponsList[1], "")
+ descListTL1 = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+ # Desc 2
+ response = translateAI(weaponsList[2], "")
+ descListTL2 = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Check Mismatch
+ if len(nameListTL) != len(weaponsList[0]) or len(descListTL1) != len(weaponsList[1]) or len(descListTL2) != len(weaponsList[2]):
+ with LOCK:
+ if filename not in MISMATCH:
+ MISMATCH.append(filename)
+ else:
+ weaponsListTL = [nameListTL, descListTL1, descListTL2]
+ translate = True
+
+ # Skills
+ if len(skillList[0]) > 0:
+ # Progress Bar
+ total = 0
+ for skillArray in skillList:
+ total += len(skillArray)
+ pbar.total = total
+ pbar.refresh()
+
+ # Name
+ response = translateAI(
+ skillList[0],
+ "Reply with only the " + LANGUAGE + " translation of the RPG skill name",
+ True,
+ )
+ nameListTL = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Desc
+ response = translateAI(
+ skillList[1],
+ "Reply with only the " + LANGUAGE + " translation of the RPG skill description",
+ True,
+ )
+ descListTL1 = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Log 1
+ response = translateAI(
+ skillList[2],
+ "reply with only the gender neutral "
+ + LANGUAGE
+ + " translation of the action log. Always start the sentence with Taro. For example, Translate 'Taroを倒した！' as 'Taro was defeated!'",
+ True,
+ )
+ descListTL2 = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Log 2
+ response = translateAI(
+ skillList[3],
+ "reply with only the gender neutral "
+ + LANGUAGE
+ + " translation of the action log. Always start the sentence with Taro. For example, Translate 'Taroを倒した！' as 'Taro was defeated!'",
+ True,
+ )
+ descListTL3 = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Log 3
+ response = translateAI(
+ skillList[4],
+ "reply with only the gender neutral "
+ + LANGUAGE
+ + " translation of the action log. Always start the sentence with Taro. For example, Translate 'Taroを倒した！' as 'Taro was defeated!'",
+ True,
+ )
+ descListTL4 = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Check Mismatch
+ if (
+ len(nameListTL) != len(skillList[0])
+ or len(descListTL1) != len(skillList[1])
+ or len(descListTL2) != len(skillList[2])
+ or len(descListTL3) != len(skillList[3])
+ or len(descListTL4) != len(skillList[4])
+ ):
+ with LOCK:
+ if filename not in MISMATCH:
+ MISMATCH.append(filename)
+ else:
+ skillListTL = [nameListTL, descListTL1, descListTL2, descListTL3, descListTL4]
+ translate = True
+
+ # State
+ for list in stateList:
+ if len(list) > 0:
+ # Progress Bar
+ total = 0
+ for stateArray in stateList:
+ total += len(stateArray)
+ pbar.total = total
+ pbar.refresh()
+
+ # Name
+ response = translateAI(
+ stateList[0],
+ f"Reply with the {LANGUAGE} translation of the status effect.",
+ True,
+ )
+ nameListTL = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Desc 1
+ response = translateAI(stateList[1], "")
+ descListTL1 = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Log 1
+ response = translateAI(
+ stateList[2],
+ f"Reply with the {LANGUAGE} translation of the status effect.",
+ True,
+ )
+ descListTL2 = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Log 2
+ response = translateAI(
+ stateList[3],
+ "reply with only the gender neutral "
+ + LANGUAGE
+ + " translation of the action log. Always start the sentence with Taro. For example, Translate 'Taroを倒した！' as 'Taro was defeated!'",
+ True,
+ )
+ descListTL3 = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Log 3
+ response = translateAI(
+ stateList[4],
+ "reply with only the gender neutral "
+ + LANGUAGE
+ + " translation of the action log. Always start the sentence with Taro. For example, Translate 'Taroを倒した！' as 'Taro was defeated!'",
+ True,
+ )
+ descListTL4 = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Log 4
+ response = translateAI(
+ stateList[5],
+ "reply with only the gender neutral "
+ + LANGUAGE
+ + " translation of the action log. Always start the sentence with Taro. For example, Translate 'Taroを倒した！' as 'Taro was defeated!'",
+ True,
+ )
+ descListTL5 = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Log 1
+ response = translateAI(
+ stateList[6],
+ "reply with only the gender neutral "
+ + LANGUAGE
+ + " translation of the action log. Always start the sentence with Taro. For example, Translate 'Taroを倒した！' as 'Taro was defeated!'",
+ True,
+ )
+ descListTL6 = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Log 1
+ response = translateAI(
+ stateList[7],
+ "reply with only the gender neutral "
+ + LANGUAGE
+ + " translation of the action log. Always start the sentence with Taro. For example, Translate 'Taroを倒した！' as 'Taro was defeated!'",
+ True,
+ )
+ descListTL7 = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Check Mismatch
+ if (
+ len(nameListTL) != len(stateList[0])
+ or len(descListTL1) != len(stateList[1])
+ or len(descListTL2) != len(stateList[2])
+ or len(descListTL3) != len(stateList[3])
+ or len(descListTL4) != len(stateList[4])
+ or len(descListTL5) != len(stateList[5])
+ or len(descListTL6) != len(stateList[6])
+ ):
+ with LOCK:
+ if filename not in MISMATCH:
+ MISMATCH.append(filename)
+ else:
+ stateListTL = [
+ nameListTL,
+ descListTL1,
+ descListTL2,
+ descListTL3,
+ descListTL4,
+ descListTL5,
+ descListTL6,
+ descListTL7,
+ ]
+ translate = True
+
+ # DB Names
+ if len(dbNameList[0]) > 0:
+ # Progress Bar
+ total = 0
+ for dbNameArray in dbNameList:
+ total += len(dbNameArray)
+ pbar.total = total
+ pbar.refresh()
+
+ # Name
+ response = translateAI(dbNameList[0], "Reply with only the " + LANGUAGE + " translation")
+ nameListTL = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Check Mismatch
+ if len(nameListTL) != len(dbNameList[0]):
+ with LOCK:
+ if filename not in MISMATCH:
+ MISMATCH.append(filename)
+ else:
+ dbNameListTL = [nameListTL]
+ translate = True
+
+ # DB Values
+ if len(dbValueList[0]) > 0:
+ # Progress Bar
+ total = 0
+ for dbValueArray in dbValueList:
+ total += len(dbValueArray)
+ pbar.total = total
+ pbar.refresh()
+
+ # Name
+ response = translateAI(dbValueList[0], "Reply with only the " + LANGUAGE + " translation")
+ valueListTL = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Check Mismatch
+ if len(valueListTL) != len(dbValueList[0]):
+ with LOCK:
+ if filename not in MISMATCH:
+ MISMATCH.append(filename)
+ else:
+ dbValueListTL = [valueListTL]
+ translate = True
+
+ # Start Pass 2
+ if translate == True:
+ jobList.append(scenarioListTL)
+ jobList.append(npcListTL)
+ jobList.append(itemListTL)
+ jobList.append(stateListTL)
+ jobList.append(armorListTL)
+ jobList.append(enemyListTL)
+ jobList.append(weaponsListTL)
+ jobList.append(skillListTL)
+ jobList.append(optionsListTL)
+ jobList.append(dbNameListTL)
+ jobList.append(dbValueListTL)
+ searchDB(events, pbar, jobList, filename)
+
+ except IndexError as e:
+ traceback.print_exc()
+ raise Exception(str(e) + "Failed to translate: " + initialJAString) from None
+ except Exception as e:
+ traceback.print_exc()
+ raise Exception(str(e) + "Failed to translate: " + initialJAString) from None
+
+ return totalTokens
+
+# Save some money and enter the character before translation
+def getSpeaker(speaker):
+ match speaker:
+ case "ファイン":
+ return ["Fine", [0, 0]]
+ case "":
+ return ["", [0, 0]]
+ case _:
+ # Find Speaker
+ for i in range(len(NAMESLIST)):
+ if speaker == NAMESLIST[i][0]:
+ return [NAMESLIST[i][1], [0, 0]]
+
+ # Translate and Store Speaker
+ response = translateAI(
+ f"{speaker}",
+ "Reply with the " + LANGUAGE + " translation of the NPC name.",
+ False,
+ )
+ response[0] = response[0].title()
+ response[0] = response[0].replace("'S", "'s")
+ response[0] = response[0].replace("Speaker: ", "")
+
+ # Retry if name doesn't translate for some reason
+ if re.search(r"([a-zA-Z？?])", response[0]) == None:
+ response = translateAI(
+ f"{speaker}",
+ "Reply with the " + LANGUAGE + " translation of the NPC name.",
+ False,
+ )
+ response[0] = response[0].title()
+ response[0] = response[0].replace("'S", "'s")
+
+ speakerList = [speaker, response[0]]
+ NAMESLIST.append(speakerList)
+ return response
+ return [speaker, [0, 0]]
+
+def translateAI(text, history, history_ctx=None):
+ """
+ Legacy wrapper function for the new shared translation utility.
+ This maintains compatibility with existing code while using the new shared implementation.
+ """
+ global PBAR, MISMATCH, FILENAME
+
+ # Update config estimate mode based on global ESTIMATE
+ TRANSLATION_CONFIG.estimateMode = bool(ESTIMATE)
+
+ # Call the new shared translation function
+ return sharedtranslateAI(
+ text=text,
+ history=history,
+ config=TRANSLATION_CONFIG,
+ filename=FILENAME,
+ pbar=PBAR,
+ lock=LOCK,
+ mismatchList=MISMATCH
+ )
diff --git a/modules/wolf2.py b/modules/wolf2.py
index 3b6ec64..194f8d2 100644
--- a/modules/wolf2.py
+++ b/modules/wolf2.py
@@ -1,420 +1,422 @@
-# Libraries
-import json
-import io
-import os
-import re
-import util.dazedwrap as dazedwrap
-import threading
-import time
-import traceback
-import tiktoken
-from pathlib import Path
-from colorama import Fore
-from dotenv import load_dotenv
-from retry import retry
-from tqdm import tqdm
-from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost
-import tempfile
-
-# OpenAI initialization centralized in util/translation.py
-
-# Globals
-MODEL = os.getenv("model")
-TIMEOUT = int(os.getenv("timeout"))
-LANGUAGE = os.getenv("language").capitalize()
-PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
-VOCAB = Path("vocab.txt").read_text(encoding="utf-8")
-LOCK = threading.Lock()
-WIDTH = int(os.getenv("width"))
-LISTWIDTH = int(os.getenv("listWidth"))
-NOTEWIDTH = 70
-MAXHISTORY = 10
-ESTIMATE = ""
-TOKENS = [0, 0]
-NAMESLIST = []
-NAMES = False # Output a list of all the character names found
-BRFLAG = False # If the game uses
instead
-FIXTEXTWRAP = True # Overwrites textwrap
-IGNORETLTEXT = False # Ignores all translated text.
-MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
-FILENAME = "" # Current file being processed, used by translateAI wrapper
-
-# tqdm Globals
-BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
-POSITION = 0
-LEAVE = False
-
-# Regex - Need to change this if you want to translate from/to other languages. Default is Japanese Regex
-LANGREGEX = r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+"
-
-# Get pricing configuration based on the model
-PRICING_CONFIG = getPricingConfig(MODEL)
-INPUTAPICOST = PRICING_CONFIG["inputAPICost"]
-OUTPUTAPICOST = PRICING_CONFIG["outputAPICost"]
-BATCHSIZE = PRICING_CONFIG["batchSize"]
-FREQUENCY_PENALTY = PRICING_CONFIG["frequencyPenalty"]
-
-# Initialize Translation Config
-TRANSLATION_CONFIG = TranslationConfig(
- model=MODEL,
- language=LANGUAGE,
- prompt=PROMPT,
- vocab=VOCAB,
- langRegex=LANGREGEX,
- batchSize=BATCHSIZE,
- maxHistory=MAXHISTORY,
- estimateMode=False # Will be set dynamically based on ESTIMATE
-)
-LEAVE = False
-
-def handleWOLF2(filename, estimate):
- global ESTIMATE, FILENAME
- ESTIMATE = estimate
- FILENAME = filename
-
- if ESTIMATE:
- start = time.time()
- translatedData = openFiles(filename)
-
- # Print Result
- end = time.time()
- tqdm.write(getResultString(translatedData, end - start, filename))
- with LOCK:
- TOKENS[0] += translatedData[1][0]
- TOKENS[1] += translatedData[1][1]
-
- # Print Total
- totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
-
- # Print any errors on maps
- if len(MISMATCH) > 0:
- return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
- else:
- return totalString
-
- else:
- try:
- with open("translated/" + filename, "w", encoding="shift_jis", errors="ignore") as outFile:
- start = time.time()
- translatedData = openFiles(filename)
-
- # Print Result
- end = time.time()
- outFile.writelines(translatedData[0])
- tqdm.write(getResultString(translatedData, end - start, filename))
- with LOCK:
- TOKENS[0] += translatedData[1][0]
- TOKENS[1] += translatedData[1][1]
- except Exception:
- traceback.print_exc()
- return "Fail"
-
- return getResultString(["", TOKENS, None], end - start, "TOTAL")
-
-
-def getResultString(translatedData, translationTime, filename):
- # File Print String
- cost = calculateCost(translatedData[1][0], translatedData[1][1], MODEL)
- totalTokenstring = (
- Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
- "[Output: "
- + str(translatedData[1][1])
- + "]" "[Cost: ${:,.4f}".format(cost)
- + "]"
- )
- timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
-
- if translatedData[2] == None:
- # Success
- return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET
-
- else:
- # Fail
- try:
- raise translatedData[2]
- except Exception as e:
- traceback.print_exc()
- errorString = str(e) + Fore.RED
- return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET
-
-
-def openFiles(filename):
- # Use a robust reader to handle cp932/Shift_JIS variants and occasional bad bytes
- def read_file_lines_with_fallback(path: str):
- encodings = ["cp932", "shift_jis", "utf-8-sig", "utf-8"]
- for enc in encodings:
- try:
- with open(path, "r", encoding=enc) as f:
- return f.readlines()
- except UnicodeDecodeError:
- continue
- # Last resort: ignore undecodable bytes under cp932
- with open(path, "rb") as f:
- raw = f.read()
- try:
- text = raw.decode("cp932", errors="ignore")
- except Exception:
- text = raw.decode("latin-1", errors="ignore")
- return text.splitlines(keepends=True)
-
- path = os.path.join("files", filename)
- lines = read_file_lines_with_fallback(path)
-
- # Keep parseWOLF API by wrapping lines in a file-like object
- with io.StringIO("".join(lines)) as readFile:
- translatedData = parseWOLF(readFile, filename)
-
- # Delete lines marked for deletion
- finalData = []
- for line in translatedData[0]:
- if line != "\\d\n":
- finalData.append(line)
- translatedData[0] = finalData
-
- return translatedData
-
-
-def parseWOLF(readFile, filename):
- totalTokens = [0, 0]
-
- # Read File into data
- data = readFile.readlines()
-
- # Create Progress Bar
- with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
- pbar.desc = filename
-
- try:
- result = translateWOLF(data, [], pbar, filename)
- totalTokens[0] += result[0]
- totalTokens[1] += result[1]
- except Exception as e:
- traceback.print_exc()
- return [data, totalTokens, e]
- return [data, totalTokens, None]
-
-
-def save_progress_lines(lines, filename, encoding="shift_jis"):
- """Atomically save current line-based translation progress."""
- try:
- if ESTIMATE:
- return
- os.makedirs("translated", exist_ok=True)
- tmp_fd, tmp_path = tempfile.mkstemp(prefix=f"{filename}.", suffix=".tmp", dir="translated")
- try:
- with os.fdopen(tmp_fd, "w", encoding=encoding, newline="\n", errors="ignore") as tmp_file:
- tmp_file.writelines(lines)
- os.replace(tmp_path, os.path.join("translated", filename))
- finally:
- if os.path.exists(tmp_path):
- try:
- os.remove(tmp_path)
- except OSError:
- pass
- except Exception:
- traceback.print_exc()
-
-
-def saveCheckLines(lines, filename, tokens=None, encoding="shift_jis"):
- try:
- if tokens is not None:
- if not (isinstance(tokens, (list, tuple)) and len(tokens) >= 2 and (tokens[0] or tokens[1])):
- return
- save_progress_lines(lines, filename, encoding=encoding)
- except Exception:
- traceback.print_exc()
-
-
-def translateWOLF(data, translatedList, pbar, filename):
- stringList = []
- currentGroup = []
- tokens = [0, 0]
- speaker = ""
- global LOCK, ESTIMATE, PBAR
- PBAR = pbar
- i = 0
-
- while i < len(data):
- # Speaker
- matchList = re.findall(r"^([^/]*)：", data[i])
- if len(matchList) != 0:
- response = getSpeaker(matchList[0])
- speaker = response[0]
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- data[i] = data[i].replace(matchList[0], f"{speaker}")
- saveCheckLines(data, filename)
- i += 1
- else:
- speaker = ""
-
- # Options
- if "//選択肢" in data[i]:
- i += 1
- choiceList = []
- initialIndex = i
- while "//" in data[i] and "の場合" not in data[i]:
- choiceList.append(re.search(r"\/\/(.*)", data[i]).group(1))
- i += 1
-
- # Translate
- response = translateAI(choiceList, "This will be a dialogue option")
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- choiceListTL = response[0]
-
- # Set Data
- if len(choiceList) == len(choiceListTL):
- # Set Data
- i = initialIndex
- while "//" in data[i] and "の場合" not in data[i]:
- choiceListTL[0] = choiceListTL[0].replace(", ", "、")
- data[i] = f"//{choiceListTL[0]}\n"
- choiceListTL.pop(0)
- i += 1
- saveCheckLines(data, filename)
-
- # Mismatch
- else:
- with LOCK:
- if filename not in MISMATCH:
- MISMATCH.append(filename)
-
- # Lines
- if r"/" not in data[i] and "@" not in data[i] and data[i] != "\n":
- # Pass 1
- if translatedList == []:
- # Grab Consecutive Strings
- currentGroup.append(data[i])
- i += 1
- while i < len(data) and r"/" not in data[i] and "@" not in data[i] and data[i] != "\n":
- currentGroup.append(data[i])
- i += 1
-
- # Join up 401 groups for better translation.
- if len(currentGroup) > 0:
- jaString = "".join(currentGroup)
- currentGroup = []
-
- # Remove any textwrap
- jaString = jaString.replace("\n", " ")
-
- # Add Speaker (If there is one)
- if speaker != "":
- jaString = f"[{speaker}]: {jaString}"
-
- # Add String
- stringList.append(jaString)
- i += 1
-
- # Pass 2
- else:
- # Insert Strings
- while i < len(data) and r"/" not in data[i] and "@" not in data[i] and data[i] != "\n":
- data.pop(i)
-
- # Get Text
- translatedText = translatedList[0]
- translatedList.pop(0)
-
- if len(translatedList) <= 0:
- translatedList = None
-
- # Remove speaker
- matchSpeakerList = re.findall(r"^(\[.+?\]\s?[|:]\s?)\s?", translatedText)
- if len(matchSpeakerList) > 0:
- translatedText = translatedText.replace(matchSpeakerList[0], "")
-
- # Textwrap
- translatedText = dazedwrap.wrapText(translatedText, width=WIDTH)
-
- # Set Data
- data.insert(i, f"{translatedText}\n")
- saveCheckLines(data, filename)
- i += 1
-
- # Nothing relevant. Skip Line.
- else:
- i += 1
-
- # EOF
- if len(stringList) > 0:
- # Set Progress
- pbar.total = len(stringList)
- pbar.refresh()
-
- # Translate
- response = translateAI(stringList, "")
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- translatedList = response[0]
-
- # Set Strings
- if len(stringList) == len(translatedList):
- translateWOLF(data, translatedList, pbar, filename)
-
- # Mismatch
- else:
- with LOCK:
- if filename not in MISMATCH:
- MISMATCH.append(filename)
- return tokens
-
-# Save some money and enter the character before translation
-def getSpeaker(speaker):
- match speaker:
- case "ファイン":
- return ["Fine", [0, 0]]
- case "":
- return ["", [0, 0]]
- case _:
- # Find Speaker
- for i in range(len(NAMESLIST)):
- if speaker == NAMESLIST[i][0]:
- return [NAMESLIST[i][1], [0, 0]]
-
- # Translate and Store Speaker
- response = translateAI(
- f"{speaker}",
- "Reply with the " + LANGUAGE + " translation of the NPC name.",
- False,
- )
- response[0] = response[0].title()
- response[0] = response[0].replace("'S", "'s")
- response[0] = response[0].replace("Speaker: ", "")
-
- # Retry if name doesn't translate for some reason
- if re.search(r"([a-zA-Z？?])", response[0]) == None:
- response = translateAI(
- f"{speaker}",
- "Reply with the " + LANGUAGE + " translation of the NPC name.",
- False,
- )
- response[0] = response[0].title()
- response[0] = response[0].replace("'S", "'s")
-
- speakerList = [speaker, response[0]]
- NAMESLIST.append(speakerList)
- return response
- return [speaker, [0, 0]]
-
-def translateAI(text, history, history_ctx=None):
- """
- Legacy wrapper function for the new shared translation utility.
- This maintains compatibility with existing code while using the new shared implementation.
- """
- global PBAR, MISMATCH, FILENAME
-
- # Update config estimate mode based on global ESTIMATE
- TRANSLATION_CONFIG.estimateMode = bool(ESTIMATE)
-
- # Call the new shared translation function
- return sharedtranslateAI(
- text=text,
- history=history,
- config=TRANSLATION_CONFIG,
- filename=FILENAME,
- pbar=PBAR,
- lock=LOCK,
- mismatchList=MISMATCH
- )
+# Libraries
+import json
+import io
+import os
+import re
+import util.dazedwrap as dazedwrap
+import threading
+import time
+import traceback
+import tiktoken
+from pathlib import Path
+from colorama import Fore
+from dotenv import load_dotenv
+from retry import retry
+from tqdm import tqdm
+from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost
+import tempfile
+
+# OpenAI initialization centralized in util/translation.py
+
+# Globals
+MODEL = os.getenv("model")
+TIMEOUT = int(os.getenv("timeout"))
+LANGUAGE = os.getenv("language").capitalize()
+from util.paths import PROMPT_PATH, VOCAB_PATH
+
+PROMPT = PROMPT_PATH.read_text(encoding="utf-8")
+VOCAB = VOCAB_PATH.read_text(encoding="utf-8")
+LOCK = threading.Lock()
+WIDTH = int(os.getenv("width"))
+LISTWIDTH = int(os.getenv("listWidth"))
+NOTEWIDTH = 70
+MAXHISTORY = 10
+ESTIMATE = ""
+TOKENS = [0, 0]
+NAMESLIST = []
+NAMES = False # Output a list of all the character names found
+BRFLAG = False # If the game uses
instead
+FIXTEXTWRAP = True # Overwrites textwrap
+IGNORETLTEXT = False # Ignores all translated text.
+MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
+FILENAME = "" # Current file being processed, used by translateAI wrapper
+
+# tqdm Globals
+BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
+POSITION = 0
+LEAVE = False
+
+# Regex - Need to change this if you want to translate from/to other languages. Default is Japanese Regex
+LANGREGEX = r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+"
+
+# Get pricing configuration based on the model
+PRICING_CONFIG = getPricingConfig(MODEL)
+INPUTAPICOST = PRICING_CONFIG["inputAPICost"]
+OUTPUTAPICOST = PRICING_CONFIG["outputAPICost"]
+BATCHSIZE = PRICING_CONFIG["batchSize"]
+FREQUENCY_PENALTY = PRICING_CONFIG["frequencyPenalty"]
+
+# Initialize Translation Config
+TRANSLATION_CONFIG = TranslationConfig(
+ model=MODEL,
+ language=LANGUAGE,
+ prompt=PROMPT,
+ vocab=VOCAB,
+ langRegex=LANGREGEX,
+ batchSize=BATCHSIZE,
+ maxHistory=MAXHISTORY,
+ estimateMode=False # Will be set dynamically based on ESTIMATE
+)
+LEAVE = False
+
+def handleWOLF2(filename, estimate):
+ global ESTIMATE, FILENAME
+ ESTIMATE = estimate
+ FILENAME = filename
+
+ if ESTIMATE:
+ start = time.time()
+ translatedData = openFiles(filename)
+
+ # Print Result
+ end = time.time()
+ tqdm.write(getResultString(translatedData, end - start, filename))
+ with LOCK:
+ TOKENS[0] += translatedData[1][0]
+ TOKENS[1] += translatedData[1][1]
+
+ # Print Total
+ totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
+
+ # Print any errors on maps
+ if len(MISMATCH) > 0:
+ return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
+ else:
+ return totalString
+
+ else:
+ try:
+ with open("translated/" + filename, "w", encoding="shift_jis", errors="ignore") as outFile:
+ start = time.time()
+ translatedData = openFiles(filename)
+
+ # Print Result
+ end = time.time()
+ outFile.writelines(translatedData[0])
+ tqdm.write(getResultString(translatedData, end - start, filename))
+ with LOCK:
+ TOKENS[0] += translatedData[1][0]
+ TOKENS[1] += translatedData[1][1]
+ except Exception:
+ traceback.print_exc()
+ return "Fail"
+
+ return getResultString(["", TOKENS, None], end - start, "TOTAL")
+
+
+def getResultString(translatedData, translationTime, filename):
+ # File Print String
+ cost = calculateCost(translatedData[1][0], translatedData[1][1], MODEL)
+ totalTokenstring = (
+ Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
+ "[Output: "
+ + str(translatedData[1][1])
+ + "]" "[Cost: ${:,.4f}".format(cost)
+ + "]"
+ )
+ timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
+
+ if translatedData[2] == None:
+ # Success
+ return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET
+
+ else:
+ # Fail
+ try:
+ raise translatedData[2]
+ except Exception as e:
+ traceback.print_exc()
+ errorString = str(e) + Fore.RED
+ return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET
+
+
+def openFiles(filename):
+ # Use a robust reader to handle cp932/Shift_JIS variants and occasional bad bytes
+ def read_file_lines_with_fallback(path: str):
+ encodings = ["cp932", "shift_jis", "utf-8-sig", "utf-8"]
+ for enc in encodings:
+ try:
+ with open(path, "r", encoding=enc) as f:
+ return f.readlines()
+ except UnicodeDecodeError:
+ continue
+ # Last resort: ignore undecodable bytes under cp932
+ with open(path, "rb") as f:
+ raw = f.read()
+ try:
+ text = raw.decode("cp932", errors="ignore")
+ except Exception:
+ text = raw.decode("latin-1", errors="ignore")
+ return text.splitlines(keepends=True)
+
+ path = os.path.join("files", filename)
+ lines = read_file_lines_with_fallback(path)
+
+ # Keep parseWOLF API by wrapping lines in a file-like object
+ with io.StringIO("".join(lines)) as readFile:
+ translatedData = parseWOLF(readFile, filename)
+
+ # Delete lines marked for deletion
+ finalData = []
+ for line in translatedData[0]:
+ if line != "\\d\n":
+ finalData.append(line)
+ translatedData[0] = finalData
+
+ return translatedData
+
+
+def parseWOLF(readFile, filename):
+ totalTokens = [0, 0]
+
+ # Read File into data
+ data = readFile.readlines()
+
+ # Create Progress Bar
+ with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
+ pbar.desc = filename
+
+ try:
+ result = translateWOLF(data, [], pbar, filename)
+ totalTokens[0] += result[0]
+ totalTokens[1] += result[1]
+ except Exception as e:
+ traceback.print_exc()
+ return [data, totalTokens, e]
+ return [data, totalTokens, None]
+
+
+def save_progress_lines(lines, filename, encoding="shift_jis"):
+ """Atomically save current line-based translation progress."""
+ try:
+ if ESTIMATE:
+ return
+ os.makedirs("translated", exist_ok=True)
+ tmp_fd, tmp_path = tempfile.mkstemp(prefix=f"{filename}.", suffix=".tmp", dir="translated")
+ try:
+ with os.fdopen(tmp_fd, "w", encoding=encoding, newline="\n", errors="ignore") as tmp_file:
+ tmp_file.writelines(lines)
+ os.replace(tmp_path, os.path.join("translated", filename))
+ finally:
+ if os.path.exists(tmp_path):
+ try:
+ os.remove(tmp_path)
+ except OSError:
+ pass
+ except Exception:
+ traceback.print_exc()
+
+
+def saveCheckLines(lines, filename, tokens=None, encoding="shift_jis"):
+ try:
+ if tokens is not None:
+ if not (isinstance(tokens, (list, tuple)) and len(tokens) >= 2 and (tokens[0] or tokens[1])):
+ return
+ save_progress_lines(lines, filename, encoding=encoding)
+ except Exception:
+ traceback.print_exc()
+
+
+def translateWOLF(data, translatedList, pbar, filename):
+ stringList = []
+ currentGroup = []
+ tokens = [0, 0]
+ speaker = ""
+ global LOCK, ESTIMATE, PBAR
+ PBAR = pbar
+ i = 0
+
+ while i < len(data):
+ # Speaker
+ matchList = re.findall(r"^([^/]*)：", data[i])
+ if len(matchList) != 0:
+ response = getSpeaker(matchList[0])
+ speaker = response[0]
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ data[i] = data[i].replace(matchList[0], f"{speaker}")
+ saveCheckLines(data, filename)
+ i += 1
+ else:
+ speaker = ""
+
+ # Options
+ if "//選択肢" in data[i]:
+ i += 1
+ choiceList = []
+ initialIndex = i
+ while "//" in data[i] and "の場合" not in data[i]:
+ choiceList.append(re.search(r"\/\/(.*)", data[i]).group(1))
+ i += 1
+
+ # Translate
+ response = translateAI(choiceList, "This will be a dialogue option")
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ choiceListTL = response[0]
+
+ # Set Data
+ if len(choiceList) == len(choiceListTL):
+ # Set Data
+ i = initialIndex
+ while "//" in data[i] and "の場合" not in data[i]:
+ choiceListTL[0] = choiceListTL[0].replace(", ", "、")
+ data[i] = f"//{choiceListTL[0]}\n"
+ choiceListTL.pop(0)
+ i += 1
+ saveCheckLines(data, filename)
+
+ # Mismatch
+ else:
+ with LOCK:
+ if filename not in MISMATCH:
+ MISMATCH.append(filename)
+
+ # Lines
+ if r"/" not in data[i] and "@" not in data[i] and data[i] != "\n":
+ # Pass 1
+ if translatedList == []:
+ # Grab Consecutive Strings
+ currentGroup.append(data[i])
+ i += 1
+ while i < len(data) and r"/" not in data[i] and "@" not in data[i] and data[i] != "\n":
+ currentGroup.append(data[i])
+ i += 1
+
+ # Join up 401 groups for better translation.
+ if len(currentGroup) > 0:
+ jaString = "".join(currentGroup)
+ currentGroup = []
+
+ # Remove any textwrap
+ jaString = jaString.replace("\n", " ")
+
+ # Add Speaker (If there is one)
+ if speaker != "":
+ jaString = f"[{speaker}]: {jaString}"
+
+ # Add String
+ stringList.append(jaString)
+ i += 1
+
+ # Pass 2
+ else:
+ # Insert Strings
+ while i < len(data) and r"/" not in data[i] and "@" not in data[i] and data[i] != "\n":
+ data.pop(i)
+
+ # Get Text
+ translatedText = translatedList[0]
+ translatedList.pop(0)
+
+ if len(translatedList) <= 0:
+ translatedList = None
+
+ # Remove speaker
+ matchSpeakerList = re.findall(r"^(\[.+?\]\s?[|:]\s?)\s?", translatedText)
+ if len(matchSpeakerList) > 0:
+ translatedText = translatedText.replace(matchSpeakerList[0], "")
+
+ # Textwrap
+ translatedText = dazedwrap.wrapText(translatedText, width=WIDTH)
+
+ # Set Data
+ data.insert(i, f"{translatedText}\n")
+ saveCheckLines(data, filename)
+ i += 1
+
+ # Nothing relevant. Skip Line.
+ else:
+ i += 1
+
+ # EOF
+ if len(stringList) > 0:
+ # Set Progress
+ pbar.total = len(stringList)
+ pbar.refresh()
+
+ # Translate
+ response = translateAI(stringList, "")
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ translatedList = response[0]
+
+ # Set Strings
+ if len(stringList) == len(translatedList):
+ translateWOLF(data, translatedList, pbar, filename)
+
+ # Mismatch
+ else:
+ with LOCK:
+ if filename not in MISMATCH:
+ MISMATCH.append(filename)
+ return tokens
+
+# Save some money and enter the character before translation
+def getSpeaker(speaker):
+ match speaker:
+ case "ファイン":
+ return ["Fine", [0, 0]]
+ case "":
+ return ["", [0, 0]]
+ case _:
+ # Find Speaker
+ for i in range(len(NAMESLIST)):
+ if speaker == NAMESLIST[i][0]:
+ return [NAMESLIST[i][1], [0, 0]]
+
+ # Translate and Store Speaker
+ response = translateAI(
+ f"{speaker}",
+ "Reply with the " + LANGUAGE + " translation of the NPC name.",
+ False,
+ )
+ response[0] = response[0].title()
+ response[0] = response[0].replace("'S", "'s")
+ response[0] = response[0].replace("Speaker: ", "")
+
+ # Retry if name doesn't translate for some reason
+ if re.search(r"([a-zA-Z？?])", response[0]) == None:
+ response = translateAI(
+ f"{speaker}",
+ "Reply with the " + LANGUAGE + " translation of the NPC name.",
+ False,
+ )
+ response[0] = response[0].title()
+ response[0] = response[0].replace("'S", "'s")
+
+ speakerList = [speaker, response[0]]
+ NAMESLIST.append(speakerList)
+ return response
+ return [speaker, [0, 0]]
+
+def translateAI(text, history, history_ctx=None):
+ """
+ Legacy wrapper function for the new shared translation utility.
+ This maintains compatibility with existing code while using the new shared implementation.
+ """
+ global PBAR, MISMATCH, FILENAME
+
+ # Update config estimate mode based on global ESTIMATE
+ TRANSLATION_CONFIG.estimateMode = bool(ESTIMATE)
+
+ # Call the new shared translation function
+ return sharedtranslateAI(
+ text=text,
+ history=history,
+ config=TRANSLATION_CONFIG,
+ filename=FILENAME,
+ pbar=PBAR,
+ lock=LOCK,
+ mismatchList=MISMATCH
+ )
diff --git a/modules/yuris.py b/modules/yuris.py
index 6995515..739b205 100644
--- a/modules/yuris.py
+++ b/modules/yuris.py
@@ -21,8 +21,10 @@ from util.translation import (
MODEL = os.getenv("model")
LANGUAGE = os.getenv("language").capitalize()
-PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
-VOCAB = Path("vocab.txt").read_text(encoding="utf-8")
+from util.paths import PROMPT_PATH, VOCAB_PATH
+
+PROMPT = PROMPT_PATH.read_text(encoding="utf-8")
+VOCAB = VOCAB_PATH.read_text(encoding="utf-8")
LOCK = threading.Lock()
WIDTH = int(os.getenv("width"))
MAXHISTORY = 10
diff --git a/scripts/launch.sh b/scripts/launch.sh
new file mode 100644
index 0000000..ec96110
--- /dev/null
+++ b/scripts/launch.sh
@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+# Launcher for DazedMTLTool.desktop (finds venv, then starts the GUI).
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+cd "$ROOT" || exit 1
+
+PYTHON="python3"
+if [[ -x "$ROOT/.venv/bin/python" ]]; then
+ PYTHON="$ROOT/.venv/bin/python"
+elif [[ -x "$ROOT/venv/bin/python" ]]; then
+ PYTHON="$ROOT/venv/bin/python"
+fi
+
+exec "$PYTHON" "$ROOT/scripts/start_gui.py"
diff --git a/start_gui.py b/scripts/start_gui.py
similarity index 76%
rename from start_gui.py
rename to scripts/start_gui.py
index 276c76b..ef1ee19 100644
--- a/start_gui.py
+++ b/scripts/start_gui.py
@@ -1,72 +1,74 @@
-#!/usr/bin/env python3
-"""
-Launch script for DazedMTLTool GUI
-"""
-
-import sys
-import os
-from pathlib import Path
-
-# Add the project root to Python path
-project_root = Path(__file__).parent
-sys.path.insert(0, str(project_root))
-
-def check_dependencies():
- """Check if required dependencies are installed."""
- missing_deps = []
-
- try:
- import PyQt5
- except ImportError:
- missing_deps.append("PyQt5")
-
- try:
- from dotenv import load_dotenv
- except ImportError:
- missing_deps.append("python-dotenv")
-
- if missing_deps:
- print("Missing dependencies:")
- for dep in missing_deps:
- print(f" - {dep}")
- print("\nPlease install them using:")
- print(" pip install -r requirements_gui.txt")
- return False
-
- return True
-
-def main():
- """Main entry point."""
- print("DazedMTLTool GUI Launcher")
- print("=" * 40)
-
- # Check dependencies
- if not check_dependencies():
- sys.exit(1)
-
- try:
- from util.ace.update_tools import seed_ace_tools
- seed_ace_tools()
- except Exception as exc:
- print(f"Warning: Ace tool setup failed ({exc}). Ace features may be unavailable.")
-
- try:
- from util.forge.update_tools import seed_forge_plugins
- seed_forge_plugins()
- except Exception as exc:
- print(f"Warning: Forge plugin setup failed ({exc}). Playtest Forge may be unavailable.")
-
- # Import and run GUI
- try:
- from gui.main import main as gui_main
- gui_main()
- except ImportError as e:
- print(f"Error importing GUI modules: {e}")
- print("Make sure all GUI files are in the 'gui' directory")
- sys.exit(1)
- except Exception as e:
- print(f"Error starting GUI: {e}")
- sys.exit(1)
-
-if __name__ == "__main__":
- main()
+#!/usr/bin/env python3
+"""Launch script for DazedMTLTool GUI."""
+
+import sys
+from pathlib import Path
+
+PROJECT_ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(PROJECT_ROOT))
+
+
+def check_dependencies():
+ """Check if required dependencies are installed."""
+ missing_deps = []
+
+ try:
+ import PyQt5 # noqa: F401
+ except ImportError:
+ missing_deps.append("PyQt5")
+
+ try:
+ from dotenv import load_dotenv # noqa: F401
+ except ImportError:
+ missing_deps.append("python-dotenv")
+
+ if missing_deps:
+ print("Missing dependencies:")
+ for dep in missing_deps:
+ print(f" - {dep}")
+ print("\nPlease install them using:")
+ print(" pip install -r requirements.txt")
+ return False
+
+ return True
+
+
+def main():
+ """Main entry point."""
+ print("DazedMTLTool GUI Launcher")
+ print("=" * 40)
+
+ from util.paths import ensure_vocab_file, migrate_root_data_files
+
+ migrate_root_data_files()
+ ensure_vocab_file()
+
+ if not check_dependencies():
+ sys.exit(1)
+
+ try:
+ from util.ace.update_tools import seed_ace_tools
+ seed_ace_tools()
+ except Exception as exc:
+ print(f"Warning: Ace tool setup failed ({exc}). Ace features may be unavailable.")
+
+ try:
+ from util.forge.update_tools import seed_forge_plugins
+ seed_forge_plugins()
+ except Exception as exc:
+ print(f"Warning: Forge plugin setup failed ({exc}). Playtest Forge may be unavailable.")
+
+ try:
+ from gui.main import main as gui_main
+ gui_main()
+ except ImportError as e:
+ print(f"Error importing GUI modules: {e}")
+ print("Make sure all GUI files are in the 'gui' directory")
+ sys.exit(1)
+ except Exception as e:
+ print(f"Error starting GUI: {e}")
+ sys.exit(1)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/set_defaults.py b/util/defaults.py
similarity index 90%
rename from set_defaults.py
rename to util/defaults.py
index 7f6c24f..2e92992 100644
--- a/set_defaults.py
+++ b/util/defaults.py
@@ -1,7 +1,9 @@
import re
from copy import deepcopy
-# Canonical defaults used by the GUI and the `set_defaults` script.
+from util.paths import PROJECT_ROOT
+
+# Canonical defaults used by the GUI and the pre-commit hook.
# Stored as booleans for easier consumption by the GUI.
DEFAULTS = {
'FIRSTLINESPEAKERS': False,
@@ -56,4 +58,4 @@ def set_defaults(file_path):
if __name__ == "__main__":
- set_defaults('modules/rpgmakermvmz.py')
+ set_defaults(PROJECT_ROOT / "modules/rpgmakermvmz.py")
diff --git a/util/linux_desktop.py b/util/linux_desktop.py
new file mode 100644
index 0000000..541c439
--- /dev/null
+++ b/util/linux_desktop.py
@@ -0,0 +1,72 @@
+"""Install/update the Linux .desktop entry for taskbar and window icons."""
+
+from __future__ import annotations
+
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+from util.paths import ICON_PATH, PROJECT_ROOT
+
+DESKTOP_ID = "DazedMTLTool"
+LAUNCH_SCRIPT = PROJECT_ROOT / "scripts" / "launch.sh"
+
+
+def installed_desktop_path() -> Path:
+ data_home = os.environ.get("XDG_DATA_HOME", "")
+ if data_home:
+ base = Path(data_home)
+ else:
+ base = Path.home() / ".local" / "share"
+ return base / "applications" / f"{DESKTOP_ID}.desktop"
+
+
+def _desktop_content(root: Path, icon: Path, launch: Path) -> str:
+ return (
+ "[Desktop Entry]\n"
+ "Type=Application\n"
+ f"Name={DESKTOP_ID}\n"
+ "GenericName=Translation Tool\n"
+ "Comment=AI translation tool for visual novels and RPG games\n"
+ f"Exec={launch}\n"
+ f"Icon={icon}\n"
+ f"Path={root}\n"
+ "Terminal=false\n"
+ "Categories=Development;Utility;\n"
+ f"StartupWMClass={DESKTOP_ID}\n"
+ )
+
+
+def ensure_linux_desktop_entry() -> None:
+ """Write ~/.local/share/applications/DazedMTLTool.desktop with absolute paths."""
+ if not sys.platform.startswith("linux"):
+ return
+ if not LAUNCH_SCRIPT.is_file() or not ICON_PATH.is_file():
+ return
+
+ desktop_path = installed_desktop_path()
+ content = _desktop_content(PROJECT_ROOT, ICON_PATH, LAUNCH_SCRIPT)
+
+ try:
+ if desktop_path.is_file() and desktop_path.read_text(encoding="utf-8") == content:
+ return
+ desktop_path.parent.mkdir(parents=True, exist_ok=True)
+ desktop_path.write_text(content, encoding="utf-8")
+ desktop_path.chmod(0o755)
+ except OSError as exc:
+ print(f"Warning: could not install Linux desktop entry ({exc})")
+ return
+
+ _refresh_desktop_database(desktop_path.parent)
+
+
+def _refresh_desktop_database(applications_dir: Path) -> None:
+ for cmd in (
+ ["update-desktop-database", str(applications_dir)],
+ ["kbuildsycoca5", "--noincremental"],
+ ):
+ try:
+ subprocess.run(cmd, check=False, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+ except OSError:
+ pass
diff --git a/util/paths.py b/util/paths.py
new file mode 100644
index 0000000..a8f1648
--- /dev/null
+++ b/util/paths.py
@@ -0,0 +1,45 @@
+"""Canonical project paths (repo root, data files, config)."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+PROJECT_ROOT = Path(__file__).resolve().parent.parent
+DATA_DIR = PROJECT_ROOT / "data"
+VOCAB_PATH = DATA_DIR / "vocab.txt"
+VOCAB_BASE_PATH = DATA_DIR / "vocab_base.txt"
+PROMPT_PATH = DATA_DIR / "prompt.txt"
+LAST_UPDATE_SHA_PATH = DATA_DIR / "last_update_sha.txt"
+ENV_PATH = PROJECT_ROOT / ".env"
+ICON_PATH = PROJECT_ROOT / "assets" / "icon.png"
+
+_ROOT_DATA_FILES = (
+ "vocab.txt",
+ "vocab_base.txt",
+ "prompt.txt",
+ "last_update_sha.txt",
+)
+
+
+def migrate_root_data_files() -> None:
+ """Move legacy root-level data files into data/ on first run."""
+ DATA_DIR.mkdir(parents=True, exist_ok=True)
+ for name in _ROOT_DATA_FILES:
+ src = PROJECT_ROOT / name
+ dst = DATA_DIR / name
+ if src.is_file() and not dst.exists():
+ src.rename(dst)
+
+
+migrate_root_data_files()
+
+
+def ensure_vocab_file() -> None:
+ """Create data/vocab.txt from vocab_base.txt when missing."""
+ migrate_root_data_files()
+ if VOCAB_PATH.is_file():
+ return
+ if VOCAB_BASE_PATH.is_file():
+ VOCAB_PATH.write_text(VOCAB_BASE_PATH.read_text(encoding="utf-8"), encoding="utf-8")
+ else:
+ VOCAB_PATH.write_text("", encoding="utf-8")
diff --git a/util/translation.py b/util/translation.py
index 39d335a..39685f4 100644
--- a/util/translation.py
+++ b/util/translation.py
@@ -1,3060 +1,3061 @@
-"""
-Shared translation utilities for DazedMTLTool.
-Centralized translation function used across all modules.
-"""
-
-import os
-import re
-import json
-import time
-import random
-import unicodedata
-import tiktoken
-import openai
-import anthropic
-import urllib.request
-from openai import APIError, APIConnectionError, RateLimitError, APIStatusError
-import hashlib
-import threading
-from contextlib import contextmanager
-from dotenv import load_dotenv
-from pathlib import Path
-from retry import retry
-
-# Set to True to enable debug logging (token counts, cache costs, etc.)
-DEBUG = True
-_debug_request_log_lock = threading.Lock()
-
-# Set to True to disable Claude prompt caching for baseline cost comparison.
-DISABLE_CACHE = False
-
-# Thread-local per-file token breakdown; read by calculateCost() for Claude.
-_thread_local = threading.local()
-
-# Cross-thread running total of accurate cache-discounted cost (protected by lock).
-_global_accurate_cost = 0.0
-_global_accurate_cost_lock = threading.Lock()
-
-
-def _usage_to_debug_dict(usage):
- """Extract token counts from provider usage objects for request debugging."""
- if not usage:
- return {}
-
- usage_dict = {}
- for field in (
- "prompt_tokens",
- "completion_tokens",
- "input_tokens",
- "output_tokens",
- "cache_read_input_tokens",
- "cache_creation_input_tokens",
- ):
- value = getattr(usage, field, None)
- if value is not None:
- usage_dict[field] = value
-
- extra = getattr(usage, "model_extra", None)
- if isinstance(extra, dict):
- for field in ("cache_read_input_tokens", "cache_creation_input_tokens"):
- value = extra.get(field)
- if value is not None and field not in usage_dict:
- usage_dict[field] = value
-
- return usage_dict
-
-
-def _write_request_debug_log(provider, request_payload, usage):
- """Write the exact SDK payload text and returned token usage."""
- if not DEBUG:
- return
-
- try:
- log_dir = Path("log")
- log_dir.mkdir(parents=True, exist_ok=True)
- usage_dict = _usage_to_debug_dict(usage)
- payload_text = json.dumps(request_payload, indent=2, ensure_ascii=False, default=str)
- usage_text = json.dumps(usage_dict, indent=2, ensure_ascii=False, default=str)
-
- with _debug_request_log_lock:
- with open(log_dir / "request_debug.log", "a", encoding="utf-8") as debug_file:
- debug_file.write("\n=== API Request ===\n")
- debug_file.write(f"Timestamp: {time.strftime('%Y-%m-%d %H:%M:%S')}\n")
- debug_file.write(f"Provider: {provider}\n")
- debug_file.write("Usage:\n")
- debug_file.write(f"{usage_text}\n")
- debug_file.write("Payload:\n")
- debug_file.write(f"{payload_text}\n")
- debug_file.flush()
- except Exception:
- pass
-
-def _normalize_openai_base_url(url: str) -> str:
- """Ensure OpenAI SDK global base_url has a trailing slash."""
- _url = (url or "").strip()
- if _url and not _url.endswith("/"):
- _url += "/"
- return _url
-
-
-def isClaudeModel(model):
- """True when the model name looks like an Anthropic Claude model."""
- return bool(model) and any(x in model.lower() for x in ("claude", "sonnet", "haiku", "opus"))
-
-
-def isClaudeNative(model):
- """True when this model routes to the native Anthropic SDK.
-
- Mirrors the routing check in translateText: the model must look like Claude
- AND the configured API URL must be unset or point at anthropic.com. Any
- other custom URL (e.g. DeepSeek, OpenAI proxy) uses the OpenAI-compatible
- path even for Claude-named models.
- """
- live_api = os.getenv("api", "").strip()
- return isClaudeModel(model) and (not live_api or "anthropic" in live_api.lower())
-
-
-def isMistralAPI():
- """True when requests go to the Mistral platform API (la Plateforme).
-
- Detection is URL-based: the adaptive rate limiter keys off x-ratelimit-*
- headers that only api.mistral.ai sends. Mistral models served through other
- OpenAI-compatible providers (Nvidia, OpenRouter, ...) use the generic path.
- """
- live_api = os.getenv("api", "").strip().lower()
- if "mistral.ai" in live_api:
- return True
- return not live_api and os.getenv("API_PROVIDER", "").strip().lower() == "mistral"
-
-
-# Models that REJECT sampling params (temperature/top_p/top_k) with a 400 —
-# Claude Opus 4.7 and up retired them in favour of adaptive thinking. Matches
-# opus-4-7, opus-4-8, opus-4-10+ and fable, but NOT opus-4-6 or Sonnet/Haiku.
-_NO_SAMPLING_RE = re.compile(r"opus-4-(?:[7-9]\b|[1-9]\d)|fable", re.I)
-
-# Tracks which distinct batch sizes have already been cache-written during this estimate run.
-# Each unique numLines value maps to a distinct output_config schema → one write per size.
-# Persisted to disk so sequential GUI subprocesses share state.
-_estimate_written_sizes: set = set()
-_ESTIMATE_SIZES_FILE = Path("log/estimate_written_sizes.json")
-
-def _load_estimate_written_sizes():
- """Load persisted written-sizes set from disk (for GUI subprocess sharing)."""
- global _estimate_written_sizes
- try:
- if _ESTIMATE_SIZES_FILE.exists():
- with open(_ESTIMATE_SIZES_FILE, "r", encoding="utf-8") as f:
- _estimate_written_sizes = set(json.load(f))
- except Exception:
- _estimate_written_sizes = set()
-
-def _save_estimate_written_sizes():
- """Persist written-sizes set to disk."""
- try:
- _ESTIMATE_SIZES_FILE.parent.mkdir(parents=True, exist_ok=True)
- with open(_ESTIMATE_SIZES_FILE, "w", encoding="utf-8") as f:
- json.dump(list(_estimate_written_sizes), f)
- except Exception:
- pass
-
-def clear_estimate_written_sizes():
- """Reset the written-sizes file at the start of a new estimate run."""
- global _estimate_written_sizes
- _estimate_written_sizes = set()
- try:
- if _ESTIMATE_SIZES_FILE.exists():
- _ESTIMATE_SIZES_FILE.unlink()
- except Exception:
- pass
-
-
-# ===== Placeholder Protection System =====
-# Patterns to protect from translation (sound effects, control codes, etc.)
-PROTECTED_PATTERNS = [
- r'\\SE\[[^\]]+\]', # \SE[sound_effect_name]
- r'\\ME\[[^\]]+\]', # \ME[music_effect_name]
- r'\\BGM\[[^\]]+\]', # \BGM[background_music_name]
- r'\\BGS\[[^\]]+\]', # \BGS[background_sound_name]
- r'_pum\[[^\]]+\]', # _pum[name]
- r'\\VS\[[^\]]+\]', # \VS[name]
-]
-
-def protect_script_codes(text):
- """
- Replace script codes (like \\SE[タイプライター]) with unique placeholders before translation.
- Returns: (protected_text, replacements_dict)
- """
- if not text or not isinstance(text, str):
- return text, {}
-
- # Normalize curly/smart quotes to ASCII equivalents BEFORE building the JSON
- # payload. When these characters appear inside a JSON string value the AI
- # tends to treat them as regular ASCII double-quotes, which makes the value
- # appear empty (e.g. `"スキルを"リセットする` → AI sees empty + stray text).
- # This mirrors the identical normalization already applied to the AI's OUTPUT
- # inside extractTranslation's translation_table.
- quote_norm_table = str.maketrans({
- '\u201C': "'", # " left double quotation mark
- '\u201D': "'", # " right double quotation mark
- '\uFF02': "'", # ＂ fullwidth quotation mark
- '\u2018': "'", # ' left single quotation mark
- '\u2019': "'", # ' right single quotation mark
- '\u201B': "'", # ‛ single high-reversed-9 quotation mark
- '\u02BC': "'", # ʼ modifier letter apostrophe
- '\uFF07': "'", # ＇ fullwidth apostrophe
- })
- text = text.translate(quote_norm_table)
-
- # Convert half-width katakana (U+FF61–U+FF9F) to full-width katakana so the
- # AI recognises them as Japanese text and translates them correctly.
- # NFKC is applied only to matched half-width kana spans to avoid altering
- # intentional fullwidth Latin/digit characters elsewhere in the string.
- text = re.sub(r'[\uFF61-\uFF9F]+', lambda m: unicodedata.normalize('NFKC', m.group(0)), text)
-
- replacements = {}
- protected_text = text
- counter = 0
-
- # Combine all patterns
- combined_pattern = '|'.join(f'({pattern})' for pattern in PROTECTED_PATTERNS)
-
- def replace_match(match):
- nonlocal counter
- original = match.group(0)
- # Create a unique placeholder that won't be translated
- placeholder = f"__PROTECTED_{counter}__"
- replacements[placeholder] = original
- counter += 1
- return placeholder
-
- if combined_pattern:
- protected_text = re.sub(combined_pattern, replace_match, protected_text)
-
- return protected_text, replacements
-
-
-def restore_script_codes(text, replacements):
- """
- Restore protected script codes from placeholders after translation.
- """
- if not text or not replacements:
- return text
-
- if isinstance(text, str):
- result = text
- for placeholder, original in replacements.items():
- result = result.replace(placeholder, original)
- return result
- elif isinstance(text, list):
- return [restore_script_codes(item, replacements) for item in text]
- else:
- return text
-
-
-def validate_placeholders(original_text, translated_text, replacements):
- """
- Validate that all placeholders from the original text appear in the translation.
- Returns: (is_valid, missing_placeholders, extra_placeholders)
- """
- if not replacements:
- return True, [], []
-
- # Get all placeholders
- all_placeholders = set(replacements.keys())
-
- # Count placeholders in original
- original_counts = {}
- for placeholder in all_placeholders:
- if isinstance(original_text, str):
- original_counts[placeholder] = original_text.count(placeholder)
- elif isinstance(original_text, list):
- original_counts[placeholder] = sum(str(item).count(placeholder) for item in original_text)
-
- # Count placeholders in translation
- translated_counts = {}
- for placeholder in all_placeholders:
- if isinstance(translated_text, str):
- translated_counts[placeholder] = translated_text.count(placeholder)
- elif isinstance(translated_text, list):
- translated_counts[placeholder] = sum(str(item).count(placeholder) for item in translated_text)
-
- # Find mismatches
- missing = []
- extra = []
- for placeholder in all_placeholders:
- orig_count = original_counts.get(placeholder, 0)
- trans_count = translated_counts.get(placeholder, 0)
-
- if trans_count < orig_count:
- missing.append(f"{placeholder} (expected {orig_count}, found {trans_count})")
- elif trans_count > orig_count:
- extra.append(f"{placeholder} (expected {orig_count}, found {trans_count})")
-
- is_valid = len(missing) == 0 and len(extra) == 0
- return is_valid, missing, extra
-
-
-def validate_translation_content(original_items, translated_items, langRegex):
- """
- Validate that translated items are not empty or nearly empty.
- Returns: (is_valid, invalid_indices, reasons)
-
- Rules:
- 1. If original has content, translation must not be empty or just whitespace
- 2. If original has Japanese text, translation must not be a single punctuation mark
- 3. Translation should have meaningful content (more than 1-2 characters for substantial originals)
- """
- if not isinstance(original_items, list):
- original_items = [original_items]
- translated_items = [translated_items]
-
- invalid_indices = []
- reasons = []
-
- for i, (orig, trans) in enumerate(zip(original_items, translated_items)):
- orig_str = str(orig).strip()
- trans_str = str(trans).strip()
-
- # Skip if original is empty or placeholder
- if not orig_str or orig_str == "Placeholder Text":
- continue
-
- # Check if original has content that needs translation
- has_source_text = bool(re.search(langRegex, orig_str))
-
- if has_source_text:
- # Original has Japanese text - translation must be substantial
-
- # Check 1: Translation is empty or just whitespace
- if not trans_str:
- invalid_indices.append(i)
- reasons.append(f"Line{i+1}: Empty translation for '{orig_str[:50]}...'")
- continue
-
- # Check 2: Translation is just a single punctuation mark or very short
- # Allow control codes like \\C[27]\\V[45] but not just ":" or ""
- # Use <= 1 so real 2-char words like "No", "Go", "Hi" are not rejected
- if len(trans_str) <= 1 and not re.search(r'\\[A-Z]\[', trans_str):
- # Exception: if original is also very short (like "回" -> "x"), that's ok
- if len(orig_str) > 3:
- invalid_indices.append(i)
- reasons.append(f"Line{i+1}: Translation too short ('{trans_str}') for '{orig_str[:50]}...'")
- continue
-
- # Check 3: For longer originals (>10 chars), translation should be more than just 1-2 chars
- # unless it's a special case like numbers or codes
- if len(orig_str) > 10 and len(trans_str) <= 2:
- # Allow if it contains control codes or is just a replacement word
- if not re.search(r'\\[A-Z]\[', trans_str) and not trans_str.isalnum():
- invalid_indices.append(i)
- reasons.append(f"Line{i+1}: Translation suspiciously short ('{trans_str}') for '{orig_str[:50]}...'")
- continue
-
- # Check 4: Runaway translation - translation is excessively long relative to original
- # Catches cases where the model repeats words endlessly (e.g. "it hurts it hurts it hurts...")
- ratio_limit = max(len(orig_str) * 8, 120)
- if len(orig_str) > 10 and len(trans_str) > ratio_limit:
- invalid_indices.append(i)
- reasons.append(f"Line{i+1}: Runaway translation (output {len(trans_str)} chars vs input {len(orig_str)} chars) for '{orig_str[:50]}...'")
- continue
- # Absolute cap: garbage outputs that are not caught by ratio alone
- if len(trans_str) > 4000 and len(trans_str) > len(orig_str) * 3:
- invalid_indices.append(i)
- reasons.append(f"Line{i+1}: Runaway translation (output {len(trans_str)} chars exceeds cap) for '{orig_str[:50]}...'")
- continue
-
- # Check 5: Same character repeated many times (common API glitch / broken JSON tail)
- if re.search(r"(.)\1{44,}", trans_str):
- invalid_indices.append(i)
- reasons.append(f"Line{i+1}: Excessive character repetition (possible model glitch) in translation")
- continue
-
- is_valid = len(invalid_indices) == 0
- return is_valid, invalid_indices, reasons
-
-# Load .env, strip accidental whitespace, set base URL / org / API key.
-# Gemini/Mistral use their endpoints only when no custom API URL is set.
-load_dotenv()
-api_provider = os.getenv("API_PROVIDER", "openai").lower()
-env_api = os.getenv("api", "").strip()
-if api_provider == "gemini" and not env_api:
- # Use Google Generative Language compatibility endpoint only as fallback.
- openai.base_url = "https://generativelanguage.googleapis.com/v1beta/openai/"
- openai.organization = None
-elif api_provider == "mistral" and not env_api:
- openai.base_url = "https://api.mistral.ai/v1/"
- openai.organization = None
-else:
- if env_api:
- openai.base_url = _normalize_openai_base_url(env_api)
- # Support both 'organization' (gui/.env.example) and legacy 'org' names
- org = os.getenv("organization") or os.getenv("org")
- if org:
- openai.organization = org.strip()
-
-# Always set API key from 'key' env var (trim whitespace)
-openai.api_key = os.getenv("key", "").strip()
-
-# Translation cache management
-CACHE_FILE = Path("log/translation_cache.json")
-CACHE_LOCK_FILE = Path("log/translation_cache.lock")
-CACHE_LOCK = threading.RLock()
-CACHE_PENDING_MARKER = "__translation_pending__"
-CACHE_PENDING_TTL = 600
-CACHE_WAIT_INTERVAL = 0.25
-_cache = None
-
-@contextmanager
-def _translation_cache_file_lock():
- """Cross-process lock for translation_cache.json."""
- CACHE_LOCK_FILE.parent.mkdir(parents=True, exist_ok=True)
- with open(CACHE_LOCK_FILE, "a+b") as lock_file:
- if os.name == "nt":
- import msvcrt
- lock_file.seek(0)
- msvcrt.locking(lock_file.fileno(), msvcrt.LK_LOCK, 1)
- try:
- yield
- finally:
- lock_file.seek(0)
- msvcrt.locking(lock_file.fileno(), msvcrt.LK_UNLCK, 1)
- else:
- import fcntl
- fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX)
- try:
- yield
- finally:
- fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN)
-
-def _read_cache_from_disk():
- """Read the disk cache; return an empty dict if it is unavailable."""
- try:
- if CACHE_FILE.exists():
- with open(CACHE_FILE, "r", encoding="utf-8") as f:
- data = json.load(f)
- return data if isinstance(data, dict) else {}
- except Exception:
- pass
- return {}
-
-def _write_cache_to_disk(cache):
- """Atomically write the cache using a process/thread-unique temp file."""
- CACHE_FILE.parent.mkdir(parents=True, exist_ok=True)
- tmp_file = CACHE_FILE.with_name(
- f"{CACHE_FILE.name}.{os.getpid()}.{threading.get_ident()}.tmp"
- )
- with open(tmp_file, "w", encoding="utf-8") as f:
- json.dump(cache, f, ensure_ascii=False, indent=2)
- tmp_file.replace(CACHE_FILE)
-
-def _is_pending_cache_entry(value):
- return isinstance(value, dict) and value.get(CACHE_PENDING_MARKER) is True
-
-def _is_stale_pending_cache_entry(value):
- if not _is_pending_cache_entry(value):
- return False
- try:
- return time.time() - float(value.get("time", 0)) > CACHE_PENDING_TTL
- except Exception:
- return True
-
-def _is_own_pending_cache_entry(value):
- return (
- _is_pending_cache_entry(value)
- and value.get("pid") == os.getpid()
- and value.get("thread") == threading.get_ident()
- )
-
-def _pending_cache_entry():
- return {
- CACHE_PENDING_MARKER: True,
- "pid": os.getpid(),
- "thread": threading.get_ident(),
- "time": time.time(),
- }
-
-def _merge_translation_caches(base, overlay):
- """Merge cache dictionaries while never replacing a translation with pending."""
- merged = dict(base or {})
- for key, value in (overlay or {}).items():
- existing = merged.get(key)
- if _is_pending_cache_entry(value) and existing is not None:
- if not _is_pending_cache_entry(existing):
- continue
- if not _is_stale_pending_cache_entry(existing):
- continue
- merged[key] = value
- return merged
-
-def clear_cache():
- """Clear the translation cache (called at start of each run)"""
- global _cache
- with CACHE_LOCK:
- _cache = {}
- with _translation_cache_file_lock():
- try:
- if CACHE_FILE.exists():
- CACHE_FILE.unlink()
- except Exception:
- pass
-
-def load_cache():
- """Load the translation cache from disk."""
- global _cache
- with CACHE_LOCK:
- with _translation_cache_file_lock():
- disk_cache = _read_cache_from_disk()
- if _cache:
- disk_cache = _merge_translation_caches(disk_cache, _cache)
- _cache = disk_cache
- return _cache
-
-def save_cache():
- """Save the translation cache to disk, preserving entries from other workers."""
- global _cache
- if _cache is None:
- return
-
- with CACHE_LOCK:
- try:
- with _translation_cache_file_lock():
- disk_cache = _read_cache_from_disk()
- disk_cache = _merge_translation_caches(disk_cache, _cache)
- _cache = disk_cache
- _write_cache_to_disk(_cache)
- except Exception:
- pass
-
-def get_cache_key(payload, language):
- """Generate a cache key for a payload (can be single string or JSON batch)"""
- # Use hash to keep keys short but unique
- payload_str = str(payload) if payload is not None else ""
- combined = f"{payload_str}|{language}"
- return hashlib.md5(combined.encode("utf-8")).hexdigest()
-
-def get_cached_translation(payload, language):
- """Get cached translation if it exists"""
- global _cache
- key = get_cache_key(payload, language)
- while True:
- with CACHE_LOCK:
- with _translation_cache_file_lock():
- cache = _read_cache_from_disk()
- if _cache:
- cache = _merge_translation_caches(cache, _cache)
-
- entry = cache.get(key)
- if (
- entry is None
- or _is_stale_pending_cache_entry(entry)
- or _is_own_pending_cache_entry(entry)
- ):
- cache[key] = _pending_cache_entry()
- _cache = cache
- _write_cache_to_disk(cache)
- return None
-
- _cache = cache
- if not _is_pending_cache_entry(entry):
- return entry
-
- time.sleep(CACHE_WAIT_INTERVAL)
-
-def cache_translation(payload, translation, language):
- """Cache a translation payload and its response"""
- global _cache
- key = get_cache_key(payload, language)
-
- with CACHE_LOCK:
- with _translation_cache_file_lock():
- cache = _read_cache_from_disk()
- if _cache:
- cache = _merge_translation_caches(cache, _cache)
- cache[key] = translation
- _cache = cache
- _write_cache_to_disk(cache)
-
-
-# Variable translation map (code 122 <-> code 111 consistency)
-VAR_MAP_FILE = Path("log/var_translation_map.json")
-VAR_MAP_LOCK = threading.Lock()
-_var_map = None
-
-def clear_var_map():
- """Clear the variable translation map (called at start of each run)"""
- global _var_map
- with VAR_MAP_LOCK:
- _var_map = {}
- try:
- if VAR_MAP_FILE.exists():
- VAR_MAP_FILE.unlink()
- except Exception:
- pass
-
-def _load_var_map():
- """Load the variable translation map from disk (always re-reads to pick up
- entries written by other subprocesses)."""
- global _var_map
- _var_map = {}
- try:
- if VAR_MAP_FILE.exists():
- with open(VAR_MAP_FILE, "r", encoding="utf-8") as f:
- _var_map = json.load(f)
- except Exception:
- _var_map = {}
- return _var_map
-
-def _save_var_map():
- """Save the variable translation map to disk.
- Re-reads the file first and merges so entries from other subprocesses
- are never lost."""
- global _var_map
- if _var_map is None:
- return
- try:
- VAR_MAP_FILE.parent.mkdir(parents=True, exist_ok=True)
- # Re-read the on-disk version and merge our entries on top
- disk_map = {}
- try:
- if VAR_MAP_FILE.exists():
- with open(VAR_MAP_FILE, "r", encoding="utf-8") as f:
- disk_map = json.load(f)
- except Exception:
- disk_map = {}
- disk_map.update(_var_map)
- _var_map = disk_map
- tmp_file = VAR_MAP_FILE.with_suffix(".tmp")
- with open(tmp_file, "w", encoding="utf-8") as f:
- json.dump(_var_map, f, ensure_ascii=False, indent=2)
- tmp_file.replace(VAR_MAP_FILE)
- except Exception:
- pass
-
-def get_var_translation(original):
- """Look up a cached variable translation. Returns the translation or None."""
- with VAR_MAP_LOCK:
- m = _load_var_map()
- return m.get(original)
-
-def set_var_translation(original, translated):
- """Store a variable translation and persist to disk.
- Skips if the translation is identical to the original (untranslated).
- """
- if original == translated:
- return
- with VAR_MAP_LOCK:
- m = _load_var_map()
- m[original] = translated
- _save_var_map()
-
-def set_var_translations_batch(pairs):
- """Store multiple variable translations at once and persist to disk.
- pairs: list of (original, translated) tuples
- Skips pairs where the translation is identical to the original (untranslated).
- """
- with VAR_MAP_LOCK:
- m = _load_var_map()
- for original, translated in pairs:
- if original != translated:
- m[original] = translated
- _save_var_map()
-
-
-# ===== Anthropic Message Batches (50% off all token usage) =====
-# Batch integration by Len — two-pass collect/consume flow; see README Credits.
-# Batch translation is a two-pass flow driven by the batch phase (kept in the
-# BATCH_PHASE env var so GUI subprocesses inherit it):
-# collect: translateAI builds each cache-missed request (byte-identical to a
-# live request, including the cached system block) and queues it
-# instead of calling the API. Text is left untranslated.
-# consume: translateAI feeds the fetched batch responses through the normal
-# validation/restore path; anything missing or invalid falls back
-# to the live API automatically.
-# Between the passes, submit/poll/fetch the queue with runTranslationBatches().
-BATCH_QUEUE_FILE = Path("log/batch_requests.json")
-BATCH_STATE_FILE = Path("log/batch_state.json")
-BATCH_RESULTS_FILE = Path("log/batch_results.json")
-BATCH_LOCK_FILE = Path("log/batch_files.lock")
-BATCH_LOCK = threading.RLock()
-# API limits are 100,000 requests / 256 MB per batch; stay safely under both.
-BATCH_MAX_REQUESTS = 100_000
-BATCH_MAX_BYTES = 200 * 1024 * 1024
-
-_batch_phase = None
-_batch_results = None # in-memory copy of BATCH_RESULTS_FILE (read-only during consume)
-_batch_queue_pending = {} # process-local queue entries not yet flushed to disk
-
-
-def set_batch_phase(phase):
- """Set the batch phase ('collect', 'consume' or None) for this process and
- any subprocesses it spawns (the GUI runs one per file)."""
- global _batch_phase, _batch_results
- _batch_phase = phase if phase in ("collect", "consume") else None
- if _batch_phase:
- os.environ["BATCH_PHASE"] = _batch_phase
- else:
- os.environ.pop("BATCH_PHASE", None)
- _batch_results = None # phase change invalidates the in-memory results copy
-
-
-def get_batch_phase():
- """Current batch phase, or None when batch translation is off."""
- if _batch_phase:
- return _batch_phase
- phase = os.getenv("BATCH_PHASE", "").strip().lower()
- return phase if phase in ("collect", "consume") else None
-
-
-@contextmanager
-def _batch_file_lock():
- """Cross-process lock for the batch queue/state/results files."""
- BATCH_LOCK_FILE.parent.mkdir(parents=True, exist_ok=True)
- with open(BATCH_LOCK_FILE, "a+b") as lock_file:
- if os.name == "nt":
- import msvcrt
- lock_file.seek(0)
- msvcrt.locking(lock_file.fileno(), msvcrt.LK_LOCK, 1)
- try:
- yield
- finally:
- lock_file.seek(0)
- msvcrt.locking(lock_file.fileno(), msvcrt.LK_UNLCK, 1)
- else:
- import fcntl
- fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX)
- try:
- yield
- finally:
- fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN)
-
-
-def _read_batch_file(path):
- """Read a batch JSON file; return an empty dict if it is unavailable."""
- try:
- if path.exists():
- with open(path, "r", encoding="utf-8") as f:
- data = json.load(f)
- return data if isinstance(data, dict) else {}
- except Exception:
- pass
- return {}
-
-
-def _write_batch_file(path, data):
- """Atomically write a batch JSON file (no indent — queues can be large)."""
- path.parent.mkdir(parents=True, exist_ok=True)
- tmp_file = path.with_name(f"{path.name}.{os.getpid()}.{threading.get_ident()}.tmp")
- with open(tmp_file, "w", encoding="utf-8") as f:
- json.dump(data, f, ensure_ascii=False)
- tmp_file.replace(path)
-
-
-def peek_cached_translation(payload, language):
- """Cache lookup that never blocks or writes a pending marker.
-
- The collect pass uses this instead of get_cached_translation so abandoned
- pending markers can't stall the consume pass for CACHE_PENDING_TTL."""
- key = get_cache_key(payload, language)
- with CACHE_LOCK:
- with _translation_cache_file_lock():
- cache = _read_cache_from_disk()
- if _cache:
- cache = _merge_translation_caches(cache, _cache)
- entry = cache.get(key)
- if entry is None or _is_pending_cache_entry(entry):
- return None
- return entry
-
-
-def queue_batch_request(payload, language, params):
- """Queue one Batches API request during the collect pass.
-
- Deduped by the same key the translation cache uses, so identical payloads
- across files are only paid for once. Entries are buffered in memory and
- merged to disk by flush_batch_queue() at the end of each translateAI call.
- """
- key = get_cache_key(payload, language)
- with BATCH_LOCK:
- _batch_queue_pending[key] = {"payload": payload, "language": language, "params": params}
- return key
-
-
-def flush_batch_queue():
- """Merge this process's pending queue entries into the on-disk queue."""
- global _batch_queue_pending
- with BATCH_LOCK:
- if not _batch_queue_pending:
- return
- pending, _batch_queue_pending = _batch_queue_pending, {}
- try:
- with _batch_file_lock():
- queue = _read_batch_file(BATCH_QUEUE_FILE)
- for key, entry in pending.items():
- queue.setdefault(key, entry)
- _write_batch_file(BATCH_QUEUE_FILE, queue)
- except Exception:
- _batch_queue_pending.update(pending) # keep entries for the next flush
-
-
-def take_batch_result(payload, language):
- """Return the fetched batch response dict for a payload, or None.
-
- The results file is loaded once per process — it is written before the
- consume pass starts and never changes mid-consume."""
- global _batch_results
- if _batch_results is None:
- with BATCH_LOCK:
- if _batch_results is None:
- with _batch_file_lock():
- _batch_results = _read_batch_file(BATCH_RESULTS_FILE)
- return _batch_results.get(get_cache_key(payload, language))
-
-
-def pendingBatchRequests():
- """Number of queued batch requests (call after the collect pass)."""
- flush_batch_queue()
- with _batch_file_lock():
- return len(_read_batch_file(BATCH_QUEUE_FILE))
-
-
-def batchRunState():
- """'submitted' when a batch is still in flight, 'fetched' when results are
- waiting to be consumed, else None. Lets an interrupted batch run resume
- instead of re-collecting and paying for a second submission."""
- with _batch_file_lock():
- if _read_batch_file(BATCH_STATE_FILE).get("batches"):
- return "submitted"
- if _read_batch_file(BATCH_RESULTS_FILE):
- return "fetched"
- return None
-
-
-def clearBatchFiles():
- """Remove queue/state/results left over from any previous batch run."""
- global _batch_results, _batch_queue_pending
- with BATCH_LOCK:
- _batch_results = None
- _batch_queue_pending = {}
- with _batch_file_lock():
- for path in (BATCH_QUEUE_FILE, BATCH_STATE_FILE, BATCH_RESULTS_FILE):
- try:
- if path.exists():
- path.unlink()
- except Exception:
- pass
-
-
-def _get_anthropic_client():
- key = os.getenv("key", "").strip()
- if not key:
- raise Exception("Batch translation requires the 'key' env var (see .env).")
- return anthropic.Anthropic(api_key=key)
-
-
-def estimateBatchCost(model=None):
- """Print a cost estimate for the queued batch requests and return it.
-
- Cached-prefix accounting mirrors what Anthropic bills: each distinct cached
- prefix is written once (2x input rate at the 1h TTL) and read by every other
- request that shares it (0.10x); everything is then halved by the batch
- discount. Cache hits inside a batch are best-effort, so the no-cache batch
- figure is the worst-case bound.
- """
- flush_batch_queue()
- with _batch_file_lock():
- queue = _read_batch_file(BATCH_QUEUE_FILE)
- if not queue:
- print("[BATCH] No batch requests queued.", flush=True)
- return None
-
- enc = tiktoken.encoding_for_model("gpt-4")
- prefix_count = {} # cached prefix text -> how many requests reuse it
- prefix_tokens = {} # cached prefix text -> token count
- dynamic_tokens = 0
- output_tokens = 0
- models = set()
- for entry in queue.values():
- params = entry.get("params", {})
- if params.get("model"):
- models.add(params["model"])
- blocks = params.get("system") or []
- cut = 0 # split system blocks at the cache breakpoint (0 = nothing cached)
- for i, b in enumerate(blocks):
- if "cache_control" in b:
- cut = i + 1
- break
- prefix = "".join(b.get("text", "") for b in blocks[:cut])
- dyn = "".join(b.get("text", "") for b in blocks[cut:])
- if prefix:
- prefix_count[prefix] = prefix_count.get(prefix, 0) + 1
- if prefix not in prefix_tokens:
- prefix_tokens[prefix] = len(enc.encode(prefix))
- msg_text = "\n".join(str(m.get("content", "")) for m in params.get("messages", []))
- dynamic_tokens += len(enc.encode(dyn)) + len(enc.encode(msg_text)) + 8
- # Output heuristic mirrors countTokens(): payload tokens x 2.5 covers
- # the echoed JSON scaffold plus EN expansion.
- output_tokens += round(len(enc.encode(str(entry.get("payload", "")))) * 2.5)
-
- est_model = model or next(iter(models), None) or os.getenv("model", "")
- # Use Anthropic's count_tokens for the exact cached-prefix size when possible.
- if prefix_tokens:
- try:
- client = _get_anthropic_client()
- for prefix in list(prefix_tokens.keys()):
- resp = client.messages.count_tokens(
- model=est_model,
- system=[{"type": "text", "text": prefix}],
- messages=[{"role": "user", "content": "x"}],
- )
- prefix_tokens[prefix] = resp.input_tokens
- except Exception:
- pass # tiktoken estimate already in place
-
- cache_write_tok = sum(prefix_tokens.values())
- cache_read_tok = sum(prefix_tokens[p] * (prefix_count[p] - 1) for p in prefix_count)
- raw_input_tok = sum(prefix_tokens[p] * prefix_count[p] for p in prefix_count) + dynamic_tokens
-
- pricing = getPricingConfig(est_model)
- in_rate = pricing["inputAPICost"] / 1_000_000
- out_rate = pricing["outputAPICost"] / 1_000_000
-
- # Batch = 50% off every token, including cache writes (2x, 1h TTL) and reads (0.10x).
- batch_cached = (cache_write_tok * in_rate * 2.00
- + cache_read_tok * in_rate * 0.10
- + dynamic_tokens * in_rate
- + output_tokens * out_rate) * 0.50
- batch_nocache = (raw_input_tok * in_rate + output_tokens * out_rate) * 0.50
- live_cost = raw_input_tok * in_rate + output_tokens * out_rate
-
- n_reread = sum(prefix_count.values()) - len(prefix_count)
- print(f"[BATCH] {len(queue)} requests queued for {est_model}", flush=True)
- print(f"[BATCH] cached prefix: {cache_write_tok:,} tokens (written once, re-read by {n_reread:,} requests)", flush=True)
- print(f"[BATCH] dynamic input: {dynamic_tokens:,} tokens | estimated output: {output_tokens:,} tokens", flush=True)
- print(f"[BATCH] estimated cost: ${batch_cached:.2f} (batch + prompt cache)", flush=True)
- print(f"[BATCH] ${batch_nocache:.2f} (batch, worst case no cache hits)", flush=True)
- print(f"[BATCH] ${live_cost:.2f} (live API, no batch discount)", flush=True)
- return {
- "requests": len(queue),
- "model": est_model,
- "cache_write_tokens": cache_write_tok,
- "cache_read_tokens": cache_read_tok,
- "dynamic_tokens": dynamic_tokens,
- "output_tokens": output_tokens,
- "batch_cached_cost": batch_cached,
- "batch_nocache_cost": batch_nocache,
- "live_cost": live_cost,
- }
-
-
-def submitTranslationBatches():
- """Submit the queued requests to the Anthropic Message Batches API.
-
- Splits at the API limits and saves the custom_id -> cache-key mapping so
- fetchTranslationBatches can route results back. Returns the batch ids."""
- flush_batch_queue()
- with _batch_file_lock():
- queue = _read_batch_file(BATCH_QUEUE_FILE)
- if not queue:
- print("[BATCH] No batch requests queued.", flush=True)
- return []
-
- client = _get_anthropic_client()
- batches = []
- requests, id_map, size = [], {}, 0
-
- def _submit():
- nonlocal requests, id_map, size
- if not requests:
- return
- batch = client.messages.batches.create(requests=requests)
- batches.append({"id": batch.id, "custom_ids": id_map})
- print(f"[BATCH] submitted {batch.id} ({len(requests)} requests)", flush=True)
- requests, id_map, size = [], {}, 0
-
- for i, (key, entry) in enumerate(queue.items()):
- custom_id = f"req-{i:06d}"
- params = entry["params"]
- requests.append({"custom_id": custom_id, "params": params})
- id_map[custom_id] = key
- size += len(json.dumps(params, ensure_ascii=False).encode("utf-8"))
- if len(requests) >= BATCH_MAX_REQUESTS or size >= BATCH_MAX_BYTES:
- _submit()
- _submit()
-
- with BATCH_LOCK:
- with _batch_file_lock():
- _write_batch_file(BATCH_STATE_FILE, {"batches": batches})
- return [b["id"] for b in batches]
-
-
-def checkTranslationBatches():
- """Print the processing status of submitted batches. True when all ended."""
- with _batch_file_lock():
- state = _read_batch_file(BATCH_STATE_FILE)
- if not state.get("batches"):
- print("[BATCH] No submitted batches — submit the queue first.", flush=True)
- return False
- client = _get_anthropic_client()
- all_ended = True
- for info in state["batches"]:
- b = client.messages.batches.retrieve(info["id"])
- counts = getattr(b, "request_counts", None)
- suffix = f" counts: {counts}" if counts else ""
- print(f"[BATCH] {time.strftime('%H:%M:%S')} {b.id}: {b.processing_status}{suffix}", flush=True)
- if b.processing_status != "ended":
- all_ended = False
- return all_ended
-
-
-def fetchTranslationBatches():
- """Download finished batch results into the local results store.
-
- Successes are stored keyed by the payload cache key for the consume pass;
- errored/expired requests are reported and simply fall back to the live API
- during consume. Returns (succeeded, errored) counts."""
- global _batch_results
- with _batch_file_lock():
- state = _read_batch_file(BATCH_STATE_FILE)
- if not state.get("batches"):
- print("[BATCH] No submitted batches — nothing to fetch.", flush=True)
- return 0, 0
- client = _get_anthropic_client()
- results, errored = {}, []
- for info in state["batches"]:
- id_map = info.get("custom_ids", {})
- for r in client.messages.batches.results(info["id"]):
- key = id_map.get(r.custom_id)
- if key is None:
- continue
- res = r.result
- if res.type != "succeeded":
- detail = res.type
- err = getattr(res, "error", None)
- if err is not None:
- inner = getattr(err, "error", err)
- detail = f"{res.type} | {getattr(inner, 'type', '')}: {str(getattr(inner, 'message', '') or err)[:200]}"
- errored.append((r.custom_id, detail))
- continue
- msg = res.message
- text = "".join(getattr(b, "text", "") or "" for b in msg.content)
- u = msg.usage
- cr = getattr(u, "cache_read_input_tokens", 0) or 0
- cw = getattr(u, "cache_creation_input_tokens", 0) or 0
- inp = getattr(u, "input_tokens", 0) or 0
- out = getattr(u, "output_tokens", 0) or 0
- results[key] = {
- "text": text,
- # prompt_tokens matches _AnthropicCompat: total incl. cache fields.
- "prompt_tokens": inp + cr + cw,
- "completion_tokens": out,
- "cache_read_input_tokens": cr,
- "cache_creation_input_tokens": cw,
- }
- with BATCH_LOCK:
- with _batch_file_lock():
- merged = _read_batch_file(BATCH_RESULTS_FILE)
- merged.update(results)
- _write_batch_file(BATCH_RESULTS_FILE, merged)
- # Queue and state are consumed; only the results store remains.
- for path in (BATCH_QUEUE_FILE, BATCH_STATE_FILE):
- try:
- if path.exists():
- path.unlink()
- except Exception:
- pass
- _batch_results = None
- print(f"[BATCH] fetched {len(results)} results ({len(errored)} errored).", flush=True)
- for cid, why in errored[:20]:
- print(f"[BATCH] ! {cid}: {why}", flush=True)
- if len(errored) > 20:
- print(f"[BATCH] ... ({len(errored) - 20} more)", flush=True)
- return len(results), len(errored)
-
-
-def runTranslationBatches(poll=60):
- """Submit the queue (unless already submitted), poll to completion, fetch."""
- with _batch_file_lock():
- state = _read_batch_file(BATCH_STATE_FILE)
- if not state.get("batches"):
- if not submitTranslationBatches():
- return 0, 0
- print(f"[BATCH] polling every {poll}s (Ctrl-C is safe — resume later with fetchTranslationBatches)...", flush=True)
- while not checkTranslationBatches():
- time.sleep(poll)
- return fetchTranslationBatches()
-
-
-# ===== Mistral (la Plateforme) adaptive rate limiting =====
-# Mistral enforces independent request and token limits, both PER-MODEL.
-# Verified live against api.mistral.ai (the response headers are authoritative):
-# * x-ratelimit-limit-req-minute — requests/minute. PER-MODEL and varies a
-# lot: mistral-medium=25, mistral-small=50, codestral=125, ministral-3b=750.
-# Mistral's dashboard shows this /60 as "RPS" (25/min = 0.42 "RPS"), so the
-# effective rate spans well below AND above 1 req/sec. There is NO
-# per-second header; we divide req-minute by 60 to get the pacing interval.
-# * x-ratelimit-limit-tokens-minute — input+output throughput, a true minute
-# window, also per-model (e.g. 50k–1.3M).
-# * Tokens per month — overall cap (not paced here; surfaces as a 429).
-# The limiter paces requests >= 1/RPS apart (so a burst of file threads can't
-# overrun the per-minute request budget) AND charges them against a
-# minute-windowed token budget. It starts from a conservative seed, then the
-# live headers on the first response pin req_per_sec / tok_per_min to the exact
-# per-model values. One limiter is shared across all file threads (this tool
-# runs one model per session). Override the seeds with
-# mistralReqPerSec / mistralTokPerMin / mistralTokenHeadroom.
-_mistral_limiter = None
-_mistral_limiter_lock = threading.Lock()
-
-
-def _estimate_mistral_tokens(text):
- # JP ~1 token/char with Mistral tokenizers; EN ~1 per 3.5 chars. Be pessimistic.
- return int(len(str(text)) * 1.1) + 8
-
-
-def _header_int(headers, *names):
- """First present header among names parsed as int, else None."""
- for n in names:
- v = headers.get(n)
- if v is not None and str(v).strip() != "":
- try:
- return int(float(v))
- except (TypeError, ValueError):
- pass
- return None
-
-
-def _header_float(headers, *names):
- """First present header among names parsed as float, else None.
-
- Used for the RPS limit header, which per-model is often FRACTIONAL
- (e.g. 0.08, 0.42, 0.83) — parsing it as int would floor those to 0."""
- for n in names:
- v = headers.get(n)
- if v is not None and str(v).strip() != "":
- try:
- return float(v)
- except (TypeError, ValueError):
- pass
- return None
-
-
-class AdaptiveLimiter:
- """Paces requests off live ratelimit headers.
-
- Two dimensions, enforced together by acquire():
- * requests: spaced >= min_interval (1/RPS) apart — Mistral's request cap
- is per-second, so this prevents bursts from tripping it.
- * tokens: a minute-windowed budget (input+output), with a headroom margin.
- """
-
- def __init__(self, req_per_sec, tok_per_min, headroom):
- self.lock = threading.Lock()
- self.req_per_sec = max(0.001, float(req_per_sec))
- self.tok_per_min = tok_per_min
- self.headroom = headroom # margin left under the TPM cap
- self.min_interval = 1.0 / self.req_per_sec
- self.next_request_at = time.monotonic()
- self.tokens_remaining = tok_per_min
- self.window_reset = time.monotonic() + 60
-
- def acquire(self, est_tokens):
- """Block until the request clears both the RPS pace and the TPM budget."""
- while True:
- with self.lock:
- now = time.monotonic()
- if now >= self.window_reset:
- self.tokens_remaining = self.tok_per_min
- self.window_reset = now + 60
- # token gate
- if self.tokens_remaining - est_tokens <= self.headroom:
- sleep_for = max(0.05, self.window_reset - now)
- # request-pace gate
- elif now < self.next_request_at:
- sleep_for = self.next_request_at - now
- else:
- # both gates clear — reserve this slot
- self.next_request_at = max(now, self.next_request_at) + self.min_interval
- self.tokens_remaining -= est_tokens
- return
- time.sleep(min(sleep_for, 5))
-
- def update(self, headers):
- """Sync budgets from the live ratelimit headers (authoritative).
-
- Accepts both the per-second request headers and (for forward/back compat)
- the older per-minute request header names; the token headers are minute."""
- if not headers:
- return
- with self.lock:
- # The authoritative request limit is x-ratelimit-limit-req-MINUTE
- # (verified live: mistral-medium=25, ministral-3b=750, codestral=125;
- # Mistral's dashboard "RPS" is just this number / 60). There is no
- # per-second header. Convert to RPS for the pacing interval. A
- # per-second header is still accepted first in case Mistral adds one.
- limit_rps = _header_float(headers, "x-ratelimit-limit-req-second",
- "x-ratelimit-limit-requests-second")
- if limit_rps is None:
- limit_rpm = _header_float(headers, "x-ratelimit-limit-req-minute",
- "x-ratelimit-limit-requests-minute")
- if limit_rpm is not None:
- limit_rps = limit_rpm / 60.0
- if limit_rps and limit_rps > 0:
- self.req_per_sec = limit_rps
- self.min_interval = 1.0 / self.req_per_sec
- limit_tok = _header_int(headers, "x-ratelimit-limit-tokens-minute",
- "x-ratelimit-limit-tokens")
- if limit_tok and limit_tok > 0:
- self.tok_per_min = limit_tok
- rem_tok = _header_int(headers, "x-ratelimit-remaining-tokens-minute",
- "x-ratelimit-remaining-tokens")
- if rem_tok is not None:
- self.tokens_remaining = rem_tok
- # Remaining requests this minute -> also cap the pace so a fresh
- # limiter that joins mid-window doesn't burst the leftover budget.
- rem_req = _header_int(headers, "x-ratelimit-remaining-req-minute",
- "x-ratelimit-remaining-requests-minute")
- if rem_req is not None and rem_req <= 0:
- # budget already spent this minute — hold off ~ a minute
- self.next_request_at = max(self.next_request_at, time.monotonic() + 60.0)
-
-
-def _get_mistral_limiter():
- global _mistral_limiter
- with _mistral_limiter_lock:
- if _mistral_limiter is None:
- # mistralReqPerMin kept as a deprecated alias: a per-minute number is
- # converted to RPS so old .env files keep pacing sanely.
- rps_env = os.getenv("mistralReqPerSec")
- if rps_env:
- req_per_sec = float(rps_env)
- elif os.getenv("mistralReqPerMin"):
- req_per_sec = max(0.05, float(os.getenv("mistralReqPerMin")) / 60.0)
- else:
- # Per-model RPS varies and is often well under 1 (free-tier
- # mistral-medium ~0.83, magistral ~0.08). Start conservative so
- # the FIRST request doesn't over-burst; the live header from that
- # first response then corrects req_per_sec to the model's exact
- # value (which is why few large batches > many tiny ones).
- req_per_sec = 0.5
- _mistral_limiter = AdaptiveLimiter(
- req_per_sec,
- int(os.getenv("mistralTokPerMin", "50000") or 50000),
- int(os.getenv("mistralTokenHeadroom", "4000") or 4000),
- )
- return _mistral_limiter
-
-
-def callMistral(params, retries=6):
- """Call the Mistral chat completions endpoint with adaptive pacing.
-
- Acquires from the shared limiter before each attempt, syncs budgets from
- the live x-ratelimit headers after each response, honours Retry-After on
- 429, backs off with jitter on 5xx/network errors, and downgrades
- json_schema -> json_object for models without structured-output support.
- """
- limiter = _get_mistral_limiter()
- est = sum(_estimate_mistral_tokens(m.get("content", "")) for m in params.get("messages", []))
- est += params.get("max_tokens", 0) // 2
- last_error = None
- for attempt in range(retries + 1):
- limiter.acquire(est)
- try:
- raw = openai.chat.completions.with_raw_response.create(**params)
- response = raw.parse()
- limiter.update(raw.headers)
- return response
- except RateLimitError as e:
- last_error = e
- resp = getattr(e, "response", None)
- limiter.update(getattr(resp, "headers", None))
- retry_after = None
- try:
- retry_after = float(resp.headers.get("retry-after"))
- except (AttributeError, TypeError, ValueError):
- pass
- time.sleep(retry_after if retry_after is not None else min(60, 2 ** attempt + random.random() * 2))
- except APIStatusError as e:
- last_error = e
- # Mistral rejects unsupported params with 400/422 — downgrade the
- # structured-output format once and retry immediately.
- if e.status_code in (400, 422) and (params.get("response_format") or {}).get("type") == "json_schema":
- params = dict(params)
- params["response_format"] = {"type": "json_object"}
- continue
- if e.status_code >= 500 and attempt < retries:
- time.sleep(min(45, 2 ** attempt + random.random()))
- continue
- raise Exception(f"Mistral API error ({e.status_code}): {e}")
- except APIConnectionError as e:
- last_error = e
- if attempt < retries:
- time.sleep(min(45, 2 ** attempt + random.random()))
- continue
- raise Exception(f"Mistral connection error: {e}")
- raise Exception(f"Mistral API failed after {retries + 1} attempts: {last_error}")
-
-
-class TranslationConfig:
- """Configuration class to hold all translation settings"""
-
- def __init__(self,
- model=None,
- language=None,
- prompt=None,
- vocab=None,
- langRegex=None,
- batchSize=None,
- maxHistory=10,
- estimateMode=False,
- logFilePath="log/translationHistory.txt",
- mismatchLogPath="log/mismatchHistory.txt"):
-
- # Load from environment if not provided
- self.model = model or os.getenv("model")
- self.language = (language or os.getenv("language", "english")).capitalize()
-
- # Load prompt and vocab files if not provided
- if prompt is None:
- try:
- self.prompt = Path("prompt.txt").read_text(encoding="utf-8")
- except FileNotFoundError:
- self.prompt = ""
- else:
- self.prompt = prompt
-
- if vocab is None:
- try:
- self.vocab = Path("vocab.txt").read_text(encoding="utf-8")
- except FileNotFoundError:
- self.vocab = ""
- else:
- self.vocab = vocab
-
- # Set language regex (default is Japanese)
- self.langRegex = langRegex or r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+"
-
- # Set batch size — derive from pricing config unless explicitly supplied
- if batchSize is None:
- self.batchSize = getPricingConfig(self.model)["batchSize"]
- else:
- self.batchSize = batchSize
-
- self.maxHistory = maxHistory
- self.estimateMode = estimateMode
- self.logFilePath = logFilePath
- self.mismatchLogPath = mismatchLogPath
-
-
-_LITELLM_PRICING_URL = (
- "https://raw.githubusercontent.com/BerriAI/litellm/main"
- "/model_prices_and_context_window.json"
-)
-_PRICING_CACHE_FILE = Path("log/litellm_pricing.json")
-_PRICING_CACHE_TTL = 86_400 # 24 hours
-_pricing_db: dict | None = None
-_pricing_db_fetched_at: float = 0.0
-_pricing_db_lock = threading.Lock()
-_pricing_fetch_warned: bool = False # print fetch-failure warning at most once per session
-
-
-def _load_litellm_pricing() -> dict | None:
- """Return the LiteLLM pricing DB, using a 24-hour disk cache."""
- global _pricing_db, _pricing_db_fetched_at, _pricing_fetch_warned
-
- with _pricing_db_lock:
- now = time.time()
-
- # In-memory cache still fresh
- if _pricing_db is not None and (now - _pricing_db_fetched_at) < _PRICING_CACHE_TTL:
- return _pricing_db
-
- # Try disk cache
- if _PRICING_CACHE_FILE.exists():
- try:
- disk = json.loads(_PRICING_CACHE_FILE.read_text(encoding="utf-8"))
- if (now - disk.get("fetched_at", 0)) < _PRICING_CACHE_TTL:
- _pricing_db = disk["prices"]
- _pricing_db_fetched_at = disk["fetched_at"]
- return _pricing_db
- except Exception:
- pass
-
- # Fetch from GitHub
- try:
- with urllib.request.urlopen(_LITELLM_PRICING_URL, timeout=5) as resp:
- data = json.loads(resp.read().decode("utf-8"))
- _pricing_db = data
- _pricing_db_fetched_at = now
- _pricing_fetch_warned = False # reset if a later fetch succeeds
- try:
- _PRICING_CACHE_FILE.parent.mkdir(parents=True, exist_ok=True)
- _PRICING_CACHE_FILE.write_text(
- json.dumps({"fetched_at": now, "prices": data}),
- encoding="utf-8",
- )
- except Exception:
- pass
- return _pricing_db
- except Exception as fetch_err:
- # No internet / GitHub unreachable — warn once, then fall back
- if not _pricing_fetch_warned:
- _pricing_fetch_warned = True
- print(
- f"[PRICING] Warning: Could not fetch live model pricing "
- f"({fetch_err}). Cost estimates may be inaccurate — "
- f"using built-in fallback prices.",
- flush=True,
- )
- # Use stale disk cache if available
- if _pricing_db is not None:
- return _pricing_db
- try:
- disk = json.loads(_PRICING_CACHE_FILE.read_text(encoding="utf-8"))
- _pricing_db = disk["prices"]
- return _pricing_db
- except Exception:
- return None
-
-
-def _lookup_model_price(model: str):
- """Look up (input_per_1M, output_per_1M) from the LiteLLM pricing DB.
-
- Returns a (float, float) tuple or None if not found.
- Matching priority:
- 1. Exact key match
- 2. Exact match on the model portion after a provider prefix (e.g. "deepseek/deepseek-chat")
- 3. The user's model name is a prefix of a DB key (handles dated suffixes like -20241022)
- 4. A DB key model-part is a prefix of the user's model name
- """
- db = _load_litellm_pricing()
- if not db:
- return None
-
- model_lower = model.lower()
-
- def _extract(entry):
- inp = entry.get("input_cost_per_token")
- out = entry.get("output_cost_per_token")
- if inp is not None and out is not None:
- return round(inp * 1_000_000, 6), round(out * 1_000_000, 6)
- return None
-
- # Pass 1: exact key
- if model_lower in db:
- result = _extract(db[model_lower])
- if result:
- return result
-
- # Build a lookup of (stripped_key → original_key) for passes 2-4
- stripped: list[tuple[str, str]] = []
- for key in db:
- stripped.append((key.split("/")[-1].lower(), key))
-
- # Pass 2: exact match on stripped key
- for skey, orig in stripped:
- if skey == model_lower:
- result = _extract(db[orig])
- if result:
- return result
-
- # Pass 3: model name is a prefix of the DB key (e.g. "claude-3-5-sonnet" matches
- # "claude-3-5-sonnet-20241022")
- candidates = [(skey, orig) for skey, orig in stripped if skey.startswith(model_lower)]
- if candidates:
- # Prefer the shortest (most generic) key
- skey, orig = min(candidates, key=lambda x: len(x[0]))
- result = _extract(db[orig])
- if result:
- return result
-
- # Pass 4: DB key is a prefix of the model name (e.g. "gemini-2.0-flash" matches
- # "gemini-2.0-flash-exp")
- candidates = [(skey, orig) for skey, orig in stripped if model_lower.startswith(skey)]
- if candidates:
- skey, orig = max(candidates, key=lambda x: len(x[0])) # longest = most specific
- result = _extract(db[orig])
- if result:
- return result
-
- return None
-
-
-def getPricingConfig(model):
- """
- Get pricing configuration for a given model.
-
- Args:
- model: The model name string
-
- Returns:
- dict: Dictionary containing inputAPICost, outputAPICost, batchSize, and frequencyPenalty
- """
- # Try to resolve pricing from the LiteLLM community pricing DB first.
- # This keeps costs accurate as providers update their prices without requiring
- # a code change. Falls back to the hardcoded table below on failure.
- live_price = _lookup_model_price(model)
- if live_price:
- inp, out = live_price
- # Preserve model-specific batch / penalty overrides from the hardcoded table
- # by still running through the if-chain but replacing the cost fields.
- _live_override = {"inputAPICost": inp, "outputAPICost": out}
- else:
- _live_override = None
-
- # Hardcoded fallback table — used for batchSize / frequencyPenalty tuning and
- # as a cost fallback when the LiteLLM DB is unavailable.
- # Batch Size: GPT-3.5 struggles past 15 lines; GPT-4 struggles past 50.
- # If you get a MISMATCH LENGTH error, lower the batch size.
- if "gpt-3.5" in model:
- cfg = {"inputAPICost": 3.00, "outputAPICost": 5.00, "batchSize": 10, "frequencyPenalty": 0.2}
- elif "gpt-4.1-mini" in model:
- cfg = {"inputAPICost": 0.40, "outputAPICost": 1.60, "batchSize": 30, "frequencyPenalty": 0.05}
- elif "gpt-4.1" in model:
- cfg = {"inputAPICost": 2.00, "outputAPICost": 8.00, "batchSize": 30, "frequencyPenalty": 0.05}
- elif "gpt-5" in model:
- cfg = {"inputAPICost": 1.25, "outputAPICost": 10.00, "batchSize": 30, "frequencyPenalty": 0.05}
- elif "deepseek" in model:
- cfg = {"inputAPICost": 0.27, "outputAPICost": 1.10, "batchSize": 30, "frequencyPenalty": 0.05}
- # Mistral — system prompt is resent per request (no prompt caching), so
- # throughput/cost favor larger batches. Live LiteLLM pricing overrides these.
- elif "mistral-large" in model or "pixtral-large" in model:
- cfg = {"inputAPICost": 2.00, "outputAPICost": 6.00, "batchSize": 40, "frequencyPenalty": 0.0}
- elif "magistral-medium" in model:
- cfg = {"inputAPICost": 2.00, "outputAPICost": 5.00, "batchSize": 40, "frequencyPenalty": 0.0}
- elif "mistral-medium-3.5" in model or "mistral-medium-3-5" in model or "mistral-medium-26" in model:
- # Medium 3.5 (v26.04) — also matches dated 26xx ids
- cfg = {"inputAPICost": 1.50, "outputAPICost": 7.50, "batchSize": 40, "frequencyPenalty": 0.0}
- elif "mistral-medium" in model:
- # Medium 3 / 3.1 (what -latest still points at)
- cfg = {"inputAPICost": 0.40, "outputAPICost": 2.00, "batchSize": 40, "frequencyPenalty": 0.0}
- elif "magistral-small" in model:
- cfg = {"inputAPICost": 0.50, "outputAPICost": 1.50, "batchSize": 40, "frequencyPenalty": 0.0}
- elif "mistral-small" in model:
- cfg = {"inputAPICost": 0.10, "outputAPICost": 0.30, "batchSize": 40, "frequencyPenalty": 0.0}
- elif "ministral" in model or "open-mistral" in model or "mistral-nemo" in model:
- cfg = {"inputAPICost": 0.10, "outputAPICost": 0.10, "batchSize": 40, "frequencyPenalty": 0.0}
- elif "codestral" in model:
- cfg = {"inputAPICost": 0.30, "outputAPICost": 0.90, "batchSize": 40, "frequencyPenalty": 0.0}
- elif "mistral" in model or "pixtral" in model:
- cfg = {"inputAPICost": 2.00, "outputAPICost": 6.00, "batchSize": 40, "frequencyPenalty": 0.0}
- elif "claude-opus-4-5" in model or "claude-opus-4-6" in model:
- cfg = {"inputAPICost": 5.00, "outputAPICost": 25.00, "batchSize": 30, "frequencyPenalty": 0.05}
- elif "claude-opus" in model or model == "claude-3-opus":
- # Opus 4, 4.1, 3 — $15/$75
- cfg = {"inputAPICost": 15.00, "outputAPICost": 75.00, "batchSize": 30, "frequencyPenalty": 0.05}
- elif "claude-haiku-4-5" in model or "claude-haiku-4-6" in model:
- cfg = {"inputAPICost": 1.00, "outputAPICost": 5.00, "batchSize": 30, "frequencyPenalty": 0.05}
- elif "claude-haiku-3-5" in model:
- cfg = {"inputAPICost": 0.80, "outputAPICost": 4.00, "batchSize": 30, "frequencyPenalty": 0.05}
- elif "claude-3-haiku" in model:
- cfg = {"inputAPICost": 0.25, "outputAPICost": 1.25, "batchSize": 30, "frequencyPenalty": 0.05}
- elif "haiku" in model:
- # Unknown haiku version — use current flagship pricing as best guess
- cfg = {"inputAPICost": 1.00, "outputAPICost": 5.00, "batchSize": 30, "frequencyPenalty": 0.05}
- elif "sonnet" in model or "claude" in model:
- cfg = {"inputAPICost": 3.00, "outputAPICost": 15.00, "batchSize": 30, "frequencyPenalty": 0.05}
- elif "gemini-2.0-flash-lite" in model:
- cfg = {"inputAPICost": 0.075, "outputAPICost": 0.30, "batchSize": 30, "frequencyPenalty": 0.0}
- elif "gemini-2.0-flash" in model:
- cfg = {"inputAPICost": 0.10, "outputAPICost": 0.40, "batchSize": 30, "frequencyPenalty": 0.0}
- elif "gemini-2.5-flash-lite" in model:
- cfg = {"inputAPICost": 0.10, "outputAPICost": 0.40, "batchSize": 30, "frequencyPenalty": 0.0}
- elif "gemini-2.5-flash" in model:
- cfg = {"inputAPICost": 0.30, "outputAPICost": 2.50, "batchSize": 30, "frequencyPenalty": 0.0}
- elif "gemini-2.5-pro" in model:
- cfg = {"inputAPICost": 1.25, "outputAPICost": 10.00, "batchSize": 30, "frequencyPenalty": 0.0}
- else:
- cfg = {
- "inputAPICost": float(os.getenv("input_cost", 3.00)),
- "outputAPICost": float(os.getenv("output_cost", 6.00)),
- "batchSize": int(os.getenv("batchsize", 10)),
- "frequencyPenalty": float(os.getenv("frequency_penalty", 0.2)),
- }
-
- # Apply live pricing from LiteLLM if available — keeps costs up-to-date
- # without requiring code changes when providers reprice their models.
- if _live_override:
- cfg.update(_live_override)
-
- return cfg
-
-
-def batchList(inputList, batchSize):
- """Split a list into batches of specified size"""
- if not isinstance(batchSize, int) or batchSize <= 0:
- raise ValueError("batchSize must be a positive integer")
-
- return [inputList[i : i + batchSize] for i in range(0, len(inputList), batchSize)]
-
-
-def parseVocabWithCategories(vocabText):
- """Parse vocabulary text and extract terms with their categories."""
- pairs = []
- seen = set()
- currentCategory = None
-
- for line in vocabText.splitlines():
- line = line.strip()
- if not line or line.startswith('```') or line.startswith('Here are some vocabulary'):
- continue
-
- # Check if this is a category header
- if line.startswith('#'):
- currentCategory = line
- continue
-
- # Parse vocabulary term - extract both Japanese and English parts.
- # Rich entries may continue after the first parenthesized translation,
- # e.g. "サンク (Sank) - Male; protagonist..."; only "Sank" is the match key.
- paren_match = re.match(r'^(.+?)\s*\(([^()]*)\)', line)
- dash_match = re.match(r'^(.+?)\s+[–-]\s+(.+)$', line)
- if paren_match:
- japanese_term = paren_match.group(1).strip()
- english_term = paren_match.group(2).strip()
-
- # Create a tuple with both terms for matching
- term_pair = (japanese_term, english_term)
- if term_pair not in seen:
- pairs.append((term_pair, line, currentCategory))
- seen.add(term_pair)
- elif dash_match:
- japanese_term = dash_match.group(1).strip()
- english_term = dash_match.group(2).strip()
-
- # Create a tuple with both terms for matching
- term_pair = (japanese_term, english_term)
- if term_pair not in seen:
- pairs.append((term_pair, line, currentCategory))
- seen.add(term_pair)
- elif line and not line.startswith('#'):
- # Fallback for lines without parentheses - treat as single term
- term = line.strip()
- if term and term not in seen:
- pairs.append((term, line, currentCategory))
- seen.add(term)
-
- return pairs
-
-
-def _japanese_term_in_text(term, text):
- """
- Check if a Japanese term appears in text as a standalone word, not as a
- substring of a longer run of the same script (katakana/hiragana/kanji).
- E.g. 'キス' will NOT match inside 'テキスト' because both neighbours are katakana.
- Falls back to plain substring check for non-Japanese or mixed terms.
- """
- if term not in text:
- return False
- KATAKANA = r'ァ-ヴーｦ-ﾟ'
- HIRAGANA = r'ぁ-ゔ'
- KANJI = r'一-龠'
- if re.search(rf'[{KATAKANA}]', term) and not re.search(rf'[{HIRAGANA}{KANJI}]', term):
- pattern = rf'(?= 500:
- raise Exception(f"API server error ({e.status_code}) - retrying... Error: {e}")
- elif e.status_code == 400 and formatType == "json" and "json_schema" in str(responseFormat):
- # Only fall back to json_object if the error is NOT "Input should be 'json_schema'"
- # (that message means json_schema IS required and json_object would also be rejected)
- if "input should be 'json_schema'" in str(e).lower() or "input should be \"json_schema\"" in str(e).lower():
- raise Exception(f"API status error ({e.status_code}): {e}")
- # Provider doesn't support json_schema (e.g. Claude) — fall back to json_object
- responseFormat = {"type": "json_object"}
- params["response_format"] = responseFormat
- try:
- response = openai.chat.completions.create(**params)
- except APIStatusError as fallback_error:
- if fallback_error.status_code == 400 and "input should be 'json_schema'" in str(fallback_error).lower():
- raise Exception(f"API requires json_schema response format but rejected the schema. Original error: {e}")
- raise Exception(f"API call failed: {e}. Fallback also failed: {fallback_error}")
- except Exception as fallback_error:
- raise Exception(f"API call failed: {e}. Fallback also failed: {fallback_error}")
- elif e.status_code == 400 and "input should be 'json_schema'" in str(e).lower():
- # response_format.type was rejected (e.g. sent "text" or "json_object" to a model
- # that only accepts json_schema). Remove response_format and retry with no constraint.
- params.pop("response_format", None)
- try:
- response = openai.chat.completions.create(**params)
- except Exception as fallback_error:
- raise Exception(f"API call failed: {e}. Fallback also failed: {fallback_error}")
- else:
- raise Exception(f"API status error ({e.status_code}): {e}")
- except (APIConnectionError, RateLimitError) as e:
- # These should always be retried
- raise Exception(f"API connection/rate limit error - retrying... Error: {e}")
- except Exception as e:
- # Check if it's a 404 error or other HTTP error that should be retried
- error_str = str(e).lower()
- if "404" in error_str or "not found" in error_str:
- raise Exception(f"API returned 404 Not Found - check your API configuration. Original error: {e}")
-
- # If structured output fails, fallback to json_object (unless the error
- # explicitly states json_schema is required — falling back would just fail again)
- if formatType == "json" and "json_schema" in str(responseFormat) and \
- "input should be 'json_schema'" not in error_str:
- responseFormat = {"type": "json_object"}
- params["response_format"] = responseFormat
- try:
- response = openai.chat.completions.create(**params)
- except Exception as fallback_error:
- # If fallback also fails, raise the original error for retry
- raise Exception(f"API call failed: {e}. Fallback also failed: {fallback_error}")
- else:
- raise e
-
- # Validate response before returning
- if not response or not hasattr(response, 'choices') or not response.choices:
- raise Exception("API returned invalid or empty response - retrying...")
-
- _write_request_debug_log(api_provider, params, getattr(response, "usage", None))
- return response
-
-
-def cleanTranslatedText(translatedText, language):
- """Clean and format translated text"""
- placeholders = {
- f"{language} Translation: ": "",
- "Translation: ": "",
- "っ": "",
- "〜": "~",
- "ッ": "",
- "。": ".",
- # Note: 「 and 」 are NOT replaced here — replacing them with ASCII " would
- # corrupt raw JSON strings before extraction. They are handled per-line
- # in _clean_extracted_line() after JSON parsing.
- "—": "―",
- "】": "]",
- "【": "[",
- "é": "e",
- "’": "'",
- "this guy": "this bastard",
- "This guy": "This bastard",
- "```json": "",
- "```": "",
- }
-
- for target, replacement in placeholders.items():
- translatedText = translatedText.replace(target, replacement)
-
- # Remove Repeating Characters
- pattern = re.compile(r"(.)\s*\1(?:\s*\1){" + str(20 - 1) + r",}")
- translatedText = pattern.sub(lambda match: match.group(0).replace(" ", "")[:20], translatedText)
-
- # Elongate Long Dashes (Since GPT Ignores them...)
- translatedText = elongateCharacters(translatedText)
- return translatedText
-
-
-def elongateCharacters(text):
- """Replace ー sequences with elongated characters"""
- # Define a pattern to match one character followed by two or more ー characters.
- # The lookbehind is restricted to non-ー Japanese/CJK characters so that:
- # - standalone ー separators (e.g. "ーーーーーーーーーー") are left untouched
- # - ー sequences preceded by a JSON quote or other non-Japanese char are not corrupted
- pattern = r"(?<=([\u3040-\u309F\u30A0-\u30FB\u30FD-\u30FF\u4E00-\u9FEF\uFF61-\uFF9F]))ー{2,}"
-
- # Define a replacement function that elongates the captured character
- def repl(match):
- char = match.group(1) # The character before the ー sequence
- count = len(match.group(0)) - 1 # Number of ー characters
- return char * count # Replace ー sequence with the character repeated
-
- # Use re.sub() to replace the pattern in the text
- return re.sub(pattern, repl, text)
-
-
-def extractTranslation(translatedTextList, isList, pbar=None):
- """Extract translation from JSON response.
-
- This function is resilient to a few common model mistakes:
- - Wraps output in code fences or outer quotes
- - Uses smart quotes instead of straight quotes
- - Inserts an extra leading quote in values (e.g. :""Word" -> :"Word")
- - Trailing commas before } or ]
-
- If strict JSON parsing fails, falls back to a regex-based extractor that
- captures LineN values in numeric order.
- """
- s = str(translatedTextList or "").strip()
-
- # Fast exit
- if not s:
- return None
-
- # Remove code fences if present
- if s.startswith("```"):
- s = re.sub(r"^```(?:json)?\s*", "", s, flags=re.IGNORECASE)
- s = re.sub(r"\s*```$", "", s)
-
- # Trim wrapping quotes around the whole JSON blob (common in logs)
- if len(s) >= 2 and s[0] == s[-1] and s[0] in {'"', "'"}:
- # Only strip if it still looks like JSON inside
- if s[1:2] == "{" and s[-2:-1] == "}":
- s = s[1:-1]
-
- # Normalize a broad set of Unicode “smart” quotes to ASCII equivalents.
- translation_table = {
- 0x201C: "'", # “ left double quotation mark
- 0x201D: "'", # ” right double quotation mark
- 0xFF02: "'", # ＂ fullwidth quotation mark
-
- 0x2018: "'", # ‘ left single quotation mark
- 0x2019: "'", # ’ right single quotation mark
- 0x201B: "'", # ‛ single high-reversed-9 quotation mark
- 0x02BC: "'", # ʼ modifier letter apostrophe
- 0xFF07: "'", # ＇ fullwidth apostrophe
- }
- s = s.translate(translation_table)
-
- # Remove trailing commas before object/array closures
- s = re.sub(r",(\s*[}\]])", r"\1", s)
-
- # Repair common doubled leading quote in values: :""Word" -> :"Word"
- # Ensure we don't alter legitimate empty strings (:"")
- s = re.sub(r":\s*\"\"(?=[^\",}\]\s])", r':"', s)
-
- # Attempt strict parse first
- try:
- lineDict = json.loads(s)
-
- # Handle array-based schema: {"translations": ["...", ...]}
- if isinstance(lineDict, dict) and "translations" in lineDict and isinstance(lineDict["translations"], list):
- stringList = [str(v) for v in lineDict["translations"]]
- return stringList if isList else (stringList[0] if stringList else None)
-
- # Build list in numeric order if keys are LineN
- numeric_keys = []
- for k in lineDict.keys():
- m = re.fullmatch(r"Line(\d+)", str(k))
- if m:
- numeric_keys.append(int(m.group(1)))
-
- if numeric_keys:
- stringList = [lineDict.get(f"Line{n}", "") for n in sorted(numeric_keys)]
- else:
- # Fallback to values order if no LineN keys found
- stringList = list(lineDict.values())
-
- return stringList if isList else (stringList[0] if stringList else None)
-
- except Exception as e:
- # Fallback: regex-based extraction tolerant to one or two opening quotes
- # Captures escaped quotes within values too
- try:
- pairs = re.findall(r'"Line(\d+)"\s*:\s*"{1,2}((?:\\.|[^"\\])*)"', s)
- if not pairs:
- raise ValueError("No LineN pairs found")
-
- # Sort numerically and unescape JSON string content
- items = []
- for n_str, v in sorted(((int(n), v) for n, v in pairs), key=lambda x: x[0]):
- try:
- # Decode JSON escapes reliably by round-tripping as a JSON string
- decoded = json.loads(f'"{v}"')
- except Exception:
- decoded = v
- items.append(decoded)
-
- return items if isList else (items[0] if items else None)
- except Exception as e2:
- if pbar:
- pbar.write(f"extractTranslation Error: {e2} after JSON error {e} on String {translatedTextList}")
- return None
-
-
-def calculateCost(inputTokens, outputTokens, model):
- """
- Calculate the cost of translation based on token usage and model pricing.
-
- For Claude models the cost is derived from the actual cache token breakdown
- recorded by translateAI, so cache discounts are reflected accurately:
- - Cache reads: 10 % of the base input rate
- - Cache writes: 125 % of the base input rate
- - Regular input: 100 % of the base input rate
-
- Call pattern (no module changes required):
- Per-file call: file_cost_ready flag is True → read thread-local per-file
- accumulators (which span all translateAI calls for the file),
- compute cost, reset accumulators, clear flag, return cost.
- TOTAL call: file_cost_ready is False (already cleared) → return the
- cross-thread _global_accurate_cost running sum.
-
- Falls back to naive token × rate calculation for non-Claude models.
- """
- _is_claude = model and any(x in model.lower() for x in ("claude", "sonnet", "haiku", "opus"))
- if _is_claude:
- if getattr(_thread_local, 'file_cost_ready', False):
- # Per-file call: compute from accumulators (may be 0 for disk-cached files),
- # reset everything, return the file cost.
- cr = getattr(_thread_local, 'file_cache_read', 0)
- cw = getattr(_thread_local, 'file_cache_write', 0)
- reg = getattr(_thread_local, 'file_regular', 0)
- out = getattr(_thread_local, 'file_output', 0)
- bcr = getattr(_thread_local, 'file_batch_read', 0)
- bcw = getattr(_thread_local, 'file_batch_write', 0)
- breg = getattr(_thread_local, 'file_batch_regular', 0)
- bout = getattr(_thread_local, 'file_batch_output', 0)
- pricing = getPricingConfig(model)
- br = pricing["inputAPICost"] / 1_000_000
- orr = pricing["outputAPICost"] / 1_000_000
- cost = (cr * br * 0.10 + cw * br * 2.00 + reg * br + out * orr
- # Batch API tokens: same rates, then the 50% batch discount.
- + (bcr * br * 0.10 + bcw * br * 2.00 + breg * br + bout * orr) * 0.50)
- _thread_local.file_cache_read = 0
- _thread_local.file_cache_write = 0
- _thread_local.file_regular = 0
- _thread_local.file_output = 0
- _thread_local.file_batch_read = 0
- _thread_local.file_batch_write = 0
- _thread_local.file_batch_regular = 0
- _thread_local.file_batch_output = 0
- _thread_local.file_cost_ready = False
- return cost
- # TOTAL call (flag already cleared): return the cross-thread running total.
- # If _global_accurate_cost is 0 it means no real API calls were made
- # (e.g. estimate mode), so fall through to the naive calculation below.
- with _global_accurate_cost_lock:
- accurate = _global_accurate_cost
- if accurate > 0:
- return accurate
-
- # Non-Claude, estimate mode, or no accurate data: naive calculation.
- # For Claude models, use the accumulated static_system token count (the portion
- # that is always cache-written at the 1hr TTL rate = 2x input rate).
- # Remaining tokens are billed at the regular input rate.
- pricing = getPricingConfig(model)
- _is_claude_naive = model and any(x in model.lower() for x in ("claude", "sonnet", "haiku", "opus"))
- if _is_claude_naive:
- static_tok = getattr(_thread_local, 'estimate_static_tokens', 0)
- regular_tok = getattr(_thread_local, 'estimate_regular_tokens', 0)
- batch_count = max(1, getattr(_thread_local, 'estimate_batch_count', 1))
- _thread_local.estimate_static_tokens = 0
- _thread_local.estimate_regular_tokens = 0
- _thread_local.estimate_batch_count = 0
- # If cache is disabled, every batch is a write (2x) — no reads ever.
- # Otherwise: each distinct batch size (= distinct output_config schema) gets exactly
- # one cache write on first use; all subsequent batches of that size are reads (0.10x).
- # Load from disk first so GUI subprocesses (one per file) share warm-cache state.
- global _estimate_written_sizes
- if DISABLE_CACHE:
- write_batches = batch_count
- read_batches = 0
- else:
- _load_estimate_written_sizes()
- seen_sizes = getattr(_thread_local, 'estimate_seen_sizes', set())
- new_sizes = seen_sizes - _estimate_written_sizes
- write_batches = len(new_sizes) # one write per newly-seen size
- read_batches = batch_count - write_batches
- _estimate_written_sizes.update(new_sizes)
- _save_estimate_written_sizes()
- _thread_local.estimate_seen_sizes = set()
- write_cost = (write_batches * static_tok / 1_000_000) * pricing["inputAPICost"] * 2.0
- read_cost = (read_batches * static_tok / 1_000_000) * pricing["inputAPICost"] * 0.10
- regular_cost = (regular_tok / 1_000_000) * pricing["inputAPICost"]
- inputCost = write_cost + read_cost + regular_cost
- else:
- inputCost = (inputTokens / 1_000_000) * pricing["inputAPICost"]
- outputCost = (outputTokens / 1_000_000) * pricing["outputAPICost"]
- return inputCost + outputCost
-
-
-def countTokens(system, user, history):
- """Count tokens for cost estimation"""
- inputTotalTokens = 0
- outputTotalTokens = 0
- enc = tiktoken.encoding_for_model("gpt-4")
-
- # Input
- if isinstance(history, list):
- for line in history:
- inputTotalTokens += len(enc.encode(line))
- else:
- inputTotalTokens += len(enc.encode(history))
- inputTotalTokens += len(enc.encode(system))
- inputTotalTokens += len(enc.encode(user))
-
- # Output
- outputTotalTokens += round(len(enc.encode(user)) * 2.5)
-
- return [inputTotalTokens, outputTotalTokens]
-
-
-@retry(exceptions=Exception, tries=5, delay=5)
-def translateAI(text, history, config, filename=None, pbar=None, lock=None, mismatchList=None):
- """
- Main translation entry point used by all modules.
-
- Returns [translatedText, [inputTokens, outputTokens]].
- """
- if not text:
- return [text, [0, 0]]
-
- # Use TRANSLATION_RUN_LOG env var as log path if set.
- run_log = os.getenv("TRANSLATION_RUN_LOG")
- if run_log:
- # Make sure parent dir exists
- try:
- Path(run_log).parent.mkdir(parents=True, exist_ok=True)
- except Exception:
- pass
- config.logFilePath = run_log
-
- # Ensure log directory exists for the configured path
- try:
- Path(config.logFilePath).parent.mkdir(parents=True, exist_ok=True)
- except Exception:
- pass
-
- # Token tracking: [input, output].
- totalTokens = [0, 0]
-
- # Init per-file accumulators on first call on this thread (never reset here —
- # they span all translateAI calls for a file; reset by calculateCost).
- if not hasattr(_thread_local, 'file_cache_read'):
- _thread_local.file_cache_read = 0
- _thread_local.file_cache_write = 0
- _thread_local.file_regular = 0
- _thread_local.file_output = 0
- # Batch API usage is billed at 50% so it is accumulated separately.
- if not hasattr(_thread_local, 'file_batch_read'):
- _thread_local.file_batch_read = 0
- _thread_local.file_batch_write = 0
- _thread_local.file_batch_regular = 0
- _thread_local.file_batch_output = 0
- # Snapshot accumulators so end-of-call delta only counts tokens from this call.
- _prev_cr = _thread_local.file_cache_read
- _prev_cw = _thread_local.file_cache_write
- _prev_reg = _thread_local.file_regular
- _prev_out = _thread_local.file_output
- _prev_bcr = _thread_local.file_batch_read
- _prev_bcw = _thread_local.file_batch_write
- _prev_breg = _thread_local.file_batch_regular
- _prev_bout = _thread_local.file_batch_output
- _thread_local.file_cost_ready = False # will be set True at end of translateAI
-
- # Anthropic batch phase ('collect'/'consume'); None when off, in estimate
- # mode, or when the model doesn't route to the native Anthropic SDK.
- batch_phase = get_batch_phase()
- if batch_phase and (config.estimateMode or not isClaudeNative(config.model)):
- batch_phase = None
-
- if isinstance(text, list):
- formatType = "json"
- tList = batchList(text, config.batchSize)
- else:
- formatType = "json"
- tList = [text]
-
- for index, tItem in enumerate(tList):
- # Check if text contains target language
- if not re.search(config.langRegex, str(tItem)):
- if pbar is not None:
- pbar.update(len(tItem) if isinstance(tItem, list) else 1)
- if isinstance(tItem, list):
- for j in range(len(tItem)):
- tItem[j] = cleanTranslatedText(tItem[j], config.language)
- tList[index] = tItem
- else:
- tList[index] = cleanTranslatedText(tItem, config.language)
- history = tItem[-config.maxHistory:] if isinstance(tItem, list) else tItem
- continue
-
- # Ellipsis-only bypass: strings whose translatable content is purely '…' characters
- # (e.g. "「………」") should never be sent to the AI — just convert brackets and pass through.
- def _is_ellipsis_only(s):
- inner = str(s).strip().lstrip('「『').rstrip('」』').strip()
- return bool(inner) and all(c in '\u2026\u30FC' for c in inner)
-
- def _convert_ellipsis(s):
- return str(s).replace('「', '"').replace('」', '"').replace('『', '"').replace('』', '"')
-
- if isinstance(tItem, list):
- if all(_is_ellipsis_only(s) for s in tItem):
- tList[index] = [_convert_ellipsis(s) for s in tItem]
- if pbar is not None:
- pbar.update(len(tItem))
- continue
- else:
- if _is_ellipsis_only(tItem):
- tList[index] = _convert_ellipsis(tItem)
- if pbar is not None:
- pbar.update(1)
- continue
-
- # Protect script codes before translation
- protected_items = []
- all_replacements = {}
-
- if isinstance(tItem, list):
- for j in range(len(tItem)):
- if not tItem[j] or not str(tItem[j]).strip():
- protected_items.append("Placeholder Text")
- all_replacements[j] = {}
- else:
- collapsed = re.sub(r'(.)\1{9,}', lambda m: m.group(1) * 10, tItem[j])
- protected_text, replacements = protect_script_codes(collapsed)
- protected_items.append(protected_text)
- all_replacements[j] = replacements
- else:
- if not tItem or not str(tItem).strip():
- protected_items = "Placeholder Text"
- all_replacements[0] = {}
- else:
- collapsed = re.sub(r'(.)\1{9,}', lambda m: m.group(1) * 10, tItem)
- protected_items, all_replacements[0] = protect_script_codes(collapsed)
-
- # Filter out corrupted/mojibake text (U+FFFD) from the batch before API call
- corrupted_map = {} # original_index -> original_text
- if isinstance(tItem, list):
- for j in range(len(tItem)):
- if tItem[j] and "\ufffd" in str(tItem[j]):
- corrupted_map[j] = tItem[j]
- elif tItem and "\ufffd" in str(tItem):
- # Single corrupted string - skip translation entirely
- tList[index] = tItem
- if pbar is not None:
- pbar.update(1)
- history = tItem
- continue
-
- # Filter out items that have content but no Japanese — they need no translation
- # and the AI tends to empty them (e.g. "「………」" -> ""). Apply the same
- # cleanup that would happen post-translation and restore them afterwards.
- no_japanese_map = {} # original_index -> already-cleaned text
- if isinstance(tItem, list):
- for j in range(len(tItem)):
- if j in corrupted_map:
- continue
- item_str = str(tItem[j]).strip() if tItem[j] else ""
- if item_str and item_str != "Placeholder Text" and not re.search(config.langRegex, item_str):
- cleaned = cleanTranslatedText(tItem[j], config.language)
- cleaned = cleaned.replace("「", '"').replace("」", '"').strip()
- no_japanese_map[j] = cleaned
-
- # Combine skip sets and rebuild protected_items / all_replacements
- skip_indices = set(corrupted_map.keys()) | set(no_japanese_map.keys())
- if isinstance(tItem, list) and skip_indices:
- clean_indices = [j for j in range(len(tItem)) if j not in skip_indices]
-
- if not clean_indices:
- # Every item is either corrupted or untranslatable — reassemble and move on
- result = []
- for j in range(len(tItem)):
- if j in corrupted_map:
- result.append(corrupted_map[j])
- elif j in no_japanese_map:
- result.append(no_japanese_map[j])
- else:
- result.append(tItem[j])
- tList[index] = result
- if pbar is not None:
- pbar.update(len(tItem))
- history = result[-config.maxHistory:]
- continue
-
- # Rebuild protected_items and all_replacements for translatable items only
- protected_items = [protected_items[j] for j in clean_indices]
- new_replacements = {}
- for new_idx, old_idx in enumerate(clean_indices):
- new_replacements[new_idx] = all_replacements.get(old_idx, {})
- all_replacements = new_replacements
-
- # Build filtered tItem for validation (excludes skipped items)
- if isinstance(tItem, list) and skip_indices:
- clean_tItem = [tItem[j] for j in range(len(tItem)) if j not in skip_indices]
- else:
- clean_tItem = tItem
-
- # Format for translation
- if isinstance(tItem, list):
- payload = {f"Line{i+1}": string for i, string in enumerate(protected_items)}
- payload = json.dumps(payload, indent=4, ensure_ascii=False)
- subbedT = payload
- else:
- subbedT = json.dumps({"Line1": protected_items}, indent=4, ensure_ascii=False)
-
- # Batch collect queues list payloads only. Single strings (speaker and
- # variable names) translate live — modules memoize them and embed the
- # results into later payloads, so they must resolve identically in both
- # passes or the consume pass couldn't match the queued payload keys.
- # This is the names-first phase; names are a tiny share of the volume.
- queue_for_batch = batch_phase == "collect" and isinstance(tItem, list)
-
- # Check cache for this exact payload (the collect pass uses a
- # non-blocking peek so no pending markers are left behind for the
- # consume pass to wait on)
- if queue_for_batch:
- cached_result = peek_cached_translation(subbedT, config.language)
- else:
- cached_result = get_cached_translation(subbedT, config.language)
- if cached_result is not None:
- # In estimate mode, never replace tList[index] from cache — the cached value
- # may have been stored for a batch with a different number of skip_indices,
- # so its length can differ from the current tItem. Keeping tList[index] as
- # the original tItem ensures the returned list always has the correct length.
- if not config.estimateMode:
- if isinstance(tItem, list):
- tList[index] = cached_result
- history = cached_result[-config.maxHistory:]
- else:
- tList[index] = cached_result
- history = cached_result
- else:
- if isinstance(cached_result, list) and cached_result:
- history = cached_result[-config.maxHistory:]
- elif cached_result:
- history = cached_result
-
- if lock and pbar is not None:
- with lock:
- pbar.update(len(tItem) if isinstance(tItem, list) else 1)
-
- continue
-
- # Create context — static_system is the stable prompt.txt content;
- # vocab_text is the per-batch matched vocabulary (dynamic).
- static_system, vocab_text, user = createContext(config, subbedT, formatType, history)
-
- # Batch collect pass: queue the request (built exactly like a live one)
- # instead of calling the API. The text stays untranslated; the consume
- # pass fills it in from the fetched batch results. History carries the
- # preceding source lines so the model still sees scene context.
- if queue_for_batch:
- numLines = len(clean_tItem) if isinstance(tItem, list) else 1
- params = buildClaudeRequest(static_system, user, history, formatType,
- config.model, numLines, vocab_text=vocab_text)
- queue_batch_request(subbedT, config.language, params)
- if lock and pbar is not None:
- with lock:
- pbar.update(len(tItem))
- history = tItem[-config.maxHistory:]
- continue
-
- # Calculate estimate if in estimate mode
- if config.estimateMode:
- estimate = countTokens(static_system + vocab_text, user, history)
- totalTokens[0] += estimate[0]
- totalTokens[1] += estimate[1]
-
- # Track exact cache write size (static_system, constant across batches)
- # and accumulate non-cached (vocab + user + history) tokens per batch.
- _est_api = os.getenv("api", "").strip()
- _is_claude_est = (
- config.model
- and any(x in config.model.lower() for x in ("claude", "sonnet", "haiku", "opus"))
- and (not _est_api or "anthropic" in _est_api.lower())
- )
- if _is_claude_est:
- # Use Anthropic's count_tokens API once to get the exact cached token count.
- # Only called on the first batch; result reused for all subsequent batches.
- if not getattr(_thread_local, 'estimate_static_tokens', 0):
- try:
- _ant_count_client = anthropic.Anthropic(api_key=openai.api_key)
- backtick = chr(96) * 3
- _sys_block = [{"type": "text", "text": backtick + "\n" + static_system + "\n" + backtick, "cache_control": {"type": "ephemeral", "ttl": "1h"}}]
- _count_resp = _ant_count_client.beta.messages.count_tokens(
- betas=["token-counting-2024-11-01"],
- model=config.model,
- system=_sys_block,
- messages=[{"role": "user", "content": "x"}]
- )
- _thread_local.estimate_static_tokens = _count_resp.input_tokens
- except Exception:
- # Fallback to tiktoken if count_tokens fails
- enc = tiktoken.encoding_for_model("gpt-4")
- _thread_local.estimate_static_tokens = len(enc.encode(static_system))
- regular_tok = max(0, estimate[0] - getattr(_thread_local, 'estimate_static_tokens', 0))
- _thread_local.estimate_regular_tokens = getattr(_thread_local, 'estimate_regular_tokens', 0) + regular_tok
- _thread_local.estimate_batch_count = getattr(_thread_local, 'estimate_batch_count', 0) + 1
- # Track unique batch sizes seen this file (each maps to a distinct schema)
- _size = len(clean_tItem) if isinstance(clean_tItem, list) else 1
- _seen = getattr(_thread_local, 'estimate_seen_sizes', set())
- _seen.add(_size)
- _thread_local.estimate_seen_sizes = _seen
-
- # Cache the payload with original text as placeholder for future estimates
- if isinstance(tItem, list):
- cache_translation(subbedT, tItem, config.language)
- else:
- cache_translation(subbedT, [tItem], config.language)
-
- continue
-
- # --- Translation and Validation Retry Block ---
- max_retries = 2 # 1 initial attempt + 2 retries
- final_translations = None
- last_raw_translation = ""
- numLines = len(clean_tItem) if isinstance(tItem, list) else 1
-
- for attempt in range(max_retries + 1):
- is_valid = True
-
- # On retries, prepend the correction note to the USER message so the
- # cached static_system block is never modified (avoids cache busting).
- current_user = user
- if attempt > 0:
- retry_note = (
- f"IMPORTANT: Your previous attempt was incorrect or incomplete. Please ensure:\n"
- f"1. The entire output is translated to {config.language} with no untranslated characters\n"
- f"2. The JSON structure is correct with NO EMPTY or near-empty translations\n"
- f" - Every line with Japanese text MUST be fully translated\n"
- f" - Do NOT leave translations empty (\"\") or as single punctuation marks (\":\")\n"
- f"3. ALL placeholders (like __PROTECTED_0__, __PROTECTED_1__, etc.) are preserved EXACTLY as they appear in the input\n"
- f" - Do not modify, translate, or remove any __PROTECTED_N__ placeholders\n"
- f" - Keep them in the exact same position in your translation\n"
- f"4. Do NOT repeat the same letter or symbol many times in a row (e.g. uuuuuuuu... or broken tails)\n"
- f" - Keep moans/effects natural; never output long runs of one character\n\n"
- )
- current_user = retry_note + user
- if pbar:
- pbar.write(f"Retrying translation... (Attempt {attempt + 1}/{max_retries + 1})")
-
- # Translate — the consume pass tries the fetched batch result first;
- # a missing or invalid result falls through to the live API.
- from_batch = False
- if batch_phase == "consume" and attempt == 0:
- batch_result = take_batch_result(subbedT, config.language)
- if batch_result is not None:
- response = _AnthropicCompat(
- batch_result.get("text", ""),
- batch_result.get("prompt_tokens", 0) or 0,
- batch_result.get("completion_tokens", 0) or 0,
- batch_result.get("cache_read_input_tokens", 0) or 0,
- batch_result.get("cache_creation_input_tokens", 0) or 0,
- )
- from_batch = True
- _write_request_debug_log("anthropic-batch", {"payload": subbedT}, response.usage)
- if not from_batch:
- try:
- response = translateText(static_system, current_user, history, 0.05, formatType, config.model, numLines, vocab_text=vocab_text)
- except Exception as api_err:
- err_msg = f"[API_ERROR] {api_err}"
- # Print to stdout so the GUI captures it immediately
- print(err_msg, flush=True)
- if pbar:
- pbar.write(err_msg)
- # Also write to the translation log file for persistence
- try:
- Path(config.logFilePath).parent.mkdir(parents=True, exist_ok=True)
- with open(config.logFilePath, "a", encoding="utf-8") as _lf:
- _lf.write(f"{err_msg}\n")
- _lf.flush()
- except Exception:
- pass
- raise # Let retry decorator handle it
- translatedText = response.choices[0].message.content
- last_raw_translation = translatedText
-
- # Update token count for this attempt
- totalTokens[0] += response.usage.prompt_tokens
- totalTokens[1] += response.usage.completion_tokens
-
- # --- Cache cost tracking (Claude only) ---
- _is_claude_model = config.model and any(x in config.model.lower() for x in ("claude", "sonnet", "haiku", "opus"))
- if _is_claude_model:
- usage = response.usage
-
- # Read cache fields from _AnthropicCompat._Usage; fall back to model_extra.
- def _get_usage_field(field):
- v = getattr(usage, field, None)
- if v is None:
- v = (getattr(usage, "model_extra", None) or {}).get(field)
- return int(v) if v else 0
-
- batch_cache_read = _get_usage_field("cache_read_input_tokens")
- batch_cache_write = _get_usage_field("cache_creation_input_tokens")
- batch_prompt_total = getattr(usage, "prompt_tokens", 0) or 0
- batch_regular = max(0, batch_prompt_total - batch_cache_read - batch_cache_write)
- batch_output = getattr(usage, "completion_tokens", 0) or 0
-
- # Accumulate into per-file thread-local counters. Batch API
- # usage is billed at 50% so it goes into its own counters.
- if from_batch:
- _thread_local.file_batch_read += batch_cache_read
- _thread_local.file_batch_write += batch_cache_write
- _thread_local.file_batch_regular += batch_regular
- _thread_local.file_batch_output += batch_output
- else:
- _thread_local.file_cache_read += batch_cache_read
- _thread_local.file_cache_write += batch_cache_write
- _thread_local.file_regular += batch_regular
- _thread_local.file_output += batch_output
-
- # --- Debug Token Logging ---
- if DEBUG:
- try:
- _dbg_dir = Path("log")
- _dbg_dir.mkdir(parents=True, exist_ok=True)
- with open(_dbg_dir / "debug.log", "a", encoding="utf-8") as _dbf:
- _dbf.write(f"\n--- Batch ({len(clean_tItem) if isinstance(tItem, list) else 1} lines) ---\n")
- _dbf.write(f"Prompt: {response.usage.prompt_tokens} tokens | Output: {response.usage.completion_tokens} tokens\n")
- if hasattr(response.usage, "cache_read_input_tokens"):
- cr = getattr(response.usage, "cache_read_input_tokens", 0) or 0
- cw = getattr(response.usage, "cache_creation_input_tokens", 0) or 0
- cache_status = "HIT" if cr > 0 else ("WRITE" if cw > 0 else "MISS")
- _dbf.write(f"Cache: {cache_status} (read={cr}, write={cw})\n")
- _dbf.flush()
- except Exception:
- pass
-
- # Clean the translation first for consistency
- cleaned_text = cleanTranslatedText(translatedText, config.language)
-
- # Process and validate translation result
- if cleaned_text:
- if isinstance(tItem, list):
- extracted = extractTranslation(cleaned_text, True, pbar)
-
- # Check 1: Mismatch in length -> still a hard failure
- if extracted is None or len(clean_tItem) != len(extracted):
- is_valid = False
- if pbar:
- pbar.write(f"Length mismatch: expected {len(clean_tItem)}, got {len(extracted) if extracted else 0}")
- else:
- # Check 2: Validate placeholders are preserved
- # Flatten all_replacements for batch validation
- all_protected_text = protected_items # The list we sent
- placeholder_valid, missing, extra = validate_placeholders(all_protected_text, extracted,
- {k: v for replacements in all_replacements.values() for k, v in replacements.items()})
-
- if not placeholder_valid:
- is_valid = False
- if pbar:
- if missing:
- pbar.write(f"Missing placeholders: {', '.join(missing)}")
- if extra:
- pbar.write(f"Extra placeholders: {', '.join(extra)}")
- else:
- # Check 3: Validate that translations are not empty or nearly empty
- content_valid, invalid_indices, content_reasons = validate_translation_content(
- clean_tItem, extracted, config.langRegex
- )
-
- if not content_valid:
- is_valid = False
- if pbar:
- pbar.write(f"Invalid translation content detected:")
- for reason in content_reasons[:5]: # Show first 5 issues
- pbar.write(f" - {reason}")
- if len(content_reasons) > 5:
- pbar.write(f" ... and {len(content_reasons) - 5} more issues")
- else:
- # Set translations (line count matches, placeholders valid, and content is good)
- # Strip "Placeholder Text" from individual lines (AI placeholder for untranslatable input)
- # Also apply the 「→" / 」→" replacements here per-line (safe now that JSON is parsed)
- def _clean_extracted_line(line):
- if not isinstance(line, str):
- return line
- line = line.replace("Placeholder Text", "").strip()
- line = line.replace("「", '"').replace("」", '"')
- line = line.replace("『", '"').replace("』", '"')
- return line
- final_translations = [_clean_extracted_line(line) for line in extracted]
- else:
- # Single string: extract from JSON schema response
- extracted = extractTranslation(cleaned_text, False, pbar)
- if extracted is None:
- is_valid = False
- if pbar:
- pbar.write(f"Failed to extract translation from response: {cleaned_text[:100]}")
- else:
- # Validate placeholders against extracted value
- placeholder_valid, missing, extra = validate_placeholders(protected_items, extracted, all_replacements[0])
-
- if not placeholder_valid:
- is_valid = False
- if pbar:
- if missing:
- pbar.write(f"Missing placeholders: {', '.join(missing)}")
- if extra:
- pbar.write(f"Extra placeholders: {', '.join(extra)}")
- else:
- # Validate content for single string
- final_cleaned = extracted.replace("Placeholder Text", "")
- content_valid, _, content_reasons = validate_translation_content(
- tItem, final_cleaned, config.langRegex
- )
-
- if not content_valid:
- is_valid = False
- if pbar:
- pbar.write(f"Invalid translation content:")
- for reason in content_reasons:
- pbar.write(f" - {reason}")
- else:
- # Accept output - all validations passed
- final_translations = final_cleaned
- else:
- is_valid = False
- if pbar: pbar.write(f"AI Refused: {tItem}\n")
-
- # If translation is valid, break the retry loop
- if is_valid:
- break
-
- # --- End of Retry Block ---
-
- # After the loop, handle the final result
- if final_translations is not None: # Success case
- # Restore protected script codes
- if isinstance(tItem, list):
- for j in range(len(final_translations)):
- if j in all_replacements:
- final_translations[j] = restore_script_codes(final_translations[j], all_replacements[j])
-
- # Re-insert corrupted / no-japanese originals at their original positions
- if corrupted_map or no_japanese_map:
- expanded = []
- clean_idx = 0
- for j in range(len(tItem)):
- if j in corrupted_map:
- expanded.append(corrupted_map[j])
- elif j in no_japanese_map:
- expanded.append(no_japanese_map[j])
- else:
- expanded.append(final_translations[clean_idx])
- clean_idx += 1
- final_translations = expanded
- else:
- final_translations = restore_script_codes(final_translations, all_replacements[0])
-
- formatted_output = last_raw_translation
- try:
- parsed_json = json.loads(last_raw_translation)
- # Normalize array-based output to LineN format for log readability
- if isinstance(parsed_json, dict) and "translations" in parsed_json and isinstance(parsed_json["translations"], list):
- parsed_json = {f"Line{i+1}": v for i, v in enumerate(parsed_json["translations"])}
- formatted_output = json.dumps(parsed_json, indent=4, ensure_ascii=False)
- except (json.JSONDecodeError, ValueError):
- pass
-
- # Only open and write to log file when we have something to log
- try:
- with open(config.logFilePath, "a", encoding="utf-8") as logFile:
- logFile.write(f"Input:\n{subbedT}\n")
- logFile.write(f"Output:\n{formatted_output}\n")
- logFile.flush() # Ensure data is written to disk immediately
- except Exception:
- pass # Don't fail if logging fails
-
- # Cache the entire payload and its translation
- if not config.estimateMode:
- cache_translation(subbedT, final_translations, config.language)
-
- if isinstance(tItem, list):
- tList[index] = final_translations
- history = final_translations[-config.maxHistory:]
- else:
- tList[index] = final_translations
- history = final_translations
-
- if lock and pbar is not None:
- with lock:
- pbar.update(len(tItem) if isinstance(tItem, list) else 1)
-
- else: # Failure case after all retries
- if pbar: pbar.write(f"Translation failed after {max_retries + 1} attempts. Check mismatch log.")
-
- # Emit a machine-readable marker on stdout so the GUI worker
- # thread can detect the mismatch reliably (stdout is captured
- # synchronously, unlike file-tail polling which can be racy).
- try:
- print(f"MISMATCH_EVENT:{filename}", flush=True)
- except Exception:
- pass
-
- formatted_mismatch_output = last_raw_translation
- try:
- parsed_json = json.loads(last_raw_translation)
- if isinstance(parsed_json, dict) and "translations" in parsed_json and isinstance(parsed_json["translations"], list):
- parsed_json = {f"Line{i+1}": v for i, v in enumerate(parsed_json["translations"])}
- formatted_mismatch_output = json.dumps(parsed_json, indent=4, ensure_ascii=False)
- except (json.JSONDecodeError, ValueError):
- pass
- with open(config.mismatchLogPath, "a+", encoding="utf-8") as mismatchFile:
- mismatchFile.write(f"Failed after retries: {filename}\n")
- mismatchFile.write(f"Input:\n{subbedT}\n")
- mismatchFile.write(f"Final Output:\n{formatted_mismatch_output}\n")
- mismatchFile.flush() # Ensure data is written to disk immediately
-
- # Also write to the main translation log so the GUI log viewer can display it
- try:
- with open(config.logFilePath, "a", encoding="utf-8") as logFile:
- logFile.write(f"[MISMATCH] Failed after retries: {filename}\n")
- logFile.write(f"[MISMATCH] Input:\n")
- for mline in subbedT.splitlines():
- logFile.write(f"[MISMATCH] {mline}\n")
- logFile.write(f"[MISMATCH] Final Output:\n")
- for mline in formatted_mismatch_output.splitlines():
- logFile.write(f"[MISMATCH] {mline}\n")
- logFile.flush()
- except Exception:
- pass # Don't fail if logging fails
-
- if filename and mismatchList is not None and filename not in mismatchList:
- mismatchList.append(filename)
-
- tList[index] = tItem
- history = text[-config.maxHistory:] if isinstance(text, list) else text
-
- # Combine if multilist
- if tList and isinstance(tList[0], list):
- tList = [t for sublist in tList for t in sublist]
-
- # Save cache after processing (for both estimate and translation modes)
- save_cache()
-
- # Batch collect pass: merge this call's queued requests into the disk queue.
- if batch_phase == "collect":
- flush_batch_queue()
-
- # For Claude: accumulate only this call's delta into the cross-thread total.
- # file_* accumulators hold full per-file totals; calculateCost() reads them.
- _is_claude_final = config.model and any(x in config.model.lower() for x in ("claude", "sonnet", "haiku", "opus"))
- if _is_claude_final and not config.estimateMode:
- _pricing = getPricingConfig(config.model)
- _br = _pricing["inputAPICost"] / 1_000_000
- _or = _pricing["outputAPICost"] / 1_000_000
- # Delta = tokens added in this call only (not earlier calls for same file).
- _delta_cr = getattr(_thread_local, 'file_cache_read', 0) - _prev_cr
- _delta_cw = getattr(_thread_local, 'file_cache_write', 0) - _prev_cw
- _delta_reg = getattr(_thread_local, 'file_regular', 0) - _prev_reg
- _delta_out = getattr(_thread_local, 'file_output', 0) - _prev_out
- _delta_bcr = getattr(_thread_local, 'file_batch_read', 0) - _prev_bcr
- _delta_bcw = getattr(_thread_local, 'file_batch_write', 0) - _prev_bcw
- _delta_breg = getattr(_thread_local, 'file_batch_regular', 0) - _prev_breg
- _delta_bout = getattr(_thread_local, 'file_batch_output', 0) - _prev_bout
- _call_cost = (
- _delta_cr * _br * 0.10 +
- _delta_cw * _br * 2.00 +
- _delta_reg * _br +
- _delta_out * _or +
- # Batch API tokens: same rates, then the 50% batch discount.
- (_delta_bcr * _br * 0.10 +
- _delta_bcw * _br * 2.00 +
- _delta_breg * _br +
- _delta_bout * _or) * 0.50
- )
- global _global_accurate_cost
- with _global_accurate_cost_lock:
- _global_accurate_cost += _call_cost
- _thread_local.file_cost_ready = True # signals calculateCost to use file accumulators
-
- # Return result
- if isinstance(text, list):
- return [tList, totalTokens]
- else:
+"""
+Shared translation utilities for DazedMTLTool.
+Centralized translation function used across all modules.
+"""
+
+import os
+import re
+import json
+import time
+import random
+import unicodedata
+import tiktoken
+import openai
+import anthropic
+import urllib.request
+from openai import APIError, APIConnectionError, RateLimitError, APIStatusError
+import hashlib
+import threading
+from contextlib import contextmanager
+from dotenv import load_dotenv
+from pathlib import Path
+from retry import retry
+
+# Set to True to enable debug logging (token counts, cache costs, etc.)
+DEBUG = True
+_debug_request_log_lock = threading.Lock()
+
+# Set to True to disable Claude prompt caching for baseline cost comparison.
+DISABLE_CACHE = False
+
+# Thread-local per-file token breakdown; read by calculateCost() for Claude.
+_thread_local = threading.local()
+
+# Cross-thread running total of accurate cache-discounted cost (protected by lock).
+_global_accurate_cost = 0.0
+_global_accurate_cost_lock = threading.Lock()
+
+
+def _usage_to_debug_dict(usage):
+ """Extract token counts from provider usage objects for request debugging."""
+ if not usage:
+ return {}
+
+ usage_dict = {}
+ for field in (
+ "prompt_tokens",
+ "completion_tokens",
+ "input_tokens",
+ "output_tokens",
+ "cache_read_input_tokens",
+ "cache_creation_input_tokens",
+ ):
+ value = getattr(usage, field, None)
+ if value is not None:
+ usage_dict[field] = value
+
+ extra = getattr(usage, "model_extra", None)
+ if isinstance(extra, dict):
+ for field in ("cache_read_input_tokens", "cache_creation_input_tokens"):
+ value = extra.get(field)
+ if value is not None and field not in usage_dict:
+ usage_dict[field] = value
+
+ return usage_dict
+
+
+def _write_request_debug_log(provider, request_payload, usage):
+ """Write the exact SDK payload text and returned token usage."""
+ if not DEBUG:
+ return
+
+ try:
+ log_dir = Path("log")
+ log_dir.mkdir(parents=True, exist_ok=True)
+ usage_dict = _usage_to_debug_dict(usage)
+ payload_text = json.dumps(request_payload, indent=2, ensure_ascii=False, default=str)
+ usage_text = json.dumps(usage_dict, indent=2, ensure_ascii=False, default=str)
+
+ with _debug_request_log_lock:
+ with open(log_dir / "request_debug.log", "a", encoding="utf-8") as debug_file:
+ debug_file.write("\n=== API Request ===\n")
+ debug_file.write(f"Timestamp: {time.strftime('%Y-%m-%d %H:%M:%S')}\n")
+ debug_file.write(f"Provider: {provider}\n")
+ debug_file.write("Usage:\n")
+ debug_file.write(f"{usage_text}\n")
+ debug_file.write("Payload:\n")
+ debug_file.write(f"{payload_text}\n")
+ debug_file.flush()
+ except Exception:
+ pass
+
+def _normalize_openai_base_url(url: str) -> str:
+ """Ensure OpenAI SDK global base_url has a trailing slash."""
+ _url = (url or "").strip()
+ if _url and not _url.endswith("/"):
+ _url += "/"
+ return _url
+
+
+def isClaudeModel(model):
+ """True when the model name looks like an Anthropic Claude model."""
+ return bool(model) and any(x in model.lower() for x in ("claude", "sonnet", "haiku", "opus"))
+
+
+def isClaudeNative(model):
+ """True when this model routes to the native Anthropic SDK.
+
+ Mirrors the routing check in translateText: the model must look like Claude
+ AND the configured API URL must be unset or point at anthropic.com. Any
+ other custom URL (e.g. DeepSeek, OpenAI proxy) uses the OpenAI-compatible
+ path even for Claude-named models.
+ """
+ live_api = os.getenv("api", "").strip()
+ return isClaudeModel(model) and (not live_api or "anthropic" in live_api.lower())
+
+
+def isMistralAPI():
+ """True when requests go to the Mistral platform API (la Plateforme).
+
+ Detection is URL-based: the adaptive rate limiter keys off x-ratelimit-*
+ headers that only api.mistral.ai sends. Mistral models served through other
+ OpenAI-compatible providers (Nvidia, OpenRouter, ...) use the generic path.
+ """
+ live_api = os.getenv("api", "").strip().lower()
+ if "mistral.ai" in live_api:
+ return True
+ return not live_api and os.getenv("API_PROVIDER", "").strip().lower() == "mistral"
+
+
+# Models that REJECT sampling params (temperature/top_p/top_k) with a 400 —
+# Claude Opus 4.7 and up retired them in favour of adaptive thinking. Matches
+# opus-4-7, opus-4-8, opus-4-10+ and fable, but NOT opus-4-6 or Sonnet/Haiku.
+_NO_SAMPLING_RE = re.compile(r"opus-4-(?:[7-9]\b|[1-9]\d)|fable", re.I)
+
+# Tracks which distinct batch sizes have already been cache-written during this estimate run.
+# Each unique numLines value maps to a distinct output_config schema → one write per size.
+# Persisted to disk so sequential GUI subprocesses share state.
+_estimate_written_sizes: set = set()
+_ESTIMATE_SIZES_FILE = Path("log/estimate_written_sizes.json")
+
+def _load_estimate_written_sizes():
+ """Load persisted written-sizes set from disk (for GUI subprocess sharing)."""
+ global _estimate_written_sizes
+ try:
+ if _ESTIMATE_SIZES_FILE.exists():
+ with open(_ESTIMATE_SIZES_FILE, "r", encoding="utf-8") as f:
+ _estimate_written_sizes = set(json.load(f))
+ except Exception:
+ _estimate_written_sizes = set()
+
+def _save_estimate_written_sizes():
+ """Persist written-sizes set to disk."""
+ try:
+ _ESTIMATE_SIZES_FILE.parent.mkdir(parents=True, exist_ok=True)
+ with open(_ESTIMATE_SIZES_FILE, "w", encoding="utf-8") as f:
+ json.dump(list(_estimate_written_sizes), f)
+ except Exception:
+ pass
+
+def clear_estimate_written_sizes():
+ """Reset the written-sizes file at the start of a new estimate run."""
+ global _estimate_written_sizes
+ _estimate_written_sizes = set()
+ try:
+ if _ESTIMATE_SIZES_FILE.exists():
+ _ESTIMATE_SIZES_FILE.unlink()
+ except Exception:
+ pass
+
+
+# ===== Placeholder Protection System =====
+# Patterns to protect from translation (sound effects, control codes, etc.)
+PROTECTED_PATTERNS = [
+ r'\\SE\[[^\]]+\]', # \SE[sound_effect_name]
+ r'\\ME\[[^\]]+\]', # \ME[music_effect_name]
+ r'\\BGM\[[^\]]+\]', # \BGM[background_music_name]
+ r'\\BGS\[[^\]]+\]', # \BGS[background_sound_name]
+ r'_pum\[[^\]]+\]', # _pum[name]
+ r'\\VS\[[^\]]+\]', # \VS[name]
+]
+
+def protect_script_codes(text):
+ """
+ Replace script codes (like \\SE[タイプライター]) with unique placeholders before translation.
+ Returns: (protected_text, replacements_dict)
+ """
+ if not text or not isinstance(text, str):
+ return text, {}
+
+ # Normalize curly/smart quotes to ASCII equivalents BEFORE building the JSON
+ # payload. When these characters appear inside a JSON string value the AI
+ # tends to treat them as regular ASCII double-quotes, which makes the value
+ # appear empty (e.g. `"スキルを"リセットする` → AI sees empty + stray text).
+ # This mirrors the identical normalization already applied to the AI's OUTPUT
+ # inside extractTranslation's translation_table.
+ quote_norm_table = str.maketrans({
+ '\u201C': "'", # " left double quotation mark
+ '\u201D': "'", # " right double quotation mark
+ '\uFF02': "'", # ＂ fullwidth quotation mark
+ '\u2018': "'", # ' left single quotation mark
+ '\u2019': "'", # ' right single quotation mark
+ '\u201B': "'", # ‛ single high-reversed-9 quotation mark
+ '\u02BC': "'", # ʼ modifier letter apostrophe
+ '\uFF07': "'", # ＇ fullwidth apostrophe
+ })
+ text = text.translate(quote_norm_table)
+
+ # Convert half-width katakana (U+FF61–U+FF9F) to full-width katakana so the
+ # AI recognises them as Japanese text and translates them correctly.
+ # NFKC is applied only to matched half-width kana spans to avoid altering
+ # intentional fullwidth Latin/digit characters elsewhere in the string.
+ text = re.sub(r'[\uFF61-\uFF9F]+', lambda m: unicodedata.normalize('NFKC', m.group(0)), text)
+
+ replacements = {}
+ protected_text = text
+ counter = 0
+
+ # Combine all patterns
+ combined_pattern = '|'.join(f'({pattern})' for pattern in PROTECTED_PATTERNS)
+
+ def replace_match(match):
+ nonlocal counter
+ original = match.group(0)
+ # Create a unique placeholder that won't be translated
+ placeholder = f"__PROTECTED_{counter}__"
+ replacements[placeholder] = original
+ counter += 1
+ return placeholder
+
+ if combined_pattern:
+ protected_text = re.sub(combined_pattern, replace_match, protected_text)
+
+ return protected_text, replacements
+
+
+def restore_script_codes(text, replacements):
+ """
+ Restore protected script codes from placeholders after translation.
+ """
+ if not text or not replacements:
+ return text
+
+ if isinstance(text, str):
+ result = text
+ for placeholder, original in replacements.items():
+ result = result.replace(placeholder, original)
+ return result
+ elif isinstance(text, list):
+ return [restore_script_codes(item, replacements) for item in text]
+ else:
+ return text
+
+
+def validate_placeholders(original_text, translated_text, replacements):
+ """
+ Validate that all placeholders from the original text appear in the translation.
+ Returns: (is_valid, missing_placeholders, extra_placeholders)
+ """
+ if not replacements:
+ return True, [], []
+
+ # Get all placeholders
+ all_placeholders = set(replacements.keys())
+
+ # Count placeholders in original
+ original_counts = {}
+ for placeholder in all_placeholders:
+ if isinstance(original_text, str):
+ original_counts[placeholder] = original_text.count(placeholder)
+ elif isinstance(original_text, list):
+ original_counts[placeholder] = sum(str(item).count(placeholder) for item in original_text)
+
+ # Count placeholders in translation
+ translated_counts = {}
+ for placeholder in all_placeholders:
+ if isinstance(translated_text, str):
+ translated_counts[placeholder] = translated_text.count(placeholder)
+ elif isinstance(translated_text, list):
+ translated_counts[placeholder] = sum(str(item).count(placeholder) for item in translated_text)
+
+ # Find mismatches
+ missing = []
+ extra = []
+ for placeholder in all_placeholders:
+ orig_count = original_counts.get(placeholder, 0)
+ trans_count = translated_counts.get(placeholder, 0)
+
+ if trans_count < orig_count:
+ missing.append(f"{placeholder} (expected {orig_count}, found {trans_count})")
+ elif trans_count > orig_count:
+ extra.append(f"{placeholder} (expected {orig_count}, found {trans_count})")
+
+ is_valid = len(missing) == 0 and len(extra) == 0
+ return is_valid, missing, extra
+
+
+def validate_translation_content(original_items, translated_items, langRegex):
+ """
+ Validate that translated items are not empty or nearly empty.
+ Returns: (is_valid, invalid_indices, reasons)
+
+ Rules:
+ 1. If original has content, translation must not be empty or just whitespace
+ 2. If original has Japanese text, translation must not be a single punctuation mark
+ 3. Translation should have meaningful content (more than 1-2 characters for substantial originals)
+ """
+ if not isinstance(original_items, list):
+ original_items = [original_items]
+ translated_items = [translated_items]
+
+ invalid_indices = []
+ reasons = []
+
+ for i, (orig, trans) in enumerate(zip(original_items, translated_items)):
+ orig_str = str(orig).strip()
+ trans_str = str(trans).strip()
+
+ # Skip if original is empty or placeholder
+ if not orig_str or orig_str == "Placeholder Text":
+ continue
+
+ # Check if original has content that needs translation
+ has_source_text = bool(re.search(langRegex, orig_str))
+
+ if has_source_text:
+ # Original has Japanese text - translation must be substantial
+
+ # Check 1: Translation is empty or just whitespace
+ if not trans_str:
+ invalid_indices.append(i)
+ reasons.append(f"Line{i+1}: Empty translation for '{orig_str[:50]}...'")
+ continue
+
+ # Check 2: Translation is just a single punctuation mark or very short
+ # Allow control codes like \\C[27]\\V[45] but not just ":" or ""
+ # Use <= 1 so real 2-char words like "No", "Go", "Hi" are not rejected
+ if len(trans_str) <= 1 and not re.search(r'\\[A-Z]\[', trans_str):
+ # Exception: if original is also very short (like "回" -> "x"), that's ok
+ if len(orig_str) > 3:
+ invalid_indices.append(i)
+ reasons.append(f"Line{i+1}: Translation too short ('{trans_str}') for '{orig_str[:50]}...'")
+ continue
+
+ # Check 3: For longer originals (>10 chars), translation should be more than just 1-2 chars
+ # unless it's a special case like numbers or codes
+ if len(orig_str) > 10 and len(trans_str) <= 2:
+ # Allow if it contains control codes or is just a replacement word
+ if not re.search(r'\\[A-Z]\[', trans_str) and not trans_str.isalnum():
+ invalid_indices.append(i)
+ reasons.append(f"Line{i+1}: Translation suspiciously short ('{trans_str}') for '{orig_str[:50]}...'")
+ continue
+
+ # Check 4: Runaway translation - translation is excessively long relative to original
+ # Catches cases where the model repeats words endlessly (e.g. "it hurts it hurts it hurts...")
+ ratio_limit = max(len(orig_str) * 8, 120)
+ if len(orig_str) > 10 and len(trans_str) > ratio_limit:
+ invalid_indices.append(i)
+ reasons.append(f"Line{i+1}: Runaway translation (output {len(trans_str)} chars vs input {len(orig_str)} chars) for '{orig_str[:50]}...'")
+ continue
+ # Absolute cap: garbage outputs that are not caught by ratio alone
+ if len(trans_str) > 4000 and len(trans_str) > len(orig_str) * 3:
+ invalid_indices.append(i)
+ reasons.append(f"Line{i+1}: Runaway translation (output {len(trans_str)} chars exceeds cap) for '{orig_str[:50]}...'")
+ continue
+
+ # Check 5: Same character repeated many times (common API glitch / broken JSON tail)
+ if re.search(r"(.)\1{44,}", trans_str):
+ invalid_indices.append(i)
+ reasons.append(f"Line{i+1}: Excessive character repetition (possible model glitch) in translation")
+ continue
+
+ is_valid = len(invalid_indices) == 0
+ return is_valid, invalid_indices, reasons
+
+# Load .env, strip accidental whitespace, set base URL / org / API key.
+# Gemini/Mistral use their endpoints only when no custom API URL is set.
+load_dotenv()
+api_provider = os.getenv("API_PROVIDER", "openai").lower()
+env_api = os.getenv("api", "").strip()
+if api_provider == "gemini" and not env_api:
+ # Use Google Generative Language compatibility endpoint only as fallback.
+ openai.base_url = "https://generativelanguage.googleapis.com/v1beta/openai/"
+ openai.organization = None
+elif api_provider == "mistral" and not env_api:
+ openai.base_url = "https://api.mistral.ai/v1/"
+ openai.organization = None
+else:
+ if env_api:
+ openai.base_url = _normalize_openai_base_url(env_api)
+ # Support both 'organization' (gui/.env.example) and legacy 'org' names
+ org = os.getenv("organization") or os.getenv("org")
+ if org:
+ openai.organization = org.strip()
+
+# Always set API key from 'key' env var (trim whitespace)
+openai.api_key = os.getenv("key", "").strip()
+
+# Translation cache management
+CACHE_FILE = Path("log/translation_cache.json")
+CACHE_LOCK_FILE = Path("log/translation_cache.lock")
+CACHE_LOCK = threading.RLock()
+CACHE_PENDING_MARKER = "__translation_pending__"
+CACHE_PENDING_TTL = 600
+CACHE_WAIT_INTERVAL = 0.25
+_cache = None
+
+@contextmanager
+def _translation_cache_file_lock():
+ """Cross-process lock for translation_cache.json."""
+ CACHE_LOCK_FILE.parent.mkdir(parents=True, exist_ok=True)
+ with open(CACHE_LOCK_FILE, "a+b") as lock_file:
+ if os.name == "nt":
+ import msvcrt
+ lock_file.seek(0)
+ msvcrt.locking(lock_file.fileno(), msvcrt.LK_LOCK, 1)
+ try:
+ yield
+ finally:
+ lock_file.seek(0)
+ msvcrt.locking(lock_file.fileno(), msvcrt.LK_UNLCK, 1)
+ else:
+ import fcntl
+ fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX)
+ try:
+ yield
+ finally:
+ fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN)
+
+def _read_cache_from_disk():
+ """Read the disk cache; return an empty dict if it is unavailable."""
+ try:
+ if CACHE_FILE.exists():
+ with open(CACHE_FILE, "r", encoding="utf-8") as f:
+ data = json.load(f)
+ return data if isinstance(data, dict) else {}
+ except Exception:
+ pass
+ return {}
+
+def _write_cache_to_disk(cache):
+ """Atomically write the cache using a process/thread-unique temp file."""
+ CACHE_FILE.parent.mkdir(parents=True, exist_ok=True)
+ tmp_file = CACHE_FILE.with_name(
+ f"{CACHE_FILE.name}.{os.getpid()}.{threading.get_ident()}.tmp"
+ )
+ with open(tmp_file, "w", encoding="utf-8") as f:
+ json.dump(cache, f, ensure_ascii=False, indent=2)
+ tmp_file.replace(CACHE_FILE)
+
+def _is_pending_cache_entry(value):
+ return isinstance(value, dict) and value.get(CACHE_PENDING_MARKER) is True
+
+def _is_stale_pending_cache_entry(value):
+ if not _is_pending_cache_entry(value):
+ return False
+ try:
+ return time.time() - float(value.get("time", 0)) > CACHE_PENDING_TTL
+ except Exception:
+ return True
+
+def _is_own_pending_cache_entry(value):
+ return (
+ _is_pending_cache_entry(value)
+ and value.get("pid") == os.getpid()
+ and value.get("thread") == threading.get_ident()
+ )
+
+def _pending_cache_entry():
+ return {
+ CACHE_PENDING_MARKER: True,
+ "pid": os.getpid(),
+ "thread": threading.get_ident(),
+ "time": time.time(),
+ }
+
+def _merge_translation_caches(base, overlay):
+ """Merge cache dictionaries while never replacing a translation with pending."""
+ merged = dict(base or {})
+ for key, value in (overlay or {}).items():
+ existing = merged.get(key)
+ if _is_pending_cache_entry(value) and existing is not None:
+ if not _is_pending_cache_entry(existing):
+ continue
+ if not _is_stale_pending_cache_entry(existing):
+ continue
+ merged[key] = value
+ return merged
+
+def clear_cache():
+ """Clear the translation cache (called at start of each run)"""
+ global _cache
+ with CACHE_LOCK:
+ _cache = {}
+ with _translation_cache_file_lock():
+ try:
+ if CACHE_FILE.exists():
+ CACHE_FILE.unlink()
+ except Exception:
+ pass
+
+def load_cache():
+ """Load the translation cache from disk."""
+ global _cache
+ with CACHE_LOCK:
+ with _translation_cache_file_lock():
+ disk_cache = _read_cache_from_disk()
+ if _cache:
+ disk_cache = _merge_translation_caches(disk_cache, _cache)
+ _cache = disk_cache
+ return _cache
+
+def save_cache():
+ """Save the translation cache to disk, preserving entries from other workers."""
+ global _cache
+ if _cache is None:
+ return
+
+ with CACHE_LOCK:
+ try:
+ with _translation_cache_file_lock():
+ disk_cache = _read_cache_from_disk()
+ disk_cache = _merge_translation_caches(disk_cache, _cache)
+ _cache = disk_cache
+ _write_cache_to_disk(_cache)
+ except Exception:
+ pass
+
+def get_cache_key(payload, language):
+ """Generate a cache key for a payload (can be single string or JSON batch)"""
+ # Use hash to keep keys short but unique
+ payload_str = str(payload) if payload is not None else ""
+ combined = f"{payload_str}|{language}"
+ return hashlib.md5(combined.encode("utf-8")).hexdigest()
+
+def get_cached_translation(payload, language):
+ """Get cached translation if it exists"""
+ global _cache
+ key = get_cache_key(payload, language)
+ while True:
+ with CACHE_LOCK:
+ with _translation_cache_file_lock():
+ cache = _read_cache_from_disk()
+ if _cache:
+ cache = _merge_translation_caches(cache, _cache)
+
+ entry = cache.get(key)
+ if (
+ entry is None
+ or _is_stale_pending_cache_entry(entry)
+ or _is_own_pending_cache_entry(entry)
+ ):
+ cache[key] = _pending_cache_entry()
+ _cache = cache
+ _write_cache_to_disk(cache)
+ return None
+
+ _cache = cache
+ if not _is_pending_cache_entry(entry):
+ return entry
+
+ time.sleep(CACHE_WAIT_INTERVAL)
+
+def cache_translation(payload, translation, language):
+ """Cache a translation payload and its response"""
+ global _cache
+ key = get_cache_key(payload, language)
+
+ with CACHE_LOCK:
+ with _translation_cache_file_lock():
+ cache = _read_cache_from_disk()
+ if _cache:
+ cache = _merge_translation_caches(cache, _cache)
+ cache[key] = translation
+ _cache = cache
+ _write_cache_to_disk(cache)
+
+
+# Variable translation map (code 122 <-> code 111 consistency)
+VAR_MAP_FILE = Path("log/var_translation_map.json")
+VAR_MAP_LOCK = threading.Lock()
+_var_map = None
+
+def clear_var_map():
+ """Clear the variable translation map (called at start of each run)"""
+ global _var_map
+ with VAR_MAP_LOCK:
+ _var_map = {}
+ try:
+ if VAR_MAP_FILE.exists():
+ VAR_MAP_FILE.unlink()
+ except Exception:
+ pass
+
+def _load_var_map():
+ """Load the variable translation map from disk (always re-reads to pick up
+ entries written by other subprocesses)."""
+ global _var_map
+ _var_map = {}
+ try:
+ if VAR_MAP_FILE.exists():
+ with open(VAR_MAP_FILE, "r", encoding="utf-8") as f:
+ _var_map = json.load(f)
+ except Exception:
+ _var_map = {}
+ return _var_map
+
+def _save_var_map():
+ """Save the variable translation map to disk.
+ Re-reads the file first and merges so entries from other subprocesses
+ are never lost."""
+ global _var_map
+ if _var_map is None:
+ return
+ try:
+ VAR_MAP_FILE.parent.mkdir(parents=True, exist_ok=True)
+ # Re-read the on-disk version and merge our entries on top
+ disk_map = {}
+ try:
+ if VAR_MAP_FILE.exists():
+ with open(VAR_MAP_FILE, "r", encoding="utf-8") as f:
+ disk_map = json.load(f)
+ except Exception:
+ disk_map = {}
+ disk_map.update(_var_map)
+ _var_map = disk_map
+ tmp_file = VAR_MAP_FILE.with_suffix(".tmp")
+ with open(tmp_file, "w", encoding="utf-8") as f:
+ json.dump(_var_map, f, ensure_ascii=False, indent=2)
+ tmp_file.replace(VAR_MAP_FILE)
+ except Exception:
+ pass
+
+def get_var_translation(original):
+ """Look up a cached variable translation. Returns the translation or None."""
+ with VAR_MAP_LOCK:
+ m = _load_var_map()
+ return m.get(original)
+
+def set_var_translation(original, translated):
+ """Store a variable translation and persist to disk.
+ Skips if the translation is identical to the original (untranslated).
+ """
+ if original == translated:
+ return
+ with VAR_MAP_LOCK:
+ m = _load_var_map()
+ m[original] = translated
+ _save_var_map()
+
+def set_var_translations_batch(pairs):
+ """Store multiple variable translations at once and persist to disk.
+ pairs: list of (original, translated) tuples
+ Skips pairs where the translation is identical to the original (untranslated).
+ """
+ with VAR_MAP_LOCK:
+ m = _load_var_map()
+ for original, translated in pairs:
+ if original != translated:
+ m[original] = translated
+ _save_var_map()
+
+
+# ===== Anthropic Message Batches (50% off all token usage) =====
+# Batch integration by Len — two-pass collect/consume flow; see README Credits.
+# Batch translation is a two-pass flow driven by the batch phase (kept in the
+# BATCH_PHASE env var so GUI subprocesses inherit it):
+# collect: translateAI builds each cache-missed request (byte-identical to a
+# live request, including the cached system block) and queues it
+# instead of calling the API. Text is left untranslated.
+# consume: translateAI feeds the fetched batch responses through the normal
+# validation/restore path; anything missing or invalid falls back
+# to the live API automatically.
+# Between the passes, submit/poll/fetch the queue with runTranslationBatches().
+BATCH_QUEUE_FILE = Path("log/batch_requests.json")
+BATCH_STATE_FILE = Path("log/batch_state.json")
+BATCH_RESULTS_FILE = Path("log/batch_results.json")
+BATCH_LOCK_FILE = Path("log/batch_files.lock")
+BATCH_LOCK = threading.RLock()
+# API limits are 100,000 requests / 256 MB per batch; stay safely under both.
+BATCH_MAX_REQUESTS = 100_000
+BATCH_MAX_BYTES = 200 * 1024 * 1024
+
+_batch_phase = None
+_batch_results = None # in-memory copy of BATCH_RESULTS_FILE (read-only during consume)
+_batch_queue_pending = {} # process-local queue entries not yet flushed to disk
+
+
+def set_batch_phase(phase):
+ """Set the batch phase ('collect', 'consume' or None) for this process and
+ any subprocesses it spawns (the GUI runs one per file)."""
+ global _batch_phase, _batch_results
+ _batch_phase = phase if phase in ("collect", "consume") else None
+ if _batch_phase:
+ os.environ["BATCH_PHASE"] = _batch_phase
+ else:
+ os.environ.pop("BATCH_PHASE", None)
+ _batch_results = None # phase change invalidates the in-memory results copy
+
+
+def get_batch_phase():
+ """Current batch phase, or None when batch translation is off."""
+ if _batch_phase:
+ return _batch_phase
+ phase = os.getenv("BATCH_PHASE", "").strip().lower()
+ return phase if phase in ("collect", "consume") else None
+
+
+@contextmanager
+def _batch_file_lock():
+ """Cross-process lock for the batch queue/state/results files."""
+ BATCH_LOCK_FILE.parent.mkdir(parents=True, exist_ok=True)
+ with open(BATCH_LOCK_FILE, "a+b") as lock_file:
+ if os.name == "nt":
+ import msvcrt
+ lock_file.seek(0)
+ msvcrt.locking(lock_file.fileno(), msvcrt.LK_LOCK, 1)
+ try:
+ yield
+ finally:
+ lock_file.seek(0)
+ msvcrt.locking(lock_file.fileno(), msvcrt.LK_UNLCK, 1)
+ else:
+ import fcntl
+ fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX)
+ try:
+ yield
+ finally:
+ fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN)
+
+
+def _read_batch_file(path):
+ """Read a batch JSON file; return an empty dict if it is unavailable."""
+ try:
+ if path.exists():
+ with open(path, "r", encoding="utf-8") as f:
+ data = json.load(f)
+ return data if isinstance(data, dict) else {}
+ except Exception:
+ pass
+ return {}
+
+
+def _write_batch_file(path, data):
+ """Atomically write a batch JSON file (no indent — queues can be large)."""
+ path.parent.mkdir(parents=True, exist_ok=True)
+ tmp_file = path.with_name(f"{path.name}.{os.getpid()}.{threading.get_ident()}.tmp")
+ with open(tmp_file, "w", encoding="utf-8") as f:
+ json.dump(data, f, ensure_ascii=False)
+ tmp_file.replace(path)
+
+
+def peek_cached_translation(payload, language):
+ """Cache lookup that never blocks or writes a pending marker.
+
+ The collect pass uses this instead of get_cached_translation so abandoned
+ pending markers can't stall the consume pass for CACHE_PENDING_TTL."""
+ key = get_cache_key(payload, language)
+ with CACHE_LOCK:
+ with _translation_cache_file_lock():
+ cache = _read_cache_from_disk()
+ if _cache:
+ cache = _merge_translation_caches(cache, _cache)
+ entry = cache.get(key)
+ if entry is None or _is_pending_cache_entry(entry):
+ return None
+ return entry
+
+
+def queue_batch_request(payload, language, params):
+ """Queue one Batches API request during the collect pass.
+
+ Deduped by the same key the translation cache uses, so identical payloads
+ across files are only paid for once. Entries are buffered in memory and
+ merged to disk by flush_batch_queue() at the end of each translateAI call.
+ """
+ key = get_cache_key(payload, language)
+ with BATCH_LOCK:
+ _batch_queue_pending[key] = {"payload": payload, "language": language, "params": params}
+ return key
+
+
+def flush_batch_queue():
+ """Merge this process's pending queue entries into the on-disk queue."""
+ global _batch_queue_pending
+ with BATCH_LOCK:
+ if not _batch_queue_pending:
+ return
+ pending, _batch_queue_pending = _batch_queue_pending, {}
+ try:
+ with _batch_file_lock():
+ queue = _read_batch_file(BATCH_QUEUE_FILE)
+ for key, entry in pending.items():
+ queue.setdefault(key, entry)
+ _write_batch_file(BATCH_QUEUE_FILE, queue)
+ except Exception:
+ _batch_queue_pending.update(pending) # keep entries for the next flush
+
+
+def take_batch_result(payload, language):
+ """Return the fetched batch response dict for a payload, or None.
+
+ The results file is loaded once per process — it is written before the
+ consume pass starts and never changes mid-consume."""
+ global _batch_results
+ if _batch_results is None:
+ with BATCH_LOCK:
+ if _batch_results is None:
+ with _batch_file_lock():
+ _batch_results = _read_batch_file(BATCH_RESULTS_FILE)
+ return _batch_results.get(get_cache_key(payload, language))
+
+
+def pendingBatchRequests():
+ """Number of queued batch requests (call after the collect pass)."""
+ flush_batch_queue()
+ with _batch_file_lock():
+ return len(_read_batch_file(BATCH_QUEUE_FILE))
+
+
+def batchRunState():
+ """'submitted' when a batch is still in flight, 'fetched' when results are
+ waiting to be consumed, else None. Lets an interrupted batch run resume
+ instead of re-collecting and paying for a second submission."""
+ with _batch_file_lock():
+ if _read_batch_file(BATCH_STATE_FILE).get("batches"):
+ return "submitted"
+ if _read_batch_file(BATCH_RESULTS_FILE):
+ return "fetched"
+ return None
+
+
+def clearBatchFiles():
+ """Remove queue/state/results left over from any previous batch run."""
+ global _batch_results, _batch_queue_pending
+ with BATCH_LOCK:
+ _batch_results = None
+ _batch_queue_pending = {}
+ with _batch_file_lock():
+ for path in (BATCH_QUEUE_FILE, BATCH_STATE_FILE, BATCH_RESULTS_FILE):
+ try:
+ if path.exists():
+ path.unlink()
+ except Exception:
+ pass
+
+
+def _get_anthropic_client():
+ key = os.getenv("key", "").strip()
+ if not key:
+ raise Exception("Batch translation requires the 'key' env var (see .env).")
+ return anthropic.Anthropic(api_key=key)
+
+
+def estimateBatchCost(model=None):
+ """Print a cost estimate for the queued batch requests and return it.
+
+ Cached-prefix accounting mirrors what Anthropic bills: each distinct cached
+ prefix is written once (2x input rate at the 1h TTL) and read by every other
+ request that shares it (0.10x); everything is then halved by the batch
+ discount. Cache hits inside a batch are best-effort, so the no-cache batch
+ figure is the worst-case bound.
+ """
+ flush_batch_queue()
+ with _batch_file_lock():
+ queue = _read_batch_file(BATCH_QUEUE_FILE)
+ if not queue:
+ print("[BATCH] No batch requests queued.", flush=True)
+ return None
+
+ enc = tiktoken.encoding_for_model("gpt-4")
+ prefix_count = {} # cached prefix text -> how many requests reuse it
+ prefix_tokens = {} # cached prefix text -> token count
+ dynamic_tokens = 0
+ output_tokens = 0
+ models = set()
+ for entry in queue.values():
+ params = entry.get("params", {})
+ if params.get("model"):
+ models.add(params["model"])
+ blocks = params.get("system") or []
+ cut = 0 # split system blocks at the cache breakpoint (0 = nothing cached)
+ for i, b in enumerate(blocks):
+ if "cache_control" in b:
+ cut = i + 1
+ break
+ prefix = "".join(b.get("text", "") for b in blocks[:cut])
+ dyn = "".join(b.get("text", "") for b in blocks[cut:])
+ if prefix:
+ prefix_count[prefix] = prefix_count.get(prefix, 0) + 1
+ if prefix not in prefix_tokens:
+ prefix_tokens[prefix] = len(enc.encode(prefix))
+ msg_text = "\n".join(str(m.get("content", "")) for m in params.get("messages", []))
+ dynamic_tokens += len(enc.encode(dyn)) + len(enc.encode(msg_text)) + 8
+ # Output heuristic mirrors countTokens(): payload tokens x 2.5 covers
+ # the echoed JSON scaffold plus EN expansion.
+ output_tokens += round(len(enc.encode(str(entry.get("payload", "")))) * 2.5)
+
+ est_model = model or next(iter(models), None) or os.getenv("model", "")
+ # Use Anthropic's count_tokens for the exact cached-prefix size when possible.
+ if prefix_tokens:
+ try:
+ client = _get_anthropic_client()
+ for prefix in list(prefix_tokens.keys()):
+ resp = client.messages.count_tokens(
+ model=est_model,
+ system=[{"type": "text", "text": prefix}],
+ messages=[{"role": "user", "content": "x"}],
+ )
+ prefix_tokens[prefix] = resp.input_tokens
+ except Exception:
+ pass # tiktoken estimate already in place
+
+ cache_write_tok = sum(prefix_tokens.values())
+ cache_read_tok = sum(prefix_tokens[p] * (prefix_count[p] - 1) for p in prefix_count)
+ raw_input_tok = sum(prefix_tokens[p] * prefix_count[p] for p in prefix_count) + dynamic_tokens
+
+ pricing = getPricingConfig(est_model)
+ in_rate = pricing["inputAPICost"] / 1_000_000
+ out_rate = pricing["outputAPICost"] / 1_000_000
+
+ # Batch = 50% off every token, including cache writes (2x, 1h TTL) and reads (0.10x).
+ batch_cached = (cache_write_tok * in_rate * 2.00
+ + cache_read_tok * in_rate * 0.10
+ + dynamic_tokens * in_rate
+ + output_tokens * out_rate) * 0.50
+ batch_nocache = (raw_input_tok * in_rate + output_tokens * out_rate) * 0.50
+ live_cost = raw_input_tok * in_rate + output_tokens * out_rate
+
+ n_reread = sum(prefix_count.values()) - len(prefix_count)
+ print(f"[BATCH] {len(queue)} requests queued for {est_model}", flush=True)
+ print(f"[BATCH] cached prefix: {cache_write_tok:,} tokens (written once, re-read by {n_reread:,} requests)", flush=True)
+ print(f"[BATCH] dynamic input: {dynamic_tokens:,} tokens | estimated output: {output_tokens:,} tokens", flush=True)
+ print(f"[BATCH] estimated cost: ${batch_cached:.2f} (batch + prompt cache)", flush=True)
+ print(f"[BATCH] ${batch_nocache:.2f} (batch, worst case no cache hits)", flush=True)
+ print(f"[BATCH] ${live_cost:.2f} (live API, no batch discount)", flush=True)
+ return {
+ "requests": len(queue),
+ "model": est_model,
+ "cache_write_tokens": cache_write_tok,
+ "cache_read_tokens": cache_read_tok,
+ "dynamic_tokens": dynamic_tokens,
+ "output_tokens": output_tokens,
+ "batch_cached_cost": batch_cached,
+ "batch_nocache_cost": batch_nocache,
+ "live_cost": live_cost,
+ }
+
+
+def submitTranslationBatches():
+ """Submit the queued requests to the Anthropic Message Batches API.
+
+ Splits at the API limits and saves the custom_id -> cache-key mapping so
+ fetchTranslationBatches can route results back. Returns the batch ids."""
+ flush_batch_queue()
+ with _batch_file_lock():
+ queue = _read_batch_file(BATCH_QUEUE_FILE)
+ if not queue:
+ print("[BATCH] No batch requests queued.", flush=True)
+ return []
+
+ client = _get_anthropic_client()
+ batches = []
+ requests, id_map, size = [], {}, 0
+
+ def _submit():
+ nonlocal requests, id_map, size
+ if not requests:
+ return
+ batch = client.messages.batches.create(requests=requests)
+ batches.append({"id": batch.id, "custom_ids": id_map})
+ print(f"[BATCH] submitted {batch.id} ({len(requests)} requests)", flush=True)
+ requests, id_map, size = [], {}, 0
+
+ for i, (key, entry) in enumerate(queue.items()):
+ custom_id = f"req-{i:06d}"
+ params = entry["params"]
+ requests.append({"custom_id": custom_id, "params": params})
+ id_map[custom_id] = key
+ size += len(json.dumps(params, ensure_ascii=False).encode("utf-8"))
+ if len(requests) >= BATCH_MAX_REQUESTS or size >= BATCH_MAX_BYTES:
+ _submit()
+ _submit()
+
+ with BATCH_LOCK:
+ with _batch_file_lock():
+ _write_batch_file(BATCH_STATE_FILE, {"batches": batches})
+ return [b["id"] for b in batches]
+
+
+def checkTranslationBatches():
+ """Print the processing status of submitted batches. True when all ended."""
+ with _batch_file_lock():
+ state = _read_batch_file(BATCH_STATE_FILE)
+ if not state.get("batches"):
+ print("[BATCH] No submitted batches — submit the queue first.", flush=True)
+ return False
+ client = _get_anthropic_client()
+ all_ended = True
+ for info in state["batches"]:
+ b = client.messages.batches.retrieve(info["id"])
+ counts = getattr(b, "request_counts", None)
+ suffix = f" counts: {counts}" if counts else ""
+ print(f"[BATCH] {time.strftime('%H:%M:%S')} {b.id}: {b.processing_status}{suffix}", flush=True)
+ if b.processing_status != "ended":
+ all_ended = False
+ return all_ended
+
+
+def fetchTranslationBatches():
+ """Download finished batch results into the local results store.
+
+ Successes are stored keyed by the payload cache key for the consume pass;
+ errored/expired requests are reported and simply fall back to the live API
+ during consume. Returns (succeeded, errored) counts."""
+ global _batch_results
+ with _batch_file_lock():
+ state = _read_batch_file(BATCH_STATE_FILE)
+ if not state.get("batches"):
+ print("[BATCH] No submitted batches — nothing to fetch.", flush=True)
+ return 0, 0
+ client = _get_anthropic_client()
+ results, errored = {}, []
+ for info in state["batches"]:
+ id_map = info.get("custom_ids", {})
+ for r in client.messages.batches.results(info["id"]):
+ key = id_map.get(r.custom_id)
+ if key is None:
+ continue
+ res = r.result
+ if res.type != "succeeded":
+ detail = res.type
+ err = getattr(res, "error", None)
+ if err is not None:
+ inner = getattr(err, "error", err)
+ detail = f"{res.type} | {getattr(inner, 'type', '')}: {str(getattr(inner, 'message', '') or err)[:200]}"
+ errored.append((r.custom_id, detail))
+ continue
+ msg = res.message
+ text = "".join(getattr(b, "text", "") or "" for b in msg.content)
+ u = msg.usage
+ cr = getattr(u, "cache_read_input_tokens", 0) or 0
+ cw = getattr(u, "cache_creation_input_tokens", 0) or 0
+ inp = getattr(u, "input_tokens", 0) or 0
+ out = getattr(u, "output_tokens", 0) or 0
+ results[key] = {
+ "text": text,
+ # prompt_tokens matches _AnthropicCompat: total incl. cache fields.
+ "prompt_tokens": inp + cr + cw,
+ "completion_tokens": out,
+ "cache_read_input_tokens": cr,
+ "cache_creation_input_tokens": cw,
+ }
+ with BATCH_LOCK:
+ with _batch_file_lock():
+ merged = _read_batch_file(BATCH_RESULTS_FILE)
+ merged.update(results)
+ _write_batch_file(BATCH_RESULTS_FILE, merged)
+ # Queue and state are consumed; only the results store remains.
+ for path in (BATCH_QUEUE_FILE, BATCH_STATE_FILE):
+ try:
+ if path.exists():
+ path.unlink()
+ except Exception:
+ pass
+ _batch_results = None
+ print(f"[BATCH] fetched {len(results)} results ({len(errored)} errored).", flush=True)
+ for cid, why in errored[:20]:
+ print(f"[BATCH] ! {cid}: {why}", flush=True)
+ if len(errored) > 20:
+ print(f"[BATCH] ... ({len(errored) - 20} more)", flush=True)
+ return len(results), len(errored)
+
+
+def runTranslationBatches(poll=60):
+ """Submit the queue (unless already submitted), poll to completion, fetch."""
+ with _batch_file_lock():
+ state = _read_batch_file(BATCH_STATE_FILE)
+ if not state.get("batches"):
+ if not submitTranslationBatches():
+ return 0, 0
+ print(f"[BATCH] polling every {poll}s (Ctrl-C is safe — resume later with fetchTranslationBatches)...", flush=True)
+ while not checkTranslationBatches():
+ time.sleep(poll)
+ return fetchTranslationBatches()
+
+
+# ===== Mistral (la Plateforme) adaptive rate limiting =====
+# Mistral enforces independent request and token limits, both PER-MODEL.
+# Verified live against api.mistral.ai (the response headers are authoritative):
+# * x-ratelimit-limit-req-minute — requests/minute. PER-MODEL and varies a
+# lot: mistral-medium=25, mistral-small=50, codestral=125, ministral-3b=750.
+# Mistral's dashboard shows this /60 as "RPS" (25/min = 0.42 "RPS"), so the
+# effective rate spans well below AND above 1 req/sec. There is NO
+# per-second header; we divide req-minute by 60 to get the pacing interval.
+# * x-ratelimit-limit-tokens-minute — input+output throughput, a true minute
+# window, also per-model (e.g. 50k–1.3M).
+# * Tokens per month — overall cap (not paced here; surfaces as a 429).
+# The limiter paces requests >= 1/RPS apart (so a burst of file threads can't
+# overrun the per-minute request budget) AND charges them against a
+# minute-windowed token budget. It starts from a conservative seed, then the
+# live headers on the first response pin req_per_sec / tok_per_min to the exact
+# per-model values. One limiter is shared across all file threads (this tool
+# runs one model per session). Override the seeds with
+# mistralReqPerSec / mistralTokPerMin / mistralTokenHeadroom.
+_mistral_limiter = None
+_mistral_limiter_lock = threading.Lock()
+
+
+def _estimate_mistral_tokens(text):
+ # JP ~1 token/char with Mistral tokenizers; EN ~1 per 3.5 chars. Be pessimistic.
+ return int(len(str(text)) * 1.1) + 8
+
+
+def _header_int(headers, *names):
+ """First present header among names parsed as int, else None."""
+ for n in names:
+ v = headers.get(n)
+ if v is not None and str(v).strip() != "":
+ try:
+ return int(float(v))
+ except (TypeError, ValueError):
+ pass
+ return None
+
+
+def _header_float(headers, *names):
+ """First present header among names parsed as float, else None.
+
+ Used for the RPS limit header, which per-model is often FRACTIONAL
+ (e.g. 0.08, 0.42, 0.83) — parsing it as int would floor those to 0."""
+ for n in names:
+ v = headers.get(n)
+ if v is not None and str(v).strip() != "":
+ try:
+ return float(v)
+ except (TypeError, ValueError):
+ pass
+ return None
+
+
+class AdaptiveLimiter:
+ """Paces requests off live ratelimit headers.
+
+ Two dimensions, enforced together by acquire():
+ * requests: spaced >= min_interval (1/RPS) apart — Mistral's request cap
+ is per-second, so this prevents bursts from tripping it.
+ * tokens: a minute-windowed budget (input+output), with a headroom margin.
+ """
+
+ def __init__(self, req_per_sec, tok_per_min, headroom):
+ self.lock = threading.Lock()
+ self.req_per_sec = max(0.001, float(req_per_sec))
+ self.tok_per_min = tok_per_min
+ self.headroom = headroom # margin left under the TPM cap
+ self.min_interval = 1.0 / self.req_per_sec
+ self.next_request_at = time.monotonic()
+ self.tokens_remaining = tok_per_min
+ self.window_reset = time.monotonic() + 60
+
+ def acquire(self, est_tokens):
+ """Block until the request clears both the RPS pace and the TPM budget."""
+ while True:
+ with self.lock:
+ now = time.monotonic()
+ if now >= self.window_reset:
+ self.tokens_remaining = self.tok_per_min
+ self.window_reset = now + 60
+ # token gate
+ if self.tokens_remaining - est_tokens <= self.headroom:
+ sleep_for = max(0.05, self.window_reset - now)
+ # request-pace gate
+ elif now < self.next_request_at:
+ sleep_for = self.next_request_at - now
+ else:
+ # both gates clear — reserve this slot
+ self.next_request_at = max(now, self.next_request_at) + self.min_interval
+ self.tokens_remaining -= est_tokens
+ return
+ time.sleep(min(sleep_for, 5))
+
+ def update(self, headers):
+ """Sync budgets from the live ratelimit headers (authoritative).
+
+ Accepts both the per-second request headers and (for forward/back compat)
+ the older per-minute request header names; the token headers are minute."""
+ if not headers:
+ return
+ with self.lock:
+ # The authoritative request limit is x-ratelimit-limit-req-MINUTE
+ # (verified live: mistral-medium=25, ministral-3b=750, codestral=125;
+ # Mistral's dashboard "RPS" is just this number / 60). There is no
+ # per-second header. Convert to RPS for the pacing interval. A
+ # per-second header is still accepted first in case Mistral adds one.
+ limit_rps = _header_float(headers, "x-ratelimit-limit-req-second",
+ "x-ratelimit-limit-requests-second")
+ if limit_rps is None:
+ limit_rpm = _header_float(headers, "x-ratelimit-limit-req-minute",
+ "x-ratelimit-limit-requests-minute")
+ if limit_rpm is not None:
+ limit_rps = limit_rpm / 60.0
+ if limit_rps and limit_rps > 0:
+ self.req_per_sec = limit_rps
+ self.min_interval = 1.0 / self.req_per_sec
+ limit_tok = _header_int(headers, "x-ratelimit-limit-tokens-minute",
+ "x-ratelimit-limit-tokens")
+ if limit_tok and limit_tok > 0:
+ self.tok_per_min = limit_tok
+ rem_tok = _header_int(headers, "x-ratelimit-remaining-tokens-minute",
+ "x-ratelimit-remaining-tokens")
+ if rem_tok is not None:
+ self.tokens_remaining = rem_tok
+ # Remaining requests this minute -> also cap the pace so a fresh
+ # limiter that joins mid-window doesn't burst the leftover budget.
+ rem_req = _header_int(headers, "x-ratelimit-remaining-req-minute",
+ "x-ratelimit-remaining-requests-minute")
+ if rem_req is not None and rem_req <= 0:
+ # budget already spent this minute — hold off ~ a minute
+ self.next_request_at = max(self.next_request_at, time.monotonic() + 60.0)
+
+
+def _get_mistral_limiter():
+ global _mistral_limiter
+ with _mistral_limiter_lock:
+ if _mistral_limiter is None:
+ # mistralReqPerMin kept as a deprecated alias: a per-minute number is
+ # converted to RPS so old .env files keep pacing sanely.
+ rps_env = os.getenv("mistralReqPerSec")
+ if rps_env:
+ req_per_sec = float(rps_env)
+ elif os.getenv("mistralReqPerMin"):
+ req_per_sec = max(0.05, float(os.getenv("mistralReqPerMin")) / 60.0)
+ else:
+ # Per-model RPS varies and is often well under 1 (free-tier
+ # mistral-medium ~0.83, magistral ~0.08). Start conservative so
+ # the FIRST request doesn't over-burst; the live header from that
+ # first response then corrects req_per_sec to the model's exact
+ # value (which is why few large batches > many tiny ones).
+ req_per_sec = 0.5
+ _mistral_limiter = AdaptiveLimiter(
+ req_per_sec,
+ int(os.getenv("mistralTokPerMin", "50000") or 50000),
+ int(os.getenv("mistralTokenHeadroom", "4000") or 4000),
+ )
+ return _mistral_limiter
+
+
+def callMistral(params, retries=6):
+ """Call the Mistral chat completions endpoint with adaptive pacing.
+
+ Acquires from the shared limiter before each attempt, syncs budgets from
+ the live x-ratelimit headers after each response, honours Retry-After on
+ 429, backs off with jitter on 5xx/network errors, and downgrades
+ json_schema -> json_object for models without structured-output support.
+ """
+ limiter = _get_mistral_limiter()
+ est = sum(_estimate_mistral_tokens(m.get("content", "")) for m in params.get("messages", []))
+ est += params.get("max_tokens", 0) // 2
+ last_error = None
+ for attempt in range(retries + 1):
+ limiter.acquire(est)
+ try:
+ raw = openai.chat.completions.with_raw_response.create(**params)
+ response = raw.parse()
+ limiter.update(raw.headers)
+ return response
+ except RateLimitError as e:
+ last_error = e
+ resp = getattr(e, "response", None)
+ limiter.update(getattr(resp, "headers", None))
+ retry_after = None
+ try:
+ retry_after = float(resp.headers.get("retry-after"))
+ except (AttributeError, TypeError, ValueError):
+ pass
+ time.sleep(retry_after if retry_after is not None else min(60, 2 ** attempt + random.random() * 2))
+ except APIStatusError as e:
+ last_error = e
+ # Mistral rejects unsupported params with 400/422 — downgrade the
+ # structured-output format once and retry immediately.
+ if e.status_code in (400, 422) and (params.get("response_format") or {}).get("type") == "json_schema":
+ params = dict(params)
+ params["response_format"] = {"type": "json_object"}
+ continue
+ if e.status_code >= 500 and attempt < retries:
+ time.sleep(min(45, 2 ** attempt + random.random()))
+ continue
+ raise Exception(f"Mistral API error ({e.status_code}): {e}")
+ except APIConnectionError as e:
+ last_error = e
+ if attempt < retries:
+ time.sleep(min(45, 2 ** attempt + random.random()))
+ continue
+ raise Exception(f"Mistral connection error: {e}")
+ raise Exception(f"Mistral API failed after {retries + 1} attempts: {last_error}")
+
+
+class TranslationConfig:
+ """Configuration class to hold all translation settings"""
+
+ def __init__(self,
+ model=None,
+ language=None,
+ prompt=None,
+ vocab=None,
+ langRegex=None,
+ batchSize=None,
+ maxHistory=10,
+ estimateMode=False,
+ logFilePath="log/translationHistory.txt",
+ mismatchLogPath="log/mismatchHistory.txt"):
+
+ # Load from environment if not provided
+ self.model = model or os.getenv("model")
+ self.language = (language or os.getenv("language", "english")).capitalize()
+
+ # Load prompt and vocab files if not provided
+ if prompt is None:
+ try:
+ from util.paths import PROMPT_PATH, VOCAB_PATH
+ self.prompt = PROMPT_PATH.read_text(encoding="utf-8")
+ except FileNotFoundError:
+ self.prompt = ""
+ else:
+ self.prompt = prompt
+
+ if vocab is None:
+ try:
+ self.vocab = VOCAB_PATH.read_text(encoding="utf-8")
+ except FileNotFoundError:
+ self.vocab = ""
+ else:
+ self.vocab = vocab
+
+ # Set language regex (default is Japanese)
+ self.langRegex = langRegex or r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+"
+
+ # Set batch size — derive from pricing config unless explicitly supplied
+ if batchSize is None:
+ self.batchSize = getPricingConfig(self.model)["batchSize"]
+ else:
+ self.batchSize = batchSize
+
+ self.maxHistory = maxHistory
+ self.estimateMode = estimateMode
+ self.logFilePath = logFilePath
+ self.mismatchLogPath = mismatchLogPath
+
+
+_LITELLM_PRICING_URL = (
+ "https://raw.githubusercontent.com/BerriAI/litellm/main"
+ "/model_prices_and_context_window.json"
+)
+_PRICING_CACHE_FILE = Path("log/litellm_pricing.json")
+_PRICING_CACHE_TTL = 86_400 # 24 hours
+_pricing_db: dict | None = None
+_pricing_db_fetched_at: float = 0.0
+_pricing_db_lock = threading.Lock()
+_pricing_fetch_warned: bool = False # print fetch-failure warning at most once per session
+
+
+def _load_litellm_pricing() -> dict | None:
+ """Return the LiteLLM pricing DB, using a 24-hour disk cache."""
+ global _pricing_db, _pricing_db_fetched_at, _pricing_fetch_warned
+
+ with _pricing_db_lock:
+ now = time.time()
+
+ # In-memory cache still fresh
+ if _pricing_db is not None and (now - _pricing_db_fetched_at) < _PRICING_CACHE_TTL:
+ return _pricing_db
+
+ # Try disk cache
+ if _PRICING_CACHE_FILE.exists():
+ try:
+ disk = json.loads(_PRICING_CACHE_FILE.read_text(encoding="utf-8"))
+ if (now - disk.get("fetched_at", 0)) < _PRICING_CACHE_TTL:
+ _pricing_db = disk["prices"]
+ _pricing_db_fetched_at = disk["fetched_at"]
+ return _pricing_db
+ except Exception:
+ pass
+
+ # Fetch from GitHub
+ try:
+ with urllib.request.urlopen(_LITELLM_PRICING_URL, timeout=5) as resp:
+ data = json.loads(resp.read().decode("utf-8"))
+ _pricing_db = data
+ _pricing_db_fetched_at = now
+ _pricing_fetch_warned = False # reset if a later fetch succeeds
+ try:
+ _PRICING_CACHE_FILE.parent.mkdir(parents=True, exist_ok=True)
+ _PRICING_CACHE_FILE.write_text(
+ json.dumps({"fetched_at": now, "prices": data}),
+ encoding="utf-8",
+ )
+ except Exception:
+ pass
+ return _pricing_db
+ except Exception as fetch_err:
+ # No internet / GitHub unreachable — warn once, then fall back
+ if not _pricing_fetch_warned:
+ _pricing_fetch_warned = True
+ print(
+ f"[PRICING] Warning: Could not fetch live model pricing "
+ f"({fetch_err}). Cost estimates may be inaccurate — "
+ f"using built-in fallback prices.",
+ flush=True,
+ )
+ # Use stale disk cache if available
+ if _pricing_db is not None:
+ return _pricing_db
+ try:
+ disk = json.loads(_PRICING_CACHE_FILE.read_text(encoding="utf-8"))
+ _pricing_db = disk["prices"]
+ return _pricing_db
+ except Exception:
+ return None
+
+
+def _lookup_model_price(model: str):
+ """Look up (input_per_1M, output_per_1M) from the LiteLLM pricing DB.
+
+ Returns a (float, float) tuple or None if not found.
+ Matching priority:
+ 1. Exact key match
+ 2. Exact match on the model portion after a provider prefix (e.g. "deepseek/deepseek-chat")
+ 3. The user's model name is a prefix of a DB key (handles dated suffixes like -20241022)
+ 4. A DB key model-part is a prefix of the user's model name
+ """
+ db = _load_litellm_pricing()
+ if not db:
+ return None
+
+ model_lower = model.lower()
+
+ def _extract(entry):
+ inp = entry.get("input_cost_per_token")
+ out = entry.get("output_cost_per_token")
+ if inp is not None and out is not None:
+ return round(inp * 1_000_000, 6), round(out * 1_000_000, 6)
+ return None
+
+ # Pass 1: exact key
+ if model_lower in db:
+ result = _extract(db[model_lower])
+ if result:
+ return result
+
+ # Build a lookup of (stripped_key → original_key) for passes 2-4
+ stripped: list[tuple[str, str]] = []
+ for key in db:
+ stripped.append((key.split("/")[-1].lower(), key))
+
+ # Pass 2: exact match on stripped key
+ for skey, orig in stripped:
+ if skey == model_lower:
+ result = _extract(db[orig])
+ if result:
+ return result
+
+ # Pass 3: model name is a prefix of the DB key (e.g. "claude-3-5-sonnet" matches
+ # "claude-3-5-sonnet-20241022")
+ candidates = [(skey, orig) for skey, orig in stripped if skey.startswith(model_lower)]
+ if candidates:
+ # Prefer the shortest (most generic) key
+ skey, orig = min(candidates, key=lambda x: len(x[0]))
+ result = _extract(db[orig])
+ if result:
+ return result
+
+ # Pass 4: DB key is a prefix of the model name (e.g. "gemini-2.0-flash" matches
+ # "gemini-2.0-flash-exp")
+ candidates = [(skey, orig) for skey, orig in stripped if model_lower.startswith(skey)]
+ if candidates:
+ skey, orig = max(candidates, key=lambda x: len(x[0])) # longest = most specific
+ result = _extract(db[orig])
+ if result:
+ return result
+
+ return None
+
+
+def getPricingConfig(model):
+ """
+ Get pricing configuration for a given model.
+
+ Args:
+ model: The model name string
+
+ Returns:
+ dict: Dictionary containing inputAPICost, outputAPICost, batchSize, and frequencyPenalty
+ """
+ # Try to resolve pricing from the LiteLLM community pricing DB first.
+ # This keeps costs accurate as providers update their prices without requiring
+ # a code change. Falls back to the hardcoded table below on failure.
+ live_price = _lookup_model_price(model)
+ if live_price:
+ inp, out = live_price
+ # Preserve model-specific batch / penalty overrides from the hardcoded table
+ # by still running through the if-chain but replacing the cost fields.
+ _live_override = {"inputAPICost": inp, "outputAPICost": out}
+ else:
+ _live_override = None
+
+ # Hardcoded fallback table — used for batchSize / frequencyPenalty tuning and
+ # as a cost fallback when the LiteLLM DB is unavailable.
+ # Batch Size: GPT-3.5 struggles past 15 lines; GPT-4 struggles past 50.
+ # If you get a MISMATCH LENGTH error, lower the batch size.
+ if "gpt-3.5" in model:
+ cfg = {"inputAPICost": 3.00, "outputAPICost": 5.00, "batchSize": 10, "frequencyPenalty": 0.2}
+ elif "gpt-4.1-mini" in model:
+ cfg = {"inputAPICost": 0.40, "outputAPICost": 1.60, "batchSize": 30, "frequencyPenalty": 0.05}
+ elif "gpt-4.1" in model:
+ cfg = {"inputAPICost": 2.00, "outputAPICost": 8.00, "batchSize": 30, "frequencyPenalty": 0.05}
+ elif "gpt-5" in model:
+ cfg = {"inputAPICost": 1.25, "outputAPICost": 10.00, "batchSize": 30, "frequencyPenalty": 0.05}
+ elif "deepseek" in model:
+ cfg = {"inputAPICost": 0.27, "outputAPICost": 1.10, "batchSize": 30, "frequencyPenalty": 0.05}
+ # Mistral — system prompt is resent per request (no prompt caching), so
+ # throughput/cost favor larger batches. Live LiteLLM pricing overrides these.
+ elif "mistral-large" in model or "pixtral-large" in model:
+ cfg = {"inputAPICost": 2.00, "outputAPICost": 6.00, "batchSize": 40, "frequencyPenalty": 0.0}
+ elif "magistral-medium" in model:
+ cfg = {"inputAPICost": 2.00, "outputAPICost": 5.00, "batchSize": 40, "frequencyPenalty": 0.0}
+ elif "mistral-medium-3.5" in model or "mistral-medium-3-5" in model or "mistral-medium-26" in model:
+ # Medium 3.5 (v26.04) — also matches dated 26xx ids
+ cfg = {"inputAPICost": 1.50, "outputAPICost": 7.50, "batchSize": 40, "frequencyPenalty": 0.0}
+ elif "mistral-medium" in model:
+ # Medium 3 / 3.1 (what -latest still points at)
+ cfg = {"inputAPICost": 0.40, "outputAPICost": 2.00, "batchSize": 40, "frequencyPenalty": 0.0}
+ elif "magistral-small" in model:
+ cfg = {"inputAPICost": 0.50, "outputAPICost": 1.50, "batchSize": 40, "frequencyPenalty": 0.0}
+ elif "mistral-small" in model:
+ cfg = {"inputAPICost": 0.10, "outputAPICost": 0.30, "batchSize": 40, "frequencyPenalty": 0.0}
+ elif "ministral" in model or "open-mistral" in model or "mistral-nemo" in model:
+ cfg = {"inputAPICost": 0.10, "outputAPICost": 0.10, "batchSize": 40, "frequencyPenalty": 0.0}
+ elif "codestral" in model:
+ cfg = {"inputAPICost": 0.30, "outputAPICost": 0.90, "batchSize": 40, "frequencyPenalty": 0.0}
+ elif "mistral" in model or "pixtral" in model:
+ cfg = {"inputAPICost": 2.00, "outputAPICost": 6.00, "batchSize": 40, "frequencyPenalty": 0.0}
+ elif "claude-opus-4-5" in model or "claude-opus-4-6" in model:
+ cfg = {"inputAPICost": 5.00, "outputAPICost": 25.00, "batchSize": 30, "frequencyPenalty": 0.05}
+ elif "claude-opus" in model or model == "claude-3-opus":
+ # Opus 4, 4.1, 3 — $15/$75
+ cfg = {"inputAPICost": 15.00, "outputAPICost": 75.00, "batchSize": 30, "frequencyPenalty": 0.05}
+ elif "claude-haiku-4-5" in model or "claude-haiku-4-6" in model:
+ cfg = {"inputAPICost": 1.00, "outputAPICost": 5.00, "batchSize": 30, "frequencyPenalty": 0.05}
+ elif "claude-haiku-3-5" in model:
+ cfg = {"inputAPICost": 0.80, "outputAPICost": 4.00, "batchSize": 30, "frequencyPenalty": 0.05}
+ elif "claude-3-haiku" in model:
+ cfg = {"inputAPICost": 0.25, "outputAPICost": 1.25, "batchSize": 30, "frequencyPenalty": 0.05}
+ elif "haiku" in model:
+ # Unknown haiku version — use current flagship pricing as best guess
+ cfg = {"inputAPICost": 1.00, "outputAPICost": 5.00, "batchSize": 30, "frequencyPenalty": 0.05}
+ elif "sonnet" in model or "claude" in model:
+ cfg = {"inputAPICost": 3.00, "outputAPICost": 15.00, "batchSize": 30, "frequencyPenalty": 0.05}
+ elif "gemini-2.0-flash-lite" in model:
+ cfg = {"inputAPICost": 0.075, "outputAPICost": 0.30, "batchSize": 30, "frequencyPenalty": 0.0}
+ elif "gemini-2.0-flash" in model:
+ cfg = {"inputAPICost": 0.10, "outputAPICost": 0.40, "batchSize": 30, "frequencyPenalty": 0.0}
+ elif "gemini-2.5-flash-lite" in model:
+ cfg = {"inputAPICost": 0.10, "outputAPICost": 0.40, "batchSize": 30, "frequencyPenalty": 0.0}
+ elif "gemini-2.5-flash" in model:
+ cfg = {"inputAPICost": 0.30, "outputAPICost": 2.50, "batchSize": 30, "frequencyPenalty": 0.0}
+ elif "gemini-2.5-pro" in model:
+ cfg = {"inputAPICost": 1.25, "outputAPICost": 10.00, "batchSize": 30, "frequencyPenalty": 0.0}
+ else:
+ cfg = {
+ "inputAPICost": float(os.getenv("input_cost", 3.00)),
+ "outputAPICost": float(os.getenv("output_cost", 6.00)),
+ "batchSize": int(os.getenv("batchsize", 10)),
+ "frequencyPenalty": float(os.getenv("frequency_penalty", 0.2)),
+ }
+
+ # Apply live pricing from LiteLLM if available — keeps costs up-to-date
+ # without requiring code changes when providers reprice their models.
+ if _live_override:
+ cfg.update(_live_override)
+
+ return cfg
+
+
+def batchList(inputList, batchSize):
+ """Split a list into batches of specified size"""
+ if not isinstance(batchSize, int) or batchSize <= 0:
+ raise ValueError("batchSize must be a positive integer")
+
+ return [inputList[i : i + batchSize] for i in range(0, len(inputList), batchSize)]
+
+
+def parseVocabWithCategories(vocabText):
+ """Parse vocabulary text and extract terms with their categories."""
+ pairs = []
+ seen = set()
+ currentCategory = None
+
+ for line in vocabText.splitlines():
+ line = line.strip()
+ if not line or line.startswith('```') or line.startswith('Here are some vocabulary'):
+ continue
+
+ # Check if this is a category header
+ if line.startswith('#'):
+ currentCategory = line
+ continue
+
+ # Parse vocabulary term - extract both Japanese and English parts.
+ # Rich entries may continue after the first parenthesized translation,
+ # e.g. "サンク (Sank) - Male; protagonist..."; only "Sank" is the match key.
+ paren_match = re.match(r'^(.+?)\s*\(([^()]*)\)', line)
+ dash_match = re.match(r'^(.+?)\s+[–-]\s+(.+)$', line)
+ if paren_match:
+ japanese_term = paren_match.group(1).strip()
+ english_term = paren_match.group(2).strip()
+
+ # Create a tuple with both terms for matching
+ term_pair = (japanese_term, english_term)
+ if term_pair not in seen:
+ pairs.append((term_pair, line, currentCategory))
+ seen.add(term_pair)
+ elif dash_match:
+ japanese_term = dash_match.group(1).strip()
+ english_term = dash_match.group(2).strip()
+
+ # Create a tuple with both terms for matching
+ term_pair = (japanese_term, english_term)
+ if term_pair not in seen:
+ pairs.append((term_pair, line, currentCategory))
+ seen.add(term_pair)
+ elif line and not line.startswith('#'):
+ # Fallback for lines without parentheses - treat as single term
+ term = line.strip()
+ if term and term not in seen:
+ pairs.append((term, line, currentCategory))
+ seen.add(term)
+
+ return pairs
+
+
+def _japanese_term_in_text(term, text):
+ """
+ Check if a Japanese term appears in text as a standalone word, not as a
+ substring of a longer run of the same script (katakana/hiragana/kanji).
+ E.g. 'キス' will NOT match inside 'テキスト' because both neighbours are katakana.
+ Falls back to plain substring check for non-Japanese or mixed terms.
+ """
+ if term not in text:
+ return False
+ KATAKANA = r'ァ-ヴーｦ-ﾟ'
+ HIRAGANA = r'ぁ-ゔ'
+ KANJI = r'一-龠'
+ if re.search(rf'[{KATAKANA}]', term) and not re.search(rf'[{HIRAGANA}{KANJI}]', term):
+ pattern = rf'(?= 500:
+ raise Exception(f"API server error ({e.status_code}) - retrying... Error: {e}")
+ elif e.status_code == 400 and formatType == "json" and "json_schema" in str(responseFormat):
+ # Only fall back to json_object if the error is NOT "Input should be 'json_schema'"
+ # (that message means json_schema IS required and json_object would also be rejected)
+ if "input should be 'json_schema'" in str(e).lower() or "input should be \"json_schema\"" in str(e).lower():
+ raise Exception(f"API status error ({e.status_code}): {e}")
+ # Provider doesn't support json_schema (e.g. Claude) — fall back to json_object
+ responseFormat = {"type": "json_object"}
+ params["response_format"] = responseFormat
+ try:
+ response = openai.chat.completions.create(**params)
+ except APIStatusError as fallback_error:
+ if fallback_error.status_code == 400 and "input should be 'json_schema'" in str(fallback_error).lower():
+ raise Exception(f"API requires json_schema response format but rejected the schema. Original error: {e}")
+ raise Exception(f"API call failed: {e}. Fallback also failed: {fallback_error}")
+ except Exception as fallback_error:
+ raise Exception(f"API call failed: {e}. Fallback also failed: {fallback_error}")
+ elif e.status_code == 400 and "input should be 'json_schema'" in str(e).lower():
+ # response_format.type was rejected (e.g. sent "text" or "json_object" to a model
+ # that only accepts json_schema). Remove response_format and retry with no constraint.
+ params.pop("response_format", None)
+ try:
+ response = openai.chat.completions.create(**params)
+ except Exception as fallback_error:
+ raise Exception(f"API call failed: {e}. Fallback also failed: {fallback_error}")
+ else:
+ raise Exception(f"API status error ({e.status_code}): {e}")
+ except (APIConnectionError, RateLimitError) as e:
+ # These should always be retried
+ raise Exception(f"API connection/rate limit error - retrying... Error: {e}")
+ except Exception as e:
+ # Check if it's a 404 error or other HTTP error that should be retried
+ error_str = str(e).lower()
+ if "404" in error_str or "not found" in error_str:
+ raise Exception(f"API returned 404 Not Found - check your API configuration. Original error: {e}")
+
+ # If structured output fails, fallback to json_object (unless the error
+ # explicitly states json_schema is required — falling back would just fail again)
+ if formatType == "json" and "json_schema" in str(responseFormat) and \
+ "input should be 'json_schema'" not in error_str:
+ responseFormat = {"type": "json_object"}
+ params["response_format"] = responseFormat
+ try:
+ response = openai.chat.completions.create(**params)
+ except Exception as fallback_error:
+ # If fallback also fails, raise the original error for retry
+ raise Exception(f"API call failed: {e}. Fallback also failed: {fallback_error}")
+ else:
+ raise e
+
+ # Validate response before returning
+ if not response or not hasattr(response, 'choices') or not response.choices:
+ raise Exception("API returned invalid or empty response - retrying...")
+
+ _write_request_debug_log(api_provider, params, getattr(response, "usage", None))
+ return response
+
+
+def cleanTranslatedText(translatedText, language):
+ """Clean and format translated text"""
+ placeholders = {
+ f"{language} Translation: ": "",
+ "Translation: ": "",
+ "っ": "",
+ "〜": "~",
+ "ッ": "",
+ "。": ".",
+ # Note: 「 and 」 are NOT replaced here — replacing them with ASCII " would
+ # corrupt raw JSON strings before extraction. They are handled per-line
+ # in _clean_extracted_line() after JSON parsing.
+ "—": "―",
+ "】": "]",
+ "【": "[",
+ "é": "e",
+ "’": "'",
+ "this guy": "this bastard",
+ "This guy": "This bastard",
+ "```json": "",
+ "```": "",
+ }
+
+ for target, replacement in placeholders.items():
+ translatedText = translatedText.replace(target, replacement)
+
+ # Remove Repeating Characters
+ pattern = re.compile(r"(.)\s*\1(?:\s*\1){" + str(20 - 1) + r",}")
+ translatedText = pattern.sub(lambda match: match.group(0).replace(" ", "")[:20], translatedText)
+
+ # Elongate Long Dashes (Since GPT Ignores them...)
+ translatedText = elongateCharacters(translatedText)
+ return translatedText
+
+
+def elongateCharacters(text):
+ """Replace ー sequences with elongated characters"""
+ # Define a pattern to match one character followed by two or more ー characters.
+ # The lookbehind is restricted to non-ー Japanese/CJK characters so that:
+ # - standalone ー separators (e.g. "ーーーーーーーーーー") are left untouched
+ # - ー sequences preceded by a JSON quote or other non-Japanese char are not corrupted
+ pattern = r"(?<=([\u3040-\u309F\u30A0-\u30FB\u30FD-\u30FF\u4E00-\u9FEF\uFF61-\uFF9F]))ー{2,}"
+
+ # Define a replacement function that elongates the captured character
+ def repl(match):
+ char = match.group(1) # The character before the ー sequence
+ count = len(match.group(0)) - 1 # Number of ー characters
+ return char * count # Replace ー sequence with the character repeated
+
+ # Use re.sub() to replace the pattern in the text
+ return re.sub(pattern, repl, text)
+
+
+def extractTranslation(translatedTextList, isList, pbar=None):
+ """Extract translation from JSON response.
+
+ This function is resilient to a few common model mistakes:
+ - Wraps output in code fences or outer quotes
+ - Uses smart quotes instead of straight quotes
+ - Inserts an extra leading quote in values (e.g. :""Word" -> :"Word")
+ - Trailing commas before } or ]
+
+ If strict JSON parsing fails, falls back to a regex-based extractor that
+ captures LineN values in numeric order.
+ """
+ s = str(translatedTextList or "").strip()
+
+ # Fast exit
+ if not s:
+ return None
+
+ # Remove code fences if present
+ if s.startswith("```"):
+ s = re.sub(r"^```(?:json)?\s*", "", s, flags=re.IGNORECASE)
+ s = re.sub(r"\s*```$", "", s)
+
+ # Trim wrapping quotes around the whole JSON blob (common in logs)
+ if len(s) >= 2 and s[0] == s[-1] and s[0] in {'"', "'"}:
+ # Only strip if it still looks like JSON inside
+ if s[1:2] == "{" and s[-2:-1] == "}":
+ s = s[1:-1]
+
+ # Normalize a broad set of Unicode “smart” quotes to ASCII equivalents.
+ translation_table = {
+ 0x201C: "'", # “ left double quotation mark
+ 0x201D: "'", # ” right double quotation mark
+ 0xFF02: "'", # ＂ fullwidth quotation mark
+
+ 0x2018: "'", # ‘ left single quotation mark
+ 0x2019: "'", # ’ right single quotation mark
+ 0x201B: "'", # ‛ single high-reversed-9 quotation mark
+ 0x02BC: "'", # ʼ modifier letter apostrophe
+ 0xFF07: "'", # ＇ fullwidth apostrophe
+ }
+ s = s.translate(translation_table)
+
+ # Remove trailing commas before object/array closures
+ s = re.sub(r",(\s*[}\]])", r"\1", s)
+
+ # Repair common doubled leading quote in values: :""Word" -> :"Word"
+ # Ensure we don't alter legitimate empty strings (:"")
+ s = re.sub(r":\s*\"\"(?=[^\",}\]\s])", r':"', s)
+
+ # Attempt strict parse first
+ try:
+ lineDict = json.loads(s)
+
+ # Handle array-based schema: {"translations": ["...", ...]}
+ if isinstance(lineDict, dict) and "translations" in lineDict and isinstance(lineDict["translations"], list):
+ stringList = [str(v) for v in lineDict["translations"]]
+ return stringList if isList else (stringList[0] if stringList else None)
+
+ # Build list in numeric order if keys are LineN
+ numeric_keys = []
+ for k in lineDict.keys():
+ m = re.fullmatch(r"Line(\d+)", str(k))
+ if m:
+ numeric_keys.append(int(m.group(1)))
+
+ if numeric_keys:
+ stringList = [lineDict.get(f"Line{n}", "") for n in sorted(numeric_keys)]
+ else:
+ # Fallback to values order if no LineN keys found
+ stringList = list(lineDict.values())
+
+ return stringList if isList else (stringList[0] if stringList else None)
+
+ except Exception as e:
+ # Fallback: regex-based extraction tolerant to one or two opening quotes
+ # Captures escaped quotes within values too
+ try:
+ pairs = re.findall(r'"Line(\d+)"\s*:\s*"{1,2}((?:\\.|[^"\\])*)"', s)
+ if not pairs:
+ raise ValueError("No LineN pairs found")
+
+ # Sort numerically and unescape JSON string content
+ items = []
+ for n_str, v in sorted(((int(n), v) for n, v in pairs), key=lambda x: x[0]):
+ try:
+ # Decode JSON escapes reliably by round-tripping as a JSON string
+ decoded = json.loads(f'"{v}"')
+ except Exception:
+ decoded = v
+ items.append(decoded)
+
+ return items if isList else (items[0] if items else None)
+ except Exception as e2:
+ if pbar:
+ pbar.write(f"extractTranslation Error: {e2} after JSON error {e} on String {translatedTextList}")
+ return None
+
+
+def calculateCost(inputTokens, outputTokens, model):
+ """
+ Calculate the cost of translation based on token usage and model pricing.
+
+ For Claude models the cost is derived from the actual cache token breakdown
+ recorded by translateAI, so cache discounts are reflected accurately:
+ - Cache reads: 10 % of the base input rate
+ - Cache writes: 125 % of the base input rate
+ - Regular input: 100 % of the base input rate
+
+ Call pattern (no module changes required):
+ Per-file call: file_cost_ready flag is True → read thread-local per-file
+ accumulators (which span all translateAI calls for the file),
+ compute cost, reset accumulators, clear flag, return cost.
+ TOTAL call: file_cost_ready is False (already cleared) → return the
+ cross-thread _global_accurate_cost running sum.
+
+ Falls back to naive token × rate calculation for non-Claude models.
+ """
+ _is_claude = model and any(x in model.lower() for x in ("claude", "sonnet", "haiku", "opus"))
+ if _is_claude:
+ if getattr(_thread_local, 'file_cost_ready', False):
+ # Per-file call: compute from accumulators (may be 0 for disk-cached files),
+ # reset everything, return the file cost.
+ cr = getattr(_thread_local, 'file_cache_read', 0)
+ cw = getattr(_thread_local, 'file_cache_write', 0)
+ reg = getattr(_thread_local, 'file_regular', 0)
+ out = getattr(_thread_local, 'file_output', 0)
+ bcr = getattr(_thread_local, 'file_batch_read', 0)
+ bcw = getattr(_thread_local, 'file_batch_write', 0)
+ breg = getattr(_thread_local, 'file_batch_regular', 0)
+ bout = getattr(_thread_local, 'file_batch_output', 0)
+ pricing = getPricingConfig(model)
+ br = pricing["inputAPICost"] / 1_000_000
+ orr = pricing["outputAPICost"] / 1_000_000
+ cost = (cr * br * 0.10 + cw * br * 2.00 + reg * br + out * orr
+ # Batch API tokens: same rates, then the 50% batch discount.
+ + (bcr * br * 0.10 + bcw * br * 2.00 + breg * br + bout * orr) * 0.50)
+ _thread_local.file_cache_read = 0
+ _thread_local.file_cache_write = 0
+ _thread_local.file_regular = 0
+ _thread_local.file_output = 0
+ _thread_local.file_batch_read = 0
+ _thread_local.file_batch_write = 0
+ _thread_local.file_batch_regular = 0
+ _thread_local.file_batch_output = 0
+ _thread_local.file_cost_ready = False
+ return cost
+ # TOTAL call (flag already cleared): return the cross-thread running total.
+ # If _global_accurate_cost is 0 it means no real API calls were made
+ # (e.g. estimate mode), so fall through to the naive calculation below.
+ with _global_accurate_cost_lock:
+ accurate = _global_accurate_cost
+ if accurate > 0:
+ return accurate
+
+ # Non-Claude, estimate mode, or no accurate data: naive calculation.
+ # For Claude models, use the accumulated static_system token count (the portion
+ # that is always cache-written at the 1hr TTL rate = 2x input rate).
+ # Remaining tokens are billed at the regular input rate.
+ pricing = getPricingConfig(model)
+ _is_claude_naive = model and any(x in model.lower() for x in ("claude", "sonnet", "haiku", "opus"))
+ if _is_claude_naive:
+ static_tok = getattr(_thread_local, 'estimate_static_tokens', 0)
+ regular_tok = getattr(_thread_local, 'estimate_regular_tokens', 0)
+ batch_count = max(1, getattr(_thread_local, 'estimate_batch_count', 1))
+ _thread_local.estimate_static_tokens = 0
+ _thread_local.estimate_regular_tokens = 0
+ _thread_local.estimate_batch_count = 0
+ # If cache is disabled, every batch is a write (2x) — no reads ever.
+ # Otherwise: each distinct batch size (= distinct output_config schema) gets exactly
+ # one cache write on first use; all subsequent batches of that size are reads (0.10x).
+ # Load from disk first so GUI subprocesses (one per file) share warm-cache state.
+ global _estimate_written_sizes
+ if DISABLE_CACHE:
+ write_batches = batch_count
+ read_batches = 0
+ else:
+ _load_estimate_written_sizes()
+ seen_sizes = getattr(_thread_local, 'estimate_seen_sizes', set())
+ new_sizes = seen_sizes - _estimate_written_sizes
+ write_batches = len(new_sizes) # one write per newly-seen size
+ read_batches = batch_count - write_batches
+ _estimate_written_sizes.update(new_sizes)
+ _save_estimate_written_sizes()
+ _thread_local.estimate_seen_sizes = set()
+ write_cost = (write_batches * static_tok / 1_000_000) * pricing["inputAPICost"] * 2.0
+ read_cost = (read_batches * static_tok / 1_000_000) * pricing["inputAPICost"] * 0.10
+ regular_cost = (regular_tok / 1_000_000) * pricing["inputAPICost"]
+ inputCost = write_cost + read_cost + regular_cost
+ else:
+ inputCost = (inputTokens / 1_000_000) * pricing["inputAPICost"]
+ outputCost = (outputTokens / 1_000_000) * pricing["outputAPICost"]
+ return inputCost + outputCost
+
+
+def countTokens(system, user, history):
+ """Count tokens for cost estimation"""
+ inputTotalTokens = 0
+ outputTotalTokens = 0
+ enc = tiktoken.encoding_for_model("gpt-4")
+
+ # Input
+ if isinstance(history, list):
+ for line in history:
+ inputTotalTokens += len(enc.encode(line))
+ else:
+ inputTotalTokens += len(enc.encode(history))
+ inputTotalTokens += len(enc.encode(system))
+ inputTotalTokens += len(enc.encode(user))
+
+ # Output
+ outputTotalTokens += round(len(enc.encode(user)) * 2.5)
+
+ return [inputTotalTokens, outputTotalTokens]
+
+
+@retry(exceptions=Exception, tries=5, delay=5)
+def translateAI(text, history, config, filename=None, pbar=None, lock=None, mismatchList=None):
+ """
+ Main translation entry point used by all modules.
+
+ Returns [translatedText, [inputTokens, outputTokens]].
+ """
+ if not text:
+ return [text, [0, 0]]
+
+ # Use TRANSLATION_RUN_LOG env var as log path if set.
+ run_log = os.getenv("TRANSLATION_RUN_LOG")
+ if run_log:
+ # Make sure parent dir exists
+ try:
+ Path(run_log).parent.mkdir(parents=True, exist_ok=True)
+ except Exception:
+ pass
+ config.logFilePath = run_log
+
+ # Ensure log directory exists for the configured path
+ try:
+ Path(config.logFilePath).parent.mkdir(parents=True, exist_ok=True)
+ except Exception:
+ pass
+
+ # Token tracking: [input, output].
+ totalTokens = [0, 0]
+
+ # Init per-file accumulators on first call on this thread (never reset here —
+ # they span all translateAI calls for a file; reset by calculateCost).
+ if not hasattr(_thread_local, 'file_cache_read'):
+ _thread_local.file_cache_read = 0
+ _thread_local.file_cache_write = 0
+ _thread_local.file_regular = 0
+ _thread_local.file_output = 0
+ # Batch API usage is billed at 50% so it is accumulated separately.
+ if not hasattr(_thread_local, 'file_batch_read'):
+ _thread_local.file_batch_read = 0
+ _thread_local.file_batch_write = 0
+ _thread_local.file_batch_regular = 0
+ _thread_local.file_batch_output = 0
+ # Snapshot accumulators so end-of-call delta only counts tokens from this call.
+ _prev_cr = _thread_local.file_cache_read
+ _prev_cw = _thread_local.file_cache_write
+ _prev_reg = _thread_local.file_regular
+ _prev_out = _thread_local.file_output
+ _prev_bcr = _thread_local.file_batch_read
+ _prev_bcw = _thread_local.file_batch_write
+ _prev_breg = _thread_local.file_batch_regular
+ _prev_bout = _thread_local.file_batch_output
+ _thread_local.file_cost_ready = False # will be set True at end of translateAI
+
+ # Anthropic batch phase ('collect'/'consume'); None when off, in estimate
+ # mode, or when the model doesn't route to the native Anthropic SDK.
+ batch_phase = get_batch_phase()
+ if batch_phase and (config.estimateMode or not isClaudeNative(config.model)):
+ batch_phase = None
+
+ if isinstance(text, list):
+ formatType = "json"
+ tList = batchList(text, config.batchSize)
+ else:
+ formatType = "json"
+ tList = [text]
+
+ for index, tItem in enumerate(tList):
+ # Check if text contains target language
+ if not re.search(config.langRegex, str(tItem)):
+ if pbar is not None:
+ pbar.update(len(tItem) if isinstance(tItem, list) else 1)
+ if isinstance(tItem, list):
+ for j in range(len(tItem)):
+ tItem[j] = cleanTranslatedText(tItem[j], config.language)
+ tList[index] = tItem
+ else:
+ tList[index] = cleanTranslatedText(tItem, config.language)
+ history = tItem[-config.maxHistory:] if isinstance(tItem, list) else tItem
+ continue
+
+ # Ellipsis-only bypass: strings whose translatable content is purely '…' characters
+ # (e.g. "「………」") should never be sent to the AI — just convert brackets and pass through.
+ def _is_ellipsis_only(s):
+ inner = str(s).strip().lstrip('「『').rstrip('」』').strip()
+ return bool(inner) and all(c in '\u2026\u30FC' for c in inner)
+
+ def _convert_ellipsis(s):
+ return str(s).replace('「', '"').replace('」', '"').replace('『', '"').replace('』', '"')
+
+ if isinstance(tItem, list):
+ if all(_is_ellipsis_only(s) for s in tItem):
+ tList[index] = [_convert_ellipsis(s) for s in tItem]
+ if pbar is not None:
+ pbar.update(len(tItem))
+ continue
+ else:
+ if _is_ellipsis_only(tItem):
+ tList[index] = _convert_ellipsis(tItem)
+ if pbar is not None:
+ pbar.update(1)
+ continue
+
+ # Protect script codes before translation
+ protected_items = []
+ all_replacements = {}
+
+ if isinstance(tItem, list):
+ for j in range(len(tItem)):
+ if not tItem[j] or not str(tItem[j]).strip():
+ protected_items.append("Placeholder Text")
+ all_replacements[j] = {}
+ else:
+ collapsed = re.sub(r'(.)\1{9,}', lambda m: m.group(1) * 10, tItem[j])
+ protected_text, replacements = protect_script_codes(collapsed)
+ protected_items.append(protected_text)
+ all_replacements[j] = replacements
+ else:
+ if not tItem or not str(tItem).strip():
+ protected_items = "Placeholder Text"
+ all_replacements[0] = {}
+ else:
+ collapsed = re.sub(r'(.)\1{9,}', lambda m: m.group(1) * 10, tItem)
+ protected_items, all_replacements[0] = protect_script_codes(collapsed)
+
+ # Filter out corrupted/mojibake text (U+FFFD) from the batch before API call
+ corrupted_map = {} # original_index -> original_text
+ if isinstance(tItem, list):
+ for j in range(len(tItem)):
+ if tItem[j] and "\ufffd" in str(tItem[j]):
+ corrupted_map[j] = tItem[j]
+ elif tItem and "\ufffd" in str(tItem):
+ # Single corrupted string - skip translation entirely
+ tList[index] = tItem
+ if pbar is not None:
+ pbar.update(1)
+ history = tItem
+ continue
+
+ # Filter out items that have content but no Japanese — they need no translation
+ # and the AI tends to empty them (e.g. "「………」" -> ""). Apply the same
+ # cleanup that would happen post-translation and restore them afterwards.
+ no_japanese_map = {} # original_index -> already-cleaned text
+ if isinstance(tItem, list):
+ for j in range(len(tItem)):
+ if j in corrupted_map:
+ continue
+ item_str = str(tItem[j]).strip() if tItem[j] else ""
+ if item_str and item_str != "Placeholder Text" and not re.search(config.langRegex, item_str):
+ cleaned = cleanTranslatedText(tItem[j], config.language)
+ cleaned = cleaned.replace("「", '"').replace("」", '"').strip()
+ no_japanese_map[j] = cleaned
+
+ # Combine skip sets and rebuild protected_items / all_replacements
+ skip_indices = set(corrupted_map.keys()) | set(no_japanese_map.keys())
+ if isinstance(tItem, list) and skip_indices:
+ clean_indices = [j for j in range(len(tItem)) if j not in skip_indices]
+
+ if not clean_indices:
+ # Every item is either corrupted or untranslatable — reassemble and move on
+ result = []
+ for j in range(len(tItem)):
+ if j in corrupted_map:
+ result.append(corrupted_map[j])
+ elif j in no_japanese_map:
+ result.append(no_japanese_map[j])
+ else:
+ result.append(tItem[j])
+ tList[index] = result
+ if pbar is not None:
+ pbar.update(len(tItem))
+ history = result[-config.maxHistory:]
+ continue
+
+ # Rebuild protected_items and all_replacements for translatable items only
+ protected_items = [protected_items[j] for j in clean_indices]
+ new_replacements = {}
+ for new_idx, old_idx in enumerate(clean_indices):
+ new_replacements[new_idx] = all_replacements.get(old_idx, {})
+ all_replacements = new_replacements
+
+ # Build filtered tItem for validation (excludes skipped items)
+ if isinstance(tItem, list) and skip_indices:
+ clean_tItem = [tItem[j] for j in range(len(tItem)) if j not in skip_indices]
+ else:
+ clean_tItem = tItem
+
+ # Format for translation
+ if isinstance(tItem, list):
+ payload = {f"Line{i+1}": string for i, string in enumerate(protected_items)}
+ payload = json.dumps(payload, indent=4, ensure_ascii=False)
+ subbedT = payload
+ else:
+ subbedT = json.dumps({"Line1": protected_items}, indent=4, ensure_ascii=False)
+
+ # Batch collect queues list payloads only. Single strings (speaker and
+ # variable names) translate live — modules memoize them and embed the
+ # results into later payloads, so they must resolve identically in both
+ # passes or the consume pass couldn't match the queued payload keys.
+ # This is the names-first phase; names are a tiny share of the volume.
+ queue_for_batch = batch_phase == "collect" and isinstance(tItem, list)
+
+ # Check cache for this exact payload (the collect pass uses a
+ # non-blocking peek so no pending markers are left behind for the
+ # consume pass to wait on)
+ if queue_for_batch:
+ cached_result = peek_cached_translation(subbedT, config.language)
+ else:
+ cached_result = get_cached_translation(subbedT, config.language)
+ if cached_result is not None:
+ # In estimate mode, never replace tList[index] from cache — the cached value
+ # may have been stored for a batch with a different number of skip_indices,
+ # so its length can differ from the current tItem. Keeping tList[index] as
+ # the original tItem ensures the returned list always has the correct length.
+ if not config.estimateMode:
+ if isinstance(tItem, list):
+ tList[index] = cached_result
+ history = cached_result[-config.maxHistory:]
+ else:
+ tList[index] = cached_result
+ history = cached_result
+ else:
+ if isinstance(cached_result, list) and cached_result:
+ history = cached_result[-config.maxHistory:]
+ elif cached_result:
+ history = cached_result
+
+ if lock and pbar is not None:
+ with lock:
+ pbar.update(len(tItem) if isinstance(tItem, list) else 1)
+
+ continue
+
+ # Create context — static_system is the stable prompt.txt content;
+ # vocab_text is the per-batch matched vocabulary (dynamic).
+ static_system, vocab_text, user = createContext(config, subbedT, formatType, history)
+
+ # Batch collect pass: queue the request (built exactly like a live one)
+ # instead of calling the API. The text stays untranslated; the consume
+ # pass fills it in from the fetched batch results. History carries the
+ # preceding source lines so the model still sees scene context.
+ if queue_for_batch:
+ numLines = len(clean_tItem) if isinstance(tItem, list) else 1
+ params = buildClaudeRequest(static_system, user, history, formatType,
+ config.model, numLines, vocab_text=vocab_text)
+ queue_batch_request(subbedT, config.language, params)
+ if lock and pbar is not None:
+ with lock:
+ pbar.update(len(tItem))
+ history = tItem[-config.maxHistory:]
+ continue
+
+ # Calculate estimate if in estimate mode
+ if config.estimateMode:
+ estimate = countTokens(static_system + vocab_text, user, history)
+ totalTokens[0] += estimate[0]
+ totalTokens[1] += estimate[1]
+
+ # Track exact cache write size (static_system, constant across batches)
+ # and accumulate non-cached (vocab + user + history) tokens per batch.
+ _est_api = os.getenv("api", "").strip()
+ _is_claude_est = (
+ config.model
+ and any(x in config.model.lower() for x in ("claude", "sonnet", "haiku", "opus"))
+ and (not _est_api or "anthropic" in _est_api.lower())
+ )
+ if _is_claude_est:
+ # Use Anthropic's count_tokens API once to get the exact cached token count.
+ # Only called on the first batch; result reused for all subsequent batches.
+ if not getattr(_thread_local, 'estimate_static_tokens', 0):
+ try:
+ _ant_count_client = anthropic.Anthropic(api_key=openai.api_key)
+ backtick = chr(96) * 3
+ _sys_block = [{"type": "text", "text": backtick + "\n" + static_system + "\n" + backtick, "cache_control": {"type": "ephemeral", "ttl": "1h"}}]
+ _count_resp = _ant_count_client.beta.messages.count_tokens(
+ betas=["token-counting-2024-11-01"],
+ model=config.model,
+ system=_sys_block,
+ messages=[{"role": "user", "content": "x"}]
+ )
+ _thread_local.estimate_static_tokens = _count_resp.input_tokens
+ except Exception:
+ # Fallback to tiktoken if count_tokens fails
+ enc = tiktoken.encoding_for_model("gpt-4")
+ _thread_local.estimate_static_tokens = len(enc.encode(static_system))
+ regular_tok = max(0, estimate[0] - getattr(_thread_local, 'estimate_static_tokens', 0))
+ _thread_local.estimate_regular_tokens = getattr(_thread_local, 'estimate_regular_tokens', 0) + regular_tok
+ _thread_local.estimate_batch_count = getattr(_thread_local, 'estimate_batch_count', 0) + 1
+ # Track unique batch sizes seen this file (each maps to a distinct schema)
+ _size = len(clean_tItem) if isinstance(clean_tItem, list) else 1
+ _seen = getattr(_thread_local, 'estimate_seen_sizes', set())
+ _seen.add(_size)
+ _thread_local.estimate_seen_sizes = _seen
+
+ # Cache the payload with original text as placeholder for future estimates
+ if isinstance(tItem, list):
+ cache_translation(subbedT, tItem, config.language)
+ else:
+ cache_translation(subbedT, [tItem], config.language)
+
+ continue
+
+ # --- Translation and Validation Retry Block ---
+ max_retries = 2 # 1 initial attempt + 2 retries
+ final_translations = None
+ last_raw_translation = ""
+ numLines = len(clean_tItem) if isinstance(tItem, list) else 1
+
+ for attempt in range(max_retries + 1):
+ is_valid = True
+
+ # On retries, prepend the correction note to the USER message so the
+ # cached static_system block is never modified (avoids cache busting).
+ current_user = user
+ if attempt > 0:
+ retry_note = (
+ f"IMPORTANT: Your previous attempt was incorrect or incomplete. Please ensure:\n"
+ f"1. The entire output is translated to {config.language} with no untranslated characters\n"
+ f"2. The JSON structure is correct with NO EMPTY or near-empty translations\n"
+ f" - Every line with Japanese text MUST be fully translated\n"
+ f" - Do NOT leave translations empty (\"\") or as single punctuation marks (\":\")\n"
+ f"3. ALL placeholders (like __PROTECTED_0__, __PROTECTED_1__, etc.) are preserved EXACTLY as they appear in the input\n"
+ f" - Do not modify, translate, or remove any __PROTECTED_N__ placeholders\n"
+ f" - Keep them in the exact same position in your translation\n"
+ f"4. Do NOT repeat the same letter or symbol many times in a row (e.g. uuuuuuuu... or broken tails)\n"
+ f" - Keep moans/effects natural; never output long runs of one character\n\n"
+ )
+ current_user = retry_note + user
+ if pbar:
+ pbar.write(f"Retrying translation... (Attempt {attempt + 1}/{max_retries + 1})")
+
+ # Translate — the consume pass tries the fetched batch result first;
+ # a missing or invalid result falls through to the live API.
+ from_batch = False
+ if batch_phase == "consume" and attempt == 0:
+ batch_result = take_batch_result(subbedT, config.language)
+ if batch_result is not None:
+ response = _AnthropicCompat(
+ batch_result.get("text", ""),
+ batch_result.get("prompt_tokens", 0) or 0,
+ batch_result.get("completion_tokens", 0) or 0,
+ batch_result.get("cache_read_input_tokens", 0) or 0,
+ batch_result.get("cache_creation_input_tokens", 0) or 0,
+ )
+ from_batch = True
+ _write_request_debug_log("anthropic-batch", {"payload": subbedT}, response.usage)
+ if not from_batch:
+ try:
+ response = translateText(static_system, current_user, history, 0.05, formatType, config.model, numLines, vocab_text=vocab_text)
+ except Exception as api_err:
+ err_msg = f"[API_ERROR] {api_err}"
+ # Print to stdout so the GUI captures it immediately
+ print(err_msg, flush=True)
+ if pbar:
+ pbar.write(err_msg)
+ # Also write to the translation log file for persistence
+ try:
+ Path(config.logFilePath).parent.mkdir(parents=True, exist_ok=True)
+ with open(config.logFilePath, "a", encoding="utf-8") as _lf:
+ _lf.write(f"{err_msg}\n")
+ _lf.flush()
+ except Exception:
+ pass
+ raise # Let retry decorator handle it
+ translatedText = response.choices[0].message.content
+ last_raw_translation = translatedText
+
+ # Update token count for this attempt
+ totalTokens[0] += response.usage.prompt_tokens
+ totalTokens[1] += response.usage.completion_tokens
+
+ # --- Cache cost tracking (Claude only) ---
+ _is_claude_model = config.model and any(x in config.model.lower() for x in ("claude", "sonnet", "haiku", "opus"))
+ if _is_claude_model:
+ usage = response.usage
+
+ # Read cache fields from _AnthropicCompat._Usage; fall back to model_extra.
+ def _get_usage_field(field):
+ v = getattr(usage, field, None)
+ if v is None:
+ v = (getattr(usage, "model_extra", None) or {}).get(field)
+ return int(v) if v else 0
+
+ batch_cache_read = _get_usage_field("cache_read_input_tokens")
+ batch_cache_write = _get_usage_field("cache_creation_input_tokens")
+ batch_prompt_total = getattr(usage, "prompt_tokens", 0) or 0
+ batch_regular = max(0, batch_prompt_total - batch_cache_read - batch_cache_write)
+ batch_output = getattr(usage, "completion_tokens", 0) or 0
+
+ # Accumulate into per-file thread-local counters. Batch API
+ # usage is billed at 50% so it goes into its own counters.
+ if from_batch:
+ _thread_local.file_batch_read += batch_cache_read
+ _thread_local.file_batch_write += batch_cache_write
+ _thread_local.file_batch_regular += batch_regular
+ _thread_local.file_batch_output += batch_output
+ else:
+ _thread_local.file_cache_read += batch_cache_read
+ _thread_local.file_cache_write += batch_cache_write
+ _thread_local.file_regular += batch_regular
+ _thread_local.file_output += batch_output
+
+ # --- Debug Token Logging ---
+ if DEBUG:
+ try:
+ _dbg_dir = Path("log")
+ _dbg_dir.mkdir(parents=True, exist_ok=True)
+ with open(_dbg_dir / "debug.log", "a", encoding="utf-8") as _dbf:
+ _dbf.write(f"\n--- Batch ({len(clean_tItem) if isinstance(tItem, list) else 1} lines) ---\n")
+ _dbf.write(f"Prompt: {response.usage.prompt_tokens} tokens | Output: {response.usage.completion_tokens} tokens\n")
+ if hasattr(response.usage, "cache_read_input_tokens"):
+ cr = getattr(response.usage, "cache_read_input_tokens", 0) or 0
+ cw = getattr(response.usage, "cache_creation_input_tokens", 0) or 0
+ cache_status = "HIT" if cr > 0 else ("WRITE" if cw > 0 else "MISS")
+ _dbf.write(f"Cache: {cache_status} (read={cr}, write={cw})\n")
+ _dbf.flush()
+ except Exception:
+ pass
+
+ # Clean the translation first for consistency
+ cleaned_text = cleanTranslatedText(translatedText, config.language)
+
+ # Process and validate translation result
+ if cleaned_text:
+ if isinstance(tItem, list):
+ extracted = extractTranslation(cleaned_text, True, pbar)
+
+ # Check 1: Mismatch in length -> still a hard failure
+ if extracted is None or len(clean_tItem) != len(extracted):
+ is_valid = False
+ if pbar:
+ pbar.write(f"Length mismatch: expected {len(clean_tItem)}, got {len(extracted) if extracted else 0}")
+ else:
+ # Check 2: Validate placeholders are preserved
+ # Flatten all_replacements for batch validation
+ all_protected_text = protected_items # The list we sent
+ placeholder_valid, missing, extra = validate_placeholders(all_protected_text, extracted,
+ {k: v for replacements in all_replacements.values() for k, v in replacements.items()})
+
+ if not placeholder_valid:
+ is_valid = False
+ if pbar:
+ if missing:
+ pbar.write(f"Missing placeholders: {', '.join(missing)}")
+ if extra:
+ pbar.write(f"Extra placeholders: {', '.join(extra)}")
+ else:
+ # Check 3: Validate that translations are not empty or nearly empty
+ content_valid, invalid_indices, content_reasons = validate_translation_content(
+ clean_tItem, extracted, config.langRegex
+ )
+
+ if not content_valid:
+ is_valid = False
+ if pbar:
+ pbar.write(f"Invalid translation content detected:")
+ for reason in content_reasons[:5]: # Show first 5 issues
+ pbar.write(f" - {reason}")
+ if len(content_reasons) > 5:
+ pbar.write(f" ... and {len(content_reasons) - 5} more issues")
+ else:
+ # Set translations (line count matches, placeholders valid, and content is good)
+ # Strip "Placeholder Text" from individual lines (AI placeholder for untranslatable input)
+ # Also apply the 「→" / 」→" replacements here per-line (safe now that JSON is parsed)
+ def _clean_extracted_line(line):
+ if not isinstance(line, str):
+ return line
+ line = line.replace("Placeholder Text", "").strip()
+ line = line.replace("「", '"').replace("」", '"')
+ line = line.replace("『", '"').replace("』", '"')
+ return line
+ final_translations = [_clean_extracted_line(line) for line in extracted]
+ else:
+ # Single string: extract from JSON schema response
+ extracted = extractTranslation(cleaned_text, False, pbar)
+ if extracted is None:
+ is_valid = False
+ if pbar:
+ pbar.write(f"Failed to extract translation from response: {cleaned_text[:100]}")
+ else:
+ # Validate placeholders against extracted value
+ placeholder_valid, missing, extra = validate_placeholders(protected_items, extracted, all_replacements[0])
+
+ if not placeholder_valid:
+ is_valid = False
+ if pbar:
+ if missing:
+ pbar.write(f"Missing placeholders: {', '.join(missing)}")
+ if extra:
+ pbar.write(f"Extra placeholders: {', '.join(extra)}")
+ else:
+ # Validate content for single string
+ final_cleaned = extracted.replace("Placeholder Text", "")
+ content_valid, _, content_reasons = validate_translation_content(
+ tItem, final_cleaned, config.langRegex
+ )
+
+ if not content_valid:
+ is_valid = False
+ if pbar:
+ pbar.write(f"Invalid translation content:")
+ for reason in content_reasons:
+ pbar.write(f" - {reason}")
+ else:
+ # Accept output - all validations passed
+ final_translations = final_cleaned
+ else:
+ is_valid = False
+ if pbar: pbar.write(f"AI Refused: {tItem}\n")
+
+ # If translation is valid, break the retry loop
+ if is_valid:
+ break
+
+ # --- End of Retry Block ---
+
+ # After the loop, handle the final result
+ if final_translations is not None: # Success case
+ # Restore protected script codes
+ if isinstance(tItem, list):
+ for j in range(len(final_translations)):
+ if j in all_replacements:
+ final_translations[j] = restore_script_codes(final_translations[j], all_replacements[j])
+
+ # Re-insert corrupted / no-japanese originals at their original positions
+ if corrupted_map or no_japanese_map:
+ expanded = []
+ clean_idx = 0
+ for j in range(len(tItem)):
+ if j in corrupted_map:
+ expanded.append(corrupted_map[j])
+ elif j in no_japanese_map:
+ expanded.append(no_japanese_map[j])
+ else:
+ expanded.append(final_translations[clean_idx])
+ clean_idx += 1
+ final_translations = expanded
+ else:
+ final_translations = restore_script_codes(final_translations, all_replacements[0])
+
+ formatted_output = last_raw_translation
+ try:
+ parsed_json = json.loads(last_raw_translation)
+ # Normalize array-based output to LineN format for log readability
+ if isinstance(parsed_json, dict) and "translations" in parsed_json and isinstance(parsed_json["translations"], list):
+ parsed_json = {f"Line{i+1}": v for i, v in enumerate(parsed_json["translations"])}
+ formatted_output = json.dumps(parsed_json, indent=4, ensure_ascii=False)
+ except (json.JSONDecodeError, ValueError):
+ pass
+
+ # Only open and write to log file when we have something to log
+ try:
+ with open(config.logFilePath, "a", encoding="utf-8") as logFile:
+ logFile.write(f"Input:\n{subbedT}\n")
+ logFile.write(f"Output:\n{formatted_output}\n")
+ logFile.flush() # Ensure data is written to disk immediately
+ except Exception:
+ pass # Don't fail if logging fails
+
+ # Cache the entire payload and its translation
+ if not config.estimateMode:
+ cache_translation(subbedT, final_translations, config.language)
+
+ if isinstance(tItem, list):
+ tList[index] = final_translations
+ history = final_translations[-config.maxHistory:]
+ else:
+ tList[index] = final_translations
+ history = final_translations
+
+ if lock and pbar is not None:
+ with lock:
+ pbar.update(len(tItem) if isinstance(tItem, list) else 1)
+
+ else: # Failure case after all retries
+ if pbar: pbar.write(f"Translation failed after {max_retries + 1} attempts. Check mismatch log.")
+
+ # Emit a machine-readable marker on stdout so the GUI worker
+ # thread can detect the mismatch reliably (stdout is captured
+ # synchronously, unlike file-tail polling which can be racy).
+ try:
+ print(f"MISMATCH_EVENT:{filename}", flush=True)
+ except Exception:
+ pass
+
+ formatted_mismatch_output = last_raw_translation
+ try:
+ parsed_json = json.loads(last_raw_translation)
+ if isinstance(parsed_json, dict) and "translations" in parsed_json and isinstance(parsed_json["translations"], list):
+ parsed_json = {f"Line{i+1}": v for i, v in enumerate(parsed_json["translations"])}
+ formatted_mismatch_output = json.dumps(parsed_json, indent=4, ensure_ascii=False)
+ except (json.JSONDecodeError, ValueError):
+ pass
+ with open(config.mismatchLogPath, "a+", encoding="utf-8") as mismatchFile:
+ mismatchFile.write(f"Failed after retries: {filename}\n")
+ mismatchFile.write(f"Input:\n{subbedT}\n")
+ mismatchFile.write(f"Final Output:\n{formatted_mismatch_output}\n")
+ mismatchFile.flush() # Ensure data is written to disk immediately
+
+ # Also write to the main translation log so the GUI log viewer can display it
+ try:
+ with open(config.logFilePath, "a", encoding="utf-8") as logFile:
+ logFile.write(f"[MISMATCH] Failed after retries: {filename}\n")
+ logFile.write(f"[MISMATCH] Input:\n")
+ for mline in subbedT.splitlines():
+ logFile.write(f"[MISMATCH] {mline}\n")
+ logFile.write(f"[MISMATCH] Final Output:\n")
+ for mline in formatted_mismatch_output.splitlines():
+ logFile.write(f"[MISMATCH] {mline}\n")
+ logFile.flush()
+ except Exception:
+ pass # Don't fail if logging fails
+
+ if filename and mismatchList is not None and filename not in mismatchList:
+ mismatchList.append(filename)
+
+ tList[index] = tItem
+ history = text[-config.maxHistory:] if isinstance(text, list) else text
+
+ # Combine if multilist
+ if tList and isinstance(tList[0], list):
+ tList = [t for sublist in tList for t in sublist]
+
+ # Save cache after processing (for both estimate and translation modes)
+ save_cache()
+
+ # Batch collect pass: merge this call's queued requests into the disk queue.
+ if batch_phase == "collect":
+ flush_batch_queue()
+
+ # For Claude: accumulate only this call's delta into the cross-thread total.
+ # file_* accumulators hold full per-file totals; calculateCost() reads them.
+ _is_claude_final = config.model and any(x in config.model.lower() for x in ("claude", "sonnet", "haiku", "opus"))
+ if _is_claude_final and not config.estimateMode:
+ _pricing = getPricingConfig(config.model)
+ _br = _pricing["inputAPICost"] / 1_000_000
+ _or = _pricing["outputAPICost"] / 1_000_000
+ # Delta = tokens added in this call only (not earlier calls for same file).
+ _delta_cr = getattr(_thread_local, 'file_cache_read', 0) - _prev_cr
+ _delta_cw = getattr(_thread_local, 'file_cache_write', 0) - _prev_cw
+ _delta_reg = getattr(_thread_local, 'file_regular', 0) - _prev_reg
+ _delta_out = getattr(_thread_local, 'file_output', 0) - _prev_out
+ _delta_bcr = getattr(_thread_local, 'file_batch_read', 0) - _prev_bcr
+ _delta_bcw = getattr(_thread_local, 'file_batch_write', 0) - _prev_bcw
+ _delta_breg = getattr(_thread_local, 'file_batch_regular', 0) - _prev_breg
+ _delta_bout = getattr(_thread_local, 'file_batch_output', 0) - _prev_bout
+ _call_cost = (
+ _delta_cr * _br * 0.10 +
+ _delta_cw * _br * 2.00 +
+ _delta_reg * _br +
+ _delta_out * _or +
+ # Batch API tokens: same rates, then the 50% batch discount.
+ (_delta_bcr * _br * 0.10 +
+ _delta_bcw * _br * 2.00 +
+ _delta_breg * _br +
+ _delta_bout * _or) * 0.50
+ )
+ global _global_accurate_cost
+ with _global_accurate_cost_lock:
+ _global_accurate_cost += _call_cost
+ _thread_local.file_cost_ready = True # signals calculateCost to use file accumulators
+
+ # Return result
+ if isinstance(text, list):
+ return [tList, totalTokens]
+ else:
return [tList[0], totalTokens]
\ No newline at end of file