dazed/DazedTL
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Clean up some translation.py stuff.

Browse source
This commit is contained in:
dazedanon 2026-03-08 16:37:22 -05:00
parent 57a7ad87cf
commit 06afa09805
1 changed files with 62 additions and 104 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

166
util/translation.py
View file

@ -1,7 +1,6 @@
""" """
Shared translation utilities for DazedMTLTool Shared translation utilities for DazedMTLTool.
This module provides a centralized translation function that can be used across all modules Centralized translation function used across all modules.
without relying on global variables.
""" """
import os import os
@ -20,30 +19,22 @@ from retry import retry
# Set to True to print input/output token counts and content for each API call # Set to True to print input/output token counts and content for each API call
DEBUG_LOGGING = False DEBUG_LOGGING = False
# Set to True to print per-batch and running-total actual costs for Claude. # Set to True to print per-batch cache costs for Claude (read/write/regular breakdown).
# Run the same file twice — once with DISABLE_CACHE=False, once with DISABLE_CACHE=True CACHE_COST_DEBUG = False
# — and compare the final "run total" lines to see real savings.
CACHE_COST_DEBUG = True
# Set to True to disable Claude prompt caching (removes cache_control from the # Set to True to disable Claude prompt caching for baseline cost comparison.
# system message). Use together with CACHE_COST_DEBUG to get the real uncached
# baseline cost for comparison against a cached run.
DISABLE_CACHE = False DISABLE_CACHE = False
# Module-level accumulators for CACHE_COST_DEBUG — persist across all translateAI # Per-process debug accumulators for CACHE_COST_DEBUG run_total line.
# calls within a single process run so the final run_total is meaningful.
_dbg_cache_read = 0 _dbg_cache_read = 0
_dbg_cache_write = 0 _dbg_cache_write = 0
_dbg_regular_input = 0 _dbg_regular_input = 0
_dbg_output = 0 _dbg_output = 0
# Thread-local storage for per-file cache token breakdown. # Thread-local per-file token breakdown; read by calculateCost() for Claude.
# Used by calculateCost() to return accurate cache-discounted costs for Claude
# without requiring changes to any module files.
_thread_local = threading.local() _thread_local = threading.local()
# Global (cross-thread) running total of accurate cache-discounted cost. # Cross-thread running total of accurate cache-discounted cost (protected by lock).
# Protected by a lock so parallel file threads can safely accumulate into it.
_global_accurate_cost = 0.0 _global_accurate_cost = 0.0
_global_accurate_cost_lock = threading.Lock() _global_accurate_cost_lock = threading.Lock()
@ -55,8 +46,8 @@ PROTECTED_PATTERNS = [
r'\\ME\[[^\]]+\]', # \ME[music_effect_name] r'\\ME\[[^\]]+\]', # \ME[music_effect_name]
r'\\BGM\[[^\]]+\]', # \BGM[background_music_name] r'\\BGM\[[^\]]+\]', # \BGM[background_music_name]
r'\\BGS\[[^\]]+\]', # \BGS[background_sound_name] r'\\BGS\[[^\]]+\]', # \BGS[background_sound_name]
r'_pum\[[^\]]+\]', # \BGS[background_sound_name] r'_pum\[[^\]]+\]', # _pum[name]
r'\\VS\[[^\]]+\]', # \BGS[background_sound_name] r'\\VS\[[^\]]+\]', # \VS[name]
] ]
def protect_script_codes(text): def protect_script_codes(text):
@ -208,8 +199,8 @@ def validate_translation_content(original_items, translated_items, langRegex):
is_valid = len(invalid_indices) == 0 is_valid = len(invalid_indices) == 0
return is_valid, invalid_indices, reasons return is_valid, invalid_indices, reasons
# (from .env if present), strip accidental whitespace, and set the base URL, # Load .env, strip accidental whitespace, set base URL / org / API key.
# organization, and API key. It also handles the Gemini compatibility layer. # Handles the Gemini compatibility layer as a special case.
load_dotenv() load_dotenv()
api_provider = os.getenv("API_PROVIDER", "openai").lower() api_provider = os.getenv("API_PROVIDER", "openai").lower()
env_api = os.getenv("api", "").strip() env_api = os.getenv("api", "").strip()
@ -433,20 +424,9 @@ class TranslationConfig:
# Set language regex (default is Japanese) # Set language regex (default is Japanese)
self.langRegex = langRegex or r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+" self.langRegex = langRegex or r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+"
# Set batch size based on model if not provided # Set batch size — derive from pricing config unless explicitly supplied
if batchSize is None: if batchSize is None:
if "gpt-3.5" in self.model: self.batchSize = getPricingConfig(self.model)["batchSize"]
self.batchSize = 10
elif "gpt-4" in self.model:
self.batchSize = 30
elif "deepseek" in self.model:
self.batchSize = 30
else:
# Try to get from environment, fallback to 10
try:
self.batchSize = int(os.getenv("batchsize", 10))
except (ValueError, TypeError):
self.batchSize = 10
else: else:
self.batchSize = batchSize self.batchSize = batchSize
@ -504,7 +484,21 @@ def getPricingConfig(model):
"batchSize": 30, "batchSize": 30,
"frequencyPenalty": 0.05 "frequencyPenalty": 0.05
} }
elif "sonnet" in model: elif "claude-opus" in model or model == "claude-3-opus":
return {
"inputAPICost": 15.00,
"outputAPICost": 75.00,
"batchSize": 30,
"frequencyPenalty": 0.05
}
elif "claude-haiku" in model or "haiku" in model:
return {
"inputAPICost": 0.80,
"outputAPICost": 4.00,
"batchSize": 30,
"frequencyPenalty": 0.05
}
elif "sonnet" in model or "claude" in model:
return { return {
"inputAPICost": 3.00, "inputAPICost": 3.00,
"outputAPICost": 15.00, "outputAPICost": 15.00,
@ -730,11 +724,9 @@ def createTranslationSchema(numLines):
def translateText(system, user, history, penalty, formatType, model, numLines=None, vocab_text=""): def translateText(system, user, history, penalty, formatType, model, numLines=None, vocab_text=""):
"""Send translation request to the selected API. """Send translation request to the selected API.
Args: system: Static system prompt (prompt.txt). Cached by Claude.
system: Static system prompt (prompt.txt content). vocab_text: Per-batch vocabulary (dynamic, never cached to avoid cache busting).
vocab_text: Per-batch matched vocabulary text (dynamic, kept separate so
it does not bust the Claude prompt cache on every call).
""" """
# Ensure system content is not empty # Ensure system content is not empty
if not system or not str(system).strip(): if not system or not str(system).strip():
@ -743,16 +735,10 @@ def translateText(system, user, history, penalty, formatType, model, numLines=No
_is_claude = model and any(x in model.lower() for x in ("claude", "sonnet", "haiku", "opus")) _is_claude = model and any(x in model.lower() for x in ("claude", "sonnet", "haiku", "opus"))
_is_deepseek = model and "deepseek" in model.lower() _is_deepseek = model and "deepseek" in model.lower()
# Prompt # Build message list.
# For Claude: the static prompt (prompt.txt) is placed in its own content block # Claude: static prompt gets cache_control; vocab appended uncached so it
# marked with cache_control so Anthropic caches it across all batches (~90% # never busts the cache. Requires ≥2048 tokens for Sonnet 4.6 to qualify.
# discount on cache reads, 25% surcharge on the one-time write). # Other providers: combine into one plain string.
# prompt.txt must be ≥2048 tokens for Sonnet 4.6 (≥1024 for older models) to
# qualify — keeping it static and vocab-free ensures the cache key never changes.
# The per-batch matched vocab is appended as a second, uncached content block so
# it never invalidates the cache.
# History is added as separate messages below.
# For all other providers: combine into one plain string as before.
if _is_claude: if _is_claude:
if DISABLE_CACHE: if DISABLE_CACHE:
# No cache_control — sends as a plain content block for a real uncached run. # No cache_control — sends as a plain content block for a real uncached run.
@ -779,11 +765,8 @@ def translateText(system, user, history, penalty, formatType, model, numLines=No
if history and str(history).strip(): if history and str(history).strip():
msg.append({"role": "assistant", "content": history}) msg.append({"role": "assistant", "content": history})
# Response Format # Response format per provider:
# - OpenAI: response_format with json_schema wrapper {name, strict, schema} # OpenAI/Gemini: json_schema | Deepseek: json_object | text: omit entirely
# - Deepseek: response_format with json_object (no schema support)
# - Claude: output_config.format via extra_body {type, schema} — no response_format
# - Gemini: response_format with json_schema (OpenAI compat)
if formatType == "json" and numLines is not None: if formatType == "json" and numLines is not None:
schema = createTranslationSchema(numLines) schema = createTranslationSchema(numLines)
@ -796,10 +779,8 @@ def translateText(system, user, history, penalty, formatType, model, numLines=No
"type": "json_schema", "type": "json_schema",
"json_schema": {"name": "translation_response", "strict": True, "schema": schema} "json_schema": {"name": "translation_response", "strict": True, "schema": schema}
} }
_claude_output_config = None
else: else:
responseFormat = {"type": "text"} responseFormat = {"type": "text"}
_claude_output_config = None
# Content to TL - ensure user content is not empty # Content to TL - ensure user content is not empty
if not user or not str(user).strip(): if not user or not str(user).strip():
@ -814,10 +795,7 @@ def translateText(system, user, history, penalty, formatType, model, numLines=No
# --- API Call Logic --- # --- API Call Logic ---
api_provider = os.getenv("API_PROVIDER", "openai").lower() api_provider = os.getenv("API_PROVIDER", "openai").lower()
# Base parameters for the API call # Omit response_format for plain text — some providers reject {"type": "text"}.
# Note: do NOT include response_format when it is plain text — many providers
# (e.g. newer OpenAI models) reject {"type": "text"} as an invalid value since
# text output is already the default when no response_format is supplied.
params = { params = {
"model": model, "model": model,
"messages": msg, "messages": msg,
@ -844,7 +822,7 @@ def translateText(system, user, history, penalty, formatType, model, numLines=No
except (ValueError, TypeError): except (ValueError, TypeError):
pass pass
# frequency_penalty is not supported via the OpenAI compatibility layer for Gemini # frequency_penalty is unsupported on the Gemini OpenAI compat layer
elif _is_claude: elif _is_claude:
params["temperature"] = 0 params["temperature"] = 0
# cache_control is set on the system message content block above. # cache_control is set on the system message content block above.
@ -855,17 +833,14 @@ def translateText(system, user, history, penalty, formatType, model, numLines=No
params["temperature"] = 0 params["temperature"] = 0
params["frequency_penalty"] = penalty params["frequency_penalty"] = penalty
# --- Native Anthropic SDK call (bypasses the OpenAI compat layer) --- # Use native Anthropic SDK — the OpenAI compat endpoint strips cache_control
# The compat endpoint strips cache_control blocks and never returns # and never returns cache_read/creation_input_tokens.
# cache_read_input_tokens / cache_creation_input_tokens. The native SDK
# exposes both, making prompt caching observable and verifiable.
if _is_claude: if _is_claude:
# system blocks were built above (with cache_control on static prompt). # system blocks were built above (with cache_control on static prompt).
ant_system = list(msg[0]["content"]) ant_system = list(msg[0]["content"])
# Convert the remaining msg entries to native-SDK format. # Convert remaining messages to native format.
# system-role entries are history markers; the assistant-role entries # Skip system wrapper lines; collect assistant history and user content.
# that follow them carry the actual previous translations.
history_items = [] history_items = []
native_msgs = [] native_msgs = []
for m in msg[1:]: for m in msg[1:]:
@ -905,8 +880,7 @@ def translateText(system, user, history, penalty, formatType, model, numLines=No
_cw = getattr(_u, "cache_creation_input_tokens", 0) or 0 _cw = getattr(_u, "cache_creation_input_tokens", 0) or 0
_inp = getattr(_u, "input_tokens", 0) or 0 _inp = getattr(_u, "input_tokens", 0) or 0
_out = getattr(_u, "output_tokens", 0) or 0 _out = getattr(_u, "output_tokens", 0) or 0
# input_tokens from native SDK = non-cached portion only; # input_tokens (native SDK) = non-cached portion; add cache fields for true total.
# rebuild the full prompt total so totalTokens stays accurate.
_total_prompt = _inp + _cr + _cw _total_prompt = _inp + _cr + _cw
class _AnthropicCompat: class _AnthropicCompat:
@ -1152,10 +1126,11 @@ def calculateCost(inputTokens, outputTokens, model):
- Regular input: 100 % of the base input rate - Regular input: 100 % of the base input rate
Call pattern (no module changes required): Call pattern (no module changes required):
Per-file call: thread-local file_* accumulators hold this file's token Per-file call: file_cost_ready flag is True → read thread-local per-file
counts → compute cost, reset accumulators to 0, return cost. accumulators (which span all translateAI calls for the file),
TOTAL call: accumulators are 0 (already reset) → return total_accurate_cost compute cost, reset accumulators, clear flag, return cost.
(running sum of all per-file costs in this thread). TOTAL call: file_cost_ready is False (already cleared) → return the
cross-thread _global_accurate_cost running sum.
Falls back to naive token × rate calculation for non-Claude models. Falls back to naive token × rate calculation for non-Claude models.
""" """
@ -1213,25 +1188,14 @@ def countTokens(system, user, history):
@retry(exceptions=Exception, tries=5, delay=5) @retry(exceptions=Exception, tries=5, delay=5)
def translateAI(text, history, fullPromptFlag, config, filename=None, pbar=None, lock=None, mismatchList=None): def translateAI(text, history, fullPromptFlag, config, filename=None, pbar=None, lock=None, mismatchList=None):
""" """
Main translation function that can be used across all modules. Main translation entry point used by all modules.
Args: Returns [translatedText, [inputTokens, outputTokens]].
text: Text to translate (string or list)
history: Translation history for context
fullPromptFlag: Whether to use full prompt or simplified version
config: TranslationConfig instance with all settings
filename: Current filename being processed (for logging)
pbar: Progress bar instance (optional)
lock: Threading lock (optional)
mismatchList: List to track mismatched files (optional)
Returns:
[translatedText, totalTokens]
""" """
if not text: if not text:
return [text, [0, 0]] return [text, [0, 0]]
# If a per-run log file is specified via environment variable, prefer it. # Use TRANSLATION_RUN_LOG env var as log path if set.
run_log = os.getenv("TRANSLATION_RUN_LOG") run_log = os.getenv("TRANSLATION_RUN_LOG")
if run_log: if run_log:
# Make sure parent dir exists # Make sure parent dir exists
@ -1247,19 +1211,17 @@ def translateAI(text, history, fullPromptFlag, config, filename=None, pbar=None,
except Exception: except Exception:
pass pass
# Don't open log file here - we'll open it only when we need to write
# Token tracking: [input, output]. # Token tracking: [input, output].
totalTokens = [0, 0] totalTokens = [0, 0]
# Ensure per-file accumulators exist on this thread (first call ever on this thread). # Init per-file accumulators on first call on this thread (never reset here —
# They are NOT reset here — they accumulate across ALL translateAI calls for one # they span all translateAI calls for a file; reset by calculateCost).
# file and are only reset by calculateCost() when the per-file cost line is printed.
if not hasattr(_thread_local, 'file_cache_read'): if not hasattr(_thread_local, 'file_cache_read'):
_thread_local.file_cache_read = 0 _thread_local.file_cache_read = 0
_thread_local.file_cache_write = 0 _thread_local.file_cache_write = 0
_thread_local.file_regular = 0 _thread_local.file_regular = 0
_thread_local.file_output = 0 _thread_local.file_output = 0
# Record snapshot so end-of-call delta only counts tokens from THIS call. # Snapshot accumulators so end-of-call delta only counts tokens from this call.
_prev_cr = _thread_local.file_cache_read _prev_cr = _thread_local.file_cache_read
_prev_cw = _thread_local.file_cache_write _prev_cw = _thread_local.file_cache_write
_prev_reg = _thread_local.file_regular _prev_reg = _thread_local.file_regular
@ -1440,8 +1402,7 @@ def translateAI(text, history, fullPromptFlag, config, filename=None, pbar=None,
if _is_claude_model: if _is_claude_model:
usage = response.usage usage = response.usage
# Cache fields are set directly on _AnthropicCompat._Usage by the # Read cache fields from _AnthropicCompat._Usage; fall back to model_extra.
# native SDK path in translateText. Fall back to model_extra for safety.
def _get_usage_field(field): def _get_usage_field(field):
v = getattr(usage, field, None) v = getattr(usage, field, None)
if v is None: if v is None:
@ -1454,8 +1415,7 @@ def translateAI(text, history, fullPromptFlag, config, filename=None, pbar=None,
batch_regular = max(0, batch_prompt_total - batch_cache_read - batch_cache_write) batch_regular = max(0, batch_prompt_total - batch_cache_read - batch_cache_write)
batch_output = getattr(usage, "completion_tokens", 0) or 0 batch_output = getattr(usage, "completion_tokens", 0) or 0
# Per-file accumulators (thread-local) — always updated so # Accumulate into per-file thread-local counters.
# calculateCost() can return accurate cache-discounted costs.
_thread_local.file_cache_read += batch_cache_read _thread_local.file_cache_read += batch_cache_read
_thread_local.file_cache_write += batch_cache_write _thread_local.file_cache_write += batch_cache_write
_thread_local.file_regular += batch_regular _thread_local.file_regular += batch_regular
@ -1694,16 +1654,14 @@ def translateAI(text, history, fullPromptFlag, config, filename=None, pbar=None,
# Save cache after processing (for both estimate and translation modes) # Save cache after processing (for both estimate and translation modes)
save_cache() save_cache()
# For Claude: add only the DELTA from this call to total_accurate_cost so # For Claude: accumulate only this call's delta into the cross-thread total.
# that multiple translateAI calls within one file don't double-count tokens. # file_* accumulators hold full per-file totals; calculateCost() reads them.
# The file_* accumulators span ALL calls for the file; calculateCost() reads
# their full accumulated values when the per-file cost line is printed.
_is_claude_final = config.model and any(x in config.model.lower() for x in ("claude", "sonnet", "haiku", "opus")) _is_claude_final = config.model and any(x in config.model.lower() for x in ("claude", "sonnet", "haiku", "opus"))
if _is_claude_final and not config.estimateMode: if _is_claude_final and not config.estimateMode:
_pricing = getPricingConfig(config.model) _pricing = getPricingConfig(config.model)
_br = _pricing["inputAPICost"] / 1_000_000 _br = _pricing["inputAPICost"] / 1_000_000
_or = _pricing["outputAPICost"] / 1_000_000 _or = _pricing["outputAPICost"] / 1_000_000
# Delta = tokens added in *this* translateAI call only # Delta = tokens added in this call only (not earlier calls for same file).
_delta_cr = getattr(_thread_local, 'file_cache_read', 0) - _prev_cr _delta_cr = getattr(_thread_local, 'file_cache_read', 0) - _prev_cr
_delta_cw = getattr(_thread_local, 'file_cache_write', 0) - _prev_cw _delta_cw = getattr(_thread_local, 'file_cache_write', 0) - _prev_cw
_delta_reg = getattr(_thread_local, 'file_regular', 0) - _prev_reg _delta_reg = getattr(_thread_local, 'file_regular', 0) - _prev_reg