Merge branch 'main' of ssh.gitgud.io:DazedAnon/DazedMTLTool
This commit is contained in:
commit
3328d898f0
6 changed files with 343 additions and 161 deletions
14
.env.example
14
.env.example
|
|
@ -1,3 +1,17 @@
|
|||
# --------------------------------------------------
|
||||
# Gemini-Specific Settings (Optional)
|
||||
# --------------------------------------------------
|
||||
|
||||
# Sets the thinking budget for Gemini models.
|
||||
# - To turn on dynamic thinking, set to -1.
|
||||
# - To disable thinking (on compatible models like 2.5 Flash), set to 0.
|
||||
# - To set a specific token budget, use a positive integer (e.g., 8192).
|
||||
# - Leave this variable out to use the model's default setting.
|
||||
GEMINI_THINKING_BUDGET=
|
||||
|
||||
# Set to "gemini" to use the Gemini API or "openai" for OpenAI (If empty it will default to openai.)
|
||||
API_PROVIDER=openai
|
||||
|
||||
#API link, leave blank to use OpenAI API
|
||||
api=""
|
||||
|
||||
|
|
|
|||
|
|
@ -239,4 +239,4 @@ def deleteFolderFiles(folderPath):
|
|||
for filename in os.listdir(folderPath):
|
||||
file_path = os.path.join(folderPath, filename)
|
||||
if file_path.endswith((".json", ".yaml", ".ks")):
|
||||
os.remove(file_path)
|
||||
os.remove(file_path)
|
||||
|
|
@ -18,9 +18,14 @@ import tempfile
|
|||
|
||||
# Open AI
|
||||
load_dotenv()
|
||||
if os.getenv("api").replace(" ", "") != "":
|
||||
openai.base_url = os.getenv("api")
|
||||
openai.organization = os.getenv("org")
|
||||
if os.getenv("API_PROVIDER") == "gemini":
|
||||
openai.base_url = "https://generativelanguage.googleapis.com/v1beta/openai/"
|
||||
openai.organization = None
|
||||
else:
|
||||
api_url = os.getenv("api")
|
||||
if api_url and api_url.replace(" ", "") != "":
|
||||
openai.base_url = api_url
|
||||
openai.organization = os.getenv("org")
|
||||
openai.api_key = os.getenv("key")
|
||||
|
||||
# Globals
|
||||
|
|
@ -429,4 +434,4 @@ def translateAI(text, history, fullPromptFlag):
|
|||
pbar=PBAR,
|
||||
lock=LOCK,
|
||||
mismatchList=MISMATCH
|
||||
)
|
||||
)
|
||||
|
|
@ -89,7 +89,7 @@ CODE102 = True
|
|||
|
||||
# Optional
|
||||
CODE101 = False
|
||||
CODE408 = False
|
||||
CODE408 = True
|
||||
|
||||
# Variables
|
||||
CODE122 = False
|
||||
|
|
@ -544,15 +544,61 @@ def parseSystem(data, filename):
|
|||
|
||||
with tqdm(total=total_units, bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
|
||||
pbar.desc = filename
|
||||
input_log = {}
|
||||
output_log = {}
|
||||
try:
|
||||
# Helper to flatten lists to Line1, Line2, ...
|
||||
def flatten_lines(prefix, values, target):
|
||||
for idx, v in enumerate(values):
|
||||
target[f"{prefix}Line{idx+1}"] = v
|
||||
|
||||
# Capture pre-translation values
|
||||
if "terms" in data:
|
||||
for key, value in data["terms"].items():
|
||||
if isinstance(value, list):
|
||||
flatten_lines(key, value, input_log)
|
||||
else:
|
||||
input_log[key] = value
|
||||
for key in ["gameTitle", "game_title", "armor_types", "skill_types", "weapon_types"]:
|
||||
if key in data:
|
||||
val = data[key]
|
||||
if isinstance(val, list):
|
||||
flatten_lines(key, val, input_log)
|
||||
else:
|
||||
input_log[key] = val
|
||||
# Run translation
|
||||
result = searchSystem(data, pbar)
|
||||
totalTokens[0] += result[0]
|
||||
totalTokens[1] += result[1]
|
||||
# Capture post-translation values
|
||||
if "terms" in data:
|
||||
for key, value in data["terms"].items():
|
||||
if isinstance(value, list):
|
||||
flatten_lines(key, value, output_log)
|
||||
else:
|
||||
output_log[key] = value
|
||||
for key in ["gameTitle", "game_title", "armor_types", "skill_types", "weapon_types"]:
|
||||
if key in data:
|
||||
val = data[key]
|
||||
if isinstance(val, list):
|
||||
flatten_lines(key, val, output_log)
|
||||
else:
|
||||
output_log[key] = val
|
||||
except Exception as e:
|
||||
traceback.print_exc()
|
||||
return [data, totalTokens, e]
|
||||
finally:
|
||||
maybe_save_progress_yaml(data, filename, result)
|
||||
# Log translation history for system.yaml in established line-by-line format
|
||||
try:
|
||||
with open("log/translationHistory.txt", "a", encoding="utf-8") as log_file:
|
||||
log_file.write("Input:\n")
|
||||
log_file.write(json.dumps(input_log, ensure_ascii=False, indent=4))
|
||||
log_file.write("\nOutput:\n")
|
||||
log_file.write(json.dumps(output_log, ensure_ascii=False, indent=4))
|
||||
log_file.write("\n")
|
||||
except Exception:
|
||||
pass
|
||||
return [data, totalTokens, None]
|
||||
|
||||
|
||||
|
|
@ -1091,7 +1137,7 @@ def searchCodes(page, pbar, jobList, filename):
|
|||
|
||||
# Brackets
|
||||
if len(speakerList) == 0:
|
||||
speakerList = re.findall(r"^【(.*?)】$|^【(.*?)】[\\]*[a-zA-Z]*\[.*\]$", jaString)
|
||||
speakerList = re.findall(r"^(?:[\\]+[rlRL]\[[\w\d\-]+\])?【(.*?)】$|^(?:[\\]+[rlRL]\[[\w\d\-]+\])?【(.*?)】[\\]*[a-zA-Z]*\[.*\]$", jaString)
|
||||
if speakerList:
|
||||
if speakerList[0][0]:
|
||||
speakerList = [speakerList[0][0]]
|
||||
|
|
@ -1114,9 +1160,9 @@ def searchCodes(page, pbar, jobList, filename):
|
|||
|
||||
# First Line Speakers
|
||||
if len(speakerList) == 0 and FIRSTLINESPEAKERS is True:
|
||||
# Remove any RPGMaker Code at start
|
||||
# Remove any RPGMaker Code at start (including \r[...] and \l[...] patterns)
|
||||
ffMatch = re.search(
|
||||
r"^((?:[\\]+[^cCnNiIkKvV]+\[[\d\w]+\])+)",
|
||||
r"^((?:[\\]+[^cCnNiIkKvV]+\[[\d\w]+\])+|(?:[\\]+[rlRL]\[[\w\d\-]+\]))",
|
||||
jaString,
|
||||
)
|
||||
if ffMatch != None:
|
||||
|
|
@ -1152,7 +1198,13 @@ def searchCodes(page, pbar, jobList, filename):
|
|||
|
||||
# Set Data
|
||||
if not setData:
|
||||
codeList[i]["p"][0] = nametag + jaString.replace(speakerList[0], speaker)
|
||||
# Check if there's a \r[...] or \l[...] code at the beginning
|
||||
codePrefix = re.search(r"^([\\]+[rlRL]\[[\w\d\-]+\])", jaString)
|
||||
if codePrefix:
|
||||
# Replace only the speaker name, preserving the code and brackets
|
||||
codeList[i]["p"][0] = jaString.replace(f"【{speakerList[0]}】", f"【{speaker}】")
|
||||
else:
|
||||
codeList[i]["p"][0] = nametag + jaString.replace(speakerList[0], speaker)
|
||||
nametag = ""
|
||||
|
||||
# Iterate to next string
|
||||
|
|
@ -1761,24 +1813,40 @@ def searchCodes(page, pbar, jobList, filename):
|
|||
|
||||
## Event Code: 408 (Script)
|
||||
if "c" in codeList[i] and (codeList[i]["c"] == 408) and CODE408 is True:
|
||||
# Remove Textwrap
|
||||
jaString = codeList[i]["p"][0]
|
||||
# jaString = jaString.replace("\n", " ")
|
||||
# Save starting index
|
||||
j = i
|
||||
jaString = codeList[i]["p"][0] if len(codeList[i]["p"]) > 0 else ""
|
||||
|
||||
# Join consecutive 408 codes
|
||||
combinedText = jaString
|
||||
if len(codeList) > i + 1:
|
||||
while i + 1 < len(codeList) and "c" in codeList[i + 1] and codeList[i + 1]["c"] == 408:
|
||||
i += 1
|
||||
if len(codeList[i]["p"]) > 0:
|
||||
combinedText += " " + codeList[i]["p"][0]
|
||||
# Mark as -1 only in Pass 2
|
||||
if not setData:
|
||||
codeList[i]["p"] = []
|
||||
codeList[i]["c"] = -1
|
||||
|
||||
# Pass 1
|
||||
if setData:
|
||||
list408.append(jaString)
|
||||
if combinedText.strip(): # Only add non-empty text
|
||||
list408.append(combinedText)
|
||||
|
||||
# Pass 2
|
||||
else:
|
||||
translatedText = list408[0]
|
||||
list408.pop(0)
|
||||
if len(list408) > 0:
|
||||
translatedText = list408[0]
|
||||
list408.pop(0)
|
||||
|
||||
# Textwrap
|
||||
translatedText = dazedwrap.wrapText(translatedText, width=1000)
|
||||
# Textwrap
|
||||
translatedText = dazedwrap.wrapText(translatedText, width=WIDTH)
|
||||
|
||||
# Set Data
|
||||
codeList[i]["p"][0] = f"{translatedText}"
|
||||
# Set Data - ensure list has at least one element
|
||||
if len(codeList[j]["p"]) == 0:
|
||||
codeList[j]["p"] = [""]
|
||||
codeList[j]["p"][0] = f"{translatedText}"
|
||||
|
||||
## Event Code: 108 (Script)
|
||||
if "c" in codeList[i] and (codeList[i]["c"] == 108) and CODE108 is True:
|
||||
|
|
@ -2449,15 +2517,23 @@ def searchSystem(data, pbar):
|
|||
totalTokens = [0, 0]
|
||||
context = "Reply with only the " + LANGUAGE + ' translation of the UI textbox."'
|
||||
|
||||
# Title
|
||||
response = translateAI(
|
||||
data["gameTitle"],
|
||||
" Reply with the " + LANGUAGE + " translation of the game title name",
|
||||
False,
|
||||
)
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
data["gameTitle"] = response[0].strip(".")
|
||||
# Title (handle both 'gameTitle' and 'game_title', skip if missing)
|
||||
game_title = data.get("gameTitle")
|
||||
if game_title is None:
|
||||
game_title = data.get("game_title")
|
||||
if game_title is not None:
|
||||
response = translateAI(
|
||||
game_title,
|
||||
" Reply with the " + LANGUAGE + " translation of the game title name",
|
||||
False,
|
||||
)
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
# Write back to whichever key existed
|
||||
if "gameTitle" in data:
|
||||
data["gameTitle"] = response[0].strip(".")
|
||||
else:
|
||||
data["game_title"] = response[0].strip(".")
|
||||
|
||||
# Terms
|
||||
for term in data["terms"]:
|
||||
|
|
@ -2510,26 +2586,44 @@ def searchSystem(data, pbar):
|
|||
# totalTokens[1] += response[1][1]
|
||||
# data['variables'][i] = response[0].replace('\"', '').strip()
|
||||
|
||||
# Messages
|
||||
# Messages and lists (handle both string and list values, log as Line1, Line2, ...)
|
||||
messages = data["terms"]
|
||||
for key, value in messages.items():
|
||||
response = translateAI(
|
||||
value,
|
||||
"Reply with only the "
|
||||
+ LANGUAGE
|
||||
+ ' translation of the battle text.\nTranslate "常時ダッシュ" as "Always Dash"\nTranslate "次の%1まで" as Next %1.',
|
||||
False,
|
||||
)
|
||||
translatedText = response[0]
|
||||
|
||||
# Remove characters that may break scripts
|
||||
charList = [".", '"', "\\n"]
|
||||
for char in charList:
|
||||
translatedText = translatedText.replace(char, "")
|
||||
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
messages[key] = translatedText
|
||||
if isinstance(value, list):
|
||||
new_list = []
|
||||
for idx, item in enumerate(value):
|
||||
if isinstance(item, str):
|
||||
response = translateAI(
|
||||
item,
|
||||
f"Reply with only the {LANGUAGE} translation of the battle text.\nTranslate '常時ダッシュ' as 'Always Dash'\nTranslate '次の%1まで' as Next %1.",
|
||||
False,
|
||||
)
|
||||
translatedText = response[0]
|
||||
charList = [".", '"', "\\n"]
|
||||
for char in charList:
|
||||
translatedText = translatedText.replace(char, "")
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
new_list.append(translatedText)
|
||||
else:
|
||||
new_list.append(item)
|
||||
# For logging: create Line1, Line2, ... keys for this list
|
||||
messages[key] = new_list
|
||||
elif isinstance(value, str):
|
||||
response = translateAI(
|
||||
value,
|
||||
f"Reply with only the {LANGUAGE} translation of the battle text.\nTranslate '常時ダッシュ' as 'Always Dash'\nTranslate '次の%1まで' as Next %1.",
|
||||
False,
|
||||
)
|
||||
translatedText = response[0]
|
||||
charList = [".", '"', "\\n"]
|
||||
for char in charList:
|
||||
translatedText = translatedText.replace(char, "")
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
messages[key] = translatedText
|
||||
else:
|
||||
messages[key] = value
|
||||
|
||||
return totalTokens
|
||||
|
||||
|
|
|
|||
|
|
@ -18,9 +18,14 @@ import tempfile
|
|||
|
||||
# Open AI
|
||||
load_dotenv()
|
||||
if os.getenv("api").replace(" ", "") != "":
|
||||
openai.base_url = os.getenv("api")
|
||||
openai.organization = os.getenv("org")
|
||||
if os.getenv("API_PROVIDER") == "gemini":
|
||||
openai.base_url = "https://generativelanguage.googleapis.com/v1beta/openai/"
|
||||
openai.organization = None
|
||||
else:
|
||||
api_url = os.getenv("api")
|
||||
if api_url and api_url.replace(" ", "") != "":
|
||||
openai.base_url = api_url
|
||||
openai.organization = os.getenv("org")
|
||||
openai.api_key = os.getenv("key")
|
||||
|
||||
# Globals
|
||||
|
|
@ -370,4 +375,4 @@ def translateAI(text, history, fullPromptFlag):
|
|||
pbar=PBAR,
|
||||
lock=LOCK,
|
||||
mismatchList=MISMATCH
|
||||
)
|
||||
)
|
||||
|
|
@ -130,6 +130,41 @@ def getPricingConfig(model):
|
|||
"batchSize": 30,
|
||||
"frequencyPenalty": 0.05
|
||||
}
|
||||
elif "gemini-2.0-flash-lite" in model:
|
||||
return {
|
||||
"inputAPICost": 0.075,
|
||||
"outputAPICost": 0.30,
|
||||
"batchSize": 30,
|
||||
"frequencyPenalty": 0.0
|
||||
}
|
||||
elif "gemini-2.0-flash" in model:
|
||||
return {
|
||||
"inputAPICost": 0.10,
|
||||
"outputAPICost": 0.40,
|
||||
"batchSize": 30,
|
||||
"frequencyPenalty": 0.0
|
||||
}
|
||||
elif "gemini-2.5-flash-lite" in model:
|
||||
return {
|
||||
"inputAPICost": 0.10,
|
||||
"outputAPICost": 0.40,
|
||||
"batchSize": 30,
|
||||
"frequencyPenalty": 0.0
|
||||
}
|
||||
elif "gemini-2.5-flash" in model:
|
||||
return {
|
||||
"inputAPICost": 0.30,
|
||||
"outputAPICost": 2.50,
|
||||
"batchSize": 30,
|
||||
"frequencyPenalty": 0.0
|
||||
}
|
||||
elif "gemini-2.5-pro" in model:
|
||||
return {
|
||||
"inputAPICost": 1.25,
|
||||
"outputAPICost": 10.00,
|
||||
"batchSize": 30,
|
||||
"frequencyPenalty": 0.0
|
||||
}
|
||||
else:
|
||||
# Fallback to environment variables
|
||||
return {
|
||||
|
|
@ -285,7 +320,7 @@ def createTranslationSchema(numLines):
|
|||
|
||||
|
||||
def translateText(system, user, history, penalty, formatType, model, numLines=None):
|
||||
"""Send translation request to OpenAI API"""
|
||||
"""Send translation request to the selected API"""
|
||||
# Ensure system content is not empty
|
||||
if not system or not str(system).strip():
|
||||
raise ValueError("System content cannot be empty")
|
||||
|
|
@ -330,42 +365,53 @@ def translateText(system, user, history, penalty, formatType, model, numLines=No
|
|||
if not message.get("content") or not str(message.get("content")).strip():
|
||||
raise ValueError(f"Message {i} has empty content: {message}")
|
||||
|
||||
# Call OpenAI API
|
||||
try:
|
||||
# --- API Call Logic ---
|
||||
api_provider = os.getenv("API_PROVIDER", "openai").lower()
|
||||
|
||||
# Base parameters for the API call
|
||||
params = {
|
||||
"model": model,
|
||||
"response_format": responseFormat,
|
||||
"messages": msg,
|
||||
}
|
||||
|
||||
# Provider-specific parameters
|
||||
if api_provider == "gemini":
|
||||
params["temperature"] = 0
|
||||
|
||||
# Handle thinking budget for Gemini
|
||||
thinking_budget_str = os.getenv("GEMINI_THINKING_BUDGET")
|
||||
if thinking_budget_str:
|
||||
try:
|
||||
thinking_budget = int(thinking_budget_str)
|
||||
params["extra_body"] = {
|
||||
'google': {
|
||||
'thinking_config': {
|
||||
'thinking_budget': thinking_budget
|
||||
}
|
||||
}
|
||||
}
|
||||
except (ValueError, TypeError):
|
||||
# Ignore if the value is not a valid integer
|
||||
pass
|
||||
|
||||
# frequency_penalty is not supported via the OpenAI compatibility layer for Gemini
|
||||
else: # Default to OpenAI behavior
|
||||
if "gpt-5" in model:
|
||||
response = openai.chat.completions.create(
|
||||
model=model,
|
||||
response_format=responseFormat,
|
||||
messages=msg,
|
||||
reasoning_effort="minimal"
|
||||
)
|
||||
params["reasoning_effort"] = "minimal"
|
||||
else:
|
||||
response = openai.chat.completions.create(
|
||||
model=model,
|
||||
response_format=responseFormat,
|
||||
messages=msg,
|
||||
temperature=0,
|
||||
frequency_penalty=penalty
|
||||
)
|
||||
params["temperature"] = 0
|
||||
params["frequency_penalty"] = penalty
|
||||
|
||||
# Call API
|
||||
try:
|
||||
response = openai.chat.completions.create(**params)
|
||||
except Exception as e:
|
||||
# If structured output fails, fallback to json_object
|
||||
if formatType == "json" and "json_schema" in str(responseFormat):
|
||||
responseFormat = {"type": "json_object"}
|
||||
if "gpt-5" in model:
|
||||
response = openai.chat.completions.create(
|
||||
model=model,
|
||||
response_format=responseFormat,
|
||||
messages=msg,
|
||||
reasoning_effort="minimal"
|
||||
)
|
||||
else:
|
||||
response = openai.chat.completions.create(
|
||||
model=model,
|
||||
response_format=responseFormat,
|
||||
messages=msg,
|
||||
temperature=0,
|
||||
frequency_penalty=penalty
|
||||
)
|
||||
params["response_format"] = responseFormat
|
||||
response = openai.chat.completions.create(**params)
|
||||
else:
|
||||
raise e
|
||||
|
||||
|
|
@ -567,7 +613,6 @@ def translateAI(text, history, fullPromptFlag, config, filename=None, pbar=None,
|
|||
return [text, [0, 0]]
|
||||
|
||||
with open(config.logFilePath, "a+", encoding="utf-8") as logFile:
|
||||
mismatch = False
|
||||
totalTokens = [0, 0]
|
||||
|
||||
if isinstance(text, list):
|
||||
|
|
@ -612,97 +657,116 @@ def translateAI(text, history, fullPromptFlag, config, filename=None, pbar=None,
|
|||
totalTokens[1] += estimate[1]
|
||||
continue
|
||||
|
||||
# Translate
|
||||
# --- Translation and Validation Retry Block ---
|
||||
max_retries = 2 # 1 initial attempt + 2 retries
|
||||
final_translations = None
|
||||
last_raw_translation = ""
|
||||
numLines = len(tItem) if isinstance(tItem, list) else None
|
||||
response = translateText(system, user, history, 0.05, formatType, config.model, numLines)
|
||||
translatedText = response.choices[0].message.content
|
||||
|
||||
# Retry if AI refused
|
||||
if not translatedText or '"error":' in translatedText:
|
||||
response = translateText(
|
||||
f"{system}\n You translate ALL content.",
|
||||
user, history, 0.1, formatType, "gpt-4.1" if config.model == "gpt-5" else "gpt-4o", numLines
|
||||
)
|
||||
for attempt in range(max_retries + 1):
|
||||
is_valid = True
|
||||
|
||||
# On retries, add a note to the system prompt
|
||||
current_system = system
|
||||
if attempt > 0:
|
||||
current_system += f"\n\nIMPORTANT: Your previous attempt was incorrect or incomplete. Please ensure the entire output is translated to {config.language} and contains no untranslated characters. Translate the following text again, ensuring the JSON structure is correct."
|
||||
if pbar:
|
||||
pbar.write(f"Retrying translation... (Attempt {attempt + 1}/{max_retries + 1})")
|
||||
|
||||
# Translate
|
||||
response = translateText(current_system, user, history, 0.05, formatType, config.model, numLines)
|
||||
translatedText = response.choices[0].message.content
|
||||
last_raw_translation = translatedText
|
||||
|
||||
# Update token count
|
||||
totalTokens[0] += response.usage.prompt_tokens
|
||||
totalTokens[1] += response.usage.completion_tokens
|
||||
# Update token count for this attempt
|
||||
totalTokens[0] += response.usage.prompt_tokens
|
||||
totalTokens[1] += response.usage.completion_tokens
|
||||
|
||||
# Clean the translation first for consistency
|
||||
translatedText = cleanTranslatedText(translatedText, config.language)
|
||||
# Clean the translation first for consistency
|
||||
cleaned_text = cleanTranslatedText(translatedText, config.language)
|
||||
|
||||
# Process translation result
|
||||
if translatedText:
|
||||
if isinstance(tItem, list):
|
||||
extractedTranslations = extractTranslation(translatedText, True, pbar)
|
||||
if extractedTranslations is None or len(tItem) != len(extractedTranslations):
|
||||
# Mismatch, try again
|
||||
response = translateText(system, user, history, 0.05, formatType, config.model, numLines)
|
||||
translatedText = response.choices[0].message.content
|
||||
totalTokens[0] += response.usage.prompt_tokens
|
||||
totalTokens[1] += response.usage.completion_tokens
|
||||
|
||||
# Clean and extract again
|
||||
translatedText = cleanTranslatedText(translatedText, config.language)
|
||||
extractedTranslations = extractTranslation(translatedText, True, pbar)
|
||||
if extractedTranslations is None or len(tItem) != len(extractedTranslations):
|
||||
# Format the JSON output for consistent mismatch logging
|
||||
formatted_mismatch_output = translatedText
|
||||
try:
|
||||
parsed_json = json.loads(translatedText)
|
||||
formatted_mismatch_output = json.dumps(parsed_json, indent=4, ensure_ascii=False)
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
pass
|
||||
|
||||
# Log mismatch
|
||||
with open(config.mismatchLogPath, "a+", encoding="utf-8") as mismatchFile:
|
||||
mismatchFile.write(f"Mismatch: {filename}\n")
|
||||
mismatchFile.write(f"Input:\n{subbedT}\n")
|
||||
mismatchFile.write(f"Output:\n{formatted_mismatch_output}\n")
|
||||
mismatch = True
|
||||
|
||||
# Format the JSON output for consistent logging
|
||||
formatted_output = translatedText
|
||||
try:
|
||||
# Try to parse and reformat the JSON for consistent formatting
|
||||
parsed_json = json.loads(translatedText)
|
||||
formatted_output = json.dumps(parsed_json, indent=4, ensure_ascii=False)
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
# If it's not valid JSON, keep the original
|
||||
pass
|
||||
|
||||
logFile.write(f"Input:\n{subbedT}\n")
|
||||
logFile.write(f"Output:\n{formatted_output}\n")
|
||||
|
||||
# Set results if no mismatch
|
||||
if not mismatch:
|
||||
tList[index] = extractedTranslations
|
||||
history = extractedTranslations[-config.maxHistory:]
|
||||
# Process and validate translation result
|
||||
if cleaned_text:
|
||||
if isinstance(tItem, list):
|
||||
extracted = extractTranslation(cleaned_text, True, pbar)
|
||||
|
||||
# Check 1: Mismatch in length
|
||||
if extracted is None or len(tItem) != len(extracted):
|
||||
is_valid = False
|
||||
|
||||
# Check 2: Untranslated content
|
||||
else:
|
||||
for line in extracted:
|
||||
if re.search(config.langRegex, str(line)):
|
||||
is_valid = False
|
||||
break
|
||||
|
||||
if is_valid:
|
||||
final_translations = extracted
|
||||
else:
|
||||
history = text[-config.maxHistory:] if isinstance(text, list) else text
|
||||
mismatch = False
|
||||
if filename and mismatchList is not None and filename not in mismatchList:
|
||||
mismatchList.append(filename)
|
||||
|
||||
# Update progress bar
|
||||
if lock and pbar is not None:
|
||||
with lock:
|
||||
pbar.update(len(tItem))
|
||||
# Check for untranslated content in single string
|
||||
if re.search(config.langRegex, cleaned_text):
|
||||
is_valid = False
|
||||
else:
|
||||
final_translations = cleaned_text.replace("Placeholder Text", "")
|
||||
else:
|
||||
# Single string translation - clean after getting the raw response
|
||||
cleanedText = cleanTranslatedText(translatedText, config.language)
|
||||
tList[index] = cleanedText.replace("Placeholder Text", "")
|
||||
else:
|
||||
if pbar:
|
||||
pbar.write(f"AI Refused: {tItem}\n")
|
||||
is_valid = False
|
||||
if pbar: pbar.write(f"AI Refused: {tItem}\n")
|
||||
|
||||
# If translation is valid, break the retry loop
|
||||
if is_valid:
|
||||
break
|
||||
|
||||
# --- End of Retry Block ---
|
||||
|
||||
# After the loop, handle the final result
|
||||
if final_translations is not None: # Success case
|
||||
formatted_output = last_raw_translation
|
||||
try:
|
||||
parsed_json = json.loads(last_raw_translation)
|
||||
formatted_output = json.dumps(parsed_json, indent=4, ensure_ascii=False)
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
pass
|
||||
logFile.write(f"Input:\n{subbedT}\n")
|
||||
logFile.write(f"Output:\n{formatted_output}\n")
|
||||
|
||||
if isinstance(tItem, list):
|
||||
tList[index] = final_translations
|
||||
history = final_translations[-config.maxHistory:]
|
||||
else:
|
||||
tList[index] = final_translations
|
||||
history = final_translations
|
||||
|
||||
if lock and pbar is not None:
|
||||
with lock:
|
||||
pbar.update(len(tItem) if isinstance(tItem, list) else 1)
|
||||
|
||||
else: # Failure case after all retries
|
||||
if pbar: pbar.write(f"Translation failed after {max_retries + 1} attempts. Check mismatch log.")
|
||||
|
||||
formatted_mismatch_output = last_raw_translation
|
||||
try:
|
||||
parsed_json = json.loads(last_raw_translation)
|
||||
formatted_mismatch_output = json.dumps(parsed_json, indent=4, ensure_ascii=False)
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
pass
|
||||
with open(config.mismatchLogPath, "a+", encoding="utf-8") as mismatchFile:
|
||||
mismatchFile.write(f"Failed after retries: {filename}\n")
|
||||
mismatchFile.write(f"Input:\n{subbedT}\n")
|
||||
mismatchFile.write(f"Final Output:\n{formatted_mismatch_output}\n")
|
||||
|
||||
if filename and mismatchList is not None and filename not in mismatchList:
|
||||
mismatchList.append(filename)
|
||||
|
||||
tList[index] = tItem
|
||||
history = text[-config.maxHistory:] if isinstance(text, list) else text
|
||||
|
||||
# Combine if multilist
|
||||
if isinstance(tList[0], list):
|
||||
if tList and isinstance(tList[0], list):
|
||||
tList = [t for sublist in tList for t in sublist]
|
||||
|
||||
# Return result
|
||||
if formatType == "json":
|
||||
return [tList, totalTokens]
|
||||
else:
|
||||
return [tList[0], totalTokens]
|
||||
return [tList[0], totalTokens]
|
||||
Loading…
Reference in a new issue