diff --git a/.env.example b/.env.example index b3b9e85..ce4081d 100644 --- a/.env.example +++ b/.env.example @@ -1,3 +1,17 @@ +# -------------------------------------------------- +# Gemini-Specific Settings (Optional) +# -------------------------------------------------- + +# Sets the thinking budget for Gemini models. +# - To turn on dynamic thinking, set to -1. +# - To disable thinking (on compatible models like 2.5 Flash), set to 0. +# - To set a specific token budget, use a positive integer (e.g., 8192). +# - Leave this variable out to use the model's default setting. +GEMINI_THINKING_BUDGET= + +# Set to "gemini" to use the Gemini API or "openai" for OpenAI (If empty it will default to openai.) +API_PROVIDER=openai + #API link, leave blank to use OpenAI API api="" diff --git a/modules/main.py b/modules/main.py index f0e7dd1..5fac3cb 100644 --- a/modules/main.py +++ b/modules/main.py @@ -239,4 +239,4 @@ def deleteFolderFiles(folderPath): for filename in os.listdir(folderPath): file_path = os.path.join(folderPath, filename) if file_path.endswith((".json", ".yaml", ".ks")): - os.remove(file_path) + os.remove(file_path) \ No newline at end of file diff --git a/modules/regex.py b/modules/regex.py index b837b1c..ffa5717 100644 --- a/modules/regex.py +++ b/modules/regex.py @@ -18,9 +18,14 @@ import tempfile # Open AI load_dotenv() -if os.getenv("api").replace(" ", "") != "": - openai.base_url = os.getenv("api") -openai.organization = os.getenv("org") +if os.getenv("API_PROVIDER") == "gemini": + openai.base_url = "https://generativelanguage.googleapis.com/v1beta/openai/" + openai.organization = None +else: + api_url = os.getenv("api") + if api_url and api_url.replace(" ", "") != "": + openai.base_url = api_url + openai.organization = os.getenv("org") openai.api_key = os.getenv("key") # Globals @@ -429,4 +434,4 @@ def translateAI(text, history, fullPromptFlag): pbar=PBAR, lock=LOCK, mismatchList=MISMATCH - ) + ) \ No newline at end of file diff --git a/modules/rpgmakerace.py b/modules/rpgmakerace.py index 2684272..9f7fbee 100644 --- a/modules/rpgmakerace.py +++ b/modules/rpgmakerace.py @@ -89,7 +89,7 @@ CODE102 = True # Optional CODE101 = False -CODE408 = False +CODE408 = True # Variables CODE122 = False @@ -544,15 +544,61 @@ def parseSystem(data, filename): with tqdm(total=total_units, bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar: pbar.desc = filename + input_log = {} + output_log = {} try: + # Helper to flatten lists to Line1, Line2, ... + def flatten_lines(prefix, values, target): + for idx, v in enumerate(values): + target[f"{prefix}Line{idx+1}"] = v + + # Capture pre-translation values + if "terms" in data: + for key, value in data["terms"].items(): + if isinstance(value, list): + flatten_lines(key, value, input_log) + else: + input_log[key] = value + for key in ["gameTitle", "game_title", "armor_types", "skill_types", "weapon_types"]: + if key in data: + val = data[key] + if isinstance(val, list): + flatten_lines(key, val, input_log) + else: + input_log[key] = val + # Run translation result = searchSystem(data, pbar) totalTokens[0] += result[0] totalTokens[1] += result[1] + # Capture post-translation values + if "terms" in data: + for key, value in data["terms"].items(): + if isinstance(value, list): + flatten_lines(key, value, output_log) + else: + output_log[key] = value + for key in ["gameTitle", "game_title", "armor_types", "skill_types", "weapon_types"]: + if key in data: + val = data[key] + if isinstance(val, list): + flatten_lines(key, val, output_log) + else: + output_log[key] = val except Exception as e: traceback.print_exc() return [data, totalTokens, e] finally: maybe_save_progress_yaml(data, filename, result) + # Log translation history for system.yaml in established line-by-line format + try: + with open("log/translationHistory.txt", "a", encoding="utf-8") as log_file: + log_file.write("Input:\n") + log_file.write(json.dumps(input_log, ensure_ascii=False, indent=4)) + log_file.write("\nOutput:\n") + log_file.write(json.dumps(output_log, ensure_ascii=False, indent=4)) + log_file.write("\n") + except Exception: + pass return [data, totalTokens, None] @@ -1091,7 +1137,7 @@ def searchCodes(page, pbar, jobList, filename): # Brackets if len(speakerList) == 0: - speakerList = re.findall(r"^【(.*?)】$|^【(.*?)】[\\]*[a-zA-Z]*\[.*\]$", jaString) + speakerList = re.findall(r"^(?:[\\]+[rlRL]\[[\w\d\-]+\])?【(.*?)】$|^(?:[\\]+[rlRL]\[[\w\d\-]+\])?【(.*?)】[\\]*[a-zA-Z]*\[.*\]$", jaString) if speakerList: if speakerList[0][0]: speakerList = [speakerList[0][0]] @@ -1114,9 +1160,9 @@ def searchCodes(page, pbar, jobList, filename): # First Line Speakers if len(speakerList) == 0 and FIRSTLINESPEAKERS is True: - # Remove any RPGMaker Code at start + # Remove any RPGMaker Code at start (including \r[...] and \l[...] patterns) ffMatch = re.search( - r"^((?:[\\]+[^cCnNiIkKvV]+\[[\d\w]+\])+)", + r"^((?:[\\]+[^cCnNiIkKvV]+\[[\d\w]+\])+|(?:[\\]+[rlRL]\[[\w\d\-]+\]))", jaString, ) if ffMatch != None: @@ -1152,7 +1198,13 @@ def searchCodes(page, pbar, jobList, filename): # Set Data if not setData: - codeList[i]["p"][0] = nametag + jaString.replace(speakerList[0], speaker) + # Check if there's a \r[...] or \l[...] code at the beginning + codePrefix = re.search(r"^([\\]+[rlRL]\[[\w\d\-]+\])", jaString) + if codePrefix: + # Replace only the speaker name, preserving the code and brackets + codeList[i]["p"][0] = jaString.replace(f"【{speakerList[0]}】", f"【{speaker}】") + else: + codeList[i]["p"][0] = nametag + jaString.replace(speakerList[0], speaker) nametag = "" # Iterate to next string @@ -1761,24 +1813,40 @@ def searchCodes(page, pbar, jobList, filename): ## Event Code: 408 (Script) if "c" in codeList[i] and (codeList[i]["c"] == 408) and CODE408 is True: - # Remove Textwrap - jaString = codeList[i]["p"][0] - # jaString = jaString.replace("\n", " ") + # Save starting index + j = i + jaString = codeList[i]["p"][0] if len(codeList[i]["p"]) > 0 else "" + + # Join consecutive 408 codes + combinedText = jaString + if len(codeList) > i + 1: + while i + 1 < len(codeList) and "c" in codeList[i + 1] and codeList[i + 1]["c"] == 408: + i += 1 + if len(codeList[i]["p"]) > 0: + combinedText += " " + codeList[i]["p"][0] + # Mark as -1 only in Pass 2 + if not setData: + codeList[i]["p"] = [] + codeList[i]["c"] = -1 # Pass 1 if setData: - list408.append(jaString) + if combinedText.strip(): # Only add non-empty text + list408.append(combinedText) # Pass 2 else: - translatedText = list408[0] - list408.pop(0) + if len(list408) > 0: + translatedText = list408[0] + list408.pop(0) - # Textwrap - translatedText = dazedwrap.wrapText(translatedText, width=1000) + # Textwrap + translatedText = dazedwrap.wrapText(translatedText, width=WIDTH) - # Set Data - codeList[i]["p"][0] = f"{translatedText}" + # Set Data - ensure list has at least one element + if len(codeList[j]["p"]) == 0: + codeList[j]["p"] = [""] + codeList[j]["p"][0] = f"{translatedText}" ## Event Code: 108 (Script) if "c" in codeList[i] and (codeList[i]["c"] == 108) and CODE108 is True: @@ -2449,15 +2517,23 @@ def searchSystem(data, pbar): totalTokens = [0, 0] context = "Reply with only the " + LANGUAGE + ' translation of the UI textbox."' - # Title - response = translateAI( - data["gameTitle"], - " Reply with the " + LANGUAGE + " translation of the game title name", - False, - ) - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - data["gameTitle"] = response[0].strip(".") + # Title (handle both 'gameTitle' and 'game_title', skip if missing) + game_title = data.get("gameTitle") + if game_title is None: + game_title = data.get("game_title") + if game_title is not None: + response = translateAI( + game_title, + " Reply with the " + LANGUAGE + " translation of the game title name", + False, + ) + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + # Write back to whichever key existed + if "gameTitle" in data: + data["gameTitle"] = response[0].strip(".") + else: + data["game_title"] = response[0].strip(".") # Terms for term in data["terms"]: @@ -2510,26 +2586,44 @@ def searchSystem(data, pbar): # totalTokens[1] += response[1][1] # data['variables'][i] = response[0].replace('\"', '').strip() - # Messages + # Messages and lists (handle both string and list values, log as Line1, Line2, ...) messages = data["terms"] for key, value in messages.items(): - response = translateAI( - value, - "Reply with only the " - + LANGUAGE - + ' translation of the battle text.\nTranslate "常時ダッシュ" as "Always Dash"\nTranslate "次の%1まで" as Next %1.', - False, - ) - translatedText = response[0] - - # Remove characters that may break scripts - charList = [".", '"', "\\n"] - for char in charList: - translatedText = translatedText.replace(char, "") - - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - messages[key] = translatedText + if isinstance(value, list): + new_list = [] + for idx, item in enumerate(value): + if isinstance(item, str): + response = translateAI( + item, + f"Reply with only the {LANGUAGE} translation of the battle text.\nTranslate '常時ダッシュ' as 'Always Dash'\nTranslate '次の%1まで' as Next %1.", + False, + ) + translatedText = response[0] + charList = [".", '"', "\\n"] + for char in charList: + translatedText = translatedText.replace(char, "") + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + new_list.append(translatedText) + else: + new_list.append(item) + # For logging: create Line1, Line2, ... keys for this list + messages[key] = new_list + elif isinstance(value, str): + response = translateAI( + value, + f"Reply with only the {LANGUAGE} translation of the battle text.\nTranslate '常時ダッシュ' as 'Always Dash'\nTranslate '次の%1まで' as Next %1.", + False, + ) + translatedText = response[0] + charList = [".", '"', "\\n"] + for char in charList: + translatedText = translatedText.replace(char, "") + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + messages[key] = translatedText + else: + messages[key] = value return totalTokens diff --git a/modules/text.py b/modules/text.py index 9e55d93..ae132ac 100644 --- a/modules/text.py +++ b/modules/text.py @@ -18,9 +18,14 @@ import tempfile # Open AI load_dotenv() -if os.getenv("api").replace(" ", "") != "": - openai.base_url = os.getenv("api") -openai.organization = os.getenv("org") +if os.getenv("API_PROVIDER") == "gemini": + openai.base_url = "https://generativelanguage.googleapis.com/v1beta/openai/" + openai.organization = None +else: + api_url = os.getenv("api") + if api_url and api_url.replace(" ", "") != "": + openai.base_url = api_url + openai.organization = os.getenv("org") openai.api_key = os.getenv("key") # Globals @@ -370,4 +375,4 @@ def translateAI(text, history, fullPromptFlag): pbar=PBAR, lock=LOCK, mismatchList=MISMATCH - ) + ) \ No newline at end of file diff --git a/util/translation.py b/util/translation.py index 3918af0..0f0505f 100644 --- a/util/translation.py +++ b/util/translation.py @@ -130,6 +130,41 @@ def getPricingConfig(model): "batchSize": 30, "frequencyPenalty": 0.05 } + elif "gemini-2.0-flash-lite" in model: + return { + "inputAPICost": 0.075, + "outputAPICost": 0.30, + "batchSize": 30, + "frequencyPenalty": 0.0 + } + elif "gemini-2.0-flash" in model: + return { + "inputAPICost": 0.10, + "outputAPICost": 0.40, + "batchSize": 30, + "frequencyPenalty": 0.0 + } + elif "gemini-2.5-flash-lite" in model: + return { + "inputAPICost": 0.10, + "outputAPICost": 0.40, + "batchSize": 30, + "frequencyPenalty": 0.0 + } + elif "gemini-2.5-flash" in model: + return { + "inputAPICost": 0.30, + "outputAPICost": 2.50, + "batchSize": 30, + "frequencyPenalty": 0.0 + } + elif "gemini-2.5-pro" in model: + return { + "inputAPICost": 1.25, + "outputAPICost": 10.00, + "batchSize": 30, + "frequencyPenalty": 0.0 + } else: # Fallback to environment variables return { @@ -285,7 +320,7 @@ def createTranslationSchema(numLines): def translateText(system, user, history, penalty, formatType, model, numLines=None): - """Send translation request to OpenAI API""" + """Send translation request to the selected API""" # Ensure system content is not empty if not system or not str(system).strip(): raise ValueError("System content cannot be empty") @@ -330,42 +365,53 @@ def translateText(system, user, history, penalty, formatType, model, numLines=No if not message.get("content") or not str(message.get("content")).strip(): raise ValueError(f"Message {i} has empty content: {message}") - # Call OpenAI API - try: + # --- API Call Logic --- + api_provider = os.getenv("API_PROVIDER", "openai").lower() + + # Base parameters for the API call + params = { + "model": model, + "response_format": responseFormat, + "messages": msg, + } + + # Provider-specific parameters + if api_provider == "gemini": + params["temperature"] = 0 + + # Handle thinking budget for Gemini + thinking_budget_str = os.getenv("GEMINI_THINKING_BUDGET") + if thinking_budget_str: + try: + thinking_budget = int(thinking_budget_str) + params["extra_body"] = { + 'google': { + 'thinking_config': { + 'thinking_budget': thinking_budget + } + } + } + except (ValueError, TypeError): + # Ignore if the value is not a valid integer + pass + + # frequency_penalty is not supported via the OpenAI compatibility layer for Gemini + else: # Default to OpenAI behavior if "gpt-5" in model: - response = openai.chat.completions.create( - model=model, - response_format=responseFormat, - messages=msg, - reasoning_effort="minimal" - ) + params["reasoning_effort"] = "minimal" else: - response = openai.chat.completions.create( - model=model, - response_format=responseFormat, - messages=msg, - temperature=0, - frequency_penalty=penalty - ) + params["temperature"] = 0 + params["frequency_penalty"] = penalty + + # Call API + try: + response = openai.chat.completions.create(**params) except Exception as e: # If structured output fails, fallback to json_object if formatType == "json" and "json_schema" in str(responseFormat): responseFormat = {"type": "json_object"} - if "gpt-5" in model: - response = openai.chat.completions.create( - model=model, - response_format=responseFormat, - messages=msg, - reasoning_effort="minimal" - ) - else: - response = openai.chat.completions.create( - model=model, - response_format=responseFormat, - messages=msg, - temperature=0, - frequency_penalty=penalty - ) + params["response_format"] = responseFormat + response = openai.chat.completions.create(**params) else: raise e @@ -567,7 +613,6 @@ def translateAI(text, history, fullPromptFlag, config, filename=None, pbar=None, return [text, [0, 0]] with open(config.logFilePath, "a+", encoding="utf-8") as logFile: - mismatch = False totalTokens = [0, 0] if isinstance(text, list): @@ -612,97 +657,116 @@ def translateAI(text, history, fullPromptFlag, config, filename=None, pbar=None, totalTokens[1] += estimate[1] continue - # Translate + # --- Translation and Validation Retry Block --- + max_retries = 2 # 1 initial attempt + 2 retries + final_translations = None + last_raw_translation = "" numLines = len(tItem) if isinstance(tItem, list) else None - response = translateText(system, user, history, 0.05, formatType, config.model, numLines) - translatedText = response.choices[0].message.content - # Retry if AI refused - if not translatedText or '"error":' in translatedText: - response = translateText( - f"{system}\n You translate ALL content.", - user, history, 0.1, formatType, "gpt-4.1" if config.model == "gpt-5" else "gpt-4o", numLines - ) + for attempt in range(max_retries + 1): + is_valid = True + + # On retries, add a note to the system prompt + current_system = system + if attempt > 0: + current_system += f"\n\nIMPORTANT: Your previous attempt was incorrect or incomplete. Please ensure the entire output is translated to {config.language} and contains no untranslated characters. Translate the following text again, ensuring the JSON structure is correct." + if pbar: + pbar.write(f"Retrying translation... (Attempt {attempt + 1}/{max_retries + 1})") + + # Translate + response = translateText(current_system, user, history, 0.05, formatType, config.model, numLines) translatedText = response.choices[0].message.content + last_raw_translation = translatedText - # Update token count - totalTokens[0] += response.usage.prompt_tokens - totalTokens[1] += response.usage.completion_tokens + # Update token count for this attempt + totalTokens[0] += response.usage.prompt_tokens + totalTokens[1] += response.usage.completion_tokens - # Clean the translation first for consistency - translatedText = cleanTranslatedText(translatedText, config.language) + # Clean the translation first for consistency + cleaned_text = cleanTranslatedText(translatedText, config.language) - # Process translation result - if translatedText: - if isinstance(tItem, list): - extractedTranslations = extractTranslation(translatedText, True, pbar) - if extractedTranslations is None or len(tItem) != len(extractedTranslations): - # Mismatch, try again - response = translateText(system, user, history, 0.05, formatType, config.model, numLines) - translatedText = response.choices[0].message.content - totalTokens[0] += response.usage.prompt_tokens - totalTokens[1] += response.usage.completion_tokens - - # Clean and extract again - translatedText = cleanTranslatedText(translatedText, config.language) - extractedTranslations = extractTranslation(translatedText, True, pbar) - if extractedTranslations is None or len(tItem) != len(extractedTranslations): - # Format the JSON output for consistent mismatch logging - formatted_mismatch_output = translatedText - try: - parsed_json = json.loads(translatedText) - formatted_mismatch_output = json.dumps(parsed_json, indent=4, ensure_ascii=False) - except (json.JSONDecodeError, ValueError): - pass - - # Log mismatch - with open(config.mismatchLogPath, "a+", encoding="utf-8") as mismatchFile: - mismatchFile.write(f"Mismatch: {filename}\n") - mismatchFile.write(f"Input:\n{subbedT}\n") - mismatchFile.write(f"Output:\n{formatted_mismatch_output}\n") - mismatch = True - - # Format the JSON output for consistent logging - formatted_output = translatedText - try: - # Try to parse and reformat the JSON for consistent formatting - parsed_json = json.loads(translatedText) - formatted_output = json.dumps(parsed_json, indent=4, ensure_ascii=False) - except (json.JSONDecodeError, ValueError): - # If it's not valid JSON, keep the original - pass - - logFile.write(f"Input:\n{subbedT}\n") - logFile.write(f"Output:\n{formatted_output}\n") - - # Set results if no mismatch - if not mismatch: - tList[index] = extractedTranslations - history = extractedTranslations[-config.maxHistory:] + # Process and validate translation result + if cleaned_text: + if isinstance(tItem, list): + extracted = extractTranslation(cleaned_text, True, pbar) + + # Check 1: Mismatch in length + if extracted is None or len(tItem) != len(extracted): + is_valid = False + + # Check 2: Untranslated content + else: + for line in extracted: + if re.search(config.langRegex, str(line)): + is_valid = False + break + + if is_valid: + final_translations = extracted else: - history = text[-config.maxHistory:] if isinstance(text, list) else text - mismatch = False - if filename and mismatchList is not None and filename not in mismatchList: - mismatchList.append(filename) - - # Update progress bar - if lock and pbar is not None: - with lock: - pbar.update(len(tItem)) + # Check for untranslated content in single string + if re.search(config.langRegex, cleaned_text): + is_valid = False + else: + final_translations = cleaned_text.replace("Placeholder Text", "") else: - # Single string translation - clean after getting the raw response - cleanedText = cleanTranslatedText(translatedText, config.language) - tList[index] = cleanedText.replace("Placeholder Text", "") - else: - if pbar: - pbar.write(f"AI Refused: {tItem}\n") + is_valid = False + if pbar: pbar.write(f"AI Refused: {tItem}\n") + + # If translation is valid, break the retry loop + if is_valid: + break + + # --- End of Retry Block --- + + # After the loop, handle the final result + if final_translations is not None: # Success case + formatted_output = last_raw_translation + try: + parsed_json = json.loads(last_raw_translation) + formatted_output = json.dumps(parsed_json, indent=4, ensure_ascii=False) + except (json.JSONDecodeError, ValueError): + pass + logFile.write(f"Input:\n{subbedT}\n") + logFile.write(f"Output:\n{formatted_output}\n") + + if isinstance(tItem, list): + tList[index] = final_translations + history = final_translations[-config.maxHistory:] + else: + tList[index] = final_translations + history = final_translations + + if lock and pbar is not None: + with lock: + pbar.update(len(tItem) if isinstance(tItem, list) else 1) + + else: # Failure case after all retries + if pbar: pbar.write(f"Translation failed after {max_retries + 1} attempts. Check mismatch log.") + + formatted_mismatch_output = last_raw_translation + try: + parsed_json = json.loads(last_raw_translation) + formatted_mismatch_output = json.dumps(parsed_json, indent=4, ensure_ascii=False) + except (json.JSONDecodeError, ValueError): + pass + with open(config.mismatchLogPath, "a+", encoding="utf-8") as mismatchFile: + mismatchFile.write(f"Failed after retries: {filename}\n") + mismatchFile.write(f"Input:\n{subbedT}\n") + mismatchFile.write(f"Final Output:\n{formatted_mismatch_output}\n") + + if filename and mismatchList is not None and filename not in mismatchList: + mismatchList.append(filename) + + tList[index] = tItem + history = text[-config.maxHistory:] if isinstance(text, list) else text # Combine if multilist - if isinstance(tList[0], list): + if tList and isinstance(tList[0], list): tList = [t for sublist in tList for t in sublist] # Return result if formatType == "json": return [tList, totalTokens] else: - return [tList[0], totalTokens] + return [tList[0], totalTokens] \ No newline at end of file