diff --git a/modules/alice.py b/modules/alice.py deleted file mode 100644 index d8edc71..0000000 --- a/modules/alice.py +++ /dev/null @@ -1,610 +0,0 @@ -# Libraries -import os -import re -import util.dazedwrap as dazedwrap -import threading -import time -import traceback -import tiktoken -import openai -from pathlib import Path -from colorama import Fore -from dotenv import load_dotenv -from retry import retry -from tqdm import tqdm - -# Open AI -load_dotenv() -if os.getenv("api").replace(" ", "") != "": - openai.base_url = os.getenv("api") -openai.organization = os.getenv("org") -openai.api_key = os.getenv("key") - -# Globals -MODEL = os.getenv("model") -TIMEOUT = int(os.getenv("timeout")) -LANGUAGE = os.getenv("language").capitalize() -PROMPT = Path("prompt.txt").read_text(encoding="utf-8") -VOCAB = Path("vocab.txt").read_text(encoding="utf-8") -THREADS = int(os.getenv("threads")) -LOCK = threading.Lock() -WIDTH = int(os.getenv("width")) -LISTWIDTH = int(os.getenv("listWidth")) -NOTEWIDTH = 70 -MAXHISTORY = 10 -ESTIMATE = "" -TOKENS = [0, 0] -NAMESLIST = [] -NAMES = False # Output a list of all the character names found -BRFLAG = False # If the game uses
instead -FIXTEXTWRAP = True # Overwrites textwrap -IGNORETLTEXT = False # Ignores all translated text. -MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong) - -# tqdm Globals -BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}" -POSITION = 0 -LEAVE = False - -# Regex - Need to change this if you want to translate from/to other languages. Default is Japanese Regex -LANGREGEX = r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+" - -# Pricing - Depends on the model https://openai.com/pricing -# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request -# If you are getting a MISMATCH LENGTH error, lower the batch size. -if "gpt-3.5" in MODEL: - INPUTAPICOST = 3.00 - OUTPUTAPICOST = 5.00 - BATCHSIZE = 10 - FREQUENCY_PENALTY = 0.2 -elif "gpt-4" in MODEL: - INPUTAPICOST = 2.0 - OUTPUTAPICOST = 8.00 - BATCHSIZE = 30 - FREQUENCY_PENALTY = 0.05 -elif "deepseek" in MODEL: - INPUTAPICOST = 0.27 - OUTPUTAPICOST = 1.10 - BATCHSIZE = 30 - FREQUENCY_PENALTY = 0.05 -else: - INPUTAPICOST = float(os.getenv("input_cost")) - OUTPUTAPICOST = float(os.getenv("output_cost")) - BATCHSIZE = int(os.getenv("batchsize")) - FREQUENCY_PENALTY = float(os.getenv("frequency_penalty")) - - -def handleAlice(filename, estimate): - global ESTIMATE - totalTokens = [0, 0] - ESTIMATE = estimate - - if estimate: - start = time.time() - translatedData = openFiles(filename) - - # Print Result - end = time.time() - tqdm.write(getResultString(translatedData, end - start, filename)) - with LOCK: - totalTokens[0] += translatedData[1][0] - totalTokens[1] += translatedData[1][1] - - # Print Total - totalString = getResultString(["", totalTokens, None], end - start, "TOTAL") - - # Print any errors on maps - if len(MISMATCH) > 0: - return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET - else: - return totalString - - else: - try: - with open("translated/" + filename, "w", encoding="utf-8", newline="\n") as outFile: - start = time.time() - translatedData = openFiles(filename) - - # Print Result - end = time.time() - outFile.writelines(translatedData[0]) - tqdm.write(getResultString(translatedData, end - start, filename)) - with LOCK: - totalTokens[0] += translatedData[1][0] - totalTokens[1] += translatedData[1][1] - except Exception: - traceback.print_exc() - return "Fail" - - return getResultString(["", totalTokens, None], end - start, "TOTAL") - - -def openFiles(filename): - with open("files/" + filename, "r", encoding="UTF-8") as f: - translatedData = parseText(f, filename) - - return translatedData - - -def getResultString(translatedData, translationTime, filename): - # File Print String - totalTokenstring = ( - Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]" - "[Output: " - + str(translatedData[1][1]) - + "]" "[Cost: ${:,.4f}".format(((translatedData[1][0] / 1000000) * INPUTAPICOST) + ((translatedData[1][1] / 1000000) * OUTPUTAPICOST)) - + "]" - ) - timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]" - - if translatedData[2] == None: - # Success - return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET - - else: - # Fail - try: - raise translatedData[2] - except Exception as e: - traceback.print_exc() - errorString = str(e) + Fore.RED - return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET - - -def parseText(data, filename): - # Get total for progress bar - linesList = data.readlines() - totalTokens = [0, 0] - totalLines = len(linesList) - global LOCK - - with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar: - pbar.desc = filename - pbar.total = totalLines - try: - result = translateLines(linesList, pbar) - totalTokens[0] += result[1][0] - totalTokens[1] += result[1][1] - except Exception as e: - traceback.print_exc() - return [linesList, totalTokens, e] - return [linesList, totalTokens, None] - - -# Grab scenario data from text file -def translateLines(linesList, pbar): - currentGroup = [] - batch = [] - textHistory = [] - tokens = [0, 0] - batchStartIndex = 0 - insertBool = False - multiLine = False - i = 0 - - try: - while i < len(linesList): - # Check if Proper Message - match = re.findall(r"s\[[0-9]+\] = \"(.*)\"", linesList[i]) - if len(match) > 0: - jaString = match[0] - - # Skip Files - if "／" in jaString: - i += 1 - continue - - ### Translate - # Remove any textwrap - jaString = re.sub(r"\\n", " ", jaString) - - # Grab Speaker - speakerMatch = re.findall(r"s\[[0-9]+\] = \"([^／]+)\"", linesList[i - 1]) - if len(speakerMatch) > 0: - # If there isn't any Japanese in the text just skip - if re.search(r"[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+", jaString) and "_" not in speakerMatch[0]: - speaker = speakerMatch[0] - else: - speaker = "" - else: - speaker = "" - - # Grab rest of the messages - currentGroup.append(jaString) - - # Check if next line should be merged - if insertBool is True: - linesList[i] = re.sub(r"(s\[[0-9]+\]) = \"(.+)\"", r'\1 = ""', linesList[i]) - linesList[i] = linesList[i].replace(";", "") - start = i - while len(linesList) > i + 1 and re.search(r"s\[[0-9]+\] = \"\s+(.*)\"", linesList[i + 1]) != None: - multiLine = True - i += 1 - match = re.findall(r"s\[[0-9]+\] = \"\s+(.*)\"", linesList[i]) - currentGroup.append(match[0]) - if insertBool is True: - linesList[i] = re.sub(r"(s\[[0-9]+\]) = \"\s+(.+)\"", r'\1 = ""', linesList[i]) - linesList[i] = linesList[i].replace(";", "") - i += 1 - - # Combine Groups and Add Speaker - finalJAString = " ".join(currentGroup) - if speaker != "": - finalJAString = f"{speaker}: {finalJAString}" - else: - finalJAString = f"{finalJAString}" - - # [Passthrough 1] Pulling From File - if insertBool is False: - # Append to List and Clear Values - batch.append(finalJAString) - - # Translate Batch if Full - if len(batch) == BATCHSIZE or i >= len(linesList) - 1: - # Translate - response = translateGPT(batch, textHistory, True) - tokens[0] += response[1][0] - tokens[1] += response[1][1] - translatedBatch = response[0] - textHistory = translatedBatch[-10:] - - # Set Values - if len(batch) == len(translatedBatch): - i = batchStartIndex - insertBool = True - - # Mismatch - else: - pbar.write(f"Mismatch: {batchStartIndex} - {i}") - MISMATCH.append(batch) - batchStartIndex = i - batch.clear() - - multiLine = False - currentGroup = [] - - # [Passthrough 2] Setting Data - else: - # Get Text - translatedText = translatedBatch[0] - - # Remove added speaker and quotes - translatedText = re.sub(r"^.+?:\s", "", translatedText) - - # Textwrap - translatedText = translatedText.replace('"', '\\"') - translatedText = dazedwrap.wrapText(translatedText, width=WIDTH) - - # Set Data - if multiLine: - textList = translatedText.split("\n") - for t in textList: - translatedText = translatedText.replace(";", "") - translatedText = re.sub( - r"(s\[[0-9]+\]) = \"(.*)\"", - rf'\1 = "{t}"', - linesList[start], - ) - translatedText = translatedText.replace(";", "") - linesList[start] = translatedText - pbar.update(1) - start += 1 - multiLine = False - translatedText = translatedText.replace(";", "") - translatedBatch.pop(0) - else: - # Remove any textwrap - translatedText = translatedText.replace("\n", " ") - translatedText = re.sub( - r"(s\[[0-9]+\]) = \"(.*)\"", - rf'\1 = "{translatedText}"', - linesList[start], - ) - translatedText = translatedText.replace(";", "") - linesList[start] = translatedText - pbar.update(1) - translatedBatch.pop(0) - - # If Batch is empty. Move on. - if len(translatedBatch) == 0: - insertBool = False - batchStartIndex = i - pbar.update(1) - batch.clear() - - currentGroup = [] - else: - if insertBool is True: - pbar.update(1) - i += 1 - - return [linesList, tokens] - except Exception: - traceback.print_exc() - return [linesList, tokens] - - -def subVars(jaString): - jaString = jaString.replace("\u3000", " ") - - # Nested - count = 0 - nestedList = re.findall(r"[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]", jaString) - nestedList = set(nestedList) - if len(nestedList) != 0: - for icon in nestedList: - jaString = jaString.replace(icon, "{Nested_" + str(count) + "}") - count += 1 - - # Icons - count = 0 - iconList = re.findall(r"[\\]+[iIkKwWaA]+\[[0-9]+\]", jaString) - iconList = set(iconList) - if len(iconList) != 0: - for icon in iconList: - jaString = jaString.replace(icon, "{Ascii_" + str(count) + "}") - count += 1 - - # Colors - count = 0 - colorList = re.findall(r"[\\]+[cC]\[[0-9]+\]", jaString) - colorList = set(colorList) - if len(colorList) != 0: - for color in colorList: - jaString = jaString.replace(color, "{Color_" + str(count) + "}") - count += 1 - - # Names - count = 0 - nameList = re.findall(r"[\\]+[nN]\[.+?\]+", jaString) - nameList = set(nameList) - if len(nameList) != 0: - for name in nameList: - jaString = jaString.replace(name, "{Noun_" + str(count) + "}") - count += 1 - - # Variables - count = 0 - varList = re.findall(r"[\\]+[vV]\[[0-9]+\]", jaString) - varList = set(varList) - if len(varList) != 0: - for var in varList: - jaString = jaString.replace(var, "{Var_" + str(count) + "}") - count += 1 - - # Formatting - count = 0 - formatList = re.findall(r"[\\]+[\w]+\[.+?\]", jaString) - formatList = set(formatList) - if len(formatList) != 0: - for var in formatList: - jaString = jaString.replace(var, "{FCode_" + str(count) + "}") - count += 1 - - # Put all lists in list and return - allList = [nestedList, iconList, colorList, nameList, varList, formatList] - return [jaString, allList] - - -def resubVars(translatedText, allList): - # Fix Spacing and ChatGPT Nonsense - matchList = re.findall(r"\[\s?.+?\s?\]", translatedText) - if len(matchList) > 0: - for match in matchList: - text = match.strip() - translatedText = translatedText.replace(match, text) - - # Nested - count = 0 - if len(allList[0]) != 0: - for var in allList[0]: - translatedText = translatedText.replace("{Nested_" + str(count) + "}", var) - count += 1 - - # Icons - count = 0 - if len(allList[1]) != 0: - for var in allList[1]: - translatedText = translatedText.replace("{Ascii_" + str(count) + "}", var) - count += 1 - - # Colors - count = 0 - if len(allList[2]) != 0: - for var in allList[2]: - translatedText = translatedText.replace("{Color_" + str(count) + "}", var) - count += 1 - - # Names - count = 0 - if len(allList[3]) != 0: - for var in allList[3]: - translatedText = translatedText.replace("{Noun_" + str(count) + "}", var) - count += 1 - - # Vars - count = 0 - if len(allList[4]) != 0: - for var in allList[4]: - translatedText = translatedText.replace("{Var_" + str(count) + "}", var) - count += 1 - - # Formatting - count = 0 - if len(allList[5]) != 0: - for var in allList[5]: - translatedText = translatedText.replace("{FCode_" + str(count) + "}", var) - count += 1 - - return translatedText - - -def batchList(input_list, batch_size): - if not isinstance(batch_size, int) or batch_size <= 0: - raise ValueError("batch_size must be a positive integer") - - return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)] - - -def createContext(fullPromptFlag, subbedT): - characters = "Game Characters:\n\ -林つかさ (Tsukasa Hayashi) - Female\n\ -山田美兎 (Miyato Yamada) - Female\n\ -鈴木赤音 (Akane Suzuki) - Female\n\ -佐藤莉伊南 (Riina Satou) - Female\n\ -佐々木万梨美 (Marimi Sasaki) - Female\n\ -渡辺登樹子 (Tokiko Watanabe) - Female\n\ -桃乃夢 (Yume Momono) - Female\n\ -吉浦美雪 (Miyuki Yoshiura) - Female\n\ -三ツ門まあな (Maana Mitsukado) - Female\n\ -モリー・ボイド (Molly Boyd) - Female\n\ -オルガ・ブヤチッチ (Olga Buyachich) - Female\n\ -アッチャラー ギッティ (Atchara Gitti) - Female\n\ -" - - system = ( - PROMPT - if fullPromptFlag - else f"Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`" - ) - user = f"{subbedT}" - return characters, system, user - - -def translateText(characters, system, user, history): - # Prompt - msg = [{"role": "system", "content": system + characters}] - - # Characters - msg.append({"role": "system", "content": characters}) - - # History - if isinstance(history, list): - msg.extend([{"role": "assistant", "content": h} for h in history]) - else: - msg.append({"role": "assistant", "content": history}) - - # Content to TL - msg.append({"role": "user", "content": f"{user}"}) - response = openai.chat.completions.create( - temperature=0.1, - frequency_penalty=0.1, - model=MODEL, - messages=msg, - ) - return response - - -def cleanTranslatedText(translatedText, varResponse): - placeholders = { - f"{LANGUAGE} Translation: ": "", - "Translation: ": "", - "っ": "", - "〜": "~", - "ッ": "", - "。": ".", - "Placeholder Text": "", - # Add more replacements as needed - } - for target, replacement in placeholders.items(): - translatedText = translatedText.replace(target, replacement) - - translatedText = resubVars(translatedText, varResponse[1]) - if "\n" in translatedText: - return [line for line in translatedText.split("\n") if line] - else: - return [line for line in translatedText.split("\\n") if line] - - -def extractTranslation(translatedTextList, is_list): - pattern = r"[\\]*`?(.*?)[\\]*?`?" - # If it's a batch (i.e., list), extract with tags; otherwise, return the single item. - if is_list: - return [re.findall(pattern, line)[0][1] for line in translatedTextList if re.search(pattern, line)] - else: - matchList = re.findall(pattern, translatedTextList) - return matchList[0][1] if matchList else translatedTextList - - -def countTokens(characters, system, user, history): - inputTotalTokens = 0 - outputTotalTokens = 0 - enc = tiktoken.encoding_for_model("gpt-4") - - # Input - if isinstance(history, list): - for line in history: - inputTotalTokens += len(enc.encode(line)) - else: - inputTotalTokens += len(enc.encode(history)) - inputTotalTokens += len(enc.encode(system)) - inputTotalTokens += len(enc.encode(characters)) - inputTotalTokens += len(enc.encode(user)) - - # Output - outputTotalTokens += round(len(enc.encode(user)) * 3) - - return [inputTotalTokens, outputTotalTokens] - - -@retry(exceptions=Exception, tries=5, delay=5) -def translateGPT(text, history, fullPromptFlag): - totalTokens = [0, 0] - if isinstance(text, list): - tList = batchList(text, BATCHSIZE) - else: - tList = [text] - - for index, tItem in enumerate(tList): - # Before sending to translation, if we have a list of items, add the formatting - if isinstance(tItem, list): - payload = "\n".join([f"`{item}`" for i, item in enumerate(tItem)]) - payload = payload.replace("``", "`Placeholder Text`") - varResponse = subVars(payload) - subbedT = varResponse[0] - else: - varResponse = subVars(tItem) - subbedT = varResponse[0] - - # Things to Check before starting translation - if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", subbedT): - continue - - # Create Message - characters, system, user = createContext(fullPromptFlag, subbedT) - - # Calculate Estimate - if ESTIMATE: - estimate = countTokens(characters, system, user, history) - totalTokens[0] += estimate[0] - totalTokens[1] += estimate[1] - continue - - # Translating - response = translateText(characters, system, user, history) - translatedText = response.choices[0].message.content - totalTokens[0] += response.usage.prompt_tokens - totalTokens[1] += response.usage.completion_tokens - - # Formatting - translatedTextList = cleanTranslatedText(translatedText, varResponse) - if isinstance(tItem, list): - extractedTranslations = extractTranslation(translatedTextList, True) - tList[index] = extractedTranslations - if len(tItem) != len(translatedTextList): - mismatch = True # Just here so breakpoint can be set - history = extractedTranslations[-10:] # Update history if we have a list - else: - # Ensure we're passing a single string to extractTranslation - extractedTranslations = extractTranslation("\n".join(translatedTextList), False) - tList[index] = extractedTranslations - - # Combine if multilist - if isinstance(tList[0], list): - tList = [t for sublist in tList for t in sublist] - - # Return - if format == "json": - return [tList, totalTokens] - else: - return [tList[0], totalTokens] diff --git a/modules/anim.py b/modules/anim.py deleted file mode 100644 index cdb0f83..0000000 --- a/modules/anim.py +++ /dev/null @@ -1,581 +0,0 @@ -# Libraries -import json -import os -import re -import util.dazedwrap as dazedwrap -import threading -import time -import traceback -import tiktoken -import openai -from pathlib import Path -from colorama import Fore -from dotenv import load_dotenv -from retry import retry -from tqdm import tqdm - -# Open AI -load_dotenv() -if os.getenv("api").replace(" ", "") != "": - openai.base_url = os.getenv("api") -openai.organization = os.getenv("org") -openai.api_key = os.getenv("key") - -# Globals -MODEL = os.getenv("model") -TIMEOUT = int(os.getenv("timeout")) -LANGUAGE = os.getenv("language").capitalize() -PROMPT = Path("prompt.txt").read_text(encoding="utf-8") -VOCAB = Path("vocab.txt").read_text(encoding="utf-8") -THREADS = int(os.getenv("threads")) -LOCK = threading.Lock() -WIDTH = int(os.getenv("width")) -LISTWIDTH = int(os.getenv("listWidth")) -NOTEWIDTH = 70 -MAXHISTORY = 10 -ESTIMATE = "" -TOKENS = [0, 0] -NAMESLIST = [] -NAMES = False # Output a list of all the character names found -BRFLAG = False # If the game uses
instead -FIXTEXTWRAP = True # Overwrites textwrap -IGNORETLTEXT = False # Ignores all translated text. -MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong) - -# tqdm Globals -BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}" -POSITION = 0 -LEAVE = False - -# Regex - Need to change this if you want to translate from/to other languages. Default is Japanese Regex -LANGREGEX = r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+" - -# Pricing - Depends on the model https://openai.com/pricing -# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request -# If you are getting a MISMATCH LENGTH error, lower the batch size. -if "gpt-3.5" in MODEL: - INPUTAPICOST = 3.00 - OUTPUTAPICOST = 5.00 - BATCHSIZE = 10 - FREQUENCY_PENALTY = 0.2 -elif "gpt-4" in MODEL: - INPUTAPICOST = 2.0 - OUTPUTAPICOST = 8.00 - BATCHSIZE = 30 - FREQUENCY_PENALTY = 0.05 -elif "deepseek" in MODEL: - INPUTAPICOST = 0.27 - OUTPUTAPICOST = 1.10 - BATCHSIZE = 30 - FREQUENCY_PENALTY = 0.05 -else: - INPUTAPICOST = float(os.getenv("input_cost")) - OUTPUTAPICOST = float(os.getenv("output_cost")) - BATCHSIZE = int(os.getenv("batchsize")) - FREQUENCY_PENALTY = float(os.getenv("frequency_penalty")) - - -def handleAnim(filename, estimate): - global ESTIMATE - totalTokens = [0, 0] - ESTIMATE = estimate - - if estimate: - start = time.time() - translatedData = openFiles(filename) - - # Print Result - end = time.time() - tqdm.write(getResultString(translatedData, end - start, filename)) - with LOCK: - totalTokens[0] += translatedData[1][0] - totalTokens[1] += translatedData[1][1] - - # Print Total - totalString = getResultString(["", totalTokens, None], end - start, "TOTAL") - - # Print any errors on maps - if len(MISMATCH) > 0: - return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET - else: - return totalString - - else: - try: - with open("translated/" + filename, "w", encoding="utf-8", newline="\n") as outFile: - start = time.time() - translatedData = openFiles(filename) - - # Print Result - end = time.time() - json.dump(translatedData[0], outFile, ensure_ascii=False, indent=4) - tqdm.write(getResultString(translatedData, end - start, filename)) - with LOCK: - totalTokens[0] += translatedData[1][0] - totalTokens[1] += translatedData[1][1] - except Exception: - return "Fail" - - return getResultString(["", totalTokens, None], end - start, "TOTAL") - - -def openFiles(filename): - with open("files/" + filename, "r", encoding="UTF-8-sig") as f: - data = json.load(f) - - # Map Files - if ".json" in filename: - translatedData = parseJSON(data, filename) - - else: - raise NameError(filename + " Not Supported") - - return translatedData - - -def getResultString(translatedData, translationTime, filename): - # File Print String - totalTokenstring = ( - Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]" - "[Output: " - + str(translatedData[1][1]) - + "]" "[Cost: ${:,.4f}".format(((translatedData[1][0] / 1000000) * INPUTAPICOST) + ((translatedData[1][1] / 1000000) * OUTPUTAPICOST)) - + "]" - ) - timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]" - - if translatedData[2] == None: - # Success - return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET - - else: - # Fail - try: - raise translatedData[2] - except Exception as e: - traceback.print_exc() - errorString = str(e) + Fore.RED - return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET - - -def parseJSON(data, filename): - keys = list(data.keys()) - batches = [keys[i : i + BATCHSIZE] for i in range(0, len(keys), BATCHSIZE)] - totalTokens = [0, 0] - totalLines = 0 - totalLines = len(batches) - global LOCK - - with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar: - pbar.desc = filename - pbar.total = totalLines - try: - result = translateJSON(batches, data, pbar) - totalTokens[0] += result[0] - totalTokens[1] += result[1] - except Exception as e: - traceback.print_exc() - return [data, totalTokens, e] - return [data, totalTokens, None] - - -def translateJSON(keys, data, pbar): - translatedBatch = [] - textHistory = [] - tokens = [0, 0] - - for batch in keys: - # Save Batch - originalBatch = batch.copy() - - # If there isn't any Japanese in the text just skip - needTL = False - for i in range(len(batch)): - t = data[batch[i]] - if re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", t) or t == "": - needTL = True - if needTL is False and IGNORETLTEXT is True: - pbar.update(1) - continue - - # Remove any textwrap and Furigana - for i in range(len(batch)): - if FIXTEXTWRAP == True: - # Textwrap - data[originalBatch[i]] = data[originalBatch[i]].replace("@b", " ") - - # Furigana - rcodeMatch = re.findall(r"(@\[(.+?):.+?\])", batch[i]) - if len(rcodeMatch) > 0: - for match in rcodeMatch: - batch[i] = batch[i].replace(match[0], match[1]) - - # Translate - if needTL is True: - response = translateGPT(batch, textHistory, True) - tokens[0] += response[1][0] - tokens[1] += response[1][1] - translatedBatch = response[0] - else: - for i in range(len(originalBatch)): - translatedBatch.append(data[originalBatch[i]]) - - # Format and Set Text - if len(batch) == len(translatedBatch): - for i in range(len(translatedBatch)): - # Remove added speaker - translatedText = translatedBatch[i] - translatedText = re.sub(r"^.+?\s\|\s?", "", translatedText) - - # Textwrap - if "@n" in translatedText: - match = re.search(r".*@n(.*)", translatedText) - if match != None: - tlText = match.group(1) - tlText = dazedwrap.wrapText(tlText, width=WIDTH) - tlText = tlText.replace("\n", "@b") - translatedText = translatedText.replace(match.group(1), tlText) - - elif "@b" not in translatedText: - translatedText = dazedwrap.wrapText(translatedText, width=WIDTH) - translatedText = translatedText.replace("\n", "@b") - - # Set Data - data[originalBatch[i]] = translatedText - textHistory = translatedBatch - translatedBatch.clear() - # Mismatch, Skip Batch - else: - MISMATCH.append(batch) - pbar.update(1) - continue - pbar.update(1) - - return tokens - - -def subVars(jaString): - jaString = jaString.replace("\u3000", " ") - - # Nested - count = 0 - nestedList = re.findall(r"[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]", jaString) - nestedList = set(nestedList) - if len(nestedList) != 0: - for icon in nestedList: - jaString = jaString.replace(icon, "[Nested_" + str(count) + "]") - count += 1 - - # Icons - count = 0 - iconList = re.findall(r"[\\]+[iIkKwWaA]+\[[0-9]+\]", jaString) - iconList = set(iconList) - if len(iconList) != 0: - for icon in iconList: - jaString = jaString.replace(icon, "[Ascii_" + str(count) + "]") - count += 1 - - # Colors - count = 0 - colorList = re.findall(r"[\\]+[cC]\[[0-9]+\]", jaString) - colorList = set(colorList) - if len(colorList) != 0: - for color in colorList: - jaString = jaString.replace(color, "[Color_" + str(count) + "]") - count += 1 - - # Names - count = 0 - nameList = re.findall(r"[\\]+[nN]\[.+?\]+", jaString) - nameList = set(nameList) - if len(nameList) != 0: - for name in nameList: - jaString = jaString.replace(name, "[Noun_" + str(count) + "]") - count += 1 - - # Variables - count = 0 - varList = re.findall(r"[\\]+[vV]\[[0-9]+\]", jaString) - varList = set(varList) - if len(varList) != 0: - for var in varList: - jaString = jaString.replace(var, "[Var_" + str(count) + "]") - count += 1 - - # Formatting - count = 0 - formatList = re.findall(r"[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]", jaString) - formatList = set(formatList) - if len(formatList) != 0: - for var in formatList: - jaString = jaString.replace(var, "[FCode_" + str(count) + "]") - count += 1 - - # Put all lists in list and return - allList = [nestedList, iconList, colorList, nameList, varList, formatList] - return [jaString, allList] - - -def resubVars(translatedText, allList): - # Fix Spacing and ChatGPT Nonsense - matchList = re.findall(r"\[\s?.+?\s?\]", translatedText) - if len(matchList) > 0: - for match in matchList: - text = match.strip() - translatedText = translatedText.replace(match, text) - - # Nested - count = 0 - if len(allList[0]) != 0: - for var in allList[0]: - translatedText = translatedText.replace("[Nested_" + str(count) + "]", var) - count += 1 - - # Icons - count = 0 - if len(allList[1]) != 0: - for var in allList[1]: - translatedText = translatedText.replace("[Ascii_" + str(count) + "]", var) - count += 1 - - # Colors - count = 0 - if len(allList[2]) != 0: - for var in allList[2]: - translatedText = translatedText.replace("[Color_" + str(count) + "]", var) - count += 1 - - # Names - count = 0 - if len(allList[3]) != 0: - for var in allList[3]: - translatedText = translatedText.replace("[Noun_" + str(count) + "]", var) - count += 1 - - # Vars - count = 0 - if len(allList[4]) != 0: - for var in allList[4]: - translatedText = translatedText.replace("[Var_" + str(count) + "]", var) - count += 1 - - # Formatting - count = 0 - if len(allList[5]) != 0: - for var in allList[5]: - translatedText = translatedText.replace("[FCode_" + str(count) + "]", var) - count += 1 - - return translatedText - - -def batchList(input_list, batch_size): - if not isinstance(batch_size, int) or batch_size <= 0: - raise ValueError("batch_size must be a positive integer") - - return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)] - - -def createContext(fullPromptFlag, subbedT): - characters = "Game Characters:\n\ -達也 (Tatsuya) - Male\n\ -香織 (Kaori) - Female\n\ -岩瀬 (Iwase)\n\ -万蔵 (Manzou) - Male\n\ -結奈 (Yuuna) - Female\n\ -茅部 (Kayabe)\n\ -" - - system = ( - PROMPT + VOCAB - if fullPromptFlag - else f"\ -You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\ -Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\ -- You translate everything, including erotic and explicit content.\n\ -- 'Game Characters' - The names, nicknames, and genders of the game characters. Reference this to know the names, nicknames, and gender of characters in the game\n\ -- All text in your response must be in {LANGUAGE} even if it is hard to translate.\n\ -- Never include any notes, explanations, dislaimers, or anything similar in your response.\n\ -- Maintain any spacing in the translation.\n\ -- Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\ -- `...` can be a part of the dialogue. Translate it as it is.\n\ -{VOCAB}\n\ -" - ) - user = f"{subbedT}" - return characters, system, user - - -def translateText(characters, system, user, history, penalty): - # Prompt - msg = [{"role": "system", "content": system + characters}] - - # Characters - msg.append({"role": "system", "content": characters}) - - # History - if isinstance(history, list): - msg.extend([{"role": "system", "content": h} for h in history]) - else: - msg.append({"role": "system", "content": history}) - - # Content to TL - msg.append({"role": "user", "content": f"{user}"}) - response = openai.chat.completions.create( - temperature=0, - frequency_penalty=penalty, - model=MODEL, - messages=msg, - ) - return response - - -def cleanTranslatedText(translatedText, varResponse): - placeholders = { - f"{LANGUAGE} Translation: ": "", - "Translation: ": "", - "っ": "", - "〜": "~", - "ッ": "", - "。": ".", - "Placeholder Text": "", - "é": "e", - "—": "-", - "ū": "u", - # Add more replacements as needed - } - for target, replacement in placeholders.items(): - translatedText = translatedText.replace(target, replacement) - - # Elongate Long Dashes (Since GPT Ignores them...) - translatedText = elongateCharacters(translatedText) - translatedText = resubVars(translatedText, varResponse[1]) - return translatedText - - -def elongateCharacters(text): - # Define a pattern to match one character followed by one or more `ー` characters - # Using a positive lookbehind assertion to capture the preceding character - pattern = r"(?<=(.))ー+" - - # Define a replacement function that elongates the captured character - def repl(match): - char = match.group(1) # The character before the ー sequence - count = len(match.group(0)) - 1 # Number of ー characters - return char * count # Replace ー sequence with the character repeated - - # Use re.sub() to replace the pattern in the text - return re.sub(pattern, repl, text) - - -def extractTranslation(translatedTextList, is_list): - pattern = r"`?<[Ll]ine\d+>([\\]*.*?[\\]*?)<\/?[Ll]ine\d+>`?" - # If it's a batch (i.e., list), extract with tags; otherwise, return the single item. - if is_list: - matchList = re.findall(pattern, translatedTextList) - return matchList - else: - matchList = re.findall(pattern, translatedTextList) - return matchList[0][0] if matchList else translatedTextList - - -def countTokens(characters, system, user, history): - inputTotalTokens = 0 - outputTotalTokens = 0 - enc = tiktoken.encoding_for_model("gpt-4") - - # Input - if isinstance(history, list): - for line in history: - inputTotalTokens += len(enc.encode(line)) - else: - inputTotalTokens += len(enc.encode(history)) - inputTotalTokens += len(enc.encode(system)) - inputTotalTokens += len(enc.encode(characters)) - inputTotalTokens += len(enc.encode(user)) - - # Output - outputTotalTokens += round(len(enc.encode(user)) * 3) - - return [inputTotalTokens, outputTotalTokens] - - -@retry(exceptions=Exception, tries=5, delay=5) -def translateGPT(text, history, fullPromptFlag): - mismatch = False - totalTokens = [0, 0] - if isinstance(text, list): - tList = batchList(text, BATCHSIZE) - else: - tList = [text] - - for index, tItem in enumerate(tList): - # Before sending to translation, if we have a list of items, add the formatting - if isinstance(tItem, list): - payload = "\n".join([f"`{item}`" for i, item in enumerate(tItem)]) - payload = re.sub(r"(<)(\/Line\d+>)", r"\1>Placeholder Text<\3", payload) - varResponse = subVars(payload) - subbedT = varResponse[0] - else: - varResponse = subVars(tItem) - subbedT = varResponse[0] - - # Things to Check before starting translation - if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", subbedT): - continue - - # Create Message - characters, system, user = createContext(fullPromptFlag, subbedT) - - # Calculate Estimate - if ESTIMATE: - estimate = countTokens(characters, system, user, history) - totalTokens[0] += estimate[0] - totalTokens[1] += estimate[1] - continue - - # Translating - response = translateText(characters, system, user, history, 0.02) - translatedText = response.choices[0].message.content - totalTokens[0] += response.usage.prompt_tokens - totalTokens[1] += response.usage.completion_tokens - - # Formatting - translatedText = cleanTranslatedText(translatedText, varResponse) - if isinstance(tItem, list): - extractedTranslations = extractTranslation(translatedText, True) - tList[index] = extractedTranslations - if len(tItem) != len(extractedTranslations): - # Mismatch. Try Again - response = translateText(characters, system, user, history, 0.1) - translatedText = response.choices[0].message.content - totalTokens[0] += response.usage.prompt_tokens - totalTokens[1] += response.usage.completion_tokens - - # Formatting - translatedText = cleanTranslatedText(translatedText, varResponse) - if isinstance(tItem, list): - extractedTranslations = extractTranslation(translatedText, True) - tList[index] = extractedTranslations - if len(tItem) != len(extractedTranslations): - mismatch = True # Just here for breakpoint - - # Create History - if not mismatch: - history = extractedTranslations[-10:] # Update history if we have a list - else: - history = text[-10:] - else: - # Ensure we're passing a single string to extractTranslation - extractedTranslations = extractTranslation(translatedText, False) - tList[index] = extractedTranslations - - # Combine if multilist - if isinstance(tList[0], list): - tList = [t for sublist in tList for t in sublist] - - # Return - if format == "json": - return [tList, totalTokens] - else: - return [tList[0], totalTokens] diff --git a/modules/atelier.py b/modules/atelier.py deleted file mode 100644 index 0ead901..0000000 --- a/modules/atelier.py +++ /dev/null @@ -1,396 +0,0 @@ -import os -from pathlib import Path -import re -import util.dazedwrap as dazedwrap -import threading -import time -import traceback -import tiktoken -from colorama import Fore -from dotenv import load_dotenv -import openai -from retry import retry -from tqdm import tqdm - -# Open AI -load_dotenv() -if os.getenv("api").replace(" ", "") != "": - openai.base_url = os.getenv("api") -openai.organization = os.getenv("org") -openai.api_key = os.getenv("key") - -# Globals -MODEL = os.getenv("model") -TIMEOUT = int(os.getenv("timeout")) -LANGUAGE = os.getenv("language").capitalize() -INPUTAPICOST = 0.002 # Depends on the model https://openai.com/pricing -OUTPUTAPICOST = 0.002 -PROMPT = Path("prompt.txt").read_text(encoding="utf-8") -VOCAB = Path("vocab.txt").read_text(encoding="utf-8") -THREADS = int(os.getenv("threads")) # Controls how many threads are working on a single file (May have to drop this) -LOCK = threading.Lock() -WIDTH = int(os.getenv("width")) -LISTWIDTH = int(os.getenv("listWidth")) -NOTEWIDTH = 40 -MAXHISTORY = 10 -ESTIMATE = "" -totalTokens = [0, 0] -NAMESLIST = [] - -# tqdm Globals -BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}" -POSITION = 0 -LEAVE = False - -# Translation Flags -FIXTEXTWRAP = True -IGNORETLTEXT = True - - -def handleAtelier(filename, estimate): - global ESTIMATE, totalTokens - ESTIMATE = estimate - - if estimate: - start = time.time() - translatedData = openFiles(filename) - - # Print Result - end = time.time() - tqdm.write(getResultString(translatedData, end - start, filename)) - with LOCK: - totalTokens[0] += translatedData[1][0] - totalTokens[1] += translatedData[1][1] - - return getResultString(["", totalTokens, None], end - start, "TOTAL") - - else: - try: - with open("translated/" + filename, "w", encoding="utf-8", newline="\n") as outFile: - start = time.time() - translatedData = openFiles(filename) - outFile.writelines(translatedData[0]) - - # Print Result - end = time.time() - tqdm.write(getResultString(translatedData, end - start, filename)) - with LOCK: - totalTokens[0] += translatedData[1][0] - totalTokens[1] += translatedData[1][1] - except Exception: - return "Fail" - - return getResultString(["", totalTokens, None], end - start, "TOTAL") - - -def openFiles(filename): - with open("files/" + filename, "r", encoding="UTF-8") as f: - translatedData = parseText(f, filename) - - return translatedData - - -def getResultString(translatedData, translationTime, filename): - # File Print String - totalTokenstring = ( - Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]" - "[Output: " - + str(translatedData[1][1]) - + "]" "[Cost: ${:,.4f}".format(((translatedData[1][0] / 1000000) * INPUTAPICOST) + ((translatedData[1][1] / 1000000) * OUTPUTAPICOST)) - + "]" - ) - timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]" - - if translatedData[2] is None: - # Success - return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET - - else: - # Fail - try: - raise translatedData[2] - except Exception as e: - errorString = str(e) + Fore.RED - return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET - - -def parseText(data, filename): - totalLines = 0 - global LOCK - - # Get total for progress bar - linesList = data.readlines() - totalLines = len(linesList) - - with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar: - pbar.desc = filename - pbar.total = totalLines - try: - response = translateText(linesList, pbar) - except Exception as e: - traceback.print_exc() - return [linesList, 0, e] - return [response[0], response[1], None] - - -def translateText(data, pbar): - textHistory = [] - maxHistory = MAXHISTORY - totalTokens = [0, 0] - syncIndex = 0 - - for i in range(len(data)): - if syncIndex > i: - i = syncIndex - - match = re.findall(r"◆.+◆(.+)", data[i]) - if len(match) > 0: - jaString = match[0] - - ### Translate - # Remove any textwrap - finalJAString = re.sub(r"\\n", " ", jaString) - - # Translate - response = translateGPT( - finalJAString, - "Previous Text for Context: " + " ".join(textHistory), - True, - ) - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - translatedText = response[0] - - # TextHistory is what we use to give GPT Context, so thats appended here. - textHistory.append('"' + translatedText + '"') - - # Keep textHistory list at length maxHistory - if len(textHistory) > maxHistory: - textHistory.pop(0) - - # Textwrap - translatedText = dazedwrap.wrapText(translatedText, width=WIDTH) - translatedText = translatedText.replace("\n", "\\n") - - # Write - data[i] = data[i].replace(match[0], translatedText) - - syncIndex = i + 1 - pbar.update() - return [data, totalTokens] - - -def subVars(jaString): - jaString = jaString.replace("\u3000", " ") - - # Nested - count = 0 - nestedList = re.findall(r"[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]", jaString) - nestedList = set(nestedList) - if len(nestedList) != 0: - for icon in nestedList: - jaString = jaString.replace(icon, "{Nested_" + str(count) + "}") - count += 1 - - # Icons - count = 0 - iconList = re.findall(r"[\\]+[iIkKwWaA]+\[[0-9]+\]", jaString) - iconList = set(iconList) - if len(iconList) != 0: - for icon in iconList: - jaString = jaString.replace(icon, "{Ascii_" + str(count) + "}") - count += 1 - - # Colors - count = 0 - colorList = re.findall(r"[\\]+[cC]\[[0-9]+\]", jaString) - colorList = set(colorList) - if len(colorList) != 0: - for color in colorList: - jaString = jaString.replace(color, "{Color_" + str(count) + "}") - count += 1 - - # Names - count = 0 - nameList = re.findall(r"[\\]+[nN]\[.+?\]+", jaString) - nameList = set(nameList) - if len(nameList) != 0: - for name in nameList: - jaString = jaString.replace(name, "{N_" + str(count) + "}") - count += 1 - - # Variables - count = 0 - varList = re.findall(r"[\\]+[vV]\[[0-9]+\]", jaString) - varList = set(varList) - if len(varList) != 0: - for var in varList: - jaString = jaString.replace(var, "{Var_" + str(count) + "}") - count += 1 - - # Formatting - count = 0 - if "笑えるよね." in jaString: - print("t") - formatList = re.findall(r"[\\]+[\w]+\[.+?\]", jaString) - formatList = set(formatList) - if len(formatList) != 0: - for var in formatList: - jaString = jaString.replace(var, "{FCode_" + str(count) + "}") - count += 1 - - # Put all lists in list and return - allList = [nestedList, iconList, colorList, nameList, varList, formatList] - return [jaString, allList] - - -def resubVars(translatedText, allList): - # Fix Spacing and ChatGPT Nonsense - matchList = re.findall(r"\[\s?.+?\s?\]", translatedText) - if len(matchList) > 0: - for match in matchList: - text = match.strip() - translatedText = translatedText.replace(match, text) - - # Nested - count = 0 - if len(allList[0]) != 0: - for var in allList[0]: - translatedText = translatedText.replace("{Nested_" + str(count) + "}", var) - count += 1 - - # Icons - count = 0 - if len(allList[1]) != 0: - for var in allList[1]: - translatedText = translatedText.replace("{Ascii_" + str(count) + "}", var) - count += 1 - - # Colors - count = 0 - if len(allList[2]) != 0: - for var in allList[2]: - translatedText = translatedText.replace("{Color_" + str(count) + "}", var) - count += 1 - - # Names - count = 0 - if len(allList[3]) != 0: - for var in allList[3]: - translatedText = translatedText.replace("{N_" + str(count) + "}", var) - count += 1 - - # Vars - count = 0 - if len(allList[4]) != 0: - for var in allList[4]: - translatedText = translatedText.replace("{Var_" + str(count) + "}", var) - count += 1 - - # Formatting - count = 0 - if len(allList[5]) != 0: - for var in allList[5]: - translatedText = translatedText.replace("{FCode_" + str(count) + "}", var) - count += 1 - - # Remove Color Variables Spaces - # if '\\c' in translatedText: - # translatedText = re.sub(r'\s*(\\+c\[[1-9]+\])\s*', r' \1', translatedText) - # translatedText = re.sub(r'\s*(\\+c\[0+\])', r'\1', translatedText) - return translatedText - - -@retry(exceptions=Exception, tries=5, delay=5) -def translateGPT(t, history, fullPromptFlag): - # Sub Vars - varResponse = subVars(t) - subbedT = varResponse[0] - - # If there isn't any Japanese in the text just skip - if not re.search(r"[一-龠]+|[ぁ-ゔ]+|[ァ-ヴ]+|[\uFF00-\uFFEF]", subbedT): - return (t, [0, 0]) - - # If ESTIMATE is True just count this as an execution and return. - if ESTIMATE: - enc = tiktoken.encoding_for_model("gpt-4") - historyRaw = "" - if isinstance(history, list): - for line in history: - historyRaw += line - else: - historyRaw = history - - inputTotalTokens = len(enc.encode(historyRaw)) + len(enc.encode(PROMPT)) - outputTotalTokens = len(enc.encode(t)) * 2 # Estimating 2x the size of the original text - totalTokens = [inputTotalTokens, outputTotalTokens] - return (t, totalTokens) - - # Characters - context = "Game Characters:\ - Character: Surname:久高 Name:有史 == Surname:Kudaka Name:Yuushi - Gender: Male\ - Character: Surname:葛城 Name:碧璃 == Surname:Katsuragi Name:Midori - Gender: Female\ - Character: Surname:葛城 Name:依理子 == Surname:Katsuragi Name:Yoriko - Gender: Female\ - Character: Surname:桐乃木 Name:奏 == Surname:Kirinogi Name:Kanade - Gender: Female\ - Character: Surname:葛城 Name:光男 == Surname:Katsuragi Name:Mitsuo - Gender: Male\ - Character: Surname:尾木 Name:優真 == Surname:Ogi Name:Yuuma - Gender: Male" - - # Prompt - if fullPromptFlag: - system = PROMPT - user = "Line to Translate = " + subbedT - else: - system = "Output ONLY the " + LANGUAGE + " translation in the following format: `Translation: <" + LANGUAGE.upper() + "_TRANSLATION>`" - user = "Line to Translate = " + subbedT - - # Create Message List - msg = [] - msg.append({"role": "system", "content": system}) - msg.append({"role": "user", "content": context}) - if isinstance(history, list): - for line in history: - msg.append({"role": "user", "content": line}) - else: - msg.append({"role": "user", "content": history}) - msg.append({"role": "user", "content": user}) - - response = openai.ChatCompletion.create( - temperature=0, - frequency_penalty=0.2, - presence_penalty=0.2, - model=MODEL, - messages=msg, - request_timeout=TIMEOUT, - ) - - # Save Translated Text - translatedText = response.choices[0].message.content - totalTokens = [response.usage.prompt_tokens, response.usage.completion_tokens] - - # Resub Vars - translatedText = resubVars(translatedText, varResponse[1]) - - # Remove Placeholder Text - translatedText = translatedText.replace(LANGUAGE + " Translation: ", "") - translatedText = translatedText.replace("Translation: ", "") - translatedText = translatedText.replace("Line to Translate = ", "") - translatedText = translatedText.replace("Translation = ", "") - translatedText = translatedText.replace("Translate = ", "") - translatedText = translatedText.replace(LANGUAGE + " Translation:", "") - translatedText = translatedText.replace("Translation:", "") - translatedText = translatedText.replace("Line to Translate =", "") - translatedText = translatedText.replace("Translation =", "") - translatedText = translatedText.replace("Translate =", "") - translatedText = translatedText.replace("っ", "") - translatedText = translatedText.replace("ッ", "") - translatedText = translatedText.replace("ぁ", "") - translatedText = translatedText.replace("。", ".") - translatedText = translatedText.replace("、", ",") - translatedText = translatedText.replace("？", "?") - translatedText = translatedText.replace("！", "!") - - # Return Translation - if len(translatedText) > 15 * len(t) or "I'm sorry, but I'm unable to assist with that translation" in translatedText: - raise Exception - else: - return [translatedText, totalTokens] diff --git a/modules/csv.py b/modules/csv.py index 0dc8e30..b794059 100644 --- a/modules/csv.py +++ b/modules/csv.py @@ -439,7 +439,6 @@ def translateCSV(data, pbar, writer, filename, translatedList, format): return totalTokens - # Save some money and enter the character before translation def getSpeaker(speaker): match speaker: @@ -453,15 +452,11 @@ def getSpeaker(speaker): if speaker == NAMESLIST[i][0]: return [NAMESLIST[i][1], [0, 0]] - # If there isn't any Japanese in the text just skip - if not re.search(LANGREGEX, speaker): - return [speaker, [0, 0]] - # Translate and Store Speaker response = translateGPT( f"{speaker}", "Reply with the " + LANGUAGE + " translation of the NPC name.", - True, + False, ) response[0] = response[0].title() response[0] = response[0].replace("'S", "'s") @@ -490,11 +485,68 @@ def batchList(input_list, batch_size): return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)] +def parseVocabWithCategories(vocabText): + """Parse vocabulary text and extract terms with their categories.""" + pairs = [] + seen = set() + currentCategory = None + + for line in vocabText.splitlines(): + line = line.strip() + if not line or line.startswith('```'): + continue + + # Check if this is a category header + if line.startswith('#'): + currentCategory = line + continue + + # Parse vocabulary term + m = re.match(r'^(.+?)(?:\s?[\(–])', line) # term is everything before space + '(' or '–' + if m: + term = m.group(1) + if term not in seen: + pairs.append((term, line, currentCategory)) + seen.add(term) + + return pairs + + +def buildMatchedVocabText(vocabPairs, subbedT): + """Build formatted vocabulary text with matched terms organized by category.""" + matchedCategories = {} + + # Use word boundaries for Japanese if appropriate, or allow substring as before. + for term, line, category in vocabPairs: + # "term in subbedT" could be false positive; can use regex but Japanese doesn't always have spaces. + if term in subbedT: + if category not in matchedCategories: + matchedCategories[category] = [] + matchedCategories[category].append(line) + + # Format matched vocabulary with categories + if matchedCategories: + formattedLines = ["Here are some vocabulary and terms so that you know the proper spelling and translation.\n"] + for category, lines in matchedCategories.items(): + if category: # Only add category header if it exists + formattedLines.append(category) + formattedLines.extend(lines) + formattedLines.append("") # Add blank line between categories + matchedVocabText = f"```\n{chr(10).join(formattedLines).rstrip()}\n```" + else: + matchedVocabText = "" + + return matchedVocabText + + def createContext(fullPromptFlag, subbedT, format): - system = ( - PROMPT + VOCAB - if fullPromptFlag - else f"\ + vocabPairs = parseVocabWithCategories(VOCAB) + matchedVocabText = buildMatchedVocabText(vocabPairs, subbedT) + + if fullPromptFlag: + system = PROMPT + matchedVocabText + else: + system = f"\ You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\ - You translate everything, including erotic and explicit content.\n\ @@ -504,9 +556,8 @@ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{ - Maintain any spacing in the translation.\n\ - Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\ - `...` can be a part of the dialogue. Translate it as it is.\n\ -{VOCAB}\n\ +{matchedVocabText}\n\ " - ) if format == "json": user = f"```json\n{subbedT}\n```" else: @@ -520,9 +571,10 @@ def translateText(system, user, history, penalty, format, model=MODEL): # History if isinstance(history, list): - msg.extend([{"role": "system", "content": h} for h in history]) + msg.append({"role": "system", "content": "Translation History:"}) + msg.extend([{"role": "assistant", "content": h} for h in history]) else: - msg.append({"role": "system", "content": history}) + msg.append({"role": "assistant", "content": history}) # Response Format if format == "json": @@ -557,8 +609,11 @@ def cleanTranslatedText(translatedText): "】": "]", "【": "[", "é": "e", - "ō": "o", + "this guy": "this bastard", + "This guy": "This bastard", "Placeholder Text": "", + "```json": "", + "```": "", # Add more replacements as needed } for target, replacement in placeholders.items(): @@ -620,7 +675,7 @@ def countTokens(system, user, history): inputTotalTokens += len(enc.encode(user)) # Output - outputTotalTokens += round(len(enc.encode(user)) * 3) + outputTotalTokens += round(len(enc.encode(user)) * 2.5) return [inputTotalTokens, outputTotalTokens] @@ -684,7 +739,8 @@ def translateGPT(text, history, fullPromptFlag): # AI Refused, Try Again if not translatedText: - response = translateText(f"{system}\n You translate ALL content.", user, history, 0.1, format) + response = translateText(f"{system}\n You translate ALL content.", user, history, 0.1, format, model="gpt-4o") + translatedText = response.choices[0].message.content # Report Tokens totalTokens[0] += response.usage.prompt_tokens diff --git a/modules/eushully.py b/modules/eushully.py deleted file mode 100644 index cb850d3..0000000 --- a/modules/eushully.py +++ /dev/null @@ -1,747 +0,0 @@ -# Libraries -import os -import re -import util.dazedwrap as dazedwrap -import threading -import time -import traceback -import tiktoken -import openai -from pathlib import Path -from colorama import Fore -from dotenv import load_dotenv -from retry import retry -from tqdm import tqdm - -# Open AI -load_dotenv() -if os.getenv("api").replace(" ", "") != "": - openai.base_url = os.getenv("api") -openai.organization = os.getenv("org") -openai.api_key = os.getenv("key") - -# Globals -MODEL = os.getenv("model") -TIMEOUT = int(os.getenv("timeout")) -LANGUAGE = os.getenv("language").capitalize() -PROMPT = Path("prompt.txt").read_text(encoding="utf-8") -VOCAB = Path("vocab.txt").read_text(encoding="utf-8") -THREADS = int(os.getenv("threads")) -LOCK = threading.Lock() -WIDTH = int(os.getenv("width")) -LISTWIDTH = int(os.getenv("listWidth")) -NOTEWIDTH = 70 -MAXHISTORY = 10 -ESTIMATE = "" -TOKENS = [0, 0] -NAMESLIST = [] -NAMES = False # Output a list of all the character names found -BRFLAG = False # If the game uses
instead -FIXTEXTWRAP = True # Overwrites textwrap -IGNORETLTEXT = False # Ignores all translated text. -MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong) - -# tqdm Globals -BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}" -POSITION = 0 -LEAVE = False -PBAR = None - -# Regex - Need to change this if you want to translate from/to other languages. Default is Japanese Regex -LANGREGEX = r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+" - -# Pricing - Depends on the model https://openai.com/pricing -# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request -# If you are getting a MISMATCH LENGTH error, lower the batch size. -if "gpt-3.5" in MODEL: - INPUTAPICOST = 3.00 - OUTPUTAPICOST = 5.00 - BATCHSIZE = 10 - FREQUENCY_PENALTY = 0.2 -elif "gpt-4" in MODEL: - INPUTAPICOST = 2.0 - OUTPUTAPICOST = 8.00 - BATCHSIZE = 30 - FREQUENCY_PENALTY = 0.05 -elif "deepseek" in MODEL: - INPUTAPICOST = 0.27 - OUTPUTAPICOST = 1.10 - BATCHSIZE = 30 - FREQUENCY_PENALTY = 0.05 -else: - INPUTAPICOST = float(os.getenv("input_cost")) - OUTPUTAPICOST = float(os.getenv("output_cost")) - BATCHSIZE = int(os.getenv("batchsize")) - FREQUENCY_PENALTY = float(os.getenv("frequency_penalty")) - - -def handleEushully(filename, estimate): - global ESTIMATE - ESTIMATE = estimate - - if ESTIMATE: - start = time.time() - translatedData = openFiles(filename) - - # Print Result - end = time.time() - tqdm.write(getResultString(translatedData, end - start, filename)) - with LOCK: - TOKENS[0] += translatedData[1][0] - TOKENS[1] += translatedData[1][1] - - # Print Total - totalString = getResultString(["", TOKENS, None], end - start, "TOTAL") - - # Print any errors on maps - if len(MISMATCH) > 0: - return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET - else: - return totalString - - else: - try: - with open("translated/" + filename, "w", encoding="utf-8", errors="ignore") as outFile: - start = time.time() - translatedData = openFiles(filename) - - # Print Result - end = time.time() - outFile.writelines(translatedData[0]) - tqdm.write(getResultString(translatedData, end - start, filename)) - with LOCK: - TOKENS[0] += translatedData[1][0] - TOKENS[1] += translatedData[1][1] - except Exception: - traceback.print_exc() - return "Fail" - - return getResultString(["", TOKENS, None], end - start, "TOTAL") - - -def getResultString(translatedData, translationTime, filename): - # File Print String - totalTokenstring = ( - Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]" - "[Output: " - + str(translatedData[1][1]) - + "]" "[Cost: ${:,.4f}".format(((translatedData[1][0] / 1000000) * INPUTAPICOST) + ((translatedData[1][1] / 1000000) * OUTPUTAPICOST)) - + "]" - ) - timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]" - - if translatedData[2] == None: - # Success - return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET - - else: - # Fail - try: - raise translatedData[2] - except Exception as e: - traceback.print_exc() - errorString = str(e) + Fore.RED - return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET - - -def openFiles(filename): - with open("files/" + filename, "r", encoding="utf-8") as readFile: - translatedData = parseRegex(readFile, filename) - - # Delete lines marked for deletion - finalData = [] - for line in translatedData[0]: - if line != "\\d\n": - finalData.append(line) - translatedData[0] = finalData - - return translatedData - - -def parseRegex(readFile, filename): - totalTokens = [0, 0] - - # Read File into data - data = readFile.readlines() - - # Create Progress Bar - with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar: - pbar.desc = filename - - try: - result = translateEushully(data, pbar, filename, []) - totalTokens[0] += result[0] - totalTokens[1] += result[1] - except Exception as e: - traceback.print_exc() - return [data, totalTokens, e] - return [data, totalTokens, None] - - -def translateEushully(data, pbar, filename, translatedList): - stringList = [] - currentGroup = [] - tokens = [0, 0] - speaker = "" - voice = False - global LOCK, ESTIMATE, PBAR - i = 0 - - while i < len(data): - voice = False - # Speaker - if "mov (global-int 46e2)" in data[i]: - # Get Speaker - speaker = re.search(r"mov \(global-int 46e2\)\s(.+)", data[i]).group(1) - response = getSpeaker(speaker) - speaker = response[0] - tokens[0] += response[1][0] - tokens[1] += response[1][1] - i += 1 - - # Show Text - if any(x in data[i] for x in ["show-text"]): - # Lines - regex = r'(.*?)"(.*)"' - match = re.search(regex, data[i]) - # Grab Strings - if match != None and match.group(2) != "": - originalString = match.group(2) - jaString = match.group(2) - currentGroup = [jaString] - while "end-text-line" in data[i + 1] and any(x in data[i + 2] for x in ["show-text"]): - match = re.search(regex, data[i + 2]) - if match != None: - currentGroup.append(match.group(2)) - if translatedList == []: - del data[i] - del data[i] - jaString = " ".join(currentGroup) - - # Pass 1 - if translatedList == []: - # Add String - if speaker: - stringList.append(f"[{speaker}]: {jaString.strip()}") - else: - stringList.append(jaString.strip()) - - # Pass 2 - else: - # Get Text - if translatedList: - # Grab and Pop - translatedText = translatedList[0] - translatedList.pop(0) - - # Set to None if empty list - if len(translatedList) <= 0: - translatedList = None - - # Replace Quotes - translatedText = translatedText.replace('"', "'") - - # Remove speaker - if speaker != "": - translatedText = re.sub(r"^\[?(.+?)\]?\s?[|:]\s?", "", translatedText) - - # Textwrap - translatedText = dazedwrap.wrapText(translatedText, width=WIDTH) - translatedTextList = translatedText.split("\n") - - # Set Data - if len(translatedTextList) > 1: - for j in range(len(translatedTextList)): - if any(x in data[i] for x in ["show-text", "set-string", "concat"]): - del data[i] - data.insert(i, f'{match.group(1)}"{translatedTextList[j]}"\n') - i += 1 - if "end-text-line" not in data[i]: - data.insert(i, "end-text-line 0\n") - i += 1 - else: - data[i] = f'{match.group(1)}"{translatedTextList[0]}"\n' - speaker = "" - i += 1 - - # Nothing relevant. Skip Line. - else: - i += 1 - - # Set String - elif "set-string" in data[i]: - # Lines - regex = r'(.*?)"(.*)"' - match = re.search(regex, data[i]) - # Grab Strings - if match != None and match.group(2) != "": - originalString = match.group(2) - jaString = match.group(2) - currentGroup = [jaString] - - # Remove Textwrap - jaString = jaString.replace("\\n", " ") - - # Pass 1 - if translatedList == []: - # Add String - stringList.append(jaString.strip()) - - # Pass 2 - else: - # Get Text - if translatedList: - # Grab and Pop - translatedText = translatedList[0] - translatedList.pop(0) - - # Set to None if empty list - if len(translatedList) <= 0: - translatedList = None - - # Replace Quotes - translatedText = translatedText.replace('"', "'") - - # Textwrap - translatedText = dazedwrap.wrapText(translatedText, width=LISTWIDTH) - translatedText = translatedText.replace("\n", "\\n") - - # Set Data - data[i] = data[i].replace(originalString, translatedText) - speaker = "" - i += 1 - - # Nothing relevant. Skip Line. - else: - i += 1 - else: - i += 1 - - # EOF - if len(stringList) > 0: - # Set Progress - pbar.total = len(stringList) - pbar.refresh() - - # Translate - PBAR = pbar - response = translateGPT(stringList, "", True) - tokens[0] += response[1][0] - tokens[1] += response[1][1] - translatedList = response[0] - - # Set Strings - if len(stringList) == len(translatedList): - translateEushully(data, pbar, filename, translatedList) - - # Mismatch - else: - with LOCK: - if filename not in MISMATCH: - MISMATCH.append(filename) - return tokens - - -# Save some money and enter the character before translation -def getSpeaker(speaker): - match speaker: - case "1": - return ["Klaus", [0, 0]] - case "2": - return ["Helmina", [0, 0]] - case "3": - return ["Juliana", [0, 0]] - case "4": - return ["Reginia", [0, 0]] - case "5": - return ["Luciel", [0, 0]] - case "6": - return ["Mavislaine", [0, 0]] - case "7": - return ["Cerouge", [0, 0]] - case "8": - return ["Maize", [0, 0]] - case "9": - return ["Elvire", [0, 0]] - case "a": - return ["Beatrice", [0, 0]] - case "295": - return ["Orc", [0, 0]] - case "232": - return ["Archangel", [0, 0]] - case "238": - return ["False Juliana", [0, 0]] - case "239": - return ["False Regina", [0, 0]] - case "23a": - return ["False Luciel", [0, 0]] - case "23d": - return ["False Mavislaine", [0, 0]] - case "cb": - return ["Olga Niza Kite", [0, 0]] - case "c9": - return ["Demon Beast Lupus", [0, 0]] - case "ca": - return ["Evelinael", [0, 0]] - case "10": - return ["Eukleia", [0, 0]] - case "15": - return ["Lily", [0, 0]] - case "16": - return ["Kupuko", [0, 0]] - case "b": - return ["Ramiel", [0, 0]] - case "c": - return ["Henriette", [0, 0]] - case "d": - return ["Camilla", [0, 0]] - case "cc": - return ["Gogonaua", [0, 0]] - case "65": - return ["Demon Lord Reyvalois", [0, 0]] - case "d0": - return ["Demon Ranwald", [0, 0]] - case "205": - return ["Vanqueor", [0, 0]] - case "66": - return ["Angel Martina", [0, 0]] - case "21f": - return ["Hiten Demon", [0, 0]] - case "d2": - return ["Lena Eli", [0, 0]] - case _: - return ["Unknown", [0, 0]] - - -def subVars(jaString): - jaString = jaString.replace("\u3000", " ") - - # Nested - count = 0 - nestedList = re.findall(r"[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]", jaString) - nestedList = set(nestedList) - if len(nestedList) != 0: - for icon in nestedList: - jaString = jaString.replace(icon, "[Nested_" + str(count) + "]") - count += 1 - - # Icons - count = 0 - iconList = re.findall(r"[\\]+[iIkKwWaA]+\[[0-9]+\]", jaString) - iconList = set(iconList) - if len(iconList) != 0: - for icon in iconList: - jaString = jaString.replace(icon, "[Ascii_" + str(count) + "]") - count += 1 - - # Colors - count = 0 - colorList = re.findall(r"[\\]+[cC]\[[0-9]+\]", jaString) - colorList = set(colorList) - if len(colorList) != 0: - for color in colorList: - jaString = jaString.replace(color, "[Color_" + str(count) + "]") - count += 1 - - # Names - count = 0 - nameList = re.findall(r"[\\]+[nN]\[.+?\]+", jaString) - nameList = set(nameList) - if len(nameList) != 0: - for name in nameList: - jaString = jaString.replace(name, "[Noun_" + str(count) + "]") - count += 1 - - # Variables - count = 0 - varList = re.findall(r"[\\]+[vV]\[[0-9]+\]", jaString) - varList = set(varList) - if len(varList) != 0: - for var in varList: - jaString = jaString.replace(var, "[Var_" + str(count) + "]") - count += 1 - - # Formatting - count = 0 - formatList = re.findall(r"[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]", jaString) - formatList = set(formatList) - if len(formatList) != 0: - for var in formatList: - jaString = jaString.replace(var, "[FCode_" + str(count) + "]") - count += 1 - - # Put all lists in list and return - allList = [nestedList, iconList, colorList, nameList, varList, formatList] - return [jaString, allList] - - -def resubVars(translatedText, allList): - # Fix Spacing and ChatGPT Nonsense - matchList = re.findall(r"\[\s?.+?\s?\]", translatedText) - if len(matchList) > 0: - for match in matchList: - text = match.strip() - translatedText = translatedText.replace(match, text) - - # Nested - count = 0 - if len(allList[0]) != 0: - for var in allList[0]: - translatedText = translatedText.replace("[Nested_" + str(count) + "]", var) - count += 1 - - # Icons - count = 0 - if len(allList[1]) != 0: - for var in allList[1]: - translatedText = translatedText.replace("[Ascii_" + str(count) + "]", var) - count += 1 - - # Colors - count = 0 - if len(allList[2]) != 0: - for var in allList[2]: - translatedText = translatedText.replace("[Color_" + str(count) + "]", var) - count += 1 - - # Names - count = 0 - if len(allList[3]) != 0: - for var in allList[3]: - translatedText = translatedText.replace("[Noun_" + str(count) + "]", var) - count += 1 - - # Vars - count = 0 - if len(allList[4]) != 0: - for var in allList[4]: - translatedText = translatedText.replace("[Var_" + str(count) + "]", var) - count += 1 - - # Formatting - count = 0 - if len(allList[5]) != 0: - for var in allList[5]: - translatedText = translatedText.replace("[FCode_" + str(count) + "]", var) - count += 1 - - return translatedText - - -def batchList(input_list, batch_size): - if not isinstance(batch_size, int) or batch_size <= 0: - raise ValueError("batch_size must be a positive integer") - - return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)] - - -def createContext(fullPromptFlag, subbedT): - characters = "Game Characters:\n\ -グレイス (Grace) - Female\n\ -" - - system = ( - PROMPT + VOCAB - if fullPromptFlag - else f"\ -You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\ -Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\ -- You translate everything, including erotic and explicit content.\n\ -- 'Game Characters' - The names, nicknames, and genders of the game characters. Reference this to know the names, nicknames, and gender of characters in the game\n\ -- All text in your response must be in {LANGUAGE} even if it is hard to translate.\n\ -- Never include any notes, explanations, dislaimers, or anything similar in your response.\n\ -- Maintain any spacing in the translation.\n\ -- Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\ -- `...` can be a part of the dialogue. Translate it as it is.\n\ -{VOCAB}\n\ -" - ) - user = f"{subbedT}" - return characters, system, user - - -def translateText(characters, system, user, history, penalty): - # Prompt - msg = [{"role": "system", "content": system + characters}] - - # Characters - msg.append({"role": "system", "content": characters}) - - # History - if isinstance(history, list): - msg.extend([{"role": "system", "content": h} for h in history]) - else: - msg.append({"role": "system", "content": history}) - - # Content to TL - msg.append({"role": "user", "content": f"{user}"}) - response = openai.chat.completions.create( - temperature=0, - frequency_penalty=penalty, - model=MODEL, - messages=msg, - ) - return response - - -def cleanTranslatedText(translatedText, varResponse): - placeholders = { - f"{LANGUAGE} Translation: ": "", - "Translation: ": "", - "っ": "", - "〜": "~", - "ッ": "", - "。": ".", - "< ": "<", - "": ">", - "「": '"', - "」": '"', - "Placeholder Text": "", - "- chan": "-chan", - "- kun": "-kun", - "- san": "-san", - # Add more replacements as needed - } - for target, replacement in placeholders.items(): - translatedText = translatedText.replace(target, replacement) - - # Elongate Long Dashes (Since GPT Ignores them...) - translatedText = elongateCharacters(translatedText) - translatedText = resubVars(translatedText, varResponse[1]) - return translatedText - - -def elongateCharacters(text): - # Define a pattern to match one character followed by one or more `ー` characters - # Using a positive lookbehind assertion to capture the preceding character - pattern = r"(?<=(.))ー+" - - # Define a replacement function that elongates the captured character - def repl(match): - char = match.group(1) # The character before the ー sequence - count = len(match.group(0)) - 1 # Number of ー characters - return char * count # Replace ー sequence with the character repeated - - # Use re.sub() to replace the pattern in the text - return re.sub(pattern, repl, text) - - -def extractTranslation(translatedTextList, is_list): - pattern = r"`?<[Ll]ine\d+>([\\]*.*?[\\]*?)<\/?[Ll]ine\d+>`?" - # If it's a batch (i.e., list), extract with tags; otherwise, return the single item. - if is_list: - matchList = re.findall(pattern, translatedTextList) - return matchList - else: - matchList = re.findall(pattern, translatedTextList) - return matchList[0][0] if matchList else translatedTextList - - -def countTokens(characters, system, user, history): - inputTotalTokens = 0 - outputTotalTokens = 0 - enc = tiktoken.encoding_for_model("gpt-4") - - # Input - if isinstance(history, list): - for line in history: - inputTotalTokens += len(enc.encode(line)) - else: - inputTotalTokens += len(enc.encode(history)) - inputTotalTokens += len(enc.encode(system)) - inputTotalTokens += len(enc.encode(characters)) - inputTotalTokens += len(enc.encode(user)) - - # Output - outputTotalTokens += round(len(enc.encode(user)) * 3) - - return [inputTotalTokens, outputTotalTokens] - - -@retry(exceptions=Exception, tries=5, delay=5) -def translateGPT(text, history, fullPromptFlag): - global PBAR - - mismatch = False - totalTokens = [0, 0] - if isinstance(text, list): - tList = batchList(text, BATCHSIZE) - else: - tList = [text] - - for index, tItem in enumerate(tList): - # Before sending to translation, if we have a list of items, add the formatting - if isinstance(tItem, list): - payload = "\n".join([f"`{item}`" for i, item in enumerate(tItem)]) - payload = re.sub(r"(<)(\/Line\d+>)", r"\1>Placeholder Text<\3", payload) - varResponse = subVars(payload) - subbedT = varResponse[0] - else: - varResponse = subVars(tItem) - subbedT = varResponse[0] - - # Things to Check before starting translation - if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", subbedT): - if PBAR is not None: - PBAR.update(len(tItem)) - continue - - # Create Message - characters, system, user = createContext(fullPromptFlag, subbedT) - - # Calculate Estimate - if ESTIMATE: - estimate = countTokens(characters, system, user, history) - totalTokens[0] += estimate[0] - totalTokens[1] += estimate[1] - continue - - # Translating - response = translateText(characters, system, user, history, 0.02) - translatedText = response.choices[0].message.content - totalTokens[0] += response.usage.prompt_tokens - totalTokens[1] += response.usage.completion_tokens - - # Formatting - translatedText = cleanTranslatedText(translatedText, varResponse) - if isinstance(tItem, list): - extractedTranslations = extractTranslation(translatedText, True) - tList[index] = extractedTranslations - if len(tItem) != len(extractedTranslations): - # Mismatch. Try Again - response = translateText(characters, system, user, history, 0.2) - translatedText = response.choices[0].message.content - totalTokens[0] += response.usage.prompt_tokens - totalTokens[1] += response.usage.completion_tokens - - # Formatting - translatedText = cleanTranslatedText(translatedText, varResponse) - if isinstance(tItem, list): - extractedTranslations = extractTranslation(translatedText, True) - tList[index] = extractedTranslations - if len(tItem) != len(extractedTranslations): - mismatch = True # Just here for breakpoint - - # Create History - with LOCK: - if PBAR is not None: - PBAR.update(len(tItem)) - if not mismatch: - history = extractedTranslations[-10:] # Update history if we have a list - else: - history = text[-10:] - else: - # Ensure we're passing a single string to extractTranslation - extractedTranslations = extractTranslation(translatedText, False) - tList[index] = extractedTranslations - - # Combine if multilist - if isinstance(tList[0], list): - tList = [t for sublist in tList for t in sublist] - - # Return - if format == "json": - return [tList, totalTokens] - else: - return [tList[0], totalTokens] diff --git a/modules/images.py b/modules/images.py index ba2a104..acdefa2 100644 --- a/modules/images.py +++ b/modules/images.py @@ -389,7 +389,6 @@ def translateImages(imageList): return [translatedList, totalTokens, None] - # Save some money and enter the character before translation def getSpeaker(speaker): match speaker: @@ -407,7 +406,7 @@ def getSpeaker(speaker): response = translateGPT( f"{speaker}", "Reply with the " + LANGUAGE + " translation of the NPC name.", - True, + False, ) response[0] = response[0].title() response[0] = response[0].replace("'S", "'s") @@ -429,40 +428,6 @@ def getSpeaker(speaker): return [speaker, [0, 0]] -def subVars(jaString): - jaString = jaString.replace("\u3000", " ") - - # Formatting - count = 0 - codeList = re.findall(r"[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]", jaString) - codeList = set(codeList) - if len(codeList) != 0: - for var in codeList: - jaString = jaString.replace(var, "[FCode_" + str(count) + "]") - count += 1 - - # Put all lists in list and return - return [jaString, codeList] - - -def resubVars(translatedText, codeList): - # Fix Spacing and ChatGPT Nonsense - matchList = re.findall(r"\[\s?.+?\s?\]", translatedText) - if len(matchList) > 0: - for match in matchList: - text = match.strip() - translatedText = translatedText.replace(match, text) - - # Formatting - count = 0 - if len(codeList) != 0: - for var in codeList: - translatedText = translatedText.replace("[FCode_" + str(count) + "]", var) - count += 1 - - return translatedText - - def batchList(input_list, batch_size): if not isinstance(batch_size, int) or batch_size <= 0: raise ValueError("batch_size must be a positive integer") @@ -470,56 +435,68 @@ def batchList(input_list, batch_size): return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)] -def createContext(fullPromptFlag, subbedT, format): - characters = "Game Characters:\n\ -ロラン (Roland) - Male\n\ -リュカ (Ryuka) - Male\n\ -レックス (Rex) - Male\n\ -タバサ (Tabasa) - Female\n\ -アルス (Ars) - Male\n\ -アマカラ (Amakara) - Male\n\ -エリー (Eri) - Female\n\ -リオ (Rio) - Female\n\ -サマル (Samal) - Male\n\ -ムーン (Moon) - Female\n\ -アリーナ (Arina) - Female\n\ -クリフト (Cliff) - Male\n\ -マーニャ (Manya) - Female\n\ -ミネア (Minea) - Female\n\ -デボラ (Debora) - Female\n\ -ビアンカ (Bianca) - Female\n\ -フローラ (Flora) - Female\n\ -バーバラ (Barbara) - Female\n\ -ミレーユ (Mireyu) - Female\n\ -アイラ (Aira) - Female\n\ -フォズ (Foz) - Female\n\ -マリベル (Maribel) - Female\n\ -ククール (Kukool) - Male\n\ -ゲルダ (Gerda) - Female\n\ -ゼシカ (Jessica) - Female\n\ -ヤンガス (Yangus) - Male\n\ -ラヴィエル (Raviel) - Female\n\ -セティア (Setia) - Female\n\ -ダイ (Dai) - Male\n\ -ヒュンケル (Hyunckel) - Male\n\ -ポップ (Pop) - Male\n\ -マァム (Maam) - Female\n\ -レオナ (Leona) - Female\n\ -アステア (Astea) - Female\n\ -イヨ (Iyo) - Female\n\ -ジャガン (Jagan) - Male\n\ -ヤオ (Yao) - Female\n\ -デイジィ (Daisy) - Female\n\ -バイシュン (Baishun) - Male\n\ -ブライ (Buraimu) - Male\n\ -ハッサン (Hassan) - Male\n\ -アロマ (Aroma) - Female\n\ -" +def parseVocabWithCategories(vocabText): + """Parse vocabulary text and extract terms with their categories.""" + pairs = [] + seen = set() + currentCategory = None + + for line in vocabText.splitlines(): + line = line.strip() + if not line or line.startswith('```'): + continue + + # Check if this is a category header + if line.startswith('#'): + currentCategory = line + continue + + # Parse vocabulary term + m = re.match(r'^(.+?)(?:\s?[\(–])', line) # term is everything before space + '(' or '–' + if m: + term = m.group(1) + if term not in seen: + pairs.append((term, line, currentCategory)) + seen.add(term) + + return pairs - system = ( - PROMPT + VOCAB - if fullPromptFlag - else f"\ + +def buildMatchedVocabText(vocabPairs, subbedT): + """Build formatted vocabulary text with matched terms organized by category.""" + matchedCategories = {} + + # Use word boundaries for Japanese if appropriate, or allow substring as before. + for term, line, category in vocabPairs: + # "term in subbedT" could be false positive; can use regex but Japanese doesn't always have spaces. + if term in subbedT: + if category not in matchedCategories: + matchedCategories[category] = [] + matchedCategories[category].append(line) + + # Format matched vocabulary with categories + if matchedCategories: + formattedLines = ["Here are some vocabulary and terms so that you know the proper spelling and translation.\n"] + for category, lines in matchedCategories.items(): + if category: # Only add category header if it exists + formattedLines.append(category) + formattedLines.extend(lines) + formattedLines.append("") # Add blank line between categories + matchedVocabText = f"```\n{chr(10).join(formattedLines).rstrip()}\n```" + else: + matchedVocabText = "" + + return matchedVocabText + + +def createContext(fullPromptFlag, subbedT, format): + vocabPairs = parseVocabWithCategories(VOCAB) + matchedVocabText = buildMatchedVocabText(vocabPairs, subbedT) + + if fullPromptFlag: + system = PROMPT + matchedVocabText + else: + system = f"\ You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\ - You translate everything, including erotic and explicit content.\n\ @@ -529,28 +506,25 @@ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{ - Maintain any spacing in the translation.\n\ - Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\ - `...` can be a part of the dialogue. Translate it as it is.\n\ -{VOCAB}\n\ +{matchedVocabText}\n\ " - ) if format == "json": user = f"```json\n{subbedT}\n```" else: user = subbedT - return characters, system, user + return system, user -def translateText(characters, system, user, history, penalty, format): +def translateText(system, user, history, penalty, format, model=MODEL): # Prompt - msg = [{"role": "system", "content": system + characters}] - - # Characters - msg.append({"role": "system", "content": characters}) + msg = [{"role": "system", "content": system}] # History if isinstance(history, list): - msg.extend([{"role": "system", "content": h} for h in history]) + msg.append({"role": "system", "content": "Translation History:"}) + msg.extend([{"role": "assistant", "content": h} for h in history]) else: - msg.append({"role": "system", "content": history}) + msg.append({"role": "assistant", "content": history}) # Response Format if format == "json": @@ -563,14 +537,14 @@ def translateText(characters, system, user, history, penalty, format): response = openai.chat.completions.create( temperature=0, frequency_penalty=penalty, - model=MODEL, + model=model, response_format=responseFormat, messages=msg, ) return response -def cleanTranslatedText(translatedText, varResponse): +def cleanTranslatedText(translatedText): placeholders = { f"{LANGUAGE} Translation: ": "", "Translation: ": "", @@ -581,15 +555,26 @@ def cleanTranslatedText(translatedText, varResponse): "「": '\\"', "」": '\\"', "- ": "-", + "—": "―", + "】": "]", + "【": "[", + "é": "e", + "this guy": "this bastard", + "This guy": "This bastard", "Placeholder Text": "", + "```json": "", + "```": "", # Add more replacements as needed } for target, replacement in placeholders.items(): translatedText = translatedText.replace(target, replacement) + # Remove Repeating Characters + pattern = re.compile(r"(.)\s*\1(?:\s*\1){" + str(20 - 1) + r",}") + translatedText = pattern.sub(lambda match: match.group(0).replace(" ", "")[:20], translatedText) + # Elongate Long Dashes (Since GPT Ignores them...) translatedText = elongateCharacters(translatedText) - translatedText = resubVars(translatedText, varResponse[1]) return translatedText @@ -610,6 +595,8 @@ def elongateCharacters(text): def extractTranslation(translatedTextList, is_list): try: + translatedTextList = re.sub(r'\\"+\"([^,\n}])', r'\\"\1', translatedTextList) + translatedTextList = re.sub(r"(? instead -FIXTEXTWRAP = True # Overwrites textwrap -IGNORETLTEXT = False # Ignores all translated text. -MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong) - -# tqdm Globals -BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}" -POSITION = 0 -LEAVE = False - -# Regex - Need to change this if you want to translate from/to other languages. Default is Japanese Regex -LANGREGEX = r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+" - -# Pricing - Depends on the model https://openai.com/pricing -# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request -# If you are getting a MISMATCH LENGTH error, lower the batch size. -if "gpt-3.5" in MODEL: - INPUTAPICOST = 3.00 - OUTPUTAPICOST = 5.00 - BATCHSIZE = 10 - FREQUENCY_PENALTY = 0.2 -elif "gpt-4" in MODEL: - INPUTAPICOST = 2.0 - OUTPUTAPICOST = 8.00 - BATCHSIZE = 30 - FREQUENCY_PENALTY = 0.05 -elif "deepseek" in MODEL: - INPUTAPICOST = 0.27 - OUTPUTAPICOST = 1.10 - BATCHSIZE = 30 - FREQUENCY_PENALTY = 0.05 -else: - INPUTAPICOST = float(os.getenv("input_cost")) - OUTPUTAPICOST = float(os.getenv("output_cost")) - BATCHSIZE = int(os.getenv("batchsize")) - FREQUENCY_PENALTY = float(os.getenv("frequency_penalty")) - - -def handleIris(filename, estimate): - global ESTIMATE - ESTIMATE = estimate - - if ESTIMATE: - start = time.time() - translatedData = openFiles(filename) - - # Print Result - end = time.time() - tqdm.write(getResultString(translatedData, end - start, filename)) - with LOCK: - TOKENS[0] += translatedData[1][0] - TOKENS[1] += translatedData[1][1] - - # Print Total - totalString = getResultString(["", TOKENS, None], end - start, "TOTAL") - - # Print any errors on maps - if len(MISMATCH) > 0: - return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET - else: - return totalString - - else: - try: - with open("translated/" + filename, "w", encoding="cp932", errors="ignore") as outFile: - start = time.time() - translatedData = openFiles(filename) - - # Print Result - end = time.time() - outFile.writelines(translatedData[0]) - tqdm.write(getResultString(translatedData, end - start, filename)) - with LOCK: - TOKENS[0] += translatedData[1][0] - TOKENS[1] += translatedData[1][1] - except Exception: - traceback.print_exc() - return "Fail" - - return getResultString(["", TOKENS, None], end - start, "TOTAL") - - -def getResultString(translatedData, translationTime, filename): - # File Print String - totalTokenstring = ( - Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]" - "[Output: " - + str(translatedData[1][1]) - + "]" "[Cost: ${:,.4f}".format(((translatedData[1][0] / 1000000) * INPUTAPICOST) + ((translatedData[1][1] / 1000000) * OUTPUTAPICOST)) - + "]" - ) - timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]" - - if translatedData[2] == None: - # Success - return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET - - else: - # Fail - try: - raise translatedData[2] - except Exception as e: - traceback.print_exc() - errorString = str(e) + Fore.RED - return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET - - -def openFiles(filename): - with open("files/" + filename, "r", encoding="shift_jis") as readFile: - translatedData = parseIris(readFile, filename) - - # Delete lines marked for deletion - finalData = [] - for line in translatedData[0]: - if line != "\\d\n": - finalData.append(line) - translatedData[0] = finalData - - return translatedData - - -def parseIris(readFile, filename): - totalTokens = [0, 0] - - # Read File into data - data = readFile.readlines() - - # Create Progress Bar - with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar: - pbar.desc = filename - - try: - result = translateIris(data, pbar, filename, []) - totalTokens[0] += result[0] - totalTokens[1] += result[1] - except Exception as e: - traceback.print_exc() - return [data, totalTokens, e] - return [data, totalTokens, None] - - -def translateIris(data, pbar, filename, translatedList): - stringList = [] - currentGroup = [] - tokens = [0, 0] - speaker = "" - voice = False - global LOCK, ESTIMATE - i = 0 - - while i < len(data): - voice = False - speaker = "" - if "#MSGVOICE" in data[i]: - i += 1 - voice = True - voiceVar = data[i] - if "#MSG," in data[i] or "#MSG\n" in data[i] or voice == True: - i += 1 - # Speaker - if re.search(r'^　?([^#\/."、。*!！（）\(\)\[\]　\n]+)\n', data[i]) and len(data[i]) < 30: - match = re.search(r"(.*)", data[i]) - if match != None: - speaker = match.group(1) - if speaker[0] == "\u3000": - speaker = speaker[1:] - response = getSpeaker(speaker, pbar, filename) - speaker = response[0] - tokens[0] += response[1][0] - tokens[1] += response[1][1] - if translatedList != []: - speaker = speaker.replace(" ", "\u3000") - data[i] = f"\u3000{speaker}\n" - else: - speaker = "" - i += 1 - - # Lines - match = re.search(r"(.*)", data[i]) - if match != None and match.group(1) != "": - # Pass 1 - if translatedList == []: - # Grab Consecutive Strings - jaString = data[i] - if data[i] != "\n": - if data[i][0] == "\u3000": - jaString = data[i][1:] - currentGroup.append(jaString) - i += 1 - while data[i] != "\n": - jaString = data[i] - if data[i] != "\n": - jaString = data[i][1:] - currentGroup.append(jaString) - i += 1 - - # Join up 401 groups for better translation. - if len(currentGroup) > 0: - jaString = "".join(currentGroup) - currentGroup = [] - - # Remove any textwrap - jaString = jaString.replace("\n", " ") - - # Temporarily convert spaces (For Textwrap Later) - jaString = jaString.replace("\u3000", " ") - - # Add Speaker (If there is one) - if speaker != "": - jaString = f"{speaker}: {jaString}" - - # Add String - stringList.append(jaString.strip()) - - # Pass 2 - else: - # Insert Strings - while data[i] != "\n": - data.pop(i) - - # Get Text - if translatedList: - translatedText = translatedList[0] - translatedList.pop(0) - if len(translatedList) <= 0: - translatedList = None - - # Remove added speaker - translatedText = re.sub(r"^.+?:\s", "", translatedText) - - # Textwrap - translatedText = dazedwrap.wrapText(translatedText, width=WIDTH) - translatedText = translatedText.replace("\n", "\n\u3000") - - # Replace Whitespace and Commas - translatedText = translatedText.replace(", ", "、") - translatedText = translatedText.replace(",\u3000", "、") - translatedText = translatedText.replace(",", "、") - translatedText = translatedText.replace(" ", "\u3000") - - # Set Data - # Game crashes on more than 3 lines. Will need to create a new MSG for long translations - if translatedText.count("\n") > 2: - # Split List - translatedTextList = splitNewlines(translatedText) - - # MSG Voice - count = 0 - for text in translatedTextList: - if count != 0: - if voice == True: - # MSG for each item in the list - data.insert(i, "#MSGVOICE,\n") - i += 1 - data.insert(i, f"{voiceVar}") - i += 1 - else: - data.insert(i, "#MSG,\n") - i += 1 - if speaker: - data[i] = f"\u3000{speaker}\n" - i += 1 - if text[0] == "\u3000": - data.insert(i, f"{text}\n") - else: - data.insert(i, f"\u3000{text}\n") - i += 1 - count += 1 - if data[i] != "\n": - data.insert(i, "\n") - data[i] = f"\n{data[i]}" - else: - data.insert(i, f"\u3000{translatedText}\n") - i += 1 - if data[i] != "\n": - data[i] = f"\n{data[i]}" - - elif "#SELECT" in data[i] and translatedList == []: - Iris = r"(.+?) +\d$" - i += 1 - match = re.search(Iris, data[i]) - if match: - choiceList = [] - choiceList.append(match.group(1)) - i += 1 - match = re.search(Iris, data[i]) - while match: - choiceList.append(match.group(1)) - i += 1 - match = re.search(Iris, data[i]) - - # Translate - question = stringList[len(stringList) - 1] - response = translateGPT( - choiceList, - f"Previous text for context: {question}\n\nThis will be a dialogue option", - True, - pbar, - filename, - ) - tokens[0] += response[1][0] - tokens[1] += response[1][1] - choiceListTL = response[0] - - # Set Data - i = i - len(choiceListTL) - for j in range(len(choiceListTL)): - # Replace Whitespace and Commas - choiceListTL[j] = choiceListTL[j].replace(", ", "、") - choiceListTL[j] = choiceListTL[j].replace(",\u3000", "、") - choiceListTL[j] = choiceListTL[j].replace(",", "、") - choiceListTL[j] = choiceListTL[j].replace(" ", "\u3000") - data[i] = data[i].replace(choiceList[j], choiceListTL[j]) - i += 1 - - # Nothing relevant. Skip Line. - else: - i += 1 - else: - i += 1 - - # EOF - if len(stringList) > 0: - # Set Progress - pbar.total = len(stringList) - pbar.refresh() - - # Translate - response = translateGPT(stringList, "", True, pbar, filename) - tokens[0] += response[1][0] - tokens[1] += response[1][1] - translatedList = response[0] - - # Set Strings - if len(stringList) == len(translatedList): - translateIris(data, pbar, filename, translatedList) - - # Mismatch - else: - with LOCK: - if filename not in MISMATCH: - MISMATCH.append(filename) - return tokens - - -def splitNewlines(text): - parts = [] - newline_count = 0 # Counts the number of newline characters encountered - start_index = 0 # Start index of the current string part - - for i, char in enumerate(text): - if char == "\n": - newline_count += 1 - if newline_count == 3: - # Append the string part from start_index to current index (inclusive) - parts.append(text[start_index : i + 1]) - # Reset newline count and update start_index for the next string part - newline_count = 0 - start_index = i + 1 - - # Edge case: if the text does not end with a newline, we still need to append the last part - if start_index < len(text): - parts.append(text[start_index:]) - - return parts - - -# Save some money and enter the character before translation -def getSpeaker(speaker, pbar, filename): - match speaker: - case "ファイン": - return ["Fine", [0, 0]] - case "": - return ["", [0, 0]] - case _: - # Store Speaker - if speaker not in str(NAMESLIST): - response = translateGPT( - speaker, - "Reply with only the " + LANGUAGE + " translation of the NPC name.", - False, - pbar, - filename, - ) - response[0] = response[0].replace("'S", "'s") - speakerList = [speaker, response[0]] - NAMESLIST.append(speakerList) - return response - - # Find Speaker - else: - for i in range(len(NAMESLIST)): - if speaker == NAMESLIST[i][0]: - return [NAMESLIST[i][1], [0, 0]] - - return [speaker, [0, 0]] - - -def subVars(jaString): - jaString = jaString.replace("\u3000", " ") - - # Nested - count = 0 - nestedList = re.findall(r"[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]", jaString) - nestedList = set(nestedList) - if len(nestedList) != 0: - for icon in nestedList: - jaString = jaString.replace(icon, "[Nested_" + str(count) + "]") - count += 1 - - # Icons - count = 0 - iconList = re.findall(r"[\\]+[iIkKwWaA]+\[[0-9]+\]", jaString) - iconList = set(iconList) - if len(iconList) != 0: - for icon in iconList: - jaString = jaString.replace(icon, "[Ascii_" + str(count) + "]") - count += 1 - - # Colors - count = 0 - colorList = re.findall(r"[\\]+[cC]\[[0-9]+\]", jaString) - colorList = set(colorList) - if len(colorList) != 0: - for color in colorList: - jaString = jaString.replace(color, "[Color_" + str(count) + "]") - count += 1 - - # Names - count = 0 - nameList = re.findall(r"[\\]+[nN]\[.+?\]+", jaString) - nameList = set(nameList) - if len(nameList) != 0: - for name in nameList: - jaString = jaString.replace(name, "[Noun_" + str(count) + "]") - count += 1 - - # Variables - count = 0 - varList = re.findall(r"[\\]+[vV]\[[0-9]+\]", jaString) - varList = set(varList) - if len(varList) != 0: - for var in varList: - jaString = jaString.replace(var, "[Var_" + str(count) + "]") - count += 1 - - # Formatting - count = 0 - formatList = re.findall(r"[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]", jaString) - formatList = set(formatList) - if len(formatList) != 0: - for var in formatList: - jaString = jaString.replace(var, "[FCode_" + str(count) + "]") - count += 1 - - # Put all lists in list and return - allList = [nestedList, iconList, colorList, nameList, varList, formatList] - return [jaString, allList] - - -def resubVars(translatedText, allList): - # Fix Spacing and ChatGPT Nonsense - matchList = re.findall(r"\[\s?.+?\s?\]", translatedText) - if len(matchList) > 0: - for match in matchList: - text = match.strip() - translatedText = translatedText.replace(match, text) - - # Nested - count = 0 - if len(allList[0]) != 0: - for var in allList[0]: - translatedText = translatedText.replace("[Nested_" + str(count) + "]", var) - count += 1 - - # Icons - count = 0 - if len(allList[1]) != 0: - for var in allList[1]: - translatedText = translatedText.replace("[Ascii_" + str(count) + "]", var) - count += 1 - - # Colors - count = 0 - if len(allList[2]) != 0: - for var in allList[2]: - translatedText = translatedText.replace("[Color_" + str(count) + "]", var) - count += 1 - - # Names - count = 0 - if len(allList[3]) != 0: - for var in allList[3]: - translatedText = translatedText.replace("[Noun_" + str(count) + "]", var) - count += 1 - - # Vars - count = 0 - if len(allList[4]) != 0: - for var in allList[4]: - translatedText = translatedText.replace("[Var_" + str(count) + "]", var) - count += 1 - - # Formatting - count = 0 - if len(allList[5]) != 0: - for var in allList[5]: - translatedText = translatedText.replace("[FCode_" + str(count) + "]", var) - count += 1 - - return translatedText - - -def batchList(input_list, batch_size): - if not isinstance(batch_size, int) or batch_size <= 0: - raise ValueError("batch_size must be a positive integer") - - return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)] - - -def createContext(fullPromptFlag, subbedT): - characters = "Game Characters:\n\ -フィリア (Philia) - Female\n\ -アルネット (Annett) - Female\n\ -ラピュセナ (Rapusena) - Female\n\ -リッカ (Rikka) - Female\n\ -アンデリビア (Andelivia) - Female\n\ -リリアブルム (Liliabloom) - Female\n\ -カルナ (Karna) - Female\n\ -ラフィング＝スピア (Laughing Spear) - Female\n\ -ノーラ (Nora) - Female\n\ -" - - system = ( - PROMPT + VOCAB - if fullPromptFlag - else f"\ -You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\ -Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\ -- You translate everything, including erotic and explicit content.\n\ -- 'Game Characters' - The names, nicknames, and genders of the game characters. Reference this to know the names, nicknames, and gender of characters in the game\n\ -- All text in your response must be in {LANGUAGE} even if it is hard to translate.\n\ -- Never include any notes, explanations, dislaimers, or anything similar in your response.\n\ -- Maintain any spacing in the translation.\n\ -- Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\ -- `...` can be a part of the dialogue. Translate it as it is.\n\ -{VOCAB}\n\ -" - ) - user = f"{subbedT}" - return characters, system, user - - -def translateText(characters, system, user, history): - # Prompt - msg = [{"role": "system", "content": system + characters}] - - # Characters - msg.append({"role": "system", "content": characters}) - - # History - if isinstance(history, list): - msg.extend([{"role": "system", "content": h} for h in history]) - else: - msg.append({"role": "system", "content": history}) - - # Content to TL - msg.append({"role": "user", "content": f"{user}"}) - response = openai.chat.completions.create( - temperature=0.1, - frequency_penalty=0.1, - model=MODEL, - messages=msg, - ) - return response - - -def cleanTranslatedText(translatedText, varResponse): - placeholders = { - f"{LANGUAGE} Translation: ": "", - "Translation: ": "", - "っ": "", - "〜": "~", - "ッ": "", - "。": ".", - "Placeholder Text": "", - # Add more replacements as needed - } - for target, replacement in placeholders.items(): - translatedText = translatedText.replace(target, replacement) - - # Elongate Long Dashes (Since GPT Ignores them...) - translatedText = elongateCharacters(translatedText) - translatedText = resubVars(translatedText, varResponse[1]) - return translatedText - - -def elongateCharacters(text): - # Define a pattern to match one character followed by one or more `ー` characters - # Using a positive lookbehind assertion to capture the preceding character - pattern = r"(?<=(.))ー+" - - # Define a replacement function that elongates the captured character - def repl(match): - char = match.group(1) # The character before the ー sequence - count = len(match.group(0)) - 1 # Number of ー characters - return char * count # Replace ー sequence with the character repeated - - # Use re.sub() to replace the pattern in the text - return re.sub(pattern, repl, text) - - -def extractTranslation(translatedTextList, is_list): - pattern = r"`?<[Ll]ine\d+>([\\]*.*?[\\]*?)<\/?[Ll]ine\d+>`?" - # If it's a batch (i.e., list), extract with tags; otherwise, return the single item. - if is_list: - matchList = re.findall(pattern, translatedTextList) - return matchList - else: - matchList = re.findall(pattern, translatedTextList) - return matchList[0][0] if matchList else translatedTextList - - -def countTokens(characters, system, user, history): - inputTotalTokens = 0 - outputTotalTokens = 0 - enc = tiktoken.encoding_for_model("gpt-4") - - # Input - if isinstance(history, list): - for line in history: - inputTotalTokens += len(enc.encode(line)) - else: - inputTotalTokens += len(enc.encode(history)) - inputTotalTokens += len(enc.encode(system)) - inputTotalTokens += len(enc.encode(characters)) - inputTotalTokens += len(enc.encode(user)) - - # Output - outputTotalTokens += round(len(enc.encode(user)) * 2) - - return [inputTotalTokens, outputTotalTokens] - - -@retry(exceptions=Exception, tries=5, delay=5) -def translateGPT(text, history, fullPromptFlag, pbar, filename): - mismatch = False - totalTokens = [0, 0] - if isinstance(text, list): - tList = batchList(text, BATCHSIZE) - else: - tList = [text] - - for index, tItem in enumerate(tList): - # Before sending to translation, if we have a list of items, add the formatting - if isinstance(tItem, list): - payload = "\n".join([f"`{item}`" for i, item in enumerate(tItem)]) - payload = re.sub(r"(<)(\/Line\d+>)", r"\1>Placeholder Text<\3", payload) - varResponse = subVars(payload) - subbedT = varResponse[0] - else: - varResponse = subVars(tItem) - subbedT = varResponse[0] - - # Things to Check before starting translation - if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", subbedT): - continue - - # Create Message - characters, system, user = createContext(fullPromptFlag, subbedT) - - # Calculate Estimate - if ESTIMATE: - estimate = countTokens(characters, system, user, history) - totalTokens[0] += estimate[0] - totalTokens[1] += estimate[1] - continue - - # Translating - response = translateText(characters, system, user, history) - translatedText = response.choices[0].message.content - totalTokens[0] += response.usage.prompt_tokens - totalTokens[1] += response.usage.completion_tokens - - # Formatting - translatedText = cleanTranslatedText(translatedText, varResponse) - if isinstance(tItem, list): - extractedTranslations = extractTranslation(translatedText, True) - if len(tItem) != len(extractedTranslations): - # Mismatch. Try Again - response = translateText(characters, system, user, history) - translatedText = response.choices[0].message.content - totalTokens[0] += response.usage.prompt_tokens - totalTokens[1] += response.usage.completion_tokens - - # Formatting - translatedText = cleanTranslatedText(translatedText, varResponse) - if isinstance(tItem, list): - extractedTranslations = extractTranslation(translatedText, True) - if len(tItem) == len(extractedTranslations): - tList[index] = extractedTranslations - else: - MISMATCH.append(filename) - else: - tList[index] = extractedTranslations - - # Create History - history = tList[index] # Update history if we have a list - pbar.update(len(tList[index])) - - else: - # Ensure we're passing a single string to extractTranslation - extractedTranslations = extractTranslation(translatedText, False) - tList[index] = extractedTranslations - - # Combine if multilist - if isinstance(tList[0], list): - tList = [t for sublist in tList for t in sublist] - - # Return - if format == "json": - return [tList, totalTokens] - else: - return [tList[0], totalTokens] diff --git a/modules/javascript.py b/modules/javascript.py deleted file mode 100644 index 40aef6a..0000000 --- a/modules/javascript.py +++ /dev/null @@ -1,537 +0,0 @@ -# Libraries -import os -import re -import util.dazedwrap as dazedwrap -import threading -import time -import traceback -import tiktoken -import openai -from pathlib import Path -from colorama import Fore -from dotenv import load_dotenv -from retry import retry -from tqdm import tqdm - -# Open AI -load_dotenv() -if os.getenv("api").replace(" ", "") != "": - openai.base_url = os.getenv("api") -openai.organization = os.getenv("org") -openai.api_key = os.getenv("key") - -# Globals -MODEL = os.getenv("model") -TIMEOUT = int(os.getenv("timeout")) -LANGUAGE = os.getenv("language").capitalize() -PROMPT = Path("prompt.txt").read_text(encoding="utf-8") -VOCAB = Path("vocab.txt").read_text(encoding="utf-8") -THREADS = int(os.getenv("threads")) -LOCK = threading.Lock() -WIDTH = int(os.getenv("width")) -LISTWIDTH = int(os.getenv("listWidth")) -NOTEWIDTH = 70 -MAXHISTORY = 10 -ESTIMATE = "" -TOKENS = [0, 0] -NAMESLIST = [] -NAMES = False # Output a list of all the character names found -BRFLAG = False # If the game uses
instead -FIXTEXTWRAP = True # Overwrites textwrap -IGNORETLTEXT = False # Ignores all translated text. -MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong) - -# tqdm Globals -BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}" -POSITION = 0 -LEAVE = False - -# Regex - Need to change this if you want to translate from/to other languages. Default is Japanese Regex -LANGREGEX = r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+" - -# Pricing - Depends on the model https://openai.com/pricing -# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request -# If you are getting a MISMATCH LENGTH error, lower the batch size. -if "gpt-3.5" in MODEL: - INPUTAPICOST = 3.00 - OUTPUTAPICOST = 5.00 - BATCHSIZE = 10 - FREQUENCY_PENALTY = 0.2 -elif "gpt-4" in MODEL: - INPUTAPICOST = 2.0 - OUTPUTAPICOST = 8.00 - BATCHSIZE = 30 - FREQUENCY_PENALTY = 0.05 -elif "deepseek" in MODEL: - INPUTAPICOST = 0.27 - OUTPUTAPICOST = 1.10 - BATCHSIZE = 30 - FREQUENCY_PENALTY = 0.05 -else: - INPUTAPICOST = float(os.getenv("input_cost")) - OUTPUTAPICOST = float(os.getenv("output_cost")) - BATCHSIZE = int(os.getenv("batchsize")) - FREQUENCY_PENALTY = float(os.getenv("frequency_penalty")) - - -def handleJavascript(filename, estimate): - global ESTIMATE - ESTIMATE = estimate - - if ESTIMATE: - start = time.time() - translatedData = openFiles(filename) - - # Print Result - end = time.time() - tqdm.write(getResultString(translatedData, end - start, filename)) - with LOCK: - TOKENS[0] += translatedData[1][0] - TOKENS[1] += translatedData[1][1] - - # Print Total - totalString = getResultString(["", TOKENS, None], end - start, "TOTAL") - - # Print any errors on maps - if len(MISMATCH) > 0: - return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET - else: - return totalString - - else: - try: - with open("translated/" + filename, "w", encoding="utf8", errors="ignore") as outFile: - start = time.time() - translatedData = openFiles(filename) - - # Print Result - end = time.time() - outFile.writelines(translatedData[0]) - tqdm.write(getResultString(translatedData, end - start, filename)) - with LOCK: - TOKENS[0] += translatedData[1][0] - TOKENS[1] += translatedData[1][1] - except Exception: - traceback.print_exc() - return "Fail" - - return getResultString(["", TOKENS, None], end - start, "TOTAL") - - -def getResultString(translatedData, translationTime, filename): - # File Print String - totalTokenstring = ( - Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]" - "[Output: " - + str(translatedData[1][1]) - + "]" "[Cost: ${:,.4f}".format(((translatedData[1][0] / 1000000) * INPUTAPICOST) + ((translatedData[1][1] / 1000000) * OUTPUTAPICOST)) - + "]" - ) - timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]" - - if translatedData[2] == None: - # Success - return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET - - else: - # Fail - try: - raise translatedData[2] - except Exception as e: - traceback.print_exc() - errorString = str(e) + Fore.RED - return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET - - -def openFiles(filename): - with open("files/" + filename, "r", encoding="utf-8") as readFile: - translatedData = parseJS(readFile, filename) - - return translatedData - - -def parseJS(readFile, filename): - totalTokens = [0, 0] - data = readFile.readlines() - - with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar: - pbar.desc = filename - - try: - result = translateJS(data, pbar) - totalTokens[0] += result[0] - totalTokens[1] += result[1] - except Exception as e: - traceback.print_exc() - return [data, totalTokens, e] - return [data, totalTokens, None] - - -def translateJS(data, pbar): - tokens = [0, 0] - i = 0 - - # Regex & Plugin Name - regex = r'ObjectiveContent[\\]+":[\\]+"[\\]+"(.*?)[\\]+"' - - # Find Plugin - while i < len(data): - # Run Search - stringList = re.findall(regex, data[i]) - if len(stringList) != 0: - pbar.total = len(stringList) - pbar.refresh() - modifiedStringList = stringList.copy() - - # Remove Wordwrap [Optional] - for j in range(len(modifiedStringList)): - modifiedStringList[j] = modifiedStringList[j].replace(r"\\\\\\\\n", r" ") - - # Translate - response = translateGPT(modifiedStringList, f"Reply with the {LANGUAGE} translation", True, pbar) - translatedList = response[0] - tokens[0] = response[1][0] - tokens[0] = response[1][1] - - # Validate Length & Replace Each Match - if len(translatedList) == len(modifiedStringList): - for j in range(len(translatedList)): - # Add escape for ' - translatedList[j] = re.sub(r"[^\\](')", "\\'", translatedList[j]) - - # Wordwrap [Optional] - translatedList[j] = dazedwrap.wrapText(translatedList[j], LISTWIDTH) - translatedList[j] = translatedList[j].replace("\n", r"\\\\\\\\n") - - # Set - data[i] = data[i].replace(stringList[j], translatedList[j]) - # Mismatch - else: - pbar.write("Mismatch Error") - i += 1 - - return tokens - - -def subVars(jaString): - jaString = jaString.replace("\u3000", " ") - - # Nested - count = 0 - nestedList = re.findall(r"[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]", jaString) - nestedList = set(nestedList) - if len(nestedList) != 0: - for icon in nestedList: - jaString = jaString.replace(icon, "[Nested_" + str(count) + "]") - count += 1 - - # Icons - count = 0 - iconList = re.findall(r"[\\]+[iIkKwWaA]+\[[0-9]+\]", jaString) - iconList = set(iconList) - if len(iconList) != 0: - for icon in iconList: - jaString = jaString.replace(icon, "[Ascii_" + str(count) + "]") - count += 1 - - # Colors - count = 0 - colorList = re.findall(r"[\\]+[cC]\[[0-9]+\]", jaString) - colorList = set(colorList) - if len(colorList) != 0: - for color in colorList: - jaString = jaString.replace(color, "[Color_" + str(count) + "]") - count += 1 - - # Names - count = 0 - nameList = re.findall(r"[\\]+[nN]\[.+?\]+", jaString) - nameList = set(nameList) - if len(nameList) != 0: - for name in nameList: - jaString = jaString.replace(name, "[Noun_" + str(count) + "]") - count += 1 - - # Variables - count = 0 - varList = re.findall(r"[\\]+[vV]\[[0-9]+\]", jaString) - varList = set(varList) - if len(varList) != 0: - for var in varList: - jaString = jaString.replace(var, "[Var_" + str(count) + "]") - count += 1 - - # Formatting - count = 0 - formatList = re.findall(r"[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]", jaString) - formatList = set(formatList) - if len(formatList) != 0: - for var in formatList: - jaString = jaString.replace(var, "[FCode_" + str(count) + "]") - count += 1 - - # Put all lists in list and return - allList = [nestedList, iconList, colorList, nameList, varList, formatList] - return [jaString, allList] - - -def resubVars(translatedText, allList): - # Fix Spacing and ChatGPT Nonsense - matchList = re.findall(r"\[\s?.+?\s?\]", translatedText) - if len(matchList) > 0: - for match in matchList: - text = match.strip() - translatedText = translatedText.replace(match, text) - - # Nested - count = 0 - if len(allList[0]) != 0: - for var in allList[0]: - translatedText = translatedText.replace("[Nested_" + str(count) + "]", var) - count += 1 - - # Icons - count = 0 - if len(allList[1]) != 0: - for var in allList[1]: - translatedText = translatedText.replace("[Ascii_" + str(count) + "]", var) - count += 1 - - # Colors - count = 0 - if len(allList[2]) != 0: - for var in allList[2]: - translatedText = translatedText.replace("[Color_" + str(count) + "]", var) - count += 1 - - # Names - count = 0 - if len(allList[3]) != 0: - for var in allList[3]: - translatedText = translatedText.replace("[Noun_" + str(count) + "]", var) - count += 1 - - # Vars - count = 0 - if len(allList[4]) != 0: - for var in allList[4]: - translatedText = translatedText.replace("[Var_" + str(count) + "]", var) - count += 1 - - # Formatting - count = 0 - if len(allList[5]) != 0: - for var in allList[5]: - translatedText = translatedText.replace("[FCode_" + str(count) + "]", var) - count += 1 - - return translatedText - - -def batchList(input_list, batch_size): - if not isinstance(batch_size, int) or batch_size <= 0: - raise ValueError("batch_size must be a positive integer") - - return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)] - - -def createContext(fullPromptFlag, subbedT): - characters = "Game Characters:\n\ -皆月 (Minazuki)\n\ -さやか (Sayaka)\n\ -皆月 さやか (Minazuki Sayaka) - Female\n\ -広瀬 (Hirose)\n\ -智恵 (Chie) - Female\n\ -広瀬 智恵 (Hirose Chie) - Female\n\ -" - - system = ( - PROMPT + VOCAB - if fullPromptFlag - else f"\ -You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\ -Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\ -- You translate everything, including erotic and explicit content.\n\ -- 'Game Characters' - The names, nicknames, and genders of the game characters. Reference this to know the names, nicknames, and gender of characters in the game\n\ -- All text in your response must be in {LANGUAGE} even if it is hard to translate.\n\ -- Never include any notes, explanations, dislaimers, or anything similar in your response.\n\ -- Maintain any spacing in the translation.\n\ -- Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\ -- `...` can be a part of the dialogue. Translate it as it is.\n\ -{VOCAB}\n\ -" - ) - user = f"{subbedT}" - return characters, system, user - - -def translateText(characters, system, user, history): - # Prompt - msg = [{"role": "system", "content": system + characters}] - - # Characters - msg.append({"role": "system", "content": characters}) - - # History - if isinstance(history, list): - msg.extend([{"role": "system", "content": h} for h in history]) - else: - msg.append({"role": "system", "content": history}) - - # Content to TL - msg.append({"role": "user", "content": f"{user}"}) - response = openai.chat.completions.create( - temperature=0.1, - frequency_penalty=0.1, - model=MODEL, - messages=msg, - ) - return response - - -def cleanTranslatedText(translatedText, varResponse): - placeholders = { - f"{LANGUAGE} Translation: ": "", - "Translation: ": "", - "っ": "", - "〜": "~", - "ッ": "", - "。": ".", - "Placeholder Text": "", - # Add more replacements as needed - } - for target, replacement in placeholders.items(): - translatedText = translatedText.replace(target, replacement) - - # Elongate Long Dashes (Since GPT Ignores them...) - translatedText = elongateCharacters(translatedText) - translatedText = resubVars(translatedText, varResponse[1]) - return translatedText - - -def elongateCharacters(text): - # Define a pattern to match one character followed by one or more `ー` characters - # Using a positive lookbehind assertion to capture the preceding character - pattern = r"(?<=(.))ー+" - - # Define a replacement function that elongates the captured character - def repl(match): - char = match.group(1) # The character before the ー sequence - count = len(match.group(0)) - 1 # Number of ー characters - return char * count # Replace ー sequence with the character repeated - - # Use re.sub() to replace the pattern in the text - return re.sub(pattern, repl, text) - - -def extractTranslation(translatedTextList, is_list): - pattern = r"`?<[Ll]ine\d+>([\\]*.*?[\\]*?)<\/?[Ll]ine\d+>`?" - # If it's a batch (i.e., list), extract with tags; otherwise, return the single item. - if is_list: - matchList = re.findall(pattern, translatedTextList) - return matchList - else: - matchList = re.findall(pattern, translatedTextList) - return matchList[0][0] if matchList else translatedTextList - - -def countTokens(characters, system, user, history): - inputTotalTokens = 0 - outputTotalTokens = 0 - enc = tiktoken.encoding_for_model("gpt-4") - - # Input - if isinstance(history, list): - for line in history: - inputTotalTokens += len(enc.encode(line)) - else: - inputTotalTokens += len(enc.encode(history)) - inputTotalTokens += len(enc.encode(system)) - inputTotalTokens += len(enc.encode(characters)) - inputTotalTokens += len(enc.encode(user)) - - # Output - outputTotalTokens += round(len(enc.encode(user)) * 3) - - return [inputTotalTokens, outputTotalTokens] - - -@retry(exceptions=Exception, tries=5, delay=5) -def translateGPT(text, history, fullPromptFlag, pbar): - mismatch = False - totalTokens = [0, 0] - if isinstance(text, list): - tList = batchList(text, BATCHSIZE) - else: - tList = [text] - - for index, tItem in enumerate(tList): - # Before sending to translation, if we have a list of items, add the formatting - if isinstance(tItem, list): - payload = "\n".join([f"`{item}`" for i, item in enumerate(tItem)]) - payload = re.sub(r"(<)(\/Line\d+>)", r"\1>Placeholder Text<\3", payload) - varResponse = subVars(payload) - subbedT = varResponse[0] - else: - varResponse = subVars(tItem) - subbedT = varResponse[0] - - # Things to Check before starting translation - if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", subbedT): - continue - - # Create Message - characters, system, user = createContext(fullPromptFlag, subbedT) - - # Calculate Estimate - if ESTIMATE: - estimate = countTokens(characters, system, user, history) - totalTokens[0] += estimate[0] - totalTokens[1] += estimate[1] - continue - - # Translating - response = translateText(characters, system, user, history) - translatedText = response.choices[0].message.content - totalTokens[0] += response.usage.prompt_tokens - totalTokens[1] += response.usage.completion_tokens - - # Formatting - translatedText = cleanTranslatedText(translatedText, varResponse) - if isinstance(tItem, list): - extractedTranslations = extractTranslation(translatedText, True) - tList[index] = extractedTranslations - if len(tItem) != len(extractedTranslations): - # Mismatch. Try Again - response = translateText(characters, system, user, history) - translatedText = response.choices[0].message.content - totalTokens[0] += response.usage.prompt_tokens - totalTokens[1] += response.usage.completion_tokens - - # Formatting - translatedText = cleanTranslatedText(translatedText, varResponse) - if isinstance(tItem, list): - extractedTranslations = extractTranslation(translatedText, True) - if len(tItem) == len(extractedTranslations): - tList[index] = extractedTranslations - else: - mismatch = True # Just here for breakpoint - - # Create History - history = tList[index] # Update history if we have a list - pbar.update(len(tList[index])) - - else: - # Ensure we're passing a single string to extractTranslation - extractedTranslations = extractTranslation(translatedText, False) - tList[index] = extractedTranslations - - # Combine if multilist - if isinstance(tList[0], list): - tList = [t for sublist in tList for t in sublist] - - # Return - if format == "json": - return [tList, totalTokens] - else: - return [tList[0], totalTokens] diff --git a/modules/json.py b/modules/json.py index f6477d9..476a109 100644 --- a/modules/json.py +++ b/modules/json.py @@ -296,7 +296,6 @@ def translateJSON(data, translatedList): translateJSON(data, [stringListTL]) return tokens - # Save some money and enter the character before translation def getSpeaker(speaker): match speaker: @@ -343,11 +342,68 @@ def batchList(input_list, batch_size): return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)] +def parseVocabWithCategories(vocabText): + """Parse vocabulary text and extract terms with their categories.""" + pairs = [] + seen = set() + currentCategory = None + + for line in vocabText.splitlines(): + line = line.strip() + if not line or line.startswith('```'): + continue + + # Check if this is a category header + if line.startswith('#'): + currentCategory = line + continue + + # Parse vocabulary term + m = re.match(r'^(.+?)(?:\s?[\(–])', line) # term is everything before space + '(' or '–' + if m: + term = m.group(1) + if term not in seen: + pairs.append((term, line, currentCategory)) + seen.add(term) + + return pairs + + +def buildMatchedVocabText(vocabPairs, subbedT): + """Build formatted vocabulary text with matched terms organized by category.""" + matchedCategories = {} + + # Use word boundaries for Japanese if appropriate, or allow substring as before. + for term, line, category in vocabPairs: + # "term in subbedT" could be false positive; can use regex but Japanese doesn't always have spaces. + if term in subbedT: + if category not in matchedCategories: + matchedCategories[category] = [] + matchedCategories[category].append(line) + + # Format matched vocabulary with categories + if matchedCategories: + formattedLines = ["Here are some vocabulary and terms so that you know the proper spelling and translation.\n"] + for category, lines in matchedCategories.items(): + if category: # Only add category header if it exists + formattedLines.append(category) + formattedLines.extend(lines) + formattedLines.append("") # Add blank line between categories + matchedVocabText = f"```\n{chr(10).join(formattedLines).rstrip()}\n```" + else: + matchedVocabText = "" + + return matchedVocabText + + def createContext(fullPromptFlag, subbedT, format): - system = ( - PROMPT + VOCAB - if fullPromptFlag - else f"\ + vocabPairs = parseVocabWithCategories(VOCAB) + matchedVocabText = buildMatchedVocabText(vocabPairs, subbedT) + + if fullPromptFlag: + system = PROMPT + matchedVocabText + else: + system = f"\ You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\ - You translate everything, including erotic and explicit content.\n\ @@ -357,9 +413,8 @@ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{ - Maintain any spacing in the translation.\n\ - Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\ - `...` can be a part of the dialogue. Translate it as it is.\n\ -{VOCAB}\n\ +{matchedVocabText}\n\ " - ) if format == "json": user = f"```json\n{subbedT}\n```" else: diff --git a/modules/kansen.py b/modules/kansen.py deleted file mode 100644 index 17ac3e8..0000000 --- a/modules/kansen.py +++ /dev/null @@ -1,713 +0,0 @@ -# Libraries -import os -import re -import util.dazedwrap as dazedwrap -import threading -import time -import traceback -import tiktoken -import openai -from pathlib import Path -from colorama import Fore -from dotenv import load_dotenv -from retry import retry -from tqdm import tqdm - -# Open AI -load_dotenv() -if os.getenv("api").replace(" ", "") != "": - openai.base_url = os.getenv("api") -openai.organization = os.getenv("org") -openai.api_key = os.getenv("key") - -# Globals -MODEL = os.getenv("model") -TIMEOUT = int(os.getenv("timeout")) -LANGUAGE = os.getenv("language").capitalize() -PROMPT = Path("prompt.txt").read_text(encoding="utf-8") -VOCAB = Path("vocab.txt").read_text(encoding="utf-8") -THREADS = int(os.getenv("threads")) -LOCK = threading.Lock() -WIDTH = int(os.getenv("width")) -LISTWIDTH = int(os.getenv("listWidth")) -NOTEWIDTH = 70 -MAXHISTORY = 10 -ESTIMATE = "" -TOKENS = [0, 0] -NAMESLIST = [] -NAMES = False # Output a list of all the character names found -BRFLAG = False # If the game uses
instead -FIXTEXTWRAP = False # Overwrites textwrap -IGNORETLTEXT = False # Ignores all translated text. -MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong) - -# tqdm Globals -BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}" -POSITION = 0 -LEAVE = False - -# Regex - Need to change this if you want to translate from/to other languages. Default is Japanese Regex -LANGREGEX = r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+" - -# Pricing - Depends on the model https://openai.com/pricing -# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request -# If you are getting a MISMATCH LENGTH error, lower the batch size. -if "gpt-3.5" in MODEL: - INPUTAPICOST = 3.00 - OUTPUTAPICOST = 5.00 - BATCHSIZE = 10 - FREQUENCY_PENALTY = 0.2 -elif "gpt-4" in MODEL: - INPUTAPICOST = 2.0 - OUTPUTAPICOST = 8.00 - BATCHSIZE = 30 - FREQUENCY_PENALTY = 0.05 -elif "deepseek" in MODEL: - INPUTAPICOST = 0.27 - OUTPUTAPICOST = 1.10 - BATCHSIZE = 30 - FREQUENCY_PENALTY = 0.05 -else: - INPUTAPICOST = float(os.getenv("input_cost")) - OUTPUTAPICOST = float(os.getenv("output_cost")) - BATCHSIZE = int(os.getenv("batchsize")) - FREQUENCY_PENALTY = float(os.getenv("frequency_penalty")) - - -def handleKansen(filename, estimate): - global ESTIMATE - ESTIMATE = estimate - - if ESTIMATE: - start = time.time() - translatedData = openFiles(filename) - - # Print Result - end = time.time() - tqdm.write(getResultString(translatedData, end - start, filename)) - with LOCK: - TOKENS[0] += translatedData[1][0] - TOKENS[1] += translatedData[1][1] - - # Print Total - totalString = getResultString(["", TOKENS, None], end - start, "TOTAL") - - # Print any errors on maps - if len(MISMATCH) > 0: - return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET - else: - return totalString - - else: - try: - with open("translated/" + filename, "w", encoding="shift_jis", errors="ignore") as outFile: - start = time.time() - translatedData = openFiles(filename) - - # Print Result - end = time.time() - outFile.writelines(translatedData[0]) - tqdm.write(getResultString(translatedData, end - start, filename)) - with LOCK: - TOKENS[0] += translatedData[1][0] - TOKENS[1] += translatedData[1][1] - except Exception: - traceback.print_exc() - return "Fail" - - return getResultString(["", TOKENS, None], end - start, "TOTAL") - - -def getResultString(translatedData, translationTime, filename): - # File Print String - totalTokenstring = ( - Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]" - "[Output: " - + str(translatedData[1][1]) - + "]" "[Cost: ${:,.4f}".format(((translatedData[1][0] / 1000000) * INPUTAPICOST) + ((translatedData[1][1] / 1000000) * OUTPUTAPICOST)) - + "]" - ) - timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]" - - if translatedData[2] == None: - # Success - return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET - - else: - # Fail - try: - raise translatedData[2] - except Exception as e: - traceback.print_exc() - errorString = str(e) + Fore.RED - return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET - - -def openFiles(filename): - with open("files/" + filename, "r", encoding="cp932") as readFile: - translatedData = parseTyrano(readFile, filename) - - # Delete lines marked for deletion - finalData = [] - for line in translatedData[0]: - if line != "\\d\n": - finalData.append(line) - translatedData[0] = finalData - - return translatedData - - -def parseTyrano(readFile, filename): - totalTokens = [0, 0] - totalLines = 0 - - # Get total for progress bar - data = readFile.readlines() - totalLines = len(data) - - with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar: - pbar.desc = filename - pbar.total = totalLines - - try: - result = translateTyrano(data, pbar, totalLines) - totalTokens[0] += result[0] - totalTokens[1] += result[1] - except Exception as e: - traceback.print_exc() - return [data, totalTokens, e] - return [data, totalTokens, None] - - -def translateTyrano(data, pbar, totalLines): - textHistory = [] - batch = [] - currentGroup = [] - maxHistory = MAXHISTORY - tokens = [0, 0] - speaker = "" - insertBool = False - global LOCK, ESTIMATE - i = 0 - batchStartIndex = 0 - - while i < len(data): - # Speaker - if "[ns]" in data[i]: - matchList = re.findall(r"\[ns\](.+?)\[", data[i]) - if len(matchList) != 0: - response = getSpeaker(matchList[0]) - speaker = response[0] - tokens[0] += response[1][0] - tokens[1] += response[1][1] - data[i] = "[ns]" + speaker + "[nse]\n" - else: - speaker = "" - - # Choices - elif "[sel" in data[i]: - matchList = re.findall(r'\[sel.+text="(.+?)".+', data[i]) - if len(matchList) != 0: - originalText = matchList[0] - if len(textHistory) > 0: - response = translateGPT( - matchList[0], - "Keep your translation as brief as possible. Previous text for context: " - + textHistory[len(textHistory) - 1] - + "\n\nReply in the style of a dialogue option.", - False, - ) - else: - response = translateGPT( - matchList[0], - "\n\nReply in the style of a dialogue option.", - False, - ) - translatedText = response[0] - tokens[0] += response[1][0] - tokens[1] += response[1][1] - - # Remove characters that may break scripts - charList = [".", '"', "\\n"] - for char in charList: - translatedText = translatedText.replace(char, "") - - # Escape all ' - translatedText = translatedText.replace("\\", "") - # translatedText = translatedText.replace("'", "\\\'") - - # Set Data - translatedText = data[i].replace(originalText, translatedText) - data[i] = translatedText - - # Lines - matchList = re.findall(r"(.+?)\[[rpcms_sel]+\]$", data[i]) - if len(matchList) > 0: - if "hisout" in matchList[0]: - i += 1 - continue - currentGroup.append(matchList[0]) - if len(data) > i + 1: - while "[r]" in data[i + 1]: - if insertBool is True: - data[i] = r"\d\n" - pbar.update(1) - i += 1 - matchList = re.findall(r"(.+?)\[r\]", data[i]) - if len(matchList) > 0: - currentGroup.append(matchList[0]) - while "[pcms]" in data[i + 1]: - if insertBool is True: - data[i] = r"\d\n" - pbar.update(1) - i += 1 - matchList = re.findall(r"(.+?)\[pcms\]", data[i]) - if len(matchList) > 0: - currentGroup.append(matchList[0]) - while "[pcms_sel]" in data[i + 1]: - if insertBool is True: - data[i] = r"\d\n" - pbar.update(1) - i += 1 - matchList = re.findall(r"(.+?)\[pcms_sel\]", data[i]) - if len(matchList) > 0: - currentGroup.append(matchList[0]) - # Join up 401 groups for better translation. - if len(currentGroup) > 0: - finalJAString = " ".join(currentGroup) - oldjaString = finalJAString - - # Remove any textwrap - if FIXTEXTWRAP == True: - finalJAString = finalJAString.replace("[r]", " ") - - # Remove Extra Stuff bad for translation. - finalJAString = finalJAString.replace("ﾞ", "") - finalJAString = finalJAString.replace("・", ".") - finalJAString = finalJAString.replace("‶", "") - finalJAString = finalJAString.replace("”", "") - finalJAString = finalJAString.replace("―", "-") - finalJAString = finalJAString.replace("…", "...") - finalJAString = re.sub(r"(\.{3}\.+)", "...", finalJAString) - finalJAString = finalJAString.replace("　", " ") - - # Furigana Removal - matchList = re.findall(r"(\[ruby\stext=.+text=\"(.+)\"\])", finalJAString) - if len(matchList) > 0: - finalJAString = finalJAString.replace(matchList[0][0], matchList[0][1]) - - # Add Speaker (If there is one) - if speaker != "": - finalJAString = f"{speaker}: {finalJAString}" - - # [Passthrough 1] Pulling From File - if insertBool is False: - # Append to List and Clear Values - batch.append(finalJAString) - speaker = "" - - # Translate Batch if Full - if len(batch) == BATCHSIZE: - # Translate - response = translateGPT(batch, textHistory, True) - tokens[0] += response[1][0] - tokens[1] += response[1][1] - translatedBatch = response[0] - textHistory = translatedBatch[-10:] - - # Set Values - if len(batch) == len(translatedBatch): - i = batchStartIndex - insertBool = True - - # Mismatch - else: - pbar.write(f"Mismatch: {batchStartIndex} - {i}") - MISMATCH.append(batch) - batchStartIndex = i - batch.clear() - - i += 1 - if insertBool is True: - pbar.update(1) - currentGroup = [] - - # [Passthrough 2] Setting Data - else: - # Get Text - translatedText = translatedBatch[0] - translatedText = translatedText.replace('\\"', '"') - translatedText = translatedText.replace("[", "(") - translatedText = translatedText.replace("]", ")") - - # Remove added speaker - translatedText = re.sub(r"^.+?:\s", "", translatedText) - - # Textwrap - translatedText = dazedwrap.wrapText(translatedText, width=WIDTH) - textList = translatedText.split("\n") - - # Set Text - data[i] = r"\d\n" - for line in textList: - # Wordwrap Text - if "[r]" not in line: - line = dazedwrap.wrapText(line, width=WIDTH) - line = line.replace("\n", "[r]") - - # Set - data.insert(i, line.strip() + "[r]\n") - i += 1 - data[i - 1] = data[i - 1].replace("[r]", "[pcms]") - translatedBatch.pop(0) - speaker = "" - currentGroup = [] - - # If Batch is empty. Move on. - if len(translatedBatch) == 0: - insertBool = False - batchStartIndex = i - batch.clear() - - # Nothing relevant. Skip Line. - else: - i += 1 - if insertBool is True: - pbar.update(1) - - # Translate Batch if not empty and EOF - if len(batch) != 0 and i >= len(data): - # Translate - response = translateGPT(batch, textHistory, True) - tokens[0] += response[1][0] - tokens[1] += response[1][1] - translatedBatch = response[0] - textHistory = translatedBatch[-10:] - - # Set Values - if len(batch) == len(translatedBatch): - i = batchStartIndex - insertBool = True - - # Mismatch - else: - pbar.write(f"Mismatch: {batchStartIndex} - {i}") - MISMATCH.append(batch) - batchStartIndex = i - batch.clear() - - currentGroup = [] - return tokens - - -# Save some money and enter the character before translation -def getSpeaker(speaker): - match speaker: - case "央": - return ["Akira", [0, 0]] - case "累": - return ["Rui", [0, 0]] - case "梨里": - return ["Riri", [0, 0]] - case "純": - return ["Jun", [0, 0]] - case "美鈴": - return ["Misuzu", [0, 0]] - case "須田": - return ["Suda", [0, 0]] - case "高橋": - return ["Takahashi", [0, 0]] - case "勇二": - return ["Yuuji", [0, 0]] - case _: - return translateGPT( - speaker, - "Reply with only the " + LANGUAGE + " translation of the NPC name.", - False, - ) - - -def subVars(jaString): - jaString = jaString.replace("\u3000", " ") - - # Nested - count = 0 - nestedList = re.findall(r"[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]", jaString) - nestedList = set(nestedList) - if len(nestedList) != 0: - for icon in nestedList: - jaString = jaString.replace(icon, "{Nested_" + str(count) + "}") - count += 1 - - # Icons - count = 0 - iconList = re.findall(r"[\\]+[iIkKwWaA]+\[[0-9]+\]", jaString) - iconList = set(iconList) - if len(iconList) != 0: - for icon in iconList: - jaString = jaString.replace(icon, "{Ascii_" + str(count) + "}") - count += 1 - - # Colors - count = 0 - colorList = re.findall(r"[\\]+[cC]\[[0-9]+\]", jaString) - colorList = set(colorList) - if len(colorList) != 0: - for color in colorList: - jaString = jaString.replace(color, "{Color_" + str(count) + "}") - count += 1 - - # Names - count = 0 - nameList = re.findall(r"[\\]+[nN]\[.+?\]+", jaString) - nameList = set(nameList) - if len(nameList) != 0: - for name in nameList: - jaString = jaString.replace(name, "{Noun_" + str(count) + "}") - count += 1 - - # Variables - count = 0 - varList = re.findall(r"[\\]+[vV]\[[0-9]+\]", jaString) - varList = set(varList) - if len(varList) != 0: - for var in varList: - jaString = jaString.replace(var, "{Var_" + str(count) + "}") - count += 1 - - # Formatting - count = 0 - formatList = re.findall(r"[\\]+[\w]+\[.+?\]", jaString) - formatList = set(formatList) - if len(formatList) != 0: - for var in formatList: - jaString = jaString.replace(var, "{FCode_" + str(count) + "}") - count += 1 - - # Put all lists in list and return - allList = [nestedList, iconList, colorList, nameList, varList, formatList] - return [jaString, allList] - - -def resubVars(translatedText, allList): - # Fix Spacing and ChatGPT Nonsense - matchList = re.findall(r"\[\s?.+?\s?\]", translatedText) - if len(matchList) > 0: - for match in matchList: - text = match.strip() - translatedText = translatedText.replace(match, text) - - # Nested - count = 0 - if len(allList[0]) != 0: - for var in allList[0]: - translatedText = translatedText.replace("{Nested_" + str(count) + "}", var) - count += 1 - - # Icons - count = 0 - if len(allList[1]) != 0: - for var in allList[1]: - translatedText = translatedText.replace("{Ascii_" + str(count) + "}", var) - count += 1 - - # Colors - count = 0 - if len(allList[2]) != 0: - for var in allList[2]: - translatedText = translatedText.replace("{Color_" + str(count) + "}", var) - count += 1 - - # Names - count = 0 - if len(allList[3]) != 0: - for var in allList[3]: - translatedText = translatedText.replace("{Noun_" + str(count) + "}", var) - count += 1 - - # Vars - count = 0 - if len(allList[4]) != 0: - for var in allList[4]: - translatedText = translatedText.replace("{Var_" + str(count) + "}", var) - count += 1 - - # Formatting - count = 0 - if len(allList[5]) != 0: - for var in allList[5]: - translatedText = translatedText.replace("{FCode_" + str(count) + "}", var) - count += 1 - - return translatedText - - -def batchList(input_list, batch_size): - if not isinstance(batch_size, int) or batch_size <= 0: - raise ValueError("batch_size must be a positive integer") - - return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)] - - -def createContext(fullPromptFlag, subbedT): - characters = "Game Characters:\n\ -渋江 央 (Shibue Akira) - Male\n\ -蘆名 累 (Ashina Rui) - Female\n\ -清原 梨里 (Kiyohara Riri) - Female\n\ -五十嵐 純 (Igarashi Jun) - Female\n\ -子野日 美鈴 (Nenohi Misuzu) - Female\n\ -須田 (Suda) - Male\n\ -高橋 (Takahashi) - Female\n\ -勇二 (Yuuji) - Male\n\ -" - - system = ( - PROMPT + VOCAB - if fullPromptFlag - else f"\ -You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\ -You are going to be translating text from a videogame.\n\ -I will give you lines of text, and you must translate each line to the best of your ability.\n\ -{VOCAB}\n\ -Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\ -" - ) - user = f"{subbedT}" - return characters, system, user - - -def translateText(characters, system, user, history): - # Prompt - msg = [{"role": "system", "content": system + characters}] - - # Characters - msg.append({"role": "system", "content": characters}) - - # History - if isinstance(history, list): - msg.extend([{"role": "system", "content": h} for h in history]) - else: - msg.append({"role": "system", "content": history}) - - # Content to TL - msg.append({"role": "user", "content": f"{user}"}) - response = openai.chat.completions.create( - temperature=0.1, - frequency_penalty=0.1, - presence_penalty=0.1, - model=MODEL, - messages=msg, - ) - return response - - -def cleanTranslatedText(translatedText, varResponse): - placeholders = { - f"{LANGUAGE} Translation: ": "", - "Translation: ": "", - "っ": "", - "〜": "~", - "ッ": "", - "。": ".", - "Placeholder Text": "", - # Add more replacements as needed - } - for target, replacement in placeholders.items(): - translatedText = translatedText.replace(target, replacement) - - translatedText = resubVars(translatedText, varResponse[1]) - return [line for line in translatedText.replace("\\n", "\n").split("\n") if line] - - -def extractTranslation(translatedTextList, is_list): - pattern = r"`?([\\]*.*?[\\]*?)<\/?Line\d+>`?" - # If it's a batch (i.e., list), extract with tags; otherwise, return the single item. - if is_list: - return [re.findall(pattern, line)[0][1] for line in translatedTextList if re.search(pattern, line)] - else: - matchList = re.findall(pattern, translatedTextList) - return matchList[0][1] if matchList else translatedTextList - - -def countTokens(characters, system, user, history): - inputTotalTokens = 0 - outputTotalTokens = 0 - enc = tiktoken.encoding_for_model("gpt-4") - - # Input - if isinstance(history, list): - for line in history: - inputTotalTokens += len(enc.encode(line)) - else: - inputTotalTokens += len(enc.encode(history)) - inputTotalTokens += len(enc.encode(system)) - inputTotalTokens += len(enc.encode(characters)) - inputTotalTokens += len(enc.encode(user)) - - # Output - outputTotalTokens += round(len(enc.encode(user)) * 3) - - return [inputTotalTokens, outputTotalTokens] - - -@retry(exceptions=Exception, tries=5, delay=5) -def translateGPT(text, history, fullPromptFlag): - totalTokens = [0, 0] - if isinstance(text, list): - tList = batchList(text, BATCHSIZE) - else: - tList = [text] - - for index, tItem in enumerate(tList): - # Before sending to translation, if we have a list of items, add the formatting - if isinstance(tItem, list): - payload = "\n".join([f"`{item}`" for i, item in enumerate(tItem)]) - payload = payload.replace("``", "`Placeholder Text`") - varResponse = subVars(payload) - subbedT = varResponse[0] - else: - varResponse = subVars(tItem) - subbedT = varResponse[0] - - # Things to Check before starting translation - if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", subbedT): - continue - - # Create Message - characters, system, user = createContext(fullPromptFlag, subbedT) - - # Calculate Estimate - if ESTIMATE: - estimate = countTokens(characters, system, user, history) - totalTokens[0] += estimate[0] - totalTokens[1] += estimate[1] - continue - - # Translating - response = translateText(characters, system, user, history) - translatedText = response.choices[0].message.content - totalTokens[0] += response.usage.prompt_tokens - totalTokens[1] += response.usage.completion_tokens - - # Formatting - translatedTextList = cleanTranslatedText(translatedText, varResponse) - if isinstance(tItem, list): - extractedTranslations = extractTranslation(translatedTextList, True) - tList[index] = extractedTranslations - if len(tItem) != len(translatedTextList): - mismatch = True # Just here so breakpoint can be set - history = extractedTranslations[-10:] # Update history if we have a list - else: - # Ensure we're passing a single string to extractTranslation - extractedTranslations = extractTranslation("\n".join(translatedTextList), False) - tList[index] = extractedTranslations - - # Combine if multilist - if isinstance(tList[0], list): - tList = [t for sublist in tList for t in sublist] - - # Return - if format == "json": - return [tList, totalTokens] - else: - return [tList[0], totalTokens] diff --git a/modules/kirikiri.py b/modules/kirikiri.py index 5ad74a5..015a417 100644 --- a/modules/kirikiri.py +++ b/modules/kirikiri.py @@ -341,7 +341,6 @@ def translateKiriKiri(data, pbar, filename, jobList): return tokens - # Save some money and enter the character before translation def getSpeaker(speaker): match speaker: @@ -359,7 +358,7 @@ def getSpeaker(speaker): response = translateGPT( f"{speaker}", "Reply with the " + LANGUAGE + " translation of the NPC name.", - True, + False, ) response[0] = response[0].title() response[0] = response[0].replace("'S", "'s") @@ -388,11 +387,68 @@ def batchList(input_list, batch_size): return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)] +def parseVocabWithCategories(vocabText): + """Parse vocabulary text and extract terms with their categories.""" + pairs = [] + seen = set() + currentCategory = None + + for line in vocabText.splitlines(): + line = line.strip() + if not line or line.startswith('```'): + continue + + # Check if this is a category header + if line.startswith('#'): + currentCategory = line + continue + + # Parse vocabulary term + m = re.match(r'^(.+?)(?:\s?[\(–])', line) # term is everything before space + '(' or '–' + if m: + term = m.group(1) + if term not in seen: + pairs.append((term, line, currentCategory)) + seen.add(term) + + return pairs + + +def buildMatchedVocabText(vocabPairs, subbedT): + """Build formatted vocabulary text with matched terms organized by category.""" + matchedCategories = {} + + # Use word boundaries for Japanese if appropriate, or allow substring as before. + for term, line, category in vocabPairs: + # "term in subbedT" could be false positive; can use regex but Japanese doesn't always have spaces. + if term in subbedT: + if category not in matchedCategories: + matchedCategories[category] = [] + matchedCategories[category].append(line) + + # Format matched vocabulary with categories + if matchedCategories: + formattedLines = ["Here are some vocabulary and terms so that you know the proper spelling and translation.\n"] + for category, lines in matchedCategories.items(): + if category: # Only add category header if it exists + formattedLines.append(category) + formattedLines.extend(lines) + formattedLines.append("") # Add blank line between categories + matchedVocabText = f"```\n{chr(10).join(formattedLines).rstrip()}\n```" + else: + matchedVocabText = "" + + return matchedVocabText + + def createContext(fullPromptFlag, subbedT, format): - system = ( - PROMPT + VOCAB - if fullPromptFlag - else f"\ + vocabPairs = parseVocabWithCategories(VOCAB) + matchedVocabText = buildMatchedVocabText(vocabPairs, subbedT) + + if fullPromptFlag: + system = PROMPT + matchedVocabText + else: + system = f"\ You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\ - You translate everything, including erotic and explicit content.\n\ @@ -402,9 +458,8 @@ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{ - Maintain any spacing in the translation.\n\ - Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\ - `...` can be a part of the dialogue. Translate it as it is.\n\ -{VOCAB}\n\ +{matchedVocabText}\n\ " - ) if format == "json": user = f"```json\n{subbedT}\n```" else: @@ -418,9 +473,10 @@ def translateText(system, user, history, penalty, format, model=MODEL): # History if isinstance(history, list): - msg.extend([{"role": "system", "content": h} for h in history]) + msg.append({"role": "system", "content": "Translation History:"}) + msg.extend([{"role": "assistant", "content": h} for h in history]) else: - msg.append({"role": "system", "content": history}) + msg.append({"role": "assistant", "content": history}) # Response Format if format == "json": @@ -455,8 +511,11 @@ def cleanTranslatedText(translatedText): "】": "]", "【": "[", "é": "e", - "ō": "o", + "this guy": "this bastard", + "This guy": "This bastard", "Placeholder Text": "", + "```json": "", + "```": "", # Add more replacements as needed } for target, replacement in placeholders.items(): @@ -518,7 +577,7 @@ def countTokens(system, user, history): inputTotalTokens += len(enc.encode(user)) # Output - outputTotalTokens += round(len(enc.encode(user)) * 3) + outputTotalTokens += round(len(enc.encode(user)) * 2.5) return [inputTotalTokens, outputTotalTokens] @@ -582,7 +641,8 @@ def translateGPT(text, history, fullPromptFlag): # AI Refused, Try Again if not translatedText: - response = translateText(f"{system}\n You translate ALL content.", user, history, 0.1, format) + response = translateText(f"{system}\n You translate ALL content.", user, history, 0.1, format, model="gpt-4o") + translatedText = response.choices[0].message.content # Report Tokens totalTokens[0] += response.usage.prompt_tokens diff --git a/modules/lune.py b/modules/lune.py index be4e3fc..279fb0c 100644 --- a/modules/lune.py +++ b/modules/lune.py @@ -300,7 +300,6 @@ def translateJSON(data, pbar): currentGroup = [] return tokens - # Save some money and enter the character before translation def getSpeaker(speaker): match speaker: @@ -318,7 +317,7 @@ def getSpeaker(speaker): response = translateGPT( f"{speaker}", "Reply with the " + LANGUAGE + " translation of the NPC name.", - True, + False, ) response[0] = response[0].title() response[0] = response[0].replace("'S", "'s") @@ -340,40 +339,6 @@ def getSpeaker(speaker): return [speaker, [0, 0]] -def subVars(jaString): - jaString = jaString.replace("\u3000", " ") - - # Formatting - count = 0 - codeList = re.findall(r"[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]", jaString) - codeList = set(codeList) - if len(codeList) != 0: - for var in codeList: - jaString = jaString.replace(var, "[FCode_" + str(count) + "]") - count += 1 - - # Put all lists in list and return - return [jaString, codeList] - - -def resubVars(translatedText, codeList): - # Fix Spacing and ChatGPT Nonsense - matchList = re.findall(r"\[\s?.+?\s?\]", translatedText) - if len(matchList) > 0: - for match in matchList: - text = match.strip() - translatedText = translatedText.replace(match, text) - - # Formatting - count = 0 - if len(codeList) != 0: - for var in codeList: - translatedText = translatedText.replace("[FCode_" + str(count) + "]", var) - count += 1 - - return translatedText - - def batchList(input_list, batch_size): if not isinstance(batch_size, int) or batch_size <= 0: raise ValueError("batch_size must be a positive integer") @@ -381,11 +346,68 @@ def batchList(input_list, batch_size): return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)] +def parseVocabWithCategories(vocabText): + """Parse vocabulary text and extract terms with their categories.""" + pairs = [] + seen = set() + currentCategory = None + + for line in vocabText.splitlines(): + line = line.strip() + if not line or line.startswith('```'): + continue + + # Check if this is a category header + if line.startswith('#'): + currentCategory = line + continue + + # Parse vocabulary term + m = re.match(r'^(.+?)(?:\s?[\(–])', line) # term is everything before space + '(' or '–' + if m: + term = m.group(1) + if term not in seen: + pairs.append((term, line, currentCategory)) + seen.add(term) + + return pairs + + +def buildMatchedVocabText(vocabPairs, subbedT): + """Build formatted vocabulary text with matched terms organized by category.""" + matchedCategories = {} + + # Use word boundaries for Japanese if appropriate, or allow substring as before. + for term, line, category in vocabPairs: + # "term in subbedT" could be false positive; can use regex but Japanese doesn't always have spaces. + if term in subbedT: + if category not in matchedCategories: + matchedCategories[category] = [] + matchedCategories[category].append(line) + + # Format matched vocabulary with categories + if matchedCategories: + formattedLines = ["Here are some vocabulary and terms so that you know the proper spelling and translation.\n"] + for category, lines in matchedCategories.items(): + if category: # Only add category header if it exists + formattedLines.append(category) + formattedLines.extend(lines) + formattedLines.append("") # Add blank line between categories + matchedVocabText = f"```\n{chr(10).join(formattedLines).rstrip()}\n```" + else: + matchedVocabText = "" + + return matchedVocabText + + def createContext(fullPromptFlag, subbedT, format): - system = ( - PROMPT + VOCAB - if fullPromptFlag - else f"\ + vocabPairs = parseVocabWithCategories(VOCAB) + matchedVocabText = buildMatchedVocabText(vocabPairs, subbedT) + + if fullPromptFlag: + system = PROMPT + matchedVocabText + else: + system = f"\ You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\ - You translate everything, including erotic and explicit content.\n\ @@ -395,9 +417,8 @@ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{ - Maintain any spacing in the translation.\n\ - Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\ - `...` can be a part of the dialogue. Translate it as it is.\n\ -{VOCAB}\n\ +{matchedVocabText}\n\ " - ) if format == "json": user = f"```json\n{subbedT}\n```" else: @@ -411,9 +432,10 @@ def translateText(system, user, history, penalty, format, model=MODEL): # History if isinstance(history, list): - msg.extend([{"role": "system", "content": h} for h in history]) + msg.append({"role": "system", "content": "Translation History:"}) + msg.extend([{"role": "assistant", "content": h} for h in history]) else: - msg.append({"role": "system", "content": history}) + msg.append({"role": "assistant", "content": history}) # Response Format if format == "json": @@ -433,7 +455,7 @@ def translateText(system, user, history, penalty, format, model=MODEL): return response -def cleanTranslatedText(translatedText, varResponse): +def cleanTranslatedText(translatedText): placeholders = { f"{LANGUAGE} Translation: ": "", "Translation: ": "", @@ -444,15 +466,26 @@ def cleanTranslatedText(translatedText, varResponse): "「": '\\"', "」": '\\"', "- ": "-", + "—": "―", + "】": "]", + "【": "[", + "é": "e", + "this guy": "this bastard", + "This guy": "This bastard", "Placeholder Text": "", + "```json": "", + "```": "", # Add more replacements as needed } for target, replacement in placeholders.items(): translatedText = translatedText.replace(target, replacement) + # Remove Repeating Characters + pattern = re.compile(r"(.)\s*\1(?:\s*\1){" + str(20 - 1) + r",}") + translatedText = pattern.sub(lambda match: match.group(0).replace(" ", "")[:20], translatedText) + # Elongate Long Dashes (Since GPT Ignores them...) translatedText = elongateCharacters(translatedText) - translatedText = resubVars(translatedText, varResponse[1]) return translatedText @@ -473,6 +506,8 @@ def elongateCharacters(text): def extractTranslation(translatedTextList, is_list): try: + translatedTextList = re.sub(r'\\"+\"([^,\n}])', r'\\"\1', translatedTextList) + translatedTextList = re.sub(r"(?`\n\ - You translate everything, including erotic and explicit content.\n\ @@ -420,9 +476,8 @@ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{ - Maintain any spacing in the translation.\n\ - Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\ - `...` can be a part of the dialogue. Translate it as it is.\n\ -{VOCAB}\n\ +{matchedVocabText}\n\ " - ) if format == "json": user = f"```json\n{subbedT}\n```" else: @@ -436,9 +491,10 @@ def translateText(system, user, history, penalty, format, model=MODEL): # History if isinstance(history, list): - msg.extend([{"role": "system", "content": h} for h in history]) + msg.append({"role": "system", "content": "Translation History:"}) + msg.extend([{"role": "assistant", "content": h} for h in history]) else: - msg.append({"role": "system", "content": history}) + msg.append({"role": "assistant", "content": history}) # Response Format if format == "json": @@ -473,8 +529,11 @@ def cleanTranslatedText(translatedText): "】": "]", "【": "[", "é": "e", - "ō": "o", + "this guy": "this bastard", + "This guy": "This bastard", "Placeholder Text": "", + "```json": "", + "```": "", # Add more replacements as needed } for target, replacement in placeholders.items(): @@ -536,7 +595,7 @@ def countTokens(system, user, history): inputTotalTokens += len(enc.encode(user)) # Output - outputTotalTokens += round(len(enc.encode(user)) * 3) + outputTotalTokens += round(len(enc.encode(user)) * 2.5) return [inputTotalTokens, outputTotalTokens] @@ -600,7 +659,8 @@ def translateGPT(text, history, fullPromptFlag): # AI Refused, Try Again if not translatedText: - response = translateText(f"{system}\n You translate ALL content.", user, history, 0.1, format) + response = translateText(f"{system}\n You translate ALL content.", user, history, 0.1, format, model="gpt-4o") + translatedText = response.choices[0].message.content # Report Tokens totalTokens[0] += response.usage.prompt_tokens diff --git a/modules/regex.py b/modules/regex.py index 9350432..24b6413 100644 --- a/modules/regex.py +++ b/modules/regex.py @@ -356,7 +356,6 @@ def translateRegex(data, translatedList): translateRegex(data, [stringListTL, choiceListTL]) return tokens - # Save some money and enter the character before translation def getSpeaker(speaker): match speaker: @@ -403,11 +402,68 @@ def batchList(input_list, batch_size): return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)] +def parseVocabWithCategories(vocabText): + """Parse vocabulary text and extract terms with their categories.""" + pairs = [] + seen = set() + currentCategory = None + + for line in vocabText.splitlines(): + line = line.strip() + if not line or line.startswith('```'): + continue + + # Check if this is a category header + if line.startswith('#'): + currentCategory = line + continue + + # Parse vocabulary term + m = re.match(r'^(.+?)(?:\s?[\(–])', line) # term is everything before space + '(' or '–' + if m: + term = m.group(1) + if term not in seen: + pairs.append((term, line, currentCategory)) + seen.add(term) + + return pairs + + +def buildMatchedVocabText(vocabPairs, subbedT): + """Build formatted vocabulary text with matched terms organized by category.""" + matchedCategories = {} + + # Use word boundaries for Japanese if appropriate, or allow substring as before. + for term, line, category in vocabPairs: + # "term in subbedT" could be false positive; can use regex but Japanese doesn't always have spaces. + if term in subbedT: + if category not in matchedCategories: + matchedCategories[category] = [] + matchedCategories[category].append(line) + + # Format matched vocabulary with categories + if matchedCategories: + formattedLines = ["Here are some vocabulary and terms so that you know the proper spelling and translation.\n"] + for category, lines in matchedCategories.items(): + if category: # Only add category header if it exists + formattedLines.append(category) + formattedLines.extend(lines) + formattedLines.append("") # Add blank line between categories + matchedVocabText = f"```\n{chr(10).join(formattedLines).rstrip()}\n```" + else: + matchedVocabText = "" + + return matchedVocabText + + def createContext(fullPromptFlag, subbedT, format): - system = ( - PROMPT + VOCAB - if fullPromptFlag - else f"\ + vocabPairs = parseVocabWithCategories(VOCAB) + matchedVocabText = buildMatchedVocabText(vocabPairs, subbedT) + + if fullPromptFlag: + system = PROMPT + matchedVocabText + else: + system = f"\ You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\ - You translate everything, including erotic and explicit content.\n\ @@ -417,9 +473,8 @@ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{ - Maintain any spacing in the translation.\n\ - Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\ - `...` can be a part of the dialogue. Translate it as it is.\n\ -{VOCAB}\n\ +{matchedVocabText}\n\ " - ) if format == "json": user = f"```json\n{subbedT}\n```" else: @@ -474,6 +529,8 @@ def cleanTranslatedText(translatedText): "this guy": "this bastard", "This guy": "This bastard", "Placeholder Text": "", + "```json": "", + "```": "", # Add more replacements as needed } for target, replacement in placeholders.items(): @@ -535,7 +592,7 @@ def countTokens(system, user, history): inputTotalTokens += len(enc.encode(user)) # Output - outputTotalTokens += round(len(enc.encode(user)) * 3) + outputTotalTokens += round(len(enc.encode(user)) * 2.5) return [inputTotalTokens, outputTotalTokens] @@ -599,7 +656,8 @@ def translateGPT(text, history, fullPromptFlag): # AI Refused, Try Again if not translatedText: - response = translateText(f"{system}\n You translate ALL content.", user, history, 0.1, format) + response = translateText(f"{system}\n You translate ALL content.", user, history, 0.1, format, model="gpt-4o") + translatedText = response.choices[0].message.content # Report Tokens totalTokens[0] += response.usage.prompt_tokens diff --git a/modules/renpy.py b/modules/renpy.py index 1b51812..5cd3e7c 100644 --- a/modules/renpy.py +++ b/modules/renpy.py @@ -280,7 +280,6 @@ def translateRenpy(data, translatedList): MISMATCH.append(FILENAME) return tokens - # Save some money and enter the character before translation def getSpeaker(speaker): match speaker: @@ -298,7 +297,7 @@ def getSpeaker(speaker): response = translateGPT( f"{speaker}", "Reply with the " + LANGUAGE + " translation of the NPC name.", - True, + False, ) response[0] = response[0].title() response[0] = response[0].replace("'S", "'s") @@ -327,11 +326,68 @@ def batchList(input_list, batch_size): return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)] +def parseVocabWithCategories(vocabText): + """Parse vocabulary text and extract terms with their categories.""" + pairs = [] + seen = set() + currentCategory = None + + for line in vocabText.splitlines(): + line = line.strip() + if not line or line.startswith('```'): + continue + + # Check if this is a category header + if line.startswith('#'): + currentCategory = line + continue + + # Parse vocabulary term + m = re.match(r'^(.+?)(?:\s?[\(–])', line) # term is everything before space + '(' or '–' + if m: + term = m.group(1) + if term not in seen: + pairs.append((term, line, currentCategory)) + seen.add(term) + + return pairs + + +def buildMatchedVocabText(vocabPairs, subbedT): + """Build formatted vocabulary text with matched terms organized by category.""" + matchedCategories = {} + + # Use word boundaries for Japanese if appropriate, or allow substring as before. + for term, line, category in vocabPairs: + # "term in subbedT" could be false positive; can use regex but Japanese doesn't always have spaces. + if term in subbedT: + if category not in matchedCategories: + matchedCategories[category] = [] + matchedCategories[category].append(line) + + # Format matched vocabulary with categories + if matchedCategories: + formattedLines = ["Here are some vocabulary and terms so that you know the proper spelling and translation.\n"] + for category, lines in matchedCategories.items(): + if category: # Only add category header if it exists + formattedLines.append(category) + formattedLines.extend(lines) + formattedLines.append("") # Add blank line between categories + matchedVocabText = f"```\n{chr(10).join(formattedLines).rstrip()}\n```" + else: + matchedVocabText = "" + + return matchedVocabText + + def createContext(fullPromptFlag, subbedT, format): - system = ( - PROMPT + VOCAB - if fullPromptFlag - else f"\ + vocabPairs = parseVocabWithCategories(VOCAB) + matchedVocabText = buildMatchedVocabText(vocabPairs, subbedT) + + if fullPromptFlag: + system = PROMPT + matchedVocabText + else: + system = f"\ You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\ - You translate everything, including erotic and explicit content.\n\ @@ -341,9 +397,8 @@ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{ - Maintain any spacing in the translation.\n\ - Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\ - `...` can be a part of the dialogue. Translate it as it is.\n\ -{VOCAB}\n\ +{matchedVocabText}\n\ " - ) if format == "json": user = f"```json\n{subbedT}\n```" else: @@ -357,9 +412,10 @@ def translateText(system, user, history, penalty, format, model=MODEL): # History if isinstance(history, list): - msg.extend([{"role": "system", "content": h} for h in history]) + msg.append({"role": "system", "content": "Translation History:"}) + msg.extend([{"role": "assistant", "content": h} for h in history]) else: - msg.append({"role": "system", "content": history}) + msg.append({"role": "assistant", "content": history}) # Response Format if format == "json": @@ -394,8 +450,11 @@ def cleanTranslatedText(translatedText): "】": "]", "【": "[", "é": "e", - "ō": "o", + "this guy": "this bastard", + "This guy": "This bastard", "Placeholder Text": "", + "```json": "", + "```": "", # Add more replacements as needed } for target, replacement in placeholders.items(): @@ -457,7 +516,7 @@ def countTokens(system, user, history): inputTotalTokens += len(enc.encode(user)) # Output - outputTotalTokens += round(len(enc.encode(user)) * 3) + outputTotalTokens += round(len(enc.encode(user)) * 2.5) return [inputTotalTokens, outputTotalTokens] @@ -521,7 +580,8 @@ def translateGPT(text, history, fullPromptFlag): # AI Refused, Try Again if not translatedText: - response = translateText(f"{system}\n You translate ALL content.", user, history, 0.1, format) + response = translateText(f"{system}\n You translate ALL content.", user, history, 0.1, format, model="gpt-4o") + translatedText = response.choices[0].message.content # Report Tokens totalTokens[0] += response.usage.prompt_tokens diff --git a/modules/rpgmakerace.py b/modules/rpgmakerace.py index 9639c44..ba858d0 100644 --- a/modules/rpgmakerace.py +++ b/modules/rpgmakerace.py @@ -2413,7 +2413,6 @@ def searchSystem(data, pbar): return totalTokens - # Save some money and enter the character before translation def getSpeaker(speaker): match speaker: @@ -2460,11 +2459,68 @@ def batchList(input_list, batch_size): return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)] +def parseVocabWithCategories(vocabText): + """Parse vocabulary text and extract terms with their categories.""" + pairs = [] + seen = set() + currentCategory = None + + for line in vocabText.splitlines(): + line = line.strip() + if not line or line.startswith('```'): + continue + + # Check if this is a category header + if line.startswith('#'): + currentCategory = line + continue + + # Parse vocabulary term + m = re.match(r'^(.+?)(?:\s?[\(–])', line) # term is everything before space + '(' or '–' + if m: + term = m.group(1) + if term not in seen: + pairs.append((term, line, currentCategory)) + seen.add(term) + + return pairs + + +def buildMatchedVocabText(vocabPairs, subbedT): + """Build formatted vocabulary text with matched terms organized by category.""" + matchedCategories = {} + + # Use word boundaries for Japanese if appropriate, or allow substring as before. + for term, line, category in vocabPairs: + # "term in subbedT" could be false positive; can use regex but Japanese doesn't always have spaces. + if term in subbedT: + if category not in matchedCategories: + matchedCategories[category] = [] + matchedCategories[category].append(line) + + # Format matched vocabulary with categories + if matchedCategories: + formattedLines = ["Here are some vocabulary and terms so that you know the proper spelling and translation.\n"] + for category, lines in matchedCategories.items(): + if category: # Only add category header if it exists + formattedLines.append(category) + formattedLines.extend(lines) + formattedLines.append("") # Add blank line between categories + matchedVocabText = f"```\n{chr(10).join(formattedLines).rstrip()}\n```" + else: + matchedVocabText = "" + + return matchedVocabText + + def createContext(fullPromptFlag, subbedT, format): - system = ( - PROMPT + VOCAB - if fullPromptFlag - else f"\ + vocabPairs = parseVocabWithCategories(VOCAB) + matchedVocabText = buildMatchedVocabText(vocabPairs, subbedT) + + if fullPromptFlag: + system = PROMPT + matchedVocabText + else: + system = f"\ You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\ - You translate everything, including erotic and explicit content.\n\ @@ -2474,9 +2530,8 @@ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{ - Maintain any spacing in the translation.\n\ - Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\ - `...` can be a part of the dialogue. Translate it as it is.\n\ -{VOCAB}\n\ +{matchedVocabText}\n\ " - ) if format == "json": user = f"```json\n{subbedT}\n```" else: diff --git a/modules/rpgmakermvmz.py b/modules/rpgmakermvmz.py index adc7efe..faf09f5 100644 --- a/modules/rpgmakermvmz.py +++ b/modules/rpgmakermvmz.py @@ -2433,7 +2433,6 @@ def searchSystem(data, pbar): return totalTokens - # Save some money and enter the character before translation def getSpeaker(speaker): match speaker: diff --git a/modules/rpgmakerplugin.py b/modules/rpgmakerplugin.py index 75b2314..c9e502c 100644 --- a/modules/rpgmakerplugin.py +++ b/modules/rpgmakerplugin.py @@ -586,7 +586,6 @@ def translatePlugin(data, pbar, filename, translatedList): translatePlugin(data, pbar, filename, [questListTL, customTL]) return tokens - # Save some money and enter the character before translation def getSpeaker(speaker): match speaker: @@ -604,7 +603,7 @@ def getSpeaker(speaker): response = translateGPT( f"{speaker}", "Reply with the " + LANGUAGE + " translation of the NPC name.", - True, + False, ) response[0] = response[0].title() response[0] = response[0].replace("'S", "'s") @@ -633,11 +632,68 @@ def batchList(input_list, batch_size): return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)] +def parseVocabWithCategories(vocabText): + """Parse vocabulary text and extract terms with their categories.""" + pairs = [] + seen = set() + currentCategory = None + + for line in vocabText.splitlines(): + line = line.strip() + if not line or line.startswith('```'): + continue + + # Check if this is a category header + if line.startswith('#'): + currentCategory = line + continue + + # Parse vocabulary term + m = re.match(r'^(.+?)(?:\s?[\(–])', line) # term is everything before space + '(' or '–' + if m: + term = m.group(1) + if term not in seen: + pairs.append((term, line, currentCategory)) + seen.add(term) + + return pairs + + +def buildMatchedVocabText(vocabPairs, subbedT): + """Build formatted vocabulary text with matched terms organized by category.""" + matchedCategories = {} + + # Use word boundaries for Japanese if appropriate, or allow substring as before. + for term, line, category in vocabPairs: + # "term in subbedT" could be false positive; can use regex but Japanese doesn't always have spaces. + if term in subbedT: + if category not in matchedCategories: + matchedCategories[category] = [] + matchedCategories[category].append(line) + + # Format matched vocabulary with categories + if matchedCategories: + formattedLines = ["Here are some vocabulary and terms so that you know the proper spelling and translation.\n"] + for category, lines in matchedCategories.items(): + if category: # Only add category header if it exists + formattedLines.append(category) + formattedLines.extend(lines) + formattedLines.append("") # Add blank line between categories + matchedVocabText = f"```\n{chr(10).join(formattedLines).rstrip()}\n```" + else: + matchedVocabText = "" + + return matchedVocabText + + def createContext(fullPromptFlag, subbedT, format): - system = ( - PROMPT + VOCAB - if fullPromptFlag - else f"\ + vocabPairs = parseVocabWithCategories(VOCAB) + matchedVocabText = buildMatchedVocabText(vocabPairs, subbedT) + + if fullPromptFlag: + system = PROMPT + matchedVocabText + else: + system = f"\ You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\ - You translate everything, including erotic and explicit content.\n\ @@ -647,9 +703,8 @@ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{ - Maintain any spacing in the translation.\n\ - Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\ - `...` can be a part of the dialogue. Translate it as it is.\n\ -{VOCAB}\n\ +{matchedVocabText}\n\ " - ) if format == "json": user = f"```json\n{subbedT}\n```" else: @@ -663,9 +718,10 @@ def translateText(system, user, history, penalty, format, model=MODEL): # History if isinstance(history, list): - msg.extend([{"role": "system", "content": h} for h in history]) + msg.append({"role": "system", "content": "Translation History:"}) + msg.extend([{"role": "assistant", "content": h} for h in history]) else: - msg.append({"role": "system", "content": history}) + msg.append({"role": "assistant", "content": history}) # Response Format if format == "json": @@ -700,8 +756,11 @@ def cleanTranslatedText(translatedText): "】": "]", "【": "[", "é": "e", - "ō": "o", + "this guy": "this bastard", + "This guy": "This bastard", "Placeholder Text": "", + "```json": "", + "```": "", # Add more replacements as needed } for target, replacement in placeholders.items(): @@ -763,7 +822,7 @@ def countTokens(system, user, history): inputTotalTokens += len(enc.encode(user)) # Output - outputTotalTokens += round(len(enc.encode(user)) * 3) + outputTotalTokens += round(len(enc.encode(user)) * 2.5) return [inputTotalTokens, outputTotalTokens] @@ -827,7 +886,8 @@ def translateGPT(text, history, fullPromptFlag): # AI Refused, Try Again if not translatedText: - response = translateText(f"{system}\n You translate ALL content.", user, history, 0.1, format) + response = translateText(f"{system}\n You translate ALL content.", user, history, 0.1, format, model="gpt-4o") + translatedText = response.choices[0].message.content # Report Tokens totalTokens[0] += response.usage.prompt_tokens diff --git a/modules/sakuranbo.py b/modules/sakuranbo.py deleted file mode 100644 index f256e75..0000000 --- a/modules/sakuranbo.py +++ /dev/null @@ -1,600 +0,0 @@ -import os -import re -import util.dazedwrap as dazedwrap -import threading -import time -import traceback -from pathlib import Path - -import openai -import tiktoken -from colorama import Fore -from dotenv import load_dotenv -from retry import retry -from tqdm import tqdm - -# Open AI -load_dotenv() -if os.getenv("api").replace(" ", "") != "": - openai.base_url = os.getenv("api") -openai.organization = os.getenv("org") -openai.api_key = os.getenv("key") - -# Globals -MODEL = os.getenv("model") -TIMEOUT = int(os.getenv("timeout")) -LANGUAGE = os.getenv("language").capitalize() -INPUTAPICOST = 0.002 # Depends on the model https://openai.com/pricing -OUTPUTAPICOST = 0.002 -PROMPT = Path("prompt.txt").read_text(encoding="utf-8") -THREADS = int(os.getenv("threads")) # Controls how many threads are working on a single file (May have to drop this) -LOCK = threading.Lock() -WIDTH = int(os.getenv("width")) -LISTWIDTH = int(os.getenv("listWidth")) -NOTEWIDTH = 40 -MAXHISTORY = 10 -ESTIMATE = "" -totalTokens = [0, 0] -NAMESLIST = [] - -# tqdm Globals -BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}" -POSITION = 0 -LEAVE = False - -# Flags -NAMES = False # Output a list of all the character names found -BRFLAG = False # If the game uses
instead -FIXTEXTWRAP = True -IGNORETLTEXT = False - - -def handleSakuranbo(filename, estimate): - global ESTIMATE - totalTokens = [0, 0] - ESTIMATE = estimate - - if estimate: - start = time.time() - translatedData = openFiles(filename) - - # Print Result - end = time.time() - tqdm.write(getResultString(translatedData, end - start, filename)) - if NAMES is True: - tqdm.write(str(NAMESLIST)) - with LOCK: - totalTokens[0] += translatedData[1][0] - totalTokens[1] += translatedData[1][1] - - return getResultString(["", totalTokens, None], end - start, "TOTAL") - - else: - try: - with open("translated/" + filename, "w", encoding="utf-16") as outFile: - start = time.time() - translatedData = openFiles(filename) - outFile.writelines(translatedData[0]) - - # Print Result - end = time.time() - tqdm.write(getResultString(translatedData, end - start, filename)) - with LOCK: - totalTokens[0] += translatedData[1][0] - totalTokens[1] += translatedData[1][1] - except Exception: - traceback.print_exc() - return "Fail" - - return getResultString(["", totalTokens, None], end - start, "TOTAL") - - -def getResultString(translatedData, translationTime, filename): - # File Print String - totalTokenstring = ( - Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]" - "[Output: " - + str(translatedData[1][1]) - + "]" "[Cost: ${:,.4f}".format(((translatedData[1][0] / 1000000) * INPUTAPICOST) + ((translatedData[1][1] / 1000000) * OUTPUTAPICOST)) - + "]" - ) - timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]" - - if translatedData[2] is None: - # Success - return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET - - else: - # Fail - try: - raise translatedData[2] - except Exception as e: - errorString = str(e) + Fore.RED - return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET - - -def openFiles(filename): - with open("files/" + filename, "r", encoding="utf-16") as readFile: - translatedData = parseTyrano(readFile, filename) - - # Delete lines marked for deletion - finalData = [] - for line in translatedData[0]: - if line != "\\d\n": - finalData.append(line) - translatedData[0] = finalData - - return translatedData - - -def parseTyrano(readFile, filename): - totalTokens = [0, 0] - totalLines = 0 - - # Get total for progress bar - data = readFile.readlines() - totalLines = len(data) - - with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar: - pbar.desc = filename - pbar.total = totalLines - - try: - response = translateTyrano(data, pbar) - totalTokens[0] = response[0] - totalTokens[1] = response[1] - except Exception as e: - traceback.print_exc() - return [data, totalTokens, e] - return [data, totalTokens, None] - - -def translateTyrano(data, pbar): - textHistory = [] - maxHistory = MAXHISTORY - tokens = [0, 0] - currentGroup = [] - syncIndex = 0 - speaker = "" - delFlag = False - global LOCK, ESTIMATE - - for i in range(len(data)): - currentGroup = [] - matchList = [] - - if syncIndex > i: - i = syncIndex - - if "[▼]" in data[i]: - data[i] = data[i].replace("[▼]".strip(), "[page]\n") - - # If there isn't any Japanese in the text just skip - if IGNORETLTEXT is True: - if not re.search(r"[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+", data[i]): - # Keep textHistory list at length maxHistory - textHistory.append('"' + data[i] + '"') - if len(textHistory) > maxHistory: - textHistory.pop(0) - currentGroup = [] - continue - - # Speaker - matchList = re.findall(r"^\[(.+)\sstorage=.+\]", data[i]) - if len(matchList) == 0: - matchList = re.findall(r"^\[([^/].+)\]$", data[i]) - if len(matchList) > 0: - if "主人公" in matchList[0]: - speaker = "Protagonist" - elif "思考" in matchList[0]: - speaker = "Protagonist Inner Thoughts" - elif "地の文" in matchList[0]: - speaker = "Narrator" - elif "マコ" in matchList[0]: - speaker = "Mako" - elif "少年" in matchList[0]: - speaker = "Boy" - elif "友達" in matchList[0]: - speaker = "Friend" - elif "少女" in matchList[0]: - speaker = "Girl" - else: - response = translateGPT( - matchList[0], - "Reply with only the " + LANGUAGE + " translation of the NPC name", - True, - ) - speaker = response[0] - tokens[0] += response[1][0] - tokens[1] += response[1][1] - # data[i] = '#' + speaker + '\n' - - # Choices - elif "glink" in data[i]: - matchList = re.findall(r"\[glink.+text=\"(.+?)\".+", data[i]) - if len(matchList) != 0: - if len(textHistory) > 0: - response = translateGPT( - matchList[0], - "Past Translated Text: " + textHistory[len(textHistory) - 1] + "\n\nReply in the style of a dialogue option.", - True, - ) - else: - response = translateGPT(matchList[0], "", False) - translatedText = response[0] - tokens[0] += response[1][0] - tokens[1] += response[1][1] - - # Remove characters that may break scripts - charList = [".", '"', "\\n"] - for char in charList: - translatedText = translatedText.replace(char, "") - - # Escape all ' - translatedText = translatedText.replace("\\", "") - translatedText = translatedText.replace("'", "\\'") - - # Set Data - translatedText = data[i].replace(matchList[0], translatedText.replace(" ", "\u00a0")) - data[i] = translatedText - - # Grab Lines - matchList = re.findall(r"^([^\n;@*\{\[].+[^;'{}\[]$)", data[i]) - if len(matchList) > 0 and ( - re.search(r"^\[(.+)\sstorage=.+\],", data[i - 1]) or re.search(r"^\[(.+)\]$", data[i - 1]) or re.search(r"^《(.+)》", data[i - 1]) - ): - currentGroup.append(matchList[0]) - if len(data) > i + 1: - matchList = re.findall(r"^([^\n;@*\{\[].+[^;'{}\[]$)", data[i + 1]) - while len(matchList) > 0: - delFlag = True - data[i] = "\d\n" # \d Marks line for deletion - i += 1 - matchList = re.findall(r"^([^\n;@*\{\[].+[^;'{}\[]$)", data[i]) - if len(matchList) > 0: - currentGroup.append(matchList[0]) - - # Join up 401 groups for better translation. - if len(currentGroup) > 0: - finalJAString = " ".join(currentGroup) - - # Remove any textwrap - if FIXTEXTWRAP is True: - finalJAString = finalJAString.replace("_", " ") - - # Check Speaker - if speaker == "": - response = translateGPT(finalJAString, textHistory, True) - tokens[0] += response[1][0] - tokens[1] += response[1][1] - translatedText = response[0] - textHistory.append('"' + translatedText + '"') - else: - response = translateGPT(speaker + ": " + finalJAString, textHistory, True) - tokens[0] += response[1][0] - tokens[1] += response[1][1] - translatedText = response[0] - textHistory.append('"' + translatedText + '"') - - # Remove added speaker - translatedText = re.sub(r"^.+:\s?", "", translatedText) - - # Set Data - translatedText = translatedText.replace("ッ", "") - translatedText = translatedText.replace("っ", "") - translatedText = translatedText.replace("ー", "") - translatedText = translatedText.replace('"', "") - translatedText = translatedText.replace("[", "") - translatedText = translatedText.replace("]", "") - - # Wordwrap Text - if "_" not in translatedText: - translatedText = dazedwrap.wrapText(translatedText, width=WIDTH) - translatedText = translatedText.replace("\n", "_") - - # Set - if delFlag is True: - data.insert(i, translatedText.strip() + "\n") - delFlag = False - else: - data[i] = translatedText.strip() + "\n" - - # Keep textHistory list at length maxHistory - if len(textHistory) > maxHistory: - textHistory.pop(0) - currentGroup = [] - speaker = "" - - pbar.update(1) - if len(data) > i + 1: - syncIndex = i + 1 - else: - break - - # Grab Lines - matchList = re.findall(r"(^\[.+\sstorage=.+\](.+)\[/.+\])", data[i]) - if len(matchList) > 0: - originalLine = matchList[0][0] - originalText = matchList[0][1] - currentGroup.append(matchList[0][1]) - if len(data) > i + 1: - matchList = re.findall(r"^([^\n;@*\{\[].+[^;'{}\[]$)", data[i + 1]) - while len(matchList) > 0: - delFlag = True - data[i] = "\d\n" # \d Marks line for deletion - i += 1 - matchList = re.findall(r"^([^\n;@*\{\[].+[^;'{}\[]$)", data[i]) - if len(matchList) > 0: - currentGroup.append(matchList[0]) - - # Join up 401 groups for better translation. - if len(currentGroup) > 0: - finalJAString = " ".join(currentGroup) - - # Remove any textwrap - if FIXTEXTWRAP is True: - finalJAString = finalJAString.replace("_", " ") - - # Check Speaker - if speaker == "": - response = translateGPT(finalJAString, textHistory, True) - tokens[0] += response[1][0] - tokens[1] += response[1][1] - translatedText = response[0] - textHistory.append('"' + translatedText + '"') - else: - response = translateGPT(speaker + ": " + finalJAString, textHistory, True) - tokens[0] += response[1][0] - tokens[1] += response[1][1] - translatedText = response[0] - textHistory.append('"' + translatedText + '"') - - # Remove added speaker - translatedText = re.sub(r"^.+:\s?", "", translatedText) - - # Set Data - translatedText = translatedText.replace("ッ", "") - translatedText = translatedText.replace("っ", "") - translatedText = translatedText.replace("ー", "") - translatedText = translatedText.replace('"', "") - translatedText = translatedText.replace("[", "") - translatedText = translatedText.replace("]", "") - - # Wordwrap Text - if "_" not in translatedText: - translatedText = dazedwrap.wrapText(translatedText, width=WIDTH) - translatedText = translatedText.replace("\n", "_") - translatedText = originalLine.replace(originalText, translatedText) - - # Set - if delFlag is True: - data.insert(i, translatedText.strip() + "\n") - delFlag = False - else: - data[i] = translatedText.strip() + "\n" - - # Keep textHistory list at length maxHistory - if len(textHistory) > maxHistory: - textHistory.pop(0) - currentGroup = [] - speaker = "" - - pbar.update(1) - if len(data) > i + 1: - syncIndex = i + 1 - else: - break - - return tokens - - -def subVars(jaString): - jaString = jaString.replace("\u3000", " ") - - # Nested - count = 0 - nestedList = re.findall(r"[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]", jaString) - nestedList = set(nestedList) - if len(nestedList) != 0: - for icon in nestedList: - jaString = jaString.replace(icon, "{Nested_" + str(count) + "}") - count += 1 - - # Icons - count = 0 - iconList = re.findall(r"[\\]+[iIkKwWaA]+\[[0-9]+\]", jaString) - iconList = set(iconList) - if len(iconList) != 0: - for icon in iconList: - jaString = jaString.replace(icon, "{Ascii_" + str(count) + "}") - count += 1 - - # Colors - count = 0 - colorList = re.findall(r"[\\]+[cC]\[[0-9]+\]", jaString) - colorList = set(colorList) - if len(colorList) != 0: - for color in colorList: - jaString = jaString.replace(color, "{Color_" + str(count) + "}") - count += 1 - - # Names - count = 0 - nameList = re.findall(r"[\\]+[nN]\[.+?\]+", jaString) - nameList = set(nameList) - if len(nameList) != 0: - for name in nameList: - jaString = jaString.replace(name, "{N_" + str(count) + "}") - count += 1 - - # Variables - count = 0 - varList = re.findall(r"[\\]+[vV]\[[0-9]+\]", jaString) - varList = set(varList) - if len(varList) != 0: - for var in varList: - jaString = jaString.replace(var, "{Var_" + str(count) + "}") - count += 1 - - # Formatting - count = 0 - if "笑えるよね." in jaString: - print("t") - formatList = re.findall(r"[\\]+[\w]+\[.+?\]", jaString) - formatList = set(formatList) - if len(formatList) != 0: - for var in formatList: - jaString = jaString.replace(var, "{FCode_" + str(count) + "}") - count += 1 - - # Put all lists in list and return - allList = [nestedList, iconList, colorList, nameList, varList, formatList] - return [jaString, allList] - - -def resubVars(translatedText, allList): - # Fix Spacing and ChatGPT Nonsense - matchList = re.findall(r"\[\s?.+?\s?\]", translatedText) - if len(matchList) > 0: - for match in matchList: - text = match.strip() - translatedText = translatedText.replace(match, text) - - # Nested - count = 0 - if len(allList[0]) != 0: - for var in allList[0]: - translatedText = translatedText.replace("{Nested_" + str(count) + "}", var) - count += 1 - - # Icons - count = 0 - if len(allList[1]) != 0: - for var in allList[1]: - translatedText = translatedText.replace("{Ascii_" + str(count) + "}", var) - count += 1 - - # Colors - count = 0 - if len(allList[2]) != 0: - for var in allList[2]: - translatedText = translatedText.replace("{Color_" + str(count) + "}", var) - count += 1 - - # Names - count = 0 - if len(allList[3]) != 0: - for var in allList[3]: - translatedText = translatedText.replace("{N_" + str(count) + "}", var) - count += 1 - - # Vars - count = 0 - if len(allList[4]) != 0: - for var in allList[4]: - translatedText = translatedText.replace("{Var_" + str(count) + "}", var) - count += 1 - - # Formatting - count = 0 - if len(allList[5]) != 0: - for var in allList[5]: - translatedText = translatedText.replace("{FCode_" + str(count) + "}", var) - count += 1 - - # Remove Color Variables Spaces - # if '\\c' in translatedText: - # translatedText = re.sub(r'\s*(\\+c\[[1-9]+\])\s*', r' \1', translatedText) - # translatedText = re.sub(r'\s*(\\+c\[0+\])', r'\1', translatedText) - return translatedText - - -@retry(exceptions=Exception, tries=5, delay=5) -def translateGPT(t, history, fullPromptFlag): - # Sub Vars - varResponse = subVars(t) - subbedT = varResponse[0] - - # If there isn't any Japanese in the text just skip - if not re.search(r"[一-龠]+|[ぁ-ゔ]+|[ァ-ヴ]+|[\uFF00-\uFFEF]", subbedT): - return (t, [0, 0]) - - # If ESTIMATE is True just count this as an execution and return. - if ESTIMATE: - enc = tiktoken.encoding_for_model("gpt-4") - historyRaw = "" - if isinstance(history, list): - for line in history: - historyRaw += line - else: - historyRaw = history - - inputTotalTokens = len(enc.encode(historyRaw)) + len(enc.encode(PROMPT)) - outputTotalTokens = len(enc.encode(t)) * 2 # Estimating 2x the size of the original text - totalTokens = [inputTotalTokens, outputTotalTokens] - return (t, totalTokens) - - # Characters - context = "Game Characters:\ - Character: マコ == Mako - Gender: Female\ - Character: 主人公 == Protagonist - Gender: Male" - - # Prompt - if fullPromptFlag: - system = PROMPT - user = "Line to Translate = " + subbedT - else: - system = "Output ONLY the " + LANGUAGE + " translation in the following format: `Translation: <" + LANGUAGE.upper() + "_TRANSLATION>`" - user = "Line to Translate = " + subbedT - - # Create Message List - msg = [] - msg.append({"role": "system", "content": system}) - msg.append({"role": "user", "content": context}) - if isinstance(history, list): - for line in history: - msg.append({"role": "user", "content": line}) - else: - msg.append({"role": "user", "content": history}) - msg.append({"role": "user", "content": user}) - - response = openai.ChatCompletion.create( - temperature=0, - frequency_penalty=0.2, - presence_penalty=0.2, - model=MODEL, - messages=msg, - request_timeout=TIMEOUT, - ) - - # Save Translated Text - translatedText = response.choices[0].message.content - totalTokens = [response.usage.prompt_tokens, response.usage.completion_tokens] - - # Resub Vars - translatedText = resubVars(translatedText, varResponse[1]) - - # Remove Placeholder Text - translatedText = translatedText.replace(LANGUAGE + " Translation: ", "") - translatedText = translatedText.replace("Translation: ", "") - translatedText = translatedText.replace("Line to Translate = ", "") - translatedText = translatedText.replace("Translation = ", "") - translatedText = translatedText.replace("Translate = ", "") - translatedText = translatedText.replace(LANGUAGE + " Translation:", "") - translatedText = translatedText.replace("Translation:", "") - translatedText = translatedText.replace("Line to Translate =", "") - translatedText = translatedText.replace("Translation =", "") - translatedText = translatedText.replace("Translate =", "") - translatedText = translatedText.replace("っ", "") - translatedText = translatedText.replace("ッ", "") - translatedText = translatedText.replace("ぁ", "") - translatedText = translatedText.replace("。", ".") - translatedText = translatedText.replace("、", ",") - translatedText = translatedText.replace("？", "?") - translatedText = translatedText.replace("！", "!") - - # Return Translation - if len(translatedText) > 15 * len(t) or "I'm sorry, but I'm unable to assist with that translation" in translatedText: - raise Exception - else: - return [translatedText, totalTokens] diff --git a/modules/text.py b/modules/text.py index 0d00965..a207691 100644 --- a/modules/text.py +++ b/modules/text.py @@ -269,7 +269,6 @@ def translateTxt(data, translatedList): translateTxt(data, [stringListTL, choiceListTL]) return tokens - # Save some money and enter the character before translation def getSpeaker(speaker): match speaker: @@ -287,7 +286,7 @@ def getSpeaker(speaker): response = translateGPT( f"{speaker}", "Reply with the " + LANGUAGE + " translation of the NPC name.", - True, + False, ) response[0] = response[0].title() response[0] = response[0].replace("'S", "'s") @@ -316,11 +315,68 @@ def batchList(input_list, batch_size): return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)] +def parseVocabWithCategories(vocabText): + """Parse vocabulary text and extract terms with their categories.""" + pairs = [] + seen = set() + currentCategory = None + + for line in vocabText.splitlines(): + line = line.strip() + if not line or line.startswith('```'): + continue + + # Check if this is a category header + if line.startswith('#'): + currentCategory = line + continue + + # Parse vocabulary term + m = re.match(r'^(.+?)(?:\s?[\(–])', line) # term is everything before space + '(' or '–' + if m: + term = m.group(1) + if term not in seen: + pairs.append((term, line, currentCategory)) + seen.add(term) + + return pairs + + +def buildMatchedVocabText(vocabPairs, subbedT): + """Build formatted vocabulary text with matched terms organized by category.""" + matchedCategories = {} + + # Use word boundaries for Japanese if appropriate, or allow substring as before. + for term, line, category in vocabPairs: + # "term in subbedT" could be false positive; can use regex but Japanese doesn't always have spaces. + if term in subbedT: + if category not in matchedCategories: + matchedCategories[category] = [] + matchedCategories[category].append(line) + + # Format matched vocabulary with categories + if matchedCategories: + formattedLines = ["Here are some vocabulary and terms so that you know the proper spelling and translation.\n"] + for category, lines in matchedCategories.items(): + if category: # Only add category header if it exists + formattedLines.append(category) + formattedLines.extend(lines) + formattedLines.append("") # Add blank line between categories + matchedVocabText = f"```\n{chr(10).join(formattedLines).rstrip()}\n```" + else: + matchedVocabText = "" + + return matchedVocabText + + def createContext(fullPromptFlag, subbedT, format): - system = ( - PROMPT + VOCAB - if fullPromptFlag - else f"\ + vocabPairs = parseVocabWithCategories(VOCAB) + matchedVocabText = buildMatchedVocabText(vocabPairs, subbedT) + + if fullPromptFlag: + system = PROMPT + matchedVocabText + else: + system = f"\ You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\ - You translate everything, including erotic and explicit content.\n\ @@ -330,9 +386,8 @@ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{ - Maintain any spacing in the translation.\n\ - Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\ - `...` can be a part of the dialogue. Translate it as it is.\n\ -{VOCAB}\n\ +{matchedVocabText}\n\ " - ) if format == "json": user = f"```json\n{subbedT}\n```" else: @@ -346,9 +401,10 @@ def translateText(system, user, history, penalty, format, model=MODEL): # History if isinstance(history, list): - msg.extend([{"role": "system", "content": h} for h in history]) + msg.append({"role": "system", "content": "Translation History:"}) + msg.extend([{"role": "assistant", "content": h} for h in history]) else: - msg.append({"role": "system", "content": history}) + msg.append({"role": "assistant", "content": history}) # Response Format if format == "json": @@ -383,8 +439,11 @@ def cleanTranslatedText(translatedText): "】": "]", "【": "[", "é": "e", - "ō": "o", + "this guy": "this bastard", + "This guy": "This bastard", "Placeholder Text": "", + "```json": "", + "```": "", # Add more replacements as needed } for target, replacement in placeholders.items(): @@ -446,7 +505,7 @@ def countTokens(system, user, history): inputTotalTokens += len(enc.encode(user)) # Output - outputTotalTokens += round(len(enc.encode(user)) * 3) + outputTotalTokens += round(len(enc.encode(user)) * 2.5) return [inputTotalTokens, outputTotalTokens] @@ -510,7 +569,8 @@ def translateGPT(text, history, fullPromptFlag): # AI Refused, Try Again if not translatedText: - response = translateText(f"{system}\n You translate ALL content.", user, history, 0.1, format) + response = translateText(f"{system}\n You translate ALL content.", user, history, 0.1, format, model="gpt-4o") + translatedText = response.choices[0].message.content # Report Tokens totalTokens[0] += response.usage.prompt_tokens diff --git a/modules/tyrano.py b/modules/tyrano.py index 81873fa..7875626 100644 --- a/modules/tyrano.py +++ b/modules/tyrano.py @@ -356,7 +356,6 @@ def translateTyrano(data, translatedList): translateTyrano(data, [stringListTL, choiceListTL]) return tokens - # Save some money and enter the character before translation def getSpeaker(speaker): match speaker: @@ -403,11 +402,68 @@ def batchList(input_list, batch_size): return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)] +def parseVocabWithCategories(vocabText): + """Parse vocabulary text and extract terms with their categories.""" + pairs = [] + seen = set() + currentCategory = None + + for line in vocabText.splitlines(): + line = line.strip() + if not line or line.startswith('```'): + continue + + # Check if this is a category header + if line.startswith('#'): + currentCategory = line + continue + + # Parse vocabulary term + m = re.match(r'^(.+?)(?:\s?[\(–])', line) # term is everything before space + '(' or '–' + if m: + term = m.group(1) + if term not in seen: + pairs.append((term, line, currentCategory)) + seen.add(term) + + return pairs + + +def buildMatchedVocabText(vocabPairs, subbedT): + """Build formatted vocabulary text with matched terms organized by category.""" + matchedCategories = {} + + # Use word boundaries for Japanese if appropriate, or allow substring as before. + for term, line, category in vocabPairs: + # "term in subbedT" could be false positive; can use regex but Japanese doesn't always have spaces. + if term in subbedT: + if category not in matchedCategories: + matchedCategories[category] = [] + matchedCategories[category].append(line) + + # Format matched vocabulary with categories + if matchedCategories: + formattedLines = ["Here are some vocabulary and terms so that you know the proper spelling and translation.\n"] + for category, lines in matchedCategories.items(): + if category: # Only add category header if it exists + formattedLines.append(category) + formattedLines.extend(lines) + formattedLines.append("") # Add blank line between categories + matchedVocabText = f"```\n{chr(10).join(formattedLines).rstrip()}\n```" + else: + matchedVocabText = "" + + return matchedVocabText + + def createContext(fullPromptFlag, subbedT, format): - system = ( - PROMPT + VOCAB - if fullPromptFlag - else f"\ + vocabPairs = parseVocabWithCategories(VOCAB) + matchedVocabText = buildMatchedVocabText(vocabPairs, subbedT) + + if fullPromptFlag: + system = PROMPT + matchedVocabText + else: + system = f"\ You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\ - You translate everything, including erotic and explicit content.\n\ @@ -417,9 +473,8 @@ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{ - Maintain any spacing in the translation.\n\ - Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\ - `...` can be a part of the dialogue. Translate it as it is.\n\ -{VOCAB}\n\ +{matchedVocabText}\n\ " - ) if format == "json": user = f"```json\n{subbedT}\n```" else: @@ -433,7 +488,7 @@ def translateText(system, user, history, penalty, format, model=MODEL): # History if isinstance(history, list): - msg.append({"role": "assistant", "content": "Translation History:"}) + msg.append({"role": "system", "content": "Translation History:"}) msg.extend([{"role": "assistant", "content": h} for h in history]) else: msg.append({"role": "assistant", "content": history}) @@ -474,6 +529,8 @@ def cleanTranslatedText(translatedText): "this guy": "this bastard", "This guy": "This bastard", "Placeholder Text": "", + "```json": "", + "```": "", # Add more replacements as needed } for target, replacement in placeholders.items(): @@ -535,7 +592,7 @@ def countTokens(system, user, history): inputTotalTokens += len(enc.encode(user)) # Output - outputTotalTokens += round(len(enc.encode(user)) * 3) + outputTotalTokens += round(len(enc.encode(user)) * 2.5) return [inputTotalTokens, outputTotalTokens] @@ -599,7 +656,8 @@ def translateGPT(text, history, fullPromptFlag): # AI Refused, Try Again if not translatedText: - response = translateText(f"{system}\n You translate ALL content.", user, history, 0.1, format) + response = translateText(f"{system}\n You translate ALL content.", user, history, 0.1, format, model="gpt-4o") + translatedText = response.choices[0].message.content # Report Tokens totalTokens[0] += response.usage.prompt_tokens diff --git a/modules/unity.py b/modules/unity.py index 6c1607b..9b37274 100644 --- a/modules/unity.py +++ b/modules/unity.py @@ -269,7 +269,6 @@ def translateUnity(data, pbar, filename, translatedList): MISMATCH.append(filename) return tokens - # Save some money and enter the character before translation def getSpeaker(speaker): match speaker: @@ -316,11 +315,68 @@ def batchList(input_list, batch_size): return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)] +def parseVocabWithCategories(vocabText): + """Parse vocabulary text and extract terms with their categories.""" + pairs = [] + seen = set() + currentCategory = None + + for line in vocabText.splitlines(): + line = line.strip() + if not line or line.startswith('```'): + continue + + # Check if this is a category header + if line.startswith('#'): + currentCategory = line + continue + + # Parse vocabulary term + m = re.match(r'^(.+?)(?:\s?[\(–])', line) # term is everything before space + '(' or '–' + if m: + term = m.group(1) + if term not in seen: + pairs.append((term, line, currentCategory)) + seen.add(term) + + return pairs + + +def buildMatchedVocabText(vocabPairs, subbedT): + """Build formatted vocabulary text with matched terms organized by category.""" + matchedCategories = {} + + # Use word boundaries for Japanese if appropriate, or allow substring as before. + for term, line, category in vocabPairs: + # "term in subbedT" could be false positive; can use regex but Japanese doesn't always have spaces. + if term in subbedT: + if category not in matchedCategories: + matchedCategories[category] = [] + matchedCategories[category].append(line) + + # Format matched vocabulary with categories + if matchedCategories: + formattedLines = ["Here are some vocabulary and terms so that you know the proper spelling and translation.\n"] + for category, lines in matchedCategories.items(): + if category: # Only add category header if it exists + formattedLines.append(category) + formattedLines.extend(lines) + formattedLines.append("") # Add blank line between categories + matchedVocabText = f"```\n{chr(10).join(formattedLines).rstrip()}\n```" + else: + matchedVocabText = "" + + return matchedVocabText + + def createContext(fullPromptFlag, subbedT, format): - system = ( - PROMPT + VOCAB - if fullPromptFlag - else f"\ + vocabPairs = parseVocabWithCategories(VOCAB) + matchedVocabText = buildMatchedVocabText(vocabPairs, subbedT) + + if fullPromptFlag: + system = PROMPT + matchedVocabText + else: + system = f"\ You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\ - You translate everything, including erotic and explicit content.\n\ @@ -330,9 +386,8 @@ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{ - Maintain any spacing in the translation.\n\ - Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\ - `...` can be a part of the dialogue. Translate it as it is.\n\ -{VOCAB}\n\ +{matchedVocabText}\n\ " - ) if format == "json": user = f"```json\n{subbedT}\n```" else: @@ -387,6 +442,8 @@ def cleanTranslatedText(translatedText): "this guy": "this bastard", "This guy": "This bastard", "Placeholder Text": "", + "```json": "", + "```": "", # Add more replacements as needed } for target, replacement in placeholders.items(): @@ -448,7 +505,7 @@ def countTokens(system, user, history): inputTotalTokens += len(enc.encode(user)) # Output - outputTotalTokens += round(len(enc.encode(user)) * 3) + outputTotalTokens += round(len(enc.encode(user)) * 2.5) return [inputTotalTokens, outputTotalTokens] @@ -512,7 +569,8 @@ def translateGPT(text, history, fullPromptFlag): # AI Refused, Try Again if not translatedText: - response = translateText(f"{system}\n You translate ALL content.", user, history, 0.1, format) + response = translateText(f"{system}\n You translate ALL content.", user, history, 0.1, format, model="gpt-4o") + translatedText = response.choices[0].message.content # Report Tokens totalTokens[0] += response.usage.prompt_tokens diff --git a/modules/wolf.py b/modules/wolf.py index b313e1e..27ee20f 100644 --- a/modules/wolf.py +++ b/modules/wolf.py @@ -2563,7 +2563,6 @@ def searchDB(events, pbar, jobList, filename): return totalTokens - # Save some money and enter the character before translation def getSpeaker(speaker): match speaker: @@ -2581,7 +2580,7 @@ def getSpeaker(speaker): response = translateGPT( f"{speaker}", "Reply with the " + LANGUAGE + " translation of the NPC name.", - True, + False, ) response[0] = response[0].title() response[0] = response[0].replace("'S", "'s") @@ -2610,11 +2609,68 @@ def batchList(input_list, batch_size): return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)] +def parseVocabWithCategories(vocabText): + """Parse vocabulary text and extract terms with their categories.""" + pairs = [] + seen = set() + currentCategory = None + + for line in vocabText.splitlines(): + line = line.strip() + if not line or line.startswith('```'): + continue + + # Check if this is a category header + if line.startswith('#'): + currentCategory = line + continue + + # Parse vocabulary term + m = re.match(r'^(.+?)(?:\s?[\(–])', line) # term is everything before space + '(' or '–' + if m: + term = m.group(1) + if term not in seen: + pairs.append((term, line, currentCategory)) + seen.add(term) + + return pairs + + +def buildMatchedVocabText(vocabPairs, subbedT): + """Build formatted vocabulary text with matched terms organized by category.""" + matchedCategories = {} + + # Use word boundaries for Japanese if appropriate, or allow substring as before. + for term, line, category in vocabPairs: + # "term in subbedT" could be false positive; can use regex but Japanese doesn't always have spaces. + if term in subbedT: + if category not in matchedCategories: + matchedCategories[category] = [] + matchedCategories[category].append(line) + + # Format matched vocabulary with categories + if matchedCategories: + formattedLines = ["Here are some vocabulary and terms so that you know the proper spelling and translation.\n"] + for category, lines in matchedCategories.items(): + if category: # Only add category header if it exists + formattedLines.append(category) + formattedLines.extend(lines) + formattedLines.append("") # Add blank line between categories + matchedVocabText = f"```\n{chr(10).join(formattedLines).rstrip()}\n```" + else: + matchedVocabText = "" + + return matchedVocabText + + def createContext(fullPromptFlag, subbedT, format): - system = ( - PROMPT + VOCAB - if fullPromptFlag - else f"\ + vocabPairs = parseVocabWithCategories(VOCAB) + matchedVocabText = buildMatchedVocabText(vocabPairs, subbedT) + + if fullPromptFlag: + system = PROMPT + matchedVocabText + else: + system = f"\ You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\ - You translate everything, including erotic and explicit content.\n\ @@ -2624,9 +2680,8 @@ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{ - Maintain any spacing in the translation.\n\ - Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\ - `...` can be a part of the dialogue. Translate it as it is.\n\ -{VOCAB}\n\ +{matchedVocabText}\n\ " - ) if format == "json": user = f"```json\n{subbedT}\n```" else: @@ -2640,9 +2695,10 @@ def translateText(system, user, history, penalty, format, model=MODEL): # History if isinstance(history, list): - msg.extend([{"role": "system", "content": h} for h in history]) + msg.append({"role": "system", "content": "Translation History:"}) + msg.extend([{"role": "assistant", "content": h} for h in history]) else: - msg.append({"role": "system", "content": history}) + msg.append({"role": "assistant", "content": history}) # Response Format if format == "json": @@ -2663,36 +2719,36 @@ def translateText(system, user, history, penalty, format, model=MODEL): def cleanTranslatedText(translatedText): - if translatedText: - placeholders = { - f"{LANGUAGE} Translation: ": "", - "Translation: ": "", - "っ": "", - "〜": "~", - "ッ": "", - "。": ".", - "「": '\\"', - "」": '\\"', - "- ": "-", - "—": "―", - "】": "]", - "【": "[", - "é": "e", - "ō": "o", - "Placeholder Text": "", - # Add more replacements as needed - } - for target, replacement in placeholders.items(): - translatedText = translatedText.replace(target, replacement) + placeholders = { + f"{LANGUAGE} Translation: ": "", + "Translation: ": "", + "っ": "", + "〜": "~", + "ッ": "", + "。": ".", + "「": '\\"', + "」": '\\"', + "- ": "-", + "—": "―", + "】": "]", + "【": "[", + "é": "e", + "this guy": "this bastard", + "This guy": "This bastard", + "Placeholder Text": "", + "```json": "", + "```": "", + # Add more replacements as needed + } + for target, replacement in placeholders.items(): + translatedText = translatedText.replace(target, replacement) - # Remove Repeating Characters - pattern = re.compile(r"(.)\s*\1(?:\s*\1){" + str(20 - 1) + r",}") - translatedText = pattern.sub(lambda match: match.group(0).replace(" ", "")[:20], translatedText) + # Remove Repeating Characters + pattern = re.compile(r"(.)\s*\1(?:\s*\1){" + str(20 - 1) + r",}") + translatedText = pattern.sub(lambda match: match.group(0).replace(" ", "")[:20], translatedText) - # Elongate Long Dashes (Since GPT Ignores them...) - translatedText = elongateCharacters(translatedText) - else: - print(translatedText) + # Elongate Long Dashes (Since GPT Ignores them...) + translatedText = elongateCharacters(translatedText) return translatedText @@ -2743,7 +2799,7 @@ def countTokens(system, user, history): inputTotalTokens += len(enc.encode(user)) # Output - outputTotalTokens += round(len(enc.encode(user)) * 3) + outputTotalTokens += round(len(enc.encode(user)) * 2.5) return [inputTotalTokens, outputTotalTokens] @@ -2800,14 +2856,15 @@ def translateGPT(text, history, fullPromptFlag): continue # Translating - response = translateText(system, user, history, 0.05, format, model="gpt-4o") + response = translateText(system, user, history, 0.05, format) # Set Tokens translatedText = response.choices[0].message.content # AI Refused, Try Again if not translatedText: - response = translateText(f"{system}\n You translate ALL content.", user, history, 0.1, format) + response = translateText(f"{system}\n You translate ALL content.", user, history, 0.1, format, model="gpt-4o") + translatedText = response.choices[0].message.content # Report Tokens totalTokens[0] += response.usage.prompt_tokens diff --git a/modules/wolf2.py b/modules/wolf2.py index cd046ee..2d66eb7 100644 --- a/modules/wolf2.py +++ b/modules/wolf2.py @@ -310,7 +310,6 @@ def translateWOLF(data, translatedList, pbar, filename): MISMATCH.append(filename) return tokens - # Save some money and enter the character before translation def getSpeaker(speaker): match speaker: @@ -328,7 +327,7 @@ def getSpeaker(speaker): response = translateGPT( f"{speaker}", "Reply with the " + LANGUAGE + " translation of the NPC name.", - True, + False, ) response[0] = response[0].title() response[0] = response[0].replace("'S", "'s") @@ -357,11 +356,68 @@ def batchList(input_list, batch_size): return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)] +def parseVocabWithCategories(vocabText): + """Parse vocabulary text and extract terms with their categories.""" + pairs = [] + seen = set() + currentCategory = None + + for line in vocabText.splitlines(): + line = line.strip() + if not line or line.startswith('```'): + continue + + # Check if this is a category header + if line.startswith('#'): + currentCategory = line + continue + + # Parse vocabulary term + m = re.match(r'^(.+?)(?:\s?[\(–])', line) # term is everything before space + '(' or '–' + if m: + term = m.group(1) + if term not in seen: + pairs.append((term, line, currentCategory)) + seen.add(term) + + return pairs + + +def buildMatchedVocabText(vocabPairs, subbedT): + """Build formatted vocabulary text with matched terms organized by category.""" + matchedCategories = {} + + # Use word boundaries for Japanese if appropriate, or allow substring as before. + for term, line, category in vocabPairs: + # "term in subbedT" could be false positive; can use regex but Japanese doesn't always have spaces. + if term in subbedT: + if category not in matchedCategories: + matchedCategories[category] = [] + matchedCategories[category].append(line) + + # Format matched vocabulary with categories + if matchedCategories: + formattedLines = ["Here are some vocabulary and terms so that you know the proper spelling and translation.\n"] + for category, lines in matchedCategories.items(): + if category: # Only add category header if it exists + formattedLines.append(category) + formattedLines.extend(lines) + formattedLines.append("") # Add blank line between categories + matchedVocabText = f"```\n{chr(10).join(formattedLines).rstrip()}\n```" + else: + matchedVocabText = "" + + return matchedVocabText + + def createContext(fullPromptFlag, subbedT, format): - system = ( - PROMPT + VOCAB - if fullPromptFlag - else f"\ + vocabPairs = parseVocabWithCategories(VOCAB) + matchedVocabText = buildMatchedVocabText(vocabPairs, subbedT) + + if fullPromptFlag: + system = PROMPT + matchedVocabText + else: + system = f"\ You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\ - You translate everything, including erotic and explicit content.\n\ @@ -371,9 +427,8 @@ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{ - Maintain any spacing in the translation.\n\ - Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\ - `...` can be a part of the dialogue. Translate it as it is.\n\ -{VOCAB}\n\ +{matchedVocabText}\n\ " - ) if format == "json": user = f"```json\n{subbedT}\n```" else: @@ -387,9 +442,10 @@ def translateText(system, user, history, penalty, format, model=MODEL): # History if isinstance(history, list): - msg.extend([{"role": "system", "content": h} for h in history]) + msg.append({"role": "system", "content": "Translation History:"}) + msg.extend([{"role": "assistant", "content": h} for h in history]) else: - msg.append({"role": "system", "content": history}) + msg.append({"role": "assistant", "content": history}) # Response Format if format == "json": @@ -424,8 +480,11 @@ def cleanTranslatedText(translatedText): "】": "]", "【": "[", "é": "e", - "ō": "o", + "this guy": "this bastard", + "This guy": "This bastard", "Placeholder Text": "", + "```json": "", + "```": "", # Add more replacements as needed } for target, replacement in placeholders.items(): @@ -487,7 +546,7 @@ def countTokens(system, user, history): inputTotalTokens += len(enc.encode(user)) # Output - outputTotalTokens += round(len(enc.encode(user)) * 3) + outputTotalTokens += round(len(enc.encode(user)) * 2.5) return [inputTotalTokens, outputTotalTokens] @@ -551,7 +610,8 @@ def translateGPT(text, history, fullPromptFlag): # AI Refused, Try Again if not translatedText: - response = translateText(f"{system}\n You translate ALL content.", user, history, 0.1, format) + response = translateText(f"{system}\n You translate ALL content.", user, history, 0.1, format, model="gpt-4o") + translatedText = response.choices[0].message.content # Report Tokens totalTokens[0] += response.usage.prompt_tokens