From 3fb50fbbf398d9443f05e0ca352c6e8e14381794 Mon Sep 17 00:00:00 2001 From: DazedAnon Date: Thu, 3 Oct 2024 10:57:56 -0500 Subject: [PATCH] Update Lune Translation function --- modules/lune.py | 337 ++++++++++++++++++++-------------------- modules/rpgmakermvmz.py | 4 +- modules/wolf.py | 28 ++-- vocab.txt | 14 +- 4 files changed, 192 insertions(+), 191 deletions(-) diff --git a/modules/lune.py b/modules/lune.py index ede0654..e28a4e6 100644 --- a/modules/lune.py +++ b/modules/lune.py @@ -41,6 +41,8 @@ BRFLAG = False # If the game uses
instead FIXTEXTWRAP = True # Overwrites textwrap IGNORETLTEXT = False # Ignores all translated text. MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong) +PBAR = None +FILENAME = None # tqdm Globals BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}" @@ -61,8 +63,9 @@ elif "gpt-4" in MODEL: def handleLune(filename, estimate): - global ESTIMATE, totalTokens + global FILENAME, ESTIMATE, totalTokens ESTIMATE = estimate + FILENAME = filename if estimate: start = time.time() @@ -176,6 +179,8 @@ def parseJSON(data, filename): def translateJSON(data, pbar): + global PBAR + PBAR = pbar textHistory = [] batch = [] maxHistory = MAXHISTORY @@ -306,85 +311,60 @@ def translateJSON(data, pbar): # Save some money and enter the character before translation def getSpeaker(speaker): match speaker: - case "セレナ": - return ["Serena", [0, 0]] - case "レナ": - return ["Rena", [0, 0]] - case "フィルス": - return ["Phils", [0, 0]] - case "レイン": - return ["Meryl", [0, 0]] + case "ファイン": + return ["Fine", [0, 0]] + case "": + return ["", [0, 0]] case _: - return translateGPT( - speaker, - "Reply with only the " + LANGUAGE + " translation of the NPC name.", - False, - ) + # Store Speaker + if speaker not in str(NAMESLIST): + response = translateGPT( + speaker, + "Reply with the " + LANGUAGE + " translation of the NPC name.", + False, + ) + response[0] = response[0].title() + response[0] = response[0].replace("'S", "'s") + + # Retry if name doesn't translate for some reason + if re.search(r"([a-zA-Z？?])", response[0]) == None: + response = translateGPT( + speaker, + "Reply with the " + LANGUAGE + " translation of the NPC name.", + False, + ) + response[0] = response[0].title() + response[0] = response[0].replace("'S", "'s") + + speakerList = [speaker, response[0]] + NAMESLIST.append(speakerList) + return response + # Find Speaker + else: + for i in range(len(NAMESLIST)): + if speaker == NAMESLIST[i][0]: + return [NAMESLIST[i][1], [0, 0]] + + return [speaker, [0, 0]] def subVars(jaString): jaString = jaString.replace("\u3000", " ") - # Nested - count = 0 - nestedList = re.findall(r"[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]", jaString) - nestedList = set(nestedList) - if len(nestedList) != 0: - for icon in nestedList: - jaString = jaString.replace(icon, "{Nested_" + str(count) + "}") - count += 1 - - # Icons - count = 0 - iconList = re.findall(r"[\\]+[iIkKwWaA]+\[[0-9]+\]", jaString) - iconList = set(iconList) - if len(iconList) != 0: - for icon in iconList: - jaString = jaString.replace(icon, "{Ascii_" + str(count) + "}") - count += 1 - - # Colors - count = 0 - colorList = re.findall(r"[\\]+[cC]\[[0-9]+\]", jaString) - colorList = set(colorList) - if len(colorList) != 0: - for color in colorList: - jaString = jaString.replace(color, "{Color_" + str(count) + "}") - count += 1 - - # Names - count = 0 - nameList = re.findall(r"[\\]+[nN]\[.+?\]+", jaString) - nameList = set(nameList) - if len(nameList) != 0: - for name in nameList: - jaString = jaString.replace(name, "{Noun_" + str(count) + "}") - count += 1 - - # Variables - count = 0 - varList = re.findall(r"[\\]+[vV]\[[0-9]+\]", jaString) - varList = set(varList) - if len(varList) != 0: - for var in varList: - jaString = jaString.replace(var, "{Var_" + str(count) + "}") - count += 1 - # Formatting count = 0 - formatList = re.findall(r"[\\]+[\w]+\[.+?\]", jaString) - formatList = set(formatList) - if len(formatList) != 0: - for var in formatList: - jaString = jaString.replace(var, "{FCode_" + str(count) + "}") + codeList = re.findall(r"[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]", jaString) + codeList = set(codeList) + if len(codeList) != 0: + for var in codeList: + jaString = jaString.replace(var, "[FCode_" + str(count) + "]") count += 1 # Put all lists in list and return - allList = [nestedList, iconList, colorList, nameList, varList, formatList] - return [jaString, allList] + return [jaString, codeList] -def resubVars(translatedText, allList): +def resubVars(translatedText, codeList): # Fix Spacing and ChatGPT Nonsense matchList = re.findall(r"\[\s?.+?\s?\]", translatedText) if len(matchList) > 0: @@ -392,46 +372,11 @@ def resubVars(translatedText, allList): text = match.strip() translatedText = translatedText.replace(match, text) - # Nested - count = 0 - if len(allList[0]) != 0: - for var in allList[0]: - translatedText = translatedText.replace("{Nested_" + str(count) + "}", var) - count += 1 - - # Icons - count = 0 - if len(allList[1]) != 0: - for var in allList[1]: - translatedText = translatedText.replace("{Ascii_" + str(count) + "}", var) - count += 1 - - # Colors - count = 0 - if len(allList[2]) != 0: - for var in allList[2]: - translatedText = translatedText.replace("{Color_" + str(count) + "}", var) - count += 1 - - # Names - count = 0 - if len(allList[3]) != 0: - for var in allList[3]: - translatedText = translatedText.replace("{Noun_" + str(count) + "}", var) - count += 1 - - # Vars - count = 0 - if len(allList[4]) != 0: - for var in allList[4]: - translatedText = translatedText.replace("{Var_" + str(count) + "}", var) - count += 1 - # Formatting count = 0 - if len(allList[5]) != 0: - for var in allList[5]: - translatedText = translatedText.replace("{FCode_" + str(count) + "}", var) + if len(codeList) != 0: + for var in codeList: + translatedText = translatedText.replace("[FCode_" + str(count) + "]", var) count += 1 return translatedText @@ -446,50 +391,53 @@ def batchList(input_list, batch_size): ] -def createContext(fullPromptFlag, subbedT): - characters = "Game Characters:\n\ -林つかさ (Tsukasa Hayashi) - Female\n\ -山田美兎 (Miyato Yamada) - Female\n\ -鈴木赤音 (Akane Suzuki) - Female\n\ -佐藤莉伊南 (Riina Satou) - Female\n\ -佐々木万梨美 (Marimi Sasaki) - Female\n\ -渡辺登樹子 (Tokiko Watanabe) - Female\n\ -桃乃夢 (Yume Momono) - Female\n\ -吉浦美雪 (Miyuki Yoshiura) - Female\n\ -三ツ門まあな (Maana Mitsukado) - Female\n\ -モリー・ボイド (Molly Boyd) - Female\n\ -オルガ・ブヤチッチ (Olga Buyachich) - Female\n\ -アッチャラー ギッティ (Atchara Gitti) - Female\n\ -" - +def createContext(fullPromptFlag, subbedT, format): system = ( - PROMPT + PROMPT + VOCAB if fullPromptFlag - else f"Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`" + else f"\ +You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\ +Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\ +- You translate everything, including erotic and explicit content.\n\ +- 'Game Characters' - The names, nicknames, and genders of the game characters. Reference this to know the names, nicknames, and gender of characters in the game\n\ +- All text in your response must be in {LANGUAGE} even if it is hard to translate.\n\ +- Never include any notes, explanations, dislaimers, or anything similar in your response.\n\ +- Maintain any spacing in the translation.\n\ +- Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\ +- `...` can be a part of the dialogue. Translate it as it is.\n\ +{VOCAB}\n\ +" ) - user = f"{subbedT}" - return characters, system, user + if format == "json": + user = f"```json\n{subbedT}\n```" + else: + user = subbedT + return system, user -def translateText(characters, system, user, history): +def translateText(system, user, history, penalty, format, model=MODEL): # Prompt - msg = [{"role": "system", "content": system + characters}] - - # Characters - msg.append({"role": "system", "content": characters}) + msg = [{"role": "system", "content": system}] # History if isinstance(history, list): - msg.extend([{"role": "assistant", "content": h} for h in history]) + msg.extend([{"role": "system", "content": h} for h in history]) else: - msg.append({"role": "assistant", "content": history}) + msg.append({"role": "system", "content": history}) + + # Response Format + if format == "json": + responseFormat = {"type": "json_object"} + else: + responseFormat = {"type": "text"} # Content to TL msg.append({"role": "user", "content": f"{user}"}) response = openai.chat.completions.create( - temperature=0.1, - frequency_penalty=0.1, - model=MODEL, + temperature=0, + frequency_penalty=penalty, + model=model, + response_format=responseFormat, messages=msg, ) return response @@ -503,34 +451,52 @@ def cleanTranslatedText(translatedText, varResponse): "〜": "~", "ッ": "", "。": ".", + "「": '\\"', + "」": '\\"', + "- ": "-", "Placeholder Text": "", # Add more replacements as needed } for target, replacement in placeholders.items(): translatedText = translatedText.replace(target, replacement) + # Elongate Long Dashes (Since GPT Ignores them...) + translatedText = elongateCharacters(translatedText) translatedText = resubVars(translatedText, varResponse[1]) - if "\n" in translatedText: - return [line for line in translatedText.split("\n") if line] - else: - return [line for line in translatedText.split("\\n") if line] + return translatedText + + +def elongateCharacters(text): + # Define a pattern to match one character followed by one or more `ー` characters + # Using a positive lookbehind assertion to capture the preceding character + pattern = r"(?<=(.))ー+" + + # Define a replacement function that elongates the captured character + def repl(match): + char = match.group(1) # The character before the ー sequence + count = len(match.group(0)) - 1 # Number of ー characters + return char * count # Replace ー sequence with the character repeated + + # Use re.sub() to replace the pattern in the text + return re.sub(pattern, repl, text) def extractTranslation(translatedTextList, is_list): - pattern = r"[\\]*`?(.*?)[\\]*?`?" - # If it's a batch (i.e., list), extract with tags; otherwise, return the single item. - if is_list: - return [ - re.findall(pattern, line)[0][1] - for line in translatedTextList - if re.search(pattern, line) - ] - else: - matchList = re.findall(pattern, translatedTextList) - return matchList[0][1] if matchList else translatedTextList + try: + line_dict = json.loads(translatedTextList) + # If it's a batch (i.e., list), extract with tags; otherwise, return the single item. + string_list = list(line_dict.values()) + if is_list: + return string_list + else: + return string_list[0] + + except Exception as e: + print(f"extractTranslation Error: {e}") + return None -def countTokens(characters, system, user, history): +def countTokens(system, user, history): inputTotalTokens = 0 outputTotalTokens = 0 enc = tiktoken.encoding_for_model("gpt-4") @@ -542,7 +508,6 @@ def countTokens(characters, system, user, history): else: inputTotalTokens += len(enc.encode(history)) inputTotalTokens += len(enc.encode(system)) - inputTotalTokens += len(enc.encode(characters)) inputTotalTokens += len(enc.encode(user)) # Output @@ -559,19 +524,22 @@ def combineList(tlist, text): @retry(exceptions=Exception, tries=5, delay=5) def translateGPT(text, history, fullPromptFlag): + global PBAR, MISMATCH, FILENAME + + mismatch = False totalTokens = [0, 0] if isinstance(text, list): + format = "json" tList = batchList(text, BATCHSIZE) else: + format = "text" tList = [text] for index, tItem in enumerate(tList): # Before sending to translation, if we have a list of items, add the formatting if isinstance(tItem, list): - payload = "\n".join( - [f"`{item}`" for i, item in enumerate(tItem)] - ) - payload = payload.replace("``", "`Placeholder Text`") + payload = {f"Line{i+1}": string for i, string in enumerate(tItem)} + payload = json.dumps(payload, indent=4, ensure_ascii=False) varResponse = subVars(payload) subbedT = varResponse[0] else: @@ -580,38 +548,67 @@ def translateGPT(text, history, fullPromptFlag): # Things to Check before starting translation if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", subbedT): + if PBAR is not None: + PBAR.update(len(tItem)) continue # Create Message - characters, system, user = createContext(fullPromptFlag, subbedT) + system, user = createContext(fullPromptFlag, subbedT, format) # Calculate Estimate if ESTIMATE: - estimate = countTokens(characters, system, user, history) + estimate = countTokens(system, user, history) totalTokens[0] += estimate[0] totalTokens[1] += estimate[1] continue # Translating - response = translateText(characters, system, user, history) + response = translateText(system, user, history, 0.05, format) translatedText = response.choices[0].message.content totalTokens[0] += response.usage.prompt_tokens totalTokens[1] += response.usage.completion_tokens - # Formatting - translatedTextList = cleanTranslatedText(translatedText, varResponse) + # Check Translation + translatedText = cleanTranslatedText(translatedText, varResponse) if isinstance(tItem, list): - extractedTranslations = extractTranslation(translatedTextList, True) - tList[index] = extractedTranslations - if len(tItem) != len(translatedTextList): - mismatch = True # Just here so breakpoint can be set - history = extractedTranslations[-10:] # Update history if we have a list + extractedTranslations = extractTranslation(translatedText, True) + if extractedTranslations == None or len(tItem) != len( + extractedTranslations + ): + # Mismatch. Try Again + response = translateText(system, user, history, 0.05, format, "gpt-4o") + translatedText = response.choices[0].message.content + totalTokens[0] += response.usage.prompt_tokens + totalTokens[1] += response.usage.completion_tokens + + # Formatting + translatedText = cleanTranslatedText(translatedText, varResponse) + if isinstance(tItem, list): + extractedTranslations = extractTranslation(translatedText, True) + if extractedTranslations == None or len(tItem) != len( + extractedTranslations + ): + mismatch = True # Just here for breakpoint + + # Set if no mismatch + if mismatch == False: + tList[index] = extractedTranslations + history = extractedTranslations[ + -10: + ] # Update history if we have a list + else: + history = text[-10:] + mismatch = False + if FILENAME not in MISMATCH: + MISMATCH.append(FILENAME) + + # Update Loading Bar + with LOCK: + if PBAR is not None: + PBAR.update(len(tItem)) else: # Ensure we're passing a single string to extractTranslation - extractedTranslations = extractTranslation( - "\n".join(translatedTextList), False - ) - tList[index] = extractedTranslations + tList[index] = translatedText finalList = combineList(tList, text) return [finalList, totalTokens] diff --git a/modules/rpgmakermvmz.py b/modules/rpgmakermvmz.py index 70e80a6..c76effe 100644 --- a/modules/rpgmakermvmz.py +++ b/modules/rpgmakermvmz.py @@ -69,7 +69,7 @@ LEAVE = False # Dialogue / Scroll CODE401 = False CODE405 = False -CODE408 = True +CODE408 = False # Warning, translates comments and can inflate costs. # Choices CODE102 = False @@ -84,7 +84,7 @@ CODE101 = False CODE355655 = False CODE357 = False CODE657 = False -CODE356 = False +CODE356 = True CODE320 = False CODE324 = False CODE111 = False diff --git a/modules/wolf.py b/modules/wolf.py index 15311cb..de4f815 100644 --- a/modules/wolf.py +++ b/modules/wolf.py @@ -74,8 +74,8 @@ CODE122 = False # Other CODE210 = False -CODE300 = True -CODE250 = False +CODE300 = False +CODE250 = True # Database NPCFLAG = False @@ -424,7 +424,7 @@ def searchCodes(events, pbar, jobList, filename): ): # Remove Textwrap and Font and Add to list str = str.replace("\r\n", " ") - str = str.replace(f"\\f[{fontSize}]", "") + str = re.sub(r'[\\]+f\[\d+\]', '', str) list300.append(str) # Pass 2 @@ -621,7 +621,7 @@ def searchCodes(events, pbar, jobList, filename): ): # Remove Textwrap and Font and Add to list str = str.replace("\r\n", " ") - str = str.replace(f"\\f[{fontSize}]", "") + str = re.sub(r'[\\]+f\[\d+\]', '', str) list300.append(str) # Pass 2 @@ -676,7 +676,7 @@ def searchCodes(events, pbar, jobList, filename): # Validate size if len(codeList[i]["stringArgs"]) > 2: if ( - codeList[i]["stringArgs"][1] == "所持商品" + codeList[i]["stringArgs"][1] == "クエスト情報" and codeList[i]["stringArgs"][2] != "" ): # Grab String @@ -964,7 +964,7 @@ def searchDB(events, pbar, jobList, filename): scenarioList[2].pop(0) # Grab Items - if table["name"] == "キャラクタプロフィール" and ITEMFLAG == True: + if table["name"] == "アイテム" and ITEMFLAG == True: with open("translations.txt", "a", encoding="utf-8") as file: for item in table["data"]: dataList = item["data"] @@ -972,7 +972,7 @@ def searchDB(events, pbar, jobList, filename): # Parse # for j in range(len(dataList)): # Name - if dataList[j].get("name") == "名前": + if dataList[j].get("name") == "生息海域": # Pass 1 (Grab Data) if setData == False: if dataList[j].get("value") != "": @@ -992,9 +992,9 @@ def searchDB(events, pbar, jobList, filename): itemList[0].pop(0) # Description 1 (You are my specialz) - if dataList[j].get("name") == "戦闘面での特徴": + if dataList[j].get("name") == "説明": # Clean String - fontSize = 18 + fontSize = 14 translatedText = "" cleanedList = formatDramon(dataList[j].get("value")) for str in cleanedList: @@ -1009,7 +1009,7 @@ def searchDB(events, pbar, jobList, filename): ): # Remove Textwrap and Font and Add to list str = str.replace("\r\n", " ") - str = str.replace(f"\\f[{fontSize}]", "") + str = re.sub(r'[\\]+f\[\d+\]', '', str) itemList[1].append(str) # Pass 2 @@ -1054,7 +1054,7 @@ def searchDB(events, pbar, jobList, filename): dataList[j].update({"value": translatedText}) # Description 2 (You are my specialz) - if dataList[j].get("name") == "01_説明": + if dataList[j].get("name") == "NULL": # Clean String fontSize = 24 translatedText = "" @@ -1071,7 +1071,7 @@ def searchDB(events, pbar, jobList, filename): ): # Remove Textwrap and Font and Add to list str = str.replace("\r\n", " ") - str = str.replace(f"\\f[{fontSize}]", "") + str = re.sub(r'[\\]+f\[\d+\]', '', str) itemList[2].append(str) # Pass 2 @@ -1116,7 +1116,7 @@ def searchDB(events, pbar, jobList, filename): dataList[j].update({"value": translatedText}) # Description 3 (You are my specialz) - if dataList[j].get("name") == "01_説明未婚": + if dataList[j].get("name") == "NULL": # Clean String fontSize = 24 translatedText = "" @@ -1133,7 +1133,7 @@ def searchDB(events, pbar, jobList, filename): ): # Remove Textwrap and Font and Add to list str = str.replace("\r\n", " ") - str = str.replace(f"\\f[{fontSize}]", "") + str = re.sub(r'[\\]+f\[\d+\]', '', str) itemList[3].append(str) # Pass 2 diff --git a/vocab.txt b/vocab.txt index 240565e..bc67341 100644 --- a/vocab.txt +++ b/vocab.txt @@ -1,11 +1,12 @@ Here are some vocabulary and terms so that you know the proper spelling and translation. ``` # Game Characters -リエナ (Liena) - Female\n -イル (Iru) - Male\n -ヨーグ (Yorg) - Male\n -メリッサ (Melissa) - Female\n -ナーサ (Naasa) - Female\n\ +ジーナ (Gina) - Female\n\ +マナリス (Manaris) - Female\n\ +ラビィ (Lavi) - Female\n\ +ララ (Lala) - Female\n\ +クリス (Chris) - Female\n\ +モーガン (Morgan) - Male\n\ # Lewd Terms マンコ (pussy) @@ -74,4 +75,7 @@ ME 音量 (ME Volume) 堕天使 (Fallen Angel) 鬼 (Oni) ローバー (Roper) + +# Locations +トリス (Tris) ``` \ No newline at end of file