From 0421c51bae130adac88217103baa8f8ed6c207ed Mon Sep 17 00:00:00 2001 From: DazedAnon Date: Tue, 17 Dec 2024 10:19:55 -0600 Subject: [PATCH] Fix single text strings --- modules/csv.py | 305 +++++--------- modules/json.py | 405 +++++++++---------- modules/kirikiri.py | 65 +-- modules/nscript.py | 336 +++++----------- modules/regex.py | 40 +- modules/renpy.py | 56 +-- modules/rpgmakerace.py | 65 +-- modules/rpgmakermvmz.py | 39 +- modules/rpgmakerplugin.py | 311 +++++---------- modules/text.py | 40 +- modules/tyrano.py | 39 +- modules/wolf.py | 811 +++++++++++++++++++++++++++++--------- modules/wolf2.py | 266 +++++-------- 13 files changed, 1301 insertions(+), 1477 deletions(-) diff --git a/modules/csv.py b/modules/csv.py index 654c2f3..d96557b 100644 --- a/modules/csv.py +++ b/modules/csv.py @@ -395,121 +395,6 @@ def getSpeaker(speaker): return [speaker, [0, 0]] -def subVars(jaString): - jaString = jaString.replace("\u3000", " ") - - # Nested - count = 0 - nestedList = re.findall(r"[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]", jaString) - nestedList = set(nestedList) - if len(nestedList) != 0: - for icon in nestedList: - jaString = jaString.replace(icon, "[Nested_" + str(count) + "]") - count += 1 - - # Icons - count = 0 - iconList = re.findall(r"[\\]+[iIkKwWaA]+\[[0-9]+\]", jaString) - iconList = set(iconList) - if len(iconList) != 0: - for icon in iconList: - jaString = jaString.replace(icon, "[Ascii_" + str(count) + "]") - count += 1 - - # Colors - count = 0 - colorList = re.findall(r"([\\]+c\[\d+\][\\]+c|[\\]+c\[\d+\])", jaString) - colorList = set(colorList) - if len(colorList) != 0: - for color in colorList: - jaString = jaString.replace(color, "[Color_" + str(count) + "]") - count += 1 - - # Names - count = 0 - nameList = re.findall(r"[\\]+[nN]\[.+?\]+", jaString) - nameList = set(nameList) - if len(nameList) != 0: - for name in nameList: - jaString = jaString.replace(name, "[Noun_" + str(count) + "]") - count += 1 - - # Variables - count = 0 - varList = re.findall(r"[\\]+[vV]\[[0-9]+\]", jaString) - varList = set(varList) - if len(varList) != 0: - for var in varList: - jaString = jaString.replace(var, "[Var_" + str(count) + "]") - count += 1 - - # Formatting - count = 0 - formatList = re.findall(r"[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]", jaString) - formatList = set(formatList) - if len(formatList) != 0: - for var in formatList: - jaString = jaString.replace(var, "[FCode_" + str(count) + "]") - count += 1 - - # Put all lists in list and return - allList = [nestedList, iconList, colorList, nameList, varList, formatList] - return [jaString, allList] - - -def resubVars(translatedText, allList): - # Fix Spacing and ChatGPT Nonsense - matchList = re.findall(r"\[\s?.+?\s?\]", translatedText) - if len(matchList) > 0: - for match in matchList: - text = match.strip() - translatedText = translatedText.replace(match, text) - - # Nested - count = 0 - if len(allList[0]) != 0: - for var in allList[0]: - translatedText = translatedText.replace("[Nested_" + str(count) + "]", var) - count += 1 - - # Icons - count = 0 - if len(allList[1]) != 0: - for var in allList[1]: - translatedText = translatedText.replace("[Ascii_" + str(count) + "]", var) - count += 1 - - # Colors - count = 0 - if len(allList[2]) != 0: - for var in allList[2]: - translatedText = translatedText.replace("[Color_" + str(count) + "]", var) - count += 1 - - # Names - count = 0 - if len(allList[3]) != 0: - for var in allList[3]: - translatedText = translatedText.replace("[Noun_" + str(count) + "]", var) - count += 1 - - # Vars - count = 0 - if len(allList[4]) != 0: - for var in allList[4]: - translatedText = translatedText.replace("[Var_" + str(count) + "]", var) - count += 1 - - # Formatting - count = 0 - if len(allList[5]) != 0: - for var in allList[5]: - translatedText = translatedText.replace("[FCode_" + str(count) + "]", var) - count += 1 - - return translatedText - - def batchList(input_list, batch_size): if not isinstance(batch_size, int) or batch_size <= 0: raise ValueError("batch_size must be a positive integer") @@ -517,12 +402,7 @@ def batchList(input_list, batch_size): return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)] -def createContext(fullPromptFlag, subbedT): - characters = "Game Characters:\n\ -クリスティーナ (Christina) - Female\n\ -リズ (Liz) - Female\n\ -" - +def createContext(fullPromptFlag, subbedT, format): system = ( PROMPT + VOCAB if fullPromptFlag @@ -539,19 +419,13 @@ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{ {VOCAB}\n\ " ) - if isinstance(subbedT, list): - user = f"```json\n{subbedT}```" - else: - user = subbedT - return characters, system, user + user = f"```json\n{subbedT}\n```" + return system, user -def translateText(characters, system, user, history, penalty, format): +def translateText(system, user, history, penalty, format, model=MODEL): # Prompt - msg = [{"role": "system", "content": system + characters}] - - # Characters - msg.append({"role": "system", "content": characters}) + msg = [{"role": "system", "content": system}] # History if isinstance(history, list): @@ -560,17 +434,14 @@ def translateText(characters, system, user, history, penalty, format): msg.append({"role": "system", "content": history}) # Response Format - if format == "json": - responseFormat = {"type": "json_object"} - else: - responseFormat = {"type": "text"} + responseFormat = {"type": "json_object"} # Content to TL msg.append({"role": "user", "content": f"{user}"}) response = openai.chat.completions.create( temperature=0, frequency_penalty=penalty, - model=MODEL, + model=model, response_format=responseFormat, messages=msg, ) @@ -588,6 +459,8 @@ def cleanTranslatedText(translatedText, varResponse): "「": '\\"', "」": '\\"', "- ": "-", + "】": "]", + "【": "[", "Placeholder Text": "", # Add more replacements as needed } @@ -596,7 +469,6 @@ def cleanTranslatedText(translatedText, varResponse): # Elongate Long Dashes (Since GPT Ignores them...) translatedText = elongateCharacters(translatedText) - translatedText = resubVars(translatedText, varResponse[1]) return translatedText @@ -618,6 +490,7 @@ def elongateCharacters(text): def extractTranslation(translatedTextList, is_list): try: translatedTextList = re.sub(r'\\"+\"([^,\n}])', r'\\"\1', translatedTextList) + translatedTextList = re.sub(r"(? 0: - for match in matchList: - text = match.strip() - translatedText = translatedText.replace(match, text) - - # Nested - count = 0 - if len(allList[0]) != 0: - for var in allList[0]: - translatedText = translatedText.replace("{Nested_" + str(count) + "}", var) - count += 1 - - # Icons - count = 0 - if len(allList[1]) != 0: - for var in allList[1]: - translatedText = translatedText.replace("{Ascii_" + str(count) + "}", var) - count += 1 - - # Colors - count = 0 - if len(allList[2]) != 0: - for var in allList[2]: - translatedText = translatedText.replace("{Color_" + str(count) + "}", var) - count += 1 - - # Names - count = 0 - if len(allList[3]) != 0: - for var in allList[3]: - translatedText = translatedText.replace("{Noun_" + str(count) + "}", var) - count += 1 - - # Vars - count = 0 - if len(allList[4]) != 0: - for var in allList[4]: - translatedText = translatedText.replace("{Var_" + str(count) + "}", var) - count += 1 - - # Formatting - count = 0 - if len(allList[5]) != 0: - for var in allList[5]: - translatedText = translatedText.replace("{FCode_" + str(count) + "}", var) - count += 1 - - return translatedText + speakerList = [speaker, response[0]] + NAMESLIST.append(speakerList) + return response + return [speaker, [0, 0]] def batchList(input_list, batch_size): @@ -438,69 +343,47 @@ def batchList(input_list, batch_size): return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)] -def createContext(fullPromptFlag, subbedT): - characters = "Game Characters:\n\ -ルナリア (Lunaria) - Female\n\ -ソニア (Sonia) - Female\n\ -マナ (Mana) - Female\n\ -マリアナ (Mariana) - Female\n\ -ディアナ (Diana) - Female\n\ -シャーリー (Shirley) - Female\n\ -エスティア (Estia) - Female\n\ -エレノア (Eleanor) - Female\n\ -メリス (Meris) - Female\n\ -サルビア (Salvia) - Female\n\ -リリ (Lili) - Female\n\ -ツキハ (Tsukiha) - Female\n\ -フィリカ (Filica) - Female\n\ -レノ (Renno) - Female\n\ -" - +def createContext(fullPromptFlag, subbedT, format): system = ( - PROMPT + PROMPT + VOCAB if fullPromptFlag else f"\ You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\ -You are going to be translating text from a videogame.\n\ -I will give you lines of text, and you must translate each line to the best of your ability.\n\ -- Translate 'マンコ' as 'pussy'\n\ -- Translate 'おまんこ' as 'pussy'\n\ -- Translate 'お尻' as 'butt'\n\ -- Translate '尻' as 'ass'\n\ -- Translate 'お股' as 'crotch'\n\ -- Translate '秘部' as 'genitals'\n\ -- Translate 'チンポ' as 'dick'\n\ -- Translate 'チンコ' as 'cock'\n\ -- Translate 'ショーツ' as 'panties\n\ -- Translate 'おねショタ' as 'Onee-shota'\n\ -- Translate 'よかった' as 'thank goodness'\n\ -Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\ +Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\ +- You translate everything, including erotic and explicit content.\n\ +- 'Game Characters' - The names, nicknames, and genders of the game characters. Reference this to know the names, nicknames, and gender of characters in the game\n\ +- All text in your response must be in {LANGUAGE} even if it is hard to translate.\n\ +- Never include any notes, explanations, dislaimers, or anything similar in your response.\n\ +- Maintain any spacing in the translation.\n\ +- Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\ +- `...` can be a part of the dialogue. Translate it as it is.\n\ +{VOCAB}\n\ " ) - user = f"{subbedT}" - return characters, system, user + user = f"```json\n{subbedT}\n```" + return system, user -def translateText(characters, system, user, history): +def translateText(system, user, history, penalty, format, model=MODEL): # Prompt - msg = [{"role": "system", "content": system + characters}] - - # Characters - msg.append({"role": "system", "content": characters}) + msg = [{"role": "system", "content": system}] # History if isinstance(history, list): - msg.extend([{"role": "assistant", "content": h} for h in history]) + msg.extend([{"role": "system", "content": h} for h in history]) else: - msg.append({"role": "assistant", "content": history}) + msg.append({"role": "system", "content": history}) + + # Response Format + responseFormat = {"type": "json_object"} # Content to TL msg.append({"role": "user", "content": f"{user}"}) response = openai.chat.completions.create( - temperature=0.1, - frequency_penalty=0.1, - presence_penalty=0.1, - model=MODEL, + temperature=0, + frequency_penalty=penalty, + model=model, + response_format=responseFormat, messages=msg, ) return response @@ -514,31 +397,55 @@ def cleanTranslatedText(translatedText, varResponse): "〜": "~", "ッ": "", "。": ".", + "「": '\\"', + "」": '\\"', + "- ": "-", + "】": "]", + "【": "[", "Placeholder Text": "", # Add more replacements as needed } for target, replacement in placeholders.items(): translatedText = translatedText.replace(target, replacement) - translatedText = resubVars(translatedText, varResponse[1]) - return [line for line in translatedText.split("\n") if line] + # Elongate Long Dashes (Since GPT Ignores them...) + translatedText = elongateCharacters(translatedText) + return translatedText + + +def elongateCharacters(text): + # Define a pattern to match one character followed by one or more `ー` characters + # Using a positive lookbehind assertion to capture the preceding character + pattern = r"(?<=(.))ー+" + + # Define a replacement function that elongates the captured character + def repl(match): + char = match.group(1) # The character before the ー sequence + count = len(match.group(0)) - 1 # Number of ー characters + return char * count # Replace ー sequence with the character repeated + + # Use re.sub() to replace the pattern in the text + return re.sub(pattern, repl, text) def extractTranslation(translatedTextList, is_list): - pattern = r"`?([\\]*.*?[\\]*?)<\/?Line\d+>`?" - # If it's a batch (i.e., list), extract with tags; otherwise, return the single item. - if is_list: - return [ - re.findall(pattern, line)[0][1] - for line in translatedTextList - if re.search(pattern, line) - ] - else: - matchList = re.findall(pattern, translatedTextList) - return matchList[0][1] if matchList else translatedTextList + try: + translatedTextList = re.sub(r'\\"+\"([^,\n}])', r'\\"\1', translatedTextList) + translatedTextList = re.sub(r"(?{item}`" for i, item in enumerate(tItem)]) - payload = payload.replace("``", "`Placeholder Text`") - varResponse = subVars(payload) - subbedT = varResponse[0] + global PBAR, MISMATCH, FILENAME + with open("log/translationHistory.txt", "a+", encoding="utf-8") as logFile: + mismatch = False + totalTokens = [0, 0] + if isinstance(text, list): + format = "json" + tList = batchList(text, BATCHSIZE) else: - varResponse = subVars(tItem) + format = "text" + tList = [text] + + for index, tItem in enumerate(tList): + # Before sending to translation, if we have a list of items, add the formatting + if not isinstance(tItem, list): + tItem = [tItem] + for j in range(len(tItem)): + if not tItem[j]: + tItem[j] = tItem[j].replace("", "Placeholder Text") + payload = {f"Line{i+1}": string for i, string in enumerate(tItem)} + payload = json.dumps(payload, indent=4, ensure_ascii=False) + varResponse = [payload, []] subbedT = varResponse[0] - # Things to Check before starting translation - if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", subbedT): - continue + # Things to Check before starting translation + if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+", subbedT): + if PBAR is not None: + PBAR.update(len(tItem)) + history = tItem[-MAXHISTORY:] + continue - # Create Message - characters, system, user = createContext(fullPromptFlag, subbedT) + # Create Message + system, user = createContext(fullPromptFlag, subbedT, format) - # Calculate Estimate - if ESTIMATE: - estimate = countTokens(characters, system, user, history) - totalTokens[0] += estimate[0] - totalTokens[1] += estimate[1] - continue + # Calculate Estimate + if ESTIMATE: + estimate = countTokens(system, user, history) + totalTokens[0] += estimate[0] + totalTokens[1] += estimate[1] + continue - # Translating - response = translateText(characters, system, user, history) - translatedText = response.choices[0].message.content - totalTokens[0] += response.usage.prompt_tokens - totalTokens[1] += response.usage.completion_tokens + # Translating + response = translateText(system, user, history, 0.05, format) + translatedText = response.choices[0].message.content + totalTokens[0] += response.usage.prompt_tokens + totalTokens[1] += response.usage.completion_tokens - # Formatting - translatedTextList = cleanTranslatedText(translatedText, varResponse) - if isinstance(tItem, list): - extractedTranslations = extractTranslation(translatedTextList, True) - tList[index] = extractedTranslations - if len(tItem) != len(translatedTextList): - mismatch = True # Just here so breakpoint can be set - history = extractedTranslations[-10:] # Update history if we have a list - else: - # Ensure we're passing a single string to extractTranslation - extractedTranslations = extractTranslation("\n".join(translatedTextList), False) - tList[index] = extractedTranslations + # Check Translation + translatedText = cleanTranslatedText(translatedText, varResponse) + if isinstance(tItem, list): + extractedTranslations = extractTranslation(translatedText, True) + if extractedTranslations == None or len(tItem) != len(extractedTranslations): + # Mismatch. Try Again + response = translateText(system, user, history, 0.05, format, "gpt-4o") + translatedText = response.choices[0].message.content + totalTokens[0] += response.usage.prompt_tokens + totalTokens[1] += response.usage.completion_tokens + + # Formatting + translatedText = cleanTranslatedText(translatedText, varResponse) + if isinstance(tItem, list): + extractedTranslations = extractTranslation(translatedText, True) + if extractedTranslations == None or len(tItem) != len( + extractedTranslations + ): + mismatch = True # Just here for breakpoint + logFile.write(f"Input:\n{subbedT}\n") + logFile.write(f"Output:\n{translatedText}\n") + + # Set if no mismatch + if mismatch == False: + tList[index] = extractedTranslations + history = extractedTranslations[ + -MAXHISTORY: + ] # Update history if we have a list + else: + history = text[-MAXHISTORY:] + mismatch = False + if FILENAME not in MISMATCH: + MISMATCH.append(FILENAME) + + # Update Loading Bar + with LOCK: + if PBAR is not None: + PBAR.update(len(tItem)) + else: + # Ensure we're passing a single string to extractTranslation + tList[index] = translatedText.replace("Placeholder Text", "") finalList = combineList(tList, text) - return [finalList, totalTokens] + if format == "json": + return [finalList, totalTokens] + else: + return [finalList[0], totalTokens] diff --git a/modules/kirikiri.py b/modules/kirikiri.py index 4266183..2598563 100644 --- a/modules/kirikiri.py +++ b/modules/kirikiri.py @@ -379,34 +379,6 @@ def getSpeaker(speaker): return [speaker, [0, 0]] -def subVars(jaString): - jaString = jaString.replace("\u3000", " ") - - # Formatting - codeList = re.findall(r"([\\]*(\w+)\[(\d+)\])|([\\]*(\w+)\[[\\]*\\w+\[(\d+)\]\])", jaString) - codeList = set(codeList) - if len(codeList) != 0: - for var in codeList: - if var[2]: - jaString = jaString.replace(var[0], f"[{var[1]}Code_" + f"{var[2]}]") - else: - jaString = jaString.replace(var[3], f"[{var[4]}Code_" + f"{var[5]}]") - - # Put all lists in list and return - return [jaString, codeList] - - -def resubVars(translatedText, codeList): - # Formatting - for var in codeList: - if var[2]: - translatedText = translatedText.replace(f"[{var[1]}Code_" + f"{var[2]}]", var[0]) - else: - translatedText = translatedText.replace(f"[{var[4]}Code_" + f"{var[5]}]", var[3]) - - return translatedText - - def batchList(input_list, batch_size): if not isinstance(batch_size, int) or batch_size <= 0: raise ValueError("batch_size must be a positive integer") @@ -431,10 +403,7 @@ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{ {VOCAB}\n\ " ) - if format == "json": - user = f"```json\n{subbedT}\n```" - else: - user = subbedT + user = f"```json\n{subbedT}\n```" return system, user @@ -449,10 +418,7 @@ def translateText(system, user, history, penalty, format, model=MODEL): msg.append({"role": "system", "content": history}) # Response Format - if format == "json": - responseFormat = {"type": "json_object"} - else: - responseFormat = {"type": "text"} + responseFormat = {"type": "json_object"} # Content to TL msg.append({"role": "user", "content": f"{user}"}) @@ -487,7 +453,6 @@ def cleanTranslatedText(translatedText, varResponse): # Elongate Long Dashes (Since GPT Ignores them...) translatedText = elongateCharacters(translatedText) - translatedText = resubVars(translatedText, varResponse[1]) return translatedText @@ -522,10 +487,6 @@ def extractTranslation(translatedTextList, is_list): PBAR.write(f"extractTranslation Error: {e} on String {translatedTextList}") return None - except Exception as e: - PBAR.write(f"extractTranslation Error: {e} on String {translatedTextList}") - return None - def countTokens(system, user, history): inputTotalTokens = 0 @@ -568,14 +529,15 @@ def translateGPT(text, history, fullPromptFlag): for index, tItem in enumerate(tList): # Before sending to translation, if we have a list of items, add the formatting - if isinstance(tItem, list): - payload = {f"Line{i+1}": string for i, string in enumerate(tItem)} - payload = json.dumps(payload, indent=4, ensure_ascii=False) - varResponse = subVars(payload) - subbedT = varResponse[0] - else: - varResponse = [tItem, []] - subbedT = varResponse[0] + if not isinstance(tItem, list): + tItem = [tItem] + for j in range(len(tItem)): + if not tItem[j]: + tItem[j] = tItem[j].replace("", "Placeholder Text") + payload = {f"Line{i+1}": string for i, string in enumerate(tItem)} + payload = json.dumps(payload, indent=4, ensure_ascii=False) + varResponse = [payload, []] + subbedT = varResponse[0] # Things to Check before starting translation if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+", subbedT): @@ -643,4 +605,7 @@ def translateGPT(text, history, fullPromptFlag): tList[index] = translatedText.replace("Placeholder Text", "") finalList = combineList(tList, text) - return [finalList, totalTokens] + if format == "json": + return [finalList, totalTokens] + else: + return [finalList[0], totalTokens] diff --git a/modules/nscript.py b/modules/nscript.py index 853f9e2..f33d430 100644 --- a/modules/nscript.py +++ b/modules/nscript.py @@ -395,121 +395,6 @@ def getSpeaker(speaker): return [speaker, [0, 0]] -def subVars(jaString): - jaString = jaString.replace("\u3000", " ") - - # Nested - count = 0 - nestedList = re.findall(r"[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]", jaString) - nestedList = set(nestedList) - if len(nestedList) != 0: - for icon in nestedList: - jaString = jaString.replace(icon, "[Nested_" + str(count) + "]") - count += 1 - - # Icons - count = 0 - iconList = re.findall(r"[\\]+[iIkKwWaA]+\[[0-9]+\]", jaString) - iconList = set(iconList) - if len(iconList) != 0: - for icon in iconList: - jaString = jaString.replace(icon, "[Ascii_" + str(count) + "]") - count += 1 - - # Colors - count = 0 - colorList = re.findall(r"([\\]+c\[\d+\][\\]+c|[\\]+c\[\d+\])", jaString) - colorList = set(colorList) - if len(colorList) != 0: - for color in colorList: - jaString = jaString.replace(color, "[Color_" + str(count) + "]") - count += 1 - - # Names - count = 0 - nameList = re.findall(r"[\\]+[nN]\[.+?\]+", jaString) - nameList = set(nameList) - if len(nameList) != 0: - for name in nameList: - jaString = jaString.replace(name, "[Noun_" + str(count) + "]") - count += 1 - - # Variables - count = 0 - varList = re.findall(r"[\\]+[vV]\[[0-9]+\]", jaString) - varList = set(varList) - if len(varList) != 0: - for var in varList: - jaString = jaString.replace(var, "[Var_" + str(count) + "]") - count += 1 - - # Formatting - count = 0 - formatList = re.findall(r"[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]", jaString) - formatList = set(formatList) - if len(formatList) != 0: - for var in formatList: - jaString = jaString.replace(var, "[FCode_" + str(count) + "]") - count += 1 - - # Put all lists in list and return - allList = [nestedList, iconList, colorList, nameList, varList, formatList] - return [jaString, allList] - - -def resubVars(translatedText, allList): - # Fix Spacing and ChatGPT Nonsense - matchList = re.findall(r"\[\s?.+?\s?\]", translatedText) - if len(matchList) > 0: - for match in matchList: - text = match.strip() - translatedText = translatedText.replace(match, text) - - # Nested - count = 0 - if len(allList[0]) != 0: - for var in allList[0]: - translatedText = translatedText.replace("[Nested_" + str(count) + "]", var) - count += 1 - - # Icons - count = 0 - if len(allList[1]) != 0: - for var in allList[1]: - translatedText = translatedText.replace("[Ascii_" + str(count) + "]", var) - count += 1 - - # Colors - count = 0 - if len(allList[2]) != 0: - for var in allList[2]: - translatedText = translatedText.replace("[Color_" + str(count) + "]", var) - count += 1 - - # Names - count = 0 - if len(allList[3]) != 0: - for var in allList[3]: - translatedText = translatedText.replace("[Noun_" + str(count) + "]", var) - count += 1 - - # Vars - count = 0 - if len(allList[4]) != 0: - for var in allList[4]: - translatedText = translatedText.replace("[Var_" + str(count) + "]", var) - count += 1 - - # Formatting - count = 0 - if len(allList[5]) != 0: - for var in allList[5]: - translatedText = translatedText.replace("[FCode_" + str(count) + "]", var) - count += 1 - - return translatedText - - def batchList(input_list, batch_size): if not isinstance(batch_size, int) or batch_size <= 0: raise ValueError("batch_size must be a positive integer") @@ -517,37 +402,7 @@ def batchList(input_list, batch_size): return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)] -def createContext(fullPromptFlag, subbedT): - characters = "Game Characters:\n\ -レナリス (Renalith) - Female\n\ -スクルー (Sukuru) - Female\n\ -シスターミサ (Sister Misa) - Female\n\ -オリン (Orin) - Female\n\ -プローテ (Prote) - Female\n\ -夜霧 (Night Fog) - Female\n\ -ワウ (Wao) - Female\n\ -ファンナ (Fanna) - Female\n\ -精霊主スクルド (Spirit God Skuld) - Female\n\ -エキドナ (Echnida) - Female\n\ -マルス (Mars) - Male\n\ -ラヴィー (Lavi) - Unknown\n\ -魅音 (Mion) - Female\n\ -ヴィオラ (Viola) - Female\n\ -リンメイ (Lin Mei) - Female\n\ -リネット (Lynette) - Female\n\ -チェロル (Cheryl) - Female\n\ -カルーア姫 (Princess Karua) - Female\n\ -田姫 (Tajirme) - Female\n\ -リュート (Luto) - Male\n\ -ホルン (Horn) - Female\n\ -ルメラ (Lumera) - Female\n\ -末嬉 (Sueki) - Female\n\ -モニカ姫 (Princess Monica) - Female\n\ -エメルーラ (Emerald) - Female\n\ -フンシス (Funsis) - Male \n\ -バゼット (Bazzet) - Female\n\ -" - +def createContext(fullPromptFlag, subbedT, format): system = ( PROMPT + VOCAB if fullPromptFlag @@ -564,16 +419,13 @@ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{ {VOCAB}\n\ " ) - user = f"```json\n{subbedT}```" - return characters, system, user + user = f"```json\n{subbedT}\n```" + return system, user -def translateText(characters, system, user, history, penalty): +def translateText(system, user, history, penalty, format, model=MODEL): # Prompt - msg = [{"role": "system", "content": system + characters}] - - # Characters - msg.append({"role": "system", "content": characters}) + msg = [{"role": "system", "content": system}] # History if isinstance(history, list): @@ -581,13 +433,16 @@ def translateText(characters, system, user, history, penalty): else: msg.append({"role": "system", "content": history}) + # Response Format + responseFormat = {"type": "json_object"} + # Content to TL msg.append({"role": "user", "content": f"{user}"}) response = openai.chat.completions.create( temperature=0, frequency_penalty=penalty, - model=MODEL, - response_format={"type": "json_object"}, + model=model, + response_format=responseFormat, messages=msg, ) return response @@ -604,6 +459,8 @@ def cleanTranslatedText(translatedText, varResponse): "「": '\\"', "」": '\\"', "- ": "-", + "】": "]", + "【": "[", "Placeholder Text": "", # Add more replacements as needed } @@ -612,7 +469,6 @@ def cleanTranslatedText(translatedText, varResponse): # Elongate Long Dashes (Since GPT Ignores them...) translatedText = elongateCharacters(translatedText) - translatedText = resubVars(translatedText, varResponse[1]) return translatedText @@ -633,17 +489,22 @@ def elongateCharacters(text): def extractTranslation(translatedTextList, is_list): try: + translatedTextList = re.sub(r'\\"+\"([^,\n}])', r'\\"\1', translatedTextList) + translatedTextList = re.sub(r"(? 0: - for match in matchList: - text = match.strip() - translatedText = translatedText.replace(match, text) - - # Nested - count = 0 - if len(allList[0]) != 0: - for var in allList[0]: - translatedText = translatedText.replace("[Nested_" + str(count) + "]", var) - count += 1 - - # Icons - count = 0 - if len(allList[1]) != 0: - for var in allList[1]: - translatedText = translatedText.replace("[Ascii_" + str(count) + "]", var) - count += 1 - - # Colors - count = 0 - if len(allList[2]) != 0: - for var in allList[2]: - translatedText = translatedText.replace("[Color_" + str(count) + "]", var) - count += 1 - - # Names - count = 0 - if len(allList[3]) != 0: - for var in allList[3]: - translatedText = translatedText.replace("[Noun_" + str(count) + "]", var) - count += 1 - - # Vars - count = 0 - if len(allList[4]) != 0: - for var in allList[4]: - translatedText = translatedText.replace("[Var_" + str(count) + "]", var) - count += 1 - - # Formatting - count = 0 - if len(allList[5]) != 0: - for var in allList[5]: - translatedText = translatedText.replace("[FCode_" + str(count) + "]", var) - count += 1 - - return translatedText - - def batchList(input_list, batch_size): if not isinstance(batch_size, int) or batch_size <= 0: raise ValueError("batch_size must be a positive integer") @@ -432,18 +317,7 @@ def batchList(input_list, batch_size): return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)] -def createContext(fullPromptFlag, subbedT): - characters = "Game Characters:\n\ -シェーア (Shea) - Female\n\ -ミューテ (Mute) - Female\n\ -タビノ (Tabino) - Female\n\ -スラミー (Slamy) - Female\n\ -クリスタ (Christa) - Female\n\ -ソフィー (Sophie) - Female\n\ -ドーラ (Dora) - Female\n\ -ミューレ (Mule) - Female\n\ -" - +def createContext(fullPromptFlag, subbedT, format): system = ( PROMPT + VOCAB if fullPromptFlag @@ -460,19 +334,13 @@ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{ {VOCAB}\n\ " ) - if isinstance(subbedT, list): - user = f"```json\n{subbedT}```" - else: - user = subbedT - return characters, system, user + user = f"```json\n{subbedT}\n```" + return system, user -def translateText(characters, system, user, history, penalty, format): +def translateText(system, user, history, penalty, format, model=MODEL): # Prompt - msg = [{"role": "system", "content": system + characters}] - - # Characters - msg.append({"role": "system", "content": characters}) + msg = [{"role": "system", "content": system}] # History if isinstance(history, list): @@ -481,17 +349,14 @@ def translateText(characters, system, user, history, penalty, format): msg.append({"role": "system", "content": history}) # Response Format - if format == "json": - responseFormat = {"type": "json_object"} - else: - responseFormat = {"type": "text"} + responseFormat = {"type": "json_object"} # Content to TL msg.append({"role": "user", "content": f"{user}"}) response = openai.chat.completions.create( temperature=0, frequency_penalty=penalty, - model=MODEL, + model=model, response_format=responseFormat, messages=msg, ) @@ -509,6 +374,8 @@ def cleanTranslatedText(translatedText, varResponse): "「": '\\"', "」": '\\"', "- ": "-", + "】": "]", + "【": "[", "Placeholder Text": "", # Add more replacements as needed } @@ -517,7 +384,6 @@ def cleanTranslatedText(translatedText, varResponse): # Elongate Long Dashes (Since GPT Ignores them...) translatedText = elongateCharacters(translatedText) - translatedText = resubVars(translatedText, varResponse[1]) return translatedText @@ -539,6 +405,7 @@ def elongateCharacters(text): def extractTranslation(translatedTextList, is_list): try: translatedTextList = re.sub(r'\\"+\"([^,\n}])', r'\\"\1', translatedTextList) + translatedTextList = re.sub(r"(? 0: + # Grab String + jaString = re.search(r"^\n?(.*)\n?$", codeList[i]["stringArgs"][0]) + if jaString: + jaString = jaString.group(1) + else: + jaString = codeList[i]["stringArgs"][0] + + # Remove Textwrap + jaString = jaString.replace("\n", " ") + jaString = jaString.replace("\r", "") + + # Translate Other Strings [Specific Files Only] + if ( + not re.search(r"\.[\w]+$", jaString) + and jaString != "" + and "_" not in jaString + and '",' not in jaString + and "/" not in jaString + ): + # Japanese Text Only + if re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", jaString): # Translate response = translateGPT( jaString, f"Reply with the {LANGUAGE} translation of the text", - False, + True, ) translatedText = response[0] totalTokens[0] += response[1][0] totalTokens[1] += response[1][1] # Textwrap - translatedText = textwrap.fill(translatedText, LISTWIDTH) + translatedText = textwrap.fill(translatedText, WIDTH) # Set String - codeList[i]["stringArgs"][0] = codeList[i]["stringArgs"][0].replace( - jaString, translatedText - ) + codeList[i]["stringArgs"][0] = translatedText ### Event Code: 300 Common Events if ( @@ -549,91 +583,37 @@ def searchCodes(events, pbar, jobList, filename): codeList[i]["stringArgs"][1] = translatedText # Dialogue - elif ( - codeList[i]["stringArgs"][0] == "Hメッセージ" - or codeList[i]["stringArgs"][0] == "Hしらべる" - or codeList[i]["stringArgs"][0] == "[共]Hメッセージ+" - or codeList[i]["stringArgs"][0] == "d[共]ポップアップ表示" - or codeList[i]["stringArgs"][0] == "m_謎冒頭" - or ( - codeList[i]["codeStr"] == "SetString" - and ( - "「" in codeList[i]["stringArgs"][0] - or "*" in codeList[i]["stringArgs"][0] - ) - ) - or codeList[i]["stringArgs"][0] == "[移]サウンドノベル" - ): - cleanedList = None - if len(codeList[i]["stringArgs"]) > 1 and not re.search( - r"^[\\]+cself\[\d+\]$", codeList[i]["stringArgs"][1] - ): - cleanedList = formatDramon(codeList[i]["stringArgs"][1]) - elif codeList[i]["code"] == 122: - cleanedList = formatDramon(codeList[i]["stringArgs"][0]) - if cleanedList: - fontSize = 24 - translatedText = "" + elif codeList[i]["stringArgs"][0] == "○【戦闘】テキスト表示": + jaString = codeList[i]["stringArgs"][1] - for str in cleanedList: - # Pass 1 - if not setData: - if ( - all(x not in str for x in ["_", "@", ">", "/"]) - and str != "\r\n" - ): - # Remove Textwrap and Font and Add to list - str = str.replace("\r\n", " ") - str = re.sub(r"[\\]+f\[\d+\]", "", str) - list300.append(str) + # Pass 1 + if not setData: + # Remove Textwrap + jaString = jaString.replace("\n", " ") + jaString = jaString.replace("\r", "") - # Pass 2 - else: - if ( - all( - x not in str - for x in [ - "_", - "@", - ">", - "/", - ] - ) - and str != "\r\n" - ): - # Decide Wrap - if codeList[i]["stringArgs"][0] == "[移]サウンドノベル": - width = 40 - else: - width = WIDTH + # Append + list300.append(jaString) - # Add Textwrap and Font - list300[0] = textwrap.fill(list300[0], width) - list300[0] = list300[0].replace("\n", f"\r\n\\f[{fontSize}]") - list300[0] = f"\\f[{fontSize}]{list300[0]}\r\n" - translatedText += list300[0] - list300.pop(0) - else: - translatedText += str + # Pass 2 + else: + # Add Textwrap and Font + translatedText = textwrap.fill(list300[0], WIDTH) + list300.pop(0) # Write to File - if setData: - # Formatting Fixes - translatedText = translatedText.replace('*"', '* "') - translatedText = translatedText.replace("\r\n\r\n", "\r\n") - translatedText = re.sub(r"[^\S\r\n]+", " ", translatedText) - if len(codeList[i]["stringArgs"]) > 1: - codeList[i]["stringArgs"][1] = translatedText - else: - codeList[i]["stringArgs"][0] = translatedText + codeList[i]["stringArgs"][1] = translatedText - ### Event Code: 250 Common Events + ### Event Code: 250 DB Read/Writes if codeList[i]["code"] == 250 and CODE250 == True: foundTerm = False # Validate size - if len(codeList[i]["stringArgs"]) > 2: - if codeList[i]["stringArgs"][2] != "": + if len(codeList[i]["stringArgs"]) == 4: + if ( + codeList[i]["stringArgs"][1] == "ワープポイント設定" + and codeList[i]["stringArgs"][2] != "" + ): # Grab String jaString = codeList[i]["stringArgs"][2] @@ -657,7 +637,7 @@ def searchCodes(events, pbar, jobList, filename): response = translateGPT( jaString, f"Reply with the {LANGUAGE} translation of the text.", - False, + True, ) translatedText = response[0] totalTokens[0] += response[1][0] @@ -792,12 +772,13 @@ def searchDB(events, pbar, jobList, filename): scenarioList = jobList[0] npcList = jobList[1] itemList = jobList[2] - collectionList = jobList[3] + stateList = jobList[3] armorList = jobList[4] enemyList = jobList[5] weaponsList = jobList[6] skillList = jobList[7] optionsList = jobList[8] + dbNameList = jobList[9] setData = True else: scenarioList = [[], [], []] @@ -806,9 +787,10 @@ def searchDB(events, pbar, jobList, filename): armorList = [[], []] enemyList = [[], []] weaponsList = [[], [], [], []] - skillList = [[], [], [], []] - collectionList = [[], [], [], []] + skillList = [[], [], [], [], []] + stateList = [[], [], [], [], [], [], [], []] optionsList = [[], [], [], []] + dbNameList = [[]] setData = False # Vars/Globals @@ -973,15 +955,32 @@ def searchDB(events, pbar, jobList, filename): # Set Data dataList[j].update({"value": translatedText}) + # Grab DB Names + if table["name"] == "マップ設定" and DBNAMEFLAG == True: + font = None + dbName = table["data"] + for j in range(len(dbName)): + # Pass 1 (Grab Data) + if setData == False: + if dbName[j].get("name") != "": + dbNameList[0].append(dbName[j].get("name")) + + # Pass 2 (Set Data) + else: + if dbName[j].get("name") != "": + dbName[j].update({"name": dbNameList[0][0]}) + dbNameList[0].pop(0) + # Grab Items - if table["name"] == "選択肢説明" and ITEMFLAG == True: + if table["name"] == "アイテム・技能作成" and ITEMFLAG == True: for item in table["data"]: dataList = item["data"] # Parse for j in range(len(dataList)): + font = None # Name - if "表示名" in dataList[j].get("name"): + if "項目文　　　　　" in dataList[j].get("name"): # Pass 1 (Grab Data) if setData == False: if dataList[j].get("value") != "": @@ -994,11 +993,11 @@ def searchDB(events, pbar, jobList, filename): itemList[0].pop(0) # Description - if "選択肢9" in dataList[j].get("name"): + if "┗未作成時" in dataList[j].get("name"): # Pass 1 (Grab Data) if setData == False: if dataList[j].get("value") != "": - # Remove Textwrap + # Remove Textwrap & Font jaString = dataList[j].get("value") jaString = jaString.replace("\n", " ") jaString = jaString.replace("\r", "") @@ -1012,13 +1011,16 @@ def searchDB(events, pbar, jobList, filename): # Textwrap translatedText = itemList[1][0] translatedText = textwrap.fill(translatedText, LISTWIDTH) - translatedText = font + translatedText + + # Font + if font: + translatedText = f"{font}{translatedText}" # Set Data dataList[j].update({"value": translatedText}) itemList[1].pop(0) # Description - if "選択肢10" in dataList[j].get("name"): + if "説明文　　　　　" in dataList[j].get("name"): # Pass 1 (Grab Data) if setData == False: if dataList[j].get("value") != "": @@ -1036,13 +1038,16 @@ def searchDB(events, pbar, jobList, filename): # Textwrap translatedText = itemList[2][0] translatedText = textwrap.fill(translatedText, LISTWIDTH) - translatedText = font + translatedText + + # Font + if font: + translatedText = f"{font}{translatedText}" # Set Data dataList[j].update({"value": translatedText}) itemList[2].pop(0) # Description - if "選択肢11" in dataList[j].get("name"): + if "┗未作成時" in dataList[j].get("name"): # Pass 1 (Grab Data) if setData == False: if dataList[j].get("value") != "": @@ -1060,7 +1065,10 @@ def searchDB(events, pbar, jobList, filename): # Textwrap translatedText = itemList[3][0] translatedText = textwrap.fill(translatedText, LISTWIDTH) - translatedText = font + translatedText + + # Font + if font: + translatedText = f"{font}{translatedText}" # Set Data dataList[j].update({"value": translatedText}) @@ -1202,14 +1210,14 @@ def searchDB(events, pbar, jobList, filename): weaponsList[1].pop(0) # Grab Skills - if table["name"] == "戦闘コマンド" and SKILLFLAG == True: + if table["name"] == "技能" and SKILLFLAG == True: for skill in table["data"]: dataList = skill["data"] # Parse for j in range(len(dataList)): # Name - if "コマンド名" in dataList[j].get("name"): + if "技能の名前" in dataList[j].get("name"): # Pass 1 (Grab Data) if setData == False: if dataList[j].get("value") != "": @@ -1222,7 +1230,7 @@ def searchDB(events, pbar, jobList, filename): skillList[0].pop(0) # Description - if "コマンドの説明文" in dataList[j].get("name"): + if "説明" in dataList[j].get("name"): # Pass 1 (Grab Data) if setData == False: if dataList[j].get("value") != "": @@ -1246,33 +1254,142 @@ def searchDB(events, pbar, jobList, filename): dataList[j].update({"value": translatedText}) skillList[1].pop(0) - # Grab Collection - if table["name"] == "鍛冶師用DB" and COLLECTIONFLAG == True: - for object in table["data"]: - dataList = object["data"] + # Log + if "使用時文章[移動](人名~" in dataList[j].get("name"): + # Pass 1 (Grab Data) + if setData == False: + if dataList[j].get("value") != "": + # Remove Textwrap + jaString = dataList[j].get("value") + jaString = jaString.replace("\n", " ") + jaString = jaString.replace("\r", "") + jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString) + + # Skill Action + if jaString[0] in [ + "は", + "を", + "の", + "に", + "が", + ]: + jaString = f"Taro{jaString}" + + # Append Data + skillList[2].append(jaString) + # Pass 2 (Set Data) + else: + if dataList[j].get("value") != "": + # Textwrap + translatedText = skillList[2][0] + translatedText = textwrap.fill(translatedText, LISTWIDTH) + translatedText = font + translatedText + + # Remove Taro + translatedText = re.sub(r"\bTaro\b", "", translatedText) + + # Set Data + dataList[j].update({"value": translatedText}) + skillList[2].pop(0) + + # Log + if "使用時文章[戦闘](人名~" in dataList[j].get("name"): + # Pass 1 (Grab Data) + if setData == False: + if dataList[j].get("value") != "": + # Remove Textwrap + jaString = dataList[j].get("value") + jaString = jaString.replace("\n", " ") + jaString = jaString.replace("\r", "") + jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString) + + # Skill Action + if jaString[0] in [ + "は", + "を", + "の", + "に", + "が", + ]: + jaString = f"Taro{jaString}" + + # Append Data + skillList[3].append(jaString) + # Pass 2 (Set Data) + else: + if dataList[j].get("value") != "": + # Textwrap + translatedText = skillList[3][0] + translatedText = textwrap.fill(translatedText, LISTWIDTH) + translatedText = font + translatedText + + # Remove Taro + translatedText = re.sub(r"\bTaro\b", "", translatedText) + + # Set Data + dataList[j].update({"value": translatedText}) + skillList[3].pop(0) + + # Log + if "失敗時文章[(対象)～]" in dataList[j].get("name"): + # Pass 1 (Grab Data) + if setData == False: + if dataList[j].get("value") != "": + # Remove Textwrap + jaString = dataList[j].get("value") + jaString = jaString.replace("\n", " ") + jaString = jaString.replace("\r", "") + jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString) + + # Skill Action + if jaString[0] in [ + "は", + "を", + "の", + "に", + "が", + ]: + jaString = f"Taro{jaString}" + + # Append Data + skillList[4].append(jaString) + # Pass 2 (Set Data) + else: + if dataList[j].get("value") != "": + # Textwrap + translatedText = skillList[4][0] + translatedText = textwrap.fill(translatedText, LISTWIDTH) + translatedText = font + translatedText + + # Remove Taro + translatedText = re.sub(r"\bTaro\b", "", translatedText) + + # Set Data + dataList[j].update({"value": translatedText}) + skillList[4].pop(0) + + # Grab States + if table["name"] == "状態設定" and STATEFLAG == True: + for state in table["data"]: + dataList = state["data"] # Parse for j in range(len(dataList)): # Name - if "作る装備" in dataList[j].get("name"): + if "状態名" in dataList[j].get("name"): # Pass 1 (Grab Data) if setData == False: if dataList[j].get("value") != "": - # Remove Textwrap - jaString = dataList[j].get("value") - jaString = jaString.replace("\n", " ") - jaString = jaString.replace("\r", "") - jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString) - collectionList[0].append(jaString) + stateList[0].append(dataList[j].get("value")) # Pass 2 (Set Data) else: if dataList[j].get("value") != "": - dataList[j].update({"value": collectionList[0][0]}) - collectionList[0].pop(0) + dataList[j].update({"value": stateList[0][0]}) + stateList[0].pop(0) # Description - if "品物の解説" in dataList[j].get("name"): + if "表示名" in dataList[j].get("name"): # Pass 1 (Grab Data) if setData == False: if dataList[j].get("value") != "": @@ -1282,29 +1399,22 @@ def searchDB(events, pbar, jobList, filename): jaString = jaString.replace("\r", "") jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString) - # Skill Action (Optional) - # jaString = f'Taro{jaString}' - # Append Data - collectionList[1].append(jaString) + stateList[1].append(jaString) # Pass 2 (Set Data) else: if dataList[j].get("value") != "": - translatedText = collectionList[1][0] - - # Remove Action (Optional) - # translatedText = translatedText.replace('Taro', '') - # Textwrap + translatedText = stateList[1][0] translatedText = textwrap.fill(translatedText, LISTWIDTH) translatedText = font + translatedText # Set Data dataList[j].update({"value": translatedText}) - collectionList[1].pop(0) + stateList[1].pop(0) - # Description 2 - if "NULL" in dataList[j].get("name"): + # Log + if "発生時の文章[(人名)～]" in dataList[j].get("name"): # Pass 1 (Grab Data) if setData == False: if dataList[j].get("value") != "": @@ -1314,37 +1424,233 @@ def searchDB(events, pbar, jobList, filename): jaString = jaString.replace("\r", "") jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString) - # Skill Action (Optional) - # jaString = f'Taro{jaString}' + # State Action + if jaString[0] in [ + "は", + "を", + "の", + "に", + "が", + ]: + jaString = f"Taro{jaString}" # Append Data - collectionList[2].append(jaString) + stateList[2].append(jaString) # Pass 2 (Set Data) else: if dataList[j].get("value") != "": - translatedText = collectionList[2][0] - - # Remove Action (Optional) - # translatedText = translatedText.replace('Taro', '') - # Textwrap + translatedText = stateList[2][0] translatedText = textwrap.fill(translatedText, LISTWIDTH) translatedText = font + translatedText + # Remove Taro + translatedText = re.sub(r"\bTaro\b", "", translatedText) + # Set Data dataList[j].update({"value": translatedText}) - collectionList[2].pop(0) + stateList[2].pop(0) + + # Log + if "行動制限時文章(空欄:ﾅｼ" in dataList[j].get("name"): + # Pass 1 (Grab Data) + if setData == False: + if dataList[j].get("value") != "": + # Remove Textwrap + jaString = dataList[j].get("value") + jaString = jaString.replace("\n", " ") + jaString = jaString.replace("\r", "") + jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString) + + # State Action + if jaString[0] in [ + "は", + "を", + "の", + "に", + "が", + ]: + jaString = f"Taro{jaString}" + + # Append Data + stateList[3].append(jaString) + # Pass 2 (Set Data) + else: + if dataList[j].get("value") != "": + # Textwrap + translatedText = stateList[3][0] + translatedText = textwrap.fill(translatedText, LISTWIDTH) + translatedText = font + translatedText + + # Remove Taro + translatedText = re.sub(r"\bTaro\b", "", translatedText) + + # Set Data + dataList[j].update({"value": translatedText}) + stateList[3].pop(0) + + # Log + if "回復時の文章[(人名)～]" in dataList[j].get("name"): + # Pass 1 (Grab Data) + if setData == False: + if dataList[j].get("value") != "": + # Remove Textwrap + jaString = dataList[j].get("value") + jaString = jaString.replace("\n", " ") + jaString = jaString.replace("\r", "") + jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString) + + # State Action + if jaString[0] in [ + "は", + "を", + "の", + "に", + "が", + ]: + jaString = f"Taro{jaString}" + + # Append Data + stateList[4].append(jaString) + # Pass 2 (Set Data) + else: + if dataList[j].get("value") != "": + # Textwrap + translatedText = stateList[4][0] + translatedText = textwrap.fill(translatedText, LISTWIDTH) + translatedText = font + translatedText + + # Remove Taro + translatedText = re.sub(r"\bTaro\b", "", translatedText) + + # Set Data + dataList[j].update({"value": translatedText}) + stateList[4].pop(0) + # Log + if "┣ ｶｳﾝﾀｰ発動文[対象～" in dataList[j].get("name"): + # Pass 1 (Grab Data) + if setData == False: + if dataList[j].get("value") != "": + # Remove Textwrap + jaString = dataList[j].get("value") + jaString = jaString.replace("\n", " ") + jaString = jaString.replace("\r", "") + jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString) + + # State Action + if jaString[0] in [ + "は", + "を", + "の", + "に", + "が", + ]: + jaString = f"Taro{jaString}" + + # Append Data + stateList[5].append(jaString) + # Pass 2 (Set Data) + else: + if dataList[j].get("value") != "": + # Textwrap + translatedText = stateList[5][0] + translatedText = textwrap.fill(translatedText, LISTWIDTH) + translatedText = font + translatedText + + # Remove Taro + translatedText = re.sub(r"\bTaro\b", "", translatedText) + + # Set Data + dataList[j].update({"value": translatedText}) + stateList[5].pop(0) + + # Log + if "尻もち　行動不能　持続3ターン" in dataList[j].get("name"): + # Pass 1 (Grab Data) + if setData == False: + if dataList[j].get("value") != "": + # Remove Textwrap + jaString = dataList[j].get("value") + jaString = jaString.replace("\n", " ") + jaString = jaString.replace("\r", "") + jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString) + + # State Action + if jaString[0] in [ + "は", + "を", + "の", + "に", + "が", + ]: + jaString = f"Taro{jaString}" + + # Append Data + stateList[6].append(jaString) + # Pass 2 (Set Data) + else: + if dataList[j].get("value") != "": + # Textwrap + translatedText = stateList[6][0] + translatedText = textwrap.fill(translatedText, LISTWIDTH) + translatedText = font + translatedText + + # Remove Taro + translatedText = re.sub(r"\bTaro\b", "", translatedText) + + # Set Data + dataList[j].update({"value": translatedText}) + stateList[6].pop(0) + + # Log + if "状態異常の説明" in dataList[j].get("name"): + # Pass 1 (Grab Data) + if setData == False: + if dataList[j].get("value") != "": + # Remove Textwrap + jaString = dataList[j].get("value") + jaString = jaString.replace("\n", " ") + jaString = jaString.replace("\r", "") + jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString) + + # State Action + if jaString[0] in [ + "は", + "を", + "の", + "に", + "が", + ]: + jaString = f"Taro{jaString}" + + # Append Data + stateList[7].append(jaString) + # Pass 2 (Set Data) + else: + if dataList[j].get("value") != "": + # Textwrap + translatedText = stateList[7][0] + translatedText = textwrap.fill(translatedText, LISTWIDTH) + translatedText = font + translatedText + + # Remove Taro + translatedText = re.sub(r"\bTaro\b", "", translatedText) + + # Set Data + dataList[j].update({"value": translatedText}) + stateList[7].pop(0) # Translation scenarioListTL = [[], [], []] npcListTL = [[], [], [], []] itemListTL = [[], [], [], []] - collectionListTL = [[], [], [], []] + stateListTL = [[], [], [], [], [], [], [], []] armorListTL = [[], []] enemyListTL = [[], []] weaponsListTL = [[], [], []] - skillListTL = [[], [], []] + skillListTL = [[], [], [], [], []] optionsListTL = [[], [], [], []] + dbNameListTL = [[]] translate = False @@ -1634,46 +1940,82 @@ def searchDB(events, pbar, jobList, filename): nameListTL = response[0] totalTokens[0] += response[1][0] totalTokens[1] += response[1][1] - # Desc 1 - response = translateGPT(skillList[1], "", True) + + # Desc + response = translateGPT( + skillList[1], + "Reply with only the " + LANGUAGE + " translation of the RPG skill description", + True, + ) descListTL1 = response[0] totalTokens[0] += response[1][0] totalTokens[1] += response[1][1] - # Desc 2 - response = translateGPT(skillList[2], "", True) + + # Log 1 + response = translateGPT( + skillList[2], + "reply with only the gender neutral " + + LANGUAGE + + " translation of the action log. Always start the sentence with Taro. For example, Translate 'Taroを倒した！' as 'Taro was defeated!'", + True, + ) descListTL2 = response[0] totalTokens[0] += response[1][0] totalTokens[1] += response[1][1] + # Log 2 + response = translateGPT( + skillList[3], + "reply with only the gender neutral " + + LANGUAGE + + " translation of the action log. Always start the sentence with Taro. For example, Translate 'Taroを倒した！' as 'Taro was defeated!'", + True, + ) + descListTL3 = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + + # Log 3 + response = translateGPT( + skillList[4], + "reply with only the gender neutral " + + LANGUAGE + + " translation of the action log. Always start the sentence with Taro. For example, Translate 'Taroを倒した！' as 'Taro was defeated!'", + True, + ) + descListTL4 = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + # Check Mismatch if ( len(nameListTL) != len(skillList[0]) or len(descListTL1) != len(skillList[1]) or len(descListTL2) != len(skillList[2]) + or len(descListTL3) != len(skillList[3]) + or len(descListTL4) != len(skillList[4]) ): with LOCK: if filename not in MISMATCH: MISMATCH.append(filename) else: - skillListTL = [nameListTL, descListTL1, descListTL2] + skillListTL = [nameListTL, descListTL1, descListTL2, descListTL3, descListTL4] translate = True - # Collection - for list in collectionList: + # State + for list in stateList: if len(list) > 0: # Progress Bar total = 0 - for collectionArray in collectionList: - total += len(collectionArray) + for stateArray in stateList: + total += len(stateArray) pbar.total = total pbar.refresh() # Name response = translateGPT( - collectionList[0], - "reply with only the gender neutral " - + LANGUAGE - + " translation of the action log. Always start the sentence with Taro. For example, Translate 'Taroを倒した！' as 'Taro was defeated!'", + stateList[0], + f"Reply with the {LANGUAGE} translation of the status effect.", True, ) nameListTL = response[0] @@ -1681,28 +2023,105 @@ def searchDB(events, pbar, jobList, filename): totalTokens[1] += response[1][1] # Desc 1 - response = translateGPT(collectionList[1], "", True) + response = translateGPT(stateList[1], "", True) descListTL1 = response[0] totalTokens[0] += response[1][0] totalTokens[1] += response[1][1] - # Desc 2 - response = translateGPT(collectionList[2], "", True) + # Log 1 + response = translateGPT( + stateList[2], + f"Reply with the {LANGUAGE} translation of the status effect.", + True, + ) descListTL2 = response[0] totalTokens[0] += response[1][0] totalTokens[1] += response[1][1] + # Log 2 + response = translateGPT( + stateList[3], + "reply with only the gender neutral " + + LANGUAGE + + " translation of the action log. Always start the sentence with Taro. For example, Translate 'Taroを倒した！' as 'Taro was defeated!'", + True, + ) + descListTL3 = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + + # Log 3 + response = translateGPT( + stateList[4], + "reply with only the gender neutral " + + LANGUAGE + + " translation of the action log. Always start the sentence with Taro. For example, Translate 'Taroを倒した！' as 'Taro was defeated!'", + True, + ) + descListTL4 = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + + # Log 4 + response = translateGPT( + stateList[5], + "reply with only the gender neutral " + + LANGUAGE + + " translation of the action log. Always start the sentence with Taro. For example, Translate 'Taroを倒した！' as 'Taro was defeated!'", + True, + ) + descListTL5 = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + + # Log 1 + response = translateGPT( + stateList[6], + "reply with only the gender neutral " + + LANGUAGE + + " translation of the action log. Always start the sentence with Taro. For example, Translate 'Taroを倒した！' as 'Taro was defeated!'", + True, + ) + descListTL6 = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + + # Log 1 + response = translateGPT( + stateList[7], + "reply with only the gender neutral " + + LANGUAGE + + " translation of the action log. Always start the sentence with Taro. For example, Translate 'Taroを倒した！' as 'Taro was defeated!'", + True, + ) + descListTL7 = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + # Check Mismatch if ( - len(nameListTL) != len(collectionList[0]) - or len(descListTL1) != len(collectionList[1]) - or len(descListTL2) != len(collectionList[2]) + len(nameListTL) != len(stateList[0]) + or len(descListTL1) != len(stateList[1]) + or len(descListTL2) != len(stateList[2]) + or len(descListTL3) != len(stateList[3]) + or len(descListTL4) != len(stateList[4]) + or len(descListTL5) != len(stateList[5]) + or len(descListTL6) != len(stateList[6]) ): with LOCK: if filename not in MISMATCH: MISMATCH.append(filename) else: - collectionListTL = [nameListTL, descListTL1, descListTL2] + stateListTL = [ + nameListTL, + descListTL1, + descListTL2, + descListTL3, + descListTL4, + descListTL5, + descListTL6, + descListTL7, + ] translate = True # OPTIONS @@ -1731,17 +2150,44 @@ def searchDB(events, pbar, jobList, filename): optionsListTL = [nameListTL] translate = True + # DB Names + if len(dbNameList[0]) > 0: + # Progress Bar + total = 0 + for dbNameArray in dbNameList: + total += len(dbNameArray) + pbar.total = total + pbar.refresh() + + # Name + response = translateGPT( + dbNameList[0], "Reply with only the " + LANGUAGE + " translation", True + ) + nameListTL = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + + # Check Mismatch + if len(nameListTL) != len(dbNameList[0]): + with LOCK: + if filename not in MISMATCH: + MISMATCH.append(filename) + else: + dbNameListTL = [nameListTL] + translate = True + # Start Pass 2 if translate == True: jobList.append(scenarioListTL) jobList.append(npcListTL) jobList.append(itemListTL) - jobList.append(collectionListTL) + jobList.append(stateListTL) jobList.append(armorListTL) jobList.append(enemyListTL) jobList.append(weaponsListTL) jobList.append(skillListTL) jobList.append(optionsListTL) + jobList.append(dbNameListTL) searchDB(events, pbar, jobList, filename) except IndexError as e: @@ -1817,10 +2263,7 @@ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{ {VOCAB}\n\ " ) - if format == "json": - user = f"```json\n{subbedT}\n```" - else: - user = subbedT + user = f"```json\n{subbedT}\n```" return system, user @@ -1835,10 +2278,7 @@ def translateText(system, user, history, penalty, format, model=MODEL): msg.append({"role": "system", "content": history}) # Response Format - if format == "json": - responseFormat = {"type": "json_object"} - else: - responseFormat = {"type": "text"} + responseFormat = {"type": "json_object"} # Content to TL msg.append({"role": "user", "content": f"{user}"}) @@ -1949,17 +2389,15 @@ def translateGPT(text, history, fullPromptFlag): for index, tItem in enumerate(tList): # Before sending to translation, if we have a list of items, add the formatting - if isinstance(tItem, list): - for j in range(len(tItem)): - if not tItem[j]: - tItem[j] = tItem[j].replace("", "Placeholder Text") - payload = {f"Line{i+1}": string for i, string in enumerate(tItem)} - payload = json.dumps(payload, indent=4, ensure_ascii=False) - varResponse = [payload, []] - subbedT = varResponse[0] - else: - varResponse = [tItem, []] - subbedT = varResponse[0] + if not isinstance(tItem, list): + tItem = [tItem] + for j in range(len(tItem)): + if not tItem[j]: + tItem[j] = tItem[j].replace("", "Placeholder Text") + payload = {f"Line{i+1}": string for i, string in enumerate(tItem)} + payload = json.dumps(payload, indent=4, ensure_ascii=False) + varResponse = [payload, []] + subbedT = varResponse[0] # Things to Check before starting translation if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+", subbedT): @@ -2027,4 +2465,7 @@ def translateGPT(text, history, fullPromptFlag): tList[index] = translatedText.replace("Placeholder Text", "") finalList = combineList(tList, text) - return [finalList, totalTokens] + if format == "json": + return [finalList, totalTokens] + else: + return [finalList[0], totalTokens] diff --git a/modules/wolf2.py b/modules/wolf2.py index e272e19..281e69b 100644 --- a/modules/wolf2.py +++ b/modules/wolf2.py @@ -353,40 +353,6 @@ def getSpeaker(speaker): return [speaker, [0, 0]] -def subVars(jaString): - jaString = jaString.replace("\u3000", " ") - - # Formatting - count = 0 - codeList = re.findall(r"[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]", jaString) - codeList = set(codeList) - if len(codeList) != 0: - for var in codeList: - jaString = jaString.replace(var, "[FCode_" + str(count) + "]") - count += 1 - - # Put all lists in list and return - return [jaString, codeList] - - -def resubVars(translatedText, codeList): - # Fix Spacing and ChatGPT Nonsense - matchList = re.findall(r"\[\s?.+?\s?\]", translatedText) - if len(matchList) > 0: - for match in matchList: - text = match.strip() - translatedText = translatedText.replace(match, text) - - # Formatting - count = 0 - if len(codeList) != 0: - for var in codeList: - translatedText = translatedText.replace("[FCode_" + str(count) + "]", var) - count += 1 - - return translatedText - - def batchList(input_list, batch_size): if not isinstance(batch_size, int) or batch_size <= 0: raise ValueError("batch_size must be a positive integer") @@ -394,54 +360,7 @@ def batchList(input_list, batch_size): return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)] -def createContext(fullPromptFlag, subbedT): - characters = "Game Characters:\n\ -ロラン (Roland) - Male\n\ -リュカ (Ryuka) - Male\n\ -レックス (Rex) - Male\n\ -タバサ (Tabasa) - Female\n\ -アルス (Ars) - Male\n\ -アマカラ (Amakara) - Male\n\ -エリー (Eri) - Female\n\ -リオ (Rio) - Female\n\ -サマル (Samal) - Male\n\ -ムーン (Moon) - Female\n\ -アリーナ (Arina) - Female\n\ -クリフト (Cliff) - Male\n\ -マーニャ (Manya) - Female\n\ -ミネア (Minea) - Female\n\ -デボラ (Debora) - Female\n\ -ビアンカ (Bianca) - Female\n\ -フローラ (Flora) - Female\n\ -バーバラ (Barbara) - Female\n\ -ミレーユ (Mireyu) - Female\n\ -アイラ (Aira) - Female\n\ -フォズ (Foz) - Female\n\ -マリベル (Maribel) - Female\n\ -ククール (Kukool) - Male\n\ -ゲルダ (Gerda) - Female\n\ -ゼシカ (Jessica) - Female\n\ -ヤンガス (Yangus) - Male\n\ -ラヴィエル (Raviel) - Female\n\ -セティア (Setia) - Female\n\ -ダイ (Dai) - Male\n\ -ヒュンケル (Hyunckel) - Male\n\ -ポップ (Pop) - Male\n\ -マァム (Maam) - Female\n\ -レオナ (Leona) - Female\n\ -アステア (Astea) - Female\n\ -イヨ (Iyo) - Female\n\ -ジャガン (Jagan) - Male\n\ -ヤオ (Yao) - Female\n\ -デイジィ (Daisy) - Female\n\ -バイシュン (Baishun) - Male\n\ -ブライ (Buraimu) - Male\n\ -ハッサン (Hassan) - Male\n\ -アロマ (Aroma) - Female\n\ -ピッケ (Pikke) - Female\n\ -ドラオ (Dorao) - Male\n\ -" - +def createContext(fullPromptFlag, subbedT, format): system = ( PROMPT + VOCAB if fullPromptFlag @@ -458,19 +377,13 @@ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{ {VOCAB}\n\ " ) - if isinstance(subbedT, list): - user = f"```json\n{subbedT}```" - else: - user = subbedT - return characters, system, user + user = f"```json\n{subbedT}\n```" + return system, user -def translateText(characters, system, user, history, penalty, format): +def translateText(system, user, history, penalty, format, model=MODEL): # Prompt - msg = [{"role": "system", "content": system + characters}] - - # Characters - msg.append({"role": "system", "content": characters}) + msg = [{"role": "system", "content": system}] # History if isinstance(history, list): @@ -479,17 +392,14 @@ def translateText(characters, system, user, history, penalty, format): msg.append({"role": "system", "content": history}) # Response Format - if format == "json": - responseFormat = {"type": "json_object"} - else: - responseFormat = {"type": "text"} + responseFormat = {"type": "json_object"} # Content to TL msg.append({"role": "user", "content": f"{user}"}) response = openai.chat.completions.create( temperature=0, frequency_penalty=penalty, - model=MODEL, + model=model, response_format=responseFormat, messages=msg, ) @@ -507,6 +417,8 @@ def cleanTranslatedText(translatedText, varResponse): "「": '\\"', "」": '\\"', "- ": "-", + "】": "]", + "【": "[", "Placeholder Text": "", # Add more replacements as needed } @@ -515,7 +427,6 @@ def cleanTranslatedText(translatedText, varResponse): # Elongate Long Dashes (Since GPT Ignores them...) translatedText = elongateCharacters(translatedText) - translatedText = resubVars(translatedText, varResponse[1]) return translatedText @@ -537,6 +448,7 @@ def elongateCharacters(text): def extractTranslation(translatedTextList, is_list): try: translatedTextList = re.sub(r'\\"+\"([^,\n}])', r'\\"\1', translatedTextList) + translatedTextList = re.sub(r"(?