Update Lune Translation function

modules/lune.py

@@ -41,6 +41,8 @@ BRFLAG = False  # If the game uses <br> instead
 FIXTEXTWRAP = True  # Overwrites textwrap
 IGNORETLTEXT = False  # Ignores all translated text.
 MISMATCH = []  # Lists files that throw a mismatch error (Length of GPT list response is wrong)
+PBAR = None
+FILENAME = None
 
 # tqdm Globals
 BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
@@ -61,8 +63,9 @@ elif "gpt-4" in MODEL:
 
 
 def handleLune(filename, estimate):
-    global ESTIMATE, totalTokens
+    global FILENAME, ESTIMATE, totalTokens
     ESTIMATE = estimate
+    FILENAME = filename
 
     if estimate:
         start = time.time()
@@ -176,6 +179,8 @@ def parseJSON(data, filename):
 
 
 def translateJSON(data, pbar):
+    global PBAR
+    PBAR = pbar
     textHistory = []
     batch = []
     maxHistory = MAXHISTORY
@@ -306,85 +311,60 @@ def translateJSON(data, pbar):
 # Save some money and enter the character before translation
 def getSpeaker(speaker):
     match speaker:
-        case "セレナ":
+        case "ファイン":
-            return ["Serena", [0, 0]]
+            return ["Fine", [0, 0]]
-        case "レナ":
+        case "":
-            return ["Rena", [0, 0]]
+            return ["", [0, 0]]
-        case "フィルス":
-            return ["Phils", [0, 0]]
-        case "レイン":
-            return ["Meryl", [0, 0]]
         case _:
-            return translateGPT(
+            # Store Speaker
-                speaker,
+            if speaker not in str(NAMESLIST):
-                "Reply with only the " + LANGUAGE + " translation of the NPC name.",
+                response = translateGPT(
-                False,
+                    speaker,
-            )
+                    "Reply with the " + LANGUAGE + " translation of the NPC name.",
+                    False,
+                )
+                response[0] = response[0].title()
+                response[0] = response[0].replace("'S", "'s")
+
+                # Retry if name doesn't translate for some reason
+                if re.search(r"([a-zA-Z？?])", response[0]) == None:
+                    response = translateGPT(
+                        speaker,
+                        "Reply with the " + LANGUAGE + " translation of the NPC name.",
+                        False,
+                    )
+                    response[0] = response[0].title()
+                    response[0] = response[0].replace("'S", "'s")
+
+                speakerList = [speaker, response[0]]
+                NAMESLIST.append(speakerList)
+                return response
+            # Find Speaker
+            else:
+                for i in range(len(NAMESLIST)):
+                    if speaker == NAMESLIST[i][0]:
+                        return [NAMESLIST[i][1], [0, 0]]
+
+    return [speaker, [0, 0]]
 
 
 def subVars(jaString):
     jaString = jaString.replace("\u3000", " ")
 
-    # Nested
-    count = 0
-    nestedList = re.findall(r"[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]", jaString)
-    nestedList = set(nestedList)
-    if len(nestedList) != 0:
-        for icon in nestedList:
-            jaString = jaString.replace(icon, "{Nested_" + str(count) + "}")
-            count += 1
-
-    # Icons
-    count = 0
-    iconList = re.findall(r"[\\]+[iIkKwWaA]+\[[0-9]+\]", jaString)
-    iconList = set(iconList)
-    if len(iconList) != 0:
-        for icon in iconList:
-            jaString = jaString.replace(icon, "{Ascii_" + str(count) + "}")
-            count += 1
-
-    # Colors
-    count = 0
-    colorList = re.findall(r"[\\]+[cC]\[[0-9]+\]", jaString)
-    colorList = set(colorList)
-    if len(colorList) != 0:
-        for color in colorList:
-            jaString = jaString.replace(color, "{Color_" + str(count) + "}")
-            count += 1
-
-    # Names
-    count = 0
-    nameList = re.findall(r"[\\]+[nN]\[.+?\]+", jaString)
-    nameList = set(nameList)
-    if len(nameList) != 0:
-        for name in nameList:
-            jaString = jaString.replace(name, "{Noun_" + str(count) + "}")
-            count += 1
-
-    # Variables
-    count = 0
-    varList = re.findall(r"[\\]+[vV]\[[0-9]+\]", jaString)
-    varList = set(varList)
-    if len(varList) != 0:
-        for var in varList:
-            jaString = jaString.replace(var, "{Var_" + str(count) + "}")
-            count += 1
-
     # Formatting
     count = 0
-    formatList = re.findall(r"[\\]+[\w]+\[.+?\]", jaString)
+    codeList = re.findall(r"[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]", jaString)
-    formatList = set(formatList)
+    codeList = set(codeList)
-    if len(formatList) != 0:
+    if len(codeList) != 0:
-        for var in formatList:
+        for var in codeList:
-            jaString = jaString.replace(var, "{FCode_" + str(count) + "}")
+            jaString = jaString.replace(var, "[FCode_" + str(count) + "]")
             count += 1
 
     # Put all lists in list and return
-    allList = [nestedList, iconList, colorList, nameList, varList, formatList]
+    return [jaString, codeList]
-    return [jaString, allList]
 
 
-def resubVars(translatedText, allList):
+def resubVars(translatedText, codeList):
     # Fix Spacing and ChatGPT Nonsense
     matchList = re.findall(r"\[\s?.+?\s?\]", translatedText)
     if len(matchList) > 0:
@@ -392,46 +372,11 @@ def resubVars(translatedText, allList):
             text = match.strip()
             translatedText = translatedText.replace(match, text)
 
-    # Nested
-    count = 0
-    if len(allList[0]) != 0:
-        for var in allList[0]:
-            translatedText = translatedText.replace("{Nested_" + str(count) + "}", var)
-            count += 1
-
-    # Icons
-    count = 0
-    if len(allList[1]) != 0:
-        for var in allList[1]:
-            translatedText = translatedText.replace("{Ascii_" + str(count) + "}", var)
-            count += 1
-
-    # Colors
-    count = 0
-    if len(allList[2]) != 0:
-        for var in allList[2]:
-            translatedText = translatedText.replace("{Color_" + str(count) + "}", var)
-            count += 1
-
-    # Names
-    count = 0
-    if len(allList[3]) != 0:
-        for var in allList[3]:
-            translatedText = translatedText.replace("{Noun_" + str(count) + "}", var)
-            count += 1
-
-    # Vars
-    count = 0
-    if len(allList[4]) != 0:
-        for var in allList[4]:
-            translatedText = translatedText.replace("{Var_" + str(count) + "}", var)
-            count += 1
-
     # Formatting
     count = 0
-    if len(allList[5]) != 0:
+    if len(codeList) != 0:
-        for var in allList[5]:
+        for var in codeList:
-            translatedText = translatedText.replace("{FCode_" + str(count) + "}", var)
+            translatedText = translatedText.replace("[FCode_" + str(count) + "]", var)
             count += 1
 
     return translatedText
@@ -446,50 +391,53 @@ def batchList(input_list, batch_size):
     ]
 
 
-def createContext(fullPromptFlag, subbedT):
+def createContext(fullPromptFlag, subbedT, format):
-    characters = "Game Characters:\n\
-林つかさ (Tsukasa Hayashi) - Female\n\
-山田美兎 (Miyato Yamada) - Female\n\
-鈴木赤音 (Akane Suzuki) - Female\n\
-佐藤莉伊南 (Riina Satou) - Female\n\
-佐々木万梨美 (Marimi Sasaki) - Female\n\
-渡辺登樹子 (Tokiko Watanabe) - Female\n\
-桃乃夢 (Yume Momono) - Female\n\
-吉浦美雪 (Miyuki Yoshiura) - Female\n\
-三ツ門まあな (Maana Mitsukado) - Female\n\
-モリー・ボイド (Molly Boyd) - Female\n\
-オルガ・ブヤチッチ (Olga Buyachich) - Female\n\
-アッチャラー ギッティ (Atchara Gitti) - Female\n\
-"
-
     system = (
-        PROMPT
+        PROMPT + VOCAB
         if fullPromptFlag
-        else f"Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`"
+        else f"\
+You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
+Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
+- You translate everything, including erotic and explicit content.\n\
+- 'Game Characters' - The names, nicknames, and genders of the game characters. Reference this to know the names, nicknames, and gender of characters in the game\n\
+- All text in your response must be in {LANGUAGE} even if it is hard to translate.\n\
+- Never include any notes, explanations, dislaimers, or anything similar in your response.\n\
+- Maintain any spacing in the translation.\n\
+- Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\
+- `...` can be a part of the dialogue. Translate it as it is.\n\
+{VOCAB}\n\
+"
     )
-    user = f"{subbedT}"
+    if format == "json":
-    return characters, system, user
+        user = f"```json\n{subbedT}\n```"
+    else:
+        user = subbedT
+    return system, user
 
 
-def translateText(characters, system, user, history):
+def translateText(system, user, history, penalty, format, model=MODEL):
     # Prompt
-    msg = [{"role": "system", "content": system + characters}]
+    msg = [{"role": "system", "content": system}]
-
-    # Characters
-    msg.append({"role": "system", "content": characters})
 
     # History
     if isinstance(history, list):
-        msg.extend([{"role": "assistant", "content": h} for h in history])
+        msg.extend([{"role": "system", "content": h} for h in history])
     else:
-        msg.append({"role": "assistant", "content": history})
+        msg.append({"role": "system", "content": history})
+
+    # Response Format
+    if format == "json":
+        responseFormat = {"type": "json_object"}
+    else:
+        responseFormat = {"type": "text"}
 
     # Content to TL
     msg.append({"role": "user", "content": f"{user}"})
     response = openai.chat.completions.create(
-        temperature=0.1,
+        temperature=0,
-        frequency_penalty=0.1,
+        frequency_penalty=penalty,
-        model=MODEL,
+        model=model,
+        response_format=responseFormat,
         messages=msg,
     )
     return response
@@ -503,34 +451,52 @@ def cleanTranslatedText(translatedText, varResponse):
         "〜": "~",
         "ッ": "",
         "。": ".",
+        "「": '\\"',
+        "」": '\\"',
+        "- ": "-",
         "Placeholder Text": "",
         # Add more replacements as needed
     }
     for target, replacement in placeholders.items():
         translatedText = translatedText.replace(target, replacement)
 
+    # Elongate Long Dashes (Since GPT Ignores them...)
+    translatedText = elongateCharacters(translatedText)
     translatedText = resubVars(translatedText, varResponse[1])
-    if "\n" in translatedText:
+    return translatedText
-        return [line for line in translatedText.split("\n") if line]
+
-    else:
+
-        return [line for line in translatedText.split("\\n") if line]
+def elongateCharacters(text):
+    # Define a pattern to match one character followed by one or more `ー` characters
+    # Using a positive lookbehind assertion to capture the preceding character
+    pattern = r"(?<=(.))ー+"
+
+    # Define a replacement function that elongates the captured character
+    def repl(match):
+        char = match.group(1)  # The character before the ー sequence
+        count = len(match.group(0)) - 1  # Number of ー characters
+        return char * count  # Replace ー sequence with the character repeated
+
+    # Use re.sub() to replace the pattern in the text
+    return re.sub(pattern, repl, text)
 
 
 def extractTranslation(translatedTextList, is_list):
-    pattern = r"<Line(\d+)>[\\]*`?(.*?)[\\]*?`?</?Line\d+>"
+    try:
-    # If it's a batch (i.e., list), extract with tags; otherwise, return the single item.
+        line_dict = json.loads(translatedTextList)
-    if is_list:
+        # If it's a batch (i.e., list), extract with tags; otherwise, return the single item.
-        return [
+        string_list = list(line_dict.values())
-            re.findall(pattern, line)[0][1]
+        if is_list:
-            for line in translatedTextList
+            return string_list
-            if re.search(pattern, line)
+        else:
-        ]
+            return string_list[0]
-    else:
+
-        matchList = re.findall(pattern, translatedTextList)
+    except Exception as e:
-        return matchList[0][1] if matchList else translatedTextList
+        print(f"extractTranslation Error: {e}")
+        return None
 
 
-def countTokens(characters, system, user, history):
+def countTokens(system, user, history):
     inputTotalTokens = 0
     outputTotalTokens = 0
     enc = tiktoken.encoding_for_model("gpt-4")
@@ -542,7 +508,6 @@ def countTokens(characters, system, user, history):
     else:
         inputTotalTokens += len(enc.encode(history))
     inputTotalTokens += len(enc.encode(system))
-    inputTotalTokens += len(enc.encode(characters))
     inputTotalTokens += len(enc.encode(user))
 
     # Output
@@ -559,19 +524,22 @@ def combineList(tlist, text):
 
 @retry(exceptions=Exception, tries=5, delay=5)
 def translateGPT(text, history, fullPromptFlag):
+    global PBAR, MISMATCH, FILENAME
+
+    mismatch = False
     totalTokens = [0, 0]
     if isinstance(text, list):
+        format = "json"
         tList = batchList(text, BATCHSIZE)
     else:
+        format = "text"
         tList = [text]
 
     for index, tItem in enumerate(tList):
         # Before sending to translation, if we have a list of items, add the formatting
         if isinstance(tItem, list):
-            payload = "\n".join(
+            payload = {f"Line{i+1}": string for i, string in enumerate(tItem)}
-                [f"<Line{i}>`{item}`</Line{i}>" for i, item in enumerate(tItem)]
+            payload = json.dumps(payload, indent=4, ensure_ascii=False)
-            )
-            payload = payload.replace("``", "`Placeholder Text`")
             varResponse = subVars(payload)
             subbedT = varResponse[0]
         else:
@@ -580,38 +548,67 @@ def translateGPT(text, history, fullPromptFlag):
 
         # Things to Check before starting translation
         if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", subbedT):
+            if PBAR is not None:
+                PBAR.update(len(tItem))
             continue
 
         # Create Message
-        characters, system, user = createContext(fullPromptFlag, subbedT)
+        system, user = createContext(fullPromptFlag, subbedT, format)
 
         # Calculate Estimate
         if ESTIMATE:
-            estimate = countTokens(characters, system, user, history)
+            estimate = countTokens(system, user, history)
             totalTokens[0] += estimate[0]
             totalTokens[1] += estimate[1]
             continue
 
         # Translating
-        response = translateText(characters, system, user, history)
+        response = translateText(system, user, history, 0.05, format)
         translatedText = response.choices[0].message.content
         totalTokens[0] += response.usage.prompt_tokens
         totalTokens[1] += response.usage.completion_tokens
 
-        # Formatting
+        # Check Translation
-        translatedTextList = cleanTranslatedText(translatedText, varResponse)
+        translatedText = cleanTranslatedText(translatedText, varResponse)
         if isinstance(tItem, list):
-            extractedTranslations = extractTranslation(translatedTextList, True)
+            extractedTranslations = extractTranslation(translatedText, True)
-            tList[index] = extractedTranslations
+            if extractedTranslations == None or len(tItem) != len(
-            if len(tItem) != len(translatedTextList):
+                extractedTranslations
-                mismatch = True  # Just here so breakpoint can be set
+            ):
-            history = extractedTranslations[-10:]  # Update history if we have a list
+                # Mismatch. Try Again
+                response = translateText(system, user, history, 0.05, format, "gpt-4o")
+                translatedText = response.choices[0].message.content
+                totalTokens[0] += response.usage.prompt_tokens
+                totalTokens[1] += response.usage.completion_tokens
+
+                # Formatting
+                translatedText = cleanTranslatedText(translatedText, varResponse)
+                if isinstance(tItem, list):
+                    extractedTranslations = extractTranslation(translatedText, True)
+                    if extractedTranslations == None or len(tItem) != len(
+                        extractedTranslations
+                    ):
+                        mismatch = True  # Just here for breakpoint
+
+            # Set if no mismatch
+            if mismatch == False:
+                tList[index] = extractedTranslations
+                history = extractedTranslations[
+                    -10:
+                ]  # Update history if we have a list
+            else:
+                history = text[-10:]
+                mismatch = False
+                if FILENAME not in MISMATCH:
+                    MISMATCH.append(FILENAME)
+
+            # Update Loading Bar
+            with LOCK:
+                if PBAR is not None:
+                    PBAR.update(len(tItem))
         else:
             # Ensure we're passing a single string to extractTranslation
-            extractedTranslations = extractTranslation(
+            tList[index] = translatedText
-                "\n".join(translatedTextList), False
-            )
-            tList[index] = extractedTranslations
 
     finalList = combineList(tList, text)
     return [finalList, totalTokens]

modules/rpgmakermvmz.py

@@ -69,7 +69,7 @@ LEAVE = False
 # Dialogue / Scroll
 CODE401 = False
 CODE405 = False
-CODE408 = True
+CODE408 = False # Warning, translates comments and can inflate costs.
 
 # Choices
 CODE102 = False
@@ -84,7 +84,7 @@ CODE101 = False
 CODE355655 = False
 CODE357 = False
 CODE657 = False
-CODE356 = False
+CODE356 = True
 CODE320 = False
 CODE324 = False
 CODE111 = False

modules/wolf.py

@@ -74,8 +74,8 @@ CODE122 = False
 
 # Other
 CODE210 = False
-CODE300 = True
+CODE300 = False
-CODE250 = False
+CODE250 = True
 
 # Database
 NPCFLAG = False
@@ -424,7 +424,7 @@ def searchCodes(events, pbar, jobList, filename):
                             ):
                                 # Remove Textwrap and Font and Add to list
                                 str = str.replace("\r\n", " ")
-                                str = str.replace(f"\\f[{fontSize}]", "")
+                                str = re.sub(r'[\\]+f\[\d+\]', '', str)
                                 list300.append(str)
 
                         # Pass 2
@@ -621,7 +621,7 @@ def searchCodes(events, pbar, jobList, filename):
                                 ):
                                     # Remove Textwrap and Font and Add to list
                                     str = str.replace("\r\n", " ")
-                                    str = str.replace(f"\\f[{fontSize}]", "")
+                                    str = re.sub(r'[\\]+f\[\d+\]', '', str)
                                     list300.append(str)
 
                             # Pass 2
@@ -676,7 +676,7 @@ def searchCodes(events, pbar, jobList, filename):
                 # Validate size
                 if len(codeList[i]["stringArgs"]) > 2:
                     if (
-                        codeList[i]["stringArgs"][1] == "所持商品"
+                        codeList[i]["stringArgs"][1] == "クエスト情報"
                         and codeList[i]["stringArgs"][2] != ""
                     ):
                         # Grab String
@@ -964,7 +964,7 @@ def searchDB(events, pbar, jobList, filename):
                             scenarioList[2].pop(0)
 
             # Grab Items
-            if table["name"] == "キャラクタプロフィール" and ITEMFLAG == True:
+            if table["name"] == "アイテム" and ITEMFLAG == True:
                 with open("translations.txt", "a", encoding="utf-8") as file:
                     for item in table["data"]:
                         dataList = item["data"]
@@ -972,7 +972,7 @@ def searchDB(events, pbar, jobList, filename):
                         # Parse #
                         for j in range(len(dataList)):
                             # Name
-                            if dataList[j].get("name") == "名前":
+                            if dataList[j].get("name") == "生息海域":
                                 # Pass 1 (Grab Data)
                                 if setData == False:
                                     if dataList[j].get("value") != "":
@@ -992,9 +992,9 @@ def searchDB(events, pbar, jobList, filename):
                                         itemList[0].pop(0)
 
                             # Description 1 (You are my specialz)
-                            if dataList[j].get("name") == "戦闘面での特徴":
+                            if dataList[j].get("name") == "説明":
                                 # Clean String
-                                fontSize = 18
+                                fontSize = 14
                                 translatedText = ""
                                 cleanedList = formatDramon(dataList[j].get("value"))
                                 for str in cleanedList:
@@ -1009,7 +1009,7 @@ def searchDB(events, pbar, jobList, filename):
                                         ):
                                             # Remove Textwrap and Font and Add to list
                                             str = str.replace("\r\n", " ")
-                                            str = str.replace(f"\\f[{fontSize}]", "")
+                                            str = re.sub(r'[\\]+f\[\d+\]', '', str)
                                             itemList[1].append(str)
 
                                     # Pass 2
@@ -1054,7 +1054,7 @@ def searchDB(events, pbar, jobList, filename):
                                     dataList[j].update({"value": translatedText})
 
                             # Description 2 (You are my specialz)
-                            if dataList[j].get("name") == "01_説明":
+                            if dataList[j].get("name") == "NULL":
                                 # Clean String
                                 fontSize = 24
                                 translatedText = ""
@@ -1071,7 +1071,7 @@ def searchDB(events, pbar, jobList, filename):
                                         ):
                                             # Remove Textwrap and Font and Add to list
                                             str = str.replace("\r\n", " ")
-                                            str = str.replace(f"\\f[{fontSize}]", "")
+                                            str = re.sub(r'[\\]+f\[\d+\]', '', str)
                                             itemList[2].append(str)
 
                                     # Pass 2
@@ -1116,7 +1116,7 @@ def searchDB(events, pbar, jobList, filename):
                                     dataList[j].update({"value": translatedText})
 
                             # Description 3 (You are my specialz)
-                            if dataList[j].get("name") == "01_説明未婚":
+                            if dataList[j].get("name") == "NULL":
                                 # Clean String
                                 fontSize = 24
                                 translatedText = ""
@@ -1133,7 +1133,7 @@ def searchDB(events, pbar, jobList, filename):
                                         ):
                                             # Remove Textwrap and Font and Add to list
                                             str = str.replace("\r\n", " ")
-                                            str = str.replace(f"\\f[{fontSize}]", "")
+                                            str = re.sub(r'[\\]+f\[\d+\]', '', str)
                                             itemList[3].append(str)
 
                                     # Pass 2

vocab.txt

@@ -1,11 +1,12 @@
 Here are some vocabulary and terms so that you know the proper spelling and translation.
 ```
 # Game Characters
-リエナ (Liena) - Female\n
+ジーナ (Gina) - Female\n\
-イル (Iru) - Male\n
+マナリス (Manaris) - Female\n\
-ヨーグ (Yorg) - Male\n
+ラビィ (Lavi) - Female\n\
-メリッサ (Melissa) - Female\n
+ララ (Lala) - Female\n\
-ナーサ (Naasa) - Female\n\
+クリス (Chris) - Female\n\
+モーガン (Morgan) - Male\n\
 
 # Lewd Terms
 マンコ (pussy)
@@ -74,4 +75,7 @@ ME 音量 (ME Volume)
 堕天使 (Fallen Angel)  
 鬼 (Oni)
 ローバー (Roper)
+
+# Locations
+トリス (Tris)
 ```