Sync wolf with rpgmv

modules/wolf.py

@@ -1776,12 +1776,13 @@ def getSpeaker(speaker):
             # Store Speaker
             if speaker not in str(NAMESLIST):
                 response = translateGPT(
-                    speaker,
+                    f'Speaker: {speaker}',
                     "Reply with the " + LANGUAGE + " translation of the NPC name.",
-                    False,
+                    True,
                 )
                 response[0] = response[0].title()
                 response[0] = response[0].replace("'S", "'s")
+                response[0] = response[0].replace("Speaker: ", "")
 
                 # Retry if name doesn't translate for some reason
                 if re.search(r"([a-zA-Z？?])", response[0]) == None:
@@ -1810,27 +1811,18 @@ def subVars(jaString):
 
     # Formatting
     count = 0
-    codeList = re.findall(r"[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]", jaString)
+    codeList = re.findall(r"[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+?\]", jaString)
     codeList = set(codeList)
     if len(codeList) != 0:
         for var in codeList:
             jaString = jaString.replace(var, "[FCode_" + str(count) + "]")
             count += 1
 
-    # WOLF Images
-    count = 0
-    humList = re.findall(r"(\\?r?\\?n?_.*?\d\r\n@?)", jaString)
-    humList = set(humList)
-    if len(humList) != 0:
-        for var in humList:
-            jaString = jaString.replace(var, "[ICode_" + str(count) + "]")
-            count += 1
-
     # Put all lists in list and return
-    return [jaString, [codeList, humList]]
+    return [jaString, codeList]
 
 
-def resubVars(translatedText, varList):
+def resubVars(translatedText, codeList):
     # Fix Spacing and ChatGPT Nonsense
     matchList = re.findall(r"\[\s?.+?\s?\]", translatedText)
     if len(matchList) > 0:
@@ -1840,18 +1832,11 @@ def resubVars(translatedText, varList):
 
     # Formatting
     count = 0
-    if len(varList[0]) != 0:
+    if len(codeList) != 0:
-        for var in varList[0]:
+        for var in codeList:
             translatedText = translatedText.replace("[FCode_" + str(count) + "]", var)
             count += 1
 
-    # Formatting
-    count = 0
-    if len(varList[1]) != 0:
-        for var in varList[1]:
-            translatedText = translatedText.replace("[ICode_" + str(count) + "]", var)
-            count += 1
-
     return translatedText
 
 
@@ -1864,54 +1849,7 @@ def batchList(input_list, batch_size):
     ]
 
 
-def createContext(fullPromptFlag, subbedT):
+def createContext(fullPromptFlag, subbedT, format):
-    characters = "Game Characters:\n\
-ロラン (Roland) - Male\n\
-リュカ (Ryuka) - Male\n\
-レックス (Rex) - Male\n\
-タバサ (Tabasa) - Female\n\
-アルス (Ars) - Male\n\
-アマカラ (Amakara) - Male\n\
-エリー (Eri) - Female\n\
-リオ (Rio) - Female\n\
-サマル (Samal) - Male\n\
-ムーン (Moon) - Female\n\
-アリーナ (Arina) - Female\n\
-クリフト (Cliff) - Male\n\
-マーニャ (Manya) - Female\n\
-ミネア (Minea) - Female\n\
-デボラ (Debora) - Female\n\
-ビアンカ (Bianca) - Female\n\
-フローラ (Flora) - Female\n\
-バーバラ (Barbara) - Female\n\
-ミレーユ (Mireyu) - Female\n\
-アイラ (Aira) - Female\n\
-フォズ (Foz) - Female\n\
-マリベル (Maribel) - Female\n\
-ククール (Kukool) - Male\n\
-ゲルダ (Gerda) - Female\n\
-ゼシカ (Jessica) - Female\n\
-ヤンガス (Yangus) - Male\n\
-ラヴィエル (Raviel) - Female\n\
-セティア (Setia) - Female\n\
-ダイ (Dai) - Male\n\
-ヒュンケル (Hyunckel) - Male\n\
-ポップ (Pop) - Male\n\
-マァム (Maam) - Female\n\
-レオナ (Leona) - Female\n\
-アステア (Astea) - Female\n\
-イヨ (Iyo) - Female\n\
-ジャガン (Jagan) - Male\n\
-ヤオ (Yao) - Female\n\
-デイジィ (Daisy) - Female\n\
-バイシュン (Baishun) - Male\n\
-ブライ (Buraimu) - Male\n\
-ハッサン (Hassan) - Male\n\
-アロマ (Aroma) - Female\n\
-ピッケ (Pikke) - Female\n\
-ドラオ (Dorao) - Male\n\
-"
-
     system = (
         PROMPT + VOCAB
         if fullPromptFlag
@@ -1928,20 +1866,17 @@ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{
 {VOCAB}\n\
 "
     )
-    if isinstance(subbedT, list):
+    if format == "json":
-        user = f"```json\n{subbedT}```"
+        user = f"```json\n{subbedT}\n```"
     else:
         user = subbedT
-    return characters, system, user
+    return system, user
 
 
-def translateText(characters, system, user, history, penalty, format):
+def translateText(system, user, history, penalty, format, model=MODEL):
     # Prompt
     msg = [{"role": "system", "content": system}]
 
-    # Characters
-    msg.append({"role": "system", "content": characters})
-
     # History
     if isinstance(history, list):
         msg.extend([{"role": "system", "content": h} for h in history])
@@ -1959,7 +1894,7 @@ def translateText(characters, system, user, history, penalty, format):
     response = openai.chat.completions.create(
         temperature=0,
         frequency_penalty=penalty,
-        model=MODEL,
+        model=model,
         response_format=responseFormat,
         messages=msg,
     )
@@ -2007,6 +1942,7 @@ def elongateCharacters(text):
 def extractTranslation(translatedTextList, is_list):
     try:
         translatedTextList = re.sub(r'\\"+\"([^,\n}])', r'\\"\1', translatedTextList)
+        translatedTextList = re.sub(r'(?<![\\])"+', r'"', translatedTextList)
         line_dict = json.loads(translatedTextList)
         # If it's a batch (i.e., list), extract with tags; otherwise, return the single item.
         string_list = list(line_dict.values())
@@ -2020,7 +1956,7 @@ def extractTranslation(translatedTextList, is_list):
         return None
 
 
-def countTokens(characters, system, user, history):
+def countTokens(system, user, history):
     inputTotalTokens = 0
     outputTotalTokens = 0
     enc = tiktoken.encoding_for_model("gpt-4")
@@ -2032,7 +1968,6 @@ def countTokens(characters, system, user, history):
     else:
         inputTotalTokens += len(enc.encode(history))
     inputTotalTokens += len(enc.encode(system))
-    inputTotalTokens += len(enc.encode(characters))
     inputTotalTokens += len(enc.encode(user))
 
     # Output
@@ -2050,92 +1985,92 @@ def combineList(tlist, text):
 @retry(exceptions=Exception, tries=5, delay=5)
 def translateGPT(text, history, fullPromptFlag):
     global PBAR, MISMATCH, FILENAME
-
+    with open("log/translationHistory.txt", "a+", encoding="utf-8") as logFile:
-    mismatch = False
+        mismatch = False
-    totalTokens = [0, 0]
+        totalTokens = [0, 0]
-    if isinstance(text, list):
+        if isinstance(text, list):
-        format = "json"
+            format = "json"
-        tList = batchList(text, BATCHSIZE)
+            tList = batchList(text, BATCHSIZE)
-    else:
-        format = "text"
-        tList = [text]
-
-    for index, tItem in enumerate(tList):
-        # Before sending to translation, if we have a list of items, add the formatting
-        if isinstance(tItem, list):
-            payload = {f"Line{i+1}": string for i, string in enumerate(tItem)}
-            payload = json.dumps(payload, indent=4, ensure_ascii=False)
-            varResponse = subVars(payload)
-            subbedT = varResponse[0]
         else:
-            varResponse = subVars(tItem)
+            format = "text"
-            subbedT = varResponse[0]
+            tList = [text]
 
-        # Things to Check before starting translation
+        for index, tItem in enumerate(tList):
-        if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", subbedT):
+            # Before sending to translation, if we have a list of items, add the formatting
-            if PBAR is not None:
+            if isinstance(tItem, list):
-                PBAR.update(len(tItem))
+                payload = {f"Line{i+1}": string for i, string in enumerate(tItem)}
-            continue
+                payload = json.dumps(payload, indent=4, ensure_ascii=False)
-
+                varResponse = subVars(payload)
-        # Create Message
+                subbedT = varResponse[0]
-        characters, system, user = createContext(fullPromptFlag, subbedT)
+                logFile.write(f'Input:\n{subbedT}\n')
-
-        # Calculate Estimate
-        if ESTIMATE:
-            estimate = countTokens(characters, system, user, history)
-            totalTokens[0] += estimate[0]
-            totalTokens[1] += estimate[1]
-            continue
-
-        # Translating
-        response = translateText(characters, system, user, history, 0.1, format)
-        translatedText = response.choices[0].message.content
-        totalTokens[0] += response.usage.prompt_tokens
-        totalTokens[1] += response.usage.completion_tokens
-
-        # Check Translation
-        translatedText = cleanTranslatedText(translatedText, varResponse)
-        if isinstance(tItem, list):
-            extractedTranslations = extractTranslation(translatedText, True)
-            if extractedTranslations == None or len(tItem) != len(
-                extractedTranslations
-            ):
-                # Mismatch. Try Again
-                response = translateText(
-                    characters, system, user, history, 0.05, format
-                )
-                translatedText = response.choices[0].message.content
-                totalTokens[0] += response.usage.prompt_tokens
-                totalTokens[1] += response.usage.completion_tokens
-
-                # Formatting
-                translatedText = cleanTranslatedText(translatedText, varResponse)
-                if isinstance(tItem, list):
-                    extractedTranslations = extractTranslation(translatedText, True)
-                    if extractedTranslations == None or len(tItem) != len(
-                        extractedTranslations
-                    ):
-                        mismatch = True  # Just here for breakpoint
-
-            # Set if no mismatch
-            if mismatch == False:
-                tList[index] = extractedTranslations
-                history = extractedTranslations[
-                    -10:
-                ]  # Update history if we have a list
             else:
-                history = text[-10:]
+                varResponse = subVars(tItem)
-                mismatch = False
+                subbedT = varResponse[0]
-                if FILENAME not in MISMATCH:
-                    MISMATCH.append(FILENAME)
 
-            # Update Loading Bar
+            # Things to Check before starting translation
-            with LOCK:
+            if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+", subbedT):
                 if PBAR is not None:
                     PBAR.update(len(tItem))
-        else:
+                continue
-            # Ensure we're passing a single string to extractTranslation
+
-            tList[index] = translatedText
+            # Create Message
+            system, user = createContext(fullPromptFlag, subbedT, format)
+
+            # Calculate Estimate
+            if ESTIMATE:
+                estimate = countTokens(system, user, history)
+                totalTokens[0] += estimate[0]
+                totalTokens[1] += estimate[1]
+                continue
+
+            # Translating
+            response = translateText(system, user, history, 0.05, format)
+            translatedText = response.choices[0].message.content
+            totalTokens[0] += response.usage.prompt_tokens
+            totalTokens[1] += response.usage.completion_tokens
+
+            # Check Translation
+            translatedText = cleanTranslatedText(translatedText, varResponse)
+            if isinstance(tItem, list):
+                extractedTranslations = extractTranslation(translatedText, True)
+                if extractedTranslations == None or len(tItem) != len(
+                    extractedTranslations
+                ):
+                    # Mismatch. Try Again
+                    response = translateText(system, user, history, 0.05, format, "gpt-4o")
+                    translatedText = response.choices[0].message.content
+                    totalTokens[0] += response.usage.prompt_tokens
+                    totalTokens[1] += response.usage.completion_tokens
+
+                    # Formatting
+                    translatedText = cleanTranslatedText(translatedText, varResponse)
+                    if isinstance(tItem, list):
+                        extractedTranslations = extractTranslation(translatedText, True)
+                        if extractedTranslations == None or len(tItem) != len(
+                            extractedTranslations
+                        ):
+                            mismatch = True  # Just here for breakpoint
+                logFile.write(f'Output:\n{translatedText}\n')
+
+                # Set if no mismatch
+                if mismatch == False:
+                    tList[index] = extractedTranslations
+                    history = extractedTranslations[
+                        -10:
+                    ]  # Update history if we have a list
+                else:
+                    history = text[-10:]
+                    mismatch = False
+                    if FILENAME not in MISMATCH:
+                        MISMATCH.append(FILENAME)
+
+                # Update Loading Bar
+                with LOCK:
+                    if PBAR is not None:
+                        PBAR.update(len(tItem))
+            else:
+                # Ensure we're passing a single string to extractTranslation
+                tList[index] = translatedText
 
     finalList = combineList(tList, text)
     return [finalList, totalTokens]

vocab.txt

@@ -1,16 +1,50 @@
 Here are some vocabulary and terms so that you know the proper spelling and translation.
 ```
 # Game Characters
-レオ (Leo) - Male
+ロラン (Roland) - Male\n\
-ルナ (Luna) - Female
+リュカ (Ryuka) - Male\n\
-クロネ (Kurone) - Female
+レックス (Rex) - Male\n\
-イビエル (Ivieru) - Female
+タバサ (Tabasa) - Female\n\
-コトリ (Kotori) - Female
+アルス (Ars) - Male\n\
-サキュ子 (Sakyuko) - Female
+アマカラ (Amakara) - Male\n\
-セリア (Seria) - Female
+エリー (Eri) - Female\n\
-メリル (Meryl) - Female
+リオ (Rio) - Female\n\
-ハヤト (Hayato) - Male
+サマル (Samal) - Male\n\
-サディ (Sadi) - Male
+ムーン (Moon) - Female\n\
+アリーナ (Arina) - Female\n\
+クリフト (Cliff) - Male\n\
+マーニャ (Manya) - Female\n\
+ミネア (Minea) - Female\n\
+デボラ (Debora) - Female\n\
+ビアンカ (Bianca) - Female\n\
+フローラ (Flora) - Female\n\
+バーバラ (Barbara) - Female\n\
+ミレーユ (Mireyu) - Female\n\
+アイラ (Aira) - Female\n\
+フォズ (Foz) - Female\n\
+マリベル (Maribel) - Female\n\
+ククール (Kukool) - Male\n\
+ゲルダ (Gerda) - Female\n\
+ゼシカ (Jessica) - Female\n\
+ヤンガス (Yangus) - Male\n\
+ラヴィエル (Raviel) - Female\n\
+セティア (Setia) - Female\n\
+ダイ (Dai) - Male\n\
+ヒュンケル (Hyunckel) - Male\n\
+ポップ (Pop) - Male\n\
+マァム (Maam) - Female\n\
+レオナ (Leona) - Female\n\
+アステア (Astea) - Female\n\
+イヨ (Iyo) - Female\n\
+ジャガン (Jagan) - Male\n\
+ヤオ (Yao) - Female\n\
+デイジィ (Daisy) - Female\n\
+バイシュン (Baishun) - Male\n\
+ブライ (Buraimu) - Male\n\
+ハッサン (Hassan) - Male\n\
+アロマ (Aroma) - Female\n\
+ピッケ (Pikke) - Female\n\
+ドラオ (Dorao) - Male\n\
 
 # Lewd Terms
 マンコ (pussy)