diff --git a/modules/alice.py b/modules/alice.py
index cf2c4ef..22da635 100644
--- a/modules/alice.py
+++ b/modules/alice.py
@@ -1,599 +1,600 @@
-# Libraries
-import os
-import re
-import textwrap
-import threading
-import time
-import traceback
-import tiktoken
-import openai
-from pathlib import Path
-from colorama import Fore
-from dotenv import load_dotenv
-from retry import retry
-from tqdm import tqdm
-
-# Open AI
-load_dotenv()
-if os.getenv("api").replace(" ", "") != "":
- openai.base_url = os.getenv("api")
-openai.organization = os.getenv("org")
-openai.api_key = os.getenv("key")
-
-# Globals
-MODEL = os.getenv("model")
-TIMEOUT = int(os.getenv("timeout"))
-LANGUAGE = os.getenv("language").capitalize()
-PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
-VOCAB = Path("vocab.txt").read_text(encoding="utf-8")
-THREADS = int(os.getenv("threads"))
-LOCK = threading.Lock()
-WIDTH = int(os.getenv("width"))
-LISTWIDTH = int(os.getenv("listWidth"))
-NOTEWIDTH = 70
-MAXHISTORY = 10
-ESTIMATE = ""
-TOKENS = [0, 0]
-NAMESLIST = []
-NAMES = False # Output a list of all the character names found
-BRFLAG = False # If the game uses
instead
-FIXTEXTWRAP = True # Overwrites textwrap
-IGNORETLTEXT = False # Ignores all translated text.
-MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
-
-# tqdm Globals
-BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
-POSITION = 0
-LEAVE = False
-
-# Pricing - Depends on the model https://openai.com/pricing
-# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request
-# If you are getting a MISMATCH LENGTH error, lower the batch size.
-if "gpt-3.5" in MODEL:
- INPUTAPICOST = 0.002
- OUTPUTAPICOST = 0.002
- BATCHSIZE = 10
-elif "gpt-4" in MODEL:
- INPUTAPICOST = 0.01
- OUTPUTAPICOST = 0.03
- BATCHSIZE = 1
-else:
- INPUTAPICOST = float(os.getenv("input_cost"))
- OUTPUTAPICOST = float(os.getenv("output_cost"))
- BATCHSIZE = int(os.getenv("batchsize"))
- FREQUENCY_PENALTY = float(os.getenv("frequency_penalty"))
-
-def handleAlice(filename, estimate):
- global ESTIMATE
- totalTokens = [0, 0]
- ESTIMATE = estimate
-
- if estimate:
- start = time.time()
- translatedData = openFiles(filename)
-
- # Print Result
- end = time.time()
- tqdm.write(getResultString(translatedData, end - start, filename))
- with LOCK:
- totalTokens[0] += translatedData[1][0]
- totalTokens[1] += translatedData[1][1]
-
- # Print Total
- totalString = getResultString(["", totalTokens, None], end - start, "TOTAL")
-
- # Print any errors on maps
- if len(MISMATCH) > 0:
- return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
- else:
- return totalString
-
- else:
- try:
- with open("translated/" + filename, "w", encoding="UTF-8") as outFile:
- start = time.time()
- translatedData = openFiles(filename)
-
- # Print Result
- end = time.time()
- outFile.writelines(translatedData[0])
- tqdm.write(getResultString(translatedData, end - start, filename))
- with LOCK:
- totalTokens[0] += translatedData[1][0]
- totalTokens[1] += translatedData[1][1]
- except Exception:
- traceback.print_exc()
- return "Fail"
-
- return getResultString(["", totalTokens, None], end - start, "TOTAL")
-
-
-def openFiles(filename):
- with open("files/" + filename, "r", encoding="UTF-8") as f:
- translatedData = parseText(f, filename)
-
- return translatedData
-
-
-def getResultString(translatedData, translationTime, filename):
- # File Print String
- totalTokenstring = (
- Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
- "[Output: "
- + str(translatedData[1][1])
- + "]" "[Cost: ${:,.4f}".format((translatedData[1][0] * 0.001 * INPUTAPICOST) + (translatedData[1][1] * 0.001 * OUTPUTAPICOST))
- + "]"
- )
- timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
-
- if translatedData[2] == None:
- # Success
- return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET
-
- else:
- # Fail
- try:
- raise translatedData[2]
- except Exception as e:
- traceback.print_exc()
- errorString = str(e) + Fore.RED
- return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET
-
-
-def parseText(data, filename):
- # Get total for progress bar
- linesList = data.readlines()
- totalTokens = [0, 0]
- totalLines = len(linesList)
- global LOCK
-
- with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar:
- pbar.desc = filename
- pbar.total = totalLines
- try:
- result = translateLines(linesList, pbar)
- totalTokens[0] += result[1][0]
- totalTokens[1] += result[1][1]
- except Exception as e:
- traceback.print_exc()
- return [linesList, totalTokens, e]
- return [linesList, totalTokens, None]
-
-
-# Grab scenario data from text file
-def translateLines(linesList, pbar):
- currentGroup = []
- batch = []
- textHistory = []
- tokens = [0, 0]
- batchStartIndex = 0
- insertBool = False
- multiLine = False
- i = 0
-
- try:
- while i < len(linesList):
- # Check if Proper Message
- match = re.findall(r"s\[[0-9]+\] = \"(.*)\"", linesList[i])
- if len(match) > 0:
- jaString = match[0]
-
- # Skip Files
- if "／" in jaString:
- i += 1
- continue
-
- ### Translate
- # Remove any textwrap
- jaString = re.sub(r"\\n", " ", jaString)
-
- # Grab Speaker
- speakerMatch = re.findall(r"s\[[0-9]+\] = \"([^／]+)\"", linesList[i - 1])
- if len(speakerMatch) > 0:
- # If there isn't any Japanese in the text just skip
- if re.search(r"[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+", jaString) and "_" not in speakerMatch[0]:
- speaker = speakerMatch[0]
- else:
- speaker = ""
- else:
- speaker = ""
-
- # Grab rest of the messages
- currentGroup.append(jaString)
-
- # Check if next line should be merged
- if insertBool is True:
- linesList[i] = re.sub(r"(s\[[0-9]+\]) = \"(.+)\"", r'\1 = ""', linesList[i])
- linesList[i] = linesList[i].replace(";", "")
- start = i
- while len(linesList) > i + 1 and re.search(r"s\[[0-9]+\] = \"\s+(.*)\"", linesList[i + 1]) != None:
- multiLine = True
- i += 1
- match = re.findall(r"s\[[0-9]+\] = \"\s+(.*)\"", linesList[i])
- currentGroup.append(match[0])
- if insertBool is True:
- linesList[i] = re.sub(r"(s\[[0-9]+\]) = \"\s+(.+)\"", r'\1 = ""', linesList[i])
- linesList[i] = linesList[i].replace(";", "")
- i += 1
-
- # Combine Groups and Add Speaker
- finalJAString = " ".join(currentGroup)
- if speaker != "":
- finalJAString = f"{speaker}: {finalJAString}"
- else:
- finalJAString = f"{finalJAString}"
-
- # [Passthrough 1] Pulling From File
- if insertBool is False:
- # Append to List and Clear Values
- batch.append(finalJAString)
-
- # Translate Batch if Full
- if len(batch) == BATCHSIZE or i >= len(linesList) - 1:
- # Translate
- response = translateGPT(batch, textHistory, True)
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- translatedBatch = response[0]
- textHistory = translatedBatch[-10:]
-
- # Set Values
- if len(batch) == len(translatedBatch):
- i = batchStartIndex
- insertBool = True
-
- # Mismatch
- else:
- pbar.write(f"Mismatch: {batchStartIndex} - {i}")
- MISMATCH.append(batch)
- batchStartIndex = i
- batch.clear()
-
- multiLine = False
- currentGroup = []
-
- # [Passthrough 2] Setting Data
- else:
- # Get Text
- translatedText = translatedBatch[0]
-
- # Remove added speaker and quotes
- translatedText = re.sub(r"^.+?:\s", "", translatedText)
-
- # Textwrap
- translatedText = translatedText.replace('"', '\\"')
- translatedText = textwrap.fill(translatedText, width=WIDTH)
-
- # Set Data
- if multiLine:
- textList = translatedText.split("\n")
- for t in textList:
- translatedText = translatedText.replace(";", "")
- translatedText = re.sub(
- r"(s\[[0-9]+\]) = \"(.*)\"",
- rf'\1 = "{t}"',
- linesList[start],
- )
- translatedText = translatedText.replace(";", "")
- linesList[start] = translatedText
- pbar.update(1)
- start += 1
- multiLine = False
- translatedText = translatedText.replace(";", "")
- translatedBatch.pop(0)
- else:
- # Remove any textwrap
- translatedText = translatedText.replace("\n", " ")
- translatedText = re.sub(
- r"(s\[[0-9]+\]) = \"(.*)\"",
- rf'\1 = "{translatedText}"',
- linesList[start],
- )
- translatedText = translatedText.replace(";", "")
- linesList[start] = translatedText
- pbar.update(1)
- translatedBatch.pop(0)
-
- # If Batch is empty. Move on.
- if len(translatedBatch) == 0:
- insertBool = False
- batchStartIndex = i
- pbar.update(1)
- batch.clear()
-
- currentGroup = []
- else:
- if insertBool is True:
- pbar.update(1)
- i += 1
-
- return [linesList, tokens]
- except Exception:
- traceback.print_exc()
- return [linesList, tokens]
-
-
-def subVars(jaString):
- jaString = jaString.replace("\u3000", " ")
-
- # Nested
- count = 0
- nestedList = re.findall(r"[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]", jaString)
- nestedList = set(nestedList)
- if len(nestedList) != 0:
- for icon in nestedList:
- jaString = jaString.replace(icon, "{Nested_" + str(count) + "}")
- count += 1
-
- # Icons
- count = 0
- iconList = re.findall(r"[\\]+[iIkKwWaA]+\[[0-9]+\]", jaString)
- iconList = set(iconList)
- if len(iconList) != 0:
- for icon in iconList:
- jaString = jaString.replace(icon, "{Ascii_" + str(count) + "}")
- count += 1
-
- # Colors
- count = 0
- colorList = re.findall(r"[\\]+[cC]\[[0-9]+\]", jaString)
- colorList = set(colorList)
- if len(colorList) != 0:
- for color in colorList:
- jaString = jaString.replace(color, "{Color_" + str(count) + "}")
- count += 1
-
- # Names
- count = 0
- nameList = re.findall(r"[\\]+[nN]\[.+?\]+", jaString)
- nameList = set(nameList)
- if len(nameList) != 0:
- for name in nameList:
- jaString = jaString.replace(name, "{Noun_" + str(count) + "}")
- count += 1
-
- # Variables
- count = 0
- varList = re.findall(r"[\\]+[vV]\[[0-9]+\]", jaString)
- varList = set(varList)
- if len(varList) != 0:
- for var in varList:
- jaString = jaString.replace(var, "{Var_" + str(count) + "}")
- count += 1
-
- # Formatting
- count = 0
- formatList = re.findall(r"[\\]+[\w]+\[.+?\]", jaString)
- formatList = set(formatList)
- if len(formatList) != 0:
- for var in formatList:
- jaString = jaString.replace(var, "{FCode_" + str(count) + "}")
- count += 1
-
- # Put all lists in list and return
- allList = [nestedList, iconList, colorList, nameList, varList, formatList]
- return [jaString, allList]
-
-
-def resubVars(translatedText, allList):
- # Fix Spacing and ChatGPT Nonsense
- matchList = re.findall(r"\[\s?.+?\s?\]", translatedText)
- if len(matchList) > 0:
- for match in matchList:
- text = match.strip()
- translatedText = translatedText.replace(match, text)
-
- # Nested
- count = 0
- if len(allList[0]) != 0:
- for var in allList[0]:
- translatedText = translatedText.replace("{Nested_" + str(count) + "}", var)
- count += 1
-
- # Icons
- count = 0
- if len(allList[1]) != 0:
- for var in allList[1]:
- translatedText = translatedText.replace("{Ascii_" + str(count) + "}", var)
- count += 1
-
- # Colors
- count = 0
- if len(allList[2]) != 0:
- for var in allList[2]:
- translatedText = translatedText.replace("{Color_" + str(count) + "}", var)
- count += 1
-
- # Names
- count = 0
- if len(allList[3]) != 0:
- for var in allList[3]:
- translatedText = translatedText.replace("{Noun_" + str(count) + "}", var)
- count += 1
-
- # Vars
- count = 0
- if len(allList[4]) != 0:
- for var in allList[4]:
- translatedText = translatedText.replace("{Var_" + str(count) + "}", var)
- count += 1
-
- # Formatting
- count = 0
- if len(allList[5]) != 0:
- for var in allList[5]:
- translatedText = translatedText.replace("{FCode_" + str(count) + "}", var)
- count += 1
-
- return translatedText
-
-
-def batchList(input_list, batch_size):
- if not isinstance(batch_size, int) or batch_size <= 0:
- raise ValueError("batch_size must be a positive integer")
-
- return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)]
-
-
-def createContext(fullPromptFlag, subbedT):
- characters = "Game Characters:\n\
-林つかさ (Tsukasa Hayashi) - Female\n\
-山田美兎 (Miyato Yamada) - Female\n\
-鈴木赤音 (Akane Suzuki) - Female\n\
-佐藤莉伊南 (Riina Satou) - Female\n\
-佐々木万梨美 (Marimi Sasaki) - Female\n\
-渡辺登樹子 (Tokiko Watanabe) - Female\n\
-桃乃夢 (Yume Momono) - Female\n\
-吉浦美雪 (Miyuki Yoshiura) - Female\n\
-三ツ門まあな (Maana Mitsukado) - Female\n\
-モリー・ボイド (Molly Boyd) - Female\n\
-オルガ・ブヤチッチ (Olga Buyachich) - Female\n\
-アッチャラー ギッティ (Atchara Gitti) - Female\n\
-"
-
- system = (
- PROMPT
- if fullPromptFlag
- else f"Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`"
- )
- user = f"{subbedT}"
- return characters, system, user
-
-
-def translateText(characters, system, user, history):
- # Prompt
- msg = [{"role": "system", "content": system + characters}]
-
- # Characters
- msg.append({"role": "system", "content": characters})
-
- # History
- if isinstance(history, list):
- msg.extend([{"role": "assistant", "content": h} for h in history])
- else:
- msg.append({"role": "assistant", "content": history})
-
- # Content to TL
- msg.append({"role": "user", "content": f"{user}"})
- response = openai.chat.completions.create(
- temperature=0.1,
- frequency_penalty=0.1,
- model=MODEL,
- messages=msg,
- )
- return response
-
-
-def cleanTranslatedText(translatedText, varResponse):
- placeholders = {
- f"{LANGUAGE} Translation: ": "",
- "Translation: ": "",
- "っ": "",
- "〜": "~",
- "ッ": "",
- "。": ".",
- "Placeholder Text": "",
- # Add more replacements as needed
- }
- for target, replacement in placeholders.items():
- translatedText = translatedText.replace(target, replacement)
-
- translatedText = resubVars(translatedText, varResponse[1])
- if "\n" in translatedText:
- return [line for line in translatedText.split("\n") if line]
- else:
- return [line for line in translatedText.split("\\n") if line]
-
-
-def extractTranslation(translatedTextList, is_list):
- pattern = r"[\\]*`?(.*?)[\\]*?`?"
- # If it's a batch (i.e., list), extract with tags; otherwise, return the single item.
- if is_list:
- return [re.findall(pattern, line)[0][1] for line in translatedTextList if re.search(pattern, line)]
- else:
- matchList = re.findall(pattern, translatedTextList)
- return matchList[0][1] if matchList else translatedTextList
-
-
-def countTokens(characters, system, user, history):
- inputTotalTokens = 0
- outputTotalTokens = 0
- enc = tiktoken.encoding_for_model("gpt-4")
-
- # Input
- if isinstance(history, list):
- for line in history:
- inputTotalTokens += len(enc.encode(line))
- else:
- inputTotalTokens += len(enc.encode(history))
- inputTotalTokens += len(enc.encode(system))
- inputTotalTokens += len(enc.encode(characters))
- inputTotalTokens += len(enc.encode(user))
-
- # Output
- outputTotalTokens += round(len(enc.encode(user)) * 3)
-
- return [inputTotalTokens, outputTotalTokens]
-
-
-@retry(exceptions=Exception, tries=5, delay=5)
-def translateGPT(text, history, fullPromptFlag):
- totalTokens = [0, 0]
- if isinstance(text, list):
- tList = batchList(text, BATCHSIZE)
- else:
- tList = [text]
-
- for index, tItem in enumerate(tList):
- # Before sending to translation, if we have a list of items, add the formatting
- if isinstance(tItem, list):
- payload = "\n".join([f"`{item}`" for i, item in enumerate(tItem)])
- payload = payload.replace("``", "`Placeholder Text`")
- varResponse = subVars(payload)
- subbedT = varResponse[0]
- else:
- varResponse = subVars(tItem)
- subbedT = varResponse[0]
-
- # Things to Check before starting translation
- if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", subbedT):
- continue
-
- # Create Message
- characters, system, user = createContext(fullPromptFlag, subbedT)
-
- # Calculate Estimate
- if ESTIMATE:
- estimate = countTokens(characters, system, user, history)
- totalTokens[0] += estimate[0]
- totalTokens[1] += estimate[1]
- continue
-
- # Translating
- response = translateText(characters, system, user, history)
- translatedText = response.choices[0].message.content
- totalTokens[0] += response.usage.prompt_tokens
- totalTokens[1] += response.usage.completion_tokens
-
- # Formatting
- translatedTextList = cleanTranslatedText(translatedText, varResponse)
- if isinstance(tItem, list):
- extractedTranslations = extractTranslation(translatedTextList, True)
- tList[index] = extractedTranslations
- if len(tItem) != len(translatedTextList):
- mismatch = True # Just here so breakpoint can be set
- history = extractedTranslations[-10:] # Update history if we have a list
- else:
- # Ensure we're passing a single string to extractTranslation
- extractedTranslations = extractTranslation("\n".join(translatedTextList), False)
- tList[index] = extractedTranslations
-
- # Combine if multilist
- if isinstance(tList[0], list):
- tList = [t for sublist in tList for t in sublist]
-
- # Return
- if format == "json":
- return [tList, totalTokens]
- else:
- return [tList[0], totalTokens]
+# Libraries
+import os
+import re
+import textwrap
+import threading
+import time
+import traceback
+import tiktoken
+import openai
+from pathlib import Path
+from colorama import Fore
+from dotenv import load_dotenv
+from retry import retry
+from tqdm import tqdm
+
+# Open AI
+load_dotenv()
+if os.getenv("api").replace(" ", "") != "":
+ openai.base_url = os.getenv("api")
+openai.organization = os.getenv("org")
+openai.api_key = os.getenv("key")
+
+# Globals
+MODEL = os.getenv("model")
+TIMEOUT = int(os.getenv("timeout"))
+LANGUAGE = os.getenv("language").capitalize()
+PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
+VOCAB = Path("vocab.txt").read_text(encoding="utf-8")
+THREADS = int(os.getenv("threads"))
+LOCK = threading.Lock()
+WIDTH = int(os.getenv("width"))
+LISTWIDTH = int(os.getenv("listWidth"))
+NOTEWIDTH = 70
+MAXHISTORY = 10
+ESTIMATE = ""
+TOKENS = [0, 0]
+NAMESLIST = []
+NAMES = False # Output a list of all the character names found
+BRFLAG = False # If the game uses
instead
+FIXTEXTWRAP = True # Overwrites textwrap
+IGNORETLTEXT = False # Ignores all translated text.
+MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
+
+# tqdm Globals
+BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
+POSITION = 0
+LEAVE = False
+
+# Pricing - Depends on the model https://openai.com/pricing
+# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request
+# If you are getting a MISMATCH LENGTH error, lower the batch size.
+if "gpt-3.5" in MODEL:
+ INPUTAPICOST = 0.002
+ OUTPUTAPICOST = 0.002
+ BATCHSIZE = 10
+elif "gpt-4" in MODEL:
+ INPUTAPICOST = 0.01
+ OUTPUTAPICOST = 0.03
+ BATCHSIZE = 1
+else:
+ INPUTAPICOST = float(os.getenv("input_cost"))
+ OUTPUTAPICOST = float(os.getenv("output_cost"))
+ BATCHSIZE = int(os.getenv("batchsize"))
+ FREQUENCY_PENALTY = float(os.getenv("frequency_penalty"))
+
+
+def handleAlice(filename, estimate):
+ global ESTIMATE
+ totalTokens = [0, 0]
+ ESTIMATE = estimate
+
+ if estimate:
+ start = time.time()
+ translatedData = openFiles(filename)
+
+ # Print Result
+ end = time.time()
+ tqdm.write(getResultString(translatedData, end - start, filename))
+ with LOCK:
+ totalTokens[0] += translatedData[1][0]
+ totalTokens[1] += translatedData[1][1]
+
+ # Print Total
+ totalString = getResultString(["", totalTokens, None], end - start, "TOTAL")
+
+ # Print any errors on maps
+ if len(MISMATCH) > 0:
+ return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
+ else:
+ return totalString
+
+ else:
+ try:
+ with open("translated/" + filename, "w", encoding="UTF-8") as outFile:
+ start = time.time()
+ translatedData = openFiles(filename)
+
+ # Print Result
+ end = time.time()
+ outFile.writelines(translatedData[0])
+ tqdm.write(getResultString(translatedData, end - start, filename))
+ with LOCK:
+ totalTokens[0] += translatedData[1][0]
+ totalTokens[1] += translatedData[1][1]
+ except Exception:
+ traceback.print_exc()
+ return "Fail"
+
+ return getResultString(["", totalTokens, None], end - start, "TOTAL")
+
+
+def openFiles(filename):
+ with open("files/" + filename, "r", encoding="UTF-8") as f:
+ translatedData = parseText(f, filename)
+
+ return translatedData
+
+
+def getResultString(translatedData, translationTime, filename):
+ # File Print String
+ totalTokenstring = (
+ Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
+ "[Output: "
+ + str(translatedData[1][1])
+ + "]" "[Cost: ${:,.4f}".format((translatedData[1][0] * 0.001 * INPUTAPICOST) + (translatedData[1][1] * 0.001 * OUTPUTAPICOST))
+ + "]"
+ )
+ timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
+
+ if translatedData[2] == None:
+ # Success
+ return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET
+
+ else:
+ # Fail
+ try:
+ raise translatedData[2]
+ except Exception as e:
+ traceback.print_exc()
+ errorString = str(e) + Fore.RED
+ return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET
+
+
+def parseText(data, filename):
+ # Get total for progress bar
+ linesList = data.readlines()
+ totalTokens = [0, 0]
+ totalLines = len(linesList)
+ global LOCK
+
+ with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar:
+ pbar.desc = filename
+ pbar.total = totalLines
+ try:
+ result = translateLines(linesList, pbar)
+ totalTokens[0] += result[1][0]
+ totalTokens[1] += result[1][1]
+ except Exception as e:
+ traceback.print_exc()
+ return [linesList, totalTokens, e]
+ return [linesList, totalTokens, None]
+
+
+# Grab scenario data from text file
+def translateLines(linesList, pbar):
+ currentGroup = []
+ batch = []
+ textHistory = []
+ tokens = [0, 0]
+ batchStartIndex = 0
+ insertBool = False
+ multiLine = False
+ i = 0
+
+ try:
+ while i < len(linesList):
+ # Check if Proper Message
+ match = re.findall(r"s\[[0-9]+\] = \"(.*)\"", linesList[i])
+ if len(match) > 0:
+ jaString = match[0]
+
+ # Skip Files
+ if "／" in jaString:
+ i += 1
+ continue
+
+ ### Translate
+ # Remove any textwrap
+ jaString = re.sub(r"\\n", " ", jaString)
+
+ # Grab Speaker
+ speakerMatch = re.findall(r"s\[[0-9]+\] = \"([^／]+)\"", linesList[i - 1])
+ if len(speakerMatch) > 0:
+ # If there isn't any Japanese in the text just skip
+ if re.search(r"[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+", jaString) and "_" not in speakerMatch[0]:
+ speaker = speakerMatch[0]
+ else:
+ speaker = ""
+ else:
+ speaker = ""
+
+ # Grab rest of the messages
+ currentGroup.append(jaString)
+
+ # Check if next line should be merged
+ if insertBool is True:
+ linesList[i] = re.sub(r"(s\[[0-9]+\]) = \"(.+)\"", r'\1 = ""', linesList[i])
+ linesList[i] = linesList[i].replace(";", "")
+ start = i
+ while len(linesList) > i + 1 and re.search(r"s\[[0-9]+\] = \"\s+(.*)\"", linesList[i + 1]) != None:
+ multiLine = True
+ i += 1
+ match = re.findall(r"s\[[0-9]+\] = \"\s+(.*)\"", linesList[i])
+ currentGroup.append(match[0])
+ if insertBool is True:
+ linesList[i] = re.sub(r"(s\[[0-9]+\]) = \"\s+(.+)\"", r'\1 = ""', linesList[i])
+ linesList[i] = linesList[i].replace(";", "")
+ i += 1
+
+ # Combine Groups and Add Speaker
+ finalJAString = " ".join(currentGroup)
+ if speaker != "":
+ finalJAString = f"{speaker}: {finalJAString}"
+ else:
+ finalJAString = f"{finalJAString}"
+
+ # [Passthrough 1] Pulling From File
+ if insertBool is False:
+ # Append to List and Clear Values
+ batch.append(finalJAString)
+
+ # Translate Batch if Full
+ if len(batch) == BATCHSIZE or i >= len(linesList) - 1:
+ # Translate
+ response = translateGPT(batch, textHistory, True)
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ translatedBatch = response[0]
+ textHistory = translatedBatch[-10:]
+
+ # Set Values
+ if len(batch) == len(translatedBatch):
+ i = batchStartIndex
+ insertBool = True
+
+ # Mismatch
+ else:
+ pbar.write(f"Mismatch: {batchStartIndex} - {i}")
+ MISMATCH.append(batch)
+ batchStartIndex = i
+ batch.clear()
+
+ multiLine = False
+ currentGroup = []
+
+ # [Passthrough 2] Setting Data
+ else:
+ # Get Text
+ translatedText = translatedBatch[0]
+
+ # Remove added speaker and quotes
+ translatedText = re.sub(r"^.+?:\s", "", translatedText)
+
+ # Textwrap
+ translatedText = translatedText.replace('"', '\\"')
+ translatedText = textwrap.fill(translatedText, width=WIDTH)
+
+ # Set Data
+ if multiLine:
+ textList = translatedText.split("\n")
+ for t in textList:
+ translatedText = translatedText.replace(";", "")
+ translatedText = re.sub(
+ r"(s\[[0-9]+\]) = \"(.*)\"",
+ rf'\1 = "{t}"',
+ linesList[start],
+ )
+ translatedText = translatedText.replace(";", "")
+ linesList[start] = translatedText
+ pbar.update(1)
+ start += 1
+ multiLine = False
+ translatedText = translatedText.replace(";", "")
+ translatedBatch.pop(0)
+ else:
+ # Remove any textwrap
+ translatedText = translatedText.replace("\n", " ")
+ translatedText = re.sub(
+ r"(s\[[0-9]+\]) = \"(.*)\"",
+ rf'\1 = "{translatedText}"',
+ linesList[start],
+ )
+ translatedText = translatedText.replace(";", "")
+ linesList[start] = translatedText
+ pbar.update(1)
+ translatedBatch.pop(0)
+
+ # If Batch is empty. Move on.
+ if len(translatedBatch) == 0:
+ insertBool = False
+ batchStartIndex = i
+ pbar.update(1)
+ batch.clear()
+
+ currentGroup = []
+ else:
+ if insertBool is True:
+ pbar.update(1)
+ i += 1
+
+ return [linesList, tokens]
+ except Exception:
+ traceback.print_exc()
+ return [linesList, tokens]
+
+
+def subVars(jaString):
+ jaString = jaString.replace("\u3000", " ")
+
+ # Nested
+ count = 0
+ nestedList = re.findall(r"[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]", jaString)
+ nestedList = set(nestedList)
+ if len(nestedList) != 0:
+ for icon in nestedList:
+ jaString = jaString.replace(icon, "{Nested_" + str(count) + "}")
+ count += 1
+
+ # Icons
+ count = 0
+ iconList = re.findall(r"[\\]+[iIkKwWaA]+\[[0-9]+\]", jaString)
+ iconList = set(iconList)
+ if len(iconList) != 0:
+ for icon in iconList:
+ jaString = jaString.replace(icon, "{Ascii_" + str(count) + "}")
+ count += 1
+
+ # Colors
+ count = 0
+ colorList = re.findall(r"[\\]+[cC]\[[0-9]+\]", jaString)
+ colorList = set(colorList)
+ if len(colorList) != 0:
+ for color in colorList:
+ jaString = jaString.replace(color, "{Color_" + str(count) + "}")
+ count += 1
+
+ # Names
+ count = 0
+ nameList = re.findall(r"[\\]+[nN]\[.+?\]+", jaString)
+ nameList = set(nameList)
+ if len(nameList) != 0:
+ for name in nameList:
+ jaString = jaString.replace(name, "{Noun_" + str(count) + "}")
+ count += 1
+
+ # Variables
+ count = 0
+ varList = re.findall(r"[\\]+[vV]\[[0-9]+\]", jaString)
+ varList = set(varList)
+ if len(varList) != 0:
+ for var in varList:
+ jaString = jaString.replace(var, "{Var_" + str(count) + "}")
+ count += 1
+
+ # Formatting
+ count = 0
+ formatList = re.findall(r"[\\]+[\w]+\[.+?\]", jaString)
+ formatList = set(formatList)
+ if len(formatList) != 0:
+ for var in formatList:
+ jaString = jaString.replace(var, "{FCode_" + str(count) + "}")
+ count += 1
+
+ # Put all lists in list and return
+ allList = [nestedList, iconList, colorList, nameList, varList, formatList]
+ return [jaString, allList]
+
+
+def resubVars(translatedText, allList):
+ # Fix Spacing and ChatGPT Nonsense
+ matchList = re.findall(r"\[\s?.+?\s?\]", translatedText)
+ if len(matchList) > 0:
+ for match in matchList:
+ text = match.strip()
+ translatedText = translatedText.replace(match, text)
+
+ # Nested
+ count = 0
+ if len(allList[0]) != 0:
+ for var in allList[0]:
+ translatedText = translatedText.replace("{Nested_" + str(count) + "}", var)
+ count += 1
+
+ # Icons
+ count = 0
+ if len(allList[1]) != 0:
+ for var in allList[1]:
+ translatedText = translatedText.replace("{Ascii_" + str(count) + "}", var)
+ count += 1
+
+ # Colors
+ count = 0
+ if len(allList[2]) != 0:
+ for var in allList[2]:
+ translatedText = translatedText.replace("{Color_" + str(count) + "}", var)
+ count += 1
+
+ # Names
+ count = 0
+ if len(allList[3]) != 0:
+ for var in allList[3]:
+ translatedText = translatedText.replace("{Noun_" + str(count) + "}", var)
+ count += 1
+
+ # Vars
+ count = 0
+ if len(allList[4]) != 0:
+ for var in allList[4]:
+ translatedText = translatedText.replace("{Var_" + str(count) + "}", var)
+ count += 1
+
+ # Formatting
+ count = 0
+ if len(allList[5]) != 0:
+ for var in allList[5]:
+ translatedText = translatedText.replace("{FCode_" + str(count) + "}", var)
+ count += 1
+
+ return translatedText
+
+
+def batchList(input_list, batch_size):
+ if not isinstance(batch_size, int) or batch_size <= 0:
+ raise ValueError("batch_size must be a positive integer")
+
+ return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)]
+
+
+def createContext(fullPromptFlag, subbedT):
+ characters = "Game Characters:\n\
+林つかさ (Tsukasa Hayashi) - Female\n\
+山田美兎 (Miyato Yamada) - Female\n\
+鈴木赤音 (Akane Suzuki) - Female\n\
+佐藤莉伊南 (Riina Satou) - Female\n\
+佐々木万梨美 (Marimi Sasaki) - Female\n\
+渡辺登樹子 (Tokiko Watanabe) - Female\n\
+桃乃夢 (Yume Momono) - Female\n\
+吉浦美雪 (Miyuki Yoshiura) - Female\n\
+三ツ門まあな (Maana Mitsukado) - Female\n\
+モリー・ボイド (Molly Boyd) - Female\n\
+オルガ・ブヤチッチ (Olga Buyachich) - Female\n\
+アッチャラー ギッティ (Atchara Gitti) - Female\n\
+"
+
+ system = (
+ PROMPT
+ if fullPromptFlag
+ else f"Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`"
+ )
+ user = f"{subbedT}"
+ return characters, system, user
+
+
+def translateText(characters, system, user, history):
+ # Prompt
+ msg = [{"role": "system", "content": system + characters}]
+
+ # Characters
+ msg.append({"role": "system", "content": characters})
+
+ # History
+ if isinstance(history, list):
+ msg.extend([{"role": "assistant", "content": h} for h in history])
+ else:
+ msg.append({"role": "assistant", "content": history})
+
+ # Content to TL
+ msg.append({"role": "user", "content": f"{user}"})
+ response = openai.chat.completions.create(
+ temperature=0.1,
+ frequency_penalty=0.1,
+ model=MODEL,
+ messages=msg,
+ )
+ return response
+
+
+def cleanTranslatedText(translatedText, varResponse):
+ placeholders = {
+ f"{LANGUAGE} Translation: ": "",
+ "Translation: ": "",
+ "っ": "",
+ "〜": "~",
+ "ッ": "",
+ "。": ".",
+ "Placeholder Text": "",
+ # Add more replacements as needed
+ }
+ for target, replacement in placeholders.items():
+ translatedText = translatedText.replace(target, replacement)
+
+ translatedText = resubVars(translatedText, varResponse[1])
+ if "\n" in translatedText:
+ return [line for line in translatedText.split("\n") if line]
+ else:
+ return [line for line in translatedText.split("\\n") if line]
+
+
+def extractTranslation(translatedTextList, is_list):
+ pattern = r"[\\]*`?(.*?)[\\]*?`?"
+ # If it's a batch (i.e., list), extract with tags; otherwise, return the single item.
+ if is_list:
+ return [re.findall(pattern, line)[0][1] for line in translatedTextList if re.search(pattern, line)]
+ else:
+ matchList = re.findall(pattern, translatedTextList)
+ return matchList[0][1] if matchList else translatedTextList
+
+
+def countTokens(characters, system, user, history):
+ inputTotalTokens = 0
+ outputTotalTokens = 0
+ enc = tiktoken.encoding_for_model("gpt-4")
+
+ # Input
+ if isinstance(history, list):
+ for line in history:
+ inputTotalTokens += len(enc.encode(line))
+ else:
+ inputTotalTokens += len(enc.encode(history))
+ inputTotalTokens += len(enc.encode(system))
+ inputTotalTokens += len(enc.encode(characters))
+ inputTotalTokens += len(enc.encode(user))
+
+ # Output
+ outputTotalTokens += round(len(enc.encode(user)) * 3)
+
+ return [inputTotalTokens, outputTotalTokens]
+
+
+@retry(exceptions=Exception, tries=5, delay=5)
+def translateGPT(text, history, fullPromptFlag):
+ totalTokens = [0, 0]
+ if isinstance(text, list):
+ tList = batchList(text, BATCHSIZE)
+ else:
+ tList = [text]
+
+ for index, tItem in enumerate(tList):
+ # Before sending to translation, if we have a list of items, add the formatting
+ if isinstance(tItem, list):
+ payload = "\n".join([f"`{item}`" for i, item in enumerate(tItem)])
+ payload = payload.replace("``", "`Placeholder Text`")
+ varResponse = subVars(payload)
+ subbedT = varResponse[0]
+ else:
+ varResponse = subVars(tItem)
+ subbedT = varResponse[0]
+
+ # Things to Check before starting translation
+ if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", subbedT):
+ continue
+
+ # Create Message
+ characters, system, user = createContext(fullPromptFlag, subbedT)
+
+ # Calculate Estimate
+ if ESTIMATE:
+ estimate = countTokens(characters, system, user, history)
+ totalTokens[0] += estimate[0]
+ totalTokens[1] += estimate[1]
+ continue
+
+ # Translating
+ response = translateText(characters, system, user, history)
+ translatedText = response.choices[0].message.content
+ totalTokens[0] += response.usage.prompt_tokens
+ totalTokens[1] += response.usage.completion_tokens
+
+ # Formatting
+ translatedTextList = cleanTranslatedText(translatedText, varResponse)
+ if isinstance(tItem, list):
+ extractedTranslations = extractTranslation(translatedTextList, True)
+ tList[index] = extractedTranslations
+ if len(tItem) != len(translatedTextList):
+ mismatch = True # Just here so breakpoint can be set
+ history = extractedTranslations[-10:] # Update history if we have a list
+ else:
+ # Ensure we're passing a single string to extractTranslation
+ extractedTranslations = extractTranslation("\n".join(translatedTextList), False)
+ tList[index] = extractedTranslations
+
+ # Combine if multilist
+ if isinstance(tList[0], list):
+ tList = [t for sublist in tList for t in sublist]
+
+ # Return
+ if format == "json":
+ return [tList, totalTokens]
+ else:
+ return [tList[0], totalTokens]
diff --git a/modules/anim.py b/modules/anim.py
index 971c5de..186695c 100644
--- a/modules/anim.py
+++ b/modules/anim.py
@@ -1,570 +1,571 @@
-# Libraries
-import json
-import os
-import re
-import textwrap
-import threading
-import time
-import traceback
-import tiktoken
-import openai
-from pathlib import Path
-from colorama import Fore
-from dotenv import load_dotenv
-from retry import retry
-from tqdm import tqdm
-
-# Open AI
-load_dotenv()
-if os.getenv("api").replace(" ", "") != "":
- openai.base_url = os.getenv("api")
-openai.organization = os.getenv("org")
-openai.api_key = os.getenv("key")
-
-# Globals
-MODEL = os.getenv("model")
-TIMEOUT = int(os.getenv("timeout"))
-LANGUAGE = os.getenv("language").capitalize()
-PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
-VOCAB = Path("vocab.txt").read_text(encoding="utf-8")
-THREADS = int(os.getenv("threads"))
-LOCK = threading.Lock()
-WIDTH = int(os.getenv("width"))
-LISTWIDTH = int(os.getenv("listWidth"))
-NOTEWIDTH = 70
-MAXHISTORY = 10
-ESTIMATE = ""
-TOKENS = [0, 0]
-NAMESLIST = []
-NAMES = False # Output a list of all the character names found
-BRFLAG = False # If the game uses
instead
-FIXTEXTWRAP = True # Overwrites textwrap
-IGNORETLTEXT = False # Ignores all translated text.
-MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
-
-# tqdm Globals
-BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
-POSITION = 0
-LEAVE = False
-
-# Pricing - Depends on the model https://openai.com/pricing
-# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request
-# If you are getting a MISMATCH LENGTH error, lower the batch size.
-if "gpt-3.5" in MODEL:
- INPUTAPICOST = 0.002
- OUTPUTAPICOST = 0.002
- BATCHSIZE = 10
-elif "gpt-4" in MODEL:
- INPUTAPICOST = 0.01
- OUTPUTAPICOST = 0.03
- BATCHSIZE = 50
-else:
- INPUTAPICOST = float(os.getenv("input_cost"))
- OUTPUTAPICOST = float(os.getenv("output_cost"))
- BATCHSIZE = int(os.getenv("batchsize"))
- FREQUENCY_PENALTY = float(os.getenv("frequency_penalty"))
-
-def handleAnim(filename, estimate):
- global ESTIMATE
- totalTokens = [0, 0]
- ESTIMATE = estimate
-
- if estimate:
- start = time.time()
- translatedData = openFiles(filename)
-
- # Print Result
- end = time.time()
- tqdm.write(getResultString(translatedData, end - start, filename))
- with LOCK:
- totalTokens[0] += translatedData[1][0]
- totalTokens[1] += translatedData[1][1]
-
- # Print Total
- totalString = getResultString(["", totalTokens, None], end - start, "TOTAL")
-
- # Print any errors on maps
- if len(MISMATCH) > 0:
- return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
- else:
- return totalString
-
- else:
- try:
- with open("translated/" + filename, "w", encoding="UTF-8") as outFile:
- start = time.time()
- translatedData = openFiles(filename)
-
- # Print Result
- end = time.time()
- json.dump(translatedData[0], outFile, ensure_ascii=False, indent=4)
- tqdm.write(getResultString(translatedData, end - start, filename))
- with LOCK:
- totalTokens[0] += translatedData[1][0]
- totalTokens[1] += translatedData[1][1]
- except Exception:
- return "Fail"
-
- return getResultString(["", totalTokens, None], end - start, "TOTAL")
-
-
-def openFiles(filename):
- with open("files/" + filename, "r", encoding="UTF-8-sig") as f:
- data = json.load(f)
-
- # Map Files
- if ".json" in filename:
- translatedData = parseJSON(data, filename)
-
- else:
- raise NameError(filename + " Not Supported")
-
- return translatedData
-
-
-def getResultString(translatedData, translationTime, filename):
- # File Print String
- totalTokenstring = (
- Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
- "[Output: "
- + str(translatedData[1][1])
- + "]" "[Cost: ${:,.4f}".format((translatedData[1][0] * 0.001 * INPUTAPICOST) + (translatedData[1][1] * 0.001 * OUTPUTAPICOST))
- + "]"
- )
- timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
-
- if translatedData[2] == None:
- # Success
- return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET
-
- else:
- # Fail
- try:
- raise translatedData[2]
- except Exception as e:
- traceback.print_exc()
- errorString = str(e) + Fore.RED
- return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET
-
-
-def parseJSON(data, filename):
- keys = list(data.keys())
- batches = [keys[i : i + BATCHSIZE] for i in range(0, len(keys), BATCHSIZE)]
- totalTokens = [0, 0]
- totalLines = 0
- totalLines = len(batches)
- global LOCK
-
- with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar:
- pbar.desc = filename
- pbar.total = totalLines
- try:
- result = translateJSON(batches, data, pbar)
- totalTokens[0] += result[0]
- totalTokens[1] += result[1]
- except Exception as e:
- traceback.print_exc()
- return [data, totalTokens, e]
- return [data, totalTokens, None]
-
-
-def translateJSON(keys, data, pbar):
- translatedBatch = []
- textHistory = []
- tokens = [0, 0]
-
- for batch in keys:
- # Save Batch
- originalBatch = batch.copy()
-
- # If there isn't any Japanese in the text just skip
- needTL = False
- for i in range(len(batch)):
- t = data[batch[i]]
- if re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", t) or t == "":
- needTL = True
- if needTL is False and IGNORETLTEXT is True:
- pbar.update(1)
- continue
-
- # Remove any textwrap and Furigana
- for i in range(len(batch)):
- if FIXTEXTWRAP == True:
- # Textwrap
- data[originalBatch[i]] = data[originalBatch[i]].replace("@b", " ")
-
- # Furigana
- rcodeMatch = re.findall(r"(@\[(.+?):.+?\])", batch[i])
- if len(rcodeMatch) > 0:
- for match in rcodeMatch:
- batch[i] = batch[i].replace(match[0], match[1])
-
- # Translate
- if needTL is True:
- response = translateGPT(batch, textHistory, True)
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- translatedBatch = response[0]
- else:
- for i in range(len(originalBatch)):
- translatedBatch.append(data[originalBatch[i]])
-
- # Format and Set Text
- if len(batch) == len(translatedBatch):
- for i in range(len(translatedBatch)):
- # Remove added speaker
- translatedText = translatedBatch[i]
- translatedText = re.sub(r"^.+?\s\|\s?", "", translatedText)
-
- # Textwrap
- if "@n" in translatedText:
- match = re.search(r".*@n(.*)", translatedText)
- if match != None:
- tlText = match.group(1)
- tlText = textwrap.fill(tlText, width=WIDTH)
- tlText = tlText.replace("\n", "@b")
- translatedText = translatedText.replace(match.group(1), tlText)
-
- elif "@b" not in translatedText:
- translatedText = textwrap.fill(translatedText, width=WIDTH)
- translatedText = translatedText.replace("\n", "@b")
-
- # Set Data
- data[originalBatch[i]] = translatedText
- textHistory = translatedBatch
- translatedBatch.clear()
- # Mismatch, Skip Batch
- else:
- MISMATCH.append(batch)
- pbar.update(1)
- continue
- pbar.update(1)
-
- return tokens
-
-
-def subVars(jaString):
- jaString = jaString.replace("\u3000", " ")
-
- # Nested
- count = 0
- nestedList = re.findall(r"[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]", jaString)
- nestedList = set(nestedList)
- if len(nestedList) != 0:
- for icon in nestedList:
- jaString = jaString.replace(icon, "[Nested_" + str(count) + "]")
- count += 1
-
- # Icons
- count = 0
- iconList = re.findall(r"[\\]+[iIkKwWaA]+\[[0-9]+\]", jaString)
- iconList = set(iconList)
- if len(iconList) != 0:
- for icon in iconList:
- jaString = jaString.replace(icon, "[Ascii_" + str(count) + "]")
- count += 1
-
- # Colors
- count = 0
- colorList = re.findall(r"[\\]+[cC]\[[0-9]+\]", jaString)
- colorList = set(colorList)
- if len(colorList) != 0:
- for color in colorList:
- jaString = jaString.replace(color, "[Color_" + str(count) + "]")
- count += 1
-
- # Names
- count = 0
- nameList = re.findall(r"[\\]+[nN]\[.+?\]+", jaString)
- nameList = set(nameList)
- if len(nameList) != 0:
- for name in nameList:
- jaString = jaString.replace(name, "[Noun_" + str(count) + "]")
- count += 1
-
- # Variables
- count = 0
- varList = re.findall(r"[\\]+[vV]\[[0-9]+\]", jaString)
- varList = set(varList)
- if len(varList) != 0:
- for var in varList:
- jaString = jaString.replace(var, "[Var_" + str(count) + "]")
- count += 1
-
- # Formatting
- count = 0
- formatList = re.findall(r"[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]", jaString)
- formatList = set(formatList)
- if len(formatList) != 0:
- for var in formatList:
- jaString = jaString.replace(var, "[FCode_" + str(count) + "]")
- count += 1
-
- # Put all lists in list and return
- allList = [nestedList, iconList, colorList, nameList, varList, formatList]
- return [jaString, allList]
-
-
-def resubVars(translatedText, allList):
- # Fix Spacing and ChatGPT Nonsense
- matchList = re.findall(r"\[\s?.+?\s?\]", translatedText)
- if len(matchList) > 0:
- for match in matchList:
- text = match.strip()
- translatedText = translatedText.replace(match, text)
-
- # Nested
- count = 0
- if len(allList[0]) != 0:
- for var in allList[0]:
- translatedText = translatedText.replace("[Nested_" + str(count) + "]", var)
- count += 1
-
- # Icons
- count = 0
- if len(allList[1]) != 0:
- for var in allList[1]:
- translatedText = translatedText.replace("[Ascii_" + str(count) + "]", var)
- count += 1
-
- # Colors
- count = 0
- if len(allList[2]) != 0:
- for var in allList[2]:
- translatedText = translatedText.replace("[Color_" + str(count) + "]", var)
- count += 1
-
- # Names
- count = 0
- if len(allList[3]) != 0:
- for var in allList[3]:
- translatedText = translatedText.replace("[Noun_" + str(count) + "]", var)
- count += 1
-
- # Vars
- count = 0
- if len(allList[4]) != 0:
- for var in allList[4]:
- translatedText = translatedText.replace("[Var_" + str(count) + "]", var)
- count += 1
-
- # Formatting
- count = 0
- if len(allList[5]) != 0:
- for var in allList[5]:
- translatedText = translatedText.replace("[FCode_" + str(count) + "]", var)
- count += 1
-
- return translatedText
-
-
-def batchList(input_list, batch_size):
- if not isinstance(batch_size, int) or batch_size <= 0:
- raise ValueError("batch_size must be a positive integer")
-
- return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)]
-
-
-def createContext(fullPromptFlag, subbedT):
- characters = "Game Characters:\n\
-達也 (Tatsuya) - Male\n\
-香織 (Kaori) - Female\n\
-岩瀬 (Iwase)\n\
-万蔵 (Manzou) - Male\n\
-結奈 (Yuuna) - Female\n\
-茅部 (Kayabe)\n\
-"
-
- system = (
- PROMPT + VOCAB
- if fullPromptFlag
- else f"\
-You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
-Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
-- You translate everything, including erotic and explicit content.\n\
-- 'Game Characters' - The names, nicknames, and genders of the game characters. Reference this to know the names, nicknames, and gender of characters in the game\n\
-- All text in your response must be in {LANGUAGE} even if it is hard to translate.\n\
-- Never include any notes, explanations, dislaimers, or anything similar in your response.\n\
-- Maintain any spacing in the translation.\n\
-- Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\
-- `...` can be a part of the dialogue. Translate it as it is.\n\
-{VOCAB}\n\
-"
- )
- user = f"{subbedT}"
- return characters, system, user
-
-
-def translateText(characters, system, user, history, penalty):
- # Prompt
- msg = [{"role": "system", "content": system + characters}]
-
- # Characters
- msg.append({"role": "system", "content": characters})
-
- # History
- if isinstance(history, list):
- msg.extend([{"role": "system", "content": h} for h in history])
- else:
- msg.append({"role": "system", "content": history})
-
- # Content to TL
- msg.append({"role": "user", "content": f"{user}"})
- response = openai.chat.completions.create(
- temperature=0,
- frequency_penalty=penalty,
- model=MODEL,
- messages=msg,
- )
- return response
-
-
-def cleanTranslatedText(translatedText, varResponse):
- placeholders = {
- f"{LANGUAGE} Translation: ": "",
- "Translation: ": "",
- "っ": "",
- "〜": "~",
- "ッ": "",
- "。": ".",
- "Placeholder Text": "",
- "é": "e",
- "—": "-",
- "ū": "u",
- # Add more replacements as needed
- }
- for target, replacement in placeholders.items():
- translatedText = translatedText.replace(target, replacement)
-
- # Elongate Long Dashes (Since GPT Ignores them...)
- translatedText = elongateCharacters(translatedText)
- translatedText = resubVars(translatedText, varResponse[1])
- return translatedText
-
-
-def elongateCharacters(text):
- # Define a pattern to match one character followed by one or more `ー` characters
- # Using a positive lookbehind assertion to capture the preceding character
- pattern = r"(?<=(.))ー+"
-
- # Define a replacement function that elongates the captured character
- def repl(match):
- char = match.group(1) # The character before the ー sequence
- count = len(match.group(0)) - 1 # Number of ー characters
- return char * count # Replace ー sequence with the character repeated
-
- # Use re.sub() to replace the pattern in the text
- return re.sub(pattern, repl, text)
-
-
-def extractTranslation(translatedTextList, is_list):
- pattern = r"`?<[Ll]ine\d+>([\\]*.*?[\\]*?)<\/?[Ll]ine\d+>`?"
- # If it's a batch (i.e., list), extract with tags; otherwise, return the single item.
- if is_list:
- matchList = re.findall(pattern, translatedTextList)
- return matchList
- else:
- matchList = re.findall(pattern, translatedTextList)
- return matchList[0][0] if matchList else translatedTextList
-
-
-def countTokens(characters, system, user, history):
- inputTotalTokens = 0
- outputTotalTokens = 0
- enc = tiktoken.encoding_for_model("gpt-4")
-
- # Input
- if isinstance(history, list):
- for line in history:
- inputTotalTokens += len(enc.encode(line))
- else:
- inputTotalTokens += len(enc.encode(history))
- inputTotalTokens += len(enc.encode(system))
- inputTotalTokens += len(enc.encode(characters))
- inputTotalTokens += len(enc.encode(user))
-
- # Output
- outputTotalTokens += round(len(enc.encode(user)) * 3)
-
- return [inputTotalTokens, outputTotalTokens]
-
-
-@retry(exceptions=Exception, tries=5, delay=5)
-def translateGPT(text, history, fullPromptFlag):
- mismatch = False
- totalTokens = [0, 0]
- if isinstance(text, list):
- tList = batchList(text, BATCHSIZE)
- else:
- tList = [text]
-
- for index, tItem in enumerate(tList):
- # Before sending to translation, if we have a list of items, add the formatting
- if isinstance(tItem, list):
- payload = "\n".join([f"`{item}`" for i, item in enumerate(tItem)])
- payload = re.sub(r"(<)(\/Line\d+>)", r"\1>Placeholder Text<\3", payload)
- varResponse = subVars(payload)
- subbedT = varResponse[0]
- else:
- varResponse = subVars(tItem)
- subbedT = varResponse[0]
-
- # Things to Check before starting translation
- if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", subbedT):
- continue
-
- # Create Message
- characters, system, user = createContext(fullPromptFlag, subbedT)
-
- # Calculate Estimate
- if ESTIMATE:
- estimate = countTokens(characters, system, user, history)
- totalTokens[0] += estimate[0]
- totalTokens[1] += estimate[1]
- continue
-
- # Translating
- response = translateText(characters, system, user, history, 0.02)
- translatedText = response.choices[0].message.content
- totalTokens[0] += response.usage.prompt_tokens
- totalTokens[1] += response.usage.completion_tokens
-
- # Formatting
- translatedText = cleanTranslatedText(translatedText, varResponse)
- if isinstance(tItem, list):
- extractedTranslations = extractTranslation(translatedText, True)
- tList[index] = extractedTranslations
- if len(tItem) != len(extractedTranslations):
- # Mismatch. Try Again
- response = translateText(characters, system, user, history, 0.1)
- translatedText = response.choices[0].message.content
- totalTokens[0] += response.usage.prompt_tokens
- totalTokens[1] += response.usage.completion_tokens
-
- # Formatting
- translatedText = cleanTranslatedText(translatedText, varResponse)
- if isinstance(tItem, list):
- extractedTranslations = extractTranslation(translatedText, True)
- tList[index] = extractedTranslations
- if len(tItem) != len(extractedTranslations):
- mismatch = True # Just here for breakpoint
-
- # Create History
- if not mismatch:
- history = extractedTranslations[-10:] # Update history if we have a list
- else:
- history = text[-10:]
- else:
- # Ensure we're passing a single string to extractTranslation
- extractedTranslations = extractTranslation(translatedText, False)
- tList[index] = extractedTranslations
-
- # Combine if multilist
- if isinstance(tList[0], list):
- tList = [t for sublist in tList for t in sublist]
-
- # Return
- if format == "json":
- return [tList, totalTokens]
- else:
- return [tList[0], totalTokens]
+# Libraries
+import json
+import os
+import re
+import textwrap
+import threading
+import time
+import traceback
+import tiktoken
+import openai
+from pathlib import Path
+from colorama import Fore
+from dotenv import load_dotenv
+from retry import retry
+from tqdm import tqdm
+
+# Open AI
+load_dotenv()
+if os.getenv("api").replace(" ", "") != "":
+ openai.base_url = os.getenv("api")
+openai.organization = os.getenv("org")
+openai.api_key = os.getenv("key")
+
+# Globals
+MODEL = os.getenv("model")
+TIMEOUT = int(os.getenv("timeout"))
+LANGUAGE = os.getenv("language").capitalize()
+PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
+VOCAB = Path("vocab.txt").read_text(encoding="utf-8")
+THREADS = int(os.getenv("threads"))
+LOCK = threading.Lock()
+WIDTH = int(os.getenv("width"))
+LISTWIDTH = int(os.getenv("listWidth"))
+NOTEWIDTH = 70
+MAXHISTORY = 10
+ESTIMATE = ""
+TOKENS = [0, 0]
+NAMESLIST = []
+NAMES = False # Output a list of all the character names found
+BRFLAG = False # If the game uses
instead
+FIXTEXTWRAP = True # Overwrites textwrap
+IGNORETLTEXT = False # Ignores all translated text.
+MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
+
+# tqdm Globals
+BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
+POSITION = 0
+LEAVE = False
+
+# Pricing - Depends on the model https://openai.com/pricing
+# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request
+# If you are getting a MISMATCH LENGTH error, lower the batch size.
+if "gpt-3.5" in MODEL:
+ INPUTAPICOST = 0.002
+ OUTPUTAPICOST = 0.002
+ BATCHSIZE = 10
+elif "gpt-4" in MODEL:
+ INPUTAPICOST = 0.01
+ OUTPUTAPICOST = 0.03
+ BATCHSIZE = 50
+else:
+ INPUTAPICOST = float(os.getenv("input_cost"))
+ OUTPUTAPICOST = float(os.getenv("output_cost"))
+ BATCHSIZE = int(os.getenv("batchsize"))
+ FREQUENCY_PENALTY = float(os.getenv("frequency_penalty"))
+
+
+def handleAnim(filename, estimate):
+ global ESTIMATE
+ totalTokens = [0, 0]
+ ESTIMATE = estimate
+
+ if estimate:
+ start = time.time()
+ translatedData = openFiles(filename)
+
+ # Print Result
+ end = time.time()
+ tqdm.write(getResultString(translatedData, end - start, filename))
+ with LOCK:
+ totalTokens[0] += translatedData[1][0]
+ totalTokens[1] += translatedData[1][1]
+
+ # Print Total
+ totalString = getResultString(["", totalTokens, None], end - start, "TOTAL")
+
+ # Print any errors on maps
+ if len(MISMATCH) > 0:
+ return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
+ else:
+ return totalString
+
+ else:
+ try:
+ with open("translated/" + filename, "w", encoding="UTF-8") as outFile:
+ start = time.time()
+ translatedData = openFiles(filename)
+
+ # Print Result
+ end = time.time()
+ json.dump(translatedData[0], outFile, ensure_ascii=False, indent=4)
+ tqdm.write(getResultString(translatedData, end - start, filename))
+ with LOCK:
+ totalTokens[0] += translatedData[1][0]
+ totalTokens[1] += translatedData[1][1]
+ except Exception:
+ return "Fail"
+
+ return getResultString(["", totalTokens, None], end - start, "TOTAL")
+
+
+def openFiles(filename):
+ with open("files/" + filename, "r", encoding="UTF-8-sig") as f:
+ data = json.load(f)
+
+ # Map Files
+ if ".json" in filename:
+ translatedData = parseJSON(data, filename)
+
+ else:
+ raise NameError(filename + " Not Supported")
+
+ return translatedData
+
+
+def getResultString(translatedData, translationTime, filename):
+ # File Print String
+ totalTokenstring = (
+ Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
+ "[Output: "
+ + str(translatedData[1][1])
+ + "]" "[Cost: ${:,.4f}".format((translatedData[1][0] * 0.001 * INPUTAPICOST) + (translatedData[1][1] * 0.001 * OUTPUTAPICOST))
+ + "]"
+ )
+ timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
+
+ if translatedData[2] == None:
+ # Success
+ return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET
+
+ else:
+ # Fail
+ try:
+ raise translatedData[2]
+ except Exception as e:
+ traceback.print_exc()
+ errorString = str(e) + Fore.RED
+ return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET
+
+
+def parseJSON(data, filename):
+ keys = list(data.keys())
+ batches = [keys[i : i + BATCHSIZE] for i in range(0, len(keys), BATCHSIZE)]
+ totalTokens = [0, 0]
+ totalLines = 0
+ totalLines = len(batches)
+ global LOCK
+
+ with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar:
+ pbar.desc = filename
+ pbar.total = totalLines
+ try:
+ result = translateJSON(batches, data, pbar)
+ totalTokens[0] += result[0]
+ totalTokens[1] += result[1]
+ except Exception as e:
+ traceback.print_exc()
+ return [data, totalTokens, e]
+ return [data, totalTokens, None]
+
+
+def translateJSON(keys, data, pbar):
+ translatedBatch = []
+ textHistory = []
+ tokens = [0, 0]
+
+ for batch in keys:
+ # Save Batch
+ originalBatch = batch.copy()
+
+ # If there isn't any Japanese in the text just skip
+ needTL = False
+ for i in range(len(batch)):
+ t = data[batch[i]]
+ if re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", t) or t == "":
+ needTL = True
+ if needTL is False and IGNORETLTEXT is True:
+ pbar.update(1)
+ continue
+
+ # Remove any textwrap and Furigana
+ for i in range(len(batch)):
+ if FIXTEXTWRAP == True:
+ # Textwrap
+ data[originalBatch[i]] = data[originalBatch[i]].replace("@b", " ")
+
+ # Furigana
+ rcodeMatch = re.findall(r"(@\[(.+?):.+?\])", batch[i])
+ if len(rcodeMatch) > 0:
+ for match in rcodeMatch:
+ batch[i] = batch[i].replace(match[0], match[1])
+
+ # Translate
+ if needTL is True:
+ response = translateGPT(batch, textHistory, True)
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ translatedBatch = response[0]
+ else:
+ for i in range(len(originalBatch)):
+ translatedBatch.append(data[originalBatch[i]])
+
+ # Format and Set Text
+ if len(batch) == len(translatedBatch):
+ for i in range(len(translatedBatch)):
+ # Remove added speaker
+ translatedText = translatedBatch[i]
+ translatedText = re.sub(r"^.+?\s\|\s?", "", translatedText)
+
+ # Textwrap
+ if "@n" in translatedText:
+ match = re.search(r".*@n(.*)", translatedText)
+ if match != None:
+ tlText = match.group(1)
+ tlText = textwrap.fill(tlText, width=WIDTH)
+ tlText = tlText.replace("\n", "@b")
+ translatedText = translatedText.replace(match.group(1), tlText)
+
+ elif "@b" not in translatedText:
+ translatedText = textwrap.fill(translatedText, width=WIDTH)
+ translatedText = translatedText.replace("\n", "@b")
+
+ # Set Data
+ data[originalBatch[i]] = translatedText
+ textHistory = translatedBatch
+ translatedBatch.clear()
+ # Mismatch, Skip Batch
+ else:
+ MISMATCH.append(batch)
+ pbar.update(1)
+ continue
+ pbar.update(1)
+
+ return tokens
+
+
+def subVars(jaString):
+ jaString = jaString.replace("\u3000", " ")
+
+ # Nested
+ count = 0
+ nestedList = re.findall(r"[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]", jaString)
+ nestedList = set(nestedList)
+ if len(nestedList) != 0:
+ for icon in nestedList:
+ jaString = jaString.replace(icon, "[Nested_" + str(count) + "]")
+ count += 1
+
+ # Icons
+ count = 0
+ iconList = re.findall(r"[\\]+[iIkKwWaA]+\[[0-9]+\]", jaString)
+ iconList = set(iconList)
+ if len(iconList) != 0:
+ for icon in iconList:
+ jaString = jaString.replace(icon, "[Ascii_" + str(count) + "]")
+ count += 1
+
+ # Colors
+ count = 0
+ colorList = re.findall(r"[\\]+[cC]\[[0-9]+\]", jaString)
+ colorList = set(colorList)
+ if len(colorList) != 0:
+ for color in colorList:
+ jaString = jaString.replace(color, "[Color_" + str(count) + "]")
+ count += 1
+
+ # Names
+ count = 0
+ nameList = re.findall(r"[\\]+[nN]\[.+?\]+", jaString)
+ nameList = set(nameList)
+ if len(nameList) != 0:
+ for name in nameList:
+ jaString = jaString.replace(name, "[Noun_" + str(count) + "]")
+ count += 1
+
+ # Variables
+ count = 0
+ varList = re.findall(r"[\\]+[vV]\[[0-9]+\]", jaString)
+ varList = set(varList)
+ if len(varList) != 0:
+ for var in varList:
+ jaString = jaString.replace(var, "[Var_" + str(count) + "]")
+ count += 1
+
+ # Formatting
+ count = 0
+ formatList = re.findall(r"[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]", jaString)
+ formatList = set(formatList)
+ if len(formatList) != 0:
+ for var in formatList:
+ jaString = jaString.replace(var, "[FCode_" + str(count) + "]")
+ count += 1
+
+ # Put all lists in list and return
+ allList = [nestedList, iconList, colorList, nameList, varList, formatList]
+ return [jaString, allList]
+
+
+def resubVars(translatedText, allList):
+ # Fix Spacing and ChatGPT Nonsense
+ matchList = re.findall(r"\[\s?.+?\s?\]", translatedText)
+ if len(matchList) > 0:
+ for match in matchList:
+ text = match.strip()
+ translatedText = translatedText.replace(match, text)
+
+ # Nested
+ count = 0
+ if len(allList[0]) != 0:
+ for var in allList[0]:
+ translatedText = translatedText.replace("[Nested_" + str(count) + "]", var)
+ count += 1
+
+ # Icons
+ count = 0
+ if len(allList[1]) != 0:
+ for var in allList[1]:
+ translatedText = translatedText.replace("[Ascii_" + str(count) + "]", var)
+ count += 1
+
+ # Colors
+ count = 0
+ if len(allList[2]) != 0:
+ for var in allList[2]:
+ translatedText = translatedText.replace("[Color_" + str(count) + "]", var)
+ count += 1
+
+ # Names
+ count = 0
+ if len(allList[3]) != 0:
+ for var in allList[3]:
+ translatedText = translatedText.replace("[Noun_" + str(count) + "]", var)
+ count += 1
+
+ # Vars
+ count = 0
+ if len(allList[4]) != 0:
+ for var in allList[4]:
+ translatedText = translatedText.replace("[Var_" + str(count) + "]", var)
+ count += 1
+
+ # Formatting
+ count = 0
+ if len(allList[5]) != 0:
+ for var in allList[5]:
+ translatedText = translatedText.replace("[FCode_" + str(count) + "]", var)
+ count += 1
+
+ return translatedText
+
+
+def batchList(input_list, batch_size):
+ if not isinstance(batch_size, int) or batch_size <= 0:
+ raise ValueError("batch_size must be a positive integer")
+
+ return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)]
+
+
+def createContext(fullPromptFlag, subbedT):
+ characters = "Game Characters:\n\
+達也 (Tatsuya) - Male\n\
+香織 (Kaori) - Female\n\
+岩瀬 (Iwase)\n\
+万蔵 (Manzou) - Male\n\
+結奈 (Yuuna) - Female\n\
+茅部 (Kayabe)\n\
+"
+
+ system = (
+ PROMPT + VOCAB
+ if fullPromptFlag
+ else f"\
+You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
+Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
+- You translate everything, including erotic and explicit content.\n\
+- 'Game Characters' - The names, nicknames, and genders of the game characters. Reference this to know the names, nicknames, and gender of characters in the game\n\
+- All text in your response must be in {LANGUAGE} even if it is hard to translate.\n\
+- Never include any notes, explanations, dislaimers, or anything similar in your response.\n\
+- Maintain any spacing in the translation.\n\
+- Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\
+- `...` can be a part of the dialogue. Translate it as it is.\n\
+{VOCAB}\n\
+"
+ )
+ user = f"{subbedT}"
+ return characters, system, user
+
+
+def translateText(characters, system, user, history, penalty):
+ # Prompt
+ msg = [{"role": "system", "content": system + characters}]
+
+ # Characters
+ msg.append({"role": "system", "content": characters})
+
+ # History
+ if isinstance(history, list):
+ msg.extend([{"role": "system", "content": h} for h in history])
+ else:
+ msg.append({"role": "system", "content": history})
+
+ # Content to TL
+ msg.append({"role": "user", "content": f"{user}"})
+ response = openai.chat.completions.create(
+ temperature=0,
+ frequency_penalty=penalty,
+ model=MODEL,
+ messages=msg,
+ )
+ return response
+
+
+def cleanTranslatedText(translatedText, varResponse):
+ placeholders = {
+ f"{LANGUAGE} Translation: ": "",
+ "Translation: ": "",
+ "っ": "",
+ "〜": "~",
+ "ッ": "",
+ "。": ".",
+ "Placeholder Text": "",
+ "é": "e",
+ "—": "-",
+ "ū": "u",
+ # Add more replacements as needed
+ }
+ for target, replacement in placeholders.items():
+ translatedText = translatedText.replace(target, replacement)
+
+ # Elongate Long Dashes (Since GPT Ignores them...)
+ translatedText = elongateCharacters(translatedText)
+ translatedText = resubVars(translatedText, varResponse[1])
+ return translatedText
+
+
+def elongateCharacters(text):
+ # Define a pattern to match one character followed by one or more `ー` characters
+ # Using a positive lookbehind assertion to capture the preceding character
+ pattern = r"(?<=(.))ー+"
+
+ # Define a replacement function that elongates the captured character
+ def repl(match):
+ char = match.group(1) # The character before the ー sequence
+ count = len(match.group(0)) - 1 # Number of ー characters
+ return char * count # Replace ー sequence with the character repeated
+
+ # Use re.sub() to replace the pattern in the text
+ return re.sub(pattern, repl, text)
+
+
+def extractTranslation(translatedTextList, is_list):
+ pattern = r"`?<[Ll]ine\d+>([\\]*.*?[\\]*?)<\/?[Ll]ine\d+>`?"
+ # If it's a batch (i.e., list), extract with tags; otherwise, return the single item.
+ if is_list:
+ matchList = re.findall(pattern, translatedTextList)
+ return matchList
+ else:
+ matchList = re.findall(pattern, translatedTextList)
+ return matchList[0][0] if matchList else translatedTextList
+
+
+def countTokens(characters, system, user, history):
+ inputTotalTokens = 0
+ outputTotalTokens = 0
+ enc = tiktoken.encoding_for_model("gpt-4")
+
+ # Input
+ if isinstance(history, list):
+ for line in history:
+ inputTotalTokens += len(enc.encode(line))
+ else:
+ inputTotalTokens += len(enc.encode(history))
+ inputTotalTokens += len(enc.encode(system))
+ inputTotalTokens += len(enc.encode(characters))
+ inputTotalTokens += len(enc.encode(user))
+
+ # Output
+ outputTotalTokens += round(len(enc.encode(user)) * 3)
+
+ return [inputTotalTokens, outputTotalTokens]
+
+
+@retry(exceptions=Exception, tries=5, delay=5)
+def translateGPT(text, history, fullPromptFlag):
+ mismatch = False
+ totalTokens = [0, 0]
+ if isinstance(text, list):
+ tList = batchList(text, BATCHSIZE)
+ else:
+ tList = [text]
+
+ for index, tItem in enumerate(tList):
+ # Before sending to translation, if we have a list of items, add the formatting
+ if isinstance(tItem, list):
+ payload = "\n".join([f"`{item}`" for i, item in enumerate(tItem)])
+ payload = re.sub(r"(<)(\/Line\d+>)", r"\1>Placeholder Text<\3", payload)
+ varResponse = subVars(payload)
+ subbedT = varResponse[0]
+ else:
+ varResponse = subVars(tItem)
+ subbedT = varResponse[0]
+
+ # Things to Check before starting translation
+ if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", subbedT):
+ continue
+
+ # Create Message
+ characters, system, user = createContext(fullPromptFlag, subbedT)
+
+ # Calculate Estimate
+ if ESTIMATE:
+ estimate = countTokens(characters, system, user, history)
+ totalTokens[0] += estimate[0]
+ totalTokens[1] += estimate[1]
+ continue
+
+ # Translating
+ response = translateText(characters, system, user, history, 0.02)
+ translatedText = response.choices[0].message.content
+ totalTokens[0] += response.usage.prompt_tokens
+ totalTokens[1] += response.usage.completion_tokens
+
+ # Formatting
+ translatedText = cleanTranslatedText(translatedText, varResponse)
+ if isinstance(tItem, list):
+ extractedTranslations = extractTranslation(translatedText, True)
+ tList[index] = extractedTranslations
+ if len(tItem) != len(extractedTranslations):
+ # Mismatch. Try Again
+ response = translateText(characters, system, user, history, 0.1)
+ translatedText = response.choices[0].message.content
+ totalTokens[0] += response.usage.prompt_tokens
+ totalTokens[1] += response.usage.completion_tokens
+
+ # Formatting
+ translatedText = cleanTranslatedText(translatedText, varResponse)
+ if isinstance(tItem, list):
+ extractedTranslations = extractTranslation(translatedText, True)
+ tList[index] = extractedTranslations
+ if len(tItem) != len(extractedTranslations):
+ mismatch = True # Just here for breakpoint
+
+ # Create History
+ if not mismatch:
+ history = extractedTranslations[-10:] # Update history if we have a list
+ else:
+ history = text[-10:]
+ else:
+ # Ensure we're passing a single string to extractTranslation
+ extractedTranslations = extractTranslation(translatedText, False)
+ tList[index] = extractedTranslations
+
+ # Combine if multilist
+ if isinstance(tList[0], list):
+ tList = [t for sublist in tList for t in sublist]
+
+ # Return
+ if format == "json":
+ return [tList, totalTokens]
+ else:
+ return [tList[0], totalTokens]
diff --git a/modules/csv.py b/modules/csv.py
index a94f795..a1d15ef 100644
--- a/modules/csv.py
+++ b/modules/csv.py
@@ -1,701 +1,701 @@
-# Libraries
-import json
-import os
-import re
-import textwrap
-import threading
-import time
-import traceback
-import tiktoken
-import openai
-import csv
-from pathlib import Path
-from colorama import Fore
-from dotenv import load_dotenv
-from retry import retry
-from tqdm import tqdm
-
-# Open AI
-load_dotenv()
-if os.getenv("api").replace(" ", "") != "":
- openai.base_url = os.getenv("api")
-openai.organization = os.getenv("org")
-openai.api_key = os.getenv("key")
-
-# Globals
-MODEL = os.getenv("model")
-TIMEOUT = int(os.getenv("timeout"))
-LANGUAGE = os.getenv("language").capitalize()
-PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
-VOCAB = Path("vocab.txt").read_text(encoding="utf-8")
-THREADS = int(os.getenv("threads"))
-LOCK = threading.Lock()
-WIDTH = int(os.getenv("width"))
-LISTWIDTH = int(os.getenv("listWidth"))
-NOTEWIDTH = int(os.getenv("noteWidth"))
-MAXHISTORY = 10
-ESTIMATE = ""
-TOKENS = [0, 0]
-NAMESLIST = []
-NAMES = False # Output a list of all the character names found
-BRFLAG = False # If the game uses
instead
-FIXTEXTWRAP = True # Overwrites textwrap
-IGNORETLTEXT = True # Ignores all translated text.
-MISMATCH = [] # Lists files that thdata a mismatch error (Length of GPT list response is wrong)
-BRACKETNAMES = False
-
-# Pricing - Depends on the model https://openai.com/pricing
-# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request
-# If you are getting a MISMATCH LENGTH error, lower the batch size.
-if "gpt-3.5" in MODEL:
- INPUTAPICOST = 0.002
- OUTPUTAPICOST = 0.002
- BATCHSIZE = 10
- FREQUENCY_PENALTY = 0.2
-elif "gpt-4" in MODEL:
- INPUTAPICOST = 0.0025
- OUTPUTAPICOST = 0.01
- BATCHSIZE = 20
- FREQUENCY_PENALTY = 0.1
-else:
- INPUTAPICOST = float(os.getenv("input_cost"))
- OUTPUTAPICOST = float(os.getenv("output_cost"))
- BATCHSIZE = int(os.getenv("batchsize"))
- FREQUENCY_PENALTY = float(os.getenv("frequency_penalty"))
-
-# tqdm Globals
-BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
-POSITION = 0
-LEAVE = False
-PBAR = None
-ENCODING = "cp932"
-
-
-def handleCSV(filename, estimate):
- global ESTIMATE, TOKENS
- ESTIMATE = estimate
-
- if not ESTIMATE:
- with open("translated/" + filename, "w+t", newline="", encoding=ENCODING) as writeFile:
- # Translate
- start = time.time()
- translatedData = openFiles(filename, writeFile)
-
- # Print Result
- end = time.time()
- tqdm.write(getResultString(translatedData, end - start, filename))
- with LOCK:
- TOKENS[0] += translatedData[1][0]
- TOKENS[1] += translatedData[1][1]
- else:
- # Translate
- start = time.time()
- translatedData = openFilesEstimate(filename)
-
- # Print Result
- end = time.time()
- tqdm.write(getResultString(translatedData, end - start, filename))
- with LOCK:
- TOKENS[0] += translatedData[1][0]
- TOKENS[1] += translatedData[1][1]
-
- # Print Total
- totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
-
- # Print any errors on maps
- if len(MISMATCH) > 0:
- return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
- else:
- return totalString
-
-
-def openFiles(filename, writeFile):
- with open("files/" + filename, "r", encoding="cp932") as readFile, writeFile:
- translatedData = parseCSV(readFile, writeFile, filename)
-
- return translatedData
-
-
-def openFilesEstimate(filename):
- with open("files/" + filename, "r", encoding="cp932") as readFile:
- translatedData = parseCSV(readFile, "", filename)
-
- return translatedData
-
-
-def getResultString(translatedData, translationTime, filename):
- # File Print String
- totalTokenstring = (
- Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
- "[Output: "
- + str(translatedData[1][1])
- + "]" "[Cost: ${:,.4f}".format((translatedData[1][0] * 0.001 * INPUTAPICOST) + (translatedData[1][1] * 0.001 * OUTPUTAPICOST))
- + "]"
- )
- timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
-
- if translatedData[2] is None:
- # Success
- return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET
- else:
- # Fail
- try:
- raise translatedData[2]
- except Exception as e:
- traceback.print_exc()
- errorString = str(e) + Fore.RED
- return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET
-
-
-def parseCSV(readFile, writeFile, filename):
- totalTokens = [0, 0]
- totalLines = 0
- global LOCK
-
- # Read from tmp files
- if os.path.isfile("csv.tmp"):
- with open("csv.tmp") as tmpFile:
- format = tmpFile.readline()
- else:
- format = ""
-
- # Choices
- while format not in ["1", "2", "3", "4"]:
- format = input("\n\nSelect the CSV Format:\n\n1. Translator++\n2. Single\n3. Multiple\n4. Speaker&Text\n")
- match format:
- case "1":
- format = "1"
- case "2":
- format = "2"
- case "3":
- format = "3"
- case "4":
- format = "4"
-
- # Write to file for later use
- with open("csv.tmp", "w", encoding="utf-8") as tmpFile:
- tmpFile.write(f"{format}")
-
- # Get total for progress bar
- totalLines = len(readFile.readlines())
- readFile.seek(0)
-
- reader = csv.reader(readFile, delimiter=",")
- if not ESTIMATE:
- writer = csv.writer(writeFile, delimiter=",")
- else:
- writer = ""
-
- with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar:
- pbar.desc = filename
- pbar.total = totalLines
-
- # Grab All Rows
- data = []
- for row in reader:
- data.append(row)
-
- try:
- response = translateCSV(data, pbar, writer, filename, None, format)
- totalTokens[0] = response[0]
- totalTokens[1] = response[1]
- except Exception:
- traceback.print_exc()
- return [data, totalTokens, None]
-
-
-def translateCSV(data, pbar, writer, filename, translatedList, format):
- global LOCK, ESTIMATE, PBAR
- PBAR = pbar
- translatedText = ""
- totalTokens = [0, 0]
- i = 0
- stringList = []
-
- try:
- # Translate
- while i < len(data):
- match format:
- # T++ Format: Source Text on column 1. TL Target on Column 2
- case "1":
- # Get String
- if i != 0:
- if data[i][1] == "":
- jaString = data[i][0]
- else:
- jaString = data[i][1]
-
- # Remove Textwrap
- jaString = jaString.replace("\\n", " ")
-
- # Pass 1
- if not translatedList:
- stringList.append(jaString)
-
- # Pass 2
- else:
- # Grab and Pop
- translatedText = translatedList[0]
- translatedList.pop(0)
-
- # Add Wordwrap
- translatedText = textwrap.fill(translatedText, WIDTH)
- translatedText = translatedText.replace("\n", "\\n")
-
- # Set Data
- data[i][1] = translatedText
-
- # Iterate
- i += 1
-
- # Target Format
- case "2":
- # Set Values
- sourceColumn = 0
- targetColumn = 1
-
- # Check if Translated
- jaString = data[i][sourceColumn]
-
- # Remove Textwrap
- jaString = jaString.replace("\\n", " ")
-
- # Pass 1
- if not translatedList:
- stringList.append(jaString)
-
- # Pass 2
- else:
- # Grab and Pop
- translatedText = translatedList[0]
- translatedList.pop(0)
-
- # Add Wordwrap
- translatedText = textwrap.fill(translatedText, WIDTH)
- translatedText = translatedText.replace("\n", "\\n")
-
- # Set Data
- data[i][targetColumn] = translatedText
-
- # Iterate
- i += 1
-
- # In Place Format
- case "3":
- # Set columns to translate. Leave empty to translate all.
- targetColumns = []
-
- # False - Place translation in source column
- # True - Place translation in next column
- targetNextRow = True
-
- for j in range(len(data[i])):
- if j not in targetColumns:
- # Check if Translated
- jaString = data[i][j]
-
- # Remove Textwrap
- jaString = jaString.replace("\\n", " ")
-
- # Pass 1
- if not translatedList:
- stringList.append(jaString)
-
- # Pass 2
- else:
- # Grab and Pop
- translatedText = translatedList[0]
- translatedList.pop(0)
-
- # Add Wordwrap
- translatedText = textwrap.fill(translatedText, WIDTH)
- translatedText = translatedText.replace("\n", "\\n")
-
- # Set Data
- if targetNextRow:
- data[i][j + 1] = translatedText
- else:
- data[i][j] = translatedText
-
- # Iterate
- i += 1
-
- # Speaker & Text Format
- case "4":
- # Set columns to translate. Leave empty to translate all.
- speakerColumn = 8
- textColumn = 20
- speaker = ""
-
- if len(data[i]) > textColumn and data[i][textColumn]:
- # Speaker
- if data[i][speakerColumn]:
- speakerResponse = getSpeaker(data[i][speakerColumn])
- totalTokens[0] += speakerResponse[1][0]
- totalTokens[1] += speakerResponse[1][1]
- speaker = speakerResponse[0]
- data[i][speakerColumn] = speaker
-
- # Get Text
- jaString = data[i][textColumn]
-
- # Remove Textwrap
- jaString = jaString.replace("\\n", " ")
-
- # Remove Furigana
- jaString = re.sub(r"＜(.*)＝.*＞", r"\1", jaString)
-
- # Pass 1
- if not translatedList:
- # Append Speaker
- if speaker:
- jaString = f"[{speaker}]: {jaString}"
-
- # Append to List
- stringList.append(jaString)
-
- # Pass 2
- else:
- # Grab and Pop
- translatedText = translatedList[0]
- translatedList.pop(0)
-
- # Remove speaker
- if speaker:
- translatedText = re.sub(r"^\[?(.+?)\]?\s?[|:]\s?", "", translatedText)
-
- # Add Wordwrap
- translatedText = textwrap.fill(translatedText, WIDTH)
- translatedText = translatedText.replace("\n", "\\n")
-
- # Set Data
- data[i][textColumn] = translatedText
-
- # Iterate
- i += 1
-
- # EOF
- if len(stringList) > 0:
- # Set Progress
- pbar.total = len(stringList)
- pbar.refresh()
-
- # Translate
- response = translateGPT(stringList, "", True)
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
- translatedList = response[0]
-
- # Set Strings
- if len(stringList) == len(translatedList):
- translateCSV(data, pbar, writer, filename, translatedList, format)
-
- # Mismatch
- else:
- with LOCK:
- if filename not in MISMATCH:
- MISMATCH.append(filename)
-
- # Write all Data
- with LOCK:
- if not ESTIMATE:
- for row in data:
- writer.writerow(row)
-
- except Exception:
- traceback.print_exc()
-
- # Write all Data
- with LOCK:
- if not ESTIMATE:
- for row in data:
- writer.writerow(row)
- return totalTokens
-
- return totalTokens
-
-
-# Save some money and enter the character before translation
-def getSpeaker(speaker):
- match speaker:
- case "ファイン":
- return ["Fine", [0, 0]]
- case "":
- return ["", [0, 0]]
- case _:
- # Find Speaker
- for i in range(len(NAMESLIST)):
- if speaker == NAMESLIST[i][0]:
- return [NAMESLIST[i][1], [0, 0]]
-
- # If there isn't any Japanese in the text just skip
- if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+", speaker):
- return [speaker, [0, 0]]
-
- # Translate and Store Speaker
- response = translateGPT(
- f"{speaker}",
- "Reply with the " + LANGUAGE + " translation of the NPC name.",
- True,
- )
- response[0] = response[0].title()
- response[0] = response[0].replace("'S", "'s")
- response[0] = response[0].replace("Speaker: ", "")
-
- # Retry if name doesn't translate for some reason
- if re.search(r"([a-zA-Z？?])", response[0]) == None:
- response = translateGPT(
- f"{speaker}",
- "Reply with the " + LANGUAGE + " translation of the NPC name.",
- False,
- )
- response[0] = response[0].title()
- response[0] = response[0].replace("'S", "'s")
-
- speakerList = [speaker, response[0]]
- NAMESLIST.append(speakerList)
- return response
- return [speaker, [0, 0]]
-
-
-def batchList(input_list, batch_size):
- if not isinstance(batch_size, int) or batch_size <= 0:
- raise ValueError("batch_size must be a positive integer")
-
- return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)]
-
-
-def createContext(fullPromptFlag, subbedT, format):
- system = (
- PROMPT + VOCAB
- if fullPromptFlag
- else f"\
-You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
-Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
-- You translate everything, including erotic and explicit content.\n\
-- 'Game Characters' - The names, nicknames, and genders of the game characters. Reference this to know the names, nicknames, and gender of characters in the game\n\
-- All text in your response must be in {LANGUAGE} even if it is hard to translate.\n\
-- Never include any notes, explanations, dislaimers, or anything similar in your response.\n\
-- Maintain any spacing in the translation.\n\
-- Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\
-- `...` can be a part of the dialogue. Translate it as it is.\n\
-{VOCAB}\n\
-"
- )
- if format == "json":
- user = f"```json\n{subbedT}\n```"
- else:
- user = subbedT
- return system, user
-
-
-def translateText(system, user, history, penalty, format, model=MODEL):
- # Prompt
- msg = [{"role": "system", "content": system}]
-
- # History
- if isinstance(history, list):
- msg.extend([{"role": "system", "content": h} for h in history])
- else:
- msg.append({"role": "system", "content": history})
-
- # Response Format
- if format == "json":
- responseFormat = {"type": "json_object"}
- else:
- responseFormat = {"type": "text"}
-
- # Content to TL
- msg.append({"role": "user", "content": f"{user}"})
- response = openai.chat.completions.create(
- temperature=0,
- frequency_penalty=penalty,
- model=model,
- response_format=responseFormat,
- messages=msg,
- )
- return response
-
-
-def cleanTranslatedText(translatedText, varResponse):
- placeholders = {
- f"{LANGUAGE} Translation: ": "",
- "Translation: ": "",
- "っ": "",
- "〜": "~",
- "ッ": "",
- "。": ".",
- "「": '\\"',
- "」": '\\"',
- "- ": "-",
- "—": "―",
- "】": "]",
- "【": "[",
- "é": "e",
- "Placeholder Text": "",
- # Add more replacements as needed
- }
- for target, replacement in placeholders.items():
- translatedText = translatedText.replace(target, replacement)
-
- # Remove Repeating Characters
- pattern = re.compile(r"(.)\s*\1(?:\s*\1){" + str(20 - 1) + r",}")
- translatedText = pattern.sub(lambda match: match.group(0).replace(" ", "")[:20], translatedText)
-
- # Elongate Long Dashes (Since GPT Ignores them...)
- translatedText = elongateCharacters(translatedText)
- return translatedText
-
-
-def elongateCharacters(text):
- # Define a pattern to match one character followed by one or more `ー` characters
- # Using a positive lookbehind assertion to capture the preceding character
- pattern = r"(?<=(.))ー+"
-
- # Define a replacement function that elongates the captured character
- def repl(match):
- char = match.group(1) # The character before the ー sequence
- count = len(match.group(0)) - 1 # Number of ー characters
- return char * count # Replace ー sequence with the character repeated
-
- # Use re.sub() to replace the pattern in the text
- return re.sub(pattern, repl, text)
-
-
-def extractTranslation(translatedTextList, is_list):
- try:
- translatedTextList = re.sub(r'\\"+\"([^,\n}])', r'\\"\1', translatedTextList)
- translatedTextList = re.sub(r"(? instead
+FIXTEXTWRAP = True # Overwrites textwrap
+IGNORETLTEXT = True # Ignores all translated text.
+MISMATCH = [] # Lists files that thdata a mismatch error (Length of GPT list response is wrong)
+BRACKETNAMES = False
+
+# Pricing - Depends on the model https://openai.com/pricing
+# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request
+# If you are getting a MISMATCH LENGTH error, lower the batch size.
+if "gpt-3.5" in MODEL:
+ INPUTAPICOST = 0.002
+ OUTPUTAPICOST = 0.002
+ BATCHSIZE = 10
+ FREQUENCY_PENALTY = 0.2
+elif "gpt-4" in MODEL:
+ INPUTAPICOST = 0.0025
+ OUTPUTAPICOST = 0.01
+ BATCHSIZE = 20
+ FREQUENCY_PENALTY = 0.1
+else:
+ INPUTAPICOST = float(os.getenv("input_cost"))
+ OUTPUTAPICOST = float(os.getenv("output_cost"))
+ BATCHSIZE = int(os.getenv("batchsize"))
+ FREQUENCY_PENALTY = float(os.getenv("frequency_penalty"))
+
+# tqdm Globals
+BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
+POSITION = 0
+LEAVE = False
+PBAR = None
+ENCODING = "cp932"
+
+
+def handleCSV(filename, estimate):
+ global ESTIMATE, TOKENS
+ ESTIMATE = estimate
+
+ if not ESTIMATE:
+ with open("translated/" + filename, "w+t", newline="", encoding=ENCODING) as writeFile:
+ # Translate
+ start = time.time()
+ translatedData = openFiles(filename, writeFile)
+
+ # Print Result
+ end = time.time()
+ tqdm.write(getResultString(translatedData, end - start, filename))
+ with LOCK:
+ TOKENS[0] += translatedData[1][0]
+ TOKENS[1] += translatedData[1][1]
+ else:
+ # Translate
+ start = time.time()
+ translatedData = openFilesEstimate(filename)
+
+ # Print Result
+ end = time.time()
+ tqdm.write(getResultString(translatedData, end - start, filename))
+ with LOCK:
+ TOKENS[0] += translatedData[1][0]
+ TOKENS[1] += translatedData[1][1]
+
+ # Print Total
+ totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
+
+ # Print any errors on maps
+ if len(MISMATCH) > 0:
+ return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
+ else:
+ return totalString
+
+
+def openFiles(filename, writeFile):
+ with open("files/" + filename, "r", encoding="cp932") as readFile, writeFile:
+ translatedData = parseCSV(readFile, writeFile, filename)
+
+ return translatedData
+
+
+def openFilesEstimate(filename):
+ with open("files/" + filename, "r", encoding="cp932") as readFile:
+ translatedData = parseCSV(readFile, "", filename)
+
+ return translatedData
+
+
+def getResultString(translatedData, translationTime, filename):
+ # File Print String
+ totalTokenstring = (
+ Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
+ "[Output: "
+ + str(translatedData[1][1])
+ + "]" "[Cost: ${:,.4f}".format((translatedData[1][0] * 0.001 * INPUTAPICOST) + (translatedData[1][1] * 0.001 * OUTPUTAPICOST))
+ + "]"
+ )
+ timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
+
+ if translatedData[2] is None:
+ # Success
+ return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET
+ else:
+ # Fail
+ try:
+ raise translatedData[2]
+ except Exception as e:
+ traceback.print_exc()
+ errorString = str(e) + Fore.RED
+ return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET
+
+
+def parseCSV(readFile, writeFile, filename):
+ totalTokens = [0, 0]
+ totalLines = 0
+ global LOCK
+
+ # Read from tmp files
+ if os.path.isfile("csv.tmp"):
+ with open("csv.tmp") as tmpFile:
+ format = tmpFile.readline()
+ else:
+ format = ""
+
+ # Choices
+ while format not in ["1", "2", "3", "4"]:
+ format = input("\n\nSelect the CSV Format:\n\n1. Translator++\n2. Single\n3. Multiple\n4. Speaker&Text\n")
+ match format:
+ case "1":
+ format = "1"
+ case "2":
+ format = "2"
+ case "3":
+ format = "3"
+ case "4":
+ format = "4"
+
+ # Write to file for later use
+ with open("csv.tmp", "w", encoding="utf-8") as tmpFile:
+ tmpFile.write(f"{format}")
+
+ # Get total for progress bar
+ totalLines = len(readFile.readlines())
+ readFile.seek(0)
+
+ reader = csv.reader(readFile, delimiter=",")
+ if not ESTIMATE:
+ writer = csv.writer(writeFile, delimiter=",")
+ else:
+ writer = ""
+
+ with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar:
+ pbar.desc = filename
+ pbar.total = totalLines
+
+ # Grab All Rows
+ data = []
+ for row in reader:
+ data.append(row)
+
+ try:
+ response = translateCSV(data, pbar, writer, filename, None, format)
+ totalTokens[0] = response[0]
+ totalTokens[1] = response[1]
+ except Exception:
+ traceback.print_exc()
+ return [data, totalTokens, None]
+
+
+def translateCSV(data, pbar, writer, filename, translatedList, format):
+ global LOCK, ESTIMATE, PBAR
+ PBAR = pbar
+ translatedText = ""
+ totalTokens = [0, 0]
+ i = 0
+ stringList = []
+
+ try:
+ # Translate
+ while i < len(data):
+ match format:
+ # T++ Format: Source Text on column 1. TL Target on Column 2
+ case "1":
+ # Get String
+ if i != 0:
+ if data[i][1] == "":
+ jaString = data[i][0]
+ else:
+ jaString = data[i][1]
+
+ # Remove Textwrap
+ jaString = jaString.replace("\\n", " ")
+
+ # Pass 1
+ if not translatedList:
+ stringList.append(jaString)
+
+ # Pass 2
+ else:
+ # Grab and Pop
+ translatedText = translatedList[0]
+ translatedList.pop(0)
+
+ # Add Wordwrap
+ translatedText = textwrap.fill(translatedText, WIDTH)
+ translatedText = translatedText.replace("\n", "\\n")
+
+ # Set Data
+ data[i][1] = translatedText
+
+ # Iterate
+ i += 1
+
+ # Target Format
+ case "2":
+ # Set Values
+ sourceColumn = 0
+ targetColumn = 1
+
+ # Check if Translated
+ jaString = data[i][sourceColumn]
+
+ # Remove Textwrap
+ jaString = jaString.replace("\\n", " ")
+
+ # Pass 1
+ if not translatedList:
+ stringList.append(jaString)
+
+ # Pass 2
+ else:
+ # Grab and Pop
+ translatedText = translatedList[0]
+ translatedList.pop(0)
+
+ # Add Wordwrap
+ translatedText = textwrap.fill(translatedText, WIDTH)
+ translatedText = translatedText.replace("\n", "\\n")
+
+ # Set Data
+ data[i][targetColumn] = translatedText
+
+ # Iterate
+ i += 1
+
+ # In Place Format
+ case "3":
+ # Set columns to translate. Leave empty to translate all.
+ targetColumns = []
+
+ # False - Place translation in source column
+ # True - Place translation in next column
+ targetNextRow = False
+
+ for j in range(len(data[i])):
+ if j not in targetColumns:
+ # Check if Translated
+ jaString = data[i][j]
+
+ # Remove Textwrap
+ jaString = jaString.replace("\\n", " ")
+
+ # Pass 1
+ if not translatedList:
+ stringList.append(jaString)
+
+ # Pass 2
+ else:
+ # Grab and Pop
+ translatedText = translatedList[0]
+ translatedList.pop(0)
+
+ # Add Wordwrap
+ translatedText = textwrap.fill(translatedText, WIDTH)
+ translatedText = translatedText.replace("\n", "\\n")
+
+ # Set Data
+ if targetNextRow:
+ data[i][j + 1] = translatedText
+ else:
+ data[i][j] = translatedText
+
+ # Iterate
+ i += 1
+
+ # Speaker & Text Format
+ case "4":
+ # Set columns to translate. Leave empty to translate all.
+ speakerColumn = 8
+ textColumn = 20
+ speaker = ""
+
+ if len(data[i]) > textColumn and data[i][textColumn]:
+ # Speaker
+ if data[i][speakerColumn]:
+ speakerResponse = getSpeaker(data[i][speakerColumn])
+ totalTokens[0] += speakerResponse[1][0]
+ totalTokens[1] += speakerResponse[1][1]
+ speaker = speakerResponse[0]
+ data[i][speakerColumn] = speaker
+
+ # Get Text
+ jaString = data[i][textColumn]
+
+ # Remove Textwrap
+ jaString = jaString.replace("\\n", " ")
+
+ # Remove Furigana
+ jaString = re.sub(r"＜(.*)＝.*＞", r"\1", jaString)
+
+ # Pass 1
+ if not translatedList:
+ # Append Speaker
+ if speaker:
+ jaString = f"[{speaker}]: {jaString}"
+
+ # Append to List
+ stringList.append(jaString)
+
+ # Pass 2
+ else:
+ # Grab and Pop
+ translatedText = translatedList[0]
+ translatedList.pop(0)
+
+ # Remove speaker
+ if speaker:
+ translatedText = re.sub(r"^\[?(.+?)\]?\s?[|:]\s?", "", translatedText)
+
+ # Add Wordwrap
+ translatedText = textwrap.fill(translatedText, WIDTH)
+ translatedText = translatedText.replace("\n", "\\n")
+
+ # Set Data
+ data[i][textColumn] = translatedText
+
+ # Iterate
+ i += 1
+
+ # EOF
+ if len(stringList) > 0:
+ # Set Progress
+ pbar.total = len(stringList)
+ pbar.refresh()
+
+ # Translate
+ response = translateGPT(stringList, "", True)
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+ translatedList = response[0]
+
+ # Set Strings
+ if len(stringList) == len(translatedList):
+ translateCSV(data, pbar, writer, filename, translatedList, format)
+
+ # Mismatch
+ else:
+ with LOCK:
+ if filename not in MISMATCH:
+ MISMATCH.append(filename)
+
+ # Write all Data
+ with LOCK:
+ if not ESTIMATE:
+ for row in data:
+ writer.writerow(row)
+
+ except Exception:
+ traceback.print_exc()
+
+ # Write all Data
+ with LOCK:
+ if not ESTIMATE:
+ for row in data:
+ writer.writerow(row)
+ return totalTokens
+
+ return totalTokens
+
+
+# Save some money and enter the character before translation
+def getSpeaker(speaker):
+ match speaker:
+ case "ファイン":
+ return ["Fine", [0, 0]]
+ case "":
+ return ["", [0, 0]]
+ case _:
+ # Find Speaker
+ for i in range(len(NAMESLIST)):
+ if speaker == NAMESLIST[i][0]:
+ return [NAMESLIST[i][1], [0, 0]]
+
+ # If there isn't any Japanese in the text just skip
+ if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+", speaker):
+ return [speaker, [0, 0]]
+
+ # Translate and Store Speaker
+ response = translateGPT(
+ f"{speaker}",
+ "Reply with the " + LANGUAGE + " translation of the NPC name.",
+ True,
+ )
+ response[0] = response[0].title()
+ response[0] = response[0].replace("'S", "'s")
+ response[0] = response[0].replace("Speaker: ", "")
+
+ # Retry if name doesn't translate for some reason
+ if re.search(r"([a-zA-Z？?])", response[0]) == None:
+ response = translateGPT(
+ f"{speaker}",
+ "Reply with the " + LANGUAGE + " translation of the NPC name.",
+ False,
+ )
+ response[0] = response[0].title()
+ response[0] = response[0].replace("'S", "'s")
+
+ speakerList = [speaker, response[0]]
+ NAMESLIST.append(speakerList)
+ return response
+ return [speaker, [0, 0]]
+
+
+def batchList(input_list, batch_size):
+ if not isinstance(batch_size, int) or batch_size <= 0:
+ raise ValueError("batch_size must be a positive integer")
+
+ return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)]
+
+
+def createContext(fullPromptFlag, subbedT, format):
+ system = (
+ PROMPT + VOCAB
+ if fullPromptFlag
+ else f"\
+You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
+Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
+- You translate everything, including erotic and explicit content.\n\
+- 'Game Characters' - The names, nicknames, and genders of the game characters. Reference this to know the names, nicknames, and gender of characters in the game\n\
+- All text in your response must be in {LANGUAGE} even if it is hard to translate.\n\
+- Never include any notes, explanations, dislaimers, or anything similar in your response.\n\
+- Maintain any spacing in the translation.\n\
+- Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\
+- `...` can be a part of the dialogue. Translate it as it is.\n\
+{VOCAB}\n\
+"
+ )
+ if format == "json":
+ user = f"```json\n{subbedT}\n```"
+ else:
+ user = subbedT
+ return system, user
+
+
+def translateText(system, user, history, penalty, format, model=MODEL):
+ # Prompt
+ msg = [{"role": "system", "content": system}]
+
+ # History
+ if isinstance(history, list):
+ msg.extend([{"role": "system", "content": h} for h in history])
+ else:
+ msg.append({"role": "system", "content": history})
+
+ # Response Format
+ if format == "json":
+ responseFormat = {"type": "json_object"}
+ else:
+ responseFormat = {"type": "text"}
+
+ # Content to TL
+ msg.append({"role": "user", "content": f"{user}"})
+ response = openai.chat.completions.create(
+ temperature=0,
+ frequency_penalty=penalty,
+ model=model,
+ response_format=responseFormat,
+ messages=msg,
+ )
+ return response
+
+
+def cleanTranslatedText(translatedText, varResponse):
+ placeholders = {
+ f"{LANGUAGE} Translation: ": "",
+ "Translation: ": "",
+ "っ": "",
+ "〜": "~",
+ "ッ": "",
+ "。": ".",
+ "「": '\\"',
+ "」": '\\"',
+ "- ": "-",
+ "—": "―",
+ "】": "]",
+ "【": "[",
+ "é": "e",
+ "Placeholder Text": "",
+ # Add more replacements as needed
+ }
+ for target, replacement in placeholders.items():
+ translatedText = translatedText.replace(target, replacement)
+
+ # Remove Repeating Characters
+ pattern = re.compile(r"(.)\s*\1(?:\s*\1){" + str(20 - 1) + r",}")
+ translatedText = pattern.sub(lambda match: match.group(0).replace(" ", "")[:20], translatedText)
+
+ # Elongate Long Dashes (Since GPT Ignores them...)
+ translatedText = elongateCharacters(translatedText)
+ return translatedText
+
+
+def elongateCharacters(text):
+ # Define a pattern to match one character followed by one or more `ー` characters
+ # Using a positive lookbehind assertion to capture the preceding character
+ pattern = r"(?<=(.))ー+"
+
+ # Define a replacement function that elongates the captured character
+ def repl(match):
+ char = match.group(1) # The character before the ー sequence
+ count = len(match.group(0)) - 1 # Number of ー characters
+ return char * count # Replace ー sequence with the character repeated
+
+ # Use re.sub() to replace the pattern in the text
+ return re.sub(pattern, repl, text)
+
+
+def extractTranslation(translatedTextList, is_list):
+ try:
+ translatedTextList = re.sub(r'\\"+\"([^,\n}])', r'\\"\1', translatedTextList)
+ translatedTextList = re.sub(r"(? instead
-FIXTEXTWRAP = True # Overwrites textwrap
-IGNORETLTEXT = False # Ignores all translated text.
-MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
-
-# tqdm Globals
-BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
-POSITION = 0
-LEAVE = False
-PBAR = None
-
-# Pricing - Depends on the model https://openai.com/pricing
-# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request
-# If you are getting a MISMATCH LENGTH error, lower the batch size.
-if "gpt-3.5" in MODEL:
- INPUTAPICOST = 0.002
- OUTPUTAPICOST = 0.002
- BATCHSIZE = 10
-elif "gpt-4" in MODEL:
- INPUTAPICOST = 0.0025
- OUTPUTAPICOST = 0.01
- BATCHSIZE = 40
-else:
- INPUTAPICOST = float(os.getenv("input_cost"))
- OUTPUTAPICOST = float(os.getenv("output_cost"))
- BATCHSIZE = int(os.getenv("batchsize"))
- FREQUENCY_PENALTY = float(os.getenv("frequency_penalty"))
-
-def handleEushully(filename, estimate):
- global ESTIMATE
- ESTIMATE = estimate
-
- if ESTIMATE:
- start = time.time()
- translatedData = openFiles(filename)
-
- # Print Result
- end = time.time()
- tqdm.write(getResultString(translatedData, end - start, filename))
- with LOCK:
- TOKENS[0] += translatedData[1][0]
- TOKENS[1] += translatedData[1][1]
-
- # Print Total
- totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
-
- # Print any errors on maps
- if len(MISMATCH) > 0:
- return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
- else:
- return totalString
-
- else:
- try:
- with open("translated/" + filename, "w", encoding="utf-8", errors="ignore") as outFile:
- start = time.time()
- translatedData = openFiles(filename)
-
- # Print Result
- end = time.time()
- outFile.writelines(translatedData[0])
- tqdm.write(getResultString(translatedData, end - start, filename))
- with LOCK:
- TOKENS[0] += translatedData[1][0]
- TOKENS[1] += translatedData[1][1]
- except Exception:
- traceback.print_exc()
- return "Fail"
-
- return getResultString(["", TOKENS, None], end - start, "TOTAL")
-
-
-def getResultString(translatedData, translationTime, filename):
- # File Print String
- totalTokenstring = (
- Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
- "[Output: "
- + str(translatedData[1][1])
- + "]" "[Cost: ${:,.4f}".format((translatedData[1][0] * 0.001 * INPUTAPICOST) + (translatedData[1][1] * 0.001 * OUTPUTAPICOST))
- + "]"
- )
- timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
-
- if translatedData[2] == None:
- # Success
- return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET
-
- else:
- # Fail
- try:
- raise translatedData[2]
- except Exception as e:
- traceback.print_exc()
- errorString = str(e) + Fore.RED
- return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET
-
-
-def openFiles(filename):
- with open("files/" + filename, "r", encoding="utf-8") as readFile:
- translatedData = parseRegex(readFile, filename)
-
- # Delete lines marked for deletion
- finalData = []
- for line in translatedData[0]:
- if line != "\\d\n":
- finalData.append(line)
- translatedData[0] = finalData
-
- return translatedData
-
-
-def parseRegex(readFile, filename):
- totalTokens = [0, 0]
-
- # Read File into data
- data = readFile.readlines()
-
- # Create Progress Bar
- with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
- pbar.desc = filename
-
- try:
- result = translateEushully(data, pbar, filename, [])
- totalTokens[0] += result[0]
- totalTokens[1] += result[1]
- except Exception as e:
- traceback.print_exc()
- return [data, totalTokens, e]
- return [data, totalTokens, None]
-
-
-def translateEushully(data, pbar, filename, translatedList):
- stringList = []
- currentGroup = []
- tokens = [0, 0]
- speaker = ""
- voice = False
- global LOCK, ESTIMATE, PBAR
- i = 0
-
- while i < len(data):
- voice = False
- # Speaker
- if "mov (global-int 46e2)" in data[i]:
- # Get Speaker
- speaker = re.search(r"mov \(global-int 46e2\)\s(.+)", data[i]).group(1)
- response = getSpeaker(speaker)
- speaker = response[0]
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- i += 1
-
- # Show Text
- if any(x in data[i] for x in ["show-text"]):
- # Lines
- regex = r'(.*?)"(.*)"'
- match = re.search(regex, data[i])
- # Grab Strings
- if match != None and match.group(2) != "":
- originalString = match.group(2)
- jaString = match.group(2)
- currentGroup = [jaString]
- while "end-text-line" in data[i + 1] and any(x in data[i + 2] for x in ["show-text"]):
- match = re.search(regex, data[i + 2])
- if match != None:
- currentGroup.append(match.group(2))
- if translatedList == []:
- del data[i]
- del data[i]
- jaString = " ".join(currentGroup)
-
- # Pass 1
- if translatedList == []:
- # Add String
- if speaker:
- stringList.append(f"[{speaker}]: {jaString.strip()}")
- else:
- stringList.append(jaString.strip())
-
- # Pass 2
- else:
- # Get Text
- if translatedList:
- # Grab and Pop
- translatedText = translatedList[0]
- translatedList.pop(0)
-
- # Set to None if empty list
- if len(translatedList) <= 0:
- translatedList = None
-
- # Replace Quotes
- translatedText = translatedText.replace('"', "'")
-
- # Remove speaker
- if speaker != "":
- translatedText = re.sub(r"^\[?(.+?)\]?\s?[|:]\s?", "", translatedText)
-
- # Textwrap
- translatedText = textwrap.fill(translatedText, width=WIDTH)
- translatedTextList = translatedText.split("\n")
-
- # Set Data
- if len(translatedTextList) > 1:
- for j in range(len(translatedTextList)):
- if any(x in data[i] for x in ["show-text", "set-string", "concat"]):
- del data[i]
- data.insert(i, f'{match.group(1)}"{translatedTextList[j]}"\n')
- i += 1
- if "end-text-line" not in data[i]:
- data.insert(i, "end-text-line 0\n")
- i += 1
- else:
- data[i] = f'{match.group(1)}"{translatedTextList[0]}"\n'
- speaker = ""
- i += 1
-
- # Nothing relevant. Skip Line.
- else:
- i += 1
-
- # Set String
- elif "set-string" in data[i]:
- # Lines
- regex = r'(.*?)"(.*)"'
- match = re.search(regex, data[i])
- # Grab Strings
- if match != None and match.group(2) != "":
- originalString = match.group(2)
- jaString = match.group(2)
- currentGroup = [jaString]
-
- # Remove Textwrap
- jaString = jaString.replace("\\n", " ")
-
- # Pass 1
- if translatedList == []:
- # Add String
- stringList.append(jaString.strip())
-
- # Pass 2
- else:
- # Get Text
- if translatedList:
- # Grab and Pop
- translatedText = translatedList[0]
- translatedList.pop(0)
-
- # Set to None if empty list
- if len(translatedList) <= 0:
- translatedList = None
-
- # Replace Quotes
- translatedText = translatedText.replace('"', "'")
-
- # Textwrap
- translatedText = textwrap.fill(translatedText, width=LISTWIDTH)
- translatedText = translatedText.replace("\n", "\\n")
-
- # Set Data
- data[i] = data[i].replace(originalString, translatedText)
- speaker = ""
- i += 1
-
- # Nothing relevant. Skip Line.
- else:
- i += 1
- else:
- i += 1
-
- # EOF
- if len(stringList) > 0:
- # Set Progress
- pbar.total = len(stringList)
- pbar.refresh()
-
- # Translate
- PBAR = pbar
- response = translateGPT(stringList, "", True)
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- translatedList = response[0]
-
- # Set Strings
- if len(stringList) == len(translatedList):
- translateEushully(data, pbar, filename, translatedList)
-
- # Mismatch
- else:
- with LOCK:
- if filename not in MISMATCH:
- MISMATCH.append(filename)
- return tokens
-
-
-# Save some money and enter the character before translation
-def getSpeaker(speaker):
- match speaker:
- case "1":
- return ["Klaus", [0, 0]]
- case "2":
- return ["Helmina", [0, 0]]
- case "3":
- return ["Juliana", [0, 0]]
- case "4":
- return ["Reginia", [0, 0]]
- case "5":
- return ["Luciel", [0, 0]]
- case "6":
- return ["Mavislaine", [0, 0]]
- case "7":
- return ["Cerouge", [0, 0]]
- case "8":
- return ["Maize", [0, 0]]
- case "9":
- return ["Elvire", [0, 0]]
- case "a":
- return ["Beatrice", [0, 0]]
- case "295":
- return ["Orc", [0, 0]]
- case "232":
- return ["Archangel", [0, 0]]
- case "238":
- return ["False Juliana", [0, 0]]
- case "239":
- return ["False Regina", [0, 0]]
- case "23a":
- return ["False Luciel", [0, 0]]
- case "23d":
- return ["False Mavislaine", [0, 0]]
- case "cb":
- return ["Olga Niza Kite", [0, 0]]
- case "c9":
- return ["Demon Beast Lupus", [0, 0]]
- case "ca":
- return ["Evelinael", [0, 0]]
- case "10":
- return ["Eukleia", [0, 0]]
- case "15":
- return ["Lily", [0, 0]]
- case "16":
- return ["Kupuko", [0, 0]]
- case "b":
- return ["Ramiel", [0, 0]]
- case "c":
- return ["Henriette", [0, 0]]
- case "d":
- return ["Camilla", [0, 0]]
- case "cc":
- return ["Gogonaua", [0, 0]]
- case "65":
- return ["Demon Lord Reyvalois", [0, 0]]
- case "d0":
- return ["Demon Ranwald", [0, 0]]
- case "205":
- return ["Vanqueor", [0, 0]]
- case "66":
- return ["Angel Martina", [0, 0]]
- case "21f":
- return ["Hiten Demon", [0, 0]]
- case "d2":
- return ["Lena Eli", [0, 0]]
- case _:
- return ["Unknown", [0, 0]]
-
-
-def subVars(jaString):
- jaString = jaString.replace("\u3000", " ")
-
- # Nested
- count = 0
- nestedList = re.findall(r"[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]", jaString)
- nestedList = set(nestedList)
- if len(nestedList) != 0:
- for icon in nestedList:
- jaString = jaString.replace(icon, "[Nested_" + str(count) + "]")
- count += 1
-
- # Icons
- count = 0
- iconList = re.findall(r"[\\]+[iIkKwWaA]+\[[0-9]+\]", jaString)
- iconList = set(iconList)
- if len(iconList) != 0:
- for icon in iconList:
- jaString = jaString.replace(icon, "[Ascii_" + str(count) + "]")
- count += 1
-
- # Colors
- count = 0
- colorList = re.findall(r"[\\]+[cC]\[[0-9]+\]", jaString)
- colorList = set(colorList)
- if len(colorList) != 0:
- for color in colorList:
- jaString = jaString.replace(color, "[Color_" + str(count) + "]")
- count += 1
-
- # Names
- count = 0
- nameList = re.findall(r"[\\]+[nN]\[.+?\]+", jaString)
- nameList = set(nameList)
- if len(nameList) != 0:
- for name in nameList:
- jaString = jaString.replace(name, "[Noun_" + str(count) + "]")
- count += 1
-
- # Variables
- count = 0
- varList = re.findall(r"[\\]+[vV]\[[0-9]+\]", jaString)
- varList = set(varList)
- if len(varList) != 0:
- for var in varList:
- jaString = jaString.replace(var, "[Var_" + str(count) + "]")
- count += 1
-
- # Formatting
- count = 0
- formatList = re.findall(r"[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]", jaString)
- formatList = set(formatList)
- if len(formatList) != 0:
- for var in formatList:
- jaString = jaString.replace(var, "[FCode_" + str(count) + "]")
- count += 1
-
- # Put all lists in list and return
- allList = [nestedList, iconList, colorList, nameList, varList, formatList]
- return [jaString, allList]
-
-
-def resubVars(translatedText, allList):
- # Fix Spacing and ChatGPT Nonsense
- matchList = re.findall(r"\[\s?.+?\s?\]", translatedText)
- if len(matchList) > 0:
- for match in matchList:
- text = match.strip()
- translatedText = translatedText.replace(match, text)
-
- # Nested
- count = 0
- if len(allList[0]) != 0:
- for var in allList[0]:
- translatedText = translatedText.replace("[Nested_" + str(count) + "]", var)
- count += 1
-
- # Icons
- count = 0
- if len(allList[1]) != 0:
- for var in allList[1]:
- translatedText = translatedText.replace("[Ascii_" + str(count) + "]", var)
- count += 1
-
- # Colors
- count = 0
- if len(allList[2]) != 0:
- for var in allList[2]:
- translatedText = translatedText.replace("[Color_" + str(count) + "]", var)
- count += 1
-
- # Names
- count = 0
- if len(allList[3]) != 0:
- for var in allList[3]:
- translatedText = translatedText.replace("[Noun_" + str(count) + "]", var)
- count += 1
-
- # Vars
- count = 0
- if len(allList[4]) != 0:
- for var in allList[4]:
- translatedText = translatedText.replace("[Var_" + str(count) + "]", var)
- count += 1
-
- # Formatting
- count = 0
- if len(allList[5]) != 0:
- for var in allList[5]:
- translatedText = translatedText.replace("[FCode_" + str(count) + "]", var)
- count += 1
-
- return translatedText
-
-
-def batchList(input_list, batch_size):
- if not isinstance(batch_size, int) or batch_size <= 0:
- raise ValueError("batch_size must be a positive integer")
-
- return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)]
-
-
-def createContext(fullPromptFlag, subbedT):
- characters = "Game Characters:\n\
-グレイス (Grace) - Female\n\
-"
-
- system = (
- PROMPT + VOCAB
- if fullPromptFlag
- else f"\
-You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
-Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
-- You translate everything, including erotic and explicit content.\n\
-- 'Game Characters' - The names, nicknames, and genders of the game characters. Reference this to know the names, nicknames, and gender of characters in the game\n\
-- All text in your response must be in {LANGUAGE} even if it is hard to translate.\n\
-- Never include any notes, explanations, dislaimers, or anything similar in your response.\n\
-- Maintain any spacing in the translation.\n\
-- Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\
-- `...` can be a part of the dialogue. Translate it as it is.\n\
-{VOCAB}\n\
-"
- )
- user = f"{subbedT}"
- return characters, system, user
-
-
-def translateText(characters, system, user, history, penalty):
- # Prompt
- msg = [{"role": "system", "content": system + characters}]
-
- # Characters
- msg.append({"role": "system", "content": characters})
-
- # History
- if isinstance(history, list):
- msg.extend([{"role": "system", "content": h} for h in history])
- else:
- msg.append({"role": "system", "content": history})
-
- # Content to TL
- msg.append({"role": "user", "content": f"{user}"})
- response = openai.chat.completions.create(
- temperature=0,
- frequency_penalty=penalty,
- model=MODEL,
- messages=msg,
- )
- return response
-
-
-def cleanTranslatedText(translatedText, varResponse):
- placeholders = {
- f"{LANGUAGE} Translation: ": "",
- "Translation: ": "",
- "っ": "",
- "〜": "~",
- "ッ": "",
- "。": ".",
- "< ": "<",
- "": ">",
- "「": '"',
- "」": '"',
- "Placeholder Text": "",
- "- chan": "-chan",
- "- kun": "-kun",
- "- san": "-san",
- # Add more replacements as needed
- }
- for target, replacement in placeholders.items():
- translatedText = translatedText.replace(target, replacement)
-
- # Elongate Long Dashes (Since GPT Ignores them...)
- translatedText = elongateCharacters(translatedText)
- translatedText = resubVars(translatedText, varResponse[1])
- return translatedText
-
-
-def elongateCharacters(text):
- # Define a pattern to match one character followed by one or more `ー` characters
- # Using a positive lookbehind assertion to capture the preceding character
- pattern = r"(?<=(.))ー+"
-
- # Define a replacement function that elongates the captured character
- def repl(match):
- char = match.group(1) # The character before the ー sequence
- count = len(match.group(0)) - 1 # Number of ー characters
- return char * count # Replace ー sequence with the character repeated
-
- # Use re.sub() to replace the pattern in the text
- return re.sub(pattern, repl, text)
-
-
-def extractTranslation(translatedTextList, is_list):
- pattern = r"`?<[Ll]ine\d+>([\\]*.*?[\\]*?)<\/?[Ll]ine\d+>`?"
- # If it's a batch (i.e., list), extract with tags; otherwise, return the single item.
- if is_list:
- matchList = re.findall(pattern, translatedTextList)
- return matchList
- else:
- matchList = re.findall(pattern, translatedTextList)
- return matchList[0][0] if matchList else translatedTextList
-
-
-def countTokens(characters, system, user, history):
- inputTotalTokens = 0
- outputTotalTokens = 0
- enc = tiktoken.encoding_for_model("gpt-4")
-
- # Input
- if isinstance(history, list):
- for line in history:
- inputTotalTokens += len(enc.encode(line))
- else:
- inputTotalTokens += len(enc.encode(history))
- inputTotalTokens += len(enc.encode(system))
- inputTotalTokens += len(enc.encode(characters))
- inputTotalTokens += len(enc.encode(user))
-
- # Output
- outputTotalTokens += round(len(enc.encode(user)) * 3)
-
- return [inputTotalTokens, outputTotalTokens]
-
-
-@retry(exceptions=Exception, tries=5, delay=5)
-def translateGPT(text, history, fullPromptFlag):
- global PBAR
-
- mismatch = False
- totalTokens = [0, 0]
- if isinstance(text, list):
- tList = batchList(text, BATCHSIZE)
- else:
- tList = [text]
-
- for index, tItem in enumerate(tList):
- # Before sending to translation, if we have a list of items, add the formatting
- if isinstance(tItem, list):
- payload = "\n".join([f"`{item}`" for i, item in enumerate(tItem)])
- payload = re.sub(r"(<)(\/Line\d+>)", r"\1>Placeholder Text<\3", payload)
- varResponse = subVars(payload)
- subbedT = varResponse[0]
- else:
- varResponse = subVars(tItem)
- subbedT = varResponse[0]
-
- # Things to Check before starting translation
- if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", subbedT):
- if PBAR is not None:
- PBAR.update(len(tItem))
- continue
-
- # Create Message
- characters, system, user = createContext(fullPromptFlag, subbedT)
-
- # Calculate Estimate
- if ESTIMATE:
- estimate = countTokens(characters, system, user, history)
- totalTokens[0] += estimate[0]
- totalTokens[1] += estimate[1]
- continue
-
- # Translating
- response = translateText(characters, system, user, history, 0.02)
- translatedText = response.choices[0].message.content
- totalTokens[0] += response.usage.prompt_tokens
- totalTokens[1] += response.usage.completion_tokens
-
- # Formatting
- translatedText = cleanTranslatedText(translatedText, varResponse)
- if isinstance(tItem, list):
- extractedTranslations = extractTranslation(translatedText, True)
- tList[index] = extractedTranslations
- if len(tItem) != len(extractedTranslations):
- # Mismatch. Try Again
- response = translateText(characters, system, user, history, 0.2)
- translatedText = response.choices[0].message.content
- totalTokens[0] += response.usage.prompt_tokens
- totalTokens[1] += response.usage.completion_tokens
-
- # Formatting
- translatedText = cleanTranslatedText(translatedText, varResponse)
- if isinstance(tItem, list):
- extractedTranslations = extractTranslation(translatedText, True)
- tList[index] = extractedTranslations
- if len(tItem) != len(extractedTranslations):
- mismatch = True # Just here for breakpoint
-
- # Create History
- with LOCK:
- if PBAR is not None:
- PBAR.update(len(tItem))
- if not mismatch:
- history = extractedTranslations[-10:] # Update history if we have a list
- else:
- history = text[-10:]
- else:
- # Ensure we're passing a single string to extractTranslation
- extractedTranslations = extractTranslation(translatedText, False)
- tList[index] = extractedTranslations
-
- # Combine if multilist
- if isinstance(tList[0], list):
- tList = [t for sublist in tList for t in sublist]
-
- # Return
- if format == "json":
- return [tList, totalTokens]
- else:
- return [tList[0], totalTokens]
+# Libraries
+import os
+import re
+import textwrap
+import threading
+import time
+import traceback
+import tiktoken
+import openai
+from pathlib import Path
+from colorama import Fore
+from dotenv import load_dotenv
+from retry import retry
+from tqdm import tqdm
+
+# Open AI
+load_dotenv()
+if os.getenv("api").replace(" ", "") != "":
+ openai.base_url = os.getenv("api")
+openai.organization = os.getenv("org")
+openai.api_key = os.getenv("key")
+
+# Globals
+MODEL = os.getenv("model")
+TIMEOUT = int(os.getenv("timeout"))
+LANGUAGE = os.getenv("language").capitalize()
+PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
+VOCAB = Path("vocab.txt").read_text(encoding="utf-8")
+THREADS = int(os.getenv("threads"))
+LOCK = threading.Lock()
+WIDTH = int(os.getenv("width"))
+LISTWIDTH = int(os.getenv("listWidth"))
+NOTEWIDTH = 70
+MAXHISTORY = 10
+ESTIMATE = ""
+TOKENS = [0, 0]
+NAMESLIST = []
+NAMES = False # Output a list of all the character names found
+BRFLAG = False # If the game uses
instead
+FIXTEXTWRAP = True # Overwrites textwrap
+IGNORETLTEXT = False # Ignores all translated text.
+MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
+
+# tqdm Globals
+BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
+POSITION = 0
+LEAVE = False
+PBAR = None
+
+# Pricing - Depends on the model https://openai.com/pricing
+# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request
+# If you are getting a MISMATCH LENGTH error, lower the batch size.
+if "gpt-3.5" in MODEL:
+ INPUTAPICOST = 0.002
+ OUTPUTAPICOST = 0.002
+ BATCHSIZE = 10
+elif "gpt-4" in MODEL:
+ INPUTAPICOST = 0.0025
+ OUTPUTAPICOST = 0.01
+ BATCHSIZE = 40
+else:
+ INPUTAPICOST = float(os.getenv("input_cost"))
+ OUTPUTAPICOST = float(os.getenv("output_cost"))
+ BATCHSIZE = int(os.getenv("batchsize"))
+ FREQUENCY_PENALTY = float(os.getenv("frequency_penalty"))
+
+
+def handleEushully(filename, estimate):
+ global ESTIMATE
+ ESTIMATE = estimate
+
+ if ESTIMATE:
+ start = time.time()
+ translatedData = openFiles(filename)
+
+ # Print Result
+ end = time.time()
+ tqdm.write(getResultString(translatedData, end - start, filename))
+ with LOCK:
+ TOKENS[0] += translatedData[1][0]
+ TOKENS[1] += translatedData[1][1]
+
+ # Print Total
+ totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
+
+ # Print any errors on maps
+ if len(MISMATCH) > 0:
+ return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
+ else:
+ return totalString
+
+ else:
+ try:
+ with open("translated/" + filename, "w", encoding="utf-8", errors="ignore") as outFile:
+ start = time.time()
+ translatedData = openFiles(filename)
+
+ # Print Result
+ end = time.time()
+ outFile.writelines(translatedData[0])
+ tqdm.write(getResultString(translatedData, end - start, filename))
+ with LOCK:
+ TOKENS[0] += translatedData[1][0]
+ TOKENS[1] += translatedData[1][1]
+ except Exception:
+ traceback.print_exc()
+ return "Fail"
+
+ return getResultString(["", TOKENS, None], end - start, "TOTAL")
+
+
+def getResultString(translatedData, translationTime, filename):
+ # File Print String
+ totalTokenstring = (
+ Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
+ "[Output: "
+ + str(translatedData[1][1])
+ + "]" "[Cost: ${:,.4f}".format((translatedData[1][0] * 0.001 * INPUTAPICOST) + (translatedData[1][1] * 0.001 * OUTPUTAPICOST))
+ + "]"
+ )
+ timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
+
+ if translatedData[2] == None:
+ # Success
+ return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET
+
+ else:
+ # Fail
+ try:
+ raise translatedData[2]
+ except Exception as e:
+ traceback.print_exc()
+ errorString = str(e) + Fore.RED
+ return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET
+
+
+def openFiles(filename):
+ with open("files/" + filename, "r", encoding="utf-8") as readFile:
+ translatedData = parseRegex(readFile, filename)
+
+ # Delete lines marked for deletion
+ finalData = []
+ for line in translatedData[0]:
+ if line != "\\d\n":
+ finalData.append(line)
+ translatedData[0] = finalData
+
+ return translatedData
+
+
+def parseRegex(readFile, filename):
+ totalTokens = [0, 0]
+
+ # Read File into data
+ data = readFile.readlines()
+
+ # Create Progress Bar
+ with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
+ pbar.desc = filename
+
+ try:
+ result = translateEushully(data, pbar, filename, [])
+ totalTokens[0] += result[0]
+ totalTokens[1] += result[1]
+ except Exception as e:
+ traceback.print_exc()
+ return [data, totalTokens, e]
+ return [data, totalTokens, None]
+
+
+def translateEushully(data, pbar, filename, translatedList):
+ stringList = []
+ currentGroup = []
+ tokens = [0, 0]
+ speaker = ""
+ voice = False
+ global LOCK, ESTIMATE, PBAR
+ i = 0
+
+ while i < len(data):
+ voice = False
+ # Speaker
+ if "mov (global-int 46e2)" in data[i]:
+ # Get Speaker
+ speaker = re.search(r"mov \(global-int 46e2\)\s(.+)", data[i]).group(1)
+ response = getSpeaker(speaker)
+ speaker = response[0]
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ i += 1
+
+ # Show Text
+ if any(x in data[i] for x in ["show-text"]):
+ # Lines
+ regex = r'(.*?)"(.*)"'
+ match = re.search(regex, data[i])
+ # Grab Strings
+ if match != None and match.group(2) != "":
+ originalString = match.group(2)
+ jaString = match.group(2)
+ currentGroup = [jaString]
+ while "end-text-line" in data[i + 1] and any(x in data[i + 2] for x in ["show-text"]):
+ match = re.search(regex, data[i + 2])
+ if match != None:
+ currentGroup.append(match.group(2))
+ if translatedList == []:
+ del data[i]
+ del data[i]
+ jaString = " ".join(currentGroup)
+
+ # Pass 1
+ if translatedList == []:
+ # Add String
+ if speaker:
+ stringList.append(f"[{speaker}]: {jaString.strip()}")
+ else:
+ stringList.append(jaString.strip())
+
+ # Pass 2
+ else:
+ # Get Text
+ if translatedList:
+ # Grab and Pop
+ translatedText = translatedList[0]
+ translatedList.pop(0)
+
+ # Set to None if empty list
+ if len(translatedList) <= 0:
+ translatedList = None
+
+ # Replace Quotes
+ translatedText = translatedText.replace('"', "'")
+
+ # Remove speaker
+ if speaker != "":
+ translatedText = re.sub(r"^\[?(.+?)\]?\s?[|:]\s?", "", translatedText)
+
+ # Textwrap
+ translatedText = textwrap.fill(translatedText, width=WIDTH)
+ translatedTextList = translatedText.split("\n")
+
+ # Set Data
+ if len(translatedTextList) > 1:
+ for j in range(len(translatedTextList)):
+ if any(x in data[i] for x in ["show-text", "set-string", "concat"]):
+ del data[i]
+ data.insert(i, f'{match.group(1)}"{translatedTextList[j]}"\n')
+ i += 1
+ if "end-text-line" not in data[i]:
+ data.insert(i, "end-text-line 0\n")
+ i += 1
+ else:
+ data[i] = f'{match.group(1)}"{translatedTextList[0]}"\n'
+ speaker = ""
+ i += 1
+
+ # Nothing relevant. Skip Line.
+ else:
+ i += 1
+
+ # Set String
+ elif "set-string" in data[i]:
+ # Lines
+ regex = r'(.*?)"(.*)"'
+ match = re.search(regex, data[i])
+ # Grab Strings
+ if match != None and match.group(2) != "":
+ originalString = match.group(2)
+ jaString = match.group(2)
+ currentGroup = [jaString]
+
+ # Remove Textwrap
+ jaString = jaString.replace("\\n", " ")
+
+ # Pass 1
+ if translatedList == []:
+ # Add String
+ stringList.append(jaString.strip())
+
+ # Pass 2
+ else:
+ # Get Text
+ if translatedList:
+ # Grab and Pop
+ translatedText = translatedList[0]
+ translatedList.pop(0)
+
+ # Set to None if empty list
+ if len(translatedList) <= 0:
+ translatedList = None
+
+ # Replace Quotes
+ translatedText = translatedText.replace('"', "'")
+
+ # Textwrap
+ translatedText = textwrap.fill(translatedText, width=LISTWIDTH)
+ translatedText = translatedText.replace("\n", "\\n")
+
+ # Set Data
+ data[i] = data[i].replace(originalString, translatedText)
+ speaker = ""
+ i += 1
+
+ # Nothing relevant. Skip Line.
+ else:
+ i += 1
+ else:
+ i += 1
+
+ # EOF
+ if len(stringList) > 0:
+ # Set Progress
+ pbar.total = len(stringList)
+ pbar.refresh()
+
+ # Translate
+ PBAR = pbar
+ response = translateGPT(stringList, "", True)
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ translatedList = response[0]
+
+ # Set Strings
+ if len(stringList) == len(translatedList):
+ translateEushully(data, pbar, filename, translatedList)
+
+ # Mismatch
+ else:
+ with LOCK:
+ if filename not in MISMATCH:
+ MISMATCH.append(filename)
+ return tokens
+
+
+# Save some money and enter the character before translation
+def getSpeaker(speaker):
+ match speaker:
+ case "1":
+ return ["Klaus", [0, 0]]
+ case "2":
+ return ["Helmina", [0, 0]]
+ case "3":
+ return ["Juliana", [0, 0]]
+ case "4":
+ return ["Reginia", [0, 0]]
+ case "5":
+ return ["Luciel", [0, 0]]
+ case "6":
+ return ["Mavislaine", [0, 0]]
+ case "7":
+ return ["Cerouge", [0, 0]]
+ case "8":
+ return ["Maize", [0, 0]]
+ case "9":
+ return ["Elvire", [0, 0]]
+ case "a":
+ return ["Beatrice", [0, 0]]
+ case "295":
+ return ["Orc", [0, 0]]
+ case "232":
+ return ["Archangel", [0, 0]]
+ case "238":
+ return ["False Juliana", [0, 0]]
+ case "239":
+ return ["False Regina", [0, 0]]
+ case "23a":
+ return ["False Luciel", [0, 0]]
+ case "23d":
+ return ["False Mavislaine", [0, 0]]
+ case "cb":
+ return ["Olga Niza Kite", [0, 0]]
+ case "c9":
+ return ["Demon Beast Lupus", [0, 0]]
+ case "ca":
+ return ["Evelinael", [0, 0]]
+ case "10":
+ return ["Eukleia", [0, 0]]
+ case "15":
+ return ["Lily", [0, 0]]
+ case "16":
+ return ["Kupuko", [0, 0]]
+ case "b":
+ return ["Ramiel", [0, 0]]
+ case "c":
+ return ["Henriette", [0, 0]]
+ case "d":
+ return ["Camilla", [0, 0]]
+ case "cc":
+ return ["Gogonaua", [0, 0]]
+ case "65":
+ return ["Demon Lord Reyvalois", [0, 0]]
+ case "d0":
+ return ["Demon Ranwald", [0, 0]]
+ case "205":
+ return ["Vanqueor", [0, 0]]
+ case "66":
+ return ["Angel Martina", [0, 0]]
+ case "21f":
+ return ["Hiten Demon", [0, 0]]
+ case "d2":
+ return ["Lena Eli", [0, 0]]
+ case _:
+ return ["Unknown", [0, 0]]
+
+
+def subVars(jaString):
+ jaString = jaString.replace("\u3000", " ")
+
+ # Nested
+ count = 0
+ nestedList = re.findall(r"[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]", jaString)
+ nestedList = set(nestedList)
+ if len(nestedList) != 0:
+ for icon in nestedList:
+ jaString = jaString.replace(icon, "[Nested_" + str(count) + "]")
+ count += 1
+
+ # Icons
+ count = 0
+ iconList = re.findall(r"[\\]+[iIkKwWaA]+\[[0-9]+\]", jaString)
+ iconList = set(iconList)
+ if len(iconList) != 0:
+ for icon in iconList:
+ jaString = jaString.replace(icon, "[Ascii_" + str(count) + "]")
+ count += 1
+
+ # Colors
+ count = 0
+ colorList = re.findall(r"[\\]+[cC]\[[0-9]+\]", jaString)
+ colorList = set(colorList)
+ if len(colorList) != 0:
+ for color in colorList:
+ jaString = jaString.replace(color, "[Color_" + str(count) + "]")
+ count += 1
+
+ # Names
+ count = 0
+ nameList = re.findall(r"[\\]+[nN]\[.+?\]+", jaString)
+ nameList = set(nameList)
+ if len(nameList) != 0:
+ for name in nameList:
+ jaString = jaString.replace(name, "[Noun_" + str(count) + "]")
+ count += 1
+
+ # Variables
+ count = 0
+ varList = re.findall(r"[\\]+[vV]\[[0-9]+\]", jaString)
+ varList = set(varList)
+ if len(varList) != 0:
+ for var in varList:
+ jaString = jaString.replace(var, "[Var_" + str(count) + "]")
+ count += 1
+
+ # Formatting
+ count = 0
+ formatList = re.findall(r"[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]", jaString)
+ formatList = set(formatList)
+ if len(formatList) != 0:
+ for var in formatList:
+ jaString = jaString.replace(var, "[FCode_" + str(count) + "]")
+ count += 1
+
+ # Put all lists in list and return
+ allList = [nestedList, iconList, colorList, nameList, varList, formatList]
+ return [jaString, allList]
+
+
+def resubVars(translatedText, allList):
+ # Fix Spacing and ChatGPT Nonsense
+ matchList = re.findall(r"\[\s?.+?\s?\]", translatedText)
+ if len(matchList) > 0:
+ for match in matchList:
+ text = match.strip()
+ translatedText = translatedText.replace(match, text)
+
+ # Nested
+ count = 0
+ if len(allList[0]) != 0:
+ for var in allList[0]:
+ translatedText = translatedText.replace("[Nested_" + str(count) + "]", var)
+ count += 1
+
+ # Icons
+ count = 0
+ if len(allList[1]) != 0:
+ for var in allList[1]:
+ translatedText = translatedText.replace("[Ascii_" + str(count) + "]", var)
+ count += 1
+
+ # Colors
+ count = 0
+ if len(allList[2]) != 0:
+ for var in allList[2]:
+ translatedText = translatedText.replace("[Color_" + str(count) + "]", var)
+ count += 1
+
+ # Names
+ count = 0
+ if len(allList[3]) != 0:
+ for var in allList[3]:
+ translatedText = translatedText.replace("[Noun_" + str(count) + "]", var)
+ count += 1
+
+ # Vars
+ count = 0
+ if len(allList[4]) != 0:
+ for var in allList[4]:
+ translatedText = translatedText.replace("[Var_" + str(count) + "]", var)
+ count += 1
+
+ # Formatting
+ count = 0
+ if len(allList[5]) != 0:
+ for var in allList[5]:
+ translatedText = translatedText.replace("[FCode_" + str(count) + "]", var)
+ count += 1
+
+ return translatedText
+
+
+def batchList(input_list, batch_size):
+ if not isinstance(batch_size, int) or batch_size <= 0:
+ raise ValueError("batch_size must be a positive integer")
+
+ return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)]
+
+
+def createContext(fullPromptFlag, subbedT):
+ characters = "Game Characters:\n\
+グレイス (Grace) - Female\n\
+"
+
+ system = (
+ PROMPT + VOCAB
+ if fullPromptFlag
+ else f"\
+You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
+Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
+- You translate everything, including erotic and explicit content.\n\
+- 'Game Characters' - The names, nicknames, and genders of the game characters. Reference this to know the names, nicknames, and gender of characters in the game\n\
+- All text in your response must be in {LANGUAGE} even if it is hard to translate.\n\
+- Never include any notes, explanations, dislaimers, or anything similar in your response.\n\
+- Maintain any spacing in the translation.\n\
+- Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\
+- `...` can be a part of the dialogue. Translate it as it is.\n\
+{VOCAB}\n\
+"
+ )
+ user = f"{subbedT}"
+ return characters, system, user
+
+
+def translateText(characters, system, user, history, penalty):
+ # Prompt
+ msg = [{"role": "system", "content": system + characters}]
+
+ # Characters
+ msg.append({"role": "system", "content": characters})
+
+ # History
+ if isinstance(history, list):
+ msg.extend([{"role": "system", "content": h} for h in history])
+ else:
+ msg.append({"role": "system", "content": history})
+
+ # Content to TL
+ msg.append({"role": "user", "content": f"{user}"})
+ response = openai.chat.completions.create(
+ temperature=0,
+ frequency_penalty=penalty,
+ model=MODEL,
+ messages=msg,
+ )
+ return response
+
+
+def cleanTranslatedText(translatedText, varResponse):
+ placeholders = {
+ f"{LANGUAGE} Translation: ": "",
+ "Translation: ": "",
+ "っ": "",
+ "〜": "~",
+ "ッ": "",
+ "。": ".",
+ "< ": "<",
+ "": ">",
+ "「": '"',
+ "」": '"',
+ "Placeholder Text": "",
+ "- chan": "-chan",
+ "- kun": "-kun",
+ "- san": "-san",
+ # Add more replacements as needed
+ }
+ for target, replacement in placeholders.items():
+ translatedText = translatedText.replace(target, replacement)
+
+ # Elongate Long Dashes (Since GPT Ignores them...)
+ translatedText = elongateCharacters(translatedText)
+ translatedText = resubVars(translatedText, varResponse[1])
+ return translatedText
+
+
+def elongateCharacters(text):
+ # Define a pattern to match one character followed by one or more `ー` characters
+ # Using a positive lookbehind assertion to capture the preceding character
+ pattern = r"(?<=(.))ー+"
+
+ # Define a replacement function that elongates the captured character
+ def repl(match):
+ char = match.group(1) # The character before the ー sequence
+ count = len(match.group(0)) - 1 # Number of ー characters
+ return char * count # Replace ー sequence with the character repeated
+
+ # Use re.sub() to replace the pattern in the text
+ return re.sub(pattern, repl, text)
+
+
+def extractTranslation(translatedTextList, is_list):
+ pattern = r"`?<[Ll]ine\d+>([\\]*.*?[\\]*?)<\/?[Ll]ine\d+>`?"
+ # If it's a batch (i.e., list), extract with tags; otherwise, return the single item.
+ if is_list:
+ matchList = re.findall(pattern, translatedTextList)
+ return matchList
+ else:
+ matchList = re.findall(pattern, translatedTextList)
+ return matchList[0][0] if matchList else translatedTextList
+
+
+def countTokens(characters, system, user, history):
+ inputTotalTokens = 0
+ outputTotalTokens = 0
+ enc = tiktoken.encoding_for_model("gpt-4")
+
+ # Input
+ if isinstance(history, list):
+ for line in history:
+ inputTotalTokens += len(enc.encode(line))
+ else:
+ inputTotalTokens += len(enc.encode(history))
+ inputTotalTokens += len(enc.encode(system))
+ inputTotalTokens += len(enc.encode(characters))
+ inputTotalTokens += len(enc.encode(user))
+
+ # Output
+ outputTotalTokens += round(len(enc.encode(user)) * 3)
+
+ return [inputTotalTokens, outputTotalTokens]
+
+
+@retry(exceptions=Exception, tries=5, delay=5)
+def translateGPT(text, history, fullPromptFlag):
+ global PBAR
+
+ mismatch = False
+ totalTokens = [0, 0]
+ if isinstance(text, list):
+ tList = batchList(text, BATCHSIZE)
+ else:
+ tList = [text]
+
+ for index, tItem in enumerate(tList):
+ # Before sending to translation, if we have a list of items, add the formatting
+ if isinstance(tItem, list):
+ payload = "\n".join([f"`{item}`" for i, item in enumerate(tItem)])
+ payload = re.sub(r"(<)(\/Line\d+>)", r"\1>Placeholder Text<\3", payload)
+ varResponse = subVars(payload)
+ subbedT = varResponse[0]
+ else:
+ varResponse = subVars(tItem)
+ subbedT = varResponse[0]
+
+ # Things to Check before starting translation
+ if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", subbedT):
+ if PBAR is not None:
+ PBAR.update(len(tItem))
+ continue
+
+ # Create Message
+ characters, system, user = createContext(fullPromptFlag, subbedT)
+
+ # Calculate Estimate
+ if ESTIMATE:
+ estimate = countTokens(characters, system, user, history)
+ totalTokens[0] += estimate[0]
+ totalTokens[1] += estimate[1]
+ continue
+
+ # Translating
+ response = translateText(characters, system, user, history, 0.02)
+ translatedText = response.choices[0].message.content
+ totalTokens[0] += response.usage.prompt_tokens
+ totalTokens[1] += response.usage.completion_tokens
+
+ # Formatting
+ translatedText = cleanTranslatedText(translatedText, varResponse)
+ if isinstance(tItem, list):
+ extractedTranslations = extractTranslation(translatedText, True)
+ tList[index] = extractedTranslations
+ if len(tItem) != len(extractedTranslations):
+ # Mismatch. Try Again
+ response = translateText(characters, system, user, history, 0.2)
+ translatedText = response.choices[0].message.content
+ totalTokens[0] += response.usage.prompt_tokens
+ totalTokens[1] += response.usage.completion_tokens
+
+ # Formatting
+ translatedText = cleanTranslatedText(translatedText, varResponse)
+ if isinstance(tItem, list):
+ extractedTranslations = extractTranslation(translatedText, True)
+ tList[index] = extractedTranslations
+ if len(tItem) != len(extractedTranslations):
+ mismatch = True # Just here for breakpoint
+
+ # Create History
+ with LOCK:
+ if PBAR is not None:
+ PBAR.update(len(tItem))
+ if not mismatch:
+ history = extractedTranslations[-10:] # Update history if we have a list
+ else:
+ history = text[-10:]
+ else:
+ # Ensure we're passing a single string to extractTranslation
+ extractedTranslations = extractTranslation(translatedText, False)
+ tList[index] = extractedTranslations
+
+ # Combine if multilist
+ if isinstance(tList[0], list):
+ tList = [t for sublist in tList for t in sublist]
+
+ # Return
+ if format == "json":
+ return [tList, totalTokens]
+ else:
+ return [tList[0], totalTokens]
diff --git a/modules/irissoft.py b/modules/irissoft.py
index 282c9ab..e82e858 100644
--- a/modules/irissoft.py
+++ b/modules/irissoft.py
@@ -1,751 +1,752 @@
-# Libraries
-import os
-import re
-import textwrap
-import threading
-import time
-import traceback
-import tiktoken
-import openai
-from pathlib import Path
-from colorama import Fore
-from dotenv import load_dotenv
-from retry import retry
-from tqdm import tqdm
-
-# Open AI
-load_dotenv()
-if os.getenv("api").replace(" ", "") != "":
- openai.base_url = os.getenv("api")
-openai.organization = os.getenv("org")
-openai.api_key = os.getenv("key")
-
-# Globals
-MODEL = os.getenv("model")
-TIMEOUT = int(os.getenv("timeout"))
-LANGUAGE = os.getenv("language").capitalize()
-PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
-VOCAB = Path("vocab.txt").read_text(encoding="utf-8")
-THREADS = int(os.getenv("threads"))
-LOCK = threading.Lock()
-WIDTH = int(os.getenv("width"))
-LISTWIDTH = int(os.getenv("listWidth"))
-NOTEWIDTH = 70
-MAXHISTORY = 10
-ESTIMATE = ""
-TOKENS = [0, 0]
-NAMESLIST = []
-NAMES = False # Output a list of all the character names found
-BRFLAG = False # If the game uses
instead
-FIXTEXTWRAP = True # Overwrites textwrap
-IGNORETLTEXT = False # Ignores all translated text.
-MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
-
-# tqdm Globals
-BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
-POSITION = 0
-LEAVE = False
-
-# Pricing - Depends on the model https://openai.com/pricing
-# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request
-# If you are getting a MISMATCH LENGTH error, lower the batch size.
-if "gpt-3.5" in MODEL:
- INPUTAPICOST = 0.002
- OUTPUTAPICOST = 0.002
- BATCHSIZE = 10
-elif "gpt-4" in MODEL:
- INPUTAPICOST = 0.0025
- OUTPUTAPICOST = 0.01
- BATCHSIZE = 40
-else:
- INPUTAPICOST = float(os.getenv("input_cost"))
- OUTPUTAPICOST = float(os.getenv("output_cost"))
- BATCHSIZE = int(os.getenv("batchsize"))
- FREQUENCY_PENALTY = float(os.getenv("frequency_penalty"))
-
-def handleIris(filename, estimate):
- global ESTIMATE
- ESTIMATE = estimate
-
- if ESTIMATE:
- start = time.time()
- translatedData = openFiles(filename)
-
- # Print Result
- end = time.time()
- tqdm.write(getResultString(translatedData, end - start, filename))
- with LOCK:
- TOKENS[0] += translatedData[1][0]
- TOKENS[1] += translatedData[1][1]
-
- # Print Total
- totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
-
- # Print any errors on maps
- if len(MISMATCH) > 0:
- return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
- else:
- return totalString
-
- else:
- try:
- with open("translated/" + filename, "w", encoding="cp932", errors="ignore") as outFile:
- start = time.time()
- translatedData = openFiles(filename)
-
- # Print Result
- end = time.time()
- outFile.writelines(translatedData[0])
- tqdm.write(getResultString(translatedData, end - start, filename))
- with LOCK:
- TOKENS[0] += translatedData[1][0]
- TOKENS[1] += translatedData[1][1]
- except Exception:
- traceback.print_exc()
- return "Fail"
-
- return getResultString(["", TOKENS, None], end - start, "TOTAL")
-
-
-def getResultString(translatedData, translationTime, filename):
- # File Print String
- totalTokenstring = (
- Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
- "[Output: "
- + str(translatedData[1][1])
- + "]" "[Cost: ${:,.4f}".format((translatedData[1][0] * 0.001 * INPUTAPICOST) + (translatedData[1][1] * 0.001 * OUTPUTAPICOST))
- + "]"
- )
- timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
-
- if translatedData[2] == None:
- # Success
- return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET
-
- else:
- # Fail
- try:
- raise translatedData[2]
- except Exception as e:
- traceback.print_exc()
- errorString = str(e) + Fore.RED
- return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET
-
-
-def openFiles(filename):
- with open("files/" + filename, "r", encoding="shift_jis") as readFile:
- translatedData = parseIris(readFile, filename)
-
- # Delete lines marked for deletion
- finalData = []
- for line in translatedData[0]:
- if line != "\\d\n":
- finalData.append(line)
- translatedData[0] = finalData
-
- return translatedData
-
-
-def parseIris(readFile, filename):
- totalTokens = [0, 0]
-
- # Read File into data
- data = readFile.readlines()
-
- # Create Progress Bar
- with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
- pbar.desc = filename
-
- try:
- result = translateIris(data, pbar, filename, [])
- totalTokens[0] += result[0]
- totalTokens[1] += result[1]
- except Exception as e:
- traceback.print_exc()
- return [data, totalTokens, e]
- return [data, totalTokens, None]
-
-
-def translateIris(data, pbar, filename, translatedList):
- stringList = []
- currentGroup = []
- tokens = [0, 0]
- speaker = ""
- voice = False
- global LOCK, ESTIMATE
- i = 0
-
- while i < len(data):
- voice = False
- speaker = ""
- if "#MSGVOICE" in data[i]:
- i += 1
- voice = True
- voiceVar = data[i]
- if "#MSG," in data[i] or "#MSG\n" in data[i] or voice == True:
- i += 1
- # Speaker
- if re.search(r'^ ?([^#\/."、。*!！（）\(\)\[\] \n]+)\n', data[i]) and len(data[i]) < 30:
- match = re.search(r"(.*)", data[i])
- if match != None:
- speaker = match.group(1)
- if speaker[0] == "\u3000":
- speaker = speaker[1:]
- response = getSpeaker(speaker, pbar, filename)
- speaker = response[0]
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- if translatedList != []:
- speaker = speaker.replace(" ", "\u3000")
- data[i] = f"\u3000{speaker}\n"
- else:
- speaker = ""
- i += 1
-
- # Lines
- match = re.search(r"(.*)", data[i])
- if match != None and match.group(1) != "":
- # Pass 1
- if translatedList == []:
- # Grab Consecutive Strings
- jaString = data[i]
- if data[i] != "\n":
- if data[i][0] == "\u3000":
- jaString = data[i][1:]
- currentGroup.append(jaString)
- i += 1
- while data[i] != "\n":
- jaString = data[i]
- if data[i] != "\n":
- jaString = data[i][1:]
- currentGroup.append(jaString)
- i += 1
-
- # Join up 401 groups for better translation.
- if len(currentGroup) > 0:
- jaString = "".join(currentGroup)
- currentGroup = []
-
- # Remove any textwrap
- jaString = jaString.replace("\n", " ")
-
- # Temporarily convert spaces (For Textwrap Later)
- jaString = jaString.replace("\u3000", " ")
-
- # Add Speaker (If there is one)
- if speaker != "":
- jaString = f"{speaker}: {jaString}"
-
- # Add String
- stringList.append(jaString.strip())
-
- # Pass 2
- else:
- # Insert Strings
- while data[i] != "\n":
- data.pop(i)
-
- # Get Text
- if translatedList:
- translatedText = translatedList[0]
- translatedList.pop(0)
- if len(translatedList) <= 0:
- translatedList = None
-
- # Remove added speaker
- translatedText = re.sub(r"^.+?:\s", "", translatedText)
-
- # Textwrap
- translatedText = textwrap.fill(translatedText, width=WIDTH)
- translatedText = translatedText.replace("\n", "\n\u3000")
-
- # Replace Whitespace and Commas
- translatedText = translatedText.replace(", ", "、")
- translatedText = translatedText.replace(",\u3000", "、")
- translatedText = translatedText.replace(",", "、")
- translatedText = translatedText.replace(" ", "\u3000")
-
- # Set Data
- # Game crashes on more than 3 lines. Will need to create a new MSG for long translations
- if translatedText.count("\n") > 2:
- # Split List
- translatedTextList = splitNewlines(translatedText)
-
- # MSG Voice
- count = 0
- for text in translatedTextList:
- if count != 0:
- if voice == True:
- # MSG for each item in the list
- data.insert(i, "#MSGVOICE,\n")
- i += 1
- data.insert(i, f"{voiceVar}")
- i += 1
- else:
- data.insert(i, "#MSG,\n")
- i += 1
- if speaker:
- data[i] = f"\u3000{speaker}\n"
- i += 1
- if text[0] == "\u3000":
- data.insert(i, f"{text}\n")
- else:
- data.insert(i, f"\u3000{text}\n")
- i += 1
- count += 1
- if data[i] != "\n":
- data.insert(i, "\n")
- data[i] = f"\n{data[i]}"
- else:
- data.insert(i, f"\u3000{translatedText}\n")
- i += 1
- if data[i] != "\n":
- data[i] = f"\n{data[i]}"
-
- elif "#SELECT" in data[i] and translatedList == []:
- Iris = r"(.+?) +\d$"
- i += 1
- match = re.search(Iris, data[i])
- if match:
- choiceList = []
- choiceList.append(match.group(1))
- i += 1
- match = re.search(Iris, data[i])
- while match:
- choiceList.append(match.group(1))
- i += 1
- match = re.search(Iris, data[i])
-
- # Translate
- question = stringList[len(stringList) - 1]
- response = translateGPT(
- choiceList,
- f"Previous text for context: {question}\n\nThis will be a dialogue option",
- True,
- pbar,
- filename,
- )
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- choiceListTL = response[0]
-
- # Set Data
- i = i - len(choiceListTL)
- for j in range(len(choiceListTL)):
- # Replace Whitespace and Commas
- choiceListTL[j] = choiceListTL[j].replace(", ", "、")
- choiceListTL[j] = choiceListTL[j].replace(",\u3000", "、")
- choiceListTL[j] = choiceListTL[j].replace(",", "、")
- choiceListTL[j] = choiceListTL[j].replace(" ", "\u3000")
- data[i] = data[i].replace(choiceList[j], choiceListTL[j])
- i += 1
-
- # Nothing relevant. Skip Line.
- else:
- i += 1
- else:
- i += 1
-
- # EOF
- if len(stringList) > 0:
- # Set Progress
- pbar.total = len(stringList)
- pbar.refresh()
-
- # Translate
- response = translateGPT(stringList, "", True, pbar, filename)
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- translatedList = response[0]
-
- # Set Strings
- if len(stringList) == len(translatedList):
- translateIris(data, pbar, filename, translatedList)
-
- # Mismatch
- else:
- with LOCK:
- if filename not in MISMATCH:
- MISMATCH.append(filename)
- return tokens
-
-
-def splitNewlines(text):
- parts = []
- newline_count = 0 # Counts the number of newline characters encountered
- start_index = 0 # Start index of the current string part
-
- for i, char in enumerate(text):
- if char == "\n":
- newline_count += 1
- if newline_count == 3:
- # Append the string part from start_index to current index (inclusive)
- parts.append(text[start_index : i + 1])
- # Reset newline count and update start_index for the next string part
- newline_count = 0
- start_index = i + 1
-
- # Edge case: if the text does not end with a newline, we still need to append the last part
- if start_index < len(text):
- parts.append(text[start_index:])
-
- return parts
-
-
-# Save some money and enter the character before translation
-def getSpeaker(speaker, pbar, filename):
- match speaker:
- case "ファイン":
- return ["Fine", [0, 0]]
- case "":
- return ["", [0, 0]]
- case _:
- # Store Speaker
- if speaker not in str(NAMESLIST):
- response = translateGPT(
- speaker,
- "Reply with only the " + LANGUAGE + " translation of the NPC name.",
- False,
- pbar,
- filename,
- )
- response[0] = response[0].replace("'S", "'s")
- speakerList = [speaker, response[0]]
- NAMESLIST.append(speakerList)
- return response
-
- # Find Speaker
- else:
- for i in range(len(NAMESLIST)):
- if speaker == NAMESLIST[i][0]:
- return [NAMESLIST[i][1], [0, 0]]
-
- return [speaker, [0, 0]]
-
-
-def subVars(jaString):
- jaString = jaString.replace("\u3000", " ")
-
- # Nested
- count = 0
- nestedList = re.findall(r"[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]", jaString)
- nestedList = set(nestedList)
- if len(nestedList) != 0:
- for icon in nestedList:
- jaString = jaString.replace(icon, "[Nested_" + str(count) + "]")
- count += 1
-
- # Icons
- count = 0
- iconList = re.findall(r"[\\]+[iIkKwWaA]+\[[0-9]+\]", jaString)
- iconList = set(iconList)
- if len(iconList) != 0:
- for icon in iconList:
- jaString = jaString.replace(icon, "[Ascii_" + str(count) + "]")
- count += 1
-
- # Colors
- count = 0
- colorList = re.findall(r"[\\]+[cC]\[[0-9]+\]", jaString)
- colorList = set(colorList)
- if len(colorList) != 0:
- for color in colorList:
- jaString = jaString.replace(color, "[Color_" + str(count) + "]")
- count += 1
-
- # Names
- count = 0
- nameList = re.findall(r"[\\]+[nN]\[.+?\]+", jaString)
- nameList = set(nameList)
- if len(nameList) != 0:
- for name in nameList:
- jaString = jaString.replace(name, "[Noun_" + str(count) + "]")
- count += 1
-
- # Variables
- count = 0
- varList = re.findall(r"[\\]+[vV]\[[0-9]+\]", jaString)
- varList = set(varList)
- if len(varList) != 0:
- for var in varList:
- jaString = jaString.replace(var, "[Var_" + str(count) + "]")
- count += 1
-
- # Formatting
- count = 0
- formatList = re.findall(r"[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]", jaString)
- formatList = set(formatList)
- if len(formatList) != 0:
- for var in formatList:
- jaString = jaString.replace(var, "[FCode_" + str(count) + "]")
- count += 1
-
- # Put all lists in list and return
- allList = [nestedList, iconList, colorList, nameList, varList, formatList]
- return [jaString, allList]
-
-
-def resubVars(translatedText, allList):
- # Fix Spacing and ChatGPT Nonsense
- matchList = re.findall(r"\[\s?.+?\s?\]", translatedText)
- if len(matchList) > 0:
- for match in matchList:
- text = match.strip()
- translatedText = translatedText.replace(match, text)
-
- # Nested
- count = 0
- if len(allList[0]) != 0:
- for var in allList[0]:
- translatedText = translatedText.replace("[Nested_" + str(count) + "]", var)
- count += 1
-
- # Icons
- count = 0
- if len(allList[1]) != 0:
- for var in allList[1]:
- translatedText = translatedText.replace("[Ascii_" + str(count) + "]", var)
- count += 1
-
- # Colors
- count = 0
- if len(allList[2]) != 0:
- for var in allList[2]:
- translatedText = translatedText.replace("[Color_" + str(count) + "]", var)
- count += 1
-
- # Names
- count = 0
- if len(allList[3]) != 0:
- for var in allList[3]:
- translatedText = translatedText.replace("[Noun_" + str(count) + "]", var)
- count += 1
-
- # Vars
- count = 0
- if len(allList[4]) != 0:
- for var in allList[4]:
- translatedText = translatedText.replace("[Var_" + str(count) + "]", var)
- count += 1
-
- # Formatting
- count = 0
- if len(allList[5]) != 0:
- for var in allList[5]:
- translatedText = translatedText.replace("[FCode_" + str(count) + "]", var)
- count += 1
-
- return translatedText
-
-
-def batchList(input_list, batch_size):
- if not isinstance(batch_size, int) or batch_size <= 0:
- raise ValueError("batch_size must be a positive integer")
-
- return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)]
-
-
-def createContext(fullPromptFlag, subbedT):
- characters = "Game Characters:\n\
-フィリア (Philia) - Female\n\
-アルネット (Annett) - Female\n\
-ラピュセナ (Rapusena) - Female\n\
-リッカ (Rikka) - Female\n\
-アンデリビア (Andelivia) - Female\n\
-リリアブルム (Liliabloom) - Female\n\
-カルナ (Karna) - Female\n\
-ラフィング＝スピア (Laughing Spear) - Female\n\
-ノーラ (Nora) - Female\n\
-"
-
- system = (
- PROMPT + VOCAB
- if fullPromptFlag
- else f"\
-You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
-Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
-- You translate everything, including erotic and explicit content.\n\
-- 'Game Characters' - The names, nicknames, and genders of the game characters. Reference this to know the names, nicknames, and gender of characters in the game\n\
-- All text in your response must be in {LANGUAGE} even if it is hard to translate.\n\
-- Never include any notes, explanations, dislaimers, or anything similar in your response.\n\
-- Maintain any spacing in the translation.\n\
-- Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\
-- `...` can be a part of the dialogue. Translate it as it is.\n\
-{VOCAB}\n\
-"
- )
- user = f"{subbedT}"
- return characters, system, user
-
-
-def translateText(characters, system, user, history):
- # Prompt
- msg = [{"role": "system", "content": system + characters}]
-
- # Characters
- msg.append({"role": "system", "content": characters})
-
- # History
- if isinstance(history, list):
- msg.extend([{"role": "system", "content": h} for h in history])
- else:
- msg.append({"role": "system", "content": history})
-
- # Content to TL
- msg.append({"role": "user", "content": f"{user}"})
- response = openai.chat.completions.create(
- temperature=0.1,
- frequency_penalty=0.1,
- model=MODEL,
- messages=msg,
- )
- return response
-
-
-def cleanTranslatedText(translatedText, varResponse):
- placeholders = {
- f"{LANGUAGE} Translation: ": "",
- "Translation: ": "",
- "っ": "",
- "〜": "~",
- "ッ": "",
- "。": ".",
- "Placeholder Text": "",
- # Add more replacements as needed
- }
- for target, replacement in placeholders.items():
- translatedText = translatedText.replace(target, replacement)
-
- # Elongate Long Dashes (Since GPT Ignores them...)
- translatedText = elongateCharacters(translatedText)
- translatedText = resubVars(translatedText, varResponse[1])
- return translatedText
-
-
-def elongateCharacters(text):
- # Define a pattern to match one character followed by one or more `ー` characters
- # Using a positive lookbehind assertion to capture the preceding character
- pattern = r"(?<=(.))ー+"
-
- # Define a replacement function that elongates the captured character
- def repl(match):
- char = match.group(1) # The character before the ー sequence
- count = len(match.group(0)) - 1 # Number of ー characters
- return char * count # Replace ー sequence with the character repeated
-
- # Use re.sub() to replace the pattern in the text
- return re.sub(pattern, repl, text)
-
-
-def extractTranslation(translatedTextList, is_list):
- pattern = r"`?<[Ll]ine\d+>([\\]*.*?[\\]*?)<\/?[Ll]ine\d+>`?"
- # If it's a batch (i.e., list), extract with tags; otherwise, return the single item.
- if is_list:
- matchList = re.findall(pattern, translatedTextList)
- return matchList
- else:
- matchList = re.findall(pattern, translatedTextList)
- return matchList[0][0] if matchList else translatedTextList
-
-
-def countTokens(characters, system, user, history):
- inputTotalTokens = 0
- outputTotalTokens = 0
- enc = tiktoken.encoding_for_model("gpt-4")
-
- # Input
- if isinstance(history, list):
- for line in history:
- inputTotalTokens += len(enc.encode(line))
- else:
- inputTotalTokens += len(enc.encode(history))
- inputTotalTokens += len(enc.encode(system))
- inputTotalTokens += len(enc.encode(characters))
- inputTotalTokens += len(enc.encode(user))
-
- # Output
- outputTotalTokens += round(len(enc.encode(user)) * 2)
-
- return [inputTotalTokens, outputTotalTokens]
-
-
-@retry(exceptions=Exception, tries=5, delay=5)
-def translateGPT(text, history, fullPromptFlag, pbar, filename):
- mismatch = False
- totalTokens = [0, 0]
- if isinstance(text, list):
- tList = batchList(text, BATCHSIZE)
- else:
- tList = [text]
-
- for index, tItem in enumerate(tList):
- # Before sending to translation, if we have a list of items, add the formatting
- if isinstance(tItem, list):
- payload = "\n".join([f"`{item}`" for i, item in enumerate(tItem)])
- payload = re.sub(r"(<)(\/Line\d+>)", r"\1>Placeholder Text<\3", payload)
- varResponse = subVars(payload)
- subbedT = varResponse[0]
- else:
- varResponse = subVars(tItem)
- subbedT = varResponse[0]
-
- # Things to Check before starting translation
- if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", subbedT):
- continue
-
- # Create Message
- characters, system, user = createContext(fullPromptFlag, subbedT)
-
- # Calculate Estimate
- if ESTIMATE:
- estimate = countTokens(characters, system, user, history)
- totalTokens[0] += estimate[0]
- totalTokens[1] += estimate[1]
- continue
-
- # Translating
- response = translateText(characters, system, user, history)
- translatedText = response.choices[0].message.content
- totalTokens[0] += response.usage.prompt_tokens
- totalTokens[1] += response.usage.completion_tokens
-
- # Formatting
- translatedText = cleanTranslatedText(translatedText, varResponse)
- if isinstance(tItem, list):
- extractedTranslations = extractTranslation(translatedText, True)
- if len(tItem) != len(extractedTranslations):
- # Mismatch. Try Again
- response = translateText(characters, system, user, history)
- translatedText = response.choices[0].message.content
- totalTokens[0] += response.usage.prompt_tokens
- totalTokens[1] += response.usage.completion_tokens
-
- # Formatting
- translatedText = cleanTranslatedText(translatedText, varResponse)
- if isinstance(tItem, list):
- extractedTranslations = extractTranslation(translatedText, True)
- if len(tItem) == len(extractedTranslations):
- tList[index] = extractedTranslations
- else:
- MISMATCH.append(filename)
- else:
- tList[index] = extractedTranslations
-
- # Create History
- history = tList[index] # Update history if we have a list
- pbar.update(len(tList[index]))
-
- else:
- # Ensure we're passing a single string to extractTranslation
- extractedTranslations = extractTranslation(translatedText, False)
- tList[index] = extractedTranslations
-
- # Combine if multilist
- if isinstance(tList[0], list):
- tList = [t for sublist in tList for t in sublist]
-
- # Return
- if format == "json":
- return [tList, totalTokens]
- else:
- return [tList[0], totalTokens]
+# Libraries
+import os
+import re
+import textwrap
+import threading
+import time
+import traceback
+import tiktoken
+import openai
+from pathlib import Path
+from colorama import Fore
+from dotenv import load_dotenv
+from retry import retry
+from tqdm import tqdm
+
+# Open AI
+load_dotenv()
+if os.getenv("api").replace(" ", "") != "":
+ openai.base_url = os.getenv("api")
+openai.organization = os.getenv("org")
+openai.api_key = os.getenv("key")
+
+# Globals
+MODEL = os.getenv("model")
+TIMEOUT = int(os.getenv("timeout"))
+LANGUAGE = os.getenv("language").capitalize()
+PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
+VOCAB = Path("vocab.txt").read_text(encoding="utf-8")
+THREADS = int(os.getenv("threads"))
+LOCK = threading.Lock()
+WIDTH = int(os.getenv("width"))
+LISTWIDTH = int(os.getenv("listWidth"))
+NOTEWIDTH = 70
+MAXHISTORY = 10
+ESTIMATE = ""
+TOKENS = [0, 0]
+NAMESLIST = []
+NAMES = False # Output a list of all the character names found
+BRFLAG = False # If the game uses
instead
+FIXTEXTWRAP = True # Overwrites textwrap
+IGNORETLTEXT = False # Ignores all translated text.
+MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
+
+# tqdm Globals
+BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
+POSITION = 0
+LEAVE = False
+
+# Pricing - Depends on the model https://openai.com/pricing
+# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request
+# If you are getting a MISMATCH LENGTH error, lower the batch size.
+if "gpt-3.5" in MODEL:
+ INPUTAPICOST = 0.002
+ OUTPUTAPICOST = 0.002
+ BATCHSIZE = 10
+elif "gpt-4" in MODEL:
+ INPUTAPICOST = 0.0025
+ OUTPUTAPICOST = 0.01
+ BATCHSIZE = 40
+else:
+ INPUTAPICOST = float(os.getenv("input_cost"))
+ OUTPUTAPICOST = float(os.getenv("output_cost"))
+ BATCHSIZE = int(os.getenv("batchsize"))
+ FREQUENCY_PENALTY = float(os.getenv("frequency_penalty"))
+
+
+def handleIris(filename, estimate):
+ global ESTIMATE
+ ESTIMATE = estimate
+
+ if ESTIMATE:
+ start = time.time()
+ translatedData = openFiles(filename)
+
+ # Print Result
+ end = time.time()
+ tqdm.write(getResultString(translatedData, end - start, filename))
+ with LOCK:
+ TOKENS[0] += translatedData[1][0]
+ TOKENS[1] += translatedData[1][1]
+
+ # Print Total
+ totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
+
+ # Print any errors on maps
+ if len(MISMATCH) > 0:
+ return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
+ else:
+ return totalString
+
+ else:
+ try:
+ with open("translated/" + filename, "w", encoding="cp932", errors="ignore") as outFile:
+ start = time.time()
+ translatedData = openFiles(filename)
+
+ # Print Result
+ end = time.time()
+ outFile.writelines(translatedData[0])
+ tqdm.write(getResultString(translatedData, end - start, filename))
+ with LOCK:
+ TOKENS[0] += translatedData[1][0]
+ TOKENS[1] += translatedData[1][1]
+ except Exception:
+ traceback.print_exc()
+ return "Fail"
+
+ return getResultString(["", TOKENS, None], end - start, "TOTAL")
+
+
+def getResultString(translatedData, translationTime, filename):
+ # File Print String
+ totalTokenstring = (
+ Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
+ "[Output: "
+ + str(translatedData[1][1])
+ + "]" "[Cost: ${:,.4f}".format((translatedData[1][0] * 0.001 * INPUTAPICOST) + (translatedData[1][1] * 0.001 * OUTPUTAPICOST))
+ + "]"
+ )
+ timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
+
+ if translatedData[2] == None:
+ # Success
+ return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET
+
+ else:
+ # Fail
+ try:
+ raise translatedData[2]
+ except Exception as e:
+ traceback.print_exc()
+ errorString = str(e) + Fore.RED
+ return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET
+
+
+def openFiles(filename):
+ with open("files/" + filename, "r", encoding="shift_jis") as readFile:
+ translatedData = parseIris(readFile, filename)
+
+ # Delete lines marked for deletion
+ finalData = []
+ for line in translatedData[0]:
+ if line != "\\d\n":
+ finalData.append(line)
+ translatedData[0] = finalData
+
+ return translatedData
+
+
+def parseIris(readFile, filename):
+ totalTokens = [0, 0]
+
+ # Read File into data
+ data = readFile.readlines()
+
+ # Create Progress Bar
+ with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
+ pbar.desc = filename
+
+ try:
+ result = translateIris(data, pbar, filename, [])
+ totalTokens[0] += result[0]
+ totalTokens[1] += result[1]
+ except Exception as e:
+ traceback.print_exc()
+ return [data, totalTokens, e]
+ return [data, totalTokens, None]
+
+
+def translateIris(data, pbar, filename, translatedList):
+ stringList = []
+ currentGroup = []
+ tokens = [0, 0]
+ speaker = ""
+ voice = False
+ global LOCK, ESTIMATE
+ i = 0
+
+ while i < len(data):
+ voice = False
+ speaker = ""
+ if "#MSGVOICE" in data[i]:
+ i += 1
+ voice = True
+ voiceVar = data[i]
+ if "#MSG," in data[i] or "#MSG\n" in data[i] or voice == True:
+ i += 1
+ # Speaker
+ if re.search(r'^ ?([^#\/."、。*!！（）\(\)\[\] \n]+)\n', data[i]) and len(data[i]) < 30:
+ match = re.search(r"(.*)", data[i])
+ if match != None:
+ speaker = match.group(1)
+ if speaker[0] == "\u3000":
+ speaker = speaker[1:]
+ response = getSpeaker(speaker, pbar, filename)
+ speaker = response[0]
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ if translatedList != []:
+ speaker = speaker.replace(" ", "\u3000")
+ data[i] = f"\u3000{speaker}\n"
+ else:
+ speaker = ""
+ i += 1
+
+ # Lines
+ match = re.search(r"(.*)", data[i])
+ if match != None and match.group(1) != "":
+ # Pass 1
+ if translatedList == []:
+ # Grab Consecutive Strings
+ jaString = data[i]
+ if data[i] != "\n":
+ if data[i][0] == "\u3000":
+ jaString = data[i][1:]
+ currentGroup.append(jaString)
+ i += 1
+ while data[i] != "\n":
+ jaString = data[i]
+ if data[i] != "\n":
+ jaString = data[i][1:]
+ currentGroup.append(jaString)
+ i += 1
+
+ # Join up 401 groups for better translation.
+ if len(currentGroup) > 0:
+ jaString = "".join(currentGroup)
+ currentGroup = []
+
+ # Remove any textwrap
+ jaString = jaString.replace("\n", " ")
+
+ # Temporarily convert spaces (For Textwrap Later)
+ jaString = jaString.replace("\u3000", " ")
+
+ # Add Speaker (If there is one)
+ if speaker != "":
+ jaString = f"{speaker}: {jaString}"
+
+ # Add String
+ stringList.append(jaString.strip())
+
+ # Pass 2
+ else:
+ # Insert Strings
+ while data[i] != "\n":
+ data.pop(i)
+
+ # Get Text
+ if translatedList:
+ translatedText = translatedList[0]
+ translatedList.pop(0)
+ if len(translatedList) <= 0:
+ translatedList = None
+
+ # Remove added speaker
+ translatedText = re.sub(r"^.+?:\s", "", translatedText)
+
+ # Textwrap
+ translatedText = textwrap.fill(translatedText, width=WIDTH)
+ translatedText = translatedText.replace("\n", "\n\u3000")
+
+ # Replace Whitespace and Commas
+ translatedText = translatedText.replace(", ", "、")
+ translatedText = translatedText.replace(",\u3000", "、")
+ translatedText = translatedText.replace(",", "、")
+ translatedText = translatedText.replace(" ", "\u3000")
+
+ # Set Data
+ # Game crashes on more than 3 lines. Will need to create a new MSG for long translations
+ if translatedText.count("\n") > 2:
+ # Split List
+ translatedTextList = splitNewlines(translatedText)
+
+ # MSG Voice
+ count = 0
+ for text in translatedTextList:
+ if count != 0:
+ if voice == True:
+ # MSG for each item in the list
+ data.insert(i, "#MSGVOICE,\n")
+ i += 1
+ data.insert(i, f"{voiceVar}")
+ i += 1
+ else:
+ data.insert(i, "#MSG,\n")
+ i += 1
+ if speaker:
+ data[i] = f"\u3000{speaker}\n"
+ i += 1
+ if text[0] == "\u3000":
+ data.insert(i, f"{text}\n")
+ else:
+ data.insert(i, f"\u3000{text}\n")
+ i += 1
+ count += 1
+ if data[i] != "\n":
+ data.insert(i, "\n")
+ data[i] = f"\n{data[i]}"
+ else:
+ data.insert(i, f"\u3000{translatedText}\n")
+ i += 1
+ if data[i] != "\n":
+ data[i] = f"\n{data[i]}"
+
+ elif "#SELECT" in data[i] and translatedList == []:
+ Iris = r"(.+?) +\d$"
+ i += 1
+ match = re.search(Iris, data[i])
+ if match:
+ choiceList = []
+ choiceList.append(match.group(1))
+ i += 1
+ match = re.search(Iris, data[i])
+ while match:
+ choiceList.append(match.group(1))
+ i += 1
+ match = re.search(Iris, data[i])
+
+ # Translate
+ question = stringList[len(stringList) - 1]
+ response = translateGPT(
+ choiceList,
+ f"Previous text for context: {question}\n\nThis will be a dialogue option",
+ True,
+ pbar,
+ filename,
+ )
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ choiceListTL = response[0]
+
+ # Set Data
+ i = i - len(choiceListTL)
+ for j in range(len(choiceListTL)):
+ # Replace Whitespace and Commas
+ choiceListTL[j] = choiceListTL[j].replace(", ", "、")
+ choiceListTL[j] = choiceListTL[j].replace(",\u3000", "、")
+ choiceListTL[j] = choiceListTL[j].replace(",", "、")
+ choiceListTL[j] = choiceListTL[j].replace(" ", "\u3000")
+ data[i] = data[i].replace(choiceList[j], choiceListTL[j])
+ i += 1
+
+ # Nothing relevant. Skip Line.
+ else:
+ i += 1
+ else:
+ i += 1
+
+ # EOF
+ if len(stringList) > 0:
+ # Set Progress
+ pbar.total = len(stringList)
+ pbar.refresh()
+
+ # Translate
+ response = translateGPT(stringList, "", True, pbar, filename)
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ translatedList = response[0]
+
+ # Set Strings
+ if len(stringList) == len(translatedList):
+ translateIris(data, pbar, filename, translatedList)
+
+ # Mismatch
+ else:
+ with LOCK:
+ if filename not in MISMATCH:
+ MISMATCH.append(filename)
+ return tokens
+
+
+def splitNewlines(text):
+ parts = []
+ newline_count = 0 # Counts the number of newline characters encountered
+ start_index = 0 # Start index of the current string part
+
+ for i, char in enumerate(text):
+ if char == "\n":
+ newline_count += 1
+ if newline_count == 3:
+ # Append the string part from start_index to current index (inclusive)
+ parts.append(text[start_index : i + 1])
+ # Reset newline count and update start_index for the next string part
+ newline_count = 0
+ start_index = i + 1
+
+ # Edge case: if the text does not end with a newline, we still need to append the last part
+ if start_index < len(text):
+ parts.append(text[start_index:])
+
+ return parts
+
+
+# Save some money and enter the character before translation
+def getSpeaker(speaker, pbar, filename):
+ match speaker:
+ case "ファイン":
+ return ["Fine", [0, 0]]
+ case "":
+ return ["", [0, 0]]
+ case _:
+ # Store Speaker
+ if speaker not in str(NAMESLIST):
+ response = translateGPT(
+ speaker,
+ "Reply with only the " + LANGUAGE + " translation of the NPC name.",
+ False,
+ pbar,
+ filename,
+ )
+ response[0] = response[0].replace("'S", "'s")
+ speakerList = [speaker, response[0]]
+ NAMESLIST.append(speakerList)
+ return response
+
+ # Find Speaker
+ else:
+ for i in range(len(NAMESLIST)):
+ if speaker == NAMESLIST[i][0]:
+ return [NAMESLIST[i][1], [0, 0]]
+
+ return [speaker, [0, 0]]
+
+
+def subVars(jaString):
+ jaString = jaString.replace("\u3000", " ")
+
+ # Nested
+ count = 0
+ nestedList = re.findall(r"[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]", jaString)
+ nestedList = set(nestedList)
+ if len(nestedList) != 0:
+ for icon in nestedList:
+ jaString = jaString.replace(icon, "[Nested_" + str(count) + "]")
+ count += 1
+
+ # Icons
+ count = 0
+ iconList = re.findall(r"[\\]+[iIkKwWaA]+\[[0-9]+\]", jaString)
+ iconList = set(iconList)
+ if len(iconList) != 0:
+ for icon in iconList:
+ jaString = jaString.replace(icon, "[Ascii_" + str(count) + "]")
+ count += 1
+
+ # Colors
+ count = 0
+ colorList = re.findall(r"[\\]+[cC]\[[0-9]+\]", jaString)
+ colorList = set(colorList)
+ if len(colorList) != 0:
+ for color in colorList:
+ jaString = jaString.replace(color, "[Color_" + str(count) + "]")
+ count += 1
+
+ # Names
+ count = 0
+ nameList = re.findall(r"[\\]+[nN]\[.+?\]+", jaString)
+ nameList = set(nameList)
+ if len(nameList) != 0:
+ for name in nameList:
+ jaString = jaString.replace(name, "[Noun_" + str(count) + "]")
+ count += 1
+
+ # Variables
+ count = 0
+ varList = re.findall(r"[\\]+[vV]\[[0-9]+\]", jaString)
+ varList = set(varList)
+ if len(varList) != 0:
+ for var in varList:
+ jaString = jaString.replace(var, "[Var_" + str(count) + "]")
+ count += 1
+
+ # Formatting
+ count = 0
+ formatList = re.findall(r"[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]", jaString)
+ formatList = set(formatList)
+ if len(formatList) != 0:
+ for var in formatList:
+ jaString = jaString.replace(var, "[FCode_" + str(count) + "]")
+ count += 1
+
+ # Put all lists in list and return
+ allList = [nestedList, iconList, colorList, nameList, varList, formatList]
+ return [jaString, allList]
+
+
+def resubVars(translatedText, allList):
+ # Fix Spacing and ChatGPT Nonsense
+ matchList = re.findall(r"\[\s?.+?\s?\]", translatedText)
+ if len(matchList) > 0:
+ for match in matchList:
+ text = match.strip()
+ translatedText = translatedText.replace(match, text)
+
+ # Nested
+ count = 0
+ if len(allList[0]) != 0:
+ for var in allList[0]:
+ translatedText = translatedText.replace("[Nested_" + str(count) + "]", var)
+ count += 1
+
+ # Icons
+ count = 0
+ if len(allList[1]) != 0:
+ for var in allList[1]:
+ translatedText = translatedText.replace("[Ascii_" + str(count) + "]", var)
+ count += 1
+
+ # Colors
+ count = 0
+ if len(allList[2]) != 0:
+ for var in allList[2]:
+ translatedText = translatedText.replace("[Color_" + str(count) + "]", var)
+ count += 1
+
+ # Names
+ count = 0
+ if len(allList[3]) != 0:
+ for var in allList[3]:
+ translatedText = translatedText.replace("[Noun_" + str(count) + "]", var)
+ count += 1
+
+ # Vars
+ count = 0
+ if len(allList[4]) != 0:
+ for var in allList[4]:
+ translatedText = translatedText.replace("[Var_" + str(count) + "]", var)
+ count += 1
+
+ # Formatting
+ count = 0
+ if len(allList[5]) != 0:
+ for var in allList[5]:
+ translatedText = translatedText.replace("[FCode_" + str(count) + "]", var)
+ count += 1
+
+ return translatedText
+
+
+def batchList(input_list, batch_size):
+ if not isinstance(batch_size, int) or batch_size <= 0:
+ raise ValueError("batch_size must be a positive integer")
+
+ return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)]
+
+
+def createContext(fullPromptFlag, subbedT):
+ characters = "Game Characters:\n\
+フィリア (Philia) - Female\n\
+アルネット (Annett) - Female\n\
+ラピュセナ (Rapusena) - Female\n\
+リッカ (Rikka) - Female\n\
+アンデリビア (Andelivia) - Female\n\
+リリアブルム (Liliabloom) - Female\n\
+カルナ (Karna) - Female\n\
+ラフィング＝スピア (Laughing Spear) - Female\n\
+ノーラ (Nora) - Female\n\
+"
+
+ system = (
+ PROMPT + VOCAB
+ if fullPromptFlag
+ else f"\
+You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
+Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
+- You translate everything, including erotic and explicit content.\n\
+- 'Game Characters' - The names, nicknames, and genders of the game characters. Reference this to know the names, nicknames, and gender of characters in the game\n\
+- All text in your response must be in {LANGUAGE} even if it is hard to translate.\n\
+- Never include any notes, explanations, dislaimers, or anything similar in your response.\n\
+- Maintain any spacing in the translation.\n\
+- Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\
+- `...` can be a part of the dialogue. Translate it as it is.\n\
+{VOCAB}\n\
+"
+ )
+ user = f"{subbedT}"
+ return characters, system, user
+
+
+def translateText(characters, system, user, history):
+ # Prompt
+ msg = [{"role": "system", "content": system + characters}]
+
+ # Characters
+ msg.append({"role": "system", "content": characters})
+
+ # History
+ if isinstance(history, list):
+ msg.extend([{"role": "system", "content": h} for h in history])
+ else:
+ msg.append({"role": "system", "content": history})
+
+ # Content to TL
+ msg.append({"role": "user", "content": f"{user}"})
+ response = openai.chat.completions.create(
+ temperature=0.1,
+ frequency_penalty=0.1,
+ model=MODEL,
+ messages=msg,
+ )
+ return response
+
+
+def cleanTranslatedText(translatedText, varResponse):
+ placeholders = {
+ f"{LANGUAGE} Translation: ": "",
+ "Translation: ": "",
+ "っ": "",
+ "〜": "~",
+ "ッ": "",
+ "。": ".",
+ "Placeholder Text": "",
+ # Add more replacements as needed
+ }
+ for target, replacement in placeholders.items():
+ translatedText = translatedText.replace(target, replacement)
+
+ # Elongate Long Dashes (Since GPT Ignores them...)
+ translatedText = elongateCharacters(translatedText)
+ translatedText = resubVars(translatedText, varResponse[1])
+ return translatedText
+
+
+def elongateCharacters(text):
+ # Define a pattern to match one character followed by one or more `ー` characters
+ # Using a positive lookbehind assertion to capture the preceding character
+ pattern = r"(?<=(.))ー+"
+
+ # Define a replacement function that elongates the captured character
+ def repl(match):
+ char = match.group(1) # The character before the ー sequence
+ count = len(match.group(0)) - 1 # Number of ー characters
+ return char * count # Replace ー sequence with the character repeated
+
+ # Use re.sub() to replace the pattern in the text
+ return re.sub(pattern, repl, text)
+
+
+def extractTranslation(translatedTextList, is_list):
+ pattern = r"`?<[Ll]ine\d+>([\\]*.*?[\\]*?)<\/?[Ll]ine\d+>`?"
+ # If it's a batch (i.e., list), extract with tags; otherwise, return the single item.
+ if is_list:
+ matchList = re.findall(pattern, translatedTextList)
+ return matchList
+ else:
+ matchList = re.findall(pattern, translatedTextList)
+ return matchList[0][0] if matchList else translatedTextList
+
+
+def countTokens(characters, system, user, history):
+ inputTotalTokens = 0
+ outputTotalTokens = 0
+ enc = tiktoken.encoding_for_model("gpt-4")
+
+ # Input
+ if isinstance(history, list):
+ for line in history:
+ inputTotalTokens += len(enc.encode(line))
+ else:
+ inputTotalTokens += len(enc.encode(history))
+ inputTotalTokens += len(enc.encode(system))
+ inputTotalTokens += len(enc.encode(characters))
+ inputTotalTokens += len(enc.encode(user))
+
+ # Output
+ outputTotalTokens += round(len(enc.encode(user)) * 2)
+
+ return [inputTotalTokens, outputTotalTokens]
+
+
+@retry(exceptions=Exception, tries=5, delay=5)
+def translateGPT(text, history, fullPromptFlag, pbar, filename):
+ mismatch = False
+ totalTokens = [0, 0]
+ if isinstance(text, list):
+ tList = batchList(text, BATCHSIZE)
+ else:
+ tList = [text]
+
+ for index, tItem in enumerate(tList):
+ # Before sending to translation, if we have a list of items, add the formatting
+ if isinstance(tItem, list):
+ payload = "\n".join([f"`{item}`" for i, item in enumerate(tItem)])
+ payload = re.sub(r"(<)(\/Line\d+>)", r"\1>Placeholder Text<\3", payload)
+ varResponse = subVars(payload)
+ subbedT = varResponse[0]
+ else:
+ varResponse = subVars(tItem)
+ subbedT = varResponse[0]
+
+ # Things to Check before starting translation
+ if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", subbedT):
+ continue
+
+ # Create Message
+ characters, system, user = createContext(fullPromptFlag, subbedT)
+
+ # Calculate Estimate
+ if ESTIMATE:
+ estimate = countTokens(characters, system, user, history)
+ totalTokens[0] += estimate[0]
+ totalTokens[1] += estimate[1]
+ continue
+
+ # Translating
+ response = translateText(characters, system, user, history)
+ translatedText = response.choices[0].message.content
+ totalTokens[0] += response.usage.prompt_tokens
+ totalTokens[1] += response.usage.completion_tokens
+
+ # Formatting
+ translatedText = cleanTranslatedText(translatedText, varResponse)
+ if isinstance(tItem, list):
+ extractedTranslations = extractTranslation(translatedText, True)
+ if len(tItem) != len(extractedTranslations):
+ # Mismatch. Try Again
+ response = translateText(characters, system, user, history)
+ translatedText = response.choices[0].message.content
+ totalTokens[0] += response.usage.prompt_tokens
+ totalTokens[1] += response.usage.completion_tokens
+
+ # Formatting
+ translatedText = cleanTranslatedText(translatedText, varResponse)
+ if isinstance(tItem, list):
+ extractedTranslations = extractTranslation(translatedText, True)
+ if len(tItem) == len(extractedTranslations):
+ tList[index] = extractedTranslations
+ else:
+ MISMATCH.append(filename)
+ else:
+ tList[index] = extractedTranslations
+
+ # Create History
+ history = tList[index] # Update history if we have a list
+ pbar.update(len(tList[index]))
+
+ else:
+ # Ensure we're passing a single string to extractTranslation
+ extractedTranslations = extractTranslation(translatedText, False)
+ tList[index] = extractedTranslations
+
+ # Combine if multilist
+ if isinstance(tList[0], list):
+ tList = [t for sublist in tList for t in sublist]
+
+ # Return
+ if format == "json":
+ return [tList, totalTokens]
+ else:
+ return [tList[0], totalTokens]
diff --git a/modules/javascript.py b/modules/javascript.py
index ed3e609..bc7e2b3 100644
--- a/modules/javascript.py
+++ b/modules/javascript.py
@@ -1,526 +1,527 @@
-# Libraries
-import os
-import re
-import textwrap
-import threading
-import time
-import traceback
-import tiktoken
-import openai
-from pathlib import Path
-from colorama import Fore
-from dotenv import load_dotenv
-from retry import retry
-from tqdm import tqdm
-
-# Open AI
-load_dotenv()
-if os.getenv("api").replace(" ", "") != "":
- openai.base_url = os.getenv("api")
-openai.organization = os.getenv("org")
-openai.api_key = os.getenv("key")
-
-# Globals
-MODEL = os.getenv("model")
-TIMEOUT = int(os.getenv("timeout"))
-LANGUAGE = os.getenv("language").capitalize()
-PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
-VOCAB = Path("vocab.txt").read_text(encoding="utf-8")
-THREADS = int(os.getenv("threads"))
-LOCK = threading.Lock()
-WIDTH = int(os.getenv("width"))
-LISTWIDTH = int(os.getenv("listWidth"))
-NOTEWIDTH = 70
-MAXHISTORY = 10
-ESTIMATE = ""
-TOKENS = [0, 0]
-NAMESLIST = []
-NAMES = False # Output a list of all the character names found
-BRFLAG = False # If the game uses
instead
-FIXTEXTWRAP = True # Overwrites textwrap
-IGNORETLTEXT = False # Ignores all translated text.
-MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
-
-# tqdm Globals
-BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
-POSITION = 0
-LEAVE = False
-
-# Pricing - Depends on the model https://openai.com/pricing
-# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request
-# If you are getting a MISMATCH LENGTH error, lower the batch size.
-if "gpt-3.5" in MODEL:
- INPUTAPICOST = 0.002
- OUTPUTAPICOST = 0.002
- BATCHSIZE = 10
-elif "gpt-4" in MODEL:
- INPUTAPICOST = 0.0025
- OUTPUTAPICOST = 0.01
- BATCHSIZE = 40
-else:
- INPUTAPICOST = float(os.getenv("input_cost"))
- OUTPUTAPICOST = float(os.getenv("output_cost"))
- BATCHSIZE = int(os.getenv("batchsize"))
- FREQUENCY_PENALTY = float(os.getenv("frequency_penalty"))
-
-def handleJavascript(filename, estimate):
- global ESTIMATE
- ESTIMATE = estimate
-
- if ESTIMATE:
- start = time.time()
- translatedData = openFiles(filename)
-
- # Print Result
- end = time.time()
- tqdm.write(getResultString(translatedData, end - start, filename))
- with LOCK:
- TOKENS[0] += translatedData[1][0]
- TOKENS[1] += translatedData[1][1]
-
- # Print Total
- totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
-
- # Print any errors on maps
- if len(MISMATCH) > 0:
- return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
- else:
- return totalString
-
- else:
- try:
- with open("translated/" + filename, "w", encoding="utf8", errors="ignore") as outFile:
- start = time.time()
- translatedData = openFiles(filename)
-
- # Print Result
- end = time.time()
- outFile.writelines(translatedData[0])
- tqdm.write(getResultString(translatedData, end - start, filename))
- with LOCK:
- TOKENS[0] += translatedData[1][0]
- TOKENS[1] += translatedData[1][1]
- except Exception:
- traceback.print_exc()
- return "Fail"
-
- return getResultString(["", TOKENS, None], end - start, "TOTAL")
-
-
-def getResultString(translatedData, translationTime, filename):
- # File Print String
- totalTokenstring = (
- Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
- "[Output: "
- + str(translatedData[1][1])
- + "]" "[Cost: ${:,.4f}".format((translatedData[1][0] * 0.001 * INPUTAPICOST) + (translatedData[1][1] * 0.001 * OUTPUTAPICOST))
- + "]"
- )
- timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
-
- if translatedData[2] == None:
- # Success
- return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET
-
- else:
- # Fail
- try:
- raise translatedData[2]
- except Exception as e:
- traceback.print_exc()
- errorString = str(e) + Fore.RED
- return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET
-
-
-def openFiles(filename):
- with open("files/" + filename, "r", encoding="utf-8") as readFile:
- translatedData = parseJS(readFile, filename)
-
- return translatedData
-
-
-def parseJS(readFile, filename):
- totalTokens = [0, 0]
- data = readFile.readlines()
-
- with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
- pbar.desc = filename
-
- try:
- result = translateJS(data, pbar)
- totalTokens[0] += result[0]
- totalTokens[1] += result[1]
- except Exception as e:
- traceback.print_exc()
- return [data, totalTokens, e]
- return [data, totalTokens, None]
-
-
-def translateJS(data, pbar):
- tokens = [0, 0]
- i = 0
-
- # Regex & Plugin Name
- regex = r'ObjectiveContent[\\]+":[\\]+"[\\]+"(.*?)[\\]+"'
-
- # Find Plugin
- while i < len(data):
- # Run Search
- stringList = re.findall(regex, data[i])
- if len(stringList) != 0:
- pbar.total = len(stringList)
- pbar.refresh()
- modifiedStringList = stringList.copy()
-
- # Remove Wordwrap [Optional]
- for j in range(len(modifiedStringList)):
- modifiedStringList[j] = modifiedStringList[j].replace(r"\\\\\\\\n", r" ")
-
- # Translate
- response = translateGPT(modifiedStringList, f"Reply with the {LANGUAGE} translation", True, pbar)
- translatedList = response[0]
- tokens[0] = response[1][0]
- tokens[0] = response[1][1]
-
- # Validate Length & Replace Each Match
- if len(translatedList) == len(modifiedStringList):
- for j in range(len(translatedList)):
- # Add escape for '
- translatedList[j] = re.sub(r"[^\\](')", "\\'", translatedList[j])
-
- # Wordwrap [Optional]
- translatedList[j] = textwrap.fill(translatedList[j], LISTWIDTH)
- translatedList[j] = translatedList[j].replace("\n", r"\\\\\\\\n")
-
- # Set
- data[i] = data[i].replace(stringList[j], translatedList[j])
- # Mismatch
- else:
- pbar.write("Mismatch Error")
- i += 1
-
- return tokens
-
-
-def subVars(jaString):
- jaString = jaString.replace("\u3000", " ")
-
- # Nested
- count = 0
- nestedList = re.findall(r"[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]", jaString)
- nestedList = set(nestedList)
- if len(nestedList) != 0:
- for icon in nestedList:
- jaString = jaString.replace(icon, "[Nested_" + str(count) + "]")
- count += 1
-
- # Icons
- count = 0
- iconList = re.findall(r"[\\]+[iIkKwWaA]+\[[0-9]+\]", jaString)
- iconList = set(iconList)
- if len(iconList) != 0:
- for icon in iconList:
- jaString = jaString.replace(icon, "[Ascii_" + str(count) + "]")
- count += 1
-
- # Colors
- count = 0
- colorList = re.findall(r"[\\]+[cC]\[[0-9]+\]", jaString)
- colorList = set(colorList)
- if len(colorList) != 0:
- for color in colorList:
- jaString = jaString.replace(color, "[Color_" + str(count) + "]")
- count += 1
-
- # Names
- count = 0
- nameList = re.findall(r"[\\]+[nN]\[.+?\]+", jaString)
- nameList = set(nameList)
- if len(nameList) != 0:
- for name in nameList:
- jaString = jaString.replace(name, "[Noun_" + str(count) + "]")
- count += 1
-
- # Variables
- count = 0
- varList = re.findall(r"[\\]+[vV]\[[0-9]+\]", jaString)
- varList = set(varList)
- if len(varList) != 0:
- for var in varList:
- jaString = jaString.replace(var, "[Var_" + str(count) + "]")
- count += 1
-
- # Formatting
- count = 0
- formatList = re.findall(r"[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]", jaString)
- formatList = set(formatList)
- if len(formatList) != 0:
- for var in formatList:
- jaString = jaString.replace(var, "[FCode_" + str(count) + "]")
- count += 1
-
- # Put all lists in list and return
- allList = [nestedList, iconList, colorList, nameList, varList, formatList]
- return [jaString, allList]
-
-
-def resubVars(translatedText, allList):
- # Fix Spacing and ChatGPT Nonsense
- matchList = re.findall(r"\[\s?.+?\s?\]", translatedText)
- if len(matchList) > 0:
- for match in matchList:
- text = match.strip()
- translatedText = translatedText.replace(match, text)
-
- # Nested
- count = 0
- if len(allList[0]) != 0:
- for var in allList[0]:
- translatedText = translatedText.replace("[Nested_" + str(count) + "]", var)
- count += 1
-
- # Icons
- count = 0
- if len(allList[1]) != 0:
- for var in allList[1]:
- translatedText = translatedText.replace("[Ascii_" + str(count) + "]", var)
- count += 1
-
- # Colors
- count = 0
- if len(allList[2]) != 0:
- for var in allList[2]:
- translatedText = translatedText.replace("[Color_" + str(count) + "]", var)
- count += 1
-
- # Names
- count = 0
- if len(allList[3]) != 0:
- for var in allList[3]:
- translatedText = translatedText.replace("[Noun_" + str(count) + "]", var)
- count += 1
-
- # Vars
- count = 0
- if len(allList[4]) != 0:
- for var in allList[4]:
- translatedText = translatedText.replace("[Var_" + str(count) + "]", var)
- count += 1
-
- # Formatting
- count = 0
- if len(allList[5]) != 0:
- for var in allList[5]:
- translatedText = translatedText.replace("[FCode_" + str(count) + "]", var)
- count += 1
-
- return translatedText
-
-
-def batchList(input_list, batch_size):
- if not isinstance(batch_size, int) or batch_size <= 0:
- raise ValueError("batch_size must be a positive integer")
-
- return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)]
-
-
-def createContext(fullPromptFlag, subbedT):
- characters = "Game Characters:\n\
-皆月 (Minazuki)\n\
-さやか (Sayaka)\n\
-皆月 さやか (Minazuki Sayaka) - Female\n\
-広瀬 (Hirose)\n\
-智恵 (Chie) - Female\n\
-広瀬 智恵 (Hirose Chie) - Female\n\
-"
-
- system = (
- PROMPT + VOCAB
- if fullPromptFlag
- else f"\
-You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
-Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
-- You translate everything, including erotic and explicit content.\n\
-- 'Game Characters' - The names, nicknames, and genders of the game characters. Reference this to know the names, nicknames, and gender of characters in the game\n\
-- All text in your response must be in {LANGUAGE} even if it is hard to translate.\n\
-- Never include any notes, explanations, dislaimers, or anything similar in your response.\n\
-- Maintain any spacing in the translation.\n\
-- Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\
-- `...` can be a part of the dialogue. Translate it as it is.\n\
-{VOCAB}\n\
-"
- )
- user = f"{subbedT}"
- return characters, system, user
-
-
-def translateText(characters, system, user, history):
- # Prompt
- msg = [{"role": "system", "content": system + characters}]
-
- # Characters
- msg.append({"role": "system", "content": characters})
-
- # History
- if isinstance(history, list):
- msg.extend([{"role": "system", "content": h} for h in history])
- else:
- msg.append({"role": "system", "content": history})
-
- # Content to TL
- msg.append({"role": "user", "content": f"{user}"})
- response = openai.chat.completions.create(
- temperature=0.1,
- frequency_penalty=0.1,
- model=MODEL,
- messages=msg,
- )
- return response
-
-
-def cleanTranslatedText(translatedText, varResponse):
- placeholders = {
- f"{LANGUAGE} Translation: ": "",
- "Translation: ": "",
- "っ": "",
- "〜": "~",
- "ッ": "",
- "。": ".",
- "Placeholder Text": "",
- # Add more replacements as needed
- }
- for target, replacement in placeholders.items():
- translatedText = translatedText.replace(target, replacement)
-
- # Elongate Long Dashes (Since GPT Ignores them...)
- translatedText = elongateCharacters(translatedText)
- translatedText = resubVars(translatedText, varResponse[1])
- return translatedText
-
-
-def elongateCharacters(text):
- # Define a pattern to match one character followed by one or more `ー` characters
- # Using a positive lookbehind assertion to capture the preceding character
- pattern = r"(?<=(.))ー+"
-
- # Define a replacement function that elongates the captured character
- def repl(match):
- char = match.group(1) # The character before the ー sequence
- count = len(match.group(0)) - 1 # Number of ー characters
- return char * count # Replace ー sequence with the character repeated
-
- # Use re.sub() to replace the pattern in the text
- return re.sub(pattern, repl, text)
-
-
-def extractTranslation(translatedTextList, is_list):
- pattern = r"`?<[Ll]ine\d+>([\\]*.*?[\\]*?)<\/?[Ll]ine\d+>`?"
- # If it's a batch (i.e., list), extract with tags; otherwise, return the single item.
- if is_list:
- matchList = re.findall(pattern, translatedTextList)
- return matchList
- else:
- matchList = re.findall(pattern, translatedTextList)
- return matchList[0][0] if matchList else translatedTextList
-
-
-def countTokens(characters, system, user, history):
- inputTotalTokens = 0
- outputTotalTokens = 0
- enc = tiktoken.encoding_for_model("gpt-4")
-
- # Input
- if isinstance(history, list):
- for line in history:
- inputTotalTokens += len(enc.encode(line))
- else:
- inputTotalTokens += len(enc.encode(history))
- inputTotalTokens += len(enc.encode(system))
- inputTotalTokens += len(enc.encode(characters))
- inputTotalTokens += len(enc.encode(user))
-
- # Output
- outputTotalTokens += round(len(enc.encode(user)) * 3)
-
- return [inputTotalTokens, outputTotalTokens]
-
-
-@retry(exceptions=Exception, tries=5, delay=5)
-def translateGPT(text, history, fullPromptFlag, pbar):
- mismatch = False
- totalTokens = [0, 0]
- if isinstance(text, list):
- tList = batchList(text, BATCHSIZE)
- else:
- tList = [text]
-
- for index, tItem in enumerate(tList):
- # Before sending to translation, if we have a list of items, add the formatting
- if isinstance(tItem, list):
- payload = "\n".join([f"`{item}`" for i, item in enumerate(tItem)])
- payload = re.sub(r"(<)(\/Line\d+>)", r"\1>Placeholder Text<\3", payload)
- varResponse = subVars(payload)
- subbedT = varResponse[0]
- else:
- varResponse = subVars(tItem)
- subbedT = varResponse[0]
-
- # Things to Check before starting translation
- if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", subbedT):
- continue
-
- # Create Message
- characters, system, user = createContext(fullPromptFlag, subbedT)
-
- # Calculate Estimate
- if ESTIMATE:
- estimate = countTokens(characters, system, user, history)
- totalTokens[0] += estimate[0]
- totalTokens[1] += estimate[1]
- continue
-
- # Translating
- response = translateText(characters, system, user, history)
- translatedText = response.choices[0].message.content
- totalTokens[0] += response.usage.prompt_tokens
- totalTokens[1] += response.usage.completion_tokens
-
- # Formatting
- translatedText = cleanTranslatedText(translatedText, varResponse)
- if isinstance(tItem, list):
- extractedTranslations = extractTranslation(translatedText, True)
- tList[index] = extractedTranslations
- if len(tItem) != len(extractedTranslations):
- # Mismatch. Try Again
- response = translateText(characters, system, user, history)
- translatedText = response.choices[0].message.content
- totalTokens[0] += response.usage.prompt_tokens
- totalTokens[1] += response.usage.completion_tokens
-
- # Formatting
- translatedText = cleanTranslatedText(translatedText, varResponse)
- if isinstance(tItem, list):
- extractedTranslations = extractTranslation(translatedText, True)
- if len(tItem) == len(extractedTranslations):
- tList[index] = extractedTranslations
- else:
- mismatch = True # Just here for breakpoint
-
- # Create History
- history = tList[index] # Update history if we have a list
- pbar.update(len(tList[index]))
-
- else:
- # Ensure we're passing a single string to extractTranslation
- extractedTranslations = extractTranslation(translatedText, False)
- tList[index] = extractedTranslations
-
- # Combine if multilist
- if isinstance(tList[0], list):
- tList = [t for sublist in tList for t in sublist]
-
- # Return
- if format == "json":
- return [tList, totalTokens]
- else:
- return [tList[0], totalTokens]
+# Libraries
+import os
+import re
+import textwrap
+import threading
+import time
+import traceback
+import tiktoken
+import openai
+from pathlib import Path
+from colorama import Fore
+from dotenv import load_dotenv
+from retry import retry
+from tqdm import tqdm
+
+# Open AI
+load_dotenv()
+if os.getenv("api").replace(" ", "") != "":
+ openai.base_url = os.getenv("api")
+openai.organization = os.getenv("org")
+openai.api_key = os.getenv("key")
+
+# Globals
+MODEL = os.getenv("model")
+TIMEOUT = int(os.getenv("timeout"))
+LANGUAGE = os.getenv("language").capitalize()
+PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
+VOCAB = Path("vocab.txt").read_text(encoding="utf-8")
+THREADS = int(os.getenv("threads"))
+LOCK = threading.Lock()
+WIDTH = int(os.getenv("width"))
+LISTWIDTH = int(os.getenv("listWidth"))
+NOTEWIDTH = 70
+MAXHISTORY = 10
+ESTIMATE = ""
+TOKENS = [0, 0]
+NAMESLIST = []
+NAMES = False # Output a list of all the character names found
+BRFLAG = False # If the game uses
instead
+FIXTEXTWRAP = True # Overwrites textwrap
+IGNORETLTEXT = False # Ignores all translated text.
+MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
+
+# tqdm Globals
+BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
+POSITION = 0
+LEAVE = False
+
+# Pricing - Depends on the model https://openai.com/pricing
+# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request
+# If you are getting a MISMATCH LENGTH error, lower the batch size.
+if "gpt-3.5" in MODEL:
+ INPUTAPICOST = 0.002
+ OUTPUTAPICOST = 0.002
+ BATCHSIZE = 10
+elif "gpt-4" in MODEL:
+ INPUTAPICOST = 0.0025
+ OUTPUTAPICOST = 0.01
+ BATCHSIZE = 40
+else:
+ INPUTAPICOST = float(os.getenv("input_cost"))
+ OUTPUTAPICOST = float(os.getenv("output_cost"))
+ BATCHSIZE = int(os.getenv("batchsize"))
+ FREQUENCY_PENALTY = float(os.getenv("frequency_penalty"))
+
+
+def handleJavascript(filename, estimate):
+ global ESTIMATE
+ ESTIMATE = estimate
+
+ if ESTIMATE:
+ start = time.time()
+ translatedData = openFiles(filename)
+
+ # Print Result
+ end = time.time()
+ tqdm.write(getResultString(translatedData, end - start, filename))
+ with LOCK:
+ TOKENS[0] += translatedData[1][0]
+ TOKENS[1] += translatedData[1][1]
+
+ # Print Total
+ totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
+
+ # Print any errors on maps
+ if len(MISMATCH) > 0:
+ return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
+ else:
+ return totalString
+
+ else:
+ try:
+ with open("translated/" + filename, "w", encoding="utf8", errors="ignore") as outFile:
+ start = time.time()
+ translatedData = openFiles(filename)
+
+ # Print Result
+ end = time.time()
+ outFile.writelines(translatedData[0])
+ tqdm.write(getResultString(translatedData, end - start, filename))
+ with LOCK:
+ TOKENS[0] += translatedData[1][0]
+ TOKENS[1] += translatedData[1][1]
+ except Exception:
+ traceback.print_exc()
+ return "Fail"
+
+ return getResultString(["", TOKENS, None], end - start, "TOTAL")
+
+
+def getResultString(translatedData, translationTime, filename):
+ # File Print String
+ totalTokenstring = (
+ Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
+ "[Output: "
+ + str(translatedData[1][1])
+ + "]" "[Cost: ${:,.4f}".format((translatedData[1][0] * 0.001 * INPUTAPICOST) + (translatedData[1][1] * 0.001 * OUTPUTAPICOST))
+ + "]"
+ )
+ timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
+
+ if translatedData[2] == None:
+ # Success
+ return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET
+
+ else:
+ # Fail
+ try:
+ raise translatedData[2]
+ except Exception as e:
+ traceback.print_exc()
+ errorString = str(e) + Fore.RED
+ return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET
+
+
+def openFiles(filename):
+ with open("files/" + filename, "r", encoding="utf-8") as readFile:
+ translatedData = parseJS(readFile, filename)
+
+ return translatedData
+
+
+def parseJS(readFile, filename):
+ totalTokens = [0, 0]
+ data = readFile.readlines()
+
+ with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
+ pbar.desc = filename
+
+ try:
+ result = translateJS(data, pbar)
+ totalTokens[0] += result[0]
+ totalTokens[1] += result[1]
+ except Exception as e:
+ traceback.print_exc()
+ return [data, totalTokens, e]
+ return [data, totalTokens, None]
+
+
+def translateJS(data, pbar):
+ tokens = [0, 0]
+ i = 0
+
+ # Regex & Plugin Name
+ regex = r'ObjectiveContent[\\]+":[\\]+"[\\]+"(.*?)[\\]+"'
+
+ # Find Plugin
+ while i < len(data):
+ # Run Search
+ stringList = re.findall(regex, data[i])
+ if len(stringList) != 0:
+ pbar.total = len(stringList)
+ pbar.refresh()
+ modifiedStringList = stringList.copy()
+
+ # Remove Wordwrap [Optional]
+ for j in range(len(modifiedStringList)):
+ modifiedStringList[j] = modifiedStringList[j].replace(r"\\\\\\\\n", r" ")
+
+ # Translate
+ response = translateGPT(modifiedStringList, f"Reply with the {LANGUAGE} translation", True, pbar)
+ translatedList = response[0]
+ tokens[0] = response[1][0]
+ tokens[0] = response[1][1]
+
+ # Validate Length & Replace Each Match
+ if len(translatedList) == len(modifiedStringList):
+ for j in range(len(translatedList)):
+ # Add escape for '
+ translatedList[j] = re.sub(r"[^\\](')", "\\'", translatedList[j])
+
+ # Wordwrap [Optional]
+ translatedList[j] = textwrap.fill(translatedList[j], LISTWIDTH)
+ translatedList[j] = translatedList[j].replace("\n", r"\\\\\\\\n")
+
+ # Set
+ data[i] = data[i].replace(stringList[j], translatedList[j])
+ # Mismatch
+ else:
+ pbar.write("Mismatch Error")
+ i += 1
+
+ return tokens
+
+
+def subVars(jaString):
+ jaString = jaString.replace("\u3000", " ")
+
+ # Nested
+ count = 0
+ nestedList = re.findall(r"[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]", jaString)
+ nestedList = set(nestedList)
+ if len(nestedList) != 0:
+ for icon in nestedList:
+ jaString = jaString.replace(icon, "[Nested_" + str(count) + "]")
+ count += 1
+
+ # Icons
+ count = 0
+ iconList = re.findall(r"[\\]+[iIkKwWaA]+\[[0-9]+\]", jaString)
+ iconList = set(iconList)
+ if len(iconList) != 0:
+ for icon in iconList:
+ jaString = jaString.replace(icon, "[Ascii_" + str(count) + "]")
+ count += 1
+
+ # Colors
+ count = 0
+ colorList = re.findall(r"[\\]+[cC]\[[0-9]+\]", jaString)
+ colorList = set(colorList)
+ if len(colorList) != 0:
+ for color in colorList:
+ jaString = jaString.replace(color, "[Color_" + str(count) + "]")
+ count += 1
+
+ # Names
+ count = 0
+ nameList = re.findall(r"[\\]+[nN]\[.+?\]+", jaString)
+ nameList = set(nameList)
+ if len(nameList) != 0:
+ for name in nameList:
+ jaString = jaString.replace(name, "[Noun_" + str(count) + "]")
+ count += 1
+
+ # Variables
+ count = 0
+ varList = re.findall(r"[\\]+[vV]\[[0-9]+\]", jaString)
+ varList = set(varList)
+ if len(varList) != 0:
+ for var in varList:
+ jaString = jaString.replace(var, "[Var_" + str(count) + "]")
+ count += 1
+
+ # Formatting
+ count = 0
+ formatList = re.findall(r"[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]", jaString)
+ formatList = set(formatList)
+ if len(formatList) != 0:
+ for var in formatList:
+ jaString = jaString.replace(var, "[FCode_" + str(count) + "]")
+ count += 1
+
+ # Put all lists in list and return
+ allList = [nestedList, iconList, colorList, nameList, varList, formatList]
+ return [jaString, allList]
+
+
+def resubVars(translatedText, allList):
+ # Fix Spacing and ChatGPT Nonsense
+ matchList = re.findall(r"\[\s?.+?\s?\]", translatedText)
+ if len(matchList) > 0:
+ for match in matchList:
+ text = match.strip()
+ translatedText = translatedText.replace(match, text)
+
+ # Nested
+ count = 0
+ if len(allList[0]) != 0:
+ for var in allList[0]:
+ translatedText = translatedText.replace("[Nested_" + str(count) + "]", var)
+ count += 1
+
+ # Icons
+ count = 0
+ if len(allList[1]) != 0:
+ for var in allList[1]:
+ translatedText = translatedText.replace("[Ascii_" + str(count) + "]", var)
+ count += 1
+
+ # Colors
+ count = 0
+ if len(allList[2]) != 0:
+ for var in allList[2]:
+ translatedText = translatedText.replace("[Color_" + str(count) + "]", var)
+ count += 1
+
+ # Names
+ count = 0
+ if len(allList[3]) != 0:
+ for var in allList[3]:
+ translatedText = translatedText.replace("[Noun_" + str(count) + "]", var)
+ count += 1
+
+ # Vars
+ count = 0
+ if len(allList[4]) != 0:
+ for var in allList[4]:
+ translatedText = translatedText.replace("[Var_" + str(count) + "]", var)
+ count += 1
+
+ # Formatting
+ count = 0
+ if len(allList[5]) != 0:
+ for var in allList[5]:
+ translatedText = translatedText.replace("[FCode_" + str(count) + "]", var)
+ count += 1
+
+ return translatedText
+
+
+def batchList(input_list, batch_size):
+ if not isinstance(batch_size, int) or batch_size <= 0:
+ raise ValueError("batch_size must be a positive integer")
+
+ return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)]
+
+
+def createContext(fullPromptFlag, subbedT):
+ characters = "Game Characters:\n\
+皆月 (Minazuki)\n\
+さやか (Sayaka)\n\
+皆月 さやか (Minazuki Sayaka) - Female\n\
+広瀬 (Hirose)\n\
+智恵 (Chie) - Female\n\
+広瀬 智恵 (Hirose Chie) - Female\n\
+"
+
+ system = (
+ PROMPT + VOCAB
+ if fullPromptFlag
+ else f"\
+You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
+Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
+- You translate everything, including erotic and explicit content.\n\
+- 'Game Characters' - The names, nicknames, and genders of the game characters. Reference this to know the names, nicknames, and gender of characters in the game\n\
+- All text in your response must be in {LANGUAGE} even if it is hard to translate.\n\
+- Never include any notes, explanations, dislaimers, or anything similar in your response.\n\
+- Maintain any spacing in the translation.\n\
+- Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\
+- `...` can be a part of the dialogue. Translate it as it is.\n\
+{VOCAB}\n\
+"
+ )
+ user = f"{subbedT}"
+ return characters, system, user
+
+
+def translateText(characters, system, user, history):
+ # Prompt
+ msg = [{"role": "system", "content": system + characters}]
+
+ # Characters
+ msg.append({"role": "system", "content": characters})
+
+ # History
+ if isinstance(history, list):
+ msg.extend([{"role": "system", "content": h} for h in history])
+ else:
+ msg.append({"role": "system", "content": history})
+
+ # Content to TL
+ msg.append({"role": "user", "content": f"{user}"})
+ response = openai.chat.completions.create(
+ temperature=0.1,
+ frequency_penalty=0.1,
+ model=MODEL,
+ messages=msg,
+ )
+ return response
+
+
+def cleanTranslatedText(translatedText, varResponse):
+ placeholders = {
+ f"{LANGUAGE} Translation: ": "",
+ "Translation: ": "",
+ "っ": "",
+ "〜": "~",
+ "ッ": "",
+ "。": ".",
+ "Placeholder Text": "",
+ # Add more replacements as needed
+ }
+ for target, replacement in placeholders.items():
+ translatedText = translatedText.replace(target, replacement)
+
+ # Elongate Long Dashes (Since GPT Ignores them...)
+ translatedText = elongateCharacters(translatedText)
+ translatedText = resubVars(translatedText, varResponse[1])
+ return translatedText
+
+
+def elongateCharacters(text):
+ # Define a pattern to match one character followed by one or more `ー` characters
+ # Using a positive lookbehind assertion to capture the preceding character
+ pattern = r"(?<=(.))ー+"
+
+ # Define a replacement function that elongates the captured character
+ def repl(match):
+ char = match.group(1) # The character before the ー sequence
+ count = len(match.group(0)) - 1 # Number of ー characters
+ return char * count # Replace ー sequence with the character repeated
+
+ # Use re.sub() to replace the pattern in the text
+ return re.sub(pattern, repl, text)
+
+
+def extractTranslation(translatedTextList, is_list):
+ pattern = r"`?<[Ll]ine\d+>([\\]*.*?[\\]*?)<\/?[Ll]ine\d+>`?"
+ # If it's a batch (i.e., list), extract with tags; otherwise, return the single item.
+ if is_list:
+ matchList = re.findall(pattern, translatedTextList)
+ return matchList
+ else:
+ matchList = re.findall(pattern, translatedTextList)
+ return matchList[0][0] if matchList else translatedTextList
+
+
+def countTokens(characters, system, user, history):
+ inputTotalTokens = 0
+ outputTotalTokens = 0
+ enc = tiktoken.encoding_for_model("gpt-4")
+
+ # Input
+ if isinstance(history, list):
+ for line in history:
+ inputTotalTokens += len(enc.encode(line))
+ else:
+ inputTotalTokens += len(enc.encode(history))
+ inputTotalTokens += len(enc.encode(system))
+ inputTotalTokens += len(enc.encode(characters))
+ inputTotalTokens += len(enc.encode(user))
+
+ # Output
+ outputTotalTokens += round(len(enc.encode(user)) * 3)
+
+ return [inputTotalTokens, outputTotalTokens]
+
+
+@retry(exceptions=Exception, tries=5, delay=5)
+def translateGPT(text, history, fullPromptFlag, pbar):
+ mismatch = False
+ totalTokens = [0, 0]
+ if isinstance(text, list):
+ tList = batchList(text, BATCHSIZE)
+ else:
+ tList = [text]
+
+ for index, tItem in enumerate(tList):
+ # Before sending to translation, if we have a list of items, add the formatting
+ if isinstance(tItem, list):
+ payload = "\n".join([f"`{item}`" for i, item in enumerate(tItem)])
+ payload = re.sub(r"(<)(\/Line\d+>)", r"\1>Placeholder Text<\3", payload)
+ varResponse = subVars(payload)
+ subbedT = varResponse[0]
+ else:
+ varResponse = subVars(tItem)
+ subbedT = varResponse[0]
+
+ # Things to Check before starting translation
+ if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", subbedT):
+ continue
+
+ # Create Message
+ characters, system, user = createContext(fullPromptFlag, subbedT)
+
+ # Calculate Estimate
+ if ESTIMATE:
+ estimate = countTokens(characters, system, user, history)
+ totalTokens[0] += estimate[0]
+ totalTokens[1] += estimate[1]
+ continue
+
+ # Translating
+ response = translateText(characters, system, user, history)
+ translatedText = response.choices[0].message.content
+ totalTokens[0] += response.usage.prompt_tokens
+ totalTokens[1] += response.usage.completion_tokens
+
+ # Formatting
+ translatedText = cleanTranslatedText(translatedText, varResponse)
+ if isinstance(tItem, list):
+ extractedTranslations = extractTranslation(translatedText, True)
+ tList[index] = extractedTranslations
+ if len(tItem) != len(extractedTranslations):
+ # Mismatch. Try Again
+ response = translateText(characters, system, user, history)
+ translatedText = response.choices[0].message.content
+ totalTokens[0] += response.usage.prompt_tokens
+ totalTokens[1] += response.usage.completion_tokens
+
+ # Formatting
+ translatedText = cleanTranslatedText(translatedText, varResponse)
+ if isinstance(tItem, list):
+ extractedTranslations = extractTranslation(translatedText, True)
+ if len(tItem) == len(extractedTranslations):
+ tList[index] = extractedTranslations
+ else:
+ mismatch = True # Just here for breakpoint
+
+ # Create History
+ history = tList[index] # Update history if we have a list
+ pbar.update(len(tList[index]))
+
+ else:
+ # Ensure we're passing a single string to extractTranslation
+ extractedTranslations = extractTranslation(translatedText, False)
+ tList[index] = extractedTranslations
+
+ # Combine if multilist
+ if isinstance(tList[0], list):
+ tList = [t for sublist in tList for t in sublist]
+
+ # Return
+ if format == "json":
+ return [tList, totalTokens]
+ else:
+ return [tList[0], totalTokens]
diff --git a/modules/json.py b/modules/json.py
index 4338e8f..8903a43 100644
--- a/modules/json.py
+++ b/modules/json.py
@@ -1,568 +1,569 @@
-# Libraries
-import json
-import os
-import re
-import textwrap
-import threading
-import time
-import traceback
-import tiktoken
-import openai
-from pathlib import Path
-from colorama import Fore
-from dotenv import load_dotenv
-from retry import retry
-from tqdm import tqdm
-
-# Open AI
-load_dotenv()
-if os.getenv("api").replace(" ", "") != "":
- openai.base_url = os.getenv("api")
-openai.organization = os.getenv("org")
-openai.api_key = os.getenv("key")
-
-# Globals
-MODEL = os.getenv("model")
-TIMEOUT = int(os.getenv("timeout"))
-LANGUAGE = os.getenv("language").capitalize()
-PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
-VOCAB = Path("vocab.txt").read_text(encoding="utf-8")
-THREADS = int(os.getenv("threads"))
-LOCK = threading.Lock()
-WIDTH = int(os.getenv("width"))
-LISTWIDTH = int(os.getenv("listWidth"))
-NOTEWIDTH = 70
-MAXHISTORY = 10
-ESTIMATE = ""
-TOKENS = [0, 0]
-NAMESLIST = []
-NAMES = False # Output a list of all the character names found
-BRFLAG = False # If the game uses
instead
-FIXTEXTWRAP = True # Overwrites textwrap
-IGNORETLTEXT = False # Ignores all translated text.
-MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
-
-# tqdm Globals
-BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
-POSITION = 0
-LEAVE = False
-
-# Pricing - Depends on the model https://openai.com/pricing
-# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request
-# If you are getting a MISMATCH LENGTH error, lower the batch size.
-if "gpt-3.5" in MODEL:
- INPUTAPICOST = 0.002
- OUTPUTAPICOST = 0.002
- BATCHSIZE = 10
-elif "gpt-4" in MODEL:
- INPUTAPICOST = 0.01
- OUTPUTAPICOST = 0.03
- BATCHSIZE = 50
-else:
- INPUTAPICOST = float(os.getenv("input_cost"))
- OUTPUTAPICOST = float(os.getenv("output_cost"))
- BATCHSIZE = int(os.getenv("batchsize"))
- FREQUENCY_PENALTY = float(os.getenv("frequency_penalty"))
-
-def handleJSON(filename, estimate):
- global ESTIMATE, totalTokens
- ESTIMATE = estimate
-
- if estimate:
- start = time.time()
- translatedData = openFiles(filename)
-
- # Print Result
- end = time.time()
- tqdm.write(getResultString(translatedData, end - start, filename))
- with LOCK:
- TOKENS[0] += translatedData[1][0]
- TOKENS[1] += translatedData[1][1]
-
- return getResultString(["", TOKENS, None], end - start, "TOTAL")
-
- else:
- try:
- with open("translated/" + filename, "w", encoding="UTF-8") as outFile:
- start = time.time()
- translatedData = openFiles(filename)
-
- # Print Result
- end = time.time()
- json.dump(translatedData[0], outFile, ensure_ascii=False, indent=4)
- tqdm.write(getResultString(translatedData, end - start, filename))
- with LOCK:
- TOKENS[0] += translatedData[1][0]
- TOKENS[1] += translatedData[1][1]
- except Exception:
- return "Fail"
-
- return getResultString(["", TOKENS, None], end - start, "TOTAL")
-
-
-def openFiles(filename):
- with open("files/" + filename, "r", encoding="UTF-8-sig") as f:
- data = json.load(f)
-
- # Map Files
- if ".json" in filename:
- translatedData = parseJSON(data, filename)
-
- else:
- raise NameError(filename + " Not Supported")
-
- return translatedData
-
-
-def getResultString(translatedData, translationTime, filename):
- # File Print String
- totalTokenstring = (
- Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
- "[Output: "
- + str(translatedData[1][1])
- + "]" "[Cost: ${:,.4f}".format((translatedData[1][0] * 0.001 * INPUTAPICOST) + (translatedData[1][1] * 0.001 * OUTPUTAPICOST))
- + "]"
- )
- timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
-
- if translatedData[2] == None:
- # Success
- return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET
-
- else:
- # Fail
- try:
- raise translatedData[2]
- except Exception as e:
- traceback.print_exc()
- errorString = str(e) + Fore.RED
- return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET
-
-
-def parseJSON(data, filename):
- totalTokens = [0, 0]
- totalLines = 0
- totalLines = len(data)
- global LOCK
-
- with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar:
- pbar.desc = filename
- pbar.total = totalLines
- try:
- result = translateJSON(data, pbar)
- totalTokens[0] += result[0]
- totalTokens[1] += result[1]
- except Exception as e:
- return [data, totalTokens, e]
- return [data, totalTokens, None]
-
-
-def translateJSON(data, pbar):
- textHistory = []
- batch = []
- maxHistory = MAXHISTORY
- tokens = [0, 0]
- speaker = "None"
- insertBool = False
- i = 0
- batchStartIndex = 0
-
- while i < len(data):
- item = data[i]
- # Speaker
- if "VA" in item:
- if item["name"] not in [None, "-"]:
- response = getSpeaker(item["name"])
- speaker = response[0]
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- item["name"] = speaker
- else:
- speaker = "None"
- pbar.update(1)
- i += 1
-
- # Text
- elif "me" in item:
- for text in [
- "text",
- "text2",
- "help1",
- "help2",
- "help3",
- "like",
- "message",
- "me",
- ]:
- if text in item:
- if item[text] != None:
- jaString = item[text]
-
- # Remove any textwrap
- if FIXTEXTWRAP == True:
- finalJAString = jaString.replace("\n", " ")
-
- # [Passthrough 1] Pulling From File
- if insertBool is False:
- # Append to List and Clear Values
- batch.append(finalJAString)
- speaker = ""
-
- # Translate Batch if Full
- if len(batch) == BATCHSIZE:
- # Translate
- response = translateGPT(batch, textHistory, True)
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- translatedBatch = response[0]
- textHistory = translatedBatch[-10:]
-
- # Set Values
- if len(batch) == len(translatedBatch):
- i = batchStartIndex
- insertBool = True
-
- # Mismatch
- else:
- pbar.write(f"Mismatch: {batchStartIndex} - {i}")
- MISMATCH.append(batch)
- batchStartIndex = i
- batch.clear()
-
- if insertBool is False:
- pbar.update(1)
- i += 1
-
- currentGroup = []
-
- # [Passthrough 2] Setting Data
- else:
- # Get Text
- translatedText = translatedBatch[0]
-
- # Remove added speaker
- translatedText = re.sub(r"^.+?:\s", "", translatedText)
-
- # Textwrap
- translatedText = textwrap.fill(translatedText, width=WIDTH)
-
- # Set Text
- item[text] = translatedText
- translatedBatch.pop(0)
- speaker = ""
- currentGroup = []
- i += 1
-
- # If Batch is empty. Move on.
- if len(translatedBatch) == 0:
- insertBool = False
- batchStartIndex = i
- batch.clear()
- else:
- i += 1
- pbar.update(1)
-
- # Translate Batch if not empty and EOF
- if len(batch) != 0 and i >= len(data):
- # Translate
- response = translateGPT(batch, textHistory, True)
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- translatedBatch = response[0]
- textHistory = translatedBatch[-10:]
-
- # Set Values
- if len(batch) == len(translatedBatch):
- i = batchStartIndex
- insertBool = True
-
- # Mismatch
- else:
- pbar.write(f"Mismatch: {batchStartIndex} - {i}")
- MISMATCH.append(batch)
- batchStartIndex = i
- batch.clear()
-
- currentGroup = []
- return tokens
-
-
-# Save some money and enter the character before translation
-def getSpeaker(speaker):
- match speaker:
- case "ファイン":
- return ["Fine", [0, 0]]
- case "":
- return ["", [0, 0]]
- case _:
- # Find Speaker
- for i in range(len(NAMESLIST)):
- if speaker == NAMESLIST[i][0]:
- return [NAMESLIST[i][1], [0, 0]]
-
- # Translate and Store Speaker
- response = translateGPT(
- f"{speaker}",
- "Reply with the " + LANGUAGE + " translation of the NPC name.",
- True,
- )
- response[0] = response[0].title()
- response[0] = response[0].replace("'S", "'s")
- response[0] = response[0].replace("Speaker: ", "")
-
- # Retry if name doesn't translate for some reason
- if re.search(r"([a-zA-Z？?])", response[0]) == None:
- response = translateGPT(
- f"{speaker}",
- "Reply with the " + LANGUAGE + " translation of the NPC name.",
- False,
- )
- response[0] = response[0].title()
- response[0] = response[0].replace("'S", "'s")
-
- speakerList = [speaker, response[0]]
- NAMESLIST.append(speakerList)
- return response
- return [speaker, [0, 0]]
-
-
-def batchList(input_list, batch_size):
- if not isinstance(batch_size, int) or batch_size <= 0:
- raise ValueError("batch_size must be a positive integer")
-
- return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)]
-
-
-def createContext(fullPromptFlag, subbedT, format):
- system = (
- PROMPT + VOCAB
- if fullPromptFlag
- else f"\
-You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
-Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
-- You translate everything, including erotic and explicit content.\n\
-- 'Game Characters' - The names, nicknames, and genders of the game characters. Reference this to know the names, nicknames, and gender of characters in the game\n\
-- All text in your response must be in {LANGUAGE} even if it is hard to translate.\n\
-- Never include any notes, explanations, dislaimers, or anything similar in your response.\n\
-- Maintain any spacing in the translation.\n\
-- Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\
-- `...` can be a part of the dialogue. Translate it as it is.\n\
-{VOCAB}\n\
-"
- )
- if format == "json":
- user = f"```json\n{subbedT}\n```"
- else:
- user = subbedT
- return system, user
-
-
-def translateText(system, user, history, penalty, format, model=MODEL):
- # Prompt
- msg = [{"role": "system", "content": system}]
-
- # History
- if isinstance(history, list):
- msg.extend([{"role": "system", "content": h} for h in history])
- else:
- msg.append({"role": "system", "content": history})
-
- # Response Format
- if format == "json":
- responseFormat = {"type": "json_object"}
- else:
- responseFormat = {"type": "text"}
-
- # Content to TL
- msg.append({"role": "user", "content": f"{user}"})
- response = openai.chat.completions.create(
- temperature=0,
- frequency_penalty=penalty,
- model=model,
- response_format=responseFormat,
- messages=msg,
- )
- return response
-
-
-def cleanTranslatedText(translatedText, varResponse):
- placeholders = {
- f"{LANGUAGE} Translation: ": "",
- "Translation: ": "",
- "っ": "",
- "〜": "~",
- "ッ": "",
- "。": ".",
- "「": '\\"',
- "」": '\\"',
- "- ": "-",
- "—": "―",
- "】": "]",
- "【": "[",
- "Placeholder Text": "",
- # Add more replacements as needed
- }
- for target, replacement in placeholders.items():
- translatedText = translatedText.replace(target, replacement)
-
- # Remove Repeating Characters
- pattern = re.compile(r"(.)\s*\1(?:\s*\1){" + str(20 - 1) + r",}")
- translatedText = pattern.sub(lambda match: match.group(0).replace(" ", "")[:20], translatedText)
-
- # Elongate Long Dashes (Since GPT Ignores them...)
- translatedText = elongateCharacters(translatedText)
- return translatedText
-
-
-def elongateCharacters(text):
- # Define a pattern to match one character followed by one or more `ー` characters
- # Using a positive lookbehind assertion to capture the preceding character
- pattern = r"(?<=(.))ー+"
-
- # Define a replacement function that elongates the captured character
- def repl(match):
- char = match.group(1) # The character before the ー sequence
- count = len(match.group(0)) - 1 # Number of ー characters
- return char * count # Replace ー sequence with the character repeated
-
- # Use re.sub() to replace the pattern in the text
- return re.sub(pattern, repl, text)
-
-
-def extractTranslation(translatedTextList, is_list):
- try:
- translatedTextList = re.sub(r'\\"+\"([^,\n}])', r'\\"\1', translatedTextList)
- translatedTextList = re.sub(r"(? instead
+FIXTEXTWRAP = True # Overwrites textwrap
+IGNORETLTEXT = False # Ignores all translated text.
+MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
+
+# tqdm Globals
+BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
+POSITION = 0
+LEAVE = False
+
+# Pricing - Depends on the model https://openai.com/pricing
+# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request
+# If you are getting a MISMATCH LENGTH error, lower the batch size.
+if "gpt-3.5" in MODEL:
+ INPUTAPICOST = 0.002
+ OUTPUTAPICOST = 0.002
+ BATCHSIZE = 10
+elif "gpt-4" in MODEL:
+ INPUTAPICOST = 0.01
+ OUTPUTAPICOST = 0.03
+ BATCHSIZE = 50
+else:
+ INPUTAPICOST = float(os.getenv("input_cost"))
+ OUTPUTAPICOST = float(os.getenv("output_cost"))
+ BATCHSIZE = int(os.getenv("batchsize"))
+ FREQUENCY_PENALTY = float(os.getenv("frequency_penalty"))
+
+
+def handleJSON(filename, estimate):
+ global ESTIMATE, totalTokens
+ ESTIMATE = estimate
+
+ if estimate:
+ start = time.time()
+ translatedData = openFiles(filename)
+
+ # Print Result
+ end = time.time()
+ tqdm.write(getResultString(translatedData, end - start, filename))
+ with LOCK:
+ TOKENS[0] += translatedData[1][0]
+ TOKENS[1] += translatedData[1][1]
+
+ return getResultString(["", TOKENS, None], end - start, "TOTAL")
+
+ else:
+ try:
+ with open("translated/" + filename, "w", encoding="UTF-8") as outFile:
+ start = time.time()
+ translatedData = openFiles(filename)
+
+ # Print Result
+ end = time.time()
+ json.dump(translatedData[0], outFile, ensure_ascii=False, indent=4)
+ tqdm.write(getResultString(translatedData, end - start, filename))
+ with LOCK:
+ TOKENS[0] += translatedData[1][0]
+ TOKENS[1] += translatedData[1][1]
+ except Exception:
+ return "Fail"
+
+ return getResultString(["", TOKENS, None], end - start, "TOTAL")
+
+
+def openFiles(filename):
+ with open("files/" + filename, "r", encoding="UTF-8-sig") as f:
+ data = json.load(f)
+
+ # Map Files
+ if ".json" in filename:
+ translatedData = parseJSON(data, filename)
+
+ else:
+ raise NameError(filename + " Not Supported")
+
+ return translatedData
+
+
+def getResultString(translatedData, translationTime, filename):
+ # File Print String
+ totalTokenstring = (
+ Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
+ "[Output: "
+ + str(translatedData[1][1])
+ + "]" "[Cost: ${:,.4f}".format((translatedData[1][0] * 0.001 * INPUTAPICOST) + (translatedData[1][1] * 0.001 * OUTPUTAPICOST))
+ + "]"
+ )
+ timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
+
+ if translatedData[2] == None:
+ # Success
+ return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET
+
+ else:
+ # Fail
+ try:
+ raise translatedData[2]
+ except Exception as e:
+ traceback.print_exc()
+ errorString = str(e) + Fore.RED
+ return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET
+
+
+def parseJSON(data, filename):
+ totalTokens = [0, 0]
+ totalLines = 0
+ totalLines = len(data)
+ global LOCK
+
+ with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar:
+ pbar.desc = filename
+ pbar.total = totalLines
+ try:
+ result = translateJSON(data, pbar)
+ totalTokens[0] += result[0]
+ totalTokens[1] += result[1]
+ except Exception as e:
+ return [data, totalTokens, e]
+ return [data, totalTokens, None]
+
+
+def translateJSON(data, pbar):
+ textHistory = []
+ batch = []
+ maxHistory = MAXHISTORY
+ tokens = [0, 0]
+ speaker = "None"
+ insertBool = False
+ i = 0
+ batchStartIndex = 0
+
+ while i < len(data):
+ item = data[i]
+ # Speaker
+ if "VA" in item:
+ if item["name"] not in [None, "-"]:
+ response = getSpeaker(item["name"])
+ speaker = response[0]
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ item["name"] = speaker
+ else:
+ speaker = "None"
+ pbar.update(1)
+ i += 1
+
+ # Text
+ elif "me" in item:
+ for text in [
+ "text",
+ "text2",
+ "help1",
+ "help2",
+ "help3",
+ "like",
+ "message",
+ "me",
+ ]:
+ if text in item:
+ if item[text] != None:
+ jaString = item[text]
+
+ # Remove any textwrap
+ if FIXTEXTWRAP == True:
+ finalJAString = jaString.replace("\n", " ")
+
+ # [Passthrough 1] Pulling From File
+ if insertBool is False:
+ # Append to List and Clear Values
+ batch.append(finalJAString)
+ speaker = ""
+
+ # Translate Batch if Full
+ if len(batch) == BATCHSIZE:
+ # Translate
+ response = translateGPT(batch, textHistory, True)
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ translatedBatch = response[0]
+ textHistory = translatedBatch[-10:]
+
+ # Set Values
+ if len(batch) == len(translatedBatch):
+ i = batchStartIndex
+ insertBool = True
+
+ # Mismatch
+ else:
+ pbar.write(f"Mismatch: {batchStartIndex} - {i}")
+ MISMATCH.append(batch)
+ batchStartIndex = i
+ batch.clear()
+
+ if insertBool is False:
+ pbar.update(1)
+ i += 1
+
+ currentGroup = []
+
+ # [Passthrough 2] Setting Data
+ else:
+ # Get Text
+ translatedText = translatedBatch[0]
+
+ # Remove added speaker
+ translatedText = re.sub(r"^.+?:\s", "", translatedText)
+
+ # Textwrap
+ translatedText = textwrap.fill(translatedText, width=WIDTH)
+
+ # Set Text
+ item[text] = translatedText
+ translatedBatch.pop(0)
+ speaker = ""
+ currentGroup = []
+ i += 1
+
+ # If Batch is empty. Move on.
+ if len(translatedBatch) == 0:
+ insertBool = False
+ batchStartIndex = i
+ batch.clear()
+ else:
+ i += 1
+ pbar.update(1)
+
+ # Translate Batch if not empty and EOF
+ if len(batch) != 0 and i >= len(data):
+ # Translate
+ response = translateGPT(batch, textHistory, True)
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ translatedBatch = response[0]
+ textHistory = translatedBatch[-10:]
+
+ # Set Values
+ if len(batch) == len(translatedBatch):
+ i = batchStartIndex
+ insertBool = True
+
+ # Mismatch
+ else:
+ pbar.write(f"Mismatch: {batchStartIndex} - {i}")
+ MISMATCH.append(batch)
+ batchStartIndex = i
+ batch.clear()
+
+ currentGroup = []
+ return tokens
+
+
+# Save some money and enter the character before translation
+def getSpeaker(speaker):
+ match speaker:
+ case "ファイン":
+ return ["Fine", [0, 0]]
+ case "":
+ return ["", [0, 0]]
+ case _:
+ # Find Speaker
+ for i in range(len(NAMESLIST)):
+ if speaker == NAMESLIST[i][0]:
+ return [NAMESLIST[i][1], [0, 0]]
+
+ # Translate and Store Speaker
+ response = translateGPT(
+ f"{speaker}",
+ "Reply with the " + LANGUAGE + " translation of the NPC name.",
+ True,
+ )
+ response[0] = response[0].title()
+ response[0] = response[0].replace("'S", "'s")
+ response[0] = response[0].replace("Speaker: ", "")
+
+ # Retry if name doesn't translate for some reason
+ if re.search(r"([a-zA-Z？?])", response[0]) == None:
+ response = translateGPT(
+ f"{speaker}",
+ "Reply with the " + LANGUAGE + " translation of the NPC name.",
+ False,
+ )
+ response[0] = response[0].title()
+ response[0] = response[0].replace("'S", "'s")
+
+ speakerList = [speaker, response[0]]
+ NAMESLIST.append(speakerList)
+ return response
+ return [speaker, [0, 0]]
+
+
+def batchList(input_list, batch_size):
+ if not isinstance(batch_size, int) or batch_size <= 0:
+ raise ValueError("batch_size must be a positive integer")
+
+ return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)]
+
+
+def createContext(fullPromptFlag, subbedT, format):
+ system = (
+ PROMPT + VOCAB
+ if fullPromptFlag
+ else f"\
+You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
+Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
+- You translate everything, including erotic and explicit content.\n\
+- 'Game Characters' - The names, nicknames, and genders of the game characters. Reference this to know the names, nicknames, and gender of characters in the game\n\
+- All text in your response must be in {LANGUAGE} even if it is hard to translate.\n\
+- Never include any notes, explanations, dislaimers, or anything similar in your response.\n\
+- Maintain any spacing in the translation.\n\
+- Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\
+- `...` can be a part of the dialogue. Translate it as it is.\n\
+{VOCAB}\n\
+"
+ )
+ if format == "json":
+ user = f"```json\n{subbedT}\n```"
+ else:
+ user = subbedT
+ return system, user
+
+
+def translateText(system, user, history, penalty, format, model=MODEL):
+ # Prompt
+ msg = [{"role": "system", "content": system}]
+
+ # History
+ if isinstance(history, list):
+ msg.extend([{"role": "system", "content": h} for h in history])
+ else:
+ msg.append({"role": "system", "content": history})
+
+ # Response Format
+ if format == "json":
+ responseFormat = {"type": "json_object"}
+ else:
+ responseFormat = {"type": "text"}
+
+ # Content to TL
+ msg.append({"role": "user", "content": f"{user}"})
+ response = openai.chat.completions.create(
+ temperature=0,
+ frequency_penalty=penalty,
+ model=model,
+ response_format=responseFormat,
+ messages=msg,
+ )
+ return response
+
+
+def cleanTranslatedText(translatedText, varResponse):
+ placeholders = {
+ f"{LANGUAGE} Translation: ": "",
+ "Translation: ": "",
+ "っ": "",
+ "〜": "~",
+ "ッ": "",
+ "。": ".",
+ "「": '\\"',
+ "」": '\\"',
+ "- ": "-",
+ "—": "―",
+ "】": "]",
+ "【": "[",
+ "Placeholder Text": "",
+ # Add more replacements as needed
+ }
+ for target, replacement in placeholders.items():
+ translatedText = translatedText.replace(target, replacement)
+
+ # Remove Repeating Characters
+ pattern = re.compile(r"(.)\s*\1(?:\s*\1){" + str(20 - 1) + r",}")
+ translatedText = pattern.sub(lambda match: match.group(0).replace(" ", "")[:20], translatedText)
+
+ # Elongate Long Dashes (Since GPT Ignores them...)
+ translatedText = elongateCharacters(translatedText)
+ return translatedText
+
+
+def elongateCharacters(text):
+ # Define a pattern to match one character followed by one or more `ー` characters
+ # Using a positive lookbehind assertion to capture the preceding character
+ pattern = r"(?<=(.))ー+"
+
+ # Define a replacement function that elongates the captured character
+ def repl(match):
+ char = match.group(1) # The character before the ー sequence
+ count = len(match.group(0)) - 1 # Number of ー characters
+ return char * count # Replace ー sequence with the character repeated
+
+ # Use re.sub() to replace the pattern in the text
+ return re.sub(pattern, repl, text)
+
+
+def extractTranslation(translatedTextList, is_list):
+ try:
+ translatedTextList = re.sub(r'\\"+\"([^,\n}])', r'\\"\1', translatedTextList)
+ translatedTextList = re.sub(r"(? instead
-FIXTEXTWRAP = False # Overwrites textwrap
-IGNORETLTEXT = False # Ignores all translated text.
-MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
-
-# tqdm Globals
-BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
-POSITION = 0
-LEAVE = False
-
-# Pricing - Depends on the model https://openai.com/pricing
-# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request
-# If you are getting a MISMATCH LENGTH error, lower the batch size.
-if "gpt-3.5" in MODEL:
- INPUTAPICOST = 0.002
- OUTPUTAPICOST = 0.002
- BATCHSIZE = 10
-elif "gpt-4" in MODEL:
- INPUTAPICOST = 0.01
- OUTPUTAPICOST = 0.03
- BATCHSIZE = 10
-else:
- INPUTAPICOST = float(os.getenv("input_cost"))
- OUTPUTAPICOST = float(os.getenv("output_cost"))
- BATCHSIZE = int(os.getenv("batchsize"))
- FREQUENCY_PENALTY = float(os.getenv("frequency_penalty"))
-
-def handleKansen(filename, estimate):
- global ESTIMATE
- ESTIMATE = estimate
-
- if ESTIMATE:
- start = time.time()
- translatedData = openFiles(filename)
-
- # Print Result
- end = time.time()
- tqdm.write(getResultString(translatedData, end - start, filename))
- with LOCK:
- TOKENS[0] += translatedData[1][0]
- TOKENS[1] += translatedData[1][1]
-
- # Print Total
- totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
-
- # Print any errors on maps
- if len(MISMATCH) > 0:
- return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
- else:
- return totalString
-
- else:
- try:
- with open("translated/" + filename, "w", encoding="shift_jis", errors="ignore") as outFile:
- start = time.time()
- translatedData = openFiles(filename)
-
- # Print Result
- end = time.time()
- outFile.writelines(translatedData[0])
- tqdm.write(getResultString(translatedData, end - start, filename))
- with LOCK:
- TOKENS[0] += translatedData[1][0]
- TOKENS[1] += translatedData[1][1]
- except Exception:
- traceback.print_exc()
- return "Fail"
-
- return getResultString(["", TOKENS, None], end - start, "TOTAL")
-
-
-def getResultString(translatedData, translationTime, filename):
- # File Print String
- totalTokenstring = (
- Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
- "[Output: "
- + str(translatedData[1][1])
- + "]" "[Cost: ${:,.4f}".format((translatedData[1][0] * 0.001 * INPUTAPICOST) + (translatedData[1][1] * 0.001 * OUTPUTAPICOST))
- + "]"
- )
- timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
-
- if translatedData[2] == None:
- # Success
- return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET
-
- else:
- # Fail
- try:
- raise translatedData[2]
- except Exception as e:
- traceback.print_exc()
- errorString = str(e) + Fore.RED
- return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET
-
-
-def openFiles(filename):
- with open("files/" + filename, "r", encoding="cp932") as readFile:
- translatedData = parseTyrano(readFile, filename)
-
- # Delete lines marked for deletion
- finalData = []
- for line in translatedData[0]:
- if line != "\\d\n":
- finalData.append(line)
- translatedData[0] = finalData
-
- return translatedData
-
-
-def parseTyrano(readFile, filename):
- totalTokens = [0, 0]
- totalLines = 0
-
- # Get total for progress bar
- data = readFile.readlines()
- totalLines = len(data)
-
- with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar:
- pbar.desc = filename
- pbar.total = totalLines
-
- try:
- result = translateTyrano(data, pbar, totalLines)
- totalTokens[0] += result[0]
- totalTokens[1] += result[1]
- except Exception as e:
- traceback.print_exc()
- return [data, totalTokens, e]
- return [data, totalTokens, None]
-
-
-def translateTyrano(data, pbar, totalLines):
- textHistory = []
- batch = []
- currentGroup = []
- maxHistory = MAXHISTORY
- tokens = [0, 0]
- speaker = ""
- insertBool = False
- global LOCK, ESTIMATE
- i = 0
- batchStartIndex = 0
-
- while i < len(data):
- # Speaker
- if "[ns]" in data[i]:
- matchList = re.findall(r"\[ns\](.+?)\[", data[i])
- if len(matchList) != 0:
- response = getSpeaker(matchList[0])
- speaker = response[0]
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- data[i] = "[ns]" + speaker + "[nse]\n"
- else:
- speaker = ""
-
- # Choices
- elif "[sel" in data[i]:
- matchList = re.findall(r'\[sel.+text="(.+?)".+', data[i])
- if len(matchList) != 0:
- originalText = matchList[0]
- if len(textHistory) > 0:
- response = translateGPT(
- matchList[0],
- "Keep your translation as brief as possible. Previous text for context: "
- + textHistory[len(textHistory) - 1]
- + "\n\nReply in the style of a dialogue option.",
- False,
- )
- else:
- response = translateGPT(
- matchList[0],
- "\n\nReply in the style of a dialogue option.",
- False,
- )
- translatedText = response[0]
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
-
- # Remove characters that may break scripts
- charList = [".", '"', "\\n"]
- for char in charList:
- translatedText = translatedText.replace(char, "")
-
- # Escape all '
- translatedText = translatedText.replace("\\", "")
- # translatedText = translatedText.replace("'", "\\\'")
-
- # Set Data
- translatedText = data[i].replace(originalText, translatedText)
- data[i] = translatedText
-
- # Lines
- matchList = re.findall(r"(.+?)\[[rpcms_sel]+\]$", data[i])
- if len(matchList) > 0:
- if "hisout" in matchList[0]:
- i += 1
- continue
- currentGroup.append(matchList[0])
- if len(data) > i + 1:
- while "[r]" in data[i + 1]:
- if insertBool is True:
- data[i] = r"\d\n"
- pbar.update(1)
- i += 1
- matchList = re.findall(r"(.+?)\[r\]", data[i])
- if len(matchList) > 0:
- currentGroup.append(matchList[0])
- while "[pcms]" in data[i + 1]:
- if insertBool is True:
- data[i] = r"\d\n"
- pbar.update(1)
- i += 1
- matchList = re.findall(r"(.+?)\[pcms\]", data[i])
- if len(matchList) > 0:
- currentGroup.append(matchList[0])
- while "[pcms_sel]" in data[i + 1]:
- if insertBool is True:
- data[i] = r"\d\n"
- pbar.update(1)
- i += 1
- matchList = re.findall(r"(.+?)\[pcms_sel\]", data[i])
- if len(matchList) > 0:
- currentGroup.append(matchList[0])
- # Join up 401 groups for better translation.
- if len(currentGroup) > 0:
- finalJAString = " ".join(currentGroup)
- oldjaString = finalJAString
-
- # Remove any textwrap
- if FIXTEXTWRAP == True:
- finalJAString = finalJAString.replace("[r]", " ")
-
- # Remove Extra Stuff bad for translation.
- finalJAString = finalJAString.replace("ﾞ", "")
- finalJAString = finalJAString.replace("・", ".")
- finalJAString = finalJAString.replace("‶", "")
- finalJAString = finalJAString.replace("”", "")
- finalJAString = finalJAString.replace("―", "-")
- finalJAString = finalJAString.replace("…", "...")
- finalJAString = re.sub(r"(\.{3}\.+)", "...", finalJAString)
- finalJAString = finalJAString.replace(" ", " ")
-
- # Furigana Removal
- matchList = re.findall(r"(\[ruby\stext=.+text=\"(.+)\"\])", finalJAString)
- if len(matchList) > 0:
- finalJAString = finalJAString.replace(matchList[0][0], matchList[0][1])
-
- # Add Speaker (If there is one)
- if speaker != "":
- finalJAString = f"{speaker}: {finalJAString}"
-
- # [Passthrough 1] Pulling From File
- if insertBool is False:
- # Append to List and Clear Values
- batch.append(finalJAString)
- speaker = ""
-
- # Translate Batch if Full
- if len(batch) == BATCHSIZE:
- # Translate
- response = translateGPT(batch, textHistory, True)
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- translatedBatch = response[0]
- textHistory = translatedBatch[-10:]
-
- # Set Values
- if len(batch) == len(translatedBatch):
- i = batchStartIndex
- insertBool = True
-
- # Mismatch
- else:
- pbar.write(f"Mismatch: {batchStartIndex} - {i}")
- MISMATCH.append(batch)
- batchStartIndex = i
- batch.clear()
-
- i += 1
- if insertBool is True:
- pbar.update(1)
- currentGroup = []
-
- # [Passthrough 2] Setting Data
- else:
- # Get Text
- translatedText = translatedBatch[0]
- translatedText = translatedText.replace('\\"', '"')
- translatedText = translatedText.replace("[", "(")
- translatedText = translatedText.replace("]", ")")
-
- # Remove added speaker
- translatedText = re.sub(r"^.+?:\s", "", translatedText)
-
- # Textwrap
- translatedText = textwrap.fill(translatedText, width=WIDTH)
- textList = translatedText.split("\n")
-
- # Set Text
- data[i] = r"\d\n"
- for line in textList:
- # Wordwrap Text
- if "[r]" not in line:
- line = textwrap.fill(line, width=WIDTH)
- line = line.replace("\n", "[r]")
-
- # Set
- data.insert(i, line.strip() + "[r]\n")
- i += 1
- data[i - 1] = data[i - 1].replace("[r]", "[pcms]")
- translatedBatch.pop(0)
- speaker = ""
- currentGroup = []
-
- # If Batch is empty. Move on.
- if len(translatedBatch) == 0:
- insertBool = False
- batchStartIndex = i
- batch.clear()
-
- # Nothing relevant. Skip Line.
- else:
- i += 1
- if insertBool is True:
- pbar.update(1)
-
- # Translate Batch if not empty and EOF
- if len(batch) != 0 and i >= len(data):
- # Translate
- response = translateGPT(batch, textHistory, True)
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- translatedBatch = response[0]
- textHistory = translatedBatch[-10:]
-
- # Set Values
- if len(batch) == len(translatedBatch):
- i = batchStartIndex
- insertBool = True
-
- # Mismatch
- else:
- pbar.write(f"Mismatch: {batchStartIndex} - {i}")
- MISMATCH.append(batch)
- batchStartIndex = i
- batch.clear()
-
- currentGroup = []
- return tokens
-
-
-# Save some money and enter the character before translation
-def getSpeaker(speaker):
- match speaker:
- case "央":
- return ["Akira", [0, 0]]
- case "累":
- return ["Rui", [0, 0]]
- case "梨里":
- return ["Riri", [0, 0]]
- case "純":
- return ["Jun", [0, 0]]
- case "美鈴":
- return ["Misuzu", [0, 0]]
- case "須田":
- return ["Suda", [0, 0]]
- case "高橋":
- return ["Takahashi", [0, 0]]
- case "勇二":
- return ["Yuuji", [0, 0]]
- case _:
- return translateGPT(
- speaker,
- "Reply with only the " + LANGUAGE + " translation of the NPC name.",
- False,
- )
-
-
-def subVars(jaString):
- jaString = jaString.replace("\u3000", " ")
-
- # Nested
- count = 0
- nestedList = re.findall(r"[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]", jaString)
- nestedList = set(nestedList)
- if len(nestedList) != 0:
- for icon in nestedList:
- jaString = jaString.replace(icon, "{Nested_" + str(count) + "}")
- count += 1
-
- # Icons
- count = 0
- iconList = re.findall(r"[\\]+[iIkKwWaA]+\[[0-9]+\]", jaString)
- iconList = set(iconList)
- if len(iconList) != 0:
- for icon in iconList:
- jaString = jaString.replace(icon, "{Ascii_" + str(count) + "}")
- count += 1
-
- # Colors
- count = 0
- colorList = re.findall(r"[\\]+[cC]\[[0-9]+\]", jaString)
- colorList = set(colorList)
- if len(colorList) != 0:
- for color in colorList:
- jaString = jaString.replace(color, "{Color_" + str(count) + "}")
- count += 1
-
- # Names
- count = 0
- nameList = re.findall(r"[\\]+[nN]\[.+?\]+", jaString)
- nameList = set(nameList)
- if len(nameList) != 0:
- for name in nameList:
- jaString = jaString.replace(name, "{Noun_" + str(count) + "}")
- count += 1
-
- # Variables
- count = 0
- varList = re.findall(r"[\\]+[vV]\[[0-9]+\]", jaString)
- varList = set(varList)
- if len(varList) != 0:
- for var in varList:
- jaString = jaString.replace(var, "{Var_" + str(count) + "}")
- count += 1
-
- # Formatting
- count = 0
- formatList = re.findall(r"[\\]+[\w]+\[.+?\]", jaString)
- formatList = set(formatList)
- if len(formatList) != 0:
- for var in formatList:
- jaString = jaString.replace(var, "{FCode_" + str(count) + "}")
- count += 1
-
- # Put all lists in list and return
- allList = [nestedList, iconList, colorList, nameList, varList, formatList]
- return [jaString, allList]
-
-
-def resubVars(translatedText, allList):
- # Fix Spacing and ChatGPT Nonsense
- matchList = re.findall(r"\[\s?.+?\s?\]", translatedText)
- if len(matchList) > 0:
- for match in matchList:
- text = match.strip()
- translatedText = translatedText.replace(match, text)
-
- # Nested
- count = 0
- if len(allList[0]) != 0:
- for var in allList[0]:
- translatedText = translatedText.replace("{Nested_" + str(count) + "}", var)
- count += 1
-
- # Icons
- count = 0
- if len(allList[1]) != 0:
- for var in allList[1]:
- translatedText = translatedText.replace("{Ascii_" + str(count) + "}", var)
- count += 1
-
- # Colors
- count = 0
- if len(allList[2]) != 0:
- for var in allList[2]:
- translatedText = translatedText.replace("{Color_" + str(count) + "}", var)
- count += 1
-
- # Names
- count = 0
- if len(allList[3]) != 0:
- for var in allList[3]:
- translatedText = translatedText.replace("{Noun_" + str(count) + "}", var)
- count += 1
-
- # Vars
- count = 0
- if len(allList[4]) != 0:
- for var in allList[4]:
- translatedText = translatedText.replace("{Var_" + str(count) + "}", var)
- count += 1
-
- # Formatting
- count = 0
- if len(allList[5]) != 0:
- for var in allList[5]:
- translatedText = translatedText.replace("{FCode_" + str(count) + "}", var)
- count += 1
-
- return translatedText
-
-
-def batchList(input_list, batch_size):
- if not isinstance(batch_size, int) or batch_size <= 0:
- raise ValueError("batch_size must be a positive integer")
-
- return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)]
-
-
-def createContext(fullPromptFlag, subbedT):
- characters = "Game Characters:\n\
-渋江 央 (Shibue Akira) - Male\n\
-蘆名 累 (Ashina Rui) - Female\n\
-清原 梨里 (Kiyohara Riri) - Female\n\
-五十嵐 純 (Igarashi Jun) - Female\n\
-子野日 美鈴 (Nenohi Misuzu) - Female\n\
-須田 (Suda) - Male\n\
-高橋 (Takahashi) - Female\n\
-勇二 (Yuuji) - Male\n\
-"
-
- system = (
- PROMPT + VOCAB
- if fullPromptFlag
- else f"\
-You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
-You are going to be translating text from a videogame.\n\
-I will give you lines of text, and you must translate each line to the best of your ability.\n\
-{VOCAB}\n\
-Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\
-"
- )
- user = f"{subbedT}"
- return characters, system, user
-
-
-def translateText(characters, system, user, history):
- # Prompt
- msg = [{"role": "system", "content": system + characters}]
-
- # Characters
- msg.append({"role": "system", "content": characters})
-
- # History
- if isinstance(history, list):
- msg.extend([{"role": "system", "content": h} for h in history])
- else:
- msg.append({"role": "system", "content": history})
-
- # Content to TL
- msg.append({"role": "user", "content": f"{user}"})
- response = openai.chat.completions.create(
- temperature=0.1,
- frequency_penalty=0.1,
- presence_penalty=0.1,
- model=MODEL,
- messages=msg,
- )
- return response
-
-
-def cleanTranslatedText(translatedText, varResponse):
- placeholders = {
- f"{LANGUAGE} Translation: ": "",
- "Translation: ": "",
- "っ": "",
- "〜": "~",
- "ッ": "",
- "。": ".",
- "Placeholder Text": "",
- # Add more replacements as needed
- }
- for target, replacement in placeholders.items():
- translatedText = translatedText.replace(target, replacement)
-
- translatedText = resubVars(translatedText, varResponse[1])
- return [line for line in translatedText.replace("\\n", "\n").split("\n") if line]
-
-
-def extractTranslation(translatedTextList, is_list):
- pattern = r"`?([\\]*.*?[\\]*?)<\/?Line\d+>`?"
- # If it's a batch (i.e., list), extract with tags; otherwise, return the single item.
- if is_list:
- return [re.findall(pattern, line)[0][1] for line in translatedTextList if re.search(pattern, line)]
- else:
- matchList = re.findall(pattern, translatedTextList)
- return matchList[0][1] if matchList else translatedTextList
-
-
-def countTokens(characters, system, user, history):
- inputTotalTokens = 0
- outputTotalTokens = 0
- enc = tiktoken.encoding_for_model("gpt-4")
-
- # Input
- if isinstance(history, list):
- for line in history:
- inputTotalTokens += len(enc.encode(line))
- else:
- inputTotalTokens += len(enc.encode(history))
- inputTotalTokens += len(enc.encode(system))
- inputTotalTokens += len(enc.encode(characters))
- inputTotalTokens += len(enc.encode(user))
-
- # Output
- outputTotalTokens += round(len(enc.encode(user)) * 3)
-
- return [inputTotalTokens, outputTotalTokens]
-
-
-@retry(exceptions=Exception, tries=5, delay=5)
-def translateGPT(text, history, fullPromptFlag):
- totalTokens = [0, 0]
- if isinstance(text, list):
- tList = batchList(text, BATCHSIZE)
- else:
- tList = [text]
-
- for index, tItem in enumerate(tList):
- # Before sending to translation, if we have a list of items, add the formatting
- if isinstance(tItem, list):
- payload = "\n".join([f"`{item}`" for i, item in enumerate(tItem)])
- payload = payload.replace("``", "`Placeholder Text`")
- varResponse = subVars(payload)
- subbedT = varResponse[0]
- else:
- varResponse = subVars(tItem)
- subbedT = varResponse[0]
-
- # Things to Check before starting translation
- if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", subbedT):
- continue
-
- # Create Message
- characters, system, user = createContext(fullPromptFlag, subbedT)
-
- # Calculate Estimate
- if ESTIMATE:
- estimate = countTokens(characters, system, user, history)
- totalTokens[0] += estimate[0]
- totalTokens[1] += estimate[1]
- continue
-
- # Translating
- response = translateText(characters, system, user, history)
- translatedText = response.choices[0].message.content
- totalTokens[0] += response.usage.prompt_tokens
- totalTokens[1] += response.usage.completion_tokens
-
- # Formatting
- translatedTextList = cleanTranslatedText(translatedText, varResponse)
- if isinstance(tItem, list):
- extractedTranslations = extractTranslation(translatedTextList, True)
- tList[index] = extractedTranslations
- if len(tItem) != len(translatedTextList):
- mismatch = True # Just here so breakpoint can be set
- history = extractedTranslations[-10:] # Update history if we have a list
- else:
- # Ensure we're passing a single string to extractTranslation
- extractedTranslations = extractTranslation("\n".join(translatedTextList), False)
- tList[index] = extractedTranslations
-
- # Combine if multilist
- if isinstance(tList[0], list):
- tList = [t for sublist in tList for t in sublist]
-
- # Return
- if format == "json":
- return [tList, totalTokens]
- else:
- return [tList[0], totalTokens]
+# Libraries
+import os
+import re
+import textwrap
+import threading
+import time
+import traceback
+import tiktoken
+import openai
+from pathlib import Path
+from colorama import Fore
+from dotenv import load_dotenv
+from retry import retry
+from tqdm import tqdm
+
+# Open AI
+load_dotenv()
+if os.getenv("api").replace(" ", "") != "":
+ openai.base_url = os.getenv("api")
+openai.organization = os.getenv("org")
+openai.api_key = os.getenv("key")
+
+# Globals
+MODEL = os.getenv("model")
+TIMEOUT = int(os.getenv("timeout"))
+LANGUAGE = os.getenv("language").capitalize()
+PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
+VOCAB = Path("vocab.txt").read_text(encoding="utf-8")
+THREADS = int(os.getenv("threads"))
+LOCK = threading.Lock()
+WIDTH = int(os.getenv("width"))
+LISTWIDTH = int(os.getenv("listWidth"))
+NOTEWIDTH = 70
+MAXHISTORY = 10
+ESTIMATE = ""
+TOKENS = [0, 0]
+NAMESLIST = []
+NAMES = False # Output a list of all the character names found
+BRFLAG = False # If the game uses
instead
+FIXTEXTWRAP = False # Overwrites textwrap
+IGNORETLTEXT = False # Ignores all translated text.
+MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
+
+# tqdm Globals
+BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
+POSITION = 0
+LEAVE = False
+
+# Pricing - Depends on the model https://openai.com/pricing
+# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request
+# If you are getting a MISMATCH LENGTH error, lower the batch size.
+if "gpt-3.5" in MODEL:
+ INPUTAPICOST = 0.002
+ OUTPUTAPICOST = 0.002
+ BATCHSIZE = 10
+elif "gpt-4" in MODEL:
+ INPUTAPICOST = 0.01
+ OUTPUTAPICOST = 0.03
+ BATCHSIZE = 10
+else:
+ INPUTAPICOST = float(os.getenv("input_cost"))
+ OUTPUTAPICOST = float(os.getenv("output_cost"))
+ BATCHSIZE = int(os.getenv("batchsize"))
+ FREQUENCY_PENALTY = float(os.getenv("frequency_penalty"))
+
+
+def handleKansen(filename, estimate):
+ global ESTIMATE
+ ESTIMATE = estimate
+
+ if ESTIMATE:
+ start = time.time()
+ translatedData = openFiles(filename)
+
+ # Print Result
+ end = time.time()
+ tqdm.write(getResultString(translatedData, end - start, filename))
+ with LOCK:
+ TOKENS[0] += translatedData[1][0]
+ TOKENS[1] += translatedData[1][1]
+
+ # Print Total
+ totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
+
+ # Print any errors on maps
+ if len(MISMATCH) > 0:
+ return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
+ else:
+ return totalString
+
+ else:
+ try:
+ with open("translated/" + filename, "w", encoding="shift_jis", errors="ignore") as outFile:
+ start = time.time()
+ translatedData = openFiles(filename)
+
+ # Print Result
+ end = time.time()
+ outFile.writelines(translatedData[0])
+ tqdm.write(getResultString(translatedData, end - start, filename))
+ with LOCK:
+ TOKENS[0] += translatedData[1][0]
+ TOKENS[1] += translatedData[1][1]
+ except Exception:
+ traceback.print_exc()
+ return "Fail"
+
+ return getResultString(["", TOKENS, None], end - start, "TOTAL")
+
+
+def getResultString(translatedData, translationTime, filename):
+ # File Print String
+ totalTokenstring = (
+ Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
+ "[Output: "
+ + str(translatedData[1][1])
+ + "]" "[Cost: ${:,.4f}".format((translatedData[1][0] * 0.001 * INPUTAPICOST) + (translatedData[1][1] * 0.001 * OUTPUTAPICOST))
+ + "]"
+ )
+ timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
+
+ if translatedData[2] == None:
+ # Success
+ return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET
+
+ else:
+ # Fail
+ try:
+ raise translatedData[2]
+ except Exception as e:
+ traceback.print_exc()
+ errorString = str(e) + Fore.RED
+ return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET
+
+
+def openFiles(filename):
+ with open("files/" + filename, "r", encoding="cp932") as readFile:
+ translatedData = parseTyrano(readFile, filename)
+
+ # Delete lines marked for deletion
+ finalData = []
+ for line in translatedData[0]:
+ if line != "\\d\n":
+ finalData.append(line)
+ translatedData[0] = finalData
+
+ return translatedData
+
+
+def parseTyrano(readFile, filename):
+ totalTokens = [0, 0]
+ totalLines = 0
+
+ # Get total for progress bar
+ data = readFile.readlines()
+ totalLines = len(data)
+
+ with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar:
+ pbar.desc = filename
+ pbar.total = totalLines
+
+ try:
+ result = translateTyrano(data, pbar, totalLines)
+ totalTokens[0] += result[0]
+ totalTokens[1] += result[1]
+ except Exception as e:
+ traceback.print_exc()
+ return [data, totalTokens, e]
+ return [data, totalTokens, None]
+
+
+def translateTyrano(data, pbar, totalLines):
+ textHistory = []
+ batch = []
+ currentGroup = []
+ maxHistory = MAXHISTORY
+ tokens = [0, 0]
+ speaker = ""
+ insertBool = False
+ global LOCK, ESTIMATE
+ i = 0
+ batchStartIndex = 0
+
+ while i < len(data):
+ # Speaker
+ if "[ns]" in data[i]:
+ matchList = re.findall(r"\[ns\](.+?)\[", data[i])
+ if len(matchList) != 0:
+ response = getSpeaker(matchList[0])
+ speaker = response[0]
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ data[i] = "[ns]" + speaker + "[nse]\n"
+ else:
+ speaker = ""
+
+ # Choices
+ elif "[sel" in data[i]:
+ matchList = re.findall(r'\[sel.+text="(.+?)".+', data[i])
+ if len(matchList) != 0:
+ originalText = matchList[0]
+ if len(textHistory) > 0:
+ response = translateGPT(
+ matchList[0],
+ "Keep your translation as brief as possible. Previous text for context: "
+ + textHistory[len(textHistory) - 1]
+ + "\n\nReply in the style of a dialogue option.",
+ False,
+ )
+ else:
+ response = translateGPT(
+ matchList[0],
+ "\n\nReply in the style of a dialogue option.",
+ False,
+ )
+ translatedText = response[0]
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+
+ # Remove characters that may break scripts
+ charList = [".", '"', "\\n"]
+ for char in charList:
+ translatedText = translatedText.replace(char, "")
+
+ # Escape all '
+ translatedText = translatedText.replace("\\", "")
+ # translatedText = translatedText.replace("'", "\\\'")
+
+ # Set Data
+ translatedText = data[i].replace(originalText, translatedText)
+ data[i] = translatedText
+
+ # Lines
+ matchList = re.findall(r"(.+?)\[[rpcms_sel]+\]$", data[i])
+ if len(matchList) > 0:
+ if "hisout" in matchList[0]:
+ i += 1
+ continue
+ currentGroup.append(matchList[0])
+ if len(data) > i + 1:
+ while "[r]" in data[i + 1]:
+ if insertBool is True:
+ data[i] = r"\d\n"
+ pbar.update(1)
+ i += 1
+ matchList = re.findall(r"(.+?)\[r\]", data[i])
+ if len(matchList) > 0:
+ currentGroup.append(matchList[0])
+ while "[pcms]" in data[i + 1]:
+ if insertBool is True:
+ data[i] = r"\d\n"
+ pbar.update(1)
+ i += 1
+ matchList = re.findall(r"(.+?)\[pcms\]", data[i])
+ if len(matchList) > 0:
+ currentGroup.append(matchList[0])
+ while "[pcms_sel]" in data[i + 1]:
+ if insertBool is True:
+ data[i] = r"\d\n"
+ pbar.update(1)
+ i += 1
+ matchList = re.findall(r"(.+?)\[pcms_sel\]", data[i])
+ if len(matchList) > 0:
+ currentGroup.append(matchList[0])
+ # Join up 401 groups for better translation.
+ if len(currentGroup) > 0:
+ finalJAString = " ".join(currentGroup)
+ oldjaString = finalJAString
+
+ # Remove any textwrap
+ if FIXTEXTWRAP == True:
+ finalJAString = finalJAString.replace("[r]", " ")
+
+ # Remove Extra Stuff bad for translation.
+ finalJAString = finalJAString.replace("ﾞ", "")
+ finalJAString = finalJAString.replace("・", ".")
+ finalJAString = finalJAString.replace("‶", "")
+ finalJAString = finalJAString.replace("”", "")
+ finalJAString = finalJAString.replace("―", "-")
+ finalJAString = finalJAString.replace("…", "...")
+ finalJAString = re.sub(r"(\.{3}\.+)", "...", finalJAString)
+ finalJAString = finalJAString.replace(" ", " ")
+
+ # Furigana Removal
+ matchList = re.findall(r"(\[ruby\stext=.+text=\"(.+)\"\])", finalJAString)
+ if len(matchList) > 0:
+ finalJAString = finalJAString.replace(matchList[0][0], matchList[0][1])
+
+ # Add Speaker (If there is one)
+ if speaker != "":
+ finalJAString = f"{speaker}: {finalJAString}"
+
+ # [Passthrough 1] Pulling From File
+ if insertBool is False:
+ # Append to List and Clear Values
+ batch.append(finalJAString)
+ speaker = ""
+
+ # Translate Batch if Full
+ if len(batch) == BATCHSIZE:
+ # Translate
+ response = translateGPT(batch, textHistory, True)
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ translatedBatch = response[0]
+ textHistory = translatedBatch[-10:]
+
+ # Set Values
+ if len(batch) == len(translatedBatch):
+ i = batchStartIndex
+ insertBool = True
+
+ # Mismatch
+ else:
+ pbar.write(f"Mismatch: {batchStartIndex} - {i}")
+ MISMATCH.append(batch)
+ batchStartIndex = i
+ batch.clear()
+
+ i += 1
+ if insertBool is True:
+ pbar.update(1)
+ currentGroup = []
+
+ # [Passthrough 2] Setting Data
+ else:
+ # Get Text
+ translatedText = translatedBatch[0]
+ translatedText = translatedText.replace('\\"', '"')
+ translatedText = translatedText.replace("[", "(")
+ translatedText = translatedText.replace("]", ")")
+
+ # Remove added speaker
+ translatedText = re.sub(r"^.+?:\s", "", translatedText)
+
+ # Textwrap
+ translatedText = textwrap.fill(translatedText, width=WIDTH)
+ textList = translatedText.split("\n")
+
+ # Set Text
+ data[i] = r"\d\n"
+ for line in textList:
+ # Wordwrap Text
+ if "[r]" not in line:
+ line = textwrap.fill(line, width=WIDTH)
+ line = line.replace("\n", "[r]")
+
+ # Set
+ data.insert(i, line.strip() + "[r]\n")
+ i += 1
+ data[i - 1] = data[i - 1].replace("[r]", "[pcms]")
+ translatedBatch.pop(0)
+ speaker = ""
+ currentGroup = []
+
+ # If Batch is empty. Move on.
+ if len(translatedBatch) == 0:
+ insertBool = False
+ batchStartIndex = i
+ batch.clear()
+
+ # Nothing relevant. Skip Line.
+ else:
+ i += 1
+ if insertBool is True:
+ pbar.update(1)
+
+ # Translate Batch if not empty and EOF
+ if len(batch) != 0 and i >= len(data):
+ # Translate
+ response = translateGPT(batch, textHistory, True)
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ translatedBatch = response[0]
+ textHistory = translatedBatch[-10:]
+
+ # Set Values
+ if len(batch) == len(translatedBatch):
+ i = batchStartIndex
+ insertBool = True
+
+ # Mismatch
+ else:
+ pbar.write(f"Mismatch: {batchStartIndex} - {i}")
+ MISMATCH.append(batch)
+ batchStartIndex = i
+ batch.clear()
+
+ currentGroup = []
+ return tokens
+
+
+# Save some money and enter the character before translation
+def getSpeaker(speaker):
+ match speaker:
+ case "央":
+ return ["Akira", [0, 0]]
+ case "累":
+ return ["Rui", [0, 0]]
+ case "梨里":
+ return ["Riri", [0, 0]]
+ case "純":
+ return ["Jun", [0, 0]]
+ case "美鈴":
+ return ["Misuzu", [0, 0]]
+ case "須田":
+ return ["Suda", [0, 0]]
+ case "高橋":
+ return ["Takahashi", [0, 0]]
+ case "勇二":
+ return ["Yuuji", [0, 0]]
+ case _:
+ return translateGPT(
+ speaker,
+ "Reply with only the " + LANGUAGE + " translation of the NPC name.",
+ False,
+ )
+
+
+def subVars(jaString):
+ jaString = jaString.replace("\u3000", " ")
+
+ # Nested
+ count = 0
+ nestedList = re.findall(r"[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]", jaString)
+ nestedList = set(nestedList)
+ if len(nestedList) != 0:
+ for icon in nestedList:
+ jaString = jaString.replace(icon, "{Nested_" + str(count) + "}")
+ count += 1
+
+ # Icons
+ count = 0
+ iconList = re.findall(r"[\\]+[iIkKwWaA]+\[[0-9]+\]", jaString)
+ iconList = set(iconList)
+ if len(iconList) != 0:
+ for icon in iconList:
+ jaString = jaString.replace(icon, "{Ascii_" + str(count) + "}")
+ count += 1
+
+ # Colors
+ count = 0
+ colorList = re.findall(r"[\\]+[cC]\[[0-9]+\]", jaString)
+ colorList = set(colorList)
+ if len(colorList) != 0:
+ for color in colorList:
+ jaString = jaString.replace(color, "{Color_" + str(count) + "}")
+ count += 1
+
+ # Names
+ count = 0
+ nameList = re.findall(r"[\\]+[nN]\[.+?\]+", jaString)
+ nameList = set(nameList)
+ if len(nameList) != 0:
+ for name in nameList:
+ jaString = jaString.replace(name, "{Noun_" + str(count) + "}")
+ count += 1
+
+ # Variables
+ count = 0
+ varList = re.findall(r"[\\]+[vV]\[[0-9]+\]", jaString)
+ varList = set(varList)
+ if len(varList) != 0:
+ for var in varList:
+ jaString = jaString.replace(var, "{Var_" + str(count) + "}")
+ count += 1
+
+ # Formatting
+ count = 0
+ formatList = re.findall(r"[\\]+[\w]+\[.+?\]", jaString)
+ formatList = set(formatList)
+ if len(formatList) != 0:
+ for var in formatList:
+ jaString = jaString.replace(var, "{FCode_" + str(count) + "}")
+ count += 1
+
+ # Put all lists in list and return
+ allList = [nestedList, iconList, colorList, nameList, varList, formatList]
+ return [jaString, allList]
+
+
+def resubVars(translatedText, allList):
+ # Fix Spacing and ChatGPT Nonsense
+ matchList = re.findall(r"\[\s?.+?\s?\]", translatedText)
+ if len(matchList) > 0:
+ for match in matchList:
+ text = match.strip()
+ translatedText = translatedText.replace(match, text)
+
+ # Nested
+ count = 0
+ if len(allList[0]) != 0:
+ for var in allList[0]:
+ translatedText = translatedText.replace("{Nested_" + str(count) + "}", var)
+ count += 1
+
+ # Icons
+ count = 0
+ if len(allList[1]) != 0:
+ for var in allList[1]:
+ translatedText = translatedText.replace("{Ascii_" + str(count) + "}", var)
+ count += 1
+
+ # Colors
+ count = 0
+ if len(allList[2]) != 0:
+ for var in allList[2]:
+ translatedText = translatedText.replace("{Color_" + str(count) + "}", var)
+ count += 1
+
+ # Names
+ count = 0
+ if len(allList[3]) != 0:
+ for var in allList[3]:
+ translatedText = translatedText.replace("{Noun_" + str(count) + "}", var)
+ count += 1
+
+ # Vars
+ count = 0
+ if len(allList[4]) != 0:
+ for var in allList[4]:
+ translatedText = translatedText.replace("{Var_" + str(count) + "}", var)
+ count += 1
+
+ # Formatting
+ count = 0
+ if len(allList[5]) != 0:
+ for var in allList[5]:
+ translatedText = translatedText.replace("{FCode_" + str(count) + "}", var)
+ count += 1
+
+ return translatedText
+
+
+def batchList(input_list, batch_size):
+ if not isinstance(batch_size, int) or batch_size <= 0:
+ raise ValueError("batch_size must be a positive integer")
+
+ return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)]
+
+
+def createContext(fullPromptFlag, subbedT):
+ characters = "Game Characters:\n\
+渋江 央 (Shibue Akira) - Male\n\
+蘆名 累 (Ashina Rui) - Female\n\
+清原 梨里 (Kiyohara Riri) - Female\n\
+五十嵐 純 (Igarashi Jun) - Female\n\
+子野日 美鈴 (Nenohi Misuzu) - Female\n\
+須田 (Suda) - Male\n\
+高橋 (Takahashi) - Female\n\
+勇二 (Yuuji) - Male\n\
+"
+
+ system = (
+ PROMPT + VOCAB
+ if fullPromptFlag
+ else f"\
+You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
+You are going to be translating text from a videogame.\n\
+I will give you lines of text, and you must translate each line to the best of your ability.\n\
+{VOCAB}\n\
+Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\
+"
+ )
+ user = f"{subbedT}"
+ return characters, system, user
+
+
+def translateText(characters, system, user, history):
+ # Prompt
+ msg = [{"role": "system", "content": system + characters}]
+
+ # Characters
+ msg.append({"role": "system", "content": characters})
+
+ # History
+ if isinstance(history, list):
+ msg.extend([{"role": "system", "content": h} for h in history])
+ else:
+ msg.append({"role": "system", "content": history})
+
+ # Content to TL
+ msg.append({"role": "user", "content": f"{user}"})
+ response = openai.chat.completions.create(
+ temperature=0.1,
+ frequency_penalty=0.1,
+ presence_penalty=0.1,
+ model=MODEL,
+ messages=msg,
+ )
+ return response
+
+
+def cleanTranslatedText(translatedText, varResponse):
+ placeholders = {
+ f"{LANGUAGE} Translation: ": "",
+ "Translation: ": "",
+ "っ": "",
+ "〜": "~",
+ "ッ": "",
+ "。": ".",
+ "Placeholder Text": "",
+ # Add more replacements as needed
+ }
+ for target, replacement in placeholders.items():
+ translatedText = translatedText.replace(target, replacement)
+
+ translatedText = resubVars(translatedText, varResponse[1])
+ return [line for line in translatedText.replace("\\n", "\n").split("\n") if line]
+
+
+def extractTranslation(translatedTextList, is_list):
+ pattern = r"`?([\\]*.*?[\\]*?)<\/?Line\d+>`?"
+ # If it's a batch (i.e., list), extract with tags; otherwise, return the single item.
+ if is_list:
+ return [re.findall(pattern, line)[0][1] for line in translatedTextList if re.search(pattern, line)]
+ else:
+ matchList = re.findall(pattern, translatedTextList)
+ return matchList[0][1] if matchList else translatedTextList
+
+
+def countTokens(characters, system, user, history):
+ inputTotalTokens = 0
+ outputTotalTokens = 0
+ enc = tiktoken.encoding_for_model("gpt-4")
+
+ # Input
+ if isinstance(history, list):
+ for line in history:
+ inputTotalTokens += len(enc.encode(line))
+ else:
+ inputTotalTokens += len(enc.encode(history))
+ inputTotalTokens += len(enc.encode(system))
+ inputTotalTokens += len(enc.encode(characters))
+ inputTotalTokens += len(enc.encode(user))
+
+ # Output
+ outputTotalTokens += round(len(enc.encode(user)) * 3)
+
+ return [inputTotalTokens, outputTotalTokens]
+
+
+@retry(exceptions=Exception, tries=5, delay=5)
+def translateGPT(text, history, fullPromptFlag):
+ totalTokens = [0, 0]
+ if isinstance(text, list):
+ tList = batchList(text, BATCHSIZE)
+ else:
+ tList = [text]
+
+ for index, tItem in enumerate(tList):
+ # Before sending to translation, if we have a list of items, add the formatting
+ if isinstance(tItem, list):
+ payload = "\n".join([f"`{item}`" for i, item in enumerate(tItem)])
+ payload = payload.replace("``", "`Placeholder Text`")
+ varResponse = subVars(payload)
+ subbedT = varResponse[0]
+ else:
+ varResponse = subVars(tItem)
+ subbedT = varResponse[0]
+
+ # Things to Check before starting translation
+ if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", subbedT):
+ continue
+
+ # Create Message
+ characters, system, user = createContext(fullPromptFlag, subbedT)
+
+ # Calculate Estimate
+ if ESTIMATE:
+ estimate = countTokens(characters, system, user, history)
+ totalTokens[0] += estimate[0]
+ totalTokens[1] += estimate[1]
+ continue
+
+ # Translating
+ response = translateText(characters, system, user, history)
+ translatedText = response.choices[0].message.content
+ totalTokens[0] += response.usage.prompt_tokens
+ totalTokens[1] += response.usage.completion_tokens
+
+ # Formatting
+ translatedTextList = cleanTranslatedText(translatedText, varResponse)
+ if isinstance(tItem, list):
+ extractedTranslations = extractTranslation(translatedTextList, True)
+ tList[index] = extractedTranslations
+ if len(tItem) != len(translatedTextList):
+ mismatch = True # Just here so breakpoint can be set
+ history = extractedTranslations[-10:] # Update history if we have a list
+ else:
+ # Ensure we're passing a single string to extractTranslation
+ extractedTranslations = extractTranslation("\n".join(translatedTextList), False)
+ tList[index] = extractedTranslations
+
+ # Combine if multilist
+ if isinstance(tList[0], list):
+ tList = [t for sublist in tList for t in sublist]
+
+ # Return
+ if format == "json":
+ return [tList, totalTokens]
+ else:
+ return [tList[0], totalTokens]
diff --git a/modules/kirikiri.py b/modules/kirikiri.py
index de94b09..b2286a1 100644
--- a/modules/kirikiri.py
+++ b/modules/kirikiri.py
@@ -1,611 +1,612 @@
-# Libraries
-import json
-import os
-import re
-import textwrap
-import threading
-import time
-import traceback
-import tiktoken
-import openai
-from pathlib import Path
-from colorama import Fore
-from dotenv import load_dotenv
-from retry import retry
-from tqdm import tqdm
-
-# Open AI
-load_dotenv()
-if os.getenv("api").replace(" ", "") != "":
- openai.base_url = os.getenv("api")
-openai.organization = os.getenv("org")
-openai.api_key = os.getenv("key")
-
-# Globals
-MODEL = os.getenv("model")
-TIMEOUT = int(os.getenv("timeout"))
-LANGUAGE = os.getenv("language").capitalize()
-PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
-VOCAB = Path("vocab.txt").read_text(encoding="utf-8")
-THREADS = int(os.getenv("threads"))
-LOCK = threading.Lock()
-WIDTH = int(os.getenv("width"))
-LISTWIDTH = int(os.getenv("listWidth"))
-NOTEWIDTH = 70
-MAXHISTORY = 10
-ESTIMATE = ""
-TOKENS = [0, 0]
-NAMESLIST = []
-NAMES = False # Output a list of all the character names found
-BRFLAG = False # If the game uses
instead
-FIXTEXTWRAP = True # Overwrites textwrap
-IGNORETLTEXT = False # Ignores all translated text.
-MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
-PBAR = None
-FILENAME = None
-
-# tqdm Globals
-BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
-POSITION = 0
-LEAVE = False
-
-# Flags
-SPEAKERS = True
-CHOICES = True
-DIALOGUE = True
-
-# Pricing - Depends on the model https://openai.com/pricing
-# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request
-# If you are getting a MISMATCH LENGTH error, lower the batch size.
-if "gpt-3.5" in MODEL:
- INPUTAPICOST = 0.002
- OUTPUTAPICOST = 0.002
- BATCHSIZE = 10
-elif "gpt-4" in MODEL:
- INPUTAPICOST = 0.0025
- OUTPUTAPICOST = 0.01
- BATCHSIZE = 40
-else:
- INPUTAPICOST = float(os.getenv("input_cost"))
- OUTPUTAPICOST = float(os.getenv("output_cost"))
- BATCHSIZE = int(os.getenv("batchsize"))
- FREQUENCY_PENALTY = float(os.getenv("frequency_penalty"))
-
-def handleKirikiri(filename, estimate):
- global ESTIMATE, FILENAME
- ESTIMATE = estimate
- FILENAME = filename
-
- if ESTIMATE:
- start = time.time()
- translatedData = openFiles(filename)
-
- # Print Result
- end = time.time()
- tqdm.write(getResultString(translatedData, end - start, filename))
- with LOCK:
- TOKENS[0] += translatedData[1][0]
- TOKENS[1] += translatedData[1][1]
-
- # Print Total
- totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
-
- # Print any errors on maps
- if len(MISMATCH) > 0:
- return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
- else:
- return totalString
-
- else:
- try:
- with open("translated/" + filename, "w", encoding="cp932", errors="ignore") as outFile:
- start = time.time()
- translatedData = openFiles(filename)
-
- # Print Result
- end = time.time()
- if translatedData[0] != []:
- outFile.writelines(translatedData[0])
- else:
- PBAR.write(f"{FILENAME} Failed to write")
- os.remove(f"translated/{filename}")
- tqdm.write(getResultString(translatedData, end - start, filename))
- with LOCK:
- TOKENS[0] += translatedData[1][0]
- TOKENS[1] += translatedData[1][1]
- except Exception:
- traceback.print_exc()
- os.remove(f"translated/{filename}")
- return "Fail"
-
- return getResultString(["", TOKENS, None], end - start, "TOTAL")
-
-
-def getResultString(translatedData, translationTime, filename):
- # File Print String
- totalTokenstring = (
- Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
- "[Output: "
- + str(translatedData[1][1])
- + "]" "[Cost: ${:,.4f}".format((translatedData[1][0] * 0.001 * INPUTAPICOST) + (translatedData[1][1] * 0.001 * OUTPUTAPICOST))
- + "]"
- )
- timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
-
- if translatedData[2] == None:
- # Success
- return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET
-
- else:
- # Fail
- try:
- raise translatedData[2]
- except Exception as e:
- traceback.print_exc()
- errorString = str(e) + Fore.RED
- return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET
-
-
-def openFiles(filename):
- with open("files/" + filename, "r", encoding="cp932") as readFile:
- translatedData = parseKiriKiri(readFile, filename)
- return translatedData
-
-
-def parseKiriKiri(readFile, filename):
- global PBAR
- totalTokens = [0, 0]
-
- # Read File into data
- data = readFile.readlines()
-
- # Create Progress Bar
- with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as PBAR:
- PBAR.desc = filename
-
- try:
- result = translateKiriKiri(data, PBAR, filename, [])
- totalTokens[0] += result[0]
- totalTokens[1] += result[1]
- except Exception as e:
- traceback.print_exc()
- return [data, totalTokens, e]
- return [data, totalTokens, None]
-
-
-def translateKiriKiri(data, pbar, filename, jobList):
- # Check Job Data
- if len(jobList) > 0:
- stringList = jobList[0]
- choiceList = jobList[1]
- setData = True
- else:
- stringList = []
- choiceList = []
- setData = False
- tokens = [0, 0]
- speaker = ""
- global LOCK, ESTIMATE
- i = 0
-
- # Regex
- speakerRegex = r"【(.*)】\[CR\]"
- dialogueRegex = r"^\[text\](.*).*\[KeyWait\]|\[\w+\](.*)\[\/\w+\].*\[KeyWait\]"
- furiganaRegex = r'(\[eruby\sstr="(.*?)"\stext.*?\])'
- choicesRegex = r"^\s*\[button\d\sclickse=sys_decide.*text='(.*?)'.*"
-
- while i < len(data):
- speaker = ""
- # Speaker
- match = re.search(speakerRegex, data[i])
- if match and SPEAKERS:
- speakerJA = match.group(1)
- response = getSpeaker(speakerJA)
- speaker = response[0]
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- data[i] = data[i].replace(speakerJA, speaker)
- i += 1
-
- # Choices
- match = re.search(choicesRegex, data[i])
- if match and CHOICES:
- jaString = match.group(1)
-
- # Pass 1
- if not setData:
- choiceList.append(jaString)
-
- # Pass 2
- else:
- # Grab and Pop and Set
- translatedText = choiceList[0]
- choiceList.pop(0)
-
- # Replace Quotes
- data[i] = data[i].replace("'", '"')
- translatedText = translatedText.replace('"', "'")
- data[i] = data[i].replace(jaString, translatedText)
-
- # Dialogue
- match = re.search(dialogueRegex, data[i])
- if match and DIALOGUE:
- jaString = match.group(1)
- if not jaString:
- jaString = match.group(2)
-
- # Pass 1
- if not setData:
- # Remove any textwrap
- jaString = jaString.replace("[r]", " ")
-
- # Remove Furigana
- matchList = re.findall(furiganaRegex, jaString)
- if matchList:
- for match in matchList:
- jaString = jaString.replace(match[0], match[1])
-
- # Add String
- if speaker:
- stringList.append(f"[{speaker}]: {jaString.strip()}")
- else:
- stringList.append(f"{jaString.strip()}")
-
- # Pass 2
- else:
- if len(stringList) > 0:
- # Grab and Pop
- translatedText = stringList[0]
- stringList.pop(0)
-
- # Remove Speaker
- translatedText = re.sub(r"\[.*?\]:\s", "", translatedText)
-
- # Textwrap
- translatedText = textwrap.fill(translatedText, width=WIDTH)
- translatedText = translatedText.replace("\n", "[r]")
-
- # Replace Quotes
- data[i] = data[i].replace("'", '"')
- translatedText = translatedText.replace('"', "'")
- data[i] = data[i].replace(jaString, translatedText)
-
- # Next Line
- i += 1
-
- # EOF
- stringListTL = []
- choiceListTL = []
-
- # Dialogue
- if len(stringList) > 0:
- # Set Progress
- pbar.total = len(stringList)
- pbar.refresh()
-
- # Translate
- response = translateGPT(
- stringList,
- "",
- True,
- )
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- stringListTL = response[0]
-
- # Validate
- if len(stringList) != len(stringListTL):
- with LOCK:
- if filename not in MISMATCH:
- MISMATCH.append(filename)
- stringListTL = stringList
-
- # Choices
- if len(choiceList) > 0:
- # Set Progress
- pbar.total = len(choiceList)
- pbar.refresh()
-
- # Translate
- response = translateGPT(
- choiceList,
- "",
- True,
- )
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- choiceListTL = response[0]
-
- # Validate
- if len(choiceList) != len(choiceListTL):
- with LOCK:
- if filename not in MISMATCH:
- MISMATCH.append(filename)
- choiceListTL = choiceList
-
- # Proceed to Pass 2
- if not setData:
- translateKiriKiri(data, pbar, filename, [stringListTL, choiceListTL])
-
- return tokens
-
-
-# Save some money and enter the character before translation
-def getSpeaker(speaker):
- match speaker:
- case "ファイン":
- return ["Fine", [0, 0]]
- case "":
- return ["", [0, 0]]
- case _:
- # Find Speaker
- for i in range(len(NAMESLIST)):
- if speaker == NAMESLIST[i][0]:
- return [NAMESLIST[i][1], [0, 0]]
-
- # Translate and Store Speaker
- response = translateGPT(
- f"{speaker}",
- "Reply with the " + LANGUAGE + " translation of the NPC name.",
- True,
- )
- response[0] = response[0].title()
- response[0] = response[0].replace("'S", "'s")
- response[0] = response[0].replace("Speaker: ", "")
-
- # Retry if name doesn't translate for some reason
- if re.search(r"([a-zA-Z？?])", response[0]) == None:
- response = translateGPT(
- f"{speaker}",
- "Reply with the " + LANGUAGE + " translation of the NPC name.",
- False,
- )
- response[0] = response[0].title()
- response[0] = response[0].replace("'S", "'s")
-
- speakerList = [speaker, response[0]]
- NAMESLIST.append(speakerList)
- return response
- return [speaker, [0, 0]]
-
-
-def batchList(input_list, batch_size):
- if not isinstance(batch_size, int) or batch_size <= 0:
- raise ValueError("batch_size must be a positive integer")
-
- return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)]
-
-
-def createContext(fullPromptFlag, subbedT, format):
- system = (
- PROMPT + VOCAB
- if fullPromptFlag
- else f"\
-You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
-Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
-- You translate everything, including erotic and explicit content.\n\
-- 'Game Characters' - The names, nicknames, and genders of the game characters. Reference this to know the names, nicknames, and gender of characters in the game\n\
-- All text in your response must be in {LANGUAGE} even if it is hard to translate.\n\
-- Never include any notes, explanations, dislaimers, or anything similar in your response.\n\
-- Maintain any spacing in the translation.\n\
-- Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\
-- `...` can be a part of the dialogue. Translate it as it is.\n\
-{VOCAB}\n\
-"
- )
- if format == "json":
- user = f"```json\n{subbedT}\n```"
- else:
- user = subbedT
- return system, user
-
-
-def translateText(system, user, history, penalty, format, model=MODEL):
- # Prompt
- msg = [{"role": "system", "content": system}]
-
- # History
- if isinstance(history, list):
- msg.extend([{"role": "system", "content": h} for h in history])
- else:
- msg.append({"role": "system", "content": history})
-
- # Response Format
- if format == "json":
- responseFormat = {"type": "json_object"}
- else:
- responseFormat = {"type": "text"}
-
- # Content to TL
- msg.append({"role": "user", "content": f"{user}"})
- response = openai.chat.completions.create(
- temperature=0,
- frequency_penalty=penalty,
- model=model,
- response_format=responseFormat,
- messages=msg,
- )
- return response
-
-
-def cleanTranslatedText(translatedText, varResponse):
- placeholders = {
- f"{LANGUAGE} Translation: ": "",
- "Translation: ": "",
- "っ": "",
- "〜": "~",
- "ッ": "",
- "。": ".",
- "「": '\\"',
- "」": '\\"',
- "- ": "-",
- "—": "―",
- "】": "]",
- "【": "[",
- "Placeholder Text": "",
- # Add more replacements as needed
- }
- for target, replacement in placeholders.items():
- translatedText = translatedText.replace(target, replacement)
-
- # Remove Repeating Characters
- pattern = re.compile(r"(.)\s*\1(?:\s*\1){" + str(20 - 1) + r",}")
- translatedText = pattern.sub(lambda match: match.group(0).replace(" ", "")[:20], translatedText)
-
- # Elongate Long Dashes (Since GPT Ignores them...)
- translatedText = elongateCharacters(translatedText)
- return translatedText
-
-
-def elongateCharacters(text):
- # Define a pattern to match one character followed by one or more `ー` characters
- # Using a positive lookbehind assertion to capture the preceding character
- pattern = r"(?<=(.))ー+"
-
- # Define a replacement function that elongates the captured character
- def repl(match):
- char = match.group(1) # The character before the ー sequence
- count = len(match.group(0)) - 1 # Number of ー characters
- return char * count # Replace ー sequence with the character repeated
-
- # Use re.sub() to replace the pattern in the text
- return re.sub(pattern, repl, text)
-
-
-def extractTranslation(translatedTextList, is_list):
- try:
- translatedTextList = re.sub(r'\\"+\"([^,\n}])', r'\\"\1', translatedTextList)
- translatedTextList = re.sub(r"(? instead
+FIXTEXTWRAP = True # Overwrites textwrap
+IGNORETLTEXT = False # Ignores all translated text.
+MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
+PBAR = None
+FILENAME = None
+
+# tqdm Globals
+BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
+POSITION = 0
+LEAVE = False
+
+# Flags
+SPEAKERS = True
+CHOICES = True
+DIALOGUE = True
+
+# Pricing - Depends on the model https://openai.com/pricing
+# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request
+# If you are getting a MISMATCH LENGTH error, lower the batch size.
+if "gpt-3.5" in MODEL:
+ INPUTAPICOST = 0.002
+ OUTPUTAPICOST = 0.002
+ BATCHSIZE = 10
+elif "gpt-4" in MODEL:
+ INPUTAPICOST = 0.0025
+ OUTPUTAPICOST = 0.01
+ BATCHSIZE = 40
+else:
+ INPUTAPICOST = float(os.getenv("input_cost"))
+ OUTPUTAPICOST = float(os.getenv("output_cost"))
+ BATCHSIZE = int(os.getenv("batchsize"))
+ FREQUENCY_PENALTY = float(os.getenv("frequency_penalty"))
+
+
+def handleKirikiri(filename, estimate):
+ global ESTIMATE, FILENAME
+ ESTIMATE = estimate
+ FILENAME = filename
+
+ if ESTIMATE:
+ start = time.time()
+ translatedData = openFiles(filename)
+
+ # Print Result
+ end = time.time()
+ tqdm.write(getResultString(translatedData, end - start, filename))
+ with LOCK:
+ TOKENS[0] += translatedData[1][0]
+ TOKENS[1] += translatedData[1][1]
+
+ # Print Total
+ totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
+
+ # Print any errors on maps
+ if len(MISMATCH) > 0:
+ return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
+ else:
+ return totalString
+
+ else:
+ try:
+ with open("translated/" + filename, "w", encoding="cp932", errors="ignore") as outFile:
+ start = time.time()
+ translatedData = openFiles(filename)
+
+ # Print Result
+ end = time.time()
+ if translatedData[0] != []:
+ outFile.writelines(translatedData[0])
+ else:
+ PBAR.write(f"{FILENAME} Failed to write")
+ os.remove(f"translated/{filename}")
+ tqdm.write(getResultString(translatedData, end - start, filename))
+ with LOCK:
+ TOKENS[0] += translatedData[1][0]
+ TOKENS[1] += translatedData[1][1]
+ except Exception:
+ traceback.print_exc()
+ os.remove(f"translated/{filename}")
+ return "Fail"
+
+ return getResultString(["", TOKENS, None], end - start, "TOTAL")
+
+
+def getResultString(translatedData, translationTime, filename):
+ # File Print String
+ totalTokenstring = (
+ Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
+ "[Output: "
+ + str(translatedData[1][1])
+ + "]" "[Cost: ${:,.4f}".format((translatedData[1][0] * 0.001 * INPUTAPICOST) + (translatedData[1][1] * 0.001 * OUTPUTAPICOST))
+ + "]"
+ )
+ timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
+
+ if translatedData[2] == None:
+ # Success
+ return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET
+
+ else:
+ # Fail
+ try:
+ raise translatedData[2]
+ except Exception as e:
+ traceback.print_exc()
+ errorString = str(e) + Fore.RED
+ return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET
+
+
+def openFiles(filename):
+ with open("files/" + filename, "r", encoding="cp932") as readFile:
+ translatedData = parseKiriKiri(readFile, filename)
+ return translatedData
+
+
+def parseKiriKiri(readFile, filename):
+ global PBAR
+ totalTokens = [0, 0]
+
+ # Read File into data
+ data = readFile.readlines()
+
+ # Create Progress Bar
+ with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as PBAR:
+ PBAR.desc = filename
+
+ try:
+ result = translateKiriKiri(data, PBAR, filename, [])
+ totalTokens[0] += result[0]
+ totalTokens[1] += result[1]
+ except Exception as e:
+ traceback.print_exc()
+ return [data, totalTokens, e]
+ return [data, totalTokens, None]
+
+
+def translateKiriKiri(data, pbar, filename, jobList):
+ # Check Job Data
+ if len(jobList) > 0:
+ stringList = jobList[0]
+ choiceList = jobList[1]
+ setData = True
+ else:
+ stringList = []
+ choiceList = []
+ setData = False
+ tokens = [0, 0]
+ speaker = ""
+ global LOCK, ESTIMATE
+ i = 0
+
+ # Regex
+ speakerRegex = r"【(.*)】\[CR\]"
+ dialogueRegex = r"^\[text\](.*).*\[KeyWait\]|\[\w+\](.*)\[\/\w+\].*\[KeyWait\]"
+ furiganaRegex = r'(\[eruby\sstr="(.*?)"\stext.*?\])'
+ choicesRegex = r"^\s*\[button\d\sclickse=sys_decide.*text='(.*?)'.*"
+
+ while i < len(data):
+ speaker = ""
+ # Speaker
+ match = re.search(speakerRegex, data[i])
+ if match and SPEAKERS:
+ speakerJA = match.group(1)
+ response = getSpeaker(speakerJA)
+ speaker = response[0]
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ data[i] = data[i].replace(speakerJA, speaker)
+ i += 1
+
+ # Choices
+ match = re.search(choicesRegex, data[i])
+ if match and CHOICES:
+ jaString = match.group(1)
+
+ # Pass 1
+ if not setData:
+ choiceList.append(jaString)
+
+ # Pass 2
+ else:
+ # Grab and Pop and Set
+ translatedText = choiceList[0]
+ choiceList.pop(0)
+
+ # Replace Quotes
+ data[i] = data[i].replace("'", '"')
+ translatedText = translatedText.replace('"', "'")
+ data[i] = data[i].replace(jaString, translatedText)
+
+ # Dialogue
+ match = re.search(dialogueRegex, data[i])
+ if match and DIALOGUE:
+ jaString = match.group(1)
+ if not jaString:
+ jaString = match.group(2)
+
+ # Pass 1
+ if not setData:
+ # Remove any textwrap
+ jaString = jaString.replace("[r]", " ")
+
+ # Remove Furigana
+ matchList = re.findall(furiganaRegex, jaString)
+ if matchList:
+ for match in matchList:
+ jaString = jaString.replace(match[0], match[1])
+
+ # Add String
+ if speaker:
+ stringList.append(f"[{speaker}]: {jaString.strip()}")
+ else:
+ stringList.append(f"{jaString.strip()}")
+
+ # Pass 2
+ else:
+ if len(stringList) > 0:
+ # Grab and Pop
+ translatedText = stringList[0]
+ stringList.pop(0)
+
+ # Remove Speaker
+ translatedText = re.sub(r"\[.*?\]:\s", "", translatedText)
+
+ # Textwrap
+ translatedText = textwrap.fill(translatedText, width=WIDTH)
+ translatedText = translatedText.replace("\n", "[r]")
+
+ # Replace Quotes
+ data[i] = data[i].replace("'", '"')
+ translatedText = translatedText.replace('"', "'")
+ data[i] = data[i].replace(jaString, translatedText)
+
+ # Next Line
+ i += 1
+
+ # EOF
+ stringListTL = []
+ choiceListTL = []
+
+ # Dialogue
+ if len(stringList) > 0:
+ # Set Progress
+ pbar.total = len(stringList)
+ pbar.refresh()
+
+ # Translate
+ response = translateGPT(
+ stringList,
+ "",
+ True,
+ )
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ stringListTL = response[0]
+
+ # Validate
+ if len(stringList) != len(stringListTL):
+ with LOCK:
+ if filename not in MISMATCH:
+ MISMATCH.append(filename)
+ stringListTL = stringList
+
+ # Choices
+ if len(choiceList) > 0:
+ # Set Progress
+ pbar.total = len(choiceList)
+ pbar.refresh()
+
+ # Translate
+ response = translateGPT(
+ choiceList,
+ "",
+ True,
+ )
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ choiceListTL = response[0]
+
+ # Validate
+ if len(choiceList) != len(choiceListTL):
+ with LOCK:
+ if filename not in MISMATCH:
+ MISMATCH.append(filename)
+ choiceListTL = choiceList
+
+ # Proceed to Pass 2
+ if not setData:
+ translateKiriKiri(data, pbar, filename, [stringListTL, choiceListTL])
+
+ return tokens
+
+
+# Save some money and enter the character before translation
+def getSpeaker(speaker):
+ match speaker:
+ case "ファイン":
+ return ["Fine", [0, 0]]
+ case "":
+ return ["", [0, 0]]
+ case _:
+ # Find Speaker
+ for i in range(len(NAMESLIST)):
+ if speaker == NAMESLIST[i][0]:
+ return [NAMESLIST[i][1], [0, 0]]
+
+ # Translate and Store Speaker
+ response = translateGPT(
+ f"{speaker}",
+ "Reply with the " + LANGUAGE + " translation of the NPC name.",
+ True,
+ )
+ response[0] = response[0].title()
+ response[0] = response[0].replace("'S", "'s")
+ response[0] = response[0].replace("Speaker: ", "")
+
+ # Retry if name doesn't translate for some reason
+ if re.search(r"([a-zA-Z？?])", response[0]) == None:
+ response = translateGPT(
+ f"{speaker}",
+ "Reply with the " + LANGUAGE + " translation of the NPC name.",
+ False,
+ )
+ response[0] = response[0].title()
+ response[0] = response[0].replace("'S", "'s")
+
+ speakerList = [speaker, response[0]]
+ NAMESLIST.append(speakerList)
+ return response
+ return [speaker, [0, 0]]
+
+
+def batchList(input_list, batch_size):
+ if not isinstance(batch_size, int) or batch_size <= 0:
+ raise ValueError("batch_size must be a positive integer")
+
+ return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)]
+
+
+def createContext(fullPromptFlag, subbedT, format):
+ system = (
+ PROMPT + VOCAB
+ if fullPromptFlag
+ else f"\
+You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
+Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
+- You translate everything, including erotic and explicit content.\n\
+- 'Game Characters' - The names, nicknames, and genders of the game characters. Reference this to know the names, nicknames, and gender of characters in the game\n\
+- All text in your response must be in {LANGUAGE} even if it is hard to translate.\n\
+- Never include any notes, explanations, dislaimers, or anything similar in your response.\n\
+- Maintain any spacing in the translation.\n\
+- Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\
+- `...` can be a part of the dialogue. Translate it as it is.\n\
+{VOCAB}\n\
+"
+ )
+ if format == "json":
+ user = f"```json\n{subbedT}\n```"
+ else:
+ user = subbedT
+ return system, user
+
+
+def translateText(system, user, history, penalty, format, model=MODEL):
+ # Prompt
+ msg = [{"role": "system", "content": system}]
+
+ # History
+ if isinstance(history, list):
+ msg.extend([{"role": "system", "content": h} for h in history])
+ else:
+ msg.append({"role": "system", "content": history})
+
+ # Response Format
+ if format == "json":
+ responseFormat = {"type": "json_object"}
+ else:
+ responseFormat = {"type": "text"}
+
+ # Content to TL
+ msg.append({"role": "user", "content": f"{user}"})
+ response = openai.chat.completions.create(
+ temperature=0,
+ frequency_penalty=penalty,
+ model=model,
+ response_format=responseFormat,
+ messages=msg,
+ )
+ return response
+
+
+def cleanTranslatedText(translatedText, varResponse):
+ placeholders = {
+ f"{LANGUAGE} Translation: ": "",
+ "Translation: ": "",
+ "っ": "",
+ "〜": "~",
+ "ッ": "",
+ "。": ".",
+ "「": '\\"',
+ "」": '\\"',
+ "- ": "-",
+ "—": "―",
+ "】": "]",
+ "【": "[",
+ "Placeholder Text": "",
+ # Add more replacements as needed
+ }
+ for target, replacement in placeholders.items():
+ translatedText = translatedText.replace(target, replacement)
+
+ # Remove Repeating Characters
+ pattern = re.compile(r"(.)\s*\1(?:\s*\1){" + str(20 - 1) + r",}")
+ translatedText = pattern.sub(lambda match: match.group(0).replace(" ", "")[:20], translatedText)
+
+ # Elongate Long Dashes (Since GPT Ignores them...)
+ translatedText = elongateCharacters(translatedText)
+ return translatedText
+
+
+def elongateCharacters(text):
+ # Define a pattern to match one character followed by one or more `ー` characters
+ # Using a positive lookbehind assertion to capture the preceding character
+ pattern = r"(?<=(.))ー+"
+
+ # Define a replacement function that elongates the captured character
+ def repl(match):
+ char = match.group(1) # The character before the ー sequence
+ count = len(match.group(0)) - 1 # Number of ー characters
+ return char * count # Replace ー sequence with the character repeated
+
+ # Use re.sub() to replace the pattern in the text
+ return re.sub(pattern, repl, text)
+
+
+def extractTranslation(translatedTextList, is_list):
+ try:
+ translatedTextList = re.sub(r'\\"+\"([^,\n}])', r'\\"\1', translatedTextList)
+ translatedTextList = re.sub(r"(? instead
-FIXTEXTWRAP = True # Overwrites textwrap
-IGNORETLTEXT = False # Ignores all translated text.
-MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
-FILENAME = None
-
-# tqdm Globals
-BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
-POSITION = 0
-LEAVE = False
-PBAR = None
-
-# Pricing - Depends on the model https://openai.com/pricing
-# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request
-# If you are getting a MISMATCH LENGTH error, lower the batch size.
-if "gpt-3.5" in MODEL:
- INPUTAPICOST = 0.002
- OUTPUTAPICOST = 0.002
- BATCHSIZE = 10
-elif "gpt-4" in MODEL:
- INPUTAPICOST = 0.0025
- OUTPUTAPICOST = 0.01
- BATCHSIZE = 20
-else:
- INPUTAPICOST = float(os.getenv("input_cost"))
- OUTPUTAPICOST = float(os.getenv("output_cost"))
- BATCHSIZE = int(os.getenv("batchsize"))
- FREQUENCY_PENALTY = float(os.getenv("frequency_penalty"))
-
-
-def handleRegex(filename, estimate):
- global ESTIMATE, TOKENS, FILENAME
- ESTIMATE = estimate
- FILENAME = filename
-
- # Translate
- start = time.time()
- translatedData = openFiles(filename)
-
- # Translate
- if not estimate:
- try:
- with open("translated/" + filename, "w", encoding="utf-8") as outFile:
- outFile.writelines(translatedData[0])
- except Exception:
- traceback.print_exc()
- return "Fail"
-
- # Print File
- end = time.time()
- tqdm.write(getResultString(translatedData, end - start, filename))
- with LOCK:
- TOKENS[0] += translatedData[1][0]
- TOKENS[1] += translatedData[1][1]
-
- # Print Total
- totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
-
- # Print any errors on maps
- if len(MISMATCH) > 0:
- return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
- else:
- return totalString
-
-
-def getResultString(translatedData, translationTime, filename):
- # File Print String
- totalTokenstring = (
- Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
- "[Output: "
- + str(translatedData[1][1])
- + "]" "[Cost: ${:,.4f}".format((translatedData[1][0] * 0.001 * INPUTAPICOST) + (translatedData[1][1] * 0.001 * OUTPUTAPICOST))
- + "]"
- )
- timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
-
- if translatedData[2] == None:
- # Success
- return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET
-
- else:
- # Fail
- try:
- raise translatedData[2]
- except Exception as e:
- traceback.print_exc()
- errorString = str(e) + Fore.RED
- return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET
-
-
-def openFiles(filename):
- with open("files/" + filename, "r", encoding="utf8") as readFile:
- translatedData = parseRegex(readFile, filename)
-
- # Delete lines marked for deletion
- finalData = []
- for line in translatedData[0]:
- if line != "\\d\n":
- finalData.append(line)
- translatedData[0] = finalData
-
- return translatedData
-
-
-def parseRegex(readFile, filename):
- global PBAR
- totalTokens = [0, 0]
-
- # Read File into data
- data = readFile.readlines()
-
- # Create Progress Bar
- with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
- pbar.desc = filename
- PBAR = pbar
-
- try:
- result = translateRegex(data, [])
- totalTokens[0] += result[0]
- totalTokens[1] += result[1]
- except Exception as e:
- traceback.print_exc()
- return [data, totalTokens, e]
- return [data, totalTokens, None]
-
-
-def translateRegex(data, translatedList):
- if translatedList:
- stringList = translatedList[0]
- choiceList = translatedList[1]
- else:
- stringList = []
- choiceList = []
- tokens = [0, 0]
- speaker = ""
- global LOCK, ESTIMATE, FILENAME, PBAR, MISMATCH
- i = 0
-
- while i < len(data):
- voice = False
- lineRegexText = r"t\s'(.*)'$"
- lineRegexSpeaker = r"n\s'(.*)'$"
- choiceRegex = r"choice:\d+\s'(.*)'"
- titleRegex = r"title\s'(.*)'$"
-
- # Title
- match = re.search(titleRegex, data[i])
- if match:
- response = translateGPT(
- match.group(1),
- f"Reply with the {LANGUAGE} translation of the chapter title",
- True,
- )
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- title = response[0]
-
- # Set
- if not translatedList:
- title = re.sub(r"(?`\n\
-- You translate everything, including erotic and explicit content.\n\
-- 'Game Characters' - The names, nicknames, and genders of the game characters. Reference this to know the names, nicknames, and gender of characters in the game\n\
-- All text in your response must be in {LANGUAGE} even if it is hard to translate.\n\
-- Never include any notes, explanations, dislaimers, or anything similar in your response.\n\
-- Maintain any spacing in the translation.\n\
-- Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\
-- `...` can be a part of the dialogue. Translate it as it is.\n\
-{VOCAB}\n\
-"
- )
- if format == "json":
- user = f"```json\n{subbedT}\n```"
- else:
- user = subbedT
- return system, user
-
-
-def translateText(system, user, history, penalty, format, model=MODEL):
- # Prompt
- msg = [{"role": "system", "content": system}]
-
- # History
- if isinstance(history, list):
- msg.extend([{"role": "system", "content": h} for h in history])
- else:
- msg.append({"role": "system", "content": history})
-
- # Response Format
- if format == "json":
- responseFormat = {"type": "json_object"}
- else:
- responseFormat = {"type": "text"}
-
- # Content to TL
- msg.append({"role": "user", "content": f"{user}"})
- response = openai.chat.completions.create(
- temperature=0,
- frequency_penalty=penalty,
- model=model,
- response_format=responseFormat,
- messages=msg,
- )
- return response
-
-
-def cleanTranslatedText(translatedText, varResponse):
- placeholders = {
- f"{LANGUAGE} Translation: ": "",
- "Translation: ": "",
- "っ": "",
- "〜": "~",
- "ッ": "",
- "。": ".",
- "「": '\\"',
- "」": '\\"',
- "- ": "-",
- "—": "―",
- "】": "]",
- "【": "[",
- "Placeholder Text": "",
- # Add more replacements as needed
- }
- for target, replacement in placeholders.items():
- translatedText = translatedText.replace(target, replacement)
-
- # Remove Repeating Characters
- pattern = re.compile(r"(.)\s*\1(?:\s*\1){" + str(20 - 1) + r",}")
- translatedText = pattern.sub(lambda match: match.group(0).replace(" ", "")[:20], translatedText)
-
- # Elongate Long Dashes (Since GPT Ignores them...)
- translatedText = elongateCharacters(translatedText)
- return translatedText
-
-
-def elongateCharacters(text):
- # Define a pattern to match one character followed by one or more `ー` characters
- # Using a positive lookbehind assertion to capture the preceding character
- pattern = r"(?<=(.))ー+"
-
- # Define a replacement function that elongates the captured character
- def repl(match):
- char = match.group(1) # The character before the ー sequence
- count = len(match.group(0)) - 1 # Number of ー characters
- return char * count # Replace ー sequence with the character repeated
-
- # Use re.sub() to replace the pattern in the text
- return re.sub(pattern, repl, text)
-
-
-def extractTranslation(translatedTextList, is_list):
- try:
- translatedTextList = re.sub(r'\\"+\"([^,\n}])', r'\\"\1', translatedTextList)
- translatedTextList = re.sub(r"(? instead
+FIXTEXTWRAP = True # Overwrites textwrap
+IGNORETLTEXT = False # Ignores all translated text.
+MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
+FILENAME = None
+
+# tqdm Globals
+BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
+POSITION = 0
+LEAVE = False
+PBAR = None
+
+# Pricing - Depends on the model https://openai.com/pricing
+# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request
+# If you are getting a MISMATCH LENGTH error, lower the batch size.
+if "gpt-3.5" in MODEL:
+ INPUTAPICOST = 0.002
+ OUTPUTAPICOST = 0.002
+ BATCHSIZE = 10
+elif "gpt-4" in MODEL:
+ INPUTAPICOST = 0.0025
+ OUTPUTAPICOST = 0.01
+ BATCHSIZE = 20
+else:
+ INPUTAPICOST = float(os.getenv("input_cost"))
+ OUTPUTAPICOST = float(os.getenv("output_cost"))
+ BATCHSIZE = int(os.getenv("batchsize"))
+ FREQUENCY_PENALTY = float(os.getenv("frequency_penalty"))
+
+
+def handleRegex(filename, estimate):
+ global ESTIMATE, TOKENS, FILENAME
+ ESTIMATE = estimate
+ FILENAME = filename
+
+ # Translate
+ start = time.time()
+ translatedData = openFiles(filename)
+
+ # Translate
+ if not estimate:
+ try:
+ with open("translated/" + filename, "w", encoding="cp932") as outFile:
+ outFile.writelines(translatedData[0])
+ except Exception:
+ traceback.print_exc()
+ return "Fail"
+
+ # Print File
+ end = time.time()
+ tqdm.write(getResultString(translatedData, end - start, filename))
+ with LOCK:
+ TOKENS[0] += translatedData[1][0]
+ TOKENS[1] += translatedData[1][1]
+
+ # Print Total
+ totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
+
+ # Print any errors on maps
+ if len(MISMATCH) > 0:
+ return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
+ else:
+ return totalString
+
+
+def getResultString(translatedData, translationTime, filename):
+ # File Print String
+ totalTokenstring = (
+ Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
+ "[Output: "
+ + str(translatedData[1][1])
+ + "]" "[Cost: ${:,.4f}".format((translatedData[1][0] * 0.001 * INPUTAPICOST) + (translatedData[1][1] * 0.001 * OUTPUTAPICOST))
+ + "]"
+ )
+ timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
+
+ if translatedData[2] == None:
+ # Success
+ return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET
+
+ else:
+ # Fail
+ try:
+ raise translatedData[2]
+ except Exception as e:
+ traceback.print_exc()
+ errorString = str(e) + Fore.RED
+ return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET
+
+
+def openFiles(filename):
+ with open("files/" + filename, "r", encoding="cp932") as readFile:
+ translatedData = parseRegex(readFile, filename)
+
+ # Delete lines marked for deletion
+ finalData = []
+ for line in translatedData[0]:
+ if line != "\\d\n":
+ finalData.append(line)
+ translatedData[0] = finalData
+
+ return translatedData
+
+
+def parseRegex(readFile, filename):
+ global PBAR
+ totalTokens = [0, 0]
+
+ # Read File into data
+ data = readFile.readlines()
+
+ # Create Progress Bar
+ with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
+ pbar.desc = filename
+ PBAR = pbar
+
+ try:
+ result = translateRegex(data, [])
+ totalTokens[0] += result[0]
+ totalTokens[1] += result[1]
+ except Exception as e:
+ traceback.print_exc()
+ return [data, totalTokens, e]
+ return [data, totalTokens, None]
+
+
+def translateRegex(data, translatedList):
+ if translatedList:
+ stringList = translatedList[0]
+ choiceList = translatedList[1]
+ else:
+ stringList = []
+ choiceList = []
+ tokens = [0, 0]
+ speaker = ""
+ global LOCK, ESTIMATE, FILENAME, PBAR, MISMATCH
+ i = 0
+
+ while i < len(data):
+ voice = False
+ lineRegexText = r"t\s'(.*)'$"
+ lineRegexSpeaker = r"n\s'(.*)'$"
+ choiceRegex = r"\$menu_item.+?,(.*?),"
+ titleRegex = r"title\s'(.*)'$"
+
+ # Title
+ match = re.search(titleRegex, data[i])
+ if match:
+ response = translateGPT(
+ match.group(1),
+ f"Reply with the {LANGUAGE} translation of the chapter title",
+ True,
+ )
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ title = response[0]
+
+ # Set
+ if not translatedList:
+ title = re.sub(r"(?`\n\
+- You translate everything, including erotic and explicit content.\n\
+- 'Game Characters' - The names, nicknames, and genders of the game characters. Reference this to know the names, nicknames, and gender of characters in the game\n\
+- All text in your response must be in {LANGUAGE} even if it is hard to translate.\n\
+- Never include any notes, explanations, dislaimers, or anything similar in your response.\n\
+- Maintain any spacing in the translation.\n\
+- Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\
+- `...` can be a part of the dialogue. Translate it as it is.\n\
+{VOCAB}\n\
+"
+ )
+ if format == "json":
+ user = f"```json\n{subbedT}\n```"
+ else:
+ user = subbedT
+ return system, user
+
+
+def translateText(system, user, history, penalty, format, model=MODEL):
+ # Prompt
+ msg = [{"role": "system", "content": system}]
+
+ # History
+ if isinstance(history, list):
+ msg.extend([{"role": "system", "content": h} for h in history])
+ else:
+ msg.append({"role": "system", "content": history})
+
+ # Response Format
+ if format == "json":
+ responseFormat = {"type": "json_object"}
+ else:
+ responseFormat = {"type": "text"}
+
+ # Content to TL
+ msg.append({"role": "user", "content": f"{user}"})
+ response = openai.chat.completions.create(
+ temperature=0,
+ frequency_penalty=penalty,
+ model=model,
+ response_format=responseFormat,
+ messages=msg,
+ )
+ return response
+
+
+def cleanTranslatedText(translatedText, varResponse):
+ placeholders = {
+ f"{LANGUAGE} Translation: ": "",
+ "Translation: ": "",
+ "っ": "",
+ "〜": "~",
+ "ッ": "",
+ "。": ".",
+ "「": '\\"',
+ "」": '\\"',
+ "- ": "-",
+ "—": "―",
+ "】": "]",
+ "【": "[",
+ "Placeholder Text": "",
+ # Add more replacements as needed
+ }
+ for target, replacement in placeholders.items():
+ translatedText = translatedText.replace(target, replacement)
+
+ # Remove Repeating Characters
+ pattern = re.compile(r"(.)\s*\1(?:\s*\1){" + str(20 - 1) + r",}")
+ translatedText = pattern.sub(lambda match: match.group(0).replace(" ", "")[:20], translatedText)
+
+ # Elongate Long Dashes (Since GPT Ignores them...)
+ translatedText = elongateCharacters(translatedText)
+ return translatedText
+
+
+def elongateCharacters(text):
+ # Define a pattern to match one character followed by one or more `ー` characters
+ # Using a positive lookbehind assertion to capture the preceding character
+ pattern = r"(?<=(.))ー+"
+
+ # Define a replacement function that elongates the captured character
+ def repl(match):
+ char = match.group(1) # The character before the ー sequence
+ count = len(match.group(0)) - 1 # Number of ー characters
+ return char * count # Replace ー sequence with the character repeated
+
+ # Use re.sub() to replace the pattern in the text
+ return re.sub(pattern, repl, text)
+
+
+def extractTranslation(translatedTextList, is_list):
+ try:
+ translatedTextList = re.sub(r'\\"+\"([^,\n}])', r'\\"\1', translatedTextList)
+ translatedTextList = re.sub(r"(? instead
-FIXTEXTWRAP = True # Overwrites textwrap
-IGNORETLTEXT = False # Ignores all translated text.
-MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
-PBAR = None
-FILENAME = None
-
-# Pricing - Depends on the model https://openai.com/pricing
-# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request
-# If you are getting a MISMATCH LENGTH error, lower the batch size.
-if "gpt-3.5" in MODEL:
- INPUTAPICOST = 0.002
- OUTPUTAPICOST = 0.002
- BATCHSIZE = 10
- FREQUENCY_PENALTY = 0.2
-elif "gpt-4" in MODEL:
- INPUTAPICOST = 0.0025
- OUTPUTAPICOST = 0.01
- BATCHSIZE = 30
- FREQUENCY_PENALTY = 0.1
-else:
- INPUTAPICOST = float(os.getenv("input_cost"))
- OUTPUTAPICOST = float(os.getenv("output_cost"))
- BATCHSIZE = int(os.getenv("batchsize"))
- FREQUENCY_PENALTY = float(os.getenv("frequency_penalty"))
-
-# tqdm Globals
-BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
-POSITION = 0
-LEAVE = False
-
-# Dialogue / Scroll / Choices (Main Codes)
-CODE401 = True
-CODE405 = True
-CODE102 = True
-
-# Optional
-CODE101 = True # Turn this one when names exist in 101
-CODE408 = False # Warning, translates comments and can inflate costs.
-
-# Variables
-CODE122 = False
-
-# Other
-CODE355655 = False
-CODE357 = False
-CODE657 = False
-CODE356 = False
-CODE320 = False
-CODE324 = False
-CODE111 = False
-CODE108 = False
-
-
-def handleMVMZ(filename, estimate):
- global ESTIMATE, TOKENS, FILENAME
- ESTIMATE = estimate
- FILENAME = filename
-
- # Translate
- start = time.time()
- translatedData = openFiles(filename)
-
- # Translate
- if not estimate:
- try:
- with open("translated/" + filename, "w", encoding="utf-8") as outFile:
- json.dump(translatedData[0], outFile, ensure_ascii=False, indent=4)
- except Exception:
- traceback.print_exc()
- return "Fail"
-
- # Print File
- end = time.time()
- tqdm.write(getResultString(translatedData, end - start, filename))
- with LOCK:
- TOKENS[0] += translatedData[1][0]
- TOKENS[1] += translatedData[1][1]
-
- # Print Total
- totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
-
- # Print any errors on maps
- if len(MISMATCH) > 0:
- return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
- else:
- return totalString
-
-
-def openFiles(filename):
- with open("files/" + filename, "r", encoding="utf-8-sig") as f:
- data = json.load(f)
-
- # Map Files
- if "Map" in filename and filename != "MapInfos.json":
- translatedData = parseMap(data, filename)
-
- # CommonEvents Files
- elif "CommonEvents" in filename:
- translatedData = parseCommonEvents(data, filename)
-
- # Actor File
- elif "Actors" in filename:
- translatedData = parseNames(data, filename, "Actors")
-
- # Armor File
- elif "Armors" in filename:
- translatedData = parseNames(data, filename, "Armors")
-
- # Weapons File
- elif "Weapons" in filename:
- translatedData = parseNames(data, filename, "Weapons")
-
- # Classes File
- elif "Classes" in filename:
- translatedData = parseNames(data, filename, "Classes")
-
- # Enemies File
- elif "Enemies" in filename:
- translatedData = parseNames(data, filename, "Enemies")
-
- # Items File
- elif "Items" in filename:
- translatedData = parseNames(data, filename, "Items")
-
- # MapInfo File
- elif "MapInfos" in filename:
- translatedData = parseNames(data, filename, "MapInfos")
-
- # Skills File
- elif "Skills" in filename:
- translatedData = parseNames(data, filename, "Skills")
-
- # Troops File
- elif "Troops" in filename:
- translatedData = parseTroops(data, filename)
-
- # States File
- elif "States" in filename:
- translatedData = parseSS(data, filename)
-
- # System File
- elif "System" in filename:
- translatedData = parseSystem(data, filename)
-
- # Scenario File
- elif "Scenario" in filename:
- translatedData = parseScenario(data, filename)
-
- else:
- raise NameError(filename + " Not Supported")
-
- return translatedData
-
-
-def getResultString(translatedData, translationTime, filename):
- # File Print String
- totalTokenstring = (
- Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
- "[Output: "
- + str(translatedData[1][1])
- + "]" "[Cost: ${:,.4f}".format((translatedData[1][0] * 0.001 * INPUTAPICOST) + (translatedData[1][1] * 0.001 * OUTPUTAPICOST))
- + "]"
- )
- timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
-
- if translatedData[2] is None:
- # Success
- return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET
- else:
- # Fail
- try:
- raise translatedData[2]
- except Exception as e:
- traceback.print_exc()
- errorString = str(e) + Fore.RED
- return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET
-
-
-def parseMap(data, filename):
- totalTokens = [0, 0]
- totalLines = 0
- events = data["events"]
- global LOCK
-
- # Translate displayName for Map files
- if "Map" in filename:
- response = translateGPT(
- data["displayName"],
- "Reply with only the " + LANGUAGE + " translation of the RPG location name",
- False,
- )
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
- data["displayName"] = response[0].replace('"', "")
-
- # Get total for progress bar
- for event in events:
- if event is not None:
- for page in event["pages"]:
- totalLines += len(page["list"])
-
- # Thread for each page in file
- with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
- pbar.desc = filename
- with ThreadPoolExecutor(max_workers=THREADS) as executor:
- for event in events:
- if event is not None:
- # This translates ID of events. (May break the game)
- if ".*")
- totalTokens[0] += response[0]
- totalTokens[1] += response[1]
- if ".*")
- totalTokens[0] += response[0]
- totalTokens[1] += response[1]
- if ".*")
- totalTokens[0] += response[0]
- totalTokens[1] += response[1]
-
- futures = [executor.submit(searchCodes, page, pbar, [], filename) for page in event["pages"] if page is not None]
- for future in as_completed(futures):
- try:
- totalTokensFuture = future.result()
- totalTokens[0] += totalTokensFuture[0]
- totalTokens[1] += totalTokensFuture[1]
- except Exception as e:
- traceback.print_exc()
- return [data, totalTokens, e]
- return [data, totalTokens, None]
-
-
-def translateNote(event, regex):
- # Regex String
- jaString = event["note"]
- match = re.findall(regex, jaString, re.DOTALL)
- if match:
- tokens = [0, 0]
- i = 0
- while i < len(match):
- initialJAString = match[i]
- # Remove any textwrap
- modifiedJAString = initialJAString.replace("\n", " ")
-
- # Translate
- response = translateGPT(
- modifiedJAString,
- "Reply with only the " + LANGUAGE + " translation.",
- False,
- )
- translatedText = response[0]
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
-
- # Textwrap
- translatedText = textwrap.fill(translatedText, width=NOTEWIDTH)
- translatedText = translatedText.replace('"', "")
- jaString = jaString.replace(initialJAString, translatedText)
- event["note"] = jaString
- i += 1
- return tokens
- return [0, 0]
-
-
-# For notes that can't have spaces.
-def translateNoteOmitSpace(event, regex):
- # Regex that only matches text inside LB.
- jaString = event["note"]
-
- match = re.findall(regex, jaString, re.DOTALL)
- if match:
- oldJAString = match[0]
- # Remove any textwrap
- jaString = re.sub(r"\n", " ", oldJAString)
-
- # Translate
- response = translateGPT(
- jaString,
- "Reply with the " + LANGUAGE + " translation of the location name.",
- False,
- )
- translatedText = response[0]
-
- translatedText = translatedText.replace('"', "")
- translatedText = translatedText.replace(" ", "_")
- event["note"] = event["note"].replace(oldJAString, translatedText)
- return response[1]
- return [0, 0]
-
-
-def parseCommonEvents(data, filename):
- totalTokens = [0, 0]
- totalLines = 0
- global LOCK
-
- # Get total for progress bar
- for page in data:
- if page is not None:
- totalLines += len(page["list"])
-
- with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
- pbar.desc = filename
- with ThreadPoolExecutor(max_workers=THREADS) as executor:
- futures = [executor.submit(searchCodes, page, pbar, [], filename) for page in data if page is not None]
- for future in as_completed(futures):
- try:
- totalTokensFuture = future.result()
- totalTokens[0] += totalTokensFuture[0]
- totalTokens[1] += totalTokensFuture[1]
- except Exception as e:
- traceback.print_exc()
- return [data, totalTokens, e]
- return [data, totalTokens, None]
-
-
-def parseTroops(data, filename):
- totalTokens = [0, 0]
- totalLines = 0
- global LOCK
-
- # Get total for progress bar
- for troop in data:
- if troop is not None:
- for page in troop["pages"]:
- totalLines += len(page["list"]) + 1 # The +1 is because each page has a name.
-
- with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
- pbar.desc = filename
- for troop in data:
- if troop is not None:
- with ThreadPoolExecutor(max_workers=THREADS) as executor:
- futures = [executor.submit(searchCodes, page, pbar, [], filename) for page in troop["pages"] if page is not None]
- for future in as_completed(futures):
- try:
- totalTokensFuture = future.result()
- totalTokens[0] += totalTokensFuture[0]
- totalTokens[1] += totalTokensFuture[1]
- except Exception as e:
- traceback.print_exc()
- return [data, totalTokens, e]
- return [data, totalTokens, None]
-
-
-def parseNames(data, filename, context):
- totalTokens = [0, 0]
- totalLines = 0
- totalLines += len(data)
-
- with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
- pbar.desc = filename
- try:
- result = searchNames(data, pbar, context)
- totalTokens[0] += result[0]
- totalTokens[1] += result[1]
- except Exception as e:
- traceback.print_exc()
- return [data, totalTokens, e]
- return [data, totalTokens, None]
-
-
-def parseSS(data, filename):
- totalTokens = [0, 0]
- totalLines = 0
- totalLines += len(data)
-
- with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
- pbar.desc = filename
- for ss in data:
- if ss is not None:
- try:
- result = searchSS(ss, pbar)
- totalTokens[0] += result[0]
- totalTokens[1] += result[1]
- except Exception as e:
- traceback.print_exc()
- return [data, totalTokens, e]
- return [data, totalTokens, None]
-
-
-def parseSystem(data, filename):
- totalTokens = [0, 0]
- totalLines = 0
-
- # Calculate Total Lines
- for term in data["terms"]:
- termList = data["terms"][term]
- totalLines += len(termList)
- totalLines += len(data["gameTitle"])
- totalLines += len(data["terms"]["messages"])
- totalLines += len(data["variables"])
- totalLines += len(data["equipTypes"])
- totalLines += len(data["armorTypes"])
- totalLines += len(data["skillTypes"])
-
- with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
- pbar.desc = filename
- try:
- result = searchSystem(data, pbar)
- totalTokens[0] += result[0]
- totalTokens[1] += result[1]
- except Exception as e:
- traceback.print_exc()
- return [data, totalTokens, e]
- return [data, totalTokens, None]
-
-
-def parseScenario(data, filename):
- totalTokens = [0, 0]
- totalLines = 0
- global LOCK
-
- # Get total for progress bar
- for page in data.items():
- totalLines += len(page[1])
-
- with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
- pbar.desc = filename
- with ThreadPoolExecutor(max_workers=THREADS) as executor:
- futures = [executor.submit(searchCodes, page[1], pbar, [], filename) for page in data.items() if page[1] is not None]
- for future in as_completed(futures):
- try:
- totalTokensFuture = future.result()
- totalTokens[0] += totalTokensFuture[0]
- totalTokens[1] += totalTokensFuture[1]
- except Exception as e:
- traceback.print_exc()
- return [data, totalTokens, e]
- return [data, totalTokens, None]
-
-
-def searchNames(data, pbar, context):
- totalTokens = [0, 0]
- nameList = []
- profileList = []
- nicknameList = []
- descriptionList = []
- noteList = []
- i = 0 # Counter
- j = 0 # Counter 2
- filling = False
- mismatch = False
- batchFull = False
-
- # Set the context of what we are translating
- if "Actors" in context:
- newContext = "Reply with only the " + LANGUAGE + " translation of the NPC name"
- if "Armors" in context:
- newContext = "Reply with only the " + LANGUAGE + " translation of the RPG equipment name"
- if "Classes" in context:
- newContext = "Reply with only the " + LANGUAGE + " translation of the RPG class name"
- if "MapInfos" in context:
- newContext = "Reply with only the " + LANGUAGE + " translation of the location name"
- if "Enemies" in context:
- newContext = "Reply with only the " + LANGUAGE + " translation of the enemy NPC name"
- if "Weapons" in context:
- newContext = "Reply with only the " + LANGUAGE + " translation of the RPG weapon name"
- if "Items" in context:
- newContext = "Reply with only the " + LANGUAGE + " translation of the RPG item name"
- if "Skills" in context:
- newContext = "Reply with only the " + LANGUAGE + " translation of the RPG skill name"
-
- # Names
- with open("translations.txt", "a", encoding="utf-8") as file:
- file.write(f"\n#{context}\n")
- while i < len(data) or filling == True:
- if i < len(data):
- # Empty Data
- if data[i] is None or data[i]["name"] == "":
- i += 1
-
- continue
-
- # Filling up Batch
- filling = True
- if context in "Actors":
- if len(nameList) < BATCHSIZE:
- if data[i]["name"] != "":
- nameList.append(data[i]["name"])
- if data[i]["nickname"] != "":
- nicknameList.append(data[i]["nickname"])
- if data[i]["profile"] != "":
- profileList.append(data[i]["profile"].replace("\n", " "))
-
- # Notes
- if "")
- totalTokens[0] += tokensResponse[0]
- totalTokens[1] += tokensResponse[1]
- if "PE拡張" in data[i]["note"]:
- tokensResponse = translateNote(data[i], r"")
- totalTokens[0] += tokensResponse[0]
- totalTokens[1] += tokensResponse[1]
- i += 1
- else:
- batchFull = True
- if context in ["Armors", "Weapons", "Items"]:
- if len(nameList) < BATCHSIZE:
- nameList.append(data[i]["name"])
- if "description" in data[i] and data[i]["description"] != "":
- descriptionList.append(data[i]["description"].replace("\n", " "))
- if "")
- totalTokens[0] += tokensResponse[0]
- totalTokens[1] += tokensResponse[1]
- if "")
- totalTokens[0] += tokensResponse[0]
- totalTokens[1] += tokensResponse[1]
- if "")
- totalTokens[0] += tokensResponse[0]
- totalTokens[1] += tokensResponse[1]
- if "")
- totalTokens[0] += tokensResponse[0]
- totalTokens[1] += tokensResponse[1]
- if "")
- totalTokens[0] += tokensResponse[0]
- totalTokens[1] += tokensResponse[1]
- if "")
- totalTokens[0] += tokensResponse[0]
- totalTokens[1] += tokensResponse[1]
- if "")
- totalTokens[0] += tokensResponse[0]
- totalTokens[1] += tokensResponse[1]
- if "Switch Shop Description" in data[i]["note"]:
- tokensResponse = translateNote(data[i], r"\n(.*)\n")
- totalTokens[0] += tokensResponse[0]
- totalTokens[1] += tokensResponse[1]
- if "")
- totalTokens[0] += tokensResponse[0]
- totalTokens[1] += tokensResponse[1]
-
- i += 1
- else:
- batchFull = True
- if context in ["Skills"]:
- if len(nameList) < BATCHSIZE:
- nameList.append(data[i]["name"])
- if "description" in data[i] and data[i]["description"]:
- descriptionList.append(data[i]["description"].replace("\n", " "))
-
- # Messages
- number = 1
- while number < 5:
- if f"message{number}" in data[i] and data[i][f"message{number}"]:
- if data[i][f"message{number}"][0] in ["は", "を", "の", "に", "が"]:
- msgResponse = translateGPT(
- "Taro" + data[i][f"message{number}"],
- "reply with only the gender neutral "
- + LANGUAGE
- + " translation of the action log. Always start the sentence with Taro. For example, Translate 'Taroを倒した！' as 'Taro was defeated!'",
- False,
- )
- data[i][f"message{number}"] = msgResponse[0].replace("Taro", "")
- totalTokens[0] += msgResponse[1][0]
- totalTokens[1] += msgResponse[1][1]
- number += 1
-
- else:
- msgResponse = translateGPT(
- data[i][f"message{number}"],
- "reply with only the gender neutral " + LANGUAGE + " translation",
- False,
- )
- data[i][f"message{number}"] = msgResponse[0]
- totalTokens[0] += msgResponse[1][0]
- totalTokens[1] += msgResponse[1][1]
- number += 1
- else:
- number += 1
-
- i += 1
- else:
- batchFull = True
- if context in ["Enemies", "Classes", "MapInfos"]:
- if len(nameList) < BATCHSIZE:
- nameList.append(data[i]["name"])
-
- # Notes
- if "note" in data[i]:
- if "")
- totalTokens[0] += tokensResponse[0]
- totalTokens[1] += tokensResponse[1]
- if "")
- totalTokens[0] += tokensResponse[0]
- totalTokens[1] += tokensResponse[1]
- i += 1
- else:
- batchFull = True
-
- # Batch Full
- if batchFull == True or i >= len(data):
- k = j # Original Index
- if context in "Actors":
- # Name
- response = translateGPT(nameList, newContext, True)
- translatedNameBatch = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Nickname
- if nicknameList:
- response = translateGPT(nicknameList, newContext, True)
- translatedNicknameBatch = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Profile
- if profileList:
- response = translateGPT(profileList, "", True)
- translatedProfileBatch = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Set Data
- if len(nameList) == len(translatedNameBatch):
- j = k
- while j < i:
- # Empty Data
- if data[j] is None or data[j]["name"] == "":
- j += 1
- continue
- else:
- # Get Text
- if data[j]["name"] != "":
- with open("translations.txt", "a", encoding="utf-8") as file:
- file.write(f'{data[j]["name"]} ({translatedNameBatch[0]})\n')
- data[j]["name"] = translatedNameBatch[0]
- translatedNameBatch.pop(0)
- if data[j]["nickname"] != "":
- data[j]["nickname"] = translatedNicknameBatch[0]
- translatedNicknameBatch.pop(0)
- if data[j]["profile"] != "":
- data[j]["profile"] = textwrap.fill(translatedProfileBatch[0], LISTWIDTH)
- translatedProfileBatch.pop(0)
-
- # If Batch is empty. Move on.
- if len(translatedNameBatch) == 0:
- nameList.clear()
- filling = False
- j += 1
- else:
- mismatch = True
-
- if context in ["Armors", "Weapons", "Items", "Skills"]:
- # Name
- response = translateGPT(nameList, newContext, True)
- translatedNameBatch = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Description
- if descriptionList:
- response = translateGPT(
- descriptionList,
- f"Reply with only the {LANGUAGE} translation of the text.",
- True,
- )
- translatedDescriptionBatch = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Set Data
- if len(nameList) == len(translatedNameBatch):
- j = k
- with open("translations.txt", "a", encoding="utf-8") as file:
- while j < i:
- # Empty Data
- if data[j] is None or data[j]["name"] == "":
- j += 1
- continue
- else:
- # Get Text
- file.write(f'{data[j]['name']} ({translatedNameBatch[0]})\n')
- data[j]["name"] = translatedNameBatch[0]
- translatedNameBatch.pop(0)
- if "description" in data[j] and data[j]["description"] != "":
- data[j]["description"] = textwrap.fill(translatedDescriptionBatch[0], LISTWIDTH)
- translatedDescriptionBatch.pop(0)
-
- # If Batch is empty. Move on.
- if len(translatedNameBatch) == 0:
- nameList.clear()
- descriptionList.clear()
- batchFull = False
- filling = False
- j += 1
- else:
- mismatch = True
- if context in ["Enemies", "Classes", "MapInfos"]:
- response = translateGPT(nameList, newContext, True)
- translatedNameBatch = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Set Data
- if len(nameList) == len(translatedNameBatch):
- j = k
- while j < i:
- # Empty Data
- if data[j] is None or data[j]["name"] == "":
- j += 1
- continue
- else:
- with open("translations.txt", "a", encoding="utf-8") as file:
- file.write(f'{data[j]["name"]} ({translatedNameBatch[0]})\n')
- # Get Text
- data[j]["name"] = translatedNameBatch[0]
- translatedNameBatch.pop(0)
-
- # If Batch is empty. Move on.
- if len(translatedNameBatch) == 0:
- nameList.clear()
- batchFull = False
- filling = False
- j += 1
- else:
- mismatch = True
-
- # Mismatch
- if mismatch == True:
- MISMATCH.append(nameList)
- nameList.clear()
- profileList.clear()
- descriptionList.clear()
- filling = False
- mismatch = False
-
- i += 1
-
- return totalTokens
-
-
-def searchCodes(page, pbar, jobList, filename):
- if len(jobList) > 0:
- list401 = jobList[0]
- list122 = jobList[1]
- list355655 = jobList[2]
- list108 = jobList[3]
- list356 = jobList[4]
- list357 = jobList[5]
- setData = True
- else:
- list401 = []
- list122 = []
- list355655 = []
- list108 = []
- list356 = []
- list357 = []
- setData = False
- textHistory = []
- match = []
- totalTokens = [0, 0]
- translatedText = ""
- speaker = ""
- speakerID = None
- syncIndex = 0
- CLFlag = False
- maxHistory = MAXHISTORY
- VNameValue = None
- speakerWindow = FIRSTLINESPEAKERS
- global LOCK
- global NAMESLIST
- global MISMATCH
- global PBAR
- with LOCK:
- PBAR = pbar
-
- # Begin Parsing File
- try:
- # Normal Format
- if "list" in page:
- codeList = page["list"]
-
- # Special Format (Scenario)
- else:
- codeList = page
-
- # Iterate through page
- i = 0
- while i < len(codeList):
- with LOCK:
- # syncIndex will keep i in sync when it gets modified
- if syncIndex > i:
- i = syncIndex
- if len(codeList) <= i:
- break
-
- # Declare Varss
- currentGroup = []
- nametag = ""
-
- ## Event Code: 401 Show Text
- if "code" in codeList[i] and codeList[i]["code"] in [401, 405, -1] and (CODE401 or CODE405):
- # Save Code and starting index (j)
- code = codeList[i]["code"]
- j = i
- endtag = ""
-
- # Grab String
- if len(codeList[i]["parameters"]) > 0:
- jaString = codeList[i]["parameters"][0]
- oldjaString = jaString
- else:
- codeList[i]["code"] = -1
- i += 1
- continue
-
- # Validate Japanese Text
- if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+", jaString) and IGNORETLTEXT:
- i += 1
- continue
-
- # # For Retarded Devs
- # retardRegex = r'([\\]+[nN]\[[\\]+V\[\d*?\]\])'
- # match = re.search(retardRegex, jaString)
- # if match:
- # if VNameValue == 1:
- # jaString = re.sub(retardRegex, 'リッカ', jaString)
- # if VNameValue == 2:
- # jaString = re.sub(retardRegex, 'ミミ', jaString)
- # if VNameValue == 3:
- # jaString = re.sub(retardRegex, 'ヒトミ', jaString)
- # if VNameValue == 4:
- # jaString = re.sub(retardRegex, 'Taro', jaString)
- # if VNameValue == 5:
- # jaString = re.sub(retardRegex, '富士見', jaString)
-
- # Speaker Check
- speakerList = []
-
- # m and z Codes
- match = re.search(r"(.*?)[\\]+m\[\d+?\][\\]+z\[\d+?\]", jaString)
- if match:
- speakerList.append(match.group(1))
- if "\\c" in speakerList[0]:
- speakerList = re.findall(
- r"^[\\]+[cC]\[[\d]+\]【?(.+?)】?[\\]+[Cc]\[[\d]\]\\?\\?$",
- speakerList[0],
- )
-
- # Brackets
- if len(speakerList) == 0:
- speakerList = re.findall(r"^【(.*?)】$", jaString)
-
- # Colors
- if len(speakerList) == 0:
- speakerList = re.findall(
- r"^[\\]+[cC]\[[\d]+\]【?(.+?)】?[\\]+[Cc]\[[\d]\]\\?\\?$",
- jaString,
- )
-
- # First Line Speakers
- if len(speakerList) == 0 and FIRSTLINESPEAKERS is True:
- # Remove any RPGMaker Code at start
- ffMatch = re.search(
- r"^(\s*[\\]+[aAbBdDeEfFgGhHjJlLmMoOpPqQrRsStTuUwWxXyYzZ]+\[[\w\d\[\]\\]+\])",
- jaString,
- )
- if ffMatch != None:
- jaString = jaString.replace(ffMatch.group(0), "")
- nametag += ffMatch.group(0)
-
- # Test Speaker
- if (
- len(jaString) < 40
- and "code" in codeList[i + 1]
- and codeList[i + 1]["code"] in [401, 405, -1]
- and len(codeList[i + 1]["parameters"]) > 0
- and len(codeList[i + 1]["parameters"][0]) > 0
- ):
- if codeList[i + 1]["parameters"] != "" and codeList[i + 1]["parameters"][0].strip()[0] in [
- "「",
- '"',
- "(",
- "（",
- "*",
- "[",
- ]:
- speakerList = re.findall(r".+", jaString)
-
- if len(speakerList) != 0 and codeList[i + 1]["code"] in [401, 405, -1]:
- # Get Speaker
- response = getSpeaker(speakerList[0])
- speaker = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Set Data
- codeList[i]["parameters"][0] = nametag + jaString.replace(speakerList[0], speaker)
- nametag = ""
-
- # Iterate to next string
- i += 1
- j = i
- while codeList[i]["code"] in [-1]:
- i += 1
- j = i
- jaString = codeList[i]["parameters"][0]
-
- # Using this to keep track of 401's in a row.
- currentGroup.append(jaString)
-
- # Join Up 401's into single string
- if len(codeList) > i + 1:
- while codeList[i + 1]["code"] in [401, 405, -1]:
- if setData == True:
- codeList[i]["parameters"] = []
- codeList[i]["code"] = -1
- i += 1
- j = i
-
- # Only add if not empty
- if len(codeList[i]["parameters"]) > 0:
- jaString = codeList[i]["parameters"][0]
- currentGroup.append(jaString)
-
- # Make sure not the end of the list.
- if len(codeList) <= i + 1:
- break
-
- # Format String
- if len(currentGroup) > 0:
- finalJAString = "\n".join(currentGroup)
- oldjaString = finalJAString
-
- # Check if Empty
- if finalJAString == "":
- i += 1
- continue
-
- # Set Back
- if setData == True:
- codeList[i]["parameters"] = [finalJAString]
-
- ### \\n
- nCase = None
- regex = r"([\\]+[kKnN][wWcCrRrEe]?[\[<](.*?)[>\]])"
- match = re.search(regex, finalJAString)
-
- # Set Name
- if match:
- nametag = match.group(1)
- speaker = match.group(2)
-
- # Translate Speaker
- response = getSpeaker(speaker)
- tledSpeaker = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Set Nametag and Remove from Final String
- finalJAString = finalJAString.replace(nametag, "")
- nametag = nametag.replace(speaker, tledSpeaker)
- speaker = tledSpeaker
-
- # Remove Extra Stuff bad for translation.
- finalJAString = finalJAString.replace("ﾞ", "")
- finalJAString = finalJAString.replace("―", "-")
- finalJAString = finalJAString.replace("…", "...")
- finalJAString = finalJAString.replace("。", ".")
- finalJAString = re.sub(r"(\.{3}\.+)", "...", finalJAString)
- finalJAString = finalJAString.replace(" ", "")
- finalJAString = finalJAString.replace("「", '"')
- finalJAString = finalJAString.replace("」", '"')
-
- ### Remove format codes
- # Furigana
- rcodeMatch = re.findall(r"([\\]+[r][b]?\[.*?,(.*?)\])", finalJAString)
- if len(rcodeMatch) > 0:
- for match in rcodeMatch:
- finalJAString = finalJAString.replace(match[0], match[1])
-
- # Formatting
- formatMatch = re.findall(r"[\\]+[!><.|#^{}]", finalJAString)
- if len(formatMatch) > 0:
- for match in formatMatch:
- finalJAString = finalJAString.replace(match, "")
-
- # Remove any RPGMaker Code at start
- ffMatch = re.search(
- r"^(\s*[\\]+[aAbBdDeEfFgGhHjJlLmMoOpPqQrRsStTuUwWxXyYzZ]+\[[\w\d\[\]\\]+?\])",
- finalJAString,
- )
- if ffMatch != None:
- finalJAString = finalJAString.replace(ffMatch.group(1), "")
- nametag += ffMatch.group(1)
-
- # Remove _ABL Codes
- ffMatch = re.search(r"^(_ABL).*", finalJAString)
- if ffMatch != None:
- finalJAString = finalJAString.replace(ffMatch.group(1), "")
- nametag += ffMatch.group(1)
-
- # Center Lines
- if "\\CL" in finalJAString or "\\ac" in finalJAString:
- finalJAString = finalJAString.replace("\\CL ", "")
- finalJAString = finalJAString.replace("\\CL", "")
- finalJAString = finalJAString.replace("\\ac ", "")
- finalJAString = finalJAString.replace("\\ac", "")
- CLFlag = True
-
- # 1st Passthrough (Grabbing Data)
- if setData == False:
- # Remove Textwrap
- if FIXTEXTWRAP:
- finalJAString = finalJAString.replace("\n", " ")
- if "\\px[200]" in finalJAString:
- finalJAString = finalJAString.replace("\\px[200]", "")
-
- # Append
- if finalJAString != "":
- if speaker == "" and finalJAString != "":
- list401.append(finalJAString)
- elif finalJAString != "":
- list401.append(f"[{speaker}]: {finalJAString}")
- else:
- list401.append(speaker)
- speaker = ""
- match = []
- nametag = ""
- currentGroup = []
- syncIndex = i + 1
-
- # Keep textHistory list at length maxHistory
- textHistory.append('"' + finalJAString + '"')
- if len(textHistory) > maxHistory:
- textHistory.pop(0)
-
- # 2nd Passthrough (Setting Data)
- else:
- # Grab Translated String
- if len(list401) > 0:
- translatedText = list401[0]
-
- # Remove speaker
- if speaker != "":
- matchSpeakerList = re.findall(r"^\[?(.+?)\]?\s?[|:]\s?", translatedText)
- if len(matchSpeakerList) > 0:
- newSpeaker = matchSpeakerList[0]
- nametag = nametag.replace(speaker, newSpeaker)
- translatedText = re.sub(r"^\[?(.+?)\]?\s?[|:]\s?", "", translatedText)
-
- # Fix '- '
- translatedText = translatedText.replace("- ", "-")
-
- # Textwrap
- if FIXTEXTWRAP is True:
- finalJAString = re.sub(r"\n", " ", finalJAString)
- finalJAString = finalJAString.replace("
", " ")
-
- if FIXTEXTWRAP is True and "_ABL" in nametag:
- translatedText = textwrap.fill(translatedText, width=100)
- elif FIXTEXTWRAP is True:
- translatedText = textwrap.fill(translatedText, width=WIDTH)
-
- # BR Flag
- if BRFLAG is True:
- translatedText = translatedText.replace("\n", "
")
-
- # px
- if "\\px[200]" in nametag:
- translatedText = translatedText.replace("\\px[200]", "")
- translatedText = translatedText.replace("\n", "\n\\px[200]")
-
- ### Add Var Strings
- # CL Flag
- if CLFlag:
- translatedText = "\\ac " + translatedText
- translatedText = translatedText.replace("\n", "\n\\ac ")
- translatedText = re.sub(r"[\\]+?ac\s+", r"\\ac ", translatedText)
- CLFlag = False
-
- # Nametag
- if nCase == 0:
- translatedText = translatedText + nametag
- else:
- translatedText = nametag + translatedText
- nametag = ""
-
- # Endtag
- if endtag != "":
- translatedText = translatedText + endtag
- endtag = ""
-
- # Set Data
- codeList[j]["parameters"] = [translatedText]
- codeList[j]["code"] = code
- speaker = ""
- match = []
- currentGroup = []
- syncIndex = i + 1
- list401.pop(0)
-
- ## Event Code: 122 [Set Variables]
- if "code" in codeList[i] and codeList[i]["code"] == 122 and CODE122 is True:
- # This is going to be the var being set. (IMPORTANT)
- if codeList[i]["parameters"][0] not in list(range(42, 45)):
- i += 1
- continue
-
- jaString = codeList[i]["parameters"][4]
-
- # # For Retarded Devs
- # VNameValue = jaString
- # i += 1
- # continue
-
- # Definitely don't want to mess with files
- # if 'gameV' in jaString or '_' in jaString:
- # i += 1
- # continue
-
- # Validate String
- if not isinstance(jaString, str):
- i += 1
- continue
-
- # Set String
- matchedText = None
- if len(re.findall(r"([\'\"\`])", jaString)) >= 2:
- matchedText = re.search(r"[\'\"\`](.*)[\'\"\`]", jaString)
- # else:
- # matchedText = re.search(r'(.*)', jaString)
-
- # Last Check
- if matchedText.group(1) != None and matchedText.group(1) != " ":
- # Remove Textwrap
- finalJAString = matchedText.group(1).replace("\\n", " ")
-
- # Pass 1
- if setData == False:
- if finalJAString != "":
- list122.append(finalJAString)
-
- # Pass 2
- else:
- if len(list122) > 0:
- # Grab and Replace
- translatedText = list122[0]
- translatedText = jaString.replace(jaString, translatedText)
-
- # Remove characters that may break scripts
- charList = ['"', "\\n"]
- for char in charList:
- translatedText = translatedText.replace(char, "")
-
- # Textwrap
- translatedText = textwrap.fill(translatedText, width=WIDTH)
- translatedText = translatedText.replace("\n", "\\n")
-
- # Set
- codeList[i]["parameters"][4] = f"`{translatedText}`"
- list122.pop(0)
-
- ## Event Code: 357 [Picture Text] [Optional]
- if "code" in codeList[i] and codeList[i]["code"] == 357 and CODE357 is True:
- headerString = codeList[i]["parameters"][0]
-
- if headerString == "LL_GalgeChoiceWindow":
- ### Message Text First
- jaString = codeList[i]["parameters"][3]["messageText"]
-
- # Remove any textwrap & TL
- jaString = re.sub(r"\n", " ", jaString)
- response = translateGPT(jaString, "", False)
- translatedText = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Textwrap & Set
- translatedText = textwrap.fill(translatedText, width=WIDTH)
- codeList[i]["parameters"][3]["messageText"] = translatedText
-
- ### Choices
- jaString = codeList[i]["parameters"][3]["choices"]
- matchList = re.findall(r'"label[\\]*":[\\]*"(.*?)[\\]', jaString)
- if matchList != None:
- # Translate
- question = codeList[i]["parameters"][3]["messageText"]
- response = translateGPT(
- matchList,
- f"Previous text for context: {question}\n\nThis will be a dialogue option",
- True,
- )
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
- translatedText = jaString
-
- # Replace Strings
- for j in range(len(matchList)):
- translatedText = translatedText.replace(matchList[j], response[0][j])
-
- # Set Data
- codeList[i]["parameters"][3]["choices"] = translatedText
-
- if "SoR_GabWindow" in headerString:
- argVar = "arg1"
- ### Message Text First
- if argVar in codeList[i]["parameters"][3]:
- jaString = codeList[i]["parameters"][3][argVar]
-
- # If there isn't any Japanese in the text just skip
- if not re.search(
- r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+",
- jaString,
- ):
- i += 1
- continue
-
- # Remove any textwrap & TL
- jaString = re.sub(r"\n", " ", jaString)
- response = translateGPT(jaString, "", False)
- translatedText = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Textwrap & Set
- translatedText = textwrap.fill(translatedText, width=WIDTH)
- codeList[i]["parameters"][3][argVar] = translatedText
- pbar.update(1)
-
- if "TorigoyaMZ_NotifyMessage" in headerString:
- argVar = "message"
- ### Message Text First
- if argVar in codeList[i]["parameters"][3]:
- jaString = codeList[i]["parameters"][3][argVar]
-
- # If there isn't any Japanese in the text just skip
- if not re.search(
- r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+",
- jaString,
- ):
- i += 1
- continue
-
- # Remove any textwrap & TL
- jaString = re.sub(r"\n", " ", jaString)
- response = translateGPT(jaString, "", False)
- translatedText = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Textwrap & Set
- translatedText = textwrap.fill(translatedText, width=WIDTH)
- codeList[i]["parameters"][3][argVar] = translatedText
- pbar.update(1)
-
- if "_TMLogWindowMZ" in headerString:
- argVar = "text"
- ### Message Text First
- if argVar in codeList[i]["parameters"][3]:
- jaString = codeList[i]["parameters"][3][argVar]
-
- # If there isn't any Japanese in the text just skip
- # if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+', jaString):
- # i += 1
- # continue
-
- # Remove any textwrap & TL
- jaString = re.sub(r"\n", " ", jaString)
- response = translateGPT(jaString, "", False)
- translatedText = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Textwrap & Set
- translatedText = textwrap.fill(translatedText, width=WIDTH)
- codeList[i]["parameters"][3][argVar] = translatedText
- pbar.update(1)
-
- if "DestinationWindow" in headerString:
- argVar = "destination"
- ### Message Text First
- if argVar in codeList[i]["parameters"][3]:
- jaString = codeList[i]["parameters"][3][argVar]
-
- # If there isn't any Japanese in the text just skip
- # if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+', jaString):
- # i += 1
- # continue
-
- # Remove any textwrap & TL
- jaString = re.sub(r"\n", " ", jaString)
- response = translateGPT(jaString, "", False)
- translatedText = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Textwrap & Set
- translatedText = textwrap.fill(translatedText, width=WIDTH)
- codeList[i]["parameters"][3][argVar] = translatedText
- pbar.update(1)
-
- if "MNKR_CommonPopupCoreMZ" in headerString:
- argVar = "text"
- ### Message Text First
- if argVar in codeList[i]["parameters"][3]:
- jaString = codeList[i]["parameters"][3][argVar]
-
- # If there isn't any Japanese in the text just skip
- # if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+', jaString):
- # i += 1
- # continue
-
- # Remove any textwrap & TL
- jaString = re.sub(r"\n", " ", jaString)
- response = translateGPT(jaString, "", False)
- translatedText = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Textwrap & Set
- translatedText = textwrap.fill(translatedText, width=WIDTH)
- codeList[i]["parameters"][3][argVar] = translatedText
- pbar.update(1)
-
- if "TextPicture" in headerString or "BalloonInBattle" in headerString:
- argVar = "text"
- font = None
- ### Message Text First
- if argVar in codeList[i]["parameters"][3]:
- acExist = False
- jaString = codeList[i]["parameters"][3][argVar]
-
- # Check ac
- if "\\ac" in jaString:
- acExist = True
- else:
- acExist = False
-
- # If there isn't any Japanese in the text just skip
- # if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+', jaString):
- # i += 1
- # continue
-
- # Remove any textwrap & TL
- jaString = re.sub(r"\n", " ", jaString)
- if acExist:
- jaString = jaString.replace("\\ac ", " ")
- jaString = jaString.replace("\\ac", "")
-
- # Pass 1
- if setData == False:
- list357.append(jaString)
-
- # Pass 2
- else:
- if len(list357) > 0:
- # Grab and Replace
- translatedText = list357[0]
- translatedText = jaString.replace(jaString, translatedText)
-
- # Remove characters that may break scripts
- charList = ['"', "\\n"]
- for char in charList:
- translatedText = translatedText.replace(char, "")
-
- # Textwrap
- translatedText = textwrap.fill(translatedText, WIDTH)
- translatedText = translatedText.replace("- ", "-")
-
- # Center Text
- if acExist:
- translatedText = f'\\ac {translatedText.replace('\n', '\n\\ac ')}'
-
- # Check and Set Font
- if "fontSize" in codeList[i]["parameters"][3]:
- if font:
- codeList[i]["parameters"][3]["fontSize"] = font
-
- # Set
- codeList[i]["parameters"][3][argVar] = translatedText
- list357.pop(0)
-
- ## Event Code: 657 [Picture Text] [Optional]
- if "code" in codeList[i] and codeList[i]["code"] == 657 and CODE657 is True:
- if "text" in codeList[i]["parameters"][0]:
- jaString = codeList[i]["parameters"][0]
- if not isinstance(jaString, str):
- i += 1
- continue
-
- # Definitely don't want to mess with files
- if "_" in jaString:
- i += 1
- continue
-
- # If there isn't any Japanese in the text just skip
- if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+", jaString):
- i += 1
- continue
-
- # Remove outside text
- startString = re.search(r"^[^一-龠ぁ-ゔァ-ヴー\<\>【】\\]+", jaString)
- jaString = re.sub(r"^[^一-龠ぁ-ゔァ-ヴー\<\>【】\\]+", "", jaString)
- endString = re.search(r"[^一-龠ぁ-ゔァ-ヴー\<\>【】。！？\\]+$", jaString)
- jaString = re.sub(r"[^一-龠ぁ-ゔァ-ヴー\<\>【】。！？\\]+$", "", jaString)
- if startString is None:
- startString = ""
- else:
- startString = startString.group()
- if endString is None:
- endString = ""
- else:
- endString = endString.group()
-
- # Remove any textwrap
- jaString = re.sub(r"\n", " ", jaString)
-
- # Translate
- response = translateGPT(jaString, "", True)
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
- translatedText = response[0]
-
- # Remove characters that may break scripts
- charList = [".", '"', "'"]
- for char in charList:
- translatedText = translatedText.replace(char, "")
-
- # Textwrap
- translatedText = textwrap.fill(translatedText, width=WIDTH)
- translatedText = startString + translatedText + endString
-
- # Set Data
- codeList[i]["parameters"][0] = translatedText
-
- ## Event Code: 101 [Name] [Optional]
- if "code" in codeList[i] and codeList[i]["code"] == 101 and CODE101 is True:
- # Check Window Type (Certain games switch between 1st line speakers and none)
- if FIRSTLINESPEAKERS:
- if codeList[i]["parameters"][2] == 0:
- speakerWindow = True
- else:
- speakerWindow = False
-
- else:
- isVar = False
-
- # Grab String
- jaString = ""
- if len(codeList[i]["parameters"]) > 4:
- jaString = codeList[i]["parameters"][4]
- # Check for Var
- elif len(codeList[i]["parameters"]) > 0:
- jaString = codeList[i]["parameters"][0]
- isVar = True
- if not isinstance(jaString, str):
- i += 1
- continue
-
- # Force Speaker using var
- if "\\ap[1左]" in jaString.lower() or "\\ap[1右]" in jaString.lower():
- speaker = "Cecily"
- i += 1
- continue
- elif "\\ap[2左]" in jaString.lower() or "\\ap[2右]" in jaString.lower():
- speaker = "Amelia"
- i += 1
- continue
- elif "\\ap[3左]" in jaString.lower() or "\\ap[3右]" in jaString.lower():
- speaker = "Henry"
- i += 1
- continue
- elif "\\ap[4左]" in jaString.lower() or "\\ap[4右]" in jaString.lower():
- speaker = "Oswald"
- i += 1
- continue
- elif "\\ap" in jaString:
- speaker = re.search(r"[\\]+AP\[(.*?)\]", jaString).group(1)
- i += 1
- continue
-
- # Get Speaker
- if "\\" not in jaString:
- response = getSpeaker(jaString)
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
- speaker = response[0]
-
- # Validate Speaker is not empty
- if len(speaker) > 0:
- if isVar == False:
- codeList[i]["parameters"][4] = speaker
- i += 1
- continue
- else:
- codeList[i]["parameters"][0] = speaker
- isVar = False
- i += 1
- continue
- else:
- speaker = ""
-
- ## Event Code: 355 or 655 Scripts [Optional]
- if "code" in codeList[i] and (codeList[i]["code"] == 355 or codeList[i]["code"] == 655) and CODE355655 is True:
- jaString = codeList[i]["parameters"][0]
- regex = r'.*subject=(.*?)"'
-
- # Var Text
- match = re.search(regex, jaString)
- if re.search(regex, jaString):
- finalJAString = match.group(1)
- # Pass 1
- if setData is False:
- list355655.append(finalJAString)
-
- # Pass 2
- else:
- # Grab and Replace
- translatedText = list355655[0]
- translatedText = translatedText.replace("'", "\\'")
-
- # Set
- codeList[i]["parameters"][0] = codeList[i]["parameters"][0].replace(finalJAString, translatedText)
- list355655.pop(0)
-
- ## Event Code: 408 (Script)
- if "code" in codeList[i] and (codeList[i]["code"] == 408) and CODE408 is True:
- jaString = codeList[i]["parameters"][0]
-
- # If there isn't any Japanese in the text just skip
- if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+", jaString):
- i += 1
- continue
-
- if "secretText" in jaString:
- regex = r"secretText:\s?(.+)"
- elif "title" in jaString:
- regex = r"title:\s?(.+)"
- else:
- regex = r"(.+)"
-
- # Need to remove outside code and put it back later
- matchList = re.findall(regex, jaString)
-
- for match in matchList:
- # Remove Textwrap
- match = match.replace("\n", " ")
- response = translateGPT(
- match,
- "Reply with the " + LANGUAGE + " translation of the achievement title.",
- False,
- )
- translatedText = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Replace
- translatedText = jaString.replace(match, translatedText)
-
- # Remove characters that may break scripts
- charList = [".", '"', "\\n"]
- for char in charList:
- translatedText = translatedText.replace(char, "")
-
- # Textwrap
- translatedText = textwrap.fill(translatedText, width=WIDTH)
-
- # Set Data
- codeList[i]["parameters"][0] = translatedText
-
- ## Event Code: 108 (Script)
- if "code" in codeList[i] and (codeList[i]["code"] == 108) and CODE108 is True:
- jaString = codeList[i]["parameters"][0]
-
- # If there isn't any Japanese in the text just skip
- if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+", jaString):
- i += 1
- continue
-
- # Translate
- if "info:" in jaString:
- regex = r"info:(.*)"
- elif "ActiveMessage:" in jaString:
- regex = r""
- elif "event_text" in jaString:
- regex = r"event_text\s*:\s*(.*)"
- else:
- i += 1
- continue
-
- # Need to remove outside code and put it back later
- match = re.search(regex, jaString)
- if match:
- # Pass 1
- if setData is False:
- list108.append(match.group(1))
-
- # Pass 2
- else:
- # Grab and Replace
- translatedText = list108[0]
- list108.pop(0)
-
- # Remove characters that may break scripts
- charList = [".", '"']
- for char in charList:
- translatedText = translatedText.replace(char, "")
- translatedText = translatedText.replace('"', '"')
- translatedText = translatedText.replace(" ", "_")
- translatedText = jaString.replace(match.group(1), translatedText)
-
- # Set Data
- codeList[i]["parameters"][0] = translatedText
-
- ## Event Code: 356
- if "code" in codeList[i] and codeList[i]["code"] == 356 and CODE356 is True:
- jaString = codeList[i]["parameters"][0]
- oldjaString = jaString
-
- # Grab Speaker
- if "Tachie showName" in jaString:
- matchList = re.findall(r"Tachie showName (.+)", jaString)
- if len(matchList) > 0:
- # Translate
- response = translateGPT(
- matchList[0],
- "Reply with the " + LANGUAGE + " translation of the NPC name.",
- False,
- )
- translatedText = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Set Text
- speaker = translatedText
- speaker = speaker.replace(" ", " ")
- codeList[i]["parameters"][0] = jaString.replace(matchList[0], speaker)
- i += 1
- continue
-
- # Want to translate this script
- if "D_TEXT " in jaString:
- regex = r"D_TEXT\s(.*?)\s.+"
- elif "ShowInfo" in jaString:
- regex = r"ShowInfo\s(.*)"
- elif "PushGab" in jaString:
- regex = r"PushGab\s(.*)"
- elif "addLog" in jaString:
- regex = r"addLog\s(.*)"
- elif "DW_" in jaString:
- regex = r"DW_.*?\s(.*)"
- elif "CommonPopup" in jaString:
- regex = r"CommonPopup\sadd\stext:(.*?)[\\]+}"
- else:
- regex = r""
-
- # Remove any textwrap
- jaString = re.sub(r"\n", "_", jaString)
-
- # Capture Arguments and text
- textMatch = re.search(regex, jaString)
- if textMatch and textMatch.group(0) != "":
- text = textMatch.group(1)
-
- # Pass 1
- if setData == False:
- text = text.replace("_", " ")
- list356.append(text)
-
- # Pass 2
- else:
- if len(list356) > 0:
- # Grab
- translatedText = list356[0]
-
- # Remove characters that may break scripts
- charList = [".", '"']
- for char in charList:
- translatedText = translatedText.replace(char, "")
-
- # Cant have spaces?
- translatedText = translatedText.replace(" ", "_")
- translatedText = translatedText.replace("__", "_")
-
- # Put Args Back
- translatedText = jaString.replace(text, translatedText)
-
- # Set Data
- codeList[i]["parameters"][0] = translatedText
- list356.pop(0)
-
- if "namePop" in jaString:
- matchList = re.findall(r"namePop\s\d+\s(.+?)\s.+", jaString)
- if len(matchList) > 0:
- # Translate
- text = matchList[0]
- response = translateGPT(text, "Reply with the " + LANGUAGE + " Translation", False)
- translatedText = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Set Data
- translatedText = jaString.replace(text, translatedText)
- codeList[i]["parameters"][0] = translatedText
-
- if "LL_InfoPopupWIndowMV" in jaString:
- matchList = re.findall(r"LL_InfoPopupWIndowMV\sshowWindow\s(.+?) .+", jaString)
- if len(matchList) > 0:
- # Translate
- text = matchList[0]
- response = translateGPT(text, "Reply with the " + LANGUAGE + " Translation", False)
- translatedText = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Set Data
- translatedText = translatedText.replace(" ", "_")
- translatedText = jaString.replace(text, translatedText)
- codeList[i]["parameters"][0] = translatedText
-
- if "OriginMenuStatus SetParam" in jaString:
- matchList = re.findall(r"OriginMenuStatus\sSetParam\sparam[\d]\s(.*)", jaString)
- if len(matchList) > 0:
- # Translate
- text = matchList[0]
- response = translateGPT(text, "Reply with the " + LANGUAGE + " Translation", False)
- translatedText = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Set Data
- translatedText = translatedText.replace(" ", "_")
- translatedText = jaString.replace(text, translatedText)
- codeList[i]["parameters"][0] = translatedText
-
- # LL_GalgeChoiceWindowMV Message
- if "LL_GalgeChoiceWindowMV setMessageText" in jaString:
- ### Message Text First
- match = re.search(r"LL_GalgeChoiceWindowMV setMessageText (.+)", jaString)
- if match:
- jaString = match.group(1)
-
- # Remove any textwrap & TL
- jaString = re.sub(r"\n", " ", jaString)
- response = translateGPT(jaString, "", False)
- translatedText = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Textwrap & Replace Whitespace
- translatedText = textwrap.fill(translatedText, width=WIDTH)
- translatedText = translatedText.replace(" ", "_")
-
- # Replace and Set
- translatedText = match.group(0).replace(match.group(1), translatedText)
- codeList[i]["parameters"][0] = translatedText
-
- # LL_GalgeChoiceWindowMV Choices
- if "LL_GalgeChoiceWindowMV setChoices":
- match = re.search(r"LL_GalgeChoiceWindowMV setChoices (.+)", jaString)
- if match:
- jaString = match.group(1)
- choiceList = jaString.split(",")
-
- # Translate
- question = translatedText
- response = translateGPT(
- choiceList,
- f"Previous text for context: {question}\n\nThis will be a dialogue option",
- True,
- )
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
- choiceListTL = response[0]
- translatedText = match.group(0)
-
- # Replace Strings
- for j in range(len(choiceListTL)):
- choiceListTL[j] = choiceListTL[j].replace(" ", "_")
- translatedText = translatedText.replace(choiceList[j], choiceListTL[j])
-
- # Set Data
- codeList[i]["parameters"][0] = translatedText
-
- ### Event Code: 102 Show Choice
- if "code" in codeList[i] and codeList[i]["code"] == 102 and CODE102 is True:
- choiceList = []
- varList = []
- for choice in range(len(codeList[i]["parameters"][0])):
- jaString = codeList[i]["parameters"][0][choice]
- jaString = jaString.replace(" 。", ".")
-
- # Avoid Empty Strings
- if jaString == "":
- i += 1
- continue
-
- # If and En Statements
- ifVar = ""
- ifList = re.findall(r"([ei][nf]\(.+?\)\)?\)?)", jaString)
- if len(ifList) != 0:
- for var in ifList:
- jaString = jaString.replace(var, "")
- ifVar += var
- varList.append(ifVar)
-
- # Append to List
- choiceList.append(jaString)
-
- # Translate
- if len(textHistory) > 0:
- response = translateGPT(
- choiceList,
- "This will be a dialogue option.\nPrevious text for context: " + str(textHistory),
- True,
- )
- translatedTextList = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
- else:
- response = translateGPT(choiceList, "This will be a dialogue option", True)
- translatedTextList = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Check Mismatch
- if len(translatedTextList) == len(choiceList):
- for choice in range(len(codeList[i]["parameters"][0])):
- translatedText = translatedTextList[choice]
-
- # Set Data
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
- if translatedText != "":
- translatedText = varList[choice] + translatedText[0].upper() + translatedText[1:]
- else:
- translatedText = varList[choice] + translatedText
- codeList[i]["parameters"][0][choice] = translatedText
- else:
- if filename not in MISMATCH:
- MISMATCH.append(filename)
-
- ### Event Code: 111 Script
- if "code" in codeList[i] and codeList[i]["code"] == 111 and CODE111 is True:
- for j in range(len(codeList[i]["parameters"])):
- jaString = codeList[i]["parameters"][j]
-
- # Check if String
- if not isinstance(jaString, str):
- i += 1
- continue
-
- # Only TL the Game Variable
- if "$gameVariables" not in jaString:
- i += 1
- continue
-
- # This is going to be the var being set. (IMPORTANT)
- if "1045" not in jaString:
- i += 1
- continue
-
- # Need to remove outside code and put it back later
- matchList = re.findall(r"'(.*?)'", jaString)
-
- for match in matchList:
- response = translateGPT(match, "", False)
- translatedText = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Remove characters that may break scripts
- charList = [".", '"', "'", "\\n"]
- for char in charList:
- translatedText = translatedText.replace(char, "")
-
- jaString = jaString.replace(match, translatedText)
-
- # Set Data
- translatedText = jaString
- codeList[i]["parameters"][j] = translatedText
-
- ### Event Code: 320 Set Variable
- if "code" in codeList[i] and codeList[i]["code"] == 320 and CODE320 is True:
- jaString = codeList[i]["parameters"][1]
- if not isinstance(jaString, str):
- i += 1
- continue
-
- # Definitely don't want to mess with files
- if "■" in jaString or "_" in jaString:
- i += 1
- continue
-
- # If there isn't any Japanese in the text just skip
- if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+", jaString):
- i += 1
- continue
-
- # Translate
- response = getSpeaker(jaString)
- translatedText = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
-
- # Remove characters that may break scripts
- charList = [".", '"', "'", "\\n"]
- for char in charList:
- translatedText = translatedText.replace(char, "")
-
- # Set Data
- codeList[i]["parameters"][1] = translatedText
-
- # Iterate
- else:
- i += 1
-
- # EOF
- list401TL = []
- list122TL = []
- list356TL = []
- list357TL = []
- list355655TL = []
- list108TL = []
- setData = False
- PBAR = pbar
-
- # 401
- if len(list401) > 0:
- response = translateGPT(list401, "", True)
- list401TL = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
- if len(list401TL) != len(list401):
- with LOCK:
- if filename not in MISMATCH:
- MISMATCH.append(filename)
- else:
- setData = True
-
- # 122
- if len(list122) > 0:
- response = translateGPT(list122, "Keep your translation as brief as possible", True)
- list122TL = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
- if len(list122TL) != len(list122):
- with LOCK:
- if filename not in MISMATCH:
- MISMATCH.append(filename)
- else:
- setData = True
-
- # 355/655
- if len(list355655) > 0:
- response = translateGPT(list355655, textHistory, True)
- list355655TL = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
- if len(list355655TL) != len(list355655):
- with LOCK:
- if filename not in MISMATCH:
- MISMATCH.append(filename)
- else:
- setData = True
-
- # 108
- if len(list108) > 0:
- response = translateGPT(list108, textHistory, True)
- list108TL = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
- if len(list108TL) != len(list108):
- with LOCK:
- if filename not in MISMATCH:
- MISMATCH.append(filename)
- else:
- setData = True
-
- # 356
- if len(list356) > 0:
- response = translateGPT(list356, textHistory, True)
- list356TL = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
- if len(list356TL) != len(list356):
- with LOCK:
- if filename not in MISMATCH:
- MISMATCH.append(filename)
- else:
- setData = True
-
- # 357
- if len(list357) > 0:
- response = translateGPT(list357, textHistory, True)
- list357TL = response[0]
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
- if len(list357TL) != len(list357):
- with LOCK:
- if filename not in MISMATCH:
- MISMATCH.append(filename)
- else:
- setData = True
-
- # Start Pass 2
- if setData:
- searchCodes(
- page,
- pbar,
- [list401TL, list122TL, list355655TL, list108TL, list356TL, list357TL],
- filename,
- )
-
- # Delete all -1 codes
- codeListFinal = []
- for i in range(len(codeList)):
- if "code" in codeList[i] and codeList[i]["code"] != -1:
- codeListFinal.append(codeList[i])
-
- # Normal Format
- if "list" in page:
- page["list"] = codeListFinal
-
- # Special Format (Scenario)
- else:
- page = codeListFinal
-
- except IndexError as e:
- traceback.print_exc()
- raise Exception(str(e) + "Failed to translate: " + oldjaString) from None
- except Exception as e:
- traceback.print_exc()
- raise Exception(str(e) + "Failed to translate: " + oldjaString) from None
-
- return totalTokens
-
-
-def searchSS(state, pbar):
- totalTokens = [0, 0]
-
- # Name
- nameResponse = (
- translateGPT(
- state["name"],
- "Reply with only the " + LANGUAGE + " translation of the RPG Skill name.",
- False,
- )
- if "name" in state
- else ""
- )
-
- # Description
- descriptionResponse = (
- translateGPT(
- state["description"],
- "Reply with only the " + LANGUAGE + " translation of the description.",
- False,
- )
- if "description" in state
- else ""
- )
-
- # Messages
- message1Response = ""
- message4Response = ""
- message2Response = ""
- message3Response = ""
-
- if "message1" in state:
- if len(state["message1"]) > 0 and state["message1"][0] in [
- "は",
- "を",
- "の",
- "に",
- "が",
- ]:
- message1Response = translateGPT(
- "Taro" + state["message1"],
- "reply with only the gender neutral "
- + LANGUAGE
- + " translation of the action log. Always start the sentence with Taro. For example,\
-Translate 'Taroを倒した！' as 'Taro was defeated!'",
- False,
- )
- else:
- message1Response = translateGPT(
- state["message1"],
- "reply with only the gender neutral " + LANGUAGE + " translation",
- False,
- )
-
- if "message2" in state:
- if len(state["message2"]) > 0 and state["message2"][0] in [
- "は",
- "を",
- "の",
- "に",
- "が",
- ]:
- message2Response = translateGPT(
- "Taro" + state["message2"],
- "reply with only the gender neutral "
- + LANGUAGE
- + " translation of the action log. Always start the sentence with Taro. For example,\
-Translate 'Taroを倒した！' as 'Taro was defeated!'",
- False,
- )
- else:
- message2Response = translateGPT(
- state["message2"],
- "reply with only the gender neutral " + LANGUAGE + " translation",
- False,
- )
-
- if "message3" in state:
- if len(state["message3"]) > 0 and state["message3"][0] in [
- "は",
- "を",
- "の",
- "に",
- "が",
- ]:
- message3Response = translateGPT(
- "Taro" + state["message3"],
- "reply with only the gender neutral "
- + LANGUAGE
- + " translation of the action log. Always start the sentence with Taro. For example,\
-Translate 'Taroを倒した！' as 'Taro was defeated!'",
- False,
- )
- else:
- message3Response = translateGPT(
- state["message3"],
- "reply with only the gender neutral " + LANGUAGE + " translation",
- False,
- )
-
- if "message4" in state:
- if len(state["message4"]) > 0 and state["message4"][0] in [
- "は",
- "を",
- "の",
- "に",
- "が",
- ]:
- message4Response = translateGPT(
- "Taro" + state["message4"],
- "reply with only the gender neutral "
- + LANGUAGE
- + " translation of the action log. Always start the sentence with Taro. For example,\
-Translate 'Taroを倒した！' as 'Taro was defeated!'",
- False,
- )
- else:
- message4Response = translateGPT(
- state["message4"],
- "reply with only the gender neutral " + LANGUAGE + " translation",
- False,
- )
-
- # Translate State Notes
- if "help" in state["note"]:
- noteResponse = translateNote(state, r"]*)>")
- totalTokens[0] += noteResponse[0]
- totalTokens[1] += noteResponse[1]
- if "STATE_HELP" in state["note"]:
- noteResponse = translateNote(state, r"\n(.*)\n")
- totalTokens[0] += noteResponse[0]
- totalTokens[1] += noteResponse[1]
-
- # Count totalTokens
- totalTokens[0] += nameResponse[1][0] if nameResponse != "" else 0
- totalTokens[1] += nameResponse[1][1] if nameResponse != "" else 0
- totalTokens[0] += descriptionResponse[1][0] if descriptionResponse != "" else 0
- totalTokens[1] += descriptionResponse[1][1] if descriptionResponse != "" else 0
- totalTokens[0] += message1Response[1][0] if message1Response != "" else 0
- totalTokens[1] += message1Response[1][1] if message1Response != "" else 0
- totalTokens[0] += message2Response[1][0] if message2Response != "" else 0
- totalTokens[1] += message2Response[1][1] if message2Response != "" else 0
- totalTokens[0] += message3Response[1][0] if message3Response != "" else 0
- totalTokens[1] += message3Response[1][1] if message3Response != "" else 0
- totalTokens[0] += message4Response[1][0] if message4Response != "" else 0
- totalTokens[1] += message4Response[1][1] if message4Response != "" else 0
-
- # Set Data
- if "name" in state:
- state["name"] = nameResponse[0].replace('"', "")
- if "description" in state:
- # Textwrap
- translatedText = descriptionResponse[0]
- translatedText = textwrap.fill(translatedText, width=LISTWIDTH)
- state["description"] = translatedText.replace('"', "")
- if "message1" in state:
- state["message1"] = message1Response[0].replace('"', "").replace("Taro", "")
- if "message2" in state:
- state["message2"] = message2Response[0].replace('"', "").replace("Taro", "")
- if "message3" in state:
- state["message3"] = message3Response[0].replace('"', "").replace("Taro", "")
- if "message4" in state:
- state["message4"] = message4Response[0].replace('"', "").replace("Taro", "")
-
- return totalTokens
-
-
-def searchSystem(data, pbar):
- totalTokens = [0, 0]
- context = "Reply with only the " + LANGUAGE + ' translation of the UI textbox."'
-
- # Title
- response = translateGPT(
- data["gameTitle"],
- " Reply with the " + LANGUAGE + " translation of the game title name",
- False,
- )
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
- data["gameTitle"] = response[0].strip(".")
-
- # Terms
- for term in data["terms"]:
- if term != "messages":
- termList = data["terms"][term]
- for i in range(len(termList)): # Last item is a messages object
- if termList[i] is not None:
- response = translateGPT(termList[i], context, False)
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
- termList[i] = response[0].replace('"', "").strip()
-
- # Armor Types
- for i in range(len(data["armorTypes"])):
- response = translateGPT(
- data["armorTypes"][i],
- "Reply with only the " + LANGUAGE + " translation of the armor type",
- False,
- )
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
- data["armorTypes"][i] = response[0].replace('"', "").strip()
-
- # Skill Types
- for i in range(len(data["skillTypes"])):
- response = translateGPT(
- data["skillTypes"][i],
- "Reply with only the " + LANGUAGE + " translation",
- False,
- )
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
- data["skillTypes"][i] = response[0].replace('"', "").strip()
-
- # Equip Types
- for i in range(len(data["equipTypes"])):
- response = translateGPT(
- data["equipTypes"][i],
- "Reply with only the " + LANGUAGE + " translation of the equipment type. No disclaimers.",
- False,
- )
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
- data["equipTypes"][i] = response[0].replace('"', "").strip()
-
- # # Variables (Optional ususally)
- # for i in range(len(data['variables'])):
- # response = translateGPT(data['variables'][i], 'Reply with only the '+ LANGUAGE +' translation of the title', False)
- # totalTokens[0] += response[1][0]
- # totalTokens[1] += response[1][1]
- # data['variables'][i] = response[0].replace('\"', '').strip()
-
- # Messages
- messages = data["terms"]["messages"]
- for key, value in messages.items():
- response = translateGPT(
- value,
- "Reply with only the "
- + LANGUAGE
- + ' translation of the battle text.\nTranslate "常時ダッシュ" as "Always Dash"\nTranslate "次の%1まで" as Next %1.',
- False,
- )
- translatedText = response[0]
-
- # Remove characters that may break scripts
- charList = [".", '"', "\\n"]
- for char in charList:
- translatedText = translatedText.replace(char, "")
-
- totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
- messages[key] = translatedText
-
- return totalTokens
-
-
-# Save some money and enter the character before translation
-def getSpeaker(speaker):
- match speaker:
- case "ファイン":
- return ["Fine", [0, 0]]
- case "":
- return ["", [0, 0]]
- case _:
- # Find Speaker
- for i in range(len(NAMESLIST)):
- if speaker == NAMESLIST[i][0]:
- return [NAMESLIST[i][1], [0, 0]]
-
- # Translate and Store Speaker
- response = translateGPT(
- f"{speaker}",
- "Reply with the " + LANGUAGE + " translation of the NPC name.",
- True,
- )
- response[0] = response[0].title()
- response[0] = response[0].replace("'S", "'s")
- response[0] = response[0].replace("Speaker: ", "")
-
- # Retry if name doesn't translate for some reason
- if re.search(r"([a-zA-Z？?])", response[0]) == None:
- response = translateGPT(
- f"{speaker}",
- "Reply with the " + LANGUAGE + " translation of the NPC name.",
- False,
- )
- response[0] = response[0].title()
- response[0] = response[0].replace("'S", "'s")
-
- speakerList = [speaker, response[0]]
- NAMESLIST.append(speakerList)
- return response
- return [speaker, [0, 0]]
-
-
-def batchList(input_list, batch_size):
- if not isinstance(batch_size, int) or batch_size <= 0:
- raise ValueError("batch_size must be a positive integer")
-
- return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)]
-
-
-def createContext(fullPromptFlag, subbedT, format):
- system = (
- PROMPT + VOCAB
- if fullPromptFlag
- else f"\
-You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
-Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
-- You translate everything, including erotic and explicit content.\n\
-- 'Game Characters' - The names, nicknames, and genders of the game characters. Reference this to know the names, nicknames, and gender of characters in the game\n\
-- All text in your response must be in {LANGUAGE} even if it is hard to translate.\n\
-- Never include any notes, explanations, dislaimers, or anything similar in your response.\n\
-- Maintain any spacing in the translation.\n\
-- Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\
-- `...` can be a part of the dialogue. Translate it as it is.\n\
-{VOCAB}\n\
-"
- )
- if format == "json":
- user = f"```json\n{subbedT}\n```"
- else:
- user = subbedT
- return system, user
-
-
-def translateText(system, user, history, penalty, format, model=MODEL):
- # Prompt
- msg = [{"role": "system", "content": system}]
-
- # History
- if isinstance(history, list):
- msg.extend([{"role": "system", "content": h} for h in history])
- else:
- msg.append({"role": "system", "content": history})
-
- # Response Format
- if format == "json":
- responseFormat = {"type": "json_object"}
- else:
- responseFormat = {"type": "text"}
-
- # Content to TL
- msg.append({"role": "user", "content": f"{user}"})
- response = openai.chat.completions.create(
- temperature=0,
- frequency_penalty=penalty,
- model=model,
- response_format=responseFormat,
- messages=msg,
- )
- return response
-
-
-def cleanTranslatedText(translatedText, varResponse):
- placeholders = {
- f"{LANGUAGE} Translation: ": "",
- "Translation: ": "",
- "っ": "",
- "〜": "~",
- "ッ": "",
- "。": ".",
- "「": '\\"',
- "」": '\\"',
- "- ": "-",
- "—": "―",
- "】": "]",
- "【": "[",
- "Placeholder Text": "",
- # Add more replacements as needed
- }
- for target, replacement in placeholders.items():
- translatedText = translatedText.replace(target, replacement)
-
- # Remove Repeating Characters
- pattern = re.compile(r"(.)\s*\1(?:\s*\1){" + str(20 - 1) + r",}")
- translatedText = pattern.sub(lambda match: match.group(0).replace(" ", "")[:20], translatedText)
-
- # Elongate Long Dashes (Since GPT Ignores them...)
- translatedText = elongateCharacters(translatedText)
- return translatedText
-
-
-def elongateCharacters(text):
- # Define a pattern to match one character followed by one or more `ー` characters
- # Using a positive lookbehind assertion to capture the preceding character
- pattern = r"(?<=(.))ー+"
-
- # Define a replacement function that elongates the captured character
- def repl(match):
- char = match.group(1) # The character before the ー sequence
- count = len(match.group(0)) - 1 # Number of ー characters
- return char * count # Replace ー sequence with the character repeated
-
- # Use re.sub() to replace the pattern in the text
- return re.sub(pattern, repl, text)
-
-
-def extractTranslation(translatedTextList, is_list):
- try:
- translatedTextList = re.sub(r'\\"+\"([^,\n}])', r'\\"\1', translatedTextList)
- translatedTextList = re.sub(r"(? instead
+FIXTEXTWRAP = True # Overwrites textwrap
+IGNORETLTEXT = False # Ignores all translated text.
+MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
+PBAR = None
+FILENAME = None
+
+# Pricing - Depends on the model https://openai.com/pricing
+# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request
+# If you are getting a MISMATCH LENGTH error, lower the batch size.
+if "gpt-3.5" in MODEL:
+ INPUTAPICOST = 0.002
+ OUTPUTAPICOST = 0.002
+ BATCHSIZE = 10
+ FREQUENCY_PENALTY = 0.2
+elif "gpt-4" in MODEL:
+ INPUTAPICOST = 0.0025
+ OUTPUTAPICOST = 0.01
+ BATCHSIZE = 30
+ FREQUENCY_PENALTY = 0.1
+else:
+ INPUTAPICOST = float(os.getenv("input_cost"))
+ OUTPUTAPICOST = float(os.getenv("output_cost"))
+ BATCHSIZE = int(os.getenv("batchsize"))
+ FREQUENCY_PENALTY = float(os.getenv("frequency_penalty"))
+
+# tqdm Globals
+BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
+POSITION = 0
+LEAVE = False
+
+# Dialogue / Scroll / Choices (Main Codes)
+CODE401 = False
+CODE405 = False
+CODE102 = False
+
+# Optional
+CODE101 = False # Turn this one when names exist in 101
+CODE408 = False # Warning, translates comments and can inflate costs.
+
+# Variables
+CODE122 = False
+
+# Other
+CODE355655 = False
+CODE357 = False
+CODE657 = False
+CODE356 = False
+CODE320 = False
+CODE324 = False
+CODE111 = False
+CODE108 = True
+
+
+def handleMVMZ(filename, estimate):
+ global ESTIMATE, TOKENS, FILENAME
+ ESTIMATE = estimate
+ FILENAME = filename
+
+ # Translate
+ start = time.time()
+ translatedData = openFiles(filename)
+
+ # Translate
+ if not estimate:
+ try:
+ with open("translated/" + filename, "w", encoding="utf-8") as outFile:
+ json.dump(translatedData[0], outFile, ensure_ascii=False, indent=4)
+ except Exception:
+ traceback.print_exc()
+ return "Fail"
+
+ # Print File
+ end = time.time()
+ tqdm.write(getResultString(translatedData, end - start, filename))
+ with LOCK:
+ TOKENS[0] += translatedData[1][0]
+ TOKENS[1] += translatedData[1][1]
+
+ # Print Total
+ totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
+
+ # Print any errors on maps
+ if len(MISMATCH) > 0:
+ return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
+ else:
+ return totalString
+
+
+def openFiles(filename):
+ with open("files/" + filename, "r", encoding="utf-8-sig") as f:
+ data = json.load(f)
+
+ # Map Files
+ if "Map" in filename and filename != "MapInfos.json":
+ translatedData = parseMap(data, filename)
+
+ # CommonEvents Files
+ elif "CommonEvents" in filename:
+ translatedData = parseCommonEvents(data, filename)
+
+ # Actor File
+ elif "Actors" in filename:
+ translatedData = parseNames(data, filename, "Actors")
+
+ # Armor File
+ elif "Armors" in filename:
+ translatedData = parseNames(data, filename, "Armors")
+
+ # Weapons File
+ elif "Weapons" in filename:
+ translatedData = parseNames(data, filename, "Weapons")
+
+ # Classes File
+ elif "Classes" in filename:
+ translatedData = parseNames(data, filename, "Classes")
+
+ # Enemies File
+ elif "Enemies" in filename:
+ translatedData = parseNames(data, filename, "Enemies")
+
+ # Items File
+ elif "Items" in filename:
+ translatedData = parseNames(data, filename, "Items")
+
+ # MapInfo File
+ elif "MapInfos" in filename:
+ translatedData = parseNames(data, filename, "MapInfos")
+
+ # Skills File
+ elif "Skills" in filename:
+ translatedData = parseNames(data, filename, "Skills")
+
+ # Troops File
+ elif "Troops" in filename:
+ translatedData = parseTroops(data, filename)
+
+ # States File
+ elif "States" in filename:
+ translatedData = parseSS(data, filename)
+
+ # System File
+ elif "System" in filename:
+ translatedData = parseSystem(data, filename)
+
+ # Scenario File
+ elif "Scenario" in filename:
+ translatedData = parseScenario(data, filename)
+
+ else:
+ raise NameError(filename + " Not Supported")
+
+ return translatedData
+
+
+def getResultString(translatedData, translationTime, filename):
+ # File Print String
+ totalTokenstring = (
+ Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
+ "[Output: "
+ + str(translatedData[1][1])
+ + "]" "[Cost: ${:,.4f}".format((translatedData[1][0] * 0.001 * INPUTAPICOST) + (translatedData[1][1] * 0.001 * OUTPUTAPICOST))
+ + "]"
+ )
+ timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
+
+ if translatedData[2] is None:
+ # Success
+ return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET
+ else:
+ # Fail
+ try:
+ raise translatedData[2]
+ except Exception as e:
+ traceback.print_exc()
+ errorString = str(e) + Fore.RED
+ return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET
+
+
+def parseMap(data, filename):
+ totalTokens = [0, 0]
+ totalLines = 0
+ events = data["events"]
+ global LOCK
+
+ # Translate displayName for Map files
+ if "Map" in filename:
+ response = translateGPT(
+ data["displayName"],
+ "Reply with only the " + LANGUAGE + " translation of the RPG location name",
+ False,
+ )
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+ data["displayName"] = response[0].replace('"', "")
+
+ # Get total for progress bar
+ for event in events:
+ if event is not None:
+ for page in event["pages"]:
+ totalLines += len(page["list"])
+
+ # Thread for each page in file
+ with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
+ pbar.desc = filename
+ with ThreadPoolExecutor(max_workers=THREADS) as executor:
+ for event in events:
+ if event is not None:
+ # This translates ID of events. (May break the game)
+ if ".*")
+ totalTokens[0] += response[0]
+ totalTokens[1] += response[1]
+ if ".*")
+ totalTokens[0] += response[0]
+ totalTokens[1] += response[1]
+ if ".*")
+ totalTokens[0] += response[0]
+ totalTokens[1] += response[1]
+
+ futures = [executor.submit(searchCodes, page, pbar, [], filename) for page in event["pages"] if page is not None]
+ for future in as_completed(futures):
+ try:
+ totalTokensFuture = future.result()
+ totalTokens[0] += totalTokensFuture[0]
+ totalTokens[1] += totalTokensFuture[1]
+ except Exception as e:
+ traceback.print_exc()
+ return [data, totalTokens, e]
+ return [data, totalTokens, None]
+
+
+def translateNote(event, regex):
+ # Regex String
+ jaString = event["note"]
+ match = re.findall(regex, jaString, re.DOTALL)
+ if match:
+ tokens = [0, 0]
+ i = 0
+ while i < len(match):
+ initialJAString = match[i]
+ # Remove any textwrap
+ modifiedJAString = initialJAString.replace("\n", " ")
+
+ # Translate
+ response = translateGPT(
+ modifiedJAString,
+ "Reply with only the " + LANGUAGE + " translation.",
+ False,
+ )
+ translatedText = response[0]
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+
+ # Textwrap
+ translatedText = textwrap.fill(translatedText, width=NOTEWIDTH)
+ translatedText = translatedText.replace('"', "")
+ jaString = jaString.replace(initialJAString, translatedText)
+ event["note"] = jaString
+ i += 1
+ return tokens
+ return [0, 0]
+
+
+# For notes that can't have spaces.
+def translateNoteOmitSpace(event, regex):
+ # Regex that only matches text inside LB.
+ jaString = event["note"]
+
+ match = re.findall(regex, jaString, re.DOTALL)
+ if match:
+ oldJAString = match[0]
+ # Remove any textwrap
+ jaString = re.sub(r"\n", " ", oldJAString)
+
+ # Translate
+ response = translateGPT(
+ jaString,
+ "Reply with the " + LANGUAGE + " translation of the location name.",
+ False,
+ )
+ translatedText = response[0]
+
+ translatedText = translatedText.replace('"', "")
+ translatedText = translatedText.replace(" ", "_")
+ event["note"] = event["note"].replace(oldJAString, translatedText)
+ return response[1]
+ return [0, 0]
+
+
+def parseCommonEvents(data, filename):
+ totalTokens = [0, 0]
+ totalLines = 0
+ global LOCK
+
+ # Get total for progress bar
+ for page in data:
+ if page is not None:
+ totalLines += len(page["list"])
+
+ with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
+ pbar.desc = filename
+ with ThreadPoolExecutor(max_workers=THREADS) as executor:
+ futures = [executor.submit(searchCodes, page, pbar, [], filename) for page in data if page is not None]
+ for future in as_completed(futures):
+ try:
+ totalTokensFuture = future.result()
+ totalTokens[0] += totalTokensFuture[0]
+ totalTokens[1] += totalTokensFuture[1]
+ except Exception as e:
+ traceback.print_exc()
+ return [data, totalTokens, e]
+ return [data, totalTokens, None]
+
+
+def parseTroops(data, filename):
+ totalTokens = [0, 0]
+ totalLines = 0
+ global LOCK
+
+ # Get total for progress bar
+ for troop in data:
+ if troop is not None:
+ for page in troop["pages"]:
+ totalLines += len(page["list"]) + 1 # The +1 is because each page has a name.
+
+ with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
+ pbar.desc = filename
+ for troop in data:
+ if troop is not None:
+ with ThreadPoolExecutor(max_workers=THREADS) as executor:
+ futures = [executor.submit(searchCodes, page, pbar, [], filename) for page in troop["pages"] if page is not None]
+ for future in as_completed(futures):
+ try:
+ totalTokensFuture = future.result()
+ totalTokens[0] += totalTokensFuture[0]
+ totalTokens[1] += totalTokensFuture[1]
+ except Exception as e:
+ traceback.print_exc()
+ return [data, totalTokens, e]
+ return [data, totalTokens, None]
+
+
+def parseNames(data, filename, context):
+ totalTokens = [0, 0]
+ totalLines = 0
+ totalLines += len(data)
+
+ with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
+ pbar.desc = filename
+ try:
+ result = searchNames(data, pbar, context)
+ totalTokens[0] += result[0]
+ totalTokens[1] += result[1]
+ except Exception as e:
+ traceback.print_exc()
+ return [data, totalTokens, e]
+ return [data, totalTokens, None]
+
+
+def parseSS(data, filename):
+ totalTokens = [0, 0]
+ totalLines = 0
+ totalLines += len(data)
+
+ with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
+ pbar.desc = filename
+ for ss in data:
+ if ss is not None:
+ try:
+ result = searchSS(ss, pbar)
+ totalTokens[0] += result[0]
+ totalTokens[1] += result[1]
+ except Exception as e:
+ traceback.print_exc()
+ return [data, totalTokens, e]
+ return [data, totalTokens, None]
+
+
+def parseSystem(data, filename):
+ totalTokens = [0, 0]
+ totalLines = 0
+
+ # Calculate Total Lines
+ for term in data["terms"]:
+ termList = data["terms"][term]
+ totalLines += len(termList)
+ totalLines += len(data["gameTitle"])
+ totalLines += len(data["terms"]["messages"])
+ totalLines += len(data["variables"])
+ totalLines += len(data["equipTypes"])
+ totalLines += len(data["armorTypes"])
+ totalLines += len(data["skillTypes"])
+
+ with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
+ pbar.desc = filename
+ try:
+ result = searchSystem(data, pbar)
+ totalTokens[0] += result[0]
+ totalTokens[1] += result[1]
+ except Exception as e:
+ traceback.print_exc()
+ return [data, totalTokens, e]
+ return [data, totalTokens, None]
+
+
+def parseScenario(data, filename):
+ totalTokens = [0, 0]
+ totalLines = 0
+ global LOCK
+
+ # Get total for progress bar
+ for page in data.items():
+ totalLines += len(page[1])
+
+ with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
+ pbar.desc = filename
+ with ThreadPoolExecutor(max_workers=THREADS) as executor:
+ futures = [executor.submit(searchCodes, page[1], pbar, [], filename) for page in data.items() if page[1] is not None]
+ for future in as_completed(futures):
+ try:
+ totalTokensFuture = future.result()
+ totalTokens[0] += totalTokensFuture[0]
+ totalTokens[1] += totalTokensFuture[1]
+ except Exception as e:
+ traceback.print_exc()
+ return [data, totalTokens, e]
+ return [data, totalTokens, None]
+
+
+def searchNames(data, pbar, context):
+ totalTokens = [0, 0]
+ nameList = []
+ profileList = []
+ nicknameList = []
+ descriptionList = []
+ noteList = []
+ i = 0 # Counter
+ j = 0 # Counter 2
+ filling = False
+ mismatch = False
+ batchFull = False
+
+ # Set the context of what we are translating
+ if "Actors" in context:
+ newContext = "Reply with only the " + LANGUAGE + " translation of the NPC name"
+ if "Armors" in context:
+ newContext = "Reply with only the " + LANGUAGE + " translation of the RPG equipment name"
+ if "Classes" in context:
+ newContext = "Reply with only the " + LANGUAGE + " translation of the RPG class name"
+ if "MapInfos" in context:
+ newContext = "Reply with only the " + LANGUAGE + " translation of the location name"
+ if "Enemies" in context:
+ newContext = "Reply with only the " + LANGUAGE + " translation of the enemy NPC name"
+ if "Weapons" in context:
+ newContext = "Reply with only the " + LANGUAGE + " translation of the RPG weapon name"
+ if "Items" in context:
+ newContext = "Reply with only the " + LANGUAGE + " translation of the RPG item name"
+ if "Skills" in context:
+ newContext = "Reply with only the " + LANGUAGE + " translation of the RPG skill name"
+
+ # Names
+ with open("translations.txt", "a", encoding="utf-8") as file:
+ file.write(f"\n#{context}\n")
+ while i < len(data) or filling == True:
+ if i < len(data):
+ # Empty Data
+ if data[i] is None or data[i]["name"] == "":
+ i += 1
+
+ continue
+
+ # Filling up Batch
+ filling = True
+ if context in "Actors":
+ if len(nameList) < BATCHSIZE:
+ if data[i]["name"] != "":
+ nameList.append(data[i]["name"])
+ if data[i]["nickname"] != "":
+ nicknameList.append(data[i]["nickname"])
+ if data[i]["profile"] != "":
+ profileList.append(data[i]["profile"].replace("\n", " "))
+
+ # Notes
+ if "")
+ totalTokens[0] += tokensResponse[0]
+ totalTokens[1] += tokensResponse[1]
+ if "PE拡張" in data[i]["note"]:
+ tokensResponse = translateNote(data[i], r"")
+ totalTokens[0] += tokensResponse[0]
+ totalTokens[1] += tokensResponse[1]
+ i += 1
+ else:
+ batchFull = True
+ if context in ["Armors", "Weapons", "Items"]:
+ if len(nameList) < BATCHSIZE:
+ nameList.append(data[i]["name"])
+ if "description" in data[i] and data[i]["description"] != "":
+ descriptionList.append(data[i]["description"].replace("\n", " "))
+ if "")
+ totalTokens[0] += tokensResponse[0]
+ totalTokens[1] += tokensResponse[1]
+ if "")
+ totalTokens[0] += tokensResponse[0]
+ totalTokens[1] += tokensResponse[1]
+ if "")
+ totalTokens[0] += tokensResponse[0]
+ totalTokens[1] += tokensResponse[1]
+ if "")
+ totalTokens[0] += tokensResponse[0]
+ totalTokens[1] += tokensResponse[1]
+ if "")
+ totalTokens[0] += tokensResponse[0]
+ totalTokens[1] += tokensResponse[1]
+ if "")
+ totalTokens[0] += tokensResponse[0]
+ totalTokens[1] += tokensResponse[1]
+ if "")
+ totalTokens[0] += tokensResponse[0]
+ totalTokens[1] += tokensResponse[1]
+ if "Switch Shop Description" in data[i]["note"]:
+ tokensResponse = translateNote(data[i], r"\n(.*)\n")
+ totalTokens[0] += tokensResponse[0]
+ totalTokens[1] += tokensResponse[1]
+ if "")
+ totalTokens[0] += tokensResponse[0]
+ totalTokens[1] += tokensResponse[1]
+
+ i += 1
+ else:
+ batchFull = True
+ if context in ["Skills"]:
+ if len(nameList) < BATCHSIZE:
+ nameList.append(data[i]["name"])
+ if "description" in data[i] and data[i]["description"]:
+ descriptionList.append(data[i]["description"].replace("\n", " "))
+
+ # Messages
+ number = 1
+ while number < 5:
+ if f"message{number}" in data[i] and data[i][f"message{number}"]:
+ if data[i][f"message{number}"][0] in ["は", "を", "の", "に", "が"]:
+ msgResponse = translateGPT(
+ "Taro" + data[i][f"message{number}"],
+ "reply with only the gender neutral "
+ + LANGUAGE
+ + " translation of the action log. Always start the sentence with Taro. For example, Translate 'Taroを倒した！' as 'Taro was defeated!'",
+ False,
+ )
+ data[i][f"message{number}"] = msgResponse[0].replace("Taro", "")
+ totalTokens[0] += msgResponse[1][0]
+ totalTokens[1] += msgResponse[1][1]
+ number += 1
+
+ else:
+ msgResponse = translateGPT(
+ data[i][f"message{number}"],
+ "reply with only the gender neutral " + LANGUAGE + " translation",
+ False,
+ )
+ data[i][f"message{number}"] = msgResponse[0]
+ totalTokens[0] += msgResponse[1][0]
+ totalTokens[1] += msgResponse[1][1]
+ number += 1
+ else:
+ number += 1
+
+ i += 1
+ else:
+ batchFull = True
+ if context in ["Enemies", "Classes", "MapInfos"]:
+ if len(nameList) < BATCHSIZE:
+ nameList.append(data[i]["name"])
+
+ # Notes
+ if "note" in data[i]:
+ if "")
+ totalTokens[0] += tokensResponse[0]
+ totalTokens[1] += tokensResponse[1]
+ if "")
+ totalTokens[0] += tokensResponse[0]
+ totalTokens[1] += tokensResponse[1]
+ i += 1
+ else:
+ batchFull = True
+
+ # Batch Full
+ if batchFull == True or i >= len(data):
+ k = j # Original Index
+ if context in "Actors":
+ # Name
+ response = translateGPT(nameList, newContext, True)
+ translatedNameBatch = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Nickname
+ if nicknameList:
+ response = translateGPT(nicknameList, newContext, True)
+ translatedNicknameBatch = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Profile
+ if profileList:
+ response = translateGPT(profileList, "", True)
+ translatedProfileBatch = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Set Data
+ if len(nameList) == len(translatedNameBatch):
+ j = k
+ while j < i:
+ # Empty Data
+ if data[j] is None or data[j]["name"] == "":
+ j += 1
+ continue
+ else:
+ # Get Text
+ if data[j]["name"] != "":
+ with open("translations.txt", "a", encoding="utf-8") as file:
+ file.write(f'{data[j]["name"]} ({translatedNameBatch[0]})\n')
+ data[j]["name"] = translatedNameBatch[0]
+ translatedNameBatch.pop(0)
+ if data[j]["nickname"] != "":
+ data[j]["nickname"] = translatedNicknameBatch[0]
+ translatedNicknameBatch.pop(0)
+ if data[j]["profile"] != "":
+ data[j]["profile"] = textwrap.fill(translatedProfileBatch[0], LISTWIDTH)
+ translatedProfileBatch.pop(0)
+
+ # If Batch is empty. Move on.
+ if len(translatedNameBatch) == 0:
+ nameList.clear()
+ filling = False
+ j += 1
+ else:
+ mismatch = True
+
+ if context in ["Armors", "Weapons", "Items", "Skills"]:
+ # Name
+ response = translateGPT(nameList, newContext, True)
+ translatedNameBatch = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Description
+ if descriptionList:
+ response = translateGPT(
+ descriptionList,
+ f"Reply with only the {LANGUAGE} translation of the text.",
+ True,
+ )
+ translatedDescriptionBatch = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Set Data
+ if len(nameList) == len(translatedNameBatch):
+ j = k
+ with open("translations.txt", "a", encoding="utf-8") as file:
+ while j < i:
+ # Empty Data
+ if data[j] is None or data[j]["name"] == "":
+ j += 1
+ continue
+ else:
+ # Get Text
+ file.write(f'{data[j]['name']} ({translatedNameBatch[0]})\n')
+ data[j]["name"] = translatedNameBatch[0]
+ translatedNameBatch.pop(0)
+ if "description" in data[j] and data[j]["description"] != "":
+ data[j]["description"] = textwrap.fill(translatedDescriptionBatch[0], LISTWIDTH)
+ translatedDescriptionBatch.pop(0)
+
+ # If Batch is empty. Move on.
+ if len(translatedNameBatch) == 0:
+ nameList.clear()
+ descriptionList.clear()
+ batchFull = False
+ filling = False
+ j += 1
+ else:
+ mismatch = True
+ if context in ["Enemies", "Classes", "MapInfos"]:
+ response = translateGPT(nameList, newContext, True)
+ translatedNameBatch = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Set Data
+ if len(nameList) == len(translatedNameBatch):
+ j = k
+ while j < i:
+ # Empty Data
+ if data[j] is None or data[j]["name"] == "":
+ j += 1
+ continue
+ else:
+ with open("translations.txt", "a", encoding="utf-8") as file:
+ file.write(f'{data[j]["name"]} ({translatedNameBatch[0]})\n')
+ # Get Text
+ data[j]["name"] = translatedNameBatch[0]
+ translatedNameBatch.pop(0)
+
+ # If Batch is empty. Move on.
+ if len(translatedNameBatch) == 0:
+ nameList.clear()
+ batchFull = False
+ filling = False
+ j += 1
+ else:
+ mismatch = True
+
+ # Mismatch
+ if mismatch == True:
+ MISMATCH.append(nameList)
+ nameList.clear()
+ profileList.clear()
+ descriptionList.clear()
+ filling = False
+ mismatch = False
+
+ i += 1
+
+ return totalTokens
+
+
+def searchCodes(page, pbar, jobList, filename):
+ if len(jobList) > 0:
+ list401 = jobList[0]
+ list122 = jobList[1]
+ list355655 = jobList[2]
+ list108 = jobList[3]
+ list356 = jobList[4]
+ list357 = jobList[5]
+ setData = True
+ else:
+ list401 = []
+ list122 = []
+ list355655 = []
+ list108 = []
+ list356 = []
+ list357 = []
+ setData = False
+ textHistory = []
+ match = []
+ totalTokens = [0, 0]
+ translatedText = ""
+ speaker = ""
+ speakerID = None
+ syncIndex = 0
+ CLFlag = False
+ maxHistory = MAXHISTORY
+ VNameValue = None
+ speakerWindow = FIRSTLINESPEAKERS
+ global LOCK
+ global NAMESLIST
+ global MISMATCH
+ global PBAR
+ with LOCK:
+ PBAR = pbar
+
+ # Begin Parsing File
+ try:
+ # Normal Format
+ if "list" in page:
+ codeList = page["list"]
+
+ # Special Format (Scenario)
+ else:
+ codeList = page
+
+ # Iterate through page
+ i = 0
+ while i < len(codeList):
+ with LOCK:
+ # syncIndex will keep i in sync when it gets modified
+ if syncIndex > i:
+ i = syncIndex
+ if len(codeList) <= i:
+ break
+
+ # Declare Varss
+ currentGroup = []
+ nametag = ""
+
+ ## Event Code: 401 Show Text
+ if "code" in codeList[i] and codeList[i]["code"] in [401, 405, -1] and (CODE401 or CODE405):
+ # Save Code and starting index (j)
+ code = codeList[i]["code"]
+ j = i
+ endtag = ""
+
+ # Grab String
+ if len(codeList[i]["parameters"]) > 0:
+ jaString = codeList[i]["parameters"][0]
+ oldjaString = jaString
+ else:
+ codeList[i]["code"] = -1
+ i += 1
+ continue
+
+ # Validate Japanese Text
+ if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+", jaString) and IGNORETLTEXT:
+ i += 1
+ continue
+
+ # # For Retarded Devs
+ # retardRegex = r'([\\]+[nN]\[[\\]+V\[\d*?\]\])'
+ # match = re.search(retardRegex, jaString)
+ # if match:
+ # if VNameValue == 1:
+ # jaString = re.sub(retardRegex, 'リッカ', jaString)
+ # if VNameValue == 2:
+ # jaString = re.sub(retardRegex, 'ミミ', jaString)
+ # if VNameValue == 3:
+ # jaString = re.sub(retardRegex, 'ヒトミ', jaString)
+ # if VNameValue == 4:
+ # jaString = re.sub(retardRegex, 'Taro', jaString)
+ # if VNameValue == 5:
+ # jaString = re.sub(retardRegex, '富士見', jaString)
+
+ # Speaker Check
+ speakerList = []
+
+ # m and z Codes
+ match = re.search(r"(.*?)[\\]+m\[\d+?\][\\]+z\[\d+?\]", jaString)
+ if match:
+ speakerList.append(match.group(1))
+ if "\\c" in speakerList[0]:
+ speakerList = re.findall(
+ r"^[\\]+[cC]\[[\d]+\]【?(.+?)】?[\\]+[Cc]\[[\d]\]\\?\\?$",
+ speakerList[0],
+ )
+
+ # Brackets
+ if len(speakerList) == 0:
+ speakerList = re.findall(r"^【(.*?)】$", jaString)
+
+ # Colors
+ if len(speakerList) == 0:
+ speakerList = re.findall(
+ r"^[\\]+[cC]\[[\d]+\]【?(.+?)】?[\\]+[Cc]\[[\d]\]\\?\\?$",
+ jaString,
+ )
+
+ # First Line Speakers
+ if len(speakerList) == 0 and FIRSTLINESPEAKERS is True:
+ # Remove any RPGMaker Code at start
+ ffMatch = re.search(
+ r"^(\s*[\\]+[aAbBdDeEfFgGhHjJlLmMoOpPqQrRsStTuUwWxXyYzZ]+\[[\w\d\[\]\\]+\])",
+ jaString,
+ )
+ if ffMatch != None:
+ jaString = jaString.replace(ffMatch.group(0), "")
+ nametag += ffMatch.group(0)
+
+ # Test Speaker
+ if (
+ len(jaString) < 40
+ and "code" in codeList[i + 1]
+ and codeList[i + 1]["code"] in [401, 405, -1]
+ and len(codeList[i + 1]["parameters"]) > 0
+ and len(codeList[i + 1]["parameters"][0]) > 0
+ ):
+ if codeList[i + 1]["parameters"] != "" and codeList[i + 1]["parameters"][0].strip()[0] in [
+ "「",
+ '"',
+ "(",
+ "（",
+ "*",
+ "[",
+ ]:
+ speakerList = re.findall(r".+", jaString)
+
+ if len(speakerList) != 0 and codeList[i + 1]["code"] in [401, 405, -1]:
+ # Get Speaker
+ response = getSpeaker(speakerList[0])
+ speaker = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Set Data
+ codeList[i]["parameters"][0] = nametag + jaString.replace(speakerList[0], speaker)
+ nametag = ""
+
+ # Iterate to next string
+ i += 1
+ j = i
+ while codeList[i]["code"] in [-1]:
+ i += 1
+ j = i
+ jaString = codeList[i]["parameters"][0]
+
+ # Using this to keep track of 401's in a row.
+ currentGroup.append(jaString)
+
+ # Join Up 401's into single string
+ if len(codeList) > i + 1:
+ while codeList[i + 1]["code"] in [401, 405, -1]:
+ if setData == True:
+ codeList[i]["parameters"] = []
+ codeList[i]["code"] = -1
+ i += 1
+ j = i
+
+ # Only add if not empty
+ if len(codeList[i]["parameters"]) > 0:
+ jaString = codeList[i]["parameters"][0]
+ currentGroup.append(jaString)
+
+ # Make sure not the end of the list.
+ if len(codeList) <= i + 1:
+ break
+
+ # Format String
+ if len(currentGroup) > 0:
+ finalJAString = "\n".join(currentGroup)
+ oldjaString = finalJAString
+
+ # Check if Empty
+ if finalJAString == "":
+ i += 1
+ continue
+
+ # Set Back
+ if setData == True:
+ codeList[i]["parameters"] = [finalJAString]
+
+ ### \\n
+ nCase = None
+ regex = r"([\\]+[kKnN][wWcCrRrEe]?[\[<](.*?)[>\]])"
+ match = re.search(regex, finalJAString)
+
+ # Set Name
+ if match:
+ nametag = match.group(1)
+ speaker = match.group(2)
+
+ # Translate Speaker
+ response = getSpeaker(speaker)
+ tledSpeaker = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Set Nametag and Remove from Final String
+ finalJAString = finalJAString.replace(nametag, "")
+ nametag = nametag.replace(speaker, tledSpeaker)
+ speaker = tledSpeaker
+
+ # Remove Extra Stuff bad for translation.
+ finalJAString = finalJAString.replace("ﾞ", "")
+ finalJAString = finalJAString.replace("―", "-")
+ finalJAString = finalJAString.replace("…", "...")
+ finalJAString = finalJAString.replace("。", ".")
+ finalJAString = re.sub(r"(\.{3}\.+)", "...", finalJAString)
+ finalJAString = finalJAString.replace(" ", "")
+ finalJAString = finalJAString.replace("「", '"')
+ finalJAString = finalJAString.replace("」", '"')
+
+ ### Remove format codes
+ # Furigana
+ rcodeMatch = re.findall(r"([\\]+[r][b]?\[.*?,(.*?)\])", finalJAString)
+ if len(rcodeMatch) > 0:
+ for match in rcodeMatch:
+ finalJAString = finalJAString.replace(match[0], match[1])
+
+ # Formatting
+ formatMatch = re.findall(r"[\\]+[!><.|#^{}]", finalJAString)
+ if len(formatMatch) > 0:
+ for match in formatMatch:
+ finalJAString = finalJAString.replace(match, "")
+
+ # Remove any RPGMaker Code at start
+ ffMatch = re.search(
+ r"^(\s*[\\]+[aAbBdDeEfFgGhHjJlLmMoOpPqQrRsStTuUwWxXyYzZ]+\[[\w\d\[\]\\]+?\])",
+ finalJAString,
+ )
+ if ffMatch != None:
+ finalJAString = finalJAString.replace(ffMatch.group(1), "")
+ nametag += ffMatch.group(1)
+
+ # Remove _ABL Codes
+ ffMatch = re.search(r"^(_ABL).*", finalJAString)
+ if ffMatch != None:
+ finalJAString = finalJAString.replace(ffMatch.group(1), "")
+ nametag += ffMatch.group(1)
+
+ # Center Lines
+ if "\\CL" in finalJAString or "\\ac" in finalJAString:
+ finalJAString = finalJAString.replace("\\CL ", "")
+ finalJAString = finalJAString.replace("\\CL", "")
+ finalJAString = finalJAString.replace("\\ac ", "")
+ finalJAString = finalJAString.replace("\\ac", "")
+ CLFlag = True
+
+ # 1st Passthrough (Grabbing Data)
+ if setData == False:
+ # Remove Textwrap
+ if FIXTEXTWRAP:
+ finalJAString = finalJAString.replace("\n", " ")
+ if "\\px[200]" in finalJAString:
+ finalJAString = finalJAString.replace("\\px[200]", "")
+
+ # Append
+ if finalJAString != "":
+ if speaker == "" and finalJAString != "":
+ list401.append(finalJAString)
+ elif finalJAString != "":
+ list401.append(f"[{speaker}]: {finalJAString}")
+ else:
+ list401.append(speaker)
+ speaker = ""
+ match = []
+ nametag = ""
+ currentGroup = []
+ syncIndex = i + 1
+
+ # Keep textHistory list at length maxHistory
+ textHistory.append('"' + finalJAString + '"')
+ if len(textHistory) > maxHistory:
+ textHistory.pop(0)
+
+ # 2nd Passthrough (Setting Data)
+ else:
+ # Grab Translated String
+ if len(list401) > 0:
+ translatedText = list401[0]
+
+ # Remove speaker
+ if speaker != "":
+ matchSpeakerList = re.findall(r"^\[?(.+?)\]?\s?[|:]\s?", translatedText)
+ if len(matchSpeakerList) > 0:
+ newSpeaker = matchSpeakerList[0]
+ nametag = nametag.replace(speaker, newSpeaker)
+ translatedText = re.sub(r"^\[?(.+?)\]?\s?[|:]\s?", "", translatedText)
+
+ # Fix '- '
+ translatedText = translatedText.replace("- ", "-")
+
+ # Textwrap
+ if FIXTEXTWRAP is True:
+ finalJAString = re.sub(r"\n", " ", finalJAString)
+ finalJAString = finalJAString.replace("
", " ")
+
+ if FIXTEXTWRAP is True and "_ABL" in nametag:
+ translatedText = textwrap.fill(translatedText, width=100)
+ elif FIXTEXTWRAP is True:
+ translatedText = textwrap.fill(translatedText, width=WIDTH)
+
+ # BR Flag
+ if BRFLAG is True:
+ translatedText = translatedText.replace("\n", "
")
+
+ # px
+ if "\\px[200]" in nametag:
+ translatedText = translatedText.replace("\\px[200]", "")
+ translatedText = translatedText.replace("\n", "\n\\px[200]")
+
+ ### Add Var Strings
+ # CL Flag
+ if CLFlag:
+ translatedText = "\\ac " + translatedText
+ translatedText = translatedText.replace("\n", "\n\\ac ")
+ translatedText = re.sub(r"[\\]+?ac\s+", r"\\ac ", translatedText)
+ CLFlag = False
+
+ # Nametag
+ if nCase == 0:
+ translatedText = translatedText + nametag
+ else:
+ translatedText = nametag + translatedText
+ nametag = ""
+
+ # Endtag
+ if endtag != "":
+ translatedText = translatedText + endtag
+ endtag = ""
+
+ # Set Data
+ codeList[j]["parameters"] = [translatedText]
+ codeList[j]["code"] = code
+ speaker = ""
+ match = []
+ currentGroup = []
+ syncIndex = i + 1
+ list401.pop(0)
+
+ ## Event Code: 122 [Set Variables]
+ if "code" in codeList[i] and codeList[i]["code"] == 122 and CODE122 is True:
+ # This is going to be the var being set. (IMPORTANT)
+ if codeList[i]["parameters"][0] not in list(range(42, 45)):
+ i += 1
+ continue
+
+ jaString = codeList[i]["parameters"][4]
+
+ # # For Retarded Devs
+ # VNameValue = jaString
+ # i += 1
+ # continue
+
+ # Definitely don't want to mess with files
+ # if 'gameV' in jaString or '_' in jaString:
+ # i += 1
+ # continue
+
+ # Validate String
+ if not isinstance(jaString, str):
+ i += 1
+ continue
+
+ # Set String
+ matchedText = None
+ if len(re.findall(r"([\'\"\`])", jaString)) >= 2:
+ matchedText = re.search(r"[\'\"\`](.*)[\'\"\`]", jaString)
+ # else:
+ # matchedText = re.search(r'(.*)', jaString)
+
+ # Last Check
+ if matchedText.group(1) != None and matchedText.group(1) != " ":
+ # Remove Textwrap
+ finalJAString = matchedText.group(1).replace("\\n", " ")
+
+ # Pass 1
+ if setData == False:
+ if finalJAString != "":
+ list122.append(finalJAString)
+
+ # Pass 2
+ else:
+ if len(list122) > 0:
+ # Grab and Replace
+ translatedText = list122[0]
+ translatedText = jaString.replace(jaString, translatedText)
+
+ # Remove characters that may break scripts
+ charList = ['"', "\\n"]
+ for char in charList:
+ translatedText = translatedText.replace(char, "")
+
+ # Textwrap
+ translatedText = textwrap.fill(translatedText, width=WIDTH)
+ translatedText = translatedText.replace("\n", "\\n")
+
+ # Set
+ codeList[i]["parameters"][4] = f"`{translatedText}`"
+ list122.pop(0)
+
+ ## Event Code: 357 [Picture Text] [Optional]
+ if "code" in codeList[i] and codeList[i]["code"] == 357 and CODE357 is True:
+ headerString = codeList[i]["parameters"][0]
+
+ if headerString == "LL_GalgeChoiceWindow":
+ ### Message Text First
+ jaString = codeList[i]["parameters"][3]["messageText"]
+
+ # Remove any textwrap & TL
+ jaString = re.sub(r"\n", " ", jaString)
+ response = translateGPT(jaString, "", False)
+ translatedText = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Textwrap & Set
+ translatedText = textwrap.fill(translatedText, width=WIDTH)
+ codeList[i]["parameters"][3]["messageText"] = translatedText
+
+ ### Choices
+ jaString = codeList[i]["parameters"][3]["choices"]
+ matchList = re.findall(r'"label[\\]*":[\\]*"(.*?)[\\]', jaString)
+ if matchList != None:
+ # Translate
+ question = codeList[i]["parameters"][3]["messageText"]
+ response = translateGPT(
+ matchList,
+ f"Previous text for context: {question}\n",
+ True,
+ )
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+ translatedText = jaString
+
+ # Replace Strings
+ for j in range(len(matchList)):
+ translatedText = translatedText.replace(matchList[j], response[0][j])
+
+ # Set Data
+ codeList[i]["parameters"][3]["choices"] = translatedText
+
+ if "SoR_GabWindow" in headerString:
+ argVar = "arg1"
+ ### Message Text First
+ if argVar in codeList[i]["parameters"][3]:
+ jaString = codeList[i]["parameters"][3][argVar]
+
+ # If there isn't any Japanese in the text just skip
+ if not re.search(
+ r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+",
+ jaString,
+ ):
+ i += 1
+ continue
+
+ # Remove any textwrap & TL
+ jaString = re.sub(r"\n", " ", jaString)
+ response = translateGPT(jaString, "", False)
+ translatedText = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Textwrap & Set
+ translatedText = textwrap.fill(translatedText, width=WIDTH)
+ codeList[i]["parameters"][3][argVar] = translatedText
+ pbar.update(1)
+
+ if "TorigoyaMZ_NotifyMessage" in headerString:
+ argVar = "message"
+ ### Message Text First
+ if argVar in codeList[i]["parameters"][3]:
+ jaString = codeList[i]["parameters"][3][argVar]
+
+ # If there isn't any Japanese in the text just skip
+ if not re.search(
+ r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+",
+ jaString,
+ ):
+ i += 1
+ continue
+
+ # Remove any textwrap & TL
+ jaString = re.sub(r"\n", " ", jaString)
+ response = translateGPT(jaString, "", False)
+ translatedText = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Textwrap & Set
+ translatedText = textwrap.fill(translatedText, width=WIDTH)
+ codeList[i]["parameters"][3][argVar] = translatedText
+ pbar.update(1)
+
+ if "_TMLogWindowMZ" in headerString:
+ argVar = "text"
+ ### Message Text First
+ if argVar in codeList[i]["parameters"][3]:
+ jaString = codeList[i]["parameters"][3][argVar]
+
+ # If there isn't any Japanese in the text just skip
+ # if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+', jaString):
+ # i += 1
+ # continue
+
+ # Remove any textwrap & TL
+ jaString = re.sub(r"\n", " ", jaString)
+ response = translateGPT(jaString, "", False)
+ translatedText = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Textwrap & Set
+ translatedText = textwrap.fill(translatedText, width=WIDTH)
+ codeList[i]["parameters"][3][argVar] = translatedText
+ pbar.update(1)
+
+ if "DestinationWindow" in headerString:
+ argVar = "destination"
+ ### Message Text First
+ if argVar in codeList[i]["parameters"][3]:
+ jaString = codeList[i]["parameters"][3][argVar]
+
+ # If there isn't any Japanese in the text just skip
+ # if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+', jaString):
+ # i += 1
+ # continue
+
+ # Remove any textwrap & TL
+ jaString = re.sub(r"\n", " ", jaString)
+ response = translateGPT(jaString, "", False)
+ translatedText = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Textwrap & Set
+ translatedText = textwrap.fill(translatedText, width=WIDTH)
+ codeList[i]["parameters"][3][argVar] = translatedText
+ pbar.update(1)
+
+ if "MNKR_CommonPopupCoreMZ" in headerString:
+ argVar = "text"
+ ### Message Text First
+ if argVar in codeList[i]["parameters"][3]:
+ jaString = codeList[i]["parameters"][3][argVar]
+
+ # If there isn't any Japanese in the text just skip
+ # if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+', jaString):
+ # i += 1
+ # continue
+
+ # Remove any textwrap & TL
+ jaString = re.sub(r"\n", " ", jaString)
+ response = translateGPT(jaString, "", False)
+ translatedText = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Textwrap & Set
+ translatedText = textwrap.fill(translatedText, width=WIDTH)
+ codeList[i]["parameters"][3][argVar] = translatedText
+ pbar.update(1)
+
+ if "TextPicture" in headerString or "BalloonInBattle" in headerString:
+ argVar = "text"
+ font = None
+ ### Message Text First
+ if argVar in codeList[i]["parameters"][3]:
+ acExist = False
+ jaString = codeList[i]["parameters"][3][argVar]
+
+ # Check ac
+ if "\\ac" in jaString:
+ acExist = True
+ else:
+ acExist = False
+
+ # If there isn't any Japanese in the text just skip
+ # if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+', jaString):
+ # i += 1
+ # continue
+
+ # Remove any textwrap & TL
+ jaString = re.sub(r"\n", " ", jaString)
+ if acExist:
+ jaString = jaString.replace("\\ac ", " ")
+ jaString = jaString.replace("\\ac", "")
+
+ # Pass 1
+ if setData == False:
+ list357.append(jaString)
+
+ # Pass 2
+ else:
+ if len(list357) > 0:
+ # Grab and Replace
+ translatedText = list357[0]
+ translatedText = jaString.replace(jaString, translatedText)
+
+ # Remove characters that may break scripts
+ charList = ['"', "\\n"]
+ for char in charList:
+ translatedText = translatedText.replace(char, "")
+
+ # Textwrap
+ translatedText = textwrap.fill(translatedText, WIDTH)
+ translatedText = translatedText.replace("- ", "-")
+
+ # Center Text
+ if acExist:
+ translatedText = f'\\ac {translatedText.replace('\n', '\n\\ac ')}'
+
+ # Check and Set Font
+ if "fontSize" in codeList[i]["parameters"][3]:
+ if font:
+ codeList[i]["parameters"][3]["fontSize"] = font
+
+ # Set
+ codeList[i]["parameters"][3][argVar] = translatedText
+ list357.pop(0)
+
+ ## Event Code: 657 [Picture Text] [Optional]
+ if "code" in codeList[i] and codeList[i]["code"] == 657 and CODE657 is True:
+ if "text" in codeList[i]["parameters"][0]:
+ jaString = codeList[i]["parameters"][0]
+ if not isinstance(jaString, str):
+ i += 1
+ continue
+
+ # Definitely don't want to mess with files
+ if "_" in jaString:
+ i += 1
+ continue
+
+ # If there isn't any Japanese in the text just skip
+ if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+", jaString):
+ i += 1
+ continue
+
+ # Remove outside text
+ startString = re.search(r"^[^一-龠ぁ-ゔァ-ヴー\<\>【】\\]+", jaString)
+ jaString = re.sub(r"^[^一-龠ぁ-ゔァ-ヴー\<\>【】\\]+", "", jaString)
+ endString = re.search(r"[^一-龠ぁ-ゔァ-ヴー\<\>【】。！？\\]+$", jaString)
+ jaString = re.sub(r"[^一-龠ぁ-ゔァ-ヴー\<\>【】。！？\\]+$", "", jaString)
+ if startString is None:
+ startString = ""
+ else:
+ startString = startString.group()
+ if endString is None:
+ endString = ""
+ else:
+ endString = endString.group()
+
+ # Remove any textwrap
+ jaString = re.sub(r"\n", " ", jaString)
+
+ # Translate
+ response = translateGPT(jaString, "", True)
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+ translatedText = response[0]
+
+ # Remove characters that may break scripts
+ charList = [".", '"', "'"]
+ for char in charList:
+ translatedText = translatedText.replace(char, "")
+
+ # Textwrap
+ translatedText = textwrap.fill(translatedText, width=WIDTH)
+ translatedText = startString + translatedText + endString
+
+ # Set Data
+ codeList[i]["parameters"][0] = translatedText
+
+ ## Event Code: 101 [Name] [Optional]
+ if "code" in codeList[i] and codeList[i]["code"] == 101 and CODE101 is True:
+ # Check Window Type (Certain games switch between 1st line speakers and none)
+ if FIRSTLINESPEAKERS:
+ if codeList[i]["parameters"][2] == 0:
+ speakerWindow = True
+ else:
+ speakerWindow = False
+
+ else:
+ isVar = False
+
+ # Grab String
+ jaString = ""
+ if len(codeList[i]["parameters"]) > 4:
+ jaString = codeList[i]["parameters"][4]
+ # Check for Var
+ elif len(codeList[i]["parameters"]) > 0:
+ jaString = codeList[i]["parameters"][0]
+ isVar = True
+ if not isinstance(jaString, str):
+ i += 1
+ continue
+
+ # Force Speaker using var
+ if "\\ap[1左]" in jaString.lower() or "\\ap[1右]" in jaString.lower():
+ speaker = "Cecily"
+ i += 1
+ continue
+ elif "\\ap[2左]" in jaString.lower() or "\\ap[2右]" in jaString.lower():
+ speaker = "Amelia"
+ i += 1
+ continue
+ elif "\\ap[3左]" in jaString.lower() or "\\ap[3右]" in jaString.lower():
+ speaker = "Henry"
+ i += 1
+ continue
+ elif "\\ap[4左]" in jaString.lower() or "\\ap[4右]" in jaString.lower():
+ speaker = "Oswald"
+ i += 1
+ continue
+ elif "\\ap" in jaString:
+ speaker = re.search(r"[\\]+AP\[(.*?)\]", jaString).group(1)
+ i += 1
+ continue
+
+ # Get Speaker
+ if "\\" not in jaString:
+ response = getSpeaker(jaString)
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+ speaker = response[0]
+
+ # Validate Speaker is not empty
+ if len(speaker) > 0:
+ if isVar == False:
+ codeList[i]["parameters"][4] = speaker
+ i += 1
+ continue
+ else:
+ codeList[i]["parameters"][0] = speaker
+ isVar = False
+ i += 1
+ continue
+ else:
+ speaker = ""
+
+ ## Event Code: 355 or 655 Scripts [Optional]
+ if "code" in codeList[i] and (codeList[i]["code"] == 355 or codeList[i]["code"] == 655) and CODE355655 is True:
+ jaString = codeList[i]["parameters"][0]
+ regex = r'.*subject=(.*?)"'
+
+ # Var Text
+ match = re.search(regex, jaString)
+ if re.search(regex, jaString):
+ finalJAString = match.group(1)
+ # Pass 1
+ if setData is False:
+ list355655.append(finalJAString)
+
+ # Pass 2
+ else:
+ # Grab and Replace
+ translatedText = list355655[0]
+ translatedText = translatedText.replace("'", "\\'")
+
+ # Set
+ codeList[i]["parameters"][0] = codeList[i]["parameters"][0].replace(finalJAString, translatedText)
+ list355655.pop(0)
+
+ ## Event Code: 408 (Script)
+ if "code" in codeList[i] and (codeList[i]["code"] == 408) and CODE408 is True:
+ jaString = codeList[i]["parameters"][0]
+
+ # If there isn't any Japanese in the text just skip
+ if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+", jaString):
+ i += 1
+ continue
+
+ if "secretText" in jaString:
+ regex = r"secretText:\s?(.+)"
+ elif "title" in jaString:
+ regex = r"title:\s?(.+)"
+ else:
+ regex = r"(.+)"
+
+ # Need to remove outside code and put it back later
+ matchList = re.findall(regex, jaString)
+
+ for match in matchList:
+ # Remove Textwrap
+ match = match.replace("\n", " ")
+ response = translateGPT(
+ match,
+ "Reply with the " + LANGUAGE + " translation of the achievement title.",
+ False,
+ )
+ translatedText = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Replace
+ translatedText = jaString.replace(match, translatedText)
+
+ # Remove characters that may break scripts
+ charList = [".", '"', "\\n"]
+ for char in charList:
+ translatedText = translatedText.replace(char, "")
+
+ # Textwrap
+ translatedText = textwrap.fill(translatedText, width=WIDTH)
+
+ # Set Data
+ codeList[i]["parameters"][0] = translatedText
+
+ ## Event Code: 108 (Script)
+ if "code" in codeList[i] and (codeList[i]["code"] == 108) and CODE108 is True:
+ jaString = codeList[i]["parameters"][0]
+
+ # If there isn't any Japanese in the text just skip
+ if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+", jaString):
+ i += 1
+ continue
+
+ # Translate
+ if "info:" in jaString:
+ regex = r"info:(.*)"
+ elif "ActiveMessage:" in jaString:
+ regex = r"?"
+ elif "event_text" in jaString:
+ regex = r"event_text\s*:\s*(.*)"
+ else:
+ i += 1
+ continue
+
+ # Need to remove outside code and put it back later
+ match = re.search(regex, jaString)
+ if match:
+ # Pass 1
+ if setData is False:
+ list108.append(match.group(1))
+
+ # Grab Next
+ j = i
+ while codeList[j + 1]["code"] == 408:
+ j += 1
+ list108[0] = list108[0] + codeList[j]["parameters"][0].replace(">", "")
+ codeList[j]["parameters"][0] = ""
+ list108[0] = list108[0].replace("\n", " ")
+
+ # Pass 2
+ else:
+ # Grab and Replace
+ translatedText = list108[0]
+ list108.pop(0)
+
+ # Textwrap
+ if codeList[i + 1]["code"] == 408:
+ translatedText = textwrap.fill(translatedText, WIDTH)
+
+ # Remove characters that may break scripts
+ charList = ['"']
+ for char in charList:
+ translatedText = translatedText.replace(char, "")
+ translatedText = translatedText.replace('"', '"')
+ translatedText = translatedText.replace(" ", "_")
+ translatedText = jaString.replace(match.group(1), translatedText)
+
+ # Add >
+ if ">" not in translatedText:
+ translatedText = translatedText + ">"
+
+ # Set Data
+ codeList[i]["parameters"][0] = translatedText
+
+ ## Event Code: 356
+ if "code" in codeList[i] and codeList[i]["code"] == 356 and CODE356 is True:
+ jaString = codeList[i]["parameters"][0]
+ oldjaString = jaString
+
+ # Grab Speaker
+ if "Tachie showName" in jaString:
+ matchList = re.findall(r"Tachie showName (.+)", jaString)
+ if len(matchList) > 0:
+ # Translate
+ response = translateGPT(
+ matchList[0],
+ "Reply with the " + LANGUAGE + " translation of the NPC name.",
+ False,
+ )
+ translatedText = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Set Text
+ speaker = translatedText
+ speaker = speaker.replace(" ", " ")
+ codeList[i]["parameters"][0] = jaString.replace(matchList[0], speaker)
+ i += 1
+ continue
+
+ # Want to translate this script
+ if "D_TEXT " in jaString:
+ regex = r"D_TEXT\s(.*?)\s.+"
+ elif "ShowInfo" in jaString:
+ regex = r"ShowInfo\s(.*)"
+ elif "PushGab" in jaString:
+ regex = r"PushGab\s(.*)"
+ elif "addLog" in jaString:
+ regex = r"addLog\s(.*)"
+ elif "DW_" in jaString:
+ regex = r"DW_.*?\s(.*)"
+ elif "CommonPopup" in jaString:
+ regex = r"CommonPopup\sadd\stext:(.*?)[\\]+}"
+ else:
+ regex = r""
+
+ # Remove any textwrap
+ jaString = re.sub(r"\n", "_", jaString)
+
+ # Capture Arguments and text
+ textMatch = re.search(regex, jaString)
+ if textMatch and textMatch.group(0) != "":
+ text = textMatch.group(1)
+
+ # Pass 1
+ if setData == False:
+ text = text.replace("_", " ")
+ list356.append(text)
+
+ # Pass 2
+ else:
+ if len(list356) > 0:
+ # Grab
+ translatedText = list356[0]
+
+ # Remove characters that may break scripts
+ charList = [".", '"']
+ for char in charList:
+ translatedText = translatedText.replace(char, "")
+
+ # Cant have spaces?
+ translatedText = translatedText.replace(" ", "_")
+ translatedText = translatedText.replace("__", "_")
+
+ # Put Args Back
+ translatedText = jaString.replace(text, translatedText)
+
+ # Set Data
+ codeList[i]["parameters"][0] = translatedText
+ list356.pop(0)
+
+ if "namePop" in jaString:
+ matchList = re.findall(r"namePop\s\d+\s(.+?)\s.+", jaString)
+ if len(matchList) > 0:
+ # Translate
+ text = matchList[0]
+ response = translateGPT(text, "Reply with the " + LANGUAGE + " Translation", False)
+ translatedText = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Set Data
+ translatedText = jaString.replace(text, translatedText)
+ codeList[i]["parameters"][0] = translatedText
+
+ if "LL_InfoPopupWIndowMV" in jaString:
+ matchList = re.findall(r"LL_InfoPopupWIndowMV\sshowWindow\s(.+?) .+", jaString)
+ if len(matchList) > 0:
+ # Translate
+ text = matchList[0]
+ response = translateGPT(text, "Reply with the " + LANGUAGE + " Translation", False)
+ translatedText = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Set Data
+ translatedText = translatedText.replace(" ", "_")
+ translatedText = jaString.replace(text, translatedText)
+ codeList[i]["parameters"][0] = translatedText
+
+ if "OriginMenuStatus SetParam" in jaString:
+ matchList = re.findall(r"OriginMenuStatus\sSetParam\sparam[\d]\s(.*)", jaString)
+ if len(matchList) > 0:
+ # Translate
+ text = matchList[0]
+ response = translateGPT(text, "Reply with the " + LANGUAGE + " Translation", False)
+ translatedText = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Set Data
+ translatedText = translatedText.replace(" ", "_")
+ translatedText = jaString.replace(text, translatedText)
+ codeList[i]["parameters"][0] = translatedText
+
+ # LL_GalgeChoiceWindowMV Message
+ if "LL_GalgeChoiceWindowMV setMessageText" in jaString:
+ ### Message Text First
+ match = re.search(r"LL_GalgeChoiceWindowMV setMessageText (.+)", jaString)
+ if match:
+ jaString = match.group(1)
+
+ # Remove any textwrap & TL
+ jaString = re.sub(r"\n", " ", jaString)
+ response = translateGPT(jaString, "", False)
+ translatedText = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Textwrap & Replace Whitespace
+ translatedText = textwrap.fill(translatedText, width=WIDTH)
+ translatedText = translatedText.replace(" ", "_")
+
+ # Replace and Set
+ translatedText = match.group(0).replace(match.group(1), translatedText)
+ codeList[i]["parameters"][0] = translatedText
+
+ # LL_GalgeChoiceWindowMV Choices
+ if "LL_GalgeChoiceWindowMV setChoices":
+ match = re.search(r"LL_GalgeChoiceWindowMV setChoices (.+)", jaString)
+ if match:
+ jaString = match.group(1)
+ choiceList = jaString.split(",")
+
+ # Translate
+ question = translatedText
+ response = translateGPT(
+ choiceList,
+ f"Previous text for context: {question}\n",
+ True,
+ )
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+ choiceListTL = response[0]
+ translatedText = match.group(0)
+
+ # Replace Strings
+ for j in range(len(choiceListTL)):
+ choiceListTL[j] = choiceListTL[j].replace(" ", "_")
+ translatedText = translatedText.replace(choiceList[j], choiceListTL[j])
+
+ # Set Data
+ codeList[i]["parameters"][0] = translatedText
+
+ ### Event Code: 102 Show Choice
+ if "code" in codeList[i] and codeList[i]["code"] == 102 and CODE102 is True:
+ choiceList = []
+ varList = []
+ for choice in range(len(codeList[i]["parameters"][0])):
+ jaString = codeList[i]["parameters"][0][choice]
+ jaString = jaString.replace(" 。", ".")
+
+ # Avoid Empty Strings
+ if jaString == "":
+ continue
+
+ # If and En Statements
+ ifVar = ""
+ ifList = re.findall(r"([ei][nf]\(.+?\)\)?\)?)", jaString)
+ if len(ifList) != 0:
+ for var in ifList:
+ jaString = jaString.replace(var, "")
+ ifVar += var
+ varList.append(ifVar)
+
+ # Append to List
+ choiceList.append(jaString)
+
+ # Translate
+ if len(textHistory) > 0:
+ response = translateGPT(
+ choiceList,
+ f"Previous text for context: {str(textHistory)}\n",
+ True,
+ )
+ translatedTextList = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+ else:
+ response = translateGPT(choiceList, "", True)
+ translatedTextList = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Check Mismatch
+ if len(translatedTextList) == len(choiceList):
+ for choice in range(len(codeList[i]["parameters"][0])):
+ jaString = codeList[i]["parameters"][0][choice]
+ jaString = jaString.replace(" 。", ".")
+
+ # Avoid Empty Strings
+ if jaString == "":
+ continue
+
+ translatedText = translatedTextList[choice]
+
+ # Set Data
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+ if translatedText != "":
+ translatedText = varList[choice] + translatedText[0].upper() + translatedText[1:]
+ else:
+ translatedText = varList[choice] + translatedText
+ codeList[i]["parameters"][0][choice] = translatedText
+ else:
+ if filename not in MISMATCH:
+ MISMATCH.append(filename)
+
+ ### Event Code: 111 Script
+ if "code" in codeList[i] and codeList[i]["code"] == 111 and CODE111 is True:
+ for j in range(len(codeList[i]["parameters"])):
+ jaString = codeList[i]["parameters"][j]
+
+ # Check if String
+ if not isinstance(jaString, str):
+ i += 1
+ continue
+
+ # Only TL the Game Variable
+ if "$gameVariables" not in jaString:
+ i += 1
+ continue
+
+ # This is going to be the var being set. (IMPORTANT)
+ if "1045" not in jaString:
+ i += 1
+ continue
+
+ # Need to remove outside code and put it back later
+ matchList = re.findall(r"'(.*?)'", jaString)
+
+ for match in matchList:
+ response = translateGPT(match, "", False)
+ translatedText = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Remove characters that may break scripts
+ charList = [".", '"', "'", "\\n"]
+ for char in charList:
+ translatedText = translatedText.replace(char, "")
+
+ jaString = jaString.replace(match, translatedText)
+
+ # Set Data
+ translatedText = jaString
+ codeList[i]["parameters"][j] = translatedText
+
+ ### Event Code: 320 Set Variable
+ if "code" in codeList[i] and codeList[i]["code"] == 320 and CODE320 is True:
+ jaString = codeList[i]["parameters"][1]
+ if not isinstance(jaString, str):
+ i += 1
+ continue
+
+ # Definitely don't want to mess with files
+ if "■" in jaString or "_" in jaString:
+ i += 1
+ continue
+
+ # If there isn't any Japanese in the text just skip
+ if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+", jaString):
+ i += 1
+ continue
+
+ # Translate
+ response = getSpeaker(jaString)
+ translatedText = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+
+ # Remove characters that may break scripts
+ charList = [".", '"', "'", "\\n"]
+ for char in charList:
+ translatedText = translatedText.replace(char, "")
+
+ # Set Data
+ codeList[i]["parameters"][1] = translatedText
+
+ # Iterate
+ else:
+ i += 1
+
+ # EOF
+ list401TL = []
+ list122TL = []
+ list356TL = []
+ list357TL = []
+ list355655TL = []
+ list108TL = []
+ setData = False
+ PBAR = pbar
+
+ # 401
+ if len(list401) > 0:
+ response = translateGPT(list401, "", True)
+ list401TL = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+ if len(list401TL) != len(list401):
+ with LOCK:
+ if filename not in MISMATCH:
+ MISMATCH.append(filename)
+ else:
+ setData = True
+
+ # 122
+ if len(list122) > 0:
+ response = translateGPT(list122, "Keep your translation as brief as possible", True)
+ list122TL = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+ if len(list122TL) != len(list122):
+ with LOCK:
+ if filename not in MISMATCH:
+ MISMATCH.append(filename)
+ else:
+ setData = True
+
+ # 355/655
+ if len(list355655) > 0:
+ response = translateGPT(list355655, textHistory, True)
+ list355655TL = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+ if len(list355655TL) != len(list355655):
+ with LOCK:
+ if filename not in MISMATCH:
+ MISMATCH.append(filename)
+ else:
+ setData = True
+
+ # 108
+ if len(list108) > 0:
+ response = translateGPT(list108, textHistory, True)
+ list108TL = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+ if len(list108TL) != len(list108):
+ with LOCK:
+ if filename not in MISMATCH:
+ MISMATCH.append(filename)
+ else:
+ setData = True
+
+ # 356
+ if len(list356) > 0:
+ response = translateGPT(list356, textHistory, True)
+ list356TL = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+ if len(list356TL) != len(list356):
+ with LOCK:
+ if filename not in MISMATCH:
+ MISMATCH.append(filename)
+ else:
+ setData = True
+
+ # 357
+ if len(list357) > 0:
+ response = translateGPT(list357, textHistory, True)
+ list357TL = response[0]
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+ if len(list357TL) != len(list357):
+ with LOCK:
+ if filename not in MISMATCH:
+ MISMATCH.append(filename)
+ else:
+ setData = True
+
+ # Start Pass 2
+ if setData:
+ searchCodes(
+ page,
+ pbar,
+ [list401TL, list122TL, list355655TL, list108TL, list356TL, list357TL],
+ filename,
+ )
+
+ # Delete all -1 codes
+ codeListFinal = []
+ for i in range(len(codeList)):
+ if "code" in codeList[i] and codeList[i]["code"] != -1:
+ codeListFinal.append(codeList[i])
+
+ # Normal Format
+ if "list" in page:
+ page["list"] = codeListFinal
+
+ # Special Format (Scenario)
+ else:
+ page = codeListFinal
+
+ except IndexError as e:
+ traceback.print_exc()
+ raise Exception(str(e) + "Failed to translate: " + oldjaString) from None
+ except Exception as e:
+ traceback.print_exc()
+ raise Exception(str(e) + "Failed to translate: " + oldjaString) from None
+
+ return totalTokens
+
+
+def searchSS(state, pbar):
+ totalTokens = [0, 0]
+
+ # Name
+ nameResponse = (
+ translateGPT(
+ state["name"],
+ "Reply with only the " + LANGUAGE + " translation of the RPG Skill name.",
+ False,
+ )
+ if "name" in state
+ else ""
+ )
+
+ # Description
+ descriptionResponse = (
+ translateGPT(
+ state["description"],
+ "Reply with only the " + LANGUAGE + " translation of the description.",
+ False,
+ )
+ if "description" in state
+ else ""
+ )
+
+ # Messages
+ message1Response = ""
+ message4Response = ""
+ message2Response = ""
+ message3Response = ""
+
+ if "message1" in state:
+ if len(state["message1"]) > 0 and state["message1"][0] in [
+ "は",
+ "を",
+ "の",
+ "に",
+ "が",
+ ]:
+ message1Response = translateGPT(
+ "Taro" + state["message1"],
+ "reply with only the gender neutral "
+ + LANGUAGE
+ + " translation of the action log. Always start the sentence with Taro. For example,\
+Translate 'Taroを倒した！' as 'Taro was defeated!'",
+ False,
+ )
+ else:
+ message1Response = translateGPT(
+ state["message1"],
+ "reply with only the gender neutral " + LANGUAGE + " translation",
+ False,
+ )
+
+ if "message2" in state:
+ if len(state["message2"]) > 0 and state["message2"][0] in [
+ "は",
+ "を",
+ "の",
+ "に",
+ "が",
+ ]:
+ message2Response = translateGPT(
+ "Taro" + state["message2"],
+ "reply with only the gender neutral "
+ + LANGUAGE
+ + " translation of the action log. Always start the sentence with Taro. For example,\
+Translate 'Taroを倒した！' as 'Taro was defeated!'",
+ False,
+ )
+ else:
+ message2Response = translateGPT(
+ state["message2"],
+ "reply with only the gender neutral " + LANGUAGE + " translation",
+ False,
+ )
+
+ if "message3" in state:
+ if len(state["message3"]) > 0 and state["message3"][0] in [
+ "は",
+ "を",
+ "の",
+ "に",
+ "が",
+ ]:
+ message3Response = translateGPT(
+ "Taro" + state["message3"],
+ "reply with only the gender neutral "
+ + LANGUAGE
+ + " translation of the action log. Always start the sentence with Taro. For example,\
+Translate 'Taroを倒した！' as 'Taro was defeated!'",
+ False,
+ )
+ else:
+ message3Response = translateGPT(
+ state["message3"],
+ "reply with only the gender neutral " + LANGUAGE + " translation",
+ False,
+ )
+
+ if "message4" in state:
+ if len(state["message4"]) > 0 and state["message4"][0] in [
+ "は",
+ "を",
+ "の",
+ "に",
+ "が",
+ ]:
+ message4Response = translateGPT(
+ "Taro" + state["message4"],
+ "reply with only the gender neutral "
+ + LANGUAGE
+ + " translation of the action log. Always start the sentence with Taro. For example,\
+Translate 'Taroを倒した！' as 'Taro was defeated!'",
+ False,
+ )
+ else:
+ message4Response = translateGPT(
+ state["message4"],
+ "reply with only the gender neutral " + LANGUAGE + " translation",
+ False,
+ )
+
+ # Translate State Notes
+ if "help" in state["note"]:
+ noteResponse = translateNote(state, r"]*)>")
+ totalTokens[0] += noteResponse[0]
+ totalTokens[1] += noteResponse[1]
+ if "STATE_HELP" in state["note"]:
+ noteResponse = translateNote(state, r"\n(.*)\n")
+ totalTokens[0] += noteResponse[0]
+ totalTokens[1] += noteResponse[1]
+
+ # Count totalTokens
+ totalTokens[0] += nameResponse[1][0] if nameResponse != "" else 0
+ totalTokens[1] += nameResponse[1][1] if nameResponse != "" else 0
+ totalTokens[0] += descriptionResponse[1][0] if descriptionResponse != "" else 0
+ totalTokens[1] += descriptionResponse[1][1] if descriptionResponse != "" else 0
+ totalTokens[0] += message1Response[1][0] if message1Response != "" else 0
+ totalTokens[1] += message1Response[1][1] if message1Response != "" else 0
+ totalTokens[0] += message2Response[1][0] if message2Response != "" else 0
+ totalTokens[1] += message2Response[1][1] if message2Response != "" else 0
+ totalTokens[0] += message3Response[1][0] if message3Response != "" else 0
+ totalTokens[1] += message3Response[1][1] if message3Response != "" else 0
+ totalTokens[0] += message4Response[1][0] if message4Response != "" else 0
+ totalTokens[1] += message4Response[1][1] if message4Response != "" else 0
+
+ # Set Data
+ if "name" in state:
+ state["name"] = nameResponse[0].replace('"', "")
+ if "description" in state:
+ # Textwrap
+ translatedText = descriptionResponse[0]
+ translatedText = textwrap.fill(translatedText, width=LISTWIDTH)
+ state["description"] = translatedText.replace('"', "")
+ if "message1" in state:
+ state["message1"] = message1Response[0].replace('"', "").replace("Taro", "")
+ if "message2" in state:
+ state["message2"] = message2Response[0].replace('"', "").replace("Taro", "")
+ if "message3" in state:
+ state["message3"] = message3Response[0].replace('"', "").replace("Taro", "")
+ if "message4" in state:
+ state["message4"] = message4Response[0].replace('"', "").replace("Taro", "")
+
+ return totalTokens
+
+
+def searchSystem(data, pbar):
+ totalTokens = [0, 0]
+ context = "Reply with only the " + LANGUAGE + ' translation of the UI textbox."'
+
+ # Title
+ response = translateGPT(
+ data["gameTitle"],
+ " Reply with the " + LANGUAGE + " translation of the game title name",
+ False,
+ )
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+ data["gameTitle"] = response[0].strip(".")
+
+ # Terms
+ for term in data["terms"]:
+ if term != "messages":
+ termList = data["terms"][term]
+ for i in range(len(termList)): # Last item is a messages object
+ if termList[i] is not None:
+ response = translateGPT(termList[i], context, False)
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+ termList[i] = response[0].replace('"', "").strip()
+
+ # Armor Types
+ for i in range(len(data["armorTypes"])):
+ response = translateGPT(
+ data["armorTypes"][i],
+ "Reply with only the " + LANGUAGE + " translation of the armor type",
+ False,
+ )
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+ data["armorTypes"][i] = response[0].replace('"', "").strip()
+
+ # Skill Types
+ for i in range(len(data["skillTypes"])):
+ response = translateGPT(
+ data["skillTypes"][i],
+ "Reply with only the " + LANGUAGE + " translation",
+ False,
+ )
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+ data["skillTypes"][i] = response[0].replace('"', "").strip()
+
+ # Equip Types
+ for i in range(len(data["equipTypes"])):
+ response = translateGPT(
+ data["equipTypes"][i],
+ "Reply with only the " + LANGUAGE + " translation of the equipment type. No disclaimers.",
+ False,
+ )
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+ data["equipTypes"][i] = response[0].replace('"', "").strip()
+
+ # # Variables (Optional ususally)
+ # for i in range(len(data['variables'])):
+ # response = translateGPT(data['variables'][i], 'Reply with only the '+ LANGUAGE +' translation of the title', False)
+ # totalTokens[0] += response[1][0]
+ # totalTokens[1] += response[1][1]
+ # data['variables'][i] = response[0].replace('\"', '').strip()
+
+ # Messages
+ messages = data["terms"]["messages"]
+ for key, value in messages.items():
+ response = translateGPT(
+ value,
+ "Reply with only the "
+ + LANGUAGE
+ + ' translation of the battle text.\nTranslate "常時ダッシュ" as "Always Dash"\nTranslate "次の%1まで" as Next %1.',
+ False,
+ )
+ translatedText = response[0]
+
+ # Remove characters that may break scripts
+ charList = [".", '"', "\\n"]
+ for char in charList:
+ translatedText = translatedText.replace(char, "")
+
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+ messages[key] = translatedText
+
+ return totalTokens
+
+
+# Save some money and enter the character before translation
+def getSpeaker(speaker):
+ match speaker:
+ case "ファイン":
+ return ["Fine", [0, 0]]
+ case "":
+ return ["", [0, 0]]
+ case _:
+ # Find Speaker
+ for i in range(len(NAMESLIST)):
+ if speaker == NAMESLIST[i][0]:
+ return [NAMESLIST[i][1], [0, 0]]
+
+ # Translate and Store Speaker
+ response = translateGPT(
+ f"{speaker}",
+ "Reply with the " + LANGUAGE + " translation of the NPC name.",
+ True,
+ )
+ response[0] = response[0].title()
+ response[0] = response[0].replace("'S", "'s")
+ response[0] = response[0].replace("Speaker: ", "")
+
+ # Retry if name doesn't translate for some reason
+ if re.search(r"([a-zA-Z？?])", response[0]) == None:
+ response = translateGPT(
+ f"{speaker}",
+ "Reply with the " + LANGUAGE + " translation of the NPC name.",
+ False,
+ )
+ response[0] = response[0].title()
+ response[0] = response[0].replace("'S", "'s")
+
+ speakerList = [speaker, response[0]]
+ NAMESLIST.append(speakerList)
+ return response
+ return [speaker, [0, 0]]
+
+
+def batchList(input_list, batch_size):
+ if not isinstance(batch_size, int) or batch_size <= 0:
+ raise ValueError("batch_size must be a positive integer")
+
+ return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)]
+
+
+def createContext(fullPromptFlag, subbedT, format):
+ system = (
+ PROMPT + VOCAB
+ if fullPromptFlag
+ else f"\
+You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
+Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
+- You translate everything, including erotic and explicit content.\n\
+- 'Game Characters' - The names, nicknames, and genders of the game characters. Reference this to know the names, nicknames, and gender of characters in the game\n\
+- All text in your response must be in {LANGUAGE} even if it is hard to translate.\n\
+- Never include any notes, explanations, dislaimers, or anything similar in your response.\n\
+- Maintain any spacing in the translation.\n\
+- Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\
+- `...` can be a part of the dialogue. Translate it as it is.\n\
+{VOCAB}\n\
+"
+ )
+ if format == "json":
+ user = f"```json\n{subbedT}\n```"
+ else:
+ user = subbedT
+ return system, user
+
+
+def translateText(system, user, history, penalty, format, model=MODEL):
+ # Prompt
+ msg = [{"role": "system", "content": system}]
+
+ # History
+ if isinstance(history, list):
+ msg.extend([{"role": "system", "content": h} for h in history])
+ else:
+ msg.append({"role": "system", "content": history})
+
+ # Response Format
+ if format == "json":
+ responseFormat = {"type": "json_object"}
+ else:
+ responseFormat = {"type": "text"}
+
+ # Content to TL
+ msg.append({"role": "user", "content": f"{user}"})
+ response = openai.chat.completions.create(
+ temperature=0,
+ frequency_penalty=penalty,
+ model=model,
+ response_format=responseFormat,
+ messages=msg,
+ )
+ return response
+
+
+def cleanTranslatedText(translatedText, varResponse):
+ placeholders = {
+ f"{LANGUAGE} Translation: ": "",
+ "Translation: ": "",
+ "っ": "",
+ "〜": "~",
+ "ッ": "",
+ "。": ".",
+ "「": '\\"',
+ "」": '\\"',
+ "- ": "-",
+ "—": "―",
+ "】": "]",
+ "【": "[",
+ "Placeholder Text": "",
+ # Add more replacements as needed
+ }
+ for target, replacement in placeholders.items():
+ translatedText = translatedText.replace(target, replacement)
+
+ # Remove Repeating Characters
+ pattern = re.compile(r"(.)\s*\1(?:\s*\1){" + str(20 - 1) + r",}")
+ translatedText = pattern.sub(lambda match: match.group(0).replace(" ", "")[:20], translatedText)
+
+ # Elongate Long Dashes (Since GPT Ignores them...)
+ translatedText = elongateCharacters(translatedText)
+ return translatedText
+
+
+def elongateCharacters(text):
+ # Define a pattern to match one character followed by one or more `ー` characters
+ # Using a positive lookbehind assertion to capture the preceding character
+ pattern = r"(?<=(.))ー+"
+
+ # Define a replacement function that elongates the captured character
+ def repl(match):
+ char = match.group(1) # The character before the ー sequence
+ count = len(match.group(0)) - 1 # Number of ー characters
+ return char * count # Replace ー sequence with the character repeated
+
+ # Use re.sub() to replace the pattern in the text
+ return re.sub(pattern, repl, text)
+
+
+def extractTranslation(translatedTextList, is_list):
+ try:
+ translatedTextList = re.sub(r'\\"+\"([^,\n}])', r'\\"\1', translatedTextList)
+ translatedTextList = re.sub(r"(? instead
-FIXTEXTWRAP = True # Overwrites textwrap
-IGNORETLTEXT = False # Ignores all translated text.
-MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
-
-# tqdm Globals
-BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
-POSITION = 0
-LEAVE = False
-PBAR = None
-
-# Pricing - Depends on the model https://openai.com/pricing
-# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request
-# If you are getting a MISMATCH LENGTH error, lower the batch size.
-if "gpt-3.5" in MODEL:
- INPUTAPICOST = 0.002
- OUTPUTAPICOST = 0.002
- BATCHSIZE = 10
-elif "gpt-4" in MODEL:
- INPUTAPICOST = 0.0025
- OUTPUTAPICOST = 0.01
- BATCHSIZE = 40
-else:
- INPUTAPICOST = float(os.getenv("input_cost"))
- OUTPUTAPICOST = float(os.getenv("output_cost"))
- BATCHSIZE = int(os.getenv("batchsize"))
- FREQUENCY_PENALTY = float(os.getenv("frequency_penalty"))
-
-def handlePlugin(filename, estimate):
- global ESTIMATE, PBAR
- ESTIMATE = estimate
-
- if ESTIMATE:
- start = time.time()
- translatedData = openFiles(filename)
-
- # Print Result
- end = time.time()
- tqdm.write(getResultString(translatedData, end - start, filename))
- with LOCK:
- TOKENS[0] += translatedData[1][0]
- TOKENS[1] += translatedData[1][1]
-
- # Print Total
- totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
-
- # Print any errors on maps
- if len(MISMATCH) > 0:
- return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
- else:
- return totalString
-
- else:
- try:
- with open("translated/" + filename, "w", encoding="utf_8", errors="ignore") as outFile:
- start = time.time()
- translatedData = openFiles(filename)
-
- # Print Result
- end = time.time()
- outFile.writelines(translatedData[0])
- tqdm.write(getResultString(translatedData, end - start, filename))
- with LOCK:
- TOKENS[0] += translatedData[1][0]
- TOKENS[1] += translatedData[1][1]
- except Exception:
- traceback.print_exc()
- return "Fail"
-
- return getResultString(["", TOKENS, None], end - start, "TOTAL")
-
-
-def getResultString(translatedData, translationTime, filename):
- # File Print String
- totalTokenstring = (
- Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
- "[Output: "
- + str(translatedData[1][1])
- + "]" "[Cost: ${:,.4f}".format((translatedData[1][0] * 0.001 * INPUTAPICOST) + (translatedData[1][1] * 0.001 * OUTPUTAPICOST))
- + "]"
- )
- timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
-
- if translatedData[2] == None:
- # Success
- return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET
-
- else:
- # Fail
- try:
- raise translatedData[2]
- except Exception as e:
- traceback.print_exc()
- errorString = str(e) + Fore.RED
- return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET
-
-
-def openFiles(filename):
- with open("files/" + filename, "r", encoding="utf_8") as readFile:
- translatedData = parsePlugin(readFile, filename)
-
- # Delete lines marked for deletion
- finalData = []
- for line in translatedData[0]:
- if line != "\\d\n":
- finalData.append(line)
- translatedData[0] = finalData
-
- return translatedData
-
-
-def parsePlugin(readFile, filename):
- totalTokens = [0, 0]
-
- # Read File into data
- data = readFile.readlines()
-
- # Create Progress Bar
- with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
- pbar.desc = filename
-
- try:
- result = translatePlugin(data, pbar, filename, [])
- totalTokens[0] += result[0]
- totalTokens[1] += result[1]
- except Exception as e:
- traceback.print_exc()
- return [data, totalTokens, e]
- return [data, totalTokens, None]
-
-
-def translatePlugin(data, pbar, filename, translatedList):
- stringList = []
- currentGroup = []
- tokens = [0, 0]
- speaker = ""
- voice = False
- global LOCK, ESTIMATE
- i = 0
-
- while i < len(data):
- voice = False
- speaker = ""
- newline = r"\n"
-
- """
- Plugin List
- Quest Name: [\\]+"QuestName[\\]+":[\\]+"(.*?)[\\]+"
- Quest Client: [\\]+"QuestClientName[\\]+":[\\]+"[\\]+"[\\]+"(.*?)[\\]+"
- Quest Location: [\\]+"QuestionLocation[\\]+":[\\]+"[\\]+"[\\]+"(.*?)[\\]+"
- Quest Targe Location: [\\]+"PlaceInformation[\\]+":[\\]+"(.*?)[\\]+"
- Quest Summary: [\\]+"QuestContent[\\]+":[\\]+"(.*?)[\\]+"
- Quest Goal: [\\]+"ObjectiveContent[\\]+":[\\]+"[\\]+"[\\]+"(.*?)[\\]+"
- Quest Goal 2: [\\]+"ObjectiveContent[\\]+":[\\]+"[\\]+"[\\]+".*?[\\]+"(.*?)[\\]+"
-
- TODO TL all of the above in one call instead of multiple
- """
- # Lines
- regex = r'this.log_output\("(.*)"'
- matchList = re.findall(regex, data[i])
- if len(matchList) > 0:
- for match in matchList:
- # Save Original String
- originalString = match
-
- # Remove any textwrap
- match = match.replace(newline, " ")
-
- # Pass 1
- if translatedList == []:
- # Add String
- if match != "\\\\\\\\":
- stringList.append(match.strip())
-
- # Pass 2
- else:
- # Get Text
- if translatedList:
- # Grab and Pop
- translatedText = translatedList[0]
- translatedList.pop(0)
-
- # Set to None if empty list
- if len(translatedList) <= 0:
- translatedList = None
-
- # Textwrap
- translatedText = textwrap.fill(translatedText, width=WIDTH)
- translatedText = translatedText.replace("\n", newline)
-
- # Replace Single Quotes
- translatedText = translatedText.replace("'", "\\'")
- translatedText = translatedText.replace('"', "\\'")
-
- # Set Data
- data[i] = data[i].replace(originalString, translatedText)
- # Next Line
- i += 1
-
- # EOF
- if len(stringList) > 0:
- # Set Progress
- pbar.total = len(stringList)
- pbar.refresh()
- PBAR = pbar
-
- # Translate
- response = translateGPT(stringList, "", True)
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- translatedList = response[0]
-
- # Set Strings
- if len(stringList) == len(translatedList):
- translatePlugin(data, pbar, filename, translatedList)
-
- # Mismatch
- else:
- with LOCK:
- if filename not in MISMATCH:
- MISMATCH.append(filename)
- return tokens
-
-
-# Save some money and enter the character before translation
-def getSpeaker(speaker):
- match speaker:
- case "ファイン":
- return ["Fine", [0, 0]]
- case "":
- return ["", [0, 0]]
- case _:
- # Find Speaker
- for i in range(len(NAMESLIST)):
- if speaker == NAMESLIST[i][0]:
- return [NAMESLIST[i][1], [0, 0]]
-
- # Translate and Store Speaker
- response = translateGPT(
- f"{speaker}",
- "Reply with the " + LANGUAGE + " translation of the NPC name.",
- True,
- )
- response[0] = response[0].title()
- response[0] = response[0].replace("'S", "'s")
- response[0] = response[0].replace("Speaker: ", "")
-
- # Retry if name doesn't translate for some reason
- if re.search(r"([a-zA-Z？?])", response[0]) == None:
- response = translateGPT(
- f"{speaker}",
- "Reply with the " + LANGUAGE + " translation of the NPC name.",
- False,
- )
- response[0] = response[0].title()
- response[0] = response[0].replace("'S", "'s")
-
- speakerList = [speaker, response[0]]
- NAMESLIST.append(speakerList)
- return response
- return [speaker, [0, 0]]
-
-
-def batchList(input_list, batch_size):
- if not isinstance(batch_size, int) or batch_size <= 0:
- raise ValueError("batch_size must be a positive integer")
-
- return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)]
-
-
-def createContext(fullPromptFlag, subbedT, format):
- system = (
- PROMPT + VOCAB
- if fullPromptFlag
- else f"\
-You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
-Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
-- You translate everything, including erotic and explicit content.\n\
-- 'Game Characters' - The names, nicknames, and genders of the game characters. Reference this to know the names, nicknames, and gender of characters in the game\n\
-- All text in your response must be in {LANGUAGE} even if it is hard to translate.\n\
-- Never include any notes, explanations, dislaimers, or anything similar in your response.\n\
-- Maintain any spacing in the translation.\n\
-- Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\
-- `...` can be a part of the dialogue. Translate it as it is.\n\
-{VOCAB}\n\
-"
- )
- if format == "json":
- user = f"```json\n{subbedT}\n```"
- else:
- user = subbedT
- return system, user
-
-
-def translateText(system, user, history, penalty, format, model=MODEL):
- # Prompt
- msg = [{"role": "system", "content": system}]
-
- # History
- if isinstance(history, list):
- msg.extend([{"role": "system", "content": h} for h in history])
- else:
- msg.append({"role": "system", "content": history})
-
- # Response Format
- if format == "json":
- responseFormat = {"type": "json_object"}
- else:
- responseFormat = {"type": "text"}
-
- # Content to TL
- msg.append({"role": "user", "content": f"{user}"})
- response = openai.chat.completions.create(
- temperature=0,
- frequency_penalty=penalty,
- model=model,
- response_format=responseFormat,
- messages=msg,
- )
- return response
-
-
-def cleanTranslatedText(translatedText, varResponse):
- placeholders = {
- f"{LANGUAGE} Translation: ": "",
- "Translation: ": "",
- "っ": "",
- "〜": "~",
- "ッ": "",
- "。": ".",
- "「": '\\"',
- "」": '\\"',
- "- ": "-",
- "—": "―",
- "】": "]",
- "【": "[",
- "Placeholder Text": "",
- # Add more replacements as needed
- }
- for target, replacement in placeholders.items():
- translatedText = translatedText.replace(target, replacement)
-
- # Remove Repeating Characters
- pattern = re.compile(r"(.)\s*\1(?:\s*\1){" + str(20 - 1) + r",}")
- translatedText = pattern.sub(lambda match: match.group(0).replace(" ", "")[:20], translatedText)
-
- # Elongate Long Dashes (Since GPT Ignores them...)
- translatedText = elongateCharacters(translatedText)
- return translatedText
-
-
-def elongateCharacters(text):
- # Define a pattern to match one character followed by one or more `ー` characters
- # Using a positive lookbehind assertion to capture the preceding character
- pattern = r"(?<=(.))ー+"
-
- # Define a replacement function that elongates the captured character
- def repl(match):
- char = match.group(1) # The character before the ー sequence
- count = len(match.group(0)) - 1 # Number of ー characters
- return char * count # Replace ー sequence with the character repeated
-
- # Use re.sub() to replace the pattern in the text
- return re.sub(pattern, repl, text)
-
-
-def extractTranslation(translatedTextList, is_list):
- try:
- translatedTextList = re.sub(r'\\"+\"([^,\n}])', r'\\"\1', translatedTextList)
- translatedTextList = re.sub(r"(? instead
+FIXTEXTWRAP = True # Overwrites textwrap
+IGNORETLTEXT = False # Ignores all translated text.
+MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
+
+# tqdm Globals
+BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
+POSITION = 0
+LEAVE = False
+PBAR = None
+
+# Pricing - Depends on the model https://openai.com/pricing
+# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request
+# If you are getting a MISMATCH LENGTH error, lower the batch size.
+if "gpt-3.5" in MODEL:
+ INPUTAPICOST = 0.002
+ OUTPUTAPICOST = 0.002
+ BATCHSIZE = 10
+elif "gpt-4" in MODEL:
+ INPUTAPICOST = 0.0025
+ OUTPUTAPICOST = 0.01
+ BATCHSIZE = 40
+else:
+ INPUTAPICOST = float(os.getenv("input_cost"))
+ OUTPUTAPICOST = float(os.getenv("output_cost"))
+ BATCHSIZE = int(os.getenv("batchsize"))
+ FREQUENCY_PENALTY = float(os.getenv("frequency_penalty"))
+
+
+def handlePlugin(filename, estimate):
+ global ESTIMATE, PBAR
+ ESTIMATE = estimate
+
+ if ESTIMATE:
+ start = time.time()
+ translatedData = openFiles(filename)
+
+ # Print Result
+ end = time.time()
+ tqdm.write(getResultString(translatedData, end - start, filename))
+ with LOCK:
+ TOKENS[0] += translatedData[1][0]
+ TOKENS[1] += translatedData[1][1]
+
+ # Print Total
+ totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
+
+ # Print any errors on maps
+ if len(MISMATCH) > 0:
+ return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
+ else:
+ return totalString
+
+ else:
+ try:
+ with open("translated/" + filename, "w", encoding="utf_8", errors="ignore") as outFile:
+ start = time.time()
+ translatedData = openFiles(filename)
+
+ # Print Result
+ end = time.time()
+ outFile.writelines(translatedData[0])
+ tqdm.write(getResultString(translatedData, end - start, filename))
+ with LOCK:
+ TOKENS[0] += translatedData[1][0]
+ TOKENS[1] += translatedData[1][1]
+ except Exception:
+ traceback.print_exc()
+ return "Fail"
+
+ return getResultString(["", TOKENS, None], end - start, "TOTAL")
+
+
+def getResultString(translatedData, translationTime, filename):
+ # File Print String
+ totalTokenstring = (
+ Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
+ "[Output: "
+ + str(translatedData[1][1])
+ + "]" "[Cost: ${:,.4f}".format((translatedData[1][0] * 0.001 * INPUTAPICOST) + (translatedData[1][1] * 0.001 * OUTPUTAPICOST))
+ + "]"
+ )
+ timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
+
+ if translatedData[2] == None:
+ # Success
+ return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET
+
+ else:
+ # Fail
+ try:
+ raise translatedData[2]
+ except Exception as e:
+ traceback.print_exc()
+ errorString = str(e) + Fore.RED
+ return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET
+
+
+def openFiles(filename):
+ with open("files/" + filename, "r", encoding="utf_8") as readFile:
+ translatedData = parsePlugin(readFile, filename)
+
+ # Delete lines marked for deletion
+ finalData = []
+ for line in translatedData[0]:
+ if line != "\\d\n":
+ finalData.append(line)
+ translatedData[0] = finalData
+
+ return translatedData
+
+
+def parsePlugin(readFile, filename):
+ totalTokens = [0, 0]
+
+ # Read File into data
+ data = readFile.readlines()
+
+ # Create Progress Bar
+ with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
+ pbar.desc = filename
+
+ try:
+ result = translatePlugin(data, pbar, filename, [])
+ totalTokens[0] += result[0]
+ totalTokens[1] += result[1]
+ except Exception as e:
+ traceback.print_exc()
+ return [data, totalTokens, e]
+ return [data, totalTokens, None]
+
+
+def translatePlugin(data, pbar, filename, translatedList):
+ stringList = []
+ currentGroup = []
+ tokens = [0, 0]
+ speaker = ""
+ voice = False
+ global LOCK, ESTIMATE
+ i = 0
+
+ while i < len(data):
+ voice = False
+ speaker = ""
+ newline = r"\n"
+
+ """
+ Plugin List
+ Quest Name: [\\]+"QuestName[\\]+":[\\]+"(.*?)[\\]+"
+ Quest Client: [\\]+"QuestClientName[\\]+":[\\]+"[\\]+"[\\]+"(.*?)[\\]+"
+ Quest Location: [\\]+"QuestionLocation[\\]+":[\\]+"[\\]+"[\\]+"(.*?)[\\]+"
+ Quest Targe Location: [\\]+"PlaceInformation[\\]+":[\\]+"(.*?)[\\]+"
+ Quest Summary: [\\]+"QuestContent[\\]+":[\\]+"(.*?)[\\]+"
+ Quest Goal: [\\]+"ObjectiveContent[\\]+":[\\]+"[\\]+"[\\]+"(.*?)[\\]+"
+ Quest Goal 2: [\\]+"ObjectiveContent[\\]+":[\\]+"[\\]+"[\\]+".*?[\\]+"(.*?)[\\]+"
+
+ TODO TL all of the above in one call instead of multiple
+ """
+ # Lines
+ regex = r'this.log_output\("(.*)"'
+ matchList = re.findall(regex, data[i])
+ if len(matchList) > 0:
+ for match in matchList:
+ # Save Original String
+ originalString = match
+
+ # Remove any textwrap
+ match = match.replace(newline, " ")
+
+ # Pass 1
+ if translatedList == []:
+ # Add String
+ if match != "\\\\\\\\":
+ stringList.append(match.strip())
+
+ # Pass 2
+ else:
+ # Get Text
+ if translatedList:
+ # Grab and Pop
+ translatedText = translatedList[0]
+ translatedList.pop(0)
+
+ # Set to None if empty list
+ if len(translatedList) <= 0:
+ translatedList = None
+
+ # Textwrap
+ translatedText = textwrap.fill(translatedText, width=WIDTH)
+ translatedText = translatedText.replace("\n", newline)
+
+ # Replace Single Quotes
+ translatedText = translatedText.replace("'", "\\'")
+ translatedText = translatedText.replace('"', "\\'")
+
+ # Set Data
+ data[i] = data[i].replace(originalString, translatedText)
+ # Next Line
+ i += 1
+
+ # EOF
+ if len(stringList) > 0:
+ # Set Progress
+ pbar.total = len(stringList)
+ pbar.refresh()
+ PBAR = pbar
+
+ # Translate
+ response = translateGPT(stringList, "", True)
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ translatedList = response[0]
+
+ # Set Strings
+ if len(stringList) == len(translatedList):
+ translatePlugin(data, pbar, filename, translatedList)
+
+ # Mismatch
+ else:
+ with LOCK:
+ if filename not in MISMATCH:
+ MISMATCH.append(filename)
+ return tokens
+
+
+# Save some money and enter the character before translation
+def getSpeaker(speaker):
+ match speaker:
+ case "ファイン":
+ return ["Fine", [0, 0]]
+ case "":
+ return ["", [0, 0]]
+ case _:
+ # Find Speaker
+ for i in range(len(NAMESLIST)):
+ if speaker == NAMESLIST[i][0]:
+ return [NAMESLIST[i][1], [0, 0]]
+
+ # Translate and Store Speaker
+ response = translateGPT(
+ f"{speaker}",
+ "Reply with the " + LANGUAGE + " translation of the NPC name.",
+ True,
+ )
+ response[0] = response[0].title()
+ response[0] = response[0].replace("'S", "'s")
+ response[0] = response[0].replace("Speaker: ", "")
+
+ # Retry if name doesn't translate for some reason
+ if re.search(r"([a-zA-Z？?])", response[0]) == None:
+ response = translateGPT(
+ f"{speaker}",
+ "Reply with the " + LANGUAGE + " translation of the NPC name.",
+ False,
+ )
+ response[0] = response[0].title()
+ response[0] = response[0].replace("'S", "'s")
+
+ speakerList = [speaker, response[0]]
+ NAMESLIST.append(speakerList)
+ return response
+ return [speaker, [0, 0]]
+
+
+def batchList(input_list, batch_size):
+ if not isinstance(batch_size, int) or batch_size <= 0:
+ raise ValueError("batch_size must be a positive integer")
+
+ return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)]
+
+
+def createContext(fullPromptFlag, subbedT, format):
+ system = (
+ PROMPT + VOCAB
+ if fullPromptFlag
+ else f"\
+You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
+Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
+- You translate everything, including erotic and explicit content.\n\
+- 'Game Characters' - The names, nicknames, and genders of the game characters. Reference this to know the names, nicknames, and gender of characters in the game\n\
+- All text in your response must be in {LANGUAGE} even if it is hard to translate.\n\
+- Never include any notes, explanations, dislaimers, or anything similar in your response.\n\
+- Maintain any spacing in the translation.\n\
+- Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\
+- `...` can be a part of the dialogue. Translate it as it is.\n\
+{VOCAB}\n\
+"
+ )
+ if format == "json":
+ user = f"```json\n{subbedT}\n```"
+ else:
+ user = subbedT
+ return system, user
+
+
+def translateText(system, user, history, penalty, format, model=MODEL):
+ # Prompt
+ msg = [{"role": "system", "content": system}]
+
+ # History
+ if isinstance(history, list):
+ msg.extend([{"role": "system", "content": h} for h in history])
+ else:
+ msg.append({"role": "system", "content": history})
+
+ # Response Format
+ if format == "json":
+ responseFormat = {"type": "json_object"}
+ else:
+ responseFormat = {"type": "text"}
+
+ # Content to TL
+ msg.append({"role": "user", "content": f"{user}"})
+ response = openai.chat.completions.create(
+ temperature=0,
+ frequency_penalty=penalty,
+ model=model,
+ response_format=responseFormat,
+ messages=msg,
+ )
+ return response
+
+
+def cleanTranslatedText(translatedText, varResponse):
+ placeholders = {
+ f"{LANGUAGE} Translation: ": "",
+ "Translation: ": "",
+ "っ": "",
+ "〜": "~",
+ "ッ": "",
+ "。": ".",
+ "「": '\\"',
+ "」": '\\"',
+ "- ": "-",
+ "—": "―",
+ "】": "]",
+ "【": "[",
+ "Placeholder Text": "",
+ # Add more replacements as needed
+ }
+ for target, replacement in placeholders.items():
+ translatedText = translatedText.replace(target, replacement)
+
+ # Remove Repeating Characters
+ pattern = re.compile(r"(.)\s*\1(?:\s*\1){" + str(20 - 1) + r",}")
+ translatedText = pattern.sub(lambda match: match.group(0).replace(" ", "")[:20], translatedText)
+
+ # Elongate Long Dashes (Since GPT Ignores them...)
+ translatedText = elongateCharacters(translatedText)
+ return translatedText
+
+
+def elongateCharacters(text):
+ # Define a pattern to match one character followed by one or more `ー` characters
+ # Using a positive lookbehind assertion to capture the preceding character
+ pattern = r"(?<=(.))ー+"
+
+ # Define a replacement function that elongates the captured character
+ def repl(match):
+ char = match.group(1) # The character before the ー sequence
+ count = len(match.group(0)) - 1 # Number of ー characters
+ return char * count # Replace ー sequence with the character repeated
+
+ # Use re.sub() to replace the pattern in the text
+ return re.sub(pattern, repl, text)
+
+
+def extractTranslation(translatedTextList, is_list):
+ try:
+ translatedTextList = re.sub(r'\\"+\"([^,\n}])', r'\\"\1', translatedTextList)
+ translatedTextList = re.sub(r"(? instead
-FIXTEXTWRAP = True # Overwrites textwrap
-IGNORETLTEXT = False # Ignores all translated text.
-MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
-FILENAME = None
-
-# tqdm Globals
-BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
-POSITION = 0
-LEAVE = False
-PBAR = None
-
-# Pricing - Depends on the model https://openai.com/pricing
-# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request
-# If you are getting a MISMATCH LENGTH error, lower the batch size.
-if "gpt-3.5" in MODEL:
- INPUTAPICOST = 0.002
- OUTPUTAPICOST = 0.002
- BATCHSIZE = 10
-elif "gpt-4" in MODEL:
- INPUTAPICOST = 0.0025
- OUTPUTAPICOST = 0.01
- BATCHSIZE = 30
-else:
- INPUTAPICOST = float(os.getenv("input_cost"))
- OUTPUTAPICOST = float(os.getenv("output_cost"))
- BATCHSIZE = int(os.getenv("batchsize"))
- FREQUENCY_PENALTY = float(os.getenv("frequency_penalty"))
-
-def handleText(filename, estimate):
- global ESTIMATE, TOKENS, FILENAME
- ESTIMATE = estimate
- FILENAME = filename
-
- # Translate
- start = time.time()
- translatedData = openFiles(filename)
-
- # Translate
- if not estimate:
- try:
- with open("translated/" + filename, "w", encoding="utf-8") as outFile:
- outFile.writelines(translatedData[0])
- except Exception:
- traceback.print_exc()
- return "Fail"
-
- # Print File
- end = time.time()
- tqdm.write(getResultString(translatedData, end - start, filename))
- with LOCK:
- TOKENS[0] += translatedData[1][0]
- TOKENS[1] += translatedData[1][1]
-
- # Print Total
- totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
-
- # Print any errors on maps
- if len(MISMATCH) > 0:
- return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
- else:
- return totalString
-
-
-def getResultString(translatedData, translationTime, filename):
- # File Print String
- totalTokenstring = (
- Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
- "[Output: "
- + str(translatedData[1][1])
- + "]" "[Cost: ${:,.4f}".format((translatedData[1][0] * 0.001 * INPUTAPICOST) + (translatedData[1][1] * 0.001 * OUTPUTAPICOST))
- + "]"
- )
- timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
-
- if translatedData[2] == None:
- # Success
- return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET
-
- else:
- # Fail
- try:
- raise translatedData[2]
- except Exception as e:
- traceback.print_exc()
- errorString = str(e) + Fore.RED
- return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET
-
-
-def openFiles(filename):
- with open("files/" + filename, "r", encoding="utf8") as readFile:
- translatedData = parseText(readFile, filename)
-
- # Delete lines marked for deletion
- finalData = []
- for line in translatedData[0]:
- if line != "\\d\n":
- finalData.append(line)
- translatedData[0] = finalData
-
- return translatedData
-
-
-def parseText(readFile, filename):
- global PBAR
- totalTokens = [0, 0]
-
- # Read File into data
- data = readFile.readlines()
-
- # Create Progress Bar
- with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
- pbar.desc = filename
- PBAR = pbar
-
- try:
- result = translateTxt(data, [])
- totalTokens[0] += result[0]
- totalTokens[1] += result[1]
- except Exception as e:
- traceback.print_exc()
- return [data, totalTokens, e]
- return [data, totalTokens, None]
-
-
-def translateTxt(data, translatedList):
- if translatedList:
- stringList = translatedList[0]
- choiceList = translatedList[1]
- else:
- stringList = []
- choiceList = []
- tokens = [0, 0]
- speaker = ""
- global LOCK, ESTIMATE, FILENAME, PBAR, MISMATCH
- i = 0
-
- while i < len(data):
- lineTextRegex = r"(.+)"
-
- # Dialogue
- match = re.search(lineTextRegex, data[i])
- jaString = None
- if match:
- # Set String
- jaString = match.group(1)
-
- # Pass 1
- if not translatedList:
- # Strip Spaces
- jaString = jaString.strip()
-
- if jaString:
- if speaker:
- stringList.append(f"[{speaker}]: {jaString}")
- else:
- stringList.append(jaString)
-
- # Pass 2
- else:
- # Get Text
- if stringList:
- # Grab and Pop
- translatedText = stringList[0]
- stringList.pop(0)
-
- # Set to None if empty list
- if len(stringList) <= 0:
- stringList = None
-
- # Remove speaker
- translatedText = re.sub(r"^\[?(.+?)\]?\s?[|:]\s?", "", translatedText)
-
- # # Textwrap
- # translatedText = textwrap.fill(translatedText, width=WIDTH)
- # translatedText = translatedText.replace("\n", "\\n")
-
- # Set Data
- data[i] = f"{translatedText}\n"
-
- i += 1
- else:
- i += 1
-
- # EOF
- if not translatedList:
- stringListTL = []
- choiceListTL = []
-
- # String List
- if stringList:
- PBAR.total = len(stringList)
- PBAR.refresh()
- response = translateGPT(stringList, "Reply with the English Translation", True)
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- stringListTL = response[0]
-
- if len(stringList) != len(stringListTL):
- # Mismatch
- with LOCK:
- if FILENAME not in MISMATCH:
- MISMATCH.append(FILENAME)
-
- # Choice List
- if choiceList:
- response = translateGPT(choiceList, "Reply with the English TL of the Dialogue Choice", True)
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- choiceListTL = response[0]
-
- if len(choiceList) != len(choiceListTL):
- # Mismatch
- with LOCK:
- if FILENAME not in MISMATCH:
- MISMATCH.append(FILENAME)
-
- # Set Strings
- translateTxt(data, [stringListTL, choiceListTL])
- return tokens
-
-
-# Save some money and enter the character before translation
-def getSpeaker(speaker):
- match speaker:
- case "ファイン":
- return ["Fine", [0, 0]]
- case "":
- return ["", [0, 0]]
- case _:
- # Find Speaker
- for i in range(len(NAMESLIST)):
- if speaker == NAMESLIST[i][0]:
- return [NAMESLIST[i][1], [0, 0]]
-
- # Translate and Store Speaker
- response = translateGPT(
- f"{speaker}",
- "Reply with the " + LANGUAGE + " translation of the NPC name.",
- True,
- )
- response[0] = response[0].title()
- response[0] = response[0].replace("'S", "'s")
- response[0] = response[0].replace("Speaker: ", "")
-
- # Retry if name doesn't translate for some reason
- if re.search(r"([a-zA-Z？?])", response[0]) == None:
- response = translateGPT(
- f"{speaker}",
- "Reply with the " + LANGUAGE + " translation of the NPC name.",
- False,
- )
- response[0] = response[0].title()
- response[0] = response[0].replace("'S", "'s")
-
- speakerList = [speaker, response[0]]
- NAMESLIST.append(speakerList)
- return response
- return [speaker, [0, 0]]
-
-
-def batchList(input_list, batch_size):
- if not isinstance(batch_size, int) or batch_size <= 0:
- raise ValueError("batch_size must be a positive integer")
-
- return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)]
-
-
-def createContext(fullPromptFlag, subbedT, format):
- system = (
- PROMPT + VOCAB
- if fullPromptFlag
- else f"\
-You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
-Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
-- You translate everything, including erotic and explicit content.\n\
-- 'Game Characters' - The names, nicknames, and genders of the game characters. Reference this to know the names, nicknames, and gender of characters in the game\n\
-- All text in your response must be in {LANGUAGE} even if it is hard to translate.\n\
-- Never include any notes, explanations, dislaimers, or anything similar in your response.\n\
-- Maintain any spacing in the translation.\n\
-- Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\
-- `...` can be a part of the dialogue. Translate it as it is.\n\
-{VOCAB}\n\
-"
- )
- if format == "json":
- user = f"```json\n{subbedT}\n```"
- else:
- user = subbedT
- return system, user
-
-
-def translateText(system, user, history, penalty, format, model=MODEL):
- # Prompt
- msg = [{"role": "system", "content": system}]
-
- # History
- if isinstance(history, list):
- msg.extend([{"role": "system", "content": h} for h in history])
- else:
- msg.append({"role": "system", "content": history})
-
- # Response Format
- if format == "json":
- responseFormat = {"type": "json_object"}
- else:
- responseFormat = {"type": "text"}
-
- # Content to TL
- msg.append({"role": "user", "content": f"{user}"})
- response = openai.chat.completions.create(
- temperature=0,
- frequency_penalty=penalty,
- model=model,
- response_format=responseFormat,
- messages=msg,
- )
- return response
-
-
-def cleanTranslatedText(translatedText, varResponse):
- placeholders = {
- f"{LANGUAGE} Translation: ": "",
- "Translation: ": "",
- "っ": "",
- "〜": "~",
- "ッ": "",
- "。": ".",
- "「": '\\"',
- "」": '\\"',
- "- ": "-",
- "—": "―",
- "】": "]",
- "【": "[",
- "Placeholder Text": "",
- # Add more replacements as needed
- }
- for target, replacement in placeholders.items():
- translatedText = translatedText.replace(target, replacement)
-
- # Remove Repeating Characters
- pattern = re.compile(r"(.)\s*\1(?:\s*\1){" + str(20 - 1) + r",}")
- translatedText = pattern.sub(lambda match: match.group(0).replace(" ", "")[:20], translatedText)
-
- # Elongate Long Dashes (Since GPT Ignores them...)
- translatedText = elongateCharacters(translatedText)
- return translatedText
-
-
-def elongateCharacters(text):
- # Define a pattern to match one character followed by one or more `ー` characters
- # Using a positive lookbehind assertion to capture the preceding character
- pattern = r"(?<=(.))ー+"
-
- # Define a replacement function that elongates the captured character
- def repl(match):
- char = match.group(1) # The character before the ー sequence
- count = len(match.group(0)) - 1 # Number of ー characters
- return char * count # Replace ー sequence with the character repeated
-
- # Use re.sub() to replace the pattern in the text
- return re.sub(pattern, repl, text)
-
-
-def extractTranslation(translatedTextList, is_list):
- try:
- translatedTextList = re.sub(r'\\"+\"([^,\n}])', r'\\"\1', translatedTextList)
- translatedTextList = re.sub(r"(? instead
+FIXTEXTWRAP = True # Overwrites textwrap
+IGNORETLTEXT = False # Ignores all translated text.
+MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
+FILENAME = None
+
+# tqdm Globals
+BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
+POSITION = 0
+LEAVE = False
+PBAR = None
+
+# Pricing - Depends on the model https://openai.com/pricing
+# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request
+# If you are getting a MISMATCH LENGTH error, lower the batch size.
+if "gpt-3.5" in MODEL:
+ INPUTAPICOST = 0.002
+ OUTPUTAPICOST = 0.002
+ BATCHSIZE = 10
+elif "gpt-4" in MODEL:
+ INPUTAPICOST = 0.0025
+ OUTPUTAPICOST = 0.01
+ BATCHSIZE = 30
+else:
+ INPUTAPICOST = float(os.getenv("input_cost"))
+ OUTPUTAPICOST = float(os.getenv("output_cost"))
+ BATCHSIZE = int(os.getenv("batchsize"))
+ FREQUENCY_PENALTY = float(os.getenv("frequency_penalty"))
+
+
+def handleText(filename, estimate):
+ global ESTIMATE, TOKENS, FILENAME
+ ESTIMATE = estimate
+ FILENAME = filename
+
+ # Translate
+ start = time.time()
+ translatedData = openFiles(filename)
+
+ # Translate
+ if not estimate:
+ try:
+ with open("translated/" + filename, "w", encoding="utf-8") as outFile:
+ outFile.writelines(translatedData[0])
+ except Exception:
+ traceback.print_exc()
+ return "Fail"
+
+ # Print File
+ end = time.time()
+ tqdm.write(getResultString(translatedData, end - start, filename))
+ with LOCK:
+ TOKENS[0] += translatedData[1][0]
+ TOKENS[1] += translatedData[1][1]
+
+ # Print Total
+ totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
+
+ # Print any errors on maps
+ if len(MISMATCH) > 0:
+ return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
+ else:
+ return totalString
+
+
+def getResultString(translatedData, translationTime, filename):
+ # File Print String
+ totalTokenstring = (
+ Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
+ "[Output: "
+ + str(translatedData[1][1])
+ + "]" "[Cost: ${:,.4f}".format((translatedData[1][0] * 0.001 * INPUTAPICOST) + (translatedData[1][1] * 0.001 * OUTPUTAPICOST))
+ + "]"
+ )
+ timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
+
+ if translatedData[2] == None:
+ # Success
+ return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET
+
+ else:
+ # Fail
+ try:
+ raise translatedData[2]
+ except Exception as e:
+ traceback.print_exc()
+ errorString = str(e) + Fore.RED
+ return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET
+
+
+def openFiles(filename):
+ with open("files/" + filename, "r", encoding="utf8") as readFile:
+ translatedData = parseText(readFile, filename)
+
+ # Delete lines marked for deletion
+ finalData = []
+ for line in translatedData[0]:
+ if line != "\\d\n":
+ finalData.append(line)
+ translatedData[0] = finalData
+
+ return translatedData
+
+
+def parseText(readFile, filename):
+ global PBAR
+ totalTokens = [0, 0]
+
+ # Read File into data
+ data = readFile.readlines()
+
+ # Create Progress Bar
+ with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
+ pbar.desc = filename
+ PBAR = pbar
+
+ try:
+ result = translateTxt(data, [])
+ totalTokens[0] += result[0]
+ totalTokens[1] += result[1]
+ except Exception as e:
+ traceback.print_exc()
+ return [data, totalTokens, e]
+ return [data, totalTokens, None]
+
+
+def translateTxt(data, translatedList):
+ if translatedList:
+ stringList = translatedList[0]
+ choiceList = translatedList[1]
+ else:
+ stringList = []
+ choiceList = []
+ tokens = [0, 0]
+ speaker = ""
+ global LOCK, ESTIMATE, FILENAME, PBAR, MISMATCH
+ i = 0
+
+ while i < len(data):
+ lineTextRegex = r"(.+)"
+
+ # Dialogue
+ match = re.search(lineTextRegex, data[i])
+ jaString = None
+ if match:
+ # Set String
+ jaString = match.group(1)
+
+ # Pass 1
+ if not translatedList:
+ # Strip Spaces
+ jaString = jaString.strip()
+
+ if jaString:
+ if speaker:
+ stringList.append(f"[{speaker}]: {jaString}")
+ else:
+ stringList.append(jaString)
+
+ # Pass 2
+ else:
+ # Get Text
+ if stringList:
+ # Grab and Pop
+ translatedText = stringList[0]
+ stringList.pop(0)
+
+ # Set to None if empty list
+ if len(stringList) <= 0:
+ stringList = None
+
+ # Remove speaker
+ translatedText = re.sub(r"^\[?(.+?)\]?\s?[|:]\s?", "", translatedText)
+
+ # # Textwrap
+ # translatedText = textwrap.fill(translatedText, width=WIDTH)
+ # translatedText = translatedText.replace("\n", "\\n")
+
+ # Set Data
+ data[i] = f"{translatedText}\n"
+
+ i += 1
+ else:
+ i += 1
+
+ # EOF
+ if not translatedList:
+ stringListTL = []
+ choiceListTL = []
+
+ # String List
+ if stringList:
+ PBAR.total = len(stringList)
+ PBAR.refresh()
+ response = translateGPT(stringList, "Reply with the English Translation", True)
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ stringListTL = response[0]
+
+ if len(stringList) != len(stringListTL):
+ # Mismatch
+ with LOCK:
+ if FILENAME not in MISMATCH:
+ MISMATCH.append(FILENAME)
+
+ # Choice List
+ if choiceList:
+ response = translateGPT(choiceList, "Reply with the English TL of the Dialogue Choice", True)
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ choiceListTL = response[0]
+
+ if len(choiceList) != len(choiceListTL):
+ # Mismatch
+ with LOCK:
+ if FILENAME not in MISMATCH:
+ MISMATCH.append(FILENAME)
+
+ # Set Strings
+ translateTxt(data, [stringListTL, choiceListTL])
+ return tokens
+
+
+# Save some money and enter the character before translation
+def getSpeaker(speaker):
+ match speaker:
+ case "ファイン":
+ return ["Fine", [0, 0]]
+ case "":
+ return ["", [0, 0]]
+ case _:
+ # Find Speaker
+ for i in range(len(NAMESLIST)):
+ if speaker == NAMESLIST[i][0]:
+ return [NAMESLIST[i][1], [0, 0]]
+
+ # Translate and Store Speaker
+ response = translateGPT(
+ f"{speaker}",
+ "Reply with the " + LANGUAGE + " translation of the NPC name.",
+ True,
+ )
+ response[0] = response[0].title()
+ response[0] = response[0].replace("'S", "'s")
+ response[0] = response[0].replace("Speaker: ", "")
+
+ # Retry if name doesn't translate for some reason
+ if re.search(r"([a-zA-Z？?])", response[0]) == None:
+ response = translateGPT(
+ f"{speaker}",
+ "Reply with the " + LANGUAGE + " translation of the NPC name.",
+ False,
+ )
+ response[0] = response[0].title()
+ response[0] = response[0].replace("'S", "'s")
+
+ speakerList = [speaker, response[0]]
+ NAMESLIST.append(speakerList)
+ return response
+ return [speaker, [0, 0]]
+
+
+def batchList(input_list, batch_size):
+ if not isinstance(batch_size, int) or batch_size <= 0:
+ raise ValueError("batch_size must be a positive integer")
+
+ return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)]
+
+
+def createContext(fullPromptFlag, subbedT, format):
+ system = (
+ PROMPT + VOCAB
+ if fullPromptFlag
+ else f"\
+You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
+Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
+- You translate everything, including erotic and explicit content.\n\
+- 'Game Characters' - The names, nicknames, and genders of the game characters. Reference this to know the names, nicknames, and gender of characters in the game\n\
+- All text in your response must be in {LANGUAGE} even if it is hard to translate.\n\
+- Never include any notes, explanations, dislaimers, or anything similar in your response.\n\
+- Maintain any spacing in the translation.\n\
+- Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\
+- `...` can be a part of the dialogue. Translate it as it is.\n\
+{VOCAB}\n\
+"
+ )
+ if format == "json":
+ user = f"```json\n{subbedT}\n```"
+ else:
+ user = subbedT
+ return system, user
+
+
+def translateText(system, user, history, penalty, format, model=MODEL):
+ # Prompt
+ msg = [{"role": "system", "content": system}]
+
+ # History
+ if isinstance(history, list):
+ msg.extend([{"role": "system", "content": h} for h in history])
+ else:
+ msg.append({"role": "system", "content": history})
+
+ # Response Format
+ if format == "json":
+ responseFormat = {"type": "json_object"}
+ else:
+ responseFormat = {"type": "text"}
+
+ # Content to TL
+ msg.append({"role": "user", "content": f"{user}"})
+ response = openai.chat.completions.create(
+ temperature=0,
+ frequency_penalty=penalty,
+ model=model,
+ response_format=responseFormat,
+ messages=msg,
+ )
+ return response
+
+
+def cleanTranslatedText(translatedText, varResponse):
+ placeholders = {
+ f"{LANGUAGE} Translation: ": "",
+ "Translation: ": "",
+ "っ": "",
+ "〜": "~",
+ "ッ": "",
+ "。": ".",
+ "「": '\\"',
+ "」": '\\"',
+ "- ": "-",
+ "—": "―",
+ "】": "]",
+ "【": "[",
+ "Placeholder Text": "",
+ # Add more replacements as needed
+ }
+ for target, replacement in placeholders.items():
+ translatedText = translatedText.replace(target, replacement)
+
+ # Remove Repeating Characters
+ pattern = re.compile(r"(.)\s*\1(?:\s*\1){" + str(20 - 1) + r",}")
+ translatedText = pattern.sub(lambda match: match.group(0).replace(" ", "")[:20], translatedText)
+
+ # Elongate Long Dashes (Since GPT Ignores them...)
+ translatedText = elongateCharacters(translatedText)
+ return translatedText
+
+
+def elongateCharacters(text):
+ # Define a pattern to match one character followed by one or more `ー` characters
+ # Using a positive lookbehind assertion to capture the preceding character
+ pattern = r"(?<=(.))ー+"
+
+ # Define a replacement function that elongates the captured character
+ def repl(match):
+ char = match.group(1) # The character before the ー sequence
+ count = len(match.group(0)) - 1 # Number of ー characters
+ return char * count # Replace ー sequence with the character repeated
+
+ # Use re.sub() to replace the pattern in the text
+ return re.sub(pattern, repl, text)
+
+
+def extractTranslation(translatedTextList, is_list):
+ try:
+ translatedTextList = re.sub(r'\\"+\"([^,\n}])', r'\\"\1', translatedTextList)
+ translatedTextList = re.sub(r"(? instead
-FIXTEXTWRAP = True # Overwrites textwrap
-IGNORETLTEXT = False # Ignores all translated text.
-MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
-FILENAME = None
-
-# tqdm Globals
-BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
-POSITION = 0
-LEAVE = False
-PBAR = None
-
-# Pricing - Depends on the model https://openai.com/pricing
-# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request
-# If you are getting a MISMATCH LENGTH error, lower the batch size.
-if "gpt-3.5" in MODEL:
- INPUTAPICOST = 0.002
- OUTPUTAPICOST = 0.002
- BATCHSIZE = 10
-elif "gpt-4" in MODEL:
- INPUTAPICOST = 0.0025
- OUTPUTAPICOST = 0.01
- BATCHSIZE = 20
-else:
- INPUTAPICOST = float(os.getenv("input_cost"))
- OUTPUTAPICOST = float(os.getenv("output_cost"))
- BATCHSIZE = int(os.getenv("batchsize"))
- FREQUENCY_PENALTY = float(os.getenv("frequency_penalty"))
-
-def handleTyrano(filename, estimate):
- global ESTIMATE, TOKENS, FILENAME
- ESTIMATE = estimate
- FILENAME = filename
-
- # Translate
- start = time.time()
- translatedData = openFiles(filename)
-
- # Translate
- if not estimate:
- try:
- with open("translated/" + filename, "w", encoding="utf-8") as outFile:
- outFile.writelines(translatedData[0])
- except Exception:
- traceback.print_exc()
- return "Fail"
-
- # Print File
- end = time.time()
- tqdm.write(getResultString(translatedData, end - start, filename))
- with LOCK:
- TOKENS[0] += translatedData[1][0]
- TOKENS[1] += translatedData[1][1]
-
- # Print Total
- totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
-
- # Print any errors on maps
- if len(MISMATCH) > 0:
- return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
- else:
- return totalString
-
-
-def getResultString(translatedData, translationTime, filename):
- # File Print String
- totalTokenstring = (
- Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
- "[Output: "
- + str(translatedData[1][1])
- + "]" "[Cost: ${:,.4f}".format((translatedData[1][0] * 0.001 * INPUTAPICOST) + (translatedData[1][1] * 0.001 * OUTPUTAPICOST))
- + "]"
- )
- timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
-
- if translatedData[2] == None:
- # Success
- return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET
-
- else:
- # Fail
- try:
- raise translatedData[2]
- except Exception as e:
- traceback.print_exc()
- errorString = str(e) + Fore.RED
- return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET
-
-
-def openFiles(filename):
- with open("files/" + filename, "r", encoding="utf8") as readFile:
- translatedData = parseTyrano(readFile, filename)
-
- # Delete lines marked for deletion
- finalData = []
- for line in translatedData[0]:
- if line != "\\d\n":
- finalData.append(line)
- translatedData[0] = finalData
-
- return translatedData
-
-
-def parseTyrano(readFile, filename):
- global PBAR
- totalTokens = [0, 0]
-
- # Read File into data
- data = readFile.readlines()
-
- # Create Progress Bar
- with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
- pbar.desc = filename
- PBAR = pbar
-
- try:
- result = translateTyrano(data, [])
- totalTokens[0] += result[0]
- totalTokens[1] += result[1]
- except Exception as e:
- traceback.print_exc()
- return [data, totalTokens, e]
- return [data, totalTokens, None]
-
-
-def translateTyrano(data, translatedList):
- if translatedList:
- stringList = translatedList[0]
- choiceList = translatedList[1]
- else:
- stringList = []
- choiceList = []
- tokens = [0, 0]
- speaker = ""
- global LOCK, ESTIMATE, FILENAME, PBAR, MISMATCH
- i = 0
-
- while i < len(data):
- voice = False
- lineRegexNoSpeaker = r"^([^\[#;*@\n]+)\[l\]\[[rp]\]|^([^\[#;*@\n]+)\[[rpl]\]|^([^\[#;*@_\n]+)\n$"
- lineRegexSpeaker = r"^#(.*)"
- furiganaRegex = r"(\[ruby\stext=(.*?)\])"
- choiceRegex = r'\[glink.+?text="(.*?)"'
-
- # Speaker
- match = re.search(lineRegexSpeaker, data[i])
- if match:
- if match.group(1):
- response = getSpeaker(match.group(1))
- speaker = response[0]
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- data[i] = data[i].replace(match.group(1), speaker)
- else:
- speaker = None
-
- # Furigana
- match = re.search(r"^\[ruby\stext", data[i])
- furiganaList = []
- if match:
- # Check next line and combine
- while match:
- furiganaList.append(data[i].replace("\n", ""))
- del data[i]
- match = re.search(r"^\[ruby\stext", data[i])
- jaString = "".join(furiganaList)
-
- # Ruby Text
- furiganaList = re.findall(furiganaRegex, jaString)
- for furigana in furiganaList:
- jaString = jaString.replace(furigana[0], furigana[1])
-
- data.insert(i, f"{jaString}[r]")
-
- # Dialogue
- match = re.search(lineRegexNoSpeaker, data[i])
- jaString = None
- if match:
- jaString = match.group(1)
- if not jaString:
- jaString = match.group(2)
- if not jaString:
- jaString = match.group(3)
-
- originalString = jaString
-
- # Pass 1
- if not translatedList:
- # Remove any textwrap and commands
- jaString = jaString.replace("[r]", " ")
- jaString = jaString.replace("[l]", "")
-
- # Ruby Text
- furiganaList = re.findall(furiganaRegex, jaString)
- for furigana in furiganaList:
- jaString = jaString.replace(furigana[0], furigana[1])
-
- # Strip Spaces
- jaString = jaString.strip()
-
- if jaString:
- if speaker:
- stringList.append(f"[{speaker}]: {jaString}")
- else:
- stringList.append(jaString)
-
- # Pass 2
- else:
- # Get Text
- if stringList:
- # Grab and Pop
- translatedText = stringList[0]
- stringList.pop(0)
-
- # Set to None if empty list
- if len(stringList) <= 0:
- stringList = None
-
- # Remove speaker
- if speaker != "":
- matchSpeakerList = re.findall(r"^\[?(.+?)\]?\s?[|:]\s?", translatedText)
- translatedText = re.sub(r"^\[?(.+?)\]?\s?[|:]\s?", "", translatedText)
-
- # # Textwrap
- # translatedText = textwrap.fill(translatedText, width=WIDTH)
- # translatedText = translatedText.replace('\n', '[r]')
-
- # Avoid Crashes
- translatedText = translatedText.replace("[", "(")
- translatedText = translatedText.replace("]", ")")
-
- # Set Data
- data[i] = data[i].replace(originalString, translatedText)
-
- # Choices
- match = re.search(choiceRegex, data[i])
- if match:
- # Pass 1
- if not translatedList:
- choiceList.append(match.group(1))
- match = re.search(choiceRegex, data[i + 1])
-
- # Pass 2
- else:
- # Grab and Pop
- translatedText = choiceList[0]
- choiceList.pop(0)
-
- # Replace Spaces
- translatedText = translatedText.replace(" ", "\u3000")
-
- # Set
- data[i] = data[i].replace(match.group(1), translatedText)
-
- i += 1
- else:
- i += 1
-
- # EOF
- if not translatedList:
- stringListTL = []
- choiceListTL = []
-
- # String List
- if stringList:
- PBAR.total = len(stringList)
- PBAR.refresh()
- response = translateGPT(stringList, "Reply with the English Translation", True)
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- stringListTL = response[0]
-
- if len(stringList) != len(stringListTL):
- # Mismatch
- with LOCK:
- if FILENAME not in MISMATCH:
- MISMATCH.append(FILENAME)
-
- # Choice List
- if choiceList:
- response = translateGPT(choiceList, "Reply with the English TL of the Dialogue Choice", True)
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- choiceListTL = response[0]
-
- if len(choiceList) != len(choiceListTL):
- # Mismatch
- with LOCK:
- if FILENAME not in MISMATCH:
- MISMATCH.append(FILENAME)
-
- # Set Strings
- translateTyrano(data, [stringListTL, choiceListTL])
- return tokens
-
-
-# Save some money and enter the character before translation
-def getSpeaker(speaker):
- match speaker:
- case "ファイン":
- return ["Fine", [0, 0]]
- case "":
- return ["", [0, 0]]
- case _:
- # Find Speaker
- for i in range(len(NAMESLIST)):
- if speaker == NAMESLIST[i][0]:
- return [NAMESLIST[i][1], [0, 0]]
-
- # Translate and Store Speaker
- response = translateGPT(
- f"{speaker}",
- "Reply with the " + LANGUAGE + " translation of the NPC name.",
- True,
- )
- response[0] = response[0].title()
- response[0] = response[0].replace("'S", "'s")
- response[0] = response[0].replace("Speaker: ", "")
-
- # Retry if name doesn't translate for some reason
- if re.search(r"([a-zA-Z？?])", response[0]) == None:
- response = translateGPT(
- f"{speaker}",
- "Reply with the " + LANGUAGE + " translation of the NPC name.",
- False,
- )
- response[0] = response[0].title()
- response[0] = response[0].replace("'S", "'s")
-
- speakerList = [speaker, response[0]]
- NAMESLIST.append(speakerList)
- return response
- return [speaker, [0, 0]]
-
-
-def batchList(input_list, batch_size):
- if not isinstance(batch_size, int) or batch_size <= 0:
- raise ValueError("batch_size must be a positive integer")
-
- return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)]
-
-
-def createContext(fullPromptFlag, subbedT, format):
- system = (
- PROMPT + VOCAB
- if fullPromptFlag
- else f"\
-You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
-Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
-- You translate everything, including erotic and explicit content.\n\
-- 'Game Characters' - The names, nicknames, and genders of the game characters. Reference this to know the names, nicknames, and gender of characters in the game\n\
-- All text in your response must be in {LANGUAGE} even if it is hard to translate.\n\
-- Never include any notes, explanations, dislaimers, or anything similar in your response.\n\
-- Maintain any spacing in the translation.\n\
-- Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\
-- `...` can be a part of the dialogue. Translate it as it is.\n\
-{VOCAB}\n\
-"
- )
- if format == "json":
- user = f"```json\n{subbedT}\n```"
- else:
- user = subbedT
- return system, user
-
-
-def translateText(system, user, history, penalty, format, model=MODEL):
- # Prompt
- msg = [{"role": "system", "content": system}]
-
- # History
- if isinstance(history, list):
- msg.extend([{"role": "system", "content": h} for h in history])
- else:
- msg.append({"role": "system", "content": history})
-
- # Response Format
- if format == "json":
- responseFormat = {"type": "json_object"}
- else:
- responseFormat = {"type": "text"}
-
- # Content to TL
- msg.append({"role": "user", "content": f"{user}"})
- response = openai.chat.completions.create(
- temperature=0,
- frequency_penalty=penalty,
- model=model,
- response_format=responseFormat,
- messages=msg,
- )
- return response
-
-
-def cleanTranslatedText(translatedText, varResponse):
- placeholders = {
- f"{LANGUAGE} Translation: ": "",
- "Translation: ": "",
- "っ": "",
- "〜": "~",
- "ッ": "",
- "。": ".",
- "「": '\\"',
- "」": '\\"',
- "- ": "-",
- "—": "―",
- "】": "]",
- "【": "[",
- "Placeholder Text": "",
- # Add more replacements as needed
- }
- for target, replacement in placeholders.items():
- translatedText = translatedText.replace(target, replacement)
-
- # Remove Repeating Characters
- pattern = re.compile(r"(.)\s*\1(?:\s*\1){" + str(20 - 1) + r",}")
- translatedText = pattern.sub(lambda match: match.group(0).replace(" ", "")[:20], translatedText)
-
- # Elongate Long Dashes (Since GPT Ignores them...)
- translatedText = elongateCharacters(translatedText)
- return translatedText
-
-
-def elongateCharacters(text):
- # Define a pattern to match one character followed by one or more `ー` characters
- # Using a positive lookbehind assertion to capture the preceding character
- pattern = r"(?<=(.))ー+"
-
- # Define a replacement function that elongates the captured character
- def repl(match):
- char = match.group(1) # The character before the ー sequence
- count = len(match.group(0)) - 1 # Number of ー characters
- return char * count # Replace ー sequence with the character repeated
-
- # Use re.sub() to replace the pattern in the text
- return re.sub(pattern, repl, text)
-
-
-def extractTranslation(translatedTextList, is_list):
- try:
- translatedTextList = re.sub(r'\\"+\"([^,\n}])', r'\\"\1', translatedTextList)
- translatedTextList = re.sub(r"(? instead
+FIXTEXTWRAP = True # Overwrites textwrap
+IGNORETLTEXT = False # Ignores all translated text.
+MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
+FILENAME = None
+
+# tqdm Globals
+BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
+POSITION = 0
+LEAVE = False
+PBAR = None
+
+# Pricing - Depends on the model https://openai.com/pricing
+# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request
+# If you are getting a MISMATCH LENGTH error, lower the batch size.
+if "gpt-3.5" in MODEL:
+ INPUTAPICOST = 0.002
+ OUTPUTAPICOST = 0.002
+ BATCHSIZE = 10
+elif "gpt-4" in MODEL:
+ INPUTAPICOST = 0.0025
+ OUTPUTAPICOST = 0.01
+ BATCHSIZE = 20
+else:
+ INPUTAPICOST = float(os.getenv("input_cost"))
+ OUTPUTAPICOST = float(os.getenv("output_cost"))
+ BATCHSIZE = int(os.getenv("batchsize"))
+ FREQUENCY_PENALTY = float(os.getenv("frequency_penalty"))
+
+
+def handleTyrano(filename, estimate):
+ global ESTIMATE, TOKENS, FILENAME
+ ESTIMATE = estimate
+ FILENAME = filename
+
+ # Translate
+ start = time.time()
+ translatedData = openFiles(filename)
+
+ # Translate
+ if not estimate:
+ try:
+ with open("translated/" + filename, "w", encoding="utf-8") as outFile:
+ outFile.writelines(translatedData[0])
+ except Exception:
+ traceback.print_exc()
+ return "Fail"
+
+ # Print File
+ end = time.time()
+ tqdm.write(getResultString(translatedData, end - start, filename))
+ with LOCK:
+ TOKENS[0] += translatedData[1][0]
+ TOKENS[1] += translatedData[1][1]
+
+ # Print Total
+ totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
+
+ # Print any errors on maps
+ if len(MISMATCH) > 0:
+ return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
+ else:
+ return totalString
+
+
+def getResultString(translatedData, translationTime, filename):
+ # File Print String
+ totalTokenstring = (
+ Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
+ "[Output: "
+ + str(translatedData[1][1])
+ + "]" "[Cost: ${:,.4f}".format((translatedData[1][0] * 0.001 * INPUTAPICOST) + (translatedData[1][1] * 0.001 * OUTPUTAPICOST))
+ + "]"
+ )
+ timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
+
+ if translatedData[2] == None:
+ # Success
+ return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET
+
+ else:
+ # Fail
+ try:
+ raise translatedData[2]
+ except Exception as e:
+ traceback.print_exc()
+ errorString = str(e) + Fore.RED
+ return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET
+
+
+def openFiles(filename):
+ with open("files/" + filename, "r", encoding="utf8") as readFile:
+ translatedData = parseTyrano(readFile, filename)
+
+ # Delete lines marked for deletion
+ finalData = []
+ for line in translatedData[0]:
+ if line != "\\d\n":
+ finalData.append(line)
+ translatedData[0] = finalData
+
+ return translatedData
+
+
+def parseTyrano(readFile, filename):
+ global PBAR
+ totalTokens = [0, 0]
+
+ # Read File into data
+ data = readFile.readlines()
+
+ # Create Progress Bar
+ with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
+ pbar.desc = filename
+ PBAR = pbar
+
+ try:
+ result = translateTyrano(data, [])
+ totalTokens[0] += result[0]
+ totalTokens[1] += result[1]
+ except Exception as e:
+ traceback.print_exc()
+ return [data, totalTokens, e]
+ return [data, totalTokens, None]
+
+
+def translateTyrano(data, translatedList):
+ if translatedList:
+ stringList = translatedList[0]
+ choiceList = translatedList[1]
+ else:
+ stringList = []
+ choiceList = []
+ tokens = [0, 0]
+ speaker = ""
+ global LOCK, ESTIMATE, FILENAME, PBAR, MISMATCH
+ i = 0
+
+ while i < len(data):
+ voice = False
+ lineRegexNoSpeaker = r"^([^\[#;*@\n]+)\[l\]\[[rp]\]|^([^\[#;*@\n]+)\[[rpl]\]|^([^\[#;*@_\n]+)\n$"
+ lineRegexSpeaker = r"^#(.*)"
+ furiganaRegex = r"(\[ruby\stext=(.*?)\])"
+ choiceRegex = r'\[glink.+?text="(.*?)"'
+
+ # Speaker
+ match = re.search(lineRegexSpeaker, data[i])
+ if match:
+ if match.group(1):
+ response = getSpeaker(match.group(1))
+ speaker = response[0]
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ data[i] = data[i].replace(match.group(1), speaker)
+ else:
+ speaker = None
+
+ # Furigana
+ match = re.search(r"^\[ruby\stext", data[i])
+ furiganaList = []
+ if match:
+ # Check next line and combine
+ while match:
+ furiganaList.append(data[i].replace("\n", ""))
+ del data[i]
+ match = re.search(r"^\[ruby\stext", data[i])
+ jaString = "".join(furiganaList)
+
+ # Ruby Text
+ furiganaList = re.findall(furiganaRegex, jaString)
+ for furigana in furiganaList:
+ jaString = jaString.replace(furigana[0], furigana[1])
+
+ data.insert(i, f"{jaString}[r]")
+
+ # Dialogue
+ match = re.search(lineRegexNoSpeaker, data[i])
+ jaString = None
+ if match:
+ jaString = match.group(1)
+ if not jaString:
+ jaString = match.group(2)
+ if not jaString:
+ jaString = match.group(3)
+
+ originalString = jaString
+
+ # Pass 1
+ if not translatedList:
+ # Remove any textwrap and commands
+ jaString = jaString.replace("[r]", " ")
+ jaString = jaString.replace("[l]", "")
+
+ # Ruby Text
+ furiganaList = re.findall(furiganaRegex, jaString)
+ for furigana in furiganaList:
+ jaString = jaString.replace(furigana[0], furigana[1])
+
+ # Strip Spaces
+ jaString = jaString.strip()
+
+ if jaString:
+ if speaker:
+ stringList.append(f"[{speaker}]: {jaString}")
+ else:
+ stringList.append(jaString)
+
+ # Pass 2
+ else:
+ # Get Text
+ if stringList:
+ # Grab and Pop
+ translatedText = stringList[0]
+ stringList.pop(0)
+
+ # Set to None if empty list
+ if len(stringList) <= 0:
+ stringList = None
+
+ # Remove speaker
+ if speaker != "":
+ matchSpeakerList = re.findall(r"^\[?(.+?)\]?\s?[|:]\s?", translatedText)
+ translatedText = re.sub(r"^\[?(.+?)\]?\s?[|:]\s?", "", translatedText)
+
+ # # Textwrap
+ # translatedText = textwrap.fill(translatedText, width=WIDTH)
+ # translatedText = translatedText.replace('\n', '[r]')
+
+ # Avoid Crashes
+ translatedText = translatedText.replace("[", "(")
+ translatedText = translatedText.replace("]", ")")
+
+ # Set Data
+ data[i] = data[i].replace(originalString, translatedText)
+
+ # Choices
+ match = re.search(choiceRegex, data[i])
+ if match:
+ # Pass 1
+ if not translatedList:
+ choiceList.append(match.group(1))
+ match = re.search(choiceRegex, data[i + 1])
+
+ # Pass 2
+ else:
+ # Grab and Pop
+ translatedText = choiceList[0]
+ choiceList.pop(0)
+
+ # Replace Spaces
+ translatedText = translatedText.replace(" ", "\u3000")
+
+ # Set
+ data[i] = data[i].replace(match.group(1), translatedText)
+
+ i += 1
+ else:
+ i += 1
+
+ # EOF
+ if not translatedList:
+ stringListTL = []
+ choiceListTL = []
+
+ # String List
+ if stringList:
+ PBAR.total = len(stringList)
+ PBAR.refresh()
+ response = translateGPT(stringList, "Reply with the English Translation", True)
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ stringListTL = response[0]
+
+ if len(stringList) != len(stringListTL):
+ # Mismatch
+ with LOCK:
+ if FILENAME not in MISMATCH:
+ MISMATCH.append(FILENAME)
+
+ # Choice List
+ if choiceList:
+ response = translateGPT(choiceList, "Reply with the English TL of the Dialogue Choice", True)
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ choiceListTL = response[0]
+
+ if len(choiceList) != len(choiceListTL):
+ # Mismatch
+ with LOCK:
+ if FILENAME not in MISMATCH:
+ MISMATCH.append(FILENAME)
+
+ # Set Strings
+ translateTyrano(data, [stringListTL, choiceListTL])
+ return tokens
+
+
+# Save some money and enter the character before translation
+def getSpeaker(speaker):
+ match speaker:
+ case "ファイン":
+ return ["Fine", [0, 0]]
+ case "":
+ return ["", [0, 0]]
+ case _:
+ # Find Speaker
+ for i in range(len(NAMESLIST)):
+ if speaker == NAMESLIST[i][0]:
+ return [NAMESLIST[i][1], [0, 0]]
+
+ # Translate and Store Speaker
+ response = translateGPT(
+ f"{speaker}",
+ "Reply with the " + LANGUAGE + " translation of the NPC name.",
+ True,
+ )
+ response[0] = response[0].title()
+ response[0] = response[0].replace("'S", "'s")
+ response[0] = response[0].replace("Speaker: ", "")
+
+ # Retry if name doesn't translate for some reason
+ if re.search(r"([a-zA-Z？?])", response[0]) == None:
+ response = translateGPT(
+ f"{speaker}",
+ "Reply with the " + LANGUAGE + " translation of the NPC name.",
+ False,
+ )
+ response[0] = response[0].title()
+ response[0] = response[0].replace("'S", "'s")
+
+ speakerList = [speaker, response[0]]
+ NAMESLIST.append(speakerList)
+ return response
+ return [speaker, [0, 0]]
+
+
+def batchList(input_list, batch_size):
+ if not isinstance(batch_size, int) or batch_size <= 0:
+ raise ValueError("batch_size must be a positive integer")
+
+ return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)]
+
+
+def createContext(fullPromptFlag, subbedT, format):
+ system = (
+ PROMPT + VOCAB
+ if fullPromptFlag
+ else f"\
+You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
+Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
+- You translate everything, including erotic and explicit content.\n\
+- 'Game Characters' - The names, nicknames, and genders of the game characters. Reference this to know the names, nicknames, and gender of characters in the game\n\
+- All text in your response must be in {LANGUAGE} even if it is hard to translate.\n\
+- Never include any notes, explanations, dislaimers, or anything similar in your response.\n\
+- Maintain any spacing in the translation.\n\
+- Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\
+- `...` can be a part of the dialogue. Translate it as it is.\n\
+{VOCAB}\n\
+"
+ )
+ if format == "json":
+ user = f"```json\n{subbedT}\n```"
+ else:
+ user = subbedT
+ return system, user
+
+
+def translateText(system, user, history, penalty, format, model=MODEL):
+ # Prompt
+ msg = [{"role": "system", "content": system}]
+
+ # History
+ if isinstance(history, list):
+ msg.extend([{"role": "system", "content": h} for h in history])
+ else:
+ msg.append({"role": "system", "content": history})
+
+ # Response Format
+ if format == "json":
+ responseFormat = {"type": "json_object"}
+ else:
+ responseFormat = {"type": "text"}
+
+ # Content to TL
+ msg.append({"role": "user", "content": f"{user}"})
+ response = openai.chat.completions.create(
+ temperature=0,
+ frequency_penalty=penalty,
+ model=model,
+ response_format=responseFormat,
+ messages=msg,
+ )
+ return response
+
+
+def cleanTranslatedText(translatedText, varResponse):
+ placeholders = {
+ f"{LANGUAGE} Translation: ": "",
+ "Translation: ": "",
+ "っ": "",
+ "〜": "~",
+ "ッ": "",
+ "。": ".",
+ "「": '\\"',
+ "」": '\\"',
+ "- ": "-",
+ "—": "―",
+ "】": "]",
+ "【": "[",
+ "Placeholder Text": "",
+ # Add more replacements as needed
+ }
+ for target, replacement in placeholders.items():
+ translatedText = translatedText.replace(target, replacement)
+
+ # Remove Repeating Characters
+ pattern = re.compile(r"(.)\s*\1(?:\s*\1){" + str(20 - 1) + r",}")
+ translatedText = pattern.sub(lambda match: match.group(0).replace(" ", "")[:20], translatedText)
+
+ # Elongate Long Dashes (Since GPT Ignores them...)
+ translatedText = elongateCharacters(translatedText)
+ return translatedText
+
+
+def elongateCharacters(text):
+ # Define a pattern to match one character followed by one or more `ー` characters
+ # Using a positive lookbehind assertion to capture the preceding character
+ pattern = r"(?<=(.))ー+"
+
+ # Define a replacement function that elongates the captured character
+ def repl(match):
+ char = match.group(1) # The character before the ー sequence
+ count = len(match.group(0)) - 1 # Number of ー characters
+ return char * count # Replace ー sequence with the character repeated
+
+ # Use re.sub() to replace the pattern in the text
+ return re.sub(pattern, repl, text)
+
+
+def extractTranslation(translatedTextList, is_list):
+ try:
+ translatedTextList = re.sub(r'\\"+\"([^,\n}])', r'\\"\1', translatedTextList)
+ translatedTextList = re.sub(r"(? instead
-FIXTEXTWRAP = True # Overwrites textwrap
-IGNORETLTEXT = False # Ignores all translated text.
-MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
-
-# tqdm Globals
-BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
-POSITION = 0
-LEAVE = False
-PBAR = None
-FILENAME = None
-
-# Full Width
-ascii_to_wide = dict((i, chr(i + 0xFEE0)) for i in range(0x21, 0x7F))
-ascii_to_wide.update({0x20: "\u3000", 0x2D: "\u2212"}) # space and minus
-wide_to_ascii = dict((i, chr(i - 0xFEE0)) for i in range(0xFF01, 0xFF5F))
-wide_to_ascii.update({0x3000: " ", 0x2212: "-"}) # space and minus
-
-# Pricing - Depends on the model https://openai.com/pricing
-# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request
-# If you are getting a MISMATCH LENGTH error, lower the batch size.
-if "gpt-3.5" in MODEL:
- INPUTAPICOST = 0.002
- OUTPUTAPICOST = 0.002
- BATCHSIZE = 10
-elif "gpt-4" in MODEL:
- INPUTAPICOST = 0.0025
- OUTPUTAPICOST = 0.01
- BATCHSIZE = 40
-else:
- INPUTAPICOST = float(os.getenv("input_cost"))
- OUTPUTAPICOST = float(os.getenv("output_cost"))
- BATCHSIZE = int(os.getenv("batchsize"))
- FREQUENCY_PENALTY = float(os.getenv("frequency_penalty"))
-
-def handleUnity(filename, estimate):
- global ESTIMATE, FILENAME
- ESTIMATE = estimate
- FILENAME = filename
-
- if ESTIMATE:
- start = time.time()
- translatedData = openFiles(filename)
-
- # Print Result
- end = time.time()
- tqdm.write(getResultString(translatedData, end - start, filename))
- with LOCK:
- TOKENS[0] += translatedData[1][0]
- TOKENS[1] += translatedData[1][1]
-
- # Print Total
- totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
-
- # Print any errors on maps
- if len(MISMATCH) > 0:
- return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
- else:
- return totalString
-
- else:
- try:
- with open("translated/" + filename, "w", encoding="utf8", errors="ignore") as outFile:
- start = time.time()
- translatedData = openFiles(filename)
-
- # Print Result
- end = time.time()
- outFile.writelines(translatedData[0])
- tqdm.write(getResultString(translatedData, end - start, filename))
- with LOCK:
- TOKENS[0] += translatedData[1][0]
- TOKENS[1] += translatedData[1][1]
- except Exception:
- traceback.print_exc()
- return "Fail"
-
- return getResultString(["", TOKENS, None], end - start, "TOTAL")
-
-
-def getResultString(translatedData, translationTime, filename):
- # File Print String
- totalTokenstring = (
- Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
- "[Output: "
- + str(translatedData[1][1])
- + "]" "[Cost: ${:,.4f}".format((translatedData[1][0] * 0.001 * INPUTAPICOST) + (translatedData[1][1] * 0.001 * OUTPUTAPICOST))
- + "]"
- )
- timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
-
- if translatedData[2] == None:
- # Success
- return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET
-
- else:
- # Fail
- try:
- raise translatedData[2]
- except Exception as e:
- traceback.print_exc()
- errorString = str(e) + Fore.RED
- return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET
-
-
-def openFiles(filename):
- with open("files/" + filename, "r", encoding="utf8") as readFile:
- translatedData = parseUnity(readFile, filename)
-
- # Delete lines marked for deletion
- finalData = []
- for line in translatedData[0]:
- if line != "\\d\n":
- finalData.append(line)
- translatedData[0] = finalData
-
- return translatedData
-
-
-def parseUnity(readFile, filename):
- totalTokens = [0, 0]
-
- # Read File into data
- data = readFile.readlines()
-
- # Create Progress Bar
- with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
- pbar.desc = filename
-
- try:
- result = translateUnity(data, pbar, filename, [])
- totalTokens[0] += result[0]
- totalTokens[1] += result[1]
- except Exception as e:
- traceback.print_exc()
- return [data, totalTokens, e]
- return [data, totalTokens, None]
-
-
-def translateUnity(data, pbar, filename, translatedList):
- stringList = []
- currentGroup = []
- tokens = [0, 0]
- speaker = ""
- voice = False
- global LOCK, ESTIMATE, PBAR
- PBAR = pbar
- i = 0
-
- # Dialogue
- while i < len(data):
- # Lines
- regex = r".*?=(.*)"
- match = re.search(regex, data[i])
- if match != None and match.group(1) != "":
- originalString = match.group(1)
- # Pass 1
- if translatedList == []:
- # Grab Consecutive Strings
- jaString = match.group(1)
-
- # Remove textwrap
- jaString = jaString.replace("\n", "")
-
- # Add String
- stringList.append(jaString.strip())
-
- # Pass 2
- else:
- # Get Text
- if translatedList:
- # Grab and Pop
- translatedText = translatedList[0]
- translatedList.pop(0)
-
- # Set to None if empty list
- if len(translatedList) <= 0:
- translatedList = None
-
- # Textwrap
- translatedText = textwrap.fill(translatedText, width=WIDTH)
- translatedText = translatedText.replace("\n", "\\n")
-
- # Remove Double Spaces and =
- translatedText = translatedText.replace(" ", " ")
- translatedText = translatedText.replace("=", "->")
-
- # Set Data
- data[i] = f"{originalString}{originalString}={translatedText}\n"
- i += 1
-
- # Nothing relevant. Skip Line.
- else:
- i += 1
-
- # EOF
- if len(stringList) > 0:
- # Set Progress
- pbar.total = len(stringList)
- pbar.refresh()
-
- # Translate
- response = translateGPT(stringList, "", True)
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- translatedList = response[0]
-
- # Set Strings
- if len(stringList) == len(translatedList):
- translateUnity(data, pbar, filename, translatedList)
-
- # Mismatch
- else:
- with LOCK:
- if filename not in MISMATCH:
- MISMATCH.append(filename)
- return tokens
-
-
-# Save some money and enter the character before translation
-def getSpeaker(speaker):
- if speaker not in str(NAMESLIST):
- response = translateGPT(
- speaker,
- "Reply with the " + LANGUAGE + " translation of the NPC name.",
- True,
- )
- response[0] = response[0].title()
- response[0] = response[0].replace("'S", "'s")
-
- # Retry if name doesn't translate for some reason
- if re.search(r"([a-zA-Z？?])", response[0]) == None:
- response = translateGPT(
- speaker,
- "Reply with the " + LANGUAGE + " translation of the NPC name.",
- False,
- )
- response[0] = response[0].title()
- response[0] = response[0].replace("'S", "'s")
-
- speakerList = [speaker, response[0]]
- NAMESLIST.append(speakerList)
- return response
-
- # Find Speaker
- else:
- for i in range(len(NAMESLIST)):
- if speaker == NAMESLIST[i][0]:
- return [NAMESLIST[i][1], [0, 0]]
-
- return [speaker, [0, 0]]
-
-
-def subVars(jaString):
- jaString = jaString.replace("\u3000", " ")
-
- # Formatting
- count = 0
- codeList = re.findall(r"[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]", jaString)
- codeList = set(codeList)
- if len(codeList) != 0:
- for var in codeList:
- jaString = jaString.replace(var, "[FCode_" + str(count) + "]")
- count += 1
-
- # Put all lists in list and return
- return [jaString, codeList]
-
-
-def resubVars(translatedText, codeList):
- # Fix Spacing and ChatGPT Nonsense
- matchList = re.findall(r"\[\s?.+?\s?\]", translatedText)
- if len(matchList) > 0:
- for match in matchList:
- text = match.strip()
- translatedText = translatedText.replace(match, text)
-
- # Formatting
- count = 0
- if len(codeList) != 0:
- for var in codeList:
- translatedText = translatedText.replace("[FCode_" + str(count) + "]", var)
- count += 1
-
- return translatedText
-
-
-def batchList(input_list, batch_size):
- if not isinstance(batch_size, int) or batch_size <= 0:
- raise ValueError("batch_size must be a positive integer")
-
- return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)]
-
-
-def createContext(fullPromptFlag, subbedT, format):
- system = (
- PROMPT + VOCAB
- if fullPromptFlag
- else f"\
-You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
-Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
-- You translate everything, including erotic and explicit content.\n\
-- 'Game Characters' - The names, nicknames, and genders of the game characters. Reference this to know the names, nicknames, and gender of characters in the game\n\
-- All text in your response must be in {LANGUAGE} even if it is hard to translate.\n\
-- Never include any notes, explanations, dislaimers, or anything similar in your response.\n\
-- Maintain any spacing in the translation.\n\
-- Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\
-- `...` can be a part of the dialogue. Translate it as it is.\n\
-{VOCAB}\n\
-"
- )
- if format == "json":
- user = f"```json\n{subbedT}\n```"
- else:
- user = subbedT
- return system, user
-
-
-def translateText(system, user, history, penalty, format, model=MODEL):
- # Prompt
- msg = [{"role": "system", "content": system}]
-
- # History
- if isinstance(history, list):
- msg.extend([{"role": "system", "content": h} for h in history])
- else:
- msg.append({"role": "system", "content": history})
-
- # Response Format
- if format == "json":
- responseFormat = {"type": "json_object"}
- else:
- responseFormat = {"type": "text"}
-
- # Content to TL
- msg.append({"role": "user", "content": f"{user}"})
- response = openai.chat.completions.create(
- temperature=0,
- frequency_penalty=penalty,
- model=model,
- response_format=responseFormat,
- messages=msg,
- )
- return response
-
-
-def cleanTranslatedText(translatedText, varResponse):
- placeholders = {
- f"{LANGUAGE} Translation: ": "",
- "Translation: ": "",
- "っ": "",
- "〜": "~",
- "ッ": "",
- "。": ".",
- "「": '\\"',
- "」": '\\"',
- "- ": "-",
- "Placeholder Text": "",
- # Add more replacements as needed
- }
- for target, replacement in placeholders.items():
- translatedText = translatedText.replace(target, replacement)
-
- # Elongate Long Dashes (Since GPT Ignores them...)
- translatedText = elongateCharacters(translatedText)
- translatedText = resubVars(translatedText, varResponse[1])
- return translatedText
-
-
-def elongateCharacters(text):
- # Define a pattern to match one character followed by one or more `ー` characters
- # Using a positive lookbehind assertion to capture the preceding character
- pattern = r"(?<=(.))ー+"
-
- # Define a replacement function that elongates the captured character
- def repl(match):
- char = match.group(1) # The character before the ー sequence
- count = len(match.group(0)) - 1 # Number of ー characters
- return char * count # Replace ー sequence with the character repeated
-
- # Use re.sub() to replace the pattern in the text
- return re.sub(pattern, repl, text)
-
-
-def extractTranslation(translatedTextList, is_list):
- try:
- line_dict = json.loads(translatedTextList)
- # If it's a batch (i.e., list), extract with tags; otherwise, return the single item.
- string_list = list(line_dict.values())
- if is_list:
- return string_list
- else:
- return string_list[0]
-
- except Exception as e:
- print(f"extractTranslation Error: {e}")
- return None
-
-
-def countTokens(system, user, history):
- inputTotalTokens = 0
- outputTotalTokens = 0
- enc = tiktoken.encoding_for_model("gpt-4")
-
- # Input
- if isinstance(history, list):
- for line in history:
- inputTotalTokens += len(enc.encode(line))
- else:
- inputTotalTokens += len(enc.encode(history))
- inputTotalTokens += len(enc.encode(system))
- inputTotalTokens += len(enc.encode(user))
-
- # Output
- outputTotalTokens += round(len(enc.encode(user)) * 3)
-
- return [inputTotalTokens, outputTotalTokens]
-
-
-@retry(exceptions=Exception, tries=5, delay=5)
-def translateGPT(text, history, fullPromptFlag):
- global PBAR, MISMATCH, FILENAME
-
- mismatch = False
- totalTokens = [0, 0]
- if isinstance(text, list):
- format = "json"
- tList = batchList(text, BATCHSIZE)
- else:
- format = "text"
- tList = [text]
-
- for index, tItem in enumerate(tList):
- # Before sending to translation, if we have a list of items, add the formatting
- if isinstance(tItem, list):
- payload = {f"Line{i+1}": string for i, string in enumerate(tItem)}
- payload = json.dumps(payload, indent=4, ensure_ascii=False)
- varResponse = subVars(payload)
- subbedT = varResponse[0]
- else:
- varResponse = subVars(tItem)
- subbedT = varResponse[0]
-
- # Things to Check before starting translation
- if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+", subbedT):
- if PBAR is not None:
- PBAR.update(len(tItem))
- continue
-
- # Create Message
- system, user = createContext(fullPromptFlag, subbedT, format)
-
- # Calculate Estimate
- if ESTIMATE:
- estimate = countTokens(system, user, history)
- totalTokens[0] += estimate[0]
- totalTokens[1] += estimate[1]
- continue
-
- # Translating
- response = translateText(system, user, history, 0.05, format)
- translatedText = response.choices[0].message.content
- totalTokens[0] += response.usage.prompt_tokens
- totalTokens[1] += response.usage.completion_tokens
-
- # Check Translation
- translatedText = cleanTranslatedText(translatedText, varResponse)
- if isinstance(tItem, list):
- extractedTranslations = extractTranslation(translatedText, True)
- if extractedTranslations == None or len(tItem) != len(extractedTranslations):
- # Mismatch. Try Again
- response = translateText(system, user, history, 0.05, format, "gpt-4o")
- translatedText = response.choices[0].message.content
- totalTokens[0] += response.usage.prompt_tokens
- totalTokens[1] += response.usage.completion_tokens
-
- # Formatting
- translatedText = cleanTranslatedText(translatedText, varResponse)
- if isinstance(tItem, list):
- extractedTranslations = extractTranslation(translatedText, True)
- if extractedTranslations == None or len(tItem) != len(extractedTranslations):
- mismatch = True # Just here for breakpoint
-
- # Set if no mismatch
- if mismatch == False:
- tList[index] = extractedTranslations
- history = extractedTranslations[-10:] # Update history if we have a list
- else:
- history = text[-10:]
- mismatch = False
- if FILENAME not in MISMATCH:
- MISMATCH.append(FILENAME)
-
- # Update Loading Bar
- with LOCK:
- if PBAR is not None:
- PBAR.update(len(tItem))
- else:
- # Ensure we're passing a single string to extractTranslation
- tList[index] = translatedText.replace("Placeholder Text", "")
-
- # Combine if multilist
- if isinstance(tList[0], list):
- tList = [t for sublist in tList for t in sublist]
-
- # Return
- if format == "json":
- return [tList, totalTokens]
- else:
- return [tList[0], totalTokens]
+# Libraries
+import json
+import os
+import re
+import textwrap
+import threading
+import time
+import traceback
+import tiktoken
+import openai
+from pathlib import Path
+from colorama import Fore
+from dotenv import load_dotenv
+from retry import retry
+from tqdm import tqdm
+
+# Open AI
+load_dotenv()
+if os.getenv("api").replace(" ", "") != "":
+ openai.base_url = os.getenv("api")
+openai.organization = os.getenv("org")
+openai.api_key = os.getenv("key")
+
+# Globals
+MODEL = os.getenv("model")
+TIMEOUT = int(os.getenv("timeout"))
+LANGUAGE = os.getenv("language").capitalize()
+PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
+VOCAB = Path("vocab.txt").read_text(encoding="utf-8")
+THREADS = int(os.getenv("threads"))
+LOCK = threading.Lock()
+WIDTH = int(os.getenv("width"))
+LISTWIDTH = int(os.getenv("listWidth"))
+NOTEWIDTH = 70
+MAXHISTORY = 10
+ESTIMATE = ""
+TOKENS = [0, 0]
+NAMESLIST = []
+NAMES = False # Output a list of all the character names found
+BRFLAG = False # If the game uses
instead
+FIXTEXTWRAP = True # Overwrites textwrap
+IGNORETLTEXT = False # Ignores all translated text.
+MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
+
+# tqdm Globals
+BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
+POSITION = 0
+LEAVE = False
+PBAR = None
+FILENAME = None
+
+# Full Width
+ascii_to_wide = dict((i, chr(i + 0xFEE0)) for i in range(0x21, 0x7F))
+ascii_to_wide.update({0x20: "\u3000", 0x2D: "\u2212"}) # space and minus
+wide_to_ascii = dict((i, chr(i - 0xFEE0)) for i in range(0xFF01, 0xFF5F))
+wide_to_ascii.update({0x3000: " ", 0x2212: "-"}) # space and minus
+
+# Pricing - Depends on the model https://openai.com/pricing
+# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request
+# If you are getting a MISMATCH LENGTH error, lower the batch size.
+if "gpt-3.5" in MODEL:
+ INPUTAPICOST = 0.002
+ OUTPUTAPICOST = 0.002
+ BATCHSIZE = 10
+elif "gpt-4" in MODEL:
+ INPUTAPICOST = 0.0025
+ OUTPUTAPICOST = 0.01
+ BATCHSIZE = 40
+else:
+ INPUTAPICOST = float(os.getenv("input_cost"))
+ OUTPUTAPICOST = float(os.getenv("output_cost"))
+ BATCHSIZE = int(os.getenv("batchsize"))
+ FREQUENCY_PENALTY = float(os.getenv("frequency_penalty"))
+
+
+def handleUnity(filename, estimate):
+ global ESTIMATE, FILENAME
+ ESTIMATE = estimate
+ FILENAME = filename
+
+ if ESTIMATE:
+ start = time.time()
+ translatedData = openFiles(filename)
+
+ # Print Result
+ end = time.time()
+ tqdm.write(getResultString(translatedData, end - start, filename))
+ with LOCK:
+ TOKENS[0] += translatedData[1][0]
+ TOKENS[1] += translatedData[1][1]
+
+ # Print Total
+ totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
+
+ # Print any errors on maps
+ if len(MISMATCH) > 0:
+ return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
+ else:
+ return totalString
+
+ else:
+ try:
+ with open("translated/" + filename, "w", encoding="utf8", errors="ignore") as outFile:
+ start = time.time()
+ translatedData = openFiles(filename)
+
+ # Print Result
+ end = time.time()
+ outFile.writelines(translatedData[0])
+ tqdm.write(getResultString(translatedData, end - start, filename))
+ with LOCK:
+ TOKENS[0] += translatedData[1][0]
+ TOKENS[1] += translatedData[1][1]
+ except Exception:
+ traceback.print_exc()
+ return "Fail"
+
+ return getResultString(["", TOKENS, None], end - start, "TOTAL")
+
+
+def getResultString(translatedData, translationTime, filename):
+ # File Print String
+ totalTokenstring = (
+ Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
+ "[Output: "
+ + str(translatedData[1][1])
+ + "]" "[Cost: ${:,.4f}".format((translatedData[1][0] * 0.001 * INPUTAPICOST) + (translatedData[1][1] * 0.001 * OUTPUTAPICOST))
+ + "]"
+ )
+ timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
+
+ if translatedData[2] == None:
+ # Success
+ return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET
+
+ else:
+ # Fail
+ try:
+ raise translatedData[2]
+ except Exception as e:
+ traceback.print_exc()
+ errorString = str(e) + Fore.RED
+ return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET
+
+
+def openFiles(filename):
+ with open("files/" + filename, "r", encoding="utf8") as readFile:
+ translatedData = parseUnity(readFile, filename)
+
+ # Delete lines marked for deletion
+ finalData = []
+ for line in translatedData[0]:
+ if line != "\\d\n":
+ finalData.append(line)
+ translatedData[0] = finalData
+
+ return translatedData
+
+
+def parseUnity(readFile, filename):
+ totalTokens = [0, 0]
+
+ # Read File into data
+ data = readFile.readlines()
+
+ # Create Progress Bar
+ with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
+ pbar.desc = filename
+
+ try:
+ result = translateUnity(data, pbar, filename, [])
+ totalTokens[0] += result[0]
+ totalTokens[1] += result[1]
+ except Exception as e:
+ traceback.print_exc()
+ return [data, totalTokens, e]
+ return [data, totalTokens, None]
+
+
+def translateUnity(data, pbar, filename, translatedList):
+ stringList = []
+ currentGroup = []
+ tokens = [0, 0]
+ speaker = ""
+ voice = False
+ global LOCK, ESTIMATE, PBAR
+ PBAR = pbar
+ i = 0
+
+ # Dialogue
+ while i < len(data):
+ # Lines
+ regex = r".*?=(.*)"
+ match = re.search(regex, data[i])
+ if match != None and match.group(1) != "":
+ originalString = match.group(1)
+ # Pass 1
+ if translatedList == []:
+ # Grab Consecutive Strings
+ jaString = match.group(1)
+
+ # Remove textwrap
+ jaString = jaString.replace("\n", "")
+
+ # Add String
+ stringList.append(jaString.strip())
+
+ # Pass 2
+ else:
+ # Get Text
+ if translatedList:
+ # Grab and Pop
+ translatedText = translatedList[0]
+ translatedList.pop(0)
+
+ # Set to None if empty list
+ if len(translatedList) <= 0:
+ translatedList = None
+
+ # Textwrap
+ translatedText = textwrap.fill(translatedText, width=WIDTH)
+ translatedText = translatedText.replace("\n", "\\n")
+
+ # Remove Double Spaces and =
+ translatedText = translatedText.replace(" ", " ")
+ translatedText = translatedText.replace("=", "->")
+
+ # Set Data
+ data[i] = f"{originalString}{originalString}={translatedText}\n"
+ i += 1
+
+ # Nothing relevant. Skip Line.
+ else:
+ i += 1
+
+ # EOF
+ if len(stringList) > 0:
+ # Set Progress
+ pbar.total = len(stringList)
+ pbar.refresh()
+
+ # Translate
+ response = translateGPT(stringList, "", True)
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ translatedList = response[0]
+
+ # Set Strings
+ if len(stringList) == len(translatedList):
+ translateUnity(data, pbar, filename, translatedList)
+
+ # Mismatch
+ else:
+ with LOCK:
+ if filename not in MISMATCH:
+ MISMATCH.append(filename)
+ return tokens
+
+
+# Save some money and enter the character before translation
+def getSpeaker(speaker):
+ if speaker not in str(NAMESLIST):
+ response = translateGPT(
+ speaker,
+ "Reply with the " + LANGUAGE + " translation of the NPC name.",
+ True,
+ )
+ response[0] = response[0].title()
+ response[0] = response[0].replace("'S", "'s")
+
+ # Retry if name doesn't translate for some reason
+ if re.search(r"([a-zA-Z？?])", response[0]) == None:
+ response = translateGPT(
+ speaker,
+ "Reply with the " + LANGUAGE + " translation of the NPC name.",
+ False,
+ )
+ response[0] = response[0].title()
+ response[0] = response[0].replace("'S", "'s")
+
+ speakerList = [speaker, response[0]]
+ NAMESLIST.append(speakerList)
+ return response
+
+ # Find Speaker
+ else:
+ for i in range(len(NAMESLIST)):
+ if speaker == NAMESLIST[i][0]:
+ return [NAMESLIST[i][1], [0, 0]]
+
+ return [speaker, [0, 0]]
+
+
+def subVars(jaString):
+ jaString = jaString.replace("\u3000", " ")
+
+ # Formatting
+ count = 0
+ codeList = re.findall(r"[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]", jaString)
+ codeList = set(codeList)
+ if len(codeList) != 0:
+ for var in codeList:
+ jaString = jaString.replace(var, "[FCode_" + str(count) + "]")
+ count += 1
+
+ # Put all lists in list and return
+ return [jaString, codeList]
+
+
+def resubVars(translatedText, codeList):
+ # Fix Spacing and ChatGPT Nonsense
+ matchList = re.findall(r"\[\s?.+?\s?\]", translatedText)
+ if len(matchList) > 0:
+ for match in matchList:
+ text = match.strip()
+ translatedText = translatedText.replace(match, text)
+
+ # Formatting
+ count = 0
+ if len(codeList) != 0:
+ for var in codeList:
+ translatedText = translatedText.replace("[FCode_" + str(count) + "]", var)
+ count += 1
+
+ return translatedText
+
+
+def batchList(input_list, batch_size):
+ if not isinstance(batch_size, int) or batch_size <= 0:
+ raise ValueError("batch_size must be a positive integer")
+
+ return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)]
+
+
+def createContext(fullPromptFlag, subbedT, format):
+ system = (
+ PROMPT + VOCAB
+ if fullPromptFlag
+ else f"\
+You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
+Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
+- You translate everything, including erotic and explicit content.\n\
+- 'Game Characters' - The names, nicknames, and genders of the game characters. Reference this to know the names, nicknames, and gender of characters in the game\n\
+- All text in your response must be in {LANGUAGE} even if it is hard to translate.\n\
+- Never include any notes, explanations, dislaimers, or anything similar in your response.\n\
+- Maintain any spacing in the translation.\n\
+- Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\
+- `...` can be a part of the dialogue. Translate it as it is.\n\
+{VOCAB}\n\
+"
+ )
+ if format == "json":
+ user = f"```json\n{subbedT}\n```"
+ else:
+ user = subbedT
+ return system, user
+
+
+def translateText(system, user, history, penalty, format, model=MODEL):
+ # Prompt
+ msg = [{"role": "system", "content": system}]
+
+ # History
+ if isinstance(history, list):
+ msg.extend([{"role": "system", "content": h} for h in history])
+ else:
+ msg.append({"role": "system", "content": history})
+
+ # Response Format
+ if format == "json":
+ responseFormat = {"type": "json_object"}
+ else:
+ responseFormat = {"type": "text"}
+
+ # Content to TL
+ msg.append({"role": "user", "content": f"{user}"})
+ response = openai.chat.completions.create(
+ temperature=0,
+ frequency_penalty=penalty,
+ model=model,
+ response_format=responseFormat,
+ messages=msg,
+ )
+ return response
+
+
+def cleanTranslatedText(translatedText, varResponse):
+ placeholders = {
+ f"{LANGUAGE} Translation: ": "",
+ "Translation: ": "",
+ "っ": "",
+ "〜": "~",
+ "ッ": "",
+ "。": ".",
+ "「": '\\"',
+ "」": '\\"',
+ "- ": "-",
+ "Placeholder Text": "",
+ # Add more replacements as needed
+ }
+ for target, replacement in placeholders.items():
+ translatedText = translatedText.replace(target, replacement)
+
+ # Elongate Long Dashes (Since GPT Ignores them...)
+ translatedText = elongateCharacters(translatedText)
+ translatedText = resubVars(translatedText, varResponse[1])
+ return translatedText
+
+
+def elongateCharacters(text):
+ # Define a pattern to match one character followed by one or more `ー` characters
+ # Using a positive lookbehind assertion to capture the preceding character
+ pattern = r"(?<=(.))ー+"
+
+ # Define a replacement function that elongates the captured character
+ def repl(match):
+ char = match.group(1) # The character before the ー sequence
+ count = len(match.group(0)) - 1 # Number of ー characters
+ return char * count # Replace ー sequence with the character repeated
+
+ # Use re.sub() to replace the pattern in the text
+ return re.sub(pattern, repl, text)
+
+
+def extractTranslation(translatedTextList, is_list):
+ try:
+ line_dict = json.loads(translatedTextList)
+ # If it's a batch (i.e., list), extract with tags; otherwise, return the single item.
+ string_list = list(line_dict.values())
+ if is_list:
+ return string_list
+ else:
+ return string_list[0]
+
+ except Exception as e:
+ print(f"extractTranslation Error: {e}")
+ return None
+
+
+def countTokens(system, user, history):
+ inputTotalTokens = 0
+ outputTotalTokens = 0
+ enc = tiktoken.encoding_for_model("gpt-4")
+
+ # Input
+ if isinstance(history, list):
+ for line in history:
+ inputTotalTokens += len(enc.encode(line))
+ else:
+ inputTotalTokens += len(enc.encode(history))
+ inputTotalTokens += len(enc.encode(system))
+ inputTotalTokens += len(enc.encode(user))
+
+ # Output
+ outputTotalTokens += round(len(enc.encode(user)) * 3)
+
+ return [inputTotalTokens, outputTotalTokens]
+
+
+@retry(exceptions=Exception, tries=5, delay=5)
+def translateGPT(text, history, fullPromptFlag):
+ global PBAR, MISMATCH, FILENAME
+
+ mismatch = False
+ totalTokens = [0, 0]
+ if isinstance(text, list):
+ format = "json"
+ tList = batchList(text, BATCHSIZE)
+ else:
+ format = "text"
+ tList = [text]
+
+ for index, tItem in enumerate(tList):
+ # Before sending to translation, if we have a list of items, add the formatting
+ if isinstance(tItem, list):
+ payload = {f"Line{i+1}": string for i, string in enumerate(tItem)}
+ payload = json.dumps(payload, indent=4, ensure_ascii=False)
+ varResponse = subVars(payload)
+ subbedT = varResponse[0]
+ else:
+ varResponse = subVars(tItem)
+ subbedT = varResponse[0]
+
+ # Things to Check before starting translation
+ if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+", subbedT):
+ if PBAR is not None:
+ PBAR.update(len(tItem))
+ continue
+
+ # Create Message
+ system, user = createContext(fullPromptFlag, subbedT, format)
+
+ # Calculate Estimate
+ if ESTIMATE:
+ estimate = countTokens(system, user, history)
+ totalTokens[0] += estimate[0]
+ totalTokens[1] += estimate[1]
+ continue
+
+ # Translating
+ response = translateText(system, user, history, 0.05, format)
+ translatedText = response.choices[0].message.content
+ totalTokens[0] += response.usage.prompt_tokens
+ totalTokens[1] += response.usage.completion_tokens
+
+ # Check Translation
+ translatedText = cleanTranslatedText(translatedText, varResponse)
+ if isinstance(tItem, list):
+ extractedTranslations = extractTranslation(translatedText, True)
+ if extractedTranslations == None or len(tItem) != len(extractedTranslations):
+ # Mismatch. Try Again
+ response = translateText(system, user, history, 0.05, format, "gpt-4o")
+ translatedText = response.choices[0].message.content
+ totalTokens[0] += response.usage.prompt_tokens
+ totalTokens[1] += response.usage.completion_tokens
+
+ # Formatting
+ translatedText = cleanTranslatedText(translatedText, varResponse)
+ if isinstance(tItem, list):
+ extractedTranslations = extractTranslation(translatedText, True)
+ if extractedTranslations == None or len(tItem) != len(extractedTranslations):
+ mismatch = True # Just here for breakpoint
+
+ # Set if no mismatch
+ if mismatch == False:
+ tList[index] = extractedTranslations
+ history = extractedTranslations[-10:] # Update history if we have a list
+ else:
+ history = text[-10:]
+ mismatch = False
+ if FILENAME not in MISMATCH:
+ MISMATCH.append(FILENAME)
+
+ # Update Loading Bar
+ with LOCK:
+ if PBAR is not None:
+ PBAR.update(len(tItem))
+ else:
+ # Ensure we're passing a single string to extractTranslation
+ tList[index] = translatedText.replace("Placeholder Text", "")
+
+ # Combine if multilist
+ if isinstance(tList[0], list):
+ tList = [t for sublist in tList for t in sublist]
+
+ # Return
+ if format == "json":
+ return [tList, totalTokens]
+ else:
+ return [tList[0], totalTokens]
diff --git a/modules/wolf2.py b/modules/wolf2.py
index 76b614e..2859761 100644
--- a/modules/wolf2.py
+++ b/modules/wolf2.py
@@ -1,579 +1,580 @@
-# Libraries
-import json
-import os
-import re
-import textwrap
-import threading
-import time
-import traceback
-import tiktoken
-import openai
-from pathlib import Path
-from colorama import Fore
-from dotenv import load_dotenv
-from retry import retry
-from tqdm import tqdm
-
-# Open AI
-load_dotenv()
-if os.getenv("api").replace(" ", "") != "":
- openai.base_url = os.getenv("api")
-openai.organization = os.getenv("org")
-openai.api_key = os.getenv("key")
-
-# Globals
-MODEL = os.getenv("model")
-TIMEOUT = int(os.getenv("timeout"))
-LANGUAGE = os.getenv("language").capitalize()
-PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
-VOCAB = Path("vocab.txt").read_text(encoding="utf-8")
-THREADS = int(os.getenv("threads"))
-LOCK = threading.Lock()
-WIDTH = int(os.getenv("width"))
-LISTWIDTH = int(os.getenv("listWidth"))
-NOTEWIDTH = 70
-MAXHISTORY = 10
-ESTIMATE = ""
-TOKENS = [0, 0]
-NAMESLIST = []
-NAMES = False # Output a list of all the character names found
-BRFLAG = False # If the game uses
instead
-FIXTEXTWRAP = True # Overwrites textwrap
-IGNORETLTEXT = False # Ignores all translated text.
-MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
-
-# tqdm Globals
-BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
-POSITION = 0
-LEAVE = False
-
-# Pricing - Depends on the model https://openai.com/pricing
-# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request
-# If you are getting a MISMATCH LENGTH error, lower the batch size.
-if "gpt-3.5" in MODEL:
- INPUTAPICOST = 0.002
- OUTPUTAPICOST = 0.002
- BATCHSIZE = 10
-elif "gpt-4" in MODEL:
- INPUTAPICOST = 0.0025
- OUTPUTAPICOST = 0.01
- BATCHSIZE = 40
-else:
- INPUTAPICOST = float(os.getenv("input_cost"))
- OUTPUTAPICOST = float(os.getenv("output_cost"))
- BATCHSIZE = int(os.getenv("batchsize"))
- FREQUENCY_PENALTY = float(os.getenv("frequency_penalty"))
-
-def handleWOLF2(filename, estimate):
- global ESTIMATE
- ESTIMATE = estimate
-
- if ESTIMATE:
- start = time.time()
- translatedData = openFiles(filename)
-
- # Print Result
- end = time.time()
- tqdm.write(getResultString(translatedData, end - start, filename))
- with LOCK:
- TOKENS[0] += translatedData[1][0]
- TOKENS[1] += translatedData[1][1]
-
- # Print Total
- totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
-
- # Print any errors on maps
- if len(MISMATCH) > 0:
- return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
- else:
- return totalString
-
- else:
- try:
- with open("translated/" + filename, "w", encoding="shift_jis", errors="ignore") as outFile:
- start = time.time()
- translatedData = openFiles(filename)
-
- # Print Result
- end = time.time()
- outFile.writelines(translatedData[0])
- tqdm.write(getResultString(translatedData, end - start, filename))
- with LOCK:
- TOKENS[0] += translatedData[1][0]
- TOKENS[1] += translatedData[1][1]
- except Exception:
- traceback.print_exc()
- return "Fail"
-
- return getResultString(["", TOKENS, None], end - start, "TOTAL")
-
-
-def getResultString(translatedData, translationTime, filename):
- # File Print String
- totalTokenstring = (
- Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
- "[Output: "
- + str(translatedData[1][1])
- + "]" "[Cost: ${:,.4f}".format((translatedData[1][0] * 0.001 * INPUTAPICOST) + (translatedData[1][1] * 0.001 * OUTPUTAPICOST))
- + "]"
- )
- timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
-
- if translatedData[2] == None:
- # Success
- return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET
-
- else:
- # Fail
- try:
- raise translatedData[2]
- except Exception as e:
- traceback.print_exc()
- errorString = str(e) + Fore.RED
- return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET
-
-
-def openFiles(filename):
- with open("files/" + filename, "r", encoding="shift_jis") as readFile:
- translatedData = parseWOLF(readFile, filename)
-
- # Delete lines marked for deletion
- finalData = []
- for line in translatedData[0]:
- if line != "\\d\n":
- finalData.append(line)
- translatedData[0] = finalData
-
- return translatedData
-
-
-def parseWOLF(readFile, filename):
- totalTokens = [0, 0]
-
- # Read File into data
- data = readFile.readlines()
-
- # Create Progress Bar
- with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
- pbar.desc = filename
-
- try:
- result = translateWOLF(data, [], pbar, filename)
- totalTokens[0] += result[0]
- totalTokens[1] += result[1]
- except Exception as e:
- traceback.print_exc()
- return [data, totalTokens, e]
- return [data, totalTokens, None]
-
-
-def translateWOLF(data, translatedList, pbar, filename):
- stringList = []
- currentGroup = []
- tokens = [0, 0]
- speaker = ""
- global LOCK, ESTIMATE, PBAR
- PBAR = pbar
- i = 0
-
- while i < len(data):
- # Speaker
- matchList = re.findall(r"(.*)：", data[i])
- if len(matchList) != 0:
- response = getSpeaker(matchList[0])
- speaker = response[0]
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- data[i] = f"{speaker}：\n"
- i += 1
- else:
- speaker = ""
-
- # Options
- if "//選択肢" in data[i]:
- i += 1
- choiceList = []
- initialIndex = i
- while "//" in data[i] and "の場合" not in data[i]:
- choiceList.append(re.search(r"\/\/(.*)", data[i]).group(1))
- i += 1
-
- # Translate
- response = translateGPT(choiceList, "This will be a dialogue option", True)
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- choiceListTL = response[0]
-
- # Set Data
- if len(choiceList) == len(choiceListTL):
- # Set Data
- i = initialIndex
- while "//" in data[i] and "の場合" not in data[i]:
- choiceListTL[0] = choiceListTL[0].replace(", ", "、")
- data[i] = f"//{choiceListTL[0]}\n"
- choiceListTL.pop(0)
- i += 1
-
- # Mismatch
- else:
- with LOCK:
- if filename not in MISMATCH:
- MISMATCH.append(filename)
-
- # Lines
- if r"/" not in data[i] and "@" not in data[i] and data[i] != "\n":
- # Pass 1
- if translatedList == []:
- # Grab Consecutive Strings
- currentGroup.append(data[i])
- i += 1
- while i < len(data) and r"/" not in data[i] and "@" not in data[i] and data[i] != "\n":
- currentGroup.append(data[i])
- i += 1
-
- # Join up 401 groups for better translation.
- if len(currentGroup) > 0:
- jaString = "".join(currentGroup)
- currentGroup = []
-
- # Remove any textwrap
- jaString = jaString.replace("\n", " ")
-
- # Add Speaker (If there is one)
- if speaker != "":
- jaString = f"{speaker}: {jaString}"
-
- # Add String
- stringList.append(jaString)
- i += 1
-
- # Pass 2
- else:
- # Insert Strings
- while i < len(data) and r"/" not in data[i] and "@" not in data[i] and data[i] != "\n":
- data.pop(i)
-
- # Get Text
- translatedText = translatedList[0]
- translatedList.pop(0)
-
- if len(translatedList) <= 0:
- translatedList = None
-
- # Remove added speaker
- # translatedText = re.sub(r"^.+?:\s", "", translatedText)
-
- # Textwrap
- translatedText = textwrap.fill(translatedText, width=WIDTH)
-
- # Set Data
- data.insert(i, f"{translatedText}\n")
- i += 1
-
- # Nothing relevant. Skip Line.
- else:
- i += 1
-
- # EOF
- if len(stringList) > 0:
- # Set Progress
- pbar.total = len(stringList)
- pbar.refresh()
-
- # Translate
- response = translateGPT(stringList, "", True)
- tokens[0] += response[1][0]
- tokens[1] += response[1][1]
- translatedList = response[0]
-
- # Set Strings
- if len(stringList) == len(translatedList):
- translateWOLF(data, translatedList, pbar, filename)
-
- # Mismatch
- else:
- with LOCK:
- if filename not in MISMATCH:
- MISMATCH.append(filename)
- return tokens
-
-
-# Save some money and enter the character before translation
-def getSpeaker(speaker):
- match speaker:
- case "ファイン":
- return ["Fine", [0, 0]]
- case "":
- return ["", [0, 0]]
- case _:
- # Find Speaker
- for i in range(len(NAMESLIST)):
- if speaker == NAMESLIST[i][0]:
- return [NAMESLIST[i][1], [0, 0]]
-
- # Translate and Store Speaker
- response = translateGPT(
- f"{speaker}",
- "Reply with the " + LANGUAGE + " translation of the NPC name.",
- True,
- )
- response[0] = response[0].title()
- response[0] = response[0].replace("'S", "'s")
- response[0] = response[0].replace("Speaker: ", "")
-
- # Retry if name doesn't translate for some reason
- if re.search(r"([a-zA-Z？?])", response[0]) == None:
- response = translateGPT(
- f"{speaker}",
- "Reply with the " + LANGUAGE + " translation of the NPC name.",
- False,
- )
- response[0] = response[0].title()
- response[0] = response[0].replace("'S", "'s")
-
- speakerList = [speaker, response[0]]
- NAMESLIST.append(speakerList)
- return response
- return [speaker, [0, 0]]
-
-
-def batchList(input_list, batch_size):
- if not isinstance(batch_size, int) or batch_size <= 0:
- raise ValueError("batch_size must be a positive integer")
-
- return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)]
-
-
-def createContext(fullPromptFlag, subbedT, format):
- system = (
- PROMPT + VOCAB
- if fullPromptFlag
- else f"\
-You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
-Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
-- You translate everything, including erotic and explicit content.\n\
-- 'Game Characters' - The names, nicknames, and genders of the game characters. Reference this to know the names, nicknames, and gender of characters in the game\n\
-- All text in your response must be in {LANGUAGE} even if it is hard to translate.\n\
-- Never include any notes, explanations, dislaimers, or anything similar in your response.\n\
-- Maintain any spacing in the translation.\n\
-- Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\
-- `...` can be a part of the dialogue. Translate it as it is.\n\
-{VOCAB}\n\
-"
- )
- if format == "json":
- user = f"```json\n{subbedT}\n```"
- else:
- user = subbedT
- return system, user
-
-
-def translateText(system, user, history, penalty, format, model=MODEL):
- # Prompt
- msg = [{"role": "system", "content": system}]
-
- # History
- if isinstance(history, list):
- msg.extend([{"role": "system", "content": h} for h in history])
- else:
- msg.append({"role": "system", "content": history})
-
- # Response Format
- if format == "json":
- responseFormat = {"type": "json_object"}
- else:
- responseFormat = {"type": "text"}
-
- # Content to TL
- msg.append({"role": "user", "content": f"{user}"})
- response = openai.chat.completions.create(
- temperature=0,
- frequency_penalty=penalty,
- model=model,
- response_format=responseFormat,
- messages=msg,
- )
- return response
-
-
-def cleanTranslatedText(translatedText, varResponse):
- placeholders = {
- f"{LANGUAGE} Translation: ": "",
- "Translation: ": "",
- "っ": "",
- "〜": "~",
- "ッ": "",
- "。": ".",
- "「": '\\"',
- "」": '\\"',
- "- ": "-",
- "—": "―",
- "】": "]",
- "【": "[",
- "Placeholder Text": "",
- # Add more replacements as needed
- }
- for target, replacement in placeholders.items():
- translatedText = translatedText.replace(target, replacement)
-
- # Remove Repeating Characters
- pattern = re.compile(r"(.)\s*\1(?:\s*\1){" + str(20 - 1) + r",}")
- translatedText = pattern.sub(lambda match: match.group(0).replace(" ", "")[:20], translatedText)
-
- # Elongate Long Dashes (Since GPT Ignores them...)
- translatedText = elongateCharacters(translatedText)
- return translatedText
-
-
-def elongateCharacters(text):
- # Define a pattern to match one character followed by one or more `ー` characters
- # Using a positive lookbehind assertion to capture the preceding character
- pattern = r"(?<=(.))ー+"
-
- # Define a replacement function that elongates the captured character
- def repl(match):
- char = match.group(1) # The character before the ー sequence
- count = len(match.group(0)) - 1 # Number of ー characters
- return char * count # Replace ー sequence with the character repeated
-
- # Use re.sub() to replace the pattern in the text
- return re.sub(pattern, repl, text)
-
-
-def extractTranslation(translatedTextList, is_list):
- try:
- translatedTextList = re.sub(r'\\"+\"([^,\n}])', r'\\"\1', translatedTextList)
- translatedTextList = re.sub(r"(? instead
+FIXTEXTWRAP = True # Overwrites textwrap
+IGNORETLTEXT = False # Ignores all translated text.
+MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
+
+# tqdm Globals
+BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
+POSITION = 0
+LEAVE = False
+
+# Pricing - Depends on the model https://openai.com/pricing
+# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request
+# If you are getting a MISMATCH LENGTH error, lower the batch size.
+if "gpt-3.5" in MODEL:
+ INPUTAPICOST = 0.002
+ OUTPUTAPICOST = 0.002
+ BATCHSIZE = 10
+elif "gpt-4" in MODEL:
+ INPUTAPICOST = 0.0025
+ OUTPUTAPICOST = 0.01
+ BATCHSIZE = 40
+else:
+ INPUTAPICOST = float(os.getenv("input_cost"))
+ OUTPUTAPICOST = float(os.getenv("output_cost"))
+ BATCHSIZE = int(os.getenv("batchsize"))
+ FREQUENCY_PENALTY = float(os.getenv("frequency_penalty"))
+
+
+def handleWOLF2(filename, estimate):
+ global ESTIMATE
+ ESTIMATE = estimate
+
+ if ESTIMATE:
+ start = time.time()
+ translatedData = openFiles(filename)
+
+ # Print Result
+ end = time.time()
+ tqdm.write(getResultString(translatedData, end - start, filename))
+ with LOCK:
+ TOKENS[0] += translatedData[1][0]
+ TOKENS[1] += translatedData[1][1]
+
+ # Print Total
+ totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
+
+ # Print any errors on maps
+ if len(MISMATCH) > 0:
+ return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
+ else:
+ return totalString
+
+ else:
+ try:
+ with open("translated/" + filename, "w", encoding="shift_jis", errors="ignore") as outFile:
+ start = time.time()
+ translatedData = openFiles(filename)
+
+ # Print Result
+ end = time.time()
+ outFile.writelines(translatedData[0])
+ tqdm.write(getResultString(translatedData, end - start, filename))
+ with LOCK:
+ TOKENS[0] += translatedData[1][0]
+ TOKENS[1] += translatedData[1][1]
+ except Exception:
+ traceback.print_exc()
+ return "Fail"
+
+ return getResultString(["", TOKENS, None], end - start, "TOTAL")
+
+
+def getResultString(translatedData, translationTime, filename):
+ # File Print String
+ totalTokenstring = (
+ Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
+ "[Output: "
+ + str(translatedData[1][1])
+ + "]" "[Cost: ${:,.4f}".format((translatedData[1][0] * 0.001 * INPUTAPICOST) + (translatedData[1][1] * 0.001 * OUTPUTAPICOST))
+ + "]"
+ )
+ timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
+
+ if translatedData[2] == None:
+ # Success
+ return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET
+
+ else:
+ # Fail
+ try:
+ raise translatedData[2]
+ except Exception as e:
+ traceback.print_exc()
+ errorString = str(e) + Fore.RED
+ return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET
+
+
+def openFiles(filename):
+ with open("files/" + filename, "r", encoding="shift_jis") as readFile:
+ translatedData = parseWOLF(readFile, filename)
+
+ # Delete lines marked for deletion
+ finalData = []
+ for line in translatedData[0]:
+ if line != "\\d\n":
+ finalData.append(line)
+ translatedData[0] = finalData
+
+ return translatedData
+
+
+def parseWOLF(readFile, filename):
+ totalTokens = [0, 0]
+
+ # Read File into data
+ data = readFile.readlines()
+
+ # Create Progress Bar
+ with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
+ pbar.desc = filename
+
+ try:
+ result = translateWOLF(data, [], pbar, filename)
+ totalTokens[0] += result[0]
+ totalTokens[1] += result[1]
+ except Exception as e:
+ traceback.print_exc()
+ return [data, totalTokens, e]
+ return [data, totalTokens, None]
+
+
+def translateWOLF(data, translatedList, pbar, filename):
+ stringList = []
+ currentGroup = []
+ tokens = [0, 0]
+ speaker = ""
+ global LOCK, ESTIMATE, PBAR
+ PBAR = pbar
+ i = 0
+
+ while i < len(data):
+ # Speaker
+ matchList = re.findall(r"(.*)：", data[i])
+ if len(matchList) != 0:
+ response = getSpeaker(matchList[0])
+ speaker = response[0]
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ data[i] = f"{speaker}：\n"
+ i += 1
+ else:
+ speaker = ""
+
+ # Options
+ if "//選択肢" in data[i]:
+ i += 1
+ choiceList = []
+ initialIndex = i
+ while "//" in data[i] and "の場合" not in data[i]:
+ choiceList.append(re.search(r"\/\/(.*)", data[i]).group(1))
+ i += 1
+
+ # Translate
+ response = translateGPT(choiceList, "This will be a dialogue option", True)
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ choiceListTL = response[0]
+
+ # Set Data
+ if len(choiceList) == len(choiceListTL):
+ # Set Data
+ i = initialIndex
+ while "//" in data[i] and "の場合" not in data[i]:
+ choiceListTL[0] = choiceListTL[0].replace(", ", "、")
+ data[i] = f"//{choiceListTL[0]}\n"
+ choiceListTL.pop(0)
+ i += 1
+
+ # Mismatch
+ else:
+ with LOCK:
+ if filename not in MISMATCH:
+ MISMATCH.append(filename)
+
+ # Lines
+ if r"/" not in data[i] and "@" not in data[i] and data[i] != "\n":
+ # Pass 1
+ if translatedList == []:
+ # Grab Consecutive Strings
+ currentGroup.append(data[i])
+ i += 1
+ while i < len(data) and r"/" not in data[i] and "@" not in data[i] and data[i] != "\n":
+ currentGroup.append(data[i])
+ i += 1
+
+ # Join up 401 groups for better translation.
+ if len(currentGroup) > 0:
+ jaString = "".join(currentGroup)
+ currentGroup = []
+
+ # Remove any textwrap
+ jaString = jaString.replace("\n", " ")
+
+ # Add Speaker (If there is one)
+ if speaker != "":
+ jaString = f"{speaker}: {jaString}"
+
+ # Add String
+ stringList.append(jaString)
+ i += 1
+
+ # Pass 2
+ else:
+ # Insert Strings
+ while i < len(data) and r"/" not in data[i] and "@" not in data[i] and data[i] != "\n":
+ data.pop(i)
+
+ # Get Text
+ translatedText = translatedList[0]
+ translatedList.pop(0)
+
+ if len(translatedList) <= 0:
+ translatedList = None
+
+ # Remove added speaker
+ # translatedText = re.sub(r"^.+?:\s", "", translatedText)
+
+ # Textwrap
+ translatedText = textwrap.fill(translatedText, width=WIDTH)
+
+ # Set Data
+ data.insert(i, f"{translatedText}\n")
+ i += 1
+
+ # Nothing relevant. Skip Line.
+ else:
+ i += 1
+
+ # EOF
+ if len(stringList) > 0:
+ # Set Progress
+ pbar.total = len(stringList)
+ pbar.refresh()
+
+ # Translate
+ response = translateGPT(stringList, "", True)
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ translatedList = response[0]
+
+ # Set Strings
+ if len(stringList) == len(translatedList):
+ translateWOLF(data, translatedList, pbar, filename)
+
+ # Mismatch
+ else:
+ with LOCK:
+ if filename not in MISMATCH:
+ MISMATCH.append(filename)
+ return tokens
+
+
+# Save some money and enter the character before translation
+def getSpeaker(speaker):
+ match speaker:
+ case "ファイン":
+ return ["Fine", [0, 0]]
+ case "":
+ return ["", [0, 0]]
+ case _:
+ # Find Speaker
+ for i in range(len(NAMESLIST)):
+ if speaker == NAMESLIST[i][0]:
+ return [NAMESLIST[i][1], [0, 0]]
+
+ # Translate and Store Speaker
+ response = translateGPT(
+ f"{speaker}",
+ "Reply with the " + LANGUAGE + " translation of the NPC name.",
+ True,
+ )
+ response[0] = response[0].title()
+ response[0] = response[0].replace("'S", "'s")
+ response[0] = response[0].replace("Speaker: ", "")
+
+ # Retry if name doesn't translate for some reason
+ if re.search(r"([a-zA-Z？?])", response[0]) == None:
+ response = translateGPT(
+ f"{speaker}",
+ "Reply with the " + LANGUAGE + " translation of the NPC name.",
+ False,
+ )
+ response[0] = response[0].title()
+ response[0] = response[0].replace("'S", "'s")
+
+ speakerList = [speaker, response[0]]
+ NAMESLIST.append(speakerList)
+ return response
+ return [speaker, [0, 0]]
+
+
+def batchList(input_list, batch_size):
+ if not isinstance(batch_size, int) or batch_size <= 0:
+ raise ValueError("batch_size must be a positive integer")
+
+ return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)]
+
+
+def createContext(fullPromptFlag, subbedT, format):
+ system = (
+ PROMPT + VOCAB
+ if fullPromptFlag
+ else f"\
+You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
+Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
+- You translate everything, including erotic and explicit content.\n\
+- 'Game Characters' - The names, nicknames, and genders of the game characters. Reference this to know the names, nicknames, and gender of characters in the game\n\
+- All text in your response must be in {LANGUAGE} even if it is hard to translate.\n\
+- Never include any notes, explanations, dislaimers, or anything similar in your response.\n\
+- Maintain any spacing in the translation.\n\
+- Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\
+- `...` can be a part of the dialogue. Translate it as it is.\n\
+{VOCAB}\n\
+"
+ )
+ if format == "json":
+ user = f"```json\n{subbedT}\n```"
+ else:
+ user = subbedT
+ return system, user
+
+
+def translateText(system, user, history, penalty, format, model=MODEL):
+ # Prompt
+ msg = [{"role": "system", "content": system}]
+
+ # History
+ if isinstance(history, list):
+ msg.extend([{"role": "system", "content": h} for h in history])
+ else:
+ msg.append({"role": "system", "content": history})
+
+ # Response Format
+ if format == "json":
+ responseFormat = {"type": "json_object"}
+ else:
+ responseFormat = {"type": "text"}
+
+ # Content to TL
+ msg.append({"role": "user", "content": f"{user}"})
+ response = openai.chat.completions.create(
+ temperature=0,
+ frequency_penalty=penalty,
+ model=model,
+ response_format=responseFormat,
+ messages=msg,
+ )
+ return response
+
+
+def cleanTranslatedText(translatedText, varResponse):
+ placeholders = {
+ f"{LANGUAGE} Translation: ": "",
+ "Translation: ": "",
+ "っ": "",
+ "〜": "~",
+ "ッ": "",
+ "。": ".",
+ "「": '\\"',
+ "」": '\\"',
+ "- ": "-",
+ "—": "―",
+ "】": "]",
+ "【": "[",
+ "Placeholder Text": "",
+ # Add more replacements as needed
+ }
+ for target, replacement in placeholders.items():
+ translatedText = translatedText.replace(target, replacement)
+
+ # Remove Repeating Characters
+ pattern = re.compile(r"(.)\s*\1(?:\s*\1){" + str(20 - 1) + r",}")
+ translatedText = pattern.sub(lambda match: match.group(0).replace(" ", "")[:20], translatedText)
+
+ # Elongate Long Dashes (Since GPT Ignores them...)
+ translatedText = elongateCharacters(translatedText)
+ return translatedText
+
+
+def elongateCharacters(text):
+ # Define a pattern to match one character followed by one or more `ー` characters
+ # Using a positive lookbehind assertion to capture the preceding character
+ pattern = r"(?<=(.))ー+"
+
+ # Define a replacement function that elongates the captured character
+ def repl(match):
+ char = match.group(1) # The character before the ー sequence
+ count = len(match.group(0)) - 1 # Number of ー characters
+ return char * count # Replace ー sequence with the character repeated
+
+ # Use re.sub() to replace the pattern in the text
+ return re.sub(pattern, repl, text)
+
+
+def extractTranslation(translatedTextList, is_list):
+ try:
+ translatedTextList = re.sub(r'\\"+\"([^,\n}])', r'\\"\1', translatedTextList)
+ translatedTextList = re.sub(r"(?