diff --git a/modules/alice.py b/modules/alice.py
index 5d38bb2..e02e99e 100644
--- a/modules/alice.py
+++ b/modules/alice.py
@@ -15,52 +15,53 @@ from tqdm import tqdm
# Open AI
load_dotenv()
-if os.getenv('api').replace(' ', '') != '':
- openai.base_url = os.getenv('api')
-openai.organization = os.getenv('org')
-openai.api_key = os.getenv('key')
+if os.getenv("api").replace(" ", "") != "":
+ openai.base_url = os.getenv("api")
+openai.organization = os.getenv("org")
+openai.api_key = os.getenv("key")
-#Globals
-MODEL = os.getenv('model')
-TIMEOUT = int(os.getenv('timeout'))
-LANGUAGE = os.getenv('language').capitalize()
-PROMPT = Path('prompt.txt').read_text(encoding='utf-8')
-VOCAB = Path('vocab.txt').read_text(encoding='utf-8')
-THREADS = int(os.getenv('threads'))
+# Globals
+MODEL = os.getenv("model")
+TIMEOUT = int(os.getenv("timeout"))
+LANGUAGE = os.getenv("language").capitalize()
+PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
+VOCAB = Path("vocab.txt").read_text(encoding="utf-8")
+THREADS = int(os.getenv("threads"))
LOCK = threading.Lock()
-WIDTH = int(os.getenv('width'))
-LISTWIDTH = int(os.getenv('listWidth'))
+WIDTH = int(os.getenv("width"))
+LISTWIDTH = int(os.getenv("listWidth"))
NOTEWIDTH = 70
MAXHISTORY = 10
-ESTIMATE = ''
+ESTIMATE = ""
TOKENS = [0, 0]
NAMESLIST = []
-NAMES = False # Output a list of all the character names found
-BRFLAG = False # If the game uses
instead
+NAMES = False # Output a list of all the character names found
+BRFLAG = False # If the game uses
instead
FIXTEXTWRAP = True # Overwrites textwrap
-IGNORETLTEXT = False # Ignores all translated text.
-MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
+IGNORETLTEXT = False # Ignores all translated text.
+MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
-#tqdm Globals
-BAR_FORMAT='{l_bar}{bar:10}{r_bar}{bar:-10b}'
+# tqdm Globals
+BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
POSITION = 0
LEAVE = False
# Pricing - Depends on the model https://openai.com/pricing
# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request
# If you are getting a MISMATCH LENGTH error, lower the batch size.
-if 'gpt-3.5' in MODEL:
- INPUTAPICOST = .002
- OUTPUTAPICOST = .002
+if "gpt-3.5" in MODEL:
+ INPUTAPICOST = 0.002
+ OUTPUTAPICOST = 0.002
BATCHSIZE = 10
-elif 'gpt-4' in MODEL:
- INPUTAPICOST = .01
- OUTPUTAPICOST = .03
+elif "gpt-4" in MODEL:
+ INPUTAPICOST = 0.01
+ OUTPUTAPICOST = 0.03
BATCHSIZE = 1
+
def handleAlice(filename, estimate):
global ESTIMATE
- totalTokens = [0,0]
+ totalTokens = [0, 0]
ESTIMATE = estimate
if estimate:
@@ -75,17 +76,19 @@ def handleAlice(filename, estimate):
totalTokens[1] += translatedData[1][1]
# Print Total
- totalString = getResultString(['', totalTokens, None], end - start, 'TOTAL')
+ totalString = getResultString(["", totalTokens, None], end - start, "TOTAL")
# Print any errors on maps
if len(MISMATCH) > 0:
- return totalString + Fore.RED + f'\nMismatch Errors: {MISMATCH}' + Fore.RESET
+ return (
+ totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
+ )
else:
return totalString
-
+
else:
try:
- with open('translated/' + filename, 'w', encoding='UTF-8') as outFile:
+ with open("translated/" + filename, "w", encoding="UTF-8") as outFile:
start = time.time()
translatedData = openFiles(filename)
@@ -98,29 +101,43 @@ def handleAlice(filename, estimate):
totalTokens[1] += translatedData[1][1]
except Exception:
traceback.print_exc()
- return 'Fail'
+ return "Fail"
+
+ return getResultString(["", totalTokens, None], end - start, "TOTAL")
- return getResultString(['', totalTokens, None], end - start, 'TOTAL')
def openFiles(filename):
- with open('files/' + filename, 'r', encoding='UTF-8') as f:
+ with open("files/" + filename, "r", encoding="UTF-8") as f:
translatedData = parseText(f, filename)
-
+
return translatedData
+
def getResultString(translatedData, translationTime, filename):
# File Print String
- totalTokenstring =\
- Fore.YELLOW +\
- '[Input: ' + str(translatedData[1][0]) + ']'\
- '[Output: ' + str(translatedData[1][1]) + ']'\
- '[Cost: ${:,.4f}'.format((translatedData[1][0] * .001 * INPUTAPICOST) +\
- (translatedData[1][1] * .001 * OUTPUTAPICOST)) + ']'
- timeString = Fore.BLUE + '[' + str(round(translationTime, 1)) + 's]'
+ totalTokenstring = (
+ Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
+ "[Output: "
+ + str(translatedData[1][1])
+ + "]" "[Cost: ${:,.4f}".format(
+ (translatedData[1][0] * 0.001 * INPUTAPICOST)
+ + (translatedData[1][1] * 0.001 * OUTPUTAPICOST)
+ )
+ + "]"
+ )
+ timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
if translatedData[2] == None:
# Success
- return filename + ': ' + totalTokenstring + timeString + Fore.GREEN + u' \u2713 ' + Fore.RESET
+ return (
+ filename
+ + ": "
+ + totalTokenstring
+ + timeString
+ + Fore.GREEN
+ + " \u2713 "
+ + Fore.RESET
+ )
else:
# Fail
@@ -129,19 +146,30 @@ def getResultString(translatedData, translationTime, filename):
except Exception as e:
traceback.print_exc()
errorString = str(e) + Fore.RED
- return filename + ': ' + totalTokenstring + timeString + Fore.RED + u' \u2717 ' +\
- errorString + Fore.RESET
-
+ return (
+ filename
+ + ": "
+ + totalTokenstring
+ + timeString
+ + Fore.RED
+ + " \u2717 "
+ + errorString
+ + Fore.RESET
+ )
+
+
def parseText(data, filename):
# Get total for progress bar
linesList = data.readlines()
totalTokens = [0, 0]
totalLines = len(linesList)
global LOCK
-
- with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar:
- pbar.desc=filename
- pbar.total=totalLines
+
+ with tqdm(
+ bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE
+ ) as pbar:
+ pbar.desc = filename
+ pbar.total = totalLines
try:
result = translateLines(linesList, pbar)
totalTokens[0] += result[1][0]
@@ -151,6 +179,7 @@ def parseText(data, filename):
return [linesList, totalTokens, e]
return [linesList, totalTokens, None]
+
# Grab scenario data from text file
def translateLines(linesList, pbar):
currentGroup = []
@@ -165,54 +194,67 @@ def translateLines(linesList, pbar):
try:
while i < len(linesList):
# Check if Proper Message
- match = re.findall(r's\[[0-9]+\] = \"(.*)\"', linesList[i])
+ match = re.findall(r"s\[[0-9]+\] = \"(.*)\"", linesList[i])
if len(match) > 0:
jaString = match[0]
# Skip Files
- if '／' in jaString:
+ if "／" in jaString:
i += 1
continue
### Translate
# Remove any textwrap
- jaString = re.sub(r'\\n', ' ', jaString)
+ jaString = re.sub(r"\\n", " ", jaString)
# Grab Speaker
- speakerMatch = re.findall(r's\[[0-9]+\] = \"([^／]+)\"', linesList[i-1])
+ speakerMatch = re.findall(
+ r"s\[[0-9]+\] = \"([^／]+)\"", linesList[i - 1]
+ )
if len(speakerMatch) > 0:
# If there isn't any Japanese in the text just skip
- if re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+', jaString) and '_' not in speakerMatch[0]:
+ if (
+ re.search(r"[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+", jaString)
+ and "_" not in speakerMatch[0]
+ ):
speaker = speakerMatch[0]
else:
- speaker = ''
+ speaker = ""
else:
- speaker = ''
+ speaker = ""
# Grab rest of the messages
currentGroup.append(jaString)
# Check if next line should be merged
if insertBool is True:
- linesList[i] = re.sub(r'(s\[[0-9]+\]) = \"(.+)\"', r'\1 = ""', linesList[i])
- linesList[i] = linesList[i].replace(';', '')
+ linesList[i] = re.sub(
+ r"(s\[[0-9]+\]) = \"(.+)\"", r'\1 = ""', linesList[i]
+ )
+ linesList[i] = linesList[i].replace(";", "")
start = i
- while (len(linesList) > i+1 and re.search(r's\[[0-9]+\] = \"\s+(.*)\"', linesList[i+1]) != None):
+ while (
+ len(linesList) > i + 1
+ and re.search(r"s\[[0-9]+\] = \"\s+(.*)\"", linesList[i + 1])
+ != None
+ ):
multiLine = True
i += 1
- match = re.findall(r's\[[0-9]+\] = \"\s+(.*)\"', linesList[i])
+ match = re.findall(r"s\[[0-9]+\] = \"\s+(.*)\"", linesList[i])
currentGroup.append(match[0])
if insertBool is True:
- linesList[i] = re.sub(r'(s\[[0-9]+\]) = \"\s+(.+)\"', r'\1 = ""', linesList[i])
- linesList[i] = linesList[i].replace(';', '')
+ linesList[i] = re.sub(
+ r"(s\[[0-9]+\]) = \"\s+(.+)\"", r'\1 = ""', linesList[i]
+ )
+ linesList[i] = linesList[i].replace(";", "")
i += 1
# Combine Groups and Add Speaker
- finalJAString = ' '.join(currentGroup)
- if speaker != '':
- finalJAString = f'{speaker}: {finalJAString}'
+ finalJAString = " ".join(currentGroup)
+ if speaker != "":
+ finalJAString = f"{speaker}: {finalJAString}"
else:
- finalJAString = f'{finalJAString}'
+ finalJAString = f"{finalJAString}"
# [Passthrough 1] Pulling From File
if insertBool is False:
@@ -235,7 +277,7 @@ def translateLines(linesList, pbar):
# Mismatch
else:
- pbar.write(f'Mismatch: {batchStartIndex} - {i}')
+ pbar.write(f"Mismatch: {batchStartIndex} - {i}")
MISMATCH.append(batch)
batchStartIndex = i
batch.clear()
@@ -249,33 +291,41 @@ def translateLines(linesList, pbar):
translatedText = translatedBatch[0]
# Remove added speaker and quotes
- translatedText = re.sub(r'^.+?:\s', '', translatedText)
+ translatedText = re.sub(r"^.+?:\s", "", translatedText)
# Textwrap
- translatedText = translatedText.replace('\"', '\\"')
+ translatedText = translatedText.replace('"', '\\"')
translatedText = textwrap.fill(translatedText, width=WIDTH)
# Set Data
if multiLine:
textList = translatedText.split("\n")
for t in textList:
- translatedText = translatedText.replace(';', '')
- translatedText = re.sub(r'(s\[[0-9]+\]) = \"(.*)\"', rf'\1 = "{t}"', linesList[start])
- translatedText = translatedText.replace(';', '')
+ translatedText = translatedText.replace(";", "")
+ translatedText = re.sub(
+ r"(s\[[0-9]+\]) = \"(.*)\"",
+ rf'\1 = "{t}"',
+ linesList[start],
+ )
+ translatedText = translatedText.replace(";", "")
linesList[start] = translatedText
pbar.update(1)
start += 1
multiLine = False
- translatedText = translatedText.replace(';', '')
- translatedBatch.pop(0)
+ translatedText = translatedText.replace(";", "")
+ translatedBatch.pop(0)
else:
# Remove any textwrap
- translatedText = translatedText.replace('\n', ' ')
- translatedText = re.sub(r'(s\[[0-9]+\]) = \"(.*)\"', rf'\1 = "{translatedText}"', linesList[start])
- translatedText = translatedText.replace(';', '')
+ translatedText = translatedText.replace("\n", " ")
+ translatedText = re.sub(
+ r"(s\[[0-9]+\]) = \"(.*)\"",
+ rf'\1 = "{translatedText}"',
+ linesList[start],
+ )
+ translatedText = translatedText.replace(";", "")
linesList[start] = translatedText
pbar.update(1)
- translatedBatch.pop(0)
+ translatedBatch.pop(0)
# If Batch is empty. Move on.
if len(translatedBatch) == 0:
@@ -283,7 +333,7 @@ def translateLines(linesList, pbar):
batchStartIndex = i
pbar.update(1)
batch.clear()
-
+
currentGroup = []
else:
if insertBool is True:
@@ -295,70 +345,72 @@ def translateLines(linesList, pbar):
traceback.print_exc()
return [linesList, tokens]
+
def subVars(jaString):
- jaString = jaString.replace('\u3000', ' ')
+ jaString = jaString.replace("\u3000", " ")
# Nested
count = 0
- nestedList = re.findall(r'[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]', jaString)
+ nestedList = re.findall(r"[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]", jaString)
nestedList = set(nestedList)
if len(nestedList) != 0:
for icon in nestedList:
- jaString = jaString.replace(icon, '{Nested_' + str(count) + '}')
+ jaString = jaString.replace(icon, "{Nested_" + str(count) + "}")
count += 1
# Icons
count = 0
- iconList = re.findall(r'[\\]+[iIkKwWaA]+\[[0-9]+\]', jaString)
+ iconList = re.findall(r"[\\]+[iIkKwWaA]+\[[0-9]+\]", jaString)
iconList = set(iconList)
if len(iconList) != 0:
for icon in iconList:
- jaString = jaString.replace(icon, '{Ascii_' + str(count) + '}')
+ jaString = jaString.replace(icon, "{Ascii_" + str(count) + "}")
count += 1
# Colors
count = 0
- colorList = re.findall(r'[\\]+[cC]\[[0-9]+\]', jaString)
+ colorList = re.findall(r"[\\]+[cC]\[[0-9]+\]", jaString)
colorList = set(colorList)
if len(colorList) != 0:
for color in colorList:
- jaString = jaString.replace(color, '{Color_' + str(count) + '}')
+ jaString = jaString.replace(color, "{Color_" + str(count) + "}")
count += 1
# Names
count = 0
- nameList = re.findall(r'[\\]+[nN]\[.+?\]+', jaString)
+ nameList = re.findall(r"[\\]+[nN]\[.+?\]+", jaString)
nameList = set(nameList)
if len(nameList) != 0:
for name in nameList:
- jaString = jaString.replace(name, '{Noun_' + str(count) + '}')
+ jaString = jaString.replace(name, "{Noun_" + str(count) + "}")
count += 1
# Variables
count = 0
- varList = re.findall(r'[\\]+[vV]\[[0-9]+\]', jaString)
+ varList = re.findall(r"[\\]+[vV]\[[0-9]+\]", jaString)
varList = set(varList)
if len(varList) != 0:
for var in varList:
- jaString = jaString.replace(var, '{Var_' + str(count) + '}')
+ jaString = jaString.replace(var, "{Var_" + str(count) + "}")
count += 1
# Formatting
count = 0
- formatList = re.findall(r'[\\]+[\w]+\[.+?\]', jaString)
+ formatList = re.findall(r"[\\]+[\w]+\[.+?\]", jaString)
formatList = set(formatList)
if len(formatList) != 0:
for var in formatList:
- jaString = jaString.replace(var, '{FCode_' + str(count) + '}')
+ jaString = jaString.replace(var, "{FCode_" + str(count) + "}")
count += 1
# Put all lists in list and return
allList = [nestedList, iconList, colorList, nameList, varList, formatList]
return [jaString, allList]
+
def resubVars(translatedText, allList):
# Fix Spacing and ChatGPT Nonsense
- matchList = re.findall(r'\[\s?.+?\s?\]', translatedText)
+ matchList = re.findall(r"\[\s?.+?\s?\]", translatedText)
if len(matchList) > 0:
for match in matchList:
text = match.strip()
@@ -368,54 +420,58 @@ def resubVars(translatedText, allList):
count = 0
if len(allList[0]) != 0:
for var in allList[0]:
- translatedText = translatedText.replace('{Nested_' + str(count) + '}', var)
+ translatedText = translatedText.replace("{Nested_" + str(count) + "}", var)
count += 1
# Icons
count = 0
if len(allList[1]) != 0:
for var in allList[1]:
- translatedText = translatedText.replace('{Ascii_' + str(count) + '}', var)
+ translatedText = translatedText.replace("{Ascii_" + str(count) + "}", var)
count += 1
# Colors
count = 0
if len(allList[2]) != 0:
for var in allList[2]:
- translatedText = translatedText.replace('{Color_' + str(count) + '}', var)
+ translatedText = translatedText.replace("{Color_" + str(count) + "}", var)
count += 1
# Names
count = 0
if len(allList[3]) != 0:
for var in allList[3]:
- translatedText = translatedText.replace('{Noun_' + str(count) + '}', var)
+ translatedText = translatedText.replace("{Noun_" + str(count) + "}", var)
count += 1
# Vars
count = 0
if len(allList[4]) != 0:
for var in allList[4]:
- translatedText = translatedText.replace('{Var_' + str(count) + '}', var)
+ translatedText = translatedText.replace("{Var_" + str(count) + "}", var)
count += 1
-
+
# Formatting
count = 0
if len(allList[5]) != 0:
for var in allList[5]:
- translatedText = translatedText.replace('{FCode_' + str(count) + '}', var)
+ translatedText = translatedText.replace("{FCode_" + str(count) + "}", var)
count += 1
return translatedText
+
def batchList(input_list, batch_size):
if not isinstance(batch_size, int) or batch_size <= 0:
raise ValueError("batch_size must be a positive integer")
-
- return [input_list[i:i + batch_size] for i in range(0, len(input_list), batch_size)]
+
+ return [
+ input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)
+ ]
+
def createContext(fullPromptFlag, subbedT):
- characters = 'Game Characters:\n\
+ characters = "Game Characters:\n\
林つかさ (Tsukasa Hayashi) - Female\n\
山田美兎 (Miyato Yamada) - Female\n\
鈴木赤音 (Akane Suzuki) - Female\n\
@@ -428,13 +484,17 @@ def createContext(fullPromptFlag, subbedT):
モリー・ボイド (Molly Boyd) - Female\n\
オルガ・ブヤチッチ (Olga Buyachich) - Female\n\
アッチャラー ギッティ (Atchara Gitti) - Female\n\
-'
-
- system = PROMPT if fullPromptFlag else \
- f'Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`'
- user = f'{subbedT}'
+"
+
+ system = (
+ PROMPT
+ if fullPromptFlag
+ else f"Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`"
+ )
+ user = f"{subbedT}"
return characters, system, user
+
def translateText(characters, system, user, history):
# Prompt
msg = [{"role": "system", "content": system + characters}]
@@ -447,9 +507,9 @@ def translateText(characters, system, user, history):
msg.extend([{"role": "assistant", "content": h} for h in history])
else:
msg.append({"role": "assistant", "content": history})
-
+
# Content to TL
- msg.append({"role": "user", "content": f'{user}'})
+ msg.append({"role": "user", "content": f"{user}"})
response = openai.chat.completions.create(
temperature=0.1,
frequency_penalty=0.1,
@@ -458,40 +518,47 @@ def translateText(characters, system, user, history):
)
return response
+
def cleanTranslatedText(translatedText, varResponse):
placeholders = {
- f'{LANGUAGE} Translation: ': '',
- 'Translation: ': '',
- 'っ': '',
- '〜': '~',
- 'ッ': '',
- '。': '.',
- 'Placeholder Text': ''
+ f"{LANGUAGE} Translation: ": "",
+ "Translation: ": "",
+ "っ": "",
+ "〜": "~",
+ "ッ": "",
+ "。": ".",
+ "Placeholder Text": "",
# Add more replacements as needed
}
for target, replacement in placeholders.items():
translatedText = translatedText.replace(target, replacement)
translatedText = resubVars(translatedText, varResponse[1])
- if '\n' in translatedText:
- return [line for line in translatedText.split('\n') if line]
+ if "\n" in translatedText:
+ return [line for line in translatedText.split("\n") if line]
else:
- return [line for line in translatedText.split('\\n') if line]
+ return [line for line in translatedText.split("\\n") if line]
+
def extractTranslation(translatedTextList, is_list):
- pattern = r'[\\]*`?(.*?)[\\]*?`?'
+ pattern = r"[\\]*`?(.*?)[\\]*?`?"
# If it's a batch (i.e., list), extract with tags; otherwise, return the single item.
if is_list:
- return [re.findall(pattern, line)[0][1] for line in translatedTextList if re.search(pattern, line)]
+ return [
+ re.findall(pattern, line)[0][1]
+ for line in translatedTextList
+ if re.search(pattern, line)
+ ]
else:
matchList = re.findall(pattern, translatedTextList)
return matchList[0][1] if matchList else translatedTextList
+
def countTokens(characters, system, user, history):
inputTotalTokens = 0
outputTotalTokens = 0
- enc = tiktoken.encoding_for_model('gpt-4')
-
+ enc = tiktoken.encoding_for_model("gpt-4")
+
# Input
if isinstance(history, list):
for line in history:
@@ -503,15 +570,17 @@ def countTokens(characters, system, user, history):
inputTotalTokens += len(enc.encode(user))
# Output
- outputTotalTokens += round(len(enc.encode(user))*3)
+ outputTotalTokens += round(len(enc.encode(user)) * 3)
return [inputTotalTokens, outputTotalTokens]
+
def combineList(tlist, text):
if isinstance(text, list):
return [t for sublist in tlist for t in sublist]
return tlist[0]
+
@retry(exceptions=Exception, tries=5, delay=5)
def translateGPT(text, history, fullPromptFlag):
totalTokens = [0, 0]
@@ -523,8 +592,10 @@ def translateGPT(text, history, fullPromptFlag):
for index, tItem in enumerate(tList):
# Before sending to translation, if we have a list of items, add the formatting
if isinstance(tItem, list):
- payload = '\n'.join([f'`{item}`' for i, item in enumerate(tItem)])
- payload = payload.replace('``', '`Placeholder Text`')
+ payload = "\n".join(
+ [f"`{item}`" for i, item in enumerate(tItem)]
+ )
+ payload = payload.replace("``", "`Placeholder Text`")
varResponse = subVars(payload)
subbedT = varResponse[0]
else:
@@ -532,7 +603,7 @@ def translateGPT(text, history, fullPromptFlag):
subbedT = varResponse[0]
# Things to Check before starting translation
- if not re.search(r'[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+', subbedT):
+ if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", subbedT):
continue
# Create Message
@@ -557,11 +628,13 @@ def translateGPT(text, history, fullPromptFlag):
extractedTranslations = extractTranslation(translatedTextList, True)
tList[index] = extractedTranslations
if len(tItem) != len(translatedTextList):
- mismatch = True # Just here so breakpoint can be set
+ mismatch = True # Just here so breakpoint can be set
history = extractedTranslations[-10:] # Update history if we have a list
else:
# Ensure we're passing a single string to extractTranslation
- extractedTranslations = extractTranslation('\n'.join(translatedTextList), False)
+ extractedTranslations = extractTranslation(
+ "\n".join(translatedTextList), False
+ )
tList[index] = extractedTranslations
finalList = combineList(tList, text)
diff --git a/modules/anim.py b/modules/anim.py
index bab2169..fa1888e 100644
--- a/modules/anim.py
+++ b/modules/anim.py
@@ -16,52 +16,53 @@ from tqdm import tqdm
# Open AI
load_dotenv()
-if os.getenv('api').replace(' ', '') != '':
- openai.base_url = os.getenv('api')
-openai.organization = os.getenv('org')
-openai.api_key = os.getenv('key')
+if os.getenv("api").replace(" ", "") != "":
+ openai.base_url = os.getenv("api")
+openai.organization = os.getenv("org")
+openai.api_key = os.getenv("key")
-#Globals
-MODEL = os.getenv('model')
-TIMEOUT = int(os.getenv('timeout'))
-LANGUAGE = os.getenv('language').capitalize()
-PROMPT = Path('prompt.txt').read_text(encoding='utf-8')
-VOCAB = Path('vocab.txt').read_text(encoding='utf-8')
-THREADS = int(os.getenv('threads'))
+# Globals
+MODEL = os.getenv("model")
+TIMEOUT = int(os.getenv("timeout"))
+LANGUAGE = os.getenv("language").capitalize()
+PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
+VOCAB = Path("vocab.txt").read_text(encoding="utf-8")
+THREADS = int(os.getenv("threads"))
LOCK = threading.Lock()
-WIDTH = int(os.getenv('width'))
-LISTWIDTH = int(os.getenv('listWidth'))
+WIDTH = int(os.getenv("width"))
+LISTWIDTH = int(os.getenv("listWidth"))
NOTEWIDTH = 70
MAXHISTORY = 10
-ESTIMATE = ''
+ESTIMATE = ""
TOKENS = [0, 0]
NAMESLIST = []
-NAMES = False # Output a list of all the character names found
-BRFLAG = False # If the game uses
instead
+NAMES = False # Output a list of all the character names found
+BRFLAG = False # If the game uses
instead
FIXTEXTWRAP = True # Overwrites textwrap
-IGNORETLTEXT = False # Ignores all translated text.
-MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
+IGNORETLTEXT = False # Ignores all translated text.
+MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
-#tqdm Globals
-BAR_FORMAT='{l_bar}{bar:10}{r_bar}{bar:-10b}'
+# tqdm Globals
+BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
POSITION = 0
LEAVE = False
# Pricing - Depends on the model https://openai.com/pricing
# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request
# If you are getting a MISMATCH LENGTH error, lower the batch size.
-if 'gpt-3.5' in MODEL:
- INPUTAPICOST = .002
- OUTPUTAPICOST = .002
+if "gpt-3.5" in MODEL:
+ INPUTAPICOST = 0.002
+ OUTPUTAPICOST = 0.002
BATCHSIZE = 10
-elif 'gpt-4' in MODEL:
- INPUTAPICOST = .01
- OUTPUTAPICOST = .03
- BATCHSIZE = 50
+elif "gpt-4" in MODEL:
+ INPUTAPICOST = 0.01
+ OUTPUTAPICOST = 0.03
+ BATCHSIZE = 50
+
def handleAnim(filename, estimate):
global ESTIMATE
- totalTokens = [0,0]
+ totalTokens = [0, 0]
ESTIMATE = estimate
if estimate:
@@ -76,17 +77,19 @@ def handleAnim(filename, estimate):
totalTokens[1] += translatedData[1][1]
# Print Total
- totalString = getResultString(['', totalTokens, None], end - start, 'TOTAL')
+ totalString = getResultString(["", totalTokens, None], end - start, "TOTAL")
# Print any errors on maps
if len(MISMATCH) > 0:
- return totalString + Fore.RED + f'\nMismatch Errors: {MISMATCH}' + Fore.RESET
+ return (
+ totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
+ )
else:
return totalString
-
+
else:
try:
- with open('translated/' + filename, 'w', encoding='UTF-8') as outFile:
+ with open("translated/" + filename, "w", encoding="UTF-8") as outFile:
start = time.time()
translatedData = openFiles(filename)
@@ -98,36 +101,50 @@ def handleAnim(filename, estimate):
totalTokens[0] += translatedData[1][0]
totalTokens[1] += translatedData[1][1]
except Exception:
- return 'Fail'
+ return "Fail"
+
+ return getResultString(["", totalTokens, None], end - start, "TOTAL")
- return getResultString(['', totalTokens, None], end - start, 'TOTAL')
def openFiles(filename):
- with open('files/' + filename, 'r', encoding='UTF-8-sig') as f:
+ with open("files/" + filename, "r", encoding="UTF-8-sig") as f:
data = json.load(f)
# Map Files
- if '.json' in filename:
+ if ".json" in filename:
translatedData = parseJSON(data, filename)
else:
- raise NameError(filename + ' Not Supported')
-
+ raise NameError(filename + " Not Supported")
+
return translatedData
+
def getResultString(translatedData, translationTime, filename):
# File Print String
- totalTokenstring =\
- Fore.YELLOW +\
- '[Input: ' + str(translatedData[1][0]) + ']'\
- '[Output: ' + str(translatedData[1][1]) + ']'\
- '[Cost: ${:,.4f}'.format((translatedData[1][0] * .001 * INPUTAPICOST) +\
- (translatedData[1][1] * .001 * OUTPUTAPICOST)) + ']'
- timeString = Fore.BLUE + '[' + str(round(translationTime, 1)) + 's]'
+ totalTokenstring = (
+ Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
+ "[Output: "
+ + str(translatedData[1][1])
+ + "]" "[Cost: ${:,.4f}".format(
+ (translatedData[1][0] * 0.001 * INPUTAPICOST)
+ + (translatedData[1][1] * 0.001 * OUTPUTAPICOST)
+ )
+ + "]"
+ )
+ timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
if translatedData[2] == None:
# Success
- return filename + ': ' + totalTokenstring + timeString + Fore.GREEN + u' \u2713 ' + Fore.RESET
+ return (
+ filename
+ + ": "
+ + totalTokenstring
+ + timeString
+ + Fore.GREEN
+ + " \u2713 "
+ + Fore.RESET
+ )
else:
# Fail
@@ -136,20 +153,31 @@ def getResultString(translatedData, translationTime, filename):
except Exception as e:
traceback.print_exc()
errorString = str(e) + Fore.RED
- return filename + ': ' + totalTokenstring + timeString + Fore.RED + u' \u2717 ' +\
- errorString + Fore.RESET
-
+ return (
+ filename
+ + ": "
+ + totalTokenstring
+ + timeString
+ + Fore.RED
+ + " \u2717 "
+ + errorString
+ + Fore.RESET
+ )
+
+
def parseJSON(data, filename):
keys = list(data.keys())
- batches = [keys[i:i + BATCHSIZE] for i in range(0, len(keys), BATCHSIZE)]
+ batches = [keys[i : i + BATCHSIZE] for i in range(0, len(keys), BATCHSIZE)]
totalTokens = [0, 0]
totalLines = 0
totalLines = len(batches)
global LOCK
-
- with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar:
- pbar.desc=filename
- pbar.total=totalLines
+
+ with tqdm(
+ bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE
+ ) as pbar:
+ pbar.desc = filename
+ pbar.total = totalLines
try:
result = translateJSON(batches, data, pbar)
totalTokens[0] += result[0]
@@ -159,6 +187,7 @@ def parseJSON(data, filename):
return [data, totalTokens, e]
return [data, totalTokens, None]
+
def translateJSON(keys, data, pbar):
translatedBatch = []
textHistory = []
@@ -172,20 +201,20 @@ def translateJSON(keys, data, pbar):
needTL = False
for i in range(len(batch)):
t = data[batch[i]]
- if re.search(r'[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+', t) or t == '':
+ if re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", t) or t == "":
needTL = True
if needTL is False and IGNORETLTEXT is True:
pbar.update(1)
- continue
+ continue
# Remove any textwrap and Furigana
for i in range(len(batch)):
if FIXTEXTWRAP == True:
# Textwrap
- data[originalBatch[i]] = data[originalBatch[i]].replace('@b', ' ')
+ data[originalBatch[i]] = data[originalBatch[i]].replace("@b", " ")
# Furigana
- rcodeMatch = re.findall(r'(@\[(.+?):.+?\])', batch[i])
+ rcodeMatch = re.findall(r"(@\[(.+?):.+?\])", batch[i])
if len(rcodeMatch) > 0:
for match in rcodeMatch:
batch[i] = batch[i].replace(match[0], match[1])
@@ -203,23 +232,22 @@ def translateJSON(keys, data, pbar):
# Format and Set Text
if len(batch) == len(translatedBatch):
for i in range(len(translatedBatch)):
-
# Remove added speaker
translatedText = translatedBatch[i]
- translatedText = re.sub(r'^.+?\s\|\s?', '', translatedText)
+ translatedText = re.sub(r"^.+?\s\|\s?", "", translatedText)
# Textwrap
- if '@n' in translatedText:
- match = re.search(r'.*@n(.*)', translatedText)
+ if "@n" in translatedText:
+ match = re.search(r".*@n(.*)", translatedText)
if match != None:
tlText = match.group(1)
tlText = textwrap.fill(tlText, width=WIDTH)
- tlText = tlText.replace('\n', '@b')
+ tlText = tlText.replace("\n", "@b")
translatedText = translatedText.replace(match.group(1), tlText)
-
- elif '@b' not in translatedText:
+
+ elif "@b" not in translatedText:
translatedText = textwrap.fill(translatedText, width=WIDTH)
- translatedText = translatedText.replace('\n', '@b')
+ translatedText = translatedText.replace("\n", "@b")
# Set Data
data[originalBatch[i]] = translatedText
@@ -232,72 +260,74 @@ def translateJSON(keys, data, pbar):
continue
pbar.update(1)
- return tokens
+ return tokens
+
def subVars(jaString):
- jaString = jaString.replace('\u3000', ' ')
+ jaString = jaString.replace("\u3000", " ")
# Nested
count = 0
- nestedList = re.findall(r'[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]', jaString)
+ nestedList = re.findall(r"[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]", jaString)
nestedList = set(nestedList)
if len(nestedList) != 0:
for icon in nestedList:
- jaString = jaString.replace(icon, '[Nested_' + str(count) + ']')
+ jaString = jaString.replace(icon, "[Nested_" + str(count) + "]")
count += 1
# Icons
count = 0
- iconList = re.findall(r'[\\]+[iIkKwWaA]+\[[0-9]+\]', jaString)
+ iconList = re.findall(r"[\\]+[iIkKwWaA]+\[[0-9]+\]", jaString)
iconList = set(iconList)
if len(iconList) != 0:
for icon in iconList:
- jaString = jaString.replace(icon, '[Ascii_' + str(count) + ']')
+ jaString = jaString.replace(icon, "[Ascii_" + str(count) + "]")
count += 1
# Colors
count = 0
- colorList = re.findall(r'[\\]+[cC]\[[0-9]+\]', jaString)
+ colorList = re.findall(r"[\\]+[cC]\[[0-9]+\]", jaString)
colorList = set(colorList)
if len(colorList) != 0:
for color in colorList:
- jaString = jaString.replace(color, '[Color_' + str(count) + ']')
+ jaString = jaString.replace(color, "[Color_" + str(count) + "]")
count += 1
# Names
count = 0
- nameList = re.findall(r'[\\]+[nN]\[.+?\]+', jaString)
+ nameList = re.findall(r"[\\]+[nN]\[.+?\]+", jaString)
nameList = set(nameList)
if len(nameList) != 0:
for name in nameList:
- jaString = jaString.replace(name, '[Noun_' + str(count) + ']')
+ jaString = jaString.replace(name, "[Noun_" + str(count) + "]")
count += 1
# Variables
count = 0
- varList = re.findall(r'[\\]+[vV]\[[0-9]+\]', jaString)
+ varList = re.findall(r"[\\]+[vV]\[[0-9]+\]", jaString)
varList = set(varList)
if len(varList) != 0:
for var in varList:
- jaString = jaString.replace(var, '[Var_' + str(count) + ']')
+ jaString = jaString.replace(var, "[Var_" + str(count) + "]")
count += 1
# Formatting
count = 0
- formatList = re.findall(r'[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]', jaString)
+ formatList = re.findall(r"[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]", jaString)
formatList = set(formatList)
if len(formatList) != 0:
for var in formatList:
- jaString = jaString.replace(var, '[FCode_' + str(count) + ']')
+ jaString = jaString.replace(var, "[FCode_" + str(count) + "]")
count += 1
# Put all lists in list and return
allList = [nestedList, iconList, colorList, nameList, varList, formatList]
return [jaString, allList]
+
def resubVars(translatedText, allList):
# Fix Spacing and ChatGPT Nonsense
- matchList = re.findall(r'\[\s?.+?\s?\]', translatedText)
+ matchList = re.findall(r"\[\s?.+?\s?\]", translatedText)
if len(matchList) > 0:
for match in matchList:
text = match.strip()
@@ -307,64 +337,70 @@ def resubVars(translatedText, allList):
count = 0
if len(allList[0]) != 0:
for var in allList[0]:
- translatedText = translatedText.replace('[Nested_' + str(count) + ']', var)
+ translatedText = translatedText.replace("[Nested_" + str(count) + "]", var)
count += 1
# Icons
count = 0
if len(allList[1]) != 0:
for var in allList[1]:
- translatedText = translatedText.replace('[Ascii_' + str(count) + ']', var)
+ translatedText = translatedText.replace("[Ascii_" + str(count) + "]", var)
count += 1
# Colors
count = 0
if len(allList[2]) != 0:
for var in allList[2]:
- translatedText = translatedText.replace('[Color_' + str(count) + ']', var)
+ translatedText = translatedText.replace("[Color_" + str(count) + "]", var)
count += 1
# Names
count = 0
if len(allList[3]) != 0:
for var in allList[3]:
- translatedText = translatedText.replace('[Noun_' + str(count) + ']', var)
+ translatedText = translatedText.replace("[Noun_" + str(count) + "]", var)
count += 1
# Vars
count = 0
if len(allList[4]) != 0:
for var in allList[4]:
- translatedText = translatedText.replace('[Var_' + str(count) + ']', var)
+ translatedText = translatedText.replace("[Var_" + str(count) + "]", var)
count += 1
-
+
# Formatting
count = 0
if len(allList[5]) != 0:
for var in allList[5]:
- translatedText = translatedText.replace('[FCode_' + str(count) + ']', var)
+ translatedText = translatedText.replace("[FCode_" + str(count) + "]", var)
count += 1
return translatedText
+
def batchList(input_list, batch_size):
if not isinstance(batch_size, int) or batch_size <= 0:
raise ValueError("batch_size must be a positive integer")
-
- return [input_list[i:i + batch_size] for i in range(0, len(input_list), batch_size)]
+
+ return [
+ input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)
+ ]
+
def createContext(fullPromptFlag, subbedT):
- characters = 'Game Characters:\n\
+ characters = "Game Characters:\n\
達也 (Tatsuya) - Male\n\
香織 (Kaori) - Female\n\
岩瀬 (Iwase)\n\
万蔵 (Manzou) - Male\n\
結奈 (Yuuna) - Female\n\
茅部 (Kayabe)\n\
-'
-
- system = PROMPT + VOCAB if fullPromptFlag else \
- f"\
+"
+
+ system = (
+ PROMPT + VOCAB
+ if fullPromptFlag
+ else f"\
You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
- You translate everything, including erotic and explicit content.\n\
@@ -376,9 +412,11 @@ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{
- `...` can be a part of the dialogue. Translate it as it is.\n\
{VOCAB}\n\
"
- user = f'{subbedT}'
+ )
+ user = f"{subbedT}"
return characters, system, user
+
def translateText(characters, system, user, history, penalty):
# Prompt
msg = [{"role": "system", "content": system + characters}]
@@ -391,9 +429,9 @@ def translateText(characters, system, user, history, penalty):
msg.extend([{"role": "system", "content": h} for h in history])
else:
msg.append({"role": "system", "content": history})
-
+
# Content to TL
- msg.append({"role": "user", "content": f'{user}'})
+ msg.append({"role": "user", "content": f"{user}"})
response = openai.chat.completions.create(
temperature=0,
frequency_penalty=penalty,
@@ -402,18 +440,19 @@ def translateText(characters, system, user, history, penalty):
)
return response
+
def cleanTranslatedText(translatedText, varResponse):
placeholders = {
- f'{LANGUAGE} Translation: ': '',
- 'Translation: ': '',
- 'っ': '',
- '〜': '~',
- 'ッ': '',
- '。': '.',
- 'Placeholder Text': '',
- 'é' : 'e',
- '—' : '-',
- 'ū' : 'u',
+ f"{LANGUAGE} Translation: ": "",
+ "Translation: ": "",
+ "っ": "",
+ "〜": "~",
+ "ッ": "",
+ "。": ".",
+ "Placeholder Text": "",
+ "é": "e",
+ "—": "-",
+ "ū": "u",
# Add more replacements as needed
}
for target, replacement in placeholders.items():
@@ -424,11 +463,12 @@ def cleanTranslatedText(translatedText, varResponse):
translatedText = resubVars(translatedText, varResponse[1])
return translatedText
+
def elongateCharacters(text):
# Define a pattern to match one character followed by one or more `ー` characters
# Using a positive lookbehind assertion to capture the preceding character
- pattern = r'(?<=(.))ー+'
-
+ pattern = r"(?<=(.))ー+"
+
# Define a replacement function that elongates the captured character
def repl(match):
char = match.group(1) # The character before the ー sequence
@@ -438,8 +478,9 @@ def elongateCharacters(text):
# Use re.sub() to replace the pattern in the text
return re.sub(pattern, repl, text)
+
def extractTranslation(translatedTextList, is_list):
- pattern = r'`?<[Ll]ine\d+>([\\]*.*?[\\]*?)<\/?[Ll]ine\d+>`?'
+ pattern = r"`?<[Ll]ine\d+>([\\]*.*?[\\]*?)<\/?[Ll]ine\d+>`?"
# If it's a batch (i.e., list), extract with tags; otherwise, return the single item.
if is_list:
matchList = re.findall(pattern, translatedTextList)
@@ -448,11 +489,12 @@ def extractTranslation(translatedTextList, is_list):
matchList = re.findall(pattern, translatedTextList)
return matchList[0][0] if matchList else translatedTextList
+
def countTokens(characters, system, user, history):
inputTotalTokens = 0
outputTotalTokens = 0
- enc = tiktoken.encoding_for_model('gpt-4')
-
+ enc = tiktoken.encoding_for_model("gpt-4")
+
# Input
if isinstance(history, list):
for line in history:
@@ -464,15 +506,17 @@ def countTokens(characters, system, user, history):
inputTotalTokens += len(enc.encode(user))
# Output
- outputTotalTokens += round(len(enc.encode(user))*3)
+ outputTotalTokens += round(len(enc.encode(user)) * 3)
return [inputTotalTokens, outputTotalTokens]
+
def combineList(tlist, text):
if isinstance(text, list):
return [t for sublist in tlist for t in sublist]
return tlist[0]
+
@retry(exceptions=Exception, tries=5, delay=5)
def translateGPT(text, history, fullPromptFlag):
mismatch = False
@@ -485,8 +529,12 @@ def translateGPT(text, history, fullPromptFlag):
for index, tItem in enumerate(tList):
# Before sending to translation, if we have a list of items, add the formatting
if isinstance(tItem, list):
- payload = '\n'.join([f'`{item}`' for i, item in enumerate(tItem)])
- payload = re.sub(r'(<)(\/Line\d+>)', r'\1>Placeholder Text<\3', payload)
+ payload = "\n".join(
+ [f"`{item}`" for i, item in enumerate(tItem)]
+ )
+ payload = re.sub(
+ r"(<)(\/Line\d+>)", r"\1>Placeholder Text<\3", payload
+ )
varResponse = subVars(payload)
subbedT = varResponse[0]
else:
@@ -494,7 +542,7 @@ def translateGPT(text, history, fullPromptFlag):
subbedT = varResponse[0]
# Things to Check before starting translation
- if not re.search(r'[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+', subbedT):
+ if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", subbedT):
continue
# Create Message
@@ -531,11 +579,13 @@ def translateGPT(text, history, fullPromptFlag):
extractedTranslations = extractTranslation(translatedText, True)
tList[index] = extractedTranslations
if len(tItem) != len(extractedTranslations):
- mismatch = True # Just here for breakpoint
+ mismatch = True # Just here for breakpoint
# Create History
if not mismatch:
- history = extractedTranslations[-10:] # Update history if we have a list
+ history = extractedTranslations[
+ -10:
+ ] # Update history if we have a list
else:
history = text[-10:]
else:
diff --git a/modules/atelier.py b/modules/atelier.py
index 6cfde80..61b389d 100644
--- a/modules/atelier.py
+++ b/modules/atelier.py
@@ -14,38 +14,41 @@ from tqdm import tqdm
# Open AI
load_dotenv()
-if os.getenv('api').replace(' ', '') != '':
- openai.base_url = os.getenv('api')
-openai.organization = os.getenv('org')
-openai.api_key = os.getenv('key')
+if os.getenv("api").replace(" ", "") != "":
+ openai.base_url = os.getenv("api")
+openai.organization = os.getenv("org")
+openai.api_key = os.getenv("key")
-#Globals
-MODEL = os.getenv('model')
-TIMEOUT = int(os.getenv('timeout'))
-LANGUAGE=os.getenv('language').capitalize()
-INPUTAPICOST = .002 # Depends on the model https://openai.com/pricing
-OUTPUTAPICOST = .002
-PROMPT = Path('prompt.txt').read_text(encoding='utf-8')
-VOCAB = Path('vocab.txt').read_text(encoding='utf-8')
-THREADS = int(os.getenv('threads')) # Controls how many threads are working on a single file (May have to drop this)
+# Globals
+MODEL = os.getenv("model")
+TIMEOUT = int(os.getenv("timeout"))
+LANGUAGE = os.getenv("language").capitalize()
+INPUTAPICOST = 0.002 # Depends on the model https://openai.com/pricing
+OUTPUTAPICOST = 0.002
+PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
+VOCAB = Path("vocab.txt").read_text(encoding="utf-8")
+THREADS = int(
+ os.getenv("threads")
+) # Controls how many threads are working on a single file (May have to drop this)
LOCK = threading.Lock()
-WIDTH = int(os.getenv('width'))
-LISTWIDTH = int(os.getenv('listWidth'))
+WIDTH = int(os.getenv("width"))
+LISTWIDTH = int(os.getenv("listWidth"))
NOTEWIDTH = 40
MAXHISTORY = 10
-ESTIMATE = ''
+ESTIMATE = ""
totalTokens = [0, 0]
NAMESLIST = []
-#tqdm Globals
-BAR_FORMAT='{l_bar}{bar:10}{r_bar}{bar:-10b}'
-POSITION=0
-LEAVE=False
+# tqdm Globals
+BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
+POSITION = 0
+LEAVE = False
# Translation Flags
FIXTEXTWRAP = True
IGNORETLTEXT = True
+
def handleAtelier(filename, estimate):
global ESTIMATE, totalTokens
ESTIMATE = estimate
@@ -61,11 +64,11 @@ def handleAtelier(filename, estimate):
totalTokens[0] += translatedData[1][0]
totalTokens[1] += translatedData[1][1]
- return getResultString(['', totalTokens, None], end - start, 'TOTAL')
+ return getResultString(["", totalTokens, None], end - start, "TOTAL")
else:
try:
- with open('translated/' + filename, 'w', encoding='utf-8') as outFile:
+ with open("translated/" + filename, "w", encoding="utf-8") as outFile:
start = time.time()
translatedData = openFiles(filename)
outFile.writelines(translatedData[0])
@@ -77,29 +80,43 @@ def handleAtelier(filename, estimate):
totalTokens[0] += translatedData[1][0]
totalTokens[1] += translatedData[1][1]
except Exception:
- return 'Fail'
+ return "Fail"
+
+ return getResultString(["", totalTokens, None], end - start, "TOTAL")
- return getResultString(['', totalTokens, None], end - start, 'TOTAL')
def openFiles(filename):
- with open('files/' + filename, 'r', encoding='UTF-8') as f:
+ with open("files/" + filename, "r", encoding="UTF-8") as f:
translatedData = parseText(f, filename)
-
+
return translatedData
+
def getResultString(translatedData, translationTime, filename):
# File Print String
- totalTokenstring =\
- Fore.YELLOW +\
- '[Input: ' + str(translatedData[1][0]) + ']'\
- '[Output: ' + str(translatedData[1][1]) + ']'\
- '[Cost: ${:,.4f}'.format((translatedData[1][0] * .001 * INPUTAPICOST) +\
- (translatedData[1][1] * .001 * OUTPUTAPICOST)) + ']'
- timeString = Fore.BLUE + '[' + str(round(translationTime, 1)) + 's]'
+ totalTokenstring = (
+ Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
+ "[Output: "
+ + str(translatedData[1][1])
+ + "]" "[Cost: ${:,.4f}".format(
+ (translatedData[1][0] * 0.001 * INPUTAPICOST)
+ + (translatedData[1][1] * 0.001 * OUTPUTAPICOST)
+ )
+ + "]"
+ )
+ timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
if translatedData[2] is None:
# Success
- return filename + ': ' + totalTokenstring + timeString + Fore.GREEN + u' \u2713 ' + Fore.RESET
+ return (
+ filename
+ + ": "
+ + totalTokenstring
+ + timeString
+ + Fore.GREEN
+ + " \u2713 "
+ + Fore.RESET
+ )
else:
# Fail
@@ -107,9 +124,18 @@ def getResultString(translatedData, translationTime, filename):
raise translatedData[2]
except Exception as e:
errorString = str(e) + Fore.RED
- return filename + ': ' + totalTokenstring + timeString + Fore.RED + u' \u2717 ' +\
- errorString + Fore.RESET
-
+ return (
+ filename
+ + ": "
+ + totalTokenstring
+ + timeString
+ + Fore.RED
+ + " \u2717 "
+ + errorString
+ + Fore.RESET
+ )
+
+
def parseText(data, filename):
totalLines = 0
global LOCK
@@ -117,10 +143,12 @@ def parseText(data, filename):
# Get total for progress bar
linesList = data.readlines()
totalLines = len(linesList)
-
- with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar:
- pbar.desc=filename
- pbar.total=totalLines
+
+ with tqdm(
+ bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE
+ ) as pbar:
+ pbar.desc = filename
+ pbar.total = totalLines
try:
response = translateText(linesList, pbar)
except Exception as e:
@@ -128,32 +156,37 @@ def parseText(data, filename):
return [linesList, 0, e]
return [response[0], response[1], None]
+
def translateText(data, pbar):
textHistory = []
maxHistory = MAXHISTORY
- totalTokens = [0,0]
+ totalTokens = [0, 0]
syncIndex = 0
for i in range(len(data)):
if syncIndex > i:
i = syncIndex
- match = re.findall(r'◆.+◆(.+)', data[i])
+ match = re.findall(r"◆.+◆(.+)", data[i])
if len(match) > 0:
jaString = match[0]
### Translate
# Remove any textwrap
- finalJAString = re.sub(r'\\n', ' ', jaString)
-
+ finalJAString = re.sub(r"\\n", " ", jaString)
+
# Translate
- response = translateGPT(finalJAString, 'Previous Text for Context: ' + ' '.join(textHistory), True)
+ response = translateGPT(
+ finalJAString,
+ "Previous Text for Context: " + " ".join(textHistory),
+ True,
+ )
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
translatedText = response[0]
-
+
# TextHistory is what we use to give GPT Context, so thats appended here.
- textHistory.append('\"' + translatedText + '\"')
+ textHistory.append('"' + translatedText + '"')
# Keep textHistory list at length maxHistory
if len(textHistory) > maxHistory:
@@ -161,81 +194,83 @@ def translateText(data, pbar):
# Textwrap
translatedText = textwrap.fill(translatedText, width=WIDTH)
- translatedText = translatedText.replace('\n', '\\n')
+ translatedText = translatedText.replace("\n", "\\n")
# Write
data[i] = data[i].replace(match[0], translatedText)
-
+
syncIndex = i + 1
pbar.update()
return [data, totalTokens]
-
+
+
def subVars(jaString):
- jaString = jaString.replace('\u3000', ' ')
+ jaString = jaString.replace("\u3000", " ")
# Nested
count = 0
- nestedList = re.findall(r'[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]', jaString)
+ nestedList = re.findall(r"[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]", jaString)
nestedList = set(nestedList)
if len(nestedList) != 0:
for icon in nestedList:
- jaString = jaString.replace(icon, '{Nested_' + str(count) + '}')
+ jaString = jaString.replace(icon, "{Nested_" + str(count) + "}")
count += 1
# Icons
count = 0
- iconList = re.findall(r'[\\]+[iIkKwWaA]+\[[0-9]+\]', jaString)
+ iconList = re.findall(r"[\\]+[iIkKwWaA]+\[[0-9]+\]", jaString)
iconList = set(iconList)
if len(iconList) != 0:
for icon in iconList:
- jaString = jaString.replace(icon, '{Ascii_' + str(count) + '}')
+ jaString = jaString.replace(icon, "{Ascii_" + str(count) + "}")
count += 1
# Colors
count = 0
- colorList = re.findall(r'[\\]+[cC]\[[0-9]+\]', jaString)
+ colorList = re.findall(r"[\\]+[cC]\[[0-9]+\]", jaString)
colorList = set(colorList)
if len(colorList) != 0:
for color in colorList:
- jaString = jaString.replace(color, '{Color_' + str(count) + '}')
+ jaString = jaString.replace(color, "{Color_" + str(count) + "}")
count += 1
# Names
count = 0
- nameList = re.findall(r'[\\]+[nN]\[.+?\]+', jaString)
+ nameList = re.findall(r"[\\]+[nN]\[.+?\]+", jaString)
nameList = set(nameList)
if len(nameList) != 0:
for name in nameList:
- jaString = jaString.replace(name, '{N_' + str(count) + '}')
+ jaString = jaString.replace(name, "{N_" + str(count) + "}")
count += 1
# Variables
count = 0
- varList = re.findall(r'[\\]+[vV]\[[0-9]+\]', jaString)
+ varList = re.findall(r"[\\]+[vV]\[[0-9]+\]", jaString)
varList = set(varList)
if len(varList) != 0:
for var in varList:
- jaString = jaString.replace(var, '{Var_' + str(count) + '}')
+ jaString = jaString.replace(var, "{Var_" + str(count) + "}")
count += 1
# Formatting
count = 0
- if '笑えるよね.' in jaString:
- print('t')
- formatList = re.findall(r'[\\]+[\w]+\[.+?\]', jaString)
+ if "笑えるよね." in jaString:
+ print("t")
+ formatList = re.findall(r"[\\]+[\w]+\[.+?\]", jaString)
formatList = set(formatList)
if len(formatList) != 0:
for var in formatList:
- jaString = jaString.replace(var, '{FCode_' + str(count) + '}')
+ jaString = jaString.replace(var, "{FCode_" + str(count) + "}")
count += 1
# Put all lists in list and return
allList = [nestedList, iconList, colorList, nameList, varList, formatList]
return [jaString, allList]
+
def resubVars(translatedText, allList):
# Fix Spacing and ChatGPT Nonsense
- matchList = re.findall(r'\[\s?.+?\s?\]', translatedText)
+ matchList = re.findall(r"\[\s?.+?\s?\]", translatedText)
if len(matchList) > 0:
for match in matchList:
text = match.strip()
@@ -245,42 +280,42 @@ def resubVars(translatedText, allList):
count = 0
if len(allList[0]) != 0:
for var in allList[0]:
- translatedText = translatedText.replace('{Nested_' + str(count) + '}', var)
+ translatedText = translatedText.replace("{Nested_" + str(count) + "}", var)
count += 1
# Icons
count = 0
if len(allList[1]) != 0:
for var in allList[1]:
- translatedText = translatedText.replace('{Ascii_' + str(count) + '}', var)
+ translatedText = translatedText.replace("{Ascii_" + str(count) + "}", var)
count += 1
# Colors
count = 0
if len(allList[2]) != 0:
for var in allList[2]:
- translatedText = translatedText.replace('{Color_' + str(count) + '}', var)
+ translatedText = translatedText.replace("{Color_" + str(count) + "}", var)
count += 1
# Names
count = 0
if len(allList[3]) != 0:
for var in allList[3]:
- translatedText = translatedText.replace('{N_' + str(count) + '}', var)
+ translatedText = translatedText.replace("{N_" + str(count) + "}", var)
count += 1
# Vars
count = 0
if len(allList[4]) != 0:
for var in allList[4]:
- translatedText = translatedText.replace('{Var_' + str(count) + '}', var)
+ translatedText = translatedText.replace("{Var_" + str(count) + "}", var)
count += 1
-
+
# Formatting
count = 0
if len(allList[5]) != 0:
for var in allList[5]:
- translatedText = translatedText.replace('{FCode_' + str(count) + '}', var)
+ translatedText = translatedText.replace("{FCode_" + str(count) + "}", var)
count += 1
# Remove Color Variables Spaces
@@ -289,6 +324,7 @@ def resubVars(translatedText, allList):
# translatedText = re.sub(r'\s*(\\+c\[0+\])', r'\1', translatedText)
return translatedText
+
@retry(exceptions=Exception, tries=5, delay=5)
def translateGPT(t, history, fullPromptFlag):
# Sub Vars
@@ -296,13 +332,13 @@ def translateGPT(t, history, fullPromptFlag):
subbedT = varResponse[0]
# If there isn't any Japanese in the text just skip
- if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴ]+|[\uFF00-\uFFEF]', subbedT):
- return(t, [0,0])
-
+ if not re.search(r"[一-龠]+|[ぁ-ゔ]+|[ァ-ヴ]+|[\uFF00-\uFFEF]", subbedT):
+ return (t, [0, 0])
+
# If ESTIMATE is True just count this as an execution and return.
if ESTIMATE:
- enc = tiktoken.encoding_for_model('gpt-4')
- historyRaw = ''
+ enc = tiktoken.encoding_for_model("gpt-4")
+ historyRaw = ""
if isinstance(history, list):
for line in history:
historyRaw += line
@@ -310,26 +346,34 @@ def translateGPT(t, history, fullPromptFlag):
historyRaw = history
inputTotalTokens = len(enc.encode(historyRaw)) + len(enc.encode(PROMPT))
- outputTotalTokens = len(enc.encode(t)) * 2 # Estimating 2x the size of the original text
+ outputTotalTokens = (
+ len(enc.encode(t)) * 2
+ ) # Estimating 2x the size of the original text
totalTokens = [inputTotalTokens, outputTotalTokens]
return (t, totalTokens)
# Characters
- context = 'Game Characters:\
+ context = "Game Characters:\
Character: Surname:久高 Name:有史 == Surname:Kudaka Name:Yuushi - Gender: Male\
Character: Surname:葛城 Name:碧璃 == Surname:Katsuragi Name:Midori - Gender: Female\
Character: Surname:葛城 Name:依理子 == Surname:Katsuragi Name:Yoriko - Gender: Female\
Character: Surname:桐乃木 Name:奏 == Surname:Kirinogi Name:Kanade - Gender: Female\
Character: Surname:葛城 Name:光男 == Surname:Katsuragi Name:Mitsuo - Gender: Male\
- Character: Surname:尾木 Name:優真 == Surname:Ogi Name:Yuuma - Gender: Male'
+ Character: Surname:尾木 Name:優真 == Surname:Ogi Name:Yuuma - Gender: Male"
# Prompt
if fullPromptFlag:
system = PROMPT
- user = 'Line to Translate = ' + subbedT
+ user = "Line to Translate = " + subbedT
else:
- system = 'Output ONLY the '+ LANGUAGE +' translation in the following format: `Translation: <'+ LANGUAGE.upper() +'_TRANSLATION>`'
- user = 'Line to Translate = ' + subbedT
+ system = (
+ "Output ONLY the "
+ + LANGUAGE
+ + " translation in the following format: `Translation: <"
+ + LANGUAGE.upper()
+ + "_TRANSLATION>`"
+ )
+ user = "Line to Translate = " + subbedT
# Create Message List
msg = []
@@ -359,26 +403,29 @@ def translateGPT(t, history, fullPromptFlag):
translatedText = resubVars(translatedText, varResponse[1])
# Remove Placeholder Text
- translatedText = translatedText.replace(LANGUAGE +' Translation: ', '')
- translatedText = translatedText.replace('Translation: ', '')
- translatedText = translatedText.replace('Line to Translate = ', '')
- translatedText = translatedText.replace('Translation = ', '')
- translatedText = translatedText.replace('Translate = ', '')
- translatedText = translatedText.replace(LANGUAGE +' Translation:', '')
- translatedText = translatedText.replace('Translation:', '')
- translatedText = translatedText.replace('Line to Translate =', '')
- translatedText = translatedText.replace('Translation =', '')
- translatedText = translatedText.replace('Translate =', '')
- translatedText = translatedText.replace('っ', '')
- translatedText = translatedText.replace('ッ', '')
- translatedText = translatedText.replace('ぁ', '')
- translatedText = translatedText.replace('。', '.')
- translatedText = translatedText.replace('、', ',')
- translatedText = translatedText.replace('？', '?')
- translatedText = translatedText.replace('！', '!')
+ translatedText = translatedText.replace(LANGUAGE + " Translation: ", "")
+ translatedText = translatedText.replace("Translation: ", "")
+ translatedText = translatedText.replace("Line to Translate = ", "")
+ translatedText = translatedText.replace("Translation = ", "")
+ translatedText = translatedText.replace("Translate = ", "")
+ translatedText = translatedText.replace(LANGUAGE + " Translation:", "")
+ translatedText = translatedText.replace("Translation:", "")
+ translatedText = translatedText.replace("Line to Translate =", "")
+ translatedText = translatedText.replace("Translation =", "")
+ translatedText = translatedText.replace("Translate =", "")
+ translatedText = translatedText.replace("っ", "")
+ translatedText = translatedText.replace("ッ", "")
+ translatedText = translatedText.replace("ぁ", "")
+ translatedText = translatedText.replace("。", ".")
+ translatedText = translatedText.replace("、", ",")
+ translatedText = translatedText.replace("？", "?")
+ translatedText = translatedText.replace("！", "!")
# Return Translation
- if len(translatedText) > 15 * len(t) or "I'm sorry, but I'm unable to assist with that translation" in translatedText:
+ if (
+ len(translatedText) > 15 * len(t)
+ or "I'm sorry, but I'm unable to assist with that translation" in translatedText
+ ):
raise Exception
else:
- return [translatedText, totalTokens]
\ No newline at end of file
+ return [translatedText, totalTokens]
diff --git a/modules/csv.py b/modules/csv.py
index 1e4ad49..ca3f873 100644
--- a/modules/csv.py
+++ b/modules/csv.py
@@ -17,63 +17,66 @@ from tqdm import tqdm
# Open AI
load_dotenv()
-if os.getenv('api').replace(' ', '') != '':
- openai.base_url = os.getenv('api')
-openai.organization = os.getenv('org')
-openai.api_key = os.getenv('key')
+if os.getenv("api").replace(" ", "") != "":
+ openai.base_url = os.getenv("api")
+openai.organization = os.getenv("org")
+openai.api_key = os.getenv("key")
-#Globals
-MODEL = os.getenv('model')
-TIMEOUT = int(os.getenv('timeout'))
-LANGUAGE = os.getenv('language').capitalize()
-PROMPT = Path('prompt.txt').read_text(encoding='utf-8')
-VOCAB = Path('vocab.txt').read_text(encoding='utf-8')
-THREADS = int(os.getenv('threads'))
+# Globals
+MODEL = os.getenv("model")
+TIMEOUT = int(os.getenv("timeout"))
+LANGUAGE = os.getenv("language").capitalize()
+PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
+VOCAB = Path("vocab.txt").read_text(encoding="utf-8")
+THREADS = int(os.getenv("threads"))
LOCK = threading.Lock()
-WIDTH = int(os.getenv('width'))
-LISTWIDTH = int(os.getenv('listWidth'))
-NOTEWIDTH = int(os.getenv('noteWidth'))
+WIDTH = int(os.getenv("width"))
+LISTWIDTH = int(os.getenv("listWidth"))
+NOTEWIDTH = int(os.getenv("noteWidth"))
MAXHISTORY = 10
-ESTIMATE = ''
+ESTIMATE = ""
TOKENS = [0, 0]
NAMESLIST = []
-NAMES = False # Output a list of all the character names found
-BRFLAG = False # If the game uses
instead
+NAMES = False # Output a list of all the character names found
+BRFLAG = False # If the game uses
instead
FIXTEXTWRAP = True # Overwrites textwrap
-IGNORETLTEXT = True # Ignores all translated text.
-MISMATCH = [] # Lists files that thdata a mismatch error (Length of GPT list response is wrong)
+IGNORETLTEXT = True # Ignores all translated text.
+MISMATCH = [] # Lists files that thdata a mismatch error (Length of GPT list response is wrong)
BRACKETNAMES = False
# Pricing - Depends on the model https://openai.com/pricing
# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request
# If you are getting a MISMATCH LENGTH error, lower the batch size.
-if 'gpt-3.5' in MODEL:
- INPUTAPICOST = .002
- OUTPUTAPICOST = .002
+if "gpt-3.5" in MODEL:
+ INPUTAPICOST = 0.002
+ OUTPUTAPICOST = 0.002
BATCHSIZE = 10
FREQUENCY_PENALTY = 0.2
-elif 'gpt-4' in MODEL:
- INPUTAPICOST = .005
- OUTPUTAPICOST = .015
+elif "gpt-4" in MODEL:
+ INPUTAPICOST = 0.005
+ OUTPUTAPICOST = 0.015
BATCHSIZE = 20
FREQUENCY_PENALTY = 0.1
-#tqdm Globals
-BAR_FORMAT='{l_bar}{bar:10}{r_bar}{bar:-10b}'
+# tqdm Globals
+BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
POSITION = 0
LEAVE = False
PBAR = None
+
def handleCSV(filename, estimate):
global ESTIMATE, TOKENS
ESTIMATE = estimate
if not ESTIMATE:
- with open('translated/' + filename, 'w+t', newline='', encoding='utf-8-sig') as writeFile:
+ with open(
+ "translated/" + filename, "w+t", newline="", encoding="utf-8-sig"
+ ) as writeFile:
# Translate
start = time.time()
translatedData = openFiles(filename, writeFile)
-
+
# Print Result
end = time.time()
tqdm.write(getResultString(translatedData, end - start, filename))
@@ -84,7 +87,7 @@ def handleCSV(filename, estimate):
# Translate
start = time.time()
translatedData = openFilesEstimate(filename)
-
+
# Print Result
end = time.time()
tqdm.write(getResultString(translatedData, end - start, filename))
@@ -92,41 +95,55 @@ def handleCSV(filename, estimate):
TOKENS[0] += translatedData[1][0]
TOKENS[1] += translatedData[1][1]
-
# Print Total
- totalString = getResultString(['', TOKENS, None], end - start, 'TOTAL')
+ totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
# Print any errors on maps
if len(MISMATCH) > 0:
- return totalString + Fore.RED + f'\nMismatch Errors: {MISMATCH}' + Fore.RESET
+ return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
else:
return totalString
+
def openFiles(filename, writeFile):
- with open('files/' + filename, 'r', encoding='utf-8-sig') as readFile, writeFile:
+ with open("files/" + filename, "r", encoding="utf-8-sig") as readFile, writeFile:
translatedData = parseCSV(readFile, writeFile, filename)
return translatedData
+
def openFilesEstimate(filename):
- with open('files/' + filename, 'r', encoding='utf-8-sig') as readFile:
- translatedData = parseCSV(readFile, '', filename)
+ with open("files/" + filename, "r", encoding="utf-8-sig") as readFile:
+ translatedData = parseCSV(readFile, "", filename)
return translatedData
+
def getResultString(translatedData, translationTime, filename):
# File Print String
- totalTokenstring =\
- Fore.YELLOW +\
- '[Input: ' + str(translatedData[1][0]) + ']'\
- '[Output: ' + str(translatedData[1][1]) + ']'\
- '[Cost: ${:,.4f}'.format((translatedData[1][0] * .001 * INPUTAPICOST) +\
- (translatedData[1][1] * .001 * OUTPUTAPICOST)) + ']'
- timeString = Fore.BLUE + '[' + str(round(translationTime, 1)) + 's]'
+ totalTokenstring = (
+ Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
+ "[Output: "
+ + str(translatedData[1][1])
+ + "]" "[Cost: ${:,.4f}".format(
+ (translatedData[1][0] * 0.001 * INPUTAPICOST)
+ + (translatedData[1][1] * 0.001 * OUTPUTAPICOST)
+ )
+ + "]"
+ )
+ timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
if translatedData[2] is None:
# Success
- return filename + ': ' + totalTokenstring + timeString + Fore.GREEN + u' \u2713 ' + Fore.RESET
+ return (
+ filename
+ + ": "
+ + totalTokenstring
+ + timeString
+ + Fore.GREEN
+ + " \u2713 "
+ + Fore.RESET
+ )
else:
# Fail
try:
@@ -134,38 +151,51 @@ def getResultString(translatedData, translationTime, filename):
except Exception as e:
traceback.print_exc()
errorString = str(e) + Fore.RED
- return filename + ': ' + totalTokenstring + timeString + Fore.RED + u' \u2717 ' +\
- errorString + Fore.RESET
-
+ return (
+ filename
+ + ": "
+ + totalTokenstring
+ + timeString
+ + Fore.RED
+ + " \u2717 "
+ + errorString
+ + Fore.RESET
+ )
+
+
def parseCSV(readFile, writeFile, filename):
- totalTokens = [0,0]
+ totalTokens = [0, 0]
totalLines = 0
global LOCK
- format = ''
- while format not in ['1', '2', '3']:
- format = input('\n\nSelect the CSV Format:\n\n1. Translator++\n2. Single\n3. Multiple\n')
+ format = ""
+ while format not in ["1", "2", "3"]:
+ format = input(
+ "\n\nSelect the CSV Format:\n\n1. Translator++\n2. Single\n3. Multiple\n"
+ )
match format:
- case '1':
- format = '1'
- case '2':
- format = '2'
- case '3':
- format = '3'
+ case "1":
+ format = "1"
+ case "2":
+ format = "2"
+ case "3":
+ format = "3"
# Get total for progress bar
totalLines = len(readFile.readlines())
readFile.seek(0)
- reader = csv.reader(readFile, delimiter=',')
+ reader = csv.reader(readFile, delimiter=",")
if not ESTIMATE:
- writer = csv.writer(writeFile, delimiter=',', quotechar='\"')
+ writer = csv.writer(writeFile, delimiter=",", quotechar='"')
else:
- writer = ''
+ writer = ""
- with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar:
- pbar.desc=filename
- pbar.total=totalLines
+ with tqdm(
+ bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE
+ ) as pbar:
+ pbar.desc = filename
+ pbar.total = totalLines
# Grab All Rows
data = []
@@ -180,11 +210,12 @@ def parseCSV(readFile, writeFile, filename):
traceback.print_exc()
return [data, totalTokens, None]
+
def translateCSV(data, pbar, writer, filename, translatedList, format):
global LOCK, ESTIMATE, PBAR
PBAR = pbar
- translatedText = ''
- totalTokens = [0,0]
+ translatedText = ""
+ totalTokens = [0, 0]
i = 0
stringList = []
@@ -193,7 +224,7 @@ def translateCSV(data, pbar, writer, filename, translatedList, format):
while i < len(data):
match format:
# T++ Format: Source Text on column 1. TL Target on Column 2
- case '1':
+ case "1":
# Get String
if i != 0:
if data[i][1] == "":
@@ -202,7 +233,7 @@ def translateCSV(data, pbar, writer, filename, translatedList, format):
jaString = data[i][1]
# Remove Textwrap
- jaString = jaString.replace('\\n', ' ')
+ jaString = jaString.replace("\\n", " ")
# Pass 1
if not translatedList:
@@ -216,16 +247,16 @@ def translateCSV(data, pbar, writer, filename, translatedList, format):
# Add Wordwrap
translatedText = textwrap.fill(translatedText, WIDTH)
- translatedText = translatedText.replace('\n', '\\n')
+ translatedText = translatedText.replace("\n", "\\n")
# Set Data
data[i][1] = translatedText
-
+
# Iterate
i += 1
-
+
# Target Format
- case '2':
+ case "2":
# Set Values
sourceColumn = 0
targetColumn = 1
@@ -234,7 +265,7 @@ def translateCSV(data, pbar, writer, filename, translatedList, format):
jaString = data[i][sourceColumn]
# Remove Textwrap
- jaString = jaString.replace('\\n', ' ')
+ jaString = jaString.replace("\\n", " ")
# Pass 1
if not translatedList:
@@ -248,22 +279,22 @@ def translateCSV(data, pbar, writer, filename, translatedList, format):
# Add Wordwrap
translatedText = textwrap.fill(translatedText, WIDTH)
- translatedText = translatedText.replace('\n', '\\n')
+ translatedText = translatedText.replace("\n", "\\n")
# Set Data
data[i][targetColumn] = translatedText
-
+
# Iterate
i += 1
# All Format
- case '3':
+ case "3":
# Set columns to translate. Leave empty to translate all.
targetColumns = []
# False - Place translation in source column
# True - Place translation in next column
- targetNextRow = True
+ targetNextRow = True
for j in range(len(data[i])):
if j not in targetColumns:
@@ -271,7 +302,7 @@ def translateCSV(data, pbar, writer, filename, translatedList, format):
jaString = data[i][j]
# Remove Textwrap
- jaString = jaString.replace('\\n', ' ')
+ jaString = jaString.replace("\\n", " ")
# Pass 1
if not translatedList:
@@ -285,14 +316,14 @@ def translateCSV(data, pbar, writer, filename, translatedList, format):
# Add Wordwrap
translatedText = textwrap.fill(translatedText, WIDTH)
- translatedText = translatedText.replace('\n', '\\n')
+ translatedText = translatedText.replace("\n", "\\n")
# Set Data
if targetNextRow:
data[i][j + 1] = translatedText
else:
data[i][j] = translatedText
-
+
# Iterate
i += 1
@@ -301,9 +332,9 @@ def translateCSV(data, pbar, writer, filename, translatedList, format):
# Set Progress
pbar.total = len(stringList)
pbar.refresh()
-
+
# Translate
- response = translateGPT(stringList, '', True)
+ response = translateGPT(stringList, "", True)
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
translatedList = response[0]
@@ -333,27 +364,35 @@ def translateCSV(data, pbar, writer, filename, translatedList, format):
for row in data:
writer.writerow(row)
return totalTokens
-
+
return totalTokens
-
+
# Save some money and enter the character before translation
def getSpeaker(speaker):
match speaker:
- case 'ファイン':
- return ['Fine', [0,0]]
- case '':
- return ['', [0,0]]
+ case "ファイン":
+ return ["Fine", [0, 0]]
+ case "":
+ return ["", [0, 0]]
case _:
# Store Speaker
if speaker not in str(NAMESLIST):
- response = translateGPT(speaker, 'Reply with the '+ LANGUAGE +' translation of the NPC name.', False)
+ response = translateGPT(
+ speaker,
+ "Reply with the " + LANGUAGE + " translation of the NPC name.",
+ False,
+ )
response[0] = response[0].title()
response[0] = response[0].replace("'S", "'s")
# Retry if name doesn't translate for some reason
- if re.search(r'([a-zA-Z？?])', response[0]) == None:
- response = translateGPT(speaker, 'Reply with the '+ LANGUAGE +' translation of the NPC name.', False)
+ if re.search(r"([a-zA-Z？?])", response[0]) == None:
+ response = translateGPT(
+ speaker,
+ "Reply with the " + LANGUAGE + " translation of the NPC name.",
+ False,
+ )
response[0] = response[0].title()
response[0] = response[0].replace("'S", "'s")
@@ -364,74 +403,76 @@ def getSpeaker(speaker):
else:
for i in range(len(NAMESLIST)):
if speaker == NAMESLIST[i][0]:
- return [NAMESLIST[i][1],[0,0]]
-
- return [speaker,[0,0]]
+ return [NAMESLIST[i][1], [0, 0]]
+
+ return [speaker, [0, 0]]
+
def subVars(jaString):
- jaString = jaString.replace('\u3000', ' ')
+ jaString = jaString.replace("\u3000", " ")
# Nested
count = 0
- nestedList = re.findall(r'[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]', jaString)
+ nestedList = re.findall(r"[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]", jaString)
nestedList = set(nestedList)
if len(nestedList) != 0:
for icon in nestedList:
- jaString = jaString.replace(icon, '[Nested_' + str(count) + ']')
+ jaString = jaString.replace(icon, "[Nested_" + str(count) + "]")
count += 1
# Icons
count = 0
- iconList = re.findall(r'[\\]+[iIkKwWaA]+\[[0-9]+\]', jaString)
+ iconList = re.findall(r"[\\]+[iIkKwWaA]+\[[0-9]+\]", jaString)
iconList = set(iconList)
if len(iconList) != 0:
for icon in iconList:
- jaString = jaString.replace(icon, '[Ascii_' + str(count) + ']')
+ jaString = jaString.replace(icon, "[Ascii_" + str(count) + "]")
count += 1
# Colors
count = 0
- colorList = re.findall(r'([\\]+c\[\d+\][\\]+c|[\\]+c\[\d+\])', jaString)
+ colorList = re.findall(r"([\\]+c\[\d+\][\\]+c|[\\]+c\[\d+\])", jaString)
colorList = set(colorList)
if len(colorList) != 0:
for color in colorList:
- jaString = jaString.replace(color, '[Color_' + str(count) + ']')
+ jaString = jaString.replace(color, "[Color_" + str(count) + "]")
count += 1
# Names
count = 0
- nameList = re.findall(r'[\\]+[nN]\[.+?\]+', jaString)
+ nameList = re.findall(r"[\\]+[nN]\[.+?\]+", jaString)
nameList = set(nameList)
if len(nameList) != 0:
for name in nameList:
- jaString = jaString.replace(name, '[Noun_' + str(count) + ']')
+ jaString = jaString.replace(name, "[Noun_" + str(count) + "]")
count += 1
# Variables
count = 0
- varList = re.findall(r'[\\]+[vV]\[[0-9]+\]', jaString)
+ varList = re.findall(r"[\\]+[vV]\[[0-9]+\]", jaString)
varList = set(varList)
if len(varList) != 0:
for var in varList:
- jaString = jaString.replace(var, '[Var_' + str(count) + ']')
+ jaString = jaString.replace(var, "[Var_" + str(count) + "]")
count += 1
# Formatting
count = 0
- formatList = re.findall(r'[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]', jaString)
+ formatList = re.findall(r"[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]", jaString)
formatList = set(formatList)
if len(formatList) != 0:
for var in formatList:
- jaString = jaString.replace(var, '[FCode_' + str(count) + ']')
+ jaString = jaString.replace(var, "[FCode_" + str(count) + "]")
count += 1
# Put all lists in list and return
allList = [nestedList, iconList, colorList, nameList, varList, formatList]
return [jaString, allList]
+
def resubVars(translatedText, allList):
# Fix Spacing and ChatGPT Nonsense
- matchList = re.findall(r'\[\s?.+?\s?\]', translatedText)
+ matchList = re.findall(r"\[\s?.+?\s?\]", translatedText)
if len(matchList) > 0:
for match in matchList:
text = match.strip()
@@ -441,60 +482,66 @@ def resubVars(translatedText, allList):
count = 0
if len(allList[0]) != 0:
for var in allList[0]:
- translatedText = translatedText.replace('[Nested_' + str(count) + ']', var)
+ translatedText = translatedText.replace("[Nested_" + str(count) + "]", var)
count += 1
# Icons
count = 0
if len(allList[1]) != 0:
for var in allList[1]:
- translatedText = translatedText.replace('[Ascii_' + str(count) + ']', var)
+ translatedText = translatedText.replace("[Ascii_" + str(count) + "]", var)
count += 1
# Colors
count = 0
if len(allList[2]) != 0:
for var in allList[2]:
- translatedText = translatedText.replace('[Color_' + str(count) + ']', var)
+ translatedText = translatedText.replace("[Color_" + str(count) + "]", var)
count += 1
# Names
count = 0
if len(allList[3]) != 0:
for var in allList[3]:
- translatedText = translatedText.replace('[Noun_' + str(count) + ']', var)
+ translatedText = translatedText.replace("[Noun_" + str(count) + "]", var)
count += 1
# Vars
count = 0
if len(allList[4]) != 0:
for var in allList[4]:
- translatedText = translatedText.replace('[Var_' + str(count) + ']', var)
+ translatedText = translatedText.replace("[Var_" + str(count) + "]", var)
count += 1
-
+
# Formatting
count = 0
if len(allList[5]) != 0:
for var in allList[5]:
- translatedText = translatedText.replace('[FCode_' + str(count) + ']', var)
+ translatedText = translatedText.replace("[FCode_" + str(count) + "]", var)
count += 1
return translatedText
+
def batchList(input_list, batch_size):
if not isinstance(batch_size, int) or batch_size <= 0:
raise ValueError("batch_size must be a positive integer")
-
- return [input_list[i:i + batch_size] for i in range(0, len(input_list), batch_size)]
+
+ return [
+ input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)
+ ]
+
def createContext(fullPromptFlag, subbedT):
- characters = 'Game Characters:\n\
+ characters = "Game Characters:\n\
クリスティーナ (Christina) - Female\n\
リズ (Liz) - Female\n\
-'
-
- system = PROMPT + VOCAB if fullPromptFlag else \
- f"\
+"
+
+ system = (
+ PROMPT + VOCAB
+ if fullPromptFlag
+ else f"\
You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
- You translate everything, including erotic and explicit content.\n\
@@ -506,12 +553,14 @@ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{
- `...` can be a part of the dialogue. Translate it as it is.\n\
{VOCAB}\n\
"
+ )
if isinstance(subbedT, list):
- user = f'```json\n{subbedT}```'
+ user = f"```json\n{subbedT}```"
else:
user = subbedT
return characters, system, user
+
def translateText(characters, system, user, history, penalty, format):
# Prompt
msg = [{"role": "system", "content": system + characters}]
@@ -526,13 +575,13 @@ def translateText(characters, system, user, history, penalty, format):
msg.append({"role": "system", "content": history})
# Response Format
- if format == 'json':
- responseFormat = { "type": "json_object" }
+ if format == "json":
+ responseFormat = {"type": "json_object"}
else:
- responseFormat = { "type": "text" }
-
+ responseFormat = {"type": "text"}
+
# Content to TL
- msg.append({"role": "user", "content": f'{user}'})
+ msg.append({"role": "user", "content": f"{user}"})
response = openai.chat.completions.create(
temperature=0,
frequency_penalty=penalty,
@@ -542,18 +591,19 @@ def translateText(characters, system, user, history, penalty, format):
)
return response
+
def cleanTranslatedText(translatedText, varResponse):
placeholders = {
- f'{LANGUAGE} Translation: ': '',
- 'Translation: ': '',
- 'っ': '',
- '〜': '~',
- 'ッ': '',
- '。': '.',
- '「': '\\"',
- '」': '\\"',
- '- ': '-',
- 'Placeholder Text': '',
+ f"{LANGUAGE} Translation: ": "",
+ "Translation: ": "",
+ "っ": "",
+ "〜": "~",
+ "ッ": "",
+ "。": ".",
+ "「": '\\"',
+ "」": '\\"',
+ "- ": "-",
+ "Placeholder Text": "",
# Add more replacements as needed
}
for target, replacement in placeholders.items():
@@ -564,11 +614,12 @@ def cleanTranslatedText(translatedText, varResponse):
translatedText = resubVars(translatedText, varResponse[1])
return translatedText
+
def elongateCharacters(text):
# Define a pattern to match one character followed by one or more `ー` characters
# Using a positive lookbehind assertion to capture the preceding character
- pattern = r'(?<=(.))ー+'
-
+ pattern = r"(?<=(.))ー+"
+
# Define a replacement function that elongates the captured character
def repl(match):
char = match.group(1) # The character before the ー sequence
@@ -578,6 +629,7 @@ def elongateCharacters(text):
# Use re.sub() to replace the pattern in the text
return re.sub(pattern, repl, text)
+
def extractTranslation(translatedTextList, is_list):
try:
translatedTextList = re.sub(r'\\"+\"([^,\n}])', r'\\"\1', translatedTextList)
@@ -590,15 +642,15 @@ def extractTranslation(translatedTextList, is_list):
return string_list[0]
except Exception as e:
- PBAR.write(f'extractTranslation Error: {e} on String {translatedTextList}')
+ PBAR.write(f"extractTranslation Error: {e} on String {translatedTextList}")
return None
def countTokens(characters, system, user, history):
inputTotalTokens = 0
outputTotalTokens = 0
- enc = tiktoken.encoding_for_model('gpt-4')
-
+ enc = tiktoken.encoding_for_model("gpt-4")
+
# Input
if isinstance(history, list):
for line in history:
@@ -610,26 +662,28 @@ def countTokens(characters, system, user, history):
inputTotalTokens += len(enc.encode(user))
# Output
- outputTotalTokens += round(len(enc.encode(user))*3)
+ outputTotalTokens += round(len(enc.encode(user)) * 3)
return [inputTotalTokens, outputTotalTokens]
+
def combineList(tlist, text):
if isinstance(text, list):
return [t for sublist in tlist for t in sublist]
return tlist[0]
+
@retry(exceptions=Exception, tries=5, delay=5)
def translateGPT(text, history, fullPromptFlag):
global PBAR
-
+
mismatch = False
totalTokens = [0, 0]
if isinstance(text, list):
- format = 'json'
+ format = "json"
tList = batchList(text, BATCHSIZE)
else:
- format = 'text'
+ format = "text"
tList = [text]
for index, tItem in enumerate(tList):
@@ -644,7 +698,7 @@ def translateGPT(text, history, fullPromptFlag):
subbedT = varResponse[0]
# Things to Check before starting translation
- if not re.search(r'[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+', subbedT):
+ if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", subbedT):
if PBAR is not None:
PBAR.update(len(tItem))
continue
@@ -669,9 +723,13 @@ def translateGPT(text, history, fullPromptFlag):
translatedText = cleanTranslatedText(translatedText, varResponse)
if isinstance(tItem, list):
extractedTranslations = extractTranslation(translatedText, True)
- if extractedTranslations == None or len(tItem) != len(extractedTranslations):
+ if extractedTranslations == None or len(tItem) != len(
+ extractedTranslations
+ ):
# Mismatch. Try Again
- response = translateText(characters, system, user, history, 0.05, format)
+ response = translateText(
+ characters, system, user, history, 0.05, format
+ )
translatedText = response.choices[0].message.content
totalTokens[0] += response.usage.prompt_tokens
totalTokens[1] += response.usage.completion_tokens
@@ -680,13 +738,17 @@ def translateGPT(text, history, fullPromptFlag):
translatedText = cleanTranslatedText(translatedText, varResponse)
if isinstance(tItem, list):
extractedTranslations = extractTranslation(translatedText, True)
- if extractedTranslations == None or len(tItem) != len(extractedTranslations):
- mismatch = True # Just here for breakpoint
-
+ if extractedTranslations == None or len(tItem) != len(
+ extractedTranslations
+ ):
+ mismatch = True # Just here for breakpoint
+
# Set if no mismatch
if mismatch == False:
tList[index] = extractedTranslations
- history = extractedTranslations[-10:] # Update history if we have a list
+ history = extractedTranslations[
+ -10:
+ ] # Update history if we have a list
else:
history = text[-10:]
mismatch = False
@@ -697,7 +759,7 @@ def translateGPT(text, history, fullPromptFlag):
PBAR.update(len(tItem))
else:
# Ensure we're passing a single string to extractTranslation
- tList[index] = translatedText
+ tList[index] = translatedText
finalList = combineList(tList, text)
return [finalList, totalTokens]
diff --git a/modules/eushully.py b/modules/eushully.py
index 7c291a1..447fe7f 100644
--- a/modules/eushully.py
+++ b/modules/eushully.py
@@ -15,34 +15,34 @@ from tqdm import tqdm
# Open AI
load_dotenv()
-if os.getenv('api').replace(' ', '') != '':
- openai.base_url = os.getenv('api')
-openai.organization = os.getenv('org')
-openai.api_key = os.getenv('key')
+if os.getenv("api").replace(" ", "") != "":
+ openai.base_url = os.getenv("api")
+openai.organization = os.getenv("org")
+openai.api_key = os.getenv("key")
-#Globals
-MODEL = os.getenv('model')
-TIMEOUT = int(os.getenv('timeout'))
-LANGUAGE = os.getenv('language').capitalize()
-PROMPT = Path('prompt.txt').read_text(encoding='utf-8')
-VOCAB = Path('vocab.txt').read_text(encoding='utf-8')
-THREADS = int(os.getenv('threads'))
+# Globals
+MODEL = os.getenv("model")
+TIMEOUT = int(os.getenv("timeout"))
+LANGUAGE = os.getenv("language").capitalize()
+PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
+VOCAB = Path("vocab.txt").read_text(encoding="utf-8")
+THREADS = int(os.getenv("threads"))
LOCK = threading.Lock()
-WIDTH = int(os.getenv('width'))
-LISTWIDTH = int(os.getenv('listWidth'))
+WIDTH = int(os.getenv("width"))
+LISTWIDTH = int(os.getenv("listWidth"))
NOTEWIDTH = 70
MAXHISTORY = 10
-ESTIMATE = ''
+ESTIMATE = ""
TOKENS = [0, 0]
NAMESLIST = []
-NAMES = False # Output a list of all the character names found
-BRFLAG = False # If the game uses
instead
+NAMES = False # Output a list of all the character names found
+BRFLAG = False # If the game uses
instead
FIXTEXTWRAP = True # Overwrites textwrap
-IGNORETLTEXT = False # Ignores all translated text.
-MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
+IGNORETLTEXT = False # Ignores all translated text.
+MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
-#tqdm Globals
-BAR_FORMAT='{l_bar}{bar:10}{r_bar}{bar:-10b}'
+# tqdm Globals
+BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
POSITION = 0
LEAVE = False
PBAR = None
@@ -50,15 +50,16 @@ PBAR = None
# Pricing - Depends on the model https://openai.com/pricing
# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request
# If you are getting a MISMATCH LENGTH error, lower the batch size.
-if 'gpt-3.5' in MODEL:
- INPUTAPICOST = .002
- OUTPUTAPICOST = .002
+if "gpt-3.5" in MODEL:
+ INPUTAPICOST = 0.002
+ OUTPUTAPICOST = 0.002
BATCHSIZE = 10
-elif 'gpt-4' in MODEL:
- INPUTAPICOST = .005
- OUTPUTAPICOST = .015
+elif "gpt-4" in MODEL:
+ INPUTAPICOST = 0.005
+ OUTPUTAPICOST = 0.015
BATCHSIZE = 40
+
def handleEushully(filename, estimate):
global ESTIMATE
ESTIMATE = estimate
@@ -75,17 +76,21 @@ def handleEushully(filename, estimate):
TOKENS[1] += translatedData[1][1]
# Print Total
- totalString = getResultString(['', TOKENS, None], end - start, 'TOTAL')
+ totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
# Print any errors on maps
if len(MISMATCH) > 0:
- return totalString + Fore.RED + f'\nMismatch Errors: {MISMATCH}' + Fore.RESET
+ return (
+ totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
+ )
else:
return totalString
-
+
else:
try:
- with open('translated/' + filename, 'w', encoding='utf-8', errors='ignore') as outFile:
+ with open(
+ "translated/" + filename, "w", encoding="utf-8", errors="ignore"
+ ) as outFile:
start = time.time()
translatedData = openFiles(filename)
@@ -98,23 +103,36 @@ def handleEushully(filename, estimate):
TOKENS[1] += translatedData[1][1]
except Exception:
traceback.print_exc()
- return 'Fail'
+ return "Fail"
+
+ return getResultString(["", TOKENS, None], end - start, "TOTAL")
- return getResultString(['', TOKENS, None], end - start, 'TOTAL')
def getResultString(translatedData, translationTime, filename):
# File Print String
- totalTokenstring =\
- Fore.YELLOW +\
- '[Input: ' + str(translatedData[1][0]) + ']'\
- '[Output: ' + str(translatedData[1][1]) + ']'\
- '[Cost: ${:,.4f}'.format((translatedData[1][0] * .001 * INPUTAPICOST) +\
- (translatedData[1][1] * .001 * OUTPUTAPICOST)) + ']'
- timeString = Fore.BLUE + '[' + str(round(translationTime, 1)) + 's]'
+ totalTokenstring = (
+ Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
+ "[Output: "
+ + str(translatedData[1][1])
+ + "]" "[Cost: ${:,.4f}".format(
+ (translatedData[1][0] * 0.001 * INPUTAPICOST)
+ + (translatedData[1][1] * 0.001 * OUTPUTAPICOST)
+ )
+ + "]"
+ )
+ timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
if translatedData[2] == None:
# Success
- return filename + ': ' + totalTokenstring + timeString + Fore.GREEN + u' \u2713 ' + Fore.RESET
+ return (
+ filename
+ + ": "
+ + totalTokenstring
+ + timeString
+ + Fore.GREEN
+ + " \u2713 "
+ + Fore.RESET
+ )
else:
# Fail
@@ -123,31 +141,41 @@ def getResultString(translatedData, translationTime, filename):
except Exception as e:
traceback.print_exc()
errorString = str(e) + Fore.RED
- return filename + ': ' + totalTokenstring + timeString + Fore.RED + u' \u2717 ' +\
- errorString + Fore.RESET
+ return (
+ filename
+ + ": "
+ + totalTokenstring
+ + timeString
+ + Fore.RED
+ + " \u2717 "
+ + errorString
+ + Fore.RESET
+ )
+
def openFiles(filename):
- with open('files/' + filename, 'r', encoding='utf-8') as readFile:
+ with open("files/" + filename, "r", encoding="utf-8") as readFile:
translatedData = parseRegex(readFile, filename)
# Delete lines marked for deletion
finalData = []
for line in translatedData[0]:
- if line != '\\d\n':
+ if line != "\\d\n":
finalData.append(line)
translatedData[0] = finalData
-
+
return translatedData
+
def parseRegex(readFile, filename):
- totalTokens = [0,0]
+ totalTokens = [0, 0]
# Read File into data
data = readFile.readlines()
# Create Progress Bar
with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
- pbar.desc=filename
+ pbar.desc = filename
try:
result = translateEushully(data, pbar, filename, [])
@@ -158,11 +186,12 @@ def parseRegex(readFile, filename):
return [data, totalTokens, e]
return [data, totalTokens, None]
+
def translateEushully(data, pbar, filename, translatedList):
stringList = []
currentGroup = []
- tokens = [0,0]
- speaker = ''
+ tokens = [0, 0]
+ speaker = ""
voice = False
global LOCK, ESTIMATE, PBAR
i = 0
@@ -170,9 +199,9 @@ def translateEushully(data, pbar, filename, translatedList):
while i < len(data):
voice = False
# Speaker
- if 'mov (global-int 46e2)' in data[i]:
+ if "mov (global-int 46e2)" in data[i]:
# Get Speaker
- speaker = re.search(r'mov \(global-int 46e2\)\s(.+)', data[i]).group(1)
+ speaker = re.search(r"mov \(global-int 46e2\)\s(.+)", data[i]).group(1)
response = getSpeaker(speaker)
speaker = response[0]
tokens[0] += response[1][0]
@@ -180,32 +209,34 @@ def translateEushully(data, pbar, filename, translatedList):
i += 1
# Show Text
- if any(x in data[i] for x in ['show-text']):
+ if any(x in data[i] for x in ["show-text"]):
# Lines
regex = r'(.*?)"(.*)"'
match = re.search(regex, data[i])
# Grab Strings
- if match != None and match.group(2) != '':
+ if match != None and match.group(2) != "":
originalString = match.group(2)
jaString = match.group(2)
currentGroup = [jaString]
- while 'end-text-line' in data[i+1] and any(x in data[i+2] for x in ['show-text']):
- match = re.search(regex, data[i+2])
+ while "end-text-line" in data[i + 1] and any(
+ x in data[i + 2] for x in ["show-text"]
+ ):
+ match = re.search(regex, data[i + 2])
if match != None:
currentGroup.append(match.group(2))
if translatedList == []:
- del(data[i])
- del(data[i])
- jaString = ' '.join(currentGroup)
-
+ del data[i]
+ del data[i]
+ jaString = " ".join(currentGroup)
+
# Pass 1
if translatedList == []:
# Add String
if speaker:
- stringList.append(f'[{speaker}]: {jaString.strip()}')
+ stringList.append(f"[{speaker}]: {jaString.strip()}")
else:
stringList.append(jaString.strip())
-
+
# Pass 2
else:
# Get Text
@@ -222,51 +253,58 @@ def translateEushully(data, pbar, filename, translatedList):
translatedText = translatedText.replace('"', "'")
# Remove speaker
- if speaker != '':
- translatedText = re.sub(r'^\[?(.+?)\]?\s?[|:]\s?', '', translatedText)
+ if speaker != "":
+ translatedText = re.sub(
+ r"^\[?(.+?)\]?\s?[|:]\s?", "", translatedText
+ )
# Textwrap
translatedText = textwrap.fill(translatedText, width=WIDTH)
- translatedTextList = translatedText.split('\n')
+ translatedTextList = translatedText.split("\n")
# Set Data
if len(translatedTextList) > 1:
for j in range(len(translatedTextList)):
- if any(x in data[i] for x in ['show-text', 'set-string', 'concat']):
- del(data[i])
- data.insert(i, f'{match.group(1)}"{translatedTextList[j]}"\n')
+ if any(
+ x in data[i]
+ for x in ["show-text", "set-string", "concat"]
+ ):
+ del data[i]
+ data.insert(
+ i, f'{match.group(1)}"{translatedTextList[j]}"\n'
+ )
i += 1
- if 'end-text-line' not in data[i]:
- data.insert(i, 'end-text-line 0\n')
+ if "end-text-line" not in data[i]:
+ data.insert(i, "end-text-line 0\n")
i += 1
else:
data[i] = f'{match.group(1)}"{translatedTextList[0]}"\n'
- speaker = ''
+ speaker = ""
i += 1
-
+
# Nothing relevant. Skip Line.
else:
i += 1
# Set String
- elif 'set-string' in data[i]:
+ elif "set-string" in data[i]:
# Lines
regex = r'(.*?)"(.*)"'
match = re.search(regex, data[i])
# Grab Strings
- if match != None and match.group(2) != '':
+ if match != None and match.group(2) != "":
originalString = match.group(2)
jaString = match.group(2)
currentGroup = [jaString]
-
+
# Remove Textwrap
- jaString = jaString.replace('\\n', ' ')
-
+ jaString = jaString.replace("\\n", " ")
+
# Pass 1
if translatedList == []:
# Add String
stringList.append(jaString.strip())
-
+
# Pass 2
else:
# Get Text
@@ -284,11 +322,11 @@ def translateEushully(data, pbar, filename, translatedList):
# Textwrap
translatedText = textwrap.fill(translatedText, width=LISTWIDTH)
- translatedText = translatedText.replace('\n', '\\n')
+ translatedText = translatedText.replace("\n", "\\n")
# Set Data
- data[i] = data[i].replace(originalString, translatedText)
- speaker = ''
+ data[i] = data[i].replace(originalString, translatedText)
+ speaker = ""
i += 1
# Nothing relevant. Skip Line.
@@ -302,10 +340,10 @@ def translateEushully(data, pbar, filename, translatedList):
# Set Progress
pbar.total = len(stringList)
pbar.refresh()
-
+
# Translate
PBAR = pbar
- response = translateGPT(stringList, '', True)
+ response = translateGPT(stringList, "", True)
tokens[0] += response[1][0]
tokens[1] += response[1][1]
translatedList = response[0]
@@ -321,140 +359,143 @@ def translateEushully(data, pbar, filename, translatedList):
MISMATCH.append(filename)
return tokens
+
# Save some money and enter the character before translation
def getSpeaker(speaker):
match speaker:
- case '1':
- return ['Klaus', [0,0]]
- case '2':
- return ['Helmina', [0,0]]
- case '3':
- return ['Juliana', [0,0]]
- case '4':
- return ['Reginia', [0,0]]
- case '5':
- return ['Luciel', [0,0]]
- case '6':
- return ['Mavislaine', [0,0]]
- case '7':
- return ['Cerouge', [0,0]]
- case '8':
- return ['Maize', [0,0]]
- case '9':
- return ['Elvire', [0,0]]
- case 'a':
- return ['Beatrice', [0,0]]
- case '295':
- return ['Orc', [0,0]]
- case '232':
- return ['Archangel', [0,0]]
- case '238':
- return ['False Juliana', [0,0]]
- case '239':
- return ['False Regina', [0,0]]
- case '23a':
- return ['False Luciel', [0,0]]
- case '23d':
- return ['False Mavislaine', [0,0]]
- case 'cb':
- return ['Olga Niza Kite', [0,0]]
- case 'c9':
- return ['Demon Beast Lupus', [0,0]]
- case 'ca':
- return ['Evelinael', [0,0]]
- case '10':
- return ['Eukleia', [0,0]]
- case '15':
- return ['Lily', [0,0]]
- case '16':
- return ['Kupuko', [0,0]]
- case 'b':
- return ['Ramiel', [0,0]]
- case 'c':
- return ['Henriette', [0,0]]
- case 'd':
- return ['Camilla', [0,0]]
- case 'cc':
- return ['Gogonaua', [0,0]]
- case '65':
- return ['Demon Lord Reyvalois', [0,0]]
- case 'd0':
- return ['Demon Ranwald', [0,0]]
- case '205':
- return ['Vanqueor', [0,0]]
- case '66':
- return ['Angel Martina', [0,0]]
- case '21f':
- return ['Hiten Demon', [0,0]]
- case 'd2':
- return ['Lena Eli', [0,0]]
+ case "1":
+ return ["Klaus", [0, 0]]
+ case "2":
+ return ["Helmina", [0, 0]]
+ case "3":
+ return ["Juliana", [0, 0]]
+ case "4":
+ return ["Reginia", [0, 0]]
+ case "5":
+ return ["Luciel", [0, 0]]
+ case "6":
+ return ["Mavislaine", [0, 0]]
+ case "7":
+ return ["Cerouge", [0, 0]]
+ case "8":
+ return ["Maize", [0, 0]]
+ case "9":
+ return ["Elvire", [0, 0]]
+ case "a":
+ return ["Beatrice", [0, 0]]
+ case "295":
+ return ["Orc", [0, 0]]
+ case "232":
+ return ["Archangel", [0, 0]]
+ case "238":
+ return ["False Juliana", [0, 0]]
+ case "239":
+ return ["False Regina", [0, 0]]
+ case "23a":
+ return ["False Luciel", [0, 0]]
+ case "23d":
+ return ["False Mavislaine", [0, 0]]
+ case "cb":
+ return ["Olga Niza Kite", [0, 0]]
+ case "c9":
+ return ["Demon Beast Lupus", [0, 0]]
+ case "ca":
+ return ["Evelinael", [0, 0]]
+ case "10":
+ return ["Eukleia", [0, 0]]
+ case "15":
+ return ["Lily", [0, 0]]
+ case "16":
+ return ["Kupuko", [0, 0]]
+ case "b":
+ return ["Ramiel", [0, 0]]
+ case "c":
+ return ["Henriette", [0, 0]]
+ case "d":
+ return ["Camilla", [0, 0]]
+ case "cc":
+ return ["Gogonaua", [0, 0]]
+ case "65":
+ return ["Demon Lord Reyvalois", [0, 0]]
+ case "d0":
+ return ["Demon Ranwald", [0, 0]]
+ case "205":
+ return ["Vanqueor", [0, 0]]
+ case "66":
+ return ["Angel Martina", [0, 0]]
+ case "21f":
+ return ["Hiten Demon", [0, 0]]
+ case "d2":
+ return ["Lena Eli", [0, 0]]
case _:
- return ['Unknown', [0,0]]
+ return ["Unknown", [0, 0]]
+
def subVars(jaString):
- jaString = jaString.replace('\u3000', ' ')
+ jaString = jaString.replace("\u3000", " ")
# Nested
count = 0
- nestedList = re.findall(r'[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]', jaString)
+ nestedList = re.findall(r"[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]", jaString)
nestedList = set(nestedList)
if len(nestedList) != 0:
for icon in nestedList:
- jaString = jaString.replace(icon, '[Nested_' + str(count) + ']')
+ jaString = jaString.replace(icon, "[Nested_" + str(count) + "]")
count += 1
# Icons
count = 0
- iconList = re.findall(r'[\\]+[iIkKwWaA]+\[[0-9]+\]', jaString)
+ iconList = re.findall(r"[\\]+[iIkKwWaA]+\[[0-9]+\]", jaString)
iconList = set(iconList)
if len(iconList) != 0:
for icon in iconList:
- jaString = jaString.replace(icon, '[Ascii_' + str(count) + ']')
+ jaString = jaString.replace(icon, "[Ascii_" + str(count) + "]")
count += 1
# Colors
count = 0
- colorList = re.findall(r'[\\]+[cC]\[[0-9]+\]', jaString)
+ colorList = re.findall(r"[\\]+[cC]\[[0-9]+\]", jaString)
colorList = set(colorList)
if len(colorList) != 0:
for color in colorList:
- jaString = jaString.replace(color, '[Color_' + str(count) + ']')
+ jaString = jaString.replace(color, "[Color_" + str(count) + "]")
count += 1
# Names
count = 0
- nameList = re.findall(r'[\\]+[nN]\[.+?\]+', jaString)
+ nameList = re.findall(r"[\\]+[nN]\[.+?\]+", jaString)
nameList = set(nameList)
if len(nameList) != 0:
for name in nameList:
- jaString = jaString.replace(name, '[Noun_' + str(count) + ']')
+ jaString = jaString.replace(name, "[Noun_" + str(count) + "]")
count += 1
# Variables
count = 0
- varList = re.findall(r'[\\]+[vV]\[[0-9]+\]', jaString)
+ varList = re.findall(r"[\\]+[vV]\[[0-9]+\]", jaString)
varList = set(varList)
if len(varList) != 0:
for var in varList:
- jaString = jaString.replace(var, '[Var_' + str(count) + ']')
+ jaString = jaString.replace(var, "[Var_" + str(count) + "]")
count += 1
# Formatting
count = 0
- formatList = re.findall(r'[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]', jaString)
+ formatList = re.findall(r"[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]", jaString)
formatList = set(formatList)
if len(formatList) != 0:
for var in formatList:
- jaString = jaString.replace(var, '[FCode_' + str(count) + ']')
+ jaString = jaString.replace(var, "[FCode_" + str(count) + "]")
count += 1
# Put all lists in list and return
allList = [nestedList, iconList, colorList, nameList, varList, formatList]
return [jaString, allList]
+
def resubVars(translatedText, allList):
# Fix Spacing and ChatGPT Nonsense
- matchList = re.findall(r'\[\s?.+?\s?\]', translatedText)
+ matchList = re.findall(r"\[\s?.+?\s?\]", translatedText)
if len(matchList) > 0:
for match in matchList:
text = match.strip()
@@ -464,59 +505,65 @@ def resubVars(translatedText, allList):
count = 0
if len(allList[0]) != 0:
for var in allList[0]:
- translatedText = translatedText.replace('[Nested_' + str(count) + ']', var)
+ translatedText = translatedText.replace("[Nested_" + str(count) + "]", var)
count += 1
# Icons
count = 0
if len(allList[1]) != 0:
for var in allList[1]:
- translatedText = translatedText.replace('[Ascii_' + str(count) + ']', var)
+ translatedText = translatedText.replace("[Ascii_" + str(count) + "]", var)
count += 1
# Colors
count = 0
if len(allList[2]) != 0:
for var in allList[2]:
- translatedText = translatedText.replace('[Color_' + str(count) + ']', var)
+ translatedText = translatedText.replace("[Color_" + str(count) + "]", var)
count += 1
# Names
count = 0
if len(allList[3]) != 0:
for var in allList[3]:
- translatedText = translatedText.replace('[Noun_' + str(count) + ']', var)
+ translatedText = translatedText.replace("[Noun_" + str(count) + "]", var)
count += 1
# Vars
count = 0
if len(allList[4]) != 0:
for var in allList[4]:
- translatedText = translatedText.replace('[Var_' + str(count) + ']', var)
+ translatedText = translatedText.replace("[Var_" + str(count) + "]", var)
count += 1
-
+
# Formatting
count = 0
if len(allList[5]) != 0:
for var in allList[5]:
- translatedText = translatedText.replace('[FCode_' + str(count) + ']', var)
+ translatedText = translatedText.replace("[FCode_" + str(count) + "]", var)
count += 1
return translatedText
+
def batchList(input_list, batch_size):
if not isinstance(batch_size, int) or batch_size <= 0:
raise ValueError("batch_size must be a positive integer")
-
- return [input_list[i:i + batch_size] for i in range(0, len(input_list), batch_size)]
+
+ return [
+ input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)
+ ]
+
def createContext(fullPromptFlag, subbedT):
- characters = 'Game Characters:\n\
+ characters = "Game Characters:\n\
グレイス (Grace) - Female\n\
-'
-
- system = PROMPT + VOCAB if fullPromptFlag else \
- f"\
+"
+
+ system = (
+ PROMPT + VOCAB
+ if fullPromptFlag
+ else f"\
You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
- You translate everything, including erotic and explicit content.\n\
@@ -528,9 +575,11 @@ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{
- `...` can be a part of the dialogue. Translate it as it is.\n\
{VOCAB}\n\
"
- user = f'{subbedT}'
+ )
+ user = f"{subbedT}"
return characters, system, user
+
def translateText(characters, system, user, history, penalty):
# Prompt
msg = [{"role": "system", "content": system + characters}]
@@ -543,9 +592,9 @@ def translateText(characters, system, user, history, penalty):
msg.extend([{"role": "system", "content": h} for h in history])
else:
msg.append({"role": "system", "content": history})
-
+
# Content to TL
- msg.append({"role": "user", "content": f'{user}'})
+ msg.append({"role": "user", "content": f"{user}"})
response = openai.chat.completions.create(
temperature=0,
frequency_penalty=penalty,
@@ -554,23 +603,24 @@ def translateText(characters, system, user, history, penalty):
)
return response
+
def cleanTranslatedText(translatedText, varResponse):
placeholders = {
- f'{LANGUAGE} Translation: ': '',
- 'Translation: ': '',
- 'っ': '',
- '〜': '~',
- 'ッ': '',
- '。': '.',
- '< ': '<',
- '': '>',
- '「': '\"',
- '」': '\"',
- 'Placeholder Text': '',
- '- chan': '-chan',
- '- kun': '-kun',
- '- san': '-san',
+ f"{LANGUAGE} Translation: ": "",
+ "Translation: ": "",
+ "っ": "",
+ "〜": "~",
+ "ッ": "",
+ "。": ".",
+ "< ": "<",
+ "": ">",
+ "「": '"',
+ "」": '"',
+ "Placeholder Text": "",
+ "- chan": "-chan",
+ "- kun": "-kun",
+ "- san": "-san",
# Add more replacements as needed
}
for target, replacement in placeholders.items():
@@ -581,11 +631,12 @@ def cleanTranslatedText(translatedText, varResponse):
translatedText = resubVars(translatedText, varResponse[1])
return translatedText
+
def elongateCharacters(text):
# Define a pattern to match one character followed by one or more `ー` characters
# Using a positive lookbehind assertion to capture the preceding character
- pattern = r'(?<=(.))ー+'
-
+ pattern = r"(?<=(.))ー+"
+
# Define a replacement function that elongates the captured character
def repl(match):
char = match.group(1) # The character before the ー sequence
@@ -595,8 +646,9 @@ def elongateCharacters(text):
# Use re.sub() to replace the pattern in the text
return re.sub(pattern, repl, text)
+
def extractTranslation(translatedTextList, is_list):
- pattern = r'`?<[Ll]ine\d+>([\\]*.*?[\\]*?)<\/?[Ll]ine\d+>`?'
+ pattern = r"`?<[Ll]ine\d+>([\\]*.*?[\\]*?)<\/?[Ll]ine\d+>`?"
# If it's a batch (i.e., list), extract with tags; otherwise, return the single item.
if is_list:
matchList = re.findall(pattern, translatedTextList)
@@ -605,11 +657,12 @@ def extractTranslation(translatedTextList, is_list):
matchList = re.findall(pattern, translatedTextList)
return matchList[0][0] if matchList else translatedTextList
+
def countTokens(characters, system, user, history):
inputTotalTokens = 0
outputTotalTokens = 0
- enc = tiktoken.encoding_for_model('gpt-4')
-
+ enc = tiktoken.encoding_for_model("gpt-4")
+
# Input
if isinstance(history, list):
for line in history:
@@ -621,19 +674,21 @@ def countTokens(characters, system, user, history):
inputTotalTokens += len(enc.encode(user))
# Output
- outputTotalTokens += round(len(enc.encode(user))*3)
+ outputTotalTokens += round(len(enc.encode(user)) * 3)
return [inputTotalTokens, outputTotalTokens]
+
def combineList(tlist, text):
if isinstance(text, list):
return [t for sublist in tlist for t in sublist]
return tlist[0]
+
@retry(exceptions=Exception, tries=5, delay=5)
def translateGPT(text, history, fullPromptFlag):
global PBAR
-
+
mismatch = False
totalTokens = [0, 0]
if isinstance(text, list):
@@ -644,8 +699,12 @@ def translateGPT(text, history, fullPromptFlag):
for index, tItem in enumerate(tList):
# Before sending to translation, if we have a list of items, add the formatting
if isinstance(tItem, list):
- payload = '\n'.join([f'`{item}`' for i, item in enumerate(tItem)])
- payload = re.sub(r'(<)(\/Line\d+>)', r'\1>Placeholder Text<\3', payload)
+ payload = "\n".join(
+ [f"`{item}`" for i, item in enumerate(tItem)]
+ )
+ payload = re.sub(
+ r"(<)(\/Line\d+>)", r"\1>Placeholder Text<\3", payload
+ )
varResponse = subVars(payload)
subbedT = varResponse[0]
else:
@@ -653,7 +712,7 @@ def translateGPT(text, history, fullPromptFlag):
subbedT = varResponse[0]
# Things to Check before starting translation
- if not re.search(r'[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+', subbedT):
+ if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", subbedT):
if PBAR is not None:
PBAR.update(len(tItem))
continue
@@ -692,14 +751,16 @@ def translateGPT(text, history, fullPromptFlag):
extractedTranslations = extractTranslation(translatedText, True)
tList[index] = extractedTranslations
if len(tItem) != len(extractedTranslations):
- mismatch = True # Just here for breakpoint
+ mismatch = True # Just here for breakpoint
# Create History
with LOCK:
if PBAR is not None:
PBAR.update(len(tItem))
if not mismatch:
- history = extractedTranslations[-10:] # Update history if we have a list
+ history = extractedTranslations[
+ -10:
+ ] # Update history if we have a list
else:
history = text[-10:]
else:
diff --git a/modules/images.py b/modules/images.py
index 2f70a13..7177878 100644
--- a/modules/images.py
+++ b/modules/images.py
@@ -14,72 +14,77 @@ from dotenv import load_dotenv
from retry import retry
from tqdm import tqdm
-#Globals
-MODEL = os.getenv('model')
-TIMEOUT = int(os.getenv('timeout'))
-LANGUAGE = os.getenv('language').capitalize()
-PROMPT = Path('prompt.txt').read_text(encoding='utf-8')
-VOCAB = Path('vocab.txt').read_text(encoding='utf-8')
-THREADS = int(os.getenv('threads'))
+# Globals
+MODEL = os.getenv("model")
+TIMEOUT = int(os.getenv("timeout"))
+LANGUAGE = os.getenv("language").capitalize()
+PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
+VOCAB = Path("vocab.txt").read_text(encoding="utf-8")
+THREADS = int(os.getenv("threads"))
LOCK = threading.Lock()
PBAR = None
-WIDTH = int(os.getenv('width'))
-LISTWIDTH = int(os.getenv('listWidth'))
-NOTEWIDTH = int(os.getenv('noteWidth'))
+WIDTH = int(os.getenv("width"))
+LISTWIDTH = int(os.getenv("listWidth"))
+NOTEWIDTH = int(os.getenv("noteWidth"))
MAXHISTORY = 10
-ESTIMATE = ''
+ESTIMATE = ""
TOKENS = [0, 0]
NAMESLIST = []
-MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
+MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
# Open AI
load_dotenv()
-if os.getenv('api').replace(' ', '') != '':
- openai.base_url = os.getenv('api')
-openai.organization = os.getenv('org')
-openai.api_key = os.getenv('key')
+if os.getenv("api").replace(" ", "") != "":
+ openai.base_url = os.getenv("api")
+openai.organization = os.getenv("org")
+openai.api_key = os.getenv("key")
# Pricing - Depends on the model https://openai.com/pricing
# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request
# If you are getting a MISMATCH LENGTH error, lower the batch size.
-if 'gpt-3.5' in MODEL:
- INPUTAPICOST = .002
- OUTPUTAPICOST = .002
+if "gpt-3.5" in MODEL:
+ INPUTAPICOST = 0.002
+ OUTPUTAPICOST = 0.002
BATCHSIZE = 10
FREQUENCY_PENALTY = 0.2
-elif 'gpt-4' in MODEL:
- INPUTAPICOST = .005
- OUTPUTAPICOST = .015
+elif "gpt-4" in MODEL:
+ INPUTAPICOST = 0.005
+ OUTPUTAPICOST = 0.015
BATCHSIZE = 20
FREQUENCY_PENALTY = 0.1
-#tqdm Globals
-BAR_FORMAT='{l_bar}{bar:10}{r_bar}{bar:-10b}'
+# tqdm Globals
+BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
POSITION = 0
LEAVE = False
+
def handleImages(folderName, estimate):
global ESTIMATE, TOKENS
ESTIMATE = estimate
start = time.time()
# Translate Strings
- translatedData = openFiles(f'files/{folderName}')
+ translatedData = openFiles(f"files/{folderName}")
# Write Strings to Images
if not ESTIMATE:
- if not os.path.exists(f'translated/{folderName}'):
- os.mkdir(f'translated/{folderName}')
+ if not os.path.exists(f"translated/{folderName}"):
+ os.mkdir(f"translated/{folderName}")
for i in range(len(translatedData[0][0])):
try:
translatedList = translatedData[0][0]
originalList = translatedData[0][1]
dimensionsList = translatedData[0][2]
- image = stringToImage(translatedList[i], dimensionsList[i][0], dimensionsList[i][1])
- image.save(rf'translated/{folderName}/{translatedList[i]}.png', quality=100)
+ image = stringToImage(
+ translatedList[i], dimensionsList[i][0], dimensionsList[i][1]
+ )
+ image.save(
+ rf"translated/{folderName}/{translatedList[i]}.png", quality=100
+ )
except Exception as e:
- PBAR.write(f'{translatedList[i]}: {str(e)}')
- #Ignore Error
+ PBAR.write(f"{translatedList[i]}: {str(e)}")
+ # Ignore Error
# Print File
end = time.time()
@@ -89,43 +94,67 @@ def handleImages(folderName, estimate):
TOKENS[1] += translatedData[1][1]
# Print Total
- totalString = getResultString(['', TOKENS, None], end - start, 'TOTAL')
+ totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
# Print any errors on maps
if len(MISMATCH) > 0:
- return totalString + Fore.RED + f'\nMismatch Errors: {MISMATCH}' + Fore.RESET
+ return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
else:
return totalString
-
+
+
def openFiles(folderName):
global PBAR
if os.path.isdir(folderName):
- imageList = [[],[]]
+ imageList = [[], []]
imageList = processImagesDir(folderName, imageList)
-
+
# Start Translation
- with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE, desc=folderName, total=len(imageList[0])) as PBAR:
+ with tqdm(
+ bar_format=BAR_FORMAT,
+ position=POSITION,
+ leave=LEAVE,
+ desc=folderName,
+ total=len(imageList[0]),
+ ) as PBAR:
translatedData = translateImages(imageList)
- translatedData = [[translatedData[0], imageList[0], imageList[1]], translatedData[1], translatedData[2]]
-
+ translatedData = [
+ [translatedData[0], imageList[0], imageList[1]],
+ translatedData[1],
+ translatedData[2],
+ ]
+
return translatedData
else:
print("The provided directory path does not exist.")
+
def getResultString(translatedData, translationTime, filename):
# File Print String
- totalTokenstring =\
- Fore.YELLOW +\
- '[Input: ' + str(translatedData[1][0]) + ']'\
- '[Output: ' + str(translatedData[1][1]) + ']'\
- '[Cost: ${:,.4f}'.format((translatedData[1][0] * .001 * INPUTAPICOST) +\
- (translatedData[1][1] * .001 * OUTPUTAPICOST)) + ']'
- timeString = Fore.BLUE + '[' + str(round(translationTime, 1)) + 's]'
+ totalTokenstring = (
+ Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
+ "[Output: "
+ + str(translatedData[1][1])
+ + "]" "[Cost: ${:,.4f}".format(
+ (translatedData[1][0] * 0.001 * INPUTAPICOST)
+ + (translatedData[1][1] * 0.001 * OUTPUTAPICOST)
+ )
+ + "]"
+ )
+ timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
if translatedData[2] is None:
# Success
- return filename + ': ' + totalTokenstring + timeString + Fore.GREEN + u' \u2713 ' + Fore.RESET
+ return (
+ filename
+ + ": "
+ + totalTokenstring
+ + timeString
+ + Fore.GREEN
+ + " \u2713 "
+ + Fore.RESET
+ )
else:
# Fail
try:
@@ -133,8 +162,17 @@ def getResultString(translatedData, translationTime, filename):
except Exception as e:
traceback.print_exc()
errorString = str(e) + Fore.RED
- return filename + ': ' + totalTokenstring + timeString + Fore.RED + u' \u2717 ' +\
- errorString + Fore.RESET
+ return (
+ filename
+ + ": "
+ + totalTokenstring
+ + timeString
+ + Fore.RED
+ + " \u2717 "
+ + errorString
+ + Fore.RESET
+ )
+
def getFontSize(text, image_width, image_height, font_path):
# Start with a high font size and keep reducing it until the text fits within the image bounds
@@ -142,50 +180,59 @@ def getFontSize(text, image_width, image_height, font_path):
while font_size > 0:
font = ImageFont.truetype(font_path, font_size)
- text_bbox = ImageDraw.Draw(Image.new('RGB', (1, 1))).textbbox((0, 0), text, font=font)
+ text_bbox = ImageDraw.Draw(Image.new("RGB", (1, 1))).textbbox(
+ (0, 0), text, font=font
+ )
text_width = text_bbox[2] - text_bbox[0]
text_height = text_bbox[3] - text_bbox[1] + 5
-
+
if text_width <= image_width and text_height <= image_height:
return font_size
font_size -= 1
return font_size
-def stringToImage(text, width, height, font_path='fonts/TsunagiGothic.ttf', scale_factor=4):
+
+def stringToImage(
+ text, width, height, font_path="fonts/TsunagiGothic.ttf", scale_factor=4
+):
# Increase the resolution
scaled_width = int(width * scale_factor)
scaled_height = int(height * scale_factor)
-
+
# Find the appropriate font size for the scaled up image
font_size = getFontSize(text, scaled_width, scaled_height, font_path)
if font_size == 0:
raise ValueError("Text is too long to fit in the supplied dimensions.")
-
+
# Create a new image with the scaled width and height and a transparent background
- image = Image.new('RGBA', (scaled_width, scaled_height), (255, 255, 255, 0))
-
+ image = Image.new("RGBA", (scaled_width, scaled_height), (255, 255, 255, 0))
+
# Create a drawing context
draw = ImageDraw.Draw(image)
-
+
# Load the appropriate font
font = ImageFont.truetype(font_path, font_size)
-
+
# Calculate the size of the text to center it
text_bbox = draw.textbbox((0, 0), text, font=font)
text_width = text_bbox[2] - text_bbox[0]
text_height = text_bbox[3] - text_bbox[1]
x = (scaled_width - text_width) // 2
y = (scaled_height - text_height) // 2
-
+
# Draw the text on the image
draw.text((x, y), text, font=font, fill=(255, 255, 255, 255))
-
+
# Resize back to the original dimensions to get a clearer text rendering
- image = image.resize((width, height), Image.LANCZOS,)
-
+ image = image.resize(
+ (width, height),
+ Image.LANCZOS,
+ )
+
return image
+
def getImageDimensions(file_path):
try:
with Image.open(file_path) as img:
@@ -195,10 +242,11 @@ def getImageDimensions(file_path):
print(f"Error reading {file_path}: {e}")
return None, None
+
def processImagesDir(directory_path, imageList):
for file_name in os.listdir(directory_path):
# .png and Japanese
- if '.png' in file_name and file_name.replace('.png', '') in VOCAB:
+ if ".png" in file_name and file_name.replace(".png", "") in VOCAB:
file_path = os.path.join(directory_path, file_name)
if os.path.isfile(file_path):
# Check if the file is an image
@@ -206,7 +254,7 @@ def processImagesDir(directory_path, imageList):
width, height = getImageDimensions(file_path)
if width is not None and height is not None:
placeholders = {
- '.png': '',
+ ".png": "",
}
for target, replacement in placeholders.items():
file_name = file_name.replace(target, replacement)
@@ -214,37 +262,49 @@ def processImagesDir(directory_path, imageList):
imageList[1].append([width, height])
except Exception as e:
print(f"Error processing {file_name}: {e}")
-
+
return imageList
+
def translateImages(imageList):
- totalTokens = [0,0]
+ totalTokens = [0, 0]
# Translate GPT
- response = translateGPT(imageList[0], 'Keep the Translation as brief as possible', True)
+ response = translateGPT(
+ imageList[0], "Keep the Translation as brief as possible", True
+ )
translatedList = response[0]
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
- return [translatedList, totalTokens, None]
+ return [translatedList, totalTokens, None]
+
# Save some money and enter the character before translation
def getSpeaker(speaker):
match speaker:
- case 'ファイン':
- return ['Fine', [0,0]]
- case '':
- return ['', [0,0]]
+ case "ファイン":
+ return ["Fine", [0, 0]]
+ case "":
+ return ["", [0, 0]]
case _:
# Store Speaker
if speaker not in str(NAMESLIST):
- response = translateGPT(speaker, 'Reply with the '+ LANGUAGE +' translation of the NPC name.', False)
+ response = translateGPT(
+ speaker,
+ "Reply with the " + LANGUAGE + " translation of the NPC name.",
+ False,
+ )
response[0] = response[0].title()
response[0] = response[0].replace("'S", "'s")
# Retry if name doesn't translate for some reason
- if re.search(r'([a-zA-Z？?])', response[0]) == None:
- response = translateGPT(speaker, 'Reply with the '+ LANGUAGE +' translation of the NPC name.', False)
+ if re.search(r"([a-zA-Z？?])", response[0]) == None:
+ response = translateGPT(
+ speaker,
+ "Reply with the " + LANGUAGE + " translation of the NPC name.",
+ False,
+ )
response[0] = response[0].title()
response[0] = response[0].replace("'S", "'s")
@@ -255,50 +315,56 @@ def getSpeaker(speaker):
else:
for i in range(len(NAMESLIST)):
if speaker == NAMESLIST[i][0]:
- return [NAMESLIST[i][1],[0,0]]
-
- return [speaker,[0,0]]
+ return [NAMESLIST[i][1], [0, 0]]
+
+ return [speaker, [0, 0]]
+
def subVars(jaString):
- jaString = jaString.replace('\u3000', ' ')
+ jaString = jaString.replace("\u3000", " ")
# Formatting
count = 0
- codeList = re.findall(r'[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]', jaString)
+ codeList = re.findall(r"[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]", jaString)
codeList = set(codeList)
if len(codeList) != 0:
for var in codeList:
- jaString = jaString.replace(var, '[FCode_' + str(count) + ']')
+ jaString = jaString.replace(var, "[FCode_" + str(count) + "]")
count += 1
# Put all lists in list and return
return [jaString, codeList]
+
def resubVars(translatedText, codeList):
# Fix Spacing and ChatGPT Nonsense
- matchList = re.findall(r'\[\s?.+?\s?\]', translatedText)
+ matchList = re.findall(r"\[\s?.+?\s?\]", translatedText)
if len(matchList) > 0:
for match in matchList:
text = match.strip()
translatedText = translatedText.replace(match, text)
-
+
# Formatting
count = 0
if len(codeList) != 0:
for var in codeList:
- translatedText = translatedText.replace('[FCode_' + str(count) + ']', var)
+ translatedText = translatedText.replace("[FCode_" + str(count) + "]", var)
count += 1
return translatedText
+
def batchList(input_list, batch_size):
if not isinstance(batch_size, int) or batch_size <= 0:
raise ValueError("batch_size must be a positive integer")
-
- return [input_list[i:i + batch_size] for i in range(0, len(input_list), batch_size)]
+
+ return [
+ input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)
+ ]
+
def createContext(fullPromptFlag, subbedT, format):
- characters = 'Game Characters:\n\
+ characters = "Game Characters:\n\
ロラン (Roland) - Male\n\
リュカ (Ryuka) - Male\n\
レックス (Rex) - Male\n\
@@ -341,10 +407,12 @@ def createContext(fullPromptFlag, subbedT, format):
ブライ (Buraimu) - Male\n\
ハッサン (Hassan) - Male\n\
アロマ (Aroma) - Female\n\
-'
-
- system = PROMPT + VOCAB if fullPromptFlag else \
- f"\
+"
+
+ system = (
+ PROMPT + VOCAB
+ if fullPromptFlag
+ else f"\
You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
- You translate everything, including erotic and explicit content.\n\
@@ -356,12 +424,14 @@ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{
- `...` can be a part of the dialogue. Translate it as it is.\n\
{VOCAB}\n\
"
- if format == 'json':
- user = f'```json\n{subbedT}\n```'
+ )
+ if format == "json":
+ user = f"```json\n{subbedT}\n```"
else:
user = subbedT
return characters, system, user
+
def translateText(characters, system, user, history, penalty, format):
# Prompt
msg = [{"role": "system", "content": system + characters}]
@@ -376,13 +446,13 @@ def translateText(characters, system, user, history, penalty, format):
msg.append({"role": "system", "content": history})
# Response Format
- if format == 'json':
- responseFormat = { "type": "json_object" }
+ if format == "json":
+ responseFormat = {"type": "json_object"}
else:
- responseFormat = { "type": "text" }
-
+ responseFormat = {"type": "text"}
+
# Content to TL
- msg.append({"role": "user", "content": f'{user}'})
+ msg.append({"role": "user", "content": f"{user}"})
response = openai.chat.completions.create(
temperature=0,
frequency_penalty=penalty,
@@ -392,18 +462,19 @@ def translateText(characters, system, user, history, penalty, format):
)
return response
+
def cleanTranslatedText(translatedText, varResponse):
placeholders = {
- f'{LANGUAGE} Translation: ': '',
- 'Translation: ': '',
- 'っ': '',
- '〜': '~',
- 'ッ': '',
- '。': '.',
- '「': '\\"',
- '」': '\\"',
- '- ': '-',
- 'Placeholder Text': '',
+ f"{LANGUAGE} Translation: ": "",
+ "Translation: ": "",
+ "っ": "",
+ "〜": "~",
+ "ッ": "",
+ "。": ".",
+ "「": '\\"',
+ "」": '\\"',
+ "- ": "-",
+ "Placeholder Text": "",
# Add more replacements as needed
}
for target, replacement in placeholders.items():
@@ -414,11 +485,12 @@ def cleanTranslatedText(translatedText, varResponse):
translatedText = resubVars(translatedText, varResponse[1])
return translatedText
+
def elongateCharacters(text):
# Define a pattern to match one character followed by one or more `ー` characters
# Using a positive lookbehind assertion to capture the preceding character
- pattern = r'(?<=(.))ー+'
-
+ pattern = r"(?<=(.))ー+"
+
# Define a replacement function that elongates the captured character
def repl(match):
char = match.group(1) # The character before the ー sequence
@@ -428,6 +500,7 @@ def elongateCharacters(text):
# Use re.sub() to replace the pattern in the text
return re.sub(pattern, repl, text)
+
def extractTranslation(translatedTextList, is_list):
try:
line_dict = json.loads(translatedTextList)
@@ -439,15 +512,15 @@ def extractTranslation(translatedTextList, is_list):
return string_list[0]
except Exception as e:
- print(f'extractTranslation Error: {e}')
+ print(f"extractTranslation Error: {e}")
return None
def countTokens(characters, system, user, history):
inputTotalTokens = 0
outputTotalTokens = 0
- enc = tiktoken.encoding_for_model('gpt-4')
-
+ enc = tiktoken.encoding_for_model("gpt-4")
+
# Input
if isinstance(history, list):
for line in history:
@@ -459,26 +532,28 @@ def countTokens(characters, system, user, history):
inputTotalTokens += len(enc.encode(user))
# Output
- outputTotalTokens += round(len(enc.encode(user))*3)
+ outputTotalTokens += round(len(enc.encode(user)) * 3)
return [inputTotalTokens, outputTotalTokens]
+
def combineList(tlist, text):
if isinstance(text, list):
return [t for sublist in tlist for t in sublist]
return tlist[0]
+
@retry(exceptions=Exception, tries=5, delay=5)
def translateGPT(text, history, fullPromptFlag):
global PBAR
-
+
mismatch = False
totalTokens = [0, 0]
if isinstance(text, list):
- format = 'json'
+ format = "json"
tList = batchList(text, BATCHSIZE)
else:
- format = 'text'
+ format = "text"
tList = [text]
for index, tItem in enumerate(tList):
@@ -493,7 +568,7 @@ def translateGPT(text, history, fullPromptFlag):
subbedT = varResponse[0]
# Things to Check before starting translation
- if not re.search(r'[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+', subbedT):
+ if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", subbedT):
if PBAR is not None:
PBAR.update(len(tItem))
continue
@@ -518,9 +593,13 @@ def translateGPT(text, history, fullPromptFlag):
translatedText = cleanTranslatedText(translatedText, varResponse)
if isinstance(tItem, list):
extractedTranslations = extractTranslation(translatedText, True)
- if extractedTranslations == None or len(tItem) != len(extractedTranslations):
+ if extractedTranslations == None or len(tItem) != len(
+ extractedTranslations
+ ):
# Mismatch. Try Again
- response = translateText(characters, system, user, history, 0.05, format)
+ response = translateText(
+ characters, system, user, history, 0.05, format
+ )
translatedText = response.choices[0].message.content
totalTokens[0] += response.usage.prompt_tokens
totalTokens[1] += response.usage.completion_tokens
@@ -529,13 +608,17 @@ def translateGPT(text, history, fullPromptFlag):
translatedText = cleanTranslatedText(translatedText, varResponse)
if isinstance(tItem, list):
extractedTranslations = extractTranslation(translatedText, True)
- if extractedTranslations == None or len(tItem) != len(extractedTranslations):
- mismatch = True # Just here for breakpoint
-
+ if extractedTranslations == None or len(tItem) != len(
+ extractedTranslations
+ ):
+ mismatch = True # Just here for breakpoint
+
# Set if no mismatch
if mismatch == False:
tList[index] = extractedTranslations
- history = extractedTranslations[-10:] # Update history if we have a list
+ history = extractedTranslations[
+ -10:
+ ] # Update history if we have a list
else:
history = text[-10:]
mismatch = False
@@ -546,7 +629,7 @@ def translateGPT(text, history, fullPromptFlag):
PBAR.update(len(tItem))
else:
# Ensure we're passing a single string to extractTranslation
- tList[index] = translatedText
+ tList[index] = translatedText
finalList = combineList(tList, text)
return [finalList, totalTokens]
diff --git a/modules/irissoft.py b/modules/irissoft.py
index 733d081..20759be 100644
--- a/modules/irissoft.py
+++ b/modules/irissoft.py
@@ -15,49 +15,50 @@ from tqdm import tqdm
# Open AI
load_dotenv()
-if os.getenv('api').replace(' ', '') != '':
- openai.base_url = os.getenv('api')
-openai.organization = os.getenv('org')
-openai.api_key = os.getenv('key')
+if os.getenv("api").replace(" ", "") != "":
+ openai.base_url = os.getenv("api")
+openai.organization = os.getenv("org")
+openai.api_key = os.getenv("key")
-#Globals
-MODEL = os.getenv('model')
-TIMEOUT = int(os.getenv('timeout'))
-LANGUAGE = os.getenv('language').capitalize()
-PROMPT = Path('prompt.txt').read_text(encoding='utf-8')
-VOCAB = Path('vocab.txt').read_text(encoding='utf-8')
-THREADS = int(os.getenv('threads'))
+# Globals
+MODEL = os.getenv("model")
+TIMEOUT = int(os.getenv("timeout"))
+LANGUAGE = os.getenv("language").capitalize()
+PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
+VOCAB = Path("vocab.txt").read_text(encoding="utf-8")
+THREADS = int(os.getenv("threads"))
LOCK = threading.Lock()
-WIDTH = int(os.getenv('width'))
-LISTWIDTH = int(os.getenv('listWidth'))
+WIDTH = int(os.getenv("width"))
+LISTWIDTH = int(os.getenv("listWidth"))
NOTEWIDTH = 70
MAXHISTORY = 10
-ESTIMATE = ''
+ESTIMATE = ""
TOKENS = [0, 0]
NAMESLIST = []
-NAMES = False # Output a list of all the character names found
-BRFLAG = False # If the game uses
instead
+NAMES = False # Output a list of all the character names found
+BRFLAG = False # If the game uses
instead
FIXTEXTWRAP = True # Overwrites textwrap
-IGNORETLTEXT = False # Ignores all translated text.
-MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
+IGNORETLTEXT = False # Ignores all translated text.
+MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
-#tqdm Globals
-BAR_FORMAT='{l_bar}{bar:10}{r_bar}{bar:-10b}'
+# tqdm Globals
+BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
POSITION = 0
LEAVE = False
# Pricing - Depends on the model https://openai.com/pricing
# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request
# If you are getting a MISMATCH LENGTH error, lower the batch size.
-if 'gpt-3.5' in MODEL:
- INPUTAPICOST = .002
- OUTPUTAPICOST = .002
+if "gpt-3.5" in MODEL:
+ INPUTAPICOST = 0.002
+ OUTPUTAPICOST = 0.002
BATCHSIZE = 10
-elif 'gpt-4' in MODEL:
- INPUTAPICOST = .005
- OUTPUTAPICOST = .015
+elif "gpt-4" in MODEL:
+ INPUTAPICOST = 0.005
+ OUTPUTAPICOST = 0.015
BATCHSIZE = 40
+
def handleIris(filename, estimate):
global ESTIMATE
ESTIMATE = estimate
@@ -74,17 +75,21 @@ def handleIris(filename, estimate):
TOKENS[1] += translatedData[1][1]
# Print Total
- totalString = getResultString(['', TOKENS, None], end - start, 'TOTAL')
+ totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
# Print any errors on maps
if len(MISMATCH) > 0:
- return totalString + Fore.RED + f'\nMismatch Errors: {MISMATCH}' + Fore.RESET
+ return (
+ totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
+ )
else:
return totalString
-
+
else:
try:
- with open('translated/' + filename, 'w', encoding='cp932', errors='ignore') as outFile:
+ with open(
+ "translated/" + filename, "w", encoding="cp932", errors="ignore"
+ ) as outFile:
start = time.time()
translatedData = openFiles(filename)
@@ -97,23 +102,36 @@ def handleIris(filename, estimate):
TOKENS[1] += translatedData[1][1]
except Exception:
traceback.print_exc()
- return 'Fail'
+ return "Fail"
+
+ return getResultString(["", TOKENS, None], end - start, "TOTAL")
- return getResultString(['', TOKENS, None], end - start, 'TOTAL')
def getResultString(translatedData, translationTime, filename):
# File Print String
- totalTokenstring =\
- Fore.YELLOW +\
- '[Input: ' + str(translatedData[1][0]) + ']'\
- '[Output: ' + str(translatedData[1][1]) + ']'\
- '[Cost: ${:,.4f}'.format((translatedData[1][0] * .001 * INPUTAPICOST) +\
- (translatedData[1][1] * .001 * OUTPUTAPICOST)) + ']'
- timeString = Fore.BLUE + '[' + str(round(translationTime, 1)) + 's]'
+ totalTokenstring = (
+ Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
+ "[Output: "
+ + str(translatedData[1][1])
+ + "]" "[Cost: ${:,.4f}".format(
+ (translatedData[1][0] * 0.001 * INPUTAPICOST)
+ + (translatedData[1][1] * 0.001 * OUTPUTAPICOST)
+ )
+ + "]"
+ )
+ timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
if translatedData[2] == None:
# Success
- return filename + ': ' + totalTokenstring + timeString + Fore.GREEN + u' \u2713 ' + Fore.RESET
+ return (
+ filename
+ + ": "
+ + totalTokenstring
+ + timeString
+ + Fore.GREEN
+ + " \u2713 "
+ + Fore.RESET
+ )
else:
# Fail
@@ -122,31 +140,41 @@ def getResultString(translatedData, translationTime, filename):
except Exception as e:
traceback.print_exc()
errorString = str(e) + Fore.RED
- return filename + ': ' + totalTokenstring + timeString + Fore.RED + u' \u2717 ' +\
- errorString + Fore.RESET
+ return (
+ filename
+ + ": "
+ + totalTokenstring
+ + timeString
+ + Fore.RED
+ + " \u2717 "
+ + errorString
+ + Fore.RESET
+ )
+
def openFiles(filename):
- with open('files/' + filename, 'r', encoding='shift_jis') as readFile:
+ with open("files/" + filename, "r", encoding="shift_jis") as readFile:
translatedData = parseIris(readFile, filename)
# Delete lines marked for deletion
finalData = []
for line in translatedData[0]:
- if line != '\\d\n':
+ if line != "\\d\n":
finalData.append(line)
translatedData[0] = finalData
-
+
return translatedData
+
def parseIris(readFile, filename):
- totalTokens = [0,0]
+ totalTokens = [0, 0]
# Read File into data
data = readFile.readlines()
# Create Progress Bar
with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
- pbar.desc=filename
+ pbar.desc = filename
try:
result = translateIris(data, pbar, filename, [])
@@ -157,83 +185,87 @@ def parseIris(readFile, filename):
return [data, totalTokens, e]
return [data, totalTokens, None]
+
def translateIris(data, pbar, filename, translatedList):
stringList = []
currentGroup = []
- tokens = [0,0]
- speaker = ''
+ tokens = [0, 0]
+ speaker = ""
voice = False
global LOCK, ESTIMATE
i = 0
while i < len(data):
voice = False
- speaker = ''
- if '#MSGVOICE' in data[i]:
+ speaker = ""
+ if "#MSGVOICE" in data[i]:
i += 1
voice = True
voiceVar = data[i]
- if '#MSG,' in data[i] or '#MSG\n' in data[i] or voice == True:
+ if "#MSG," in data[i] or "#MSG\n" in data[i] or voice == True:
i += 1
# Speaker
- if re.search(r'^ ?([^#\/."、。*!！（）\(\)\[\] \n]+)\n', data[i]) and len(data[i]) < 30:
- match = re.search(r'(.*)', data[i])
+ if (
+ re.search(r'^ ?([^#\/."、。*!！（）\(\)\[\] \n]+)\n', data[i])
+ and len(data[i]) < 30
+ ):
+ match = re.search(r"(.*)", data[i])
if match != None:
speaker = match.group(1)
- if speaker[0] == '\u3000':
+ if speaker[0] == "\u3000":
speaker = speaker[1:]
response = getSpeaker(speaker, pbar, filename)
speaker = response[0]
tokens[0] += response[1][0]
tokens[1] += response[1][1]
if translatedList != []:
- speaker = speaker.replace(' ', '\u3000')
- data[i] = f'\u3000{speaker}\n'
+ speaker = speaker.replace(" ", "\u3000")
+ data[i] = f"\u3000{speaker}\n"
else:
- speaker = ''
+ speaker = ""
i += 1
# Lines
- match = re.search(r'(.*)', data[i])
- if match != None and match.group(1) != '':
+ match = re.search(r"(.*)", data[i])
+ if match != None and match.group(1) != "":
# Pass 1
if translatedList == []:
# Grab Consecutive Strings
jaString = data[i]
- if data[i] != '\n':
- if data[i][0] == '\u3000':
+ if data[i] != "\n":
+ if data[i][0] == "\u3000":
jaString = data[i][1:]
currentGroup.append(jaString)
i += 1
- while data[i] != '\n':
+ while data[i] != "\n":
jaString = data[i]
- if data[i] != '\n':
+ if data[i] != "\n":
jaString = data[i][1:]
currentGroup.append(jaString)
i += 1
-
+
# Join up 401 groups for better translation.
if len(currentGroup) > 0:
- jaString = ''.join(currentGroup)
+ jaString = "".join(currentGroup)
currentGroup = []
-
+
# Remove any textwrap
- jaString = jaString.replace('\n', ' ')
+ jaString = jaString.replace("\n", " ")
# Temporarily convert spaces (For Textwrap Later)
- jaString = jaString.replace('\u3000', ' ')
+ jaString = jaString.replace("\u3000", " ")
# Add Speaker (If there is one)
- if speaker != '':
- jaString = f'{speaker}: {jaString}'
+ if speaker != "":
+ jaString = f"{speaker}: {jaString}"
# Add String
stringList.append(jaString.strip())
-
+
# Pass 2
else:
# Insert Strings
- while data[i] != '\n':
+ while data[i] != "\n":
data.pop(i)
# Get Text
@@ -244,21 +276,21 @@ def translateIris(data, pbar, filename, translatedList):
translatedList = None
# Remove added speaker
- translatedText = re.sub(r'^.+?:\s', '', translatedText)
+ translatedText = re.sub(r"^.+?:\s", "", translatedText)
# Textwrap
translatedText = textwrap.fill(translatedText, width=WIDTH)
- translatedText = translatedText.replace('\n', '\n\u3000')
+ translatedText = translatedText.replace("\n", "\n\u3000")
# Replace Whitespace and Commas
- translatedText = translatedText.replace(', ', '、')
- translatedText = translatedText.replace(',\u3000', '、')
- translatedText = translatedText.replace(',', '、')
- translatedText = translatedText.replace(' ', '\u3000')
+ translatedText = translatedText.replace(", ", "、")
+ translatedText = translatedText.replace(",\u3000", "、")
+ translatedText = translatedText.replace(",", "、")
+ translatedText = translatedText.replace(" ", "\u3000")
# Set Data
# Game crashes on more than 3 lines. Will need to create a new MSG for long translations
- if translatedText.count('\n') > 2:
+ if translatedText.count("\n") > 2:
# Split List
translatedTextList = splitNewlines(translatedText)
@@ -267,34 +299,34 @@ def translateIris(data, pbar, filename, translatedList):
for text in translatedTextList:
if count != 0:
if voice == True:
- #MSG for each item in the list
- data.insert(i, '#MSGVOICE,\n')
+ # MSG for each item in the list
+ data.insert(i, "#MSGVOICE,\n")
i += 1
- data.insert(i, f'{voiceVar}')
+ data.insert(i, f"{voiceVar}")
i += 1
else:
- data.insert(i, '#MSG,\n')
+ data.insert(i, "#MSG,\n")
i += 1
if speaker:
- data[i] = f'\u3000{speaker}\n'
+ data[i] = f"\u3000{speaker}\n"
i += 1
- if text[0] == '\u3000':
- data.insert(i, f'{text}\n')
+ if text[0] == "\u3000":
+ data.insert(i, f"{text}\n")
else:
- data.insert(i, f'\u3000{text}\n')
+ data.insert(i, f"\u3000{text}\n")
i += 1
count += 1
- if data[i] != '\n':
- data.insert(i, '\n')
- data[i] = f'\n{data[i]}'
+ if data[i] != "\n":
+ data.insert(i, "\n")
+ data[i] = f"\n{data[i]}"
else:
- data.insert(i, f'\u3000{translatedText}\n')
+ data.insert(i, f"\u3000{translatedText}\n")
i += 1
- if data[i] != '\n':
- data[i] = f'\n{data[i]}'
+ if data[i] != "\n":
+ data[i] = f"\n{data[i]}"
- elif '#SELECT' in data[i] and translatedList == []:
- Iris = r'(.+?) +\d$'
+ elif "#SELECT" in data[i] and translatedList == []:
+ Iris = r"(.+?) +\d$"
i += 1
match = re.search(Iris, data[i])
if match:
@@ -302,14 +334,20 @@ def translateIris(data, pbar, filename, translatedList):
choiceList.append(match.group(1))
i += 1
match = re.search(Iris, data[i])
- while(match):
+ while match:
choiceList.append(match.group(1))
i += 1
match = re.search(Iris, data[i])
-
+
# Translate
question = stringList[len(stringList) - 1]
- response = translateGPT(choiceList, f'Previous text for context: {question}\n\nThis will be a dialogue option', True, pbar, filename)
+ response = translateGPT(
+ choiceList,
+ f"Previous text for context: {question}\n\nThis will be a dialogue option",
+ True,
+ pbar,
+ filename,
+ )
tokens[0] += response[1][0]
tokens[1] += response[1][1]
choiceListTL = response[0]
@@ -318,10 +356,10 @@ def translateIris(data, pbar, filename, translatedList):
i = i - len(choiceListTL)
for j in range(len(choiceListTL)):
# Replace Whitespace and Commas
- choiceListTL[j] = choiceListTL[j].replace(', ', '、')
- choiceListTL[j] = choiceListTL[j].replace(',\u3000', '、')
- choiceListTL[j] = choiceListTL[j].replace(',', '、')
- choiceListTL[j] = choiceListTL[j].replace(' ', '\u3000')
+ choiceListTL[j] = choiceListTL[j].replace(", ", "、")
+ choiceListTL[j] = choiceListTL[j].replace(",\u3000", "、")
+ choiceListTL[j] = choiceListTL[j].replace(",", "、")
+ choiceListTL[j] = choiceListTL[j].replace(" ", "\u3000")
data[i] = data[i].replace(choiceList[j], choiceListTL[j])
i += 1
@@ -336,9 +374,9 @@ def translateIris(data, pbar, filename, translatedList):
# Set Progress
pbar.total = len(stringList)
pbar.refresh()
-
+
# Translate
- response = translateGPT(stringList, '', True, pbar, filename)
+ response = translateGPT(stringList, "", True, pbar, filename)
tokens[0] += response[1][0]
tokens[1] += response[1][1]
translatedList = response[0]
@@ -354,20 +392,21 @@ def translateIris(data, pbar, filename, translatedList):
MISMATCH.append(filename)
return tokens
+
def splitNewlines(text):
parts = []
newline_count = 0 # Counts the number of newline characters encountered
start_index = 0 # Start index of the current string part
for i, char in enumerate(text):
- if char == '\n':
+ if char == "\n":
newline_count += 1
if newline_count == 3:
# Append the string part from start_index to current index (inclusive)
- parts.append(text[start_index:i+1])
+ parts.append(text[start_index : i + 1])
# Reset newline count and update start_index for the next string part
newline_count = 0
- start_index = i+1
+ start_index = i + 1
# Edge case: if the text does not end with a newline, we still need to append the last part
if start_index < len(text):
@@ -375,94 +414,103 @@ def splitNewlines(text):
return parts
+
# Save some money and enter the character before translation
def getSpeaker(speaker, pbar, filename):
match speaker:
- case 'ファイン':
- return ['Fine', [0,0]]
- case '':
- return ['', [0,0]]
+ case "ファイン":
+ return ["Fine", [0, 0]]
+ case "":
+ return ["", [0, 0]]
case _:
# Store Speaker
if speaker not in str(NAMESLIST):
- response = translateGPT(speaker, 'Reply with only the '+ LANGUAGE +' translation of the NPC name.', False, pbar, filename)
+ response = translateGPT(
+ speaker,
+ "Reply with only the " + LANGUAGE + " translation of the NPC name.",
+ False,
+ pbar,
+ filename,
+ )
response[0] = response[0].replace("'S", "'s")
speakerList = [speaker, response[0]]
NAMESLIST.append(speakerList)
return response
-
+
# Find Speaker
else:
for i in range(len(NAMESLIST)):
if speaker == NAMESLIST[i][0]:
- return [NAMESLIST[i][1],[0,0]]
-
- return [speaker,[0,0]]
+ return [NAMESLIST[i][1], [0, 0]]
+
+ return [speaker, [0, 0]]
+
def subVars(jaString):
- jaString = jaString.replace('\u3000', ' ')
+ jaString = jaString.replace("\u3000", " ")
# Nested
count = 0
- nestedList = re.findall(r'[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]', jaString)
+ nestedList = re.findall(r"[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]", jaString)
nestedList = set(nestedList)
if len(nestedList) != 0:
for icon in nestedList:
- jaString = jaString.replace(icon, '[Nested_' + str(count) + ']')
+ jaString = jaString.replace(icon, "[Nested_" + str(count) + "]")
count += 1
# Icons
count = 0
- iconList = re.findall(r'[\\]+[iIkKwWaA]+\[[0-9]+\]', jaString)
+ iconList = re.findall(r"[\\]+[iIkKwWaA]+\[[0-9]+\]", jaString)
iconList = set(iconList)
if len(iconList) != 0:
for icon in iconList:
- jaString = jaString.replace(icon, '[Ascii_' + str(count) + ']')
+ jaString = jaString.replace(icon, "[Ascii_" + str(count) + "]")
count += 1
# Colors
count = 0
- colorList = re.findall(r'[\\]+[cC]\[[0-9]+\]', jaString)
+ colorList = re.findall(r"[\\]+[cC]\[[0-9]+\]", jaString)
colorList = set(colorList)
if len(colorList) != 0:
for color in colorList:
- jaString = jaString.replace(color, '[Color_' + str(count) + ']')
+ jaString = jaString.replace(color, "[Color_" + str(count) + "]")
count += 1
# Names
count = 0
- nameList = re.findall(r'[\\]+[nN]\[.+?\]+', jaString)
+ nameList = re.findall(r"[\\]+[nN]\[.+?\]+", jaString)
nameList = set(nameList)
if len(nameList) != 0:
for name in nameList:
- jaString = jaString.replace(name, '[Noun_' + str(count) + ']')
+ jaString = jaString.replace(name, "[Noun_" + str(count) + "]")
count += 1
# Variables
count = 0
- varList = re.findall(r'[\\]+[vV]\[[0-9]+\]', jaString)
+ varList = re.findall(r"[\\]+[vV]\[[0-9]+\]", jaString)
varList = set(varList)
if len(varList) != 0:
for var in varList:
- jaString = jaString.replace(var, '[Var_' + str(count) + ']')
+ jaString = jaString.replace(var, "[Var_" + str(count) + "]")
count += 1
# Formatting
count = 0
- formatList = re.findall(r'[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]', jaString)
+ formatList = re.findall(r"[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]", jaString)
formatList = set(formatList)
if len(formatList) != 0:
for var in formatList:
- jaString = jaString.replace(var, '[FCode_' + str(count) + ']')
+ jaString = jaString.replace(var, "[FCode_" + str(count) + "]")
count += 1
# Put all lists in list and return
allList = [nestedList, iconList, colorList, nameList, varList, formatList]
return [jaString, allList]
+
def resubVars(translatedText, allList):
# Fix Spacing and ChatGPT Nonsense
- matchList = re.findall(r'\[\s?.+?\s?\]', translatedText)
+ matchList = re.findall(r"\[\s?.+?\s?\]", translatedText)
if len(matchList) > 0:
for match in matchList:
text = match.strip()
@@ -472,54 +520,58 @@ def resubVars(translatedText, allList):
count = 0
if len(allList[0]) != 0:
for var in allList[0]:
- translatedText = translatedText.replace('[Nested_' + str(count) + ']', var)
+ translatedText = translatedText.replace("[Nested_" + str(count) + "]", var)
count += 1
# Icons
count = 0
if len(allList[1]) != 0:
for var in allList[1]:
- translatedText = translatedText.replace('[Ascii_' + str(count) + ']', var)
+ translatedText = translatedText.replace("[Ascii_" + str(count) + "]", var)
count += 1
# Colors
count = 0
if len(allList[2]) != 0:
for var in allList[2]:
- translatedText = translatedText.replace('[Color_' + str(count) + ']', var)
+ translatedText = translatedText.replace("[Color_" + str(count) + "]", var)
count += 1
# Names
count = 0
if len(allList[3]) != 0:
for var in allList[3]:
- translatedText = translatedText.replace('[Noun_' + str(count) + ']', var)
+ translatedText = translatedText.replace("[Noun_" + str(count) + "]", var)
count += 1
# Vars
count = 0
if len(allList[4]) != 0:
for var in allList[4]:
- translatedText = translatedText.replace('[Var_' + str(count) + ']', var)
+ translatedText = translatedText.replace("[Var_" + str(count) + "]", var)
count += 1
-
+
# Formatting
count = 0
if len(allList[5]) != 0:
for var in allList[5]:
- translatedText = translatedText.replace('[FCode_' + str(count) + ']', var)
+ translatedText = translatedText.replace("[FCode_" + str(count) + "]", var)
count += 1
return translatedText
+
def batchList(input_list, batch_size):
if not isinstance(batch_size, int) or batch_size <= 0:
raise ValueError("batch_size must be a positive integer")
-
- return [input_list[i:i + batch_size] for i in range(0, len(input_list), batch_size)]
+
+ return [
+ input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)
+ ]
+
def createContext(fullPromptFlag, subbedT):
- characters = 'Game Characters:\n\
+ characters = "Game Characters:\n\
フィリア (Philia) - Female\n\
アルネット (Annett) - Female\n\
ラピュセナ (Rapusena) - Female\n\
@@ -529,10 +581,12 @@ def createContext(fullPromptFlag, subbedT):
カルナ (Karna) - Female\n\
ラフィング＝スピア (Laughing Spear) - Female\n\
ノーラ (Nora) - Female\n\
-'
-
- system = PROMPT + VOCAB if fullPromptFlag else \
- f"\
+"
+
+ system = (
+ PROMPT + VOCAB
+ if fullPromptFlag
+ else f"\
You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
- You translate everything, including erotic and explicit content.\n\
@@ -544,9 +598,11 @@ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{
- `...` can be a part of the dialogue. Translate it as it is.\n\
{VOCAB}\n\
"
- user = f'{subbedT}'
+ )
+ user = f"{subbedT}"
return characters, system, user
+
def translateText(characters, system, user, history):
# Prompt
msg = [{"role": "system", "content": system + characters}]
@@ -559,9 +615,9 @@ def translateText(characters, system, user, history):
msg.extend([{"role": "system", "content": h} for h in history])
else:
msg.append({"role": "system", "content": history})
-
+
# Content to TL
- msg.append({"role": "user", "content": f'{user}'})
+ msg.append({"role": "user", "content": f"{user}"})
response = openai.chat.completions.create(
temperature=0.1,
frequency_penalty=0.1,
@@ -570,15 +626,16 @@ def translateText(characters, system, user, history):
)
return response
+
def cleanTranslatedText(translatedText, varResponse):
placeholders = {
- f'{LANGUAGE} Translation: ': '',
- 'Translation: ': '',
- 'っ': '',
- '〜': '~',
- 'ッ': '',
- '。': '.',
- 'Placeholder Text': ''
+ f"{LANGUAGE} Translation: ": "",
+ "Translation: ": "",
+ "っ": "",
+ "〜": "~",
+ "ッ": "",
+ "。": ".",
+ "Placeholder Text": "",
# Add more replacements as needed
}
for target, replacement in placeholders.items():
@@ -589,11 +646,12 @@ def cleanTranslatedText(translatedText, varResponse):
translatedText = resubVars(translatedText, varResponse[1])
return translatedText
+
def elongateCharacters(text):
# Define a pattern to match one character followed by one or more `ー` characters
# Using a positive lookbehind assertion to capture the preceding character
- pattern = r'(?<=(.))ー+'
-
+ pattern = r"(?<=(.))ー+"
+
# Define a replacement function that elongates the captured character
def repl(match):
char = match.group(1) # The character before the ー sequence
@@ -603,8 +661,9 @@ def elongateCharacters(text):
# Use re.sub() to replace the pattern in the text
return re.sub(pattern, repl, text)
+
def extractTranslation(translatedTextList, is_list):
- pattern = r'`?<[Ll]ine\d+>([\\]*.*?[\\]*?)<\/?[Ll]ine\d+>`?'
+ pattern = r"`?<[Ll]ine\d+>([\\]*.*?[\\]*?)<\/?[Ll]ine\d+>`?"
# If it's a batch (i.e., list), extract with tags; otherwise, return the single item.
if is_list:
matchList = re.findall(pattern, translatedTextList)
@@ -613,11 +672,12 @@ def extractTranslation(translatedTextList, is_list):
matchList = re.findall(pattern, translatedTextList)
return matchList[0][0] if matchList else translatedTextList
+
def countTokens(characters, system, user, history):
inputTotalTokens = 0
outputTotalTokens = 0
- enc = tiktoken.encoding_for_model('gpt-4')
-
+ enc = tiktoken.encoding_for_model("gpt-4")
+
# Input
if isinstance(history, list):
for line in history:
@@ -629,15 +689,17 @@ def countTokens(characters, system, user, history):
inputTotalTokens += len(enc.encode(user))
# Output
- outputTotalTokens += round(len(enc.encode(user))*2)
+ outputTotalTokens += round(len(enc.encode(user)) * 2)
return [inputTotalTokens, outputTotalTokens]
+
def combineList(tlist, text):
if isinstance(text, list):
return [t for sublist in tlist for t in sublist]
return tlist[0]
+
@retry(exceptions=Exception, tries=5, delay=5)
def translateGPT(text, history, fullPromptFlag, pbar, filename):
mismatch = False
@@ -650,8 +712,12 @@ def translateGPT(text, history, fullPromptFlag, pbar, filename):
for index, tItem in enumerate(tList):
# Before sending to translation, if we have a list of items, add the formatting
if isinstance(tItem, list):
- payload = '\n'.join([f'`{item}`' for i, item in enumerate(tItem)])
- payload = re.sub(r'(<)(\/Line\d+>)', r'\1>Placeholder Text<\3', payload)
+ payload = "\n".join(
+ [f"`{item}`" for i, item in enumerate(tItem)]
+ )
+ payload = re.sub(
+ r"(<)(\/Line\d+>)", r"\1>Placeholder Text<\3", payload
+ )
varResponse = subVars(payload)
subbedT = varResponse[0]
else:
@@ -659,7 +725,7 @@ def translateGPT(text, history, fullPromptFlag, pbar, filename):
subbedT = varResponse[0]
# Things to Check before starting translation
- if not re.search(r'[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+', subbedT):
+ if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", subbedT):
continue
# Create Message
diff --git a/modules/javascript.py b/modules/javascript.py
index 8080db0..89c9c3e 100644
--- a/modules/javascript.py
+++ b/modules/javascript.py
@@ -15,49 +15,50 @@ from tqdm import tqdm
# Open AI
load_dotenv()
-if os.getenv('api').replace(' ', '') != '':
- openai.base_url = os.getenv('api')
-openai.organization = os.getenv('org')
-openai.api_key = os.getenv('key')
+if os.getenv("api").replace(" ", "") != "":
+ openai.base_url = os.getenv("api")
+openai.organization = os.getenv("org")
+openai.api_key = os.getenv("key")
-#Globals
-MODEL = os.getenv('model')
-TIMEOUT = int(os.getenv('timeout'))
-LANGUAGE = os.getenv('language').capitalize()
-PROMPT = Path('prompt.txt').read_text(encoding='utf-8')
-VOCAB = Path('vocab.txt').read_text(encoding='utf-8')
-THREADS = int(os.getenv('threads'))
+# Globals
+MODEL = os.getenv("model")
+TIMEOUT = int(os.getenv("timeout"))
+LANGUAGE = os.getenv("language").capitalize()
+PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
+VOCAB = Path("vocab.txt").read_text(encoding="utf-8")
+THREADS = int(os.getenv("threads"))
LOCK = threading.Lock()
-WIDTH = int(os.getenv('width'))
-LISTWIDTH = int(os.getenv('listWidth'))
+WIDTH = int(os.getenv("width"))
+LISTWIDTH = int(os.getenv("listWidth"))
NOTEWIDTH = 70
MAXHISTORY = 10
-ESTIMATE = ''
+ESTIMATE = ""
TOKENS = [0, 0]
NAMESLIST = []
-NAMES = False # Output a list of all the character names found
-BRFLAG = False # If the game uses
instead
+NAMES = False # Output a list of all the character names found
+BRFLAG = False # If the game uses
instead
FIXTEXTWRAP = True # Overwrites textwrap
-IGNORETLTEXT = False # Ignores all translated text.
-MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
+IGNORETLTEXT = False # Ignores all translated text.
+MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
-#tqdm Globals
-BAR_FORMAT='{l_bar}{bar:10}{r_bar}{bar:-10b}'
+# tqdm Globals
+BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
POSITION = 0
LEAVE = False
# Pricing - Depends on the model https://openai.com/pricing
# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request
# If you are getting a MISMATCH LENGTH error, lower the batch size.
-if 'gpt-3.5' in MODEL:
- INPUTAPICOST = .002
- OUTPUTAPICOST = .002
+if "gpt-3.5" in MODEL:
+ INPUTAPICOST = 0.002
+ OUTPUTAPICOST = 0.002
BATCHSIZE = 10
-elif 'gpt-4' in MODEL:
- INPUTAPICOST = .005
- OUTPUTAPICOST = .015
+elif "gpt-4" in MODEL:
+ INPUTAPICOST = 0.005
+ OUTPUTAPICOST = 0.015
BATCHSIZE = 40
+
def handleJavascript(filename, estimate):
global ESTIMATE
ESTIMATE = estimate
@@ -74,17 +75,21 @@ def handleJavascript(filename, estimate):
TOKENS[1] += translatedData[1][1]
# Print Total
- totalString = getResultString(['', TOKENS, None], end - start, 'TOTAL')
+ totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
# Print any errors on maps
if len(MISMATCH) > 0:
- return totalString + Fore.RED + f'\nMismatch Errors: {MISMATCH}' + Fore.RESET
+ return (
+ totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
+ )
else:
return totalString
-
+
else:
try:
- with open('translated/' + filename, 'w', encoding='utf8', errors='ignore') as outFile:
+ with open(
+ "translated/" + filename, "w", encoding="utf8", errors="ignore"
+ ) as outFile:
start = time.time()
translatedData = openFiles(filename)
@@ -97,23 +102,36 @@ def handleJavascript(filename, estimate):
TOKENS[1] += translatedData[1][1]
except Exception:
traceback.print_exc()
- return 'Fail'
+ return "Fail"
+
+ return getResultString(["", TOKENS, None], end - start, "TOTAL")
- return getResultString(['', TOKENS, None], end - start, 'TOTAL')
def getResultString(translatedData, translationTime, filename):
# File Print String
- totalTokenstring =\
- Fore.YELLOW +\
- '[Input: ' + str(translatedData[1][0]) + ']'\
- '[Output: ' + str(translatedData[1][1]) + ']'\
- '[Cost: ${:,.4f}'.format((translatedData[1][0] * .001 * INPUTAPICOST) +\
- (translatedData[1][1] * .001 * OUTPUTAPICOST)) + ']'
- timeString = Fore.BLUE + '[' + str(round(translationTime, 1)) + 's]'
+ totalTokenstring = (
+ Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
+ "[Output: "
+ + str(translatedData[1][1])
+ + "]" "[Cost: ${:,.4f}".format(
+ (translatedData[1][0] * 0.001 * INPUTAPICOST)
+ + (translatedData[1][1] * 0.001 * OUTPUTAPICOST)
+ )
+ + "]"
+ )
+ timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
if translatedData[2] == None:
# Success
- return filename + ': ' + totalTokenstring + timeString + Fore.GREEN + u' \u2713 ' + Fore.RESET
+ return (
+ filename
+ + ": "
+ + totalTokenstring
+ + timeString
+ + Fore.GREEN
+ + " \u2713 "
+ + Fore.RESET
+ )
else:
# Fail
@@ -122,21 +140,31 @@ def getResultString(translatedData, translationTime, filename):
except Exception as e:
traceback.print_exc()
errorString = str(e) + Fore.RED
- return filename + ': ' + totalTokenstring + timeString + Fore.RED + u' \u2717 ' +\
- errorString + Fore.RESET
+ return (
+ filename
+ + ": "
+ + totalTokenstring
+ + timeString
+ + Fore.RED
+ + " \u2717 "
+ + errorString
+ + Fore.RESET
+ )
+
def openFiles(filename):
- with open('files/' + filename, 'r', encoding='utf-8') as readFile:
+ with open("files/" + filename, "r", encoding="utf-8") as readFile:
translatedData = parseJS(readFile, filename)
-
+
return translatedData
+
def parseJS(readFile, filename):
- totalTokens = [0,0]
+ totalTokens = [0, 0]
data = readFile.readlines()
with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
- pbar.desc=filename
+ pbar.desc = filename
try:
result = translateJS(data, pbar)
@@ -147,8 +175,9 @@ def parseJS(readFile, filename):
return [data, totalTokens, e]
return [data, totalTokens, None]
+
def translateJS(data, pbar):
- tokens = [0,0]
+ tokens = [0, 0]
i = 0
# Regex & Plugin Name
@@ -165,10 +194,14 @@ def translateJS(data, pbar):
# Remove Wordwrap [Optional]
for j in range(len(modifiedStringList)):
- modifiedStringList[j] = modifiedStringList[j].replace(r'\\\\\\\\n', r' ')
+ modifiedStringList[j] = modifiedStringList[j].replace(
+ r"\\\\\\\\n", r" "
+ )
# Translate
- response = translateGPT(modifiedStringList, f'Reply with the {LANGUAGE} translation', True, pbar)
+ response = translateGPT(
+ modifiedStringList, f"Reply with the {LANGUAGE} translation", True, pbar
+ )
translatedList = response[0]
tokens[0] = response[1][0]
tokens[0] = response[1][1]
@@ -181,82 +214,83 @@ def translateJS(data, pbar):
# Wordwrap [Optional]
translatedList[j] = textwrap.fill(translatedList[j], LISTWIDTH)
- translatedList[j] = translatedList[j].replace('\n', r'\\\\\\\\n')
+ translatedList[j] = translatedList[j].replace("\n", r"\\\\\\\\n")
# Set
data[i] = data[i].replace(stringList[j], translatedList[j])
# Mismatch
else:
- pbar.write('Mismatch Error')
+ pbar.write("Mismatch Error")
i += 1
-
return tokens
+
def subVars(jaString):
- jaString = jaString.replace('\u3000', ' ')
+ jaString = jaString.replace("\u3000", " ")
# Nested
count = 0
- nestedList = re.findall(r'[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]', jaString)
+ nestedList = re.findall(r"[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]", jaString)
nestedList = set(nestedList)
if len(nestedList) != 0:
for icon in nestedList:
- jaString = jaString.replace(icon, '[Nested_' + str(count) + ']')
+ jaString = jaString.replace(icon, "[Nested_" + str(count) + "]")
count += 1
# Icons
count = 0
- iconList = re.findall(r'[\\]+[iIkKwWaA]+\[[0-9]+\]', jaString)
+ iconList = re.findall(r"[\\]+[iIkKwWaA]+\[[0-9]+\]", jaString)
iconList = set(iconList)
if len(iconList) != 0:
for icon in iconList:
- jaString = jaString.replace(icon, '[Ascii_' + str(count) + ']')
+ jaString = jaString.replace(icon, "[Ascii_" + str(count) + "]")
count += 1
# Colors
count = 0
- colorList = re.findall(r'[\\]+[cC]\[[0-9]+\]', jaString)
+ colorList = re.findall(r"[\\]+[cC]\[[0-9]+\]", jaString)
colorList = set(colorList)
if len(colorList) != 0:
for color in colorList:
- jaString = jaString.replace(color, '[Color_' + str(count) + ']')
+ jaString = jaString.replace(color, "[Color_" + str(count) + "]")
count += 1
# Names
count = 0
- nameList = re.findall(r'[\\]+[nN]\[.+?\]+', jaString)
+ nameList = re.findall(r"[\\]+[nN]\[.+?\]+", jaString)
nameList = set(nameList)
if len(nameList) != 0:
for name in nameList:
- jaString = jaString.replace(name, '[Noun_' + str(count) + ']')
+ jaString = jaString.replace(name, "[Noun_" + str(count) + "]")
count += 1
# Variables
count = 0
- varList = re.findall(r'[\\]+[vV]\[[0-9]+\]', jaString)
+ varList = re.findall(r"[\\]+[vV]\[[0-9]+\]", jaString)
varList = set(varList)
if len(varList) != 0:
for var in varList:
- jaString = jaString.replace(var, '[Var_' + str(count) + ']')
+ jaString = jaString.replace(var, "[Var_" + str(count) + "]")
count += 1
# Formatting
count = 0
- formatList = re.findall(r'[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]', jaString)
+ formatList = re.findall(r"[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]", jaString)
formatList = set(formatList)
if len(formatList) != 0:
for var in formatList:
- jaString = jaString.replace(var, '[FCode_' + str(count) + ']')
+ jaString = jaString.replace(var, "[FCode_" + str(count) + "]")
count += 1
# Put all lists in list and return
allList = [nestedList, iconList, colorList, nameList, varList, formatList]
return [jaString, allList]
+
def resubVars(translatedText, allList):
# Fix Spacing and ChatGPT Nonsense
- matchList = re.findall(r'\[\s?.+?\s?\]', translatedText)
+ matchList = re.findall(r"\[\s?.+?\s?\]", translatedText)
if len(matchList) > 0:
for match in matchList:
text = match.strip()
@@ -266,64 +300,70 @@ def resubVars(translatedText, allList):
count = 0
if len(allList[0]) != 0:
for var in allList[0]:
- translatedText = translatedText.replace('[Nested_' + str(count) + ']', var)
+ translatedText = translatedText.replace("[Nested_" + str(count) + "]", var)
count += 1
# Icons
count = 0
if len(allList[1]) != 0:
for var in allList[1]:
- translatedText = translatedText.replace('[Ascii_' + str(count) + ']', var)
+ translatedText = translatedText.replace("[Ascii_" + str(count) + "]", var)
count += 1
# Colors
count = 0
if len(allList[2]) != 0:
for var in allList[2]:
- translatedText = translatedText.replace('[Color_' + str(count) + ']', var)
+ translatedText = translatedText.replace("[Color_" + str(count) + "]", var)
count += 1
# Names
count = 0
if len(allList[3]) != 0:
for var in allList[3]:
- translatedText = translatedText.replace('[Noun_' + str(count) + ']', var)
+ translatedText = translatedText.replace("[Noun_" + str(count) + "]", var)
count += 1
# Vars
count = 0
if len(allList[4]) != 0:
for var in allList[4]:
- translatedText = translatedText.replace('[Var_' + str(count) + ']', var)
+ translatedText = translatedText.replace("[Var_" + str(count) + "]", var)
count += 1
-
+
# Formatting
count = 0
if len(allList[5]) != 0:
for var in allList[5]:
- translatedText = translatedText.replace('[FCode_' + str(count) + ']', var)
+ translatedText = translatedText.replace("[FCode_" + str(count) + "]", var)
count += 1
return translatedText
+
def batchList(input_list, batch_size):
if not isinstance(batch_size, int) or batch_size <= 0:
raise ValueError("batch_size must be a positive integer")
-
- return [input_list[i:i + batch_size] for i in range(0, len(input_list), batch_size)]
+
+ return [
+ input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)
+ ]
+
def createContext(fullPromptFlag, subbedT):
- characters = 'Game Characters:\n\
+ characters = "Game Characters:\n\
皆月 (Minazuki)\n\
さやか (Sayaka)\n\
皆月 さやか (Minazuki Sayaka) - Female\n\
広瀬 (Hirose)\n\
智恵 (Chie) - Female\n\
広瀬 智恵 (Hirose Chie) - Female\n\
-'
-
- system = PROMPT + VOCAB if fullPromptFlag else \
- f"\
+"
+
+ system = (
+ PROMPT + VOCAB
+ if fullPromptFlag
+ else f"\
You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
- You translate everything, including erotic and explicit content.\n\
@@ -335,9 +375,11 @@ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{
- `...` can be a part of the dialogue. Translate it as it is.\n\
{VOCAB}\n\
"
- user = f'{subbedT}'
+ )
+ user = f"{subbedT}"
return characters, system, user
+
def translateText(characters, system, user, history):
# Prompt
msg = [{"role": "system", "content": system + characters}]
@@ -350,9 +392,9 @@ def translateText(characters, system, user, history):
msg.extend([{"role": "system", "content": h} for h in history])
else:
msg.append({"role": "system", "content": history})
-
+
# Content to TL
- msg.append({"role": "user", "content": f'{user}'})
+ msg.append({"role": "user", "content": f"{user}"})
response = openai.chat.completions.create(
temperature=0.1,
frequency_penalty=0.1,
@@ -361,15 +403,16 @@ def translateText(characters, system, user, history):
)
return response
+
def cleanTranslatedText(translatedText, varResponse):
placeholders = {
- f'{LANGUAGE} Translation: ': '',
- 'Translation: ': '',
- 'っ': '',
- '〜': '~',
- 'ッ': '',
- '。': '.',
- 'Placeholder Text': ''
+ f"{LANGUAGE} Translation: ": "",
+ "Translation: ": "",
+ "っ": "",
+ "〜": "~",
+ "ッ": "",
+ "。": ".",
+ "Placeholder Text": "",
# Add more replacements as needed
}
for target, replacement in placeholders.items():
@@ -380,11 +423,12 @@ def cleanTranslatedText(translatedText, varResponse):
translatedText = resubVars(translatedText, varResponse[1])
return translatedText
+
def elongateCharacters(text):
# Define a pattern to match one character followed by one or more `ー` characters
# Using a positive lookbehind assertion to capture the preceding character
- pattern = r'(?<=(.))ー+'
-
+ pattern = r"(?<=(.))ー+"
+
# Define a replacement function that elongates the captured character
def repl(match):
char = match.group(1) # The character before the ー sequence
@@ -394,8 +438,9 @@ def elongateCharacters(text):
# Use re.sub() to replace the pattern in the text
return re.sub(pattern, repl, text)
+
def extractTranslation(translatedTextList, is_list):
- pattern = r'`?<[Ll]ine\d+>([\\]*.*?[\\]*?)<\/?[Ll]ine\d+>`?'
+ pattern = r"`?<[Ll]ine\d+>([\\]*.*?[\\]*?)<\/?[Ll]ine\d+>`?"
# If it's a batch (i.e., list), extract with tags; otherwise, return the single item.
if is_list:
matchList = re.findall(pattern, translatedTextList)
@@ -404,11 +449,12 @@ def extractTranslation(translatedTextList, is_list):
matchList = re.findall(pattern, translatedTextList)
return matchList[0][0] if matchList else translatedTextList
+
def countTokens(characters, system, user, history):
inputTotalTokens = 0
outputTotalTokens = 0
- enc = tiktoken.encoding_for_model('gpt-4')
-
+ enc = tiktoken.encoding_for_model("gpt-4")
+
# Input
if isinstance(history, list):
for line in history:
@@ -420,15 +466,17 @@ def countTokens(characters, system, user, history):
inputTotalTokens += len(enc.encode(user))
# Output
- outputTotalTokens += round(len(enc.encode(user))*3)
+ outputTotalTokens += round(len(enc.encode(user)) * 3)
return [inputTotalTokens, outputTotalTokens]
+
def combineList(tlist, text):
if isinstance(text, list):
return [t for sublist in tlist for t in sublist]
return tlist[0]
+
@retry(exceptions=Exception, tries=5, delay=5)
def translateGPT(text, history, fullPromptFlag, pbar):
mismatch = False
@@ -441,8 +489,12 @@ def translateGPT(text, history, fullPromptFlag, pbar):
for index, tItem in enumerate(tList):
# Before sending to translation, if we have a list of items, add the formatting
if isinstance(tItem, list):
- payload = '\n'.join([f'`{item}`' for i, item in enumerate(tItem)])
- payload = re.sub(r'(<)(\/Line\d+>)', r'\1>Placeholder Text<\3', payload)
+ payload = "\n".join(
+ [f"`{item}`" for i, item in enumerate(tItem)]
+ )
+ payload = re.sub(
+ r"(<)(\/Line\d+>)", r"\1>Placeholder Text<\3", payload
+ )
varResponse = subVars(payload)
subbedT = varResponse[0]
else:
@@ -450,7 +502,7 @@ def translateGPT(text, history, fullPromptFlag, pbar):
subbedT = varResponse[0]
# Things to Check before starting translation
- if not re.search(r'[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+', subbedT):
+ if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", subbedT):
continue
# Create Message
@@ -488,7 +540,7 @@ def translateGPT(text, history, fullPromptFlag, pbar):
if len(tItem) == len(extractedTranslations):
tList[index] = extractedTranslations
else:
- mismatch = True # Just here for breakpoint
+ mismatch = True # Just here for breakpoint
# Create History
history = tList[index] # Update history if we have a list
diff --git a/modules/json.py b/modules/json.py
index b67431c..d686a5f 100644
--- a/modules/json.py
+++ b/modules/json.py
@@ -16,49 +16,50 @@ from tqdm import tqdm
# Open AI
load_dotenv()
-if os.getenv('api').replace(' ', '') != '':
- openai.base_url = os.getenv('api')
-openai.organization = os.getenv('org')
-openai.api_key = os.getenv('key')
+if os.getenv("api").replace(" ", "") != "":
+ openai.base_url = os.getenv("api")
+openai.organization = os.getenv("org")
+openai.api_key = os.getenv("key")
-#Globals
-MODEL = os.getenv('model')
-TIMEOUT = int(os.getenv('timeout'))
-LANGUAGE = os.getenv('language').capitalize()
-PROMPT = Path('prompt.txt').read_text(encoding='utf-8')
-VOCAB = Path('vocab.txt').read_text(encoding='utf-8')
-THREADS = int(os.getenv('threads'))
+# Globals
+MODEL = os.getenv("model")
+TIMEOUT = int(os.getenv("timeout"))
+LANGUAGE = os.getenv("language").capitalize()
+PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
+VOCAB = Path("vocab.txt").read_text(encoding="utf-8")
+THREADS = int(os.getenv("threads"))
LOCK = threading.Lock()
-WIDTH = int(os.getenv('width'))
-LISTWIDTH = int(os.getenv('listWidth'))
+WIDTH = int(os.getenv("width"))
+LISTWIDTH = int(os.getenv("listWidth"))
NOTEWIDTH = 70
MAXHISTORY = 10
-ESTIMATE = ''
+ESTIMATE = ""
TOKENS = [0, 0]
NAMESLIST = []
-NAMES = False # Output a list of all the character names found
-BRFLAG = False # If the game uses
instead
+NAMES = False # Output a list of all the character names found
+BRFLAG = False # If the game uses
instead
FIXTEXTWRAP = True # Overwrites textwrap
-IGNORETLTEXT = False # Ignores all translated text.
-MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
+IGNORETLTEXT = False # Ignores all translated text.
+MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
-#tqdm Globals
-BAR_FORMAT='{l_bar}{bar:10}{r_bar}{bar:-10b}'
+# tqdm Globals
+BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
POSITION = 0
LEAVE = False
# Pricing - Depends on the model https://openai.com/pricing
# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request
# If you are getting a MISMATCH LENGTH error, lower the batch size.
-if 'gpt-3.5' in MODEL:
- INPUTAPICOST = .002
- OUTPUTAPICOST = .002
+if "gpt-3.5" in MODEL:
+ INPUTAPICOST = 0.002
+ OUTPUTAPICOST = 0.002
BATCHSIZE = 10
-elif 'gpt-4' in MODEL:
- INPUTAPICOST = .01
- OUTPUTAPICOST = .03
+elif "gpt-4" in MODEL:
+ INPUTAPICOST = 0.01
+ OUTPUTAPICOST = 0.03
BATCHSIZE = 50
+
def handleJSON(filename, estimate):
global ESTIMATE, totalTokens
ESTIMATE = estimate
@@ -74,11 +75,11 @@ def handleJSON(filename, estimate):
TOKENS[0] += translatedData[1][0]
TOKENS[1] += translatedData[1][1]
- return getResultString(['', TOKENS, None], end - start, 'TOTAL')
-
+ return getResultString(["", TOKENS, None], end - start, "TOTAL")
+
else:
try:
- with open('translated/' + filename, 'w', encoding='UTF-8') as outFile:
+ with open("translated/" + filename, "w", encoding="UTF-8") as outFile:
start = time.time()
translatedData = openFiles(filename)
@@ -90,36 +91,50 @@ def handleJSON(filename, estimate):
TOKENS[0] += translatedData[1][0]
TOKENS[1] += translatedData[1][1]
except Exception:
- return 'Fail'
+ return "Fail"
+
+ return getResultString(["", TOKENS, None], end - start, "TOTAL")
- return getResultString(['', TOKENS, None], end - start, 'TOTAL')
def openFiles(filename):
- with open('files/' + filename, 'r', encoding='UTF-8-sig') as f:
+ with open("files/" + filename, "r", encoding="UTF-8-sig") as f:
data = json.load(f)
# Map Files
- if '.json' in filename:
+ if ".json" in filename:
translatedData = parseJSON(data, filename)
else:
- raise NameError(filename + ' Not Supported')
-
+ raise NameError(filename + " Not Supported")
+
return translatedData
+
def getResultString(translatedData, translationTime, filename):
# File Print String
- totalTokenstring =\
- Fore.YELLOW +\
- '[Input: ' + str(translatedData[1][0]) + ']'\
- '[Output: ' + str(translatedData[1][1]) + ']'\
- '[Cost: ${:,.4f}'.format((translatedData[1][0] * .001 * INPUTAPICOST) +\
- (translatedData[1][1] * .001 * OUTPUTAPICOST)) + ']'
- timeString = Fore.BLUE + '[' + str(round(translationTime, 1)) + 's]'
+ totalTokenstring = (
+ Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
+ "[Output: "
+ + str(translatedData[1][1])
+ + "]" "[Cost: ${:,.4f}".format(
+ (translatedData[1][0] * 0.001 * INPUTAPICOST)
+ + (translatedData[1][1] * 0.001 * OUTPUTAPICOST)
+ )
+ + "]"
+ )
+ timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
if translatedData[2] == None:
# Success
- return filename + ': ' + totalTokenstring + timeString + Fore.GREEN + u' \u2713 ' + Fore.RESET
+ return (
+ filename
+ + ": "
+ + totalTokenstring
+ + timeString
+ + Fore.GREEN
+ + " \u2713 "
+ + Fore.RESET
+ )
else:
# Fail
@@ -128,18 +143,29 @@ def getResultString(translatedData, translationTime, filename):
except Exception as e:
traceback.print_exc()
errorString = str(e) + Fore.RED
- return filename + ': ' + totalTokenstring + timeString + Fore.RED + u' \u2717 ' +\
- errorString + Fore.RESET
-
+ return (
+ filename
+ + ": "
+ + totalTokenstring
+ + timeString
+ + Fore.RED
+ + " \u2717 "
+ + errorString
+ + Fore.RESET
+ )
+
+
def parseJSON(data, filename):
totalTokens = [0, 0]
totalLines = 0
totalLines = len(data)
global LOCK
-
- with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar:
- pbar.desc=filename
- pbar.total=totalLines
+
+ with tqdm(
+ bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE
+ ) as pbar:
+ pbar.desc = filename
+ pbar.total = totalLines
try:
result = translateJSON(data, pbar)
totalTokens[0] += result[0]
@@ -148,12 +174,13 @@ def parseJSON(data, filename):
return [data, totalTokens, e]
return [data, totalTokens, None]
+
def translateJSON(data, pbar):
textHistory = []
batch = []
maxHistory = MAXHISTORY
tokens = [0, 0]
- speaker = 'None'
+ speaker = "None"
insertBool = False
i = 0
batchStartIndex = 0
@@ -161,35 +188,43 @@ def translateJSON(data, pbar):
while i < len(data):
item = data[i]
# Speaker
- if 'name' in item:
- if item['name'] not in [None, '-']:
- response = getSpeaker(item['name'])
+ if "name" in item:
+ if item["name"] not in [None, "-"]:
+ response = getSpeaker(item["name"])
speaker = response[0]
tokens[0] += response[1][0]
tokens[1] += response[1][1]
- item['name'] = speaker
+ item["name"] = speaker
else:
- speaker = 'None'
+ speaker = "None"
pbar.update(1)
i += 1
-
# Text
- elif 'me' in item:
- for text in ['text', 'text2', 'help1', 'help2', 'help3', 'like', 'message', 'me']:
+ elif "me" in item:
+ for text in [
+ "text",
+ "text2",
+ "help1",
+ "help2",
+ "help3",
+ "like",
+ "message",
+ "me",
+ ]:
if text in item:
if item[text] != None:
jaString = item[text]
# Remove any textwrap
if FIXTEXTWRAP == True:
- finalJAString = jaString.replace('\n', ' ')
+ finalJAString = jaString.replace("\n", " ")
# [Passthrough 1] Pulling From File
if insertBool is False:
# Append to List and Clear Values
batch.append(finalJAString)
- speaker = ''
+ speaker = ""
# Translate Batch if Full
if len(batch) == BATCHSIZE:
@@ -207,7 +242,7 @@ def translateJSON(data, pbar):
# Mismatch
else:
- pbar.write(f'Mismatch: {batchStartIndex} - {i}')
+ pbar.write(f"Mismatch: {batchStartIndex} - {i}")
MISMATCH.append(batch)
batchStartIndex = i
batch.clear()
@@ -215,7 +250,7 @@ def translateJSON(data, pbar):
if insertBool is False:
pbar.update(1)
i += 1
-
+
currentGroup = []
# [Passthrough 2] Setting Data
@@ -224,15 +259,15 @@ def translateJSON(data, pbar):
translatedText = translatedBatch[0]
# Remove added speaker
- translatedText = re.sub(r'^.+?:\s', '', translatedText)
+ translatedText = re.sub(r"^.+?:\s", "", translatedText)
# Textwrap
translatedText = textwrap.fill(translatedText, width=WIDTH)
-
+
# Set Text
item[text] = translatedText
translatedBatch.pop(0)
- speaker = ''
+ speaker = ""
currentGroup = []
i += 1
@@ -261,93 +296,99 @@ def translateJSON(data, pbar):
# Mismatch
else:
- pbar.write(f'Mismatch: {batchStartIndex} - {i}')
+ pbar.write(f"Mismatch: {batchStartIndex} - {i}")
MISMATCH.append(batch)
batchStartIndex = i
batch.clear()
currentGroup = []
- return tokens
+ return tokens
+
# Save some money and enter the character before translation
def getSpeaker(speaker):
match speaker:
- case 'セレナ':
- return ['Serena', [0,0]]
- case 'レナ':
- return ['Rena', [0,0]]
- case 'フィルス':
- return ['Phils', [0,0]]
- case 'レイン':
- return ['Meryl', [0,0]]
+ case "セレナ":
+ return ["Serena", [0, 0]]
+ case "レナ":
+ return ["Rena", [0, 0]]
+ case "フィルス":
+ return ["Phils", [0, 0]]
+ case "レイン":
+ return ["Meryl", [0, 0]]
case _:
- return translateGPT(speaker, 'Reply with only the '+ LANGUAGE +' translation of the NPC name.', False)
+ return translateGPT(
+ speaker,
+ "Reply with only the " + LANGUAGE + " translation of the NPC name.",
+ False,
+ )
def subVars(jaString):
- jaString = jaString.replace('\u3000', ' ')
+ jaString = jaString.replace("\u3000", " ")
# Nested
count = 0
- nestedList = re.findall(r'[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]', jaString)
+ nestedList = re.findall(r"[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]", jaString)
nestedList = set(nestedList)
if len(nestedList) != 0:
for icon in nestedList:
- jaString = jaString.replace(icon, '{Nested_' + str(count) + '}')
+ jaString = jaString.replace(icon, "{Nested_" + str(count) + "}")
count += 1
# Icons
count = 0
- iconList = re.findall(r'[\\]+[iIkKwWaA]+\[[0-9]+\]', jaString)
+ iconList = re.findall(r"[\\]+[iIkKwWaA]+\[[0-9]+\]", jaString)
iconList = set(iconList)
if len(iconList) != 0:
for icon in iconList:
- jaString = jaString.replace(icon, '{Ascii_' + str(count) + '}')
+ jaString = jaString.replace(icon, "{Ascii_" + str(count) + "}")
count += 1
# Colors
count = 0
- colorList = re.findall(r'[\\]+[cC]\[[0-9]+\]', jaString)
+ colorList = re.findall(r"[\\]+[cC]\[[0-9]+\]", jaString)
colorList = set(colorList)
if len(colorList) != 0:
for color in colorList:
- jaString = jaString.replace(color, '{Color_' + str(count) + '}')
+ jaString = jaString.replace(color, "{Color_" + str(count) + "}")
count += 1
# Names
count = 0
- nameList = re.findall(r'[\\]+[nN]\[.+?\]+', jaString)
+ nameList = re.findall(r"[\\]+[nN]\[.+?\]+", jaString)
nameList = set(nameList)
if len(nameList) != 0:
for name in nameList:
- jaString = jaString.replace(name, '{Noun_' + str(count) + '}')
+ jaString = jaString.replace(name, "{Noun_" + str(count) + "}")
count += 1
# Variables
count = 0
- varList = re.findall(r'[\\]+[vV]\[[0-9]+\]', jaString)
+ varList = re.findall(r"[\\]+[vV]\[[0-9]+\]", jaString)
varList = set(varList)
if len(varList) != 0:
for var in varList:
- jaString = jaString.replace(var, '{Var_' + str(count) + '}')
+ jaString = jaString.replace(var, "{Var_" + str(count) + "}")
count += 1
# Formatting
count = 0
- formatList = re.findall(r'[\\]+[\w]+\[.+?\]', jaString)
+ formatList = re.findall(r"[\\]+[\w]+\[.+?\]", jaString)
formatList = set(formatList)
if len(formatList) != 0:
for var in formatList:
- jaString = jaString.replace(var, '{FCode_' + str(count) + '}')
+ jaString = jaString.replace(var, "{FCode_" + str(count) + "}")
count += 1
# Put all lists in list and return
allList = [nestedList, iconList, colorList, nameList, varList, formatList]
return [jaString, allList]
+
def resubVars(translatedText, allList):
# Fix Spacing and ChatGPT Nonsense
- matchList = re.findall(r'\[\s?.+?\s?\]', translatedText)
+ matchList = re.findall(r"\[\s?.+?\s?\]", translatedText)
if len(matchList) > 0:
for match in matchList:
text = match.strip()
@@ -357,54 +398,58 @@ def resubVars(translatedText, allList):
count = 0
if len(allList[0]) != 0:
for var in allList[0]:
- translatedText = translatedText.replace('{Nested_' + str(count) + '}', var)
+ translatedText = translatedText.replace("{Nested_" + str(count) + "}", var)
count += 1
# Icons
count = 0
if len(allList[1]) != 0:
for var in allList[1]:
- translatedText = translatedText.replace('{Ascii_' + str(count) + '}', var)
+ translatedText = translatedText.replace("{Ascii_" + str(count) + "}", var)
count += 1
# Colors
count = 0
if len(allList[2]) != 0:
for var in allList[2]:
- translatedText = translatedText.replace('{Color_' + str(count) + '}', var)
+ translatedText = translatedText.replace("{Color_" + str(count) + "}", var)
count += 1
# Names
count = 0
if len(allList[3]) != 0:
for var in allList[3]:
- translatedText = translatedText.replace('{Noun_' + str(count) + '}', var)
+ translatedText = translatedText.replace("{Noun_" + str(count) + "}", var)
count += 1
# Vars
count = 0
if len(allList[4]) != 0:
for var in allList[4]:
- translatedText = translatedText.replace('{Var_' + str(count) + '}', var)
+ translatedText = translatedText.replace("{Var_" + str(count) + "}", var)
count += 1
-
+
# Formatting
count = 0
if len(allList[5]) != 0:
for var in allList[5]:
- translatedText = translatedText.replace('{FCode_' + str(count) + '}', var)
+ translatedText = translatedText.replace("{FCode_" + str(count) + "}", var)
count += 1
return translatedText
+
def batchList(input_list, batch_size):
if not isinstance(batch_size, int) or batch_size <= 0:
raise ValueError("batch_size must be a positive integer")
-
- return [input_list[i:i + batch_size] for i in range(0, len(input_list), batch_size)]
+
+ return [
+ input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)
+ ]
+
def createContext(fullPromptFlag, subbedT):
- characters = 'Game Characters:\n\
+ characters = "Game Characters:\n\
ルナリア (Lunaria) - Female\n\
ソニア (Sonia) - Female\n\
マナ (Mana) - Female\n\
@@ -419,10 +464,12 @@ def createContext(fullPromptFlag, subbedT):
ツキハ (Tsukiha) - Female\n\
フィリカ (Filica) - Female\n\
レノ (Renno) - Female\n\
-'
-
- system = PROMPT if fullPromptFlag else \
- f"\
+"
+
+ system = (
+ PROMPT
+ if fullPromptFlag
+ else f"\
You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
You are going to be translating text from a videogame.\n\
I will give you lines of text, and you must translate each line to the best of your ability.\n\
@@ -439,9 +486,11 @@ I will give you lines of text, and you must translate each line to the best of y
- Translate 'よかった' as 'thank goodness'\n\
Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\
"
- user = f'{subbedT}'
+ )
+ user = f"{subbedT}"
return characters, system, user
+
def translateText(characters, system, user, history):
# Prompt
msg = [{"role": "system", "content": system + characters}]
@@ -454,9 +503,9 @@ def translateText(characters, system, user, history):
msg.extend([{"role": "assistant", "content": h} for h in history])
else:
msg.append({"role": "assistant", "content": history})
-
+
# Content to TL
- msg.append({"role": "user", "content": f'{user}'})
+ msg.append({"role": "user", "content": f"{user}"})
response = openai.chat.completions.create(
temperature=0.1,
frequency_penalty=0.1,
@@ -466,37 +515,44 @@ def translateText(characters, system, user, history):
)
return response
+
def cleanTranslatedText(translatedText, varResponse):
placeholders = {
- f'{LANGUAGE} Translation: ': '',
- 'Translation: ': '',
- 'っ': '',
- '〜': '~',
- 'ッ': '',
- '。': '.',
- 'Placeholder Text': ''
+ f"{LANGUAGE} Translation: ": "",
+ "Translation: ": "",
+ "っ": "",
+ "〜": "~",
+ "ッ": "",
+ "。": ".",
+ "Placeholder Text": "",
# Add more replacements as needed
}
for target, replacement in placeholders.items():
translatedText = translatedText.replace(target, replacement)
translatedText = resubVars(translatedText, varResponse[1])
- return [line for line in translatedText.split('\n') if line]
+ return [line for line in translatedText.split("\n") if line]
+
def extractTranslation(translatedTextList, is_list):
- pattern = r'`?([\\]*.*?[\\]*?)<\/?Line\d+>`?'
+ pattern = r"`?([\\]*.*?[\\]*?)<\/?Line\d+>`?"
# If it's a batch (i.e., list), extract with tags; otherwise, return the single item.
if is_list:
- return [re.findall(pattern, line)[0][1] for line in translatedTextList if re.search(pattern, line)]
+ return [
+ re.findall(pattern, line)[0][1]
+ for line in translatedTextList
+ if re.search(pattern, line)
+ ]
else:
matchList = re.findall(pattern, translatedTextList)
return matchList[0][1] if matchList else translatedTextList
+
def countTokens(characters, system, user, history):
inputTotalTokens = 0
outputTotalTokens = 0
- enc = tiktoken.encoding_for_model('gpt-4')
-
+ enc = tiktoken.encoding_for_model("gpt-4")
+
# Input
if isinstance(history, list):
for line in history:
@@ -508,15 +564,17 @@ def countTokens(characters, system, user, history):
inputTotalTokens += len(enc.encode(user))
# Output
- outputTotalTokens += round(len(enc.encode(user))*3)
+ outputTotalTokens += round(len(enc.encode(user)) * 3)
return [inputTotalTokens, outputTotalTokens]
+
def combineList(tlist, text):
if isinstance(text, list):
return [t for sublist in tlist for t in sublist]
return tlist[0]
+
@retry(exceptions=Exception, tries=5, delay=5)
def translateGPT(text, history, fullPromptFlag):
totalTokens = [0, 0]
@@ -528,8 +586,10 @@ def translateGPT(text, history, fullPromptFlag):
for index, tItem in enumerate(tList):
# Before sending to translation, if we have a list of items, add the formatting
if isinstance(tItem, list):
- payload = '\n'.join([f'`{item}`' for i, item in enumerate(tItem)])
- payload = payload.replace('``', '`Placeholder Text`')
+ payload = "\n".join(
+ [f"`{item}`" for i, item in enumerate(tItem)]
+ )
+ payload = payload.replace("``", "`Placeholder Text`")
varResponse = subVars(payload)
subbedT = varResponse[0]
else:
@@ -537,7 +597,7 @@ def translateGPT(text, history, fullPromptFlag):
subbedT = varResponse[0]
# Things to Check before starting translation
- if not re.search(r'[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+', subbedT):
+ if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", subbedT):
continue
# Create Message
@@ -562,12 +622,14 @@ def translateGPT(text, history, fullPromptFlag):
extractedTranslations = extractTranslation(translatedTextList, True)
tList[index] = extractedTranslations
if len(tItem) != len(translatedTextList):
- mismatch = True # Just here so breakpoint can be set
+ mismatch = True # Just here so breakpoint can be set
history = extractedTranslations[-10:] # Update history if we have a list
else:
# Ensure we're passing a single string to extractTranslation
- extractedTranslations = extractTranslation('\n'.join(translatedTextList), False)
+ extractedTranslations = extractTranslation(
+ "\n".join(translatedTextList), False
+ )
tList[index] = extractedTranslations
finalList = combineList(tList, text)
- return [finalList, totalTokens]
\ No newline at end of file
+ return [finalList, totalTokens]
diff --git a/modules/kansen.py b/modules/kansen.py
index fb341bf..178fd3f 100644
--- a/modules/kansen.py
+++ b/modules/kansen.py
@@ -15,49 +15,50 @@ from tqdm import tqdm
# Open AI
load_dotenv()
-if os.getenv('api').replace(' ', '') != '':
- openai.base_url = os.getenv('api')
-openai.organization = os.getenv('org')
-openai.api_key = os.getenv('key')
+if os.getenv("api").replace(" ", "") != "":
+ openai.base_url = os.getenv("api")
+openai.organization = os.getenv("org")
+openai.api_key = os.getenv("key")
-#Globals
-MODEL = os.getenv('model')
-TIMEOUT = int(os.getenv('timeout'))
-LANGUAGE = os.getenv('language').capitalize()
-PROMPT = Path('prompt.txt').read_text(encoding='utf-8')
-VOCAB = Path('vocab.txt').read_text(encoding='utf-8')
-THREADS = int(os.getenv('threads'))
+# Globals
+MODEL = os.getenv("model")
+TIMEOUT = int(os.getenv("timeout"))
+LANGUAGE = os.getenv("language").capitalize()
+PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
+VOCAB = Path("vocab.txt").read_text(encoding="utf-8")
+THREADS = int(os.getenv("threads"))
LOCK = threading.Lock()
-WIDTH = int(os.getenv('width'))
-LISTWIDTH = int(os.getenv('listWidth'))
+WIDTH = int(os.getenv("width"))
+LISTWIDTH = int(os.getenv("listWidth"))
NOTEWIDTH = 70
MAXHISTORY = 10
-ESTIMATE = ''
+ESTIMATE = ""
TOKENS = [0, 0]
NAMESLIST = []
-NAMES = False # Output a list of all the character names found
-BRFLAG = False # If the game uses
instead
+NAMES = False # Output a list of all the character names found
+BRFLAG = False # If the game uses
instead
FIXTEXTWRAP = False # Overwrites textwrap
-IGNORETLTEXT = False # Ignores all translated text.
-MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
+IGNORETLTEXT = False # Ignores all translated text.
+MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
-#tqdm Globals
-BAR_FORMAT='{l_bar}{bar:10}{r_bar}{bar:-10b}'
+# tqdm Globals
+BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
POSITION = 0
LEAVE = False
# Pricing - Depends on the model https://openai.com/pricing
# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request
# If you are getting a MISMATCH LENGTH error, lower the batch size.
-if 'gpt-3.5' in MODEL:
- INPUTAPICOST = .002
- OUTPUTAPICOST = .002
+if "gpt-3.5" in MODEL:
+ INPUTAPICOST = 0.002
+ OUTPUTAPICOST = 0.002
BATCHSIZE = 10
-elif 'gpt-4' in MODEL:
- INPUTAPICOST = .01
- OUTPUTAPICOST = .03
+elif "gpt-4" in MODEL:
+ INPUTAPICOST = 0.01
+ OUTPUTAPICOST = 0.03
BATCHSIZE = 10
+
def handleKansen(filename, estimate):
global ESTIMATE
ESTIMATE = estimate
@@ -74,17 +75,21 @@ def handleKansen(filename, estimate):
TOKENS[1] += translatedData[1][1]
# Print Total
- totalString = getResultString(['', TOKENS, None], end - start, 'TOTAL')
+ totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
# Print any errors on maps
if len(MISMATCH) > 0:
- return totalString + Fore.RED + f'\nMismatch Errors: {MISMATCH}' + Fore.RESET
+ return (
+ totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
+ )
else:
return totalString
-
+
else:
try:
- with open('translated/' + filename, 'w', encoding='shift_jis', errors='ignore') as outFile:
+ with open(
+ "translated/" + filename, "w", encoding="shift_jis", errors="ignore"
+ ) as outFile:
start = time.time()
translatedData = openFiles(filename)
@@ -97,23 +102,36 @@ def handleKansen(filename, estimate):
TOKENS[1] += translatedData[1][1]
except Exception:
traceback.print_exc()
- return 'Fail'
+ return "Fail"
+
+ return getResultString(["", TOKENS, None], end - start, "TOTAL")
- return getResultString(['', TOKENS, None], end - start, 'TOTAL')
def getResultString(translatedData, translationTime, filename):
# File Print String
- totalTokenstring =\
- Fore.YELLOW +\
- '[Input: ' + str(translatedData[1][0]) + ']'\
- '[Output: ' + str(translatedData[1][1]) + ']'\
- '[Cost: ${:,.4f}'.format((translatedData[1][0] * .001 * INPUTAPICOST) +\
- (translatedData[1][1] * .001 * OUTPUTAPICOST)) + ']'
- timeString = Fore.BLUE + '[' + str(round(translationTime, 1)) + 's]'
+ totalTokenstring = (
+ Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
+ "[Output: "
+ + str(translatedData[1][1])
+ + "]" "[Cost: ${:,.4f}".format(
+ (translatedData[1][0] * 0.001 * INPUTAPICOST)
+ + (translatedData[1][1] * 0.001 * OUTPUTAPICOST)
+ )
+ + "]"
+ )
+ timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
if translatedData[2] == None:
# Success
- return filename + ': ' + totalTokenstring + timeString + Fore.GREEN + u' \u2713 ' + Fore.RESET
+ return (
+ filename
+ + ": "
+ + totalTokenstring
+ + timeString
+ + Fore.GREEN
+ + " \u2713 "
+ + Fore.RESET
+ )
else:
# Fail
@@ -122,33 +140,45 @@ def getResultString(translatedData, translationTime, filename):
except Exception as e:
traceback.print_exc()
errorString = str(e) + Fore.RED
- return filename + ': ' + totalTokenstring + timeString + Fore.RED + u' \u2717 ' +\
- errorString + Fore.RESET
+ return (
+ filename
+ + ": "
+ + totalTokenstring
+ + timeString
+ + Fore.RED
+ + " \u2717 "
+ + errorString
+ + Fore.RESET
+ )
+
def openFiles(filename):
- with open('files/' + filename, 'r', encoding='cp932') as readFile:
+ with open("files/" + filename, "r", encoding="cp932") as readFile:
translatedData = parseTyrano(readFile, filename)
# Delete lines marked for deletion
finalData = []
for line in translatedData[0]:
- if line != '\\d\n':
+ if line != "\\d\n":
finalData.append(line)
translatedData[0] = finalData
-
+
return translatedData
+
def parseTyrano(readFile, filename):
- totalTokens = [0,0]
+ totalTokens = [0, 0]
totalLines = 0
# Get total for progress bar
data = readFile.readlines()
totalLines = len(data)
- with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar:
- pbar.desc=filename
- pbar.total=totalLines
+ with tqdm(
+ bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE
+ ) as pbar:
+ pbar.desc = filename
+ pbar.total = totalLines
try:
result = translateTyrano(data, pbar, totalLines)
@@ -159,13 +189,14 @@ def parseTyrano(readFile, filename):
return [data, totalTokens, e]
return [data, totalTokens, None]
+
def translateTyrano(data, pbar, totalLines):
textHistory = []
batch = []
currentGroup = []
maxHistory = MAXHISTORY
- tokens = [0,0]
- speaker = ''
+ tokens = [0, 0]
+ speaker = ""
insertBool = False
global LOCK, ESTIMATE
i = 0
@@ -173,108 +204,118 @@ def translateTyrano(data, pbar, totalLines):
while i < len(data):
# Speaker
- if '[ns]' in data[i]:
- matchList = re.findall(r'\[ns\](.+?)\[', data[i])
+ if "[ns]" in data[i]:
+ matchList = re.findall(r"\[ns\](.+?)\[", data[i])
if len(matchList) != 0:
response = getSpeaker(matchList[0])
speaker = response[0]
tokens[0] += response[1][0]
tokens[1] += response[1][1]
- data[i] = '[ns]' + speaker + '[nse]\n'
+ data[i] = "[ns]" + speaker + "[nse]\n"
else:
- speaker = ''
+ speaker = ""
# Choices
- elif '[sel' in data[i]:
+ elif "[sel" in data[i]:
matchList = re.findall(r'\[sel.+text="(.+?)".+', data[i])
if len(matchList) != 0:
originalText = matchList[0]
if len(textHistory) > 0:
- response = translateGPT(matchList[0], 'Keep your translation as brief as possible. Previous text for context: ' + textHistory[len(textHistory)-1] + '\n\nReply in the style of a dialogue option.', False)
+ response = translateGPT(
+ matchList[0],
+ "Keep your translation as brief as possible. Previous text for context: "
+ + textHistory[len(textHistory) - 1]
+ + "\n\nReply in the style of a dialogue option.",
+ False,
+ )
else:
- response = translateGPT(matchList[0], '\n\nReply in the style of a dialogue option.', False)
+ response = translateGPT(
+ matchList[0],
+ "\n\nReply in the style of a dialogue option.",
+ False,
+ )
translatedText = response[0]
tokens[0] += response[1][0]
tokens[1] += response[1][1]
# Remove characters that may break scripts
- charList = ['.', '\"', '\\n']
+ charList = [".", '"', "\\n"]
for char in charList:
- translatedText = translatedText.replace(char, '')
+ translatedText = translatedText.replace(char, "")
# Escape all '
- translatedText = translatedText.replace('\\', '')
+ translatedText = translatedText.replace("\\", "")
# translatedText = translatedText.replace("'", "\\\'")
# Set Data
translatedText = data[i].replace(originalText, translatedText)
- data[i] = translatedText
+ data[i] = translatedText
# Lines
- matchList = re.findall(r'(.+?)\[[rpcms_sel]+\]$', data[i])
+ matchList = re.findall(r"(.+?)\[[rpcms_sel]+\]$", data[i])
if len(matchList) > 0:
- if 'hisout' in matchList[0]:
+ if "hisout" in matchList[0]:
i += 1
continue
currentGroup.append(matchList[0])
- if len(data) > i+1:
- while '[r]' in data[i+1]:
+ if len(data) > i + 1:
+ while "[r]" in data[i + 1]:
if insertBool is True:
- data[i] = r'\d\n'
+ data[i] = r"\d\n"
pbar.update(1)
i += 1
- matchList = re.findall(r'(.+?)\[r\]', data[i])
+ matchList = re.findall(r"(.+?)\[r\]", data[i])
if len(matchList) > 0:
currentGroup.append(matchList[0])
- while '[pcms]' in data[i+1]:
+ while "[pcms]" in data[i + 1]:
if insertBool is True:
- data[i] = r'\d\n'
+ data[i] = r"\d\n"
pbar.update(1)
i += 1
- matchList = re.findall(r'(.+?)\[pcms\]', data[i])
+ matchList = re.findall(r"(.+?)\[pcms\]", data[i])
if len(matchList) > 0:
currentGroup.append(matchList[0])
- while '[pcms_sel]' in data[i+1]:
+ while "[pcms_sel]" in data[i + 1]:
if insertBool is True:
- data[i] = r'\d\n'
+ data[i] = r"\d\n"
pbar.update(1)
i += 1
- matchList = re.findall(r'(.+?)\[pcms_sel\]', data[i])
+ matchList = re.findall(r"(.+?)\[pcms_sel\]", data[i])
if len(matchList) > 0:
currentGroup.append(matchList[0])
# Join up 401 groups for better translation.
if len(currentGroup) > 0:
- finalJAString = ' '.join(currentGroup)
+ finalJAString = " ".join(currentGroup)
oldjaString = finalJAString
# Remove any textwrap
if FIXTEXTWRAP == True:
- finalJAString = finalJAString.replace('[r]', ' ')
+ finalJAString = finalJAString.replace("[r]", " ")
# Remove Extra Stuff bad for translation.
- finalJAString = finalJAString.replace('ﾞ', '')
- finalJAString = finalJAString.replace('・', '.')
- finalJAString = finalJAString.replace('‶', '')
- finalJAString = finalJAString.replace('”', '')
- finalJAString = finalJAString.replace('―', '-')
- finalJAString = finalJAString.replace('…', '...')
- finalJAString = re.sub(r'(\.{3}\.+)', '...', finalJAString)
- finalJAString = finalJAString.replace(' ', ' ')
+ finalJAString = finalJAString.replace("ﾞ", "")
+ finalJAString = finalJAString.replace("・", ".")
+ finalJAString = finalJAString.replace("‶", "")
+ finalJAString = finalJAString.replace("”", "")
+ finalJAString = finalJAString.replace("―", "-")
+ finalJAString = finalJAString.replace("…", "...")
+ finalJAString = re.sub(r"(\.{3}\.+)", "...", finalJAString)
+ finalJAString = finalJAString.replace(" ", " ")
# Furigana Removal
- matchList = re.findall(r'(\[ruby\stext=.+text=\"(.+)\"\])', finalJAString)
+ matchList = re.findall(r"(\[ruby\stext=.+text=\"(.+)\"\])", finalJAString)
if len(matchList) > 0:
finalJAString = finalJAString.replace(matchList[0][0], matchList[0][1])
# Add Speaker (If there is one)
- if speaker != '':
- finalJAString = f'{speaker}: {finalJAString}'
+ if speaker != "":
+ finalJAString = f"{speaker}: {finalJAString}"
# [Passthrough 1] Pulling From File
if insertBool is False:
# Append to List and Clear Values
batch.append(finalJAString)
- speaker = ''
+ speaker = ""
# Translate Batch if Full
if len(batch) == BATCHSIZE:
@@ -292,7 +333,7 @@ def translateTyrano(data, pbar, totalLines):
# Mismatch
else:
- pbar.write(f'Mismatch: {batchStartIndex} - {i}')
+ pbar.write(f"Mismatch: {batchStartIndex} - {i}")
MISMATCH.append(batch)
batchStartIndex = i
batch.clear()
@@ -306,31 +347,31 @@ def translateTyrano(data, pbar, totalLines):
else:
# Get Text
translatedText = translatedBatch[0]
- translatedText = translatedText.replace('\\"', '\"')
- translatedText = translatedText.replace('[', '(')
- translatedText = translatedText.replace(']', ')')
+ translatedText = translatedText.replace('\\"', '"')
+ translatedText = translatedText.replace("[", "(")
+ translatedText = translatedText.replace("]", ")")
# Remove added speaker
- translatedText = re.sub(r'^.+?:\s', '', translatedText)
+ translatedText = re.sub(r"^.+?:\s", "", translatedText)
# Textwrap
translatedText = textwrap.fill(translatedText, width=WIDTH)
- textList = translatedText.split('\n')
-
+ textList = translatedText.split("\n")
+
# Set Text
- data[i] = r'\d\n'
+ data[i] = r"\d\n"
for line in textList:
# Wordwrap Text
- if '[r]' not in line:
+ if "[r]" not in line:
line = textwrap.fill(line, width=WIDTH)
- line = line.replace('\n', '[r]')
-
+ line = line.replace("\n", "[r]")
+
# Set
- data.insert(i, line.strip() + '[r]\n')
- i+=1
- data[i-1] = data[i-1].replace('[r]', '[pcms]')
+ data.insert(i, line.strip() + "[r]\n")
+ i += 1
+ data[i - 1] = data[i - 1].replace("[r]", "[pcms]")
translatedBatch.pop(0)
- speaker = ''
+ speaker = ""
currentGroup = []
# If Batch is empty. Move on.
@@ -361,7 +402,7 @@ def translateTyrano(data, pbar, totalLines):
# Mismatch
else:
- pbar.write(f'Mismatch: {batchStartIndex} - {i}')
+ pbar.write(f"Mismatch: {batchStartIndex} - {i}")
MISMATCH.append(batch)
batchStartIndex = i
batch.clear()
@@ -369,92 +410,99 @@ def translateTyrano(data, pbar, totalLines):
currentGroup = []
return tokens
+
# Save some money and enter the character before translation
def getSpeaker(speaker):
match speaker:
- case '央':
- return ['Akira', [0,0]]
- case '累':
- return ['Rui', [0,0]]
- case '梨里':
- return ['Riri', [0,0]]
- case '純':
- return ['Jun', [0,0]]
- case '美鈴':
- return ['Misuzu', [0,0]]
- case '須田':
- return ['Suda', [0,0]]
- case '高橋':
- return ['Takahashi', [0,0]]
- case '勇二':
- return ['Yuuji', [0,0]]
+ case "央":
+ return ["Akira", [0, 0]]
+ case "累":
+ return ["Rui", [0, 0]]
+ case "梨里":
+ return ["Riri", [0, 0]]
+ case "純":
+ return ["Jun", [0, 0]]
+ case "美鈴":
+ return ["Misuzu", [0, 0]]
+ case "須田":
+ return ["Suda", [0, 0]]
+ case "高橋":
+ return ["Takahashi", [0, 0]]
+ case "勇二":
+ return ["Yuuji", [0, 0]]
case _:
- return translateGPT(speaker, 'Reply with only the '+ LANGUAGE +' translation of the NPC name.', False)
-
+ return translateGPT(
+ speaker,
+ "Reply with only the " + LANGUAGE + " translation of the NPC name.",
+ False,
+ )
+
+
def subVars(jaString):
- jaString = jaString.replace('\u3000', ' ')
+ jaString = jaString.replace("\u3000", " ")
# Nested
count = 0
- nestedList = re.findall(r'[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]', jaString)
+ nestedList = re.findall(r"[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]", jaString)
nestedList = set(nestedList)
if len(nestedList) != 0:
for icon in nestedList:
- jaString = jaString.replace(icon, '{Nested_' + str(count) + '}')
+ jaString = jaString.replace(icon, "{Nested_" + str(count) + "}")
count += 1
# Icons
count = 0
- iconList = re.findall(r'[\\]+[iIkKwWaA]+\[[0-9]+\]', jaString)
+ iconList = re.findall(r"[\\]+[iIkKwWaA]+\[[0-9]+\]", jaString)
iconList = set(iconList)
if len(iconList) != 0:
for icon in iconList:
- jaString = jaString.replace(icon, '{Ascii_' + str(count) + '}')
+ jaString = jaString.replace(icon, "{Ascii_" + str(count) + "}")
count += 1
# Colors
count = 0
- colorList = re.findall(r'[\\]+[cC]\[[0-9]+\]', jaString)
+ colorList = re.findall(r"[\\]+[cC]\[[0-9]+\]", jaString)
colorList = set(colorList)
if len(colorList) != 0:
for color in colorList:
- jaString = jaString.replace(color, '{Color_' + str(count) + '}')
+ jaString = jaString.replace(color, "{Color_" + str(count) + "}")
count += 1
# Names
count = 0
- nameList = re.findall(r'[\\]+[nN]\[.+?\]+', jaString)
+ nameList = re.findall(r"[\\]+[nN]\[.+?\]+", jaString)
nameList = set(nameList)
if len(nameList) != 0:
for name in nameList:
- jaString = jaString.replace(name, '{Noun_' + str(count) + '}')
+ jaString = jaString.replace(name, "{Noun_" + str(count) + "}")
count += 1
# Variables
count = 0
- varList = re.findall(r'[\\]+[vV]\[[0-9]+\]', jaString)
+ varList = re.findall(r"[\\]+[vV]\[[0-9]+\]", jaString)
varList = set(varList)
if len(varList) != 0:
for var in varList:
- jaString = jaString.replace(var, '{Var_' + str(count) + '}')
+ jaString = jaString.replace(var, "{Var_" + str(count) + "}")
count += 1
# Formatting
count = 0
- formatList = re.findall(r'[\\]+[\w]+\[.+?\]', jaString)
+ formatList = re.findall(r"[\\]+[\w]+\[.+?\]", jaString)
formatList = set(formatList)
if len(formatList) != 0:
for var in formatList:
- jaString = jaString.replace(var, '{FCode_' + str(count) + '}')
+ jaString = jaString.replace(var, "{FCode_" + str(count) + "}")
count += 1
# Put all lists in list and return
allList = [nestedList, iconList, colorList, nameList, varList, formatList]
return [jaString, allList]
+
def resubVars(translatedText, allList):
# Fix Spacing and ChatGPT Nonsense
- matchList = re.findall(r'\[\s?.+?\s?\]', translatedText)
+ matchList = re.findall(r"\[\s?.+?\s?\]", translatedText)
if len(matchList) > 0:
for match in matchList:
text = match.strip()
@@ -464,54 +512,58 @@ def resubVars(translatedText, allList):
count = 0
if len(allList[0]) != 0:
for var in allList[0]:
- translatedText = translatedText.replace('{Nested_' + str(count) + '}', var)
+ translatedText = translatedText.replace("{Nested_" + str(count) + "}", var)
count += 1
# Icons
count = 0
if len(allList[1]) != 0:
for var in allList[1]:
- translatedText = translatedText.replace('{Ascii_' + str(count) + '}', var)
+ translatedText = translatedText.replace("{Ascii_" + str(count) + "}", var)
count += 1
# Colors
count = 0
if len(allList[2]) != 0:
for var in allList[2]:
- translatedText = translatedText.replace('{Color_' + str(count) + '}', var)
+ translatedText = translatedText.replace("{Color_" + str(count) + "}", var)
count += 1
# Names
count = 0
if len(allList[3]) != 0:
for var in allList[3]:
- translatedText = translatedText.replace('{Noun_' + str(count) + '}', var)
+ translatedText = translatedText.replace("{Noun_" + str(count) + "}", var)
count += 1
# Vars
count = 0
if len(allList[4]) != 0:
for var in allList[4]:
- translatedText = translatedText.replace('{Var_' + str(count) + '}', var)
+ translatedText = translatedText.replace("{Var_" + str(count) + "}", var)
count += 1
-
+
# Formatting
count = 0
if len(allList[5]) != 0:
for var in allList[5]:
- translatedText = translatedText.replace('{FCode_' + str(count) + '}', var)
+ translatedText = translatedText.replace("{FCode_" + str(count) + "}", var)
count += 1
return translatedText
+
def batchList(input_list, batch_size):
if not isinstance(batch_size, int) or batch_size <= 0:
raise ValueError("batch_size must be a positive integer")
-
- return [input_list[i:i + batch_size] for i in range(0, len(input_list), batch_size)]
+
+ return [
+ input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)
+ ]
+
def createContext(fullPromptFlag, subbedT):
- characters = 'Game Characters:\n\
+ characters = "Game Characters:\n\
渋江 央 (Shibue Akira) - Male\n\
蘆名 累 (Ashina Rui) - Female\n\
清原 梨里 (Kiyohara Riri) - Female\n\
@@ -520,19 +572,23 @@ def createContext(fullPromptFlag, subbedT):
須田 (Suda) - Male\n\
高橋 (Takahashi) - Female\n\
勇二 (Yuuji) - Male\n\
-'
-
- system = PROMPT + VOCAB if fullPromptFlag else \
- f"\
+"
+
+ system = (
+ PROMPT + VOCAB
+ if fullPromptFlag
+ else f"\
You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
You are going to be translating text from a videogame.\n\
I will give you lines of text, and you must translate each line to the best of your ability.\n\
{VOCAB}\n\
Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\
"
- user = f'{subbedT}'
+ )
+ user = f"{subbedT}"
return characters, system, user
+
def translateText(characters, system, user, history):
# Prompt
msg = [{"role": "system", "content": system + characters}]
@@ -545,9 +601,9 @@ def translateText(characters, system, user, history):
msg.extend([{"role": "system", "content": h} for h in history])
else:
msg.append({"role": "system", "content": history})
-
+
# Content to TL
- msg.append({"role": "user", "content": f'{user}'})
+ msg.append({"role": "user", "content": f"{user}"})
response = openai.chat.completions.create(
temperature=0.1,
frequency_penalty=0.1,
@@ -557,37 +613,44 @@ def translateText(characters, system, user, history):
)
return response
+
def cleanTranslatedText(translatedText, varResponse):
placeholders = {
- f'{LANGUAGE} Translation: ': '',
- 'Translation: ': '',
- 'っ': '',
- '〜': '~',
- 'ッ': '',
- '。': '.',
- 'Placeholder Text': ''
+ f"{LANGUAGE} Translation: ": "",
+ "Translation: ": "",
+ "っ": "",
+ "〜": "~",
+ "ッ": "",
+ "。": ".",
+ "Placeholder Text": "",
# Add more replacements as needed
}
for target, replacement in placeholders.items():
translatedText = translatedText.replace(target, replacement)
translatedText = resubVars(translatedText, varResponse[1])
- return [line for line in translatedText.replace('\\n', '\n').split('\n') if line]
+ return [line for line in translatedText.replace("\\n", "\n").split("\n") if line]
+
def extractTranslation(translatedTextList, is_list):
- pattern = r'`?([\\]*.*?[\\]*?)<\/?Line\d+>`?'
+ pattern = r"`?([\\]*.*?[\\]*?)<\/?Line\d+>`?"
# If it's a batch (i.e., list), extract with tags; otherwise, return the single item.
if is_list:
- return [re.findall(pattern, line)[0][1] for line in translatedTextList if re.search(pattern, line)]
+ return [
+ re.findall(pattern, line)[0][1]
+ for line in translatedTextList
+ if re.search(pattern, line)
+ ]
else:
matchList = re.findall(pattern, translatedTextList)
return matchList[0][1] if matchList else translatedTextList
+
def countTokens(characters, system, user, history):
inputTotalTokens = 0
outputTotalTokens = 0
- enc = tiktoken.encoding_for_model('gpt-4')
-
+ enc = tiktoken.encoding_for_model("gpt-4")
+
# Input
if isinstance(history, list):
for line in history:
@@ -599,15 +662,17 @@ def countTokens(characters, system, user, history):
inputTotalTokens += len(enc.encode(user))
# Output
- outputTotalTokens += round(len(enc.encode(user))*3)
+ outputTotalTokens += round(len(enc.encode(user)) * 3)
return [inputTotalTokens, outputTotalTokens]
+
def combineList(tlist, text):
if isinstance(text, list):
return [t for sublist in tlist for t in sublist]
return tlist[0]
+
@retry(exceptions=Exception, tries=5, delay=5)
def translateGPT(text, history, fullPromptFlag):
totalTokens = [0, 0]
@@ -619,8 +684,10 @@ def translateGPT(text, history, fullPromptFlag):
for index, tItem in enumerate(tList):
# Before sending to translation, if we have a list of items, add the formatting
if isinstance(tItem, list):
- payload = '\n'.join([f'`{item}`' for i, item in enumerate(tItem)])
- payload = payload.replace('``', '`Placeholder Text`')
+ payload = "\n".join(
+ [f"`{item}`" for i, item in enumerate(tItem)]
+ )
+ payload = payload.replace("``", "`Placeholder Text`")
varResponse = subVars(payload)
subbedT = varResponse[0]
else:
@@ -628,7 +695,7 @@ def translateGPT(text, history, fullPromptFlag):
subbedT = varResponse[0]
# Things to Check before starting translation
- if not re.search(r'[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+', subbedT):
+ if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", subbedT):
continue
# Create Message
@@ -653,11 +720,13 @@ def translateGPT(text, history, fullPromptFlag):
extractedTranslations = extractTranslation(translatedTextList, True)
tList[index] = extractedTranslations
if len(tItem) != len(translatedTextList):
- mismatch = True # Just here so breakpoint can be set
+ mismatch = True # Just here so breakpoint can be set
history = extractedTranslations[-10:] # Update history if we have a list
else:
# Ensure we're passing a single string to extractTranslation
- extractedTranslations = extractTranslation('\n'.join(translatedTextList), False)
+ extractedTranslations = extractTranslation(
+ "\n".join(translatedTextList), False
+ )
tList[index] = extractedTranslations
finalList = combineList(tList, text)
diff --git a/modules/lune.py b/modules/lune.py
index 879e839..ede0654 100644
--- a/modules/lune.py
+++ b/modules/lune.py
@@ -16,49 +16,50 @@ from tqdm import tqdm
# Open AI
load_dotenv()
-if os.getenv('api').replace(' ', '') != '':
- openai.base_url = os.getenv('api')
-openai.organization = os.getenv('org')
-openai.api_key = os.getenv('key')
+if os.getenv("api").replace(" ", "") != "":
+ openai.base_url = os.getenv("api")
+openai.organization = os.getenv("org")
+openai.api_key = os.getenv("key")
-#Globals
-MODEL = os.getenv('model')
-TIMEOUT = int(os.getenv('timeout'))
-LANGUAGE = os.getenv('language').capitalize()
-PROMPT = Path('prompt.txt').read_text(encoding='utf-8')
-VOCAB = Path('vocab.txt').read_text(encoding='utf-8')
-THREADS = int(os.getenv('threads'))
+# Globals
+MODEL = os.getenv("model")
+TIMEOUT = int(os.getenv("timeout"))
+LANGUAGE = os.getenv("language").capitalize()
+PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
+VOCAB = Path("vocab.txt").read_text(encoding="utf-8")
+THREADS = int(os.getenv("threads"))
LOCK = threading.Lock()
-WIDTH = int(os.getenv('width'))
-LISTWIDTH = int(os.getenv('listWidth'))
+WIDTH = int(os.getenv("width"))
+LISTWIDTH = int(os.getenv("listWidth"))
NOTEWIDTH = 70
MAXHISTORY = 10
-ESTIMATE = ''
+ESTIMATE = ""
TOKENS = [0, 0]
NAMESLIST = []
-NAMES = False # Output a list of all the character names found
-BRFLAG = False # If the game uses
instead
+NAMES = False # Output a list of all the character names found
+BRFLAG = False # If the game uses
instead
FIXTEXTWRAP = True # Overwrites textwrap
-IGNORETLTEXT = False # Ignores all translated text.
-MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
+IGNORETLTEXT = False # Ignores all translated text.
+MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
-#tqdm Globals
-BAR_FORMAT='{l_bar}{bar:10}{r_bar}{bar:-10b}'
+# tqdm Globals
+BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
POSITION = 0
LEAVE = False
# Pricing - Depends on the model https://openai.com/pricing
# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request
# If you are getting a MISMATCH LENGTH error, lower the batch size.
-if 'gpt-3.5' in MODEL:
- INPUTAPICOST = .002
- OUTPUTAPICOST = .002
+if "gpt-3.5" in MODEL:
+ INPUTAPICOST = 0.002
+ OUTPUTAPICOST = 0.002
BATCHSIZE = 10
-elif 'gpt-4' in MODEL:
- INPUTAPICOST = .01
- OUTPUTAPICOST = .03
+elif "gpt-4" in MODEL:
+ INPUTAPICOST = 0.01
+ OUTPUTAPICOST = 0.03
BATCHSIZE = 50
+
def handleLune(filename, estimate):
global ESTIMATE, totalTokens
ESTIMATE = estimate
@@ -74,11 +75,11 @@ def handleLune(filename, estimate):
TOKENS[0] += translatedData[1][0]
TOKENS[1] += translatedData[1][1]
- return getResultString(['', TOKENS, None], end - start, 'TOTAL')
-
+ return getResultString(["", TOKENS, None], end - start, "TOTAL")
+
else:
try:
- with open('translated/' + filename, 'w', encoding='UTF-8') as outFile:
+ with open("translated/" + filename, "w", encoding="UTF-8") as outFile:
start = time.time()
translatedData = openFiles(filename)
@@ -90,36 +91,50 @@ def handleLune(filename, estimate):
TOKENS[0] += translatedData[1][0]
TOKENS[1] += translatedData[1][1]
except Exception:
- return 'Fail'
+ return "Fail"
+
+ return getResultString(["", TOKENS, None], end - start, "TOTAL")
- return getResultString(['', TOKENS, None], end - start, 'TOTAL')
def openFiles(filename):
- with open('files/' + filename, 'r', encoding='UTF-8-sig') as f:
+ with open("files/" + filename, "r", encoding="UTF-8-sig") as f:
data = json.load(f)
# Map Files
- if '.json' in filename:
+ if ".json" in filename:
translatedData = parseJSON(data, filename)
else:
- raise NameError(filename + ' Not Supported')
-
+ raise NameError(filename + " Not Supported")
+
return translatedData
+
def getResultString(translatedData, translationTime, filename):
# File Print String
- totalTokenstring =\
- Fore.YELLOW +\
- '[Input: ' + str(translatedData[1][0]) + ']'\
- '[Output: ' + str(translatedData[1][1]) + ']'\
- '[Cost: ${:,.4f}'.format((translatedData[1][0] * .001 * INPUTAPICOST) +\
- (translatedData[1][1] * .001 * OUTPUTAPICOST)) + ']'
- timeString = Fore.BLUE + '[' + str(round(translationTime, 1)) + 's]'
+ totalTokenstring = (
+ Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
+ "[Output: "
+ + str(translatedData[1][1])
+ + "]" "[Cost: ${:,.4f}".format(
+ (translatedData[1][0] * 0.001 * INPUTAPICOST)
+ + (translatedData[1][1] * 0.001 * OUTPUTAPICOST)
+ )
+ + "]"
+ )
+ timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
if translatedData[2] == None:
# Success
- return filename + ': ' + totalTokenstring + timeString + Fore.GREEN + u' \u2713 ' + Fore.RESET
+ return (
+ filename
+ + ": "
+ + totalTokenstring
+ + timeString
+ + Fore.GREEN
+ + " \u2713 "
+ + Fore.RESET
+ )
else:
# Fail
@@ -128,18 +143,29 @@ def getResultString(translatedData, translationTime, filename):
except Exception as e:
traceback.print_exc()
errorString = str(e) + Fore.RED
- return filename + ': ' + totalTokenstring + timeString + Fore.RED + u' \u2717 ' +\
- errorString + Fore.RESET
-
+ return (
+ filename
+ + ": "
+ + totalTokenstring
+ + timeString
+ + Fore.RED
+ + " \u2717 "
+ + errorString
+ + Fore.RESET
+ )
+
+
def parseJSON(data, filename):
totalTokens = [0, 0]
totalLines = 0
totalLines = len(data)
global LOCK
-
- with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar:
- pbar.desc=filename
- pbar.total=totalLines
+
+ with tqdm(
+ bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE
+ ) as pbar:
+ pbar.desc = filename
+ pbar.total = totalLines
try:
result = translateJSON(data, pbar)
totalTokens[0] += result[0]
@@ -148,12 +174,13 @@ def parseJSON(data, filename):
return [data, totalTokens, e]
return [data, totalTokens, None]
+
def translateJSON(data, pbar):
textHistory = []
batch = []
maxHistory = MAXHISTORY
tokens = [0, 0]
- speaker = 'None'
+ speaker = "None"
insertBool = False
i = 0
batchStartIndex = 0
@@ -161,32 +188,41 @@ def translateJSON(data, pbar):
while i < len(data):
item = data[i]
# Speaker
- if 'name' in item:
- if item['name'] not in [None, '-']:
- response = getSpeaker(item['name'])
+ if "name" in item:
+ if item["name"] not in [None, "-"]:
+ response = getSpeaker(item["name"])
speaker = response[0]
tokens[0] += response[1][0]
tokens[1] += response[1][1]
- item['name'] = speaker
+ item["name"] = speaker
else:
- speaker = 'None'
+ speaker = "None"
# Text
- if 'message' in item:
- for text in ['text', 'text2', 'help1', 'help2', 'help3', 'like', 'message', 'me']:
+ if "message" in item:
+ for text in [
+ "text",
+ "text2",
+ "help1",
+ "help2",
+ "help3",
+ "like",
+ "message",
+ "me",
+ ]:
if text in item:
if item[text] != None:
jaString = item[text]
# Remove any textwrap
if FIXTEXTWRAP == True:
- finalJAString = jaString.replace('\n', ' ')
+ finalJAString = jaString.replace("\n", " ")
# [Passthrough 1] Pulling From File
if insertBool is False:
# Append to List and Clear Values
batch.append(finalJAString)
- speaker = ''
+ speaker = ""
# Translate Batch if Full
if len(batch) == BATCHSIZE:
@@ -204,7 +240,7 @@ def translateJSON(data, pbar):
# Mismatch
else:
- pbar.write(f'Mismatch: {batchStartIndex} - {i}')
+ pbar.write(f"Mismatch: {batchStartIndex} - {i}")
MISMATCH.append(batch)
batchStartIndex = i
batch.clear()
@@ -212,7 +248,7 @@ def translateJSON(data, pbar):
if insertBool is False:
pbar.update(1)
i += 1
-
+
currentGroup = []
# [Passthrough 2] Setting Data
@@ -221,15 +257,15 @@ def translateJSON(data, pbar):
translatedText = translatedBatch[0]
# Remove added speaker
- translatedText = re.sub(r'^.+?:\s', '', translatedText)
+ translatedText = re.sub(r"^.+?:\s", "", translatedText)
# Textwrap
translatedText = textwrap.fill(translatedText, width=WIDTH)
-
+
# Set Text
item[text] = translatedText
translatedBatch.pop(0)
- speaker = ''
+ speaker = ""
currentGroup = []
i += 1
@@ -258,92 +294,99 @@ def translateJSON(data, pbar):
# Mismatch
else:
- pbar.write(f'Mismatch: {batchStartIndex} - {i}')
+ pbar.write(f"Mismatch: {batchStartIndex} - {i}")
MISMATCH.append(batch)
batchStartIndex = i
batch.clear()
currentGroup = []
- return tokens
+ return tokens
+
# Save some money and enter the character before translation
def getSpeaker(speaker):
match speaker:
- case 'セレナ':
- return ['Serena', [0,0]]
- case 'レナ':
- return ['Rena', [0,0]]
- case 'フィルス':
- return ['Phils', [0,0]]
- case 'レイン':
- return ['Meryl', [0,0]]
+ case "セレナ":
+ return ["Serena", [0, 0]]
+ case "レナ":
+ return ["Rena", [0, 0]]
+ case "フィルス":
+ return ["Phils", [0, 0]]
+ case "レイン":
+ return ["Meryl", [0, 0]]
case _:
- return translateGPT(speaker, 'Reply with only the '+ LANGUAGE +' translation of the NPC name.', False)
+ return translateGPT(
+ speaker,
+ "Reply with only the " + LANGUAGE + " translation of the NPC name.",
+ False,
+ )
+
def subVars(jaString):
- jaString = jaString.replace('\u3000', ' ')
+ jaString = jaString.replace("\u3000", " ")
# Nested
count = 0
- nestedList = re.findall(r'[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]', jaString)
+ nestedList = re.findall(r"[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]", jaString)
nestedList = set(nestedList)
if len(nestedList) != 0:
for icon in nestedList:
- jaString = jaString.replace(icon, '{Nested_' + str(count) + '}')
+ jaString = jaString.replace(icon, "{Nested_" + str(count) + "}")
count += 1
# Icons
count = 0
- iconList = re.findall(r'[\\]+[iIkKwWaA]+\[[0-9]+\]', jaString)
+ iconList = re.findall(r"[\\]+[iIkKwWaA]+\[[0-9]+\]", jaString)
iconList = set(iconList)
if len(iconList) != 0:
for icon in iconList:
- jaString = jaString.replace(icon, '{Ascii_' + str(count) + '}')
+ jaString = jaString.replace(icon, "{Ascii_" + str(count) + "}")
count += 1
# Colors
count = 0
- colorList = re.findall(r'[\\]+[cC]\[[0-9]+\]', jaString)
+ colorList = re.findall(r"[\\]+[cC]\[[0-9]+\]", jaString)
colorList = set(colorList)
if len(colorList) != 0:
for color in colorList:
- jaString = jaString.replace(color, '{Color_' + str(count) + '}')
+ jaString = jaString.replace(color, "{Color_" + str(count) + "}")
count += 1
# Names
count = 0
- nameList = re.findall(r'[\\]+[nN]\[.+?\]+', jaString)
+ nameList = re.findall(r"[\\]+[nN]\[.+?\]+", jaString)
nameList = set(nameList)
if len(nameList) != 0:
for name in nameList:
- jaString = jaString.replace(name, '{Noun_' + str(count) + '}')
+ jaString = jaString.replace(name, "{Noun_" + str(count) + "}")
count += 1
# Variables
count = 0
- varList = re.findall(r'[\\]+[vV]\[[0-9]+\]', jaString)
+ varList = re.findall(r"[\\]+[vV]\[[0-9]+\]", jaString)
varList = set(varList)
if len(varList) != 0:
for var in varList:
- jaString = jaString.replace(var, '{Var_' + str(count) + '}')
+ jaString = jaString.replace(var, "{Var_" + str(count) + "}")
count += 1
# Formatting
count = 0
- formatList = re.findall(r'[\\]+[\w]+\[.+?\]', jaString)
+ formatList = re.findall(r"[\\]+[\w]+\[.+?\]", jaString)
formatList = set(formatList)
if len(formatList) != 0:
for var in formatList:
- jaString = jaString.replace(var, '{FCode_' + str(count) + '}')
+ jaString = jaString.replace(var, "{FCode_" + str(count) + "}")
count += 1
# Put all lists in list and return
allList = [nestedList, iconList, colorList, nameList, varList, formatList]
return [jaString, allList]
+
def resubVars(translatedText, allList):
# Fix Spacing and ChatGPT Nonsense
- matchList = re.findall(r'\[\s?.+?\s?\]', translatedText)
+ matchList = re.findall(r"\[\s?.+?\s?\]", translatedText)
if len(matchList) > 0:
for match in matchList:
text = match.strip()
@@ -353,54 +396,58 @@ def resubVars(translatedText, allList):
count = 0
if len(allList[0]) != 0:
for var in allList[0]:
- translatedText = translatedText.replace('{Nested_' + str(count) + '}', var)
+ translatedText = translatedText.replace("{Nested_" + str(count) + "}", var)
count += 1
# Icons
count = 0
if len(allList[1]) != 0:
for var in allList[1]:
- translatedText = translatedText.replace('{Ascii_' + str(count) + '}', var)
+ translatedText = translatedText.replace("{Ascii_" + str(count) + "}", var)
count += 1
# Colors
count = 0
if len(allList[2]) != 0:
for var in allList[2]:
- translatedText = translatedText.replace('{Color_' + str(count) + '}', var)
+ translatedText = translatedText.replace("{Color_" + str(count) + "}", var)
count += 1
# Names
count = 0
if len(allList[3]) != 0:
for var in allList[3]:
- translatedText = translatedText.replace('{Noun_' + str(count) + '}', var)
+ translatedText = translatedText.replace("{Noun_" + str(count) + "}", var)
count += 1
# Vars
count = 0
if len(allList[4]) != 0:
for var in allList[4]:
- translatedText = translatedText.replace('{Var_' + str(count) + '}', var)
+ translatedText = translatedText.replace("{Var_" + str(count) + "}", var)
count += 1
-
+
# Formatting
count = 0
if len(allList[5]) != 0:
for var in allList[5]:
- translatedText = translatedText.replace('{FCode_' + str(count) + '}', var)
+ translatedText = translatedText.replace("{FCode_" + str(count) + "}", var)
count += 1
return translatedText
+
def batchList(input_list, batch_size):
if not isinstance(batch_size, int) or batch_size <= 0:
raise ValueError("batch_size must be a positive integer")
-
- return [input_list[i:i + batch_size] for i in range(0, len(input_list), batch_size)]
+
+ return [
+ input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)
+ ]
+
def createContext(fullPromptFlag, subbedT):
- characters = 'Game Characters:\n\
+ characters = "Game Characters:\n\
林つかさ (Tsukasa Hayashi) - Female\n\
山田美兎 (Miyato Yamada) - Female\n\
鈴木赤音 (Akane Suzuki) - Female\n\
@@ -413,13 +460,17 @@ def createContext(fullPromptFlag, subbedT):
モリー・ボイド (Molly Boyd) - Female\n\
オルガ・ブヤチッチ (Olga Buyachich) - Female\n\
アッチャラー ギッティ (Atchara Gitti) - Female\n\
-'
-
- system = PROMPT if fullPromptFlag else \
- f'Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`'
- user = f'{subbedT}'
+"
+
+ system = (
+ PROMPT
+ if fullPromptFlag
+ else f"Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`"
+ )
+ user = f"{subbedT}"
return characters, system, user
+
def translateText(characters, system, user, history):
# Prompt
msg = [{"role": "system", "content": system + characters}]
@@ -432,9 +483,9 @@ def translateText(characters, system, user, history):
msg.extend([{"role": "assistant", "content": h} for h in history])
else:
msg.append({"role": "assistant", "content": history})
-
+
# Content to TL
- msg.append({"role": "user", "content": f'{user}'})
+ msg.append({"role": "user", "content": f"{user}"})
response = openai.chat.completions.create(
temperature=0.1,
frequency_penalty=0.1,
@@ -443,40 +494,47 @@ def translateText(characters, system, user, history):
)
return response
+
def cleanTranslatedText(translatedText, varResponse):
placeholders = {
- f'{LANGUAGE} Translation: ': '',
- 'Translation: ': '',
- 'っ': '',
- '〜': '~',
- 'ッ': '',
- '。': '.',
- 'Placeholder Text': ''
+ f"{LANGUAGE} Translation: ": "",
+ "Translation: ": "",
+ "っ": "",
+ "〜": "~",
+ "ッ": "",
+ "。": ".",
+ "Placeholder Text": "",
# Add more replacements as needed
}
for target, replacement in placeholders.items():
translatedText = translatedText.replace(target, replacement)
translatedText = resubVars(translatedText, varResponse[1])
- if '\n' in translatedText:
- return [line for line in translatedText.split('\n') if line]
+ if "\n" in translatedText:
+ return [line for line in translatedText.split("\n") if line]
else:
- return [line for line in translatedText.split('\\n') if line]
+ return [line for line in translatedText.split("\\n") if line]
+
def extractTranslation(translatedTextList, is_list):
- pattern = r'[\\]*`?(.*?)[\\]*?`?'
+ pattern = r"[\\]*`?(.*?)[\\]*?`?"
# If it's a batch (i.e., list), extract with tags; otherwise, return the single item.
if is_list:
- return [re.findall(pattern, line)[0][1] for line in translatedTextList if re.search(pattern, line)]
+ return [
+ re.findall(pattern, line)[0][1]
+ for line in translatedTextList
+ if re.search(pattern, line)
+ ]
else:
matchList = re.findall(pattern, translatedTextList)
return matchList[0][1] if matchList else translatedTextList
+
def countTokens(characters, system, user, history):
inputTotalTokens = 0
outputTotalTokens = 0
- enc = tiktoken.encoding_for_model('gpt-4')
-
+ enc = tiktoken.encoding_for_model("gpt-4")
+
# Input
if isinstance(history, list):
for line in history:
@@ -488,15 +546,17 @@ def countTokens(characters, system, user, history):
inputTotalTokens += len(enc.encode(user))
# Output
- outputTotalTokens += round(len(enc.encode(user))*3)
+ outputTotalTokens += round(len(enc.encode(user)) * 3)
return [inputTotalTokens, outputTotalTokens]
+
def combineList(tlist, text):
if isinstance(text, list):
return [t for sublist in tlist for t in sublist]
return tlist[0]
+
@retry(exceptions=Exception, tries=5, delay=5)
def translateGPT(text, history, fullPromptFlag):
totalTokens = [0, 0]
@@ -508,8 +568,10 @@ def translateGPT(text, history, fullPromptFlag):
for index, tItem in enumerate(tList):
# Before sending to translation, if we have a list of items, add the formatting
if isinstance(tItem, list):
- payload = '\n'.join([f'`{item}`' for i, item in enumerate(tItem)])
- payload = payload.replace('``', '`Placeholder Text`')
+ payload = "\n".join(
+ [f"`{item}`" for i, item in enumerate(tItem)]
+ )
+ payload = payload.replace("``", "`Placeholder Text`")
varResponse = subVars(payload)
subbedT = varResponse[0]
else:
@@ -517,7 +579,7 @@ def translateGPT(text, history, fullPromptFlag):
subbedT = varResponse[0]
# Things to Check before starting translation
- if not re.search(r'[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+', subbedT):
+ if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", subbedT):
continue
# Create Message
@@ -542,11 +604,13 @@ def translateGPT(text, history, fullPromptFlag):
extractedTranslations = extractTranslation(translatedTextList, True)
tList[index] = extractedTranslations
if len(tItem) != len(translatedTextList):
- mismatch = True # Just here so breakpoint can be set
+ mismatch = True # Just here so breakpoint can be set
history = extractedTranslations[-10:] # Update history if we have a list
else:
# Ensure we're passing a single string to extractTranslation
- extractedTranslations = extractTranslation('\n'.join(translatedTextList), False)
+ extractedTranslations = extractTranslation(
+ "\n".join(translatedTextList), False
+ )
tList[index] = extractedTranslations
finalList = combineList(tList, text)
diff --git a/modules/main.py b/modules/main.py
index 934f007..5bbd029 100644
--- a/modules/main.py
+++ b/modules/main.py
@@ -10,13 +10,27 @@ from dotenv import load_dotenv
# upon import, in which case if they are unset the script will crash before we can output these messages.
envMissing = False
load_dotenv()
-for env in ['api','key','organization','model','language','timeout','fileThreads','threads','width','listWidth']:
- if os.getenv(env) is None or str(os.getenv(env))[:1] == '<':
- tqdm.write(Fore.RED + f'Environment variable {env} is not set!')
+for env in [
+ "api",
+ "key",
+ "organization",
+ "model",
+ "language",
+ "timeout",
+ "fileThreads",
+ "threads",
+ "width",
+ "listWidth",
+]:
+ if os.getenv(env) is None or str(os.getenv(env))[:1] == "<":
+ tqdm.write(Fore.RED + f"Environment variable {env} is not set!")
envMissing = True
if envMissing:
- tqdm.write(Fore.RED + 'Some of the required environment values may not be set correctly. You can set \
-these values using an .env file, for an example see .env.example')
+ tqdm.write(
+ Fore.RED
+ + "Some of the required environment values may not be set correctly. You can set \
+these values using an .env file, for an example see .env.example"
+ )
from modules.rpgmakermvmz import handleMVMZ
from modules.rpgmakerace import handleACE
@@ -40,7 +54,7 @@ from modules.rpgmakerplugin import handlePlugin
# For GPT4 rate limit will be hit if you have more than 1 thread.
# 1 Thread for each file. Controls how many files are worked on at once.
-THREADS = int(os.getenv('fileThreads'))
+THREADS = int(os.getenv("fileThreads"))
# [Display name, file extension, handle function]
MODULES = [
@@ -66,59 +80,76 @@ MODULES = [
]
# Info Message
-tqdm.write(Fore.LIGHTYELLOW_EX + "WARNING: Once the translation starts do not close it unless you want to lose your \
+tqdm.write(
+ Fore.LIGHTYELLOW_EX
+ + "WARNING: Once the translation starts do not close it unless you want to lose your \
translated data. If a file fails or gets stuck, translated lines will remain translated so you don't have \
to worry about being charged twice. You can simply copy the file generated in /translations back over to \
-/files and start the script again. It will skip over any translated text." + Fore.RESET, end='\n\n')
+/files and start the script again. It will skip over any translated text."
+ + Fore.RESET,
+ end="\n\n",
+)
+
def main():
- estimate = ''
- while estimate == '':
- estimate = input('Select Translation or Cost Estimation:\n\n 1. Translate\n 2. Estimate\n')
+ estimate = ""
+ while estimate == "":
+ estimate = input(
+ "Select Translation or Cost Estimation:\n\n 1. Translate\n 2. Estimate\n"
+ )
match estimate:
- case '1':
+ case "1":
estimate = False
- case '2':
+ case "2":
estimate = True
case _:
- estimate = ''
-
- version = ''
+ estimate = ""
+
+ version = ""
while True:
tqdm.write("Select game engine:\n")
for position, module in enumerate(MODULES):
- tqdm.write(f'{str(position + 1).rjust(2)}. {module[0]} (.{module[1]})')
+ tqdm.write(f"{str(position + 1).rjust(2)}. {module[0]} (.{module[1]})")
version = input()
try:
version = int(version) - 1
except:
continue
if version in range(len(MODULES)):
- break
+ break
- totalCost = Fore.RED + 'Translation module didn\'t return the total cost. Make sure the \
-files to translate are in the /files folder and that you picked the right game engine.'
+ totalCost = (
+ Fore.RED
+ + "Translation module didn't return the total cost. Make sure the \
+files to translate are in the /files folder and that you picked the right game engine."
+ )
# Open File (Threads)
with ThreadPoolExecutor(max_workers=THREADS) as executor:
- futures = [executor.submit(MODULES[version][2], filename, estimate) \
- for filename in os.listdir("files") if filename.endswith(MODULES[version][1]) and filename != '.gitkeep']
+ futures = [
+ executor.submit(MODULES[version][2], filename, estimate)
+ for filename in os.listdir("files")
+ if filename.endswith(MODULES[version][1]) and filename != ".gitkeep"
+ ]
for future in as_completed(futures):
try:
totalCost = future.result()
except Exception as e:
- tracebackLineNo = str(traceback.extract_tb(sys.exc_info()[2])[-1].lineno)
- tqdm.write(Fore.RED + str(e) + '|' + tracebackLineNo + Fore.RESET)
+ tracebackLineNo = str(
+ traceback.extract_tb(sys.exc_info()[2])[-1].lineno
+ )
+ tqdm.write(Fore.RED + str(e) + "|" + tracebackLineNo + Fore.RESET)
- if totalCost != 'Fail':
+ if totalCost != "Fail":
if estimate is False:
# This is to encourage people to grab what's in /translated instead
- deleteFolderFiles('files')
+ deleteFolderFiles("files")
tqdm.write(str(totalCost))
+
def deleteFolderFiles(folderPath):
for filename in os.listdir(folderPath):
file_path = os.path.join(folderPath, filename)
- if file_path.endswith(('.json', '.yaml', '.ks')):
- os.remove(file_path)
+ if file_path.endswith((".json", ".yaml", ".ks")):
+ os.remove(file_path)
diff --git a/modules/nscript.py b/modules/nscript.py
index c22b700..2643b70 100644
--- a/modules/nscript.py
+++ b/modules/nscript.py
@@ -16,57 +16,58 @@ from tqdm import tqdm
# Open AI
load_dotenv()
-if os.getenv('api').replace(' ', '') != '':
- openai.base_url = os.getenv('api')
-openai.organization = os.getenv('org')
-openai.api_key = os.getenv('key')
+if os.getenv("api").replace(" ", "") != "":
+ openai.base_url = os.getenv("api")
+openai.organization = os.getenv("org")
+openai.api_key = os.getenv("key")
-#Globals
-MODEL = os.getenv('model')
-TIMEOUT = int(os.getenv('timeout'))
-LANGUAGE = os.getenv('language').capitalize()
-PROMPT = Path('prompt.txt').read_text(encoding='utf-8')
-VOCAB = Path('vocab.txt').read_text(encoding='utf-8')
-THREADS = int(os.getenv('threads'))
+# Globals
+MODEL = os.getenv("model")
+TIMEOUT = int(os.getenv("timeout"))
+LANGUAGE = os.getenv("language").capitalize()
+PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
+VOCAB = Path("vocab.txt").read_text(encoding="utf-8")
+THREADS = int(os.getenv("threads"))
LOCK = threading.Lock()
-WIDTH = int(os.getenv('width'))
-LISTWIDTH = int(os.getenv('listWidth'))
+WIDTH = int(os.getenv("width"))
+LISTWIDTH = int(os.getenv("listWidth"))
NOTEWIDTH = 70
MAXHISTORY = 10
-ESTIMATE = ''
+ESTIMATE = ""
TOKENS = [0, 0]
NAMESLIST = []
-NAMES = False # Output a list of all the character names found
-BRFLAG = False # If the game uses
instead
+NAMES = False # Output a list of all the character names found
+BRFLAG = False # If the game uses
instead
FIXTEXTWRAP = True # Overwrites textwrap
-IGNORETLTEXT = False # Ignores all translated text.
-MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
+IGNORETLTEXT = False # Ignores all translated text.
+MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
-#tqdm Globals
-BAR_FORMAT='{l_bar}{bar:10}{r_bar}{bar:-10b}'
+# tqdm Globals
+BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
POSITION = 0
LEAVE = False
PBAR = None
FILENAME = None
# Full Width
-ascii_to_wide = dict((i, chr(i + 0xfee0)) for i in range(0x21, 0x7f))
-ascii_to_wide.update({0x20: u'\u3000', 0x2D: u'\u2212'}) # space and minus
-wide_to_ascii = dict((i, chr(i - 0xfee0)) for i in range(0xff01, 0xff5f))
-wide_to_ascii.update({0x3000: u' ', 0x2212: u'-'}) # space and minus
+ascii_to_wide = dict((i, chr(i + 0xFEE0)) for i in range(0x21, 0x7F))
+ascii_to_wide.update({0x20: "\u3000", 0x2D: "\u2212"}) # space and minus
+wide_to_ascii = dict((i, chr(i - 0xFEE0)) for i in range(0xFF01, 0xFF5F))
+wide_to_ascii.update({0x3000: " ", 0x2212: "-"}) # space and minus
# Pricing - Depends on the model https://openai.com/pricing
# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request
# If you are getting a MISMATCH LENGTH error, lower the batch size.
-if 'gpt-3.5' in MODEL:
- INPUTAPICOST = .002
- OUTPUTAPICOST = .002
+if "gpt-3.5" in MODEL:
+ INPUTAPICOST = 0.002
+ OUTPUTAPICOST = 0.002
BATCHSIZE = 10
-elif 'gpt-4' in MODEL:
- INPUTAPICOST = .005
- OUTPUTAPICOST = .015
+elif "gpt-4" in MODEL:
+ INPUTAPICOST = 0.005
+ OUTPUTAPICOST = 0.015
BATCHSIZE = 40
+
def handleOnscripter(filename, estimate):
global ESTIMATE, FILENAME
ESTIMATE = estimate
@@ -84,17 +85,21 @@ def handleOnscripter(filename, estimate):
TOKENS[1] += translatedData[1][1]
# Print Total
- totalString = getResultString(['', TOKENS, None], end - start, 'TOTAL')
+ totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
# Print any errors on maps
if len(MISMATCH) > 0:
- return totalString + Fore.RED + f'\nMismatch Errors: {MISMATCH}' + Fore.RESET
+ return (
+ totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
+ )
else:
return totalString
-
+
else:
try:
- with open('translated/' + filename, 'w', encoding='cp932', errors='ignore') as outFile:
+ with open(
+ "translated/" + filename, "w", encoding="cp932", errors="ignore"
+ ) as outFile:
start = time.time()
translatedData = openFiles(filename)
@@ -107,23 +112,36 @@ def handleOnscripter(filename, estimate):
TOKENS[1] += translatedData[1][1]
except Exception:
traceback.print_exc()
- return 'Fail'
+ return "Fail"
+
+ return getResultString(["", TOKENS, None], end - start, "TOTAL")
- return getResultString(['', TOKENS, None], end - start, 'TOTAL')
def getResultString(translatedData, translationTime, filename):
# File Print String
- totalTokenstring =\
- Fore.YELLOW +\
- '[Input: ' + str(translatedData[1][0]) + ']'\
- '[Output: ' + str(translatedData[1][1]) + ']'\
- '[Cost: ${:,.4f}'.format((translatedData[1][0] * .001 * INPUTAPICOST) +\
- (translatedData[1][1] * .001 * OUTPUTAPICOST)) + ']'
- timeString = Fore.BLUE + '[' + str(round(translationTime, 1)) + 's]'
+ totalTokenstring = (
+ Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
+ "[Output: "
+ + str(translatedData[1][1])
+ + "]" "[Cost: ${:,.4f}".format(
+ (translatedData[1][0] * 0.001 * INPUTAPICOST)
+ + (translatedData[1][1] * 0.001 * OUTPUTAPICOST)
+ )
+ + "]"
+ )
+ timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
if translatedData[2] == None:
# Success
- return filename + ': ' + totalTokenstring + timeString + Fore.GREEN + u' \u2713 ' + Fore.RESET
+ return (
+ filename
+ + ": "
+ + totalTokenstring
+ + timeString
+ + Fore.GREEN
+ + " \u2713 "
+ + Fore.RESET
+ )
else:
# Fail
@@ -132,31 +150,41 @@ def getResultString(translatedData, translationTime, filename):
except Exception as e:
traceback.print_exc()
errorString = str(e) + Fore.RED
- return filename + ': ' + totalTokenstring + timeString + Fore.RED + u' \u2717 ' +\
- errorString + Fore.RESET
+ return (
+ filename
+ + ": "
+ + totalTokenstring
+ + timeString
+ + Fore.RED
+ + " \u2717 "
+ + errorString
+ + Fore.RESET
+ )
+
def openFiles(filename):
- with open('files/' + filename, 'r', encoding='cp932') as readFile:
+ with open("files/" + filename, "r", encoding="cp932") as readFile:
translatedData = parseOnscripter(readFile, filename)
# Delete lines marked for deletion
finalData = []
for line in translatedData[0]:
- if line != '\\d\n':
+ if line != "\\d\n":
finalData.append(line)
translatedData[0] = finalData
-
+
return translatedData
+
def parseOnscripter(readFile, filename):
- totalTokens = [0,0]
+ totalTokens = [0, 0]
# Read File into data
data = readFile.readlines()
# Create Progress Bar
with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
- pbar.desc=filename
+ pbar.desc = filename
try:
result = translateOnscripter(data, pbar, filename, [])
@@ -167,11 +195,12 @@ def parseOnscripter(readFile, filename):
return [data, totalTokens, e]
return [data, totalTokens, None]
+
def translateOnscripter(data, pbar, filename, translatedList):
stringList = []
currentGroup = []
- tokens = [0,0]
- speaker = ''
+ tokens = [0, 0]
+ speaker = ""
voice = False
global LOCK, ESTIMATE, PBAR
PBAR = pbar
@@ -180,38 +209,38 @@ def translateOnscripter(data, pbar, filename, translatedList):
# Dialogue
while i < len(data):
# Lines
- regex = r'^([\u3000「（【[][^\n]+)'
+ regex = r"^([\u3000「（【[][^\n]+)"
match = re.search(regex, data[i])
- if match != None and match.group(1) != '':
+ if match != None and match.group(1) != "":
originalString = match.group(1)
# Pass 1
if translatedList == []:
# Grab Consecutive Strings
jaString = match.group(1)
- while len(data) > i+1 and re.match(regex, data[i+1]):
- data[i] = ''
+ while len(data) > i + 1 and re.match(regex, data[i + 1]):
+ data[i] = ""
i += 1
- jaString = f'{jaString} {data[i]}'
+ jaString = f"{jaString} {data[i]}"
# Convert from Wide
jaString = jaString.translate(wide_to_ascii)
-
+
# Remove any textwrap and \u3000 and \
- jaString = jaString.replace('\n', '')
- jaString = jaString.replace('\u3000', '')
- jaString = jaString.replace('\\', '')
- jaString = jaString.replace(' ＞', ')')
- jaString = jaString.replace('＜ ', '(')
+ jaString = jaString.replace("\n", "")
+ jaString = jaString.replace("\u3000", "")
+ jaString = jaString.replace("\\", "")
+ jaString = jaString.replace(" ＞", ")")
+ jaString = jaString.replace("＜ ", "(")
# Remove Furigana
- furiMatch = re.findall(r'({(.+?)\/(.+?)})', jaString)
+ furiMatch = re.findall(r"({(.+?)\/(.+?)})", jaString)
if furiMatch:
for match in furiMatch:
jaString = jaString.replace(match[0], match[2])
# Add String
stringList.append(jaString.strip())
-
+
# Pass 2
else:
# Get Text
@@ -226,24 +255,24 @@ def translateOnscripter(data, pbar, filename, translatedList):
# Textwrap & Other Text
translatedText = textwrap.fill(translatedText, width=WIDTH)
- translatedText = translatedText.replace('\n', '\n\u3000')
+ translatedText = translatedText.replace("\n", "\n\u3000")
# Split the string into lines
- lines = translatedText.split('\n')
-
+ lines = translatedText.split("\n")
+
# Add a backslash after every 3rd line
j = 0
while j < len(lines):
if j == 4:
- lines[j-1] = f'{lines[j-1]}\\'
- lines[j] = f'\n{lines[j]}'
+ lines[j - 1] = f"{lines[j-1]}\\"
+ lines[j] = f"\n{lines[j]}"
j += 1
-
+
# Join the lines back into a single string
- translatedText = '\n'.join(lines)
+ translatedText = "\n".join(lines)
# Remove Double Spaces
- translatedText = translatedText.replace(' ', ' ')
+ translatedText = translatedText.replace(" ", " ")
# Convert to Wide
translatedText = translatedText.translate(ascii_to_wide)
@@ -252,18 +281,20 @@ def translateOnscripter(data, pbar, filename, translatedList):
translatedText = fixText(translatedText)
# Set Data
- data[i] = data[i].replace(originalString, f'{translatedText}')
+ data[i] = data[i].replace(originalString, f"{translatedText}")
i += 1
# Choices
- elif 'csel' in data[i] and translatedList != []:
+ elif "csel" in data[i] and translatedList != []:
choiceList = []
jaString = data[i]
- choiceList = re.findall(r'\"(.*?)\"', jaString)
+ choiceList = re.findall(r"\"(.*?)\"", jaString)
if len(choiceList) > 0:
# Translate
- response = translateGPT(choiceList, 'This will be a dialogue option', True)
+ response = translateGPT(
+ choiceList, "This will be a dialogue option", True
+ )
translatedTextList = response[0]
tokens[0] += response[1][0]
tokens[1] += response[1][1]
@@ -286,9 +317,9 @@ def translateOnscripter(data, pbar, filename, translatedList):
# Set Progress
pbar.total = len(stringList)
pbar.refresh()
-
+
# Translate
- response = translateGPT(stringList, '', True)
+ response = translateGPT(stringList, "", True)
tokens[0] += response[1][0]
tokens[1] += response[1][1]
translatedList = response[0]
@@ -304,54 +335,72 @@ def translateOnscripter(data, pbar, filename, translatedList):
MISMATCH.append(filename)
return tokens
+
def fixText(translatedText):
# Add Break
- translatedText = translatedText.replace('\"', '\'')
- translatedText = f'\u3000{translatedText}\\'
+ translatedText = translatedText.replace('"', "'")
+ translatedText = f"\u3000{translatedText}\\"
# Unconvert Codes
- matchList = re.findall(r'([＄].+?)[^\w]', translatedText)
+ matchList = re.findall(r"([＄].+?)[^\w]", translatedText)
if matchList:
for match in matchList:
- translatedText = translatedText.replace(match, match.translate(wide_to_ascii))
+ translatedText = translatedText.replace(
+ match, match.translate(wide_to_ascii)
+ )
# Unconvert Color Codes
- matchList = re.findall(r'([＃][\w\d]{6})', translatedText)
+ matchList = re.findall(r"([＃][\w\d]{6})", translatedText)
if matchList:
for match in matchList:
- translatedText = translatedText.replace(match, match.translate(wide_to_ascii))
+ translatedText = translatedText.replace(
+ match, match.translate(wide_to_ascii)
+ )
# Unconvert Variables
- matchList = re.findall(r'([％]\w.+?)[^\w＿]', translatedText)
+ matchList = re.findall(r"([％]\w.+?)[^\w＿]", translatedText)
if matchList:
for match in matchList:
- translatedText = translatedText.replace(match, match.translate(wide_to_ascii))
+ translatedText = translatedText.replace(
+ match, match.translate(wide_to_ascii)
+ )
# Unconvert Backslashes
- matchList = re.findall(r'＼', translatedText)
+ matchList = re.findall(r"＼", translatedText)
if matchList:
for match in matchList:
- translatedText = translatedText.replace(match, match.translate(wide_to_ascii))
+ translatedText = translatedText.replace(
+ match, match.translate(wide_to_ascii)
+ )
return translatedText
+
# Save some money and enter the character before translation
def getSpeaker(speaker):
match speaker:
- case 'ファイン':
- return ['Fine', [0,0]]
- case '':
- return ['', [0,0]]
+ case "ファイン":
+ return ["Fine", [0, 0]]
+ case "":
+ return ["", [0, 0]]
case _:
# Store Speaker
if speaker not in str(NAMESLIST):
- response = translateGPT(speaker, 'Reply with the '+ LANGUAGE +' translation of the NPC name.', False)
+ response = translateGPT(
+ speaker,
+ "Reply with the " + LANGUAGE + " translation of the NPC name.",
+ False,
+ )
response[0] = response[0].title()
response[0] = response[0].replace("'S", "'s")
# Retry if name doesn't translate for some reason
- if re.search(r'([a-zA-Z？?])', response[0]) == None:
- response = translateGPT(speaker, 'Reply with the '+ LANGUAGE +' translation of the NPC name.', False)
+ if re.search(r"([a-zA-Z？?])", response[0]) == None:
+ response = translateGPT(
+ speaker,
+ "Reply with the " + LANGUAGE + " translation of the NPC name.",
+ False,
+ )
response[0] = response[0].title()
response[0] = response[0].replace("'S", "'s")
@@ -362,74 +411,76 @@ def getSpeaker(speaker):
else:
for i in range(len(NAMESLIST)):
if speaker == NAMESLIST[i][0]:
- return [NAMESLIST[i][1],[0,0]]
-
- return [speaker,[0,0]]
+ return [NAMESLIST[i][1], [0, 0]]
+
+ return [speaker, [0, 0]]
+
def subVars(jaString):
- jaString = jaString.replace('\u3000', ' ')
+ jaString = jaString.replace("\u3000", " ")
# Nested
count = 0
- nestedList = re.findall(r'[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]', jaString)
+ nestedList = re.findall(r"[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]", jaString)
nestedList = set(nestedList)
if len(nestedList) != 0:
for icon in nestedList:
- jaString = jaString.replace(icon, '[Nested_' + str(count) + ']')
+ jaString = jaString.replace(icon, "[Nested_" + str(count) + "]")
count += 1
# Icons
count = 0
- iconList = re.findall(r'[\\]+[iIkKwWaA]+\[[0-9]+\]', jaString)
+ iconList = re.findall(r"[\\]+[iIkKwWaA]+\[[0-9]+\]", jaString)
iconList = set(iconList)
if len(iconList) != 0:
for icon in iconList:
- jaString = jaString.replace(icon, '[Ascii_' + str(count) + ']')
+ jaString = jaString.replace(icon, "[Ascii_" + str(count) + "]")
count += 1
# Colors
count = 0
- colorList = re.findall(r'([\\]+c\[\d+\][\\]+c|[\\]+c\[\d+\])', jaString)
+ colorList = re.findall(r"([\\]+c\[\d+\][\\]+c|[\\]+c\[\d+\])", jaString)
colorList = set(colorList)
if len(colorList) != 0:
for color in colorList:
- jaString = jaString.replace(color, '[Color_' + str(count) + ']')
+ jaString = jaString.replace(color, "[Color_" + str(count) + "]")
count += 1
# Names
count = 0
- nameList = re.findall(r'[\\]+[nN]\[.+?\]+', jaString)
+ nameList = re.findall(r"[\\]+[nN]\[.+?\]+", jaString)
nameList = set(nameList)
if len(nameList) != 0:
for name in nameList:
- jaString = jaString.replace(name, '[Noun_' + str(count) + ']')
+ jaString = jaString.replace(name, "[Noun_" + str(count) + "]")
count += 1
# Variables
count = 0
- varList = re.findall(r'[\\]+[vV]\[[0-9]+\]', jaString)
+ varList = re.findall(r"[\\]+[vV]\[[0-9]+\]", jaString)
varList = set(varList)
if len(varList) != 0:
for var in varList:
- jaString = jaString.replace(var, '[Var_' + str(count) + ']')
+ jaString = jaString.replace(var, "[Var_" + str(count) + "]")
count += 1
# Formatting
count = 0
- formatList = re.findall(r'[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]', jaString)
+ formatList = re.findall(r"[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]", jaString)
formatList = set(formatList)
if len(formatList) != 0:
for var in formatList:
- jaString = jaString.replace(var, '[FCode_' + str(count) + ']')
+ jaString = jaString.replace(var, "[FCode_" + str(count) + "]")
count += 1
# Put all lists in list and return
allList = [nestedList, iconList, colorList, nameList, varList, formatList]
return [jaString, allList]
+
def resubVars(translatedText, allList):
# Fix Spacing and ChatGPT Nonsense
- matchList = re.findall(r'\[\s?.+?\s?\]', translatedText)
+ matchList = re.findall(r"\[\s?.+?\s?\]", translatedText)
if len(matchList) > 0:
for match in matchList:
text = match.strip()
@@ -439,54 +490,58 @@ def resubVars(translatedText, allList):
count = 0
if len(allList[0]) != 0:
for var in allList[0]:
- translatedText = translatedText.replace('[Nested_' + str(count) + ']', var)
+ translatedText = translatedText.replace("[Nested_" + str(count) + "]", var)
count += 1
# Icons
count = 0
if len(allList[1]) != 0:
for var in allList[1]:
- translatedText = translatedText.replace('[Ascii_' + str(count) + ']', var)
+ translatedText = translatedText.replace("[Ascii_" + str(count) + "]", var)
count += 1
# Colors
count = 0
if len(allList[2]) != 0:
for var in allList[2]:
- translatedText = translatedText.replace('[Color_' + str(count) + ']', var)
+ translatedText = translatedText.replace("[Color_" + str(count) + "]", var)
count += 1
# Names
count = 0
if len(allList[3]) != 0:
for var in allList[3]:
- translatedText = translatedText.replace('[Noun_' + str(count) + ']', var)
+ translatedText = translatedText.replace("[Noun_" + str(count) + "]", var)
count += 1
# Vars
count = 0
if len(allList[4]) != 0:
for var in allList[4]:
- translatedText = translatedText.replace('[Var_' + str(count) + ']', var)
+ translatedText = translatedText.replace("[Var_" + str(count) + "]", var)
count += 1
-
+
# Formatting
count = 0
if len(allList[5]) != 0:
for var in allList[5]:
- translatedText = translatedText.replace('[FCode_' + str(count) + ']', var)
+ translatedText = translatedText.replace("[FCode_" + str(count) + "]", var)
count += 1
return translatedText
+
def batchList(input_list, batch_size):
if not isinstance(batch_size, int) or batch_size <= 0:
raise ValueError("batch_size must be a positive integer")
-
- return [input_list[i:i + batch_size] for i in range(0, len(input_list), batch_size)]
+
+ return [
+ input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)
+ ]
+
def createContext(fullPromptFlag, subbedT):
- characters = 'Game Characters:\n\
+ characters = "Game Characters:\n\
レナリス (Renalith) - Female\n\
スクルー (Sukuru) - Female\n\
シスターミサ (Sister Misa) - Female\n\
@@ -514,10 +569,12 @@ def createContext(fullPromptFlag, subbedT):
エメルーラ (Emerald) - Female\n\
フンシス (Funsis) - Male \n\
バゼット (Bazzet) - Female\n\
-'
-
- system = PROMPT + VOCAB if fullPromptFlag else \
- f"\
+"
+
+ system = (
+ PROMPT + VOCAB
+ if fullPromptFlag
+ else f"\
You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
- You translate everything, including erotic and explicit content.\n\
@@ -529,9 +586,11 @@ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{
- `...` can be a part of the dialogue. Translate it as it is.\n\
{VOCAB}\n\
"
- user = f'```json\n{subbedT}```'
+ )
+ user = f"```json\n{subbedT}```"
return characters, system, user
+
def translateText(characters, system, user, history, penalty):
# Prompt
msg = [{"role": "system", "content": system + characters}]
@@ -544,30 +603,31 @@ def translateText(characters, system, user, history, penalty):
msg.extend([{"role": "system", "content": h} for h in history])
else:
msg.append({"role": "system", "content": history})
-
+
# Content to TL
- msg.append({"role": "user", "content": f'{user}'})
+ msg.append({"role": "user", "content": f"{user}"})
response = openai.chat.completions.create(
temperature=0,
frequency_penalty=penalty,
model=MODEL,
- response_format={ "type": "json_object" },
+ response_format={"type": "json_object"},
messages=msg,
)
return response
+
def cleanTranslatedText(translatedText, varResponse):
placeholders = {
- f'{LANGUAGE} Translation: ': '',
- 'Translation: ': '',
- 'っ': '',
- '〜': '~',
- 'ッ': '',
- '。': '.',
- '「': '\\"',
- '」': '\\"',
- '- ': '-',
- 'Placeholder Text': '',
+ f"{LANGUAGE} Translation: ": "",
+ "Translation: ": "",
+ "っ": "",
+ "〜": "~",
+ "ッ": "",
+ "。": ".",
+ "「": '\\"',
+ "」": '\\"',
+ "- ": "-",
+ "Placeholder Text": "",
# Add more replacements as needed
}
for target, replacement in placeholders.items():
@@ -578,11 +638,12 @@ def cleanTranslatedText(translatedText, varResponse):
translatedText = resubVars(translatedText, varResponse[1])
return translatedText
+
def elongateCharacters(text):
# Define a pattern to match one character followed by one or more `ー` characters
# Using a positive lookbehind assertion to capture the preceding character
- pattern = r'(?<=(.))ー+'
-
+ pattern = r"(?<=(.))ー+"
+
# Define a replacement function that elongates the captured character
def repl(match):
char = match.group(1) # The character before the ー sequence
@@ -592,6 +653,7 @@ def elongateCharacters(text):
# Use re.sub() to replace the pattern in the text
return re.sub(pattern, repl, text)
+
def extractTranslation(translatedTextList, is_list):
try:
line_dict = json.loads(translatedTextList)
@@ -603,11 +665,12 @@ def extractTranslation(translatedTextList, is_list):
print(e)
return translatedTextList
+
def countTokens(characters, system, user, history):
inputTotalTokens = 0
outputTotalTokens = 0
- enc = tiktoken.encoding_for_model('gpt-4')
-
+ enc = tiktoken.encoding_for_model("gpt-4")
+
# Input
if isinstance(history, list):
for line in history:
@@ -619,19 +682,21 @@ def countTokens(characters, system, user, history):
inputTotalTokens += len(enc.encode(user))
# Output
- outputTotalTokens += round(len(enc.encode(user))*3)
+ outputTotalTokens += round(len(enc.encode(user)) * 3)
return [inputTotalTokens, outputTotalTokens]
+
def combineList(tlist, text):
if isinstance(text, list):
return [t for sublist in tlist for t in sublist]
return tlist[0]
+
@retry(exceptions=Exception, tries=5, delay=5)
def translateGPT(text, history, fullPromptFlag):
global PBAR
-
+
mismatch = False
totalTokens = [0, 0]
if isinstance(text, list):
@@ -651,7 +716,7 @@ def translateGPT(text, history, fullPromptFlag):
subbedT = varResponse[0]
# Things to Check before starting translation
- if not re.search(r'[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+', subbedT):
+ if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", subbedT):
if PBAR is not None:
PBAR.update(len(tItem))
continue
@@ -688,12 +753,14 @@ def translateGPT(text, history, fullPromptFlag):
if isinstance(tItem, list):
extractedTranslations = extractTranslation(translatedText, True)
if len(tItem) != len(extractedTranslations):
- mismatch = True # Just here for breakpoint
-
+ mismatch = True # Just here for breakpoint
+
# Set if no mismatch
if mismatch == False:
tList[index] = extractedTranslations
- history = extractedTranslations[-10:] # Update history if we have a list
+ history = extractedTranslations[
+ -10:
+ ] # Update history if we have a list
else:
history = text[-10:]
mismatch = False
diff --git a/modules/regex.py b/modules/regex.py
index 0dccff4..376f100 100644
--- a/modules/regex.py
+++ b/modules/regex.py
@@ -15,49 +15,50 @@ from tqdm import tqdm
# Open AI
load_dotenv()
-if os.getenv('api').replace(' ', '') != '':
- openai.base_url = os.getenv('api')
-openai.organization = os.getenv('org')
-openai.api_key = os.getenv('key')
+if os.getenv("api").replace(" ", "") != "":
+ openai.base_url = os.getenv("api")
+openai.organization = os.getenv("org")
+openai.api_key = os.getenv("key")
-#Globals
-MODEL = os.getenv('model')
-TIMEOUT = int(os.getenv('timeout'))
-LANGUAGE = os.getenv('language').capitalize()
-PROMPT = Path('prompt.txt').read_text(encoding='utf-8')
-VOCAB = Path('vocab.txt').read_text(encoding='utf-8')
-THREADS = int(os.getenv('threads'))
+# Globals
+MODEL = os.getenv("model")
+TIMEOUT = int(os.getenv("timeout"))
+LANGUAGE = os.getenv("language").capitalize()
+PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
+VOCAB = Path("vocab.txt").read_text(encoding="utf-8")
+THREADS = int(os.getenv("threads"))
LOCK = threading.Lock()
-WIDTH = int(os.getenv('width'))
-LISTWIDTH = int(os.getenv('listWidth'))
+WIDTH = int(os.getenv("width"))
+LISTWIDTH = int(os.getenv("listWidth"))
NOTEWIDTH = 70
MAXHISTORY = 10
-ESTIMATE = ''
+ESTIMATE = ""
TOKENS = [0, 0]
NAMESLIST = []
-NAMES = False # Output a list of all the character names found
-BRFLAG = False # If the game uses
instead
+NAMES = False # Output a list of all the character names found
+BRFLAG = False # If the game uses
instead
FIXTEXTWRAP = True # Overwrites textwrap
-IGNORETLTEXT = False # Ignores all translated text.
-MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
+IGNORETLTEXT = False # Ignores all translated text.
+MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
-#tqdm Globals
-BAR_FORMAT='{l_bar}{bar:10}{r_bar}{bar:-10b}'
+# tqdm Globals
+BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
POSITION = 0
LEAVE = False
# Pricing - Depends on the model https://openai.com/pricing
# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request
# If you are getting a MISMATCH LENGTH error, lower the batch size.
-if 'gpt-3.5' in MODEL:
- INPUTAPICOST = .002
- OUTPUTAPICOST = .002
+if "gpt-3.5" in MODEL:
+ INPUTAPICOST = 0.002
+ OUTPUTAPICOST = 0.002
BATCHSIZE = 10
-elif 'gpt-4' in MODEL:
- INPUTAPICOST = .005
- OUTPUTAPICOST = .015
+elif "gpt-4" in MODEL:
+ INPUTAPICOST = 0.005
+ OUTPUTAPICOST = 0.015
BATCHSIZE = 40
+
def handleRegex(filename, estimate):
global ESTIMATE
ESTIMATE = estimate
@@ -74,17 +75,21 @@ def handleRegex(filename, estimate):
TOKENS[1] += translatedData[1][1]
# Print Total
- totalString = getResultString(['', TOKENS, None], end - start, 'TOTAL')
+ totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
# Print any errors on maps
if len(MISMATCH) > 0:
- return totalString + Fore.RED + f'\nMismatch Errors: {MISMATCH}' + Fore.RESET
+ return (
+ totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
+ )
else:
return totalString
-
+
else:
try:
- with open('translated/' + filename, 'w', encoding='cp932', errors='ignore') as outFile:
+ with open(
+ "translated/" + filename, "w", encoding="cp932", errors="ignore"
+ ) as outFile:
start = time.time()
translatedData = openFiles(filename)
@@ -97,23 +102,36 @@ def handleRegex(filename, estimate):
TOKENS[1] += translatedData[1][1]
except Exception:
traceback.print_exc()
- return 'Fail'
+ return "Fail"
+
+ return getResultString(["", TOKENS, None], end - start, "TOTAL")
- return getResultString(['', TOKENS, None], end - start, 'TOTAL')
def getResultString(translatedData, translationTime, filename):
# File Print String
- totalTokenstring =\
- Fore.YELLOW +\
- '[Input: ' + str(translatedData[1][0]) + ']'\
- '[Output: ' + str(translatedData[1][1]) + ']'\
- '[Cost: ${:,.4f}'.format((translatedData[1][0] * .001 * INPUTAPICOST) +\
- (translatedData[1][1] * .001 * OUTPUTAPICOST)) + ']'
- timeString = Fore.BLUE + '[' + str(round(translationTime, 1)) + 's]'
+ totalTokenstring = (
+ Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
+ "[Output: "
+ + str(translatedData[1][1])
+ + "]" "[Cost: ${:,.4f}".format(
+ (translatedData[1][0] * 0.001 * INPUTAPICOST)
+ + (translatedData[1][1] * 0.001 * OUTPUTAPICOST)
+ )
+ + "]"
+ )
+ timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
if translatedData[2] == None:
# Success
- return filename + ': ' + totalTokenstring + timeString + Fore.GREEN + u' \u2713 ' + Fore.RESET
+ return (
+ filename
+ + ": "
+ + totalTokenstring
+ + timeString
+ + Fore.GREEN
+ + " \u2713 "
+ + Fore.RESET
+ )
else:
# Fail
@@ -122,31 +140,41 @@ def getResultString(translatedData, translationTime, filename):
except Exception as e:
traceback.print_exc()
errorString = str(e) + Fore.RED
- return filename + ': ' + totalTokenstring + timeString + Fore.RED + u' \u2717 ' +\
- errorString + Fore.RESET
+ return (
+ filename
+ + ": "
+ + totalTokenstring
+ + timeString
+ + Fore.RED
+ + " \u2717 "
+ + errorString
+ + Fore.RESET
+ )
+
def openFiles(filename):
- with open('files/' + filename, 'r', encoding='shift_jis') as readFile:
+ with open("files/" + filename, "r", encoding="shift_jis") as readFile:
translatedData = parseRegex(readFile, filename)
# Delete lines marked for deletion
finalData = []
for line in translatedData[0]:
- if line != '\\d\n':
+ if line != "\\d\n":
finalData.append(line)
translatedData[0] = finalData
-
+
return translatedData
+
def parseRegex(readFile, filename):
- totalTokens = [0,0]
+ totalTokens = [0, 0]
# Read File into data
data = readFile.readlines()
# Create Progress Bar
with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
- pbar.desc=filename
+ pbar.desc = filename
try:
result = translateRegex(data, pbar, filename, [])
@@ -157,36 +185,37 @@ def parseRegex(readFile, filename):
return [data, totalTokens, e]
return [data, totalTokens, None]
+
def translateRegex(data, pbar, filename, translatedList):
stringList = []
currentGroup = []
- tokens = [0,0]
- speaker = ''
+ tokens = [0, 0]
+ speaker = ""
voice = False
global LOCK, ESTIMATE
i = 0
while i < len(data):
voice = False
- speaker = ''
- if 'actorID' in data[i]:
+ speaker = ""
+ if "actorID" in data[i]:
# Lines
- match = re.search(r'label[\\]+\":[\\]+\"(.*?)\"', data[i])
+ match = re.search(r"label[\\]+\":[\\]+\"(.*?)\"", data[i])
if match == None:
- match = re.search(r'label[\\]+\":[\\]+\"(.*?)\"', data[i])
- if match != None and match.group(1) != '':
+ match = re.search(r"label[\\]+\":[\\]+\"(.*?)\"", data[i])
+ if match != None and match.group(1) != "":
originalString = match.group(1)
# Pass 1
if translatedList == []:
# Grab Consecutive Strings
jaString = match.group(1)
-
+
# Remove any textwrap
- jaString = jaString.replace('\n', ' ')
+ jaString = jaString.replace("\n", " ")
# Add String
stringList.append(jaString.strip())
-
+
# Pass 2
else:
# Get Text
@@ -217,9 +246,15 @@ def translateRegex(data, pbar, filename, translatedList):
# Set Progress
pbar.total = len(stringList)
pbar.refresh()
-
+
# Translate
- response = translateGPT(stringList, 'Reply with the English TL of the NPC Name', True, pbar, filename)
+ response = translateGPT(
+ stringList,
+ "Reply with the English TL of the NPC Name",
+ True,
+ pbar,
+ filename,
+ )
tokens[0] += response[1][0]
tokens[1] += response[1][1]
translatedList = response[0]
@@ -235,94 +270,103 @@ def translateRegex(data, pbar, filename, translatedList):
MISMATCH.append(filename)
return tokens
+
# Save some money and enter the character before translation
def getSpeaker(speaker, pbar, filename):
match speaker:
- case 'ファイン':
- return ['Fine', [0,0]]
- case '':
- return ['', [0,0]]
+ case "ファイン":
+ return ["Fine", [0, 0]]
+ case "":
+ return ["", [0, 0]]
case _:
# Store Speaker
if speaker not in str(NAMESLIST):
- response = translateGPT(speaker, 'Reply with only the '+ LANGUAGE +' translation of the NPC name.', False, pbar, filename)
+ response = translateGPT(
+ speaker,
+ "Reply with only the " + LANGUAGE + " translation of the NPC name.",
+ False,
+ pbar,
+ filename,
+ )
response[0] = response[0].replace("'S", "'s")
speakerList = [speaker, response[0]]
NAMESLIST.append(speakerList)
return response
-
+
# Find Speaker
else:
for i in range(len(NAMESLIST)):
if speaker == NAMESLIST[i][0]:
- return [NAMESLIST[i][1],[0,0]]
-
- return [speaker,[0,0]]
+ return [NAMESLIST[i][1], [0, 0]]
+
+ return [speaker, [0, 0]]
+
def subVars(jaString):
- jaString = jaString.replace('\u3000', ' ')
+ jaString = jaString.replace("\u3000", " ")
# Nested
count = 0
- nestedList = re.findall(r'[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]', jaString)
+ nestedList = re.findall(r"[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]", jaString)
nestedList = set(nestedList)
if len(nestedList) != 0:
for icon in nestedList:
- jaString = jaString.replace(icon, '[Nested_' + str(count) + ']')
+ jaString = jaString.replace(icon, "[Nested_" + str(count) + "]")
count += 1
# Icons
count = 0
- iconList = re.findall(r'[\\]+[iIkKwWaA]+\[[0-9]+\]', jaString)
+ iconList = re.findall(r"[\\]+[iIkKwWaA]+\[[0-9]+\]", jaString)
iconList = set(iconList)
if len(iconList) != 0:
for icon in iconList:
- jaString = jaString.replace(icon, '[Ascii_' + str(count) + ']')
+ jaString = jaString.replace(icon, "[Ascii_" + str(count) + "]")
count += 1
# Colors
count = 0
- colorList = re.findall(r'[\\]+[cC]\[[0-9]+\]', jaString)
+ colorList = re.findall(r"[\\]+[cC]\[[0-9]+\]", jaString)
colorList = set(colorList)
if len(colorList) != 0:
for color in colorList:
- jaString = jaString.replace(color, '[Color_' + str(count) + ']')
+ jaString = jaString.replace(color, "[Color_" + str(count) + "]")
count += 1
# Names
count = 0
- nameList = re.findall(r'[\\]+[nN]\[.+?\]+', jaString)
+ nameList = re.findall(r"[\\]+[nN]\[.+?\]+", jaString)
nameList = set(nameList)
if len(nameList) != 0:
for name in nameList:
- jaString = jaString.replace(name, '[Noun_' + str(count) + ']')
+ jaString = jaString.replace(name, "[Noun_" + str(count) + "]")
count += 1
# Variables
count = 0
- varList = re.findall(r'[\\]+[vV]\[[0-9]+\]', jaString)
+ varList = re.findall(r"[\\]+[vV]\[[0-9]+\]", jaString)
varList = set(varList)
if len(varList) != 0:
for var in varList:
- jaString = jaString.replace(var, '[Var_' + str(count) + ']')
+ jaString = jaString.replace(var, "[Var_" + str(count) + "]")
count += 1
# Formatting
count = 0
- formatList = re.findall(r'[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]', jaString)
+ formatList = re.findall(r"[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]", jaString)
formatList = set(formatList)
if len(formatList) != 0:
for var in formatList:
- jaString = jaString.replace(var, '[FCode_' + str(count) + ']')
+ jaString = jaString.replace(var, "[FCode_" + str(count) + "]")
count += 1
# Put all lists in list and return
allList = [nestedList, iconList, colorList, nameList, varList, formatList]
return [jaString, allList]
+
def resubVars(translatedText, allList):
# Fix Spacing and ChatGPT Nonsense
- matchList = re.findall(r'\[\s?.+?\s?\]', translatedText)
+ matchList = re.findall(r"\[\s?.+?\s?\]", translatedText)
if len(matchList) > 0:
for match in matchList:
text = match.strip()
@@ -332,54 +376,58 @@ def resubVars(translatedText, allList):
count = 0
if len(allList[0]) != 0:
for var in allList[0]:
- translatedText = translatedText.replace('[Nested_' + str(count) + ']', var)
+ translatedText = translatedText.replace("[Nested_" + str(count) + "]", var)
count += 1
# Icons
count = 0
if len(allList[1]) != 0:
for var in allList[1]:
- translatedText = translatedText.replace('[Ascii_' + str(count) + ']', var)
+ translatedText = translatedText.replace("[Ascii_" + str(count) + "]", var)
count += 1
# Colors
count = 0
if len(allList[2]) != 0:
for var in allList[2]:
- translatedText = translatedText.replace('[Color_' + str(count) + ']', var)
+ translatedText = translatedText.replace("[Color_" + str(count) + "]", var)
count += 1
# Names
count = 0
if len(allList[3]) != 0:
for var in allList[3]:
- translatedText = translatedText.replace('[Noun_' + str(count) + ']', var)
+ translatedText = translatedText.replace("[Noun_" + str(count) + "]", var)
count += 1
# Vars
count = 0
if len(allList[4]) != 0:
for var in allList[4]:
- translatedText = translatedText.replace('[Var_' + str(count) + ']', var)
+ translatedText = translatedText.replace("[Var_" + str(count) + "]", var)
count += 1
-
+
# Formatting
count = 0
if len(allList[5]) != 0:
for var in allList[5]:
- translatedText = translatedText.replace('[FCode_' + str(count) + ']', var)
+ translatedText = translatedText.replace("[FCode_" + str(count) + "]", var)
count += 1
return translatedText
+
def batchList(input_list, batch_size):
if not isinstance(batch_size, int) or batch_size <= 0:
raise ValueError("batch_size must be a positive integer")
-
- return [input_list[i:i + batch_size] for i in range(0, len(input_list), batch_size)]
+
+ return [
+ input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)
+ ]
+
def createContext(fullPromptFlag, subbedT):
- characters = 'Game Characters:\n\
+ characters = "Game Characters:\n\
フィリア (Philia) - Female\n\
アルネット (Annett) - Female\n\
ラピュセナ (Rapusena) - Female\n\
@@ -389,10 +437,12 @@ def createContext(fullPromptFlag, subbedT):
カルナ (Karna) - Female\n\
ラフィング＝スピア (Laughing Spear) - Female\n\
ノーラ (Nora) - Female\n\
-'
-
- system = PROMPT + VOCAB if fullPromptFlag else \
- f"\
+"
+
+ system = (
+ PROMPT + VOCAB
+ if fullPromptFlag
+ else f"\
You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
- You translate everything, including erotic and explicit content.\n\
@@ -404,9 +454,11 @@ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{
- `...` can be a part of the dialogue. Translate it as it is.\n\
{VOCAB}\n\
"
- user = f'{subbedT}'
+ )
+ user = f"{subbedT}"
return characters, system, user
+
def translateText(characters, system, user, history):
# Prompt
msg = [{"role": "system", "content": system + characters}]
@@ -419,9 +471,9 @@ def translateText(characters, system, user, history):
msg.extend([{"role": "system", "content": h} for h in history])
else:
msg.append({"role": "system", "content": history})
-
+
# Content to TL
- msg.append({"role": "user", "content": f'{user}'})
+ msg.append({"role": "user", "content": f"{user}"})
response = openai.chat.completions.create(
temperature=0.1,
frequency_penalty=0.1,
@@ -430,15 +482,16 @@ def translateText(characters, system, user, history):
)
return response
+
def cleanTranslatedText(translatedText, varResponse):
placeholders = {
- f'{LANGUAGE} Translation: ': '',
- 'Translation: ': '',
- 'っ': '',
- '〜': '~',
- 'ッ': '',
- '。': '.',
- 'Placeholder Text': ''
+ f"{LANGUAGE} Translation: ": "",
+ "Translation: ": "",
+ "っ": "",
+ "〜": "~",
+ "ッ": "",
+ "。": ".",
+ "Placeholder Text": "",
# Add more replacements as needed
}
for target, replacement in placeholders.items():
@@ -449,11 +502,12 @@ def cleanTranslatedText(translatedText, varResponse):
translatedText = resubVars(translatedText, varResponse[1])
return translatedText
+
def elongateCharacters(text):
# Define a pattern to match one character followed by one or more `ー` characters
# Using a positive lookbehind assertion to capture the preceding character
- pattern = r'(?<=(.))ー+'
-
+ pattern = r"(?<=(.))ー+"
+
# Define a replacement function that elongates the captured character
def repl(match):
char = match.group(1) # The character before the ー sequence
@@ -463,8 +517,9 @@ def elongateCharacters(text):
# Use re.sub() to replace the pattern in the text
return re.sub(pattern, repl, text)
+
def extractTranslation(translatedTextList, is_list):
- pattern = r'`?<[Ll]ine\d+>([\\]*.*?[\\]*?)<\/?[Ll]ine\d+>`?'
+ pattern = r"`?<[Ll]ine\d+>([\\]*.*?[\\]*?)<\/?[Ll]ine\d+>`?"
# If it's a batch (i.e., list), extract with tags; otherwise, return the single item.
if is_list:
matchList = re.findall(pattern, translatedTextList)
@@ -473,11 +528,12 @@ def extractTranslation(translatedTextList, is_list):
matchList = re.findall(pattern, translatedTextList)
return matchList[0][0] if matchList else translatedTextList
+
def countTokens(characters, system, user, history):
inputTotalTokens = 0
outputTotalTokens = 0
- enc = tiktoken.encoding_for_model('gpt-4')
-
+ enc = tiktoken.encoding_for_model("gpt-4")
+
# Input
if isinstance(history, list):
for line in history:
@@ -489,15 +545,17 @@ def countTokens(characters, system, user, history):
inputTotalTokens += len(enc.encode(user))
# Output
- outputTotalTokens += round(len(enc.encode(user))*2)
+ outputTotalTokens += round(len(enc.encode(user)) * 2)
return [inputTotalTokens, outputTotalTokens]
+
def combineList(tlist, text):
if isinstance(text, list):
return [t for sublist in tlist for t in sublist]
return tlist[0]
+
@retry(exceptions=Exception, tries=5, delay=5)
def translateGPT(text, history, fullPromptFlag, pbar, filename):
mismatch = False
@@ -510,8 +568,12 @@ def translateGPT(text, history, fullPromptFlag, pbar, filename):
for index, tItem in enumerate(tList):
# Before sending to translation, if we have a list of items, add the formatting
if isinstance(tItem, list):
- payload = '\n'.join([f'`{item}`' for i, item in enumerate(tItem)])
- payload = re.sub(r'(<)(\/Line\d+>)', r'\1>Placeholder Text<\3', payload)
+ payload = "\n".join(
+ [f"`{item}`" for i, item in enumerate(tItem)]
+ )
+ payload = re.sub(
+ r"(<)(\/Line\d+>)", r"\1>Placeholder Text<\3", payload
+ )
varResponse = subVars(payload)
subbedT = varResponse[0]
else:
@@ -519,7 +581,7 @@ def translateGPT(text, history, fullPromptFlag, pbar, filename):
subbedT = varResponse[0]
# Things to Check before starting translation
- if not re.search(r'[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+', subbedT):
+ if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", subbedT):
continue
# Create Message
diff --git a/modules/rpgmakerace.py b/modules/rpgmakerace.py
index dc5275e..610ef25 100644
--- a/modules/rpgmakerace.py
+++ b/modules/rpgmakerace.py
@@ -19,32 +19,32 @@ from ruamel.yaml import YAML
# Open AI
load_dotenv()
-if os.getenv('api').replace(' ', '') != '':
- openai.base_url = os.getenv('api')
-openai.organization = os.getenv('org')
-openai.api_key = os.getenv('key')
+if os.getenv("api").replace(" ", "") != "":
+ openai.base_url = os.getenv("api")
+openai.organization = os.getenv("org")
+openai.api_key = os.getenv("key")
-#Globals
-MODEL = os.getenv('model')
-TIMEOUT = int(os.getenv('timeout'))
-LANGUAGE = os.getenv('language').capitalize()
-PROMPT = Path('prompt.txt').read_text(encoding='utf-8')
-VOCAB = Path('vocab.txt').read_text(encoding='utf-8')
-THREADS = int(os.getenv('threads'))
+# Globals
+MODEL = os.getenv("model")
+TIMEOUT = int(os.getenv("timeout"))
+LANGUAGE = os.getenv("language").capitalize()
+PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
+VOCAB = Path("vocab.txt").read_text(encoding="utf-8")
+THREADS = int(os.getenv("threads"))
LOCK = threading.Lock()
-WIDTH = int(os.getenv('width'))
-LISTWIDTH = int(os.getenv('listWidth'))
-NOTEWIDTH = int(os.getenv('noteWidth'))
+WIDTH = int(os.getenv("width"))
+LISTWIDTH = int(os.getenv("listWidth"))
+NOTEWIDTH = int(os.getenv("noteWidth"))
MAXHISTORY = 10
-ESTIMATE = ''
+ESTIMATE = ""
TOKENS = [0, 0]
NAMESLIST = []
-FIRSTLINESPEAKERS = True # If 1st line of dialogue is a speaker, set to True
-NAMES = False # Output a list of all the character names found
-BRFLAG = False # If the game uses
instead
+FIRSTLINESPEAKERS = True # If 1st line of dialogue is a speaker, set to True
+NAMES = False # Output a list of all the character names found
+BRFLAG = False # If the game uses
instead
FIXTEXTWRAP = True # Overwrites textwrap
-IGNORETLTEXT = False # Ignores all translated text.
-MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
+IGNORETLTEXT = False # Ignores all translated text.
+MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
BRACKETNAMES = False
PBAR = None
FILENAME = None
@@ -52,19 +52,19 @@ FILENAME = None
# Pricing - Depends on the model https://openai.com/pricing
# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request
# If you are getting a MISMATCH LENGTH error, lower the batch size.
-if 'gpt-3.5' in MODEL:
- INPUTAPICOST = .002
- OUTPUTAPICOST = .002
+if "gpt-3.5" in MODEL:
+ INPUTAPICOST = 0.002
+ OUTPUTAPICOST = 0.002
BATCHSIZE = 10
FREQUENCY_PENALTY = 0.2
-elif 'gpt-4' in MODEL:
- INPUTAPICOST = .0025
- OUTPUTAPICOST = .01
+elif "gpt-4" in MODEL:
+ INPUTAPICOST = 0.0025
+ OUTPUTAPICOST = 0.01
BATCHSIZE = 20
FREQUENCY_PENALTY = 0.1
-#tqdm Globals
-BAR_FORMAT='{l_bar}{bar:10}{r_bar}{bar:-10b}'
+# tqdm Globals
+BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
POSITION = 0
LEAVE = False
@@ -92,6 +92,7 @@ CODE324 = False
CODE111 = False
CODE108 = False
+
def handleACE(filename, estimate):
global ESTIMATE, TOKENS, FILENAME
ESTIMATE = estimate
@@ -100,19 +101,19 @@ def handleACE(filename, estimate):
# Translate
start = time.time()
translatedData = openFiles(filename)
-
+
# Translate
if not estimate:
try:
- with open('translated/' + filename, 'w', encoding='utf-8') as outFile:
- yaml=YAML(pure=True)
+ with open("translated/" + filename, "w", encoding="utf-8") as outFile:
+ yaml = YAML(pure=True)
yaml.width = 4096
yaml.default_style = "'"
yaml.dump(translatedData[0], outFile)
except Exception:
traceback.print_exc()
- return 'Fail'
-
+ return "Fail"
+
# Print File
end = time.time()
tqdm.write(getResultString(translatedData, end - start, filename))
@@ -121,108 +122,122 @@ def handleACE(filename, estimate):
TOKENS[1] += translatedData[1][1]
# Print Total
- totalString = getResultString(['', TOKENS, None], end - start, 'TOTAL')
+ totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
# Print any errors on maps
if len(MISMATCH) > 0:
- return totalString + Fore.RED + f'\nMismatch Errors: {MISMATCH}' + Fore.RESET
+ return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
else:
return totalString
+
def openFiles(filename):
- yaml=YAML(pure=True) # Need a yaml instance per thread.
+ yaml = YAML(pure=True) # Need a yaml instance per thread.
yaml.width = 4096
yaml.default_style = "'"
- with open('files/' + filename, 'r', encoding='UTF-8') as f:
+ with open("files/" + filename, "r", encoding="UTF-8") as f:
# Map Files
- if 'Map' in filename and filename != 'MapInfos.json':
+ if "Map" in filename and filename != "MapInfos.json":
data = yaml.load(f)
translatedData = parseMap(data, filename)
# CommonEvents Files
- elif 'CommonEvents' in filename:
+ elif "CommonEvents" in filename:
data = yaml.load(f)
translatedData = parseCommonEvents(data, filename)
# Actor File
- elif 'Actors' in filename:
+ elif "Actors" in filename:
data = yaml.load(f)
- translatedData = parseNames(data, filename, 'Actors')
+ translatedData = parseNames(data, filename, "Actors")
# Armor File
- elif 'Armors' in filename:
+ elif "Armors" in filename:
data = yaml.load(f)
- translatedData = parseNames(data, filename, 'Armors')
+ translatedData = parseNames(data, filename, "Armors")
# Weapons File
- elif 'Weapons' in filename:
+ elif "Weapons" in filename:
data = yaml.load(f)
- translatedData = parseNames(data, filename, 'Weapons')
-
+ translatedData = parseNames(data, filename, "Weapons")
+
# Classes File
- elif 'Classes' in filename:
+ elif "Classes" in filename:
data = yaml.load(f)
- translatedData = parseNames(data, filename, 'Classes')
+ translatedData = parseNames(data, filename, "Classes")
# Enemies File
- elif 'Enemies' in filename:
+ elif "Enemies" in filename:
data = yaml.load(f)
- translatedData = parseNames(data, filename, 'Enemies')
+ translatedData = parseNames(data, filename, "Enemies")
# Items File
- elif 'Items' in filename:
+ elif "Items" in filename:
data = yaml.load(f)
- translatedData = parseNames(data, filename, 'Items')
+ translatedData = parseNames(data, filename, "Items")
# MapInfo File
- elif 'MapInfos' in filename:
+ elif "MapInfos" in filename:
data = yaml.load(f)
- translatedData = parseNames(data, filename, 'MapInfos')
+ translatedData = parseNames(data, filename, "MapInfos")
# Skills File
- elif 'Skills' in filename:
+ elif "Skills" in filename:
data = yaml.load(f)
- translatedData = parseNames(data, filename, 'Skills')
+ translatedData = parseNames(data, filename, "Skills")
# Troops File
- elif 'Troops' in filename:
+ elif "Troops" in filename:
data = yaml.load(f)
translatedData = parseTroops(data, filename)
# States File
- elif 'States' in filename:
+ elif "States" in filename:
data = yaml.load(f)
translatedData = parseSS(data, filename)
# System File
- elif 'System' in filename:
+ elif "System" in filename:
data = yaml.load(f)
translatedData = parseSystem(data, filename)
# Scenario File
- elif 'Scenario' in filename:
+ elif "Scenario" in filename:
data = yaml.load(f)
translatedData = parseScenario(data, filename)
else:
- raise NameError(filename + ' Not Supported')
-
+ raise NameError(filename + " Not Supported")
+
return translatedData
+
def getResultString(translatedData, translationTime, filename):
# File Print String
- totalTokenstring =\
- Fore.YELLOW +\
- '[Input: ' + str(translatedData[1][0]) + ']'\
- '[Output: ' + str(translatedData[1][1]) + ']'\
- '[Cost: ${:,.4f}'.format((translatedData[1][0] * .001 * INPUTAPICOST) +\
- (translatedData[1][1] * .001 * OUTPUTAPICOST)) + ']'
- timeString = Fore.BLUE + '[' + str(round(translationTime, 1)) + 's]'
+ totalTokenstring = (
+ Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
+ "[Output: "
+ + str(translatedData[1][1])
+ + "]" "[Cost: ${:,.4f}".format(
+ (translatedData[1][0] * 0.001 * INPUTAPICOST)
+ + (translatedData[1][1] * 0.001 * OUTPUTAPICOST)
+ )
+ + "]"
+ )
+ timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
if translatedData[2] is None:
# Success
- return filename + ': ' + totalTokenstring + timeString + Fore.GREEN + u' \u2713 ' + Fore.RESET
+ return (
+ filename
+ + ": "
+ + totalTokenstring
+ + timeString
+ + Fore.GREEN
+ + " \u2713 "
+ + Fore.RESET
+ )
else:
# Fail
try:
@@ -230,29 +245,46 @@ def getResultString(translatedData, translationTime, filename):
except Exception as e:
traceback.print_exc()
errorString = str(e) + Fore.RED
- return filename + ': ' + totalTokenstring + timeString + Fore.RED + u' \u2717 ' +\
- errorString + Fore.RESET
+ return (
+ filename
+ + ": "
+ + totalTokenstring
+ + timeString
+ + Fore.RED
+ + " \u2717 "
+ + errorString
+ + Fore.RESET
+ )
+
def parseMap(data, filename):
totalTokens = [0, 0]
totalLines = 0
- events = data['events']
+ events = data["events"]
global LOCK
# Translate displayName for Map files
- if 'Map' in filename:
- response = translateGPT(data['display_name'], 'Reply with only the '+ LANGUAGE +' translation of the RPG location name', False)
+ if "Map" in filename:
+ response = translateGPT(
+ data["display_name"],
+ "Reply with only the " + LANGUAGE + " translation of the RPG location name",
+ False,
+ )
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
- data['display_name'] = response[0].replace('\"', '')
-
+ data["display_name"] = response[0].replace('"', "")
+
# Thread for each page in file
with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
- pbar.desc=filename
+ pbar.desc = filename
with ThreadPoolExecutor(max_workers=THREADS) as executor:
for key in events:
if key is not None:
- futures = [executor.submit(searchCodes, page, pbar, [], filename) for page in events[key]['pages'] if page is not None]
+ futures = [
+ executor.submit(searchCodes, page, pbar, [], filename)
+ for page in events[key]["pages"]
+ if page is not None
+ ]
for future in as_completed(futures):
try:
totalTokensFuture = future.result()
@@ -263,53 +295,64 @@ def parseMap(data, filename):
return [data, totalTokens, e]
return [data, totalTokens, None]
+
def translateNote(event, regex):
# Regex String
- jaString = event['note']
+ jaString = event["note"]
match = re.findall(regex, jaString, re.DOTALL)
if match:
- tokens = [0,0]
+ tokens = [0, 0]
i = 0
while i < len(match):
initialJAString = match[i]
# Remove any textwrap
- modifiedJAString = initialJAString.replace('\n', ' ')
+ modifiedJAString = initialJAString.replace("\n", " ")
# Translate
- response = translateGPT(modifiedJAString, 'Reply with only the '+ LANGUAGE +' translation.', False)
+ response = translateGPT(
+ modifiedJAString,
+ "Reply with only the " + LANGUAGE + " translation.",
+ False,
+ )
translatedText = response[0]
tokens[0] += response[1][0]
tokens[1] += response[1][1]
# Textwrap
translatedText = textwrap.fill(translatedText, width=NOTEWIDTH)
- translatedText = translatedText.replace('\"', '')
+ translatedText = translatedText.replace('"', "")
jaString = jaString.replace(initialJAString, translatedText)
- event['note'] = jaString
+ event["note"] = jaString
i += 1
return tokens
- return [0,0]
+ return [0, 0]
+
# For notes that can't have spaces.
def translateNoteOmitSpace(event, regex):
# Regex that only matches text inside LB.
- jaString = event['note']
+ jaString = event["note"]
match = re.findall(regex, jaString, re.DOTALL)
if match:
oldJAString = match[0]
# Remove any textwrap
- jaString = re.sub(r'\n', ' ', oldJAString)
+ jaString = re.sub(r"\n", " ", oldJAString)
# Translate
- response = translateGPT(jaString, 'Reply with the '+ LANGUAGE +' translation of the location name.', False)
+ response = translateGPT(
+ jaString,
+ "Reply with the " + LANGUAGE + " translation of the location name.",
+ False,
+ )
translatedText = response[0]
- translatedText = translatedText.replace('\"', '')
- translatedText = translatedText.replace(' ', '_')
- event['note'] = event['note'].replace(oldJAString, translatedText)
+ translatedText = translatedText.replace('"', "")
+ translatedText = translatedText.replace(" ", "_")
+ event["note"] = event["note"].replace(oldJAString, translatedText)
return response[1]
- return [0,0]
+ return [0, 0]
+
def parseCommonEvents(data, filename):
totalTokens = [0, 0]
@@ -319,12 +362,16 @@ def parseCommonEvents(data, filename):
# Get total for progress bar
for page in data:
if page is not None:
- totalLines += len(page['list'])
+ totalLines += len(page["list"])
with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
- pbar.desc=filename
+ pbar.desc = filename
with ThreadPoolExecutor(max_workers=THREADS) as executor:
- futures = [executor.submit(searchCodes, page, pbar, [], filename) for page in data if page is not None]
+ futures = [
+ executor.submit(searchCodes, page, pbar, [], filename)
+ for page in data
+ if page is not None
+ ]
for future in as_completed(futures):
try:
totalTokensFuture = future.result()
@@ -335,6 +382,7 @@ def parseCommonEvents(data, filename):
return [data, totalTokens, e]
return [data, totalTokens, None]
+
def parseTroops(data, filename):
totalTokens = [0, 0]
totalLines = 0
@@ -343,15 +391,21 @@ def parseTroops(data, filename):
# Get total for progress bar
for troop in data:
if troop is not None:
- for page in troop['pages']:
- totalLines += len(page['list']) + 1 # The +1 is because each page has a name.
+ for page in troop["pages"]:
+ totalLines += (
+ len(page["list"]) + 1
+ ) # The +1 is because each page has a name.
with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
- pbar.desc=filename
+ pbar.desc = filename
for troop in data:
if troop is not None:
with ThreadPoolExecutor(max_workers=THREADS) as executor:
- futures = [executor.submit(searchCodes, page, pbar, [], filename) for page in troop['pages'] if page is not None]
+ futures = [
+ executor.submit(searchCodes, page, pbar, [], filename)
+ for page in troop["pages"]
+ if page is not None
+ ]
for future in as_completed(futures):
try:
totalTokensFuture = future.result()
@@ -361,59 +415,17 @@ def parseTroops(data, filename):
traceback.print_exc()
return [data, totalTokens, e]
return [data, totalTokens, None]
-
+
+
def parseNames(data, filename, context):
totalTokens = [0, 0]
totalLines = 0
totalLines += len(data)
-
- with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
- pbar.desc=filename
- try:
- result = searchNames(data, pbar, context)
- totalTokens[0] += result[0]
- totalTokens[1] += result[1]
- except Exception as e:
- traceback.print_exc()
- return [data, totalTokens, e]
- return [data, totalTokens, None]
-def parseSS(data, filename):
- totalTokens = [0, 0]
- totalLines = 0
- totalLines += len(data)
-
with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
- pbar.desc=filename
- for ss in data:
- if ss is not None:
- try:
- result = searchSS(ss, pbar)
- totalTokens[0] += result[0]
- totalTokens[1] += result[1]
- except Exception as e:
- traceback.print_exc()
- return [data, totalTokens, e]
- return [data, totalTokens, None]
-
-def parseSystem(data, filename):
- totalTokens = [0, 0]
- totalLines = 0
-
- # Calculate Total Lines
- for term in data['terms']:
- termList = data['terms'][term]
- totalLines += len(termList)
- totalLines += len(data['game_title'])
- totalLines += len(data['variables'])
- totalLines += len(data['weapon_types'])
- totalLines += len(data['armor_types'])
- totalLines += len(data['skill_types'])
-
- with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
- pbar.desc=filename
+ pbar.desc = filename
try:
- result = searchSystem(data, pbar)
+ result = searchNames(data, pbar, context)
totalTokens[0] += result[0]
totalTokens[1] += result[1]
except Exception as e:
@@ -421,6 +433,52 @@ def parseSystem(data, filename):
return [data, totalTokens, e]
return [data, totalTokens, None]
+
+def parseSS(data, filename):
+ totalTokens = [0, 0]
+ totalLines = 0
+ totalLines += len(data)
+
+ with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
+ pbar.desc = filename
+ for ss in data:
+ if ss is not None:
+ try:
+ result = searchSS(ss, pbar)
+ totalTokens[0] += result[0]
+ totalTokens[1] += result[1]
+ except Exception as e:
+ traceback.print_exc()
+ return [data, totalTokens, e]
+ return [data, totalTokens, None]
+
+
+def parseSystem(data, filename):
+ totalTokens = [0, 0]
+ totalLines = 0
+
+ # Calculate Total Lines
+ for term in data["terms"]:
+ termList = data["terms"][term]
+ totalLines += len(termList)
+ totalLines += len(data["game_title"])
+ totalLines += len(data["variables"])
+ totalLines += len(data["weapon_types"])
+ totalLines += len(data["armor_types"])
+ totalLines += len(data["skill_types"])
+
+ with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
+ pbar.desc = filename
+ try:
+ result = searchSystem(data, pbar)
+ totalTokens[0] += result[0]
+ totalTokens[1] += result[1]
+ except Exception as e:
+ traceback.print_exc()
+ return [data, totalTokens, e]
+ return [data, totalTokens, None]
+
+
def parseScenario(data, filename):
totalTokens = [0, 0]
totalLines = 0
@@ -431,9 +489,13 @@ def parseScenario(data, filename):
totalLines += len(page[1])
with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
- pbar.desc=filename
+ pbar.desc = filename
with ThreadPoolExecutor(max_workers=THREADS) as executor:
- futures = [executor.submit(searchCodes, page[1], pbar, [], filename) for page in data.items() if page[1] is not None]
+ futures = [
+ executor.submit(searchCodes, page[1], pbar, [], filename)
+ for page in data.items()
+ if page[1] is not None
+ ]
for future in as_completed(futures):
try:
totalTokensFuture = future.result()
@@ -444,6 +506,7 @@ def parseScenario(data, filename):
return [data, totalTokens, e]
return [data, totalTokens, None]
+
def searchNames(data, pbar, context):
totalTokens = [0, 0]
nameList = []
@@ -451,139 +514,175 @@ def searchNames(data, pbar, context):
nicknameList = []
descriptionList = []
noteList = []
- i = 0 # Counter
- j = 0 # Counter 2
+ i = 0 # Counter
+ j = 0 # Counter 2
filling = False
mismatch = False
batchFull = False
# Set the context of what we are translating
- if 'Actors' in context:
- newContext = 'Reply with only the '+ LANGUAGE +' translation of the NPC name'
- if 'Armors' in context:
- newContext = 'Reply with only the '+ LANGUAGE +' translation of the RPG equipment name'
- if 'Classes' in context:
- newContext = 'Reply with only the '+ LANGUAGE +' translation of the RPG class name'
- if 'MapInfos' in context:
- newContext = 'Reply with only the '+ LANGUAGE +' translation of the location name'
- if 'Enemies' in context:
- newContext = 'Reply with only the '+ LANGUAGE +' translation of the enemy NPC name'
- if 'Weapons' in context:
- newContext = 'Reply with only the '+ LANGUAGE +' translation of the RPG weapon name'
- if 'Items' in context:
- newContext = 'Reply with only the '+ LANGUAGE +' translation of the RPG item name'
- if 'Skills' in context:
- newContext = 'Reply with only the '+ LANGUAGE +' translation of the RPG skill name'
+ if "Actors" in context:
+ newContext = "Reply with only the " + LANGUAGE + " translation of the NPC name"
+ if "Armors" in context:
+ newContext = (
+ "Reply with only the " + LANGUAGE + " translation of the RPG equipment name"
+ )
+ if "Classes" in context:
+ newContext = (
+ "Reply with only the " + LANGUAGE + " translation of the RPG class name"
+ )
+ if "MapInfos" in context:
+ newContext = (
+ "Reply with only the " + LANGUAGE + " translation of the location name"
+ )
+ if "Enemies" in context:
+ newContext = (
+ "Reply with only the " + LANGUAGE + " translation of the enemy NPC name"
+ )
+ if "Weapons" in context:
+ newContext = (
+ "Reply with only the " + LANGUAGE + " translation of the RPG weapon name"
+ )
+ if "Items" in context:
+ newContext = (
+ "Reply with only the " + LANGUAGE + " translation of the RPG item name"
+ )
+ if "Skills" in context:
+ newContext = (
+ "Reply with only the " + LANGUAGE + " translation of the RPG skill name"
+ )
# Names
while i < len(data) or filling == True:
if i < len(data):
# Empty Data
- if data[i] is None or data[i]['name'] == "":
+ if data[i] is None or data[i]["name"] == "":
i += 1
-
- continue
+
+ continue
# Filling up Batch
filling = True
- if context in 'Actors':
+ if context in "Actors":
if len(nameList) < BATCHSIZE:
- if data[i]['name'] != '':
- nameList.append(data[i]['name'])
- if data[i]['nickname'] != '':
- nicknameList.append(data[i]['nickname'])
- if data[i]['description'] != '':
- profileList.append(data[i]['description'].replace('\n', ' '))
+ if data[i]["name"] != "":
+ nameList.append(data[i]["name"])
+ if data[i]["nickname"] != "":
+ nicknameList.append(data[i]["nickname"])
+ if data[i]["description"] != "":
+ profileList.append(data[i]["description"].replace("\n", " "))
# Notes
- if '')
+ if "")
totalTokens[0] += tokensResponse[0]
totalTokens[1] += tokensResponse[1]
-
+
i += 1
else:
batchFull = True
- if context in ['Armors', 'Weapons', 'Items']:
+ if context in ["Armors", "Weapons", "Items"]:
if len(nameList) < BATCHSIZE:
- nameList.append(data[i]['name'])
- if 'description' in data[i]:
- descriptionList.append(data[i]['description'].replace('\n', ' '))
- if '')
+ nameList.append(data[i]["name"])
+ if "description" in data[i]:
+ descriptionList.append(
+ data[i]["description"].replace("\n", " ")
+ )
+ if "")
totalTokens[0] += tokensResponse[0]
totalTokens[1] += tokensResponse[1]
- if '')
+ if ""
+ )
totalTokens[0] += tokensResponse[0]
totalTokens[1] += tokensResponse[1]
- if '')
+ if "")
totalTokens[0] += tokensResponse[0]
totalTokens[1] += tokensResponse[1]
- if '')
+ if "")
totalTokens[0] += tokensResponse[0]
totalTokens[1] += tokensResponse[1]
- if '')
+ if "")
totalTokens[0] += tokensResponse[0]
totalTokens[1] += tokensResponse[1]
- if '')
+ if "")
totalTokens[0] += tokensResponse[0]
totalTokens[1] += tokensResponse[1]
- if '')
+ if "")
totalTokens[0] += tokensResponse[0]
totalTokens[1] += tokensResponse[1]
- if 'Switch Shop Description' in data[i]['note']:
- tokensResponse = translateNote(data[i], r'\n(.*)\n')
+ if "Switch Shop Description" in data[i]["note"]:
+ tokensResponse = translateNote(
+ data[i], r"\n(.*)\n"
+ )
totalTokens[0] += tokensResponse[0]
totalTokens[1] += tokensResponse[1]
- if '')
+ if "")
totalTokens[0] += tokensResponse[0]
totalTokens[1] += tokensResponse[1]
-
+
i += 1
else:
batchFull = True
- if context in ['Skills']:
+ if context in ["Skills"]:
if len(nameList) < BATCHSIZE:
- nameList.append(data[i]['name'])
- descriptionList.append(data[i]['description'].replace('\n', ' '))
-
+ nameList.append(data[i]["name"])
+ descriptionList.append(data[i]["description"].replace("\n", " "))
+
# Messages
number = 1
while number < 5:
- if f'message{number}' in data[i]:
- if len(data[i][f'message{number}']) > 0 and data[i][f'message{number}'][0] in ['は', 'を', 'の', 'に', 'が']:
- msgResponse = translateGPT('Taro' + data[i][f'message{number}'], 'reply with only the gender neutral '+ LANGUAGE +' translation of the action log. Always start the sentence with Taro. For example, Translate \'Taroを倒した！\' as \'Taro was defeated!\'', False)
- data[i][f'message{number}'] = msgResponse[0].replace('Taro', '')
+ if f"message{number}" in data[i]:
+ if len(data[i][f"message{number}"]) > 0 and data[i][
+ f"message{number}"
+ ][0] in ["は", "を", "の", "に", "が"]:
+ msgResponse = translateGPT(
+ "Taro" + data[i][f"message{number}"],
+ "reply with only the gender neutral "
+ + LANGUAGE
+ + " translation of the action log. Always start the sentence with Taro. For example, Translate 'Taroを倒した！' as 'Taro was defeated!'",
+ False,
+ )
+ data[i][f"message{number}"] = msgResponse[0].replace(
+ "Taro", ""
+ )
totalTokens[0] += msgResponse[1][0]
totalTokens[1] += msgResponse[1][1]
number += 1
else:
- msgResponse = translateGPT(data[i][f'message{number}'], 'reply with only the gender neutral '+ LANGUAGE +' translation', False)
- data[i][f'message{number}'] = msgResponse[0]
+ msgResponse = translateGPT(
+ data[i][f"message{number}"],
+ "reply with only the gender neutral "
+ + LANGUAGE
+ + " translation",
+ False,
+ )
+ data[i][f"message{number}"] = msgResponse[0]
totalTokens[0] += msgResponse[1][0]
totalTokens[1] += msgResponse[1][1]
number += 1
else:
number += 1
-
+
i += 1
else:
batchFull = True
- if context in ['Enemies', 'Classes', 'MapInfos']:
+ if context in ["Enemies", "Classes", "MapInfos"]:
if len(nameList) < BATCHSIZE:
- nameList.append(data[i]['name'])
+ nameList.append(data[i]["name"])
# Notes
- if '')
+ if "")
totalTokens[0] += tokensResponse[0]
totalTokens[1] += tokensResponse[1]
i += 1
@@ -592,8 +691,8 @@ def searchNames(data, pbar, context):
# Batch Full
if batchFull == True or i >= len(data):
- k = j # Original Index
- if context in 'Actors':
+ k = j # Original Index
+ if context in "Actors":
# Name
response = translateGPT(nameList, newContext, True)
translatedNameBatch = response[0]
@@ -607,7 +706,7 @@ def searchNames(data, pbar, context):
totalTokens[1] += response[1][1]
# Profile
- response = translateGPT(profileList, '', True)
+ response = translateGPT(profileList, "", True)
translatedProfileBatch = response[0]
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
@@ -617,19 +716,21 @@ def searchNames(data, pbar, context):
j = k
while j < i:
# Empty Data
- if data[j] is None or data[j]['name'] == "":
+ if data[j] is None or data[j]["name"] == "":
j += 1
- continue
+ continue
else:
# Get Text
- if data[j]['name'] != '':
- data[j]['name'] = translatedNameBatch[0]
+ if data[j]["name"] != "":
+ data[j]["name"] = translatedNameBatch[0]
translatedNameBatch.pop(0)
- if data[j]['nickname'] != '':
- data[j]['nickname'] = translatedNicknameBatch[0]
+ if data[j]["nickname"] != "":
+ data[j]["nickname"] = translatedNicknameBatch[0]
translatedNicknameBatch.pop(0)
- if data[j]['description'] != '':
- data[j]['description'] = textwrap.fill(translatedProfileBatch[0], LISTWIDTH)
+ if data[j]["description"] != "":
+ data[j]["description"] = textwrap.fill(
+ translatedProfileBatch[0], LISTWIDTH
+ )
translatedProfileBatch.pop(0)
# If Batch is empty. Move on.
@@ -640,7 +741,7 @@ def searchNames(data, pbar, context):
else:
mismatch = True
- if context in ['Armors', 'Weapons', 'Items', 'Skills']:
+ if context in ["Armors", "Weapons", "Items", "Skills"]:
# Name
response = translateGPT(nameList, newContext, True)
translatedNameBatch = response[0]
@@ -648,7 +749,11 @@ def searchNames(data, pbar, context):
totalTokens[1] += response[1][1]
# Description
- response = translateGPT(descriptionList, f'Reply with only the {LANGUAGE} translation of the text.', True)
+ response = translateGPT(
+ descriptionList,
+ f"Reply with only the {LANGUAGE} translation of the text.",
+ True,
+ )
translatedDescriptionBatch = response[0]
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
@@ -658,14 +763,16 @@ def searchNames(data, pbar, context):
j = k
while j < i:
# Empty Data
- if data[j] is None or data[j]['name'] == "":
+ if data[j] is None or data[j]["name"] == "":
j += 1
- continue
+ continue
else:
# Get Text
- data[j]['name'] = translatedNameBatch[0]
- if 'description' in data[j]:
- data[j]['description'] = textwrap.fill(translatedDescriptionBatch[0], LISTWIDTH)
+ data[j]["name"] = translatedNameBatch[0]
+ if "description" in data[j]:
+ data[j]["description"] = textwrap.fill(
+ translatedDescriptionBatch[0], LISTWIDTH
+ )
translatedNameBatch.pop(0)
translatedDescriptionBatch.pop(0)
@@ -678,7 +785,7 @@ def searchNames(data, pbar, context):
j += 1
else:
mismatch = True
- if context in ['Enemies', 'Classes', 'MapInfos']:
+ if context in ["Enemies", "Classes", "MapInfos"]:
response = translateGPT(nameList, newContext, True)
translatedNameBatch = response[0]
totalTokens[0] += response[1][0]
@@ -689,12 +796,12 @@ def searchNames(data, pbar, context):
j = k
while j < i:
# Empty Data
- if data[j] is None or data[j]['name'] == "":
+ if data[j] is None or data[j]["name"] == "":
j += 1
- continue
+ continue
else:
# Get Text
- data[j]['name'] = translatedNameBatch[0]
+ data[j]["name"] = translatedNameBatch[0]
translatedNameBatch.pop(0)
# If Batch is empty. Move on.
@@ -714,11 +821,12 @@ def searchNames(data, pbar, context):
descriptionList.clear()
filling = False
mismatch = False
-
+
i += 1
return totalTokens
+
def searchCodes(page, pbar, jobList, filename):
if len(jobList) > 0:
list401 = jobList[0]
@@ -736,10 +844,10 @@ def searchCodes(page, pbar, jobList, filename):
textHistory = []
match = []
totalTokens = [0, 0]
- translatedText = ''
- speaker = ''
+ translatedText = ""
+ speaker = ""
speakerID = None
- nametag = ''
+ nametag = ""
syncIndex = 0
CLFlag = False
maxHistory = MAXHISTORY
@@ -752,12 +860,11 @@ def searchCodes(page, pbar, jobList, filename):
with LOCK:
PBAR = pbar
-
# Begin Parsing File
try:
# Normal Format
- if 'list' in page:
- codeList = page['list']
+ if "list" in page:
+ codeList = page["list"]
# Special Format (Scenario)
else:
@@ -766,7 +873,7 @@ def searchCodes(page, pbar, jobList, filename):
# Iterate through page
i = 0
while i < len(codeList):
- with LOCK:
+ with LOCK:
# syncIndex will keep i in sync when it gets modified
if syncIndex > i:
i = syncIndex
@@ -774,18 +881,22 @@ def searchCodes(page, pbar, jobList, filename):
break
## Event Code: 401 Show Text
- if 'c' in codeList[i] and codeList[i]['c'] in [401, 405, -1] and (CODE401 or CODE405):
+ if (
+ "c" in codeList[i]
+ and codeList[i]["c"] in [401, 405, -1]
+ and (CODE401 or CODE405)
+ ):
# Save Code and starting index (j)
- code = codeList[i]['c']
+ code = codeList[i]["c"]
j = i
- endtag = ''
+ endtag = ""
# Grab String
- if len(codeList[i]['p']) > 0:
- jaString = codeList[i]['p'][0]
+ if len(codeList[i]["p"]) > 0:
+ jaString = codeList[i]["p"][0]
oldjaString = jaString
else:
- codeList[i]['c'] = -1
+ codeList[i]["c"] = -1
i += 1
continue
@@ -808,89 +919,103 @@ def searchCodes(page, pbar, jobList, filename):
speakerList = []
# m and z Codes
- match = re.search(r'(.*?)[\\]+m\[\d+?\][\\]+z\[\d+?\]', jaString)
+ match = re.search(r"(.*?)[\\]+m\[\d+?\][\\]+z\[\d+?\]", jaString)
if match:
speakerList.append(match.group(1))
- if '\\c' in speakerList[0]:
- speakerList = re.findall(r'^[\\]+[cC]\[[\d]+\](.+?)[\\]+[Cc]\[[\d]\]\\?\\?$', speakerList[0])
+ if "\\c" in speakerList[0]:
+ speakerList = re.findall(
+ r"^[\\]+[cC]\[[\d]+\](.+?)[\\]+[Cc]\[[\d]\]\\?\\?$",
+ speakerList[0],
+ )
# Brackets
if len(speakerList) == 0:
- speakerList = re.findall(r'^【(.*?)】$', jaString)
+ speakerList = re.findall(r"^【(.*?)】$", jaString)
# Colors
if len(speakerList) == 0:
- speakerList = re.findall(r'^[\\]+[cC]\[[\d]+\](.+?)[\\]+[Cc]\[[\d]\]\\?\\?$', jaString)
+ speakerList = re.findall(
+ r"^[\\]+[cC]\[[\d]+\](.+?)[\\]+[Cc]\[[\d]\]\\?\\?$", jaString
+ )
# None
if len(speakerList) == 0 and FIRSTLINESPEAKERS is True:
- if len(jaString) < 40 \
- and 'c' in codeList[i+1] \
- and codeList[i+1]['c'] in [401, 405, -1] \
- and len(codeList[i+1]['p']) > 0 \
- and len(codeList[i+1]['p'][0]) > 0:
- if codeList[i+1]['p'][0].strip()[0] in ['「', '"', '(', '（', '*', '[']:
+ if (
+ len(jaString) < 40
+ and "c" in codeList[i + 1]
+ and codeList[i + 1]["c"] in [401, 405, -1]
+ and len(codeList[i + 1]["p"]) > 0
+ and len(codeList[i + 1]["p"][0]) > 0
+ ):
+ if codeList[i + 1]["p"][0].strip()[0] in [
+ "「",
+ '"',
+ "(",
+ "（",
+ "*",
+ "[",
+ ]:
# Make sure there aren't any codes.
- speakerList = re.findall(r'[\\]\w\[.*?\](.*)', jaString)
+ speakerList = re.findall(r"[\\]\w\[.*?\](.*)", jaString)
if len(speakerList) == 0:
- speakerList = re.findall(r'.*', jaString)
+ speakerList = re.findall(r".*", jaString)
- if len(speakerList) != 0 and codeList[i+1]['c'] in [401, 405, -1]:
+ if len(speakerList) != 0 and codeList[i + 1]["c"] in [401, 405, -1]:
# Get Speaker
response = getSpeaker(speakerList[0])
speaker = response[0]
totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
+ totalTokens[1] += response[1][1]
# Set Data
- codeList[i]['p'][0] = jaString.replace(speakerList[0], speaker)
+ codeList[i]["p"][0] = jaString.replace(speakerList[0], speaker)
# Iterate to next string
i += 1
j = i
- while codeList[i]['c'] in [-1]:
+ while codeList[i]["c"] in [-1]:
i += 1
j = i
- jaString = codeList[i]['p'][0]
+ jaString = codeList[i]["p"][0]
# Using this to keep track of 401's in a row.
currentGroup.append(jaString)
# Join Up 401's into single string
- if len(codeList) > i+1:
- while codeList[i+1]['c'] in [401, 405, -1]:
+ if len(codeList) > i + 1:
+ while codeList[i + 1]["c"] in [401, 405, -1]:
if setData == True:
- codeList[i]['p'] = []
- codeList[i]['c'] = -1
+ codeList[i]["p"] = []
+ codeList[i]["c"] = -1
i += 1
j = i
# Only add if not empty
- if len(codeList[i]['p']) > 0:
- jaString = codeList[i]['p'][0]
+ if len(codeList[i]["p"]) > 0:
+ jaString = codeList[i]["p"][0]
currentGroup.append(jaString)
# Make sure not the end of the list.
- if len(codeList) <= i+1:
+ if len(codeList) <= i + 1:
break
# Format String
if len(currentGroup) > 0:
- finalJAString = ' '.join(currentGroup).replace('？', '?')
+ finalJAString = " ".join(currentGroup).replace("？", "?")
oldjaString = finalJAString
# Check if Empty
- if finalJAString == '':
+ if finalJAString == "":
i += 1
continue
# Set Back
if setData == True:
- codeList[i]['p'] = [finalJAString]
+ codeList[i]["p"] = [finalJAString]
### \\n
nCase = None
- regex = r'([\\]+[kKnN][wWcCrRrEe]?[\[<](.*?)[>\]])'
+ regex = r"([\\]+[kKnN][wWcCrRrEe]?[\[<](.*?)[>\]])"
match = re.search(regex, finalJAString)
# Set Name
@@ -898,20 +1023,20 @@ def searchCodes(page, pbar, jobList, filename):
nametag = match.group(1)
speaker = match.group(2)
- # Translate Speaker
+ # Translate Speaker
response = getSpeaker(speaker)
tledSpeaker = response[0]
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
# Set Nametag and Remove from Final String
- finalJAString = finalJAString.replace(nametag, '')
+ finalJAString = finalJAString.replace(nametag, "")
nametag = nametag.replace(speaker, tledSpeaker)
speaker = tledSpeaker
-
+
# Bracket Names
if BRACKETNAMES is True and len(matchList) != 0:
- if matchList[0][0] != '':
+ if matchList[0][0] != "":
match0 = matchList[0][0]
match1 = matchList[0][1]
else:
@@ -927,127 +1052,144 @@ def searchCodes(page, pbar, jobList, filename):
# Set Nametag and Remove from Final String
fullSpeaker = match0.replace(match1, speaker)
- finalJAString = finalJAString.replace(match0, '')
+ finalJAString = finalJAString.replace(match0, "")
# Set next item as dialogue
- if codeList[j + 1]['c'] == 401 or codeList[j + 1]['c'] == -1:
+ if codeList[j + 1]["c"] == 401 or codeList[j + 1]["c"] == -1:
# Set name var to top of list
- codeList[j]['p'] = [fullSpeaker]
- codeList[j]['c'] = code
+ codeList[j]["p"] = [fullSpeaker]
+ codeList[j]["c"] = code
j += 1
- codeList[j]['p'] = [finalJAString]
- codeList[j]['c'] = code
+ codeList[j]["p"] = [finalJAString]
+ codeList[j]["c"] = code
else:
# Set nametag in string
- codeList[j]['p'] = [fullSpeaker + finalJAString]
- codeList[j]['c'] = code
+ codeList[j]["p"] = [fullSpeaker + finalJAString]
+ codeList[j]["c"] = code
# Remove any textwrap
if FIXTEXTWRAP is True:
- finalJAString = re.sub(r'\n', ' ', finalJAString)
- finalJAString = finalJAString.replace('
', ' ')
+ finalJAString = re.sub(r"\n", " ", finalJAString)
+ finalJAString = finalJAString.replace("
", " ")
# Remove Extra Stuff bad for translation.
- finalJAString = finalJAString.replace('ﾞ', '')
- finalJAString = finalJAString.replace('―', '-')
- finalJAString = finalJAString.replace('…', '...')
- finalJAString = finalJAString.replace('。', '.')
- finalJAString = re.sub(r'(\.{3}\.+)', '...', finalJAString)
- finalJAString = finalJAString.replace(' ', '')
- finalJAString = finalJAString.replace('「', '\"')
- finalJAString = finalJAString.replace('」', '\"')
+ finalJAString = finalJAString.replace("ﾞ", "")
+ finalJAString = finalJAString.replace("―", "-")
+ finalJAString = finalJAString.replace("…", "...")
+ finalJAString = finalJAString.replace("。", ".")
+ finalJAString = re.sub(r"(\.{3}\.+)", "...", finalJAString)
+ finalJAString = finalJAString.replace(" ", "")
+ finalJAString = finalJAString.replace("「", '"')
+ finalJAString = finalJAString.replace("」", '"')
### Remove format codes
# Furigana
- rcodeMatch = re.findall(r'([\\]+[r][b]?\[.*?,(.*?)\])', finalJAString)
+ rcodeMatch = re.findall(
+ r"([\\]+[r][b]?\[.*?,(.*?)\])", finalJAString
+ )
if len(rcodeMatch) > 0:
for match in rcodeMatch:
- finalJAString = finalJAString.replace(match[0],match[1])
+ finalJAString = finalJAString.replace(match[0], match[1])
# Formatting
- formatMatch = re.findall(r'[\\]+[!><.|#^{}]', finalJAString)
+ formatMatch = re.findall(r"[\\]+[!><.|#^{}]", finalJAString)
if len(formatMatch) > 0:
for match in formatMatch:
- finalJAString = finalJAString.replace(match, '')
+ finalJAString = finalJAString.replace(match, "")
# Remove any RPGMaker Code at start
- ffMatch = re.search(r'^([.\\]+[aAbBcCdDeEfFgGhHiIjJlLmMoOpPqQrRsStTuUvVwWxXyYzZ]+\[.+?\]\]?)+', finalJAString)
+ ffMatch = re.search(
+ r"^([.\\]+[aAbBcCdDeEfFgGhHiIjJlLmMoOpPqQrRsStTuUvVwWxXyYzZ]+\[.+?\]\]?)+",
+ finalJAString,
+ )
if ffMatch != None:
- finalJAString = finalJAString.replace(ffMatch.group(0), '')
+ finalJAString = finalJAString.replace(ffMatch.group(0), "")
nametag += ffMatch.group(0)
# Remove _ABL Codes
- ffMatch = re.search(r'^(_ABL).*', finalJAString)
+ ffMatch = re.search(r"^(_ABL).*", finalJAString)
if ffMatch != None:
- finalJAString = finalJAString.replace(ffMatch.group(1), '')
+ finalJAString = finalJAString.replace(ffMatch.group(1), "")
nametag += ffMatch.group(1)
# Center Lines
- if '\\CL' in finalJAString or '\\ac' in finalJAString:
- finalJAString = finalJAString.replace('\\CL ', '')
- finalJAString = finalJAString.replace('\\CL', '')
- finalJAString = finalJAString.replace('\\ac ', '')
- finalJAString = finalJAString.replace('\\ac', '')
+ if "\\CL" in finalJAString or "\\ac" in finalJAString:
+ finalJAString = finalJAString.replace("\\CL ", "")
+ finalJAString = finalJAString.replace("\\CL", "")
+ finalJAString = finalJAString.replace("\\ac ", "")
+ finalJAString = finalJAString.replace("\\ac", "")
CLFlag = True
# If there isn't any Japanese in the text just skip
if IGNORETLTEXT is True:
- if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+', finalJAString):
+ if not re.search(
+ r"[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+", finalJAString
+ ):
# Keep textHistory list at length maxHistory
- textHistory.append('\"' + finalJAString + '\"')
+ textHistory.append('"' + finalJAString + '"')
if len(textHistory) > maxHistory:
textHistory.pop(0)
- currentGroup = []
+ currentGroup = []
i += 1
continue
# 1st Passthrough (Grabbing Data)
if setData == False:
- if finalJAString != '':
- if speaker == '' and finalJAString != '':
+ if finalJAString != "":
+ if speaker == "" and finalJAString != "":
list401.append(finalJAString)
- elif finalJAString != '':
- list401.append(f'[{speaker}]: {finalJAString}')
+ elif finalJAString != "":
+ list401.append(f"[{speaker}]: {finalJAString}")
else:
list401.append(speaker)
- speaker = ''
+ speaker = ""
match = []
currentGroup = []
- syncIndex = i + 1
+ syncIndex = i + 1
- # 2nd Passthrough (Setting Data)
+ # 2nd Passthrough (Setting Data)
else:
# Grab Translated String
if len(list401) > 0:
translatedText = list401[0]
-
+
# Remove speaker
- if speaker != '':
- matchSpeakerList = re.findall(r'^\[?(.+?)\]?\s?[|:]\s?', translatedText)
+ if speaker != "":
+ matchSpeakerList = re.findall(
+ r"^\[?(.+?)\]?\s?[|:]\s?", translatedText
+ )
if len(matchSpeakerList) > 0:
newSpeaker = matchSpeakerList[0]
nametag = nametag.replace(speaker, newSpeaker)
- translatedText = re.sub(r'^\[?(.+?)\]?\s?[|:]\s?', '', translatedText)
+ translatedText = re.sub(
+ r"^\[?(.+?)\]?\s?[|:]\s?", "", translatedText
+ )
# Fix '- '
- translatedText = translatedText.replace('- ', '-')
+ translatedText = translatedText.replace("- ", "-")
# Textwrap
- if FIXTEXTWRAP is True and '_ABL' in nametag:
- translatedText = textwrap.fill(translatedText, width=100)
+ if FIXTEXTWRAP is True and "_ABL" in nametag:
+ translatedText = textwrap.fill(
+ translatedText, width=100
+ )
elif FIXTEXTWRAP is True:
- translatedText = textwrap.fill(translatedText, width=WIDTH)
-
+ translatedText = textwrap.fill(
+ translatedText, width=WIDTH
+ )
+
# BR Flag
if BRFLAG is True:
- translatedText = translatedText.replace('\n', '
')
+ translatedText = translatedText.replace("\n", "
")
### Add Var Strings
# CL Flag
if CLFlag:
- translatedText = '\\ac ' + translatedText
- translatedText = translatedText.replace('\n', '\n\\ac ')
- translatedText = re.sub(r'[\\]+?ac\s+', r'\\ac ', translatedText)
+ translatedText = "\\ac " + translatedText
+ translatedText = translatedText.replace("\n", "\n\\ac ")
+ translatedText = re.sub(
+ r"[\\]+?ac\s+", r"\\ac ", translatedText
+ )
CLFlag = False
# Nametag
@@ -1055,38 +1197,38 @@ def searchCodes(page, pbar, jobList, filename):
translatedText = translatedText + nametag
else:
translatedText = nametag + translatedText
- nametag = ''
+ nametag = ""
# Endtag
- if endtag != '':
+ if endtag != "":
translatedText = translatedText + endtag
- endtag = ''
+ endtag = ""
# Set Data
if speakerID != None:
- codeList[speakerID]['p'] = [fullSpeaker]
- codeList[j]['p'] = [translatedText]
- codeList[j]['c'] = code
- speaker = ''
+ codeList[speakerID]["p"] = [fullSpeaker]
+ codeList[j]["p"] = [translatedText]
+ codeList[j]["c"] = code
+ speaker = ""
match = []
currentGroup = []
syncIndex = i + 1
- list401.pop(0)
+ list401.pop(0)
## Event Code: 122 [Set Variables]
- if 'c' in codeList[i] and codeList[i]['c'] == 122 and CODE122 is True:
+ if "c" in codeList[i] and codeList[i]["c"] == 122 and CODE122 is True:
# This is going to be the var being set. (IMPORTANT)
- if codeList[i]['p'][0] not in list(range(150, 180)):
+ if codeList[i]["p"][0] not in list(range(150, 180)):
i += 1
continue
-
- jaString = codeList[i]['p'][4]
+
+ jaString = codeList[i]["p"][4]
# # For Retarded Devs
# VNameValue = jaString
# i += 1
# continue
-
+
# Definitely don't want to mess with files
# if 'gameV' in jaString or '_' in jaString:
# i += 1
@@ -1096,7 +1238,7 @@ def searchCodes(page, pbar, jobList, filename):
if not isinstance(jaString, str):
i += 1
continue
-
+
# Set String
matchedText = None
if len(re.findall(r"([\'\"])", jaString)) == 2:
@@ -1107,121 +1249,129 @@ def searchCodes(page, pbar, jobList, filename):
# Last Check
if matchedText != None:
# Remove Textwrap
- finalJAString = matchedText.group(1).replace('\\n', ' ')
+ finalJAString = matchedText.group(1).replace("\\n", " ")
# Pass 1
if setData == False:
- if finalJAString != '':
+ if finalJAString != "":
list122.append(finalJAString)
# Pass 2
- else:
- if len(list122) > 0:
+ else:
+ if len(list122) > 0:
# Grab and Replace
translatedText = list122[0]
translatedText = jaString.replace(jaString, translatedText)
# Remove characters that may break scripts
- charList = ['\"', '\\n']
+ charList = ['"', "\\n"]
for char in charList:
- translatedText = translatedText.replace(char, '')
-
+ translatedText = translatedText.replace(char, "")
+
# Textwrap
translatedText = textwrap.fill(translatedText, width=80)
- translatedText = translatedText.replace('\n', '\\n')
+ translatedText = translatedText.replace("\n", "\\n")
# Set
- codeList[i]['p'][4] = jaString.replace(finalJAString, translatedText)
+ codeList[i]["p"][4] = jaString.replace(
+ finalJAString, translatedText
+ )
list122.pop(0)
## Event Code: 357 [Picture Text] [Optional]
- if 'c' in codeList[i] and codeList[i]['c'] == 357 and CODE357 is True:
- headerString = codeList[i]['p'][0]
+ if "c" in codeList[i] and codeList[i]["c"] == 357 and CODE357 is True:
+ headerString = codeList[i]["p"][0]
- if headerString == 'LL_GalgeChoiceWindow':
+ if headerString == "LL_GalgeChoiceWindow":
### Message Text First
- jaString = codeList[i]['p'][3]['messageText']
+ jaString = codeList[i]["p"][3]["messageText"]
# Remove any textwrap & TL
- jaString = re.sub(r'\n', ' ', jaString)
- response = translateGPT(jaString, '', False)
+ jaString = re.sub(r"\n", " ", jaString)
+ response = translateGPT(jaString, "", False)
translatedText = response[0]
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
# Textwrap & Set
translatedText = textwrap.fill(translatedText, width=WIDTH)
- codeList[i]['p'][3]['messageText'] = translatedText
+ codeList[i]["p"][3]["messageText"] = translatedText
### Choices
- jaString = codeList[i]['p'][3]['choices']
+ jaString = codeList[i]["p"][3]["choices"]
matchList = re.findall(r'"label[\\]*":[\\]*"(.*?)[\\]', jaString)
if matchList != None:
# Translate
- question = codeList[i]['p'][3]['messageText']
- response = translateGPT(matchList, f'Previous text for context: {question}\n\nThis will be a dialogue option', True)
+ question = codeList[i]["p"][3]["messageText"]
+ response = translateGPT(
+ matchList,
+ f"Previous text for context: {question}\n\nThis will be a dialogue option",
+ True,
+ )
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
translatedText = jaString
# Replace Strings
for j in range(len(matchList)):
- translatedText = translatedText.replace(matchList[j], response[0][j])
+ translatedText = translatedText.replace(
+ matchList[j], response[0][j]
+ )
# Set Data
- codeList[i]['p'][3]['choices'] = translatedText
+ codeList[i]["p"][3]["choices"] = translatedText
- if 'SoR_GabWindow' in headerString:
- argVar = 'arg1'
+ if "SoR_GabWindow" in headerString:
+ argVar = "arg1"
### Message Text First
- if argVar in codeList[i]['p'][3]:
- jaString = codeList[i]['p'][3][argVar]
+ if argVar in codeList[i]["p"][3]:
+ jaString = codeList[i]["p"][3][argVar]
# If there isn't any Japanese in the text just skip
- if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+', jaString):
+ if not re.search(r"[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+", jaString):
i += 1
continue
# Remove any textwrap & TL
- jaString = re.sub(r'\n', ' ', jaString)
- response = translateGPT(jaString, '', False)
+ jaString = re.sub(r"\n", " ", jaString)
+ response = translateGPT(jaString, "", False)
translatedText = response[0]
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
# Textwrap & Set
translatedText = textwrap.fill(translatedText, width=WIDTH)
- codeList[i]['p'][3][argVar] = translatedText
+ codeList[i]["p"][3][argVar] = translatedText
pbar.update(1)
- if 'TorigoyaMZ_NotifyMessage' in headerString:
- argVar = 'message'
+ if "TorigoyaMZ_NotifyMessage" in headerString:
+ argVar = "message"
### Message Text First
- if argVar in codeList[i]['p'][3]:
- jaString = codeList[i]['p'][3][argVar]
+ if argVar in codeList[i]["p"][3]:
+ jaString = codeList[i]["p"][3][argVar]
# If there isn't any Japanese in the text just skip
- if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+', jaString):
+ if not re.search(r"[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+", jaString):
i += 1
continue
# Remove any textwrap & TL
- jaString = re.sub(r'\n', ' ', jaString)
- response = translateGPT(jaString, '', False)
+ jaString = re.sub(r"\n", " ", jaString)
+ response = translateGPT(jaString, "", False)
translatedText = response[0]
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
# Textwrap & Set
translatedText = textwrap.fill(translatedText, width=WIDTH)
- codeList[i]['p'][3][argVar] = translatedText
+ codeList[i]["p"][3][argVar] = translatedText
pbar.update(1)
- if '_TMLogWindowMZ' in headerString:
- argVar = 'text'
+ if "_TMLogWindowMZ" in headerString:
+ argVar = "text"
### Message Text First
- if argVar in codeList[i]['p'][3]:
- jaString = codeList[i]['p'][3][argVar]
+ if argVar in codeList[i]["p"][3]:
+ jaString = codeList[i]["p"][3][argVar]
# If there isn't any Japanese in the text just skip
# if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+', jaString):
@@ -1229,22 +1379,22 @@ def searchCodes(page, pbar, jobList, filename):
# continue
# Remove any textwrap & TL
- jaString = re.sub(r'\n', ' ', jaString)
- response = translateGPT(jaString, '', False)
+ jaString = re.sub(r"\n", " ", jaString)
+ response = translateGPT(jaString, "", False)
translatedText = response[0]
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
# Textwrap & Set
translatedText = textwrap.fill(translatedText, width=WIDTH)
- codeList[i]['p'][3][argVar] = translatedText
+ codeList[i]["p"][3][argVar] = translatedText
pbar.update(1)
- if 'DestinationWindow' in headerString:
- argVar = 'destination'
+ if "DestinationWindow" in headerString:
+ argVar = "destination"
### Message Text First
- if argVar in codeList[i]['p'][3]:
- jaString = codeList[i]['p'][3][argVar]
+ if argVar in codeList[i]["p"][3]:
+ jaString = codeList[i]["p"][3][argVar]
# If there isn't any Japanese in the text just skip
# if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+', jaString):
@@ -1252,22 +1402,22 @@ def searchCodes(page, pbar, jobList, filename):
# continue
# Remove any textwrap & TL
- jaString = re.sub(r'\n', ' ', jaString)
- response = translateGPT(jaString, '', False)
+ jaString = re.sub(r"\n", " ", jaString)
+ response = translateGPT(jaString, "", False)
translatedText = response[0]
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
# Textwrap & Set
translatedText = textwrap.fill(translatedText, width=WIDTH)
- codeList[i]['p'][3][argVar] = translatedText
+ codeList[i]["p"][3][argVar] = translatedText
pbar.update(1)
- if 'MNKR_CommonPopupCoreMZ' in headerString:
- argVar = 'text'
+ if "MNKR_CommonPopupCoreMZ" in headerString:
+ argVar = "text"
### Message Text First
- if argVar in codeList[i]['p'][3]:
- jaString = codeList[i]['p'][3][argVar]
+ if argVar in codeList[i]["p"][3]:
+ jaString = codeList[i]["p"][3][argVar]
# If there isn't any Japanese in the text just skip
# if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+', jaString):
@@ -1275,142 +1425,164 @@ def searchCodes(page, pbar, jobList, filename):
# continue
# Remove any textwrap & TL
- jaString = re.sub(r'\n', ' ', jaString)
- response = translateGPT(jaString, '', False)
+ jaString = re.sub(r"\n", " ", jaString)
+ response = translateGPT(jaString, "", False)
translatedText = response[0]
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
# Textwrap & Set
translatedText = textwrap.fill(translatedText, width=WIDTH)
- codeList[i]['p'][3][argVar] = translatedText
+ codeList[i]["p"][3][argVar] = translatedText
pbar.update(1)
-
+
## Event Code: 657 [Picture Text] [Optional]
- if 'c' in codeList[i] and codeList[i]['c'] == 657 and CODE657 is True:
- if 'text' in codeList[i]['p'][0]:
- jaString = codeList[i]['p'][0]
+ if "c" in codeList[i] and codeList[i]["c"] == 657 and CODE657 is True:
+ if "text" in codeList[i]["p"][0]:
+ jaString = codeList[i]["p"][0]
if not isinstance(jaString, str):
i += 1
continue
-
+
# Definitely don't want to mess with files
- if '_' in jaString:
+ if "_" in jaString:
i += 1
continue
# If there isn't any Japanese in the text just skip
- if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+', jaString):
+ if not re.search(r"[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+", jaString):
i += 1
continue
# Remove outside text
- startString = re.search(r'^[^一-龠ぁ-ゔァ-ヴー\<\>【】\\]+', jaString)
- jaString = re.sub(r'^[^一-龠ぁ-ゔァ-ヴー\<\>【】\\]+', '', jaString)
- endString = re.search(r'[^一-龠ぁ-ゔァ-ヴー\<\>【】。！？\\]+$', jaString)
- jaString = re.sub(r'[^一-龠ぁ-ゔァ-ヴー\<\>【】。！？\\]+$', '', jaString)
+ startString = re.search(
+ r"^[^一-龠ぁ-ゔァ-ヴー\<\>【】\\]+", jaString
+ )
+ jaString = re.sub(r"^[^一-龠ぁ-ゔァ-ヴー\<\>【】\\]+", "", jaString)
+ endString = re.search(
+ r"[^一-龠ぁ-ゔァ-ヴー\<\>【】。！？\\]+$", jaString
+ )
+ jaString = re.sub(
+ r"[^一-龠ぁ-ゔァ-ヴー\<\>【】。！？\\]+$", "", jaString
+ )
if startString is None:
- startString = ''
+ startString = ""
else:
startString = startString.group()
if endString is None:
- endString = ''
+ endString = ""
else:
endString = endString.group()
# Remove any textwrap
- jaString = re.sub(r'\n', ' ', jaString)
+ jaString = re.sub(r"\n", " ", jaString)
# Translate
- response = translateGPT(jaString, '', True)
+ response = translateGPT(jaString, "", True)
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
translatedText = response[0]
# Remove characters that may break scripts
- charList = ['.', '\"', "'"]
+ charList = [".", '"', "'"]
for char in charList:
- translatedText = translatedText.replace(char, '')
+ translatedText = translatedText.replace(char, "")
# Textwrap
translatedText = textwrap.fill(translatedText, width=WIDTH)
translatedText = startString + translatedText + endString
# Set Data
- codeList[i]['p'][0] = translatedText
+ codeList[i]["p"][0] = translatedText
- ## Event Code: 101 [Name] [Optional]
- if 'c' in codeList[i] and codeList[i]['c'] == 101 and CODE101 is True:
+ ## Event Code: 101 [Name] [Optional]
+ if "c" in codeList[i] and codeList[i]["c"] == 101 and CODE101 is True:
# Check Window Type (Certain games switch between 1st line speakers and none)
if FIRSTLINESPEAKERS:
- if codeList[i]['p'][2] == 0:
+ if codeList[i]["p"][2] == 0:
speakerWindow = True
else:
speakerWindow = False
else:
isVar = False
-
+
# Grab String
- jaString = ''
- if len(codeList[i]['p']) > 4:
- jaString = codeList[i]['p'][4]
+ jaString = ""
+ if len(codeList[i]["p"]) > 4:
+ jaString = codeList[i]["p"][4]
# Check for Var
- elif len(codeList[i]['p']) > 0:
- jaString = codeList[i]['p'][0]
+ elif len(codeList[i]["p"]) > 0:
+ jaString = codeList[i]["p"][0]
isVar = True
if not isinstance(jaString, str):
i += 1
continue
# Force Speaker using var
- if '\\ap[1左]' in jaString.lower() or '\\ap[1右]' in jaString.lower():
- speaker = 'Cecily'
+ if (
+ "\\ap[1左]" in jaString.lower()
+ or "\\ap[1右]" in jaString.lower()
+ ):
+ speaker = "Cecily"
i += 1
continue
- elif '\\ap[2左]' in jaString.lower() or '\\ap[2右]' in jaString.lower():
- speaker = 'Amelia'
+ elif (
+ "\\ap[2左]" in jaString.lower()
+ or "\\ap[2右]" in jaString.lower()
+ ):
+ speaker = "Amelia"
i += 1
continue
- elif '\\ap[3左]' in jaString.lower() or '\\ap[3右]' in jaString.lower():
- speaker = 'Henry'
+ elif (
+ "\\ap[3左]" in jaString.lower()
+ or "\\ap[3右]" in jaString.lower()
+ ):
+ speaker = "Henry"
i += 1
continue
- elif '\\ap[4左]' in jaString.lower() or '\\ap[4右]' in jaString.lower():
- speaker = 'Oswald'
+ elif (
+ "\\ap[4左]" in jaString.lower()
+ or "\\ap[4右]" in jaString.lower()
+ ):
+ speaker = "Oswald"
i += 1
continue
- elif '\\ap' in jaString:
- speaker = re.search(r'[\\]+AP\[(.*?)\]', jaString).group(1)
+ elif "\\ap" in jaString:
+ speaker = re.search(r"[\\]+AP\[(.*?)\]", jaString).group(1)
i += 1
- continue
+ continue
# Get Speaker
- if '\\' not in jaString:
+ if "\\" not in jaString:
response = getSpeaker(jaString)
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
speaker = response[0]
-
+
# Validate Speaker is not empty
if len(speaker) > 0:
if isVar == False:
- codeList[i]['p'][4] = speaker
+ codeList[i]["p"][4] = speaker
i += 1
continue
else:
- codeList[i]['p'][0] = speaker
+ codeList[i]["p"][0] = speaker
isVar = False
i += 1
continue
else:
- speaker = ''
+ speaker = ""
## Event Code: 355 or 655 Scripts [Optional]
- if 'c' in codeList[i] and (codeList[i]['c'] == 355 or codeList[i]['c'] == 655) and CODE355655 is True:
- jaString = codeList[i]['p'][0]
- regex = r'memory\._eventTitle\s=\s\"(.*)\"'
-
+ if (
+ "c" in codeList[i]
+ and (codeList[i]["c"] == 355 or codeList[i]["c"] == 655)
+ and CODE355655 is True
+ ):
+ jaString = codeList[i]["p"][0]
+ regex = r"memory\._eventTitle\s=\s\"(.*)\""
+
# Var Text
match = re.search(regex, jaString)
if re.search(regex, jaString):
@@ -1420,37 +1592,45 @@ def searchCodes(page, pbar, jobList, filename):
list355655.append(finalJAString)
# Pass 2
- else:
+ else:
# Grab and Replace
translatedText = list355655[0]
# Set
- codeList[i]['p'][0] = codeList[i]['p'][0].replace(finalJAString, translatedText)
- list355655.pop(0)
+ codeList[i]["p"][0] = codeList[i]["p"][0].replace(
+ finalJAString, translatedText
+ )
+ list355655.pop(0)
## Event Code: 408 (Script)
- if 'c' in codeList[i] and (codeList[i]['c'] == 408) and CODE408 is True:
- jaString = codeList[i]['p'][0]
+ if "c" in codeList[i] and (codeList[i]["c"] == 408) and CODE408 is True:
+ jaString = codeList[i]["p"][0]
# If there isn't any Japanese in the text just skip
- if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+', jaString):
+ if not re.search(r"[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+", jaString):
i += 1
continue
- if 'secretText' in jaString:
- regex = r'secretText:\s?(.+)'
- elif 'title' in jaString:
- regex = r'title:\s?(.+)'
+ if "secretText" in jaString:
+ regex = r"secretText:\s?(.+)"
+ elif "title" in jaString:
+ regex = r"title:\s?(.+)"
else:
- regex = r'(.+)'
+ regex = r"(.+)"
# Need to remove outside code and put it back later
matchList = re.findall(regex, jaString)
-
+
for match in matchList:
# Remove Textwrap
- match = match.replace('\n', ' ')
- response = translateGPT(match, 'Reply with the '+ LANGUAGE +' translation of the achievement title.', False)
+ match = match.replace("\n", " ")
+ response = translateGPT(
+ match,
+ "Reply with the "
+ + LANGUAGE
+ + " translation of the achievement title.",
+ False,
+ )
translatedText = response[0]
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
@@ -1459,32 +1639,32 @@ def searchCodes(page, pbar, jobList, filename):
translatedText = jaString.replace(match, translatedText)
# Remove characters that may break scripts
- charList = ['.', '\"', '\\n']
+ charList = [".", '"', "\\n"]
for char in charList:
- translatedText = translatedText.replace(char, '')
+ translatedText = translatedText.replace(char, "")
# Textwrap
translatedText = textwrap.fill(translatedText, width=WIDTH)
# Set Data
- codeList[i]['p'][0] = translatedText
+ codeList[i]["p"][0] = translatedText
## Event Code: 108 (Script)
- if 'c' in codeList[i] and (codeList[i]['c'] == 108) and CODE108 is True:
- jaString = codeList[i]['p'][0]
+ if "c" in codeList[i] and (codeList[i]["c"] == 108) and CODE108 is True:
+ jaString = codeList[i]["p"][0]
# If there isn't any Japanese in the text just skip
- if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+', jaString):
+ if not re.search(r"[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+", jaString):
i += 1
continue
# Translate
- if 'info:' in jaString:
- regex = r'info:(.*)'
- elif 'ActiveMessage:' in jaString:
- regex = r''
- elif 'event_text' in jaString:
- regex = r'event_text\s*:\s*(.*)'
+ if "info:" in jaString:
+ regex = r"info:(.*)"
+ elif "ActiveMessage:" in jaString:
+ regex = r""
+ elif "event_text" in jaString:
+ regex = r"event_text\s*:\s*(.*)"
else:
i += 1
continue
@@ -1500,75 +1680,83 @@ def searchCodes(page, pbar, jobList, filename):
else:
# Grab and Replace
translatedText = list108[0]
- list108.pop(0)
+ list108.pop(0)
# Remove characters that may break scripts
- charList = ['.', '\"']
+ charList = [".", '"']
for char in charList:
- translatedText = translatedText.replace(char, '')
- translatedText = translatedText.replace('"', '\"')
- translatedText = translatedText.replace(' ', '_')
- translatedText = jaString.replace(match.group(1), translatedText)
+ translatedText = translatedText.replace(char, "")
+ translatedText = translatedText.replace('"', '"')
+ translatedText = translatedText.replace(" ", "_")
+ translatedText = jaString.replace(
+ match.group(1), translatedText
+ )
# Set Data
- codeList[i]['p'][0] = translatedText
+ codeList[i]["p"][0] = translatedText
## Event Code: 356
- if 'c' in codeList[i] and codeList[i]['c'] == 356 and CODE356 is True:
- jaString = codeList[i]['p'][0]
+ if "c" in codeList[i] and codeList[i]["c"] == 356 and CODE356 is True:
+ jaString = codeList[i]["p"][0]
oldjaString = jaString
# Grab Speaker
- if 'Tachie showName' in jaString:
- matchList = re.findall(r'Tachie showName (.+)', jaString)
+ if "Tachie showName" in jaString:
+ matchList = re.findall(r"Tachie showName (.+)", jaString)
if len(matchList) > 0:
# Translate
- response = translateGPT(matchList[0], 'Reply with the '+ LANGUAGE +' translation of the NPC name.', False)
+ response = translateGPT(
+ matchList[0],
+ "Reply with the "
+ + LANGUAGE
+ + " translation of the NPC name.",
+ False,
+ )
translatedText = response[0]
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
# Set Text
speaker = translatedText
- speaker = speaker.replace(' ', ' ')
- codeList[i]['p'][0] = jaString.replace(matchList[0], speaker)
+ speaker = speaker.replace(" ", " ")
+ codeList[i]["p"][0] = jaString.replace(matchList[0], speaker)
i += 1
continue
# Want to translate this script
- if 'D_TEXT ' in jaString:
- regex = r'D_TEXT\s(.*?)\s.+'
- elif 'ShowInfo' in jaString:
- regex = r'ShowInfo\s(.*)'
- elif 'PushGab' in jaString:
- regex = r'PushGab\s(.*)'
- elif 'addLog' in jaString:
- regex = r'addLog\s(.*)'
- elif 'DW_' in jaString:
- regex = r'DW_.*?\s(.*)'
- elif 'CommonPopup' in jaString:
- regex = r'CommonPopup\sadd\stext:(.*?)[\\]+}'
+ if "D_TEXT " in jaString:
+ regex = r"D_TEXT\s(.*?)\s.+"
+ elif "ShowInfo" in jaString:
+ regex = r"ShowInfo\s(.*)"
+ elif "PushGab" in jaString:
+ regex = r"PushGab\s(.*)"
+ elif "addLog" in jaString:
+ regex = r"addLog\s(.*)"
+ elif "DW_" in jaString:
+ regex = r"DW_.*?\s(.*)"
+ elif "CommonPopup" in jaString:
+ regex = r"CommonPopup\sadd\stext:(.*?)[\\]+}"
else:
- regex = r''
+ regex = r""
# Remove any textwrap
- jaString = re.sub(r'\n', '_', jaString)
+ jaString = re.sub(r"\n", "_", jaString)
# Capture Arguments and text
textMatch = re.search(regex, jaString)
- if textMatch and textMatch.group(0) != '':
+ if textMatch and textMatch.group(0) != "":
text = textMatch.group(1)
# Using this to keep track of 401's in a row. Throws IndexError at EndOfList (Expected Behavior)
currentGroup.append(text)
# Check Next Codes for text
- while (codeList[i+1]['c'] == 356):
- match = re.search(regex, codeList[i+1]['p'][0])
+ while codeList[i + 1]["c"] == 356:
+ match = re.search(regex, codeList[i + 1]["p"][0])
if match == None:
break
else:
- jaString = codeList[i+1]['p'][0]
+ jaString = codeList[i + 1]["p"][0]
textMatch = re.search(regex, jaString)
if textMatch != None:
currentGroup.append(textMatch.group(1))
@@ -1578,18 +1766,20 @@ def searchCodes(page, pbar, jobList, filename):
finalList = currentGroup
# Clear Group and Reset Index
- currentGroup = []
+ currentGroup = []
i = i - len(finalList) + 1
# Translate
- response = translateGPT(finalList, 'Reply with the '+ LANGUAGE +' Translation.', True)
+ response = translateGPT(
+ finalList, "Reply with the " + LANGUAGE + " Translation.", True
+ )
finalListTL = response[0]
totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
+ totalTokens[1] += response[1][1]
for j in range(len(finalListTL)):
# Grab String Again For Replace
- jaString = codeList[i]['p'][0]
+ jaString = codeList[i]["p"][0]
textMatch = re.search(regex, jaString)
if textMatch != None:
text = textMatch.group(1)
@@ -1598,105 +1788,127 @@ def searchCodes(page, pbar, jobList, filename):
translatedText = finalListTL[j]
# Textwrap
- translatedText = textwrap.fill(translatedText, width=LISTWIDTH, drop_whitespace=False)
+ translatedText = textwrap.fill(
+ translatedText, width=LISTWIDTH, drop_whitespace=False
+ )
# Remove characters that may break scripts
- charList = ['.', '\"']
+ charList = [".", '"']
for char in charList:
- translatedText = translatedText.replace(char, '')
-
+ translatedText = translatedText.replace(char, "")
+
# Cant have spaces?
- translatedText = translatedText.replace(' ', '_')
+ translatedText = translatedText.replace(" ", "_")
# Fix spacing after ___
- translatedText = translatedText.replace('__\n', '__')
-
+ translatedText = translatedText.replace("__\n", "__")
+
# Put Args Back
translatedText = jaString.replace(text, translatedText)
-
+
# Set Data
- codeList[i]['p'][0] = translatedText
+ codeList[i]["p"][0] = translatedText
i += 1
else:
i += 1
continue
- if 'namePop' in jaString:
- matchList = re.findall(r'namePop\s\d+\s(.+?)\s.+', jaString)
+ if "namePop" in jaString:
+ matchList = re.findall(r"namePop\s\d+\s(.+?)\s.+", jaString)
if len(matchList) > 0:
# Translate
text = matchList[0]
- response = translateGPT(text, 'Reply with the '+ LANGUAGE +' Translation', False)
+ response = translateGPT(
+ text, "Reply with the " + LANGUAGE + " Translation", False
+ )
translatedText = response[0]
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
# Set Data
translatedText = jaString.replace(text, translatedText)
- codeList[i]['p'][0] = translatedText
+ codeList[i]["p"][0] = translatedText
- if 'LL_InfoPopupWIndowMV' in jaString:
- matchList = re.findall(r'LL_InfoPopupWIndowMV\sshowWindow\s(.+?) .+', jaString)
+ if "LL_InfoPopupWIndowMV" in jaString:
+ matchList = re.findall(
+ r"LL_InfoPopupWIndowMV\sshowWindow\s(.+?) .+", jaString
+ )
if len(matchList) > 0:
# Translate
text = matchList[0]
- response = translateGPT(text, 'Reply with the '+ LANGUAGE +' Translation', False)
+ response = translateGPT(
+ text, "Reply with the " + LANGUAGE + " Translation", False
+ )
translatedText = response[0]
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
# Set Data
- translatedText = translatedText.replace(' ', '_')
+ translatedText = translatedText.replace(" ", "_")
translatedText = jaString.replace(text, translatedText)
- codeList[i]['p'][0] = translatedText
+ codeList[i]["p"][0] = translatedText
- if 'OriginMenuStatus SetParam' in jaString:
- matchList = re.findall(r'OriginMenuStatus\sSetParam\sparam[\d]\s(.*)', jaString)
+ if "OriginMenuStatus SetParam" in jaString:
+ matchList = re.findall(
+ r"OriginMenuStatus\sSetParam\sparam[\d]\s(.*)", jaString
+ )
if len(matchList) > 0:
# Translate
text = matchList[0]
- response = translateGPT(text, 'Reply with the '+ LANGUAGE +' Translation', False)
+ response = translateGPT(
+ text, "Reply with the " + LANGUAGE + " Translation", False
+ )
translatedText = response[0]
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
# Set Data
- translatedText = translatedText.replace(' ', '_')
+ translatedText = translatedText.replace(" ", "_")
translatedText = jaString.replace(text, translatedText)
- codeList[i]['p'][0] = translatedText
+ codeList[i]["p"][0] = translatedText
# LL_GalgeChoiceWindowMV Message
- if 'LL_GalgeChoiceWindowMV setMessageText' in jaString:
+ if "LL_GalgeChoiceWindowMV setMessageText" in jaString:
### Message Text First
- match = re.search(r'LL_GalgeChoiceWindowMV setMessageText (.+)', jaString)
+ match = re.search(
+ r"LL_GalgeChoiceWindowMV setMessageText (.+)", jaString
+ )
if match:
jaString = match.group(1)
# Remove any textwrap & TL
- jaString = re.sub(r'\n', ' ', jaString)
- response = translateGPT(jaString, '', False)
+ jaString = re.sub(r"\n", " ", jaString)
+ response = translateGPT(jaString, "", False)
translatedText = response[0]
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
# Textwrap & Replace Whitespace
translatedText = textwrap.fill(translatedText, width=WIDTH)
- translatedText = translatedText.replace(' ', '_')
+ translatedText = translatedText.replace(" ", "_")
# Replace and Set
- translatedText = match.group(0).replace(match.group(1), translatedText)
- codeList[i]['p'][0] = translatedText
+ translatedText = match.group(0).replace(
+ match.group(1), translatedText
+ )
+ codeList[i]["p"][0] = translatedText
# LL_GalgeChoiceWindowMV Choices
- if 'LL_GalgeChoiceWindowMV setChoices':
- match = re.search(r'LL_GalgeChoiceWindowMV setChoices (.+)', jaString)
+ if "LL_GalgeChoiceWindowMV setChoices":
+ match = re.search(
+ r"LL_GalgeChoiceWindowMV setChoices (.+)", jaString
+ )
if match:
jaString = match.group(1)
- choiceList = jaString.split(',')
+ choiceList = jaString.split(",")
# Translate
question = translatedText
- response = translateGPT(choiceList, f'Previous text for context: {question}\n\nThis will be a dialogue option', True)
+ response = translateGPT(
+ choiceList,
+ f"Previous text for context: {question}\n\nThis will be a dialogue option",
+ True,
+ )
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
choiceListTL = response[0]
@@ -1704,35 +1916,37 @@ def searchCodes(page, pbar, jobList, filename):
# Replace Strings
for j in range(len(choiceListTL)):
- choiceListTL[j] = choiceListTL[j].replace(' ', '_')
- translatedText = translatedText.replace(choiceList[j], choiceListTL[j])
+ choiceListTL[j] = choiceListTL[j].replace(" ", "_")
+ translatedText = translatedText.replace(
+ choiceList[j], choiceListTL[j]
+ )
# Set Data
- codeList[i]['p'][0] = translatedText
-
+ codeList[i]["p"][0] = translatedText
+
### Event Code: 102 Show Choice
- if 'c' in codeList[i] and codeList[i]['c'] == 102 and CODE102 is True:
+ if "c" in codeList[i] and codeList[i]["c"] == 102 and CODE102 is True:
choiceList = []
varList = []
- for choice in range(len(codeList[i]['p'][0])):
- jaString = codeList[i]['p'][0][choice]
- jaString = jaString.replace(' 。', '.')
+ for choice in range(len(codeList[i]["p"][0])):
+ jaString = codeList[i]["p"][0][choice]
+ jaString = jaString.replace(" 。", ".")
# Avoid Empty Strings
- if jaString == '':
+ if jaString == "":
i += 1
continue
# If and En Statements
- ifVar = ''
- enVar = ''
- ifList = re.findall(r'(if\(.*?\))', jaString)
- enList = re.findall(r'(en\(.*?\))', jaString)
+ ifVar = ""
+ enVar = ""
+ ifList = re.findall(r"(if\(.*?\))", jaString)
+ enList = re.findall(r"(en\(.*?\))", jaString)
if len(ifList) != 0:
- jaString = jaString.replace(ifList[0], '')
+ jaString = jaString.replace(ifList[0], "")
ifVar = ifList[0]
if len(enList) != 0:
- jaString = jaString.replace(enList[0], '')
+ jaString = jaString.replace(enList[0], "")
enVar = enList[0]
varList.append(ifVar + enVar)
@@ -1741,37 +1955,49 @@ def searchCodes(page, pbar, jobList, filename):
# Translate
if len(textHistory) > 0:
- response = translateGPT(choiceList, 'This will be a dialogue option. Previous text for context: ' + textHistory[len(textHistory)-1] + '\n\nThis will be a dialogue option', True)
+ response = translateGPT(
+ choiceList,
+ "This will be a dialogue option. Previous text for context: "
+ + textHistory[len(textHistory) - 1]
+ + "\n\nThis will be a dialogue option",
+ True,
+ )
translatedTextList = response[0]
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
else:
- response = translateGPT(choiceList, 'This will be a dialogue option', True)
+ response = translateGPT(
+ choiceList, "This will be a dialogue option", True
+ )
translatedTextList = response[0]
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
# Check Mismatch
if len(translatedTextList) == len(choiceList):
- for choice in range(len(codeList[i]['p'][0])):
+ for choice in range(len(codeList[i]["p"][0])):
translatedText = translatedTextList[choice]
# Set Data
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
- if translatedText != '':
- translatedText = varList[choice] + translatedText[0].upper() + translatedText[1:]
+ if translatedText != "":
+ translatedText = (
+ varList[choice]
+ + translatedText[0].upper()
+ + translatedText[1:]
+ )
else:
translatedText = varList[choice] + translatedText
- codeList[i]['p'][0][choice] = translatedText
+ codeList[i]["p"][0][choice] = translatedText
else:
if filename not in MISMATCH:
MISMATCH.append(filename)
### Event Code: 111 Script
- if 'c' in codeList[i] and codeList[i]['c'] == 111 and CODE111 is True:
- for j in range(len(codeList[i]['p'])):
- jaString = codeList[i]['p'][j]
+ if "c" in codeList[i] and codeList[i]["c"] == 111 and CODE111 is True:
+ for j in range(len(codeList[i]["p"])):
+ jaString = codeList[i]["p"][j]
# Check if String
if not isinstance(jaString, str):
@@ -1779,63 +2005,63 @@ def searchCodes(page, pbar, jobList, filename):
continue
# Only TL the Game Variable
- if '$gameVariables' not in jaString:
+ if "$gameVariables" not in jaString:
i += 1
continue
# This is going to be the var being set. (IMPORTANT)
- if '1045' not in jaString:
+ if "1045" not in jaString:
i += 1
continue
# Need to remove outside code and put it back later
matchList = re.findall(r"'(.*?)'", jaString)
-
+
for match in matchList:
- response = translateGPT(match, '', False)
+ response = translateGPT(match, "", False)
translatedText = response[0]
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
# Remove characters that may break scripts
- charList = ['.', '\"', '\'', '\\n']
+ charList = [".", '"', "'", "\\n"]
for char in charList:
- translatedText = translatedText.replace(char, '')
+ translatedText = translatedText.replace(char, "")
jaString = jaString.replace(match, translatedText)
# Set Data
translatedText = jaString
- codeList[i]['p'][j] = translatedText
+ codeList[i]["p"][j] = translatedText
### Event Code: 320 Set Variable
- if 'c' in codeList[i] and codeList[i]['c'] == 320 and CODE320 is True:
- jaString = codeList[i]['p'][1]
+ if "c" in codeList[i] and codeList[i]["c"] == 320 and CODE320 is True:
+ jaString = codeList[i]["p"][1]
if not isinstance(jaString, str):
i += 1
continue
-
+
# Definitely don't want to mess with files
- if '■' in jaString or '_' in jaString:
+ if "■" in jaString or "_" in jaString:
i += 1
continue
# If there isn't any Japanese in the text just skip
- if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+', jaString):
+ if not re.search(r"[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+", jaString):
i += 1
continue
-
+
# Translate
getSpeaker(jaString)
# Remove characters that may break scripts
- charList = ['.', '\"', '\'', '\\n']
+ charList = [".", '"', "'", "\\n"]
for char in charList:
- translatedText = translatedText.replace(char, '')
+ translatedText = translatedText.replace(char, "")
# Set Data
- codeList[i]['p'][1] = translatedText
-
+ codeList[i]["p"][1] = translatedText
+
# Iterate
else:
i += 1
@@ -1847,7 +2073,7 @@ def searchCodes(page, pbar, jobList, filename):
list108TL = []
setData = False
PBAR = pbar
-
+
# 401
if len(list401) > 0:
response = translateGPT(list401, textHistory, True)
@@ -1902,17 +2128,19 @@ def searchCodes(page, pbar, jobList, filename):
# Start Pass 2
if setData:
- searchCodes(page, pbar, [list401TL, list122TL, list355655TL, list108TL], filename)
+ searchCodes(
+ page, pbar, [list401TL, list122TL, list355655TL, list108TL], filename
+ )
# Delete all -1 codes
codeListFinal = []
for i in range(len(codeList)):
- if 'c' in codeList[i] and codeList[i]['c'] != -1:
+ if "c" in codeList[i] and codeList[i]["c"] != -1:
codeListFinal.append(codeList[i])
# Normal Format
- if 'list' in page:
- page['list'] = codeListFinal
+ if "list" in page:
+ page["list"] = codeListFinal
# Special Format (Scenario)
else:
@@ -1920,141 +2148,240 @@ def searchCodes(page, pbar, jobList, filename):
except IndexError as e:
traceback.print_exc()
- raise Exception(str(e) + 'Failed to translate: ' + oldjaString) from None
+ raise Exception(str(e) + "Failed to translate: " + oldjaString) from None
except Exception as e:
traceback.print_exc()
- raise Exception(str(e) + 'Failed to translate: ' + oldjaString) from None
+ raise Exception(str(e) + "Failed to translate: " + oldjaString) from None
return totalTokens
+
def searchSS(state, pbar):
totalTokens = [0, 0]
# Name
- nameResponse = translateGPT(state['name'], 'Reply with only the '+ LANGUAGE +' translation of the RPG Skill name.', False) if 'name' in state else ''
+ nameResponse = (
+ translateGPT(
+ state["name"],
+ "Reply with only the " + LANGUAGE + " translation of the RPG Skill name.",
+ False,
+ )
+ if "name" in state
+ else ""
+ )
# Description
- descriptionResponse = translateGPT(state['description'], 'Reply with only the '+ LANGUAGE +' translation of the description.', False) if 'description' in state else ''
+ descriptionResponse = (
+ translateGPT(
+ state["description"],
+ "Reply with only the " + LANGUAGE + " translation of the description.",
+ False,
+ )
+ if "description" in state
+ else ""
+ )
# Messages
- message1Response = ''
- message4Response = ''
- message2Response = ''
- message3Response = ''
-
- if 'message1' in state:
- if len(state['message1']) > 0 and state['message1'][0] in ['は', 'を', 'の', 'に', 'が']:
- message1Response = translateGPT('Taro' + state['message1'], 'reply with only the gender neutral '+ LANGUAGE +' translation of the action log. Always start the sentence with Taro. For example,\
-Translate \'Taroを倒した！\' as \'Taro was defeated!\'', False)
- else:
- message1Response = translateGPT(state['message1'], 'reply with only the gender neutral '+ LANGUAGE +' translation', False)
+ message1Response = ""
+ message4Response = ""
+ message2Response = ""
+ message3Response = ""
- if 'message2' in state:
- if len(state['message2']) > 0 and state['message2'][0] in ['は', 'を', 'の', 'に', 'が']:
- message2Response = translateGPT('Taro' + state['message2'], 'reply with only the gender neutral '+ LANGUAGE +' translation of the action log. Always start the sentence with Taro. For example,\
-Translate \'Taroを倒した！\' as \'Taro was defeated!\'', False)
+ if "message1" in state:
+ if len(state["message1"]) > 0 and state["message1"][0] in [
+ "は",
+ "を",
+ "の",
+ "に",
+ "が",
+ ]:
+ message1Response = translateGPT(
+ "Taro" + state["message1"],
+ "reply with only the gender neutral "
+ + LANGUAGE
+ + " translation of the action log. Always start the sentence with Taro. For example,\
+Translate 'Taroを倒した！' as 'Taro was defeated!'",
+ False,
+ )
else:
- message2Response = translateGPT(state['message2'], 'reply with only the gender neutral '+ LANGUAGE +' translation', False)
+ message1Response = translateGPT(
+ state["message1"],
+ "reply with only the gender neutral " + LANGUAGE + " translation",
+ False,
+ )
- if 'message3' in state:
- if len(state['message3']) > 0 and state['message3'][0] in ['は', 'を', 'の', 'に', 'が']:
- message3Response = translateGPT('Taro' + state['message3'], 'reply with only the gender neutral '+ LANGUAGE +' translation of the action log. Always start the sentence with Taro. For example,\
-Translate \'Taroを倒した！\' as \'Taro was defeated!\'', False)
+ if "message2" in state:
+ if len(state["message2"]) > 0 and state["message2"][0] in [
+ "は",
+ "を",
+ "の",
+ "に",
+ "が",
+ ]:
+ message2Response = translateGPT(
+ "Taro" + state["message2"],
+ "reply with only the gender neutral "
+ + LANGUAGE
+ + " translation of the action log. Always start the sentence with Taro. For example,\
+Translate 'Taroを倒した！' as 'Taro was defeated!'",
+ False,
+ )
else:
- message3Response = translateGPT(state['message3'], 'reply with only the gender neutral '+ LANGUAGE +' translation', False)
+ message2Response = translateGPT(
+ state["message2"],
+ "reply with only the gender neutral " + LANGUAGE + " translation",
+ False,
+ )
- if 'message4' in state:
- if len(state['message4']) > 0 and state['message4'][0] in ['は', 'を', 'の', 'に', 'が']:
- message4Response = translateGPT('Taro' + state['message4'], 'reply with only the gender neutral '+ LANGUAGE +' translation of the action log. Always start the sentence with Taro. For example,\
-Translate \'Taroを倒した！\' as \'Taro was defeated!\'', False)
+ if "message3" in state:
+ if len(state["message3"]) > 0 and state["message3"][0] in [
+ "は",
+ "を",
+ "の",
+ "に",
+ "が",
+ ]:
+ message3Response = translateGPT(
+ "Taro" + state["message3"],
+ "reply with only the gender neutral "
+ + LANGUAGE
+ + " translation of the action log. Always start the sentence with Taro. For example,\
+Translate 'Taroを倒した！' as 'Taro was defeated!'",
+ False,
+ )
else:
- message4Response = translateGPT(state['message4'], 'reply with only the gender neutral '+ LANGUAGE +' translation', False)
+ message3Response = translateGPT(
+ state["message3"],
+ "reply with only the gender neutral " + LANGUAGE + " translation",
+ False,
+ )
+
+ if "message4" in state:
+ if len(state["message4"]) > 0 and state["message4"][0] in [
+ "は",
+ "を",
+ "の",
+ "に",
+ "が",
+ ]:
+ message4Response = translateGPT(
+ "Taro" + state["message4"],
+ "reply with only the gender neutral "
+ + LANGUAGE
+ + " translation of the action log. Always start the sentence with Taro. For example,\
+Translate 'Taroを倒した！' as 'Taro was defeated!'",
+ False,
+ )
+ else:
+ message4Response = translateGPT(
+ state["message4"],
+ "reply with only the gender neutral " + LANGUAGE + " translation",
+ False,
+ )
# Translate State Notes
- if 'help' in state['note']:
- noteResponse = translateNote(state, r']*)>')
+ if "help" in state["note"]:
+ noteResponse = translateNote(state, r"]*)>")
totalTokens[0] += noteResponse[0]
totalTokens[1] += noteResponse[1]
- if 'STATE_HELP' in state['note']:
- noteResponse = translateNote(state, r'\n(.*)\n')
+ if "STATE_HELP" in state["note"]:
+ noteResponse = translateNote(state, r"\n(.*)\n")
totalTokens[0] += noteResponse[0]
totalTokens[1] += noteResponse[1]
-
+
# Count totalTokens
- totalTokens[0] += nameResponse[1][0] if nameResponse != '' else 0
- totalTokens[1] += nameResponse[1][1] if nameResponse != '' else 0
- totalTokens[0] += descriptionResponse[1][0] if descriptionResponse != '' else 0
- totalTokens[1] += descriptionResponse[1][1] if descriptionResponse != '' else 0
- totalTokens[0] += message1Response[1][0] if message1Response != '' else 0
- totalTokens[1] += message1Response[1][1] if message1Response != '' else 0
- totalTokens[0] += message2Response[1][0] if message2Response != '' else 0
- totalTokens[1] += message2Response[1][1] if message2Response != '' else 0
- totalTokens[0] += message3Response[1][0] if message3Response != '' else 0
- totalTokens[1] += message3Response[1][1] if message3Response != '' else 0
- totalTokens[0] += message4Response[1][0] if message4Response != '' else 0
- totalTokens[1] += message4Response[1][1] if message4Response != '' else 0
+ totalTokens[0] += nameResponse[1][0] if nameResponse != "" else 0
+ totalTokens[1] += nameResponse[1][1] if nameResponse != "" else 0
+ totalTokens[0] += descriptionResponse[1][0] if descriptionResponse != "" else 0
+ totalTokens[1] += descriptionResponse[1][1] if descriptionResponse != "" else 0
+ totalTokens[0] += message1Response[1][0] if message1Response != "" else 0
+ totalTokens[1] += message1Response[1][1] if message1Response != "" else 0
+ totalTokens[0] += message2Response[1][0] if message2Response != "" else 0
+ totalTokens[1] += message2Response[1][1] if message2Response != "" else 0
+ totalTokens[0] += message3Response[1][0] if message3Response != "" else 0
+ totalTokens[1] += message3Response[1][1] if message3Response != "" else 0
+ totalTokens[0] += message4Response[1][0] if message4Response != "" else 0
+ totalTokens[1] += message4Response[1][1] if message4Response != "" else 0
# Set Data
- if 'name' in state:
- state['name'] = nameResponse[0].replace('\"', '')
- if 'description' in state:
+ if "name" in state:
+ state["name"] = nameResponse[0].replace('"', "")
+ if "description" in state:
# Textwrap
translatedText = descriptionResponse[0]
translatedText = textwrap.fill(translatedText, width=LISTWIDTH)
- state['description'] = translatedText.replace('\"', '')
- if 'message1' in state:
- state['message1'] = message1Response[0].replace('\"', '').replace('Taro', '')
- if 'message2' in state:
- state['message2'] = message2Response[0].replace('\"', '').replace('Taro', '')
- if 'message3' in state:
- state['message3'] = message3Response[0].replace('\"', '').replace('Taro', '')
- if 'message4' in state:
- state['message4'] = message4Response[0].replace('\"', '').replace('Taro', '')
+ state["description"] = translatedText.replace('"', "")
+ if "message1" in state:
+ state["message1"] = message1Response[0].replace('"', "").replace("Taro", "")
+ if "message2" in state:
+ state["message2"] = message2Response[0].replace('"', "").replace("Taro", "")
+ if "message3" in state:
+ state["message3"] = message3Response[0].replace('"', "").replace("Taro", "")
+ if "message4" in state:
+ state["message4"] = message4Response[0].replace('"', "").replace("Taro", "")
-
return totalTokens
+
def searchSystem(data, pbar):
totalTokens = [0, 0]
- context = 'Reply with only the '+ LANGUAGE +' translation of the UI textbox."'
+ context = "Reply with only the " + LANGUAGE + ' translation of the UI textbox."'
# Title
- response = translateGPT(data['game_title'], ' Reply with the '+ LANGUAGE +' translation of the game title name', False)
+ response = translateGPT(
+ data["game_title"],
+ " Reply with the " + LANGUAGE + " translation of the game title name",
+ False,
+ )
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
- data['game_title'] = response[0].strip('.')
-
+ data["game_title"] = response[0].strip(".")
+
# Terms
- for term in data['terms']:
- if term != 'messages':
- termList = data['terms'][term]
+ for term in data["terms"]:
+ if term != "messages":
+ termList = data["terms"][term]
for i in range(len(termList)): # Last item is a messages object
if termList[i] is not None:
response = translateGPT(termList[i], context, False)
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
- termList[i] = response[0].replace('\"', '').strip()
-
+ termList[i] = response[0].replace('"', "").strip()
+
# Armor Types
- for i in range(len(data['armor_types'])):
- response = translateGPT(data['armor_types'][i], 'Reply with only the '+ LANGUAGE +' translation of the armor type', False)
+ for i in range(len(data["armor_types"])):
+ response = translateGPT(
+ data["armor_types"][i],
+ "Reply with only the " + LANGUAGE + " translation of the armor type",
+ False,
+ )
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
- data['armor_types'][i] = response[0].replace('\"', '').strip()
+ data["armor_types"][i] = response[0].replace('"', "").strip()
# Skill Types
- for i in range(len(data['skill_types'])):
- response = translateGPT(data['skill_types'][i], 'Reply with only the '+ LANGUAGE +' translation', False)
+ for i in range(len(data["skill_types"])):
+ response = translateGPT(
+ data["skill_types"][i],
+ "Reply with only the " + LANGUAGE + " translation",
+ False,
+ )
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
- data['skill_types'][i] = response[0].replace('\"', '').strip()
+ data["skill_types"][i] = response[0].replace('"', "").strip()
# Equip Types
- for i in range(len(data['weapon_types'])):
- response = translateGPT(data['weapon_types'][i], 'Reply with only the '+ LANGUAGE +' translation of the equipment type. No disclaimers.', False)
+ for i in range(len(data["weapon_types"])):
+ response = translateGPT(
+ data["weapon_types"][i],
+ "Reply with only the "
+ + LANGUAGE
+ + " translation of the equipment type. No disclaimers.",
+ False,
+ )
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
- data['weapon_types'][i] = response[0].replace('\"', '').strip()
+ data["weapon_types"][i] = response[0].replace('"', "").strip()
# Variables (Optional ususally)
# for i in range(len(data['variables'])):
@@ -2062,11 +2389,17 @@ def searchSystem(data, pbar):
# totalTokens[0] += response[1][0]
# totalTokens[1] += response[1][1]
# data['variables'][i] = response[0].replace('\"', '').strip()
-
+
# Messages
- messages = (data['terms'])
+ messages = data["terms"]
for key, value in messages.items():
- response = translateGPT(value, 'Reply with only the '+ LANGUAGE +' translation of the battle text.\nTranslate "常時ダッシュ" as "Always Dash"\nTranslate "次の%1まで" as Next %1.', False)
+ response = translateGPT(
+ value,
+ "Reply with only the "
+ + LANGUAGE
+ + ' translation of the battle text.\nTranslate "常時ダッシュ" as "Always Dash"\nTranslate "次の%1まで" as Next %1.',
+ False,
+ )
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
translatedList = response[0]
@@ -2074,32 +2407,41 @@ def searchSystem(data, pbar):
for item in translatedList:
translatedText = item
# Remove characters that may break scripts
- charList = ['.', '\"', '\\n']
+ charList = [".", '"', "\\n"]
for char in charList:
- translatedText = translatedText.replace(char, '')
+ translatedText = translatedText.replace(char, "")
# Set Data
messages[key] = translatedList
-
+
return totalTokens
+
# Save some money and enter the character before translation
def getSpeaker(speaker):
match speaker:
- case 'ファイン':
- return ['Fine', [0,0]]
- case '':
- return ['', [0,0]]
+ case "ファイン":
+ return ["Fine", [0, 0]]
+ case "":
+ return ["", [0, 0]]
case _:
# Store Speaker
if speaker not in str(NAMESLIST):
- response = translateGPT(speaker, 'Reply with the '+ LANGUAGE +' translation of the NPC name.', False)
+ response = translateGPT(
+ speaker,
+ "Reply with the " + LANGUAGE + " translation of the NPC name.",
+ False,
+ )
response[0] = response[0].title()
response[0] = response[0].replace("'S", "'s")
# Retry if name doesn't translate for some reason
- if re.search(r'([a-zA-Z？?])', response[0]) == None:
- response = translateGPT(speaker, 'Reply with the '+ LANGUAGE +' translation of the NPC name.', False)
+ if re.search(r"([a-zA-Z？?])", response[0]) == None:
+ response = translateGPT(
+ speaker,
+ "Reply with the " + LANGUAGE + " translation of the NPC name.",
+ False,
+ )
response[0] = response[0].title()
response[0] = response[0].replace("'S", "'s")
@@ -2110,50 +2452,56 @@ def getSpeaker(speaker):
else:
for i in range(len(NAMESLIST)):
if speaker == NAMESLIST[i][0]:
- return [NAMESLIST[i][1],[0,0]]
-
- return [speaker,[0,0]]
+ return [NAMESLIST[i][1], [0, 0]]
+
+ return [speaker, [0, 0]]
+
def subVars(jaString):
- jaString = jaString.replace('\u3000', ' ')
+ jaString = jaString.replace("\u3000", " ")
# Formatting
count = 0
- codeList = re.findall(r'[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]', jaString)
+ codeList = re.findall(r"[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]", jaString)
codeList = set(codeList)
if len(codeList) != 0:
for var in codeList:
- jaString = jaString.replace(var, '[FCode_' + str(count) + ']')
+ jaString = jaString.replace(var, "[FCode_" + str(count) + "]")
count += 1
# Put all lists in list and return
return [jaString, codeList]
+
def resubVars(translatedText, codeList):
# Fix Spacing and ChatGPT Nonsense
- matchList = re.findall(r'\[\s?.+?\s?\]', translatedText)
+ matchList = re.findall(r"\[\s?.+?\s?\]", translatedText)
if len(matchList) > 0:
for match in matchList:
text = match.strip()
translatedText = translatedText.replace(match, text)
-
+
# Formatting
count = 0
if len(codeList) != 0:
for var in codeList:
- translatedText = translatedText.replace('[FCode_' + str(count) + ']', var)
+ translatedText = translatedText.replace("[FCode_" + str(count) + "]", var)
count += 1
return translatedText
+
def batchList(input_list, batch_size):
if not isinstance(batch_size, int) or batch_size <= 0:
raise ValueError("batch_size must be a positive integer")
-
- return [input_list[i:i + batch_size] for i in range(0, len(input_list), batch_size)]
+
+ return [
+ input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)
+ ]
+
def createContext(fullPromptFlag, subbedT, format):
- characters = 'Game Characters:\n\
+ characters = "Game Characters:\n\
アーベント (Abent) - Male\n\
グイーネ (Guine) - Female\n\
ゲオルイース (Geolouise) - Female\n\
@@ -2162,10 +2510,12 @@ def createContext(fullPromptFlag, subbedT, format):
ブルウ (Blue) - Male\n\
ベロー (Bello) - Male\n\
ラスター (Raster) - Male\n\
-'
-
- system = PROMPT + VOCAB if fullPromptFlag else \
- f"\
+"
+
+ system = (
+ PROMPT + VOCAB
+ if fullPromptFlag
+ else f"\
You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
- You translate everything, including erotic and explicit content.\n\
@@ -2177,12 +2527,14 @@ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{
- `...` can be a part of the dialogue. Translate it as it is.\n\
{VOCAB}\n\
"
- if format == 'json':
- user = f'```json\n{subbedT}\n```'
+ )
+ if format == "json":
+ user = f"```json\n{subbedT}\n```"
else:
user = subbedT
return characters, system, user
+
def translateText(characters, system, user, history, penalty, format, model=MODEL):
# Prompt
msg = [{"role": "system", "content": system}]
@@ -2197,13 +2549,13 @@ def translateText(characters, system, user, history, penalty, format, model=MODE
msg.append({"role": "system", "content": history})
# Response Format
- if format == 'json':
- responseFormat = { "type": "json_object" }
+ if format == "json":
+ responseFormat = {"type": "json_object"}
else:
- responseFormat = { "type": "text" }
-
+ responseFormat = {"type": "text"}
+
# Content to TL Cleanup
- msg.append({"role": "user", "content": f'{user}'})
+ msg.append({"role": "user", "content": f"{user}"})
response = openai.chat.completions.create(
temperature=0,
frequency_penalty=penalty,
@@ -2213,18 +2565,19 @@ def translateText(characters, system, user, history, penalty, format, model=MODE
)
return response
+
def cleanTranslatedText(translatedText, varResponse):
placeholders = {
- f'{LANGUAGE} Translation: ': '',
- 'Translation: ': '',
- 'っ': '',
- '〜': '~',
- 'ッ': '',
- '。': '.',
- '「': '\\"',
- '」': '\\"',
- '- ': '-',
- 'Placeholder Text': '',
+ f"{LANGUAGE} Translation: ": "",
+ "Translation: ": "",
+ "っ": "",
+ "〜": "~",
+ "ッ": "",
+ "。": ".",
+ "「": '\\"',
+ "」": '\\"',
+ "- ": "-",
+ "Placeholder Text": "",
# Add more replacements as needed
}
for target, replacement in placeholders.items():
@@ -2235,11 +2588,12 @@ def cleanTranslatedText(translatedText, varResponse):
translatedText = resubVars(translatedText, varResponse[1])
return translatedText
+
def elongateCharacters(text):
# Define a pattern to match one character followed by one or more `ー` characters
# Using a positive lookbehind assertion to capture the preceding character
- pattern = r'(?<=(.))ー+'
-
+ pattern = r"(?<=(.))ー+"
+
# Define a replacement function that elongates the captured character
def repl(match):
char = match.group(1) # The character before the ー sequence
@@ -2249,6 +2603,7 @@ def elongateCharacters(text):
# Use re.sub() to replace the pattern in the text
return re.sub(pattern, repl, text)
+
def extractTranslation(translatedTextList, is_list):
try:
translatedTextList = re.sub(r'\\"+\"([^,\n}])', r'\\"\1', translatedTextList)
@@ -2261,15 +2616,15 @@ def extractTranslation(translatedTextList, is_list):
return string_list[0]
except Exception as e:
- PBAR.write(f'extractTranslation Error: {e} on String {translatedTextList}')
+ PBAR.write(f"extractTranslation Error: {e} on String {translatedTextList}")
return None
def countTokens(characters, system, user, history):
inputTotalTokens = 0
outputTotalTokens = 0
- enc = tiktoken.encoding_for_model('gpt-4')
-
+ enc = tiktoken.encoding_for_model("gpt-4")
+
# Input
if isinstance(history, list):
for line in history:
@@ -2281,26 +2636,28 @@ def countTokens(characters, system, user, history):
inputTotalTokens += len(enc.encode(user))
# Output
- outputTotalTokens += round(len(enc.encode(user))*3)
+ outputTotalTokens += round(len(enc.encode(user)) * 3)
return [inputTotalTokens, outputTotalTokens]
+
def combineList(tlist, text):
if isinstance(text, list):
return [t for sublist in tlist for t in sublist]
return tlist[0]
+
@retry(exceptions=Exception, tries=5, delay=5)
def translateGPT(text, history, fullPromptFlag):
global PBAR, MISMATCH, FILENAME
-
+
mismatch = False
totalTokens = [0, 0]
if isinstance(text, list):
- format = 'json'
+ format = "json"
tList = batchList(text, BATCHSIZE)
else:
- format = 'text'
+ format = "text"
tList = [text]
for index, tItem in enumerate(tList):
@@ -2315,7 +2672,7 @@ def translateGPT(text, history, fullPromptFlag):
subbedT = varResponse[0]
# Things to Check before starting translation
- if not re.search(r'[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+', subbedT):
+ if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", subbedT):
if PBAR is not None:
PBAR.update(len(tItem))
continue
@@ -2340,9 +2697,13 @@ def translateGPT(text, history, fullPromptFlag):
translatedText = cleanTranslatedText(translatedText, varResponse)
if isinstance(tItem, list):
extractedTranslations = extractTranslation(translatedText, True)
- if extractedTranslations == None or len(tItem) != len(extractedTranslations):
+ if extractedTranslations == None or len(tItem) != len(
+ extractedTranslations
+ ):
# Mismatch. Try Again w/ Different Model
- response = translateText(characters, system, user, history, 0.05, format, 'gpt-4o')
+ response = translateText(
+ characters, system, user, history, 0.05, format, "gpt-4o"
+ )
translatedText = response.choices[0].message.content
totalTokens[0] += response.usage.prompt_tokens
totalTokens[1] += response.usage.completion_tokens
@@ -2351,14 +2712,18 @@ def translateGPT(text, history, fullPromptFlag):
translatedText = cleanTranslatedText(translatedText, varResponse)
if isinstance(tItem, list):
extractedTranslations = extractTranslation(translatedText, True)
- if extractedTranslations == None or len(tItem) != len(extractedTranslations):
- mismatch = True # Just here for breakpoint
-
+ if extractedTranslations == None or len(tItem) != len(
+ extractedTranslations
+ ):
+ mismatch = True # Just here for breakpoint
+
# Set if no mismatch
if mismatch == False:
tList[index] = extractedTranslations
- history = extractedTranslations[-10:] # Update history if we have a list
-
+ history = extractedTranslations[
+ -10:
+ ] # Update history if we have a list
+
else:
history = text[-10:]
mismatch = False
@@ -2371,7 +2736,7 @@ def translateGPT(text, history, fullPromptFlag):
PBAR.update(len(tItem))
else:
# Ensure we're passing a single string to extractTranslation
- tList[index] = translatedText
+ tList[index] = translatedText
finalList = combineList(tList, text)
return [finalList, totalTokens]
diff --git a/modules/rpgmakermvmz.py b/modules/rpgmakermvmz.py
index fafca94..ffc7775 100644
--- a/modules/rpgmakermvmz.py
+++ b/modules/rpgmakermvmz.py
@@ -17,32 +17,32 @@ from tqdm import tqdm
# Open AI
load_dotenv()
-if os.getenv('api').replace(' ', '') != '':
- openai.base_url = os.getenv('api')
-openai.organization = os.getenv('org')
-openai.api_key = os.getenv('key')
+if os.getenv("api").replace(" ", "") != "":
+ openai.base_url = os.getenv("api")
+openai.organization = os.getenv("org")
+openai.api_key = os.getenv("key")
-#Globals
-MODEL = os.getenv('model')
-TIMEOUT = int(os.getenv('timeout'))
-LANGUAGE = os.getenv('language').capitalize()
-PROMPT = Path('prompt.txt').read_text(encoding='utf-8')
-VOCAB = Path('vocab.txt').read_text(encoding='utf-8')
-THREADS = int(os.getenv('threads'))
+# Globals
+MODEL = os.getenv("model")
+TIMEOUT = int(os.getenv("timeout"))
+LANGUAGE = os.getenv("language").capitalize()
+PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
+VOCAB = Path("vocab.txt").read_text(encoding="utf-8")
+THREADS = int(os.getenv("threads"))
LOCK = threading.Lock()
-WIDTH = int(os.getenv('width'))
-LISTWIDTH = int(os.getenv('listWidth'))
-NOTEWIDTH = int(os.getenv('noteWidth'))
+WIDTH = int(os.getenv("width"))
+LISTWIDTH = int(os.getenv("listWidth"))
+NOTEWIDTH = int(os.getenv("noteWidth"))
MAXHISTORY = 10
-ESTIMATE = ''
+ESTIMATE = ""
TOKENS = [0, 0]
NAMESLIST = []
-FIRSTLINESPEAKERS = True # If 1st line of dialogue is a speaker, set to True
-NAMES = False # Output a list of all the character names found
-BRFLAG = False # If the game uses
instead
+FIRSTLINESPEAKERS = True # If 1st line of dialogue is a speaker, set to True
+NAMES = False # Output a list of all the character names found
+BRFLAG = False # If the game uses
instead
FIXTEXTWRAP = True # Overwrites textwrap
-IGNORETLTEXT = False # Ignores all translated text.
-MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
+IGNORETLTEXT = False # Ignores all translated text.
+MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
BRACKETNAMES = False
PBAR = None
FILENAME = None
@@ -50,19 +50,19 @@ FILENAME = None
# Pricing - Depends on the model https://openai.com/pricing
# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request
# If you are getting a MISMATCH LENGTH error, lower the batch size.
-if 'gpt-3.5' in MODEL:
- INPUTAPICOST = .002
- OUTPUTAPICOST = .002
+if "gpt-3.5" in MODEL:
+ INPUTAPICOST = 0.002
+ OUTPUTAPICOST = 0.002
BATCHSIZE = 10
FREQUENCY_PENALTY = 0.2
-elif 'gpt-4' in MODEL:
- INPUTAPICOST = .005
- OUTPUTAPICOST = .015
+elif "gpt-4" in MODEL:
+ INPUTAPICOST = 0.005
+ OUTPUTAPICOST = 0.015
BATCHSIZE = 20
FREQUENCY_PENALTY = 0.1
-#tqdm Globals
-BAR_FORMAT='{l_bar}{bar:10}{r_bar}{bar:-10b}'
+# tqdm Globals
+BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
POSITION = 0
LEAVE = False
@@ -90,6 +90,7 @@ CODE324 = False
CODE111 = False
CODE108 = False
+
def handleMVMZ(filename, estimate):
global ESTIMATE, TOKENS, FILENAME
ESTIMATE = estimate
@@ -98,16 +99,16 @@ def handleMVMZ(filename, estimate):
# Translate
start = time.time()
translatedData = openFiles(filename)
-
+
# Translate
if not estimate:
try:
- with open('translated/' + filename, 'w', encoding='utf-8') as outFile:
+ with open("translated/" + filename, "w", encoding="utf-8") as outFile:
json.dump(translatedData[0], outFile, ensure_ascii=False, indent=4)
except Exception:
traceback.print_exc()
- return 'Fail'
-
+ return "Fail"
+
# Print File
end = time.time()
tqdm.write(getResultString(translatedData, end - start, filename))
@@ -116,92 +117,106 @@ def handleMVMZ(filename, estimate):
TOKENS[1] += translatedData[1][1]
# Print Total
- totalString = getResultString(['', TOKENS, None], end - start, 'TOTAL')
+ totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
# Print any errors on maps
if len(MISMATCH) > 0:
- return totalString + Fore.RED + f'\nMismatch Errors: {MISMATCH}' + Fore.RESET
+ return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
else:
return totalString
+
def openFiles(filename):
- with open('files/' + filename, 'r', encoding='utf-8-sig') as f:
+ with open("files/" + filename, "r", encoding="utf-8-sig") as f:
data = json.load(f)
# Map Files
- if 'Map' in filename and filename != 'MapInfos.json':
+ if "Map" in filename and filename != "MapInfos.json":
translatedData = parseMap(data, filename)
# CommonEvents Files
- elif 'CommonEvents' in filename:
+ elif "CommonEvents" in filename:
translatedData = parseCommonEvents(data, filename)
# Actor File
- elif 'Actors' in filename:
- translatedData = parseNames(data, filename, 'Actors')
+ elif "Actors" in filename:
+ translatedData = parseNames(data, filename, "Actors")
# Armor File
- elif 'Armors' in filename:
- translatedData = parseNames(data, filename, 'Armors')
+ elif "Armors" in filename:
+ translatedData = parseNames(data, filename, "Armors")
# Weapons File
- elif 'Weapons' in filename:
- translatedData = parseNames(data, filename, 'Weapons')
-
+ elif "Weapons" in filename:
+ translatedData = parseNames(data, filename, "Weapons")
+
# Classes File
- elif 'Classes' in filename:
- translatedData = parseNames(data, filename, 'Classes')
+ elif "Classes" in filename:
+ translatedData = parseNames(data, filename, "Classes")
# Enemies File
- elif 'Enemies' in filename:
- translatedData = parseNames(data, filename, 'Enemies')
+ elif "Enemies" in filename:
+ translatedData = parseNames(data, filename, "Enemies")
# Items File
- elif 'Items' in filename:
- translatedData = parseNames(data, filename, 'Items')
+ elif "Items" in filename:
+ translatedData = parseNames(data, filename, "Items")
# MapInfo File
- elif 'MapInfos' in filename:
- translatedData = parseNames(data, filename, 'MapInfos')
+ elif "MapInfos" in filename:
+ translatedData = parseNames(data, filename, "MapInfos")
# Skills File
- elif 'Skills' in filename:
- translatedData = parseNames(data, filename, 'Skills')
+ elif "Skills" in filename:
+ translatedData = parseNames(data, filename, "Skills")
# Troops File
- elif 'Troops' in filename:
+ elif "Troops" in filename:
translatedData = parseTroops(data, filename)
# States File
- elif 'States' in filename:
+ elif "States" in filename:
translatedData = parseSS(data, filename)
# System File
- elif 'System' in filename:
+ elif "System" in filename:
translatedData = parseSystem(data, filename)
# Scenario File
- elif 'Scenario' in filename:
+ elif "Scenario" in filename:
translatedData = parseScenario(data, filename)
else:
- raise NameError(filename + ' Not Supported')
-
+ raise NameError(filename + " Not Supported")
+
return translatedData
+
def getResultString(translatedData, translationTime, filename):
# File Print String
- totalTokenstring =\
- Fore.YELLOW +\
- '[Input: ' + str(translatedData[1][0]) + ']'\
- '[Output: ' + str(translatedData[1][1]) + ']'\
- '[Cost: ${:,.4f}'.format((translatedData[1][0] * .001 * INPUTAPICOST) +\
- (translatedData[1][1] * .001 * OUTPUTAPICOST)) + ']'
- timeString = Fore.BLUE + '[' + str(round(translationTime, 1)) + 's]'
+ totalTokenstring = (
+ Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
+ "[Output: "
+ + str(translatedData[1][1])
+ + "]" "[Cost: ${:,.4f}".format(
+ (translatedData[1][0] * 0.001 * INPUTAPICOST)
+ + (translatedData[1][1] * 0.001 * OUTPUTAPICOST)
+ )
+ + "]"
+ )
+ timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
if translatedData[2] is None:
# Success
- return filename + ': ' + totalTokenstring + timeString + Fore.GREEN + u' \u2713 ' + Fore.RESET
+ return (
+ filename
+ + ": "
+ + totalTokenstring
+ + timeString
+ + Fore.GREEN
+ + " \u2713 "
+ + Fore.RESET
+ )
else:
# Fail
try:
@@ -209,45 +224,64 @@ def getResultString(translatedData, translationTime, filename):
except Exception as e:
traceback.print_exc()
errorString = str(e) + Fore.RED
- return filename + ': ' + totalTokenstring + timeString + Fore.RED + u' \u2717 ' +\
- errorString + Fore.RESET
+ return (
+ filename
+ + ": "
+ + totalTokenstring
+ + timeString
+ + Fore.RED
+ + " \u2717 "
+ + errorString
+ + Fore.RESET
+ )
+
def parseMap(data, filename):
totalTokens = [0, 0]
totalLines = 0
- events = data['events']
+ events = data["events"]
global LOCK
# Translate displayName for Map files
- if 'Map' in filename:
- response = translateGPT(data['displayName'], 'Reply with only the '+ LANGUAGE +' translation of the RPG location name', False)
+ if "Map" in filename:
+ response = translateGPT(
+ data["displayName"],
+ "Reply with only the " + LANGUAGE + " translation of the RPG location name",
+ False,
+ )
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
- data['displayName'] = response[0].replace('\"', '')
+ data["displayName"] = response[0].replace('"', "")
# Get total for progress bar
for event in events:
if event is not None:
- for page in event['pages']:
- totalLines += len(page['list'])
-
+ for page in event["pages"]:
+ totalLines += len(page["list"])
+
# Thread for each page in file
with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
- pbar.desc=filename
+ pbar.desc = filename
with ThreadPoolExecutor(max_workers=THREADS) as executor:
for event in events:
if event is not None:
# This translates ID of events. (May break the game)
- if '.*')
+ if ".*"
+ )
totalTokens[0] += response[0]
totalTokens[1] += response[1]
- if '.*')
+ if ".*")
totalTokens[0] += response[0]
totalTokens[1] += response[1]
- futures = [executor.submit(searchCodes, page, pbar, [], filename) for page in event['pages'] if page is not None]
+ futures = [
+ executor.submit(searchCodes, page, pbar, [], filename)
+ for page in event["pages"]
+ if page is not None
+ ]
for future in as_completed(futures):
try:
totalTokensFuture = future.result()
@@ -258,53 +292,64 @@ def parseMap(data, filename):
return [data, totalTokens, e]
return [data, totalTokens, None]
+
def translateNote(event, regex):
# Regex String
- jaString = event['note']
+ jaString = event["note"]
match = re.findall(regex, jaString, re.DOTALL)
if match:
- tokens = [0,0]
+ tokens = [0, 0]
i = 0
while i < len(match):
initialJAString = match[i]
# Remove any textwrap
- modifiedJAString = initialJAString.replace('\n', ' ')
+ modifiedJAString = initialJAString.replace("\n", " ")
# Translate
- response = translateGPT(modifiedJAString, 'Reply with only the '+ LANGUAGE +' translation.', False)
+ response = translateGPT(
+ modifiedJAString,
+ "Reply with only the " + LANGUAGE + " translation.",
+ False,
+ )
translatedText = response[0]
tokens[0] += response[1][0]
tokens[1] += response[1][1]
# Textwrap
translatedText = textwrap.fill(translatedText, width=NOTEWIDTH)
- translatedText = translatedText.replace('\"', '')
+ translatedText = translatedText.replace('"', "")
jaString = jaString.replace(initialJAString, translatedText)
- event['note'] = jaString
+ event["note"] = jaString
i += 1
return tokens
- return [0,0]
+ return [0, 0]
+
# For notes that can't have spaces.
def translateNoteOmitSpace(event, regex):
# Regex that only matches text inside LB.
- jaString = event['note']
+ jaString = event["note"]
match = re.findall(regex, jaString, re.DOTALL)
if match:
oldJAString = match[0]
# Remove any textwrap
- jaString = re.sub(r'\n', ' ', oldJAString)
+ jaString = re.sub(r"\n", " ", oldJAString)
# Translate
- response = translateGPT(jaString, 'Reply with the '+ LANGUAGE +' translation of the location name.', False)
+ response = translateGPT(
+ jaString,
+ "Reply with the " + LANGUAGE + " translation of the location name.",
+ False,
+ )
translatedText = response[0]
- translatedText = translatedText.replace('\"', '')
- translatedText = translatedText.replace(' ', '_')
- event['note'] = event['note'].replace(oldJAString, translatedText)
+ translatedText = translatedText.replace('"', "")
+ translatedText = translatedText.replace(" ", "_")
+ event["note"] = event["note"].replace(oldJAString, translatedText)
return response[1]
- return [0,0]
+ return [0, 0]
+
def parseCommonEvents(data, filename):
totalTokens = [0, 0]
@@ -314,12 +359,16 @@ def parseCommonEvents(data, filename):
# Get total for progress bar
for page in data:
if page is not None:
- totalLines += len(page['list'])
+ totalLines += len(page["list"])
with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
- pbar.desc=filename
+ pbar.desc = filename
with ThreadPoolExecutor(max_workers=THREADS) as executor:
- futures = [executor.submit(searchCodes, page, pbar, [], filename) for page in data if page is not None]
+ futures = [
+ executor.submit(searchCodes, page, pbar, [], filename)
+ for page in data
+ if page is not None
+ ]
for future in as_completed(futures):
try:
totalTokensFuture = future.result()
@@ -330,6 +379,7 @@ def parseCommonEvents(data, filename):
return [data, totalTokens, e]
return [data, totalTokens, None]
+
def parseTroops(data, filename):
totalTokens = [0, 0]
totalLines = 0
@@ -338,15 +388,21 @@ def parseTroops(data, filename):
# Get total for progress bar
for troop in data:
if troop is not None:
- for page in troop['pages']:
- totalLines += len(page['list']) + 1 # The +1 is because each page has a name.
+ for page in troop["pages"]:
+ totalLines += (
+ len(page["list"]) + 1
+ ) # The +1 is because each page has a name.
with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
- pbar.desc=filename
+ pbar.desc = filename
for troop in data:
if troop is not None:
with ThreadPoolExecutor(max_workers=THREADS) as executor:
- futures = [executor.submit(searchCodes, page, pbar, [], filename) for page in troop['pages'] if page is not None]
+ futures = [
+ executor.submit(searchCodes, page, pbar, [], filename)
+ for page in troop["pages"]
+ if page is not None
+ ]
for future in as_completed(futures):
try:
totalTokensFuture = future.result()
@@ -356,60 +412,17 @@ def parseTroops(data, filename):
traceback.print_exc()
return [data, totalTokens, e]
return [data, totalTokens, None]
-
+
+
def parseNames(data, filename, context):
totalTokens = [0, 0]
totalLines = 0
totalLines += len(data)
-
- with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
- pbar.desc=filename
- try:
- result = searchNames(data, pbar, context)
- totalTokens[0] += result[0]
- totalTokens[1] += result[1]
- except Exception as e:
- traceback.print_exc()
- return [data, totalTokens, e]
- return [data, totalTokens, None]
-def parseSS(data, filename):
- totalTokens = [0, 0]
- totalLines = 0
- totalLines += len(data)
-
with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
- pbar.desc=filename
- for ss in data:
- if ss is not None:
- try:
- result = searchSS(ss, pbar)
- totalTokens[0] += result[0]
- totalTokens[1] += result[1]
- except Exception as e:
- traceback.print_exc()
- return [data, totalTokens, e]
- return [data, totalTokens, None]
-
-def parseSystem(data, filename):
- totalTokens = [0, 0]
- totalLines = 0
-
- # Calculate Total Lines
- for term in data['terms']:
- termList = data['terms'][term]
- totalLines += len(termList)
- totalLines += len(data['gameTitle'])
- totalLines += len(data['terms']['messages'])
- totalLines += len(data['variables'])
- totalLines += len(data['equipTypes'])
- totalLines += len(data['armorTypes'])
- totalLines += len(data['skillTypes'])
-
- with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
- pbar.desc=filename
+ pbar.desc = filename
try:
- result = searchSystem(data, pbar)
+ result = searchNames(data, pbar, context)
totalTokens[0] += result[0]
totalTokens[1] += result[1]
except Exception as e:
@@ -417,6 +430,53 @@ def parseSystem(data, filename):
return [data, totalTokens, e]
return [data, totalTokens, None]
+
+def parseSS(data, filename):
+ totalTokens = [0, 0]
+ totalLines = 0
+ totalLines += len(data)
+
+ with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
+ pbar.desc = filename
+ for ss in data:
+ if ss is not None:
+ try:
+ result = searchSS(ss, pbar)
+ totalTokens[0] += result[0]
+ totalTokens[1] += result[1]
+ except Exception as e:
+ traceback.print_exc()
+ return [data, totalTokens, e]
+ return [data, totalTokens, None]
+
+
+def parseSystem(data, filename):
+ totalTokens = [0, 0]
+ totalLines = 0
+
+ # Calculate Total Lines
+ for term in data["terms"]:
+ termList = data["terms"][term]
+ totalLines += len(termList)
+ totalLines += len(data["gameTitle"])
+ totalLines += len(data["terms"]["messages"])
+ totalLines += len(data["variables"])
+ totalLines += len(data["equipTypes"])
+ totalLines += len(data["armorTypes"])
+ totalLines += len(data["skillTypes"])
+
+ with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
+ pbar.desc = filename
+ try:
+ result = searchSystem(data, pbar)
+ totalTokens[0] += result[0]
+ totalTokens[1] += result[1]
+ except Exception as e:
+ traceback.print_exc()
+ return [data, totalTokens, e]
+ return [data, totalTokens, None]
+
+
def parseScenario(data, filename):
totalTokens = [0, 0]
totalLines = 0
@@ -427,9 +487,13 @@ def parseScenario(data, filename):
totalLines += len(page[1])
with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
- pbar.desc=filename
+ pbar.desc = filename
with ThreadPoolExecutor(max_workers=THREADS) as executor:
- futures = [executor.submit(searchCodes, page[1], pbar, [], filename) for page in data.items() if page[1] is not None]
+ futures = [
+ executor.submit(searchCodes, page[1], pbar, [], filename)
+ for page in data.items()
+ if page[1] is not None
+ ]
for future in as_completed(futures):
try:
totalTokensFuture = future.result()
@@ -440,6 +504,7 @@ def parseScenario(data, filename):
return [data, totalTokens, e]
return [data, totalTokens, None]
+
def searchNames(data, pbar, context):
totalTokens = [0, 0]
nameList = []
@@ -447,142 +512,178 @@ def searchNames(data, pbar, context):
nicknameList = []
descriptionList = []
noteList = []
- i = 0 # Counter
- j = 0 # Counter 2
+ i = 0 # Counter
+ j = 0 # Counter 2
filling = False
mismatch = False
batchFull = False
# Set the context of what we are translating
- if 'Actors' in context:
- newContext = 'Reply with only the '+ LANGUAGE +' translation of the NPC name'
- if 'Armors' in context:
- newContext = 'Reply with only the '+ LANGUAGE +' translation of the RPG equipment name'
- if 'Classes' in context:
- newContext = 'Reply with only the '+ LANGUAGE +' translation of the RPG class name'
- if 'MapInfos' in context:
- newContext = 'Reply with only the '+ LANGUAGE +' translation of the location name'
- if 'Enemies' in context:
- newContext = 'Reply with only the '+ LANGUAGE +' translation of the enemy NPC name'
- if 'Weapons' in context:
- newContext = 'Reply with only the '+ LANGUAGE +' translation of the RPG weapon name'
- if 'Items' in context:
- newContext = 'Reply with only the '+ LANGUAGE +' translation of the RPG item name'
- if 'Skills' in context:
- newContext = 'Reply with only the '+ LANGUAGE +' translation of the RPG skill name'
+ if "Actors" in context:
+ newContext = "Reply with only the " + LANGUAGE + " translation of the NPC name"
+ if "Armors" in context:
+ newContext = (
+ "Reply with only the " + LANGUAGE + " translation of the RPG equipment name"
+ )
+ if "Classes" in context:
+ newContext = (
+ "Reply with only the " + LANGUAGE + " translation of the RPG class name"
+ )
+ if "MapInfos" in context:
+ newContext = (
+ "Reply with only the " + LANGUAGE + " translation of the location name"
+ )
+ if "Enemies" in context:
+ newContext = (
+ "Reply with only the " + LANGUAGE + " translation of the enemy NPC name"
+ )
+ if "Weapons" in context:
+ newContext = (
+ "Reply with only the " + LANGUAGE + " translation of the RPG weapon name"
+ )
+ if "Items" in context:
+ newContext = (
+ "Reply with only the " + LANGUAGE + " translation of the RPG item name"
+ )
+ if "Skills" in context:
+ newContext = (
+ "Reply with only the " + LANGUAGE + " translation of the RPG skill name"
+ )
# Names
while i < len(data) or filling == True:
if i < len(data):
# Empty Data
- if data[i] is None or data[i]['name'] == "":
+ if data[i] is None or data[i]["name"] == "":
i += 1
-
- continue
+
+ continue
# Filling up Batch
filling = True
- if context in 'Actors':
+ if context in "Actors":
if len(nameList) < BATCHSIZE:
- if data[i]['name'] != '':
- nameList.append(data[i]['name'])
- if data[i]['nickname'] != '':
- nicknameList.append(data[i]['nickname'])
- if data[i]['profile'] != '':
- profileList.append(data[i]['profile'].replace('\n', ' '))
+ if data[i]["name"] != "":
+ nameList.append(data[i]["name"])
+ if data[i]["nickname"] != "":
+ nicknameList.append(data[i]["nickname"])
+ if data[i]["profile"] != "":
+ profileList.append(data[i]["profile"].replace("\n", " "))
# Notes
- if '')
+ if "")
totalTokens[0] += tokensResponse[0]
totalTokens[1] += tokensResponse[1]
- if 'PE拡張' in data[i]['note']:
- tokensResponse = translateNote(data[i], r'')
+ if "PE拡張" in data[i]["note"]:
+ tokensResponse = translateNote(data[i], r"")
totalTokens[0] += tokensResponse[0]
totalTokens[1] += tokensResponse[1]
i += 1
else:
batchFull = True
- if context in ['Armors', 'Weapons', 'Items']:
+ if context in ["Armors", "Weapons", "Items"]:
if len(nameList) < BATCHSIZE:
- nameList.append(data[i]['name'])
- if 'description' in data[i]:
- descriptionList.append(data[i]['description'].replace('\n', ' '))
- if '')
+ nameList.append(data[i]["name"])
+ if "description" in data[i]:
+ descriptionList.append(
+ data[i]["description"].replace("\n", " ")
+ )
+ if "")
totalTokens[0] += tokensResponse[0]
totalTokens[1] += tokensResponse[1]
- if '')
+ if ""
+ )
totalTokens[0] += tokensResponse[0]
totalTokens[1] += tokensResponse[1]
- if '')
+ if "")
totalTokens[0] += tokensResponse[0]
totalTokens[1] += tokensResponse[1]
- if '')
+ if "")
totalTokens[0] += tokensResponse[0]
totalTokens[1] += tokensResponse[1]
- if '')
+ if "")
totalTokens[0] += tokensResponse[0]
totalTokens[1] += tokensResponse[1]
- if '')
+ if "")
totalTokens[0] += tokensResponse[0]
totalTokens[1] += tokensResponse[1]
- if '')
+ if "")
totalTokens[0] += tokensResponse[0]
totalTokens[1] += tokensResponse[1]
- if 'Switch Shop Description' in data[i]['note']:
- tokensResponse = translateNote(data[i], r'\n(.*)\n')
+ if "Switch Shop Description" in data[i]["note"]:
+ tokensResponse = translateNote(
+ data[i], r"\n(.*)\n"
+ )
totalTokens[0] += tokensResponse[0]
totalTokens[1] += tokensResponse[1]
- if '')
+ if "")
totalTokens[0] += tokensResponse[0]
totalTokens[1] += tokensResponse[1]
-
+
i += 1
else:
batchFull = True
- if context in ['Skills']:
+ if context in ["Skills"]:
if len(nameList) < BATCHSIZE:
- nameList.append(data[i]['name'])
- descriptionList.append(data[i]['description'].replace('\n', ' '))
-
+ nameList.append(data[i]["name"])
+ descriptionList.append(data[i]["description"].replace("\n", " "))
+
# Messages
number = 1
while number < 5:
- if f'message{number}' in data[i]:
- if len(data[i][f'message{number}']) > 0 and data[i][f'message{number}'][0] in ['は', 'を', 'の', 'に', 'が']:
- msgResponse = translateGPT('Taro' + data[i][f'message{number}'], 'reply with only the gender neutral '+ LANGUAGE +' translation of the action log. Always start the sentence with Taro. For example, Translate \'Taroを倒した！\' as \'Taro was defeated!\'', False)
- data[i][f'message{number}'] = msgResponse[0].replace('Taro', '')
+ if f"message{number}" in data[i]:
+ if len(data[i][f"message{number}"]) > 0 and data[i][
+ f"message{number}"
+ ][0] in ["は", "を", "の", "に", "が"]:
+ msgResponse = translateGPT(
+ "Taro" + data[i][f"message{number}"],
+ "reply with only the gender neutral "
+ + LANGUAGE
+ + " translation of the action log. Always start the sentence with Taro. For example, Translate 'Taroを倒した！' as 'Taro was defeated!'",
+ False,
+ )
+ data[i][f"message{number}"] = msgResponse[0].replace(
+ "Taro", ""
+ )
totalTokens[0] += msgResponse[1][0]
totalTokens[1] += msgResponse[1][1]
number += 1
else:
- msgResponse = translateGPT(data[i][f'message{number}'], 'reply with only the gender neutral '+ LANGUAGE +' translation', False)
- data[i][f'message{number}'] = msgResponse[0]
+ msgResponse = translateGPT(
+ data[i][f"message{number}"],
+ "reply with only the gender neutral "
+ + LANGUAGE
+ + " translation",
+ False,
+ )
+ data[i][f"message{number}"] = msgResponse[0]
totalTokens[0] += msgResponse[1][0]
totalTokens[1] += msgResponse[1][1]
number += 1
else:
number += 1
-
+
i += 1
else:
batchFull = True
- if context in ['Enemies', 'Classes', 'MapInfos']:
+ if context in ["Enemies", "Classes", "MapInfos"]:
if len(nameList) < BATCHSIZE:
- nameList.append(data[i]['name'])
+ nameList.append(data[i]["name"])
# Notes
- if '')
+ if "")
totalTokens[0] += tokensResponse[0]
totalTokens[1] += tokensResponse[1]
i += 1
@@ -591,8 +692,8 @@ def searchNames(data, pbar, context):
# Batch Full
if batchFull == True or i >= len(data):
- k = j # Original Index
- if context in 'Actors':
+ k = j # Original Index
+ if context in "Actors":
# Name
response = translateGPT(nameList, newContext, True)
translatedNameBatch = response[0]
@@ -606,7 +707,7 @@ def searchNames(data, pbar, context):
totalTokens[1] += response[1][1]
# Profile
- response = translateGPT(profileList, '', True)
+ response = translateGPT(profileList, "", True)
translatedProfileBatch = response[0]
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
@@ -616,21 +717,27 @@ def searchNames(data, pbar, context):
j = k
while j < i:
# Empty Data
- if data[j] is None or data[j]['name'] == "":
+ if data[j] is None or data[j]["name"] == "":
j += 1
- continue
+ continue
else:
# Get Text
- if data[j]['name'] != '':
- with open('translations.txt', 'a', encoding='utf-8') as file:
- file.write(f'{data[j]['name']} ({translatedNameBatch[0]})\n')
- data[j]['name'] = translatedNameBatch[0]
+ if data[j]["name"] != "":
+ with open(
+ "translations.txt", "a", encoding="utf-8"
+ ) as file:
+ file.write(
+ f'{data[j]['name']} ({translatedNameBatch[0]})\n'
+ )
+ data[j]["name"] = translatedNameBatch[0]
translatedNameBatch.pop(0)
- if data[j]['nickname'] != '':
- data[j]['nickname'] = translatedNicknameBatch[0]
+ if data[j]["nickname"] != "":
+ data[j]["nickname"] = translatedNicknameBatch[0]
translatedNicknameBatch.pop(0)
- if data[j]['profile'] != '':
- data[j]['profile'] = textwrap.fill(translatedProfileBatch[0], LISTWIDTH)
+ if data[j]["profile"] != "":
+ data[j]["profile"] = textwrap.fill(
+ translatedProfileBatch[0], LISTWIDTH
+ )
translatedProfileBatch.pop(0)
# If Batch is empty. Move on.
@@ -641,7 +748,7 @@ def searchNames(data, pbar, context):
else:
mismatch = True
- if context in ['Armors', 'Weapons', 'Items', 'Skills']:
+ if context in ["Armors", "Weapons", "Items", "Skills"]:
# Name
response = translateGPT(nameList, newContext, True)
translatedNameBatch = response[0]
@@ -649,7 +756,11 @@ def searchNames(data, pbar, context):
totalTokens[1] += response[1][1]
# Description
- response = translateGPT(descriptionList, f'Reply with only the {LANGUAGE} translation of the text.', True)
+ response = translateGPT(
+ descriptionList,
+ f"Reply with only the {LANGUAGE} translation of the text.",
+ True,
+ )
translatedDescriptionBatch = response[0]
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
@@ -657,19 +768,23 @@ def searchNames(data, pbar, context):
# Set Data
if len(nameList) == len(translatedNameBatch):
j = k
- with open('translations.txt', 'a', encoding='utf-8') as file:
- file.write('# Items\n')
+ with open("translations.txt", "a", encoding="utf-8") as file:
+ file.write("# Items\n")
while j < i:
# Empty Data
- if data[j] is None or data[j]['name'] == "":
+ if data[j] is None or data[j]["name"] == "":
j += 1
- continue
+ continue
else:
# Get Text
- file.write(f'{data[j]['name']} ({translatedNameBatch[0]})\n')
- data[j]['name'] = translatedNameBatch[0]
- if 'description' in data[j]:
- data[j]['description'] = textwrap.fill(translatedDescriptionBatch[0], LISTWIDTH)
+ file.write(
+ f'{data[j]['name']} ({translatedNameBatch[0]})\n'
+ )
+ data[j]["name"] = translatedNameBatch[0]
+ if "description" in data[j]:
+ data[j]["description"] = textwrap.fill(
+ translatedDescriptionBatch[0], LISTWIDTH
+ )
translatedNameBatch.pop(0)
translatedDescriptionBatch.pop(0)
@@ -682,7 +797,7 @@ def searchNames(data, pbar, context):
j += 1
else:
mismatch = True
- if context in ['Enemies', 'Classes', 'MapInfos']:
+ if context in ["Enemies", "Classes", "MapInfos"]:
response = translateGPT(nameList, newContext, True)
translatedNameBatch = response[0]
totalTokens[0] += response[1][0]
@@ -693,12 +808,12 @@ def searchNames(data, pbar, context):
j = k
while j < i:
# Empty Data
- if data[j] is None or data[j]['name'] == "":
+ if data[j] is None or data[j]["name"] == "":
j += 1
- continue
+ continue
else:
# Get Text
- data[j]['name'] = translatedNameBatch[0]
+ data[j]["name"] = translatedNameBatch[0]
translatedNameBatch.pop(0)
# If Batch is empty. Move on.
@@ -718,11 +833,12 @@ def searchNames(data, pbar, context):
descriptionList.clear()
filling = False
mismatch = False
-
+
i += 1
return totalTokens
+
def searchCodes(page, pbar, jobList, filename):
if len(jobList) > 0:
list401 = jobList[0]
@@ -739,8 +855,8 @@ def searchCodes(page, pbar, jobList, filename):
textHistory = []
match = []
totalTokens = [0, 0]
- translatedText = ''
- speaker = ''
+ translatedText = ""
+ speaker = ""
speakerID = None
syncIndex = 0
CLFlag = False
@@ -754,12 +870,11 @@ def searchCodes(page, pbar, jobList, filename):
with LOCK:
PBAR = pbar
-
# Begin Parsing File
try:
# Normal Format
- if 'list' in page:
- codeList = page['list']
+ if "list" in page:
+ codeList = page["list"]
# Special Format (Scenario)
else:
@@ -768,7 +883,7 @@ def searchCodes(page, pbar, jobList, filename):
# Iterate through page
i = 0
while i < len(codeList):
- with LOCK:
+ with LOCK:
# syncIndex will keep i in sync when it gets modified
if syncIndex > i:
i = syncIndex
@@ -777,21 +892,25 @@ def searchCodes(page, pbar, jobList, filename):
# Declare Varss
currentGroup = []
- nametag = ''
+ nametag = ""
## Event Code: 401 Show Text
- if 'code' in codeList[i] and codeList[i]['code'] in [401, 405, -1] and (CODE401 or CODE405):
+ if (
+ "code" in codeList[i]
+ and codeList[i]["code"] in [401, 405, -1]
+ and (CODE401 or CODE405)
+ ):
# Save Code and starting index (j)
- code = codeList[i]['code']
+ code = codeList[i]["code"]
j = i
- endtag = ''
+ endtag = ""
# Grab String
- if len(codeList[i]['parameters']) > 0:
- jaString = codeList[i]['parameters'][0]
+ if len(codeList[i]["parameters"]) > 0:
+ jaString = codeList[i]["parameters"][0]
oldjaString = jaString
else:
- codeList[i]['code'] = -1
+ codeList[i]["code"] = -1
i += 1
continue
@@ -814,95 +933,114 @@ def searchCodes(page, pbar, jobList, filename):
speakerList = []
# m and z Codes
- match = re.search(r'(.*?)[\\]+m\[\d+?\][\\]+z\[\d+?\]', jaString)
+ match = re.search(r"(.*?)[\\]+m\[\d+?\][\\]+z\[\d+?\]", jaString)
if match:
speakerList.append(match.group(1))
- if '\\c' in speakerList[0]:
- speakerList = re.findall(r'^[\\]+[cC]\[[\d]+\](.+?)[\\]+[Cc]\[[\d]\]\\?\\?$', speakerList[0])
+ if "\\c" in speakerList[0]:
+ speakerList = re.findall(
+ r"^[\\]+[cC]\[[\d]+\](.+?)[\\]+[Cc]\[[\d]\]\\?\\?$",
+ speakerList[0],
+ )
# Brackets
if len(speakerList) == 0:
- speakerList = re.findall(r'^【(.*?)】$', jaString)
+ speakerList = re.findall(r"^【(.*?)】$", jaString)
# Colors
if len(speakerList) == 0:
- speakerList = re.findall(r'^[\\]+[cC]\[[\d]+\](.+?)[\\]+[Cc]\[[\d]\]\\?\\?$', jaString)
+ speakerList = re.findall(
+ r"^[\\]+[cC]\[[\d]+\](.+?)[\\]+[Cc]\[[\d]\]\\?\\?$", jaString
+ )
# First Line Speakers
if len(speakerList) == 0 and FIRSTLINESPEAKERS is True:
# Remove any RPGMaker Code at start
- ffMatch = re.search(r'^(\s*[\\]+[aAbBdDeEfFgGhHiIjJlLmMoOpPqQrRsStTuUvVwWxXyYzZ]+\[[\w\d\[\]\\]+\])', jaString)
+ ffMatch = re.search(
+ r"^(\s*[\\]+[aAbBdDeEfFgGhHiIjJlLmMoOpPqQrRsStTuUvVwWxXyYzZ]+\[[\w\d\[\]\\]+\])",
+ jaString,
+ )
if ffMatch != None:
- jaString = jaString.replace(ffMatch.group(0), '')
+ jaString = jaString.replace(ffMatch.group(0), "")
nametag += ffMatch.group(0)
-
- # Test Speaker
- if len(jaString) < 40 \
- and 'code' in codeList[i+1] \
- and codeList[i+1]['code'] in [401, 405, -1] \
- and len(codeList[i+1]['parameters']) > 0 \
- and len(codeList[i+1]['parameters'][0]) > 0:
- if codeList[i+1]['parameters'][0].strip()[0] in ['「', '"', '(', '（', '*', '[']:
- speakerList = re.findall(r'.+', jaString)
- if len(speakerList) != 0 and codeList[i+1]['code'] in [401, 405, -1]:
+ # Test Speaker
+ if (
+ len(jaString) < 40
+ and "code" in codeList[i + 1]
+ and codeList[i + 1]["code"] in [401, 405, -1]
+ and len(codeList[i + 1]["parameters"]) > 0
+ and len(codeList[i + 1]["parameters"][0]) > 0
+ ):
+ if codeList[i + 1]["parameters"][0].strip()[0] in [
+ "「",
+ '"',
+ "(",
+ "（",
+ "*",
+ "[",
+ ]:
+ speakerList = re.findall(r".+", jaString)
+
+ if len(speakerList) != 0 and codeList[i + 1]["code"] in [401, 405, -1]:
# Get Speaker
response = getSpeaker(speakerList[0])
speaker = response[0]
totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
+ totalTokens[1] += response[1][1]
# Set Data
- codeList[i]['parameters'][0] = nametag + jaString.replace(speakerList[0], speaker)
- nametag = ''
+ codeList[i]["parameters"][0] = nametag + jaString.replace(
+ speakerList[0], speaker
+ )
+ nametag = ""
# Iterate to next string
i += 1
j = i
- while codeList[i]['code'] in [-1]:
+ while codeList[i]["code"] in [-1]:
i += 1
j = i
- jaString = codeList[i]['parameters'][0]
+ jaString = codeList[i]["parameters"][0]
# Using this to keep track of 401's in a row.
currentGroup.append(jaString)
# Join Up 401's into single string
- if len(codeList) > i+1:
- while codeList[i+1]['code'] in [401, 405, -1]:
+ if len(codeList) > i + 1:
+ while codeList[i + 1]["code"] in [401, 405, -1]:
if setData == True:
- codeList[i]['parameters'] = []
- codeList[i]['code'] = -1
+ codeList[i]["parameters"] = []
+ codeList[i]["code"] = -1
i += 1
j = i
# Only add if not empty
- if len(codeList[i]['parameters']) > 0:
- jaString = codeList[i]['parameters'][0]
+ if len(codeList[i]["parameters"]) > 0:
+ jaString = codeList[i]["parameters"][0]
currentGroup.append(jaString)
# Make sure not the end of the list.
- if len(codeList) <= i+1:
+ if len(codeList) <= i + 1:
break
# Format String
if len(currentGroup) > 0:
- finalJAString = ''
- finalJAString = ''.join(currentGroup).replace('？', '?')
+ finalJAString = ""
+ finalJAString = "".join(currentGroup).replace("？", "?")
oldjaString = finalJAString
# Check if Empty
- if finalJAString == '':
+ if finalJAString == "":
i += 1
continue
# Set Back
if setData == True:
- codeList[i]['parameters'] = [finalJAString]
+ codeList[i]["parameters"] = [finalJAString]
### \\n
nCase = None
- regex = r'([\\]+[kKnN][wWcCrRrEe]?[\[<](.*?)[>\]])'
+ regex = r"([\\]+[kKnN][wWcCrRrEe]?[\[<](.*?)[>\]])"
match = re.search(regex, finalJAString)
# Set Name
@@ -910,20 +1048,20 @@ def searchCodes(page, pbar, jobList, filename):
nametag = match.group(1)
speaker = match.group(2)
- # Translate Speaker
+ # Translate Speaker
response = getSpeaker(speaker)
tledSpeaker = response[0]
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
# Set Nametag and Remove from Final String
- finalJAString = finalJAString.replace(nametag, '')
+ finalJAString = finalJAString.replace(nametag, "")
nametag = nametag.replace(speaker, tledSpeaker)
speaker = tledSpeaker
-
+
# Bracket Names
if BRACKETNAMES is True and len(matchList) != 0:
- if matchList[0][0] != '':
+ if matchList[0][0] != "":
match0 = matchList[0][0]
match1 = matchList[0][1]
else:
@@ -939,128 +1077,148 @@ def searchCodes(page, pbar, jobList, filename):
# Set Nametag and Remove from Final String
fullSpeaker = match0.replace(match1, speaker)
- finalJAString = finalJAString.replace(match0, '')
+ finalJAString = finalJAString.replace(match0, "")
# Set next item as dialogue
- if codeList[j + 1]['code'] == 401 or codeList[j + 1]['code'] == -1:
+ if (
+ codeList[j + 1]["code"] == 401
+ or codeList[j + 1]["code"] == -1
+ ):
# Set name var to top of list
- codeList[j]['parameters'] = [fullSpeaker]
- codeList[j]['code'] = code
+ codeList[j]["parameters"] = [fullSpeaker]
+ codeList[j]["code"] = code
j += 1
- codeList[j]['parameters'] = [finalJAString]
- codeList[j]['code'] = code
+ codeList[j]["parameters"] = [finalJAString]
+ codeList[j]["code"] = code
else:
# Set nametag in string
- codeList[j]['parameters'] = [fullSpeaker + finalJAString]
- codeList[j]['code'] = code
+ codeList[j]["parameters"] = [fullSpeaker + finalJAString]
+ codeList[j]["code"] = code
# Remove any textwrap
if FIXTEXTWRAP is True:
- finalJAString = re.sub(r'\n', ' ', finalJAString)
- finalJAString = finalJAString.replace('
', ' ')
+ finalJAString = re.sub(r"\n", " ", finalJAString)
+ finalJAString = finalJAString.replace("
", " ")
# Remove Extra Stuff bad for translation.
- finalJAString = finalJAString.replace('ﾞ', '')
- finalJAString = finalJAString.replace('―', '-')
- finalJAString = finalJAString.replace('…', '...')
- finalJAString = finalJAString.replace('。', '.')
- finalJAString = re.sub(r'(\.{3}\.+)', '...', finalJAString)
- finalJAString = finalJAString.replace(' ', '')
- finalJAString = finalJAString.replace('「', '\"')
- finalJAString = finalJAString.replace('」', '\"')
+ finalJAString = finalJAString.replace("ﾞ", "")
+ finalJAString = finalJAString.replace("―", "-")
+ finalJAString = finalJAString.replace("…", "...")
+ finalJAString = finalJAString.replace("。", ".")
+ finalJAString = re.sub(r"(\.{3}\.+)", "...", finalJAString)
+ finalJAString = finalJAString.replace(" ", "")
+ finalJAString = finalJAString.replace("「", '"')
+ finalJAString = finalJAString.replace("」", '"')
### Remove format codes
# Furigana
- rcodeMatch = re.findall(r'([\\]+[r][b]?\[.*?,(.*?)\])', finalJAString)
+ rcodeMatch = re.findall(
+ r"([\\]+[r][b]?\[.*?,(.*?)\])", finalJAString
+ )
if len(rcodeMatch) > 0:
for match in rcodeMatch:
- finalJAString = finalJAString.replace(match[0],match[1])
+ finalJAString = finalJAString.replace(match[0], match[1])
# Formatting
- formatMatch = re.findall(r'[\\]+[!><.|#^{}]', finalJAString)
+ formatMatch = re.findall(r"[\\]+[!><.|#^{}]", finalJAString)
if len(formatMatch) > 0:
for match in formatMatch:
- finalJAString = finalJAString.replace(match, '')
+ finalJAString = finalJAString.replace(match, "")
# Remove any RPGMaker Code at start
- ffMatch = re.search(r'^(\s*[\\]+[aAbBdDeEfFgGhHiIjJlLmMoOpPqQrRsStTuUvVwWxXyYzZ]+\[[\w\d\[\]\\]+\])', finalJAString)
+ ffMatch = re.search(
+ r"^(\s*[\\]+[aAbBdDeEfFgGhHiIjJlLmMoOpPqQrRsStTuUvVwWxXyYzZ]+\[[\w\d\[\]\\]+\])",
+ finalJAString,
+ )
if ffMatch != None:
- finalJAString = finalJAString.replace(ffMatch.group(0), '')
+ finalJAString = finalJAString.replace(ffMatch.group(0), "")
nametag += ffMatch.group(0)
# Remove _ABL Codes
- ffMatch = re.search(r'^(_ABL).*', finalJAString)
+ ffMatch = re.search(r"^(_ABL).*", finalJAString)
if ffMatch != None:
- finalJAString = finalJAString.replace(ffMatch.group(1), '')
+ finalJAString = finalJAString.replace(ffMatch.group(1), "")
nametag += ffMatch.group(1)
# Center Lines
- if '\\CL' in finalJAString or '\\ac' in finalJAString:
- finalJAString = finalJAString.replace('\\CL ', '')
- finalJAString = finalJAString.replace('\\CL', '')
- finalJAString = finalJAString.replace('\\ac ', '')
- finalJAString = finalJAString.replace('\\ac', '')
+ if "\\CL" in finalJAString or "\\ac" in finalJAString:
+ finalJAString = finalJAString.replace("\\CL ", "")
+ finalJAString = finalJAString.replace("\\CL", "")
+ finalJAString = finalJAString.replace("\\ac ", "")
+ finalJAString = finalJAString.replace("\\ac", "")
CLFlag = True
# If there isn't any Japanese in the text just skip
if IGNORETLTEXT is True:
- if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+', finalJAString):
+ if not re.search(
+ r"[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+", finalJAString
+ ):
# Keep textHistory list at length maxHistory
- textHistory.append('\"' + finalJAString + '\"')
+ textHistory.append('"' + finalJAString + '"')
if len(textHistory) > maxHistory:
textHistory.pop(0)
- currentGroup = []
+ currentGroup = []
i += 1
continue
# 1st Passthrough (Grabbing Data)
if setData == False:
- if finalJAString != '':
- if speaker == '' and finalJAString != '':
+ if finalJAString != "":
+ if speaker == "" and finalJAString != "":
list401.append(finalJAString)
- elif finalJAString != '':
- list401.append(f'[{speaker}]: {finalJAString}')
+ elif finalJAString != "":
+ list401.append(f"[{speaker}]: {finalJAString}")
else:
list401.append(speaker)
- speaker = ''
+ speaker = ""
match = []
- nametag = ''
+ nametag = ""
currentGroup = []
- syncIndex = i + 1
+ syncIndex = i + 1
- # 2nd Passthrough (Setting Data)
+ # 2nd Passthrough (Setting Data)
else:
# Grab Translated String
if len(list401) > 0:
translatedText = list401[0]
-
+
# Remove speaker
- if speaker != '':
- matchSpeakerList = re.findall(r'^\[?(.+?)\]?\s?[|:]\s?', translatedText)
+ if speaker != "":
+ matchSpeakerList = re.findall(
+ r"^\[?(.+?)\]?\s?[|:]\s?", translatedText
+ )
if len(matchSpeakerList) > 0:
newSpeaker = matchSpeakerList[0]
nametag = nametag.replace(speaker, newSpeaker)
- translatedText = re.sub(r'^\[?(.+?)\]?\s?[|:]\s?', '', translatedText)
+ translatedText = re.sub(
+ r"^\[?(.+?)\]?\s?[|:]\s?", "", translatedText
+ )
# Fix '- '
- translatedText = translatedText.replace('- ', '-')
+ translatedText = translatedText.replace("- ", "-")
# Textwrap
- if FIXTEXTWRAP is True and '_ABL' in nametag:
- translatedText = textwrap.fill(translatedText, width=100)
+ if FIXTEXTWRAP is True and "_ABL" in nametag:
+ translatedText = textwrap.fill(
+ translatedText, width=100
+ )
elif FIXTEXTWRAP is True:
- translatedText = textwrap.fill(translatedText, width=WIDTH)
-
+ translatedText = textwrap.fill(
+ translatedText, width=WIDTH
+ )
+
# BR Flag
if BRFLAG is True:
- translatedText = translatedText.replace('\n', '
')
+ translatedText = translatedText.replace("\n", "
")
### Add Var Strings
# CL Flag
if CLFlag:
- translatedText = '\\ac ' + translatedText
- translatedText = translatedText.replace('\n', '\n\\ac ')
- translatedText = re.sub(r'[\\]+?ac\s+', r'\\ac ', translatedText)
+ translatedText = "\\ac " + translatedText
+ translatedText = translatedText.replace("\n", "\n\\ac ")
+ translatedText = re.sub(
+ r"[\\]+?ac\s+", r"\\ac ", translatedText
+ )
CLFlag = False
# Nametag
@@ -1068,38 +1226,38 @@ def searchCodes(page, pbar, jobList, filename):
translatedText = translatedText + nametag
else:
translatedText = nametag + translatedText
- nametag = ''
+ nametag = ""
# Endtag
- if endtag != '':
+ if endtag != "":
translatedText = translatedText + endtag
- endtag = ''
+ endtag = ""
# Set Data
if speakerID != None:
- codeList[speakerID]['parameters'] = [fullSpeaker]
- codeList[j]['parameters'] = [translatedText]
- codeList[j]['code'] = code
- speaker = ''
+ codeList[speakerID]["parameters"] = [fullSpeaker]
+ codeList[j]["parameters"] = [translatedText]
+ codeList[j]["code"] = code
+ speaker = ""
match = []
currentGroup = []
syncIndex = i + 1
- list401.pop(0)
+ list401.pop(0)
## Event Code: 122 [Set Variables]
- if 'code' in codeList[i] and codeList[i]['code'] == 122 and CODE122 is True:
+ if "code" in codeList[i] and codeList[i]["code"] == 122 and CODE122 is True:
# This is going to be the var being set. (IMPORTANT)
- if codeList[i]['parameters'][0] not in list(range(150, 180)):
+ if codeList[i]["parameters"][0] not in list(range(150, 180)):
i += 1
continue
-
- jaString = codeList[i]['parameters'][4]
+
+ jaString = codeList[i]["parameters"][4]
# # For Retarded Devs
# VNameValue = jaString
# i += 1
# continue
-
+
# Definitely don't want to mess with files
# if 'gameV' in jaString or '_' in jaString:
# i += 1
@@ -1109,7 +1267,7 @@ def searchCodes(page, pbar, jobList, filename):
if not isinstance(jaString, str):
i += 1
continue
-
+
# Set String
matchedText = None
if len(re.findall(r"([\'\"])", jaString)) == 2:
@@ -1120,121 +1278,129 @@ def searchCodes(page, pbar, jobList, filename):
# Last Check
if matchedText != None:
# Remove Textwrap
- finalJAString = matchedText.group(1).replace('\\n', ' ')
+ finalJAString = matchedText.group(1).replace("\\n", " ")
# Pass 1
if setData == False:
- if finalJAString != '':
+ if finalJAString != "":
list122.append(finalJAString)
# Pass 2
- else:
- if len(list122) > 0:
+ else:
+ if len(list122) > 0:
# Grab and Replace
translatedText = list122[0]
translatedText = jaString.replace(jaString, translatedText)
# Remove characters that may break scripts
- charList = ['\"', '\\n']
+ charList = ['"', "\\n"]
for char in charList:
- translatedText = translatedText.replace(char, '')
-
+ translatedText = translatedText.replace(char, "")
+
# Textwrap
translatedText = textwrap.fill(translatedText, width=80)
- translatedText = translatedText.replace('\n', '\\n')
+ translatedText = translatedText.replace("\n", "\\n")
# Set
- codeList[i]['parameters'][4] = jaString.replace(finalJAString, translatedText)
+ codeList[i]["parameters"][4] = jaString.replace(
+ finalJAString, translatedText
+ )
list122.pop(0)
## Event Code: 357 [Picture Text] [Optional]
- if 'code' in codeList[i] and codeList[i]['code'] == 357 and CODE357 is True:
- headerString = codeList[i]['parameters'][0]
+ if "code" in codeList[i] and codeList[i]["code"] == 357 and CODE357 is True:
+ headerString = codeList[i]["parameters"][0]
- if headerString == 'LL_GalgeChoiceWindow':
+ if headerString == "LL_GalgeChoiceWindow":
### Message Text First
- jaString = codeList[i]['parameters'][3]['messageText']
+ jaString = codeList[i]["parameters"][3]["messageText"]
# Remove any textwrap & TL
- jaString = re.sub(r'\n', ' ', jaString)
- response = translateGPT(jaString, '', False)
+ jaString = re.sub(r"\n", " ", jaString)
+ response = translateGPT(jaString, "", False)
translatedText = response[0]
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
# Textwrap & Set
translatedText = textwrap.fill(translatedText, width=WIDTH)
- codeList[i]['parameters'][3]['messageText'] = translatedText
+ codeList[i]["parameters"][3]["messageText"] = translatedText
### Choices
- jaString = codeList[i]['parameters'][3]['choices']
+ jaString = codeList[i]["parameters"][3]["choices"]
matchList = re.findall(r'"label[\\]*":[\\]*"(.*?)[\\]', jaString)
if matchList != None:
# Translate
- question = codeList[i]['parameters'][3]['messageText']
- response = translateGPT(matchList, f'Previous text for context: {question}\n\nThis will be a dialogue option', True)
+ question = codeList[i]["parameters"][3]["messageText"]
+ response = translateGPT(
+ matchList,
+ f"Previous text for context: {question}\n\nThis will be a dialogue option",
+ True,
+ )
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
translatedText = jaString
# Replace Strings
for j in range(len(matchList)):
- translatedText = translatedText.replace(matchList[j], response[0][j])
+ translatedText = translatedText.replace(
+ matchList[j], response[0][j]
+ )
# Set Data
- codeList[i]['parameters'][3]['choices'] = translatedText
+ codeList[i]["parameters"][3]["choices"] = translatedText
- if 'SoR_GabWindow' in headerString:
- argVar = 'arg1'
+ if "SoR_GabWindow" in headerString:
+ argVar = "arg1"
### Message Text First
- if argVar in codeList[i]['parameters'][3]:
- jaString = codeList[i]['parameters'][3][argVar]
+ if argVar in codeList[i]["parameters"][3]:
+ jaString = codeList[i]["parameters"][3][argVar]
# If there isn't any Japanese in the text just skip
- if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+', jaString):
+ if not re.search(r"[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+", jaString):
i += 1
continue
# Remove any textwrap & TL
- jaString = re.sub(r'\n', ' ', jaString)
- response = translateGPT(jaString, '', False)
+ jaString = re.sub(r"\n", " ", jaString)
+ response = translateGPT(jaString, "", False)
translatedText = response[0]
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
# Textwrap & Set
translatedText = textwrap.fill(translatedText, width=WIDTH)
- codeList[i]['parameters'][3][argVar] = translatedText
+ codeList[i]["parameters"][3][argVar] = translatedText
pbar.update(1)
- if 'TorigoyaMZ_NotifyMessage' in headerString:
- argVar = 'message'
+ if "TorigoyaMZ_NotifyMessage" in headerString:
+ argVar = "message"
### Message Text First
- if argVar in codeList[i]['parameters'][3]:
- jaString = codeList[i]['parameters'][3][argVar]
+ if argVar in codeList[i]["parameters"][3]:
+ jaString = codeList[i]["parameters"][3][argVar]
# If there isn't any Japanese in the text just skip
- if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+', jaString):
+ if not re.search(r"[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+", jaString):
i += 1
continue
# Remove any textwrap & TL
- jaString = re.sub(r'\n', ' ', jaString)
- response = translateGPT(jaString, '', False)
+ jaString = re.sub(r"\n", " ", jaString)
+ response = translateGPT(jaString, "", False)
translatedText = response[0]
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
# Textwrap & Set
translatedText = textwrap.fill(translatedText, width=WIDTH)
- codeList[i]['parameters'][3][argVar] = translatedText
+ codeList[i]["parameters"][3][argVar] = translatedText
pbar.update(1)
- if '_TMLogWindowMZ' in headerString:
- argVar = 'text'
+ if "_TMLogWindowMZ" in headerString:
+ argVar = "text"
### Message Text First
- if argVar in codeList[i]['parameters'][3]:
- jaString = codeList[i]['parameters'][3][argVar]
+ if argVar in codeList[i]["parameters"][3]:
+ jaString = codeList[i]["parameters"][3][argVar]
# If there isn't any Japanese in the text just skip
# if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+', jaString):
@@ -1242,22 +1408,22 @@ def searchCodes(page, pbar, jobList, filename):
# continue
# Remove any textwrap & TL
- jaString = re.sub(r'\n', ' ', jaString)
- response = translateGPT(jaString, '', False)
+ jaString = re.sub(r"\n", " ", jaString)
+ response = translateGPT(jaString, "", False)
translatedText = response[0]
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
# Textwrap & Set
translatedText = textwrap.fill(translatedText, width=WIDTH)
- codeList[i]['parameters'][3][argVar] = translatedText
+ codeList[i]["parameters"][3][argVar] = translatedText
pbar.update(1)
- if 'DestinationWindow' in headerString:
- argVar = 'destination'
+ if "DestinationWindow" in headerString:
+ argVar = "destination"
### Message Text First
- if argVar in codeList[i]['parameters'][3]:
- jaString = codeList[i]['parameters'][3][argVar]
+ if argVar in codeList[i]["parameters"][3]:
+ jaString = codeList[i]["parameters"][3][argVar]
# If there isn't any Japanese in the text just skip
# if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+', jaString):
@@ -1265,22 +1431,22 @@ def searchCodes(page, pbar, jobList, filename):
# continue
# Remove any textwrap & TL
- jaString = re.sub(r'\n', ' ', jaString)
- response = translateGPT(jaString, '', False)
+ jaString = re.sub(r"\n", " ", jaString)
+ response = translateGPT(jaString, "", False)
translatedText = response[0]
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
# Textwrap & Set
translatedText = textwrap.fill(translatedText, width=WIDTH)
- codeList[i]['parameters'][3][argVar] = translatedText
+ codeList[i]["parameters"][3][argVar] = translatedText
pbar.update(1)
- if 'MNKR_CommonPopupCoreMZ' in headerString:
- argVar = 'text'
+ if "MNKR_CommonPopupCoreMZ" in headerString:
+ argVar = "text"
### Message Text First
- if argVar in codeList[i]['parameters'][3]:
- jaString = codeList[i]['parameters'][3][argVar]
+ if argVar in codeList[i]["parameters"][3]:
+ jaString = codeList[i]["parameters"][3][argVar]
# If there isn't any Japanese in the text just skip
# if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+', jaString):
@@ -1288,142 +1454,164 @@ def searchCodes(page, pbar, jobList, filename):
# continue
# Remove any textwrap & TL
- jaString = re.sub(r'\n', ' ', jaString)
- response = translateGPT(jaString, '', False)
+ jaString = re.sub(r"\n", " ", jaString)
+ response = translateGPT(jaString, "", False)
translatedText = response[0]
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
# Textwrap & Set
translatedText = textwrap.fill(translatedText, width=WIDTH)
- codeList[i]['parameters'][3][argVar] = translatedText
+ codeList[i]["parameters"][3][argVar] = translatedText
pbar.update(1)
-
+
## Event Code: 657 [Picture Text] [Optional]
- if 'code' in codeList[i] and codeList[i]['code'] == 657 and CODE657 is True:
- if 'text' in codeList[i]['parameters'][0]:
- jaString = codeList[i]['parameters'][0]
+ if "code" in codeList[i] and codeList[i]["code"] == 657 and CODE657 is True:
+ if "text" in codeList[i]["parameters"][0]:
+ jaString = codeList[i]["parameters"][0]
if not isinstance(jaString, str):
i += 1
continue
-
+
# Definitely don't want to mess with files
- if '_' in jaString:
+ if "_" in jaString:
i += 1
continue
# If there isn't any Japanese in the text just skip
- if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+', jaString):
+ if not re.search(r"[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+", jaString):
i += 1
continue
# Remove outside text
- startString = re.search(r'^[^一-龠ぁ-ゔァ-ヴー\<\>【】\\]+', jaString)
- jaString = re.sub(r'^[^一-龠ぁ-ゔァ-ヴー\<\>【】\\]+', '', jaString)
- endString = re.search(r'[^一-龠ぁ-ゔァ-ヴー\<\>【】。！？\\]+$', jaString)
- jaString = re.sub(r'[^一-龠ぁ-ゔァ-ヴー\<\>【】。！？\\]+$', '', jaString)
+ startString = re.search(
+ r"^[^一-龠ぁ-ゔァ-ヴー\<\>【】\\]+", jaString
+ )
+ jaString = re.sub(r"^[^一-龠ぁ-ゔァ-ヴー\<\>【】\\]+", "", jaString)
+ endString = re.search(
+ r"[^一-龠ぁ-ゔァ-ヴー\<\>【】。！？\\]+$", jaString
+ )
+ jaString = re.sub(
+ r"[^一-龠ぁ-ゔァ-ヴー\<\>【】。！？\\]+$", "", jaString
+ )
if startString is None:
- startString = ''
+ startString = ""
else:
startString = startString.group()
if endString is None:
- endString = ''
+ endString = ""
else:
endString = endString.group()
# Remove any textwrap
- jaString = re.sub(r'\n', ' ', jaString)
+ jaString = re.sub(r"\n", " ", jaString)
# Translate
- response = translateGPT(jaString, '', True)
+ response = translateGPT(jaString, "", True)
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
translatedText = response[0]
# Remove characters that may break scripts
- charList = ['.', '\"', "'"]
+ charList = [".", '"', "'"]
for char in charList:
- translatedText = translatedText.replace(char, '')
+ translatedText = translatedText.replace(char, "")
# Textwrap
translatedText = textwrap.fill(translatedText, width=WIDTH)
translatedText = startString + translatedText + endString
# Set Data
- codeList[i]['parameters'][0] = translatedText
+ codeList[i]["parameters"][0] = translatedText
- ## Event Code: 101 [Name] [Optional]
- if 'code' in codeList[i] and codeList[i]['code'] == 101 and CODE101 is True:
+ ## Event Code: 101 [Name] [Optional]
+ if "code" in codeList[i] and codeList[i]["code"] == 101 and CODE101 is True:
# Check Window Type (Certain games switch between 1st line speakers and none)
if FIRSTLINESPEAKERS:
- if codeList[i]['parameters'][2] == 0:
+ if codeList[i]["parameters"][2] == 0:
speakerWindow = True
else:
speakerWindow = False
else:
isVar = False
-
+
# Grab String
- jaString = ''
- if len(codeList[i]['parameters']) > 4:
- jaString = codeList[i]['parameters'][4]
+ jaString = ""
+ if len(codeList[i]["parameters"]) > 4:
+ jaString = codeList[i]["parameters"][4]
# Check for Var
- elif len(codeList[i]['parameters']) > 0:
- jaString = codeList[i]['parameters'][0]
+ elif len(codeList[i]["parameters"]) > 0:
+ jaString = codeList[i]["parameters"][0]
isVar = True
if not isinstance(jaString, str):
i += 1
continue
# Force Speaker using var
- if '\\ap[1左]' in jaString.lower() or '\\ap[1右]' in jaString.lower():
- speaker = 'Cecily'
+ if (
+ "\\ap[1左]" in jaString.lower()
+ or "\\ap[1右]" in jaString.lower()
+ ):
+ speaker = "Cecily"
i += 1
continue
- elif '\\ap[2左]' in jaString.lower() or '\\ap[2右]' in jaString.lower():
- speaker = 'Amelia'
+ elif (
+ "\\ap[2左]" in jaString.lower()
+ or "\\ap[2右]" in jaString.lower()
+ ):
+ speaker = "Amelia"
i += 1
continue
- elif '\\ap[3左]' in jaString.lower() or '\\ap[3右]' in jaString.lower():
- speaker = 'Henry'
+ elif (
+ "\\ap[3左]" in jaString.lower()
+ or "\\ap[3右]" in jaString.lower()
+ ):
+ speaker = "Henry"
i += 1
continue
- elif '\\ap[4左]' in jaString.lower() or '\\ap[4右]' in jaString.lower():
- speaker = 'Oswald'
+ elif (
+ "\\ap[4左]" in jaString.lower()
+ or "\\ap[4右]" in jaString.lower()
+ ):
+ speaker = "Oswald"
i += 1
continue
- elif '\\ap' in jaString:
- speaker = re.search(r'[\\]+AP\[(.*?)\]', jaString).group(1)
+ elif "\\ap" in jaString:
+ speaker = re.search(r"[\\]+AP\[(.*?)\]", jaString).group(1)
i += 1
- continue
+ continue
# Get Speaker
- if '\\' not in jaString:
+ if "\\" not in jaString:
response = getSpeaker(jaString)
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
speaker = response[0]
-
+
# Validate Speaker is not empty
if len(speaker) > 0:
if isVar == False:
- codeList[i]['parameters'][4] = speaker
+ codeList[i]["parameters"][4] = speaker
i += 1
continue
else:
- codeList[i]['parameters'][0] = speaker
+ codeList[i]["parameters"][0] = speaker
isVar = False
i += 1
continue
else:
- speaker = ''
+ speaker = ""
## Event Code: 355 or 655 Scripts [Optional]
- if 'code' in codeList[i] and (codeList[i]['code'] == 355 or codeList[i]['code'] == 655) and CODE355655 is True:
- jaString = codeList[i]['parameters'][0]
- regex = r'memory\._eventTitle\s=\s\"(.*)\"'
-
+ if (
+ "code" in codeList[i]
+ and (codeList[i]["code"] == 355 or codeList[i]["code"] == 655)
+ and CODE355655 is True
+ ):
+ jaString = codeList[i]["parameters"][0]
+ regex = r"memory\._eventTitle\s=\s\"(.*)\""
+
# Var Text
match = re.search(regex, jaString)
if re.search(regex, jaString):
@@ -1433,37 +1621,49 @@ def searchCodes(page, pbar, jobList, filename):
list355655.append(finalJAString)
# Pass 2
- else:
+ else:
# Grab and Replace
translatedText = list355655[0]
# Set
- codeList[i]['parameters'][0] = codeList[i]['parameters'][0].replace(finalJAString, translatedText)
- list355655.pop(0)
+ codeList[i]["parameters"][0] = codeList[i]["parameters"][
+ 0
+ ].replace(finalJAString, translatedText)
+ list355655.pop(0)
## Event Code: 408 (Script)
- if 'code' in codeList[i] and (codeList[i]['code'] == 408) and CODE408 is True:
- jaString = codeList[i]['parameters'][0]
+ if (
+ "code" in codeList[i]
+ and (codeList[i]["code"] == 408)
+ and CODE408 is True
+ ):
+ jaString = codeList[i]["parameters"][0]
# If there isn't any Japanese in the text just skip
- if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+', jaString):
+ if not re.search(r"[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+", jaString):
i += 1
continue
- if 'secretText' in jaString:
- regex = r'secretText:\s?(.+)'
- elif 'title' in jaString:
- regex = r'title:\s?(.+)'
+ if "secretText" in jaString:
+ regex = r"secretText:\s?(.+)"
+ elif "title" in jaString:
+ regex = r"title:\s?(.+)"
else:
- regex = r'(.+)'
+ regex = r"(.+)"
# Need to remove outside code and put it back later
matchList = re.findall(regex, jaString)
-
+
for match in matchList:
# Remove Textwrap
- match = match.replace('\n', ' ')
- response = translateGPT(match, 'Reply with the '+ LANGUAGE +' translation of the achievement title.', False)
+ match = match.replace("\n", " ")
+ response = translateGPT(
+ match,
+ "Reply with the "
+ + LANGUAGE
+ + " translation of the achievement title.",
+ False,
+ )
translatedText = response[0]
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
@@ -1472,32 +1672,36 @@ def searchCodes(page, pbar, jobList, filename):
translatedText = jaString.replace(match, translatedText)
# Remove characters that may break scripts
- charList = ['.', '\"', '\\n']
+ charList = [".", '"', "\\n"]
for char in charList:
- translatedText = translatedText.replace(char, '')
+ translatedText = translatedText.replace(char, "")
# Textwrap
translatedText = textwrap.fill(translatedText, width=WIDTH)
# Set Data
- codeList[i]['parameters'][0] = translatedText
+ codeList[i]["parameters"][0] = translatedText
## Event Code: 108 (Script)
- if 'code' in codeList[i] and (codeList[i]['code'] == 108) and CODE108 is True:
- jaString = codeList[i]['parameters'][0]
+ if (
+ "code" in codeList[i]
+ and (codeList[i]["code"] == 108)
+ and CODE108 is True
+ ):
+ jaString = codeList[i]["parameters"][0]
# If there isn't any Japanese in the text just skip
- if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+', jaString):
+ if not re.search(r"[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+", jaString):
i += 1
continue
# Translate
- if 'info:' in jaString:
- regex = r'info:(.*)'
- elif 'ActiveMessage:' in jaString:
- regex = r''
- elif 'event_text' in jaString:
- regex = r'event_text\s*:\s*(.*)'
+ if "info:" in jaString:
+ regex = r"info:(.*)"
+ elif "ActiveMessage:" in jaString:
+ regex = r""
+ elif "event_text" in jaString:
+ regex = r"event_text\s*:\s*(.*)"
else:
i += 1
continue
@@ -1513,75 +1717,85 @@ def searchCodes(page, pbar, jobList, filename):
else:
# Grab and Replace
translatedText = list108[0]
- list108.pop(0)
+ list108.pop(0)
# Remove characters that may break scripts
- charList = ['.', '\"']
+ charList = [".", '"']
for char in charList:
- translatedText = translatedText.replace(char, '')
- translatedText = translatedText.replace('"', '\"')
- translatedText = translatedText.replace(' ', '_')
- translatedText = jaString.replace(match.group(1), translatedText)
+ translatedText = translatedText.replace(char, "")
+ translatedText = translatedText.replace('"', '"')
+ translatedText = translatedText.replace(" ", "_")
+ translatedText = jaString.replace(
+ match.group(1), translatedText
+ )
# Set Data
- codeList[i]['parameters'][0] = translatedText
+ codeList[i]["parameters"][0] = translatedText
## Event Code: 356
- if 'code' in codeList[i] and codeList[i]['code'] == 356 and CODE356 is True:
- jaString = codeList[i]['parameters'][0]
+ if "code" in codeList[i] and codeList[i]["code"] == 356 and CODE356 is True:
+ jaString = codeList[i]["parameters"][0]
oldjaString = jaString
# Grab Speaker
- if 'Tachie showName' in jaString:
- matchList = re.findall(r'Tachie showName (.+)', jaString)
+ if "Tachie showName" in jaString:
+ matchList = re.findall(r"Tachie showName (.+)", jaString)
if len(matchList) > 0:
# Translate
- response = translateGPT(matchList[0], 'Reply with the '+ LANGUAGE +' translation of the NPC name.', False)
+ response = translateGPT(
+ matchList[0],
+ "Reply with the "
+ + LANGUAGE
+ + " translation of the NPC name.",
+ False,
+ )
translatedText = response[0]
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
# Set Text
speaker = translatedText
- speaker = speaker.replace(' ', ' ')
- codeList[i]['parameters'][0] = jaString.replace(matchList[0], speaker)
+ speaker = speaker.replace(" ", " ")
+ codeList[i]["parameters"][0] = jaString.replace(
+ matchList[0], speaker
+ )
i += 1
continue
# Want to translate this script
- if 'D_TEXT ' in jaString:
- regex = r'D_TEXT\s(.*?)\s.+'
- elif 'ShowInfo' in jaString:
- regex = r'ShowInfo\s(.*)'
- elif 'PushGab' in jaString:
- regex = r'PushGab\s(.*)'
- elif 'addLog' in jaString:
- regex = r'addLog\s(.*)'
- elif 'DW_' in jaString:
- regex = r'DW_.*?\s(.*)'
- elif 'CommonPopup' in jaString:
- regex = r'CommonPopup\sadd\stext:(.*?)[\\]+}'
+ if "D_TEXT " in jaString:
+ regex = r"D_TEXT\s(.*?)\s.+"
+ elif "ShowInfo" in jaString:
+ regex = r"ShowInfo\s(.*)"
+ elif "PushGab" in jaString:
+ regex = r"PushGab\s(.*)"
+ elif "addLog" in jaString:
+ regex = r"addLog\s(.*)"
+ elif "DW_" in jaString:
+ regex = r"DW_.*?\s(.*)"
+ elif "CommonPopup" in jaString:
+ regex = r"CommonPopup\sadd\stext:(.*?)[\\]+}"
else:
- regex = r''
+ regex = r""
# Remove any textwrap
- jaString = re.sub(r'\n', '_', jaString)
+ jaString = re.sub(r"\n", "_", jaString)
# Capture Arguments and text
textMatch = re.search(regex, jaString)
- if textMatch and textMatch.group(0) != '':
+ if textMatch and textMatch.group(0) != "":
text = textMatch.group(1)
# Using this to keep track of 401's in a row. Throws IndexError at EndOfList (Expected Behavior)
currentGroup.append(text)
# Check Next Codes for text
- while (codeList[i+1]['code'] == 356):
- match = re.search(regex, codeList[i+1]['parameters'][0])
+ while codeList[i + 1]["code"] == 356:
+ match = re.search(regex, codeList[i + 1]["parameters"][0])
if match == None:
break
else:
- jaString = codeList[i+1]['parameters'][0]
+ jaString = codeList[i + 1]["parameters"][0]
textMatch = re.search(regex, jaString)
if textMatch != None:
currentGroup.append(textMatch.group(1))
@@ -1591,18 +1805,20 @@ def searchCodes(page, pbar, jobList, filename):
finalList = currentGroup
# Clear Group and Reset Index
- currentGroup = []
+ currentGroup = []
i = i - len(finalList) + 1
# Translate
- response = translateGPT(finalList, 'Reply with the '+ LANGUAGE +' Translation.', True)
+ response = translateGPT(
+ finalList, "Reply with the " + LANGUAGE + " Translation.", True
+ )
finalListTL = response[0]
totalTokens[0] += response[1][0]
- totalTokens[1] += response[1][1]
+ totalTokens[1] += response[1][1]
for j in range(len(finalListTL)):
# Grab String Again For Replace
- jaString = codeList[i]['parameters'][0]
+ jaString = codeList[i]["parameters"][0]
textMatch = re.search(regex, jaString)
if textMatch != None:
text = textMatch.group(1)
@@ -1611,105 +1827,127 @@ def searchCodes(page, pbar, jobList, filename):
translatedText = finalListTL[j]
# Textwrap
- translatedText = textwrap.fill(translatedText, width=LISTWIDTH, drop_whitespace=False)
+ translatedText = textwrap.fill(
+ translatedText, width=LISTWIDTH, drop_whitespace=False
+ )
# Remove characters that may break scripts
- charList = ['.', '\"']
+ charList = [".", '"']
for char in charList:
- translatedText = translatedText.replace(char, '')
-
+ translatedText = translatedText.replace(char, "")
+
# Cant have spaces?
- translatedText = translatedText.replace(' ', '_')
+ translatedText = translatedText.replace(" ", "_")
# Fix spacing after ___
- translatedText = translatedText.replace('__\n', '__')
-
+ translatedText = translatedText.replace("__\n", "__")
+
# Put Args Back
translatedText = jaString.replace(text, translatedText)
-
+
# Set Data
- codeList[i]['parameters'][0] = translatedText
+ codeList[i]["parameters"][0] = translatedText
i += 1
else:
i += 1
continue
- if 'namePop' in jaString:
- matchList = re.findall(r'namePop\s\d+\s(.+?)\s.+', jaString)
+ if "namePop" in jaString:
+ matchList = re.findall(r"namePop\s\d+\s(.+?)\s.+", jaString)
if len(matchList) > 0:
# Translate
text = matchList[0]
- response = translateGPT(text, 'Reply with the '+ LANGUAGE +' Translation', False)
+ response = translateGPT(
+ text, "Reply with the " + LANGUAGE + " Translation", False
+ )
translatedText = response[0]
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
# Set Data
translatedText = jaString.replace(text, translatedText)
- codeList[i]['parameters'][0] = translatedText
+ codeList[i]["parameters"][0] = translatedText
- if 'LL_InfoPopupWIndowMV' in jaString:
- matchList = re.findall(r'LL_InfoPopupWIndowMV\sshowWindow\s(.+?) .+', jaString)
+ if "LL_InfoPopupWIndowMV" in jaString:
+ matchList = re.findall(
+ r"LL_InfoPopupWIndowMV\sshowWindow\s(.+?) .+", jaString
+ )
if len(matchList) > 0:
# Translate
text = matchList[0]
- response = translateGPT(text, 'Reply with the '+ LANGUAGE +' Translation', False)
+ response = translateGPT(
+ text, "Reply with the " + LANGUAGE + " Translation", False
+ )
translatedText = response[0]
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
# Set Data
- translatedText = translatedText.replace(' ', '_')
+ translatedText = translatedText.replace(" ", "_")
translatedText = jaString.replace(text, translatedText)
- codeList[i]['parameters'][0] = translatedText
+ codeList[i]["parameters"][0] = translatedText
- if 'OriginMenuStatus SetParam' in jaString:
- matchList = re.findall(r'OriginMenuStatus\sSetParam\sparam[\d]\s(.*)', jaString)
+ if "OriginMenuStatus SetParam" in jaString:
+ matchList = re.findall(
+ r"OriginMenuStatus\sSetParam\sparam[\d]\s(.*)", jaString
+ )
if len(matchList) > 0:
# Translate
text = matchList[0]
- response = translateGPT(text, 'Reply with the '+ LANGUAGE +' Translation', False)
+ response = translateGPT(
+ text, "Reply with the " + LANGUAGE + " Translation", False
+ )
translatedText = response[0]
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
# Set Data
- translatedText = translatedText.replace(' ', '_')
+ translatedText = translatedText.replace(" ", "_")
translatedText = jaString.replace(text, translatedText)
- codeList[i]['parameters'][0] = translatedText
+ codeList[i]["parameters"][0] = translatedText
# LL_GalgeChoiceWindowMV Message
- if 'LL_GalgeChoiceWindowMV setMessageText' in jaString:
+ if "LL_GalgeChoiceWindowMV setMessageText" in jaString:
### Message Text First
- match = re.search(r'LL_GalgeChoiceWindowMV setMessageText (.+)', jaString)
+ match = re.search(
+ r"LL_GalgeChoiceWindowMV setMessageText (.+)", jaString
+ )
if match:
jaString = match.group(1)
# Remove any textwrap & TL
- jaString = re.sub(r'\n', ' ', jaString)
- response = translateGPT(jaString, '', False)
+ jaString = re.sub(r"\n", " ", jaString)
+ response = translateGPT(jaString, "", False)
translatedText = response[0]
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
# Textwrap & Replace Whitespace
translatedText = textwrap.fill(translatedText, width=WIDTH)
- translatedText = translatedText.replace(' ', '_')
+ translatedText = translatedText.replace(" ", "_")
# Replace and Set
- translatedText = match.group(0).replace(match.group(1), translatedText)
- codeList[i]['parameters'][0] = translatedText
+ translatedText = match.group(0).replace(
+ match.group(1), translatedText
+ )
+ codeList[i]["parameters"][0] = translatedText
# LL_GalgeChoiceWindowMV Choices
- if 'LL_GalgeChoiceWindowMV setChoices':
- match = re.search(r'LL_GalgeChoiceWindowMV setChoices (.+)', jaString)
+ if "LL_GalgeChoiceWindowMV setChoices":
+ match = re.search(
+ r"LL_GalgeChoiceWindowMV setChoices (.+)", jaString
+ )
if match:
jaString = match.group(1)
- choiceList = jaString.split(',')
+ choiceList = jaString.split(",")
# Translate
question = translatedText
- response = translateGPT(choiceList, f'Previous text for context: {question}\n\nThis will be a dialogue option', True)
+ response = translateGPT(
+ choiceList,
+ f"Previous text for context: {question}\n\nThis will be a dialogue option",
+ True,
+ )
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
choiceListTL = response[0]
@@ -1717,35 +1955,37 @@ def searchCodes(page, pbar, jobList, filename):
# Replace Strings
for j in range(len(choiceListTL)):
- choiceListTL[j] = choiceListTL[j].replace(' ', '_')
- translatedText = translatedText.replace(choiceList[j], choiceListTL[j])
+ choiceListTL[j] = choiceListTL[j].replace(" ", "_")
+ translatedText = translatedText.replace(
+ choiceList[j], choiceListTL[j]
+ )
# Set Data
- codeList[i]['parameters'][0] = translatedText
-
+ codeList[i]["parameters"][0] = translatedText
+
### Event Code: 102 Show Choice
- if 'code' in codeList[i] and codeList[i]['code'] == 102 and CODE102 is True:
+ if "code" in codeList[i] and codeList[i]["code"] == 102 and CODE102 is True:
choiceList = []
varList = []
- for choice in range(len(codeList[i]['parameters'][0])):
- jaString = codeList[i]['parameters'][0][choice]
- jaString = jaString.replace(' 。', '.')
+ for choice in range(len(codeList[i]["parameters"][0])):
+ jaString = codeList[i]["parameters"][0][choice]
+ jaString = jaString.replace(" 。", ".")
# Avoid Empty Strings
- if jaString == '':
+ if jaString == "":
i += 1
continue
# If and En Statements
- ifVar = ''
- enVar = ''
- ifList = re.findall(r'(if\(.*?\))', jaString)
- enList = re.findall(r'(en\(.*?\))', jaString)
+ ifVar = ""
+ enVar = ""
+ ifList = re.findall(r"(if\(.*?\))", jaString)
+ enList = re.findall(r"(en\(.*?\))", jaString)
if len(ifList) != 0:
- jaString = jaString.replace(ifList[0], '')
+ jaString = jaString.replace(ifList[0], "")
ifVar = ifList[0]
if len(enList) != 0:
- jaString = jaString.replace(enList[0], '')
+ jaString = jaString.replace(enList[0], "")
enVar = enList[0]
varList.append(ifVar + enVar)
@@ -1754,37 +1994,49 @@ def searchCodes(page, pbar, jobList, filename):
# Translate
if len(textHistory) > 0:
- response = translateGPT(choiceList, 'This will be a dialogue option. Previous text for context: ' + textHistory[len(textHistory)-1] + '\n\nThis will be a dialogue option', True)
+ response = translateGPT(
+ choiceList,
+ "This will be a dialogue option. Previous text for context: "
+ + textHistory[len(textHistory) - 1]
+ + "\n\nThis will be a dialogue option",
+ True,
+ )
translatedTextList = response[0]
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
else:
- response = translateGPT(choiceList, 'This will be a dialogue option', True)
+ response = translateGPT(
+ choiceList, "This will be a dialogue option", True
+ )
translatedTextList = response[0]
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
# Check Mismatch
if len(translatedTextList) == len(choiceList):
- for choice in range(len(codeList[i]['parameters'][0])):
+ for choice in range(len(codeList[i]["parameters"][0])):
translatedText = translatedTextList[choice]
# Set Data
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
- if translatedText != '':
- translatedText = varList[choice] + translatedText[0].upper() + translatedText[1:]
+ if translatedText != "":
+ translatedText = (
+ varList[choice]
+ + translatedText[0].upper()
+ + translatedText[1:]
+ )
else:
translatedText = varList[choice] + translatedText
- codeList[i]['parameters'][0][choice] = translatedText
+ codeList[i]["parameters"][0][choice] = translatedText
else:
if filename not in MISMATCH:
MISMATCH.append(filename)
### Event Code: 111 Script
- if 'code' in codeList[i] and codeList[i]['code'] == 111 and CODE111 is True:
- for j in range(len(codeList[i]['parameters'])):
- jaString = codeList[i]['parameters'][j]
+ if "code" in codeList[i] and codeList[i]["code"] == 111 and CODE111 is True:
+ for j in range(len(codeList[i]["parameters"])):
+ jaString = codeList[i]["parameters"][j]
# Check if String
if not isinstance(jaString, str):
@@ -1792,63 +2044,63 @@ def searchCodes(page, pbar, jobList, filename):
continue
# Only TL the Game Variable
- if '$gameVariables' not in jaString:
+ if "$gameVariables" not in jaString:
i += 1
continue
# This is going to be the var being set. (IMPORTANT)
- if '1045' not in jaString:
+ if "1045" not in jaString:
i += 1
continue
# Need to remove outside code and put it back later
matchList = re.findall(r"'(.*?)'", jaString)
-
+
for match in matchList:
- response = translateGPT(match, '', False)
+ response = translateGPT(match, "", False)
translatedText = response[0]
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
# Remove characters that may break scripts
- charList = ['.', '\"', '\'', '\\n']
+ charList = [".", '"', "'", "\\n"]
for char in charList:
- translatedText = translatedText.replace(char, '')
+ translatedText = translatedText.replace(char, "")
jaString = jaString.replace(match, translatedText)
# Set Data
translatedText = jaString
- codeList[i]['parameters'][j] = translatedText
+ codeList[i]["parameters"][j] = translatedText
### Event Code: 320 Set Variable
- if 'code' in codeList[i] and codeList[i]['code'] == 320 and CODE320 is True:
- jaString = codeList[i]['parameters'][1]
+ if "code" in codeList[i] and codeList[i]["code"] == 320 and CODE320 is True:
+ jaString = codeList[i]["parameters"][1]
if not isinstance(jaString, str):
i += 1
continue
-
+
# Definitely don't want to mess with files
- if '■' in jaString or '_' in jaString:
+ if "■" in jaString or "_" in jaString:
i += 1
continue
# If there isn't any Japanese in the text just skip
- if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+', jaString):
+ if not re.search(r"[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+", jaString):
i += 1
continue
-
+
# Translate
getSpeaker(jaString)
# Remove characters that may break scripts
- charList = ['.', '\"', '\'', '\\n']
+ charList = [".", '"', "'", "\\n"]
for char in charList:
- translatedText = translatedText.replace(char, '')
+ translatedText = translatedText.replace(char, "")
# Set Data
- codeList[i]['parameters'][1] = translatedText
-
+ codeList[i]["parameters"][1] = translatedText
+
# Iterate
else:
i += 1
@@ -1860,7 +2112,7 @@ def searchCodes(page, pbar, jobList, filename):
list108TL = []
setData = False
PBAR = pbar
-
+
# 401
if len(list401) > 0:
response = translateGPT(list401, textHistory, True)
@@ -1915,17 +2167,19 @@ def searchCodes(page, pbar, jobList, filename):
# Start Pass 2
if setData:
- searchCodes(page, pbar, [list401TL, list122TL, list355655TL, list108TL], filename)
+ searchCodes(
+ page, pbar, [list401TL, list122TL, list355655TL, list108TL], filename
+ )
# Delete all -1 codes
codeListFinal = []
for i in range(len(codeList)):
- if 'code' in codeList[i] and codeList[i]['code'] != -1:
+ if "code" in codeList[i] and codeList[i]["code"] != -1:
codeListFinal.append(codeList[i])
# Normal Format
- if 'list' in page:
- page['list'] = codeListFinal
+ if "list" in page:
+ page["list"] = codeListFinal
# Special Format (Scenario)
else:
@@ -1933,141 +2187,240 @@ def searchCodes(page, pbar, jobList, filename):
except IndexError as e:
traceback.print_exc()
- raise Exception(str(e) + 'Failed to translate: ' + oldjaString) from None
+ raise Exception(str(e) + "Failed to translate: " + oldjaString) from None
except Exception as e:
traceback.print_exc()
- raise Exception(str(e) + 'Failed to translate: ' + oldjaString) from None
+ raise Exception(str(e) + "Failed to translate: " + oldjaString) from None
return totalTokens
+
def searchSS(state, pbar):
totalTokens = [0, 0]
# Name
- nameResponse = translateGPT(state['name'], 'Reply with only the '+ LANGUAGE +' translation of the RPG Skill name.', False) if 'name' in state else ''
+ nameResponse = (
+ translateGPT(
+ state["name"],
+ "Reply with only the " + LANGUAGE + " translation of the RPG Skill name.",
+ False,
+ )
+ if "name" in state
+ else ""
+ )
# Description
- descriptionResponse = translateGPT(state['description'], 'Reply with only the '+ LANGUAGE +' translation of the description.', False) if 'description' in state else ''
+ descriptionResponse = (
+ translateGPT(
+ state["description"],
+ "Reply with only the " + LANGUAGE + " translation of the description.",
+ False,
+ )
+ if "description" in state
+ else ""
+ )
# Messages
- message1Response = ''
- message4Response = ''
- message2Response = ''
- message3Response = ''
-
- if 'message1' in state:
- if len(state['message1']) > 0 and state['message1'][0] in ['は', 'を', 'の', 'に', 'が']:
- message1Response = translateGPT('Taro' + state['message1'], 'reply with only the gender neutral '+ LANGUAGE +' translation of the action log. Always start the sentence with Taro. For example,\
-Translate \'Taroを倒した！\' as \'Taro was defeated!\'', False)
- else:
- message1Response = translateGPT(state['message1'], 'reply with only the gender neutral '+ LANGUAGE +' translation', False)
+ message1Response = ""
+ message4Response = ""
+ message2Response = ""
+ message3Response = ""
- if 'message2' in state:
- if len(state['message2']) > 0 and state['message2'][0] in ['は', 'を', 'の', 'に', 'が']:
- message2Response = translateGPT('Taro' + state['message2'], 'reply with only the gender neutral '+ LANGUAGE +' translation of the action log. Always start the sentence with Taro. For example,\
-Translate \'Taroを倒した！\' as \'Taro was defeated!\'', False)
+ if "message1" in state:
+ if len(state["message1"]) > 0 and state["message1"][0] in [
+ "は",
+ "を",
+ "の",
+ "に",
+ "が",
+ ]:
+ message1Response = translateGPT(
+ "Taro" + state["message1"],
+ "reply with only the gender neutral "
+ + LANGUAGE
+ + " translation of the action log. Always start the sentence with Taro. For example,\
+Translate 'Taroを倒した！' as 'Taro was defeated!'",
+ False,
+ )
else:
- message2Response = translateGPT(state['message2'], 'reply with only the gender neutral '+ LANGUAGE +' translation', False)
+ message1Response = translateGPT(
+ state["message1"],
+ "reply with only the gender neutral " + LANGUAGE + " translation",
+ False,
+ )
- if 'message3' in state:
- if len(state['message3']) > 0 and state['message3'][0] in ['は', 'を', 'の', 'に', 'が']:
- message3Response = translateGPT('Taro' + state['message3'], 'reply with only the gender neutral '+ LANGUAGE +' translation of the action log. Always start the sentence with Taro. For example,\
-Translate \'Taroを倒した！\' as \'Taro was defeated!\'', False)
+ if "message2" in state:
+ if len(state["message2"]) > 0 and state["message2"][0] in [
+ "は",
+ "を",
+ "の",
+ "に",
+ "が",
+ ]:
+ message2Response = translateGPT(
+ "Taro" + state["message2"],
+ "reply with only the gender neutral "
+ + LANGUAGE
+ + " translation of the action log. Always start the sentence with Taro. For example,\
+Translate 'Taroを倒した！' as 'Taro was defeated!'",
+ False,
+ )
else:
- message3Response = translateGPT(state['message3'], 'reply with only the gender neutral '+ LANGUAGE +' translation', False)
+ message2Response = translateGPT(
+ state["message2"],
+ "reply with only the gender neutral " + LANGUAGE + " translation",
+ False,
+ )
- if 'message4' in state:
- if len(state['message4']) > 0 and state['message4'][0] in ['は', 'を', 'の', 'に', 'が']:
- message4Response = translateGPT('Taro' + state['message4'], 'reply with only the gender neutral '+ LANGUAGE +' translation of the action log. Always start the sentence with Taro. For example,\
-Translate \'Taroを倒した！\' as \'Taro was defeated!\'', False)
+ if "message3" in state:
+ if len(state["message3"]) > 0 and state["message3"][0] in [
+ "は",
+ "を",
+ "の",
+ "に",
+ "が",
+ ]:
+ message3Response = translateGPT(
+ "Taro" + state["message3"],
+ "reply with only the gender neutral "
+ + LANGUAGE
+ + " translation of the action log. Always start the sentence with Taro. For example,\
+Translate 'Taroを倒した！' as 'Taro was defeated!'",
+ False,
+ )
else:
- message4Response = translateGPT(state['message4'], 'reply with only the gender neutral '+ LANGUAGE +' translation', False)
+ message3Response = translateGPT(
+ state["message3"],
+ "reply with only the gender neutral " + LANGUAGE + " translation",
+ False,
+ )
+
+ if "message4" in state:
+ if len(state["message4"]) > 0 and state["message4"][0] in [
+ "は",
+ "を",
+ "の",
+ "に",
+ "が",
+ ]:
+ message4Response = translateGPT(
+ "Taro" + state["message4"],
+ "reply with only the gender neutral "
+ + LANGUAGE
+ + " translation of the action log. Always start the sentence with Taro. For example,\
+Translate 'Taroを倒した！' as 'Taro was defeated!'",
+ False,
+ )
+ else:
+ message4Response = translateGPT(
+ state["message4"],
+ "reply with only the gender neutral " + LANGUAGE + " translation",
+ False,
+ )
# Translate State Notes
- if 'help' in state['note']:
- noteResponse = translateNote(state, r']*)>')
+ if "help" in state["note"]:
+ noteResponse = translateNote(state, r"]*)>")
totalTokens[0] += noteResponse[0]
totalTokens[1] += noteResponse[1]
- if 'STATE_HELP' in state['note']:
- noteResponse = translateNote(state, r'\n(.*)\n')
+ if "STATE_HELP" in state["note"]:
+ noteResponse = translateNote(state, r"\n(.*)\n")
totalTokens[0] += noteResponse[0]
totalTokens[1] += noteResponse[1]
-
+
# Count totalTokens
- totalTokens[0] += nameResponse[1][0] if nameResponse != '' else 0
- totalTokens[1] += nameResponse[1][1] if nameResponse != '' else 0
- totalTokens[0] += descriptionResponse[1][0] if descriptionResponse != '' else 0
- totalTokens[1] += descriptionResponse[1][1] if descriptionResponse != '' else 0
- totalTokens[0] += message1Response[1][0] if message1Response != '' else 0
- totalTokens[1] += message1Response[1][1] if message1Response != '' else 0
- totalTokens[0] += message2Response[1][0] if message2Response != '' else 0
- totalTokens[1] += message2Response[1][1] if message2Response != '' else 0
- totalTokens[0] += message3Response[1][0] if message3Response != '' else 0
- totalTokens[1] += message3Response[1][1] if message3Response != '' else 0
- totalTokens[0] += message4Response[1][0] if message4Response != '' else 0
- totalTokens[1] += message4Response[1][1] if message4Response != '' else 0
+ totalTokens[0] += nameResponse[1][0] if nameResponse != "" else 0
+ totalTokens[1] += nameResponse[1][1] if nameResponse != "" else 0
+ totalTokens[0] += descriptionResponse[1][0] if descriptionResponse != "" else 0
+ totalTokens[1] += descriptionResponse[1][1] if descriptionResponse != "" else 0
+ totalTokens[0] += message1Response[1][0] if message1Response != "" else 0
+ totalTokens[1] += message1Response[1][1] if message1Response != "" else 0
+ totalTokens[0] += message2Response[1][0] if message2Response != "" else 0
+ totalTokens[1] += message2Response[1][1] if message2Response != "" else 0
+ totalTokens[0] += message3Response[1][0] if message3Response != "" else 0
+ totalTokens[1] += message3Response[1][1] if message3Response != "" else 0
+ totalTokens[0] += message4Response[1][0] if message4Response != "" else 0
+ totalTokens[1] += message4Response[1][1] if message4Response != "" else 0
# Set Data
- if 'name' in state:
- state['name'] = nameResponse[0].replace('\"', '')
- if 'description' in state:
+ if "name" in state:
+ state["name"] = nameResponse[0].replace('"', "")
+ if "description" in state:
# Textwrap
translatedText = descriptionResponse[0]
translatedText = textwrap.fill(translatedText, width=LISTWIDTH)
- state['description'] = translatedText.replace('\"', '')
- if 'message1' in state:
- state['message1'] = message1Response[0].replace('\"', '').replace('Taro', '')
- if 'message2' in state:
- state['message2'] = message2Response[0].replace('\"', '').replace('Taro', '')
- if 'message3' in state:
- state['message3'] = message3Response[0].replace('\"', '').replace('Taro', '')
- if 'message4' in state:
- state['message4'] = message4Response[0].replace('\"', '').replace('Taro', '')
+ state["description"] = translatedText.replace('"', "")
+ if "message1" in state:
+ state["message1"] = message1Response[0].replace('"', "").replace("Taro", "")
+ if "message2" in state:
+ state["message2"] = message2Response[0].replace('"', "").replace("Taro", "")
+ if "message3" in state:
+ state["message3"] = message3Response[0].replace('"', "").replace("Taro", "")
+ if "message4" in state:
+ state["message4"] = message4Response[0].replace('"', "").replace("Taro", "")
-
return totalTokens
+
def searchSystem(data, pbar):
totalTokens = [0, 0]
- context = 'Reply with only the '+ LANGUAGE +' translation of the UI textbox."'
+ context = "Reply with only the " + LANGUAGE + ' translation of the UI textbox."'
# Title
- response = translateGPT(data['gameTitle'], ' Reply with the '+ LANGUAGE +' translation of the game title name', False)
+ response = translateGPT(
+ data["gameTitle"],
+ " Reply with the " + LANGUAGE + " translation of the game title name",
+ False,
+ )
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
- data['gameTitle'] = response[0].strip('.')
-
+ data["gameTitle"] = response[0].strip(".")
+
# Terms
- for term in data['terms']:
- if term != 'messages':
- termList = data['terms'][term]
+ for term in data["terms"]:
+ if term != "messages":
+ termList = data["terms"][term]
for i in range(len(termList)): # Last item is a messages object
if termList[i] is not None:
response = translateGPT(termList[i], context, False)
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
- termList[i] = response[0].replace('\"', '').strip()
-
+ termList[i] = response[0].replace('"', "").strip()
+
# Armor Types
- for i in range(len(data['armorTypes'])):
- response = translateGPT(data['armorTypes'][i], 'Reply with only the '+ LANGUAGE +' translation of the armor type', False)
+ for i in range(len(data["armorTypes"])):
+ response = translateGPT(
+ data["armorTypes"][i],
+ "Reply with only the " + LANGUAGE + " translation of the armor type",
+ False,
+ )
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
- data['armorTypes'][i] = response[0].replace('\"', '').strip()
+ data["armorTypes"][i] = response[0].replace('"', "").strip()
# Skill Types
- for i in range(len(data['skillTypes'])):
- response = translateGPT(data['skillTypes'][i], 'Reply with only the '+ LANGUAGE +' translation', False)
+ for i in range(len(data["skillTypes"])):
+ response = translateGPT(
+ data["skillTypes"][i],
+ "Reply with only the " + LANGUAGE + " translation",
+ False,
+ )
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
- data['skillTypes'][i] = response[0].replace('\"', '').strip()
+ data["skillTypes"][i] = response[0].replace('"', "").strip()
# Equip Types
- for i in range(len(data['equipTypes'])):
- response = translateGPT(data['equipTypes'][i], 'Reply with only the '+ LANGUAGE +' translation of the equipment type. No disclaimers.', False)
+ for i in range(len(data["equipTypes"])):
+ response = translateGPT(
+ data["equipTypes"][i],
+ "Reply with only the "
+ + LANGUAGE
+ + " translation of the equipment type. No disclaimers.",
+ False,
+ )
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
- data['equipTypes'][i] = response[0].replace('\"', '').strip()
+ data["equipTypes"][i] = response[0].replace('"', "").strip()
# # Variables (Optional ususally)
# for i in range(len(data['variables'])):
@@ -2075,42 +2428,56 @@ def searchSystem(data, pbar):
# totalTokens[0] += response[1][0]
# totalTokens[1] += response[1][1]
# data['variables'][i] = response[0].replace('\"', '').strip()
-
# Messages
- messages = (data['terms']['messages'])
+ messages = data["terms"]["messages"]
for key, value in messages.items():
- response = translateGPT(value, 'Reply with only the '+ LANGUAGE +' translation of the battle text.\nTranslate "常時ダッシュ" as "Always Dash"\nTranslate "次の%1まで" as Next %1.', False)
+ response = translateGPT(
+ value,
+ "Reply with only the "
+ + LANGUAGE
+ + ' translation of the battle text.\nTranslate "常時ダッシュ" as "Always Dash"\nTranslate "次の%1まで" as Next %1.',
+ False,
+ )
translatedText = response[0]
# Remove characters that may break scripts
- charList = ['.', '\"', '\\n']
+ charList = [".", '"', "\\n"]
for char in charList:
- translatedText = translatedText.replace(char, '')
+ translatedText = translatedText.replace(char, "")
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
messages[key] = translatedText
-
+
return totalTokens
+
# Save some money and enter the character before translation
def getSpeaker(speaker):
match speaker:
- case 'ファイン':
- return ['Fine', [0,0]]
- case '':
- return ['', [0,0]]
+ case "ファイン":
+ return ["Fine", [0, 0]]
+ case "":
+ return ["", [0, 0]]
case _:
# Store Speaker
if speaker not in str(NAMESLIST):
- response = translateGPT(speaker, 'Reply with the '+ LANGUAGE +' translation of the NPC name.', False)
+ response = translateGPT(
+ speaker,
+ "Reply with the " + LANGUAGE + " translation of the NPC name.",
+ False,
+ )
response[0] = response[0].title()
response[0] = response[0].replace("'S", "'s")
# Retry if name doesn't translate for some reason
- if re.search(r'([a-zA-Z？?])', response[0]) == None:
- response = translateGPT(speaker, 'Reply with the '+ LANGUAGE +' translation of the NPC name.', False)
+ if re.search(r"([a-zA-Z？?])", response[0]) == None:
+ response = translateGPT(
+ speaker,
+ "Reply with the " + LANGUAGE + " translation of the NPC name.",
+ False,
+ )
response[0] = response[0].title()
response[0] = response[0].replace("'S", "'s")
@@ -2121,51 +2488,59 @@ def getSpeaker(speaker):
else:
for i in range(len(NAMESLIST)):
if speaker == NAMESLIST[i][0]:
- return [NAMESLIST[i][1],[0,0]]
-
- return [speaker,[0,0]]
+ return [NAMESLIST[i][1], [0, 0]]
+
+ return [speaker, [0, 0]]
+
def subVars(jaString):
- jaString = jaString.replace('\u3000', ' ')
+ jaString = jaString.replace("\u3000", " ")
# Formatting
count = 0
- codeList = re.findall(r'[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]', jaString)
+ codeList = re.findall(r"[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]", jaString)
codeList = set(codeList)
if len(codeList) != 0:
for var in codeList:
- jaString = jaString.replace(var, '[FCode_' + str(count) + ']')
+ jaString = jaString.replace(var, "[FCode_" + str(count) + "]")
count += 1
# Put all lists in list and return
return [jaString, codeList]
+
def resubVars(translatedText, codeList):
# Fix Spacing and ChatGPT Nonsense
- matchList = re.findall(r'\[\s?.+?\s?\]', translatedText)
+ matchList = re.findall(r"\[\s?.+?\s?\]", translatedText)
if len(matchList) > 0:
for match in matchList:
text = match.strip()
translatedText = translatedText.replace(match, text)
-
+
# Formatting
count = 0
if len(codeList) != 0:
for var in codeList:
- translatedText = translatedText.replace('[FCode_' + str(count) + ']', var)
+ translatedText = translatedText.replace("[FCode_" + str(count) + "]", var)
count += 1
return translatedText
+
def batchList(input_list, batch_size):
if not isinstance(batch_size, int) or batch_size <= 0:
raise ValueError("batch_size must be a positive integer")
-
- return [input_list[i:i + batch_size] for i in range(0, len(input_list), batch_size)]
+
+ return [
+ input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)
+ ]
+
def createContext(fullPromptFlag, subbedT, format):
- system = PROMPT + VOCAB if fullPromptFlag else \
- f"\
+ system = (
+ PROMPT + VOCAB
+ if fullPromptFlag
+ else f"\
You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
- You translate everything, including erotic and explicit content.\n\
@@ -2177,12 +2552,14 @@ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{
- `...` can be a part of the dialogue. Translate it as it is.\n\
{VOCAB}\n\
"
- if format == 'json':
- user = f'```json\n{subbedT}\n```'
+ )
+ if format == "json":
+ user = f"```json\n{subbedT}\n```"
else:
user = subbedT
return system, user
+
def translateText(system, user, history, penalty, format, model=MODEL):
# Prompt
msg = [{"role": "system", "content": system}]
@@ -2194,13 +2571,13 @@ def translateText(system, user, history, penalty, format, model=MODEL):
msg.append({"role": "system", "content": history})
# Response Format
- if format == 'json':
- responseFormat = { "type": "json_object" }
+ if format == "json":
+ responseFormat = {"type": "json_object"}
else:
- responseFormat = { "type": "text" }
-
+ responseFormat = {"type": "text"}
+
# Content to TL
- msg.append({"role": "user", "content": f'{user}'})
+ msg.append({"role": "user", "content": f"{user}"})
response = openai.chat.completions.create(
temperature=0,
frequency_penalty=penalty,
@@ -2210,18 +2587,19 @@ def translateText(system, user, history, penalty, format, model=MODEL):
)
return response
+
def cleanTranslatedText(translatedText, varResponse):
placeholders = {
- f'{LANGUAGE} Translation: ': '',
- 'Translation: ': '',
- 'っ': '',
- '〜': '~',
- 'ッ': '',
- '。': '.',
- '「': '\\"',
- '」': '\\"',
- '- ': '-',
- 'Placeholder Text': '',
+ f"{LANGUAGE} Translation: ": "",
+ "Translation: ": "",
+ "っ": "",
+ "〜": "~",
+ "ッ": "",
+ "。": ".",
+ "「": '\\"',
+ "」": '\\"',
+ "- ": "-",
+ "Placeholder Text": "",
# Add more replacements as needed
}
for target, replacement in placeholders.items():
@@ -2232,11 +2610,12 @@ def cleanTranslatedText(translatedText, varResponse):
translatedText = resubVars(translatedText, varResponse[1])
return translatedText
+
def elongateCharacters(text):
# Define a pattern to match one character followed by one or more `ー` characters
# Using a positive lookbehind assertion to capture the preceding character
- pattern = r'(?<=(.))ー+'
-
+ pattern = r"(?<=(.))ー+"
+
# Define a replacement function that elongates the captured character
def repl(match):
char = match.group(1) # The character before the ー sequence
@@ -2246,6 +2625,7 @@ def elongateCharacters(text):
# Use re.sub() to replace the pattern in the text
return re.sub(pattern, repl, text)
+
def extractTranslation(translatedTextList, is_list):
try:
line_dict = json.loads(translatedTextList)
@@ -2257,15 +2637,15 @@ def extractTranslation(translatedTextList, is_list):
return string_list[0]
except Exception as e:
- print(f'extractTranslation Error: {e}')
+ print(f"extractTranslation Error: {e}")
return None
def countTokens(system, user, history):
inputTotalTokens = 0
outputTotalTokens = 0
- enc = tiktoken.encoding_for_model('gpt-4')
-
+ enc = tiktoken.encoding_for_model("gpt-4")
+
# Input
if isinstance(history, list):
for line in history:
@@ -2276,26 +2656,28 @@ def countTokens(system, user, history):
inputTotalTokens += len(enc.encode(user))
# Output
- outputTotalTokens += round(len(enc.encode(user))*3)
+ outputTotalTokens += round(len(enc.encode(user)) * 3)
return [inputTotalTokens, outputTotalTokens]
+
def combineList(tlist, text):
if isinstance(text, list):
return [t for sublist in tlist for t in sublist]
return tlist[0]
+
@retry(exceptions=Exception, tries=5, delay=5)
def translateGPT(text, history, fullPromptFlag):
global PBAR, MISMATCH, FILENAME
-
+
mismatch = False
totalTokens = [0, 0]
if isinstance(text, list):
- format = 'json'
+ format = "json"
tList = batchList(text, BATCHSIZE)
else:
- format = 'text'
+ format = "text"
tList = [text]
for index, tItem in enumerate(tList):
@@ -2310,7 +2692,7 @@ def translateGPT(text, history, fullPromptFlag):
subbedT = varResponse[0]
# Things to Check before starting translation
- if not re.search(r'[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+', subbedT):
+ if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", subbedT):
if PBAR is not None:
PBAR.update(len(tItem))
continue
@@ -2335,9 +2717,11 @@ def translateGPT(text, history, fullPromptFlag):
translatedText = cleanTranslatedText(translatedText, varResponse)
if isinstance(tItem, list):
extractedTranslations = extractTranslation(translatedText, True)
- if extractedTranslations == None or len(tItem) != len(extractedTranslations):
+ if extractedTranslations == None or len(tItem) != len(
+ extractedTranslations
+ ):
# Mismatch. Try Again
- response = translateText(system, user, history, 0.05, format, 'gpt-4o')
+ response = translateText(system, user, history, 0.05, format, "gpt-4o")
translatedText = response.choices[0].message.content
totalTokens[0] += response.usage.prompt_tokens
totalTokens[1] += response.usage.completion_tokens
@@ -2346,13 +2730,17 @@ def translateGPT(text, history, fullPromptFlag):
translatedText = cleanTranslatedText(translatedText, varResponse)
if isinstance(tItem, list):
extractedTranslations = extractTranslation(translatedText, True)
- if extractedTranslations == None or len(tItem) != len(extractedTranslations):
- mismatch = True # Just here for breakpoint
-
+ if extractedTranslations == None or len(tItem) != len(
+ extractedTranslations
+ ):
+ mismatch = True # Just here for breakpoint
+
# Set if no mismatch
if mismatch == False:
tList[index] = extractedTranslations
- history = extractedTranslations[-10:] # Update history if we have a list
+ history = extractedTranslations[
+ -10:
+ ] # Update history if we have a list
else:
history = text[-10:]
mismatch = False
@@ -2365,7 +2753,7 @@ def translateGPT(text, history, fullPromptFlag):
PBAR.update(len(tItem))
else:
# Ensure we're passing a single string to extractTranslation
- tList[index] = translatedText
+ tList[index] = translatedText
finalList = combineList(tList, text)
return [finalList, totalTokens]
diff --git a/modules/rpgmakerplugin.py b/modules/rpgmakerplugin.py
index ce4fb36..3d3b493 100644
--- a/modules/rpgmakerplugin.py
+++ b/modules/rpgmakerplugin.py
@@ -16,34 +16,34 @@ from tqdm import tqdm
# Open AI
load_dotenv()
-if os.getenv('api').replace(' ', '') != '':
- openai.base_url = os.getenv('api')
-openai.organization = os.getenv('org')
-openai.api_key = os.getenv('key')
+if os.getenv("api").replace(" ", "") != "":
+ openai.base_url = os.getenv("api")
+openai.organization = os.getenv("org")
+openai.api_key = os.getenv("key")
-#Globals
-MODEL = os.getenv('model')
-TIMEOUT = int(os.getenv('timeout'))
-LANGUAGE = os.getenv('language').capitalize()
-PROMPT = Path('prompt.txt').read_text(encoding='utf-8')
-VOCAB = Path('vocab.txt').read_text(encoding='utf-8')
-THREADS = int(os.getenv('threads'))
+# Globals
+MODEL = os.getenv("model")
+TIMEOUT = int(os.getenv("timeout"))
+LANGUAGE = os.getenv("language").capitalize()
+PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
+VOCAB = Path("vocab.txt").read_text(encoding="utf-8")
+THREADS = int(os.getenv("threads"))
LOCK = threading.Lock()
-WIDTH = int(os.getenv('width'))
-LISTWIDTH = int(os.getenv('listWidth'))
+WIDTH = int(os.getenv("width"))
+LISTWIDTH = int(os.getenv("listWidth"))
NOTEWIDTH = 70
MAXHISTORY = 10
-ESTIMATE = ''
+ESTIMATE = ""
TOKENS = [0, 0]
NAMESLIST = []
-NAMES = False # Output a list of all the character names found
-BRFLAG = False # If the game uses
instead
+NAMES = False # Output a list of all the character names found
+BRFLAG = False # If the game uses
instead
FIXTEXTWRAP = True # Overwrites textwrap
-IGNORETLTEXT = False # Ignores all translated text.
-MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
+IGNORETLTEXT = False # Ignores all translated text.
+MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
-#tqdm Globals
-BAR_FORMAT='{l_bar}{bar:10}{r_bar}{bar:-10b}'
+# tqdm Globals
+BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
POSITION = 0
LEAVE = False
PBAR = None
@@ -51,15 +51,16 @@ PBAR = None
# Pricing - Depends on the model https://openai.com/pricing
# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request
# If you are getting a MISMATCH LENGTH error, lower the batch size.
-if 'gpt-3.5' in MODEL:
- INPUTAPICOST = .002
- OUTPUTAPICOST = .002
+if "gpt-3.5" in MODEL:
+ INPUTAPICOST = 0.002
+ OUTPUTAPICOST = 0.002
BATCHSIZE = 10
-elif 'gpt-4' in MODEL:
- INPUTAPICOST = .005
- OUTPUTAPICOST = .015
+elif "gpt-4" in MODEL:
+ INPUTAPICOST = 0.005
+ OUTPUTAPICOST = 0.015
BATCHSIZE = 40
+
def handlePlugin(filename, estimate):
global ESTIMATE, PBAR
ESTIMATE = estimate
@@ -76,17 +77,21 @@ def handlePlugin(filename, estimate):
TOKENS[1] += translatedData[1][1]
# Print Total
- totalString = getResultString(['', TOKENS, None], end - start, 'TOTAL')
+ totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
# Print any errors on maps
if len(MISMATCH) > 0:
- return totalString + Fore.RED + f'\nMismatch Errors: {MISMATCH}' + Fore.RESET
+ return (
+ totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
+ )
else:
return totalString
-
+
else:
try:
- with open('translated/' + filename, 'w', encoding='utf_8', errors='ignore') as outFile:
+ with open(
+ "translated/" + filename, "w", encoding="utf_8", errors="ignore"
+ ) as outFile:
start = time.time()
translatedData = openFiles(filename)
@@ -99,23 +104,36 @@ def handlePlugin(filename, estimate):
TOKENS[1] += translatedData[1][1]
except Exception:
traceback.print_exc()
- return 'Fail'
+ return "Fail"
+
+ return getResultString(["", TOKENS, None], end - start, "TOTAL")
- return getResultString(['', TOKENS, None], end - start, 'TOTAL')
def getResultString(translatedData, translationTime, filename):
# File Print String
- totalTokenstring =\
- Fore.YELLOW +\
- '[Input: ' + str(translatedData[1][0]) + ']'\
- '[Output: ' + str(translatedData[1][1]) + ']'\
- '[Cost: ${:,.4f}'.format((translatedData[1][0] * .001 * INPUTAPICOST) +\
- (translatedData[1][1] * .001 * OUTPUTAPICOST)) + ']'
- timeString = Fore.BLUE + '[' + str(round(translationTime, 1)) + 's]'
+ totalTokenstring = (
+ Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
+ "[Output: "
+ + str(translatedData[1][1])
+ + "]" "[Cost: ${:,.4f}".format(
+ (translatedData[1][0] * 0.001 * INPUTAPICOST)
+ + (translatedData[1][1] * 0.001 * OUTPUTAPICOST)
+ )
+ + "]"
+ )
+ timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
if translatedData[2] == None:
# Success
- return filename + ': ' + totalTokenstring + timeString + Fore.GREEN + u' \u2713 ' + Fore.RESET
+ return (
+ filename
+ + ": "
+ + totalTokenstring
+ + timeString
+ + Fore.GREEN
+ + " \u2713 "
+ + Fore.RESET
+ )
else:
# Fail
@@ -124,31 +142,41 @@ def getResultString(translatedData, translationTime, filename):
except Exception as e:
traceback.print_exc()
errorString = str(e) + Fore.RED
- return filename + ': ' + totalTokenstring + timeString + Fore.RED + u' \u2717 ' +\
- errorString + Fore.RESET
+ return (
+ filename
+ + ": "
+ + totalTokenstring
+ + timeString
+ + Fore.RED
+ + " \u2717 "
+ + errorString
+ + Fore.RESET
+ )
+
def openFiles(filename):
- with open('files/' + filename, 'r', encoding='utf_8') as readFile:
+ with open("files/" + filename, "r", encoding="utf_8") as readFile:
translatedData = parsePlugin(readFile, filename)
# Delete lines marked for deletion
finalData = []
for line in translatedData[0]:
- if line != '\\d\n':
+ if line != "\\d\n":
finalData.append(line)
translatedData[0] = finalData
-
+
return translatedData
+
def parsePlugin(readFile, filename):
- totalTokens = [0,0]
+ totalTokens = [0, 0]
# Read File into data
data = readFile.readlines()
# Create Progress Bar
with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
- pbar.desc=filename
+ pbar.desc = filename
try:
result = translatePlugin(data, pbar, filename, [])
@@ -159,19 +187,20 @@ def parsePlugin(readFile, filename):
return [data, totalTokens, e]
return [data, totalTokens, None]
+
def translatePlugin(data, pbar, filename, translatedList):
stringList = []
currentGroup = []
- tokens = [0,0]
- speaker = ''
+ tokens = [0, 0]
+ speaker = ""
voice = False
global LOCK, ESTIMATE
i = 0
while i < len(data):
voice = False
- speaker = ''
- newline = r'\n'
+ speaker = ""
+ newline = r"\n"
"""
Plugin List
@@ -192,16 +221,16 @@ def translatePlugin(data, pbar, filename, translatedList):
for match in matchList:
# Save Original String
originalString = match
-
+
# Remove any textwrap
- match = match.replace(newline, ' ')
+ match = match.replace(newline, " ")
# Pass 1
- if translatedList == []:
+ if translatedList == []:
# Add String
- if match != '\\\\\\\\':
+ if match != "\\\\\\\\":
stringList.append(match.strip())
-
+
# Pass 2
else:
# Get Text
@@ -216,7 +245,7 @@ def translatePlugin(data, pbar, filename, translatedList):
# Textwrap
translatedText = textwrap.fill(translatedText, width=WIDTH)
- translatedText = translatedText.replace('\n', newline)
+ translatedText = translatedText.replace("\n", newline)
# Replace Single Quotes
translatedText = translatedText.replace("'", "\\'")
@@ -224,7 +253,7 @@ def translatePlugin(data, pbar, filename, translatedList):
# Set Data
data[i] = data[i].replace(originalString, translatedText)
- # Next Line
+ # Next Line
i += 1
# EOF
@@ -233,9 +262,9 @@ def translatePlugin(data, pbar, filename, translatedList):
pbar.total = len(stringList)
pbar.refresh()
PBAR = pbar
-
+
# Translate
- response = translateGPT(stringList, '', True)
+ response = translateGPT(stringList, "", True)
tokens[0] += response[1][0]
tokens[1] += response[1][1]
translatedList = response[0]
@@ -251,23 +280,32 @@ def translatePlugin(data, pbar, filename, translatedList):
MISMATCH.append(filename)
return tokens
+
# Save some money and enter the character before translation
def getSpeaker(speaker):
match speaker:
- case 'ファイン':
- return ['Fine', [0,0]]
- case '':
- return ['', [0,0]]
+ case "ファイン":
+ return ["Fine", [0, 0]]
+ case "":
+ return ["", [0, 0]]
case _:
# Store Speaker
if speaker not in str(NAMESLIST):
- response = translateGPT(speaker, 'Reply with the '+ LANGUAGE +' translation of the NPC name.', False)
+ response = translateGPT(
+ speaker,
+ "Reply with the " + LANGUAGE + " translation of the NPC name.",
+ False,
+ )
response[0] = response[0].title()
response[0] = response[0].replace("'S", "'s")
# Retry if name doesn't translate for some reason
- if re.search(r'([a-zA-Z？?])', response[0]) == None:
- response = translateGPT(speaker, 'Reply with the '+ LANGUAGE +' translation of the NPC name.', False)
+ if re.search(r"([a-zA-Z？?])", response[0]) == None:
+ response = translateGPT(
+ speaker,
+ "Reply with the " + LANGUAGE + " translation of the NPC name.",
+ False,
+ )
response[0] = response[0].title()
response[0] = response[0].replace("'S", "'s")
@@ -278,74 +316,76 @@ def getSpeaker(speaker):
else:
for i in range(len(NAMESLIST)):
if speaker == NAMESLIST[i][0]:
- return [NAMESLIST[i][1],[0,0]]
-
- return [speaker,[0,0]]
+ return [NAMESLIST[i][1], [0, 0]]
+
+ return [speaker, [0, 0]]
+
def subVars(jaString):
- jaString = jaString.replace('\u3000', ' ')
+ jaString = jaString.replace("\u3000", " ")
# Nested
count = 0
- nestedList = re.findall(r'[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]', jaString)
+ nestedList = re.findall(r"[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]", jaString)
nestedList = set(nestedList)
if len(nestedList) != 0:
for icon in nestedList:
- jaString = jaString.replace(icon, '[Nested_' + str(count) + ']')
+ jaString = jaString.replace(icon, "[Nested_" + str(count) + "]")
count += 1
# Icons
count = 0
- iconList = re.findall(r'[\\]+[iIkKwWaA]+\[[0-9]+\]', jaString)
+ iconList = re.findall(r"[\\]+[iIkKwWaA]+\[[0-9]+\]", jaString)
iconList = set(iconList)
if len(iconList) != 0:
for icon in iconList:
- jaString = jaString.replace(icon, '[Ascii_' + str(count) + ']')
+ jaString = jaString.replace(icon, "[Ascii_" + str(count) + "]")
count += 1
# Colors
count = 0
- colorList = re.findall(r'([\\]+c\[\d+\][\\]+c|[\\]+c\[\d+\])', jaString)
+ colorList = re.findall(r"([\\]+c\[\d+\][\\]+c|[\\]+c\[\d+\])", jaString)
colorList = set(colorList)
if len(colorList) != 0:
for color in colorList:
- jaString = jaString.replace(color, '[Color_' + str(count) + ']')
+ jaString = jaString.replace(color, "[Color_" + str(count) + "]")
count += 1
# Names
count = 0
- nameList = re.findall(r'[\\]+[nN]\[.+?\]+', jaString)
+ nameList = re.findall(r"[\\]+[nN]\[.+?\]+", jaString)
nameList = set(nameList)
if len(nameList) != 0:
for name in nameList:
- jaString = jaString.replace(name, '[Noun_' + str(count) + ']')
+ jaString = jaString.replace(name, "[Noun_" + str(count) + "]")
count += 1
# Variables
count = 0
- varList = re.findall(r'[\\]+[vV]\[[0-9]+\]', jaString)
+ varList = re.findall(r"[\\]+[vV]\[[0-9]+\]", jaString)
varList = set(varList)
if len(varList) != 0:
for var in varList:
- jaString = jaString.replace(var, '[Var_' + str(count) + ']')
+ jaString = jaString.replace(var, "[Var_" + str(count) + "]")
count += 1
# Formatting
count = 0
- formatList = re.findall(r'[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]', jaString)
+ formatList = re.findall(r"[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]", jaString)
formatList = set(formatList)
if len(formatList) != 0:
for var in formatList:
- jaString = jaString.replace(var, '[FCode_' + str(count) + ']')
+ jaString = jaString.replace(var, "[FCode_" + str(count) + "]")
count += 1
# Put all lists in list and return
allList = [nestedList, iconList, colorList, nameList, varList, formatList]
return [jaString, allList]
+
def resubVars(translatedText, allList):
# Fix Spacing and ChatGPT Nonsense
- matchList = re.findall(r'\[\s?.+?\s?\]', translatedText)
+ matchList = re.findall(r"\[\s?.+?\s?\]", translatedText)
if len(matchList) > 0:
for match in matchList:
text = match.strip()
@@ -355,54 +395,58 @@ def resubVars(translatedText, allList):
count = 0
if len(allList[0]) != 0:
for var in allList[0]:
- translatedText = translatedText.replace('[Nested_' + str(count) + ']', var)
+ translatedText = translatedText.replace("[Nested_" + str(count) + "]", var)
count += 1
# Icons
count = 0
if len(allList[1]) != 0:
for var in allList[1]:
- translatedText = translatedText.replace('[Ascii_' + str(count) + ']', var)
+ translatedText = translatedText.replace("[Ascii_" + str(count) + "]", var)
count += 1
# Colors
count = 0
if len(allList[2]) != 0:
for var in allList[2]:
- translatedText = translatedText.replace('[Color_' + str(count) + ']', var)
+ translatedText = translatedText.replace("[Color_" + str(count) + "]", var)
count += 1
# Names
count = 0
if len(allList[3]) != 0:
for var in allList[3]:
- translatedText = translatedText.replace('[Noun_' + str(count) + ']', var)
+ translatedText = translatedText.replace("[Noun_" + str(count) + "]", var)
count += 1
# Vars
count = 0
if len(allList[4]) != 0:
for var in allList[4]:
- translatedText = translatedText.replace('[Var_' + str(count) + ']', var)
+ translatedText = translatedText.replace("[Var_" + str(count) + "]", var)
count += 1
-
+
# Formatting
count = 0
if len(allList[5]) != 0:
for var in allList[5]:
- translatedText = translatedText.replace('[FCode_' + str(count) + ']', var)
+ translatedText = translatedText.replace("[FCode_" + str(count) + "]", var)
count += 1
return translatedText
+
def batchList(input_list, batch_size):
if not isinstance(batch_size, int) or batch_size <= 0:
raise ValueError("batch_size must be a positive integer")
-
- return [input_list[i:i + batch_size] for i in range(0, len(input_list), batch_size)]
+
+ return [
+ input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)
+ ]
+
def createContext(fullPromptFlag, subbedT):
- characters = 'Game Characters:\n\
+ characters = "Game Characters:\n\
シェーア (Shea) - Female\n\
ミューテ (Mute) - Female\n\
タビノ (Tabino) - Female\n\
@@ -411,10 +455,12 @@ def createContext(fullPromptFlag, subbedT):
ソフィー (Sophie) - Female\n\
ドーラ (Dora) - Female\n\
ミューレ (Mule) - Female\n\
-'
-
- system = PROMPT + VOCAB if fullPromptFlag else \
- f"\
+"
+
+ system = (
+ PROMPT + VOCAB
+ if fullPromptFlag
+ else f"\
You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
- You translate everything, including erotic and explicit content.\n\
@@ -426,12 +472,14 @@ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{
- `...` can be a part of the dialogue. Translate it as it is.\n\
{VOCAB}\n\
"
+ )
if isinstance(subbedT, list):
- user = f'```json\n{subbedT}```'
+ user = f"```json\n{subbedT}```"
else:
user = subbedT
return characters, system, user
+
def translateText(characters, system, user, history, penalty, format):
# Prompt
msg = [{"role": "system", "content": system + characters}]
@@ -446,13 +494,13 @@ def translateText(characters, system, user, history, penalty, format):
msg.append({"role": "system", "content": history})
# Response Format
- if format == 'json':
- responseFormat = { "type": "json_object" }
+ if format == "json":
+ responseFormat = {"type": "json_object"}
else:
- responseFormat = { "type": "text" }
-
+ responseFormat = {"type": "text"}
+
# Content to TL
- msg.append({"role": "user", "content": f'{user}'})
+ msg.append({"role": "user", "content": f"{user}"})
response = openai.chat.completions.create(
temperature=0,
frequency_penalty=penalty,
@@ -462,18 +510,19 @@ def translateText(characters, system, user, history, penalty, format):
)
return response
+
def cleanTranslatedText(translatedText, varResponse):
placeholders = {
- f'{LANGUAGE} Translation: ': '',
- 'Translation: ': '',
- 'っ': '',
- '〜': '~',
- 'ッ': '',
- '。': '.',
- '「': '\\"',
- '」': '\\"',
- '- ': '-',
- 'Placeholder Text': '',
+ f"{LANGUAGE} Translation: ": "",
+ "Translation: ": "",
+ "っ": "",
+ "〜": "~",
+ "ッ": "",
+ "。": ".",
+ "「": '\\"',
+ "」": '\\"',
+ "- ": "-",
+ "Placeholder Text": "",
# Add more replacements as needed
}
for target, replacement in placeholders.items():
@@ -484,11 +533,12 @@ def cleanTranslatedText(translatedText, varResponse):
translatedText = resubVars(translatedText, varResponse[1])
return translatedText
+
def elongateCharacters(text):
# Define a pattern to match one character followed by one or more `ー` characters
# Using a positive lookbehind assertion to capture the preceding character
- pattern = r'(?<=(.))ー+'
-
+ pattern = r"(?<=(.))ー+"
+
# Define a replacement function that elongates the captured character
def repl(match):
char = match.group(1) # The character before the ー sequence
@@ -498,6 +548,7 @@ def elongateCharacters(text):
# Use re.sub() to replace the pattern in the text
return re.sub(pattern, repl, text)
+
def extractTranslation(translatedTextList, is_list):
try:
translatedTextList = re.sub(r'\\"+\"([^,\n}])', r'\\"\1', translatedTextList)
@@ -510,15 +561,15 @@ def extractTranslation(translatedTextList, is_list):
return string_list[0]
except Exception as e:
- PBAR.write(f'extractTranslation Error: {e} on String {translatedTextList}')
+ PBAR.write(f"extractTranslation Error: {e} on String {translatedTextList}")
return None
def countTokens(characters, system, user, history):
inputTotalTokens = 0
outputTotalTokens = 0
- enc = tiktoken.encoding_for_model('gpt-4')
-
+ enc = tiktoken.encoding_for_model("gpt-4")
+
# Input
if isinstance(history, list):
for line in history:
@@ -530,26 +581,28 @@ def countTokens(characters, system, user, history):
inputTotalTokens += len(enc.encode(user))
# Output
- outputTotalTokens += round(len(enc.encode(user))*3)
+ outputTotalTokens += round(len(enc.encode(user)) * 3)
return [inputTotalTokens, outputTotalTokens]
+
def combineList(tlist, text):
if isinstance(text, list):
return [t for sublist in tlist for t in sublist]
return tlist[0]
+
@retry(exceptions=Exception, tries=5, delay=5)
def translateGPT(text, history, fullPromptFlag):
global PBAR
-
+
mismatch = False
totalTokens = [0, 0]
if isinstance(text, list):
- format = 'json'
+ format = "json"
tList = batchList(text, BATCHSIZE)
else:
- format = 'text'
+ format = "text"
tList = [text]
for index, tItem in enumerate(tList):
@@ -564,7 +617,7 @@ def translateGPT(text, history, fullPromptFlag):
subbedT = varResponse[0]
# Things to Check before starting translation
- if not re.search(r'[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+', subbedT):
+ if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", subbedT):
if PBAR is not None:
PBAR.update(len(tItem))
continue
@@ -589,9 +642,13 @@ def translateGPT(text, history, fullPromptFlag):
translatedText = cleanTranslatedText(translatedText, varResponse)
if isinstance(tItem, list):
extractedTranslations = extractTranslation(translatedText, True)
- if extractedTranslations == None or len(tItem) != len(extractedTranslations):
+ if extractedTranslations == None or len(tItem) != len(
+ extractedTranslations
+ ):
# Mismatch. Try Again
- response = translateText(characters, system, user, history, 0.05, format)
+ response = translateText(
+ characters, system, user, history, 0.05, format
+ )
translatedText = response.choices[0].message.content
totalTokens[0] += response.usage.prompt_tokens
totalTokens[1] += response.usage.completion_tokens
@@ -600,13 +657,17 @@ def translateGPT(text, history, fullPromptFlag):
translatedText = cleanTranslatedText(translatedText, varResponse)
if isinstance(tItem, list):
extractedTranslations = extractTranslation(translatedText, True)
- if extractedTranslations == None or len(tItem) != len(extractedTranslations):
- mismatch = True # Just here for breakpoint
-
+ if extractedTranslations == None or len(tItem) != len(
+ extractedTranslations
+ ):
+ mismatch = True # Just here for breakpoint
+
# Set if no mismatch
if mismatch == False:
tList[index] = extractedTranslations
- history = extractedTranslations[-10:] # Update history if we have a list
+ history = extractedTranslations[
+ -10:
+ ] # Update history if we have a list
else:
history = text[-10:]
mismatch = False
@@ -617,7 +678,7 @@ def translateGPT(text, history, fullPromptFlag):
PBAR.update(len(tItem))
else:
# Ensure we're passing a single string to extractTranslation
- tList[index] = translatedText
+ tList[index] = translatedText
finalList = combineList(tList, text)
return [finalList, totalTokens]
diff --git a/modules/sakuranbo.py b/modules/sakuranbo.py
index e45c177..b0e4c29 100644
--- a/modules/sakuranbo.py
+++ b/modules/sakuranbo.py
@@ -16,7 +16,7 @@ from tqdm import tqdm
# Open AI
load_dotenv()
if os.getenv("api").replace(" ", "") != "":
- openai.base_url = os.getenv('api')
+ openai.base_url = os.getenv("api")
openai.organization = os.getenv("org")
openai.api_key = os.getenv("key")
@@ -95,8 +95,9 @@ def getResultString(translatedData, translationTime, filename):
# File Print String
totalTokenstring = (
Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
- "[Output: " + str(translatedData[1][1]) + "]"
- "[Cost: ${:,.4f}".format(
+ "[Output: "
+ + str(translatedData[1][1])
+ + "]" "[Cost: ${:,.4f}".format(
(translatedData[1][0] * 0.001 * INPUTAPICOST)
+ (translatedData[1][1] * 0.001 * OUTPUTAPICOST)
)
@@ -189,17 +190,17 @@ def translateTyrano(data, pbar):
if syncIndex > i:
i = syncIndex
- if '[▼]' in data[i]:
- data[i] = data[i].replace('[▼]'.strip(), '[page]\n')
+ if "[▼]" in data[i]:
+ data[i] = data[i].replace("[▼]".strip(), "[page]\n")
# If there isn't any Japanese in the text just skip
if IGNORETLTEXT is True:
- if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+', data[i]):
+ if not re.search(r"[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+", data[i]):
# Keep textHistory list at length maxHistory
- textHistory.append('\"' + data[i] + '\"')
+ textHistory.append('"' + data[i] + '"')
if len(textHistory) > maxHistory:
textHistory.pop(0)
- currentGroup = []
+ currentGroup = []
continue
# Speaker
@@ -215,18 +216,16 @@ def translateTyrano(data, pbar):
speaker = "Narrator"
elif "マコ" in matchList[0]:
speaker = "Mako"
- elif '少年' in matchList[0]:
+ elif "少年" in matchList[0]:
speaker = "Boy"
- elif '友達' in matchList[0]:
+ elif "友達" in matchList[0]:
speaker = "Friend"
- elif '少女' in matchList[0]:
+ elif "少女" in matchList[0]:
speaker = "Girl"
else:
response = translateGPT(
matchList[0],
- "Reply with only the "
- + LANGUAGE
- + " translation of the NPC name",
+ "Reply with only the " + LANGUAGE + " translation of the NPC name",
True,
)
speaker = response[0]
@@ -263,13 +262,17 @@ def translateTyrano(data, pbar):
# Set Data
translatedText = data[i].replace(
- matchList[0], translatedText.replace(" ", "\u00A0")
+ matchList[0], translatedText.replace(" ", "\u00a0")
)
data[i] = translatedText
# Grab Lines
matchList = re.findall(r"^([^\n;@*\{\[].+[^;'{}\[]$)", data[i])
- if len(matchList) > 0 and (re.search(r'^\[(.+)\sstorage=.+\],', data[i-1]) or re.search(r'^\[(.+)\]$', data[i-1]) or re.search(r'^《(.+)》', data[i-1])):
+ if len(matchList) > 0 and (
+ re.search(r"^\[(.+)\sstorage=.+\],", data[i - 1])
+ or re.search(r"^\[(.+)\]$", data[i - 1])
+ or re.search(r"^《(.+)》", data[i - 1])
+ ):
currentGroup.append(matchList[0])
if len(data) > i + 1:
matchList = re.findall(r"^([^\n;@*\{\[].+[^;'{}\[]$)", data[i + 1])
@@ -323,10 +326,10 @@ def translateTyrano(data, pbar):
# Set
if delFlag is True:
- data.insert(i, translatedText.strip() + '\n')
+ data.insert(i, translatedText.strip() + "\n")
delFlag = False
else:
- data[i] = translatedText.strip() + '\n'
+ data[i] = translatedText.strip() + "\n"
# Keep textHistory list at length maxHistory
if len(textHistory) > maxHistory:
@@ -399,10 +402,10 @@ def translateTyrano(data, pbar):
# Set
if delFlag is True:
- data.insert(i, translatedText.strip() + '\n')
+ data.insert(i, translatedText.strip() + "\n")
delFlag = False
else:
- data[i] = translatedText.strip() + '\n'
+ data[i] = translatedText.strip() + "\n"
# Keep textHistory list at length maxHistory
if len(textHistory) > maxHistory:
@@ -418,6 +421,7 @@ def translateTyrano(data, pbar):
return tokens
+
def subVars(jaString):
jaString = jaString.replace("\u3000", " ")
@@ -551,7 +555,7 @@ def translateGPT(t, history, fullPromptFlag):
# If ESTIMATE is True just count this as an execution and return.
if ESTIMATE:
- enc = tiktoken.encoding_for_model('gpt-4')
+ enc = tiktoken.encoding_for_model("gpt-4")
historyRaw = ""
if isinstance(history, list):
for line in history:
diff --git a/modules/tyrano.py b/modules/tyrano.py
index b63a5d6..f52814f 100644
--- a/modules/tyrano.py
+++ b/modules/tyrano.py
@@ -15,35 +15,35 @@ from tqdm import tqdm
# Open AI
load_dotenv()
-if os.getenv('api').replace(' ', '') != '':
- openai.base_url = os.getenv('api')
-openai.organization = os.getenv('org')
-openai.api_key = os.getenv('key')
+if os.getenv("api").replace(" ", "") != "":
+ openai.base_url = os.getenv("api")
+openai.organization = os.getenv("org")
+openai.api_key = os.getenv("key")
-#Globals
+# Globals
PBAR = None
-MODEL = os.getenv('model')
-TIMEOUT = int(os.getenv('timeout'))
-LANGUAGE = os.getenv('language').capitalize()
-PROMPT = Path('prompt.txt').read_text(encoding='utf-8')
-VOCAB = Path('vocab.txt').read_text(encoding='utf-8')
-THREADS = int(os.getenv('threads'))
+MODEL = os.getenv("model")
+TIMEOUT = int(os.getenv("timeout"))
+LANGUAGE = os.getenv("language").capitalize()
+PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
+VOCAB = Path("vocab.txt").read_text(encoding="utf-8")
+THREADS = int(os.getenv("threads"))
LOCK = threading.Lock()
-WIDTH = int(os.getenv('width'))
-LISTWIDTH = int(os.getenv('listWidth'))
+WIDTH = int(os.getenv("width"))
+LISTWIDTH = int(os.getenv("listWidth"))
NOTEWIDTH = 70
MAXHISTORY = 10
-ESTIMATE = ''
+ESTIMATE = ""
TOKENS = [0, 0]
NAMESLIST = []
-NAMES = False # Output a list of all the character names found
-BRFLAG = False # If the game uses
instead
+NAMES = False # Output a list of all the character names found
+BRFLAG = False # If the game uses
instead
FIXTEXTWRAP = False # Overwrites textwrap
-IGNORETLTEXT = False # Ignores all translated text.
-MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
+IGNORETLTEXT = False # Ignores all translated text.
+MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
-#tqdm Globals
-BAR_FORMAT='{l_bar}{bar:10}{r_bar}{bar:-10b}'
+# tqdm Globals
+BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
POSITION = 0
LEAVE = False
@@ -54,15 +54,16 @@ TEXTWRAPCHOICES = True
# Pricing - Depends on the model https://openai.com/pricing
# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request
# If you are getting a MISMATCH LENGTH error, lower the batch size.
-if 'gpt-3.5' in MODEL:
- INPUTAPICOST = .002
- OUTPUTAPICOST = .002
+if "gpt-3.5" in MODEL:
+ INPUTAPICOST = 0.002
+ OUTPUTAPICOST = 0.002
BATCHSIZE = 10
-elif 'gpt-4' in MODEL:
- INPUTAPICOST = .005
- OUTPUTAPICOST = .015
+elif "gpt-4" in MODEL:
+ INPUTAPICOST = 0.005
+ OUTPUTAPICOST = 0.015
BATCHSIZE = 40
+
def handleTyrano(filename, estimate):
global ESTIMATE
ESTIMATE = estimate
@@ -79,17 +80,21 @@ def handleTyrano(filename, estimate):
TOKENS[1] += translatedData[1][1]
# Print Total
- totalString = getResultString(['', TOKENS, None], end - start, 'TOTAL')
+ totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
# Print any errors on maps
if len(MISMATCH) > 0:
- return totalString + Fore.RED + f'\nMismatch Errors: {MISMATCH}' + Fore.RESET
+ return (
+ totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
+ )
else:
return totalString
-
+
else:
try:
- with open('translated/' + filename, 'w', encoding='utf8', errors='ignore') as outFile:
+ with open(
+ "translated/" + filename, "w", encoding="utf8", errors="ignore"
+ ) as outFile:
start = time.time()
translatedData = openFiles(filename)
@@ -102,23 +107,36 @@ def handleTyrano(filename, estimate):
TOKENS[1] += translatedData[1][1]
except Exception:
traceback.print_exc()
- return 'Fail'
+ return "Fail"
+
+ return getResultString(["", TOKENS, None], end - start, "TOTAL")
- return getResultString(['', TOKENS, None], end - start, 'TOTAL')
def getResultString(translatedData, translationTime, filename):
# File Print String
- totalTokenstring =\
- Fore.YELLOW +\
- '[Input: ' + str(translatedData[1][0]) + ']'\
- '[Output: ' + str(translatedData[1][1]) + ']'\
- '[Cost: ${:,.4f}'.format((translatedData[1][0] * .001 * INPUTAPICOST) +\
- (translatedData[1][1] * .001 * OUTPUTAPICOST)) + ']'
- timeString = Fore.BLUE + '[' + str(round(translationTime, 1)) + 's]'
+ totalTokenstring = (
+ Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
+ "[Output: "
+ + str(translatedData[1][1])
+ + "]" "[Cost: ${:,.4f}".format(
+ (translatedData[1][0] * 0.001 * INPUTAPICOST)
+ + (translatedData[1][1] * 0.001 * OUTPUTAPICOST)
+ )
+ + "]"
+ )
+ timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
if translatedData[2] == None:
# Success
- return filename + ': ' + totalTokenstring + timeString + Fore.GREEN + u' \u2713 ' + Fore.RESET
+ return (
+ filename
+ + ": "
+ + totalTokenstring
+ + timeString
+ + Fore.GREEN
+ + " \u2713 "
+ + Fore.RESET
+ )
else:
# Fail
@@ -127,34 +145,46 @@ def getResultString(translatedData, translationTime, filename):
except Exception as e:
traceback.print_exc()
errorString = str(e) + Fore.RED
- return filename + ': ' + totalTokenstring + timeString + Fore.RED + u' \u2717 ' +\
- errorString + Fore.RESET
+ return (
+ filename
+ + ": "
+ + totalTokenstring
+ + timeString
+ + Fore.RED
+ + " \u2717 "
+ + errorString
+ + Fore.RESET
+ )
+
def openFiles(filename):
- with open('files/' + filename, 'r', encoding='utf8') as readFile:
+ with open("files/" + filename, "r", encoding="utf8") as readFile:
translatedData = parseTyrano(readFile, filename)
# Delete lines marked for deletion
finalData = []
for line in translatedData[0]:
- if line != '\\d\n':
+ if line != "\\d\n":
finalData.append(line)
translatedData[0] = finalData
-
+
return translatedData
+
def parseTyrano(readFile, filename):
- totalTokens = [0,0]
+ totalTokens = [0, 0]
totalLines = 0
# Get total for progress bar
data = readFile.readlines()
- with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar:
- pbar.desc=filename
+ with tqdm(
+ bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE
+ ) as pbar:
+ pbar.desc = filename
try:
- result = translateTyrano(data, pbar, filename, False, [[],[]])
+ result = translateTyrano(data, pbar, filename, False, [[], []])
totalTokens[0] += result[0]
totalTokens[1] += result[1]
except Exception as e:
@@ -162,58 +192,63 @@ def parseTyrano(readFile, filename):
return [data, totalTokens, e]
return [data, totalTokens, None]
+
def translateTyrano(data, pbar, filename, setData, jobList):
textHistory = []
lineList = jobList[0]
- totalTokens = [0,0]
- speaker = ''
+ totalTokens = [0, 0]
+ speaker = ""
global LOCK, ESTIMATE
i = 0
# Set Progress Bar
- global PBAR
+ global PBAR
PBAR = pbar
while i < len(data):
# Choices
choiceList = []
- choiceRegex = r'[sS]tatus.+?\](.+)'
- if 'tatus' in data[i]:
+ choiceRegex = r"[sS]tatus.+?\](.+)"
+ if "tatus" in data[i]:
match = re.search(choiceRegex, data[i])
if match != None:
jaString = match.group(1)
# Remove Textwrap
if TEXTWRAPCHOICES is True:
- jaString = jaString.replace('[r]', ' ')
- data[i] = data[i].replace('[r]', ' ')
-
+ jaString = jaString.replace("[r]", " ")
+ data[i] = data[i].replace("[r]", " ")
+
# Add to list
choiceList.append(jaString)
i += 1
# Grab them all up for list
- while(i < len(data) and 'tatus' in data[i]):
+ while i < len(data) and "tatus" in data[i]:
match = re.search(choiceRegex, data[i])
if match != None:
jaString = match.group(1)
# Remove Textwrap
if TEXTWRAPCHOICES is True:
- jaString = jaString.replace('[r]', ' ')
- data[i] = data[i].replace('[r]', ' ')
+ jaString = jaString.replace("[r]", " ")
+ data[i] = data[i].replace("[r]", " ")
# Add to list
choiceList.append(jaString)
i += 1
-
+
# Translate
if len(choiceList) != 0:
- response = translateGPT(choiceList, 'Reply with the {LANGUAGE} translation of the text', True)
+ response = translateGPT(
+ choiceList,
+ "Reply with the {LANGUAGE} translation of the text",
+ True,
+ )
choiceListTL = response[0]
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
-
+
# Set Data
if len(choiceList) == len(choiceListTL):
i = i - len(choiceListTL)
@@ -223,75 +258,83 @@ def translateTyrano(data, pbar, filename, setData, jobList):
# Textwrap
if TEXTWRAPCHOICES is True:
translatedText = textwrap.fill(translatedText, WIDTH)
- translatedText = translatedText.replace('\n', '[r]')
+ translatedText = translatedText.replace("\n", "[r]")
data[i] = data[i].replace(choiceList[j], translatedText)
i += 1
else:
with LOCK:
if filename not in MISMATCH:
MISMATCH.append(filename)
-
+
if DIALOGUEFLAG is True:
# Speaker
- if '[@]' in data[i]:
- if 'FACE' not in data[i]:
- matchList = re.findall(r'\[(.*?)\].+\[.*\]', data[i])
+ if "[@]" in data[i]:
+ if "FACE" not in data[i]:
+ matchList = re.findall(r"\[(.*?)\].+\[.*\]", data[i])
else:
- matchList = re.findall(r'face=.+?\]\[(.+?)\]', data[i])
- if len(matchList) != 0 and '=' not in matchList[0] and re.search(r'\[.+\]', matchList[0]) == None:
+ matchList = re.findall(r"face=.+?\]\[(.+?)\]", data[i])
+ if (
+ len(matchList) != 0
+ and "=" not in matchList[0]
+ and re.search(r"\[.+\]", matchList[0]) == None
+ ):
response = getSpeaker(matchList[0])
speaker = response[0]
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
# data[i] = data[i].replace(matchList[0], f'{speaker}')
else:
- speaker = ''
-
+ speaker = ""
+
# Lines
- if 'FACE' not in data[i]:
- matchList = re.findall(r'\[.+?\](.+)\[.+\]', data[i])
+ if "FACE" not in data[i]:
+ matchList = re.findall(r"\[.+?\](.+)\[.+\]", data[i])
else:
- matchList = re.findall(r'face=.+?\]\[.+?\](.+)\[.+\]', data[i])
- if len(matchList) > 0 and '=' not in matchList[0]:
+ matchList = re.findall(r"face=.+?\]\[.+?\](.+)\[.+\]", data[i])
+ if len(matchList) > 0 and "=" not in matchList[0]:
# No Japanese text
- if not re.search(r'[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+', matchList[0]):
+ if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", matchList[0]):
i += 1
continue
# Remove [r] and [l]
oldjaString = matchList[0]
jaString = oldjaString
- jaString = jaString.replace('[r]', ' ')
- jaString = jaString.replace('[l]', '')
+ jaString = jaString.replace("[r]", " ")
+ jaString = jaString.replace("[l]", "")
# Join up 401 groups for better translation.
finalJAString = jaString
# Remove Extra Stuff bad for translation.
- finalJAString = finalJAString.replace('ﾞ', '')
- finalJAString = finalJAString.replace('・', '.')
- finalJAString = finalJAString.replace('‶', '')
- finalJAString = finalJAString.replace('”', '')
- finalJAString = finalJAString.replace('―', '-')
- finalJAString = finalJAString.replace('…', '...')
- finalJAString = re.sub(r'(\.{3}\.+)', '...', finalJAString)
- finalJAString = finalJAString.replace(' ', ' ')
- finalJAString = finalJAString.replace('】', ')')
- finalJAString = finalJAString.replace('【 ', '(')
+ finalJAString = finalJAString.replace("ﾞ", "")
+ finalJAString = finalJAString.replace("・", ".")
+ finalJAString = finalJAString.replace("‶", "")
+ finalJAString = finalJAString.replace("”", "")
+ finalJAString = finalJAString.replace("―", "-")
+ finalJAString = finalJAString.replace("…", "...")
+ finalJAString = re.sub(r"(\.{3}\.+)", "...", finalJAString)
+ finalJAString = finalJAString.replace(" ", " ")
+ finalJAString = finalJAString.replace("】", ")")
+ finalJAString = finalJAString.replace("【 ", "(")
# Furigana Removal
- matchList = re.findall(r'(\[ruby\stext=.+text=\"(.+)\"\])', finalJAString)
+ matchList = re.findall(
+ r"(\[ruby\stext=.+text=\"(.+)\"\])", finalJAString
+ )
if len(matchList) > 0:
- finalJAString = finalJAString.replace(matchList[0][0], matchList[0][1])
+ finalJAString = finalJAString.replace(
+ matchList[0][0], matchList[0][1]
+ )
# Add Speaker (If there is one)
- if speaker != '':
- finalJAString = f'{speaker}: {finalJAString}'
+ if speaker != "":
+ finalJAString = f"{speaker}: {finalJAString}"
# [Passthrough 1] Append To List
if setData is False:
lineList.append(finalJAString)
-
+
# [Passthrough 2] Set Data
else:
# Grab and Pop
@@ -299,22 +342,24 @@ def translateTyrano(data, pbar, filename, setData, jobList):
lineList.pop(0)
# Remove speaker
- translatedText = re.sub(r'^\[?(.+?)\]?\s?[|:]\s?', '', translatedText)
+ translatedText = re.sub(
+ r"^\[?(.+?)\]?\s?[|:]\s?", "", translatedText
+ )
# Textwrap
translatedText = textwrap.fill(translatedText, WIDTH)
- translatedText = translatedText.replace('\n', '[r]')
+ translatedText = translatedText.replace("\n", "[r]")
# Set Data
data[i] = data[i].replace(oldjaString, translatedText)
# Next Line
i += 1
-
+
# Translate Data
lineListTL = []
setData = False
-
+
# Line List
if len(lineList) > 0:
pbar.total = len(lineList)
@@ -335,17 +380,23 @@ def translateTyrano(data, pbar, filename, setData, jobList):
translateTyrano(data, pbar, filename, True, [lineListTL])
return totalTokens
+
+
# Save some money and enter the character before translation
def getSpeaker(speaker):
match speaker:
- case 'ファイン':
- return ['Fine', [0,0]]
- case '':
- return ['', [0,0]]
+ case "ファイン":
+ return ["Fine", [0, 0]]
+ case "":
+ return ["", [0, 0]]
case _:
# Store Speaker
if speaker not in str(NAMESLIST):
- response = translateGPT(speaker, 'Reply with only the '+ LANGUAGE +' translation of the NPC name.', False)
+ response = translateGPT(
+ speaker,
+ "Reply with only the " + LANGUAGE + " translation of the NPC name.",
+ False,
+ )
response[0] = response[0].title()
speakerList = [speaker, response[0]]
NAMESLIST.append(speakerList)
@@ -354,74 +405,76 @@ def getSpeaker(speaker):
else:
for i in range(len(NAMESLIST)):
if speaker == NAMESLIST[i][0]:
- return [NAMESLIST[i][1],[0,0]]
-
- return [speaker,[0,0]]
+ return [NAMESLIST[i][1], [0, 0]]
+
+ return [speaker, [0, 0]]
+
def subVars(jaString):
- jaString = jaString.replace('\u3000', ' ')
+ jaString = jaString.replace("\u3000", " ")
# Nested
count = 0
- nestedList = re.findall(r'[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]', jaString)
+ nestedList = re.findall(r"[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]", jaString)
nestedList = set(nestedList)
if len(nestedList) != 0:
for icon in nestedList:
- jaString = jaString.replace(icon, '[Nested_' + str(count) + ']')
+ jaString = jaString.replace(icon, "[Nested_" + str(count) + "]")
count += 1
# Icons
count = 0
- iconList = re.findall(r'[\\]+[iIkKwWaA]+\[[0-9]+\]', jaString)
+ iconList = re.findall(r"[\\]+[iIkKwWaA]+\[[0-9]+\]", jaString)
iconList = set(iconList)
if len(iconList) != 0:
for icon in iconList:
- jaString = jaString.replace(icon, '[Ascii_' + str(count) + ']')
+ jaString = jaString.replace(icon, "[Ascii_" + str(count) + "]")
count += 1
# Colors
count = 0
- colorList = re.findall(r'[\\]+[cC]\[[0-9]+\]', jaString)
+ colorList = re.findall(r"[\\]+[cC]\[[0-9]+\]", jaString)
colorList = set(colorList)
if len(colorList) != 0:
for color in colorList:
- jaString = jaString.replace(color, '[Color_' + str(count) + ']')
+ jaString = jaString.replace(color, "[Color_" + str(count) + "]")
count += 1
# Names
count = 0
- nameList = re.findall(r'[\\]+[nN]\[.+?\]+', jaString)
+ nameList = re.findall(r"[\\]+[nN]\[.+?\]+", jaString)
nameList = set(nameList)
if len(nameList) != 0:
for name in nameList:
- jaString = jaString.replace(name, '[Noun_' + str(count) + ']')
+ jaString = jaString.replace(name, "[Noun_" + str(count) + "]")
count += 1
# Variables
count = 0
- varList = re.findall(r'[\\]+[vV]\[[0-9]+\]', jaString)
+ varList = re.findall(r"[\\]+[vV]\[[0-9]+\]", jaString)
varList = set(varList)
if len(varList) != 0:
for var in varList:
- jaString = jaString.replace(var, '[Var_' + str(count) + ']')
+ jaString = jaString.replace(var, "[Var_" + str(count) + "]")
count += 1
# Formatting
count = 0
- formatList = re.findall(r'[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]', jaString)
+ formatList = re.findall(r"[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]", jaString)
formatList = set(formatList)
if len(formatList) != 0:
for var in formatList:
- jaString = jaString.replace(var, '[FCode_' + str(count) + ']')
+ jaString = jaString.replace(var, "[FCode_" + str(count) + "]")
count += 1
# Put all lists in list and return
allList = [nestedList, iconList, colorList, nameList, varList, formatList]
return [jaString, allList]
+
def resubVars(translatedText, allList):
# Fix Spacing and ChatGPT Nonsense
- matchList = re.findall(r'\[\s?.+?\s?\]', translatedText)
+ matchList = re.findall(r"\[\s?.+?\s?\]", translatedText)
if len(matchList) > 0:
for match in matchList:
text = match.strip()
@@ -431,60 +484,66 @@ def resubVars(translatedText, allList):
count = 0
if len(allList[0]) != 0:
for var in allList[0]:
- translatedText = translatedText.replace('[Nested_' + str(count) + ']', var)
+ translatedText = translatedText.replace("[Nested_" + str(count) + "]", var)
count += 1
# Icons
count = 0
if len(allList[1]) != 0:
for var in allList[1]:
- translatedText = translatedText.replace('[Ascii_' + str(count) + ']', var)
+ translatedText = translatedText.replace("[Ascii_" + str(count) + "]", var)
count += 1
# Colors
count = 0
if len(allList[2]) != 0:
for var in allList[2]:
- translatedText = translatedText.replace('[Color_' + str(count) + ']', var)
+ translatedText = translatedText.replace("[Color_" + str(count) + "]", var)
count += 1
# Names
count = 0
if len(allList[3]) != 0:
for var in allList[3]:
- translatedText = translatedText.replace('[Noun_' + str(count) + ']', var)
+ translatedText = translatedText.replace("[Noun_" + str(count) + "]", var)
count += 1
# Vars
count = 0
if len(allList[4]) != 0:
for var in allList[4]:
- translatedText = translatedText.replace('[Var_' + str(count) + ']', var)
+ translatedText = translatedText.replace("[Var_" + str(count) + "]", var)
count += 1
-
+
# Formatting
count = 0
if len(allList[5]) != 0:
for var in allList[5]:
- translatedText = translatedText.replace('[FCode_' + str(count) + ']', var)
+ translatedText = translatedText.replace("[FCode_" + str(count) + "]", var)
count += 1
return translatedText
+
def batchList(input_list, batch_size):
if not isinstance(batch_size, int) or batch_size <= 0:
raise ValueError("batch_size must be a positive integer")
-
- return [input_list[i:i + batch_size] for i in range(0, len(input_list), batch_size)]
+
+ return [
+ input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)
+ ]
+
def createContext(fullPromptFlag, subbedT):
- characters = 'Game Characters:\n\
+ characters = "Game Characters:\n\
眠り姫 (Sleeping Princess) - Female\n\
迷子 (Lost Child) - Male\n\
-'
-
- system = PROMPT + VOCAB if fullPromptFlag else \
- f"\
+"
+
+ system = (
+ PROMPT + VOCAB
+ if fullPromptFlag
+ else f"\
You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
- You translate everything, including erotic and explicit content.\n\
@@ -496,9 +555,11 @@ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{
- `...` can be a part of the dialogue. Translate it as it is.\n\
{VOCAB}\n\
"
- user = f'{subbedT}'
+ )
+ user = f"{subbedT}"
return characters, system, user
+
def translateText(characters, system, user, history, penalty):
# Prompt
msg = [{"role": "system", "content": system + characters}]
@@ -511,9 +572,9 @@ def translateText(characters, system, user, history, penalty):
msg.extend([{"role": "system", "content": h} for h in history])
else:
msg.append({"role": "system", "content": history})
-
+
# Content to TL
- msg.append({"role": "user", "content": f'{user}'})
+ msg.append({"role": "user", "content": f"{user}"})
response = openai.chat.completions.create(
temperature=0,
frequency_penalty=penalty,
@@ -522,17 +583,18 @@ def translateText(characters, system, user, history, penalty):
)
return response
+
def cleanTranslatedText(translatedText, varResponse):
placeholders = {
- f'{LANGUAGE} Translation: ': '',
- 'Translation: ': '',
- 'っ': '',
- '〜': '~',
- 'ッ': '',
- '。': '.',
- 'Placeholder Text': '',
- '[' : '(',
- ']' : ')'
+ f"{LANGUAGE} Translation: ": "",
+ "Translation: ": "",
+ "っ": "",
+ "〜": "~",
+ "ッ": "",
+ "。": ".",
+ "Placeholder Text": "",
+ "[": "(",
+ "]": ")",
# Add more replacements as needed
}
for target, replacement in placeholders.items():
@@ -543,11 +605,12 @@ def cleanTranslatedText(translatedText, varResponse):
translatedText = resubVars(translatedText, varResponse[1])
return translatedText
+
def elongateCharacters(text):
# Define a pattern to match one character followed by one or more `ー` characters
# Using a positive lookbehind assertion to capture the preceding character
- pattern = r'(?<=(.))ー+'
-
+ pattern = r"(?<=(.))ー+"
+
# Define a replacement function that elongates the captured character
def repl(match):
char = match.group(1) # The character before the ー sequence
@@ -557,8 +620,9 @@ def elongateCharacters(text):
# Use re.sub() to replace the pattern in the text
return re.sub(pattern, repl, text)
+
def extractTranslation(translatedTextList, is_list):
- pattern = r'`?<[Ll]ine\d+>([\\]*.*?[\\]*?)<\/?[Ll]ine\d+>`?'
+ pattern = r"`?<[Ll]ine\d+>([\\]*.*?[\\]*?)<\/?[Ll]ine\d+>`?"
# If it's a batch (i.e., list), extract with tags; otherwise, return the single item.
if is_list:
matchList = re.findall(pattern, translatedTextList)
@@ -567,11 +631,12 @@ def extractTranslation(translatedTextList, is_list):
matchList = re.findall(pattern, translatedTextList)
return matchList[0][0] if matchList else translatedTextList
+
def countTokens(characters, system, user, history):
inputTotalTokens = 0
outputTotalTokens = 0
- enc = tiktoken.encoding_for_model('gpt-4')
-
+ enc = tiktoken.encoding_for_model("gpt-4")
+
# Input
if isinstance(history, list):
for line in history:
@@ -583,15 +648,17 @@ def countTokens(characters, system, user, history):
inputTotalTokens += len(enc.encode(user))
# Output
- outputTotalTokens += round(len(enc.encode(user))*3)
+ outputTotalTokens += round(len(enc.encode(user)) * 3)
return [inputTotalTokens, outputTotalTokens]
+
def combineList(tlist, text):
if isinstance(text, list):
return [t for sublist in tlist for t in sublist]
return tlist[0]
+
@retry(exceptions=Exception, tries=5, delay=5)
def translateGPT(text, history, fullPromptFlag):
global PBAR
@@ -605,8 +672,12 @@ def translateGPT(text, history, fullPromptFlag):
for index, tItem in enumerate(tList):
# Before sending to translation, if we have a list of items, add the formatting
if isinstance(tItem, list):
- payload = '\n'.join([f'`{item}`' for i, item in enumerate(tItem)])
- payload = re.sub(r'(<)(\/Line\d+>)', r'\1>Placeholder Text<\3', payload)
+ payload = "\n".join(
+ [f"`{item}`" for i, item in enumerate(tItem)]
+ )
+ payload = re.sub(
+ r"(<)(\/Line\d+>)", r"\1>Placeholder Text<\3", payload
+ )
varResponse = subVars(payload)
subbedT = varResponse[0]
else:
@@ -614,7 +685,7 @@ def translateGPT(text, history, fullPromptFlag):
subbedT = varResponse[0]
# Things to Check before starting translation
- if not re.search(r'[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+', subbedT):
+ if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", subbedT):
continue
# Create Message
@@ -651,11 +722,13 @@ def translateGPT(text, history, fullPromptFlag):
extractedTranslations = extractTranslation(translatedText, True)
tList[index] = extractedTranslations
if len(tItem) != len(extractedTranslations):
- mismatch = True # Just here for breakpoint
+ mismatch = True # Just here for breakpoint
# Create History
if not mismatch:
- history = extractedTranslations[-10:] # Update history if we have a list
+ history = extractedTranslations[
+ -10:
+ ] # Update history if we have a list
else:
history = text[-10:]
PBAR.update(len(tItem))
diff --git a/modules/wolf.py b/modules/wolf.py
index f302d8b..159b6e4 100644
--- a/modules/wolf.py
+++ b/modules/wolf.py
@@ -17,51 +17,51 @@ from tqdm import tqdm
# Open AI
load_dotenv()
-if os.getenv('api').replace(' ', '') != '':
- openai.base_url = os.getenv('api')
-openai.organization = os.getenv('org')
-openai.api_key = os.getenv('key')
+if os.getenv("api").replace(" ", "") != "":
+ openai.base_url = os.getenv("api")
+openai.organization = os.getenv("org")
+openai.api_key = os.getenv("key")
-#Globals
-MODEL = os.getenv('model')
-TIMEOUT = int(os.getenv('timeout'))
-LANGUAGE = os.getenv('language').capitalize()
-PROMPT = Path('prompt.txt').read_text(encoding='utf-8')
-VOCAB = Path('vocab.txt').read_text(encoding='utf-8')
-THREADS = int(os.getenv('threads'))
+# Globals
+MODEL = os.getenv("model")
+TIMEOUT = int(os.getenv("timeout"))
+LANGUAGE = os.getenv("language").capitalize()
+PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
+VOCAB = Path("vocab.txt").read_text(encoding="utf-8")
+THREADS = int(os.getenv("threads"))
LOCK = threading.Lock()
-WIDTH = int(os.getenv('width'))
-LISTWIDTH = int(os.getenv('listWidth'))
-NOTEWIDTH = int(os.getenv('noteWidth'))
+WIDTH = int(os.getenv("width"))
+LISTWIDTH = int(os.getenv("listWidth"))
+NOTEWIDTH = int(os.getenv("noteWidth"))
MAXHISTORY = 10
-ESTIMATE = ''
+ESTIMATE = ""
TOKENS = [0, 0]
-NAMESLIST = [] # Keep list for consistency
-TERMSLIST = [] # Keep list for consistency
-NAMES = False # Output a list of all the character names found
-BRFLAG = False # If the game uses
instead
+NAMESLIST = [] # Keep list for consistency
+TERMSLIST = [] # Keep list for consistency
+NAMES = False # Output a list of all the character names found
+BRFLAG = False # If the game uses
instead
FIXTEXTWRAP = True # Overwrites textwrap
-IGNORETLTEXT = False # Ignores all translated text.
-MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
+IGNORETLTEXT = False # Ignores all translated text.
+MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
FILENAME = None
BRACKETNAMES = False
# Pricing - Depends on the model https://openai.com/pricing
# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request
# If you are getting a MISMATCH LENGTH error, lower the batch size.
-if 'gpt-3.5' in MODEL:
- INPUTAPICOST = .002
- OUTPUTAPICOST = .002
+if "gpt-3.5" in MODEL:
+ INPUTAPICOST = 0.002
+ OUTPUTAPICOST = 0.002
BATCHSIZE = 10
FREQUENCY_PENALTY = 0.2
-elif 'gpt-4' in MODEL:
- INPUTAPICOST = .005
- OUTPUTAPICOST = .015
+elif "gpt-4" in MODEL:
+ INPUTAPICOST = 0.005
+ OUTPUTAPICOST = 0.015
BATCHSIZE = 20
FREQUENCY_PENALTY = 0.1
-#tqdm Globals
-BAR_FORMAT='{l_bar}{bar:10}{r_bar}{bar:-10b}'
+# tqdm Globals
+BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
POSITION = 0
LEAVE = False
PBAR = None
@@ -86,6 +86,7 @@ ARMORFLAG = False
ENEMYFLAG = False
WEAPONFLAG = False
+
def handleWOLF(filename, estimate):
global ESTIMATE, TOKENS, FILENAME
ESTIMATE = estimate
@@ -94,16 +95,16 @@ def handleWOLF(filename, estimate):
# Translate
start = time.time()
translatedData = openFiles(filename)
-
+
# Translate
if not estimate:
try:
- with open('translated/' + filename, 'w', encoding='utf-8') as outFile:
+ with open("translated/" + filename, "w", encoding="utf-8") as outFile:
json.dump(translatedData[0], outFile, ensure_ascii=False, indent=4)
except Exception:
traceback.print_exc()
- return 'Fail'
-
+ return "Fail"
+
# Print File
end = time.time()
tqdm.write(getResultString(translatedData, end - start, filename))
@@ -112,24 +113,25 @@ def handleWOLF(filename, estimate):
TOKENS[1] += translatedData[1][1]
# Print Total
- totalString = getResultString(['', TOKENS, None], end - start, 'TOTAL')
+ totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
# Print any errors on maps
if len(MISMATCH) > 0:
- return totalString + Fore.RED + f'\nMismatch Errors: {MISMATCH}' + Fore.RESET
+ return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
else:
return totalString
+
def openFiles(filename):
- with open('files/' + filename, 'r', encoding='utf-8-sig') as f:
+ with open("files/" + filename, "r", encoding="utf-8-sig") as f:
data = json.load(f)
# Map Files
if "'events':" in str(data):
- if len(data['events']) > 0:
+ if len(data["events"]) > 0:
translatedData = parseMap(data, filename)
else:
- return [data, [0,0], None]
+ return [data, [0, 0], None]
# Map Files
elif "'types':" in str(data):
@@ -138,25 +140,38 @@ def openFiles(filename):
# Other Files
elif "'commands':" in str(data):
translatedData = parseOther(data, filename)
-
+
else:
- raise NameError(filename + ' Not Supported')
-
+ raise NameError(filename + " Not Supported")
+
return translatedData
+
def getResultString(translatedData, translationTime, filename):
# File Print String
- totalTokenstring =\
- Fore.YELLOW +\
- '[Input: ' + str(translatedData[1][0]) + ']'\
- '[Output: ' + str(translatedData[1][1]) + ']'\
- '[Cost: ${:,.4f}'.format((translatedData[1][0] * .001 * INPUTAPICOST) +\
- (translatedData[1][1] * .001 * OUTPUTAPICOST)) + ']'
- timeString = Fore.BLUE + '[' + str(round(translationTime, 1)) + 's]'
+ totalTokenstring = (
+ Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
+ "[Output: "
+ + str(translatedData[1][1])
+ + "]" "[Cost: ${:,.4f}".format(
+ (translatedData[1][0] * 0.001 * INPUTAPICOST)
+ + (translatedData[1][1] * 0.001 * OUTPUTAPICOST)
+ )
+ + "]"
+ )
+ timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
if translatedData[2] is None:
# Success
- return filename + ': ' + totalTokenstring + timeString + Fore.GREEN + u' \u2713 ' + Fore.RESET
+ return (
+ filename
+ + ": "
+ + totalTokenstring
+ + timeString
+ + Fore.GREEN
+ + " \u2713 "
+ + Fore.RESET
+ )
else:
# Fail
try:
@@ -164,19 +179,28 @@ def getResultString(translatedData, translationTime, filename):
except Exception as e:
traceback.print_exc()
errorString = str(e) + Fore.RED
- return filename + ': ' + totalTokenstring + timeString + Fore.RED + u' \u2717 ' +\
- errorString + Fore.RESET
+ return (
+ filename
+ + ": "
+ + totalTokenstring
+ + timeString
+ + Fore.RED
+ + " \u2717 "
+ + errorString
+ + Fore.RESET
+ )
+
def parseOther(data, filename):
totalTokens = [0, 0]
totalLines = 0
- events = data['commands']
+ events = data["commands"]
global LOCK
-
+
# Thread for each page in file
with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
- pbar.desc=filename
- pbar.total=totalLines
+ pbar.desc = filename
+ pbar.total = totalLines
translationData = searchCodes(events, pbar, [], filename)
try:
totalTokens[0] += translationData[0]
@@ -185,16 +209,17 @@ def parseOther(data, filename):
return [data, totalTokens, e]
return [data, totalTokens, None]
+
def parseDB(data, filename):
totalTokens = [0, 0]
totalLines = 0
- events = data['types']
+ events = data["types"]
global LOCK
-
+
# Thread for each page in file
with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
- pbar.desc=filename
- pbar.total=totalLines
+ pbar.desc = filename
+ pbar.total = totalLines
translationData = searchDB(events, pbar, [], filename)
try:
totalTokens[0] += translationData[0]
@@ -203,26 +228,33 @@ def parseDB(data, filename):
return [data, totalTokens, e]
return [data, totalTokens, None]
+
def parseMap(data, filename):
totalTokens = [0, 0]
totalLines = 0
- events = data['events']
+ events = data["events"]
global LOCK
# Get total for progress bar
for event in events:
if event is not None:
- for page in event['pages']:
- totalLines += len(page['list'])
-
+ for page in event["pages"]:
+ totalLines += len(page["list"])
+
# Thread for each page in file
- with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar:
- pbar.desc=filename
- pbar.total=totalLines
+ with tqdm(
+ bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE
+ ) as pbar:
+ pbar.desc = filename
+ pbar.total = totalLines
with ThreadPoolExecutor(max_workers=THREADS) as executor:
for event in events:
if event is not None:
- futures = [executor.submit(searchCodes, page['list'], pbar, None, filename) for page in event['pages'] if page is not None]
+ futures = [
+ executor.submit(searchCodes, page["list"], pbar, None, filename)
+ for page in event["pages"]
+ if page is not None
+ ]
for future in as_completed(futures):
try:
totalTokensFuture = future.result()
@@ -232,8 +264,9 @@ def parseMap(data, filename):
return [data, totalTokens, e]
return [data, totalTokens, None]
+
def searchCodes(events, pbar, jobList, filename):
- #Lists
+ # Lists
if jobList:
stringList = jobList[0]
list300 = jobList[1]
@@ -247,79 +280,78 @@ def searchCodes(events, pbar, jobList, filename):
codeList = events
textHistory = []
totalTokens = [0, 0]
- translatedText = ''
- speaker = ''
- nametag = ''
- initialJAString = ''
- global LOCK, NAMESLIST, MISMATCH, PBAR , FILENAME
+ translatedText = ""
+ speaker = ""
+ nametag = ""
+ initialJAString = ""
+ global LOCK, NAMESLIST, MISMATCH, PBAR, FILENAME
FILENAME = filename
PBAR = pbar
# Calculate Total Length
- code_flags = {
- 102: CODE102,
- 122: CODE122,
- 300: CODE300,
- 250: CODE250
- }
+ code_flags = {102: CODE102, 122: CODE122, 300: CODE300, 250: CODE250}
totalList = 0
for code_item in codeList:
- if code_flags.get(code_item['code'], False):
+ if code_flags.get(code_item["code"], False):
totalList += 1
pbar.total = totalList
pbar.refresh()
-
+
# Begin Parsing File
try:
# Iterate through events
i = 0
while i < len(codeList):
### Event Code: 101 Message
- if codeList[i]['code'] == 101 and CODE101 == True:
+ if codeList[i]["code"] == 101 and CODE101 == True:
# Grab String
- jaString = codeList[i]['stringArgs'][0]
+ jaString = codeList[i]["stringArgs"][0]
initialJAString = jaString
# Catch Vars that may break the TL
- varString = ''
- matchList = re.findall(r'^[\\_]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]', jaString)
+ varString = ""
+ matchList = re.findall(
+ r"^[\\_]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]", jaString
+ )
if len(matchList) != 0:
varString = matchList[0]
- jaString = jaString.replace(matchList[0], '')
+ jaString = jaString.replace(matchList[0], "")
# Grab Speaker
- if '：\n' in jaString:
- nameList = re.findall(r'(.*)：\n', jaString)
+ if "：\n" in jaString:
+ nameList = re.findall(r"(.*)：\n", jaString)
if nameList is not None:
# TL Speaker
response = getSpeaker(nameList[0])
speaker = response[0]
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
-
+
# Set nametag and remove from string
- nametag = f'{speaker}：\n'
- jaString = jaString.replace(f'{nameList[0]}：\n', '')
+ nametag = f"{speaker}：\n"
+ jaString = jaString.replace(f"{nameList[0]}：\n", "")
# Remove Textwrap
- jaString = jaString.replace('\n', ' ')
+ jaString = jaString.replace("\n", " ")
# 1st Pass (Save Text to List)
if not setData:
- if speaker == '':
+ if speaker == "":
stringList.append(jaString)
else:
- stringList.append(f'[{speaker}]: {jaString}')
+ stringList.append(f"[{speaker}]: {jaString}")
# 2nd Pass (Set Text)
else:
# Grab Translated String
translatedText = stringList[0]
-
+
# Remove speaker
- matchSpeakerList = re.findall(r'^(\[.+?\]\s?[|:]\s?)\s?', translatedText)
+ matchSpeakerList = re.findall(
+ r"^(\[.+?\]\s?[|:]\s?)\s?", translatedText
+ )
if len(matchSpeakerList) > 0:
- translatedText = translatedText.replace(matchSpeakerList[0], '')
+ translatedText = translatedText.replace(matchSpeakerList[0], "")
# Textwrap
if FIXTEXTWRAP is True:
@@ -327,35 +359,39 @@ def searchCodes(events, pbar, jobList, filename):
# Add back Nametag
translatedText = nametag + translatedText
- nametag = ''
+ nametag = ""
# Add back Potential Variables in String
translatedText = varString + translatedText
# Set Data
- codeList[i]['stringArgs'][0] = translatedText
+ codeList[i]["stringArgs"][0] = translatedText
# Reset Data and Pop Item
- speaker = ''
+ speaker = ""
stringList.pop(0)
### Event Code: 102 Choices
- if codeList[i]['code'] == 102 and CODE102 == True:
+ if codeList[i]["code"] == 102 and CODE102 == True:
# Grab Choice List
- choiceList = codeList[i]['stringArgs']
+ choiceList = codeList[i]["stringArgs"]
# Translate
- response = translateGPT(choiceList, f'Reply with the {LANGUAGE} translation of the dialogue choice', True)
+ response = translateGPT(
+ choiceList,
+ f"Reply with the {LANGUAGE} translation of the dialogue choice",
+ True,
+ )
translatedChoiceList = response[0]
totalTokens[0] = response[1][0]
totalTokens[1] = response[1][1]
# Validate and Set Data
if len(choiceList) == len(translatedChoiceList):
- codeList[i]['stringArgs'] = translatedChoiceList
+ codeList[i]["stringArgs"] = translatedChoiceList
### Event Code: 210 Common Event
- if codeList[i]['code'] == 210 and CODE210 == True:
+ if codeList[i]["code"] == 210 and CODE210 == True:
# if 'stringArgs' in codeList[i] and len(codeList[i]['stringArgs']) > 1:
# # Grab Event List
# jaString = codeList[i]['stringArgs'][1]
@@ -374,33 +410,49 @@ def searchCodes(events, pbar, jobList, filename):
# # Validate and Set Data
# codeList[i]['stringArgs'][1] = translatedText
- if 'stringArgs' in codeList[i] and len(codeList[i]['stringArgs']) > 1:
- cleanedList = formatDramon(codeList[i]['stringArgs'][1])
+ if "stringArgs" in codeList[i] and len(codeList[i]["stringArgs"]) > 1:
+ cleanedList = formatDramon(codeList[i]["stringArgs"][1])
fontSize = 24
- translatedText = ''
-
+ translatedText = ""
+
for str in cleanedList:
# Pass 1
if not setData:
- if all(x not in str for x in ['_', '@', '>', '/']) and str != '\r\n':
+ if (
+ all(x not in str for x in ["_", "@", ">", "/"])
+ and str != "\r\n"
+ ):
# Remove Textwrap and Font and Add to list
- str = str.replace('\r\n', ' ')
- str = str.replace(f'\\f[{fontSize}]', '')
+ str = str.replace("\r\n", " ")
+ str = str.replace(f"\\f[{fontSize}]", "")
list300.append(str)
# Pass 2
else:
- if all(x not in str for x in ['_', '@', '>', '/',]) and str != '\r\n':
+ if (
+ all(
+ x not in str
+ for x in [
+ "_",
+ "@",
+ ">",
+ "/",
+ ]
+ )
+ and str != "\r\n"
+ ):
# Decide Wrap
- if codeList[i]['stringArgs'][0] == '[移]サウンドノベル':
+ if codeList[i]["stringArgs"][0] == "[移]サウンドノベル":
width = 40
else:
width = WIDTH
# Add Textwrap and Font
list300[0] = textwrap.fill(list300[0], width)
- list300[0] = list300[0].replace('\n', f'\r\n\\f[{fontSize}]')
- list300[0] = f'\\f[{fontSize}]{list300[0]}\r\n'
+ list300[0] = list300[0].replace(
+ "\n", f"\r\n\\f[{fontSize}]"
+ )
+ list300[0] = f"\\f[{fontSize}]{list300[0]}\r\n"
translatedText += list300[0]
list300.pop(0)
else:
@@ -410,28 +462,31 @@ def searchCodes(events, pbar, jobList, filename):
if setData:
# Formatting Fixes
translatedText = translatedText.replace('*"', '* "')
- translatedText = translatedText.replace('\r\n\r\n', '\r\n')
- translatedText = re.sub(r'[^\S\r\n]+', ' ', translatedText)
- codeList[i]['stringArgs'][1] = translatedText
-
+ translatedText = translatedText.replace("\r\n\r\n", "\r\n")
+ translatedText = re.sub(r"[^\S\r\n]+", " ", translatedText)
+ codeList[i]["stringArgs"][1] = translatedText
### Event Code: 122 SetString
- if codeList[i]['code'] == 122 and CODE122 == True:
- if 'stringArgs' in codeList[i] and len(codeList[i]['stringArgs']) > 0:
+ if codeList[i]["code"] == 122 and CODE122 == True:
+ if "stringArgs" in codeList[i] and len(codeList[i]["stringArgs"]) > 0:
# Grab String
- jaString = codeList[i]['stringArgs'][0]
+ jaString = codeList[i]["stringArgs"][0]
# Translate Conversations
- if '：Nothing' in jaString:
+ if "：Nothing" in jaString:
# Separate into list
- list122 = jaString.split('\n\n')
+ list122 = jaString.split("\n\n")
# Remove Textwrap
for j in range(len(list122)):
- list122[j] = list122[j].replace('\n', ' ')
+ list122[j] = list122[j].replace("\n", " ")
# Translate
- response = translateGPT(list122, f'Reply with the {LANGUAGE} translation of the text', True)
+ response = translateGPT(
+ list122,
+ f"Reply with the {LANGUAGE} translation of the text",
+ True,
+ )
list122TL = response[0]
totalTokens[0] = response[1][0]
totalTokens[1] = response[1][1]
@@ -441,30 +496,40 @@ def searchCodes(events, pbar, jobList, filename):
# Adjust Speaker and Add Textwrap
for j in range(len(list122TL)):
list122TL[j] = textwrap.fill(list122TL[j], WIDTH)
- list122TL[j] = re.sub(r'^\[?(.+?)\]?:', r'\1：', list122TL[j])
- list122TL[j] = list122TL[j].replace('：', '：\n')
- list122TL[j] = list122TL[j].replace('：\n ', '：\n')
+ list122TL[j] = re.sub(
+ r"^\[?(.+?)\]?:", r"\1：", list122TL[j]
+ )
+ list122TL[j] = list122TL[j].replace("：", "：\n")
+ list122TL[j] = list122TL[j].replace("：\n ", "：\n")
# Join back into single string
- list122TL = '\n\n'.join(list122TL)
-
+ list122TL = "\n\n".join(list122TL)
+
# Set String
- codeList[i]['stringArgs'][0] = list122TL
-
+ codeList[i]["stringArgs"][0] = list122TL
+
# Translate Other Strings [Specific Files Only]
else:
- if not re.search(r'\.[\w]+$', jaString)\
- and jaString != ''\
- and '_' not in jaString\
- and '",' not in jaString\
- and '/' not in jaString:
+ if (
+ not re.search(r"\.[\w]+$", jaString)
+ and jaString != ""
+ and "_" not in jaString
+ and '",' not in jaString
+ and "/" not in jaString
+ ):
# Things to Check before starting translation
- if re.search(r'[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+', jaString):
+ if re.search(
+ r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", jaString
+ ):
# Remove Textwrap
- jaString = jaString.replace('\n', ' ')
+ jaString = jaString.replace("\n", " ")
# Translate
- response = translateGPT(jaString, f'Reply with the {LANGUAGE} translation of the text', False)
+ response = translateGPT(
+ jaString,
+ f"Reply with the {LANGUAGE} translation of the text",
+ False,
+ )
translatedText = response[0]
totalTokens[0] = response[1][0]
totalTokens[1] = response[1][1]
@@ -473,15 +538,22 @@ def searchCodes(events, pbar, jobList, filename):
translatedText = textwrap.fill(translatedText, WIDTH)
# Set String
- codeList[i]['stringArgs'][0] = translatedText
+ codeList[i]["stringArgs"][0] = translatedText
### Event Code: 300 Common Events
- if codeList[i]['code'] == 300 and CODE300 == True and 'stringArgs' in codeList[i] and len(codeList[i]['stringArgs']) > 1:
+ if (
+ codeList[i]["code"] == 300
+ and CODE300 == True
+ and "stringArgs" in codeList[i]
+ and len(codeList[i]["stringArgs"]) > 1
+ ):
# Choices
- if codeList[i]['stringArgs'][0] == "[共]汎用ウィンドウ生成" \
- or codeList[i]['stringArgs'][0] == "[共]選択生成":
+ if (
+ codeList[i]["stringArgs"][0] == "[共]汎用ウィンドウ生成"
+ or codeList[i]["stringArgs"][0] == "[共]選択生成"
+ ):
# Grab String
- choiceList = codeList[i]['stringArgs'][1].split('\r\n')
+ choiceList = codeList[i]["stringArgs"][1].split("\r\n")
# # Translate Question
# question = codeList[i]['stringArgs'][2]
@@ -501,51 +573,80 @@ def searchCodes(events, pbar, jobList, filename):
# Replace Commas
for j in range(len(choiceListTL)):
- choiceListTL[j] = choiceListTL[j].replace(', ', '、')
+ choiceListTL[j] = choiceListTL[j].replace(", ", "、")
# Convert to String and Set
- translatedText = '\r\n'.join(choiceListTL)
- codeList[i]['stringArgs'][1] = translatedText
+ translatedText = "\r\n".join(choiceListTL)
+ codeList[i]["stringArgs"][1] = translatedText
# Dialogue
- elif codeList[i]['stringArgs'][0] == "Hメッセージ" \
- or codeList[i]['stringArgs'][0] == 'Hしらべる' \
- or codeList[i]['stringArgs'][0] == '[共]Hメッセージ+' \
- or codeList[i]['stringArgs'][0] == 'd[共]ポップアップ表示' \
- or codeList[i]['stringArgs'][0] == 'm_謎冒頭' \
- or (codeList[i]['codeStr'] == 'SetString' and ('「' in codeList[i]['stringArgs'][0] or '*' in codeList[i]['stringArgs'][0]))\
- or codeList[i]['stringArgs'][0] == '[移]サウンドノベル':
+ elif (
+ codeList[i]["stringArgs"][0] == "Hメッセージ"
+ or codeList[i]["stringArgs"][0] == "Hしらべる"
+ or codeList[i]["stringArgs"][0] == "[共]Hメッセージ+"
+ or codeList[i]["stringArgs"][0] == "d[共]ポップアップ表示"
+ or codeList[i]["stringArgs"][0] == "m_謎冒頭"
+ or (
+ codeList[i]["codeStr"] == "SetString"
+ and (
+ "「" in codeList[i]["stringArgs"][0]
+ or "*" in codeList[i]["stringArgs"][0]
+ )
+ )
+ or codeList[i]["stringArgs"][0] == "[移]サウンドノベル"
+ ):
cleanedList = None
- if len(codeList[i]['stringArgs']) > 1 and not re.search(r'^[\\]+cself\[\d+\]$', codeList[i]['stringArgs'][1]):
- cleanedList = formatDramon(codeList[i]['stringArgs'][1])
- elif codeList[i]['code'] == 122:
- cleanedList = formatDramon(codeList[i]['stringArgs'][0])
+ if len(codeList[i]["stringArgs"]) > 1 and not re.search(
+ r"^[\\]+cself\[\d+\]$", codeList[i]["stringArgs"][1]
+ ):
+ cleanedList = formatDramon(codeList[i]["stringArgs"][1])
+ elif codeList[i]["code"] == 122:
+ cleanedList = formatDramon(codeList[i]["stringArgs"][0])
if cleanedList:
fontSize = 24
- translatedText = ''
-
+ translatedText = ""
+
for str in cleanedList:
# Pass 1
if not setData:
- if all(x not in str for x in ['_', '@', '>', '/']) and str != '\r\n':
+ if (
+ all(x not in str for x in ["_", "@", ">", "/"])
+ and str != "\r\n"
+ ):
# Remove Textwrap and Font and Add to list
- str = str.replace('\r\n', ' ')
- str = str.replace(f'\\f[{fontSize}]', '')
+ str = str.replace("\r\n", " ")
+ str = str.replace(f"\\f[{fontSize}]", "")
list300.append(str)
# Pass 2
else:
- if all(x not in str for x in ['_', '@', '>', '/',]) and str != '\r\n':
+ if (
+ all(
+ x not in str
+ for x in [
+ "_",
+ "@",
+ ">",
+ "/",
+ ]
+ )
+ and str != "\r\n"
+ ):
# Decide Wrap
- if codeList[i]['stringArgs'][0] == '[移]サウンドノベル':
+ if (
+ codeList[i]["stringArgs"][0]
+ == "[移]サウンドノベル"
+ ):
width = 40
else:
width = WIDTH
# Add Textwrap and Font
list300[0] = textwrap.fill(list300[0], width)
- list300[0] = list300[0].replace('\n', f'\r\n\\f[{fontSize}]')
- list300[0] = f'\\f[{fontSize}]{list300[0]}\r\n'
+ list300[0] = list300[0].replace(
+ "\n", f"\r\n\\f[{fontSize}]"
+ )
+ list300[0] = f"\\f[{fontSize}]{list300[0]}\r\n"
translatedText += list300[0]
list300.pop(0)
else:
@@ -555,29 +656,34 @@ def searchCodes(events, pbar, jobList, filename):
if setData:
# Formatting Fixes
translatedText = translatedText.replace('*"', '* "')
- translatedText = translatedText.replace('\r\n\r\n', '\r\n')
- translatedText = re.sub(r'[^\S\r\n]+', ' ', translatedText)
- if len(codeList[i]['stringArgs']) > 1:
- codeList[i]['stringArgs'][1] = translatedText
+ translatedText = translatedText.replace("\r\n\r\n", "\r\n")
+ translatedText = re.sub(r"[^\S\r\n]+", " ", translatedText)
+ if len(codeList[i]["stringArgs"]) > 1:
+ codeList[i]["stringArgs"][1] = translatedText
else:
- codeList[i]['stringArgs'][0] = translatedText
+ codeList[i]["stringArgs"][0] = translatedText
### Event Code: 250 Common Events
- if codeList[i]['code'] == 250 and CODE250 == True:
+ if codeList[i]["code"] == 250 and CODE250 == True:
foundTerm = False
# Validate size
- if len(codeList[i]['stringArgs']) > 2:
- if codeList[i]['stringArgs'][1] == "┗所持防具個数" and codeList[i]['stringArgs'][2] != '':
+ if len(codeList[i]["stringArgs"]) > 2:
+ if (
+ codeList[i]["stringArgs"][1] == "┗所持防具個数"
+ and codeList[i]["stringArgs"][2] != ""
+ ):
# Grab String
- jaString = codeList[i]['stringArgs'][2]
+ jaString = codeList[i]["stringArgs"][2]
# Catch Vars that may break the TL
- varString = ''
- matchList = re.findall(r'^[\\_]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]', jaString)
+ varString = ""
+ matchList = re.findall(
+ r"^[\\_]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]", jaString
+ )
if len(matchList) != 0:
varString = matchList[0]
- jaString = jaString.replace(matchList[0], '')
+ jaString = jaString.replace(matchList[0], "")
# Check if term already translated
for j in range(len(TERMSLIST)):
@@ -587,7 +693,11 @@ def searchCodes(events, pbar, jobList, filename):
# Translate
if foundTerm == False:
- response = translateGPT(jaString, f'Reply with the {LANGUAGE} translation of the text.', False)
+ response = translateGPT(
+ jaString,
+ f"Reply with the {LANGUAGE} translation of the text.",
+ False,
+ )
translatedText = response[0]
totalTokens[0] = response[1][0]
totalTokens[1] = response[1][1]
@@ -597,11 +707,11 @@ def searchCodes(events, pbar, jobList, filename):
translatedText = varString + translatedText
# Set Data
- codeList[i]['stringArgs'][2] = translatedText
-
+ codeList[i]["stringArgs"][2] = translatedText
+
### Iterate
i += 1
-
+
# EOF
stringListTL = []
list300TL = []
@@ -621,7 +731,7 @@ def searchCodes(events, pbar, jobList, filename):
MISMATCH.append(filename)
else:
setData = True
-
+
# 300 List
if len(list300) > 0:
pbar.total = len(list300)
@@ -636,54 +746,70 @@ def searchCodes(events, pbar, jobList, filename):
MISMATCH.append(filename)
else:
setData = True
-
+
# Pass 2
if setData:
stringList = []
searchCodes(events, pbar, [stringListTL, list300TL], filename)
- else:
+ else:
# Set Data
events = codeList
except IndexError as e:
traceback.print_exc()
- raise Exception(str(e) + 'Failed to translate: ' + initialJAString) from None
+ raise Exception(str(e) + "Failed to translate: " + initialJAString) from None
except Exception as e:
traceback.print_exc()
- raise Exception(str(e) + 'Failed to translate: ' + initialJAString) from None
+ raise Exception(str(e) + "Failed to translate: " + initialJAString) from None
return totalTokens
-def formatDramon(jaString):
- imageRegex = r'(\r?\n?_[a-zA-Z_\d/.]+\r?\n?)|(@-?\d?\r\n)|(@-?\d?)([^\r]\d?-?\d?[^\r]+?)(\r\n|$)|(_PDC)|(>\r\n)|(^[#])|(\r\n@$)|(/)|(_SS_)|(\r\n[@/]\s?-?\d?\r\n)'
- jaString = jaString.replace('\u3000', ' ')
- jaString = jaString.replace('#', '')
- jaString = re.sub(r'([^\r])\n', r'\1\r\n', jaString)
+def formatDramon(jaString):
+ imageRegex = r"(\r?\n?_[a-zA-Z_\d/.]+\r?\n?)|(@-?\d?\r\n)|(@-?\d?)([^\r]\d?-?\d?[^\r]+?)(\r\n|$)|(_PDC)|(>\r\n)|(^[#])|(\r\n@$)|(/)|(_SS_)|(\r\n[@/]\s?-?\d?\r\n)"
+
+ jaString = jaString.replace("\u3000", " ")
+ jaString = jaString.replace("#", "")
+ jaString = re.sub(r"([^\r])\n", r"\1\r\n", jaString)
# Grab and Split
jaStringList = re.split(imageRegex, jaString)
# Clean List
- cleanedList = [x for x in jaStringList if x is not None and x != '' and x != '\r\n' and x != '_SS_']
+ cleanedList = [
+ x
+ for x in jaStringList
+ if x is not None and x != "" and x != "\r\n" and x != "_SS_"
+ ]
# Iterate Through List
j = 0
- translatedText = ''
+ translatedText = ""
while j < len(cleanedList):
- if ('@' in cleanedList[j] or '/' in cleanedList[j]) and j < len(cleanedList)-1 and re.search(r'([@/]-?\d?\r\n)', cleanedList[j]) is None and '.ogg' not in cleanedList[j]:
+ if (
+ ("@" in cleanedList[j] or "/" in cleanedList[j])
+ and j < len(cleanedList) - 1
+ and re.search(r"([@/]-?\d?\r\n)", cleanedList[j]) is None
+ and ".ogg" not in cleanedList[j]
+ ):
# Setup @
- if j > 0 and '@' not in cleanedList[j-1] and '/' not in cleanedList[j-1] and '_' not in cleanedList[j-1]:
- cleanedList[j-1] = cleanedList[j-1] + cleanedList[j+1]
+ if (
+ j > 0
+ and "@" not in cleanedList[j - 1]
+ and "/" not in cleanedList[j - 1]
+ and "_" not in cleanedList[j - 1]
+ ):
+ cleanedList[j - 1] = cleanedList[j - 1] + cleanedList[j + 1]
else:
- cleanedList.insert(j, cleanedList[j+1])
+ cleanedList.insert(j, cleanedList[j + 1])
j += 1
- cleanedList[j] = f'\r\n{cleanedList[j]}\r\n'
- cleanedList.pop(j+1)
+ cleanedList[j] = f"\r\n{cleanedList[j]}\r\n"
+ cleanedList.pop(j + 1)
j += 1
-
+
return cleanedList
+
# DatabaseDatabase
def searchDB(events, pbar, jobList, filename):
# Set Lists
@@ -697,187 +823,209 @@ def searchDB(events, pbar, jobList, filename):
weaponsList = jobList[6]
setData = True
else:
- scenarioList = [[],[],[]]
- NPCList = [[],[],[],[]]
- itemList = [[],[],[],[]]
- armorList = [[],[]]
- enemyList = [[],[]]
- weaponsList = [[],[],[],[]]
- collectionList = [[],[],[],[]]
+ scenarioList = [[], [], []]
+ NPCList = [[], [], [], []]
+ itemList = [[], [], [], []]
+ armorList = [[], []]
+ enemyList = [[], []]
+ weaponsList = [[], [], [], []]
+ collectionList = [[], [], [], []]
setData = False
-
+
# Vars/Globals
totalTokens = [0, 0]
- initialJAString = ''
+ initialJAString = ""
tableList = events
- font = ''
+ font = ""
global LOCK
global NAMESLIST
global MISMATCH
-
+
# Calculate Total
totalLines = 0
for table in tableList:
- if table['name'] == 'NPC' and NPCFLAG == True:
- for NPC in table['data']:
- totalLines += len(NPC['data'])
- if table['name'] == 'Hシナリオ' and SCENARIOFLAG == True:
- for hScenario in table['data']:
- totalLines += len(hScenario['data'])
+ if table["name"] == "NPC" and NPCFLAG == True:
+ for NPC in table["data"]:
+ totalLines += len(NPC["data"])
+ if table["name"] == "Hシナリオ" and SCENARIOFLAG == True:
+ for hScenario in table["data"]:
+ totalLines += len(hScenario["data"])
pbar.total = totalLines
pbar.refresh()
# Begin Parsing File
try:
for table in tableList:
-
# Translate NPC
- if table['name'] == 'キャラ会話' and NPCFLAG == True:
- for npc in table['data']:
- dataList = npc['data']
+ if table["name"] == "キャラ会話" and NPCFLAG == True:
+ for npc in table["data"]:
+ dataList = npc["data"]
# Parse
for j in range(len(dataList)):
# Name
- if 'キャラ名' in dataList[j].get('name'):
+ if "キャラ名" in dataList[j].get("name"):
# Pass 1 (Grab Data)
if setData == False:
- if dataList[j].get('value') != '':
- NPCList[0].append(dataList[j].get('value'))
+ if dataList[j].get("value") != "":
+ NPCList[0].append(dataList[j].get("value"))
# Pass 2 (Set Data)
else:
- if dataList[j].get('value') != '':
- dataList[j].update({'value': NPCList[0][0]})
+ if dataList[j].get("value") != "":
+ dataList[j].update({"value": NPCList[0][0]})
NPCList[0].pop(0)
-
+
# Description
- if '菊池' in dataList[j].get('name'):
+ if "菊池" in dataList[j].get("name"):
# Pass 1 (Grab Data)
if setData == False:
- if dataList[j].get('value') != '':
+ if dataList[j].get("value") != "":
# Remove Textwrap
- jaString = dataList[j].get('value')
- jaString = jaString.replace('\n', ' ')
- jaString = jaString.replace('\r', '')
- jaString = re.sub(r'[\\]+f\[\d+\]', '', jaString)
+ jaString = dataList[j].get("value")
+ jaString = jaString.replace("\n", " ")
+ jaString = jaString.replace("\r", "")
+ jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString)
# Append Data
NPCList[1].append(jaString)
# Pass 2 (Set Data)
else:
- if dataList[j].get('value') != '':
+ if dataList[j].get("value") != "":
# Textwrap
translatedText = NPCList[1][0]
translatedText = textwrap.fill(translatedText, 30)
translatedText = font + translatedText
# Set Data
- dataList[j].update({'value': translatedText})
+ dataList[j].update({"value": translatedText})
NPCList[1].pop(0)
# Description
- if '篠宮' in dataList[j].get('name'):
+ if "篠宮" in dataList[j].get("name"):
# Pass 1 (Grab Data)
if setData == False:
- if dataList[j].get('value') != '':
+ if dataList[j].get("value") != "":
# Remove Textwrap
- jaString = dataList[j].get('value')
- jaString = jaString.replace('\n', ' ')
- jaString = jaString.replace('\r', '')
- jaString = re.sub(r'[\\]+f\[\d+\]', '', jaString)
+ jaString = dataList[j].get("value")
+ jaString = jaString.replace("\n", " ")
+ jaString = jaString.replace("\r", "")
+ jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString)
# Append Data
NPCList[2].append(jaString)
# Pass 2 (Set Data)
else:
- if dataList[j].get('value') != '':
+ if dataList[j].get("value") != "":
# Textwrap
translatedText = NPCList[2][0]
- translatedText = textwrap.fill(translatedText, LISTWIDTH)
+ translatedText = textwrap.fill(
+ translatedText, LISTWIDTH
+ )
translatedText = font + translatedText
# Set Data
- dataList[j].update({'value': translatedText})
+ dataList[j].update({"value": translatedText})
NPCList[2].pop(0)
# Grab Scenarios
- if table['name'] == 'Hシナリオ' and SCENARIOFLAG == True:
- for hScenario in table['data']:
- dataList = hScenario['data']
+ if table["name"] == "Hシナリオ" and SCENARIOFLAG == True:
+ for hScenario in table["data"]:
+ dataList = hScenario["data"]
# Parse
# Pass 1 (Grab Data)
if setData == False:
- if dataList[1].get('value') != '':
- scenarioList[0].append(dataList[1].get('value'))
- if dataList[44].get('value') != '':
- scenarioList[1].append(dataList[44].get('value'))
- if dataList[45].get('value') != '':
- scenarioList[2].append(dataList[45].get('value'))
+ if dataList[1].get("value") != "":
+ scenarioList[0].append(dataList[1].get("value"))
+ if dataList[44].get("value") != "":
+ scenarioList[1].append(dataList[44].get("value"))
+ if dataList[45].get("value") != "":
+ scenarioList[2].append(dataList[45].get("value"))
# Pass 2 (Set Data)
else:
- if dataList[1].get('value') != '':
- dataList[1].update({'value': scenarioList[0][0]})
+ if dataList[1].get("value") != "":
+ dataList[1].update({"value": scenarioList[0][0]})
scenarioList[0].pop(0)
- if dataList[44].get('value') != '':
- dataList[44].update({'value': scenarioList[1][0]})
+ if dataList[44].get("value") != "":
+ dataList[44].update({"value": scenarioList[1][0]})
scenarioList[1].pop(0)
- if dataList[45].get('value') != '':
- dataList[45].update({'value': scenarioList[2][0]})
+ if dataList[45].get("value") != "":
+ dataList[45].update({"value": scenarioList[2][0]})
scenarioList[2].pop(0)
# Grab Items
- if table['name'] == 'オーブ' and ITEMFLAG == True:
- with open('translations.txt', 'a', encoding='utf-8') as file:
- for item in table['data']:
- dataList = item['data']
+ if table["name"] == "オーブ" and ITEMFLAG == True:
+ with open("translations.txt", "a", encoding="utf-8") as file:
+ for item in table["data"]:
+ dataList = item["data"]
# Parse #
for j in range(len(dataList)):
# Name
- if dataList[j].get('name') == 'オーブの名前':
+ if dataList[j].get("name") == "オーブの名前":
# Pass 1 (Grab Data)
if setData == False:
- if dataList[j].get('value') != '':
- itemList[0].append(dataList[j].get('value'))
+ if dataList[j].get("value") != "":
+ itemList[0].append(dataList[j].get("value"))
# Pass 2 (Set Data)
else:
- if dataList[j].get('value') != '':
- file.write(f'{dataList[j].get('value')} ({itemList[0][0]})\n')
- dataList[j].update({'value': itemList[0][0]})
+ if dataList[j].get("value") != "":
+ file.write(
+ f'{dataList[j].get('value')} ({itemList[0][0]})\n'
+ )
+ dataList[j].update({"value": itemList[0][0]})
itemList[0].pop(0)
-
+
# Description 1 (You are my specialz)
- if dataList[j].get('name') == 'オーブの説明':
+ if dataList[j].get("name") == "オーブの説明":
# Clean String
fontSize = 18
- translatedText = ''
- cleanedList = formatDramon(dataList[j].get('value'))
+ translatedText = ""
+ cleanedList = formatDramon(dataList[j].get("value"))
for str in cleanedList:
# Pass 1
if not setData:
- if all(x not in str for x in ['_', '@', '>', '/']) and str != '\r\n':
+ if (
+ all(
+ x not in str
+ for x in ["_", "@", ">", "/"]
+ )
+ and str != "\r\n"
+ ):
# Remove Textwrap and Font and Add to list
- str = str.replace('\r\n', ' ')
- str = str.replace(f'\\f[{fontSize}]', '')
+ str = str.replace("\r\n", " ")
+ str = str.replace(f"\\f[{fontSize}]", "")
itemList[1].append(str)
# Pass 2
else:
- if all(x not in str for x in ['_', '@', '>', '/',]) and str != '\r\n':
+ if (
+ all(
+ x not in str
+ for x in [
+ "_",
+ "@",
+ ">",
+ "/",
+ ]
+ )
+ and str != "\r\n"
+ ):
# Decide Wrap
width = WIDTH
# Add Textwrap and Font
tempText = itemList[1][0]
tempText = textwrap.fill(tempText, width)
- tempText = tempText.replace('\n', f'\r\n\\f[{fontSize}]')
- tempText = f'\\f[{fontSize}]{tempText}\r\n'
+ tempText = tempText.replace(
+ "\n", f"\r\n\\f[{fontSize}]"
+ )
+ tempText = f"\\f[{fontSize}]{tempText}\r\n"
translatedText += tempText
itemList[1].pop(0)
else:
@@ -887,36 +1035,59 @@ def searchDB(events, pbar, jobList, filename):
if setData:
# Formatting Fixes
translatedText = translatedText.replace('*"', '* "')
- translatedText = translatedText.replace('\r\n\r\n', '\r\n')
- translatedText = re.sub(r'[^\S\r\n]+', ' ', translatedText)
- dataList[j].update({'value': translatedText})
+ translatedText = translatedText.replace(
+ "\r\n\r\n", "\r\n"
+ )
+ translatedText = re.sub(
+ r"[^\S\r\n]+", " ", translatedText
+ )
+ dataList[j].update({"value": translatedText})
# Description 2 (You are my specialz)
- if dataList[j].get('name') == 'NULL':
+ if dataList[j].get("name") == "NULL":
# Clean String
fontSize = 24
- translatedText = ''
- cleanedList = formatDramon(dataList[j].get('value'))
+ translatedText = ""
+ cleanedList = formatDramon(dataList[j].get("value"))
for str in cleanedList:
# Pass 1
if not setData:
- if all(x not in str for x in ['_', '@', '>', '/']) and str != '\r\n':
+ if (
+ all(
+ x not in str
+ for x in ["_", "@", ">", "/"]
+ )
+ and str != "\r\n"
+ ):
# Remove Textwrap and Font and Add to list
- str = str.replace('\r\n', ' ')
- str = str.replace(f'\\f[{fontSize}]', '')
+ str = str.replace("\r\n", " ")
+ str = str.replace(f"\\f[{fontSize}]", "")
itemList[2].append(str)
# Pass 2
else:
- if all(x not in str for x in ['_', '@', '>', '/',]) and str != '\r\n':
+ if (
+ all(
+ x not in str
+ for x in [
+ "_",
+ "@",
+ ">",
+ "/",
+ ]
+ )
+ and str != "\r\n"
+ ):
# Decide Wrap
width = WIDTH
# Add Textwrap and Font
tempText = itemList[2][0]
tempText = textwrap.fill(tempText, width)
- tempText = tempText.replace('\n', f'\r\n\\f[{fontSize}]')
- tempText = f'\\f[{fontSize}]{tempText}\r\n'
+ tempText = tempText.replace(
+ "\n", f"\r\n\\f[{fontSize}]"
+ )
+ tempText = f"\\f[{fontSize}]{tempText}\r\n"
translatedText += tempText
itemList[2].pop(0)
else:
@@ -926,28 +1097,49 @@ def searchDB(events, pbar, jobList, filename):
if setData:
# Formatting Fixes
translatedText = translatedText.replace('*"', '* "')
- translatedText = translatedText.replace('\r\n\r\n', '\r\n')
- translatedText = re.sub(r'[^\S\r\n]+', ' ', translatedText)
- dataList[j].update({'value': translatedText})
+ translatedText = translatedText.replace(
+ "\r\n\r\n", "\r\n"
+ )
+ translatedText = re.sub(
+ r"[^\S\r\n]+", " ", translatedText
+ )
+ dataList[j].update({"value": translatedText})
# Description 3 (You are my specialz)
- if dataList[j].get('name') == 'NULL':
+ if dataList[j].get("name") == "NULL":
# Clean String
fontSize = 24
- translatedText = ''
- cleanedList = formatDramon(dataList[j].get('value'))
+ translatedText = ""
+ cleanedList = formatDramon(dataList[j].get("value"))
for str in cleanedList:
# Pass 1
if not setData:
- if all(x not in str for x in ['_', '@', '>', '/']) and str != '\r\n':
+ if (
+ all(
+ x not in str
+ for x in ["_", "@", ">", "/"]
+ )
+ and str != "\r\n"
+ ):
# Remove Textwrap and Font and Add to list
- str = str.replace('\r\n', ' ')
- str = str.replace(f'\\f[{fontSize}]', '')
+ str = str.replace("\r\n", " ")
+ str = str.replace(f"\\f[{fontSize}]", "")
itemList[3].append(str)
# Pass 2
else:
- if all(x not in str for x in ['_', '@', '>', '/',]) and str != '\r\n':
+ if (
+ all(
+ x not in str
+ for x in [
+ "_",
+ "@",
+ ">",
+ "/",
+ ]
+ )
+ and str != "\r\n"
+ ):
# Decide Wrap
width = WIDTH
@@ -965,180 +1157,190 @@ def searchDB(events, pbar, jobList, filename):
if setData:
# Formatting Fixes
translatedText = translatedText.replace('*"', '* "')
- translatedText = translatedText.replace('\r\n\r\n', '\r\n')
- translatedText = re.sub(r'[^\S\r\n]+', ' ', translatedText)
- dataList[j].update({'value': translatedText})
+ translatedText = translatedText.replace(
+ "\r\n\r\n", "\r\n"
+ )
+ translatedText = re.sub(
+ r"[^\S\r\n]+", " ", translatedText
+ )
+ dataList[j].update({"value": translatedText})
# Grab Armors
- if table['name'] == '防具' and ARMORFLAG == True:
- for armor in table['data']:
- dataList = armor['data']
+ if table["name"] == "防具" and ARMORFLAG == True:
+ for armor in table["data"]:
+ dataList = armor["data"]
# Parse
for j in range(len(dataList)):
# Name
- if '防具の名前' in dataList[j].get('name'):
+ if "防具の名前" in dataList[j].get("name"):
# Pass 1 (Grab Data)
if setData == False:
- if dataList[j].get('value') != '':
- armorList[0].append(dataList[j].get('value'))
+ if dataList[j].get("value") != "":
+ armorList[0].append(dataList[j].get("value"))
# Pass 2 (Set Data)
else:
- if dataList[j].get('value') != '':
- dataList[j].update({'value': armorList[0][0]})
+ if dataList[j].get("value") != "":
+ dataList[j].update({"value": armorList[0][0]})
armorList[0].pop(0)
-
+
# Description
- if '防具の説明' in dataList[j].get('name'):
+ if "防具の説明" in dataList[j].get("name"):
# Pass 1 (Grab Data)
if setData == False:
- if dataList[j].get('value') != '':
+ if dataList[j].get("value") != "":
# Remove Textwrap
- jaString = dataList[j].get('value')
- jaString = jaString.replace('\n', ' ')
- jaString = jaString.replace('\r', '')
- jaString = re.sub(r'[\\]+f\[\d+\]', '', jaString)
+ jaString = dataList[j].get("value")
+ jaString = jaString.replace("\n", " ")
+ jaString = jaString.replace("\r", "")
+ jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString)
# Append Data
armorList[1].append(jaString)
# Pass 2 (Set Data)
else:
- if dataList[j].get('value') != '':
+ if dataList[j].get("value") != "":
# Textwrap
translatedText = armorList[1][0]
- translatedText = textwrap.fill(translatedText, LISTWIDTH)
+ translatedText = textwrap.fill(
+ translatedText, LISTWIDTH
+ )
translatedText = font + translatedText
# Set Data
- dataList[j].update({'value': translatedText})
+ dataList[j].update({"value": translatedText})
armorList[1].pop(0)
# Grab Enemies
- if table['name'] == '敵ｷｬﾗ個体ﾃﾞｰﾀ' and ENEMYFLAG == True:
- for enemy in table['data']:
- dataList = enemy['data']
+ if table["name"] == "敵ｷｬﾗ個体ﾃﾞｰﾀ" and ENEMYFLAG == True:
+ for enemy in table["data"]:
+ dataList = enemy["data"]
# Parse
for j in range(len(dataList)):
# Name
- if '敵キャラ名' in dataList[j].get('name'):
+ if "敵キャラ名" in dataList[j].get("name"):
# Pass 1 (Grab Data)
if setData == False:
- if dataList[j].get('value') != '':
- enemyList[0].append(dataList[j].get('value'))
+ if dataList[j].get("value") != "":
+ enemyList[0].append(dataList[j].get("value"))
# Pass 2 (Set Data)
else:
- if dataList[j].get('value') != '':
- dataList[j].update({'value': enemyList[0][0]})
+ if dataList[j].get("value") != "":
+ dataList[j].update({"value": enemyList[0][0]})
enemyList[0].pop(0)
-
+
# Description
- if 'NULL' in dataList[j].get('name'):
+ if "NULL" in dataList[j].get("name"):
# Pass 1 (Grab Data)
if setData == False:
- if dataList[j].get('value') != '':
+ if dataList[j].get("value") != "":
# Remove Textwrap
- jaString = dataList[j].get('value')
- jaString = jaString.replace('\n', ' ')
- jaString = jaString.replace('\r', '')
- jaString = re.sub(r'[\\]+f\[\d+\]', '', jaString)
+ jaString = dataList[j].get("value")
+ jaString = jaString.replace("\n", " ")
+ jaString = jaString.replace("\r", "")
+ jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString)
# Append Data
enemyList[1].append(jaString)
# Pass 2 (Set Data)
else:
- if dataList[j].get('value') != '':
+ if dataList[j].get("value") != "":
# Textwrap
translatedText = enemyList[1][0]
- translatedText = textwrap.fill(translatedText, LISTWIDTH)
+ translatedText = textwrap.fill(
+ translatedText, LISTWIDTH
+ )
translatedText = font + translatedText
# Set Data
- dataList[j].update({'value': translatedText})
+ dataList[j].update({"value": translatedText})
enemyList[1].pop(0)
# Grab Weapons
- if table['name'] == '武器' and WEAPONFLAG == True:
- for weapon in table['data']:
- dataList = weapon['data']
+ if table["name"] == "武器" and WEAPONFLAG == True:
+ for weapon in table["data"]:
+ dataList = weapon["data"]
# Parse
for j in range(len(dataList)):
# Name
- if '武器の名前' in dataList[j].get('name'):
+ if "武器の名前" in dataList[j].get("name"):
# Pass 1 (Grab Data)
if setData == False:
- if dataList[j].get('value') != '':
- weaponsList[0].append(dataList[j].get('value'))
+ if dataList[j].get("value") != "":
+ weaponsList[0].append(dataList[j].get("value"))
# Pass 2 (Set Data)
else:
- if dataList[j].get('value') != '':
- dataList[j].update({'value': weaponsList[0][0]})
+ if dataList[j].get("value") != "":
+ dataList[j].update({"value": weaponsList[0][0]})
weaponsList[0].pop(0)
-
+
# Description
- if '武器の説明' in dataList[j].get('name'):
+ if "武器の説明" in dataList[j].get("name"):
# Pass 1 (Grab Data)
if setData == False:
- if dataList[j].get('value') != '':
+ if dataList[j].get("value") != "":
# Remove Textwrap
- jaString = dataList[j].get('value')
- jaString = jaString.replace('\n', ' ')
- jaString = jaString.replace('\r', '')
- jaString = re.sub(r'[\\]+f\[\d+\]', '', jaString)
+ jaString = dataList[j].get("value")
+ jaString = jaString.replace("\n", " ")
+ jaString = jaString.replace("\r", "")
+ jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString)
# Append Data
weaponsList[1].append(jaString)
# Pass 2 (Set Data)
else:
- if dataList[j].get('value') != '':
+ if dataList[j].get("value") != "":
# Textwrap
translatedText = weaponsList[1][0]
- translatedText = textwrap.fill(translatedText, LISTWIDTH)
+ translatedText = textwrap.fill(
+ translatedText, LISTWIDTH
+ )
translatedText = font + translatedText
# Set Data
- dataList[j].update({'value': translatedText})
+ dataList[j].update({"value": translatedText})
weaponsList[1].pop(0)
# Grab Collection
- if table['name'] == '鍛冶師用DB' and COLLECTIONFLAG == True:
- for object in table['data']:
- dataList = object['data']
+ if table["name"] == "鍛冶師用DB" and COLLECTIONFLAG == True:
+ for object in table["data"]:
+ dataList = object["data"]
# Parse
for j in range(len(dataList)):
# Name
- if '作る装備' in dataList[j].get('name'):
+ if "作る装備" in dataList[j].get("name"):
# Pass 1 (Grab Data)
if setData == False:
- if dataList[j].get('value') != '':
+ if dataList[j].get("value") != "":
# Remove Textwrap
- jaString = dataList[j].get('value')
- jaString = jaString.replace('\n', ' ')
- jaString = jaString.replace('\r', '')
- jaString = re.sub(r'[\\]+f\[\d+\]', '', jaString)
+ jaString = dataList[j].get("value")
+ jaString = jaString.replace("\n", " ")
+ jaString = jaString.replace("\r", "")
+ jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString)
collectionList[0].append(jaString)
# Pass 2 (Set Data)
else:
- if dataList[j].get('value') != '':
- dataList[j].update({'value': collectionList[0][0]})
+ if dataList[j].get("value") != "":
+ dataList[j].update({"value": collectionList[0][0]})
collectionList[0].pop(0)
- # Description
- if '品物の解説' in dataList[j].get('name'):
+ # Description
+ if "品物の解説" in dataList[j].get("name"):
# Pass 1 (Grab Data)
if setData == False:
- if dataList[j].get('value') != '':
+ if dataList[j].get("value") != "":
# Remove Textwrap
- jaString = dataList[j].get('value')
- jaString = jaString.replace('\n', ' ')
- jaString = jaString.replace('\r', '')
- jaString = re.sub(r'[\\]+f\[\d+\]', '', jaString)
+ jaString = dataList[j].get("value")
+ jaString = jaString.replace("\n", " ")
+ jaString = jaString.replace("\r", "")
+ jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString)
# Skill Action (Optional)
# jaString = f'Taro{jaString}'
@@ -1147,30 +1349,32 @@ def searchDB(events, pbar, jobList, filename):
collectionList[1].append(jaString)
# Pass 2 (Set Data)
else:
- if dataList[j].get('value') != '':
+ if dataList[j].get("value") != "":
translatedText = collectionList[1][0]
# Remove Action (Optional)
# translatedText = translatedText.replace('Taro', '')
# Textwrap
- translatedText = textwrap.fill(translatedText, LISTWIDTH)
+ translatedText = textwrap.fill(
+ translatedText, LISTWIDTH
+ )
translatedText = font + translatedText
# Set Data
- dataList[j].update({'value': translatedText})
+ dataList[j].update({"value": translatedText})
collectionList[1].pop(0)
- # Description 2
- if 'NULL' in dataList[j].get('name'):
+ # Description 2
+ if "NULL" in dataList[j].get("name"):
# Pass 1 (Grab Data)
if setData == False:
- if dataList[j].get('value') != '':
+ if dataList[j].get("value") != "":
# Remove Textwrap
- jaString = dataList[j].get('value')
- jaString = jaString.replace('\n', ' ')
- jaString = jaString.replace('\r', '')
- jaString = re.sub(r'[\\]+f\[\d+\]', '', jaString)
+ jaString = dataList[j].get("value")
+ jaString = jaString.replace("\n", " ")
+ jaString = jaString.replace("\r", "")
+ jaString = re.sub(r"[\\]+f\[\d+\]", "", jaString)
# Skill Action (Optional)
# jaString = f'Taro{jaString}'
@@ -1179,28 +1383,30 @@ def searchDB(events, pbar, jobList, filename):
collectionList[2].append(jaString)
# Pass 2 (Set Data)
else:
- if dataList[j].get('value') != '':
+ if dataList[j].get("value") != "":
translatedText = collectionList[2][0]
# Remove Action (Optional)
# translatedText = translatedText.replace('Taro', '')
# Textwrap
- translatedText = textwrap.fill(translatedText, LISTWIDTH)
+ translatedText = textwrap.fill(
+ translatedText, LISTWIDTH
+ )
translatedText = font + translatedText
# Set Data
- dataList[j].update({'value': translatedText})
+ dataList[j].update({"value": translatedText})
collectionList[2].pop(0)
# Translation
- scenarioListTL = [[],[],[]]
- NPCListTL = [[],[],[],[]]
- itemListTL = [[],[],[],[]]
- collectionListTL = [[],[],[],[]]
- armorListTL = [[],[]]
- enemyListTL = [[],[]]
- weaponsListTL = [[],[],[]]
+ scenarioListTL = [[], [], []]
+ NPCListTL = [[], [], [], []]
+ itemListTL = [[], [], [], []]
+ collectionListTL = [[], [], [], []]
+ armorListTL = [[], []]
+ enemyListTL = [[], []]
+ weaponsListTL = [[], [], []]
translate = False
@@ -1214,37 +1420,51 @@ def searchDB(events, pbar, jobList, filename):
pbar.refresh()
# Name
- response = translateGPT(NPCList[0], 'Reply with only the '+ LANGUAGE +' translation of the RPG enemy name', True)
+ response = translateGPT(
+ NPCList[0],
+ "Reply with only the "
+ + LANGUAGE
+ + " translation of the RPG enemy name",
+ True,
+ )
nameListTL = response[0]
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
# Desc 1
- response = translateGPT(NPCList[1], 'Reply with only the '+ LANGUAGE +' translation', True)
+ response = translateGPT(
+ NPCList[1], "Reply with only the " + LANGUAGE + " translation", True
+ )
descListTL1 = response[0]
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
# Desc 2
- response = translateGPT(NPCList[2], 'Reply with only the '+ LANGUAGE +' translation', True)
+ response = translateGPT(
+ NPCList[2], "Reply with only the " + LANGUAGE + " translation", True
+ )
descListTL2 = response[0]
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
# Desc 3
- response = translateGPT(NPCList[3], 'Reply with only the '+ LANGUAGE +' translation', True)
+ response = translateGPT(
+ NPCList[3], "Reply with only the " + LANGUAGE + " translation", True
+ )
descListTL3 = response[0]
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
# Check Mismatch
- if len(nameListTL) != len(NPCList[0]) or\
- len(descListTL1) != len(NPCList[1]) or\
- len(descListTL2) != len(NPCList[2])or\
- len(descListTL3) != len(NPCList[3]):
+ if (
+ len(nameListTL) != len(NPCList[0])
+ or len(descListTL1) != len(NPCList[1])
+ or len(descListTL2) != len(NPCList[2])
+ or len(descListTL3) != len(NPCList[3])
+ ):
with LOCK:
if filename not in MISMATCH:
MISMATCH.append(filename)
else:
NPCListTL = [nameListTL, descListTL1, descListTL2, descListTL3]
- translate = True
+ translate = True
# SCENARIO
if len(scenarioList[0]) > 0:
@@ -1256,31 +1476,49 @@ def searchDB(events, pbar, jobList, filename):
pbar.refresh()
# Name
- response = translateGPT(scenarioList[0], 'Reply with only the '+ LANGUAGE +' translation', True)
+ response = translateGPT(
+ scenarioList[0],
+ "Reply with only the " + LANGUAGE + " translation",
+ True,
+ )
nameListTL = response[0]
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
# Desc 1
- response = translateGPT(scenarioList[1], 'reply with only the gender neutral '+ LANGUAGE +' translation of the NPC name', True)
+ response = translateGPT(
+ scenarioList[1],
+ "reply with only the gender neutral "
+ + LANGUAGE
+ + " translation of the NPC name",
+ True,
+ )
descListTL1 = response[0]
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
# Desc 2
- response = translateGPT(scenarioList[2], 'reply with only the gender neutral '+ LANGUAGE +' translation of the NPC name', True)
+ response = translateGPT(
+ scenarioList[2],
+ "reply with only the gender neutral "
+ + LANGUAGE
+ + " translation of the NPC name",
+ True,
+ )
descListTL2 = response[0]
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
# Check Mismatch
- if len(nameListTL) != len(scenarioList[0]) or\
- len(descListTL1) != len(scenarioList[1]) or\
- len(descListTL2) != len(scenarioList[2]):
+ if (
+ len(nameListTL) != len(scenarioList[0])
+ or len(descListTL1) != len(scenarioList[1])
+ or len(descListTL2) != len(scenarioList[2])
+ ):
with LOCK:
if filename not in MISMATCH:
MISMATCH.append(filename)
else:
scenarioListTL = [nameListTL, descListTL1, descListTL2]
- translate = True
+ translate = True
# ITEMS
if len(itemList[0]) > 0 or len(itemList[1]) > 0:
@@ -1292,37 +1530,47 @@ def searchDB(events, pbar, jobList, filename):
pbar.refresh()
# Name
- response = translateGPT(itemList[0], 'Reply with only the '+ LANGUAGE +' translation', True)
+ response = translateGPT(
+ itemList[0], "Reply with only the " + LANGUAGE + " translation", True
+ )
nameListTL = response[0]
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
# Desc 1
- response = translateGPT(itemList[1], 'Reply with only the '+ LANGUAGE +' translation', True)
+ response = translateGPT(
+ itemList[1], "Reply with only the " + LANGUAGE + " translation", True
+ )
descListTL1 = response[0]
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
# Desc 2
- response = translateGPT(itemList[2], 'Reply with only the '+ LANGUAGE +' translation', True)
+ response = translateGPT(
+ itemList[2], "Reply with only the " + LANGUAGE + " translation", True
+ )
descListTL2 = response[0]
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
# Desc 3
- response = translateGPT(itemList[3], 'Reply with only the '+ LANGUAGE +' translation', True)
+ response = translateGPT(
+ itemList[3], "Reply with only the " + LANGUAGE + " translation", True
+ )
descListTL3 = response[0]
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
# Check Mismatch
- if len(nameListTL) != len(itemList[0]) or\
- len(descListTL1) != len(itemList[1]) or\
- len(descListTL2) != len(itemList[2])or\
- len(descListTL3) != len(itemList[3]):
+ if (
+ len(nameListTL) != len(itemList[0])
+ or len(descListTL1) != len(itemList[1])
+ or len(descListTL2) != len(itemList[2])
+ or len(descListTL3) != len(itemList[3])
+ ):
with LOCK:
if filename not in MISMATCH:
MISMATCH.append(filename)
else:
itemListTL = [nameListTL, descListTL1, descListTL2, descListTL3]
- translate = True
+ translate = True
# Armor
if len(armorList[0]) > 0:
@@ -1334,25 +1582,32 @@ def searchDB(events, pbar, jobList, filename):
pbar.refresh()
# Name
- response = translateGPT(armorList[0], 'Reply with only the '+ LANGUAGE +' translation of the NPC name', True)
+ response = translateGPT(
+ armorList[0],
+ "Reply with only the " + LANGUAGE + " translation of the NPC name",
+ True,
+ )
nameListTL = response[0]
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
# Desc 1
- response = translateGPT(armorList[1], 'Reply with only the '+ LANGUAGE +' translation', True)
+ response = translateGPT(
+ armorList[1], "Reply with only the " + LANGUAGE + " translation", True
+ )
descListTL1 = response[0]
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
# Check Mismatch
- if len(nameListTL) != len(armorList[0]) or\
- len(descListTL1) != len(armorList[1]):
+ if len(nameListTL) != len(armorList[0]) or len(descListTL1) != len(
+ armorList[1]
+ ):
with LOCK:
if filename not in MISMATCH:
MISMATCH.append(filename)
else:
armorListTL = [nameListTL, descListTL1]
- translate = True
+ translate = True
# Enemies
if len(enemyList[0]) > 0:
@@ -1364,25 +1619,32 @@ def searchDB(events, pbar, jobList, filename):
pbar.refresh()
# Name
- response = translateGPT(enemyList[0], 'Reply with only the '+ LANGUAGE +' translation of the RPG item name', True)
+ response = translateGPT(
+ enemyList[0],
+ "Reply with only the " + LANGUAGE + " translation of the RPG item name",
+ True,
+ )
nameListTL = response[0]
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
# Desc 1
- response = translateGPT(enemyList[1], 'Reply with only the '+ LANGUAGE +' translation', True)
+ response = translateGPT(
+ enemyList[1], "Reply with only the " + LANGUAGE + " translation", True
+ )
descListTL1 = response[0]
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
# Check Mismatch
- if len(nameListTL) != len(enemyList[0]) or\
- len(descListTL1) != len(enemyList[1]):
+ if len(nameListTL) != len(enemyList[0]) or len(descListTL1) != len(
+ enemyList[1]
+ ):
with LOCK:
if filename not in MISMATCH:
MISMATCH.append(filename)
else:
enemyListTL = [nameListTL, descListTL1]
- translate = True
+ translate = True
# Weapons
if len(weaponsList[0]) > 0:
@@ -1394,31 +1656,37 @@ def searchDB(events, pbar, jobList, filename):
pbar.refresh()
# Name
- response = translateGPT(weaponsList[0], 'Reply with only the '+ LANGUAGE +' translation of the RPG item name', True)
+ response = translateGPT(
+ weaponsList[0],
+ "Reply with only the " + LANGUAGE + " translation of the RPG item name",
+ True,
+ )
nameListTL = response[0]
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
# Desc 1
- response = translateGPT(weaponsList[1], '', True)
+ response = translateGPT(weaponsList[1], "", True)
descListTL1 = response[0]
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
# Desc 2
- response = translateGPT(weaponsList[2], '', True)
+ response = translateGPT(weaponsList[2], "", True)
descListTL2 = response[0]
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
# Check Mismatch
- if len(nameListTL) != len(weaponsList[0]) or\
- len(descListTL1) != len(weaponsList[1]) or\
- len(descListTL2) != len(weaponsList[2]):
+ if (
+ len(nameListTL) != len(weaponsList[0])
+ or len(descListTL1) != len(weaponsList[1])
+ or len(descListTL2) != len(weaponsList[2])
+ ):
with LOCK:
if filename not in MISMATCH:
MISMATCH.append(filename)
else:
weaponsListTL = [nameListTL, descListTL1, descListTL2]
- translate = True
+ translate = True
# Collection
for list in collectionList:
@@ -1431,34 +1699,42 @@ def searchDB(events, pbar, jobList, filename):
pbar.refresh()
# Name
- response = translateGPT(collectionList[0], 'reply with only the gender neutral '+ LANGUAGE +' translation of the action log. Always start the sentence with Taro. For example, Translate \'Taroを倒した！\' as \'Taro was defeated!\'', True)
+ response = translateGPT(
+ collectionList[0],
+ "reply with only the gender neutral "
+ + LANGUAGE
+ + " translation of the action log. Always start the sentence with Taro. For example, Translate 'Taroを倒した！' as 'Taro was defeated!'",
+ True,
+ )
nameListTL = response[0]
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
# Desc 1
- response = translateGPT(collectionList[1], '', True)
+ response = translateGPT(collectionList[1], "", True)
descListTL1 = response[0]
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
# Desc 2
- response = translateGPT(collectionList[2], '', True)
+ response = translateGPT(collectionList[2], "", True)
descListTL2 = response[0]
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
# Check Mismatch
- if len(nameListTL) != len(collectionList[0]) or\
- len(descListTL1) != len(collectionList[1]) or\
- len(descListTL2) != len(collectionList[2]):
+ if (
+ len(nameListTL) != len(collectionList[0])
+ or len(descListTL1) != len(collectionList[1])
+ or len(descListTL2) != len(collectionList[2])
+ ):
with LOCK:
if filename not in MISMATCH:
MISMATCH.append(filename)
else:
collectionListTL = [nameListTL, descListTL1, descListTL2]
- translate = True
-
+ translate = True
+
# Start Pass 2
if translate == True:
jobList.append(scenarioListTL)
@@ -1468,34 +1744,43 @@ def searchDB(events, pbar, jobList, filename):
jobList.append(armorListTL)
jobList.append(enemyListTL)
jobList.append(weaponsListTL)
- searchDB(events, pbar, jobList, filename)
+ searchDB(events, pbar, jobList, filename)
except IndexError as e:
traceback.print_exc()
- raise Exception(str(e) + 'Failed to translate: ' + initialJAString) from None
+ raise Exception(str(e) + "Failed to translate: " + initialJAString) from None
except Exception as e:
traceback.print_exc()
- raise Exception(str(e) + 'Failed to translate: ' + initialJAString) from None
+ raise Exception(str(e) + "Failed to translate: " + initialJAString) from None
return totalTokens
+
# Save some money and enter the character before translation
def getSpeaker(speaker):
match speaker:
- case 'ファイン':
- return ['Fine', [0,0]]
- case '':
- return ['', [0,0]]
+ case "ファイン":
+ return ["Fine", [0, 0]]
+ case "":
+ return ["", [0, 0]]
case _:
# Store Speaker
if speaker not in str(NAMESLIST):
- response = translateGPT(speaker, 'Reply with the '+ LANGUAGE +' translation of the NPC name.', False)
+ response = translateGPT(
+ speaker,
+ "Reply with the " + LANGUAGE + " translation of the NPC name.",
+ False,
+ )
response[0] = response[0].title()
response[0] = response[0].replace("'S", "'s")
# Retry if name doesn't translate for some reason
- if re.search(r'([a-zA-Z？?])', response[0]) == None:
- response = translateGPT(speaker, 'Reply with the '+ LANGUAGE +' translation of the NPC name.', False)
+ if re.search(r"([a-zA-Z？?])", response[0]) == None:
+ response = translateGPT(
+ speaker,
+ "Reply with the " + LANGUAGE + " translation of the NPC name.",
+ False,
+ )
response[0] = response[0].title()
response[0] = response[0].replace("'S", "'s")
@@ -1506,66 +1791,72 @@ def getSpeaker(speaker):
else:
for i in range(len(NAMESLIST)):
if speaker == NAMESLIST[i][0]:
- return [NAMESLIST[i][1],[0,0]]
-
- return [speaker,[0,0]]
+ return [NAMESLIST[i][1], [0, 0]]
+
+ return [speaker, [0, 0]]
+
def subVars(jaString):
- jaString = jaString.replace('\u3000', ' ')
+ jaString = jaString.replace("\u3000", " ")
# Formatting
count = 0
- codeList = re.findall(r'[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]', jaString)
+ codeList = re.findall(r"[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]", jaString)
codeList = set(codeList)
if len(codeList) != 0:
for var in codeList:
- jaString = jaString.replace(var, '[FCode_' + str(count) + ']')
+ jaString = jaString.replace(var, "[FCode_" + str(count) + "]")
count += 1
# WOLF Images
count = 0
- humList = re.findall(r'(\\?r?\\?n?_.*?\d\r\n@?)', jaString)
+ humList = re.findall(r"(\\?r?\\?n?_.*?\d\r\n@?)", jaString)
humList = set(humList)
if len(humList) != 0:
for var in humList:
- jaString = jaString.replace(var, '[ICode_' + str(count) + ']')
+ jaString = jaString.replace(var, "[ICode_" + str(count) + "]")
count += 1
# Put all lists in list and return
return [jaString, [codeList, humList]]
+
def resubVars(translatedText, varList):
# Fix Spacing and ChatGPT Nonsense
- matchList = re.findall(r'\[\s?.+?\s?\]', translatedText)
+ matchList = re.findall(r"\[\s?.+?\s?\]", translatedText)
if len(matchList) > 0:
for match in matchList:
text = match.strip()
translatedText = translatedText.replace(match, text)
-
+
# Formatting
count = 0
if len(varList[0]) != 0:
for var in varList[0]:
- translatedText = translatedText.replace('[FCode_' + str(count) + ']', var)
+ translatedText = translatedText.replace("[FCode_" + str(count) + "]", var)
count += 1
# Formatting
count = 0
if len(varList[1]) != 0:
for var in varList[1]:
- translatedText = translatedText.replace('[ICode_' + str(count) + ']', var)
+ translatedText = translatedText.replace("[ICode_" + str(count) + "]", var)
count += 1
return translatedText
+
def batchList(input_list, batch_size):
if not isinstance(batch_size, int) or batch_size <= 0:
raise ValueError("batch_size must be a positive integer")
-
- return [input_list[i:i + batch_size] for i in range(0, len(input_list), batch_size)]
+
+ return [
+ input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)
+ ]
+
def createContext(fullPromptFlag, subbedT):
- characters = 'Game Characters:\n\
+ characters = "Game Characters:\n\
ロラン (Roland) - Male\n\
リュカ (Ryuka) - Male\n\
レックス (Rex) - Male\n\
@@ -1610,10 +1901,12 @@ def createContext(fullPromptFlag, subbedT):
アロマ (Aroma) - Female\n\
ピッケ (Pikke) - Female\n\
ドラオ (Dorao) - Male\n\
-'
-
- system = PROMPT + VOCAB if fullPromptFlag else \
- f"\
+"
+
+ system = (
+ PROMPT + VOCAB
+ if fullPromptFlag
+ else f"\
You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
- You translate everything, including erotic and explicit content.\n\
@@ -1625,12 +1918,14 @@ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{
- `...` can be a part of the dialogue. Translate it as it is.\n\
{VOCAB}\n\
"
+ )
if isinstance(subbedT, list):
- user = f'```json\n{subbedT}```'
+ user = f"```json\n{subbedT}```"
else:
user = subbedT
return characters, system, user
+
def translateText(characters, system, user, history, penalty, format):
# Prompt
msg = [{"role": "system", "content": system}]
@@ -1645,13 +1940,13 @@ def translateText(characters, system, user, history, penalty, format):
msg.append({"role": "system", "content": history})
# Response Format
- if format == 'json':
- responseFormat = { "type": "json_object" }
+ if format == "json":
+ responseFormat = {"type": "json_object"}
else:
- responseFormat = { "type": "text" }
-
+ responseFormat = {"type": "text"}
+
# Content to TL
- msg.append({"role": "user", "content": f'{user}'})
+ msg.append({"role": "user", "content": f"{user}"})
response = openai.chat.completions.create(
temperature=0,
frequency_penalty=penalty,
@@ -1661,18 +1956,19 @@ def translateText(characters, system, user, history, penalty, format):
)
return response
+
def cleanTranslatedText(translatedText, varResponse):
placeholders = {
- f'{LANGUAGE} Translation: ': '',
- 'Translation: ': '',
- 'っ': '',
- '〜': '~',
- 'ッ': '',
- '。': '.',
- '「': '\\"',
- '」': '\\"',
- '- ': '-',
- 'Placeholder Text': '',
+ f"{LANGUAGE} Translation: ": "",
+ "Translation: ": "",
+ "っ": "",
+ "〜": "~",
+ "ッ": "",
+ "。": ".",
+ "「": '\\"',
+ "」": '\\"',
+ "- ": "-",
+ "Placeholder Text": "",
# Add more replacements as needed
}
for target, replacement in placeholders.items():
@@ -1683,11 +1979,12 @@ def cleanTranslatedText(translatedText, varResponse):
translatedText = resubVars(translatedText, varResponse[1])
return translatedText
+
def elongateCharacters(text):
# Define a pattern to match one character followed by one or more `ー` characters
# Using a positive lookbehind assertion to capture the preceding character
- pattern = r'(?<=(.))ー+'
-
+ pattern = r"(?<=(.))ー+"
+
# Define a replacement function that elongates the captured character
def repl(match):
char = match.group(1) # The character before the ー sequence
@@ -1697,6 +1994,7 @@ def elongateCharacters(text):
# Use re.sub() to replace the pattern in the text
return re.sub(pattern, repl, text)
+
def extractTranslation(translatedTextList, is_list):
try:
translatedTextList = re.sub(r'\\"+\"([^,\n}])', r'\\"\1', translatedTextList)
@@ -1709,15 +2007,15 @@ def extractTranslation(translatedTextList, is_list):
return string_list[0]
except Exception as e:
- PBAR.write(f'extractTranslation Error: {e} on String {translatedTextList}')
+ PBAR.write(f"extractTranslation Error: {e} on String {translatedTextList}")
return None
def countTokens(characters, system, user, history):
inputTotalTokens = 0
outputTotalTokens = 0
- enc = tiktoken.encoding_for_model('gpt-4')
-
+ enc = tiktoken.encoding_for_model("gpt-4")
+
# Input
if isinstance(history, list):
for line in history:
@@ -1729,26 +2027,28 @@ def countTokens(characters, system, user, history):
inputTotalTokens += len(enc.encode(user))
# Output
- outputTotalTokens += round(len(enc.encode(user))*3)
+ outputTotalTokens += round(len(enc.encode(user)) * 3)
return [inputTotalTokens, outputTotalTokens]
+
def combineList(tlist, text):
if isinstance(text, list):
return [t for sublist in tlist for t in sublist]
return tlist[0]
+
@retry(exceptions=Exception, tries=5, delay=5)
def translateGPT(text, history, fullPromptFlag):
global PBAR, MISMATCH, FILENAME
-
+
mismatch = False
totalTokens = [0, 0]
if isinstance(text, list):
- format = 'json'
+ format = "json"
tList = batchList(text, BATCHSIZE)
else:
- format = 'text'
+ format = "text"
tList = [text]
for index, tItem in enumerate(tList):
@@ -1763,7 +2063,7 @@ def translateGPT(text, history, fullPromptFlag):
subbedT = varResponse[0]
# Things to Check before starting translation
- if not re.search(r'[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+', subbedT):
+ if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", subbedT):
if PBAR is not None:
PBAR.update(len(tItem))
continue
@@ -1788,9 +2088,13 @@ def translateGPT(text, history, fullPromptFlag):
translatedText = cleanTranslatedText(translatedText, varResponse)
if isinstance(tItem, list):
extractedTranslations = extractTranslation(translatedText, True)
- if extractedTranslations == None or len(tItem) != len(extractedTranslations):
+ if extractedTranslations == None or len(tItem) != len(
+ extractedTranslations
+ ):
# Mismatch. Try Again
- response = translateText(characters, system, user, history, 0.05, format)
+ response = translateText(
+ characters, system, user, history, 0.05, format
+ )
translatedText = response.choices[0].message.content
totalTokens[0] += response.usage.prompt_tokens
totalTokens[1] += response.usage.completion_tokens
@@ -1799,13 +2103,17 @@ def translateGPT(text, history, fullPromptFlag):
translatedText = cleanTranslatedText(translatedText, varResponse)
if isinstance(tItem, list):
extractedTranslations = extractTranslation(translatedText, True)
- if extractedTranslations == None or len(tItem) != len(extractedTranslations):
- mismatch = True # Just here for breakpoint
-
+ if extractedTranslations == None or len(tItem) != len(
+ extractedTranslations
+ ):
+ mismatch = True # Just here for breakpoint
+
# Set if no mismatch
if mismatch == False:
tList[index] = extractedTranslations
- history = extractedTranslations[-10:] # Update history if we have a list
+ history = extractedTranslations[
+ -10:
+ ] # Update history if we have a list
else:
history = text[-10:]
mismatch = False
@@ -1818,7 +2126,7 @@ def translateGPT(text, history, fullPromptFlag):
PBAR.update(len(tItem))
else:
# Ensure we're passing a single string to extractTranslation
- tList[index] = translatedText
+ tList[index] = translatedText
finalList = combineList(tList, text)
return [finalList, totalTokens]
diff --git a/modules/wolf2.py b/modules/wolf2.py
index aa51a47..a66b7d8 100644
--- a/modules/wolf2.py
+++ b/modules/wolf2.py
@@ -16,49 +16,50 @@ from tqdm import tqdm
# Open AI
load_dotenv()
-if os.getenv('api').replace(' ', '') != '':
- openai.base_url = os.getenv('api')
-openai.organization = os.getenv('org')
-openai.api_key = os.getenv('key')
+if os.getenv("api").replace(" ", "") != "":
+ openai.base_url = os.getenv("api")
+openai.organization = os.getenv("org")
+openai.api_key = os.getenv("key")
-#Globals
-MODEL = os.getenv('model')
-TIMEOUT = int(os.getenv('timeout'))
-LANGUAGE = os.getenv('language').capitalize()
-PROMPT = Path('prompt.txt').read_text(encoding='utf-8')
-VOCAB = Path('vocab.txt').read_text(encoding='utf-8')
-THREADS = int(os.getenv('threads'))
+# Globals
+MODEL = os.getenv("model")
+TIMEOUT = int(os.getenv("timeout"))
+LANGUAGE = os.getenv("language").capitalize()
+PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
+VOCAB = Path("vocab.txt").read_text(encoding="utf-8")
+THREADS = int(os.getenv("threads"))
LOCK = threading.Lock()
-WIDTH = int(os.getenv('width'))
-LISTWIDTH = int(os.getenv('listWidth'))
+WIDTH = int(os.getenv("width"))
+LISTWIDTH = int(os.getenv("listWidth"))
NOTEWIDTH = 70
MAXHISTORY = 10
-ESTIMATE = ''
+ESTIMATE = ""
TOKENS = [0, 0]
NAMESLIST = []
-NAMES = False # Output a list of all the character names found
-BRFLAG = False # If the game uses
instead
+NAMES = False # Output a list of all the character names found
+BRFLAG = False # If the game uses
instead
FIXTEXTWRAP = True # Overwrites textwrap
-IGNORETLTEXT = False # Ignores all translated text.
-MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
+IGNORETLTEXT = False # Ignores all translated text.
+MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
-#tqdm Globals
-BAR_FORMAT='{l_bar}{bar:10}{r_bar}{bar:-10b}'
+# tqdm Globals
+BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
POSITION = 0
LEAVE = False
# Pricing - Depends on the model https://openai.com/pricing
# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request
# If you are getting a MISMATCH LENGTH error, lower the batch size.
-if 'gpt-3.5' in MODEL:
- INPUTAPICOST = .002
- OUTPUTAPICOST = .002
+if "gpt-3.5" in MODEL:
+ INPUTAPICOST = 0.002
+ OUTPUTAPICOST = 0.002
BATCHSIZE = 10
-elif 'gpt-4' in MODEL:
- INPUTAPICOST = .005
- OUTPUTAPICOST = .015
+elif "gpt-4" in MODEL:
+ INPUTAPICOST = 0.005
+ OUTPUTAPICOST = 0.015
BATCHSIZE = 40
+
def handleWOLF2(filename, estimate):
global ESTIMATE
ESTIMATE = estimate
@@ -75,17 +76,21 @@ def handleWOLF2(filename, estimate):
TOKENS[1] += translatedData[1][1]
# Print Total
- totalString = getResultString(['', TOKENS, None], end - start, 'TOTAL')
+ totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
# Print any errors on maps
if len(MISMATCH) > 0:
- return totalString + Fore.RED + f'\nMismatch Errors: {MISMATCH}' + Fore.RESET
+ return (
+ totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
+ )
else:
return totalString
-
+
else:
try:
- with open('translated/' + filename, 'w', encoding='shift_jis', errors='ignore') as outFile:
+ with open(
+ "translated/" + filename, "w", encoding="shift_jis", errors="ignore"
+ ) as outFile:
start = time.time()
translatedData = openFiles(filename)
@@ -98,23 +103,36 @@ def handleWOLF2(filename, estimate):
TOKENS[1] += translatedData[1][1]
except Exception:
traceback.print_exc()
- return 'Fail'
+ return "Fail"
+
+ return getResultString(["", TOKENS, None], end - start, "TOTAL")
- return getResultString(['', TOKENS, None], end - start, 'TOTAL')
def getResultString(translatedData, translationTime, filename):
# File Print String
- totalTokenstring =\
- Fore.YELLOW +\
- '[Input: ' + str(translatedData[1][0]) + ']'\
- '[Output: ' + str(translatedData[1][1]) + ']'\
- '[Cost: ${:,.4f}'.format((translatedData[1][0] * .001 * INPUTAPICOST) +\
- (translatedData[1][1] * .001 * OUTPUTAPICOST)) + ']'
- timeString = Fore.BLUE + '[' + str(round(translationTime, 1)) + 's]'
+ totalTokenstring = (
+ Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
+ "[Output: "
+ + str(translatedData[1][1])
+ + "]" "[Cost: ${:,.4f}".format(
+ (translatedData[1][0] * 0.001 * INPUTAPICOST)
+ + (translatedData[1][1] * 0.001 * OUTPUTAPICOST)
+ )
+ + "]"
+ )
+ timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
if translatedData[2] == None:
# Success
- return filename + ': ' + totalTokenstring + timeString + Fore.GREEN + u' \u2713 ' + Fore.RESET
+ return (
+ filename
+ + ": "
+ + totalTokenstring
+ + timeString
+ + Fore.GREEN
+ + " \u2713 "
+ + Fore.RESET
+ )
else:
# Fail
@@ -123,31 +141,41 @@ def getResultString(translatedData, translationTime, filename):
except Exception as e:
traceback.print_exc()
errorString = str(e) + Fore.RED
- return filename + ': ' + totalTokenstring + timeString + Fore.RED + u' \u2717 ' +\
- errorString + Fore.RESET
+ return (
+ filename
+ + ": "
+ + totalTokenstring
+ + timeString
+ + Fore.RED
+ + " \u2717 "
+ + errorString
+ + Fore.RESET
+ )
+
def openFiles(filename):
- with open('files/' + filename, 'r', encoding='shift_jis') as readFile:
+ with open("files/" + filename, "r", encoding="shift_jis") as readFile:
translatedData = parseWOLF(readFile, filename)
# Delete lines marked for deletion
finalData = []
for line in translatedData[0]:
- if line != '\\d\n':
+ if line != "\\d\n":
finalData.append(line)
translatedData[0] = finalData
-
+
return translatedData
+
def parseWOLF(readFile, filename):
- totalTokens = [0,0]
+ totalTokens = [0, 0]
# Read File into data
data = readFile.readlines()
# Create Progress Bar
with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
- pbar.desc=filename
+ pbar.desc = filename
try:
result = translateWOLF(data, [], pbar, filename)
@@ -158,39 +186,40 @@ def parseWOLF(readFile, filename):
return [data, totalTokens, e]
return [data, totalTokens, None]
+
def translateWOLF(data, translatedList, pbar, filename):
stringList = []
currentGroup = []
- tokens = [0,0]
- speaker = ''
+ tokens = [0, 0]
+ speaker = ""
global LOCK, ESTIMATE, PBAR
PBAR = pbar
i = 0
while i < len(data):
# Speaker
- matchList = re.findall(r'(.*)：', data[i])
+ matchList = re.findall(r"(.*)：", data[i])
if len(matchList) != 0:
response = getSpeaker(matchList[0])
speaker = response[0]
tokens[0] += response[1][0]
tokens[1] += response[1][1]
- data[i] = f'{speaker}：\n'
+ data[i] = f"{speaker}：\n"
i += 1
else:
- speaker = ''
+ speaker = ""
# Options
- if '//選択肢' in data[i]:
+ if "//選択肢" in data[i]:
i += 1
choiceList = []
initialIndex = i
- while('//' in data[i] and 'の場合' not in data[i]):
- choiceList.append(re.search(r'\/\/(.*)', data[i]).group(1))
+ while "//" in data[i] and "の場合" not in data[i]:
+ choiceList.append(re.search(r"\/\/(.*)", data[i]).group(1))
i += 1
-
+
# Translate
- response = translateGPT(choiceList, 'This will be a dialogue option', True)
+ response = translateGPT(choiceList, "This will be a dialogue option", True)
tokens[0] += response[1][0]
tokens[1] += response[1][1]
choiceListTL = response[0]
@@ -199,9 +228,9 @@ def translateWOLF(data, translatedList, pbar, filename):
if len(choiceList) == len(choiceListTL):
# Set Data
i = initialIndex
- while('//' in data[i] and 'の場合' not in data[i]):
- choiceListTL[0] = choiceListTL[0].replace(', ', '、')
- data[i] = f'//{choiceListTL[0]}\n'
+ while "//" in data[i] and "の場合" not in data[i]:
+ choiceListTL[0] = choiceListTL[0].replace(", ", "、")
+ data[i] = f"//{choiceListTL[0]}\n"
choiceListTL.pop(0)
i += 1
@@ -212,36 +241,46 @@ def translateWOLF(data, translatedList, pbar, filename):
MISMATCH.append(filename)
# Lines
- if r'/' not in data[i] and '@' not in data[i] and data[i] != '\n':
+ if r"/" not in data[i] and "@" not in data[i] and data[i] != "\n":
# Pass 1
if translatedList == []:
# Grab Consecutive Strings
currentGroup.append(data[i])
i += 1
- while i < len(data) and r'/' not in data[i] and '@' not in data[i] and data[i] != '\n':
+ while (
+ i < len(data)
+ and r"/" not in data[i]
+ and "@" not in data[i]
+ and data[i] != "\n"
+ ):
currentGroup.append(data[i])
i += 1
-
+
# Join up 401 groups for better translation.
if len(currentGroup) > 0:
- jaString = ''.join(currentGroup)
+ jaString = "".join(currentGroup)
currentGroup = []
-
+
# Remove any textwrap
- jaString = jaString.replace('\n', ' ')
+ jaString = jaString.replace("\n", " ")
# Add Speaker (If there is one)
- if speaker != '':
- jaString = f'{speaker}: {jaString}'
+ if speaker != "":
+ jaString = f"{speaker}: {jaString}"
# Add String
stringList.append(jaString)
i += 1
-
+
# Pass 2
else:
# Insert Strings
- while i < len(data) and r'/' not in data[i] and '@' not in data[i] and data[i] != '\n':
+ while (
+ i < len(data)
+ and r"/" not in data[i]
+ and "@" not in data[i]
+ and data[i] != "\n"
+ ):
data.pop(i)
# Get Text
@@ -252,13 +291,13 @@ def translateWOLF(data, translatedList, pbar, filename):
translatedList = None
# Remove added speaker
- translatedText = re.sub(r'^.+?:\s', '', translatedText)
+ translatedText = re.sub(r"^.+?:\s", "", translatedText)
# Textwrap
translatedText = textwrap.fill(translatedText, width=WIDTH)
# Set Data
- data.insert(i, f'{translatedText}\n')
+ data.insert(i, f"{translatedText}\n")
i += 1
# Nothing relevant. Skip Line.
@@ -270,9 +309,9 @@ def translateWOLF(data, translatedList, pbar, filename):
# Set Progress
pbar.total = len(stringList)
pbar.refresh()
-
+
# Translate
- response = translateGPT(stringList, '', True)
+ response = translateGPT(stringList, "", True)
tokens[0] += response[1][0]
tokens[1] += response[1][1]
translatedList = response[0]
@@ -288,23 +327,32 @@ def translateWOLF(data, translatedList, pbar, filename):
MISMATCH.append(filename)
return tokens
+
# Save some money and enter the character before translation
def getSpeaker(speaker):
match speaker:
- case 'ファイン':
- return ['Fine', [0,0]]
- case '':
- return ['', [0,0]]
+ case "ファイン":
+ return ["Fine", [0, 0]]
+ case "":
+ return ["", [0, 0]]
case _:
# Store Speaker
if speaker not in str(NAMESLIST):
- response = translateGPT(speaker, 'Reply with the '+ LANGUAGE +' translation of the NPC name.', False)
+ response = translateGPT(
+ speaker,
+ "Reply with the " + LANGUAGE + " translation of the NPC name.",
+ False,
+ )
response[0] = response[0].title()
response[0] = response[0].replace("'S", "'s")
# Retry if name doesn't translate for some reason
- if re.search(r'([a-zA-Z？?])', response[0]) == None:
- response = translateGPT(speaker, 'Reply with the '+ LANGUAGE +' translation of the NPC name.', False)
+ if re.search(r"([a-zA-Z？?])", response[0]) == None:
+ response = translateGPT(
+ speaker,
+ "Reply with the " + LANGUAGE + " translation of the NPC name.",
+ False,
+ )
response[0] = response[0].title()
response[0] = response[0].replace("'S", "'s")
@@ -315,50 +363,56 @@ def getSpeaker(speaker):
else:
for i in range(len(NAMESLIST)):
if speaker == NAMESLIST[i][0]:
- return [NAMESLIST[i][1],[0,0]]
-
- return [speaker,[0,0]]
+ return [NAMESLIST[i][1], [0, 0]]
+
+ return [speaker, [0, 0]]
+
def subVars(jaString):
- jaString = jaString.replace('\u3000', ' ')
+ jaString = jaString.replace("\u3000", " ")
# Formatting
count = 0
- codeList = re.findall(r'[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]', jaString)
+ codeList = re.findall(r"[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]", jaString)
codeList = set(codeList)
if len(codeList) != 0:
for var in codeList:
- jaString = jaString.replace(var, '[FCode_' + str(count) + ']')
+ jaString = jaString.replace(var, "[FCode_" + str(count) + "]")
count += 1
# Put all lists in list and return
return [jaString, codeList]
+
def resubVars(translatedText, codeList):
# Fix Spacing and ChatGPT Nonsense
- matchList = re.findall(r'\[\s?.+?\s?\]', translatedText)
+ matchList = re.findall(r"\[\s?.+?\s?\]", translatedText)
if len(matchList) > 0:
for match in matchList:
text = match.strip()
translatedText = translatedText.replace(match, text)
-
+
# Formatting
count = 0
if len(codeList) != 0:
for var in codeList:
- translatedText = translatedText.replace('[FCode_' + str(count) + ']', var)
+ translatedText = translatedText.replace("[FCode_" + str(count) + "]", var)
count += 1
return translatedText
+
def batchList(input_list, batch_size):
if not isinstance(batch_size, int) or batch_size <= 0:
raise ValueError("batch_size must be a positive integer")
-
- return [input_list[i:i + batch_size] for i in range(0, len(input_list), batch_size)]
+
+ return [
+ input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)
+ ]
+
def createContext(fullPromptFlag, subbedT):
- characters = 'Game Characters:\n\
+ characters = "Game Characters:\n\
ロラン (Roland) - Male\n\
リュカ (Ryuka) - Male\n\
レックス (Rex) - Male\n\
@@ -403,10 +457,12 @@ def createContext(fullPromptFlag, subbedT):
アロマ (Aroma) - Female\n\
ピッケ (Pikke) - Female\n\
ドラオ (Dorao) - Male\n\
-'
-
- system = PROMPT + VOCAB if fullPromptFlag else \
- f"\
+"
+
+ system = (
+ PROMPT + VOCAB
+ if fullPromptFlag
+ else f"\
You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
- You translate everything, including erotic and explicit content.\n\
@@ -418,12 +474,14 @@ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{
- `...` can be a part of the dialogue. Translate it as it is.\n\
{VOCAB}\n\
"
+ )
if isinstance(subbedT, list):
- user = f'```json\n{subbedT}```'
+ user = f"```json\n{subbedT}```"
else:
user = subbedT
return characters, system, user
+
def translateText(characters, system, user, history, penalty, format):
# Prompt
msg = [{"role": "system", "content": system + characters}]
@@ -438,13 +496,13 @@ def translateText(characters, system, user, history, penalty, format):
msg.append({"role": "system", "content": history})
# Response Format
- if format == 'json':
- responseFormat = { "type": "json_object" }
+ if format == "json":
+ responseFormat = {"type": "json_object"}
else:
- responseFormat = { "type": "text" }
-
+ responseFormat = {"type": "text"}
+
# Content to TL
- msg.append({"role": "user", "content": f'{user}'})
+ msg.append({"role": "user", "content": f"{user}"})
response = openai.chat.completions.create(
temperature=0,
frequency_penalty=penalty,
@@ -454,18 +512,19 @@ def translateText(characters, system, user, history, penalty, format):
)
return response
+
def cleanTranslatedText(translatedText, varResponse):
placeholders = {
- f'{LANGUAGE} Translation: ': '',
- 'Translation: ': '',
- 'っ': '',
- '〜': '~',
- 'ッ': '',
- '。': '.',
- '「': '\\"',
- '」': '\\"',
- '- ': '-',
- 'Placeholder Text': '',
+ f"{LANGUAGE} Translation: ": "",
+ "Translation: ": "",
+ "っ": "",
+ "〜": "~",
+ "ッ": "",
+ "。": ".",
+ "「": '\\"',
+ "」": '\\"',
+ "- ": "-",
+ "Placeholder Text": "",
# Add more replacements as needed
}
for target, replacement in placeholders.items():
@@ -476,11 +535,12 @@ def cleanTranslatedText(translatedText, varResponse):
translatedText = resubVars(translatedText, varResponse[1])
return translatedText
+
def elongateCharacters(text):
# Define a pattern to match one character followed by one or more `ー` characters
# Using a positive lookbehind assertion to capture the preceding character
- pattern = r'(?<=(.))ー+'
-
+ pattern = r"(?<=(.))ー+"
+
# Define a replacement function that elongates the captured character
def repl(match):
char = match.group(1) # The character before the ー sequence
@@ -490,6 +550,7 @@ def elongateCharacters(text):
# Use re.sub() to replace the pattern in the text
return re.sub(pattern, repl, text)
+
def extractTranslation(translatedTextList, is_list):
try:
translatedTextList = re.sub(r'\\"+\"([^,\n}])', r'\\"\1', translatedTextList)
@@ -502,15 +563,15 @@ def extractTranslation(translatedTextList, is_list):
return string_list[0]
except Exception as e:
- PBAR.write(f'extractTranslation Error: {e} on String {translatedTextList}')
+ PBAR.write(f"extractTranslation Error: {e} on String {translatedTextList}")
return None
def countTokens(characters, system, user, history):
inputTotalTokens = 0
outputTotalTokens = 0
- enc = tiktoken.encoding_for_model('gpt-4')
-
+ enc = tiktoken.encoding_for_model("gpt-4")
+
# Input
if isinstance(history, list):
for line in history:
@@ -522,26 +583,28 @@ def countTokens(characters, system, user, history):
inputTotalTokens += len(enc.encode(user))
# Output
- outputTotalTokens += round(len(enc.encode(user))*3)
+ outputTotalTokens += round(len(enc.encode(user)) * 3)
return [inputTotalTokens, outputTotalTokens]
+
def combineList(tlist, text):
if isinstance(text, list):
return [t for sublist in tlist for t in sublist]
return tlist[0]
+
@retry(exceptions=Exception, tries=5, delay=5)
def translateGPT(text, history, fullPromptFlag):
global PBAR
-
+
mismatch = False
totalTokens = [0, 0]
if isinstance(text, list):
- format = 'json'
+ format = "json"
tList = batchList(text, BATCHSIZE)
else:
- format = 'text'
+ format = "text"
tList = [text]
for index, tItem in enumerate(tList):
@@ -556,7 +619,7 @@ def translateGPT(text, history, fullPromptFlag):
subbedT = varResponse[0]
# Things to Check before starting translation
- if not re.search(r'[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+', subbedT):
+ if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", subbedT):
if PBAR is not None:
PBAR.update(len(tItem))
continue
@@ -581,9 +644,13 @@ def translateGPT(text, history, fullPromptFlag):
translatedText = cleanTranslatedText(translatedText, varResponse)
if isinstance(tItem, list):
extractedTranslations = extractTranslation(translatedText, True)
- if extractedTranslations == None or len(tItem) != len(extractedTranslations):
+ if extractedTranslations == None or len(tItem) != len(
+ extractedTranslations
+ ):
# Mismatch. Try Again
- response = translateText(characters, system, user, history, 0.05, format)
+ response = translateText(
+ characters, system, user, history, 0.05, format
+ )
translatedText = response.choices[0].message.content
totalTokens[0] += response.usage.prompt_tokens
totalTokens[1] += response.usage.completion_tokens
@@ -592,13 +659,17 @@ def translateGPT(text, history, fullPromptFlag):
translatedText = cleanTranslatedText(translatedText, varResponse)
if isinstance(tItem, list):
extractedTranslations = extractTranslation(translatedText, True)
- if extractedTranslations == None or len(tItem) != len(extractedTranslations):
- mismatch = True # Just here for breakpoint
-
+ if extractedTranslations == None or len(tItem) != len(
+ extractedTranslations
+ ):
+ mismatch = True # Just here for breakpoint
+
# Set if no mismatch
if mismatch == False:
tList[index] = extractedTranslations
- history = extractedTranslations[-10:] # Update history if we have a list
+ history = extractedTranslations[
+ -10:
+ ] # Update history if we have a list
else:
history = text[-10:]
mismatch = False
@@ -609,7 +680,7 @@ def translateGPT(text, history, fullPromptFlag):
PBAR.update(len(tItem))
else:
# Ensure we're passing a single string to extractTranslation
- tList[index] = translatedText
+ tList[index] = translatedText
finalList = combineList(tList, text)
return [finalList, totalTokens]