From fe62cbb11e2ee6aae04154b123320ac7db16af89 Mon Sep 17 00:00:00 2001 From: Dazed Date: Thu, 23 Nov 2023 02:27:07 -0600 Subject: [PATCH] Save latest --- modules/atelier.py | 380 +++++++++++++++++++ modules/main.py | 29 +- modules/rpgmakermvmz.py | 142 ++++---- modules/tyrano.py | 784 ++++++++++++++++++++-------------------- 4 files changed, 867 insertions(+), 468 deletions(-) create mode 100644 modules/atelier.py diff --git a/modules/atelier.py b/modules/atelier.py new file mode 100644 index 0000000..d214749 --- /dev/null +++ b/modules/atelier.py @@ -0,0 +1,380 @@ +import os +from pathlib import Path +import re +import textwrap +import threading +import time +import traceback +import tiktoken +from colorama import Fore +from dotenv import load_dotenv +import openai +from retry import retry +from tqdm import tqdm + +# Open AI +load_dotenv() +if os.getenv('api').replace(' ', '') != '': + openai.api_base = os.getenv('api') +openai.organization = os.getenv('org') +openai.api_key = os.getenv('key') + +#Globals +MODEL = os.getenv('model') +TIMEOUT = int(os.getenv('timeout')) +LANGUAGE=os.getenv('language').capitalize() +INPUTAPICOST = .002 # Depends on the model https://openai.com/pricing +OUTPUTAPICOST = .002 +PROMPT = Path('prompt.txt').read_text(encoding='utf-8') +THREADS = int(os.getenv('threads')) # Controls how many threads are working on a single file (May have to drop this) +LOCK = threading.Lock() +WIDTH = int(os.getenv('width')) +LISTWIDTH = int(os.getenv('listWidth')) +NOTEWIDTH = 40 +MAXHISTORY = 10 +ESTIMATE = '' +totalTokens = [0, 0] +NAMESLIST = [] + +#tqdm Globals +BAR_FORMAT='{l_bar}{bar:10}{r_bar}{bar:-10b}' +POSITION=0 +LEAVE=False + +# Translation Flags +FIXTEXTWRAP = True +IGNORETLTEXT = True + +def handleAtelier(filename, estimate): + global ESTIMATE, totalTokens + ESTIMATE = estimate + + if estimate: + start = time.time() + translatedData = openFiles(filename) + + # Print Result + end = time.time() + tqdm.write(getResultString(translatedData, end - start, filename)) + with LOCK: + totalTokens[0] += translatedData[1][0] + totalTokens[1] += translatedData[1][1] + + return getResultString(['', totalTokens, None], end - start, 'TOTAL') + + else: + try: + with open('translated/' + filename, 'w', encoding='utf-8'): + start = time.time() + translatedData = openFiles(filename) + + # Print Result + end = time.time() + tqdm.write(getResultString(translatedData, end - start, filename)) + with LOCK: + totalTokens[0] += translatedData[1][0] + totalTokens[1] += translatedData[1][1] + except Exception: + return 'Fail' + + return getResultString(['', totalTokens, None], end - start, 'TOTAL') + +def openFiles(filename): + with open('files/' + filename, 'r', encoding='UTF-8') as f: + translatedData = parseText(f, filename) + + return translatedData + +def getResultString(translatedData, translationTime, filename): + # File Print String + totalTokenstring =\ + Fore.YELLOW +\ + '[Input: ' + str(translatedData[1][0]) + ']'\ + '[Output: ' + str(translatedData[1][1]) + ']'\ + '[Cost: ${:,.4f}'.format((translatedData[1][0] * .001 * INPUTAPICOST) +\ + (translatedData[1][1] * .001 * OUTPUTAPICOST)) + ']' + timeString = Fore.BLUE + '[' + str(round(translationTime, 1)) + 's]' + + if translatedData[2] is None: + # Success + return filename + ': ' + totalTokenstring + timeString + Fore.GREEN + u' \u2713 ' + Fore.RESET + + else: + # Fail + try: + raise translatedData[2] + except Exception as e: + errorString = str(e) + Fore.RED + return filename + ': ' + totalTokenstring + timeString + Fore.RED + u' \u2717 ' +\ + errorString + Fore.RESET + +def parseText(data, filename): + totalLines = 0 + global LOCK + + # Get total for progress bar + linesList = data.readlines() + totalLines = len(linesList) + + with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar: + pbar.desc=filename + pbar.total=totalLines + try: + response = translateText(linesList, pbar) + except Exception as e: + traceback.print_exc() + return [linesList, 0, e] + return [response[0], response[1], None] + +def translateText(data, pbar): + textHistory = [] + maxHistory = MAXHISTORY + totalTokens = [0,0] + syncIndex = 0 + + for i in range(len(data)): + if syncIndex > i: + i = syncIndex + + match = re.findall(r'◆.+◆(.+)', data[i]) + if len(match) > 0: + jaString = match[0] + + ### Translate + # Remove any textwrap + finalJAString = re.sub(r'\\n', ' ', jaString) + + # Translate + response = translateGPT(finalJAString, 'Previous Text for Context: ' + ' '.join(textHistory), True) + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] + translatedText = response[0] + + # TextHistory is what we use to give GPT Context, so thats appended here. + textHistory.append('\"' + translatedText + '\"') + + # Keep textHistory list at length maxHistory + if len(textHistory) > maxHistory: + textHistory.pop(0) + + # Textwrap + translatedText = translatedText.replace('\"', '\\"') + translatedText = textwrap.fill(translatedText, width=WIDTH) + + # Write + data[i] = data[i].replace(match[0], translatedText) + + syncIndex = i + 1 + pbar.update() + return [data, totalTokens] + +def subVars(jaString): + jaString = jaString.replace('\u3000', ' ') + + # Nested + count = 0 + nestedList = re.findall(r'[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]', jaString) + nestedList = set(nestedList) + if len(nestedList) != 0: + for icon in nestedList: + jaString = jaString.replace(icon, '{Nested_' + str(count) + '}') + count += 1 + + # Icons + count = 0 + iconList = re.findall(r'[\\]+[iIkKwWaA]+\[[0-9]+\]', jaString) + iconList = set(iconList) + if len(iconList) != 0: + for icon in iconList: + jaString = jaString.replace(icon, '{Ascii_' + str(count) + '}') + count += 1 + + # Colors + count = 0 + colorList = re.findall(r'[\\]+[cC]\[[0-9]+\]', jaString) + colorList = set(colorList) + if len(colorList) != 0: + for color in colorList: + jaString = jaString.replace(color, '{Color_' + str(count) + '}') + count += 1 + + # Names + count = 0 + nameList = re.findall(r'[\\]+[nN]\[.+?\]+', jaString) + nameList = set(nameList) + if len(nameList) != 0: + for name in nameList: + jaString = jaString.replace(name, '{N_' + str(count) + '}') + count += 1 + + # Variables + count = 0 + varList = re.findall(r'[\\]+[vV]\[[0-9]+\]', jaString) + varList = set(varList) + if len(varList) != 0: + for var in varList: + jaString = jaString.replace(var, '{Var_' + str(count) + '}') + count += 1 + + # Formatting + count = 0 + if '笑えるよね.' in jaString: + print('t') + formatList = re.findall(r'[\\]+[\w]+\[.+?\]', jaString) + formatList = set(formatList) + if len(formatList) != 0: + for var in formatList: + jaString = jaString.replace(var, '{FCode_' + str(count) + '}') + count += 1 + + # Put all lists in list and return + allList = [nestedList, iconList, colorList, nameList, varList, formatList] + return [jaString, allList] + +def resubVars(translatedText, allList): + # Fix Spacing and ChatGPT Nonsense + matchList = re.findall(r'\[\s?.+?\s?\]', translatedText) + if len(matchList) > 0: + for match in matchList: + text = match.strip() + translatedText = translatedText.replace(match, text) + + # Nested + count = 0 + if len(allList[0]) != 0: + for var in allList[0]: + translatedText = translatedText.replace('{Nested_' + str(count) + '}', var) + count += 1 + + # Icons + count = 0 + if len(allList[1]) != 0: + for var in allList[1]: + translatedText = translatedText.replace('{Ascii_' + str(count) + '}', var) + count += 1 + + # Colors + count = 0 + if len(allList[2]) != 0: + for var in allList[2]: + translatedText = translatedText.replace('{Color_' + str(count) + '}', var) + count += 1 + + # Names + count = 0 + if len(allList[3]) != 0: + for var in allList[3]: + translatedText = translatedText.replace('{N_' + str(count) + '}', var) + count += 1 + + # Vars + count = 0 + if len(allList[4]) != 0: + for var in allList[4]: + translatedText = translatedText.replace('{Var_' + str(count) + '}', var) + count += 1 + + # Formatting + count = 0 + if len(allList[5]) != 0: + for var in allList[5]: + translatedText = translatedText.replace('{FCode_' + str(count) + '}', var) + count += 1 + + # Remove Color Variables Spaces + # if '\\c' in translatedText: + # translatedText = re.sub(r'\s*(\\+c\[[1-9]+\])\s*', r' \1', translatedText) + # translatedText = re.sub(r'\s*(\\+c\[0+\])', r'\1', translatedText) + return translatedText + +@retry(exceptions=Exception, tries=5, delay=5) +def translateGPT(t, history, fullPromptFlag): + # Sub Vars + varResponse = subVars(t) + subbedT = varResponse[0] + + # If there isn't any Japanese in the text just skip + if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴ]+|[\uFF00-\uFFEF]', subbedT): + return(t, [0,0]) + + # If ESTIMATE is True just count this as an execution and return. + if ESTIMATE: + enc = tiktoken.encoding_for_model(MODEL) + historyRaw = '' + if isinstance(history, list): + for line in history: + historyRaw += line + else: + historyRaw = history + + inputTotalTokens = len(enc.encode(historyRaw)) + len(enc.encode(PROMPT)) + outputTotalTokens = len(enc.encode(t)) * 2 # Estimating 2x the size of the original text + totalTokens = [inputTotalTokens, outputTotalTokens] + return (t, totalTokens) + + # Characters + context = 'Game Characters:\ + Character: リッカ == Ricca - Gender: Female\ + Character: シーナ == Sina - Gender: Female\ + Character: ヘレナ == Helena - Gender: Female\ + Character: Miko == Miko - Gender: Female' + + # Prompt + if fullPromptFlag: + system = PROMPT + user = 'Line to Translate = ' + subbedT + else: + system = 'Output ONLY the '+ LANGUAGE +' translation in the following format: `Translation: <'+ LANGUAGE.upper() +'_TRANSLATION>`' + user = 'Line to Translate = ' + subbedT + + # Create Message List + msg = [] + msg.append({"role": "system", "content": system}) + msg.append({"role": "user", "content": context}) + if isinstance(history, list): + for line in history: + msg.append({"role": "user", "content": line}) + else: + msg.append({"role": "user", "content": history}) + msg.append({"role": "user", "content": user}) + + response = openai.ChatCompletion.create( + temperature=0, + frequency_penalty=0.2, + presence_penalty=0.2, + model=MODEL, + messages=msg, + request_timeout=TIMEOUT, + ) + + # Save Translated Text + translatedText = response.choices[0].message.content + totalTokens = [response.usage.prompt_tokens, response.usage.completion_tokens] + + # Resub Vars + translatedText = resubVars(translatedText, varResponse[1]) + + # Remove Placeholder Text + translatedText = translatedText.replace(LANGUAGE +' Translation: ', '') + translatedText = translatedText.replace('Translation: ', '') + translatedText = translatedText.replace('Line to Translate = ', '') + translatedText = translatedText.replace('Translation = ', '') + translatedText = translatedText.replace('Translate = ', '') + translatedText = translatedText.replace(LANGUAGE +' Translation:', '') + translatedText = translatedText.replace('Translation:', '') + translatedText = translatedText.replace('Line to Translate =', '') + translatedText = translatedText.replace('Translation =', '') + translatedText = translatedText.replace('Translate =', '') + translatedText = translatedText.replace('っ', '') + translatedText = translatedText.replace('ッ', '') + translatedText = translatedText.replace('ぁ', '') + translatedText = translatedText.replace('。', '.') + translatedText = translatedText.replace('、', ',') + translatedText = translatedText.replace('？', '?') + translatedText = translatedText.replace('！', '!') + + # Return Translation + if len(translatedText) > 15 * len(t) or "I'm sorry, but I'm unable to assist with that translation" in translatedText: + raise Exception + else: + return [translatedText, totalTokens] \ No newline at end of file diff --git a/modules/main.py b/modules/main.py index 5bbc342..8520a6d 100644 --- a/modules/main.py +++ b/modules/main.py @@ -12,8 +12,8 @@ from modules.txt import handleTXT from modules.tyrano import handleTyrano from modules.json import handleJSON from modules.kansen import handleKansen -from modules.lune import handleLune from modules.lune2 import handleLuneTxt +from modules.atelier import handleAtelier # For GPT4 rate limit will be hit if you have more than 1 thread. # 1 Thread for each file. Controls how many files are worked on at once. @@ -30,14 +30,17 @@ def main(): while estimate == '': estimate = input('Select Translation or Cost Estimation:\n\n1. Translate\n2. Estimate\n') match estimate: - case '1': estimate = False - case '2': estimate = True - case _: estimate = '' + case '1': + estimate = False + case '2': + estimate = True + case _: + estimate = '' totalCost = 0 version = '' while version == '': - version = input('Select the RPGMaker Version:\n\n1. MV/MZ\n2. ACE\n3. CSV (From Translator++)\n4. Text (Custom)\n5. Tyrano\n6. JSON\n7. Kansen\n8. Lune\n') + version = input('Select the RPGMaker Version:\n\n1. MV/MZ\n2. ACE\n3. CSV (From Translator++)\n4. Text (Custom)\n5. Tyrano\n6. JSON\n7. Kansen\n8. Lune\n9. Atelier\n') match version: case '1': # Open File (Threads) @@ -149,11 +152,25 @@ def main(): tracebackLineNo = str(traceback.extract_tb(sys.exc_info()[2])[-1].lineno) tqdm.write(Fore.RED + str(e) + '|' + tracebackLineNo + Fore.RESET) + case '9': + # Open File (Threads) + with ThreadPoolExecutor(max_workers=THREADS) as executor: + futures = [executor.submit(handleAtelier, filename, estimate) \ + for filename in os.listdir("files") if filename.endswith('txt')] + + for future in as_completed(futures): + try: + totalCost = future.result() + + except Exception as e: + tracebackLineNo = str(traceback.extract_tb(sys.exc_info()[2])[-1].lineno) + tqdm.write(Fore.RED + str(e) + '|' + tracebackLineNo + Fore.RESET) + case _: version = '' if totalCost != 'Fail': - if estimate == False: + if estimate is False: # This is to encourage people to grab what's in /translated instead deleteFolderFiles('files') diff --git a/modules/rpgmakermvmz.py b/modules/rpgmakermvmz.py index bb0898e..e897e9c 100644 --- a/modules/rpgmakermvmz.py +++ b/modules/rpgmakermvmz.py @@ -3,30 +3,28 @@ import json import os from pathlib import Path import re -import sys import textwrap import threading import time import traceback import tiktoken - from colorama import Fore from dotenv import load_dotenv import openai from retry import retry from tqdm import tqdm -#Globals +# Open AI load_dotenv() if os.getenv('api').replace(' ', '') != '': openai.api_base = os.getenv('api') - openai.organization = os.getenv('org') openai.api_key = os.getenv('key') + +#Globals MODEL = os.getenv('model') TIMEOUT = int(os.getenv('timeout')) LANGUAGE=os.getenv('language').capitalize() - INPUTAPICOST = .002 # Depends on the model https://openai.com/pricing OUTPUTAPICOST = .002 PROMPT = Path('prompt.txt').read_text(encoding='utf-8') @@ -49,17 +47,17 @@ LEAVE=False CODE401 = True CODE405 = False CODE102 = True -CODE122 = True +CODE122 = False CODE101 = False CODE355655 = False CODE357 = False CODE657 = False -CODE356 = True +CODE356 = False CODE320 = False CODE324 = False CODE111 = False CODE408 = False -CODE108 = True +CODE108 = False NAMES = False # Output a list of all the character names found BRFLAG = False # If the game uses
instead FIXTEXTWRAP = True @@ -76,7 +74,7 @@ def handleMVMZ(filename, estimate): # Print Result end = time.time() tqdm.write(getResultString(translatedData, end - start, filename)) - if NAMES == True: + if NAMES is True: tqdm.write(str(NAMESLIST)) with LOCK: totalTokens[0] += translatedData[1][0] @@ -86,7 +84,7 @@ def handleMVMZ(filename, estimate): else: try: - with open('translated/' + filename, 'w', encoding='UTF-8') as outFile: + with open('translated/' + filename, 'w', encoding='utf-8') as outFile: start = time.time() translatedData = openFiles(filename) @@ -97,13 +95,13 @@ def handleMVMZ(filename, estimate): with LOCK: totalTokens[0] += translatedData[1][0] totalTokens[1] += translatedData[1][1] - except Exception as e: + except Exception: return 'Fail' return getResultString(['', totalTokens, None], end - start, 'TOTAL') def openFiles(filename): - with open('files/' + filename, 'r', encoding='UTF-8') as f: + with open('files/' + filename, 'r', encoding='utf-8-sig') as f: data = json.load(f) # Map Files @@ -177,7 +175,7 @@ def getResultString(translatedData, translationTime, filename): (translatedData[1][1] * .001 * OUTPUTAPICOST)) + ']' timeString = Fore.BLUE + '[' + str(round(translationTime, 1)) + 's]' - if translatedData[2] == None: + if translatedData[2] is None: # Success return filename + ': ' + totalTokenstring + timeString + Fore.GREEN + u' \u2713 ' + Fore.RESET @@ -431,7 +429,7 @@ def searchThings(name, pbar): totalTokens = [0, 0] # If there isn't any Japanese in the text just skip - if IGNORETLTEXT == True: + if IGNORETLTEXT is True: if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+', name['name']) and re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+', name['description']): pbar.update(1) return totalTokens @@ -551,7 +549,6 @@ def searchCodes(page, pbar): maxHistory = MAXHISTORY totalTokens = [0, 0] speaker = '' - speakerVar = '' nametag = '' match = [] syncIndex = 0 @@ -576,7 +573,7 @@ def searchCodes(page, pbar): ### IF these crash or fail your game will do the same. Use the flags to skip codes. ## Event Code: 401 Show Text - if codeList[i]['code'] == 401 and CODE401 == True or codeList[i]['code'] == 405 and CODE405: + if codeList[i]['code'] == 401 and CODE401 is True or codeList[i]['code'] == 405 and CODE405: # Use this to place text later code = codeList[i]['code'] j = i @@ -584,13 +581,12 @@ def searchCodes(page, pbar): # Grab String if len(codeList[i]['parameters']) > 0: jaString = codeList[i]['parameters'][0] - firstJAString = jaString else: codeList[i]['code'] = -1 continue # If there isn't any Japanese in the text just skip - if IGNORETLTEXT == True: + if IGNORETLTEXT is True: if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+', jaString): # Keep textHistory list at length maxHistory textHistory.append('\"' + jaString + '\"') @@ -732,25 +728,23 @@ def searchCodes(page, pbar): finalJAString = finalJAString.replace(matchList[0], '') # Remove any textwrap - if FIXTEXTWRAP == True: + if FIXTEXTWRAP is True: finalJAString = re.sub(r'\n', ' ', finalJAString) finalJAString = finalJAString.replace('
', ' ') # Remove Extra Stuff finalJAString = finalJAString.replace('ﾞ', '') - finalJAString = finalJAString.replace('。', '.') - finalJAString = finalJAString.replace('？', '?') - finalJAString = finalJAString.replace('！', '!') finalJAString = finalJAString.replace('・', '.') finalJAString = finalJAString.replace('‶', '') finalJAString = finalJAString.replace('”', '') finalJAString = finalJAString.replace('―', '-') + finalJAString = finalJAString.replace('ー', '-') finalJAString = finalJAString.replace('…', '...') finalJAString = finalJAString.replace('　', '') # finalJAString = finalJAString.replace('〇', '*') # Remove any RPGMaker Code at start - ffMatchList = re.findall(r'[\\]+[fF]+\[.+?\]', finalJAString) + ffMatchList = re.findall(r'[\\]+[fFaA]+\[.+?\]', finalJAString) if len(ffMatchList) > 0: finalJAString = finalJAString.replace(ffMatchList[0], '') nametag += ffMatchList[0] @@ -785,29 +779,27 @@ def searchCodes(page, pbar): # Sub Vars varResponse = subVars(translatedText) - subbedT = varResponse[0] textHistory.append('\"' + varResponse[0] + '\"') elif finalJAString != '': response = translateGPT(speaker + ' | ' + finalJAString, textHistory, True) totalTokens[0] += response[1][0] totalTokens[1] += response[1][1] translatedText = response[0] - + # Remove added speaker translatedText = re.sub(r'(^.+?)\s?[|:]\s?', '', translatedText) # Sub Vars varResponse = subVars(translatedText) - subbedT = varResponse[0] textHistory.append('\"' + speaker + ' | ' + varResponse[0] + '\"') speaker = '' else: translatedText = finalJAString # Textwrap - if FIXTEXTWRAP == True: + if FIXTEXTWRAP is True: translatedText = textwrap.fill(translatedText, width=WIDTH) - if BRFLAG == True: + if BRFLAG is True: translatedText = translatedText.replace('\n', '
') # Add Beginning Text @@ -834,14 +826,13 @@ def searchCodes(page, pbar): currentGroup = [] ## Event Code: 122 [Set Variables] - if codeList[i]['code'] == 122 and CODE122 == True: + if codeList[i]['code'] == 122 and CODE122 is True: # This is going to be the var being set. (IMPORTANT) - varNum = codeList[i]['parameters'][0] # if varNum not in [1178]: # continue jaString = codeList[i]['parameters'][4] - if type(jaString) != str: + if isinstance(jaString, str): continue # Definitely don't want to mess with files @@ -880,11 +871,11 @@ def searchCodes(page, pbar): # Set Data codeList[i]['parameters'][4] = translatedText - ## Event Code: 357 [Picture Text] [Optional] - if codeList[i]['code'] == 357 and CODE357 == True: + ## Event Code: 357 [Picture Text] [Optional] + if codeList[i]['code'] == 357 and CODE357 is True: if 'message' in codeList[i]['parameters'][3]: jaString = codeList[i]['parameters'][3]['message'] - if type(jaString) != str: + if not isinstance(jaString, str): continue # Definitely don't want to mess with files @@ -899,8 +890,10 @@ def searchCodes(page, pbar): oldjaString = jaString startString = re.search(r'^[^一-龠ぁ-ゔァ-ヴー【】（）「」a-zA-ZＡ-Ｚ０-９\\]+', jaString) finalJAString = re.sub(r'^[^一-龠ぁ-ゔァ-ヴー【】（）「」a-zA-ZＡ-Ｚ０-９\\]+', '', jaString) - if startString is None: startString = '' - else: startString = startString.group() + if startString is None: + startString = '' + else: + startString = startString.group() # Remove any textwrap finalJAString = re.sub(r'\n', ' ', finalJAString) @@ -917,11 +910,11 @@ def searchCodes(page, pbar): # Set Data codeList[i]['parameters'][3]['message'] = startString + translatedText - ## Event Code: 657 [Picture Text] [Optional] - if codeList[i]['code'] == 657 and CODE657 == True: + ## Event Code: 657 [Picture Text] [Optional] + if codeList[i]['code'] == 657 and CODE657 is True: if 'text' in codeList[i]['parameters'][0]: jaString = codeList[i]['parameters'][0] - if type(jaString) != str: + if not isinstance(jaString, str): continue # Definitely don't want to mess with files @@ -937,10 +930,14 @@ def searchCodes(page, pbar): jaString = re.sub(r'^[^一-龠ぁ-ゔァ-ヴー\<\>【】\\]+', '', jaString) endString = re.search(r'[^一-龠ぁ-ゔァ-ヴー\<\>【】。！？\\]+$', jaString) jaString = re.sub(r'[^一-龠ぁ-ゔァ-ヴー\<\>【】。！？\\]+$', '', jaString) - if startString is None: startString = '' - else: startString = startString.group() - if endString is None: endString = '' - else: endString = endString.group() + if startString is None: + startString = '' + else: + startString = startString.group() + if endString is None: + endString = '' + else: + endString = endString.group() # Remove any textwrap jaString = re.sub(r'\n', ' ', jaString) @@ -966,12 +963,12 @@ def searchCodes(page, pbar): codeList[i]['parameters'][0] = translatedText ## Event Code: 101 [Name] [Optional] - if codeList[i]['code'] == 101 and CODE101 == True: + if codeList[i]['code'] == 101 and CODE101 is True: # Grab String jaString = '' if len(codeList[i]['parameters']) > 4: jaString = codeList[i]['parameters'][4] - if type(jaString) != str: + if not isinstance(jaString, str): continue # Force Speaker @@ -1032,7 +1029,7 @@ def searchCodes(page, pbar): NAMESLIST.append(speaker) ## Event Code: 355 or 655 Scripts [Optional] - if (codeList[i]['code'] == 355 or codeList[i]['code'] == 655) and CODE355655 == True: + if (codeList[i]['code'] == 355 or codeList[i]['code'] == 655) and CODE355655 is True: jaString = codeList[i]['parameters'][0] # If there isn't any Japanese in the text just skip @@ -1072,7 +1069,7 @@ def searchCodes(page, pbar): codeList[i]['parameters'][0] = translatedText ## Event Code: 408 (Script) - if (codeList[i]['code'] == 408) and CODE408 == True: + if (codeList[i]['code'] == 408) and CODE408 is True: jaString = codeList[i]['parameters'][0] # If there isn't any Japanese in the text just skip @@ -1112,7 +1109,7 @@ def searchCodes(page, pbar): codeList[i]['parameters'][0] = translatedText ## Event Code: 108 (Script) - if (codeList[i]['code'] == 108) and CODE108 == True: + if (codeList[i]['code'] == 108) and CODE108 is True: jaString = codeList[i]['parameters'][0] # If there isn't any Japanese in the text just skip @@ -1145,7 +1142,7 @@ def searchCodes(page, pbar): codeList[i]['parameters'][0] = translatedText ## Event Code: 356 - if codeList[i]['code'] == 356 and CODE356 == True: + if codeList[i]['code'] == 356 and CODE356 is True: jaString = codeList[i]['parameters'][0] oldjaString = jaString @@ -1433,7 +1430,7 @@ def searchCodes(page, pbar): continue ### Event Code: 102 Show Choice - if codeList[i]['code'] == 102 and CODE102 == True: + if codeList[i]['code'] == 102 and CODE102 is True: for choice in range(len(codeList[i]['parameters'][0])): jaString = codeList[i]['parameters'][0][choice] jaString = jaString.replace(' 。', '.') @@ -1466,12 +1463,12 @@ def searchCodes(page, pbar): codeList[i]['parameters'][0][choice] = startString + translatedText + endString ### Event Code: 111 Script - if codeList[i]['code'] == 111 and CODE111 == True: + if codeList[i]['code'] == 111 and CODE111 is True: for j in range(len(codeList[i]['parameters'])): jaString = codeList[i]['parameters'][j] # Check if String - if type(jaString) != str: + if not isinstance(jaString, str): continue # Only TL the Game Variable @@ -1503,9 +1500,9 @@ def searchCodes(page, pbar): codeList[i]['parameters'][j] = translatedText ### Event Code: 320 Set Variable - if codeList[i]['code'] == 320 and CODE320 == True: + if codeList[i]['code'] == 320 and CODE320 is True: jaString = codeList[i]['parameters'][1] - if type(jaString) != str: + if not isinstance(jaString, str): continue # Definitely don't want to mess with files @@ -1544,7 +1541,7 @@ def searchCodes(page, pbar): # raise Exception(str(e) + '|Line:' + tracebackLineNo) except Exception as e: traceback.print_exc() - raise Exception(str(e) + 'Failed to translate: ' + oldjaString) + raise Exception(str(e) + 'Failed to translate: ' + oldjaString) from None # Append leftover groups in 401 if len(currentGroup) > 0: @@ -1576,7 +1573,6 @@ def searchCodes(page, pbar): return totalTokens def searchSS(state, pbar): - '''Searches skills and states json files''' totalTokens = [0, 0] # Name @@ -1791,7 +1787,7 @@ def subVars(jaString): count = 0 if '笑えるよね.' in jaString: print('t') - formatList = re.findall(r'[\\]+CL', jaString) + formatList = re.findall(r'[\\]+[\w]+\[.+?\]', jaString) formatList = set(formatList) if len(formatList) != 0: for var in formatList: @@ -1860,6 +1856,14 @@ def resubVars(translatedText, allList): @retry(exceptions=Exception, tries=5, delay=5) def translateGPT(t, history, fullPromptFlag): + # Sub Vars + varResponse = subVars(t) + subbedT = varResponse[0] + + # If there isn't any Japanese in the text just skip + if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴ]+|[\uFF00-\uFFEF]', subbedT): + return(t, [0,0]) + # If ESTIMATE is True just count this as an execution and return. if ESTIMATE: enc = tiktoken.encoding_for_model(MODEL) @@ -1874,23 +1878,13 @@ def translateGPT(t, history, fullPromptFlag): outputTotalTokens = len(enc.encode(t)) * 2 # Estimating 2x the size of the original text totalTokens = [inputTotalTokens, outputTotalTokens] return (t, totalTokens) - - # Sub Vars - varResponse = subVars(t) - subbedT = varResponse[0] - - # If there isn't any Japanese in the text just skip - if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴ]+|[\uFF00-\uFFEF]', subbedT): - return(t, [0,0]) # Characters context = 'Game Characters:\ - Character: アーテル == Artel - Gender: Female\ - Character: コリウス == Coleus - Gender: Male\ - Character: ギア == Gear - Gender: Male\ - Character: ロック == Rock - Gender: Male\ - Character: テア == Thea - Gender: Male\ - Character: ヘンジ == Henge - Gender: Male' + Character: リッカ == Ricca - Gender: Female\ + Character: シーナ == Sina - Gender: Female\ + Character: ヘレナ == Helena - Gender: Female\ + Character: Miko == Miko - Gender: Female' # Prompt if fullPromptFlag: @@ -1939,6 +1933,12 @@ def translateGPT(t, history, fullPromptFlag): translatedText = translatedText.replace('Translation =', '') translatedText = translatedText.replace('Translate =', '') translatedText = translatedText.replace('っ', '') + translatedText = translatedText.replace('ッ', '') + translatedText = translatedText.replace('ぁ', '') + translatedText = translatedText.replace('。', '.') + translatedText = translatedText.replace('、', ',') + translatedText = translatedText.replace('？', '?') + translatedText = translatedText.replace('！', '!') # Return Translation if len(translatedText) > 15 * len(t) or "I'm sorry, but I'm unable to assist with that translation" in translatedText: diff --git a/modules/tyrano.py b/modules/tyrano.py index 3c0343c..259b8ab 100644 --- a/modules/tyrano.py +++ b/modules/tyrano.py @@ -1,55 +1,59 @@ -from concurrent.futures import ThreadPoolExecutor, as_completed import os -from pathlib import Path import re import textwrap import threading import time import traceback -import tiktoken +from pathlib import Path +import openai +import tiktoken from colorama import Fore from dotenv import load_dotenv -import openai from retry import retry from tqdm import tqdm -#Globals +# Open AI load_dotenv() -if os.getenv('api').replace(' ', '') != '': - openai.api_base = os.getenv('api') +if os.getenv("api").replace(" ", "") != "": + openai.api_base = os.getenv("api") +openai.organization = os.getenv("org") +openai.api_key = os.getenv("key") -openai.organization = os.getenv('org') -openai.api_key = os.getenv('key') -MODEL = os.getenv('model') -TIMEOUT = int(os.getenv('timeout')) -LANGUAGE=os.getenv('language').capitalize() - -APICOST = .002 # Depends on the model https://openai.com/pricing -PROMPT = Path('prompt.txt').read_text(encoding='utf-8') -THREADS = int(os.getenv('threads')) # For GPT4 rate limit will be hit if you have more than 1 thread. +# Globals +MODEL = os.getenv("model") +TIMEOUT = int(os.getenv("timeout")) +LANGUAGE = os.getenv("language").capitalize() +INPUTAPICOST = 0.002 # Depends on the model https://openai.com/pricing +OUTPUTAPICOST = 0.002 +PROMPT = Path("prompt.txt").read_text(encoding="utf-8") +THREADS = int( + os.getenv("threads") +) # Controls how many threads are working on a single file (May have to drop this) LOCK = threading.Lock() -WIDTH = int(os.getenv('width')) -LISTWIDTH = int(os.getenv('listWidth')) +WIDTH = int(os.getenv("width")) +LISTWIDTH = int(os.getenv("listWidth")) +NOTEWIDTH = 40 MAXHISTORY = 10 -ESTIMATE = '' -TOTALCOST = 0 -TOKENS = 0 -TOTALTOKENS = 0 +ESTIMATE = "" +totalTokens = [0, 0] NAMESLIST = [] -#tqdm Globals -BAR_FORMAT='{l_bar}{bar:10}{r_bar}{bar:-10b}' -POSITION=0 -LEAVE=False +# tqdm Globals +BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}" +POSITION = 0 +LEAVE = False # Flags -NAMES = False # Output a list of all the character names found +NAMES = False # Output a list of all the character names found +BRFLAG = False # If the game uses
instead FIXTEXTWRAP = True -IGNORETLTEXT = True +IGNORETLTEXT = False + def handleTyrano(filename, estimate): - global ESTIMATE, TOKENS, TOTALTOKENS, TOTALCOST + global ESTIMATE + totalTokens = [0, 0] ESTIMATE = estimate if estimate: @@ -59,416 +63,390 @@ def handleTyrano(filename, estimate): # Print Result end = time.time() tqdm.write(getResultString(translatedData, end - start, filename)) + if NAMES is True: + tqdm.write(str(NAMESLIST)) with LOCK: - TOTALCOST += translatedData[1] * .001 * APICOST - TOTALTOKENS += translatedData[1] + totalTokens[0] += translatedData[1][0] + totalTokens[1] += translatedData[1][1] + + return getResultString(["", totalTokens, None], end - start, "TOTAL") - return getResultString(['', TOTALTOKENS, None], end - start, 'TOTAL') - else: try: - with open('translated/' + filename, 'w', encoding='utf-8', errors='ignore') as outFile: + with open("translated/" + filename, "w", encoding="utf-8") as outFile: start = time.time() translatedData = openFiles(filename) + outFile.writelines(translatedData[0]) # Print Result - outFile.writelines(translatedData[0]) end = time.time() tqdm.write(getResultString(translatedData, end - start, filename)) with LOCK: - TOTALCOST += translatedData[1] * .001 * APICOST - TOTALTOKENS += translatedData[1] - except Exception as e: + totalTokens[0] += translatedData[1][0] + totalTokens[1] += translatedData[1][1] + except Exception: traceback.print_exc() - return 'Fail' + return "Fail" - return getResultString(['', TOTALTOKENS, None], end - start, 'TOTAL') + return getResultString(["", totalTokens, None], end - start, "TOTAL") -def openFiles(filename): - with open('files/' + filename, 'r', encoding='utf-8') as readFile: - translatedData = parseTyrano(readFile, filename) - # Delete lines marked for deletion - finalData = [] - for line in translatedData[0]: - if line != '\\d\n': - finalData.append(line) - translatedData[0] = finalData - - return translatedData - -def parseTyrano(readFile, filename): - totalTokens = 0 - totalLines = 0 - - # Get total for progress bar - data = readFile.readlines() - totalLines = len(data) - - with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar: - pbar.desc=filename - pbar.total=totalLines - - try: - totalTokens += translateTyrano(data, pbar) - except Exception as e: - traceback.print_exc() - return [data, totalTokens, e] - return [data, totalTokens, None] - -def translateTyrano(data, pbar): - textHistory = [] - maxHistory = MAXHISTORY - tokens = 0 - currentGroup = [] - syncIndex = 0 - speaker = '' - global LOCK, ESTIMATE - - for i in range(len(data)): - if syncIndex > i: - i = syncIndex - - # Speaker - if '#' in data[i]: - matchList = re.findall(r'#(.+)', data[i]) - if len(matchList) != 0: - response = translateGPT(matchList[0], 'Reply with only the '+ LANGUAGE +' translation of the NPC name', True) - speaker = response[0] - tokens += response[1] - data[i] = '#' + speaker + '\n' - else: - speaker = '' - - # Choices - elif 'glink' in data[i]: - matchList = re.findall(r'\[glink.+text=\"(.+?)\".+', data[i]) - if len(matchList) != 0: - if len(textHistory) > 0: - originalText = matchList[0] - response = translateGPT(matchList[0], 'Past Translated Text: ' + textHistory[len(textHistory)-1] + '\n\nReply in the style of a dialogue option.', True) - else: - response = translateGPT(matchList[0], '', False) - translatedText = response[0] - tokens += response[1] - - # Remove characters that may break scripts - charList = ['.', '\"', '\\n'] - for char in charList: - translatedText = translatedText.replace(char, '') - - # Escape all ' - translatedText = translatedText.replace('\\', '') - translatedText = translatedText.replace("'", "\\\'") - - # Set Data - translatedText = data[i].replace(matchList[0], translatedText.replace(' ', '\u00A0')) - data[i] = translatedText - - # Grab Lines - matchList = re.findall(r'(.+?)\[p\]', data[i]) - if len(matchList) > 0: - matchList[0] = matchList[0].replace('「', '') - matchList[0] = matchList[0].replace('」', '') - currentGroup.append(matchList[0]) - if len(data) > i+1: - while '[r]' in data[i+1]: - data[i] = '\d\n' # \d Marks line for deletion - i += 1 - matchList = re.findall(r'(.+?)\[p\]', data[i]) - if len(matchList) > 0: - matchList[0] = matchList[0].replace('「', '') - matchList[0] = matchList[0].replace('」', '') - currentGroup.append(matchList[0]) - while '[p][cm]' in data[i+1]: - data[i] = '\d\n' - i += 1 - matchList = re.findall(r'(.+?)\[p\]', data[i]) - if len(matchList) > 0: - matchList[0] = matchList[0].replace('「', '') - matchList[0] = matchList[0].replace('」', '') - currentGroup.append(matchList[0]) - # Join up 401 groups for better translation. - if len(currentGroup) > 0: - finalJAString = ' '.join(currentGroup) - oldjaString = finalJAString - - # Remove any textwrap - if FIXTEXTWRAP == True: - finalJAString = finalJAString.replace('[r]', ' ') - - #Check Speaker - if speaker == '': - response = translateGPT(finalJAString, textHistory, True) - tokens += response[1] - translatedText = response[0] - textHistory.append('\"' + translatedText + '\"') - else: - response = translateGPT(speaker + ': ' + finalJAString, textHistory, True) - tokens += response[1] - translatedText = response[0] - textHistory.append('\"' + translatedText + '\"') - - # Remove added speaker - translatedText = re.sub(r'^.+:\s?', '', translatedText) - - # Set Data - translatedText = translatedText.replace('ッ', '') - translatedText = translatedText.replace('っ', '') - translatedText = translatedText.replace('ー', '') - translatedText = translatedText.replace('\"', '') - translatedText = translatedText.replace('[', '') - translatedText = translatedText.replace(']', '') - - # Split final string into full sentences. - matchList = re.findall(r'(.+?[)\.\?\!）。・]+)', translatedText) - translatedText = re.sub(r'(.+?[)\.\?\!）。・]+)', '', translatedText) - for l in range(len(matchList)): - if any(t in matchList[l] for t in ['Mr.', 'Ms.', 'Mrs.']): - if len(matchList) > l+1: - matchList[l] = matchList[l] + matchList[l+1] - matchList[l+1] = '' - - # Combine Lists - for k in range(len(matchList)): - matchList[k] = matchList[k].strip() - j=0 - while(len(matchList) > j+1): - while len(matchList[j]) < 30 and len(matchList) > j: - matchList[j:j+2] = [' '.join(matchList[j:j+2])] - if len(matchList) == j+1: - matchList[j] = matchList[j] + ' ' + translatedText - translatedText = '' - break - j+=1 - - # Normal Lines - if len(matchList) > 0: - data[i] = '\d\n' - for line in matchList: - # Wordwrap Text - if '[r]' not in line: - line = textwrap.fill(line, width=WIDTH) - line = line.replace('\n', '[r]') - - # Set - data.insert(i, line.strip() + '[p][cm]\n') - i+=1 - data[i-1] = data[i-1].replace('[l][er]', '[p][cm]') - # else: - # print ('No Matches') - - # Backup TL - if translatedText != '': - # Wordwrap Text - if '[r]' not in translatedText: - translatedText = textwrap.fill(translatedText, width=WIDTH) - translatedText = translatedText.replace('\n', '[r]') - - # Set Last Line - data[i] = translatedText.strip() + '[p][cm]\n' - - # Keep textHistory list at length maxHistory - if len(textHistory) > maxHistory: - textHistory.pop(0) - currentGroup = [] - speaker = '' - - # Grab Lines [p][cm] - matchList = re.findall(r'(.+?)\[p\]\[cm\]', data[i]) - if len(matchList) > 0: - matchList[0] = matchList[0].replace('「', '') - matchList[0] = matchList[0].replace('」', '') - finalJAString = matchList[0] - - # Remove any textwrap - if FIXTEXTWRAP == True: - finalJAString = finalJAString.replace('[r]', ' ') - - #Check Speaker - if speaker == '': - response = translateGPT(finalJAString, 'Previous Dialogue: ' + '\n\n'.join(textHistory), True) - tokens += response[1] - translatedText = response[0] - textHistory.append('\"' + translatedText + '\"') - else: - response = translateGPT(speaker + ': ' + finalJAString, 'Previous Dialogue: ' + '\n\n'.join(textHistory), True) - tokens += response[1] - translatedText = response[0] - textHistory.append('\"' + translatedText + '\"') - - # Remove added speaker - translatedText = re.sub(r'^.+:\s?', '', translatedText) - - # Set Data - translatedText = translatedText.replace('ッ', '') - translatedText = translatedText.replace('っ', '') - translatedText = translatedText.replace('ー', '') - translatedText = translatedText.replace('\"', '') - translatedText = translatedText.replace('[', '') - translatedText = translatedText.replace(']', '') - - # Split final string into full sentences. - matchList = re.findall(r'(.+?[)\.\?\!）。・]+)', translatedText) - translatedText = re.sub(r'(.+?[)\.\?\!）。・]+)', '', translatedText) - for l in range(len(matchList)): - if any(t in matchList[l] for t in ['Mr.', 'Ms.', 'Mrs.']): - if len(matchList) > l+1: - matchList[l] = matchList[l] + matchList[l+1] - matchList[l+1] = '' - - # Get rid of whitespace for each item and add wordwrap - for k in range(len(matchList)): - matchList[k] = matchList[k].strip() - - # Combine Sentences with a max limit (Wordwrap basically) - j=0 - while(len(matchList) > j+1): - while len(matchList[j]) < 30 and len(matchList) > j: - matchList[j:j+2] = [' '.join(matchList[j:j+2])] - if len(matchList) == j+1: - matchList[j] = matchList[j] + ' ' + translatedText - translatedText = '' - break - j+=1 - - # Set Data - if len(matchList) > 0: - data[i] = '\d\n' - for line in matchList: - # Wordwrap Text - if '[r]' not in line: - line = textwrap.fill(line, width=WIDTH) - line = line.replace('\n', '[r]') - - # Set - data.insert(i, line.strip() + '[p][cm]\n') - i+=1 - # Set last line as [pcm] instead of [r] - data[i-1] = data[i-1].replace('[l][er]', '[p][cm]') - # else: - # print ('No Matches') - if translatedText != '': - # Wordwrap Text - if '[r]' not in translatedText: - translatedText = textwrap.fill(translatedText, width=WIDTH) - translatedText = translatedText.replace('\n', '[r]') - - # Set Backup - data[i] = translatedText.strip() + '[p][cm]\n' - - # Keep textHistory list at length maxHistory - if len(textHistory) > maxHistory: - textHistory.pop(0) - currentGroup = [] - speaker = '' - - currentGroup = [] - pbar.update(1) - if len(data) > i+1: - syncIndex = i+1 - else: - break - - return tokens - def getResultString(translatedData, translationTime, filename): # File Print String - tokenString = Fore.YELLOW + '[' + str(translatedData[1]) + \ - ' Tokens/${:,.4f}'.format(translatedData[1] * .001 * APICOST) + ']' - timeString = Fore.BLUE + '[' + str(round(translationTime, 1)) + 's]' + totalTokenstring = ( + Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]" + "[Output: " + str(translatedData[1][1]) + "]" + "[Cost: ${:,.4f}".format( + (translatedData[1][0] * 0.001 * INPUTAPICOST) + + (translatedData[1][1] * 0.001 * OUTPUTAPICOST) + ) + + "]" + ) + timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]" - if translatedData[2] == None: + if translatedData[2] is None: # Success - return filename + ': ' + tokenString + timeString + Fore.GREEN + u' \u2713 ' + Fore.RESET + return ( + filename + + ": " + + totalTokenstring + + timeString + + Fore.GREEN + + " \u2713 " + + Fore.RESET + ) else: # Fail try: raise translatedData[2] except Exception as e: - traceback.print_exc() errorString = str(e) + Fore.RED - return filename + ': ' + tokenString + timeString + Fore.RED + u' \u2717 ' +\ - errorString + Fore.RESET - + return ( + filename + + ": " + + totalTokenstring + + timeString + + Fore.RED + + " \u2717 " + + errorString + + Fore.RESET + ) + + +def openFiles(filename): + with open("files/" + filename, "r", encoding="utf-8") as readFile: + translatedData = parseTyrano(readFile, filename) + + # Delete lines marked for deletion + finalData = [] + for line in translatedData[0]: + if line != "\\d\n": + finalData.append(line) + translatedData[0] = finalData + + return translatedData + + +def parseTyrano(readFile, filename): + totalTokens = [0, 0] + totalLines = 0 + + # Get total for progress bar + data = readFile.readlines() + totalLines = len(data) + + with tqdm( + bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE + ) as pbar: + pbar.desc = filename + pbar.total = totalLines + + try: + response = translateTyrano(data, pbar) + totalTokens[0] = response[0] + totalTokens[1] = response[1] + except Exception as e: + traceback.print_exc() + return [data, totalTokens, e] + return [data, totalTokens, None] + + +def translateTyrano(data, pbar): + textHistory = [] + maxHistory = MAXHISTORY + tokens = [0, 0] + currentGroup = [] + syncIndex = 0 + speaker = "" + delFlag = False + global LOCK, ESTIMATE + + for i in range(len(data)): + if syncIndex > i: + i = syncIndex + + if '[▼]' in data[i]: + data[i] = data[i].replace('[▼]'.strip(), '[page]\n') + + # If there isn't any Japanese in the text just skip + if IGNORETLTEXT is True: + if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+', data[i]): + # Keep textHistory list at length maxHistory + textHistory.append('\"' + data[i] + '\"') + if len(textHistory) > maxHistory: + textHistory.pop(0) + currentGroup = [] + continue + + # Speaker + matchList = re.findall(r"^\[(.+)\sstorage=.+\]", data[i]) + if len(matchList) == 0: + matchList = re.findall(r"^\[(.+)\]$", data[i]) + if len(matchList) != 0: + if "主人公" in matchList[0]: + speaker = "Protagonist" + elif "思考" in matchList[0]: + speaker = "Protagonist Inner Thoughts" + elif "地の文" in matchList[0]: + speaker = "Narrator" + elif "マコ" in matchList[0]: + speaker = "Mako" + else: + response = translateGPT( + matchList[0], + "Reply with only the " + + LANGUAGE + + " translation of the NPC name", + True, + ) + speaker = response[0] + tokens[0] += response[1][0] + tokens[1] += response[1][1] + # data[i] = '#' + speaker + '\n' + + # Choices + elif "glink" in data[i]: + matchList = re.findall(r"\[glink.+text=\"(.+?)\".+", data[i]) + if len(matchList) != 0: + if len(textHistory) > 0: + response = translateGPT( + matchList[0], + "Past Translated Text: " + + textHistory[len(textHistory) - 1] + + "\n\nReply in the style of a dialogue option.", + True, + ) + else: + response = translateGPT(matchList[0], "", False) + translatedText = response[0] + tokens[0] += response[1][0] + tokens[1] += response[1][1] + + # Remove characters that may break scripts + charList = [".", '"', "\\n"] + for char in charList: + translatedText = translatedText.replace(char, "") + + # Escape all ' + translatedText = translatedText.replace("\\", "") + translatedText = translatedText.replace("'", "\\'") + + # Set Data + translatedText = data[i].replace( + matchList[0], translatedText.replace(" ", "\u00A0") + ) + data[i] = translatedText + + # Grab Lines + matchList = re.findall(r"^([^\n;@*\[].+)", data[i]) + if len(matchList) > 0 and ('storage' in data[i-1] or re.search(r'^\[(.+)\]$', data[i-1])): + currentGroup.append(matchList[0]) + if len(data) > i + 1: + matchList = re.findall(r"^([^\n;@*\[].+)", data[i + 1]) + while len(matchList) > 0: + delFlag = True + data[i] = "\d\n" # \d Marks line for deletion + i += 1 + matchList = re.findall(r"^([^\n;@*\[].+)", data[i]) + if len(matchList) > 0: + currentGroup.append(matchList[0]) + + # Join up 401 groups for better translation. + if len(currentGroup) > 0: + finalJAString = " ".join(currentGroup) + + # Remove any textwrap + if FIXTEXTWRAP is True: + finalJAString = finalJAString.replace("_", " ") + + # Check Speaker + if speaker == "": + response = translateGPT(finalJAString, textHistory, True) + tokens[0] += response[1][0] + tokens[1] += response[1][1] + translatedText = response[0] + textHistory.append('"' + translatedText + '"') + else: + response = translateGPT( + speaker + ": " + finalJAString, textHistory, True + ) + tokens[0] += response[1][0] + tokens[1] += response[1][1] + translatedText = response[0] + textHistory.append('"' + translatedText + '"') + + # Remove added speaker + translatedText = re.sub(r"^.+:\s?", "", translatedText) + + # Set Data + translatedText = translatedText.replace("ッ", "") + translatedText = translatedText.replace("っ", "") + translatedText = translatedText.replace("ー", "") + translatedText = translatedText.replace('"', "") + translatedText = translatedText.replace("[", "") + translatedText = translatedText.replace("]", "") + + # Wordwrap Text + if "_" not in translatedText: + translatedText = textwrap.fill(translatedText, width=WIDTH) + translatedText = translatedText.replace("\n", "_") + + # Set + if delFlag is True: + data.insert(i, translatedText.strip() + '\n') + delFlag = False + else: + data[i] = translatedText.strip() + '\n' + + # Keep textHistory list at length maxHistory + if len(textHistory) > maxHistory: + textHistory.pop(0) + currentGroup = [] + speaker = "" + + currentGroup = [] + pbar.update(1) + if len(data) > i + 1: + syncIndex = i + 1 + else: + break + + return tokens + + def subVars(jaString): - jaString = jaString.replace('\u3000', ' ') + jaString = jaString.replace("\u3000", " ") + + # Nested + count = 0 + nestedList = re.findall(r"[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]", jaString) + nestedList = set(nestedList) + if len(nestedList) != 0: + for icon in nestedList: + jaString = jaString.replace(icon, "{Nested_" + str(count) + "}") + count += 1 # Icons count = 0 - iconList = re.findall(r'[\\]+[iIkKwW]+\[[0-9]+\]', jaString) + iconList = re.findall(r"[\\]+[iIkKwWaA]+\[[0-9]+\]", jaString) iconList = set(iconList) if len(iconList) != 0: for icon in iconList: - jaString = jaString.replace(icon, '[Icon' + str(count) + ']') + jaString = jaString.replace(icon, "{Ascii_" + str(count) + "}") count += 1 # Colors count = 0 - colorList = re.findall(r'[\\]+[cC]\[[0-9]+\]', jaString) + colorList = re.findall(r"[\\]+[cC]\[[0-9]+\]", jaString) colorList = set(colorList) if len(colorList) != 0: for color in colorList: - jaString = jaString.replace(color, '[Color' + str(count) + ']') + jaString = jaString.replace(color, "{Color_" + str(count) + "}") count += 1 # Names count = 0 - nameList = re.findall(r'[\\]+[nN]\[[0-9]+\]', jaString) + nameList = re.findall(r"[\\]+[nN]\[.+?\]+", jaString) nameList = set(nameList) if len(nameList) != 0: for name in nameList: - jaString = jaString.replace(name, '[Name' + str(count) + ']') + jaString = jaString.replace(name, "{N_" + str(count) + "}") count += 1 # Variables count = 0 - varList = re.findall(r'[\\]+[vV]\[[0-9]+\]', jaString) + varList = re.findall(r"[\\]+[vV]\[[0-9]+\]", jaString) varList = set(varList) if len(varList) != 0: for var in varList: - jaString = jaString.replace(var, '[Var' + str(count) + ']') + jaString = jaString.replace(var, "{Var_" + str(count) + "}") + count += 1 + + # Formatting + count = 0 + if "笑えるよね." in jaString: + print("t") + formatList = re.findall(r"[\\]+[\w]+\[.+?\]", jaString) + formatList = set(formatList) + if len(formatList) != 0: + for var in formatList: + jaString = jaString.replace(var, "{FCode_" + str(count) + "}") count += 1 # Put all lists in list and return - allList = [iconList, colorList, nameList, varList] + allList = [nestedList, iconList, colorList, nameList, varList, formatList] return [jaString, allList] + def resubVars(translatedText, allList): # Fix Spacing and ChatGPT Nonsense - matchList = re.findall(r'\[\s?.+?\s?\]', translatedText) + matchList = re.findall(r"\[\s?.+?\s?\]", translatedText) if len(matchList) > 0: for match in matchList: text = match.strip() translatedText = translatedText.replace(match, text) - # Icons + # Nested count = 0 if len(allList[0]) != 0: for var in allList[0]: - translatedText = translatedText.replace('[Icon' + str(count) + ']', var) + translatedText = translatedText.replace("{Nested_" + str(count) + "}", var) + count += 1 + + # Icons + count = 0 + if len(allList[1]) != 0: + for var in allList[1]: + translatedText = translatedText.replace("{Ascii_" + str(count) + "}", var) count += 1 # Colors count = 0 - if len(allList[1]) != 0: - for var in allList[1]: - translatedText = translatedText.replace('[Color' + str(count) + ']', var) + if len(allList[2]) != 0: + for var in allList[2]: + translatedText = translatedText.replace("{Color_" + str(count) + "}", var) count += 1 # Names count = 0 - if len(allList[2]) != 0: - for var in allList[2]: - translatedText = translatedText.replace('[Name' + str(count) + ']', var) + if len(allList[3]) != 0: + for var in allList[3]: + translatedText = translatedText.replace("{N_" + str(count) + "}", var) count += 1 # Vars count = 0 - if len(allList[3]) != 0: - for var in allList[3]: - translatedText = translatedText.replace('[Var' + str(count) + ']', var) + if len(allList[4]) != 0: + for var in allList[4]: + translatedText = translatedText.replace("{Var_" + str(count) + "}", var) + count += 1 + + # Formatting + count = 0 + if len(allList[5]) != 0: + for var in allList[5]: + translatedText = translatedText.replace("{FCode_" + str(count) + "}", var) count += 1 # Remove Color Variables Spaces @@ -477,38 +455,54 @@ def resubVars(translatedText, allList): # translatedText = re.sub(r'\s*(\\+c\[0+\])', r'\1', translatedText) return translatedText + @retry(exceptions=Exception, tries=5, delay=5) def translateGPT(t, history, fullPromptFlag): - # If ESTIMATE is True just count this as an execution and return. - if ESTIMATE: - enc = tiktoken.encoding_for_model(MODEL) - tokens = len(enc.encode(t)) * 2 + len(enc.encode(str(history))) + len(enc.encode(PROMPT)) - return (t, tokens) - # Sub Vars varResponse = subVars(t) subbedT = varResponse[0] # If there isn't any Japanese in the text just skip - if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴ]+|[\uFF00-\uFFEF]', subbedT): - return(t, 0) + if not re.search(r"[一-龠]+|[ぁ-ゔ]+|[ァ-ヴ]+|[\uFF00-\uFFEF]", subbedT): + return (t, [0, 0]) + + # If ESTIMATE is True just count this as an execution and return. + if ESTIMATE: + enc = tiktoken.encoding_for_model(MODEL) + historyRaw = "" + if isinstance(history, list): + for line in history: + historyRaw += line + else: + historyRaw = history + + inputTotalTokens = len(enc.encode(historyRaw)) + len(enc.encode(PROMPT)) + outputTotalTokens = ( + len(enc.encode(t)) * 2 + ) # Estimating 2x the size of the original text + totalTokens = [inputTotalTokens, outputTotalTokens] + return (t, totalTokens) # Characters - context = '```\ - Game Characters:\ - Character: 亜紀子 == Akiko - Gender: Female\ - Character: 明日香 == Asuka - Gender: Female\ - ```' + context = "Game Characters:\ + Character: マコ == Mako - Gender: Female\ + Character: 主人公 == Protagonist - Gender: Male" # Prompt if fullPromptFlag: system = PROMPT - user = 'Line to Translate = ' + subbedT + user = "Line to Translate = " + subbedT else: - system = 'Output ONLY the '+ LANGUAGE +' translation in the following format: `Translation: <'+ LANGUAGE.upper() +'_TRANSLATION>`' - user = 'Line to Translate = ' + subbedT + system = ( + "Output ONLY the " + + LANGUAGE + + " translation in the following format: `Translation: <" + + LANGUAGE.upper() + + "_TRANSLATION>`" + ) + user = "Line to Translate = " + subbedT - # Create Message List + # Create Message List msg = [] msg.append({"role": "system", "content": system}) msg.append({"role": "user", "content": context}) @@ -520,7 +514,7 @@ def translateGPT(t, history, fullPromptFlag): msg.append({"role": "user", "content": user}) response = openai.ChatCompletion.create( - temperature=0.1, + temperature=0, frequency_penalty=0.2, presence_penalty=0.2, model=MODEL, @@ -530,27 +524,35 @@ def translateGPT(t, history, fullPromptFlag): # Save Translated Text translatedText = response.choices[0].message.content - tokens = response.usage.total_tokens + totalTokens = [response.usage.prompt_tokens, response.usage.completion_tokens] # Resub Vars translatedText = resubVars(translatedText, varResponse[1]) # Remove Placeholder Text - translatedText = translatedText.replace(LANGUAGE +' Translation: ', '') - translatedText = translatedText.replace('Translation: ', '') - translatedText = translatedText.replace('Line to Translate = ', '') - translatedText = translatedText.replace('Translation = ', '') - translatedText = translatedText.replace('Translate = ', '') - translatedText = translatedText.replace(LANGUAGE +' Translation:', '') - translatedText = translatedText.replace('Translation:', '') - translatedText = translatedText.replace('Line to Translate =', '') - translatedText = translatedText.replace('Translation =', '') - translatedText = translatedText.replace('Translate =', '') - translatedText = re.sub(r'Note:.*', '', translatedText) - translatedText = translatedText.replace('っ', '') + translatedText = translatedText.replace(LANGUAGE + " Translation: ", "") + translatedText = translatedText.replace("Translation: ", "") + translatedText = translatedText.replace("Line to Translate = ", "") + translatedText = translatedText.replace("Translation = ", "") + translatedText = translatedText.replace("Translate = ", "") + translatedText = translatedText.replace(LANGUAGE + " Translation:", "") + translatedText = translatedText.replace("Translation:", "") + translatedText = translatedText.replace("Line to Translate =", "") + translatedText = translatedText.replace("Translation =", "") + translatedText = translatedText.replace("Translate =", "") + translatedText = translatedText.replace("っ", "") + translatedText = translatedText.replace("ッ", "") + translatedText = translatedText.replace("ぁ", "") + translatedText = translatedText.replace("。", ".") + translatedText = translatedText.replace("、", ",") + translatedText = translatedText.replace("？", "?") + translatedText = translatedText.replace("！", "!") # Return Translation - if len(translatedText) > 15 * len(t) or "I'm sorry, but I'm unable to assist with that translation" in translatedText: + if ( + len(translatedText) > 15 * len(t) + or "I'm sorry, but I'm unable to assist with that translation" in translatedText + ): raise Exception else: - return [translatedText, tokens] + return [translatedText, totalTokens]