diff --git a/modules/atelier.py b/modules/atelier.py
new file mode 100644
index 0000000..d214749
--- /dev/null
+++ b/modules/atelier.py
@@ -0,0 +1,380 @@
+import os
+from pathlib import Path
+import re
+import textwrap
+import threading
+import time
+import traceback
+import tiktoken
+from colorama import Fore
+from dotenv import load_dotenv
+import openai
+from retry import retry
+from tqdm import tqdm
+
+# Open AI
+load_dotenv()
+if os.getenv('api').replace(' ', '') != '':
+ openai.api_base = os.getenv('api')
+openai.organization = os.getenv('org')
+openai.api_key = os.getenv('key')
+
+#Globals
+MODEL = os.getenv('model')
+TIMEOUT = int(os.getenv('timeout'))
+LANGUAGE=os.getenv('language').capitalize()
+INPUTAPICOST = .002 # Depends on the model https://openai.com/pricing
+OUTPUTAPICOST = .002
+PROMPT = Path('prompt.txt').read_text(encoding='utf-8')
+THREADS = int(os.getenv('threads')) # Controls how many threads are working on a single file (May have to drop this)
+LOCK = threading.Lock()
+WIDTH = int(os.getenv('width'))
+LISTWIDTH = int(os.getenv('listWidth'))
+NOTEWIDTH = 40
+MAXHISTORY = 10
+ESTIMATE = ''
+totalTokens = [0, 0]
+NAMESLIST = []
+
+#tqdm Globals
+BAR_FORMAT='{l_bar}{bar:10}{r_bar}{bar:-10b}'
+POSITION=0
+LEAVE=False
+
+# Translation Flags
+FIXTEXTWRAP = True
+IGNORETLTEXT = True
+
+def handleAtelier(filename, estimate):
+ global ESTIMATE, totalTokens
+ ESTIMATE = estimate
+
+ if estimate:
+ start = time.time()
+ translatedData = openFiles(filename)
+
+ # Print Result
+ end = time.time()
+ tqdm.write(getResultString(translatedData, end - start, filename))
+ with LOCK:
+ totalTokens[0] += translatedData[1][0]
+ totalTokens[1] += translatedData[1][1]
+
+ return getResultString(['', totalTokens, None], end - start, 'TOTAL')
+
+ else:
+ try:
+ with open('translated/' + filename, 'w', encoding='utf-8'):
+ start = time.time()
+ translatedData = openFiles(filename)
+
+ # Print Result
+ end = time.time()
+ tqdm.write(getResultString(translatedData, end - start, filename))
+ with LOCK:
+ totalTokens[0] += translatedData[1][0]
+ totalTokens[1] += translatedData[1][1]
+ except Exception:
+ return 'Fail'
+
+ return getResultString(['', totalTokens, None], end - start, 'TOTAL')
+
+def openFiles(filename):
+ with open('files/' + filename, 'r', encoding='UTF-8') as f:
+ translatedData = parseText(f, filename)
+
+ return translatedData
+
+def getResultString(translatedData, translationTime, filename):
+ # File Print String
+ totalTokenstring =\
+ Fore.YELLOW +\
+ '[Input: ' + str(translatedData[1][0]) + ']'\
+ '[Output: ' + str(translatedData[1][1]) + ']'\
+ '[Cost: ${:,.4f}'.format((translatedData[1][0] * .001 * INPUTAPICOST) +\
+ (translatedData[1][1] * .001 * OUTPUTAPICOST)) + ']'
+ timeString = Fore.BLUE + '[' + str(round(translationTime, 1)) + 's]'
+
+ if translatedData[2] is None:
+ # Success
+ return filename + ': ' + totalTokenstring + timeString + Fore.GREEN + u' \u2713 ' + Fore.RESET
+
+ else:
+ # Fail
+ try:
+ raise translatedData[2]
+ except Exception as e:
+ errorString = str(e) + Fore.RED
+ return filename + ': ' + totalTokenstring + timeString + Fore.RED + u' \u2717 ' +\
+ errorString + Fore.RESET
+
+def parseText(data, filename):
+ totalLines = 0
+ global LOCK
+
+ # Get total for progress bar
+ linesList = data.readlines()
+ totalLines = len(linesList)
+
+ with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar:
+ pbar.desc=filename
+ pbar.total=totalLines
+ try:
+ response = translateText(linesList, pbar)
+ except Exception as e:
+ traceback.print_exc()
+ return [linesList, 0, e]
+ return [response[0], response[1], None]
+
+def translateText(data, pbar):
+ textHistory = []
+ maxHistory = MAXHISTORY
+ totalTokens = [0,0]
+ syncIndex = 0
+
+ for i in range(len(data)):
+ if syncIndex > i:
+ i = syncIndex
+
+ match = re.findall(r'◆.+◆(.+)', data[i])
+ if len(match) > 0:
+ jaString = match[0]
+
+ ### Translate
+ # Remove any textwrap
+ finalJAString = re.sub(r'\\n', ' ', jaString)
+
+ # Translate
+ response = translateGPT(finalJAString, 'Previous Text for Context: ' + ' '.join(textHistory), True)
+ totalTokens[0] += response[1][0]
+ totalTokens[1] += response[1][1]
+ translatedText = response[0]
+
+ # TextHistory is what we use to give GPT Context, so thats appended here.
+ textHistory.append('\"' + translatedText + '\"')
+
+ # Keep textHistory list at length maxHistory
+ if len(textHistory) > maxHistory:
+ textHistory.pop(0)
+
+ # Textwrap
+ translatedText = translatedText.replace('\"', '\\"')
+ translatedText = textwrap.fill(translatedText, width=WIDTH)
+
+ # Write
+ data[i] = data[i].replace(match[0], translatedText)
+
+ syncIndex = i + 1
+ pbar.update()
+ return [data, totalTokens]
+
+def subVars(jaString):
+ jaString = jaString.replace('\u3000', ' ')
+
+ # Nested
+ count = 0
+ nestedList = re.findall(r'[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]', jaString)
+ nestedList = set(nestedList)
+ if len(nestedList) != 0:
+ for icon in nestedList:
+ jaString = jaString.replace(icon, '{Nested_' + str(count) + '}')
+ count += 1
+
+ # Icons
+ count = 0
+ iconList = re.findall(r'[\\]+[iIkKwWaA]+\[[0-9]+\]', jaString)
+ iconList = set(iconList)
+ if len(iconList) != 0:
+ for icon in iconList:
+ jaString = jaString.replace(icon, '{Ascii_' + str(count) + '}')
+ count += 1
+
+ # Colors
+ count = 0
+ colorList = re.findall(r'[\\]+[cC]\[[0-9]+\]', jaString)
+ colorList = set(colorList)
+ if len(colorList) != 0:
+ for color in colorList:
+ jaString = jaString.replace(color, '{Color_' + str(count) + '}')
+ count += 1
+
+ # Names
+ count = 0
+ nameList = re.findall(r'[\\]+[nN]\[.+?\]+', jaString)
+ nameList = set(nameList)
+ if len(nameList) != 0:
+ for name in nameList:
+ jaString = jaString.replace(name, '{N_' + str(count) + '}')
+ count += 1
+
+ # Variables
+ count = 0
+ varList = re.findall(r'[\\]+[vV]\[[0-9]+\]', jaString)
+ varList = set(varList)
+ if len(varList) != 0:
+ for var in varList:
+ jaString = jaString.replace(var, '{Var_' + str(count) + '}')
+ count += 1
+
+ # Formatting
+ count = 0
+ if '笑えるよね.' in jaString:
+ print('t')
+ formatList = re.findall(r'[\\]+[\w]+\[.+?\]', jaString)
+ formatList = set(formatList)
+ if len(formatList) != 0:
+ for var in formatList:
+ jaString = jaString.replace(var, '{FCode_' + str(count) + '}')
+ count += 1
+
+ # Put all lists in list and return
+ allList = [nestedList, iconList, colorList, nameList, varList, formatList]
+ return [jaString, allList]
+
+def resubVars(translatedText, allList):
+ # Fix Spacing and ChatGPT Nonsense
+ matchList = re.findall(r'\[\s?.+?\s?\]', translatedText)
+ if len(matchList) > 0:
+ for match in matchList:
+ text = match.strip()
+ translatedText = translatedText.replace(match, text)
+
+ # Nested
+ count = 0
+ if len(allList[0]) != 0:
+ for var in allList[0]:
+ translatedText = translatedText.replace('{Nested_' + str(count) + '}', var)
+ count += 1
+
+ # Icons
+ count = 0
+ if len(allList[1]) != 0:
+ for var in allList[1]:
+ translatedText = translatedText.replace('{Ascii_' + str(count) + '}', var)
+ count += 1
+
+ # Colors
+ count = 0
+ if len(allList[2]) != 0:
+ for var in allList[2]:
+ translatedText = translatedText.replace('{Color_' + str(count) + '}', var)
+ count += 1
+
+ # Names
+ count = 0
+ if len(allList[3]) != 0:
+ for var in allList[3]:
+ translatedText = translatedText.replace('{N_' + str(count) + '}', var)
+ count += 1
+
+ # Vars
+ count = 0
+ if len(allList[4]) != 0:
+ for var in allList[4]:
+ translatedText = translatedText.replace('{Var_' + str(count) + '}', var)
+ count += 1
+
+ # Formatting
+ count = 0
+ if len(allList[5]) != 0:
+ for var in allList[5]:
+ translatedText = translatedText.replace('{FCode_' + str(count) + '}', var)
+ count += 1
+
+ # Remove Color Variables Spaces
+ # if '\\c' in translatedText:
+ # translatedText = re.sub(r'\s*(\\+c\[[1-9]+\])\s*', r' \1', translatedText)
+ # translatedText = re.sub(r'\s*(\\+c\[0+\])', r'\1', translatedText)
+ return translatedText
+
+@retry(exceptions=Exception, tries=5, delay=5)
+def translateGPT(t, history, fullPromptFlag):
+ # Sub Vars
+ varResponse = subVars(t)
+ subbedT = varResponse[0]
+
+ # If there isn't any Japanese in the text just skip
+ if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴ]+|[\uFF00-\uFFEF]', subbedT):
+ return(t, [0,0])
+
+ # If ESTIMATE is True just count this as an execution and return.
+ if ESTIMATE:
+ enc = tiktoken.encoding_for_model(MODEL)
+ historyRaw = ''
+ if isinstance(history, list):
+ for line in history:
+ historyRaw += line
+ else:
+ historyRaw = history
+
+ inputTotalTokens = len(enc.encode(historyRaw)) + len(enc.encode(PROMPT))
+ outputTotalTokens = len(enc.encode(t)) * 2 # Estimating 2x the size of the original text
+ totalTokens = [inputTotalTokens, outputTotalTokens]
+ return (t, totalTokens)
+
+ # Characters
+ context = 'Game Characters:\
+ Character: リッカ == Ricca - Gender: Female\
+ Character: シーナ == Sina - Gender: Female\
+ Character: ヘレナ == Helena - Gender: Female\
+ Character: Miko == Miko - Gender: Female'
+
+ # Prompt
+ if fullPromptFlag:
+ system = PROMPT
+ user = 'Line to Translate = ' + subbedT
+ else:
+ system = 'Output ONLY the '+ LANGUAGE +' translation in the following format: `Translation: <'+ LANGUAGE.upper() +'_TRANSLATION>`'
+ user = 'Line to Translate = ' + subbedT
+
+ # Create Message List
+ msg = []
+ msg.append({"role": "system", "content": system})
+ msg.append({"role": "user", "content": context})
+ if isinstance(history, list):
+ for line in history:
+ msg.append({"role": "user", "content": line})
+ else:
+ msg.append({"role": "user", "content": history})
+ msg.append({"role": "user", "content": user})
+
+ response = openai.ChatCompletion.create(
+ temperature=0,
+ frequency_penalty=0.2,
+ presence_penalty=0.2,
+ model=MODEL,
+ messages=msg,
+ request_timeout=TIMEOUT,
+ )
+
+ # Save Translated Text
+ translatedText = response.choices[0].message.content
+ totalTokens = [response.usage.prompt_tokens, response.usage.completion_tokens]
+
+ # Resub Vars
+ translatedText = resubVars(translatedText, varResponse[1])
+
+ # Remove Placeholder Text
+ translatedText = translatedText.replace(LANGUAGE +' Translation: ', '')
+ translatedText = translatedText.replace('Translation: ', '')
+ translatedText = translatedText.replace('Line to Translate = ', '')
+ translatedText = translatedText.replace('Translation = ', '')
+ translatedText = translatedText.replace('Translate = ', '')
+ translatedText = translatedText.replace(LANGUAGE +' Translation:', '')
+ translatedText = translatedText.replace('Translation:', '')
+ translatedText = translatedText.replace('Line to Translate =', '')
+ translatedText = translatedText.replace('Translation =', '')
+ translatedText = translatedText.replace('Translate =', '')
+ translatedText = translatedText.replace('っ', '')
+ translatedText = translatedText.replace('ッ', '')
+ translatedText = translatedText.replace('ぁ', '')
+ translatedText = translatedText.replace('。', '.')
+ translatedText = translatedText.replace('、', ',')
+ translatedText = translatedText.replace('?', '?')
+ translatedText = translatedText.replace('!', '!')
+
+ # Return Translation
+ if len(translatedText) > 15 * len(t) or "I'm sorry, but I'm unable to assist with that translation" in translatedText:
+ raise Exception
+ else:
+ return [translatedText, totalTokens]
\ No newline at end of file
diff --git a/modules/main.py b/modules/main.py
index 5bbc342..8520a6d 100644
--- a/modules/main.py
+++ b/modules/main.py
@@ -12,8 +12,8 @@ from modules.txt import handleTXT
from modules.tyrano import handleTyrano
from modules.json import handleJSON
from modules.kansen import handleKansen
-from modules.lune import handleLune
from modules.lune2 import handleLuneTxt
+from modules.atelier import handleAtelier
# For GPT4 rate limit will be hit if you have more than 1 thread.
# 1 Thread for each file. Controls how many files are worked on at once.
@@ -30,14 +30,17 @@ def main():
while estimate == '':
estimate = input('Select Translation or Cost Estimation:\n\n1. Translate\n2. Estimate\n')
match estimate:
- case '1': estimate = False
- case '2': estimate = True
- case _: estimate = ''
+ case '1':
+ estimate = False
+ case '2':
+ estimate = True
+ case _:
+ estimate = ''
totalCost = 0
version = ''
while version == '':
- version = input('Select the RPGMaker Version:\n\n1. MV/MZ\n2. ACE\n3. CSV (From Translator++)\n4. Text (Custom)\n5. Tyrano\n6. JSON\n7. Kansen\n8. Lune\n')
+ version = input('Select the RPGMaker Version:\n\n1. MV/MZ\n2. ACE\n3. CSV (From Translator++)\n4. Text (Custom)\n5. Tyrano\n6. JSON\n7. Kansen\n8. Lune\n9. Atelier\n')
match version:
case '1':
# Open File (Threads)
@@ -149,11 +152,25 @@ def main():
tracebackLineNo = str(traceback.extract_tb(sys.exc_info()[2])[-1].lineno)
tqdm.write(Fore.RED + str(e) + '|' + tracebackLineNo + Fore.RESET)
+ case '9':
+ # Open File (Threads)
+ with ThreadPoolExecutor(max_workers=THREADS) as executor:
+ futures = [executor.submit(handleAtelier, filename, estimate) \
+ for filename in os.listdir("files") if filename.endswith('txt')]
+
+ for future in as_completed(futures):
+ try:
+ totalCost = future.result()
+
+ except Exception as e:
+ tracebackLineNo = str(traceback.extract_tb(sys.exc_info()[2])[-1].lineno)
+ tqdm.write(Fore.RED + str(e) + '|' + tracebackLineNo + Fore.RESET)
+
case _:
version = ''
if totalCost != 'Fail':
- if estimate == False:
+ if estimate is False:
# This is to encourage people to grab what's in /translated instead
deleteFolderFiles('files')
diff --git a/modules/rpgmakermvmz.py b/modules/rpgmakermvmz.py
index bb0898e..e897e9c 100644
--- a/modules/rpgmakermvmz.py
+++ b/modules/rpgmakermvmz.py
@@ -3,30 +3,28 @@ import json
import os
from pathlib import Path
import re
-import sys
import textwrap
import threading
import time
import traceback
import tiktoken
-
from colorama import Fore
from dotenv import load_dotenv
import openai
from retry import retry
from tqdm import tqdm
-#Globals
+# Open AI
load_dotenv()
if os.getenv('api').replace(' ', '') != '':
openai.api_base = os.getenv('api')
-
openai.organization = os.getenv('org')
openai.api_key = os.getenv('key')
+
+#Globals
MODEL = os.getenv('model')
TIMEOUT = int(os.getenv('timeout'))
LANGUAGE=os.getenv('language').capitalize()
-
INPUTAPICOST = .002 # Depends on the model https://openai.com/pricing
OUTPUTAPICOST = .002
PROMPT = Path('prompt.txt').read_text(encoding='utf-8')
@@ -49,17 +47,17 @@ LEAVE=False
CODE401 = True
CODE405 = False
CODE102 = True
-CODE122 = True
+CODE122 = False
CODE101 = False
CODE355655 = False
CODE357 = False
CODE657 = False
-CODE356 = True
+CODE356 = False
CODE320 = False
CODE324 = False
CODE111 = False
CODE408 = False
-CODE108 = True
+CODE108 = False
NAMES = False # Output a list of all the character names found
BRFLAG = False # If the game uses
instead
FIXTEXTWRAP = True
@@ -76,7 +74,7 @@ def handleMVMZ(filename, estimate):
# Print Result
end = time.time()
tqdm.write(getResultString(translatedData, end - start, filename))
- if NAMES == True:
+ if NAMES is True:
tqdm.write(str(NAMESLIST))
with LOCK:
totalTokens[0] += translatedData[1][0]
@@ -86,7 +84,7 @@ def handleMVMZ(filename, estimate):
else:
try:
- with open('translated/' + filename, 'w', encoding='UTF-8') as outFile:
+ with open('translated/' + filename, 'w', encoding='utf-8') as outFile:
start = time.time()
translatedData = openFiles(filename)
@@ -97,13 +95,13 @@ def handleMVMZ(filename, estimate):
with LOCK:
totalTokens[0] += translatedData[1][0]
totalTokens[1] += translatedData[1][1]
- except Exception as e:
+ except Exception:
return 'Fail'
return getResultString(['', totalTokens, None], end - start, 'TOTAL')
def openFiles(filename):
- with open('files/' + filename, 'r', encoding='UTF-8') as f:
+ with open('files/' + filename, 'r', encoding='utf-8-sig') as f:
data = json.load(f)
# Map Files
@@ -177,7 +175,7 @@ def getResultString(translatedData, translationTime, filename):
(translatedData[1][1] * .001 * OUTPUTAPICOST)) + ']'
timeString = Fore.BLUE + '[' + str(round(translationTime, 1)) + 's]'
- if translatedData[2] == None:
+ if translatedData[2] is None:
# Success
return filename + ': ' + totalTokenstring + timeString + Fore.GREEN + u' \u2713 ' + Fore.RESET
@@ -431,7 +429,7 @@ def searchThings(name, pbar):
totalTokens = [0, 0]
# If there isn't any Japanese in the text just skip
- if IGNORETLTEXT == True:
+ if IGNORETLTEXT is True:
if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+', name['name']) and re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+', name['description']):
pbar.update(1)
return totalTokens
@@ -551,7 +549,6 @@ def searchCodes(page, pbar):
maxHistory = MAXHISTORY
totalTokens = [0, 0]
speaker = ''
- speakerVar = ''
nametag = ''
match = []
syncIndex = 0
@@ -576,7 +573,7 @@ def searchCodes(page, pbar):
### IF these crash or fail your game will do the same. Use the flags to skip codes.
## Event Code: 401 Show Text
- if codeList[i]['code'] == 401 and CODE401 == True or codeList[i]['code'] == 405 and CODE405:
+ if codeList[i]['code'] == 401 and CODE401 is True or codeList[i]['code'] == 405 and CODE405:
# Use this to place text later
code = codeList[i]['code']
j = i
@@ -584,13 +581,12 @@ def searchCodes(page, pbar):
# Grab String
if len(codeList[i]['parameters']) > 0:
jaString = codeList[i]['parameters'][0]
- firstJAString = jaString
else:
codeList[i]['code'] = -1
continue
# If there isn't any Japanese in the text just skip
- if IGNORETLTEXT == True:
+ if IGNORETLTEXT is True:
if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+', jaString):
# Keep textHistory list at length maxHistory
textHistory.append('\"' + jaString + '\"')
@@ -732,25 +728,23 @@ def searchCodes(page, pbar):
finalJAString = finalJAString.replace(matchList[0], '')
# Remove any textwrap
- if FIXTEXTWRAP == True:
+ if FIXTEXTWRAP is True:
finalJAString = re.sub(r'\n', ' ', finalJAString)
finalJAString = finalJAString.replace('
', ' ')
# Remove Extra Stuff
finalJAString = finalJAString.replace('゙', '')
- finalJAString = finalJAString.replace('。', '.')
- finalJAString = finalJAString.replace('?', '?')
- finalJAString = finalJAString.replace('!', '!')
finalJAString = finalJAString.replace('・', '.')
finalJAString = finalJAString.replace('‶', '')
finalJAString = finalJAString.replace('”', '')
finalJAString = finalJAString.replace('―', '-')
+ finalJAString = finalJAString.replace('ー', '-')
finalJAString = finalJAString.replace('…', '...')
finalJAString = finalJAString.replace(' ', '')
# finalJAString = finalJAString.replace('〇', '*')
# Remove any RPGMaker Code at start
- ffMatchList = re.findall(r'[\\]+[fF]+\[.+?\]', finalJAString)
+ ffMatchList = re.findall(r'[\\]+[fFaA]+\[.+?\]', finalJAString)
if len(ffMatchList) > 0:
finalJAString = finalJAString.replace(ffMatchList[0], '')
nametag += ffMatchList[0]
@@ -785,29 +779,27 @@ def searchCodes(page, pbar):
# Sub Vars
varResponse = subVars(translatedText)
- subbedT = varResponse[0]
textHistory.append('\"' + varResponse[0] + '\"')
elif finalJAString != '':
response = translateGPT(speaker + ' | ' + finalJAString, textHistory, True)
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
translatedText = response[0]
-
+
# Remove added speaker
translatedText = re.sub(r'(^.+?)\s?[|:]\s?', '', translatedText)
# Sub Vars
varResponse = subVars(translatedText)
- subbedT = varResponse[0]
textHistory.append('\"' + speaker + ' | ' + varResponse[0] + '\"')
speaker = ''
else:
translatedText = finalJAString
# Textwrap
- if FIXTEXTWRAP == True:
+ if FIXTEXTWRAP is True:
translatedText = textwrap.fill(translatedText, width=WIDTH)
- if BRFLAG == True:
+ if BRFLAG is True:
translatedText = translatedText.replace('\n', '
')
# Add Beginning Text
@@ -834,14 +826,13 @@ def searchCodes(page, pbar):
currentGroup = []
## Event Code: 122 [Set Variables]
- if codeList[i]['code'] == 122 and CODE122 == True:
+ if codeList[i]['code'] == 122 and CODE122 is True:
# This is going to be the var being set. (IMPORTANT)
- varNum = codeList[i]['parameters'][0]
# if varNum not in [1178]:
# continue
jaString = codeList[i]['parameters'][4]
- if type(jaString) != str:
+ if isinstance(jaString, str):
continue
# Definitely don't want to mess with files
@@ -880,11 +871,11 @@ def searchCodes(page, pbar):
# Set Data
codeList[i]['parameters'][4] = translatedText
- ## Event Code: 357 [Picture Text] [Optional]
- if codeList[i]['code'] == 357 and CODE357 == True:
+ ## Event Code: 357 [Picture Text] [Optional]
+ if codeList[i]['code'] == 357 and CODE357 is True:
if 'message' in codeList[i]['parameters'][3]:
jaString = codeList[i]['parameters'][3]['message']
- if type(jaString) != str:
+ if not isinstance(jaString, str):
continue
# Definitely don't want to mess with files
@@ -899,8 +890,10 @@ def searchCodes(page, pbar):
oldjaString = jaString
startString = re.search(r'^[^一-龠ぁ-ゔァ-ヴー【】()「」a-zA-ZA-Z0-9\\]+', jaString)
finalJAString = re.sub(r'^[^一-龠ぁ-ゔァ-ヴー【】()「」a-zA-ZA-Z0-9\\]+', '', jaString)
- if startString is None: startString = ''
- else: startString = startString.group()
+ if startString is None:
+ startString = ''
+ else:
+ startString = startString.group()
# Remove any textwrap
finalJAString = re.sub(r'\n', ' ', finalJAString)
@@ -917,11 +910,11 @@ def searchCodes(page, pbar):
# Set Data
codeList[i]['parameters'][3]['message'] = startString + translatedText
- ## Event Code: 657 [Picture Text] [Optional]
- if codeList[i]['code'] == 657 and CODE657 == True:
+ ## Event Code: 657 [Picture Text] [Optional]
+ if codeList[i]['code'] == 657 and CODE657 is True:
if 'text' in codeList[i]['parameters'][0]:
jaString = codeList[i]['parameters'][0]
- if type(jaString) != str:
+ if not isinstance(jaString, str):
continue
# Definitely don't want to mess with files
@@ -937,10 +930,14 @@ def searchCodes(page, pbar):
jaString = re.sub(r'^[^一-龠ぁ-ゔァ-ヴー\<\>【】\\]+', '', jaString)
endString = re.search(r'[^一-龠ぁ-ゔァ-ヴー\<\>【】。!?\\]+$', jaString)
jaString = re.sub(r'[^一-龠ぁ-ゔァ-ヴー\<\>【】。!?\\]+$', '', jaString)
- if startString is None: startString = ''
- else: startString = startString.group()
- if endString is None: endString = ''
- else: endString = endString.group()
+ if startString is None:
+ startString = ''
+ else:
+ startString = startString.group()
+ if endString is None:
+ endString = ''
+ else:
+ endString = endString.group()
# Remove any textwrap
jaString = re.sub(r'\n', ' ', jaString)
@@ -966,12 +963,12 @@ def searchCodes(page, pbar):
codeList[i]['parameters'][0] = translatedText
## Event Code: 101 [Name] [Optional]
- if codeList[i]['code'] == 101 and CODE101 == True:
+ if codeList[i]['code'] == 101 and CODE101 is True:
# Grab String
jaString = ''
if len(codeList[i]['parameters']) > 4:
jaString = codeList[i]['parameters'][4]
- if type(jaString) != str:
+ if not isinstance(jaString, str):
continue
# Force Speaker
@@ -1032,7 +1029,7 @@ def searchCodes(page, pbar):
NAMESLIST.append(speaker)
## Event Code: 355 or 655 Scripts [Optional]
- if (codeList[i]['code'] == 355 or codeList[i]['code'] == 655) and CODE355655 == True:
+ if (codeList[i]['code'] == 355 or codeList[i]['code'] == 655) and CODE355655 is True:
jaString = codeList[i]['parameters'][0]
# If there isn't any Japanese in the text just skip
@@ -1072,7 +1069,7 @@ def searchCodes(page, pbar):
codeList[i]['parameters'][0] = translatedText
## Event Code: 408 (Script)
- if (codeList[i]['code'] == 408) and CODE408 == True:
+ if (codeList[i]['code'] == 408) and CODE408 is True:
jaString = codeList[i]['parameters'][0]
# If there isn't any Japanese in the text just skip
@@ -1112,7 +1109,7 @@ def searchCodes(page, pbar):
codeList[i]['parameters'][0] = translatedText
## Event Code: 108 (Script)
- if (codeList[i]['code'] == 108) and CODE108 == True:
+ if (codeList[i]['code'] == 108) and CODE108 is True:
jaString = codeList[i]['parameters'][0]
# If there isn't any Japanese in the text just skip
@@ -1145,7 +1142,7 @@ def searchCodes(page, pbar):
codeList[i]['parameters'][0] = translatedText
## Event Code: 356
- if codeList[i]['code'] == 356 and CODE356 == True:
+ if codeList[i]['code'] == 356 and CODE356 is True:
jaString = codeList[i]['parameters'][0]
oldjaString = jaString
@@ -1433,7 +1430,7 @@ def searchCodes(page, pbar):
continue
### Event Code: 102 Show Choice
- if codeList[i]['code'] == 102 and CODE102 == True:
+ if codeList[i]['code'] == 102 and CODE102 is True:
for choice in range(len(codeList[i]['parameters'][0])):
jaString = codeList[i]['parameters'][0][choice]
jaString = jaString.replace(' 。', '.')
@@ -1466,12 +1463,12 @@ def searchCodes(page, pbar):
codeList[i]['parameters'][0][choice] = startString + translatedText + endString
### Event Code: 111 Script
- if codeList[i]['code'] == 111 and CODE111 == True:
+ if codeList[i]['code'] == 111 and CODE111 is True:
for j in range(len(codeList[i]['parameters'])):
jaString = codeList[i]['parameters'][j]
# Check if String
- if type(jaString) != str:
+ if not isinstance(jaString, str):
continue
# Only TL the Game Variable
@@ -1503,9 +1500,9 @@ def searchCodes(page, pbar):
codeList[i]['parameters'][j] = translatedText
### Event Code: 320 Set Variable
- if codeList[i]['code'] == 320 and CODE320 == True:
+ if codeList[i]['code'] == 320 and CODE320 is True:
jaString = codeList[i]['parameters'][1]
- if type(jaString) != str:
+ if not isinstance(jaString, str):
continue
# Definitely don't want to mess with files
@@ -1544,7 +1541,7 @@ def searchCodes(page, pbar):
# raise Exception(str(e) + '|Line:' + tracebackLineNo)
except Exception as e:
traceback.print_exc()
- raise Exception(str(e) + 'Failed to translate: ' + oldjaString)
+ raise Exception(str(e) + 'Failed to translate: ' + oldjaString) from None
# Append leftover groups in 401
if len(currentGroup) > 0:
@@ -1576,7 +1573,6 @@ def searchCodes(page, pbar):
return totalTokens
def searchSS(state, pbar):
- '''Searches skills and states json files'''
totalTokens = [0, 0]
# Name
@@ -1791,7 +1787,7 @@ def subVars(jaString):
count = 0
if '笑えるよね.' in jaString:
print('t')
- formatList = re.findall(r'[\\]+CL', jaString)
+ formatList = re.findall(r'[\\]+[\w]+\[.+?\]', jaString)
formatList = set(formatList)
if len(formatList) != 0:
for var in formatList:
@@ -1860,6 +1856,14 @@ def resubVars(translatedText, allList):
@retry(exceptions=Exception, tries=5, delay=5)
def translateGPT(t, history, fullPromptFlag):
+ # Sub Vars
+ varResponse = subVars(t)
+ subbedT = varResponse[0]
+
+ # If there isn't any Japanese in the text just skip
+ if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴ]+|[\uFF00-\uFFEF]', subbedT):
+ return(t, [0,0])
+
# If ESTIMATE is True just count this as an execution and return.
if ESTIMATE:
enc = tiktoken.encoding_for_model(MODEL)
@@ -1874,23 +1878,13 @@ def translateGPT(t, history, fullPromptFlag):
outputTotalTokens = len(enc.encode(t)) * 2 # Estimating 2x the size of the original text
totalTokens = [inputTotalTokens, outputTotalTokens]
return (t, totalTokens)
-
- # Sub Vars
- varResponse = subVars(t)
- subbedT = varResponse[0]
-
- # If there isn't any Japanese in the text just skip
- if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴ]+|[\uFF00-\uFFEF]', subbedT):
- return(t, [0,0])
# Characters
context = 'Game Characters:\
- Character: アーテル == Artel - Gender: Female\
- Character: コリウス == Coleus - Gender: Male\
- Character: ギア == Gear - Gender: Male\
- Character: ロック == Rock - Gender: Male\
- Character: テア == Thea - Gender: Male\
- Character: ヘンジ == Henge - Gender: Male'
+ Character: リッカ == Ricca - Gender: Female\
+ Character: シーナ == Sina - Gender: Female\
+ Character: ヘレナ == Helena - Gender: Female\
+ Character: Miko == Miko - Gender: Female'
# Prompt
if fullPromptFlag:
@@ -1939,6 +1933,12 @@ def translateGPT(t, history, fullPromptFlag):
translatedText = translatedText.replace('Translation =', '')
translatedText = translatedText.replace('Translate =', '')
translatedText = translatedText.replace('っ', '')
+ translatedText = translatedText.replace('ッ', '')
+ translatedText = translatedText.replace('ぁ', '')
+ translatedText = translatedText.replace('。', '.')
+ translatedText = translatedText.replace('、', ',')
+ translatedText = translatedText.replace('?', '?')
+ translatedText = translatedText.replace('!', '!')
# Return Translation
if len(translatedText) > 15 * len(t) or "I'm sorry, but I'm unable to assist with that translation" in translatedText:
diff --git a/modules/tyrano.py b/modules/tyrano.py
index 3c0343c..259b8ab 100644
--- a/modules/tyrano.py
+++ b/modules/tyrano.py
@@ -1,55 +1,59 @@
-from concurrent.futures import ThreadPoolExecutor, as_completed
import os
-from pathlib import Path
import re
import textwrap
import threading
import time
import traceback
-import tiktoken
+from pathlib import Path
+import openai
+import tiktoken
from colorama import Fore
from dotenv import load_dotenv
-import openai
from retry import retry
from tqdm import tqdm
-#Globals
+# Open AI
load_dotenv()
-if os.getenv('api').replace(' ', '') != '':
- openai.api_base = os.getenv('api')
+if os.getenv("api").replace(" ", "") != "":
+ openai.api_base = os.getenv("api")
+openai.organization = os.getenv("org")
+openai.api_key = os.getenv("key")
-openai.organization = os.getenv('org')
-openai.api_key = os.getenv('key')
-MODEL = os.getenv('model')
-TIMEOUT = int(os.getenv('timeout'))
-LANGUAGE=os.getenv('language').capitalize()
-
-APICOST = .002 # Depends on the model https://openai.com/pricing
-PROMPT = Path('prompt.txt').read_text(encoding='utf-8')
-THREADS = int(os.getenv('threads')) # For GPT4 rate limit will be hit if you have more than 1 thread.
+# Globals
+MODEL = os.getenv("model")
+TIMEOUT = int(os.getenv("timeout"))
+LANGUAGE = os.getenv("language").capitalize()
+INPUTAPICOST = 0.002 # Depends on the model https://openai.com/pricing
+OUTPUTAPICOST = 0.002
+PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
+THREADS = int(
+ os.getenv("threads")
+) # Controls how many threads are working on a single file (May have to drop this)
LOCK = threading.Lock()
-WIDTH = int(os.getenv('width'))
-LISTWIDTH = int(os.getenv('listWidth'))
+WIDTH = int(os.getenv("width"))
+LISTWIDTH = int(os.getenv("listWidth"))
+NOTEWIDTH = 40
MAXHISTORY = 10
-ESTIMATE = ''
-TOTALCOST = 0
-TOKENS = 0
-TOTALTOKENS = 0
+ESTIMATE = ""
+totalTokens = [0, 0]
NAMESLIST = []
-#tqdm Globals
-BAR_FORMAT='{l_bar}{bar:10}{r_bar}{bar:-10b}'
-POSITION=0
-LEAVE=False
+# tqdm Globals
+BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
+POSITION = 0
+LEAVE = False
# Flags
-NAMES = False # Output a list of all the character names found
+NAMES = False # Output a list of all the character names found
+BRFLAG = False # If the game uses
instead
FIXTEXTWRAP = True
-IGNORETLTEXT = True
+IGNORETLTEXT = False
+
def handleTyrano(filename, estimate):
- global ESTIMATE, TOKENS, TOTALTOKENS, TOTALCOST
+ global ESTIMATE
+ totalTokens = [0, 0]
ESTIMATE = estimate
if estimate:
@@ -59,416 +63,390 @@ def handleTyrano(filename, estimate):
# Print Result
end = time.time()
tqdm.write(getResultString(translatedData, end - start, filename))
+ if NAMES is True:
+ tqdm.write(str(NAMESLIST))
with LOCK:
- TOTALCOST += translatedData[1] * .001 * APICOST
- TOTALTOKENS += translatedData[1]
+ totalTokens[0] += translatedData[1][0]
+ totalTokens[1] += translatedData[1][1]
+
+ return getResultString(["", totalTokens, None], end - start, "TOTAL")
- return getResultString(['', TOTALTOKENS, None], end - start, 'TOTAL')
-
else:
try:
- with open('translated/' + filename, 'w', encoding='utf-8', errors='ignore') as outFile:
+ with open("translated/" + filename, "w", encoding="utf-8") as outFile:
start = time.time()
translatedData = openFiles(filename)
+ outFile.writelines(translatedData[0])
# Print Result
- outFile.writelines(translatedData[0])
end = time.time()
tqdm.write(getResultString(translatedData, end - start, filename))
with LOCK:
- TOTALCOST += translatedData[1] * .001 * APICOST
- TOTALTOKENS += translatedData[1]
- except Exception as e:
+ totalTokens[0] += translatedData[1][0]
+ totalTokens[1] += translatedData[1][1]
+ except Exception:
traceback.print_exc()
- return 'Fail'
+ return "Fail"
- return getResultString(['', TOTALTOKENS, None], end - start, 'TOTAL')
+ return getResultString(["", totalTokens, None], end - start, "TOTAL")
-def openFiles(filename):
- with open('files/' + filename, 'r', encoding='utf-8') as readFile:
- translatedData = parseTyrano(readFile, filename)
- # Delete lines marked for deletion
- finalData = []
- for line in translatedData[0]:
- if line != '\\d\n':
- finalData.append(line)
- translatedData[0] = finalData
-
- return translatedData
-
-def parseTyrano(readFile, filename):
- totalTokens = 0
- totalLines = 0
-
- # Get total for progress bar
- data = readFile.readlines()
- totalLines = len(data)
-
- with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar:
- pbar.desc=filename
- pbar.total=totalLines
-
- try:
- totalTokens += translateTyrano(data, pbar)
- except Exception as e:
- traceback.print_exc()
- return [data, totalTokens, e]
- return [data, totalTokens, None]
-
-def translateTyrano(data, pbar):
- textHistory = []
- maxHistory = MAXHISTORY
- tokens = 0
- currentGroup = []
- syncIndex = 0
- speaker = ''
- global LOCK, ESTIMATE
-
- for i in range(len(data)):
- if syncIndex > i:
- i = syncIndex
-
- # Speaker
- if '#' in data[i]:
- matchList = re.findall(r'#(.+)', data[i])
- if len(matchList) != 0:
- response = translateGPT(matchList[0], 'Reply with only the '+ LANGUAGE +' translation of the NPC name', True)
- speaker = response[0]
- tokens += response[1]
- data[i] = '#' + speaker + '\n'
- else:
- speaker = ''
-
- # Choices
- elif 'glink' in data[i]:
- matchList = re.findall(r'\[glink.+text=\"(.+?)\".+', data[i])
- if len(matchList) != 0:
- if len(textHistory) > 0:
- originalText = matchList[0]
- response = translateGPT(matchList[0], 'Past Translated Text: ' + textHistory[len(textHistory)-1] + '\n\nReply in the style of a dialogue option.', True)
- else:
- response = translateGPT(matchList[0], '', False)
- translatedText = response[0]
- tokens += response[1]
-
- # Remove characters that may break scripts
- charList = ['.', '\"', '\\n']
- for char in charList:
- translatedText = translatedText.replace(char, '')
-
- # Escape all '
- translatedText = translatedText.replace('\\', '')
- translatedText = translatedText.replace("'", "\\\'")
-
- # Set Data
- translatedText = data[i].replace(matchList[0], translatedText.replace(' ', '\u00A0'))
- data[i] = translatedText
-
- # Grab Lines
- matchList = re.findall(r'(.+?)\[p\]', data[i])
- if len(matchList) > 0:
- matchList[0] = matchList[0].replace('「', '')
- matchList[0] = matchList[0].replace('」', '')
- currentGroup.append(matchList[0])
- if len(data) > i+1:
- while '[r]' in data[i+1]:
- data[i] = '\d\n' # \d Marks line for deletion
- i += 1
- matchList = re.findall(r'(.+?)\[p\]', data[i])
- if len(matchList) > 0:
- matchList[0] = matchList[0].replace('「', '')
- matchList[0] = matchList[0].replace('」', '')
- currentGroup.append(matchList[0])
- while '[p][cm]' in data[i+1]:
- data[i] = '\d\n'
- i += 1
- matchList = re.findall(r'(.+?)\[p\]', data[i])
- if len(matchList) > 0:
- matchList[0] = matchList[0].replace('「', '')
- matchList[0] = matchList[0].replace('」', '')
- currentGroup.append(matchList[0])
- # Join up 401 groups for better translation.
- if len(currentGroup) > 0:
- finalJAString = ' '.join(currentGroup)
- oldjaString = finalJAString
-
- # Remove any textwrap
- if FIXTEXTWRAP == True:
- finalJAString = finalJAString.replace('[r]', ' ')
-
- #Check Speaker
- if speaker == '':
- response = translateGPT(finalJAString, textHistory, True)
- tokens += response[1]
- translatedText = response[0]
- textHistory.append('\"' + translatedText + '\"')
- else:
- response = translateGPT(speaker + ': ' + finalJAString, textHistory, True)
- tokens += response[1]
- translatedText = response[0]
- textHistory.append('\"' + translatedText + '\"')
-
- # Remove added speaker
- translatedText = re.sub(r'^.+:\s?', '', translatedText)
-
- # Set Data
- translatedText = translatedText.replace('ッ', '')
- translatedText = translatedText.replace('っ', '')
- translatedText = translatedText.replace('ー', '')
- translatedText = translatedText.replace('\"', '')
- translatedText = translatedText.replace('[', '')
- translatedText = translatedText.replace(']', '')
-
- # Split final string into full sentences.
- matchList = re.findall(r'(.+?[)\.\?\!)。・]+)', translatedText)
- translatedText = re.sub(r'(.+?[)\.\?\!)。・]+)', '', translatedText)
- for l in range(len(matchList)):
- if any(t in matchList[l] for t in ['Mr.', 'Ms.', 'Mrs.']):
- if len(matchList) > l+1:
- matchList[l] = matchList[l] + matchList[l+1]
- matchList[l+1] = ''
-
- # Combine Lists
- for k in range(len(matchList)):
- matchList[k] = matchList[k].strip()
- j=0
- while(len(matchList) > j+1):
- while len(matchList[j]) < 30 and len(matchList) > j:
- matchList[j:j+2] = [' '.join(matchList[j:j+2])]
- if len(matchList) == j+1:
- matchList[j] = matchList[j] + ' ' + translatedText
- translatedText = ''
- break
- j+=1
-
- # Normal Lines
- if len(matchList) > 0:
- data[i] = '\d\n'
- for line in matchList:
- # Wordwrap Text
- if '[r]' not in line:
- line = textwrap.fill(line, width=WIDTH)
- line = line.replace('\n', '[r]')
-
- # Set
- data.insert(i, line.strip() + '[p][cm]\n')
- i+=1
- data[i-1] = data[i-1].replace('[l][er]', '[p][cm]')
- # else:
- # print ('No Matches')
-
- # Backup TL
- if translatedText != '':
- # Wordwrap Text
- if '[r]' not in translatedText:
- translatedText = textwrap.fill(translatedText, width=WIDTH)
- translatedText = translatedText.replace('\n', '[r]')
-
- # Set Last Line
- data[i] = translatedText.strip() + '[p][cm]\n'
-
- # Keep textHistory list at length maxHistory
- if len(textHistory) > maxHistory:
- textHistory.pop(0)
- currentGroup = []
- speaker = ''
-
- # Grab Lines [p][cm]
- matchList = re.findall(r'(.+?)\[p\]\[cm\]', data[i])
- if len(matchList) > 0:
- matchList[0] = matchList[0].replace('「', '')
- matchList[0] = matchList[0].replace('」', '')
- finalJAString = matchList[0]
-
- # Remove any textwrap
- if FIXTEXTWRAP == True:
- finalJAString = finalJAString.replace('[r]', ' ')
-
- #Check Speaker
- if speaker == '':
- response = translateGPT(finalJAString, 'Previous Dialogue: ' + '\n\n'.join(textHistory), True)
- tokens += response[1]
- translatedText = response[0]
- textHistory.append('\"' + translatedText + '\"')
- else:
- response = translateGPT(speaker + ': ' + finalJAString, 'Previous Dialogue: ' + '\n\n'.join(textHistory), True)
- tokens += response[1]
- translatedText = response[0]
- textHistory.append('\"' + translatedText + '\"')
-
- # Remove added speaker
- translatedText = re.sub(r'^.+:\s?', '', translatedText)
-
- # Set Data
- translatedText = translatedText.replace('ッ', '')
- translatedText = translatedText.replace('っ', '')
- translatedText = translatedText.replace('ー', '')
- translatedText = translatedText.replace('\"', '')
- translatedText = translatedText.replace('[', '')
- translatedText = translatedText.replace(']', '')
-
- # Split final string into full sentences.
- matchList = re.findall(r'(.+?[)\.\?\!)。・]+)', translatedText)
- translatedText = re.sub(r'(.+?[)\.\?\!)。・]+)', '', translatedText)
- for l in range(len(matchList)):
- if any(t in matchList[l] for t in ['Mr.', 'Ms.', 'Mrs.']):
- if len(matchList) > l+1:
- matchList[l] = matchList[l] + matchList[l+1]
- matchList[l+1] = ''
-
- # Get rid of whitespace for each item and add wordwrap
- for k in range(len(matchList)):
- matchList[k] = matchList[k].strip()
-
- # Combine Sentences with a max limit (Wordwrap basically)
- j=0
- while(len(matchList) > j+1):
- while len(matchList[j]) < 30 and len(matchList) > j:
- matchList[j:j+2] = [' '.join(matchList[j:j+2])]
- if len(matchList) == j+1:
- matchList[j] = matchList[j] + ' ' + translatedText
- translatedText = ''
- break
- j+=1
-
- # Set Data
- if len(matchList) > 0:
- data[i] = '\d\n'
- for line in matchList:
- # Wordwrap Text
- if '[r]' not in line:
- line = textwrap.fill(line, width=WIDTH)
- line = line.replace('\n', '[r]')
-
- # Set
- data.insert(i, line.strip() + '[p][cm]\n')
- i+=1
- # Set last line as [pcm] instead of [r]
- data[i-1] = data[i-1].replace('[l][er]', '[p][cm]')
- # else:
- # print ('No Matches')
- if translatedText != '':
- # Wordwrap Text
- if '[r]' not in translatedText:
- translatedText = textwrap.fill(translatedText, width=WIDTH)
- translatedText = translatedText.replace('\n', '[r]')
-
- # Set Backup
- data[i] = translatedText.strip() + '[p][cm]\n'
-
- # Keep textHistory list at length maxHistory
- if len(textHistory) > maxHistory:
- textHistory.pop(0)
- currentGroup = []
- speaker = ''
-
- currentGroup = []
- pbar.update(1)
- if len(data) > i+1:
- syncIndex = i+1
- else:
- break
-
- return tokens
-
def getResultString(translatedData, translationTime, filename):
# File Print String
- tokenString = Fore.YELLOW + '[' + str(translatedData[1]) + \
- ' Tokens/${:,.4f}'.format(translatedData[1] * .001 * APICOST) + ']'
- timeString = Fore.BLUE + '[' + str(round(translationTime, 1)) + 's]'
+ totalTokenstring = (
+ Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
+ "[Output: " + str(translatedData[1][1]) + "]"
+ "[Cost: ${:,.4f}".format(
+ (translatedData[1][0] * 0.001 * INPUTAPICOST)
+ + (translatedData[1][1] * 0.001 * OUTPUTAPICOST)
+ )
+ + "]"
+ )
+ timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
- if translatedData[2] == None:
+ if translatedData[2] is None:
# Success
- return filename + ': ' + tokenString + timeString + Fore.GREEN + u' \u2713 ' + Fore.RESET
+ return (
+ filename
+ + ": "
+ + totalTokenstring
+ + timeString
+ + Fore.GREEN
+ + " \u2713 "
+ + Fore.RESET
+ )
else:
# Fail
try:
raise translatedData[2]
except Exception as e:
- traceback.print_exc()
errorString = str(e) + Fore.RED
- return filename + ': ' + tokenString + timeString + Fore.RED + u' \u2717 ' +\
- errorString + Fore.RESET
-
+ return (
+ filename
+ + ": "
+ + totalTokenstring
+ + timeString
+ + Fore.RED
+ + " \u2717 "
+ + errorString
+ + Fore.RESET
+ )
+
+
+def openFiles(filename):
+ with open("files/" + filename, "r", encoding="utf-8") as readFile:
+ translatedData = parseTyrano(readFile, filename)
+
+ # Delete lines marked for deletion
+ finalData = []
+ for line in translatedData[0]:
+ if line != "\\d\n":
+ finalData.append(line)
+ translatedData[0] = finalData
+
+ return translatedData
+
+
+def parseTyrano(readFile, filename):
+ totalTokens = [0, 0]
+ totalLines = 0
+
+ # Get total for progress bar
+ data = readFile.readlines()
+ totalLines = len(data)
+
+ with tqdm(
+ bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE
+ ) as pbar:
+ pbar.desc = filename
+ pbar.total = totalLines
+
+ try:
+ response = translateTyrano(data, pbar)
+ totalTokens[0] = response[0]
+ totalTokens[1] = response[1]
+ except Exception as e:
+ traceback.print_exc()
+ return [data, totalTokens, e]
+ return [data, totalTokens, None]
+
+
+def translateTyrano(data, pbar):
+ textHistory = []
+ maxHistory = MAXHISTORY
+ tokens = [0, 0]
+ currentGroup = []
+ syncIndex = 0
+ speaker = ""
+ delFlag = False
+ global LOCK, ESTIMATE
+
+ for i in range(len(data)):
+ if syncIndex > i:
+ i = syncIndex
+
+ if '[▼]' in data[i]:
+ data[i] = data[i].replace('[▼]'.strip(), '[page]\n')
+
+ # If there isn't any Japanese in the text just skip
+ if IGNORETLTEXT is True:
+ if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+', data[i]):
+ # Keep textHistory list at length maxHistory
+ textHistory.append('\"' + data[i] + '\"')
+ if len(textHistory) > maxHistory:
+ textHistory.pop(0)
+ currentGroup = []
+ continue
+
+ # Speaker
+ matchList = re.findall(r"^\[(.+)\sstorage=.+\]", data[i])
+ if len(matchList) == 0:
+ matchList = re.findall(r"^\[(.+)\]$", data[i])
+ if len(matchList) != 0:
+ if "主人公" in matchList[0]:
+ speaker = "Protagonist"
+ elif "思考" in matchList[0]:
+ speaker = "Protagonist Inner Thoughts"
+ elif "地の文" in matchList[0]:
+ speaker = "Narrator"
+ elif "マコ" in matchList[0]:
+ speaker = "Mako"
+ else:
+ response = translateGPT(
+ matchList[0],
+ "Reply with only the "
+ + LANGUAGE
+ + " translation of the NPC name",
+ True,
+ )
+ speaker = response[0]
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ # data[i] = '#' + speaker + '\n'
+
+ # Choices
+ elif "glink" in data[i]:
+ matchList = re.findall(r"\[glink.+text=\"(.+?)\".+", data[i])
+ if len(matchList) != 0:
+ if len(textHistory) > 0:
+ response = translateGPT(
+ matchList[0],
+ "Past Translated Text: "
+ + textHistory[len(textHistory) - 1]
+ + "\n\nReply in the style of a dialogue option.",
+ True,
+ )
+ else:
+ response = translateGPT(matchList[0], "", False)
+ translatedText = response[0]
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+
+ # Remove characters that may break scripts
+ charList = [".", '"', "\\n"]
+ for char in charList:
+ translatedText = translatedText.replace(char, "")
+
+ # Escape all '
+ translatedText = translatedText.replace("\\", "")
+ translatedText = translatedText.replace("'", "\\'")
+
+ # Set Data
+ translatedText = data[i].replace(
+ matchList[0], translatedText.replace(" ", "\u00A0")
+ )
+ data[i] = translatedText
+
+ # Grab Lines
+ matchList = re.findall(r"^([^\n;@*\[].+)", data[i])
+ if len(matchList) > 0 and ('storage' in data[i-1] or re.search(r'^\[(.+)\]$', data[i-1])):
+ currentGroup.append(matchList[0])
+ if len(data) > i + 1:
+ matchList = re.findall(r"^([^\n;@*\[].+)", data[i + 1])
+ while len(matchList) > 0:
+ delFlag = True
+ data[i] = "\d\n" # \d Marks line for deletion
+ i += 1
+ matchList = re.findall(r"^([^\n;@*\[].+)", data[i])
+ if len(matchList) > 0:
+ currentGroup.append(matchList[0])
+
+ # Join up 401 groups for better translation.
+ if len(currentGroup) > 0:
+ finalJAString = " ".join(currentGroup)
+
+ # Remove any textwrap
+ if FIXTEXTWRAP is True:
+ finalJAString = finalJAString.replace("_", " ")
+
+ # Check Speaker
+ if speaker == "":
+ response = translateGPT(finalJAString, textHistory, True)
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ translatedText = response[0]
+ textHistory.append('"' + translatedText + '"')
+ else:
+ response = translateGPT(
+ speaker + ": " + finalJAString, textHistory, True
+ )
+ tokens[0] += response[1][0]
+ tokens[1] += response[1][1]
+ translatedText = response[0]
+ textHistory.append('"' + translatedText + '"')
+
+ # Remove added speaker
+ translatedText = re.sub(r"^.+:\s?", "", translatedText)
+
+ # Set Data
+ translatedText = translatedText.replace("ッ", "")
+ translatedText = translatedText.replace("っ", "")
+ translatedText = translatedText.replace("ー", "")
+ translatedText = translatedText.replace('"', "")
+ translatedText = translatedText.replace("[", "")
+ translatedText = translatedText.replace("]", "")
+
+ # Wordwrap Text
+ if "_" not in translatedText:
+ translatedText = textwrap.fill(translatedText, width=WIDTH)
+ translatedText = translatedText.replace("\n", "_")
+
+ # Set
+ if delFlag is True:
+ data.insert(i, translatedText.strip() + '\n')
+ delFlag = False
+ else:
+ data[i] = translatedText.strip() + '\n'
+
+ # Keep textHistory list at length maxHistory
+ if len(textHistory) > maxHistory:
+ textHistory.pop(0)
+ currentGroup = []
+ speaker = ""
+
+ currentGroup = []
+ pbar.update(1)
+ if len(data) > i + 1:
+ syncIndex = i + 1
+ else:
+ break
+
+ return tokens
+
+
def subVars(jaString):
- jaString = jaString.replace('\u3000', ' ')
+ jaString = jaString.replace("\u3000", " ")
+
+ # Nested
+ count = 0
+ nestedList = re.findall(r"[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]", jaString)
+ nestedList = set(nestedList)
+ if len(nestedList) != 0:
+ for icon in nestedList:
+ jaString = jaString.replace(icon, "{Nested_" + str(count) + "}")
+ count += 1
# Icons
count = 0
- iconList = re.findall(r'[\\]+[iIkKwW]+\[[0-9]+\]', jaString)
+ iconList = re.findall(r"[\\]+[iIkKwWaA]+\[[0-9]+\]", jaString)
iconList = set(iconList)
if len(iconList) != 0:
for icon in iconList:
- jaString = jaString.replace(icon, '[Icon' + str(count) + ']')
+ jaString = jaString.replace(icon, "{Ascii_" + str(count) + "}")
count += 1
# Colors
count = 0
- colorList = re.findall(r'[\\]+[cC]\[[0-9]+\]', jaString)
+ colorList = re.findall(r"[\\]+[cC]\[[0-9]+\]", jaString)
colorList = set(colorList)
if len(colorList) != 0:
for color in colorList:
- jaString = jaString.replace(color, '[Color' + str(count) + ']')
+ jaString = jaString.replace(color, "{Color_" + str(count) + "}")
count += 1
# Names
count = 0
- nameList = re.findall(r'[\\]+[nN]\[[0-9]+\]', jaString)
+ nameList = re.findall(r"[\\]+[nN]\[.+?\]+", jaString)
nameList = set(nameList)
if len(nameList) != 0:
for name in nameList:
- jaString = jaString.replace(name, '[Name' + str(count) + ']')
+ jaString = jaString.replace(name, "{N_" + str(count) + "}")
count += 1
# Variables
count = 0
- varList = re.findall(r'[\\]+[vV]\[[0-9]+\]', jaString)
+ varList = re.findall(r"[\\]+[vV]\[[0-9]+\]", jaString)
varList = set(varList)
if len(varList) != 0:
for var in varList:
- jaString = jaString.replace(var, '[Var' + str(count) + ']')
+ jaString = jaString.replace(var, "{Var_" + str(count) + "}")
+ count += 1
+
+ # Formatting
+ count = 0
+ if "笑えるよね." in jaString:
+ print("t")
+ formatList = re.findall(r"[\\]+[\w]+\[.+?\]", jaString)
+ formatList = set(formatList)
+ if len(formatList) != 0:
+ for var in formatList:
+ jaString = jaString.replace(var, "{FCode_" + str(count) + "}")
count += 1
# Put all lists in list and return
- allList = [iconList, colorList, nameList, varList]
+ allList = [nestedList, iconList, colorList, nameList, varList, formatList]
return [jaString, allList]
+
def resubVars(translatedText, allList):
# Fix Spacing and ChatGPT Nonsense
- matchList = re.findall(r'\[\s?.+?\s?\]', translatedText)
+ matchList = re.findall(r"\[\s?.+?\s?\]", translatedText)
if len(matchList) > 0:
for match in matchList:
text = match.strip()
translatedText = translatedText.replace(match, text)
- # Icons
+ # Nested
count = 0
if len(allList[0]) != 0:
for var in allList[0]:
- translatedText = translatedText.replace('[Icon' + str(count) + ']', var)
+ translatedText = translatedText.replace("{Nested_" + str(count) + "}", var)
+ count += 1
+
+ # Icons
+ count = 0
+ if len(allList[1]) != 0:
+ for var in allList[1]:
+ translatedText = translatedText.replace("{Ascii_" + str(count) + "}", var)
count += 1
# Colors
count = 0
- if len(allList[1]) != 0:
- for var in allList[1]:
- translatedText = translatedText.replace('[Color' + str(count) + ']', var)
+ if len(allList[2]) != 0:
+ for var in allList[2]:
+ translatedText = translatedText.replace("{Color_" + str(count) + "}", var)
count += 1
# Names
count = 0
- if len(allList[2]) != 0:
- for var in allList[2]:
- translatedText = translatedText.replace('[Name' + str(count) + ']', var)
+ if len(allList[3]) != 0:
+ for var in allList[3]:
+ translatedText = translatedText.replace("{N_" + str(count) + "}", var)
count += 1
# Vars
count = 0
- if len(allList[3]) != 0:
- for var in allList[3]:
- translatedText = translatedText.replace('[Var' + str(count) + ']', var)
+ if len(allList[4]) != 0:
+ for var in allList[4]:
+ translatedText = translatedText.replace("{Var_" + str(count) + "}", var)
+ count += 1
+
+ # Formatting
+ count = 0
+ if len(allList[5]) != 0:
+ for var in allList[5]:
+ translatedText = translatedText.replace("{FCode_" + str(count) + "}", var)
count += 1
# Remove Color Variables Spaces
@@ -477,38 +455,54 @@ def resubVars(translatedText, allList):
# translatedText = re.sub(r'\s*(\\+c\[0+\])', r'\1', translatedText)
return translatedText
+
@retry(exceptions=Exception, tries=5, delay=5)
def translateGPT(t, history, fullPromptFlag):
- # If ESTIMATE is True just count this as an execution and return.
- if ESTIMATE:
- enc = tiktoken.encoding_for_model(MODEL)
- tokens = len(enc.encode(t)) * 2 + len(enc.encode(str(history))) + len(enc.encode(PROMPT))
- return (t, tokens)
-
# Sub Vars
varResponse = subVars(t)
subbedT = varResponse[0]
# If there isn't any Japanese in the text just skip
- if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴ]+|[\uFF00-\uFFEF]', subbedT):
- return(t, 0)
+ if not re.search(r"[一-龠]+|[ぁ-ゔ]+|[ァ-ヴ]+|[\uFF00-\uFFEF]", subbedT):
+ return (t, [0, 0])
+
+ # If ESTIMATE is True just count this as an execution and return.
+ if ESTIMATE:
+ enc = tiktoken.encoding_for_model(MODEL)
+ historyRaw = ""
+ if isinstance(history, list):
+ for line in history:
+ historyRaw += line
+ else:
+ historyRaw = history
+
+ inputTotalTokens = len(enc.encode(historyRaw)) + len(enc.encode(PROMPT))
+ outputTotalTokens = (
+ len(enc.encode(t)) * 2
+ ) # Estimating 2x the size of the original text
+ totalTokens = [inputTotalTokens, outputTotalTokens]
+ return (t, totalTokens)
# Characters
- context = '```\
- Game Characters:\
- Character: 亜紀子 == Akiko - Gender: Female\
- Character: 明日香 == Asuka - Gender: Female\
- ```'
+ context = "Game Characters:\
+ Character: マコ == Mako - Gender: Female\
+ Character: 主人公 == Protagonist - Gender: Male"
# Prompt
if fullPromptFlag:
system = PROMPT
- user = 'Line to Translate = ' + subbedT
+ user = "Line to Translate = " + subbedT
else:
- system = 'Output ONLY the '+ LANGUAGE +' translation in the following format: `Translation: <'+ LANGUAGE.upper() +'_TRANSLATION>`'
- user = 'Line to Translate = ' + subbedT
+ system = (
+ "Output ONLY the "
+ + LANGUAGE
+ + " translation in the following format: `Translation: <"
+ + LANGUAGE.upper()
+ + "_TRANSLATION>`"
+ )
+ user = "Line to Translate = " + subbedT
- # Create Message List
+ # Create Message List
msg = []
msg.append({"role": "system", "content": system})
msg.append({"role": "user", "content": context})
@@ -520,7 +514,7 @@ def translateGPT(t, history, fullPromptFlag):
msg.append({"role": "user", "content": user})
response = openai.ChatCompletion.create(
- temperature=0.1,
+ temperature=0,
frequency_penalty=0.2,
presence_penalty=0.2,
model=MODEL,
@@ -530,27 +524,35 @@ def translateGPT(t, history, fullPromptFlag):
# Save Translated Text
translatedText = response.choices[0].message.content
- tokens = response.usage.total_tokens
+ totalTokens = [response.usage.prompt_tokens, response.usage.completion_tokens]
# Resub Vars
translatedText = resubVars(translatedText, varResponse[1])
# Remove Placeholder Text
- translatedText = translatedText.replace(LANGUAGE +' Translation: ', '')
- translatedText = translatedText.replace('Translation: ', '')
- translatedText = translatedText.replace('Line to Translate = ', '')
- translatedText = translatedText.replace('Translation = ', '')
- translatedText = translatedText.replace('Translate = ', '')
- translatedText = translatedText.replace(LANGUAGE +' Translation:', '')
- translatedText = translatedText.replace('Translation:', '')
- translatedText = translatedText.replace('Line to Translate =', '')
- translatedText = translatedText.replace('Translation =', '')
- translatedText = translatedText.replace('Translate =', '')
- translatedText = re.sub(r'Note:.*', '', translatedText)
- translatedText = translatedText.replace('っ', '')
+ translatedText = translatedText.replace(LANGUAGE + " Translation: ", "")
+ translatedText = translatedText.replace("Translation: ", "")
+ translatedText = translatedText.replace("Line to Translate = ", "")
+ translatedText = translatedText.replace("Translation = ", "")
+ translatedText = translatedText.replace("Translate = ", "")
+ translatedText = translatedText.replace(LANGUAGE + " Translation:", "")
+ translatedText = translatedText.replace("Translation:", "")
+ translatedText = translatedText.replace("Line to Translate =", "")
+ translatedText = translatedText.replace("Translation =", "")
+ translatedText = translatedText.replace("Translate =", "")
+ translatedText = translatedText.replace("っ", "")
+ translatedText = translatedText.replace("ッ", "")
+ translatedText = translatedText.replace("ぁ", "")
+ translatedText = translatedText.replace("。", ".")
+ translatedText = translatedText.replace("、", ",")
+ translatedText = translatedText.replace("?", "?")
+ translatedText = translatedText.replace("!", "!")
# Return Translation
- if len(translatedText) > 15 * len(t) or "I'm sorry, but I'm unable to assist with that translation" in translatedText:
+ if (
+ len(translatedText) > 15 * len(t)
+ or "I'm sorry, but I'm unable to assist with that translation" in translatedText
+ ):
raise Exception
else:
- return [translatedText, tokens]
+ return [translatedText, totalTokens]