Save latest
This commit is contained in:
parent
a2bb551395
commit
fe62cbb11e
4 changed files with 867 additions and 468 deletions
380
modules/atelier.py
Normal file
380
modules/atelier.py
Normal file
|
|
@ -0,0 +1,380 @@
|
|||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import textwrap
|
||||
import threading
|
||||
import time
|
||||
import traceback
|
||||
import tiktoken
|
||||
from colorama import Fore
|
||||
from dotenv import load_dotenv
|
||||
import openai
|
||||
from retry import retry
|
||||
from tqdm import tqdm
|
||||
|
||||
# Open AI
|
||||
load_dotenv()
|
||||
if os.getenv('api').replace(' ', '') != '':
|
||||
openai.api_base = os.getenv('api')
|
||||
openai.organization = os.getenv('org')
|
||||
openai.api_key = os.getenv('key')
|
||||
|
||||
#Globals
|
||||
MODEL = os.getenv('model')
|
||||
TIMEOUT = int(os.getenv('timeout'))
|
||||
LANGUAGE=os.getenv('language').capitalize()
|
||||
INPUTAPICOST = .002 # Depends on the model https://openai.com/pricing
|
||||
OUTPUTAPICOST = .002
|
||||
PROMPT = Path('prompt.txt').read_text(encoding='utf-8')
|
||||
THREADS = int(os.getenv('threads')) # Controls how many threads are working on a single file (May have to drop this)
|
||||
LOCK = threading.Lock()
|
||||
WIDTH = int(os.getenv('width'))
|
||||
LISTWIDTH = int(os.getenv('listWidth'))
|
||||
NOTEWIDTH = 40
|
||||
MAXHISTORY = 10
|
||||
ESTIMATE = ''
|
||||
totalTokens = [0, 0]
|
||||
NAMESLIST = []
|
||||
|
||||
#tqdm Globals
|
||||
BAR_FORMAT='{l_bar}{bar:10}{r_bar}{bar:-10b}'
|
||||
POSITION=0
|
||||
LEAVE=False
|
||||
|
||||
# Translation Flags
|
||||
FIXTEXTWRAP = True
|
||||
IGNORETLTEXT = True
|
||||
|
||||
def handleAtelier(filename, estimate):
|
||||
global ESTIMATE, totalTokens
|
||||
ESTIMATE = estimate
|
||||
|
||||
if estimate:
|
||||
start = time.time()
|
||||
translatedData = openFiles(filename)
|
||||
|
||||
# Print Result
|
||||
end = time.time()
|
||||
tqdm.write(getResultString(translatedData, end - start, filename))
|
||||
with LOCK:
|
||||
totalTokens[0] += translatedData[1][0]
|
||||
totalTokens[1] += translatedData[1][1]
|
||||
|
||||
return getResultString(['', totalTokens, None], end - start, 'TOTAL')
|
||||
|
||||
else:
|
||||
try:
|
||||
with open('translated/' + filename, 'w', encoding='utf-8'):
|
||||
start = time.time()
|
||||
translatedData = openFiles(filename)
|
||||
|
||||
# Print Result
|
||||
end = time.time()
|
||||
tqdm.write(getResultString(translatedData, end - start, filename))
|
||||
with LOCK:
|
||||
totalTokens[0] += translatedData[1][0]
|
||||
totalTokens[1] += translatedData[1][1]
|
||||
except Exception:
|
||||
return 'Fail'
|
||||
|
||||
return getResultString(['', totalTokens, None], end - start, 'TOTAL')
|
||||
|
||||
def openFiles(filename):
|
||||
with open('files/' + filename, 'r', encoding='UTF-8') as f:
|
||||
translatedData = parseText(f, filename)
|
||||
|
||||
return translatedData
|
||||
|
||||
def getResultString(translatedData, translationTime, filename):
|
||||
# File Print String
|
||||
totalTokenstring =\
|
||||
Fore.YELLOW +\
|
||||
'[Input: ' + str(translatedData[1][0]) + ']'\
|
||||
'[Output: ' + str(translatedData[1][1]) + ']'\
|
||||
'[Cost: ${:,.4f}'.format((translatedData[1][0] * .001 * INPUTAPICOST) +\
|
||||
(translatedData[1][1] * .001 * OUTPUTAPICOST)) + ']'
|
||||
timeString = Fore.BLUE + '[' + str(round(translationTime, 1)) + 's]'
|
||||
|
||||
if translatedData[2] is None:
|
||||
# Success
|
||||
return filename + ': ' + totalTokenstring + timeString + Fore.GREEN + u' \u2713 ' + Fore.RESET
|
||||
|
||||
else:
|
||||
# Fail
|
||||
try:
|
||||
raise translatedData[2]
|
||||
except Exception as e:
|
||||
errorString = str(e) + Fore.RED
|
||||
return filename + ': ' + totalTokenstring + timeString + Fore.RED + u' \u2717 ' +\
|
||||
errorString + Fore.RESET
|
||||
|
||||
def parseText(data, filename):
|
||||
totalLines = 0
|
||||
global LOCK
|
||||
|
||||
# Get total for progress bar
|
||||
linesList = data.readlines()
|
||||
totalLines = len(linesList)
|
||||
|
||||
with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar:
|
||||
pbar.desc=filename
|
||||
pbar.total=totalLines
|
||||
try:
|
||||
response = translateText(linesList, pbar)
|
||||
except Exception as e:
|
||||
traceback.print_exc()
|
||||
return [linesList, 0, e]
|
||||
return [response[0], response[1], None]
|
||||
|
||||
def translateText(data, pbar):
|
||||
textHistory = []
|
||||
maxHistory = MAXHISTORY
|
||||
totalTokens = [0,0]
|
||||
syncIndex = 0
|
||||
|
||||
for i in range(len(data)):
|
||||
if syncIndex > i:
|
||||
i = syncIndex
|
||||
|
||||
match = re.findall(r'◆.+◆(.+)', data[i])
|
||||
if len(match) > 0:
|
||||
jaString = match[0]
|
||||
|
||||
### Translate
|
||||
# Remove any textwrap
|
||||
finalJAString = re.sub(r'\\n', ' ', jaString)
|
||||
|
||||
# Translate
|
||||
response = translateGPT(finalJAString, 'Previous Text for Context: ' + ' '.join(textHistory), True)
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
translatedText = response[0]
|
||||
|
||||
# TextHistory is what we use to give GPT Context, so thats appended here.
|
||||
textHistory.append('\"' + translatedText + '\"')
|
||||
|
||||
# Keep textHistory list at length maxHistory
|
||||
if len(textHistory) > maxHistory:
|
||||
textHistory.pop(0)
|
||||
|
||||
# Textwrap
|
||||
translatedText = translatedText.replace('\"', '\\"')
|
||||
translatedText = textwrap.fill(translatedText, width=WIDTH)
|
||||
|
||||
# Write
|
||||
data[i] = data[i].replace(match[0], translatedText)
|
||||
|
||||
syncIndex = i + 1
|
||||
pbar.update()
|
||||
return [data, totalTokens]
|
||||
|
||||
def subVars(jaString):
|
||||
jaString = jaString.replace('\u3000', ' ')
|
||||
|
||||
# Nested
|
||||
count = 0
|
||||
nestedList = re.findall(r'[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]', jaString)
|
||||
nestedList = set(nestedList)
|
||||
if len(nestedList) != 0:
|
||||
for icon in nestedList:
|
||||
jaString = jaString.replace(icon, '{Nested_' + str(count) + '}')
|
||||
count += 1
|
||||
|
||||
# Icons
|
||||
count = 0
|
||||
iconList = re.findall(r'[\\]+[iIkKwWaA]+\[[0-9]+\]', jaString)
|
||||
iconList = set(iconList)
|
||||
if len(iconList) != 0:
|
||||
for icon in iconList:
|
||||
jaString = jaString.replace(icon, '{Ascii_' + str(count) + '}')
|
||||
count += 1
|
||||
|
||||
# Colors
|
||||
count = 0
|
||||
colorList = re.findall(r'[\\]+[cC]\[[0-9]+\]', jaString)
|
||||
colorList = set(colorList)
|
||||
if len(colorList) != 0:
|
||||
for color in colorList:
|
||||
jaString = jaString.replace(color, '{Color_' + str(count) + '}')
|
||||
count += 1
|
||||
|
||||
# Names
|
||||
count = 0
|
||||
nameList = re.findall(r'[\\]+[nN]\[.+?\]+', jaString)
|
||||
nameList = set(nameList)
|
||||
if len(nameList) != 0:
|
||||
for name in nameList:
|
||||
jaString = jaString.replace(name, '{N_' + str(count) + '}')
|
||||
count += 1
|
||||
|
||||
# Variables
|
||||
count = 0
|
||||
varList = re.findall(r'[\\]+[vV]\[[0-9]+\]', jaString)
|
||||
varList = set(varList)
|
||||
if len(varList) != 0:
|
||||
for var in varList:
|
||||
jaString = jaString.replace(var, '{Var_' + str(count) + '}')
|
||||
count += 1
|
||||
|
||||
# Formatting
|
||||
count = 0
|
||||
if '笑えるよね.' in jaString:
|
||||
print('t')
|
||||
formatList = re.findall(r'[\\]+[\w]+\[.+?\]', jaString)
|
||||
formatList = set(formatList)
|
||||
if len(formatList) != 0:
|
||||
for var in formatList:
|
||||
jaString = jaString.replace(var, '{FCode_' + str(count) + '}')
|
||||
count += 1
|
||||
|
||||
# Put all lists in list and return
|
||||
allList = [nestedList, iconList, colorList, nameList, varList, formatList]
|
||||
return [jaString, allList]
|
||||
|
||||
def resubVars(translatedText, allList):
|
||||
# Fix Spacing and ChatGPT Nonsense
|
||||
matchList = re.findall(r'\[\s?.+?\s?\]', translatedText)
|
||||
if len(matchList) > 0:
|
||||
for match in matchList:
|
||||
text = match.strip()
|
||||
translatedText = translatedText.replace(match, text)
|
||||
|
||||
# Nested
|
||||
count = 0
|
||||
if len(allList[0]) != 0:
|
||||
for var in allList[0]:
|
||||
translatedText = translatedText.replace('{Nested_' + str(count) + '}', var)
|
||||
count += 1
|
||||
|
||||
# Icons
|
||||
count = 0
|
||||
if len(allList[1]) != 0:
|
||||
for var in allList[1]:
|
||||
translatedText = translatedText.replace('{Ascii_' + str(count) + '}', var)
|
||||
count += 1
|
||||
|
||||
# Colors
|
||||
count = 0
|
||||
if len(allList[2]) != 0:
|
||||
for var in allList[2]:
|
||||
translatedText = translatedText.replace('{Color_' + str(count) + '}', var)
|
||||
count += 1
|
||||
|
||||
# Names
|
||||
count = 0
|
||||
if len(allList[3]) != 0:
|
||||
for var in allList[3]:
|
||||
translatedText = translatedText.replace('{N_' + str(count) + '}', var)
|
||||
count += 1
|
||||
|
||||
# Vars
|
||||
count = 0
|
||||
if len(allList[4]) != 0:
|
||||
for var in allList[4]:
|
||||
translatedText = translatedText.replace('{Var_' + str(count) + '}', var)
|
||||
count += 1
|
||||
|
||||
# Formatting
|
||||
count = 0
|
||||
if len(allList[5]) != 0:
|
||||
for var in allList[5]:
|
||||
translatedText = translatedText.replace('{FCode_' + str(count) + '}', var)
|
||||
count += 1
|
||||
|
||||
# Remove Color Variables Spaces
|
||||
# if '\\c' in translatedText:
|
||||
# translatedText = re.sub(r'\s*(\\+c\[[1-9]+\])\s*', r' \1', translatedText)
|
||||
# translatedText = re.sub(r'\s*(\\+c\[0+\])', r'\1', translatedText)
|
||||
return translatedText
|
||||
|
||||
@retry(exceptions=Exception, tries=5, delay=5)
|
||||
def translateGPT(t, history, fullPromptFlag):
|
||||
# Sub Vars
|
||||
varResponse = subVars(t)
|
||||
subbedT = varResponse[0]
|
||||
|
||||
# If there isn't any Japanese in the text just skip
|
||||
if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴ]+|[\uFF00-\uFFEF]', subbedT):
|
||||
return(t, [0,0])
|
||||
|
||||
# If ESTIMATE is True just count this as an execution and return.
|
||||
if ESTIMATE:
|
||||
enc = tiktoken.encoding_for_model(MODEL)
|
||||
historyRaw = ''
|
||||
if isinstance(history, list):
|
||||
for line in history:
|
||||
historyRaw += line
|
||||
else:
|
||||
historyRaw = history
|
||||
|
||||
inputTotalTokens = len(enc.encode(historyRaw)) + len(enc.encode(PROMPT))
|
||||
outputTotalTokens = len(enc.encode(t)) * 2 # Estimating 2x the size of the original text
|
||||
totalTokens = [inputTotalTokens, outputTotalTokens]
|
||||
return (t, totalTokens)
|
||||
|
||||
# Characters
|
||||
context = 'Game Characters:\
|
||||
Character: リッカ == Ricca - Gender: Female\
|
||||
Character: シーナ == Sina - Gender: Female\
|
||||
Character: ヘレナ == Helena - Gender: Female\
|
||||
Character: Miko == Miko - Gender: Female'
|
||||
|
||||
# Prompt
|
||||
if fullPromptFlag:
|
||||
system = PROMPT
|
||||
user = 'Line to Translate = ' + subbedT
|
||||
else:
|
||||
system = 'Output ONLY the '+ LANGUAGE +' translation in the following format: `Translation: <'+ LANGUAGE.upper() +'_TRANSLATION>`'
|
||||
user = 'Line to Translate = ' + subbedT
|
||||
|
||||
# Create Message List
|
||||
msg = []
|
||||
msg.append({"role": "system", "content": system})
|
||||
msg.append({"role": "user", "content": context})
|
||||
if isinstance(history, list):
|
||||
for line in history:
|
||||
msg.append({"role": "user", "content": line})
|
||||
else:
|
||||
msg.append({"role": "user", "content": history})
|
||||
msg.append({"role": "user", "content": user})
|
||||
|
||||
response = openai.ChatCompletion.create(
|
||||
temperature=0,
|
||||
frequency_penalty=0.2,
|
||||
presence_penalty=0.2,
|
||||
model=MODEL,
|
||||
messages=msg,
|
||||
request_timeout=TIMEOUT,
|
||||
)
|
||||
|
||||
# Save Translated Text
|
||||
translatedText = response.choices[0].message.content
|
||||
totalTokens = [response.usage.prompt_tokens, response.usage.completion_tokens]
|
||||
|
||||
# Resub Vars
|
||||
translatedText = resubVars(translatedText, varResponse[1])
|
||||
|
||||
# Remove Placeholder Text
|
||||
translatedText = translatedText.replace(LANGUAGE +' Translation: ', '')
|
||||
translatedText = translatedText.replace('Translation: ', '')
|
||||
translatedText = translatedText.replace('Line to Translate = ', '')
|
||||
translatedText = translatedText.replace('Translation = ', '')
|
||||
translatedText = translatedText.replace('Translate = ', '')
|
||||
translatedText = translatedText.replace(LANGUAGE +' Translation:', '')
|
||||
translatedText = translatedText.replace('Translation:', '')
|
||||
translatedText = translatedText.replace('Line to Translate =', '')
|
||||
translatedText = translatedText.replace('Translation =', '')
|
||||
translatedText = translatedText.replace('Translate =', '')
|
||||
translatedText = translatedText.replace('っ', '')
|
||||
translatedText = translatedText.replace('ッ', '')
|
||||
translatedText = translatedText.replace('ぁ', '')
|
||||
translatedText = translatedText.replace('。', '.')
|
||||
translatedText = translatedText.replace('、', ',')
|
||||
translatedText = translatedText.replace('？', '?')
|
||||
translatedText = translatedText.replace('！', '!')
|
||||
|
||||
# Return Translation
|
||||
if len(translatedText) > 15 * len(t) or "I'm sorry, but I'm unable to assist with that translation" in translatedText:
|
||||
raise Exception
|
||||
else:
|
||||
return [translatedText, totalTokens]
|
||||
|
|
@ -12,8 +12,8 @@ from modules.txt import handleTXT
|
|||
from modules.tyrano import handleTyrano
|
||||
from modules.json import handleJSON
|
||||
from modules.kansen import handleKansen
|
||||
from modules.lune import handleLune
|
||||
from modules.lune2 import handleLuneTxt
|
||||
from modules.atelier import handleAtelier
|
||||
|
||||
# For GPT4 rate limit will be hit if you have more than 1 thread.
|
||||
# 1 Thread for each file. Controls how many files are worked on at once.
|
||||
|
|
@ -30,14 +30,17 @@ def main():
|
|||
while estimate == '':
|
||||
estimate = input('Select Translation or Cost Estimation:\n\n1. Translate\n2. Estimate\n')
|
||||
match estimate:
|
||||
case '1': estimate = False
|
||||
case '2': estimate = True
|
||||
case _: estimate = ''
|
||||
case '1':
|
||||
estimate = False
|
||||
case '2':
|
||||
estimate = True
|
||||
case _:
|
||||
estimate = ''
|
||||
|
||||
totalCost = 0
|
||||
version = ''
|
||||
while version == '':
|
||||
version = input('Select the RPGMaker Version:\n\n1. MV/MZ\n2. ACE\n3. CSV (From Translator++)\n4. Text (Custom)\n5. Tyrano\n6. JSON\n7. Kansen\n8. Lune\n')
|
||||
version = input('Select the RPGMaker Version:\n\n1. MV/MZ\n2. ACE\n3. CSV (From Translator++)\n4. Text (Custom)\n5. Tyrano\n6. JSON\n7. Kansen\n8. Lune\n9. Atelier\n')
|
||||
match version:
|
||||
case '1':
|
||||
# Open File (Threads)
|
||||
|
|
@ -149,11 +152,25 @@ def main():
|
|||
tracebackLineNo = str(traceback.extract_tb(sys.exc_info()[2])[-1].lineno)
|
||||
tqdm.write(Fore.RED + str(e) + '|' + tracebackLineNo + Fore.RESET)
|
||||
|
||||
case '9':
|
||||
# Open File (Threads)
|
||||
with ThreadPoolExecutor(max_workers=THREADS) as executor:
|
||||
futures = [executor.submit(handleAtelier, filename, estimate) \
|
||||
for filename in os.listdir("files") if filename.endswith('txt')]
|
||||
|
||||
for future in as_completed(futures):
|
||||
try:
|
||||
totalCost = future.result()
|
||||
|
||||
except Exception as e:
|
||||
tracebackLineNo = str(traceback.extract_tb(sys.exc_info()[2])[-1].lineno)
|
||||
tqdm.write(Fore.RED + str(e) + '|' + tracebackLineNo + Fore.RESET)
|
||||
|
||||
case _:
|
||||
version = ''
|
||||
|
||||
if totalCost != 'Fail':
|
||||
if estimate == False:
|
||||
if estimate is False:
|
||||
# This is to encourage people to grab what's in /translated instead
|
||||
deleteFolderFiles('files')
|
||||
|
||||
|
|
|
|||
|
|
@ -3,30 +3,28 @@ import json
|
|||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import sys
|
||||
import textwrap
|
||||
import threading
|
||||
import time
|
||||
import traceback
|
||||
import tiktoken
|
||||
|
||||
from colorama import Fore
|
||||
from dotenv import load_dotenv
|
||||
import openai
|
||||
from retry import retry
|
||||
from tqdm import tqdm
|
||||
|
||||
#Globals
|
||||
# Open AI
|
||||
load_dotenv()
|
||||
if os.getenv('api').replace(' ', '') != '':
|
||||
openai.api_base = os.getenv('api')
|
||||
|
||||
openai.organization = os.getenv('org')
|
||||
openai.api_key = os.getenv('key')
|
||||
|
||||
#Globals
|
||||
MODEL = os.getenv('model')
|
||||
TIMEOUT = int(os.getenv('timeout'))
|
||||
LANGUAGE=os.getenv('language').capitalize()
|
||||
|
||||
INPUTAPICOST = .002 # Depends on the model https://openai.com/pricing
|
||||
OUTPUTAPICOST = .002
|
||||
PROMPT = Path('prompt.txt').read_text(encoding='utf-8')
|
||||
|
|
@ -49,17 +47,17 @@ LEAVE=False
|
|||
CODE401 = True
|
||||
CODE405 = False
|
||||
CODE102 = True
|
||||
CODE122 = True
|
||||
CODE122 = False
|
||||
CODE101 = False
|
||||
CODE355655 = False
|
||||
CODE357 = False
|
||||
CODE657 = False
|
||||
CODE356 = True
|
||||
CODE356 = False
|
||||
CODE320 = False
|
||||
CODE324 = False
|
||||
CODE111 = False
|
||||
CODE408 = False
|
||||
CODE108 = True
|
||||
CODE108 = False
|
||||
NAMES = False # Output a list of all the character names found
|
||||
BRFLAG = False # If the game uses <br> instead
|
||||
FIXTEXTWRAP = True
|
||||
|
|
@ -76,7 +74,7 @@ def handleMVMZ(filename, estimate):
|
|||
# Print Result
|
||||
end = time.time()
|
||||
tqdm.write(getResultString(translatedData, end - start, filename))
|
||||
if NAMES == True:
|
||||
if NAMES is True:
|
||||
tqdm.write(str(NAMESLIST))
|
||||
with LOCK:
|
||||
totalTokens[0] += translatedData[1][0]
|
||||
|
|
@ -86,7 +84,7 @@ def handleMVMZ(filename, estimate):
|
|||
|
||||
else:
|
||||
try:
|
||||
with open('translated/' + filename, 'w', encoding='UTF-8') as outFile:
|
||||
with open('translated/' + filename, 'w', encoding='utf-8') as outFile:
|
||||
start = time.time()
|
||||
translatedData = openFiles(filename)
|
||||
|
||||
|
|
@ -97,13 +95,13 @@ def handleMVMZ(filename, estimate):
|
|||
with LOCK:
|
||||
totalTokens[0] += translatedData[1][0]
|
||||
totalTokens[1] += translatedData[1][1]
|
||||
except Exception as e:
|
||||
except Exception:
|
||||
return 'Fail'
|
||||
|
||||
return getResultString(['', totalTokens, None], end - start, 'TOTAL')
|
||||
|
||||
def openFiles(filename):
|
||||
with open('files/' + filename, 'r', encoding='UTF-8') as f:
|
||||
with open('files/' + filename, 'r', encoding='utf-8-sig') as f:
|
||||
data = json.load(f)
|
||||
|
||||
# Map Files
|
||||
|
|
@ -177,7 +175,7 @@ def getResultString(translatedData, translationTime, filename):
|
|||
(translatedData[1][1] * .001 * OUTPUTAPICOST)) + ']'
|
||||
timeString = Fore.BLUE + '[' + str(round(translationTime, 1)) + 's]'
|
||||
|
||||
if translatedData[2] == None:
|
||||
if translatedData[2] is None:
|
||||
# Success
|
||||
return filename + ': ' + totalTokenstring + timeString + Fore.GREEN + u' \u2713 ' + Fore.RESET
|
||||
|
||||
|
|
@ -431,7 +429,7 @@ def searchThings(name, pbar):
|
|||
totalTokens = [0, 0]
|
||||
|
||||
# If there isn't any Japanese in the text just skip
|
||||
if IGNORETLTEXT == True:
|
||||
if IGNORETLTEXT is True:
|
||||
if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+', name['name']) and re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+', name['description']):
|
||||
pbar.update(1)
|
||||
return totalTokens
|
||||
|
|
@ -551,7 +549,6 @@ def searchCodes(page, pbar):
|
|||
maxHistory = MAXHISTORY
|
||||
totalTokens = [0, 0]
|
||||
speaker = ''
|
||||
speakerVar = ''
|
||||
nametag = ''
|
||||
match = []
|
||||
syncIndex = 0
|
||||
|
|
@ -576,7 +573,7 @@ def searchCodes(page, pbar):
|
|||
### IF these crash or fail your game will do the same. Use the flags to skip codes.
|
||||
|
||||
## Event Code: 401 Show Text
|
||||
if codeList[i]['code'] == 401 and CODE401 == True or codeList[i]['code'] == 405 and CODE405:
|
||||
if codeList[i]['code'] == 401 and CODE401 is True or codeList[i]['code'] == 405 and CODE405:
|
||||
# Use this to place text later
|
||||
code = codeList[i]['code']
|
||||
j = i
|
||||
|
|
@ -584,13 +581,12 @@ def searchCodes(page, pbar):
|
|||
# Grab String
|
||||
if len(codeList[i]['parameters']) > 0:
|
||||
jaString = codeList[i]['parameters'][0]
|
||||
firstJAString = jaString
|
||||
else:
|
||||
codeList[i]['code'] = -1
|
||||
continue
|
||||
|
||||
# If there isn't any Japanese in the text just skip
|
||||
if IGNORETLTEXT == True:
|
||||
if IGNORETLTEXT is True:
|
||||
if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+', jaString):
|
||||
# Keep textHistory list at length maxHistory
|
||||
textHistory.append('\"' + jaString + '\"')
|
||||
|
|
@ -732,25 +728,23 @@ def searchCodes(page, pbar):
|
|||
finalJAString = finalJAString.replace(matchList[0], '')
|
||||
|
||||
# Remove any textwrap
|
||||
if FIXTEXTWRAP == True:
|
||||
if FIXTEXTWRAP is True:
|
||||
finalJAString = re.sub(r'\n', ' ', finalJAString)
|
||||
finalJAString = finalJAString.replace('<br>', ' ')
|
||||
|
||||
# Remove Extra Stuff
|
||||
finalJAString = finalJAString.replace('ﾞ', '')
|
||||
finalJAString = finalJAString.replace('。', '.')
|
||||
finalJAString = finalJAString.replace('？', '?')
|
||||
finalJAString = finalJAString.replace('！', '!')
|
||||
finalJAString = finalJAString.replace('・', '.')
|
||||
finalJAString = finalJAString.replace('‶', '')
|
||||
finalJAString = finalJAString.replace('”', '')
|
||||
finalJAString = finalJAString.replace('―', '-')
|
||||
finalJAString = finalJAString.replace('ー', '-')
|
||||
finalJAString = finalJAString.replace('…', '...')
|
||||
finalJAString = finalJAString.replace(' ', '')
|
||||
# finalJAString = finalJAString.replace('〇', '*')
|
||||
|
||||
# Remove any RPGMaker Code at start
|
||||
ffMatchList = re.findall(r'[\\]+[fF]+\[.+?\]', finalJAString)
|
||||
ffMatchList = re.findall(r'[\\]+[fFaA]+\[.+?\]', finalJAString)
|
||||
if len(ffMatchList) > 0:
|
||||
finalJAString = finalJAString.replace(ffMatchList[0], '')
|
||||
nametag += ffMatchList[0]
|
||||
|
|
@ -785,29 +779,27 @@ def searchCodes(page, pbar):
|
|||
|
||||
# Sub Vars
|
||||
varResponse = subVars(translatedText)
|
||||
subbedT = varResponse[0]
|
||||
textHistory.append('\"' + varResponse[0] + '\"')
|
||||
elif finalJAString != '':
|
||||
response = translateGPT(speaker + ' | ' + finalJAString, textHistory, True)
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
translatedText = response[0]
|
||||
|
||||
|
||||
# Remove added speaker
|
||||
translatedText = re.sub(r'(^.+?)\s?[|:]\s?', '', translatedText)
|
||||
|
||||
# Sub Vars
|
||||
varResponse = subVars(translatedText)
|
||||
subbedT = varResponse[0]
|
||||
textHistory.append('\"' + speaker + ' | ' + varResponse[0] + '\"')
|
||||
speaker = ''
|
||||
else:
|
||||
translatedText = finalJAString
|
||||
|
||||
# Textwrap
|
||||
if FIXTEXTWRAP == True:
|
||||
if FIXTEXTWRAP is True:
|
||||
translatedText = textwrap.fill(translatedText, width=WIDTH)
|
||||
if BRFLAG == True:
|
||||
if BRFLAG is True:
|
||||
translatedText = translatedText.replace('\n', '<br>')
|
||||
|
||||
# Add Beginning Text
|
||||
|
|
@ -834,14 +826,13 @@ def searchCodes(page, pbar):
|
|||
currentGroup = []
|
||||
|
||||
## Event Code: 122 [Set Variables]
|
||||
if codeList[i]['code'] == 122 and CODE122 == True:
|
||||
if codeList[i]['code'] == 122 and CODE122 is True:
|
||||
# This is going to be the var being set. (IMPORTANT)
|
||||
varNum = codeList[i]['parameters'][0]
|
||||
# if varNum not in [1178]:
|
||||
# continue
|
||||
|
||||
jaString = codeList[i]['parameters'][4]
|
||||
if type(jaString) != str:
|
||||
if isinstance(jaString, str):
|
||||
continue
|
||||
|
||||
# Definitely don't want to mess with files
|
||||
|
|
@ -880,11 +871,11 @@ def searchCodes(page, pbar):
|
|||
# Set Data
|
||||
codeList[i]['parameters'][4] = translatedText
|
||||
|
||||
## Event Code: 357 [Picture Text] [Optional]
|
||||
if codeList[i]['code'] == 357 and CODE357 == True:
|
||||
## Event Code: 357 [Picture Text] [Optional]
|
||||
if codeList[i]['code'] == 357 and CODE357 is True:
|
||||
if 'message' in codeList[i]['parameters'][3]:
|
||||
jaString = codeList[i]['parameters'][3]['message']
|
||||
if type(jaString) != str:
|
||||
if not isinstance(jaString, str):
|
||||
continue
|
||||
|
||||
# Definitely don't want to mess with files
|
||||
|
|
@ -899,8 +890,10 @@ def searchCodes(page, pbar):
|
|||
oldjaString = jaString
|
||||
startString = re.search(r'^[^一-龠ぁ-ゔァ-ヴー【】（）「」a-zA-ZＡ-Ｚ０-９\\]+', jaString)
|
||||
finalJAString = re.sub(r'^[^一-龠ぁ-ゔァ-ヴー【】（）「」a-zA-ZＡ-Ｚ０-９\\]+', '', jaString)
|
||||
if startString is None: startString = ''
|
||||
else: startString = startString.group()
|
||||
if startString is None:
|
||||
startString = ''
|
||||
else:
|
||||
startString = startString.group()
|
||||
|
||||
# Remove any textwrap
|
||||
finalJAString = re.sub(r'\n', ' ', finalJAString)
|
||||
|
|
@ -917,11 +910,11 @@ def searchCodes(page, pbar):
|
|||
# Set Data
|
||||
codeList[i]['parameters'][3]['message'] = startString + translatedText
|
||||
|
||||
## Event Code: 657 [Picture Text] [Optional]
|
||||
if codeList[i]['code'] == 657 and CODE657 == True:
|
||||
## Event Code: 657 [Picture Text] [Optional]
|
||||
if codeList[i]['code'] == 657 and CODE657 is True:
|
||||
if 'text' in codeList[i]['parameters'][0]:
|
||||
jaString = codeList[i]['parameters'][0]
|
||||
if type(jaString) != str:
|
||||
if not isinstance(jaString, str):
|
||||
continue
|
||||
|
||||
# Definitely don't want to mess with files
|
||||
|
|
@ -937,10 +930,14 @@ def searchCodes(page, pbar):
|
|||
jaString = re.sub(r'^[^一-龠ぁ-ゔァ-ヴー\<\>【】\\]+', '', jaString)
|
||||
endString = re.search(r'[^一-龠ぁ-ゔァ-ヴー\<\>【】。！？\\]+$', jaString)
|
||||
jaString = re.sub(r'[^一-龠ぁ-ゔァ-ヴー\<\>【】。！？\\]+$', '', jaString)
|
||||
if startString is None: startString = ''
|
||||
else: startString = startString.group()
|
||||
if endString is None: endString = ''
|
||||
else: endString = endString.group()
|
||||
if startString is None:
|
||||
startString = ''
|
||||
else:
|
||||
startString = startString.group()
|
||||
if endString is None:
|
||||
endString = ''
|
||||
else:
|
||||
endString = endString.group()
|
||||
|
||||
# Remove any textwrap
|
||||
jaString = re.sub(r'\n', ' ', jaString)
|
||||
|
|
@ -966,12 +963,12 @@ def searchCodes(page, pbar):
|
|||
codeList[i]['parameters'][0] = translatedText
|
||||
|
||||
## Event Code: 101 [Name] [Optional]
|
||||
if codeList[i]['code'] == 101 and CODE101 == True:
|
||||
if codeList[i]['code'] == 101 and CODE101 is True:
|
||||
# Grab String
|
||||
jaString = ''
|
||||
if len(codeList[i]['parameters']) > 4:
|
||||
jaString = codeList[i]['parameters'][4]
|
||||
if type(jaString) != str:
|
||||
if not isinstance(jaString, str):
|
||||
continue
|
||||
|
||||
# Force Speaker
|
||||
|
|
@ -1032,7 +1029,7 @@ def searchCodes(page, pbar):
|
|||
NAMESLIST.append(speaker)
|
||||
|
||||
## Event Code: 355 or 655 Scripts [Optional]
|
||||
if (codeList[i]['code'] == 355 or codeList[i]['code'] == 655) and CODE355655 == True:
|
||||
if (codeList[i]['code'] == 355 or codeList[i]['code'] == 655) and CODE355655 is True:
|
||||
jaString = codeList[i]['parameters'][0]
|
||||
|
||||
# If there isn't any Japanese in the text just skip
|
||||
|
|
@ -1072,7 +1069,7 @@ def searchCodes(page, pbar):
|
|||
codeList[i]['parameters'][0] = translatedText
|
||||
|
||||
## Event Code: 408 (Script)
|
||||
if (codeList[i]['code'] == 408) and CODE408 == True:
|
||||
if (codeList[i]['code'] == 408) and CODE408 is True:
|
||||
jaString = codeList[i]['parameters'][0]
|
||||
|
||||
# If there isn't any Japanese in the text just skip
|
||||
|
|
@ -1112,7 +1109,7 @@ def searchCodes(page, pbar):
|
|||
codeList[i]['parameters'][0] = translatedText
|
||||
|
||||
## Event Code: 108 (Script)
|
||||
if (codeList[i]['code'] == 108) and CODE108 == True:
|
||||
if (codeList[i]['code'] == 108) and CODE108 is True:
|
||||
jaString = codeList[i]['parameters'][0]
|
||||
|
||||
# If there isn't any Japanese in the text just skip
|
||||
|
|
@ -1145,7 +1142,7 @@ def searchCodes(page, pbar):
|
|||
codeList[i]['parameters'][0] = translatedText
|
||||
|
||||
## Event Code: 356
|
||||
if codeList[i]['code'] == 356 and CODE356 == True:
|
||||
if codeList[i]['code'] == 356 and CODE356 is True:
|
||||
jaString = codeList[i]['parameters'][0]
|
||||
oldjaString = jaString
|
||||
|
||||
|
|
@ -1433,7 +1430,7 @@ def searchCodes(page, pbar):
|
|||
continue
|
||||
|
||||
### Event Code: 102 Show Choice
|
||||
if codeList[i]['code'] == 102 and CODE102 == True:
|
||||
if codeList[i]['code'] == 102 and CODE102 is True:
|
||||
for choice in range(len(codeList[i]['parameters'][0])):
|
||||
jaString = codeList[i]['parameters'][0][choice]
|
||||
jaString = jaString.replace(' 。', '.')
|
||||
|
|
@ -1466,12 +1463,12 @@ def searchCodes(page, pbar):
|
|||
codeList[i]['parameters'][0][choice] = startString + translatedText + endString
|
||||
|
||||
### Event Code: 111 Script
|
||||
if codeList[i]['code'] == 111 and CODE111 == True:
|
||||
if codeList[i]['code'] == 111 and CODE111 is True:
|
||||
for j in range(len(codeList[i]['parameters'])):
|
||||
jaString = codeList[i]['parameters'][j]
|
||||
|
||||
# Check if String
|
||||
if type(jaString) != str:
|
||||
if not isinstance(jaString, str):
|
||||
continue
|
||||
|
||||
# Only TL the Game Variable
|
||||
|
|
@ -1503,9 +1500,9 @@ def searchCodes(page, pbar):
|
|||
codeList[i]['parameters'][j] = translatedText
|
||||
|
||||
### Event Code: 320 Set Variable
|
||||
if codeList[i]['code'] == 320 and CODE320 == True:
|
||||
if codeList[i]['code'] == 320 and CODE320 is True:
|
||||
jaString = codeList[i]['parameters'][1]
|
||||
if type(jaString) != str:
|
||||
if not isinstance(jaString, str):
|
||||
continue
|
||||
|
||||
# Definitely don't want to mess with files
|
||||
|
|
@ -1544,7 +1541,7 @@ def searchCodes(page, pbar):
|
|||
# raise Exception(str(e) + '|Line:' + tracebackLineNo)
|
||||
except Exception as e:
|
||||
traceback.print_exc()
|
||||
raise Exception(str(e) + 'Failed to translate: ' + oldjaString)
|
||||
raise Exception(str(e) + 'Failed to translate: ' + oldjaString) from None
|
||||
|
||||
# Append leftover groups in 401
|
||||
if len(currentGroup) > 0:
|
||||
|
|
@ -1576,7 +1573,6 @@ def searchCodes(page, pbar):
|
|||
return totalTokens
|
||||
|
||||
def searchSS(state, pbar):
|
||||
'''Searches skills and states json files'''
|
||||
totalTokens = [0, 0]
|
||||
|
||||
# Name
|
||||
|
|
@ -1791,7 +1787,7 @@ def subVars(jaString):
|
|||
count = 0
|
||||
if '笑えるよね.' in jaString:
|
||||
print('t')
|
||||
formatList = re.findall(r'[\\]+CL', jaString)
|
||||
formatList = re.findall(r'[\\]+[\w]+\[.+?\]', jaString)
|
||||
formatList = set(formatList)
|
||||
if len(formatList) != 0:
|
||||
for var in formatList:
|
||||
|
|
@ -1860,6 +1856,14 @@ def resubVars(translatedText, allList):
|
|||
|
||||
@retry(exceptions=Exception, tries=5, delay=5)
|
||||
def translateGPT(t, history, fullPromptFlag):
|
||||
# Sub Vars
|
||||
varResponse = subVars(t)
|
||||
subbedT = varResponse[0]
|
||||
|
||||
# If there isn't any Japanese in the text just skip
|
||||
if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴ]+|[\uFF00-\uFFEF]', subbedT):
|
||||
return(t, [0,0])
|
||||
|
||||
# If ESTIMATE is True just count this as an execution and return.
|
||||
if ESTIMATE:
|
||||
enc = tiktoken.encoding_for_model(MODEL)
|
||||
|
|
@ -1874,23 +1878,13 @@ def translateGPT(t, history, fullPromptFlag):
|
|||
outputTotalTokens = len(enc.encode(t)) * 2 # Estimating 2x the size of the original text
|
||||
totalTokens = [inputTotalTokens, outputTotalTokens]
|
||||
return (t, totalTokens)
|
||||
|
||||
# Sub Vars
|
||||
varResponse = subVars(t)
|
||||
subbedT = varResponse[0]
|
||||
|
||||
# If there isn't any Japanese in the text just skip
|
||||
if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴ]+|[\uFF00-\uFFEF]', subbedT):
|
||||
return(t, [0,0])
|
||||
|
||||
# Characters
|
||||
context = 'Game Characters:\
|
||||
Character: アーテル == Artel - Gender: Female\
|
||||
Character: コリウス == Coleus - Gender: Male\
|
||||
Character: ギア == Gear - Gender: Male\
|
||||
Character: ロック == Rock - Gender: Male\
|
||||
Character: テア == Thea - Gender: Male\
|
||||
Character: ヘンジ == Henge - Gender: Male'
|
||||
Character: リッカ == Ricca - Gender: Female\
|
||||
Character: シーナ == Sina - Gender: Female\
|
||||
Character: ヘレナ == Helena - Gender: Female\
|
||||
Character: Miko == Miko - Gender: Female'
|
||||
|
||||
# Prompt
|
||||
if fullPromptFlag:
|
||||
|
|
@ -1939,6 +1933,12 @@ def translateGPT(t, history, fullPromptFlag):
|
|||
translatedText = translatedText.replace('Translation =', '')
|
||||
translatedText = translatedText.replace('Translate =', '')
|
||||
translatedText = translatedText.replace('っ', '')
|
||||
translatedText = translatedText.replace('ッ', '')
|
||||
translatedText = translatedText.replace('ぁ', '')
|
||||
translatedText = translatedText.replace('。', '.')
|
||||
translatedText = translatedText.replace('、', ',')
|
||||
translatedText = translatedText.replace('？', '?')
|
||||
translatedText = translatedText.replace('！', '!')
|
||||
|
||||
# Return Translation
|
||||
if len(translatedText) > 15 * len(t) or "I'm sorry, but I'm unable to assist with that translation" in translatedText:
|
||||
|
|
|
|||
|
|
@ -1,55 +1,59 @@
|
|||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import textwrap
|
||||
import threading
|
||||
import time
|
||||
import traceback
|
||||
import tiktoken
|
||||
from pathlib import Path
|
||||
|
||||
import openai
|
||||
import tiktoken
|
||||
from colorama import Fore
|
||||
from dotenv import load_dotenv
|
||||
import openai
|
||||
from retry import retry
|
||||
from tqdm import tqdm
|
||||
|
||||
#Globals
|
||||
# Open AI
|
||||
load_dotenv()
|
||||
if os.getenv('api').replace(' ', '') != '':
|
||||
openai.api_base = os.getenv('api')
|
||||
if os.getenv("api").replace(" ", "") != "":
|
||||
openai.api_base = os.getenv("api")
|
||||
openai.organization = os.getenv("org")
|
||||
openai.api_key = os.getenv("key")
|
||||
|
||||
openai.organization = os.getenv('org')
|
||||
openai.api_key = os.getenv('key')
|
||||
MODEL = os.getenv('model')
|
||||
TIMEOUT = int(os.getenv('timeout'))
|
||||
LANGUAGE=os.getenv('language').capitalize()
|
||||
|
||||
APICOST = .002 # Depends on the model https://openai.com/pricing
|
||||
PROMPT = Path('prompt.txt').read_text(encoding='utf-8')
|
||||
THREADS = int(os.getenv('threads')) # For GPT4 rate limit will be hit if you have more than 1 thread.
|
||||
# Globals
|
||||
MODEL = os.getenv("model")
|
||||
TIMEOUT = int(os.getenv("timeout"))
|
||||
LANGUAGE = os.getenv("language").capitalize()
|
||||
INPUTAPICOST = 0.002 # Depends on the model https://openai.com/pricing
|
||||
OUTPUTAPICOST = 0.002
|
||||
PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
|
||||
THREADS = int(
|
||||
os.getenv("threads")
|
||||
) # Controls how many threads are working on a single file (May have to drop this)
|
||||
LOCK = threading.Lock()
|
||||
WIDTH = int(os.getenv('width'))
|
||||
LISTWIDTH = int(os.getenv('listWidth'))
|
||||
WIDTH = int(os.getenv("width"))
|
||||
LISTWIDTH = int(os.getenv("listWidth"))
|
||||
NOTEWIDTH = 40
|
||||
MAXHISTORY = 10
|
||||
ESTIMATE = ''
|
||||
TOTALCOST = 0
|
||||
TOKENS = 0
|
||||
TOTALTOKENS = 0
|
||||
ESTIMATE = ""
|
||||
totalTokens = [0, 0]
|
||||
NAMESLIST = []
|
||||
|
||||
#tqdm Globals
|
||||
BAR_FORMAT='{l_bar}{bar:10}{r_bar}{bar:-10b}'
|
||||
POSITION=0
|
||||
LEAVE=False
|
||||
# tqdm Globals
|
||||
BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
|
||||
POSITION = 0
|
||||
LEAVE = False
|
||||
|
||||
# Flags
|
||||
NAMES = False # Output a list of all the character names found
|
||||
NAMES = False # Output a list of all the character names found
|
||||
BRFLAG = False # If the game uses <br> instead
|
||||
FIXTEXTWRAP = True
|
||||
IGNORETLTEXT = True
|
||||
IGNORETLTEXT = False
|
||||
|
||||
|
||||
def handleTyrano(filename, estimate):
|
||||
global ESTIMATE, TOKENS, TOTALTOKENS, TOTALCOST
|
||||
global ESTIMATE
|
||||
totalTokens = [0, 0]
|
||||
ESTIMATE = estimate
|
||||
|
||||
if estimate:
|
||||
|
|
@ -59,416 +63,390 @@ def handleTyrano(filename, estimate):
|
|||
# Print Result
|
||||
end = time.time()
|
||||
tqdm.write(getResultString(translatedData, end - start, filename))
|
||||
if NAMES is True:
|
||||
tqdm.write(str(NAMESLIST))
|
||||
with LOCK:
|
||||
TOTALCOST += translatedData[1] * .001 * APICOST
|
||||
TOTALTOKENS += translatedData[1]
|
||||
totalTokens[0] += translatedData[1][0]
|
||||
totalTokens[1] += translatedData[1][1]
|
||||
|
||||
return getResultString(["", totalTokens, None], end - start, "TOTAL")
|
||||
|
||||
return getResultString(['', TOTALTOKENS, None], end - start, 'TOTAL')
|
||||
|
||||
else:
|
||||
try:
|
||||
with open('translated/' + filename, 'w', encoding='utf-8', errors='ignore') as outFile:
|
||||
with open("translated/" + filename, "w", encoding="utf-8") as outFile:
|
||||
start = time.time()
|
||||
translatedData = openFiles(filename)
|
||||
outFile.writelines(translatedData[0])
|
||||
|
||||
# Print Result
|
||||
outFile.writelines(translatedData[0])
|
||||
end = time.time()
|
||||
tqdm.write(getResultString(translatedData, end - start, filename))
|
||||
with LOCK:
|
||||
TOTALCOST += translatedData[1] * .001 * APICOST
|
||||
TOTALTOKENS += translatedData[1]
|
||||
except Exception as e:
|
||||
totalTokens[0] += translatedData[1][0]
|
||||
totalTokens[1] += translatedData[1][1]
|
||||
except Exception:
|
||||
traceback.print_exc()
|
||||
return 'Fail'
|
||||
return "Fail"
|
||||
|
||||
return getResultString(['', TOTALTOKENS, None], end - start, 'TOTAL')
|
||||
return getResultString(["", totalTokens, None], end - start, "TOTAL")
|
||||
|
||||
def openFiles(filename):
|
||||
with open('files/' + filename, 'r', encoding='utf-8') as readFile:
|
||||
translatedData = parseTyrano(readFile, filename)
|
||||
|
||||
# Delete lines marked for deletion
|
||||
finalData = []
|
||||
for line in translatedData[0]:
|
||||
if line != '\\d\n':
|
||||
finalData.append(line)
|
||||
translatedData[0] = finalData
|
||||
|
||||
return translatedData
|
||||
|
||||
def parseTyrano(readFile, filename):
|
||||
totalTokens = 0
|
||||
totalLines = 0
|
||||
|
||||
# Get total for progress bar
|
||||
data = readFile.readlines()
|
||||
totalLines = len(data)
|
||||
|
||||
with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar:
|
||||
pbar.desc=filename
|
||||
pbar.total=totalLines
|
||||
|
||||
try:
|
||||
totalTokens += translateTyrano(data, pbar)
|
||||
except Exception as e:
|
||||
traceback.print_exc()
|
||||
return [data, totalTokens, e]
|
||||
return [data, totalTokens, None]
|
||||
|
||||
def translateTyrano(data, pbar):
|
||||
textHistory = []
|
||||
maxHistory = MAXHISTORY
|
||||
tokens = 0
|
||||
currentGroup = []
|
||||
syncIndex = 0
|
||||
speaker = ''
|
||||
global LOCK, ESTIMATE
|
||||
|
||||
for i in range(len(data)):
|
||||
if syncIndex > i:
|
||||
i = syncIndex
|
||||
|
||||
# Speaker
|
||||
if '#' in data[i]:
|
||||
matchList = re.findall(r'#(.+)', data[i])
|
||||
if len(matchList) != 0:
|
||||
response = translateGPT(matchList[0], 'Reply with only the '+ LANGUAGE +' translation of the NPC name', True)
|
||||
speaker = response[0]
|
||||
tokens += response[1]
|
||||
data[i] = '#' + speaker + '\n'
|
||||
else:
|
||||
speaker = ''
|
||||
|
||||
# Choices
|
||||
elif 'glink' in data[i]:
|
||||
matchList = re.findall(r'\[glink.+text=\"(.+?)\".+', data[i])
|
||||
if len(matchList) != 0:
|
||||
if len(textHistory) > 0:
|
||||
originalText = matchList[0]
|
||||
response = translateGPT(matchList[0], 'Past Translated Text: ' + textHistory[len(textHistory)-1] + '\n\nReply in the style of a dialogue option.', True)
|
||||
else:
|
||||
response = translateGPT(matchList[0], '', False)
|
||||
translatedText = response[0]
|
||||
tokens += response[1]
|
||||
|
||||
# Remove characters that may break scripts
|
||||
charList = ['.', '\"', '\\n']
|
||||
for char in charList:
|
||||
translatedText = translatedText.replace(char, '')
|
||||
|
||||
# Escape all '
|
||||
translatedText = translatedText.replace('\\', '')
|
||||
translatedText = translatedText.replace("'", "\\\'")
|
||||
|
||||
# Set Data
|
||||
translatedText = data[i].replace(matchList[0], translatedText.replace(' ', '\u00A0'))
|
||||
data[i] = translatedText
|
||||
|
||||
# Grab Lines
|
||||
matchList = re.findall(r'(.+?)\[p\]', data[i])
|
||||
if len(matchList) > 0:
|
||||
matchList[0] = matchList[0].replace('「', '')
|
||||
matchList[0] = matchList[0].replace('」', '')
|
||||
currentGroup.append(matchList[0])
|
||||
if len(data) > i+1:
|
||||
while '[r]' in data[i+1]:
|
||||
data[i] = '\d\n' # \d Marks line for deletion
|
||||
i += 1
|
||||
matchList = re.findall(r'(.+?)\[p\]', data[i])
|
||||
if len(matchList) > 0:
|
||||
matchList[0] = matchList[0].replace('「', '')
|
||||
matchList[0] = matchList[0].replace('」', '')
|
||||
currentGroup.append(matchList[0])
|
||||
while '[p][cm]' in data[i+1]:
|
||||
data[i] = '\d\n'
|
||||
i += 1
|
||||
matchList = re.findall(r'(.+?)\[p\]', data[i])
|
||||
if len(matchList) > 0:
|
||||
matchList[0] = matchList[0].replace('「', '')
|
||||
matchList[0] = matchList[0].replace('」', '')
|
||||
currentGroup.append(matchList[0])
|
||||
# Join up 401 groups for better translation.
|
||||
if len(currentGroup) > 0:
|
||||
finalJAString = ' '.join(currentGroup)
|
||||
oldjaString = finalJAString
|
||||
|
||||
# Remove any textwrap
|
||||
if FIXTEXTWRAP == True:
|
||||
finalJAString = finalJAString.replace('[r]', ' ')
|
||||
|
||||
#Check Speaker
|
||||
if speaker == '':
|
||||
response = translateGPT(finalJAString, textHistory, True)
|
||||
tokens += response[1]
|
||||
translatedText = response[0]
|
||||
textHistory.append('\"' + translatedText + '\"')
|
||||
else:
|
||||
response = translateGPT(speaker + ': ' + finalJAString, textHistory, True)
|
||||
tokens += response[1]
|
||||
translatedText = response[0]
|
||||
textHistory.append('\"' + translatedText + '\"')
|
||||
|
||||
# Remove added speaker
|
||||
translatedText = re.sub(r'^.+:\s?', '', translatedText)
|
||||
|
||||
# Set Data
|
||||
translatedText = translatedText.replace('ッ', '')
|
||||
translatedText = translatedText.replace('っ', '')
|
||||
translatedText = translatedText.replace('ー', '')
|
||||
translatedText = translatedText.replace('\"', '')
|
||||
translatedText = translatedText.replace('[', '')
|
||||
translatedText = translatedText.replace(']', '')
|
||||
|
||||
# Split final string into full sentences.
|
||||
matchList = re.findall(r'(.+?[)\.\?\!）。・]+)', translatedText)
|
||||
translatedText = re.sub(r'(.+?[)\.\?\!）。・]+)', '', translatedText)
|
||||
for l in range(len(matchList)):
|
||||
if any(t in matchList[l] for t in ['Mr.', 'Ms.', 'Mrs.']):
|
||||
if len(matchList) > l+1:
|
||||
matchList[l] = matchList[l] + matchList[l+1]
|
||||
matchList[l+1] = ''
|
||||
|
||||
# Combine Lists
|
||||
for k in range(len(matchList)):
|
||||
matchList[k] = matchList[k].strip()
|
||||
j=0
|
||||
while(len(matchList) > j+1):
|
||||
while len(matchList[j]) < 30 and len(matchList) > j:
|
||||
matchList[j:j+2] = [' '.join(matchList[j:j+2])]
|
||||
if len(matchList) == j+1:
|
||||
matchList[j] = matchList[j] + ' ' + translatedText
|
||||
translatedText = ''
|
||||
break
|
||||
j+=1
|
||||
|
||||
# Normal Lines
|
||||
if len(matchList) > 0:
|
||||
data[i] = '\d\n'
|
||||
for line in matchList:
|
||||
# Wordwrap Text
|
||||
if '[r]' not in line:
|
||||
line = textwrap.fill(line, width=WIDTH)
|
||||
line = line.replace('\n', '[r]')
|
||||
|
||||
# Set
|
||||
data.insert(i, line.strip() + '[p][cm]\n')
|
||||
i+=1
|
||||
data[i-1] = data[i-1].replace('[l][er]', '[p][cm]')
|
||||
# else:
|
||||
# print ('No Matches')
|
||||
|
||||
# Backup TL
|
||||
if translatedText != '':
|
||||
# Wordwrap Text
|
||||
if '[r]' not in translatedText:
|
||||
translatedText = textwrap.fill(translatedText, width=WIDTH)
|
||||
translatedText = translatedText.replace('\n', '[r]')
|
||||
|
||||
# Set Last Line
|
||||
data[i] = translatedText.strip() + '[p][cm]\n'
|
||||
|
||||
# Keep textHistory list at length maxHistory
|
||||
if len(textHistory) > maxHistory:
|
||||
textHistory.pop(0)
|
||||
currentGroup = []
|
||||
speaker = ''
|
||||
|
||||
# Grab Lines [p][cm]
|
||||
matchList = re.findall(r'(.+?)\[p\]\[cm\]', data[i])
|
||||
if len(matchList) > 0:
|
||||
matchList[0] = matchList[0].replace('「', '')
|
||||
matchList[0] = matchList[0].replace('」', '')
|
||||
finalJAString = matchList[0]
|
||||
|
||||
# Remove any textwrap
|
||||
if FIXTEXTWRAP == True:
|
||||
finalJAString = finalJAString.replace('[r]', ' ')
|
||||
|
||||
#Check Speaker
|
||||
if speaker == '':
|
||||
response = translateGPT(finalJAString, 'Previous Dialogue: ' + '\n\n'.join(textHistory), True)
|
||||
tokens += response[1]
|
||||
translatedText = response[0]
|
||||
textHistory.append('\"' + translatedText + '\"')
|
||||
else:
|
||||
response = translateGPT(speaker + ': ' + finalJAString, 'Previous Dialogue: ' + '\n\n'.join(textHistory), True)
|
||||
tokens += response[1]
|
||||
translatedText = response[0]
|
||||
textHistory.append('\"' + translatedText + '\"')
|
||||
|
||||
# Remove added speaker
|
||||
translatedText = re.sub(r'^.+:\s?', '', translatedText)
|
||||
|
||||
# Set Data
|
||||
translatedText = translatedText.replace('ッ', '')
|
||||
translatedText = translatedText.replace('っ', '')
|
||||
translatedText = translatedText.replace('ー', '')
|
||||
translatedText = translatedText.replace('\"', '')
|
||||
translatedText = translatedText.replace('[', '')
|
||||
translatedText = translatedText.replace(']', '')
|
||||
|
||||
# Split final string into full sentences.
|
||||
matchList = re.findall(r'(.+?[)\.\?\!）。・]+)', translatedText)
|
||||
translatedText = re.sub(r'(.+?[)\.\?\!）。・]+)', '', translatedText)
|
||||
for l in range(len(matchList)):
|
||||
if any(t in matchList[l] for t in ['Mr.', 'Ms.', 'Mrs.']):
|
||||
if len(matchList) > l+1:
|
||||
matchList[l] = matchList[l] + matchList[l+1]
|
||||
matchList[l+1] = ''
|
||||
|
||||
# Get rid of whitespace for each item and add wordwrap
|
||||
for k in range(len(matchList)):
|
||||
matchList[k] = matchList[k].strip()
|
||||
|
||||
# Combine Sentences with a max limit (Wordwrap basically)
|
||||
j=0
|
||||
while(len(matchList) > j+1):
|
||||
while len(matchList[j]) < 30 and len(matchList) > j:
|
||||
matchList[j:j+2] = [' '.join(matchList[j:j+2])]
|
||||
if len(matchList) == j+1:
|
||||
matchList[j] = matchList[j] + ' ' + translatedText
|
||||
translatedText = ''
|
||||
break
|
||||
j+=1
|
||||
|
||||
# Set Data
|
||||
if len(matchList) > 0:
|
||||
data[i] = '\d\n'
|
||||
for line in matchList:
|
||||
# Wordwrap Text
|
||||
if '[r]' not in line:
|
||||
line = textwrap.fill(line, width=WIDTH)
|
||||
line = line.replace('\n', '[r]')
|
||||
|
||||
# Set
|
||||
data.insert(i, line.strip() + '[p][cm]\n')
|
||||
i+=1
|
||||
# Set last line as [pcm] instead of [r]
|
||||
data[i-1] = data[i-1].replace('[l][er]', '[p][cm]')
|
||||
# else:
|
||||
# print ('No Matches')
|
||||
if translatedText != '':
|
||||
# Wordwrap Text
|
||||
if '[r]' not in translatedText:
|
||||
translatedText = textwrap.fill(translatedText, width=WIDTH)
|
||||
translatedText = translatedText.replace('\n', '[r]')
|
||||
|
||||
# Set Backup
|
||||
data[i] = translatedText.strip() + '[p][cm]\n'
|
||||
|
||||
# Keep textHistory list at length maxHistory
|
||||
if len(textHistory) > maxHistory:
|
||||
textHistory.pop(0)
|
||||
currentGroup = []
|
||||
speaker = ''
|
||||
|
||||
currentGroup = []
|
||||
pbar.update(1)
|
||||
if len(data) > i+1:
|
||||
syncIndex = i+1
|
||||
else:
|
||||
break
|
||||
|
||||
return tokens
|
||||
|
||||
def getResultString(translatedData, translationTime, filename):
|
||||
# File Print String
|
||||
tokenString = Fore.YELLOW + '[' + str(translatedData[1]) + \
|
||||
' Tokens/${:,.4f}'.format(translatedData[1] * .001 * APICOST) + ']'
|
||||
timeString = Fore.BLUE + '[' + str(round(translationTime, 1)) + 's]'
|
||||
totalTokenstring = (
|
||||
Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
|
||||
"[Output: " + str(translatedData[1][1]) + "]"
|
||||
"[Cost: ${:,.4f}".format(
|
||||
(translatedData[1][0] * 0.001 * INPUTAPICOST)
|
||||
+ (translatedData[1][1] * 0.001 * OUTPUTAPICOST)
|
||||
)
|
||||
+ "]"
|
||||
)
|
||||
timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
|
||||
|
||||
if translatedData[2] == None:
|
||||
if translatedData[2] is None:
|
||||
# Success
|
||||
return filename + ': ' + tokenString + timeString + Fore.GREEN + u' \u2713 ' + Fore.RESET
|
||||
return (
|
||||
filename
|
||||
+ ": "
|
||||
+ totalTokenstring
|
||||
+ timeString
|
||||
+ Fore.GREEN
|
||||
+ " \u2713 "
|
||||
+ Fore.RESET
|
||||
)
|
||||
|
||||
else:
|
||||
# Fail
|
||||
try:
|
||||
raise translatedData[2]
|
||||
except Exception as e:
|
||||
traceback.print_exc()
|
||||
errorString = str(e) + Fore.RED
|
||||
return filename + ': ' + tokenString + timeString + Fore.RED + u' \u2717 ' +\
|
||||
errorString + Fore.RESET
|
||||
|
||||
return (
|
||||
filename
|
||||
+ ": "
|
||||
+ totalTokenstring
|
||||
+ timeString
|
||||
+ Fore.RED
|
||||
+ " \u2717 "
|
||||
+ errorString
|
||||
+ Fore.RESET
|
||||
)
|
||||
|
||||
|
||||
def openFiles(filename):
|
||||
with open("files/" + filename, "r", encoding="utf-8") as readFile:
|
||||
translatedData = parseTyrano(readFile, filename)
|
||||
|
||||
# Delete lines marked for deletion
|
||||
finalData = []
|
||||
for line in translatedData[0]:
|
||||
if line != "\\d\n":
|
||||
finalData.append(line)
|
||||
translatedData[0] = finalData
|
||||
|
||||
return translatedData
|
||||
|
||||
|
||||
def parseTyrano(readFile, filename):
|
||||
totalTokens = [0, 0]
|
||||
totalLines = 0
|
||||
|
||||
# Get total for progress bar
|
||||
data = readFile.readlines()
|
||||
totalLines = len(data)
|
||||
|
||||
with tqdm(
|
||||
bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE
|
||||
) as pbar:
|
||||
pbar.desc = filename
|
||||
pbar.total = totalLines
|
||||
|
||||
try:
|
||||
response = translateTyrano(data, pbar)
|
||||
totalTokens[0] = response[0]
|
||||
totalTokens[1] = response[1]
|
||||
except Exception as e:
|
||||
traceback.print_exc()
|
||||
return [data, totalTokens, e]
|
||||
return [data, totalTokens, None]
|
||||
|
||||
|
||||
def translateTyrano(data, pbar):
|
||||
textHistory = []
|
||||
maxHistory = MAXHISTORY
|
||||
tokens = [0, 0]
|
||||
currentGroup = []
|
||||
syncIndex = 0
|
||||
speaker = ""
|
||||
delFlag = False
|
||||
global LOCK, ESTIMATE
|
||||
|
||||
for i in range(len(data)):
|
||||
if syncIndex > i:
|
||||
i = syncIndex
|
||||
|
||||
if '[▼]' in data[i]:
|
||||
data[i] = data[i].replace('[▼]'.strip(), '[page]\n')
|
||||
|
||||
# If there isn't any Japanese in the text just skip
|
||||
if IGNORETLTEXT is True:
|
||||
if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+', data[i]):
|
||||
# Keep textHistory list at length maxHistory
|
||||
textHistory.append('\"' + data[i] + '\"')
|
||||
if len(textHistory) > maxHistory:
|
||||
textHistory.pop(0)
|
||||
currentGroup = []
|
||||
continue
|
||||
|
||||
# Speaker
|
||||
matchList = re.findall(r"^\[(.+)\sstorage=.+\]", data[i])
|
||||
if len(matchList) == 0:
|
||||
matchList = re.findall(r"^\[(.+)\]$", data[i])
|
||||
if len(matchList) != 0:
|
||||
if "主人公" in matchList[0]:
|
||||
speaker = "Protagonist"
|
||||
elif "思考" in matchList[0]:
|
||||
speaker = "Protagonist Inner Thoughts"
|
||||
elif "地の文" in matchList[0]:
|
||||
speaker = "Narrator"
|
||||
elif "マコ" in matchList[0]:
|
||||
speaker = "Mako"
|
||||
else:
|
||||
response = translateGPT(
|
||||
matchList[0],
|
||||
"Reply with only the "
|
||||
+ LANGUAGE
|
||||
+ " translation of the NPC name",
|
||||
True,
|
||||
)
|
||||
speaker = response[0]
|
||||
tokens[0] += response[1][0]
|
||||
tokens[1] += response[1][1]
|
||||
# data[i] = '#' + speaker + '\n'
|
||||
|
||||
# Choices
|
||||
elif "glink" in data[i]:
|
||||
matchList = re.findall(r"\[glink.+text=\"(.+?)\".+", data[i])
|
||||
if len(matchList) != 0:
|
||||
if len(textHistory) > 0:
|
||||
response = translateGPT(
|
||||
matchList[0],
|
||||
"Past Translated Text: "
|
||||
+ textHistory[len(textHistory) - 1]
|
||||
+ "\n\nReply in the style of a dialogue option.",
|
||||
True,
|
||||
)
|
||||
else:
|
||||
response = translateGPT(matchList[0], "", False)
|
||||
translatedText = response[0]
|
||||
tokens[0] += response[1][0]
|
||||
tokens[1] += response[1][1]
|
||||
|
||||
# Remove characters that may break scripts
|
||||
charList = [".", '"', "\\n"]
|
||||
for char in charList:
|
||||
translatedText = translatedText.replace(char, "")
|
||||
|
||||
# Escape all '
|
||||
translatedText = translatedText.replace("\\", "")
|
||||
translatedText = translatedText.replace("'", "\\'")
|
||||
|
||||
# Set Data
|
||||
translatedText = data[i].replace(
|
||||
matchList[0], translatedText.replace(" ", "\u00A0")
|
||||
)
|
||||
data[i] = translatedText
|
||||
|
||||
# Grab Lines
|
||||
matchList = re.findall(r"^([^\n;@*\[].+)", data[i])
|
||||
if len(matchList) > 0 and ('storage' in data[i-1] or re.search(r'^\[(.+)\]$', data[i-1])):
|
||||
currentGroup.append(matchList[0])
|
||||
if len(data) > i + 1:
|
||||
matchList = re.findall(r"^([^\n;@*\[].+)", data[i + 1])
|
||||
while len(matchList) > 0:
|
||||
delFlag = True
|
||||
data[i] = "\d\n" # \d Marks line for deletion
|
||||
i += 1
|
||||
matchList = re.findall(r"^([^\n;@*\[].+)", data[i])
|
||||
if len(matchList) > 0:
|
||||
currentGroup.append(matchList[0])
|
||||
|
||||
# Join up 401 groups for better translation.
|
||||
if len(currentGroup) > 0:
|
||||
finalJAString = " ".join(currentGroup)
|
||||
|
||||
# Remove any textwrap
|
||||
if FIXTEXTWRAP is True:
|
||||
finalJAString = finalJAString.replace("_", " ")
|
||||
|
||||
# Check Speaker
|
||||
if speaker == "":
|
||||
response = translateGPT(finalJAString, textHistory, True)
|
||||
tokens[0] += response[1][0]
|
||||
tokens[1] += response[1][1]
|
||||
translatedText = response[0]
|
||||
textHistory.append('"' + translatedText + '"')
|
||||
else:
|
||||
response = translateGPT(
|
||||
speaker + ": " + finalJAString, textHistory, True
|
||||
)
|
||||
tokens[0] += response[1][0]
|
||||
tokens[1] += response[1][1]
|
||||
translatedText = response[0]
|
||||
textHistory.append('"' + translatedText + '"')
|
||||
|
||||
# Remove added speaker
|
||||
translatedText = re.sub(r"^.+:\s?", "", translatedText)
|
||||
|
||||
# Set Data
|
||||
translatedText = translatedText.replace("ッ", "")
|
||||
translatedText = translatedText.replace("っ", "")
|
||||
translatedText = translatedText.replace("ー", "")
|
||||
translatedText = translatedText.replace('"', "")
|
||||
translatedText = translatedText.replace("[", "")
|
||||
translatedText = translatedText.replace("]", "")
|
||||
|
||||
# Wordwrap Text
|
||||
if "_" not in translatedText:
|
||||
translatedText = textwrap.fill(translatedText, width=WIDTH)
|
||||
translatedText = translatedText.replace("\n", "_")
|
||||
|
||||
# Set
|
||||
if delFlag is True:
|
||||
data.insert(i, translatedText.strip() + '\n')
|
||||
delFlag = False
|
||||
else:
|
||||
data[i] = translatedText.strip() + '\n'
|
||||
|
||||
# Keep textHistory list at length maxHistory
|
||||
if len(textHistory) > maxHistory:
|
||||
textHistory.pop(0)
|
||||
currentGroup = []
|
||||
speaker = ""
|
||||
|
||||
currentGroup = []
|
||||
pbar.update(1)
|
||||
if len(data) > i + 1:
|
||||
syncIndex = i + 1
|
||||
else:
|
||||
break
|
||||
|
||||
return tokens
|
||||
|
||||
|
||||
def subVars(jaString):
|
||||
jaString = jaString.replace('\u3000', ' ')
|
||||
jaString = jaString.replace("\u3000", " ")
|
||||
|
||||
# Nested
|
||||
count = 0
|
||||
nestedList = re.findall(r"[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]", jaString)
|
||||
nestedList = set(nestedList)
|
||||
if len(nestedList) != 0:
|
||||
for icon in nestedList:
|
||||
jaString = jaString.replace(icon, "{Nested_" + str(count) + "}")
|
||||
count += 1
|
||||
|
||||
# Icons
|
||||
count = 0
|
||||
iconList = re.findall(r'[\\]+[iIkKwW]+\[[0-9]+\]', jaString)
|
||||
iconList = re.findall(r"[\\]+[iIkKwWaA]+\[[0-9]+\]", jaString)
|
||||
iconList = set(iconList)
|
||||
if len(iconList) != 0:
|
||||
for icon in iconList:
|
||||
jaString = jaString.replace(icon, '[Icon' + str(count) + ']')
|
||||
jaString = jaString.replace(icon, "{Ascii_" + str(count) + "}")
|
||||
count += 1
|
||||
|
||||
# Colors
|
||||
count = 0
|
||||
colorList = re.findall(r'[\\]+[cC]\[[0-9]+\]', jaString)
|
||||
colorList = re.findall(r"[\\]+[cC]\[[0-9]+\]", jaString)
|
||||
colorList = set(colorList)
|
||||
if len(colorList) != 0:
|
||||
for color in colorList:
|
||||
jaString = jaString.replace(color, '[Color' + str(count) + ']')
|
||||
jaString = jaString.replace(color, "{Color_" + str(count) + "}")
|
||||
count += 1
|
||||
|
||||
# Names
|
||||
count = 0
|
||||
nameList = re.findall(r'[\\]+[nN]\[[0-9]+\]', jaString)
|
||||
nameList = re.findall(r"[\\]+[nN]\[.+?\]+", jaString)
|
||||
nameList = set(nameList)
|
||||
if len(nameList) != 0:
|
||||
for name in nameList:
|
||||
jaString = jaString.replace(name, '[Name' + str(count) + ']')
|
||||
jaString = jaString.replace(name, "{N_" + str(count) + "}")
|
||||
count += 1
|
||||
|
||||
# Variables
|
||||
count = 0
|
||||
varList = re.findall(r'[\\]+[vV]\[[0-9]+\]', jaString)
|
||||
varList = re.findall(r"[\\]+[vV]\[[0-9]+\]", jaString)
|
||||
varList = set(varList)
|
||||
if len(varList) != 0:
|
||||
for var in varList:
|
||||
jaString = jaString.replace(var, '[Var' + str(count) + ']')
|
||||
jaString = jaString.replace(var, "{Var_" + str(count) + "}")
|
||||
count += 1
|
||||
|
||||
# Formatting
|
||||
count = 0
|
||||
if "笑えるよね." in jaString:
|
||||
print("t")
|
||||
formatList = re.findall(r"[\\]+[\w]+\[.+?\]", jaString)
|
||||
formatList = set(formatList)
|
||||
if len(formatList) != 0:
|
||||
for var in formatList:
|
||||
jaString = jaString.replace(var, "{FCode_" + str(count) + "}")
|
||||
count += 1
|
||||
|
||||
# Put all lists in list and return
|
||||
allList = [iconList, colorList, nameList, varList]
|
||||
allList = [nestedList, iconList, colorList, nameList, varList, formatList]
|
||||
return [jaString, allList]
|
||||
|
||||
|
||||
def resubVars(translatedText, allList):
|
||||
# Fix Spacing and ChatGPT Nonsense
|
||||
matchList = re.findall(r'\[\s?.+?\s?\]', translatedText)
|
||||
matchList = re.findall(r"\[\s?.+?\s?\]", translatedText)
|
||||
if len(matchList) > 0:
|
||||
for match in matchList:
|
||||
text = match.strip()
|
||||
translatedText = translatedText.replace(match, text)
|
||||
|
||||
# Icons
|
||||
# Nested
|
||||
count = 0
|
||||
if len(allList[0]) != 0:
|
||||
for var in allList[0]:
|
||||
translatedText = translatedText.replace('[Icon' + str(count) + ']', var)
|
||||
translatedText = translatedText.replace("{Nested_" + str(count) + "}", var)
|
||||
count += 1
|
||||
|
||||
# Icons
|
||||
count = 0
|
||||
if len(allList[1]) != 0:
|
||||
for var in allList[1]:
|
||||
translatedText = translatedText.replace("{Ascii_" + str(count) + "}", var)
|
||||
count += 1
|
||||
|
||||
# Colors
|
||||
count = 0
|
||||
if len(allList[1]) != 0:
|
||||
for var in allList[1]:
|
||||
translatedText = translatedText.replace('[Color' + str(count) + ']', var)
|
||||
if len(allList[2]) != 0:
|
||||
for var in allList[2]:
|
||||
translatedText = translatedText.replace("{Color_" + str(count) + "}", var)
|
||||
count += 1
|
||||
|
||||
# Names
|
||||
count = 0
|
||||
if len(allList[2]) != 0:
|
||||
for var in allList[2]:
|
||||
translatedText = translatedText.replace('[Name' + str(count) + ']', var)
|
||||
if len(allList[3]) != 0:
|
||||
for var in allList[3]:
|
||||
translatedText = translatedText.replace("{N_" + str(count) + "}", var)
|
||||
count += 1
|
||||
|
||||
# Vars
|
||||
count = 0
|
||||
if len(allList[3]) != 0:
|
||||
for var in allList[3]:
|
||||
translatedText = translatedText.replace('[Var' + str(count) + ']', var)
|
||||
if len(allList[4]) != 0:
|
||||
for var in allList[4]:
|
||||
translatedText = translatedText.replace("{Var_" + str(count) + "}", var)
|
||||
count += 1
|
||||
|
||||
# Formatting
|
||||
count = 0
|
||||
if len(allList[5]) != 0:
|
||||
for var in allList[5]:
|
||||
translatedText = translatedText.replace("{FCode_" + str(count) + "}", var)
|
||||
count += 1
|
||||
|
||||
# Remove Color Variables Spaces
|
||||
|
|
@ -477,38 +455,54 @@ def resubVars(translatedText, allList):
|
|||
# translatedText = re.sub(r'\s*(\\+c\[0+\])', r'\1', translatedText)
|
||||
return translatedText
|
||||
|
||||
|
||||
@retry(exceptions=Exception, tries=5, delay=5)
|
||||
def translateGPT(t, history, fullPromptFlag):
|
||||
# If ESTIMATE is True just count this as an execution and return.
|
||||
if ESTIMATE:
|
||||
enc = tiktoken.encoding_for_model(MODEL)
|
||||
tokens = len(enc.encode(t)) * 2 + len(enc.encode(str(history))) + len(enc.encode(PROMPT))
|
||||
return (t, tokens)
|
||||
|
||||
# Sub Vars
|
||||
varResponse = subVars(t)
|
||||
subbedT = varResponse[0]
|
||||
|
||||
# If there isn't any Japanese in the text just skip
|
||||
if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴ]+|[\uFF00-\uFFEF]', subbedT):
|
||||
return(t, 0)
|
||||
if not re.search(r"[一-龠]+|[ぁ-ゔ]+|[ァ-ヴ]+|[\uFF00-\uFFEF]", subbedT):
|
||||
return (t, [0, 0])
|
||||
|
||||
# If ESTIMATE is True just count this as an execution and return.
|
||||
if ESTIMATE:
|
||||
enc = tiktoken.encoding_for_model(MODEL)
|
||||
historyRaw = ""
|
||||
if isinstance(history, list):
|
||||
for line in history:
|
||||
historyRaw += line
|
||||
else:
|
||||
historyRaw = history
|
||||
|
||||
inputTotalTokens = len(enc.encode(historyRaw)) + len(enc.encode(PROMPT))
|
||||
outputTotalTokens = (
|
||||
len(enc.encode(t)) * 2
|
||||
) # Estimating 2x the size of the original text
|
||||
totalTokens = [inputTotalTokens, outputTotalTokens]
|
||||
return (t, totalTokens)
|
||||
|
||||
# Characters
|
||||
context = '```\
|
||||
Game Characters:\
|
||||
Character: 亜紀子 == Akiko - Gender: Female\
|
||||
Character: 明日香 == Asuka - Gender: Female\
|
||||
```'
|
||||
context = "Game Characters:\
|
||||
Character: マコ == Mako - Gender: Female\
|
||||
Character: 主人公 == Protagonist - Gender: Male"
|
||||
|
||||
# Prompt
|
||||
if fullPromptFlag:
|
||||
system = PROMPT
|
||||
user = 'Line to Translate = ' + subbedT
|
||||
user = "Line to Translate = " + subbedT
|
||||
else:
|
||||
system = 'Output ONLY the '+ LANGUAGE +' translation in the following format: `Translation: <'+ LANGUAGE.upper() +'_TRANSLATION>`'
|
||||
user = 'Line to Translate = ' + subbedT
|
||||
system = (
|
||||
"Output ONLY the "
|
||||
+ LANGUAGE
|
||||
+ " translation in the following format: `Translation: <"
|
||||
+ LANGUAGE.upper()
|
||||
+ "_TRANSLATION>`"
|
||||
)
|
||||
user = "Line to Translate = " + subbedT
|
||||
|
||||
# Create Message List
|
||||
# Create Message List
|
||||
msg = []
|
||||
msg.append({"role": "system", "content": system})
|
||||
msg.append({"role": "user", "content": context})
|
||||
|
|
@ -520,7 +514,7 @@ def translateGPT(t, history, fullPromptFlag):
|
|||
msg.append({"role": "user", "content": user})
|
||||
|
||||
response = openai.ChatCompletion.create(
|
||||
temperature=0.1,
|
||||
temperature=0,
|
||||
frequency_penalty=0.2,
|
||||
presence_penalty=0.2,
|
||||
model=MODEL,
|
||||
|
|
@ -530,27 +524,35 @@ def translateGPT(t, history, fullPromptFlag):
|
|||
|
||||
# Save Translated Text
|
||||
translatedText = response.choices[0].message.content
|
||||
tokens = response.usage.total_tokens
|
||||
totalTokens = [response.usage.prompt_tokens, response.usage.completion_tokens]
|
||||
|
||||
# Resub Vars
|
||||
translatedText = resubVars(translatedText, varResponse[1])
|
||||
|
||||
# Remove Placeholder Text
|
||||
translatedText = translatedText.replace(LANGUAGE +' Translation: ', '')
|
||||
translatedText = translatedText.replace('Translation: ', '')
|
||||
translatedText = translatedText.replace('Line to Translate = ', '')
|
||||
translatedText = translatedText.replace('Translation = ', '')
|
||||
translatedText = translatedText.replace('Translate = ', '')
|
||||
translatedText = translatedText.replace(LANGUAGE +' Translation:', '')
|
||||
translatedText = translatedText.replace('Translation:', '')
|
||||
translatedText = translatedText.replace('Line to Translate =', '')
|
||||
translatedText = translatedText.replace('Translation =', '')
|
||||
translatedText = translatedText.replace('Translate =', '')
|
||||
translatedText = re.sub(r'Note:.*', '', translatedText)
|
||||
translatedText = translatedText.replace('っ', '')
|
||||
translatedText = translatedText.replace(LANGUAGE + " Translation: ", "")
|
||||
translatedText = translatedText.replace("Translation: ", "")
|
||||
translatedText = translatedText.replace("Line to Translate = ", "")
|
||||
translatedText = translatedText.replace("Translation = ", "")
|
||||
translatedText = translatedText.replace("Translate = ", "")
|
||||
translatedText = translatedText.replace(LANGUAGE + " Translation:", "")
|
||||
translatedText = translatedText.replace("Translation:", "")
|
||||
translatedText = translatedText.replace("Line to Translate =", "")
|
||||
translatedText = translatedText.replace("Translation =", "")
|
||||
translatedText = translatedText.replace("Translate =", "")
|
||||
translatedText = translatedText.replace("っ", "")
|
||||
translatedText = translatedText.replace("ッ", "")
|
||||
translatedText = translatedText.replace("ぁ", "")
|
||||
translatedText = translatedText.replace("。", ".")
|
||||
translatedText = translatedText.replace("、", ",")
|
||||
translatedText = translatedText.replace("？", "?")
|
||||
translatedText = translatedText.replace("！", "!")
|
||||
|
||||
# Return Translation
|
||||
if len(translatedText) > 15 * len(t) or "I'm sorry, but I'm unable to assist with that translation" in translatedText:
|
||||
if (
|
||||
len(translatedText) > 15 * len(t)
|
||||
or "I'm sorry, but I'm unable to assist with that translation" in translatedText
|
||||
):
|
||||
raise Exception
|
||||
else:
|
||||
return [translatedText, tokens]
|
||||
return [translatedText, totalTokens]
|
||||
|
|
|
|||
Loading…
Reference in a new issue