dazed/DazedTL
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Save latest changes

Browse source
This commit is contained in:
Dazed 2023-09-25 15:04:09 -05:00
parent cabaf9f82a
commit 47647abe40
4 changed files with 158 additions and 40 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -5,4 +5,5 @@
*.txt
!requirements.txt
*.csv
*.ks
__pycache__

92
modules/alltext.py Normal file
View file

@ -0,0 +1,92 @@
from concurrent.futures import ThreadPoolExecutor, as_completed
import json
import os
from pathlib import Path
import re
import sys
import textwrap
import threading
import time
import traceback
import tiktoken
from colorama import Fore
from dotenv import load_dotenv
import openai
from retry import retry
from tqdm import tqdm
#Globals
load_dotenv()
openai.organization = os.getenv('org')
openai.api_key = os.getenv('key')
APICOST = .002 # Depends on the model https://openai.com/pricing
PROMPT = Path('prompt.txt').read_text(encoding='utf-8')
THREADS = 10 # For GPT4 rate limit will be hit if you have more than 1 thread.
LOCK = threading.Lock()
WIDTH = 60
LISTWIDTH = 60
MAXHISTORY = 10
ESTIMATE = ''
TOTALCOST = 0
TOKENS = 0
TOTALTOKENS = 0
NAMESLIST = []
#tqdm Globals
BAR_FORMAT='{l_bar}{bar:10}{r_bar}{bar:-10b}'
POSITION=0
LEAVE=False
def handleAllText(filename, estimate):
global ESTIMATE, TOKENS, TOTALTOKENS, TOTALCOST
ESTIMATE = estimate
with open('translated/' + filename, 'w+t', newline='', encoding='utf-16-le') as writeFile:
start = time.time()
translatedData = openFiles(filename, writeFile)
if estimate:
# Print Result
end = time.time()
tqdm.write(getResultString(['', TOKENS, None], end - start, filename))
TOTALCOST += TOKENS * .001 * APICOST
TOTALTOKENS += TOKENS
TOKENS = 0
os.remove('translated/' + filename)
else:
# Print Result
end = time.time()
tqdm.write(getResultString(translatedData, end - start, filename))
TOTALCOST += translatedData[1] * .001 * APICOST
TOTALTOKENS += translatedData[1]
return getResultString(['', TOTALTOKENS, None], end - start, 'TOTAL')
def openFiles(filename, writeFile):
with open('files/' + filename, 'r', encoding='utf-16-le') as readFile, writeFile:
translatedData = parseText(readFile, writeFile, filename)
return translatedData
def getResultString(translatedData, translationTime, filename):
# File Print String
tokenString = Fore.YELLOW + '[' + str(translatedData[1]) + \
' Tokens/${:,.4f}'.format(translatedData[1] * .001 * APICOST) + ']'
timeString = Fore.BLUE + '[' + str(round(translationTime, 1)) + 's]'
if translatedData[2] == None:
# Success
return filename + ': ' + tokenString + timeString + Fore.GREEN + u' \u2713 ' + Fore.RESET
else:
# Fail
try:
raise translatedData[2]
except Exception as e:
errorString = str(e) + '|' + translatedData[3] + Fore.RED
return filename + ': ' + tokenString + timeString + Fore.RED + u' \u2717 ' +\
errorString + Fore.RESET

2
modules/main.py
View file

@ -9,7 +9,7 @@ from modules.rpgmakerace import handleACE
from modules.csvtl import handleCSV
from modules.textfile import handleTextfile
THREADS = 20 # For GPT4 rate limit will be hit if you have more than 1 thread.
THREADS = 10 # For GPT4 rate limit will be hit if you have more than 1 thread.
# Info Message
print(Fore.LIGHTYELLOW_EX + "WARNING: Once a translation starts do not close it unless you want to lose your\

103
modules/rpgmakermvmz.py
View file

@ -25,7 +25,7 @@ APICOST = .002 # Depends on the model https://openai.com/pricing
PROMPT = Path('prompt.txt').read_text(encoding='utf-8')
THREADS = 10 # For GPT4 rate limit will be hit if you have more than 1 thread.
LOCK = threading.Lock()
WIDTH = 40
WIDTH = 60
LISTWIDTH = 60
MAXHISTORY = 10
ESTIMATE = ''
@ -54,7 +54,9 @@ CODE324 = False
CODE111 = False
CODE408 = False
CODE108 = False
NAMES = True # Output a list of all the character names found
NAMES = True # Output a list of all the character names found
BRFLAG = False # If the game uses <br> instead
FIXTEXTWRAP = False
def handleMVMZ(filename, estimate):
global ESTIMATE, TOKENS, TOTALTOKENS, TOTALCOST
@ -189,7 +191,7 @@ def parseMap(data, filename):
if 'Map' in filename:
response = translateGPT(data['displayName'], 'Reply with only the english translation of the RPG location name', False)
totalTokens += response[1]
data['displayName'] = response[0].strip('.\"')
data['displayName'] = response[0].replace('\"', '')
# Get total for progress bar
for event in events:
@ -401,14 +403,14 @@ def searchThings(name, pbar):
# Set Data
if 'name' in name:
name['name'] = nameResponse[0].strip('\"')
name['name'] = nameResponse[0].replace('\"', '')
if 'description' in name:
description = descriptionResponse[0]
# Remove Textwrap
description = description.replace('\n', ' ')
description = textwrap.fill(descriptionResponse[0], LISTWIDTH)
name['description'] = description.strip('\"')
name['description'] = description.replace('\"', '')
pbar.update(1)
return tokens
@ -461,19 +463,19 @@ def searchNames(name, pbar, context):
responseList[i] = responseList[i][0]
# Set Data
name['name'] = responseList[0].strip('.\"')
name['name'] = responseList[0].replace('\"', '')
if 'Actors' in context:
translatedText = textwrap.fill(responseList[1], LISTWIDTH)
name['profile'] = translatedText.strip('\"')
name['profile'] = translatedText.replace('\"', '')
translatedText = textwrap.fill(responseList[2], LISTWIDTH)
name['nickname'] = translatedText.strip('\"')
name['nickname'] = translatedText.replace('\"', '')
if '<特徴1:' in name['note']:
tokens += translateNote(name, r'<特徴1:([^>]*)>')
if 'Armors' in context or 'Weapons' in context:
translatedText = textwrap.fill(responseList[1], LISTWIDTH)
if 'description' in name:
name['description'] = translatedText.strip('\"')
name['description'] = translatedText.replace('\"', '')
if '<SG説明:' in name['note']:
tokens += translateNote(name, r'<Info Text Bottom>\n([\s\S]*?)\n</Info Text Bottom>')
pbar.update(1)
@ -581,10 +583,30 @@ def searchCodes(page, pbar):
codeList[j]['code'] = code
# Remove nametag from final string
finalJAString = finalJAString.replace(match[0], '')
finalJAString = finalJAString.replace(match[0], '')
elif '\\nc' in finalJAString:
matchList = re.findall(r'(\\+nc<(.*?)>)(.+)?', finalJAString)
if len(matchList) != 0:
# Translate Speaker
response = translateGPT(matchList[0][1], 'Reply with only the english translation of the NPC name', True)
tokens += response[1]
speaker = response[0].strip('.')
nametag = matchList[0][0].replace(matchList[0][1], speaker)
finalJAString = finalJAString.replace(matchList[0][0], '')
# Set dialogue
codeList[j]['parameters'][0] = matchList[0][2]
codeList[j]['code'] = 401
# Remove nametag from final string
finalJAString = finalJAString.replace(nametag, '')
# Remove any textwrap
finalJAString = re.sub(r'\n', ' ', finalJAString)
if FIXTEXTWRAP == True:
finalJAString = re.sub(r'\n', ' ', finalJAString)
finalJAString = finalJAString.replace('<br>', ' ')
# Remove Extra Stuff
finalJAString = finalJAString.replace('ﾞ', '')
finalJAString = finalJAString.replace('。', '.')
finalJAString = finalJAString.replace('・', '.')
@ -628,11 +650,10 @@ def searchCodes(page, pbar):
translatedText = finalJAString
# Textwrap
textList = re.findall(r'(^.+?)(\s.+)$', translatedText) # Strip out vars
if len(textList) > 0:
translatedText = textList[0][0] + textwrap.fill(textList[0][1], width=WIDTH)
else:
if '\n' not in translatedText and '<br>' not in translatedText:
translatedText = textwrap.fill(translatedText, width=WIDTH)
if BRFLAG == True:
translatedText = translatedText.replace('\n', '<br>')
# Add Beginning Text
translatedText = startString + translatedText
@ -660,8 +681,8 @@ def searchCodes(page, pbar):
if codeList[i]['code'] == 122 and CODE122 == True:
# This is going to be the var being set. (IMPORTANT)
varNum = codeList[i]['parameters'][0]
if varNum != 328:
continue
# if varNum != 328:
# continue
jaString = codeList[i]['parameters'][4]
if type(jaString) != str:
@ -671,6 +692,10 @@ def searchCodes(page, pbar):
if '■' in jaString or '_' in jaString:
continue
# Definitely don't want to mess with files
if '\"' not in jaString:
continue
# Need to remove outside code and put it back later
matchList = re.findall(r"[\'\"\`](.*?)[\'\"\`]", jaString)
@ -691,10 +716,10 @@ def searchCodes(page, pbar):
translatedText = translatedText.replace(char, '')
# Textwrap
translatedText = textwrap.fill(translatedText, width=30)
translatedText = translatedText.replace('\n', '\\n')
translatedText = translatedText.replace('\'', '\\\'')
translatedText = '\'' + translatedText + '\''
# translatedText = textwrap.fill(translatedText, width=30)
# translatedText = translatedText.replace('\n', '\\n')
# translatedText = translatedText.replace('\'', '\\\'')
translatedText = '\"' + translatedText + '\"'
# Set Data
codeList[i]['parameters'][4] = translatedText
@ -827,7 +852,7 @@ def searchCodes(page, pbar):
NAMESLIST.append(speaker)
## Event Code: 355 or 655 Scripts [Optional]
if (codeList[i]['code'] == 355) and CODE355655 == True:
if (codeList[i]['code'] == 355 or codeList[i]['code'] == 655) and CODE355655 == True:
jaString = codeList[i]['parameters'][0]
# If there isn't any Japanese in the text just skip
@ -835,11 +860,11 @@ def searchCodes(page, pbar):
continue
# Want to translate this script
if codeList[i]['code'] == 355 and 'BattleManager._logWindow.addText' not in jaString:
if codeList[i]['code'] == 355 and '$gameSystem.addLog' not in jaString:
continue
# Don't want to touch certain scripts
if codeList[i]['code'] == 655 and '.' in jaString:
if codeList[i]['code'] == 655 and '$gameSystem.addLog' not in jaString:
continue
# Need to remove outside code and put it back later
@ -1198,20 +1223,20 @@ def searchSS(state, pbar):
# Set Data
if 'name' in state:
state['name'] = nameResponse[0].strip('\"')
state['name'] = nameResponse[0].replace('\"', '')
if 'description' in state:
# Textwrap
translatedText = descriptionResponse[0]
translatedText = textwrap.fill(translatedText, width=LISTWIDTH)
state['description'] = translatedText.strip('\"')
state['description'] = translatedText.replace('\"', '')
if 'message1' in state:
state['message1'] = message1Response[0].strip('\"').replace('Taro', '')
state['message1'] = message1Response[0].replace('\"', '').replace('Taro', '')
if 'message2' in state:
state['message2'] = message2Response[0].strip('\"').replace('Taro', '')
state['message2'] = message2Response[0].replace('\"', '').replace('Taro', '')
if 'message3' in state:
state['message3'] = message3Response[0].strip('\"').replace('Taro', '')
state['message3'] = message3Response[0].replace('\"', '').replace('Taro', '')
if 'message4' in state:
state['message4'] = message4Response[0].strip('\"').replace('Taro', '')
state['message4'] = message4Response[0].replace('\"', '').replace('Taro', '')
pbar.update(1)
return tokens
@ -1234,35 +1259,35 @@ def searchSystem(data, pbar):
if termList[i] is not None:
response = translateGPT(termList[i], context, False)
tokens += response[1]
termList[i] = response[0].strip('.\"')
termList[i] = response[0].replace('\"', '')
pbar.update(1)
# Armor Types
for i in range(len(data['armorTypes'])):
response = translateGPT(data['armorTypes'][i], 'Reply with only the english translation of the armor type', False)
tokens += response[1]
data['armorTypes'][i] = response[0].strip('.\"')
data['armorTypes'][i] = response[0].replace('\"', '')
pbar.update(1)
# Skill Types
for i in range(len(data['skillTypes'])):
response = translateGPT(data['skillTypes'][i], 'Reply with only the english translation', False)
tokens += response[1]
data['skillTypes'][i] = response[0].strip('.\"')
data['skillTypes'][i] = response[0].replace('\"', '')
pbar.update(1)
# Equip Types
for i in range(len(data['equipTypes'])):
response = translateGPT(data['equipTypes'][i], 'Reply with only the english translation of the equipment type. No disclaimers.', False)
tokens += response[1]
data['equipTypes'][i] = response[0].strip('.\"')
data['equipTypes'][i] = response[0].replace('\"', '')
pbar.update(1)
# Variables (Optional ususally)
for i in range(len(data['variables'])):
response = translateGPT(data['variables'][i], 'Reply with only the english translation of the title', False)
tokens += response[1]
data['variables'][i] = response[0].strip('.\"')
data['variables'][i] = response[0].replace('\"', '')
pbar.update(1)
# Messages
@ -1312,9 +1337,9 @@ def resubVars(translatedText, varList):
count += 1
# Remove Color Variables Spaces
if '\\c' in translatedText:
translatedText = re.sub(r'\s*(\\+c\[[1-9]+\])\s*', r' \1', translatedText)
translatedText = re.sub(r'\s*(\\+c\[0+\])', r'\1', translatedText)
# if '\\c' in translatedText:
# translatedText = re.sub(r'\s*(\\+c\[[1-9]+\])\s*', r' \1', translatedText)
# translatedText = re.sub(r'\s*(\\+c\[0+\])', r'\1', translatedText)
return translatedText
@retry(exceptions=Exception, tries=5, delay=5)
@ -1336,7 +1361,7 @@ def translateGPT(t, history, fullPromptFlag):
return(t, 0)
"""Translate text using GPT"""
context = 'Eroge Names Context: ボク == Boku | Male'
context = 'Eroge Names Context: 圭 == Kei | Female, レヴァンティア == Levantia | Female, カシューナッツ == Cashew Nut | Male, レイナ == Reina | Female, ピンクキャンディー == Pink Candy | Female, ルーシャーベット == Lusha Sorbet | Female, ブラックマッスル == Black Muscle | Male, ミスターヒプノシス == Mr. Hypnosis | Male, リリス == Lilith | Female, 侵食体 == Invasoid | Female'
if fullPromptFlag:
system = PROMPT
user = 'Line to Translate: ' + subbedT