dazed/DazedTL
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'rework-translation-system' of https://github.com/dazedanon/GPT4-TL into rework-translation-system

Browse source
This commit is contained in:
Dazed 2023-12-12 18:22:03 -06:00
commit 4f5c4f1e09
4 changed files with 102 additions and 198 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
modules/alice.py
View file

@ -427,6 +427,9 @@ def translateText(characters, system, user, history):
# Prompt
msg = [{"role": "system", "content": system + characters}]
# Characters
msg.append({"role": "system", "content": characters})
# History
if isinstance(history, list):
msg.extend([{"role": "assistant", "content": h} for h in history])

218
modules/main.py
View file

@ -1,10 +1,19 @@
import sys, os, traceback
from concurrent.futures import ThreadPoolExecutor, as_completed
import sys
import traceback
from colorama import Fore
import os
from tqdm import tqdm
# This needs to be before the module imports as some of them currently try to read and use some of these values
# upon import, in which case if they are unset the script will crash before we can output these messages.
envMissing = False
for env in ['api','key','organization','model','language','timeout','fileThreads','threads','width','listWidth']:
if os.getenv(env) is None or str(os.getenv(env))[:1] == '<':
tqdm.write(Fore.RED + f'Environment variable {env} is not set!')
envMissing = True
if envMissing:
tqdm.write(Fore.RED + f'Some of the required environment values may not be set correctly. You can set \
these values using an .env file, for an example see .env.example')
from modules.rpgmakermvmz import handleMVMZ
from modules.rpgmakerace import handleACE
from modules.csv import handleCSV
@ -20,8 +29,22 @@ from modules.anim import handleAnim
# 1 Thread for each file. Controls how many files are worked on at once.
THREADS = int(os.getenv('fileThreads'))
# [Display name, file extension, handle function]
MODULES = [
["RPGMaker MV/MZ", "json", handleMVMZ],
["RPGMaker ACE", "yaml", handleACE],
["CSV (From Translator++)", "csv", handleCSV],
["Alice", "txt", handleAlice],
["Tyrano", "ks", handleTyrano],
["JSON", "json", handleJSON],
["Kansen", "ks", handleKansen],
["Lune", "txt", handleLuneTxt],
["Atelier", "txt", handleAtelier],
["Anim", "json", handleAnim],
]
# Info Message
tqdm.write(Fore.LIGHTYELLOW_EX + "WARNING: Once a translation starts do not close it unless you want to lose your\
tqdm.write(Fore.LIGHTYELLOW_EX + "WARNING: Once the translation starts do not close it unless you want to lose your \
translated data. If a file fails or gets stuck, translated lines will remain translated so you don't have \
to worry about being charged twice. You can simply copy the file generated in /translations back over to \
/files and start the script again. It will skip over any translated text." + Fore.RESET, end='\n\n')
@ -29,7 +52,7 @@ to worry about being charged twice. You can simply copy the file generated in /t
def main():
estimate = ''
while estimate == '':
estimate = input('Select Translation or Cost Estimation:\n\n1. Translate\n2. Estimate\n')
estimate = input('Select Translation or Cost Estimation:\n\n 1. Translate\n 2. Estimate\n')
match estimate:
case '1':
estimate = False
@ -37,172 +60,41 @@ def main():
estimate = True
case _:
estimate = ''
totalCost = 0
version = ''
while version == '':
version = input('Select the RPGMaker Version:\n\n\
1. MV/MZ\n\
2. ACE\n\
3. CSV (From Translator++)\n\
4. Alice\n\
5. Tyrano\n\
6. JSON\n\
7. Kansen\n\
8. Lune\n\
9. Atelier\n\
10. Anim\n'
)
match version:
case '1':
# Open File (Threads)
with ThreadPoolExecutor(max_workers=THREADS) as executor:
futures = [executor.submit(handleMVMZ, filename, estimate) \
for filename in os.listdir("files") if filename.endswith('json')]
while True:
tqdm.write("Select game engine:\n")
for position, module in enumerate(MODULES):
tqdm.write(f'{str(position + 1).rjust(2)}. {module[0]} (.{module[1]})')
version = input()
try:
version = int(version) - 1
except:
continue
if version in range(len(MODULES)):
break
totalCost = Fore.RED + 'Translation module didn\'t return the total cost. Make sure the \
files to translate are in the /files folder and that you picked the right game engine.'
# Open File (Threads)
with ThreadPoolExecutor(max_workers=THREADS) as executor:
futures = [executor.submit(MODULES[version][2], filename, estimate) \
for filename in os.listdir("files") if filename.endswith(MODULES[version][1])]
for future in as_completed(futures):
try:
totalCost = future.result()
except Exception as e:
tracebackLineNo = str(traceback.extract_tb(sys.exc_info()[2])[-1].lineno)
tqdm.write(Fore.RED + str(e) + '|' + tracebackLineNo + Fore.RESET)
for future in as_completed(futures):
try:
totalCost = future.result()
except Exception as e:
tracebackLineNo = str(traceback.extract_tb(sys.exc_info()[2])[-1].lineno)
tqdm.write(Fore.RED + str(e) + '|' + tracebackLineNo + Fore.RESET)
case '2':
# Open File (Threads)
with ThreadPoolExecutor(max_workers=THREADS) as executor:
futures = [executor.submit(handleACE, filename, estimate) \
for filename in os.listdir("files") if filename.endswith('yaml')]
for future in as_completed(futures):
try:
totalCost = future.result()
except Exception as e:
tracebackLineNo = str(traceback.extract_tb(sys.exc_info()[2])[-1].lineno)
tqdm.write(Fore.RED + str(e) + '|' + tracebackLineNo + Fore.RESET)
case '3':
# Open File (Threads)
with ThreadPoolExecutor(max_workers=THREADS) as executor:
futures = [executor.submit(handleCSV, filename, estimate) \
for filename in os.listdir("files") if filename.endswith('csv')]
for future in as_completed(futures):
try:
totalCost = future.result()
except Exception as e:
tracebackLineNo = str(traceback.extract_tb(sys.exc_info()[2])[-1].lineno)
tqdm.write(Fore.RED + str(e) + '|' + tracebackLineNo + Fore.RESET)
case '4':
# Open File (Threads)
with ThreadPoolExecutor(max_workers=THREADS) as executor:
futures = [executor.submit(handleAlice, filename, estimate) \
for filename in os.listdir("files") if filename.endswith('txt')]
for future in as_completed(futures):
try:
totalCost = future.result()
except Exception as e:
tracebackLineNo = str(traceback.extract_tb(sys.exc_info()[2])[-1].lineno)
tqdm.write(Fore.RED + str(e) + '|' + tracebackLineNo + Fore.RESET)
case '5':
# Open File (Threads)
with ThreadPoolExecutor(max_workers=THREADS) as executor:
futures = [executor.submit(handleTyrano, filename, estimate) \
for filename in os.listdir("files") if filename.endswith('ks')]
for future in as_completed(futures):
try:
totalCost = future.result()
except Exception as e:
tracebackLineNo = str(traceback.extract_tb(sys.exc_info()[2])[-1].lineno)
tqdm.write(Fore.RED + str(e) + '|' + tracebackLineNo + Fore.RESET)
case '6':
# Open File (Threads)
with ThreadPoolExecutor(max_workers=THREADS) as executor:
futures = [executor.submit(handleJSON, filename, estimate) \
for filename in os.listdir("files") if filename.endswith('json')]
for future in as_completed(futures):
try:
totalCost = future.result()
except Exception as e:
tracebackLineNo = str(traceback.extract_tb(sys.exc_info()[2])[-1].lineno)
tqdm.write(Fore.RED + str(e) + '|' + tracebackLineNo + Fore.RESET)
case '7':
# Open File (Threads)
with ThreadPoolExecutor(max_workers=THREADS) as executor:
futures = [executor.submit(handleKansen, filename, estimate) \
for filename in os.listdir("files") if filename.endswith('ks')]
for future in as_completed(futures):
try:
totalCost = future.result()
except Exception as e:
tracebackLineNo = str(traceback.extract_tb(sys.exc_info()[2])[-1].lineno)
tqdm.write(Fore.RED + str(e) + '|' + tracebackLineNo + Fore.RESET)
case '8':
# Open File (Threads)
with ThreadPoolExecutor(max_workers=THREADS) as executor:
futures = [executor.submit(handleLuneTxt, filename, estimate) \
for filename in os.listdir("files") if filename.endswith('txt')]
for future in as_completed(futures):
try:
totalCost = future.result()
except Exception as e:
tracebackLineNo = str(traceback.extract_tb(sys.exc_info()[2])[-1].lineno)
tqdm.write(Fore.RED + str(e) + '|' + tracebackLineNo + Fore.RESET)
case '9':
# Open File (Threads)
with ThreadPoolExecutor(max_workers=THREADS) as executor:
futures = [executor.submit(handleAtelier, filename, estimate) \
for filename in os.listdir("files") if filename.endswith('txt')]
for future in as_completed(futures):
try:
totalCost = future.result()
except Exception as e:
tracebackLineNo = str(traceback.extract_tb(sys.exc_info()[2])[-1].lineno)
tqdm.write(Fore.RED + str(e) + '|' + tracebackLineNo + Fore.RESET)
case '10':
# Open File (Threads)
with ThreadPoolExecutor(max_workers=THREADS) as executor:
futures = [executor.submit(handleAnim, filename, estimate) \
for filename in os.listdir("files") if filename.endswith('json')]
for future in as_completed(futures):
try:
totalCost = future.result()
except Exception as e:
tracebackLineNo = str(traceback.extract_tb(sys.exc_info()[2])[-1].lineno)
tqdm.write(Fore.RED + str(e) + '|' + tracebackLineNo + Fore.RESET)
case _:
version = ''
if totalCost != 'Fail':
if estimate is False:
# This is to encourage people to grab what's in /translated instead
deleteFolderFiles('files')
# Prevent immediately closing of CLI
tqdm.write(totalCost)
# input('Done! Press Enter to close.')
tqdm.write(str(totalCost))
def deleteFolderFiles(folderPath):
for filename in os.listdir(folderPath):

58
modules/rpgmakermvmz.py
View file

@ -41,10 +41,12 @@ if 'gpt-3.5' in MODEL:
INPUTAPICOST = .002
OUTPUTAPICOST = .002
BATCHSIZE = 10
FREQUENCY_PENALTY = 0.2
elif 'gpt-4' in MODEL:
INPUTAPICOST = .01
OUTPUTAPICOST = .03
BATCHSIZE = 40
FREQUENCY_PENALTY = 0
#tqdm Globals
BAR_FORMAT='{l_bar}{bar:10}{r_bar}{bar:-10b}'
@ -52,11 +54,11 @@ POSITION = 0
LEAVE = False
# Dialogue / Scroll
CODE401 = False
CODE401 = True
CODE405 = False
# Choices
CODE102 = False
CODE102 = True
# Variables
CODE122 = False
@ -72,7 +74,7 @@ CODE356 = False
CODE320 = False
CODE324 = False
CODE111 = False
CODE108 = True
CODE108 = False
CODE408 = False
def handleMVMZ(filename, estimate):
@ -1514,7 +1516,8 @@ def searchCodes(page, pbar, fillList, filename):
if len(fillList) != len(docList):
global MISMATCH
with LOCK:
MISMATCH.append(filename)
if filename not in MISMATCH:
MISMATCH.append(filename)
else:
docList = []
searchCodes(page, pbar, fillList, filename)
@ -1840,12 +1843,22 @@ def batchList(input_list, batch_size):
return [input_list[i:i + batch_size] for i in range(0, len(input_list), batch_size)]
def createContext(fullPromptFlag, subbedT):
characters = 'Game Characters:\
篠崎 誠一 == Shinozaki Seiichi - Male\
宮前 遥奈 == Miyamae Haruna - Female\
榛名 悠真 == Haruna Yuuma - Male\
浪川 時宗 == Namikawa Tokimune - Male\
高嶋 美雪 == Takashima Miyuki - Female'
characters = 'Game Characters (Format: Last Name, First Name - Gender):\
護 == Mamoru - Male\
神代 一騎 == Kamishiro, Ikki - Male\
神代 琴音 == Kamishiro, Kotone - Female\
神代 莉々子 == Kamishiro, Ririko - Female\
神代 紗夜 == Kamishiro, Saya - Female\
篠原漣 == Shinohara, Ren - Male\
藪井 == Yabui - Male\
舟木 == Funaki - Male\
貞二 == Jouji - Male\
兼田 響子 == Kaneda, Kyouko - Female\
兼田 真人 == Kaneda, Masato - Male\
小出 == Koide - Male\
進士 == Shinji - Male\
雪乃 == Yukino - Male'
system = PROMPT if fullPromptFlag else \
f'Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`'
user = f'{subbedT}'
@ -1853,26 +1866,23 @@ def createContext(fullPromptFlag, subbedT):
def translateText(characters, system, user, history):
# Prompt
msg = [{"role": "system", "content": system}]
# Characters
msg.append({"role": "user", "content": characters})
msg = [{"role": "system", "content": system + characters}]
# History
if isinstance(history, list):
msg.extend([{"role": "user", "content": h} for h in history])
msg.extend([{"role": "assistant", "content": h} for h in history])
else:
msg.append({"role": "user", "content": history})
msg.append({"role": "assistant", "content": history})
# Content to TL
msg.append({"role": "user", "content": user})
response = openai.ChatCompletion.create(
temperature=0,
frequency_penalty=0,
presence_penalty=0,
msg.append({"role": "user", "content": f'{user}'})
response = openai.chat.completions.create(
temperature=0.1,
top_p = 0.2,
frequency_penalty=FREQUENCY_PENALTY,
presence_penalty=FREQUENCY_PENALTY,
model=MODEL,
messages=msg,
request_timeout=TIMEOUT,
)
return response
@ -1890,7 +1900,7 @@ def cleanTranslatedText(translatedText, varResponse):
return [line for line in translatedText.split('\n') if line]
def extractTranslation(translatedTextList, is_list):
pattern = r'<Line(\d+)>(.*)</Line\d+>'
pattern = r'<Line(\d+)>?(.*)</Line\d+>'
# If it's a batch (i.e., list), extract with tags; otherwise, return the single item.
if is_list:
return [re.findall(pattern, line)[0][1] for line in translatedTextList if re.search(pattern, line)]
@ -1966,6 +1976,8 @@ def translateGPT(text, history, fullPromptFlag):
if isinstance(tItem, list):
extractedTranslations = extractTranslation(translatedTextList, True)
tList[index] = extractedTranslations
if len(tList[index]) != len(translatedTextList):
print('Test')
history = extractedTranslations[-10:] # Update history if we have a list
else:
# Ensure we're passing a single string to extractTranslation

21
prompt.example
View file

@ -5,30 +5,27 @@ I will give you lines of text, and you must translate each line to the best of y
Use the following step-by-step instructions to respond to user inputs.
Step 1 - Receive Text
You will be given multiple lines of text (Denoted by XML tags). Translate each line separately and avoid combining or omitting any.
You will be given multiple lines of text (Denoted by XML). Translate each line separately and avoid combining or omitting any.
Step 2 - Output Text
You output only the English translation of each line. For example:
<Line0>English Translation of Line 0</Line0>
<Line1>English Translation of Line 1</Line1>
<Line2>English Translation of Line 2</Line2>
You output the English translation of each line and include the same XML tags. These XML tags are not optional, you must always include them.
Step 3 - Check Work
Double check that the number of lines/xml-tags in your response match the number of lines/xml-tags from the user message.
For example:
<Line0>English Translation</Line0>\n<Line1>English Translation</Line1>\n<Line2>English Translation</Line2>
Other Notes:
Notes:
- "Game Characters" - The names, nicknames, and genders of the game characters. Reference this to know the names, nicknames, and gender of characters in the game.
- Always include both the start and end xml tag in your translation.
- Only reply with the English Translation even if it may be hard to translate.
- If a line is already translated, empty, or can't be translated, do not change it.
- Denote speakers with ':' if given. For example L1 - Speaker: "Spoken Text"
- If a line is already in English, empty, or can't be translated, leave it in English and include it in your response.
- Each line must have the translation of the original text inside.
- Denote speakers with ':' if given. For example <Line0>Speaker: "Spoken Text"</Line0>
- If the speaker is '???' then they are unknown so leave it as is.
- Pay attention to the gender of the subjects and characters. Avoid misgendering characters.
- Maintain any spacing in the translation.
- Maintain any code in brackets if given.
- Maintain the speaker if given.
- Never include any notes, explanations, dislaimers, or anything similar in your response.
- Maintain Japanese Honorifics. For example: 'サクラねえちゃん' == 'Sakura Onee-san'
- "--" is intentional and indicates a blank in the text. Leave it as is.
- Translate 'おまんこ' as 'pussy'.
- Translate 'お尻' as 'butt'
- Translate '尻' as 'ass'