diff --git a/modules/alice.py b/modules/alice.py index 76ebe52..d4629c6 100644 --- a/modules/alice.py +++ b/modules/alice.py @@ -427,6 +427,9 @@ def translateText(characters, system, user, history): # Prompt msg = [{"role": "system", "content": system + characters}] + # Characters + msg.append({"role": "system", "content": characters}) + # History if isinstance(history, list): msg.extend([{"role": "assistant", "content": h} for h in history]) diff --git a/modules/main.py b/modules/main.py index 2000fcc..9b04478 100644 --- a/modules/main.py +++ b/modules/main.py @@ -1,10 +1,19 @@ +import sys, os, traceback from concurrent.futures import ThreadPoolExecutor, as_completed -import sys -import traceback from colorama import Fore -import os from tqdm import tqdm +# This needs to be before the module imports as some of them currently try to read and use some of these values +# upon import, in which case if they are unset the script will crash before we can output these messages. +envMissing = False +for env in ['api','key','organization','model','language','timeout','fileThreads','threads','width','listWidth']: + if os.getenv(env) is None or str(os.getenv(env))[:1] == '<': + tqdm.write(Fore.RED + f'Environment variable {env} is not set!') + envMissing = True +if envMissing: + tqdm.write(Fore.RED + f'Some of the required environment values may not be set correctly. You can set \ +these values using an .env file, for an example see .env.example') + from modules.rpgmakermvmz import handleMVMZ from modules.rpgmakerace import handleACE from modules.csv import handleCSV @@ -20,8 +29,22 @@ from modules.anim import handleAnim # 1 Thread for each file. Controls how many files are worked on at once. THREADS = int(os.getenv('fileThreads')) +# [Display name, file extension, handle function] +MODULES = [ + ["RPGMaker MV/MZ", "json", handleMVMZ], + ["RPGMaker ACE", "yaml", handleACE], + ["CSV (From Translator++)", "csv", handleCSV], + ["Alice", "txt", handleAlice], + ["Tyrano", "ks", handleTyrano], + ["JSON", "json", handleJSON], + ["Kansen", "ks", handleKansen], + ["Lune", "txt", handleLuneTxt], + ["Atelier", "txt", handleAtelier], + ["Anim", "json", handleAnim], +] + # Info Message -tqdm.write(Fore.LIGHTYELLOW_EX + "WARNING: Once a translation starts do not close it unless you want to lose your\ +tqdm.write(Fore.LIGHTYELLOW_EX + "WARNING: Once the translation starts do not close it unless you want to lose your \ translated data. If a file fails or gets stuck, translated lines will remain translated so you don't have \ to worry about being charged twice. You can simply copy the file generated in /translations back over to \ /files and start the script again. It will skip over any translated text." + Fore.RESET, end='\n\n') @@ -29,7 +52,7 @@ to worry about being charged twice. You can simply copy the file generated in /t def main(): estimate = '' while estimate == '': - estimate = input('Select Translation or Cost Estimation:\n\n1. Translate\n2. Estimate\n') + estimate = input('Select Translation or Cost Estimation:\n\n 1. Translate\n 2. Estimate\n') match estimate: case '1': estimate = False @@ -37,172 +60,41 @@ def main(): estimate = True case _: estimate = '' - - totalCost = 0 + version = '' - while version == '': - version = input('Select the RPGMaker Version:\n\n\ -1. MV/MZ\n\ -2. ACE\n\ -3. CSV (From Translator++)\n\ -4. Alice\n\ -5. Tyrano\n\ -6. JSON\n\ -7. Kansen\n\ -8. Lune\n\ -9. Atelier\n\ -10. Anim\n' - ) - match version: - case '1': - # Open File (Threads) - with ThreadPoolExecutor(max_workers=THREADS) as executor: - futures = [executor.submit(handleMVMZ, filename, estimate) \ - for filename in os.listdir("files") if filename.endswith('json')] + while True: + tqdm.write("Select game engine:\n") + for position, module in enumerate(MODULES): + tqdm.write(f'{str(position + 1).rjust(2)}. {module[0]} (.{module[1]})') + version = input() + try: + version = int(version) - 1 + except: + continue + if version in range(len(MODULES)): + break + + totalCost = Fore.RED + 'Translation module didn\'t return the total cost. Make sure the \ +files to translate are in the /files folder and that you picked the right game engine.' + + # Open File (Threads) + with ThreadPoolExecutor(max_workers=THREADS) as executor: + futures = [executor.submit(MODULES[version][2], filename, estimate) \ + for filename in os.listdir("files") if filename.endswith(MODULES[version][1])] - for future in as_completed(futures): - try: - totalCost = future.result() - except Exception as e: - tracebackLineNo = str(traceback.extract_tb(sys.exc_info()[2])[-1].lineno) - tqdm.write(Fore.RED + str(e) + '|' + tracebackLineNo + Fore.RESET) + for future in as_completed(futures): + try: + totalCost = future.result() + except Exception as e: + tracebackLineNo = str(traceback.extract_tb(sys.exc_info()[2])[-1].lineno) + tqdm.write(Fore.RED + str(e) + '|' + tracebackLineNo + Fore.RESET) - case '2': - # Open File (Threads) - with ThreadPoolExecutor(max_workers=THREADS) as executor: - futures = [executor.submit(handleACE, filename, estimate) \ - for filename in os.listdir("files") if filename.endswith('yaml')] - - for future in as_completed(futures): - try: - totalCost = future.result() - except Exception as e: - tracebackLineNo = str(traceback.extract_tb(sys.exc_info()[2])[-1].lineno) - tqdm.write(Fore.RED + str(e) + '|' + tracebackLineNo + Fore.RESET) - - case '3': - # Open File (Threads) - with ThreadPoolExecutor(max_workers=THREADS) as executor: - futures = [executor.submit(handleCSV, filename, estimate) \ - for filename in os.listdir("files") if filename.endswith('csv')] - - for future in as_completed(futures): - try: - totalCost = future.result() - - except Exception as e: - tracebackLineNo = str(traceback.extract_tb(sys.exc_info()[2])[-1].lineno) - tqdm.write(Fore.RED + str(e) + '|' + tracebackLineNo + Fore.RESET) - - case '4': - # Open File (Threads) - with ThreadPoolExecutor(max_workers=THREADS) as executor: - futures = [executor.submit(handleAlice, filename, estimate) \ - for filename in os.listdir("files") if filename.endswith('txt')] - - for future in as_completed(futures): - try: - totalCost = future.result() - - except Exception as e: - tracebackLineNo = str(traceback.extract_tb(sys.exc_info()[2])[-1].lineno) - tqdm.write(Fore.RED + str(e) + '|' + tracebackLineNo + Fore.RESET) - - case '5': - # Open File (Threads) - with ThreadPoolExecutor(max_workers=THREADS) as executor: - futures = [executor.submit(handleTyrano, filename, estimate) \ - for filename in os.listdir("files") if filename.endswith('ks')] - - for future in as_completed(futures): - try: - totalCost = future.result() - - except Exception as e: - tracebackLineNo = str(traceback.extract_tb(sys.exc_info()[2])[-1].lineno) - tqdm.write(Fore.RED + str(e) + '|' + tracebackLineNo + Fore.RESET) - - case '6': - # Open File (Threads) - with ThreadPoolExecutor(max_workers=THREADS) as executor: - futures = [executor.submit(handleJSON, filename, estimate) \ - for filename in os.listdir("files") if filename.endswith('json')] - - for future in as_completed(futures): - try: - totalCost = future.result() - - except Exception as e: - tracebackLineNo = str(traceback.extract_tb(sys.exc_info()[2])[-1].lineno) - tqdm.write(Fore.RED + str(e) + '|' + tracebackLineNo + Fore.RESET) - - case '7': - # Open File (Threads) - with ThreadPoolExecutor(max_workers=THREADS) as executor: - futures = [executor.submit(handleKansen, filename, estimate) \ - for filename in os.listdir("files") if filename.endswith('ks')] - - for future in as_completed(futures): - try: - totalCost = future.result() - - except Exception as e: - tracebackLineNo = str(traceback.extract_tb(sys.exc_info()[2])[-1].lineno) - tqdm.write(Fore.RED + str(e) + '|' + tracebackLineNo + Fore.RESET) - - case '8': - # Open File (Threads) - with ThreadPoolExecutor(max_workers=THREADS) as executor: - futures = [executor.submit(handleLuneTxt, filename, estimate) \ - for filename in os.listdir("files") if filename.endswith('txt')] - - for future in as_completed(futures): - try: - totalCost = future.result() - - except Exception as e: - tracebackLineNo = str(traceback.extract_tb(sys.exc_info()[2])[-1].lineno) - tqdm.write(Fore.RED + str(e) + '|' + tracebackLineNo + Fore.RESET) - - case '9': - # Open File (Threads) - with ThreadPoolExecutor(max_workers=THREADS) as executor: - futures = [executor.submit(handleAtelier, filename, estimate) \ - for filename in os.listdir("files") if filename.endswith('txt')] - - for future in as_completed(futures): - try: - totalCost = future.result() - - except Exception as e: - tracebackLineNo = str(traceback.extract_tb(sys.exc_info()[2])[-1].lineno) - tqdm.write(Fore.RED + str(e) + '|' + tracebackLineNo + Fore.RESET) - - case '10': - # Open File (Threads) - with ThreadPoolExecutor(max_workers=THREADS) as executor: - futures = [executor.submit(handleAnim, filename, estimate) \ - for filename in os.listdir("files") if filename.endswith('json')] - - for future in as_completed(futures): - try: - totalCost = future.result() - - except Exception as e: - tracebackLineNo = str(traceback.extract_tb(sys.exc_info()[2])[-1].lineno) - tqdm.write(Fore.RED + str(e) + '|' + tracebackLineNo + Fore.RESET) - - case _: - version = '' - if totalCost != 'Fail': if estimate is False: # This is to encourage people to grab what's in /translated instead deleteFolderFiles('files') - # Prevent immediately closing of CLI - tqdm.write(totalCost) - # input('Done! Press Enter to close.') + tqdm.write(str(totalCost)) def deleteFolderFiles(folderPath): for filename in os.listdir(folderPath): diff --git a/modules/rpgmakermvmz.py b/modules/rpgmakermvmz.py index 3fe1e9e..8b5c9bd 100644 --- a/modules/rpgmakermvmz.py +++ b/modules/rpgmakermvmz.py @@ -41,10 +41,12 @@ if 'gpt-3.5' in MODEL: INPUTAPICOST = .002 OUTPUTAPICOST = .002 BATCHSIZE = 10 + FREQUENCY_PENALTY = 0.2 elif 'gpt-4' in MODEL: INPUTAPICOST = .01 OUTPUTAPICOST = .03 BATCHSIZE = 40 + FREQUENCY_PENALTY = 0 #tqdm Globals BAR_FORMAT='{l_bar}{bar:10}{r_bar}{bar:-10b}' @@ -52,11 +54,11 @@ POSITION = 0 LEAVE = False # Dialogue / Scroll -CODE401 = False +CODE401 = True CODE405 = False # Choices -CODE102 = False +CODE102 = True # Variables CODE122 = False @@ -72,7 +74,7 @@ CODE356 = False CODE320 = False CODE324 = False CODE111 = False -CODE108 = True +CODE108 = False CODE408 = False def handleMVMZ(filename, estimate): @@ -1514,7 +1516,8 @@ def searchCodes(page, pbar, fillList, filename): if len(fillList) != len(docList): global MISMATCH with LOCK: - MISMATCH.append(filename) + if filename not in MISMATCH: + MISMATCH.append(filename) else: docList = [] searchCodes(page, pbar, fillList, filename) @@ -1840,12 +1843,22 @@ def batchList(input_list, batch_size): return [input_list[i:i + batch_size] for i in range(0, len(input_list), batch_size)] def createContext(fullPromptFlag, subbedT): - characters = 'Game Characters:\ - 篠崎 誠一 == Shinozaki Seiichi - Male\ - 宮前 遥奈 == Miyamae Haruna - Female\ - 榛名 悠真 == Haruna Yuuma - Male\ - 浪川 時宗 == Namikawa Tokimune - Male\ - 高嶋 美雪 == Takashima Miyuki - Female' + characters = 'Game Characters (Format: Last Name, First Name - Gender):\ + 護 == Mamoru - Male\ + 神代 一騎 == Kamishiro, Ikki - Male\ + 神代 琴音 == Kamishiro, Kotone - Female\ + 神代 莉々子 == Kamishiro, Ririko - Female\ + 神代 紗夜 == Kamishiro, Saya - Female\ + 篠原漣 == Shinohara, Ren - Male\ + 藪井 == Yabui - Male\ + 舟木 == Funaki - Male\ + 貞二 == Jouji - Male\ + 兼田 響子 == Kaneda, Kyouko - Female\ + 兼田 真人 == Kaneda, Masato - Male\ + 小出 == Koide - Male\ + 進士 == Shinji - Male\ + 雪乃 == Yukino - Male' + system = PROMPT if fullPromptFlag else \ f'Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`' user = f'{subbedT}' @@ -1853,26 +1866,23 @@ def createContext(fullPromptFlag, subbedT): def translateText(characters, system, user, history): # Prompt - msg = [{"role": "system", "content": system}] - - # Characters - msg.append({"role": "user", "content": characters}) + msg = [{"role": "system", "content": system + characters}] # History if isinstance(history, list): - msg.extend([{"role": "user", "content": h} for h in history]) + msg.extend([{"role": "assistant", "content": h} for h in history]) else: - msg.append({"role": "user", "content": history}) + msg.append({"role": "assistant", "content": history}) # Content to TL - msg.append({"role": "user", "content": user}) - response = openai.ChatCompletion.create( - temperature=0, - frequency_penalty=0, - presence_penalty=0, + msg.append({"role": "user", "content": f'{user}'}) + response = openai.chat.completions.create( + temperature=0.1, + top_p = 0.2, + frequency_penalty=FREQUENCY_PENALTY, + presence_penalty=FREQUENCY_PENALTY, model=MODEL, messages=msg, - request_timeout=TIMEOUT, ) return response @@ -1890,7 +1900,7 @@ def cleanTranslatedText(translatedText, varResponse): return [line for line in translatedText.split('\n') if line] def extractTranslation(translatedTextList, is_list): - pattern = r'(.*)' + pattern = r'?(.*)' # If it's a batch (i.e., list), extract with tags; otherwise, return the single item. if is_list: return [re.findall(pattern, line)[0][1] for line in translatedTextList if re.search(pattern, line)] @@ -1966,6 +1976,8 @@ def translateGPT(text, history, fullPromptFlag): if isinstance(tItem, list): extractedTranslations = extractTranslation(translatedTextList, True) tList[index] = extractedTranslations + if len(tList[index]) != len(translatedTextList): + print('Test') history = extractedTranslations[-10:] # Update history if we have a list else: # Ensure we're passing a single string to extractTranslation diff --git a/prompt.example b/prompt.example index 563ccdd..5b86fc0 100644 --- a/prompt.example +++ b/prompt.example @@ -5,30 +5,27 @@ I will give you lines of text, and you must translate each line to the best of y Use the following step-by-step instructions to respond to user inputs. Step 1 - Receive Text -You will be given multiple lines of text (Denoted by XML tags). Translate each line separately and avoid combining or omitting any. +You will be given multiple lines of text (Denoted by XML). Translate each line separately and avoid combining or omitting any. Step 2 - Output Text -You output only the English translation of each line. For example: -English Translation of Line 0 -English Translation of Line 1 -English Translation of Line 2 +You output the English translation of each line and include the same XML tags. These XML tags are not optional, you must always include them. -Step 3 - Check Work -Double check that the number of lines/xml-tags in your response match the number of lines/xml-tags from the user message. +For example: +English Translation\nEnglish Translation\nEnglish Translation -Other Notes: +Notes: - "Game Characters" - The names, nicknames, and genders of the game characters. Reference this to know the names, nicknames, and gender of characters in the game. +- Always include both the start and end xml tag in your translation. - Only reply with the English Translation even if it may be hard to translate. -- If a line is already translated, empty, or can't be translated, do not change it. -- Denote speakers with ':' if given. For example L1 - Speaker: "Spoken Text" +- If a line is already in English, empty, or can't be translated, leave it in English and include it in your response. +- Each line must have the translation of the original text inside. +- Denote speakers with ':' if given. For example Speaker: "Spoken Text" - If the speaker is '???' then they are unknown so leave it as is. - Pay attention to the gender of the subjects and characters. Avoid misgendering characters. - Maintain any spacing in the translation. - Maintain any code in brackets if given. -- Maintain the speaker if given. - Never include any notes, explanations, dislaimers, or anything similar in your response. - Maintain Japanese Honorifics. For example: 'サクラねえちゃん' == 'Sakura Onee-san' -- "--" is intentional and indicates a blank in the text. Leave it as is. - Translate 'おまんこ' as 'pussy'. - Translate 'お尻' as 'butt' - Translate '尻' as 'ass'