# Libraries import json import os import re import util.dazedwrap as dazedwrap import threading import time import traceback import tiktoken import openai from pathlib import Path from colorama import Fore from dotenv import load_dotenv from retry import retry from tqdm import tqdm from util.translation import TranslationConfig, translateAI as sharedtranslateAI # Open AI load_dotenv() if os.getenv("api").replace(" ", "") != "": openai.base_url = os.getenv("api") openai.organization = os.getenv("org") openai.api_key = os.getenv("key") # Globals MODEL = os.getenv("model") TIMEOUT = int(os.getenv("timeout")) LANGUAGE = os.getenv("language").capitalize() PROMPT = Path("prompt.txt").read_text(encoding="utf-8") VOCAB = Path("vocab.txt").read_text(encoding="utf-8") THREADS = int(os.getenv("threads")) LOCK = threading.Lock() WIDTH = int(os.getenv("width")) LISTWIDTH = int(os.getenv("listWidth")) NOTEWIDTH = 70 MAXHISTORY = 10 ESTIMATE = "" TOKENS = [0, 0] NAMESLIST = [] NAMES = False # Output a list of all the character names found BRFLAG = False # If the game uses
instead FIXTEXTWRAP = True # Overwrites textwrap IGNORETLTEXT = False # Ignores all translated text. MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong) FILENAME = None TIMETOTAL = 0 # Total Time Taken for all translations TIMETOTAL = 0 # Total Time Taken for all translations # tqdm Globals BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}" POSITION = 0 LEAVE = False PBAR = None # Regex - Need to change this if you want to translate from/to other languages. Default is Japanese Regex LANGREGEX = r"[一-龠ぁ-ゔァ-ヴーa-zA-Z0-9\uFF61-\uFF9F]+" # Pricing - Depends on the model https://openai.com/pricing # Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request # If you are getting a MISMATCH LENGTH error, lower the batch size. if "gpt-3.5" in MODEL: INPUTAPICOST = 3.00 OUTPUTAPICOST = 5.00 BATCHSIZE = 10 FREQUENCY_PENALTY = 0.2 elif "gpt-5" in MODEL: INPUTAPICOST = 1.25 OUTPUTAPICOST = 10.00 BATCHSIZE = 30 FREQUENCY_PENALTY = 0.05 elif "deepseek" in MODEL: INPUTAPICOST = 0.27 OUTPUTAPICOST = 1.10 BATCHSIZE = 30 FREQUENCY_PENALTY = 0.05 else: INPUTAPICOST = float(os.getenv("input_cost")) OUTPUTAPICOST = float(os.getenv("output_cost")) BATCHSIZE = int(os.getenv("batchsize")) FREQUENCY_PENALTY = float(os.getenv("frequency_penalty")) # Initialize Translation Config TRANSLATION_CONFIG = TranslationConfig( model=MODEL, language=LANGUAGE, prompt=PROMPT, vocab=VOCAB, langRegex=LANGREGEX, batchSize=BATCHSIZE, maxHistory=MAXHISTORY, estimateMode=False # Will be set dynamically based on ESTIMATE ) LEAVE = False def handleRegex(filename, estimate): global ESTIMATE, TOKENS, FILENAME ESTIMATE = estimate FILENAME = filename # Translate start = time.time() translatedData = openFiles(filename) # Translate if not estimate: try: with open("translated/" + filename, "w", encoding="utf-8-sig") as outFile: outFile.writelines(translatedData[0]) except Exception: traceback.print_exc() return "Fail" # Print File end = time.time() tqdm.write(getResultString(translatedData, end - start, filename)) with LOCK: TOKENS[0] += translatedData[1][0] TOKENS[1] += translatedData[1][1] # Print Total totalString = getResultString(["", TOKENS, None], end - start, "TOTAL") # Print any errors on maps if len(MISMATCH) > 0: return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET else: return totalString def getResultString(translatedData, translationTime, filename): global TIMETOTAL # File Print String totalTokenstring = ( Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]" "[Output: " + str(translatedData[1][1]) + "]" "[Cost: ${:,.4f}".format(((translatedData[1][0] / 1000000) * INPUTAPICOST) + ((translatedData[1][1] / 1000000) * OUTPUTAPICOST)) + "]" ) if filename != "TOTAL": timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]" TIMETOTAL += round(translationTime, 1) else: timeString = Fore.BLUE + "[" + str(round(TIMETOTAL, 1)) + "s]" if translatedData[2] == None: # Success return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET else: # Fail try: raise translatedData[2] except Exception as e: traceback.print_exc() errorString = str(e) + Fore.RED return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET def openFiles(filename): with open("files/" + filename, "r", encoding="utf-8-sig") as readFile: translatedData = parseRegex(readFile, filename) # Delete lines marked for deletion finalData = [] for line in translatedData[0]: if line != "\\d\n": finalData.append(line) translatedData[0] = finalData return translatedData def parseRegex(readFile, filename): global PBAR totalTokens = [0, 0] # Read File into data data = readFile.readlines() # Create Progress Bar with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar: pbar.desc = filename PBAR = pbar try: result = translateRegex(data, []) totalTokens[0] += result[0] totalTokens[1] += result[1] except Exception as e: traceback.print_exc() return [data, totalTokens, e] return [data, totalTokens, None] def translateRegex(data, translatedList): if translatedList: stringList = translatedList[0] choiceList = translatedList[1] else: stringList = [] choiceList = [] tokens = [0, 0] speaker = "" global LOCK, ESTIMATE, FILENAME, PBAR, MISMATCH i = 0 while i < len(data): voice = False lineRegexText = r'"profile": "(.*)"' lineRegexSpeaker = r"◆A.+?◆(.+)" choiceRegex = r"\$menu_item.+?,(.*?)," titleRegex = r"title\s'(.*)'$" speaker = "" # Title match = re.search(titleRegex, data[i]) if match: response = translateAI( match.group(1), f"Reply with the {LANGUAGE} translation of the chapter title", True, ) tokens[0] += response[1][0] tokens[1] += response[1][1] title = response[0] # Set if not translatedList: title = re.sub(r"(?", ")") # Textwrap translatedText = dazedwrap.wrapText(translatedText, width=WIDTH).replace("\n", "\\n") # Set Data if "「" in data[i-1] and "」" not in translatedText: data[i] = data[i].replace(originalString, f"「{translatedText}」") else: data[i] = data[i].replace(originalString, f"{translatedText}") # Choices match = re.search(choiceRegex, data[i]) if match: # Pass 1 if not translatedList: choiceList.append(match.group(1)) # Pass 2 else: # Grab and Pop translatedText = choiceList[0] choiceList.pop(0) # Replace Spaces translatedText = translatedText.replace("\u3000", " ") # Escape Quotes translatedText = re.sub(r'(?