from concurrent.futures import ThreadPoolExecutor, as_completed import json import os from pathlib import Path import re import sys import textwrap import threading import time import traceback import tiktoken from colorama import Fore from dotenv import load_dotenv import openai from retry import retry from tqdm import tqdm # Globals load_dotenv() if os.getenv('api').replace(' ', '') != '': openai.api_base = os.getenv('api') openai.organization = os.getenv('org') openai.api_key = os.getenv('key') MODEL = os.getenv('model') TIMEOUT = int(os.getenv('timeout')) LANGUAGE = os.getenv('language').capitalize() APICOST = .002 # Depends on the model https://openai.com/pricing PROMPT = Path('prompt.txt').read_text(encoding='utf-8') THREADS = int(os.getenv('threads')) # For GPT4 rate limit will be hit if you have more than 1 thread. LOCK = threading.Lock() WIDTH = int(os.getenv('width')) LISTWIDTH = int(os.getenv('listWidth')) MAXHISTORY = 10 ESTIMATE = '' TOTALCOST = 0 TOKENS = 0 TOTALTOKENS = 0 NAMESLIST = [] # tqdm Globals BAR_FORMAT = '{l_bar}{bar:10}{r_bar}{bar:-10b}' POSITION = 0 LEAVE = False def handleAllText(filename, estimate): global ESTIMATE, TOKENS, TOTALTOKENS, TOTALCOST ESTIMATE = estimate with open('translated/' + filename, 'w+t', newline='', encoding='utf-16-le') as writeFile: start = time.time() translatedData = openFiles(filename, writeFile) if estimate: # Print Result end = time.time() tqdm.write(getResultString(['', TOKENS, None], end - start, filename)) TOTALCOST += TOKENS * .001 * APICOST TOTALTOKENS += TOKENS TOKENS = 0 os.remove('translated/' + filename) else: # Print Result end = time.time() tqdm.write(getResultString(translatedData, end - start, filename)) TOTALCOST += translatedData[1] * .001 * APICOST TOTALTOKENS += translatedData[1] return getResultString(['', TOTALTOKENS, None], end - start, 'TOTAL') def openFiles(filename, writeFile): with open('files/' + filename, 'r', encoding='utf-16-le') as readFile, writeFile: translatedData = parseText(readFile, writeFile, filename) return translatedData def getResultString(translatedData, translationTime, filename): # File Print String tokenString = Fore.YELLOW + '[' + str(translatedData[1]) + \ ' Tokens/${:,.4f}'.format(translatedData[1] * .001 * APICOST) + ']' timeString = Fore.BLUE + '[' + str(round(translationTime, 1)) + 's]' if translatedData[2] == None: # Success return filename + ': ' + tokenString + timeString + Fore.GREEN + u' \u2713 ' + Fore.RESET else: # Fail try: raise translatedData[2] except Exception as e: errorString = str(e) + '|' + translatedData[3] + Fore.RED return filename + ': ' + tokenString + timeString + Fore.RED + u' \u2717 ' + \ errorString + Fore.RESET @retry(exceptions=Exception, tries=5, delay=5) def translateGPT(t, history, fullPromptFlag): # If ESTIMATE is True just count this as an execution and return. if ESTIMATE: enc = tiktoken.encoding_for_model(MODEL) tokens = len(enc.encode(t)) * 2 + len(enc.encode(str(history))) + len(enc.encode(PROMPT)) return (t, tokens) # Sub Vars varResponse = subVars(t) subbedT = varResponse[0] # If there isn't any Japanese in the text just skip if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴ]+|[\uFF00-\uFFEF]', subbedT): return (t, 0) # Characters context = '```\ Character: 池ノ上 拓海 == Ikenoue Takumi - Gender: Male\ Character: 福永 こはる == Fukunaga Koharu - Gender: Female\ Character: 神泉 理央 == Kamiizumi Rio - Gender: Female\ Character: 吉祥寺 アリサ == Kisshouji Arisa - Gender: Female\ Character: 久我 友里子 == Kuga Yuriko - Gender: Female\ ```' # Prompt if fullPromptFlag: system = PROMPT user = 'Line to Translate = ' + subbedT else: system = 'Output ONLY the ' + LANGUAGE + ' translation in the following format: `Translation: <' + LANGUAGE.upper() + '_TRANSLATION>`' user = 'Line to Translate = ' + subbedT # Create Message List msg = [] msg.append({"role": "system", "content": system}) msg.append({"role": "user", "content": context}) if isinstance(history, list): for line in history: msg.append({"role": "user", "content": line}) else: msg.append({"role": "user", "content": history}) msg.append({"role": "user", "content": user}) response = openai.ChatCompletion.create( temperature=0.1, frequency_penalty=0.2, presence_penalty=0.2, model=MODEL, messages=msg, request_timeout=TIMEOUT, ) # Save Translated Text translatedText = response.choices[0].message.content tokens = response.usage.total_tokens # Resub Vars translatedText = resubVars(translatedText, varResponse[1]) # Remove Placeholder Text translatedText = translatedText.replace(LANGUAGE + ' Translation: ', '') translatedText = translatedText.replace('Translation: ', '') translatedText = translatedText.replace('Line to Translate = ', '') translatedText = translatedText.replace(LANGUAGE + ' Translation = ', '') translatedText = translatedText.replace('Translate = ', '') translatedText = re.sub(r'Note:.*', '', translatedText) translatedText = translatedText.replace('っ', '') # Return Translation if len(translatedText) > 15 * len(t) or "I'm sorry, but I'm unable to assist with that translation" in translatedText: raise Exception else: return [translatedText, tokens]