From 178a09367fee67c435e14a8e6e84d39f303c0c4a Mon Sep 17 00:00:00 2001 From: Dazed Date: Sun, 26 Nov 2023 08:14:24 -0600 Subject: [PATCH] Remove outdated alltext.py --- modules/alltext.py | 172 --------------------------------------------- 1 file changed, 172 deletions(-) delete mode 100644 modules/alltext.py diff --git a/modules/alltext.py b/modules/alltext.py deleted file mode 100644 index a26c921..0000000 --- a/modules/alltext.py +++ /dev/null @@ -1,172 +0,0 @@ -from concurrent.futures import ThreadPoolExecutor, as_completed -import json -import os -from pathlib import Path -import re -import sys -import textwrap -import threading -import time -import traceback -import tiktoken - -from colorama import Fore -from dotenv import load_dotenv -import openai -from retry import retry -from tqdm import tqdm - -# Globals -load_dotenv() -if os.getenv('api').replace(' ', '') != '': - openai.api_base = os.getenv('api') - -openai.organization = os.getenv('org') -openai.api_key = os.getenv('key') -MODEL = os.getenv('model') -TIMEOUT = int(os.getenv('timeout')) -LANGUAGE = os.getenv('language').capitalize() - -APICOST = .002 # Depends on the model https://openai.com/pricing -PROMPT = Path('prompt.txt').read_text(encoding='utf-8') -THREADS = int(os.getenv('threads')) # For GPT4 rate limit will be hit if you have more than 1 thread. -LOCK = threading.Lock() -WIDTH = int(os.getenv('width')) -LISTWIDTH = int(os.getenv('listWidth')) -MAXHISTORY = 10 -ESTIMATE = '' -TOTALCOST = 0 -TOKENS = 0 -TOTALTOKENS = 0 -NAMESLIST = [] - -# tqdm Globals -BAR_FORMAT = '{l_bar}{bar:10}{r_bar}{bar:-10b}' -POSITION = 0 -LEAVE = False - -def handleAllText(filename, estimate): - global ESTIMATE, TOKENS, TOTALTOKENS, TOTALCOST - ESTIMATE = estimate - - with open('translated/' + filename, 'w+t', newline='', encoding='utf-16-le') as writeFile: - start = time.time() - translatedData = openFiles(filename, writeFile) - - if estimate: - # Print Result - end = time.time() - tqdm.write(getResultString(['', TOKENS, None], end - start, filename)) - TOTALCOST += TOKENS * .001 * APICOST - TOTALTOKENS += TOKENS - TOKENS = 0 - os.remove('translated/' + filename) - - else: - # Print Result - end = time.time() - tqdm.write(getResultString(translatedData, end - start, filename)) - TOTALCOST += translatedData[1] * .001 * APICOST - TOTALTOKENS += translatedData[1] - - return getResultString(['', TOTALTOKENS, None], end - start, 'TOTAL') - -def openFiles(filename, writeFile): - with open('files/' + filename, 'r', encoding='utf-16-le') as readFile, writeFile: - translatedData = parseText(readFile, writeFile, filename) - - return translatedData - -def getResultString(translatedData, translationTime, filename): - # File Print String - tokenString = Fore.YELLOW + '[' + str(translatedData[1]) + \ - ' Tokens/${:,.4f}'.format(translatedData[1] * .001 * APICOST) + ']' - timeString = Fore.BLUE + '[' + str(round(translationTime, 1)) + 's]' - - if translatedData[2] == None: - # Success - return filename + ': ' + tokenString + timeString + Fore.GREEN + u' \u2713 ' + Fore.RESET - - else: - # Fail - try: - raise translatedData[2] - except Exception as e: - errorString = str(e) + '|' + translatedData[3] + Fore.RED - return filename + ': ' + tokenString + timeString + Fore.RED + u' \u2717 ' + \ - errorString + Fore.RESET - -@retry(exceptions=Exception, tries=5, delay=5) -def translateGPT(t, history, fullPromptFlag): - # If ESTIMATE is True just count this as an execution and return. - if ESTIMATE: - enc = tiktoken.encoding_for_model(MODEL) - tokens = len(enc.encode(t)) * 2 + len(enc.encode(str(history))) + len(enc.encode(PROMPT)) - return (t, tokens) - - # Sub Vars - varResponse = subVars(t) - subbedT = varResponse[0] - - # If there isn't any Japanese in the text just skip - if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴ]+|[\uFF00-\uFFEF]', subbedT): - return (t, 0) - - # Characters - context = '```\ - Character: 池ノ上 拓海 == Ikenoue Takumi - Gender: Male\ - Character: 福永 こはる == Fukunaga Koharu - Gender: Female\ - Character: 神泉 理央 == Kamiizumi Rio - Gender: Female\ - Character: 吉祥寺 アリサ == Kisshouji Arisa - Gender: Female\ - Character: 久我 友里子 == Kuga Yuriko - Gender: Female\ - ```' - - # Prompt - if fullPromptFlag: - system = PROMPT - user = 'Line to Translate = ' + subbedT - else: - system = 'Output ONLY the ' + LANGUAGE + ' translation in the following format: `Translation: <' + LANGUAGE.upper() + '_TRANSLATION>`' - user = 'Line to Translate = ' + subbedT - - # Create Message List - msg = [] - msg.append({"role": "system", "content": system}) - msg.append({"role": "user", "content": context}) - if isinstance(history, list): - for line in history: - msg.append({"role": "user", "content": line}) - else: - msg.append({"role": "user", "content": history}) - msg.append({"role": "user", "content": user}) - - response = openai.ChatCompletion.create( - temperature=0.1, - frequency_penalty=0.2, - presence_penalty=0.2, - model=MODEL, - messages=msg, - request_timeout=TIMEOUT, - ) - - # Save Translated Text - translatedText = response.choices[0].message.content - tokens = response.usage.total_tokens - - # Resub Vars - translatedText = resubVars(translatedText, varResponse[1]) - - # Remove Placeholder Text - translatedText = translatedText.replace(LANGUAGE + ' Translation: ', '') - translatedText = translatedText.replace('Translation: ', '') - translatedText = translatedText.replace('Line to Translate = ', '') - translatedText = translatedText.replace(LANGUAGE + ' Translation = ', '') - translatedText = translatedText.replace('Translate = ', '') - translatedText = re.sub(r'Note:.*', '', translatedText) - translatedText = translatedText.replace('っ', '') - - # Return Translation - if len(translatedText) > 15 * len(t) or "I'm sorry, but I'm unable to assist with that translation" in translatedText: - raise Exception - else: - return [translatedText, tokens]