From e4b4ae27854eed5a303cb66e2723cd39654945e9 Mon Sep 17 00:00:00 2001 From: Dazed Date: Sun, 23 Apr 2023 07:53:51 -0500 Subject: [PATCH] refactor: Moving a couple things around --- main.py | 70 -------------------- src/csv.py | 69 ++++++++++++++++++++ src/main.py | 90 ++++++++++++++++++++++++++ rpgmakerace.py => src/rpgmakerace.py | 0 rpgmakermvmz.py => src/rpgmakermvmz.py | 0 5 files changed, 159 insertions(+), 70 deletions(-) delete mode 100644 main.py create mode 100644 src/csv.py create mode 100644 src/main.py rename rpgmakerace.py => src/rpgmakerace.py (100%) rename rpgmakermvmz.py => src/rpgmakermvmz.py (100%) diff --git a/main.py b/main.py deleted file mode 100644 index 7b5a11f..0000000 --- a/main.py +++ /dev/null @@ -1,70 +0,0 @@ -from concurrent.futures import ThreadPoolExecutor, as_completed -import sys -import traceback -from colorama import Fore -import os - -from rpgmakermvmz import handleMVMZ -from rpgmakerace import handleACE - -THREADS = 20 - -# Info Message -print(Fore.LIGHTYELLOW_EX + "Do not close while translation is in progress. If a file fails or gets stuck, \ -Translated lines will remain translated so you don't have to worry about being charged \ -twice. You can simply copy the file generated in /translations back over to /files and \ -start the script again. It will skip over any translated text." + Fore.RESET, end='\n\n') - -def main(): - estimate = '' - while estimate == '': - estimate = input('Select Translation or Cost Estimation:\n\n1. Translate\n2. Estimate\n') - match estimate: - case '1': estimate = False - case '2': estimate = True - case _: estimate = '' - - version = input('Select the RPGMaker Version:\n\n1. MV/MZ\n2. ACE\n') - - totalCost = 0 - match version: - case '1': - # Open File (Threads) - with ThreadPoolExecutor(max_workers=THREADS) as executor: - futures = [executor.submit(handleMVMZ, filename, estimate) \ - for filename in os.listdir("files") if filename.endswith('json')] - - for future in as_completed(futures): - try: - totalCost = future.result() - except Exception as e: - tracebackLineNo = str(traceback.extract_tb(sys.exc_info()[2])[-1].lineno) - print(Fore.RED + str(e) + '|' + tracebackLineNo + Fore.RESET) - - case '2': - # Open File (Threads) - with ThreadPoolExecutor(max_workers=THREADS) as executor: - for filename in os.listdir("files"): - if filename.endswith('json'): - future = executor.submit(handleACE, filename, estimate) - - try: - totalCost = future.result() - except Exception as e: - print(Fore.RED + str(e)) - - if estimate == False: - # This is to encourage people to grab what's in /translated instead - deleteFolderFiles('files') - - # Prevent immediately closing of CLI - print(totalCost) - input('Done! Press Enter to close.') - -def deleteFolderFiles(folderPath): - for filename in os.listdir(folderPath): - file_path = os.path.join(folderPath, filename) - if file_path.endswith('.json'): - os.remove(file_path) - -main() diff --git a/src/csv.py b/src/csv.py new file mode 100644 index 0000000..d63f7c3 --- /dev/null +++ b/src/csv.py @@ -0,0 +1,69 @@ +from concurrent.futures import ThreadPoolExecutor, as_completed +import json +import os +from pathlib import Path +import re +import sys +import textwrap +import threading +import time +import traceback +import tiktoken + +from colorama import Fore +from dotenv import load_dotenv +import openai +from retry import retry +from tqdm import tqdm + +#Globals +load_dotenv() +openai.organization = os.getenv('org') +openai.api_key = os.getenv('key') +APICOST = .002 # Depends on the model https://openai.com/pricing + +PROMPT = Path('prompt.txt').read_text(encoding='utf-8') +THREADS = 20 +LOCK = threading.Lock() +WIDTH = 60 +MAXHISTORY = 10 +ESTIMATE = '' +TOTALCOST = 0 +TOKENS = 0 +TOTALTOKENS = 0 + +#tqdm Globals +BAR_FORMAT='{l_bar}{bar:10}{r_bar}{bar:-10b}' +POSITION=0 +LEAVE=False + +def handleCSV(filename, estimate): + global ESTIMATE, TOKENS, TOTALTOKENS, TOTALCOST + ESTIMATE = estimate + + if estimate: + start = time.time() + translatedData = openFiles(filename) + + # Print Result + end = time.time() + tqdm.write(getResultString(['', TOKENS, None], end - start, filename)) + TOTALCOST += TOKENS * .001 * APICOST + TOTALTOKENS += TOKENS + TOKENS = 0 + + return getResultString(['', TOTALTOKENS, None], end - start, 'TOTAL') + + else: + with open('translated/' + filename, 'w', encoding='UTF-8') as outFile: + start = time.time() + translatedData = openFiles(filename) + + # Print Result + end = time.time() + json.dump(translatedData[0], outFile, ensure_ascii=False) + tqdm.write(getResultString(translatedData, end - start, filename)) + TOTALCOST += translatedData[1] * .001 * APICOST + TOTALTOKENS += translatedData[1] + + return getResultString(['', TOTALTOKENS, None], end - start, 'TOTAL') \ No newline at end of file diff --git a/src/main.py b/src/main.py new file mode 100644 index 0000000..358eae5 --- /dev/null +++ b/src/main.py @@ -0,0 +1,90 @@ +from concurrent.futures import ThreadPoolExecutor, as_completed +import sys +import traceback +from colorama import Fore +import os + +from rpgmakermvmz import handleMVMZ +from rpgmakerace import handleACE +from csv import handleCSV + +THREADS = 20 + +# Info Message +print(Fore.LIGHTYELLOW_EX + "Do not close while translation is in progress. If a file fails or gets stuck, \ +Translated lines will remain translated so you don't have to worry about being charged \ +twice. You can simply copy the file generated in /translations back over to /files and \ +start the script again. It will skip over any translated text." + Fore.RESET, end='\n\n') + +def main(): + estimate = '' + while estimate == '': + estimate = input('Select Translation or Cost Estimation:\n\n1. Translate\n2. Estimate\n') + match estimate: + case '1': estimate = False + case '2': estimate = True + case _: estimate = '' + + version = input('Select the RPGMaker Version:\n\n1. MV/MZ\n2. ACE\n') + + totalCost = 0 + version = '' + while version == '': + match version: + case '1': + # Open File (Threads) + with ThreadPoolExecutor(max_workers=THREADS) as executor: + futures = [executor.submit(handleMVMZ, filename, estimate) \ + for filename in os.listdir("files") if filename.endswith('json')] + + for future in as_completed(futures): + try: + totalCost = future.result() + except Exception as e: + tracebackLineNo = str(traceback.extract_tb(sys.exc_info()[2])[-1].lineno) + print(Fore.RED + str(e) + '|' + tracebackLineNo + Fore.RESET) + + case '2': + # Open File (Threads) + with ThreadPoolExecutor(max_workers=THREADS) as executor: + futures = [executor.submit(handleACE, filename, estimate) \ + for filename in os.listdir("files") if filename.endswith('json')] + + for future in as_completed(futures): + try: + totalCost = future.result() + except Exception as e: + tracebackLineNo = str(traceback.extract_tb(sys.exc_info()[2])[-1].lineno) + print(Fore.RED + str(e) + '|' + tracebackLineNo + Fore.RESET) + + case '3': + # Open File (Threads) + with ThreadPoolExecutor(max_workers=THREADS) as executor: + futures = [executor.submit(handleCSV, filename, estimate) \ + for filename in os.listdir("files") if filename.endswith('json')] + + for future in as_completed(futures): + try: + totalCost = future.result() + except Exception as e: + tracebackLineNo = str(traceback.extract_tb(sys.exc_info()[2])[-1].lineno) + print(Fore.RED + str(e) + '|' + tracebackLineNo + Fore.RESET) + + case _: + version = '' + + if estimate == False: + # This is to encourage people to grab what's in /translated instead + deleteFolderFiles('files') + + # Prevent immediately closing of CLI + print(totalCost) + input('Done! Press Enter to close.') + +def deleteFolderFiles(folderPath): + for filename in os.listdir(folderPath): + file_path = os.path.join(folderPath, filename) + if file_path.endswith('.json'): + os.remove(file_path) + +main() diff --git a/rpgmakerace.py b/src/rpgmakerace.py similarity index 100% rename from rpgmakerace.py rename to src/rpgmakerace.py diff --git a/rpgmakermvmz.py b/src/rpgmakermvmz.py similarity index 100% rename from rpgmakermvmz.py rename to src/rpgmakermvmz.py