From 0f724dc6cd7ce4964dfba53444f5796dd46aeceb Mon Sep 17 00:00:00 2001 From: Dazed Date: Sat, 22 Apr 2023 13:20:30 -0500 Subject: [PATCH] feat: Estimate Cost feature implemented --- main.py | 21 +++--- rpgmakerace.py | 172 ++++++++++++++++++++++++------------------------ rpgmakermvmz.py | 148 +++++++++++++++++++++-------------------- 3 files changed, 174 insertions(+), 167 deletions(-) diff --git a/main.py b/main.py index 869728e..7b5a11f 100644 --- a/main.py +++ b/main.py @@ -1,4 +1,6 @@ -from concurrent.futures import ThreadPoolExecutor +from concurrent.futures import ThreadPoolExecutor, as_completed +import sys +import traceback from colorama import Fore import os @@ -29,14 +31,15 @@ def main(): case '1': # Open File (Threads) with ThreadPoolExecutor(max_workers=THREADS) as executor: - for filename in os.listdir("files"): - if filename.endswith('json'): - future = executor.submit(handleMVMZ, filename, estimate) - - try: - totalCost = future.result() - except Exception as e: - print(Fore.RED + str(e)) + futures = [executor.submit(handleMVMZ, filename, estimate) \ + for filename in os.listdir("files") if filename.endswith('json')] + + for future in as_completed(futures): + try: + totalCost = future.result() + except Exception as e: + tracebackLineNo = str(traceback.extract_tb(sys.exc_info()[2])[-1].lineno) + print(Fore.RED + str(e) + '|' + tracebackLineNo + Fore.RESET) case '2': # Open File (Threads) diff --git a/rpgmakerace.py b/rpgmakerace.py index b7f6142..ac4e7e2 100644 --- a/rpgmakerace.py +++ b/rpgmakerace.py @@ -8,6 +8,7 @@ import textwrap import threading import time import traceback +import tiktoken from colorama import Fore from dotenv import load_dotenv @@ -27,8 +28,8 @@ LOCK = threading.Lock() WIDTH = 60 MAXHISTORY = 10 ESTIMATE = '' -CHARACTERS = 0 TOTALCOST = 0 +TOKENS = 0 TOTALTOKENS = 0 #tqdm Globals @@ -38,97 +39,93 @@ LEAVE=False # Flags CODE401 = True -CODE102 = True +CODE102 = False CODE122 = False CODE101 = False CODE355655 = False CODE357 = False def handleACE(filename, estimate): - global ESTIMATE + global ESTIMATE, TOKENS, TOTALTOKENS, TOTALCOST ESTIMATE = estimate - totalStart = time.time() - with open('translated/' + filename, 'w', encoding='UTF-8') as outFile: - with open('files/' + filename, 'r', encoding='UTF-8') as f: - data = json.load(f) - - # Map Files - if 'Map' in filename and filename != 'MapInfos.json': - start = time.time() - translatedData = parseMap(data, filename) - - # CommonEvents Files - if 'CommonEvents' in filename: - start = time.time() - translatedData = parseCommonEvents(data, filename) - - # Actor File - if 'Actors' in filename: - start = time.time() - translatedData = parseNames(data, filename, 'Actors') - - # Armor File - if 'Actors' in filename: - start = time.time() - translatedData = parseThings(data, filename, 'Armor') - - # Classes File - if 'Classes' in filename: - start = time.time() - translatedData = parseNames(data, filename, 'Classes') - - # Items File - if 'Items' in filename: - start = time.time() - translatedData = parseThings(data, filename, 'Items') - - # MapInfo File - if 'MapInfos' in filename: - start = time.time() - translatedData = parseNames(data, filename, 'MapInfos') - - # Skills File - if 'Skills' in filename: - start = time.time() - translatedData = parseSS(data, filename) - - # States File - if 'States' in filename: - start = time.time() - translatedData = parseSS(data, filename) - - # System File - if 'System' in filename: - start = time.time() - translatedData = parseSystem(data, filename) - - end = time.time() - json.dump(translatedData[0], outFile, ensure_ascii=False) + if estimate: + start = time.time() + translatedData = openFiles(filename) # Print Result - if estimate: - global CHARACTERS - print('CHARACTERS Total: ' + str(CHARACTERS)) - printString(['', round(CHARACTERS/.325, 1), None], end - start, f.name) - - # Reset CHARACTERS* - CHARACTERS = 0 - else: - printString(translatedData, end - start, f.name) + end = time.time() + tqdm.write(getResultString(['', TOKENS, None], end - start, filename)) + TOTALCOST += TOKENS * .001 * APICOST + TOTALTOKENS += TOKENS + TOKENS = 0 + + return getResultString(['', TOTALTOKENS, None], end - start, 'TOTAL') - # Final Output - totalEnd = time.time() - printString(['', round(TOTALTOKENS, 1), None], totalEnd - totalStart, 'TOTAL') + else: + with open('translated/' + filename, 'w', encoding='UTF-8') as outFile: + start = time.time() + translatedData = openFiles(filename) -def printString(translatedData, translationTime, filename): - global TOTALCOST, TOTALTOKENS + # Print Result + end = time.time() + json.dump(translatedData[0], outFile, ensure_ascii=False) + tqdm.write(getResultString(translatedData, end - start, filename)) + TOTALCOST += translatedData[1] * .001 * APICOST + TOTALTOKENS += translatedData[1] - # Cost Estimation - cost = translatedData[1] * .001 * APICOST - TOTALCOST += cost - TOTALTOKENS += translatedData[1] + return getResultString(['', TOTALTOKENS, None], end - start, 'TOTAL') +def openFiles(filename): + with open('files/' + filename, 'r', encoding='UTF-8') as f: + data = json.load(f) + + # Map Files + if 'Map' in filename and filename != 'MapInfos.json': + translatedData = parseMap(data, filename) + + # CommonEvents Files + elif 'CommonEvents' in filename: + translatedData = parseCommonEvents(data, filename) + + # Actor File + elif 'Actors' in filename: + translatedData = parseNames(data, filename, 'Actors') + + # Armor File + elif 'Actors' in filename: + translatedData = parseThings(data, filename, 'Armor') + + # Classes File + elif 'Classes' in filename: + translatedData = parseNames(data, filename, 'Classes') + + # Items File + elif 'Items' in filename: + translatedData = parseThings(data, filename, 'Items') + + # MapInfo File + elif 'MapInfos' in filename: + translatedData = parseNames(data, filename, 'MapInfos') + + # Skills File + elif 'Skills' in filename: + translatedData = parseSS(data, filename) + + # States File + elif 'States' in filename: + translatedData = parseSS(data, filename) + + # System File + elif 'System' in filename: + translatedData = parseSystem(data, filename) + + else: + raise NameError(filename + ' Not Supported') + + return translatedData + +def getResultString(translatedData, translationTime, filename): # File Print String tokenString = Fore.YELLOW + '[' + str(translatedData[1]) + \ ' Tokens/${:,.4f}'.format(translatedData[1] * .001 * APICOST) + ']' @@ -136,15 +133,16 @@ def printString(translatedData, translationTime, filename): if translatedData[2] == None: # Success - tqdm.write(filename + ': ' + tokenString + timeString + Fore.GREEN + u' \u2713 ' + Fore.RESET) + return filename + ': ' + tokenString + timeString + Fore.GREEN + u' \u2713 ' + Fore.RESET + else: # Fail try: raise translatedData[2] except Exception as e: errorString = str(e) + Fore.RED - tqdm.write(filename + ': ' + tokenString + timeString + Fore.RED + u' \u2717 ' +\ - errorString + Fore.RESET) + return filename + ': ' + tokenString + timeString + Fore.RED + u' \u2717 ' +\ + errorString + Fore.RESET def parseMap(data, filename): totalTokens = 0 @@ -670,11 +668,13 @@ def searchSystem(data, pbar): @retry(exceptions=Exception, tries=5, delay=5) def translateGPT(t, history): - # If ESTIMATE is True just count this as an execution and return. - if ESTIMATE: - global CHARACTERS - CHARACTERS += len(t) + len(history) - return (t, 0) + with LOCK: + # If ESTIMATE is True just count this as an execution and return. + if ESTIMATE: + global TOKENS + enc = tiktoken.encoding_for_model("gpt-3.5-turbo") + TOKENS += len(enc.encode(t)) * 2 + len(enc.encode(history)) + len(enc.encode(PROMPT)) + return (t, 0) # If there isn't any Japanese in the text just skip if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴ]+', t): diff --git a/rpgmakermvmz.py b/rpgmakermvmz.py index 2b01878..d487cf3 100644 --- a/rpgmakermvmz.py +++ b/rpgmakermvmz.py @@ -39,7 +39,7 @@ LEAVE=False # Flags CODE401 = True -CODE102 = True +CODE102 = False CODE122 = False CODE101 = False CODE355655 = False @@ -49,78 +49,82 @@ def handleMVMZ(filename, estimate): global ESTIMATE, TOKENS, TOTALTOKENS, TOTALCOST ESTIMATE = estimate - with open('translated/' + filename, 'w', encoding='UTF-8') as outFile: - with open('files/' + filename, 'r', encoding='UTF-8') as f: - data = json.load(f) - - # Map Files - if 'Map' in filename and filename != 'MapInfos.json': - start = time.time() - translatedData = parseMap(data, filename) - - # CommonEvents Files - if 'CommonEvents' in filename: - start = time.time() - translatedData = parseCommonEvents(data, filename) - - # Actor File - if 'Actors' in filename: - start = time.time() - translatedData = parseNames(data, filename, 'Actors') - - # Armor File - if 'Actors' in filename: - start = time.time() - translatedData = parseThings(data, filename, 'Armor') - - # Classes File - if 'Classes' in filename: - start = time.time() - translatedData = parseNames(data, filename, 'Classes') - - # Items File - if 'Items' in filename: - start = time.time() - translatedData = parseThings(data, filename, 'Items') - - # MapInfo File - if 'MapInfos' in filename: - start = time.time() - translatedData = parseNames(data, filename, 'MapInfos') - - # Skills File - if 'Skills' in filename: - start = time.time() - translatedData = parseSS(data, filename) - - # States File - if 'States' in filename: - start = time.time() - translatedData = parseSS(data, filename) - - # System File - if 'System' in filename: - start = time.time() - translatedData = parseSystem(data, filename) - - end = time.time() - json.dump(translatedData[0], outFile, ensure_ascii=False) + if estimate: + start = time.time() + translatedData = openFiles(filename) # Print Result - if estimate: - tqdm.write(getResultString(['', TOKENS, None], end - start, f.name)) + end = time.time() + tqdm.write(getResultString(['', TOKENS, None], end - start, filename)) + TOTALCOST += TOKENS * .001 * APICOST + TOTALTOKENS += TOKENS + TOKENS = 0 - TOTALCOST += TOKENS * .001 * APICOST - TOTALTOKENS += TOKENS - TOKENS = 0 - else: - tqdm.write(getResultString(translatedData, end - start, f.name)) + return getResultString(['', TOTALTOKENS, None], end - start, 'TOTAL') + + else: + with open('translated/' + filename, 'w', encoding='UTF-8') as outFile: + start = time.time() + translatedData = openFiles(filename) + # Print Result + end = time.time() + json.dump(translatedData[0], outFile, ensure_ascii=False) + tqdm.write(getResultString(translatedData, end - start, filename)) TOTALCOST += translatedData[1] * .001 * APICOST TOTALTOKENS += translatedData[1] return getResultString(['', TOTALTOKENS, None], end - start, 'TOTAL') +def openFiles(filename): + with open('files/' + filename, 'r', encoding='UTF-8') as f: + data = json.load(f) + + # Map Files + if 'Map' in filename and filename != 'MapInfos.json': + translatedData = parseMap(data, filename) + + # CommonEvents Files + elif 'CommonEvents' in filename: + translatedData = parseCommonEvents(data, filename) + + # Actor File + elif 'Actors' in filename: + translatedData = parseNames(data, filename, 'Actors') + + # Armor File + elif 'Actors' in filename: + translatedData = parseThings(data, filename, 'Armor') + + # Classes File + elif 'Classes' in filename: + translatedData = parseNames(data, filename, 'Classes') + + # Items File + elif 'Items' in filename: + translatedData = parseThings(data, filename, 'Items') + + # MapInfo File + elif 'MapInfos' in filename: + translatedData = parseNames(data, filename, 'MapInfos') + + # Skills File + elif 'Skills' in filename: + translatedData = parseSS(data, filename) + + # States File + elif 'States' in filename: + translatedData = parseSS(data, filename) + + # System File + elif 'System' in filename: + translatedData = parseSystem(data, filename) + + else: + raise NameError(filename + ' Not Supported') + + return translatedData + def getResultString(translatedData, translationTime, filename): # File Print String tokenString = Fore.YELLOW + '[' + str(translatedData[1]) + \ @@ -315,7 +319,6 @@ def searchNames(name, pbar, context): return tokens def searchCodes(page, pbar): - text = '' translatedText = '' currentGroup = [] textHistory = [] @@ -330,7 +333,7 @@ def searchCodes(page, pbar): pbar.update(1) ### All the codes are here which translate specific functions in the MAP files. - ### IF these crash or fail your game will do the same. Just comment out anything not needed. + ### IF these crash or fail your game will do the same. Use the flags to skip codes. ## Event Code: 401 Show Text if page['list'][i]['code'] == 401 and CODE401 == True: @@ -660,12 +663,13 @@ def searchSystem(data, pbar): @retry(exceptions=Exception, tries=5, delay=5) def translateGPT(t, history): - # If ESTIMATE is True just count this as an execution and return. - if ESTIMATE: - global TOKENS - enc = tiktoken.encoding_for_model("gpt-3.5-turbo") - TOKENS += len(enc.encode(t) + enc.encode(history) + enc.encode(PROMPT)) - return (t, 0) + with LOCK: + # If ESTIMATE is True just count this as an execution and return. + if ESTIMATE: + global TOKENS + enc = tiktoken.encoding_for_model("gpt-3.5-turbo") + TOKENS += len(enc.encode(t)) * 2 + len(enc.encode(history)) + len(enc.encode(PROMPT)) + return (t, 0) # If there isn't any Japanese in the text just skip if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴ]+', t):