From 5fecfe20842fccfa6e397554552d7348c0c14166 Mon Sep 17 00:00:00 2001 From: Dazed Date: Mon, 25 Sep 2023 20:35:30 -0500 Subject: [PATCH] Fix Estimate and more tyrano changes --- modules/csvtl.py | 12 +++++------ modules/rpgmakerace.py | 22 +++++++++------------ modules/rpgmakermvmz.py | 22 +++++++++------------ modules/tyrano.py | 44 +++++++++++++++++++++-------------------- start.py | 2 +- 5 files changed, 47 insertions(+), 55 deletions(-) diff --git a/modules/csvtl.py b/modules/csvtl.py index c11b411..598cad5 100644 --- a/modules/csvtl.py +++ b/modules/csvtl.py @@ -253,13 +253,11 @@ def resubVars(translatedText, varList): @retry(exceptions=Exception, tries=5, delay=5) def translateGPT(t, history, fullPromptFlag): - with LOCK: - # If ESTIMATE is True just count this as an execution and return. - if ESTIMATE: - global TOKENS - enc = tiktoken.encoding_for_model("gpt-3.5-turbo-0613") - TOKENS += len(enc.encode(t)) * 2 + len(enc.encode(history)) + len(enc.encode(PROMPT)) - return (t, 0) + # If ESTIMATE is True just count this as an execution and return. + if ESTIMATE: + enc = tiktoken.encoding_for_model("gpt-3.5-turbo-0613") + TOKENS = len(enc.encode(t)) * 2 + len(enc.encode(history)) + len(enc.encode(PROMPT)) + return (t, 0) # Sub Vars varResponse = subVars(t) diff --git a/modules/rpgmakerace.py b/modules/rpgmakerace.py index 544f9c4..d72f786 100644 --- a/modules/rpgmakerace.py +++ b/modules/rpgmakerace.py @@ -31,7 +31,6 @@ LISTWIDTH = 70 MAXHISTORY = 10 ESTIMATE = '' TOTALCOST = 0 -TOKENS = 0 TOTALTOKENS = 0 NAMESLIST = [] @@ -58,7 +57,7 @@ CODE108 = False NAMES = False # Output a list of all the character names found def handleACE(filename, estimate): - global ESTIMATE, TOKENS, TOTALTOKENS, TOTALCOST + global ESTIMATE, TOTALTOKENS, TOTALCOST ESTIMATE = estimate if estimate: @@ -67,13 +66,12 @@ def handleACE(filename, estimate): # Print Result end = time.time() - tqdm.write(getResultString(['', TOKENS, None], end - start, filename)) + tqdm.write(getResultString(translatedData, end - start, filename)) if NAMES is True: tqdm.write(str(NAMESLIST)) with LOCK: - TOTALCOST += TOKENS * .001 * APICOST - TOTALTOKENS += TOKENS - TOKENS = 0 + TOTALCOST += translatedData[1] * .001 * APICOST + TOTALTOKENS += translatedData[1] return getResultString(['', TOTALTOKENS, None], end - start, 'TOTAL') @@ -1318,13 +1316,11 @@ def resubVars(translatedText, varList): @retry(exceptions=Exception, tries=5, delay=5) def translateGPT(t, history, fullPromptFlag): - with LOCK: - # If ESTIMATE is True just count this as an execution and return. - if ESTIMATE: - global TOKENS - enc = tiktoken.encoding_for_model("gpt-3.5-turbo") - TOKENS += len(enc.encode(t)) * 2 + len(enc.encode(history)) + len(enc.encode(PROMPT)) - return (t, 0) + # If ESTIMATE is True just count this as an execution and return. + if ESTIMATE: + enc = tiktoken.encoding_for_model("gpt-3.5-turbo") + tokens = len(enc.encode(t)) * 2 + len(enc.encode(history)) + len(enc.encode(PROMPT)) + return (t, tokens) # Sub Vars varResponse = subVars(t) diff --git a/modules/rpgmakermvmz.py b/modules/rpgmakermvmz.py index 3b3eb4b..3ae2991 100644 --- a/modules/rpgmakermvmz.py +++ b/modules/rpgmakermvmz.py @@ -30,7 +30,6 @@ LISTWIDTH = 60 MAXHISTORY = 10 ESTIMATE = '' TOTALCOST = 0 -TOKENS = 0 TOTALTOKENS = 0 NAMESLIST = [] @@ -54,7 +53,7 @@ CODE324 = False CODE111 = False CODE408 = False CODE108 = False -NAMES = True # Output a list of all the character names found +NAMES = False # Output a list of all the character names found BRFLAG = False # If the game uses
instead FIXTEXTWRAP = False @@ -68,13 +67,12 @@ def handleMVMZ(filename, estimate): # Print Result end = time.time() - tqdm.write(getResultString(['', TOKENS, None], end - start, filename)) + tqdm.write(getResultString(translatedData, end - start, filename)) if NAMES == True: tqdm.write(str(NAMESLIST)) with LOCK: - TOTALCOST += TOKENS * .001 * APICOST - TOTALTOKENS += TOKENS - TOKENS = 0 + TOTALCOST += translatedData[1] * .001 * APICOST + TOTALTOKENS += translatedData[1] return getResultString(['', TOTALTOKENS, None], end - start, 'TOTAL') @@ -1344,13 +1342,11 @@ def resubVars(translatedText, varList): @retry(exceptions=Exception, tries=5, delay=5) def translateGPT(t, history, fullPromptFlag): - with LOCK: - # If ESTIMATE is True just count this as an execution and return. - if ESTIMATE: - global TOKENS - enc = tiktoken.encoding_for_model("gpt-3.5-turbo") - TOKENS += len(enc.encode(t)) * 2 + len(enc.encode(history)) + len(enc.encode(PROMPT)) - return (t, 0) + # If ESTIMATE is True just count this as an execution and return. + if ESTIMATE: + enc = tiktoken.encoding_for_model("gpt-3.5-turbo") + tokens = len(enc.encode(t)) * 2 + len(enc.encode(history)) + len(enc.encode(PROMPT)) + return (t, tokens) # Sub Vars varResponse = subVars(t) diff --git a/modules/tyrano.py b/modules/tyrano.py index 1d5b365..b7a30e6 100644 --- a/modules/tyrano.py +++ b/modules/tyrano.py @@ -40,7 +40,7 @@ POSITION=0 LEAVE=False # Flags -NAMES = True # Output a list of all the character names found +NAMES = False # Output a list of all the character names found BRFLAG = False # If the game uses
instead FIXTEXTWRAP = False @@ -54,13 +54,10 @@ def handleTyrano(filename, estimate): # Print Result end = time.time() - tqdm.write(getResultString(['', TOKENS, None], end - start, filename)) - if NAMES == True: - tqdm.write(str(NAMESLIST)) + tqdm.write(getResultString(translatedData, end - start, filename)) with LOCK: - TOTALCOST += TOKENS * .001 * APICOST - TOTALTOKENS += TOKENS - TOKENS = 0 + TOTALCOST += translatedData[1] * .001 * APICOST + TOTALTOKENS += translatedData[1] return getResultString(['', TOTALTOKENS, None], end - start, 'TOTAL') @@ -68,7 +65,7 @@ def handleTyrano(filename, estimate): try: with open('translated/' + filename, 'w', encoding='UTF-8') as outFile: start = time.time() - translatedData = openFiles(filename, outFile) + translatedData = openFiles(filename) # Print Result outFile.writelines(translatedData[0]) @@ -83,9 +80,9 @@ def handleTyrano(filename, estimate): return getResultString(['', TOTALTOKENS, None], end - start, 'TOTAL') -def openFiles(filename, outFile): +def openFiles(filename): with open('files/' + filename, 'r', encoding='utf-8') as readFile: - translatedData = parseTyrano(readFile, outFile, filename) + translatedData = parseTyrano(readFile, filename) # Delete lines marked for deletion finalData = [] @@ -96,10 +93,9 @@ def openFiles(filename, outFile): return translatedData -def parseTyrano(readFile, outFile, filename): +def parseTyrano(readFile, filename): totalTokens = 0 totalLines = 0 - global LOCK # Get total for progress bar data = readFile.readlines() @@ -110,18 +106,19 @@ def parseTyrano(readFile, outFile, filename): pbar.total=totalLines try: - totalTokens += translateTyrano(data, outFile, pbar) + totalTokens += translateTyrano(data, pbar) except Exception as e: traceback.print_exc() return [data, totalTokens, e] return [data, totalTokens, None] -def translateTyrano(data, outFile, pbar): +def translateTyrano(data, pbar): textHistory = [] maxHistory = MAXHISTORY tokens = 0 currentGroup = [] syncIndex = 0 + speaker = '' global LOCK, ESTIMATE for i in range(len(data)): @@ -179,18 +176,23 @@ def translateTyrano(data, outFile, pbar): translatedText = translatedText.replace('\"', '') # Format Text - matchList = re.findall(r'(.+?[\.\?\!]+)', translatedText) - translatedText = re.sub(r'(.+?[\.\?\!]+)', '', translatedText) + matchList = re.findall(r'(.+?[\.\?\!）。・]+)', translatedText) + translatedText = re.sub(r'(.+?[\.\?\!）。・]+)', '', translatedText) if len(matchList) > 0: del data[i] for line in matchList: data.insert(i, line.strip() + '[p]\n') i+=1 - else: - print ('SUS') + # else: + # print ('No Matches') if translatedText != '': data[i] = translatedText.strip() + '[p]\n' + # Keep textHistory list at length maxHistory + if len(textHistory) > maxHistory: + textHistory.pop(0) + currentGroup = [] + currentGroup = [] pbar.update(1) if len(data) > i+1: @@ -255,13 +257,13 @@ def resubVars(translatedText, varList): @retry(exceptions=Exception, tries=5, delay=5) def translateGPT(t, history, fullPromptFlag): + global LOCK, TOKENS with LOCK: # If ESTIMATE is True just count this as an execution and return. if ESTIMATE: - global TOKENS enc = tiktoken.encoding_for_model("gpt-3.5-turbo") - TOKENS += len(enc.encode(t)) * 2 + len(enc.encode(history)) + len(enc.encode(PROMPT)) - return (t, 0) + tokens = len(enc.encode(t)) * 2 + len(enc.encode(history)) + len(enc.encode(PROMPT)) + return (t, tokens) # Sub Vars varResponse = subVars(t) diff --git a/start.py b/start.py index 1973eb2..3956884 100644 --- a/start.py +++ b/start.py @@ -1,3 +1,3 @@ from modules.main import main -main() \ No newline at end of file +main()