diff --git a/modules/main.py b/modules/main.py index 460f996..2808975 100644 --- a/modules/main.py +++ b/modules/main.py @@ -3,6 +3,7 @@ import sys import traceback from colorama import Fore import os +from tqdm import tqdm from modules.rpgmakermvmz import handleMVMZ from modules.rpgmakerace import handleACE @@ -16,7 +17,7 @@ from modules.lune import handleLune THREADS = 5 # For GPT4 rate limit will be hit if you have more than 1 thread. # Info Message -print(Fore.LIGHTYELLOW_EX + "WARNING: Once a translation starts do not close it unless you want to lose your\ +tqdm.write(Fore.LIGHTYELLOW_EX + "WARNING: Once a translation starts do not close it unless you want to lose your\ translated data. If a file fails or gets stuck, translated lines will remain translated so you don't have \ to worry about being charged twice. You can simply copy the file generated in /translations back over to \ /files and start the script again. It will skip over any translated text." + Fore.RESET, end='\n\n') @@ -46,7 +47,7 @@ def main(): totalCost = future.result() except Exception as e: tracebackLineNo = str(traceback.extract_tb(sys.exc_info()[2])[-1].lineno) - print(Fore.RED + str(e) + '|' + tracebackLineNo + Fore.RESET) + tqdm.write(Fore.RED + str(e) + '|' + tracebackLineNo + Fore.RESET) case '2': # Open File (Threads) @@ -59,7 +60,7 @@ def main(): totalCost = future.result() except Exception as e: tracebackLineNo = str(traceback.extract_tb(sys.exc_info()[2])[-1].lineno) - print(Fore.RED + str(e) + '|' + tracebackLineNo + Fore.RESET) + tqdm.write(Fore.RED + str(e) + '|' + tracebackLineNo + Fore.RESET) case '3': # Open File (Threads) @@ -73,7 +74,7 @@ def main(): except Exception as e: tracebackLineNo = str(traceback.extract_tb(sys.exc_info()[2])[-1].lineno) - print(Fore.RED + str(e) + '|' + tracebackLineNo + Fore.RESET) + tqdm.write(Fore.RED + str(e) + '|' + tracebackLineNo + Fore.RESET) case '4': # Open File (Threads) @@ -87,7 +88,7 @@ def main(): except Exception as e: tracebackLineNo = str(traceback.extract_tb(sys.exc_info()[2])[-1].lineno) - print(Fore.RED + str(e) + '|' + tracebackLineNo + Fore.RESET) + tqdm.write(Fore.RED + str(e) + '|' + tracebackLineNo + Fore.RESET) case '5': # Open File (Threads) @@ -101,7 +102,7 @@ def main(): except Exception as e: tracebackLineNo = str(traceback.extract_tb(sys.exc_info()[2])[-1].lineno) - print(Fore.RED + str(e) + '|' + tracebackLineNo + Fore.RESET) + tqdm.write(Fore.RED + str(e) + '|' + tracebackLineNo + Fore.RESET) case '6': # Open File (Threads) @@ -115,7 +116,7 @@ def main(): except Exception as e: tracebackLineNo = str(traceback.extract_tb(sys.exc_info()[2])[-1].lineno) - print(Fore.RED + str(e) + '|' + tracebackLineNo + Fore.RESET) + tqdm.write(Fore.RED + str(e) + '|' + tracebackLineNo + Fore.RESET) case '7': # Open File (Threads) @@ -129,7 +130,7 @@ def main(): except Exception as e: tracebackLineNo = str(traceback.extract_tb(sys.exc_info()[2])[-1].lineno) - print(Fore.RED + str(e) + '|' + tracebackLineNo + Fore.RESET) + tqdm.write(Fore.RED + str(e) + '|' + tracebackLineNo + Fore.RESET) case '8': # Open File (Threads) @@ -143,7 +144,7 @@ def main(): except Exception as e: tracebackLineNo = str(traceback.extract_tb(sys.exc_info()[2])[-1].lineno) - print(Fore.RED + str(e) + '|' + tracebackLineNo + Fore.RESET) + tqdm.write(Fore.RED + str(e) + '|' + tracebackLineNo + Fore.RESET) case _: version = '' @@ -154,7 +155,7 @@ def main(): deleteFolderFiles('files') # Prevent immediately closing of CLI - print(totalCost) + tqdm.write(totalCost) # input('Done! Press Enter to close.') def deleteFolderFiles(folderPath): diff --git a/modules/rpgmakermvmz.py b/modules/rpgmakermvmz.py index 89bfebd..bf36712 100644 --- a/modules/rpgmakermvmz.py +++ b/modules/rpgmakermvmz.py @@ -21,7 +21,8 @@ load_dotenv() openai.organization = os.getenv('org') openai.api_key = os.getenv('key') -APICOST = .002 # Depends on the model https://openai.com/pricing +INPUTAPICOST = .002 # Depends on the model https://openai.com/pricing +OUTPUTAPICOST = .002 PROMPT = Path('prompt.txt').read_text(encoding='utf-8') THREADS = 10 LOCK = threading.Lock() @@ -30,8 +31,7 @@ LISTWIDTH = 90 NOTEWIDTH = 50 MAXHISTORY = 10 ESTIMATE = '' -TOTALCOST = 0 -TOTALTOKENS = 0 +totalTokens = [0, 0] NAMESLIST = [] #tqdm Globals @@ -60,7 +60,7 @@ FIXTEXTWRAP = True IGNORETLTEXT = False def handleMVMZ(filename, estimate): - global ESTIMATE, TOKENS, TOTALTOKENS, TOTALCOST + global ESTIMATE, totalTokens ESTIMATE = estimate if estimate: @@ -73,10 +73,10 @@ def handleMVMZ(filename, estimate): if NAMES == True: tqdm.write(str(NAMESLIST)) with LOCK: - TOTALCOST += translatedData[1] * .001 * APICOST - TOTALTOKENS += translatedData[1] + totalTokens[0] += translatedData[1][0] + totalTokens[1] += translatedData[1][1] - return getResultString(['', TOTALTOKENS, None], end - start, 'TOTAL') + return getResultString(['', totalTokens, None], end - start, 'TOTAL') else: try: @@ -89,13 +89,12 @@ def handleMVMZ(filename, estimate): json.dump(translatedData[0], outFile, ensure_ascii=False) tqdm.write(getResultString(translatedData, end - start, filename)) with LOCK: - TOTALCOST += translatedData[1] * .001 * APICOST - TOTALTOKENS += translatedData[1] + totalTokens[0] += translatedData[1][0] + totalTokens[1] += translatedData[1][1] except Exception as e: - traceback.print_exc() return 'Fail' - return getResultString(['', TOTALTOKENS, None], end - start, 'TOTAL') + return getResultString(['', totalTokens, None], end - start, 'TOTAL') def openFiles(filename): with open('files/' + filename, 'r', encoding='UTF-8-sig') as f: @@ -164,13 +163,17 @@ def openFiles(filename): def getResultString(translatedData, translationTime, filename): # File Print String - tokenString = Fore.YELLOW + '[' + str(translatedData[1]) + \ - ' Tokens/${:,.4f}'.format(translatedData[1] * .001 * APICOST) + totalTokenstring =\ + Fore.YELLOW +\ + '[Input: ' + str(translatedData[1][0]) + ']'\ + '[Output: ' + str(translatedData[1][1]) + ']'\ + '[Cost: ${:,.4f}'.format((translatedData[1][0] * .001 * INPUTAPICOST) +\ + (translatedData[1][1] * .001 * OUTPUTAPICOST)) + ']' timeString = Fore.BLUE + '[' + str(round(translationTime, 1)) + 's]' if translatedData[2] == None: # Success - return filename + ': ' + tokenString + timeString + Fore.GREEN + u' \u2713 ' + Fore.RESET + return filename + ': ' + totalTokenstring + timeString + Fore.GREEN + u' \u2713 ' + Fore.RESET else: # Fail @@ -178,11 +181,11 @@ def getResultString(translatedData, translationTime, filename): raise translatedData[2] except Exception as e: errorString = str(e) + Fore.RED - return filename + ': ' + tokenString + timeString + Fore.RED + u' \u2717 ' +\ + return filename + ': ' + totalTokenstring + timeString + Fore.RED + u' \u2717 ' +\ errorString + Fore.RESET def parseMap(data, filename): - totalTokens = 0 + totalTokens = [0, 0] totalLines = 0 events = data['events'] global LOCK @@ -190,7 +193,8 @@ def parseMap(data, filename): # Translate displayName for Map files if 'Map' in filename: response = translateGPT(data['displayName'], 'Reply with only the english translation of the RPG location name', False) - totalTokens += response[1] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] data['displayName'] = response[0].replace('\"', '') # Get total for progress bar @@ -208,12 +212,15 @@ def parseMap(data, filename): # This translates ID of events. (May break the game) # response = translateGPT(event['name'], 'Reply with the English translation of the Title.', True) # event['name'] = response[0].replace('\"', '') - # totalTokens += response[1] + # totalTokens[0] += response[1][0] + # totalTokens[1] += response[1][1] futures = [executor.submit(searchCodes, page, pbar) for page in event['pages'] if page is not None] for future in as_completed(futures): try: - totalTokens += future.result() + totalTokensFuture = future.result() + totalTokens[0] += totalTokensFuture[0] + totalTokens[1] += totalTokensFuture[1] except Exception as e: return [data, totalTokens, e] return [data, totalTokens, None] @@ -238,10 +245,10 @@ def translateNote(event, regex): translatedText = translatedText.replace('\"', '') event['note'] = event['note'].replace(oldJAString, translatedText) return response[1] - return 0 + return [0,0] def parseCommonEvents(data, filename): - totalTokens = 0 + totalTokens = [0, 0] totalLines = 0 global LOCK @@ -257,13 +264,15 @@ def parseCommonEvents(data, filename): futures = [executor.submit(searchCodes, page, pbar) for page in data if page is not None] for future in as_completed(futures): try: - totalTokens += future.result() + totalTokensFuture = future.result() + totalTokens[0] += totalTokensFuture[0] + totalTokens[1] += totalTokensFuture[1] except Exception as e: return [data, totalTokens, e] return [data, totalTokens, None] def parseTroops(data, filename): - totalTokens = 0 + totalTokens = [0, 0] totalLines = 0 global LOCK @@ -282,13 +291,15 @@ def parseTroops(data, filename): futures = [executor.submit(searchCodes, page, pbar) for page in troop['pages'] if page is not None] for future in as_completed(futures): try: - totalTokens += future.result() + totalTokensFuture = future.result() + totalTokens[0] += totalTokensFuture[0] + totalTokens[1] += totalTokensFuture[1] except Exception as e: return [data, totalTokens, e] return [data, totalTokens, None] def parseNames(data, filename, context): - totalTokens = 0 + totalTokens = [0, 0] totalLines = 0 totalLines += len(data) @@ -299,13 +310,15 @@ def parseNames(data, filename, context): if name is not None: try: result = searchNames(name, pbar, context) - totalTokens += result + totalTokens[0] += result[0] + totalTokens[1] += result[1] except Exception as e: + traceback.print_exc() return [data, totalTokens, e] return [data, totalTokens, None] def parseThings(data, filename): - totalTokens = 0 + totalTokens = [0, 0] totalLines = 0 totalLines += len(data) @@ -316,13 +329,14 @@ def parseThings(data, filename): if name is not None: try: result = searchThings(name, pbar) - totalTokens += result + totalTokens[0] += result[0] + totalTokens[1] += result[1] except Exception as e: return [data, totalTokens, e] return [data, totalTokens, None] def parseSS(data, filename): - totalTokens = 0 + totalTokens = [0, 0] totalLines = 0 totalLines += len(data) @@ -333,13 +347,14 @@ def parseSS(data, filename): if ss is not None: try: result = searchSS(ss, pbar) - totalTokens += result + totalTokens[0] += result[0] + totalTokens[1] += result[1] except Exception as e: return [data, totalTokens, e] return [data, totalTokens, None] def parseSystem(data, filename): - totalTokens = 0 + totalTokens = [0, 0] totalLines = 0 # Calculate Total Lines @@ -358,13 +373,14 @@ def parseSystem(data, filename): pbar.total=totalLines try: result = searchSystem(data, pbar) - totalTokens += result + totalTokens[0] += result[0] + totalTokens[1] += result[1] except Exception as e: return [data, totalTokens, e] return [data, totalTokens, None] def parseScenario(data, filename): - totalTokens = 0 + totalTokens = [0, 0] totalLines = 0 global LOCK @@ -379,13 +395,15 @@ def parseScenario(data, filename): futures = [executor.submit(searchCodes, page[1], pbar) for page in data.items() if page[1] is not None] for future in as_completed(futures): try: - totalTokens += future.result() + totalTokensFuture = future.result() + totalTokens[0] += totalTokensFuture[0] + totalTokens[1] += totalTokensFuture[1] except Exception as e: return [data, totalTokens, e] return [data, totalTokens, None] def searchThings(name, pbar): - tokens = 0 + totalTokens = [0, 0] # Name nameResponse = translateGPT(name['name'], 'Reply with only the english translation of the RPG item name.', False) if 'name' in name else '' @@ -395,13 +413,17 @@ def searchThings(name, pbar): # Note if '') + totalTokens[0] += translateNote(name, r'')[0] + totalTokens[1] += translateNote(name, r'')[1] if '') + totalTokens[0] += translateNote(name, r'')[0] + totalTokens[1] += translateNote(name, r'')[1] - # Count Tokens - tokens += nameResponse[1] if nameResponse != '' else 0 - tokens += descriptionResponse[1] if descriptionResponse != '' else 0 + # Count totalTokens + totalTokens[0] += nameResponse[1][0] if nameResponse != '' else 0 + totalTokens[1] += nameResponse[1][1] if nameResponse != '' else 0 + totalTokens[0] += descriptionResponse[1][0] if descriptionResponse != '' else 0 + totalTokens[1] += descriptionResponse[1][1] if descriptionResponse != '' else 0 # Set Data if 'name' in name: @@ -415,10 +437,10 @@ def searchThings(name, pbar): name['description'] = description.replace('\"', '') pbar.update(1) - return tokens + return totalTokens def searchNames(name, pbar, context): - tokens = 0 + totalTokens = [0, 0] # Set the context of what we are translating if 'Actors' in context: @@ -447,21 +469,26 @@ def searchNames(name, pbar, context): else: responseList.append(['', 0]) if 'hint' in name['note']: - tokens += translateNote(name, r'') + totalTokens[0] += translateNote(name, r'')[0] + totalTokens[1] += translateNote(name, r'')[1] if 'Enemies' in context: if 'variable_update_skill' in name['note']: - tokens += translateNote(name, r'111:(.+?)\n') + totalTokens[0] += translateNote(name, r'111:(.+?)\n')[0] + totalTokens[1] += translateNote(name, r'111:(.+?)\n')[1] if 'desc2' in name['note']: - tokens += translateNote(name, r']*)>') + totalTokens[0] += translateNote(name, r']*)>')[0] + totalTokens[1] += translateNote(name, r']*)>')[1] if 'desc3' in name['note']: - tokens += translateNote(name, r']*)>') + totalTokens[0] += translateNote(name, r']*)>')[0] + totalTokens[1] += translateNote(name, r']*)>')[1] # Extract all our translations in a list from response for i in range(len(responseList)): - tokens += responseList[i][1] + totalTokens[0] += responseList[i][1][0] + totalTokens[1] += responseList[i][1][1] responseList[i] = responseList[i][0] # Set Data @@ -472,24 +499,26 @@ def searchNames(name, pbar, context): translatedText = textwrap.fill(responseList[2], LISTWIDTH) name['nickname'] = translatedText.replace('\"', '') if '<特徴1:' in name['note']: - tokens += translateNote(name, r'<特徴1:([^>]*)>') + totalTokens[0] += translateNote(name, r'<特徴1:([^>]*)>')[0] + totalTokens[1] += translateNote(name, r'<特徴1:([^>]*)>')[1] if 'Armors' in context or 'Weapons' in context: translatedText = textwrap.fill(responseList[1], LISTWIDTH) if 'description' in name: name['description'] = translatedText.replace('\"', '') if '\n([\s\S]*?)\n') + totalTokens[0] += translateNote(name, r'\n([\s\S]*?)\n')[0] + totalTokens[1] += translateNote(name, r'\n([\s\S]*?)\n')[1] pbar.update(1) - return tokens + return totalTokens def searchCodes(page, pbar): translatedText = '' currentGroup = [] textHistory = [] maxHistory = MAXHISTORY - tokens = 0 + totalTokens = [0, 0] speaker = '' speakerVar = '' nametag = '' @@ -560,7 +589,8 @@ def searchCodes(page, pbar): matchList = re.findall(r'(.*?([\\]+[nN]<(.+?)>).*)', finalJAString) if len(matchList) > 0: response = translateGPT(matchList[0][2], 'Reply with only the english translation of the NPC name', False) - tokens += response[1] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] speaker = response[0].strip('.') nametag = matchList[0][1].replace(matchList[0][2], speaker) finalJAString = finalJAString.replace(matchList[0][1], '') @@ -608,7 +638,8 @@ def searchCodes(page, pbar): if len(matchList) != 0: # Translate Speaker response = translateGPT(matchList[0][1], 'Reply with only the english translation of the NPC name', True) - tokens += response[1] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] speaker = response[0].strip('.') nametag = matchList[0][0].replace(matchList[0][1], speaker) finalJAString = finalJAString.replace(matchList[0][0], '') @@ -623,7 +654,8 @@ def searchCodes(page, pbar): matchList = re.findall(r'([\\]+[nN][wW]\[(.+?)\]+)(.+)', finalJAString) if len(matchList) != 0: response = translateGPT(matchList[0][1], 'Reply with only the english translation of the NPC name', True) - tokens += response[1] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] speaker = response[0].strip('.') # Set Nametag and Remove from Final String @@ -653,7 +685,8 @@ def searchCodes(page, pbar): # response = translateGPT(matchList[0][1], 'Reply with only the english translation of the NPC name', True) # else: # print('wtf') - # tokens += response[1] + # totalTokens[0] += response[1][0] + # totalTokens[1] += response[1][1] # speaker = response[0].strip('.') # # Set Nametag and Remove from Final String @@ -725,7 +758,8 @@ def searchCodes(page, pbar): # Translate if speaker == '' and finalJAString != '': response = translateGPT(finalJAString, textHistory, True) - tokens += response[1] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] translatedText = response[0] # Remove added speaker @@ -737,7 +771,8 @@ def searchCodes(page, pbar): textHistory.append('\"' + varResponse[0] + '\"') elif finalJAString != '': response = translateGPT(speaker + ': ' + finalJAString, textHistory, True) - tokens += response[1] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] translatedText = response[0] # Remove added speaker @@ -807,7 +842,8 @@ def searchCodes(page, pbar): match = match.replace('\\n', ' ') response = translateGPT(match, 'Reply with the English translation.', True) translatedText = response[0] - tokens += response[1] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] # Replace translatedText = jaString.replace(jaString, translatedText) @@ -853,7 +889,8 @@ def searchCodes(page, pbar): # Translate response = translateGPT(finalJAString, '', True) - tokens += response[1] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] translatedText = response[0] # Textwrap @@ -892,7 +929,8 @@ def searchCodes(page, pbar): # Translate response = translateGPT(jaString, '', True) - tokens += response[1] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] translatedText = response[0] # Remove characters that may break scripts @@ -957,7 +995,8 @@ def searchCodes(page, pbar): # Translate response = translateGPT(jaString, 'Reply with only the english translation of the NPC name.', False) - tokens += response[1] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] translatedText = response[0] # Remove characters that may break scripts @@ -999,7 +1038,8 @@ def searchCodes(page, pbar): continue response = translateGPT(matchList[0], 'Reply with the English translation Stat Title. Keep it brief.', True) - tokens += response[1] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] translatedText = response[0] # Remove characters that may break scripts @@ -1037,7 +1077,8 @@ def searchCodes(page, pbar): # Translate response = translateGPT(jaString, '', True) - tokens += response[1] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] translatedText = response[0] # Remove characters that may break scripts @@ -1070,7 +1111,8 @@ def searchCodes(page, pbar): # Translate if len(matchList) > 0: response = translateGPT(matchList[0], 'Reply with the English translation of the Location Title', True) - tokens += response[1] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] translatedText = response[0] # Remove characters that may break scripts @@ -1096,7 +1138,8 @@ def searchCodes(page, pbar): # Translate response = translateGPT(matchList[0], 'Reply with the English translation of the NPC name.', True) translatedText = response[0] - tokens += response[1] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] # Set Text speaker = translatedText @@ -1153,7 +1196,8 @@ def searchCodes(page, pbar): # Translate response = translateGPT(finalJAString, 'Reply with the English Translation.', True) translatedText = response[0] - tokens += response[1] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] # Textwrap translatedText = textwrap.fill(translatedText, width=20, drop_whitespace=False) @@ -1221,7 +1265,8 @@ def searchCodes(page, pbar): # Translate response = translateGPT(finalJAString, 'Reply with the English Translation.', True) translatedText = response[0] - tokens += response[1] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] # Remove characters that may break scripts charList = ['.', '\"'] @@ -1283,7 +1328,8 @@ def searchCodes(page, pbar): # Translate response = translateGPT(finalJAString, 'Reply with the English Translation.', True) translatedText = response[0] - tokens += response[1] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] # Remove characters that may break scripts charList = ['.', '\"'] @@ -1330,7 +1376,8 @@ def searchCodes(page, pbar): translatedText = translatedText.replace(char, '') # Set Data - tokens += response[1] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] codeList[i]['parameters'][0][choice] = startString + translatedText + endString ### Event Code: 111 Script @@ -1356,7 +1403,8 @@ def searchCodes(page, pbar): for match in matchList: response = translateGPT(match, '', True) translatedText = response[0] - tokens += response[1] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] # Remove characters that may break scripts charList = ['.', '\"', '\'', '\\n'] @@ -1385,7 +1433,8 @@ def searchCodes(page, pbar): response = translateGPT(jaString, 'Reply with the English translation of the NPC name.', True) translatedText = response[0] - tokens += response[1] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] # Remove characters that may break scripts charList = ['.', '\"', '\'', '\\n'] @@ -1409,7 +1458,8 @@ def searchCodes(page, pbar): if len(currentGroup) > 0: # Translate response = translateGPT(finalJAString, 'Previous Translated Text for Context: ' + '\n\n'.join(textHistory), True) - tokens += response[1] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] translatedText = response[0] # TextHistory is what we use to give GPT Context, so thats appended here. @@ -1431,11 +1481,11 @@ def searchCodes(page, pbar): textHistory.pop(0) currentGroup = [] - return tokens + return totalTokens def searchSS(state, pbar): '''Searches skills and states json files''' - tokens = 0 + totalTokens = [0, 0] # Name nameResponse = translateGPT(state['name'], 'Reply with only the english translation of the RPG Skill name.', True) if 'name' in state else '' @@ -1475,15 +1525,22 @@ def searchSS(state, pbar): # if 'note' in state: if 'help' in state['note']: - tokens += translateNote(state, r']*)>') + totalTokens[0] += translateNote(state, r']*)>')[0] + totalTokens[1] += translateNote(state, r']*)>')[1] - # Count Tokens - tokens += nameResponse[1] if nameResponse != '' else 0 - tokens += descriptionResponse[1] if descriptionResponse != '' else 0 - tokens += message1Response[1] if message1Response != '' else 0 - tokens += message2Response[1] if message2Response != '' else 0 - tokens += message3Response[1] if message3Response != '' else 0 - tokens += message4Response[1] if message4Response != '' else 0 + # Count totalTokens + totalTokens[0] += nameResponse[1][0] if nameResponse != '' else 0 + totalTokens[1] += nameResponse[1][1] if nameResponse != '' else 0 + totalTokens[0] += descriptionResponse[1][0] if descriptionResponse != '' else 0 + totalTokens[1] += descriptionResponse[1][1] if descriptionResponse != '' else 0 + totalTokens[0] += message1Response[1][0] if message1Response != '' else 0 + totalTokens[1] += message1Response[1][1] if message1Response != '' else 0 + totalTokens[0] += message2Response[1][0] if message2Response != '' else 0 + totalTokens[1] += message2Response[1][1] if message2Response != '' else 0 + totalTokens[0] += message3Response[1][0] if message3Response != '' else 0 + totalTokens[1] += message3Response[1][1] if message3Response != '' else 0 + totalTokens[0] += message4Response[1][0] if message4Response != '' else 0 + totalTokens[1] += message4Response[1][1] if message4Response != '' else 0 # Set Data if 'name' in state: @@ -1503,10 +1560,10 @@ def searchSS(state, pbar): state['message4'] = message4Response[0].replace('\"', '').replace('Taro', '') pbar.update(1) - return tokens + return totalTokens def searchSystem(data, pbar): - tokens = 0 + totalTokens = [0, 0] context = 'UI Text Items:\ "逃げる" == "Escape"\ "大事なもの" == "Key Items"\ @@ -1523,7 +1580,8 @@ def searchSystem(data, pbar): # Title response = translateGPT(data['gameTitle'], ' Reply with the English translation of the game title name', False) - tokens += response[1] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] data['gameTitle'] = response[0].strip('.') pbar.update(1) @@ -1534,35 +1592,40 @@ def searchSystem(data, pbar): for i in range(len(termList)): # Last item is a messages object if termList[i] is not None: response = translateGPT(termList[i], context, False) - tokens += response[1] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] termList[i] = response[0].replace('\"', '').strip() pbar.update(1) # Armor Types for i in range(len(data['armorTypes'])): response = translateGPT(data['armorTypes'][i], 'Reply with only the english translation of the armor type', False) - tokens += response[1] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] data['armorTypes'][i] = response[0].replace('\"', '').strip() pbar.update(1) # Skill Types for i in range(len(data['skillTypes'])): response = translateGPT(data['skillTypes'][i], 'Reply with only the english translation', False) - tokens += response[1] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] data['skillTypes'][i] = response[0].replace('\"', '').strip() pbar.update(1) # Equip Types for i in range(len(data['equipTypes'])): response = translateGPT(data['equipTypes'][i], 'Reply with only the english translation of the equipment type. No disclaimers.', False) - tokens += response[1] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] data['equipTypes'][i] = response[0].replace('\"', '').strip() pbar.update(1) # Variables (Optional ususally) for i in range(len(data['variables'])): response = translateGPT(data['variables'][i], 'Reply with only the english translation of the title', False) - tokens += response[1] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] data['variables'][i] = response[0].replace('\"', '').strip() pbar.update(1) @@ -1577,11 +1640,12 @@ def searchSystem(data, pbar): for char in charList: translatedText = translatedText.replace(char, '') - tokens += response[1] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] messages[key] = translatedText pbar.update(1) - return tokens + return totalTokens def subVars(jaString): jaString = jaString.replace('\u3000', ' ') @@ -1690,9 +1754,18 @@ def resubVars(translatedText, allList): def translateGPT(t, history, fullPromptFlag): # If ESTIMATE is True just count this as an execution and return. if ESTIMATE: - enc = tiktoken.encoding_for_model("gpt-4") - tokens = len(enc.encode(t)) * 2 + len(enc.encode(str(history))) + len(enc.encode(PROMPT)) - return (t, tokens) + enc = tiktoken.encoding_for_model("gpt-3.5-turbo") + historyRaw = '' + if isinstance(history, list): + for line in history: + historyRaw += line + else: + historyRaw = history + + inputTotalTokens = len(enc.encode(historyRaw)) + len(enc.encode(PROMPT)) + outputTotalTokens = len(enc.encode(t)) * 2 # Estimating 2x the size of the original text + totalTokens = [inputTotalTokens, outputTotalTokens] + return (t, totalTokens) # Sub Vars varResponse = subVars(t) @@ -1700,7 +1773,7 @@ def translateGPT(t, history, fullPromptFlag): # If there isn't any Japanese in the text just skip if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴ]+|[\uFF00-\uFFEF]', subbedT): - return(t, 0) + return(t, [0,0]) # Characters context = '```\ @@ -1742,7 +1815,7 @@ def translateGPT(t, history, fullPromptFlag): # Save Translated Text translatedText = response.choices[0].message.content - tokens = response.usage.total_tokens + totalTokens = [response.usage.prompt_tokens, response.usage.completion_tokens] # Resub Vars translatedText = resubVars(translatedText, varResponse[1]) @@ -1765,4 +1838,4 @@ def translateGPT(t, history, fullPromptFlag): if len(translatedText) > 15 * len(t) or "I'm sorry, but I'm unable to assist with that translation" in translatedText: raise Exception else: - return [translatedText, tokens] \ No newline at end of file + return [translatedText, totalTokens] \ No newline at end of file