From 3d573fcaab04cd464f2bd70fb7d27e74e794c691 Mon Sep 17 00:00:00 2001 From: DazedAnon Date: Tue, 24 Dec 2024 15:18:40 -0600 Subject: [PATCH 1/3] Check if text is null first --- modules/csv.py | 173 +++++++++++++++++++------------------- modules/json.py | 173 +++++++++++++++++++------------------- modules/kirikiri.py | 173 +++++++++++++++++++------------------- modules/nscript.py | 173 +++++++++++++++++++------------------- modules/renpy.py | 173 +++++++++++++++++++------------------- modules/rpgmakerace.py | 173 +++++++++++++++++++------------------- modules/rpgmakermvmz.py | 173 +++++++++++++++++++------------------- modules/rpgmakerplugin.py | 173 +++++++++++++++++++------------------- modules/text.py | 173 +++++++++++++++++++------------------- modules/tyrano.py | 173 +++++++++++++++++++------------------- modules/wolf.py | 173 +++++++++++++++++++------------------- modules/wolf2.py | 173 +++++++++++++++++++------------------- 12 files changed, 1056 insertions(+), 1020 deletions(-) diff --git a/modules/csv.py b/modules/csv.py index 2d25146..8c76ebc 100644 --- a/modules/csv.py +++ b/modules/csv.py @@ -597,97 +597,100 @@ def countTokens(system, user, history): @retry(exceptions=Exception, tries=5, delay=5) def translateGPT(text, history, fullPromptFlag): global PBAR, MISMATCH, FILENAME - with open("log/translationHistory.txt", "a+", encoding="utf-8") as logFile: - mismatch = False - totalTokens = [0, 0] - if isinstance(text, list): - format = "json" - tList = batchList(text, BATCHSIZE) - else: - format = "text" - tList = [text] - - for index, tItem in enumerate(tList): - # Before sending to translation, if we have a list of items, add the formatting - if isinstance(tItem, list): - for j in range(len(tItem)): - if not tItem[j]: - tItem[j] = tItem[j].replace("", "Placeholder Text") - payload = {f"Line{i+1}": string for i, string in enumerate(tItem)} - payload = json.dumps(payload, indent=4, ensure_ascii=False) - varResponse = [payload, []] - subbedT = varResponse[0] + if text: + with open("log/translationHistory.txt", "a+", encoding="utf-8") as logFile: + mismatch = False + totalTokens = [0, 0] + if isinstance(text, list): + format = "json" + tList = batchList(text, BATCHSIZE) else: - varResponse = [tItem, []] - subbedT = varResponse[0] + format = "text" + tList = [text] - # Things to Check before starting translation - if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+", subbedT): - if PBAR is not None: - PBAR.update(len(tItem)) - history = tItem[-MAXHISTORY:] - continue - - # Create Message - system, user = createContext(fullPromptFlag, subbedT, format) - - # Calculate Estimate - if ESTIMATE: - estimate = countTokens(system, user, history) - totalTokens[0] += estimate[0] - totalTokens[1] += estimate[1] - continue - - # Translating - response = translateText(system, user, history, 0.05, format) - translatedText = response.choices[0].message.content - totalTokens[0] += response.usage.prompt_tokens - totalTokens[1] += response.usage.completion_tokens - - # Check Translation - translatedText = cleanTranslatedText(translatedText, varResponse) - if isinstance(tItem, list): - extractedTranslations = extractTranslation(translatedText, True) - if extractedTranslations == None or len(tItem) != len(extractedTranslations): - # Mismatch. Try Again - response = translateText(system, user, history, 0.05, format, "gpt-4o") - translatedText = response.choices[0].message.content - totalTokens[0] += response.usage.prompt_tokens - totalTokens[1] += response.usage.completion_tokens - - # Formatting - translatedText = cleanTranslatedText(translatedText, varResponse) - if isinstance(tItem, list): - extractedTranslations = extractTranslation(translatedText, True) - if extractedTranslations == None or len(tItem) != len(extractedTranslations): - mismatch = True # Just here for breakpoint - logFile.write(f"Input:\n{subbedT}\n") - logFile.write(f"Output:\n{translatedText}\n") - - # Set if no mismatch - if mismatch == False: - tList[index] = extractedTranslations - history = extractedTranslations[-MAXHISTORY:] # Update history if we have a list + for index, tItem in enumerate(tList): + # Before sending to translation, if we have a list of items, add the formatting + if isinstance(tItem, list): + for j in range(len(tItem)): + if not tItem[j]: + tItem[j] = tItem[j].replace("", "Placeholder Text") + payload = {f"Line{i+1}": string for i, string in enumerate(tItem)} + payload = json.dumps(payload, indent=4, ensure_ascii=False) + varResponse = [payload, []] + subbedT = varResponse[0] else: - history = text[-MAXHISTORY:] - mismatch = False - if FILENAME not in MISMATCH: - MISMATCH.append(FILENAME) + varResponse = [tItem, []] + subbedT = varResponse[0] - # Update Loading Bar - with LOCK: + # Things to Check before starting translation + if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+", subbedT): if PBAR is not None: PBAR.update(len(tItem)) - else: - # Ensure we're passing a single string to extractTranslation - tList[index] = translatedText.replace("Placeholder Text", "") + history = tItem[-MAXHISTORY:] + continue - # Combine if multilist - if isinstance(tList[0], list): - tList = [t for sublist in tList for t in sublist] + # Create Message + system, user = createContext(fullPromptFlag, subbedT, format) - # Return - if format == "json": - return [tList, totalTokens] + # Calculate Estimate + if ESTIMATE: + estimate = countTokens(system, user, history) + totalTokens[0] += estimate[0] + totalTokens[1] += estimate[1] + continue + + # Translating + response = translateText(system, user, history, 0.05, format) + translatedText = response.choices[0].message.content + totalTokens[0] += response.usage.prompt_tokens + totalTokens[1] += response.usage.completion_tokens + + # Check Translation + translatedText = cleanTranslatedText(translatedText, varResponse) + if isinstance(tItem, list): + extractedTranslations = extractTranslation(translatedText, True) + if extractedTranslations == None or len(tItem) != len(extractedTranslations): + # Mismatch. Try Again + response = translateText(system, user, history, 0.05, format, "gpt-4o") + translatedText = response.choices[0].message.content + totalTokens[0] += response.usage.prompt_tokens + totalTokens[1] += response.usage.completion_tokens + + # Formatting + translatedText = cleanTranslatedText(translatedText, varResponse) + if isinstance(tItem, list): + extractedTranslations = extractTranslation(translatedText, True) + if extractedTranslations == None or len(tItem) != len(extractedTranslations): + mismatch = True # Just here for breakpoint + logFile.write(f"Input:\n{subbedT}\n") + logFile.write(f"Output:\n{translatedText}\n") + + # Set if no mismatch + if mismatch == False: + tList[index] = extractedTranslations + history = extractedTranslations[-MAXHISTORY:] # Update history if we have a list + else: + history = text[-MAXHISTORY:] + mismatch = False + if FILENAME not in MISMATCH: + MISMATCH.append(FILENAME) + + # Update Loading Bar + with LOCK: + if PBAR is not None: + PBAR.update(len(tItem)) + else: + # Ensure we're passing a single string to extractTranslation + tList[index] = translatedText.replace("Placeholder Text", "") + + # Combine if multilist + if isinstance(tList[0], list): + tList = [t for sublist in tList for t in sublist] + + # Return + if format == "json": + return [tList, totalTokens] + else: + return [tList[0], totalTokens] else: - return [tList[0], totalTokens] + return [text, [0,0]] diff --git a/modules/json.py b/modules/json.py index fc44b09..a0309de 100644 --- a/modules/json.py +++ b/modules/json.py @@ -465,97 +465,100 @@ def countTokens(system, user, history): @retry(exceptions=Exception, tries=5, delay=5) def translateGPT(text, history, fullPromptFlag): global PBAR, MISMATCH, FILENAME - with open("log/translationHistory.txt", "a+", encoding="utf-8") as logFile: - mismatch = False - totalTokens = [0, 0] - if isinstance(text, list): - format = "json" - tList = batchList(text, BATCHSIZE) - else: - format = "text" - tList = [text] - - for index, tItem in enumerate(tList): - # Before sending to translation, if we have a list of items, add the formatting - if isinstance(tItem, list): - for j in range(len(tItem)): - if not tItem[j]: - tItem[j] = tItem[j].replace("", "Placeholder Text") - payload = {f"Line{i+1}": string for i, string in enumerate(tItem)} - payload = json.dumps(payload, indent=4, ensure_ascii=False) - varResponse = [payload, []] - subbedT = varResponse[0] + if text: + with open("log/translationHistory.txt", "a+", encoding="utf-8") as logFile: + mismatch = False + totalTokens = [0, 0] + if isinstance(text, list): + format = "json" + tList = batchList(text, BATCHSIZE) else: - varResponse = [tItem, []] - subbedT = varResponse[0] + format = "text" + tList = [text] - # Things to Check before starting translation - if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+", subbedT): - if PBAR is not None: - PBAR.update(len(tItem)) - history = tItem[-MAXHISTORY:] - continue - - # Create Message - system, user = createContext(fullPromptFlag, subbedT, format) - - # Calculate Estimate - if ESTIMATE: - estimate = countTokens(system, user, history) - totalTokens[0] += estimate[0] - totalTokens[1] += estimate[1] - continue - - # Translating - response = translateText(system, user, history, 0.05, format) - translatedText = response.choices[0].message.content - totalTokens[0] += response.usage.prompt_tokens - totalTokens[1] += response.usage.completion_tokens - - # Check Translation - translatedText = cleanTranslatedText(translatedText, varResponse) - if isinstance(tItem, list): - extractedTranslations = extractTranslation(translatedText, True) - if extractedTranslations == None or len(tItem) != len(extractedTranslations): - # Mismatch. Try Again - response = translateText(system, user, history, 0.05, format, "gpt-4o") - translatedText = response.choices[0].message.content - totalTokens[0] += response.usage.prompt_tokens - totalTokens[1] += response.usage.completion_tokens - - # Formatting - translatedText = cleanTranslatedText(translatedText, varResponse) - if isinstance(tItem, list): - extractedTranslations = extractTranslation(translatedText, True) - if extractedTranslations == None or len(tItem) != len(extractedTranslations): - mismatch = True # Just here for breakpoint - logFile.write(f"Input:\n{subbedT}\n") - logFile.write(f"Output:\n{translatedText}\n") - - # Set if no mismatch - if mismatch == False: - tList[index] = extractedTranslations - history = extractedTranslations[-MAXHISTORY:] # Update history if we have a list + for index, tItem in enumerate(tList): + # Before sending to translation, if we have a list of items, add the formatting + if isinstance(tItem, list): + for j in range(len(tItem)): + if not tItem[j]: + tItem[j] = tItem[j].replace("", "Placeholder Text") + payload = {f"Line{i+1}": string for i, string in enumerate(tItem)} + payload = json.dumps(payload, indent=4, ensure_ascii=False) + varResponse = [payload, []] + subbedT = varResponse[0] else: - history = text[-MAXHISTORY:] - mismatch = False - if FILENAME not in MISMATCH: - MISMATCH.append(FILENAME) + varResponse = [tItem, []] + subbedT = varResponse[0] - # Update Loading Bar - with LOCK: + # Things to Check before starting translation + if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+", subbedT): if PBAR is not None: PBAR.update(len(tItem)) - else: - # Ensure we're passing a single string to extractTranslation - tList[index] = translatedText.replace("Placeholder Text", "") + history = tItem[-MAXHISTORY:] + continue - # Combine if multilist - if isinstance(tList[0], list): - tList = [t for sublist in tList for t in sublist] + # Create Message + system, user = createContext(fullPromptFlag, subbedT, format) - # Return - if format == "json": - return [tList, totalTokens] + # Calculate Estimate + if ESTIMATE: + estimate = countTokens(system, user, history) + totalTokens[0] += estimate[0] + totalTokens[1] += estimate[1] + continue + + # Translating + response = translateText(system, user, history, 0.05, format) + translatedText = response.choices[0].message.content + totalTokens[0] += response.usage.prompt_tokens + totalTokens[1] += response.usage.completion_tokens + + # Check Translation + translatedText = cleanTranslatedText(translatedText, varResponse) + if isinstance(tItem, list): + extractedTranslations = extractTranslation(translatedText, True) + if extractedTranslations == None or len(tItem) != len(extractedTranslations): + # Mismatch. Try Again + response = translateText(system, user, history, 0.05, format, "gpt-4o") + translatedText = response.choices[0].message.content + totalTokens[0] += response.usage.prompt_tokens + totalTokens[1] += response.usage.completion_tokens + + # Formatting + translatedText = cleanTranslatedText(translatedText, varResponse) + if isinstance(tItem, list): + extractedTranslations = extractTranslation(translatedText, True) + if extractedTranslations == None or len(tItem) != len(extractedTranslations): + mismatch = True # Just here for breakpoint + logFile.write(f"Input:\n{subbedT}\n") + logFile.write(f"Output:\n{translatedText}\n") + + # Set if no mismatch + if mismatch == False: + tList[index] = extractedTranslations + history = extractedTranslations[-MAXHISTORY:] # Update history if we have a list + else: + history = text[-MAXHISTORY:] + mismatch = False + if FILENAME not in MISMATCH: + MISMATCH.append(FILENAME) + + # Update Loading Bar + with LOCK: + if PBAR is not None: + PBAR.update(len(tItem)) + else: + # Ensure we're passing a single string to extractTranslation + tList[index] = translatedText.replace("Placeholder Text", "") + + # Combine if multilist + if isinstance(tList[0], list): + tList = [t for sublist in tList for t in sublist] + + # Return + if format == "json": + return [tList, totalTokens] + else: + return [tList[0], totalTokens] else: - return [tList[0], totalTokens] + return [text, [0,0]] diff --git a/modules/kirikiri.py b/modules/kirikiri.py index 0f18a02..6488180 100644 --- a/modules/kirikiri.py +++ b/modules/kirikiri.py @@ -508,97 +508,100 @@ def countTokens(system, user, history): @retry(exceptions=Exception, tries=5, delay=5) def translateGPT(text, history, fullPromptFlag): global PBAR, MISMATCH, FILENAME - with open("log/translationHistory.txt", "a+", encoding="utf-8") as logFile: - mismatch = False - totalTokens = [0, 0] - if isinstance(text, list): - format = "json" - tList = batchList(text, BATCHSIZE) - else: - format = "text" - tList = [text] - - for index, tItem in enumerate(tList): - # Before sending to translation, if we have a list of items, add the formatting - if isinstance(tItem, list): - for j in range(len(tItem)): - if not tItem[j]: - tItem[j] = tItem[j].replace("", "Placeholder Text") - payload = {f"Line{i+1}": string for i, string in enumerate(tItem)} - payload = json.dumps(payload, indent=4, ensure_ascii=False) - varResponse = [payload, []] - subbedT = varResponse[0] + if text: + with open("log/translationHistory.txt", "a+", encoding="utf-8") as logFile: + mismatch = False + totalTokens = [0, 0] + if isinstance(text, list): + format = "json" + tList = batchList(text, BATCHSIZE) else: - varResponse = [tItem, []] - subbedT = varResponse[0] + format = "text" + tList = [text] - # Things to Check before starting translation - if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+", subbedT): - if PBAR is not None: - PBAR.update(len(tItem)) - history = tItem[-MAXHISTORY:] - continue - - # Create Message - system, user = createContext(fullPromptFlag, subbedT, format) - - # Calculate Estimate - if ESTIMATE: - estimate = countTokens(system, user, history) - totalTokens[0] += estimate[0] - totalTokens[1] += estimate[1] - continue - - # Translating - response = translateText(system, user, history, 0.05, format) - translatedText = response.choices[0].message.content - totalTokens[0] += response.usage.prompt_tokens - totalTokens[1] += response.usage.completion_tokens - - # Check Translation - translatedText = cleanTranslatedText(translatedText, varResponse) - if isinstance(tItem, list): - extractedTranslations = extractTranslation(translatedText, True) - if extractedTranslations == None or len(tItem) != len(extractedTranslations): - # Mismatch. Try Again - response = translateText(system, user, history, 0.05, format, "gpt-4o") - translatedText = response.choices[0].message.content - totalTokens[0] += response.usage.prompt_tokens - totalTokens[1] += response.usage.completion_tokens - - # Formatting - translatedText = cleanTranslatedText(translatedText, varResponse) - if isinstance(tItem, list): - extractedTranslations = extractTranslation(translatedText, True) - if extractedTranslations == None or len(tItem) != len(extractedTranslations): - mismatch = True # Just here for breakpoint - logFile.write(f"Input:\n{subbedT}\n") - logFile.write(f"Output:\n{translatedText}\n") - - # Set if no mismatch - if mismatch == False: - tList[index] = extractedTranslations - history = extractedTranslations[-MAXHISTORY:] # Update history if we have a list + for index, tItem in enumerate(tList): + # Before sending to translation, if we have a list of items, add the formatting + if isinstance(tItem, list): + for j in range(len(tItem)): + if not tItem[j]: + tItem[j] = tItem[j].replace("", "Placeholder Text") + payload = {f"Line{i+1}": string for i, string in enumerate(tItem)} + payload = json.dumps(payload, indent=4, ensure_ascii=False) + varResponse = [payload, []] + subbedT = varResponse[0] else: - history = text[-MAXHISTORY:] - mismatch = False - if FILENAME not in MISMATCH: - MISMATCH.append(FILENAME) + varResponse = [tItem, []] + subbedT = varResponse[0] - # Update Loading Bar - with LOCK: + # Things to Check before starting translation + if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+", subbedT): if PBAR is not None: PBAR.update(len(tItem)) - else: - # Ensure we're passing a single string to extractTranslation - tList[index] = translatedText.replace("Placeholder Text", "") + history = tItem[-MAXHISTORY:] + continue - # Combine if multilist - if isinstance(tList[0], list): - tList = [t for sublist in tList for t in sublist] + # Create Message + system, user = createContext(fullPromptFlag, subbedT, format) - # Return - if format == "json": - return [tList, totalTokens] + # Calculate Estimate + if ESTIMATE: + estimate = countTokens(system, user, history) + totalTokens[0] += estimate[0] + totalTokens[1] += estimate[1] + continue + + # Translating + response = translateText(system, user, history, 0.05, format) + translatedText = response.choices[0].message.content + totalTokens[0] += response.usage.prompt_tokens + totalTokens[1] += response.usage.completion_tokens + + # Check Translation + translatedText = cleanTranslatedText(translatedText, varResponse) + if isinstance(tItem, list): + extractedTranslations = extractTranslation(translatedText, True) + if extractedTranslations == None or len(tItem) != len(extractedTranslations): + # Mismatch. Try Again + response = translateText(system, user, history, 0.05, format, "gpt-4o") + translatedText = response.choices[0].message.content + totalTokens[0] += response.usage.prompt_tokens + totalTokens[1] += response.usage.completion_tokens + + # Formatting + translatedText = cleanTranslatedText(translatedText, varResponse) + if isinstance(tItem, list): + extractedTranslations = extractTranslation(translatedText, True) + if extractedTranslations == None or len(tItem) != len(extractedTranslations): + mismatch = True # Just here for breakpoint + logFile.write(f"Input:\n{subbedT}\n") + logFile.write(f"Output:\n{translatedText}\n") + + # Set if no mismatch + if mismatch == False: + tList[index] = extractedTranslations + history = extractedTranslations[-MAXHISTORY:] # Update history if we have a list + else: + history = text[-MAXHISTORY:] + mismatch = False + if FILENAME not in MISMATCH: + MISMATCH.append(FILENAME) + + # Update Loading Bar + with LOCK: + if PBAR is not None: + PBAR.update(len(tItem)) + else: + # Ensure we're passing a single string to extractTranslation + tList[index] = translatedText.replace("Placeholder Text", "") + + # Combine if multilist + if isinstance(tList[0], list): + tList = [t for sublist in tList for t in sublist] + + # Return + if format == "json": + return [tList, totalTokens] + else: + return [tList[0], totalTokens] else: - return [tList[0], totalTokens] + return [text, [0,0]] diff --git a/modules/nscript.py b/modules/nscript.py index fef80e4..aaa2dd9 100644 --- a/modules/nscript.py +++ b/modules/nscript.py @@ -524,97 +524,100 @@ def countTokens(system, user, history): @retry(exceptions=Exception, tries=5, delay=5) def translateGPT(text, history, fullPromptFlag): global PBAR, MISMATCH, FILENAME - with open("log/translationHistory.txt", "a+", encoding="utf-8") as logFile: - mismatch = False - totalTokens = [0, 0] - if isinstance(text, list): - format = "json" - tList = batchList(text, BATCHSIZE) - else: - format = "text" - tList = [text] - - for index, tItem in enumerate(tList): - # Before sending to translation, if we have a list of items, add the formatting - if isinstance(tItem, list): - for j in range(len(tItem)): - if not tItem[j]: - tItem[j] = tItem[j].replace("", "Placeholder Text") - payload = {f"Line{i+1}": string for i, string in enumerate(tItem)} - payload = json.dumps(payload, indent=4, ensure_ascii=False) - varResponse = [payload, []] - subbedT = varResponse[0] + if text: + with open("log/translationHistory.txt", "a+", encoding="utf-8") as logFile: + mismatch = False + totalTokens = [0, 0] + if isinstance(text, list): + format = "json" + tList = batchList(text, BATCHSIZE) else: - varResponse = [tItem, []] - subbedT = varResponse[0] + format = "text" + tList = [text] - # Things to Check before starting translation - if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+", subbedT): - if PBAR is not None: - PBAR.update(len(tItem)) - history = tItem[-MAXHISTORY:] - continue - - # Create Message - system, user = createContext(fullPromptFlag, subbedT, format) - - # Calculate Estimate - if ESTIMATE: - estimate = countTokens(system, user, history) - totalTokens[0] += estimate[0] - totalTokens[1] += estimate[1] - continue - - # Translating - response = translateText(system, user, history, 0.05, format) - translatedText = response.choices[0].message.content - totalTokens[0] += response.usage.prompt_tokens - totalTokens[1] += response.usage.completion_tokens - - # Check Translation - translatedText = cleanTranslatedText(translatedText, varResponse) - if isinstance(tItem, list): - extractedTranslations = extractTranslation(translatedText, True) - if extractedTranslations == None or len(tItem) != len(extractedTranslations): - # Mismatch. Try Again - response = translateText(system, user, history, 0.05, format, "gpt-4o") - translatedText = response.choices[0].message.content - totalTokens[0] += response.usage.prompt_tokens - totalTokens[1] += response.usage.completion_tokens - - # Formatting - translatedText = cleanTranslatedText(translatedText, varResponse) - if isinstance(tItem, list): - extractedTranslations = extractTranslation(translatedText, True) - if extractedTranslations == None or len(tItem) != len(extractedTranslations): - mismatch = True # Just here for breakpoint - logFile.write(f"Input:\n{subbedT}\n") - logFile.write(f"Output:\n{translatedText}\n") - - # Set if no mismatch - if mismatch == False: - tList[index] = extractedTranslations - history = extractedTranslations[-MAXHISTORY:] # Update history if we have a list + for index, tItem in enumerate(tList): + # Before sending to translation, if we have a list of items, add the formatting + if isinstance(tItem, list): + for j in range(len(tItem)): + if not tItem[j]: + tItem[j] = tItem[j].replace("", "Placeholder Text") + payload = {f"Line{i+1}": string for i, string in enumerate(tItem)} + payload = json.dumps(payload, indent=4, ensure_ascii=False) + varResponse = [payload, []] + subbedT = varResponse[0] else: - history = text[-MAXHISTORY:] - mismatch = False - if FILENAME not in MISMATCH: - MISMATCH.append(FILENAME) + varResponse = [tItem, []] + subbedT = varResponse[0] - # Update Loading Bar - with LOCK: + # Things to Check before starting translation + if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+", subbedT): if PBAR is not None: PBAR.update(len(tItem)) - else: - # Ensure we're passing a single string to extractTranslation - tList[index] = translatedText.replace("Placeholder Text", "") + history = tItem[-MAXHISTORY:] + continue - # Combine if multilist - if isinstance(tList[0], list): - tList = [t for sublist in tList for t in sublist] + # Create Message + system, user = createContext(fullPromptFlag, subbedT, format) - # Return - if format == "json": - return [tList, totalTokens] + # Calculate Estimate + if ESTIMATE: + estimate = countTokens(system, user, history) + totalTokens[0] += estimate[0] + totalTokens[1] += estimate[1] + continue + + # Translating + response = translateText(system, user, history, 0.05, format) + translatedText = response.choices[0].message.content + totalTokens[0] += response.usage.prompt_tokens + totalTokens[1] += response.usage.completion_tokens + + # Check Translation + translatedText = cleanTranslatedText(translatedText, varResponse) + if isinstance(tItem, list): + extractedTranslations = extractTranslation(translatedText, True) + if extractedTranslations == None or len(tItem) != len(extractedTranslations): + # Mismatch. Try Again + response = translateText(system, user, history, 0.05, format, "gpt-4o") + translatedText = response.choices[0].message.content + totalTokens[0] += response.usage.prompt_tokens + totalTokens[1] += response.usage.completion_tokens + + # Formatting + translatedText = cleanTranslatedText(translatedText, varResponse) + if isinstance(tItem, list): + extractedTranslations = extractTranslation(translatedText, True) + if extractedTranslations == None or len(tItem) != len(extractedTranslations): + mismatch = True # Just here for breakpoint + logFile.write(f"Input:\n{subbedT}\n") + logFile.write(f"Output:\n{translatedText}\n") + + # Set if no mismatch + if mismatch == False: + tList[index] = extractedTranslations + history = extractedTranslations[-MAXHISTORY:] # Update history if we have a list + else: + history = text[-MAXHISTORY:] + mismatch = False + if FILENAME not in MISMATCH: + MISMATCH.append(FILENAME) + + # Update Loading Bar + with LOCK: + if PBAR is not None: + PBAR.update(len(tItem)) + else: + # Ensure we're passing a single string to extractTranslation + tList[index] = translatedText.replace("Placeholder Text", "") + + # Combine if multilist + if isinstance(tList[0], list): + tList = [t for sublist in tList for t in sublist] + + # Return + if format == "json": + return [tList, totalTokens] + else: + return [tList[0], totalTokens] else: - return [tList[0], totalTokens] + return [text, [0,0]] diff --git a/modules/renpy.py b/modules/renpy.py index 699a8c4..0dd8fc2 100644 --- a/modules/renpy.py +++ b/modules/renpy.py @@ -448,97 +448,100 @@ def countTokens(system, user, history): @retry(exceptions=Exception, tries=5, delay=5) def translateGPT(text, history, fullPromptFlag): global PBAR, MISMATCH, FILENAME - with open("log/translationHistory.txt", "a+", encoding="utf-8") as logFile: - mismatch = False - totalTokens = [0, 0] - if isinstance(text, list): - format = "json" - tList = batchList(text, BATCHSIZE) - else: - format = "text" - tList = [text] - - for index, tItem in enumerate(tList): - # Before sending to translation, if we have a list of items, add the formatting - if isinstance(tItem, list): - for j in range(len(tItem)): - if not tItem[j]: - tItem[j] = tItem[j].replace("", "Placeholder Text") - payload = {f"Line{i+1}": string for i, string in enumerate(tItem)} - payload = json.dumps(payload, indent=4, ensure_ascii=False) - varResponse = [payload, []] - subbedT = varResponse[0] + if text: + with open("log/translationHistory.txt", "a+", encoding="utf-8") as logFile: + mismatch = False + totalTokens = [0, 0] + if isinstance(text, list): + format = "json" + tList = batchList(text, BATCHSIZE) else: - varResponse = [tItem, []] - subbedT = varResponse[0] + format = "text" + tList = [text] - # Things to Check before starting translation - if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+", subbedT): - if PBAR is not None: - PBAR.update(len(tItem)) - history = tItem[-MAXHISTORY:] - continue - - # Create Message - system, user = createContext(fullPromptFlag, subbedT, format) - - # Calculate Estimate - if ESTIMATE: - estimate = countTokens(system, user, history) - totalTokens[0] += estimate[0] - totalTokens[1] += estimate[1] - continue - - # Translating - response = translateText(system, user, history, 0.05, format) - translatedText = response.choices[0].message.content - totalTokens[0] += response.usage.prompt_tokens - totalTokens[1] += response.usage.completion_tokens - - # Check Translation - translatedText = cleanTranslatedText(translatedText, varResponse) - if isinstance(tItem, list): - extractedTranslations = extractTranslation(translatedText, True) - if extractedTranslations == None or len(tItem) != len(extractedTranslations): - # Mismatch. Try Again - response = translateText(system, user, history, 0.05, format, "gpt-4o") - translatedText = response.choices[0].message.content - totalTokens[0] += response.usage.prompt_tokens - totalTokens[1] += response.usage.completion_tokens - - # Formatting - translatedText = cleanTranslatedText(translatedText, varResponse) - if isinstance(tItem, list): - extractedTranslations = extractTranslation(translatedText, True) - if extractedTranslations == None or len(tItem) != len(extractedTranslations): - mismatch = True # Just here for breakpoint - logFile.write(f"Input:\n{subbedT}\n") - logFile.write(f"Output:\n{translatedText}\n") - - # Set if no mismatch - if mismatch == False: - tList[index] = extractedTranslations - history = extractedTranslations[-MAXHISTORY:] # Update history if we have a list + for index, tItem in enumerate(tList): + # Before sending to translation, if we have a list of items, add the formatting + if isinstance(tItem, list): + for j in range(len(tItem)): + if not tItem[j]: + tItem[j] = tItem[j].replace("", "Placeholder Text") + payload = {f"Line{i+1}": string for i, string in enumerate(tItem)} + payload = json.dumps(payload, indent=4, ensure_ascii=False) + varResponse = [payload, []] + subbedT = varResponse[0] else: - history = text[-MAXHISTORY:] - mismatch = False - if FILENAME not in MISMATCH: - MISMATCH.append(FILENAME) + varResponse = [tItem, []] + subbedT = varResponse[0] - # Update Loading Bar - with LOCK: + # Things to Check before starting translation + if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+", subbedT): if PBAR is not None: PBAR.update(len(tItem)) - else: - # Ensure we're passing a single string to extractTranslation - tList[index] = translatedText.replace("Placeholder Text", "") + history = tItem[-MAXHISTORY:] + continue - # Combine if multilist - if isinstance(tList[0], list): - tList = [t for sublist in tList for t in sublist] + # Create Message + system, user = createContext(fullPromptFlag, subbedT, format) - # Return - if format == "json": - return [tList, totalTokens] + # Calculate Estimate + if ESTIMATE: + estimate = countTokens(system, user, history) + totalTokens[0] += estimate[0] + totalTokens[1] += estimate[1] + continue + + # Translating + response = translateText(system, user, history, 0.05, format) + translatedText = response.choices[0].message.content + totalTokens[0] += response.usage.prompt_tokens + totalTokens[1] += response.usage.completion_tokens + + # Check Translation + translatedText = cleanTranslatedText(translatedText, varResponse) + if isinstance(tItem, list): + extractedTranslations = extractTranslation(translatedText, True) + if extractedTranslations == None or len(tItem) != len(extractedTranslations): + # Mismatch. Try Again + response = translateText(system, user, history, 0.05, format, "gpt-4o") + translatedText = response.choices[0].message.content + totalTokens[0] += response.usage.prompt_tokens + totalTokens[1] += response.usage.completion_tokens + + # Formatting + translatedText = cleanTranslatedText(translatedText, varResponse) + if isinstance(tItem, list): + extractedTranslations = extractTranslation(translatedText, True) + if extractedTranslations == None or len(tItem) != len(extractedTranslations): + mismatch = True # Just here for breakpoint + logFile.write(f"Input:\n{subbedT}\n") + logFile.write(f"Output:\n{translatedText}\n") + + # Set if no mismatch + if mismatch == False: + tList[index] = extractedTranslations + history = extractedTranslations[-MAXHISTORY:] # Update history if we have a list + else: + history = text[-MAXHISTORY:] + mismatch = False + if FILENAME not in MISMATCH: + MISMATCH.append(FILENAME) + + # Update Loading Bar + with LOCK: + if PBAR is not None: + PBAR.update(len(tItem)) + else: + # Ensure we're passing a single string to extractTranslation + tList[index] = translatedText.replace("Placeholder Text", "") + + # Combine if multilist + if isinstance(tList[0], list): + tList = [t for sublist in tList for t in sublist] + + # Return + if format == "json": + return [tList, totalTokens] + else: + return [tList[0], totalTokens] else: - return [tList[0], totalTokens] + return [text, [0,0]] diff --git a/modules/rpgmakerace.py b/modules/rpgmakerace.py index d566f93..a4f17e3 100644 --- a/modules/rpgmakerace.py +++ b/modules/rpgmakerace.py @@ -2531,97 +2531,100 @@ def countTokens(system, user, history): @retry(exceptions=Exception, tries=5, delay=5) def translateGPT(text, history, fullPromptFlag): global PBAR, MISMATCH, FILENAME - with open("log/translationHistory.txt", "a+", encoding="utf-8") as logFile: - mismatch = False - totalTokens = [0, 0] - if isinstance(text, list): - format = "json" - tList = batchList(text, BATCHSIZE) - else: - format = "text" - tList = [text] - - for index, tItem in enumerate(tList): - # Before sending to translation, if we have a list of items, add the formatting - if isinstance(tItem, list): - for j in range(len(tItem)): - if not tItem[j]: - tItem[j] = tItem[j].replace("", "Placeholder Text") - payload = {f"Line{i+1}": string for i, string in enumerate(tItem)} - payload = json.dumps(payload, indent=4, ensure_ascii=False) - varResponse = [payload, []] - subbedT = varResponse[0] + if text: + with open("log/translationHistory.txt", "a+", encoding="utf-8") as logFile: + mismatch = False + totalTokens = [0, 0] + if isinstance(text, list): + format = "json" + tList = batchList(text, BATCHSIZE) else: - varResponse = [tItem, []] - subbedT = varResponse[0] + format = "text" + tList = [text] - # Things to Check before starting translation - if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+", subbedT): - if PBAR is not None: - PBAR.update(len(tItem)) - history = tItem[-MAXHISTORY:] - continue - - # Create Message - system, user = createContext(fullPromptFlag, subbedT, format) - - # Calculate Estimate - if ESTIMATE: - estimate = countTokens(system, user, history) - totalTokens[0] += estimate[0] - totalTokens[1] += estimate[1] - continue - - # Translating - response = translateText(system, user, history, 0.05, format) - translatedText = response.choices[0].message.content - totalTokens[0] += response.usage.prompt_tokens - totalTokens[1] += response.usage.completion_tokens - - # Check Translation - translatedText = cleanTranslatedText(translatedText, varResponse) - if isinstance(tItem, list): - extractedTranslations = extractTranslation(translatedText, True) - if extractedTranslations == None or len(tItem) != len(extractedTranslations): - # Mismatch. Try Again - response = translateText(system, user, history, 0.05, format, "gpt-4o") - translatedText = response.choices[0].message.content - totalTokens[0] += response.usage.prompt_tokens - totalTokens[1] += response.usage.completion_tokens - - # Formatting - translatedText = cleanTranslatedText(translatedText, varResponse) - if isinstance(tItem, list): - extractedTranslations = extractTranslation(translatedText, True) - if extractedTranslations == None or len(tItem) != len(extractedTranslations): - mismatch = True # Just here for breakpoint - logFile.write(f"Input:\n{subbedT}\n") - logFile.write(f"Output:\n{translatedText}\n") - - # Set if no mismatch - if mismatch == False: - tList[index] = extractedTranslations - history = extractedTranslations[-MAXHISTORY:] # Update history if we have a list + for index, tItem in enumerate(tList): + # Before sending to translation, if we have a list of items, add the formatting + if isinstance(tItem, list): + for j in range(len(tItem)): + if not tItem[j]: + tItem[j] = tItem[j].replace("", "Placeholder Text") + payload = {f"Line{i+1}": string for i, string in enumerate(tItem)} + payload = json.dumps(payload, indent=4, ensure_ascii=False) + varResponse = [payload, []] + subbedT = varResponse[0] else: - history = text[-MAXHISTORY:] - mismatch = False - if FILENAME not in MISMATCH: - MISMATCH.append(FILENAME) + varResponse = [tItem, []] + subbedT = varResponse[0] - # Update Loading Bar - with LOCK: + # Things to Check before starting translation + if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+", subbedT): if PBAR is not None: PBAR.update(len(tItem)) - else: - # Ensure we're passing a single string to extractTranslation - tList[index] = translatedText.replace("Placeholder Text", "") + history = tItem[-MAXHISTORY:] + continue - # Combine if multilist - if isinstance(tList[0], list): - tList = [t for sublist in tList for t in sublist] + # Create Message + system, user = createContext(fullPromptFlag, subbedT, format) - # Return - if format == "json": - return [tList, totalTokens] + # Calculate Estimate + if ESTIMATE: + estimate = countTokens(system, user, history) + totalTokens[0] += estimate[0] + totalTokens[1] += estimate[1] + continue + + # Translating + response = translateText(system, user, history, 0.05, format) + translatedText = response.choices[0].message.content + totalTokens[0] += response.usage.prompt_tokens + totalTokens[1] += response.usage.completion_tokens + + # Check Translation + translatedText = cleanTranslatedText(translatedText, varResponse) + if isinstance(tItem, list): + extractedTranslations = extractTranslation(translatedText, True) + if extractedTranslations == None or len(tItem) != len(extractedTranslations): + # Mismatch. Try Again + response = translateText(system, user, history, 0.05, format, "gpt-4o") + translatedText = response.choices[0].message.content + totalTokens[0] += response.usage.prompt_tokens + totalTokens[1] += response.usage.completion_tokens + + # Formatting + translatedText = cleanTranslatedText(translatedText, varResponse) + if isinstance(tItem, list): + extractedTranslations = extractTranslation(translatedText, True) + if extractedTranslations == None or len(tItem) != len(extractedTranslations): + mismatch = True # Just here for breakpoint + logFile.write(f"Input:\n{subbedT}\n") + logFile.write(f"Output:\n{translatedText}\n") + + # Set if no mismatch + if mismatch == False: + tList[index] = extractedTranslations + history = extractedTranslations[-MAXHISTORY:] # Update history if we have a list + else: + history = text[-MAXHISTORY:] + mismatch = False + if FILENAME not in MISMATCH: + MISMATCH.append(FILENAME) + + # Update Loading Bar + with LOCK: + if PBAR is not None: + PBAR.update(len(tItem)) + else: + # Ensure we're passing a single string to extractTranslation + tList[index] = translatedText.replace("Placeholder Text", "") + + # Combine if multilist + if isinstance(tList[0], list): + tList = [t for sublist in tList for t in sublist] + + # Return + if format == "json": + return [tList, totalTokens] + else: + return [tList[0], totalTokens] else: - return [tList[0], totalTokens] + return [text, [0,0]] diff --git a/modules/rpgmakermvmz.py b/modules/rpgmakermvmz.py index 426d154..6127663 100644 --- a/modules/rpgmakermvmz.py +++ b/modules/rpgmakermvmz.py @@ -2510,97 +2510,100 @@ def countTokens(system, user, history): @retry(exceptions=Exception, tries=5, delay=5) def translateGPT(text, history, fullPromptFlag): global PBAR, MISMATCH, FILENAME - with open("log/translationHistory.txt", "a+", encoding="utf-8") as logFile: - mismatch = False - totalTokens = [0, 0] - if isinstance(text, list): - format = "json" - tList = batchList(text, BATCHSIZE) - else: - format = "text" - tList = [text] - - for index, tItem in enumerate(tList): - # Before sending to translation, if we have a list of items, add the formatting - if isinstance(tItem, list): - for j in range(len(tItem)): - if not tItem[j]: - tItem[j] = tItem[j].replace("", "Placeholder Text") - payload = {f"Line{i+1}": string for i, string in enumerate(tItem)} - payload = json.dumps(payload, indent=4, ensure_ascii=False) - varResponse = [payload, []] - subbedT = varResponse[0] + if text: + with open("log/translationHistory.txt", "a+", encoding="utf-8") as logFile: + mismatch = False + totalTokens = [0, 0] + if isinstance(text, list): + format = "json" + tList = batchList(text, BATCHSIZE) else: - varResponse = [tItem, []] - subbedT = varResponse[0] + format = "text" + tList = [text] - # Things to Check before starting translation - if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+", subbedT): - if PBAR is not None: - PBAR.update(len(tItem)) - history = tItem[-MAXHISTORY:] - continue - - # Create Message - system, user = createContext(fullPromptFlag, subbedT, format) - - # Calculate Estimate - if ESTIMATE: - estimate = countTokens(system, user, history) - totalTokens[0] += estimate[0] - totalTokens[1] += estimate[1] - continue - - # Translating - response = translateText(system, user, history, 0.05, format) - translatedText = response.choices[0].message.content - totalTokens[0] += response.usage.prompt_tokens - totalTokens[1] += response.usage.completion_tokens - - # Check Translation - translatedText = cleanTranslatedText(translatedText, varResponse) - if isinstance(tItem, list): - extractedTranslations = extractTranslation(translatedText, True) - if extractedTranslations == None or len(tItem) != len(extractedTranslations): - # Mismatch. Try Again - response = translateText(system, user, history, 0.05, format, "gpt-4o") - translatedText = response.choices[0].message.content - totalTokens[0] += response.usage.prompt_tokens - totalTokens[1] += response.usage.completion_tokens - - # Formatting - translatedText = cleanTranslatedText(translatedText, varResponse) - if isinstance(tItem, list): - extractedTranslations = extractTranslation(translatedText, True) - if extractedTranslations == None or len(tItem) != len(extractedTranslations): - mismatch = True # Just here for breakpoint - logFile.write(f"Input:\n{subbedT}\n") - logFile.write(f"Output:\n{translatedText}\n") - - # Set if no mismatch - if mismatch == False: - tList[index] = extractedTranslations - history = extractedTranslations[-MAXHISTORY:] # Update history if we have a list + for index, tItem in enumerate(tList): + # Before sending to translation, if we have a list of items, add the formatting + if isinstance(tItem, list): + for j in range(len(tItem)): + if not tItem[j]: + tItem[j] = tItem[j].replace("", "Placeholder Text") + payload = {f"Line{i+1}": string for i, string in enumerate(tItem)} + payload = json.dumps(payload, indent=4, ensure_ascii=False) + varResponse = [payload, []] + subbedT = varResponse[0] else: - history = text[-MAXHISTORY:] - mismatch = False - if FILENAME not in MISMATCH: - MISMATCH.append(FILENAME) + varResponse = [tItem, []] + subbedT = varResponse[0] - # Update Loading Bar - with LOCK: + # Things to Check before starting translation + if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+", subbedT): if PBAR is not None: PBAR.update(len(tItem)) - else: - # Ensure we're passing a single string to extractTranslation - tList[index] = translatedText.replace("Placeholder Text", "") + history = tItem[-MAXHISTORY:] + continue - # Combine if multilist - if isinstance(tList[0], list): - tList = [t for sublist in tList for t in sublist] + # Create Message + system, user = createContext(fullPromptFlag, subbedT, format) - # Return - if format == "json": - return [tList, totalTokens] + # Calculate Estimate + if ESTIMATE: + estimate = countTokens(system, user, history) + totalTokens[0] += estimate[0] + totalTokens[1] += estimate[1] + continue + + # Translating + response = translateText(system, user, history, 0.05, format) + translatedText = response.choices[0].message.content + totalTokens[0] += response.usage.prompt_tokens + totalTokens[1] += response.usage.completion_tokens + + # Check Translation + translatedText = cleanTranslatedText(translatedText, varResponse) + if isinstance(tItem, list): + extractedTranslations = extractTranslation(translatedText, True) + if extractedTranslations == None or len(tItem) != len(extractedTranslations): + # Mismatch. Try Again + response = translateText(system, user, history, 0.05, format, "gpt-4o") + translatedText = response.choices[0].message.content + totalTokens[0] += response.usage.prompt_tokens + totalTokens[1] += response.usage.completion_tokens + + # Formatting + translatedText = cleanTranslatedText(translatedText, varResponse) + if isinstance(tItem, list): + extractedTranslations = extractTranslation(translatedText, True) + if extractedTranslations == None or len(tItem) != len(extractedTranslations): + mismatch = True # Just here for breakpoint + logFile.write(f"Input:\n{subbedT}\n") + logFile.write(f"Output:\n{translatedText}\n") + + # Set if no mismatch + if mismatch == False: + tList[index] = extractedTranslations + history = extractedTranslations[-MAXHISTORY:] # Update history if we have a list + else: + history = text[-MAXHISTORY:] + mismatch = False + if FILENAME not in MISMATCH: + MISMATCH.append(FILENAME) + + # Update Loading Bar + with LOCK: + if PBAR is not None: + PBAR.update(len(tItem)) + else: + # Ensure we're passing a single string to extractTranslation + tList[index] = translatedText.replace("Placeholder Text", "") + + # Combine if multilist + if isinstance(tList[0], list): + tList = [t for sublist in tList for t in sublist] + + # Return + if format == "json": + return [tList, totalTokens] + else: + return [tList[0], totalTokens] else: - return [tList[0], totalTokens] + return [text, [0,0]] diff --git a/modules/rpgmakerplugin.py b/modules/rpgmakerplugin.py index 7ac94c5..4a49b2f 100644 --- a/modules/rpgmakerplugin.py +++ b/modules/rpgmakerplugin.py @@ -439,97 +439,100 @@ def countTokens(system, user, history): @retry(exceptions=Exception, tries=5, delay=5) def translateGPT(text, history, fullPromptFlag): global PBAR, MISMATCH, FILENAME - with open("log/translationHistory.txt", "a+", encoding="utf-8") as logFile: - mismatch = False - totalTokens = [0, 0] - if isinstance(text, list): - format = "json" - tList = batchList(text, BATCHSIZE) - else: - format = "text" - tList = [text] - - for index, tItem in enumerate(tList): - # Before sending to translation, if we have a list of items, add the formatting - if isinstance(tItem, list): - for j in range(len(tItem)): - if not tItem[j]: - tItem[j] = tItem[j].replace("", "Placeholder Text") - payload = {f"Line{i+1}": string for i, string in enumerate(tItem)} - payload = json.dumps(payload, indent=4, ensure_ascii=False) - varResponse = [payload, []] - subbedT = varResponse[0] + if text: + with open("log/translationHistory.txt", "a+", encoding="utf-8") as logFile: + mismatch = False + totalTokens = [0, 0] + if isinstance(text, list): + format = "json" + tList = batchList(text, BATCHSIZE) else: - varResponse = [tItem, []] - subbedT = varResponse[0] + format = "text" + tList = [text] - # Things to Check before starting translation - if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+", subbedT): - if PBAR is not None: - PBAR.update(len(tItem)) - history = tItem[-MAXHISTORY:] - continue - - # Create Message - system, user = createContext(fullPromptFlag, subbedT, format) - - # Calculate Estimate - if ESTIMATE: - estimate = countTokens(system, user, history) - totalTokens[0] += estimate[0] - totalTokens[1] += estimate[1] - continue - - # Translating - response = translateText(system, user, history, 0.05, format) - translatedText = response.choices[0].message.content - totalTokens[0] += response.usage.prompt_tokens - totalTokens[1] += response.usage.completion_tokens - - # Check Translation - translatedText = cleanTranslatedText(translatedText, varResponse) - if isinstance(tItem, list): - extractedTranslations = extractTranslation(translatedText, True) - if extractedTranslations == None or len(tItem) != len(extractedTranslations): - # Mismatch. Try Again - response = translateText(system, user, history, 0.05, format, "gpt-4o") - translatedText = response.choices[0].message.content - totalTokens[0] += response.usage.prompt_tokens - totalTokens[1] += response.usage.completion_tokens - - # Formatting - translatedText = cleanTranslatedText(translatedText, varResponse) - if isinstance(tItem, list): - extractedTranslations = extractTranslation(translatedText, True) - if extractedTranslations == None or len(tItem) != len(extractedTranslations): - mismatch = True # Just here for breakpoint - logFile.write(f"Input:\n{subbedT}\n") - logFile.write(f"Output:\n{translatedText}\n") - - # Set if no mismatch - if mismatch == False: - tList[index] = extractedTranslations - history = extractedTranslations[-MAXHISTORY:] # Update history if we have a list + for index, tItem in enumerate(tList): + # Before sending to translation, if we have a list of items, add the formatting + if isinstance(tItem, list): + for j in range(len(tItem)): + if not tItem[j]: + tItem[j] = tItem[j].replace("", "Placeholder Text") + payload = {f"Line{i+1}": string for i, string in enumerate(tItem)} + payload = json.dumps(payload, indent=4, ensure_ascii=False) + varResponse = [payload, []] + subbedT = varResponse[0] else: - history = text[-MAXHISTORY:] - mismatch = False - if FILENAME not in MISMATCH: - MISMATCH.append(FILENAME) + varResponse = [tItem, []] + subbedT = varResponse[0] - # Update Loading Bar - with LOCK: + # Things to Check before starting translation + if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+", subbedT): if PBAR is not None: PBAR.update(len(tItem)) - else: - # Ensure we're passing a single string to extractTranslation - tList[index] = translatedText.replace("Placeholder Text", "") + history = tItem[-MAXHISTORY:] + continue - # Combine if multilist - if isinstance(tList[0], list): - tList = [t for sublist in tList for t in sublist] + # Create Message + system, user = createContext(fullPromptFlag, subbedT, format) - # Return - if format == "json": - return [tList, totalTokens] + # Calculate Estimate + if ESTIMATE: + estimate = countTokens(system, user, history) + totalTokens[0] += estimate[0] + totalTokens[1] += estimate[1] + continue + + # Translating + response = translateText(system, user, history, 0.05, format) + translatedText = response.choices[0].message.content + totalTokens[0] += response.usage.prompt_tokens + totalTokens[1] += response.usage.completion_tokens + + # Check Translation + translatedText = cleanTranslatedText(translatedText, varResponse) + if isinstance(tItem, list): + extractedTranslations = extractTranslation(translatedText, True) + if extractedTranslations == None or len(tItem) != len(extractedTranslations): + # Mismatch. Try Again + response = translateText(system, user, history, 0.05, format, "gpt-4o") + translatedText = response.choices[0].message.content + totalTokens[0] += response.usage.prompt_tokens + totalTokens[1] += response.usage.completion_tokens + + # Formatting + translatedText = cleanTranslatedText(translatedText, varResponse) + if isinstance(tItem, list): + extractedTranslations = extractTranslation(translatedText, True) + if extractedTranslations == None or len(tItem) != len(extractedTranslations): + mismatch = True # Just here for breakpoint + logFile.write(f"Input:\n{subbedT}\n") + logFile.write(f"Output:\n{translatedText}\n") + + # Set if no mismatch + if mismatch == False: + tList[index] = extractedTranslations + history = extractedTranslations[-MAXHISTORY:] # Update history if we have a list + else: + history = text[-MAXHISTORY:] + mismatch = False + if FILENAME not in MISMATCH: + MISMATCH.append(FILENAME) + + # Update Loading Bar + with LOCK: + if PBAR is not None: + PBAR.update(len(tItem)) + else: + # Ensure we're passing a single string to extractTranslation + tList[index] = translatedText.replace("Placeholder Text", "") + + # Combine if multilist + if isinstance(tList[0], list): + tList = [t for sublist in tList for t in sublist] + + # Return + if format == "json": + return [tList, totalTokens] + else: + return [tList[0], totalTokens] else: - return [tList[0], totalTokens] + return [text, [0,0]] diff --git a/modules/text.py b/modules/text.py index af710df..d4702bd 100644 --- a/modules/text.py +++ b/modules/text.py @@ -437,97 +437,100 @@ def countTokens(system, user, history): @retry(exceptions=Exception, tries=5, delay=5) def translateGPT(text, history, fullPromptFlag): global PBAR, MISMATCH, FILENAME - with open("log/translationHistory.txt", "a+", encoding="utf-8") as logFile: - mismatch = False - totalTokens = [0, 0] - if isinstance(text, list): - format = "json" - tList = batchList(text, BATCHSIZE) - else: - format = "text" - tList = [text] - - for index, tItem in enumerate(tList): - # Before sending to translation, if we have a list of items, add the formatting - if isinstance(tItem, list): - for j in range(len(tItem)): - if not tItem[j]: - tItem[j] = tItem[j].replace("", "Placeholder Text") - payload = {f"Line{i+1}": string for i, string in enumerate(tItem)} - payload = json.dumps(payload, indent=4, ensure_ascii=False) - varResponse = [payload, []] - subbedT = varResponse[0] + if text: + with open("log/translationHistory.txt", "a+", encoding="utf-8") as logFile: + mismatch = False + totalTokens = [0, 0] + if isinstance(text, list): + format = "json" + tList = batchList(text, BATCHSIZE) else: - varResponse = [tItem, []] - subbedT = varResponse[0] + format = "text" + tList = [text] - # Things to Check before starting translation - if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+", subbedT): - if PBAR is not None: - PBAR.update(len(tItem)) - history = tItem[-MAXHISTORY:] - continue - - # Create Message - system, user = createContext(fullPromptFlag, subbedT, format) - - # Calculate Estimate - if ESTIMATE: - estimate = countTokens(system, user, history) - totalTokens[0] += estimate[0] - totalTokens[1] += estimate[1] - continue - - # Translating - response = translateText(system, user, history, 0.05, format) - translatedText = response.choices[0].message.content - totalTokens[0] += response.usage.prompt_tokens - totalTokens[1] += response.usage.completion_tokens - - # Check Translation - translatedText = cleanTranslatedText(translatedText, varResponse) - if isinstance(tItem, list): - extractedTranslations = extractTranslation(translatedText, True) - if extractedTranslations == None or len(tItem) != len(extractedTranslations): - # Mismatch. Try Again - response = translateText(system, user, history, 0.05, format, "gpt-4o") - translatedText = response.choices[0].message.content - totalTokens[0] += response.usage.prompt_tokens - totalTokens[1] += response.usage.completion_tokens - - # Formatting - translatedText = cleanTranslatedText(translatedText, varResponse) - if isinstance(tItem, list): - extractedTranslations = extractTranslation(translatedText, True) - if extractedTranslations == None or len(tItem) != len(extractedTranslations): - mismatch = True # Just here for breakpoint - logFile.write(f"Input:\n{subbedT}\n") - logFile.write(f"Output:\n{translatedText}\n") - - # Set if no mismatch - if mismatch == False: - tList[index] = extractedTranslations - history = extractedTranslations[-MAXHISTORY:] # Update history if we have a list + for index, tItem in enumerate(tList): + # Before sending to translation, if we have a list of items, add the formatting + if isinstance(tItem, list): + for j in range(len(tItem)): + if not tItem[j]: + tItem[j] = tItem[j].replace("", "Placeholder Text") + payload = {f"Line{i+1}": string for i, string in enumerate(tItem)} + payload = json.dumps(payload, indent=4, ensure_ascii=False) + varResponse = [payload, []] + subbedT = varResponse[0] else: - history = text[-MAXHISTORY:] - mismatch = False - if FILENAME not in MISMATCH: - MISMATCH.append(FILENAME) + varResponse = [tItem, []] + subbedT = varResponse[0] - # Update Loading Bar - with LOCK: + # Things to Check before starting translation + if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+", subbedT): if PBAR is not None: PBAR.update(len(tItem)) - else: - # Ensure we're passing a single string to extractTranslation - tList[index] = translatedText.replace("Placeholder Text", "") + history = tItem[-MAXHISTORY:] + continue - # Combine if multilist - if isinstance(tList[0], list): - tList = [t for sublist in tList for t in sublist] + # Create Message + system, user = createContext(fullPromptFlag, subbedT, format) - # Return - if format == "json": - return [tList, totalTokens] + # Calculate Estimate + if ESTIMATE: + estimate = countTokens(system, user, history) + totalTokens[0] += estimate[0] + totalTokens[1] += estimate[1] + continue + + # Translating + response = translateText(system, user, history, 0.05, format) + translatedText = response.choices[0].message.content + totalTokens[0] += response.usage.prompt_tokens + totalTokens[1] += response.usage.completion_tokens + + # Check Translation + translatedText = cleanTranslatedText(translatedText, varResponse) + if isinstance(tItem, list): + extractedTranslations = extractTranslation(translatedText, True) + if extractedTranslations == None or len(tItem) != len(extractedTranslations): + # Mismatch. Try Again + response = translateText(system, user, history, 0.05, format, "gpt-4o") + translatedText = response.choices[0].message.content + totalTokens[0] += response.usage.prompt_tokens + totalTokens[1] += response.usage.completion_tokens + + # Formatting + translatedText = cleanTranslatedText(translatedText, varResponse) + if isinstance(tItem, list): + extractedTranslations = extractTranslation(translatedText, True) + if extractedTranslations == None or len(tItem) != len(extractedTranslations): + mismatch = True # Just here for breakpoint + logFile.write(f"Input:\n{subbedT}\n") + logFile.write(f"Output:\n{translatedText}\n") + + # Set if no mismatch + if mismatch == False: + tList[index] = extractedTranslations + history = extractedTranslations[-MAXHISTORY:] # Update history if we have a list + else: + history = text[-MAXHISTORY:] + mismatch = False + if FILENAME not in MISMATCH: + MISMATCH.append(FILENAME) + + # Update Loading Bar + with LOCK: + if PBAR is not None: + PBAR.update(len(tItem)) + else: + # Ensure we're passing a single string to extractTranslation + tList[index] = translatedText.replace("Placeholder Text", "") + + # Combine if multilist + if isinstance(tList[0], list): + tList = [t for sublist in tList for t in sublist] + + # Return + if format == "json": + return [tList, totalTokens] + else: + return [tList[0], totalTokens] else: - return [tList[0], totalTokens] + return [text, [0,0]] diff --git a/modules/tyrano.py b/modules/tyrano.py index 7e9ff55..515327d 100644 --- a/modules/tyrano.py +++ b/modules/tyrano.py @@ -511,97 +511,100 @@ def countTokens(system, user, history): @retry(exceptions=Exception, tries=5, delay=5) def translateGPT(text, history, fullPromptFlag): global PBAR, MISMATCH, FILENAME - with open("log/translationHistory.txt", "a+", encoding="utf-8") as logFile: - mismatch = False - totalTokens = [0, 0] - if isinstance(text, list): - format = "json" - tList = batchList(text, BATCHSIZE) - else: - format = "text" - tList = [text] - - for index, tItem in enumerate(tList): - # Before sending to translation, if we have a list of items, add the formatting - if isinstance(tItem, list): - for j in range(len(tItem)): - if not tItem[j]: - tItem[j] = tItem[j].replace("", "Placeholder Text") - payload = {f"Line{i+1}": string for i, string in enumerate(tItem)} - payload = json.dumps(payload, indent=4, ensure_ascii=False) - varResponse = [payload, []] - subbedT = varResponse[0] + if text: + with open("log/translationHistory.txt", "a+", encoding="utf-8") as logFile: + mismatch = False + totalTokens = [0, 0] + if isinstance(text, list): + format = "json" + tList = batchList(text, BATCHSIZE) else: - varResponse = [tItem, []] - subbedT = varResponse[0] + format = "text" + tList = [text] - # Things to Check before starting translation - if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+", subbedT): - if PBAR is not None: - PBAR.update(len(tItem)) - history = tItem[-MAXHISTORY:] - continue - - # Create Message - system, user = createContext(fullPromptFlag, subbedT, format) - - # Calculate Estimate - if ESTIMATE: - estimate = countTokens(system, user, history) - totalTokens[0] += estimate[0] - totalTokens[1] += estimate[1] - continue - - # Translating - response = translateText(system, user, history, 0.05, format) - translatedText = response.choices[0].message.content - totalTokens[0] += response.usage.prompt_tokens - totalTokens[1] += response.usage.completion_tokens - - # Check Translation - translatedText = cleanTranslatedText(translatedText, varResponse) - if isinstance(tItem, list): - extractedTranslations = extractTranslation(translatedText, True) - if extractedTranslations == None or len(tItem) != len(extractedTranslations): - # Mismatch. Try Again - response = translateText(system, user, history, 0.05, format, "gpt-4o") - translatedText = response.choices[0].message.content - totalTokens[0] += response.usage.prompt_tokens - totalTokens[1] += response.usage.completion_tokens - - # Formatting - translatedText = cleanTranslatedText(translatedText, varResponse) - if isinstance(tItem, list): - extractedTranslations = extractTranslation(translatedText, True) - if extractedTranslations == None or len(tItem) != len(extractedTranslations): - mismatch = True # Just here for breakpoint - logFile.write(f"Input:\n{subbedT}\n") - logFile.write(f"Output:\n{translatedText}\n") - - # Set if no mismatch - if mismatch == False: - tList[index] = extractedTranslations - history = extractedTranslations[-MAXHISTORY:] # Update history if we have a list + for index, tItem in enumerate(tList): + # Before sending to translation, if we have a list of items, add the formatting + if isinstance(tItem, list): + for j in range(len(tItem)): + if not tItem[j]: + tItem[j] = tItem[j].replace("", "Placeholder Text") + payload = {f"Line{i+1}": string for i, string in enumerate(tItem)} + payload = json.dumps(payload, indent=4, ensure_ascii=False) + varResponse = [payload, []] + subbedT = varResponse[0] else: - history = text[-MAXHISTORY:] - mismatch = False - if FILENAME not in MISMATCH: - MISMATCH.append(FILENAME) + varResponse = [tItem, []] + subbedT = varResponse[0] - # Update Loading Bar - with LOCK: + # Things to Check before starting translation + if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+", subbedT): if PBAR is not None: PBAR.update(len(tItem)) - else: - # Ensure we're passing a single string to extractTranslation - tList[index] = translatedText.replace("Placeholder Text", "") + history = tItem[-MAXHISTORY:] + continue - # Combine if multilist - if isinstance(tList[0], list): - tList = [t for sublist in tList for t in sublist] + # Create Message + system, user = createContext(fullPromptFlag, subbedT, format) - # Return - if format == "json": - return [tList, totalTokens] + # Calculate Estimate + if ESTIMATE: + estimate = countTokens(system, user, history) + totalTokens[0] += estimate[0] + totalTokens[1] += estimate[1] + continue + + # Translating + response = translateText(system, user, history, 0.05, format) + translatedText = response.choices[0].message.content + totalTokens[0] += response.usage.prompt_tokens + totalTokens[1] += response.usage.completion_tokens + + # Check Translation + translatedText = cleanTranslatedText(translatedText, varResponse) + if isinstance(tItem, list): + extractedTranslations = extractTranslation(translatedText, True) + if extractedTranslations == None or len(tItem) != len(extractedTranslations): + # Mismatch. Try Again + response = translateText(system, user, history, 0.05, format, "gpt-4o") + translatedText = response.choices[0].message.content + totalTokens[0] += response.usage.prompt_tokens + totalTokens[1] += response.usage.completion_tokens + + # Formatting + translatedText = cleanTranslatedText(translatedText, varResponse) + if isinstance(tItem, list): + extractedTranslations = extractTranslation(translatedText, True) + if extractedTranslations == None or len(tItem) != len(extractedTranslations): + mismatch = True # Just here for breakpoint + logFile.write(f"Input:\n{subbedT}\n") + logFile.write(f"Output:\n{translatedText}\n") + + # Set if no mismatch + if mismatch == False: + tList[index] = extractedTranslations + history = extractedTranslations[-MAXHISTORY:] # Update history if we have a list + else: + history = text[-MAXHISTORY:] + mismatch = False + if FILENAME not in MISMATCH: + MISMATCH.append(FILENAME) + + # Update Loading Bar + with LOCK: + if PBAR is not None: + PBAR.update(len(tItem)) + else: + # Ensure we're passing a single string to extractTranslation + tList[index] = translatedText.replace("Placeholder Text", "") + + # Combine if multilist + if isinstance(tList[0], list): + tList = [t for sublist in tList for t in sublist] + + # Return + if format == "json": + return [tList, totalTokens] + else: + return [tList[0], totalTokens] else: - return [tList[0], totalTokens] + return [text, [0,0]] diff --git a/modules/wolf.py b/modules/wolf.py index 2bd99e7..75e76ae 100644 --- a/modules/wolf.py +++ b/modules/wolf.py @@ -2313,97 +2313,100 @@ def countTokens(system, user, history): @retry(exceptions=Exception, tries=5, delay=5) def translateGPT(text, history, fullPromptFlag): global PBAR, MISMATCH, FILENAME - with open("log/translationHistory.txt", "a+", encoding="utf-8") as logFile: - mismatch = False - totalTokens = [0, 0] - if isinstance(text, list): - format = "json" - tList = batchList(text, BATCHSIZE) - else: - format = "text" - tList = [text] - - for index, tItem in enumerate(tList): - # Before sending to translation, if we have a list of items, add the formatting - if isinstance(tItem, list): - for j in range(len(tItem)): - if not tItem[j]: - tItem[j] = tItem[j].replace("", "Placeholder Text") - payload = {f"Line{i+1}": string for i, string in enumerate(tItem)} - payload = json.dumps(payload, indent=4, ensure_ascii=False) - varResponse = [payload, []] - subbedT = varResponse[0] + if text: + with open("log/translationHistory.txt", "a+", encoding="utf-8") as logFile: + mismatch = False + totalTokens = [0, 0] + if isinstance(text, list): + format = "json" + tList = batchList(text, BATCHSIZE) else: - varResponse = [tItem, []] - subbedT = varResponse[0] + format = "text" + tList = [text] - # Things to Check before starting translation - if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+", subbedT): - if PBAR is not None: - PBAR.update(len(tItem)) - history = tItem[-MAXHISTORY:] - continue - - # Create Message - system, user = createContext(fullPromptFlag, subbedT, format) - - # Calculate Estimate - if ESTIMATE: - estimate = countTokens(system, user, history) - totalTokens[0] += estimate[0] - totalTokens[1] += estimate[1] - continue - - # Translating - response = translateText(system, user, history, 0.05, format) - translatedText = response.choices[0].message.content - totalTokens[0] += response.usage.prompt_tokens - totalTokens[1] += response.usage.completion_tokens - - # Check Translation - translatedText = cleanTranslatedText(translatedText, varResponse) - if isinstance(tItem, list): - extractedTranslations = extractTranslation(translatedText, True) - if extractedTranslations == None or len(tItem) != len(extractedTranslations): - # Mismatch. Try Again - response = translateText(system, user, history, 0.05, format, "gpt-4o") - translatedText = response.choices[0].message.content - totalTokens[0] += response.usage.prompt_tokens - totalTokens[1] += response.usage.completion_tokens - - # Formatting - translatedText = cleanTranslatedText(translatedText, varResponse) - if isinstance(tItem, list): - extractedTranslations = extractTranslation(translatedText, True) - if extractedTranslations == None or len(tItem) != len(extractedTranslations): - mismatch = True # Just here for breakpoint - logFile.write(f"Input:\n{subbedT}\n") - logFile.write(f"Output:\n{translatedText}\n") - - # Set if no mismatch - if mismatch == False: - tList[index] = extractedTranslations - history = extractedTranslations[-MAXHISTORY:] # Update history if we have a list + for index, tItem in enumerate(tList): + # Before sending to translation, if we have a list of items, add the formatting + if isinstance(tItem, list): + for j in range(len(tItem)): + if not tItem[j]: + tItem[j] = tItem[j].replace("", "Placeholder Text") + payload = {f"Line{i+1}": string for i, string in enumerate(tItem)} + payload = json.dumps(payload, indent=4, ensure_ascii=False) + varResponse = [payload, []] + subbedT = varResponse[0] else: - history = text[-MAXHISTORY:] - mismatch = False - if FILENAME not in MISMATCH: - MISMATCH.append(FILENAME) + varResponse = [tItem, []] + subbedT = varResponse[0] - # Update Loading Bar - with LOCK: + # Things to Check before starting translation + if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+", subbedT): if PBAR is not None: PBAR.update(len(tItem)) - else: - # Ensure we're passing a single string to extractTranslation - tList[index] = translatedText.replace("Placeholder Text", "") + history = tItem[-MAXHISTORY:] + continue - # Combine if multilist - if isinstance(tList[0], list): - tList = [t for sublist in tList for t in sublist] + # Create Message + system, user = createContext(fullPromptFlag, subbedT, format) - # Return - if format == "json": - return [tList, totalTokens] + # Calculate Estimate + if ESTIMATE: + estimate = countTokens(system, user, history) + totalTokens[0] += estimate[0] + totalTokens[1] += estimate[1] + continue + + # Translating + response = translateText(system, user, history, 0.05, format) + translatedText = response.choices[0].message.content + totalTokens[0] += response.usage.prompt_tokens + totalTokens[1] += response.usage.completion_tokens + + # Check Translation + translatedText = cleanTranslatedText(translatedText, varResponse) + if isinstance(tItem, list): + extractedTranslations = extractTranslation(translatedText, True) + if extractedTranslations == None or len(tItem) != len(extractedTranslations): + # Mismatch. Try Again + response = translateText(system, user, history, 0.05, format, "gpt-4o") + translatedText = response.choices[0].message.content + totalTokens[0] += response.usage.prompt_tokens + totalTokens[1] += response.usage.completion_tokens + + # Formatting + translatedText = cleanTranslatedText(translatedText, varResponse) + if isinstance(tItem, list): + extractedTranslations = extractTranslation(translatedText, True) + if extractedTranslations == None or len(tItem) != len(extractedTranslations): + mismatch = True # Just here for breakpoint + logFile.write(f"Input:\n{subbedT}\n") + logFile.write(f"Output:\n{translatedText}\n") + + # Set if no mismatch + if mismatch == False: + tList[index] = extractedTranslations + history = extractedTranslations[-MAXHISTORY:] # Update history if we have a list + else: + history = text[-MAXHISTORY:] + mismatch = False + if FILENAME not in MISMATCH: + MISMATCH.append(FILENAME) + + # Update Loading Bar + with LOCK: + if PBAR is not None: + PBAR.update(len(tItem)) + else: + # Ensure we're passing a single string to extractTranslation + tList[index] = translatedText.replace("Placeholder Text", "") + + # Combine if multilist + if isinstance(tList[0], list): + tList = [t for sublist in tList for t in sublist] + + # Return + if format == "json": + return [tList, totalTokens] + else: + return [tList[0], totalTokens] else: - return [tList[0], totalTokens] + return [text, [0,0]] diff --git a/modules/wolf2.py b/modules/wolf2.py index 114870d..8a4f5e6 100644 --- a/modules/wolf2.py +++ b/modules/wolf2.py @@ -476,97 +476,100 @@ def countTokens(system, user, history): @retry(exceptions=Exception, tries=5, delay=5) def translateGPT(text, history, fullPromptFlag): global PBAR, MISMATCH, FILENAME - with open("log/translationHistory.txt", "a+", encoding="utf-8") as logFile: - mismatch = False - totalTokens = [0, 0] - if isinstance(text, list): - format = "json" - tList = batchList(text, BATCHSIZE) - else: - format = "text" - tList = [text] - - for index, tItem in enumerate(tList): - # Before sending to translation, if we have a list of items, add the formatting - if isinstance(tItem, list): - for j in range(len(tItem)): - if not tItem[j]: - tItem[j] = tItem[j].replace("", "Placeholder Text") - payload = {f"Line{i+1}": string for i, string in enumerate(tItem)} - payload = json.dumps(payload, indent=4, ensure_ascii=False) - varResponse = [payload, []] - subbedT = varResponse[0] + if text: + with open("log/translationHistory.txt", "a+", encoding="utf-8") as logFile: + mismatch = False + totalTokens = [0, 0] + if isinstance(text, list): + format = "json" + tList = batchList(text, BATCHSIZE) else: - varResponse = [tItem, []] - subbedT = varResponse[0] + format = "text" + tList = [text] - # Things to Check before starting translation - if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+", subbedT): - if PBAR is not None: - PBAR.update(len(tItem)) - history = tItem[-MAXHISTORY:] - continue - - # Create Message - system, user = createContext(fullPromptFlag, subbedT, format) - - # Calculate Estimate - if ESTIMATE: - estimate = countTokens(system, user, history) - totalTokens[0] += estimate[0] - totalTokens[1] += estimate[1] - continue - - # Translating - response = translateText(system, user, history, 0.05, format) - translatedText = response.choices[0].message.content - totalTokens[0] += response.usage.prompt_tokens - totalTokens[1] += response.usage.completion_tokens - - # Check Translation - translatedText = cleanTranslatedText(translatedText, varResponse) - if isinstance(tItem, list): - extractedTranslations = extractTranslation(translatedText, True) - if extractedTranslations == None or len(tItem) != len(extractedTranslations): - # Mismatch. Try Again - response = translateText(system, user, history, 0.05, format, "gpt-4o") - translatedText = response.choices[0].message.content - totalTokens[0] += response.usage.prompt_tokens - totalTokens[1] += response.usage.completion_tokens - - # Formatting - translatedText = cleanTranslatedText(translatedText, varResponse) - if isinstance(tItem, list): - extractedTranslations = extractTranslation(translatedText, True) - if extractedTranslations == None or len(tItem) != len(extractedTranslations): - mismatch = True # Just here for breakpoint - logFile.write(f"Input:\n{subbedT}\n") - logFile.write(f"Output:\n{translatedText}\n") - - # Set if no mismatch - if mismatch == False: - tList[index] = extractedTranslations - history = extractedTranslations[-MAXHISTORY:] # Update history if we have a list + for index, tItem in enumerate(tList): + # Before sending to translation, if we have a list of items, add the formatting + if isinstance(tItem, list): + for j in range(len(tItem)): + if not tItem[j]: + tItem[j] = tItem[j].replace("", "Placeholder Text") + payload = {f"Line{i+1}": string for i, string in enumerate(tItem)} + payload = json.dumps(payload, indent=4, ensure_ascii=False) + varResponse = [payload, []] + subbedT = varResponse[0] else: - history = text[-MAXHISTORY:] - mismatch = False - if FILENAME not in MISMATCH: - MISMATCH.append(FILENAME) + varResponse = [tItem, []] + subbedT = varResponse[0] - # Update Loading Bar - with LOCK: + # Things to Check before starting translation + if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+", subbedT): if PBAR is not None: PBAR.update(len(tItem)) - else: - # Ensure we're passing a single string to extractTranslation - tList[index] = translatedText.replace("Placeholder Text", "") + history = tItem[-MAXHISTORY:] + continue - # Combine if multilist - if isinstance(tList[0], list): - tList = [t for sublist in tList for t in sublist] + # Create Message + system, user = createContext(fullPromptFlag, subbedT, format) - # Return - if format == "json": - return [tList, totalTokens] + # Calculate Estimate + if ESTIMATE: + estimate = countTokens(system, user, history) + totalTokens[0] += estimate[0] + totalTokens[1] += estimate[1] + continue + + # Translating + response = translateText(system, user, history, 0.05, format) + translatedText = response.choices[0].message.content + totalTokens[0] += response.usage.prompt_tokens + totalTokens[1] += response.usage.completion_tokens + + # Check Translation + translatedText = cleanTranslatedText(translatedText, varResponse) + if isinstance(tItem, list): + extractedTranslations = extractTranslation(translatedText, True) + if extractedTranslations == None or len(tItem) != len(extractedTranslations): + # Mismatch. Try Again + response = translateText(system, user, history, 0.05, format, "gpt-4o") + translatedText = response.choices[0].message.content + totalTokens[0] += response.usage.prompt_tokens + totalTokens[1] += response.usage.completion_tokens + + # Formatting + translatedText = cleanTranslatedText(translatedText, varResponse) + if isinstance(tItem, list): + extractedTranslations = extractTranslation(translatedText, True) + if extractedTranslations == None or len(tItem) != len(extractedTranslations): + mismatch = True # Just here for breakpoint + logFile.write(f"Input:\n{subbedT}\n") + logFile.write(f"Output:\n{translatedText}\n") + + # Set if no mismatch + if mismatch == False: + tList[index] = extractedTranslations + history = extractedTranslations[-MAXHISTORY:] # Update history if we have a list + else: + history = text[-MAXHISTORY:] + mismatch = False + if FILENAME not in MISMATCH: + MISMATCH.append(FILENAME) + + # Update Loading Bar + with LOCK: + if PBAR is not None: + PBAR.update(len(tItem)) + else: + # Ensure we're passing a single string to extractTranslation + tList[index] = translatedText.replace("Placeholder Text", "") + + # Combine if multilist + if isinstance(tList[0], list): + tList = [t for sublist in tList for t in sublist] + + # Return + if format == "json": + return [tList, totalTokens] + else: + return [tList[0], totalTokens] else: - return [tList[0], totalTokens] + return [text, [0,0]] From 7f3e0d8fa4339349a7d7e0645165b22ce727cecf Mon Sep 17 00:00:00 2001 From: DazedAnon Date: Tue, 24 Dec 2024 15:23:35 -0600 Subject: [PATCH 2/3] Don't bother sending description list if its empty --- modules/rpgmakermvmz.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/modules/rpgmakermvmz.py b/modules/rpgmakermvmz.py index 6127663..c6047b0 100644 --- a/modules/rpgmakermvmz.py +++ b/modules/rpgmakermvmz.py @@ -579,7 +579,7 @@ def searchNames(data, pbar, context): if context in ["Skills"]: if len(nameList) < BATCHSIZE: nameList.append(data[i]["name"]) - if "description" in data[i] and data[i]["description"] != "": + if "description" in data[i] and data[i]["description"]: descriptionList.append(data[i]["description"].replace("\n", " ")) # Messages @@ -694,14 +694,15 @@ def searchNames(data, pbar, context): totalTokens[1] += response[1][1] # Description - response = translateGPT( - descriptionList, - f"Reply with only the {LANGUAGE} translation of the text.", - True, - ) - translatedDescriptionBatch = response[0] - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] + if descriptionList: + response = translateGPT( + descriptionList, + f"Reply with only the {LANGUAGE} translation of the text.", + True, + ) + translatedDescriptionBatch = response[0] + totalTokens[0] += response[1][0] + totalTokens[1] += response[1][1] # Set Data if len(nameList) == len(translatedNameBatch): From 915ffd9b84abdd4a06ba46c38ad167295e4c3cf4 Mon Sep 17 00:00:00 2001 From: DazedAnon Date: Tue, 24 Dec 2024 15:28:03 -0600 Subject: [PATCH 3/3] Don't forget formatting --- modules/csv.py | 4 ++-- modules/json.py | 2 +- modules/kirikiri.py | 2 +- modules/nscript.py | 2 +- modules/renpy.py | 2 +- modules/rpgmakerace.py | 2 +- modules/rpgmakermvmz.py | 2 +- modules/rpgmakerplugin.py | 2 +- modules/text.py | 2 +- modules/tyrano.py | 2 +- modules/wolf.py | 2 +- modules/wolf2.py | 2 +- 12 files changed, 13 insertions(+), 13 deletions(-) diff --git a/modules/csv.py b/modules/csv.py index 8c76ebc..8ca8a2e 100644 --- a/modules/csv.py +++ b/modules/csv.py @@ -166,7 +166,7 @@ def parseCSV(readFile, writeFile, filename): format = "3" case "4": format = "4" - + # Write to file for later use with open("csv.tmp", "w", encoding="utf-8") as tmpFile: tmpFile.write(f"{format}") @@ -693,4 +693,4 @@ def translateGPT(text, history, fullPromptFlag): else: return [tList[0], totalTokens] else: - return [text, [0,0]] + return [text, [0, 0]] diff --git a/modules/json.py b/modules/json.py index a0309de..b5ea104 100644 --- a/modules/json.py +++ b/modules/json.py @@ -561,4 +561,4 @@ def translateGPT(text, history, fullPromptFlag): else: return [tList[0], totalTokens] else: - return [text, [0,0]] + return [text, [0, 0]] diff --git a/modules/kirikiri.py b/modules/kirikiri.py index 6488180..857902b 100644 --- a/modules/kirikiri.py +++ b/modules/kirikiri.py @@ -604,4 +604,4 @@ def translateGPT(text, history, fullPromptFlag): else: return [tList[0], totalTokens] else: - return [text, [0,0]] + return [text, [0, 0]] diff --git a/modules/nscript.py b/modules/nscript.py index aaa2dd9..59cb31d 100644 --- a/modules/nscript.py +++ b/modules/nscript.py @@ -620,4 +620,4 @@ def translateGPT(text, history, fullPromptFlag): else: return [tList[0], totalTokens] else: - return [text, [0,0]] + return [text, [0, 0]] diff --git a/modules/renpy.py b/modules/renpy.py index 0dd8fc2..0135308 100644 --- a/modules/renpy.py +++ b/modules/renpy.py @@ -544,4 +544,4 @@ def translateGPT(text, history, fullPromptFlag): else: return [tList[0], totalTokens] else: - return [text, [0,0]] + return [text, [0, 0]] diff --git a/modules/rpgmakerace.py b/modules/rpgmakerace.py index a4f17e3..e596d22 100644 --- a/modules/rpgmakerace.py +++ b/modules/rpgmakerace.py @@ -2627,4 +2627,4 @@ def translateGPT(text, history, fullPromptFlag): else: return [tList[0], totalTokens] else: - return [text, [0,0]] + return [text, [0, 0]] diff --git a/modules/rpgmakermvmz.py b/modules/rpgmakermvmz.py index c6047b0..041f359 100644 --- a/modules/rpgmakermvmz.py +++ b/modules/rpgmakermvmz.py @@ -2607,4 +2607,4 @@ def translateGPT(text, history, fullPromptFlag): else: return [tList[0], totalTokens] else: - return [text, [0,0]] + return [text, [0, 0]] diff --git a/modules/rpgmakerplugin.py b/modules/rpgmakerplugin.py index 4a49b2f..377e9ef 100644 --- a/modules/rpgmakerplugin.py +++ b/modules/rpgmakerplugin.py @@ -535,4 +535,4 @@ def translateGPT(text, history, fullPromptFlag): else: return [tList[0], totalTokens] else: - return [text, [0,0]] + return [text, [0, 0]] diff --git a/modules/text.py b/modules/text.py index d4702bd..4ec6cc3 100644 --- a/modules/text.py +++ b/modules/text.py @@ -533,4 +533,4 @@ def translateGPT(text, history, fullPromptFlag): else: return [tList[0], totalTokens] else: - return [text, [0,0]] + return [text, [0, 0]] diff --git a/modules/tyrano.py b/modules/tyrano.py index 515327d..1556ee6 100644 --- a/modules/tyrano.py +++ b/modules/tyrano.py @@ -607,4 +607,4 @@ def translateGPT(text, history, fullPromptFlag): else: return [tList[0], totalTokens] else: - return [text, [0,0]] + return [text, [0, 0]] diff --git a/modules/wolf.py b/modules/wolf.py index 75e76ae..9933b3e 100644 --- a/modules/wolf.py +++ b/modules/wolf.py @@ -2409,4 +2409,4 @@ def translateGPT(text, history, fullPromptFlag): else: return [tList[0], totalTokens] else: - return [text, [0,0]] + return [text, [0, 0]] diff --git a/modules/wolf2.py b/modules/wolf2.py index 8a4f5e6..77a9ae1 100644 --- a/modules/wolf2.py +++ b/modules/wolf2.py @@ -572,4 +572,4 @@ def translateGPT(text, history, fullPromptFlag): else: return [tList[0], totalTokens] else: - return [text, [0,0]] + return [text, [0, 0]]