diff --git a/modules/csv.py b/modules/csv.py index 3287b25..a94f795 100644 --- a/modules/csv.py +++ b/modules/csv.py @@ -171,7 +171,7 @@ def parseCSV(readFile, writeFile, filename): format = "3" case "4": format = "4" - + # Write to file for later use with open("csv.tmp", "w", encoding="utf-8") as tmpFile: tmpFile.write(f"{format}") @@ -602,97 +602,100 @@ def countTokens(system, user, history): @retry(exceptions=Exception, tries=5, delay=5) def translateGPT(text, history, fullPromptFlag): global PBAR, MISMATCH, FILENAME - with open("log/translationHistory.txt", "a+", encoding="utf-8") as logFile: - mismatch = False - totalTokens = [0, 0] - if isinstance(text, list): - format = "json" - tList = batchList(text, BATCHSIZE) - else: - format = "text" - tList = [text] - - for index, tItem in enumerate(tList): - # Before sending to translation, if we have a list of items, add the formatting - if isinstance(tItem, list): - for j in range(len(tItem)): - if not tItem[j]: - tItem[j] = tItem[j].replace("", "Placeholder Text") - payload = {f"Line{i+1}": string for i, string in enumerate(tItem)} - payload = json.dumps(payload, indent=4, ensure_ascii=False) - varResponse = [payload, []] - subbedT = varResponse[0] + if text: + with open("log/translationHistory.txt", "a+", encoding="utf-8") as logFile: + mismatch = False + totalTokens = [0, 0] + if isinstance(text, list): + format = "json" + tList = batchList(text, BATCHSIZE) else: - varResponse = [tItem, []] - subbedT = varResponse[0] + format = "text" + tList = [text] - # Things to Check before starting translation - if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+", subbedT): - if PBAR is not None: - PBAR.update(len(tItem)) - history = tItem[-MAXHISTORY:] - continue - - # Create Message - system, user = createContext(fullPromptFlag, subbedT, format) - - # Calculate Estimate - if ESTIMATE: - estimate = countTokens(system, user, history) - totalTokens[0] += estimate[0] - totalTokens[1] += estimate[1] - continue - - # Translating - response = translateText(system, user, history, 0.05, format) - translatedText = response.choices[0].message.content - totalTokens[0] += response.usage.prompt_tokens - totalTokens[1] += response.usage.completion_tokens - - # Check Translation - translatedText = cleanTranslatedText(translatedText, varResponse) - if isinstance(tItem, list): - extractedTranslations = extractTranslation(translatedText, True) - if extractedTranslations == None or len(tItem) != len(extractedTranslations): - # Mismatch. Try Again - response = translateText(system, user, history, 0.05, format, "gpt-4o") - translatedText = response.choices[0].message.content - totalTokens[0] += response.usage.prompt_tokens - totalTokens[1] += response.usage.completion_tokens - - # Formatting - translatedText = cleanTranslatedText(translatedText, varResponse) - if isinstance(tItem, list): - extractedTranslations = extractTranslation(translatedText, True) - if extractedTranslations == None or len(tItem) != len(extractedTranslations): - mismatch = True # Just here for breakpoint - logFile.write(f"Input:\n{subbedT}\n") - logFile.write(f"Output:\n{translatedText}\n") - - # Set if no mismatch - if mismatch == False: - tList[index] = extractedTranslations - history = extractedTranslations[-MAXHISTORY:] # Update history if we have a list + for index, tItem in enumerate(tList): + # Before sending to translation, if we have a list of items, add the formatting + if isinstance(tItem, list): + for j in range(len(tItem)): + if not tItem[j]: + tItem[j] = tItem[j].replace("", "Placeholder Text") + payload = {f"Line{i+1}": string for i, string in enumerate(tItem)} + payload = json.dumps(payload, indent=4, ensure_ascii=False) + varResponse = [payload, []] + subbedT = varResponse[0] else: - history = text[-MAXHISTORY:] - mismatch = False - if FILENAME not in MISMATCH: - MISMATCH.append(FILENAME) + varResponse = [tItem, []] + subbedT = varResponse[0] - # Update Loading Bar - with LOCK: + # Things to Check before starting translation + if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+", subbedT): if PBAR is not None: PBAR.update(len(tItem)) - else: - # Ensure we're passing a single string to extractTranslation - tList[index] = translatedText.replace("Placeholder Text", "") + history = tItem[-MAXHISTORY:] + continue - # Combine if multilist - if isinstance(tList[0], list): - tList = [t for sublist in tList for t in sublist] + # Create Message + system, user = createContext(fullPromptFlag, subbedT, format) - # Return - if format == "json": - return [tList, totalTokens] + # Calculate Estimate + if ESTIMATE: + estimate = countTokens(system, user, history) + totalTokens[0] += estimate[0] + totalTokens[1] += estimate[1] + continue + + # Translating + response = translateText(system, user, history, 0.05, format) + translatedText = response.choices[0].message.content + totalTokens[0] += response.usage.prompt_tokens + totalTokens[1] += response.usage.completion_tokens + + # Check Translation + translatedText = cleanTranslatedText(translatedText, varResponse) + if isinstance(tItem, list): + extractedTranslations = extractTranslation(translatedText, True) + if extractedTranslations == None or len(tItem) != len(extractedTranslations): + # Mismatch. Try Again + response = translateText(system, user, history, 0.05, format, "gpt-4o") + translatedText = response.choices[0].message.content + totalTokens[0] += response.usage.prompt_tokens + totalTokens[1] += response.usage.completion_tokens + + # Formatting + translatedText = cleanTranslatedText(translatedText, varResponse) + if isinstance(tItem, list): + extractedTranslations = extractTranslation(translatedText, True) + if extractedTranslations == None or len(tItem) != len(extractedTranslations): + mismatch = True # Just here for breakpoint + logFile.write(f"Input:\n{subbedT}\n") + logFile.write(f"Output:\n{translatedText}\n") + + # Set if no mismatch + if mismatch == False: + tList[index] = extractedTranslations + history = extractedTranslations[-MAXHISTORY:] # Update history if we have a list + else: + history = text[-MAXHISTORY:] + mismatch = False + if FILENAME not in MISMATCH: + MISMATCH.append(FILENAME) + + # Update Loading Bar + with LOCK: + if PBAR is not None: + PBAR.update(len(tItem)) + else: + # Ensure we're passing a single string to extractTranslation + tList[index] = translatedText.replace("Placeholder Text", "") + + # Combine if multilist + if isinstance(tList[0], list): + tList = [t for sublist in tList for t in sublist] + + # Return + if format == "json": + return [tList, totalTokens] + else: + return [tList[0], totalTokens] else: - return [tList[0], totalTokens] + return [text, [0, 0]] diff --git a/modules/json.py b/modules/json.py index 3bbbc35..4338e8f 100644 --- a/modules/json.py +++ b/modules/json.py @@ -469,97 +469,100 @@ def countTokens(system, user, history): @retry(exceptions=Exception, tries=5, delay=5) def translateGPT(text, history, fullPromptFlag): global PBAR, MISMATCH, FILENAME - with open("log/translationHistory.txt", "a+", encoding="utf-8") as logFile: - mismatch = False - totalTokens = [0, 0] - if isinstance(text, list): - format = "json" - tList = batchList(text, BATCHSIZE) - else: - format = "text" - tList = [text] - - for index, tItem in enumerate(tList): - # Before sending to translation, if we have a list of items, add the formatting - if isinstance(tItem, list): - for j in range(len(tItem)): - if not tItem[j]: - tItem[j] = tItem[j].replace("", "Placeholder Text") - payload = {f"Line{i+1}": string for i, string in enumerate(tItem)} - payload = json.dumps(payload, indent=4, ensure_ascii=False) - varResponse = [payload, []] - subbedT = varResponse[0] + if text: + with open("log/translationHistory.txt", "a+", encoding="utf-8") as logFile: + mismatch = False + totalTokens = [0, 0] + if isinstance(text, list): + format = "json" + tList = batchList(text, BATCHSIZE) else: - varResponse = [tItem, []] - subbedT = varResponse[0] + format = "text" + tList = [text] - # Things to Check before starting translation - if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+", subbedT): - if PBAR is not None: - PBAR.update(len(tItem)) - history = tItem[-MAXHISTORY:] - continue - - # Create Message - system, user = createContext(fullPromptFlag, subbedT, format) - - # Calculate Estimate - if ESTIMATE: - estimate = countTokens(system, user, history) - totalTokens[0] += estimate[0] - totalTokens[1] += estimate[1] - continue - - # Translating - response = translateText(system, user, history, 0.05, format) - translatedText = response.choices[0].message.content - totalTokens[0] += response.usage.prompt_tokens - totalTokens[1] += response.usage.completion_tokens - - # Check Translation - translatedText = cleanTranslatedText(translatedText, varResponse) - if isinstance(tItem, list): - extractedTranslations = extractTranslation(translatedText, True) - if extractedTranslations == None or len(tItem) != len(extractedTranslations): - # Mismatch. Try Again - response = translateText(system, user, history, 0.05, format, "gpt-4o") - translatedText = response.choices[0].message.content - totalTokens[0] += response.usage.prompt_tokens - totalTokens[1] += response.usage.completion_tokens - - # Formatting - translatedText = cleanTranslatedText(translatedText, varResponse) - if isinstance(tItem, list): - extractedTranslations = extractTranslation(translatedText, True) - if extractedTranslations == None or len(tItem) != len(extractedTranslations): - mismatch = True # Just here for breakpoint - logFile.write(f"Input:\n{subbedT}\n") - logFile.write(f"Output:\n{translatedText}\n") - - # Set if no mismatch - if mismatch == False: - tList[index] = extractedTranslations - history = extractedTranslations[-MAXHISTORY:] # Update history if we have a list + for index, tItem in enumerate(tList): + # Before sending to translation, if we have a list of items, add the formatting + if isinstance(tItem, list): + for j in range(len(tItem)): + if not tItem[j]: + tItem[j] = tItem[j].replace("", "Placeholder Text") + payload = {f"Line{i+1}": string for i, string in enumerate(tItem)} + payload = json.dumps(payload, indent=4, ensure_ascii=False) + varResponse = [payload, []] + subbedT = varResponse[0] else: - history = text[-MAXHISTORY:] - mismatch = False - if FILENAME not in MISMATCH: - MISMATCH.append(FILENAME) + varResponse = [tItem, []] + subbedT = varResponse[0] - # Update Loading Bar - with LOCK: + # Things to Check before starting translation + if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+", subbedT): if PBAR is not None: PBAR.update(len(tItem)) - else: - # Ensure we're passing a single string to extractTranslation - tList[index] = translatedText.replace("Placeholder Text", "") + history = tItem[-MAXHISTORY:] + continue - # Combine if multilist - if isinstance(tList[0], list): - tList = [t for sublist in tList for t in sublist] + # Create Message + system, user = createContext(fullPromptFlag, subbedT, format) - # Return - if format == "json": - return [tList, totalTokens] + # Calculate Estimate + if ESTIMATE: + estimate = countTokens(system, user, history) + totalTokens[0] += estimate[0] + totalTokens[1] += estimate[1] + continue + + # Translating + response = translateText(system, user, history, 0.05, format) + translatedText = response.choices[0].message.content + totalTokens[0] += response.usage.prompt_tokens + totalTokens[1] += response.usage.completion_tokens + + # Check Translation + translatedText = cleanTranslatedText(translatedText, varResponse) + if isinstance(tItem, list): + extractedTranslations = extractTranslation(translatedText, True) + if extractedTranslations == None or len(tItem) != len(extractedTranslations): + # Mismatch. Try Again + response = translateText(system, user, history, 0.05, format, "gpt-4o") + translatedText = response.choices[0].message.content + totalTokens[0] += response.usage.prompt_tokens + totalTokens[1] += response.usage.completion_tokens + + # Formatting + translatedText = cleanTranslatedText(translatedText, varResponse) + if isinstance(tItem, list): + extractedTranslations = extractTranslation(translatedText, True) + if extractedTranslations == None or len(tItem) != len(extractedTranslations): + mismatch = True # Just here for breakpoint + logFile.write(f"Input:\n{subbedT}\n") + logFile.write(f"Output:\n{translatedText}\n") + + # Set if no mismatch + if mismatch == False: + tList[index] = extractedTranslations + history = extractedTranslations[-MAXHISTORY:] # Update history if we have a list + else: + history = text[-MAXHISTORY:] + mismatch = False + if FILENAME not in MISMATCH: + MISMATCH.append(FILENAME) + + # Update Loading Bar + with LOCK: + if PBAR is not None: + PBAR.update(len(tItem)) + else: + # Ensure we're passing a single string to extractTranslation + tList[index] = translatedText.replace("Placeholder Text", "") + + # Combine if multilist + if isinstance(tList[0], list): + tList = [t for sublist in tList for t in sublist] + + # Return + if format == "json": + return [tList, totalTokens] + else: + return [tList[0], totalTokens] else: - return [tList[0], totalTokens] + return [text, [0, 0]] diff --git a/modules/kirikiri.py b/modules/kirikiri.py index 7c7ff59..de94b09 100644 --- a/modules/kirikiri.py +++ b/modules/kirikiri.py @@ -512,97 +512,100 @@ def countTokens(system, user, history): @retry(exceptions=Exception, tries=5, delay=5) def translateGPT(text, history, fullPromptFlag): global PBAR, MISMATCH, FILENAME - with open("log/translationHistory.txt", "a+", encoding="utf-8") as logFile: - mismatch = False - totalTokens = [0, 0] - if isinstance(text, list): - format = "json" - tList = batchList(text, BATCHSIZE) - else: - format = "text" - tList = [text] - - for index, tItem in enumerate(tList): - # Before sending to translation, if we have a list of items, add the formatting - if isinstance(tItem, list): - for j in range(len(tItem)): - if not tItem[j]: - tItem[j] = tItem[j].replace("", "Placeholder Text") - payload = {f"Line{i+1}": string for i, string in enumerate(tItem)} - payload = json.dumps(payload, indent=4, ensure_ascii=False) - varResponse = [payload, []] - subbedT = varResponse[0] + if text: + with open("log/translationHistory.txt", "a+", encoding="utf-8") as logFile: + mismatch = False + totalTokens = [0, 0] + if isinstance(text, list): + format = "json" + tList = batchList(text, BATCHSIZE) else: - varResponse = [tItem, []] - subbedT = varResponse[0] + format = "text" + tList = [text] - # Things to Check before starting translation - if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+", subbedT): - if PBAR is not None: - PBAR.update(len(tItem)) - history = tItem[-MAXHISTORY:] - continue - - # Create Message - system, user = createContext(fullPromptFlag, subbedT, format) - - # Calculate Estimate - if ESTIMATE: - estimate = countTokens(system, user, history) - totalTokens[0] += estimate[0] - totalTokens[1] += estimate[1] - continue - - # Translating - response = translateText(system, user, history, 0.05, format) - translatedText = response.choices[0].message.content - totalTokens[0] += response.usage.prompt_tokens - totalTokens[1] += response.usage.completion_tokens - - # Check Translation - translatedText = cleanTranslatedText(translatedText, varResponse) - if isinstance(tItem, list): - extractedTranslations = extractTranslation(translatedText, True) - if extractedTranslations == None or len(tItem) != len(extractedTranslations): - # Mismatch. Try Again - response = translateText(system, user, history, 0.05, format, "gpt-4o") - translatedText = response.choices[0].message.content - totalTokens[0] += response.usage.prompt_tokens - totalTokens[1] += response.usage.completion_tokens - - # Formatting - translatedText = cleanTranslatedText(translatedText, varResponse) - if isinstance(tItem, list): - extractedTranslations = extractTranslation(translatedText, True) - if extractedTranslations == None or len(tItem) != len(extractedTranslations): - mismatch = True # Just here for breakpoint - logFile.write(f"Input:\n{subbedT}\n") - logFile.write(f"Output:\n{translatedText}\n") - - # Set if no mismatch - if mismatch == False: - tList[index] = extractedTranslations - history = extractedTranslations[-MAXHISTORY:] # Update history if we have a list + for index, tItem in enumerate(tList): + # Before sending to translation, if we have a list of items, add the formatting + if isinstance(tItem, list): + for j in range(len(tItem)): + if not tItem[j]: + tItem[j] = tItem[j].replace("", "Placeholder Text") + payload = {f"Line{i+1}": string for i, string in enumerate(tItem)} + payload = json.dumps(payload, indent=4, ensure_ascii=False) + varResponse = [payload, []] + subbedT = varResponse[0] else: - history = text[-MAXHISTORY:] - mismatch = False - if FILENAME not in MISMATCH: - MISMATCH.append(FILENAME) + varResponse = [tItem, []] + subbedT = varResponse[0] - # Update Loading Bar - with LOCK: + # Things to Check before starting translation + if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+", subbedT): if PBAR is not None: PBAR.update(len(tItem)) - else: - # Ensure we're passing a single string to extractTranslation - tList[index] = translatedText.replace("Placeholder Text", "") + history = tItem[-MAXHISTORY:] + continue - # Combine if multilist - if isinstance(tList[0], list): - tList = [t for sublist in tList for t in sublist] + # Create Message + system, user = createContext(fullPromptFlag, subbedT, format) - # Return - if format == "json": - return [tList, totalTokens] + # Calculate Estimate + if ESTIMATE: + estimate = countTokens(system, user, history) + totalTokens[0] += estimate[0] + totalTokens[1] += estimate[1] + continue + + # Translating + response = translateText(system, user, history, 0.05, format) + translatedText = response.choices[0].message.content + totalTokens[0] += response.usage.prompt_tokens + totalTokens[1] += response.usage.completion_tokens + + # Check Translation + translatedText = cleanTranslatedText(translatedText, varResponse) + if isinstance(tItem, list): + extractedTranslations = extractTranslation(translatedText, True) + if extractedTranslations == None or len(tItem) != len(extractedTranslations): + # Mismatch. Try Again + response = translateText(system, user, history, 0.05, format, "gpt-4o") + translatedText = response.choices[0].message.content + totalTokens[0] += response.usage.prompt_tokens + totalTokens[1] += response.usage.completion_tokens + + # Formatting + translatedText = cleanTranslatedText(translatedText, varResponse) + if isinstance(tItem, list): + extractedTranslations = extractTranslation(translatedText, True) + if extractedTranslations == None or len(tItem) != len(extractedTranslations): + mismatch = True # Just here for breakpoint + logFile.write(f"Input:\n{subbedT}\n") + logFile.write(f"Output:\n{translatedText}\n") + + # Set if no mismatch + if mismatch == False: + tList[index] = extractedTranslations + history = extractedTranslations[-MAXHISTORY:] # Update history if we have a list + else: + history = text[-MAXHISTORY:] + mismatch = False + if FILENAME not in MISMATCH: + MISMATCH.append(FILENAME) + + # Update Loading Bar + with LOCK: + if PBAR is not None: + PBAR.update(len(tItem)) + else: + # Ensure we're passing a single string to extractTranslation + tList[index] = translatedText.replace("Placeholder Text", "") + + # Combine if multilist + if isinstance(tList[0], list): + tList = [t for sublist in tList for t in sublist] + + # Return + if format == "json": + return [tList, totalTokens] + else: + return [tList[0], totalTokens] else: - return [tList[0], totalTokens] + return [text, [0, 0]] diff --git a/modules/nscript.py b/modules/nscript.py index 6bc7720..0a6a91c 100644 --- a/modules/nscript.py +++ b/modules/nscript.py @@ -529,97 +529,100 @@ def countTokens(system, user, history): @retry(exceptions=Exception, tries=5, delay=5) def translateGPT(text, history, fullPromptFlag): global PBAR, MISMATCH, FILENAME - with open("log/translationHistory.txt", "a+", encoding="utf-8") as logFile: - mismatch = False - totalTokens = [0, 0] - if isinstance(text, list): - format = "json" - tList = batchList(text, BATCHSIZE) - else: - format = "text" - tList = [text] - - for index, tItem in enumerate(tList): - # Before sending to translation, if we have a list of items, add the formatting - if isinstance(tItem, list): - for j in range(len(tItem)): - if not tItem[j]: - tItem[j] = tItem[j].replace("", "Placeholder Text") - payload = {f"Line{i+1}": string for i, string in enumerate(tItem)} - payload = json.dumps(payload, indent=4, ensure_ascii=False) - varResponse = [payload, []] - subbedT = varResponse[0] + if text: + with open("log/translationHistory.txt", "a+", encoding="utf-8") as logFile: + mismatch = False + totalTokens = [0, 0] + if isinstance(text, list): + format = "json" + tList = batchList(text, BATCHSIZE) else: - varResponse = [tItem, []] - subbedT = varResponse[0] + format = "text" + tList = [text] - # Things to Check before starting translation - if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+", subbedT): - if PBAR is not None: - PBAR.update(len(tItem)) - history = tItem[-MAXHISTORY:] - continue - - # Create Message - system, user = createContext(fullPromptFlag, subbedT, format) - - # Calculate Estimate - if ESTIMATE: - estimate = countTokens(system, user, history) - totalTokens[0] += estimate[0] - totalTokens[1] += estimate[1] - continue - - # Translating - response = translateText(system, user, history, 0.05, format) - translatedText = response.choices[0].message.content - totalTokens[0] += response.usage.prompt_tokens - totalTokens[1] += response.usage.completion_tokens - - # Check Translation - translatedText = cleanTranslatedText(translatedText, varResponse) - if isinstance(tItem, list): - extractedTranslations = extractTranslation(translatedText, True) - if extractedTranslations == None or len(tItem) != len(extractedTranslations): - # Mismatch. Try Again - response = translateText(system, user, history, 0.05, format, "gpt-4o") - translatedText = response.choices[0].message.content - totalTokens[0] += response.usage.prompt_tokens - totalTokens[1] += response.usage.completion_tokens - - # Formatting - translatedText = cleanTranslatedText(translatedText, varResponse) - if isinstance(tItem, list): - extractedTranslations = extractTranslation(translatedText, True) - if extractedTranslations == None or len(tItem) != len(extractedTranslations): - mismatch = True # Just here for breakpoint - logFile.write(f"Input:\n{subbedT}\n") - logFile.write(f"Output:\n{translatedText}\n") - - # Set if no mismatch - if mismatch == False: - tList[index] = extractedTranslations - history = extractedTranslations[-MAXHISTORY:] # Update history if we have a list + for index, tItem in enumerate(tList): + # Before sending to translation, if we have a list of items, add the formatting + if isinstance(tItem, list): + for j in range(len(tItem)): + if not tItem[j]: + tItem[j] = tItem[j].replace("", "Placeholder Text") + payload = {f"Line{i+1}": string for i, string in enumerate(tItem)} + payload = json.dumps(payload, indent=4, ensure_ascii=False) + varResponse = [payload, []] + subbedT = varResponse[0] else: - history = text[-MAXHISTORY:] - mismatch = False - if FILENAME not in MISMATCH: - MISMATCH.append(FILENAME) + varResponse = [tItem, []] + subbedT = varResponse[0] - # Update Loading Bar - with LOCK: + # Things to Check before starting translation + if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+", subbedT): if PBAR is not None: PBAR.update(len(tItem)) - else: - # Ensure we're passing a single string to extractTranslation - tList[index] = translatedText.replace("Placeholder Text", "") + history = tItem[-MAXHISTORY:] + continue - # Combine if multilist - if isinstance(tList[0], list): - tList = [t for sublist in tList for t in sublist] + # Create Message + system, user = createContext(fullPromptFlag, subbedT, format) - # Return - if format == "json": - return [tList, totalTokens] + # Calculate Estimate + if ESTIMATE: + estimate = countTokens(system, user, history) + totalTokens[0] += estimate[0] + totalTokens[1] += estimate[1] + continue + + # Translating + response = translateText(system, user, history, 0.05, format) + translatedText = response.choices[0].message.content + totalTokens[0] += response.usage.prompt_tokens + totalTokens[1] += response.usage.completion_tokens + + # Check Translation + translatedText = cleanTranslatedText(translatedText, varResponse) + if isinstance(tItem, list): + extractedTranslations = extractTranslation(translatedText, True) + if extractedTranslations == None or len(tItem) != len(extractedTranslations): + # Mismatch. Try Again + response = translateText(system, user, history, 0.05, format, "gpt-4o") + translatedText = response.choices[0].message.content + totalTokens[0] += response.usage.prompt_tokens + totalTokens[1] += response.usage.completion_tokens + + # Formatting + translatedText = cleanTranslatedText(translatedText, varResponse) + if isinstance(tItem, list): + extractedTranslations = extractTranslation(translatedText, True) + if extractedTranslations == None or len(tItem) != len(extractedTranslations): + mismatch = True # Just here for breakpoint + logFile.write(f"Input:\n{subbedT}\n") + logFile.write(f"Output:\n{translatedText}\n") + + # Set if no mismatch + if mismatch == False: + tList[index] = extractedTranslations + history = extractedTranslations[-MAXHISTORY:] # Update history if we have a list + else: + history = text[-MAXHISTORY:] + mismatch = False + if FILENAME not in MISMATCH: + MISMATCH.append(FILENAME) + + # Update Loading Bar + with LOCK: + if PBAR is not None: + PBAR.update(len(tItem)) + else: + # Ensure we're passing a single string to extractTranslation + tList[index] = translatedText.replace("Placeholder Text", "") + + # Combine if multilist + if isinstance(tList[0], list): + tList = [t for sublist in tList for t in sublist] + + # Return + if format == "json": + return [tList, totalTokens] + else: + return [tList[0], totalTokens] else: - return [tList[0], totalTokens] + return [text, [0, 0]] diff --git a/modules/renpy.py b/modules/renpy.py index 6fab545..48029b7 100644 --- a/modules/renpy.py +++ b/modules/renpy.py @@ -453,97 +453,100 @@ def countTokens(system, user, history): @retry(exceptions=Exception, tries=5, delay=5) def translateGPT(text, history, fullPromptFlag): global PBAR, MISMATCH, FILENAME - with open("log/translationHistory.txt", "a+", encoding="utf-8") as logFile: - mismatch = False - totalTokens = [0, 0] - if isinstance(text, list): - format = "json" - tList = batchList(text, BATCHSIZE) - else: - format = "text" - tList = [text] - - for index, tItem in enumerate(tList): - # Before sending to translation, if we have a list of items, add the formatting - if isinstance(tItem, list): - for j in range(len(tItem)): - if not tItem[j]: - tItem[j] = tItem[j].replace("", "Placeholder Text") - payload = {f"Line{i+1}": string for i, string in enumerate(tItem)} - payload = json.dumps(payload, indent=4, ensure_ascii=False) - varResponse = [payload, []] - subbedT = varResponse[0] + if text: + with open("log/translationHistory.txt", "a+", encoding="utf-8") as logFile: + mismatch = False + totalTokens = [0, 0] + if isinstance(text, list): + format = "json" + tList = batchList(text, BATCHSIZE) else: - varResponse = [tItem, []] - subbedT = varResponse[0] + format = "text" + tList = [text] - # Things to Check before starting translation - if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+", subbedT): - if PBAR is not None: - PBAR.update(len(tItem)) - history = tItem[-MAXHISTORY:] - continue - - # Create Message - system, user = createContext(fullPromptFlag, subbedT, format) - - # Calculate Estimate - if ESTIMATE: - estimate = countTokens(system, user, history) - totalTokens[0] += estimate[0] - totalTokens[1] += estimate[1] - continue - - # Translating - response = translateText(system, user, history, 0.05, format) - translatedText = response.choices[0].message.content - totalTokens[0] += response.usage.prompt_tokens - totalTokens[1] += response.usage.completion_tokens - - # Check Translation - translatedText = cleanTranslatedText(translatedText, varResponse) - if isinstance(tItem, list): - extractedTranslations = extractTranslation(translatedText, True) - if extractedTranslations == None or len(tItem) != len(extractedTranslations): - # Mismatch. Try Again - response = translateText(system, user, history, 0.05, format, "gpt-4o") - translatedText = response.choices[0].message.content - totalTokens[0] += response.usage.prompt_tokens - totalTokens[1] += response.usage.completion_tokens - - # Formatting - translatedText = cleanTranslatedText(translatedText, varResponse) - if isinstance(tItem, list): - extractedTranslations = extractTranslation(translatedText, True) - if extractedTranslations == None or len(tItem) != len(extractedTranslations): - mismatch = True # Just here for breakpoint - logFile.write(f"Input:\n{subbedT}\n") - logFile.write(f"Output:\n{translatedText}\n") - - # Set if no mismatch - if mismatch == False: - tList[index] = extractedTranslations - history = extractedTranslations[-MAXHISTORY:] # Update history if we have a list + for index, tItem in enumerate(tList): + # Before sending to translation, if we have a list of items, add the formatting + if isinstance(tItem, list): + for j in range(len(tItem)): + if not tItem[j]: + tItem[j] = tItem[j].replace("", "Placeholder Text") + payload = {f"Line{i+1}": string for i, string in enumerate(tItem)} + payload = json.dumps(payload, indent=4, ensure_ascii=False) + varResponse = [payload, []] + subbedT = varResponse[0] else: - history = text[-MAXHISTORY:] - mismatch = False - if FILENAME not in MISMATCH: - MISMATCH.append(FILENAME) + varResponse = [tItem, []] + subbedT = varResponse[0] - # Update Loading Bar - with LOCK: + # Things to Check before starting translation + if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+", subbedT): if PBAR is not None: PBAR.update(len(tItem)) - else: - # Ensure we're passing a single string to extractTranslation - tList[index] = translatedText.replace("Placeholder Text", "") + history = tItem[-MAXHISTORY:] + continue - # Combine if multilist - if isinstance(tList[0], list): - tList = [t for sublist in tList for t in sublist] + # Create Message + system, user = createContext(fullPromptFlag, subbedT, format) - # Return - if format == "json": - return [tList, totalTokens] + # Calculate Estimate + if ESTIMATE: + estimate = countTokens(system, user, history) + totalTokens[0] += estimate[0] + totalTokens[1] += estimate[1] + continue + + # Translating + response = translateText(system, user, history, 0.05, format) + translatedText = response.choices[0].message.content + totalTokens[0] += response.usage.prompt_tokens + totalTokens[1] += response.usage.completion_tokens + + # Check Translation + translatedText = cleanTranslatedText(translatedText, varResponse) + if isinstance(tItem, list): + extractedTranslations = extractTranslation(translatedText, True) + if extractedTranslations == None or len(tItem) != len(extractedTranslations): + # Mismatch. Try Again + response = translateText(system, user, history, 0.05, format, "gpt-4o") + translatedText = response.choices[0].message.content + totalTokens[0] += response.usage.prompt_tokens + totalTokens[1] += response.usage.completion_tokens + + # Formatting + translatedText = cleanTranslatedText(translatedText, varResponse) + if isinstance(tItem, list): + extractedTranslations = extractTranslation(translatedText, True) + if extractedTranslations == None or len(tItem) != len(extractedTranslations): + mismatch = True # Just here for breakpoint + logFile.write(f"Input:\n{subbedT}\n") + logFile.write(f"Output:\n{translatedText}\n") + + # Set if no mismatch + if mismatch == False: + tList[index] = extractedTranslations + history = extractedTranslations[-MAXHISTORY:] # Update history if we have a list + else: + history = text[-MAXHISTORY:] + mismatch = False + if FILENAME not in MISMATCH: + MISMATCH.append(FILENAME) + + # Update Loading Bar + with LOCK: + if PBAR is not None: + PBAR.update(len(tItem)) + else: + # Ensure we're passing a single string to extractTranslation + tList[index] = translatedText.replace("Placeholder Text", "") + + # Combine if multilist + if isinstance(tList[0], list): + tList = [t for sublist in tList for t in sublist] + + # Return + if format == "json": + return [tList, totalTokens] + else: + return [tList[0], totalTokens] else: - return [tList[0], totalTokens] + return [text, [0, 0]] diff --git a/modules/rpgmakerace.py b/modules/rpgmakerace.py index 897986d..ef6ff27 100644 --- a/modules/rpgmakerace.py +++ b/modules/rpgmakerace.py @@ -2536,97 +2536,100 @@ def countTokens(system, user, history): @retry(exceptions=Exception, tries=5, delay=5) def translateGPT(text, history, fullPromptFlag): global PBAR, MISMATCH, FILENAME - with open("log/translationHistory.txt", "a+", encoding="utf-8") as logFile: - mismatch = False - totalTokens = [0, 0] - if isinstance(text, list): - format = "json" - tList = batchList(text, BATCHSIZE) - else: - format = "text" - tList = [text] - - for index, tItem in enumerate(tList): - # Before sending to translation, if we have a list of items, add the formatting - if isinstance(tItem, list): - for j in range(len(tItem)): - if not tItem[j]: - tItem[j] = tItem[j].replace("", "Placeholder Text") - payload = {f"Line{i+1}": string for i, string in enumerate(tItem)} - payload = json.dumps(payload, indent=4, ensure_ascii=False) - varResponse = [payload, []] - subbedT = varResponse[0] + if text: + with open("log/translationHistory.txt", "a+", encoding="utf-8") as logFile: + mismatch = False + totalTokens = [0, 0] + if isinstance(text, list): + format = "json" + tList = batchList(text, BATCHSIZE) else: - varResponse = [tItem, []] - subbedT = varResponse[0] + format = "text" + tList = [text] - # Things to Check before starting translation - if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+", subbedT): - if PBAR is not None: - PBAR.update(len(tItem)) - history = tItem[-MAXHISTORY:] - continue - - # Create Message - system, user = createContext(fullPromptFlag, subbedT, format) - - # Calculate Estimate - if ESTIMATE: - estimate = countTokens(system, user, history) - totalTokens[0] += estimate[0] - totalTokens[1] += estimate[1] - continue - - # Translating - response = translateText(system, user, history, 0.05, format) - translatedText = response.choices[0].message.content - totalTokens[0] += response.usage.prompt_tokens - totalTokens[1] += response.usage.completion_tokens - - # Check Translation - translatedText = cleanTranslatedText(translatedText, varResponse) - if isinstance(tItem, list): - extractedTranslations = extractTranslation(translatedText, True) - if extractedTranslations == None or len(tItem) != len(extractedTranslations): - # Mismatch. Try Again - response = translateText(system, user, history, 0.05, format, "gpt-4o") - translatedText = response.choices[0].message.content - totalTokens[0] += response.usage.prompt_tokens - totalTokens[1] += response.usage.completion_tokens - - # Formatting - translatedText = cleanTranslatedText(translatedText, varResponse) - if isinstance(tItem, list): - extractedTranslations = extractTranslation(translatedText, True) - if extractedTranslations == None or len(tItem) != len(extractedTranslations): - mismatch = True # Just here for breakpoint - logFile.write(f"Input:\n{subbedT}\n") - logFile.write(f"Output:\n{translatedText}\n") - - # Set if no mismatch - if mismatch == False: - tList[index] = extractedTranslations - history = extractedTranslations[-MAXHISTORY:] # Update history if we have a list + for index, tItem in enumerate(tList): + # Before sending to translation, if we have a list of items, add the formatting + if isinstance(tItem, list): + for j in range(len(tItem)): + if not tItem[j]: + tItem[j] = tItem[j].replace("", "Placeholder Text") + payload = {f"Line{i+1}": string for i, string in enumerate(tItem)} + payload = json.dumps(payload, indent=4, ensure_ascii=False) + varResponse = [payload, []] + subbedT = varResponse[0] else: - history = text[-MAXHISTORY:] - mismatch = False - if FILENAME not in MISMATCH: - MISMATCH.append(FILENAME) + varResponse = [tItem, []] + subbedT = varResponse[0] - # Update Loading Bar - with LOCK: + # Things to Check before starting translation + if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+", subbedT): if PBAR is not None: PBAR.update(len(tItem)) - else: - # Ensure we're passing a single string to extractTranslation - tList[index] = translatedText.replace("Placeholder Text", "") + history = tItem[-MAXHISTORY:] + continue - # Combine if multilist - if isinstance(tList[0], list): - tList = [t for sublist in tList for t in sublist] + # Create Message + system, user = createContext(fullPromptFlag, subbedT, format) - # Return - if format == "json": - return [tList, totalTokens] + # Calculate Estimate + if ESTIMATE: + estimate = countTokens(system, user, history) + totalTokens[0] += estimate[0] + totalTokens[1] += estimate[1] + continue + + # Translating + response = translateText(system, user, history, 0.05, format) + translatedText = response.choices[0].message.content + totalTokens[0] += response.usage.prompt_tokens + totalTokens[1] += response.usage.completion_tokens + + # Check Translation + translatedText = cleanTranslatedText(translatedText, varResponse) + if isinstance(tItem, list): + extractedTranslations = extractTranslation(translatedText, True) + if extractedTranslations == None or len(tItem) != len(extractedTranslations): + # Mismatch. Try Again + response = translateText(system, user, history, 0.05, format, "gpt-4o") + translatedText = response.choices[0].message.content + totalTokens[0] += response.usage.prompt_tokens + totalTokens[1] += response.usage.completion_tokens + + # Formatting + translatedText = cleanTranslatedText(translatedText, varResponse) + if isinstance(tItem, list): + extractedTranslations = extractTranslation(translatedText, True) + if extractedTranslations == None or len(tItem) != len(extractedTranslations): + mismatch = True # Just here for breakpoint + logFile.write(f"Input:\n{subbedT}\n") + logFile.write(f"Output:\n{translatedText}\n") + + # Set if no mismatch + if mismatch == False: + tList[index] = extractedTranslations + history = extractedTranslations[-MAXHISTORY:] # Update history if we have a list + else: + history = text[-MAXHISTORY:] + mismatch = False + if FILENAME not in MISMATCH: + MISMATCH.append(FILENAME) + + # Update Loading Bar + with LOCK: + if PBAR is not None: + PBAR.update(len(tItem)) + else: + # Ensure we're passing a single string to extractTranslation + tList[index] = translatedText.replace("Placeholder Text", "") + + # Combine if multilist + if isinstance(tList[0], list): + tList = [t for sublist in tList for t in sublist] + + # Return + if format == "json": + return [tList, totalTokens] + else: + return [tList[0], totalTokens] else: - return [tList[0], totalTokens] + return [text, [0, 0]] diff --git a/modules/rpgmakermvmz.py b/modules/rpgmakermvmz.py index 513f782..0464f40 100644 --- a/modules/rpgmakermvmz.py +++ b/modules/rpgmakermvmz.py @@ -584,7 +584,7 @@ def searchNames(data, pbar, context): if context in ["Skills"]: if len(nameList) < BATCHSIZE: nameList.append(data[i]["name"]) - if "description" in data[i] and data[i]["description"] != "": + if "description" in data[i] and data[i]["description"]: descriptionList.append(data[i]["description"].replace("\n", " ")) # Messages @@ -2517,97 +2517,100 @@ def countTokens(system, user, history): @retry(exceptions=Exception, tries=5, delay=5) def translateGPT(text, history, fullPromptFlag): global PBAR, MISMATCH, FILENAME - with open("log/translationHistory.txt", "a+", encoding="utf-8") as logFile: - mismatch = False - totalTokens = [0, 0] - if isinstance(text, list): - format = "json" - tList = batchList(text, BATCHSIZE) - else: - format = "text" - tList = [text] - - for index, tItem in enumerate(tList): - # Before sending to translation, if we have a list of items, add the formatting - if isinstance(tItem, list): - for j in range(len(tItem)): - if not tItem[j]: - tItem[j] = tItem[j].replace("", "Placeholder Text") - payload = {f"Line{i+1}": string for i, string in enumerate(tItem)} - payload = json.dumps(payload, indent=4, ensure_ascii=False) - varResponse = [payload, []] - subbedT = varResponse[0] + if text: + with open("log/translationHistory.txt", "a+", encoding="utf-8") as logFile: + mismatch = False + totalTokens = [0, 0] + if isinstance(text, list): + format = "json" + tList = batchList(text, BATCHSIZE) else: - varResponse = [tItem, []] - subbedT = varResponse[0] + format = "text" + tList = [text] - # Things to Check before starting translation - if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+", subbedT): - if PBAR is not None: - PBAR.update(len(tItem)) - history = tItem[-MAXHISTORY:] - continue - - # Create Message - system, user = createContext(fullPromptFlag, subbedT, format) - - # Calculate Estimate - if ESTIMATE: - estimate = countTokens(system, user, history) - totalTokens[0] += estimate[0] - totalTokens[1] += estimate[1] - continue - - # Translating - response = translateText(system, user, history, 0.05, format) - translatedText = response.choices[0].message.content - totalTokens[0] += response.usage.prompt_tokens - totalTokens[1] += response.usage.completion_tokens - - # Check Translation - translatedText = cleanTranslatedText(translatedText, varResponse) - if isinstance(tItem, list): - extractedTranslations = extractTranslation(translatedText, True) - if extractedTranslations == None or len(tItem) != len(extractedTranslations): - # Mismatch. Try Again - response = translateText(system, user, history, 0.05, format, "gpt-4o") - translatedText = response.choices[0].message.content - totalTokens[0] += response.usage.prompt_tokens - totalTokens[1] += response.usage.completion_tokens - - # Formatting - translatedText = cleanTranslatedText(translatedText, varResponse) - if isinstance(tItem, list): - extractedTranslations = extractTranslation(translatedText, True) - if extractedTranslations == None or len(tItem) != len(extractedTranslations): - mismatch = True # Just here for breakpoint - logFile.write(f"Input:\n{subbedT}\n") - logFile.write(f"Output:\n{translatedText}\n") - - # Set if no mismatch - if mismatch == False: - tList[index] = extractedTranslations - history = extractedTranslations[-MAXHISTORY:] # Update history if we have a list + for index, tItem in enumerate(tList): + # Before sending to translation, if we have a list of items, add the formatting + if isinstance(tItem, list): + for j in range(len(tItem)): + if not tItem[j]: + tItem[j] = tItem[j].replace("", "Placeholder Text") + payload = {f"Line{i+1}": string for i, string in enumerate(tItem)} + payload = json.dumps(payload, indent=4, ensure_ascii=False) + varResponse = [payload, []] + subbedT = varResponse[0] else: - history = text[-MAXHISTORY:] - mismatch = False - if FILENAME not in MISMATCH: - MISMATCH.append(FILENAME) + varResponse = [tItem, []] + subbedT = varResponse[0] - # Update Loading Bar - with LOCK: + # Things to Check before starting translation + if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+", subbedT): if PBAR is not None: PBAR.update(len(tItem)) - else: - # Ensure we're passing a single string to extractTranslation - tList[index] = translatedText.replace("Placeholder Text", "") + history = tItem[-MAXHISTORY:] + continue - # Combine if multilist - if isinstance(tList[0], list): - tList = [t for sublist in tList for t in sublist] + # Create Message + system, user = createContext(fullPromptFlag, subbedT, format) - # Return - if format == "json": - return [tList, totalTokens] + # Calculate Estimate + if ESTIMATE: + estimate = countTokens(system, user, history) + totalTokens[0] += estimate[0] + totalTokens[1] += estimate[1] + continue + + # Translating + response = translateText(system, user, history, 0.05, format) + translatedText = response.choices[0].message.content + totalTokens[0] += response.usage.prompt_tokens + totalTokens[1] += response.usage.completion_tokens + + # Check Translation + translatedText = cleanTranslatedText(translatedText, varResponse) + if isinstance(tItem, list): + extractedTranslations = extractTranslation(translatedText, True) + if extractedTranslations == None or len(tItem) != len(extractedTranslations): + # Mismatch. Try Again + response = translateText(system, user, history, 0.05, format, "gpt-4o") + translatedText = response.choices[0].message.content + totalTokens[0] += response.usage.prompt_tokens + totalTokens[1] += response.usage.completion_tokens + + # Formatting + translatedText = cleanTranslatedText(translatedText, varResponse) + if isinstance(tItem, list): + extractedTranslations = extractTranslation(translatedText, True) + if extractedTranslations == None or len(tItem) != len(extractedTranslations): + mismatch = True # Just here for breakpoint + logFile.write(f"Input:\n{subbedT}\n") + logFile.write(f"Output:\n{translatedText}\n") + + # Set if no mismatch + if mismatch == False: + tList[index] = extractedTranslations + history = extractedTranslations[-MAXHISTORY:] # Update history if we have a list + else: + history = text[-MAXHISTORY:] + mismatch = False + if FILENAME not in MISMATCH: + MISMATCH.append(FILENAME) + + # Update Loading Bar + with LOCK: + if PBAR is not None: + PBAR.update(len(tItem)) + else: + # Ensure we're passing a single string to extractTranslation + tList[index] = translatedText.replace("Placeholder Text", "") + + # Combine if multilist + if isinstance(tList[0], list): + tList = [t for sublist in tList for t in sublist] + + # Return + if format == "json": + return [tList, totalTokens] + else: + return [tList[0], totalTokens] else: - return [tList[0], totalTokens] + return [text, [0, 0]] diff --git a/modules/rpgmakerplugin.py b/modules/rpgmakerplugin.py index b22aef9..790685c 100644 --- a/modules/rpgmakerplugin.py +++ b/modules/rpgmakerplugin.py @@ -443,97 +443,100 @@ def countTokens(system, user, history): @retry(exceptions=Exception, tries=5, delay=5) def translateGPT(text, history, fullPromptFlag): global PBAR, MISMATCH, FILENAME - with open("log/translationHistory.txt", "a+", encoding="utf-8") as logFile: - mismatch = False - totalTokens = [0, 0] - if isinstance(text, list): - format = "json" - tList = batchList(text, BATCHSIZE) - else: - format = "text" - tList = [text] - - for index, tItem in enumerate(tList): - # Before sending to translation, if we have a list of items, add the formatting - if isinstance(tItem, list): - for j in range(len(tItem)): - if not tItem[j]: - tItem[j] = tItem[j].replace("", "Placeholder Text") - payload = {f"Line{i+1}": string for i, string in enumerate(tItem)} - payload = json.dumps(payload, indent=4, ensure_ascii=False) - varResponse = [payload, []] - subbedT = varResponse[0] + if text: + with open("log/translationHistory.txt", "a+", encoding="utf-8") as logFile: + mismatch = False + totalTokens = [0, 0] + if isinstance(text, list): + format = "json" + tList = batchList(text, BATCHSIZE) else: - varResponse = [tItem, []] - subbedT = varResponse[0] + format = "text" + tList = [text] - # Things to Check before starting translation - if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+", subbedT): - if PBAR is not None: - PBAR.update(len(tItem)) - history = tItem[-MAXHISTORY:] - continue - - # Create Message - system, user = createContext(fullPromptFlag, subbedT, format) - - # Calculate Estimate - if ESTIMATE: - estimate = countTokens(system, user, history) - totalTokens[0] += estimate[0] - totalTokens[1] += estimate[1] - continue - - # Translating - response = translateText(system, user, history, 0.05, format) - translatedText = response.choices[0].message.content - totalTokens[0] += response.usage.prompt_tokens - totalTokens[1] += response.usage.completion_tokens - - # Check Translation - translatedText = cleanTranslatedText(translatedText, varResponse) - if isinstance(tItem, list): - extractedTranslations = extractTranslation(translatedText, True) - if extractedTranslations == None or len(tItem) != len(extractedTranslations): - # Mismatch. Try Again - response = translateText(system, user, history, 0.05, format, "gpt-4o") - translatedText = response.choices[0].message.content - totalTokens[0] += response.usage.prompt_tokens - totalTokens[1] += response.usage.completion_tokens - - # Formatting - translatedText = cleanTranslatedText(translatedText, varResponse) - if isinstance(tItem, list): - extractedTranslations = extractTranslation(translatedText, True) - if extractedTranslations == None or len(tItem) != len(extractedTranslations): - mismatch = True # Just here for breakpoint - logFile.write(f"Input:\n{subbedT}\n") - logFile.write(f"Output:\n{translatedText}\n") - - # Set if no mismatch - if mismatch == False: - tList[index] = extractedTranslations - history = extractedTranslations[-MAXHISTORY:] # Update history if we have a list + for index, tItem in enumerate(tList): + # Before sending to translation, if we have a list of items, add the formatting + if isinstance(tItem, list): + for j in range(len(tItem)): + if not tItem[j]: + tItem[j] = tItem[j].replace("", "Placeholder Text") + payload = {f"Line{i+1}": string for i, string in enumerate(tItem)} + payload = json.dumps(payload, indent=4, ensure_ascii=False) + varResponse = [payload, []] + subbedT = varResponse[0] else: - history = text[-MAXHISTORY:] - mismatch = False - if FILENAME not in MISMATCH: - MISMATCH.append(FILENAME) + varResponse = [tItem, []] + subbedT = varResponse[0] - # Update Loading Bar - with LOCK: + # Things to Check before starting translation + if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+", subbedT): if PBAR is not None: PBAR.update(len(tItem)) - else: - # Ensure we're passing a single string to extractTranslation - tList[index] = translatedText.replace("Placeholder Text", "") + history = tItem[-MAXHISTORY:] + continue - # Combine if multilist - if isinstance(tList[0], list): - tList = [t for sublist in tList for t in sublist] + # Create Message + system, user = createContext(fullPromptFlag, subbedT, format) - # Return - if format == "json": - return [tList, totalTokens] + # Calculate Estimate + if ESTIMATE: + estimate = countTokens(system, user, history) + totalTokens[0] += estimate[0] + totalTokens[1] += estimate[1] + continue + + # Translating + response = translateText(system, user, history, 0.05, format) + translatedText = response.choices[0].message.content + totalTokens[0] += response.usage.prompt_tokens + totalTokens[1] += response.usage.completion_tokens + + # Check Translation + translatedText = cleanTranslatedText(translatedText, varResponse) + if isinstance(tItem, list): + extractedTranslations = extractTranslation(translatedText, True) + if extractedTranslations == None or len(tItem) != len(extractedTranslations): + # Mismatch. Try Again + response = translateText(system, user, history, 0.05, format, "gpt-4o") + translatedText = response.choices[0].message.content + totalTokens[0] += response.usage.prompt_tokens + totalTokens[1] += response.usage.completion_tokens + + # Formatting + translatedText = cleanTranslatedText(translatedText, varResponse) + if isinstance(tItem, list): + extractedTranslations = extractTranslation(translatedText, True) + if extractedTranslations == None or len(tItem) != len(extractedTranslations): + mismatch = True # Just here for breakpoint + logFile.write(f"Input:\n{subbedT}\n") + logFile.write(f"Output:\n{translatedText}\n") + + # Set if no mismatch + if mismatch == False: + tList[index] = extractedTranslations + history = extractedTranslations[-MAXHISTORY:] # Update history if we have a list + else: + history = text[-MAXHISTORY:] + mismatch = False + if FILENAME not in MISMATCH: + MISMATCH.append(FILENAME) + + # Update Loading Bar + with LOCK: + if PBAR is not None: + PBAR.update(len(tItem)) + else: + # Ensure we're passing a single string to extractTranslation + tList[index] = translatedText.replace("Placeholder Text", "") + + # Combine if multilist + if isinstance(tList[0], list): + tList = [t for sublist in tList for t in sublist] + + # Return + if format == "json": + return [tList, totalTokens] + else: + return [tList[0], totalTokens] else: - return [tList[0], totalTokens] + return [text, [0, 0]] diff --git a/modules/text.py b/modules/text.py index 3654b97..45f9bd1 100644 --- a/modules/text.py +++ b/modules/text.py @@ -441,97 +441,100 @@ def countTokens(system, user, history): @retry(exceptions=Exception, tries=5, delay=5) def translateGPT(text, history, fullPromptFlag): global PBAR, MISMATCH, FILENAME - with open("log/translationHistory.txt", "a+", encoding="utf-8") as logFile: - mismatch = False - totalTokens = [0, 0] - if isinstance(text, list): - format = "json" - tList = batchList(text, BATCHSIZE) - else: - format = "text" - tList = [text] - - for index, tItem in enumerate(tList): - # Before sending to translation, if we have a list of items, add the formatting - if isinstance(tItem, list): - for j in range(len(tItem)): - if not tItem[j]: - tItem[j] = tItem[j].replace("", "Placeholder Text") - payload = {f"Line{i+1}": string for i, string in enumerate(tItem)} - payload = json.dumps(payload, indent=4, ensure_ascii=False) - varResponse = [payload, []] - subbedT = varResponse[0] + if text: + with open("log/translationHistory.txt", "a+", encoding="utf-8") as logFile: + mismatch = False + totalTokens = [0, 0] + if isinstance(text, list): + format = "json" + tList = batchList(text, BATCHSIZE) else: - varResponse = [tItem, []] - subbedT = varResponse[0] + format = "text" + tList = [text] - # Things to Check before starting translation - if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+", subbedT): - if PBAR is not None: - PBAR.update(len(tItem)) - history = tItem[-MAXHISTORY:] - continue - - # Create Message - system, user = createContext(fullPromptFlag, subbedT, format) - - # Calculate Estimate - if ESTIMATE: - estimate = countTokens(system, user, history) - totalTokens[0] += estimate[0] - totalTokens[1] += estimate[1] - continue - - # Translating - response = translateText(system, user, history, 0.05, format) - translatedText = response.choices[0].message.content - totalTokens[0] += response.usage.prompt_tokens - totalTokens[1] += response.usage.completion_tokens - - # Check Translation - translatedText = cleanTranslatedText(translatedText, varResponse) - if isinstance(tItem, list): - extractedTranslations = extractTranslation(translatedText, True) - if extractedTranslations == None or len(tItem) != len(extractedTranslations): - # Mismatch. Try Again - response = translateText(system, user, history, 0.05, format, "gpt-4o") - translatedText = response.choices[0].message.content - totalTokens[0] += response.usage.prompt_tokens - totalTokens[1] += response.usage.completion_tokens - - # Formatting - translatedText = cleanTranslatedText(translatedText, varResponse) - if isinstance(tItem, list): - extractedTranslations = extractTranslation(translatedText, True) - if extractedTranslations == None or len(tItem) != len(extractedTranslations): - mismatch = True # Just here for breakpoint - logFile.write(f"Input:\n{subbedT}\n") - logFile.write(f"Output:\n{translatedText}\n") - - # Set if no mismatch - if mismatch == False: - tList[index] = extractedTranslations - history = extractedTranslations[-MAXHISTORY:] # Update history if we have a list + for index, tItem in enumerate(tList): + # Before sending to translation, if we have a list of items, add the formatting + if isinstance(tItem, list): + for j in range(len(tItem)): + if not tItem[j]: + tItem[j] = tItem[j].replace("", "Placeholder Text") + payload = {f"Line{i+1}": string for i, string in enumerate(tItem)} + payload = json.dumps(payload, indent=4, ensure_ascii=False) + varResponse = [payload, []] + subbedT = varResponse[0] else: - history = text[-MAXHISTORY:] - mismatch = False - if FILENAME not in MISMATCH: - MISMATCH.append(FILENAME) + varResponse = [tItem, []] + subbedT = varResponse[0] - # Update Loading Bar - with LOCK: + # Things to Check before starting translation + if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+", subbedT): if PBAR is not None: PBAR.update(len(tItem)) - else: - # Ensure we're passing a single string to extractTranslation - tList[index] = translatedText.replace("Placeholder Text", "") + history = tItem[-MAXHISTORY:] + continue - # Combine if multilist - if isinstance(tList[0], list): - tList = [t for sublist in tList for t in sublist] + # Create Message + system, user = createContext(fullPromptFlag, subbedT, format) - # Return - if format == "json": - return [tList, totalTokens] + # Calculate Estimate + if ESTIMATE: + estimate = countTokens(system, user, history) + totalTokens[0] += estimate[0] + totalTokens[1] += estimate[1] + continue + + # Translating + response = translateText(system, user, history, 0.05, format) + translatedText = response.choices[0].message.content + totalTokens[0] += response.usage.prompt_tokens + totalTokens[1] += response.usage.completion_tokens + + # Check Translation + translatedText = cleanTranslatedText(translatedText, varResponse) + if isinstance(tItem, list): + extractedTranslations = extractTranslation(translatedText, True) + if extractedTranslations == None or len(tItem) != len(extractedTranslations): + # Mismatch. Try Again + response = translateText(system, user, history, 0.05, format, "gpt-4o") + translatedText = response.choices[0].message.content + totalTokens[0] += response.usage.prompt_tokens + totalTokens[1] += response.usage.completion_tokens + + # Formatting + translatedText = cleanTranslatedText(translatedText, varResponse) + if isinstance(tItem, list): + extractedTranslations = extractTranslation(translatedText, True) + if extractedTranslations == None or len(tItem) != len(extractedTranslations): + mismatch = True # Just here for breakpoint + logFile.write(f"Input:\n{subbedT}\n") + logFile.write(f"Output:\n{translatedText}\n") + + # Set if no mismatch + if mismatch == False: + tList[index] = extractedTranslations + history = extractedTranslations[-MAXHISTORY:] # Update history if we have a list + else: + history = text[-MAXHISTORY:] + mismatch = False + if FILENAME not in MISMATCH: + MISMATCH.append(FILENAME) + + # Update Loading Bar + with LOCK: + if PBAR is not None: + PBAR.update(len(tItem)) + else: + # Ensure we're passing a single string to extractTranslation + tList[index] = translatedText.replace("Placeholder Text", "") + + # Combine if multilist + if isinstance(tList[0], list): + tList = [t for sublist in tList for t in sublist] + + # Return + if format == "json": + return [tList, totalTokens] + else: + return [tList[0], totalTokens] else: - return [tList[0], totalTokens] + return [text, [0, 0]] diff --git a/modules/tyrano.py b/modules/tyrano.py index b653e9c..1b970cd 100644 --- a/modules/tyrano.py +++ b/modules/tyrano.py @@ -515,97 +515,100 @@ def countTokens(system, user, history): @retry(exceptions=Exception, tries=5, delay=5) def translateGPT(text, history, fullPromptFlag): global PBAR, MISMATCH, FILENAME - with open("log/translationHistory.txt", "a+", encoding="utf-8") as logFile: - mismatch = False - totalTokens = [0, 0] - if isinstance(text, list): - format = "json" - tList = batchList(text, BATCHSIZE) - else: - format = "text" - tList = [text] - - for index, tItem in enumerate(tList): - # Before sending to translation, if we have a list of items, add the formatting - if isinstance(tItem, list): - for j in range(len(tItem)): - if not tItem[j]: - tItem[j] = tItem[j].replace("", "Placeholder Text") - payload = {f"Line{i+1}": string for i, string in enumerate(tItem)} - payload = json.dumps(payload, indent=4, ensure_ascii=False) - varResponse = [payload, []] - subbedT = varResponse[0] + if text: + with open("log/translationHistory.txt", "a+", encoding="utf-8") as logFile: + mismatch = False + totalTokens = [0, 0] + if isinstance(text, list): + format = "json" + tList = batchList(text, BATCHSIZE) else: - varResponse = [tItem, []] - subbedT = varResponse[0] + format = "text" + tList = [text] - # Things to Check before starting translation - if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+", subbedT): - if PBAR is not None: - PBAR.update(len(tItem)) - history = tItem[-MAXHISTORY:] - continue - - # Create Message - system, user = createContext(fullPromptFlag, subbedT, format) - - # Calculate Estimate - if ESTIMATE: - estimate = countTokens(system, user, history) - totalTokens[0] += estimate[0] - totalTokens[1] += estimate[1] - continue - - # Translating - response = translateText(system, user, history, 0.05, format) - translatedText = response.choices[0].message.content - totalTokens[0] += response.usage.prompt_tokens - totalTokens[1] += response.usage.completion_tokens - - # Check Translation - translatedText = cleanTranslatedText(translatedText, varResponse) - if isinstance(tItem, list): - extractedTranslations = extractTranslation(translatedText, True) - if extractedTranslations == None or len(tItem) != len(extractedTranslations): - # Mismatch. Try Again - response = translateText(system, user, history, 0.05, format, "gpt-4o") - translatedText = response.choices[0].message.content - totalTokens[0] += response.usage.prompt_tokens - totalTokens[1] += response.usage.completion_tokens - - # Formatting - translatedText = cleanTranslatedText(translatedText, varResponse) - if isinstance(tItem, list): - extractedTranslations = extractTranslation(translatedText, True) - if extractedTranslations == None or len(tItem) != len(extractedTranslations): - mismatch = True # Just here for breakpoint - logFile.write(f"Input:\n{subbedT}\n") - logFile.write(f"Output:\n{translatedText}\n") - - # Set if no mismatch - if mismatch == False: - tList[index] = extractedTranslations - history = extractedTranslations[-MAXHISTORY:] # Update history if we have a list + for index, tItem in enumerate(tList): + # Before sending to translation, if we have a list of items, add the formatting + if isinstance(tItem, list): + for j in range(len(tItem)): + if not tItem[j]: + tItem[j] = tItem[j].replace("", "Placeholder Text") + payload = {f"Line{i+1}": string for i, string in enumerate(tItem)} + payload = json.dumps(payload, indent=4, ensure_ascii=False) + varResponse = [payload, []] + subbedT = varResponse[0] else: - history = text[-MAXHISTORY:] - mismatch = False - if FILENAME not in MISMATCH: - MISMATCH.append(FILENAME) + varResponse = [tItem, []] + subbedT = varResponse[0] - # Update Loading Bar - with LOCK: + # Things to Check before starting translation + if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+", subbedT): if PBAR is not None: PBAR.update(len(tItem)) - else: - # Ensure we're passing a single string to extractTranslation - tList[index] = translatedText.replace("Placeholder Text", "") + history = tItem[-MAXHISTORY:] + continue - # Combine if multilist - if isinstance(tList[0], list): - tList = [t for sublist in tList for t in sublist] + # Create Message + system, user = createContext(fullPromptFlag, subbedT, format) - # Return - if format == "json": - return [tList, totalTokens] + # Calculate Estimate + if ESTIMATE: + estimate = countTokens(system, user, history) + totalTokens[0] += estimate[0] + totalTokens[1] += estimate[1] + continue + + # Translating + response = translateText(system, user, history, 0.05, format) + translatedText = response.choices[0].message.content + totalTokens[0] += response.usage.prompt_tokens + totalTokens[1] += response.usage.completion_tokens + + # Check Translation + translatedText = cleanTranslatedText(translatedText, varResponse) + if isinstance(tItem, list): + extractedTranslations = extractTranslation(translatedText, True) + if extractedTranslations == None or len(tItem) != len(extractedTranslations): + # Mismatch. Try Again + response = translateText(system, user, history, 0.05, format, "gpt-4o") + translatedText = response.choices[0].message.content + totalTokens[0] += response.usage.prompt_tokens + totalTokens[1] += response.usage.completion_tokens + + # Formatting + translatedText = cleanTranslatedText(translatedText, varResponse) + if isinstance(tItem, list): + extractedTranslations = extractTranslation(translatedText, True) + if extractedTranslations == None or len(tItem) != len(extractedTranslations): + mismatch = True # Just here for breakpoint + logFile.write(f"Input:\n{subbedT}\n") + logFile.write(f"Output:\n{translatedText}\n") + + # Set if no mismatch + if mismatch == False: + tList[index] = extractedTranslations + history = extractedTranslations[-MAXHISTORY:] # Update history if we have a list + else: + history = text[-MAXHISTORY:] + mismatch = False + if FILENAME not in MISMATCH: + MISMATCH.append(FILENAME) + + # Update Loading Bar + with LOCK: + if PBAR is not None: + PBAR.update(len(tItem)) + else: + # Ensure we're passing a single string to extractTranslation + tList[index] = translatedText.replace("Placeholder Text", "") + + # Combine if multilist + if isinstance(tList[0], list): + tList = [t for sublist in tList for t in sublist] + + # Return + if format == "json": + return [tList, totalTokens] + else: + return [tList[0], totalTokens] else: - return [tList[0], totalTokens] + return [text, [0, 0]] diff --git a/modules/wolf.py b/modules/wolf.py index 7fd7efd..3b15e2e 100644 --- a/modules/wolf.py +++ b/modules/wolf.py @@ -2318,97 +2318,100 @@ def countTokens(system, user, history): @retry(exceptions=Exception, tries=5, delay=5) def translateGPT(text, history, fullPromptFlag): global PBAR, MISMATCH, FILENAME - with open("log/translationHistory.txt", "a+", encoding="utf-8") as logFile: - mismatch = False - totalTokens = [0, 0] - if isinstance(text, list): - format = "json" - tList = batchList(text, BATCHSIZE) - else: - format = "text" - tList = [text] - - for index, tItem in enumerate(tList): - # Before sending to translation, if we have a list of items, add the formatting - if isinstance(tItem, list): - for j in range(len(tItem)): - if not tItem[j]: - tItem[j] = tItem[j].replace("", "Placeholder Text") - payload = {f"Line{i+1}": string for i, string in enumerate(tItem)} - payload = json.dumps(payload, indent=4, ensure_ascii=False) - varResponse = [payload, []] - subbedT = varResponse[0] + if text: + with open("log/translationHistory.txt", "a+", encoding="utf-8") as logFile: + mismatch = False + totalTokens = [0, 0] + if isinstance(text, list): + format = "json" + tList = batchList(text, BATCHSIZE) else: - varResponse = [tItem, []] - subbedT = varResponse[0] + format = "text" + tList = [text] - # Things to Check before starting translation - if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+", subbedT): - if PBAR is not None: - PBAR.update(len(tItem)) - history = tItem[-MAXHISTORY:] - continue - - # Create Message - system, user = createContext(fullPromptFlag, subbedT, format) - - # Calculate Estimate - if ESTIMATE: - estimate = countTokens(system, user, history) - totalTokens[0] += estimate[0] - totalTokens[1] += estimate[1] - continue - - # Translating - response = translateText(system, user, history, 0.05, format) - translatedText = response.choices[0].message.content - totalTokens[0] += response.usage.prompt_tokens - totalTokens[1] += response.usage.completion_tokens - - # Check Translation - translatedText = cleanTranslatedText(translatedText, varResponse) - if isinstance(tItem, list): - extractedTranslations = extractTranslation(translatedText, True) - if extractedTranslations == None or len(tItem) != len(extractedTranslations): - # Mismatch. Try Again - response = translateText(system, user, history, 0.05, format, "gpt-4o") - translatedText = response.choices[0].message.content - totalTokens[0] += response.usage.prompt_tokens - totalTokens[1] += response.usage.completion_tokens - - # Formatting - translatedText = cleanTranslatedText(translatedText, varResponse) - if isinstance(tItem, list): - extractedTranslations = extractTranslation(translatedText, True) - if extractedTranslations == None or len(tItem) != len(extractedTranslations): - mismatch = True # Just here for breakpoint - logFile.write(f"Input:\n{subbedT}\n") - logFile.write(f"Output:\n{translatedText}\n") - - # Set if no mismatch - if mismatch == False: - tList[index] = extractedTranslations - history = extractedTranslations[-MAXHISTORY:] # Update history if we have a list + for index, tItem in enumerate(tList): + # Before sending to translation, if we have a list of items, add the formatting + if isinstance(tItem, list): + for j in range(len(tItem)): + if not tItem[j]: + tItem[j] = tItem[j].replace("", "Placeholder Text") + payload = {f"Line{i+1}": string for i, string in enumerate(tItem)} + payload = json.dumps(payload, indent=4, ensure_ascii=False) + varResponse = [payload, []] + subbedT = varResponse[0] else: - history = text[-MAXHISTORY:] - mismatch = False - if FILENAME not in MISMATCH: - MISMATCH.append(FILENAME) + varResponse = [tItem, []] + subbedT = varResponse[0] - # Update Loading Bar - with LOCK: + # Things to Check before starting translation + if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+", subbedT): if PBAR is not None: PBAR.update(len(tItem)) - else: - # Ensure we're passing a single string to extractTranslation - tList[index] = translatedText.replace("Placeholder Text", "") + history = tItem[-MAXHISTORY:] + continue - # Combine if multilist - if isinstance(tList[0], list): - tList = [t for sublist in tList for t in sublist] + # Create Message + system, user = createContext(fullPromptFlag, subbedT, format) - # Return - if format == "json": - return [tList, totalTokens] + # Calculate Estimate + if ESTIMATE: + estimate = countTokens(system, user, history) + totalTokens[0] += estimate[0] + totalTokens[1] += estimate[1] + continue + + # Translating + response = translateText(system, user, history, 0.05, format) + translatedText = response.choices[0].message.content + totalTokens[0] += response.usage.prompt_tokens + totalTokens[1] += response.usage.completion_tokens + + # Check Translation + translatedText = cleanTranslatedText(translatedText, varResponse) + if isinstance(tItem, list): + extractedTranslations = extractTranslation(translatedText, True) + if extractedTranslations == None or len(tItem) != len(extractedTranslations): + # Mismatch. Try Again + response = translateText(system, user, history, 0.05, format, "gpt-4o") + translatedText = response.choices[0].message.content + totalTokens[0] += response.usage.prompt_tokens + totalTokens[1] += response.usage.completion_tokens + + # Formatting + translatedText = cleanTranslatedText(translatedText, varResponse) + if isinstance(tItem, list): + extractedTranslations = extractTranslation(translatedText, True) + if extractedTranslations == None or len(tItem) != len(extractedTranslations): + mismatch = True # Just here for breakpoint + logFile.write(f"Input:\n{subbedT}\n") + logFile.write(f"Output:\n{translatedText}\n") + + # Set if no mismatch + if mismatch == False: + tList[index] = extractedTranslations + history = extractedTranslations[-MAXHISTORY:] # Update history if we have a list + else: + history = text[-MAXHISTORY:] + mismatch = False + if FILENAME not in MISMATCH: + MISMATCH.append(FILENAME) + + # Update Loading Bar + with LOCK: + if PBAR is not None: + PBAR.update(len(tItem)) + else: + # Ensure we're passing a single string to extractTranslation + tList[index] = translatedText.replace("Placeholder Text", "") + + # Combine if multilist + if isinstance(tList[0], list): + tList = [t for sublist in tList for t in sublist] + + # Return + if format == "json": + return [tList, totalTokens] + else: + return [tList[0], totalTokens] else: - return [tList[0], totalTokens] + return [text, [0, 0]] diff --git a/modules/wolf2.py b/modules/wolf2.py index f01c74d..76b614e 100644 --- a/modules/wolf2.py +++ b/modules/wolf2.py @@ -480,97 +480,100 @@ def countTokens(system, user, history): @retry(exceptions=Exception, tries=5, delay=5) def translateGPT(text, history, fullPromptFlag): global PBAR, MISMATCH, FILENAME - with open("log/translationHistory.txt", "a+", encoding="utf-8") as logFile: - mismatch = False - totalTokens = [0, 0] - if isinstance(text, list): - format = "json" - tList = batchList(text, BATCHSIZE) - else: - format = "text" - tList = [text] - - for index, tItem in enumerate(tList): - # Before sending to translation, if we have a list of items, add the formatting - if isinstance(tItem, list): - for j in range(len(tItem)): - if not tItem[j]: - tItem[j] = tItem[j].replace("", "Placeholder Text") - payload = {f"Line{i+1}": string for i, string in enumerate(tItem)} - payload = json.dumps(payload, indent=4, ensure_ascii=False) - varResponse = [payload, []] - subbedT = varResponse[0] + if text: + with open("log/translationHistory.txt", "a+", encoding="utf-8") as logFile: + mismatch = False + totalTokens = [0, 0] + if isinstance(text, list): + format = "json" + tList = batchList(text, BATCHSIZE) else: - varResponse = [tItem, []] - subbedT = varResponse[0] + format = "text" + tList = [text] - # Things to Check before starting translation - if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+", subbedT): - if PBAR is not None: - PBAR.update(len(tItem)) - history = tItem[-MAXHISTORY:] - continue - - # Create Message - system, user = createContext(fullPromptFlag, subbedT, format) - - # Calculate Estimate - if ESTIMATE: - estimate = countTokens(system, user, history) - totalTokens[0] += estimate[0] - totalTokens[1] += estimate[1] - continue - - # Translating - response = translateText(system, user, history, 0.05, format) - translatedText = response.choices[0].message.content - totalTokens[0] += response.usage.prompt_tokens - totalTokens[1] += response.usage.completion_tokens - - # Check Translation - translatedText = cleanTranslatedText(translatedText, varResponse) - if isinstance(tItem, list): - extractedTranslations = extractTranslation(translatedText, True) - if extractedTranslations == None or len(tItem) != len(extractedTranslations): - # Mismatch. Try Again - response = translateText(system, user, history, 0.05, format, "gpt-4o") - translatedText = response.choices[0].message.content - totalTokens[0] += response.usage.prompt_tokens - totalTokens[1] += response.usage.completion_tokens - - # Formatting - translatedText = cleanTranslatedText(translatedText, varResponse) - if isinstance(tItem, list): - extractedTranslations = extractTranslation(translatedText, True) - if extractedTranslations == None or len(tItem) != len(extractedTranslations): - mismatch = True # Just here for breakpoint - logFile.write(f"Input:\n{subbedT}\n") - logFile.write(f"Output:\n{translatedText}\n") - - # Set if no mismatch - if mismatch == False: - tList[index] = extractedTranslations - history = extractedTranslations[-MAXHISTORY:] # Update history if we have a list + for index, tItem in enumerate(tList): + # Before sending to translation, if we have a list of items, add the formatting + if isinstance(tItem, list): + for j in range(len(tItem)): + if not tItem[j]: + tItem[j] = tItem[j].replace("", "Placeholder Text") + payload = {f"Line{i+1}": string for i, string in enumerate(tItem)} + payload = json.dumps(payload, indent=4, ensure_ascii=False) + varResponse = [payload, []] + subbedT = varResponse[0] else: - history = text[-MAXHISTORY:] - mismatch = False - if FILENAME not in MISMATCH: - MISMATCH.append(FILENAME) + varResponse = [tItem, []] + subbedT = varResponse[0] - # Update Loading Bar - with LOCK: + # Things to Check before starting translation + if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+", subbedT): if PBAR is not None: PBAR.update(len(tItem)) - else: - # Ensure we're passing a single string to extractTranslation - tList[index] = translatedText.replace("Placeholder Text", "") + history = tItem[-MAXHISTORY:] + continue - # Combine if multilist - if isinstance(tList[0], list): - tList = [t for sublist in tList for t in sublist] + # Create Message + system, user = createContext(fullPromptFlag, subbedT, format) - # Return - if format == "json": - return [tList, totalTokens] + # Calculate Estimate + if ESTIMATE: + estimate = countTokens(system, user, history) + totalTokens[0] += estimate[0] + totalTokens[1] += estimate[1] + continue + + # Translating + response = translateText(system, user, history, 0.05, format) + translatedText = response.choices[0].message.content + totalTokens[0] += response.usage.prompt_tokens + totalTokens[1] += response.usage.completion_tokens + + # Check Translation + translatedText = cleanTranslatedText(translatedText, varResponse) + if isinstance(tItem, list): + extractedTranslations = extractTranslation(translatedText, True) + if extractedTranslations == None or len(tItem) != len(extractedTranslations): + # Mismatch. Try Again + response = translateText(system, user, history, 0.05, format, "gpt-4o") + translatedText = response.choices[0].message.content + totalTokens[0] += response.usage.prompt_tokens + totalTokens[1] += response.usage.completion_tokens + + # Formatting + translatedText = cleanTranslatedText(translatedText, varResponse) + if isinstance(tItem, list): + extractedTranslations = extractTranslation(translatedText, True) + if extractedTranslations == None or len(tItem) != len(extractedTranslations): + mismatch = True # Just here for breakpoint + logFile.write(f"Input:\n{subbedT}\n") + logFile.write(f"Output:\n{translatedText}\n") + + # Set if no mismatch + if mismatch == False: + tList[index] = extractedTranslations + history = extractedTranslations[-MAXHISTORY:] # Update history if we have a list + else: + history = text[-MAXHISTORY:] + mismatch = False + if FILENAME not in MISMATCH: + MISMATCH.append(FILENAME) + + # Update Loading Bar + with LOCK: + if PBAR is not None: + PBAR.update(len(tItem)) + else: + # Ensure we're passing a single string to extractTranslation + tList[index] = translatedText.replace("Placeholder Text", "") + + # Combine if multilist + if isinstance(tList[0], list): + tList = [t for sublist in tList for t in sublist] + + # Return + if format == "json": + return [tList, totalTokens] + else: + return [tList[0], totalTokens] else: - return [tList[0], totalTokens] + return [text, [0, 0]]