Praise the AI Gods. Refactor translateAI into a single function HALLEFUCKINGLUJAH
This commit is contained in:
parent
f51ef7f7e2
commit
8606a5c7b8
17 changed files with 1149 additions and 5391 deletions
359
modules/csv.py
359
modules/csv.py
|
|
@ -14,6 +14,7 @@ from colorama import Fore
|
|||
from dotenv import load_dotenv
|
||||
from retry import retry
|
||||
from tqdm import tqdm
|
||||
from util.translation import TranslationConfig, translateAI as sharedtranslateAI
|
||||
|
||||
# Open AI
|
||||
load_dotenv()
|
||||
|
|
@ -78,6 +79,18 @@ LEAVE = False
|
|||
PBAR = None
|
||||
ENCODING = "utf8"
|
||||
|
||||
# Initialize Translation Config
|
||||
TRANSLATION_CONFIG = TranslationConfig(
|
||||
model=MODEL,
|
||||
language=LANGUAGE,
|
||||
prompt=PROMPT,
|
||||
vocab=VOCAB,
|
||||
langRegex=LANGREGEX,
|
||||
batchSize=BATCHSIZE,
|
||||
maxHistory=MAXHISTORY,
|
||||
estimateMode=False # Will be set dynamically based on ESTIMATE
|
||||
)
|
||||
LEAVE = False
|
||||
|
||||
def handleCSV(filename, estimate):
|
||||
global ESTIMATE, TOKENS
|
||||
|
|
@ -406,7 +419,7 @@ def translateCSV(data, pbar, writer, filename, translatedList, format):
|
|||
pbar.refresh()
|
||||
|
||||
# Translate
|
||||
response = translateGPT(stringList, "", True)
|
||||
response = translateAI(stringList, "", True)
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
translatedList = response[0]
|
||||
|
|
@ -453,7 +466,7 @@ def getSpeaker(speaker):
|
|||
return [NAMESLIST[i][1], [0, 0]]
|
||||
|
||||
# Translate and Store Speaker
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
f"{speaker}",
|
||||
"Reply with the " + LANGUAGE + " translation of the NPC name.",
|
||||
False,
|
||||
|
|
@ -464,7 +477,7 @@ def getSpeaker(speaker):
|
|||
|
||||
# Retry if name doesn't translate for some reason
|
||||
if re.search(r"([a-zA-Z？?])", response[0]) == None:
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
f"{speaker}",
|
||||
"Reply with the " + LANGUAGE + " translation of the NPC name.",
|
||||
False,
|
||||
|
|
@ -477,328 +490,24 @@ def getSpeaker(speaker):
|
|||
return response
|
||||
return [speaker, [0, 0]]
|
||||
|
||||
|
||||
def batchList(input_list, batch_size):
|
||||
if not isinstance(batch_size, int) or batch_size <= 0:
|
||||
raise ValueError("batch_size must be a positive integer")
|
||||
|
||||
return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)]
|
||||
|
||||
|
||||
def parseVocabWithCategories(vocabText):
|
||||
"""Parse vocabulary text and extract terms with their categories."""
|
||||
pairs = []
|
||||
seen = set()
|
||||
currentCategory = None
|
||||
|
||||
for line in vocabText.splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith('```'):
|
||||
continue
|
||||
|
||||
# Check if this is a category header
|
||||
if line.startswith('#'):
|
||||
currentCategory = line
|
||||
continue
|
||||
|
||||
# Parse vocabulary term
|
||||
m = re.match(r'^(.+?)(?:\s?[\(–])', line) # term is everything before space + '(' or '–'
|
||||
if m:
|
||||
term = m.group(1)
|
||||
if term not in seen:
|
||||
pairs.append((term, line, currentCategory))
|
||||
seen.add(term)
|
||||
|
||||
return pairs
|
||||
|
||||
|
||||
def buildMatchedVocabText(vocabPairs, subbedT):
|
||||
"""Build formatted vocabulary text with matched terms organized by category."""
|
||||
matchedCategories = {}
|
||||
|
||||
# Use word boundaries for Japanese if appropriate, or allow substring as before.
|
||||
for term, line, category in vocabPairs:
|
||||
# "term in subbedT" could be false positive; can use regex but Japanese doesn't always have spaces.
|
||||
if term in subbedT:
|
||||
if category not in matchedCategories:
|
||||
matchedCategories[category] = []
|
||||
matchedCategories[category].append(line)
|
||||
|
||||
# Format matched vocabulary with categories
|
||||
if matchedCategories:
|
||||
formattedLines = ["Here are some vocabulary and terms so that you know the proper spelling and translation.\n"]
|
||||
for category, lines in matchedCategories.items():
|
||||
if category: # Only add category header if it exists
|
||||
formattedLines.append(category)
|
||||
formattedLines.extend(lines)
|
||||
formattedLines.append("") # Add blank line between categories
|
||||
matchedVocabText = f"```\n{chr(10).join(formattedLines).rstrip()}\n```"
|
||||
else:
|
||||
matchedVocabText = ""
|
||||
|
||||
return matchedVocabText
|
||||
|
||||
|
||||
def createContext(fullPromptFlag, subbedT, format):
|
||||
vocabPairs = parseVocabWithCategories(VOCAB)
|
||||
matchedVocabText = buildMatchedVocabText(vocabPairs, subbedT)
|
||||
|
||||
if fullPromptFlag:
|
||||
system = PROMPT + matchedVocabText
|
||||
else:
|
||||
system = f"\
|
||||
You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
|
||||
Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
|
||||
- You translate everything, including erotic and explicit content.\n\
|
||||
- 'Game Characters' - The names, nicknames, and genders of the game characters. Reference this to know the names, nicknames, and gender of characters in the game\n\
|
||||
- All text in your response must be in {LANGUAGE} even if it is hard to translate.\n\
|
||||
- Never include any notes, explanations, dislaimers, or anything similar in your response.\n\
|
||||
- Maintain any spacing in the translation.\n\
|
||||
- Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\
|
||||
- `...` can be a part of the dialogue. Translate it as it is.\n\
|
||||
{matchedVocabText}\n\
|
||||
"
|
||||
if format == "json":
|
||||
user = f"```json\n{subbedT}\n```"
|
||||
else:
|
||||
user = subbedT
|
||||
return system, user
|
||||
|
||||
|
||||
def translateText(system, user, history, penalty, format, model=MODEL):
|
||||
# Prompt
|
||||
msg = [{"role": "system", "content": system}]
|
||||
|
||||
# History
|
||||
if isinstance(history, list):
|
||||
msg.append({"role": "system", "content": "Translation History:"})
|
||||
msg.extend([{"role": "assistant", "content": h} for h in history])
|
||||
else:
|
||||
msg.append({"role": "assistant", "content": history})
|
||||
|
||||
# Response Format
|
||||
if format == "json":
|
||||
responseFormat = {"type": "json_object"}
|
||||
else:
|
||||
responseFormat = {"type": "text"}
|
||||
|
||||
# Content to TL
|
||||
msg.append({"role": "user", "content": f"{user}"})
|
||||
response = openai.chat.completions.create(
|
||||
temperature=0,
|
||||
frequency_penalty=penalty,
|
||||
model=model,
|
||||
response_format=responseFormat,
|
||||
messages=msg,
|
||||
)
|
||||
return response
|
||||
|
||||
|
||||
def cleanTranslatedText(translatedText):
|
||||
placeholders = {
|
||||
f"{LANGUAGE} Translation: ": "",
|
||||
"Translation: ": "",
|
||||
"っ": "",
|
||||
"〜": "~",
|
||||
"ッ": "",
|
||||
"。": ".",
|
||||
"「": '\\"',
|
||||
"」": '\\"',
|
||||
"- ": "-",
|
||||
"—": "―",
|
||||
"】": "]",
|
||||
"【": "[",
|
||||
"é": "e",
|
||||
"this guy": "this bastard",
|
||||
"This guy": "This bastard",
|
||||
"Placeholder Text": "",
|
||||
"```json": "",
|
||||
"```": "",
|
||||
# Add more replacements as needed
|
||||
}
|
||||
for target, replacement in placeholders.items():
|
||||
translatedText = translatedText.replace(target, replacement)
|
||||
|
||||
# Remove Repeating Characters
|
||||
pattern = re.compile(r"(.)\s*\1(?:\s*\1){" + str(20 - 1) + r",}")
|
||||
translatedText = pattern.sub(lambda match: match.group(0).replace(" ", "")[:20], translatedText)
|
||||
|
||||
# Elongate Long Dashes (Since GPT Ignores them...)
|
||||
translatedText = elongateCharacters(translatedText)
|
||||
return translatedText
|
||||
|
||||
|
||||
def elongateCharacters(text):
|
||||
# Define a pattern to match one character followed by one or more `ー` characters
|
||||
# Using a positive lookbehind assertion to capture the preceding character
|
||||
pattern = r"(?<=(.))ー+"
|
||||
|
||||
# Define a replacement function that elongates the captured character
|
||||
def repl(match):
|
||||
char = match.group(1) # The character before the ー sequence
|
||||
count = len(match.group(0)) - 1 # Number of ー characters
|
||||
return char * count # Replace ー sequence with the character repeated
|
||||
|
||||
# Use re.sub() to replace the pattern in the text
|
||||
return re.sub(pattern, repl, text)
|
||||
|
||||
|
||||
def extractTranslation(translatedTextList, is_list):
|
||||
try:
|
||||
translatedTextList = re.sub(r'\\"+\"([^,\n}])', r'\\"\1', translatedTextList)
|
||||
translatedTextList = re.sub(r"(?<![\\])\"+(?![\n,])", r'"', translatedTextList)
|
||||
line_dict = json.loads(translatedTextList)
|
||||
# If it's a batch (i.e., list), extract with tags; otherwise, return the single item.
|
||||
string_list = list(line_dict.values())
|
||||
if is_list:
|
||||
return string_list
|
||||
else:
|
||||
return string_list[0]
|
||||
|
||||
except Exception as e:
|
||||
PBAR.write(f"extractTranslation Error: {e} on String {translatedTextList}")
|
||||
return None
|
||||
|
||||
|
||||
def countTokens(system, user, history):
|
||||
inputTotalTokens = 0
|
||||
outputTotalTokens = 0
|
||||
enc = tiktoken.encoding_for_model("gpt-4")
|
||||
|
||||
# Input
|
||||
if isinstance(history, list):
|
||||
for line in history:
|
||||
inputTotalTokens += len(enc.encode(line))
|
||||
else:
|
||||
inputTotalTokens += len(enc.encode(history))
|
||||
inputTotalTokens += len(enc.encode(system))
|
||||
inputTotalTokens += len(enc.encode(user))
|
||||
|
||||
# Output
|
||||
outputTotalTokens += round(len(enc.encode(user)) * 2.5)
|
||||
|
||||
return [inputTotalTokens, outputTotalTokens]
|
||||
|
||||
|
||||
@retry(exceptions=Exception, tries=5, delay=5)
|
||||
def translateGPT(text, history, fullPromptFlag):
|
||||
def translateAI(text, history, fullPromptFlag):
|
||||
"""
|
||||
Legacy wrapper function for the new shared translation utility.
|
||||
This maintains compatibility with existing code while using the new shared implementation.
|
||||
"""
|
||||
global PBAR, MISMATCH, FILENAME
|
||||
if text:
|
||||
with open("log/translationHistory.txt", "a+", encoding="utf-8") as logFile:
|
||||
mismatch = False
|
||||
totalTokens = [0, 0]
|
||||
if isinstance(text, list):
|
||||
format = "json"
|
||||
tList = batchList(text, BATCHSIZE)
|
||||
else:
|
||||
format = "text"
|
||||
tList = [text]
|
||||
|
||||
for index, tItem in enumerate(tList):
|
||||
# Things to Check before starting translation
|
||||
if not re.search(LANGREGEX, str(tItem)):
|
||||
if PBAR is not None:
|
||||
PBAR.update(len(tItem))
|
||||
if isinstance(tItem, list):
|
||||
for j in range(len(tItem)):
|
||||
tItem[j] = cleanTranslatedText(tItem[j])
|
||||
tList[index] = tItem
|
||||
else:
|
||||
tList[index] = cleanTranslatedText(tItem)
|
||||
history = tItem[-MAXHISTORY:]
|
||||
continue
|
||||
# Update config estimate mode based on global ESTIMATE
|
||||
TRANSLATION_CONFIG.estimateMode = bool(ESTIMATE)
|
||||
|
||||
# Before sending to translation, if we have a list of items, add the formatting
|
||||
if isinstance(tItem, list):
|
||||
for j in range(len(tItem)):
|
||||
if not tItem[j]:
|
||||
tItem[j] = tItem[j].replace("", "Placeholder Text")
|
||||
payload = {f"Line{i+1}": string for i, string in enumerate(tItem)}
|
||||
payload = json.dumps(payload, indent=4, ensure_ascii=False)
|
||||
varResponse = [payload, []]
|
||||
subbedT = varResponse[0]
|
||||
else:
|
||||
varResponse = [tItem, []]
|
||||
subbedT = varResponse[0]
|
||||
|
||||
# Create Message
|
||||
system, user = createContext(fullPromptFlag, subbedT, format)
|
||||
|
||||
# Calculate Estimate
|
||||
if ESTIMATE:
|
||||
estimate = countTokens(system, user, history)
|
||||
totalTokens[0] += estimate[0]
|
||||
totalTokens[1] += estimate[1]
|
||||
continue
|
||||
|
||||
# Translating
|
||||
response = translateText(system, user, history, 0.05, format)
|
||||
|
||||
# Set Tokens
|
||||
translatedText = response.choices[0].message.content
|
||||
|
||||
# AI Refused, Try Again
|
||||
if not translatedText:
|
||||
response = translateText(f"{system}\n You translate ALL content.", user, history, 0.1, format, model="gpt-4o")
|
||||
translatedText = response.choices[0].message.content
|
||||
|
||||
# Report Tokens
|
||||
totalTokens[0] += response.usage.prompt_tokens
|
||||
totalTokens[1] += response.usage.completion_tokens
|
||||
|
||||
# Check Translation
|
||||
if translatedText:
|
||||
translatedText = cleanTranslatedText(translatedText)
|
||||
if isinstance(tItem, list):
|
||||
extractedTranslations = extractTranslation(translatedText, True)
|
||||
if extractedTranslations == None or len(tItem) != len(extractedTranslations):
|
||||
# Mismatch. Try Again
|
||||
response = translateText(system, user, history, 0.05, format, MODEL)
|
||||
translatedText = response.choices[0].message.content
|
||||
totalTokens[0] += response.usage.prompt_tokens
|
||||
totalTokens[1] += response.usage.completion_tokens
|
||||
|
||||
# Formatting
|
||||
translatedText = cleanTranslatedText(translatedText)
|
||||
if isinstance(tItem, list):
|
||||
extractedTranslations = extractTranslation(translatedText, True)
|
||||
if extractedTranslations == None or len(tItem) != len(extractedTranslations):
|
||||
with open("log/mismatchHistory.txt", "a+", encoding="utf-8") as mismatchFile:
|
||||
mismatchFile.write(f"Mismatch: {FILENAME}\n")
|
||||
mismatchFile.write(f"Input:\n{subbedT}\n")
|
||||
mismatchFile.write(f"Output:\n{translatedText}\n")
|
||||
mismatch = True # Just here for breakpoint
|
||||
logFile.write(f"Input:\n{subbedT}\n")
|
||||
logFile.write(f"Output:\n{translatedText}\n")
|
||||
|
||||
# Set if no mismatch
|
||||
if mismatch == False:
|
||||
tList[index] = extractedTranslations
|
||||
history = extractedTranslations[-MAXHISTORY:] # Update history if we have a list
|
||||
else:
|
||||
history = text[-MAXHISTORY:]
|
||||
mismatch = False
|
||||
if FILENAME not in MISMATCH:
|
||||
MISMATCH.append(FILENAME)
|
||||
|
||||
# Update Loading Bar
|
||||
with LOCK:
|
||||
if PBAR is not None:
|
||||
PBAR.update(len(tItem))
|
||||
else:
|
||||
# Ensure we're passing a single string to extractTranslation
|
||||
tList[index] = translatedText.replace("Placeholder Text", "")
|
||||
else:
|
||||
PBAR.write(f"AI Refused:{tItem}\n")
|
||||
|
||||
# Combine if multilist
|
||||
if isinstance(tList[0], list):
|
||||
tList = [t for sublist in tList for t in sublist]
|
||||
|
||||
# Return
|
||||
if format == "json":
|
||||
return [tList, totalTokens]
|
||||
else:
|
||||
return [tList[0], totalTokens]
|
||||
else:
|
||||
return [text, [0, 0]]
|
||||
# Call the new shared translation function
|
||||
return sharedtranslateAI(
|
||||
text=text,
|
||||
history=history,
|
||||
fullPromptFlag=fullPromptFlag,
|
||||
config=TRANSLATION_CONFIG,
|
||||
filename=FILENAME,
|
||||
pbar=PBAR,
|
||||
lock=LOCK,
|
||||
mismatchList=MISMATCH
|
||||
)
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ from colorama import Fore
|
|||
from dotenv import load_dotenv
|
||||
from retry import retry
|
||||
from tqdm import tqdm
|
||||
from util.translation import TranslationConfig, translateAI as sharedtranslateAI
|
||||
|
||||
# Globals
|
||||
MODEL = os.getenv("model")
|
||||
|
|
@ -66,6 +67,19 @@ else:
|
|||
BATCHSIZE = int(os.getenv("batchsize"))
|
||||
FREQUENCY_PENALTY = float(os.getenv("frequency_penalty"))
|
||||
|
||||
# Initialize Translation Config
|
||||
TRANSLATION_CONFIG = TranslationConfig(
|
||||
model=MODEL,
|
||||
language=LANGUAGE,
|
||||
prompt=PROMPT,
|
||||
vocab=VOCAB,
|
||||
langRegex=LANGREGEX,
|
||||
batchSize=BATCHSIZE,
|
||||
maxHistory=MAXHISTORY,
|
||||
estimateMode=False # Will be set dynamically based on ESTIMATE
|
||||
)
|
||||
LEAVE = False
|
||||
|
||||
# tqdm Globals
|
||||
BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
|
||||
POSITION = 0
|
||||
|
|
@ -382,7 +396,7 @@ def translateImages(imageList):
|
|||
totalTokens = [0, 0]
|
||||
|
||||
# Translate GPT
|
||||
response = translateGPT(imageList[0], "Keep the Translation as brief as possible", True)
|
||||
response = translateAI(imageList[0], "Keep the Translation as brief as possible", True)
|
||||
translatedList = response[0]
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
|
|
@ -403,7 +417,7 @@ def getSpeaker(speaker):
|
|||
return [NAMESLIST[i][1], [0, 0]]
|
||||
|
||||
# Translate and Store Speaker
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
f"{speaker}",
|
||||
"Reply with the " + LANGUAGE + " translation of the NPC name.",
|
||||
False,
|
||||
|
|
@ -414,7 +428,7 @@ def getSpeaker(speaker):
|
|||
|
||||
# Retry if name doesn't translate for some reason
|
||||
if re.search(r"([a-zA-Z？?])", response[0]) == None:
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
f"{speaker}",
|
||||
"Reply with the " + LANGUAGE + " translation of the NPC name.",
|
||||
False,
|
||||
|
|
@ -426,329 +440,24 @@ def getSpeaker(speaker):
|
|||
NAMESLIST.append(speakerList)
|
||||
return response
|
||||
return [speaker, [0, 0]]
|
||||
|
||||
|
||||
def batchList(input_list, batch_size):
|
||||
if not isinstance(batch_size, int) or batch_size <= 0:
|
||||
raise ValueError("batch_size must be a positive integer")
|
||||
|
||||
return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)]
|
||||
|
||||
|
||||
def parseVocabWithCategories(vocabText):
|
||||
"""Parse vocabulary text and extract terms with their categories."""
|
||||
pairs = []
|
||||
seen = set()
|
||||
currentCategory = None
|
||||
|
||||
for line in vocabText.splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith('```'):
|
||||
continue
|
||||
|
||||
# Check if this is a category header
|
||||
if line.startswith('#'):
|
||||
currentCategory = line
|
||||
continue
|
||||
|
||||
# Parse vocabulary term
|
||||
m = re.match(r'^(.+?)(?:\s?[\(–])', line) # term is everything before space + '(' or '–'
|
||||
if m:
|
||||
term = m.group(1)
|
||||
if term not in seen:
|
||||
pairs.append((term, line, currentCategory))
|
||||
seen.add(term)
|
||||
|
||||
return pairs
|
||||
|
||||
|
||||
def buildMatchedVocabText(vocabPairs, subbedT):
|
||||
"""Build formatted vocabulary text with matched terms organized by category."""
|
||||
matchedCategories = {}
|
||||
|
||||
# Use word boundaries for Japanese if appropriate, or allow substring as before.
|
||||
for term, line, category in vocabPairs:
|
||||
# "term in subbedT" could be false positive; can use regex but Japanese doesn't always have spaces.
|
||||
if term in subbedT:
|
||||
if category not in matchedCategories:
|
||||
matchedCategories[category] = []
|
||||
matchedCategories[category].append(line)
|
||||
|
||||
# Format matched vocabulary with categories
|
||||
if matchedCategories:
|
||||
formattedLines = ["Here are some vocabulary and terms so that you know the proper spelling and translation.\n"]
|
||||
for category, lines in matchedCategories.items():
|
||||
if category: # Only add category header if it exists
|
||||
formattedLines.append(category)
|
||||
formattedLines.extend(lines)
|
||||
formattedLines.append("") # Add blank line between categories
|
||||
matchedVocabText = f"```\n{chr(10).join(formattedLines).rstrip()}\n```"
|
||||
else:
|
||||
matchedVocabText = ""
|
||||
|
||||
return matchedVocabText
|
||||
|
||||
|
||||
def createContext(fullPromptFlag, subbedT, format):
|
||||
vocabPairs = parseVocabWithCategories(VOCAB)
|
||||
matchedVocabText = buildMatchedVocabText(vocabPairs, subbedT)
|
||||
|
||||
if fullPromptFlag:
|
||||
system = PROMPT + matchedVocabText
|
||||
else:
|
||||
system = f"\
|
||||
You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
|
||||
Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
|
||||
- You translate everything, including erotic and explicit content.\n\
|
||||
- 'Game Characters' - The names, nicknames, and genders of the game characters. Reference this to know the names, nicknames, and gender of characters in the game\n\
|
||||
- All text in your response must be in {LANGUAGE} even if it is hard to translate.\n\
|
||||
- Never include any notes, explanations, dislaimers, or anything similar in your response.\n\
|
||||
- Maintain any spacing in the translation.\n\
|
||||
- Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\
|
||||
- `...` can be a part of the dialogue. Translate it as it is.\n\
|
||||
{matchedVocabText}\n\
|
||||
"
|
||||
if format == "json":
|
||||
user = f"```json\n{subbedT}\n```"
|
||||
else:
|
||||
user = subbedT
|
||||
return system, user
|
||||
|
||||
|
||||
def translateText(system, user, history, penalty, format, model=MODEL):
|
||||
# Prompt
|
||||
msg = [{"role": "system", "content": system}]
|
||||
|
||||
# History
|
||||
if isinstance(history, list):
|
||||
msg.append({"role": "system", "content": "Translation History:"})
|
||||
msg.extend([{"role": "assistant", "content": h} for h in history])
|
||||
else:
|
||||
msg.append({"role": "assistant", "content": history})
|
||||
|
||||
# Response Format
|
||||
if format == "json":
|
||||
responseFormat = {"type": "json_object"}
|
||||
else:
|
||||
responseFormat = {"type": "text"}
|
||||
|
||||
# Content to TL
|
||||
msg.append({"role": "user", "content": f"{user}"})
|
||||
response = openai.chat.completions.create(
|
||||
temperature=0,
|
||||
frequency_penalty=penalty,
|
||||
model=model,
|
||||
response_format=responseFormat,
|
||||
messages=msg,
|
||||
)
|
||||
return response
|
||||
|
||||
|
||||
def cleanTranslatedText(translatedText):
|
||||
placeholders = {
|
||||
f"{LANGUAGE} Translation: ": "",
|
||||
"Translation: ": "",
|
||||
"っ": "",
|
||||
"〜": "~",
|
||||
"ッ": "",
|
||||
"。": ".",
|
||||
"「": '\\"',
|
||||
"」": '\\"',
|
||||
"- ": "-",
|
||||
"—": "―",
|
||||
"】": "]",
|
||||
"【": "[",
|
||||
"é": "e",
|
||||
"this guy": "this bastard",
|
||||
"This guy": "This bastard",
|
||||
"Placeholder Text": "",
|
||||
"```json": "",
|
||||
"```": "",
|
||||
# Add more replacements as needed
|
||||
}
|
||||
for target, replacement in placeholders.items():
|
||||
translatedText = translatedText.replace(target, replacement)
|
||||
|
||||
# Remove Repeating Characters
|
||||
pattern = re.compile(r"(.)\s*\1(?:\s*\1){" + str(20 - 1) + r",}")
|
||||
translatedText = pattern.sub(lambda match: match.group(0).replace(" ", "")[:20], translatedText)
|
||||
|
||||
# Elongate Long Dashes (Since GPT Ignores them...)
|
||||
translatedText = elongateCharacters(translatedText)
|
||||
return translatedText
|
||||
|
||||
|
||||
def elongateCharacters(text):
|
||||
# Define a pattern to match one character followed by one or more `ー` characters
|
||||
# Using a positive lookbehind assertion to capture the preceding character
|
||||
pattern = r"(?<=(.))ー+"
|
||||
|
||||
# Define a replacement function that elongates the captured character
|
||||
def repl(match):
|
||||
char = match.group(1) # The character before the ー sequence
|
||||
count = len(match.group(0)) - 1 # Number of ー characters
|
||||
return char * count # Replace ー sequence with the character repeated
|
||||
|
||||
# Use re.sub() to replace the pattern in the text
|
||||
return re.sub(pattern, repl, text)
|
||||
|
||||
|
||||
def extractTranslation(translatedTextList, is_list):
|
||||
try:
|
||||
translatedTextList = re.sub(r'\\"+\"([^,\n}])', r'\\"\1', translatedTextList)
|
||||
translatedTextList = re.sub(r"(?<![\\])\"+(?![\n,])", r'"', translatedTextList)
|
||||
line_dict = json.loads(translatedTextList)
|
||||
# If it's a batch (i.e., list), extract with tags; otherwise, return the single item.
|
||||
string_list = list(line_dict.values())
|
||||
if is_list:
|
||||
return string_list
|
||||
else:
|
||||
return string_list[0]
|
||||
|
||||
except Exception as e:
|
||||
PBAR.write(f"extractTranslation Error: {e} on String {translatedTextList}")
|
||||
return None
|
||||
|
||||
|
||||
def countTokens(system, user, history):
|
||||
inputTotalTokens = 0
|
||||
outputTotalTokens = 0
|
||||
enc = tiktoken.encoding_for_model("gpt-4")
|
||||
|
||||
# Input
|
||||
if isinstance(history, list):
|
||||
for line in history:
|
||||
inputTotalTokens += len(enc.encode(line))
|
||||
else:
|
||||
inputTotalTokens += len(enc.encode(history))
|
||||
inputTotalTokens += len(enc.encode(system))
|
||||
inputTotalTokens += len(enc.encode(user))
|
||||
|
||||
# Output
|
||||
outputTotalTokens += round(len(enc.encode(user)) * 2.5)
|
||||
|
||||
return [inputTotalTokens, outputTotalTokens]
|
||||
|
||||
|
||||
@retry(exceptions=Exception, tries=5, delay=5)
|
||||
def translateGPT(text, history, fullPromptFlag):
|
||||
def translateAI(text, history, fullPromptFlag):
|
||||
"""
|
||||
Legacy wrapper function for the new shared translation utility.
|
||||
This maintains compatibility with existing code while using the new shared implementation.
|
||||
"""
|
||||
global PBAR, MISMATCH, FILENAME
|
||||
if text:
|
||||
with open("log/translationHistory.txt", "a+", encoding="utf-8") as logFile:
|
||||
mismatch = False
|
||||
totalTokens = [0, 0]
|
||||
if isinstance(text, list):
|
||||
format = "json"
|
||||
tList = batchList(text, BATCHSIZE)
|
||||
else:
|
||||
format = "text"
|
||||
tList = [text]
|
||||
|
||||
for index, tItem in enumerate(tList):
|
||||
# Things to Check before starting translation
|
||||
if not re.search(LANGREGEX, str(tItem)):
|
||||
if PBAR is not None:
|
||||
PBAR.update(len(tItem))
|
||||
if isinstance(tItem, list):
|
||||
for j in range(len(tItem)):
|
||||
tItem[j] = cleanTranslatedText(tItem[j])
|
||||
tList[index] = tItem
|
||||
else:
|
||||
tList[index] = cleanTranslatedText(tItem)
|
||||
history = tItem[-MAXHISTORY:]
|
||||
continue
|
||||
# Update config estimate mode based on global ESTIMATE
|
||||
TRANSLATION_CONFIG.estimateMode = bool(ESTIMATE)
|
||||
|
||||
# Before sending to translation, if we have a list of items, add the formatting
|
||||
if isinstance(tItem, list):
|
||||
for j in range(len(tItem)):
|
||||
if not tItem[j]:
|
||||
tItem[j] = tItem[j].replace("", "Placeholder Text")
|
||||
payload = {f"Line{i+1}": string for i, string in enumerate(tItem)}
|
||||
payload = json.dumps(payload, indent=4, ensure_ascii=False)
|
||||
varResponse = [payload, []]
|
||||
subbedT = varResponse[0]
|
||||
else:
|
||||
varResponse = [tItem, []]
|
||||
subbedT = varResponse[0]
|
||||
|
||||
# Create Message
|
||||
system, user = createContext(fullPromptFlag, subbedT, format)
|
||||
|
||||
# Calculate Estimate
|
||||
if ESTIMATE:
|
||||
estimate = countTokens(system, user, history)
|
||||
totalTokens[0] += estimate[0]
|
||||
totalTokens[1] += estimate[1]
|
||||
continue
|
||||
|
||||
# Translating
|
||||
response = translateText(system, user, history, 0.05, format)
|
||||
|
||||
# Set Tokens
|
||||
translatedText = response.choices[0].message.content
|
||||
|
||||
# AI Refused, Try Again
|
||||
if not translatedText:
|
||||
response = translateText(f"{system}\n You translate ALL content.", user, history, 0.1, format, model="gpt-4o")
|
||||
translatedText = response.choices[0].message.content
|
||||
|
||||
# Report Tokens
|
||||
totalTokens[0] += response.usage.prompt_tokens
|
||||
totalTokens[1] += response.usage.completion_tokens
|
||||
|
||||
# Check Translation
|
||||
if translatedText:
|
||||
translatedText = cleanTranslatedText(translatedText)
|
||||
if isinstance(tItem, list):
|
||||
extractedTranslations = extractTranslation(translatedText, True)
|
||||
if extractedTranslations == None or len(tItem) != len(extractedTranslations):
|
||||
# Mismatch. Try Again
|
||||
response = translateText(system, user, history, 0.05, format, MODEL)
|
||||
translatedText = response.choices[0].message.content
|
||||
totalTokens[0] += response.usage.prompt_tokens
|
||||
totalTokens[1] += response.usage.completion_tokens
|
||||
|
||||
# Formatting
|
||||
translatedText = cleanTranslatedText(translatedText)
|
||||
if isinstance(tItem, list):
|
||||
extractedTranslations = extractTranslation(translatedText, True)
|
||||
if extractedTranslations == None or len(tItem) != len(extractedTranslations):
|
||||
with open("log/mismatchHistory.txt", "a+", encoding="utf-8") as mismatchFile:
|
||||
mismatchFile.write(f"Mismatch: {FILENAME}\n")
|
||||
mismatchFile.write(f"Input:\n{subbedT}\n")
|
||||
mismatchFile.write(f"Output:\n{translatedText}\n")
|
||||
mismatch = True # Just here for breakpoint
|
||||
logFile.write(f"Input:\n{subbedT}\n")
|
||||
logFile.write(f"Output:\n{translatedText}\n")
|
||||
|
||||
# Set if no mismatch
|
||||
if mismatch == False:
|
||||
tList[index] = extractedTranslations
|
||||
history = extractedTranslations[-MAXHISTORY:] # Update history if we have a list
|
||||
else:
|
||||
history = text[-MAXHISTORY:]
|
||||
mismatch = False
|
||||
if FILENAME not in MISMATCH:
|
||||
MISMATCH.append(FILENAME)
|
||||
|
||||
# Update Loading Bar
|
||||
with LOCK:
|
||||
if PBAR is not None:
|
||||
PBAR.update(len(tItem))
|
||||
else:
|
||||
# Ensure we're passing a single string to extractTranslation
|
||||
tList[index] = translatedText.replace("Placeholder Text", "")
|
||||
else:
|
||||
PBAR.write(f"AI Refused:{tItem}\n")
|
||||
|
||||
# Combine if multilist
|
||||
if isinstance(tList[0], list):
|
||||
tList = [t for sublist in tList for t in sublist]
|
||||
|
||||
# Return
|
||||
if format == "json":
|
||||
return [tList, totalTokens]
|
||||
else:
|
||||
return [tList[0], totalTokens]
|
||||
else:
|
||||
return [text, [0, 0]]
|
||||
# Call the new shared translation function
|
||||
return sharedtranslateAI(
|
||||
text=text,
|
||||
history=history,
|
||||
fullPromptFlag=fullPromptFlag,
|
||||
config=TRANSLATION_CONFIG,
|
||||
filename=FILENAME,
|
||||
pbar=PBAR,
|
||||
lock=LOCK,
|
||||
mismatchList=MISMATCH
|
||||
)
|
||||
|
|
|
|||
359
modules/json.py
359
modules/json.py
|
|
@ -13,6 +13,7 @@ from colorama import Fore
|
|||
from dotenv import load_dotenv
|
||||
from retry import retry
|
||||
from tqdm import tqdm
|
||||
from util.translation import TranslationConfig, translateAI as sharedtranslateAI
|
||||
|
||||
# Open AI
|
||||
load_dotenv()
|
||||
|
|
@ -77,6 +78,18 @@ else:
|
|||
BATCHSIZE = int(os.getenv("batchsize"))
|
||||
FREQUENCY_PENALTY = float(os.getenv("frequency_penalty"))
|
||||
|
||||
# Initialize Translation Config
|
||||
TRANSLATION_CONFIG = TranslationConfig(
|
||||
model=MODEL,
|
||||
language=LANGUAGE,
|
||||
prompt=PROMPT,
|
||||
vocab=VOCAB,
|
||||
langRegex=LANGREGEX,
|
||||
batchSize=BATCHSIZE,
|
||||
maxHistory=MAXHISTORY,
|
||||
estimateMode=False # Will be set dynamically based on ESTIMATE
|
||||
)
|
||||
LEAVE = False
|
||||
|
||||
def handleJSON(filename, estimate):
|
||||
global ESTIMATE, totalTokens
|
||||
|
|
@ -281,7 +294,7 @@ def translateJSON(data, translatedList):
|
|||
if stringList:
|
||||
PBAR.total = len(stringList)
|
||||
PBAR.refresh()
|
||||
response = translateGPT(stringList, "Reply with the English Translation", True)
|
||||
response = translateAI(stringList, "Reply with the English Translation", True)
|
||||
tokens[0] += response[1][0]
|
||||
tokens[1] += response[1][1]
|
||||
stringListTL = response[0]
|
||||
|
|
@ -310,7 +323,7 @@ def getSpeaker(speaker):
|
|||
return [NAMESLIST[i][1], [0, 0]]
|
||||
|
||||
# Translate and Store Speaker
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
f"{speaker}",
|
||||
"Reply with the " + LANGUAGE + " translation of the NPC name.",
|
||||
False,
|
||||
|
|
@ -321,7 +334,7 @@ def getSpeaker(speaker):
|
|||
|
||||
# Retry if name doesn't translate for some reason
|
||||
if re.search(r"([a-zA-Z？?])", response[0]) == None:
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
f"{speaker}",
|
||||
"Reply with the " + LANGUAGE + " translation of the NPC name.",
|
||||
False,
|
||||
|
|
@ -334,328 +347,24 @@ def getSpeaker(speaker):
|
|||
return response
|
||||
return [speaker, [0, 0]]
|
||||
|
||||
|
||||
def batchList(input_list, batch_size):
|
||||
if not isinstance(batch_size, int) or batch_size <= 0:
|
||||
raise ValueError("batch_size must be a positive integer")
|
||||
|
||||
return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)]
|
||||
|
||||
|
||||
def parseVocabWithCategories(vocabText):
|
||||
"""Parse vocabulary text and extract terms with their categories."""
|
||||
pairs = []
|
||||
seen = set()
|
||||
currentCategory = None
|
||||
|
||||
for line in vocabText.splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith('```'):
|
||||
continue
|
||||
|
||||
# Check if this is a category header
|
||||
if line.startswith('#'):
|
||||
currentCategory = line
|
||||
continue
|
||||
|
||||
# Parse vocabulary term
|
||||
m = re.match(r'^(.+?)(?:\s?[\(–])', line) # term is everything before space + '(' or '–'
|
||||
if m:
|
||||
term = m.group(1)
|
||||
if term not in seen:
|
||||
pairs.append((term, line, currentCategory))
|
||||
seen.add(term)
|
||||
|
||||
return pairs
|
||||
|
||||
|
||||
def buildMatchedVocabText(vocabPairs, subbedT):
|
||||
"""Build formatted vocabulary text with matched terms organized by category."""
|
||||
matchedCategories = {}
|
||||
|
||||
# Use word boundaries for Japanese if appropriate, or allow substring as before.
|
||||
for term, line, category in vocabPairs:
|
||||
# "term in subbedT" could be false positive; can use regex but Japanese doesn't always have spaces.
|
||||
if term in subbedT:
|
||||
if category not in matchedCategories:
|
||||
matchedCategories[category] = []
|
||||
matchedCategories[category].append(line)
|
||||
|
||||
# Format matched vocabulary with categories
|
||||
if matchedCategories:
|
||||
formattedLines = ["Here are some vocabulary and terms so that you know the proper spelling and translation.\n"]
|
||||
for category, lines in matchedCategories.items():
|
||||
if category: # Only add category header if it exists
|
||||
formattedLines.append(category)
|
||||
formattedLines.extend(lines)
|
||||
formattedLines.append("") # Add blank line between categories
|
||||
matchedVocabText = f"```\n{chr(10).join(formattedLines).rstrip()}\n```"
|
||||
else:
|
||||
matchedVocabText = ""
|
||||
|
||||
return matchedVocabText
|
||||
|
||||
|
||||
def createContext(fullPromptFlag, subbedT, format):
|
||||
vocabPairs = parseVocabWithCategories(VOCAB)
|
||||
matchedVocabText = buildMatchedVocabText(vocabPairs, subbedT)
|
||||
|
||||
if fullPromptFlag:
|
||||
system = PROMPT + matchedVocabText
|
||||
else:
|
||||
system = f"\
|
||||
You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
|
||||
Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
|
||||
- You translate everything, including erotic and explicit content.\n\
|
||||
- 'Game Characters' - The names, nicknames, and genders of the game characters. Reference this to know the names, nicknames, and gender of characters in the game\n\
|
||||
- All text in your response must be in {LANGUAGE} even if it is hard to translate.\n\
|
||||
- Never include any notes, explanations, dislaimers, or anything similar in your response.\n\
|
||||
- Maintain any spacing in the translation.\n\
|
||||
- Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\
|
||||
- `...` can be a part of the dialogue. Translate it as it is.\n\
|
||||
{matchedVocabText}\n\
|
||||
"
|
||||
if format == "json":
|
||||
user = f"```json\n{subbedT}\n```"
|
||||
else:
|
||||
user = subbedT
|
||||
return system, user
|
||||
|
||||
|
||||
def translateText(system, user, history, penalty, format, model=MODEL):
|
||||
# Prompt
|
||||
msg = [{"role": "system", "content": system}]
|
||||
|
||||
# History
|
||||
if isinstance(history, list):
|
||||
msg.append({"role": "system", "content": "Translation History:"})
|
||||
msg.extend([{"role": "assistant", "content": h} for h in history])
|
||||
else:
|
||||
msg.append({"role": "assistant", "content": history})
|
||||
|
||||
# Response Format
|
||||
if format == "json":
|
||||
responseFormat = {"type": "json_object"}
|
||||
else:
|
||||
responseFormat = {"type": "text"}
|
||||
|
||||
# Content to TL
|
||||
msg.append({"role": "user", "content": f"{user}"})
|
||||
response = openai.chat.completions.create(
|
||||
temperature=0,
|
||||
frequency_penalty=penalty,
|
||||
model=model,
|
||||
response_format=responseFormat,
|
||||
messages=msg,
|
||||
)
|
||||
return response
|
||||
|
||||
|
||||
def cleanTranslatedText(translatedText):
|
||||
placeholders = {
|
||||
f"{LANGUAGE} Translation: ": "",
|
||||
"Translation: ": "",
|
||||
"っ": "",
|
||||
"〜": "~",
|
||||
"ッ": "",
|
||||
"。": ".",
|
||||
"「": '\\"',
|
||||
"」": '\\"',
|
||||
"- ": "-",
|
||||
"—": "―",
|
||||
"】": "]",
|
||||
"【": "[",
|
||||
"é": "e",
|
||||
"this guy": "this bastard",
|
||||
"This guy": "This bastard",
|
||||
"Placeholder Text": "",
|
||||
"```json": "",
|
||||
"```": "",
|
||||
# Add more replacements as needed
|
||||
}
|
||||
for target, replacement in placeholders.items():
|
||||
translatedText = translatedText.replace(target, replacement)
|
||||
|
||||
# Remove Repeating Characters
|
||||
pattern = re.compile(r"(.)\s*\1(?:\s*\1){" + str(20 - 1) + r",}")
|
||||
translatedText = pattern.sub(lambda match: match.group(0).replace(" ", "")[:20], translatedText)
|
||||
|
||||
# Elongate Long Dashes (Since GPT Ignores them...)
|
||||
translatedText = elongateCharacters(translatedText)
|
||||
return translatedText
|
||||
|
||||
|
||||
def elongateCharacters(text):
|
||||
# Define a pattern to match one character followed by one or more `ー` characters
|
||||
# Using a positive lookbehind assertion to capture the preceding character
|
||||
pattern = r"(?<=(.))ー+"
|
||||
|
||||
# Define a replacement function that elongates the captured character
|
||||
def repl(match):
|
||||
char = match.group(1) # The character before the ー sequence
|
||||
count = len(match.group(0)) - 1 # Number of ー characters
|
||||
return char * count # Replace ー sequence with the character repeated
|
||||
|
||||
# Use re.sub() to replace the pattern in the text
|
||||
return re.sub(pattern, repl, text)
|
||||
|
||||
|
||||
def extractTranslation(translatedTextList, is_list):
|
||||
try:
|
||||
translatedTextList = re.sub(r'\\"+\"([^,\n}])', r'\\"\1', translatedTextList)
|
||||
translatedTextList = re.sub(r"(?<![\\])\"+(?![\n,])", r'"', translatedTextList)
|
||||
line_dict = json.loads(translatedTextList)
|
||||
# If it's a batch (i.e., list), extract with tags; otherwise, return the single item.
|
||||
string_list = list(line_dict.values())
|
||||
if is_list:
|
||||
return string_list
|
||||
else:
|
||||
return string_list[0]
|
||||
|
||||
except Exception as e:
|
||||
PBAR.write(f"extractTranslation Error: {e} on String {translatedTextList}")
|
||||
return None
|
||||
|
||||
|
||||
def countTokens(system, user, history):
|
||||
inputTotalTokens = 0
|
||||
outputTotalTokens = 0
|
||||
enc = tiktoken.encoding_for_model("gpt-4")
|
||||
|
||||
# Input
|
||||
if isinstance(history, list):
|
||||
for line in history:
|
||||
inputTotalTokens += len(enc.encode(line))
|
||||
else:
|
||||
inputTotalTokens += len(enc.encode(history))
|
||||
inputTotalTokens += len(enc.encode(system))
|
||||
inputTotalTokens += len(enc.encode(user))
|
||||
|
||||
# Output
|
||||
outputTotalTokens += round(len(enc.encode(user)) * 2.5)
|
||||
|
||||
return [inputTotalTokens, outputTotalTokens]
|
||||
|
||||
|
||||
@retry(exceptions=Exception, tries=5, delay=5)
|
||||
def translateGPT(text, history, fullPromptFlag):
|
||||
def translateAI(text, history, fullPromptFlag):
|
||||
"""
|
||||
Legacy wrapper function for the new shared translation utility.
|
||||
This maintains compatibility with existing code while using the new shared implementation.
|
||||
"""
|
||||
global PBAR, MISMATCH, FILENAME
|
||||
if text:
|
||||
with open("log/translationHistory.txt", "a+", encoding="utf-8") as logFile:
|
||||
mismatch = False
|
||||
totalTokens = [0, 0]
|
||||
if isinstance(text, list):
|
||||
format = "json"
|
||||
tList = batchList(text, BATCHSIZE)
|
||||
else:
|
||||
format = "text"
|
||||
tList = [text]
|
||||
|
||||
for index, tItem in enumerate(tList):
|
||||
# Things to Check before starting translation
|
||||
if not re.search(LANGREGEX, str(tItem)):
|
||||
if PBAR is not None:
|
||||
PBAR.update(len(tItem))
|
||||
if isinstance(tItem, list):
|
||||
for j in range(len(tItem)):
|
||||
tItem[j] = cleanTranslatedText(tItem[j])
|
||||
tList[index] = tItem
|
||||
else:
|
||||
tList[index] = cleanTranslatedText(tItem)
|
||||
history = tItem[-MAXHISTORY:]
|
||||
continue
|
||||
# Update config estimate mode based on global ESTIMATE
|
||||
TRANSLATION_CONFIG.estimateMode = bool(ESTIMATE)
|
||||
|
||||
# Before sending to translation, if we have a list of items, add the formatting
|
||||
if isinstance(tItem, list):
|
||||
for j in range(len(tItem)):
|
||||
if not tItem[j]:
|
||||
tItem[j] = tItem[j].replace("", "Placeholder Text")
|
||||
payload = {f"Line{i+1}": string for i, string in enumerate(tItem)}
|
||||
payload = json.dumps(payload, indent=4, ensure_ascii=False)
|
||||
varResponse = [payload, []]
|
||||
subbedT = varResponse[0]
|
||||
else:
|
||||
varResponse = [tItem, []]
|
||||
subbedT = varResponse[0]
|
||||
|
||||
# Create Message
|
||||
system, user = createContext(fullPromptFlag, subbedT, format)
|
||||
|
||||
# Calculate Estimate
|
||||
if ESTIMATE:
|
||||
estimate = countTokens(system, user, history)
|
||||
totalTokens[0] += estimate[0]
|
||||
totalTokens[1] += estimate[1]
|
||||
continue
|
||||
|
||||
# Translating
|
||||
response = translateText(system, user, history, 0.05, format)
|
||||
|
||||
# Set Tokens
|
||||
translatedText = response.choices[0].message.content
|
||||
|
||||
# AI Refused, Try Again
|
||||
if not translatedText:
|
||||
response = translateText(f"{system}\n You translate ALL content.", user, history, 0.1, format, model="gpt-4o")
|
||||
translatedText = response.choices[0].message.content
|
||||
|
||||
# Report Tokens
|
||||
totalTokens[0] += response.usage.prompt_tokens
|
||||
totalTokens[1] += response.usage.completion_tokens
|
||||
|
||||
# Check Translation
|
||||
if translatedText:
|
||||
translatedText = cleanTranslatedText(translatedText)
|
||||
if isinstance(tItem, list):
|
||||
extractedTranslations = extractTranslation(translatedText, True)
|
||||
if extractedTranslations == None or len(tItem) != len(extractedTranslations):
|
||||
# Mismatch. Try Again
|
||||
response = translateText(system, user, history, 0.05, format, MODEL)
|
||||
translatedText = response.choices[0].message.content
|
||||
totalTokens[0] += response.usage.prompt_tokens
|
||||
totalTokens[1] += response.usage.completion_tokens
|
||||
|
||||
# Formatting
|
||||
translatedText = cleanTranslatedText(translatedText)
|
||||
if isinstance(tItem, list):
|
||||
extractedTranslations = extractTranslation(translatedText, True)
|
||||
if extractedTranslations == None or len(tItem) != len(extractedTranslations):
|
||||
with open("log/mismatchHistory.txt", "a+", encoding="utf-8") as mismatchFile:
|
||||
mismatchFile.write(f"Mismatch: {FILENAME}\n")
|
||||
mismatchFile.write(f"Input:\n{subbedT}\n")
|
||||
mismatchFile.write(f"Output:\n{translatedText}\n")
|
||||
mismatch = True # Just here for breakpoint
|
||||
logFile.write(f"Input:\n{subbedT}\n")
|
||||
logFile.write(f"Output:\n{translatedText}\n")
|
||||
|
||||
# Set if no mismatch
|
||||
if mismatch == False:
|
||||
tList[index] = extractedTranslations
|
||||
history = extractedTranslations[-MAXHISTORY:] # Update history if we have a list
|
||||
else:
|
||||
history = text[-MAXHISTORY:]
|
||||
mismatch = False
|
||||
if FILENAME not in MISMATCH:
|
||||
MISMATCH.append(FILENAME)
|
||||
|
||||
# Update Loading Bar
|
||||
with LOCK:
|
||||
if PBAR is not None:
|
||||
PBAR.update(len(tItem))
|
||||
else:
|
||||
# Ensure we're passing a single string to extractTranslation
|
||||
tList[index] = translatedText.replace("Placeholder Text", "")
|
||||
else:
|
||||
PBAR.write(f"AI Refused:{tItem}\n")
|
||||
|
||||
# Combine if multilist
|
||||
if isinstance(tList[0], list):
|
||||
tList = [t for sublist in tList for t in sublist]
|
||||
|
||||
# Return
|
||||
if format == "json":
|
||||
return [tList, totalTokens]
|
||||
else:
|
||||
return [tList[0], totalTokens]
|
||||
else:
|
||||
return [text, [0, 0]]
|
||||
# Call the new shared translation function
|
||||
return sharedtranslateAI(
|
||||
text=text,
|
||||
history=history,
|
||||
fullPromptFlag=fullPromptFlag,
|
||||
config=TRANSLATION_CONFIG,
|
||||
filename=FILENAME,
|
||||
pbar=PBAR,
|
||||
lock=LOCK,
|
||||
mismatchList=MISMATCH
|
||||
)
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ from colorama import Fore
|
|||
from dotenv import load_dotenv
|
||||
from retry import retry
|
||||
from tqdm import tqdm
|
||||
from util.translation import TranslationConfig, translateAI as sharedtranslateAI
|
||||
|
||||
# Open AI
|
||||
load_dotenv()
|
||||
|
|
@ -54,7 +55,6 @@ SPEAKERS = True
|
|||
CHOICES = True
|
||||
DIALOGUE = True
|
||||
|
||||
|
||||
# Regex - Need to change this if you want to translate from/to other languages. Default is Japanese Regex
|
||||
LANGREGEX = r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+"
|
||||
|
||||
|
|
@ -82,6 +82,18 @@ else:
|
|||
BATCHSIZE = int(os.getenv("batchsize"))
|
||||
FREQUENCY_PENALTY = float(os.getenv("frequency_penalty"))
|
||||
|
||||
# Initialize Translation Config
|
||||
TRANSLATION_CONFIG = TranslationConfig(
|
||||
model=MODEL,
|
||||
language=LANGUAGE,
|
||||
prompt=PROMPT,
|
||||
vocab=VOCAB,
|
||||
langRegex=LANGREGEX,
|
||||
batchSize=BATCHSIZE,
|
||||
maxHistory=MAXHISTORY,
|
||||
estimateMode=False # Will be set dynamically based on ESTIMATE
|
||||
)
|
||||
LEAVE = False
|
||||
|
||||
def handleKirikiri(filename, estimate):
|
||||
global ESTIMATE, FILENAME
|
||||
|
|
@ -296,7 +308,7 @@ def translateKiriKiri(data, pbar, filename, jobList):
|
|||
pbar.refresh()
|
||||
|
||||
# Translate
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
stringList,
|
||||
"",
|
||||
True,
|
||||
|
|
@ -319,7 +331,7 @@ def translateKiriKiri(data, pbar, filename, jobList):
|
|||
pbar.refresh()
|
||||
|
||||
# Translate
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
choiceList,
|
||||
"",
|
||||
True,
|
||||
|
|
@ -355,7 +367,7 @@ def getSpeaker(speaker):
|
|||
return [NAMESLIST[i][1], [0, 0]]
|
||||
|
||||
# Translate and Store Speaker
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
f"{speaker}",
|
||||
"Reply with the " + LANGUAGE + " translation of the NPC name.",
|
||||
False,
|
||||
|
|
@ -366,7 +378,7 @@ def getSpeaker(speaker):
|
|||
|
||||
# Retry if name doesn't translate for some reason
|
||||
if re.search(r"([a-zA-Z？?])", response[0]) == None:
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
f"{speaker}",
|
||||
"Reply with the " + LANGUAGE + " translation of the NPC name.",
|
||||
False,
|
||||
|
|
@ -379,328 +391,24 @@ def getSpeaker(speaker):
|
|||
return response
|
||||
return [speaker, [0, 0]]
|
||||
|
||||
|
||||
def batchList(input_list, batch_size):
|
||||
if not isinstance(batch_size, int) or batch_size <= 0:
|
||||
raise ValueError("batch_size must be a positive integer")
|
||||
|
||||
return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)]
|
||||
|
||||
|
||||
def parseVocabWithCategories(vocabText):
|
||||
"""Parse vocabulary text and extract terms with their categories."""
|
||||
pairs = []
|
||||
seen = set()
|
||||
currentCategory = None
|
||||
|
||||
for line in vocabText.splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith('```'):
|
||||
continue
|
||||
|
||||
# Check if this is a category header
|
||||
if line.startswith('#'):
|
||||
currentCategory = line
|
||||
continue
|
||||
|
||||
# Parse vocabulary term
|
||||
m = re.match(r'^(.+?)(?:\s?[\(–])', line) # term is everything before space + '(' or '–'
|
||||
if m:
|
||||
term = m.group(1)
|
||||
if term not in seen:
|
||||
pairs.append((term, line, currentCategory))
|
||||
seen.add(term)
|
||||
|
||||
return pairs
|
||||
|
||||
|
||||
def buildMatchedVocabText(vocabPairs, subbedT):
|
||||
"""Build formatted vocabulary text with matched terms organized by category."""
|
||||
matchedCategories = {}
|
||||
|
||||
# Use word boundaries for Japanese if appropriate, or allow substring as before.
|
||||
for term, line, category in vocabPairs:
|
||||
# "term in subbedT" could be false positive; can use regex but Japanese doesn't always have spaces.
|
||||
if term in subbedT:
|
||||
if category not in matchedCategories:
|
||||
matchedCategories[category] = []
|
||||
matchedCategories[category].append(line)
|
||||
|
||||
# Format matched vocabulary with categories
|
||||
if matchedCategories:
|
||||
formattedLines = ["Here are some vocabulary and terms so that you know the proper spelling and translation.\n"]
|
||||
for category, lines in matchedCategories.items():
|
||||
if category: # Only add category header if it exists
|
||||
formattedLines.append(category)
|
||||
formattedLines.extend(lines)
|
||||
formattedLines.append("") # Add blank line between categories
|
||||
matchedVocabText = f"```\n{chr(10).join(formattedLines).rstrip()}\n```"
|
||||
else:
|
||||
matchedVocabText = ""
|
||||
|
||||
return matchedVocabText
|
||||
|
||||
|
||||
def createContext(fullPromptFlag, subbedT, format):
|
||||
vocabPairs = parseVocabWithCategories(VOCAB)
|
||||
matchedVocabText = buildMatchedVocabText(vocabPairs, subbedT)
|
||||
|
||||
if fullPromptFlag:
|
||||
system = PROMPT + matchedVocabText
|
||||
else:
|
||||
system = f"\
|
||||
You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
|
||||
Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
|
||||
- You translate everything, including erotic and explicit content.\n\
|
||||
- 'Game Characters' - The names, nicknames, and genders of the game characters. Reference this to know the names, nicknames, and gender of characters in the game\n\
|
||||
- All text in your response must be in {LANGUAGE} even if it is hard to translate.\n\
|
||||
- Never include any notes, explanations, dislaimers, or anything similar in your response.\n\
|
||||
- Maintain any spacing in the translation.\n\
|
||||
- Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\
|
||||
- `...` can be a part of the dialogue. Translate it as it is.\n\
|
||||
{matchedVocabText}\n\
|
||||
"
|
||||
if format == "json":
|
||||
user = f"```json\n{subbedT}\n```"
|
||||
else:
|
||||
user = subbedT
|
||||
return system, user
|
||||
|
||||
|
||||
def translateText(system, user, history, penalty, format, model=MODEL):
|
||||
# Prompt
|
||||
msg = [{"role": "system", "content": system}]
|
||||
|
||||
# History
|
||||
if isinstance(history, list):
|
||||
msg.append({"role": "system", "content": "Translation History:"})
|
||||
msg.extend([{"role": "assistant", "content": h} for h in history])
|
||||
else:
|
||||
msg.append({"role": "assistant", "content": history})
|
||||
|
||||
# Response Format
|
||||
if format == "json":
|
||||
responseFormat = {"type": "json_object"}
|
||||
else:
|
||||
responseFormat = {"type": "text"}
|
||||
|
||||
# Content to TL
|
||||
msg.append({"role": "user", "content": f"{user}"})
|
||||
response = openai.chat.completions.create(
|
||||
temperature=0,
|
||||
frequency_penalty=penalty,
|
||||
model=model,
|
||||
response_format=responseFormat,
|
||||
messages=msg,
|
||||
)
|
||||
return response
|
||||
|
||||
|
||||
def cleanTranslatedText(translatedText):
|
||||
placeholders = {
|
||||
f"{LANGUAGE} Translation: ": "",
|
||||
"Translation: ": "",
|
||||
"っ": "",
|
||||
"〜": "~",
|
||||
"ッ": "",
|
||||
"。": ".",
|
||||
"「": '\\"',
|
||||
"」": '\\"',
|
||||
"- ": "-",
|
||||
"—": "―",
|
||||
"】": "]",
|
||||
"【": "[",
|
||||
"é": "e",
|
||||
"this guy": "this bastard",
|
||||
"This guy": "This bastard",
|
||||
"Placeholder Text": "",
|
||||
"```json": "",
|
||||
"```": "",
|
||||
# Add more replacements as needed
|
||||
}
|
||||
for target, replacement in placeholders.items():
|
||||
translatedText = translatedText.replace(target, replacement)
|
||||
|
||||
# Remove Repeating Characters
|
||||
pattern = re.compile(r"(.)\s*\1(?:\s*\1){" + str(20 - 1) + r",}")
|
||||
translatedText = pattern.sub(lambda match: match.group(0).replace(" ", "")[:20], translatedText)
|
||||
|
||||
# Elongate Long Dashes (Since GPT Ignores them...)
|
||||
translatedText = elongateCharacters(translatedText)
|
||||
return translatedText
|
||||
|
||||
|
||||
def elongateCharacters(text):
|
||||
# Define a pattern to match one character followed by one or more `ー` characters
|
||||
# Using a positive lookbehind assertion to capture the preceding character
|
||||
pattern = r"(?<=(.))ー+"
|
||||
|
||||
# Define a replacement function that elongates the captured character
|
||||
def repl(match):
|
||||
char = match.group(1) # The character before the ー sequence
|
||||
count = len(match.group(0)) - 1 # Number of ー characters
|
||||
return char * count # Replace ー sequence with the character repeated
|
||||
|
||||
# Use re.sub() to replace the pattern in the text
|
||||
return re.sub(pattern, repl, text)
|
||||
|
||||
|
||||
def extractTranslation(translatedTextList, is_list):
|
||||
try:
|
||||
translatedTextList = re.sub(r'\\"+\"([^,\n}])', r'\\"\1', translatedTextList)
|
||||
translatedTextList = re.sub(r"(?<![\\])\"+(?![\n,])", r'"', translatedTextList)
|
||||
line_dict = json.loads(translatedTextList)
|
||||
# If it's a batch (i.e., list), extract with tags; otherwise, return the single item.
|
||||
string_list = list(line_dict.values())
|
||||
if is_list:
|
||||
return string_list
|
||||
else:
|
||||
return string_list[0]
|
||||
|
||||
except Exception as e:
|
||||
PBAR.write(f"extractTranslation Error: {e} on String {translatedTextList}")
|
||||
return None
|
||||
|
||||
|
||||
def countTokens(system, user, history):
|
||||
inputTotalTokens = 0
|
||||
outputTotalTokens = 0
|
||||
enc = tiktoken.encoding_for_model("gpt-4")
|
||||
|
||||
# Input
|
||||
if isinstance(history, list):
|
||||
for line in history:
|
||||
inputTotalTokens += len(enc.encode(line))
|
||||
else:
|
||||
inputTotalTokens += len(enc.encode(history))
|
||||
inputTotalTokens += len(enc.encode(system))
|
||||
inputTotalTokens += len(enc.encode(user))
|
||||
|
||||
# Output
|
||||
outputTotalTokens += round(len(enc.encode(user)) * 2.5)
|
||||
|
||||
return [inputTotalTokens, outputTotalTokens]
|
||||
|
||||
|
||||
@retry(exceptions=Exception, tries=5, delay=5)
|
||||
def translateGPT(text, history, fullPromptFlag):
|
||||
def translateAI(text, history, fullPromptFlag):
|
||||
"""
|
||||
Legacy wrapper function for the new shared translation utility.
|
||||
This maintains compatibility with existing code while using the new shared implementation.
|
||||
"""
|
||||
global PBAR, MISMATCH, FILENAME
|
||||
if text:
|
||||
with open("log/translationHistory.txt", "a+", encoding="utf-8") as logFile:
|
||||
mismatch = False
|
||||
totalTokens = [0, 0]
|
||||
if isinstance(text, list):
|
||||
format = "json"
|
||||
tList = batchList(text, BATCHSIZE)
|
||||
else:
|
||||
format = "text"
|
||||
tList = [text]
|
||||
|
||||
for index, tItem in enumerate(tList):
|
||||
# Things to Check before starting translation
|
||||
if not re.search(LANGREGEX, str(tItem)):
|
||||
if PBAR is not None:
|
||||
PBAR.update(len(tItem))
|
||||
if isinstance(tItem, list):
|
||||
for j in range(len(tItem)):
|
||||
tItem[j] = cleanTranslatedText(tItem[j])
|
||||
tList[index] = tItem
|
||||
else:
|
||||
tList[index] = cleanTranslatedText(tItem)
|
||||
history = tItem[-MAXHISTORY:]
|
||||
continue
|
||||
# Update config estimate mode based on global ESTIMATE
|
||||
TRANSLATION_CONFIG.estimateMode = bool(ESTIMATE)
|
||||
|
||||
# Before sending to translation, if we have a list of items, add the formatting
|
||||
if isinstance(tItem, list):
|
||||
for j in range(len(tItem)):
|
||||
if not tItem[j]:
|
||||
tItem[j] = tItem[j].replace("", "Placeholder Text")
|
||||
payload = {f"Line{i+1}": string for i, string in enumerate(tItem)}
|
||||
payload = json.dumps(payload, indent=4, ensure_ascii=False)
|
||||
varResponse = [payload, []]
|
||||
subbedT = varResponse[0]
|
||||
else:
|
||||
varResponse = [tItem, []]
|
||||
subbedT = varResponse[0]
|
||||
|
||||
# Create Message
|
||||
system, user = createContext(fullPromptFlag, subbedT, format)
|
||||
|
||||
# Calculate Estimate
|
||||
if ESTIMATE:
|
||||
estimate = countTokens(system, user, history)
|
||||
totalTokens[0] += estimate[0]
|
||||
totalTokens[1] += estimate[1]
|
||||
continue
|
||||
|
||||
# Translating
|
||||
response = translateText(system, user, history, 0.05, format)
|
||||
|
||||
# Set Tokens
|
||||
translatedText = response.choices[0].message.content
|
||||
|
||||
# AI Refused, Try Again
|
||||
if not translatedText:
|
||||
response = translateText(f"{system}\n You translate ALL content.", user, history, 0.1, format, model="gpt-4o")
|
||||
translatedText = response.choices[0].message.content
|
||||
|
||||
# Report Tokens
|
||||
totalTokens[0] += response.usage.prompt_tokens
|
||||
totalTokens[1] += response.usage.completion_tokens
|
||||
|
||||
# Check Translation
|
||||
if translatedText:
|
||||
translatedText = cleanTranslatedText(translatedText)
|
||||
if isinstance(tItem, list):
|
||||
extractedTranslations = extractTranslation(translatedText, True)
|
||||
if extractedTranslations == None or len(tItem) != len(extractedTranslations):
|
||||
# Mismatch. Try Again
|
||||
response = translateText(system, user, history, 0.05, format, MODEL)
|
||||
translatedText = response.choices[0].message.content
|
||||
totalTokens[0] += response.usage.prompt_tokens
|
||||
totalTokens[1] += response.usage.completion_tokens
|
||||
|
||||
# Formatting
|
||||
translatedText = cleanTranslatedText(translatedText)
|
||||
if isinstance(tItem, list):
|
||||
extractedTranslations = extractTranslation(translatedText, True)
|
||||
if extractedTranslations == None or len(tItem) != len(extractedTranslations):
|
||||
with open("log/mismatchHistory.txt", "a+", encoding="utf-8") as mismatchFile:
|
||||
mismatchFile.write(f"Mismatch: {FILENAME}\n")
|
||||
mismatchFile.write(f"Input:\n{subbedT}\n")
|
||||
mismatchFile.write(f"Output:\n{translatedText}\n")
|
||||
mismatch = True # Just here for breakpoint
|
||||
logFile.write(f"Input:\n{subbedT}\n")
|
||||
logFile.write(f"Output:\n{translatedText}\n")
|
||||
|
||||
# Set if no mismatch
|
||||
if mismatch == False:
|
||||
tList[index] = extractedTranslations
|
||||
history = extractedTranslations[-MAXHISTORY:] # Update history if we have a list
|
||||
else:
|
||||
history = text[-MAXHISTORY:]
|
||||
mismatch = False
|
||||
if FILENAME not in MISMATCH:
|
||||
MISMATCH.append(FILENAME)
|
||||
|
||||
# Update Loading Bar
|
||||
with LOCK:
|
||||
if PBAR is not None:
|
||||
PBAR.update(len(tItem))
|
||||
else:
|
||||
# Ensure we're passing a single string to extractTranslation
|
||||
tList[index] = translatedText.replace("Placeholder Text", "")
|
||||
else:
|
||||
PBAR.write(f"AI Refused:{tItem}\n")
|
||||
|
||||
# Combine if multilist
|
||||
if isinstance(tList[0], list):
|
||||
tList = [t for sublist in tList for t in sublist]
|
||||
|
||||
# Return
|
||||
if format == "json":
|
||||
return [tList, totalTokens]
|
||||
else:
|
||||
return [tList[0], totalTokens]
|
||||
else:
|
||||
return [text, [0, 0]]
|
||||
# Call the new shared translation function
|
||||
return sharedtranslateAI(
|
||||
text=text,
|
||||
history=history,
|
||||
fullPromptFlag=fullPromptFlag,
|
||||
config=TRANSLATION_CONFIG,
|
||||
filename=FILENAME,
|
||||
pbar=PBAR,
|
||||
lock=LOCK,
|
||||
mismatchList=MISMATCH
|
||||
)
|
||||
|
|
|
|||
361
modules/lune.py
361
modules/lune.py
|
|
@ -13,6 +13,7 @@ from colorama import Fore
|
|||
from dotenv import load_dotenv
|
||||
from retry import retry
|
||||
from tqdm import tqdm
|
||||
from util.translation import TranslationConfig, translateAI as sharedtranslateAI
|
||||
|
||||
# Open AI
|
||||
load_dotenv()
|
||||
|
|
@ -76,6 +77,18 @@ else:
|
|||
BATCHSIZE = int(os.getenv("batchsize"))
|
||||
FREQUENCY_PENALTY = float(os.getenv("frequency_penalty"))
|
||||
|
||||
# Initialize Translation Config
|
||||
TRANSLATION_CONFIG = TranslationConfig(
|
||||
model=MODEL,
|
||||
language=LANGUAGE,
|
||||
prompt=PROMPT,
|
||||
vocab=VOCAB,
|
||||
langRegex=LANGREGEX,
|
||||
batchSize=BATCHSIZE,
|
||||
maxHistory=MAXHISTORY,
|
||||
estimateMode=False # Will be set dynamically based on ESTIMATE
|
||||
)
|
||||
LEAVE = False
|
||||
|
||||
def handleLune(filename, estimate):
|
||||
global FILENAME, ESTIMATE, totalTokens
|
||||
|
|
@ -225,7 +238,7 @@ def translateJSON(data, pbar):
|
|||
# Translate Batch if Full
|
||||
if len(batch) == BATCHSIZE:
|
||||
# Translate
|
||||
response = translateGPT(batch, textHistory, True)
|
||||
response = translateAI(batch, textHistory, True)
|
||||
tokens[0] += response[1][0]
|
||||
tokens[1] += response[1][1]
|
||||
translatedBatch = response[0]
|
||||
|
|
@ -279,7 +292,7 @@ def translateJSON(data, pbar):
|
|||
# Translate Batch if not empty and EOF
|
||||
if len(batch) != 0 and i >= len(data):
|
||||
# Translate
|
||||
response = translateGPT(batch, textHistory, True)
|
||||
response = translateAI(batch, textHistory, True)
|
||||
tokens[0] += response[1][0]
|
||||
tokens[1] += response[1][1]
|
||||
translatedBatch = response[0]
|
||||
|
|
@ -314,7 +327,7 @@ def getSpeaker(speaker):
|
|||
return [NAMESLIST[i][1], [0, 0]]
|
||||
|
||||
# Translate and Store Speaker
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
f"{speaker}",
|
||||
"Reply with the " + LANGUAGE + " translation of the NPC name.",
|
||||
False,
|
||||
|
|
@ -325,7 +338,7 @@ def getSpeaker(speaker):
|
|||
|
||||
# Retry if name doesn't translate for some reason
|
||||
if re.search(r"([a-zA-Z？?])", response[0]) == None:
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
f"{speaker}",
|
||||
"Reply with the " + LANGUAGE + " translation of the NPC name.",
|
||||
False,
|
||||
|
|
@ -338,328 +351,24 @@ def getSpeaker(speaker):
|
|||
return response
|
||||
return [speaker, [0, 0]]
|
||||
|
||||
|
||||
def batchList(input_list, batch_size):
|
||||
if not isinstance(batch_size, int) or batch_size <= 0:
|
||||
raise ValueError("batch_size must be a positive integer")
|
||||
|
||||
return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)]
|
||||
|
||||
|
||||
def parseVocabWithCategories(vocabText):
|
||||
"""Parse vocabulary text and extract terms with their categories."""
|
||||
pairs = []
|
||||
seen = set()
|
||||
currentCategory = None
|
||||
|
||||
for line in vocabText.splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith('```'):
|
||||
continue
|
||||
|
||||
# Check if this is a category header
|
||||
if line.startswith('#'):
|
||||
currentCategory = line
|
||||
continue
|
||||
|
||||
# Parse vocabulary term
|
||||
m = re.match(r'^(.+?)(?:\s?[\(–])', line) # term is everything before space + '(' or '–'
|
||||
if m:
|
||||
term = m.group(1)
|
||||
if term not in seen:
|
||||
pairs.append((term, line, currentCategory))
|
||||
seen.add(term)
|
||||
|
||||
return pairs
|
||||
|
||||
|
||||
def buildMatchedVocabText(vocabPairs, subbedT):
|
||||
"""Build formatted vocabulary text with matched terms organized by category."""
|
||||
matchedCategories = {}
|
||||
|
||||
# Use word boundaries for Japanese if appropriate, or allow substring as before.
|
||||
for term, line, category in vocabPairs:
|
||||
# "term in subbedT" could be false positive; can use regex but Japanese doesn't always have spaces.
|
||||
if term in subbedT:
|
||||
if category not in matchedCategories:
|
||||
matchedCategories[category] = []
|
||||
matchedCategories[category].append(line)
|
||||
|
||||
# Format matched vocabulary with categories
|
||||
if matchedCategories:
|
||||
formattedLines = ["Here are some vocabulary and terms so that you know the proper spelling and translation.\n"]
|
||||
for category, lines in matchedCategories.items():
|
||||
if category: # Only add category header if it exists
|
||||
formattedLines.append(category)
|
||||
formattedLines.extend(lines)
|
||||
formattedLines.append("") # Add blank line between categories
|
||||
matchedVocabText = f"```\n{chr(10).join(formattedLines).rstrip()}\n```"
|
||||
else:
|
||||
matchedVocabText = ""
|
||||
|
||||
return matchedVocabText
|
||||
|
||||
|
||||
def createContext(fullPromptFlag, subbedT, format):
|
||||
vocabPairs = parseVocabWithCategories(VOCAB)
|
||||
matchedVocabText = buildMatchedVocabText(vocabPairs, subbedT)
|
||||
|
||||
if fullPromptFlag:
|
||||
system = PROMPT + matchedVocabText
|
||||
else:
|
||||
system = f"\
|
||||
You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
|
||||
Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
|
||||
- You translate everything, including erotic and explicit content.\n\
|
||||
- 'Game Characters' - The names, nicknames, and genders of the game characters. Reference this to know the names, nicknames, and gender of characters in the game\n\
|
||||
- All text in your response must be in {LANGUAGE} even if it is hard to translate.\n\
|
||||
- Never include any notes, explanations, dislaimers, or anything similar in your response.\n\
|
||||
- Maintain any spacing in the translation.\n\
|
||||
- Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\
|
||||
- `...` can be a part of the dialogue. Translate it as it is.\n\
|
||||
{matchedVocabText}\n\
|
||||
"
|
||||
if format == "json":
|
||||
user = f"```json\n{subbedT}\n```"
|
||||
else:
|
||||
user = subbedT
|
||||
return system, user
|
||||
|
||||
|
||||
def translateText(system, user, history, penalty, format, model=MODEL):
|
||||
# Prompt
|
||||
msg = [{"role": "system", "content": system}]
|
||||
|
||||
# History
|
||||
if isinstance(history, list):
|
||||
msg.append({"role": "system", "content": "Translation History:"})
|
||||
msg.extend([{"role": "assistant", "content": h} for h in history])
|
||||
else:
|
||||
msg.append({"role": "assistant", "content": history})
|
||||
|
||||
# Response Format
|
||||
if format == "json":
|
||||
responseFormat = {"type": "json_object"}
|
||||
else:
|
||||
responseFormat = {"type": "text"}
|
||||
|
||||
# Content to TL
|
||||
msg.append({"role": "user", "content": f"{user}"})
|
||||
response = openai.chat.completions.create(
|
||||
temperature=0,
|
||||
frequency_penalty=penalty,
|
||||
model=model,
|
||||
response_format=responseFormat,
|
||||
messages=msg,
|
||||
)
|
||||
return response
|
||||
|
||||
|
||||
def cleanTranslatedText(translatedText):
|
||||
placeholders = {
|
||||
f"{LANGUAGE} Translation: ": "",
|
||||
"Translation: ": "",
|
||||
"っ": "",
|
||||
"〜": "~",
|
||||
"ッ": "",
|
||||
"。": ".",
|
||||
"「": '\\"',
|
||||
"」": '\\"',
|
||||
"- ": "-",
|
||||
"—": "―",
|
||||
"】": "]",
|
||||
"【": "[",
|
||||
"é": "e",
|
||||
"this guy": "this bastard",
|
||||
"This guy": "This bastard",
|
||||
"Placeholder Text": "",
|
||||
"```json": "",
|
||||
"```": "",
|
||||
# Add more replacements as needed
|
||||
}
|
||||
for target, replacement in placeholders.items():
|
||||
translatedText = translatedText.replace(target, replacement)
|
||||
|
||||
# Remove Repeating Characters
|
||||
pattern = re.compile(r"(.)\s*\1(?:\s*\1){" + str(20 - 1) + r",}")
|
||||
translatedText = pattern.sub(lambda match: match.group(0).replace(" ", "")[:20], translatedText)
|
||||
|
||||
# Elongate Long Dashes (Since GPT Ignores them...)
|
||||
translatedText = elongateCharacters(translatedText)
|
||||
return translatedText
|
||||
|
||||
|
||||
def elongateCharacters(text):
|
||||
# Define a pattern to match one character followed by one or more `ー` characters
|
||||
# Using a positive lookbehind assertion to capture the preceding character
|
||||
pattern = r"(?<=(.))ー+"
|
||||
|
||||
# Define a replacement function that elongates the captured character
|
||||
def repl(match):
|
||||
char = match.group(1) # The character before the ー sequence
|
||||
count = len(match.group(0)) - 1 # Number of ー characters
|
||||
return char * count # Replace ー sequence with the character repeated
|
||||
|
||||
# Use re.sub() to replace the pattern in the text
|
||||
return re.sub(pattern, repl, text)
|
||||
|
||||
|
||||
def extractTranslation(translatedTextList, is_list):
|
||||
try:
|
||||
translatedTextList = re.sub(r'\\"+\"([^,\n}])', r'\\"\1', translatedTextList)
|
||||
translatedTextList = re.sub(r"(?<![\\])\"+(?![\n,])", r'"', translatedTextList)
|
||||
line_dict = json.loads(translatedTextList)
|
||||
# If it's a batch (i.e., list), extract with tags; otherwise, return the single item.
|
||||
string_list = list(line_dict.values())
|
||||
if is_list:
|
||||
return string_list
|
||||
else:
|
||||
return string_list[0]
|
||||
|
||||
except Exception as e:
|
||||
PBAR.write(f"extractTranslation Error: {e} on String {translatedTextList}")
|
||||
return None
|
||||
|
||||
|
||||
def countTokens(system, user, history):
|
||||
inputTotalTokens = 0
|
||||
outputTotalTokens = 0
|
||||
enc = tiktoken.encoding_for_model("gpt-4")
|
||||
|
||||
# Input
|
||||
if isinstance(history, list):
|
||||
for line in history:
|
||||
inputTotalTokens += len(enc.encode(line))
|
||||
else:
|
||||
inputTotalTokens += len(enc.encode(history))
|
||||
inputTotalTokens += len(enc.encode(system))
|
||||
inputTotalTokens += len(enc.encode(user))
|
||||
|
||||
# Output
|
||||
outputTotalTokens += round(len(enc.encode(user)) * 2.5)
|
||||
|
||||
return [inputTotalTokens, outputTotalTokens]
|
||||
|
||||
|
||||
@retry(exceptions=Exception, tries=5, delay=5)
|
||||
def translateGPT(text, history, fullPromptFlag):
|
||||
def translateAI(text, history, fullPromptFlag):
|
||||
"""
|
||||
Legacy wrapper function for the new shared translation utility.
|
||||
This maintains compatibility with existing code while using the new shared implementation.
|
||||
"""
|
||||
global PBAR, MISMATCH, FILENAME
|
||||
if text:
|
||||
with open("log/translationHistory.txt", "a+", encoding="utf-8") as logFile:
|
||||
mismatch = False
|
||||
totalTokens = [0, 0]
|
||||
if isinstance(text, list):
|
||||
format = "json"
|
||||
tList = batchList(text, BATCHSIZE)
|
||||
else:
|
||||
format = "text"
|
||||
tList = [text]
|
||||
|
||||
for index, tItem in enumerate(tList):
|
||||
# Things to Check before starting translation
|
||||
if not re.search(LANGREGEX, str(tItem)):
|
||||
if PBAR is not None:
|
||||
PBAR.update(len(tItem))
|
||||
if isinstance(tItem, list):
|
||||
for j in range(len(tItem)):
|
||||
tItem[j] = cleanTranslatedText(tItem[j])
|
||||
tList[index] = tItem
|
||||
else:
|
||||
tList[index] = cleanTranslatedText(tItem)
|
||||
history = tItem[-MAXHISTORY:]
|
||||
continue
|
||||
# Update config estimate mode based on global ESTIMATE
|
||||
TRANSLATION_CONFIG.estimateMode = bool(ESTIMATE)
|
||||
|
||||
# Before sending to translation, if we have a list of items, add the formatting
|
||||
if isinstance(tItem, list):
|
||||
for j in range(len(tItem)):
|
||||
if not tItem[j]:
|
||||
tItem[j] = tItem[j].replace("", "Placeholder Text")
|
||||
payload = {f"Line{i+1}": string for i, string in enumerate(tItem)}
|
||||
payload = json.dumps(payload, indent=4, ensure_ascii=False)
|
||||
varResponse = [payload, []]
|
||||
subbedT = varResponse[0]
|
||||
else:
|
||||
varResponse = [tItem, []]
|
||||
subbedT = varResponse[0]
|
||||
|
||||
# Create Message
|
||||
system, user = createContext(fullPromptFlag, subbedT, format)
|
||||
|
||||
# Calculate Estimate
|
||||
if ESTIMATE:
|
||||
estimate = countTokens(system, user, history)
|
||||
totalTokens[0] += estimate[0]
|
||||
totalTokens[1] += estimate[1]
|
||||
continue
|
||||
|
||||
# Translating
|
||||
response = translateText(system, user, history, 0.05, format)
|
||||
|
||||
# Set Tokens
|
||||
translatedText = response.choices[0].message.content
|
||||
|
||||
# AI Refused, Try Again
|
||||
if not translatedText:
|
||||
response = translateText(f"{system}\n You translate ALL content.", user, history, 0.1, format, model="gpt-4o")
|
||||
translatedText = response.choices[0].message.content
|
||||
|
||||
# Report Tokens
|
||||
totalTokens[0] += response.usage.prompt_tokens
|
||||
totalTokens[1] += response.usage.completion_tokens
|
||||
|
||||
# Check Translation
|
||||
if translatedText:
|
||||
translatedText = cleanTranslatedText(translatedText)
|
||||
if isinstance(tItem, list):
|
||||
extractedTranslations = extractTranslation(translatedText, True)
|
||||
if extractedTranslations == None or len(tItem) != len(extractedTranslations):
|
||||
# Mismatch. Try Again
|
||||
response = translateText(system, user, history, 0.05, format, MODEL)
|
||||
translatedText = response.choices[0].message.content
|
||||
totalTokens[0] += response.usage.prompt_tokens
|
||||
totalTokens[1] += response.usage.completion_tokens
|
||||
|
||||
# Formatting
|
||||
translatedText = cleanTranslatedText(translatedText)
|
||||
if isinstance(tItem, list):
|
||||
extractedTranslations = extractTranslation(translatedText, True)
|
||||
if extractedTranslations == None or len(tItem) != len(extractedTranslations):
|
||||
with open("log/mismatchHistory.txt", "a+", encoding="utf-8") as mismatchFile:
|
||||
mismatchFile.write(f"Mismatch: {FILENAME}\n")
|
||||
mismatchFile.write(f"Input:\n{subbedT}\n")
|
||||
mismatchFile.write(f"Output:\n{translatedText}\n")
|
||||
mismatch = True # Just here for breakpoint
|
||||
logFile.write(f"Input:\n{subbedT}\n")
|
||||
logFile.write(f"Output:\n{translatedText}\n")
|
||||
|
||||
# Set if no mismatch
|
||||
if mismatch == False:
|
||||
tList[index] = extractedTranslations
|
||||
history = extractedTranslations[-MAXHISTORY:] # Update history if we have a list
|
||||
else:
|
||||
history = text[-MAXHISTORY:]
|
||||
mismatch = False
|
||||
if FILENAME not in MISMATCH:
|
||||
MISMATCH.append(FILENAME)
|
||||
|
||||
# Update Loading Bar
|
||||
with LOCK:
|
||||
if PBAR is not None:
|
||||
PBAR.update(len(tItem))
|
||||
else:
|
||||
# Ensure we're passing a single string to extractTranslation
|
||||
tList[index] = translatedText.replace("Placeholder Text", "")
|
||||
else:
|
||||
PBAR.write(f"AI Refused:{tItem}\n")
|
||||
|
||||
# Combine if multilist
|
||||
if isinstance(tList[0], list):
|
||||
tList = [t for sublist in tList for t in sublist]
|
||||
|
||||
# Return
|
||||
if format == "json":
|
||||
return [tList, totalTokens]
|
||||
else:
|
||||
return [tList[0], totalTokens]
|
||||
else:
|
||||
return [text, [0, 0]]
|
||||
# Call the new shared translation function
|
||||
return sharedtranslateAI(
|
||||
text=text,
|
||||
history=history,
|
||||
fullPromptFlag=fullPromptFlag,
|
||||
config=TRANSLATION_CONFIG,
|
||||
filename=FILENAME,
|
||||
pbar=PBAR,
|
||||
lock=LOCK,
|
||||
mismatchList=MISMATCH
|
||||
)
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ from colorama import Fore
|
|||
from dotenv import load_dotenv
|
||||
from retry import retry
|
||||
from tqdm import tqdm
|
||||
from util.translation import TranslationConfig, translateAI as sharedtranslateAI
|
||||
|
||||
# Open AI
|
||||
load_dotenv()
|
||||
|
|
@ -83,6 +84,18 @@ else:
|
|||
BATCHSIZE = int(os.getenv("batchsize"))
|
||||
FREQUENCY_PENALTY = float(os.getenv("frequency_penalty"))
|
||||
|
||||
# Initialize Translation Config
|
||||
TRANSLATION_CONFIG = TranslationConfig(
|
||||
model=MODEL,
|
||||
language=LANGUAGE,
|
||||
prompt=PROMPT,
|
||||
vocab=VOCAB,
|
||||
langRegex=LANGREGEX,
|
||||
batchSize=BATCHSIZE,
|
||||
maxHistory=MAXHISTORY,
|
||||
estimateMode=False # Will be set dynamically based on ESTIMATE
|
||||
)
|
||||
LEAVE = False
|
||||
|
||||
def handleOnscripter(filename, estimate):
|
||||
global ESTIMATE, FILENAME
|
||||
|
|
@ -285,7 +298,7 @@ def translateOnscripter(data, pbar, filename, translatedList):
|
|||
choiceList = re.findall(r"\"(.*?)\"", jaString)
|
||||
if len(choiceList) > 0:
|
||||
# Translate
|
||||
response = translateGPT(choiceList, "This will be a dialogue option", True)
|
||||
response = translateAI(choiceList, "This will be a dialogue option", True)
|
||||
translatedTextList = response[0]
|
||||
tokens[0] += response[1][0]
|
||||
tokens[1] += response[1][1]
|
||||
|
|
@ -311,7 +324,7 @@ def translateOnscripter(data, pbar, filename, translatedList):
|
|||
pbar.refresh()
|
||||
|
||||
# Translate
|
||||
response = translateGPT(stringList, "", True)
|
||||
response = translateAI(stringList, "", True)
|
||||
tokens[0] += response[1][0]
|
||||
tokens[1] += response[1][1]
|
||||
translatedList = response[0]
|
||||
|
|
@ -373,7 +386,7 @@ def getSpeaker(speaker):
|
|||
return [NAMESLIST[i][1], [0, 0]]
|
||||
|
||||
# Translate and Store Speaker
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
f"{speaker}",
|
||||
"Reply with the " + LANGUAGE + " translation of the NPC name.",
|
||||
False,
|
||||
|
|
@ -384,7 +397,7 @@ def getSpeaker(speaker):
|
|||
|
||||
# Retry if name doesn't translate for some reason
|
||||
if re.search(r"([a-zA-Z？?])", response[0]) == None:
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
f"{speaker}",
|
||||
"Reply with the " + LANGUAGE + " translation of the NPC name.",
|
||||
False,
|
||||
|
|
@ -398,327 +411,9 @@ def getSpeaker(speaker):
|
|||
return [speaker, [0, 0]]
|
||||
|
||||
|
||||
def batchList(input_list, batch_size):
|
||||
if not isinstance(batch_size, int) or batch_size <= 0:
|
||||
raise ValueError("batch_size must be a positive integer")
|
||||
|
||||
return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)]
|
||||
|
||||
|
||||
def parseVocabWithCategories(vocabText):
|
||||
"""Parse vocabulary text and extract terms with their categories."""
|
||||
pairs = []
|
||||
seen = set()
|
||||
currentCategory = None
|
||||
|
||||
for line in vocabText.splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith('```'):
|
||||
continue
|
||||
|
||||
# Check if this is a category header
|
||||
if line.startswith('#'):
|
||||
currentCategory = line
|
||||
continue
|
||||
|
||||
# Parse vocabulary term
|
||||
m = re.match(r'^(.+?)(?:\s?[\(–])', line) # term is everything before space + '(' or '–'
|
||||
if m:
|
||||
term = m.group(1)
|
||||
if term not in seen:
|
||||
pairs.append((term, line, currentCategory))
|
||||
seen.add(term)
|
||||
|
||||
return pairs
|
||||
|
||||
|
||||
def buildMatchedVocabText(vocabPairs, subbedT):
|
||||
"""Build formatted vocabulary text with matched terms organized by category."""
|
||||
matchedCategories = {}
|
||||
|
||||
# Use word boundaries for Japanese if appropriate, or allow substring as before.
|
||||
for term, line, category in vocabPairs:
|
||||
# "term in subbedT" could be false positive; can use regex but Japanese doesn't always have spaces.
|
||||
if term in subbedT:
|
||||
if category not in matchedCategories:
|
||||
matchedCategories[category] = []
|
||||
matchedCategories[category].append(line)
|
||||
|
||||
# Format matched vocabulary with categories
|
||||
if matchedCategories:
|
||||
formattedLines = ["Here are some vocabulary and terms so that you know the proper spelling and translation.\n"]
|
||||
for category, lines in matchedCategories.items():
|
||||
if category: # Only add category header if it exists
|
||||
formattedLines.append(category)
|
||||
formattedLines.extend(lines)
|
||||
formattedLines.append("") # Add blank line between categories
|
||||
matchedVocabText = f"```\n{chr(10).join(formattedLines).rstrip()}\n```"
|
||||
else:
|
||||
matchedVocabText = ""
|
||||
|
||||
return matchedVocabText
|
||||
|
||||
|
||||
def createContext(fullPromptFlag, subbedT, format):
|
||||
vocabPairs = parseVocabWithCategories(VOCAB)
|
||||
matchedVocabText = buildMatchedVocabText(vocabPairs, subbedT)
|
||||
|
||||
if fullPromptFlag:
|
||||
system = PROMPT + matchedVocabText
|
||||
else:
|
||||
system = f"\
|
||||
You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
|
||||
Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
|
||||
- You translate everything, including erotic and explicit content.\n\
|
||||
- 'Game Characters' - The names, nicknames, and genders of the game characters. Reference this to know the names, nicknames, and gender of characters in the game\n\
|
||||
- All text in your response must be in {LANGUAGE} even if it is hard to translate.\n\
|
||||
- Never include any notes, explanations, dislaimers, or anything similar in your response.\n\
|
||||
- Maintain any spacing in the translation.\n\
|
||||
- Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\
|
||||
- `...` can be a part of the dialogue. Translate it as it is.\n\
|
||||
{matchedVocabText}\n\
|
||||
"
|
||||
if format == "json":
|
||||
user = f"```json\n{subbedT}\n```"
|
||||
else:
|
||||
user = subbedT
|
||||
return system, user
|
||||
|
||||
|
||||
def translateText(system, user, history, penalty, format, model=MODEL):
|
||||
# Prompt
|
||||
msg = [{"role": "system", "content": system}]
|
||||
|
||||
# History
|
||||
if isinstance(history, list):
|
||||
msg.append({"role": "system", "content": "Translation History:"})
|
||||
msg.extend([{"role": "assistant", "content": h} for h in history])
|
||||
else:
|
||||
msg.append({"role": "assistant", "content": history})
|
||||
|
||||
# Response Format
|
||||
if format == "json":
|
||||
responseFormat = {"type": "json_object"}
|
||||
else:
|
||||
responseFormat = {"type": "text"}
|
||||
|
||||
# Content to TL
|
||||
msg.append({"role": "user", "content": f"{user}"})
|
||||
response = openai.chat.completions.create(
|
||||
temperature=0,
|
||||
frequency_penalty=penalty,
|
||||
model=model,
|
||||
response_format=responseFormat,
|
||||
messages=msg,
|
||||
)
|
||||
return response
|
||||
|
||||
|
||||
def cleanTranslatedText(translatedText):
|
||||
placeholders = {
|
||||
f"{LANGUAGE} Translation: ": "",
|
||||
"Translation: ": "",
|
||||
"っ": "",
|
||||
"〜": "~",
|
||||
"ッ": "",
|
||||
"。": ".",
|
||||
"「": '\\"',
|
||||
"」": '\\"',
|
||||
"- ": "-",
|
||||
"—": "―",
|
||||
"】": "]",
|
||||
"【": "[",
|
||||
"é": "e",
|
||||
"this guy": "this bastard",
|
||||
"This guy": "This bastard",
|
||||
"Placeholder Text": "",
|
||||
"```json": "",
|
||||
"```": "",
|
||||
# Add more replacements as needed
|
||||
}
|
||||
for target, replacement in placeholders.items():
|
||||
translatedText = translatedText.replace(target, replacement)
|
||||
|
||||
# Remove Repeating Characters
|
||||
pattern = re.compile(r"(.)\s*\1(?:\s*\1){" + str(20 - 1) + r",}")
|
||||
translatedText = pattern.sub(lambda match: match.group(0).replace(" ", "")[:20], translatedText)
|
||||
|
||||
# Elongate Long Dashes (Since GPT Ignores them...)
|
||||
translatedText = elongateCharacters(translatedText)
|
||||
return translatedText
|
||||
|
||||
|
||||
def elongateCharacters(text):
|
||||
# Define a pattern to match one character followed by one or more `ー` characters
|
||||
# Using a positive lookbehind assertion to capture the preceding character
|
||||
pattern = r"(?<=(.))ー+"
|
||||
|
||||
# Define a replacement function that elongates the captured character
|
||||
def repl(match):
|
||||
char = match.group(1) # The character before the ー sequence
|
||||
count = len(match.group(0)) - 1 # Number of ー characters
|
||||
return char * count # Replace ー sequence with the character repeated
|
||||
|
||||
# Use re.sub() to replace the pattern in the text
|
||||
return re.sub(pattern, repl, text)
|
||||
|
||||
|
||||
def extractTranslation(translatedTextList, is_list):
|
||||
try:
|
||||
translatedTextList = re.sub(r'\\"+\"([^,\n}])', r'\\"\1', translatedTextList)
|
||||
translatedTextList = re.sub(r"(?<![\\])\"+(?![\n,])", r'"', translatedTextList)
|
||||
line_dict = json.loads(translatedTextList)
|
||||
# If it's a batch (i.e., list), extract with tags; otherwise, return the single item.
|
||||
string_list = list(line_dict.values())
|
||||
if is_list:
|
||||
return string_list
|
||||
else:
|
||||
return string_list[0]
|
||||
|
||||
except Exception as e:
|
||||
PBAR.write(f"extractTranslation Error: {e} on String {translatedTextList}")
|
||||
return None
|
||||
|
||||
|
||||
def countTokens(system, user, history):
|
||||
inputTotalTokens = 0
|
||||
outputTotalTokens = 0
|
||||
enc = tiktoken.encoding_for_model("gpt-4")
|
||||
|
||||
# Input
|
||||
if isinstance(history, list):
|
||||
for line in history:
|
||||
inputTotalTokens += len(enc.encode(line))
|
||||
else:
|
||||
inputTotalTokens += len(enc.encode(history))
|
||||
inputTotalTokens += len(enc.encode(system))
|
||||
inputTotalTokens += len(enc.encode(user))
|
||||
|
||||
# Output
|
||||
outputTotalTokens += round(len(enc.encode(user)) * 2.5)
|
||||
|
||||
return [inputTotalTokens, outputTotalTokens]
|
||||
|
||||
|
||||
@retry(exceptions=Exception, tries=5, delay=5)
|
||||
def translateGPT(text, history, fullPromptFlag):
|
||||
global PBAR, MISMATCH, FILENAME
|
||||
if text:
|
||||
with open("log/translationHistory.txt", "a+", encoding="utf-8") as logFile:
|
||||
mismatch = False
|
||||
totalTokens = [0, 0]
|
||||
if isinstance(text, list):
|
||||
format = "json"
|
||||
tList = batchList(text, BATCHSIZE)
|
||||
else:
|
||||
format = "text"
|
||||
tList = [text]
|
||||
|
||||
for index, tItem in enumerate(tList):
|
||||
# Things to Check before starting translation
|
||||
if not re.search(LANGREGEX, str(tItem)):
|
||||
if PBAR is not None:
|
||||
PBAR.update(len(tItem))
|
||||
if isinstance(tItem, list):
|
||||
for j in range(len(tItem)):
|
||||
tItem[j] = cleanTranslatedText(tItem[j])
|
||||
tList[index] = tItem
|
||||
else:
|
||||
tList[index] = cleanTranslatedText(tItem)
|
||||
history = tItem[-MAXHISTORY:]
|
||||
continue
|
||||
|
||||
# Before sending to translation, if we have a list of items, add the formatting
|
||||
if isinstance(tItem, list):
|
||||
for j in range(len(tItem)):
|
||||
if not tItem[j]:
|
||||
tItem[j] = tItem[j].replace("", "Placeholder Text")
|
||||
payload = {f"Line{i+1}": string for i, string in enumerate(tItem)}
|
||||
payload = json.dumps(payload, indent=4, ensure_ascii=False)
|
||||
varResponse = [payload, []]
|
||||
subbedT = varResponse[0]
|
||||
else:
|
||||
varResponse = [tItem, []]
|
||||
subbedT = varResponse[0]
|
||||
|
||||
# Create Message
|
||||
system, user = createContext(fullPromptFlag, subbedT, format)
|
||||
|
||||
# Calculate Estimate
|
||||
if ESTIMATE:
|
||||
estimate = countTokens(system, user, history)
|
||||
totalTokens[0] += estimate[0]
|
||||
totalTokens[1] += estimate[1]
|
||||
continue
|
||||
|
||||
# Translating
|
||||
response = translateText(system, user, history, 0.05, format)
|
||||
|
||||
# Set Tokens
|
||||
translatedText = response.choices[0].message.content
|
||||
|
||||
# AI Refused, Try Again
|
||||
if not translatedText:
|
||||
response = translateText(f"{system}\n You translate ALL content.", user, history, 0.1, format, model="gpt-4o")
|
||||
translatedText = response.choices[0].message.content
|
||||
|
||||
# Report Tokens
|
||||
totalTokens[0] += response.usage.prompt_tokens
|
||||
totalTokens[1] += response.usage.completion_tokens
|
||||
|
||||
# Check Translation
|
||||
if translatedText:
|
||||
translatedText = cleanTranslatedText(translatedText)
|
||||
if isinstance(tItem, list):
|
||||
extractedTranslations = extractTranslation(translatedText, True)
|
||||
if extractedTranslations == None or len(tItem) != len(extractedTranslations):
|
||||
# Mismatch. Try Again
|
||||
response = translateText(system, user, history, 0.05, format, MODEL)
|
||||
translatedText = response.choices[0].message.content
|
||||
totalTokens[0] += response.usage.prompt_tokens
|
||||
totalTokens[1] += response.usage.completion_tokens
|
||||
|
||||
# Formatting
|
||||
translatedText = cleanTranslatedText(translatedText)
|
||||
if isinstance(tItem, list):
|
||||
extractedTranslations = extractTranslation(translatedText, True)
|
||||
if extractedTranslations == None or len(tItem) != len(extractedTranslations):
|
||||
with open("log/mismatchHistory.txt", "a+", encoding="utf-8") as mismatchFile:
|
||||
mismatchFile.write(f"Mismatch: {FILENAME}\n")
|
||||
mismatchFile.write(f"Input:\n{subbedT}\n")
|
||||
mismatchFile.write(f"Output:\n{translatedText}\n")
|
||||
mismatch = True # Just here for breakpoint
|
||||
logFile.write(f"Input:\n{subbedT}\n")
|
||||
logFile.write(f"Output:\n{translatedText}\n")
|
||||
|
||||
# Set if no mismatch
|
||||
if mismatch == False:
|
||||
tList[index] = extractedTranslations
|
||||
history = extractedTranslations[-MAXHISTORY:] # Update history if we have a list
|
||||
else:
|
||||
history = text[-MAXHISTORY:]
|
||||
mismatch = False
|
||||
if FILENAME not in MISMATCH:
|
||||
MISMATCH.append(FILENAME)
|
||||
|
||||
# Update Loading Bar
|
||||
with LOCK:
|
||||
if PBAR is not None:
|
||||
PBAR.update(len(tItem))
|
||||
else:
|
||||
# Ensure we're passing a single string to extractTranslation
|
||||
tList[index] = translatedText.replace("Placeholder Text", "")
|
||||
else:
|
||||
PBAR.write(f"AI Refused:{tItem}\n")
|
||||
|
||||
# Combine if multilist
|
||||
if isinstance(tList[0], list):
|
||||
tList = [t for sublist in tList for t in sublist]
|
||||
|
||||
# Return
|
||||
if format == "json":
|
||||
return [tList, totalTokens]
|
||||
else:
|
||||
return [tList[0], totalTokens]
|
||||
else:
|
||||
return [text, [0, 0]]
|
||||
def translateAI(text, history, fullPromptFlag):
|
||||
"""
|
||||
Translate text using the shared translation utility.
|
||||
This function maintains compatibility with existing code while using the new shared implementation.
|
||||
"""
|
||||
return sharedtranslateAI(TRANSLATION_CONFIG, text, history, fullPromptFlag)
|
||||
|
|
|
|||
365
modules/regex.py
365
modules/regex.py
|
|
@ -13,6 +13,7 @@ from colorama import Fore
|
|||
from dotenv import load_dotenv
|
||||
from retry import retry
|
||||
from tqdm import tqdm
|
||||
from util.translation import TranslationConfig, translateAI as sharedtranslateAI
|
||||
|
||||
# Open AI
|
||||
load_dotenv()
|
||||
|
|
@ -78,6 +79,18 @@ else:
|
|||
BATCHSIZE = int(os.getenv("batchsize"))
|
||||
FREQUENCY_PENALTY = float(os.getenv("frequency_penalty"))
|
||||
|
||||
# Initialize Translation Config
|
||||
TRANSLATION_CONFIG = TranslationConfig(
|
||||
model=MODEL,
|
||||
language=LANGUAGE,
|
||||
prompt=PROMPT,
|
||||
vocab=VOCAB,
|
||||
langRegex=LANGREGEX,
|
||||
batchSize=BATCHSIZE,
|
||||
maxHistory=MAXHISTORY,
|
||||
estimateMode=False # Will be set dynamically based on ESTIMATE
|
||||
)
|
||||
LEAVE = False
|
||||
|
||||
def handleRegex(filename, estimate):
|
||||
global ESTIMATE, TOKENS, FILENAME
|
||||
|
|
@ -194,7 +207,7 @@ def translateRegex(data, translatedList):
|
|||
|
||||
while i < len(data):
|
||||
voice = False
|
||||
lineRegexText = r'"Text":\s"(.+)",'
|
||||
lineRegexText = r'"profile": "(.*)"'
|
||||
lineRegexSpeaker = r"◆A.+?◆(.+)"
|
||||
choiceRegex = r"\$menu_item.+?,(.*?),"
|
||||
titleRegex = r"title\s'(.*)'$"
|
||||
|
|
@ -203,7 +216,7 @@ def translateRegex(data, translatedList):
|
|||
# Title
|
||||
match = re.search(titleRegex, data[i])
|
||||
if match:
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
match.group(1),
|
||||
f"Reply with the {LANGUAGE} translation of the chapter title",
|
||||
True,
|
||||
|
|
@ -328,7 +341,7 @@ def translateRegex(data, translatedList):
|
|||
if stringList:
|
||||
PBAR.total = len(stringList)
|
||||
PBAR.refresh()
|
||||
response = translateGPT(stringList, "Reply with the English Translation", True)
|
||||
response = translateAI(stringList, "Reply with the English Translation", True)
|
||||
tokens[0] += response[1][0]
|
||||
tokens[1] += response[1][1]
|
||||
stringListTL = response[0]
|
||||
|
|
@ -341,7 +354,7 @@ def translateRegex(data, translatedList):
|
|||
|
||||
# Choice List
|
||||
if choiceList:
|
||||
response = translateGPT(choiceList, "Reply with the English TL of the Dialogue Choice", True)
|
||||
response = translateAI(choiceList, "Reply with the English TL of the Dialogue Choice", True)
|
||||
tokens[0] += response[1][0]
|
||||
tokens[1] += response[1][1]
|
||||
choiceListTL = response[0]
|
||||
|
|
@ -370,7 +383,7 @@ def getSpeaker(speaker):
|
|||
return [NAMESLIST[i][1], [0, 0]]
|
||||
|
||||
# Translate and Store Speaker
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
f"{speaker}",
|
||||
"Reply with the " + LANGUAGE + " translation of the NPC name.",
|
||||
False,
|
||||
|
|
@ -381,7 +394,7 @@ def getSpeaker(speaker):
|
|||
|
||||
# Retry if name doesn't translate for some reason
|
||||
if re.search(r"([a-zA-Z？?])", response[0]) == None:
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
f"{speaker}",
|
||||
"Reply with the " + LANGUAGE + " translation of the NPC name.",
|
||||
False,
|
||||
|
|
@ -394,328 +407,24 @@ def getSpeaker(speaker):
|
|||
return response
|
||||
return [speaker, [0, 0]]
|
||||
|
||||
|
||||
def batchList(input_list, batch_size):
|
||||
if not isinstance(batch_size, int) or batch_size <= 0:
|
||||
raise ValueError("batch_size must be a positive integer")
|
||||
|
||||
return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)]
|
||||
|
||||
|
||||
def parseVocabWithCategories(vocabText):
|
||||
"""Parse vocabulary text and extract terms with their categories."""
|
||||
pairs = []
|
||||
seen = set()
|
||||
currentCategory = None
|
||||
|
||||
for line in vocabText.splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith('```'):
|
||||
continue
|
||||
|
||||
# Check if this is a category header
|
||||
if line.startswith('#'):
|
||||
currentCategory = line
|
||||
continue
|
||||
|
||||
# Parse vocabulary term
|
||||
m = re.match(r'^(.+?)(?:\s?[\(–])', line) # term is everything before space + '(' or '–'
|
||||
if m:
|
||||
term = m.group(1)
|
||||
if term not in seen:
|
||||
pairs.append((term, line, currentCategory))
|
||||
seen.add(term)
|
||||
|
||||
return pairs
|
||||
|
||||
|
||||
def buildMatchedVocabText(vocabPairs, subbedT):
|
||||
"""Build formatted vocabulary text with matched terms organized by category."""
|
||||
matchedCategories = {}
|
||||
|
||||
# Use word boundaries for Japanese if appropriate, or allow substring as before.
|
||||
for term, line, category in vocabPairs:
|
||||
# "term in subbedT" could be false positive; can use regex but Japanese doesn't always have spaces.
|
||||
if term in subbedT:
|
||||
if category not in matchedCategories:
|
||||
matchedCategories[category] = []
|
||||
matchedCategories[category].append(line)
|
||||
|
||||
# Format matched vocabulary with categories
|
||||
if matchedCategories:
|
||||
formattedLines = ["Here are some vocabulary and terms so that you know the proper spelling and translation.\n"]
|
||||
for category, lines in matchedCategories.items():
|
||||
if category: # Only add category header if it exists
|
||||
formattedLines.append(category)
|
||||
formattedLines.extend(lines)
|
||||
formattedLines.append("") # Add blank line between categories
|
||||
matchedVocabText = f"```\n{chr(10).join(formattedLines).rstrip()}\n```"
|
||||
else:
|
||||
matchedVocabText = ""
|
||||
|
||||
return matchedVocabText
|
||||
|
||||
|
||||
def createContext(fullPromptFlag, subbedT, format):
|
||||
vocabPairs = parseVocabWithCategories(VOCAB)
|
||||
matchedVocabText = buildMatchedVocabText(vocabPairs, subbedT)
|
||||
|
||||
if fullPromptFlag:
|
||||
system = PROMPT + matchedVocabText
|
||||
else:
|
||||
system = f"\
|
||||
You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
|
||||
Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
|
||||
- You translate everything, including erotic and explicit content.\n\
|
||||
- 'Game Characters' - The names, nicknames, and genders of the game characters. Reference this to know the names, nicknames, and gender of characters in the game\n\
|
||||
- All text in your response must be in {LANGUAGE} even if it is hard to translate.\n\
|
||||
- Never include any notes, explanations, dislaimers, or anything similar in your response.\n\
|
||||
- Maintain any spacing in the translation.\n\
|
||||
- Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\
|
||||
- `...` can be a part of the dialogue. Translate it as it is.\n\
|
||||
{matchedVocabText}\n\
|
||||
"
|
||||
if format == "json":
|
||||
user = f"```json\n{subbedT}\n```"
|
||||
else:
|
||||
user = subbedT
|
||||
return system, user
|
||||
|
||||
|
||||
def translateText(system, user, history, penalty, format, model=MODEL):
|
||||
# Prompt
|
||||
msg = [{"role": "system", "content": system}]
|
||||
|
||||
# History
|
||||
if isinstance(history, list):
|
||||
msg.append({"role": "system", "content": "Translation History:"})
|
||||
msg.extend([{"role": "assistant", "content": h} for h in history])
|
||||
else:
|
||||
msg.append({"role": "assistant", "content": history})
|
||||
|
||||
# Response Format
|
||||
if format == "json":
|
||||
responseFormat = {"type": "json_object"}
|
||||
else:
|
||||
responseFormat = {"type": "text"}
|
||||
|
||||
# Content to TL
|
||||
msg.append({"role": "user", "content": f"{user}"})
|
||||
response = openai.chat.completions.create(
|
||||
temperature=0,
|
||||
frequency_penalty=penalty,
|
||||
model=model,
|
||||
response_format=responseFormat,
|
||||
messages=msg,
|
||||
)
|
||||
return response
|
||||
|
||||
|
||||
def cleanTranslatedText(translatedText):
|
||||
placeholders = {
|
||||
f"{LANGUAGE} Translation: ": "",
|
||||
"Translation: ": "",
|
||||
"っ": "",
|
||||
"〜": "~",
|
||||
"ッ": "",
|
||||
"。": ".",
|
||||
"「": '\\"',
|
||||
"」": '\\"',
|
||||
"- ": "-",
|
||||
"—": "―",
|
||||
"】": "]",
|
||||
"【": "[",
|
||||
"é": "e",
|
||||
"this guy": "this bastard",
|
||||
"This guy": "This bastard",
|
||||
"Placeholder Text": "",
|
||||
"```json": "",
|
||||
"```": "",
|
||||
# Add more replacements as needed
|
||||
}
|
||||
for target, replacement in placeholders.items():
|
||||
translatedText = translatedText.replace(target, replacement)
|
||||
|
||||
# Remove Repeating Characters
|
||||
pattern = re.compile(r"(.)\s*\1(?:\s*\1){" + str(20 - 1) + r",}")
|
||||
translatedText = pattern.sub(lambda match: match.group(0).replace(" ", "")[:20], translatedText)
|
||||
|
||||
# Elongate Long Dashes (Since GPT Ignores them...)
|
||||
translatedText = elongateCharacters(translatedText)
|
||||
return translatedText
|
||||
|
||||
|
||||
def elongateCharacters(text):
|
||||
# Define a pattern to match one character followed by one or more `ー` characters
|
||||
# Using a positive lookbehind assertion to capture the preceding character
|
||||
pattern = r"(?<=(.))ー+"
|
||||
|
||||
# Define a replacement function that elongates the captured character
|
||||
def repl(match):
|
||||
char = match.group(1) # The character before the ー sequence
|
||||
count = len(match.group(0)) - 1 # Number of ー characters
|
||||
return char * count # Replace ー sequence with the character repeated
|
||||
|
||||
# Use re.sub() to replace the pattern in the text
|
||||
return re.sub(pattern, repl, text)
|
||||
|
||||
|
||||
def extractTranslation(translatedTextList, is_list):
|
||||
try:
|
||||
translatedTextList = re.sub(r'\\"+\"([^,\n}])', r'\\"\1', translatedTextList)
|
||||
translatedTextList = re.sub(r"(?<![\\])\"+(?![\n,])", r'"', translatedTextList)
|
||||
line_dict = json.loads(translatedTextList)
|
||||
# If it's a batch (i.e., list), extract with tags; otherwise, return the single item.
|
||||
string_list = list(line_dict.values())
|
||||
if is_list:
|
||||
return string_list
|
||||
else:
|
||||
return string_list[0]
|
||||
|
||||
except Exception as e:
|
||||
PBAR.write(f"extractTranslation Error: {e} on String {translatedTextList}")
|
||||
return None
|
||||
|
||||
|
||||
def countTokens(system, user, history):
|
||||
inputTotalTokens = 0
|
||||
outputTotalTokens = 0
|
||||
enc = tiktoken.encoding_for_model("gpt-4")
|
||||
|
||||
# Input
|
||||
if isinstance(history, list):
|
||||
for line in history:
|
||||
inputTotalTokens += len(enc.encode(line))
|
||||
else:
|
||||
inputTotalTokens += len(enc.encode(history))
|
||||
inputTotalTokens += len(enc.encode(system))
|
||||
inputTotalTokens += len(enc.encode(user))
|
||||
|
||||
# Output
|
||||
outputTotalTokens += round(len(enc.encode(user)) * 2.5)
|
||||
|
||||
return [inputTotalTokens, outputTotalTokens]
|
||||
|
||||
|
||||
@retry(exceptions=Exception, tries=5, delay=5)
|
||||
def translateGPT(text, history, fullPromptFlag):
|
||||
def translateAI(text, history, fullPromptFlag):
|
||||
"""
|
||||
Legacy wrapper function for the new shared translation utility.
|
||||
This maintains compatibility with existing code while using the new shared implementation.
|
||||
"""
|
||||
global PBAR, MISMATCH, FILENAME
|
||||
if text:
|
||||
with open("log/translationHistory.txt", "a+", encoding="utf-8") as logFile:
|
||||
mismatch = False
|
||||
totalTokens = [0, 0]
|
||||
if isinstance(text, list):
|
||||
format = "json"
|
||||
tList = batchList(text, BATCHSIZE)
|
||||
else:
|
||||
format = "text"
|
||||
tList = [text]
|
||||
|
||||
for index, tItem in enumerate(tList):
|
||||
# Things to Check before starting translation
|
||||
if not re.search(LANGREGEX, str(tItem)):
|
||||
if PBAR is not None:
|
||||
PBAR.update(len(tItem))
|
||||
if isinstance(tItem, list):
|
||||
for j in range(len(tItem)):
|
||||
tItem[j] = cleanTranslatedText(tItem[j])
|
||||
tList[index] = tItem
|
||||
else:
|
||||
tList[index] = cleanTranslatedText(tItem)
|
||||
history = tItem[-MAXHISTORY:]
|
||||
continue
|
||||
# Update config estimate mode based on global ESTIMATE
|
||||
TRANSLATION_CONFIG.estimateMode = bool(ESTIMATE)
|
||||
|
||||
# Before sending to translation, if we have a list of items, add the formatting
|
||||
if isinstance(tItem, list):
|
||||
for j in range(len(tItem)):
|
||||
if not tItem[j]:
|
||||
tItem[j] = tItem[j].replace("", "Placeholder Text")
|
||||
payload = {f"Line{i+1}": string for i, string in enumerate(tItem)}
|
||||
payload = json.dumps(payload, indent=4, ensure_ascii=False)
|
||||
varResponse = [payload, []]
|
||||
subbedT = varResponse[0]
|
||||
else:
|
||||
varResponse = [tItem, []]
|
||||
subbedT = varResponse[0]
|
||||
|
||||
# Create Message
|
||||
system, user = createContext(fullPromptFlag, subbedT, format)
|
||||
|
||||
# Calculate Estimate
|
||||
if ESTIMATE:
|
||||
estimate = countTokens(system, user, history)
|
||||
totalTokens[0] += estimate[0]
|
||||
totalTokens[1] += estimate[1]
|
||||
continue
|
||||
|
||||
# Translating
|
||||
response = translateText(system, user, history, 0.05, format)
|
||||
|
||||
# Set Tokens
|
||||
translatedText = response.choices[0].message.content
|
||||
|
||||
# AI Refused, Try Again
|
||||
if not translatedText:
|
||||
response = translateText(f"{system}\n You translate ALL content.", user, history, 0.1, format, model="gpt-4o")
|
||||
translatedText = response.choices[0].message.content
|
||||
|
||||
# Report Tokens
|
||||
totalTokens[0] += response.usage.prompt_tokens
|
||||
totalTokens[1] += response.usage.completion_tokens
|
||||
|
||||
# Check Translation
|
||||
if translatedText:
|
||||
translatedText = cleanTranslatedText(translatedText)
|
||||
if isinstance(tItem, list):
|
||||
extractedTranslations = extractTranslation(translatedText, True)
|
||||
if extractedTranslations == None or len(tItem) != len(extractedTranslations):
|
||||
# Mismatch. Try Again
|
||||
response = translateText(system, user, history, 0.05, format, MODEL)
|
||||
translatedText = response.choices[0].message.content
|
||||
totalTokens[0] += response.usage.prompt_tokens
|
||||
totalTokens[1] += response.usage.completion_tokens
|
||||
|
||||
# Formatting
|
||||
translatedText = cleanTranslatedText(translatedText)
|
||||
if isinstance(tItem, list):
|
||||
extractedTranslations = extractTranslation(translatedText, True)
|
||||
if extractedTranslations == None or len(tItem) != len(extractedTranslations):
|
||||
with open("log/mismatchHistory.txt", "a+", encoding="utf-8") as mismatchFile:
|
||||
mismatchFile.write(f"Mismatch: {FILENAME}\n")
|
||||
mismatchFile.write(f"Input:\n{subbedT}\n")
|
||||
mismatchFile.write(f"Output:\n{translatedText}\n")
|
||||
mismatch = True # Just here for breakpoint
|
||||
logFile.write(f"Input:\n{subbedT}\n")
|
||||
logFile.write(f"Output:\n{translatedText}\n")
|
||||
|
||||
# Set if no mismatch
|
||||
if mismatch == False:
|
||||
tList[index] = extractedTranslations
|
||||
history = extractedTranslations[-MAXHISTORY:] # Update history if we have a list
|
||||
else:
|
||||
history = text[-MAXHISTORY:]
|
||||
mismatch = False
|
||||
if FILENAME not in MISMATCH:
|
||||
MISMATCH.append(FILENAME)
|
||||
|
||||
# Update Loading Bar
|
||||
with LOCK:
|
||||
if PBAR is not None:
|
||||
PBAR.update(len(tItem))
|
||||
else:
|
||||
# Ensure we're passing a single string to extractTranslation
|
||||
tList[index] = translatedText.replace("Placeholder Text", "")
|
||||
else:
|
||||
PBAR.write(f"AI Refused:{tItem}\n")
|
||||
|
||||
# Combine if multilist
|
||||
if isinstance(tList[0], list):
|
||||
tList = [t for sublist in tList for t in sublist]
|
||||
|
||||
# Return
|
||||
if format == "json":
|
||||
return [tList, totalTokens]
|
||||
else:
|
||||
return [tList[0], totalTokens]
|
||||
else:
|
||||
return [text, [0, 0]]
|
||||
# Call the new shared translation function
|
||||
return sharedtranslateAI(
|
||||
text=text,
|
||||
history=history,
|
||||
fullPromptFlag=fullPromptFlag,
|
||||
config=TRANSLATION_CONFIG,
|
||||
filename=FILENAME,
|
||||
pbar=PBAR,
|
||||
lock=LOCK,
|
||||
mismatchList=MISMATCH
|
||||
)
|
||||
|
|
|
|||
359
modules/renpy.py
359
modules/renpy.py
|
|
@ -13,6 +13,7 @@ from colorama import Fore
|
|||
from dotenv import load_dotenv
|
||||
from retry import retry
|
||||
from tqdm import tqdm
|
||||
from util.translation import TranslationConfig, translateAI as sharedtranslateAI
|
||||
|
||||
# Open AI
|
||||
load_dotenv()
|
||||
|
|
@ -76,6 +77,18 @@ else:
|
|||
BATCHSIZE = int(os.getenv("batchsize"))
|
||||
FREQUENCY_PENALTY = float(os.getenv("frequency_penalty"))
|
||||
|
||||
# Initialize Translation Config
|
||||
TRANSLATION_CONFIG = TranslationConfig(
|
||||
model=MODEL,
|
||||
language=LANGUAGE,
|
||||
prompt=PROMPT,
|
||||
vocab=VOCAB,
|
||||
langRegex=LANGREGEX,
|
||||
batchSize=BATCHSIZE,
|
||||
maxHistory=MAXHISTORY,
|
||||
estimateMode=False # Will be set dynamically based on ESTIMATE
|
||||
)
|
||||
LEAVE = False
|
||||
|
||||
def handleRenpy(filename, estimate):
|
||||
global ESTIMATE
|
||||
|
|
@ -264,7 +277,7 @@ def translateRenpy(data, translatedList):
|
|||
PBAR.refresh()
|
||||
|
||||
# Translate
|
||||
response = translateGPT(stringList, "Reply with the English TL of the NPC Name", True)
|
||||
response = translateAI(stringList, "Reply with the English TL of the NPC Name", True)
|
||||
tokens[0] += response[1][0]
|
||||
tokens[1] += response[1][1]
|
||||
translatedList = response[0]
|
||||
|
|
@ -294,7 +307,7 @@ def getSpeaker(speaker):
|
|||
return [NAMESLIST[i][1], [0, 0]]
|
||||
|
||||
# Translate and Store Speaker
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
f"{speaker}",
|
||||
"Reply with the " + LANGUAGE + " translation of the NPC name.",
|
||||
False,
|
||||
|
|
@ -305,7 +318,7 @@ def getSpeaker(speaker):
|
|||
|
||||
# Retry if name doesn't translate for some reason
|
||||
if re.search(r"([a-zA-Z？?])", response[0]) == None:
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
f"{speaker}",
|
||||
"Reply with the " + LANGUAGE + " translation of the NPC name.",
|
||||
False,
|
||||
|
|
@ -318,328 +331,24 @@ def getSpeaker(speaker):
|
|||
return response
|
||||
return [speaker, [0, 0]]
|
||||
|
||||
|
||||
def batchList(input_list, batch_size):
|
||||
if not isinstance(batch_size, int) or batch_size <= 0:
|
||||
raise ValueError("batch_size must be a positive integer")
|
||||
|
||||
return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)]
|
||||
|
||||
|
||||
def parseVocabWithCategories(vocabText):
|
||||
"""Parse vocabulary text and extract terms with their categories."""
|
||||
pairs = []
|
||||
seen = set()
|
||||
currentCategory = None
|
||||
|
||||
for line in vocabText.splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith('```'):
|
||||
continue
|
||||
|
||||
# Check if this is a category header
|
||||
if line.startswith('#'):
|
||||
currentCategory = line
|
||||
continue
|
||||
|
||||
# Parse vocabulary term
|
||||
m = re.match(r'^(.+?)(?:\s?[\(–])', line) # term is everything before space + '(' or '–'
|
||||
if m:
|
||||
term = m.group(1)
|
||||
if term not in seen:
|
||||
pairs.append((term, line, currentCategory))
|
||||
seen.add(term)
|
||||
|
||||
return pairs
|
||||
|
||||
|
||||
def buildMatchedVocabText(vocabPairs, subbedT):
|
||||
"""Build formatted vocabulary text with matched terms organized by category."""
|
||||
matchedCategories = {}
|
||||
|
||||
# Use word boundaries for Japanese if appropriate, or allow substring as before.
|
||||
for term, line, category in vocabPairs:
|
||||
# "term in subbedT" could be false positive; can use regex but Japanese doesn't always have spaces.
|
||||
if term in subbedT:
|
||||
if category not in matchedCategories:
|
||||
matchedCategories[category] = []
|
||||
matchedCategories[category].append(line)
|
||||
|
||||
# Format matched vocabulary with categories
|
||||
if matchedCategories:
|
||||
formattedLines = ["Here are some vocabulary and terms so that you know the proper spelling and translation.\n"]
|
||||
for category, lines in matchedCategories.items():
|
||||
if category: # Only add category header if it exists
|
||||
formattedLines.append(category)
|
||||
formattedLines.extend(lines)
|
||||
formattedLines.append("") # Add blank line between categories
|
||||
matchedVocabText = f"```\n{chr(10).join(formattedLines).rstrip()}\n```"
|
||||
else:
|
||||
matchedVocabText = ""
|
||||
|
||||
return matchedVocabText
|
||||
|
||||
|
||||
def createContext(fullPromptFlag, subbedT, format):
|
||||
vocabPairs = parseVocabWithCategories(VOCAB)
|
||||
matchedVocabText = buildMatchedVocabText(vocabPairs, subbedT)
|
||||
|
||||
if fullPromptFlag:
|
||||
system = PROMPT + matchedVocabText
|
||||
else:
|
||||
system = f"\
|
||||
You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
|
||||
Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
|
||||
- You translate everything, including erotic and explicit content.\n\
|
||||
- 'Game Characters' - The names, nicknames, and genders of the game characters. Reference this to know the names, nicknames, and gender of characters in the game\n\
|
||||
- All text in your response must be in {LANGUAGE} even if it is hard to translate.\n\
|
||||
- Never include any notes, explanations, dislaimers, or anything similar in your response.\n\
|
||||
- Maintain any spacing in the translation.\n\
|
||||
- Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\
|
||||
- `...` can be a part of the dialogue. Translate it as it is.\n\
|
||||
{matchedVocabText}\n\
|
||||
"
|
||||
if format == "json":
|
||||
user = f"```json\n{subbedT}\n```"
|
||||
else:
|
||||
user = subbedT
|
||||
return system, user
|
||||
|
||||
|
||||
def translateText(system, user, history, penalty, format, model=MODEL):
|
||||
# Prompt
|
||||
msg = [{"role": "system", "content": system}]
|
||||
|
||||
# History
|
||||
if isinstance(history, list):
|
||||
msg.append({"role": "system", "content": "Translation History:"})
|
||||
msg.extend([{"role": "assistant", "content": h} for h in history])
|
||||
else:
|
||||
msg.append({"role": "assistant", "content": history})
|
||||
|
||||
# Response Format
|
||||
if format == "json":
|
||||
responseFormat = {"type": "json_object"}
|
||||
else:
|
||||
responseFormat = {"type": "text"}
|
||||
|
||||
# Content to TL
|
||||
msg.append({"role": "user", "content": f"{user}"})
|
||||
response = openai.chat.completions.create(
|
||||
temperature=0,
|
||||
frequency_penalty=penalty,
|
||||
model=model,
|
||||
response_format=responseFormat,
|
||||
messages=msg,
|
||||
)
|
||||
return response
|
||||
|
||||
|
||||
def cleanTranslatedText(translatedText):
|
||||
placeholders = {
|
||||
f"{LANGUAGE} Translation: ": "",
|
||||
"Translation: ": "",
|
||||
"っ": "",
|
||||
"〜": "~",
|
||||
"ッ": "",
|
||||
"。": ".",
|
||||
"「": '\\"',
|
||||
"」": '\\"',
|
||||
"- ": "-",
|
||||
"—": "―",
|
||||
"】": "]",
|
||||
"【": "[",
|
||||
"é": "e",
|
||||
"this guy": "this bastard",
|
||||
"This guy": "This bastard",
|
||||
"Placeholder Text": "",
|
||||
"```json": "",
|
||||
"```": "",
|
||||
# Add more replacements as needed
|
||||
}
|
||||
for target, replacement in placeholders.items():
|
||||
translatedText = translatedText.replace(target, replacement)
|
||||
|
||||
# Remove Repeating Characters
|
||||
pattern = re.compile(r"(.)\s*\1(?:\s*\1){" + str(20 - 1) + r",}")
|
||||
translatedText = pattern.sub(lambda match: match.group(0).replace(" ", "")[:20], translatedText)
|
||||
|
||||
# Elongate Long Dashes (Since GPT Ignores them...)
|
||||
translatedText = elongateCharacters(translatedText)
|
||||
return translatedText
|
||||
|
||||
|
||||
def elongateCharacters(text):
|
||||
# Define a pattern to match one character followed by one or more `ー` characters
|
||||
# Using a positive lookbehind assertion to capture the preceding character
|
||||
pattern = r"(?<=(.))ー+"
|
||||
|
||||
# Define a replacement function that elongates the captured character
|
||||
def repl(match):
|
||||
char = match.group(1) # The character before the ー sequence
|
||||
count = len(match.group(0)) - 1 # Number of ー characters
|
||||
return char * count # Replace ー sequence with the character repeated
|
||||
|
||||
# Use re.sub() to replace the pattern in the text
|
||||
return re.sub(pattern, repl, text)
|
||||
|
||||
|
||||
def extractTranslation(translatedTextList, is_list):
|
||||
try:
|
||||
translatedTextList = re.sub(r'\\"+\"([^,\n}])', r'\\"\1', translatedTextList)
|
||||
translatedTextList = re.sub(r"(?<![\\])\"+(?![\n,])", r'"', translatedTextList)
|
||||
line_dict = json.loads(translatedTextList)
|
||||
# If it's a batch (i.e., list), extract with tags; otherwise, return the single item.
|
||||
string_list = list(line_dict.values())
|
||||
if is_list:
|
||||
return string_list
|
||||
else:
|
||||
return string_list[0]
|
||||
|
||||
except Exception as e:
|
||||
PBAR.write(f"extractTranslation Error: {e} on String {translatedTextList}")
|
||||
return None
|
||||
|
||||
|
||||
def countTokens(system, user, history):
|
||||
inputTotalTokens = 0
|
||||
outputTotalTokens = 0
|
||||
enc = tiktoken.encoding_for_model("gpt-4")
|
||||
|
||||
# Input
|
||||
if isinstance(history, list):
|
||||
for line in history:
|
||||
inputTotalTokens += len(enc.encode(line))
|
||||
else:
|
||||
inputTotalTokens += len(enc.encode(history))
|
||||
inputTotalTokens += len(enc.encode(system))
|
||||
inputTotalTokens += len(enc.encode(user))
|
||||
|
||||
# Output
|
||||
outputTotalTokens += round(len(enc.encode(user)) * 2.5)
|
||||
|
||||
return [inputTotalTokens, outputTotalTokens]
|
||||
|
||||
|
||||
@retry(exceptions=Exception, tries=5, delay=5)
|
||||
def translateGPT(text, history, fullPromptFlag):
|
||||
def translateAI(text, history, fullPromptFlag):
|
||||
"""
|
||||
Legacy wrapper function for the new shared translation utility.
|
||||
This maintains compatibility with existing code while using the new shared implementation.
|
||||
"""
|
||||
global PBAR, MISMATCH, FILENAME
|
||||
if text:
|
||||
with open("log/translationHistory.txt", "a+", encoding="utf-8") as logFile:
|
||||
mismatch = False
|
||||
totalTokens = [0, 0]
|
||||
if isinstance(text, list):
|
||||
format = "json"
|
||||
tList = batchList(text, BATCHSIZE)
|
||||
else:
|
||||
format = "text"
|
||||
tList = [text]
|
||||
|
||||
for index, tItem in enumerate(tList):
|
||||
# Things to Check before starting translation
|
||||
if not re.search(LANGREGEX, str(tItem)):
|
||||
if PBAR is not None:
|
||||
PBAR.update(len(tItem))
|
||||
if isinstance(tItem, list):
|
||||
for j in range(len(tItem)):
|
||||
tItem[j] = cleanTranslatedText(tItem[j])
|
||||
tList[index] = tItem
|
||||
else:
|
||||
tList[index] = cleanTranslatedText(tItem)
|
||||
history = tItem[-MAXHISTORY:]
|
||||
continue
|
||||
# Update config estimate mode based on global ESTIMATE
|
||||
TRANSLATION_CONFIG.estimateMode = bool(ESTIMATE)
|
||||
|
||||
# Before sending to translation, if we have a list of items, add the formatting
|
||||
if isinstance(tItem, list):
|
||||
for j in range(len(tItem)):
|
||||
if not tItem[j]:
|
||||
tItem[j] = tItem[j].replace("", "Placeholder Text")
|
||||
payload = {f"Line{i+1}": string for i, string in enumerate(tItem)}
|
||||
payload = json.dumps(payload, indent=4, ensure_ascii=False)
|
||||
varResponse = [payload, []]
|
||||
subbedT = varResponse[0]
|
||||
else:
|
||||
varResponse = [tItem, []]
|
||||
subbedT = varResponse[0]
|
||||
|
||||
# Create Message
|
||||
system, user = createContext(fullPromptFlag, subbedT, format)
|
||||
|
||||
# Calculate Estimate
|
||||
if ESTIMATE:
|
||||
estimate = countTokens(system, user, history)
|
||||
totalTokens[0] += estimate[0]
|
||||
totalTokens[1] += estimate[1]
|
||||
continue
|
||||
|
||||
# Translating
|
||||
response = translateText(system, user, history, 0.05, format)
|
||||
|
||||
# Set Tokens
|
||||
translatedText = response.choices[0].message.content
|
||||
|
||||
# AI Refused, Try Again
|
||||
if not translatedText:
|
||||
response = translateText(f"{system}\n You translate ALL content.", user, history, 0.1, format, model="gpt-4o")
|
||||
translatedText = response.choices[0].message.content
|
||||
|
||||
# Report Tokens
|
||||
totalTokens[0] += response.usage.prompt_tokens
|
||||
totalTokens[1] += response.usage.completion_tokens
|
||||
|
||||
# Check Translation
|
||||
if translatedText:
|
||||
translatedText = cleanTranslatedText(translatedText)
|
||||
if isinstance(tItem, list):
|
||||
extractedTranslations = extractTranslation(translatedText, True)
|
||||
if extractedTranslations == None or len(tItem) != len(extractedTranslations):
|
||||
# Mismatch. Try Again
|
||||
response = translateText(system, user, history, 0.05, format, MODEL)
|
||||
translatedText = response.choices[0].message.content
|
||||
totalTokens[0] += response.usage.prompt_tokens
|
||||
totalTokens[1] += response.usage.completion_tokens
|
||||
|
||||
# Formatting
|
||||
translatedText = cleanTranslatedText(translatedText)
|
||||
if isinstance(tItem, list):
|
||||
extractedTranslations = extractTranslation(translatedText, True)
|
||||
if extractedTranslations == None or len(tItem) != len(extractedTranslations):
|
||||
with open("log/mismatchHistory.txt", "a+", encoding="utf-8") as mismatchFile:
|
||||
mismatchFile.write(f"Mismatch: {FILENAME}\n")
|
||||
mismatchFile.write(f"Input:\n{subbedT}\n")
|
||||
mismatchFile.write(f"Output:\n{translatedText}\n")
|
||||
mismatch = True # Just here for breakpoint
|
||||
logFile.write(f"Input:\n{subbedT}\n")
|
||||
logFile.write(f"Output:\n{translatedText}\n")
|
||||
|
||||
# Set if no mismatch
|
||||
if mismatch == False:
|
||||
tList[index] = extractedTranslations
|
||||
history = extractedTranslations[-MAXHISTORY:] # Update history if we have a list
|
||||
else:
|
||||
history = text[-MAXHISTORY:]
|
||||
mismatch = False
|
||||
if FILENAME not in MISMATCH:
|
||||
MISMATCH.append(FILENAME)
|
||||
|
||||
# Update Loading Bar
|
||||
with LOCK:
|
||||
if PBAR is not None:
|
||||
PBAR.update(len(tItem))
|
||||
else:
|
||||
# Ensure we're passing a single string to extractTranslation
|
||||
tList[index] = translatedText.replace("Placeholder Text", "")
|
||||
else:
|
||||
PBAR.write(f"AI Refused:{tItem}\n")
|
||||
|
||||
# Combine if multilist
|
||||
if isinstance(tList[0], list):
|
||||
tList = [t for sublist in tList for t in sublist]
|
||||
|
||||
# Return
|
||||
if format == "json":
|
||||
return [tList, totalTokens]
|
||||
else:
|
||||
return [tList[0], totalTokens]
|
||||
else:
|
||||
return [text, [0, 0]]
|
||||
# Call the new shared translation function
|
||||
return sharedtranslateAI(
|
||||
text=text,
|
||||
history=history,
|
||||
fullPromptFlag=fullPromptFlag,
|
||||
config=TRANSLATION_CONFIG,
|
||||
filename=FILENAME,
|
||||
pbar=PBAR,
|
||||
lock=LOCK,
|
||||
mismatchList=MISMATCH
|
||||
)
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ from colorama import Fore
|
|||
from dotenv import load_dotenv
|
||||
from retry import retry
|
||||
from tqdm import tqdm
|
||||
from util.translation import TranslationConfig, translateAI as sharedtranslateAI
|
||||
from ruamel.yaml import YAML
|
||||
|
||||
|
||||
|
|
@ -77,6 +78,19 @@ BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
|
|||
POSITION = 0
|
||||
LEAVE = False
|
||||
|
||||
# Initialize Translation Config
|
||||
TRANSLATION_CONFIG = TranslationConfig(
|
||||
model=MODEL,
|
||||
language=LANGUAGE,
|
||||
prompt=PROMPT,
|
||||
vocab=VOCAB,
|
||||
langRegex=LANGREGEX,
|
||||
batchSize=BATCHSIZE,
|
||||
maxHistory=MAXHISTORY,
|
||||
estimateMode=False # Will be set dynamically based on ESTIMATE
|
||||
)
|
||||
LEAVE = False
|
||||
|
||||
# Config (Default)
|
||||
FIRSTLINESPEAKERS = False # If 1st line of 401 is a speaker, set to True (False)
|
||||
FACENAME101 = False # Find Speakers in 101 Codes based on Face Name (False)
|
||||
|
|
@ -252,7 +266,7 @@ def parseMap(data, filename):
|
|||
|
||||
# Translate display_name for Map files
|
||||
if "Map" in filename:
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
data["display_name"],
|
||||
"Reply with only the " + LANGUAGE + " translation of the RPG location name",
|
||||
False,
|
||||
|
|
@ -299,7 +313,7 @@ def translateNote(event, regex, wordwrap=False):
|
|||
modifiedJAString = modifiedJAString.replace("\n", " ")
|
||||
|
||||
# Translate
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
modifiedJAString,
|
||||
"Reply with only the " + LANGUAGE + " translation.",
|
||||
False,
|
||||
|
|
@ -332,7 +346,7 @@ def translateNoteOmitSpace(event, regex):
|
|||
jaString = re.sub(r"\n", " ", oldJAString)
|
||||
|
||||
# Translate
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
jaString,
|
||||
"Reply with the " + LANGUAGE + " translation of the location name.",
|
||||
False,
|
||||
|
|
@ -644,7 +658,7 @@ def searchNames(data, pbar, context):
|
|||
while number < 5:
|
||||
if f"message{number}" in data[i] and data[i][f"message{number}"]:
|
||||
if data[i][f"message{number}"][0] in ["は", "を", "の", "に", "が"]:
|
||||
msgResponse = translateGPT(
|
||||
msgResponse = translateAI(
|
||||
"Taro" + data[i][f"message{number}"],
|
||||
"reply with only the gender neutral "
|
||||
+ LANGUAGE
|
||||
|
|
@ -657,7 +671,7 @@ def searchNames(data, pbar, context):
|
|||
number += 1
|
||||
|
||||
else:
|
||||
msgResponse = translateGPT(
|
||||
msgResponse = translateAI(
|
||||
data[i][f"message{number}"],
|
||||
"reply with only the gender neutral " + LANGUAGE + " translation",
|
||||
False,
|
||||
|
|
@ -721,21 +735,21 @@ def searchNames(data, pbar, context):
|
|||
k = j # Original Index
|
||||
if context in "Actors":
|
||||
# Name
|
||||
response = translateGPT(nameList, newContext, True)
|
||||
response = translateAI(nameList, newContext, True)
|
||||
translatedNameBatch = response[0]
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
|
||||
# Nickname
|
||||
if nicknameList:
|
||||
response = translateGPT(nicknameList, newContext, True)
|
||||
response = translateAI(nicknameList, newContext, True)
|
||||
translatedNicknameBatch = response[0]
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
|
||||
# Profile
|
||||
if profileList:
|
||||
response = translateGPT(profileList, "", True)
|
||||
response = translateAI(profileList, "", True)
|
||||
translatedProfileBatch = response[0]
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
|
|
@ -775,14 +789,14 @@ def searchNames(data, pbar, context):
|
|||
|
||||
if context in ["Armors", "Weapons", "Items", "Skills"]:
|
||||
# Name
|
||||
response = translateGPT(nameList, newContext, True)
|
||||
response = translateAI(nameList, newContext, True)
|
||||
translatedNameBatch = response[0]
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
|
||||
# Description
|
||||
if descriptionList:
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
descriptionList,
|
||||
f"Reply with only the {LANGUAGE} translation of the text.",
|
||||
True,
|
||||
|
|
@ -820,7 +834,7 @@ def searchNames(data, pbar, context):
|
|||
else:
|
||||
mismatch = True
|
||||
if context in ["Enemies", "Classes", "MapInfos"]:
|
||||
response = translateGPT(nameList, newContext, True)
|
||||
response = translateAI(nameList, newContext, True)
|
||||
translatedNameBatch = response[0]
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
|
|
@ -1434,7 +1448,7 @@ def searchCodes(page, pbar, jobList, filename):
|
|||
|
||||
# Remove any textwrap & TL
|
||||
jaString = re.sub(r"\n", " ", jaString)
|
||||
response = translateGPT(jaString, "", False)
|
||||
response = translateAI(jaString, "", False)
|
||||
translatedText = response[0]
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
|
|
@ -1449,7 +1463,7 @@ def searchCodes(page, pbar, jobList, filename):
|
|||
if matchList != None:
|
||||
# Translate
|
||||
question = codeList[i]["p"][3]["messageText"]
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
matchList,
|
||||
f"Previous text for context: {question}\n",
|
||||
True,
|
||||
|
|
@ -1501,7 +1515,7 @@ def searchCodes(page, pbar, jobList, filename):
|
|||
jaString = re.sub(r"\n", " ", jaString)
|
||||
|
||||
# Translate
|
||||
response = translateGPT(jaString, "", True)
|
||||
response = translateAI(jaString, "", True)
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
translatedText = response[0]
|
||||
|
|
@ -1732,7 +1746,7 @@ def searchCodes(page, pbar, jobList, filename):
|
|||
matchList = re.findall(r"Tachie showName (.+)", jaString)
|
||||
if len(matchList) > 0:
|
||||
# Translate
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
matchList[0],
|
||||
"Reply with the " + LANGUAGE + " translation of the NPC name.",
|
||||
False,
|
||||
|
|
@ -1806,7 +1820,7 @@ def searchCodes(page, pbar, jobList, filename):
|
|||
if len(matchList) > 0:
|
||||
# Translate
|
||||
text = matchList[0]
|
||||
response = translateGPT(text, "Reply with the " + LANGUAGE + " Translation", False)
|
||||
response = translateAI(text, "Reply with the " + LANGUAGE + " Translation", False)
|
||||
translatedText = response[0]
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
|
|
@ -1820,7 +1834,7 @@ def searchCodes(page, pbar, jobList, filename):
|
|||
if len(matchList) > 0:
|
||||
# Translate
|
||||
text = matchList[0]
|
||||
response = translateGPT(text, "Reply with the " + LANGUAGE + " Translation", False)
|
||||
response = translateAI(text, "Reply with the " + LANGUAGE + " Translation", False)
|
||||
translatedText = response[0]
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
|
|
@ -1835,7 +1849,7 @@ def searchCodes(page, pbar, jobList, filename):
|
|||
if len(matchList) > 0:
|
||||
# Translate
|
||||
text = matchList[0]
|
||||
response = translateGPT(text, "Reply with the " + LANGUAGE + " Translation", False)
|
||||
response = translateAI(text, "Reply with the " + LANGUAGE + " Translation", False)
|
||||
translatedText = response[0]
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
|
|
@ -1854,7 +1868,7 @@ def searchCodes(page, pbar, jobList, filename):
|
|||
|
||||
# Remove any textwrap & TL
|
||||
jaString = re.sub(r"\n", " ", jaString)
|
||||
response = translateGPT(jaString, "", False)
|
||||
response = translateAI(jaString, "", False)
|
||||
translatedText = response[0]
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
|
|
@ -1876,7 +1890,7 @@ def searchCodes(page, pbar, jobList, filename):
|
|||
|
||||
# Translate
|
||||
question = translatedText
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
choiceList,
|
||||
f"Previous text for context: {question}\n",
|
||||
True,
|
||||
|
|
@ -1920,7 +1934,7 @@ def searchCodes(page, pbar, jobList, filename):
|
|||
|
||||
# Translate
|
||||
if len(textHistory) > 0:
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
choiceList,
|
||||
f"Reply with the English translation of the dialogue choice.\n\nPrevious text for context: {str(textHistory)}\n",
|
||||
True,
|
||||
|
|
@ -1929,7 +1943,7 @@ def searchCodes(page, pbar, jobList, filename):
|
|||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
else:
|
||||
response = translateGPT(choiceList, "Reply with the English translation of the dialogue choice.", True)
|
||||
response = translateAI(choiceList, "Reply with the English translation of the dialogue choice.", True)
|
||||
translatedTextList = response[0]
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
|
|
@ -1982,7 +1996,7 @@ def searchCodes(page, pbar, jobList, filename):
|
|||
matchList = re.findall(r"'(.*?)'", jaString)
|
||||
|
||||
for match in matchList:
|
||||
response = translateGPT(match, "", False)
|
||||
response = translateAI(match, "", False)
|
||||
translatedText = response[0]
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
|
|
@ -2045,7 +2059,7 @@ def searchCodes(page, pbar, jobList, filename):
|
|||
|
||||
# 401
|
||||
if len(list401) > 0:
|
||||
response = translateGPT(list401, "", True)
|
||||
response = translateAI(list401, "", True)
|
||||
list401TL = response[0]
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
|
|
@ -2056,7 +2070,7 @@ def searchCodes(page, pbar, jobList, filename):
|
|||
|
||||
# 122
|
||||
if len(list122) > 0:
|
||||
response = translateGPT(list122, "Keep your translation as brief as possible", True)
|
||||
response = translateAI(list122, "Keep your translation as brief as possible", True)
|
||||
list122TL = response[0]
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
|
|
@ -2067,7 +2081,7 @@ def searchCodes(page, pbar, jobList, filename):
|
|||
|
||||
# 355/655
|
||||
if len(list355655) > 0:
|
||||
response = translateGPT(list355655, textHistory, True)
|
||||
response = translateAI(list355655, textHistory, True)
|
||||
list355655TL = response[0]
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
|
|
@ -2078,7 +2092,7 @@ def searchCodes(page, pbar, jobList, filename):
|
|||
|
||||
# 108
|
||||
if len(list108) > 0:
|
||||
response = translateGPT(list108, textHistory, True)
|
||||
response = translateAI(list108, textHistory, True)
|
||||
list108TL = response[0]
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
|
|
@ -2089,7 +2103,7 @@ def searchCodes(page, pbar, jobList, filename):
|
|||
|
||||
# 356
|
||||
if len(list356) > 0:
|
||||
response = translateGPT(list356, textHistory, True)
|
||||
response = translateAI(list356, textHistory, True)
|
||||
list356TL = response[0]
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
|
|
@ -2100,7 +2114,7 @@ def searchCodes(page, pbar, jobList, filename):
|
|||
|
||||
# 357
|
||||
if len(list357) > 0:
|
||||
response = translateGPT(list357, textHistory, True)
|
||||
response = translateAI(list357, textHistory, True)
|
||||
list357TL = response[0]
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
|
|
@ -2111,7 +2125,7 @@ def searchCodes(page, pbar, jobList, filename):
|
|||
|
||||
# 408
|
||||
if len(list408) > 0:
|
||||
response = translateGPT(list408, "", True)
|
||||
response = translateAI(list408, "", True)
|
||||
list408TL = response[0]
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
|
|
@ -2155,7 +2169,7 @@ def searchSS(state, pbar):
|
|||
|
||||
# Name
|
||||
nameResponse = (
|
||||
translateGPT(
|
||||
translateAI(
|
||||
state["name"],
|
||||
"Reply with only the " + LANGUAGE + " translation of the RPG Skill name.",
|
||||
False,
|
||||
|
|
@ -2166,7 +2180,7 @@ def searchSS(state, pbar):
|
|||
|
||||
# Description
|
||||
descriptionResponse = (
|
||||
translateGPT(
|
||||
translateAI(
|
||||
state["description"],
|
||||
"Reply with only the " + LANGUAGE + " translation of the description.",
|
||||
False,
|
||||
|
|
@ -2189,7 +2203,7 @@ def searchSS(state, pbar):
|
|||
"に",
|
||||
"が",
|
||||
]:
|
||||
message1Response = translateGPT(
|
||||
message1Response = translateAI(
|
||||
"Taro" + state["message1"],
|
||||
"reply with only the gender neutral "
|
||||
+ LANGUAGE
|
||||
|
|
@ -2198,7 +2212,7 @@ Translate 'Taroを倒した！' as 'Taro was defeated!'",
|
|||
False,
|
||||
)
|
||||
else:
|
||||
message1Response = translateGPT(
|
||||
message1Response = translateAI(
|
||||
state["message1"],
|
||||
"reply with only the gender neutral " + LANGUAGE + " translation",
|
||||
False,
|
||||
|
|
@ -2212,7 +2226,7 @@ Translate 'Taroを倒した！' as 'Taro was defeated!'",
|
|||
"に",
|
||||
"が",
|
||||
]:
|
||||
message2Response = translateGPT(
|
||||
message2Response = translateAI(
|
||||
"Taro" + state["message2"],
|
||||
"reply with only the gender neutral "
|
||||
+ LANGUAGE
|
||||
|
|
@ -2221,7 +2235,7 @@ Translate 'Taroを倒した！' as 'Taro was defeated!'",
|
|||
False,
|
||||
)
|
||||
else:
|
||||
message2Response = translateGPT(
|
||||
message2Response = translateAI(
|
||||
state["message2"],
|
||||
"reply with only the gender neutral " + LANGUAGE + " translation",
|
||||
False,
|
||||
|
|
@ -2235,7 +2249,7 @@ Translate 'Taroを倒した！' as 'Taro was defeated!'",
|
|||
"に",
|
||||
"が",
|
||||
]:
|
||||
message3Response = translateGPT(
|
||||
message3Response = translateAI(
|
||||
"Taro" + state["message3"],
|
||||
"reply with only the gender neutral "
|
||||
+ LANGUAGE
|
||||
|
|
@ -2244,7 +2258,7 @@ Translate 'Taroを倒した！' as 'Taro was defeated!'",
|
|||
False,
|
||||
)
|
||||
else:
|
||||
message3Response = translateGPT(
|
||||
message3Response = translateAI(
|
||||
state["message3"],
|
||||
"reply with only the gender neutral " + LANGUAGE + " translation",
|
||||
False,
|
||||
|
|
@ -2258,7 +2272,7 @@ Translate 'Taroを倒した！' as 'Taro was defeated!'",
|
|||
"に",
|
||||
"が",
|
||||
]:
|
||||
message4Response = translateGPT(
|
||||
message4Response = translateAI(
|
||||
"Taro" + state["message4"],
|
||||
"reply with only the gender neutral "
|
||||
+ LANGUAGE
|
||||
|
|
@ -2267,7 +2281,7 @@ Translate 'Taroを倒した！' as 'Taro was defeated!'",
|
|||
False,
|
||||
)
|
||||
else:
|
||||
message4Response = translateGPT(
|
||||
message4Response = translateAI(
|
||||
state["message4"],
|
||||
"reply with only the gender neutral " + LANGUAGE + " translation",
|
||||
False,
|
||||
|
|
@ -2330,7 +2344,7 @@ def searchSystem(data, pbar):
|
|||
context = "Reply with only the " + LANGUAGE + ' translation of the UI textbox."'
|
||||
|
||||
# Title
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
data["gameTitle"],
|
||||
" Reply with the " + LANGUAGE + " translation of the game title name",
|
||||
False,
|
||||
|
|
@ -2345,14 +2359,14 @@ def searchSystem(data, pbar):
|
|||
termList = data["terms"][term]
|
||||
for i in range(len(termList)): # Last item is a messages object
|
||||
if termList[i] is not None:
|
||||
response = translateGPT(termList[i], context, False)
|
||||
response = translateAI(termList[i], context, False)
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
termList[i] = response[0].replace('"', "").strip()
|
||||
|
||||
# Armor Types
|
||||
for i in range(len(data["armor_types"])):
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
data["armor_types"][i],
|
||||
"Reply with only the " + LANGUAGE + " translation of the armor type",
|
||||
False,
|
||||
|
|
@ -2363,7 +2377,7 @@ def searchSystem(data, pbar):
|
|||
|
||||
# Skill Types
|
||||
for i in range(len(data["skill_types"])):
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
data["skill_types"][i],
|
||||
"Reply with only the " + LANGUAGE + " translation",
|
||||
False,
|
||||
|
|
@ -2374,7 +2388,7 @@ def searchSystem(data, pbar):
|
|||
|
||||
# Equip Types
|
||||
for i in range(len(data["weapon_types"])):
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
data["weapon_types"][i],
|
||||
"Reply with only the " + LANGUAGE + " translation of the equipment type. No disclaimers.",
|
||||
False,
|
||||
|
|
@ -2385,7 +2399,7 @@ def searchSystem(data, pbar):
|
|||
|
||||
# # Variables (Optional ususally)
|
||||
# for i in range(len(data['variables'])):
|
||||
# response = translateGPT(data['variables'][i], 'Reply with only the '+ LANGUAGE +' translation of the title', False)
|
||||
# response = translateAI(data['variables'][i], 'Reply with only the '+ LANGUAGE +' translation of the title', False)
|
||||
# totalTokens[0] += response[1][0]
|
||||
# totalTokens[1] += response[1][1]
|
||||
# data['variables'][i] = response[0].replace('\"', '').strip()
|
||||
|
|
@ -2393,7 +2407,7 @@ def searchSystem(data, pbar):
|
|||
# Messages
|
||||
messages = data["terms"]
|
||||
for key, value in messages.items():
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
value,
|
||||
"Reply with only the "
|
||||
+ LANGUAGE
|
||||
|
|
@ -2427,7 +2441,7 @@ def getSpeaker(speaker):
|
|||
return [NAMESLIST[i][1], [0, 0]]
|
||||
|
||||
# Translate and Store Speaker
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
f"{speaker}",
|
||||
"Reply with the " + LANGUAGE + " translation of the NPC name.",
|
||||
False,
|
||||
|
|
@ -2438,7 +2452,7 @@ def getSpeaker(speaker):
|
|||
|
||||
# Retry if name doesn't translate for some reason
|
||||
if re.search(r"([a-zA-Z？?])", response[0]) == None:
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
f"{speaker}",
|
||||
"Reply with the " + LANGUAGE + " translation of the NPC name.",
|
||||
False,
|
||||
|
|
@ -2451,328 +2465,24 @@ def getSpeaker(speaker):
|
|||
return response
|
||||
return [speaker, [0, 0]]
|
||||
|
||||
|
||||
def batchList(input_list, batch_size):
|
||||
if not isinstance(batch_size, int) or batch_size <= 0:
|
||||
raise ValueError("batch_size must be a positive integer")
|
||||
|
||||
return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)]
|
||||
|
||||
|
||||
def parseVocabWithCategories(vocabText):
|
||||
"""Parse vocabulary text and extract terms with their categories."""
|
||||
pairs = []
|
||||
seen = set()
|
||||
currentCategory = None
|
||||
|
||||
for line in vocabText.splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith('```'):
|
||||
continue
|
||||
|
||||
# Check if this is a category header
|
||||
if line.startswith('#'):
|
||||
currentCategory = line
|
||||
continue
|
||||
|
||||
# Parse vocabulary term
|
||||
m = re.match(r'^(.+?)(?:\s?[\(–])', line) # term is everything before space + '(' or '–'
|
||||
if m:
|
||||
term = m.group(1)
|
||||
if term not in seen:
|
||||
pairs.append((term, line, currentCategory))
|
||||
seen.add(term)
|
||||
|
||||
return pairs
|
||||
|
||||
|
||||
def buildMatchedVocabText(vocabPairs, subbedT):
|
||||
"""Build formatted vocabulary text with matched terms organized by category."""
|
||||
matchedCategories = {}
|
||||
|
||||
# Use word boundaries for Japanese if appropriate, or allow substring as before.
|
||||
for term, line, category in vocabPairs:
|
||||
# "term in subbedT" could be false positive; can use regex but Japanese doesn't always have spaces.
|
||||
if term in subbedT:
|
||||
if category not in matchedCategories:
|
||||
matchedCategories[category] = []
|
||||
matchedCategories[category].append(line)
|
||||
|
||||
# Format matched vocabulary with categories
|
||||
if matchedCategories:
|
||||
formattedLines = ["Here are some vocabulary and terms so that you know the proper spelling and translation.\n"]
|
||||
for category, lines in matchedCategories.items():
|
||||
if category: # Only add category header if it exists
|
||||
formattedLines.append(category)
|
||||
formattedLines.extend(lines)
|
||||
formattedLines.append("") # Add blank line between categories
|
||||
matchedVocabText = f"```\n{chr(10).join(formattedLines).rstrip()}\n```"
|
||||
else:
|
||||
matchedVocabText = ""
|
||||
|
||||
return matchedVocabText
|
||||
|
||||
|
||||
def createContext(fullPromptFlag, subbedT, format):
|
||||
vocabPairs = parseVocabWithCategories(VOCAB)
|
||||
matchedVocabText = buildMatchedVocabText(vocabPairs, subbedT)
|
||||
|
||||
if fullPromptFlag:
|
||||
system = PROMPT + matchedVocabText
|
||||
else:
|
||||
system = f"\
|
||||
You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
|
||||
Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
|
||||
- You translate everything, including erotic and explicit content.\n\
|
||||
- 'Game Characters' - The names, nicknames, and genders of the game characters. Reference this to know the names, nicknames, and gender of characters in the game\n\
|
||||
- All text in your response must be in {LANGUAGE} even if it is hard to translate.\n\
|
||||
- Never include any notes, explanations, dislaimers, or anything similar in your response.\n\
|
||||
- Maintain any spacing in the translation.\n\
|
||||
- Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\
|
||||
- `...` can be a part of the dialogue. Translate it as it is.\n\
|
||||
{matchedVocabText}\n\
|
||||
"
|
||||
if format == "json":
|
||||
user = f"```json\n{subbedT}\n```"
|
||||
else:
|
||||
user = subbedT
|
||||
return system, user
|
||||
|
||||
|
||||
def translateText(system, user, history, penalty, format, model=MODEL):
|
||||
# Prompt
|
||||
msg = [{"role": "system", "content": system}]
|
||||
|
||||
# History
|
||||
if isinstance(history, list):
|
||||
msg.append({"role": "system", "content": "Translation History:"})
|
||||
msg.extend([{"role": "assistant", "content": h} for h in history])
|
||||
else:
|
||||
msg.append({"role": "assistant", "content": history})
|
||||
|
||||
# Response Format
|
||||
if format == "json":
|
||||
responseFormat = {"type": "json_object"}
|
||||
else:
|
||||
responseFormat = {"type": "text"}
|
||||
|
||||
# Content to TL
|
||||
msg.append({"role": "user", "content": f"{user}"})
|
||||
response = openai.chat.completions.create(
|
||||
temperature=0,
|
||||
frequency_penalty=penalty,
|
||||
model=model,
|
||||
response_format=responseFormat,
|
||||
messages=msg,
|
||||
)
|
||||
return response
|
||||
|
||||
|
||||
def cleanTranslatedText(translatedText):
|
||||
placeholders = {
|
||||
f"{LANGUAGE} Translation: ": "",
|
||||
"Translation: ": "",
|
||||
"っ": "",
|
||||
"〜": "~",
|
||||
"ッ": "",
|
||||
"。": ".",
|
||||
"「": '\\"',
|
||||
"」": '\\"',
|
||||
"- ": "-",
|
||||
"—": "―",
|
||||
"】": "]",
|
||||
"【": "[",
|
||||
"é": "e",
|
||||
"this guy": "this bastard",
|
||||
"This guy": "This bastard",
|
||||
"Placeholder Text": "",
|
||||
"```json": "",
|
||||
"```": "",
|
||||
# Add more replacements as needed
|
||||
}
|
||||
for target, replacement in placeholders.items():
|
||||
translatedText = translatedText.replace(target, replacement)
|
||||
|
||||
# Remove Repeating Characters
|
||||
pattern = re.compile(r"(.)\s*\1(?:\s*\1){" + str(20 - 1) + r",}")
|
||||
translatedText = pattern.sub(lambda match: match.group(0).replace(" ", "")[:20], translatedText)
|
||||
|
||||
# Elongate Long Dashes (Since GPT Ignores them...)
|
||||
translatedText = elongateCharacters(translatedText)
|
||||
return translatedText
|
||||
|
||||
|
||||
def elongateCharacters(text):
|
||||
# Define a pattern to match one character followed by one or more `ー` characters
|
||||
# Using a positive lookbehind assertion to capture the preceding character
|
||||
pattern = r"(?<=(.))ー+"
|
||||
|
||||
# Define a replacement function that elongates the captured character
|
||||
def repl(match):
|
||||
char = match.group(1) # The character before the ー sequence
|
||||
count = len(match.group(0)) - 1 # Number of ー characters
|
||||
return char * count # Replace ー sequence with the character repeated
|
||||
|
||||
# Use re.sub() to replace the pattern in the text
|
||||
return re.sub(pattern, repl, text)
|
||||
|
||||
|
||||
def extractTranslation(translatedTextList, is_list):
|
||||
try:
|
||||
translatedTextList = re.sub(r'\\"+\"([^,\n}])', r'\\"\1', translatedTextList)
|
||||
translatedTextList = re.sub(r"(?<![\\])\"+(?![\n,])", r'"', translatedTextList)
|
||||
line_dict = json.loads(translatedTextList)
|
||||
# If it's a batch (i.e., list), extract with tags; otherwise, return the single item.
|
||||
string_list = list(line_dict.values())
|
||||
if is_list:
|
||||
return string_list
|
||||
else:
|
||||
return string_list[0]
|
||||
|
||||
except Exception as e:
|
||||
PBAR.write(f"extractTranslation Error: {e} on String {translatedTextList}")
|
||||
return None
|
||||
|
||||
|
||||
def countTokens(system, user, history):
|
||||
inputTotalTokens = 0
|
||||
outputTotalTokens = 0
|
||||
enc = tiktoken.encoding_for_model("gpt-4")
|
||||
|
||||
# Input
|
||||
if isinstance(history, list):
|
||||
for line in history:
|
||||
inputTotalTokens += len(enc.encode(line))
|
||||
else:
|
||||
inputTotalTokens += len(enc.encode(history))
|
||||
inputTotalTokens += len(enc.encode(system))
|
||||
inputTotalTokens += len(enc.encode(user))
|
||||
|
||||
# Output
|
||||
outputTotalTokens += round(len(enc.encode(user)) * 2.5)
|
||||
|
||||
return [inputTotalTokens, outputTotalTokens]
|
||||
|
||||
|
||||
@retry(exceptions=Exception, tries=5, delay=5)
|
||||
def translateGPT(text, history, fullPromptFlag):
|
||||
def translateAI(text, history, fullPromptFlag):
|
||||
"""
|
||||
Legacy wrapper function for the new shared translation utility.
|
||||
This maintains compatibility with existing code while using the new shared implementation.
|
||||
"""
|
||||
global PBAR, MISMATCH, FILENAME
|
||||
if text:
|
||||
with open("log/translationHistory.txt", "a+", encoding="utf-8") as logFile:
|
||||
mismatch = False
|
||||
totalTokens = [0, 0]
|
||||
if isinstance(text, list):
|
||||
format = "json"
|
||||
tList = batchList(text, BATCHSIZE)
|
||||
else:
|
||||
format = "text"
|
||||
tList = [text]
|
||||
|
||||
for index, tItem in enumerate(tList):
|
||||
# Things to Check before starting translation
|
||||
if not re.search(LANGREGEX, str(tItem)):
|
||||
if PBAR is not None:
|
||||
PBAR.update(len(tItem))
|
||||
if isinstance(tItem, list):
|
||||
for j in range(len(tItem)):
|
||||
tItem[j] = cleanTranslatedText(tItem[j])
|
||||
tList[index] = tItem
|
||||
else:
|
||||
tList[index] = cleanTranslatedText(tItem)
|
||||
history = tItem[-MAXHISTORY:]
|
||||
continue
|
||||
# Update config estimate mode based on global ESTIMATE
|
||||
TRANSLATION_CONFIG.estimateMode = bool(ESTIMATE)
|
||||
|
||||
# Before sending to translation, if we have a list of items, add the formatting
|
||||
if isinstance(tItem, list):
|
||||
for j in range(len(tItem)):
|
||||
if not tItem[j]:
|
||||
tItem[j] = tItem[j].replace("", "Placeholder Text")
|
||||
payload = {f"Line{i+1}": string for i, string in enumerate(tItem)}
|
||||
payload = json.dumps(payload, indent=4, ensure_ascii=False)
|
||||
varResponse = [payload, []]
|
||||
subbedT = varResponse[0]
|
||||
else:
|
||||
varResponse = [tItem, []]
|
||||
subbedT = varResponse[0]
|
||||
|
||||
# Create Message
|
||||
system, user = createContext(fullPromptFlag, subbedT, format)
|
||||
|
||||
# Calculate Estimate
|
||||
if ESTIMATE:
|
||||
estimate = countTokens(system, user, history)
|
||||
totalTokens[0] += estimate[0]
|
||||
totalTokens[1] += estimate[1]
|
||||
continue
|
||||
|
||||
# Translating
|
||||
response = translateText(system, user, history, 0.05, format)
|
||||
|
||||
# Set Tokens
|
||||
translatedText = response.choices[0].message.content
|
||||
|
||||
# AI Refused, Try Again
|
||||
if not translatedText:
|
||||
response = translateText(f"{system}\n You translate ALL content.", user, history, 0.1, format, model="gpt-4o")
|
||||
translatedText = response.choices[0].message.content
|
||||
|
||||
# Report Tokens
|
||||
totalTokens[0] += response.usage.prompt_tokens
|
||||
totalTokens[1] += response.usage.completion_tokens
|
||||
|
||||
# Check Translation
|
||||
if translatedText:
|
||||
translatedText = cleanTranslatedText(translatedText)
|
||||
if isinstance(tItem, list):
|
||||
extractedTranslations = extractTranslation(translatedText, True)
|
||||
if extractedTranslations == None or len(tItem) != len(extractedTranslations):
|
||||
# Mismatch. Try Again
|
||||
response = translateText(system, user, history, 0.05, format, MODEL)
|
||||
translatedText = response.choices[0].message.content
|
||||
totalTokens[0] += response.usage.prompt_tokens
|
||||
totalTokens[1] += response.usage.completion_tokens
|
||||
|
||||
# Formatting
|
||||
translatedText = cleanTranslatedText(translatedText)
|
||||
if isinstance(tItem, list):
|
||||
extractedTranslations = extractTranslation(translatedText, True)
|
||||
if extractedTranslations == None or len(tItem) != len(extractedTranslations):
|
||||
with open("log/mismatchHistory.txt", "a+", encoding="utf-8") as mismatchFile:
|
||||
mismatchFile.write(f"Mismatch: {FILENAME}\n")
|
||||
mismatchFile.write(f"Input:\n{subbedT}\n")
|
||||
mismatchFile.write(f"Output:\n{translatedText}\n")
|
||||
mismatch = True # Just here for breakpoint
|
||||
logFile.write(f"Input:\n{subbedT}\n")
|
||||
logFile.write(f"Output:\n{translatedText}\n")
|
||||
|
||||
# Set if no mismatch
|
||||
if mismatch == False:
|
||||
tList[index] = extractedTranslations
|
||||
history = extractedTranslations[-MAXHISTORY:] # Update history if we have a list
|
||||
else:
|
||||
history = text[-MAXHISTORY:]
|
||||
mismatch = False
|
||||
if FILENAME not in MISMATCH:
|
||||
MISMATCH.append(FILENAME)
|
||||
|
||||
# Update Loading Bar
|
||||
with LOCK:
|
||||
if PBAR is not None:
|
||||
PBAR.update(len(tItem))
|
||||
else:
|
||||
# Ensure we're passing a single string to extractTranslation
|
||||
tList[index] = translatedText.replace("Placeholder Text", "")
|
||||
else:
|
||||
PBAR.write(f"AI Refused:{tItem}\n")
|
||||
|
||||
# Combine if multilist
|
||||
if isinstance(tList[0], list):
|
||||
tList = [t for sublist in tList for t in sublist]
|
||||
|
||||
# Return
|
||||
if format == "json":
|
||||
return [tList, totalTokens]
|
||||
else:
|
||||
return [tList[0], totalTokens]
|
||||
else:
|
||||
return [text, [0, 0]]
|
||||
# Call the new shared translation function
|
||||
return sharedtranslateAI(
|
||||
text=text,
|
||||
history=history,
|
||||
fullPromptFlag=fullPromptFlag,
|
||||
config=TRANSLATION_CONFIG,
|
||||
filename=FILENAME,
|
||||
pbar=PBAR,
|
||||
lock=LOCK,
|
||||
mismatchList=MISMATCH
|
||||
)
|
||||
|
|
|
|||
|
|
@ -6,7 +6,6 @@ import util.dazedwrap as dazedwrap
|
|||
import threading
|
||||
import time
|
||||
import traceback
|
||||
import tiktoken
|
||||
import openai
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from pathlib import Path
|
||||
|
|
@ -14,6 +13,7 @@ from colorama import Fore
|
|||
from dotenv import load_dotenv
|
||||
from retry import retry
|
||||
from tqdm import tqdm
|
||||
from util.translation import TranslationConfig, translateAI as sharedtranslateAI
|
||||
|
||||
# Open AI
|
||||
load_dotenv()
|
||||
|
|
@ -72,6 +72,18 @@ else:
|
|||
# tqdm Globals
|
||||
BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
|
||||
POSITION = 0
|
||||
|
||||
# Initialize Translation Config
|
||||
TRANSLATION_CONFIG = TranslationConfig(
|
||||
model=MODEL,
|
||||
language=LANGUAGE,
|
||||
prompt=PROMPT,
|
||||
vocab=VOCAB,
|
||||
langRegex=LANGREGEX,
|
||||
batchSize=BATCHSIZE,
|
||||
maxHistory=MAXHISTORY,
|
||||
estimateMode=False # Will be set dynamically based on ESTIMATE
|
||||
)
|
||||
LEAVE = False
|
||||
|
||||
# Config (Default)
|
||||
|
|
@ -243,7 +255,7 @@ def parseMap(data, filename):
|
|||
|
||||
# Translate displayName for Map files
|
||||
if "Map" in filename:
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
data["displayName"],
|
||||
"Reply with only the " + LANGUAGE + " translation of the RPG location name",
|
||||
False,
|
||||
|
|
@ -256,7 +268,7 @@ def parseMap(data, filename):
|
|||
for event in events:
|
||||
if event:
|
||||
if "<LB>" in event["note"]:
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
event["name"],
|
||||
"Reply with only the " + LANGUAGE + " translation of the RPG location name",
|
||||
False,
|
||||
|
|
@ -318,7 +330,7 @@ def translateNote(event, regex, wordwrap=False):
|
|||
modifiedJAString = modifiedJAString.replace("\n", " ")
|
||||
|
||||
# Translate
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
modifiedJAString,
|
||||
"Reply with only the " + LANGUAGE + " translation.",
|
||||
False,
|
||||
|
|
@ -351,7 +363,7 @@ def translateNoteOmitSpace(event, regex):
|
|||
jaString = re.sub(r"\n", " ", oldJAString)
|
||||
|
||||
# Translate
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
jaString,
|
||||
"Reply with the " + LANGUAGE + " translation of the location name.",
|
||||
False,
|
||||
|
|
@ -593,7 +605,7 @@ def searchNames(data, pbar, context):
|
|||
# --- Batch translate all notes ---
|
||||
translatedNotesBatch = []
|
||||
if notesBatch:
|
||||
response = translateGPT(notesBatch, f"Reply with only the {LANGUAGE} translation of the note text.", True)
|
||||
response = translateAI(notesBatch, f"Reply with only the {LANGUAGE} translation of the note text.", True)
|
||||
translatedNotesBatch = response[0]
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
|
|
@ -655,7 +667,7 @@ def searchNames(data, pbar, context):
|
|||
while number < 5:
|
||||
if f"message{number}" in data[i] and data[i][f"message{number}"]:
|
||||
if data[i][f"message{number}"][0] in ["は", "を", "の", "に", "が"]:
|
||||
msgResponse = translateGPT(
|
||||
msgResponse = translateAI(
|
||||
"Taro" + data[i][f"message{number}"],
|
||||
"reply with only the gender neutral "
|
||||
+ LANGUAGE
|
||||
|
|
@ -667,7 +679,7 @@ def searchNames(data, pbar, context):
|
|||
totalTokens[1] += msgResponse[1][1]
|
||||
number += 1
|
||||
else:
|
||||
msgResponse = translateGPT(
|
||||
msgResponse = translateAI(
|
||||
data[i][f"message{number}"],
|
||||
"reply with only the gender neutral " + LANGUAGE + " translation",
|
||||
False,
|
||||
|
|
@ -693,21 +705,21 @@ def searchNames(data, pbar, context):
|
|||
k = j # Original Index
|
||||
if context in "Actors":
|
||||
# Name
|
||||
response = translateGPT(nameList, newContext, True)
|
||||
response = translateAI(nameList, newContext, True)
|
||||
translatedNameBatch = response[0]
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
|
||||
# Nickname
|
||||
if nicknameList:
|
||||
response = translateGPT(nicknameList, newContext, True)
|
||||
response = translateAI(nicknameList, newContext, True)
|
||||
translatedNicknameBatch = response[0]
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
|
||||
# Profile
|
||||
if profileList:
|
||||
response = translateGPT(profileList, "", True)
|
||||
response = translateAI(profileList, "", True)
|
||||
translatedProfileBatch = response[0]
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
|
|
@ -747,14 +759,14 @@ def searchNames(data, pbar, context):
|
|||
|
||||
if context in ["Armors", "Weapons", "Items", "Skills"]:
|
||||
# Name
|
||||
response = translateGPT(nameList, newContext, True)
|
||||
response = translateAI(nameList, newContext, True)
|
||||
translatedNameBatch = response[0]
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
|
||||
# Description
|
||||
if descriptionList:
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
descriptionList,
|
||||
f"Reply with only the {LANGUAGE} translation of the text.",
|
||||
True,
|
||||
|
|
@ -792,7 +804,7 @@ def searchNames(data, pbar, context):
|
|||
else:
|
||||
mismatch = True
|
||||
if context in ["Enemies", "Classes", "MapInfos"]:
|
||||
response = translateGPT(nameList, newContext, True)
|
||||
response = translateAI(nameList, newContext, True)
|
||||
translatedNameBatch = response[0]
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
|
|
@ -1414,7 +1426,7 @@ def searchCodes(page, pbar, jobList, filename):
|
|||
|
||||
# Remove any textwrap & TL
|
||||
jaString = re.sub(r"\n", " ", jaString)
|
||||
response = translateGPT(jaString, "", False)
|
||||
response = translateAI(jaString, "", False)
|
||||
translatedText = response[0]
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
|
|
@ -1429,7 +1441,7 @@ def searchCodes(page, pbar, jobList, filename):
|
|||
if matchList != None:
|
||||
# Translate
|
||||
question = codeList[i]["parameters"][3]["messageText"]
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
matchList,
|
||||
f"Previous text for context: {question}\n",
|
||||
True,
|
||||
|
|
@ -1481,7 +1493,7 @@ def searchCodes(page, pbar, jobList, filename):
|
|||
jaString = re.sub(r"\n", " ", jaString)
|
||||
|
||||
# Translate
|
||||
response = translateGPT(jaString, "", True)
|
||||
response = translateAI(jaString, "", True)
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
translatedText = response[0]
|
||||
|
|
@ -1712,7 +1724,7 @@ def searchCodes(page, pbar, jobList, filename):
|
|||
matchList = re.findall(r"Tachie showName (.+)", jaString)
|
||||
if len(matchList) > 0:
|
||||
# Translate
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
matchList[0],
|
||||
"Reply with the " + LANGUAGE + " translation of the NPC name.",
|
||||
False,
|
||||
|
|
@ -1802,7 +1814,7 @@ def searchCodes(page, pbar, jobList, filename):
|
|||
if len(matchList) > 0:
|
||||
# Translate
|
||||
text = matchList[0]
|
||||
response = translateGPT(text, "Reply with the " + LANGUAGE + " Translation", False)
|
||||
response = translateAI(text, "Reply with the " + LANGUAGE + " Translation", False)
|
||||
translatedText = response[0]
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
|
|
@ -1816,7 +1828,7 @@ def searchCodes(page, pbar, jobList, filename):
|
|||
if len(matchList) > 0:
|
||||
# Translate
|
||||
text = matchList[0]
|
||||
response = translateGPT(text, "Reply with the " + LANGUAGE + " Translation", False)
|
||||
response = translateAI(text, "Reply with the " + LANGUAGE + " Translation", False)
|
||||
translatedText = response[0]
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
|
|
@ -1831,7 +1843,7 @@ def searchCodes(page, pbar, jobList, filename):
|
|||
if len(matchList) > 0:
|
||||
# Translate
|
||||
text = matchList[0]
|
||||
response = translateGPT(text, "Reply with the " + LANGUAGE + " Translation", False)
|
||||
response = translateAI(text, "Reply with the " + LANGUAGE + " Translation", False)
|
||||
translatedText = response[0]
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
|
|
@ -1850,7 +1862,7 @@ def searchCodes(page, pbar, jobList, filename):
|
|||
|
||||
# Remove any textwrap & TL
|
||||
jaString = re.sub(r"\n", " ", jaString)
|
||||
response = translateGPT(jaString, "", False)
|
||||
response = translateAI(jaString, "", False)
|
||||
translatedText = response[0]
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
|
|
@ -1872,7 +1884,7 @@ def searchCodes(page, pbar, jobList, filename):
|
|||
|
||||
# Translate
|
||||
question = translatedText
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
choiceList,
|
||||
f"Previous text for context: {question}\n",
|
||||
True,
|
||||
|
|
@ -1916,7 +1928,7 @@ def searchCodes(page, pbar, jobList, filename):
|
|||
|
||||
# Translate
|
||||
if len(textHistory) > 0:
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
choiceList,
|
||||
f"Reply with the English translation of the dialogue choice.\n\nPrevious text for context: {str(textHistory)}\n",
|
||||
True,
|
||||
|
|
@ -1925,7 +1937,7 @@ def searchCodes(page, pbar, jobList, filename):
|
|||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
else:
|
||||
response = translateGPT(choiceList, "Reply with the English translation of the dialogue choice.", True)
|
||||
response = translateAI(choiceList, "Reply with the English translation of the dialogue choice.", True)
|
||||
translatedTextList = response[0]
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
|
|
@ -1978,7 +1990,7 @@ def searchCodes(page, pbar, jobList, filename):
|
|||
matchList = re.findall(r"'(.*?)'", jaString)
|
||||
|
||||
for match in matchList:
|
||||
response = translateGPT(match, "", False)
|
||||
response = translateAI(match, "", False)
|
||||
translatedText = response[0]
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
|
|
@ -2041,7 +2053,7 @@ def searchCodes(page, pbar, jobList, filename):
|
|||
|
||||
# 401
|
||||
if len(list401) > 0:
|
||||
response = translateGPT(list401, "", True)
|
||||
response = translateAI(list401, "", True)
|
||||
list401TL = response[0]
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
|
|
@ -2052,7 +2064,7 @@ def searchCodes(page, pbar, jobList, filename):
|
|||
|
||||
# 122
|
||||
if len(list122) > 0:
|
||||
response = translateGPT(list122, "Keep your translation as brief as possible", True)
|
||||
response = translateAI(list122, "Keep your translation as brief as possible", True)
|
||||
list122TL = response[0]
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
|
|
@ -2063,7 +2075,7 @@ def searchCodes(page, pbar, jobList, filename):
|
|||
|
||||
# 355/655
|
||||
if len(list355655) > 0:
|
||||
response = translateGPT(list355655, textHistory, True)
|
||||
response = translateAI(list355655, textHistory, True)
|
||||
list355655TL = response[0]
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
|
|
@ -2074,7 +2086,7 @@ def searchCodes(page, pbar, jobList, filename):
|
|||
|
||||
# 108
|
||||
if len(list108) > 0:
|
||||
response = translateGPT(list108, textHistory, True)
|
||||
response = translateAI(list108, textHistory, True)
|
||||
list108TL = response[0]
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
|
|
@ -2085,7 +2097,7 @@ def searchCodes(page, pbar, jobList, filename):
|
|||
|
||||
# 356
|
||||
if len(list356) > 0:
|
||||
response = translateGPT(list356, textHistory, True)
|
||||
response = translateAI(list356, textHistory, True)
|
||||
list356TL = response[0]
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
|
|
@ -2096,7 +2108,7 @@ def searchCodes(page, pbar, jobList, filename):
|
|||
|
||||
# 357
|
||||
if len(list357) > 0:
|
||||
response = translateGPT(list357, textHistory, True)
|
||||
response = translateAI(list357, textHistory, True)
|
||||
list357TL = response[0]
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
|
|
@ -2107,7 +2119,7 @@ def searchCodes(page, pbar, jobList, filename):
|
|||
|
||||
# 408
|
||||
if len(list408) > 0:
|
||||
response = translateGPT(list408, "", True)
|
||||
response = translateAI(list408, "", True)
|
||||
list408TL = response[0]
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
|
|
@ -2151,7 +2163,7 @@ def searchSS(state, pbar):
|
|||
|
||||
# Name
|
||||
nameResponse = (
|
||||
translateGPT(
|
||||
translateAI(
|
||||
state["name"],
|
||||
"Reply with only the " + LANGUAGE + " translation of the RPG Skill name.",
|
||||
False,
|
||||
|
|
@ -2162,7 +2174,7 @@ def searchSS(state, pbar):
|
|||
|
||||
# Description
|
||||
descriptionResponse = (
|
||||
translateGPT(
|
||||
translateAI(
|
||||
state["description"],
|
||||
"Reply with only the " + LANGUAGE + " translation of the description.",
|
||||
False,
|
||||
|
|
@ -2185,7 +2197,7 @@ def searchSS(state, pbar):
|
|||
"に",
|
||||
"が",
|
||||
]:
|
||||
message1Response = translateGPT(
|
||||
message1Response = translateAI(
|
||||
"Taro" + state["message1"],
|
||||
"reply with only the gender neutral "
|
||||
+ LANGUAGE
|
||||
|
|
@ -2194,7 +2206,7 @@ Translate 'Taroを倒した！' as 'Taro was defeated!'",
|
|||
False,
|
||||
)
|
||||
else:
|
||||
message1Response = translateGPT(
|
||||
message1Response = translateAI(
|
||||
state["message1"],
|
||||
"reply with only the gender neutral " + LANGUAGE + " translation",
|
||||
False,
|
||||
|
|
@ -2208,7 +2220,7 @@ Translate 'Taroを倒した！' as 'Taro was defeated!'",
|
|||
"に",
|
||||
"が",
|
||||
]:
|
||||
message2Response = translateGPT(
|
||||
message2Response = translateAI(
|
||||
"Taro" + state["message2"],
|
||||
"reply with only the gender neutral "
|
||||
+ LANGUAGE
|
||||
|
|
@ -2217,7 +2229,7 @@ Translate 'Taroを倒した！' as 'Taro was defeated!'",
|
|||
False,
|
||||
)
|
||||
else:
|
||||
message2Response = translateGPT(
|
||||
message2Response = translateAI(
|
||||
state["message2"],
|
||||
"reply with only the gender neutral " + LANGUAGE + " translation",
|
||||
False,
|
||||
|
|
@ -2231,7 +2243,7 @@ Translate 'Taroを倒した！' as 'Taro was defeated!'",
|
|||
"に",
|
||||
"が",
|
||||
]:
|
||||
message3Response = translateGPT(
|
||||
message3Response = translateAI(
|
||||
"Taro" + state["message3"],
|
||||
"reply with only the gender neutral "
|
||||
+ LANGUAGE
|
||||
|
|
@ -2240,7 +2252,7 @@ Translate 'Taroを倒した！' as 'Taro was defeated!'",
|
|||
False,
|
||||
)
|
||||
else:
|
||||
message3Response = translateGPT(
|
||||
message3Response = translateAI(
|
||||
state["message3"],
|
||||
"reply with only the gender neutral " + LANGUAGE + " translation",
|
||||
False,
|
||||
|
|
@ -2254,7 +2266,7 @@ Translate 'Taroを倒した！' as 'Taro was defeated!'",
|
|||
"に",
|
||||
"が",
|
||||
]:
|
||||
message4Response = translateGPT(
|
||||
message4Response = translateAI(
|
||||
"Taro" + state["message4"],
|
||||
"reply with only the gender neutral "
|
||||
+ LANGUAGE
|
||||
|
|
@ -2263,7 +2275,7 @@ Translate 'Taroを倒した！' as 'Taro was defeated!'",
|
|||
False,
|
||||
)
|
||||
else:
|
||||
message4Response = translateGPT(
|
||||
message4Response = translateAI(
|
||||
state["message4"],
|
||||
"reply with only the gender neutral " + LANGUAGE + " translation",
|
||||
False,
|
||||
|
|
@ -2290,7 +2302,7 @@ Translate 'Taroを倒した！' as 'Taro was defeated!'",
|
|||
# --- Batch translate all notes ---
|
||||
translatedNotesBatch = []
|
||||
if notesBatch:
|
||||
response = translateGPT(notesBatch, f"Reply with only the {LANGUAGE} translation of the note text.", True)
|
||||
response = translateAI(notesBatch, f"Reply with only the {LANGUAGE} translation of the note text.", True)
|
||||
translatedNotesBatch = response[0]
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
|
|
@ -2348,7 +2360,7 @@ def searchSystem(data, pbar):
|
|||
context = "Reply with only the " + LANGUAGE + ' translation of the UI textbox."'
|
||||
|
||||
# Title
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
data["gameTitle"],
|
||||
" Reply with the " + LANGUAGE + " translation of the game title name",
|
||||
False,
|
||||
|
|
@ -2363,14 +2375,14 @@ def searchSystem(data, pbar):
|
|||
termList = data["terms"][term]
|
||||
for i in range(len(termList)): # Last item is a messages object
|
||||
if termList[i] is not None:
|
||||
response = translateGPT(termList[i], context, False)
|
||||
response = translateAI(termList[i], context, False)
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
termList[i] = response[0].replace('"', "").strip()
|
||||
|
||||
# Armor Types
|
||||
for i in range(len(data["armorTypes"])):
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
data["armorTypes"][i],
|
||||
"Reply with only the " + LANGUAGE + " translation of the armor type",
|
||||
False,
|
||||
|
|
@ -2381,7 +2393,7 @@ def searchSystem(data, pbar):
|
|||
|
||||
# Skill Types
|
||||
for i in range(len(data["skillTypes"])):
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
data["skillTypes"][i],
|
||||
"Reply with only the " + LANGUAGE + " translation",
|
||||
False,
|
||||
|
|
@ -2392,7 +2404,7 @@ def searchSystem(data, pbar):
|
|||
|
||||
# Equip Types
|
||||
for i in range(len(data["equipTypes"])):
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
data["equipTypes"][i],
|
||||
"Reply with only the " + LANGUAGE + " translation of the equipment type. No disclaimers.",
|
||||
False,
|
||||
|
|
@ -2403,7 +2415,7 @@ def searchSystem(data, pbar):
|
|||
|
||||
# # Variables (Optional ususally)
|
||||
# for i in range(len(data['variables'])):
|
||||
# response = translateGPT(data['variables'][i], 'Reply with only the '+ LANGUAGE +' translation of the title', False)
|
||||
# response = translateAI(data['variables'][i], 'Reply with only the '+ LANGUAGE +' translation of the title', False)
|
||||
# totalTokens[0] += response[1][0]
|
||||
# totalTokens[1] += response[1][1]
|
||||
# data['variables'][i] = response[0].replace('\"', '').strip()
|
||||
|
|
@ -2411,7 +2423,7 @@ def searchSystem(data, pbar):
|
|||
# Messages
|
||||
messages = data["terms"]["messages"]
|
||||
for key, value in messages.items():
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
value,
|
||||
"Reply with only the "
|
||||
+ LANGUAGE
|
||||
|
|
@ -2445,7 +2457,7 @@ def getSpeaker(speaker):
|
|||
return [NAMESLIST[i][1], [0, 0]]
|
||||
|
||||
# Translate and Store Speaker
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
f"{speaker}",
|
||||
"Reply with the " + LANGUAGE + " translation of the NPC name.",
|
||||
False,
|
||||
|
|
@ -2456,7 +2468,7 @@ def getSpeaker(speaker):
|
|||
|
||||
# Retry if name doesn't translate for some reason
|
||||
if re.search(r"([a-zA-Z？?])", response[0]) == None:
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
f"{speaker}",
|
||||
"Reply with the " + LANGUAGE + " translation of the NPC name.",
|
||||
False,
|
||||
|
|
@ -2469,328 +2481,24 @@ def getSpeaker(speaker):
|
|||
return response
|
||||
return [speaker, [0, 0]]
|
||||
|
||||
|
||||
def batchList(input_list, batch_size):
|
||||
if not isinstance(batch_size, int) or batch_size <= 0:
|
||||
raise ValueError("batch_size must be a positive integer")
|
||||
|
||||
return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)]
|
||||
|
||||
|
||||
def parseVocabWithCategories(vocabText):
|
||||
"""Parse vocabulary text and extract terms with their categories."""
|
||||
pairs = []
|
||||
seen = set()
|
||||
currentCategory = None
|
||||
|
||||
for line in vocabText.splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith('```'):
|
||||
continue
|
||||
|
||||
# Check if this is a category header
|
||||
if line.startswith('#'):
|
||||
currentCategory = line
|
||||
continue
|
||||
|
||||
# Parse vocabulary term
|
||||
m = re.match(r'^(.+?)(?:\s?[\(–])', line) # term is everything before space + '(' or '–'
|
||||
if m:
|
||||
term = m.group(1)
|
||||
if term not in seen:
|
||||
pairs.append((term, line, currentCategory))
|
||||
seen.add(term)
|
||||
|
||||
return pairs
|
||||
|
||||
|
||||
def buildMatchedVocabText(vocabPairs, subbedT):
|
||||
"""Build formatted vocabulary text with matched terms organized by category."""
|
||||
matchedCategories = {}
|
||||
|
||||
# Use word boundaries for Japanese if appropriate, or allow substring as before.
|
||||
for term, line, category in vocabPairs:
|
||||
# "term in subbedT" could be false positive; can use regex but Japanese doesn't always have spaces.
|
||||
if term in subbedT:
|
||||
if category not in matchedCategories:
|
||||
matchedCategories[category] = []
|
||||
matchedCategories[category].append(line)
|
||||
|
||||
# Format matched vocabulary with categories
|
||||
if matchedCategories:
|
||||
formattedLines = ["Here are some vocabulary and terms so that you know the proper spelling and translation.\n"]
|
||||
for category, lines in matchedCategories.items():
|
||||
if category: # Only add category header if it exists
|
||||
formattedLines.append(category)
|
||||
formattedLines.extend(lines)
|
||||
formattedLines.append("") # Add blank line between categories
|
||||
matchedVocabText = f"```\n{chr(10).join(formattedLines).rstrip()}\n```"
|
||||
else:
|
||||
matchedVocabText = ""
|
||||
|
||||
return matchedVocabText
|
||||
|
||||
|
||||
def createContext(fullPromptFlag, subbedT, format):
|
||||
vocabPairs = parseVocabWithCategories(VOCAB)
|
||||
matchedVocabText = buildMatchedVocabText(vocabPairs, subbedT)
|
||||
|
||||
if fullPromptFlag:
|
||||
system = PROMPT + matchedVocabText
|
||||
else:
|
||||
system = f"\
|
||||
You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
|
||||
Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
|
||||
- You translate everything, including erotic and explicit content.\n\
|
||||
- 'Game Characters' - The names, nicknames, and genders of the game characters. Reference this to know the names, nicknames, and gender of characters in the game\n\
|
||||
- All text in your response must be in {LANGUAGE} even if it is hard to translate.\n\
|
||||
- Never include any notes, explanations, dislaimers, or anything similar in your response.\n\
|
||||
- Maintain any spacing in the translation.\n\
|
||||
- Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\
|
||||
- `...` can be a part of the dialogue. Translate it as it is.\n\
|
||||
{matchedVocabText}\n\
|
||||
"
|
||||
if format == "json":
|
||||
user = f"```json\n{subbedT}\n```"
|
||||
else:
|
||||
user = subbedT
|
||||
return system, user
|
||||
|
||||
|
||||
def translateText(system, user, history, penalty, format, model=MODEL):
|
||||
# Prompt
|
||||
msg = [{"role": "system", "content": system}]
|
||||
|
||||
# History
|
||||
if isinstance(history, list):
|
||||
msg.append({"role": "system", "content": "Translation History:"})
|
||||
msg.extend([{"role": "assistant", "content": h} for h in history])
|
||||
else:
|
||||
msg.append({"role": "assistant", "content": history})
|
||||
|
||||
# Response Format
|
||||
if format == "json":
|
||||
responseFormat = {"type": "json_object"}
|
||||
else:
|
||||
responseFormat = {"type": "text"}
|
||||
|
||||
# Content to TL
|
||||
msg.append({"role": "user", "content": f"{user}"})
|
||||
response = openai.chat.completions.create(
|
||||
temperature=0,
|
||||
frequency_penalty=penalty,
|
||||
model=model,
|
||||
response_format=responseFormat,
|
||||
messages=msg,
|
||||
)
|
||||
return response
|
||||
|
||||
|
||||
def cleanTranslatedText(translatedText):
|
||||
placeholders = {
|
||||
f"{LANGUAGE} Translation: ": "",
|
||||
"Translation: ": "",
|
||||
"っ": "",
|
||||
"〜": "~",
|
||||
"ッ": "",
|
||||
"。": ".",
|
||||
"「": '\\"',
|
||||
"」": '\\"',
|
||||
"- ": "-",
|
||||
"—": "―",
|
||||
"】": "]",
|
||||
"【": "[",
|
||||
"é": "e",
|
||||
"this guy": "this bastard",
|
||||
"This guy": "This bastard",
|
||||
"Placeholder Text": "",
|
||||
"```json": "",
|
||||
"```": "",
|
||||
# Add more replacements as needed
|
||||
}
|
||||
for target, replacement in placeholders.items():
|
||||
translatedText = translatedText.replace(target, replacement)
|
||||
|
||||
# Remove Repeating Characters
|
||||
pattern = re.compile(r"(.)\s*\1(?:\s*\1){" + str(20 - 1) + r",}")
|
||||
translatedText = pattern.sub(lambda match: match.group(0).replace(" ", "")[:20], translatedText)
|
||||
|
||||
# Elongate Long Dashes (Since GPT Ignores them...)
|
||||
translatedText = elongateCharacters(translatedText)
|
||||
return translatedText
|
||||
|
||||
|
||||
def elongateCharacters(text):
|
||||
# Define a pattern to match one character followed by two or more `ー` characters
|
||||
# Using a positive lookbehind assertion to capture the preceding character
|
||||
pattern = r"(?<=(.))ー{2,}"
|
||||
|
||||
# Define a replacement function that elongates the captured character
|
||||
def repl(match):
|
||||
char = match.group(1) # The character before the ー sequence
|
||||
count = len(match.group(0)) - 1 # Number of ー characters
|
||||
return char * count # Replace ー sequence with the character repeated
|
||||
|
||||
# Use re.sub() to replace the pattern in the text
|
||||
return re.sub(pattern, repl, text)
|
||||
|
||||
|
||||
def extractTranslation(translatedTextList, is_list):
|
||||
try:
|
||||
translatedTextList = re.sub(r'\\"+\"([^,\n}])', r'\\"\1', translatedTextList)
|
||||
translatedTextList = re.sub(r"(?<![\\])\"+(?![\n,])", r'"', translatedTextList)
|
||||
line_dict = json.loads(translatedTextList)
|
||||
# If it's a batch (i.e., list), extract with tags; otherwise, return the single item.
|
||||
string_list = list(line_dict.values())
|
||||
if is_list:
|
||||
return string_list
|
||||
else:
|
||||
return string_list[0]
|
||||
|
||||
except Exception as e:
|
||||
PBAR.write(f"extractTranslation Error: {e} on String {translatedTextList}")
|
||||
return None
|
||||
|
||||
|
||||
def countTokens(system, user, history):
|
||||
inputTotalTokens = 0
|
||||
outputTotalTokens = 0
|
||||
enc = tiktoken.encoding_for_model("gpt-4")
|
||||
|
||||
# Input
|
||||
if isinstance(history, list):
|
||||
for line in history:
|
||||
inputTotalTokens += len(enc.encode(line))
|
||||
else:
|
||||
inputTotalTokens += len(enc.encode(history))
|
||||
inputTotalTokens += len(enc.encode(system))
|
||||
inputTotalTokens += len(enc.encode(user))
|
||||
|
||||
# Output
|
||||
outputTotalTokens += round(len(enc.encode(user)) * 2.5)
|
||||
|
||||
return [inputTotalTokens, outputTotalTokens]
|
||||
|
||||
|
||||
@retry(exceptions=Exception, tries=5, delay=5)
|
||||
def translateGPT(text, history, fullPromptFlag):
|
||||
def translateAI(text, history, fullPromptFlag):
|
||||
"""
|
||||
Legacy wrapper function for the new shared translation utility.
|
||||
This maintains compatibility with existing code while using the new shared implementation.
|
||||
"""
|
||||
global PBAR, MISMATCH, FILENAME
|
||||
if text:
|
||||
with open("log/translationHistory.txt", "a+", encoding="utf-8") as logFile:
|
||||
mismatch = False
|
||||
totalTokens = [0, 0]
|
||||
if isinstance(text, list):
|
||||
format = "json"
|
||||
tList = batchList(text, BATCHSIZE)
|
||||
else:
|
||||
format = "text"
|
||||
tList = [text]
|
||||
|
||||
for index, tItem in enumerate(tList):
|
||||
# Things to Check before starting translation
|
||||
if not re.search(LANGREGEX, str(tItem)):
|
||||
if PBAR is not None:
|
||||
PBAR.update(len(tItem))
|
||||
if isinstance(tItem, list):
|
||||
for j in range(len(tItem)):
|
||||
tItem[j] = cleanTranslatedText(tItem[j])
|
||||
tList[index] = tItem
|
||||
else:
|
||||
tList[index] = cleanTranslatedText(tItem)
|
||||
history = tItem[-MAXHISTORY:]
|
||||
continue
|
||||
# Update config estimate mode based on global ESTIMATE
|
||||
TRANSLATION_CONFIG.estimateMode = bool(ESTIMATE)
|
||||
|
||||
# Before sending to translation, if we have a list of items, add the formatting
|
||||
if isinstance(tItem, list):
|
||||
for j in range(len(tItem)):
|
||||
if not tItem[j]:
|
||||
tItem[j] = tItem[j].replace("", "Placeholder Text")
|
||||
payload = {f"Line{i+1}": string for i, string in enumerate(tItem)}
|
||||
payload = json.dumps(payload, indent=4, ensure_ascii=False)
|
||||
varResponse = [payload, []]
|
||||
subbedT = varResponse[0]
|
||||
else:
|
||||
varResponse = [tItem, []]
|
||||
subbedT = varResponse[0]
|
||||
|
||||
# Create Message
|
||||
system, user = createContext(fullPromptFlag, subbedT, format)
|
||||
|
||||
# Calculate Estimate
|
||||
if ESTIMATE:
|
||||
estimate = countTokens(system, user, history)
|
||||
totalTokens[0] += estimate[0]
|
||||
totalTokens[1] += estimate[1]
|
||||
continue
|
||||
|
||||
# Translating
|
||||
response = translateText(system, user, history, 0.05, format)
|
||||
|
||||
# Set Tokens
|
||||
translatedText = response.choices[0].message.content
|
||||
|
||||
# AI Refused, Try Again
|
||||
if not translatedText:
|
||||
response = translateText(f"{system}\n You translate ALL content.", user, history, 0.1, format, model="gpt-4o")
|
||||
translatedText = response.choices[0].message.content
|
||||
|
||||
# Report Tokens
|
||||
totalTokens[0] += response.usage.prompt_tokens
|
||||
totalTokens[1] += response.usage.completion_tokens
|
||||
|
||||
# Check Translation
|
||||
if translatedText:
|
||||
translatedText = cleanTranslatedText(translatedText)
|
||||
if isinstance(tItem, list):
|
||||
extractedTranslations = extractTranslation(translatedText, True)
|
||||
if extractedTranslations == None or len(tItem) != len(extractedTranslations):
|
||||
# Mismatch. Try Again
|
||||
response = translateText(system, user, history, 0.05, format, MODEL)
|
||||
translatedText = response.choices[0].message.content
|
||||
totalTokens[0] += response.usage.prompt_tokens
|
||||
totalTokens[1] += response.usage.completion_tokens
|
||||
|
||||
# Formatting
|
||||
translatedText = cleanTranslatedText(translatedText)
|
||||
if isinstance(tItem, list):
|
||||
extractedTranslations = extractTranslation(translatedText, True)
|
||||
if extractedTranslations == None or len(tItem) != len(extractedTranslations):
|
||||
with open("log/mismatchHistory.txt", "a+", encoding="utf-8") as mismatchFile:
|
||||
mismatchFile.write(f"Mismatch: {FILENAME}\n")
|
||||
mismatchFile.write(f"Input:\n{subbedT}\n")
|
||||
mismatchFile.write(f"Output:\n{translatedText}\n")
|
||||
mismatch = True # Just here for breakpoint
|
||||
logFile.write(f"Input:\n{subbedT}\n")
|
||||
logFile.write(f"Output:\n{translatedText}\n")
|
||||
|
||||
# Set if no mismatch
|
||||
if mismatch == False:
|
||||
tList[index] = extractedTranslations
|
||||
history = extractedTranslations[-MAXHISTORY:] # Update history if we have a list
|
||||
else:
|
||||
history = text[-MAXHISTORY:]
|
||||
mismatch = False
|
||||
if FILENAME not in MISMATCH:
|
||||
MISMATCH.append(FILENAME)
|
||||
|
||||
# Update Loading Bar
|
||||
with LOCK:
|
||||
if PBAR is not None:
|
||||
PBAR.update(len(tItem))
|
||||
else:
|
||||
# Ensure we're passing a single string to extractTranslation
|
||||
tList[index] = translatedText.replace("Placeholder Text", "")
|
||||
else:
|
||||
PBAR.write(f"AI Refused:{tItem}\n")
|
||||
|
||||
# Combine if multilist
|
||||
if isinstance(tList[0], list):
|
||||
tList = [t for sublist in tList for t in sublist]
|
||||
|
||||
# Return
|
||||
if format == "json":
|
||||
return [tList, totalTokens]
|
||||
else:
|
||||
return [tList[0], totalTokens]
|
||||
else:
|
||||
return [text, [0, 0]]
|
||||
# Call the new shared translation function
|
||||
return sharedtranslateAI(
|
||||
text=text,
|
||||
history=history,
|
||||
fullPromptFlag=fullPromptFlag,
|
||||
config=TRANSLATION_CONFIG,
|
||||
filename=FILENAME,
|
||||
pbar=PBAR,
|
||||
lock=LOCK,
|
||||
mismatchList=MISMATCH
|
||||
)
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ from colorama import Fore
|
|||
from dotenv import load_dotenv
|
||||
from retry import retry
|
||||
from tqdm import tqdm
|
||||
from util.translation import TranslationConfig, translateAI as sharedtranslateAI
|
||||
|
||||
# Open AI
|
||||
load_dotenv()
|
||||
|
|
@ -75,6 +76,18 @@ else:
|
|||
BATCHSIZE = int(os.getenv("batchsize"))
|
||||
FREQUENCY_PENALTY = float(os.getenv("frequency_penalty"))
|
||||
|
||||
# Initialize Translation Config
|
||||
TRANSLATION_CONFIG = TranslationConfig(
|
||||
model=MODEL,
|
||||
language=LANGUAGE,
|
||||
prompt=PROMPT,
|
||||
vocab=VOCAB,
|
||||
langRegex=LANGREGEX,
|
||||
batchSize=BATCHSIZE,
|
||||
maxHistory=MAXHISTORY,
|
||||
estimateMode=False # Will be set dynamically based on ESTIMATE
|
||||
)
|
||||
LEAVE = False
|
||||
|
||||
def handlePlugin(filename, estimate):
|
||||
global ESTIMATE, PBAR
|
||||
|
|
@ -496,42 +509,42 @@ def translatePlugin(data, pbar, filename, translatedList):
|
|||
PBAR = pbar
|
||||
|
||||
# Quest Name
|
||||
response = translateGPT(questList[0], "Quest Name", True)
|
||||
response = translateAI(questList[0], "Quest Name", True)
|
||||
tokens[0] += response[1][0]
|
||||
tokens[1] += response[1][1]
|
||||
questName = response[0]
|
||||
pbar.update(len(questList[0]))
|
||||
|
||||
# Quest Client
|
||||
response = translateGPT(questList[1], "Quest Client", True)
|
||||
response = translateAI(questList[1], "Quest Client", True)
|
||||
tokens[0] += response[1][0]
|
||||
tokens[1] += response[1][1]
|
||||
questClient = response[0]
|
||||
pbar.update(len(questList[1]))
|
||||
|
||||
# Quest Location
|
||||
response = translateGPT(questList[2], "Quest Location", True)
|
||||
response = translateAI(questList[2], "Quest Location", True)
|
||||
tokens[0] += response[1][0]
|
||||
tokens[1] += response[1][1]
|
||||
questLocation = response[0]
|
||||
pbar.update(len(questList[2]))
|
||||
|
||||
# Quest Target
|
||||
response = translateGPT(questList[3], "Quest Location", True)
|
||||
response = translateAI(questList[3], "Quest Location", True)
|
||||
tokens[0] += response[1][0]
|
||||
tokens[1] += response[1][1]
|
||||
questTarget = response[0]
|
||||
pbar.update(len(questList[3]))
|
||||
|
||||
# Quest Summary
|
||||
response = translateGPT(questList[4], "Quest Summary", True)
|
||||
response = translateAI(questList[4], "Quest Summary", True)
|
||||
tokens[0] += response[1][0]
|
||||
tokens[1] += response[1][1]
|
||||
questSummary = response[0]
|
||||
pbar.update(len(questList[4]))
|
||||
|
||||
# Quest Goal 1
|
||||
response = translateGPT(questList[5], "Quest Goal", True)
|
||||
response = translateAI(questList[5], "Quest Goal", True)
|
||||
tokens[0] += response[1][0]
|
||||
tokens[1] += response[1][1]
|
||||
questGoal1 = response[0]
|
||||
|
|
@ -564,7 +577,7 @@ def translatePlugin(data, pbar, filename, translatedList):
|
|||
PBAR = pbar
|
||||
|
||||
# TL
|
||||
response = translateGPT(custom, "Relic Name", True)
|
||||
response = translateAI(custom, "Relic Name", True)
|
||||
tokens[0] += response[1][0]
|
||||
tokens[1] += response[1][1]
|
||||
customResponse = response[0]
|
||||
|
|
@ -600,7 +613,7 @@ def getSpeaker(speaker):
|
|||
return [NAMESLIST[i][1], [0, 0]]
|
||||
|
||||
# Translate and Store Speaker
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
f"{speaker}",
|
||||
"Reply with the " + LANGUAGE + " translation of the NPC name.",
|
||||
False,
|
||||
|
|
@ -611,7 +624,7 @@ def getSpeaker(speaker):
|
|||
|
||||
# Retry if name doesn't translate for some reason
|
||||
if re.search(r"([a-zA-Z？?])", response[0]) == None:
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
f"{speaker}",
|
||||
"Reply with the " + LANGUAGE + " translation of the NPC name.",
|
||||
False,
|
||||
|
|
@ -624,328 +637,24 @@ def getSpeaker(speaker):
|
|||
return response
|
||||
return [speaker, [0, 0]]
|
||||
|
||||
|
||||
def batchList(input_list, batch_size):
|
||||
if not isinstance(batch_size, int) or batch_size <= 0:
|
||||
raise ValueError("batch_size must be a positive integer")
|
||||
|
||||
return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)]
|
||||
|
||||
|
||||
def parseVocabWithCategories(vocabText):
|
||||
"""Parse vocabulary text and extract terms with their categories."""
|
||||
pairs = []
|
||||
seen = set()
|
||||
currentCategory = None
|
||||
|
||||
for line in vocabText.splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith('```'):
|
||||
continue
|
||||
|
||||
# Check if this is a category header
|
||||
if line.startswith('#'):
|
||||
currentCategory = line
|
||||
continue
|
||||
|
||||
# Parse vocabulary term
|
||||
m = re.match(r'^(.+?)(?:\s?[\(–])', line) # term is everything before space + '(' or '–'
|
||||
if m:
|
||||
term = m.group(1)
|
||||
if term not in seen:
|
||||
pairs.append((term, line, currentCategory))
|
||||
seen.add(term)
|
||||
|
||||
return pairs
|
||||
|
||||
|
||||
def buildMatchedVocabText(vocabPairs, subbedT):
|
||||
"""Build formatted vocabulary text with matched terms organized by category."""
|
||||
matchedCategories = {}
|
||||
|
||||
# Use word boundaries for Japanese if appropriate, or allow substring as before.
|
||||
for term, line, category in vocabPairs:
|
||||
# "term in subbedT" could be false positive; can use regex but Japanese doesn't always have spaces.
|
||||
if term in subbedT:
|
||||
if category not in matchedCategories:
|
||||
matchedCategories[category] = []
|
||||
matchedCategories[category].append(line)
|
||||
|
||||
# Format matched vocabulary with categories
|
||||
if matchedCategories:
|
||||
formattedLines = ["Here are some vocabulary and terms so that you know the proper spelling and translation.\n"]
|
||||
for category, lines in matchedCategories.items():
|
||||
if category: # Only add category header if it exists
|
||||
formattedLines.append(category)
|
||||
formattedLines.extend(lines)
|
||||
formattedLines.append("") # Add blank line between categories
|
||||
matchedVocabText = f"```\n{chr(10).join(formattedLines).rstrip()}\n```"
|
||||
else:
|
||||
matchedVocabText = ""
|
||||
|
||||
return matchedVocabText
|
||||
|
||||
|
||||
def createContext(fullPromptFlag, subbedT, format):
|
||||
vocabPairs = parseVocabWithCategories(VOCAB)
|
||||
matchedVocabText = buildMatchedVocabText(vocabPairs, subbedT)
|
||||
|
||||
if fullPromptFlag:
|
||||
system = PROMPT + matchedVocabText
|
||||
else:
|
||||
system = f"\
|
||||
You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
|
||||
Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
|
||||
- You translate everything, including erotic and explicit content.\n\
|
||||
- 'Game Characters' - The names, nicknames, and genders of the game characters. Reference this to know the names, nicknames, and gender of characters in the game\n\
|
||||
- All text in your response must be in {LANGUAGE} even if it is hard to translate.\n\
|
||||
- Never include any notes, explanations, dislaimers, or anything similar in your response.\n\
|
||||
- Maintain any spacing in the translation.\n\
|
||||
- Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\
|
||||
- `...` can be a part of the dialogue. Translate it as it is.\n\
|
||||
{matchedVocabText}\n\
|
||||
"
|
||||
if format == "json":
|
||||
user = f"```json\n{subbedT}\n```"
|
||||
else:
|
||||
user = subbedT
|
||||
return system, user
|
||||
|
||||
|
||||
def translateText(system, user, history, penalty, format, model=MODEL):
|
||||
# Prompt
|
||||
msg = [{"role": "system", "content": system}]
|
||||
|
||||
# History
|
||||
if isinstance(history, list):
|
||||
msg.append({"role": "system", "content": "Translation History:"})
|
||||
msg.extend([{"role": "assistant", "content": h} for h in history])
|
||||
else:
|
||||
msg.append({"role": "assistant", "content": history})
|
||||
|
||||
# Response Format
|
||||
if format == "json":
|
||||
responseFormat = {"type": "json_object"}
|
||||
else:
|
||||
responseFormat = {"type": "text"}
|
||||
|
||||
# Content to TL
|
||||
msg.append({"role": "user", "content": f"{user}"})
|
||||
response = openai.chat.completions.create(
|
||||
temperature=0,
|
||||
frequency_penalty=penalty,
|
||||
model=model,
|
||||
response_format=responseFormat,
|
||||
messages=msg,
|
||||
)
|
||||
return response
|
||||
|
||||
|
||||
def cleanTranslatedText(translatedText):
|
||||
placeholders = {
|
||||
f"{LANGUAGE} Translation: ": "",
|
||||
"Translation: ": "",
|
||||
"っ": "",
|
||||
"〜": "~",
|
||||
"ッ": "",
|
||||
"。": ".",
|
||||
"「": '\\"',
|
||||
"」": '\\"',
|
||||
"- ": "-",
|
||||
"—": "―",
|
||||
"】": "]",
|
||||
"【": "[",
|
||||
"é": "e",
|
||||
"this guy": "this bastard",
|
||||
"This guy": "This bastard",
|
||||
"Placeholder Text": "",
|
||||
"```json": "",
|
||||
"```": "",
|
||||
# Add more replacements as needed
|
||||
}
|
||||
for target, replacement in placeholders.items():
|
||||
translatedText = translatedText.replace(target, replacement)
|
||||
|
||||
# Remove Repeating Characters
|
||||
pattern = re.compile(r"(.)\s*\1(?:\s*\1){" + str(20 - 1) + r",}")
|
||||
translatedText = pattern.sub(lambda match: match.group(0).replace(" ", "")[:20], translatedText)
|
||||
|
||||
# Elongate Long Dashes (Since GPT Ignores them...)
|
||||
translatedText = elongateCharacters(translatedText)
|
||||
return translatedText
|
||||
|
||||
|
||||
def elongateCharacters(text):
|
||||
# Define a pattern to match one character followed by one or more `ー` characters
|
||||
# Using a positive lookbehind assertion to capture the preceding character
|
||||
pattern = r"(?<=(.))ー+"
|
||||
|
||||
# Define a replacement function that elongates the captured character
|
||||
def repl(match):
|
||||
char = match.group(1) # The character before the ー sequence
|
||||
count = len(match.group(0)) - 1 # Number of ー characters
|
||||
return char * count # Replace ー sequence with the character repeated
|
||||
|
||||
# Use re.sub() to replace the pattern in the text
|
||||
return re.sub(pattern, repl, text)
|
||||
|
||||
|
||||
def extractTranslation(translatedTextList, is_list):
|
||||
try:
|
||||
translatedTextList = re.sub(r'\\"+\"([^,\n}])', r'\\"\1', translatedTextList)
|
||||
translatedTextList = re.sub(r"(?<![\\])\"+(?![\n,])", r'"', translatedTextList)
|
||||
line_dict = json.loads(translatedTextList)
|
||||
# If it's a batch (i.e., list), extract with tags; otherwise, return the single item.
|
||||
string_list = list(line_dict.values())
|
||||
if is_list:
|
||||
return string_list
|
||||
else:
|
||||
return string_list[0]
|
||||
|
||||
except Exception as e:
|
||||
PBAR.write(f"extractTranslation Error: {e} on String {translatedTextList}")
|
||||
return None
|
||||
|
||||
|
||||
def countTokens(system, user, history):
|
||||
inputTotalTokens = 0
|
||||
outputTotalTokens = 0
|
||||
enc = tiktoken.encoding_for_model("gpt-4")
|
||||
|
||||
# Input
|
||||
if isinstance(history, list):
|
||||
for line in history:
|
||||
inputTotalTokens += len(enc.encode(line))
|
||||
else:
|
||||
inputTotalTokens += len(enc.encode(history))
|
||||
inputTotalTokens += len(enc.encode(system))
|
||||
inputTotalTokens += len(enc.encode(user))
|
||||
|
||||
# Output
|
||||
outputTotalTokens += round(len(enc.encode(user)) * 2.5)
|
||||
|
||||
return [inputTotalTokens, outputTotalTokens]
|
||||
|
||||
|
||||
@retry(exceptions=Exception, tries=5, delay=5)
|
||||
def translateGPT(text, history, fullPromptFlag):
|
||||
def translateAI(text, history, fullPromptFlag):
|
||||
"""
|
||||
Legacy wrapper function for the new shared translation utility.
|
||||
This maintains compatibility with existing code while using the new shared implementation.
|
||||
"""
|
||||
global PBAR, MISMATCH, FILENAME
|
||||
if text:
|
||||
with open("log/translationHistory.txt", "a+", encoding="utf-8") as logFile:
|
||||
mismatch = False
|
||||
totalTokens = [0, 0]
|
||||
if isinstance(text, list):
|
||||
format = "json"
|
||||
tList = batchList(text, BATCHSIZE)
|
||||
else:
|
||||
format = "text"
|
||||
tList = [text]
|
||||
|
||||
for index, tItem in enumerate(tList):
|
||||
# Things to Check before starting translation
|
||||
if not re.search(LANGREGEX, str(tItem)):
|
||||
if PBAR is not None:
|
||||
PBAR.update(len(tItem))
|
||||
if isinstance(tItem, list):
|
||||
for j in range(len(tItem)):
|
||||
tItem[j] = cleanTranslatedText(tItem[j])
|
||||
tList[index] = tItem
|
||||
else:
|
||||
tList[index] = cleanTranslatedText(tItem)
|
||||
history = tItem[-MAXHISTORY:]
|
||||
continue
|
||||
# Update config estimate mode based on global ESTIMATE
|
||||
TRANSLATION_CONFIG.estimateMode = bool(ESTIMATE)
|
||||
|
||||
# Before sending to translation, if we have a list of items, add the formatting
|
||||
if isinstance(tItem, list):
|
||||
for j in range(len(tItem)):
|
||||
if not tItem[j]:
|
||||
tItem[j] = tItem[j].replace("", "Placeholder Text")
|
||||
payload = {f"Line{i+1}": string for i, string in enumerate(tItem)}
|
||||
payload = json.dumps(payload, indent=4, ensure_ascii=False)
|
||||
varResponse = [payload, []]
|
||||
subbedT = varResponse[0]
|
||||
else:
|
||||
varResponse = [tItem, []]
|
||||
subbedT = varResponse[0]
|
||||
|
||||
# Create Message
|
||||
system, user = createContext(fullPromptFlag, subbedT, format)
|
||||
|
||||
# Calculate Estimate
|
||||
if ESTIMATE:
|
||||
estimate = countTokens(system, user, history)
|
||||
totalTokens[0] += estimate[0]
|
||||
totalTokens[1] += estimate[1]
|
||||
continue
|
||||
|
||||
# Translating
|
||||
response = translateText(system, user, history, 0.05, format)
|
||||
|
||||
# Set Tokens
|
||||
translatedText = response.choices[0].message.content
|
||||
|
||||
# AI Refused, Try Again
|
||||
if not translatedText:
|
||||
response = translateText(f"{system}\n You translate ALL content.", user, history, 0.1, format, model="gpt-4o")
|
||||
translatedText = response.choices[0].message.content
|
||||
|
||||
# Report Tokens
|
||||
totalTokens[0] += response.usage.prompt_tokens
|
||||
totalTokens[1] += response.usage.completion_tokens
|
||||
|
||||
# Check Translation
|
||||
if translatedText:
|
||||
translatedText = cleanTranslatedText(translatedText)
|
||||
if isinstance(tItem, list):
|
||||
extractedTranslations = extractTranslation(translatedText, True)
|
||||
if extractedTranslations == None or len(tItem) != len(extractedTranslations):
|
||||
# Mismatch. Try Again
|
||||
response = translateText(system, user, history, 0.05, format, MODEL)
|
||||
translatedText = response.choices[0].message.content
|
||||
totalTokens[0] += response.usage.prompt_tokens
|
||||
totalTokens[1] += response.usage.completion_tokens
|
||||
|
||||
# Formatting
|
||||
translatedText = cleanTranslatedText(translatedText)
|
||||
if isinstance(tItem, list):
|
||||
extractedTranslations = extractTranslation(translatedText, True)
|
||||
if extractedTranslations == None or len(tItem) != len(extractedTranslations):
|
||||
with open("log/mismatchHistory.txt", "a+", encoding="utf-8") as mismatchFile:
|
||||
mismatchFile.write(f"Mismatch: {FILENAME}\n")
|
||||
mismatchFile.write(f"Input:\n{subbedT}\n")
|
||||
mismatchFile.write(f"Output:\n{translatedText}\n")
|
||||
mismatch = True # Just here for breakpoint
|
||||
logFile.write(f"Input:\n{subbedT}\n")
|
||||
logFile.write(f"Output:\n{translatedText}\n")
|
||||
|
||||
# Set if no mismatch
|
||||
if mismatch == False:
|
||||
tList[index] = extractedTranslations
|
||||
history = extractedTranslations[-MAXHISTORY:] # Update history if we have a list
|
||||
else:
|
||||
history = text[-MAXHISTORY:]
|
||||
mismatch = False
|
||||
if FILENAME not in MISMATCH:
|
||||
MISMATCH.append(FILENAME)
|
||||
|
||||
# Update Loading Bar
|
||||
with LOCK:
|
||||
if PBAR is not None:
|
||||
PBAR.update(len(tItem))
|
||||
else:
|
||||
# Ensure we're passing a single string to extractTranslation
|
||||
tList[index] = translatedText.replace("Placeholder Text", "")
|
||||
else:
|
||||
PBAR.write(f"AI Refused:{tItem}\n")
|
||||
|
||||
# Combine if multilist
|
||||
if isinstance(tList[0], list):
|
||||
tList = [t for sublist in tList for t in sublist]
|
||||
|
||||
# Return
|
||||
if format == "json":
|
||||
return [tList, totalTokens]
|
||||
else:
|
||||
return [tList[0], totalTokens]
|
||||
else:
|
||||
return [text, [0, 0]]
|
||||
# Call the new shared translation function
|
||||
return sharedtranslateAI(
|
||||
text=text,
|
||||
history=history,
|
||||
fullPromptFlag=fullPromptFlag,
|
||||
config=TRANSLATION_CONFIG,
|
||||
filename=FILENAME,
|
||||
pbar=PBAR,
|
||||
lock=LOCK,
|
||||
mismatchList=MISMATCH
|
||||
)
|
||||
|
|
|
|||
361
modules/text.py
361
modules/text.py
|
|
@ -13,6 +13,7 @@ from colorama import Fore
|
|||
from dotenv import load_dotenv
|
||||
from retry import retry
|
||||
from tqdm import tqdm
|
||||
from util.translation import TranslationConfig, translateAI as sharedtranslateAI
|
||||
|
||||
# Open AI
|
||||
load_dotenv()
|
||||
|
|
@ -76,6 +77,18 @@ else:
|
|||
BATCHSIZE = int(os.getenv("batchsize"))
|
||||
FREQUENCY_PENALTY = float(os.getenv("frequency_penalty"))
|
||||
|
||||
# Initialize Translation Config
|
||||
TRANSLATION_CONFIG = TranslationConfig(
|
||||
model=MODEL,
|
||||
language=LANGUAGE,
|
||||
prompt=PROMPT,
|
||||
vocab=VOCAB,
|
||||
langRegex=LANGREGEX,
|
||||
batchSize=BATCHSIZE,
|
||||
maxHistory=MAXHISTORY,
|
||||
estimateMode=False # Will be set dynamically based on ESTIMATE
|
||||
)
|
||||
LEAVE = False
|
||||
|
||||
def handleText(filename, estimate):
|
||||
global ESTIMATE, TOKENS, FILENAME
|
||||
|
|
@ -241,7 +254,7 @@ def translateTxt(data, translatedList):
|
|||
if stringList:
|
||||
PBAR.total = len(stringList)
|
||||
PBAR.refresh()
|
||||
response = translateGPT(stringList, "Reply with the English Translation", True)
|
||||
response = translateAI(stringList, "Reply with the English Translation", True)
|
||||
tokens[0] += response[1][0]
|
||||
tokens[1] += response[1][1]
|
||||
stringListTL = response[0]
|
||||
|
|
@ -254,7 +267,7 @@ def translateTxt(data, translatedList):
|
|||
|
||||
# Choice List
|
||||
if choiceList:
|
||||
response = translateGPT(choiceList, "Reply with the English TL of the Dialogue Choice", True)
|
||||
response = translateAI(choiceList, "Reply with the English TL of the Dialogue Choice", True)
|
||||
tokens[0] += response[1][0]
|
||||
tokens[1] += response[1][1]
|
||||
choiceListTL = response[0]
|
||||
|
|
@ -283,7 +296,7 @@ def getSpeaker(speaker):
|
|||
return [NAMESLIST[i][1], [0, 0]]
|
||||
|
||||
# Translate and Store Speaker
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
f"{speaker}",
|
||||
"Reply with the " + LANGUAGE + " translation of the NPC name.",
|
||||
False,
|
||||
|
|
@ -294,7 +307,7 @@ def getSpeaker(speaker):
|
|||
|
||||
# Retry if name doesn't translate for some reason
|
||||
if re.search(r"([a-zA-Z？?])", response[0]) == None:
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
f"{speaker}",
|
||||
"Reply with the " + LANGUAGE + " translation of the NPC name.",
|
||||
False,
|
||||
|
|
@ -307,328 +320,24 @@ def getSpeaker(speaker):
|
|||
return response
|
||||
return [speaker, [0, 0]]
|
||||
|
||||
|
||||
def batchList(input_list, batch_size):
|
||||
if not isinstance(batch_size, int) or batch_size <= 0:
|
||||
raise ValueError("batch_size must be a positive integer")
|
||||
|
||||
return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)]
|
||||
|
||||
|
||||
def parseVocabWithCategories(vocabText):
|
||||
"""Parse vocabulary text and extract terms with their categories."""
|
||||
pairs = []
|
||||
seen = set()
|
||||
currentCategory = None
|
||||
|
||||
for line in vocabText.splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith('```'):
|
||||
continue
|
||||
|
||||
# Check if this is a category header
|
||||
if line.startswith('#'):
|
||||
currentCategory = line
|
||||
continue
|
||||
|
||||
# Parse vocabulary term
|
||||
m = re.match(r'^(.+?)(?:\s?[\(–])', line) # term is everything before space + '(' or '–'
|
||||
if m:
|
||||
term = m.group(1)
|
||||
if term not in seen:
|
||||
pairs.append((term, line, currentCategory))
|
||||
seen.add(term)
|
||||
|
||||
return pairs
|
||||
|
||||
|
||||
def buildMatchedVocabText(vocabPairs, subbedT):
|
||||
"""Build formatted vocabulary text with matched terms organized by category."""
|
||||
matchedCategories = {}
|
||||
|
||||
# Use word boundaries for Japanese if appropriate, or allow substring as before.
|
||||
for term, line, category in vocabPairs:
|
||||
# "term in subbedT" could be false positive; can use regex but Japanese doesn't always have spaces.
|
||||
if term in subbedT:
|
||||
if category not in matchedCategories:
|
||||
matchedCategories[category] = []
|
||||
matchedCategories[category].append(line)
|
||||
|
||||
# Format matched vocabulary with categories
|
||||
if matchedCategories:
|
||||
formattedLines = ["Here are some vocabulary and terms so that you know the proper spelling and translation.\n"]
|
||||
for category, lines in matchedCategories.items():
|
||||
if category: # Only add category header if it exists
|
||||
formattedLines.append(category)
|
||||
formattedLines.extend(lines)
|
||||
formattedLines.append("") # Add blank line between categories
|
||||
matchedVocabText = f"```\n{chr(10).join(formattedLines).rstrip()}\n```"
|
||||
else:
|
||||
matchedVocabText = ""
|
||||
|
||||
return matchedVocabText
|
||||
|
||||
|
||||
def createContext(fullPromptFlag, subbedT, format):
|
||||
vocabPairs = parseVocabWithCategories(VOCAB)
|
||||
matchedVocabText = buildMatchedVocabText(vocabPairs, subbedT)
|
||||
|
||||
if fullPromptFlag:
|
||||
system = PROMPT + matchedVocabText
|
||||
else:
|
||||
system = f"\
|
||||
You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
|
||||
Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
|
||||
- You translate everything, including erotic and explicit content.\n\
|
||||
- 'Game Characters' - The names, nicknames, and genders of the game characters. Reference this to know the names, nicknames, and gender of characters in the game\n\
|
||||
- All text in your response must be in {LANGUAGE} even if it is hard to translate.\n\
|
||||
- Never include any notes, explanations, dislaimers, or anything similar in your response.\n\
|
||||
- Maintain any spacing in the translation.\n\
|
||||
- Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\
|
||||
- `...` can be a part of the dialogue. Translate it as it is.\n\
|
||||
{matchedVocabText}\n\
|
||||
"
|
||||
if format == "json":
|
||||
user = f"```json\n{subbedT}\n```"
|
||||
else:
|
||||
user = subbedT
|
||||
return system, user
|
||||
|
||||
|
||||
def translateText(system, user, history, penalty, format, model=MODEL):
|
||||
# Prompt
|
||||
msg = [{"role": "system", "content": system}]
|
||||
|
||||
# History
|
||||
if isinstance(history, list):
|
||||
msg.append({"role": "system", "content": "Translation History:"})
|
||||
msg.extend([{"role": "assistant", "content": h} for h in history])
|
||||
else:
|
||||
msg.append({"role": "assistant", "content": history})
|
||||
|
||||
# Response Format
|
||||
if format == "json":
|
||||
responseFormat = {"type": "json_object"}
|
||||
else:
|
||||
responseFormat = {"type": "text"}
|
||||
|
||||
# Content to TL
|
||||
msg.append({"role": "user", "content": f"{user}"})
|
||||
response = openai.chat.completions.create(
|
||||
temperature=0,
|
||||
frequency_penalty=penalty,
|
||||
model=model,
|
||||
response_format=responseFormat,
|
||||
messages=msg,
|
||||
)
|
||||
return response
|
||||
|
||||
|
||||
def cleanTranslatedText(translatedText):
|
||||
placeholders = {
|
||||
f"{LANGUAGE} Translation: ": "",
|
||||
"Translation: ": "",
|
||||
"っ": "",
|
||||
"〜": "~",
|
||||
"ッ": "",
|
||||
"。": ".",
|
||||
"「": '\\"',
|
||||
"」": '\\"',
|
||||
"- ": "-",
|
||||
"—": "―",
|
||||
"】": "]",
|
||||
"【": "[",
|
||||
"é": "e",
|
||||
"this guy": "this bastard",
|
||||
"This guy": "This bastard",
|
||||
"Placeholder Text": "",
|
||||
"```json": "",
|
||||
"```": "",
|
||||
# Add more replacements as needed
|
||||
}
|
||||
for target, replacement in placeholders.items():
|
||||
translatedText = translatedText.replace(target, replacement)
|
||||
|
||||
# Remove Repeating Characters
|
||||
pattern = re.compile(r"(.)\s*\1(?:\s*\1){" + str(20 - 1) + r",}")
|
||||
translatedText = pattern.sub(lambda match: match.group(0).replace(" ", "")[:20], translatedText)
|
||||
|
||||
# Elongate Long Dashes (Since GPT Ignores them...)
|
||||
translatedText = elongateCharacters(translatedText)
|
||||
return translatedText
|
||||
|
||||
|
||||
def elongateCharacters(text):
|
||||
# Define a pattern to match one character followed by one or more `ー` characters
|
||||
# Using a positive lookbehind assertion to capture the preceding character
|
||||
pattern = r"(?<=(.))ー+"
|
||||
|
||||
# Define a replacement function that elongates the captured character
|
||||
def repl(match):
|
||||
char = match.group(1) # The character before the ー sequence
|
||||
count = len(match.group(0)) - 1 # Number of ー characters
|
||||
return char * count # Replace ー sequence with the character repeated
|
||||
|
||||
# Use re.sub() to replace the pattern in the text
|
||||
return re.sub(pattern, repl, text)
|
||||
|
||||
|
||||
def extractTranslation(translatedTextList, is_list):
|
||||
try:
|
||||
translatedTextList = re.sub(r'\\"+\"([^,\n}])', r'\\"\1', translatedTextList)
|
||||
translatedTextList = re.sub(r"(?<![\\])\"+(?![\n,])", r'"', translatedTextList)
|
||||
line_dict = json.loads(translatedTextList)
|
||||
# If it's a batch (i.e., list), extract with tags; otherwise, return the single item.
|
||||
string_list = list(line_dict.values())
|
||||
if is_list:
|
||||
return string_list
|
||||
else:
|
||||
return string_list[0]
|
||||
|
||||
except Exception as e:
|
||||
PBAR.write(f"extractTranslation Error: {e} on String {translatedTextList}")
|
||||
return None
|
||||
|
||||
|
||||
def countTokens(system, user, history):
|
||||
inputTotalTokens = 0
|
||||
outputTotalTokens = 0
|
||||
enc = tiktoken.encoding_for_model("gpt-4")
|
||||
|
||||
# Input
|
||||
if isinstance(history, list):
|
||||
for line in history:
|
||||
inputTotalTokens += len(enc.encode(line))
|
||||
else:
|
||||
inputTotalTokens += len(enc.encode(history))
|
||||
inputTotalTokens += len(enc.encode(system))
|
||||
inputTotalTokens += len(enc.encode(user))
|
||||
|
||||
# Output
|
||||
outputTotalTokens += round(len(enc.encode(user)) * 2.5)
|
||||
|
||||
return [inputTotalTokens, outputTotalTokens]
|
||||
|
||||
|
||||
@retry(exceptions=Exception, tries=5, delay=5)
|
||||
def translateGPT(text, history, fullPromptFlag):
|
||||
def translateAI(text, history, fullPromptFlag):
|
||||
"""
|
||||
Legacy wrapper function for the new shared translation utility.
|
||||
This maintains compatibility with existing code while using the new shared implementation.
|
||||
"""
|
||||
global PBAR, MISMATCH, FILENAME
|
||||
if text:
|
||||
with open("log/translationHistory.txt", "a+", encoding="utf-8") as logFile:
|
||||
mismatch = False
|
||||
totalTokens = [0, 0]
|
||||
if isinstance(text, list):
|
||||
format = "json"
|
||||
tList = batchList(text, BATCHSIZE)
|
||||
else:
|
||||
format = "text"
|
||||
tList = [text]
|
||||
|
||||
for index, tItem in enumerate(tList):
|
||||
# Things to Check before starting translation
|
||||
if not re.search(LANGREGEX, str(tItem)):
|
||||
if PBAR is not None:
|
||||
PBAR.update(len(tItem))
|
||||
if isinstance(tItem, list):
|
||||
for j in range(len(tItem)):
|
||||
tItem[j] = cleanTranslatedText(tItem[j])
|
||||
tList[index] = tItem
|
||||
else:
|
||||
tList[index] = cleanTranslatedText(tItem)
|
||||
history = tItem[-MAXHISTORY:]
|
||||
continue
|
||||
# Update config estimate mode based on global ESTIMATE
|
||||
TRANSLATION_CONFIG.estimateMode = bool(ESTIMATE)
|
||||
|
||||
# Before sending to translation, if we have a list of items, add the formatting
|
||||
if isinstance(tItem, list):
|
||||
for j in range(len(tItem)):
|
||||
if not tItem[j]:
|
||||
tItem[j] = tItem[j].replace("", "Placeholder Text")
|
||||
payload = {f"Line{i+1}": string for i, string in enumerate(tItem)}
|
||||
payload = json.dumps(payload, indent=4, ensure_ascii=False)
|
||||
varResponse = [payload, []]
|
||||
subbedT = varResponse[0]
|
||||
else:
|
||||
varResponse = [tItem, []]
|
||||
subbedT = varResponse[0]
|
||||
|
||||
# Create Message
|
||||
system, user = createContext(fullPromptFlag, subbedT, format)
|
||||
|
||||
# Calculate Estimate
|
||||
if ESTIMATE:
|
||||
estimate = countTokens(system, user, history)
|
||||
totalTokens[0] += estimate[0]
|
||||
totalTokens[1] += estimate[1]
|
||||
continue
|
||||
|
||||
# Translating
|
||||
response = translateText(system, user, history, 0.05, format)
|
||||
|
||||
# Set Tokens
|
||||
translatedText = response.choices[0].message.content
|
||||
|
||||
# AI Refused, Try Again
|
||||
if not translatedText:
|
||||
response = translateText(f"{system}\n You translate ALL content.", user, history, 0.1, format, model="gpt-4o")
|
||||
translatedText = response.choices[0].message.content
|
||||
|
||||
# Report Tokens
|
||||
totalTokens[0] += response.usage.prompt_tokens
|
||||
totalTokens[1] += response.usage.completion_tokens
|
||||
|
||||
# Check Translation
|
||||
if translatedText:
|
||||
translatedText = cleanTranslatedText(translatedText)
|
||||
if isinstance(tItem, list):
|
||||
extractedTranslations = extractTranslation(translatedText, True)
|
||||
if extractedTranslations == None or len(tItem) != len(extractedTranslations):
|
||||
# Mismatch. Try Again
|
||||
response = translateText(system, user, history, 0.05, format, MODEL)
|
||||
translatedText = response.choices[0].message.content
|
||||
totalTokens[0] += response.usage.prompt_tokens
|
||||
totalTokens[1] += response.usage.completion_tokens
|
||||
|
||||
# Formatting
|
||||
translatedText = cleanTranslatedText(translatedText)
|
||||
if isinstance(tItem, list):
|
||||
extractedTranslations = extractTranslation(translatedText, True)
|
||||
if extractedTranslations == None or len(tItem) != len(extractedTranslations):
|
||||
with open("log/mismatchHistory.txt", "a+", encoding="utf-8") as mismatchFile:
|
||||
mismatchFile.write(f"Mismatch: {FILENAME}\n")
|
||||
mismatchFile.write(f"Input:\n{subbedT}\n")
|
||||
mismatchFile.write(f"Output:\n{translatedText}\n")
|
||||
mismatch = True # Just here for breakpoint
|
||||
logFile.write(f"Input:\n{subbedT}\n")
|
||||
logFile.write(f"Output:\n{translatedText}\n")
|
||||
|
||||
# Set if no mismatch
|
||||
if mismatch == False:
|
||||
tList[index] = extractedTranslations
|
||||
history = extractedTranslations[-MAXHISTORY:] # Update history if we have a list
|
||||
else:
|
||||
history = text[-MAXHISTORY:]
|
||||
mismatch = False
|
||||
if FILENAME not in MISMATCH:
|
||||
MISMATCH.append(FILENAME)
|
||||
|
||||
# Update Loading Bar
|
||||
with LOCK:
|
||||
if PBAR is not None:
|
||||
PBAR.update(len(tItem))
|
||||
else:
|
||||
# Ensure we're passing a single string to extractTranslation
|
||||
tList[index] = translatedText.replace("Placeholder Text", "")
|
||||
else:
|
||||
PBAR.write(f"AI Refused:{tItem}\n")
|
||||
|
||||
# Combine if multilist
|
||||
if isinstance(tList[0], list):
|
||||
tList = [t for sublist in tList for t in sublist]
|
||||
|
||||
# Return
|
||||
if format == "json":
|
||||
return [tList, totalTokens]
|
||||
else:
|
||||
return [tList[0], totalTokens]
|
||||
else:
|
||||
return [text, [0, 0]]
|
||||
# Call the new shared translation function
|
||||
return sharedtranslateAI(
|
||||
text=text,
|
||||
history=history,
|
||||
fullPromptFlag=fullPromptFlag,
|
||||
config=TRANSLATION_CONFIG,
|
||||
filename=FILENAME,
|
||||
pbar=PBAR,
|
||||
lock=LOCK,
|
||||
mismatchList=MISMATCH
|
||||
)
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ from colorama import Fore
|
|||
from dotenv import load_dotenv
|
||||
from retry import retry
|
||||
from tqdm import tqdm
|
||||
from util.translation import TranslationConfig, translateAI as sharedtranslateAI
|
||||
|
||||
# Open AI
|
||||
load_dotenv()
|
||||
|
|
@ -76,6 +77,18 @@ else:
|
|||
BATCHSIZE = int(os.getenv("batchsize"))
|
||||
FREQUENCY_PENALTY = float(os.getenv("frequency_penalty"))
|
||||
|
||||
# Initialize Translation Config
|
||||
TRANSLATION_CONFIG = TranslationConfig(
|
||||
model=MODEL,
|
||||
language=LANGUAGE,
|
||||
prompt=PROMPT,
|
||||
vocab=VOCAB,
|
||||
langRegex=LANGREGEX,
|
||||
batchSize=BATCHSIZE,
|
||||
maxHistory=MAXHISTORY,
|
||||
estimateMode=False # Will be set dynamically based on ESTIMATE
|
||||
)
|
||||
LEAVE = False
|
||||
|
||||
def handleTyrano(filename, estimate):
|
||||
global ESTIMATE, TOKENS, FILENAME
|
||||
|
|
@ -328,7 +341,7 @@ def translateTyrano(data, translatedList):
|
|||
if stringList:
|
||||
PBAR.total = len(stringList)
|
||||
PBAR.refresh()
|
||||
response = translateGPT(stringList, "Reply with the English Translation", True)
|
||||
response = translateAI(stringList, "Reply with the English Translation", True)
|
||||
tokens[0] += response[1][0]
|
||||
tokens[1] += response[1][1]
|
||||
stringListTL = response[0]
|
||||
|
|
@ -341,7 +354,7 @@ def translateTyrano(data, translatedList):
|
|||
|
||||
# Choice List
|
||||
if choiceList:
|
||||
response = translateGPT(choiceList, "Reply with the English TL of the Dialogue Choice", True)
|
||||
response = translateAI(choiceList, "Reply with the English TL of the Dialogue Choice", True)
|
||||
tokens[0] += response[1][0]
|
||||
tokens[1] += response[1][1]
|
||||
choiceListTL = response[0]
|
||||
|
|
@ -370,7 +383,7 @@ def getSpeaker(speaker):
|
|||
return [NAMESLIST[i][1], [0, 0]]
|
||||
|
||||
# Translate and Store Speaker
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
f"{speaker}",
|
||||
"Reply with the " + LANGUAGE + " translation of the NPC name.",
|
||||
False,
|
||||
|
|
@ -381,7 +394,7 @@ def getSpeaker(speaker):
|
|||
|
||||
# Retry if name doesn't translate for some reason
|
||||
if re.search(r"([a-zA-Z？?])", response[0]) == None:
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
f"{speaker}",
|
||||
"Reply with the " + LANGUAGE + " translation of the NPC name.",
|
||||
False,
|
||||
|
|
@ -394,328 +407,24 @@ def getSpeaker(speaker):
|
|||
return response
|
||||
return [speaker, [0, 0]]
|
||||
|
||||
|
||||
def batchList(input_list, batch_size):
|
||||
if not isinstance(batch_size, int) or batch_size <= 0:
|
||||
raise ValueError("batch_size must be a positive integer")
|
||||
|
||||
return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)]
|
||||
|
||||
|
||||
def parseVocabWithCategories(vocabText):
|
||||
"""Parse vocabulary text and extract terms with their categories."""
|
||||
pairs = []
|
||||
seen = set()
|
||||
currentCategory = None
|
||||
|
||||
for line in vocabText.splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith('```'):
|
||||
continue
|
||||
|
||||
# Check if this is a category header
|
||||
if line.startswith('#'):
|
||||
currentCategory = line
|
||||
continue
|
||||
|
||||
# Parse vocabulary term
|
||||
m = re.match(r'^(.+?)(?:\s?[\(–])', line) # term is everything before space + '(' or '–'
|
||||
if m:
|
||||
term = m.group(1)
|
||||
if term not in seen:
|
||||
pairs.append((term, line, currentCategory))
|
||||
seen.add(term)
|
||||
|
||||
return pairs
|
||||
|
||||
|
||||
def buildMatchedVocabText(vocabPairs, subbedT):
|
||||
"""Build formatted vocabulary text with matched terms organized by category."""
|
||||
matchedCategories = {}
|
||||
|
||||
# Use word boundaries for Japanese if appropriate, or allow substring as before.
|
||||
for term, line, category in vocabPairs:
|
||||
# "term in subbedT" could be false positive; can use regex but Japanese doesn't always have spaces.
|
||||
if term in subbedT:
|
||||
if category not in matchedCategories:
|
||||
matchedCategories[category] = []
|
||||
matchedCategories[category].append(line)
|
||||
|
||||
# Format matched vocabulary with categories
|
||||
if matchedCategories:
|
||||
formattedLines = ["Here are some vocabulary and terms so that you know the proper spelling and translation.\n"]
|
||||
for category, lines in matchedCategories.items():
|
||||
if category: # Only add category header if it exists
|
||||
formattedLines.append(category)
|
||||
formattedLines.extend(lines)
|
||||
formattedLines.append("") # Add blank line between categories
|
||||
matchedVocabText = f"```\n{chr(10).join(formattedLines).rstrip()}\n```"
|
||||
else:
|
||||
matchedVocabText = ""
|
||||
|
||||
return matchedVocabText
|
||||
|
||||
|
||||
def createContext(fullPromptFlag, subbedT, format):
|
||||
vocabPairs = parseVocabWithCategories(VOCAB)
|
||||
matchedVocabText = buildMatchedVocabText(vocabPairs, subbedT)
|
||||
|
||||
if fullPromptFlag:
|
||||
system = PROMPT + matchedVocabText
|
||||
else:
|
||||
system = f"\
|
||||
You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
|
||||
Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
|
||||
- You translate everything, including erotic and explicit content.\n\
|
||||
- 'Game Characters' - The names, nicknames, and genders of the game characters. Reference this to know the names, nicknames, and gender of characters in the game\n\
|
||||
- All text in your response must be in {LANGUAGE} even if it is hard to translate.\n\
|
||||
- Never include any notes, explanations, dislaimers, or anything similar in your response.\n\
|
||||
- Maintain any spacing in the translation.\n\
|
||||
- Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\
|
||||
- `...` can be a part of the dialogue. Translate it as it is.\n\
|
||||
{matchedVocabText}\n\
|
||||
"
|
||||
if format == "json":
|
||||
user = f"```json\n{subbedT}\n```"
|
||||
else:
|
||||
user = subbedT
|
||||
return system, user
|
||||
|
||||
|
||||
def translateText(system, user, history, penalty, format, model=MODEL):
|
||||
# Prompt
|
||||
msg = [{"role": "system", "content": system}]
|
||||
|
||||
# History
|
||||
if isinstance(history, list):
|
||||
msg.append({"role": "system", "content": "Translation History:"})
|
||||
msg.extend([{"role": "assistant", "content": h} for h in history])
|
||||
else:
|
||||
msg.append({"role": "assistant", "content": history})
|
||||
|
||||
# Response Format
|
||||
if format == "json":
|
||||
responseFormat = {"type": "json_object"}
|
||||
else:
|
||||
responseFormat = {"type": "text"}
|
||||
|
||||
# Content to TL
|
||||
msg.append({"role": "user", "content": f"{user}"})
|
||||
response = openai.chat.completions.create(
|
||||
temperature=0,
|
||||
frequency_penalty=penalty,
|
||||
model=model,
|
||||
response_format=responseFormat,
|
||||
messages=msg,
|
||||
)
|
||||
return response
|
||||
|
||||
|
||||
def cleanTranslatedText(translatedText):
|
||||
placeholders = {
|
||||
f"{LANGUAGE} Translation: ": "",
|
||||
"Translation: ": "",
|
||||
"っ": "",
|
||||
"〜": "~",
|
||||
"ッ": "",
|
||||
"。": ".",
|
||||
"「": '\\"',
|
||||
"」": '\\"',
|
||||
"- ": "-",
|
||||
"—": "―",
|
||||
"】": "]",
|
||||
"【": "[",
|
||||
"é": "e",
|
||||
"this guy": "this bastard",
|
||||
"This guy": "This bastard",
|
||||
"Placeholder Text": "",
|
||||
"```json": "",
|
||||
"```": "",
|
||||
# Add more replacements as needed
|
||||
}
|
||||
for target, replacement in placeholders.items():
|
||||
translatedText = translatedText.replace(target, replacement)
|
||||
|
||||
# Remove Repeating Characters
|
||||
pattern = re.compile(r"(.)\s*\1(?:\s*\1){" + str(20 - 1) + r",}")
|
||||
translatedText = pattern.sub(lambda match: match.group(0).replace(" ", "")[:20], translatedText)
|
||||
|
||||
# Elongate Long Dashes (Since GPT Ignores them...)
|
||||
translatedText = elongateCharacters(translatedText)
|
||||
return translatedText
|
||||
|
||||
|
||||
def elongateCharacters(text):
|
||||
# Define a pattern to match one character followed by one or more `ー` characters
|
||||
# Using a positive lookbehind assertion to capture the preceding character
|
||||
pattern = r"(?<=(.))ー+"
|
||||
|
||||
# Define a replacement function that elongates the captured character
|
||||
def repl(match):
|
||||
char = match.group(1) # The character before the ー sequence
|
||||
count = len(match.group(0)) - 1 # Number of ー characters
|
||||
return char * count # Replace ー sequence with the character repeated
|
||||
|
||||
# Use re.sub() to replace the pattern in the text
|
||||
return re.sub(pattern, repl, text)
|
||||
|
||||
|
||||
def extractTranslation(translatedTextList, is_list):
|
||||
try:
|
||||
translatedTextList = re.sub(r'\\"+\"([^,\n}])', r'\\"\1', translatedTextList)
|
||||
translatedTextList = re.sub(r"(?<![\\])\"+(?![\n,])", r'"', translatedTextList)
|
||||
line_dict = json.loads(translatedTextList)
|
||||
# If it's a batch (i.e., list), extract with tags; otherwise, return the single item.
|
||||
string_list = list(line_dict.values())
|
||||
if is_list:
|
||||
return string_list
|
||||
else:
|
||||
return string_list[0]
|
||||
|
||||
except Exception as e:
|
||||
PBAR.write(f"extractTranslation Error: {e} on String {translatedTextList}")
|
||||
return None
|
||||
|
||||
|
||||
def countTokens(system, user, history):
|
||||
inputTotalTokens = 0
|
||||
outputTotalTokens = 0
|
||||
enc = tiktoken.encoding_for_model("gpt-4")
|
||||
|
||||
# Input
|
||||
if isinstance(history, list):
|
||||
for line in history:
|
||||
inputTotalTokens += len(enc.encode(line))
|
||||
else:
|
||||
inputTotalTokens += len(enc.encode(history))
|
||||
inputTotalTokens += len(enc.encode(system))
|
||||
inputTotalTokens += len(enc.encode(user))
|
||||
|
||||
# Output
|
||||
outputTotalTokens += round(len(enc.encode(user)) * 2.5)
|
||||
|
||||
return [inputTotalTokens, outputTotalTokens]
|
||||
|
||||
|
||||
@retry(exceptions=Exception, tries=5, delay=5)
|
||||
def translateGPT(text, history, fullPromptFlag):
|
||||
def translateAI(text, history, fullPromptFlag):
|
||||
"""
|
||||
Legacy wrapper function for the new shared translation utility.
|
||||
This maintains compatibility with existing code while using the new shared implementation.
|
||||
"""
|
||||
global PBAR, MISMATCH, FILENAME
|
||||
if text:
|
||||
with open("log/translationHistory.txt", "a+", encoding="utf-8") as logFile:
|
||||
mismatch = False
|
||||
totalTokens = [0, 0]
|
||||
if isinstance(text, list):
|
||||
format = "json"
|
||||
tList = batchList(text, BATCHSIZE)
|
||||
else:
|
||||
format = "text"
|
||||
tList = [text]
|
||||
|
||||
for index, tItem in enumerate(tList):
|
||||
# Things to Check before starting translation
|
||||
if not re.search(LANGREGEX, str(tItem)):
|
||||
if PBAR is not None:
|
||||
PBAR.update(len(tItem))
|
||||
if isinstance(tItem, list):
|
||||
for j in range(len(tItem)):
|
||||
tItem[j] = cleanTranslatedText(tItem[j])
|
||||
tList[index] = tItem
|
||||
else:
|
||||
tList[index] = cleanTranslatedText(tItem)
|
||||
history = tItem[-MAXHISTORY:]
|
||||
continue
|
||||
# Update config estimate mode based on global ESTIMATE
|
||||
TRANSLATION_CONFIG.estimateMode = bool(ESTIMATE)
|
||||
|
||||
# Before sending to translation, if we have a list of items, add the formatting
|
||||
if isinstance(tItem, list):
|
||||
for j in range(len(tItem)):
|
||||
if not tItem[j]:
|
||||
tItem[j] = tItem[j].replace("", "Placeholder Text")
|
||||
payload = {f"Line{i+1}": string for i, string in enumerate(tItem)}
|
||||
payload = json.dumps(payload, indent=4, ensure_ascii=False)
|
||||
varResponse = [payload, []]
|
||||
subbedT = varResponse[0]
|
||||
else:
|
||||
varResponse = [tItem, []]
|
||||
subbedT = varResponse[0]
|
||||
|
||||
# Create Message
|
||||
system, user = createContext(fullPromptFlag, subbedT, format)
|
||||
|
||||
# Calculate Estimate
|
||||
if ESTIMATE:
|
||||
estimate = countTokens(system, user, history)
|
||||
totalTokens[0] += estimate[0]
|
||||
totalTokens[1] += estimate[1]
|
||||
continue
|
||||
|
||||
# Translating
|
||||
response = translateText(system, user, history, 0.05, format)
|
||||
|
||||
# Set Tokens
|
||||
translatedText = response.choices[0].message.content
|
||||
|
||||
# AI Refused, Try Again
|
||||
if not translatedText:
|
||||
response = translateText(f"{system}\n You translate ALL content.", user, history, 0.1, format, model="gpt-4o")
|
||||
translatedText = response.choices[0].message.content
|
||||
|
||||
# Report Tokens
|
||||
totalTokens[0] += response.usage.prompt_tokens
|
||||
totalTokens[1] += response.usage.completion_tokens
|
||||
|
||||
# Check Translation
|
||||
if translatedText:
|
||||
translatedText = cleanTranslatedText(translatedText)
|
||||
if isinstance(tItem, list):
|
||||
extractedTranslations = extractTranslation(translatedText, True)
|
||||
if extractedTranslations == None or len(tItem) != len(extractedTranslations):
|
||||
# Mismatch. Try Again
|
||||
response = translateText(system, user, history, 0.05, format, MODEL)
|
||||
translatedText = response.choices[0].message.content
|
||||
totalTokens[0] += response.usage.prompt_tokens
|
||||
totalTokens[1] += response.usage.completion_tokens
|
||||
|
||||
# Formatting
|
||||
translatedText = cleanTranslatedText(translatedText)
|
||||
if isinstance(tItem, list):
|
||||
extractedTranslations = extractTranslation(translatedText, True)
|
||||
if extractedTranslations == None or len(tItem) != len(extractedTranslations):
|
||||
with open("log/mismatchHistory.txt", "a+", encoding="utf-8") as mismatchFile:
|
||||
mismatchFile.write(f"Mismatch: {FILENAME}\n")
|
||||
mismatchFile.write(f"Input:\n{subbedT}\n")
|
||||
mismatchFile.write(f"Output:\n{translatedText}\n")
|
||||
mismatch = True # Just here for breakpoint
|
||||
logFile.write(f"Input:\n{subbedT}\n")
|
||||
logFile.write(f"Output:\n{translatedText}\n")
|
||||
|
||||
# Set if no mismatch
|
||||
if mismatch == False:
|
||||
tList[index] = extractedTranslations
|
||||
history = extractedTranslations[-MAXHISTORY:] # Update history if we have a list
|
||||
else:
|
||||
history = text[-MAXHISTORY:]
|
||||
mismatch = False
|
||||
if FILENAME not in MISMATCH:
|
||||
MISMATCH.append(FILENAME)
|
||||
|
||||
# Update Loading Bar
|
||||
with LOCK:
|
||||
if PBAR is not None:
|
||||
PBAR.update(len(tItem))
|
||||
else:
|
||||
# Ensure we're passing a single string to extractTranslation
|
||||
tList[index] = translatedText.replace("Placeholder Text", "")
|
||||
else:
|
||||
PBAR.write(f"AI Refused:{tItem}\n")
|
||||
|
||||
# Combine if multilist
|
||||
if isinstance(tList[0], list):
|
||||
tList = [t for sublist in tList for t in sublist]
|
||||
|
||||
# Return
|
||||
if format == "json":
|
||||
return [tList, totalTokens]
|
||||
else:
|
||||
return [tList[0], totalTokens]
|
||||
else:
|
||||
return [text, [0, 0]]
|
||||
# Call the new shared translation function
|
||||
return sharedtranslateAI(
|
||||
text=text,
|
||||
history=history,
|
||||
fullPromptFlag=fullPromptFlag,
|
||||
config=TRANSLATION_CONFIG,
|
||||
filename=FILENAME,
|
||||
pbar=PBAR,
|
||||
lock=LOCK,
|
||||
mismatchList=MISMATCH
|
||||
)
|
||||
|
|
|
|||
359
modules/unity.py
359
modules/unity.py
|
|
@ -13,6 +13,7 @@ from colorama import Fore
|
|||
from dotenv import load_dotenv
|
||||
from retry import retry
|
||||
from tqdm import tqdm
|
||||
from util.translation import TranslationConfig, translateAI as sharedtranslateAI
|
||||
|
||||
# Open AI
|
||||
load_dotenv()
|
||||
|
|
@ -82,6 +83,18 @@ else:
|
|||
BATCHSIZE = int(os.getenv("batchsize"))
|
||||
FREQUENCY_PENALTY = float(os.getenv("frequency_penalty"))
|
||||
|
||||
# Initialize Translation Config
|
||||
TRANSLATION_CONFIG = TranslationConfig(
|
||||
model=MODEL,
|
||||
language=LANGUAGE,
|
||||
prompt=PROMPT,
|
||||
vocab=VOCAB,
|
||||
langRegex=LANGREGEX,
|
||||
batchSize=BATCHSIZE,
|
||||
maxHistory=MAXHISTORY,
|
||||
estimateMode=False # Will be set dynamically based on ESTIMATE
|
||||
)
|
||||
LEAVE = False
|
||||
|
||||
def handleUnity(filename, estimate):
|
||||
global ESTIMATE, FILENAME
|
||||
|
|
@ -253,7 +266,7 @@ def translateUnity(data, pbar, filename, translatedList):
|
|||
pbar.refresh()
|
||||
|
||||
# Translate
|
||||
response = translateGPT(stringList, "", True)
|
||||
response = translateAI(stringList, "", True)
|
||||
tokens[0] += response[1][0]
|
||||
tokens[1] += response[1][1]
|
||||
translatedList = response[0]
|
||||
|
|
@ -283,7 +296,7 @@ def getSpeaker(speaker):
|
|||
return [NAMESLIST[i][1], [0, 0]]
|
||||
|
||||
# Translate and Store Speaker
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
f"{speaker}",
|
||||
"Reply with the " + LANGUAGE + " translation of the NPC name.",
|
||||
False,
|
||||
|
|
@ -294,7 +307,7 @@ def getSpeaker(speaker):
|
|||
|
||||
# Retry if name doesn't translate for some reason
|
||||
if re.search(r"([a-zA-Z？?])", response[0]) == None:
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
f"{speaker}",
|
||||
"Reply with the " + LANGUAGE + " translation of the NPC name.",
|
||||
False,
|
||||
|
|
@ -307,328 +320,24 @@ def getSpeaker(speaker):
|
|||
return response
|
||||
return [speaker, [0, 0]]
|
||||
|
||||
|
||||
def batchList(input_list, batch_size):
|
||||
if not isinstance(batch_size, int) or batch_size <= 0:
|
||||
raise ValueError("batch_size must be a positive integer")
|
||||
|
||||
return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)]
|
||||
|
||||
|
||||
def parseVocabWithCategories(vocabText):
|
||||
"""Parse vocabulary text and extract terms with their categories."""
|
||||
pairs = []
|
||||
seen = set()
|
||||
currentCategory = None
|
||||
|
||||
for line in vocabText.splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith('```'):
|
||||
continue
|
||||
|
||||
# Check if this is a category header
|
||||
if line.startswith('#'):
|
||||
currentCategory = line
|
||||
continue
|
||||
|
||||
# Parse vocabulary term
|
||||
m = re.match(r'^(.+?)(?:\s?[\(–])', line) # term is everything before space + '(' or '–'
|
||||
if m:
|
||||
term = m.group(1)
|
||||
if term not in seen:
|
||||
pairs.append((term, line, currentCategory))
|
||||
seen.add(term)
|
||||
|
||||
return pairs
|
||||
|
||||
|
||||
def buildMatchedVocabText(vocabPairs, subbedT):
|
||||
"""Build formatted vocabulary text with matched terms organized by category."""
|
||||
matchedCategories = {}
|
||||
|
||||
# Use word boundaries for Japanese if appropriate, or allow substring as before.
|
||||
for term, line, category in vocabPairs:
|
||||
# "term in subbedT" could be false positive; can use regex but Japanese doesn't always have spaces.
|
||||
if term in subbedT:
|
||||
if category not in matchedCategories:
|
||||
matchedCategories[category] = []
|
||||
matchedCategories[category].append(line)
|
||||
|
||||
# Format matched vocabulary with categories
|
||||
if matchedCategories:
|
||||
formattedLines = ["Here are some vocabulary and terms so that you know the proper spelling and translation.\n"]
|
||||
for category, lines in matchedCategories.items():
|
||||
if category: # Only add category header if it exists
|
||||
formattedLines.append(category)
|
||||
formattedLines.extend(lines)
|
||||
formattedLines.append("") # Add blank line between categories
|
||||
matchedVocabText = f"```\n{chr(10).join(formattedLines).rstrip()}\n```"
|
||||
else:
|
||||
matchedVocabText = ""
|
||||
|
||||
return matchedVocabText
|
||||
|
||||
|
||||
def createContext(fullPromptFlag, subbedT, format):
|
||||
vocabPairs = parseVocabWithCategories(VOCAB)
|
||||
matchedVocabText = buildMatchedVocabText(vocabPairs, subbedT)
|
||||
|
||||
if fullPromptFlag:
|
||||
system = PROMPT + matchedVocabText
|
||||
else:
|
||||
system = f"\
|
||||
You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
|
||||
Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
|
||||
- You translate everything, including erotic and explicit content.\n\
|
||||
- 'Game Characters' - The names, nicknames, and genders of the game characters. Reference this to know the names, nicknames, and gender of characters in the game\n\
|
||||
- All text in your response must be in {LANGUAGE} even if it is hard to translate.\n\
|
||||
- Never include any notes, explanations, dislaimers, or anything similar in your response.\n\
|
||||
- Maintain any spacing in the translation.\n\
|
||||
- Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\
|
||||
- `...` can be a part of the dialogue. Translate it as it is.\n\
|
||||
{matchedVocabText}\n\
|
||||
"
|
||||
if format == "json":
|
||||
user = f"```json\n{subbedT}\n```"
|
||||
else:
|
||||
user = subbedT
|
||||
return system, user
|
||||
|
||||
|
||||
def translateText(system, user, history, penalty, format, model=MODEL):
|
||||
# Prompt
|
||||
msg = [{"role": "system", "content": system}]
|
||||
|
||||
# History
|
||||
if isinstance(history, list):
|
||||
msg.append({"role": "system", "content": "Translation History:"})
|
||||
msg.extend([{"role": "assistant", "content": h} for h in history])
|
||||
else:
|
||||
msg.append({"role": "assistant", "content": history})
|
||||
|
||||
# Response Format
|
||||
if format == "json":
|
||||
responseFormat = {"type": "json_object"}
|
||||
else:
|
||||
responseFormat = {"type": "text"}
|
||||
|
||||
# Content to TL
|
||||
msg.append({"role": "user", "content": f"{user}"})
|
||||
response = openai.chat.completions.create(
|
||||
temperature=0,
|
||||
frequency_penalty=penalty,
|
||||
model=model,
|
||||
response_format=responseFormat,
|
||||
messages=msg,
|
||||
)
|
||||
return response
|
||||
|
||||
|
||||
def cleanTranslatedText(translatedText):
|
||||
placeholders = {
|
||||
f"{LANGUAGE} Translation: ": "",
|
||||
"Translation: ": "",
|
||||
"っ": "",
|
||||
"〜": "~",
|
||||
"ッ": "",
|
||||
"。": ".",
|
||||
"「": '\\"',
|
||||
"」": '\\"',
|
||||
"- ": "-",
|
||||
"—": "―",
|
||||
"】": "]",
|
||||
"【": "[",
|
||||
"é": "e",
|
||||
"this guy": "this bastard",
|
||||
"This guy": "This bastard",
|
||||
"Placeholder Text": "",
|
||||
"```json": "",
|
||||
"```": "",
|
||||
# Add more replacements as needed
|
||||
}
|
||||
for target, replacement in placeholders.items():
|
||||
translatedText = translatedText.replace(target, replacement)
|
||||
|
||||
# Remove Repeating Characters
|
||||
pattern = re.compile(r"(.)\s*\1(?:\s*\1){" + str(20 - 1) + r",}")
|
||||
translatedText = pattern.sub(lambda match: match.group(0).replace(" ", "")[:20], translatedText)
|
||||
|
||||
# Elongate Long Dashes (Since GPT Ignores them...)
|
||||
translatedText = elongateCharacters(translatedText)
|
||||
return translatedText
|
||||
|
||||
|
||||
def elongateCharacters(text):
|
||||
# Define a pattern to match one character followed by one or more `ー` characters
|
||||
# Using a positive lookbehind assertion to capture the preceding character
|
||||
pattern = r"(?<=(.))ー+"
|
||||
|
||||
# Define a replacement function that elongates the captured character
|
||||
def repl(match):
|
||||
char = match.group(1) # The character before the ー sequence
|
||||
count = len(match.group(0)) - 1 # Number of ー characters
|
||||
return char * count # Replace ー sequence with the character repeated
|
||||
|
||||
# Use re.sub() to replace the pattern in the text
|
||||
return re.sub(pattern, repl, text)
|
||||
|
||||
|
||||
def extractTranslation(translatedTextList, is_list):
|
||||
try:
|
||||
translatedTextList = re.sub(r'\\"+\"([^,\n}])', r'\\"\1', translatedTextList)
|
||||
translatedTextList = re.sub(r"(?<![\\])\"+(?![\n,])", r'"', translatedTextList)
|
||||
line_dict = json.loads(translatedTextList)
|
||||
# If it's a batch (i.e., list), extract with tags; otherwise, return the single item.
|
||||
string_list = list(line_dict.values())
|
||||
if is_list:
|
||||
return string_list
|
||||
else:
|
||||
return string_list[0]
|
||||
|
||||
except Exception as e:
|
||||
PBAR.write(f"extractTranslation Error: {e} on String {translatedTextList}")
|
||||
return None
|
||||
|
||||
|
||||
def countTokens(system, user, history):
|
||||
inputTotalTokens = 0
|
||||
outputTotalTokens = 0
|
||||
enc = tiktoken.encoding_for_model("gpt-4")
|
||||
|
||||
# Input
|
||||
if isinstance(history, list):
|
||||
for line in history:
|
||||
inputTotalTokens += len(enc.encode(line))
|
||||
else:
|
||||
inputTotalTokens += len(enc.encode(history))
|
||||
inputTotalTokens += len(enc.encode(system))
|
||||
inputTotalTokens += len(enc.encode(user))
|
||||
|
||||
# Output
|
||||
outputTotalTokens += round(len(enc.encode(user)) * 2.5)
|
||||
|
||||
return [inputTotalTokens, outputTotalTokens]
|
||||
|
||||
|
||||
@retry(exceptions=Exception, tries=5, delay=5)
|
||||
def translateGPT(text, history, fullPromptFlag):
|
||||
def translateAI(text, history, fullPromptFlag):
|
||||
"""
|
||||
Legacy wrapper function for the new shared translation utility.
|
||||
This maintains compatibility with existing code while using the new shared implementation.
|
||||
"""
|
||||
global PBAR, MISMATCH, FILENAME
|
||||
if text:
|
||||
with open("log/translationHistory.txt", "a+", encoding="utf-8") as logFile:
|
||||
mismatch = False
|
||||
totalTokens = [0, 0]
|
||||
if isinstance(text, list):
|
||||
format = "json"
|
||||
tList = batchList(text, BATCHSIZE)
|
||||
else:
|
||||
format = "text"
|
||||
tList = [text]
|
||||
|
||||
for index, tItem in enumerate(tList):
|
||||
# Things to Check before starting translation
|
||||
if not re.search(LANGREGEX, str(tItem)):
|
||||
if PBAR is not None:
|
||||
PBAR.update(len(tItem))
|
||||
if isinstance(tItem, list):
|
||||
for j in range(len(tItem)):
|
||||
tItem[j] = cleanTranslatedText(tItem[j])
|
||||
tList[index] = tItem
|
||||
else:
|
||||
tList[index] = cleanTranslatedText(tItem)
|
||||
history = tItem[-MAXHISTORY:]
|
||||
continue
|
||||
# Update config estimate mode based on global ESTIMATE
|
||||
TRANSLATION_CONFIG.estimateMode = bool(ESTIMATE)
|
||||
|
||||
# Before sending to translation, if we have a list of items, add the formatting
|
||||
if isinstance(tItem, list):
|
||||
for j in range(len(tItem)):
|
||||
if not tItem[j]:
|
||||
tItem[j] = tItem[j].replace("", "Placeholder Text")
|
||||
payload = {f"Line{i+1}": string for i, string in enumerate(tItem)}
|
||||
payload = json.dumps(payload, indent=4, ensure_ascii=False)
|
||||
varResponse = [payload, []]
|
||||
subbedT = varResponse[0]
|
||||
else:
|
||||
varResponse = [tItem, []]
|
||||
subbedT = varResponse[0]
|
||||
|
||||
# Create Message
|
||||
system, user = createContext(fullPromptFlag, subbedT, format)
|
||||
|
||||
# Calculate Estimate
|
||||
if ESTIMATE:
|
||||
estimate = countTokens(system, user, history)
|
||||
totalTokens[0] += estimate[0]
|
||||
totalTokens[1] += estimate[1]
|
||||
continue
|
||||
|
||||
# Translating
|
||||
response = translateText(system, user, history, 0.05, format)
|
||||
|
||||
# Set Tokens
|
||||
translatedText = response.choices[0].message.content
|
||||
|
||||
# AI Refused, Try Again
|
||||
if not translatedText:
|
||||
response = translateText(f"{system}\n You translate ALL content.", user, history, 0.1, format, model="gpt-4o")
|
||||
translatedText = response.choices[0].message.content
|
||||
|
||||
# Report Tokens
|
||||
totalTokens[0] += response.usage.prompt_tokens
|
||||
totalTokens[1] += response.usage.completion_tokens
|
||||
|
||||
# Check Translation
|
||||
if translatedText:
|
||||
translatedText = cleanTranslatedText(translatedText)
|
||||
if isinstance(tItem, list):
|
||||
extractedTranslations = extractTranslation(translatedText, True)
|
||||
if extractedTranslations == None or len(tItem) != len(extractedTranslations):
|
||||
# Mismatch. Try Again
|
||||
response = translateText(system, user, history, 0.05, format, MODEL)
|
||||
translatedText = response.choices[0].message.content
|
||||
totalTokens[0] += response.usage.prompt_tokens
|
||||
totalTokens[1] += response.usage.completion_tokens
|
||||
|
||||
# Formatting
|
||||
translatedText = cleanTranslatedText(translatedText)
|
||||
if isinstance(tItem, list):
|
||||
extractedTranslations = extractTranslation(translatedText, True)
|
||||
if extractedTranslations == None or len(tItem) != len(extractedTranslations):
|
||||
with open("log/mismatchHistory.txt", "a+", encoding="utf-8") as mismatchFile:
|
||||
mismatchFile.write(f"Mismatch: {FILENAME}\n")
|
||||
mismatchFile.write(f"Input:\n{subbedT}\n")
|
||||
mismatchFile.write(f"Output:\n{translatedText}\n")
|
||||
mismatch = True # Just here for breakpoint
|
||||
logFile.write(f"Input:\n{subbedT}\n")
|
||||
logFile.write(f"Output:\n{translatedText}\n")
|
||||
|
||||
# Set if no mismatch
|
||||
if mismatch == False:
|
||||
tList[index] = extractedTranslations
|
||||
history = extractedTranslations[-MAXHISTORY:] # Update history if we have a list
|
||||
else:
|
||||
history = text[-MAXHISTORY:]
|
||||
mismatch = False
|
||||
if FILENAME not in MISMATCH:
|
||||
MISMATCH.append(FILENAME)
|
||||
|
||||
# Update Loading Bar
|
||||
with LOCK:
|
||||
if PBAR is not None:
|
||||
PBAR.update(len(tItem))
|
||||
else:
|
||||
# Ensure we're passing a single string to extractTranslation
|
||||
tList[index] = translatedText.replace("Placeholder Text", "")
|
||||
else:
|
||||
PBAR.write(f"AI Refused:{tItem}\n")
|
||||
|
||||
# Combine if multilist
|
||||
if isinstance(tList[0], list):
|
||||
tList = [t for sublist in tList for t in sublist]
|
||||
|
||||
# Return
|
||||
if format == "json":
|
||||
return [tList, totalTokens]
|
||||
else:
|
||||
return [tList[0], totalTokens]
|
||||
else:
|
||||
return [text, [0, 0]]
|
||||
# Call the new shared translation function
|
||||
return sharedtranslateAI(
|
||||
text=text,
|
||||
history=history,
|
||||
fullPromptFlag=fullPromptFlag,
|
||||
config=TRANSLATION_CONFIG,
|
||||
filename=FILENAME,
|
||||
pbar=PBAR,
|
||||
lock=LOCK,
|
||||
mismatchList=MISMATCH
|
||||
)
|
||||
|
|
|
|||
451
modules/wolf.py
451
modules/wolf.py
|
|
@ -14,6 +14,7 @@ from colorama import Fore
|
|||
from dotenv import load_dotenv
|
||||
from retry import retry
|
||||
from tqdm import tqdm
|
||||
from util.translation import TranslationConfig, translateAI as sharedtranslateAI
|
||||
|
||||
# Open AI
|
||||
load_dotenv()
|
||||
|
|
@ -106,6 +107,18 @@ ARMORFLAG = False
|
|||
WEAPONFLAG = False
|
||||
SKILLFLAG = False
|
||||
|
||||
# Initialize Translation Config
|
||||
TRANSLATION_CONFIG = TranslationConfig(
|
||||
model=MODEL,
|
||||
language=LANGUAGE,
|
||||
prompt=PROMPT,
|
||||
vocab=VOCAB,
|
||||
langRegex=LANGREGEX,
|
||||
batchSize=BATCHSIZE,
|
||||
maxHistory=MAXHISTORY,
|
||||
estimateMode=False # Will be set dynamically based on ESTIMATE
|
||||
)
|
||||
LEAVE = False
|
||||
|
||||
def handleWOLF(filename, estimate):
|
||||
global ESTIMATE, TOKENS, FILENAME
|
||||
|
|
@ -378,7 +391,7 @@ def searchCodes(events, pbar, jobList, filename):
|
|||
choiceString = f"Previous Line: {jaString}\n\nReply with the {LANGUAGE} translation of the dialogue choice"
|
||||
else:
|
||||
choiceString = f"Reply with the {LANGUAGE} translation of the dialogue choice"
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
choiceList,
|
||||
choiceString,
|
||||
True,
|
||||
|
|
@ -459,7 +472,7 @@ def searchCodes(events, pbar, jobList, filename):
|
|||
# list122[j] = list122[j].replace("\n", " ")
|
||||
|
||||
# Translate
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
list122,
|
||||
f"Reply with the {LANGUAGE} translation of the text",
|
||||
True,
|
||||
|
|
@ -495,7 +508,7 @@ def searchCodes(events, pbar, jobList, filename):
|
|||
# Japanese Text Only
|
||||
# if re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", jaString):
|
||||
# Translate
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
jaString,
|
||||
f"Maintain any newlines and Reply with the {LANGUAGE} translation of the text",
|
||||
True,
|
||||
|
|
@ -566,7 +579,7 @@ def searchCodes(events, pbar, jobList, filename):
|
|||
|
||||
# Translate Question
|
||||
question = codeList[i]['stringArgs'][2]
|
||||
response = translateGPT(question, "Reply with the {LANGUAGE} translation", False)
|
||||
response = translateAI(question, "Reply with the {LANGUAGE} translation", False)
|
||||
translatedText = response[0]
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
|
|
@ -577,9 +590,9 @@ def searchCodes(events, pbar, jobList, filename):
|
|||
|
||||
# Translate Choices
|
||||
if 'jaString' in locals() and jaString:
|
||||
response = translateGPT(choiceList, jaString, True)
|
||||
response = translateAI(choiceList, jaString, True)
|
||||
else:
|
||||
response = translateGPT(choiceList, f"Reply with the {LANGUAGE} translation of the dialogue choice", True)
|
||||
response = translateAI(choiceList, f"Reply with the {LANGUAGE} translation of the dialogue choice", True)
|
||||
choiceListTL = response[0]
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
|
|
@ -729,7 +742,7 @@ def searchCodes(events, pbar, jobList, filename):
|
|||
if len(stringList) > 0:
|
||||
pbar.total = len(stringList)
|
||||
pbar.refresh()
|
||||
response = translateGPT(stringList, textHistory, True)
|
||||
response = translateAI(stringList, textHistory, True)
|
||||
stringListTL = response[0]
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
|
|
@ -744,7 +757,7 @@ def searchCodes(events, pbar, jobList, filename):
|
|||
if len(list210) > 0:
|
||||
pbar.total = len(list210)
|
||||
pbar.refresh()
|
||||
response = translateGPT(list210, textHistory, True)
|
||||
response = translateAI(list210, textHistory, True)
|
||||
list210TL = response[0]
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
|
|
@ -759,7 +772,7 @@ def searchCodes(events, pbar, jobList, filename):
|
|||
if len(list250) > 0:
|
||||
pbar.total = len(list250)
|
||||
pbar.refresh()
|
||||
response = translateGPT(list250, textHistory, True)
|
||||
response = translateAI(list250, textHistory, True)
|
||||
list250TL = response[0]
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
|
|
@ -774,7 +787,7 @@ def searchCodes(events, pbar, jobList, filename):
|
|||
if len(list300) > 0:
|
||||
pbar.total = len(list300)
|
||||
pbar.refresh()
|
||||
response = translateGPT(list300, textHistory, True)
|
||||
response = translateAI(list300, textHistory, True)
|
||||
list300TL = response[0]
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
|
|
@ -792,7 +805,7 @@ def searchCodes(events, pbar, jobList, filename):
|
|||
pbar.refresh()
|
||||
|
||||
# Translate
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
list150,
|
||||
textHistory,
|
||||
True,
|
||||
|
|
@ -2011,7 +2024,7 @@ def searchDB(events, pbar, jobList, filename):
|
|||
pbar.refresh()
|
||||
|
||||
# Name
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
npcList[0],
|
||||
"Reply with only the " + LANGUAGE + " translation of the RPG enemy name",
|
||||
True,
|
||||
|
|
@ -2020,17 +2033,17 @@ def searchDB(events, pbar, jobList, filename):
|
|||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
# Desc 1
|
||||
response = translateGPT(npcList[1], "Reply with only the " + LANGUAGE + " translation", True)
|
||||
response = translateAI(npcList[1], "Reply with only the " + LANGUAGE + " translation", True)
|
||||
descListTL1 = response[0]
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
# Desc 2
|
||||
response = translateGPT(npcList[2], "Reply with only the " + LANGUAGE + " translation", True)
|
||||
response = translateAI(npcList[2], "Reply with only the " + LANGUAGE + " translation", True)
|
||||
descListTL2 = response[0]
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
# Desc 3
|
||||
response = translateGPT(npcList[3], "Reply with only the " + LANGUAGE + " translation", True)
|
||||
response = translateAI(npcList[3], "Reply with only the " + LANGUAGE + " translation", True)
|
||||
descListTL3 = response[0]
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
|
|
@ -2059,7 +2072,7 @@ def searchDB(events, pbar, jobList, filename):
|
|||
pbar.refresh()
|
||||
|
||||
# Name
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
scenarioList[0],
|
||||
"Reply with only the " + LANGUAGE + " translation",
|
||||
True,
|
||||
|
|
@ -2068,7 +2081,7 @@ def searchDB(events, pbar, jobList, filename):
|
|||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
# Desc 1
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
scenarioList[1],
|
||||
"reply with only the gender neutral " + LANGUAGE + " translation of the NPC name",
|
||||
True,
|
||||
|
|
@ -2077,7 +2090,7 @@ def searchDB(events, pbar, jobList, filename):
|
|||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
# Desc 2
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
scenarioList[2],
|
||||
"reply with only the gender neutral " + LANGUAGE + " translation of the NPC name",
|
||||
True,
|
||||
|
|
@ -2105,7 +2118,7 @@ def searchDB(events, pbar, jobList, filename):
|
|||
pbar.refresh()
|
||||
|
||||
# Name
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
optionsList[0],
|
||||
"Reply with only the " + LANGUAGE + " translation",
|
||||
True,
|
||||
|
|
@ -2114,7 +2127,7 @@ def searchDB(events, pbar, jobList, filename):
|
|||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
# Desc 1
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
optionsList[1],
|
||||
"reply with only the gender neutral " + LANGUAGE + " translation of the NPC name",
|
||||
True,
|
||||
|
|
@ -2123,7 +2136,7 @@ def searchDB(events, pbar, jobList, filename):
|
|||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
# Desc 2
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
optionsList[2],
|
||||
"reply with only the gender neutral " + LANGUAGE + " translation of the NPC name",
|
||||
True,
|
||||
|
|
@ -2151,22 +2164,22 @@ def searchDB(events, pbar, jobList, filename):
|
|||
pbar.refresh()
|
||||
|
||||
# Name
|
||||
response = translateGPT(itemList[0], "Reply with only the " + LANGUAGE + " translation", True)
|
||||
response = translateAI(itemList[0], "Reply with only the " + LANGUAGE + " translation", True)
|
||||
nameListTL = response[0]
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
# Desc 1
|
||||
response = translateGPT(itemList[1], "Reply with only the " + LANGUAGE + " translation", True)
|
||||
response = translateAI(itemList[1], "Reply with only the " + LANGUAGE + " translation", True)
|
||||
descListTL1 = response[0]
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
# Desc 2
|
||||
response = translateGPT(itemList[2], "Reply with only the " + LANGUAGE + " translation", True)
|
||||
response = translateAI(itemList[2], "Reply with only the " + LANGUAGE + " translation", True)
|
||||
descListTL2 = response[0]
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
# Desc 3
|
||||
response = translateGPT(itemList[3], "Reply with only the " + LANGUAGE + " translation", True)
|
||||
response = translateAI(itemList[3], "Reply with only the " + LANGUAGE + " translation", True)
|
||||
descListTL3 = response[0]
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
|
|
@ -2195,7 +2208,7 @@ def searchDB(events, pbar, jobList, filename):
|
|||
pbar.refresh()
|
||||
|
||||
# Name
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
armorList[0],
|
||||
"Reply with only the " + LANGUAGE + " translation of the NPC name",
|
||||
True,
|
||||
|
|
@ -2204,7 +2217,7 @@ def searchDB(events, pbar, jobList, filename):
|
|||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
# Desc 1
|
||||
response = translateGPT(armorList[1], "Reply with only the " + LANGUAGE + " translation", True)
|
||||
response = translateAI(armorList[1], "Reply with only the " + LANGUAGE + " translation", True)
|
||||
descListTL1 = response[0]
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
|
|
@ -2228,7 +2241,7 @@ def searchDB(events, pbar, jobList, filename):
|
|||
pbar.refresh()
|
||||
|
||||
# Name
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
enemyList[0],
|
||||
"Reply with only the " + LANGUAGE + " translation of the enemy NPC name",
|
||||
True,
|
||||
|
|
@ -2237,7 +2250,7 @@ def searchDB(events, pbar, jobList, filename):
|
|||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
# Desc 1
|
||||
response = translateGPT(enemyList[1], "Reply with only the " + LANGUAGE + " translation", True)
|
||||
response = translateAI(enemyList[1], "Reply with only the " + LANGUAGE + " translation", True)
|
||||
descListTL1 = response[0]
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
|
|
@ -2261,7 +2274,7 @@ def searchDB(events, pbar, jobList, filename):
|
|||
pbar.refresh()
|
||||
|
||||
# Name
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
weaponsList[0],
|
||||
"Reply with only the " + LANGUAGE + " translation of the RPG weapon name",
|
||||
True,
|
||||
|
|
@ -2270,12 +2283,12 @@ def searchDB(events, pbar, jobList, filename):
|
|||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
# Desc 1
|
||||
response = translateGPT(weaponsList[1], "", True)
|
||||
response = translateAI(weaponsList[1], "", True)
|
||||
descListTL1 = response[0]
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
# Desc 2
|
||||
response = translateGPT(weaponsList[2], "", True)
|
||||
response = translateAI(weaponsList[2], "", True)
|
||||
descListTL2 = response[0]
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
|
|
@ -2299,7 +2312,7 @@ def searchDB(events, pbar, jobList, filename):
|
|||
pbar.refresh()
|
||||
|
||||
# Name
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
skillList[0],
|
||||
"Reply with only the " + LANGUAGE + " translation of the RPG skill name",
|
||||
True,
|
||||
|
|
@ -2309,7 +2322,7 @@ def searchDB(events, pbar, jobList, filename):
|
|||
totalTokens[1] += response[1][1]
|
||||
|
||||
# Desc
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
skillList[1],
|
||||
"Reply with only the " + LANGUAGE + " translation of the RPG skill description",
|
||||
True,
|
||||
|
|
@ -2319,7 +2332,7 @@ def searchDB(events, pbar, jobList, filename):
|
|||
totalTokens[1] += response[1][1]
|
||||
|
||||
# Log 1
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
skillList[2],
|
||||
"reply with only the gender neutral "
|
||||
+ LANGUAGE
|
||||
|
|
@ -2331,7 +2344,7 @@ def searchDB(events, pbar, jobList, filename):
|
|||
totalTokens[1] += response[1][1]
|
||||
|
||||
# Log 2
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
skillList[3],
|
||||
"reply with only the gender neutral "
|
||||
+ LANGUAGE
|
||||
|
|
@ -2343,7 +2356,7 @@ def searchDB(events, pbar, jobList, filename):
|
|||
totalTokens[1] += response[1][1]
|
||||
|
||||
# Log 3
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
skillList[4],
|
||||
"reply with only the gender neutral "
|
||||
+ LANGUAGE
|
||||
|
|
@ -2380,7 +2393,7 @@ def searchDB(events, pbar, jobList, filename):
|
|||
pbar.refresh()
|
||||
|
||||
# Name
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
stateList[0],
|
||||
f"Reply with the {LANGUAGE} translation of the status effect.",
|
||||
True,
|
||||
|
|
@ -2390,13 +2403,13 @@ def searchDB(events, pbar, jobList, filename):
|
|||
totalTokens[1] += response[1][1]
|
||||
|
||||
# Desc 1
|
||||
response = translateGPT(stateList[1], "", True)
|
||||
response = translateAI(stateList[1], "", True)
|
||||
descListTL1 = response[0]
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
|
||||
# Log 1
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
stateList[2],
|
||||
f"Reply with the {LANGUAGE} translation of the status effect.",
|
||||
True,
|
||||
|
|
@ -2406,7 +2419,7 @@ def searchDB(events, pbar, jobList, filename):
|
|||
totalTokens[1] += response[1][1]
|
||||
|
||||
# Log 2
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
stateList[3],
|
||||
"reply with only the gender neutral "
|
||||
+ LANGUAGE
|
||||
|
|
@ -2418,7 +2431,7 @@ def searchDB(events, pbar, jobList, filename):
|
|||
totalTokens[1] += response[1][1]
|
||||
|
||||
# Log 3
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
stateList[4],
|
||||
"reply with only the gender neutral "
|
||||
+ LANGUAGE
|
||||
|
|
@ -2430,7 +2443,7 @@ def searchDB(events, pbar, jobList, filename):
|
|||
totalTokens[1] += response[1][1]
|
||||
|
||||
# Log 4
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
stateList[5],
|
||||
"reply with only the gender neutral "
|
||||
+ LANGUAGE
|
||||
|
|
@ -2442,7 +2455,7 @@ def searchDB(events, pbar, jobList, filename):
|
|||
totalTokens[1] += response[1][1]
|
||||
|
||||
# Log 1
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
stateList[6],
|
||||
"reply with only the gender neutral "
|
||||
+ LANGUAGE
|
||||
|
|
@ -2454,7 +2467,7 @@ def searchDB(events, pbar, jobList, filename):
|
|||
totalTokens[1] += response[1][1]
|
||||
|
||||
# Log 1
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
stateList[7],
|
||||
"reply with only the gender neutral "
|
||||
+ LANGUAGE
|
||||
|
|
@ -2501,7 +2514,7 @@ def searchDB(events, pbar, jobList, filename):
|
|||
pbar.refresh()
|
||||
|
||||
# Name
|
||||
response = translateGPT(dbNameList[0], "Reply with only the " + LANGUAGE + " translation", True)
|
||||
response = translateAI(dbNameList[0], "Reply with only the " + LANGUAGE + " translation", True)
|
||||
nameListTL = response[0]
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
|
|
@ -2525,7 +2538,7 @@ def searchDB(events, pbar, jobList, filename):
|
|||
pbar.refresh()
|
||||
|
||||
# Name
|
||||
response = translateGPT(dbValueList[0], "Reply with only the " + LANGUAGE + " translation", True)
|
||||
response = translateAI(dbValueList[0], "Reply with only the " + LANGUAGE + " translation", True)
|
||||
valueListTL = response[0]
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
|
|
@ -2577,7 +2590,7 @@ def getSpeaker(speaker):
|
|||
return [NAMESLIST[i][1], [0, 0]]
|
||||
|
||||
# Translate and Store Speaker
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
f"{speaker}",
|
||||
"Reply with the " + LANGUAGE + " translation of the NPC name.",
|
||||
False,
|
||||
|
|
@ -2588,7 +2601,7 @@ def getSpeaker(speaker):
|
|||
|
||||
# Retry if name doesn't translate for some reason
|
||||
if re.search(r"([a-zA-Z？?])", response[0]) == None:
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
f"{speaker}",
|
||||
"Reply with the " + LANGUAGE + " translation of the NPC name.",
|
||||
False,
|
||||
|
|
@ -2601,328 +2614,24 @@ def getSpeaker(speaker):
|
|||
return response
|
||||
return [speaker, [0, 0]]
|
||||
|
||||
|
||||
def batchList(input_list, batch_size):
|
||||
if not isinstance(batch_size, int) or batch_size <= 0:
|
||||
raise ValueError("batch_size must be a positive integer")
|
||||
|
||||
return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)]
|
||||
|
||||
|
||||
def parseVocabWithCategories(vocabText):
|
||||
"""Parse vocabulary text and extract terms with their categories."""
|
||||
pairs = []
|
||||
seen = set()
|
||||
currentCategory = None
|
||||
|
||||
for line in vocabText.splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith('```'):
|
||||
continue
|
||||
|
||||
# Check if this is a category header
|
||||
if line.startswith('#'):
|
||||
currentCategory = line
|
||||
continue
|
||||
|
||||
# Parse vocabulary term
|
||||
m = re.match(r'^(.+?)(?:\s?[\(–])', line) # term is everything before space + '(' or '–'
|
||||
if m:
|
||||
term = m.group(1)
|
||||
if term not in seen:
|
||||
pairs.append((term, line, currentCategory))
|
||||
seen.add(term)
|
||||
|
||||
return pairs
|
||||
|
||||
|
||||
def buildMatchedVocabText(vocabPairs, subbedT):
|
||||
"""Build formatted vocabulary text with matched terms organized by category."""
|
||||
matchedCategories = {}
|
||||
|
||||
# Use word boundaries for Japanese if appropriate, or allow substring as before.
|
||||
for term, line, category in vocabPairs:
|
||||
# "term in subbedT" could be false positive; can use regex but Japanese doesn't always have spaces.
|
||||
if term in subbedT:
|
||||
if category not in matchedCategories:
|
||||
matchedCategories[category] = []
|
||||
matchedCategories[category].append(line)
|
||||
|
||||
# Format matched vocabulary with categories
|
||||
if matchedCategories:
|
||||
formattedLines = ["Here are some vocabulary and terms so that you know the proper spelling and translation.\n"]
|
||||
for category, lines in matchedCategories.items():
|
||||
if category: # Only add category header if it exists
|
||||
formattedLines.append(category)
|
||||
formattedLines.extend(lines)
|
||||
formattedLines.append("") # Add blank line between categories
|
||||
matchedVocabText = f"```\n{chr(10).join(formattedLines).rstrip()}\n```"
|
||||
else:
|
||||
matchedVocabText = ""
|
||||
|
||||
return matchedVocabText
|
||||
|
||||
|
||||
def createContext(fullPromptFlag, subbedT, format):
|
||||
vocabPairs = parseVocabWithCategories(VOCAB)
|
||||
matchedVocabText = buildMatchedVocabText(vocabPairs, subbedT)
|
||||
|
||||
if fullPromptFlag:
|
||||
system = PROMPT + matchedVocabText
|
||||
else:
|
||||
system = f"\
|
||||
You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
|
||||
Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
|
||||
- You translate everything, including erotic and explicit content.\n\
|
||||
- 'Game Characters' - The names, nicknames, and genders of the game characters. Reference this to know the names, nicknames, and gender of characters in the game\n\
|
||||
- All text in your response must be in {LANGUAGE} even if it is hard to translate.\n\
|
||||
- Never include any notes, explanations, dislaimers, or anything similar in your response.\n\
|
||||
- Maintain any spacing in the translation.\n\
|
||||
- Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\
|
||||
- `...` can be a part of the dialogue. Translate it as it is.\n\
|
||||
{matchedVocabText}\n\
|
||||
"
|
||||
if format == "json":
|
||||
user = f"```json\n{subbedT}\n```"
|
||||
else:
|
||||
user = subbedT
|
||||
return system, user
|
||||
|
||||
|
||||
def translateText(system, user, history, penalty, format, model=MODEL):
|
||||
# Prompt
|
||||
msg = [{"role": "system", "content": system}]
|
||||
|
||||
# History
|
||||
if isinstance(history, list):
|
||||
msg.append({"role": "system", "content": "Translation History:"})
|
||||
msg.extend([{"role": "assistant", "content": h} for h in history])
|
||||
else:
|
||||
msg.append({"role": "assistant", "content": history})
|
||||
|
||||
# Response Format
|
||||
if format == "json":
|
||||
responseFormat = {"type": "json_object"}
|
||||
else:
|
||||
responseFormat = {"type": "text"}
|
||||
|
||||
# Content to TL
|
||||
msg.append({"role": "user", "content": f"{user}"})
|
||||
response = openai.chat.completions.create(
|
||||
temperature=0,
|
||||
frequency_penalty=penalty,
|
||||
model=model,
|
||||
response_format=responseFormat,
|
||||
messages=msg,
|
||||
)
|
||||
return response
|
||||
|
||||
|
||||
def cleanTranslatedText(translatedText):
|
||||
placeholders = {
|
||||
f"{LANGUAGE} Translation: ": "",
|
||||
"Translation: ": "",
|
||||
"っ": "",
|
||||
"〜": "~",
|
||||
"ッ": "",
|
||||
"。": ".",
|
||||
"「": '\\"',
|
||||
"」": '\\"',
|
||||
"- ": "-",
|
||||
"—": "―",
|
||||
"】": "]",
|
||||
"【": "[",
|
||||
"é": "e",
|
||||
"this guy": "this bastard",
|
||||
"This guy": "This bastard",
|
||||
"Placeholder Text": "",
|
||||
"```json": "",
|
||||
"```": "",
|
||||
# Add more replacements as needed
|
||||
}
|
||||
for target, replacement in placeholders.items():
|
||||
translatedText = translatedText.replace(target, replacement)
|
||||
|
||||
# Remove Repeating Characters
|
||||
pattern = re.compile(r"(.)\s*\1(?:\s*\1){" + str(20 - 1) + r",}")
|
||||
translatedText = pattern.sub(lambda match: match.group(0).replace(" ", "")[:20], translatedText)
|
||||
|
||||
# Elongate Long Dashes (Since GPT Ignores them...)
|
||||
translatedText = elongateCharacters(translatedText)
|
||||
return translatedText
|
||||
|
||||
|
||||
def elongateCharacters(text):
|
||||
# Define a pattern to match one character followed by one or more `ー` characters
|
||||
# Using a positive lookbehind assertion to capture the preceding character
|
||||
pattern = r"(?<=(.))ー+"
|
||||
|
||||
# Define a replacement function that elongates the captured character
|
||||
def repl(match):
|
||||
char = match.group(1) # The character before the ー sequence
|
||||
count = len(match.group(0)) - 1 # Number of ー characters
|
||||
return char * count # Replace ー sequence with the character repeated
|
||||
|
||||
# Use re.sub() to replace the pattern in the text
|
||||
return re.sub(pattern, repl, text)
|
||||
|
||||
|
||||
def extractTranslation(translatedTextList, is_list):
|
||||
try:
|
||||
translatedTextList = re.sub(r'\\"+\"([^,\n}])', r'\\"\1', translatedTextList)
|
||||
translatedTextList = re.sub(r"(?<![\\])\"+(?![\n,])", r'"', translatedTextList)
|
||||
line_dict = json.loads(translatedTextList)
|
||||
# If it's a batch (i.e., list), extract with tags; otherwise, return the single item.
|
||||
string_list = list(line_dict.values())
|
||||
if is_list:
|
||||
return string_list
|
||||
else:
|
||||
return string_list[0]
|
||||
|
||||
except Exception as e:
|
||||
PBAR.write(f"extractTranslation Error: {e} on String {translatedTextList}")
|
||||
return None
|
||||
|
||||
|
||||
def countTokens(system, user, history):
|
||||
inputTotalTokens = 0
|
||||
outputTotalTokens = 0
|
||||
enc = tiktoken.encoding_for_model("gpt-4")
|
||||
|
||||
# Input
|
||||
if isinstance(history, list):
|
||||
for line in history:
|
||||
inputTotalTokens += len(enc.encode(line))
|
||||
else:
|
||||
inputTotalTokens += len(enc.encode(history))
|
||||
inputTotalTokens += len(enc.encode(system))
|
||||
inputTotalTokens += len(enc.encode(user))
|
||||
|
||||
# Output
|
||||
outputTotalTokens += round(len(enc.encode(user)) * 2.5)
|
||||
|
||||
return [inputTotalTokens, outputTotalTokens]
|
||||
|
||||
|
||||
@retry(exceptions=Exception, tries=5, delay=5)
|
||||
def translateGPT(text, history, fullPromptFlag):
|
||||
def translateAI(text, history, fullPromptFlag):
|
||||
"""
|
||||
Legacy wrapper function for the new shared translation utility.
|
||||
This maintains compatibility with existing code while using the new shared implementation.
|
||||
"""
|
||||
global PBAR, MISMATCH, FILENAME
|
||||
if text:
|
||||
with open("log/translationHistory.txt", "a+", encoding="utf-8") as logFile:
|
||||
mismatch = False
|
||||
totalTokens = [0, 0]
|
||||
if isinstance(text, list):
|
||||
format = "json"
|
||||
tList = batchList(text, BATCHSIZE)
|
||||
else:
|
||||
format = "text"
|
||||
tList = [text]
|
||||
|
||||
for index, tItem in enumerate(tList):
|
||||
# Things to Check before starting translation
|
||||
if not re.search(LANGREGEX, str(tItem)):
|
||||
if PBAR is not None:
|
||||
PBAR.update(len(tItem))
|
||||
if isinstance(tItem, list):
|
||||
for j in range(len(tItem)):
|
||||
tItem[j] = cleanTranslatedText(tItem[j])
|
||||
tList[index] = tItem
|
||||
else:
|
||||
tList[index] = cleanTranslatedText(tItem)
|
||||
history = tItem[-MAXHISTORY:]
|
||||
continue
|
||||
# Update config estimate mode based on global ESTIMATE
|
||||
TRANSLATION_CONFIG.estimateMode = bool(ESTIMATE)
|
||||
|
||||
# Before sending to translation, if we have a list of items, add the formatting
|
||||
if isinstance(tItem, list):
|
||||
for j in range(len(tItem)):
|
||||
if not tItem[j]:
|
||||
tItem[j] = tItem[j].replace("", "Placeholder Text")
|
||||
payload = {f"Line{i+1}": string for i, string in enumerate(tItem)}
|
||||
payload = json.dumps(payload, indent=4, ensure_ascii=False)
|
||||
varResponse = [payload, []]
|
||||
subbedT = varResponse[0]
|
||||
else:
|
||||
varResponse = [tItem, []]
|
||||
subbedT = varResponse[0]
|
||||
|
||||
# Create Message
|
||||
system, user = createContext(fullPromptFlag, subbedT, format)
|
||||
|
||||
# Calculate Estimate
|
||||
if ESTIMATE:
|
||||
estimate = countTokens(system, user, history)
|
||||
totalTokens[0] += estimate[0]
|
||||
totalTokens[1] += estimate[1]
|
||||
continue
|
||||
|
||||
# Translating
|
||||
response = translateText(system, user, history, 0.05, format)
|
||||
|
||||
# Set Tokens
|
||||
translatedText = response.choices[0].message.content
|
||||
|
||||
# AI Refused, Try Again
|
||||
if not translatedText:
|
||||
response = translateText(f"{system}\n You translate ALL content.", user, history, 0.1, format, model="gpt-4o")
|
||||
translatedText = response.choices[0].message.content
|
||||
|
||||
# Report Tokens
|
||||
totalTokens[0] += response.usage.prompt_tokens
|
||||
totalTokens[1] += response.usage.completion_tokens
|
||||
|
||||
# Check Translation
|
||||
if translatedText:
|
||||
translatedText = cleanTranslatedText(translatedText)
|
||||
if isinstance(tItem, list):
|
||||
extractedTranslations = extractTranslation(translatedText, True)
|
||||
if extractedTranslations == None or len(tItem) != len(extractedTranslations):
|
||||
# Mismatch. Try Again
|
||||
response = translateText(system, user, history, 0.05, format, MODEL)
|
||||
translatedText = response.choices[0].message.content
|
||||
totalTokens[0] += response.usage.prompt_tokens
|
||||
totalTokens[1] += response.usage.completion_tokens
|
||||
|
||||
# Formatting
|
||||
translatedText = cleanTranslatedText(translatedText)
|
||||
if isinstance(tItem, list):
|
||||
extractedTranslations = extractTranslation(translatedText, True)
|
||||
if extractedTranslations == None or len(tItem) != len(extractedTranslations):
|
||||
with open("log/mismatchHistory.txt", "a+", encoding="utf-8") as mismatchFile:
|
||||
mismatchFile.write(f"Mismatch: {FILENAME}\n")
|
||||
mismatchFile.write(f"Input:\n{subbedT}\n")
|
||||
mismatchFile.write(f"Output:\n{translatedText}\n")
|
||||
mismatch = True # Just here for breakpoint
|
||||
logFile.write(f"Input:\n{subbedT}\n")
|
||||
logFile.write(f"Output:\n{translatedText}\n")
|
||||
|
||||
# Set if no mismatch
|
||||
if mismatch == False:
|
||||
tList[index] = extractedTranslations
|
||||
history = extractedTranslations[-MAXHISTORY:] # Update history if we have a list
|
||||
else:
|
||||
history = text[-MAXHISTORY:]
|
||||
mismatch = False
|
||||
if FILENAME not in MISMATCH:
|
||||
MISMATCH.append(FILENAME)
|
||||
|
||||
# Update Loading Bar
|
||||
with LOCK:
|
||||
if PBAR is not None:
|
||||
PBAR.update(len(tItem))
|
||||
else:
|
||||
# Ensure we're passing a single string to extractTranslation
|
||||
tList[index] = translatedText.replace("Placeholder Text", "")
|
||||
else:
|
||||
PBAR.write(f"AI Refused:{tItem}\n")
|
||||
|
||||
# Combine if multilist
|
||||
if isinstance(tList[0], list):
|
||||
tList = [t for sublist in tList for t in sublist]
|
||||
|
||||
# Return
|
||||
if format == "json":
|
||||
return [tList, totalTokens]
|
||||
else:
|
||||
return [tList[0], totalTokens]
|
||||
else:
|
||||
return [text, [0, 0]]
|
||||
# Call the new shared translation function
|
||||
return sharedtranslateAI(
|
||||
text=text,
|
||||
history=history,
|
||||
fullPromptFlag=fullPromptFlag,
|
||||
config=TRANSLATION_CONFIG,
|
||||
filename=FILENAME,
|
||||
pbar=PBAR,
|
||||
lock=LOCK,
|
||||
mismatchList=MISMATCH
|
||||
)
|
||||
|
|
|
|||
361
modules/wolf2.py
361
modules/wolf2.py
|
|
@ -13,6 +13,7 @@ from colorama import Fore
|
|||
from dotenv import load_dotenv
|
||||
from retry import retry
|
||||
from tqdm import tqdm
|
||||
from util.translation import TranslationConfig, translateAI as sharedtranslateAI
|
||||
|
||||
# Open AI
|
||||
load_dotenv()
|
||||
|
|
@ -74,6 +75,18 @@ else:
|
|||
BATCHSIZE = int(os.getenv("batchsize"))
|
||||
FREQUENCY_PENALTY = float(os.getenv("frequency_penalty"))
|
||||
|
||||
# Initialize Translation Config
|
||||
TRANSLATION_CONFIG = TranslationConfig(
|
||||
model=MODEL,
|
||||
language=LANGUAGE,
|
||||
prompt=PROMPT,
|
||||
vocab=VOCAB,
|
||||
langRegex=LANGREGEX,
|
||||
batchSize=BATCHSIZE,
|
||||
maxHistory=MAXHISTORY,
|
||||
estimateMode=False # Will be set dynamically based on ESTIMATE
|
||||
)
|
||||
LEAVE = False
|
||||
|
||||
def handleWOLF2(filename, estimate):
|
||||
global ESTIMATE
|
||||
|
|
@ -210,7 +223,7 @@ def translateWOLF(data, translatedList, pbar, filename):
|
|||
i += 1
|
||||
|
||||
# Translate
|
||||
response = translateGPT(choiceList, "This will be a dialogue option", True)
|
||||
response = translateAI(choiceList, "This will be a dialogue option", True)
|
||||
tokens[0] += response[1][0]
|
||||
tokens[1] += response[1][1]
|
||||
choiceListTL = response[0]
|
||||
|
|
@ -294,7 +307,7 @@ def translateWOLF(data, translatedList, pbar, filename):
|
|||
pbar.refresh()
|
||||
|
||||
# Translate
|
||||
response = translateGPT(stringList, "", True)
|
||||
response = translateAI(stringList, "", True)
|
||||
tokens[0] += response[1][0]
|
||||
tokens[1] += response[1][1]
|
||||
translatedList = response[0]
|
||||
|
|
@ -324,7 +337,7 @@ def getSpeaker(speaker):
|
|||
return [NAMESLIST[i][1], [0, 0]]
|
||||
|
||||
# Translate and Store Speaker
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
f"{speaker}",
|
||||
"Reply with the " + LANGUAGE + " translation of the NPC name.",
|
||||
False,
|
||||
|
|
@ -335,7 +348,7 @@ def getSpeaker(speaker):
|
|||
|
||||
# Retry if name doesn't translate for some reason
|
||||
if re.search(r"([a-zA-Z？?])", response[0]) == None:
|
||||
response = translateGPT(
|
||||
response = translateAI(
|
||||
f"{speaker}",
|
||||
"Reply with the " + LANGUAGE + " translation of the NPC name.",
|
||||
False,
|
||||
|
|
@ -348,328 +361,24 @@ def getSpeaker(speaker):
|
|||
return response
|
||||
return [speaker, [0, 0]]
|
||||
|
||||
|
||||
def batchList(input_list, batch_size):
|
||||
if not isinstance(batch_size, int) or batch_size <= 0:
|
||||
raise ValueError("batch_size must be a positive integer")
|
||||
|
||||
return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)]
|
||||
|
||||
|
||||
def parseVocabWithCategories(vocabText):
|
||||
"""Parse vocabulary text and extract terms with their categories."""
|
||||
pairs = []
|
||||
seen = set()
|
||||
currentCategory = None
|
||||
|
||||
for line in vocabText.splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith('```'):
|
||||
continue
|
||||
|
||||
# Check if this is a category header
|
||||
if line.startswith('#'):
|
||||
currentCategory = line
|
||||
continue
|
||||
|
||||
# Parse vocabulary term
|
||||
m = re.match(r'^(.+?)(?:\s?[\(–])', line) # term is everything before space + '(' or '–'
|
||||
if m:
|
||||
term = m.group(1)
|
||||
if term not in seen:
|
||||
pairs.append((term, line, currentCategory))
|
||||
seen.add(term)
|
||||
|
||||
return pairs
|
||||
|
||||
|
||||
def buildMatchedVocabText(vocabPairs, subbedT):
|
||||
"""Build formatted vocabulary text with matched terms organized by category."""
|
||||
matchedCategories = {}
|
||||
|
||||
# Use word boundaries for Japanese if appropriate, or allow substring as before.
|
||||
for term, line, category in vocabPairs:
|
||||
# "term in subbedT" could be false positive; can use regex but Japanese doesn't always have spaces.
|
||||
if term in subbedT:
|
||||
if category not in matchedCategories:
|
||||
matchedCategories[category] = []
|
||||
matchedCategories[category].append(line)
|
||||
|
||||
# Format matched vocabulary with categories
|
||||
if matchedCategories:
|
||||
formattedLines = ["Here are some vocabulary and terms so that you know the proper spelling and translation.\n"]
|
||||
for category, lines in matchedCategories.items():
|
||||
if category: # Only add category header if it exists
|
||||
formattedLines.append(category)
|
||||
formattedLines.extend(lines)
|
||||
formattedLines.append("") # Add blank line between categories
|
||||
matchedVocabText = f"```\n{chr(10).join(formattedLines).rstrip()}\n```"
|
||||
else:
|
||||
matchedVocabText = ""
|
||||
|
||||
return matchedVocabText
|
||||
|
||||
|
||||
def createContext(fullPromptFlag, subbedT, format):
|
||||
vocabPairs = parseVocabWithCategories(VOCAB)
|
||||
matchedVocabText = buildMatchedVocabText(vocabPairs, subbedT)
|
||||
|
||||
if fullPromptFlag:
|
||||
system = PROMPT + matchedVocabText
|
||||
else:
|
||||
system = f"\
|
||||
You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
|
||||
Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
|
||||
- You translate everything, including erotic and explicit content.\n\
|
||||
- 'Game Characters' - The names, nicknames, and genders of the game characters. Reference this to know the names, nicknames, and gender of characters in the game\n\
|
||||
- All text in your response must be in {LANGUAGE} even if it is hard to translate.\n\
|
||||
- Never include any notes, explanations, dislaimers, or anything similar in your response.\n\
|
||||
- Maintain any spacing in the translation.\n\
|
||||
- Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\
|
||||
- `...` can be a part of the dialogue. Translate it as it is.\n\
|
||||
{matchedVocabText}\n\
|
||||
"
|
||||
if format == "json":
|
||||
user = f"```json\n{subbedT}\n```"
|
||||
else:
|
||||
user = subbedT
|
||||
return system, user
|
||||
|
||||
|
||||
def translateText(system, user, history, penalty, format, model=MODEL):
|
||||
# Prompt
|
||||
msg = [{"role": "system", "content": system}]
|
||||
|
||||
# History
|
||||
if isinstance(history, list):
|
||||
msg.append({"role": "system", "content": "Translation History:"})
|
||||
msg.extend([{"role": "assistant", "content": h} for h in history])
|
||||
else:
|
||||
msg.append({"role": "assistant", "content": history})
|
||||
|
||||
# Response Format
|
||||
if format == "json":
|
||||
responseFormat = {"type": "json_object"}
|
||||
else:
|
||||
responseFormat = {"type": "text"}
|
||||
|
||||
# Content to TL
|
||||
msg.append({"role": "user", "content": f"{user}"})
|
||||
response = openai.chat.completions.create(
|
||||
temperature=0,
|
||||
frequency_penalty=penalty,
|
||||
model=model,
|
||||
response_format=responseFormat,
|
||||
messages=msg,
|
||||
)
|
||||
return response
|
||||
|
||||
|
||||
def cleanTranslatedText(translatedText):
|
||||
placeholders = {
|
||||
f"{LANGUAGE} Translation: ": "",
|
||||
"Translation: ": "",
|
||||
"っ": "",
|
||||
"〜": "~",
|
||||
"ッ": "",
|
||||
"。": ".",
|
||||
"「": '\\"',
|
||||
"」": '\\"',
|
||||
"- ": "-",
|
||||
"—": "―",
|
||||
"】": "]",
|
||||
"【": "[",
|
||||
"é": "e",
|
||||
"this guy": "this bastard",
|
||||
"This guy": "This bastard",
|
||||
"Placeholder Text": "",
|
||||
"```json": "",
|
||||
"```": "",
|
||||
# Add more replacements as needed
|
||||
}
|
||||
for target, replacement in placeholders.items():
|
||||
translatedText = translatedText.replace(target, replacement)
|
||||
|
||||
# Remove Repeating Characters
|
||||
pattern = re.compile(r"(.)\s*\1(?:\s*\1){" + str(20 - 1) + r",}")
|
||||
translatedText = pattern.sub(lambda match: match.group(0).replace(" ", "")[:20], translatedText)
|
||||
|
||||
# Elongate Long Dashes (Since GPT Ignores them...)
|
||||
translatedText = elongateCharacters(translatedText)
|
||||
return translatedText
|
||||
|
||||
|
||||
def elongateCharacters(text):
|
||||
# Define a pattern to match one character followed by one or more `ー` characters
|
||||
# Using a positive lookbehind assertion to capture the preceding character
|
||||
pattern = r"(?<=(.))ー+"
|
||||
|
||||
# Define a replacement function that elongates the captured character
|
||||
def repl(match):
|
||||
char = match.group(1) # The character before the ー sequence
|
||||
count = len(match.group(0)) - 1 # Number of ー characters
|
||||
return char * count # Replace ー sequence with the character repeated
|
||||
|
||||
# Use re.sub() to replace the pattern in the text
|
||||
return re.sub(pattern, repl, text)
|
||||
|
||||
|
||||
def extractTranslation(translatedTextList, is_list):
|
||||
try:
|
||||
translatedTextList = re.sub(r'\\"+\"([^,\n}])', r'\\"\1', translatedTextList)
|
||||
translatedTextList = re.sub(r"(?<![\\])\"+(?![\n,])", r'"', translatedTextList)
|
||||
line_dict = json.loads(translatedTextList)
|
||||
# If it's a batch (i.e., list), extract with tags; otherwise, return the single item.
|
||||
string_list = list(line_dict.values())
|
||||
if is_list:
|
||||
return string_list
|
||||
else:
|
||||
return string_list[0]
|
||||
|
||||
except Exception as e:
|
||||
PBAR.write(f"extractTranslation Error: {e} on String {translatedTextList}")
|
||||
return None
|
||||
|
||||
|
||||
def countTokens(system, user, history):
|
||||
inputTotalTokens = 0
|
||||
outputTotalTokens = 0
|
||||
enc = tiktoken.encoding_for_model("gpt-4")
|
||||
|
||||
# Input
|
||||
if isinstance(history, list):
|
||||
for line in history:
|
||||
inputTotalTokens += len(enc.encode(line))
|
||||
else:
|
||||
inputTotalTokens += len(enc.encode(history))
|
||||
inputTotalTokens += len(enc.encode(system))
|
||||
inputTotalTokens += len(enc.encode(user))
|
||||
|
||||
# Output
|
||||
outputTotalTokens += round(len(enc.encode(user)) * 2.5)
|
||||
|
||||
return [inputTotalTokens, outputTotalTokens]
|
||||
|
||||
|
||||
@retry(exceptions=Exception, tries=5, delay=5)
|
||||
def translateGPT(text, history, fullPromptFlag):
|
||||
def translateAI(text, history, fullPromptFlag):
|
||||
"""
|
||||
Legacy wrapper function for the new shared translation utility.
|
||||
This maintains compatibility with existing code while using the new shared implementation.
|
||||
"""
|
||||
global PBAR, MISMATCH, FILENAME
|
||||
if text:
|
||||
with open("log/translationHistory.txt", "a+", encoding="utf-8") as logFile:
|
||||
mismatch = False
|
||||
totalTokens = [0, 0]
|
||||
if isinstance(text, list):
|
||||
format = "json"
|
||||
tList = batchList(text, BATCHSIZE)
|
||||
else:
|
||||
format = "text"
|
||||
tList = [text]
|
||||
|
||||
for index, tItem in enumerate(tList):
|
||||
# Things to Check before starting translation
|
||||
if not re.search(LANGREGEX, str(tItem)):
|
||||
if PBAR is not None:
|
||||
PBAR.update(len(tItem))
|
||||
if isinstance(tItem, list):
|
||||
for j in range(len(tItem)):
|
||||
tItem[j] = cleanTranslatedText(tItem[j])
|
||||
tList[index] = tItem
|
||||
else:
|
||||
tList[index] = cleanTranslatedText(tItem)
|
||||
history = tItem[-MAXHISTORY:]
|
||||
continue
|
||||
# Update config estimate mode based on global ESTIMATE
|
||||
TRANSLATION_CONFIG.estimateMode = bool(ESTIMATE)
|
||||
|
||||
# Before sending to translation, if we have a list of items, add the formatting
|
||||
if isinstance(tItem, list):
|
||||
for j in range(len(tItem)):
|
||||
if not tItem[j]:
|
||||
tItem[j] = tItem[j].replace("", "Placeholder Text")
|
||||
payload = {f"Line{i+1}": string for i, string in enumerate(tItem)}
|
||||
payload = json.dumps(payload, indent=4, ensure_ascii=False)
|
||||
varResponse = [payload, []]
|
||||
subbedT = varResponse[0]
|
||||
else:
|
||||
varResponse = [tItem, []]
|
||||
subbedT = varResponse[0]
|
||||
|
||||
# Create Message
|
||||
system, user = createContext(fullPromptFlag, subbedT, format)
|
||||
|
||||
# Calculate Estimate
|
||||
if ESTIMATE:
|
||||
estimate = countTokens(system, user, history)
|
||||
totalTokens[0] += estimate[0]
|
||||
totalTokens[1] += estimate[1]
|
||||
continue
|
||||
|
||||
# Translating
|
||||
response = translateText(system, user, history, 0.05, format)
|
||||
|
||||
# Set Tokens
|
||||
translatedText = response.choices[0].message.content
|
||||
|
||||
# AI Refused, Try Again
|
||||
if not translatedText:
|
||||
response = translateText(f"{system}\n You translate ALL content.", user, history, 0.1, format, model="gpt-4o")
|
||||
translatedText = response.choices[0].message.content
|
||||
|
||||
# Report Tokens
|
||||
totalTokens[0] += response.usage.prompt_tokens
|
||||
totalTokens[1] += response.usage.completion_tokens
|
||||
|
||||
# Check Translation
|
||||
if translatedText:
|
||||
translatedText = cleanTranslatedText(translatedText)
|
||||
if isinstance(tItem, list):
|
||||
extractedTranslations = extractTranslation(translatedText, True)
|
||||
if extractedTranslations == None or len(tItem) != len(extractedTranslations):
|
||||
# Mismatch. Try Again
|
||||
response = translateText(system, user, history, 0.05, format, MODEL)
|
||||
translatedText = response.choices[0].message.content
|
||||
totalTokens[0] += response.usage.prompt_tokens
|
||||
totalTokens[1] += response.usage.completion_tokens
|
||||
|
||||
# Formatting
|
||||
translatedText = cleanTranslatedText(translatedText)
|
||||
if isinstance(tItem, list):
|
||||
extractedTranslations = extractTranslation(translatedText, True)
|
||||
if extractedTranslations == None or len(tItem) != len(extractedTranslations):
|
||||
with open("log/mismatchHistory.txt", "a+", encoding="utf-8") as mismatchFile:
|
||||
mismatchFile.write(f"Mismatch: {FILENAME}\n")
|
||||
mismatchFile.write(f"Input:\n{subbedT}\n")
|
||||
mismatchFile.write(f"Output:\n{translatedText}\n")
|
||||
mismatch = True # Just here for breakpoint
|
||||
logFile.write(f"Input:\n{subbedT}\n")
|
||||
logFile.write(f"Output:\n{translatedText}\n")
|
||||
|
||||
# Set if no mismatch
|
||||
if mismatch == False:
|
||||
tList[index] = extractedTranslations
|
||||
history = extractedTranslations[-MAXHISTORY:] # Update history if we have a list
|
||||
else:
|
||||
history = text[-MAXHISTORY:]
|
||||
mismatch = False
|
||||
if FILENAME not in MISMATCH:
|
||||
MISMATCH.append(FILENAME)
|
||||
|
||||
# Update Loading Bar
|
||||
with LOCK:
|
||||
if PBAR is not None:
|
||||
PBAR.update(len(tItem))
|
||||
else:
|
||||
# Ensure we're passing a single string to extractTranslation
|
||||
tList[index] = translatedText.replace("Placeholder Text", "")
|
||||
else:
|
||||
PBAR.write(f"AI Refused:{tItem}\n")
|
||||
|
||||
# Combine if multilist
|
||||
if isinstance(tList[0], list):
|
||||
tList = [t for sublist in tList for t in sublist]
|
||||
|
||||
# Return
|
||||
if format == "json":
|
||||
return [tList, totalTokens]
|
||||
else:
|
||||
return [tList[0], totalTokens]
|
||||
else:
|
||||
return [text, [0, 0]]
|
||||
# Call the new shared translation function
|
||||
return sharedtranslateAI(
|
||||
text=text,
|
||||
history=history,
|
||||
fullPromptFlag=fullPromptFlag,
|
||||
config=TRANSLATION_CONFIG,
|
||||
filename=FILENAME,
|
||||
pbar=PBAR,
|
||||
lock=LOCK,
|
||||
mismatchList=MISMATCH
|
||||
)
|
||||
|
|
|
|||
429
util/translation.py
Normal file
429
util/translation.py
Normal file
|
|
@ -0,0 +1,429 @@
|
|||
"""
|
||||
Shared translation utilities for DazedMTLTool
|
||||
This module provides a centralized translation function that can be used across all modules
|
||||
without relying on global variables.
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import json
|
||||
import tiktoken
|
||||
import openai
|
||||
from pathlib import Path
|
||||
from retry import retry
|
||||
|
||||
|
||||
class TranslationConfig:
|
||||
"""Configuration class to hold all translation settings"""
|
||||
|
||||
def __init__(self,
|
||||
model=None,
|
||||
language=None,
|
||||
prompt=None,
|
||||
vocab=None,
|
||||
langRegex=None,
|
||||
batchSize=None,
|
||||
maxHistory=10,
|
||||
estimateMode=False,
|
||||
logFilePath="log/translationHistory.txt",
|
||||
mismatchLogPath="log/mismatchHistory.txt"):
|
||||
|
||||
# Load from environment if not provided
|
||||
self.model = model or os.getenv("model")
|
||||
self.language = (language or os.getenv("language", "english")).capitalize()
|
||||
|
||||
# Load prompt and vocab files if not provided
|
||||
if prompt is None:
|
||||
try:
|
||||
self.prompt = Path("prompt.txt").read_text(encoding="utf-8")
|
||||
except FileNotFoundError:
|
||||
self.prompt = ""
|
||||
else:
|
||||
self.prompt = prompt
|
||||
|
||||
if vocab is None:
|
||||
try:
|
||||
self.vocab = Path("vocab.txt").read_text(encoding="utf-8")
|
||||
except FileNotFoundError:
|
||||
self.vocab = ""
|
||||
else:
|
||||
self.vocab = vocab
|
||||
|
||||
# Set language regex (default is Japanese)
|
||||
self.langRegex = langRegex or r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+"
|
||||
|
||||
# Set batch size based on model if not provided
|
||||
if batchSize is None:
|
||||
if "gpt-3.5" in self.model:
|
||||
self.batchSize = 10
|
||||
elif "gpt-4" in self.model:
|
||||
self.batchSize = 30
|
||||
elif "deepseek" in self.model:
|
||||
self.batchSize = 30
|
||||
else:
|
||||
# Try to get from environment, fallback to 10
|
||||
try:
|
||||
self.batchSize = int(os.getenv("batchsize", 10))
|
||||
except (ValueError, TypeError):
|
||||
self.batchSize = 10
|
||||
else:
|
||||
self.batchSize = batchSize
|
||||
|
||||
self.maxHistory = maxHistory
|
||||
self.estimateMode = estimateMode
|
||||
self.logFilePath = logFilePath
|
||||
self.mismatchLogPath = mismatchLogPath
|
||||
|
||||
|
||||
def batchList(inputList, batchSize):
|
||||
"""Split a list into batches of specified size"""
|
||||
if not isinstance(batchSize, int) or batchSize <= 0:
|
||||
raise ValueError("batchSize must be a positive integer")
|
||||
|
||||
return [inputList[i : i + batchSize] for i in range(0, len(inputList), batchSize)]
|
||||
|
||||
|
||||
def parseVocabWithCategories(vocabText):
|
||||
"""Parse vocabulary text and extract terms with their categories."""
|
||||
pairs = []
|
||||
seen = set()
|
||||
currentCategory = None
|
||||
|
||||
for line in vocabText.splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith('```'):
|
||||
continue
|
||||
|
||||
# Check if this is a category header
|
||||
if line.startswith('#'):
|
||||
currentCategory = line
|
||||
continue
|
||||
|
||||
# Parse vocabulary term
|
||||
m = re.match(r'^(.+?)(?:\s?[\(–])', line) # term is everything before space + '(' or '–'
|
||||
if m:
|
||||
term = m.group(1)
|
||||
if term not in seen:
|
||||
pairs.append((term, line, currentCategory))
|
||||
seen.add(term)
|
||||
|
||||
return pairs
|
||||
|
||||
|
||||
def buildMatchedVocabText(vocabPairs, subbedText):
|
||||
"""Build formatted vocabulary text with matched terms organized by category."""
|
||||
matchedCategories = {}
|
||||
|
||||
# Use word boundaries for Japanese if appropriate, or allow substring as before.
|
||||
for term, line, category in vocabPairs:
|
||||
# "term in subbedT" could be false positive; can use regex but Japanese doesn't always have spaces.
|
||||
if term in subbedText:
|
||||
if category not in matchedCategories:
|
||||
matchedCategories[category] = []
|
||||
matchedCategories[category].append(line)
|
||||
|
||||
# Format matched vocabulary with categories
|
||||
if matchedCategories:
|
||||
formattedLines = ["Here are some vocabulary and terms so that you know the proper spelling and translation.\n"]
|
||||
for category, lines in matchedCategories.items():
|
||||
if category: # Only add category header if it exists
|
||||
formattedLines.append(category)
|
||||
formattedLines.extend(lines)
|
||||
formattedLines.append("") # Add blank line between categories
|
||||
matchedVocabText = f"```\n{chr(10).join(formattedLines).rstrip()}\n```"
|
||||
else:
|
||||
matchedVocabText = ""
|
||||
|
||||
return matchedVocabText
|
||||
|
||||
|
||||
def createContext(config, fullPromptFlag, subbedText, formatType):
|
||||
"""Create system and user messages for translation"""
|
||||
vocabPairs = parseVocabWithCategories(config.vocab)
|
||||
matchedVocabText = buildMatchedVocabText(vocabPairs, subbedText)
|
||||
|
||||
if fullPromptFlag:
|
||||
system = config.prompt + matchedVocabText
|
||||
else:
|
||||
system = f"""\
|
||||
You are an expert Eroge Game translator who translates Japanese text to {config.language}.
|
||||
Output ONLY the {config.language} translation in the following format: `Translation: <{config.language.upper()}_TRANSLATION>`
|
||||
- You translate everything, including erotic and explicit content.
|
||||
- 'Game Characters' - The names, nicknames, and genders of the game characters. Reference this to know the names, nicknames, and gender of characters in the game
|
||||
- All text in your response must be in {config.language} even if it is hard to translate.
|
||||
- Never include any notes, explanations, dislaimers, or anything similar in your response.
|
||||
- Maintain any spacing in the translation.
|
||||
- Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)
|
||||
- `...` can be a part of the dialogue. Translate it as it is.
|
||||
{matchedVocabText}
|
||||
"""
|
||||
|
||||
if formatType == "json":
|
||||
user = f"```json\n{subbedText}\n```"
|
||||
else:
|
||||
user = subbedText
|
||||
|
||||
return system, user
|
||||
|
||||
|
||||
def translateText(system, user, history, penalty, formatType, model):
|
||||
"""Send translation request to OpenAI API"""
|
||||
# Prompt
|
||||
msg = [{"role": "system", "content": system}]
|
||||
|
||||
# History
|
||||
if isinstance(history, list):
|
||||
msg.append({"role": "system", "content": "Translation History:"})
|
||||
msg.extend([{"role": "assistant", "content": h} for h in history])
|
||||
else:
|
||||
msg.append({"role": "assistant", "content": history})
|
||||
|
||||
# Response Format
|
||||
if formatType == "json":
|
||||
responseFormat = {"type": "json_object"}
|
||||
else:
|
||||
responseFormat = {"type": "text"}
|
||||
|
||||
# Content to TL
|
||||
msg.append({"role": "user", "content": f"{user}"})
|
||||
response = openai.chat.completions.create(
|
||||
temperature=0,
|
||||
frequency_penalty=penalty,
|
||||
model=model,
|
||||
response_format=responseFormat,
|
||||
messages=msg,
|
||||
)
|
||||
return response
|
||||
|
||||
|
||||
def cleanTranslatedText(translatedText, language):
|
||||
"""Clean and format translated text"""
|
||||
placeholders = {
|
||||
f"{language} Translation: ": "",
|
||||
"Translation: ": "",
|
||||
"っ": "",
|
||||
"〜": "~",
|
||||
"ッ": "",
|
||||
"。": ".",
|
||||
"「": '\"',
|
||||
"」": '\"',
|
||||
"- ": "-",
|
||||
"—": "―",
|
||||
"】": "]",
|
||||
"【": "[",
|
||||
"é": "e",
|
||||
"this guy": "this bastard",
|
||||
"This guy": "This bastard",
|
||||
"Placeholder Text": "",
|
||||
"```json": "",
|
||||
"```": "",
|
||||
}
|
||||
|
||||
for target, replacement in placeholders.items():
|
||||
translatedText = translatedText.replace(target, replacement)
|
||||
|
||||
# Remove Repeating Characters
|
||||
pattern = re.compile(r"(.)\s*\1(?:\s*\1){" + str(20 - 1) + r",}")
|
||||
translatedText = pattern.sub(lambda match: match.group(0).replace(" ", "")[:20], translatedText)
|
||||
|
||||
# Elongate Long Dashes (Since GPT Ignores them...)
|
||||
translatedText = elongateCharacters(translatedText)
|
||||
return translatedText
|
||||
|
||||
|
||||
def elongateCharacters(text):
|
||||
"""Replace ー sequences with elongated characters"""
|
||||
# Define a pattern to match one character followed by two or more ー characters
|
||||
pattern = r"(?<=(.))ー{2,}"
|
||||
|
||||
# Define a replacement function that elongates the captured character
|
||||
def repl(match):
|
||||
char = match.group(1) # The character before the ー sequence
|
||||
count = len(match.group(0)) - 1 # Number of ー characters
|
||||
return char * count # Replace ー sequence with the character repeated
|
||||
|
||||
# Use re.sub() to replace the pattern in the text
|
||||
return re.sub(pattern, repl, text)
|
||||
|
||||
|
||||
def extractTranslation(translatedTextList, isList, pbar=None):
|
||||
"""Extract translation from JSON response"""
|
||||
try:
|
||||
translatedTextList = re.sub(r'\\"+\"([^,\n}])', r'\\"\1', translatedTextList)
|
||||
translatedTextList = re.sub(r"(?<![\\])\"+(?![\n,])", r'"', translatedTextList)
|
||||
lineDict = json.loads(translatedTextList)
|
||||
# If it's a batch (i.e., list), extract with tags; otherwise, return the single item.
|
||||
stringList = list(lineDict.values())
|
||||
if isList:
|
||||
return stringList
|
||||
else:
|
||||
return stringList[0]
|
||||
|
||||
except Exception as e:
|
||||
if pbar:
|
||||
pbar.write(f"extractTranslation Error: {e} on String {translatedTextList}")
|
||||
return None
|
||||
|
||||
|
||||
def countTokens(system, user, history):
|
||||
"""Count tokens for cost estimation"""
|
||||
inputTotalTokens = 0
|
||||
outputTotalTokens = 0
|
||||
enc = tiktoken.encoding_for_model("gpt-4")
|
||||
|
||||
# Input
|
||||
if isinstance(history, list):
|
||||
for line in history:
|
||||
inputTotalTokens += len(enc.encode(line))
|
||||
else:
|
||||
inputTotalTokens += len(enc.encode(history))
|
||||
inputTotalTokens += len(enc.encode(system))
|
||||
inputTotalTokens += len(enc.encode(user))
|
||||
|
||||
# Output
|
||||
outputTotalTokens += round(len(enc.encode(user)) * 2.5)
|
||||
|
||||
return [inputTotalTokens, outputTotalTokens]
|
||||
|
||||
|
||||
@retry(exceptions=Exception, tries=5, delay=5)
|
||||
def translateAI(text, history, fullPromptFlag, config, filename=None, pbar=None, lock=None, mismatchList=None):
|
||||
"""
|
||||
Main translation function that can be used across all modules.
|
||||
|
||||
Args:
|
||||
text: Text to translate (string or list)
|
||||
history: Translation history for context
|
||||
fullPromptFlag: Whether to use full prompt or simplified version
|
||||
config: TranslationConfig instance with all settings
|
||||
filename: Current filename being processed (for logging)
|
||||
pbar: Progress bar instance (optional)
|
||||
lock: Threading lock (optional)
|
||||
mismatchList: List to track mismatched files (optional)
|
||||
|
||||
Returns:
|
||||
[translatedText, totalTokens]
|
||||
"""
|
||||
if not text:
|
||||
return [text, [0, 0]]
|
||||
|
||||
with open(config.logFilePath, "a+", encoding="utf-8") as logFile:
|
||||
mismatch = False
|
||||
totalTokens = [0, 0]
|
||||
|
||||
if isinstance(text, list):
|
||||
formatType = "json"
|
||||
tList = batchList(text, config.batchSize)
|
||||
else:
|
||||
formatType = "text"
|
||||
tList = [text]
|
||||
|
||||
for index, tItem in enumerate(tList):
|
||||
# Check if text contains target language
|
||||
if not re.search(config.langRegex, str(tItem)):
|
||||
if pbar is not None:
|
||||
pbar.update(len(tItem) if isinstance(tItem, list) else 1)
|
||||
if isinstance(tItem, list):
|
||||
for j in range(len(tItem)):
|
||||
tItem[j] = cleanTranslatedText(tItem[j], config.language)
|
||||
tList[index] = tItem
|
||||
else:
|
||||
tList[index] = cleanTranslatedText(tItem, config.language)
|
||||
history = tItem[-config.maxHistory:] if isinstance(tItem, list) else tItem
|
||||
continue
|
||||
|
||||
# Format for translation
|
||||
if isinstance(tItem, list):
|
||||
for j in range(len(tItem)):
|
||||
if not tItem[j]:
|
||||
tItem[j] = tItem[j].replace("", "Placeholder Text")
|
||||
payload = {f"Line{i+1}": string for i, string in enumerate(tItem)}
|
||||
payload = json.dumps(payload, indent=4, ensure_ascii=False)
|
||||
subbedT = payload
|
||||
else:
|
||||
subbedT = tItem
|
||||
|
||||
# Create context
|
||||
system, user = createContext(config, fullPromptFlag, subbedT, formatType)
|
||||
|
||||
# Calculate estimate if in estimate mode
|
||||
if config.estimateMode:
|
||||
estimate = countTokens(system, user, history)
|
||||
totalTokens[0] += estimate[0]
|
||||
totalTokens[1] += estimate[1]
|
||||
continue
|
||||
|
||||
# Translate
|
||||
response = translateText(system, user, history, 0.05, formatType, config.model)
|
||||
translatedText = response.choices[0].message.content
|
||||
|
||||
# Retry if AI refused
|
||||
if not translatedText:
|
||||
response = translateText(
|
||||
f"{system}\n You translate ALL content.",
|
||||
user, history, 0.1, formatType, "gpt-4o"
|
||||
)
|
||||
translatedText = response.choices[0].message.content
|
||||
|
||||
# Update token count
|
||||
totalTokens[0] += response.usage.prompt_tokens
|
||||
totalTokens[1] += response.usage.completion_tokens
|
||||
|
||||
# Process translation result
|
||||
if translatedText:
|
||||
translatedText = cleanTranslatedText(translatedText, config.language)
|
||||
|
||||
if isinstance(tItem, list):
|
||||
extractedTranslations = extractTranslation(translatedText, True, pbar)
|
||||
if extractedTranslations is None or len(tItem) != len(extractedTranslations):
|
||||
# Mismatch, try again
|
||||
response = translateText(system, user, history, 0.05, formatType, config.model)
|
||||
translatedText = response.choices[0].message.content
|
||||
totalTokens[0] += response.usage.prompt_tokens
|
||||
totalTokens[1] += response.usage.completion_tokens
|
||||
|
||||
# Format and check again
|
||||
translatedText = cleanTranslatedText(translatedText, config.language)
|
||||
if isinstance(tItem, list):
|
||||
extractedTranslations = extractTranslation(translatedText, True, pbar)
|
||||
if extractedTranslations is None or len(tItem) != len(extractedTranslations):
|
||||
# Log mismatch
|
||||
with open(config.mismatchLogPath, "a+", encoding="utf-8") as mismatchFile:
|
||||
mismatchFile.write(f"Mismatch: {filename}\n")
|
||||
mismatchFile.write(f"Input:\n{subbedT}\n")
|
||||
mismatchFile.write(f"Output:\n{translatedText}\n")
|
||||
mismatch = True
|
||||
|
||||
logFile.write(f"Input:\n{subbedT}\n")
|
||||
logFile.write(f"Output:\n{translatedText}\n")
|
||||
|
||||
# Set results if no mismatch
|
||||
if not mismatch:
|
||||
tList[index] = extractedTranslations
|
||||
history = extractedTranslations[-config.maxHistory:]
|
||||
else:
|
||||
history = text[-config.maxHistory:] if isinstance(text, list) else text
|
||||
mismatch = False
|
||||
if filename and mismatchList is not None and filename not in mismatchList:
|
||||
mismatchList.append(filename)
|
||||
|
||||
# Update progress bar
|
||||
if lock and pbar is not None:
|
||||
with lock:
|
||||
pbar.update(len(tItem))
|
||||
else:
|
||||
# Single string translation
|
||||
tList[index] = translatedText.replace("Placeholder Text", "")
|
||||
else:
|
||||
if pbar:
|
||||
pbar.write(f"AI Refused: {tItem}\n")
|
||||
|
||||
# Combine if multilist
|
||||
if isinstance(tList[0], list):
|
||||
tList = [t for sublist in tList for t in sublist]
|
||||
|
||||
# Return result
|
||||
if formatType == "json":
|
||||
return [tList, totalTokens]
|
||||
else:
|
||||
return [tList[0], totalTokens]
|
||||
Loading…
Reference in a new issue