Update Lune Translation function

This commit is contained in:
DazedAnon 2024-10-03 10:57:56 -05:00
parent 69fe538a0e
commit 3fb50fbbf3
4 changed files with 192 additions and 191 deletions

View file

@ -41,6 +41,8 @@ BRFLAG = False # If the game uses <br> instead
FIXTEXTWRAP = True # Overwrites textwrap
IGNORETLTEXT = False # Ignores all translated text.
MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
PBAR = None
FILENAME = None
# tqdm Globals
BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
@ -61,8 +63,9 @@ elif "gpt-4" in MODEL:
def handleLune(filename, estimate):
global ESTIMATE, totalTokens
global FILENAME, ESTIMATE, totalTokens
ESTIMATE = estimate
FILENAME = filename
if estimate:
start = time.time()
@ -176,6 +179,8 @@ def parseJSON(data, filename):
def translateJSON(data, pbar):
global PBAR
PBAR = pbar
textHistory = []
batch = []
maxHistory = MAXHISTORY
@ -306,85 +311,60 @@ def translateJSON(data, pbar):
# Save some money and enter the character before translation
def getSpeaker(speaker):
match speaker:
case "セレナ":
return ["Serena", [0, 0]]
case "レナ":
return ["Rena", [0, 0]]
case "フィルス":
return ["Phils", [0, 0]]
case "レイン":
return ["Meryl", [0, 0]]
case "ファイン":
return ["Fine", [0, 0]]
case "":
return ["", [0, 0]]
case _:
return translateGPT(
speaker,
"Reply with only the " + LANGUAGE + " translation of the NPC name.",
False,
)
# Store Speaker
if speaker not in str(NAMESLIST):
response = translateGPT(
speaker,
"Reply with the " + LANGUAGE + " translation of the NPC name.",
False,
)
response[0] = response[0].title()
response[0] = response[0].replace("'S", "'s")
# Retry if name doesn't translate for some reason
if re.search(r"([a-zA-Z?])", response[0]) == None:
response = translateGPT(
speaker,
"Reply with the " + LANGUAGE + " translation of the NPC name.",
False,
)
response[0] = response[0].title()
response[0] = response[0].replace("'S", "'s")
speakerList = [speaker, response[0]]
NAMESLIST.append(speakerList)
return response
# Find Speaker
else:
for i in range(len(NAMESLIST)):
if speaker == NAMESLIST[i][0]:
return [NAMESLIST[i][1], [0, 0]]
return [speaker, [0, 0]]
def subVars(jaString):
jaString = jaString.replace("\u3000", " ")
# Nested
count = 0
nestedList = re.findall(r"[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]", jaString)
nestedList = set(nestedList)
if len(nestedList) != 0:
for icon in nestedList:
jaString = jaString.replace(icon, "{Nested_" + str(count) + "}")
count += 1
# Icons
count = 0
iconList = re.findall(r"[\\]+[iIkKwWaA]+\[[0-9]+\]", jaString)
iconList = set(iconList)
if len(iconList) != 0:
for icon in iconList:
jaString = jaString.replace(icon, "{Ascii_" + str(count) + "}")
count += 1
# Colors
count = 0
colorList = re.findall(r"[\\]+[cC]\[[0-9]+\]", jaString)
colorList = set(colorList)
if len(colorList) != 0:
for color in colorList:
jaString = jaString.replace(color, "{Color_" + str(count) + "}")
count += 1
# Names
count = 0
nameList = re.findall(r"[\\]+[nN]\[.+?\]+", jaString)
nameList = set(nameList)
if len(nameList) != 0:
for name in nameList:
jaString = jaString.replace(name, "{Noun_" + str(count) + "}")
count += 1
# Variables
count = 0
varList = re.findall(r"[\\]+[vV]\[[0-9]+\]", jaString)
varList = set(varList)
if len(varList) != 0:
for var in varList:
jaString = jaString.replace(var, "{Var_" + str(count) + "}")
count += 1
# Formatting
count = 0
formatList = re.findall(r"[\\]+[\w]+\[.+?\]", jaString)
formatList = set(formatList)
if len(formatList) != 0:
for var in formatList:
jaString = jaString.replace(var, "{FCode_" + str(count) + "}")
codeList = re.findall(r"[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]", jaString)
codeList = set(codeList)
if len(codeList) != 0:
for var in codeList:
jaString = jaString.replace(var, "[FCode_" + str(count) + "]")
count += 1
# Put all lists in list and return
allList = [nestedList, iconList, colorList, nameList, varList, formatList]
return [jaString, allList]
return [jaString, codeList]
def resubVars(translatedText, allList):
def resubVars(translatedText, codeList):
# Fix Spacing and ChatGPT Nonsense
matchList = re.findall(r"\[\s?.+?\s?\]", translatedText)
if len(matchList) > 0:
@ -392,46 +372,11 @@ def resubVars(translatedText, allList):
text = match.strip()
translatedText = translatedText.replace(match, text)
# Nested
count = 0
if len(allList[0]) != 0:
for var in allList[0]:
translatedText = translatedText.replace("{Nested_" + str(count) + "}", var)
count += 1
# Icons
count = 0
if len(allList[1]) != 0:
for var in allList[1]:
translatedText = translatedText.replace("{Ascii_" + str(count) + "}", var)
count += 1
# Colors
count = 0
if len(allList[2]) != 0:
for var in allList[2]:
translatedText = translatedText.replace("{Color_" + str(count) + "}", var)
count += 1
# Names
count = 0
if len(allList[3]) != 0:
for var in allList[3]:
translatedText = translatedText.replace("{Noun_" + str(count) + "}", var)
count += 1
# Vars
count = 0
if len(allList[4]) != 0:
for var in allList[4]:
translatedText = translatedText.replace("{Var_" + str(count) + "}", var)
count += 1
# Formatting
count = 0
if len(allList[5]) != 0:
for var in allList[5]:
translatedText = translatedText.replace("{FCode_" + str(count) + "}", var)
if len(codeList) != 0:
for var in codeList:
translatedText = translatedText.replace("[FCode_" + str(count) + "]", var)
count += 1
return translatedText
@ -446,50 +391,53 @@ def batchList(input_list, batch_size):
]
def createContext(fullPromptFlag, subbedT):
characters = "Game Characters:\n\
林つかさ (Tsukasa Hayashi) - Female\n\
山田美兎 (Miyato Yamada) - Female\n\
鈴木赤音 (Akane Suzuki) - Female\n\
佐藤莉伊南 (Riina Satou) - Female\n\
佐々木万梨美 (Marimi Sasaki) - Female\n\
渡辺登樹子 (Tokiko Watanabe) - Female\n\
桃乃夢 (Yume Momono) - Female\n\
吉浦美雪 (Miyuki Yoshiura) - Female\n\
三ツ門まあな (Maana Mitsukado) - Female\n\
モリーボイド (Molly Boyd) - Female\n\
オルガブヤチッチ (Olga Buyachich) - Female\n\
アッチャラー ギッティ (Atchara Gitti) - Female\n\
"
def createContext(fullPromptFlag, subbedT, format):
system = (
PROMPT
PROMPT + VOCAB
if fullPromptFlag
else f"Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`"
else f"\
You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
- You translate everything, including erotic and explicit content.\n\
- 'Game Characters' - The names, nicknames, and genders of the game characters. Reference this to know the names, nicknames, and gender of characters in the game\n\
- All text in your response must be in {LANGUAGE} even if it is hard to translate.\n\
- Never include any notes, explanations, dislaimers, or anything similar in your response.\n\
- Maintain any spacing in the translation.\n\
- Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\
- `...` can be a part of the dialogue. Translate it as it is.\n\
{VOCAB}\n\
"
)
user = f"{subbedT}"
return characters, system, user
if format == "json":
user = f"```json\n{subbedT}\n```"
else:
user = subbedT
return system, user
def translateText(characters, system, user, history):
def translateText(system, user, history, penalty, format, model=MODEL):
# Prompt
msg = [{"role": "system", "content": system + characters}]
# Characters
msg.append({"role": "system", "content": characters})
msg = [{"role": "system", "content": system}]
# History
if isinstance(history, list):
msg.extend([{"role": "assistant", "content": h} for h in history])
msg.extend([{"role": "system", "content": h} for h in history])
else:
msg.append({"role": "assistant", "content": history})
msg.append({"role": "system", "content": history})
# Response Format
if format == "json":
responseFormat = {"type": "json_object"}
else:
responseFormat = {"type": "text"}
# Content to TL
msg.append({"role": "user", "content": f"{user}"})
response = openai.chat.completions.create(
temperature=0.1,
frequency_penalty=0.1,
model=MODEL,
temperature=0,
frequency_penalty=penalty,
model=model,
response_format=responseFormat,
messages=msg,
)
return response
@ -503,34 +451,52 @@ def cleanTranslatedText(translatedText, varResponse):
"": "~",
"": "",
"": ".",
"": '\\"',
"": '\\"',
"- ": "-",
"Placeholder Text": "",
# Add more replacements as needed
}
for target, replacement in placeholders.items():
translatedText = translatedText.replace(target, replacement)
# Elongate Long Dashes (Since GPT Ignores them...)
translatedText = elongateCharacters(translatedText)
translatedText = resubVars(translatedText, varResponse[1])
if "\n" in translatedText:
return [line for line in translatedText.split("\n") if line]
else:
return [line for line in translatedText.split("\\n") if line]
return translatedText
def elongateCharacters(text):
# Define a pattern to match one character followed by one or more `ー` characters
# Using a positive lookbehind assertion to capture the preceding character
pattern = r"(?<=(.))ー+"
# Define a replacement function that elongates the captured character
def repl(match):
char = match.group(1) # The character before the ー sequence
count = len(match.group(0)) - 1 # Number of ー characters
return char * count # Replace ー sequence with the character repeated
# Use re.sub() to replace the pattern in the text
return re.sub(pattern, repl, text)
def extractTranslation(translatedTextList, is_list):
pattern = r"<Line(\d+)>[\\]*`?(.*?)[\\]*?`?</?Line\d+>"
# If it's a batch (i.e., list), extract with tags; otherwise, return the single item.
if is_list:
return [
re.findall(pattern, line)[0][1]
for line in translatedTextList
if re.search(pattern, line)
]
else:
matchList = re.findall(pattern, translatedTextList)
return matchList[0][1] if matchList else translatedTextList
try:
line_dict = json.loads(translatedTextList)
# If it's a batch (i.e., list), extract with tags; otherwise, return the single item.
string_list = list(line_dict.values())
if is_list:
return string_list
else:
return string_list[0]
except Exception as e:
print(f"extractTranslation Error: {e}")
return None
def countTokens(characters, system, user, history):
def countTokens(system, user, history):
inputTotalTokens = 0
outputTotalTokens = 0
enc = tiktoken.encoding_for_model("gpt-4")
@ -542,7 +508,6 @@ def countTokens(characters, system, user, history):
else:
inputTotalTokens += len(enc.encode(history))
inputTotalTokens += len(enc.encode(system))
inputTotalTokens += len(enc.encode(characters))
inputTotalTokens += len(enc.encode(user))
# Output
@ -559,19 +524,22 @@ def combineList(tlist, text):
@retry(exceptions=Exception, tries=5, delay=5)
def translateGPT(text, history, fullPromptFlag):
global PBAR, MISMATCH, FILENAME
mismatch = False
totalTokens = [0, 0]
if isinstance(text, list):
format = "json"
tList = batchList(text, BATCHSIZE)
else:
format = "text"
tList = [text]
for index, tItem in enumerate(tList):
# Before sending to translation, if we have a list of items, add the formatting
if isinstance(tItem, list):
payload = "\n".join(
[f"<Line{i}>`{item}`</Line{i}>" for i, item in enumerate(tItem)]
)
payload = payload.replace("``", "`Placeholder Text`")
payload = {f"Line{i+1}": string for i, string in enumerate(tItem)}
payload = json.dumps(payload, indent=4, ensure_ascii=False)
varResponse = subVars(payload)
subbedT = varResponse[0]
else:
@ -580,38 +548,67 @@ def translateGPT(text, history, fullPromptFlag):
# Things to Check before starting translation
if not re.search(r"[一-龠ぁ-ゔァ-ヴーa---]+", subbedT):
if PBAR is not None:
PBAR.update(len(tItem))
continue
# Create Message
characters, system, user = createContext(fullPromptFlag, subbedT)
system, user = createContext(fullPromptFlag, subbedT, format)
# Calculate Estimate
if ESTIMATE:
estimate = countTokens(characters, system, user, history)
estimate = countTokens(system, user, history)
totalTokens[0] += estimate[0]
totalTokens[1] += estimate[1]
continue
# Translating
response = translateText(characters, system, user, history)
response = translateText(system, user, history, 0.05, format)
translatedText = response.choices[0].message.content
totalTokens[0] += response.usage.prompt_tokens
totalTokens[1] += response.usage.completion_tokens
# Formatting
translatedTextList = cleanTranslatedText(translatedText, varResponse)
# Check Translation
translatedText = cleanTranslatedText(translatedText, varResponse)
if isinstance(tItem, list):
extractedTranslations = extractTranslation(translatedTextList, True)
tList[index] = extractedTranslations
if len(tItem) != len(translatedTextList):
mismatch = True # Just here so breakpoint can be set
history = extractedTranslations[-10:] # Update history if we have a list
extractedTranslations = extractTranslation(translatedText, True)
if extractedTranslations == None or len(tItem) != len(
extractedTranslations
):
# Mismatch. Try Again
response = translateText(system, user, history, 0.05, format, "gpt-4o")
translatedText = response.choices[0].message.content
totalTokens[0] += response.usage.prompt_tokens
totalTokens[1] += response.usage.completion_tokens
# Formatting
translatedText = cleanTranslatedText(translatedText, varResponse)
if isinstance(tItem, list):
extractedTranslations = extractTranslation(translatedText, True)
if extractedTranslations == None or len(tItem) != len(
extractedTranslations
):
mismatch = True # Just here for breakpoint
# Set if no mismatch
if mismatch == False:
tList[index] = extractedTranslations
history = extractedTranslations[
-10:
] # Update history if we have a list
else:
history = text[-10:]
mismatch = False
if FILENAME not in MISMATCH:
MISMATCH.append(FILENAME)
# Update Loading Bar
with LOCK:
if PBAR is not None:
PBAR.update(len(tItem))
else:
# Ensure we're passing a single string to extractTranslation
extractedTranslations = extractTranslation(
"\n".join(translatedTextList), False
)
tList[index] = extractedTranslations
tList[index] = translatedText
finalList = combineList(tList, text)
return [finalList, totalTokens]

View file

@ -69,7 +69,7 @@ LEAVE = False
# Dialogue / Scroll
CODE401 = False
CODE405 = False
CODE408 = True
CODE408 = False # Warning, translates comments and can inflate costs.
# Choices
CODE102 = False
@ -84,7 +84,7 @@ CODE101 = False
CODE355655 = False
CODE357 = False
CODE657 = False
CODE356 = False
CODE356 = True
CODE320 = False
CODE324 = False
CODE111 = False

View file

@ -74,8 +74,8 @@ CODE122 = False
# Other
CODE210 = False
CODE300 = True
CODE250 = False
CODE300 = False
CODE250 = True
# Database
NPCFLAG = False
@ -424,7 +424,7 @@ def searchCodes(events, pbar, jobList, filename):
):
# Remove Textwrap and Font and Add to list
str = str.replace("\r\n", " ")
str = str.replace(f"\\f[{fontSize}]", "")
str = re.sub(r'[\\]+f\[\d+\]', '', str)
list300.append(str)
# Pass 2
@ -621,7 +621,7 @@ def searchCodes(events, pbar, jobList, filename):
):
# Remove Textwrap and Font and Add to list
str = str.replace("\r\n", " ")
str = str.replace(f"\\f[{fontSize}]", "")
str = re.sub(r'[\\]+f\[\d+\]', '', str)
list300.append(str)
# Pass 2
@ -676,7 +676,7 @@ def searchCodes(events, pbar, jobList, filename):
# Validate size
if len(codeList[i]["stringArgs"]) > 2:
if (
codeList[i]["stringArgs"][1] == "所持商品"
codeList[i]["stringArgs"][1] == "クエスト情報"
and codeList[i]["stringArgs"][2] != ""
):
# Grab String
@ -964,7 +964,7 @@ def searchDB(events, pbar, jobList, filename):
scenarioList[2].pop(0)
# Grab Items
if table["name"] == "キャラクタプロフィール" and ITEMFLAG == True:
if table["name"] == "アイテム" and ITEMFLAG == True:
with open("translations.txt", "a", encoding="utf-8") as file:
for item in table["data"]:
dataList = item["data"]
@ -972,7 +972,7 @@ def searchDB(events, pbar, jobList, filename):
# Parse #
for j in range(len(dataList)):
# Name
if dataList[j].get("name") == "名前":
if dataList[j].get("name") == "生息海域":
# Pass 1 (Grab Data)
if setData == False:
if dataList[j].get("value") != "":
@ -992,9 +992,9 @@ def searchDB(events, pbar, jobList, filename):
itemList[0].pop(0)
# Description 1 (You are my specialz)
if dataList[j].get("name") == "戦闘面での特徴":
if dataList[j].get("name") == "説明":
# Clean String
fontSize = 18
fontSize = 14
translatedText = ""
cleanedList = formatDramon(dataList[j].get("value"))
for str in cleanedList:
@ -1009,7 +1009,7 @@ def searchDB(events, pbar, jobList, filename):
):
# Remove Textwrap and Font and Add to list
str = str.replace("\r\n", " ")
str = str.replace(f"\\f[{fontSize}]", "")
str = re.sub(r'[\\]+f\[\d+\]', '', str)
itemList[1].append(str)
# Pass 2
@ -1054,7 +1054,7 @@ def searchDB(events, pbar, jobList, filename):
dataList[j].update({"value": translatedText})
# Description 2 (You are my specialz)
if dataList[j].get("name") == "01_説明":
if dataList[j].get("name") == "NULL":
# Clean String
fontSize = 24
translatedText = ""
@ -1071,7 +1071,7 @@ def searchDB(events, pbar, jobList, filename):
):
# Remove Textwrap and Font and Add to list
str = str.replace("\r\n", " ")
str = str.replace(f"\\f[{fontSize}]", "")
str = re.sub(r'[\\]+f\[\d+\]', '', str)
itemList[2].append(str)
# Pass 2
@ -1116,7 +1116,7 @@ def searchDB(events, pbar, jobList, filename):
dataList[j].update({"value": translatedText})
# Description 3 (You are my specialz)
if dataList[j].get("name") == "01_説明未婚":
if dataList[j].get("name") == "NULL":
# Clean String
fontSize = 24
translatedText = ""
@ -1133,7 +1133,7 @@ def searchDB(events, pbar, jobList, filename):
):
# Remove Textwrap and Font and Add to list
str = str.replace("\r\n", " ")
str = str.replace(f"\\f[{fontSize}]", "")
str = re.sub(r'[\\]+f\[\d+\]', '', str)
itemList[3].append(str)
# Pass 2

View file

@ -1,11 +1,12 @@
Here are some vocabulary and terms so that you know the proper spelling and translation.
```
# Game Characters
リエナ (Liena) - Female\n
イル (Iru) - Male\n
ヨーグ (Yorg) - Male\n
メリッサ (Melissa) - Female\n
ナーサ (Naasa) - Female\n\
ジーナ (Gina) - Female\n\
マナリス (Manaris) - Female\n\
ラビィ (Lavi) - Female\n\
ララ (Lala) - Female\n\
クリス (Chris) - Female\n\
モーガン (Morgan) - Male\n\
# Lewd Terms
マンコ (pussy)
@ -74,4 +75,7 @@ ME 音量 (ME Volume)
堕天使 (Fallen Angel)
鬼 (Oni)
ローバー (Roper)
# Locations
トリス (Tris)
```