Push latest changes

This commit is contained in:
DazedAnon 2024-12-07 10:42:50 -06:00
parent f684cf1eef
commit cc39b4bb3d
7 changed files with 104 additions and 190 deletions

View file

@ -149,8 +149,8 @@ files to translate are in the /files folder and that you picked the right game e
if totalCost != "Fail":
# if estimate is False:
# This is to encourage people to grab what's in /translated instead
# deleteFolderFiles("files")
# This is to encourage people to grab what's in /translated instead
# deleteFolderFiles("files")
tqdm.write(str(totalCost))

View file

@ -195,38 +195,38 @@ def translateRegex(data, translatedList):
lineRegexText = r"t\s'(.*)'$"
lineRegexSpeaker = r"n\s'(.*)'$"
choiceRegex = r"choice:\d+\s'(.*)'"
# Speaker
match = re.search(lineRegexSpeaker, data[i])
match = re.search(lineRegexSpeaker, data[i])
if match:
if match.group(1):
response = getSpeaker(match.group(1))
speaker = response[0]
tokens[0] += response[1][0]
tokens[1] += response[1][1]
data[i] = data[i].replace(match.group(1), speaker)
data[i] = data[i].replace(match.group(1), speaker)
else:
speaker = None
speaker = None
# Dialogue
match = re.search(lineRegexText, data[i])
match = re.search(lineRegexText, data[i])
jaString = None
if match:
if match:
# Set String
jaString = match.group(1)
originalString = jaString
# Check if next lines are strings
jaStringLines = [jaString]
match = re.search(lineRegexText, data[i+1])
match = re.search(lineRegexText, data[i + 1])
while match:
jaStringLines.append(match.group(1))
del(data[i+1])
match = re.search(lineRegexText, data[i+1])
del data[i + 1]
match = re.search(lineRegexText, data[i + 1])
# Combine
jaString = " ".join(jaStringLines)
# Pass 1
if not translatedList:
# Strip Spaces
@ -284,7 +284,11 @@ def translateRegex(data, translatedList):
choiceList.pop(0)
# Replace Spaces
translatedText = translatedText.replace(' ', '\u3000')
translatedText = translatedText.replace("\u3000", " ")
# Escape Quotes
translatedText = re.sub(r'(?<!\\)"', r'\\"', translatedText)
translatedText = re.sub(r"(?<!\\)'", r"\\'", translatedText)
# Set
data[i] = data[i].replace(match.group(1), translatedText)
@ -296,23 +300,21 @@ def translateRegex(data, translatedList):
# EOF
if not translatedList:
stringListTL = []
choiceListTL = []
choiceListTL = []
# String List
if stringList:
PBAR.total = len(stringList)
PBAR.refresh()
response = translateGPT(
stringList,
"Reply with the English Translation",
True
stringList, "Reply with the English Translation", True
)
tokens[0] += response[1][0]
tokens[1] += response[1][1]
stringListTL = response[0]
if len(stringList) != len(stringListTL):
# Mismatch
# Mismatch
with LOCK:
if FILENAME not in MISMATCH:
MISMATCH.append(FILENAME)
@ -320,16 +322,14 @@ def translateRegex(data, translatedList):
# Choice List
if choiceList:
response = translateGPT(
choiceList,
"Reply with the English TL of the Dialogue Choice",
True
choiceList, "Reply with the English TL of the Dialogue Choice", True
)
tokens[0] += response[1][0]
tokens[1] += response[1][1]
choiceListTL = response[0]
if len(choiceList) != len(choiceListTL):
# Mismatch
# Mismatch
with LOCK:
if FILENAME not in MISMATCH:
MISMATCH.append(FILENAME)
@ -450,7 +450,7 @@ def cleanTranslatedText(translatedText, varResponse):
"": '\\"',
"": '\\"',
"- ": "-",
"" : "-",
"": "-",
"": "]",
"": "[",
"Placeholder Text": "",

View file

@ -207,9 +207,9 @@ def translateRenpy(data, translatedList):
speaker = ""
lineRegexNoSpeaker = r'^\s\s\s\s"(.*)"'
lineRegexSpeaker = r'^\s\s\s\s(.+?)\s"(.*)"'
# Grab Line
match = re.search(lineRegexSpeaker, data[i])
match = re.search(lineRegexSpeaker, data[i])
if match:
response = getSpeaker(match.group(1))
jaString = match.group(2)
@ -222,7 +222,7 @@ def translateRenpy(data, translatedList):
jaString = match.group(1)
# Valid Line
if match and 'voice' not in data[i]:
if match and "voice" not in data[i]:
originalString = jaString
# Pass 1
if translatedList == []:
@ -262,7 +262,7 @@ def translateRenpy(data, translatedList):
# Textwrap
translatedText = textwrap.fill(translatedText, width=WIDTH)
translatedText = translatedText.replace('\n', '\\n')
translatedText = translatedText.replace("\n", "\\n")
# Set Data
data[i] = data[i].replace(originalString, translatedText)
@ -278,9 +278,7 @@ def translateRenpy(data, translatedList):
# Translate
response = translateGPT(
stringList,
"Reply with the English TL of the NPC Name",
True
stringList, "Reply with the English TL of the NPC Name", True
)
tokens[0] += response[1][0]
tokens[1] += response[1][1]

View file

@ -1146,10 +1146,10 @@ def searchCodes(page, pbar, jobList, filename):
if setData == False:
# Remove Textwrap
if FIXTEXTWRAP:
finalJAString = finalJAString.replace('\n', ' ')
finalJAString = finalJAString.replace("\n", " ")
if "\\px[200]" in finalJAString:
finalJAString = finalJAString.replace('\\px[200]', '')
finalJAString = finalJAString.replace("\\px[200]", "")
# Append
if finalJAString != "":
if speaker == "" and finalJAString != "":
@ -1209,9 +1209,11 @@ def searchCodes(page, pbar, jobList, filename):
translatedText = translatedText.replace("\n", "<br>")
# px
if '\\px[200]' in nametag:
if "\\px[200]" in nametag:
translatedText = translatedText.replace("\\px[200]", "")
translatedText = translatedText.replace("\n", "\n\\px[200]")
translatedText = translatedText.replace(
"\n", "\n\\px[200]"
)
### Add Var Strings
# CL Flag

View file

@ -192,32 +192,34 @@ def translateTyrano(data, translatedList):
while i < len(data):
voice = False
lineRegexNoSpeaker = r'^([^\[#;*@\n]+)\[l\]\[[rp]\]|^([^\[#;*@\n]+)\[[rpl]\]|^([^\[#;*@_\n]+)\n$'
lineRegexSpeaker = r'^#(.*)'
furiganaRegex = r'(\[ruby\stext=(.*?)\])'
lineRegexNoSpeaker = (
r"^([^\[#;*@\n]+)\[l\]\[[rp]\]|^([^\[#;*@\n]+)\[[rpl]\]|^([^\[#;*@_\n]+)\n$"
)
lineRegexSpeaker = r"^#(.*)"
furiganaRegex = r"(\[ruby\stext=(.*?)\])"
choiceRegex = r'\[glink.+?text="(.*?)"'
# Speaker
match = re.search(lineRegexSpeaker, data[i])
match = re.search(lineRegexSpeaker, data[i])
if match:
if match.group(1):
response = getSpeaker(match.group(1))
speaker = response[0]
tokens[0] += response[1][0]
tokens[1] += response[1][1]
data[i] = data[i].replace(match.group(1), speaker)
data[i] = data[i].replace(match.group(1), speaker)
else:
speaker = None
speaker = None
# Furigana
match = re.search(r'^\[ruby\stext', data[i])
match = re.search(r"^\[ruby\stext", data[i])
furiganaList = []
if match:
# Check next line and combine
while match:
furiganaList.append(data[i].replace('\n', ''))
furiganaList.append(data[i].replace("\n", ""))
del data[i]
match = re.search(r'^\[ruby\stext', data[i])
match = re.search(r"^\[ruby\stext", data[i])
jaString = "".join(furiganaList)
# Ruby Text
@ -228,7 +230,7 @@ def translateTyrano(data, translatedList):
data.insert(i, f"{jaString}[r]")
# Dialogue
match = re.search(lineRegexNoSpeaker, data[i])
match = re.search(lineRegexNoSpeaker, data[i])
jaString = None
if match:
jaString = match.group(1)
@ -238,7 +240,7 @@ def translateTyrano(data, translatedList):
jaString = match.group(3)
originalString = jaString
# Pass 1
if not translatedList:
# Remove any textwrap and commands
@ -285,8 +287,8 @@ def translateTyrano(data, translatedList):
# translatedText = translatedText.replace('\n', '[r]')
# Avoid Crashes
translatedText = translatedText.replace('[', '(')
translatedText = translatedText.replace(']', ')')
translatedText = translatedText.replace("[", "(")
translatedText = translatedText.replace("]", ")")
# Set Data
data[i] = data[i].replace(originalString, translatedText)
@ -297,7 +299,7 @@ def translateTyrano(data, translatedList):
# Pass 1
if not translatedList:
choiceList.append(match.group(1))
match = re.search(choiceRegex, data[i+1])
match = re.search(choiceRegex, data[i + 1])
# Pass 2
else:
@ -306,7 +308,7 @@ def translateTyrano(data, translatedList):
choiceList.pop(0)
# Replace Spaces
translatedText = translatedText.replace(' ', '\u3000')
translatedText = translatedText.replace(" ", "\u3000")
# Set
data[i] = data[i].replace(match.group(1), translatedText)
@ -318,23 +320,21 @@ def translateTyrano(data, translatedList):
# EOF
if not translatedList:
stringListTL = []
choiceListTL = []
choiceListTL = []
# String List
if stringList:
PBAR.total = len(stringList)
PBAR.refresh()
response = translateGPT(
stringList,
"Reply with the English Translation",
True
stringList, "Reply with the English Translation", True
)
tokens[0] += response[1][0]
tokens[1] += response[1][1]
stringListTL = response[0]
if len(stringList) != len(stringListTL):
# Mismatch
# Mismatch
with LOCK:
if FILENAME not in MISMATCH:
MISMATCH.append(FILENAME)
@ -342,16 +342,14 @@ def translateTyrano(data, translatedList):
# Choice List
if choiceList:
response = translateGPT(
choiceList,
"Reply with the English TL of the Dialogue Choice",
True
choiceList, "Reply with the English TL of the Dialogue Choice", True
)
tokens[0] += response[1][0]
tokens[1] += response[1][1]
choiceListTL = response[0]
if len(choiceList) != len(choiceListTL):
# Mismatch
# Mismatch
with LOCK:
if FILENAME not in MISMATCH:
MISMATCH.append(FILENAME)

View file

@ -57,7 +57,7 @@ if "gpt-3.5" in MODEL:
elif "gpt-4" in MODEL:
INPUTAPICOST = 0.0025
OUTPUTAPICOST = 0.01
BATCHSIZE = 20
BATCHSIZE = 30
FREQUENCY_PENALTY = 0.1
# tqdm Globals
@ -68,14 +68,16 @@ PBAR = None
FILENAME = None
# Dialogue / Scroll
CODE101 = False
CODE102 = False
CODE101 = True
CODE102 = True
# Set String (Fragile but necessary)
CODE122 = False
# Other
CODE210 = False
CODE210 = True
CODE300 = False
CODE250 = True
CODE250 = False
# Database
NPCFLAG = False
@ -392,79 +394,28 @@ def searchCodes(events, pbar, jobList, filename):
### Event Code: 210 Common Event
if codeList[i]["code"] == 210 and CODE210 == True:
# if 'stringArgs' in codeList[i] and len(codeList[i]['stringArgs']) > 1:
# # Grab Event List
# jaString = codeList[i]['stringArgs'][1]
# Add Speaker
if (
"stringArgs" in codeList[i]
and codeList[i]["intArgs"][0] == 500529
and len(codeList[i]["stringArgs"]) == 2
):
response = getSpeaker(codeList[i]["stringArgs"][1])
totalTokens[1] += response[1][0]
totalTokens[1] += response[1][1]
speaker = response[0]
lastSpeaker = speaker
# # Remove Textwrap
# jaString = jaString.replace("\n", ' ')
# # Translate
# response = translateGPT(jaString, f'Reply with the {LANGUAGE} translation of the location', False)
# translatedText = response[0]
# totalTokens[0] += response[1][0]
# totalTokens[1] += response[1][1]
# # Textwrap
# translatedText = textwrap.fill(translatedText, WIDTH)
# # Validate and Set Data
# codeList[i]['stringArgs'][1] = translatedText
if "stringArgs" in codeList[i] and len(codeList[i]["stringArgs"]) > 1:
cleanedList = formatDramon(codeList[i]["stringArgs"][1])
fontSize = 24
translatedText = ""
for str in cleanedList:
# Pass 1
if not setData:
if (
all(x not in str for x in ["_", "@", ">", "/"])
and str != "\r\n"
):
# Remove Textwrap and Font and Add to list
str = str.replace("\r\n", " ")
str = re.sub(r"[\\]+f\[\d+\]", "", str)
list300.append(str)
# Pass 2
else:
if (
all(
x not in str
for x in [
"_",
"@",
">",
"/",
]
)
and str != "\r\n"
):
# Decide Wrap
if codeList[i]["stringArgs"][0] == "[移]サウンドノベル":
width = 40
else:
width = WIDTH
# Add Textwrap and Font
list300[0] = textwrap.fill(list300[0], width)
list300[0] = list300[0].replace(
"\n", f"\r\n\\f[{fontSize}]"
)
list300[0] = f"\\f[{fontSize}]{list300[0]}\r\n"
translatedText += list300[0]
list300.pop(0)
else:
translatedText += str
# Write to File
if setData:
# Formatting Fixes
translatedText = translatedText.replace('*"', '* "')
translatedText = translatedText.replace("\r\n\r\n", "\r\n")
translatedText = re.sub(r"[^\S\r\n]+", " ", translatedText)
codeList[i]["stringArgs"][1] = translatedText
# Set Data
codeList[i]["stringArgs"][1] = speaker
# Reuse Last Speaker
elif codeList[i]["intArgs"][0] == 500501 and lastSpeaker != "":
speaker = lastSpeaker
# Erase Speaker
elif codeList[i]["intArgs"][0] == 500529:
speaker = ""
### Event Code: 122 SetString
if codeList[i]["code"] == 122 and CODE122 == True:
@ -1802,40 +1753,6 @@ def getSpeaker(speaker):
return [speaker, [0, 0]]
def subVars(jaString):
jaString = jaString.replace("\u3000", " ")
# Formatting
count = 0
codeList = re.findall(r"[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+?\]", jaString)
codeList = set(codeList)
if len(codeList) != 0:
for var in codeList:
jaString = jaString.replace(var, "[FCode_" + str(count) + "]")
count += 1
# Put all lists in list and return
return [jaString, codeList]
def resubVars(translatedText, codeList):
# Fix Spacing and ChatGPT Nonsense
matchList = re.findall(r"\[\s?.+?\s?\]", translatedText)
if len(matchList) > 0:
for match in matchList:
text = match.strip()
translatedText = translatedText.replace(match, text)
# Formatting
count = 0
if len(codeList) != 0:
for var in codeList:
translatedText = translatedText.replace("[FCode_" + str(count) + "]", var)
count += 1
return translatedText
def batchList(input_list, batch_size):
if not isinstance(batch_size, int) or batch_size <= 0:
raise ValueError("batch_size must be a positive integer")
@ -1908,6 +1825,8 @@ def cleanTranslatedText(translatedText, varResponse):
"": '\\"',
"": '\\"',
"- ": "-",
"": "]",
"": "[",
"Placeholder Text": "",
# Add more replacements as needed
}
@ -1916,7 +1835,6 @@ def cleanTranslatedText(translatedText, varResponse):
# Elongate Long Dashes (Since GPT Ignores them...)
translatedText = elongateCharacters(translatedText)
translatedText = resubVars(translatedText, varResponse[1])
return translatedText
@ -1938,7 +1856,7 @@ def elongateCharacters(text):
def extractTranslation(translatedTextList, is_list):
try:
translatedTextList = re.sub(r'\\"+\"([^,\n}])', r'\\"\1', translatedTextList)
translatedTextList = re.sub(r'(?<![\\])"+', r'"', translatedTextList)
translatedTextList = re.sub(r"(?<![\\])\"+(?!\n)", r'"', translatedTextList)
line_dict = json.loads(translatedTextList)
# If it's a batch (i.e., list), extract with tags; otherwise, return the single item.
string_list = list(line_dict.values())
@ -1996,11 +1914,10 @@ def translateGPT(text, history, fullPromptFlag):
if isinstance(tItem, list):
payload = {f"Line{i+1}": string for i, string in enumerate(tItem)}
payload = json.dumps(payload, indent=4, ensure_ascii=False)
varResponse = subVars(payload)
varResponse = [payload, []]
subbedT = varResponse[0]
logFile.write(f"Input:\n{subbedT}\n")
else:
varResponse = subVars(tItem)
varResponse = [tItem, []]
subbedT = varResponse[0]
# Things to Check before starting translation
@ -2009,6 +1926,7 @@ def translateGPT(text, history, fullPromptFlag):
):
if PBAR is not None:
PBAR.update(len(tItem))
history = tItem[-MAXHISTORY:]
continue
# Create Message
@ -2050,16 +1968,17 @@ def translateGPT(text, history, fullPromptFlag):
extractedTranslations
):
mismatch = True # Just here for breakpoint
logFile.write(f"Input:\n{subbedT}\n")
logFile.write(f"Output:\n{translatedText}\n")
# Set if no mismatch
if mismatch == False:
tList[index] = extractedTranslations
history = extractedTranslations[
-10:
-MAXHISTORY:
] # Update history if we have a list
else:
history = text[-10:]
history = text[-MAXHISTORY:]
mismatch = False
if FILENAME not in MISMATCH:
MISMATCH.append(FILENAME)

View file

@ -1,14 +1,11 @@
Here are some vocabulary and terms so that you know the proper spelling and translation.
```
# Game Characters
ルカミラ・ハル・ロッシュ (Lukamilla Harre Roche) - Female
マリアーヌ・ローエン・グリン (Marianne Roen Green) - Female
リアン (Rianne) - Female
スカーレット・ルンベルク (Scarlet Lumberg) - Female
ソフィア (Sofia) - Female
アグファス・ブロウニン (Agfas Blonin) - Male
マルック・スタンレー (Marc Stanley) - Male
モノキオス (Monokios) - Male
黒井 衣 (Kuroi Koromo) - Female
ころも (Koromo) - Female
鉄仮面 (Iron Mask) - Male
ジライヤ (Jiraiya) - Male
REMU - Female
# Lewd Terms
マンコ (pussy)