dazed/DazedTL
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Implement in rest and delete some old modules

Browse source
This commit is contained in:
dazedanon 2025-07-15 15:17:10 -05:00
parent 9c36e6d7f7
commit f39663bbdd
24 changed files with 1317 additions and 5422 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

610
modules/alice.py
View file

@ -1,610 +0,0 @@
# Libraries
import os
import re
import util.dazedwrap as dazedwrap
import threading
import time
import traceback
import tiktoken
import openai
from pathlib import Path
from colorama import Fore
from dotenv import load_dotenv
from retry import retry
from tqdm import tqdm
# Open AI
load_dotenv()
if os.getenv("api").replace(" ", "") != "":
openai.base_url = os.getenv("api")
openai.organization = os.getenv("org")
openai.api_key = os.getenv("key")
# Globals
MODEL = os.getenv("model")
TIMEOUT = int(os.getenv("timeout"))
LANGUAGE = os.getenv("language").capitalize()
PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
VOCAB = Path("vocab.txt").read_text(encoding="utf-8")
THREADS = int(os.getenv("threads"))
LOCK = threading.Lock()
WIDTH = int(os.getenv("width"))
LISTWIDTH = int(os.getenv("listWidth"))
NOTEWIDTH = 70
MAXHISTORY = 10
ESTIMATE = ""
TOKENS = [0, 0]
NAMESLIST = []
NAMES = False # Output a list of all the character names found
BRFLAG = False # If the game uses <br> instead
FIXTEXTWRAP = True # Overwrites textwrap
IGNORETLTEXT = False # Ignores all translated text.
MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
# tqdm Globals
BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
POSITION = 0
LEAVE = False
# Regex - Need to change this if you want to translate from/to other languages. Default is Japanese Regex
LANGREGEX = r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+"
# Pricing - Depends on the model https://openai.com/pricing
# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request
# If you are getting a MISMATCH LENGTH error, lower the batch size.
if "gpt-3.5" in MODEL:
INPUTAPICOST = 3.00
OUTPUTAPICOST = 5.00
BATCHSIZE = 10
FREQUENCY_PENALTY = 0.2
elif "gpt-4" in MODEL:
INPUTAPICOST = 2.0
OUTPUTAPICOST = 8.00
BATCHSIZE = 30
FREQUENCY_PENALTY = 0.05
elif "deepseek" in MODEL:
INPUTAPICOST = 0.27
OUTPUTAPICOST = 1.10
BATCHSIZE = 30
FREQUENCY_PENALTY = 0.05
else:
INPUTAPICOST = float(os.getenv("input_cost"))
OUTPUTAPICOST = float(os.getenv("output_cost"))
BATCHSIZE = int(os.getenv("batchsize"))
FREQUENCY_PENALTY = float(os.getenv("frequency_penalty"))
def handleAlice(filename, estimate):
global ESTIMATE
totalTokens = [0, 0]
ESTIMATE = estimate
if estimate:
start = time.time()
translatedData = openFiles(filename)
# Print Result
end = time.time()
tqdm.write(getResultString(translatedData, end - start, filename))
with LOCK:
totalTokens[0] += translatedData[1][0]
totalTokens[1] += translatedData[1][1]
# Print Total
totalString = getResultString(["", totalTokens, None], end - start, "TOTAL")
# Print any errors on maps
if len(MISMATCH) > 0:
return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
else:
return totalString
else:
try:
with open("translated/" + filename, "w", encoding="utf-8", newline="\n") as outFile:
start = time.time()
translatedData = openFiles(filename)
# Print Result
end = time.time()
outFile.writelines(translatedData[0])
tqdm.write(getResultString(translatedData, end - start, filename))
with LOCK:
totalTokens[0] += translatedData[1][0]
totalTokens[1] += translatedData[1][1]
except Exception:
traceback.print_exc()
return "Fail"
return getResultString(["", totalTokens, None], end - start, "TOTAL")
def openFiles(filename):
with open("files/" + filename, "r", encoding="UTF-8") as f:
translatedData = parseText(f, filename)
return translatedData
def getResultString(translatedData, translationTime, filename):
# File Print String
totalTokenstring = (
Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
"[Output: "
+ str(translatedData[1][1])
+ "]" "[Cost: ${:,.4f}".format(((translatedData[1][0] / 1000000) * INPUTAPICOST) + ((translatedData[1][1] / 1000000) * OUTPUTAPICOST))
+ "]"
)
timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
if translatedData[2] == None:
# Success
return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET
else:
# Fail
try:
raise translatedData[2]
except Exception as e:
traceback.print_exc()
errorString = str(e) + Fore.RED
return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET
def parseText(data, filename):
# Get total for progress bar
linesList = data.readlines()
totalTokens = [0, 0]
totalLines = len(linesList)
global LOCK
with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar:
pbar.desc = filename
pbar.total = totalLines
try:
result = translateLines(linesList, pbar)
totalTokens[0] += result[1][0]
totalTokens[1] += result[1][1]
except Exception as e:
traceback.print_exc()
return [linesList, totalTokens, e]
return [linesList, totalTokens, None]
# Grab scenario data from text file
def translateLines(linesList, pbar):
currentGroup = []
batch = []
textHistory = []
tokens = [0, 0]
batchStartIndex = 0
insertBool = False
multiLine = False
i = 0
try:
while i < len(linesList):
# Check if Proper Message
match = re.findall(r"s\[[0-9]+\] = \"(.*)\"", linesList[i])
if len(match) > 0:
jaString = match[0]
# Skip Files
if "／" in jaString:
i += 1
continue
### Translate
# Remove any textwrap
jaString = re.sub(r"\\n", " ", jaString)
# Grab Speaker
speakerMatch = re.findall(r"s\[[0-9]+\] = \"([^／]+)\"", linesList[i - 1])
if len(speakerMatch) > 0:
# If there isn't any Japanese in the text just skip
if re.search(r"[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+", jaString) and "_" not in speakerMatch[0]:
speaker = speakerMatch[0]
else:
speaker = ""
else:
speaker = ""
# Grab rest of the messages
currentGroup.append(jaString)
# Check if next line should be merged
if insertBool is True:
linesList[i] = re.sub(r"(s\[[0-9]+\]) = \"(.+)\"", r'\1 = ""', linesList[i])
linesList[i] = linesList[i].replace(";", "")
start = i
while len(linesList) > i + 1 and re.search(r"s\[[0-9]+\] = \"\s+(.*)\"", linesList[i + 1]) != None:
multiLine = True
i += 1
match = re.findall(r"s\[[0-9]+\] = \"\s+(.*)\"", linesList[i])
currentGroup.append(match[0])
if insertBool is True:
linesList[i] = re.sub(r"(s\[[0-9]+\]) = \"\s+(.+)\"", r'\1 = ""', linesList[i])
linesList[i] = linesList[i].replace(";", "")
i += 1
# Combine Groups and Add Speaker
finalJAString = " ".join(currentGroup)
if speaker != "":
finalJAString = f"{speaker}: {finalJAString}"
else:
finalJAString = f"{finalJAString}"
# [Passthrough 1] Pulling From File
if insertBool is False:
# Append to List and Clear Values
batch.append(finalJAString)
# Translate Batch if Full
if len(batch) == BATCHSIZE or i >= len(linesList) - 1:
# Translate
response = translateGPT(batch, textHistory, True)
tokens[0] += response[1][0]
tokens[1] += response[1][1]
translatedBatch = response[0]
textHistory = translatedBatch[-10:]
# Set Values
if len(batch) == len(translatedBatch):
i = batchStartIndex
insertBool = True
# Mismatch
else:
pbar.write(f"Mismatch: {batchStartIndex} - {i}")
MISMATCH.append(batch)
batchStartIndex = i
batch.clear()
multiLine = False
currentGroup = []
# [Passthrough 2] Setting Data
else:
# Get Text
translatedText = translatedBatch[0]
# Remove added speaker and quotes
translatedText = re.sub(r"^.+?:\s", "", translatedText)
# Textwrap
translatedText = translatedText.replace('"', '\\"')
translatedText = dazedwrap.wrapText(translatedText, width=WIDTH)
# Set Data
if multiLine:
textList = translatedText.split("\n")
for t in textList:
translatedText = translatedText.replace(";", "")
translatedText = re.sub(
r"(s\[[0-9]+\]) = \"(.*)\"",
rf'\1 = "{t}"',
linesList[start],
)
translatedText = translatedText.replace(";", "")
linesList[start] = translatedText
pbar.update(1)
start += 1
multiLine = False
translatedText = translatedText.replace(";", "")
translatedBatch.pop(0)
else:
# Remove any textwrap
translatedText = translatedText.replace("\n", " ")
translatedText = re.sub(
r"(s\[[0-9]+\]) = \"(.*)\"",
rf'\1 = "{translatedText}"',
linesList[start],
)
translatedText = translatedText.replace(";", "")
linesList[start] = translatedText
pbar.update(1)
translatedBatch.pop(0)
# If Batch is empty. Move on.
if len(translatedBatch) == 0:
insertBool = False
batchStartIndex = i
pbar.update(1)
batch.clear()
currentGroup = []
else:
if insertBool is True:
pbar.update(1)
i += 1
return [linesList, tokens]
except Exception:
traceback.print_exc()
return [linesList, tokens]
def subVars(jaString):
jaString = jaString.replace("\u3000", " ")
# Nested
count = 0
nestedList = re.findall(r"[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]", jaString)
nestedList = set(nestedList)
if len(nestedList) != 0:
for icon in nestedList:
jaString = jaString.replace(icon, "{Nested_" + str(count) + "}")
count += 1
# Icons
count = 0
iconList = re.findall(r"[\\]+[iIkKwWaA]+\[[0-9]+\]", jaString)
iconList = set(iconList)
if len(iconList) != 0:
for icon in iconList:
jaString = jaString.replace(icon, "{Ascii_" + str(count) + "}")
count += 1
# Colors
count = 0
colorList = re.findall(r"[\\]+[cC]\[[0-9]+\]", jaString)
colorList = set(colorList)
if len(colorList) != 0:
for color in colorList:
jaString = jaString.replace(color, "{Color_" + str(count) + "}")
count += 1
# Names
count = 0
nameList = re.findall(r"[\\]+[nN]\[.+?\]+", jaString)
nameList = set(nameList)
if len(nameList) != 0:
for name in nameList:
jaString = jaString.replace(name, "{Noun_" + str(count) + "}")
count += 1
# Variables
count = 0
varList = re.findall(r"[\\]+[vV]\[[0-9]+\]", jaString)
varList = set(varList)
if len(varList) != 0:
for var in varList:
jaString = jaString.replace(var, "{Var_" + str(count) + "}")
count += 1
# Formatting
count = 0
formatList = re.findall(r"[\\]+[\w]+\[.+?\]", jaString)
formatList = set(formatList)
if len(formatList) != 0:
for var in formatList:
jaString = jaString.replace(var, "{FCode_" + str(count) + "}")
count += 1
# Put all lists in list and return
allList = [nestedList, iconList, colorList, nameList, varList, formatList]
return [jaString, allList]
def resubVars(translatedText, allList):
# Fix Spacing and ChatGPT Nonsense
matchList = re.findall(r"\[\s?.+?\s?\]", translatedText)
if len(matchList) > 0:
for match in matchList:
text = match.strip()
translatedText = translatedText.replace(match, text)
# Nested
count = 0
if len(allList[0]) != 0:
for var in allList[0]:
translatedText = translatedText.replace("{Nested_" + str(count) + "}", var)
count += 1
# Icons
count = 0
if len(allList[1]) != 0:
for var in allList[1]:
translatedText = translatedText.replace("{Ascii_" + str(count) + "}", var)
count += 1
# Colors
count = 0
if len(allList[2]) != 0:
for var in allList[2]:
translatedText = translatedText.replace("{Color_" + str(count) + "}", var)
count += 1
# Names
count = 0
if len(allList[3]) != 0:
for var in allList[3]:
translatedText = translatedText.replace("{Noun_" + str(count) + "}", var)
count += 1
# Vars
count = 0
if len(allList[4]) != 0:
for var in allList[4]:
translatedText = translatedText.replace("{Var_" + str(count) + "}", var)
count += 1
# Formatting
count = 0
if len(allList[5]) != 0:
for var in allList[5]:
translatedText = translatedText.replace("{FCode_" + str(count) + "}", var)
count += 1
return translatedText
def batchList(input_list, batch_size):
if not isinstance(batch_size, int) or batch_size <= 0:
raise ValueError("batch_size must be a positive integer")
return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)]
def createContext(fullPromptFlag, subbedT):
characters = "Game Characters:\n\
林つかさ (Tsukasa Hayashi) - Female\n\
山田美兎 (Miyato Yamada) - Female\n\
鈴木赤音 (Akane Suzuki) - Female\n\
佐藤莉伊南 (Riina Satou) - Female\n\
佐々木万梨美 (Marimi Sasaki) - Female\n\
渡辺登樹子 (Tokiko Watanabe) - Female\n\
桃乃夢 (Yume Momono) - Female\n\
吉浦美雪 (Miyuki Yoshiura) - Female\n\
三ツ門まあな (Maana Mitsukado) - Female\n\
モリー・ボイド (Molly Boyd) - Female\n\
オルガ・ブヤチッチ (Olga Buyachich) - Female\n\
アッチャラー ギッティ (Atchara Gitti) - Female\n\
"
system = (
PROMPT
if fullPromptFlag
else f"Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`"
)
user = f"{subbedT}"
return characters, system, user
def translateText(characters, system, user, history):
# Prompt
msg = [{"role": "system", "content": system + characters}]
# Characters
msg.append({"role": "system", "content": characters})
# History
if isinstance(history, list):
msg.extend([{"role": "assistant", "content": h} for h in history])
else:
msg.append({"role": "assistant", "content": history})
# Content to TL
msg.append({"role": "user", "content": f"{user}"})
response = openai.chat.completions.create(
temperature=0.1,
frequency_penalty=0.1,
model=MODEL,
messages=msg,
)
return response
def cleanTranslatedText(translatedText, varResponse):
placeholders = {
f"{LANGUAGE} Translation: ": "",
"Translation: ": "",
"っ": "",
"〜": "~",
"ッ": "",
"。": ".",
"Placeholder Text": "",
# Add more replacements as needed
}
for target, replacement in placeholders.items():
translatedText = translatedText.replace(target, replacement)
translatedText = resubVars(translatedText, varResponse[1])
if "\n" in translatedText:
return [line for line in translatedText.split("\n") if line]
else:
return [line for line in translatedText.split("\\n") if line]
def extractTranslation(translatedTextList, is_list):
pattern = r"<Line(\d+)>[\\]*`?(.*?)[\\]*?`?</?Line\d+>"
# If it's a batch (i.e., list), extract with tags; otherwise, return the single item.
if is_list:
return [re.findall(pattern, line)[0][1] for line in translatedTextList if re.search(pattern, line)]
else:
matchList = re.findall(pattern, translatedTextList)
return matchList[0][1] if matchList else translatedTextList
def countTokens(characters, system, user, history):
inputTotalTokens = 0
outputTotalTokens = 0
enc = tiktoken.encoding_for_model("gpt-4")
# Input
if isinstance(history, list):
for line in history:
inputTotalTokens += len(enc.encode(line))
else:
inputTotalTokens += len(enc.encode(history))
inputTotalTokens += len(enc.encode(system))
inputTotalTokens += len(enc.encode(characters))
inputTotalTokens += len(enc.encode(user))
# Output
outputTotalTokens += round(len(enc.encode(user)) * 3)
return [inputTotalTokens, outputTotalTokens]
@retry(exceptions=Exception, tries=5, delay=5)
def translateGPT(text, history, fullPromptFlag):
totalTokens = [0, 0]
if isinstance(text, list):
tList = batchList(text, BATCHSIZE)
else:
tList = [text]
for index, tItem in enumerate(tList):
# Before sending to translation, if we have a list of items, add the formatting
if isinstance(tItem, list):
payload = "\n".join([f"<Line{i}>`{item}`</Line{i}>" for i, item in enumerate(tItem)])
payload = payload.replace("``", "`Placeholder Text`")
varResponse = subVars(payload)
subbedT = varResponse[0]
else:
varResponse = subVars(tItem)
subbedT = varResponse[0]
# Things to Check before starting translation
if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", subbedT):
continue
# Create Message
characters, system, user = createContext(fullPromptFlag, subbedT)
# Calculate Estimate
if ESTIMATE:
estimate = countTokens(characters, system, user, history)
totalTokens[0] += estimate[0]
totalTokens[1] += estimate[1]
continue
# Translating
response = translateText(characters, system, user, history)
translatedText = response.choices[0].message.content
totalTokens[0] += response.usage.prompt_tokens
totalTokens[1] += response.usage.completion_tokens
# Formatting
translatedTextList = cleanTranslatedText(translatedText, varResponse)
if isinstance(tItem, list):
extractedTranslations = extractTranslation(translatedTextList, True)
tList[index] = extractedTranslations
if len(tItem) != len(translatedTextList):
mismatch = True # Just here so breakpoint can be set
history = extractedTranslations[-10:] # Update history if we have a list
else:
# Ensure we're passing a single string to extractTranslation
extractedTranslations = extractTranslation("\n".join(translatedTextList), False)
tList[index] = extractedTranslations
# Combine if multilist
if isinstance(tList[0], list):
tList = [t for sublist in tList for t in sublist]
# Return
if format == "json":
return [tList, totalTokens]
else:
return [tList[0], totalTokens]

581
modules/anim.py
View file

@ -1,581 +0,0 @@
# Libraries
import json
import os
import re
import util.dazedwrap as dazedwrap
import threading
import time
import traceback
import tiktoken
import openai
from pathlib import Path
from colorama import Fore
from dotenv import load_dotenv
from retry import retry
from tqdm import tqdm
# Open AI
load_dotenv()
if os.getenv("api").replace(" ", "") != "":
openai.base_url = os.getenv("api")
openai.organization = os.getenv("org")
openai.api_key = os.getenv("key")
# Globals
MODEL = os.getenv("model")
TIMEOUT = int(os.getenv("timeout"))
LANGUAGE = os.getenv("language").capitalize()
PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
VOCAB = Path("vocab.txt").read_text(encoding="utf-8")
THREADS = int(os.getenv("threads"))
LOCK = threading.Lock()
WIDTH = int(os.getenv("width"))
LISTWIDTH = int(os.getenv("listWidth"))
NOTEWIDTH = 70
MAXHISTORY = 10
ESTIMATE = ""
TOKENS = [0, 0]
NAMESLIST = []
NAMES = False # Output a list of all the character names found
BRFLAG = False # If the game uses <br> instead
FIXTEXTWRAP = True # Overwrites textwrap
IGNORETLTEXT = False # Ignores all translated text.
MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
# tqdm Globals
BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
POSITION = 0
LEAVE = False
# Regex - Need to change this if you want to translate from/to other languages. Default is Japanese Regex
LANGREGEX = r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+"
# Pricing - Depends on the model https://openai.com/pricing
# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request
# If you are getting a MISMATCH LENGTH error, lower the batch size.
if "gpt-3.5" in MODEL:
INPUTAPICOST = 3.00
OUTPUTAPICOST = 5.00
BATCHSIZE = 10
FREQUENCY_PENALTY = 0.2
elif "gpt-4" in MODEL:
INPUTAPICOST = 2.0
OUTPUTAPICOST = 8.00
BATCHSIZE = 30
FREQUENCY_PENALTY = 0.05
elif "deepseek" in MODEL:
INPUTAPICOST = 0.27
OUTPUTAPICOST = 1.10
BATCHSIZE = 30
FREQUENCY_PENALTY = 0.05
else:
INPUTAPICOST = float(os.getenv("input_cost"))
OUTPUTAPICOST = float(os.getenv("output_cost"))
BATCHSIZE = int(os.getenv("batchsize"))
FREQUENCY_PENALTY = float(os.getenv("frequency_penalty"))
def handleAnim(filename, estimate):
global ESTIMATE
totalTokens = [0, 0]
ESTIMATE = estimate
if estimate:
start = time.time()
translatedData = openFiles(filename)
# Print Result
end = time.time()
tqdm.write(getResultString(translatedData, end - start, filename))
with LOCK:
totalTokens[0] += translatedData[1][0]
totalTokens[1] += translatedData[1][1]
# Print Total
totalString = getResultString(["", totalTokens, None], end - start, "TOTAL")
# Print any errors on maps
if len(MISMATCH) > 0:
return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
else:
return totalString
else:
try:
with open("translated/" + filename, "w", encoding="utf-8", newline="\n") as outFile:
start = time.time()
translatedData = openFiles(filename)
# Print Result
end = time.time()
json.dump(translatedData[0], outFile, ensure_ascii=False, indent=4)
tqdm.write(getResultString(translatedData, end - start, filename))
with LOCK:
totalTokens[0] += translatedData[1][0]
totalTokens[1] += translatedData[1][1]
except Exception:
return "Fail"
return getResultString(["", totalTokens, None], end - start, "TOTAL")
def openFiles(filename):
with open("files/" + filename, "r", encoding="UTF-8-sig") as f:
data = json.load(f)
# Map Files
if ".json" in filename:
translatedData = parseJSON(data, filename)
else:
raise NameError(filename + " Not Supported")
return translatedData
def getResultString(translatedData, translationTime, filename):
# File Print String
totalTokenstring = (
Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
"[Output: "
+ str(translatedData[1][1])
+ "]" "[Cost: ${:,.4f}".format(((translatedData[1][0] / 1000000) * INPUTAPICOST) + ((translatedData[1][1] / 1000000) * OUTPUTAPICOST))
+ "]"
)
timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
if translatedData[2] == None:
# Success
return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET
else:
# Fail
try:
raise translatedData[2]
except Exception as e:
traceback.print_exc()
errorString = str(e) + Fore.RED
return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET
def parseJSON(data, filename):
keys = list(data.keys())
batches = [keys[i : i + BATCHSIZE] for i in range(0, len(keys), BATCHSIZE)]
totalTokens = [0, 0]
totalLines = 0
totalLines = len(batches)
global LOCK
with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar:
pbar.desc = filename
pbar.total = totalLines
try:
result = translateJSON(batches, data, pbar)
totalTokens[0] += result[0]
totalTokens[1] += result[1]
except Exception as e:
traceback.print_exc()
return [data, totalTokens, e]
return [data, totalTokens, None]
def translateJSON(keys, data, pbar):
translatedBatch = []
textHistory = []
tokens = [0, 0]
for batch in keys:
# Save Batch
originalBatch = batch.copy()
# If there isn't any Japanese in the text just skip
needTL = False
for i in range(len(batch)):
t = data[batch[i]]
if re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", t) or t == "":
needTL = True
if needTL is False and IGNORETLTEXT is True:
pbar.update(1)
continue
# Remove any textwrap and Furigana
for i in range(len(batch)):
if FIXTEXTWRAP == True:
# Textwrap
data[originalBatch[i]] = data[originalBatch[i]].replace("@b", " ")
# Furigana
rcodeMatch = re.findall(r"(@\[(.+?):.+?\])", batch[i])
if len(rcodeMatch) > 0:
for match in rcodeMatch:
batch[i] = batch[i].replace(match[0], match[1])
# Translate
if needTL is True:
response = translateGPT(batch, textHistory, True)
tokens[0] += response[1][0]
tokens[1] += response[1][1]
translatedBatch = response[0]
else:
for i in range(len(originalBatch)):
translatedBatch.append(data[originalBatch[i]])
# Format and Set Text
if len(batch) == len(translatedBatch):
for i in range(len(translatedBatch)):
# Remove added speaker
translatedText = translatedBatch[i]
translatedText = re.sub(r"^.+?\s\|\s?", "", translatedText)
# Textwrap
if "@n" in translatedText:
match = re.search(r".*@n(.*)", translatedText)
if match != None:
tlText = match.group(1)
tlText = dazedwrap.wrapText(tlText, width=WIDTH)
tlText = tlText.replace("\n", "@b")
translatedText = translatedText.replace(match.group(1), tlText)
elif "@b" not in translatedText:
translatedText = dazedwrap.wrapText(translatedText, width=WIDTH)
translatedText = translatedText.replace("\n", "@b")
# Set Data
data[originalBatch[i]] = translatedText
textHistory = translatedBatch
translatedBatch.clear()
# Mismatch, Skip Batch
else:
MISMATCH.append(batch)
pbar.update(1)
continue
pbar.update(1)
return tokens
def subVars(jaString):
jaString = jaString.replace("\u3000", " ")
# Nested
count = 0
nestedList = re.findall(r"[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]", jaString)
nestedList = set(nestedList)
if len(nestedList) != 0:
for icon in nestedList:
jaString = jaString.replace(icon, "[Nested_" + str(count) + "]")
count += 1
# Icons
count = 0
iconList = re.findall(r"[\\]+[iIkKwWaA]+\[[0-9]+\]", jaString)
iconList = set(iconList)
if len(iconList) != 0:
for icon in iconList:
jaString = jaString.replace(icon, "[Ascii_" + str(count) + "]")
count += 1
# Colors
count = 0
colorList = re.findall(r"[\\]+[cC]\[[0-9]+\]", jaString)
colorList = set(colorList)
if len(colorList) != 0:
for color in colorList:
jaString = jaString.replace(color, "[Color_" + str(count) + "]")
count += 1
# Names
count = 0
nameList = re.findall(r"[\\]+[nN]\[.+?\]+", jaString)
nameList = set(nameList)
if len(nameList) != 0:
for name in nameList:
jaString = jaString.replace(name, "[Noun_" + str(count) + "]")
count += 1
# Variables
count = 0
varList = re.findall(r"[\\]+[vV]\[[0-9]+\]", jaString)
varList = set(varList)
if len(varList) != 0:
for var in varList:
jaString = jaString.replace(var, "[Var_" + str(count) + "]")
count += 1
# Formatting
count = 0
formatList = re.findall(r"[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]", jaString)
formatList = set(formatList)
if len(formatList) != 0:
for var in formatList:
jaString = jaString.replace(var, "[FCode_" + str(count) + "]")
count += 1
# Put all lists in list and return
allList = [nestedList, iconList, colorList, nameList, varList, formatList]
return [jaString, allList]
def resubVars(translatedText, allList):
# Fix Spacing and ChatGPT Nonsense
matchList = re.findall(r"\[\s?.+?\s?\]", translatedText)
if len(matchList) > 0:
for match in matchList:
text = match.strip()
translatedText = translatedText.replace(match, text)
# Nested
count = 0
if len(allList[0]) != 0:
for var in allList[0]:
translatedText = translatedText.replace("[Nested_" + str(count) + "]", var)
count += 1
# Icons
count = 0
if len(allList[1]) != 0:
for var in allList[1]:
translatedText = translatedText.replace("[Ascii_" + str(count) + "]", var)
count += 1
# Colors
count = 0
if len(allList[2]) != 0:
for var in allList[2]:
translatedText = translatedText.replace("[Color_" + str(count) + "]", var)
count += 1
# Names
count = 0
if len(allList[3]) != 0:
for var in allList[3]:
translatedText = translatedText.replace("[Noun_" + str(count) + "]", var)
count += 1
# Vars
count = 0
if len(allList[4]) != 0:
for var in allList[4]:
translatedText = translatedText.replace("[Var_" + str(count) + "]", var)
count += 1
# Formatting
count = 0
if len(allList[5]) != 0:
for var in allList[5]:
translatedText = translatedText.replace("[FCode_" + str(count) + "]", var)
count += 1
return translatedText
def batchList(input_list, batch_size):
if not isinstance(batch_size, int) or batch_size <= 0:
raise ValueError("batch_size must be a positive integer")
return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)]
def createContext(fullPromptFlag, subbedT):
characters = "Game Characters:\n\
達也 (Tatsuya) - Male\n\
香織 (Kaori) - Female\n\
岩瀬 (Iwase)\n\
万蔵 (Manzou) - Male\n\
結奈 (Yuuna) - Female\n\
茅部 (Kayabe)\n\
"
system = (
PROMPT + VOCAB
if fullPromptFlag
else f"\
You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
- You translate everything, including erotic and explicit content.\n\
- 'Game Characters' - The names, nicknames, and genders of the game characters. Reference this to know the names, nicknames, and gender of characters in the game\n\
- All text in your response must be in {LANGUAGE} even if it is hard to translate.\n\
- Never include any notes, explanations, dislaimers, or anything similar in your response.\n\
- Maintain any spacing in the translation.\n\
- Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\
- `...` can be a part of the dialogue. Translate it as it is.\n\
{VOCAB}\n\
"
)
user = f"{subbedT}"
return characters, system, user
def translateText(characters, system, user, history, penalty):
# Prompt
msg = [{"role": "system", "content": system + characters}]
# Characters
msg.append({"role": "system", "content": characters})
# History
if isinstance(history, list):
msg.extend([{"role": "system", "content": h} for h in history])
else:
msg.append({"role": "system", "content": history})
# Content to TL
msg.append({"role": "user", "content": f"{user}"})
response = openai.chat.completions.create(
temperature=0,
frequency_penalty=penalty,
model=MODEL,
messages=msg,
)
return response
def cleanTranslatedText(translatedText, varResponse):
placeholders = {
f"{LANGUAGE} Translation: ": "",
"Translation: ": "",
"っ": "",
"〜": "~",
"ッ": "",
"。": ".",
"Placeholder Text": "",
"é": "e",
"—": "-",
"ū": "u",
# Add more replacements as needed
}
for target, replacement in placeholders.items():
translatedText = translatedText.replace(target, replacement)
# Elongate Long Dashes (Since GPT Ignores them...)
translatedText = elongateCharacters(translatedText)
translatedText = resubVars(translatedText, varResponse[1])
return translatedText
def elongateCharacters(text):
# Define a pattern to match one character followed by one or more `ー` characters
# Using a positive lookbehind assertion to capture the preceding character
pattern = r"(?<=(.))ー+"
# Define a replacement function that elongates the captured character
def repl(match):
char = match.group(1) # The character before the ー sequence
count = len(match.group(0)) - 1 # Number of ー characters
return char * count # Replace ー sequence with the character repeated
# Use re.sub() to replace the pattern in the text
return re.sub(pattern, repl, text)
def extractTranslation(translatedTextList, is_list):
pattern = r"`?<[Ll]ine\d+>([\\]*.*?[\\]*?)<\/?[Ll]ine\d+>`?"
# If it's a batch (i.e., list), extract with tags; otherwise, return the single item.
if is_list:
matchList = re.findall(pattern, translatedTextList)
return matchList
else:
matchList = re.findall(pattern, translatedTextList)
return matchList[0][0] if matchList else translatedTextList
def countTokens(characters, system, user, history):
inputTotalTokens = 0
outputTotalTokens = 0
enc = tiktoken.encoding_for_model("gpt-4")
# Input
if isinstance(history, list):
for line in history:
inputTotalTokens += len(enc.encode(line))
else:
inputTotalTokens += len(enc.encode(history))
inputTotalTokens += len(enc.encode(system))
inputTotalTokens += len(enc.encode(characters))
inputTotalTokens += len(enc.encode(user))
# Output
outputTotalTokens += round(len(enc.encode(user)) * 3)
return [inputTotalTokens, outputTotalTokens]
@retry(exceptions=Exception, tries=5, delay=5)
def translateGPT(text, history, fullPromptFlag):
mismatch = False
totalTokens = [0, 0]
if isinstance(text, list):
tList = batchList(text, BATCHSIZE)
else:
tList = [text]
for index, tItem in enumerate(tList):
# Before sending to translation, if we have a list of items, add the formatting
if isinstance(tItem, list):
payload = "\n".join([f"`<Line{i}>{item}</Line{i}>`" for i, item in enumerate(tItem)])
payload = re.sub(r"(<Line\d+)(><)(\/Line\d+>)", r"\1>Placeholder Text<\3", payload)
varResponse = subVars(payload)
subbedT = varResponse[0]
else:
varResponse = subVars(tItem)
subbedT = varResponse[0]
# Things to Check before starting translation
if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", subbedT):
continue
# Create Message
characters, system, user = createContext(fullPromptFlag, subbedT)
# Calculate Estimate
if ESTIMATE:
estimate = countTokens(characters, system, user, history)
totalTokens[0] += estimate[0]
totalTokens[1] += estimate[1]
continue
# Translating
response = translateText(characters, system, user, history, 0.02)
translatedText = response.choices[0].message.content
totalTokens[0] += response.usage.prompt_tokens
totalTokens[1] += response.usage.completion_tokens
# Formatting
translatedText = cleanTranslatedText(translatedText, varResponse)
if isinstance(tItem, list):
extractedTranslations = extractTranslation(translatedText, True)
tList[index] = extractedTranslations
if len(tItem) != len(extractedTranslations):
# Mismatch. Try Again
response = translateText(characters, system, user, history, 0.1)
translatedText = response.choices[0].message.content
totalTokens[0] += response.usage.prompt_tokens
totalTokens[1] += response.usage.completion_tokens
# Formatting
translatedText = cleanTranslatedText(translatedText, varResponse)
if isinstance(tItem, list):
extractedTranslations = extractTranslation(translatedText, True)
tList[index] = extractedTranslations
if len(tItem) != len(extractedTranslations):
mismatch = True # Just here for breakpoint
# Create History
if not mismatch:
history = extractedTranslations[-10:] # Update history if we have a list
else:
history = text[-10:]
else:
# Ensure we're passing a single string to extractTranslation
extractedTranslations = extractTranslation(translatedText, False)
tList[index] = extractedTranslations
# Combine if multilist
if isinstance(tList[0], list):
tList = [t for sublist in tList for t in sublist]
# Return
if format == "json":
return [tList, totalTokens]
else:
return [tList[0], totalTokens]

396
modules/atelier.py
View file

@ -1,396 +0,0 @@
import os
from pathlib import Path
import re
import util.dazedwrap as dazedwrap
import threading
import time
import traceback
import tiktoken
from colorama import Fore
from dotenv import load_dotenv
import openai
from retry import retry
from tqdm import tqdm
# Open AI
load_dotenv()
if os.getenv("api").replace(" ", "") != "":
openai.base_url = os.getenv("api")
openai.organization = os.getenv("org")
openai.api_key = os.getenv("key")
# Globals
MODEL = os.getenv("model")
TIMEOUT = int(os.getenv("timeout"))
LANGUAGE = os.getenv("language").capitalize()
INPUTAPICOST = 0.002 # Depends on the model https://openai.com/pricing
OUTPUTAPICOST = 0.002
PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
VOCAB = Path("vocab.txt").read_text(encoding="utf-8")
THREADS = int(os.getenv("threads")) # Controls how many threads are working on a single file (May have to drop this)
LOCK = threading.Lock()
WIDTH = int(os.getenv("width"))
LISTWIDTH = int(os.getenv("listWidth"))
NOTEWIDTH = 40
MAXHISTORY = 10
ESTIMATE = ""
totalTokens = [0, 0]
NAMESLIST = []
# tqdm Globals
BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
POSITION = 0
LEAVE = False
# Translation Flags
FIXTEXTWRAP = True
IGNORETLTEXT = True
def handleAtelier(filename, estimate):
global ESTIMATE, totalTokens
ESTIMATE = estimate
if estimate:
start = time.time()
translatedData = openFiles(filename)
# Print Result
end = time.time()
tqdm.write(getResultString(translatedData, end - start, filename))
with LOCK:
totalTokens[0] += translatedData[1][0]
totalTokens[1] += translatedData[1][1]
return getResultString(["", totalTokens, None], end - start, "TOTAL")
else:
try:
with open("translated/" + filename, "w", encoding="utf-8", newline="\n") as outFile:
start = time.time()
translatedData = openFiles(filename)
outFile.writelines(translatedData[0])
# Print Result
end = time.time()
tqdm.write(getResultString(translatedData, end - start, filename))
with LOCK:
totalTokens[0] += translatedData[1][0]
totalTokens[1] += translatedData[1][1]
except Exception:
return "Fail"
return getResultString(["", totalTokens, None], end - start, "TOTAL")
def openFiles(filename):
with open("files/" + filename, "r", encoding="UTF-8") as f:
translatedData = parseText(f, filename)
return translatedData
def getResultString(translatedData, translationTime, filename):
# File Print String
totalTokenstring = (
Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
"[Output: "
+ str(translatedData[1][1])
+ "]" "[Cost: ${:,.4f}".format(((translatedData[1][0] / 1000000) * INPUTAPICOST) + ((translatedData[1][1] / 1000000) * OUTPUTAPICOST))
+ "]"
)
timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
if translatedData[2] is None:
# Success
return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET
else:
# Fail
try:
raise translatedData[2]
except Exception as e:
errorString = str(e) + Fore.RED
return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET
def parseText(data, filename):
totalLines = 0
global LOCK
# Get total for progress bar
linesList = data.readlines()
totalLines = len(linesList)
with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar:
pbar.desc = filename
pbar.total = totalLines
try:
response = translateText(linesList, pbar)
except Exception as e:
traceback.print_exc()
return [linesList, 0, e]
return [response[0], response[1], None]
def translateText(data, pbar):
textHistory = []
maxHistory = MAXHISTORY
totalTokens = [0, 0]
syncIndex = 0
for i in range(len(data)):
if syncIndex > i:
i = syncIndex
match = re.findall(r"◆.+◆(.+)", data[i])
if len(match) > 0:
jaString = match[0]
### Translate
# Remove any textwrap
finalJAString = re.sub(r"\\n", " ", jaString)
# Translate
response = translateGPT(
finalJAString,
"Previous Text for Context: " + " ".join(textHistory),
True,
)
totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1]
translatedText = response[0]
# TextHistory is what we use to give GPT Context, so thats appended here.
textHistory.append('"' + translatedText + '"')
# Keep textHistory list at length maxHistory
if len(textHistory) > maxHistory:
textHistory.pop(0)
# Textwrap
translatedText = dazedwrap.wrapText(translatedText, width=WIDTH)
translatedText = translatedText.replace("\n", "\\n")
# Write
data[i] = data[i].replace(match[0], translatedText)
syncIndex = i + 1
pbar.update()
return [data, totalTokens]
def subVars(jaString):
jaString = jaString.replace("\u3000", " ")
# Nested
count = 0
nestedList = re.findall(r"[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]", jaString)
nestedList = set(nestedList)
if len(nestedList) != 0:
for icon in nestedList:
jaString = jaString.replace(icon, "{Nested_" + str(count) + "}")
count += 1
# Icons
count = 0
iconList = re.findall(r"[\\]+[iIkKwWaA]+\[[0-9]+\]", jaString)
iconList = set(iconList)
if len(iconList) != 0:
for icon in iconList:
jaString = jaString.replace(icon, "{Ascii_" + str(count) + "}")
count += 1
# Colors
count = 0
colorList = re.findall(r"[\\]+[cC]\[[0-9]+\]", jaString)
colorList = set(colorList)
if len(colorList) != 0:
for color in colorList:
jaString = jaString.replace(color, "{Color_" + str(count) + "}")
count += 1
# Names
count = 0
nameList = re.findall(r"[\\]+[nN]\[.+?\]+", jaString)
nameList = set(nameList)
if len(nameList) != 0:
for name in nameList:
jaString = jaString.replace(name, "{N_" + str(count) + "}")
count += 1
# Variables
count = 0
varList = re.findall(r"[\\]+[vV]\[[0-9]+\]", jaString)
varList = set(varList)
if len(varList) != 0:
for var in varList:
jaString = jaString.replace(var, "{Var_" + str(count) + "}")
count += 1
# Formatting
count = 0
if "笑えるよね." in jaString:
print("t")
formatList = re.findall(r"[\\]+[\w]+\[.+?\]", jaString)
formatList = set(formatList)
if len(formatList) != 0:
for var in formatList:
jaString = jaString.replace(var, "{FCode_" + str(count) + "}")
count += 1
# Put all lists in list and return
allList = [nestedList, iconList, colorList, nameList, varList, formatList]
return [jaString, allList]
def resubVars(translatedText, allList):
# Fix Spacing and ChatGPT Nonsense
matchList = re.findall(r"\[\s?.+?\s?\]", translatedText)
if len(matchList) > 0:
for match in matchList:
text = match.strip()
translatedText = translatedText.replace(match, text)
# Nested
count = 0
if len(allList[0]) != 0:
for var in allList[0]:
translatedText = translatedText.replace("{Nested_" + str(count) + "}", var)
count += 1
# Icons
count = 0
if len(allList[1]) != 0:
for var in allList[1]:
translatedText = translatedText.replace("{Ascii_" + str(count) + "}", var)
count += 1
# Colors
count = 0
if len(allList[2]) != 0:
for var in allList[2]:
translatedText = translatedText.replace("{Color_" + str(count) + "}", var)
count += 1
# Names
count = 0
if len(allList[3]) != 0:
for var in allList[3]:
translatedText = translatedText.replace("{N_" + str(count) + "}", var)
count += 1
# Vars
count = 0
if len(allList[4]) != 0:
for var in allList[4]:
translatedText = translatedText.replace("{Var_" + str(count) + "}", var)
count += 1
# Formatting
count = 0
if len(allList[5]) != 0:
for var in allList[5]:
translatedText = translatedText.replace("{FCode_" + str(count) + "}", var)
count += 1
# Remove Color Variables Spaces
# if '\\c' in translatedText:
# translatedText = re.sub(r'\s*(\\+c\[[1-9]+\])\s*', r' \1', translatedText)
# translatedText = re.sub(r'\s*(\\+c\[0+\])', r'\1', translatedText)
return translatedText
@retry(exceptions=Exception, tries=5, delay=5)
def translateGPT(t, history, fullPromptFlag):
# Sub Vars
varResponse = subVars(t)
subbedT = varResponse[0]
# If there isn't any Japanese in the text just skip
if not re.search(r"[一-龠]+|[ぁ-ゔ]+|[ァ-ヴ]+|[\uFF00-\uFFEF]", subbedT):
return (t, [0, 0])
# If ESTIMATE is True just count this as an execution and return.
if ESTIMATE:
enc = tiktoken.encoding_for_model("gpt-4")
historyRaw = ""
if isinstance(history, list):
for line in history:
historyRaw += line
else:
historyRaw = history
inputTotalTokens = len(enc.encode(historyRaw)) + len(enc.encode(PROMPT))
outputTotalTokens = len(enc.encode(t)) * 2 # Estimating 2x the size of the original text
totalTokens = [inputTotalTokens, outputTotalTokens]
return (t, totalTokens)
# Characters
context = "Game Characters:\
Character: Surname:久高 Name:有史 == Surname:Kudaka Name:Yuushi - Gender: Male\
Character: Surname:葛城 Name:碧璃 == Surname:Katsuragi Name:Midori - Gender: Female\
Character: Surname:葛城 Name:依理子 == Surname:Katsuragi Name:Yoriko - Gender: Female\
Character: Surname:桐乃木 Name:奏 == Surname:Kirinogi Name:Kanade - Gender: Female\
Character: Surname:葛城 Name:光男 == Surname:Katsuragi Name:Mitsuo - Gender: Male\
Character: Surname:尾木 Name:優真 == Surname:Ogi Name:Yuuma - Gender: Male"
# Prompt
if fullPromptFlag:
system = PROMPT
user = "Line to Translate = " + subbedT
else:
system = "Output ONLY the " + LANGUAGE + " translation in the following format: `Translation: <" + LANGUAGE.upper() + "_TRANSLATION>`"
user = "Line to Translate = " + subbedT
# Create Message List
msg = []
msg.append({"role": "system", "content": system})
msg.append({"role": "user", "content": context})
if isinstance(history, list):
for line in history:
msg.append({"role": "user", "content": line})
else:
msg.append({"role": "user", "content": history})
msg.append({"role": "user", "content": user})
response = openai.ChatCompletion.create(
temperature=0,
frequency_penalty=0.2,
presence_penalty=0.2,
model=MODEL,
messages=msg,
request_timeout=TIMEOUT,
)
# Save Translated Text
translatedText = response.choices[0].message.content
totalTokens = [response.usage.prompt_tokens, response.usage.completion_tokens]
# Resub Vars
translatedText = resubVars(translatedText, varResponse[1])
# Remove Placeholder Text
translatedText = translatedText.replace(LANGUAGE + " Translation: ", "")
translatedText = translatedText.replace("Translation: ", "")
translatedText = translatedText.replace("Line to Translate = ", "")
translatedText = translatedText.replace("Translation = ", "")
translatedText = translatedText.replace("Translate = ", "")
translatedText = translatedText.replace(LANGUAGE + " Translation:", "")
translatedText = translatedText.replace("Translation:", "")
translatedText = translatedText.replace("Line to Translate =", "")
translatedText = translatedText.replace("Translation =", "")
translatedText = translatedText.replace("Translate =", "")
translatedText = translatedText.replace("っ", "")
translatedText = translatedText.replace("ッ", "")
translatedText = translatedText.replace("ぁ", "")
translatedText = translatedText.replace("。", ".")
translatedText = translatedText.replace("、", ",")
translatedText = translatedText.replace("？", "?")
translatedText = translatedText.replace("！", "!")
# Return Translation
if len(translatedText) > 15 * len(t) or "I'm sorry, but I'm unable to assist with that translation" in translatedText:
raise Exception
else:
return [translatedText, totalTokens]

90
modules/csv.py
View file

@ -439,7 +439,6 @@ def translateCSV(data, pbar, writer, filename, translatedList, format):
return totalTokens
# Save some money and enter the character before translation
def getSpeaker(speaker):
match speaker:
@ -453,15 +452,11 @@ def getSpeaker(speaker):
if speaker == NAMESLIST[i][0]:
return [NAMESLIST[i][1], [0, 0]]
# If there isn't any Japanese in the text just skip
if not re.search(LANGREGEX, speaker):
return [speaker, [0, 0]]
# Translate and Store Speaker
response = translateGPT(
f"{speaker}",
"Reply with the " + LANGUAGE + " translation of the NPC name.",
True,
False,
)
response[0] = response[0].title()
response[0] = response[0].replace("'S", "'s")
@ -490,11 +485,68 @@ def batchList(input_list, batch_size):
return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)]
def parseVocabWithCategories(vocabText):
"""Parse vocabulary text and extract terms with their categories."""
pairs = []
seen = set()
currentCategory = None
for line in vocabText.splitlines():
line = line.strip()
if not line or line.startswith('```'):
continue
# Check if this is a category header
if line.startswith('#'):
currentCategory = line
continue
# Parse vocabulary term
m = re.match(r'^(.+?)(?:\s?[\(–])', line) # term is everything before space + '(' or '–'
if m:
term = m.group(1)
if term not in seen:
pairs.append((term, line, currentCategory))
seen.add(term)
return pairs
def buildMatchedVocabText(vocabPairs, subbedT):
"""Build formatted vocabulary text with matched terms organized by category."""
matchedCategories = {}
# Use word boundaries for Japanese if appropriate, or allow substring as before.
for term, line, category in vocabPairs:
# "term in subbedT" could be false positive; can use regex but Japanese doesn't always have spaces.
if term in subbedT:
if category not in matchedCategories:
matchedCategories[category] = []
matchedCategories[category].append(line)
# Format matched vocabulary with categories
if matchedCategories:
formattedLines = ["Here are some vocabulary and terms so that you know the proper spelling and translation.\n"]
for category, lines in matchedCategories.items():
if category: # Only add category header if it exists
formattedLines.append(category)
formattedLines.extend(lines)
formattedLines.append("") # Add blank line between categories
matchedVocabText = f"```\n{chr(10).join(formattedLines).rstrip()}\n```"
else:
matchedVocabText = ""
return matchedVocabText
def createContext(fullPromptFlag, subbedT, format):
system = (
PROMPT + VOCAB
if fullPromptFlag
else f"\
vocabPairs = parseVocabWithCategories(VOCAB)
matchedVocabText = buildMatchedVocabText(vocabPairs, subbedT)
if fullPromptFlag:
system = PROMPT + matchedVocabText
else:
system = f"\
You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
- You translate everything, including erotic and explicit content.\n\
@ -504,9 +556,8 @@ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{
- Maintain any spacing in the translation.\n\
- Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\
- `...` can be a part of the dialogue. Translate it as it is.\n\
{VOCAB}\n\
{matchedVocabText}\n\
"
)
if format == "json":
user = f"```json\n{subbedT}\n```"
else:
@ -520,9 +571,10 @@ def translateText(system, user, history, penalty, format, model=MODEL):
# History
if isinstance(history, list):
msg.extend([{"role": "system", "content": h} for h in history])
msg.append({"role": "system", "content": "Translation History:"})
msg.extend([{"role": "assistant", "content": h} for h in history])
else:
msg.append({"role": "system", "content": history})
msg.append({"role": "assistant", "content": history})
# Response Format
if format == "json":
@ -557,8 +609,11 @@ def cleanTranslatedText(translatedText):
"】": "]",
"【": "[",
"é": "e",
"ō": "o",
"this guy": "this bastard",
"This guy": "This bastard",
"Placeholder Text": "",
"```json": "",
"```": "",
# Add more replacements as needed
}
for target, replacement in placeholders.items():
@ -620,7 +675,7 @@ def countTokens(system, user, history):
inputTotalTokens += len(enc.encode(user))
# Output
outputTotalTokens += round(len(enc.encode(user)) * 3)
outputTotalTokens += round(len(enc.encode(user)) * 2.5)
return [inputTotalTokens, outputTotalTokens]
@ -684,7 +739,8 @@ def translateGPT(text, history, fullPromptFlag):
# AI Refused, Try Again
if not translatedText:
response = translateText(f"{system}\n You translate ALL content.", user, history, 0.1, format)
response = translateText(f"{system}\n You translate ALL content.", user, history, 0.1, format, model="gpt-4o")
translatedText = response.choices[0].message.content
# Report Tokens
totalTokens[0] += response.usage.prompt_tokens

747
modules/eushully.py
View file

@ -1,747 +0,0 @@
# Libraries
import os
import re
import util.dazedwrap as dazedwrap
import threading
import time
import traceback
import tiktoken
import openai
from pathlib import Path
from colorama import Fore
from dotenv import load_dotenv
from retry import retry
from tqdm import tqdm
# Open AI
load_dotenv()
if os.getenv("api").replace(" ", "") != "":
openai.base_url = os.getenv("api")
openai.organization = os.getenv("org")
openai.api_key = os.getenv("key")
# Globals
MODEL = os.getenv("model")
TIMEOUT = int(os.getenv("timeout"))
LANGUAGE = os.getenv("language").capitalize()
PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
VOCAB = Path("vocab.txt").read_text(encoding="utf-8")
THREADS = int(os.getenv("threads"))
LOCK = threading.Lock()
WIDTH = int(os.getenv("width"))
LISTWIDTH = int(os.getenv("listWidth"))
NOTEWIDTH = 70
MAXHISTORY = 10
ESTIMATE = ""
TOKENS = [0, 0]
NAMESLIST = []
NAMES = False # Output a list of all the character names found
BRFLAG = False # If the game uses <br> instead
FIXTEXTWRAP = True # Overwrites textwrap
IGNORETLTEXT = False # Ignores all translated text.
MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
# tqdm Globals
BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
POSITION = 0
LEAVE = False
PBAR = None
# Regex - Need to change this if you want to translate from/to other languages. Default is Japanese Regex
LANGREGEX = r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+"
# Pricing - Depends on the model https://openai.com/pricing
# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request
# If you are getting a MISMATCH LENGTH error, lower the batch size.
if "gpt-3.5" in MODEL:
INPUTAPICOST = 3.00
OUTPUTAPICOST = 5.00
BATCHSIZE = 10
FREQUENCY_PENALTY = 0.2
elif "gpt-4" in MODEL:
INPUTAPICOST = 2.0
OUTPUTAPICOST = 8.00
BATCHSIZE = 30
FREQUENCY_PENALTY = 0.05
elif "deepseek" in MODEL:
INPUTAPICOST = 0.27
OUTPUTAPICOST = 1.10
BATCHSIZE = 30
FREQUENCY_PENALTY = 0.05
else:
INPUTAPICOST = float(os.getenv("input_cost"))
OUTPUTAPICOST = float(os.getenv("output_cost"))
BATCHSIZE = int(os.getenv("batchsize"))
FREQUENCY_PENALTY = float(os.getenv("frequency_penalty"))
def handleEushully(filename, estimate):
global ESTIMATE
ESTIMATE = estimate
if ESTIMATE:
start = time.time()
translatedData = openFiles(filename)
# Print Result
end = time.time()
tqdm.write(getResultString(translatedData, end - start, filename))
with LOCK:
TOKENS[0] += translatedData[1][0]
TOKENS[1] += translatedData[1][1]
# Print Total
totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
# Print any errors on maps
if len(MISMATCH) > 0:
return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
else:
return totalString
else:
try:
with open("translated/" + filename, "w", encoding="utf-8", errors="ignore") as outFile:
start = time.time()
translatedData = openFiles(filename)
# Print Result
end = time.time()
outFile.writelines(translatedData[0])
tqdm.write(getResultString(translatedData, end - start, filename))
with LOCK:
TOKENS[0] += translatedData[1][0]
TOKENS[1] += translatedData[1][1]
except Exception:
traceback.print_exc()
return "Fail"
return getResultString(["", TOKENS, None], end - start, "TOTAL")
def getResultString(translatedData, translationTime, filename):
# File Print String
totalTokenstring = (
Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
"[Output: "
+ str(translatedData[1][1])
+ "]" "[Cost: ${:,.4f}".format(((translatedData[1][0] / 1000000) * INPUTAPICOST) + ((translatedData[1][1] / 1000000) * OUTPUTAPICOST))
+ "]"
)
timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
if translatedData[2] == None:
# Success
return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET
else:
# Fail
try:
raise translatedData[2]
except Exception as e:
traceback.print_exc()
errorString = str(e) + Fore.RED
return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET
def openFiles(filename):
with open("files/" + filename, "r", encoding="utf-8") as readFile:
translatedData = parseRegex(readFile, filename)
# Delete lines marked for deletion
finalData = []
for line in translatedData[0]:
if line != "\\d\n":
finalData.append(line)
translatedData[0] = finalData
return translatedData
def parseRegex(readFile, filename):
totalTokens = [0, 0]
# Read File into data
data = readFile.readlines()
# Create Progress Bar
with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
pbar.desc = filename
try:
result = translateEushully(data, pbar, filename, [])
totalTokens[0] += result[0]
totalTokens[1] += result[1]
except Exception as e:
traceback.print_exc()
return [data, totalTokens, e]
return [data, totalTokens, None]
def translateEushully(data, pbar, filename, translatedList):
stringList = []
currentGroup = []
tokens = [0, 0]
speaker = ""
voice = False
global LOCK, ESTIMATE, PBAR
i = 0
while i < len(data):
voice = False
# Speaker
if "mov (global-int 46e2)" in data[i]:
# Get Speaker
speaker = re.search(r"mov \(global-int 46e2\)\s(.+)", data[i]).group(1)
response = getSpeaker(speaker)
speaker = response[0]
tokens[0] += response[1][0]
tokens[1] += response[1][1]
i += 1
# Show Text
if any(x in data[i] for x in ["show-text"]):
# Lines
regex = r'(.*?)"(.*)"'
match = re.search(regex, data[i])
# Grab Strings
if match != None and match.group(2) != "":
originalString = match.group(2)
jaString = match.group(2)
currentGroup = [jaString]
while "end-text-line" in data[i + 1] and any(x in data[i + 2] for x in ["show-text"]):
match = re.search(regex, data[i + 2])
if match != None:
currentGroup.append(match.group(2))
if translatedList == []:
del data[i]
del data[i]
jaString = " ".join(currentGroup)
# Pass 1
if translatedList == []:
# Add String
if speaker:
stringList.append(f"[{speaker}]: {jaString.strip()}")
else:
stringList.append(jaString.strip())
# Pass 2
else:
# Get Text
if translatedList:
# Grab and Pop
translatedText = translatedList[0]
translatedList.pop(0)
# Set to None if empty list
if len(translatedList) <= 0:
translatedList = None
# Replace Quotes
translatedText = translatedText.replace('"', "'")
# Remove speaker
if speaker != "":
translatedText = re.sub(r"^\[?(.+?)\]?\s?[|:]\s?", "", translatedText)
# Textwrap
translatedText = dazedwrap.wrapText(translatedText, width=WIDTH)
translatedTextList = translatedText.split("\n")
# Set Data
if len(translatedTextList) > 1:
for j in range(len(translatedTextList)):
if any(x in data[i] for x in ["show-text", "set-string", "concat"]):
del data[i]
data.insert(i, f'{match.group(1)}"{translatedTextList[j]}"\n')
i += 1
if "end-text-line" not in data[i]:
data.insert(i, "end-text-line 0\n")
i += 1
else:
data[i] = f'{match.group(1)}"{translatedTextList[0]}"\n'
speaker = ""
i += 1
# Nothing relevant. Skip Line.
else:
i += 1
# Set String
elif "set-string" in data[i]:
# Lines
regex = r'(.*?)"(.*)"'
match = re.search(regex, data[i])
# Grab Strings
if match != None and match.group(2) != "":
originalString = match.group(2)
jaString = match.group(2)
currentGroup = [jaString]
# Remove Textwrap
jaString = jaString.replace("\\n", " ")
# Pass 1
if translatedList == []:
# Add String
stringList.append(jaString.strip())
# Pass 2
else:
# Get Text
if translatedList:
# Grab and Pop
translatedText = translatedList[0]
translatedList.pop(0)
# Set to None if empty list
if len(translatedList) <= 0:
translatedList = None
# Replace Quotes
translatedText = translatedText.replace('"', "'")
# Textwrap
translatedText = dazedwrap.wrapText(translatedText, width=LISTWIDTH)
translatedText = translatedText.replace("\n", "\\n")
# Set Data
data[i] = data[i].replace(originalString, translatedText)
speaker = ""
i += 1
# Nothing relevant. Skip Line.
else:
i += 1
else:
i += 1
# EOF
if len(stringList) > 0:
# Set Progress
pbar.total = len(stringList)
pbar.refresh()
# Translate
PBAR = pbar
response = translateGPT(stringList, "", True)
tokens[0] += response[1][0]
tokens[1] += response[1][1]
translatedList = response[0]
# Set Strings
if len(stringList) == len(translatedList):
translateEushully(data, pbar, filename, translatedList)
# Mismatch
else:
with LOCK:
if filename not in MISMATCH:
MISMATCH.append(filename)
return tokens
# Save some money and enter the character before translation
def getSpeaker(speaker):
match speaker:
case "1":
return ["Klaus", [0, 0]]
case "2":
return ["Helmina", [0, 0]]
case "3":
return ["Juliana", [0, 0]]
case "4":
return ["Reginia", [0, 0]]
case "5":
return ["Luciel", [0, 0]]
case "6":
return ["Mavislaine", [0, 0]]
case "7":
return ["Cerouge", [0, 0]]
case "8":
return ["Maize", [0, 0]]
case "9":
return ["Elvire", [0, 0]]
case "a":
return ["Beatrice", [0, 0]]
case "295":
return ["Orc", [0, 0]]
case "232":
return ["Archangel", [0, 0]]
case "238":
return ["False Juliana", [0, 0]]
case "239":
return ["False Regina", [0, 0]]
case "23a":
return ["False Luciel", [0, 0]]
case "23d":
return ["False Mavislaine", [0, 0]]
case "cb":
return ["Olga Niza Kite", [0, 0]]
case "c9":
return ["Demon Beast Lupus", [0, 0]]
case "ca":
return ["Evelinael", [0, 0]]
case "10":
return ["Eukleia", [0, 0]]
case "15":
return ["Lily", [0, 0]]
case "16":
return ["Kupuko", [0, 0]]
case "b":
return ["Ramiel", [0, 0]]
case "c":
return ["Henriette", [0, 0]]
case "d":
return ["Camilla", [0, 0]]
case "cc":
return ["Gogonaua", [0, 0]]
case "65":
return ["Demon Lord Reyvalois", [0, 0]]
case "d0":
return ["Demon Ranwald", [0, 0]]
case "205":
return ["Vanqueor", [0, 0]]
case "66":
return ["Angel Martina", [0, 0]]
case "21f":
return ["Hiten Demon", [0, 0]]
case "d2":
return ["Lena Eli", [0, 0]]
case _:
return ["Unknown", [0, 0]]
def subVars(jaString):
jaString = jaString.replace("\u3000", " ")
# Nested
count = 0
nestedList = re.findall(r"[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]", jaString)
nestedList = set(nestedList)
if len(nestedList) != 0:
for icon in nestedList:
jaString = jaString.replace(icon, "[Nested_" + str(count) + "]")
count += 1
# Icons
count = 0
iconList = re.findall(r"[\\]+[iIkKwWaA]+\[[0-9]+\]", jaString)
iconList = set(iconList)
if len(iconList) != 0:
for icon in iconList:
jaString = jaString.replace(icon, "[Ascii_" + str(count) + "]")
count += 1
# Colors
count = 0
colorList = re.findall(r"[\\]+[cC]\[[0-9]+\]", jaString)
colorList = set(colorList)
if len(colorList) != 0:
for color in colorList:
jaString = jaString.replace(color, "[Color_" + str(count) + "]")
count += 1
# Names
count = 0
nameList = re.findall(r"[\\]+[nN]\[.+?\]+", jaString)
nameList = set(nameList)
if len(nameList) != 0:
for name in nameList:
jaString = jaString.replace(name, "[Noun_" + str(count) + "]")
count += 1
# Variables
count = 0
varList = re.findall(r"[\\]+[vV]\[[0-9]+\]", jaString)
varList = set(varList)
if len(varList) != 0:
for var in varList:
jaString = jaString.replace(var, "[Var_" + str(count) + "]")
count += 1
# Formatting
count = 0
formatList = re.findall(r"[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]", jaString)
formatList = set(formatList)
if len(formatList) != 0:
for var in formatList:
jaString = jaString.replace(var, "[FCode_" + str(count) + "]")
count += 1
# Put all lists in list and return
allList = [nestedList, iconList, colorList, nameList, varList, formatList]
return [jaString, allList]
def resubVars(translatedText, allList):
# Fix Spacing and ChatGPT Nonsense
matchList = re.findall(r"\[\s?.+?\s?\]", translatedText)
if len(matchList) > 0:
for match in matchList:
text = match.strip()
translatedText = translatedText.replace(match, text)
# Nested
count = 0
if len(allList[0]) != 0:
for var in allList[0]:
translatedText = translatedText.replace("[Nested_" + str(count) + "]", var)
count += 1
# Icons
count = 0
if len(allList[1]) != 0:
for var in allList[1]:
translatedText = translatedText.replace("[Ascii_" + str(count) + "]", var)
count += 1
# Colors
count = 0
if len(allList[2]) != 0:
for var in allList[2]:
translatedText = translatedText.replace("[Color_" + str(count) + "]", var)
count += 1
# Names
count = 0
if len(allList[3]) != 0:
for var in allList[3]:
translatedText = translatedText.replace("[Noun_" + str(count) + "]", var)
count += 1
# Vars
count = 0
if len(allList[4]) != 0:
for var in allList[4]:
translatedText = translatedText.replace("[Var_" + str(count) + "]", var)
count += 1
# Formatting
count = 0
if len(allList[5]) != 0:
for var in allList[5]:
translatedText = translatedText.replace("[FCode_" + str(count) + "]", var)
count += 1
return translatedText
def batchList(input_list, batch_size):
if not isinstance(batch_size, int) or batch_size <= 0:
raise ValueError("batch_size must be a positive integer")
return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)]
def createContext(fullPromptFlag, subbedT):
characters = "Game Characters:\n\
グレイス (Grace) - Female\n\
"
system = (
PROMPT + VOCAB
if fullPromptFlag
else f"\
You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
- You translate everything, including erotic and explicit content.\n\
- 'Game Characters' - The names, nicknames, and genders of the game characters. Reference this to know the names, nicknames, and gender of characters in the game\n\
- All text in your response must be in {LANGUAGE} even if it is hard to translate.\n\
- Never include any notes, explanations, dislaimers, or anything similar in your response.\n\
- Maintain any spacing in the translation.\n\
- Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\
- `...` can be a part of the dialogue. Translate it as it is.\n\
{VOCAB}\n\
"
)
user = f"{subbedT}"
return characters, system, user
def translateText(characters, system, user, history, penalty):
# Prompt
msg = [{"role": "system", "content": system + characters}]
# Characters
msg.append({"role": "system", "content": characters})
# History
if isinstance(history, list):
msg.extend([{"role": "system", "content": h} for h in history])
else:
msg.append({"role": "system", "content": history})
# Content to TL
msg.append({"role": "user", "content": f"{user}"})
response = openai.chat.completions.create(
temperature=0,
frequency_penalty=penalty,
model=MODEL,
messages=msg,
)
return response
def cleanTranslatedText(translatedText, varResponse):
placeholders = {
f"{LANGUAGE} Translation: ": "",
"Translation: ": "",
"っ": "",
"〜": "~",
"ッ": "",
"。": ".",
"< ": "<",
"</ ": "</",
" >": ">",
"「": '"',
"」": '"',
"Placeholder Text": "",
"- chan": "-chan",
"- kun": "-kun",
"- san": "-san",
# Add more replacements as needed
}
for target, replacement in placeholders.items():
translatedText = translatedText.replace(target, replacement)
# Elongate Long Dashes (Since GPT Ignores them...)
translatedText = elongateCharacters(translatedText)
translatedText = resubVars(translatedText, varResponse[1])
return translatedText
def elongateCharacters(text):
# Define a pattern to match one character followed by one or more `ー` characters
# Using a positive lookbehind assertion to capture the preceding character
pattern = r"(?<=(.))ー+"
# Define a replacement function that elongates the captured character
def repl(match):
char = match.group(1) # The character before the ー sequence
count = len(match.group(0)) - 1 # Number of ー characters
return char * count # Replace ー sequence with the character repeated
# Use re.sub() to replace the pattern in the text
return re.sub(pattern, repl, text)
def extractTranslation(translatedTextList, is_list):
pattern = r"`?<[Ll]ine\d+>([\\]*.*?[\\]*?)<\/?[Ll]ine\d+>`?"
# If it's a batch (i.e., list), extract with tags; otherwise, return the single item.
if is_list:
matchList = re.findall(pattern, translatedTextList)
return matchList
else:
matchList = re.findall(pattern, translatedTextList)
return matchList[0][0] if matchList else translatedTextList
def countTokens(characters, system, user, history):
inputTotalTokens = 0
outputTotalTokens = 0
enc = tiktoken.encoding_for_model("gpt-4")
# Input
if isinstance(history, list):
for line in history:
inputTotalTokens += len(enc.encode(line))
else:
inputTotalTokens += len(enc.encode(history))
inputTotalTokens += len(enc.encode(system))
inputTotalTokens += len(enc.encode(characters))
inputTotalTokens += len(enc.encode(user))
# Output
outputTotalTokens += round(len(enc.encode(user)) * 3)
return [inputTotalTokens, outputTotalTokens]
@retry(exceptions=Exception, tries=5, delay=5)
def translateGPT(text, history, fullPromptFlag):
global PBAR
mismatch = False
totalTokens = [0, 0]
if isinstance(text, list):
tList = batchList(text, BATCHSIZE)
else:
tList = [text]
for index, tItem in enumerate(tList):
# Before sending to translation, if we have a list of items, add the formatting
if isinstance(tItem, list):
payload = "\n".join([f"`<Line{i}>{item}</Line{i}>`" for i, item in enumerate(tItem)])
payload = re.sub(r"(<Line\d+)(><)(\/Line\d+>)", r"\1>Placeholder Text<\3", payload)
varResponse = subVars(payload)
subbedT = varResponse[0]
else:
varResponse = subVars(tItem)
subbedT = varResponse[0]
# Things to Check before starting translation
if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", subbedT):
if PBAR is not None:
PBAR.update(len(tItem))
continue
# Create Message
characters, system, user = createContext(fullPromptFlag, subbedT)
# Calculate Estimate
if ESTIMATE:
estimate = countTokens(characters, system, user, history)
totalTokens[0] += estimate[0]
totalTokens[1] += estimate[1]
continue
# Translating
response = translateText(characters, system, user, history, 0.02)
translatedText = response.choices[0].message.content
totalTokens[0] += response.usage.prompt_tokens
totalTokens[1] += response.usage.completion_tokens
# Formatting
translatedText = cleanTranslatedText(translatedText, varResponse)
if isinstance(tItem, list):
extractedTranslations = extractTranslation(translatedText, True)
tList[index] = extractedTranslations
if len(tItem) != len(extractedTranslations):
# Mismatch. Try Again
response = translateText(characters, system, user, history, 0.2)
translatedText = response.choices[0].message.content
totalTokens[0] += response.usage.prompt_tokens
totalTokens[1] += response.usage.completion_tokens
# Formatting
translatedText = cleanTranslatedText(translatedText, varResponse)
if isinstance(tItem, list):
extractedTranslations = extractTranslation(translatedText, True)
tList[index] = extractedTranslations
if len(tItem) != len(extractedTranslations):
mismatch = True # Just here for breakpoint
# Create History
with LOCK:
if PBAR is not None:
PBAR.update(len(tItem))
if not mismatch:
history = extractedTranslations[-10:] # Update history if we have a list
else:
history = text[-10:]
else:
# Ensure we're passing a single string to extractTranslation
extractedTranslations = extractTranslation(translatedText, False)
tList[index] = extractedTranslations
# Combine if multilist
if isinstance(tList[0], list):
tList = [t for sublist in tList for t in sublist]
# Return
if format == "json":
return [tList, totalTokens]
else:
return [tList[0], totalTokens]

369
modules/images.py
View file

@ -389,7 +389,6 @@ def translateImages(imageList):
return [translatedList, totalTokens, None]
# Save some money and enter the character before translation
def getSpeaker(speaker):
match speaker:
@ -407,7 +406,7 @@ def getSpeaker(speaker):
response = translateGPT(
f"{speaker}",
"Reply with the " + LANGUAGE + " translation of the NPC name.",
True,
False,
)
response[0] = response[0].title()
response[0] = response[0].replace("'S", "'s")
@ -429,40 +428,6 @@ def getSpeaker(speaker):
return [speaker, [0, 0]]
def subVars(jaString):
jaString = jaString.replace("\u3000", " ")
# Formatting
count = 0
codeList = re.findall(r"[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]", jaString)
codeList = set(codeList)
if len(codeList) != 0:
for var in codeList:
jaString = jaString.replace(var, "[FCode_" + str(count) + "]")
count += 1
# Put all lists in list and return
return [jaString, codeList]
def resubVars(translatedText, codeList):
# Fix Spacing and ChatGPT Nonsense
matchList = re.findall(r"\[\s?.+?\s?\]", translatedText)
if len(matchList) > 0:
for match in matchList:
text = match.strip()
translatedText = translatedText.replace(match, text)
# Formatting
count = 0
if len(codeList) != 0:
for var in codeList:
translatedText = translatedText.replace("[FCode_" + str(count) + "]", var)
count += 1
return translatedText
def batchList(input_list, batch_size):
if not isinstance(batch_size, int) or batch_size <= 0:
raise ValueError("batch_size must be a positive integer")
@ -470,56 +435,68 @@ def batchList(input_list, batch_size):
return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)]
def createContext(fullPromptFlag, subbedT, format):
characters = "Game Characters:\n\
ロラン (Roland) - Male\n\
リュカ (Ryuka) - Male\n\
レックス (Rex) - Male\n\
タバサ (Tabasa) - Female\n\
アルス (Ars) - Male\n\
アマカラ (Amakara) - Male\n\
エリー (Eri) - Female\n\
リオ (Rio) - Female\n\
サマル (Samal) - Male\n\
ムーン (Moon) - Female\n\
アリーナ (Arina) - Female\n\
クリフト (Cliff) - Male\n\
マーニャ (Manya) - Female\n\
ミネア (Minea) - Female\n\
デボラ (Debora) - Female\n\
ビアンカ (Bianca) - Female\n\
フローラ (Flora) - Female\n\
バーバラ (Barbara) - Female\n\
ミレーユ (Mireyu) - Female\n\
アイラ (Aira) - Female\n\
フォズ (Foz) - Female\n\
マリベル (Maribel) - Female\n\
ククール (Kukool) - Male\n\
ゲルダ (Gerda) - Female\n\
ゼシカ (Jessica) - Female\n\
ヤンガス (Yangus) - Male\n\
ラヴィエル (Raviel) - Female\n\
セティア (Setia) - Female\n\
ダイ (Dai) - Male\n\
ヒュンケル (Hyunckel) - Male\n\
ポップ (Pop) - Male\n\
マァム (Maam) - Female\n\
レオナ (Leona) - Female\n\
アステア (Astea) - Female\n\
イヨ (Iyo) - Female\n\
ジャガン (Jagan) - Male\n\
ヤオ (Yao) - Female\n\
デイジィ (Daisy) - Female\n\
バイシュン (Baishun) - Male\n\
ブライ (Buraimu) - Male\n\
ハッサン (Hassan) - Male\n\
アロマ (Aroma) - Female\n\
"
def parseVocabWithCategories(vocabText):
"""Parse vocabulary text and extract terms with their categories."""
pairs = []
seen = set()
currentCategory = None
for line in vocabText.splitlines():
line = line.strip()
if not line or line.startswith('```'):
continue
# Check if this is a category header
if line.startswith('#'):
currentCategory = line
continue
# Parse vocabulary term
m = re.match(r'^(.+?)(?:\s?[\(–])', line) # term is everything before space + '(' or '–'
if m:
term = m.group(1)
if term not in seen:
pairs.append((term, line, currentCategory))
seen.add(term)
return pairs
system = (
PROMPT + VOCAB
if fullPromptFlag
else f"\
def buildMatchedVocabText(vocabPairs, subbedT):
"""Build formatted vocabulary text with matched terms organized by category."""
matchedCategories = {}
# Use word boundaries for Japanese if appropriate, or allow substring as before.
for term, line, category in vocabPairs:
# "term in subbedT" could be false positive; can use regex but Japanese doesn't always have spaces.
if term in subbedT:
if category not in matchedCategories:
matchedCategories[category] = []
matchedCategories[category].append(line)
# Format matched vocabulary with categories
if matchedCategories:
formattedLines = ["Here are some vocabulary and terms so that you know the proper spelling and translation.\n"]
for category, lines in matchedCategories.items():
if category: # Only add category header if it exists
formattedLines.append(category)
formattedLines.extend(lines)
formattedLines.append("") # Add blank line between categories
matchedVocabText = f"```\n{chr(10).join(formattedLines).rstrip()}\n```"
else:
matchedVocabText = ""
return matchedVocabText
def createContext(fullPromptFlag, subbedT, format):
vocabPairs = parseVocabWithCategories(VOCAB)
matchedVocabText = buildMatchedVocabText(vocabPairs, subbedT)
if fullPromptFlag:
system = PROMPT + matchedVocabText
else:
system = f"\
You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
- You translate everything, including erotic and explicit content.\n\
@ -529,28 +506,25 @@ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{
- Maintain any spacing in the translation.\n\
- Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\
- `...` can be a part of the dialogue. Translate it as it is.\n\
{VOCAB}\n\
{matchedVocabText}\n\
"
)
if format == "json":
user = f"```json\n{subbedT}\n```"
else:
user = subbedT
return characters, system, user
return system, user
def translateText(characters, system, user, history, penalty, format):
def translateText(system, user, history, penalty, format, model=MODEL):
# Prompt
msg = [{"role": "system", "content": system + characters}]
# Characters
msg.append({"role": "system", "content": characters})
msg = [{"role": "system", "content": system}]
# History
if isinstance(history, list):
msg.extend([{"role": "system", "content": h} for h in history])
msg.append({"role": "system", "content": "Translation History:"})
msg.extend([{"role": "assistant", "content": h} for h in history])
else:
msg.append({"role": "system", "content": history})
msg.append({"role": "assistant", "content": history})
# Response Format
if format == "json":
@ -563,14 +537,14 @@ def translateText(characters, system, user, history, penalty, format):
response = openai.chat.completions.create(
temperature=0,
frequency_penalty=penalty,
model=MODEL,
model=model,
response_format=responseFormat,
messages=msg,
)
return response
def cleanTranslatedText(translatedText, varResponse):
def cleanTranslatedText(translatedText):
placeholders = {
f"{LANGUAGE} Translation: ": "",
"Translation: ": "",
@ -581,15 +555,26 @@ def cleanTranslatedText(translatedText, varResponse):
"「": '\\"',
"」": '\\"',
"- ": "-",
"—": "―",
"】": "]",
"【": "[",
"é": "e",
"this guy": "this bastard",
"This guy": "This bastard",
"Placeholder Text": "",
"```json": "",
"```": "",
# Add more replacements as needed
}
for target, replacement in placeholders.items():
translatedText = translatedText.replace(target, replacement)
# Remove Repeating Characters
pattern = re.compile(r"(.)\s*\1(?:\s*\1){" + str(20 - 1) + r",}")
translatedText = pattern.sub(lambda match: match.group(0).replace(" ", "")[:20], translatedText)
# Elongate Long Dashes (Since GPT Ignores them...)
translatedText = elongateCharacters(translatedText)
translatedText = resubVars(translatedText, varResponse[1])
return translatedText
@ -610,6 +595,8 @@ def elongateCharacters(text):
def extractTranslation(translatedTextList, is_list):
try:
translatedTextList = re.sub(r'\\"+\"([^,\n}])', r'\\"\1', translatedTextList)
translatedTextList = re.sub(r"(?<![\\])\"+(?![\n,])", r'"', translatedTextList)
line_dict = json.loads(translatedTextList)
# If it's a batch (i.e., list), extract with tags; otherwise, return the single item.
string_list = list(line_dict.values())
@ -619,11 +606,11 @@ def extractTranslation(translatedTextList, is_list):
return string_list[0]
except Exception as e:
print(f"extractTranslation Error: {e}")
PBAR.write(f"extractTranslation Error: {e} on String {translatedTextList}")
return None
def countTokens(characters, system, user, history):
def countTokens(system, user, history):
inputTotalTokens = 0
outputTotalTokens = 0
enc = tiktoken.encoding_for_model("gpt-4")
@ -635,101 +622,133 @@ def countTokens(characters, system, user, history):
else:
inputTotalTokens += len(enc.encode(history))
inputTotalTokens += len(enc.encode(system))
inputTotalTokens += len(enc.encode(characters))
inputTotalTokens += len(enc.encode(user))
# Output
outputTotalTokens += round(len(enc.encode(user)) * 3)
outputTotalTokens += round(len(enc.encode(user)) * 2.5)
return [inputTotalTokens, outputTotalTokens]
@retry(exceptions=Exception, tries=5, delay=5)
def translateGPT(text, history, fullPromptFlag):
global PBAR
global PBAR, MISMATCH, FILENAME
if text:
with open("log/translationHistory.txt", "a+", encoding="utf-8") as logFile:
mismatch = False
totalTokens = [0, 0]
if isinstance(text, list):
format = "json"
tList = batchList(text, BATCHSIZE)
else:
format = "text"
tList = [text]
mismatch = False
totalTokens = [0, 0]
if isinstance(text, list):
format = "json"
tList = batchList(text, BATCHSIZE)
else:
format = "text"
tList = [text]
for index, tItem in enumerate(tList):
# Things to Check before starting translation
if not re.search(LANGREGEX, str(tItem)):
if PBAR is not None:
PBAR.update(len(tItem))
if isinstance(tItem, list):
for j in range(len(tItem)):
tItem[j] = cleanTranslatedText(tItem[j])
tList[index] = tItem
else:
tList[index] = cleanTranslatedText(tItem)
history = tItem[-MAXHISTORY:]
continue
for index, tItem in enumerate(tList):
# Before sending to translation, if we have a list of items, add the formatting
if isinstance(tItem, list):
payload = {f"Line{i+1}": string for i, string in enumerate(tItem)}
payload = json.dumps(payload, indent=4, ensure_ascii=False)
varResponse = subVars(payload)
subbedT = varResponse[0]
else:
varResponse = subVars(tItem)
subbedT = varResponse[0]
# Before sending to translation, if we have a list of items, add the formatting
if isinstance(tItem, list):
for j in range(len(tItem)):
if not tItem[j]:
tItem[j] = tItem[j].replace("", "Placeholder Text")
payload = {f"Line{i+1}": string for i, string in enumerate(tItem)}
payload = json.dumps(payload, indent=4, ensure_ascii=False)
varResponse = [payload, []]
subbedT = varResponse[0]
else:
varResponse = [tItem, []]
subbedT = varResponse[0]
# Things to Check before starting translation
if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", subbedT):
if PBAR is not None:
PBAR.update(len(tItem))
continue
# Create Message
system, user = createContext(fullPromptFlag, subbedT, format)
# Create Message
characters, system, user = createContext(fullPromptFlag, subbedT, format)
# Calculate Estimate
if ESTIMATE:
estimate = countTokens(system, user, history)
totalTokens[0] += estimate[0]
totalTokens[1] += estimate[1]
continue
# Calculate Estimate
if ESTIMATE:
estimate = countTokens(characters, system, user, history)
totalTokens[0] += estimate[0]
totalTokens[1] += estimate[1]
continue
# Translating
response = translateText(system, user, history, 0.05, format)
# Translating
response = translateText(characters, system, user, history, 0.05, format)
translatedText = response.choices[0].message.content
totalTokens[0] += response.usage.prompt_tokens
totalTokens[1] += response.usage.completion_tokens
# Check Translation
translatedText = cleanTranslatedText(translatedText, varResponse)
if isinstance(tItem, list):
extractedTranslations = extractTranslation(translatedText, True)
if extractedTranslations == None or len(tItem) != len(extractedTranslations):
# Mismatch. Try Again
response = translateText(characters, system, user, history, 0.05, format)
# Set Tokens
translatedText = response.choices[0].message.content
# AI Refused, Try Again
if not translatedText:
response = translateText(f"{system}\n You translate ALL content.", user, history, 0.1, format, model="gpt-4o")
translatedText = response.choices[0].message.content
# Report Tokens
totalTokens[0] += response.usage.prompt_tokens
totalTokens[1] += response.usage.completion_tokens
# Formatting
translatedText = cleanTranslatedText(translatedText, varResponse)
if isinstance(tItem, list):
extractedTranslations = extractTranslation(translatedText, True)
if extractedTranslations == None or len(tItem) != len(extractedTranslations):
mismatch = True # Just here for breakpoint
# Check Translation
if translatedText:
translatedText = cleanTranslatedText(translatedText)
if isinstance(tItem, list):
extractedTranslations = extractTranslation(translatedText, True)
if extractedTranslations == None or len(tItem) != len(extractedTranslations):
# Mismatch. Try Again
response = translateText(system, user, history, 0.05, format, MODEL)
translatedText = response.choices[0].message.content
totalTokens[0] += response.usage.prompt_tokens
totalTokens[1] += response.usage.completion_tokens
# Set if no mismatch
if mismatch == False:
tList[index] = extractedTranslations
history = extractedTranslations[-10:] # Update history if we have a list
else:
history = text[-10:]
mismatch = False
# Formatting
translatedText = cleanTranslatedText(translatedText)
if isinstance(tItem, list):
extractedTranslations = extractTranslation(translatedText, True)
if extractedTranslations == None or len(tItem) != len(extractedTranslations):
with open("log/mismatchHistory.txt", "a+", encoding="utf-8") as mismatchFile:
mismatchFile.write(f"Mismatch: {FILENAME}\n")
mismatchFile.write(f"Input:\n{subbedT}\n")
mismatchFile.write(f"Output:\n{translatedText}\n")
mismatch = True # Just here for breakpoint
logFile.write(f"Input:\n{subbedT}\n")
logFile.write(f"Output:\n{translatedText}\n")
# Update Loading Bar
with LOCK:
if PBAR is not None:
PBAR.update(len(tItem))
# Set if no mismatch
if mismatch == False:
tList[index] = extractedTranslations
history = extractedTranslations[-MAXHISTORY:] # Update history if we have a list
else:
history = text[-MAXHISTORY:]
mismatch = False
if FILENAME not in MISMATCH:
MISMATCH.append(FILENAME)
# Update Loading Bar
with LOCK:
if PBAR is not None:
PBAR.update(len(tItem))
else:
# Ensure we're passing a single string to extractTranslation
tList[index] = translatedText.replace("Placeholder Text", "")
else:
PBAR.write(f"AI Refused:{tItem}\n")
# Combine if multilist
if isinstance(tList[0], list):
tList = [t for sublist in tList for t in sublist]
# Return
if format == "json":
return [tList, totalTokens]
else:
# Ensure we're passing a single string to extractTranslation
tList[index] = translatedText.replace("Placeholder Text", "")
# Combine if multilist
if isinstance(tList[0], list):
tList = [t for sublist in tList for t in sublist]
# Return
if format == "json":
return [tList, totalTokens]
return [tList[0], totalTokens]
else:
return [tList[0], totalTokens]
return [text, [0, 0]]

762
modules/irissoft.py
View file

@ -1,762 +0,0 @@
# Libraries
import os
import re
import util.dazedwrap as dazedwrap
import threading
import time
import traceback
import tiktoken
import openai
from pathlib import Path
from colorama import Fore
from dotenv import load_dotenv
from retry import retry
from tqdm import tqdm
# Open AI
load_dotenv()
if os.getenv("api").replace(" ", "") != "":
openai.base_url = os.getenv("api")
openai.organization = os.getenv("org")
openai.api_key = os.getenv("key")
# Globals
MODEL = os.getenv("model")
TIMEOUT = int(os.getenv("timeout"))
LANGUAGE = os.getenv("language").capitalize()
PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
VOCAB = Path("vocab.txt").read_text(encoding="utf-8")
THREADS = int(os.getenv("threads"))
LOCK = threading.Lock()
WIDTH = int(os.getenv("width"))
LISTWIDTH = int(os.getenv("listWidth"))
NOTEWIDTH = 70
MAXHISTORY = 10
ESTIMATE = ""
TOKENS = [0, 0]
NAMESLIST = []
NAMES = False # Output a list of all the character names found
BRFLAG = False # If the game uses <br> instead
FIXTEXTWRAP = True # Overwrites textwrap
IGNORETLTEXT = False # Ignores all translated text.
MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
# tqdm Globals
BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
POSITION = 0
LEAVE = False
# Regex - Need to change this if you want to translate from/to other languages. Default is Japanese Regex
LANGREGEX = r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+"
# Pricing - Depends on the model https://openai.com/pricing
# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request
# If you are getting a MISMATCH LENGTH error, lower the batch size.
if "gpt-3.5" in MODEL:
INPUTAPICOST = 3.00
OUTPUTAPICOST = 5.00
BATCHSIZE = 10
FREQUENCY_PENALTY = 0.2
elif "gpt-4" in MODEL:
INPUTAPICOST = 2.0
OUTPUTAPICOST = 8.00
BATCHSIZE = 30
FREQUENCY_PENALTY = 0.05
elif "deepseek" in MODEL:
INPUTAPICOST = 0.27
OUTPUTAPICOST = 1.10
BATCHSIZE = 30
FREQUENCY_PENALTY = 0.05
else:
INPUTAPICOST = float(os.getenv("input_cost"))
OUTPUTAPICOST = float(os.getenv("output_cost"))
BATCHSIZE = int(os.getenv("batchsize"))
FREQUENCY_PENALTY = float(os.getenv("frequency_penalty"))
def handleIris(filename, estimate):
global ESTIMATE
ESTIMATE = estimate
if ESTIMATE:
start = time.time()
translatedData = openFiles(filename)
# Print Result
end = time.time()
tqdm.write(getResultString(translatedData, end - start, filename))
with LOCK:
TOKENS[0] += translatedData[1][0]
TOKENS[1] += translatedData[1][1]
# Print Total
totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
# Print any errors on maps
if len(MISMATCH) > 0:
return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
else:
return totalString
else:
try:
with open("translated/" + filename, "w", encoding="cp932", errors="ignore") as outFile:
start = time.time()
translatedData = openFiles(filename)
# Print Result
end = time.time()
outFile.writelines(translatedData[0])
tqdm.write(getResultString(translatedData, end - start, filename))
with LOCK:
TOKENS[0] += translatedData[1][0]
TOKENS[1] += translatedData[1][1]
except Exception:
traceback.print_exc()
return "Fail"
return getResultString(["", TOKENS, None], end - start, "TOTAL")
def getResultString(translatedData, translationTime, filename):
# File Print String
totalTokenstring = (
Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
"[Output: "
+ str(translatedData[1][1])
+ "]" "[Cost: ${:,.4f}".format(((translatedData[1][0] / 1000000) * INPUTAPICOST) + ((translatedData[1][1] / 1000000) * OUTPUTAPICOST))
+ "]"
)
timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
if translatedData[2] == None:
# Success
return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET
else:
# Fail
try:
raise translatedData[2]
except Exception as e:
traceback.print_exc()
errorString = str(e) + Fore.RED
return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET
def openFiles(filename):
with open("files/" + filename, "r", encoding="shift_jis") as readFile:
translatedData = parseIris(readFile, filename)
# Delete lines marked for deletion
finalData = []
for line in translatedData[0]:
if line != "\\d\n":
finalData.append(line)
translatedData[0] = finalData
return translatedData
def parseIris(readFile, filename):
totalTokens = [0, 0]
# Read File into data
data = readFile.readlines()
# Create Progress Bar
with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
pbar.desc = filename
try:
result = translateIris(data, pbar, filename, [])
totalTokens[0] += result[0]
totalTokens[1] += result[1]
except Exception as e:
traceback.print_exc()
return [data, totalTokens, e]
return [data, totalTokens, None]
def translateIris(data, pbar, filename, translatedList):
stringList = []
currentGroup = []
tokens = [0, 0]
speaker = ""
voice = False
global LOCK, ESTIMATE
i = 0
while i < len(data):
voice = False
speaker = ""
if "#MSGVOICE" in data[i]:
i += 1
voice = True
voiceVar = data[i]
if "#MSG," in data[i] or "#MSG\n" in data[i] or voice == True:
i += 1
# Speaker
if re.search(r'^　?([^#\/."、。*!！（）\(\)\[\]　\n]+)\n', data[i]) and len(data[i]) < 30:
match = re.search(r"(.*)", data[i])
if match != None:
speaker = match.group(1)
if speaker[0] == "\u3000":
speaker = speaker[1:]
response = getSpeaker(speaker, pbar, filename)
speaker = response[0]
tokens[0] += response[1][0]
tokens[1] += response[1][1]
if translatedList != []:
speaker = speaker.replace(" ", "\u3000")
data[i] = f"\u3000{speaker}\n"
else:
speaker = ""
i += 1
# Lines
match = re.search(r"(.*)", data[i])
if match != None and match.group(1) != "":
# Pass 1
if translatedList == []:
# Grab Consecutive Strings
jaString = data[i]
if data[i] != "\n":
if data[i][0] == "\u3000":
jaString = data[i][1:]
currentGroup.append(jaString)
i += 1
while data[i] != "\n":
jaString = data[i]
if data[i] != "\n":
jaString = data[i][1:]
currentGroup.append(jaString)
i += 1
# Join up 401 groups for better translation.
if len(currentGroup) > 0:
jaString = "".join(currentGroup)
currentGroup = []
# Remove any textwrap
jaString = jaString.replace("\n", " ")
# Temporarily convert spaces (For Textwrap Later)
jaString = jaString.replace("\u3000", " ")
# Add Speaker (If there is one)
if speaker != "":
jaString = f"{speaker}: {jaString}"
# Add String
stringList.append(jaString.strip())
# Pass 2
else:
# Insert Strings
while data[i] != "\n":
data.pop(i)
# Get Text
if translatedList:
translatedText = translatedList[0]
translatedList.pop(0)
if len(translatedList) <= 0:
translatedList = None
# Remove added speaker
translatedText = re.sub(r"^.+?:\s", "", translatedText)
# Textwrap
translatedText = dazedwrap.wrapText(translatedText, width=WIDTH)
translatedText = translatedText.replace("\n", "\n\u3000")
# Replace Whitespace and Commas
translatedText = translatedText.replace(", ", "、")
translatedText = translatedText.replace(",\u3000", "、")
translatedText = translatedText.replace(",", "、")
translatedText = translatedText.replace(" ", "\u3000")
# Set Data
# Game crashes on more than 3 lines. Will need to create a new MSG for long translations
if translatedText.count("\n") > 2:
# Split List
translatedTextList = splitNewlines(translatedText)
# MSG Voice
count = 0
for text in translatedTextList:
if count != 0:
if voice == True:
# MSG for each item in the list
data.insert(i, "#MSGVOICE,\n")
i += 1
data.insert(i, f"{voiceVar}")
i += 1
else:
data.insert(i, "#MSG,\n")
i += 1
if speaker:
data[i] = f"\u3000{speaker}\n"
i += 1
if text[0] == "\u3000":
data.insert(i, f"{text}\n")
else:
data.insert(i, f"\u3000{text}\n")
i += 1
count += 1
if data[i] != "\n":
data.insert(i, "\n")
data[i] = f"\n{data[i]}"
else:
data.insert(i, f"\u3000{translatedText}\n")
i += 1
if data[i] != "\n":
data[i] = f"\n{data[i]}"
elif "#SELECT" in data[i] and translatedList == []:
Iris = r"(.+?) +\d$"
i += 1
match = re.search(Iris, data[i])
if match:
choiceList = []
choiceList.append(match.group(1))
i += 1
match = re.search(Iris, data[i])
while match:
choiceList.append(match.group(1))
i += 1
match = re.search(Iris, data[i])
# Translate
question = stringList[len(stringList) - 1]
response = translateGPT(
choiceList,
f"Previous text for context: {question}\n\nThis will be a dialogue option",
True,
pbar,
filename,
)
tokens[0] += response[1][0]
tokens[1] += response[1][1]
choiceListTL = response[0]
# Set Data
i = i - len(choiceListTL)
for j in range(len(choiceListTL)):
# Replace Whitespace and Commas
choiceListTL[j] = choiceListTL[j].replace(", ", "、")
choiceListTL[j] = choiceListTL[j].replace(",\u3000", "、")
choiceListTL[j] = choiceListTL[j].replace(",", "、")
choiceListTL[j] = choiceListTL[j].replace(" ", "\u3000")
data[i] = data[i].replace(choiceList[j], choiceListTL[j])
i += 1
# Nothing relevant. Skip Line.
else:
i += 1
else:
i += 1
# EOF
if len(stringList) > 0:
# Set Progress
pbar.total = len(stringList)
pbar.refresh()
# Translate
response = translateGPT(stringList, "", True, pbar, filename)
tokens[0] += response[1][0]
tokens[1] += response[1][1]
translatedList = response[0]
# Set Strings
if len(stringList) == len(translatedList):
translateIris(data, pbar, filename, translatedList)
# Mismatch
else:
with LOCK:
if filename not in MISMATCH:
MISMATCH.append(filename)
return tokens
def splitNewlines(text):
parts = []
newline_count = 0 # Counts the number of newline characters encountered
start_index = 0 # Start index of the current string part
for i, char in enumerate(text):
if char == "\n":
newline_count += 1
if newline_count == 3:
# Append the string part from start_index to current index (inclusive)
parts.append(text[start_index : i + 1])
# Reset newline count and update start_index for the next string part
newline_count = 0
start_index = i + 1
# Edge case: if the text does not end with a newline, we still need to append the last part
if start_index < len(text):
parts.append(text[start_index:])
return parts
# Save some money and enter the character before translation
def getSpeaker(speaker, pbar, filename):
match speaker:
case "ファイン":
return ["Fine", [0, 0]]
case "":
return ["", [0, 0]]
case _:
# Store Speaker
if speaker not in str(NAMESLIST):
response = translateGPT(
speaker,
"Reply with only the " + LANGUAGE + " translation of the NPC name.",
False,
pbar,
filename,
)
response[0] = response[0].replace("'S", "'s")
speakerList = [speaker, response[0]]
NAMESLIST.append(speakerList)
return response
# Find Speaker
else:
for i in range(len(NAMESLIST)):
if speaker == NAMESLIST[i][0]:
return [NAMESLIST[i][1], [0, 0]]
return [speaker, [0, 0]]
def subVars(jaString):
jaString = jaString.replace("\u3000", " ")
# Nested
count = 0
nestedList = re.findall(r"[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]", jaString)
nestedList = set(nestedList)
if len(nestedList) != 0:
for icon in nestedList:
jaString = jaString.replace(icon, "[Nested_" + str(count) + "]")
count += 1
# Icons
count = 0
iconList = re.findall(r"[\\]+[iIkKwWaA]+\[[0-9]+\]", jaString)
iconList = set(iconList)
if len(iconList) != 0:
for icon in iconList:
jaString = jaString.replace(icon, "[Ascii_" + str(count) + "]")
count += 1
# Colors
count = 0
colorList = re.findall(r"[\\]+[cC]\[[0-9]+\]", jaString)
colorList = set(colorList)
if len(colorList) != 0:
for color in colorList:
jaString = jaString.replace(color, "[Color_" + str(count) + "]")
count += 1
# Names
count = 0
nameList = re.findall(r"[\\]+[nN]\[.+?\]+", jaString)
nameList = set(nameList)
if len(nameList) != 0:
for name in nameList:
jaString = jaString.replace(name, "[Noun_" + str(count) + "]")
count += 1
# Variables
count = 0
varList = re.findall(r"[\\]+[vV]\[[0-9]+\]", jaString)
varList = set(varList)
if len(varList) != 0:
for var in varList:
jaString = jaString.replace(var, "[Var_" + str(count) + "]")
count += 1
# Formatting
count = 0
formatList = re.findall(r"[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]", jaString)
formatList = set(formatList)
if len(formatList) != 0:
for var in formatList:
jaString = jaString.replace(var, "[FCode_" + str(count) + "]")
count += 1
# Put all lists in list and return
allList = [nestedList, iconList, colorList, nameList, varList, formatList]
return [jaString, allList]
def resubVars(translatedText, allList):
# Fix Spacing and ChatGPT Nonsense
matchList = re.findall(r"\[\s?.+?\s?\]", translatedText)
if len(matchList) > 0:
for match in matchList:
text = match.strip()
translatedText = translatedText.replace(match, text)
# Nested
count = 0
if len(allList[0]) != 0:
for var in allList[0]:
translatedText = translatedText.replace("[Nested_" + str(count) + "]", var)
count += 1
# Icons
count = 0
if len(allList[1]) != 0:
for var in allList[1]:
translatedText = translatedText.replace("[Ascii_" + str(count) + "]", var)
count += 1
# Colors
count = 0
if len(allList[2]) != 0:
for var in allList[2]:
translatedText = translatedText.replace("[Color_" + str(count) + "]", var)
count += 1
# Names
count = 0
if len(allList[3]) != 0:
for var in allList[3]:
translatedText = translatedText.replace("[Noun_" + str(count) + "]", var)
count += 1
# Vars
count = 0
if len(allList[4]) != 0:
for var in allList[4]:
translatedText = translatedText.replace("[Var_" + str(count) + "]", var)
count += 1
# Formatting
count = 0
if len(allList[5]) != 0:
for var in allList[5]:
translatedText = translatedText.replace("[FCode_" + str(count) + "]", var)
count += 1
return translatedText
def batchList(input_list, batch_size):
if not isinstance(batch_size, int) or batch_size <= 0:
raise ValueError("batch_size must be a positive integer")
return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)]
def createContext(fullPromptFlag, subbedT):
characters = "Game Characters:\n\
フィリア (Philia) - Female\n\
アルネット (Annett) - Female\n\
ラピュセナ (Rapusena) - Female\n\
リッカ (Rikka) - Female\n\
アンデリビア (Andelivia) - Female\n\
リリアブルム (Liliabloom) - Female\n\
カルナ (Karna) - Female\n\
ラフィング＝スピア (Laughing Spear) - Female\n\
ノーラ (Nora) - Female\n\
"
system = (
PROMPT + VOCAB
if fullPromptFlag
else f"\
You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
- You translate everything, including erotic and explicit content.\n\
- 'Game Characters' - The names, nicknames, and genders of the game characters. Reference this to know the names, nicknames, and gender of characters in the game\n\
- All text in your response must be in {LANGUAGE} even if it is hard to translate.\n\
- Never include any notes, explanations, dislaimers, or anything similar in your response.\n\
- Maintain any spacing in the translation.\n\
- Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\
- `...` can be a part of the dialogue. Translate it as it is.\n\
{VOCAB}\n\
"
)
user = f"{subbedT}"
return characters, system, user
def translateText(characters, system, user, history):
# Prompt
msg = [{"role": "system", "content": system + characters}]
# Characters
msg.append({"role": "system", "content": characters})
# History
if isinstance(history, list):
msg.extend([{"role": "system", "content": h} for h in history])
else:
msg.append({"role": "system", "content": history})
# Content to TL
msg.append({"role": "user", "content": f"{user}"})
response = openai.chat.completions.create(
temperature=0.1,
frequency_penalty=0.1,
model=MODEL,
messages=msg,
)
return response
def cleanTranslatedText(translatedText, varResponse):
placeholders = {
f"{LANGUAGE} Translation: ": "",
"Translation: ": "",
"っ": "",
"〜": "~",
"ッ": "",
"。": ".",
"Placeholder Text": "",
# Add more replacements as needed
}
for target, replacement in placeholders.items():
translatedText = translatedText.replace(target, replacement)
# Elongate Long Dashes (Since GPT Ignores them...)
translatedText = elongateCharacters(translatedText)
translatedText = resubVars(translatedText, varResponse[1])
return translatedText
def elongateCharacters(text):
# Define a pattern to match one character followed by one or more `ー` characters
# Using a positive lookbehind assertion to capture the preceding character
pattern = r"(?<=(.))ー+"
# Define a replacement function that elongates the captured character
def repl(match):
char = match.group(1) # The character before the ー sequence
count = len(match.group(0)) - 1 # Number of ー characters
return char * count # Replace ー sequence with the character repeated
# Use re.sub() to replace the pattern in the text
return re.sub(pattern, repl, text)
def extractTranslation(translatedTextList, is_list):
pattern = r"`?<[Ll]ine\d+>([\\]*.*?[\\]*?)<\/?[Ll]ine\d+>`?"
# If it's a batch (i.e., list), extract with tags; otherwise, return the single item.
if is_list:
matchList = re.findall(pattern, translatedTextList)
return matchList
else:
matchList = re.findall(pattern, translatedTextList)
return matchList[0][0] if matchList else translatedTextList
def countTokens(characters, system, user, history):
inputTotalTokens = 0
outputTotalTokens = 0
enc = tiktoken.encoding_for_model("gpt-4")
# Input
if isinstance(history, list):
for line in history:
inputTotalTokens += len(enc.encode(line))
else:
inputTotalTokens += len(enc.encode(history))
inputTotalTokens += len(enc.encode(system))
inputTotalTokens += len(enc.encode(characters))
inputTotalTokens += len(enc.encode(user))
# Output
outputTotalTokens += round(len(enc.encode(user)) * 2)
return [inputTotalTokens, outputTotalTokens]
@retry(exceptions=Exception, tries=5, delay=5)
def translateGPT(text, history, fullPromptFlag, pbar, filename):
mismatch = False
totalTokens = [0, 0]
if isinstance(text, list):
tList = batchList(text, BATCHSIZE)
else:
tList = [text]
for index, tItem in enumerate(tList):
# Before sending to translation, if we have a list of items, add the formatting
if isinstance(tItem, list):
payload = "\n".join([f"`<Line{i}>{item}</Line{i}>`" for i, item in enumerate(tItem)])
payload = re.sub(r"(<Line\d+)(><)(\/Line\d+>)", r"\1>Placeholder Text<\3", payload)
varResponse = subVars(payload)
subbedT = varResponse[0]
else:
varResponse = subVars(tItem)
subbedT = varResponse[0]
# Things to Check before starting translation
if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", subbedT):
continue
# Create Message
characters, system, user = createContext(fullPromptFlag, subbedT)
# Calculate Estimate
if ESTIMATE:
estimate = countTokens(characters, system, user, history)
totalTokens[0] += estimate[0]
totalTokens[1] += estimate[1]
continue
# Translating
response = translateText(characters, system, user, history)
translatedText = response.choices[0].message.content
totalTokens[0] += response.usage.prompt_tokens
totalTokens[1] += response.usage.completion_tokens
# Formatting
translatedText = cleanTranslatedText(translatedText, varResponse)
if isinstance(tItem, list):
extractedTranslations = extractTranslation(translatedText, True)
if len(tItem) != len(extractedTranslations):
# Mismatch. Try Again
response = translateText(characters, system, user, history)
translatedText = response.choices[0].message.content
totalTokens[0] += response.usage.prompt_tokens
totalTokens[1] += response.usage.completion_tokens
# Formatting
translatedText = cleanTranslatedText(translatedText, varResponse)
if isinstance(tItem, list):
extractedTranslations = extractTranslation(translatedText, True)
if len(tItem) == len(extractedTranslations):
tList[index] = extractedTranslations
else:
MISMATCH.append(filename)
else:
tList[index] = extractedTranslations
# Create History
history = tList[index] # Update history if we have a list
pbar.update(len(tList[index]))
else:
# Ensure we're passing a single string to extractTranslation
extractedTranslations = extractTranslation(translatedText, False)
tList[index] = extractedTranslations
# Combine if multilist
if isinstance(tList[0], list):
tList = [t for sublist in tList for t in sublist]
# Return
if format == "json":
return [tList, totalTokens]
else:
return [tList[0], totalTokens]

537
modules/javascript.py
View file

@ -1,537 +0,0 @@
# Libraries
import os
import re
import util.dazedwrap as dazedwrap
import threading
import time
import traceback
import tiktoken
import openai
from pathlib import Path
from colorama import Fore
from dotenv import load_dotenv
from retry import retry
from tqdm import tqdm
# Open AI
load_dotenv()
if os.getenv("api").replace(" ", "") != "":
openai.base_url = os.getenv("api")
openai.organization = os.getenv("org")
openai.api_key = os.getenv("key")
# Globals
MODEL = os.getenv("model")
TIMEOUT = int(os.getenv("timeout"))
LANGUAGE = os.getenv("language").capitalize()
PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
VOCAB = Path("vocab.txt").read_text(encoding="utf-8")
THREADS = int(os.getenv("threads"))
LOCK = threading.Lock()
WIDTH = int(os.getenv("width"))
LISTWIDTH = int(os.getenv("listWidth"))
NOTEWIDTH = 70
MAXHISTORY = 10
ESTIMATE = ""
TOKENS = [0, 0]
NAMESLIST = []
NAMES = False # Output a list of all the character names found
BRFLAG = False # If the game uses <br> instead
FIXTEXTWRAP = True # Overwrites textwrap
IGNORETLTEXT = False # Ignores all translated text.
MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
# tqdm Globals
BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
POSITION = 0
LEAVE = False
# Regex - Need to change this if you want to translate from/to other languages. Default is Japanese Regex
LANGREGEX = r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+"
# Pricing - Depends on the model https://openai.com/pricing
# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request
# If you are getting a MISMATCH LENGTH error, lower the batch size.
if "gpt-3.5" in MODEL:
INPUTAPICOST = 3.00
OUTPUTAPICOST = 5.00
BATCHSIZE = 10
FREQUENCY_PENALTY = 0.2
elif "gpt-4" in MODEL:
INPUTAPICOST = 2.0
OUTPUTAPICOST = 8.00
BATCHSIZE = 30
FREQUENCY_PENALTY = 0.05
elif "deepseek" in MODEL:
INPUTAPICOST = 0.27
OUTPUTAPICOST = 1.10
BATCHSIZE = 30
FREQUENCY_PENALTY = 0.05
else:
INPUTAPICOST = float(os.getenv("input_cost"))
OUTPUTAPICOST = float(os.getenv("output_cost"))
BATCHSIZE = int(os.getenv("batchsize"))
FREQUENCY_PENALTY = float(os.getenv("frequency_penalty"))
def handleJavascript(filename, estimate):
global ESTIMATE
ESTIMATE = estimate
if ESTIMATE:
start = time.time()
translatedData = openFiles(filename)
# Print Result
end = time.time()
tqdm.write(getResultString(translatedData, end - start, filename))
with LOCK:
TOKENS[0] += translatedData[1][0]
TOKENS[1] += translatedData[1][1]
# Print Total
totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
# Print any errors on maps
if len(MISMATCH) > 0:
return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
else:
return totalString
else:
try:
with open("translated/" + filename, "w", encoding="utf8", errors="ignore") as outFile:
start = time.time()
translatedData = openFiles(filename)
# Print Result
end = time.time()
outFile.writelines(translatedData[0])
tqdm.write(getResultString(translatedData, end - start, filename))
with LOCK:
TOKENS[0] += translatedData[1][0]
TOKENS[1] += translatedData[1][1]
except Exception:
traceback.print_exc()
return "Fail"
return getResultString(["", TOKENS, None], end - start, "TOTAL")
def getResultString(translatedData, translationTime, filename):
# File Print String
totalTokenstring = (
Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
"[Output: "
+ str(translatedData[1][1])
+ "]" "[Cost: ${:,.4f}".format(((translatedData[1][0] / 1000000) * INPUTAPICOST) + ((translatedData[1][1] / 1000000) * OUTPUTAPICOST))
+ "]"
)
timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
if translatedData[2] == None:
# Success
return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET
else:
# Fail
try:
raise translatedData[2]
except Exception as e:
traceback.print_exc()
errorString = str(e) + Fore.RED
return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET
def openFiles(filename):
with open("files/" + filename, "r", encoding="utf-8") as readFile:
translatedData = parseJS(readFile, filename)
return translatedData
def parseJS(readFile, filename):
totalTokens = [0, 0]
data = readFile.readlines()
with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
pbar.desc = filename
try:
result = translateJS(data, pbar)
totalTokens[0] += result[0]
totalTokens[1] += result[1]
except Exception as e:
traceback.print_exc()
return [data, totalTokens, e]
return [data, totalTokens, None]
def translateJS(data, pbar):
tokens = [0, 0]
i = 0
# Regex & Plugin Name
regex = r'ObjectiveContent[\\]+":[\\]+"[\\]+"(.*?)[\\]+"'
# Find Plugin
while i < len(data):
# Run Search
stringList = re.findall(regex, data[i])
if len(stringList) != 0:
pbar.total = len(stringList)
pbar.refresh()
modifiedStringList = stringList.copy()
# Remove Wordwrap [Optional]
for j in range(len(modifiedStringList)):
modifiedStringList[j] = modifiedStringList[j].replace(r"\\\\\\\\n", r" ")
# Translate
response = translateGPT(modifiedStringList, f"Reply with the {LANGUAGE} translation", True, pbar)
translatedList = response[0]
tokens[0] = response[1][0]
tokens[0] = response[1][1]
# Validate Length & Replace Each Match
if len(translatedList) == len(modifiedStringList):
for j in range(len(translatedList)):
# Add escape for '
translatedList[j] = re.sub(r"[^\\](')", "\\'", translatedList[j])
# Wordwrap [Optional]
translatedList[j] = dazedwrap.wrapText(translatedList[j], LISTWIDTH)
translatedList[j] = translatedList[j].replace("\n", r"\\\\\\\\n")
# Set
data[i] = data[i].replace(stringList[j], translatedList[j])
# Mismatch
else:
pbar.write("Mismatch Error")
i += 1
return tokens
def subVars(jaString):
jaString = jaString.replace("\u3000", " ")
# Nested
count = 0
nestedList = re.findall(r"[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]", jaString)
nestedList = set(nestedList)
if len(nestedList) != 0:
for icon in nestedList:
jaString = jaString.replace(icon, "[Nested_" + str(count) + "]")
count += 1
# Icons
count = 0
iconList = re.findall(r"[\\]+[iIkKwWaA]+\[[0-9]+\]", jaString)
iconList = set(iconList)
if len(iconList) != 0:
for icon in iconList:
jaString = jaString.replace(icon, "[Ascii_" + str(count) + "]")
count += 1
# Colors
count = 0
colorList = re.findall(r"[\\]+[cC]\[[0-9]+\]", jaString)
colorList = set(colorList)
if len(colorList) != 0:
for color in colorList:
jaString = jaString.replace(color, "[Color_" + str(count) + "]")
count += 1
# Names
count = 0
nameList = re.findall(r"[\\]+[nN]\[.+?\]+", jaString)
nameList = set(nameList)
if len(nameList) != 0:
for name in nameList:
jaString = jaString.replace(name, "[Noun_" + str(count) + "]")
count += 1
# Variables
count = 0
varList = re.findall(r"[\\]+[vV]\[[0-9]+\]", jaString)
varList = set(varList)
if len(varList) != 0:
for var in varList:
jaString = jaString.replace(var, "[Var_" + str(count) + "]")
count += 1
# Formatting
count = 0
formatList = re.findall(r"[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]", jaString)
formatList = set(formatList)
if len(formatList) != 0:
for var in formatList:
jaString = jaString.replace(var, "[FCode_" + str(count) + "]")
count += 1
# Put all lists in list and return
allList = [nestedList, iconList, colorList, nameList, varList, formatList]
return [jaString, allList]
def resubVars(translatedText, allList):
# Fix Spacing and ChatGPT Nonsense
matchList = re.findall(r"\[\s?.+?\s?\]", translatedText)
if len(matchList) > 0:
for match in matchList:
text = match.strip()
translatedText = translatedText.replace(match, text)
# Nested
count = 0
if len(allList[0]) != 0:
for var in allList[0]:
translatedText = translatedText.replace("[Nested_" + str(count) + "]", var)
count += 1
# Icons
count = 0
if len(allList[1]) != 0:
for var in allList[1]:
translatedText = translatedText.replace("[Ascii_" + str(count) + "]", var)
count += 1
# Colors
count = 0
if len(allList[2]) != 0:
for var in allList[2]:
translatedText = translatedText.replace("[Color_" + str(count) + "]", var)
count += 1
# Names
count = 0
if len(allList[3]) != 0:
for var in allList[3]:
translatedText = translatedText.replace("[Noun_" + str(count) + "]", var)
count += 1
# Vars
count = 0
if len(allList[4]) != 0:
for var in allList[4]:
translatedText = translatedText.replace("[Var_" + str(count) + "]", var)
count += 1
# Formatting
count = 0
if len(allList[5]) != 0:
for var in allList[5]:
translatedText = translatedText.replace("[FCode_" + str(count) + "]", var)
count += 1
return translatedText
def batchList(input_list, batch_size):
if not isinstance(batch_size, int) or batch_size <= 0:
raise ValueError("batch_size must be a positive integer")
return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)]
def createContext(fullPromptFlag, subbedT):
characters = "Game Characters:\n\
皆月 (Minazuki)\n\
さやか (Sayaka)\n\
皆月 さやか (Minazuki Sayaka) - Female\n\
広瀬 (Hirose)\n\
智恵 (Chie) - Female\n\
広瀬 智恵 (Hirose Chie) - Female\n\
"
system = (
PROMPT + VOCAB
if fullPromptFlag
else f"\
You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
- You translate everything, including erotic and explicit content.\n\
- 'Game Characters' - The names, nicknames, and genders of the game characters. Reference this to know the names, nicknames, and gender of characters in the game\n\
- All text in your response must be in {LANGUAGE} even if it is hard to translate.\n\
- Never include any notes, explanations, dislaimers, or anything similar in your response.\n\
- Maintain any spacing in the translation.\n\
- Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\
- `...` can be a part of the dialogue. Translate it as it is.\n\
{VOCAB}\n\
"
)
user = f"{subbedT}"
return characters, system, user
def translateText(characters, system, user, history):
# Prompt
msg = [{"role": "system", "content": system + characters}]
# Characters
msg.append({"role": "system", "content": characters})
# History
if isinstance(history, list):
msg.extend([{"role": "system", "content": h} for h in history])
else:
msg.append({"role": "system", "content": history})
# Content to TL
msg.append({"role": "user", "content": f"{user}"})
response = openai.chat.completions.create(
temperature=0.1,
frequency_penalty=0.1,
model=MODEL,
messages=msg,
)
return response
def cleanTranslatedText(translatedText, varResponse):
placeholders = {
f"{LANGUAGE} Translation: ": "",
"Translation: ": "",
"っ": "",
"〜": "~",
"ッ": "",
"。": ".",
"Placeholder Text": "",
# Add more replacements as needed
}
for target, replacement in placeholders.items():
translatedText = translatedText.replace(target, replacement)
# Elongate Long Dashes (Since GPT Ignores them...)
translatedText = elongateCharacters(translatedText)
translatedText = resubVars(translatedText, varResponse[1])
return translatedText
def elongateCharacters(text):
# Define a pattern to match one character followed by one or more `ー` characters
# Using a positive lookbehind assertion to capture the preceding character
pattern = r"(?<=(.))ー+"
# Define a replacement function that elongates the captured character
def repl(match):
char = match.group(1) # The character before the ー sequence
count = len(match.group(0)) - 1 # Number of ー characters
return char * count # Replace ー sequence with the character repeated
# Use re.sub() to replace the pattern in the text
return re.sub(pattern, repl, text)
def extractTranslation(translatedTextList, is_list):
pattern = r"`?<[Ll]ine\d+>([\\]*.*?[\\]*?)<\/?[Ll]ine\d+>`?"
# If it's a batch (i.e., list), extract with tags; otherwise, return the single item.
if is_list:
matchList = re.findall(pattern, translatedTextList)
return matchList
else:
matchList = re.findall(pattern, translatedTextList)
return matchList[0][0] if matchList else translatedTextList
def countTokens(characters, system, user, history):
inputTotalTokens = 0
outputTotalTokens = 0
enc = tiktoken.encoding_for_model("gpt-4")
# Input
if isinstance(history, list):
for line in history:
inputTotalTokens += len(enc.encode(line))
else:
inputTotalTokens += len(enc.encode(history))
inputTotalTokens += len(enc.encode(system))
inputTotalTokens += len(enc.encode(characters))
inputTotalTokens += len(enc.encode(user))
# Output
outputTotalTokens += round(len(enc.encode(user)) * 3)
return [inputTotalTokens, outputTotalTokens]
@retry(exceptions=Exception, tries=5, delay=5)
def translateGPT(text, history, fullPromptFlag, pbar):
mismatch = False
totalTokens = [0, 0]
if isinstance(text, list):
tList = batchList(text, BATCHSIZE)
else:
tList = [text]
for index, tItem in enumerate(tList):
# Before sending to translation, if we have a list of items, add the formatting
if isinstance(tItem, list):
payload = "\n".join([f"`<Line{i}>{item}</Line{i}>`" for i, item in enumerate(tItem)])
payload = re.sub(r"(<Line\d+)(><)(\/Line\d+>)", r"\1>Placeholder Text<\3", payload)
varResponse = subVars(payload)
subbedT = varResponse[0]
else:
varResponse = subVars(tItem)
subbedT = varResponse[0]
# Things to Check before starting translation
if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", subbedT):
continue
# Create Message
characters, system, user = createContext(fullPromptFlag, subbedT)
# Calculate Estimate
if ESTIMATE:
estimate = countTokens(characters, system, user, history)
totalTokens[0] += estimate[0]
totalTokens[1] += estimate[1]
continue
# Translating
response = translateText(characters, system, user, history)
translatedText = response.choices[0].message.content
totalTokens[0] += response.usage.prompt_tokens
totalTokens[1] += response.usage.completion_tokens
# Formatting
translatedText = cleanTranslatedText(translatedText, varResponse)
if isinstance(tItem, list):
extractedTranslations = extractTranslation(translatedText, True)
tList[index] = extractedTranslations
if len(tItem) != len(extractedTranslations):
# Mismatch. Try Again
response = translateText(characters, system, user, history)
translatedText = response.choices[0].message.content
totalTokens[0] += response.usage.prompt_tokens
totalTokens[1] += response.usage.completion_tokens
# Formatting
translatedText = cleanTranslatedText(translatedText, varResponse)
if isinstance(tItem, list):
extractedTranslations = extractTranslation(translatedText, True)
if len(tItem) == len(extractedTranslations):
tList[index] = extractedTranslations
else:
mismatch = True # Just here for breakpoint
# Create History
history = tList[index] # Update history if we have a list
pbar.update(len(tList[index]))
else:
# Ensure we're passing a single string to extractTranslation
extractedTranslations = extractTranslation(translatedText, False)
tList[index] = extractedTranslations
# Combine if multilist
if isinstance(tList[0], list):
tList = [t for sublist in tList for t in sublist]
# Return
if format == "json":
return [tList, totalTokens]
else:
return [tList[0], totalTokens]

69
modules/json.py
View file

@ -296,7 +296,6 @@ def translateJSON(data, translatedList):
translateJSON(data, [stringListTL])
return tokens
# Save some money and enter the character before translation
def getSpeaker(speaker):
match speaker:
@ -343,11 +342,68 @@ def batchList(input_list, batch_size):
return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)]
def parseVocabWithCategories(vocabText):
"""Parse vocabulary text and extract terms with their categories."""
pairs = []
seen = set()
currentCategory = None
for line in vocabText.splitlines():
line = line.strip()
if not line or line.startswith('```'):
continue
# Check if this is a category header
if line.startswith('#'):
currentCategory = line
continue
# Parse vocabulary term
m = re.match(r'^(.+?)(?:\s?[\(–])', line) # term is everything before space + '(' or '–'
if m:
term = m.group(1)
if term not in seen:
pairs.append((term, line, currentCategory))
seen.add(term)
return pairs
def buildMatchedVocabText(vocabPairs, subbedT):
"""Build formatted vocabulary text with matched terms organized by category."""
matchedCategories = {}
# Use word boundaries for Japanese if appropriate, or allow substring as before.
for term, line, category in vocabPairs:
# "term in subbedT" could be false positive; can use regex but Japanese doesn't always have spaces.
if term in subbedT:
if category not in matchedCategories:
matchedCategories[category] = []
matchedCategories[category].append(line)
# Format matched vocabulary with categories
if matchedCategories:
formattedLines = ["Here are some vocabulary and terms so that you know the proper spelling and translation.\n"]
for category, lines in matchedCategories.items():
if category: # Only add category header if it exists
formattedLines.append(category)
formattedLines.extend(lines)
formattedLines.append("") # Add blank line between categories
matchedVocabText = f"```\n{chr(10).join(formattedLines).rstrip()}\n```"
else:
matchedVocabText = ""
return matchedVocabText
def createContext(fullPromptFlag, subbedT, format):
system = (
PROMPT + VOCAB
if fullPromptFlag
else f"\
vocabPairs = parseVocabWithCategories(VOCAB)
matchedVocabText = buildMatchedVocabText(vocabPairs, subbedT)
if fullPromptFlag:
system = PROMPT + matchedVocabText
else:
system = f"\
You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
- You translate everything, including erotic and explicit content.\n\
@ -357,9 +413,8 @@ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{
- Maintain any spacing in the translation.\n\
- Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\
- `...` can be a part of the dialogue. Translate it as it is.\n\
{VOCAB}\n\
{matchedVocabText}\n\
"
)
if format == "json":
user = f"```json\n{subbedT}\n```"
else:

713
modules/kansen.py
View file

@ -1,713 +0,0 @@
# Libraries
import os
import re
import util.dazedwrap as dazedwrap
import threading
import time
import traceback
import tiktoken
import openai
from pathlib import Path
from colorama import Fore
from dotenv import load_dotenv
from retry import retry
from tqdm import tqdm
# Open AI
load_dotenv()
if os.getenv("api").replace(" ", "") != "":
openai.base_url = os.getenv("api")
openai.organization = os.getenv("org")
openai.api_key = os.getenv("key")
# Globals
MODEL = os.getenv("model")
TIMEOUT = int(os.getenv("timeout"))
LANGUAGE = os.getenv("language").capitalize()
PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
VOCAB = Path("vocab.txt").read_text(encoding="utf-8")
THREADS = int(os.getenv("threads"))
LOCK = threading.Lock()
WIDTH = int(os.getenv("width"))
LISTWIDTH = int(os.getenv("listWidth"))
NOTEWIDTH = 70
MAXHISTORY = 10
ESTIMATE = ""
TOKENS = [0, 0]
NAMESLIST = []
NAMES = False # Output a list of all the character names found
BRFLAG = False # If the game uses <br> instead
FIXTEXTWRAP = False # Overwrites textwrap
IGNORETLTEXT = False # Ignores all translated text.
MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
# tqdm Globals
BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
POSITION = 0
LEAVE = False
# Regex - Need to change this if you want to translate from/to other languages. Default is Japanese Regex
LANGREGEX = r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+"
# Pricing - Depends on the model https://openai.com/pricing
# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request
# If you are getting a MISMATCH LENGTH error, lower the batch size.
if "gpt-3.5" in MODEL:
INPUTAPICOST = 3.00
OUTPUTAPICOST = 5.00
BATCHSIZE = 10
FREQUENCY_PENALTY = 0.2
elif "gpt-4" in MODEL:
INPUTAPICOST = 2.0
OUTPUTAPICOST = 8.00
BATCHSIZE = 30
FREQUENCY_PENALTY = 0.05
elif "deepseek" in MODEL:
INPUTAPICOST = 0.27
OUTPUTAPICOST = 1.10
BATCHSIZE = 30
FREQUENCY_PENALTY = 0.05
else:
INPUTAPICOST = float(os.getenv("input_cost"))
OUTPUTAPICOST = float(os.getenv("output_cost"))
BATCHSIZE = int(os.getenv("batchsize"))
FREQUENCY_PENALTY = float(os.getenv("frequency_penalty"))
def handleKansen(filename, estimate):
global ESTIMATE
ESTIMATE = estimate
if ESTIMATE:
start = time.time()
translatedData = openFiles(filename)
# Print Result
end = time.time()
tqdm.write(getResultString(translatedData, end - start, filename))
with LOCK:
TOKENS[0] += translatedData[1][0]
TOKENS[1] += translatedData[1][1]
# Print Total
totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
# Print any errors on maps
if len(MISMATCH) > 0:
return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
else:
return totalString
else:
try:
with open("translated/" + filename, "w", encoding="shift_jis", errors="ignore") as outFile:
start = time.time()
translatedData = openFiles(filename)
# Print Result
end = time.time()
outFile.writelines(translatedData[0])
tqdm.write(getResultString(translatedData, end - start, filename))
with LOCK:
TOKENS[0] += translatedData[1][0]
TOKENS[1] += translatedData[1][1]
except Exception:
traceback.print_exc()
return "Fail"
return getResultString(["", TOKENS, None], end - start, "TOTAL")
def getResultString(translatedData, translationTime, filename):
# File Print String
totalTokenstring = (
Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
"[Output: "
+ str(translatedData[1][1])
+ "]" "[Cost: ${:,.4f}".format(((translatedData[1][0] / 1000000) * INPUTAPICOST) + ((translatedData[1][1] / 1000000) * OUTPUTAPICOST))
+ "]"
)
timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
if translatedData[2] == None:
# Success
return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET
else:
# Fail
try:
raise translatedData[2]
except Exception as e:
traceback.print_exc()
errorString = str(e) + Fore.RED
return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET
def openFiles(filename):
with open("files/" + filename, "r", encoding="cp932") as readFile:
translatedData = parseTyrano(readFile, filename)
# Delete lines marked for deletion
finalData = []
for line in translatedData[0]:
if line != "\\d\n":
finalData.append(line)
translatedData[0] = finalData
return translatedData
def parseTyrano(readFile, filename):
totalTokens = [0, 0]
totalLines = 0
# Get total for progress bar
data = readFile.readlines()
totalLines = len(data)
with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar:
pbar.desc = filename
pbar.total = totalLines
try:
result = translateTyrano(data, pbar, totalLines)
totalTokens[0] += result[0]
totalTokens[1] += result[1]
except Exception as e:
traceback.print_exc()
return [data, totalTokens, e]
return [data, totalTokens, None]
def translateTyrano(data, pbar, totalLines):
textHistory = []
batch = []
currentGroup = []
maxHistory = MAXHISTORY
tokens = [0, 0]
speaker = ""
insertBool = False
global LOCK, ESTIMATE
i = 0
batchStartIndex = 0
while i < len(data):
# Speaker
if "[ns]" in data[i]:
matchList = re.findall(r"\[ns\](.+?)\[", data[i])
if len(matchList) != 0:
response = getSpeaker(matchList[0])
speaker = response[0]
tokens[0] += response[1][0]
tokens[1] += response[1][1]
data[i] = "[ns]" + speaker + "[nse]\n"
else:
speaker = ""
# Choices
elif "[sel" in data[i]:
matchList = re.findall(r'\[sel.+text="(.+?)".+', data[i])
if len(matchList) != 0:
originalText = matchList[0]
if len(textHistory) > 0:
response = translateGPT(
matchList[0],
"Keep your translation as brief as possible. Previous text for context: "
+ textHistory[len(textHistory) - 1]
+ "\n\nReply in the style of a dialogue option.",
False,
)
else:
response = translateGPT(
matchList[0],
"\n\nReply in the style of a dialogue option.",
False,
)
translatedText = response[0]
tokens[0] += response[1][0]
tokens[1] += response[1][1]
# Remove characters that may break scripts
charList = [".", '"', "\\n"]
for char in charList:
translatedText = translatedText.replace(char, "")
# Escape all '
translatedText = translatedText.replace("\\", "")
# translatedText = translatedText.replace("'", "\\\'")
# Set Data
translatedText = data[i].replace(originalText, translatedText)
data[i] = translatedText
# Lines
matchList = re.findall(r"(.+?)\[[rpcms_sel]+\]$", data[i])
if len(matchList) > 0:
if "hisout" in matchList[0]:
i += 1
continue
currentGroup.append(matchList[0])
if len(data) > i + 1:
while "[r]" in data[i + 1]:
if insertBool is True:
data[i] = r"\d\n"
pbar.update(1)
i += 1
matchList = re.findall(r"(.+?)\[r\]", data[i])
if len(matchList) > 0:
currentGroup.append(matchList[0])
while "[pcms]" in data[i + 1]:
if insertBool is True:
data[i] = r"\d\n"
pbar.update(1)
i += 1
matchList = re.findall(r"(.+?)\[pcms\]", data[i])
if len(matchList) > 0:
currentGroup.append(matchList[0])
while "[pcms_sel]" in data[i + 1]:
if insertBool is True:
data[i] = r"\d\n"
pbar.update(1)
i += 1
matchList = re.findall(r"(.+?)\[pcms_sel\]", data[i])
if len(matchList) > 0:
currentGroup.append(matchList[0])
# Join up 401 groups for better translation.
if len(currentGroup) > 0:
finalJAString = " ".join(currentGroup)
oldjaString = finalJAString
# Remove any textwrap
if FIXTEXTWRAP == True:
finalJAString = finalJAString.replace("[r]", " ")
# Remove Extra Stuff bad for translation.
finalJAString = finalJAString.replace("ﾞ", "")
finalJAString = finalJAString.replace("・", ".")
finalJAString = finalJAString.replace("‶", "")
finalJAString = finalJAString.replace("”", "")
finalJAString = finalJAString.replace("―", "-")
finalJAString = finalJAString.replace("…", "...")
finalJAString = re.sub(r"(\.{3}\.+)", "...", finalJAString)
finalJAString = finalJAString.replace("　", " ")
# Furigana Removal
matchList = re.findall(r"(\[ruby\stext=.+text=\"(.+)\"\])", finalJAString)
if len(matchList) > 0:
finalJAString = finalJAString.replace(matchList[0][0], matchList[0][1])
# Add Speaker (If there is one)
if speaker != "":
finalJAString = f"{speaker}: {finalJAString}"
# [Passthrough 1] Pulling From File
if insertBool is False:
# Append to List and Clear Values
batch.append(finalJAString)
speaker = ""
# Translate Batch if Full
if len(batch) == BATCHSIZE:
# Translate
response = translateGPT(batch, textHistory, True)
tokens[0] += response[1][0]
tokens[1] += response[1][1]
translatedBatch = response[0]
textHistory = translatedBatch[-10:]
# Set Values
if len(batch) == len(translatedBatch):
i = batchStartIndex
insertBool = True
# Mismatch
else:
pbar.write(f"Mismatch: {batchStartIndex} - {i}")
MISMATCH.append(batch)
batchStartIndex = i
batch.clear()
i += 1
if insertBool is True:
pbar.update(1)
currentGroup = []
# [Passthrough 2] Setting Data
else:
# Get Text
translatedText = translatedBatch[0]
translatedText = translatedText.replace('\\"', '"')
translatedText = translatedText.replace("[", "(")
translatedText = translatedText.replace("]", ")")
# Remove added speaker
translatedText = re.sub(r"^.+?:\s", "", translatedText)
# Textwrap
translatedText = dazedwrap.wrapText(translatedText, width=WIDTH)
textList = translatedText.split("\n")
# Set Text
data[i] = r"\d\n"
for line in textList:
# Wordwrap Text
if "[r]" not in line:
line = dazedwrap.wrapText(line, width=WIDTH)
line = line.replace("\n", "[r]")
# Set
data.insert(i, line.strip() + "[r]\n")
i += 1
data[i - 1] = data[i - 1].replace("[r]", "[pcms]")
translatedBatch.pop(0)
speaker = ""
currentGroup = []
# If Batch is empty. Move on.
if len(translatedBatch) == 0:
insertBool = False
batchStartIndex = i
batch.clear()
# Nothing relevant. Skip Line.
else:
i += 1
if insertBool is True:
pbar.update(1)
# Translate Batch if not empty and EOF
if len(batch) != 0 and i >= len(data):
# Translate
response = translateGPT(batch, textHistory, True)
tokens[0] += response[1][0]
tokens[1] += response[1][1]
translatedBatch = response[0]
textHistory = translatedBatch[-10:]
# Set Values
if len(batch) == len(translatedBatch):
i = batchStartIndex
insertBool = True
# Mismatch
else:
pbar.write(f"Mismatch: {batchStartIndex} - {i}")
MISMATCH.append(batch)
batchStartIndex = i
batch.clear()
currentGroup = []
return tokens
# Save some money and enter the character before translation
def getSpeaker(speaker):
match speaker:
case "央":
return ["Akira", [0, 0]]
case "累":
return ["Rui", [0, 0]]
case "梨里":
return ["Riri", [0, 0]]
case "純":
return ["Jun", [0, 0]]
case "美鈴":
return ["Misuzu", [0, 0]]
case "須田":
return ["Suda", [0, 0]]
case "高橋":
return ["Takahashi", [0, 0]]
case "勇二":
return ["Yuuji", [0, 0]]
case _:
return translateGPT(
speaker,
"Reply with only the " + LANGUAGE + " translation of the NPC name.",
False,
)
def subVars(jaString):
jaString = jaString.replace("\u3000", " ")
# Nested
count = 0
nestedList = re.findall(r"[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]", jaString)
nestedList = set(nestedList)
if len(nestedList) != 0:
for icon in nestedList:
jaString = jaString.replace(icon, "{Nested_" + str(count) + "}")
count += 1
# Icons
count = 0
iconList = re.findall(r"[\\]+[iIkKwWaA]+\[[0-9]+\]", jaString)
iconList = set(iconList)
if len(iconList) != 0:
for icon in iconList:
jaString = jaString.replace(icon, "{Ascii_" + str(count) + "}")
count += 1
# Colors
count = 0
colorList = re.findall(r"[\\]+[cC]\[[0-9]+\]", jaString)
colorList = set(colorList)
if len(colorList) != 0:
for color in colorList:
jaString = jaString.replace(color, "{Color_" + str(count) + "}")
count += 1
# Names
count = 0
nameList = re.findall(r"[\\]+[nN]\[.+?\]+", jaString)
nameList = set(nameList)
if len(nameList) != 0:
for name in nameList:
jaString = jaString.replace(name, "{Noun_" + str(count) + "}")
count += 1
# Variables
count = 0
varList = re.findall(r"[\\]+[vV]\[[0-9]+\]", jaString)
varList = set(varList)
if len(varList) != 0:
for var in varList:
jaString = jaString.replace(var, "{Var_" + str(count) + "}")
count += 1
# Formatting
count = 0
formatList = re.findall(r"[\\]+[\w]+\[.+?\]", jaString)
formatList = set(formatList)
if len(formatList) != 0:
for var in formatList:
jaString = jaString.replace(var, "{FCode_" + str(count) + "}")
count += 1
# Put all lists in list and return
allList = [nestedList, iconList, colorList, nameList, varList, formatList]
return [jaString, allList]
def resubVars(translatedText, allList):
# Fix Spacing and ChatGPT Nonsense
matchList = re.findall(r"\[\s?.+?\s?\]", translatedText)
if len(matchList) > 0:
for match in matchList:
text = match.strip()
translatedText = translatedText.replace(match, text)
# Nested
count = 0
if len(allList[0]) != 0:
for var in allList[0]:
translatedText = translatedText.replace("{Nested_" + str(count) + "}", var)
count += 1
# Icons
count = 0
if len(allList[1]) != 0:
for var in allList[1]:
translatedText = translatedText.replace("{Ascii_" + str(count) + "}", var)
count += 1
# Colors
count = 0
if len(allList[2]) != 0:
for var in allList[2]:
translatedText = translatedText.replace("{Color_" + str(count) + "}", var)
count += 1
# Names
count = 0
if len(allList[3]) != 0:
for var in allList[3]:
translatedText = translatedText.replace("{Noun_" + str(count) + "}", var)
count += 1
# Vars
count = 0
if len(allList[4]) != 0:
for var in allList[4]:
translatedText = translatedText.replace("{Var_" + str(count) + "}", var)
count += 1
# Formatting
count = 0
if len(allList[5]) != 0:
for var in allList[5]:
translatedText = translatedText.replace("{FCode_" + str(count) + "}", var)
count += 1
return translatedText
def batchList(input_list, batch_size):
if not isinstance(batch_size, int) or batch_size <= 0:
raise ValueError("batch_size must be a positive integer")
return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)]
def createContext(fullPromptFlag, subbedT):
characters = "Game Characters:\n\
渋江 央 (Shibue Akira) - Male\n\
蘆名 累 (Ashina Rui) - Female\n\
清原 梨里 (Kiyohara Riri) - Female\n\
五十嵐 純 (Igarashi Jun) - Female\n\
子野日 美鈴 (Nenohi Misuzu) - Female\n\
須田 (Suda) - Male\n\
高橋 (Takahashi) - Female\n\
勇二 (Yuuji) - Male\n\
"
system = (
PROMPT + VOCAB
if fullPromptFlag
else f"\
You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
You are going to be translating text from a videogame.\n\
I will give you lines of text, and you must translate each line to the best of your ability.\n\
{VOCAB}\n\
Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\
"
)
user = f"{subbedT}"
return characters, system, user
def translateText(characters, system, user, history):
# Prompt
msg = [{"role": "system", "content": system + characters}]
# Characters
msg.append({"role": "system", "content": characters})
# History
if isinstance(history, list):
msg.extend([{"role": "system", "content": h} for h in history])
else:
msg.append({"role": "system", "content": history})
# Content to TL
msg.append({"role": "user", "content": f"{user}"})
response = openai.chat.completions.create(
temperature=0.1,
frequency_penalty=0.1,
presence_penalty=0.1,
model=MODEL,
messages=msg,
)
return response
def cleanTranslatedText(translatedText, varResponse):
placeholders = {
f"{LANGUAGE} Translation: ": "",
"Translation: ": "",
"っ": "",
"〜": "~",
"ッ": "",
"。": ".",
"Placeholder Text": "",
# Add more replacements as needed
}
for target, replacement in placeholders.items():
translatedText = translatedText.replace(target, replacement)
translatedText = resubVars(translatedText, varResponse[1])
return [line for line in translatedText.replace("\\n", "\n").split("\n") if line]
def extractTranslation(translatedTextList, is_list):
pattern = r"`?<Line(\d+)>([\\]*.*?[\\]*?)<\/?Line\d+>`?"
# If it's a batch (i.e., list), extract with tags; otherwise, return the single item.
if is_list:
return [re.findall(pattern, line)[0][1] for line in translatedTextList if re.search(pattern, line)]
else:
matchList = re.findall(pattern, translatedTextList)
return matchList[0][1] if matchList else translatedTextList
def countTokens(characters, system, user, history):
inputTotalTokens = 0
outputTotalTokens = 0
enc = tiktoken.encoding_for_model("gpt-4")
# Input
if isinstance(history, list):
for line in history:
inputTotalTokens += len(enc.encode(line))
else:
inputTotalTokens += len(enc.encode(history))
inputTotalTokens += len(enc.encode(system))
inputTotalTokens += len(enc.encode(characters))
inputTotalTokens += len(enc.encode(user))
# Output
outputTotalTokens += round(len(enc.encode(user)) * 3)
return [inputTotalTokens, outputTotalTokens]
@retry(exceptions=Exception, tries=5, delay=5)
def translateGPT(text, history, fullPromptFlag):
totalTokens = [0, 0]
if isinstance(text, list):
tList = batchList(text, BATCHSIZE)
else:
tList = [text]
for index, tItem in enumerate(tList):
# Before sending to translation, if we have a list of items, add the formatting
if isinstance(tItem, list):
payload = "\n".join([f"`<Line{i}>{item}</Line{i}>`" for i, item in enumerate(tItem)])
payload = payload.replace("``", "`Placeholder Text`")
varResponse = subVars(payload)
subbedT = varResponse[0]
else:
varResponse = subVars(tItem)
subbedT = varResponse[0]
# Things to Check before starting translation
if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", subbedT):
continue
# Create Message
characters, system, user = createContext(fullPromptFlag, subbedT)
# Calculate Estimate
if ESTIMATE:
estimate = countTokens(characters, system, user, history)
totalTokens[0] += estimate[0]
totalTokens[1] += estimate[1]
continue
# Translating
response = translateText(characters, system, user, history)
translatedText = response.choices[0].message.content
totalTokens[0] += response.usage.prompt_tokens
totalTokens[1] += response.usage.completion_tokens
# Formatting
translatedTextList = cleanTranslatedText(translatedText, varResponse)
if isinstance(tItem, list):
extractedTranslations = extractTranslation(translatedTextList, True)
tList[index] = extractedTranslations
if len(tItem) != len(translatedTextList):
mismatch = True # Just here so breakpoint can be set
history = extractedTranslations[-10:] # Update history if we have a list
else:
# Ensure we're passing a single string to extractTranslation
extractedTranslations = extractTranslation("\n".join(translatedTextList), False)
tList[index] = extractedTranslations
# Combine if multilist
if isinstance(tList[0], list):
tList = [t for sublist in tList for t in sublist]
# Return
if format == "json":
return [tList, totalTokens]
else:
return [tList[0], totalTokens]

86
modules/kirikiri.py
View file

@ -341,7 +341,6 @@ def translateKiriKiri(data, pbar, filename, jobList):
return tokens
# Save some money and enter the character before translation
def getSpeaker(speaker):
match speaker:
@ -359,7 +358,7 @@ def getSpeaker(speaker):
response = translateGPT(
f"{speaker}",
"Reply with the " + LANGUAGE + " translation of the NPC name.",
True,
False,
)
response[0] = response[0].title()
response[0] = response[0].replace("'S", "'s")
@ -388,11 +387,68 @@ def batchList(input_list, batch_size):
return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)]
def parseVocabWithCategories(vocabText):
"""Parse vocabulary text and extract terms with their categories."""
pairs = []
seen = set()
currentCategory = None
for line in vocabText.splitlines():
line = line.strip()
if not line or line.startswith('```'):
continue
# Check if this is a category header
if line.startswith('#'):
currentCategory = line
continue
# Parse vocabulary term
m = re.match(r'^(.+?)(?:\s?[\(–])', line) # term is everything before space + '(' or '–'
if m:
term = m.group(1)
if term not in seen:
pairs.append((term, line, currentCategory))
seen.add(term)
return pairs
def buildMatchedVocabText(vocabPairs, subbedT):
"""Build formatted vocabulary text with matched terms organized by category."""
matchedCategories = {}
# Use word boundaries for Japanese if appropriate, or allow substring as before.
for term, line, category in vocabPairs:
# "term in subbedT" could be false positive; can use regex but Japanese doesn't always have spaces.
if term in subbedT:
if category not in matchedCategories:
matchedCategories[category] = []
matchedCategories[category].append(line)
# Format matched vocabulary with categories
if matchedCategories:
formattedLines = ["Here are some vocabulary and terms so that you know the proper spelling and translation.\n"]
for category, lines in matchedCategories.items():
if category: # Only add category header if it exists
formattedLines.append(category)
formattedLines.extend(lines)
formattedLines.append("") # Add blank line between categories
matchedVocabText = f"```\n{chr(10).join(formattedLines).rstrip()}\n```"
else:
matchedVocabText = ""
return matchedVocabText
def createContext(fullPromptFlag, subbedT, format):
system = (
PROMPT + VOCAB
if fullPromptFlag
else f"\
vocabPairs = parseVocabWithCategories(VOCAB)
matchedVocabText = buildMatchedVocabText(vocabPairs, subbedT)
if fullPromptFlag:
system = PROMPT + matchedVocabText
else:
system = f"\
You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
- You translate everything, including erotic and explicit content.\n\
@ -402,9 +458,8 @@ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{
- Maintain any spacing in the translation.\n\
- Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\
- `...` can be a part of the dialogue. Translate it as it is.\n\
{VOCAB}\n\
{matchedVocabText}\n\
"
)
if format == "json":
user = f"```json\n{subbedT}\n```"
else:
@ -418,9 +473,10 @@ def translateText(system, user, history, penalty, format, model=MODEL):
# History
if isinstance(history, list):
msg.extend([{"role": "system", "content": h} for h in history])
msg.append({"role": "system", "content": "Translation History:"})
msg.extend([{"role": "assistant", "content": h} for h in history])
else:
msg.append({"role": "system", "content": history})
msg.append({"role": "assistant", "content": history})
# Response Format
if format == "json":
@ -455,8 +511,11 @@ def cleanTranslatedText(translatedText):
"】": "]",
"【": "[",
"é": "e",
"ō": "o",
"this guy": "this bastard",
"This guy": "This bastard",
"Placeholder Text": "",
"```json": "",
"```": "",
# Add more replacements as needed
}
for target, replacement in placeholders.items():
@ -518,7 +577,7 @@ def countTokens(system, user, history):
inputTotalTokens += len(enc.encode(user))
# Output
outputTotalTokens += round(len(enc.encode(user)) * 3)
outputTotalTokens += round(len(enc.encode(user)) * 2.5)
return [inputTotalTokens, outputTotalTokens]
@ -582,7 +641,8 @@ def translateGPT(text, history, fullPromptFlag):
# AI Refused, Try Again
if not translatedText:
response = translateText(f"{system}\n You translate ALL content.", user, history, 0.1, format)
response = translateText(f"{system}\n You translate ALL content.", user, history, 0.1, format, model="gpt-4o")
translatedText = response.choices[0].message.content
# Report Tokens
totalTokens[0] += response.usage.prompt_tokens

310
modules/lune.py
View file

@ -300,7 +300,6 @@ def translateJSON(data, pbar):
currentGroup = []
return tokens
# Save some money and enter the character before translation
def getSpeaker(speaker):
match speaker:
@ -318,7 +317,7 @@ def getSpeaker(speaker):
response = translateGPT(
f"{speaker}",
"Reply with the " + LANGUAGE + " translation of the NPC name.",
True,
False,
)
response[0] = response[0].title()
response[0] = response[0].replace("'S", "'s")
@ -340,40 +339,6 @@ def getSpeaker(speaker):
return [speaker, [0, 0]]
def subVars(jaString):
jaString = jaString.replace("\u3000", " ")
# Formatting
count = 0
codeList = re.findall(r"[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]", jaString)
codeList = set(codeList)
if len(codeList) != 0:
for var in codeList:
jaString = jaString.replace(var, "[FCode_" + str(count) + "]")
count += 1
# Put all lists in list and return
return [jaString, codeList]
def resubVars(translatedText, codeList):
# Fix Spacing and ChatGPT Nonsense
matchList = re.findall(r"\[\s?.+?\s?\]", translatedText)
if len(matchList) > 0:
for match in matchList:
text = match.strip()
translatedText = translatedText.replace(match, text)
# Formatting
count = 0
if len(codeList) != 0:
for var in codeList:
translatedText = translatedText.replace("[FCode_" + str(count) + "]", var)
count += 1
return translatedText
def batchList(input_list, batch_size):
if not isinstance(batch_size, int) or batch_size <= 0:
raise ValueError("batch_size must be a positive integer")
@ -381,11 +346,68 @@ def batchList(input_list, batch_size):
return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)]
def parseVocabWithCategories(vocabText):
"""Parse vocabulary text and extract terms with their categories."""
pairs = []
seen = set()
currentCategory = None
for line in vocabText.splitlines():
line = line.strip()
if not line or line.startswith('```'):
continue
# Check if this is a category header
if line.startswith('#'):
currentCategory = line
continue
# Parse vocabulary term
m = re.match(r'^(.+?)(?:\s?[\(–])', line) # term is everything before space + '(' or '–'
if m:
term = m.group(1)
if term not in seen:
pairs.append((term, line, currentCategory))
seen.add(term)
return pairs
def buildMatchedVocabText(vocabPairs, subbedT):
"""Build formatted vocabulary text with matched terms organized by category."""
matchedCategories = {}
# Use word boundaries for Japanese if appropriate, or allow substring as before.
for term, line, category in vocabPairs:
# "term in subbedT" could be false positive; can use regex but Japanese doesn't always have spaces.
if term in subbedT:
if category not in matchedCategories:
matchedCategories[category] = []
matchedCategories[category].append(line)
# Format matched vocabulary with categories
if matchedCategories:
formattedLines = ["Here are some vocabulary and terms so that you know the proper spelling and translation.\n"]
for category, lines in matchedCategories.items():
if category: # Only add category header if it exists
formattedLines.append(category)
formattedLines.extend(lines)
formattedLines.append("") # Add blank line between categories
matchedVocabText = f"```\n{chr(10).join(formattedLines).rstrip()}\n```"
else:
matchedVocabText = ""
return matchedVocabText
def createContext(fullPromptFlag, subbedT, format):
system = (
PROMPT + VOCAB
if fullPromptFlag
else f"\
vocabPairs = parseVocabWithCategories(VOCAB)
matchedVocabText = buildMatchedVocabText(vocabPairs, subbedT)
if fullPromptFlag:
system = PROMPT + matchedVocabText
else:
system = f"\
You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
- You translate everything, including erotic and explicit content.\n\
@ -395,9 +417,8 @@ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{
- Maintain any spacing in the translation.\n\
- Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\
- `...` can be a part of the dialogue. Translate it as it is.\n\
{VOCAB}\n\
{matchedVocabText}\n\
"
)
if format == "json":
user = f"```json\n{subbedT}\n```"
else:
@ -411,9 +432,10 @@ def translateText(system, user, history, penalty, format, model=MODEL):
# History
if isinstance(history, list):
msg.extend([{"role": "system", "content": h} for h in history])
msg.append({"role": "system", "content": "Translation History:"})
msg.extend([{"role": "assistant", "content": h} for h in history])
else:
msg.append({"role": "system", "content": history})
msg.append({"role": "assistant", "content": history})
# Response Format
if format == "json":
@ -433,7 +455,7 @@ def translateText(system, user, history, penalty, format, model=MODEL):
return response
def cleanTranslatedText(translatedText, varResponse):
def cleanTranslatedText(translatedText):
placeholders = {
f"{LANGUAGE} Translation: ": "",
"Translation: ": "",
@ -444,15 +466,26 @@ def cleanTranslatedText(translatedText, varResponse):
"「": '\\"',
"」": '\\"',
"- ": "-",
"—": "―",
"】": "]",
"【": "[",
"é": "e",
"this guy": "this bastard",
"This guy": "This bastard",
"Placeholder Text": "",
"```json": "",
"```": "",
# Add more replacements as needed
}
for target, replacement in placeholders.items():
translatedText = translatedText.replace(target, replacement)
# Remove Repeating Characters
pattern = re.compile(r"(.)\s*\1(?:\s*\1){" + str(20 - 1) + r",}")
translatedText = pattern.sub(lambda match: match.group(0).replace(" ", "")[:20], translatedText)
# Elongate Long Dashes (Since GPT Ignores them...)
translatedText = elongateCharacters(translatedText)
translatedText = resubVars(translatedText, varResponse[1])
return translatedText
@ -473,6 +506,8 @@ def elongateCharacters(text):
def extractTranslation(translatedTextList, is_list):
try:
translatedTextList = re.sub(r'\\"+\"([^,\n}])', r'\\"\1', translatedTextList)
translatedTextList = re.sub(r"(?<![\\])\"+(?![\n,])", r'"', translatedTextList)
line_dict = json.loads(translatedTextList)
# If it's a batch (i.e., list), extract with tags; otherwise, return the single item.
string_list = list(line_dict.values())
@ -482,7 +517,7 @@ def extractTranslation(translatedTextList, is_list):
return string_list[0]
except Exception as e:
print(f"extractTranslation Error: {e}")
PBAR.write(f"extractTranslation Error: {e} on String {translatedTextList}")
return None
@ -501,7 +536,7 @@ def countTokens(system, user, history):
inputTotalTokens += len(enc.encode(user))
# Output
outputTotalTokens += round(len(enc.encode(user)) * 3)
outputTotalTokens += round(len(enc.encode(user)) * 2.5)
return [inputTotalTokens, outputTotalTokens]
@ -509,91 +544,122 @@ def countTokens(system, user, history):
@retry(exceptions=Exception, tries=5, delay=5)
def translateGPT(text, history, fullPromptFlag):
global PBAR, MISMATCH, FILENAME
if text:
with open("log/translationHistory.txt", "a+", encoding="utf-8") as logFile:
mismatch = False
totalTokens = [0, 0]
if isinstance(text, list):
format = "json"
tList = batchList(text, BATCHSIZE)
else:
format = "text"
tList = [text]
mismatch = False
totalTokens = [0, 0]
if isinstance(text, list):
format = "json"
tList = batchList(text, BATCHSIZE)
else:
format = "text"
tList = [text]
for index, tItem in enumerate(tList):
# Things to Check before starting translation
if not re.search(LANGREGEX, str(tItem)):
if PBAR is not None:
PBAR.update(len(tItem))
if isinstance(tItem, list):
for j in range(len(tItem)):
tItem[j] = cleanTranslatedText(tItem[j])
tList[index] = tItem
else:
tList[index] = cleanTranslatedText(tItem)
history = tItem[-MAXHISTORY:]
continue
for index, tItem in enumerate(tList):
# Before sending to translation, if we have a list of items, add the formatting
if isinstance(tItem, list):
payload = {f"Line{i+1}": string for i, string in enumerate(tItem)}
payload = json.dumps(payload, indent=4, ensure_ascii=False)
varResponse = subVars(payload)
subbedT = varResponse[0]
else:
varResponse = subVars(tItem)
subbedT = varResponse[0]
# Before sending to translation, if we have a list of items, add the formatting
if isinstance(tItem, list):
for j in range(len(tItem)):
if not tItem[j]:
tItem[j] = tItem[j].replace("", "Placeholder Text")
payload = {f"Line{i+1}": string for i, string in enumerate(tItem)}
payload = json.dumps(payload, indent=4, ensure_ascii=False)
varResponse = [payload, []]
subbedT = varResponse[0]
else:
varResponse = [tItem, []]
subbedT = varResponse[0]
# Things to Check before starting translation
if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", subbedT):
if PBAR is not None:
PBAR.update(len(tItem))
continue
# Create Message
system, user = createContext(fullPromptFlag, subbedT, format)
# Create Message
system, user = createContext(fullPromptFlag, subbedT, format)
# Calculate Estimate
if ESTIMATE:
estimate = countTokens(system, user, history)
totalTokens[0] += estimate[0]
totalTokens[1] += estimate[1]
continue
# Calculate Estimate
if ESTIMATE:
estimate = countTokens(system, user, history)
totalTokens[0] += estimate[0]
totalTokens[1] += estimate[1]
continue
# Translating
response = translateText(system, user, history, 0.05, format)
# Translating
response = translateText(system, user, history, 0.05, format)
translatedText = response.choices[0].message.content
totalTokens[0] += response.usage.prompt_tokens
totalTokens[1] += response.usage.completion_tokens
# Check Translation
translatedText = cleanTranslatedText(translatedText, varResponse)
if isinstance(tItem, list):
extractedTranslations = extractTranslation(translatedText, True)
if extractedTranslations == None or len(tItem) != len(extractedTranslations):
# Mismatch. Try Again
response = translateText(system, user, history, 0.05, format, MODEL)
# Set Tokens
translatedText = response.choices[0].message.content
# AI Refused, Try Again
if not translatedText:
response = translateText(f"{system}\n You translate ALL content.", user, history, 0.1, format, model="gpt-4o")
translatedText = response.choices[0].message.content
# Report Tokens
totalTokens[0] += response.usage.prompt_tokens
totalTokens[1] += response.usage.completion_tokens
# Formatting
translatedText = cleanTranslatedText(translatedText, varResponse)
if isinstance(tItem, list):
extractedTranslations = extractTranslation(translatedText, True)
if extractedTranslations == None or len(tItem) != len(extractedTranslations):
mismatch = True # Just here for breakpoint
# Check Translation
if translatedText:
translatedText = cleanTranslatedText(translatedText)
if isinstance(tItem, list):
extractedTranslations = extractTranslation(translatedText, True)
if extractedTranslations == None or len(tItem) != len(extractedTranslations):
# Mismatch. Try Again
response = translateText(system, user, history, 0.05, format, MODEL)
translatedText = response.choices[0].message.content
totalTokens[0] += response.usage.prompt_tokens
totalTokens[1] += response.usage.completion_tokens
# Set if no mismatch
if mismatch == False:
tList[index] = extractedTranslations
history = extractedTranslations[-10:] # Update history if we have a list
else:
history = text[-10:]
mismatch = False
if FILENAME not in MISMATCH:
MISMATCH.append(FILENAME)
# Formatting
translatedText = cleanTranslatedText(translatedText)
if isinstance(tItem, list):
extractedTranslations = extractTranslation(translatedText, True)
if extractedTranslations == None or len(tItem) != len(extractedTranslations):
with open("log/mismatchHistory.txt", "a+", encoding="utf-8") as mismatchFile:
mismatchFile.write(f"Mismatch: {FILENAME}\n")
mismatchFile.write(f"Input:\n{subbedT}\n")
mismatchFile.write(f"Output:\n{translatedText}\n")
mismatch = True # Just here for breakpoint
logFile.write(f"Input:\n{subbedT}\n")
logFile.write(f"Output:\n{translatedText}\n")
# Update Loading Bar
with LOCK:
if PBAR is not None:
PBAR.update(len(tItem))
# Set if no mismatch
if mismatch == False:
tList[index] = extractedTranslations
history = extractedTranslations[-MAXHISTORY:] # Update history if we have a list
else:
history = text[-MAXHISTORY:]
mismatch = False
if FILENAME not in MISMATCH:
MISMATCH.append(FILENAME)
# Update Loading Bar
with LOCK:
if PBAR is not None:
PBAR.update(len(tItem))
else:
# Ensure we're passing a single string to extractTranslation
tList[index] = translatedText.replace("Placeholder Text", "")
else:
PBAR.write(f"AI Refused:{tItem}\n")
# Combine if multilist
if isinstance(tList[0], list):
tList = [t for sublist in tList for t in sublist]
# Return
if format == "json":
return [tList, totalTokens]
else:
# Ensure we're passing a single string to extractTranslation
tList[index] = translatedText.replace("Placeholder Text", "")
# Combine if multilist
if isinstance(tList[0], list):
tList = [t for sublist in tList for t in sublist]
# Return
if format == "json":
return [tList, totalTokens]
return [tList[0], totalTokens]
else:
return [tList[0], totalTokens]
return [text, [0, 0]]

86
modules/nscript.py
View file

@ -359,7 +359,6 @@ def fixText(translatedText):
return translatedText
# Save some money and enter the character before translation
def getSpeaker(speaker):
match speaker:
@ -377,7 +376,7 @@ def getSpeaker(speaker):
response = translateGPT(
f"{speaker}",
"Reply with the " + LANGUAGE + " translation of the NPC name.",
True,
False,
)
response[0] = response[0].title()
response[0] = response[0].replace("'S", "'s")
@ -406,11 +405,68 @@ def batchList(input_list, batch_size):
return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)]
def parseVocabWithCategories(vocabText):
"""Parse vocabulary text and extract terms with their categories."""
pairs = []
seen = set()
currentCategory = None
for line in vocabText.splitlines():
line = line.strip()
if not line or line.startswith('```'):
continue
# Check if this is a category header
if line.startswith('#'):
currentCategory = line
continue
# Parse vocabulary term
m = re.match(r'^(.+?)(?:\s?[\(–])', line) # term is everything before space + '(' or '–'
if m:
term = m.group(1)
if term not in seen:
pairs.append((term, line, currentCategory))
seen.add(term)
return pairs
def buildMatchedVocabText(vocabPairs, subbedT):
"""Build formatted vocabulary text with matched terms organized by category."""
matchedCategories = {}
# Use word boundaries for Japanese if appropriate, or allow substring as before.
for term, line, category in vocabPairs:
# "term in subbedT" could be false positive; can use regex but Japanese doesn't always have spaces.
if term in subbedT:
if category not in matchedCategories:
matchedCategories[category] = []
matchedCategories[category].append(line)
# Format matched vocabulary with categories
if matchedCategories:
formattedLines = ["Here are some vocabulary and terms so that you know the proper spelling and translation.\n"]
for category, lines in matchedCategories.items():
if category: # Only add category header if it exists
formattedLines.append(category)
formattedLines.extend(lines)
formattedLines.append("") # Add blank line between categories
matchedVocabText = f"```\n{chr(10).join(formattedLines).rstrip()}\n```"
else:
matchedVocabText = ""
return matchedVocabText
def createContext(fullPromptFlag, subbedT, format):
system = (
PROMPT + VOCAB
if fullPromptFlag
else f"\
vocabPairs = parseVocabWithCategories(VOCAB)
matchedVocabText = buildMatchedVocabText(vocabPairs, subbedT)
if fullPromptFlag:
system = PROMPT + matchedVocabText
else:
system = f"\
You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
- You translate everything, including erotic and explicit content.\n\
@ -420,9 +476,8 @@ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{
- Maintain any spacing in the translation.\n\
- Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\
- `...` can be a part of the dialogue. Translate it as it is.\n\
{VOCAB}\n\
{matchedVocabText}\n\
"
)
if format == "json":
user = f"```json\n{subbedT}\n```"
else:
@ -436,9 +491,10 @@ def translateText(system, user, history, penalty, format, model=MODEL):
# History
if isinstance(history, list):
msg.extend([{"role": "system", "content": h} for h in history])
msg.append({"role": "system", "content": "Translation History:"})
msg.extend([{"role": "assistant", "content": h} for h in history])
else:
msg.append({"role": "system", "content": history})
msg.append({"role": "assistant", "content": history})
# Response Format
if format == "json":
@ -473,8 +529,11 @@ def cleanTranslatedText(translatedText):
"】": "]",
"【": "[",
"é": "e",
"ō": "o",
"this guy": "this bastard",
"This guy": "This bastard",
"Placeholder Text": "",
"```json": "",
"```": "",
# Add more replacements as needed
}
for target, replacement in placeholders.items():
@ -536,7 +595,7 @@ def countTokens(system, user, history):
inputTotalTokens += len(enc.encode(user))
# Output
outputTotalTokens += round(len(enc.encode(user)) * 3)
outputTotalTokens += round(len(enc.encode(user)) * 2.5)
return [inputTotalTokens, outputTotalTokens]
@ -600,7 +659,8 @@ def translateGPT(text, history, fullPromptFlag):
# AI Refused, Try Again
if not translatedText:
response = translateText(f"{system}\n You translate ALL content.", user, history, 0.1, format)
response = translateText(f"{system}\n You translate ALL content.", user, history, 0.1, format, model="gpt-4o")
translatedText = response.choices[0].message.content
# Report Tokens
totalTokens[0] += response.usage.prompt_tokens

76
modules/regex.py
View file

@ -356,7 +356,6 @@ def translateRegex(data, translatedList):
translateRegex(data, [stringListTL, choiceListTL])
return tokens
# Save some money and enter the character before translation
def getSpeaker(speaker):
match speaker:
@ -403,11 +402,68 @@ def batchList(input_list, batch_size):
return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)]
def parseVocabWithCategories(vocabText):
"""Parse vocabulary text and extract terms with their categories."""
pairs = []
seen = set()
currentCategory = None
for line in vocabText.splitlines():
line = line.strip()
if not line or line.startswith('```'):
continue
# Check if this is a category header
if line.startswith('#'):
currentCategory = line
continue
# Parse vocabulary term
m = re.match(r'^(.+?)(?:\s?[\(–])', line) # term is everything before space + '(' or '–'
if m:
term = m.group(1)
if term not in seen:
pairs.append((term, line, currentCategory))
seen.add(term)
return pairs
def buildMatchedVocabText(vocabPairs, subbedT):
"""Build formatted vocabulary text with matched terms organized by category."""
matchedCategories = {}
# Use word boundaries for Japanese if appropriate, or allow substring as before.
for term, line, category in vocabPairs:
# "term in subbedT" could be false positive; can use regex but Japanese doesn't always have spaces.
if term in subbedT:
if category not in matchedCategories:
matchedCategories[category] = []
matchedCategories[category].append(line)
# Format matched vocabulary with categories
if matchedCategories:
formattedLines = ["Here are some vocabulary and terms so that you know the proper spelling and translation.\n"]
for category, lines in matchedCategories.items():
if category: # Only add category header if it exists
formattedLines.append(category)
formattedLines.extend(lines)
formattedLines.append("") # Add blank line between categories
matchedVocabText = f"```\n{chr(10).join(formattedLines).rstrip()}\n```"
else:
matchedVocabText = ""
return matchedVocabText
def createContext(fullPromptFlag, subbedT, format):
system = (
PROMPT + VOCAB
if fullPromptFlag
else f"\
vocabPairs = parseVocabWithCategories(VOCAB)
matchedVocabText = buildMatchedVocabText(vocabPairs, subbedT)
if fullPromptFlag:
system = PROMPT + matchedVocabText
else:
system = f"\
You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
- You translate everything, including erotic and explicit content.\n\
@ -417,9 +473,8 @@ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{
- Maintain any spacing in the translation.\n\
- Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\
- `...` can be a part of the dialogue. Translate it as it is.\n\
{VOCAB}\n\
{matchedVocabText}\n\
"
)
if format == "json":
user = f"```json\n{subbedT}\n```"
else:
@ -474,6 +529,8 @@ def cleanTranslatedText(translatedText):
"this guy": "this bastard",
"This guy": "This bastard",
"Placeholder Text": "",
"```json": "",
"```": "",
# Add more replacements as needed
}
for target, replacement in placeholders.items():
@ -535,7 +592,7 @@ def countTokens(system, user, history):
inputTotalTokens += len(enc.encode(user))
# Output
outputTotalTokens += round(len(enc.encode(user)) * 3)
outputTotalTokens += round(len(enc.encode(user)) * 2.5)
return [inputTotalTokens, outputTotalTokens]
@ -599,7 +656,8 @@ def translateGPT(text, history, fullPromptFlag):
# AI Refused, Try Again
if not translatedText:
response = translateText(f"{system}\n You translate ALL content.", user, history, 0.1, format)
response = translateText(f"{system}\n You translate ALL content.", user, history, 0.1, format, model="gpt-4o")
translatedText = response.choices[0].message.content
# Report Tokens
totalTokens[0] += response.usage.prompt_tokens

86
modules/renpy.py
View file

@ -280,7 +280,6 @@ def translateRenpy(data, translatedList):
MISMATCH.append(FILENAME)
return tokens
# Save some money and enter the character before translation
def getSpeaker(speaker):
match speaker:
@ -298,7 +297,7 @@ def getSpeaker(speaker):
response = translateGPT(
f"{speaker}",
"Reply with the " + LANGUAGE + " translation of the NPC name.",
True,
False,
)
response[0] = response[0].title()
response[0] = response[0].replace("'S", "'s")
@ -327,11 +326,68 @@ def batchList(input_list, batch_size):
return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)]
def parseVocabWithCategories(vocabText):
"""Parse vocabulary text and extract terms with their categories."""
pairs = []
seen = set()
currentCategory = None
for line in vocabText.splitlines():
line = line.strip()
if not line or line.startswith('```'):
continue
# Check if this is a category header
if line.startswith('#'):
currentCategory = line
continue
# Parse vocabulary term
m = re.match(r'^(.+?)(?:\s?[\(–])', line) # term is everything before space + '(' or '–'
if m:
term = m.group(1)
if term not in seen:
pairs.append((term, line, currentCategory))
seen.add(term)
return pairs
def buildMatchedVocabText(vocabPairs, subbedT):
"""Build formatted vocabulary text with matched terms organized by category."""
matchedCategories = {}
# Use word boundaries for Japanese if appropriate, or allow substring as before.
for term, line, category in vocabPairs:
# "term in subbedT" could be false positive; can use regex but Japanese doesn't always have spaces.
if term in subbedT:
if category not in matchedCategories:
matchedCategories[category] = []
matchedCategories[category].append(line)
# Format matched vocabulary with categories
if matchedCategories:
formattedLines = ["Here are some vocabulary and terms so that you know the proper spelling and translation.\n"]
for category, lines in matchedCategories.items():
if category: # Only add category header if it exists
formattedLines.append(category)
formattedLines.extend(lines)
formattedLines.append("") # Add blank line between categories
matchedVocabText = f"```\n{chr(10).join(formattedLines).rstrip()}\n```"
else:
matchedVocabText = ""
return matchedVocabText
def createContext(fullPromptFlag, subbedT, format):
system = (
PROMPT + VOCAB
if fullPromptFlag
else f"\
vocabPairs = parseVocabWithCategories(VOCAB)
matchedVocabText = buildMatchedVocabText(vocabPairs, subbedT)
if fullPromptFlag:
system = PROMPT + matchedVocabText
else:
system = f"\
You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
- You translate everything, including erotic and explicit content.\n\
@ -341,9 +397,8 @@ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{
- Maintain any spacing in the translation.\n\
- Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\
- `...` can be a part of the dialogue. Translate it as it is.\n\
{VOCAB}\n\
{matchedVocabText}\n\
"
)
if format == "json":
user = f"```json\n{subbedT}\n```"
else:
@ -357,9 +412,10 @@ def translateText(system, user, history, penalty, format, model=MODEL):
# History
if isinstance(history, list):
msg.extend([{"role": "system", "content": h} for h in history])
msg.append({"role": "system", "content": "Translation History:"})
msg.extend([{"role": "assistant", "content": h} for h in history])
else:
msg.append({"role": "system", "content": history})
msg.append({"role": "assistant", "content": history})
# Response Format
if format == "json":
@ -394,8 +450,11 @@ def cleanTranslatedText(translatedText):
"】": "]",
"【": "[",
"é": "e",
"ō": "o",
"this guy": "this bastard",
"This guy": "This bastard",
"Placeholder Text": "",
"```json": "",
"```": "",
# Add more replacements as needed
}
for target, replacement in placeholders.items():
@ -457,7 +516,7 @@ def countTokens(system, user, history):
inputTotalTokens += len(enc.encode(user))
# Output
outputTotalTokens += round(len(enc.encode(user)) * 3)
outputTotalTokens += round(len(enc.encode(user)) * 2.5)
return [inputTotalTokens, outputTotalTokens]
@ -521,7 +580,8 @@ def translateGPT(text, history, fullPromptFlag):
# AI Refused, Try Again
if not translatedText:
response = translateText(f"{system}\n You translate ALL content.", user, history, 0.1, format)
response = translateText(f"{system}\n You translate ALL content.", user, history, 0.1, format, model="gpt-4o")
translatedText = response.choices[0].message.content
# Report Tokens
totalTokens[0] += response.usage.prompt_tokens

69
modules/rpgmakerace.py
View file

@ -2413,7 +2413,6 @@ def searchSystem(data, pbar):
return totalTokens
# Save some money and enter the character before translation
def getSpeaker(speaker):
match speaker:
@ -2460,11 +2459,68 @@ def batchList(input_list, batch_size):
return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)]
def parseVocabWithCategories(vocabText):
"""Parse vocabulary text and extract terms with their categories."""
pairs = []
seen = set()
currentCategory = None
for line in vocabText.splitlines():
line = line.strip()
if not line or line.startswith('```'):
continue
# Check if this is a category header
if line.startswith('#'):
currentCategory = line
continue
# Parse vocabulary term
m = re.match(r'^(.+?)(?:\s?[\(–])', line) # term is everything before space + '(' or '–'
if m:
term = m.group(1)
if term not in seen:
pairs.append((term, line, currentCategory))
seen.add(term)
return pairs
def buildMatchedVocabText(vocabPairs, subbedT):
"""Build formatted vocabulary text with matched terms organized by category."""
matchedCategories = {}
# Use word boundaries for Japanese if appropriate, or allow substring as before.
for term, line, category in vocabPairs:
# "term in subbedT" could be false positive; can use regex but Japanese doesn't always have spaces.
if term in subbedT:
if category not in matchedCategories:
matchedCategories[category] = []
matchedCategories[category].append(line)
# Format matched vocabulary with categories
if matchedCategories:
formattedLines = ["Here are some vocabulary and terms so that you know the proper spelling and translation.\n"]
for category, lines in matchedCategories.items():
if category: # Only add category header if it exists
formattedLines.append(category)
formattedLines.extend(lines)
formattedLines.append("") # Add blank line between categories
matchedVocabText = f"```\n{chr(10).join(formattedLines).rstrip()}\n```"
else:
matchedVocabText = ""
return matchedVocabText
def createContext(fullPromptFlag, subbedT, format):
system = (
PROMPT + VOCAB
if fullPromptFlag
else f"\
vocabPairs = parseVocabWithCategories(VOCAB)
matchedVocabText = buildMatchedVocabText(vocabPairs, subbedT)
if fullPromptFlag:
system = PROMPT + matchedVocabText
else:
system = f"\
You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
- You translate everything, including erotic and explicit content.\n\
@ -2474,9 +2530,8 @@ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{
- Maintain any spacing in the translation.\n\
- Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\
- `...` can be a part of the dialogue. Translate it as it is.\n\
{VOCAB}\n\
{matchedVocabText}\n\
"
)
if format == "json":
user = f"```json\n{subbedT}\n```"
else:

1
modules/rpgmakermvmz.py
View file

@ -2433,7 +2433,6 @@ def searchSystem(data, pbar):
return totalTokens
# Save some money and enter the character before translation
def getSpeaker(speaker):
match speaker:

86
modules/rpgmakerplugin.py
View file

@ -586,7 +586,6 @@ def translatePlugin(data, pbar, filename, translatedList):
translatePlugin(data, pbar, filename, [questListTL, customTL])
return tokens
# Save some money and enter the character before translation
def getSpeaker(speaker):
match speaker:
@ -604,7 +603,7 @@ def getSpeaker(speaker):
response = translateGPT(
f"{speaker}",
"Reply with the " + LANGUAGE + " translation of the NPC name.",
True,
False,
)
response[0] = response[0].title()
response[0] = response[0].replace("'S", "'s")
@ -633,11 +632,68 @@ def batchList(input_list, batch_size):
return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)]
def parseVocabWithCategories(vocabText):
"""Parse vocabulary text and extract terms with their categories."""
pairs = []
seen = set()
currentCategory = None
for line in vocabText.splitlines():
line = line.strip()
if not line or line.startswith('```'):
continue
# Check if this is a category header
if line.startswith('#'):
currentCategory = line
continue
# Parse vocabulary term
m = re.match(r'^(.+?)(?:\s?[\(–])', line) # term is everything before space + '(' or '–'
if m:
term = m.group(1)
if term not in seen:
pairs.append((term, line, currentCategory))
seen.add(term)
return pairs
def buildMatchedVocabText(vocabPairs, subbedT):
"""Build formatted vocabulary text with matched terms organized by category."""
matchedCategories = {}
# Use word boundaries for Japanese if appropriate, or allow substring as before.
for term, line, category in vocabPairs:
# "term in subbedT" could be false positive; can use regex but Japanese doesn't always have spaces.
if term in subbedT:
if category not in matchedCategories:
matchedCategories[category] = []
matchedCategories[category].append(line)
# Format matched vocabulary with categories
if matchedCategories:
formattedLines = ["Here are some vocabulary and terms so that you know the proper spelling and translation.\n"]
for category, lines in matchedCategories.items():
if category: # Only add category header if it exists
formattedLines.append(category)
formattedLines.extend(lines)
formattedLines.append("") # Add blank line between categories
matchedVocabText = f"```\n{chr(10).join(formattedLines).rstrip()}\n```"
else:
matchedVocabText = ""
return matchedVocabText
def createContext(fullPromptFlag, subbedT, format):
system = (
PROMPT + VOCAB
if fullPromptFlag
else f"\
vocabPairs = parseVocabWithCategories(VOCAB)
matchedVocabText = buildMatchedVocabText(vocabPairs, subbedT)
if fullPromptFlag:
system = PROMPT + matchedVocabText
else:
system = f"\
You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
- You translate everything, including erotic and explicit content.\n\
@ -647,9 +703,8 @@ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{
- Maintain any spacing in the translation.\n\
- Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\
- `...` can be a part of the dialogue. Translate it as it is.\n\
{VOCAB}\n\
{matchedVocabText}\n\
"
)
if format == "json":
user = f"```json\n{subbedT}\n```"
else:
@ -663,9 +718,10 @@ def translateText(system, user, history, penalty, format, model=MODEL):
# History
if isinstance(history, list):
msg.extend([{"role": "system", "content": h} for h in history])
msg.append({"role": "system", "content": "Translation History:"})
msg.extend([{"role": "assistant", "content": h} for h in history])
else:
msg.append({"role": "system", "content": history})
msg.append({"role": "assistant", "content": history})
# Response Format
if format == "json":
@ -700,8 +756,11 @@ def cleanTranslatedText(translatedText):
"】": "]",
"【": "[",
"é": "e",
"ō": "o",
"this guy": "this bastard",
"This guy": "This bastard",
"Placeholder Text": "",
"```json": "",
"```": "",
# Add more replacements as needed
}
for target, replacement in placeholders.items():
@ -763,7 +822,7 @@ def countTokens(system, user, history):
inputTotalTokens += len(enc.encode(user))
# Output
outputTotalTokens += round(len(enc.encode(user)) * 3)
outputTotalTokens += round(len(enc.encode(user)) * 2.5)
return [inputTotalTokens, outputTotalTokens]
@ -827,7 +886,8 @@ def translateGPT(text, history, fullPromptFlag):
# AI Refused, Try Again
if not translatedText:
response = translateText(f"{system}\n You translate ALL content.", user, history, 0.1, format)
response = translateText(f"{system}\n You translate ALL content.", user, history, 0.1, format, model="gpt-4o")
translatedText = response.choices[0].message.content
# Report Tokens
totalTokens[0] += response.usage.prompt_tokens

600
modules/sakuranbo.py
View file

@ -1,600 +0,0 @@
import os
import re
import util.dazedwrap as dazedwrap
import threading
import time
import traceback
from pathlib import Path
import openai
import tiktoken
from colorama import Fore
from dotenv import load_dotenv
from retry import retry
from tqdm import tqdm
# Open AI
load_dotenv()
if os.getenv("api").replace(" ", "") != "":
openai.base_url = os.getenv("api")
openai.organization = os.getenv("org")
openai.api_key = os.getenv("key")
# Globals
MODEL = os.getenv("model")
TIMEOUT = int(os.getenv("timeout"))
LANGUAGE = os.getenv("language").capitalize()
INPUTAPICOST = 0.002 # Depends on the model https://openai.com/pricing
OUTPUTAPICOST = 0.002
PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
THREADS = int(os.getenv("threads")) # Controls how many threads are working on a single file (May have to drop this)
LOCK = threading.Lock()
WIDTH = int(os.getenv("width"))
LISTWIDTH = int(os.getenv("listWidth"))
NOTEWIDTH = 40
MAXHISTORY = 10
ESTIMATE = ""
totalTokens = [0, 0]
NAMESLIST = []
# tqdm Globals
BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
POSITION = 0
LEAVE = False
# Flags
NAMES = False # Output a list of all the character names found
BRFLAG = False # If the game uses <br> instead
FIXTEXTWRAP = True
IGNORETLTEXT = False
def handleSakuranbo(filename, estimate):
global ESTIMATE
totalTokens = [0, 0]
ESTIMATE = estimate
if estimate:
start = time.time()
translatedData = openFiles(filename)
# Print Result
end = time.time()
tqdm.write(getResultString(translatedData, end - start, filename))
if NAMES is True:
tqdm.write(str(NAMESLIST))
with LOCK:
totalTokens[0] += translatedData[1][0]
totalTokens[1] += translatedData[1][1]
return getResultString(["", totalTokens, None], end - start, "TOTAL")
else:
try:
with open("translated/" + filename, "w", encoding="utf-16") as outFile:
start = time.time()
translatedData = openFiles(filename)
outFile.writelines(translatedData[0])
# Print Result
end = time.time()
tqdm.write(getResultString(translatedData, end - start, filename))
with LOCK:
totalTokens[0] += translatedData[1][0]
totalTokens[1] += translatedData[1][1]
except Exception:
traceback.print_exc()
return "Fail"
return getResultString(["", totalTokens, None], end - start, "TOTAL")
def getResultString(translatedData, translationTime, filename):
# File Print String
totalTokenstring = (
Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
"[Output: "
+ str(translatedData[1][1])
+ "]" "[Cost: ${:,.4f}".format(((translatedData[1][0] / 1000000) * INPUTAPICOST) + ((translatedData[1][1] / 1000000) * OUTPUTAPICOST))
+ "]"
)
timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
if translatedData[2] is None:
# Success
return filename + ": " + totalTokenstring + timeString + Fore.GREEN + " \u2713 " + Fore.RESET
else:
# Fail
try:
raise translatedData[2]
except Exception as e:
errorString = str(e) + Fore.RED
return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET
def openFiles(filename):
with open("files/" + filename, "r", encoding="utf-16") as readFile:
translatedData = parseTyrano(readFile, filename)
# Delete lines marked for deletion
finalData = []
for line in translatedData[0]:
if line != "\\d\n":
finalData.append(line)
translatedData[0] = finalData
return translatedData
def parseTyrano(readFile, filename):
totalTokens = [0, 0]
totalLines = 0
# Get total for progress bar
data = readFile.readlines()
totalLines = len(data)
with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar:
pbar.desc = filename
pbar.total = totalLines
try:
response = translateTyrano(data, pbar)
totalTokens[0] = response[0]
totalTokens[1] = response[1]
except Exception as e:
traceback.print_exc()
return [data, totalTokens, e]
return [data, totalTokens, None]
def translateTyrano(data, pbar):
textHistory = []
maxHistory = MAXHISTORY
tokens = [0, 0]
currentGroup = []
syncIndex = 0
speaker = ""
delFlag = False
global LOCK, ESTIMATE
for i in range(len(data)):
currentGroup = []
matchList = []
if syncIndex > i:
i = syncIndex
if "[▼]" in data[i]:
data[i] = data[i].replace("[▼]".strip(), "[page]\n")
# If there isn't any Japanese in the text just skip
if IGNORETLTEXT is True:
if not re.search(r"[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+", data[i]):
# Keep textHistory list at length maxHistory
textHistory.append('"' + data[i] + '"')
if len(textHistory) > maxHistory:
textHistory.pop(0)
currentGroup = []
continue
# Speaker
matchList = re.findall(r"^\[(.+)\sstorage=.+\]", data[i])
if len(matchList) == 0:
matchList = re.findall(r"^\[([^/].+)\]$", data[i])
if len(matchList) > 0:
if "主人公" in matchList[0]:
speaker = "Protagonist"
elif "思考" in matchList[0]:
speaker = "Protagonist Inner Thoughts"
elif "地の文" in matchList[0]:
speaker = "Narrator"
elif "マコ" in matchList[0]:
speaker = "Mako"
elif "少年" in matchList[0]:
speaker = "Boy"
elif "友達" in matchList[0]:
speaker = "Friend"
elif "少女" in matchList[0]:
speaker = "Girl"
else:
response = translateGPT(
matchList[0],
"Reply with only the " + LANGUAGE + " translation of the NPC name",
True,
)
speaker = response[0]
tokens[0] += response[1][0]
tokens[1] += response[1][1]
# data[i] = '#' + speaker + '\n'
# Choices
elif "glink" in data[i]:
matchList = re.findall(r"\[glink.+text=\"(.+?)\".+", data[i])
if len(matchList) != 0:
if len(textHistory) > 0:
response = translateGPT(
matchList[0],
"Past Translated Text: " + textHistory[len(textHistory) - 1] + "\n\nReply in the style of a dialogue option.",
True,
)
else:
response = translateGPT(matchList[0], "", False)
translatedText = response[0]
tokens[0] += response[1][0]
tokens[1] += response[1][1]
# Remove characters that may break scripts
charList = [".", '"', "\\n"]
for char in charList:
translatedText = translatedText.replace(char, "")
# Escape all '
translatedText = translatedText.replace("\\", "")
translatedText = translatedText.replace("'", "\\'")
# Set Data
translatedText = data[i].replace(matchList[0], translatedText.replace(" ", "\u00a0"))
data[i] = translatedText
# Grab Lines
matchList = re.findall(r"^([^\n;@*\{\[].+[^;'{}\[]$)", data[i])
if len(matchList) > 0 and (
re.search(r"^\[(.+)\sstorage=.+\],", data[i - 1]) or re.search(r"^\[(.+)\]$", data[i - 1]) or re.search(r"^《(.+)》", data[i - 1])
):
currentGroup.append(matchList[0])
if len(data) > i + 1:
matchList = re.findall(r"^([^\n;@*\{\[].+[^;'{}\[]$)", data[i + 1])
while len(matchList) > 0:
delFlag = True
data[i] = "\d\n" # \d Marks line for deletion
i += 1
matchList = re.findall(r"^([^\n;@*\{\[].+[^;'{}\[]$)", data[i])
if len(matchList) > 0:
currentGroup.append(matchList[0])
# Join up 401 groups for better translation.
if len(currentGroup) > 0:
finalJAString = " ".join(currentGroup)
# Remove any textwrap
if FIXTEXTWRAP is True:
finalJAString = finalJAString.replace("_", " ")
# Check Speaker
if speaker == "":
response = translateGPT(finalJAString, textHistory, True)
tokens[0] += response[1][0]
tokens[1] += response[1][1]
translatedText = response[0]
textHistory.append('"' + translatedText + '"')
else:
response = translateGPT(speaker + ": " + finalJAString, textHistory, True)
tokens[0] += response[1][0]
tokens[1] += response[1][1]
translatedText = response[0]
textHistory.append('"' + translatedText + '"')
# Remove added speaker
translatedText = re.sub(r"^.+:\s?", "", translatedText)
# Set Data
translatedText = translatedText.replace("ッ", "")
translatedText = translatedText.replace("っ", "")
translatedText = translatedText.replace("ー", "")
translatedText = translatedText.replace('"', "")
translatedText = translatedText.replace("[", "")
translatedText = translatedText.replace("]", "")
# Wordwrap Text
if "_" not in translatedText:
translatedText = dazedwrap.wrapText(translatedText, width=WIDTH)
translatedText = translatedText.replace("\n", "_")
# Set
if delFlag is True:
data.insert(i, translatedText.strip() + "\n")
delFlag = False
else:
data[i] = translatedText.strip() + "\n"
# Keep textHistory list at length maxHistory
if len(textHistory) > maxHistory:
textHistory.pop(0)
currentGroup = []
speaker = ""
pbar.update(1)
if len(data) > i + 1:
syncIndex = i + 1
else:
break
# Grab Lines
matchList = re.findall(r"(^\[.+\sstorage=.+\](.+)\[/.+\])", data[i])
if len(matchList) > 0:
originalLine = matchList[0][0]
originalText = matchList[0][1]
currentGroup.append(matchList[0][1])
if len(data) > i + 1:
matchList = re.findall(r"^([^\n;@*\{\[].+[^;'{}\[]$)", data[i + 1])
while len(matchList) > 0:
delFlag = True
data[i] = "\d\n" # \d Marks line for deletion
i += 1
matchList = re.findall(r"^([^\n;@*\{\[].+[^;'{}\[]$)", data[i])
if len(matchList) > 0:
currentGroup.append(matchList[0])
# Join up 401 groups for better translation.
if len(currentGroup) > 0:
finalJAString = " ".join(currentGroup)
# Remove any textwrap
if FIXTEXTWRAP is True:
finalJAString = finalJAString.replace("_", " ")
# Check Speaker
if speaker == "":
response = translateGPT(finalJAString, textHistory, True)
tokens[0] += response[1][0]
tokens[1] += response[1][1]
translatedText = response[0]
textHistory.append('"' + translatedText + '"')
else:
response = translateGPT(speaker + ": " + finalJAString, textHistory, True)
tokens[0] += response[1][0]
tokens[1] += response[1][1]
translatedText = response[0]
textHistory.append('"' + translatedText + '"')
# Remove added speaker
translatedText = re.sub(r"^.+:\s?", "", translatedText)
# Set Data
translatedText = translatedText.replace("ッ", "")
translatedText = translatedText.replace("っ", "")
translatedText = translatedText.replace("ー", "")
translatedText = translatedText.replace('"', "")
translatedText = translatedText.replace("[", "")
translatedText = translatedText.replace("]", "")
# Wordwrap Text
if "_" not in translatedText:
translatedText = dazedwrap.wrapText(translatedText, width=WIDTH)
translatedText = translatedText.replace("\n", "_")
translatedText = originalLine.replace(originalText, translatedText)
# Set
if delFlag is True:
data.insert(i, translatedText.strip() + "\n")
delFlag = False
else:
data[i] = translatedText.strip() + "\n"
# Keep textHistory list at length maxHistory
if len(textHistory) > maxHistory:
textHistory.pop(0)
currentGroup = []
speaker = ""
pbar.update(1)
if len(data) > i + 1:
syncIndex = i + 1
else:
break
return tokens
def subVars(jaString):
jaString = jaString.replace("\u3000", " ")
# Nested
count = 0
nestedList = re.findall(r"[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]", jaString)
nestedList = set(nestedList)
if len(nestedList) != 0:
for icon in nestedList:
jaString = jaString.replace(icon, "{Nested_" + str(count) + "}")
count += 1
# Icons
count = 0
iconList = re.findall(r"[\\]+[iIkKwWaA]+\[[0-9]+\]", jaString)
iconList = set(iconList)
if len(iconList) != 0:
for icon in iconList:
jaString = jaString.replace(icon, "{Ascii_" + str(count) + "}")
count += 1
# Colors
count = 0
colorList = re.findall(r"[\\]+[cC]\[[0-9]+\]", jaString)
colorList = set(colorList)
if len(colorList) != 0:
for color in colorList:
jaString = jaString.replace(color, "{Color_" + str(count) + "}")
count += 1
# Names
count = 0
nameList = re.findall(r"[\\]+[nN]\[.+?\]+", jaString)
nameList = set(nameList)
if len(nameList) != 0:
for name in nameList:
jaString = jaString.replace(name, "{N_" + str(count) + "}")
count += 1
# Variables
count = 0
varList = re.findall(r"[\\]+[vV]\[[0-9]+\]", jaString)
varList = set(varList)
if len(varList) != 0:
for var in varList:
jaString = jaString.replace(var, "{Var_" + str(count) + "}")
count += 1
# Formatting
count = 0
if "笑えるよね." in jaString:
print("t")
formatList = re.findall(r"[\\]+[\w]+\[.+?\]", jaString)
formatList = set(formatList)
if len(formatList) != 0:
for var in formatList:
jaString = jaString.replace(var, "{FCode_" + str(count) + "}")
count += 1
# Put all lists in list and return
allList = [nestedList, iconList, colorList, nameList, varList, formatList]
return [jaString, allList]
def resubVars(translatedText, allList):
# Fix Spacing and ChatGPT Nonsense
matchList = re.findall(r"\[\s?.+?\s?\]", translatedText)
if len(matchList) > 0:
for match in matchList:
text = match.strip()
translatedText = translatedText.replace(match, text)
# Nested
count = 0
if len(allList[0]) != 0:
for var in allList[0]:
translatedText = translatedText.replace("{Nested_" + str(count) + "}", var)
count += 1
# Icons
count = 0
if len(allList[1]) != 0:
for var in allList[1]:
translatedText = translatedText.replace("{Ascii_" + str(count) + "}", var)
count += 1
# Colors
count = 0
if len(allList[2]) != 0:
for var in allList[2]:
translatedText = translatedText.replace("{Color_" + str(count) + "}", var)
count += 1
# Names
count = 0
if len(allList[3]) != 0:
for var in allList[3]:
translatedText = translatedText.replace("{N_" + str(count) + "}", var)
count += 1
# Vars
count = 0
if len(allList[4]) != 0:
for var in allList[4]:
translatedText = translatedText.replace("{Var_" + str(count) + "}", var)
count += 1
# Formatting
count = 0
if len(allList[5]) != 0:
for var in allList[5]:
translatedText = translatedText.replace("{FCode_" + str(count) + "}", var)
count += 1
# Remove Color Variables Spaces
# if '\\c' in translatedText:
# translatedText = re.sub(r'\s*(\\+c\[[1-9]+\])\s*', r' \1', translatedText)
# translatedText = re.sub(r'\s*(\\+c\[0+\])', r'\1', translatedText)
return translatedText
@retry(exceptions=Exception, tries=5, delay=5)
def translateGPT(t, history, fullPromptFlag):
# Sub Vars
varResponse = subVars(t)
subbedT = varResponse[0]
# If there isn't any Japanese in the text just skip
if not re.search(r"[一-龠]+|[ぁ-ゔ]+|[ァ-ヴ]+|[\uFF00-\uFFEF]", subbedT):
return (t, [0, 0])
# If ESTIMATE is True just count this as an execution and return.
if ESTIMATE:
enc = tiktoken.encoding_for_model("gpt-4")
historyRaw = ""
if isinstance(history, list):
for line in history:
historyRaw += line
else:
historyRaw = history
inputTotalTokens = len(enc.encode(historyRaw)) + len(enc.encode(PROMPT))
outputTotalTokens = len(enc.encode(t)) * 2 # Estimating 2x the size of the original text
totalTokens = [inputTotalTokens, outputTotalTokens]
return (t, totalTokens)
# Characters
context = "Game Characters:\
Character: マコ == Mako - Gender: Female\
Character: 主人公 == Protagonist - Gender: Male"
# Prompt
if fullPromptFlag:
system = PROMPT
user = "Line to Translate = " + subbedT
else:
system = "Output ONLY the " + LANGUAGE + " translation in the following format: `Translation: <" + LANGUAGE.upper() + "_TRANSLATION>`"
user = "Line to Translate = " + subbedT
# Create Message List
msg = []
msg.append({"role": "system", "content": system})
msg.append({"role": "user", "content": context})
if isinstance(history, list):
for line in history:
msg.append({"role": "user", "content": line})
else:
msg.append({"role": "user", "content": history})
msg.append({"role": "user", "content": user})
response = openai.ChatCompletion.create(
temperature=0,
frequency_penalty=0.2,
presence_penalty=0.2,
model=MODEL,
messages=msg,
request_timeout=TIMEOUT,
)
# Save Translated Text
translatedText = response.choices[0].message.content
totalTokens = [response.usage.prompt_tokens, response.usage.completion_tokens]
# Resub Vars
translatedText = resubVars(translatedText, varResponse[1])
# Remove Placeholder Text
translatedText = translatedText.replace(LANGUAGE + " Translation: ", "")
translatedText = translatedText.replace("Translation: ", "")
translatedText = translatedText.replace("Line to Translate = ", "")
translatedText = translatedText.replace("Translation = ", "")
translatedText = translatedText.replace("Translate = ", "")
translatedText = translatedText.replace(LANGUAGE + " Translation:", "")
translatedText = translatedText.replace("Translation:", "")
translatedText = translatedText.replace("Line to Translate =", "")
translatedText = translatedText.replace("Translation =", "")
translatedText = translatedText.replace("Translate =", "")
translatedText = translatedText.replace("っ", "")
translatedText = translatedText.replace("ッ", "")
translatedText = translatedText.replace("ぁ", "")
translatedText = translatedText.replace("。", ".")
translatedText = translatedText.replace("、", ",")
translatedText = translatedText.replace("？", "?")
translatedText = translatedText.replace("！", "!")
# Return Translation
if len(translatedText) > 15 * len(t) or "I'm sorry, but I'm unable to assist with that translation" in translatedText:
raise Exception
else:
return [translatedText, totalTokens]

86
modules/text.py
View file

@ -269,7 +269,6 @@ def translateTxt(data, translatedList):
translateTxt(data, [stringListTL, choiceListTL])
return tokens
# Save some money and enter the character before translation
def getSpeaker(speaker):
match speaker:
@ -287,7 +286,7 @@ def getSpeaker(speaker):
response = translateGPT(
f"{speaker}",
"Reply with the " + LANGUAGE + " translation of the NPC name.",
True,
False,
)
response[0] = response[0].title()
response[0] = response[0].replace("'S", "'s")
@ -316,11 +315,68 @@ def batchList(input_list, batch_size):
return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)]
def parseVocabWithCategories(vocabText):
"""Parse vocabulary text and extract terms with their categories."""
pairs = []
seen = set()
currentCategory = None
for line in vocabText.splitlines():
line = line.strip()
if not line or line.startswith('```'):
continue
# Check if this is a category header
if line.startswith('#'):
currentCategory = line
continue
# Parse vocabulary term
m = re.match(r'^(.+?)(?:\s?[\(–])', line) # term is everything before space + '(' or '–'
if m:
term = m.group(1)
if term not in seen:
pairs.append((term, line, currentCategory))
seen.add(term)
return pairs
def buildMatchedVocabText(vocabPairs, subbedT):
"""Build formatted vocabulary text with matched terms organized by category."""
matchedCategories = {}
# Use word boundaries for Japanese if appropriate, or allow substring as before.
for term, line, category in vocabPairs:
# "term in subbedT" could be false positive; can use regex but Japanese doesn't always have spaces.
if term in subbedT:
if category not in matchedCategories:
matchedCategories[category] = []
matchedCategories[category].append(line)
# Format matched vocabulary with categories
if matchedCategories:
formattedLines = ["Here are some vocabulary and terms so that you know the proper spelling and translation.\n"]
for category, lines in matchedCategories.items():
if category: # Only add category header if it exists
formattedLines.append(category)
formattedLines.extend(lines)
formattedLines.append("") # Add blank line between categories
matchedVocabText = f"```\n{chr(10).join(formattedLines).rstrip()}\n```"
else:
matchedVocabText = ""
return matchedVocabText
def createContext(fullPromptFlag, subbedT, format):
system = (
PROMPT + VOCAB
if fullPromptFlag
else f"\
vocabPairs = parseVocabWithCategories(VOCAB)
matchedVocabText = buildMatchedVocabText(vocabPairs, subbedT)
if fullPromptFlag:
system = PROMPT + matchedVocabText
else:
system = f"\
You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
- You translate everything, including erotic and explicit content.\n\
@ -330,9 +386,8 @@ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{
- Maintain any spacing in the translation.\n\
- Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\
- `...` can be a part of the dialogue. Translate it as it is.\n\
{VOCAB}\n\
{matchedVocabText}\n\
"
)
if format == "json":
user = f"```json\n{subbedT}\n```"
else:
@ -346,9 +401,10 @@ def translateText(system, user, history, penalty, format, model=MODEL):
# History
if isinstance(history, list):
msg.extend([{"role": "system", "content": h} for h in history])
msg.append({"role": "system", "content": "Translation History:"})
msg.extend([{"role": "assistant", "content": h} for h in history])
else:
msg.append({"role": "system", "content": history})
msg.append({"role": "assistant", "content": history})
# Response Format
if format == "json":
@ -383,8 +439,11 @@ def cleanTranslatedText(translatedText):
"】": "]",
"【": "[",
"é": "e",
"ō": "o",
"this guy": "this bastard",
"This guy": "This bastard",
"Placeholder Text": "",
"```json": "",
"```": "",
# Add more replacements as needed
}
for target, replacement in placeholders.items():
@ -446,7 +505,7 @@ def countTokens(system, user, history):
inputTotalTokens += len(enc.encode(user))
# Output
outputTotalTokens += round(len(enc.encode(user)) * 3)
outputTotalTokens += round(len(enc.encode(user)) * 2.5)
return [inputTotalTokens, outputTotalTokens]
@ -510,7 +569,8 @@ def translateGPT(text, history, fullPromptFlag):
# AI Refused, Try Again
if not translatedText:
response = translateText(f"{system}\n You translate ALL content.", user, history, 0.1, format)
response = translateText(f"{system}\n You translate ALL content.", user, history, 0.1, format, model="gpt-4o")
translatedText = response.choices[0].message.content
# Report Tokens
totalTokens[0] += response.usage.prompt_tokens

78
modules/tyrano.py
View file

@ -356,7 +356,6 @@ def translateTyrano(data, translatedList):
translateTyrano(data, [stringListTL, choiceListTL])
return tokens
# Save some money and enter the character before translation
def getSpeaker(speaker):
match speaker:
@ -403,11 +402,68 @@ def batchList(input_list, batch_size):
return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)]
def parseVocabWithCategories(vocabText):
"""Parse vocabulary text and extract terms with their categories."""
pairs = []
seen = set()
currentCategory = None
for line in vocabText.splitlines():
line = line.strip()
if not line or line.startswith('```'):
continue
# Check if this is a category header
if line.startswith('#'):
currentCategory = line
continue
# Parse vocabulary term
m = re.match(r'^(.+?)(?:\s?[\(–])', line) # term is everything before space + '(' or '–'
if m:
term = m.group(1)
if term not in seen:
pairs.append((term, line, currentCategory))
seen.add(term)
return pairs
def buildMatchedVocabText(vocabPairs, subbedT):
"""Build formatted vocabulary text with matched terms organized by category."""
matchedCategories = {}
# Use word boundaries for Japanese if appropriate, or allow substring as before.
for term, line, category in vocabPairs:
# "term in subbedT" could be false positive; can use regex but Japanese doesn't always have spaces.
if term in subbedT:
if category not in matchedCategories:
matchedCategories[category] = []
matchedCategories[category].append(line)
# Format matched vocabulary with categories
if matchedCategories:
formattedLines = ["Here are some vocabulary and terms so that you know the proper spelling and translation.\n"]
for category, lines in matchedCategories.items():
if category: # Only add category header if it exists
formattedLines.append(category)
formattedLines.extend(lines)
formattedLines.append("") # Add blank line between categories
matchedVocabText = f"```\n{chr(10).join(formattedLines).rstrip()}\n```"
else:
matchedVocabText = ""
return matchedVocabText
def createContext(fullPromptFlag, subbedT, format):
system = (
PROMPT + VOCAB
if fullPromptFlag
else f"\
vocabPairs = parseVocabWithCategories(VOCAB)
matchedVocabText = buildMatchedVocabText(vocabPairs, subbedT)
if fullPromptFlag:
system = PROMPT + matchedVocabText
else:
system = f"\
You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
- You translate everything, including erotic and explicit content.\n\
@ -417,9 +473,8 @@ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{
- Maintain any spacing in the translation.\n\
- Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\
- `...` can be a part of the dialogue. Translate it as it is.\n\
{VOCAB}\n\
{matchedVocabText}\n\
"
)
if format == "json":
user = f"```json\n{subbedT}\n```"
else:
@ -433,7 +488,7 @@ def translateText(system, user, history, penalty, format, model=MODEL):
# History
if isinstance(history, list):
msg.append({"role": "assistant", "content": "Translation History:"})
msg.append({"role": "system", "content": "Translation History:"})
msg.extend([{"role": "assistant", "content": h} for h in history])
else:
msg.append({"role": "assistant", "content": history})
@ -474,6 +529,8 @@ def cleanTranslatedText(translatedText):
"this guy": "this bastard",
"This guy": "This bastard",
"Placeholder Text": "",
"```json": "",
"```": "",
# Add more replacements as needed
}
for target, replacement in placeholders.items():
@ -535,7 +592,7 @@ def countTokens(system, user, history):
inputTotalTokens += len(enc.encode(user))
# Output
outputTotalTokens += round(len(enc.encode(user)) * 3)
outputTotalTokens += round(len(enc.encode(user)) * 2.5)
return [inputTotalTokens, outputTotalTokens]
@ -599,7 +656,8 @@ def translateGPT(text, history, fullPromptFlag):
# AI Refused, Try Again
if not translatedText:
response = translateText(f"{system}\n You translate ALL content.", user, history, 0.1, format)
response = translateText(f"{system}\n You translate ALL content.", user, history, 0.1, format, model="gpt-4o")
translatedText = response.choices[0].message.content
# Report Tokens
totalTokens[0] += response.usage.prompt_tokens

76
modules/unity.py
View file

@ -269,7 +269,6 @@ def translateUnity(data, pbar, filename, translatedList):
MISMATCH.append(filename)
return tokens
# Save some money and enter the character before translation
def getSpeaker(speaker):
match speaker:
@ -316,11 +315,68 @@ def batchList(input_list, batch_size):
return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)]
def parseVocabWithCategories(vocabText):
"""Parse vocabulary text and extract terms with their categories."""
pairs = []
seen = set()
currentCategory = None
for line in vocabText.splitlines():
line = line.strip()
if not line or line.startswith('```'):
continue
# Check if this is a category header
if line.startswith('#'):
currentCategory = line
continue
# Parse vocabulary term
m = re.match(r'^(.+?)(?:\s?[\(–])', line) # term is everything before space + '(' or '–'
if m:
term = m.group(1)
if term not in seen:
pairs.append((term, line, currentCategory))
seen.add(term)
return pairs
def buildMatchedVocabText(vocabPairs, subbedT):
"""Build formatted vocabulary text with matched terms organized by category."""
matchedCategories = {}
# Use word boundaries for Japanese if appropriate, or allow substring as before.
for term, line, category in vocabPairs:
# "term in subbedT" could be false positive; can use regex but Japanese doesn't always have spaces.
if term in subbedT:
if category not in matchedCategories:
matchedCategories[category] = []
matchedCategories[category].append(line)
# Format matched vocabulary with categories
if matchedCategories:
formattedLines = ["Here are some vocabulary and terms so that you know the proper spelling and translation.\n"]
for category, lines in matchedCategories.items():
if category: # Only add category header if it exists
formattedLines.append(category)
formattedLines.extend(lines)
formattedLines.append("") # Add blank line between categories
matchedVocabText = f"```\n{chr(10).join(formattedLines).rstrip()}\n```"
else:
matchedVocabText = ""
return matchedVocabText
def createContext(fullPromptFlag, subbedT, format):
system = (
PROMPT + VOCAB
if fullPromptFlag
else f"\
vocabPairs = parseVocabWithCategories(VOCAB)
matchedVocabText = buildMatchedVocabText(vocabPairs, subbedT)
if fullPromptFlag:
system = PROMPT + matchedVocabText
else:
system = f"\
You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
- You translate everything, including erotic and explicit content.\n\
@ -330,9 +386,8 @@ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{
- Maintain any spacing in the translation.\n\
- Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\
- `...` can be a part of the dialogue. Translate it as it is.\n\
{VOCAB}\n\
{matchedVocabText}\n\
"
)
if format == "json":
user = f"```json\n{subbedT}\n```"
else:
@ -387,6 +442,8 @@ def cleanTranslatedText(translatedText):
"this guy": "this bastard",
"This guy": "This bastard",
"Placeholder Text": "",
"```json": "",
"```": "",
# Add more replacements as needed
}
for target, replacement in placeholders.items():
@ -448,7 +505,7 @@ def countTokens(system, user, history):
inputTotalTokens += len(enc.encode(user))
# Output
outputTotalTokens += round(len(enc.encode(user)) * 3)
outputTotalTokens += round(len(enc.encode(user)) * 2.5)
return [inputTotalTokens, outputTotalTokens]
@ -512,7 +569,8 @@ def translateGPT(text, history, fullPromptFlag):
# AI Refused, Try Again
if not translatedText:
response = translateText(f"{system}\n You translate ALL content.", user, history, 0.1, format)
response = translateText(f"{system}\n You translate ALL content.", user, history, 0.1, format, model="gpt-4o")
translatedText = response.choices[0].message.content
# Report Tokens
totalTokens[0] += response.usage.prompt_tokens

139
modules/wolf.py
View file

@ -2563,7 +2563,6 @@ def searchDB(events, pbar, jobList, filename):
return totalTokens
# Save some money and enter the character before translation
def getSpeaker(speaker):
match speaker:
@ -2581,7 +2580,7 @@ def getSpeaker(speaker):
response = translateGPT(
f"{speaker}",
"Reply with the " + LANGUAGE + " translation of the NPC name.",
True,
False,
)
response[0] = response[0].title()
response[0] = response[0].replace("'S", "'s")
@ -2610,11 +2609,68 @@ def batchList(input_list, batch_size):
return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)]
def parseVocabWithCategories(vocabText):
"""Parse vocabulary text and extract terms with their categories."""
pairs = []
seen = set()
currentCategory = None
for line in vocabText.splitlines():
line = line.strip()
if not line or line.startswith('```'):
continue
# Check if this is a category header
if line.startswith('#'):
currentCategory = line
continue
# Parse vocabulary term
m = re.match(r'^(.+?)(?:\s?[\(–])', line) # term is everything before space + '(' or '–'
if m:
term = m.group(1)
if term not in seen:
pairs.append((term, line, currentCategory))
seen.add(term)
return pairs
def buildMatchedVocabText(vocabPairs, subbedT):
"""Build formatted vocabulary text with matched terms organized by category."""
matchedCategories = {}
# Use word boundaries for Japanese if appropriate, or allow substring as before.
for term, line, category in vocabPairs:
# "term in subbedT" could be false positive; can use regex but Japanese doesn't always have spaces.
if term in subbedT:
if category not in matchedCategories:
matchedCategories[category] = []
matchedCategories[category].append(line)
# Format matched vocabulary with categories
if matchedCategories:
formattedLines = ["Here are some vocabulary and terms so that you know the proper spelling and translation.\n"]
for category, lines in matchedCategories.items():
if category: # Only add category header if it exists
formattedLines.append(category)
formattedLines.extend(lines)
formattedLines.append("") # Add blank line between categories
matchedVocabText = f"```\n{chr(10).join(formattedLines).rstrip()}\n```"
else:
matchedVocabText = ""
return matchedVocabText
def createContext(fullPromptFlag, subbedT, format):
system = (
PROMPT + VOCAB
if fullPromptFlag
else f"\
vocabPairs = parseVocabWithCategories(VOCAB)
matchedVocabText = buildMatchedVocabText(vocabPairs, subbedT)
if fullPromptFlag:
system = PROMPT + matchedVocabText
else:
system = f"\
You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
- You translate everything, including erotic and explicit content.\n\
@ -2624,9 +2680,8 @@ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{
- Maintain any spacing in the translation.\n\
- Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\
- `...` can be a part of the dialogue. Translate it as it is.\n\
{VOCAB}\n\
{matchedVocabText}\n\
"
)
if format == "json":
user = f"```json\n{subbedT}\n```"
else:
@ -2640,9 +2695,10 @@ def translateText(system, user, history, penalty, format, model=MODEL):
# History
if isinstance(history, list):
msg.extend([{"role": "system", "content": h} for h in history])
msg.append({"role": "system", "content": "Translation History:"})
msg.extend([{"role": "assistant", "content": h} for h in history])
else:
msg.append({"role": "system", "content": history})
msg.append({"role": "assistant", "content": history})
# Response Format
if format == "json":
@ -2663,36 +2719,36 @@ def translateText(system, user, history, penalty, format, model=MODEL):
def cleanTranslatedText(translatedText):
if translatedText:
placeholders = {
f"{LANGUAGE} Translation: ": "",
"Translation: ": "",
"っ": "",
"〜": "~",
"ッ": "",
"。": ".",
"「": '\\"',
"」": '\\"',
"- ": "-",
"—": "―",
"】": "]",
"【": "[",
"é": "e",
"ō": "o",
"Placeholder Text": "",
# Add more replacements as needed
}
for target, replacement in placeholders.items():
translatedText = translatedText.replace(target, replacement)
placeholders = {
f"{LANGUAGE} Translation: ": "",
"Translation: ": "",
"っ": "",
"〜": "~",
"ッ": "",
"。": ".",
"「": '\\"',
"」": '\\"',
"- ": "-",
"—": "―",
"】": "]",
"【": "[",
"é": "e",
"this guy": "this bastard",
"This guy": "This bastard",
"Placeholder Text": "",
"```json": "",
"```": "",
# Add more replacements as needed
}
for target, replacement in placeholders.items():
translatedText = translatedText.replace(target, replacement)
# Remove Repeating Characters
pattern = re.compile(r"(.)\s*\1(?:\s*\1){" + str(20 - 1) + r",}")
translatedText = pattern.sub(lambda match: match.group(0).replace(" ", "")[:20], translatedText)
# Remove Repeating Characters
pattern = re.compile(r"(.)\s*\1(?:\s*\1){" + str(20 - 1) + r",}")
translatedText = pattern.sub(lambda match: match.group(0).replace(" ", "")[:20], translatedText)
# Elongate Long Dashes (Since GPT Ignores them...)
translatedText = elongateCharacters(translatedText)
else:
print(translatedText)
# Elongate Long Dashes (Since GPT Ignores them...)
translatedText = elongateCharacters(translatedText)
return translatedText
@ -2743,7 +2799,7 @@ def countTokens(system, user, history):
inputTotalTokens += len(enc.encode(user))
# Output
outputTotalTokens += round(len(enc.encode(user)) * 3)
outputTotalTokens += round(len(enc.encode(user)) * 2.5)
return [inputTotalTokens, outputTotalTokens]
@ -2800,14 +2856,15 @@ def translateGPT(text, history, fullPromptFlag):
continue
# Translating
response = translateText(system, user, history, 0.05, format, model="gpt-4o")
response = translateText(system, user, history, 0.05, format)
# Set Tokens
translatedText = response.choices[0].message.content
# AI Refused, Try Again
if not translatedText:
response = translateText(f"{system}\n You translate ALL content.", user, history, 0.1, format)
response = translateText(f"{system}\n You translate ALL content.", user, history, 0.1, format, model="gpt-4o")
translatedText = response.choices[0].message.content
# Report Tokens
totalTokens[0] += response.usage.prompt_tokens

86
modules/wolf2.py
View file

@ -310,7 +310,6 @@ def translateWOLF(data, translatedList, pbar, filename):
MISMATCH.append(filename)
return tokens
# Save some money and enter the character before translation
def getSpeaker(speaker):
match speaker:
@ -328,7 +327,7 @@ def getSpeaker(speaker):
response = translateGPT(
f"{speaker}",
"Reply with the " + LANGUAGE + " translation of the NPC name.",
True,
False,
)
response[0] = response[0].title()
response[0] = response[0].replace("'S", "'s")
@ -357,11 +356,68 @@ def batchList(input_list, batch_size):
return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)]
def parseVocabWithCategories(vocabText):
"""Parse vocabulary text and extract terms with their categories."""
pairs = []
seen = set()
currentCategory = None
for line in vocabText.splitlines():
line = line.strip()
if not line or line.startswith('```'):
continue
# Check if this is a category header
if line.startswith('#'):
currentCategory = line
continue
# Parse vocabulary term
m = re.match(r'^(.+?)(?:\s?[\(–])', line) # term is everything before space + '(' or '–'
if m:
term = m.group(1)
if term not in seen:
pairs.append((term, line, currentCategory))
seen.add(term)
return pairs
def buildMatchedVocabText(vocabPairs, subbedT):
"""Build formatted vocabulary text with matched terms organized by category."""
matchedCategories = {}
# Use word boundaries for Japanese if appropriate, or allow substring as before.
for term, line, category in vocabPairs:
# "term in subbedT" could be false positive; can use regex but Japanese doesn't always have spaces.
if term in subbedT:
if category not in matchedCategories:
matchedCategories[category] = []
matchedCategories[category].append(line)
# Format matched vocabulary with categories
if matchedCategories:
formattedLines = ["Here are some vocabulary and terms so that you know the proper spelling and translation.\n"]
for category, lines in matchedCategories.items():
if category: # Only add category header if it exists
formattedLines.append(category)
formattedLines.extend(lines)
formattedLines.append("") # Add blank line between categories
matchedVocabText = f"```\n{chr(10).join(formattedLines).rstrip()}\n```"
else:
matchedVocabText = ""
return matchedVocabText
def createContext(fullPromptFlag, subbedT, format):
system = (
PROMPT + VOCAB
if fullPromptFlag
else f"\
vocabPairs = parseVocabWithCategories(VOCAB)
matchedVocabText = buildMatchedVocabText(vocabPairs, subbedT)
if fullPromptFlag:
system = PROMPT + matchedVocabText
else:
system = f"\
You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
- You translate everything, including erotic and explicit content.\n\
@ -371,9 +427,8 @@ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{
- Maintain any spacing in the translation.\n\
- Maintain any code text in brackets if given. (e.g `[Color_0]`, `[Ascii_0]`, `[FCode_1`], etc)\n\
- `...` can be a part of the dialogue. Translate it as it is.\n\
{VOCAB}\n\
{matchedVocabText}\n\
"
)
if format == "json":
user = f"```json\n{subbedT}\n```"
else:
@ -387,9 +442,10 @@ def translateText(system, user, history, penalty, format, model=MODEL):
# History
if isinstance(history, list):
msg.extend([{"role": "system", "content": h} for h in history])
msg.append({"role": "system", "content": "Translation History:"})
msg.extend([{"role": "assistant", "content": h} for h in history])
else:
msg.append({"role": "system", "content": history})
msg.append({"role": "assistant", "content": history})
# Response Format
if format == "json":
@ -424,8 +480,11 @@ def cleanTranslatedText(translatedText):
"】": "]",
"【": "[",
"é": "e",
"ō": "o",
"this guy": "this bastard",
"This guy": "This bastard",
"Placeholder Text": "",
"```json": "",
"```": "",
# Add more replacements as needed
}
for target, replacement in placeholders.items():
@ -487,7 +546,7 @@ def countTokens(system, user, history):
inputTotalTokens += len(enc.encode(user))
# Output
outputTotalTokens += round(len(enc.encode(user)) * 3)
outputTotalTokens += round(len(enc.encode(user)) * 2.5)
return [inputTotalTokens, outputTotalTokens]
@ -551,7 +610,8 @@ def translateGPT(text, history, fullPromptFlag):
# AI Refused, Try Again
if not translatedText:
response = translateText(f"{system}\n You translate ALL content.", user, history, 0.1, format)
response = translateText(f"{system}\n You translate ALL content.", user, history, 0.1, format, model="gpt-4o")
translatedText = response.choices[0].message.content
# Report Tokens
totalTokens[0] += response.usage.prompt_tokens